diff --git a/dm/host/hle.c b/dm/host/hle.c index 63642a3..52c2f8b 100644 --- a/dm/host/hle.c +++ b/dm/host/hle.c @@ -56,19 +56,19 @@ void stubs_report(void) { LOG("stubs=%u", g_nstub); } // ---------- Host gate dispatch ---------- // The rewriter replaces every svc #n / mrs cntpct with a call to dm_gate; here we remember, by gate block return point (absolute address), what it originally was. // Lookup: open-addressing hash, one hit (it used to do module_of + a linear compare over 110 entries on every gate call, keyed by in-module offset, which would collide across modules). -enum { G_SVC, G_CNTPCT, G_CNTVCT, G_OBS }; +enum { G_SVC, G_CNTPCT, G_CNTVCT, G_OBS, G_SPIN }; // Observation gates (inserted offline from the rewriter's OBS table): number → read-only callback; with no callback attached the gate returns after one pass in and out static void (*g_obs_fn[OBS_MAX])(const uint64_t *regs); static int g_obs_have[OBS_MAX]; void gate_obs_set(int id, void (*fn)(const uint64_t *regs)) { if (id <= 0 || id >= OBS_MAX) die("observation gate number out of range %d", id); g_obs_fn[id] = fn; } int gate_obs_present(int id) { return id > 0 && id < OBS_MAX && g_obs_have[id]; } typedef struct { uint64_t ret; int kind, arg; uint64_t orig; } gate_t; -#define GATE_BITS 12 // 4096 slots, gate count ≤ 1024, load factor ≤ 1/4 +#define GATE_BITS 15 // 32768 slots, gate count ≤ 8192, load factor ≤ 1/4 (spin wait gates: Tears of the Kingdom's main has 843) static gate_t g_gate[1u << GATE_BITS]; static int g_ngate; static inline unsigned gate_h(uint64_t a) { return (unsigned)(((a >> 2) * 0x9E3779B97F4A7C15ull) >> (64 - GATE_BITS)); } void gate_register(uint64_t ret_abs, const char *kind, int arg, uint64_t orig) { - int k = !strcmp(kind, "svc") ? G_SVC : !strcmp(kind, "cntpct") ? G_CNTPCT : !strcmp(kind, "cntvct") ? G_CNTVCT : !strcmp(kind, "obs") ? G_OBS : -1; + int k = !strcmp(kind, "svc") ? G_SVC : !strcmp(kind, "cntpct") ? G_CNTPCT : !strcmp(kind, "cntvct") ? G_CNTVCT : !strcmp(kind, "obs") ? G_OBS : !strcmp(kind, "spin") ? G_SPIN : -1; if (k < 0) die("unknown gate kind %s", kind); if (k == G_OBS) { if (arg <= 0 || arg >= OBS_MAX) die("observation gate number out of range %d", arg); g_obs_have[arg] = 1; } if (++g_ngate > (1 << GATE_BITS) / 4) die("gate overflow"); @@ -90,6 +90,9 @@ uint64_t gate_dispatch(uint64_t ret_addr, uint64_t *regs) { return t; } case G_OBS: if (g_obs_fn[g->arg]) g_obs_fn[g->arg](regs); return 0; // Registers are read-only; the x0–x8 that dm_gate writes back are the original values + case G_SPIN: { // a pass of a spin wait loop (patch_nso.py "Spin waits"), where the thread may sleep (hle_thread.c dm_spin_wait): inside an SVC frame, so that a thread paused there has its context + void thr_svc_enter(uint64_t *, uint64_t), thr_svc_leave(void), dm_spin_wait(uint64_t, uint64_t); + thr_svc_enter(regs, g->orig); dm_spin_wait(ret_addr, g->orig); thr_svc_leave(); return 0; } default: { // register the gate frame on SVC entry / exit: it is the context of a paused thread (hle_thread.c) void thr_svc_enter(uint64_t *, uint64_t), thr_svc_leave(void); thr_svc_enter(regs, g->orig); uint64_t r = svc_dispatch(g->arg, regs, g->orig); thr_svc_leave(); return r; } diff --git a/dm/host/hle_thread.c b/dm/host/hle_thread.c index 01ee798..4f15663 100644 --- a/dm/host/hle_thread.c +++ b/dm/host/hle_thread.c @@ -65,6 +65,8 @@ typedef struct thr { int closed; // Handle closed: the object is freed only when this holds together with exited (Horizon: the thread object goes away with the last reference) uint64_t nsvc; // SVC count (for the state dump) uint64_t nyield, nyield_seen, nbackoff, nbackoff_seen; // Yields (SleepThread 0/-1/-2) and those the back-off turned into short sleeps, with both counts at the last power report (dm_yield_top) + uint64_t nspin, nspin_seen, spin_site; const char *spin_mod; // Spin wait passes (dm_spin_wait; those that yield count in nyield too) and where the last one was + uint64_t srun_svc, srun_t0, spin_ret; // Current spin wait: nsvc at its passes, when it began, when the last pass returned uint64_t cpu_seen_us; // CPU time (µs) at the last power report (dm_thread_cpu) uint64_t yrun_svc, yrun_t0; // Current run of yields: nsvc of the last yield, and when the run began (a run lasts while no other SVC comes between two yields) mach_port_t mport; // Host thread port (to get CPU time) @@ -559,17 +561,58 @@ static uint64_t yield_backoff_ns(uint64_t run_ns) { uint64_t s = run_ns / 8; if (s < 10000) s = 10000; return s < (uint64_t)cap ? s : (uint64_t)cap; } -// The threads that yielded most since the last call (the power line every 300 frames, hle_nvn.c): "handle name N (back-off sleeps M)", at most two +// sched_yield, or a short sleep once the wait has lasted run_ns past the back-off's start +static void yield_for(thr_t *me, uint64_t run_ns) { + uint64_t bo = yield_backoff_ns(run_ns); + if (bo) { me->nbackoff++; me->sleeping = 1; struct timespec ts = {0, (long)bo}; nanosleep(&ts, 0); me->sleeping = 0; } + else sched_yield(); +} +// One yield SVC of the calling thread (me may be NULL) +static void thr_yield(thr_t *me) { + if (!me) { sched_yield(); return; } + uint64_t y0 = now_ns(); + me->nyield++; if (me->yrun_svc + 1 != me->nsvc) me->yrun_t0 = y0; me->yrun_svc = me->nsvc; + yield_for(me, y0 - me->yrun_t0); + if (me->handle == 0x1001) { g_main_yield_ns += now_ns() - y0; g_main_yield_n++; } +} +// Spin waits (patch_nso.py "Spin waits"): a guest loop that polls memory with no SVC, which the rewriter has made go through a spin gate on every pass after the first. On the Switch +// such a poll costs nothing while another thread of its core has work (a higher-priority one preempts it); here it takes a host core from the threads doing the work: Paper Mario: +// The Thousand-Year Door's three job workers (main+0xa8e680) spun at 94% of a core each in Rogueport on an iPhone, which ran at 24 fps. The gate alone slows the poll down, and on the +// Mac that was the whole gain (the workers' CPU fell about tenfold, see docs/development.md): most waits are short handoffs between workers. A wait (passes with no SVC between them +// and at most SPIN_GAP_NS of guest work between two) therefore polls on for its first DM_SPIN_POLL µs (2000, the yield back-off's start), and then sleeps as a run of yields does +// (yield_backoff_ns, DM_YIELD_BACKOFF), which leaves the core to the threads doing the work. A sched_yield on every pass instead (DM_SPIN_POLL=0) raised the Mac's frame p95 by 2-5 ms, +// as a yield there hands the core to other processes. DM_SPIN=0: no sleeps either. gate = the gate's address, site = the loop head (module offset) +#define SPIN_GAP_NS 20000 +void dm_spin_wait(uint64_t gate, uint64_t site) { + static int on = -1; static uint64_t poll_ns; + if (on < 0) { const char *e = getenv("DM_SPIN"), *p = getenv("DM_SPIN_POLL"); poll_ns = (p && *p ? strtoull(p, 0, 10) : 2000) * 1000; on = !(e && *e && atoi(e) == 0); + if (!on) LOG("DM_SPIN=0: spin waits do not yield or sleep"); else if (p && *p) LOG("DM_SPIN_POLL: spin waits poll for %llu µs, then yield", (unsigned long long)(poll_ns / 1000)); } + thr_t *me = pthread_getspecific(g_keythr); + if (!me) return; + uint64_t now = now_ns(); + if (me->srun_svc != me->nsvc || now - me->spin_ret > SPIN_GAP_NS) me->srun_t0 = now; // a new wait + me->srun_svc = me->nsvc; me->nspin++; + if (site != me->spin_site || !me->spin_mod) { const module_t *m = module_of((void *)gate); me->spin_mod = m ? m->name : "?"; me->spin_site = site; } + if (on && now - me->srun_t0 >= poll_ns) { me->nyield++; yield_for(me, now - me->srun_t0); } + me->spin_ret = now_ns(); +} +// The threads that yielded or passed through spin waits most since the last call (the power line every 300 frames, hle_nvn.c): "handle name N (back-off sleeps M)", at most two, +// N the yields (spin wait passes that yielded among them), with the spin wait passes and the last one's site ("; spin passes K, last at main+0x…") void dm_yield_top(char *out, size_t n) { - thr_t *top[2] = {0}; uint64_t d[2] = {0}, b[2] = {0}; + thr_t *top[2] = {0}; uint64_t d[2] = {0}, b[2] = {0}, s[2] = {0}, r[2] = {0}; pthread_mutex_lock(&G); for (thr_t *t = g_all; t; t = t->all) { - uint64_t y = t->nyield - t->nyield_seen, bo = t->nbackoff - t->nbackoff_seen; t->nyield_seen = t->nyield; t->nbackoff_seen = t->nbackoff; - if (y > d[0]) { top[1] = top[0]; d[1] = d[0]; b[1] = b[0]; top[0] = t; d[0] = y; b[0] = bo; } else if (y > d[1]) { top[1] = t; d[1] = y; b[1] = bo; } + uint64_t y = t->nyield - t->nyield_seen, bo = t->nbackoff - t->nbackoff_seen, sp = t->nspin - t->nspin_seen, k = y + sp; + t->nyield_seen = t->nyield; t->nbackoff_seen = t->nbackoff; t->nspin_seen = t->nspin; + if (k > r[0]) { top[1] = top[0]; d[1] = d[0]; b[1] = b[0]; s[1] = s[0]; r[1] = r[0]; top[0] = t; d[0] = y; b[0] = bo; s[0] = sp; r[0] = k; } + else if (k > r[1]) { top[1] = t; d[1] = y; b[1] = bo; s[1] = sp; r[1] = k; } } size_t o = 0; char nb[32]; - for (int i = 0; i < 2 && top[i] && o < n; i++) - o += snprintf(out + o, n - o, "%s%#x '%s' %llu times (back-off sleeps %llu)", i ? ", " : "", top[i]->handle, thr_name(top[i], nb, sizeof nb), (unsigned long long)d[i], (unsigned long long)b[i]); + for (int i = 0; i < 2 && top[i] && o < n; i++) { + o += snprintf(out + o, n - o, "%s%#x '%s' %llu times (back-off sleeps %llu", i ? ", " : "", top[i]->handle, thr_name(top[i], nb, sizeof nb), (unsigned long long)d[i], (unsigned long long)b[i]); + if (o < n && s[i]) o += snprintf(out + o, n - o, "; spin passes %llu, last at %s+%#llx", (unsigned long long)s[i], top[i]->spin_mod, (unsigned long long)top[i]->spin_site); + if (o < n) o += snprintf(out + o, n - o, ")"); + } pthread_mutex_unlock(&G); if (!o && n) snprintf(out, n, "none"); } @@ -663,15 +706,7 @@ static int svc_thread_impl(int num, uint64_t *regs) { // Yield (0 / -1 / -2), and sleeps shorter than 20 µs, which sleep_precise would return from at once: Super Mario Galaxy's Hagi polls with SleepThread(100 ns) (main+0x517c), // which kept a core at 100% (1.4 billion calls in 200 s); as yields they get the back-off below. // Count and time of the main thread's yields are recorded: a render task graph's "wait for everything to finish" is often pure polling + yield; measure how much of a frame it takes - else { - thr_t *me = pthread_getspecific(g_keythr); - uint64_t y0 = me ? now_ns() : 0; - if (me) { me->nyield++; if (me->yrun_svc + 1 != me->nsvc) me->yrun_t0 = y0; me->yrun_svc = me->nsvc; } - uint64_t bo = me ? yield_backoff_ns(y0 - me->yrun_t0) : 0; - if (bo) { me->nbackoff++; me->sleeping = 1; struct timespec ts = {0, (long)bo}; nanosleep(&ts, 0); me->sleeping = 0; } - else sched_yield(); - if (me && me->handle == 0x1001) { g_main_yield_ns += now_ns() - y0; g_main_yield_n++; } - } + else thr_yield(pthread_getspecific(g_keythr)); return 1; } case 0x0C: { thr_t *t = h_get((uint32_t)regs[1], O_THREAD); regs[0] = t ? 0 : R_BAD_HANDLE; regs[1] = t ? (uint64_t)t->prio : 0; return 1; } diff --git a/dm/tools/patch_nso.py b/dm/tools/patch_nso.py index 8227724..016c741 100644 --- a/dm/tools/patch_nso.py +++ b/dm/tools/patch_nso.py @@ -6,6 +6,7 @@ # mrs tpidrro_el0 → read thread slot KTLR (Horizon ThreadLocalRegion pointer) # svc #n, mrs cntpct_el0 → host gate (gate address taken from thread slot KGATE; once in C, gates.tsv is looked up by return point) # sub sp, sp, xN → xN rounded up to 16 first (Apple's kernel faults on a misaligned sp, Horizon does not; site_sp_align) +# spin wait loops → a copy whose back-edges pass a spin gate (the host yields there; a busy poll costs nothing on the Switch, see "Spin waits") # Registered cuts (CUTS, restricted by NSO build ID): before rewriting, the original bytes are asserted one by one and any mismatch aborts; results are listed in .cuts.tsv # Registered observation gates (OBS, restricted by build ID): one instruction at the entry is replaced with "enter the host observation callback → execute the original instruction → jump back", so diagnostics modify zero code at runtime; listed in .obs.tsv # The key page only stores "actual key number × 8" at the end of the data segment, and patch code fetches thread slots indirectly through the key page: the loader may use any key number. @@ -14,6 +15,7 @@ import struct, re, sys, os from itertools import combinations import lz4.block from capstone import * +from capstone.arm64 import ARM64_OP_IMM, ARM64_OP_MEM, ARM64_OP_REG src, outdir, NAME = sys.argv[1], sys.argv[2], sys.argv[3] d = open(src, 'rb').read() @@ -676,9 +678,34 @@ starts = sorted(x for x in starts if x < TEXT_END) sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) import nso_code DATA_BLOCKS = nso_code.data_blocks(IMG, TEXT_END, md) +CODE_RANGES = list(zip([0] + [e for _, e in DATA_BLOCKS], [s for s, _ in DATA_BLOCKS] + [TEXT_END])) +md_lite = Cs(CS_ARCH_ARM64, CS_MODE_ARM) # no details: the x18 scan reads only the operand text +R18 = re.compile(r'\b[xw]18\b') +def x18_words(lo, hi): + """offsets in [lo, hi) of the instructions whose operands name x18 / w18 (as has18). Decoded 16 KB at a time without details, going on after a word capstone cannot decode: + a third of the time of decoding one word at a time with details (Paper Mario: The Thousand-Year Door main, 437 sites in 300,000 words either way)""" + out, a = set(), lo + while a < hi: + end = min(hi, a + 0x4000); last = a - 4 + for addr, size, mn, op in md_lite.disasm_lite(bytes(IMG[a:end]), a): + last = addr + if R18.search(op): out.add(addr) + a = last + 4 if last + 4 >= end else last + 8 # the chunk is done, or the word after the last decoded one is no instruction + return out +x18 = set().union(*(x18_words(s, e) for s, e in CODE_RANGES)) kinds = {} -for o in (o for s, e in zip([0] + [e for _, e in DATA_BLOCKS], [s for s, _ in DATA_BLOCKS] + [TEXT_END]) for o in range(s, e, 4)): +# Spin wait candidates (see "Spin waits" below): backward direct branches (b, b.cond, cbz / cbnz, tbz / tbnz) at most SPIN_WIN bytes back, by target +SPIN_WIN, SPIN_TOPS = 512, {0x16, 0x17, 0x54, 0x34, 0x35, 0xB4, 0xB5, 0x36, 0x37, 0xB6, 0xB7} +spin_cand = {} +for o in (o for s, e in CODE_RANGES for o in range(s, e, 4)): w = struct.unpack_from('> 24 in SPIN_TOPS: + t = w >> 24 + if t in (0x16, 0x17): d = ((w & 0x3FFFFFF) - (1 << 26)) * 4 + elif t & 0x7E == 0x36: d = (((w >> 5) & 0x3FFF) - (1 << 14)) * 4 if w & 0x40000 else 4 + elif t == 0x54: d = (((w >> 5) & 0x7FFFF) - (1 << 19)) * 4 if w & 0x800000 and not w & 0x10 else 4 + else: d = (((w >> 5) & 0x7FFFF) - (1 << 19)) * 4 if w & 0x800000 else 4 + if -SPIN_WIN <= d <= 0: spin_cand.setdefault(o + d, []).append(o) if w & 0xFFE0001F == 0xD4000001: kinds[o] = ('svc', (w >> 5) & 0xFFFF); continue if w & 0xFFFFFFE0 == 0xD53BD040: kinds[o] = ('tpidr_r', None); continue if w & 0xFFFFFFE0 == 0xD51BD040: kinds[o] = ('tpidr_w', None); continue @@ -689,8 +716,7 @@ for o in (o for s, e in zip([0] + [e for _, e in DATA_BLOCKS], [s for s, _ in DA if w & 0xFFFFFFE0 == 0xD53B0020: kinds[o] = ('ctr', w & 31); continue # sub sp, sp, xN (extended register, LSL #0): sp may end up misaligned, which Horizon allows and Apple's kernel does not (site_sp_align); with x18 it is an x18 site (site_sp) if w & 0xFFE0FFFF == 0xCB2063FF and (w >> 16) & 31 != 18: kinds[o] = ('sp_align', (w >> 16) & 31); continue - for ins in md.disasm(IMG[o:o + 4], o): - if has18(ins): kinds[o] = ('x18', None) + if o in x18: kinds[o] = ('x18', None) # Take the instruction table per function, find the windows and jump tables import bisect @@ -1267,6 +1293,215 @@ if FS_STATE_OFF and BUILD_ID == '0effe4af1dec3a7966b934d4a7c3d2bf': emit(0xbd004660); emit(B(here(), 0x2e77fc)) # orig: str s0,[x19,#0x44]; back fs_put(0x2e77f8, [B(0x2e77f8, start)], 'frame step: the base clock steps by the host\'s vsyncs (state page target fps; 0 = the game\'s measured step)') +# ---- Spin waits (every module) ---- +# A guest loop that polls memory until another thread changes it fills only its own core on the Switch, where any higher-priority thread pinned there preempts it, so the +# spin costs the game nothing; on the host it takes a whole core from the threads doing the work. Paper Mario: The Thousand-Year Door's gfx::ParallelManager workers +# (main+0xa8e680, three, pinned at priority 45) poll their job list this way while a batch is pending; on an iPhone Rogueport ran at 24 fps with the three at 94% of a core each. +# A loop counts as a spin wait only when it provably cannot leave by itself. From its head T, every pass along the region R (the paths from T back to T, within SPIN_WIN bytes +# after T) runs only loads, register and flag arithmetic, direct branches and barriers: no store, call, SVC, system register or event wait; and no register or flag that is +# live at T is written in R (no counter, cursor or delay). One exception: an exclusive add of a constant (ld(a)xr; add/sub #k; st(l)xr; cbnz) that every path back to T +# undoes on the same address (a worker that registers with a job, sees it finished and backs out). Such a pass leaves memory and registers as it found them, so only another +# thread can end the loop, and a pause there changes only timing. R is copied into the patch area; the original back-edges jump to the copy (no pause on the first pass, so a +# single retry costs nothing), the copy's back-edges go through a spin gate, which the host takes as a yield with the yield back-off (hle_thread.c dm_spin_wait), and exits +# from the copy go back to the original code. The gate keeps every register and the flags (entry.S) +SPIN_PATHS, SPIN_STEPS = 256, 4096 # complete paths back to T; search steps (R itself lies within SPIN_WIN bytes) +SPIN_LSE = ('ldadd', 'ldclr', 'ldeor', 'ldset', 'ldsmax', 'ldsmin', 'ldumax', 'ldumin') +SPIN_BAD = {'bl', 'blr', 'br', 'ret', 'eret', 'svc', 'hvc', 'smc', 'brk', 'hlt', 'udf', 'mrs', 'msr', 'sys', 'sysl', 'wfe', 'wfi', 'sev', 'sevl', 'clrex', 'dc', 'ic', 'at', + 'tlbi', 'drps', 'dcps1', 'dcps2', 'dcps3', 'blraa', 'blrab', 'blraaz', 'blrabz', 'braa', 'brab', 'braaz', 'brabz', 'retaa', 'retab', 'eretaa', 'eretab'} +spin_dec = {} +def spin_ins(a): + if a not in spin_dec: spin_dec[a] = next(md.disasm(bytes(IMG[a:a + 4]), a), None) + return spin_dec[a] +def spin_norm(n): + n = {'fp': 'x29', 'lr': 'x30', 'wsp': 'sp'}.get(n, n) + if n in ('wzr', 'xzr'): return None + m = re.fullmatch(r'[wx](\d+)', n) + if m: return 'x' + m.group(1) + m = re.fullmatch(r'[bhsdqv](\d+)', n) + return 'v' + m.group(1) if m else n +def spin_rw(ins): + """(registers read, registers written), w / x and the vector views merged. Capstone's own lists are not enough: it lists a compare's first operand as written and the flags + as read by cbz / tbz, and leaves out the destination of adds (sdk+0x31033c in Paper Mario: The Thousand-Year Door, adds w9,w9,#8; b.ne: a counted loop). So every register + operand counts as well: operand 0 written and the others read, or for a load all of them written (its address registers are in capstone's reads)""" + m = ins.mnemonic + regs = [spin_norm(ins.reg_name(o.reg)) for o in ins.operands if o.type == ARM64_OP_REG] + ops = set(regs) - {None} + if m in ('cbz', 'cbnz', 'tbz', 'tbnz'): return ops, set() + if m.startswith('b.'): return {'nzcv'}, set() + if m == 'b': return set(), set() + if m in ('cmp', 'cmn', 'tst', 'fcmp', 'fcmpe'): return ops, {'nzcv'} + if m in ('ccmp', 'ccmn', 'fccmp', 'fccmpe'): return ops | {'nzcv'}, {'nzcv'} + r, w = ins.regs_access() + r, w = {spin_norm(ins.reg_name(x)) for x in r} - {None}, {spin_norm(ins.reg_name(x)) for x in w} - {None} + if m.startswith(('ld', 'prfm', 'prfum')): return r, w | ops + return r | (set(regs[1:]) - {None}), w | (set(regs[:1]) - {None}) +def spin_kind(ins): + """'br' direct branch, 'pc' adr / adrp, 'ok' a load or register / flag arithmetic, None anything else""" + if ins is None: return None + m = ins.mnemonic + if m in ('b', 'cbz', 'cbnz', 'tbz', 'tbnz') or m.startswith('b.'): return 'br' + if m in ('adr', 'adrp'): return 'pc' + if m in SPIN_BAD or m.startswith(('st', 'cas', 'swp') + SPIN_LSE): return None + mem = any(o.type == ARM64_OP_MEM for o in ins.operands) + if m.startswith(('ld', 'prfm', 'prfum')): return 'ok' if mem else None # no memory operand: a pc-relative literal load + return None if mem else 'ok' +def spin_rmw(a): + """ld(a)xr wA,[xN]; add|sub wB,wA,#k; st(l)xr wS,wB,[xN]; cbnz wS, at a → (xN, ±k, registers written, the four instructions), or None""" + l = [spin_ins(a + 4 * i) for i in range(4)] + if None in l or l[0].mnemonic not in ('ldxr', 'ldaxr') or l[1].mnemonic not in ('add', 'sub') or l[2].mnemonic not in ('stxr', 'stlxr') or l[3].mnemonic != 'cbnz': return None + r = lambda x, i: x.reg_name(x.operands[i].reg) + if len(l[1].operands) != 3 or l[1].operands[2].type != ARM64_OP_IMM or 'lsl' in l[1].op_str: return None + base = l[0].operands[1].mem + if base.disp or base.index or l[2].operands[2].mem.base != base.base or l[2].operands[2].mem.disp: return None + if r(l[1], 1) != r(l[0], 0) or r(l[2], 1) != r(l[1], 0) or r(l[3], 0) != r(l[2], 0) or l[3].operands[1].imm != a: return None + k = l[1].operands[2].imm * (1 if l[1].mnemonic == 'add' else -1) + return spin_norm(l[0].reg_name(base.base)), k, {spin_norm(r(l[0], 0)), spin_norm(r(l[1], 0)), spin_norm(r(l[2], 0))} - {None}, l +def spin_rebranch(ins, pc, tgt): + """the direct branch ins re-encoded at pc with target tgt""" + w = struct.unpack('> 2) & 0x3FFF) << 5) + assert -(1 << 20) <= off < (1 << 20) and off % 4 == 0, f'spin wait branch out of range {pc:#x}' + return (w & ~(0x7FFFF << 5)) | (((off >> 2) & 0x7FFFF) << 5) +spin_touched = set(kinds) | done | {o for off, oid, ow, why in obs for o in (off,)} | {o for off, ob, nb, why in cuts for o in range(off, off + len(ob) // 2, 4)} +# Instruction words that end every path through them, told apart by their encoding alone: bl; br / blr / ret; svc / brk; str / strb / strh and the vector str (unsigned offset; +# unscaled, pre / post-index; register offset); stp / stnp; ldadd and the other LSE atomics. Exclusive stores are left to the full check (the exclusive add), and so are the pre / +# post-index loads: a seqlock read in SDK 15's nn::time (sdk+0x3d3270) moves a base it sets again on each pass +SPIN_STOP = ((0xFC000000, 0x94000000), (0xFE000000, 0xD6000000), (0xFF000000, 0xD4000000), (0x3FC00000, 0x39000000), (0x3FC00000, 0x3D000000), (0x3FC00000, 0x3D800000), + (0x3FE00000, 0x38000000), (0x3FE00C00, 0x38200800), (0x3FE00000, 0x3C000000), (0x3FE00000, 0x3C800000), (0x3A400000, 0x28000000), (0x3F200C00, 0x38200000)) +def spin_quick(T, hi): + """a back-edge to T can be reached from T without passing a word of SPIN_STOP (only such heads get the full check, which decodes)""" + seen, todo = set(), [T] + while todo: + a = todo.pop() + if a in seen or not T <= a <= hi: continue + seen.add(a); w = struct.unpack_from('> 24 + if 0x14 <= t <= 0x17: succ = (a + (((w & 0x3FFFFFF) ^ 0x2000000) - 0x2000000) * 4,) + elif t == 0x54 or t & 0x7E == 0x34: succ = (a + ((((w >> 5) & 0x7FFFF) ^ 0x40000) - 0x40000) * 4, a + 4) + elif t & 0x7E == 0x36: succ = (a + ((((w >> 5) & 0x3FFF) ^ 0x2000) - 0x2000) * 4, a + 4) + elif any(w & m == v for m, v in SPIN_STOP): continue + else: succ = (a + 4,) + if T in succ: return True + todo.extend(succ) + return False +class SpinNo(Exception): pass +def spin_region(T, bes): + """the spin wait region with head T (bes: the backward branches to T found by the scan) → dict, or None""" + hi = max(bes) + if any(s < hi + 4 and T < e for s, e in DATA_BLOCKS) or not spin_quick(T, hi): return None + nodes, back, bad, inner = {}, set(), set(), set() + todo = [T] + while todo: + a = todo.pop() + if a in nodes or a in bad: continue + rmw = spin_rmw(a) if a + 12 <= hi and struct.unpack_from(' SPIN_STEPS: raise SpinNo + n = nodes[a] + if n['kind'] == 'rmw': + key = (n['base'], ver.get(n['base'], 0)); delta = dict(delta); delta[key] = delta.get(key, 0) + n['k'] + if n['defs']: ver = dict(ver); ver.update({r: ver.get(r, 0) + 1 for r in n['defs']}) + for s in n['succ']: + if s == T: + cnt[0] += 1 + if cnt[0] > SPIN_PATHS: raise SpinNo + if any(delta.values()): bad_back.add(a) + elif s in R: + if s in onpath: raise SpinNo + walk(s, onpath | {s}, ver, delta) + try: walk(T, {T}, {}, {}) + except SpinNo: return None + valid = back - bad_back + if not valid: return None + R = prune(valid) + # no state carried from one pass to the next: nothing live at T is written in R + live, more = {a: set() for a in R}, True + while more: + more = False + for a in sorted(R, reverse=True): + n, out = nodes[a], set() + for s in n['succ']: + if s == T: out |= live[T] if a in valid else set() + elif s in R: out |= live[s] + new = n['uses'] | (out - n['defs']) + if new != live[a]: live[a] = new; more = True + if live[T] & set().union(*(nodes[a]['defs'] for a in R)): return None + return {'T': T, 'R': sorted(R), 'nodes': nodes, 'valid': valid} +def spin_emit(g): + T, R, nodes, valid = g['T'], g['R'], g['nodes'], g['valid'] + cpy, fixes, tramp = {}, [], {} + for i, a in enumerate(R): + n = nodes[a]; cpy[a] = here(); x = n['ins'][0] + if n['kind'] == 'rmw': + for y in n['ins'][:3]: emit(struct.unpack(':