From 99c3f91481203c67f778d326f7f4bd742804d2f7 Mon Sep 17 00:00:00 2001 From: seedds Date: Thu, 22 Oct 2026 16:32:01 +0800 Subject: [PATCH] Spin waits: the rewriter finds guest poll loops in every module and the host lets them sleep (Paper Mario: The Thousand-Year Door's job workers at 24 fps on the iPhone) A loop that polls memory with no SVC costs nothing on the Switch, where a higher-priority thread of its core preempts it; here it takes a host core. TTYD's three gfx::ParallelManager workers (main+0xa8e680) spun at 94% of a core each in Rogueport on an iPhone. patch_nso.py now copies every loop that provably cannot leave by itself (only loads, register and flag arithmetic, direct branches and barriers on every path back to its head, nothing live at the head written; an exclusive add undone on every path allowed) into the patch area; the copy's back-edges pass a spin gate, where the host polls on for 2 ms and then sleeps with the yield back-off (dm_spin_wait; DM_SPIN, DM_SPIN_POLL). Mac Rogueport route beside the committed build: frame p95 29-37 -> 18-22 ms, CPU 93-129% -> 38-57%. The x18 scan decodes in blocks without details, so the rewrite is faster than before (TTYD main 20.5 -> 11.7 s, Tears of the Kingdom 101.5 -> 64.2 s); its sites are unchanged. --- dm/host/hle.c | 9 +- dm/host/hle_thread.c | 65 +++++++--- dm/tools/patch_nso.py | 242 +++++++++++++++++++++++++++++++++++- docs/agents/05-debugging.md | 6 +- docs/development.md | 9 ++ 5 files changed, 309 insertions(+), 22 deletions(-) diff --git a/dm/host/hle.c b/dm/host/hle.c index 63642a3..52c2f8b 100644 --- a/dm/host/hle.c +++ b/dm/host/hle.c @@ -56,19 +56,19 @@ void stubs_report(void) { LOG("stubs=%u", g_nstub); } // ---------- Host gate dispatch ---------- // The rewriter replaces every svc #n / mrs cntpct with a call to dm_gate; here we remember, by gate block return point (absolute address), what it originally was. // Lookup: open-addressing hash, one hit (it used to do module_of + a linear compare over 110 entries on every gate call, keyed by in-module offset, which would collide across modules). -enum { G_SVC, G_CNTPCT, G_CNTVCT, G_OBS }; +enum { G_SVC, G_CNTPCT, G_CNTVCT, G_OBS, G_SPIN }; // Observation gates (inserted offline from the rewriter's OBS table): number → read-only callback; with no callback attached the gate returns after one pass in and out static void (*g_obs_fn[OBS_MAX])(const uint64_t *regs); static int g_obs_have[OBS_MAX]; void gate_obs_set(int id, void (*fn)(const uint64_t *regs)) { if (id <= 0 || id >= OBS_MAX) die("observation gate number out of range %d", id); g_obs_fn[id] = fn; } int gate_obs_present(int id) { return id > 0 && id < OBS_MAX && g_obs_have[id]; } typedef struct { uint64_t ret; int kind, arg; uint64_t orig; } gate_t; -#define GATE_BITS 12 // 4096 slots, gate count ≤ 1024, load factor ≤ 1/4 +#define GATE_BITS 15 // 32768 slots, gate count ≤ 8192, load factor ≤ 1/4 (spin wait gates: Tears of the Kingdom's main has 843) static gate_t g_gate[1u << GATE_BITS]; static int g_ngate; static inline unsigned gate_h(uint64_t a) { return (unsigned)(((a >> 2) * 0x9E3779B97F4A7C15ull) >> (64 - GATE_BITS)); } void gate_register(uint64_t ret_abs, const char *kind, int arg, uint64_t orig) { - int k = !strcmp(kind, "svc") ? G_SVC : !strcmp(kind, "cntpct") ? G_CNTPCT : !strcmp(kind, "cntvct") ? G_CNTVCT : !strcmp(kind, "obs") ? G_OBS : -1; + int k = !strcmp(kind, "svc") ? G_SVC : !strcmp(kind, "cntpct") ? G_CNTPCT : !strcmp(kind, "cntvct") ? G_CNTVCT : !strcmp(kind, "obs") ? G_OBS : !strcmp(kind, "spin") ? G_SPIN : -1; if (k < 0) die("unknown gate kind %s", kind); if (k == G_OBS) { if (arg <= 0 || arg >= OBS_MAX) die("observation gate number out of range %d", arg); g_obs_have[arg] = 1; } if (++g_ngate > (1 << GATE_BITS) / 4) die("gate overflow"); @@ -90,6 +90,9 @@ uint64_t gate_dispatch(uint64_t ret_addr, uint64_t *regs) { return t; } case G_OBS: if (g_obs_fn[g->arg]) g_obs_fn[g->arg](regs); return 0; // Registers are read-only; the x0–x8 that dm_gate writes back are the original values + case G_SPIN: { // a pass of a spin wait loop (patch_nso.py "Spin waits"), where the thread may sleep (hle_thread.c dm_spin_wait): inside an SVC frame, so that a thread paused there has its context + void thr_svc_enter(uint64_t *, uint64_t), thr_svc_leave(void), dm_spin_wait(uint64_t, uint64_t); + thr_svc_enter(regs, g->orig); dm_spin_wait(ret_addr, g->orig); thr_svc_leave(); return 0; } default: { // register the gate frame on SVC entry / exit: it is the context of a paused thread (hle_thread.c) void thr_svc_enter(uint64_t *, uint64_t), thr_svc_leave(void); thr_svc_enter(regs, g->orig); uint64_t r = svc_dispatch(g->arg, regs, g->orig); thr_svc_leave(); return r; } diff --git a/dm/host/hle_thread.c b/dm/host/hle_thread.c index 01ee798..4f15663 100644 --- a/dm/host/hle_thread.c +++ b/dm/host/hle_thread.c @@ -65,6 +65,8 @@ typedef struct thr { int closed; // Handle closed: the object is freed only when this holds together with exited (Horizon: the thread object goes away with the last reference) uint64_t nsvc; // SVC count (for the state dump) uint64_t nyield, nyield_seen, nbackoff, nbackoff_seen; // Yields (SleepThread 0/-1/-2) and those the back-off turned into short sleeps, with both counts at the last power report (dm_yield_top) + uint64_t nspin, nspin_seen, spin_site; const char *spin_mod; // Spin wait passes (dm_spin_wait; those that yield count in nyield too) and where the last one was + uint64_t srun_svc, srun_t0, spin_ret; // Current spin wait: nsvc at its passes, when it began, when the last pass returned uint64_t cpu_seen_us; // CPU time (µs) at the last power report (dm_thread_cpu) uint64_t yrun_svc, yrun_t0; // Current run of yields: nsvc of the last yield, and when the run began (a run lasts while no other SVC comes between two yields) mach_port_t mport; // Host thread port (to get CPU time) @@ -559,17 +561,58 @@ static uint64_t yield_backoff_ns(uint64_t run_ns) { uint64_t s = run_ns / 8; if (s < 10000) s = 10000; return s < (uint64_t)cap ? s : (uint64_t)cap; } -// The threads that yielded most since the last call (the power line every 300 frames, hle_nvn.c): "handle name N (back-off sleeps M)", at most two +// sched_yield, or a short sleep once the wait has lasted run_ns past the back-off's start +static void yield_for(thr_t *me, uint64_t run_ns) { + uint64_t bo = yield_backoff_ns(run_ns); + if (bo) { me->nbackoff++; me->sleeping = 1; struct timespec ts = {0, (long)bo}; nanosleep(&ts, 0); me->sleeping = 0; } + else sched_yield(); +} +// One yield SVC of the calling thread (me may be NULL) +static void thr_yield(thr_t *me) { + if (!me) { sched_yield(); return; } + uint64_t y0 = now_ns(); + me->nyield++; if (me->yrun_svc + 1 != me->nsvc) me->yrun_t0 = y0; me->yrun_svc = me->nsvc; + yield_for(me, y0 - me->yrun_t0); + if (me->handle == 0x1001) { g_main_yield_ns += now_ns() - y0; g_main_yield_n++; } +} +// Spin waits (patch_nso.py "Spin waits"): a guest loop that polls memory with no SVC, which the rewriter has made go through a spin gate on every pass after the first. On the Switch +// such a poll costs nothing while another thread of its core has work (a higher-priority one preempts it); here it takes a host core from the threads doing the work: Paper Mario: +// The Thousand-Year Door's three job workers (main+0xa8e680) spun at 94% of a core each in Rogueport on an iPhone, which ran at 24 fps. The gate alone slows the poll down, and on the +// Mac that was the whole gain (the workers' CPU fell about tenfold, see docs/development.md): most waits are short handoffs between workers. A wait (passes with no SVC between them +// and at most SPIN_GAP_NS of guest work between two) therefore polls on for its first DM_SPIN_POLL µs (2000, the yield back-off's start), and then sleeps as a run of yields does +// (yield_backoff_ns, DM_YIELD_BACKOFF), which leaves the core to the threads doing the work. A sched_yield on every pass instead (DM_SPIN_POLL=0) raised the Mac's frame p95 by 2-5 ms, +// as a yield there hands the core to other processes. DM_SPIN=0: no sleeps either. gate = the gate's address, site = the loop head (module offset) +#define SPIN_GAP_NS 20000 +void dm_spin_wait(uint64_t gate, uint64_t site) { + static int on = -1; static uint64_t poll_ns; + if (on < 0) { const char *e = getenv("DM_SPIN"), *p = getenv("DM_SPIN_POLL"); poll_ns = (p && *p ? strtoull(p, 0, 10) : 2000) * 1000; on = !(e && *e && atoi(e) == 0); + if (!on) LOG("DM_SPIN=0: spin waits do not yield or sleep"); else if (p && *p) LOG("DM_SPIN_POLL: spin waits poll for %llu µs, then yield", (unsigned long long)(poll_ns / 1000)); } + thr_t *me = pthread_getspecific(g_keythr); + if (!me) return; + uint64_t now = now_ns(); + if (me->srun_svc != me->nsvc || now - me->spin_ret > SPIN_GAP_NS) me->srun_t0 = now; // a new wait + me->srun_svc = me->nsvc; me->nspin++; + if (site != me->spin_site || !me->spin_mod) { const module_t *m = module_of((void *)gate); me->spin_mod = m ? m->name : "?"; me->spin_site = site; } + if (on && now - me->srun_t0 >= poll_ns) { me->nyield++; yield_for(me, now - me->srun_t0); } + me->spin_ret = now_ns(); +} +// The threads that yielded or passed through spin waits most since the last call (the power line every 300 frames, hle_nvn.c): "handle name N (back-off sleeps M)", at most two, +// N the yields (spin wait passes that yielded among them), with the spin wait passes and the last one's site ("; spin passes K, last at main+0x…") void dm_yield_top(char *out, size_t n) { - thr_t *top[2] = {0}; uint64_t d[2] = {0}, b[2] = {0}; + thr_t *top[2] = {0}; uint64_t d[2] = {0}, b[2] = {0}, s[2] = {0}, r[2] = {0}; pthread_mutex_lock(&G); for (thr_t *t = g_all; t; t = t->all) { - uint64_t y = t->nyield - t->nyield_seen, bo = t->nbackoff - t->nbackoff_seen; t->nyield_seen = t->nyield; t->nbackoff_seen = t->nbackoff; - if (y > d[0]) { top[1] = top[0]; d[1] = d[0]; b[1] = b[0]; top[0] = t; d[0] = y; b[0] = bo; } else if (y > d[1]) { top[1] = t; d[1] = y; b[1] = bo; } + uint64_t y = t->nyield - t->nyield_seen, bo = t->nbackoff - t->nbackoff_seen, sp = t->nspin - t->nspin_seen, k = y + sp; + t->nyield_seen = t->nyield; t->nbackoff_seen = t->nbackoff; t->nspin_seen = t->nspin; + if (k > r[0]) { top[1] = top[0]; d[1] = d[0]; b[1] = b[0]; s[1] = s[0]; r[1] = r[0]; top[0] = t; d[0] = y; b[0] = bo; s[0] = sp; r[0] = k; } + else if (k > r[1]) { top[1] = t; d[1] = y; b[1] = bo; s[1] = sp; r[1] = k; } } size_t o = 0; char nb[32]; - for (int i = 0; i < 2 && top[i] && o < n; i++) - o += snprintf(out + o, n - o, "%s%#x '%s' %llu times (back-off sleeps %llu)", i ? ", " : "", top[i]->handle, thr_name(top[i], nb, sizeof nb), (unsigned long long)d[i], (unsigned long long)b[i]); + for (int i = 0; i < 2 && top[i] && o < n; i++) { + o += snprintf(out + o, n - o, "%s%#x '%s' %llu times (back-off sleeps %llu", i ? ", " : "", top[i]->handle, thr_name(top[i], nb, sizeof nb), (unsigned long long)d[i], (unsigned long long)b[i]); + if (o < n && s[i]) o += snprintf(out + o, n - o, "; spin passes %llu, last at %s+%#llx", (unsigned long long)s[i], top[i]->spin_mod, (unsigned long long)top[i]->spin_site); + if (o < n) o += snprintf(out + o, n - o, ")"); + } pthread_mutex_unlock(&G); if (!o && n) snprintf(out, n, "none"); } @@ -663,15 +706,7 @@ static int svc_thread_impl(int num, uint64_t *regs) { // Yield (0 / -1 / -2), and sleeps shorter than 20 µs, which sleep_precise would return from at once: Super Mario Galaxy's Hagi polls with SleepThread(100 ns) (main+0x517c), // which kept a core at 100% (1.4 billion calls in 200 s); as yields they get the back-off below. // Count and time of the main thread's yields are recorded: a render task graph's "wait for everything to finish" is often pure polling + yield; measure how much of a frame it takes - else { - thr_t *me = pthread_getspecific(g_keythr); - uint64_t y0 = me ? now_ns() : 0; - if (me) { me->nyield++; if (me->yrun_svc + 1 != me->nsvc) me->yrun_t0 = y0; me->yrun_svc = me->nsvc; } - uint64_t bo = me ? yield_backoff_ns(y0 - me->yrun_t0) : 0; - if (bo) { me->nbackoff++; me->sleeping = 1; struct timespec ts = {0, (long)bo}; nanosleep(&ts, 0); me->sleeping = 0; } - else sched_yield(); - if (me && me->handle == 0x1001) { g_main_yield_ns += now_ns() - y0; g_main_yield_n++; } - } + else thr_yield(pthread_getspecific(g_keythr)); return 1; } case 0x0C: { thr_t *t = h_get((uint32_t)regs[1], O_THREAD); regs[0] = t ? 0 : R_BAD_HANDLE; regs[1] = t ? (uint64_t)t->prio : 0; return 1; } diff --git a/dm/tools/patch_nso.py b/dm/tools/patch_nso.py index 8227724..016c741 100644 --- a/dm/tools/patch_nso.py +++ b/dm/tools/patch_nso.py @@ -6,6 +6,7 @@ # mrs tpidrro_el0 → read thread slot KTLR (Horizon ThreadLocalRegion pointer) # svc #n, mrs cntpct_el0 → host gate (gate address taken from thread slot KGATE; once in C, gates.tsv is looked up by return point) # sub sp, sp, xN → xN rounded up to 16 first (Apple's kernel faults on a misaligned sp, Horizon does not; site_sp_align) +# spin wait loops → a copy whose back-edges pass a spin gate (the host yields there; a busy poll costs nothing on the Switch, see "Spin waits") # Registered cuts (CUTS, restricted by NSO build ID): before rewriting, the original bytes are asserted one by one and any mismatch aborts; results are listed in .cuts.tsv # Registered observation gates (OBS, restricted by build ID): one instruction at the entry is replaced with "enter the host observation callback → execute the original instruction → jump back", so diagnostics modify zero code at runtime; listed in .obs.tsv # The key page only stores "actual key number × 8" at the end of the data segment, and patch code fetches thread slots indirectly through the key page: the loader may use any key number. @@ -14,6 +15,7 @@ import struct, re, sys, os from itertools import combinations import lz4.block from capstone import * +from capstone.arm64 import ARM64_OP_IMM, ARM64_OP_MEM, ARM64_OP_REG src, outdir, NAME = sys.argv[1], sys.argv[2], sys.argv[3] d = open(src, 'rb').read() @@ -676,9 +678,34 @@ starts = sorted(x for x in starts if x < TEXT_END) sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) import nso_code DATA_BLOCKS = nso_code.data_blocks(IMG, TEXT_END, md) +CODE_RANGES = list(zip([0] + [e for _, e in DATA_BLOCKS], [s for s, _ in DATA_BLOCKS] + [TEXT_END])) +md_lite = Cs(CS_ARCH_ARM64, CS_MODE_ARM) # no details: the x18 scan reads only the operand text +R18 = re.compile(r'\b[xw]18\b') +def x18_words(lo, hi): + """offsets in [lo, hi) of the instructions whose operands name x18 / w18 (as has18). Decoded 16 KB at a time without details, going on after a word capstone cannot decode: + a third of the time of decoding one word at a time with details (Paper Mario: The Thousand-Year Door main, 437 sites in 300,000 words either way)""" + out, a = set(), lo + while a < hi: + end = min(hi, a + 0x4000); last = a - 4 + for addr, size, mn, op in md_lite.disasm_lite(bytes(IMG[a:end]), a): + last = addr + if R18.search(op): out.add(addr) + a = last + 4 if last + 4 >= end else last + 8 # the chunk is done, or the word after the last decoded one is no instruction + return out +x18 = set().union(*(x18_words(s, e) for s, e in CODE_RANGES)) kinds = {} -for o in (o for s, e in zip([0] + [e for _, e in DATA_BLOCKS], [s for s, _ in DATA_BLOCKS] + [TEXT_END]) for o in range(s, e, 4)): +# Spin wait candidates (see "Spin waits" below): backward direct branches (b, b.cond, cbz / cbnz, tbz / tbnz) at most SPIN_WIN bytes back, by target +SPIN_WIN, SPIN_TOPS = 512, {0x16, 0x17, 0x54, 0x34, 0x35, 0xB4, 0xB5, 0x36, 0x37, 0xB6, 0xB7} +spin_cand = {} +for o in (o for s, e in CODE_RANGES for o in range(s, e, 4)): w = struct.unpack_from('> 24 in SPIN_TOPS: + t = w >> 24 + if t in (0x16, 0x17): d = ((w & 0x3FFFFFF) - (1 << 26)) * 4 + elif t & 0x7E == 0x36: d = (((w >> 5) & 0x3FFF) - (1 << 14)) * 4 if w & 0x40000 else 4 + elif t == 0x54: d = (((w >> 5) & 0x7FFFF) - (1 << 19)) * 4 if w & 0x800000 and not w & 0x10 else 4 + else: d = (((w >> 5) & 0x7FFFF) - (1 << 19)) * 4 if w & 0x800000 else 4 + if -SPIN_WIN <= d <= 0: spin_cand.setdefault(o + d, []).append(o) if w & 0xFFE0001F == 0xD4000001: kinds[o] = ('svc', (w >> 5) & 0xFFFF); continue if w & 0xFFFFFFE0 == 0xD53BD040: kinds[o] = ('tpidr_r', None); continue if w & 0xFFFFFFE0 == 0xD51BD040: kinds[o] = ('tpidr_w', None); continue @@ -689,8 +716,7 @@ for o in (o for s, e in zip([0] + [e for _, e in DATA_BLOCKS], [s for s, _ in DA if w & 0xFFFFFFE0 == 0xD53B0020: kinds[o] = ('ctr', w & 31); continue # sub sp, sp, xN (extended register, LSL #0): sp may end up misaligned, which Horizon allows and Apple's kernel does not (site_sp_align); with x18 it is an x18 site (site_sp) if w & 0xFFE0FFFF == 0xCB2063FF and (w >> 16) & 31 != 18: kinds[o] = ('sp_align', (w >> 16) & 31); continue - for ins in md.disasm(IMG[o:o + 4], o): - if has18(ins): kinds[o] = ('x18', None) + if o in x18: kinds[o] = ('x18', None) # Take the instruction table per function, find the windows and jump tables import bisect @@ -1267,6 +1293,215 @@ if FS_STATE_OFF and BUILD_ID == '0effe4af1dec3a7966b934d4a7c3d2bf': emit(0xbd004660); emit(B(here(), 0x2e77fc)) # orig: str s0,[x19,#0x44]; back fs_put(0x2e77f8, [B(0x2e77f8, start)], 'frame step: the base clock steps by the host\'s vsyncs (state page target fps; 0 = the game\'s measured step)') +# ---- Spin waits (every module) ---- +# A guest loop that polls memory until another thread changes it fills only its own core on the Switch, where any higher-priority thread pinned there preempts it, so the +# spin costs the game nothing; on the host it takes a whole core from the threads doing the work. Paper Mario: The Thousand-Year Door's gfx::ParallelManager workers +# (main+0xa8e680, three, pinned at priority 45) poll their job list this way while a batch is pending; on an iPhone Rogueport ran at 24 fps with the three at 94% of a core each. +# A loop counts as a spin wait only when it provably cannot leave by itself. From its head T, every pass along the region R (the paths from T back to T, within SPIN_WIN bytes +# after T) runs only loads, register and flag arithmetic, direct branches and barriers: no store, call, SVC, system register or event wait; and no register or flag that is +# live at T is written in R (no counter, cursor or delay). One exception: an exclusive add of a constant (ld(a)xr; add/sub #k; st(l)xr; cbnz) that every path back to T +# undoes on the same address (a worker that registers with a job, sees it finished and backs out). Such a pass leaves memory and registers as it found them, so only another +# thread can end the loop, and a pause there changes only timing. R is copied into the patch area; the original back-edges jump to the copy (no pause on the first pass, so a +# single retry costs nothing), the copy's back-edges go through a spin gate, which the host takes as a yield with the yield back-off (hle_thread.c dm_spin_wait), and exits +# from the copy go back to the original code. The gate keeps every register and the flags (entry.S) +SPIN_PATHS, SPIN_STEPS = 256, 4096 # complete paths back to T; search steps (R itself lies within SPIN_WIN bytes) +SPIN_LSE = ('ldadd', 'ldclr', 'ldeor', 'ldset', 'ldsmax', 'ldsmin', 'ldumax', 'ldumin') +SPIN_BAD = {'bl', 'blr', 'br', 'ret', 'eret', 'svc', 'hvc', 'smc', 'brk', 'hlt', 'udf', 'mrs', 'msr', 'sys', 'sysl', 'wfe', 'wfi', 'sev', 'sevl', 'clrex', 'dc', 'ic', 'at', + 'tlbi', 'drps', 'dcps1', 'dcps2', 'dcps3', 'blraa', 'blrab', 'blraaz', 'blrabz', 'braa', 'brab', 'braaz', 'brabz', 'retaa', 'retab', 'eretaa', 'eretab'} +spin_dec = {} +def spin_ins(a): + if a not in spin_dec: spin_dec[a] = next(md.disasm(bytes(IMG[a:a + 4]), a), None) + return spin_dec[a] +def spin_norm(n): + n = {'fp': 'x29', 'lr': 'x30', 'wsp': 'sp'}.get(n, n) + if n in ('wzr', 'xzr'): return None + m = re.fullmatch(r'[wx](\d+)', n) + if m: return 'x' + m.group(1) + m = re.fullmatch(r'[bhsdqv](\d+)', n) + return 'v' + m.group(1) if m else n +def spin_rw(ins): + """(registers read, registers written), w / x and the vector views merged. Capstone's own lists are not enough: it lists a compare's first operand as written and the flags + as read by cbz / tbz, and leaves out the destination of adds (sdk+0x31033c in Paper Mario: The Thousand-Year Door, adds w9,w9,#8; b.ne: a counted loop). So every register + operand counts as well: operand 0 written and the others read, or for a load all of them written (its address registers are in capstone's reads)""" + m = ins.mnemonic + regs = [spin_norm(ins.reg_name(o.reg)) for o in ins.operands if o.type == ARM64_OP_REG] + ops = set(regs) - {None} + if m in ('cbz', 'cbnz', 'tbz', 'tbnz'): return ops, set() + if m.startswith('b.'): return {'nzcv'}, set() + if m == 'b': return set(), set() + if m in ('cmp', 'cmn', 'tst', 'fcmp', 'fcmpe'): return ops, {'nzcv'} + if m in ('ccmp', 'ccmn', 'fccmp', 'fccmpe'): return ops | {'nzcv'}, {'nzcv'} + r, w = ins.regs_access() + r, w = {spin_norm(ins.reg_name(x)) for x in r} - {None}, {spin_norm(ins.reg_name(x)) for x in w} - {None} + if m.startswith(('ld', 'prfm', 'prfum')): return r, w | ops + return r | (set(regs[1:]) - {None}), w | (set(regs[:1]) - {None}) +def spin_kind(ins): + """'br' direct branch, 'pc' adr / adrp, 'ok' a load or register / flag arithmetic, None anything else""" + if ins is None: return None + m = ins.mnemonic + if m in ('b', 'cbz', 'cbnz', 'tbz', 'tbnz') or m.startswith('b.'): return 'br' + if m in ('adr', 'adrp'): return 'pc' + if m in SPIN_BAD or m.startswith(('st', 'cas', 'swp') + SPIN_LSE): return None + mem = any(o.type == ARM64_OP_MEM for o in ins.operands) + if m.startswith(('ld', 'prfm', 'prfum')): return 'ok' if mem else None # no memory operand: a pc-relative literal load + return None if mem else 'ok' +def spin_rmw(a): + """ld(a)xr wA,[xN]; add|sub wB,wA,#k; st(l)xr wS,wB,[xN]; cbnz wS, at a → (xN, ±k, registers written, the four instructions), or None""" + l = [spin_ins(a + 4 * i) for i in range(4)] + if None in l or l[0].mnemonic not in ('ldxr', 'ldaxr') or l[1].mnemonic not in ('add', 'sub') or l[2].mnemonic not in ('stxr', 'stlxr') or l[3].mnemonic != 'cbnz': return None + r = lambda x, i: x.reg_name(x.operands[i].reg) + if len(l[1].operands) != 3 or l[1].operands[2].type != ARM64_OP_IMM or 'lsl' in l[1].op_str: return None + base = l[0].operands[1].mem + if base.disp or base.index or l[2].operands[2].mem.base != base.base or l[2].operands[2].mem.disp: return None + if r(l[1], 1) != r(l[0], 0) or r(l[2], 1) != r(l[1], 0) or r(l[3], 0) != r(l[2], 0) or l[3].operands[1].imm != a: return None + k = l[1].operands[2].imm * (1 if l[1].mnemonic == 'add' else -1) + return spin_norm(l[0].reg_name(base.base)), k, {spin_norm(r(l[0], 0)), spin_norm(r(l[1], 0)), spin_norm(r(l[2], 0))} - {None}, l +def spin_rebranch(ins, pc, tgt): + """the direct branch ins re-encoded at pc with target tgt""" + w = struct.unpack('> 2) & 0x3FFF) << 5) + assert -(1 << 20) <= off < (1 << 20) and off % 4 == 0, f'spin wait branch out of range {pc:#x}' + return (w & ~(0x7FFFF << 5)) | (((off >> 2) & 0x7FFFF) << 5) +spin_touched = set(kinds) | done | {o for off, oid, ow, why in obs for o in (off,)} | {o for off, ob, nb, why in cuts for o in range(off, off + len(ob) // 2, 4)} +# Instruction words that end every path through them, told apart by their encoding alone: bl; br / blr / ret; svc / brk; str / strb / strh and the vector str (unsigned offset; +# unscaled, pre / post-index; register offset); stp / stnp; ldadd and the other LSE atomics. Exclusive stores are left to the full check (the exclusive add), and so are the pre / +# post-index loads: a seqlock read in SDK 15's nn::time (sdk+0x3d3270) moves a base it sets again on each pass +SPIN_STOP = ((0xFC000000, 0x94000000), (0xFE000000, 0xD6000000), (0xFF000000, 0xD4000000), (0x3FC00000, 0x39000000), (0x3FC00000, 0x3D000000), (0x3FC00000, 0x3D800000), + (0x3FE00000, 0x38000000), (0x3FE00C00, 0x38200800), (0x3FE00000, 0x3C000000), (0x3FE00000, 0x3C800000), (0x3A400000, 0x28000000), (0x3F200C00, 0x38200000)) +def spin_quick(T, hi): + """a back-edge to T can be reached from T without passing a word of SPIN_STOP (only such heads get the full check, which decodes)""" + seen, todo = set(), [T] + while todo: + a = todo.pop() + if a in seen or not T <= a <= hi: continue + seen.add(a); w = struct.unpack_from('> 24 + if 0x14 <= t <= 0x17: succ = (a + (((w & 0x3FFFFFF) ^ 0x2000000) - 0x2000000) * 4,) + elif t == 0x54 or t & 0x7E == 0x34: succ = (a + ((((w >> 5) & 0x7FFFF) ^ 0x40000) - 0x40000) * 4, a + 4) + elif t & 0x7E == 0x36: succ = (a + ((((w >> 5) & 0x3FFF) ^ 0x2000) - 0x2000) * 4, a + 4) + elif any(w & m == v for m, v in SPIN_STOP): continue + else: succ = (a + 4,) + if T in succ: return True + todo.extend(succ) + return False +class SpinNo(Exception): pass +def spin_region(T, bes): + """the spin wait region with head T (bes: the backward branches to T found by the scan) → dict, or None""" + hi = max(bes) + if any(s < hi + 4 and T < e for s, e in DATA_BLOCKS) or not spin_quick(T, hi): return None + nodes, back, bad, inner = {}, set(), set(), set() + todo = [T] + while todo: + a = todo.pop() + if a in nodes or a in bad: continue + rmw = spin_rmw(a) if a + 12 <= hi and struct.unpack_from(' SPIN_STEPS: raise SpinNo + n = nodes[a] + if n['kind'] == 'rmw': + key = (n['base'], ver.get(n['base'], 0)); delta = dict(delta); delta[key] = delta.get(key, 0) + n['k'] + if n['defs']: ver = dict(ver); ver.update({r: ver.get(r, 0) + 1 for r in n['defs']}) + for s in n['succ']: + if s == T: + cnt[0] += 1 + if cnt[0] > SPIN_PATHS: raise SpinNo + if any(delta.values()): bad_back.add(a) + elif s in R: + if s in onpath: raise SpinNo + walk(s, onpath | {s}, ver, delta) + try: walk(T, {T}, {}, {}) + except SpinNo: return None + valid = back - bad_back + if not valid: return None + R = prune(valid) + # no state carried from one pass to the next: nothing live at T is written in R + live, more = {a: set() for a in R}, True + while more: + more = False + for a in sorted(R, reverse=True): + n, out = nodes[a], set() + for s in n['succ']: + if s == T: out |= live[T] if a in valid else set() + elif s in R: out |= live[s] + new = n['uses'] | (out - n['defs']) + if new != live[a]: live[a] = new; more = True + if live[T] & set().union(*(nodes[a]['defs'] for a in R)): return None + return {'T': T, 'R': sorted(R), 'nodes': nodes, 'valid': valid} +def spin_emit(g): + T, R, nodes, valid = g['T'], g['R'], g['nodes'], g['valid'] + cpy, fixes, tramp = {}, [], {} + for i, a in enumerate(R): + n = nodes[a]; cpy[a] = here(); x = n['ins'][0] + if n['kind'] == 'rmw': + for y in n['ins'][:3]: emit(struct.unpack(':