okooooooo/nxnative-easy
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Keep guest memory queries, budgets and stack ownership consistent

Browse Source 
Incorporate the active SVC improvements from PR #151 around the existing
alias bitmap and mapping registry. Serialize heap changes, physical maps,
stack ownership and memory queries with one lock; report committed alias
runs and borrowed sources; initialize reused logical pages while preserving
live neighbors and lazy first exposure. Retain PR #127's fail-fast host
protection policy and the log formats parsed by the NSP heap probe.

Game smoke tests exposed two integration requirements: Raging Loop's SDK
extends committed windows across borrowed stack sources, so allow recommit
that preserves existing bytes; Tears of the Kingdom's audio helper needs
guest-backed stacks, so allocate host-created guest-thread stacks through
SDK malloc and free them after thread destruction and copyback.

Keep SDK reservation ownership and existing patches. The unused host
reservation allocator from #151 is not included.

Validation: full host suite; focused UBSan, ASan and ThreadSanitizer tests;
real dirty-tracker concurrency tests; heap-probe parser compatibility; Mac
host builds; iOS arm64 syntax checks; all 60 profile layout checks.
Raging Loop and Tears of the Kingdom passed guest-thread self-checks and
ran to deliberate timeouts after the startup fixes. Their shader caches
are stale for this checkout, so rendering/performance are not validated.
No IPA build or iOS device validation was performed.

Co-authored-by: b4f8abbb24dbf56a7d0280f296e16eeb <51352400+b4f8abbb24dbf56a7d0280f296e16eeb@users.noreply.github.com>
This commit is contained in:
seeddsandb4f8abbb24dbf56a7d0280f296e16eeb committed 2026-10-25 12:23:01 +08:00
1 parent 66abbc3d30
commit cae5ec2155
8 files changed
+818 -43
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

dm/host/hle_guestthread.c
+19 -7
View File
@@ -11,20 +11,32 @@ typedef void (*tt_fn)(void *tt);
static void *need(const char *n) { void *p = guest_sym(n); if (!p) die("host guest thread: missing symbol %s", n); return p; }
// an entry shim: first register the thread as host-created (hle_thread.c dm_yield_to_core_peers does not wait for them), then run fn
typedef struct { void (*fn)(void *); void *arg; } spawn_arg;
typedef struct { _Alignas(16) uint8_t tt[0x200]; void *stack_alloc; } host_thread;
void dm_thread_mark_host(void);
static void spawn_tramp(void *p) { spawn_arg a = *(spawn_arg *)p; free(p); dm_thread_mark_host(); a.fn(a.arg); }
void *dm_guest_thread_spawn(void (*fn)(void *), void *arg, int prio, size_t stack_size) {
spawn_arg *sa = malloc(sizeof *sa); if (!sa) die("host guest thread: allocation failed"); sa->fn = fn; sa->arg = arg; fn = spawn_tramp; arg = sa;
stack_size = (stack_size + 0xfff) & ~(size_t)0xfff;
void *tt = calloc(1, 0x200); // nn::os::ThreadType (0x1c0 bytes, with headroom)
void *stack = aligned_alloc(0x1000, stack_size); // the SDK requires the stack to be 4 KiB aligned and its size a multiple of 4 KiB
if (!tt || !stack) die("host guest thread: allocation failed");
int rc = ((create_fn)need("_ZN2nn2os12CreateThreadEPNS0_10ThreadTypeEPFvPvES3_S3_mi"))(tt, fn, arg, stack, stack_size, prio);
host_thread *t = calloc(1, sizeof *t); // ThreadType is first, 0x1c0 bytes with headroom
if (!t) die("host guest thread: allocation failed");
// CreateThread borrows this range through MapMemory. Host malloc has no
// guest backing/state (the audio worker failed here in Tears of the Kingdom).
// Use SDK malloc, retaining the raw pointer until DestroyThread copies back.
t->stack_alloc = ((void *(*)(size_t))need("malloc"))(stack_size + 0xfff);
if (!t->stack_alloc) die("host guest thread: stack allocation failed");
void *stack = (void *)(((uintptr_t)t->stack_alloc + 0xfff) & ~(uintptr_t)0xfff);
int rc = ((create_fn)need("_ZN2nn2os12CreateThreadEPNS0_10ThreadTypeEPFvPvES3_S3_mi"))(t->tt, fn, arg, stack, stack_size, prio);
if (rc) die("host guest thread: CreateThread failed %#x", rc);
((tt_fn)need("_ZN2nn2os11StartThreadEPNS0_10ThreadTypeE"))(tt);
return tt;
((tt_fn)need("_ZN2nn2os11StartThreadEPNS0_10ThreadTypeE"))(t->tt);
return t;
}
void dm_guest_thread_join(void *tt) {
((tt_fn)need("_ZN2nn2os10WaitThreadEPNS0_10ThreadTypeE"))(tt);
((tt_fn)need("_ZN2nn2os13DestroyThreadEPNS0_10ThreadTypeE"))(tt);
host_thread *t = tt;
((void (*)(void *))need("free"))(t->stack_alloc);
free(t);
}
void dm_guest_thread_join(void *tt) { ((tt_fn)need("_ZN2nn2os10WaitThreadEPNS0_10ThreadTypeE"))(tt); ((tt_fn)need("_ZN2nn2os13DestroyThreadEPNS0_10ThreadTypeE"))(tt); }
static struct { int malloc_ok, lock_owned, handle; } g_st;
static void selftest_body(void *arg) {
dm/host/hle_svc.c
+224 -32
View File
@@ -17,18 +17,23 @@
// with an 8 GB reservation, builds from developer accounts that cannot sign that entitlement (free accounts) would exit at this step.
#define RESERVE DM_GAME_MEM_RESERVE // SMBW: the heap needs only 1.47 GB, so the block stays at 5 GB (the total proven on a real device; a single 6 GB block can be reserved, but Metal etc. still need address space)
static uint8_t *g_as; // ASLR region base = the whole block
static uint8_t *g_heap; static size_t g_heap_cur, g_heap_cap;
static uint8_t *g_heap; static _Atomic size_t g_heap_cur; static size_t g_heap_cap;
// The address range lies entirely within the currently opened (readable and writable) part of the guest heap: such reads need no kernel trap to check, memcpy directly (used where guest memory is read every tick, such as diag.c tk_rd)
int dm_guest_heap_readable(uint64_t a, size_t n) { uint64_t b = (uint64_t)(uintptr_t)g_heap; size_t cur = g_heap_cur; return g_heap && a >= b && n <= cur && a - b <= cur - n; }
static uint8_t *g_alias; static size_t g_alias_cap;
static _Atomic uint64_t *g_alias_bits; // alias region page bitmap (alias_pages)
static uint64_t *g_alias_exposed; // one bit per host page ever opened; protected by g_map_lock
static uint8_t *g_stack; static size_t g_stack_cap;
// The address lies in the guest address block (every guest thread stack is here: the game's own heap or the stack region); the big-stack entry (hle_bigstack.c) uses it to decide whether to switch stacks
int dm_guest_as_contains(uintptr_t a) { uintptr_t b = (uintptr_t)g_as; return g_as && a >= b && a - b < RESERVE; }
static void *alias_windows_thread(void *arg);
static void mem_init(void) {
if (g_as) return;
static void mem_init_once(void) {
size_t pg = (size_t)getpagesize();
if (RESERVE < 0x10000000 || DM_GAME_MEM_ALIAS > RESERVE - 0x10000000 ||
DM_GAME_MEM_STACK > RESERVE - 0x10000000 - DM_GAME_MEM_ALIAS ||
((RESERVE | DM_GAME_MEM_ALIAS | DM_GAME_MEM_STACK) & (pg - 1)))
die("guest address-space profile has invalid native boundaries");
g_as = mmap(NULL, RESERVE, PROT_NONE, MAP_PRIVATE | MAP_ANON, -1, 0);
if (g_as == MAP_FAILED) die("reserve guest AS");
// Fixed partition (all inside the reserved block): alias 2.875 GB, stack 256 MB, heap the remaining 1.625 GB.
@@ -41,10 +46,15 @@ static void mem_init(void) {
g_stack = g_alias + g_alias_cap; g_stack_cap = DM_GAME_MEM_STACK;
g_heap = g_stack + g_stack_cap; g_heap_cap = RESERVE - (size_t)(g_heap - g_as);
if (!(g_alias_bits = calloc((g_alias_cap >> 18) + 1, sizeof *g_alias_bits))) die("alias region page bitmap calloc");
if (!(g_alias_exposed = calloc(g_alias_cap / pg / 64 + 1, sizeof *g_alias_exposed))) die("alias exposure bitmap calloc");
LOG("guest AS %p +%#llx heap %p alias %p stack %p", g_as, (unsigned long long)RESERVE, g_heap, g_alias, g_stack);
const char *aw = getenv("DM_ALIAS_WINDOWS");
if (aw && atoi(aw) > 0) { pthread_t th; if (!pthread_create(&th, NULL, alias_windows_thread, (void *)(uintptr_t)atoi(aw))) pthread_detach(th); }
}
static void mem_init(void) {
static pthread_once_t once = PTHREAD_ONCE_INIT;
pthread_once(&once, mem_init_once);
}
// Application memory pool = the 3285 MB the Switch (4 GB model) gives applications. At startup (main+0x1b66160) the game sets the whole total reported by QueryMemoryInfo as the heap, retrying with 2 MB less on failure,
// and hands the whole block to its own StandardAllocator (main+0x32137a0) -- so heap size = pool − image footprint, it has only as much as the host reports.
@@ -68,7 +78,9 @@ enum { I_CoreMask=0, I_PriorityMask=1, I_AliasAddr=2, I_AliasSize=3, I_HeapAddr=
static uint8_t g_seen[256];
// Registry of mapped ranges (MapMemory stacks, MapPhysicalMemory alias pages), QueryMemory reports from it; the SDK relies on it to find free space.
// Stack destinations and their borrowed sources. Physical alias commits live in
// g_alias_bits. One lock covers memory validation, protections, copying, budgets
// and publication; QueryMemory and used-memory queries take the same snapshot.
// g_map_lock is held over a whole lookup + check + change (and over QueryMemory's reads): map_find's index is only valid under it, since map_del moves the last entry into the freed slot.
// The lock used to cover only the add and the delete, so two threads unmapping at once (stacks of exiting threads) could delete through a stale index and drop another live mapping
#define MAX_MAP 4096
@@ -107,7 +119,115 @@ static int64_t alias_unmapped(uint64_t off, uint64_t n) {
return c;
}
static _Atomic int64_t g_alias_mapped; // bytes mapped in the alias region (page bitmap)
static pthread_mutex_t g_phys_mu = PTHREAD_MUTEX_INITIALIZER; // mem_map_limit: the pool check and the map it allows are one step
// Word-at-a-time boundaries of the bitmap run containing p (guest page index).
static uint64_t alias_mask(unsigned n) { return n == 64 ? UINT64_MAX : (UINT64_C(1) << n) - 1; }
static uint64_t alias_run_end(uint64_t p, uint64_t end, int mapped) {
for (uint64_t first = p + 1; first < end;) {
unsigned shift = first & 63, n = (unsigned)(end - first < 64 - shift ? end - first : 64 - shift);
uint64_t bits = atomic_load(&g_alias_bits[first >> 6]);
uint64_t different = (mapped ? ~bits : bits) & (alias_mask(n) << shift);
if (different) return (first & ~UINT64_C(63)) + __builtin_ctzll(different);
first += n;
}
return end;
}
static void alias_query(uint64_t addr, uint64_t *base, uint64_t *size, int *mapped) {
uint64_t p = (addr - (uintptr_t)g_alias) >> 12, lo = p;
*mapped = (atomic_load(&g_alias_bits[p >> 6]) >> (p & 63)) & 1;
while (lo) {
uint64_t first = (lo - 1) & ~UINT64_C(63);
uint64_t bits = atomic_load(&g_alias_bits[first >> 6]);
uint64_t different = (*mapped ? ~bits : bits) & alias_mask((unsigned)(lo - first));
if (different) { lo = first + 64 - __builtin_clzll(different); break; }
lo = first;
}
uint64_t hi = alias_run_end(p, g_alias_cap >> 12, *mapped);
*base = (uintptr_t)g_alias + (lo << 12); *size = (hi - lo) << 12;
}
// Initialize holes using the OLD commit bitmap. Fresh anonymous host pages stay
// lazy; after any exposure, even an untouched 4 KiB sibling may hold stale bytes.
static void alias_initialize(uint64_t off, size_t size) {
size_t pg = (size_t)getpagesize(), ratio = pg >> 12;
uint64_t end = (off + size) >> 12;
for (uint64_t p = off >> 12; p < end;) {
int mapped = (atomic_load(&g_alias_bits[p >> 6]) >> (p & 63)) & 1;
uint64_t next = alias_run_end(p, end, mapped);
if (!mapped) {
uint64_t clear = p;
for (uint64_t q = p; q < next;) {
uint64_t host = q / ratio, stop = (host + 1) * ratio;
if (stop > next) stop = next;
if (!(g_alias_exposed[host >> 6] & (UINT64_C(1) << (host & 63)))) {
if (clear < q) memset(g_alias + (clear << 12), 0, (q - clear) << 12);
clear = stop;
}
q = stop;
}
if (clear < next) memset(g_alias + (clear << 12), 0, (next - clear) << 12);
}
p = next;
}
for (uint64_t h = off / pg, last = (off + size - 1) / pg; h <= last; ++h)
g_alias_exposed[h >> 6] |= UINT64_C(1) << (h & 63);
}
static int memory_contains(uint64_t base, uint64_t capacity, uint64_t addr, uint64_t size) {
return addr >= base && size <= capacity && addr - base <= capacity - size;
}
static uint32_t map_arguments(uint64_t dst, uint64_t src, uint64_t size) {
if ((dst | src) & 0xfff) return 0xCC01;
if (!size || (size & 0xfff)) return 0xCA01;
if (size > UINT64_MAX - src || size > UINT64_MAX - dst) return 0xD401;
return memory_contains((uintptr_t)g_stack, g_stack_cap, dst, size) ? 0 : 0xDC01;
}
static int map_source_busy(uint64_t addr, uint64_t size) { // g_map_lock held; nonwrapping ranges
for (int i = 0; i < g_nmap; ++i)
if (addr < g_map[i].src + g_map[i].size && g_map[i].src < addr + size) return 1;
return 0;
}
// Loader text/patch seals use 16 KiB boundaries, independently of host page size.
static void module_readonly_run(uint64_t off, uint64_t start, uint64_t end,
uint64_t *lo, uint64_t *hi, uint64_t *perm) {
if (start >= end) return;
if (off < start) { if (start < *hi) *hi = start; }
else if (off >= end) { if (end > *lo) *lo = end; }
else { if (start > *lo) *lo = start; if (end < *hi) *hi = end; *perm &= ~UINT64_C(2); }
}
static void module_data_query(const module_t *m, uint64_t off, uint64_t *lo, uint64_t *hi, uint64_t *perm) {
module_readonly_run(off, 0, (m->text_end + 0x3fff) & ~UINT64_C(0x3fff), lo, hi, perm);
if (m->patch_off > 0) module_readonly_run(off, (uint64_t)m->patch_off & ~UINT64_C(0x3fff), m->size, lo, hi, perm);
module_readonly_run(off, m->key_page_off, m->key_page_off + m->key_page_size, lo, hi, perm);
}
static int map_source_backed(uint64_t src, uint64_t size) {
if (memory_contains((uintptr_t)g_heap, g_heap_cur, src, size)) return 1;
if (memory_contains((uintptr_t)g_alias, g_alias_cap, src, size))
return !alias_unmapped(src - (uintptr_t)g_alias, size);
module_t *m = module_of((void *)(uintptr_t)src);
// NRO reload restores backing outside this lock; it cannot be borrowed until
// that lifetime is integrated. Static module data is permanent.
if (!m || m->nro || !m->loaded || !memory_contains((uintptr_t)m->base, m->size, src, size)) return 0;
uint64_t lo = m->data_off ? m->data_off : (m->ro_off ? m->ro_off : m->text_end);
uint64_t hi = m->patch_off > 0 ? (uint64_t)m->patch_off : m->size;
uint64_t off = src - (uintptr_t)m->base, perm = 3;
module_data_query(m, off, &lo, &hi, &perm);
return perm == 3 && hi >= lo && memory_contains(lo, hi - lo, off, size);
}
static void map_source_query(uint64_t addr, uint64_t *base, uint64_t *size, uint32_t *attr, uint64_t *perm) {
uint64_t lo = *base, remaining = *size; // the final free page can end at 2^64
for (int i = 0; i < g_nmap; ++i) {
uint64_t src = g_map[i].src, end = src + g_map[i].size;
uint64_t lower = lo, upper = remaining;
if (addr >= src && addr < end) {
if (src > lower) lower = src;
if (end - lo < upper) upper = end - lo;
*attr = 1; *perm = 0; // Locked / IsBorrowed, not PermissionLocked (bit 4)
} else if (end <= addr && end > lower) lower = end;
else if (src > addr && src - lo < upper) upper = src - lo;
remaining = upper - (lower - lo); lo = lower;
}
*base = lo; *size = remaining;
}
// Diagnostic DM_ALIAS_WINDOWS=<window MB>: every 30 s, the mapped memory of each window of that size in the alias region (windows aligned to the size in absolute addresses, counted from the page
// bitmap; windows with nothing mapped are left out). For sizing an allocator that reserves fixed, aligned windows and commits pages from each window's start: Unreal's FMallocBinned3 takes
@@ -116,12 +236,14 @@ static void *alias_windows_thread(void *arg) {
uint64_t win = (uint64_t)(uintptr_t)arg << 20, a0 = (uint64_t)(uintptr_t)g_alias, a1 = a0 + g_alias_cap;
for (;;) {
sleep(30);
char line[8192]; size_t o = 0; uint64_t tot = 0;
char line[8192] = {0}; size_t o = 0; uint64_t tot = 0;
pthread_mutex_lock(&g_map_lock);
for (uint64_t w = a0 & ~(win - 1); w < a1 && o < sizeof line - 64; w += win) {
uint64_t n = 0, p0 = (w > a0 ? w - a0 : 0) >> 12, p1 = ((w + win < a1 ? w + win : a1) - a0) >> 12;
for (uint64_t p = p0; p < p1; p++) n += atomic_load(&g_alias_bits[p >> 6]) >> (p & 63) & 1;
if (n) { o += snprintf(line + o, sizeof line - o, " +%#llx:%.1f", (unsigned long long)(w - (a0 & ~(win - 1))), n * 4096 / 1048576.0); tot += n; }
}
pthread_mutex_unlock(&g_map_lock);
LOG("[mem] alias windows of %llu MB (window offset:MB mapped), %.1f MB in all:%s", (unsigned long long)(win >> 20), tot * 4096 / 1048576.0, line);
}
return NULL;
@@ -137,12 +259,21 @@ uint64_t svc_dispatch(int num, uint64_t *regs, uint64_t orig_addr) {
case 0x01: { // SetHeapSize(out_addr*, size) → x1 = heap base
size_t size = regs[1];
if (size > g_heap_cap) { LOG("SetHeapSize OOM: req %#zx > cap %#zx", size, g_heap_cap); regs[0] = 0xD401; return 0; }
if (size + image_bytes() > APP_POOL) { // exceeds the application memory pool: ResultLimitReached. At startup the game retries in 2 MB decrements until it fits (main+0x1b66184)
pthread_mutex_lock(&g_map_lock);
size_t previous = g_heap_cur, image = image_bytes();
if (size < previous && map_source_busy((uintptr_t)g_heap + size, previous - size)) {
pthread_mutex_unlock(&g_map_lock); regs[0] = 0xD401; return 0;
}
if (image > APP_POOL || size > APP_POOL - image) { // ResultLimitReached; games retry in 2 MB decrements
// Logged once (Mario's retries would repeat it): a game that sizes its heap as pool − used memory and does not retry stops here when its modules are larger than the used memory reported
// (profile mem_used; the NSP script's probe boot reads this line, and sets mem_used when no SetHeapSize succeeds afterwards)
static int once; if (!once++) LOG("[mem] SetHeapSize %#zx + modules %#zx > app pool %#llx (used memory reported %#llx + bss %#zx)", size, image_bytes() - big_bss_bytes(),
(unsigned long long)APP_POOL, (unsigned long long)DM_GAME_MEM_USED, big_bss_bytes());
regs[0] = 0x10801; return 0;
pthread_mutex_unlock(&g_map_lock); regs[0] = 0x10801; return 0;
}
uint64_t charged = (uint64_t)DM_GAME_MEM_USED + big_bss_bytes() + atomic_load(&g_alias_mapped);
if (DM_GAME_MEM_MAP_LIMIT && (charged > APP_POOL || size > APP_POOL - charged)) {
pthread_mutex_unlock(&g_map_lock); regs[0] = 0x10801; return 0;
}
int rc = 0;
if (size > g_heap_cur) rc = mprotect(g_heap + g_heap_cur, size - g_heap_cur, PROT_READ | PROT_WRITE);
@@ -151,18 +282,29 @@ uint64_t svc_dispatch(int num, uint64_t *regs, uint64_t orig_addr) {
if (g_flag.svc) LOG("SetHeapSize %#zx -> base %p mprotect=%d errno=%d", size, g_heap, rc, errno);
else if (size > g_heap_cur) LOG("[mem] heap %.2f GB (heap region %.2f GB)", size / 1073741824.0, g_heap_cap / 1073741824.0);
g_heap_cur = size;
pthread_mutex_unlock(&g_map_lock);
regs[0] = 0; regs[1] = (uint64_t)g_heap; return 0;
}
case 0x02: regs[0] = 0; return 0; // SetMemoryPermission
case 0x03: regs[0] = 0; return 0; // SetMemoryAttribute
case 0x02: case 0x03: { // Permission/attribute stubs must not modify a borrowed source.
uint64_t addr = regs[0], size = regs[1];
pthread_mutex_lock(&g_map_lock);
int busy = size > UINT64_MAX - addr || (size && map_source_busy(addr, size));
pthread_mutex_unlock(&g_map_lock);
regs[0] = busy ? 0xD401 : 0; return 0;
}
case 0x04: { // MapMemory(dst, src, size): when creating a thread the SDK moves the stack on the heap into the stack region
uint64_t dst = regs[0], src = regs[1], size = regs[2];
uint32_t result = map_arguments(dst, src, size);
if (result) { regs[0] = result; return 0; }
pthread_mutex_lock(&g_map_lock);
if ((uint8_t *)dst < g_stack || (uint8_t *)dst + size > g_stack + g_stack_cap || map_find(dst, size)) {
pthread_mutex_unlock(&g_map_lock); LOG("MapMemory target invalid or in use %#llx+%#llx", dst, size); regs[0] = 0xDC01; return 0; }
if (map_find(dst, size) || map_source_busy(src, size) || !map_source_backed(src, size)) result = 0xD401;
else if (g_nmap == MAX_MAP) result = 0xCE01;
if (result) { pthread_mutex_unlock(&g_map_lock); regs[0] = result; return 0; }
// Apple pages are 16K, Horizon pages 4K: the accessible region is extended to a 16K boundary (the extra pages opened are unused pages of the same region, semantics unaffected)
uint64_t lo = dst & ~0x3FFFull, hi = (dst + size + 0x3FFF) & ~0x3FFFull;
uint64_t mask = (size_t)getpagesize() - 1;
uint64_t lo = dst & ~mask, hi = (dst + size + mask) & ~mask;
if (mprotect((void *)lo, hi - lo, PROT_READ | PROT_WRITE)) die("MapMemory mprotect errno=%d", errno);
dm_guest_write_prepare((void *)lo, hi - lo);
memcpy((void *)dst, (void *)src, size); // move semantics: the source region is not accessed while mapped (Horizon makes it inaccessible)
map_add(dst, size, src, 0x0B /*MappedMemory*/);
pthread_mutex_unlock(&g_map_lock);
@@ -171,10 +313,13 @@ uint64_t svc_dispatch(int num, uint64_t *regs, uint64_t orig_addr) {
}
case 0x05: { // UnmapMemory(dst, src, size): contents are moved back to the source region; a 16K page may be shared with an adjacent mapping, permissions are not revoked
uint64_t dst = regs[0], src = regs[1], size = regs[2];
uint32_t result = map_arguments(dst, src, size);
if (result) { regs[0] = result; return 0; }
pthread_mutex_lock(&g_map_lock);
int k = map_find(dst, size);
if (!k || g_map[k - 1].base != dst || g_map[k - 1].size != size || g_map[k - 1].src != src) {
pthread_mutex_unlock(&g_map_lock); LOG("UnmapMemory does not match a mapping %#llx+%#llx", dst, size); regs[0] = 0xD401; return 0; }
dm_guest_write_prepare((void *)src, size);
memcpy((void *)src, (void *)dst, size);
map_del(k - 1);
pthread_mutex_unlock(&g_map_lock);
@@ -185,14 +330,31 @@ uint64_t svc_dispatch(int num, uint64_t *regs, uint64_t orig_addr) {
// MemoryInfo: base(0) size(8) state(0x10) attr(0x14) perm(0x18) ...
uint8_t *a = (uint8_t *)addr;
uint64_t base, size, state, perm;
uint32_t attr = 0;
int k;
pthread_mutex_lock(&g_map_lock); // the registry branches read g_map; the other branches are cheap, so the lock simply spans the whole chain
uint64_t ab = (uintptr_t)g_alias, ae = ab + g_alias_cap, rb = 0, rn = 0;
int mapped = 0, in_alias = addr >= ab && addr < ae;
if (in_alias) alias_query(addr, &rb, &rn, &mapped);
if (a >= g_heap && a < g_heap + g_heap_cur) { base=(uint64_t)g_heap; size=g_heap_cur; state=5/*Heap*/; perm=3; }
else if ((k = map_find(addr, 1))) { base=g_map[k-1].base; size=g_map[k-1].size; state=g_map[k-1].state; perm=3; }
else if (in_alias && mapped) { base=rb; size=rn; state=5; perm=3; }
else if (a >= g_as && a < g_as + RESERVE) { // free: up to the adjacent mapping/heap boundary
uint64_t lo = (uint64_t)g_as, hi = (uint64_t)g_as + RESERVE;
if (a >= g_heap) lo = (uint64_t)g_heap + g_heap_cur;
else if (hi > (uint64_t)g_heap) hi = (uint64_t)g_heap;
if (in_alias) {
if (rb > ab && rb > lo) lo = rb;
if (rb + rn < ae && rb + rn < hi) hi = rb + rn;
} else if (g_alias_cap && addr < ab) {
alias_query(ab, &rb, &rn, &mapped);
uint64_t edge = mapped ? ab : rb + rn;
if ((mapped || edge < ae) && edge < hi) hi = edge;
} else if (g_alias_cap) {
alias_query(ae - 1, &rb, &rn, &mapped);
uint64_t edge = mapped ? ae : rb;
if ((mapped || edge > ab) && edge > lo) lo = edge;
}
for (int i = 0; i < g_nmap; i++) {
uint64_t e = g_map[i].base + g_map[i].size;
if (e <= addr && e > lo) lo = e;
@@ -212,6 +374,11 @@ uint64_t svc_dispatch(int num, uint64_t *regs, uint64_t orig_addr) {
else if (off < da) { base = mb + ro; size = da - ro; state = 3/*Code*/; perm = 1; }
else if (off < pa) { base = mb + da; size = pa - da; state = 4/*CodeData*/; perm = 3; }
else { base = mb + pa; size = mm->size - pa; state = 4/*CodeData*/; perm = 5; }
if (state == 4 && perm == 3) {
uint64_t lo = base - mb, hi = lo + size;
module_data_query(mm, off, &lo, &hi, &perm);
base = mb + lo; size = hi - lo;
}
}
else {
// The address lies in "the current host thread's stack": report the whole stack (state 0xB = stack). When the SDK initializes the main thread record it queries here with the sp at that moment and takes the reported range as the main thread stack;
@@ -221,45 +388,68 @@ uint64_t svc_dispatch(int num, uint64_t *regs, uint64_t orig_addr) {
if (addr < top && addr >= top - ssz) { base = top - ssz; size = ssz; state = 0xB/*Stack*/; perm = 3; }
// Other host memory: report "free, no permissions" (the 4 KiB page around it) as Horizon does for unmapped addresses. It used to report Code+RX, and GetExceptionInfo walking back from main merged on through host memory down to 0
else { base=addr & ~0xFFFull; size=0x1000; state=0/*Free*/; perm=0; } }
map_source_query(addr, &base, &size, &attr, &perm);
pthread_mutex_unlock(&g_map_lock);
if (mi) { uint64_t *m=(uint64_t*)mi; m[0]=base; m[1]=size; ((uint32_t*)mi)[4]=state; ((uint32_t*)mi)[5]=0; ((uint32_t*)mi)[6]=perm; ((uint32_t*)mi)[7]=0; ((uint32_t*)mi)[8]=0; }
if (mi) {
struct { uint64_t base, size; uint32_t state, attr, perm, ipc, device, reserved; }
out = {base, size, (uint32_t)state, attr, (uint32_t)perm, 0, 0, 0};
_Static_assert(sizeof out == 40, "MemoryInfo ABI size");
memcpy((void *)(uintptr_t)mi, &out, sizeof out);
}
regs[0] = 0; regs[1] = 0; return 0;
}
case 0x2C: case 0x2D: { // MapPhysicalMemory / UnmapPhysicalMemory(addr, size): only allowed inside the alias region (used by the SDK's VammManager)
uint8_t *a = (uint8_t *)regs[0]; size_t size = regs[1];
if (a < g_alias || a + size > g_alias + g_alias_cap) { LOG("SVC %#x outside the alias region %p+%#zx", num, a, size); regs[0] = 0xD401; return 0; }
uintptr_t a = regs[0]; size_t size = regs[1];
if (!size || ((a | size) & 0xfff) || !memory_contains((uintptr_t)g_alias, g_alias_cap, a, size)) {
LOG("SVC %#x outside the alias region %p+%#zx", num, (void *)a, size); regs[0] = 0xD401; return 0;
}
pthread_mutex_lock(&g_map_lock);
// The SDK extends committed windows across borrowed stack sources
// (Raging Loop, StandardAllocator). Recommit preserves every live byte;
// only decommit can invalidate backing required for eventual copyback.
if (num == 0x2D && map_source_busy(a, size)) { pthread_mutex_unlock(&g_map_lock); regs[0] = 0xD401; return 0; }
uint64_t off = a - (uintptr_t)g_alias;
int64_t need = alias_unmapped(off, size) * 4096;
// Profile mem_map_limit: a map that would take the process past the application pool fails with ResultLimitReached, as on Horizon (the pool less the used memory GetInfo reports).
// Pokémon Legends: Z-A's nninitStartup allocates and frees everything allocatable (pool − 64 MB, mapped at the top of its StandardAllocator region: 12 GB, 4 GB after the cut in its patches.py); the SDK keeps the freed pages mapped
// and maps new ones for later allocations, until a map fails: then it unmaps the cached block and retries (measured on the Mac: the refused 4 MB map at +0x43e2c000 was followed by
// UnmapPhysicalMemory of the 3.15 GB block and the same map again). Without the limit the mapped total went past the pool in the first minute and the freed block stayed mapped.
// Off by default: Ace Attorney Investigations Collection maps a little more than the 1 GB pool its profile reports
if (DM_GAME_MEM_MAP_LIMIT && num == 0x2C) {
pthread_mutex_lock(&g_phys_mu);
int64_t need = alias_unmapped((uint64_t)(a - g_alias), size) << 12, room = (int64_t)APP_POOL - (int64_t)(g_heap_cur + DM_GAME_MEM_USED + big_bss_bytes()) - atomic_load(&g_alias_mapped); // the pool less the used memory GetInfo reports
if (need > room) {
pthread_mutex_unlock(&g_phys_mu);
int64_t room = (int64_t)APP_POOL - (int64_t)(g_heap_cur + DM_GAME_MEM_USED + big_bss_bytes()) - atomic_load(&g_alias_mapped);
if (need && need > room) {
pthread_mutex_unlock(&g_map_lock);
static _Atomic int nref; if (atomic_fetch_add(&nref, 1) < 8) LOG("[mem] MapPhysicalMemory +%#zx refused: %.2f GB mapped, the application pool is full (ResultLimitReached, the SDK frees cached pages and retries)",
size, atomic_load(&g_alias_mapped) / 1073741824.0);
regs[0] = 0x10801; return 0;
}
}
// Host pages are 16 KB, Horizon pages 4 KB: a map is rounded out to host pages (the extra neighboring 4K pages are inside the alias region and merely become writable early, harmless);
// an unmap only reclaims host pages lying entirely inside the range (a partly covered page may still be used by an adjacent mapping, so it stays writable)
// An edge host page is reclaimed only when no live logical neighbor remains.
uintptr_t pg = (uintptr_t)getpagesize(), lo = (uintptr_t)a, hi = (uintptr_t)a + size;
uintptr_t mlo = num == 0x2C ? lo & ~(pg - 1) : (lo + pg - 1) & ~(pg - 1), mhi = num == 0x2C ? (hi + pg - 1) & ~(pg - 1) : hi & ~(pg - 1);
int rc = 0;
if (mhi > mlo) rc = num == 0x2C ? mprotect((void *)mlo, mhi - mlo, PROT_READ | PROT_WRITE)
: (madvise((void *)mlo, mhi - mlo, MADV_FREE), mprotect((void *)mlo, mhi - mlo, PROT_NONE));
if (rc) { if (DM_GAME_MEM_MAP_LIMIT && num == 0x2C) pthread_mutex_unlock(&g_phys_mu); die("SVC %#x mprotect %p+%#zx errno=%d", num, a, size, errno); }
if (g_flag.svc) LOG("%sPhysicalMemory %p+%#zx", num == 0x2C ? "Map" : "Unmap", a, size);
uintptr_t mlo = lo & ~(pg - 1), mhi = (hi + pg - 1) & ~(pg - 1);
int commit = num == 0x2C;
int64_t changed = commit ? need : (int64_t)size - need;
if (!commit) {
if ((lo - mlo) / 4096 != (uint64_t)alias_unmapped(mlo - (uintptr_t)g_alias, lo - mlo)) mlo += pg;
if ((mhi - hi) / 4096 != (uint64_t)alias_unmapped(hi - (uintptr_t)g_alias, mhi - hi)) mhi -= pg;
}
if (changed && mhi > mlo) {
if (mprotect((void *)mlo, mhi - mlo, commit ? PROT_READ | PROT_WRITE : PROT_NONE))
die("SVC %#x mprotect %p+%#zx errno=%d", num, (void *)a, size, errno);
if (commit) dm_guest_write_prepare((void *)mlo, mhi - mlo);
else (void)madvise((void *)mlo, mhi - mlo, MADV_FREE);
}
if (commit && changed) alias_initialize(off, size);
if (g_flag.svc) LOG("%sPhysicalMemory %p+%#zx", commit ? "Map" : "Unmap", (void *)a, size);
// how deep the alias region is used (one line per 256 MB crossed): for sizing the reservation and alias region (iOS caps a single reservation at 6 GB)
if (num == 0x2C) { static _Atomic uint64_t hw; uint64_t end = (uint64_t)(a + size - g_alias), old = atomic_load(&hw);
if (num == 0x2C) { static _Atomic uint64_t hw; uint64_t end = off + size, old = atomic_load(&hw);
while (end > old && !atomic_compare_exchange_weak(&hw, &old, end)) {}
if (end > old && end >> 28 != old >> 28) LOG("[mem] alias region mapped up to +%.2f GB (alias region %.2f GB)", end / 1073741824.0, g_alias_cap / 1073741824.0); }
// Total actually mapped in the alias region (one line per 256 MB crossed, up or down): how much physical memory the game uses and whether it keeps growing. Counted with a page bitmap, only pages whose state really changed --
// the SDK maps pages that are already mapped and unmaps ranges larger than what it mapped (Engage unmaps 0x1000000 and then 0x103f000 at the same address), so counting requested sizes went negative
{ int64_t d = alias_pages((uint64_t)(a - g_alias), size, num == 0x2C) << 12, before = atomic_fetch_add(&g_alias_mapped, d), after = before + d;
if (DM_GAME_MEM_MAP_LIMIT && num == 0x2C) pthread_mutex_unlock(&g_phys_mu);
{ int64_t d = alias_pages(off, size, commit) * 4096, before = atomic_fetch_add(&g_alias_mapped, d), after = before + d;
pthread_mutex_unlock(&g_map_lock);
if (before >> 28 != after >> 28) LOG("[mem] alias region mapped %.2f GB (application pool %.2f GB)", after / 1073741824.0, APP_POOL / 1073741824.0); }
regs[0] = 0; return 0;
}
@@ -287,9 +477,11 @@ uint64_t svc_dispatch(int num, uint64_t *regs, uint64_t orig_addr) {
case I_AslrAddr: v = (uint64_t)g_as; break; case I_AslrSize: v = RESERVE; break;
case I_StackAddr: v = (uint64_t)g_stack; break; case I_StackSize: v = g_stack_cap; break;
case I_TotalMem: case I_TotalNonSys: v = APP_POOL; break;
case I_UsedMem: case I_UsedNonSys: v = g_heap_cur + DM_GAME_MEM_USED + big_bss_bytes(); // profile mem_used, 64 MB unless the game needs more (see mem_used in dm/games/README.md)
case I_UsedMem: case I_UsedNonSys: pthread_mutex_lock(&g_map_lock);
v = g_heap_cur + DM_GAME_MEM_USED + big_bss_bytes(); // profile mem_used, 64 MB unless the game needs more (see mem_used in dm/games/README.md)
// with mem_map_limit, the pages mapped in the alias region count too, as on Horizon (the limit in MapPhysicalMemory is the pool less this)
if (DM_GAME_MEM_MAP_LIMIT) v += (uint64_t)atomic_load(&g_alias_mapped); break;
if (DM_GAME_MEM_MAP_LIMIT) v += (uint64_t)atomic_load(&g_alias_mapped);
pthread_mutex_unlock(&g_map_lock); break;
// the system resource the NPDM declares (profile sys_resource; 32 MB without the key). The SDK's IsVirtualAddressMemoryEnabled is "total != 0": Big Brain Academy's nninitStartup
// asks StandardAllocator for 60 GB of virtual address memory when it is, and sets a heap only when it is not (its NPDM declares none)
case I_SysResTotal: v = DM_GAME_SYS_RESOURCE; break; case I_SysResUsed: v = DM_GAME_SYS_RESOURCE ? 0x400000 : 0; break;
dm/host/tests/Makefile
+26
View File
@@ -1,5 +1,6 @@
# dm/host unit tests in one command: make -C dm/host/tests GAME=<base game title ID> run (takes seconds; build output goes to build/ and is not committed)
B = build
PYTHON ?= python3
# game profile header (dm.h needs game_profile.h, generated by the host Makefile into ../gen/<profile>/)
# tests that use the game profile need GAME=<base game title ID> (there is no default game); GINC only expands when used, so tests without the profile build as before
GINC = $(if $(strip $(GAME)),-I../gen/$(GAME) -I../../games/$(GAME),$(error GAME is not set: make GAME=<base game title ID>: $(shell python3 ../../games/game.py list | awk -F'\t' '{printf "%s (%s) ", $$1, $$3}')))
@@ -91,3 +92,28 @@ $(B):
.PHONY: run tsan
tsan: $(B)/t_audio_spsc_tsan
$(B)/t_audio_spsc_tsan
MEM_FLAGS = -O1 -Wall -Werror -Wno-unused-function -Imemory_profile -I..
MEM_DEPS = t_svc_memory.c memory_profile/game_profile.h ../hle_svc.c ../hle_guestthread.c ../hle.h ../dm.h
$(B)/t_svc_memory: $(MEM_DEPS) | $(B)
clang $(MEM_FLAGS) -fsanitize=undefined -o $@ t_svc_memory.c
$(B)/t_svc_memory_unlimited: $(MEM_DEPS) | $(B)
clang $(MEM_FLAGS) -fsanitize=undefined -DDM_GAME_MEM_MAP_LIMIT=0 -o $@ t_svc_memory.c
$(B)/t_svc_memory_16k: $(MEM_DEPS) | $(B)
clang $(MEM_FLAGS) -fsanitize=address,undefined -DTEST_PAGE_16K -o $@ t_svc_memory.c
$(B)/t_svc_memory_tsan: $(MEM_DEPS) | $(B)
clang $(MEM_FLAGS) -fsanitize=thread -o $@ t_svc_memory.c
$(B)/t_svc_memory_dirty: $(MEM_DEPS) ../../gfx/nvn/nvn_dirty.c ../../gfx/nvn/nvn_dirty.h | $(B)
clang $(MEM_FLAGS) -DTEST_REAL_DIRTY -o $@ t_svc_memory.c ../../gfx/nvn/nvn_dirty.c
.PHONY: memory memory-tsan memory-dirty memory-profile
memory: $(B)/t_svc_memory $(B)/t_svc_memory_unlimited $(B)/t_svc_memory_16k
$(B)/t_svc_memory
$(B)/t_svc_memory_unlimited
$(B)/t_svc_memory_16k
memory-tsan: $(B)/t_svc_memory_tsan
$(B)/t_svc_memory_tsan
memory-dirty: $(B)/t_svc_memory_dirty
$(B)/t_svc_memory_dirty
memory-profile: $(B)/t_svc_memory
$(PYTHON) test_memory_profile.py
run: memory memory-dirty memory-profile
dm/host/tests/memory_profile/game_profile.h
+12
View File
@@ -0,0 +1,12 @@
// Small synthetic address layout for the production memory SVC tests.
#pragma once
#define DM_GAME_MEM_RESERVE UINT64_C(0x14400000)
#define DM_GAME_MEM_ALIAS UINT64_C(0x02000000)
#define DM_GAME_MEM_STACK UINT64_C(0x01400000)
#define DM_GAME_APP_POOL UINT64_C(0x01000000)
#define DM_GAME_MEM_USED UINT64_C(0x00200000)
#define DM_GAME_SYS_RESOURCE UINT64_C(0x02000000)
#define DM_GAME_PROGRAM_ID UINT64_C(0)
#ifndef DM_GAME_MEM_MAP_LIMIT
#define DM_GAME_MEM_MAP_LIMIT 1
#endif
dm/host/tests/t_heap_size.c
+5 -4
View File
@@ -30,6 +30,7 @@ void guest_clock_read_wait(void) {}
uint64_t guest_ns(void) { return 0; }
uint64_t dm_entropy64(void) { return 0; }
int thr_limit_used(void) { return 0; }
void dm_guest_write_prepare(void *addr, size_t size) { (void)addr; (void)size; }
static size_t page, old_size, requested;
static int calls, injected_error, last_prot;
@@ -97,11 +98,10 @@ static void used_memory(size_t heap_size) {
assert(r[0] == 0 && r[1] == heap_size + DM_GAME_MEM_USED);
}
int main(void) {
mem_init();
page = (size_t)getpagesize(); g_heap_cap = 4 * page;
assert(g_heap_cap < APP_POOL);
g_heap = mmap(NULL, g_heap_cap, PROT_NONE, MAP_PRIVATE | MAP_ANON, -1, 0);
assert(g_heap != MAP_FAILED);
g_as = g_heap; // Skip the multi-GB profile reservation; use only these synthetic pages.
// The reservation stays lazy; only four host pages of its heap are used.
success(2 * page); used_memory(2 * page); // initial growth
success(3 * page); used_memory(3 * page); // further growth
success(page); used_memory(page); // shrink
@@ -114,7 +114,8 @@ int main(void) {
success(0); used_memory(0);
success(0); // unchanged empty heap
success(page); success(0); // regrowth after release
assert(munmap(g_heap, g_heap_cap) == 0);
free(g_alias_bits); free(g_alias_exposed);
assert(munmap(g_as, RESERVE) == 0);
puts("SetHeapSize production dispatcher: growth, shrink, unchanged, zero, failure state and fail-fast passed");
return 0;
}
dm/host/tests/t_svc_memory.c
+471
View File
@@ -0,0 +1,471 @@
// Exercise the production dispatcher with real backing, including protection
// failures and concurrent operations. No alternate allocator implementation.
#include "../dm.h"
#include <assert.h>
#include <errno.h>
#include <signal.h>
#include <stdatomic.h>
#include <sys/mman.h>
#include <sys/wait.h>
#include <unistd.h>
static int memory_mprotect(void *addr, size_t size, int prot);
static _Noreturn void memory_abort(void);
static int memory_getpagesize(void) {
#ifdef TEST_PAGE_16K
assert(getpagesize() <= 0x4000);
return 0x4000;
#else
return getpagesize();
#endif
}
#define getpagesize memory_getpagesize
#define mprotect memory_mprotect
#define abort memory_abort
#include "../hle_svc.c"
#undef abort
#undef mprotect
#undef getpagesize
module_t g_mod[MAX_MOD];
int g_nmod;
struct dm_flags g_flag;
void hlecount_svc(int num) { (void)num; }
int svc_thread(int num, uint64_t *regs) { (void)num; (void)regs; return 0; }
void dm_backtrace(uint64_t fp) { (void)fp; }
void guest_clock_read_wait(void) {}
uint64_t guest_ns(void) { return 0; }
uint64_t dm_entropy64(void) { return 0; }
int thr_limit_used(void) { return 0; }
module_t *module_of(const void *p) {
uintptr_t a = (uintptr_t)p;
for (int i = 0; i < g_nmod; ++i) {
uintptr_t b = (uintptr_t)g_mod[i].base;
if (a >= b && a - b < g_mod[i].size) return &g_mod[i];
if (g_mod[i].patch_off < 0 && a >= b + g_mod[i].patch_off && a < b) return &g_mod[i];
}
return NULL;
}
#ifdef TEST_REAL_DIRTY
#include "../../gfx/nvn/nvn_dirty.h"
static void (*write_prepare)(void *, size_t);
static int (*fault_hook)(void *, int, void *);
void dm_fault_hook_register(int (*fn)(void *, int, void *)) { fault_hook = fn; }
void dm_guest_write_prep_register(void (*fn)(void *, size_t)) { write_prepare = fn; }
static void memory_fault(int sig, siginfo_t *info, void *ctx) {
if (fault_hook && fault_hook(info->si_addr, 1, ctx)) return;
_exit(128 + sig);
}
#endif
static _Atomic unsigned prepares, protects;
static uintptr_t readonly_base;
static size_t readonly_size;
void dm_guest_write_prepare(void *addr, size_t size) {
atomic_fetch_add(&prepares, 1);
#ifdef TEST_REAL_DIRTY
if (write_prepare) write_prepare(addr, size);
#endif
if (readonly_size && (uintptr_t)addr < readonly_base + readonly_size && readonly_base < (uintptr_t)addr + size) {
assert(!mprotect((void *)readonly_base, readonly_size, PROT_READ | PROT_WRITE));
readonly_size = 0;
}
}
static int fail_protect;
static uint64_t *failure_regs, failure_input[3];
static int64_t failure_mapped;
static size_t failure_heap;
static int failure_nmap;
static unsigned failure_prepares;
static int memory_mprotect(void *addr, size_t size, int prot) {
size_t pg = memory_getpagesize();
assert(!((uintptr_t)addr & (pg - 1)) && !(size & (pg - 1)) && size);
atomic_fetch_add(&protects, 1);
if (fail_protect) { errno = EACCES; return -1; }
return mprotect(addr, size, prot);
}
static _Noreturn void memory_abort(void) {
// Only the production fatal path reaches this wrapper; assertion failures
// cannot pass these checks. No guest return or metadata publication allowed.
if (!fail_protect || errno != EACCES || !failure_regs ||
memcmp(failure_regs, failure_input, sizeof failure_input) ||
g_alias_mapped != failure_mapped || g_heap_cur != failure_heap ||
g_nmap != failure_nmap || prepares != failure_prepares) _exit(78);
_exit(77);
}
enum { P = 0x1000, BAD = 0xd401, LIMIT = 0x10801 };
static uintptr_t alias, stack, heap;
static uint64_t call(unsigned svc, uint64_t x0, uint64_t x1, uint64_t x2) {
uint64_t r[31] = {x0, x1, x2}; svc_dispatch(svc, r, 0); return r[0];
}
static uint64_t pages(int commit, uintptr_t a, size_t n) { return call(commit ? 0x2c : 0x2d, a, n, 0); }
static uint64_t resize(size_t n) { return call(1, 0, n, 0); }
static uint64_t map(uintptr_t d, uintptr_t s, size_t n) { return call(4, d, s, n); }
static uint64_t unmap(uintptr_t d, uintptr_t s, size_t n) { return call(5, d, s, n); }
static uint64_t info(unsigned id) {
uint64_t r[31] = {0, id}; svc_dispatch(0x29, r, 0); assert(!r[0]); return r[1];
}
typedef struct { uint64_t base, size; uint32_t state, attr, perm, ipc, device, reserved; } memory_info;
static memory_info query(uintptr_t a) {
uint8_t out[57]; memset(out, 0xa7, sizeof out);
uint64_t r[31] = {(uintptr_t)(out + 1), 0, a}; svc_dispatch(6, r, 0);
assert(!r[0] && !r[1] && out[0] == 0xa7);
for (unsigned i = 41; i < sizeof out; ++i) assert(out[i] == 0xa7);
memory_info m; memcpy(&m, out + 1, sizeof m);
assert(!m.ipc && !m.device && !m.reserved && m.size && a >= m.base && a - m.base < m.size);
return m;
}
static void expect(uintptr_t a, uintptr_t base, size_t size, unsigned state, unsigned attr, unsigned perm) {
memory_info m = query(a);
assert(m.base == base && m.size == size && m.state == state && m.attr == attr && m.perm == perm);
}
static void fatal(unsigned svc, uint64_t x0, uint64_t x1, uint64_t x2) {
fflush(NULL); pid_t pid = fork(); assert(pid >= 0);
if (!pid) {
alarm(5);
uint64_t r[31] = {x0, x1, x2}; failure_regs = r;
memcpy(failure_input, r, sizeof failure_input);
failure_mapped = g_alias_mapped; failure_heap = g_heap_cur;
failure_nmap = g_nmap; failure_prepares = prepares; fail_protect = 1;
svc_dispatch(svc, r, 0); _exit(79);
}
int status; assert(waitpid(pid, &status, 0) == pid);
assert(WIFEXITED(status) && WEXITSTATUS(status) == 77);
}
static void alias_lifecycle(void) {
assert(!pages(1, alias + 4 * P, 4 * P));
memset((void *)(alias + 4 * P), 0x53, 4 * P);
assert(!pages(1, alias + 6 * P, 4 * P));
assert(g_alias_mapped == 6 * P);
expect(alias + 7 * P, alias + 4 * P, 6 * P, 5, 0, 3);
expect((uintptr_t)g_as, (uintptr_t)g_as, alias + 4 * P - (uintptr_t)g_as, 0, 0, 0);
expect(stack, alias + 10 * P, heap - alias - 10 * P, 0, 0, 0);
for (size_t i = 0; i < 4 * P; ++i) assert(((uint8_t *)alias)[4 * P + i] == 0x53);
for (size_t i = 8 * P; i < 10 * P; ++i) assert(((uint8_t *)alias)[i] == 0);
unsigned before = protects;
assert(!pages(1, alias + 4 * P, 6 * P) && protects == before);
assert(!pages(0, alias + 5 * P, P));
expect(alias + 5 * P, alias + 5 * P, P, 0, 0, 0);
expect(alias + 6 * P, alias + 6 * P, 4 * P, 5, 0, 3);
assert(*(uint8_t *)(alias + 4 * P) == 0x53 && *(uint8_t *)(alias + 6 * P) == 0x53);
assert(!pages(1, alias + 5 * P, P));
for (size_t i = 0; i < P; ++i) assert(((uint8_t *)alias)[5 * P + i] == 0);
assert(*(uint8_t *)(alias + 4 * P) == 0x53);
assert(!pages(0, alias, 16 * P) && !pages(0, alias, 16 * P) && !g_alias_mapped);
// First commit of a logical sibling after its host page was writable must
// clear retained bytes, even though that logical page was never committed.
size_t pg = memory_getpagesize();
assert(!pages(1, alias, P));
if (pg > P) {
memset((void *)(alias + P), 0xa9, P);
assert(!pages(1, alias + P, P));
for (size_t i = 0; i < P; ++i) assert(((uint8_t *)alias)[P + i] == 0);
}
assert(!pages(0, alias, pg));
assert(pages(1, alias, 0) == BAD && pages(1, alias + 1, P) == BAD);
assert(pages(1, alias, P + 1) == BAD && pages(1, alias - P, P) == BAD);
assert(pages(1, UINTPTR_MAX - P + 1, 2 * P) == BAD);
assert(pages(1, alias + g_alias_cap - P, 2 * P) == BAD);
}
static void ownership(void) {
assert(!pages(1, alias, 16 * P));
memset((void *)alias, 0x34, 16 * P);
assert(!map(stack + P, alias + P, 2 * P));
expect(alias + P, alias + P, 2 * P, 5, 1, 0);
expect(alias, alias, P, 5, 0, 3);
expect(alias + 3 * P, alias + 3 * P, 13 * P, 5, 0, 3);
expect(stack + P, stack + P, 2 * P, 0xb, 0, 3);
assert(map(stack + 8 * P, alias + 2 * P, P) == BAD);
assert(map(stack + 2 * P, alias + 8 * P, P) == BAD);
assert(pages(0, alias, 4 * P) == BAD);
// SDK StandardAllocator extends committed windows across existing stacks.
// Recommit must preserve the borrowed source and its authoritative copy.
assert(!pages(1, alias, 4 * P));
assert(!pages(0, alias, P));
assert(!pages(1, alias, 4 * P)); // initializes a hole beside the borrowed source
assert(!pages(1, alias + 15 * P, 2 * P));
expect(alias + P, alias + P, 2 * P, 5, 1, 0);
assert(*(uint8_t *)(alias + P) == 0x34);
expect(UINTPTR_MAX, UINTPTR_MAX - P + 1, P, 0, 0, 0);
assert(call(2, alias + P, P, 3) == BAD && call(3, alias + P, P, 0) == BAD);
assert(unmap(stack + P, alias + P, P) == BAD);
assert(unmap(stack + P, alias + 4 * P, 2 * P) == BAD);
assert(!pages(0, alias, P) && !pages(1, alias, P));
assert(*(uint8_t *)(alias + P) == 0x34);
memset((void *)(stack + P), 0x76, 2 * P);
readonly_base = alias; readonly_size = memory_getpagesize();
assert(!mprotect((void *)readonly_base, readonly_size, PROT_READ));
assert(!unmap(stack + P, alias + P, 2 * P) && !readonly_size);
for (size_t i = P; i < 3 * P; ++i) assert(((uint8_t *)alias)[i] == 0x76);
expect(alias + P, alias, 17 * P, 5, 0, 3);
assert(!pages(0, alias, 17 * P));
assert(map(stack, alias, P) == BAD);
assert(map(stack + 1, alias, P) == 0xcc01 && map(stack, alias, 0) == 0xca01);
assert(map(alias, alias, P) == 0xdc01 && map(stack, UINTPTR_MAX - P + 1, 2 * P) == BAD);
assert(!resize(4 * 1024 * 1024));
assert(!map(stack, heap + 3 * 1024 * 1024, P));
assert(resize(2 * 1024 * 1024) == BAD && g_heap_cur == 4 * 1024 * 1024);
assert(!resize(6 * 1024 * 1024));
*(uint8_t *)stack = 0x91;
assert(!unmap(stack, heap + 3 * 1024 * 1024, P));
assert(*(uint8_t *)(heap + 3 * 1024 * 1024) == 0x91);
assert(!resize(0));
}
static void module_sources(void) {
uint8_t *b = mmap(NULL, 0x20000, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0);
assert(b != MAP_FAILED);
g_mod[0] = (module_t){.base=b, .size=0x20000, .text_end=0x2800, .ro_off=0x2800,
.data_off=0x3000, .patch_off=0x1e000, .key_page_off=0x8000, .key_page_size=0x4000, .loaded=1};
g_nmod = 1;
assert(!mprotect(b, 0x4000, PROT_READ));
assert(!mprotect(b + 0x8000, 0x4000, PROT_READ));
assert(!mprotect(b + 0x1c000, 0x4000, PROT_READ));
assert(map(stack, (uintptr_t)b + 0x3000, P) == BAD);
assert(map(stack, (uintptr_t)b + 0x7000, 2 * P) == BAD);
assert(map(stack, (uintptr_t)b + 0x1b000, 2 * P) == BAD);
assert(!map(stack, (uintptr_t)b + 0x4000, P));
expect((uintptr_t)b + 0x4000, (uintptr_t)b + 0x4000, P, 4, 1, 0);
expect((uintptr_t)b + 0x8000, (uintptr_t)b + 0x8000, 0x4000, 4, 0, 1);
*(uint8_t *)stack = 0x62;
assert(!unmap(stack, (uintptr_t)b + 0x4000, P) && b[0x4000] == 0x62);
g_mod[0].nro = 1;
assert(map(stack, (uintptr_t)b + 0x4000, P) == BAD);
g_nmod = 0; memset(g_mod, 0, sizeof g_mod); assert(!munmap(b, 0x20000));
}
// Apple has no pthread_barrier. The gate makes races deterministic at entry,
// without assumptions about which operation wins the memory-state lock.
typedef struct { pthread_mutex_t mu; pthread_cond_t cv; unsigned waiting; } gate;
static void gate_wait(gate *g) {
assert(!pthread_mutex_lock(&g->mu));
if (++g->waiting == 2) assert(!pthread_cond_broadcast(&g->cv));
else while (g->waiting < 2) assert(!pthread_cond_wait(&g->cv, &g->mu));
assert(!pthread_mutex_unlock(&g->mu));
}
typedef struct { gate *start; unsigned svc; uint64_t a, b, c, result; } task;
static void *compete(void *arg) {
task *t = arg; gate_wait(t->start); t->result = call(t->svc, t->a, t->b, t->c); return NULL;
}
static void race(task *a, task *b) {
gate g = {PTHREAD_MUTEX_INITIALIZER, PTHREAD_COND_INITIALIZER, 0};
a->start = b->start = &g; pthread_t ta, tb;
assert(!pthread_create(&ta, NULL, compete, a) && !pthread_create(&tb, NULL, compete, b));
assert(!pthread_join(ta, NULL) && !pthread_join(tb, NULL));
assert(!pthread_cond_destroy(&g.cv) && !pthread_mutex_destroy(&g.mu));
}
static void concurrency(void) {
for (unsigned i = 0; i < 40; ++i) {
task h = {.svc=1, .b=8 * 1024 * 1024}, p = {.svc=0x2c, .a=alias, .b=8 * 1024 * 1024};
race(&h, &p);
if (DM_GAME_MEM_MAP_LIMIT) assert((!h.result && p.result == LIMIT) || (h.result == LIMIT && !p.result));
else assert(!h.result && !p.result);
assert(info(I_UsedMem) == DM_GAME_MEM_USED + g_heap_cur + (DM_GAME_MEM_MAP_LIMIT ? g_alias_mapped : 0));
assert(!pages(0, alias, p.b) && !resize(0));
assert(!pages(1, alias, P));
task m = {.svc=4, .a=stack, .b=alias, .c=P}, d = {.svc=0x2d, .a=alias, .b=P};
race(&m, &d);
assert((!m.result && d.result == BAD) || (m.result == BAD && !d.result));
if (!m.result) assert(!unmap(stack, alias, P));
assert(!pages(0, alias, P));
assert(!resize(4 * 1024 * 1024));
m.b = heap + 3 * 1024 * 1024; h.b = 2 * 1024 * 1024;
race(&m, &h);
assert((!m.result && h.result == BAD) || (m.result == BAD && !h.result));
if (!m.result) assert(!unmap(stack, m.b, P));
assert(!resize(0));
assert(!pages(1, alias, 2 * P));
m.b = alias;
task other = {.svc=4, .a=stack + 0x4000, .b=alias, .c=P};
race(&m, &other);
assert((!m.result && other.result == BAD) || (m.result == BAD && !other.result));
assert(!unmap(m.result ? other.a : m.a, alias, P));
other.a = stack; other.b = alias + P;
race(&m, &other);
assert((!m.result && other.result == BAD) || (m.result == BAD && !other.result));
assert(!unmap(stack, m.result ? other.b : m.b, P));
assert(!pages(0, alias, 2 * P));
}
}
static void fragmented_queries(void) {
// Independent byte-sized model checks word boundaries and free runs across
// the alias partition. Preserve every live page's marker on overlapping maps.
enum { N = 160 };
unsigned char live[N] = {0}; uint32_t rng = 0x13579b;
for (unsigned step = 0; step < 1000; ++step) {
rng = rng * 1664525 + 1013904223; unsigned first = rng % N;
rng = rng * 1664525 + 1013904223; unsigned n = 1 + rng % (N - first);
int commit = (rng >> 20) & 1;
assert(!pages(commit, alias + first * P, n * P));
for (unsigned i = first; i < first + n; ++i) {
if (commit) {
assert(*(uint8_t *)(alias + i * P) == (live[i] ? 0x57 : 0));
*(uint8_t *)(alias + i * P) = 0x57;
}
live[i] = commit;
}
unsigned total = 0;
for (unsigned i = 0; i < N; ++i) {
total += live[i];
unsigned lo = i, hi = i + 1;
while (lo && live[lo - 1] == live[i]) --lo;
while (hi < N && live[hi] == live[i]) ++hi;
uintptr_t b = !live[i] && !lo ? (uintptr_t)g_as : alias + lo * P;
uintptr_t e = !live[i] && hi == N ? heap : alias + hi * P;
expect(alias + i * P, b, e - b, live[i] ? 5 : 0, 0, live[i] ? 3 : 0);
}
assert(g_alias_mapped == total * P);
}
assert(!pages(0, alias, N * P));
}
static void lazy_commit(void) {
size_t n = DM_GAME_APP_POOL - DM_GAME_MEM_USED, pg = memory_getpagesize();
char *resident = malloc(n / pg); assert(resident);
assert(!pages(1, alias, n));
assert(!mincore((void *)alias, n, resident));
size_t count = 0;
for (size_t i = 0; i < n / pg; ++i) count += !!(resident[i] & MINCORE_INCORE);
assert(count < 32); // metadata scans must not fault the anonymous backing in
((uint8_t *)alias)[0] = 0x98;
assert(!pages(0, alias, pg) && !pages(1, alias, pg));
for (size_t i = 0; i < pg; ++i) assert(((uint8_t *)alias)[i] == 0);
assert(!pages(0, alias, n)); free(resident);
}
static void metadata_limit(void) {
assert(!pages(1, alias, (MAX_MAP + 1) * P));
for (unsigned i = 0; i < MAX_MAP; ++i) assert(!map(stack + i * P, alias + i * P, P));
unsigned before = protects;
assert(map(stack + MAX_MAP * P, alias + MAX_MAP * P, P) == 0xce01);
assert(protects == before && g_nmap == MAX_MAP);
expect(alias + MAX_MAP * P, alias + MAX_MAP * P, P, 5, 0, 3);
// Releasing one destination makes its metadata available again.
assert(!unmap(stack + 123 * P, alias + 123 * P, P));
assert(!map(stack + 123 * P, alias + MAX_MAP * P, P));
assert(protects == before + 1);
for (unsigned i = 0; i < MAX_MAP; ++i)
assert(!unmap(stack + i * P, alias + (i == 123 ? MAX_MAP : i) * P, P));
assert(!pages(0, alias, (MAX_MAP + 1) * P));
}
#ifdef TEST_REAL_DIRTY
static _Atomic int dirty_done;
static void *protect_worker(void *unused) {
(void)unused;
while (!atomic_load(&dirty_done)) nvn_dirty_protect((void *)alias, P);
return NULL;
}
static void dirty_integration(void) {
struct sigaction sa = {.sa_sigaction=memory_fault, .sa_flags=SA_SIGINFO};
sigemptyset(&sa.sa_mask);
assert(!sigaction(SIGBUS, &sa, NULL) && !sigaction(SIGSEGV, &sa, NULL));
size_t pg = memory_getpagesize(); assert(pg > P);
assert(!pages(1, alias, 2 * pg));
nvn_dirty_add_range((void *)alias, P);
nvn_dirty_add_range((void *)(alias + P), P);
*(uint8_t *)alias = 0x29;
for (unsigned i = 0; i < 20; ++i) {
nvn_dirty_protect((void *)alias, P);
nvn_dirty_protect((void *)(alias + P), P);
uint64_t stamp = nvn_dirty_now();
assert(!pages(0, alias + P, P) && !pages(1, alias + P, P));
assert(nvn_dirty_since((void *)alias, P, stamp));
assert(nvn_dirty_since((void *)(alias + P), P, stamp));
assert(*(uint8_t *)alias == 0x29 && *(uint8_t *)(alias + P) == 0);
assert(!map(stack, alias, P));
*(uint8_t *)stack = 0x29;
nvn_dirty_protect((void *)alias, P); stamp = nvn_dirty_now();
assert(!unmap(stack, alias, P));
assert(nvn_dirty_since((void *)alias, P, stamp));
}
pthread_t thread; assert(!pthread_create(&thread, NULL, protect_worker, NULL));
for (unsigned i = 0; i < 1000; ++i) {
assert(!pages(0, alias + P, P) && !pages(1, alias + P, P));
assert(!map(stack, alias, P)); *(uint8_t *)stack = 0x29;
assert(!unmap(stack, alias, P));
assert(*(uint8_t *)alias == 0x29 && *(uint8_t *)(alias + P) == 0);
}
atomic_store(&dirty_done, 1); assert(!pthread_join(thread, NULL));
nvn_dirty_remove_range((void *)alias); nvn_dirty_remove_range((void *)(alias + P));
assert(!pages(0, alias, 2 * pg));
puts("real GPU dirty tracking: shared-page recommit, copyback and concurrent protection passed");
}
#endif
static void failures(void) {
fatal(0x2c, alias, 0x4000, 0);
assert(!pages(1, alias, 0x4000));
fatal(0x2d, alias, 0x4000, 0);
fatal(4, stack, alias, P);
assert(!pages(0, alias, 0x4000));
fatal(1, 0, 0x200000, 0);
assert(!resize(0x200000)); fatal(1, 0, 0, 0); assert(!resize(0));
}
// Exercise the real host helper through SDK-shaped test entry points. Its stack
// allocation must reach MapMemory as guest backing and outlive SDK copyback.
#include "../hle_guestthread.c"
static void (*helper_entry)(void *);
static void *helper_arg, *helper_stack;
static size_t helper_size;
static int helper_started, helper_destroyed, helper_freed;
void dm_thread_mark_host(void) {}
uint32_t thr_cur_handle(void) { return 0x1001; }
static void *helper_malloc(size_t n) {
assert(n < 0x200000); assert(!resize(0x200000));
return (void *)(heap + 32); // unaligned allocation, like SDK malloc
}
static void helper_free(void *p) {
assert(p == (void *)(heap + 32) && helper_destroyed);
assert(*(uint8_t *)helper_stack == 0x87); // DestroyThread already copied back
helper_freed = 1; assert(!resize(0));
}
static int helper_create(void *tt, void (*fn)(void *), void *arg, void *source, size_t n, int prio) {
(void)tt; (void)prio; helper_entry = fn; helper_arg = arg; helper_stack = source; helper_size = n;
return (int)map(stack, (uintptr_t)source, n);
}
static void helper_start(void *tt) { (void)tt; helper_entry(helper_arg); helper_started = 1; }
static void helper_wait(void *tt) { (void)tt; assert(helper_started); }
static void helper_destroy(void *tt) {
(void)tt; assert(!unmap(stack, (uintptr_t)helper_stack, helper_size)); helper_destroyed = 1;
}
void *guest_sym(const char *name) {
if (!strcmp(name, "malloc")) return helper_malloc;
if (!strcmp(name, "free")) return helper_free;
if (!strcmp(name, "_ZN2nn2os12CreateThreadEPNS0_10ThreadTypeEPFvPvES3_S3_mi")) return helper_create;
if (!strcmp(name, "_ZN2nn2os11StartThreadEPNS0_10ThreadTypeE")) return helper_start;
if (!strcmp(name, "_ZN2nn2os10WaitThreadEPNS0_10ThreadTypeE")) return helper_wait;
if (!strcmp(name, "_ZN2nn2os13DestroyThreadEPNS0_10ThreadTypeE")) return helper_destroy;
assert(0); return NULL;
}
static void helper_body(void *arg) { assert(arg == &helper_started); *(uint8_t *)stack = 0x87; }
static void guest_helper(void) {
void *tt = dm_guest_thread_spawn(helper_body, &helper_started, 9, 0x10000);
assert(helper_started && !helper_destroyed && !helper_freed);
assert(resize(0) == BAD);
dm_guest_thread_join(tt); assert(helper_freed && !g_nmap && !g_heap_cur);
}
static void *init_worker(void *arg) { *(uint64_t *)arg = info(I_AliasAddr); return NULL; }
int main(int argc, char **argv) {
if (argc == 2 && !strcmp(argv[1], "--heap-probe")) {
mem_init();
assert(resize(g_heap_cap + 0x200000) == BAD);
g_nmod = 1; g_mod[0].size = 0xa00000;
assert(resize(0x800000) == LIMIT);
return 0;
}
pthread_t threads[8]; uint64_t addresses[8];
for (unsigned i = 0; i < 8; ++i) assert(!pthread_create(&threads[i], NULL, init_worker, &addresses[i]));
for (unsigned i = 0; i < 8; ++i) { assert(!pthread_join(threads[i], NULL)); assert(addresses[i] == addresses[0]); }
alias = addresses[0]; stack = info(I_StackAddr); heap = info(I_HeapAddr);
lazy_commit(); alias_lifecycle(); ownership(); module_sources(); failures(); concurrency(); fragmented_queries();
guest_helper();
if (!DM_GAME_MEM_MAP_LIMIT) metadata_limit();
#ifdef TEST_REAL_DIRTY
dirty_integration();
#endif
assert(!g_nmap && !g_alias_mapped && !g_heap_cur);
assert(info(I_UsedMem) == DM_GAME_MEM_USED);
free(g_alias_bits); free(g_alias_exposed); assert(!munmap(g_as, RESERVE));
printf("memory SVCs passed (map limit=%d, host page=%d)\n", DM_GAME_MEM_MAP_LIMIT, memory_getpagesize());
return 0;
}
dm/host/tests/test_memory_profile.py
+26
View File
@@ -0,0 +1,26 @@
"""The NSP heap probe must still understand real dispatcher diagnostics."""
import subprocess
import sys
import unittest
from pathlib import Path
HERE = Path(__file__).resolve().parent
sys.path.insert(0, str(HERE.parents[1] / 'games'))
from profile_guess import heap_signals
class MemoryProfileTests(unittest.TestCase):
def test_dispatcher_logs(self):
result = subprocess.run([str(HERE / 'build/t_svc_memory'), '--heap-probe'],
capture_output=True, text=True, check=True)
signals = heap_signals(result.stderr, {
'mem_reserve': '0x140000000', 'mem_stack': '0x10000000',
'mem_alias': '0xb8000000', 'mem_used': '0x200000',
'app_pool': '0x1000000', 'sys_resource': '0x2000000',
})
self.assertEqual(signals['mem_alias'][0], '0x118000000')
self.assertEqual(signals['mem_used'][0], '0xe00000')
if __name__ == '__main__':
unittest.main()
docs/development.md
+35
View File
@@ -14,6 +14,41 @@ The README covers packing and playing. This page is for working on nxnative itse
| iOS app | One unsigned IPA for iPhone and iPad. The rewritten code is embedded in the executable; the shader libraries, the RomFS and the game icon are inside the app. A settings screen comes before the game (`Launcher.swift`, drawn from `dm/launcher.json`, which `dm/games/settings.py` writes from `dm/games/settings.tsv`); the host's loading starts at app launch meanwhile, and the frame rate and render scale go into the frame step state when the game starts (`loader.c` `dm_fps_state_apply`) |
| Known gaps | Conditional render is always drawn; `MultiDraw*IndirectCount` waits on the CPU for the GPU-written count; geometry shaders do not translate. TBDR optimizations are off by default (they don't understand bindless textures yet). No amiibo. The bundled prewarm lists are empty. On the Mac, Mario's menu and dialogue text render blank (fine on the iPhone) |
## Guest memory SVCs
`dm/host/hle_svc.c` serializes heap resizing, physical map/unmap, stack
map/unmap and memory queries with one memory-state lock. The alias bitmap is
the source of committed/free runs for `QueryMemory` and of committed-byte
accounting. With `mem_map_limit`, heap growth and physical maps check the same
application-pool budget; overlapping maps charge only newly committed pages.
Unexpected host protection failures abort before publishing state, including
the `SetHeapSize` fail-fast check from PR #127.
`MapMemory` borrows opened heap memory, committed alias memory or writable
static-module data until an exact unmap copies it back. Another borrow,
decommit, heap shrink or permission/attribute request cannot invalidate that
source. Queries report its original state with borrowed attribute bit 0 and
no user permissions. The implementation still copies bytes: source permissions
are logical, and native host pages cannot isolate adjacent 4 KiB slices.
Dynamic NRO backing is rejected because reload can restore it outside this
ownership lifetime. Host-created guest threads allocate their stack backing
through SDK malloc and free it after SDK thread destruction/copyback. Physical
recommit may overlap a borrowed source: SDK allocators extend committed windows
this way, and existing committed bytes are preserved.
A host-page exposure bitmap preserves lazy zero-fill on first commitment.
Later commits clear only newly committed logical slices, preserving live
neighbors on a 16 KiB host page. GPU dirty tracking is notified when a mapping
opens a host page and before stack copyback. The SDK retains address-region
reservation ownership and its existing build patches.
`make -C dm/host/tests GAME=<id> run` includes the production-dispatcher memory
tests, real GPU dirty-tracker integration and heap-probe log compatibility.
The focused targets are `heap_size`, `memory`, `memory-dirty`, `memory-profile`
and `memory-tsan`; `memory` includes undefined-behavior and address sanitizers.
Use `PYTHON=<venv>/bin/python` when the NSP tools' Python dependencies are in a
virtual environment; the heap-probe check imports the production profile reader.
## The packing script in detail
`scripts/build_ipa_from_nsp.py` (`-h` lists every option):