From 62a640b037da2eb80eeb46c50e6240b77700cd32 Mon Sep 17 00:00:00 2001 From: seedds Date: Mon, 26 Oct 2026 10:36:23 +0800 Subject: [PATCH] Fix custom key loading for compressed game inputs (#204) nsz 4.6.1 rejects --keys and loads keys.txt beside argv[0] during import. Run the converter in its temporary output folder with the selected key file linked there before import, using absolute input and output paths. Remove the temporary link on both successful and failed conversions. Integrate PR #204 with XCZ and relative-path regressions, rename the test flag that shadowed unittest.TestCase.fail, and qualify the documentation for the verified nsz release. Record the packing finding and provenance. Validation: reproduced the exact parser failure using real nsz 4.6.1; a synthetic NSZ conversion verifies selected-key lookup and link cleanup; 52 NSP tests, four documentation tests and git diff --check pass. The synthetic payload is unencrypted and uses dummy keys. Real-game decryption, actual XCZ conversion and a full IPA build remain unverified. Co-authored-by: Fernando Messias --- dm/games/0100000000010000/README.md | 4 ++- dm/tools/tests/test_nsp_nsz.py | 47 ++++++++++++++++++++++------- docs/agents/symptoms.md | 4 +++ docs/development.md | 2 +- docs/reference/README.md | 1 + docs/reference/packing.md | 10 ++++++ scripts/build_ipa_from_nsp.py | 6 ++-- 7 files changed, 59 insertions(+), 15 deletions(-) create mode 100644 docs/reference/packing.md diff --git a/dm/games/0100000000010000/README.md b/dm/games/0100000000010000/README.md index f34a07d..3141732 100644 --- a/dm/games/0100000000010000/README.md +++ b/dm/games/0100000000010000/README.md @@ -52,7 +52,9 @@ No `patches.py`. `op_mode` stays docked: Odyssey only takes its motion moves in ## Found on this game -General host changes first needed here; each applies to every game: +General host and tool changes first needed here; each applies to every game: + +- **Resolved: NSZ decompression rejected `--keys`** when packing v0: [NSZ decompression rejects `--keys`](../../../docs/reference/packing.md#nsz-decompression-rejects---keys). - **Largest point size at `c0 + 0x610`**: [04, section 3](../../../docs/agents/04-graphics.md#3-find-the-binding-layout-by-census) and [Graphics `c0 + 0x610`: the largest point size](../../../docs/reference/shaders.md#graphics-c0--0x610-the-largest-point-size). - **Invariant positions** (`-fpreserve-invariance` in `dmsh_build.py` and `ios_metal.py`): [Position invariance across shaders](../../../docs/reference/shaders.md#position-invariance-across-shaders). diff --git a/dm/tools/tests/test_nsp_nsz.py b/dm/tools/tests/test_nsp_nsz.py index 596b729..dc924b5 100644 --- a/dm/tools/tests/test_nsp_nsz.py +++ b/dm/tools/tests/test_nsp_nsz.py @@ -28,38 +28,63 @@ class Nsz(unittest.TestCase): self.tmp = Path(tempfile.mkdtemp()); self.addCleanup(shutil.rmtree, self.tmp) self.base = self.tmp / 'Game [v0].nsz'; pfs0(self.base, [f'{APP:016x}0000000000000011.tik', 'a.ncz']) self.upd = self.tmp / 'Game [v65536].nsz'; pfs0(self.upd, [f'{APP | 0x800:016x}0000000000000011.tik', 'b.ncz']) - self.runs, self.fail = [], False + self.keys = self.tmp / 'custom keys' / 'console.keys'; self.keys.parent.mkdir(); self.keys.write_text('test keys\n') + self.runs, self.fail_conversion = [], False for cm in (mock.patch.object(nsp, 'PROG', mock.Mock()), mock.patch.object(nsp, 'say', lambda m: None), mock.patch.object(nsp, 'run', self.fake_nsz), mock.patch.object(importlib.util, 'find_spec', lambda m, *a: object() if m == 'nsz' else None)): cm.__enter__(); self.addCleanup(cm.__exit__, None, None, None) - def fake_nsz(self, cmd, log, env=None, cwd=None, on_line=None): # nsz -D --keys K -o OUT SRC: writes OUT/.nsp, or a partial one and a non-zero exit + def fake_nsz(self, cmd, log, env=None, cwd=None, on_line=None): # nsz -D -o OUT SRC, with keys.txt available before import; or a partial output and a non-zero exit src, out = Path(cmd[-1]), Path(cmd[cmd.index('-o') + 1]) + self.assertTrue(src.is_absolute()) + self.assertTrue(out.is_absolute()) + self.assertNotIn('--keys', cmd) # nsz 4.6.1's argument parser rejects it + self.assertEqual(cmd[2], nsp.NSZ_MAIN) + self.assertEqual(Path(cwd), out) + staged = Path(cwd) / 'keys.txt' + self.assertTrue(staged.is_symlink()) + self.assertEqual(staged.resolve(), self.keys.resolve()) + self.assertEqual(staged.read_text(), 'test keys\n') self.runs.append(src.name) - (out / (src.stem + '.nsp')).write_bytes(b'partial' if self.fail else b'nsp of ' + src.name.encode()) - if self.fail: raise nsp.BuildError('nsz exit code 1') + (out / (src.stem + nsp.COMPRESSED[src.suffix])).write_bytes(b'partial' if self.fail_conversion else b'nsp of ' + src.name.encode()) + if self.fail_conversion: raise nsp.BuildError('nsz exit code 1') def test_scan_and_convert(self): app, fam = nsp.scan_nsps(self.tmp) self.assertEqual((app, fam['base'], fam['updates']), (APP, self.base, [(65536, self.upd)])) - fam = nsp.decompress_nsz(self.tmp, app, fam, self.tmp / 'prod.keys', {}) + fam = nsp.decompress_nsz(self.tmp, app, fam, self.keys, {}) out = self.tmp / f'{APP:016X}_build' / 'nsp' self.assertEqual((fam['base'], fam['updates']), (out / 'Game [v0].nsp', [(65536, out / 'Game [v65536].nsp')])) self.assertEqual(fam['base'].read_bytes(), b'nsp of Game [v0].nsz') self.assertFalse((out / 'partial').exists()) - nsp.decompress_nsz(self.tmp, app, nsp.scan_nsps(self.tmp)[1], self.tmp / 'prod.keys', {}) # unchanged sources: reused + nsp.decompress_nsz(self.tmp, app, nsp.scan_nsps(self.tmp)[1], self.keys, {}) # unchanged sources: reused self.assertEqual(len(self.runs), 2) os.utime(self.upd, ns=(0, 0)) # a changed source: converted again - nsp.decompress_nsz(self.tmp, app, nsp.scan_nsps(self.tmp)[1], self.tmp / 'prod.keys', {}) + nsp.decompress_nsz(self.tmp, app, nsp.scan_nsps(self.tmp)[1], self.keys, {}) self.assertEqual(self.runs[2:], [self.upd.name]) + def test_relative_paths(self): + previous_cwd = Path.cwd(); self.addCleanup(os.chdir, previous_cwd); os.chdir(self.tmp) + app, fam = nsp.scan_nsps(Path('.')) + fam = nsp.decompress_nsz(Path('.'), app, fam, self.keys.relative_to(self.tmp), {}) + self.assertEqual(fam['base'].resolve(), (self.tmp / f'{APP:016X}_build/nsp/Game [v0].nsp').resolve()) + self.assertFalse((fam['base'].parent / 'partial').exists()) + + def test_xcz_uses_the_selected_keys(self): + self.base.rename(self.base.with_suffix('.xcz')); self.upd.unlink() + app, fam = nsp.scan_nsps(self.tmp) + fam = nsp.decompress_nsz(self.tmp, app, fam, self.keys, {}) + self.assertEqual(fam['base'].suffix, '.xci') + self.assertEqual(fam['base'].read_bytes(), b'nsp of Game [v0].xcz') + self.assertFalse((fam['base'].parent / 'partial').exists()) + def test_failed_conversion_is_not_kept(self): - app, fam = nsp.scan_nsps(self.tmp); self.fail = True - with self.assertRaises(nsp.BuildError): nsp.decompress_nsz(self.tmp, app, fam, self.tmp / 'prod.keys', {}) + app, fam = nsp.scan_nsps(self.tmp); self.fail_conversion = True + with self.assertRaises(nsp.BuildError): nsp.decompress_nsz(self.tmp, app, fam, self.keys, {}) out = self.tmp / f'{APP:016X}_build' / 'nsp' self.assertEqual(sorted(p.name for p in out.iterdir()), []) # neither the partial file nor a key - self.fail = False - nsp.decompress_nsz(self.tmp, app, fam, self.tmp / 'prod.keys', {}) + self.fail_conversion = False + nsp.decompress_nsz(self.tmp, app, fam, self.keys, {}) self.assertEqual((out / 'Game [v0].nsp').read_bytes(), b'nsp of Game [v0].nsz') def test_an_nsp_beside_wins_and_nsps_need_no_nsz(self): diff --git a/docs/agents/symptoms.md b/docs/agents/symptoms.md index 0b0826d..7daff48 100644 --- a/docs/agents/symptoms.md +++ b/docs/agents/symptoms.md @@ -2,6 +2,10 @@ One line per symptom in the [subsystem references](../reference/): what you see (an assertion, a `[RED]` line, an abort or crash site, a visible result) and the entry that explains it. Search this page for the text in your log first (`grep -i` the function name, the `[RED]` text or the address form), then read the entry and the game notes it names. Generated by `dm/tools/docs_symptoms.py` from the entries' Symptom bullets: change the entry, then run it. +## Packing + +- `decompress NSZ` stops immediately with `nsz: error: unrecognized arguments: --keys …`. → [NSZ decompression rejects `--keys`](../reference/packing.md#nsz-decompression-rejects---keys) + ## Rewriter - an updated game loses a needed cut because its NSO build ID changed, although the relevant code still matches a verified version; the rewriter reports `[RED] function patch ... skipped` when that match cannot be established. → [Function-matched cuts for unregistered builds](../reference/rewriter.md#function-matched-cuts-for-unregistered-builds) diff --git a/docs/development.md b/docs/development.md index de6b2a5..c7c4367 100644 --- a/docs/development.md +++ b/docs/development.md @@ -141,7 +141,7 @@ When the output goes to a file or a pipe, the script prints a plain line instead `--progress json` is for a program that shows the build (a GUI): stdout carries one JSON object per line, each with `ev` and the time `t`. The events are `stage` (`k`, `n`, `name`, `step`), `step`, `progress` (`done`, `total`, `unit`, `est`, `note`, `frac`, `eta`, `elapsed`; `null` while unknown), `stage_end` (`ok`, `summary`, `elapsed`) and `say` (`msg`: every message, errors and a failed step's log tail included). A count change goes out at most every 0.5 s, any other change at once. This mode never asks for the NSP folder or the keys; argument errors and an unexpected error's traceback still go to stderr. `dm/tools/tests/test_nsp_progress.py` tests it without game files, keys or Xcode (`python3 -m unittest discover -s dm/tools/tests`). -Everything it writes stays in the NSP folder: intermediate files and logs in `_build/`, the IPA next to the NSPs. NSZ and XCZ files are converted with `nsz -D` (from the `nsz` package in the Python running the script) into `_build/nsp/`, through a temporary folder so that only a conversion nsz finished is used, and reused while the source keeps its name, size and modification time; `nsz -D` reports an NCA hash mismatch but goes on, so a damaged file is caught by nx_extract's own hash check (`--allow-damaged`). The extracted game files stay in `<id>_build/extract/` for the next build; `--no-keep-extracted` (`KEEP_EXTRACTED = False` in Spyder) deletes them at the end. It exits with 0 when done and 1 on an error. The app's bundle ID is `org.nxnative.<base game title ID>` (for example `org.nxnative.0100a6301214e000`), and its version is the game's display version: the update's when an update is packed, else the base game's (Engage 2.0.0); `--app-name` / `--bundle-id` (or `APP_NAME` / `BUNDLE_ID` in Spyder) override the name and ID. +Everything it writes stays in the NSP folder: intermediate files and logs in `<id>_build/`, the IPA next to the NSPs. NSZ and XCZ files are converted with `nsz -D` (from the `nsz` package in the Python running the script) into `<title ID>_build/nsp/`, through a temporary folder so that only a conversion nsz finished is used, and reused while the source keeps its name, size and modification time; `nsz -D` reports an NCA hash mismatch but goes on, so a damaged file is caught by nx_extract's own hash check (`--allow-damaged`). The keys selected by `-k` are linked temporarily as `keys.txt` for nsz's import-time lookup ([NSZ keys](reference/packing.md#nsz-decompression-rejects---keys)). The extracted game files stay in `<id>_build/extract/` for the next build; `--no-keep-extracted` (`KEEP_EXTRACTED = False` in Spyder) deletes them at the end. It exits with 0 when done and 1 on an error. The app's bundle ID is `org.nxnative.<base game title ID>` (for example `org.nxnative.0100a6301214e000`), and its version is the game's display version: the update's when an update is packed, else the base game's (Engage 2.0.0); `--app-name` / `--bundle-id` (or `APP_NAME` / `BUNDLE_ID` in Spyder) override the name and ID. The game language (`--language`, `LANGUAGE`; English by default) is what `nn::oe::GetDesiredLanguage` returns. As on the console, it must be one the game's control data lists (`SupportedLanguageFlag`, `nacp.py --languages`); the packed version's control data counts, since an update may add languages. The same language for another region stands in (`en-GB` for `en-US`, `es` for `es-419`, `fr` for `fr-CA`, `pt` for `pt-BR`, and back). A language the game lacks stops the build with the game's list; English, the default, falls back to the game's first language. On the device, environment switches go in `Documents/env.txt` (one `KEY=VALUE` per line), and game files are also accepted at `Documents/romfs`. diff --git a/docs/reference/README.md b/docs/reference/README.md index c1065f8..e6ad375 100644 --- a/docs/reference/README.md +++ b/docs/reference/README.md @@ -4,6 +4,7 @@ How the host meets each part of a Switch game, and every failure met so far, one | File | Covers | |---|---| +| [packing.md](packing.md) | NSP script, compressed inputs and build artifacts | | [rewriter.md](rewriter.md) | The NSO rewriter `patch_nso.py`: jump tables, exclusive windows, patch area reach, cuts and observation gates, functions without unwind entries, stack and system registers, relocations | | [memory.md](memory.md) | Guest layout, heap and application pool, virtual address memory and the alias region (the SDK's node area), engine allocators, mapping SVCs, running out of memory | | [threads.md](threads.md) | Thread creation and cores, address arbitration, yields and spin waits, events | diff --git a/docs/reference/packing.md b/docs/reference/packing.md new file mode 100644 index 0000000..c455192 --- /dev/null +++ b/docs/reference/packing.md @@ -0,0 +1,10 @@ +# Packing + +## Compressed inputs + +### NSZ decompression rejects `--keys` + +- **Symptom:** `decompress NSZ` stops immediately with `nsz: error: unrecognized arguments: --keys …`. +- **Cause:** `nsz` 4.6.1 has no `--keys` argument. It loads `keys.txt` beside `sys.argv[0]`, or falls back to `~/.switch/prod.keys`, while importing its modules, before parsing arguments ([4.6.1 key loader](https://github.com/nicoboss/nsz/blob/4.6.1/nsz/nut/Keys.py)). Newer upstream code supports `--keys`; staging `keys.txt` also supports 4.6.1. +- **Fix:** `scripts/build_ipa_from_nsp.py` `decompress_nsz` links the file selected by `-k` as `keys.txt` in the conversion's temporary folder and runs `nsz -D` there, with `sys.argv[0]` set to `nsz` before import. Input and output paths are absolute. The temporary folder and link are removed on success or failure; the user's default keys are untouched. +- **Found on:** [Super Mario Odyssey](../../dm/games/0100000000010000/README.md), converting its v0 NSZ. Applies to NSZ and XCZ inputs for every game. diff --git a/scripts/build_ipa_from_nsp.py b/scripts/build_ipa_from_nsp.py index d549fe0..b024610 100755 --- a/scripts/build_ipa_from_nsp.py +++ b/scripts/build_ipa_from_nsp.py @@ -435,7 +435,7 @@ def decompress_nsz(nsp_dir, app, fam, keys, env): if not todo: return fam if importlib.util.find_spec('nsz') is None: raise BuildError(f'NSZ / XCZ files need the nsz tool in the current Python ({sys.executable}): pip install nsz') - out = nsp_dir / f'{app:016X}_build' / 'nsp'; logs, tmp = out.parent / 'logs', out / 'partial' + out = nsp_dir.absolute() / f'{app:016X}_build' / 'nsp'; logs, tmp = out.parent / 'logs', out / 'partial' PROG.plan(1); PROG.stage('decompress NSZ', done=0, total=len(todo)) conv, made = {}, 0 for i, p in enumerate(todo): @@ -446,7 +446,9 @@ def decompress_nsz(nsp_dir, app, fam, keys, env): rmtree(tmp); tmp.mkdir(parents=True); rmtree(dst); keyfile.unlink(missing_ok=True) log = logs / f'nsz_{p.stem}.log' try: - run([sys.executable, '-c', NSZ_MAIN, '-D', '--keys', keys, '-o', tmp, p], log, env=env) + # nsz 4.6.1 loads keys.txt beside argv[0] at import time; with argv[0] = 'nsz' that is cwd. That release has no --keys option. + (tmp / 'keys.txt').symlink_to(keys.resolve()) + run([sys.executable, '-c', NSZ_MAIN, '-D', '-o', tmp, p.resolve()], log, env=env, cwd=tmp) if not (tmp / dst.name).is_file(): raise BuildError(f'nsz made no {dst.name} (full output: {log})') (tmp / dst.name).rename(dst) finally: rmtree(tmp)