diff --git a/dm/games/0100000000010000/README.md b/dm/games/0100000000010000/README.md index f34a07d..3141732 100644 --- a/dm/games/0100000000010000/README.md +++ b/dm/games/0100000000010000/README.md @@ -52,7 +52,9 @@ No `patches.py`. `op_mode` stays docked: Odyssey only takes its motion moves in ## Found on this game -General host changes first needed here; each applies to every game: +General host and tool changes first needed here; each applies to every game: + +- **Resolved: NSZ decompression rejected `--keys`** when packing v0: [NSZ decompression rejects `--keys`](../../../docs/reference/packing.md#nsz-decompression-rejects---keys). - **Largest point size at `c0 + 0x610`**: [04, section 3](../../../docs/agents/04-graphics.md#3-find-the-binding-layout-by-census) and [Graphics `c0 + 0x610`: the largest point size](../../../docs/reference/shaders.md#graphics-c0--0x610-the-largest-point-size). - **Invariant positions** (`-fpreserve-invariance` in `dmsh_build.py` and `ios_metal.py`): [Position invariance across shaders](../../../docs/reference/shaders.md#position-invariance-across-shaders). diff --git a/dm/tools/tests/test_nsp_nsz.py b/dm/tools/tests/test_nsp_nsz.py index 596b729..dc924b5 100644 --- a/dm/tools/tests/test_nsp_nsz.py +++ b/dm/tools/tests/test_nsp_nsz.py @@ -28,38 +28,63 @@ class Nsz(unittest.TestCase): self.tmp = Path(tempfile.mkdtemp()); self.addCleanup(shutil.rmtree, self.tmp) self.base = self.tmp / 'Game [v0].nsz'; pfs0(self.base, [f'{APP:016x}0000000000000011.tik', 'a.ncz']) self.upd = self.tmp / 'Game [v65536].nsz'; pfs0(self.upd, [f'{APP | 0x800:016x}0000000000000011.tik', 'b.ncz']) - self.runs, self.fail = [], False + self.keys = self.tmp / 'custom keys' / 'console.keys'; self.keys.parent.mkdir(); self.keys.write_text('test keys\n') + self.runs, self.fail_conversion = [], False for cm in (mock.patch.object(nsp, 'PROG', mock.Mock()), mock.patch.object(nsp, 'say', lambda m: None), mock.patch.object(nsp, 'run', self.fake_nsz), mock.patch.object(importlib.util, 'find_spec', lambda m, *a: object() if m == 'nsz' else None)): cm.__enter__(); self.addCleanup(cm.__exit__, None, None, None) - def fake_nsz(self, cmd, log, env=None, cwd=None, on_line=None): # nsz -D --keys K -o OUT SRC: writes OUT/.nsp, or a partial one and a non-zero exit + def fake_nsz(self, cmd, log, env=None, cwd=None, on_line=None): # nsz -D -o OUT SRC, with keys.txt available before import; or a partial output and a non-zero exit src, out = Path(cmd[-1]), Path(cmd[cmd.index('-o') + 1]) + self.assertTrue(src.is_absolute()) + self.assertTrue(out.is_absolute()) + self.assertNotIn('--keys', cmd) # nsz 4.6.1's argument parser rejects it + self.assertEqual(cmd[2], nsp.NSZ_MAIN) + self.assertEqual(Path(cwd), out) + staged = Path(cwd) / 'keys.txt' + self.assertTrue(staged.is_symlink()) + self.assertEqual(staged.resolve(), self.keys.resolve()) + self.assertEqual(staged.read_text(), 'test keys\n') self.runs.append(src.name) - (out / (src.stem + '.nsp')).write_bytes(b'partial' if self.fail else b'nsp of ' + src.name.encode()) - if self.fail: raise nsp.BuildError('nsz exit code 1') + (out / (src.stem + nsp.COMPRESSED[src.suffix])).write_bytes(b'partial' if self.fail_conversion else b'nsp of ' + src.name.encode()) + if self.fail_conversion: raise nsp.BuildError('nsz exit code 1') def test_scan_and_convert(self): app, fam = nsp.scan_nsps(self.tmp) self.assertEqual((app, fam['base'], fam['updates']), (APP, self.base, [(65536, self.upd)])) - fam = nsp.decompress_nsz(self.tmp, app, fam, self.tmp / 'prod.keys', {}) + fam = nsp.decompress_nsz(self.tmp, app, fam, self.keys, {}) out = self.tmp / f'{APP:016X}_build' / 'nsp' self.assertEqual((fam['base'], fam['updates']), (out / 'Game [v0].nsp', [(65536, out / 'Game [v65536].nsp')])) self.assertEqual(fam['base'].read_bytes(), b'nsp of Game [v0].nsz') self.assertFalse((out / 'partial').exists()) - nsp.decompress_nsz(self.tmp, app, nsp.scan_nsps(self.tmp)[1], self.tmp / 'prod.keys', {}) # unchanged sources: reused + nsp.decompress_nsz(self.tmp, app, nsp.scan_nsps(self.tmp)[1], self.keys, {}) # unchanged sources: reused self.assertEqual(len(self.runs), 2) os.utime(self.upd, ns=(0, 0)) # a changed source: converted again - nsp.decompress_nsz(self.tmp, app, nsp.scan_nsps(self.tmp)[1], self.tmp / 'prod.keys', {}) + nsp.decompress_nsz(self.tmp, app, nsp.scan_nsps(self.tmp)[1], self.keys, {}) self.assertEqual(self.runs[2:], [self.upd.name]) + def test_relative_paths(self): + previous_cwd = Path.cwd(); self.addCleanup(os.chdir, previous_cwd); os.chdir(self.tmp) + app, fam = nsp.scan_nsps(Path('.')) + fam = nsp.decompress_nsz(Path('.'), app, fam, self.keys.relative_to(self.tmp), {}) + self.assertEqual(fam['base'].resolve(), (self.tmp / f'{APP:016X}_build/nsp/Game [v0].nsp').resolve()) + self.assertFalse((fam['base'].parent / 'partial').exists()) + + def test_xcz_uses_the_selected_keys(self): + self.base.rename(self.base.with_suffix('.xcz')); self.upd.unlink() + app, fam = nsp.scan_nsps(self.tmp) + fam = nsp.decompress_nsz(self.tmp, app, fam, self.keys, {}) + self.assertEqual(fam['base'].suffix, '.xci') + self.assertEqual(fam['base'].read_bytes(), b'nsp of Game [v0].xcz') + self.assertFalse((fam['base'].parent / 'partial').exists()) + def test_failed_conversion_is_not_kept(self): - app, fam = nsp.scan_nsps(self.tmp); self.fail = True - with self.assertRaises(nsp.BuildError): nsp.decompress_nsz(self.tmp, app, fam, self.tmp / 'prod.keys', {}) + app, fam = nsp.scan_nsps(self.tmp); self.fail_conversion = True + with self.assertRaises(nsp.BuildError): nsp.decompress_nsz(self.tmp, app, fam, self.keys, {}) out = self.tmp / f'{APP:016X}_build' / 'nsp' self.assertEqual(sorted(p.name for p in out.iterdir()), []) # neither the partial file nor a key - self.fail = False - nsp.decompress_nsz(self.tmp, app, fam, self.tmp / 'prod.keys', {}) + self.fail_conversion = False + nsp.decompress_nsz(self.tmp, app, fam, self.keys, {}) self.assertEqual((out / 'Game [v0].nsp').read_bytes(), b'nsp of Game [v0].nsz') def test_an_nsp_beside_wins_and_nsps_need_no_nsz(self): diff --git a/docs/agents/symptoms.md b/docs/agents/symptoms.md index 0b0826d..7daff48 100644 --- a/docs/agents/symptoms.md +++ b/docs/agents/symptoms.md @@ -2,6 +2,10 @@ One line per symptom in the [subsystem references](../reference/): what you see (an assertion, a `[RED]` line, an abort or crash site, a visible result) and the entry that explains it. Search this page for the text in your log first (`grep -i` the function name, the `[RED]` text or the address form), then read the entry and the game notes it names. Generated by `dm/tools/docs_symptoms.py` from the entries' Symptom bullets: change the entry, then run it. +## Packing + +- `decompress NSZ` stops immediately with `nsz: error: unrecognized arguments: --keys …`. → [NSZ decompression rejects `--keys`](../reference/packing.md#nsz-decompression-rejects---keys) + ## Rewriter - an updated game loses a needed cut because its NSO build ID changed, although the relevant code still matches a verified version; the rewriter reports `[RED] function patch ... skipped` when that match cannot be established. → [Function-matched cuts for unregistered builds](../reference/rewriter.md#function-matched-cuts-for-unregistered-builds) diff --git a/docs/development.md b/docs/development.md index de6b2a5..c7c4367 100644 --- a/docs/development.md +++ b/docs/development.md @@ -141,7 +141,7 @@ When the output goes to a file or a pipe, the script prints a plain line instead `--progress json` is for a program that shows the build (a GUI): stdout carries one JSON object per line, each with `ev` and the time `t`. The events are `stage` (`k`, `n`, `name`, `step`), `step`, `progress` (`done`, `total`, `unit`, `est`, `note`, `frac`, `eta`, `elapsed`; `null` while unknown), `stage_end` (`ok`, `summary`, `elapsed`) and `say` (`msg`: every message, errors and a failed step's log tail included). A count change goes out at most every 0.5 s, any other change at once. This mode never asks for the NSP folder or the keys; argument errors and an unexpected error's traceback still go to stderr. `dm/tools/tests/test_nsp_progress.py` tests it without game files, keys or Xcode (`python3 -m unittest discover -s dm/tools/tests`). -Everything it writes stays in the NSP folder: intermediate files and logs in `_build/`, the IPA next to the NSPs. NSZ and XCZ files are converted with `nsz -D` (from the `nsz` package in the Python running the script) into `_build/nsp/`, through a temporary folder so that only a conversion nsz finished is used, and reused while the source keeps its name, size and modification time; `nsz -D` reports an NCA hash mismatch but goes on, so a damaged file is caught by nx_extract's own hash check (`--allow-damaged`). The extracted game files stay in `<id>_build/extract/` for the next build; `--no-keep-extracted` (`KEEP_EXTRACTED = False` in Spyder) deletes them at the end. It exits with 0 when done and 1 on an error. The app's bundle ID is `org.nxnative.<base game title ID>` (for example `org.nxnative.0100a6301214e000`), and its version is the game's display version: the update's when an update is packed, else the base game's (Engage 2.0.0); `--app-name` / `--bundle-id` (or `APP_NAME` / `BUNDLE_ID` in Spyder) override the name and ID. +Everything it writes stays in the NSP folder: intermediate files and logs in `<id>_build/`, the IPA next to the NSPs. NSZ and XCZ files are converted with `nsz -D` (from the `nsz` package in the Python running the script) into `<title ID>_build/nsp/`, through a temporary folder so that only a conversion nsz finished is used, and reused while the source keeps its name, size and modification time; `nsz -D` reports an NCA hash mismatch but goes on, so a damaged file is caught by nx_extract's own hash check (`--allow-damaged`). The keys selected by `-k` are linked temporarily as `keys.txt` for nsz's import-time lookup ([NSZ keys](reference/packing.md#nsz-decompression-rejects---keys)). The extracted game files stay in `<id>_build/extract/` for the next build; `--no-keep-extracted` (`KEEP_EXTRACTED = False` in Spyder) deletes them at the end. It exits with 0 when done and 1 on an error. The app's bundle ID is `org.nxnative.<base game title ID>` (for example `org.nxnative.0100a6301214e000`), and its version is the game's display version: the update's when an update is packed, else the base game's (Engage 2.0.0); `--app-name` / `--bundle-id` (or `APP_NAME` / `BUNDLE_ID` in Spyder) override the name and ID. The game language (`--language`, `LANGUAGE`; English by default) is what `nn::oe::GetDesiredLanguage` returns. As on the console, it must be one the game's control data lists (`SupportedLanguageFlag`, `nacp.py --languages`); the packed version's control data counts, since an update may add languages. The same language for another region stands in (`en-GB` for `en-US`, `es` for `es-419`, `fr` for `fr-CA`, `pt` for `pt-BR`, and back). A language the game lacks stops the build with the game's list; English, the default, falls back to the game's first language. On the device, environment switches go in `Documents/env.txt` (one `KEY=VALUE` per line), and game files are also accepted at `Documents/romfs`. diff --git a/docs/reference/README.md b/docs/reference/README.md index c1065f8..e6ad375 100644 --- a/docs/reference/README.md +++ b/docs/reference/README.md @@ -4,6 +4,7 @@ How the host meets each part of a Switch game, and every failure met so far, one | File | Covers | |---|---| +| [packing.md](packing.md) | NSP script, compressed inputs and build artifacts | | [rewriter.md](rewriter.md) | The NSO rewriter `patch_nso.py`: jump tables, exclusive windows, patch area reach, cuts and observation gates, functions without unwind entries, stack and system registers, relocations | | [memory.md](memory.md) | Guest layout, heap and application pool, virtual address memory and the alias region (the SDK's node area), engine allocators, mapping SVCs, running out of memory | | [threads.md](threads.md) | Thread creation and cores, address arbitration, yields and spin waits, events | diff --git a/docs/reference/packing.md b/docs/reference/packing.md new file mode 100644 index 0000000..c455192 --- /dev/null +++ b/docs/reference/packing.md @@ -0,0 +1,10 @@ +# Packing + +## Compressed inputs + +### NSZ decompression rejects `--keys` + +- **Symptom:** `decompress NSZ` stops immediately with `nsz: error: unrecognized arguments: --keys …`. +- **Cause:** `nsz` 4.6.1 has no `--keys` argument. It loads `keys.txt` beside `sys.argv[0]`, or falls back to `~/.switch/prod.keys`, while importing its modules, before parsing arguments ([4.6.1 key loader](https://github.com/nicoboss/nsz/blob/4.6.1/nsz/nut/Keys.py)). Newer upstream code supports `--keys`; staging `keys.txt` also supports 4.6.1. +- **Fix:** `scripts/build_ipa_from_nsp.py` `decompress_nsz` links the file selected by `-k` as `keys.txt` in the conversion's temporary folder and runs `nsz -D` there, with `sys.argv[0]` set to `nsz` before import. Input and output paths are absolute. The temporary folder and link are removed on success or failure; the user's default keys are untouched. +- **Found on:** [Super Mario Odyssey](../../dm/games/0100000000010000/README.md), converting its v0 NSZ. Applies to NSZ and XCZ inputs for every game. diff --git a/scripts/build_ipa_from_nsp.py b/scripts/build_ipa_from_nsp.py index d549fe0..b024610 100755 --- a/scripts/build_ipa_from_nsp.py +++ b/scripts/build_ipa_from_nsp.py @@ -435,7 +435,7 @@ def decompress_nsz(nsp_dir, app, fam, keys, env): if not todo: return fam if importlib.util.find_spec('nsz') is None: raise BuildError(f'NSZ / XCZ files need the nsz tool in the current Python ({sys.executable}): pip install nsz') - out = nsp_dir / f'{app:016X}_build' / 'nsp'; logs, tmp = out.parent / 'logs', out / 'partial' + out = nsp_dir.absolute() / f'{app:016X}_build' / 'nsp'; logs, tmp = out.parent / 'logs', out / 'partial' PROG.plan(1); PROG.stage('decompress NSZ', done=0, total=len(todo)) conv, made = {}, 0 for i, p in enumerate(todo): @@ -446,7 +446,9 @@ def decompress_nsz(nsp_dir, app, fam, keys, env): rmtree(tmp); tmp.mkdir(parents=True); rmtree(dst); keyfile.unlink(missing_ok=True) log = logs / f'nsz_{p.stem}.log' try: - run([sys.executable, '-c', NSZ_MAIN, '-D', '--keys', keys, '-o', tmp, p], log, env=env) + # nsz 4.6.1 loads keys.txt beside argv[0] at import time; with argv[0] = 'nsz' that is cwd. That release has no --keys option. + (tmp / 'keys.txt').symlink_to(keys.resolve()) + run([sys.executable, '-c', NSZ_MAIN, '-D', '-o', tmp, p.resolve()], log, env=env, cwd=tmp) if not (tmp / dst.name).is_file(): raise BuildError(f'nsz made no {dst.name} (full output: {log})') (tmp / dst.name).rename(dst) finally: rmtree(tmp)