Make repeated IPA and Mac app builds incremental (#279)

Reuse unchanged IPA entries through the packing index and APFS clones,
and keep Mac app and derived-profile shader outputs while their inputs
match. Preserve main's shader regression checks and .NET discovery.

Count uncompressed bytes in streamed files so a size change aborts the
pack without replacing the previous IPA or index. Cover both deflated
and stored streams with growth and shrinkage regression tests, and
record the cause and fix in the packing reference.

Co-authored-by: Suraj Mandal <surajmandalcell@gmail.com>

README.md

@@ -183,7 +183,7 @@ Any version works, with or without an update: the script packs the newest versio
 python3 scripts/build_ipa_from_nsp.py ~/Games/MarioWonder -k ~/keys/prod.keys
 ```
 
-The IPA appears in the same folder (here `~/Games/MarioWonder/SMBWMobile.ipa`). The first build takes 30–60 minutes, later ones about 10. You need free disk space of about twice the NSPs' size plus 10 GB. Options such as `-o` (output path) and `--app-name` are listed by `-h`.
+The IPA appears in the same folder (here `~/Games/MarioWonder/SMBWMobile.ipa`). The first build takes 30–60 minutes. A later one reuses every step whose inputs did not change, so after a pull that changes only the host it takes about a minute. The small `.idx` file beside the IPA (`.app.key` beside a Mac app) records what was packed; keep it for the next build. You need free disk space of about twice the NSPs' size plus 10 GB. Options such as `-o` (output path) and `--app-name` are listed by `-h`.
 
 **4. Install** the IPA with your sideloading tool (AltStore, Sideloadly, LiveContainer, …). The app gets the game's name, icon and version (the update's version, for example 1.2.1 for Mario Wonder; the base game's when there is no update). Sign it with the **Increased Memory Limit** capability (`com.apple.developer.kernel.increased-memory-limit`) if your tool can: without it iOS gives the app about 3.4 GB and closes the bigger games (Hyrule Warriors: Age of Calamity and Hades II use more, Breath of the Wild and Cuphead come close); with it a recent iPhone gives about 6 GB.
 

dm/ios/tests/test_pack_ipa.py

@@ -0,0 +1,311 @@
+"""Repeat IPA packs match fresh packs, including clone and compressed-copy paths."""
+import contextlib
+import importlib.util
+import io
+import json
+import os
+from pathlib import Path
+import stat
+import sys
+import tempfile
+import time
+import unittest
+from unittest.mock import patch
+import zipfile
+
+REPO = Path(__file__).resolve().parents[3]
+spec = importlib.util.spec_from_file_location('pack_ipa', REPO / 'dm/ios/tools/pack_ipa.py')
+pack = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(pack)
+spec = importlib.util.spec_from_file_location('build_ipa_from_nsp', REPO / 'scripts/build_ipa_from_nsp.py')
+nsp = importlib.util.module_from_spec(spec); spec.loader.exec_module(nsp)
+
+
+class PackIpa(unittest.TestCase):
+    def setUp(self):
+        self.tmpdir = tempfile.TemporaryDirectory(); self.addCleanup(self.tmpdir.cleanup)
+        self.tmp = Path(self.tmpdir.name)
+        self.app, self.rom = self.tmp / 'app', self.tmp / 'romfs'
+        (self.app / 'sub').mkdir(parents=True); (self.rom / 'déjà').mkdir(parents=True)
+        (self.app / 'exe').write_bytes(b'code' * 3000); os.chmod(self.app / 'exe', 0o755)
+        (self.app / 'sub/info.plist').write_bytes(b'x' * 5000)
+        os.symlink('info.plist', self.app / 'sub/link')
+        (self.rom / 'random.bin').write_bytes(os.urandom(30000))
+        (self.rom / 'text.txt').write_bytes(b'hello world ' * 4000)
+        (self.rom / 'déjà/empty').write_bytes(b'')
+        (self.rom / 'gone.dat').write_bytes(b'abc' * 2000)
+        self.maps = [f'Payload/DmIOS.app/romfs={self.rom}', f'Payload/DmIOS.app={self.app}']
+        self.out = self.tmp / 'a.ipa'
+
+    def run_pack(self, out=None, *opts, big=pack.BIG):
+        out = out or self.out
+        self.kept = self.reused = 0
+        replace, clone, kept, reuse = os.replace, pack.clone, pack.Zip.kept, pack.Zip.reuse
+        def cloned(old, dst):
+            self.assertEqual(str(dst), str(out) + '.clone')
+            self.assertFalse(Path(str(out) + '.partial').exists())
+            result = clone(old, dst)
+            self.assertFalse(Path(str(out) + '.partial').exists())
+            return result
+        def publish(src, dst):
+            if str(src).endswith('.clone'):
+                self.assertEqual(str(dst), str(out) + '.partial')
+                self.assertFalse(Path(dst).exists())
+                self.kept = Path(src).stat().st_size
+                self.assertLess(self.kept, out.stat().st_size)   # the old central directory was cut before .partial appears
+            return replace(src, dst)
+        def kept_entry(z, e, *args):
+            if e.st and stat.S_ISREG(e.st.st_mode): self.reused += 1
+            return kept(z, e, *args)
+        def copy_entry(z, *args):
+            self.reused += 1
+            return reuse(z, *args)
+        with patch.object(sys, 'argv', ['pack_ipa', str(out), *opts, *self.maps]), patch.object(pack, 'BIG', big), patch.object(pack, 'CHUNK', 4096), \
+                patch.object(pack.os, 'replace', publish), patch.object(pack, 'clone', cloned), patch.object(pack.Zip, 'kept', kept_entry), patch.object(pack.Zip, 'reuse', copy_entry), contextlib.redirect_stdout(io.StringIO()) as output:
+            pack.main()
+        self.output = output.getvalue()
+        for suffix in ('.partial', '.clone', '.idx.partial'): self.assertFalse(Path(str(out) + suffix).exists())
+        idx = json.loads(Path(str(out) + '.idx').read_text())
+        self.assertEqual(set(idx), {'version', 'level', 'zlib', 'ipa', 'files'})
+        self.assertTrue(all(len(row) == 5 for row in idx['files'].values()))
+        self.assertEqual(idx['ipa'], [out.stat().st_size, out.stat().st_mtime_ns])
+        return idx
+
+    def same_as_fresh(self, *opts, big=pack.BIG):
+        fresh = self.tmp / 'fresh.ipa'; self.run_pack(fresh, '--no-reuse', *opts, big=big)
+        with zipfile.ZipFile(self.out) as a, zipfile.ZipFile(fresh) as b:
+            self.assertIsNone(a.testzip()); self.assertEqual(a.namelist(), b.namelist())
+            for ia, ib in zip(a.infolist(), b.infolist()):
+                for key in ('CRC', 'file_size', 'compress_size', 'external_attr', 'date_time', 'compress_type', 'flag_bits', 'header_offset'):
+                    self.assertEqual(getattr(ia, key), getattr(ib, key), (ia.filename, key))
+                self.assertEqual(a.read(ia), b.read(ib), ia.filename)
+        self.assertEqual(self.out.read_bytes(), fresh.read_bytes())
+
+    def offset(self, name):
+        with zipfile.ZipFile(self.out) as z: return z.getinfo('Payload/DmIOS.app/' + name).header_offset
+
+    def test_unchanged_clone(self):
+        self.run_pack()
+        with zipfile.ZipFile(self.out) as z: end = z.start_dir
+        with patch.object(pack, 'load', side_effect=AssertionError('unchanged file compressed')):
+            self.run_pack()
+        self.assertEqual(self.reused, 6); self.assertEqual(self.kept, end)
+        self.same_as_fresh()
+
+    def test_app_change_keeps_romfs_and_ignores_app_folder_time(self):
+        self.run_pack()
+        (self.app / 'replacement').write_bytes(b'new!' * 3000)
+        os.replace(self.app / 'replacement', self.app / 'exe'); os.chmod(self.app / 'exe', 0o755)
+        os.utime(self.app, (time.time() - 7200,) * 2)
+        self.run_pack()
+        self.assertEqual(self.reused, 5); self.assertEqual(self.kept, self.offset('exe'))
+        self.assertGreater(self.kept, self.offset('romfs/text.txt')); self.same_as_fresh()
+
+    def test_change_in_first_mapping(self):
+        self.run_pack(); (self.rom / 'déjà/empty').write_bytes(b'changed')
+        self.run_pack(); self.assertEqual(self.reused, 5)
+        self.assertEqual(self.kept, self.offset('romfs/déjà/empty')); self.same_as_fresh()
+
+    def test_changed_added_removed_and_same_size_rewrite(self):
+        self.run_pack()
+        (self.rom / 'text.txt').write_bytes(b'changed' * 4000); (self.rom / 'gone.dat').unlink(); (self.rom / 'new.bin').write_bytes(b'n' * 9000)
+        source = self.app / 'sub/info.plist'; before = source.stat()
+        source.write_bytes(b'y' * before.st_size); os.utime(source, ns=(before.st_atime_ns, before.st_mtime_ns))
+        self.assertNotEqual(source.stat().st_ctime_ns, before.st_ctime_ns)
+        self.run_pack(); self.assertEqual(self.reused, 3); self.same_as_fresh()
+
+    def test_caller_order_and_parents_once_before_children(self):
+        msl, aoc, pso = [self.tmp / n for n in ('msl', 'aoc', 'pso')]
+        for folder in (msl / 'lib', aoc, pso): folder.mkdir(parents=True)
+        for file in (msl / 'lib/a.metallib', aoc / 'content', pso / 'archive'): file.write_bytes(b'fixture')
+        self.maps = [self.maps[0], f'Payload/DmIOS.app/aoc={aoc}', f'Payload/DmIOS.app/pso={pso}', f'Payload/DmIOS.app/msl/lib={msl / "lib"}', self.maps[1]]
+        self.run_pack()
+        with zipfile.ZipFile(self.out) as z:
+            names = z.namelist(); self.assertEqual(len(names), len(set(names)))
+            for i, name in enumerate(names):
+                parts = name.rstrip('/').split('/')
+                for end in range(1, len(parts)): self.assertIn('/'.join(parts[:end]) + '/', names[:i])
+            for name in ('Payload/', 'Payload/DmIOS.app/', 'Payload/DmIOS.app/msl/'):
+                self.assertEqual(z.getinfo(name).date_time, (1980, 1, 1, 0, 0, 0))
+                self.assertEqual(z.getinfo(name).external_attr >> 16, pack.DIR_MODE)
+            positions = [names.index('Payload/DmIOS.app/' + n) for n in ('romfs/text.txt', 'aoc/content', 'pso/archive', 'msl/lib/a.metallib', 'exe')]
+            self.assertEqual(positions, sorted(positions))
+        self.same_as_fresh()
+        self.maps.reverse(); self.run_pack()
+        self.assertLess(self.offset('exe'), self.offset('romfs/text.txt')); self.same_as_fresh()
+        self.maps.append(self.maps[0])
+        with self.assertRaisesRegex(SystemExit, 'twice'): self.run_pack()
+
+    def test_nested_mapping_before_parent(self):
+        extra = self.tmp / 'extra'; extra.mkdir(); (extra / 'child').write_bytes(b'child')
+        self.maps.insert(0, f'Payload/DmIOS.app/romfs/deep={extra}')
+        self.run_pack(); self.run_pack(); self.assertEqual(self.reused, 7); self.same_as_fresh()
+
+    def test_clone_failure_copies_compressed(self):
+        self.run_pack()
+        with patch.object(pack, 'clone', return_value=False), patch.object(pack, 'load', side_effect=AssertionError('recompressed')):
+            self.run_pack()
+        self.assertEqual(self.kept, 0); self.assertEqual(self.reused, 6); self.same_as_fresh()
+
+    def test_symlink_and_folder_headers(self):
+        self.run_pack(); (self.app / 'sub/link').unlink(); os.symlink('../exe', self.app / 'sub/link')
+        self.run_pack(); self.assertEqual(self.kept, self.offset('sub/link')); self.same_as_fresh()
+        os.utime(self.app / 'sub', (time.time() - 7200,) * 2)
+        self.run_pack(); self.assertEqual(self.kept, self.offset('sub/')); self.same_as_fresh()
+        os.chmod(self.app / 'sub', 0o700)
+        self.run_pack(); self.assertGreater(self.kept, self.offset('sub/link')); self.same_as_fresh()
+
+    def test_store_flag_change(self):
+        self.run_pack(); self.run_pack(None, '--store', '.TXT'); self.assertEqual(self.reused, 5)
+        self.same_as_fresh('--store', '.txt')
+
+    def test_stale_replaced_and_malformed_index(self):
+        self.run_pack(); index = Path(str(self.out) + '.idx')
+        original = index.read_text()
+        other = self.tmp / 'other.ipa'; self.run_pack(other)
+        os.replace(other, self.out); index.write_text(original)
+        self.run_pack(); self.assertEqual(self.reused, 0); self.assertEqual(self.kept, 0)
+        for bad in ('{not json', '[]', '{}'):
+            index.write_text(bad); self.run_pack(); self.assertEqual(self.reused, 0); self.same_as_fresh()
+        index.unlink(); self.run_pack(); self.assertEqual(self.reused, 0)
+        idx = json.loads(index.read_text()); idx['files']['Payload/DmIOS.app/exe'] = ['bad']
+        index.write_text(json.dumps(idx)); self.run_pack(); self.assertEqual(self.reused, 5); self.same_as_fresh()
+
+    def test_zlib_version_mismatch_forces_full_pack(self):
+        idx = self.run_pack(); idx['zlib'] = 'different'
+        Path(str(self.out) + '.idx').write_text(json.dumps(idx))
+        self.run_pack(); self.assertEqual(self.kept, 0); self.assertEqual(self.reused, 0); self.same_as_fresh()
+
+    def test_streamed_reuse_and_fallback_to_stored(self):
+        self.run_pack(big=1024); self.run_pack(big=1024); self.assertEqual(self.reused, 6)
+        (self.rom / 'random.bin').write_bytes(os.urandom(30000))
+        self.run_pack(big=1024); self.assertEqual(self.reused, 5); self.same_as_fresh(big=1024)
+        self.run_pack(); self.assertEqual(self.reused, 6); self.same_as_fresh()
+
+    def test_stream_drains_pool_before_later_jobs(self):
+        load, stream, events, submitted = pack.load, pack.Zip.stream, [], []
+        class Pool(pack.cf.ThreadPoolExecutor):
+            def submit(self, fn, path, deflate):
+                submitted.append(Path(path).name)
+                return super().submit(fn, path, deflate)
+        def loaded(path, deflate):
+            result = load(path, deflate); events.append(Path(path).name); return result
+        def streamed(z, e, deflate):
+            self.assertIn('random.bin', events); self.assertNotIn('exe', submitted)
+            return stream(z, e, deflate)
+        with patch.object(pack, 'load', loaded), patch.object(pack.Zip, 'stream', streamed), patch.object(pack.cf, 'ThreadPoolExecutor', Pool): self.run_pack(big=40000)
+        self.same_as_fresh(big=40000)
+
+    def test_local_header_mismatch_recompresses(self):
+        self.run_pack(); offset = self.offset('exe')
+        with open(self.out, 'r+b') as f: f.seek(offset + 10); f.write(b'\x00\x00')
+        idxp = Path(str(self.out) + '.idx'); idx = json.loads(idxp.read_text()); idx['ipa'] = [self.out.stat().st_size, self.out.stat().st_mtime_ns]; idxp.write_text(json.dumps(idx))
+        self.run_pack(); self.assertEqual(self.reused, 5); self.same_as_fresh()
+
+    def test_zip64_local_headers(self):
+        header = pack.header
+        def zip64(e, method, crc, csize, usize, z64=False): return header(e, method, crc, csize, usize, True)
+        with patch.object(pack, 'header', zip64):
+            self.run_pack(); self.run_pack(); self.assertEqual(self.reused, 6)
+            self.assertGreater(self.kept, self.offset('sub/link')); self.same_as_fresh()
+        self.run_pack(); self.assertEqual(self.reused, 0); self.assertEqual(self.kept, 0); self.same_as_fresh()
+
+    def test_clone_uses_checked_inode_after_path_replaced(self):
+        self.run_pack(); clone = pack.clone; old_bytes = self.out.read_bytes()
+        def checked_clone(old, dst):
+            self.assertEqual(os.pread(old.fileno(), len(old_bytes), 0), old_bytes)
+            replacement = self.tmp / 'replacement'; replacement.write_bytes(b'wrong inode')
+            with open(replacement, 'r+b') as f: f.truncate(len(old_bytes))
+            os.replace(replacement, self.out)
+            return clone(old, dst)
+        with patch.object(pack, 'clone', checked_clone): self.run_pack()
+        self.assertGreater(self.kept, 0); self.same_as_fresh()
+
+    def test_stat_before_publish_and_stale_partial_cleanup(self):
+        self.run_pack()
+        target = self.tmp / 'protected'; target.write_bytes(b'unchanged')
+        Path(str(self.out) + '.idx.partial').symlink_to(target)
+        replace, expected = os.replace, []
+        def publish(src, dst):
+            if str(src).endswith('.partial') and str(dst) == str(self.out):
+                st = os.stat(src); expected.append([st.st_size, st.st_mtime_ns])
+                replace(src, dst); os.utime(dst, ns=(st.st_atime_ns, st.st_mtime_ns + 1_000_000))
+            else: replace(src, dst)
+        with patch.object(pack.os, 'replace', publish), patch.object(sys, 'argv', ['pack_ipa', str(self.out), *self.maps]), contextlib.redirect_stdout(io.StringIO()): pack.main()
+        self.assertFalse(Path(str(self.out) + '.idx.partial').exists())
+        self.assertEqual(json.loads(Path(str(self.out) + '.idx').read_text())['ipa'], expected[0])
+        self.assertEqual(target.read_bytes(), b'unchanged')
+
+    def test_failure_preserves_old_ipa_and_index(self):
+        self.run_pack(); before, idx = self.out.read_bytes(), Path(str(self.out) + '.idx').read_bytes()
+        (self.app / 'exe').write_bytes(b'changed')
+        with patch.object(pack, 'load', side_effect=OSError('read failed')):
+            with self.assertRaisesRegex(OSError, 'read failed'): self.run_pack()
+        self.assertEqual(self.out.read_bytes(), before); self.assertEqual(Path(str(self.out) + '.idx').read_bytes(), idx)
+        self.assertFalse(Path(str(self.out) + '.partial').exists()); self.assertFalse(Path(str(self.out) + '.clone').exists())
+
+    def test_pool_size_change_preserves_old_ipa_and_index(self):
+        self.run_pack(); before, index = self.out.read_bytes(), Path(str(self.out) + '.idx').read_bytes()
+        source, load = self.app / 'exe', pack.load
+        for store in (False, True):
+            for grow in (False, True):
+                with self.subTest(store=store, grow=grow):
+                    source.write_bytes(b'code' * 3000)
+                    def changed(path, deflate):
+                        if path == str(source):
+                            data = source.read_bytes(); source.write_bytes(data + b'grow' if grow else data[:-4])
+                        return load(path, deflate)
+                    with patch.object(pack, 'load', changed), self.assertRaises(SystemExit) as error:
+                        self.run_pack(None, *(['--store', 'exe'] if store else []))
+                    self.assertEqual(str(error.exception), f'[RED] pack_ipa: {source} changed size while it was packed')
+                    self.assertEqual(self.out.read_bytes(), before); self.assertEqual(Path(str(self.out) + '.idx').read_bytes(), index)
+                    for suffix in ('.partial', '.clone'): self.assertFalse(Path(str(self.out) + suffix).exists())
+
+    def test_stream_size_change_preserves_old_ipa_and_index(self):
+        self.run_pack(); before, index = self.out.read_bytes(), Path(str(self.out) + '.idx').read_bytes()
+        source, stream = self.app / 'exe', pack.Zip.stream
+        for store in (False, True):
+            for grow in (False, True):
+                with self.subTest(store=store, grow=grow):
+                    source.write_bytes(b'code' * 3000)
+                    def changed(z, e, deflate):
+                        if e.path == str(source):
+                            data = source.read_bytes(); source.write_bytes(data + b'grow' if grow else data[:-4])
+                        return stream(z, e, deflate)
+                    with patch.object(pack.Zip, 'stream', changed), self.assertRaisesRegex(SystemExit, 'changed size while it was packed'):
+                        self.run_pack(None, *(['--store', 'exe'] if store else []), big=1024)
+                    self.assertEqual(self.out.read_bytes(), before); self.assertEqual(Path(str(self.out) + '.idx').read_bytes(), index)
+                    for suffix in ('.partial', '.clone'): self.assertFalse(Path(str(self.out) + suffix).exists())
+
+    def test_non_apfs_mapping_never_reuses_files(self):
+        self.assertEqual(pack.fs_type(self.tmp / 'missing'), b'')
+        self.run_pack()
+        for unsafe, filesystem, expected in (({str(self.rom)}, b'exfat', 2), ({str(self.app)}, b'exfat', 4), ({str(self.app), str(self.rom)}, b'', 0)):
+            with self.subTest(unsafe=unsafe, filesystem=filesystem):
+                with patch.object(pack, 'fs_type', side_effect=lambda root: filesystem if root in unsafe else b'apfs') as fs, \
+                        patch.object(pack, 'load', wraps=pack.load) as loaded:
+                    self.run_pack()
+                self.assertEqual(self.reused, expected)
+                self.assertEqual({call.args[0] for call in fs.call_args_list}, {str(self.app), str(self.rom)})
+                self.assertEqual(fs.call_count, 2)
+                self.assertEqual(len(loaded.call_args_list), 6 - expected)
+                self.assertTrue(all(any(Path(call.args[0]).is_relative_to(root) for root in unsafe) for call in loaded.call_args_list))
+                for arc, root in (mapping.split('=', 1) for mapping in self.maps):
+                    message = f'reuse off for {arc}: not APFS'
+                    self.assertEqual(self.output.count(message), int(root in unsafe))
+                    if root in unsafe: self.assertIn(message, self.output.splitlines()[-1])
+                if expected == 4: self.assertGreater(self.kept, self.offset('romfs/text.txt'))
+                self.same_as_fresh()
+
+    def test_progress_excludes_kept_bytes(self):
+        with patch.object(nsp, 'PROG'):
+            self.out.write_bytes(b'0' * 1000)
+            watch = nsp.IpaWatch(REPO, self.tmp, self.out)
+            watch.line('pack IPA'); watch.line('pack: 8 entries, 2000 bytes')
+            watch.part.write_bytes(b'0' * 1000); self.assertIsNone(watch.poll())
+            watch.line('pack: keeping 800 bytes')
+            watch.part.write_bytes(b'0' * 850)
+            self.assertEqual(watch.poll(), {'done': 50, 'total': 200, 'est': True})
+
+
+if __name__ == '__main__': unittest.main()

dm/ios/tests/test_xcode_workflow.py

@@ -97,7 +97,7 @@ class XcodePipelineTests(unittest.TestCase):
                 self.assertEqual(mocks['split_shaders'].call_args.args[8], derived)
             else:
                 builder = mocks['build_vk_shaders' if graphics == 'vulkan' else 'build_gl_shaders']
-                self.assertEqual(builder.call_args.kwargs['reuse'], not derived)
+                self.assertTrue(builder.call_args.kwargs['reuse'])
             warnings = [event.get('msg', '') for event in events]
             self.assertEqual(any('this Xcode project uses' in message for message in warnings), derived)
             if derived:

dm/ios/tools/build_ipa.sh

@@ -148,11 +148,12 @@ APP=$DD/Build/Products/Release-iphoneos/DmIOS.app
 # Straight from the sources: the app without its signature, and without the folders an app built before pack_ipa.py still has (DerivedData keeps resources that left the project;
 # msl/ is also deleted there, it took 1.5 GB of the home disk), then the shader libraries, the add-on content (the RomFS extracted from the player's DLC NSPs, one folder per index:
 # the app sets DM_AOC_DIR at launch, the host reports them as owned and mounts them from it) and the RomFS
-rm -rf "$APP/msl" "$O/Payload" "$O/out.ipa"   # (the last two: what the copy-and-zip packing left behind)
-P=Payload/DmIOS.app; MAPS=("$P=$APP" "$P/msl/$MSLDIR=$MSL_IOS/$MSLDIR")
+rm -rf "$APP/msl" "$O/Payload"   # Payload/: what the copy-and-zip packing left behind, as was out.ipa (and its index) unless it is the IPA packed here, which a repeat pack reuses
+[ "$O/out.ipa" -ef "$OUT_IPA" ] || rm -f "$O/out.ipa"; [ "$O/out.ipa.idx" -ef "$OUT_IPA.idx" ] || rm -f "$O/out.ipa.idx"
+P=Payload/DmIOS.app; MAPS=("$P/romfs=$ROMFS")
 if [ -n "$AOC" ] && [ -d "$AOC" ]; then MAPS+=("$P/aoc=$AOC"); t "add-on content: $(ls "$AOC" | tr '\n' ' ')"; fi
 if [ -n "${PSO:-}" ]; then MAPS+=("$P/pso=$PSO"); fi
-MAPS+=("$P/romfs=$ROMFS")
+MAPS+=("$P/msl/$MSLDIR=$MSL_IOS/$MSLDIR" "$P=$APP")
 t "pack IPA (already-compressed game files stored, everything else deflated)"
 # The pulling vertex libraries (<key>_bp, dm/gfx/tools/dmsh_pull.py) serve the hybrid pipelines. Default to the same pso_mtl4 setting compiled by gen_build.py; IOS_PULL=0 / 1 overrides it.
 # The pipeline archive's bases are made of them, so an IPA with an archive (pso/) always carries them, whatever IOS_PULL says (an Xcode build copies them always).

dm/ios/tools/pack_ipa.py

@@ -5,27 +5,35 @@
 # Speed (Info-ZIP managed about 100 MB/s on the T7): files up to BIG are read, and deflated, by a thread pool (zlib releases the GIL) while this thread writes them in order;
 # bigger files are streamed here, their CRC patched into the local header afterwards (no data descriptors). Zip64 fields wherever a size, an offset or the entry count passes its 32 / 16-bit field.
 # Usage: pack_ipa.py <out.ipa> [--store .a:.b] [--skip <archive path>]... <archive path>=<source folder>...
+#   Mappings are packed in command-line order; put the files that change least first, and the app last
 #   --skip: an archive path left out of the walk of the folder that holds it (folders a stale app build still has, the signature); a mapping of its own still adds it
 #   --skip-suffix: files whose names end so are left out of every walk (build_ipa.sh: the pulling vertex libraries _bp.metallib when the profile's pso_mtl4 is off, unless IOS_PULL overrides it)
 #   Writes <out.ipa>.partial and renames it at the end; prints "pack: N entries, X bytes" first (the NSP script's progress total, X = the files' bytes) and a summary line last
-import argparse, collections, concurrent.futures as cf, os, stat, struct, sys, time, zlib
+#   Repeat packs reuse a file when its size, mtime_ns, ctime_ns and inode are unchanged (true on APFS). A tool that rewrites in place and restores mtime defeats this if ctime also stays
+#   unchanged; --no-reuse forces a full pack. The index holds only source metadata and compression settings; ZIP metadata comes from the old IPA.
+#   Only mappings whose source folder is on APFS reuse files; other mappings pack their files again. The kept run needs an APFS clone of the old IPA; if cloning fails, eligible files are copied compressed.
+#   Reused bytes are not checked against their CRC: damage inside the old IPA carries over. Use --no-reuse or delete <out.ipa>.idx to rebuild
+import argparse, collections, concurrent.futures as cf, ctypes, json, os, stat, struct, sys, time, zipfile, zlib
 
+LEVEL = 6
+IDX_VERSION = 3
 BIG = 256 << 20      # files above this are streamed by the writer; smaller ones go through the pool
 WINDOW = 2 << 30     # input bytes in the pool at once
 QUEUE = 4096         # entries in the pool at once (folders of tiny files)
 CHUNK = 16 << 20
 L32, L16 = 0xFFFFFFFF, 0xFFFF
-DIR_MODE = stat.S_IFDIR | 0o755   # a parent folder no source folder stands for (Payload/, msl/)
-Entry = collections.namedtuple('Entry', 'name path st')   # st None: such a parent folder
+DIR_MODE = stat.S_IFDIR | 0o755   # a parent folder no source folder stands for (Payload/, msl/); its time is 1980-01-01, so it is the same in every pack
+Entry = collections.namedtuple('Entry', 'name path st root')   # st None: a synthetic parent; root: the mapping's source folder
 
 
 def entries(maps, skip, skip_suffix=()):
     """The archive's entries in order: for each mapping its parent folders (unless an earlier entry gave them), its root, then its tree depth first, sorted by name,
-    without the skipped paths and without other mappings' roots (each adds its own)"""
-    roots, out, seen = {a for a, _ in maps}, [], set()
+    without the skipped paths and without other mappings' roots (each adds its own). Parents needed before their mapping are synthetic folders with fixed metadata."""
+    roots, out, seen, parents = {a for a, _ in maps}, [], set(), set()
     def add(name, path, st):
+        if name in parents and st is not None and stat.S_ISDIR(st.st_mode): return   # written already, as a parent folder
         if name in seen: sys.exit(f'[RED] pack_ipa: {name} twice in the archive')
-        seen.add(name); out.append(Entry(name, path, st))
+        seen.add(name); out.append(Entry(name, path, st, src))
     def walk(d, arc):
         with os.scandir(d) as it: es = sorted(it, key=lambda e: e.name)
         for e in es:
@@ -38,19 +46,20 @@ def entries(maps, skip, skip_suffix=()):
     for arc, src in maps:
         parts = arc.split('/')
         for i in range(1, len(parts)):
-            p = '/'.join(parts[:i]) + '/'
-            if p not in seen: add(p, None, None)
+            p = '/'.join(parts[:i])
+            if p + '/' not in seen:
+                add(p + '/', None, None); parents.add(p + '/')
         add(arc + '/', src, os.stat(src)); walk(src, arc)
     return out
 
 
-def load(path, deflate):   # in the pool: (method, CRC, the bytes to write)
+def load(path, deflate):   # in the pool: (method, CRC, the bytes to write, uncompressed length)
     with open(path, 'rb') as f: data = f.read()
-    crc = zlib.crc32(data)
+    crc, usize = zlib.crc32(data), len(data)
     if deflate and data:
-        c = zlib.compressobj(6, zlib.DEFLATED, -15); z = c.compress(data) + c.flush()
-        if len(z) < len(data): return 8, crc, z
-    return 0, crc, data
+        c = zlib.compressobj(LEVEL, zlib.DEFLATED, -15); z = c.compress(data) + c.flush()
+        if len(z) < len(data): return 8, crc, z, usize
+    return 0, crc, data, usize
 
 
 def dos_time(t):
@@ -59,20 +68,46 @@ def dos_time(t):
     return (h << 11) | (mi << 5) | (s // 2), ((y - 1980) << 9) | (mo << 5) | d
 
 
+def header(e, method, crc, csize, usize, z64=False):   # a local header: (its bytes, name bytes, flags, time, date)
+    nb = e.name.encode('utf-8', 'surrogateescape'); fl = 0 if e.name.isascii() else 0x800
+    t, d = dos_time(e.st.st_mtime if e.st else 0)
+    ext = struct.pack('<HHQQ', 1, 16, usize, csize) if z64 else b''
+    return (struct.pack('<IHHHHHIIIHH', 0x04034B50, 45 if z64 else 20 if method == 8 else 10, fl, method, t, d, crc, L32 if z64 else csize, L32 if z64 else usize, len(nb), len(ext))
+            + nb + ext, nb, fl, t, d)
+
+
+def clone(old, dst):   # clone the checked inode, even if its path was replaced; AT_FDCWD is -2 on Darwin
+    try: return ctypes.CDLL(None, use_errno=True).fclonefileat(old.fileno(), -2, os.fsencode(dst), 0) == 0
+    except (AttributeError, OSError): return False
+
+
+class Statfs64(ctypes.Structure):   # Darwin's statfs64 layout from sys/mount.h, including the fields after f_fstypename
+    _fields_ = [('f_bsize', ctypes.c_uint32), ('f_iosize', ctypes.c_int32), ('counts', ctypes.c_uint64 * 5), ('f_fsid', ctypes.c_int32 * 2),
+                ('f_owner', ctypes.c_uint32), ('f_type', ctypes.c_uint32), ('f_flags', ctypes.c_uint32), ('f_fssubtype', ctypes.c_uint32),
+                ('f_fstypename', ctypes.c_char * 16), ('f_mntonname', ctypes.c_char * 1024), ('f_mntfromname', ctypes.c_char * 1024), ('reserved', ctypes.c_uint32 * 8)]
+
+
+def fs_type(root):
+    if sys.platform != 'darwin': return b''
+    fs = Statfs64()
+    try:
+        if ctypes.CDLL(None).statfs64(os.fsencode(root), ctypes.byref(fs)) == 0: return fs.f_fstypename
+    except (AttributeError, OSError): pass
+    return b''   # unknown identity: do not reuse
+
+
 class Zip:
-    def __init__(self, path):
-        self.f, self.pos, self.cd, self.now = open(path, 'wb', buffering=1 << 20), 0, [], time.time()
+    def __init__(self, path, keep=0):   # keep: the bytes of a clone at path that stay (the leading entries kept)
+        self.f = open(path, 'r+b' if keep else 'wb', buffering=1 << 20)
+        if keep: self.f.seek(keep)
+        self.pos, self.cd, self.written = keep, [], 0   # written: the bytes this pack writes (entries, header patches, the central directory; main adds the index)
         self.stats = collections.Counter()
 
     def w(self, b):
-        self.f.write(b); self.pos += len(b)
+        self.f.write(b); self.pos += len(b); self.written += len(b)
 
-    def local(self, e, method, crc, csize, usize, z64=False):   # the local header; returns (its offset, name bytes, flags, time, date)
-        nb = e.name.encode('utf-8', 'surrogateescape'); fl = 0 if e.name.isascii() else 0x800
-        t, d = dos_time(e.st.st_mtime if e.st else self.now)
-        ext = struct.pack('<HHQQ', 1, 16, usize, csize) if z64 else b''
-        off = self.pos
-        self.w(struct.pack('<IHHHHHIIIHH', 0x04034B50, 45 if z64 else 20 if method == 8 else 10, fl, method, t, d, crc, L32 if z64 else csize, L32 if z64 else usize, len(nb), len(ext)) + nb + ext)
+    def local(self, e, method, crc, csize, usize, z64=False):   # writes the local header; returns (its offset, name bytes, flags, time, date)
+        b, nb, fl, t, d = header(e, method, crc, csize, usize, z64); off = self.pos; self.w(b)
         return off, nb, fl, t, d
 
     def central(self, e, h, method, crc, csize, usize):
@@ -82,53 +117,119 @@ class Zip:
         mode = e.st.st_mode if e.st else DIR_MODE
         self.cd.append(struct.pack('<IHHHHHHIIIHHHHHII', 0x02014B50, 0x031E, 45 if z else 20 if method == 8 else 10, fl, method, t, d, crc, min(csize, L32), min(usize, L32),
                                    len(nb), len(ext), 0, 0, 0, (mode << 16) | (0x10 if stat.S_ISDIR(mode) else 0), min(off, L32)) + nb + ext)
+        self.stats['deflated' if method == 8 else 'stored'] += usize
+
+    def kept(self, e, info, h):
+        self.central(e, (info.header_offset,) + h[1:], info.compress_type, info.CRC, info.compress_size, info.file_size)
+        if e.st and stat.S_ISREG(e.st.st_mode): self.stats['reused'] += 1
 
     def put(self, e, method, crc, data, usize):   # an entry whose bytes are at hand
         h = self.local(e, method, crc, len(data), usize)
         if data: self.w(data)
         self.central(e, h, method, crc, len(data), usize)
-        self.stats['deflated' if method == 8 else 'stored'] += usize; self.stats['out'] += len(data)
 
     def other(self, e):   # a folder or a symlink
         if e.st and stat.S_ISLNK(e.st.st_mode):
             t = os.readlink(e.path).encode('utf-8', 'surrogateescape'); self.put(e, 0, zlib.crc32(t), t, len(t))
         else: self.put(e, 0, 0, b'', 0)
 
+    def patch(self, at, b):
+        self.f.seek(at); self.f.write(b); self.written += len(b)
+
     def stream(self, e, deflate):   # a big file: the local header first, its CRC (and a deflated size) patched in afterwards
         usize = e.st.st_size; z64 = usize >= L32   # a deflate that is not smaller is redone stored, so the deflated size stays below usize
         h = self.local(e, 8 if deflate else 0, 0, 0 if deflate else usize, usize, z64)
-        crc, csize, c = 0, 0, zlib.compressobj(6, zlib.DEFLATED, -15) if deflate else None
+        crc, csize, read, c = 0, 0, 0, zlib.compressobj(LEVEL, zlib.DEFLATED, -15) if deflate else None
         with open(e.path, 'rb') as f:
             while b := f.read(CHUNK):
-                crc = zlib.crc32(b, crc)
+                read += len(b); crc = zlib.crc32(b, crc)
                 if c: b = c.compress(b)
                 self.w(b); csize += len(b)
             if c: b = c.flush(); self.w(b); csize += len(b)
-        if csize != usize and not c: sys.exit(f'[RED] pack_ipa: {e.path} changed size while it was packed')
+        if read != usize: sys.exit(f'[RED] pack_ipa: {e.path} changed size while it was packed')
         if c and csize >= usize:   # not smaller: stored instead
             self.f.seek(h[0]); self.f.truncate(); self.pos = h[0]; return self.stream(e, False)
-        self.f.seek(h[0] + 14); self.f.write(struct.pack('<I', crc) + (b'' if z64 else struct.pack('<II', csize, usize)))
-        if z64: self.f.seek(h[0] + 30 + len(h[1]) + 4); self.f.write(struct.pack('<QQ', usize, csize))
+        self.patch(h[0] + 14, struct.pack('<I', crc) + (b'' if z64 else struct.pack('<II', csize, usize)))
+        if z64: self.patch(h[0] + 30 + len(h[1]) + 4, struct.pack('<QQ', usize, csize))
         self.f.seek(self.pos)
         self.central(e, h, 8 if c else 0, crc, csize, usize)
-        self.stats['deflated' if c else 'stored'] += usize; self.stats['out'] += csize
+
+    def reuse(self, e, info, h, old):   # copy already compressed bytes, after the checked local header
+        method, crc, csize, usize = info.compress_type, info.CRC, info.compress_size, info.file_size
+        nl, el = struct.unpack_from('<HH', h[0], 26)
+        left, at = csize, info.header_offset + 30 + nl + el
+        h = self.local(e, method, crc, csize, usize, usize >= L32 or csize >= L32)
+        while left:
+            b = os.pread(old.fileno(), min(CHUNK, left), at)
+            if not b: sys.exit(f'[RED] pack_ipa: the old IPA ended while {e.name} was copied from it')
+            self.w(b); left -= len(b); at += len(b)
+        self.central(e, h, method, crc, csize, usize)
+        self.stats['reused'] += 1
 
     def close(self):   # the central directory and the end records (zip64 ones when the count, the size or the offset does not fit)
-        cd_off, n = self.pos, len(self.cd); blob = b''.join(self.cd); self.w(blob)
-        if n >= L16 or cd_off >= L32 or len(blob) >= L32:
+        self.cd_off, n = self.pos, len(self.cd); blob = b''.join(self.cd); self.w(blob)
+        if n >= L16 or self.cd_off >= L32 or len(blob) >= L32:
             z = self.pos
-            self.w(struct.pack('<IQHHIIQQQQ', 0x06064B50, 44, 0x031E, 45, 0, 0, n, n, len(blob), cd_off))
+            self.w(struct.pack('<IQHHIIQQQQ', 0x06064B50, 44, 0x031E, 45, 0, 0, n, n, len(blob), self.cd_off))
             self.w(struct.pack('<IIQI', 0x07064B50, 0, z, 1))
-        self.w(struct.pack('<IHHHHIIH', 0x06054B50, 0, 0, min(n, L16), min(n, L16), min(len(blob), L32), min(cd_off, L32), 0))
+        self.w(struct.pack('<IHHHHIIH', 0x06054B50, 0, 0, min(n, L16), min(n, L16), min(len(blob), L32), min(self.cd_off, L32), 0))
         self.f.close()
 
 
+def load_index(out):   # (the old IPA opened, its index) when <out>.idx describes the IPA there now and this deflate level, else None
+    old = None
+    try:
+        with open(out + '.idx') as f: idx = json.load(f)
+        old = open(out, 'rb'); st = os.fstat(old.fileno())
+        if (isinstance(idx, dict) and idx.get('version') == IDX_VERSION and idx.get('level') == LEVEL and idx.get('zlib') == zlib.ZLIB_RUNTIME_VERSION
+                and idx.get('ipa') == [st.st_size, st.st_mtime_ns] and isinstance(idx.get('files'), dict)):
+            with zipfile.ZipFile(old) as archive: infos = {i.filename: i for i in archive.infolist()}
+            return old, idx, infos
+    except (OSError, ValueError, zipfile.BadZipFile): pass
+    if old: old.close()
+    return None
+
+
+def fingerprint(e, deflate):
+    st = e.st
+    return [st.st_size, st.st_mtime_ns, st.st_ctime_ns, st.st_ino, deflate]
+
+
+def match(e, row, info, old, idx, apfs):   # one header comparison for both the kept run and later compressed copies
+    if info is None: return None
+    mode, data = e.st.st_mode if e.st else DIR_MODE, b''
+    if not stat.S_ISDIR(mode) and not apfs[e.root]: return None
+    if stat.S_ISREG(mode):
+        if idx['files'].get(e.name) != row or info.file_size != e.st.st_size or info.compress_type not in (0, 8): return None
+        method, crc, csize, usize = info.compress_type, info.CRC, info.compress_size, info.file_size
+    else:
+        if stat.S_ISLNK(mode): data = os.readlink(e.path).encode('utf-8', 'surrogateescape')
+        method, crc, csize, usize = 0, zlib.crc32(data), len(data), len(data)
+        if (info.compress_type, info.CRC, info.compress_size, info.file_size) != (method, crc, csize, usize): return None
+    h = header(e, method, crc, csize, usize, usize >= L32 or csize >= L32)
+    if info.header_offset < 0 or info.header_offset + len(h[0]) + csize > idx['ipa'][0]: return None
+    if os.pread(old.fileno(), len(h[0]) + len(data), info.header_offset) != h[0] + data: return None
+    return h
+
+
+def prefix(es, files, infos, old, idx, apfs):
+    keep, pos = [], 0
+    for e in es:
+        info = infos.get(e.name)
+        if info is None or info.header_offset != pos: break
+        h = match(e, files.get(e.name), info, old, idx, apfs)
+        if h is None: break
+        keep.append((e, info, h)); pos += len(h[0]) + info.compress_size
+    return keep, pos
+
+
 def main():
     ap = argparse.ArgumentParser(description='Pack an IPA (zip) from source folders mapped to archive paths, in parallel')
     ap.add_argument('out'); ap.add_argument('maps', nargs='+', metavar='archive_path=folder')
     ap.add_argument('--store', default='', help='suffixes stored without compressing, separated by ":"')
     ap.add_argument('--skip', action='append', default=[], help='an archive path left out of the folder walk that would add it')
     ap.add_argument('--skip-suffix', action='append', default=[], help='files whose names end so are left out of every folder walk')
+    ap.add_argument('--no-reuse', action='store_true', help='pack every file again, without copying from the old IPA')
     a = ap.parse_args()
     maps = []
     for m in a.maps:
@@ -137,34 +238,68 @@ def main():
         maps.append((arc.strip('/'), src))
     store = tuple(s.lower() for s in a.store.split(':') if s)
     t0 = time.time()
+    if len({arc for arc, _ in maps}) != len(maps): sys.exit('[RED] pack_ipa: mapping twice in the archive')
     es = entries(maps, {s.strip('/') for s in a.skip}, tuple(a.skip_suffix))
     total = sum(e.st.st_size for e in es if e.st and stat.S_ISREG(e.st.st_mode))
     print(f'pack: {len(es)} entries, {total} bytes', flush=True)
-    part = a.out + '.partial'
-    z = Zip(part)
+    apfs = {root: fs_type(root) == b'apfs' for root in {src for _, src in maps}}
+    disabled = '; '.join(f'reuse off for {arc}: not APFS' for arc, src in maps if not apfs[src])
+    flags = [not e.name.lower().endswith(store) if store else True for e in es]   # each file's deflate flag
+    files = {e.name: fingerprint(e, f) for e, f in zip(es, flags) if e.st and stat.S_ISREG(e.st.st_mode)}
+    part, tmp, ipart = a.out + '.partial', a.out + '.clone', a.out + '.idx.partial'
+    for path in (part, tmp, ipart):
+        if os.path.lexists(path): os.unlink(path)
+    old, idx, infos = (None, None, {}) if a.no_reuse else load_index(a.out) or (None, None, {})
+    keep, kept = prefix(es, files, infos, old, idx, apfs) if old else ([], 0)
+    z = None
     try:
+        if kept:
+            if clone(old, tmp):
+                with open(tmp, 'r+b') as f: f.truncate(kept)
+                os.replace(tmp, part)
+            else: keep, kept = [], 0   # no clone: everything is written, files still copied compressed
+        if os.path.lexists(tmp): os.unlink(tmp)
+        print(f'pack: keeping {kept} bytes', flush=True)
+        z = Zip(part, kept)
+        for k in keep: z.kept(*k)
         pend, inflight = collections.deque(), 0
         def one():   # write the oldest pending entry
-            e, fut = pend.popleft()
-            if fut is None: z.other(e); return 0
-            method, crc, data = fut.result(); z.put(e, method, crc, data, e.st.st_size); return e.st.st_size
+            kind, e, value = pend.popleft()
+            if kind == 'other': z.other(e); return 0
+            if kind == 'reuse': z.reuse(e, *value, old); return 0
+            method, crc, data, usize = value.result()
+            if usize != e.st.st_size: sys.exit(f'[RED] pack_ipa: {e.path} changed size while it was packed')
+            z.put(e, method, crc, data, usize); return usize
         with cf.ThreadPoolExecutor(os.cpu_count()) as pool:
-            for e in es:
-                if not (e.st and stat.S_ISREG(e.st.st_mode)): pend.append((e, None)); continue   # folders and symlinks keep their place in the queue
-                deflate = not e.name.lower().endswith(store) if store else True
+            for e, deflate in zip(es[len(keep):], flags[len(keep):]):
+                if not (e.st and stat.S_ISREG(e.st.st_mode)): pend.append(('other', e, None)); continue
+                info = infos.get(e.name)
+                h = match(e, files[e.name], info, old, idx, apfs) if old else None
+                if h is not None: pend.append(('reuse', e, (info, h))); continue
                 if e.st.st_size > BIG:
                     while pend: inflight -= one()
                     z.stream(e, deflate); continue
                 while pend and (len(pend) >= QUEUE or inflight + e.st.st_size > WINDOW): inflight -= one()
-                pend.append((e, pool.submit(load, e.path, deflate))); inflight += e.st.st_size
+                pend.append(('load', e, pool.submit(load, e.path, deflate))); inflight += e.st.st_size
             while pend: inflight -= one()
         z.close()
+        st = os.stat(part)   # bind the index to this file before publishing; rename preserves its mtime
+        os.replace(part, a.out)
     except BaseException:
-        z.f.close(); os.unlink(part); raise
-    os.replace(part, a.out)
+        if z: z.f.close()
+        for path in (part, tmp):
+            if os.path.lexists(path): os.unlink(path)
+        if old: old.close()
+        raise
+    if old: old.close()
+    try:   # the index only after the rename: an IPA it does not describe (size, time) makes it ignored
+        blob = json.dumps({'version': IDX_VERSION, 'level': LEVEL, 'zlib': zlib.ZLIB_RUNTIME_VERSION, 'ipa': [st.st_size, st.st_mtime_ns], 'files': files})
+        with open(ipart, 'w') as f: f.write(blob)
+        os.replace(ipart, a.out + '.idx'); z.written += len(blob)
+    except OSError as x: print(f'[RED] pack_ipa: index not written, the next pack packs every file again: {x}', flush=True)
     s, dt = z.stats, time.time() - t0
-    print(f'packed: {len(es)} entries, stored {s["stored"] / 2**30:.2f} GB, deflated {s["deflated"] / 2**30:.2f} GB, archive {os.path.getsize(a.out) / 2**30:.2f} GB '
-          f'in {dt:.0f} s ({total / max(dt, 1e-3) / 2**20:.0f} MB/s)', flush=True)
+    print(f'packed: {len(es)} entries, stored {s["stored"] / 2**30:.2f} GB, deflated {s["deflated"] / 2**30:.2f} GB, archive {st.st_size / 2**30:.2f} GB, '
+          f'kept {kept / 2**30:.2f} GB, reused {s["reused"]} files in {dt:.0f} s, wrote {z.written / 2**30:.2f} GB' + (f'; {disabled}' if disabled else ''), flush=True)
 
 
 if __name__ == '__main__': main()

dm/mac/pack_app.py

@@ -8,8 +8,8 @@
 # Signing: the inner host first, then the bundle (codesign --force --sign <identity>; '-' = ad-hoc), without the hardened runtime: the host maps the game's code executable, which the hardened
 # runtime refuses without entitlements, and so the app cannot be notarized.
 # Usage: pack_app.py <out.app> --host <dmhost> --patched <dir> --romfs <dir> --msl <dir> --tables <profile dir> --font <dir> --title-id <id> --name <name> [options]
-#   Writes <out.app>.partial and swaps it in at the end (the old app stays as it was on failure); prints one line per step ("step: ...")
-import argparse, ctypes, os, plistlib, re, shutil, subprocess, sys, tempfile
+#   Reuses <out.app> while <out.app>.key matches; otherwise writes <out.app>.partial and swaps it in at the end. Prints "step: ..." and "app: <path>[, reused (inputs unchanged)]".
+import argparse, ctypes, hashlib, json, os, plistlib, re, shutil, stat, subprocess, sys, tempfile
 from pathlib import Path
 
 HERE = Path(__file__).resolve().parent
@@ -75,12 +75,12 @@ def min_os(host):   # the host's deployment target (LC_BUILD_VERSION minos): the
     return m.group(1) if m else '14.0'
 
 
-def build_launcher(out, minos):
-    run(['xcrun', 'clang', '-arch', 'arm64', f'-mmacosx-version-min={minos}', '-O2', '-Wall', '-o', out, HERE / 'launcher.c', '-framework', 'CoreFoundation'])
+def build_launcher(out, minos, clang, sdk):
+    run([clang, '-isysroot', sdk, '-arch', 'arm64', f'-mmacosx-version-min={minos}', '-O2', '-Wall', '-o', out, HERE / 'launcher.c', '-framework', 'CoreFoundation'])
 
 
 def make_icns(jpg, out):   # the game's icon (256x256 JPEG) at every size an .icns holds
-    with tempfile.TemporaryDirectory() as t:
+    with tempfile.TemporaryDirectory(dir=out.parent) as t:
         s = Path(t) / 'AppIcon.iconset'; s.mkdir()
         for n in ICON_SIZES:
             for k, suffix in ((1, ''), (2, '@2x')):
@@ -97,65 +97,149 @@ def info_plist(a, minos, icon):
     return p
 
 
-def sign(app, identity):   # the inner host first, then the bundle (its resources and the launcher); no --options runtime (see the top)
+def sign(app, identity, codesign):   # the inner host first, then the bundle (its resources and the launcher); no --options runtime (see the top)
     host = app / 'Contents/MacOS/dmhost'
     for target in (host, app):
-        try: run(['codesign', '--force', '--sign', identity, target])
+        try: run([codesign, '--force', '--sign', identity, target])
         except SystemExit as e:
             if 'detritus' not in str(e): raise
-            run(['xattr', '-cr', app]); run(['codesign', '--force', '--sign', identity, target])   # Finder information or resource forks on copied files: codesign refuses them
+            run(['xattr', '-cr', app]); run([codesign, '--force', '--sign', identity, target])   # Finder information or resource forks on copied files: codesign refuses them
+
+
+def digest(path):
+    h = hashlib.sha256()
+    with Path(path).open('rb') as f:
+        for b in iter(lambda: f.read(1 << 20), b''): h.update(b)
+    return h.hexdigest()
+
+
+def inventory(root):
+    """Stat trees avoid reading multi-GB game data, and detect changes to executable modes in the app."""
+    def walk_error(e): raise e
+    paths = [root]
+    for parent, dirs, files in os.walk(root, onerror=walk_error):
+        paths.extend(Path(parent) / n for n in dirs + files)
+    rows = []
+    for p in sorted(paths):
+        s = p.lstat()
+        rows.append([str(p.relative_to(root)), s.st_mode, s.st_size, s.st_mtime_ns, s.st_ctime_ns,
+                     os.readlink(p) if stat.S_ISLNK(s.st_mode) else ''])
+    return rows
+
+
+def selected_inputs(a):
+    """Resolve once: these same lists supply both the reuse key and the copies."""
+    for name in ('patched', 'romfs', 'msl', 'tables', 'font', 'aoc', 'pso'):
+        p = getattr(a, name)
+        if p and not Path(p).is_dir(): raise SystemExit(f'no {name} directory at {p}')
+    if a.marker and not (Path(a.romfs) / a.marker).exists(): raise SystemExit(f'no RomFS at {a.romfs} (missing {a.marker})')
+    mods = sorted({p for pat in MODULE_FILES for p in Path(a.patched).glob(pat)})
+    if not any(p.name.endswith('.patched.bin') for p in mods): raise SystemExit(f'no rewritten module in {a.patched}')
+    for p in mods:
+        if p.name.endswith('.patched.bin') and not p.with_name(p.name.removesuffix('.patched.bin') + '.fixups.tsv').is_file():
+            raise SystemExit(f'no rewriter fixups for {p}')
+    msls = sorted(p for pat in MSL_FILES for p in Path(a.msl).glob(pat) if p.is_file() and p.name != '_b_build.json')
+    if not any(p.suffix == '.metallib' for p in msls): raise SystemExit(f'no shader libraries at {a.msl}')
+    pso = sorted({p for pat in PSO_FILES for p in Path(a.pso).glob(pat)}) if a.pso else []
+    if a.pso and (not any(p.suffix == '.mtl4a' for p in pso) or not (Path(a.pso) / 'prebuilt.tsv').is_file()):
+        raise SystemExit(f'no pipeline archive with its prebuilt.tsv in {a.pso}')
+    tables = [Path(a.tables) / n for n in TABLES if (Path(a.tables) / n).is_file()]
+    pairs = Path(a.pso_pairs) if a.pso_pairs and not any(p.name == 'pso_pairs.tsv' for p in tables) else None
+    fonts = sorted(Path(a.font).glob('*.ttf'))
+    files = [Path(a.host)] + mods + msls + pso + tables + fonts + ([pairs] if pairs else [])
+    files += [Path(p) for p in (a.env_file, a.icon) if p]
+    for p in files:
+        if not p.is_file(): raise SystemExit(f'no input file at {p}')
+    return mods, msls, pso, tables, pairs, fonts, files
+
+
+def replace_app(tmp, out):
+    if out.exists():
+        # RENAME_SWAP exchanges complete directories atomically; a failed swap leaves the old app at out.
+        swap = ctypes.CDLL(None, use_errno=True).renamex_np
+        swap.argtypes = (ctypes.c_char_p, ctypes.c_char_p, ctypes.c_uint)
+        swap.restype = ctypes.c_int
+        if swap(os.fsencode(tmp), os.fsencode(out), 2):
+            err = ctypes.get_errno(); raise OSError(err, os.strerror(err), str(out))
+    else:
+        tmp.rename(out)
 
 
 def pack(a):
     out = Path(a.out).resolve()
     if out.suffix != '.app': raise SystemExit(f'the output must end in .app: {out}')
+    key_file = out.with_name(out.name + '.key')
+    try:
+        previous = json.loads(key_file.read_text())
+    except (OSError, ValueError): previous = None
+    try:
+        a = argparse.Namespace(**vars(a))
+        for name in ('out', 'host', 'patched', 'romfs', 'msl', 'tables', 'font', 'aoc', 'pso', 'pso_pairs', 'env_file', 'icon'):
+            if getattr(a, name): setattr(a, name, str(Path(getattr(a, name)).resolve()))
+        mods, msls, pso, tables, pairs, fonts, files = selected_inputs(a)
+        clang = Path(run(['xcrun', '--find', 'clang']).strip()).resolve()
+        sdk = Path(run(['xcrun', '--sdk', 'macosx', '--show-sdk-path']).strip()).resolve()
+        if not clang.is_file() or not sdk.is_dir(): raise SystemExit(f'no compiler or SDK at {clang}, {sdk}')
+        codesign = shutil.which('codesign')
+        if not codesign: raise SystemExit('codesign is not available')
+        codesign = Path(codesign).resolve()
+        minos = min_os(a.host)
+        clang_stat = clang.stat()
+        inputs = {'options': vars(a), 'files': [(str(p), p.stat().st_mode, digest(p), os.readlink(p) if p.is_symlink() else None) for p in files + [Path(__file__).resolve(), HERE / 'launcher.c']],
+                  'trees': [(p, inventory(Path(p))) for p in (a.romfs, a.aoc) if p],
+                  'tools': [str(clang), clang_stat.st_mode, clang_stat.st_size, clang_stat.st_mtime_ns, clang_stat.st_ctime_ns, run([clang, '--version']).strip(),
+                            str(sdk), run(['xcrun', '--sdk', 'macosx', '--show-sdk-version']).strip(),
+                            str(codesign), codesign.stat().st_mode, digest(codesign), minos]}
+        key = hashlib.sha256(json.dumps(inputs, sort_keys=True).encode()).hexdigest()
+        if out.is_dir() and isinstance(previous, dict) and previous.get('inputs') == key and previous.get('app') == inventory(out):
+            print(f'app: {out}, reused (inputs unchanged)', flush=True)
+            return
+    except BaseException:
+        key_file.unlink(missing_ok=True)
+        raise
+    key_file.unlink(missing_ok=True)   # no failed attempt may leave a valid reuse key
     tmp = out.with_name(out.name + '.partial')
     shutil.rmtree(tmp, ignore_errors=True)
     c = tmp / 'Contents'; mac, res = c / 'MacOS', c / 'Resources'; dm = res / 'dm'
     for d in (mac, dm / 'font'): d.mkdir(parents=True)
     romfs, msl, aoc = Path(a.romfs), Path(a.msl), Path(a.aoc) if a.aoc else None
-    if a.marker and not (romfs / a.marker).exists(): raise SystemExit(f'no RomFS at {romfs} (missing {a.marker})')
-    msls = sorted(p for pat in MSL_FILES for p in msl.glob(pat) if p.is_file() and p.name != '_b_build.json') if msl.is_dir() else []
-    if not any(p.suffix == '.metallib' for p in msls): raise SystemExit(f'no shader libraries at {msl}')
-    pso = sorted({p for pat in PSO_FILES for p in Path(a.pso).glob(pat)}) if a.pso else []
-    if a.pso and (not any(p.suffix == '.mtl4a' for p in pso) or not (Path(a.pso) / 'prebuilt.tsv').is_file()): raise SystemExit(f'no pipeline archive with its prebuilt.tsv in {a.pso}')
     clone = can_clone(romfs, res); pso_clone = bool(pso) and can_clone(Path(a.pso), res)
     if not clone or (pso and not pso_clone):
         need = (0 if clone else sum(tree_bytes(d) for d in [romfs] + ([aoc] if aoc else [])) + sum(p.stat().st_size for p in msls)) + (0 if pso_clone else sum(p.stat().st_size for p in pso))
         free = shutil.disk_usage(res).free
         if need + (1 << 30) > free: raise SystemExit(f'the app needs {need / 2**30:.1f} GB on the volume of {out.parent} (no APFS clones from the build there) and {free / 2**30:.1f} GB are free')
     step(f'launcher and host ({"clones" if clone else "copies"} of the game files)')
-    minos = min_os(a.host)
-    build_launcher(mac / EXE, minos)
+    build_launcher(mac / EXE, minos, clang, sdk)
     shutil.copy2(a.host, mac / 'dmhost')
-    mods = sorted({p for pat in MODULE_FILES for p in Path(a.patched).glob(pat)})
-    if not any(p.name.endswith('.patched.bin') for p in mods): raise SystemExit(f'no rewritten module in {a.patched}')
-    for p in mods:
-        if p.name.endswith('.patched.bin') and not p.with_name(p.name.removesuffix('.patched.bin') + '.fixups.tsv').is_file():
-            raise SystemExit(f'no rewriter fixups for {p}')
     copy(mods, dm, clone)
-    tables = [Path(a.tables) / n for n in TABLES if (Path(a.tables) / n).is_file()]
     if tables: copy(tables, dm, False)
-    if a.pso_pairs and not (dm / 'pso_pairs.tsv').is_file(): shutil.copyfile(a.pso_pairs, dm / 'pso_pairs.tsv')   # the split's copy, for a profile without the table
-    fonts = sorted(Path(a.font).glob('*.ttf'))
+    if pairs: shutil.copyfile(pairs, dm / 'pso_pairs.tsv')   # the split's copy, for a profile without the table
     if fonts: copy(fonts, dm / 'font', False)
     if a.env_file: shutil.copyfile(a.env_file, dm / 'env.txt')
     step('shader libraries'); msl_dst = res / 'msl' / (a.msl_name or msl.name); msl_dst.mkdir(parents=True); copy(msls, msl_dst, clone)
     if aoc: step('add-on content'); copy([aoc], res / 'aoc', clone)
     if pso: step('pipeline archive'); (res / 'pso').mkdir(); copy(pso, res / 'pso', pso_clone)
     step('game files'); copy([romfs], res / 'romfs', clone)
-    icon = bool(a.icon and Path(a.icon).is_file())
+    icon = bool(a.icon)
     if icon: step('icon'); make_icns(a.icon, res / 'AppIcon.icns')
     info = info_plist(a, minos, icon)
     info['DMModuleFiles'] = [p.name for p in mods]
     (c / 'Info.plist').write_bytes(plistlib.dumps(info))
     (c / 'PkgInfo').write_text('APPL????')
-    step(f'signing ({"ad-hoc" if a.sign == "-" else a.sign})'); sign(tmp, a.sign)
-    old = out.with_name(out.name + '.old')
-    shutil.rmtree(old, ignore_errors=True)
-    if out.exists(): out.rename(old)
-    tmp.rename(out)
-    shutil.rmtree(old, ignore_errors=True)
+    step(f'signing ({"ad-hoc" if a.sign == "-" else a.sign})'); sign(tmp, a.sign, codesign)
+    had_old = out.exists()
+    replace_app(tmp, out)
+    key_tmp = key_file.with_name(key_file.name + '.partial')
+    try:
+        key_tmp.write_text(json.dumps({'inputs': key, 'app': inventory(out)}, sort_keys=True) + '\n')
+        key_tmp.replace(key_file)
+    except BaseException:
+        if had_old: replace_app(tmp, out)
+        else: out.rename(tmp)
+        raise
+    finally:
+        key_tmp.unlink(missing_ok=True)
+    shutil.rmtree(tmp, ignore_errors=True)   # the old app after a successful swap
     print(f'app: {out}', flush=True)
 
 

dm/tools/tests/test_nsp_incremental.py

@@ -0,0 +1,392 @@
+"""NSP script reuse across derived and provisional builds. Real fingerprints and mock tool runs; no game files, keys or Xcode needed."""
+import importlib.util
+import os
+from pathlib import Path
+import shutil
+import sys
+import tempfile
+import unittest
+from unittest import mock
+
+REPO = Path(__file__).resolve().parents[3]
+sys.path.insert(0, str(REPO / 'dm/games')); sys.path.insert(0, str(REPO / 'dm/tools'))
+import game, nacp, profile_guess
+spec = importlib.util.spec_from_file_location('build_ipa_from_nsp', REPO / 'scripts/build_ipa_from_nsp.py')
+nsp = importlib.util.module_from_spec(spec); spec.loader.exec_module(nsp)
+nsp.pg, nsp.nacp = profile_guess, nacp   # main sets them
+
+TID = 0x0100AAAA00000000
+BID, BID2 = 'ab' * 16, 'cd' * 16
+
+
+def nso(path, bid):   # an NSO header: the build ID at +0x40
+    path.parent.mkdir(parents=True, exist_ok=True); path.write_bytes(b'NSO0' + bytes(0x3c) + bytes.fromhex(bid) + bytes(0x50))
+
+
+def control(path, disp):   # control.nacp with the display version at 0x3060
+    path.parent.mkdir(parents=True, exist_ok=True); path.write_bytes(bytes(0x3060) + disp.encode().ljust(0x10, b'\0') + bytes(0xa0))
+
+
+class Base(unittest.TestCase):
+    def setUp(self):
+        self.tmp = Path(tempfile.mkdtemp()).resolve(); self.addCleanup(shutil.rmtree, self.tmp)
+        self.src = self.tmp / 'games' / f'{TID:016X}'; self.src.mkdir(parents=True)
+        (self.src / 'game.tsv').write_text('\n'.join([f'title_id\t{TID:016X}', f'program_id\t{TID:016X}', 'name\tTest Game', 'modules\tmain', 'app_category\tpublic.app-category.games',
+                                                      'romfs_marker\ta/b', f'build\tv0\t1.0.0\tmain={BID}', 'package\tv0', 'shader_versions\tv0']) + '\n')
+        (self.src / 'shader_map.tsv').write_text('# registered\npkg.bin\t0\tvertex\tregistered\tx\n')
+        (self.src / 'shader_facts_manual.tsv').write_text('k\t{"manual": 1}\n')
+        (self.src / 'pso_manifest.tsv').write_text('S\tk\tvertex\n')
+        (self.src / 'dmsh_chain.h').write_text('#define DMSH_CHAIN "old"\n')
+        self.ex = self.tmp / 'extract/v65536'; nso(self.ex / 'ExeFS/main', BID2); control(self.ex / 'control.nacp', '1.1.0')
+        (self.ex / 'RomFS').mkdir(); (self.ex / 'RomFS/a.bin').write_bytes(b'romfs')
+        self.pdir = self.tmp / 'derived' / f'{TID:016X}'
+        for cm in (mock.patch.dict(os.environ, {'DM_GAMES_DIR': str(self.tmp / 'games')}), mock.patch.object(nsp, 'PROG', mock.Mock()), mock.patch.object(nsp, 'say', lambda m: None)):
+            cm.__enter__(); self.addCleanup(cm.__exit__, None, None, None)
+
+    def derive(self, ver='v65536', mods=('main',), app=None):
+        return nsp.derive_profile(game.Game(self.src), self.pdir, ver, self.ex, list(mods), app)
+
+    def redump(self, path):   # identical bytes and size, but a fresh extraction or capture timestamp
+        before = path.stat()
+        os.utime(path, ns=(before.st_atime_ns, before.st_mtime_ns + 1_000_000_000))
+
+
+class DeriveProfile(Base):
+    def later_stages(self):   # what refine_after_rewrite and the shader split write into the derived profile
+        nsp.set_keys(game.Game(self.pdir), {'mem_used': ('0x4800000', 'rewritten modules')}, 'After the rewrite')
+        (self.pdir / 'shader_map.tsv').write_text('# split\npkg.bin\t0\tvertex\tderived\tx\n')
+
+    def assert_kept(self):
+        self.assertEqual(game.Game(self.pdir).get('mem_used'), '0x4800000')
+        self.assertIn('derived', (self.pdir / 'shader_map.tsv').read_text())
+
+    def assert_fresh(self):
+        self.assertNotIn('mem_used', [k for k, _ in game.Game(self.pdir).rows])
+        self.assertFalse((self.pdir / 'shader_map.tsv').exists())
+
+    def test_unchanged_inputs_keep_later_stages(self):
+        self.assertEqual(self.derive(), '1.1.0')
+        self.later_stages()
+        self.assertEqual(self.derive(), '1.1.0')
+        self.assert_kept()
+        self.assertEqual(game.Game(self.pdir).builds()['v65536']['main'], BID2)
+        self.assertEqual(self.derive(), '1.1.0'); self.assert_kept()   # still kept on a third run
+
+    def test_changed_inputs_derive_again(self):
+        changes = {
+            'source game.tsv': lambda: (self.src / 'game.tsv').write_text((self.src / 'game.tsv').read_text() + 'pad_layout\tfull\n'),
+            'source manual table': lambda: (self.src / 'shader_facts_manual.tsv').write_text('k\t{"manual": 2}\n'),
+            'source chain header': lambda: (self.src / 'dmsh_chain.h').write_text('#define DMSH_CHAIN "new"\n'),
+            'new source file': lambda: (self.src / 'bf_want.tsv').write_text('# new\n'),
+            'main build': lambda: nso(self.ex / 'ExeFS/main', 'ef' * 16),
+            'display version': lambda: control(self.ex / 'control.nacp', '1.2.0'),
+        }
+        for what, change in changes.items():
+            with self.subTest(what):
+                self.derive(); self.later_stages(); self.derive(); self.assert_kept()
+                change(); self.derive(); self.assert_fresh()
+                for p in self.src.iterdir():
+                    if p.name not in nsp.DERIVED_SKIP: self.assertEqual((self.pdir / p.name).read_bytes(), p.read_bytes())
+        self.assertEqual(game.Game(self.pdir).builds()['v65536'], {'display': '1.2.0', 'main': 'ef' * 16})
+
+    def test_changed_arguments_derive_again(self):
+        nso(self.ex / 'ExeFS/sdk', BID)   # in the build row only when the modules name it
+        for what, kw in (('version', {'ver': 'v131072'}), ('modules', {'mods': ('main', 'sdk')}), ('title', {'app': 0x0100AAAA00FFF000})):
+            with self.subTest(what):
+                self.derive(); self.later_stages(); self.derive(); self.assert_kept()
+                self.derive(**kw); self.assert_fresh()
+        self.assertEqual(game.Game(self.pdir).title_id, 0x0100AAAA00FFF000)
+
+    def test_removed_source_file_goes(self):
+        self.derive(); self.later_stages(); self.derive(); self.assert_kept()
+        (self.src / 'pso_manifest.tsv').unlink(); self.derive()
+        self.assertFalse((self.pdir / 'pso_manifest.tsv').exists()); self.assert_fresh()
+
+    def test_edited_copy_is_kept_until_inputs_change(self):   # the stored digest is of the inputs, not of the derived files
+        self.derive(); (self.pdir / 'pso_manifest.tsv').write_text('S\tk2\tpixel\n'); self.derive()
+        self.assertEqual((self.pdir / 'pso_manifest.tsv').read_text(), 'S\tk2\tpixel\n')
+
+    def test_missing_corrupt_and_duplicate_override_rows_derive_again(self):
+        for change in ('missing', 'corrupt', 'override', 'indented override', 'truncated'):
+            with self.subTest(change):
+                self.derive(); p = self.pdir / 'game.tsv'; original = p.read_text()
+                if change == 'missing': broken = original.replace('romfs_marker\ta/b\n', '')
+                elif change == 'corrupt': broken = original.replace('romfs_marker\ta/b', 'romfs_marker\tx/y')
+                elif change == 'override': broken = 'romfs_marker\tx/y\n' + original
+                elif change == 'indented override': broken = ' romfs_marker\tx/y\n' + original
+                else: broken = original[:original.index('romfs_marker')]
+                p.write_text(broken)
+                self.derive()
+                self.assertEqual(p.read_text(), original)
+
+    def test_repeated_source_rows_and_computed_additions_are_kept(self):
+        p = self.src / 'game.tsv'; p.write_text(p.read_text() + 'extra\tone\nextra\tone\n')
+        self.derive(); self.later_stages(); self.derive(); self.assert_kept()
+        p = self.pdir / 'game.tsv'; p.write_text(p.read_text().replace('extra\tone\n', '', 1))
+        self.derive()
+        self.assertEqual(game.Game(self.pdir).values('extra'), [['one'], ['one']])
+
+    def test_provisional_computed_mem_used_grows_shrinks_and_stays_stable(self):
+        self.derive(); g = game.Game(self.pdir)
+        # These reason comments were written by earlier versions of this script.
+        for reason in ('the rewritten modules are larger than the default 64 MB of used memory (Age of Calamity: 66.6 MB → 72 MB)',
+                       'used memory for the rewritten modules (above the default 64 MB)',
+                       'used memory for the current rewritten modules, at least the default 64 MB'):
+            g = nsp.set_keys(g, {'mem_used': ('0x4800000', reason)}, 'After the rewrite')
+            with mock.patch.object(nsp.pg, 'mem_used') as used:
+                for value in (0x5000000, None, 0x4800000):
+                    used.return_value = value
+                    g = nsp.refine_after_rewrite(g, self.ex, self.tmp / 'patched')
+                    self.assertEqual(int(g.get('mem_used'), 0), value or 0x4000000)
+                    before = g.table('game.tsv').read_bytes()
+                    nsp.refine_after_rewrite(g, self.ex, self.tmp / 'patched')
+                    self.assertEqual(g.table('game.tsv').read_bytes(), before)
+
+    def test_explicit_mem_used_without_registered_profile_is_kept(self):
+        self.derive(); p = self.pdir / 'game.tsv'; p.write_text(p.read_text() + 'mem_used\t0x5800000\n')
+        g = game.Game(self.pdir)
+        with mock.patch.object(nsp.pg, 'mem_used', side_effect=AssertionError('explicit value must win')):
+            self.assertIs(nsp.refine_after_rewrite(g, self.ex, self.tmp / 'patched'), g)
+            self.assertIs(nsp.refine_after_rewrite(g, self.ex, self.tmp / 'patched', game.Game(self.src)), g)
+
+    def test_computed_equal_numeric_value_keeps_bytes(self):
+        self.derive()
+        g = nsp.set_keys(game.Game(self.pdir), {'mem_used': ('75497472', 'used memory for the current rewritten modules, at least the default 64 MB')}, 'After the rewrite')
+        before = g.table('game.tsv').read_bytes()
+        with mock.patch.object(nsp.pg, 'mem_used', return_value=0x4800000) as used:
+            nsp.refine_after_rewrite(g, self.ex, self.tmp / 'patched')
+        used.assert_called_once()
+        self.assertEqual(g.table('game.tsv').read_bytes(), before)
+
+    def test_derived_replace_failure_preserves_original(self):
+        self.derive(); p = self.pdir / 'game.tsv'; before = p.read_bytes()
+        with mock.patch.object(nsp.os, 'replace', side_effect=OSError('interrupted replace')):
+            with self.assertRaisesRegex(OSError, 'interrupted replace'): self.derive(ver='v131072')
+        self.assertEqual(p.read_bytes(), before)
+
+    def test_set_keys_replace_failure_preserves_original(self):
+        self.derive(); g = game.Game(self.pdir); before = g.table('game.tsv').read_bytes()
+        with mock.patch.object(nsp.os, 'replace', side_effect=OSError('interrupted replace')):
+            with self.assertRaisesRegex(OSError, 'interrupted replace'):
+                nsp.set_keys(g, {'mem_used': ('0x4800000', 'rewritten modules')}, 'After the rewrite')
+        self.assertEqual(g.table('game.tsv').read_bytes(), before)
+        self.assertEqual(sorted(p.name for p in self.pdir.iterdir() if p.name.startswith('.game.tsv')), [])
+
+    def test_heap_stamp_replace_failure_preserves_original(self):
+        self.derive(); g = game.Game(self.pdir); before = g.table('game.tsv').read_bytes()
+        logs = self.tmp / 'logs'; logs.mkdir()
+        def boot(*args): (logs / 'heap_check.log').write_text('>>> nnMain\n'); return 0
+        with mock.patch.object(nsp, 'build_mac_host'), mock.patch.object(nsp, 'boot_mac', side_effect=boot), \
+             mock.patch.object(nsp.pg, 'heap_signals', return_value={}), mock.patch.object(nsp.os, 'replace', side_effect=OSError('interrupted replace')):
+            with self.assertRaisesRegex(OSError, 'interrupted replace'):
+                nsp.heap_check(REPO, self.ex, self.tmp / 'patched', self.tmp / 'work', {}, logs, g)
+        self.assertEqual(g.table('game.tsv').read_bytes(), before)
+
+    def test_rewrite_changes_update_auto_mem_used(self):
+        self.derive(); g = game.Game(self.pdir); reg = game.Game(self.src)
+        with mock.patch.object(nsp.pg, 'mem_used') as used:
+            for value in (0x4800000, 0x5000000, None):
+                used.return_value = value
+                g = nsp.refine_after_rewrite(g, self.ex, self.tmp / 'patched', reg)
+                self.assertEqual(int(g.get('mem_used'), 0), value or 0x4000000)
+                before = g.table('game.tsv').read_bytes()
+                self.derive(); g = game.Game(self.pdir)
+                g = nsp.refine_after_rewrite(g, self.ex, self.tmp / 'patched', reg)
+                self.assertEqual(g.table('game.tsv').read_bytes(), before)
+
+    def test_explicit_mem_used_is_kept_and_equal_values_do_not_rewrite(self):
+        self.derive(); g = nsp.set_keys(game.Game(self.pdir), {'mem_used': ('75497472', '')}, 'After the rewrite')
+        before = g.table('game.tsv').read_bytes()
+        with mock.patch.object(nsp.pg, 'mem_used', return_value=0x4800000):
+            nsp.refine_after_rewrite(g, self.ex, self.tmp / 'patched', game.Game(self.src))
+        self.assertEqual(g.table('game.tsv').read_bytes(), before)
+        with (self.src / 'game.tsv').open('a') as f: f.write('mem_used\t0x5800000\n')
+        self.derive(); g = game.Game(self.pdir)
+        with mock.patch.object(nsp.pg, 'mem_used', side_effect=AssertionError('explicit value must win')):
+            self.assertIs(nsp.refine_after_rewrite(g, self.ex, self.tmp / 'patched', game.Game(self.src)), g)
+        self.assertEqual(g.get('mem_used'), '0x5800000')
+
+    def test_provisional_run_invalidates_derivation(self):
+        self.derive()
+        key = nsp.derived_key(self.pdir)
+        self.assertTrue(key.is_file())
+        nsp.set_keys(game.Game(self.pdir), {'mem_alias': ('0x80000000', '')}, 'Heap check')
+        keys = self.tmp / 'prod.keys'; keys.write_text('test')
+        fam = {'base': self.tmp / 'base.nsp', 'updates': [], 'dlc': []}
+        args = nsp.parse_args([str(self.tmp), '-k', str(keys), '--platform', 'mac', '--provisional', '--profile-out', str(self.pdir.parent)])
+        with mock.patch.object(nsp, 'find_repo', return_value=REPO), mock.patch.object(nsp, 'apple_silicon', return_value=True), \
+             mock.patch.object(nsp, '_importable', return_value=True), mock.patch.object(nsp.shutil, 'which', return_value='/dotnet'), \
+             mock.patch.object(nsp, 'scan_nsps', return_value=(TID, fam)), mock.patch.object(nsp, 'decompress_nsz', return_value=fam), \
+             mock.patch.object(nsp, 'nsp_versions', side_effect=RuntimeError('profile selected')):
+            with self.assertRaisesRegex(RuntimeError, 'profile selected'): nsp.main(args)
+        self.assertFalse(key.exists())
+        self.derive()
+        self.assertFalse(game.Game(self.pdir).values('mem_alias'))
+
+
+class SplitShaders(Base):
+    NAMES = ('shader_map.tsv', 'shader_facts.tsv', 'shader_pairs.tsv', 'pso_pairs.tsv')
+
+    def setUp(self):
+        super().setUp()
+        self.repo = self.tmp / 'repo'
+        for f in ('dm/gfx/tools/shader_split.py', 'dm/gfx/tools/dmsh_chain.lock', 'dm/games/game.py', 'dm/ios/tools/ios_metal.py'):
+            p = self.repo / f; p.parent.mkdir(parents=True, exist_ok=True); p.write_text('v1')
+        self.pkg = self.tmp / 'work/shader/pkg'; self.logs = self.tmp / 'logs'; self.logs.mkdir()
+        self.splits, self.rows = 0, 'derived'
+        cm = mock.patch.object(nsp, 'run', self.split); cm.__enter__(); self.addCleanup(cm.__exit__, None, None, None)
+
+    def split(self, cmd, log, env=None, cwd=None, on_line=None):   # shader_split.py: the package and the four tables, deterministic for the same rows
+        self.splits += 1; out = Path(cmd[4]); out.mkdir(parents=True); (out / 'main.bin').write_bytes(self.rows.encode())
+        for n, t in zip(self.NAMES, cmd[5:]): Path(t).write_text(self.table(n))
+        Path(log).write_text('shaders: 3 (3 vertex)\n')
+
+    def table(self, n): return f'# split header\n{n}\t{self.rows}\n'
+
+    def go(self, g, own=True, oodle=None):
+        return nsp.split_shaders(sys.executable, self.repo, [self.ex / 'RomFS'], [self.ex / 'ExeFS/main'], self.pkg, {}, self.logs, g, own, oodle)
+
+    def own(self):   # a derived profile with the empty tables, as main makes it
+        self.derive(); g = game.Game(self.pdir); nsp.provisional_prep(sys.executable, self.repo, {}, self.logs, g); return g
+
+    def test_own_profile_reuses_and_returns_the_package_key(self):
+        g = self.own()
+        k1 = self.go(g)
+        self.assertEqual((self.splits, k1), (1, nsp.package_key(self.pkg)))
+        for n in self.NAMES: self.assertEqual(g.table(n).read_text(), self.table(n))
+        k2 = self.go(g)
+        self.assertEqual((self.splits, k2), (1, k1))   # the key stored after the tables were written: the second run sees the same inputs
+
+    def test_redumped_inputs_invalidate_only_own_split(self):
+        for n in self.NAMES[:3]: (self.src / n).write_text(self.table(n))
+        captures = [self.ex / 'glslc_capture/a.glslc', self.ex / 'ue_shaders/a.ushaderbytecode']
+        for p in captures: p.parent.mkdir(); p.write_bytes(b'shader')
+        for own in (False, True):
+            g = self.own() if own else game.Game(self.src)
+            self.go(g, own=own)
+            for p in [self.ex / 'RomFS/a.bin', *captures]:
+                with self.subTest(own=own, path=p):
+                    before = self.splits; self.redump(p); self.go(g, own=own)
+                    self.assertEqual(self.splits, before + int(own))
+                    self.go(g, own=own); self.assertEqual(self.splits, before + int(own))
+
+    def test_decoder_host_and_modules_invalidate_split(self):
+        g = self.own()
+        host = self.tmp / 'decoder-host'; host.write_bytes(b'host')
+        modules = self.tmp / 'patched'; modules.mkdir()
+        module = modules / 'main.patched.bin'; module.write_bytes(b'code')
+        decoder = mock.Mock(return_value={'DM_OODLE_HOST': str(host), 'DM_OODLE_MODULES': str(modules)})
+        self.go(g, oodle=decoder); self.go(g, oodle=decoder); self.assertEqual(self.splits, 1)
+        host.write_bytes(b'fixed host')
+        self.go(g, oodle=decoder); self.go(g, oodle=decoder); self.assertEqual(self.splits, 2)
+        module.write_bytes(b'new rewritten module')
+        self.go(g, oodle=decoder); self.go(g, oodle=decoder); self.assertEqual(self.splits, 3)
+        self.assertEqual(decoder.call_count, 6)
+
+    def test_own_profile_changes_split_again(self):
+        changes = {'game.tsv': lambda g: nsp.set_keys(g, {'mem_used': ('0x4800000', '')}, 'After the rewrite'),
+                   'manual table': lambda g: g.table('shader_facts_manual.tsv').write_text('k\t{"manual": 3}\n'),
+                   'main': lambda g: nso(self.ex / 'ExeFS/main', 'ef' * 16),
+                   'RomFS': lambda g: (self.ex / 'RomFS/b.bin').write_bytes(b'more'),
+                   'tool': lambda g: (self.repo / 'dm/gfx/tools/shader_split.py').write_text('v2')}
+        g = self.own(); self.go(g)
+        for i, (what, change) in enumerate(changes.items(), 2):
+            with self.subTest(what):
+                change(g); self.assertEqual(self.go(g), nsp.package_key(self.pkg)); self.assertEqual(self.splits, i)
+                self.go(g); self.assertEqual(self.splits, i)
+
+    def test_own_generated_tables_are_checked(self):   # an edited or missing table of the split is never taken as is: the split's output comes back (split again, or restored)
+        g = self.own(); self.go(g)
+        for n in self.NAMES:
+            with self.subTest(edited=n):
+                g.table(n).write_text('# edited\nstale\trow\n'); self.go(g)
+                self.assertEqual(g.table(n).read_text(), self.table(n))
+            with self.subTest(missing=n):
+                g.table(n).unlink(); self.go(g)
+                self.assertEqual(g.table(n).read_text(), self.table(n))
+        runs = self.splits; self.go(g); self.assertEqual(self.splits, runs)
+
+    def test_own_new_output_replaces_tables(self):   # a changed input whose split writes other rows: the profile takes them
+        g = self.own(); self.go(g)
+        self.rows = 'other'; (self.ex / 'RomFS/b.bin').write_bytes(b'more'); self.go(g)
+        self.assertEqual(g.table('shader_map.tsv').read_text(), self.table('shader_map.tsv'))
+        self.go(g); self.assertEqual(self.splits, 2)
+
+    def test_mode_is_in_the_key(self):   # the same files split as an own profile, then checked against them as a registered one
+        g = self.own(); self.go(g)
+        self.assertEqual(self.go(g, own=False), nsp.package_key(self.pkg)); self.assertEqual(self.splits, 2)
+        self.go(g, own=False); self.assertEqual(self.splits, 2)
+        self.go(g); self.assertEqual(self.splits, 3)
+
+    def test_registered_tables_are_compared(self):
+        for n in self.NAMES[:3]: (self.src / n).write_text(f'# older header\n{n}\t{self.rows}\n')
+        g = game.Game(self.src)
+        k = self.go(g, own=False)
+        self.assertEqual((self.splits, k), (1, nsp.package_key(self.pkg)))
+        self.assertEqual(self.go(g, own=False), k); self.assertEqual(self.splits, 1)
+        self.assertEqual((self.src / 'shader_map.tsv').read_text(), '# older header\nshader_map.tsv\tderived\n')   # a registered profile's tables are not written
+        self.rows = 'other'; (self.ex / 'RomFS/b.bin').write_bytes(b'more')
+        with self.assertRaisesRegex(nsp.BuildError, 'differ from game profile'): self.go(g, own=False)
+        self.assertEqual((self.src / 'shader_map.tsv').read_text(), '# older header\nshader_map.tsv\tderived\n')
+        with self.assertRaisesRegex(nsp.BuildError, 'differ from game profile'): self.go(g, own=False)   # a failed check is not reused
+        self.assertEqual(self.splits, 3)
+
+    def test_damaged_never_reuses(self):
+        g = self.own()
+        with mock.patch.object(nsp, 'ALLOW_DAMAGED', True):
+            self.assertIsNone(self.go(g)); self.assertIsNone(self.go(g))
+        self.assertEqual(self.splits, 2)
+        self.go(g); self.assertEqual(self.splits, 3)   # a damaged run leaves no key behind
+
+    def test_derived_rerun_is_incremental(self):   # the whole derived path of a second run: derive, prepare, rewrite keys, split
+        g = self.own(); g = nsp.set_keys(g, {'mem_used': ('0x4800000', '')}, 'After the rewrite'); k = self.go(g)
+        g = self.own(); self.assertEqual(g.get('mem_used'), '0x4800000')
+        self.assertEqual(self.go(g), k); self.assertEqual(self.splits, 1)
+
+
+class LayerShaders(Base):
+    def test_redumped_romfs_and_captures_invalidate_only_own_libraries(self):
+        cap = self.ex / 'capture'; cap.mkdir(); (cap / 'shader.bin').write_bytes(b'shader')
+        self.derive(); g = game.Game(self.pdir)
+        logs = self.tmp / 'logs'; logs.mkdir(); runs = []
+        def translate(cmd, log, **kw):
+            api = Path(cmd[1]).stem.split('_')[0]
+            lib = Path(cmd[-1]) / api / f'{api}_shaders.metallib'
+            lib.parent.mkdir(parents=True); lib.write_bytes(b'library')
+            Path(log).write_text('macosx: 1\n'); runs.append(api)
+        with mock.patch.object(nsp, 'run', side_effect=translate), mock.patch.object(nsp, 'toolchain', return_value=()), \
+             mock.patch.object(nsp.shutil, 'which', return_value='/glslangValidator'), \
+             mock.patch.object(nsp.subprocess, 'run', return_value=mock.Mock(stdout='compiler', stderr='')):
+            for api, build in (('vk', nsp.build_vk_shaders), ('gl', nsp.build_gl_shaders)):
+                roots = [self.ex / 'RomFS', cap] + ([self.ex / 'ExeFS/main'] if api == 'gl' else [])
+                for own in (False, True):
+                    sh = self.tmp / f'{api}-{own}'
+                    def go(): build(sys.executable, REPO, roots, sh, {'WORK': str(self.tmp), 'PATH': ''}, logs, g, mac=True, provisional=own)
+                    go(); go()
+                    for p in (self.ex / 'RomFS/a.bin', cap / 'shader.bin'):
+                        with self.subTest(api=api, own=own, path=p):
+                            before = len(runs); self.redump(p); go()
+                            self.assertEqual(len(runs), before + int(own))
+                            go(); self.assertEqual(len(runs), before + int(own))
+
+
+class MacApp(Base):
+    def test_packer_reports_reuse_and_first_pack_steps(self):
+        t = self.tmp; out = t / 'Game.app'; packs = []
+        def pack(cmd, log, env=None, on_line=None):
+            packs.append(cmd)
+            (out / 'Contents/MacOS').mkdir(parents=True, exist_ok=True)
+            (out / 'Contents/MacOS/dmhost').write_bytes(b'host')
+            if len(packs) == 1: on_line('step: launcher and host (clones of the game files)')
+            on_line(f'app: {out}' + (', reused (inputs unchanged)' if len(packs) > 1 else ''))
+        with mock.patch.object(nsp, 'run', side_effect=pack):
+            for i in range(2):
+                result = nsp.mac_app(sys.executable, REPO, out, TID, t, self.ex, t / 'patched', t / 'host', t / 'font', t / 'shader', None, 'vulkan', 'en-US',
+                                     game.Game(self.src), 'Game', '1.0.0', 'org.nxnative.test', None, '', {}, t)
+                self.assertIsNone(result)
+                if i: nsp.PROG.end.assert_called_with('Game.app, reused (inputs unchanged)')
+                else: nsp.PROG.set.assert_called_with(step='launcher and host (clones of the game files)')
+        self.assertEqual(len(packs), 2)   # pack_app owns the reuse decision and sidecar key
+
+
+if __name__ == '__main__': unittest.main()

dm/tools/tests/test_nsp_mac_app.py

@@ -25,6 +25,7 @@ nsp = importlib.util.module_from_spec(spec); spec.loader.exec_module(nsp)
 spec = importlib.util.spec_from_file_location('pack_app', REPO / 'dm/mac/pack_app.py')
 pack_app = importlib.util.module_from_spec(spec); spec.loader.exec_module(pack_app)
 TID = '01008CF01BAAC000'
+TEST_ROOT = Path(os.environ.get('NXNATIVE_TEST_ROOT', tempfile.gettempdir()))
 
 HOST_C = r'''#include <signal.h>
 #include <stdio.h>
@@ -72,7 +73,7 @@ class Copy(unittest.TestCase):
 
     @unittest.skipUnless(sys.platform == 'darwin', 'exercises the macOS exec limit')
     def test_long_paths(self):
-        with tempfile.TemporaryDirectory() as tmp:
+        with tempfile.TemporaryDirectory(dir=TEST_ROOT) as tmp:
             src = Path(tmp) / ('a' * 200) / ('b' * 200) / ('c' * 200); src.mkdir(parents=True)
             dst = Path(tmp) / 'out'; dst.mkdir()
             files = [src / f'{i:04d}_b.metallib' for i in range(2000)]
@@ -83,11 +84,245 @@ class Copy(unittest.TestCase):
             for p in files: self.assertEqual((dst / p.name).read_bytes(), p.name.encode())
 
 
+@unittest.skipUnless(sys.platform == 'darwin', 'uses macOS atomic bundle replacement')
+class PackReuse(unittest.TestCase):
+    """Call the real pack entry point; only external compiler, icon and signing tools are faked. No host is run."""
+    def setUp(self):
+        TEST_ROOT.mkdir(parents=True, exist_ok=True)
+        self.tmp = Path(tempfile.mkdtemp(prefix='pack-reuse-', dir=TEST_ROOT)).resolve()
+        self.addCleanup(shutil.rmtree, self.tmp)
+        t = self.tmp
+        for d in ('patched', 'romfs/empty', 'msl', 'profile', 'font', 'aoc/1', 'pso', 'bin', 'sdk', 'source'):
+            (t / d).mkdir(parents=True)
+        files = {
+            'dmhost': b'host', 'patched/main.patched.bin': b'code', 'patched/main.fixups.tsv': b'fixups',
+            'patched/main.gates.tsv': b'gates', 'patched/main.cuts.tsv': b'cuts', 'patched/main.obs.tsv': b'obs', 'patched/nro.tsv': b'nro',
+            'romfs/marker': b'game', 'msl/k.metallib': b'lib', 'msl/k.json': b'{}', 'msl/inputs.tsv': b'stamp',
+            'font/nintendo_ext.ttf': b'font', 'aoc/1/data': b'dlc', 'pso/gpu.mtl4a': b'archive',
+            'pso/prebuilt.tsv': b'prebuilt', 'pso/rest_pairs.tsv': b'rest', 'pso/archive.tsv': b'index',
+            'env.txt': b'DM_TEST=1', 'split.tsv': b'split', 'icon.jpg': b'jpeg', 'source/launcher.c': b'launcher',
+        }
+        for n in pack_app.TABLES: files['profile/' + n] = n.encode()
+        for n, content in files.items(): (t / n).write_bytes(content)
+        for tool in ('clang', 'codesign', 'other-codesign'):
+            (t / 'bin' / tool).write_text('fake ' + tool); (t / 'bin' / tool).chmod(0o755)
+        self.args = pack_app.parse([str(t / 'Game.app'), '--host', str(t / 'dmhost'), '--patched', str(t / 'patched'),
+                                   '--romfs', str(t / 'romfs'), '--msl', str(t / 'msl'), '--tables', str(t / 'profile'), '--font', str(t / 'font'),
+                                   '--aoc', str(t / 'aoc'), '--pso', str(t / 'pso'), '--pso-pairs', str(t / 'split.tsv'), '--env-file', str(t / 'env.txt'),
+                                   '--title-id', TID, '--name', 'Test Game', '--marker', 'marker', '--icon', str(t / 'icon.jpg')])
+        self.commands = []
+        self.clang_version, self.sdk_version = 'clang fixture 1', '27.0'
+        self.clang, self.sdk = t / 'bin/clang', t / 'sdk'
+        self.codesign = t / 'bin/codesign'
+        self.failed_tool = None
+        real_run, real_which = pack_app.run, shutil.which
+        def fake_run(cmd):
+            cmd = [str(c) for c in cmd]; self.commands.append(cmd)
+            name = Path(cmd[0]).name
+            if self.failed_tool == name and (name != 'clang' or '-o' in cmd): raise SystemExit('fixture ' + name + ' failed')
+            if name == 'otool': return 'cmd LC_BUILD_VERSION\n minos 14.0\n'
+            if name == 'xcrun':
+                if '--find' in cmd: return str(self.clang) + '\n'
+                if '--show-sdk-path' in cmd: return str(self.sdk) + '\n'
+                if '--show-sdk-version' in cmd: return self.sdk_version + '\n'
+                if 'clang' in cmd: name = 'clang'
+            if name == 'clang':
+                if '--version' in cmd: return self.clang_version + '\n'
+                out = Path(cmd[cmd.index('-o') + 1]); out.write_bytes(b'fake launcher'); out.chmod(0o755); return ''
+            if name in ('codesign', 'other-codesign'): return ''
+            if name == 'sips': Path(cmd[cmd.index('--out') + 1]).write_bytes(b'png'); return ''
+            if name == 'iconutil': Path(cmd[cmd.index('-o') + 1]).write_bytes(b'icns'); return ''
+            return real_run(cmd)
+        def which(name): return str(self.codesign) if name == 'codesign' else real_which(name)
+        for patch in (mock.patch.object(pack_app, 'run', fake_run), mock.patch.object(pack_app, 'HERE', t / 'source'),
+                      mock.patch.object(pack_app, 'can_clone', return_value=False), mock.patch.object(shutil, 'which', which)):
+            patch.start(); self.addCleanup(patch.stop)
+        self.clone = pack_app.can_clone
+
+    def pack(self):
+        self.commands.clear(); self.clone.reset_mock()
+        with contextlib.redirect_stdout(io.StringIO()) as output: pack_app.pack(self.args)
+        return output.getvalue()
+
+    @property
+    def key(self): return Path(self.args.out + '.key')
+
+    def assert_reused(self):
+        self.assertEqual(self.pack(), f'app: {Path(self.args.out).resolve()}, reused (inputs unchanged)\n')
+        self.clone.assert_not_called()
+        self.assertFalse(any('-o' in c or '--sign' in c or c[0] == '/bin/cp' for c in self.commands))
+
+    def assert_repacked(self):
+        self.assertNotIn('reused (inputs unchanged)', self.pack())
+        self.assertTrue(any('--sign' in c for c in self.commands))
+        self.assertTrue(self.key.is_file())
+
+    def test_retargeted_input_symlink_repacks(self):
+        msl = self.tmp / 'msl'
+        for name in ('left.metallib', 'right.metallib'): (msl / name).write_bytes(b'same library')
+        link = msl / 'linked.metallib'; link.symlink_to('left.metallib')
+        self.pack(); self.assert_reused()
+        link.unlink(); link.symlink_to('right.metallib')
+        self.assert_repacked()
+        packed = Path(self.args.out) / 'Contents/Resources/msl/msl/linked.metallib'
+        self.assertEqual(os.readlink(packed), 'right.metallib')
+        self.assert_reused()
+
+    def test_unchanged_reuses_before_partial_clone_compile_and_sign(self):
+        self.pack()
+        partial = Path(self.args.out + '.partial'); partial.mkdir(); (partial / 'leave').write_text('untouched')
+        self.assert_reused()
+        self.assertEqual((partial / 'leave').read_text(), 'untouched')
+
+    def test_each_selected_file_content_invalidates_even_with_same_stat(self):
+        self.pack()
+        names = ['dmhost', 'patched/main.patched.bin', 'patched/main.fixups.tsv', 'patched/main.gates.tsv', 'patched/main.cuts.tsv',
+                 'patched/main.obs.tsv', 'patched/nro.tsv', 'msl/k.metallib', 'msl/k.json', 'msl/inputs.tsv', 'font/nintendo_ext.ttf',
+                 'pso/gpu.mtl4a', 'pso/prebuilt.tsv', 'pso/rest_pairs.tsv', 'pso/archive.tsv', 'env.txt', 'icon.jpg', 'source/launcher.c']
+        names += ['profile/' + n for n in pack_app.TABLES]
+        for name in names:
+            with self.subTest(name=name):
+                p = self.tmp / name; st = p.stat(); data = p.read_bytes(); p.write_bytes(bytes([data[0] ^ 1]) + data[1:])
+                os.utime(p, ns=(st.st_atime_ns, st.st_mtime_ns)); self.assert_repacked()
+
+    def test_romfs_and_aoc_stat_trees_include_paths_modes_and_empty_directories(self):
+        self.pack()
+        for name in ('romfs/marker', 'aoc/1/data'):
+            with self.subTest(name=name):
+                p = self.tmp / name; p.write_bytes(b'changed size'); self.assert_repacked()
+                p.chmod(0o600); self.assert_repacked()
+                p.with_name('added').write_bytes(b'new'); self.assert_repacked()
+                p.with_name('added').rename(p.with_name('renamed')); self.assert_repacked()
+        (self.tmp / 'romfs/new-empty').mkdir(); self.assert_repacked()
+
+    def test_all_metadata_sign_identity_and_selected_paths_invalidate(self):
+        self.pack()
+        for name, value in [('name', 'New Game'), ('version', '2.0'), ('bundle_id', 'org.test.new'), ('category', 'public.app-category.action-games'),
+                            ('lang', 'ja'), ('title_id', '0100000000000000'), ('marker', 'empty'), ('msl_name', 'custom'), ('sign', 'Fixture identity')]:
+            with self.subTest(option=name): setattr(self.args, name, value); self.assert_repacked()
+        for name in ('host', 'patched', 'romfs', 'msl', 'tables', 'font', 'aoc', 'pso', 'env_file', 'icon', 'pso_pairs'):
+            with self.subTest(path=name):
+                old = Path(getattr(self.args, name)); new = old.with_name(old.name + '-other')
+                if old.is_dir(): shutil.copytree(old, new)
+                else: shutil.copy2(old, new)
+                setattr(self.args, name, str(new)); self.assert_repacked()
+
+    def test_compiler_sdk_and_resolved_sign_tool_invalidate(self):
+        self.pack()
+        self.clang_version = 'clang fixture 2'; self.assert_repacked()
+        self.clang.write_text('changed compiler'); self.assert_repacked()
+        self.sdk_version = '28.0'; self.assert_repacked()
+        self.sdk = self.tmp / 'other-sdk'; self.sdk.mkdir(); self.assert_repacked()
+        self.codesign = self.tmp / 'bin/other-codesign'; self.assert_repacked()
+        self.codesign.write_text('changed signing tool'); self.assert_repacked()
+
+    def test_selected_file_modes_and_file_additions_and_removals_invalidate(self):
+        self.pack()
+        (self.tmp / 'dmhost').chmod(0o755); self.assert_repacked()
+        for name in ('patched/new.patched.bin', 'patched/new.fixups.tsv', 'msl/new.metallib', 'font/new.ttf', 'pso/new.mtl4a'):
+            (self.tmp / name).write_text('new')
+        self.assert_repacked()
+        dm = Path(self.args.out) / 'Contents/Resources/dm'
+        self.assertEqual((dm / 'new.patched.bin').read_text(), 'new')
+        (self.tmp / 'profile/c1_full.tsv').unlink(); self.assert_repacked()
+        self.assertFalse((dm / 'c1_full.tsv').exists())
+        (self.tmp / 'patched/new.patched.bin').unlink(); (self.tmp / 'patched/new.fixups.tsv').unlink(); self.assert_repacked()
+
+    def test_output_path_and_optional_inputs_invalidate(self):
+        self.pack()
+        old = Path(self.args.out); new = old.with_name('Other.app')
+        shutil.copytree(old, new); shutil.copy2(self.key, Path(str(new) + '.key'))
+        self.args.out = str(new); self.assert_repacked()
+        for name in ('icon', 'env_file', 'aoc', 'pso', 'pso_pairs'):
+            with self.subTest(option=name): setattr(self.args, name, None); self.assert_repacked()
+
+    def test_packer_source_content_invalidates(self):
+        source = self.tmp / 'source/pack_app.py'; shutil.copy2(pack_app.__file__, source)
+        with mock.patch.object(pack_app, '__file__', str(source)):
+            self.pack(); source.write_text(source.read_text() + '\n# fixture change\n'); self.assert_repacked()
+
+    def test_output_missing_modified_mode_time_path_and_directory_invalidate(self):
+        self.pack()
+        launcher = Path(self.args.out) / 'Contents/MacOS/nxnative'
+        launcher.chmod(0o644); self.assert_repacked()
+        launcher.unlink(); self.assert_repacked()
+        launcher.write_bytes(b'broken'); self.assert_repacked()
+        st = launcher.stat(); os.utime(launcher, ns=(st.st_atime_ns, st.st_mtime_ns + 1000000000)); self.assert_repacked()
+        extra = Path(self.args.out) / 'Contents/extra'; extra.mkdir(); self.assert_repacked()
+        empty = Path(self.args.out) / 'Contents/Resources/romfs/empty'
+        empty.chmod(0o700); self.assert_repacked()
+        empty.rename(empty.with_name('renamed')); self.assert_repacked()
+        shutil.rmtree(self.args.out); self.assert_repacked()
+
+    def test_unpacked_files_and_unselected_split_pairs_do_not_invalidate(self):
+        self.pack()
+        for name in ('msl/k.metal', 'msl/_b_build.json', 'msl/analysis/report.txt', 'patched/main.sites.tsv', 'profile/game.tsv',
+                     'font/readme.txt', 'pso/analysis.txt', 'split.tsv'):
+            p = self.tmp / name; p.parent.mkdir(exist_ok=True); p.write_text('not packed'); self.assert_reused()
+        (self.tmp / 'split.tsv').unlink(); self.assert_reused()
+
+    def test_split_pairs_fallback_is_selected_only_without_profile_pairs(self):
+        (self.tmp / 'profile/pso_pairs.tsv').unlink(); self.pack()
+        target = Path(self.args.out) / 'Contents/Resources/dm/pso_pairs.tsv'
+        self.assertEqual(target.read_bytes(), b'split')
+        (self.tmp / 'split.tsv').write_bytes(b'new split'); self.assert_repacked()
+        self.assertEqual(target.read_bytes(), b'new split')
+
+    def test_missing_selected_inputs_fail_before_reuse_or_partial_creation(self):
+        self.pack()
+        for name in ('dmhost', 'patched/main.fixups.tsv', 'msl/k.metallib', 'romfs/marker', 'pso/prebuilt.tsv', 'env.txt', 'icon.jpg',
+                     'font', 'aoc', 'profile', 'bin/clang', 'sdk', 'bin/codesign', 'source/launcher.c'):
+            with self.subTest(name=name):
+                p = self.tmp / name; hidden = p.with_name(p.name + '.hidden'); p.rename(hidden)
+                try:
+                    with self.assertRaises((SystemExit, OSError)): self.pack()
+                    self.assertFalse(self.key.exists())
+                    self.assertFalse(Path(self.args.out + '.partial').exists())
+                finally: hidden.rename(p)
+                self.pack()
+
+    def test_failed_repack_drops_key_and_keeps_old_app(self):
+        self.pack()
+        before = (Path(self.args.out) / 'Contents/Info.plist').read_bytes()
+        for tool in ('clang', 'codesign'):
+            with self.subTest(tool=tool):
+                self.args.name += ' changed'; self.failed_tool = tool
+                with self.assertRaisesRegex(SystemExit, 'fixture'): self.pack()
+                self.assertFalse(self.key.exists())
+                self.assertEqual((Path(self.args.out) / 'Contents/Info.plist').read_bytes(), before)
+                self.failed_tool = None
+        self.assert_repacked()
+
+    def test_failed_atomic_replacement_drops_key_and_keeps_old_app(self):
+        self.pack(); before = (Path(self.args.out) / 'Contents/Info.plist').read_bytes()
+        self.args.name = 'changed'
+        library = types.SimpleNamespace(renamex_np=mock.Mock(return_value=-1))
+        with mock.patch.object(pack_app.ctypes, 'CDLL', return_value=library), mock.patch.object(pack_app.ctypes, 'get_errno', return_value=5):
+            with self.assertRaises(OSError): self.pack()
+        self.assertFalse(self.key.exists())
+        self.assertEqual((Path(self.args.out) / 'Contents/Info.plist').read_bytes(), before)
+        self.assert_repacked()
+
+    def test_failed_key_write_rolls_back_app_and_drops_key(self):
+        self.pack(); before = (Path(self.args.out) / 'Contents/Info.plist').read_bytes()
+        self.args.name = 'changed'
+        real_replace = Path.replace
+        def fail_key(path, target):
+            if str(target).endswith('.app.key'): raise OSError('fixture key write failed')
+            return real_replace(path, target)
+        with mock.patch.object(Path, 'replace', fail_key):
+            with self.assertRaisesRegex(OSError, 'fixture key write'): self.pack()
+        self.assertFalse(self.key.exists())
+        self.assertFalse(Path(str(self.key) + '.partial').exists())
+        self.assertEqual((Path(self.args.out) / 'Contents/Info.plist').read_bytes(), before)
+        self.assert_repacked()
+
+
 @unittest.skipUnless(sys.platform == 'darwin' and shutil.which('codesign'), 'needs macOS (clang, codesign, sips, iconutil)')
 class App(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        cls.tmp = Path(tempfile.mkdtemp()).resolve()
+        cls.tmp = Path(tempfile.mkdtemp(dir=TEST_ROOT)).resolve()
         t = cls.tmp
         (t / 'host.c').write_text(HOST_C)
         subprocess.run(['xcrun', 'clang', '-arch', 'arm64', '-o', t / 'dmhost', t / 'host.c'], check=True)
@@ -152,7 +387,7 @@ class App(unittest.TestCase):
         self.assertEqual((dm / 'pso_pairs.tsv').read_text(), 'other-vertex\tother-pixel\n')
 
     def test_split_pso_pairs(self):   # a profile without pso_pairs.tsv: the shader split's copy (--pso-pairs); the profile's own wins when it has one
-        with tempfile.TemporaryDirectory() as d, mock.patch.object(pack_app, 'sign', lambda *a: None), mock.patch('sys.stdout'):
+        with tempfile.TemporaryDirectory(dir=TEST_ROOT) as d, mock.patch.object(pack_app, 'sign', lambda *a: None), mock.patch('sys.stdout'):
             d = Path(d); (d / 'profile').mkdir(); (d / 'profile/shader_map.tsv').write_text('m'); (d / 'split.tsv').write_text('split-vertex\tsplit-pixel\n')
             a = types.SimpleNamespace(**vars(self.args)); a.out = str(self.tmp / 'Split.app'); a.icon = None; a.tables = str(d / 'profile'); a.pso_pairs = str(d / 'split.tsv')
             pack_app.pack(a)
@@ -239,7 +474,7 @@ class App(unittest.TestCase):
         self.assertIn('game data inside this app is missing', args[3])
 
     def test_relocated(self):   # run with the entire build folder unavailable
-        with tempfile.TemporaryDirectory() as dst:
+        with tempfile.TemporaryDirectory(dir=TEST_ROOT) as dst:
             dst = Path(dst).resolve(); moved = dst / self.app.name; home = dst / 'home'; home.mkdir()
             self.app.rename(moved)
             hidden = self.tmp.with_name(self.tmp.name + '.unavailable')
@@ -256,7 +491,7 @@ class App(unittest.TestCase):
                 moved.rename(self.app)
 
     def test_pack_missing_fixups(self):
-        with tempfile.TemporaryDirectory() as src:
+        with tempfile.TemporaryDirectory(dir=TEST_ROOT) as src:
             src = Path(src); (src / 'main.patched.bin').write_bytes(b'code')
             a = types.SimpleNamespace(**vars(self.args)); a.patched = str(src); a.out = str(self.tmp / 'Incomplete.app')
             with contextlib.redirect_stdout(io.StringIO()), self.assertRaisesRegex(SystemExit, 'no rewriter fixups'):
@@ -264,7 +499,7 @@ class App(unittest.TestCase):
             self.assertFalse(Path(a.out).exists())
 
     def test_pack_bad_archive(self):   # a pipeline archive folder without its archive or stamp is refused before anything is copied
-        with tempfile.TemporaryDirectory() as src:
+        with tempfile.TemporaryDirectory(dir=TEST_ROOT) as src:
             src = Path(src); (src / 'prebuilt.tsv').write_text('chain\t0123456789ab\n')
             a = types.SimpleNamespace(**vars(self.args)); a.pso = str(src); a.out = str(self.tmp / 'NoArchive.app')
             with contextlib.redirect_stdout(io.StringIO()) as o, self.assertRaisesRegex(SystemExit, 'no pipeline archive with its prebuilt.tsv'):
@@ -299,7 +534,7 @@ class Script(unittest.TestCase):
         self.assertEqual(nsp.mac_app_name('AC/DC'), 'AC - DC'); self.assertEqual(nsp.mac_app_name(':'), 'nxnative')
 
     def pack_cmd(self, gfx, **kw):
-        ran, tmp = [], Path(tempfile.mkdtemp()); self.addCleanup(shutil.rmtree, tmp); self.said = []
+        ran, tmp = [], Path(tempfile.mkdtemp(dir=TEST_ROOT)); self.addCleanup(shutil.rmtree, tmp); self.said = []
         if kw.get('work'): tmp = kw['work']
         out = tmp / 'G.app'; shutil.rmtree(out, ignore_errors=True)
         def fake_run(cmd, log, env=None, cwd=None, on_line=None):
@@ -310,13 +545,13 @@ class Script(unittest.TestCase):
                 return
             (out / 'Contents/MacOS').mkdir(parents=True); (out / 'Contents/MacOS/dmhost').write_bytes(b'x')
             on_line('step: launcher and host (clones of the game files)')
+            on_line(f'app: {out}')
         g = types.SimpleNamespace(table=lambda n: tmp / 'profile' / n, get={'app_category': 'public.app-category.games', 'romfs_marker': 'a/b'}.get)
         g.id = '01008CF01BAAC000'
         with mock.patch.object(nsp, 'run', fake_run), mock.patch.object(nsp, 'PROG', mock.Mock()), mock.patch.object(nsp, 'chain_of', lambda repo: '0123456789ab'), \
              mock.patch.object(nsp, 'mac_gpu_arch', lambda *a: 'applegpu_g14s'), mock.patch.object(nsp, 'say', self.said.append):
-            clones = nsp.mac_app(sys.executable, REPO, out, 0x01008CF01BAAC000, tmp, tmp / 'ex', tmp / 'patched', tmp / 'dmhost', tmp / 'font', tmp / 'shader/cache' if gfx == 'nvn' else tmp / 'shader',
-                                 kw.get('aoc'), gfx, 'en-US', g, 'Game', '1.0.0', 'org.nxnative.01008cf01baac000', None, kw.get('sign', ''), {}, tmp, gpus=kw.get('gpus', ''))
-        self.assertTrue(clones)
+            nsp.mac_app(sys.executable, REPO, out, 0x01008CF01BAAC000, tmp, tmp / 'ex', tmp / 'patched', tmp / 'dmhost', tmp / 'font', tmp / 'shader/cache' if gfx == 'nvn' else tmp / 'shader',
+                        kw.get('aoc'), gfx, 'en-US', g, 'Game', '1.0.0', 'org.nxnative.01008cf01baac000', None, kw.get('sign', ''), {}, tmp, gpus=kw.get('gpus', ''))
         self.pso_cmd = ran[0] if len(ran) > 1 else None   # pso_offline.py, before the packer
         c = ran[-1]; return {c[i]: c[i + 1] for i in range(3, len(c) - 1) if c[i].startswith('--')}, tmp
 
@@ -335,7 +570,7 @@ class Script(unittest.TestCase):
         self.assertEqual((o['--msl'], o['--aoc'], o['--sign']), (str(tmp / 'shader/vk'), '/x/aoc', 'Developer ID Application: X (T)'))
 
     def test_pso_archive(self):   # an NVN game with shader pairs: its pipeline archive for this Mac's GPU, made by pso_offline.py, goes into the app
-        tmp = Path(tempfile.mkdtemp()); self.addCleanup(shutil.rmtree, tmp)
+        tmp = Path(tempfile.mkdtemp(dir=TEST_ROOT)); self.addCleanup(shutil.rmtree, tmp)
         (tmp / 'profile').mkdir(); (tmp / 'profile/shader_pairs.tsv').write_text('v\tf\n')
         o, _ = self.pack_cmd('nvn', work=tmp)
         self.assertEqual(o['--pso'], str(tmp / 'pso/mac'))

docs/agents/symptoms.md

@@ -5,6 +5,7 @@ One line per symptom in the [subsystem references](../reference/): what you see
 ## Packing
 
 - the first extraction stops with `nx_extract exit code 131` and `.NET location: Not found`, even though `dotnet` is on `PATH`. The shader translator can fail the same way. → [Homebrew .NET is on PATH but apphosts cannot find the runtime](../reference/packing.md#homebrew-net-is-on-path-but-apphosts-cannot-find-the-runtime)
+- packing succeeds, but `unzip -t` or Python's `ZipFile.testzip()` reports a bad CRC for a large deflated file. → [A streamed file changes size and the IPA fails its CRC check](../reference/packing.md#a-streamed-file-changes-size-and-the-ipa-fails-its-crc-check)
 - `decompress NSZ` stops immediately with `nsz: error: unrecognized arguments: --keys …`. → [NSZ decompression rejects `--keys`](../reference/packing.md#nsz-decompression-rejects---keys)
 
 ## Rewriter

docs/development.md

@@ -128,10 +128,14 @@ What the script does:
 5. It splits out the shaders and checks them against the profile's tables (a Vulkan profile: it translates the RomFS's SPIR-V and the capture with `vk_shaders.py` instead; an OpenGL one: the GLSL sources and the capture with `gl_shaders.py`).
 6. It builds the iOS shader libraries, with the Mac's own Metal compiler and spirv-cross when they differ from those in `dmsh_chain.lock` (`dm/gfx/tools/own_compilers.py`; one line in the output names them). If a registered profile's `dmsh_chain.h` does not hold the version the source tree gives (a chain change left it behind), the script writes it again before any host is built, so it shows as changed in the checkout (`dmsh_build.py` would stop otherwise). Then the script makes the profile's `c1_full.tsv` and `cb_extent.tsv` from this build's Mac outputs when they are missing or name another chain. These two tables are not committed and git ignores them; `build_ipa.sh` stops without a current `c1_full.tsv`. Each line in the output names the file.
    It then compares the build with the game's previous one, the newest other `b-*/` in `<id>_build/shader/cache/` (`dm/gfx/tools/shader_regressions.py`, which reads the two `_b_build.json` reports). The translation chain is one for every game, so a translator change is tested only on the games its author owns; shaders that translated before and fail now are listed in `logs/shader_regressions.log` and stop the build ([GitHub issue #272](https://github.com/seedds/nxnative/issues/272)). Report them; `--allow-shader-regressions` (`ALLOW_SHADER_REGRESSIONS = True` in Spyder) packs the app without them.
-7. It calls `build_ipa.sh`. Xcode builds the app in the game's own build folder (`<id>_build/ipa/build/ios/pkg/dd`: `build_ipa.sh` sets `SYMROOT` and `OBJROOT`, which a custom build location in Xcode's settings does not override), so a repeat build compiles only what changed and two games built at once never write the same app. Then `dm/ios/tools/pack_ipa.py` packs it with the shader libraries, the add-on content and the RomFS straight from where they are. It reads and deflates files on all cores and writes at about the T7's write speed (300 MB/s): Breath of the Wild's 16 GB IPA takes 57 s, Info-ZIP's copy and zip took 2 min 50 s.
+7. It calls `build_ipa.sh`. Xcode builds the app in the game's own build folder (`<id>_build/ipa/build/ios/pkg/dd`: `build_ipa.sh` sets `SYMROOT` and `OBJROOT`, which a custom build location in Xcode's settings does not override), so a repeat build compiles only what changed and two games built at once never write the same app. Then `dm/ios/tools/pack_ipa.py` packs it with the shader libraries, the add-on content and the RomFS straight from where they are. It reads and deflates files on all cores and writes at about the T7's write speed (300 MB/s): Breath of the Wild's 16 GB IPA takes 57 s, Info-ZIP's copy and zip took 2 min 50 s. A repeat pack reuses the previous IPA through `<ipa>.idx`, which records each packed file's size, times and inode: the unchanged leading entries stay in place in an APFS clone of the old IPA, and after the first changed entry, an unchanged file is copied compressed from it (both only when the source folder is on APFS; elsewhere every file is deflated again). `build_ipa.sh` passes the RomFS, the add-on content, the pipeline archive and the shader libraries before the app, so after a host fix only the app is written: Breath of the Wild repacks in 12-23 s instead of 105-163 s.
 
 A repeat build skips work whose inputs have not changed. The extraction is kept (`extract/`). The DLC, the rewrite, the shader split and the iOS (or Vulkan) shaders each write a key beside their output: a sha256 of their inputs (the game files, the tools, the profile, Xcode's and spirv-cross's versions; for the DLC, the DLC NSPs' names, sizes and times and nx_extract's sources). When the key matches, the stage line says `reused (inputs unchanged)` and the step does not run. Any change to a tool or the profile runs the step again; deleting the `.key` file forces it. The iOS shaders' inputs are the shader package's content and only the tools `dmsh_build.py` loads (the modules it imports, at any depth): a split that runs again because a reader for another engine changed (a pull that touches `shader_split.py` or `unityfs.py`) writes the same package, and the iOS shaders stay reused.
 
+Derived profiles keep generated files while `<id>.derived.key` matches and every source-generated data row remains intact, including repeated rows. Provisional and derived shader keys include their profile files and each RomFS or capture file's path, size and modification time. Re-extraction or a new capture timestamp therefore runs the split or Vulkan/GL translation again, even with identical bytes. Registered RomFS keys use paths and sizes; NVN split tables are still checked against the profile.
+
+For `mac-app`, `pack_app.py` owns the sidecar `<game>.app.key`. It checks packed inputs, options, compiler, SDK and app files; unchanged bundles skip cloning, launcher compilation and signing. The script uses the packer's reuse message. `pack_ipa.py --no-reuse` forces a full pack; a missing or stale `.idx` also causes a full pack.
+
 The rewrite also collects NRO modules registered by the game's NRR files, including modules inside supported archives (`nro_collect.py`). The host maps the rewritten modules at startup and activates them on `nn::ro::LoadModule`, with symbol precedence following activation order; unloading and reloading restores their initial data and bss. `DM_NRO=0` disables this loading. Epic Online Services modules are identified by their exports and refused, preserving offline play. Collection reuse includes the archive reader and symbol parser; the rewrite key includes `nro.tsv`, even when it lists only refused modules. On iOS the rewritten NROs are packed into three sections of the executable (`dm/ios/tools/gen_embed.py`: a Mach-O file has at most 255 sections, and Super Smash Bros. Ultimate's 98 NROs would take three each) with a table of their pieces, from which the host maps them at start (`loader.c` `nro_pieces`). The synthetic loader test (`make -C dm/host/tests GAME=<id> nro`) covers load order, rebinding, weak symbols, init/fini, data reset and refusal without game files.
 
 While it runs, the console shows one line per stage: building nx_extract, extracting each version, the DLC, the rewrite, the run-time shader capture (only for such profiles), the shader split, the iOS shaders, the app build and packing the IPA. Each line shows `[k/n]`, the stage and its current step. Where the work can be counted, it adds a bar with done/total, percent, elapsed time and ETA. Counted work is bytes extracted, modules rewritten, shaders (cache check, translation, compile), files compiled by Xcode, and bytes packed into the IPA. Elsewhere a spinner shows the elapsed time. A total marked `~` is an estimate. A version's first extraction is estimated from the NSPs' size, and later extractions use the previous run's exact size. The IPA is estimated from the previous IPA, or from the files' size before compression. In a terminal or in Spyder the line is redrawn in place. When the stage ends, it becomes a `✓` line with the elapsed time and a summary:

docs/reference/packing.md

@@ -9,6 +9,15 @@
 - **Fix:** `scripts/build_ipa_from_nsp.py` `configure_dotnet` derives the root from the `Microsoft.NETCore.App` entry in `dotnet --list-runtimes`, using the build's environment, before any extraction or survey. It exports `DOTNET_ROOT` both to that environment and to `os.environ`, for tools launched with an explicit environment and those that inherit it. Caller-supplied roots, including architecture-specific overrides, retain precedence. A failed or malformed probe leaves the environment unchanged rather than guessing the wrapper's directory.
 - **Found on:** [Balatro](../../dm/games/0100CD801CE5E000/README.md), first extraction with Homebrew .NET ([PR #284](https://github.com/seedds/nxnative/pull/284)). Applies to every game's build on that installation setup.
 
+## IPA archives
+
+### A streamed file changes size and the IPA fails its CRC check
+
+- **Symptom:** packing succeeds, but `unzip -t` or Python's `ZipFile.testzip()` reports a bad CRC for a large deflated file.
+- **Cause:** `pack_ipa.py` wrote the uncompressed size collected before reading. Its streaming check compared the output size only for stored files, so a compressible file that grew or shrank still published an archive with the old size.
+- **Fix:** count uncompressed bytes as they are read and compare them with the listed size before completing either streamed method. A mismatch stops the pack and removes the partial archive, preserving the previous IPA and its reuse index. Regression tests cover growth and shrinkage with both deflate and store.
+- **Found on:** synthetic IPA inputs while reviewing [PR #279](https://github.com/seedds/nxnative/pull/279); no game files required.
+
 ## Compressed inputs
 
 ### NSZ decompression rejects `--keys`

scripts/build_ipa_from_nsp.py

@@ -704,15 +704,18 @@ def write_profile(pdir, app, f):
 DERIVED_SKIP = {'game.tsv', 'shader_map.tsv', 'shader_facts.tsv', 'shader_pairs.tsv', 'pso_pairs.tsv', 'c1_full.tsv', 'cb_extent.tsv', 'vk_capture.tsv', 'gl_capture.tsv', 'patches.py'}   # made again from the packed version's own files (patches.py: the rewriter's patches of the registered builds)
 
 
+def derived_key(profile_dir):
+    return profile_dir.with_name(profile_dir.name + '.derived.key')
+
+
 def derive_profile(g, pdir, ver, d, mods, app=None):
     """The game's profile for a version it was not brought up on (the folder has no update, another update, files of other builds, or a registered build under another title ID: app),
     in pdir (outside the repository, where a provisional profile goes): its keys as they are, with this version's build row, package and shader versions, and app's title ID and
     main.npdm's program ID when app is not the profile's title. The shader tables, the chain header and the run-time capture list are
     left out: the build makes them again from this version's own files, as for a provisional profile. The hand-written tables (the prewarm list, shader_facts_manual.tsv) are copied:
-    they name shaders by content key. Returns the version's display version"""
-    rmtree(pdir); pdir.mkdir(parents=True)
-    for p in g.dir.iterdir():
-        if p.is_file() and p.name not in DERIVED_SKIP: shutil.copyfile(p, pdir / p.name)
+    they name shaders by content key. A folder derived from the same inputs (the files copied and the game.tsv written, kept in <pdir>.derived.key) is kept as it is, with what later
+    stages wrote into it (mem_used, the heap check, the split's tables), so their own reuse holds. Returns the version's display version"""
+    src = sorted(p for p in g.dir.iterdir() if p.is_file() and p.name not in DERIVED_SKIP)
     T, disp = '\t', nacp.version(d / 'control.nacp') or '?'
     row = T.join(['build', ver, disp] + [f'{m}={build_id(d / "ExeFS" / m)}' for m in mods if (d / 'ExeFS' / m).exists()])
     out, put = [f'# Derived from game profile {g.dir} by scripts/build_ipa_from_nsp.py for {ver} ({disp}), a version that profile was not brought up on:',
@@ -727,7 +730,21 @@ def derive_profile(g, pdir, ver, d, mods, app=None):
             if not put: out += [row, f'package{T}{ver}', f'shader_versions{T}{ver}']; put = True
             continue
         out.append(line)
-    (pdir / 'game.tsv').write_text('\n'.join(out) + '\n', encoding='utf-8')
+    text, kf = '\n'.join(out) + '\n', derived_key(pdir)   # beside the profile, not in it: no tool reads it as a table
+    key = fingerprint(('files', src), text)
+    if (pdir / 'game.tsv').is_file() and reused(kf, key) is not None:
+        def rows(text):   # the same whitespace and comment rules as game.Game
+            return [tuple(c.strip() for c in l.split(T)) for l in text.splitlines() if l.strip() and not l.lstrip().startswith('#')]
+        expected = collections.Counter(rows(text)); keys = {r[0] for r in expected}
+        try: kept = collections.Counter(r for r in rows((pdir / 'game.tsv').read_text(encoding='utf-8')) if r[0] in keys)
+        except (OSError, UnicodeError): kept = None
+        if kept == expected: return disp
+    rmtree(kf); pdir.mkdir(parents=True, exist_ok=True)
+    atomic_text(pdir / 'game.tsv', text)
+    for p in pdir.iterdir():
+        if p.name != 'game.tsv': rmtree(p)
+    for p in src: shutil.copyfile(p, pdir / p.name)
+    kf.write_text(f'{key}\n')
     return disp
 
 
@@ -1183,6 +1200,14 @@ def capture_shaders(repo, d, patched, work, env, logs, g, api):
 
 
 # ---- A provisional profile after the rewrite: mem_used and the heap check ----
+def atomic_text(path, text):
+    f = tempfile.NamedTemporaryFile(mode='w', encoding='utf-8', dir=path.parent, prefix=f'.{path.name}.', delete=False)
+    try:
+        with f: f.write(text)
+        os.replace(f.name, path)
+    finally: Path(f.name).unlink(missing_ok=True)
+
+
 def set_keys(g, changes, section):
     """Write keys into a provisional game.tsv, each under its reason, in a section of their own at the end; earlier rows of the same keys (and the reason line right above one) go.
     Returns the profile reloaded"""
@@ -1193,7 +1218,7 @@ def set_keys(g, changes, section):
             continue
         out.append(line)
     out += ['', f'# ---- {section} ----'] + [x for k, (v, why) in changes.items() for x in ([f'# {why}'] if why else []) + [f'{k}\t{v}']]
-    p.write_text('\n'.join(out) + '\n', encoding='utf-8')
+    atomic_text(p, '\n'.join(out) + '\n')
     return type(g)(g.dir)
 
 
@@ -1254,13 +1279,21 @@ def chain_tables(py, repo, sh, env, logs, g, chain):
     return True
 
 
-def refine_after_rewrite(g, ex_dir, patched):
-    """Keys of a provisional profile that need the rewritten modules: mem_used (dm/games/profile_guess.py), for a game without virtual address memory that sizes its heap as pool − used memory"""
-    if any(k == 'mem_used' for k, _ in g.rows) or pg.npdm(ex_dir / 'ExeFS/main.npdm')[0]: return g
-    used = pg.mem_used(patched, g.words('modules'))
-    if not used: return g
-    say(f'mem_used {used:#x}: the rewritten modules are larger than the 64 MB of used memory the game takes off the pool for its heap')
-    return set_keys(g, {'mem_used': (f'{used:#x}', 'the rewritten modules are larger than the default 64 MB of used memory (Age of Calamity: 66.6 MB → 72 MB)')}, 'After the rewrite')
+def refine_after_rewrite(g, ex_dir, patched, reg=None):
+    """Refresh computed mem_used for the current modules; explicit profile values always win."""
+    for profile in (reg, g) if reg else (g,):
+        section, reason = '', ''
+        for line in (profile.dir / 'game.tsv').read_text(encoding='utf-8').splitlines():
+            if line.startswith('# ----'): section = line
+            if line.split('\t', 1)[0].strip() == 'mem_used':
+                computed = section == '# ---- After the rewrite ----' and reason.startswith(('# used memory for the rewritten modules', '# used memory for the current rewritten modules', '# the rewritten modules are larger than the default 64 MB'))
+                if not computed: return g
+            reason = line
+    if pg.npdm(ex_dir / 'ExeFS/main.npdm')[0]: return g
+    used = pg.mem_used(patched, g.words('modules')) or 0x4000000
+    if int(g.get('mem_used', '0x4000000'), 0) == used: return g
+    say(f'mem_used {used:#x}: from the current rewritten modules (64 MB when the default covers them)')
+    return set_keys(g, {'mem_used': (f'{used:#x}', 'used memory for the current rewritten modules, at least the default 64 MB')}, 'After the rewrite')
 
 
 def heap_checked(g):   # the heap check's stamp in the profile ("# heap check: ..."): its result is kept, later runs skip it
@@ -1295,7 +1328,7 @@ def heap_check(repo, d, patched, work, env, logs, g):
     how = ('the heap request did not fit' if 'SetHeapSize OOM' in text else f'heap {m.group(1)}' if m else 'no heap set' if '>>> nnMain' in text
            else f'the host stopped before nnMain (exit {rc})' if rc is not None else 'no nnMain within the time limit')
     if sig: g = set_keys(g, sig, 'Heap check on the Mac')
-    p = g.dir / 'game.tsv'; p.write_text(p.read_text(encoding='utf-8') + f'\n# heap check: {time.strftime("%Y-%m-%d")}, {how}' + (f'; set {", ".join(sig)}' if sig else '') + '\n', encoding='utf-8')
+    p = g.dir / 'game.tsv'; atomic_text(p, p.read_text(encoding='utf-8') + f'\n# heap check: {time.strftime("%Y-%m-%d")}, {how}' + (f'; set {", ".join(sig)}' if sig else '') + '\n')
     if not m and '>>> nnMain' not in text and 'SetHeapSize OOM' not in text:   # stopped before any heap request: worth a look before the IPA is tried
         PROG.say(f'[RED] heap check: {how}; ---- last lines of {log} ----\n' + ''.join(collections.deque(open(log, errors='replace'), maxlen=20)).rstrip('\n'))
     PROG.end(f'{how} in {hms(time.time() - t0)}' + (f'; set {", ".join(sig)}' if sig else ''))
@@ -1308,19 +1341,22 @@ def data_rows(b):   # a table's data rows (without # comment lines: the header c
 
 def split_shaders(py, repo, roms, mains, pkg, env, logs, g, provisional=False, oodle=None):
     """The shader split into pkg, checked against the profile's tables. The package is reused when the tools, the profile and each version's main, RomFS (paths and sizes) and run-time capture
-    are as at the last successful split. Returns the package's content key, the iOS shaders' input (None for a provisional profile or damaged files, which never reuse them): a split that runs
+    are as at the last successful split. Returns the package's content key, the shader build's input (None for damaged files, which never reuse them): a split that runs
     again because a reader of another engine changed writes the same package, and the iOS shaders stay reused.
-    oodle: for a game with Trinity packs (Pokémon Legends: Z-A; profile_guess.py shader_sources), a function that builds the Mac host and returns the split's DM_OODLE_HOST / DM_OODLE_MODULES; called only when the split runs"""
+    oodle: for decoder-backed packs, builds or reuses the Mac host and returns DM_OODLE_HOST / DM_OODLE_MODULES; their current files are split inputs too"""
     names = ('shader_map.tsv', 'shader_facts.tsv', 'shader_pairs.tsv')
     PROG.stage('shader split', step='checking the inputs')   # then a spinner: shader_split.py writes the package and the tables only at the end
     kf = pkg.with_name(pkg.name + '.key')
-    key = None if provisional or ALLOW_DAMAGED else fingerprint(('files', tool_files(repo)), ('files', profile_files(g)),
-        *[x for r, m in zip(roms, mains) for x in (('files', [m]), ('tree', r), ('files', sorted((r.parent / 'glslc_capture').glob('*.glslc'))),
-                                                   ('files', sorted((r.parent / 'ue_shaders').glob('*.ushaderbytecode'))))])
+    inputs = [('files', tool_files(repo)), bool(provisional)] + [x for r, m in zip(roms, mains) for x in
+        (('files', [m]), ('stat' if provisional else 'tree', r),
+         ('stat', r.parent / 'glslc_capture') if provisional else ('files', sorted((r.parent / 'glslc_capture').glob('*.glslc'))),
+         ('stat', r.parent / 'ue_shaders') if provisional else ('files', sorted((r.parent / 'ue_shaders').glob('*.ushaderbytecode'))))]
+    if oodle:   # a host or rewriter fix can change what the game's decoder produces
+        PROG.set(step='building the Mac host for the game\'s Oodle decoder'); env = dict(env, **oodle())
+        inputs += [('files', [env['DM_OODLE_HOST']]), ('stat', Path(env['DM_OODLE_MODULES']))]
+    key = None if ALLOW_DAMAGED else fingerprint(*inputs, ('files', profile_files(g)))
     if key and pkg.is_dir() and (note := reused(kf, key)) is not None:
         PROG.end(f'{note}, reused (inputs unchanged)'); return package_key(pkg)
-    if oodle:   # the packs are decoded by the game's own Oodle in the Mac host (dm/host/hle_oodle.c)
-        PROG.set(step='building the Mac host for the game\'s Oodle decoder'); env = dict(env, **oodle())
     PROG.set(step='splitting')
     tmp = pkg.with_name(pkg.name + '.partial'); rmtree(tmp); (tmp / 'tables').mkdir(parents=True)
     run([py, repo / 'dm/gfx/tools/shader_split.py', ','.join(map(str, roms)), ','.join(map(str, mains)), tmp / 'pkg'] + [tmp / 'tables' / n for n in names + ('pso_pairs.tsv',)],
@@ -1340,7 +1376,9 @@ def split_shaders(py, repo, roms, mains, pkg, env, logs, g, provisional=False, o
         if diff: raise BuildError(f'the split shader tables differ from game profile {g.id}\'s: {", ".join(diff)} (game files not the versions the profile registers?)')
     rmtree(kf); swap_in(tmp / 'pkg', pkg); rmtree(tmp)
     note = f'{int(m.group(1)):,} shaders' if m else ''
-    if key: kf.write_text(f'{key}\n{note}\n')
+    if key:   # an own profile's tables are outputs too; the next split checks the profile after they were written
+        key = fingerprint(*inputs, ('files', profile_files(g)))
+        kf.write_text(f'{key}\n{note}\n')
     PROG.end(note)
     return key and package_key(pkg)
 
@@ -1484,20 +1522,20 @@ def layer_shaders(py, repo, api, name, roots, sh, key, logs, env, mac):
     return out
 
 
-def build_vk_shaders(py, repo, roots, sh, env, logs, g, reuse=True, mac=False):
+def build_vk_shaders(py, repo, roots, sh, env, logs, g, reuse=True, mac=False, provisional=False):
     """A Vulkan game's SPIR-V shaders (roots: the RomFS, and a run-time capture folder) → one Metal library each for the Mac (sh/vk/) and the iPhone (sh/ios/vk/, what build_ipa.sh takes from MSL_IOS;
     not with mac), replacing the NVN split and chain build (layer_shaders). Reused when the roots (paths and sizes), the tools, the profile and the compilers are as at the last successful run (reuse False: never)"""
     PROG.stage('Vulkan shaders', step='checking the inputs')
-    key = fingerprint(*[('tree', r) for r in roots], ('files', tool_files(repo)), ('files', profile_files(g)), *toolchain(), env=env) if reuse else None
+    key = fingerprint(*[('stat' if provisional else 'tree', r) for r in roots], ('files', tool_files(repo)), ('files', profile_files(g)), *toolchain(), env=env) if reuse else None
     return layer_shaders(py, repo, 'vk', 'Vulkan', roots, sh, key, logs, env, mac)
 
 
-def build_gl_shaders(py, repo, roots, sh, env, logs, g, reuse=True, mac=False):
+def build_gl_shaders(py, repo, roots, sh, env, logs, g, reuse=True, mac=False, provisional=False):
     """An OpenGL game's GLSL sources (roots: the RomFS, main, a run-time capture folder) → one Metal library each for the Mac (sh/gl/) and the iPhone (sh/ios/gl/, what build_ipa.sh takes from MSL_IOS;
     not with mac), replacing the NVN split and chain build, as build_vk_shaders does for a Vulkan game; glslang (glslangValidator) compiles the GLSL to SPIR-V for spirv-cross"""
     PROG.stage('GL shaders', step='checking the inputs')
     if not shutil.which('glslangValidator', path=env['PATH']): raise BuildError('glslangValidator not found (brew install glslang): it compiles an OpenGL game\'s GLSL shaders')
-    key = fingerprint(*[('tree', r) if Path(r).is_dir() else ('files', [r]) for r in roots], ('files', tool_files(repo)), ('files', profile_files(g)), *toolchain(),
+    key = fingerprint(*[('stat' if provisional else 'tree', r) if Path(r).is_dir() else ('files', [r]) for r in roots], ('files', tool_files(repo)), ('files', profile_files(g)), *toolchain(),
                       ('cmd', ['glslangValidator', '--version']), env=env) if reuse else None
     return layer_shaders(py, repo, 'gl', 'GL', roots, sh, key, logs, env, mac)
 
@@ -1511,6 +1549,7 @@ class IpaWatch:
     def __init__(self, repo, w, out_ipa):
         self.ios, self.o, self.part = repo / 'dm/ios', w / 'build/ios/pkg', out_ipa.with_name(out_ipa.name + '.partial')
         self.last = out_ipa.stat().st_size if out_ipa.is_file() else None   # the last IPA's size: the expected size of the new archive
+        self.kept = 0
         self.state, self.t, self.off, self.compiled, self.what, self.total, self.most, self.lock = None, 0, 0, 0, '', None, 0, threading.Lock()
 
     def line(self, line):   # a line of build_ipa.sh's output
@@ -1527,7 +1566,9 @@ class IpaWatch:
                 PROG.end(f'{self.compiled:,} file{"" if self.compiled == 1 else "s"} compiled')
                 self.state, self.total, self.most = 'list', None, 0; PROG.stage('IPA pack', poll=self.poll, step='listing the files')
             elif self.state == 'list' and (m := re.match(r'pack: (\d+) entries, (\d+) bytes', s)):   # the files' bytes: the archive's size when nothing deflates, else the last IPA's size is closer
-                self.state, self.total = 'pack', self.last or int(m.group(2)); PROG.set(step='packing', unit='B', note=f'{int(m.group(1)):,} entries')
+                self.state, self.total = 'keep', self.last or int(m.group(2)); PROG.set(step='packing', unit='B', note=f'{int(m.group(1)):,} entries')
+            elif self.state == 'keep' and (m := re.match(r'pack: keeping (\d+) bytes', s)):
+                self.kept = int(m.group(1)); self.state = 'pack'; self.most = 0; self.total = max(0, self.total - self.kept)
 
     def poll(self):   # from the poll thread
         with self.lock: return self._poll()
@@ -1546,7 +1587,7 @@ class IpaWatch:
                 self.what = self.XCODE.get(w, self.what)
             r = {'done': self.compiled, 'note': self.what}
         elif state == 'pack':
-            try: self.most = max(self.most, self.part.stat().st_size)   # never backwards: the archive is renamed at the end
+            try: self.most = max(self.most, self.part.stat().st_size - self.kept)   # never backwards: the archive is renamed at the end
             except OSError: pass
             r = {'done': self.most, 'total': self.total, 'est': True}
         else: return None
@@ -1667,13 +1708,14 @@ def mac_app(py, repo, out, app, work, game, patched, host, font, msl, aoc, gfx,
                    ('--pso', pso), ('--pso-pairs', split_pairs if gfx == 'nvn' and not g.table('pso_pairs.tsv').is_file() and split_pairs.is_file() else None),
                    ('--env-file', Path(os.path.expanduser(env_file)).resolve() if env_file else None)):
         if v: cmd += [opt, v]
-    steps = []
+    unchanged = False
     def line(ln):
-        if ln.startswith('step: '): steps.append(ln[6:].strip()); PROG.set(step=steps[-1])
+        nonlocal unchanged
+        if ln.startswith('step: '): PROG.set(step=ln[6:].strip())
+        if ln.startswith('app: ') and ln.endswith(', reused (inputs unchanged)'): unchanged = True
     run(cmd, logs / 'mac_app.log', env=env, on_line=line)
     if not (out / 'Contents/MacOS/dmhost').is_file(): raise BuildError(f'no app at {out}')
-    PROG.end(f'{out.name}, {amount(tree_size(out)[1], "B")}')
-    return any('(clones of the game files)' in s for s in steps)   # whether the game files are APFS clones of the build's (pack_app.py's first step says)
+    PROG.end(f'{out.name}, ' + ('reused (inputs unchanged)' if unchanged else amount(tree_size(out)[1], 'B')))
 
 
 def find_repo(here):   # this file lives in the repository's scripts/; walk up to the level that has dm/tools/nx_extract/Program.cs
@@ -1784,6 +1826,7 @@ def main(args=None):
         if missing: raise BuildError(f'the current Python ({sys.executable}) is missing modules: {", ".join(missing)}')
         if not shutil.which('dotnet', path=env['PATH']): raise BuildError('dotnet not found (.NET SDK 10)')
         pdir = Path(os.path.expanduser(args.profile_out or PROFILE_OUT)).resolve() if (args.profile_out or PROFILE_OUT) else nsp_dir / f'{app:016X}_build' / 'profile'
+        if not (args.survey_only or SURVEY_ONLY): rmtree(derived_key(pdir / f'{app:016X}'))
         if (pdir / f'{app:016X}' / 'game.tsv').exists() and not (args.survey_only or SURVEY_ONLY):
             say(f'no game profile for {app:016X}: reusing the provisional profile from the last run (delete it to survey again)')
         else:
@@ -1906,7 +1949,7 @@ def main(args=None):
     chain_header(py, repo, env, logs)   # before any host is built
     if provisional or derived:   # the keys that need the rewritten modules, then (provisional) the heap check (both only change keys the host is built with, not the rewrite or the shader steps)
         provisional_prep(py, repo, env, logs, g)
-        g = refine_after_rewrite(g, game, patched)
+        g = refine_after_rewrite(g, game, patched, reg if derived else None)
         if provisional and heapck and pg.heap_check_needed(game / 'ExeFS'):
             if apple_silicon(): PROG.add(1); g = heap_check(repo, game, patched, work, env, logs, g)
             else: say('no heap check: it boots the game in the Mac host, which needs a Mac with Apple silicon; the profile keeps its guessed heap keys')
@@ -1915,10 +1958,10 @@ def main(args=None):
     sh = work / 'shader'
     if gfx == 'vulkan':   # the packed version's merged RomFS holds the SPIR-V files the game loads; a capture holds those it generates
         cap = capture_shaders(repo, game, patched, work, env, logs, g, 'vk') if vkcap else None
-        ios = build_vk_shaders(py, repo, [game / 'RomFS'] + ([cap] if cap else []), sh, env, logs, g, reuse=not (provisional or derived or ALLOW_DAMAGED), mac=mac)
+        ios = build_vk_shaders(py, repo, [game / 'RomFS'] + ([cap] if cap else []), sh, env, logs, g, reuse=not ALLOW_DAMAGED, mac=mac, provisional=bool(provisional or derived))
     elif gfx == 'gl':   # the GLSL sources in the packed version's archives and in its main (the game compiles them at run time; the host finds each by the hash of its text), and a capture of those it puts together
         cap = capture_shaders(repo, game, patched, work, env, logs, g, 'gl') if glcap else None
-        ios = build_gl_shaders(py, repo, [game / 'RomFS', game / 'ExeFS/main'] + ([cap] if cap else []), sh, env, logs, g, reuse=not (provisional or derived or ALLOW_DAMAGED), mac=mac)
+        ios = build_gl_shaders(py, repo, [game / 'RomFS', game / 'ExeFS/main'] + ([cap] if cap else []), sh, env, logs, g, reuse=not ALLOW_DAMAGED, mac=mac, provisional=bool(provisional or derived))
     else:
         # Trinity packs and legacy Unreal paks: the split decodes them with the packed version's own decoder (Oodle, LZ5), in the Mac host built for the profile
         oodle = (lambda: {'DM_OODLE_HOST': str(build_mac_host(repo, work / 'tools/dmhost', env, logs, g)), 'DM_OODLE_MODULES': str(patched)}) if src['oodle_packs'][0] or src['ue_legacy'][0] else None
@@ -1929,10 +1972,10 @@ def main(args=None):
     if mac_pack:   # a Mac app: the Mac host, then the app packed from it and the build's files
         host, font = mac_host(py, repo, work, env, logs, g)
         if not icon: say('[RED] no game icon in the extraction (an older extraction did not save it: delete extract/ and run again); the app\'s icon will be blank')
-        clones = mac_app(py, repo, out_ipa, app, work, game, patched, host, font, ios, aoc, gfx, lang, g, app_name, app_version, bundle_id, icon, args.mac_sign or MAC_SIGN, env, logs, args.env_file or DM_ENV_FILE,
-                         (args.gpu or PSO_GPUS).strip())
+        mac_app(py, repo, out_ipa, app, work, game, patched, host, font, ios, aoc, gfx, lang, g, app_name, app_version, bundle_id, icon, args.mac_sign or MAC_SIGN, env, logs, args.env_file or DM_ENV_FILE,
+                (args.gpu or PSO_GPUS).strip())
         rmtree(work / 'tmp')
-        say(f'done: {out_ipa} ({tree_size(out_ipa)[1] / 2**30:.1f} GB{", its game files APFS clones of the extracted ones, which take no space of their own" if clones else ""}); '
+        say(f'done: {out_ipa} ({tree_size(out_ipa)[1] / 2**30:.1f} GB); '
             f'open it from Finder. Its save is in ~/Library/Application Support/nxnative/{app:016X}/, its log in ~/Library/Logs/nxnative/{app:016X}.log')
         report_profile_status(g, reg, pkg_ver, derived, provisional, 'Mac app')
         return