seeddsandmarco
ff44c59e12
Read both shader pair tables, including the generated BAHS fallback, for padding, build order and shader cache dependencies. Include the optional profile table in the input stamp and the generated table in step reuse; regenerate a missing fallback before reusing the shader split. Add native Metal and cache/order regressions, update the shared chain, and document full validation on TotK and Odyssey. All 301 affected pairs compile for g13s and g16s, both full Mac shader/host builds pass, all 57,039 pair interfaces match, and TotK's 24,153-base archive succeeds. The separate Odyssey Max Unstruct ID compiler limit remains open. Fixes #283 Refs #276, #298 Co-authored-by: marco <6410852+marcolaux@users.noreply.github.com>
402 lines
25 KiB
Python
402 lines
25 KiB
Python
"""NSP script reuse across derived and provisional builds. Real fingerprints and mock tool runs; no game files, keys or Xcode needed."""
|
|
import importlib.util
|
|
import os
|
|
from pathlib import Path
|
|
import shutil
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from unittest import mock
|
|
|
|
REPO = Path(__file__).resolve().parents[3]
|
|
sys.path.insert(0, str(REPO / 'dm/games')); sys.path.insert(0, str(REPO / 'dm/tools'))
|
|
import game, nacp, profile_guess
|
|
spec = importlib.util.spec_from_file_location('build_ipa_from_nsp', REPO / 'scripts/build_ipa_from_nsp.py')
|
|
nsp = importlib.util.module_from_spec(spec); spec.loader.exec_module(nsp)
|
|
nsp.pg, nsp.nacp = profile_guess, nacp # main sets them
|
|
|
|
TID = 0x0100AAAA00000000
|
|
BID, BID2 = 'ab' * 16, 'cd' * 16
|
|
|
|
|
|
def nso(path, bid): # an NSO header: the build ID at +0x40
|
|
path.parent.mkdir(parents=True, exist_ok=True); path.write_bytes(b'NSO0' + bytes(0x3c) + bytes.fromhex(bid) + bytes(0x50))
|
|
|
|
|
|
def control(path, disp): # control.nacp with the display version at 0x3060
|
|
path.parent.mkdir(parents=True, exist_ok=True); path.write_bytes(bytes(0x3060) + disp.encode().ljust(0x10, b'\0') + bytes(0xa0))
|
|
|
|
|
|
class Base(unittest.TestCase):
|
|
def setUp(self):
|
|
self.tmp = Path(tempfile.mkdtemp()).resolve(); self.addCleanup(shutil.rmtree, self.tmp)
|
|
self.src = self.tmp / 'games' / f'{TID:016X}'; self.src.mkdir(parents=True)
|
|
(self.src / 'game.tsv').write_text('\n'.join([f'title_id\t{TID:016X}', f'program_id\t{TID:016X}', 'name\tTest Game', 'modules\tmain', 'app_category\tpublic.app-category.games',
|
|
'romfs_marker\ta/b', f'build\tv0\t1.0.0\tmain={BID}', 'package\tv0', 'shader_versions\tv0']) + '\n')
|
|
(self.src / 'shader_map.tsv').write_text('# registered\npkg.bin\t0\tvertex\tregistered\tx\n')
|
|
(self.src / 'shader_facts_manual.tsv').write_text('k\t{"manual": 1}\n')
|
|
(self.src / 'pso_manifest.tsv').write_text('S\tk\tvertex\n')
|
|
(self.src / 'dmsh_chain.h').write_text('#define DMSH_CHAIN "old"\n')
|
|
self.ex = self.tmp / 'extract/v65536'; nso(self.ex / 'ExeFS/main', BID2); control(self.ex / 'control.nacp', '1.1.0')
|
|
(self.ex / 'RomFS').mkdir(); (self.ex / 'RomFS/a.bin').write_bytes(b'romfs')
|
|
self.pdir = self.tmp / 'derived' / f'{TID:016X}'
|
|
for cm in (mock.patch.dict(os.environ, {'DM_GAMES_DIR': str(self.tmp / 'games')}), mock.patch.object(nsp, 'PROG', mock.Mock()), mock.patch.object(nsp, 'say', lambda m: None)):
|
|
cm.__enter__(); self.addCleanup(cm.__exit__, None, None, None)
|
|
|
|
def derive(self, ver='v65536', mods=('main',), app=None):
|
|
return nsp.derive_profile(game.Game(self.src), self.pdir, ver, self.ex, list(mods), app)
|
|
|
|
def redump(self, path): # identical bytes and size, but a fresh extraction or capture timestamp
|
|
before = path.stat()
|
|
os.utime(path, ns=(before.st_atime_ns, before.st_mtime_ns + 1_000_000_000))
|
|
|
|
|
|
class DeriveProfile(Base):
|
|
def later_stages(self): # what refine_after_rewrite and the shader split write into the derived profile
|
|
nsp.set_keys(game.Game(self.pdir), {'mem_used': ('0x4800000', 'rewritten modules')}, 'After the rewrite')
|
|
(self.pdir / 'shader_map.tsv').write_text('# split\npkg.bin\t0\tvertex\tderived\tx\n')
|
|
|
|
def assert_kept(self):
|
|
self.assertEqual(game.Game(self.pdir).get('mem_used'), '0x4800000')
|
|
self.assertIn('derived', (self.pdir / 'shader_map.tsv').read_text())
|
|
|
|
def assert_fresh(self):
|
|
self.assertNotIn('mem_used', [k for k, _ in game.Game(self.pdir).rows])
|
|
self.assertFalse((self.pdir / 'shader_map.tsv').exists())
|
|
|
|
def test_unchanged_inputs_keep_later_stages(self):
|
|
self.assertEqual(self.derive(), '1.1.0')
|
|
self.later_stages()
|
|
self.assertEqual(self.derive(), '1.1.0')
|
|
self.assert_kept()
|
|
self.assertEqual(game.Game(self.pdir).builds()['v65536']['main'], BID2)
|
|
self.assertEqual(self.derive(), '1.1.0'); self.assert_kept() # still kept on a third run
|
|
|
|
def test_changed_inputs_derive_again(self):
|
|
changes = {
|
|
'source game.tsv': lambda: (self.src / 'game.tsv').write_text((self.src / 'game.tsv').read_text() + 'pad_layout\tfull\n'),
|
|
'source manual table': lambda: (self.src / 'shader_facts_manual.tsv').write_text('k\t{"manual": 2}\n'),
|
|
'source chain header': lambda: (self.src / 'dmsh_chain.h').write_text('#define DMSH_CHAIN "new"\n'),
|
|
'new source file': lambda: (self.src / 'bf_want.tsv').write_text('# new\n'),
|
|
'main build': lambda: nso(self.ex / 'ExeFS/main', 'ef' * 16),
|
|
'display version': lambda: control(self.ex / 'control.nacp', '1.2.0'),
|
|
}
|
|
for what, change in changes.items():
|
|
with self.subTest(what):
|
|
self.derive(); self.later_stages(); self.derive(); self.assert_kept()
|
|
change(); self.derive(); self.assert_fresh()
|
|
for p in self.src.iterdir():
|
|
if p.name not in nsp.DERIVED_SKIP: self.assertEqual((self.pdir / p.name).read_bytes(), p.read_bytes())
|
|
self.assertEqual(game.Game(self.pdir).builds()['v65536'], {'display': '1.2.0', 'main': 'ef' * 16})
|
|
|
|
def test_changed_arguments_derive_again(self):
|
|
nso(self.ex / 'ExeFS/sdk', BID) # in the build row only when the modules name it
|
|
for what, kw in (('version', {'ver': 'v131072'}), ('modules', {'mods': ('main', 'sdk')}), ('title', {'app': 0x0100AAAA00FFF000})):
|
|
with self.subTest(what):
|
|
self.derive(); self.later_stages(); self.derive(); self.assert_kept()
|
|
self.derive(**kw); self.assert_fresh()
|
|
self.assertEqual(game.Game(self.pdir).title_id, 0x0100AAAA00FFF000)
|
|
|
|
def test_removed_source_file_goes(self):
|
|
self.derive(); self.later_stages(); self.derive(); self.assert_kept()
|
|
(self.src / 'pso_manifest.tsv').unlink(); self.derive()
|
|
self.assertFalse((self.pdir / 'pso_manifest.tsv').exists()); self.assert_fresh()
|
|
|
|
def test_edited_copy_is_kept_until_inputs_change(self): # the stored digest is of the inputs, not of the derived files
|
|
self.derive(); (self.pdir / 'pso_manifest.tsv').write_text('S\tk2\tpixel\n'); self.derive()
|
|
self.assertEqual((self.pdir / 'pso_manifest.tsv').read_text(), 'S\tk2\tpixel\n')
|
|
|
|
def test_missing_corrupt_and_duplicate_override_rows_derive_again(self):
|
|
for change in ('missing', 'corrupt', 'override', 'indented override', 'truncated'):
|
|
with self.subTest(change):
|
|
self.derive(); p = self.pdir / 'game.tsv'; original = p.read_text()
|
|
if change == 'missing': broken = original.replace('romfs_marker\ta/b\n', '')
|
|
elif change == 'corrupt': broken = original.replace('romfs_marker\ta/b', 'romfs_marker\tx/y')
|
|
elif change == 'override': broken = 'romfs_marker\tx/y\n' + original
|
|
elif change == 'indented override': broken = ' romfs_marker\tx/y\n' + original
|
|
else: broken = original[:original.index('romfs_marker')]
|
|
p.write_text(broken)
|
|
self.derive()
|
|
self.assertEqual(p.read_text(), original)
|
|
|
|
def test_repeated_source_rows_and_computed_additions_are_kept(self):
|
|
p = self.src / 'game.tsv'; p.write_text(p.read_text() + 'extra\tone\nextra\tone\n')
|
|
self.derive(); self.later_stages(); self.derive(); self.assert_kept()
|
|
p = self.pdir / 'game.tsv'; p.write_text(p.read_text().replace('extra\tone\n', '', 1))
|
|
self.derive()
|
|
self.assertEqual(game.Game(self.pdir).values('extra'), [['one'], ['one']])
|
|
|
|
def test_provisional_computed_mem_used_grows_shrinks_and_stays_stable(self):
|
|
self.derive(); g = game.Game(self.pdir)
|
|
# These reason comments were written by earlier versions of this script.
|
|
for reason in ('the rewritten modules are larger than the default 64 MB of used memory (Age of Calamity: 66.6 MB → 72 MB)',
|
|
'used memory for the rewritten modules (above the default 64 MB)',
|
|
'used memory for the current rewritten modules, at least the default 64 MB'):
|
|
g = nsp.set_keys(g, {'mem_used': ('0x4800000', reason)}, 'After the rewrite')
|
|
with mock.patch.object(nsp.pg, 'mem_used') as used:
|
|
for value in (0x5000000, None, 0x4800000):
|
|
used.return_value = value
|
|
g = nsp.refine_after_rewrite(g, self.ex, self.tmp / 'patched')
|
|
self.assertEqual(int(g.get('mem_used'), 0), value or 0x4000000)
|
|
before = g.table('game.tsv').read_bytes()
|
|
nsp.refine_after_rewrite(g, self.ex, self.tmp / 'patched')
|
|
self.assertEqual(g.table('game.tsv').read_bytes(), before)
|
|
|
|
def test_explicit_mem_used_without_registered_profile_is_kept(self):
|
|
self.derive(); p = self.pdir / 'game.tsv'; p.write_text(p.read_text() + 'mem_used\t0x5800000\n')
|
|
g = game.Game(self.pdir)
|
|
with mock.patch.object(nsp.pg, 'mem_used', side_effect=AssertionError('explicit value must win')):
|
|
self.assertIs(nsp.refine_after_rewrite(g, self.ex, self.tmp / 'patched'), g)
|
|
self.assertIs(nsp.refine_after_rewrite(g, self.ex, self.tmp / 'patched', game.Game(self.src)), g)
|
|
|
|
def test_computed_equal_numeric_value_keeps_bytes(self):
|
|
self.derive()
|
|
g = nsp.set_keys(game.Game(self.pdir), {'mem_used': ('75497472', 'used memory for the current rewritten modules, at least the default 64 MB')}, 'After the rewrite')
|
|
before = g.table('game.tsv').read_bytes()
|
|
with mock.patch.object(nsp.pg, 'mem_used', return_value=0x4800000) as used:
|
|
nsp.refine_after_rewrite(g, self.ex, self.tmp / 'patched')
|
|
used.assert_called_once()
|
|
self.assertEqual(g.table('game.tsv').read_bytes(), before)
|
|
|
|
def test_derived_replace_failure_preserves_original(self):
|
|
self.derive(); p = self.pdir / 'game.tsv'; before = p.read_bytes()
|
|
with mock.patch.object(nsp.os, 'replace', side_effect=OSError('interrupted replace')):
|
|
with self.assertRaisesRegex(OSError, 'interrupted replace'): self.derive(ver='v131072')
|
|
self.assertEqual(p.read_bytes(), before)
|
|
|
|
def test_set_keys_replace_failure_preserves_original(self):
|
|
self.derive(); g = game.Game(self.pdir); before = g.table('game.tsv').read_bytes()
|
|
with mock.patch.object(nsp.os, 'replace', side_effect=OSError('interrupted replace')):
|
|
with self.assertRaisesRegex(OSError, 'interrupted replace'):
|
|
nsp.set_keys(g, {'mem_used': ('0x4800000', 'rewritten modules')}, 'After the rewrite')
|
|
self.assertEqual(g.table('game.tsv').read_bytes(), before)
|
|
self.assertEqual(sorted(p.name for p in self.pdir.iterdir() if p.name.startswith('.game.tsv')), [])
|
|
|
|
def test_heap_stamp_replace_failure_preserves_original(self):
|
|
self.derive(); g = game.Game(self.pdir); before = g.table('game.tsv').read_bytes()
|
|
logs = self.tmp / 'logs'; logs.mkdir()
|
|
def boot(*args): (logs / 'heap_check.log').write_text('>>> nnMain\n'); return 0
|
|
with mock.patch.object(nsp, 'build_mac_host'), mock.patch.object(nsp, 'boot_mac', side_effect=boot), \
|
|
mock.patch.object(nsp.pg, 'heap_signals', return_value={}), mock.patch.object(nsp.os, 'replace', side_effect=OSError('interrupted replace')):
|
|
with self.assertRaisesRegex(OSError, 'interrupted replace'):
|
|
nsp.heap_check(REPO, self.ex, self.tmp / 'patched', self.tmp / 'work', {}, logs, g)
|
|
self.assertEqual(g.table('game.tsv').read_bytes(), before)
|
|
|
|
def test_rewrite_changes_update_auto_mem_used(self):
|
|
self.derive(); g = game.Game(self.pdir); reg = game.Game(self.src)
|
|
with mock.patch.object(nsp.pg, 'mem_used') as used:
|
|
for value in (0x4800000, 0x5000000, None):
|
|
used.return_value = value
|
|
g = nsp.refine_after_rewrite(g, self.ex, self.tmp / 'patched', reg)
|
|
self.assertEqual(int(g.get('mem_used'), 0), value or 0x4000000)
|
|
before = g.table('game.tsv').read_bytes()
|
|
self.derive(); g = game.Game(self.pdir)
|
|
g = nsp.refine_after_rewrite(g, self.ex, self.tmp / 'patched', reg)
|
|
self.assertEqual(g.table('game.tsv').read_bytes(), before)
|
|
|
|
def test_explicit_mem_used_is_kept_and_equal_values_do_not_rewrite(self):
|
|
self.derive(); g = nsp.set_keys(game.Game(self.pdir), {'mem_used': ('75497472', '')}, 'After the rewrite')
|
|
before = g.table('game.tsv').read_bytes()
|
|
with mock.patch.object(nsp.pg, 'mem_used', return_value=0x4800000):
|
|
nsp.refine_after_rewrite(g, self.ex, self.tmp / 'patched', game.Game(self.src))
|
|
self.assertEqual(g.table('game.tsv').read_bytes(), before)
|
|
with (self.src / 'game.tsv').open('a') as f: f.write('mem_used\t0x5800000\n')
|
|
self.derive(); g = game.Game(self.pdir)
|
|
with mock.patch.object(nsp.pg, 'mem_used', side_effect=AssertionError('explicit value must win')):
|
|
self.assertIs(nsp.refine_after_rewrite(g, self.ex, self.tmp / 'patched', game.Game(self.src)), g)
|
|
self.assertEqual(g.get('mem_used'), '0x5800000')
|
|
|
|
def test_provisional_run_invalidates_derivation(self):
|
|
self.derive()
|
|
key = nsp.derived_key(self.pdir)
|
|
self.assertTrue(key.is_file())
|
|
nsp.set_keys(game.Game(self.pdir), {'mem_alias': ('0x80000000', '')}, 'Heap check')
|
|
keys = self.tmp / 'prod.keys'; keys.write_text('test')
|
|
fam = {'base': self.tmp / 'base.nsp', 'updates': [], 'dlc': []}
|
|
args = nsp.parse_args([str(self.tmp), '-k', str(keys), '--platform', 'mac', '--provisional', '--profile-out', str(self.pdir.parent)])
|
|
with mock.patch.object(nsp, 'find_repo', return_value=REPO), mock.patch.object(nsp, 'apple_silicon', return_value=True), \
|
|
mock.patch.object(nsp, '_importable', return_value=True), mock.patch.object(nsp.shutil, 'which', return_value='/dotnet'), \
|
|
mock.patch.object(nsp, 'scan_nsps', return_value=(TID, fam)), mock.patch.object(nsp, 'decompress_nsz', return_value=fam), \
|
|
mock.patch.object(nsp, 'nsp_versions', side_effect=RuntimeError('profile selected')):
|
|
with self.assertRaisesRegex(RuntimeError, 'profile selected'): nsp.main(args)
|
|
self.assertFalse(key.exists())
|
|
self.derive()
|
|
self.assertFalse(game.Game(self.pdir).values('mem_alias'))
|
|
|
|
|
|
class SplitShaders(Base):
|
|
NAMES = ('shader_map.tsv', 'shader_facts.tsv', 'shader_pairs.tsv', 'pso_pairs.tsv')
|
|
|
|
def setUp(self):
|
|
super().setUp()
|
|
self.repo = self.tmp / 'repo'
|
|
for f in ('dm/gfx/tools/shader_split.py', 'dm/gfx/tools/dmsh_chain.lock', 'dm/games/game.py', 'dm/ios/tools/ios_metal.py'):
|
|
p = self.repo / f; p.parent.mkdir(parents=True, exist_ok=True); p.write_text('v1')
|
|
self.pkg = self.tmp / 'work/shader/pkg'; self.logs = self.tmp / 'logs'; self.logs.mkdir()
|
|
self.splits, self.rows = 0, 'derived'
|
|
cm = mock.patch.object(nsp, 'run', self.split); cm.__enter__(); self.addCleanup(cm.__exit__, None, None, None)
|
|
|
|
def split(self, cmd, log, env=None, cwd=None, on_line=None): # shader_split.py: the package and the four tables, deterministic for the same rows
|
|
self.splits += 1; out = Path(cmd[4]); out.mkdir(parents=True); (out / 'main.bin').write_bytes(self.rows.encode())
|
|
for n, t in zip(self.NAMES, cmd[5:]): Path(t).write_text(self.table(n))
|
|
Path(log).write_text('shaders: 3 (3 vertex)\n')
|
|
|
|
def table(self, n): return f'# split header\n{n}\t{self.rows}\n'
|
|
|
|
def go(self, g, own=True, oodle=None):
|
|
return nsp.split_shaders(sys.executable, self.repo, [self.ex / 'RomFS'], [self.ex / 'ExeFS/main'], self.pkg, {}, self.logs, g, own, oodle)
|
|
|
|
def own(self): # a derived profile with the empty tables, as main makes it
|
|
self.derive(); g = game.Game(self.pdir); nsp.provisional_prep(sys.executable, self.repo, {}, self.logs, g); return g
|
|
|
|
def test_own_profile_reuses_and_returns_the_package_key(self):
|
|
g = self.own()
|
|
k1 = self.go(g)
|
|
self.assertEqual((self.splits, k1), (1, nsp.package_key(self.pkg)))
|
|
for n in self.NAMES: self.assertEqual(g.table(n).read_text(), self.table(n))
|
|
k2 = self.go(g)
|
|
self.assertEqual((self.splits, k2), (1, k1)) # the key stored after the tables were written: the second run sees the same inputs
|
|
|
|
def test_redumped_inputs_invalidate_only_own_split(self):
|
|
for n in self.NAMES[:3]: (self.src / n).write_text(self.table(n))
|
|
captures = [self.ex / 'glslc_capture/a.glslc', self.ex / 'ue_shaders/a.ushaderbytecode']
|
|
for p in captures: p.parent.mkdir(); p.write_bytes(b'shader')
|
|
for own in (False, True):
|
|
g = self.own() if own else game.Game(self.src)
|
|
self.go(g, own=own)
|
|
for p in [self.ex / 'RomFS/a.bin', *captures]:
|
|
with self.subTest(own=own, path=p):
|
|
before = self.splits; self.redump(p); self.go(g, own=own)
|
|
self.assertEqual(self.splits, before + int(own))
|
|
self.go(g, own=own); self.assertEqual(self.splits, before + int(own))
|
|
|
|
def test_decoder_host_and_modules_invalidate_split(self):
|
|
g = self.own()
|
|
host = self.tmp / 'decoder-host'; host.write_bytes(b'host')
|
|
modules = self.tmp / 'patched'; modules.mkdir()
|
|
module = modules / 'main.patched.bin'; module.write_bytes(b'code')
|
|
decoder = mock.Mock(return_value={'DM_OODLE_HOST': str(host), 'DM_OODLE_MODULES': str(modules)})
|
|
self.go(g, oodle=decoder); self.go(g, oodle=decoder); self.assertEqual(self.splits, 1)
|
|
host.write_bytes(b'fixed host')
|
|
self.go(g, oodle=decoder); self.go(g, oodle=decoder); self.assertEqual(self.splits, 2)
|
|
module.write_bytes(b'new rewritten module')
|
|
self.go(g, oodle=decoder); self.go(g, oodle=decoder); self.assertEqual(self.splits, 3)
|
|
self.assertEqual(decoder.call_count, 6)
|
|
|
|
def test_own_profile_changes_split_again(self):
|
|
changes = {'game.tsv': lambda g: nsp.set_keys(g, {'mem_used': ('0x4800000', '')}, 'After the rewrite'),
|
|
'manual table': lambda g: g.table('shader_facts_manual.tsv').write_text('k\t{"manual": 3}\n'),
|
|
'main': lambda g: nso(self.ex / 'ExeFS/main', 'ef' * 16),
|
|
'RomFS': lambda g: (self.ex / 'RomFS/b.bin').write_bytes(b'more'),
|
|
'tool': lambda g: (self.repo / 'dm/gfx/tools/shader_split.py').write_text('v2')}
|
|
g = self.own(); self.go(g)
|
|
for i, (what, change) in enumerate(changes.items(), 2):
|
|
with self.subTest(what):
|
|
change(g); self.assertEqual(self.go(g), nsp.package_key(self.pkg)); self.assertEqual(self.splits, i)
|
|
self.go(g); self.assertEqual(self.splits, i)
|
|
|
|
def test_own_generated_tables_are_checked(self): # an edited or missing table of the split is never taken as is: the split's output comes back (split again, or restored)
|
|
g = self.own(); self.go(g)
|
|
for n in self.NAMES:
|
|
with self.subTest(edited=n):
|
|
g.table(n).write_text('# edited\nstale\trow\n'); self.go(g)
|
|
self.assertEqual(g.table(n).read_text(), self.table(n))
|
|
with self.subTest(missing=n):
|
|
g.table(n).unlink(); self.go(g)
|
|
self.assertEqual(g.table(n).read_text(), self.table(n))
|
|
runs = self.splits; self.go(g); self.assertEqual(self.splits, runs)
|
|
|
|
def test_own_new_output_replaces_tables(self): # a changed input whose split writes other rows: the profile takes them
|
|
g = self.own(); self.go(g)
|
|
self.rows = 'other'; (self.ex / 'RomFS/b.bin').write_bytes(b'more'); self.go(g)
|
|
self.assertEqual(g.table('shader_map.tsv').read_text(), self.table('shader_map.tsv'))
|
|
self.go(g); self.assertEqual(self.splits, 2)
|
|
|
|
def test_mode_is_in_the_key(self): # the same files split as an own profile, then checked against them as a registered one
|
|
g = self.own(); self.go(g)
|
|
self.assertEqual(self.go(g, own=False), nsp.package_key(self.pkg)); self.assertEqual(self.splits, 2)
|
|
self.go(g, own=False); self.assertEqual(self.splits, 2)
|
|
self.go(g); self.assertEqual(self.splits, 3)
|
|
|
|
def test_registered_tables_are_compared(self):
|
|
for n in self.NAMES[:3]: (self.src / n).write_text(f'# older header\n{n}\t{self.rows}\n')
|
|
g = game.Game(self.src)
|
|
k = self.go(g, own=False)
|
|
self.assertEqual((self.splits, k), (1, nsp.package_key(self.pkg)))
|
|
self.assertEqual(self.go(g, own=False), k); self.assertEqual(self.splits, 1)
|
|
self.assertEqual((self.src / 'shader_map.tsv').read_text(), '# older header\nshader_map.tsv\tderived\n') # a registered profile's tables are not written
|
|
self.rows = 'other'; (self.ex / 'RomFS/b.bin').write_bytes(b'more')
|
|
with self.assertRaisesRegex(nsp.BuildError, 'differ from game profile'): self.go(g, own=False)
|
|
self.assertEqual((self.src / 'shader_map.tsv').read_text(), '# older header\nshader_map.tsv\tderived\n')
|
|
with self.assertRaisesRegex(nsp.BuildError, 'differ from game profile'): self.go(g, own=False) # a failed check is not reused
|
|
self.assertEqual(self.splits, 3)
|
|
|
|
def test_damaged_never_reuses(self):
|
|
g = self.own()
|
|
with mock.patch.object(nsp, 'ALLOW_DAMAGED', True):
|
|
self.assertIsNone(self.go(g)); self.assertIsNone(self.go(g))
|
|
self.assertEqual(self.splits, 2)
|
|
self.go(g); self.assertEqual(self.splits, 3) # a damaged run leaves no key behind
|
|
|
|
def test_missing_split_pairs_are_generated_again(self):
|
|
for n in self.NAMES[:3]: (self.src / n).write_text(self.table(n))
|
|
g = game.Game(self.src)
|
|
self.go(g, own=False)
|
|
pairs = self.pkg.with_name('pso_pairs.tsv'); pairs.unlink()
|
|
self.go(g, own=False)
|
|
self.assertEqual(self.splits, 2)
|
|
self.assertEqual(pairs.read_text(), self.table('pso_pairs.tsv'))
|
|
|
|
def test_derived_rerun_is_incremental(self): # the whole derived path of a second run: derive, prepare, rewrite keys, split
|
|
g = self.own(); g = nsp.set_keys(g, {'mem_used': ('0x4800000', '')}, 'After the rewrite'); k = self.go(g)
|
|
g = self.own(); self.assertEqual(g.get('mem_used'), '0x4800000')
|
|
self.assertEqual(self.go(g), k); self.assertEqual(self.splits, 1)
|
|
|
|
|
|
class LayerShaders(Base):
|
|
def test_redumped_romfs_and_captures_invalidate_only_own_libraries(self):
|
|
cap = self.ex / 'capture'; cap.mkdir(); (cap / 'shader.bin').write_bytes(b'shader')
|
|
self.derive(); g = game.Game(self.pdir)
|
|
logs = self.tmp / 'logs'; logs.mkdir(); runs = []
|
|
def translate(cmd, log, **kw):
|
|
api = Path(cmd[1]).stem.split('_')[0]
|
|
lib = Path(cmd[-1]) / api / f'{api}_shaders.metallib'
|
|
lib.parent.mkdir(parents=True); lib.write_bytes(b'library')
|
|
Path(log).write_text('macosx: 1\n'); runs.append(api)
|
|
with mock.patch.object(nsp, 'run', side_effect=translate), mock.patch.object(nsp, 'toolchain', return_value=()), \
|
|
mock.patch.object(nsp.shutil, 'which', return_value='/glslangValidator'), \
|
|
mock.patch.object(nsp.subprocess, 'run', return_value=mock.Mock(stdout='compiler', stderr='')):
|
|
for api, build in (('vk', nsp.build_vk_shaders), ('gl', nsp.build_gl_shaders)):
|
|
roots = [self.ex / 'RomFS', cap] + ([self.ex / 'ExeFS/main'] if api == 'gl' else [])
|
|
for own in (False, True):
|
|
sh = self.tmp / f'{api}-{own}'
|
|
def go(): build(sys.executable, REPO, roots, sh, {'WORK': str(self.tmp), 'PATH': ''}, logs, g, mac=True, provisional=own)
|
|
go(); go()
|
|
for p in (self.ex / 'RomFS/a.bin', cap / 'shader.bin'):
|
|
with self.subTest(api=api, own=own, path=p):
|
|
before = len(runs); self.redump(p); go()
|
|
self.assertEqual(len(runs), before + int(own))
|
|
go(); self.assertEqual(len(runs), before + int(own))
|
|
|
|
|
|
class MacApp(Base):
|
|
def test_packer_reports_reuse_and_first_pack_steps(self):
|
|
t = self.tmp; out = t / 'Game.app'; packs = []
|
|
def pack(cmd, log, env=None, on_line=None):
|
|
packs.append(cmd)
|
|
(out / 'Contents/MacOS').mkdir(parents=True, exist_ok=True)
|
|
(out / 'Contents/MacOS/dmhost').write_bytes(b'host')
|
|
if len(packs) == 1: on_line('step: launcher and host (clones of the game files)')
|
|
on_line(f'app: {out}' + (', reused (inputs unchanged)' if len(packs) > 1 else ''))
|
|
with mock.patch.object(nsp, 'run', side_effect=pack):
|
|
for i in range(2):
|
|
result = nsp.mac_app(sys.executable, REPO, out, TID, t, self.ex, t / 'patched', t / 'host', t / 'font', t / 'shader', None, 'vulkan', 'en-US',
|
|
game.Game(self.src), 'Game', '1.0.0', 'org.nxnative.test', None, '', {}, t)
|
|
self.assertIsNone(result)
|
|
if i: nsp.PROG.end.assert_called_with('Game.app, reused (inputs unchanged)')
|
|
else: nsp.PROG.set.assert_called_with(step='launcher and host (clones of the game files)')
|
|
self.assertEqual(len(packs), 2) # pack_app owns the reuse decision and sidecar key
|
|
|
|
|
|
if __name__ == '__main__': unittest.main()
|