NSP script: a title ID no profile has is looked up by its main build (general, from issue #114). A regional release of a build a profile registers has another title ID (Advance Wars 1+2: Re-Boot Camp's main 320a17744aefd67fec5738ca0c629795 under 0100300012FFF000 as well as 0100300012F2A000), so game.py by_title found nothing and the script surveyed it as an unknown game with a provisional profile. same_build now extracts only the code modules (nx_extract --manifest: no game files written) of the base game with its newest update, then of the base game alone, and game.py by_build looks main's build ID up in every profile's build rows; a match packs from a profile derived from that one (derive_profile), with the folder's title ID and main.npdm's program ID (the host reports that program ID and answers save data queries for it) and shaders split from the title's own files, since its RomFS may differ. The closing message says the derived build is the profile's registered build under another title ID instead of a version it was not brought up on. No two of the 56 registered main builds are in more than one profile. New test test_nsp_alias.py (by_build, the update-then-base order, the derived profile's IDs); the 44 tool tests pass. Checked: the script on Advance Wars' NSP with its title ID made to read as 0100300012FFF000 found profile 0100300012F2A000's v0 in 2 s, packed for the Mac (20,424 shaders) and the build ran 90 s headless with no red line; the real title packs as before, with no lookup. docs/development.md says so

dm/games/game.py

@@ -84,6 +84,10 @@ def by_title(tid):
     return next((g for g in all_games() if g.title_id == tid), None)
 
 
+def by_build(main_id):   # the profile whose build rows register this main build ID (a release of the same build under another title ID: the NSP script's same_build)
+    return next((g for g in all_games() if any(b.get('main') == main_id for b in g.builds().values())), None)
+
+
 def header(g):
     """the host's game_profile.h: the profile constants the host uses, as macros"""
     q = lambda s: '"' + s.replace('\\', '\\\\').replace('"', '\\"') + '"'

dm/tools/tests/test_nsp_alias.py

@@ -0,0 +1,69 @@
+"""NSP script: a title ID no profile has, whose main is a build a profile registers (a regional release of the same build, GitHub issue #114), packs from a profile derived
+from that one (same_build, game.py by_build, derive_profile with the folder's title ID and main.npdm's program ID). No game files, keys or Xcode needed (temporary files only)."""
+import importlib.util
+import os
+from pathlib import Path
+import shutil
+import struct
+import sys
+import tempfile
+import unittest
+from unittest import mock
+
+REPO = Path(__file__).resolve().parents[3]
+sys.path.insert(0, str(REPO / 'dm/games')); sys.path.insert(0, str(REPO / 'dm/tools'))
+import game, nacp, profile_guess
+spec = importlib.util.spec_from_file_location('build_ipa_from_nsp', REPO / 'scripts/build_ipa_from_nsp.py')
+nsp = importlib.util.module_from_spec(spec); spec.loader.exec_module(nsp)
+nsp.pg, nsp.nacp = profile_guess, nacp   # main sets them
+
+PROFILE, OTHER, PROGRAM = 0x0100AAAA00000000, 0x0100AAAA00FFF000, 0x0100AAAA00FFF000
+BID, UPD_BID = 'ab' * 16, 'cd' * 16
+
+
+def nso(path, bid):   # an NSO header: the build ID at +0x40
+    path.parent.mkdir(parents=True, exist_ok=True); path.write_bytes(b'NSO0' + bytes(0x3c) + bytes.fromhex(bid) + bytes(0x50))
+
+
+class Alias(unittest.TestCase):
+    def setUp(self):
+        self.tmp = Path(tempfile.mkdtemp()); self.addCleanup(shutil.rmtree, self.tmp)
+        prof = self.tmp / 'games' / f'{PROFILE:016X}'; prof.mkdir(parents=True)
+        (prof / 'game.tsv').write_text('\n'.join([f'title_id\t{PROFILE:016X}', f'program_id\t{PROFILE:016X}', 'name\tTest Game', 'modules\tmain',
+                                                  f'build\tv0\t1.0.0\tmain={BID}', 'package\tv0', 'shader_versions\tv0', 'pad_layout\tfull']) + '\n')
+        (prof / 'shader_map.tsv').write_text('# made again from the build\n')
+        for cm in (mock.patch.dict(os.environ, {'DM_GAMES_DIR': str(self.tmp / 'games')}), mock.patch.object(nsp, 'PROG', mock.Mock()), mock.patch.object(nsp, 'say', lambda m: None)):
+            cm.__enter__(); self.addCleanup(cm.__exit__, None, None, None)
+
+    def test_by_build(self):
+        self.assertEqual(game.by_build(BID).id, f'{PROFILE:016X}')
+        self.assertIsNone(game.by_build(UPD_BID))
+        self.assertIsNone(game.by_title(OTHER))
+
+    def test_same_build_tries_the_update_then_the_base(self):
+        seen = []
+        def fake_run(cmd, log, env=None, cwd=None, on_line=None):   # nx_extract: the update's main is another build, the base game's is the profile's
+            seen.append(cmd[3]); nso(Path(cmd[4]) / 'ExeFS/main', UPD_BID if cmd[3] != '-' else BID)
+        fam = {'base': self.tmp / 'base.nsp', 'updates': [(65536, self.tmp / 'upd.nsp')], 'dlc': []}
+        with mock.patch.object(nsp, 'build_nx_extract', lambda *a: 'nx'), mock.patch.object(nsp, 'run', fake_run):
+            g, ver, bid = nsp.same_build(self.tmp, OTHER, fam, self.tmp / 'prod.keys', REPO, {})
+        self.assertEqual((g.id, ver, bid), (f'{PROFILE:016X}', 'v0', BID))
+        self.assertEqual(seen, [fam['updates'][0][1], '-'])
+        with mock.patch.object(nsp, 'build_nx_extract', lambda *a: 'nx'), mock.patch.object(nsp, 'run', lambda cmd, log, **k: nso(Path(cmd[4]) / 'ExeFS/main', UPD_BID)):
+            self.assertIsNone(nsp.same_build(self.tmp, OTHER, fam, self.tmp / 'prod.keys', REPO, {}))
+
+    def test_derived_profile_takes_the_title_and_program_ids(self):
+        d = self.tmp / 'extract/v0'; nso(d / 'ExeFS/main', BID)
+        (d / 'ExeFS/main.npdm').write_bytes(b'META' + bytes(0x7c) + b'ACI0' + bytes(0xc) + struct.pack('<Q', PROGRAM) + bytes(0x20))
+        g = game.load(f'{PROFILE:016X}'); out = self.tmp / 'derived' / g.id
+        nsp.derive_profile(g, out, 'v0', d, ['main'], OTHER)
+        h = game.Game(out)
+        self.assertEqual((h.title_id, int(h.get('program_id'), 16)), (OTHER, PROGRAM))
+        self.assertEqual((h.get('name'), h.get('pad_layout'), h.builds()['v0']['main']), ('Test Game', 'full', BID))
+        self.assertFalse((out / 'shader_map.tsv').exists())   # made again from the build's own files
+        nsp.derive_profile(g, out, 'v0', d, ['main'], PROFILE)   # the profile's own title: its IDs as they are
+        self.assertEqual(game.Game(out).title_id, PROFILE)
+
+
+if __name__ == '__main__':
+    unittest.main()

docs/development.md

@@ -55,7 +55,7 @@ In Spyder, open the script, fill in the settings at the top (`NSP_DIR`, `PROD_KE
 What the script does:
 1. It finds the game profile from the title ID (the `[title ID]` in the file names, or the NSP's ticket; the update by its `[vN]` tag). The update is optional: it packs the newest version in the folder (`-p` / `PACKAGE` picks another).
 2. It extracts the NSPs.
-3. It rewrites the code modules, checking each build ID against the profile. A version the profile does not register (the base game alone when the profile was brought up on an update, another update, or files whose build IDs differ), or one whose shader tables need a version that is not in the folder, is built from a profile derived from it in `<id>_build/profile/<id>/`: the profile's keys, that version's build row, and shader tables, chain header, `c1_full.tsv` and `cb_extent.tsv` made from that version's own files (as for a provisional profile, without the heap check). The rewriter's cuts and the host's fixes for a registered build do not reach a module of another build; the script says which modules those are. A provisional profile (a game without one in `dm/games`) then gets `mem_used` from the rewritten modules, and, when the game has virtual address memory and sets a heap, a heap check: the Mac host is built into `<id>_build/tools/dmhost` and the game boots in it until its first heap request (a few seconds, log `<id>_build/logs/heap_check.log`), which sizes the heap region (see [dm/games/README.md](../dm/games/README.md), Adding a game).
+3. It rewrites the code modules, checking each build ID against the profile. A version the profile does not register (the base game alone when the profile was brought up on an update, another update, or files whose build IDs differ), or one whose shader tables need a version that is not in the folder, is built from a profile derived from it in `<id>_build/profile/<id>/`: the profile's keys, that version's build row, and shader tables, chain header, `c1_full.tsv` and `cb_extent.tsv` made from that version's own files (as for a provisional profile, without the heap check). The rewriter's cuts and the host's fixes for a registered build do not reach a module of another build; the script says which modules those are. A title ID no profile has is first looked up by build: when its `main` (of the base game with its newest update, else of the base game alone, read from a code-only extraction) is a build a profile registers, a regional release of the same build (Advance Wars 1+2: Re-Boot Camp under `0100300012FFF000`, issue #114), it packs from a profile derived from that one, with the folder's title ID and `main.npdm`'s program ID (`same_build`, `game.py` `by_build`); only otherwise is it surveyed. A provisional profile (a game without one in `dm/games`) then gets `mem_used` from the rewritten modules, and, when the game has virtual address memory and sets a heap, a heap check: the Mac host is built into `<id>_build/tools/dmhost` and the game boots in it until its first heap request (a few seconds, log `<id>_build/logs/heap_check.log`), which sizes the heap region (see [dm/games/README.md](../dm/games/README.md), Adding a game).
 4. For a game whose `main` holds GLSL sources and calls `glslcCompile` (Three Houses), it captures the shaders the game compiles at run time, which no game file holds. It builds the Mac host into `<id>_build/tools/dmhost` and boots the game in it, headless and silent, with `DM_GLSLC_CAPTURE` pointing at `<id>_build/extract/<version>/glslc_capture`. The save, temporary storage and log all stay in `<id>_build/`. It stops the game as soon as every capture file named in the profile's shader map is there (Three Houses: 9, about a second after launch), and gives up after 30 s. A folder that already holds them all is reused.
    A Vulkan game without SPIR-V files in its RomFS (Super Mario Galaxy) gets the same kind of run: the game boots in the Mac host with `DM_VK_CAPTURE` pointing at `<id>_build/extract/<version>/vk_capture` until every SPIR-V key in the profile's `vk_capture.tsv` is there (Galaxy: 3,089, about 7 s after launch), and gives up after 120 s.
    So does an OpenGL game with no GLSL source in its RomFS or `main` (Balatro: LÖVE puts its shaders together at run time, in Lua): `DM_GL_CAPTURE` at `<id>_build/extract/<version>/gl_capture` until every file in the profile's `gl_capture.tsv` is there (Balatro: 35, made before its first frame). Without the profile's tables (a provisional profile) a capture stops once nothing has been added for 20 s.

scripts/build_ipa_from_nsp.py

@@ -451,6 +451,29 @@ def can_clone(d):   # whether this volume supports APFS clones (nx_extract uses
 
 
 # ---- Survey of a new game (when no game profile matches) ----
+def same_build(nsp_dir, app, fam, keys, repo, env):
+    """A title ID no profile has may be a known build under another title ID (a regional release: Advance Wars 1+2: Re-Boot Camp's main 320a1774… under 0100300012FFF000 as well as
+    0100300012F2A000, GitHub issue #114): main's build ID of the base game with its newest update, then of the base game alone, against every profile's build rows (game.py by_build).
+    Only the code modules are extracted (a RomFS manifest, no game files written). Returns (profile, version, main build ID) or None"""
+    import game as gamemod   # dm/games (main put it on the path)
+    work = nsp_dir / f'{app:016X}_survey'; logs = work / 'logs'
+    PROG.plan(1); PROG.stage('build ID lookup', step='build nx_extract')
+    nx = build_nx_extract(repo, work, logs, env)
+    ups = sorted(fam['updates'], key=lambda u: u[0] or 0)
+    for v, upd in ([ups[-1]] if ups else []) + [(0, None)]:
+        PROG.set(step=f'main of {"the base game" if upd is None else "update v" + str(v)}')
+        tmp = work / 'exefs.partial'; rmtree(tmp)
+        run([nx, keys, fam['base'], upd or '-', tmp, '--manifest'] + (['--allow-damaged'] if ALLOW_DAMAGED else []), logs / 'exefs.log')
+        main = tmp / 'ExeFS/main'
+        bid = build_id(main) if main.exists() else None
+        g = gamemod.by_build(bid) if bid else None
+        rmtree(tmp)
+        if g:
+            ver = next(ver for ver, b in g.builds().items() if b.get('main') == bid)
+            PROG.end(f'main {bid} is game profile {g.id}\'s {ver}'); return g, ver, bid
+    PROG.end('no profile registers its main build'); return None
+
+
 def survey(nsp_dir, app, fam, keys, repo, py, env):
     work = nsp_dir / f'{app:016X}_survey'; logs, ex = work / 'logs', work / 'extract'
     say(f'no game profile for {app:016X}: surveying first')
@@ -611,9 +634,10 @@ def write_profile(pdir, app, f):
 DERIVED_SKIP = {'game.tsv', 'shader_map.tsv', 'shader_facts.tsv', 'shader_pairs.tsv', 'c1_full.tsv', 'cb_extent.tsv', 'vk_capture.tsv', 'gl_capture.tsv', 'patches.py'}   # made again from the packed version's own files (patches.py: the rewriter's patches of the registered builds)
 
 
-def derive_profile(g, pdir, ver, d, mods):
-    """The game's profile for a version it was not brought up on (the folder has no update, another update, or files of other builds), in pdir (outside the repository, where a
-    provisional profile goes): its keys as they are, with this version's build row, package and shader versions. The shader tables, the chain header and the run-time capture list are
+def derive_profile(g, pdir, ver, d, mods, app=None):
+    """The game's profile for a version it was not brought up on (the folder has no update, another update, files of other builds, or a registered build under another title ID: app),
+    in pdir (outside the repository, where a provisional profile goes): its keys as they are, with this version's build row, package and shader versions, and app's title ID and
+    main.npdm's program ID when app is not the profile's title. The shader tables, the chain header and the run-time capture list are
     left out: the build makes them again from this version's own files, as for a provisional profile. The hand-written tables (the prewarm list, shader_facts_manual.tsv) are copied:
     they name shaders by content key. Returns the version's display version"""
     rmtree(pdir); pdir.mkdir(parents=True)
@@ -623,7 +647,12 @@ def derive_profile(g, pdir, ver, d, mods):
     row = T.join(['build', ver, disp] + [f'{m}={build_id(d / "ExeFS" / m)}' for m in mods if (d / 'ExeFS' / m).exists()])
     out, put = [f'# Derived from game profile {g.dir} by scripts/build_ipa_from_nsp.py for {ver} ({disp}), a version that profile was not brought up on:',
                 '# its keys as they are, this version\'s build row, and shader tables made from this version\'s own files'], False
+    ids = {}
+    if app is not None and app != g.title_id:   # the host reports the program ID and answers save data queries for it (hle_svc.c, hle_fs.c)
+        ids = {'title_id': f'title_id{T}{app:016X}', 'program_id': f'program_id{T}{(pg.npdm(d / "ExeFS/main.npdm")[1] or app):016X}'}
+        out.insert(1, f'# under title ID {app:016X}, not the profile\'s {g.title_id:016X}')
     for line in (g.dir / 'game.tsv').read_text(encoding='utf-8').splitlines():
+        if line.split(T, 1)[0] in ids: out.append(ids[line.split(T, 1)[0]]); continue
         if line.split(T, 1)[0] in ('build', 'package', 'shader_versions'):
             if not put: out += [row, f'package{T}{ver}', f'shader_versions{T}{ver}']; put = True
             continue
@@ -1491,7 +1520,9 @@ def report_profile_status(g, reg, pkg_ver, derived, provisional, artifact='IPA')
         known = {(m, b) for row in reg.builds().values() for m, b in row.items() if m != 'display'}
         other = [m for m, b in g.builds()[pkg_ver].items() if m != 'display' and (m, b) not in known]
         registered_builds = ", ".join(f'{v} {b["display"]}' for v, b in reg.builds().items())
-        say(f'[RED] this {artifact} uses {pkg_ver} ({disp}), which game profile {reg.id} was not brought up on ({registered_builds}): '
+        if g.title_id != reg.title_id and not other and pkg_ver in reg.builds():   # a registered build under another title ID (same_build): every fix applies
+            say(f'this {artifact} is game profile {reg.id}\'s {pkg_ver} ({disp}) under title ID {g.title_id:016X}: its settings and every fix for its builds, with shaders split from this title\'s own files')
+        else: say(f'[RED] this {artifact} uses {pkg_ver} ({disp}), which game profile {reg.id} was not brought up on ({registered_builds}): '
             'its settings, with this version\'s own shaders' + (f'; {", ".join(other)} {"is a build" if len(other) == 1 else "are builds"} the profile does not register, so the fixes made for '
             'the registered builds\' code (patch_nso.py, the host) do not apply there and the game may not run as well' if other else '; every module is a registered build'))
     if provisional:
@@ -1532,6 +1563,9 @@ def main(args=None):
     app, fam = scan_nsps(nsp_dir)
     force = args.provisional or PROVISIONAL   # a provisional profile even when dm/games has one (testing the guesses)
     g, provisional, shadowed = None if force else gamemod.by_title(app), False, False
+    alias = None if force or g else same_build(nsp_dir, app, fam, keys, repo, env)   # a known build under another title ID: packed from a profile derived from that build's
+    if alias:
+        g = alias[0]; say(f'no game profile for {app:016X}, but its main build {alias[2]} is game profile {g.id}\'s {alias[1]} ({g.get("name", "")}) under another title ID')
     if g is None:   # no game profile: survey, then (unless --survey-only) a provisional profile outside the repository and a best-effort IPA
         missing = [m for m in ('lz4.block', 'capstone') if not _importable(m)]
         if missing: raise BuildError(f'the current Python ({sys.executable}) is missing modules: {", ".join(missing)}')
@@ -1564,6 +1598,7 @@ def main(args=None):
             registered_builds = ", ".join(f'{v} {b["display"]}' for v, b in builds.items())
             derived = f'{pkg_ver} is not a version game profile {g.id} registers ({registered_builds})'
         elif not all(v in have for v in need): derived = f'the folder has no {", ".join(v for v in need if v not in have)}, whose files game profile {g.id}\'s shader tables were made from'
+        if alias and not derived: derived = f'title {app:016X} is not game profile {g.id}\'s title {g.title_id:016X}'
         if derived: say(f'{derived}: packing {pkg_ver} from a profile derived from it')
     versions = [pkg_ver] if derived else need + ([pkg_ver] if pkg_ver not in need else [])
     out_ipa = Path(os.path.expanduser(args.out or OUT_IPA)).resolve() if (args.out or OUT_IPA) else nsp_dir / g.get('ipa')   # by default in the NSP folder, next to the NSPs and <id>_build/
@@ -1613,7 +1648,7 @@ def main(args=None):
     reg = g   # the profile as registered (a derived build reports what it lacks against it)
     if derived:   # the derived profile goes where a provisional one would, and every tool (game.py, make, build_ipa.sh) takes it first (DM_GAMES_FIRST)
         pdir = Path(os.path.expanduser(args.profile_out or PROFILE_OUT)).resolve() if (args.profile_out or PROFILE_OUT) else nsp_dir / f'{app:016X}_build' / 'profile'
-        disp = derive_profile(g, pdir / g.id, pkg_ver, dirs[pkg_ver], g.words('modules'))
+        disp = derive_profile(g, pdir / g.id, pkg_ver, dirs[pkg_ver], g.words('modules'), app)
         os.environ['DM_GAMES_DIR'] = env['DM_GAMES_DIR'] = os.pathsep.join([str(pdir)] + [d for d in os.environ.get('DM_GAMES_DIR', '').split(os.pathsep) if d and d != str(pdir)])
         os.environ['DM_GAMES_FIRST'] = env['DM_GAMES_FIRST'] = '1'
         g = gamemod.load(g.id)