diff --git a/dm/tools/tests/test_nsp_mac.py b/dm/tools/tests/test_nsp_mac.py new file mode 100644 index 0000000..b4c93d2 --- /dev/null +++ b/dm/tools/tests/test_nsp_mac.py @@ -0,0 +1,124 @@ +"""NSP script Mac-only build (scripts/build_ipa_from_nsp.py --platform mac): the shader steps build only the Mac's libraries, each output keeps its own reuse key, a Mac-only build is +reused after an iOS build (which builds the Mac's too) and an iOS build is not reused after a Mac-only one, neither key without the Mac libraries' stamp (inputs.tsv, which the one +chain version's c1_full.tsv / cb_extent.tsv name); the run command's environment file is read as the iOS app reads it. +No game files, keys or Xcode needed (temporary files only in a temporary folder).""" +import importlib.util +from pathlib import Path +import shutil +import sys +import tempfile +import types +import unittest +from unittest import mock + +REPO = Path(__file__).resolve().parents[3] +SCRIPT = REPO / 'scripts/build_ipa_from_nsp.py' +spec = importlib.util.spec_from_file_location('build_ipa_from_nsp', SCRIPT) +nsp = importlib.util.module_from_spec(spec); spec.loader.exec_module(nsp) +CHAIN, INPUTS = '0123456789ab', 'aaaaaaaaaaaa' + + +class Base(unittest.TestCase): + def setUp(self): + self.tmp = Path(tempfile.mkdtemp()); self.addCleanup(shutil.rmtree, self.tmp) + self.sh, self.logs, self.ran = self.tmp / 'shader', self.tmp / 'logs', [] + self.logs.mkdir() + for cm in (mock.patch.object(nsp, 'PROG', mock.Mock()), mock.patch.object(nsp, 'say', lambda m: None), mock.patch.object(nsp, 'run', self.fake_run)): + cm.__enter__(); self.addCleanup(cm.__exit__, None, None, None) + + def fake_run(self, cmd, log, env=None, cwd=None, on_line=None): # stands in for the translators: writes the libraries a real run writes + self.ran.append(([str(c) for c in cmd], dict(env or {}))) + tool = Path(cmd[1]).name + if tool in ('vk_shaders.py', 'gl_shaders.py'): + api, a = tool[:2], [str(c) for c in cmd] + outs = [Path(a[a.index('--ios') - 1]), Path(a[a.index('--ios') + 1])] if '--ios' in a else [Path(a[-1])] + for o in outs: (o / api).mkdir(parents=True, exist_ok=True); (o / api / f'{api}_shaders.metallib').write_bytes(b'lib') + log.write_text('macosx: 3 shaders\n' + ('iphoneos: 3 shaders\n' if '--ios' in a else '')) + elif tool == 'own_compilers.py': # a full dmsh_build.py run: the libraries, their stamp, and the profile's tables naming it + outs = [self.sh / 'cache'] + ([Path(env['IOSOUT'])] if 'IOSOUT' in env else []) + for o in outs: + (o / f'b-{CHAIN}').mkdir(parents=True, exist_ok=True); (o / f'b-{CHAIN}' / 'k_b.metallib').write_bytes(b'lib') + (o / f'b-{CHAIN}' / 'inputs.tsv').write_text(f'# stamp\ngame\t0100535012974000\ninputs\t{INPUTS}\n') + for n in ('c1_full.tsv', 'cb_extent.tsv'): (self.tmp / n).write_text(f'# chain b-{CHAIN} inputs {INPUTS} 2026-10-23T00:00:00+00:00\n') + log.write_text('shaders 1: ok 1\n') + + +class Layer(Base): + def vk(self, mac, key='K'): + self.ran.clear() + return nsp.layer_shaders(sys.executable, REPO, 'vk', 'Vulkan', [self.tmp / 'romfs'], self.sh, key, self.logs, {'WORK': str(self.tmp)}, mac) + + def test_mac_only(self): # no --ios, only the Mac key, the Mac folder for DM_MSL_DIR + self.assertEqual(self.vk(True), self.sh) + self.assertNotIn('--ios', self.ran[0][0]) + self.assertTrue((self.sh / 'vk.key').is_file()); self.assertFalse((self.sh / 'ios/vk.key').exists()) + self.assertEqual(self.vk(True), self.sh); self.assertEqual(self.ran, []) # reused + + def test_mac_after_ios(self): # an iOS build makes the Mac library too: a Mac-only build is reused + self.assertEqual(self.vk(False), self.sh / 'ios') + self.assertIn('--ios', self.ran[0][0]) + (self.sh / 'vk.key').unlink() # an iOS build from before Mac-only builds wrote only its own key + self.assertEqual(self.vk(True), self.sh); self.assertEqual(self.ran, []) + + def test_ios_after_mac(self): # a Mac-only build made no iPhone library: an iOS build runs + self.vk(True) + self.assertEqual(self.vk(False), self.sh / 'ios'); self.assertEqual(len(self.ran), 1) + self.assertTrue((self.sh / 'ios/vk/vk_shaders.metallib').is_file()) + + def test_changed_inputs(self): + self.vk(False) + self.vk(True, 'K2'); self.assertEqual(len(self.ran), 1) + self.assertTrue((self.sh / 'ios/vk/vk_shaders.metallib').is_file()) # the iPhone library stays, under its own (older) key + self.assertEqual(nsp.reused(self.sh / 'ios/vk.key', 'K'), '3 shaders for the Mac and iOS') + + +class Nvn(Base): + def setUp(self): + super().setUp() + self.g = types.SimpleNamespace(table=lambda n: self.tmp / n, dir=self.tmp) + for cm in (mock.patch.object(nsp, 'fingerprint', lambda *a, **k: 'K'), mock.patch.object(nsp, 'chain_of', lambda repo: CHAIN)): # one chain version for every game + cm.__enter__(); self.addCleanup(cm.__exit__, None, None, None) + + def build(self, mac): + self.ran.clear() + return nsp.build_shaders(sys.executable, REPO, self.sh, REPO / 'dm/gfx/tools/dmsh_xlate_bin/dmsh_xlate', {'WORK': str(self.tmp), 'IOSOUT': '/elsewhere'}, self.logs, self.g, pkg_key='P', mac=mac) + + def test_mac_only(self): # no IOSOUT (not even the caller's), the Mac key beside the cache folder, not in it + self.assertEqual(self.build(True), self.sh / 'cache') + self.assertNotIn('IOSOUT', self.ran[0][1]) + self.assertTrue((self.sh / f'mac-b-{CHAIN}.key').is_file()); self.assertEqual([p.name for p in (self.sh / 'cache').iterdir()], [f'b-{CHAIN}']) + self.assertEqual(self.build(True), self.sh / 'cache'); self.assertEqual(self.ran, []) + + def test_mac_after_ios(self): + self.assertEqual(self.build(False), self.sh / 'ios') + self.assertEqual(self.ran[0][1]['IOSOUT'], str(self.sh / 'ios')) + (self.sh / f'mac-b-{CHAIN}.key').unlink() # an iOS build from before Mac-only builds + self.assertEqual(self.build(True), self.sh / 'cache'); self.assertEqual(self.ran, []) + + def test_ios_after_mac(self): + self.build(True) + self.assertEqual(self.build(False), self.sh / 'ios'); self.assertEqual(len(self.ran), 1) + + def test_no_stamp(self): # Mac libraries no full build stamped (an interrupted one): no table can name them, so neither key reuses them + self.build(False) + (self.sh / 'cache' / f'b-{CHAIN}' / 'inputs.tsv').unlink() + self.assertEqual(self.build(True), self.sh / 'cache'); self.assertEqual(len(self.ran), 1) + self.assertTrue((self.sh / 'cache' / f'b-{CHAIN}' / 'inputs.tsv').is_file()) + + def test_stale_tables(self): # a reused build makes a table again when it does not name the Mac libraries' stamp + self.build(True) + (self.tmp / 'cb_extent.tsv').write_text(f'# chain b-{CHAIN} inputs bbbbbbbbbbbb 2026-10-23T00:00:00+00:00\n') + self.assertEqual(self.build(True), self.sh / 'cache'); self.assertEqual([Path(c[1]).name for c, _ in self.ran], ['cb_extent.py']) + + +class EnvFile(Base): + def test_pairs(self): + p = self.tmp / 'env.txt'; p.write_text('# a comment\nDM_A=1\n DM_B = x=y\n=nokey\nnoeq\n\nDM_C=\n') + self.assertEqual(nsp.env_file_pairs(p), {'DM_A': '1', 'DM_B': ' x=y', 'DM_C': ''}) + + def test_missing(self): + with self.assertRaises(nsp.BuildError): nsp.env_file_pairs(self.tmp / 'none.txt') + + +if __name__ == '__main__': + unittest.main() diff --git a/docs/development.md b/docs/development.md index 6e599c1..9b1cdc6 100644 --- a/docs/development.md +++ b/docs/development.md @@ -45,9 +45,12 @@ python3 scripts/build_ipa_from_nsp.py ~/Games/NewGame -k ~/keys/prod.keys # test the guesses on a game that has a profile: build from a provisional one (the IPA is named -provisional.ipa) python3 scripts/build_ipa_from_nsp.py ~/Games/MarioGalaxy -k ~/keys/prod.keys --provisional --allow-damaged + +# development: build only what running the game on this Mac needs (no iOS shaders, Xcode project or IPA) and print the command that runs it +python3 scripts/build_ipa_from_nsp.py ~/Games/MarioWonder -k ~/keys/prod.keys --platform mac ``` -In Spyder, open the script, fill in the settings at the top (`NSP_DIR`, `PROD_KEYS`, and optionally `PACKAGE`, `OUT_IPA`, `APP_NAME`, `BUNDLE_ID`, `LANGUAGE`, `KEEP_EXTRACTED`, `SHOW_FPS`; for a game without a profile `HEAP_CHECK`, `HEAP_CHECK_SECS`, `PROVISIONAL`), and press F5. Empty paths are asked for at run time. +In Spyder, open the script, fill in the settings at the top (`NSP_DIR`, `PROD_KEYS`, and optionally `PACKAGE`, `OUT_IPA`, `APP_NAME`, `BUNDLE_ID`, `LANGUAGE`, `KEEP_EXTRACTED`, `SHOW_FPS`, `PLATFORM`; for a game without a profile `HEAP_CHECK`, `HEAP_CHECK_SECS`, `PROVISIONAL`), and press F5. Empty paths are asked for at run time. What the script does: 1. It finds the game profile from the title ID (the `[title ID]` in the file names, or the NSP's ticket; the update by its `[vN]` tag). The update is optional: it packs the newest version in the folder (`-p` / `PACKAGE` picks another). @@ -77,6 +80,24 @@ The game language (`--language`, `LANGUAGE`; English by default) is what `nn::oe On the device, environment switches go in `Documents/env.txt` (one `KEY=VALUE` per line), and game files are also accepted at `Documents/romfs`. +### A Mac-only build for development + +The Mac is the development host, not a platform for players: there is no Mac app. To test or debug a game on the Mac, `--platform mac` (`PLATFORM = 'mac'` in Spyder) builds only what running it in the Mac host needs, from the same NSP folder: +- the same steps up to the shaders: extraction, rewrite, run-time shader captures and decodes, a provisional profile's heap check and the shader split; +- the Mac shader libraries only: `dmsh_build.py` without `IOSOUT` into `_build/shader/cache/b-/`, which a full build stamps with the profile's inputs (`inputs.tsv`) and from which it makes the profile's `c1_full.tsv` and `cb_extent.tsv`, or a Vulkan or GL game's library without `--ios` into `_build/shader/vk/` or `gl/`; +- the Mac host for the profile into `_build/tools/dmhost`: the compile line of `make -C dm/host GAME= dmhost`, with a provisional or derived profile through `DM_GAMES_DIR` / `DM_GAMES_FIRST` as for the script's other host builds. It is reused while its sources (the chain header `dm/gfx/nvn/dmsh_chain.h` among them), the generated profile header, libvpx and the compiler are unchanged; +- the button-icon font (`make_ext_font.py`) into `_build/font/`. + +It skips the iOS shaders, the Xcode project and the IPA. For Mario Kart 8 Deluxe that saves about 30,000 iOS shader compiles and the 12 GB IPA. It keeps the extracted files even with `--no-keep-extracted`, because the game runs from them. It needs a Mac with Apple silicon, and it cannot be combined with `--prepare-xcode`. The last line it prints is the command that runs the game, built from the run's own paths (step 5 of [Build by hand (Mac)](#build-by-hand-mac)), for example: + +``` +cd "/_build/run" && DM_ROOT="/_build" DM_ROMFS="/_build/extract//RomFS" DM_SAVE="/_build/run/save" DM_TEMP="/_build/run/temp" DM_MSL_DIR="/_build/shader/cache" DM_FONT_DIR="/_build/font" DM_NVN_SHMAP="/dm/games//shader_map.tsv" DM_LANG=en-US DM_WINDOW=1 DM_AUDIO=1 "/_build/tools/dmhost" "/_build/patched_" +``` + +The save is in `_build/run/save` and stays there from run to run. `DM_AOC_DIR` is added when the folder has DLC. `DM_NVN_SHMAP` is added for an NVN game; it names the profile's shader map, or the derived or provisional profile's. A Vulkan or GL game's `DM_MSL_DIR` is `_build/shader`. The switches of `--env-file` are added to the command. + +Reuse works as for the IPA. Each shader output has its own key, named after the one chain version: `shader/ios/b-.key` for the iOS libraries and `shader/mac-b-.key` for the Mac's (`shader/ios/vk.key` and `shader/vk.key` for a Vulkan game, the same with `gl` for a GL game). An iOS build writes both keys, since it builds the Mac's libraries too, so a Mac-only build of a folder already packed as an IPA reuses its shaders. Either key is taken only while the Mac libraries carry a full build's stamp; `c1_full.tsv` or `cb_extent.tsv` that does not name it is made again from them, as for the IPA. An iOS build after a Mac-only build still has the iPhone's libraries to make. It translates the shaders again, because `dmsh_build.py`'s per-shader cache keys include the iOS compile convention. It then takes the Mac libraries from the Metal compile cache (`shader/cache/cas/metallib`) and compiles only the iPhone's. + ## Build and run on a device with Xcode With an Apple developer account, Xcode can sign and install the app itself: @@ -121,7 +142,7 @@ To add a game: ## Build by hand (Mac) -You need the same tools as the script. The shader translator is the prebuilt `dm/gfx/tools/dmsh_xlate_bin`, the build that matches `dmsh_chain.lock`; a fresh `dotnet publish` gives different fingerprints. Its launcher `dmsh_xlate` is a universal binary (arm64 and x86_64), so an Intel Mac runs it too, with the same output; after a rebuild, join the new launcher with lipo to the one `dotnet build -r osx-x64` makes of the same .NET version (the lock does not cover the launcher). What an Intel Mac cannot run is the Mac host, which runs the game's own arm64 code: the NSP script stops at a step that boots the game there (a run-time shader capture, a decode with the game's own Oodle or LZ5) and skips a provisional profile's heap check. The commands use three variables: +`scripts/build_ipa_from_nsp.py --platform mac` does steps 1 to 4 in the game's NSP folder and prints the command of step 5 ([A Mac-only build for development](#a-mac-only-build-for-development)). The steps by hand follow. You need the same tools as the script. The shader translator is the prebuilt `dm/gfx/tools/dmsh_xlate_bin`, the build that matches `dmsh_chain.lock`; a fresh `dotnet publish` gives different fingerprints. Its launcher `dmsh_xlate` is a universal binary (arm64 and x86_64), so an Intel Mac runs it too, with the same output; after a rebuild, join the new launcher with lipo to the one `dotnet build -r osx-x64` makes of the same .NET version (the lock does not cover the launcher). What an Intel Mac cannot run is the Mac host, which runs the game's own arm64 code: the NSP script stops at a step that boots the game there (a run-time shader capture, a decode with the game's own Oodle or LZ5) and skips a provisional profile's heap check. The commands use three variables: - `G`, the profile; - `W`, the game's work folder: `-work` inside the game's NSP folder, outside this repository and off the home folder (it grows to the size of the game files); - `VER`, a version folder named after the NSP version. diff --git a/scripts/build_ipa_from_nsp.py b/scripts/build_ipa_from_nsp.py index 19de191..755d96d 100755 --- a/scripts/build_ipa_from_nsp.py +++ b/scripts/build_ipa_from_nsp.py @@ -6,6 +6,9 @@ # → iOS shader libraries (dmsh_build.py) → build_ipa.sh. A Vulkan game (main imports Vulkan, not NVN: Hades) skips the NVN shader steps: vk_shaders.py translates its SPIR-V shaders instead. # Which of these steps a game needs follows from its files (dm/games/profile_guess.py shader_sources), not from its profile. A game with no matching profile is surveyed (<NSP folder>/<title ID>_survey/survey.md), then gets a provisional profile guessed from # the survey (<NSP folder>/<title ID>_build/profile/<title ID>/, outside the repository, found through DM_GAMES_DIR) and a best-effort IPA that may not boot; SURVEY_ONLY / --survey-only stops after the survey. +# PLATFORM = 'mac' / --platform mac is a development option (the Mac is the development host, not a platform for players): the same steps up to the shaders, but only the Mac's +# (the shader build without IOSOUT, a Vulkan or GL game's libraries without --ios), then the Mac host for the profile and the button-icon font; no iOS shaders, Xcode project or IPA. +# The extracted files are kept whatever KEEP_EXTRACTED says (the game runs from them), and the last line is the command that runs the game in the Mac host, its save in <id>_build/run/save. # Two ways to run it: open this file in Spyder and press F5 (it uses the values under "Settings" below; an empty NSP folder or prod.keys is asked for at run time); # or in a terminal: python3 scripts/build_ipa_from_nsp.py <NSP folder> -k <prod.keys> [options] (values given on the command line override "Settings", -h lists every option; # without keyboard input a missing value is not asked for but reported as an error; exit code 0 on success, 1 on error, 130 when interrupted). Command-line options entered in Spyder's run configuration work the same way. @@ -33,6 +36,8 @@ OUT_IPA = '' # output IPA (empty = <the profile's ipa APP_NAME = '' # app name on the home screen (empty = the game name in the base NSP) BUNDLE_ID = '' # the app's bundle ID (empty = saved per-game Xcode ID, else org.nxnative.<base game title ID in lowercase>) PREPARE_XCODE = False # prepare a per-game Xcode project and add it to the workspace instead of packing an IPA; requires KEEP_EXTRACTED +PLATFORM = 'ios' # 'ios' = the IPA (or the Xcode project); 'mac' = development: build only what running the game on this Mac needs (Mac shaders, the Mac host), no iOS shaders, + # Xcode project or IPA; keeps the extracted files and prints the command that runs the game (needs Apple silicon) TEAM_ID = '' # optional Apple team ID; empty = DM_TEAM_ID environment, saved Xcode selection, or select in Xcode XCODE_WORKSPACE = '' # shared .xcworkspace (empty = NxNative.xcworkspace beside the game's NSP folder) LANGUAGE = '' # game language, a code or a name: 'ja', 'fr', 'de', 'es', 'zh-Hans', 'zh-Hant', 'ko', 'Japanese', ... (empty = English); it must be one the game supports (the console line lists them); @@ -690,13 +695,14 @@ def profile_files(g): # every file of the game profile (game.tsv and its table # ---- Steps ---- -def check_prereqs(env, xlate): +def check_prereqs(env, xlate, mac=False): # mac: a Mac-only build (PLATFORM 'mac'), which needs no xcodegen and compiles shaders only for macOS missing = [m for m in ('lz4.block', 'capstone', 'fontTools') if not _importable(m)] + ([] if _importable('compression.zstd') or _importable('backports.zstd') else ['backports.zstd']) # zstd: built in from Python 3.14 if missing: raise BuildError(f'the current Python ({sys.executable}) is missing modules: {", ".join(missing)} (pip install lz4 capstone fonttools backports.zstd; backports.zstd only before Python 3.14)') tools = {'xcodebuild': 'Xcode', 'xcodegen': 'brew install xcodegen', 'dotnet': '.NET SDK 10', 'zstd': 'brew install zstd', 'spirv-cross': 'brew install spirv-cross', 'make': 'Xcode'} + if mac: del tools['xcodegen'] gone = [f'{t} ({how})' for t, how in tools.items() if not shutil.which(t, path=env['PATH'])] if gone: raise BuildError('tools not found: ' + ', '.join(gone)) - mtl = subprocess.run(['xcrun', '-sdk', 'iphoneos', 'metal', '--version'], capture_output=True, text=True, env=env) # Xcode 26 and later install the Metal toolchain separately + mtl = subprocess.run(['xcrun', '-sdk', 'macosx' if mac else 'iphoneos', 'metal', '--version'], capture_output=True, text=True, env=env) # Xcode 26 and later install the Metal toolchain separately if mtl.returncode: raise BuildError(f'the Metal compiler does not run ({(mtl.stderr.strip().splitlines() or ["no output"])[-1]}): install it with xcodebuild -downloadComponent MetalToolchain') if not xlate.exists(): raise BuildError(f'the shader translator is missing from the repository: {xlate}') @@ -831,9 +837,13 @@ def apple_silicon(): # hw.optional.arm64 is 1 on Apple silicon (for a Python u return subprocess.run(['sysctl', '-n', 'hw.optional.arm64'], capture_output=True, text=True).stdout.strip() == '1' +HOST_DIRS = ('dm/host', 'dm/gfx', 'dm/gfx/nvn', 'dm/gfx/nvn/include', 'dm/gfx/tex', 'dm/gfx/vk', 'dm/gfx/gl') # the folders the host's compile line takes its sources and headers from (dm/host/Makefile SRC, GFX, GFX_H) + + def build_mac_host(repo, out, env, logs, g): """The Mac host for the profile, built into out: dm/host's own compile line (make -n) with the output moved, since make takes no target path with spaces and dm/host/dmhost may be another build's. - The host runs the game's own arm64 code natively, so an Intel Mac stops here (GitHub issue #54)""" + Reused when the compile line and its inputs (the source files of HOST_DIRS, the chain header dm/gfx/nvn/dmsh_chain.h among them, the profile header make generates, libvpx and the third-party headers, the compiler + and the macOS SDK) are as at the last build into out. The host runs the game's own arm64 code natively, so an Intel Mac stops here (GitHub issue #54)""" if not apple_silicon(): raise BuildError('this step boots the game in the Mac host, which runs the game\'s own arm64 code natively: it needs a Mac with Apple silicon, and this Mac has an Intel processor') host = repo / 'dm/host' run(['make', '-s', f'GAME={g.id}', f'gen/{g.id}/game_profile.h', 'gen/vpx/libvpx.a'], logs / 'mac_host_make.log', env=env, cwd=host) @@ -841,8 +851,14 @@ def build_mac_host(repo, out, env, logs, g): line = next((l for l in dry.stdout.splitlines() if ' -o dmhost ' in l), None) if not line: raise BuildError(f'make -n gave no compile line for the Mac host: {dry.stderr.strip()[-500:]}') cmd = shlex.split(line); cmd[cmd.index('-o') + 1] = out - out.parent.mkdir(parents=True, exist_ok=True) + kf, tp = out.with_name(out.name + '.key'), repo / 'dm/third_party' + srcs = [p for d in HOST_DIRS for p in sorted((repo / d).iterdir()) if p.is_file() and p.suffix in ('.c', '.m', '.h', '.S', '.inc')] + key = fingerprint(cmd, ('files', srcs + [host / f'gen/{g.id}/game_profile.h', host / 'gen/vpx/libvpx.a', tp / 'stb_vorbis/stb_vorbis.c']), + ('tree', tp / 'vulkan-headers/include'), ('tree', tp / 'libvpx/vpx'), ('cmd', ['clang', '--version']), ('cmd', ['xcrun', '--show-sdk-build-version']), env=env) + if out.is_file() and reused(kf, key) is not None: return out + rmtree(kf); out.parent.mkdir(parents=True, exist_ok=True) run(cmd, logs / 'mac_host_build.log', env=env, cwd=host) + kf.write_text(f'{key}\n') return out @@ -1172,10 +1188,10 @@ def split_shaders(py, repo, roms, mains, pkg, env, logs, g, provisional=False, o class ShaderWatch: """Progress of dmsh_build.py, read from the files it writes (it prints nothing until it ends). It works in three phases: - the cache check (a hit restores <key>_b.json into the iOS output b-<chain>/, a miss writes <key>.bin into its temporary folder dmsh_* under WORK), + the cache check (a hit restores <key>_b.json into the library directory b-<chain>/: the iOS one, or the Mac's for a Mac-only build; a miss writes <key>.bin into its temporary folder dmsh_* under WORK), the translation of the misses (<key>.spv in the temporary folder), and the compile, which takes the keys in a fixed order (sorted, the vertex shaders with pixel pairs last) and writes <key>_b.json again for every miss and every pixel shader: the furthest key written so far tells how far it got""" - def __init__(self, g, out, tmp): # out: the iOS library directory b-<chain>/ + def __init__(self, g, out, tmp): # out: the library directory b-<chain>/ (the iOS one, or the Mac's for a Mac-only build) keys = set() for ln in open(g.table('shader_map.tsv'), errors='replace'): # the keys dmsh_build.py builds: column 4 of the rows from a shader package c = ln.rstrip('\n').split('\t') @@ -1186,7 +1202,7 @@ class ShaderWatch: self.out, self.tmp, self.t0, self.t1, self.w = out, tmp, time.time(), None, None self.hits, self.fin, self.top = set(), set(), -1 - def written(self, keys, since): # those of keys whose <key>_b.json in the iOS output was written after since + def written(self, keys, since): # those of keys whose <key>_b.json in the output was written after since found = set() try: with os.scandir(self.out) as it: @@ -1217,30 +1233,41 @@ def toolchain(): # the compilers' versions, part of the shader stages' keys return ('cmd', ['xcodebuild', '-version']), ('cmd', ['spirv-cross', '--revision']) -def build_shaders(py, repo, sh, xlate, env, logs, g, pkg_key=None): - """The iOS shader libraries (dmsh_build.py). Reused when the shader package (pkg_key, its content; None never reuses) and the chain's tools, translator binaries, profile and compilers - are as at the last successful build: dmsh_build.py would only restore every shader from its cache (63,000 for Breath of the Wild: about 100 s)""" +def build_shaders(py, repo, sh, xlate, env, logs, g, pkg_key=None, mac=False): + """The iOS shader libraries (dmsh_build.py, which builds the Mac's into sh/cache in the same run), or with mac only the Mac's (a Mac-only build: no IOSOUT). Returns the folder the host takes + them from. Reused when the shader package (pkg_key, its content; None never reuses) and the chain's tools, translator binaries, profile and compilers are as at the last successful build: + dmsh_build.py would only restore every shader from its cache (63,000 for Breath of the Wild: about 100 s). The key of the iOS libraries is sh/ios/b-<chain>.key, that of the Mac's + sh/mac-b-<chain>.key (not in sh/cache, whose b-* entries dmsh_build.py reads as library folders), which an iOS build writes too, so a Mac-only build after it is reused; either is taken only + while the Mac libraries carry a full build's stamp (inputs.tsv) for c1_full.tsv and cb_extent.tsv to name (chain_tables). + An iOS build after a Mac-only one translates the shaders again (dmsh_build.py's per-shader cache keys include the iOS compile convention), but takes the Mac libraries from its + Metal compile cache (dmsh_mtlcache.py), so only the iPhone's are compiled""" ios, log = sh / 'ios', logs / 'dmsh_build.log' - PROG.stage('iOS shaders') + PROG.stage('Mac shaders' if mac else 'iOS shaders') chain = chain_of(repo) - libs = ios / f'b-{chain}'; kf = libs.with_name(libs.name + '.key') + out = sh / 'cache' if mac else ios + libs, kf, kmac = out / f'b-{chain}', ios / f'b-{chain}.key', sh / f'mac-b-{chain}.key' if pkg_key: PROG.set(step='checking the inputs') key = fingerprint(pkg_key, ('files', chain_files(repo)), ('stat', xlate.parent), ('files', profile_files(g)), *toolchain(), str(xlate), {k: env.get(k) for k in ('XLATE_DEBUG', 'XLATE_REDUCED')}, env=env) + note = next((n for k in ((kmac, kf) if mac else (kf,)) if (n := reused(k, key)) is not None), None) # the libraries are the last build's, so are tables that name their stamp; a missing or stale one is made from that build's Mac outputs, and without them the step runs again - if next(libs.glob('*_b.metallib'), None) and (note := reused(kf, key)) is not None and chain_tables(py, repo, sh, env, logs, g, chain): - PROG.end(f'{note}, reused (inputs unchanged)'); return ios + if next(libs.glob('*_b.metallib'), None) and note is not None and chain_tables(py, repo, sh, env, logs, g, chain): + PROG.end(f'{note}, reused (inputs unchanged)'); return out else: key = None - rmtree(kf) + rmtree(kmac) + if not mac: rmtree(kf) try: PROG.watch(ShaderWatch(g, libs, Path(env['WORK']))) except Exception: pass # progress never stops the build (a missing table is dmsh_build.py's to report) - run([py, repo / 'dm/gfx/tools/own_compilers.py', sh / 'pkg', sh / 'cache'], log, env=dict(env, XLATE=str(xlate), IOSOUT=str(ios))) # dmsh_build.py with this Mac's Metal compiler and spirv-cross + benv = dict(env, XLATE=str(xlate), IOSOUT=str(ios)) + if mac: del benv['IOSOUT'] # dmsh_build.py builds the iOS libraries only with IOSOUT + run([py, repo / 'dm/gfx/tools/own_compilers.py', sh / 'pkg', sh / 'cache'], log, env=benv) # dmsh_build.py with this Mac's Metal compiler and spirv-cross PROG.watch(None) if not chain_tables(py, repo, sh, env, logs, g, chain): raise BuildError(f'no stamped Mac shader outputs in {sh / "cache" / f"b-{chain}"} (inputs.tsv): dmsh_build.py did not finish a full build') + what = 'Mac' if mac else 'iOS' n = sum(1 for _ in libs.glob('*_b.metallib')) if libs.is_dir() else 0 - if n == 0: raise BuildError(f'no iOS shader libraries in {libs}') - say(f'{n} iOS shader libraries ({libs})') + if n == 0: raise BuildError(f'no {what} shader libraries in {libs}') + say(f'{n} {what} shader libraries ({libs})') t = log.read_text(errors='replace') if own := re.search(r'^note: (built with this Mac\'s compilers.*)$', t, re.M): say(own.group(1)) m, c = re.search(r'shaders (\d+): ok (\d+)', t), re.search(r'shader cache: hit (\d+)', t) @@ -1251,49 +1278,50 @@ def build_shaders(py, repo, sh, xlate, env, logs, g, pkg_key=None): note = f'{ok:,} of {total:,} shaders ready{failed}' PROG.end(f'{ok:,} of {total:,} shaders ready' + (f' ({int(c.group(1)):,} from the cache)' if c else '') + failed) else: PROG.end() - if key: kf.write_text(f'{key}\n{note}\n') - return ios + if key: + for k in (kmac,) if mac else (kf, kmac): k.write_text(f'{key}\n{note}\n') + return out -def build_vk_shaders(py, repo, roots, sh, env, logs, g, reuse=True): - """A Vulkan game's SPIR-V shaders (roots: the RomFS, and a run-time capture folder) → one Metal library each for the Mac (sh/vk/) and the iPhone (sh/ios/vk/, what build_ipa.sh takes from MSL_IOS), - replacing the NVN split and chain build. Reused when the roots (paths and sizes), the tools, the profile and the compilers are as at the last successful run (reuse False: never). - Otherwise the old output is deleted first, so a failed run leaves nothing for build_ipa.sh to pick up; vk_shaders.py's temporary files go to WORK (on the NSP folder's volume)""" - ios = sh / 'ios'; kf = ios / 'vk.key' +def layer_shaders(py, repo, api, name, roots, sh, key, logs, env, mac): + """What build_vk_shaders and build_gl_shaders share: dm/gfx/tools/<api>_shaders.py on the roots → sh/<api>/ for the Mac and, unless mac (a Mac-only build), sh/ios/<api>/ for the iPhone. + Returns the folder the host (DM_MSL_DIR) or build_ipa.sh (MSL_IOS) takes the libraries from. key: the inputs' fingerprint (None never reuses). The iOS libraries' key is sh/ios/<api>.key, + the Mac's sh/<api>.key, which an iOS build writes too (it builds both), so a Mac-only build is reused after either; an iOS build after a Mac-only one translates again. + Otherwise the old output is deleted first, so a failed run leaves nothing for build_ipa.sh or the host to pick up; the translator's temporary files go to WORK (on the NSP folder's volume)""" + ios = sh / 'ios'; out = sh if mac else ios + kf, kmac, lib, log = ios / f'{api}.key', sh / f'{api}.key', out / api / f'{api}_shaders.metallib', logs / f'{api}_shaders.log' + note = next((n for k in ((kmac, kf) if mac else (kf,)) if (n := reused(k, key)) is not None), None) if key else None + if lib.is_file() and note is not None: + PROG.end(f'{note}, reused (inputs unchanged)'); return out + PROG.set(step='translating') + rmtree(kmac); rmtree(sh / api) + if not mac: rmtree(kf); rmtree(ios / api) + run([py, repo / f'dm/gfx/tools/{api}_shaders.py', *roots, sh] + ([] if mac else ['--ios', ios]), log, env=dict(env, TMPDIR=env['WORK'])) + if not lib.is_file(): raise BuildError(f'no {"Mac" if mac else "iOS"} {name} shader library in {lib.parent}') + m = re.search(r'macosx: (\d+)' if mac else r'iphoneos: (\d+)', log.read_text(errors='replace')) + note = f'{int(m.group(1)):,} shaders for the Mac' + ('' if mac else ' and iOS') if m else '' + if key: + for k in (kmac,) if mac else (kf, kmac): k.write_text(f'{key}\n{note}\n') + PROG.end(note) + return out + + +def build_vk_shaders(py, repo, roots, sh, env, logs, g, reuse=True, mac=False): + """A Vulkan game's SPIR-V shaders (roots: the RomFS, and a run-time capture folder) → one Metal library each for the Mac (sh/vk/) and the iPhone (sh/ios/vk/, what build_ipa.sh takes from MSL_IOS; + not with mac), replacing the NVN split and chain build (layer_shaders). Reused when the roots (paths and sizes), the tools, the profile and the compilers are as at the last successful run (reuse False: never)""" PROG.stage('Vulkan shaders', step='checking the inputs') key = fingerprint(*[('tree', r) for r in roots], ('files', tool_files(repo)), ('files', profile_files(g)), *toolchain(), env=env) if reuse else None - if key and (ios / 'vk/vk_shaders.metallib').is_file() and (note := reused(kf, key)) is not None: - PROG.end(f'{note}, reused (inputs unchanged)'); return ios - PROG.set(step='translating') - rmtree(kf); rmtree(sh / 'vk'); rmtree(ios / 'vk') - run([py, repo / 'dm/gfx/tools/vk_shaders.py', *roots, sh, '--ios', ios], logs / 'vk_shaders.log', env=dict(env, TMPDIR=env['WORK'])) - if not (ios / 'vk/vk_shaders.metallib').is_file(): raise BuildError(f'no iOS Vulkan shader library in {ios / "vk"}') - m = re.search(r'iphoneos: (\d+)', (logs / 'vk_shaders.log').read_text(errors='replace')) - note = f'{int(m.group(1)):,} shaders for the Mac and iOS' if m else '' - if key: kf.write_text(f'{key}\n{note}\n') - PROG.end(note) - return ios + return layer_shaders(py, repo, 'vk', 'Vulkan', roots, sh, key, logs, env, mac) -def build_gl_shaders(py, repo, roots, sh, env, logs, g, reuse=True): - """An OpenGL game's GLSL sources (roots: the RomFS, main, a run-time capture folder) → one Metal library each for the Mac (sh/gl/) and the iPhone (sh/ios/gl/, what build_ipa.sh takes from MSL_IOS), - replacing the NVN split and chain build, as build_vk_shaders does for a Vulkan game; glslang (glslangValidator) compiles the GLSL to SPIR-V for spirv-cross""" - ios = sh / 'ios'; kf = ios / 'gl.key' +def build_gl_shaders(py, repo, roots, sh, env, logs, g, reuse=True, mac=False): + """An OpenGL game's GLSL sources (roots: the RomFS, main, a run-time capture folder) → one Metal library each for the Mac (sh/gl/) and the iPhone (sh/ios/gl/, what build_ipa.sh takes from MSL_IOS; + not with mac), replacing the NVN split and chain build, as build_vk_shaders does for a Vulkan game; glslang (glslangValidator) compiles the GLSL to SPIR-V for spirv-cross""" PROG.stage('GL shaders', step='checking the inputs') if not shutil.which('glslangValidator', path=env['PATH']): raise BuildError('glslangValidator not found (brew install glslang): it compiles an OpenGL game\'s GLSL shaders') key = fingerprint(*[('tree', r) if Path(r).is_dir() else ('files', [r]) for r in roots], ('files', tool_files(repo)), ('files', profile_files(g)), *toolchain(), ('cmd', ['glslangValidator', '--version']), env=env) if reuse else None - if key and (ios / 'gl/gl_shaders.metallib').is_file() and (note := reused(kf, key)) is not None: - PROG.end(f'{note}, reused (inputs unchanged)'); return ios - PROG.set(step='translating') - rmtree(kf); rmtree(sh / 'gl'); rmtree(ios / 'gl') - run([py, repo / 'dm/gfx/tools/gl_shaders.py', *roots, sh, '--ios', ios], logs / 'gl_shaders.log', env=dict(env, TMPDIR=env['WORK'])) - if not (ios / 'gl/gl_shaders.metallib').is_file(): raise BuildError(f'no iOS GL shader library in {ios / "gl"}') - m = re.search(r'iphoneos: (\d+)', (logs / 'gl_shaders.log').read_text(errors='replace')) - note = f'{int(m.group(1)):,} shaders for the Mac and iOS' if m else '' - if key: kf.write_text(f'{key}\n{note}\n') - PROG.end(note) - return ios + return layer_shaders(py, repo, 'gl', 'GL', roots, sh, key, logs, env, mac) class IpaWatch: @@ -1346,6 +1374,43 @@ class IpaWatch: return r +def env_file_pairs(path): # an environment file's switches (one KEY=VALUE per line, # starts a comment), read as the iOS app reads them (dm/ios/Sources/main.m) + out = {} + try: text = Path(os.path.expanduser(path)).read_text(errors='replace') + except OSError as e: raise BuildError(f'cannot read the environment file (--env-file / DM_ENV_FILE): {e}') from e + for ln in text.splitlines(): + k, eq, v = ln.strip().partition('=') + if k.strip() and eq and not k.startswith('#'): out[k.strip()] = v + return out + + +def mac_run(py, repo, work, game, patched, msl, aoc, gfx, lang, env, logs, g, env_file=''): + """The last stage of a Mac-only build (PLATFORM 'mac'): the Mac host for the profile (build_mac_host, into <id>_build/tools/dmhost, where the run-time captures build it too; DM_GAMES_DIR / + DM_GAMES_FIRST in env pick a provisional or derived profile) and the button-icon font the host gives a game for the Nintendo extension shared font (make_ext_font.py, into <id>_build/font, + made again when the tool changed). Returns the command that runs the game in a window with sound (docs/development.md, Build by hand (Mac), step 5): its save and temporary storage in + <id>_build/run (kept from run to run), the extracted RomFS, the Mac shader libraries (msl), the DLC, the profile's shader map (a host outside dm/host does not find it beside itself; + c1_full.tsv and cb_extent.tsv are read beside it), the game language, and the switches of an environment file (DM_ENV_FILE / --env-file)""" + PROG.stage('Mac host', step='building') + host = build_mac_host(repo, work / 'tools/dmhost', env, logs, g) + PROG.set(step='button-icon font') + font = work / 'font'; ttf, kf = font / 'nintendo_ext.ttf', font / 'nintendo_ext.key' + key = fingerprint(('files', [repo / 'dm/tools/make_ext_font.py'])) + if not (ttf.is_file() and reused(kf, key) is not None): + rmtree(kf); font.mkdir(parents=True, exist_ok=True) + run([py, repo / 'dm/tools/make_ext_font.py', ttf], logs / 'make_ext_font.log', env=env) + kf.write_text(f'{key}\n') + rd = work / 'run' + for d in ('save', 'temp'): (rd / d).mkdir(parents=True, exist_ok=True) + renv = {'DM_ROOT': work, 'DM_ROMFS': game / 'RomFS', 'DM_SAVE': rd / 'save', 'DM_TEMP': rd / 'temp', 'DM_MSL_DIR': msl, 'DM_FONT_DIR': font} + if aoc: renv['DM_AOC_DIR'] = aoc + if gfx == 'nvn': renv['DM_NVN_SHMAP'] = g.table('shader_map.tsv') + renv.update(DM_LANG=lang, DM_WINDOW=1, DM_AUDIO=1) + if env_file: renv.update(env_file_pairs(env_file)) + PROG.end(str(host)) + q = lambda v: shlex.quote(str(v)) + return f'cd {q(rd)} && ' + ' '.join(f'{k}={q(v)}' for k, v in renv.items()) + f' {q(host)} {q(patched)}' + + def find_repo(here): # this file lives in the repository's scripts/; walk up to the level that has dm/tools/nx_extract/Program.cs for d in here.parents: if (d / 'dm/tools/nx_extract/Program.cs').exists(): return d @@ -1365,6 +1430,8 @@ def parse_args(argv): ap.add_argument('--app-name', help='app name on the home screen (default: the game name in the base NSP)') ap.add_argument('--bundle-id', help='the app\'s bundle ID (default: saved Xcode ID when preparing again, else org.nxnative.<base game title ID in lowercase>)') ap.add_argument('--prepare-xcode', action='store_true', help='prepare a per-game Xcode project and add it to the shared workspace instead of packing an IPA (requires keeping extracted files)') + ap.add_argument('--platform', choices=('ios', 'mac'), help='ios = the IPA (or the Xcode project); mac = for development: build only what running the game on this Mac needs (the Mac shaders and the Mac host, ' + 'no iOS shaders, Xcode project or IPA), keep the extracted files and print the command that runs the game (default: the PLATFORM setting, ios)') ap.add_argument('--team-id', help='Apple developer team ID (default: TEAM_ID setting, DM_TEAM_ID environment, or saved Xcode selection)') ap.add_argument('--xcode-workspace', help='workspace collecting prepared games (default: NxNative.xcworkspace beside the NSP folder; use with --prepare-xcode)') ap.add_argument('-l', '--language', help='game language, a code or a name: ja, fr, de, es, it, nl, pt, ru, ko, zh-Hans, zh-Hant, en-GB, fr-CA, es-419, pt-BR, Japanese, ... (default: the LANGUAGE setting, else English); ' @@ -1382,9 +1449,8 @@ def parse_args(argv): return ap.parse_args(argv) -def report_profile_status(g, reg, pkg_ver, derived, provisional, prepare_xcode=False): - """Keep build-specific warnings visible for both IPA and Xcode output.""" - artifact = 'Xcode project' if prepare_xcode else 'IPA' +def report_profile_status(g, reg, pkg_ver, derived, provisional, artifact='IPA'): + """Keep build-specific warnings visible for the IPA, the Xcode project and a Mac-only build""" disp = g.builds()[pkg_ver]['display'] if derived else '' if derived: # the modules whose build no registered version has: the rewriter's cuts and the host's fixes made for a registered build's code do not reach them known = {(m, b) for row in reg.builds().values() for m, b in row.items() if m != 'display'} @@ -1404,7 +1470,13 @@ def main(args=None): here = Path(globals().get('__file__') or sys.argv[0]).resolve() repo = find_repo(here) prepare_xcode = args.prepare_xcode or PREPARE_XCODE + platform = args.platform or PLATFORM + if platform not in ('ios', 'mac'): raise BuildError(f'PLATFORM / --platform must be ios or mac: {platform}') + mac = platform == 'mac' # a Mac-only build for development: no iOS shaders, Xcode project or IPA (mac_run) + if mac and prepare_xcode: raise BuildError('--prepare-xcode makes an iPhone project: it cannot be combined with --platform mac') + if mac and not apple_silicon(): raise BuildError('--platform mac runs the game in the Mac host, which runs the game\'s own arm64 code natively: it needs a Mac with Apple silicon, and this Mac has an Intel processor') keep = KEEP_EXTRACTED if args.keep_extracted is None else args.keep_extracted + if mac and not keep: say('keeping the extracted game files all the same: the game runs from them on the Mac'); keep = True if prepare_xcode and not keep: raise BuildError('--prepare-xcode requires --keep-extracted: Xcode builds from the extracted files') if args.xcode_workspace and not prepare_xcode: raise BuildError('--xcode-workspace requires --prepare-xcode') nsp_dir = Path(os.path.expanduser(args.nsp_dir or NSP_DIR or ask('NSP folder: ', 'the NSP folder'))).resolve() @@ -1472,20 +1544,20 @@ def main(args=None): if not saved_signing and legacy_config.exists() and f'/gen/{g.id}/' in legacy_config.read_text(): saved_signing = xcode_projects.signing_settings(repo / 'dm/ios/DmIOS.xcodeproj') except ValueError as e: raise BuildError(str(e)) from e - say(f'Xcode project: {project}' if prepare_xcode else f'output IPA: {out_ipa}') + say('Mac-only build for development: the Mac shaders and the Mac host, no iOS shaders, Xcode project or IPA' if mac else f'Xcode project: {project}' if prepare_xcode else f'output IPA: {out_ipa}') say(f'work folder (extracted game files in extract/, {"kept for the next build" if keep else "deleted after the IPA is built"}): {work}') xlate = repo / 'dm/gfx/tools/dmsh_xlate_bin/dmsh_xlate' # DM_GAME picks the profile; DM_ROOT / WORK point at <id>_build: the tools' default paths and temporary directories all stay inside it env.update(DM_GAME=g.id, DM_ROOT=str(work), WORK=str(work / 'tmp')) logs = work / 'logs' - check_prereqs(env, xlate) + check_prereqs(env, xlate, mac) picks = pick_nsps(fam, have, versions) ex = work / 'extract'; ex.mkdir(parents=True, exist_ok=True); (work / 'tmp').mkdir(exist_ok=True) gb = sum({p.stat().st_size for b, u in picks.values() for p in (b, u) if p}) / 2**30 + 6 # extraction + shader cache + slack if free_gb(work) < gb and not all((ex / v).exists() for v in versions): raise BuildError(f'less than {gb:.0f} GB free on the volume of {work}') heapck = provisional and HEAP_CHECK and not args.no_heap_check and not heap_checked(g) # the heap check, when the game needs one (once per provisional profile: it stamps the profile) - PROG.plan(1 + len(versions) + bool(fam['dlc']) + 4 - prepare_xcode) # prepare Xcode replaces app build + IPA pack with one stage + PROG.plan(1 + len(versions) + bool(fam['dlc']) + 4 - (prepare_xcode or mac)) # prepare Xcode (a Mac-only build: the Mac host) replaces app build + IPA pack with one stage PROG.stage('build nx_extract') nx = build_nx_extract(repo, work, logs, env) PROG.end() @@ -1525,7 +1597,7 @@ def main(args=None): base = next((dirs[v] for v in versions if picks[v][1] is None), dirs[versions[0]]) # the folder the base game was extracted to alone # the app name and bundle ID both come from the base NSP: the game name in its control data, and a bundle ID from its title ID; a setting / the command line overrides them app_name = args.app_name or APP_NAME or nacp.name(base / 'control.nacp') - if not app_name: raise BuildError(f'no game name in the base extraction ({base / "control.nacp"}; an older extraction did not save it: delete extract/ and run again, or give --app-name)') + if not app_name and not mac: raise BuildError(f'no game name in the base extraction ({base / "control.nacp"}; an older extraction did not save it: delete extract/ and run again, or give --app-name)') bundle_id = args.bundle_id or BUNDLE_ID or saved_signing.get('PRODUCT_BUNDLE_IDENTIFIER') or f'org.nxnative.{app:016x}' team_id = args.team_id or TEAM_ID or env.get('DM_TEAM_ID') or saved_signing.get('DEVELOPMENT_TEAM') or '0000000000' if prepare_xcode: @@ -1533,12 +1605,12 @@ def main(args=None): except ValueError as e: raise BuildError(str(e)) from e # the app version is the packed game's display version: the update's control data when an update is packed (nx_extract saves it next to the RomFS), else the base game's app_version = nacp.version(game / 'control.nacp') or nacp.version(base / 'control.nacp') - say(f'app name: {app_name}; version: {app_version or "1.0"}; bundle ID: {bundle_id}') + if not mac: say(f'app name: {app_name}; version: {app_version or "1.0"}; bundle ID: {bundle_id}') # the game language: Horizon gives the game one of the languages its control data lists (the packed version's, which an update may extend), so the one packed must be among them have = nacp.languages(game / 'control.nacp') or nacp.languages(base / 'control.nacp') lang = game_language(want_lang, bool(lang_in), have) say(f'game language: {lang} ({dict(nacp.LANGUAGES)[lang]})' + (f', the game has no {want_lang}' if lang != want_lang else '') - + (f'; the game\'s languages: {" ".join(have)}' if have else '; the control data lists no languages (not checked)') + ' (DM_LANG=<code> in the app\'s Documents/env.txt changes it)') + + (f'; the game\'s languages: {" ".join(have)}' if have else '; the control data lists no languages (not checked)') + (' (DM_LANG=<code> in the run command changes it)' if mac else ' (DM_LANG=<code> in the app\'s Documents/env.txt changes it)')) patched = work / f'patched_{pkg_ver}' rewrite(py, repo, game / 'ExeFS', patched, env, logs, g) @@ -1554,15 +1626,22 @@ def main(args=None): sh = work / 'shader' if gfx == 'vulkan': # the packed version's merged RomFS holds the SPIR-V files the game loads; a capture holds those it generates cap = capture_vk(repo, game, patched, work, env, logs, g) if vkcap else None - ios = build_vk_shaders(py, repo, [game / 'RomFS'] + ([cap] if cap else []), sh, env, logs, g, reuse=not (provisional or derived or ALLOW_DAMAGED)) + ios = build_vk_shaders(py, repo, [game / 'RomFS'] + ([cap] if cap else []), sh, env, logs, g, reuse=not (provisional or derived or ALLOW_DAMAGED), mac=mac) elif gfx == 'gl': # the GLSL sources in the packed version's archives and in its main (the game compiles them at run time; the host finds each by the hash of its text) - ios = build_gl_shaders(py, repo, [game / 'RomFS', game / 'ExeFS/main'], sh, env, logs, g, reuse=not (provisional or derived or ALLOW_DAMAGED)) + ios = build_gl_shaders(py, repo, [game / 'RomFS', game / 'ExeFS/main'], sh, env, logs, g, reuse=not (provisional or derived or ALLOW_DAMAGED), mac=mac) else: # Trinity packs and legacy Unreal paks: the split decodes them with the packed version's own decoder (Oodle, LZ5), in the Mac host built for the profile oodle = (lambda: {'DM_OODLE_HOST': str(build_mac_host(repo, work / 'tools/dmhost', env, logs, g)), 'DM_OODLE_MODULES': str(patched)}) if src['oodle_packs'][0] or src['ue_legacy'][0] else None own = bool(provisional or derived) # the shader tables come from this build's split (the profile's are checked otherwise) pkey = split_shaders(py, repo, [dirs[v] / 'RomFS' for v in sv], [dirs[v] / 'ExeFS/main' for v in sv], sh / 'pkg', env, logs, g, own, oodle) - ios = build_shaders(py, repo, sh, xlate, env, logs, g, pkey) + ios = build_shaders(py, repo, sh, xlate, env, logs, g, pkey, mac) + if mac: # a Mac-only build ends with the Mac host and the command that runs the game + cmd = mac_run(py, repo, work, game, patched, ios, aoc, gfx, lang, env, logs, g, args.env_file or DM_ENV_FILE) + rmtree(work / 'tmp') + say(f'done: the game runs on the Mac with this command (its save in {work / "run/save"}, kept from run to run; DM_WINDOW=1 opens a window, DM_AUDIO=1 plays sound):') + PROG.say(cmd) + report_profile_status(g, reg, pkg_ver, derived, provisional, 'Mac build') + return if not prepare_xcode and free_gb(out_ipa.parent) < 5: raise BuildError(f'less than 5 GB free on the volume of {out_ipa.parent}') say('preparing the Xcode project' if prepare_xcode else f'building the IPA → {out_ipa}') @@ -1589,7 +1668,7 @@ def main(args=None): say(f'game project: {project}') say(f'Xcode derived data: {work / "xcode-derived-data"}') say('choose the game scheme in Xcode, select your Team and iPhone or iPad, then Run (Release)') - report_profile_status(g, reg, pkg_ver, derived, provisional, prepare_xcode=True) + report_profile_status(g, reg, pkg_ver, derived, provisional, 'Xcode project') return watch = IpaWatch(repo, work / 'ipa', out_ipa) PROG.stage('app build', poll=watch.poll, step='starting') # build_ipa.sh's "pack IPA" line starts the stage "IPA pack"