NSP script: --platform mac (PLATFORM = 'mac') builds only what running a game on this Mac needs, from PR #82

A development option; the Mac is the development host, not a platform for players. The same steps up to the
shaders (extraction, DLC, rewrite, run-time captures and decodes, heap check, shader split), then only the
Mac's: dmsh_build.py without IOSOUT (an inherited one is dropped), a Vulkan or GL game's library without --ios,
the Mac host for the profile into <id>_build/tools/dmhost (build_mac_host, through DM_GAMES_DIR /
DM_GAMES_FIRST for a provisional or derived profile) and the button-icon font into <id>_build/font. No iOS
shaders, Xcode project or IPA; the extracted files are kept whatever KEEP_EXTRACTED says, and the last line is
the command that runs the game (its save in <id>_build/run/save, DM_NVN_SHMAP for an NVN game, DM_AOC_DIR
with DLC, the --env-file switches). It refuses --prepare-xcode and an Intel Mac.

Reuse: the Mac libraries have a key of their own (shader/mac-b-<chain>.key beside the cache folder,
shader/vk.key / gl.key), which an iOS build writes too, so a Mac-only build after an IPA reuses its shaders;
either key only while the Mac libraries carry a full build's stamp (inputs.tsv). An iOS build after a Mac-only
one translates again (dmsh_build.py's per-shader keys include the iOS compile convention) and takes the Mac
libraries from the Metal compile cache. build_mac_host now has a reuse key too (the compile line, the sources and
headers of the folders it compiles, the generated profile header, libvpx, the third-party headers, clang and the
SDK), which also saves the rebuild for the Oodle / LZ5 decodes and the run-time captures. build_vk_shaders and
build_gl_shaders share layer_shaders. The default (ios) path builds as before.

Checked: dm/tools/tests 34 of 34 (11 in the new test_nsp_mac.py); Hades in Mac mode ran the Mac stages only,
printed the run command and exited 0, and a second run reused every stage, the host included (2 s); the host it
built boots Hades from that command. Breath of the Wild (NVN, with DLC) in Mac mode restored its 19,592 shaders
from the cache, printed a command with DM_AOC_DIR and DM_NVN_SHMAP, and the host booted it from that command into
play. Not run end to end here: a GL, provisional or derived game in Mac mode, and an IPA after a Mac-only build
(unit tests with the tools stubbed).

Co-authored-by: v0lumehi <34629096+v0lumehi@users.noreply.github.com>
This commit is contained in:
seeddsandv0lumehi committed 2026-10-24 01:40:58 +08:00
1 parent e98a11562a
commit 98a91a6749
3 files changed
+290 -66

No files matched your search

+124
View File
@@ -0,0 +1,124 @@
"""NSP script Mac-only build (scripts/build_ipa_from_nsp.py --platform mac): the shader steps build only the Mac's libraries, each output keeps its own reuse key, a Mac-only build is
reused after an iOS build (which builds the Mac's too) and an iOS build is not reused after a Mac-only one, neither key without the Mac libraries' stamp (inputs.tsv, which the one
chain version's c1_full.tsv / cb_extent.tsv name); the run command's environment file is read as the iOS app reads it.
No game files, keys or Xcode needed (temporary files only in a temporary folder)."""
import importlib.util
from pathlib import Path
import shutil
import sys
import tempfile
import types
import unittest
from unittest import mock
REPO = Path(__file__).resolve().parents[3]
SCRIPT = REPO / 'scripts/build_ipa_from_nsp.py'
spec = importlib.util.spec_from_file_location('build_ipa_from_nsp', SCRIPT)
nsp = importlib.util.module_from_spec(spec); spec.loader.exec_module(nsp)
CHAIN, INPUTS = '0123456789ab', 'aaaaaaaaaaaa'
class Base(unittest.TestCase):
def setUp(self):
self.tmp = Path(tempfile.mkdtemp()); self.addCleanup(shutil.rmtree, self.tmp)
self.sh, self.logs, self.ran = self.tmp / 'shader', self.tmp / 'logs', []
self.logs.mkdir()
for cm in (mock.patch.object(nsp, 'PROG', mock.Mock()), mock.patch.object(nsp, 'say', lambda m: None), mock.patch.object(nsp, 'run', self.fake_run)):
cm.__enter__(); self.addCleanup(cm.__exit__, None, None, None)
def fake_run(self, cmd, log, env=None, cwd=None, on_line=None): # stands in for the translators: writes the libraries a real run writes
self.ran.append(([str(c) for c in cmd], dict(env or {})))
tool = Path(cmd[1]).name
if tool in ('vk_shaders.py', 'gl_shaders.py'):
api, a = tool[:2], [str(c) for c in cmd]
outs = [Path(a[a.index('--ios') - 1]), Path(a[a.index('--ios') + 1])] if '--ios' in a else [Path(a[-1])]
for o in outs: (o / api).mkdir(parents=True, exist_ok=True); (o / api / f'{api}_shaders.metallib').write_bytes(b'lib')
log.write_text('macosx: 3 shaders\n' + ('iphoneos: 3 shaders\n' if '--ios' in a else ''))
elif tool == 'own_compilers.py': # a full dmsh_build.py run: the libraries, their stamp, and the profile's tables naming it
outs = [self.sh / 'cache'] + ([Path(env['IOSOUT'])] if 'IOSOUT' in env else [])
for o in outs:
(o / f'b-{CHAIN}').mkdir(parents=True, exist_ok=True); (o / f'b-{CHAIN}' / 'k_b.metallib').write_bytes(b'lib')
(o / f'b-{CHAIN}' / 'inputs.tsv').write_text(f'# stamp\ngame\t0100535012974000\ninputs\t{INPUTS}\n')
for n in ('c1_full.tsv', 'cb_extent.tsv'): (self.tmp / n).write_text(f'# chain b-{CHAIN} inputs {INPUTS} 2026-10-23T00:00:00+00:00\n')
log.write_text('shaders 1: ok 1\n')
class Layer(Base):
def vk(self, mac, key='K'):
self.ran.clear()
return nsp.layer_shaders(sys.executable, REPO, 'vk', 'Vulkan', [self.tmp / 'romfs'], self.sh, key, self.logs, {'WORK': str(self.tmp)}, mac)
def test_mac_only(self): # no --ios, only the Mac key, the Mac folder for DM_MSL_DIR
self.assertEqual(self.vk(True), self.sh)
self.assertNotIn('--ios', self.ran[0][0])
self.assertTrue((self.sh / 'vk.key').is_file()); self.assertFalse((self.sh / 'ios/vk.key').exists())
self.assertEqual(self.vk(True), self.sh); self.assertEqual(self.ran, []) # reused
def test_mac_after_ios(self): # an iOS build makes the Mac library too: a Mac-only build is reused
self.assertEqual(self.vk(False), self.sh / 'ios')
self.assertIn('--ios', self.ran[0][0])
(self.sh / 'vk.key').unlink() # an iOS build from before Mac-only builds wrote only its own key
self.assertEqual(self.vk(True), self.sh); self.assertEqual(self.ran, [])
def test_ios_after_mac(self): # a Mac-only build made no iPhone library: an iOS build runs
self.vk(True)
self.assertEqual(self.vk(False), self.sh / 'ios'); self.assertEqual(len(self.ran), 1)
self.assertTrue((self.sh / 'ios/vk/vk_shaders.metallib').is_file())
def test_changed_inputs(self):
self.vk(False)
self.vk(True, 'K2'); self.assertEqual(len(self.ran), 1)
self.assertTrue((self.sh / 'ios/vk/vk_shaders.metallib').is_file()) # the iPhone library stays, under its own (older) key
self.assertEqual(nsp.reused(self.sh / 'ios/vk.key', 'K'), '3 shaders for the Mac and iOS')
class Nvn(Base):
def setUp(self):
super().setUp()
self.g = types.SimpleNamespace(table=lambda n: self.tmp / n, dir=self.tmp)
for cm in (mock.patch.object(nsp, 'fingerprint', lambda *a, **k: 'K'), mock.patch.object(nsp, 'chain_of', lambda repo: CHAIN)): # one chain version for every game
cm.__enter__(); self.addCleanup(cm.__exit__, None, None, None)
def build(self, mac):
self.ran.clear()
return nsp.build_shaders(sys.executable, REPO, self.sh, REPO / 'dm/gfx/tools/dmsh_xlate_bin/dmsh_xlate', {'WORK': str(self.tmp), 'IOSOUT': '/elsewhere'}, self.logs, self.g, pkg_key='P', mac=mac)
def test_mac_only(self): # no IOSOUT (not even the caller's), the Mac key beside the cache folder, not in it
self.assertEqual(self.build(True), self.sh / 'cache')
self.assertNotIn('IOSOUT', self.ran[0][1])
self.assertTrue((self.sh / f'mac-b-{CHAIN}.key').is_file()); self.assertEqual([p.name for p in (self.sh / 'cache').iterdir()], [f'b-{CHAIN}'])
self.assertEqual(self.build(True), self.sh / 'cache'); self.assertEqual(self.ran, [])
def test_mac_after_ios(self):
self.assertEqual(self.build(False), self.sh / 'ios')
self.assertEqual(self.ran[0][1]['IOSOUT'], str(self.sh / 'ios'))
(self.sh / f'mac-b-{CHAIN}.key').unlink() # an iOS build from before Mac-only builds
self.assertEqual(self.build(True), self.sh / 'cache'); self.assertEqual(self.ran, [])
def test_ios_after_mac(self):
self.build(True)
self.assertEqual(self.build(False), self.sh / 'ios'); self.assertEqual(len(self.ran), 1)
def test_no_stamp(self): # Mac libraries no full build stamped (an interrupted one): no table can name them, so neither key reuses them
self.build(False)
(self.sh / 'cache' / f'b-{CHAIN}' / 'inputs.tsv').unlink()
self.assertEqual(self.build(True), self.sh / 'cache'); self.assertEqual(len(self.ran), 1)
self.assertTrue((self.sh / 'cache' / f'b-{CHAIN}' / 'inputs.tsv').is_file())
def test_stale_tables(self): # a reused build makes a table again when it does not name the Mac libraries' stamp
self.build(True)
(self.tmp / 'cb_extent.tsv').write_text(f'# chain b-{CHAIN} inputs bbbbbbbbbbbb 2026-10-23T00:00:00+00:00\n')
self.assertEqual(self.build(True), self.sh / 'cache'); self.assertEqual([Path(c[1]).name for c, _ in self.ran], ['cb_extent.py'])
class EnvFile(Base):
def test_pairs(self):
p = self.tmp / 'env.txt'; p.write_text('# a comment\nDM_A=1\n DM_B = x=y\n=nokey\nnoeq\n\nDM_C=\n')
self.assertEqual(nsp.env_file_pairs(p), {'DM_A': '1', 'DM_B': ' x=y', 'DM_C': ''})
def test_missing(self):
with self.assertRaises(nsp.BuildError): nsp.env_file_pairs(self.tmp / 'none.txt')
if __name__ == '__main__':
unittest.main()
+23 -2
View File
@@ -45,9 +45,12 @@ python3 scripts/build_ipa_from_nsp.py ~/Games/NewGame -k ~/keys/prod.keys
# test the guesses on a game that has a profile: build from a provisional one (the IPA is named <ipa>-provisional.ipa) # test the guesses on a game that has a profile: build from a provisional one (the IPA is named <ipa>-provisional.ipa)
python3 scripts/build_ipa_from_nsp.py ~/Games/MarioGalaxy -k ~/keys/prod.keys --provisional --allow-damaged python3 scripts/build_ipa_from_nsp.py ~/Games/MarioGalaxy -k ~/keys/prod.keys --provisional --allow-damaged
# development: build only what running the game on this Mac needs (no iOS shaders, Xcode project or IPA) and print the command that runs it
python3 scripts/build_ipa_from_nsp.py ~/Games/MarioWonder -k ~/keys/prod.keys --platform mac
``` ```
In Spyder, open the script, fill in the settings at the top (`NSP_DIR`, `PROD_KEYS`, and optionally `PACKAGE`, `OUT_IPA`, `APP_NAME`, `BUNDLE_ID`, `LANGUAGE`, `KEEP_EXTRACTED`, `SHOW_FPS`; for a game without a profile `HEAP_CHECK`, `HEAP_CHECK_SECS`, `PROVISIONAL`), and press F5. Empty paths are asked for at run time. In Spyder, open the script, fill in the settings at the top (`NSP_DIR`, `PROD_KEYS`, and optionally `PACKAGE`, `OUT_IPA`, `APP_NAME`, `BUNDLE_ID`, `LANGUAGE`, `KEEP_EXTRACTED`, `SHOW_FPS`, `PLATFORM`; for a game without a profile `HEAP_CHECK`, `HEAP_CHECK_SECS`, `PROVISIONAL`), and press F5. Empty paths are asked for at run time.
What the script does: What the script does:
1. It finds the game profile from the title ID (the `[title ID]` in the file names, or the NSP's ticket; the update by its `[vN]` tag). The update is optional: it packs the newest version in the folder (`-p` / `PACKAGE` picks another). 1. It finds the game profile from the title ID (the `[title ID]` in the file names, or the NSP's ticket; the update by its `[vN]` tag). The update is optional: it packs the newest version in the folder (`-p` / `PACKAGE` picks another).
@@ -77,6 +80,24 @@ The game language (`--language`, `LANGUAGE`; English by default) is what `nn::oe
On the device, environment switches go in `Documents/env.txt` (one `KEY=VALUE` per line), and game files are also accepted at `Documents/romfs`. On the device, environment switches go in `Documents/env.txt` (one `KEY=VALUE` per line), and game files are also accepted at `Documents/romfs`.
### A Mac-only build for development
The Mac is the development host, not a platform for players: there is no Mac app. To test or debug a game on the Mac, `--platform mac` (`PLATFORM = 'mac'` in Spyder) builds only what running it in the Mac host needs, from the same NSP folder:
- the same steps up to the shaders: extraction, rewrite, run-time shader captures and decodes, a provisional profile's heap check and the shader split;
- the Mac shader libraries only: `dmsh_build.py` without `IOSOUT` into `<id>_build/shader/cache/b-<chain>/`, which a full build stamps with the profile's inputs (`inputs.tsv`) and from which it makes the profile's `c1_full.tsv` and `cb_extent.tsv`, or a Vulkan or GL game's library without `--ios` into `<id>_build/shader/vk/` or `gl/`;
- the Mac host for the profile into `<id>_build/tools/dmhost`: the compile line of `make -C dm/host GAME=<id> dmhost`, with a provisional or derived profile through `DM_GAMES_DIR` / `DM_GAMES_FIRST` as for the script's other host builds. It is reused while its sources (the chain header `dm/gfx/nvn/dmsh_chain.h` among them), the generated profile header, libvpx and the compiler are unchanged;
- the button-icon font (`make_ext_font.py`) into `<id>_build/font/`.
It skips the iOS shaders, the Xcode project and the IPA. For Mario Kart 8 Deluxe that saves about 30,000 iOS shader compiles and the 12 GB IPA. It keeps the extracted files even with `--no-keep-extracted`, because the game runs from them. It needs a Mac with Apple silicon, and it cannot be combined with `--prepare-xcode`. The last line it prints is the command that runs the game, built from the run's own paths (step 5 of [Build by hand (Mac)](#build-by-hand-mac)), for example:
```
cd "<NSP folder>/<id>_build/run" && DM_ROOT="<NSP folder>/<id>_build" DM_ROMFS="<NSP folder>/<id>_build/extract/<version>/RomFS" DM_SAVE="<NSP folder>/<id>_build/run/save" DM_TEMP="<NSP folder>/<id>_build/run/temp" DM_MSL_DIR="<NSP folder>/<id>_build/shader/cache" DM_FONT_DIR="<NSP folder>/<id>_build/font" DM_NVN_SHMAP="<repository>/dm/games/<id>/shader_map.tsv" DM_LANG=en-US DM_WINDOW=1 DM_AUDIO=1 "<NSP folder>/<id>_build/tools/dmhost" "<NSP folder>/<id>_build/patched_<version>"
```
The save is in `<id>_build/run/save` and stays there from run to run. `DM_AOC_DIR` is added when the folder has DLC. `DM_NVN_SHMAP` is added for an NVN game; it names the profile's shader map, or the derived or provisional profile's. A Vulkan or GL game's `DM_MSL_DIR` is `<id>_build/shader`. The switches of `--env-file` are added to the command.
Reuse works as for the IPA. Each shader output has its own key, named after the one chain version: `shader/ios/b-<chain>.key` for the iOS libraries and `shader/mac-b-<chain>.key` for the Mac's (`shader/ios/vk.key` and `shader/vk.key` for a Vulkan game, the same with `gl` for a GL game). An iOS build writes both keys, since it builds the Mac's libraries too, so a Mac-only build of a folder already packed as an IPA reuses its shaders. Either key is taken only while the Mac libraries carry a full build's stamp; `c1_full.tsv` or `cb_extent.tsv` that does not name it is made again from them, as for the IPA. An iOS build after a Mac-only build still has the iPhone's libraries to make. It translates the shaders again, because `dmsh_build.py`'s per-shader cache keys include the iOS compile convention. It then takes the Mac libraries from the Metal compile cache (`shader/cache/cas/metallib`) and compiles only the iPhone's.
## Build and run on a device with Xcode ## Build and run on a device with Xcode
With an Apple developer account, Xcode can sign and install the app itself: With an Apple developer account, Xcode can sign and install the app itself:
@@ -121,7 +142,7 @@ To add a game:
## Build by hand (Mac) ## Build by hand (Mac)
You need the same tools as the script. The shader translator is the prebuilt `dm/gfx/tools/dmsh_xlate_bin`, the build that matches `dmsh_chain.lock`; a fresh `dotnet publish` gives different fingerprints. Its launcher `dmsh_xlate` is a universal binary (arm64 and x86_64), so an Intel Mac runs it too, with the same output; after a rebuild, join the new launcher with lipo to the one `dotnet build -r osx-x64` makes of the same .NET version (the lock does not cover the launcher). What an Intel Mac cannot run is the Mac host, which runs the game's own arm64 code: the NSP script stops at a step that boots the game there (a run-time shader capture, a decode with the game's own Oodle or LZ5) and skips a provisional profile's heap check. The commands use three variables: `scripts/build_ipa_from_nsp.py --platform mac` does steps 1 to 4 in the game's NSP folder and prints the command of step 5 ([A Mac-only build for development](#a-mac-only-build-for-development)). The steps by hand follow. You need the same tools as the script. The shader translator is the prebuilt `dm/gfx/tools/dmsh_xlate_bin`, the build that matches `dmsh_chain.lock`; a fresh `dotnet publish` gives different fingerprints. Its launcher `dmsh_xlate` is a universal binary (arm64 and x86_64), so an Intel Mac runs it too, with the same output; after a rebuild, join the new launcher with lipo to the one `dotnet build -r osx-x64` makes of the same .NET version (the lock does not cover the launcher). What an Intel Mac cannot run is the Mac host, which runs the game's own arm64 code: the NSP script stops at a step that boots the game there (a run-time shader capture, a decode with the game's own Oodle or LZ5) and skips a provisional profile's heap check. The commands use three variables:
- `G`, the profile; - `G`, the profile;
- `W`, the game's work folder: `<title ID>-work` inside the game's NSP folder, outside this repository and off the home folder (it grows to the size of the game files); - `W`, the game's work folder: `<title ID>-work` inside the game's NSP folder, outside this repository and off the home folder (it grows to the size of the game files);
- `VER`, a version folder named after the NSP version. - `VER`, a version folder named after the NSP version.
+143 -64
View File
@@ -6,6 +6,9 @@
# → iOS shader libraries (dmsh_build.py) → build_ipa.sh. A Vulkan game (main imports Vulkan, not NVN: Hades) skips the NVN shader steps: vk_shaders.py translates its SPIR-V shaders instead. # → iOS shader libraries (dmsh_build.py) → build_ipa.sh. A Vulkan game (main imports Vulkan, not NVN: Hades) skips the NVN shader steps: vk_shaders.py translates its SPIR-V shaders instead.
# Which of these steps a game needs follows from its files (dm/games/profile_guess.py shader_sources), not from its profile. A game with no matching profile is surveyed (<NSP folder>/<title ID>_survey/survey.md), then gets a provisional profile guessed from # Which of these steps a game needs follows from its files (dm/games/profile_guess.py shader_sources), not from its profile. A game with no matching profile is surveyed (<NSP folder>/<title ID>_survey/survey.md), then gets a provisional profile guessed from
# the survey (<NSP folder>/<title ID>_build/profile/<title ID>/, outside the repository, found through DM_GAMES_DIR) and a best-effort IPA that may not boot; SURVEY_ONLY / --survey-only stops after the survey. # the survey (<NSP folder>/<title ID>_build/profile/<title ID>/, outside the repository, found through DM_GAMES_DIR) and a best-effort IPA that may not boot; SURVEY_ONLY / --survey-only stops after the survey.
# PLATFORM = 'mac' / --platform mac is a development option (the Mac is the development host, not a platform for players): the same steps up to the shaders, but only the Mac's
# (the shader build without IOSOUT, a Vulkan or GL game's libraries without --ios), then the Mac host for the profile and the button-icon font; no iOS shaders, Xcode project or IPA.
# The extracted files are kept whatever KEEP_EXTRACTED says (the game runs from them), and the last line is the command that runs the game in the Mac host, its save in <id>_build/run/save.
# Two ways to run it: open this file in Spyder and press F5 (it uses the values under "Settings" below; an empty NSP folder or prod.keys is asked for at run time); # Two ways to run it: open this file in Spyder and press F5 (it uses the values under "Settings" below; an empty NSP folder or prod.keys is asked for at run time);
# or in a terminal: python3 scripts/build_ipa_from_nsp.py <NSP folder> -k <prod.keys> [options] (values given on the command line override "Settings", -h lists every option; # or in a terminal: python3 scripts/build_ipa_from_nsp.py <NSP folder> -k <prod.keys> [options] (values given on the command line override "Settings", -h lists every option;
# without keyboard input a missing value is not asked for but reported as an error; exit code 0 on success, 1 on error, 130 when interrupted). Command-line options entered in Spyder's run configuration work the same way. # without keyboard input a missing value is not asked for but reported as an error; exit code 0 on success, 1 on error, 130 when interrupted). Command-line options entered in Spyder's run configuration work the same way.
@@ -33,6 +36,8 @@ OUT_IPA = '' # output IPA (empty = <the profile's ipa
APP_NAME = '' # app name on the home screen (empty = the game name in the base NSP) APP_NAME = '' # app name on the home screen (empty = the game name in the base NSP)
BUNDLE_ID = '' # the app's bundle ID (empty = saved per-game Xcode ID, else org.nxnative.<base game title ID in lowercase>) BUNDLE_ID = '' # the app's bundle ID (empty = saved per-game Xcode ID, else org.nxnative.<base game title ID in lowercase>)
PREPARE_XCODE = False # prepare a per-game Xcode project and add it to the workspace instead of packing an IPA; requires KEEP_EXTRACTED PREPARE_XCODE = False # prepare a per-game Xcode project and add it to the workspace instead of packing an IPA; requires KEEP_EXTRACTED
PLATFORM = 'ios' # 'ios' = the IPA (or the Xcode project); 'mac' = development: build only what running the game on this Mac needs (Mac shaders, the Mac host), no iOS shaders,
# Xcode project or IPA; keeps the extracted files and prints the command that runs the game (needs Apple silicon)
TEAM_ID = '' # optional Apple team ID; empty = DM_TEAM_ID environment, saved Xcode selection, or select in Xcode TEAM_ID = '' # optional Apple team ID; empty = DM_TEAM_ID environment, saved Xcode selection, or select in Xcode
XCODE_WORKSPACE = '' # shared .xcworkspace (empty = NxNative.xcworkspace beside the game's NSP folder) XCODE_WORKSPACE = '' # shared .xcworkspace (empty = NxNative.xcworkspace beside the game's NSP folder)
LANGUAGE = '' # game language, a code or a name: 'ja', 'fr', 'de', 'es', 'zh-Hans', 'zh-Hant', 'ko', 'Japanese', ... (empty = English); it must be one the game supports (the console line lists them); LANGUAGE = '' # game language, a code or a name: 'ja', 'fr', 'de', 'es', 'zh-Hans', 'zh-Hant', 'ko', 'Japanese', ... (empty = English); it must be one the game supports (the console line lists them);
@@ -690,13 +695,14 @@ def profile_files(g): # every file of the game profile (game.tsv and its table
# ---- Steps ---- # ---- Steps ----
def check_prereqs(env, xlate): def check_prereqs(env, xlate, mac=False): # mac: a Mac-only build (PLATFORM 'mac'), which needs no xcodegen and compiles shaders only for macOS
missing = [m for m in ('lz4.block', 'capstone', 'fontTools') if not _importable(m)] + ([] if _importable('compression.zstd') or _importable('backports.zstd') else ['backports.zstd']) # zstd: built in from Python 3.14 missing = [m for m in ('lz4.block', 'capstone', 'fontTools') if not _importable(m)] + ([] if _importable('compression.zstd') or _importable('backports.zstd') else ['backports.zstd']) # zstd: built in from Python 3.14
if missing: raise BuildError(f'the current Python ({sys.executable}) is missing modules: {", ".join(missing)} (pip install lz4 capstone fonttools backports.zstd; backports.zstd only before Python 3.14)') if missing: raise BuildError(f'the current Python ({sys.executable}) is missing modules: {", ".join(missing)} (pip install lz4 capstone fonttools backports.zstd; backports.zstd only before Python 3.14)')
tools = {'xcodebuild': 'Xcode', 'xcodegen': 'brew install xcodegen', 'dotnet': '.NET SDK 10', 'zstd': 'brew install zstd', 'spirv-cross': 'brew install spirv-cross', 'make': 'Xcode'} tools = {'xcodebuild': 'Xcode', 'xcodegen': 'brew install xcodegen', 'dotnet': '.NET SDK 10', 'zstd': 'brew install zstd', 'spirv-cross': 'brew install spirv-cross', 'make': 'Xcode'}
if mac: del tools['xcodegen']
gone = [f'{t} ({how})' for t, how in tools.items() if not shutil.which(t, path=env['PATH'])] gone = [f'{t} ({how})' for t, how in tools.items() if not shutil.which(t, path=env['PATH'])]
if gone: raise BuildError('tools not found: ' + ', '.join(gone)) if gone: raise BuildError('tools not found: ' + ', '.join(gone))
mtl = subprocess.run(['xcrun', '-sdk', 'iphoneos', 'metal', '--version'], capture_output=True, text=True, env=env) # Xcode 26 and later install the Metal toolchain separately mtl = subprocess.run(['xcrun', '-sdk', 'macosx' if mac else 'iphoneos', 'metal', '--version'], capture_output=True, text=True, env=env) # Xcode 26 and later install the Metal toolchain separately
if mtl.returncode: raise BuildError(f'the Metal compiler does not run ({(mtl.stderr.strip().splitlines() or ["no output"])[-1]}): install it with xcodebuild -downloadComponent MetalToolchain') if mtl.returncode: raise BuildError(f'the Metal compiler does not run ({(mtl.stderr.strip().splitlines() or ["no output"])[-1]}): install it with xcodebuild -downloadComponent MetalToolchain')
if not xlate.exists(): raise BuildError(f'the shader translator is missing from the repository: {xlate}') if not xlate.exists(): raise BuildError(f'the shader translator is missing from the repository: {xlate}')
@@ -831,9 +837,13 @@ def apple_silicon(): # hw.optional.arm64 is 1 on Apple silicon (for a Python u
return subprocess.run(['sysctl', '-n', 'hw.optional.arm64'], capture_output=True, text=True).stdout.strip() == '1' return subprocess.run(['sysctl', '-n', 'hw.optional.arm64'], capture_output=True, text=True).stdout.strip() == '1'
HOST_DIRS = ('dm/host', 'dm/gfx', 'dm/gfx/nvn', 'dm/gfx/nvn/include', 'dm/gfx/tex', 'dm/gfx/vk', 'dm/gfx/gl') # the folders the host's compile line takes its sources and headers from (dm/host/Makefile SRC, GFX, GFX_H)
def build_mac_host(repo, out, env, logs, g): def build_mac_host(repo, out, env, logs, g):
"""The Mac host for the profile, built into out: dm/host's own compile line (make -n) with the output moved, since make takes no target path with spaces and dm/host/dmhost may be another build's. """The Mac host for the profile, built into out: dm/host's own compile line (make -n) with the output moved, since make takes no target path with spaces and dm/host/dmhost may be another build's.
The host runs the game's own arm64 code natively, so an Intel Mac stops here (GitHub issue #54)""" Reused when the compile line and its inputs (the source files of HOST_DIRS, the chain header dm/gfx/nvn/dmsh_chain.h among them, the profile header make generates, libvpx and the third-party headers, the compiler
and the macOS SDK) are as at the last build into out. The host runs the game's own arm64 code natively, so an Intel Mac stops here (GitHub issue #54)"""
if not apple_silicon(): raise BuildError('this step boots the game in the Mac host, which runs the game\'s own arm64 code natively: it needs a Mac with Apple silicon, and this Mac has an Intel processor') if not apple_silicon(): raise BuildError('this step boots the game in the Mac host, which runs the game\'s own arm64 code natively: it needs a Mac with Apple silicon, and this Mac has an Intel processor')
host = repo / 'dm/host' host = repo / 'dm/host'
run(['make', '-s', f'GAME={g.id}', f'gen/{g.id}/game_profile.h', 'gen/vpx/libvpx.a'], logs / 'mac_host_make.log', env=env, cwd=host) run(['make', '-s', f'GAME={g.id}', f'gen/{g.id}/game_profile.h', 'gen/vpx/libvpx.a'], logs / 'mac_host_make.log', env=env, cwd=host)
@@ -841,8 +851,14 @@ def build_mac_host(repo, out, env, logs, g):
line = next((l for l in dry.stdout.splitlines() if ' -o dmhost ' in l), None) line = next((l for l in dry.stdout.splitlines() if ' -o dmhost ' in l), None)
if not line: raise BuildError(f'make -n gave no compile line for the Mac host: {dry.stderr.strip()[-500:]}') if not line: raise BuildError(f'make -n gave no compile line for the Mac host: {dry.stderr.strip()[-500:]}')
cmd = shlex.split(line); cmd[cmd.index('-o') + 1] = out cmd = shlex.split(line); cmd[cmd.index('-o') + 1] = out
out.parent.mkdir(parents=True, exist_ok=True) kf, tp = out.with_name(out.name + '.key'), repo / 'dm/third_party'
srcs = [p for d in HOST_DIRS for p in sorted((repo / d).iterdir()) if p.is_file() and p.suffix in ('.c', '.m', '.h', '.S', '.inc')]
key = fingerprint(cmd, ('files', srcs + [host / f'gen/{g.id}/game_profile.h', host / 'gen/vpx/libvpx.a', tp / 'stb_vorbis/stb_vorbis.c']),
('tree', tp / 'vulkan-headers/include'), ('tree', tp / 'libvpx/vpx'), ('cmd', ['clang', '--version']), ('cmd', ['xcrun', '--show-sdk-build-version']), env=env)
if out.is_file() and reused(kf, key) is not None: return out
rmtree(kf); out.parent.mkdir(parents=True, exist_ok=True)
run(cmd, logs / 'mac_host_build.log', env=env, cwd=host) run(cmd, logs / 'mac_host_build.log', env=env, cwd=host)
kf.write_text(f'{key}\n')
return out return out
@@ -1172,10 +1188,10 @@ def split_shaders(py, repo, roms, mains, pkg, env, logs, g, provisional=False, o
class ShaderWatch: class ShaderWatch:
"""Progress of dmsh_build.py, read from the files it writes (it prints nothing until it ends). It works in three phases: """Progress of dmsh_build.py, read from the files it writes (it prints nothing until it ends). It works in three phases:
the cache check (a hit restores <key>_b.json into the iOS output b-<chain>/, a miss writes <key>.bin into its temporary folder dmsh_* under WORK), the cache check (a hit restores <key>_b.json into the library directory b-<chain>/: the iOS one, or the Mac's for a Mac-only build; a miss writes <key>.bin into its temporary folder dmsh_* under WORK),
the translation of the misses (<key>.spv in the temporary folder), and the compile, which takes the keys in a fixed order (sorted, the vertex shaders with pixel pairs last) the translation of the misses (<key>.spv in the temporary folder), and the compile, which takes the keys in a fixed order (sorted, the vertex shaders with pixel pairs last)
and writes <key>_b.json again for every miss and every pixel shader: the furthest key written so far tells how far it got""" and writes <key>_b.json again for every miss and every pixel shader: the furthest key written so far tells how far it got"""
def __init__(self, g, out, tmp): # out: the iOS library directory b-<chain>/ def __init__(self, g, out, tmp): # out: the library directory b-<chain>/ (the iOS one, or the Mac's for a Mac-only build)
keys = set() keys = set()
for ln in open(g.table('shader_map.tsv'), errors='replace'): # the keys dmsh_build.py builds: column 4 of the rows from a shader package for ln in open(g.table('shader_map.tsv'), errors='replace'): # the keys dmsh_build.py builds: column 4 of the rows from a shader package
c = ln.rstrip('\n').split('\t') c = ln.rstrip('\n').split('\t')
@@ -1186,7 +1202,7 @@ class ShaderWatch:
self.out, self.tmp, self.t0, self.t1, self.w = out, tmp, time.time(), None, None self.out, self.tmp, self.t0, self.t1, self.w = out, tmp, time.time(), None, None
self.hits, self.fin, self.top = set(), set(), -1 self.hits, self.fin, self.top = set(), set(), -1
def written(self, keys, since): # those of keys whose <key>_b.json in the iOS output was written after since def written(self, keys, since): # those of keys whose <key>_b.json in the output was written after since
found = set() found = set()
try: try:
with os.scandir(self.out) as it: with os.scandir(self.out) as it:
@@ -1217,30 +1233,41 @@ def toolchain(): # the compilers' versions, part of the shader stages' keys
return ('cmd', ['xcodebuild', '-version']), ('cmd', ['spirv-cross', '--revision']) return ('cmd', ['xcodebuild', '-version']), ('cmd', ['spirv-cross', '--revision'])
def build_shaders(py, repo, sh, xlate, env, logs, g, pkg_key=None): def build_shaders(py, repo, sh, xlate, env, logs, g, pkg_key=None, mac=False):
"""The iOS shader libraries (dmsh_build.py). Reused when the shader package (pkg_key, its content; None never reuses) and the chain's tools, translator binaries, profile and compilers """The iOS shader libraries (dmsh_build.py, which builds the Mac's into sh/cache in the same run), or with mac only the Mac's (a Mac-only build: no IOSOUT). Returns the folder the host takes
are as at the last successful build: dmsh_build.py would only restore every shader from its cache (63,000 for Breath of the Wild: about 100 s)""" them from. Reused when the shader package (pkg_key, its content; None never reuses) and the chain's tools, translator binaries, profile and compilers are as at the last successful build:
dmsh_build.py would only restore every shader from its cache (63,000 for Breath of the Wild: about 100 s). The key of the iOS libraries is sh/ios/b-<chain>.key, that of the Mac's
sh/mac-b-<chain>.key (not in sh/cache, whose b-* entries dmsh_build.py reads as library folders), which an iOS build writes too, so a Mac-only build after it is reused; either is taken only
while the Mac libraries carry a full build's stamp (inputs.tsv) for c1_full.tsv and cb_extent.tsv to name (chain_tables).
An iOS build after a Mac-only one translates the shaders again (dmsh_build.py's per-shader cache keys include the iOS compile convention), but takes the Mac libraries from its
Metal compile cache (dmsh_mtlcache.py), so only the iPhone's are compiled"""
ios, log = sh / 'ios', logs / 'dmsh_build.log' ios, log = sh / 'ios', logs / 'dmsh_build.log'
PROG.stage('iOS shaders') PROG.stage('Mac shaders' if mac else 'iOS shaders')
chain = chain_of(repo) chain = chain_of(repo)
libs = ios / f'b-{chain}'; kf = libs.with_name(libs.name + '.key') out = sh / 'cache' if mac else ios
libs, kf, kmac = out / f'b-{chain}', ios / f'b-{chain}.key', sh / f'mac-b-{chain}.key'
if pkg_key: if pkg_key:
PROG.set(step='checking the inputs') PROG.set(step='checking the inputs')
key = fingerprint(pkg_key, ('files', chain_files(repo)), ('stat', xlate.parent), ('files', profile_files(g)), *toolchain(), str(xlate), key = fingerprint(pkg_key, ('files', chain_files(repo)), ('stat', xlate.parent), ('files', profile_files(g)), *toolchain(), str(xlate),
{k: env.get(k) for k in ('XLATE_DEBUG', 'XLATE_REDUCED')}, env=env) {k: env.get(k) for k in ('XLATE_DEBUG', 'XLATE_REDUCED')}, env=env)
note = next((n for k in ((kmac, kf) if mac else (kf,)) if (n := reused(k, key)) is not None), None)
# the libraries are the last build's, so are tables that name their stamp; a missing or stale one is made from that build's Mac outputs, and without them the step runs again # the libraries are the last build's, so are tables that name their stamp; a missing or stale one is made from that build's Mac outputs, and without them the step runs again
if next(libs.glob('*_b.metallib'), None) and (note := reused(kf, key)) is not None and chain_tables(py, repo, sh, env, logs, g, chain): if next(libs.glob('*_b.metallib'), None) and note is not None and chain_tables(py, repo, sh, env, logs, g, chain):
PROG.end(f'{note}, reused (inputs unchanged)'); return ios PROG.end(f'{note}, reused (inputs unchanged)'); return out
else: key = None else: key = None
rmtree(kf) rmtree(kmac)
if not mac: rmtree(kf)
try: PROG.watch(ShaderWatch(g, libs, Path(env['WORK']))) try: PROG.watch(ShaderWatch(g, libs, Path(env['WORK'])))
except Exception: pass # progress never stops the build (a missing table is dmsh_build.py's to report) except Exception: pass # progress never stops the build (a missing table is dmsh_build.py's to report)
run([py, repo / 'dm/gfx/tools/own_compilers.py', sh / 'pkg', sh / 'cache'], log, env=dict(env, XLATE=str(xlate), IOSOUT=str(ios))) # dmsh_build.py with this Mac's Metal compiler and spirv-cross benv = dict(env, XLATE=str(xlate), IOSOUT=str(ios))
if mac: del benv['IOSOUT'] # dmsh_build.py builds the iOS libraries only with IOSOUT
run([py, repo / 'dm/gfx/tools/own_compilers.py', sh / 'pkg', sh / 'cache'], log, env=benv) # dmsh_build.py with this Mac's Metal compiler and spirv-cross
PROG.watch(None) PROG.watch(None)
if not chain_tables(py, repo, sh, env, logs, g, chain): raise BuildError(f'no stamped Mac shader outputs in {sh / "cache" / f"b-{chain}"} (inputs.tsv): dmsh_build.py did not finish a full build') if not chain_tables(py, repo, sh, env, logs, g, chain): raise BuildError(f'no stamped Mac shader outputs in {sh / "cache" / f"b-{chain}"} (inputs.tsv): dmsh_build.py did not finish a full build')
what = 'Mac' if mac else 'iOS'
n = sum(1 for _ in libs.glob('*_b.metallib')) if libs.is_dir() else 0 n = sum(1 for _ in libs.glob('*_b.metallib')) if libs.is_dir() else 0
if n == 0: raise BuildError(f'no iOS shader libraries in {libs}') if n == 0: raise BuildError(f'no {what} shader libraries in {libs}')
say(f'{n} iOS shader libraries ({libs})') say(f'{n} {what} shader libraries ({libs})')
t = log.read_text(errors='replace') t = log.read_text(errors='replace')
if own := re.search(r'^note: (built with this Mac\'s compilers.*)$', t, re.M): say(own.group(1)) if own := re.search(r'^note: (built with this Mac\'s compilers.*)$', t, re.M): say(own.group(1))
m, c = re.search(r'shaders (\d+): ok (\d+)', t), re.search(r'shader cache: hit (\d+)', t) m, c = re.search(r'shaders (\d+): ok (\d+)', t), re.search(r'shader cache: hit (\d+)', t)
@@ -1251,49 +1278,50 @@ def build_shaders(py, repo, sh, xlate, env, logs, g, pkg_key=None):
note = f'{ok:,} of {total:,} shaders ready{failed}' note = f'{ok:,} of {total:,} shaders ready{failed}'
PROG.end(f'{ok:,} of {total:,} shaders ready' + (f' ({int(c.group(1)):,} from the cache)' if c else '') + failed) PROG.end(f'{ok:,} of {total:,} shaders ready' + (f' ({int(c.group(1)):,} from the cache)' if c else '') + failed)
else: PROG.end() else: PROG.end()
if key: kf.write_text(f'{key}\n{note}\n') if key:
return ios for k in (kmac,) if mac else (kf, kmac): k.write_text(f'{key}\n{note}\n')
return out
def build_vk_shaders(py, repo, roots, sh, env, logs, g, reuse=True): def layer_shaders(py, repo, api, name, roots, sh, key, logs, env, mac):
"""A Vulkan game's SPIR-V shaders (roots: the RomFS, and a run-time capture folder) → one Metal library each for the Mac (sh/vk/) and the iPhone (sh/ios/vk/, what build_ipa.sh takes from MSL_IOS), """What build_vk_shaders and build_gl_shaders share: dm/gfx/tools/<api>_shaders.py on the roots → sh/<api>/ for the Mac and, unless mac (a Mac-only build), sh/ios/<api>/ for the iPhone.
replacing the NVN split and chain build. Reused when the roots (paths and sizes), the tools, the profile and the compilers are as at the last successful run (reuse False: never). Returns the folder the host (DM_MSL_DIR) or build_ipa.sh (MSL_IOS) takes the libraries from. key: the inputs' fingerprint (None never reuses). The iOS libraries' key is sh/ios/<api>.key,
Otherwise the old output is deleted first, so a failed run leaves nothing for build_ipa.sh to pick up; vk_shaders.py's temporary files go to WORK (on the NSP folder's volume)""" the Mac's sh/<api>.key, which an iOS build writes too (it builds both), so a Mac-only build is reused after either; an iOS build after a Mac-only one translates again.
ios = sh / 'ios'; kf = ios / 'vk.key' Otherwise the old output is deleted first, so a failed run leaves nothing for build_ipa.sh or the host to pick up; the translator's temporary files go to WORK (on the NSP folder's volume)"""
ios = sh / 'ios'; out = sh if mac else ios
kf, kmac, lib, log = ios / f'{api}.key', sh / f'{api}.key', out / api / f'{api}_shaders.metallib', logs / f'{api}_shaders.log'
note = next((n for k in ((kmac, kf) if mac else (kf,)) if (n := reused(k, key)) is not None), None) if key else None
if lib.is_file() and note is not None:
PROG.end(f'{note}, reused (inputs unchanged)'); return out
PROG.set(step='translating')
rmtree(kmac); rmtree(sh / api)
if not mac: rmtree(kf); rmtree(ios / api)
run([py, repo / f'dm/gfx/tools/{api}_shaders.py', *roots, sh] + ([] if mac else ['--ios', ios]), log, env=dict(env, TMPDIR=env['WORK']))
if not lib.is_file(): raise BuildError(f'no {"Mac" if mac else "iOS"} {name} shader library in {lib.parent}')
m = re.search(r'macosx: (\d+)' if mac else r'iphoneos: (\d+)', log.read_text(errors='replace'))
note = f'{int(m.group(1)):,} shaders for the Mac' + ('' if mac else ' and iOS') if m else ''
if key:
for k in (kmac,) if mac else (kf, kmac): k.write_text(f'{key}\n{note}\n')
PROG.end(note)
return out
def build_vk_shaders(py, repo, roots, sh, env, logs, g, reuse=True, mac=False):
"""A Vulkan game's SPIR-V shaders (roots: the RomFS, and a run-time capture folder) → one Metal library each for the Mac (sh/vk/) and the iPhone (sh/ios/vk/, what build_ipa.sh takes from MSL_IOS;
not with mac), replacing the NVN split and chain build (layer_shaders). Reused when the roots (paths and sizes), the tools, the profile and the compilers are as at the last successful run (reuse False: never)"""
PROG.stage('Vulkan shaders', step='checking the inputs') PROG.stage('Vulkan shaders', step='checking the inputs')
key = fingerprint(*[('tree', r) for r in roots], ('files', tool_files(repo)), ('files', profile_files(g)), *toolchain(), env=env) if reuse else None key = fingerprint(*[('tree', r) for r in roots], ('files', tool_files(repo)), ('files', profile_files(g)), *toolchain(), env=env) if reuse else None
if key and (ios / 'vk/vk_shaders.metallib').is_file() and (note := reused(kf, key)) is not None: return layer_shaders(py, repo, 'vk', 'Vulkan', roots, sh, key, logs, env, mac)
PROG.end(f'{note}, reused (inputs unchanged)'); return ios
PROG.set(step='translating')
rmtree(kf); rmtree(sh / 'vk'); rmtree(ios / 'vk')
run([py, repo / 'dm/gfx/tools/vk_shaders.py', *roots, sh, '--ios', ios], logs / 'vk_shaders.log', env=dict(env, TMPDIR=env['WORK']))
if not (ios / 'vk/vk_shaders.metallib').is_file(): raise BuildError(f'no iOS Vulkan shader library in {ios / "vk"}')
m = re.search(r'iphoneos: (\d+)', (logs / 'vk_shaders.log').read_text(errors='replace'))
note = f'{int(m.group(1)):,} shaders for the Mac and iOS' if m else ''
if key: kf.write_text(f'{key}\n{note}\n')
PROG.end(note)
return ios
def build_gl_shaders(py, repo, roots, sh, env, logs, g, reuse=True): def build_gl_shaders(py, repo, roots, sh, env, logs, g, reuse=True, mac=False):
"""An OpenGL game's GLSL sources (roots: the RomFS, main, a run-time capture folder) → one Metal library each for the Mac (sh/gl/) and the iPhone (sh/ios/gl/, what build_ipa.sh takes from MSL_IOS), """An OpenGL game's GLSL sources (roots: the RomFS, main, a run-time capture folder) → one Metal library each for the Mac (sh/gl/) and the iPhone (sh/ios/gl/, what build_ipa.sh takes from MSL_IOS;
replacing the NVN split and chain build, as build_vk_shaders does for a Vulkan game; glslang (glslangValidator) compiles the GLSL to SPIR-V for spirv-cross""" not with mac), replacing the NVN split and chain build, as build_vk_shaders does for a Vulkan game; glslang (glslangValidator) compiles the GLSL to SPIR-V for spirv-cross"""
ios = sh / 'ios'; kf = ios / 'gl.key'
PROG.stage('GL shaders', step='checking the inputs') PROG.stage('GL shaders', step='checking the inputs')
if not shutil.which('glslangValidator', path=env['PATH']): raise BuildError('glslangValidator not found (brew install glslang): it compiles an OpenGL game\'s GLSL shaders') if not shutil.which('glslangValidator', path=env['PATH']): raise BuildError('glslangValidator not found (brew install glslang): it compiles an OpenGL game\'s GLSL shaders')
key = fingerprint(*[('tree', r) if Path(r).is_dir() else ('files', [r]) for r in roots], ('files', tool_files(repo)), ('files', profile_files(g)), *toolchain(), key = fingerprint(*[('tree', r) if Path(r).is_dir() else ('files', [r]) for r in roots], ('files', tool_files(repo)), ('files', profile_files(g)), *toolchain(),
('cmd', ['glslangValidator', '--version']), env=env) if reuse else None ('cmd', ['glslangValidator', '--version']), env=env) if reuse else None
if key and (ios / 'gl/gl_shaders.metallib').is_file() and (note := reused(kf, key)) is not None: return layer_shaders(py, repo, 'gl', 'GL', roots, sh, key, logs, env, mac)
PROG.end(f'{note}, reused (inputs unchanged)'); return ios
PROG.set(step='translating')
rmtree(kf); rmtree(sh / 'gl'); rmtree(ios / 'gl')
run([py, repo / 'dm/gfx/tools/gl_shaders.py', *roots, sh, '--ios', ios], logs / 'gl_shaders.log', env=dict(env, TMPDIR=env['WORK']))
if not (ios / 'gl/gl_shaders.metallib').is_file(): raise BuildError(f'no iOS GL shader library in {ios / "gl"}')
m = re.search(r'iphoneos: (\d+)', (logs / 'gl_shaders.log').read_text(errors='replace'))
note = f'{int(m.group(1)):,} shaders for the Mac and iOS' if m else ''
if key: kf.write_text(f'{key}\n{note}\n')
PROG.end(note)
return ios
class IpaWatch: class IpaWatch:
@@ -1346,6 +1374,43 @@ class IpaWatch:
return r return r
def env_file_pairs(path): # an environment file's switches (one KEY=VALUE per line, # starts a comment), read as the iOS app reads them (dm/ios/Sources/main.m)
out = {}
try: text = Path(os.path.expanduser(path)).read_text(errors='replace')
except OSError as e: raise BuildError(f'cannot read the environment file (--env-file / DM_ENV_FILE): {e}') from e
for ln in text.splitlines():
k, eq, v = ln.strip().partition('=')
if k.strip() and eq and not k.startswith('#'): out[k.strip()] = v
return out
def mac_run(py, repo, work, game, patched, msl, aoc, gfx, lang, env, logs, g, env_file=''):
"""The last stage of a Mac-only build (PLATFORM 'mac'): the Mac host for the profile (build_mac_host, into <id>_build/tools/dmhost, where the run-time captures build it too; DM_GAMES_DIR /
DM_GAMES_FIRST in env pick a provisional or derived profile) and the button-icon font the host gives a game for the Nintendo extension shared font (make_ext_font.py, into <id>_build/font,
made again when the tool changed). Returns the command that runs the game in a window with sound (docs/development.md, Build by hand (Mac), step 5): its save and temporary storage in
<id>_build/run (kept from run to run), the extracted RomFS, the Mac shader libraries (msl), the DLC, the profile's shader map (a host outside dm/host does not find it beside itself;
c1_full.tsv and cb_extent.tsv are read beside it), the game language, and the switches of an environment file (DM_ENV_FILE / --env-file)"""
PROG.stage('Mac host', step='building')
host = build_mac_host(repo, work / 'tools/dmhost', env, logs, g)
PROG.set(step='button-icon font')
font = work / 'font'; ttf, kf = font / 'nintendo_ext.ttf', font / 'nintendo_ext.key'
key = fingerprint(('files', [repo / 'dm/tools/make_ext_font.py']))
if not (ttf.is_file() and reused(kf, key) is not None):
rmtree(kf); font.mkdir(parents=True, exist_ok=True)
run([py, repo / 'dm/tools/make_ext_font.py', ttf], logs / 'make_ext_font.log', env=env)
kf.write_text(f'{key}\n')
rd = work / 'run'
for d in ('save', 'temp'): (rd / d).mkdir(parents=True, exist_ok=True)
renv = {'DM_ROOT': work, 'DM_ROMFS': game / 'RomFS', 'DM_SAVE': rd / 'save', 'DM_TEMP': rd / 'temp', 'DM_MSL_DIR': msl, 'DM_FONT_DIR': font}
if aoc: renv['DM_AOC_DIR'] = aoc
if gfx == 'nvn': renv['DM_NVN_SHMAP'] = g.table('shader_map.tsv')
renv.update(DM_LANG=lang, DM_WINDOW=1, DM_AUDIO=1)
if env_file: renv.update(env_file_pairs(env_file))
PROG.end(str(host))
q = lambda v: shlex.quote(str(v))
return f'cd {q(rd)} && ' + ' '.join(f'{k}={q(v)}' for k, v in renv.items()) + f' {q(host)} {q(patched)}'
def find_repo(here): # this file lives in the repository's scripts/; walk up to the level that has dm/tools/nx_extract/Program.cs def find_repo(here): # this file lives in the repository's scripts/; walk up to the level that has dm/tools/nx_extract/Program.cs
for d in here.parents: for d in here.parents:
if (d / 'dm/tools/nx_extract/Program.cs').exists(): return d if (d / 'dm/tools/nx_extract/Program.cs').exists(): return d
@@ -1365,6 +1430,8 @@ def parse_args(argv):
ap.add_argument('--app-name', help='app name on the home screen (default: the game name in the base NSP)') ap.add_argument('--app-name', help='app name on the home screen (default: the game name in the base NSP)')
ap.add_argument('--bundle-id', help='the app\'s bundle ID (default: saved Xcode ID when preparing again, else org.nxnative.<base game title ID in lowercase>)') ap.add_argument('--bundle-id', help='the app\'s bundle ID (default: saved Xcode ID when preparing again, else org.nxnative.<base game title ID in lowercase>)')
ap.add_argument('--prepare-xcode', action='store_true', help='prepare a per-game Xcode project and add it to the shared workspace instead of packing an IPA (requires keeping extracted files)') ap.add_argument('--prepare-xcode', action='store_true', help='prepare a per-game Xcode project and add it to the shared workspace instead of packing an IPA (requires keeping extracted files)')
ap.add_argument('--platform', choices=('ios', 'mac'), help='ios = the IPA (or the Xcode project); mac = for development: build only what running the game on this Mac needs (the Mac shaders and the Mac host, '
'no iOS shaders, Xcode project or IPA), keep the extracted files and print the command that runs the game (default: the PLATFORM setting, ios)')
ap.add_argument('--team-id', help='Apple developer team ID (default: TEAM_ID setting, DM_TEAM_ID environment, or saved Xcode selection)') ap.add_argument('--team-id', help='Apple developer team ID (default: TEAM_ID setting, DM_TEAM_ID environment, or saved Xcode selection)')
ap.add_argument('--xcode-workspace', help='workspace collecting prepared games (default: NxNative.xcworkspace beside the NSP folder; use with --prepare-xcode)') ap.add_argument('--xcode-workspace', help='workspace collecting prepared games (default: NxNative.xcworkspace beside the NSP folder; use with --prepare-xcode)')
ap.add_argument('-l', '--language', help='game language, a code or a name: ja, fr, de, es, it, nl, pt, ru, ko, zh-Hans, zh-Hant, en-GB, fr-CA, es-419, pt-BR, Japanese, ... (default: the LANGUAGE setting, else English); ' ap.add_argument('-l', '--language', help='game language, a code or a name: ja, fr, de, es, it, nl, pt, ru, ko, zh-Hans, zh-Hant, en-GB, fr-CA, es-419, pt-BR, Japanese, ... (default: the LANGUAGE setting, else English); '
@@ -1382,9 +1449,8 @@ def parse_args(argv):
return ap.parse_args(argv) return ap.parse_args(argv)
def report_profile_status(g, reg, pkg_ver, derived, provisional, prepare_xcode=False): def report_profile_status(g, reg, pkg_ver, derived, provisional, artifact='IPA'):
"""Keep build-specific warnings visible for both IPA and Xcode output.""" """Keep build-specific warnings visible for the IPA, the Xcode project and a Mac-only build"""
artifact = 'Xcode project' if prepare_xcode else 'IPA'
disp = g.builds()[pkg_ver]['display'] if derived else '' disp = g.builds()[pkg_ver]['display'] if derived else ''
if derived: # the modules whose build no registered version has: the rewriter's cuts and the host's fixes made for a registered build's code do not reach them if derived: # the modules whose build no registered version has: the rewriter's cuts and the host's fixes made for a registered build's code do not reach them
known = {(m, b) for row in reg.builds().values() for m, b in row.items() if m != 'display'} known = {(m, b) for row in reg.builds().values() for m, b in row.items() if m != 'display'}
@@ -1404,7 +1470,13 @@ def main(args=None):
here = Path(globals().get('__file__') or sys.argv[0]).resolve() here = Path(globals().get('__file__') or sys.argv[0]).resolve()
repo = find_repo(here) repo = find_repo(here)
prepare_xcode = args.prepare_xcode or PREPARE_XCODE prepare_xcode = args.prepare_xcode or PREPARE_XCODE
platform = args.platform or PLATFORM
if platform not in ('ios', 'mac'): raise BuildError(f'PLATFORM / --platform must be ios or mac: {platform}')
mac = platform == 'mac' # a Mac-only build for development: no iOS shaders, Xcode project or IPA (mac_run)
if mac and prepare_xcode: raise BuildError('--prepare-xcode makes an iPhone project: it cannot be combined with --platform mac')
if mac and not apple_silicon(): raise BuildError('--platform mac runs the game in the Mac host, which runs the game\'s own arm64 code natively: it needs a Mac with Apple silicon, and this Mac has an Intel processor')
keep = KEEP_EXTRACTED if args.keep_extracted is None else args.keep_extracted keep = KEEP_EXTRACTED if args.keep_extracted is None else args.keep_extracted
if mac and not keep: say('keeping the extracted game files all the same: the game runs from them on the Mac'); keep = True
if prepare_xcode and not keep: raise BuildError('--prepare-xcode requires --keep-extracted: Xcode builds from the extracted files') if prepare_xcode and not keep: raise BuildError('--prepare-xcode requires --keep-extracted: Xcode builds from the extracted files')
if args.xcode_workspace and not prepare_xcode: raise BuildError('--xcode-workspace requires --prepare-xcode') if args.xcode_workspace and not prepare_xcode: raise BuildError('--xcode-workspace requires --prepare-xcode')
nsp_dir = Path(os.path.expanduser(args.nsp_dir or NSP_DIR or ask('NSP folder: ', 'the NSP folder'))).resolve() nsp_dir = Path(os.path.expanduser(args.nsp_dir or NSP_DIR or ask('NSP folder: ', 'the NSP folder'))).resolve()
@@ -1472,20 +1544,20 @@ def main(args=None):
if not saved_signing and legacy_config.exists() and f'/gen/{g.id}/' in legacy_config.read_text(): if not saved_signing and legacy_config.exists() and f'/gen/{g.id}/' in legacy_config.read_text():
saved_signing = xcode_projects.signing_settings(repo / 'dm/ios/DmIOS.xcodeproj') saved_signing = xcode_projects.signing_settings(repo / 'dm/ios/DmIOS.xcodeproj')
except ValueError as e: raise BuildError(str(e)) from e except ValueError as e: raise BuildError(str(e)) from e
say(f'Xcode project: {project}' if prepare_xcode else f'output IPA: {out_ipa}') say('Mac-only build for development: the Mac shaders and the Mac host, no iOS shaders, Xcode project or IPA' if mac else f'Xcode project: {project}' if prepare_xcode else f'output IPA: {out_ipa}')
say(f'work folder (extracted game files in extract/, {"kept for the next build" if keep else "deleted after the IPA is built"}): {work}') say(f'work folder (extracted game files in extract/, {"kept for the next build" if keep else "deleted after the IPA is built"}): {work}')
xlate = repo / 'dm/gfx/tools/dmsh_xlate_bin/dmsh_xlate' xlate = repo / 'dm/gfx/tools/dmsh_xlate_bin/dmsh_xlate'
# DM_GAME picks the profile; DM_ROOT / WORK point at <id>_build: the tools' default paths and temporary directories all stay inside it # DM_GAME picks the profile; DM_ROOT / WORK point at <id>_build: the tools' default paths and temporary directories all stay inside it
env.update(DM_GAME=g.id, DM_ROOT=str(work), WORK=str(work / 'tmp')) env.update(DM_GAME=g.id, DM_ROOT=str(work), WORK=str(work / 'tmp'))
logs = work / 'logs' logs = work / 'logs'
check_prereqs(env, xlate) check_prereqs(env, xlate, mac)
picks = pick_nsps(fam, have, versions) picks = pick_nsps(fam, have, versions)
ex = work / 'extract'; ex.mkdir(parents=True, exist_ok=True); (work / 'tmp').mkdir(exist_ok=True) ex = work / 'extract'; ex.mkdir(parents=True, exist_ok=True); (work / 'tmp').mkdir(exist_ok=True)
gb = sum({p.stat().st_size for b, u in picks.values() for p in (b, u) if p}) / 2**30 + 6 # extraction + shader cache + slack gb = sum({p.stat().st_size for b, u in picks.values() for p in (b, u) if p}) / 2**30 + 6 # extraction + shader cache + slack
if free_gb(work) < gb and not all((ex / v).exists() for v in versions): raise BuildError(f'less than {gb:.0f} GB free on the volume of {work}') if free_gb(work) < gb and not all((ex / v).exists() for v in versions): raise BuildError(f'less than {gb:.0f} GB free on the volume of {work}')
heapck = provisional and HEAP_CHECK and not args.no_heap_check and not heap_checked(g) # the heap check, when the game needs one (once per provisional profile: it stamps the profile) heapck = provisional and HEAP_CHECK and not args.no_heap_check and not heap_checked(g) # the heap check, when the game needs one (once per provisional profile: it stamps the profile)
PROG.plan(1 + len(versions) + bool(fam['dlc']) + 4 - prepare_xcode) # prepare Xcode replaces app build + IPA pack with one stage PROG.plan(1 + len(versions) + bool(fam['dlc']) + 4 - (prepare_xcode or mac)) # prepare Xcode (a Mac-only build: the Mac host) replaces app build + IPA pack with one stage
PROG.stage('build nx_extract') PROG.stage('build nx_extract')
nx = build_nx_extract(repo, work, logs, env) nx = build_nx_extract(repo, work, logs, env)
PROG.end() PROG.end()
@@ -1525,7 +1597,7 @@ def main(args=None):
base = next((dirs[v] for v in versions if picks[v][1] is None), dirs[versions[0]]) # the folder the base game was extracted to alone base = next((dirs[v] for v in versions if picks[v][1] is None), dirs[versions[0]]) # the folder the base game was extracted to alone
# the app name and bundle ID both come from the base NSP: the game name in its control data, and a bundle ID from its title ID; a setting / the command line overrides them # the app name and bundle ID both come from the base NSP: the game name in its control data, and a bundle ID from its title ID; a setting / the command line overrides them
app_name = args.app_name or APP_NAME or nacp.name(base / 'control.nacp') app_name = args.app_name or APP_NAME or nacp.name(base / 'control.nacp')
if not app_name: raise BuildError(f'no game name in the base extraction ({base / "control.nacp"}; an older extraction did not save it: delete extract/ and run again, or give --app-name)') if not app_name and not mac: raise BuildError(f'no game name in the base extraction ({base / "control.nacp"}; an older extraction did not save it: delete extract/ and run again, or give --app-name)')
bundle_id = args.bundle_id or BUNDLE_ID or saved_signing.get('PRODUCT_BUNDLE_IDENTIFIER') or f'org.nxnative.{app:016x}' bundle_id = args.bundle_id or BUNDLE_ID or saved_signing.get('PRODUCT_BUNDLE_IDENTIFIER') or f'org.nxnative.{app:016x}'
team_id = args.team_id or TEAM_ID or env.get('DM_TEAM_ID') or saved_signing.get('DEVELOPMENT_TEAM') or '0000000000' team_id = args.team_id or TEAM_ID or env.get('DM_TEAM_ID') or saved_signing.get('DEVELOPMENT_TEAM') or '0000000000'
if prepare_xcode: if prepare_xcode:
@@ -1533,12 +1605,12 @@ def main(args=None):
except ValueError as e: raise BuildError(str(e)) from e except ValueError as e: raise BuildError(str(e)) from e
# the app version is the packed game's display version: the update's control data when an update is packed (nx_extract saves it next to the RomFS), else the base game's # the app version is the packed game's display version: the update's control data when an update is packed (nx_extract saves it next to the RomFS), else the base game's
app_version = nacp.version(game / 'control.nacp') or nacp.version(base / 'control.nacp') app_version = nacp.version(game / 'control.nacp') or nacp.version(base / 'control.nacp')
say(f'app name: {app_name}; version: {app_version or "1.0"}; bundle ID: {bundle_id}') if not mac: say(f'app name: {app_name}; version: {app_version or "1.0"}; bundle ID: {bundle_id}')
# the game language: Horizon gives the game one of the languages its control data lists (the packed version's, which an update may extend), so the one packed must be among them # the game language: Horizon gives the game one of the languages its control data lists (the packed version's, which an update may extend), so the one packed must be among them
have = nacp.languages(game / 'control.nacp') or nacp.languages(base / 'control.nacp') have = nacp.languages(game / 'control.nacp') or nacp.languages(base / 'control.nacp')
lang = game_language(want_lang, bool(lang_in), have) lang = game_language(want_lang, bool(lang_in), have)
say(f'game language: {lang} ({dict(nacp.LANGUAGES)[lang]})' + (f', the game has no {want_lang}' if lang != want_lang else '') say(f'game language: {lang} ({dict(nacp.LANGUAGES)[lang]})' + (f', the game has no {want_lang}' if lang != want_lang else '')
+ (f'; the game\'s languages: {" ".join(have)}' if have else '; the control data lists no languages (not checked)') + ' (DM_LANG=<code> in the app\'s Documents/env.txt changes it)') + (f'; the game\'s languages: {" ".join(have)}' if have else '; the control data lists no languages (not checked)') + (' (DM_LANG=<code> in the run command changes it)' if mac else ' (DM_LANG=<code> in the app\'s Documents/env.txt changes it)'))
patched = work / f'patched_{pkg_ver}' patched = work / f'patched_{pkg_ver}'
rewrite(py, repo, game / 'ExeFS', patched, env, logs, g) rewrite(py, repo, game / 'ExeFS', patched, env, logs, g)
@@ -1554,15 +1626,22 @@ def main(args=None):
sh = work / 'shader' sh = work / 'shader'
if gfx == 'vulkan': # the packed version's merged RomFS holds the SPIR-V files the game loads; a capture holds those it generates if gfx == 'vulkan': # the packed version's merged RomFS holds the SPIR-V files the game loads; a capture holds those it generates
cap = capture_vk(repo, game, patched, work, env, logs, g) if vkcap else None cap = capture_vk(repo, game, patched, work, env, logs, g) if vkcap else None
ios = build_vk_shaders(py, repo, [game / 'RomFS'] + ([cap] if cap else []), sh, env, logs, g, reuse=not (provisional or derived or ALLOW_DAMAGED)) ios = build_vk_shaders(py, repo, [game / 'RomFS'] + ([cap] if cap else []), sh, env, logs, g, reuse=not (provisional or derived or ALLOW_DAMAGED), mac=mac)
elif gfx == 'gl': # the GLSL sources in the packed version's archives and in its main (the game compiles them at run time; the host finds each by the hash of its text) elif gfx == 'gl': # the GLSL sources in the packed version's archives and in its main (the game compiles them at run time; the host finds each by the hash of its text)
ios = build_gl_shaders(py, repo, [game / 'RomFS', game / 'ExeFS/main'], sh, env, logs, g, reuse=not (provisional or derived or ALLOW_DAMAGED)) ios = build_gl_shaders(py, repo, [game / 'RomFS', game / 'ExeFS/main'], sh, env, logs, g, reuse=not (provisional or derived or ALLOW_DAMAGED), mac=mac)
else: else:
# Trinity packs and legacy Unreal paks: the split decodes them with the packed version's own decoder (Oodle, LZ5), in the Mac host built for the profile # Trinity packs and legacy Unreal paks: the split decodes them with the packed version's own decoder (Oodle, LZ5), in the Mac host built for the profile
oodle = (lambda: {'DM_OODLE_HOST': str(build_mac_host(repo, work / 'tools/dmhost', env, logs, g)), 'DM_OODLE_MODULES': str(patched)}) if src['oodle_packs'][0] or src['ue_legacy'][0] else None oodle = (lambda: {'DM_OODLE_HOST': str(build_mac_host(repo, work / 'tools/dmhost', env, logs, g)), 'DM_OODLE_MODULES': str(patched)}) if src['oodle_packs'][0] or src['ue_legacy'][0] else None
own = bool(provisional or derived) # the shader tables come from this build's split (the profile's are checked otherwise) own = bool(provisional or derived) # the shader tables come from this build's split (the profile's are checked otherwise)
pkey = split_shaders(py, repo, [dirs[v] / 'RomFS' for v in sv], [dirs[v] / 'ExeFS/main' for v in sv], sh / 'pkg', env, logs, g, own, oodle) pkey = split_shaders(py, repo, [dirs[v] / 'RomFS' for v in sv], [dirs[v] / 'ExeFS/main' for v in sv], sh / 'pkg', env, logs, g, own, oodle)
ios = build_shaders(py, repo, sh, xlate, env, logs, g, pkey) ios = build_shaders(py, repo, sh, xlate, env, logs, g, pkey, mac)
if mac: # a Mac-only build ends with the Mac host and the command that runs the game
cmd = mac_run(py, repo, work, game, patched, ios, aoc, gfx, lang, env, logs, g, args.env_file or DM_ENV_FILE)
rmtree(work / 'tmp')
say(f'done: the game runs on the Mac with this command (its save in {work / "run/save"}, kept from run to run; DM_WINDOW=1 opens a window, DM_AUDIO=1 plays sound):')
PROG.say(cmd)
report_profile_status(g, reg, pkg_ver, derived, provisional, 'Mac build')
return
if not prepare_xcode and free_gb(out_ipa.parent) < 5: raise BuildError(f'less than 5 GB free on the volume of {out_ipa.parent}') if not prepare_xcode and free_gb(out_ipa.parent) < 5: raise BuildError(f'less than 5 GB free on the volume of {out_ipa.parent}')
say('preparing the Xcode project' if prepare_xcode else f'building the IPA → {out_ipa}') say('preparing the Xcode project' if prepare_xcode else f'building the IPA → {out_ipa}')
@@ -1589,7 +1668,7 @@ def main(args=None):
say(f'game project: {project}') say(f'game project: {project}')
say(f'Xcode derived data: {work / "xcode-derived-data"}') say(f'Xcode derived data: {work / "xcode-derived-data"}')
say('choose the game scheme in Xcode, select your Team and iPhone or iPad, then Run (Release)') say('choose the game scheme in Xcode, select your Team and iPhone or iPad, then Run (Release)')
report_profile_status(g, reg, pkg_ver, derived, provisional, prepare_xcode=True) report_profile_status(g, reg, pkg_ver, derived, provisional, 'Xcode project')
return return
watch = IpaWatch(repo, work / 'ipa', out_ipa) watch = IpaWatch(repo, work / 'ipa', out_ipa)
PROG.stage('app build', poll=watch.poll, step='starting') # build_ipa.sh's "pack IPA" line starts the stage "IPA pack" PROG.stage('app build', poll=watch.poll, step='starting') # build_ipa.sh's "pack IPA" line starts the stage "IPA pack"