NSP script: --platform mac (PLATFORM = 'mac') builds only what running a game on this Mac needs, from PR #82
A development option; the Mac is the development host, not a platform for players. The same steps up to the shaders (extraction, DLC, rewrite, run-time captures and decodes, heap check, shader split), then only the Mac's: dmsh_build.py without IOSOUT (an inherited one is dropped), a Vulkan or GL game's library without --ios, the Mac host for the profile into <id>_build/tools/dmhost (build_mac_host, through DM_GAMES_DIR / DM_GAMES_FIRST for a provisional or derived profile) and the button-icon font into <id>_build/font. No iOS shaders, Xcode project or IPA; the extracted files are kept whatever KEEP_EXTRACTED says, and the last line is the command that runs the game (its save in <id>_build/run/save, DM_NVN_SHMAP for an NVN game, DM_AOC_DIR with DLC, the --env-file switches). It refuses --prepare-xcode and an Intel Mac. Reuse: the Mac libraries have a key of their own (shader/mac-b-<chain>.key beside the cache folder, shader/vk.key / gl.key), which an iOS build writes too, so a Mac-only build after an IPA reuses its shaders; either key only while the Mac libraries carry a full build's stamp (inputs.tsv). An iOS build after a Mac-only one translates again (dmsh_build.py's per-shader keys include the iOS compile convention) and takes the Mac libraries from the Metal compile cache. build_mac_host now has a reuse key too (the compile line, the sources and headers of the folders it compiles, the generated profile header, libvpx, the third-party headers, clang and the SDK), which also saves the rebuild for the Oodle / LZ5 decodes and the run-time captures. build_vk_shaders and build_gl_shaders share layer_shaders. The default (ios) path builds as before. Checked: dm/tools/tests 34 of 34 (11 in the new test_nsp_mac.py); Hades in Mac mode ran the Mac stages only, printed the run command and exited 0, and a second run reused every stage, the host included (2 s); the host it built boots Hades from that command. Breath of the Wild (NVN, with DLC) in Mac mode restored its 19,592 shaders from the cache, printed a command with DM_AOC_DIR and DM_NVN_SHMAP, and the host booted it from that command into play. Not run end to end here: a GL, provisional or derived game in Mac mode, and an IPA after a Mac-only build (unit tests with the tools stubbed). Co-authored-by: v0lumehi <34629096+v0lumehi@users.noreply.github.com>
This commit is contained in:
1 parent
e98a11562a
commit
98a91a6749
3 files changed
+290
-66
No files matched your search
@@ -0,0 +1,124 @@
|
||||
"""NSP script Mac-only build (scripts/build_ipa_from_nsp.py --platform mac): the shader steps build only the Mac's libraries, each output keeps its own reuse key, a Mac-only build is
|
||||
reused after an iOS build (which builds the Mac's too) and an iOS build is not reused after a Mac-only one, neither key without the Mac libraries' stamp (inputs.tsv, which the one
|
||||
chain version's c1_full.tsv / cb_extent.tsv name); the run command's environment file is read as the iOS app reads it.
|
||||
No game files, keys or Xcode needed (temporary files only in a temporary folder)."""
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
import types
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
REPO = Path(__file__).resolve().parents[3]
|
||||
SCRIPT = REPO / 'scripts/build_ipa_from_nsp.py'
|
||||
spec = importlib.util.spec_from_file_location('build_ipa_from_nsp', SCRIPT)
|
||||
nsp = importlib.util.module_from_spec(spec); spec.loader.exec_module(nsp)
|
||||
CHAIN, INPUTS = '0123456789ab', 'aaaaaaaaaaaa'
|
||||
|
||||
|
||||
class Base(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = Path(tempfile.mkdtemp()); self.addCleanup(shutil.rmtree, self.tmp)
|
||||
self.sh, self.logs, self.ran = self.tmp / 'shader', self.tmp / 'logs', []
|
||||
self.logs.mkdir()
|
||||
for cm in (mock.patch.object(nsp, 'PROG', mock.Mock()), mock.patch.object(nsp, 'say', lambda m: None), mock.patch.object(nsp, 'run', self.fake_run)):
|
||||
cm.__enter__(); self.addCleanup(cm.__exit__, None, None, None)
|
||||
|
||||
def fake_run(self, cmd, log, env=None, cwd=None, on_line=None): # stands in for the translators: writes the libraries a real run writes
|
||||
self.ran.append(([str(c) for c in cmd], dict(env or {})))
|
||||
tool = Path(cmd[1]).name
|
||||
if tool in ('vk_shaders.py', 'gl_shaders.py'):
|
||||
api, a = tool[:2], [str(c) for c in cmd]
|
||||
outs = [Path(a[a.index('--ios') - 1]), Path(a[a.index('--ios') + 1])] if '--ios' in a else [Path(a[-1])]
|
||||
for o in outs: (o / api).mkdir(parents=True, exist_ok=True); (o / api / f'{api}_shaders.metallib').write_bytes(b'lib')
|
||||
log.write_text('macosx: 3 shaders\n' + ('iphoneos: 3 shaders\n' if '--ios' in a else ''))
|
||||
elif tool == 'own_compilers.py': # a full dmsh_build.py run: the libraries, their stamp, and the profile's tables naming it
|
||||
outs = [self.sh / 'cache'] + ([Path(env['IOSOUT'])] if 'IOSOUT' in env else [])
|
||||
for o in outs:
|
||||
(o / f'b-{CHAIN}').mkdir(parents=True, exist_ok=True); (o / f'b-{CHAIN}' / 'k_b.metallib').write_bytes(b'lib')
|
||||
(o / f'b-{CHAIN}' / 'inputs.tsv').write_text(f'# stamp\ngame\t0100535012974000\ninputs\t{INPUTS}\n')
|
||||
for n in ('c1_full.tsv', 'cb_extent.tsv'): (self.tmp / n).write_text(f'# chain b-{CHAIN} inputs {INPUTS} 2026-10-23T00:00:00+00:00\n')
|
||||
log.write_text('shaders 1: ok 1\n')
|
||||
|
||||
|
||||
class Layer(Base):
|
||||
def vk(self, mac, key='K'):
|
||||
self.ran.clear()
|
||||
return nsp.layer_shaders(sys.executable, REPO, 'vk', 'Vulkan', [self.tmp / 'romfs'], self.sh, key, self.logs, {'WORK': str(self.tmp)}, mac)
|
||||
|
||||
def test_mac_only(self): # no --ios, only the Mac key, the Mac folder for DM_MSL_DIR
|
||||
self.assertEqual(self.vk(True), self.sh)
|
||||
self.assertNotIn('--ios', self.ran[0][0])
|
||||
self.assertTrue((self.sh / 'vk.key').is_file()); self.assertFalse((self.sh / 'ios/vk.key').exists())
|
||||
self.assertEqual(self.vk(True), self.sh); self.assertEqual(self.ran, []) # reused
|
||||
|
||||
def test_mac_after_ios(self): # an iOS build makes the Mac library too: a Mac-only build is reused
|
||||
self.assertEqual(self.vk(False), self.sh / 'ios')
|
||||
self.assertIn('--ios', self.ran[0][0])
|
||||
(self.sh / 'vk.key').unlink() # an iOS build from before Mac-only builds wrote only its own key
|
||||
self.assertEqual(self.vk(True), self.sh); self.assertEqual(self.ran, [])
|
||||
|
||||
def test_ios_after_mac(self): # a Mac-only build made no iPhone library: an iOS build runs
|
||||
self.vk(True)
|
||||
self.assertEqual(self.vk(False), self.sh / 'ios'); self.assertEqual(len(self.ran), 1)
|
||||
self.assertTrue((self.sh / 'ios/vk/vk_shaders.metallib').is_file())
|
||||
|
||||
def test_changed_inputs(self):
|
||||
self.vk(False)
|
||||
self.vk(True, 'K2'); self.assertEqual(len(self.ran), 1)
|
||||
self.assertTrue((self.sh / 'ios/vk/vk_shaders.metallib').is_file()) # the iPhone library stays, under its own (older) key
|
||||
self.assertEqual(nsp.reused(self.sh / 'ios/vk.key', 'K'), '3 shaders for the Mac and iOS')
|
||||
|
||||
|
||||
class Nvn(Base):
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
self.g = types.SimpleNamespace(table=lambda n: self.tmp / n, dir=self.tmp)
|
||||
for cm in (mock.patch.object(nsp, 'fingerprint', lambda *a, **k: 'K'), mock.patch.object(nsp, 'chain_of', lambda repo: CHAIN)): # one chain version for every game
|
||||
cm.__enter__(); self.addCleanup(cm.__exit__, None, None, None)
|
||||
|
||||
def build(self, mac):
|
||||
self.ran.clear()
|
||||
return nsp.build_shaders(sys.executable, REPO, self.sh, REPO / 'dm/gfx/tools/dmsh_xlate_bin/dmsh_xlate', {'WORK': str(self.tmp), 'IOSOUT': '/elsewhere'}, self.logs, self.g, pkg_key='P', mac=mac)
|
||||
|
||||
def test_mac_only(self): # no IOSOUT (not even the caller's), the Mac key beside the cache folder, not in it
|
||||
self.assertEqual(self.build(True), self.sh / 'cache')
|
||||
self.assertNotIn('IOSOUT', self.ran[0][1])
|
||||
self.assertTrue((self.sh / f'mac-b-{CHAIN}.key').is_file()); self.assertEqual([p.name for p in (self.sh / 'cache').iterdir()], [f'b-{CHAIN}'])
|
||||
self.assertEqual(self.build(True), self.sh / 'cache'); self.assertEqual(self.ran, [])
|
||||
|
||||
def test_mac_after_ios(self):
|
||||
self.assertEqual(self.build(False), self.sh / 'ios')
|
||||
self.assertEqual(self.ran[0][1]['IOSOUT'], str(self.sh / 'ios'))
|
||||
(self.sh / f'mac-b-{CHAIN}.key').unlink() # an iOS build from before Mac-only builds
|
||||
self.assertEqual(self.build(True), self.sh / 'cache'); self.assertEqual(self.ran, [])
|
||||
|
||||
def test_ios_after_mac(self):
|
||||
self.build(True)
|
||||
self.assertEqual(self.build(False), self.sh / 'ios'); self.assertEqual(len(self.ran), 1)
|
||||
|
||||
def test_no_stamp(self): # Mac libraries no full build stamped (an interrupted one): no table can name them, so neither key reuses them
|
||||
self.build(False)
|
||||
(self.sh / 'cache' / f'b-{CHAIN}' / 'inputs.tsv').unlink()
|
||||
self.assertEqual(self.build(True), self.sh / 'cache'); self.assertEqual(len(self.ran), 1)
|
||||
self.assertTrue((self.sh / 'cache' / f'b-{CHAIN}' / 'inputs.tsv').is_file())
|
||||
|
||||
def test_stale_tables(self): # a reused build makes a table again when it does not name the Mac libraries' stamp
|
||||
self.build(True)
|
||||
(self.tmp / 'cb_extent.tsv').write_text(f'# chain b-{CHAIN} inputs bbbbbbbbbbbb 2026-10-23T00:00:00+00:00\n')
|
||||
self.assertEqual(self.build(True), self.sh / 'cache'); self.assertEqual([Path(c[1]).name for c, _ in self.ran], ['cb_extent.py'])
|
||||
|
||||
|
||||
class EnvFile(Base):
|
||||
def test_pairs(self):
|
||||
p = self.tmp / 'env.txt'; p.write_text('# a comment\nDM_A=1\n DM_B = x=y\n=nokey\nnoeq\n\nDM_C=\n')
|
||||
self.assertEqual(nsp.env_file_pairs(p), {'DM_A': '1', 'DM_B': ' x=y', 'DM_C': ''})
|
||||
|
||||
def test_missing(self):
|
||||
with self.assertRaises(nsp.BuildError): nsp.env_file_pairs(self.tmp / 'none.txt')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
+23
-2
@@ -45,9 +45,12 @@ python3 scripts/build_ipa_from_nsp.py ~/Games/NewGame -k ~/keys/prod.keys
|
||||
|
||||
# test the guesses on a game that has a profile: build from a provisional one (the IPA is named <ipa>-provisional.ipa)
|
||||
python3 scripts/build_ipa_from_nsp.py ~/Games/MarioGalaxy -k ~/keys/prod.keys --provisional --allow-damaged
|
||||
|
||||
# development: build only what running the game on this Mac needs (no iOS shaders, Xcode project or IPA) and print the command that runs it
|
||||
python3 scripts/build_ipa_from_nsp.py ~/Games/MarioWonder -k ~/keys/prod.keys --platform mac
|
||||
```
|
||||
|
||||
In Spyder, open the script, fill in the settings at the top (`NSP_DIR`, `PROD_KEYS`, and optionally `PACKAGE`, `OUT_IPA`, `APP_NAME`, `BUNDLE_ID`, `LANGUAGE`, `KEEP_EXTRACTED`, `SHOW_FPS`; for a game without a profile `HEAP_CHECK`, `HEAP_CHECK_SECS`, `PROVISIONAL`), and press F5. Empty paths are asked for at run time.
|
||||
In Spyder, open the script, fill in the settings at the top (`NSP_DIR`, `PROD_KEYS`, and optionally `PACKAGE`, `OUT_IPA`, `APP_NAME`, `BUNDLE_ID`, `LANGUAGE`, `KEEP_EXTRACTED`, `SHOW_FPS`, `PLATFORM`; for a game without a profile `HEAP_CHECK`, `HEAP_CHECK_SECS`, `PROVISIONAL`), and press F5. Empty paths are asked for at run time.
|
||||
|
||||
What the script does:
|
||||
1. It finds the game profile from the title ID (the `[title ID]` in the file names, or the NSP's ticket; the update by its `[vN]` tag). The update is optional: it packs the newest version in the folder (`-p` / `PACKAGE` picks another).
|
||||
@@ -77,6 +80,24 @@ The game language (`--language`, `LANGUAGE`; English by default) is what `nn::oe
|
||||
|
||||
On the device, environment switches go in `Documents/env.txt` (one `KEY=VALUE` per line), and game files are also accepted at `Documents/romfs`.
|
||||
|
||||
### A Mac-only build for development
|
||||
|
||||
The Mac is the development host, not a platform for players: there is no Mac app. To test or debug a game on the Mac, `--platform mac` (`PLATFORM = 'mac'` in Spyder) builds only what running it in the Mac host needs, from the same NSP folder:
|
||||
- the same steps up to the shaders: extraction, rewrite, run-time shader captures and decodes, a provisional profile's heap check and the shader split;
|
||||
- the Mac shader libraries only: `dmsh_build.py` without `IOSOUT` into `<id>_build/shader/cache/b-<chain>/`, which a full build stamps with the profile's inputs (`inputs.tsv`) and from which it makes the profile's `c1_full.tsv` and `cb_extent.tsv`, or a Vulkan or GL game's library without `--ios` into `<id>_build/shader/vk/` or `gl/`;
|
||||
- the Mac host for the profile into `<id>_build/tools/dmhost`: the compile line of `make -C dm/host GAME=<id> dmhost`, with a provisional or derived profile through `DM_GAMES_DIR` / `DM_GAMES_FIRST` as for the script's other host builds. It is reused while its sources (the chain header `dm/gfx/nvn/dmsh_chain.h` among them), the generated profile header, libvpx and the compiler are unchanged;
|
||||
- the button-icon font (`make_ext_font.py`) into `<id>_build/font/`.
|
||||
|
||||
It skips the iOS shaders, the Xcode project and the IPA. For Mario Kart 8 Deluxe that saves about 30,000 iOS shader compiles and the 12 GB IPA. It keeps the extracted files even with `--no-keep-extracted`, because the game runs from them. It needs a Mac with Apple silicon, and it cannot be combined with `--prepare-xcode`. The last line it prints is the command that runs the game, built from the run's own paths (step 5 of [Build by hand (Mac)](#build-by-hand-mac)), for example:
|
||||
|
||||
```
|
||||
cd "<NSP folder>/<id>_build/run" && DM_ROOT="<NSP folder>/<id>_build" DM_ROMFS="<NSP folder>/<id>_build/extract/<version>/RomFS" DM_SAVE="<NSP folder>/<id>_build/run/save" DM_TEMP="<NSP folder>/<id>_build/run/temp" DM_MSL_DIR="<NSP folder>/<id>_build/shader/cache" DM_FONT_DIR="<NSP folder>/<id>_build/font" DM_NVN_SHMAP="<repository>/dm/games/<id>/shader_map.tsv" DM_LANG=en-US DM_WINDOW=1 DM_AUDIO=1 "<NSP folder>/<id>_build/tools/dmhost" "<NSP folder>/<id>_build/patched_<version>"
|
||||
```
|
||||
|
||||
The save is in `<id>_build/run/save` and stays there from run to run. `DM_AOC_DIR` is added when the folder has DLC. `DM_NVN_SHMAP` is added for an NVN game; it names the profile's shader map, or the derived or provisional profile's. A Vulkan or GL game's `DM_MSL_DIR` is `<id>_build/shader`. The switches of `--env-file` are added to the command.
|
||||
|
||||
Reuse works as for the IPA. Each shader output has its own key, named after the one chain version: `shader/ios/b-<chain>.key` for the iOS libraries and `shader/mac-b-<chain>.key` for the Mac's (`shader/ios/vk.key` and `shader/vk.key` for a Vulkan game, the same with `gl` for a GL game). An iOS build writes both keys, since it builds the Mac's libraries too, so a Mac-only build of a folder already packed as an IPA reuses its shaders. Either key is taken only while the Mac libraries carry a full build's stamp; `c1_full.tsv` or `cb_extent.tsv` that does not name it is made again from them, as for the IPA. An iOS build after a Mac-only build still has the iPhone's libraries to make. It translates the shaders again, because `dmsh_build.py`'s per-shader cache keys include the iOS compile convention. It then takes the Mac libraries from the Metal compile cache (`shader/cache/cas/metallib`) and compiles only the iPhone's.
|
||||
|
||||
## Build and run on a device with Xcode
|
||||
|
||||
With an Apple developer account, Xcode can sign and install the app itself:
|
||||
@@ -121,7 +142,7 @@ To add a game:
|
||||
|
||||
## Build by hand (Mac)
|
||||
|
||||
You need the same tools as the script. The shader translator is the prebuilt `dm/gfx/tools/dmsh_xlate_bin`, the build that matches `dmsh_chain.lock`; a fresh `dotnet publish` gives different fingerprints. Its launcher `dmsh_xlate` is a universal binary (arm64 and x86_64), so an Intel Mac runs it too, with the same output; after a rebuild, join the new launcher with lipo to the one `dotnet build -r osx-x64` makes of the same .NET version (the lock does not cover the launcher). What an Intel Mac cannot run is the Mac host, which runs the game's own arm64 code: the NSP script stops at a step that boots the game there (a run-time shader capture, a decode with the game's own Oodle or LZ5) and skips a provisional profile's heap check. The commands use three variables:
|
||||
`scripts/build_ipa_from_nsp.py --platform mac` does steps 1 to 4 in the game's NSP folder and prints the command of step 5 ([A Mac-only build for development](#a-mac-only-build-for-development)). The steps by hand follow. You need the same tools as the script. The shader translator is the prebuilt `dm/gfx/tools/dmsh_xlate_bin`, the build that matches `dmsh_chain.lock`; a fresh `dotnet publish` gives different fingerprints. Its launcher `dmsh_xlate` is a universal binary (arm64 and x86_64), so an Intel Mac runs it too, with the same output; after a rebuild, join the new launcher with lipo to the one `dotnet build -r osx-x64` makes of the same .NET version (the lock does not cover the launcher). What an Intel Mac cannot run is the Mac host, which runs the game's own arm64 code: the NSP script stops at a step that boots the game there (a run-time shader capture, a decode with the game's own Oodle or LZ5) and skips a provisional profile's heap check. The commands use three variables:
|
||||
- `G`, the profile;
|
||||
- `W`, the game's work folder: `<title ID>-work` inside the game's NSP folder, outside this repository and off the home folder (it grows to the size of the game files);
|
||||
- `VER`, a version folder named after the NSP version.
|
||||
|
||||
+143
-64
@@ -6,6 +6,9 @@
|
||||
# → iOS shader libraries (dmsh_build.py) → build_ipa.sh. A Vulkan game (main imports Vulkan, not NVN: Hades) skips the NVN shader steps: vk_shaders.py translates its SPIR-V shaders instead.
|
||||
# Which of these steps a game needs follows from its files (dm/games/profile_guess.py shader_sources), not from its profile. A game with no matching profile is surveyed (<NSP folder>/<title ID>_survey/survey.md), then gets a provisional profile guessed from
|
||||
# the survey (<NSP folder>/<title ID>_build/profile/<title ID>/, outside the repository, found through DM_GAMES_DIR) and a best-effort IPA that may not boot; SURVEY_ONLY / --survey-only stops after the survey.
|
||||
# PLATFORM = 'mac' / --platform mac is a development option (the Mac is the development host, not a platform for players): the same steps up to the shaders, but only the Mac's
|
||||
# (the shader build without IOSOUT, a Vulkan or GL game's libraries without --ios), then the Mac host for the profile and the button-icon font; no iOS shaders, Xcode project or IPA.
|
||||
# The extracted files are kept whatever KEEP_EXTRACTED says (the game runs from them), and the last line is the command that runs the game in the Mac host, its save in <id>_build/run/save.
|
||||
# Two ways to run it: open this file in Spyder and press F5 (it uses the values under "Settings" below; an empty NSP folder or prod.keys is asked for at run time);
|
||||
# or in a terminal: python3 scripts/build_ipa_from_nsp.py <NSP folder> -k <prod.keys> [options] (values given on the command line override "Settings", -h lists every option;
|
||||
# without keyboard input a missing value is not asked for but reported as an error; exit code 0 on success, 1 on error, 130 when interrupted). Command-line options entered in Spyder's run configuration work the same way.
|
||||
@@ -33,6 +36,8 @@ OUT_IPA = '' # output IPA (empty = <the profile's ipa
|
||||
APP_NAME = '' # app name on the home screen (empty = the game name in the base NSP)
|
||||
BUNDLE_ID = '' # the app's bundle ID (empty = saved per-game Xcode ID, else org.nxnative.<base game title ID in lowercase>)
|
||||
PREPARE_XCODE = False # prepare a per-game Xcode project and add it to the workspace instead of packing an IPA; requires KEEP_EXTRACTED
|
||||
PLATFORM = 'ios' # 'ios' = the IPA (or the Xcode project); 'mac' = development: build only what running the game on this Mac needs (Mac shaders, the Mac host), no iOS shaders,
|
||||
# Xcode project or IPA; keeps the extracted files and prints the command that runs the game (needs Apple silicon)
|
||||
TEAM_ID = '' # optional Apple team ID; empty = DM_TEAM_ID environment, saved Xcode selection, or select in Xcode
|
||||
XCODE_WORKSPACE = '' # shared .xcworkspace (empty = NxNative.xcworkspace beside the game's NSP folder)
|
||||
LANGUAGE = '' # game language, a code or a name: 'ja', 'fr', 'de', 'es', 'zh-Hans', 'zh-Hant', 'ko', 'Japanese', ... (empty = English); it must be one the game supports (the console line lists them);
|
||||
@@ -690,13 +695,14 @@ def profile_files(g): # every file of the game profile (game.tsv and its table
|
||||
|
||||
|
||||
# ---- Steps ----
|
||||
def check_prereqs(env, xlate):
|
||||
def check_prereqs(env, xlate, mac=False): # mac: a Mac-only build (PLATFORM 'mac'), which needs no xcodegen and compiles shaders only for macOS
|
||||
missing = [m for m in ('lz4.block', 'capstone', 'fontTools') if not _importable(m)] + ([] if _importable('compression.zstd') or _importable('backports.zstd') else ['backports.zstd']) # zstd: built in from Python 3.14
|
||||
if missing: raise BuildError(f'the current Python ({sys.executable}) is missing modules: {", ".join(missing)} (pip install lz4 capstone fonttools backports.zstd; backports.zstd only before Python 3.14)')
|
||||
tools = {'xcodebuild': 'Xcode', 'xcodegen': 'brew install xcodegen', 'dotnet': '.NET SDK 10', 'zstd': 'brew install zstd', 'spirv-cross': 'brew install spirv-cross', 'make': 'Xcode'}
|
||||
if mac: del tools['xcodegen']
|
||||
gone = [f'{t} ({how})' for t, how in tools.items() if not shutil.which(t, path=env['PATH'])]
|
||||
if gone: raise BuildError('tools not found: ' + ', '.join(gone))
|
||||
mtl = subprocess.run(['xcrun', '-sdk', 'iphoneos', 'metal', '--version'], capture_output=True, text=True, env=env) # Xcode 26 and later install the Metal toolchain separately
|
||||
mtl = subprocess.run(['xcrun', '-sdk', 'macosx' if mac else 'iphoneos', 'metal', '--version'], capture_output=True, text=True, env=env) # Xcode 26 and later install the Metal toolchain separately
|
||||
if mtl.returncode: raise BuildError(f'the Metal compiler does not run ({(mtl.stderr.strip().splitlines() or ["no output"])[-1]}): install it with xcodebuild -downloadComponent MetalToolchain')
|
||||
if not xlate.exists(): raise BuildError(f'the shader translator is missing from the repository: {xlate}')
|
||||
|
||||
@@ -831,9 +837,13 @@ def apple_silicon(): # hw.optional.arm64 is 1 on Apple silicon (for a Python u
|
||||
return subprocess.run(['sysctl', '-n', 'hw.optional.arm64'], capture_output=True, text=True).stdout.strip() == '1'
|
||||
|
||||
|
||||
HOST_DIRS = ('dm/host', 'dm/gfx', 'dm/gfx/nvn', 'dm/gfx/nvn/include', 'dm/gfx/tex', 'dm/gfx/vk', 'dm/gfx/gl') # the folders the host's compile line takes its sources and headers from (dm/host/Makefile SRC, GFX, GFX_H)
|
||||
|
||||
|
||||
def build_mac_host(repo, out, env, logs, g):
|
||||
"""The Mac host for the profile, built into out: dm/host's own compile line (make -n) with the output moved, since make takes no target path with spaces and dm/host/dmhost may be another build's.
|
||||
The host runs the game's own arm64 code natively, so an Intel Mac stops here (GitHub issue #54)"""
|
||||
Reused when the compile line and its inputs (the source files of HOST_DIRS, the chain header dm/gfx/nvn/dmsh_chain.h among them, the profile header make generates, libvpx and the third-party headers, the compiler
|
||||
and the macOS SDK) are as at the last build into out. The host runs the game's own arm64 code natively, so an Intel Mac stops here (GitHub issue #54)"""
|
||||
if not apple_silicon(): raise BuildError('this step boots the game in the Mac host, which runs the game\'s own arm64 code natively: it needs a Mac with Apple silicon, and this Mac has an Intel processor')
|
||||
host = repo / 'dm/host'
|
||||
run(['make', '-s', f'GAME={g.id}', f'gen/{g.id}/game_profile.h', 'gen/vpx/libvpx.a'], logs / 'mac_host_make.log', env=env, cwd=host)
|
||||
@@ -841,8 +851,14 @@ def build_mac_host(repo, out, env, logs, g):
|
||||
line = next((l for l in dry.stdout.splitlines() if ' -o dmhost ' in l), None)
|
||||
if not line: raise BuildError(f'make -n gave no compile line for the Mac host: {dry.stderr.strip()[-500:]}')
|
||||
cmd = shlex.split(line); cmd[cmd.index('-o') + 1] = out
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
kf, tp = out.with_name(out.name + '.key'), repo / 'dm/third_party'
|
||||
srcs = [p for d in HOST_DIRS for p in sorted((repo / d).iterdir()) if p.is_file() and p.suffix in ('.c', '.m', '.h', '.S', '.inc')]
|
||||
key = fingerprint(cmd, ('files', srcs + [host / f'gen/{g.id}/game_profile.h', host / 'gen/vpx/libvpx.a', tp / 'stb_vorbis/stb_vorbis.c']),
|
||||
('tree', tp / 'vulkan-headers/include'), ('tree', tp / 'libvpx/vpx'), ('cmd', ['clang', '--version']), ('cmd', ['xcrun', '--show-sdk-build-version']), env=env)
|
||||
if out.is_file() and reused(kf, key) is not None: return out
|
||||
rmtree(kf); out.parent.mkdir(parents=True, exist_ok=True)
|
||||
run(cmd, logs / 'mac_host_build.log', env=env, cwd=host)
|
||||
kf.write_text(f'{key}\n')
|
||||
return out
|
||||
|
||||
|
||||
@@ -1172,10 +1188,10 @@ def split_shaders(py, repo, roms, mains, pkg, env, logs, g, provisional=False, o
|
||||
|
||||
class ShaderWatch:
|
||||
"""Progress of dmsh_build.py, read from the files it writes (it prints nothing until it ends). It works in three phases:
|
||||
the cache check (a hit restores <key>_b.json into the iOS output b-<chain>/, a miss writes <key>.bin into its temporary folder dmsh_* under WORK),
|
||||
the cache check (a hit restores <key>_b.json into the library directory b-<chain>/: the iOS one, or the Mac's for a Mac-only build; a miss writes <key>.bin into its temporary folder dmsh_* under WORK),
|
||||
the translation of the misses (<key>.spv in the temporary folder), and the compile, which takes the keys in a fixed order (sorted, the vertex shaders with pixel pairs last)
|
||||
and writes <key>_b.json again for every miss and every pixel shader: the furthest key written so far tells how far it got"""
|
||||
def __init__(self, g, out, tmp): # out: the iOS library directory b-<chain>/
|
||||
def __init__(self, g, out, tmp): # out: the library directory b-<chain>/ (the iOS one, or the Mac's for a Mac-only build)
|
||||
keys = set()
|
||||
for ln in open(g.table('shader_map.tsv'), errors='replace'): # the keys dmsh_build.py builds: column 4 of the rows from a shader package
|
||||
c = ln.rstrip('\n').split('\t')
|
||||
@@ -1186,7 +1202,7 @@ class ShaderWatch:
|
||||
self.out, self.tmp, self.t0, self.t1, self.w = out, tmp, time.time(), None, None
|
||||
self.hits, self.fin, self.top = set(), set(), -1
|
||||
|
||||
def written(self, keys, since): # those of keys whose <key>_b.json in the iOS output was written after since
|
||||
def written(self, keys, since): # those of keys whose <key>_b.json in the output was written after since
|
||||
found = set()
|
||||
try:
|
||||
with os.scandir(self.out) as it:
|
||||
@@ -1217,30 +1233,41 @@ def toolchain(): # the compilers' versions, part of the shader stages' keys
|
||||
return ('cmd', ['xcodebuild', '-version']), ('cmd', ['spirv-cross', '--revision'])
|
||||
|
||||
|
||||
def build_shaders(py, repo, sh, xlate, env, logs, g, pkg_key=None):
|
||||
"""The iOS shader libraries (dmsh_build.py). Reused when the shader package (pkg_key, its content; None never reuses) and the chain's tools, translator binaries, profile and compilers
|
||||
are as at the last successful build: dmsh_build.py would only restore every shader from its cache (63,000 for Breath of the Wild: about 100 s)"""
|
||||
def build_shaders(py, repo, sh, xlate, env, logs, g, pkg_key=None, mac=False):
|
||||
"""The iOS shader libraries (dmsh_build.py, which builds the Mac's into sh/cache in the same run), or with mac only the Mac's (a Mac-only build: no IOSOUT). Returns the folder the host takes
|
||||
them from. Reused when the shader package (pkg_key, its content; None never reuses) and the chain's tools, translator binaries, profile and compilers are as at the last successful build:
|
||||
dmsh_build.py would only restore every shader from its cache (63,000 for Breath of the Wild: about 100 s). The key of the iOS libraries is sh/ios/b-<chain>.key, that of the Mac's
|
||||
sh/mac-b-<chain>.key (not in sh/cache, whose b-* entries dmsh_build.py reads as library folders), which an iOS build writes too, so a Mac-only build after it is reused; either is taken only
|
||||
while the Mac libraries carry a full build's stamp (inputs.tsv) for c1_full.tsv and cb_extent.tsv to name (chain_tables).
|
||||
An iOS build after a Mac-only one translates the shaders again (dmsh_build.py's per-shader cache keys include the iOS compile convention), but takes the Mac libraries from its
|
||||
Metal compile cache (dmsh_mtlcache.py), so only the iPhone's are compiled"""
|
||||
ios, log = sh / 'ios', logs / 'dmsh_build.log'
|
||||
PROG.stage('iOS shaders')
|
||||
PROG.stage('Mac shaders' if mac else 'iOS shaders')
|
||||
chain = chain_of(repo)
|
||||
libs = ios / f'b-{chain}'; kf = libs.with_name(libs.name + '.key')
|
||||
out = sh / 'cache' if mac else ios
|
||||
libs, kf, kmac = out / f'b-{chain}', ios / f'b-{chain}.key', sh / f'mac-b-{chain}.key'
|
||||
if pkg_key:
|
||||
PROG.set(step='checking the inputs')
|
||||
key = fingerprint(pkg_key, ('files', chain_files(repo)), ('stat', xlate.parent), ('files', profile_files(g)), *toolchain(), str(xlate),
|
||||
{k: env.get(k) for k in ('XLATE_DEBUG', 'XLATE_REDUCED')}, env=env)
|
||||
note = next((n for k in ((kmac, kf) if mac else (kf,)) if (n := reused(k, key)) is not None), None)
|
||||
# the libraries are the last build's, so are tables that name their stamp; a missing or stale one is made from that build's Mac outputs, and without them the step runs again
|
||||
if next(libs.glob('*_b.metallib'), None) and (note := reused(kf, key)) is not None and chain_tables(py, repo, sh, env, logs, g, chain):
|
||||
PROG.end(f'{note}, reused (inputs unchanged)'); return ios
|
||||
if next(libs.glob('*_b.metallib'), None) and note is not None and chain_tables(py, repo, sh, env, logs, g, chain):
|
||||
PROG.end(f'{note}, reused (inputs unchanged)'); return out
|
||||
else: key = None
|
||||
rmtree(kf)
|
||||
rmtree(kmac)
|
||||
if not mac: rmtree(kf)
|
||||
try: PROG.watch(ShaderWatch(g, libs, Path(env['WORK'])))
|
||||
except Exception: pass # progress never stops the build (a missing table is dmsh_build.py's to report)
|
||||
run([py, repo / 'dm/gfx/tools/own_compilers.py', sh / 'pkg', sh / 'cache'], log, env=dict(env, XLATE=str(xlate), IOSOUT=str(ios))) # dmsh_build.py with this Mac's Metal compiler and spirv-cross
|
||||
benv = dict(env, XLATE=str(xlate), IOSOUT=str(ios))
|
||||
if mac: del benv['IOSOUT'] # dmsh_build.py builds the iOS libraries only with IOSOUT
|
||||
run([py, repo / 'dm/gfx/tools/own_compilers.py', sh / 'pkg', sh / 'cache'], log, env=benv) # dmsh_build.py with this Mac's Metal compiler and spirv-cross
|
||||
PROG.watch(None)
|
||||
if not chain_tables(py, repo, sh, env, logs, g, chain): raise BuildError(f'no stamped Mac shader outputs in {sh / "cache" / f"b-{chain}"} (inputs.tsv): dmsh_build.py did not finish a full build')
|
||||
what = 'Mac' if mac else 'iOS'
|
||||
n = sum(1 for _ in libs.glob('*_b.metallib')) if libs.is_dir() else 0
|
||||
if n == 0: raise BuildError(f'no iOS shader libraries in {libs}')
|
||||
say(f'{n} iOS shader libraries ({libs})')
|
||||
if n == 0: raise BuildError(f'no {what} shader libraries in {libs}')
|
||||
say(f'{n} {what} shader libraries ({libs})')
|
||||
t = log.read_text(errors='replace')
|
||||
if own := re.search(r'^note: (built with this Mac\'s compilers.*)$', t, re.M): say(own.group(1))
|
||||
m, c = re.search(r'shaders (\d+): ok (\d+)', t), re.search(r'shader cache: hit (\d+)', t)
|
||||
@@ -1251,49 +1278,50 @@ def build_shaders(py, repo, sh, xlate, env, logs, g, pkg_key=None):
|
||||
note = f'{ok:,} of {total:,} shaders ready{failed}'
|
||||
PROG.end(f'{ok:,} of {total:,} shaders ready' + (f' ({int(c.group(1)):,} from the cache)' if c else '') + failed)
|
||||
else: PROG.end()
|
||||
if key: kf.write_text(f'{key}\n{note}\n')
|
||||
return ios
|
||||
if key:
|
||||
for k in (kmac,) if mac else (kf, kmac): k.write_text(f'{key}\n{note}\n')
|
||||
return out
|
||||
|
||||
|
||||
def build_vk_shaders(py, repo, roots, sh, env, logs, g, reuse=True):
|
||||
"""A Vulkan game's SPIR-V shaders (roots: the RomFS, and a run-time capture folder) → one Metal library each for the Mac (sh/vk/) and the iPhone (sh/ios/vk/, what build_ipa.sh takes from MSL_IOS),
|
||||
replacing the NVN split and chain build. Reused when the roots (paths and sizes), the tools, the profile and the compilers are as at the last successful run (reuse False: never).
|
||||
Otherwise the old output is deleted first, so a failed run leaves nothing for build_ipa.sh to pick up; vk_shaders.py's temporary files go to WORK (on the NSP folder's volume)"""
|
||||
ios = sh / 'ios'; kf = ios / 'vk.key'
|
||||
def layer_shaders(py, repo, api, name, roots, sh, key, logs, env, mac):
|
||||
"""What build_vk_shaders and build_gl_shaders share: dm/gfx/tools/<api>_shaders.py on the roots → sh/<api>/ for the Mac and, unless mac (a Mac-only build), sh/ios/<api>/ for the iPhone.
|
||||
Returns the folder the host (DM_MSL_DIR) or build_ipa.sh (MSL_IOS) takes the libraries from. key: the inputs' fingerprint (None never reuses). The iOS libraries' key is sh/ios/<api>.key,
|
||||
the Mac's sh/<api>.key, which an iOS build writes too (it builds both), so a Mac-only build is reused after either; an iOS build after a Mac-only one translates again.
|
||||
Otherwise the old output is deleted first, so a failed run leaves nothing for build_ipa.sh or the host to pick up; the translator's temporary files go to WORK (on the NSP folder's volume)"""
|
||||
ios = sh / 'ios'; out = sh if mac else ios
|
||||
kf, kmac, lib, log = ios / f'{api}.key', sh / f'{api}.key', out / api / f'{api}_shaders.metallib', logs / f'{api}_shaders.log'
|
||||
note = next((n for k in ((kmac, kf) if mac else (kf,)) if (n := reused(k, key)) is not None), None) if key else None
|
||||
if lib.is_file() and note is not None:
|
||||
PROG.end(f'{note}, reused (inputs unchanged)'); return out
|
||||
PROG.set(step='translating')
|
||||
rmtree(kmac); rmtree(sh / api)
|
||||
if not mac: rmtree(kf); rmtree(ios / api)
|
||||
run([py, repo / f'dm/gfx/tools/{api}_shaders.py', *roots, sh] + ([] if mac else ['--ios', ios]), log, env=dict(env, TMPDIR=env['WORK']))
|
||||
if not lib.is_file(): raise BuildError(f'no {"Mac" if mac else "iOS"} {name} shader library in {lib.parent}')
|
||||
m = re.search(r'macosx: (\d+)' if mac else r'iphoneos: (\d+)', log.read_text(errors='replace'))
|
||||
note = f'{int(m.group(1)):,} shaders for the Mac' + ('' if mac else ' and iOS') if m else ''
|
||||
if key:
|
||||
for k in (kmac,) if mac else (kf, kmac): k.write_text(f'{key}\n{note}\n')
|
||||
PROG.end(note)
|
||||
return out
|
||||
|
||||
|
||||
def build_vk_shaders(py, repo, roots, sh, env, logs, g, reuse=True, mac=False):
|
||||
"""A Vulkan game's SPIR-V shaders (roots: the RomFS, and a run-time capture folder) → one Metal library each for the Mac (sh/vk/) and the iPhone (sh/ios/vk/, what build_ipa.sh takes from MSL_IOS;
|
||||
not with mac), replacing the NVN split and chain build (layer_shaders). Reused when the roots (paths and sizes), the tools, the profile and the compilers are as at the last successful run (reuse False: never)"""
|
||||
PROG.stage('Vulkan shaders', step='checking the inputs')
|
||||
key = fingerprint(*[('tree', r) for r in roots], ('files', tool_files(repo)), ('files', profile_files(g)), *toolchain(), env=env) if reuse else None
|
||||
if key and (ios / 'vk/vk_shaders.metallib').is_file() and (note := reused(kf, key)) is not None:
|
||||
PROG.end(f'{note}, reused (inputs unchanged)'); return ios
|
||||
PROG.set(step='translating')
|
||||
rmtree(kf); rmtree(sh / 'vk'); rmtree(ios / 'vk')
|
||||
run([py, repo / 'dm/gfx/tools/vk_shaders.py', *roots, sh, '--ios', ios], logs / 'vk_shaders.log', env=dict(env, TMPDIR=env['WORK']))
|
||||
if not (ios / 'vk/vk_shaders.metallib').is_file(): raise BuildError(f'no iOS Vulkan shader library in {ios / "vk"}')
|
||||
m = re.search(r'iphoneos: (\d+)', (logs / 'vk_shaders.log').read_text(errors='replace'))
|
||||
note = f'{int(m.group(1)):,} shaders for the Mac and iOS' if m else ''
|
||||
if key: kf.write_text(f'{key}\n{note}\n')
|
||||
PROG.end(note)
|
||||
return ios
|
||||
return layer_shaders(py, repo, 'vk', 'Vulkan', roots, sh, key, logs, env, mac)
|
||||
|
||||
|
||||
def build_gl_shaders(py, repo, roots, sh, env, logs, g, reuse=True):
|
||||
"""An OpenGL game's GLSL sources (roots: the RomFS, main, a run-time capture folder) → one Metal library each for the Mac (sh/gl/) and the iPhone (sh/ios/gl/, what build_ipa.sh takes from MSL_IOS),
|
||||
replacing the NVN split and chain build, as build_vk_shaders does for a Vulkan game; glslang (glslangValidator) compiles the GLSL to SPIR-V for spirv-cross"""
|
||||
ios = sh / 'ios'; kf = ios / 'gl.key'
|
||||
def build_gl_shaders(py, repo, roots, sh, env, logs, g, reuse=True, mac=False):
|
||||
"""An OpenGL game's GLSL sources (roots: the RomFS, main, a run-time capture folder) → one Metal library each for the Mac (sh/gl/) and the iPhone (sh/ios/gl/, what build_ipa.sh takes from MSL_IOS;
|
||||
not with mac), replacing the NVN split and chain build, as build_vk_shaders does for a Vulkan game; glslang (glslangValidator) compiles the GLSL to SPIR-V for spirv-cross"""
|
||||
PROG.stage('GL shaders', step='checking the inputs')
|
||||
if not shutil.which('glslangValidator', path=env['PATH']): raise BuildError('glslangValidator not found (brew install glslang): it compiles an OpenGL game\'s GLSL shaders')
|
||||
key = fingerprint(*[('tree', r) if Path(r).is_dir() else ('files', [r]) for r in roots], ('files', tool_files(repo)), ('files', profile_files(g)), *toolchain(),
|
||||
('cmd', ['glslangValidator', '--version']), env=env) if reuse else None
|
||||
if key and (ios / 'gl/gl_shaders.metallib').is_file() and (note := reused(kf, key)) is not None:
|
||||
PROG.end(f'{note}, reused (inputs unchanged)'); return ios
|
||||
PROG.set(step='translating')
|
||||
rmtree(kf); rmtree(sh / 'gl'); rmtree(ios / 'gl')
|
||||
run([py, repo / 'dm/gfx/tools/gl_shaders.py', *roots, sh, '--ios', ios], logs / 'gl_shaders.log', env=dict(env, TMPDIR=env['WORK']))
|
||||
if not (ios / 'gl/gl_shaders.metallib').is_file(): raise BuildError(f'no iOS GL shader library in {ios / "gl"}')
|
||||
m = re.search(r'iphoneos: (\d+)', (logs / 'gl_shaders.log').read_text(errors='replace'))
|
||||
note = f'{int(m.group(1)):,} shaders for the Mac and iOS' if m else ''
|
||||
if key: kf.write_text(f'{key}\n{note}\n')
|
||||
PROG.end(note)
|
||||
return ios
|
||||
return layer_shaders(py, repo, 'gl', 'GL', roots, sh, key, logs, env, mac)
|
||||
|
||||
|
||||
class IpaWatch:
|
||||
@@ -1346,6 +1374,43 @@ class IpaWatch:
|
||||
return r
|
||||
|
||||
|
||||
def env_file_pairs(path): # an environment file's switches (one KEY=VALUE per line, # starts a comment), read as the iOS app reads them (dm/ios/Sources/main.m)
|
||||
out = {}
|
||||
try: text = Path(os.path.expanduser(path)).read_text(errors='replace')
|
||||
except OSError as e: raise BuildError(f'cannot read the environment file (--env-file / DM_ENV_FILE): {e}') from e
|
||||
for ln in text.splitlines():
|
||||
k, eq, v = ln.strip().partition('=')
|
||||
if k.strip() and eq and not k.startswith('#'): out[k.strip()] = v
|
||||
return out
|
||||
|
||||
|
||||
def mac_run(py, repo, work, game, patched, msl, aoc, gfx, lang, env, logs, g, env_file=''):
|
||||
"""The last stage of a Mac-only build (PLATFORM 'mac'): the Mac host for the profile (build_mac_host, into <id>_build/tools/dmhost, where the run-time captures build it too; DM_GAMES_DIR /
|
||||
DM_GAMES_FIRST in env pick a provisional or derived profile) and the button-icon font the host gives a game for the Nintendo extension shared font (make_ext_font.py, into <id>_build/font,
|
||||
made again when the tool changed). Returns the command that runs the game in a window with sound (docs/development.md, Build by hand (Mac), step 5): its save and temporary storage in
|
||||
<id>_build/run (kept from run to run), the extracted RomFS, the Mac shader libraries (msl), the DLC, the profile's shader map (a host outside dm/host does not find it beside itself;
|
||||
c1_full.tsv and cb_extent.tsv are read beside it), the game language, and the switches of an environment file (DM_ENV_FILE / --env-file)"""
|
||||
PROG.stage('Mac host', step='building')
|
||||
host = build_mac_host(repo, work / 'tools/dmhost', env, logs, g)
|
||||
PROG.set(step='button-icon font')
|
||||
font = work / 'font'; ttf, kf = font / 'nintendo_ext.ttf', font / 'nintendo_ext.key'
|
||||
key = fingerprint(('files', [repo / 'dm/tools/make_ext_font.py']))
|
||||
if not (ttf.is_file() and reused(kf, key) is not None):
|
||||
rmtree(kf); font.mkdir(parents=True, exist_ok=True)
|
||||
run([py, repo / 'dm/tools/make_ext_font.py', ttf], logs / 'make_ext_font.log', env=env)
|
||||
kf.write_text(f'{key}\n')
|
||||
rd = work / 'run'
|
||||
for d in ('save', 'temp'): (rd / d).mkdir(parents=True, exist_ok=True)
|
||||
renv = {'DM_ROOT': work, 'DM_ROMFS': game / 'RomFS', 'DM_SAVE': rd / 'save', 'DM_TEMP': rd / 'temp', 'DM_MSL_DIR': msl, 'DM_FONT_DIR': font}
|
||||
if aoc: renv['DM_AOC_DIR'] = aoc
|
||||
if gfx == 'nvn': renv['DM_NVN_SHMAP'] = g.table('shader_map.tsv')
|
||||
renv.update(DM_LANG=lang, DM_WINDOW=1, DM_AUDIO=1)
|
||||
if env_file: renv.update(env_file_pairs(env_file))
|
||||
PROG.end(str(host))
|
||||
q = lambda v: shlex.quote(str(v))
|
||||
return f'cd {q(rd)} && ' + ' '.join(f'{k}={q(v)}' for k, v in renv.items()) + f' {q(host)} {q(patched)}'
|
||||
|
||||
|
||||
def find_repo(here): # this file lives in the repository's scripts/; walk up to the level that has dm/tools/nx_extract/Program.cs
|
||||
for d in here.parents:
|
||||
if (d / 'dm/tools/nx_extract/Program.cs').exists(): return d
|
||||
@@ -1365,6 +1430,8 @@ def parse_args(argv):
|
||||
ap.add_argument('--app-name', help='app name on the home screen (default: the game name in the base NSP)')
|
||||
ap.add_argument('--bundle-id', help='the app\'s bundle ID (default: saved Xcode ID when preparing again, else org.nxnative.<base game title ID in lowercase>)')
|
||||
ap.add_argument('--prepare-xcode', action='store_true', help='prepare a per-game Xcode project and add it to the shared workspace instead of packing an IPA (requires keeping extracted files)')
|
||||
ap.add_argument('--platform', choices=('ios', 'mac'), help='ios = the IPA (or the Xcode project); mac = for development: build only what running the game on this Mac needs (the Mac shaders and the Mac host, '
|
||||
'no iOS shaders, Xcode project or IPA), keep the extracted files and print the command that runs the game (default: the PLATFORM setting, ios)')
|
||||
ap.add_argument('--team-id', help='Apple developer team ID (default: TEAM_ID setting, DM_TEAM_ID environment, or saved Xcode selection)')
|
||||
ap.add_argument('--xcode-workspace', help='workspace collecting prepared games (default: NxNative.xcworkspace beside the NSP folder; use with --prepare-xcode)')
|
||||
ap.add_argument('-l', '--language', help='game language, a code or a name: ja, fr, de, es, it, nl, pt, ru, ko, zh-Hans, zh-Hant, en-GB, fr-CA, es-419, pt-BR, Japanese, ... (default: the LANGUAGE setting, else English); '
|
||||
@@ -1382,9 +1449,8 @@ def parse_args(argv):
|
||||
return ap.parse_args(argv)
|
||||
|
||||
|
||||
def report_profile_status(g, reg, pkg_ver, derived, provisional, prepare_xcode=False):
|
||||
"""Keep build-specific warnings visible for both IPA and Xcode output."""
|
||||
artifact = 'Xcode project' if prepare_xcode else 'IPA'
|
||||
def report_profile_status(g, reg, pkg_ver, derived, provisional, artifact='IPA'):
|
||||
"""Keep build-specific warnings visible for the IPA, the Xcode project and a Mac-only build"""
|
||||
disp = g.builds()[pkg_ver]['display'] if derived else ''
|
||||
if derived: # the modules whose build no registered version has: the rewriter's cuts and the host's fixes made for a registered build's code do not reach them
|
||||
known = {(m, b) for row in reg.builds().values() for m, b in row.items() if m != 'display'}
|
||||
@@ -1404,7 +1470,13 @@ def main(args=None):
|
||||
here = Path(globals().get('__file__') or sys.argv[0]).resolve()
|
||||
repo = find_repo(here)
|
||||
prepare_xcode = args.prepare_xcode or PREPARE_XCODE
|
||||
platform = args.platform or PLATFORM
|
||||
if platform not in ('ios', 'mac'): raise BuildError(f'PLATFORM / --platform must be ios or mac: {platform}')
|
||||
mac = platform == 'mac' # a Mac-only build for development: no iOS shaders, Xcode project or IPA (mac_run)
|
||||
if mac and prepare_xcode: raise BuildError('--prepare-xcode makes an iPhone project: it cannot be combined with --platform mac')
|
||||
if mac and not apple_silicon(): raise BuildError('--platform mac runs the game in the Mac host, which runs the game\'s own arm64 code natively: it needs a Mac with Apple silicon, and this Mac has an Intel processor')
|
||||
keep = KEEP_EXTRACTED if args.keep_extracted is None else args.keep_extracted
|
||||
if mac and not keep: say('keeping the extracted game files all the same: the game runs from them on the Mac'); keep = True
|
||||
if prepare_xcode and not keep: raise BuildError('--prepare-xcode requires --keep-extracted: Xcode builds from the extracted files')
|
||||
if args.xcode_workspace and not prepare_xcode: raise BuildError('--xcode-workspace requires --prepare-xcode')
|
||||
nsp_dir = Path(os.path.expanduser(args.nsp_dir or NSP_DIR or ask('NSP folder: ', 'the NSP folder'))).resolve()
|
||||
@@ -1472,20 +1544,20 @@ def main(args=None):
|
||||
if not saved_signing and legacy_config.exists() and f'/gen/{g.id}/' in legacy_config.read_text():
|
||||
saved_signing = xcode_projects.signing_settings(repo / 'dm/ios/DmIOS.xcodeproj')
|
||||
except ValueError as e: raise BuildError(str(e)) from e
|
||||
say(f'Xcode project: {project}' if prepare_xcode else f'output IPA: {out_ipa}')
|
||||
say('Mac-only build for development: the Mac shaders and the Mac host, no iOS shaders, Xcode project or IPA' if mac else f'Xcode project: {project}' if prepare_xcode else f'output IPA: {out_ipa}')
|
||||
say(f'work folder (extracted game files in extract/, {"kept for the next build" if keep else "deleted after the IPA is built"}): {work}')
|
||||
xlate = repo / 'dm/gfx/tools/dmsh_xlate_bin/dmsh_xlate'
|
||||
# DM_GAME picks the profile; DM_ROOT / WORK point at <id>_build: the tools' default paths and temporary directories all stay inside it
|
||||
env.update(DM_GAME=g.id, DM_ROOT=str(work), WORK=str(work / 'tmp'))
|
||||
logs = work / 'logs'
|
||||
check_prereqs(env, xlate)
|
||||
check_prereqs(env, xlate, mac)
|
||||
picks = pick_nsps(fam, have, versions)
|
||||
|
||||
ex = work / 'extract'; ex.mkdir(parents=True, exist_ok=True); (work / 'tmp').mkdir(exist_ok=True)
|
||||
gb = sum({p.stat().st_size for b, u in picks.values() for p in (b, u) if p}) / 2**30 + 6 # extraction + shader cache + slack
|
||||
if free_gb(work) < gb and not all((ex / v).exists() for v in versions): raise BuildError(f'less than {gb:.0f} GB free on the volume of {work}')
|
||||
heapck = provisional and HEAP_CHECK and not args.no_heap_check and not heap_checked(g) # the heap check, when the game needs one (once per provisional profile: it stamps the profile)
|
||||
PROG.plan(1 + len(versions) + bool(fam['dlc']) + 4 - prepare_xcode) # prepare Xcode replaces app build + IPA pack with one stage
|
||||
PROG.plan(1 + len(versions) + bool(fam['dlc']) + 4 - (prepare_xcode or mac)) # prepare Xcode (a Mac-only build: the Mac host) replaces app build + IPA pack with one stage
|
||||
PROG.stage('build nx_extract')
|
||||
nx = build_nx_extract(repo, work, logs, env)
|
||||
PROG.end()
|
||||
@@ -1525,7 +1597,7 @@ def main(args=None):
|
||||
base = next((dirs[v] for v in versions if picks[v][1] is None), dirs[versions[0]]) # the folder the base game was extracted to alone
|
||||
# the app name and bundle ID both come from the base NSP: the game name in its control data, and a bundle ID from its title ID; a setting / the command line overrides them
|
||||
app_name = args.app_name or APP_NAME or nacp.name(base / 'control.nacp')
|
||||
if not app_name: raise BuildError(f'no game name in the base extraction ({base / "control.nacp"}; an older extraction did not save it: delete extract/ and run again, or give --app-name)')
|
||||
if not app_name and not mac: raise BuildError(f'no game name in the base extraction ({base / "control.nacp"}; an older extraction did not save it: delete extract/ and run again, or give --app-name)')
|
||||
bundle_id = args.bundle_id or BUNDLE_ID or saved_signing.get('PRODUCT_BUNDLE_IDENTIFIER') or f'org.nxnative.{app:016x}'
|
||||
team_id = args.team_id or TEAM_ID or env.get('DM_TEAM_ID') or saved_signing.get('DEVELOPMENT_TEAM') or '0000000000'
|
||||
if prepare_xcode:
|
||||
@@ -1533,12 +1605,12 @@ def main(args=None):
|
||||
except ValueError as e: raise BuildError(str(e)) from e
|
||||
# the app version is the packed game's display version: the update's control data when an update is packed (nx_extract saves it next to the RomFS), else the base game's
|
||||
app_version = nacp.version(game / 'control.nacp') or nacp.version(base / 'control.nacp')
|
||||
say(f'app name: {app_name}; version: {app_version or "1.0"}; bundle ID: {bundle_id}')
|
||||
if not mac: say(f'app name: {app_name}; version: {app_version or "1.0"}; bundle ID: {bundle_id}')
|
||||
# the game language: Horizon gives the game one of the languages its control data lists (the packed version's, which an update may extend), so the one packed must be among them
|
||||
have = nacp.languages(game / 'control.nacp') or nacp.languages(base / 'control.nacp')
|
||||
lang = game_language(want_lang, bool(lang_in), have)
|
||||
say(f'game language: {lang} ({dict(nacp.LANGUAGES)[lang]})' + (f', the game has no {want_lang}' if lang != want_lang else '')
|
||||
+ (f'; the game\'s languages: {" ".join(have)}' if have else '; the control data lists no languages (not checked)') + ' (DM_LANG=<code> in the app\'s Documents/env.txt changes it)')
|
||||
+ (f'; the game\'s languages: {" ".join(have)}' if have else '; the control data lists no languages (not checked)') + (' (DM_LANG=<code> in the run command changes it)' if mac else ' (DM_LANG=<code> in the app\'s Documents/env.txt changes it)'))
|
||||
|
||||
patched = work / f'patched_{pkg_ver}'
|
||||
rewrite(py, repo, game / 'ExeFS', patched, env, logs, g)
|
||||
@@ -1554,15 +1626,22 @@ def main(args=None):
|
||||
sh = work / 'shader'
|
||||
if gfx == 'vulkan': # the packed version's merged RomFS holds the SPIR-V files the game loads; a capture holds those it generates
|
||||
cap = capture_vk(repo, game, patched, work, env, logs, g) if vkcap else None
|
||||
ios = build_vk_shaders(py, repo, [game / 'RomFS'] + ([cap] if cap else []), sh, env, logs, g, reuse=not (provisional or derived or ALLOW_DAMAGED))
|
||||
ios = build_vk_shaders(py, repo, [game / 'RomFS'] + ([cap] if cap else []), sh, env, logs, g, reuse=not (provisional or derived or ALLOW_DAMAGED), mac=mac)
|
||||
elif gfx == 'gl': # the GLSL sources in the packed version's archives and in its main (the game compiles them at run time; the host finds each by the hash of its text)
|
||||
ios = build_gl_shaders(py, repo, [game / 'RomFS', game / 'ExeFS/main'], sh, env, logs, g, reuse=not (provisional or derived or ALLOW_DAMAGED))
|
||||
ios = build_gl_shaders(py, repo, [game / 'RomFS', game / 'ExeFS/main'], sh, env, logs, g, reuse=not (provisional or derived or ALLOW_DAMAGED), mac=mac)
|
||||
else:
|
||||
# Trinity packs and legacy Unreal paks: the split decodes them with the packed version's own decoder (Oodle, LZ5), in the Mac host built for the profile
|
||||
oodle = (lambda: {'DM_OODLE_HOST': str(build_mac_host(repo, work / 'tools/dmhost', env, logs, g)), 'DM_OODLE_MODULES': str(patched)}) if src['oodle_packs'][0] or src['ue_legacy'][0] else None
|
||||
own = bool(provisional or derived) # the shader tables come from this build's split (the profile's are checked otherwise)
|
||||
pkey = split_shaders(py, repo, [dirs[v] / 'RomFS' for v in sv], [dirs[v] / 'ExeFS/main' for v in sv], sh / 'pkg', env, logs, g, own, oodle)
|
||||
ios = build_shaders(py, repo, sh, xlate, env, logs, g, pkey)
|
||||
ios = build_shaders(py, repo, sh, xlate, env, logs, g, pkey, mac)
|
||||
if mac: # a Mac-only build ends with the Mac host and the command that runs the game
|
||||
cmd = mac_run(py, repo, work, game, patched, ios, aoc, gfx, lang, env, logs, g, args.env_file or DM_ENV_FILE)
|
||||
rmtree(work / 'tmp')
|
||||
say(f'done: the game runs on the Mac with this command (its save in {work / "run/save"}, kept from run to run; DM_WINDOW=1 opens a window, DM_AUDIO=1 plays sound):')
|
||||
PROG.say(cmd)
|
||||
report_profile_status(g, reg, pkg_ver, derived, provisional, 'Mac build')
|
||||
return
|
||||
|
||||
if not prepare_xcode and free_gb(out_ipa.parent) < 5: raise BuildError(f'less than 5 GB free on the volume of {out_ipa.parent}')
|
||||
say('preparing the Xcode project' if prepare_xcode else f'building the IPA → {out_ipa}')
|
||||
@@ -1589,7 +1668,7 @@ def main(args=None):
|
||||
say(f'game project: {project}')
|
||||
say(f'Xcode derived data: {work / "xcode-derived-data"}')
|
||||
say('choose the game scheme in Xcode, select your Team and iPhone or iPad, then Run (Release)')
|
||||
report_profile_status(g, reg, pkg_ver, derived, provisional, prepare_xcode=True)
|
||||
report_profile_status(g, reg, pkg_ver, derived, provisional, 'Xcode project')
|
||||
return
|
||||
watch = IpaWatch(repo, work / 'ipa', out_ipa)
|
||||
PROG.stage('app build', poll=watch.poll, step='starting') # build_ipa.sh's "pack IPA" line starts the stage "IPA pack"
|
||||
|
||||
Reference in new issue
Block a user