diff --git a/dm/ios/project.yml b/dm/ios/project.yml index e0fcfb6..6ecf41e 100644 --- a/dm/ios/project.yml +++ b/dm/ios/project.yml @@ -1,4 +1,4 @@ -# iOS project (xcodegen); the app name, bundle ID and category come from the game profile (dm/games//game.tsv, passed by build_ipa.sh through environment variables). tools/build_ipa.sh generates Gen/ first (host source links, the embedded rewritten modules, iOS shader libraries, host tables), then packages. +# iOS project (xcodegen); the app name, bundle ID and category come from the game profile (dm/games//game.tsv, passed by build_ipa.sh through environment variables). tools/build_ipa.sh generates Gen/ first (host source links, the embedded rewritten modules, host tables), then packs the app with the iOS shader libraries (DmIOS.app/msl) and the RomFS (tools/pack_ipa.py). name: DmIOS options: bundleIdPrefix: ${DM_BUNDLE_ID} @@ -17,9 +17,6 @@ targets: - path: Gen/dm # Rewriter tables (*.fixups.tsv / gates / params / cuts / obs) + shader map / c1 list / warm-up list type: folder buildPhase: resources - - path: Gen/msl # iOS metallib + reflection json (DM_MSL_DIR) - type: folder - buildPhase: resources - path: Gen/Assets.xcassets # app icon (generated by build_ipa.sh from the game icon; an empty catalog when there is none, since xcodegen lists it in the project whether or not it exists) - path: Gen/src # Source files from the compile commands of dm/host/Makefile (linked in by tools/gen_build.py) - path: Gen/main/main.c diff --git a/dm/ios/tools/build_ipa.sh b/dm/ios/tools/build_ipa.sh index 716f3db..db2a00c 100755 --- a/dm/ios/tools/build_ipa.sh +++ b/dm/ios/tools/build_ipa.sh @@ -1,5 +1,6 @@ #!/bin/bash -# Build one unsigned all-in-one IPA: the rewritten modules compiled into the executable, iOS shader libraries, and the game files (RomFS) at DmIOS.app/romfs. At install time a signing tool (AltStore / Sideloadly etc.) swaps in your own certificate and bundle ID. +# Build one unsigned all-in-one IPA: the rewritten modules compiled into the executable, iOS shader libraries at DmIOS.app/msl, and the game files (RomFS) at DmIOS.app/romfs. +# Xcode builds only the app; tools/pack_ipa.py then packs it with the shader libraries, the add-on content and the RomFS straight from where they are (no Payload/ copy, nothing through the home disk). At install time a signing tool (AltStore / Sideloadly etc.) swaps in your own certificate and bundle ID. # Prerequisites: the rewriter (patched directory) and the iOS shader build (IOSOUT) have been run as in the README; Xcode and xcodegen are installed. # Usage: dm/ios/tools/build_ipa.sh [output.ipa]; environment: GAME=game profile (dm/games/, or one in DM_GAMES_DIR; required), W=the game's work folder (required: /-work, never the home folder), PATCHED / ROMFS / MSL_IOS can each be overridden # AOC=add-on content directory (default $W/data/aoc; when it exists it goes to DmIOS.app/aoc/<index>/; an empty string = none) @@ -33,8 +34,8 @@ if [ "$GFXAPI" = vulkan ]; then [ -f "$MSL_IOS/vk/vk_shaders.metallib" ] || die else [ -n "$V" ] && ls "$MSL_IOS/b-$V" 2> /dev/null | grep -q '_b.metallib$' || die "no iOS shader libraries $MSL_IOS/b-$V (IOSOUT=$MSL_IOS dmsh_build.py ...)"; fi t "app: $DM_DISPLAY_NAME $DM_APP_VERSION ($DM_BUNDLE_ID)" -t "generate Gen/ (host sources, embedded modules, tables, shader libraries)" -rm -rf Gen && mkdir -p Gen/dm Gen/msl || die "mkdir Gen" +t "generate Gen/ (host sources, embedded modules, tables)" +rm -rf Gen && mkdir -p Gen/dm || die "mkdir Gen" python3 tools/gen_build.py ../host Gen > Gen/dm.xcconfig || die "gen_build" F=$(python3 tools/gen_embed.py "$PATCHED" Gen/embed.S $MODS) || die "gen_embed" echo "OTHER_LDFLAGS = \$(inherited) \$(DM_FRAMEWORKS) $F" >> Gen/dm.xcconfig @@ -43,8 +44,8 @@ if [ "$GFXAPI" = vulkan ]; then cp "$GD/shader_map.tsv" "$GD/pso_manifest.tsv" G else cp "$GD/shader_map.tsv" "$GD/pso_manifest.tsv" "$GD/c1_full.tsv" Gen/dm/ || die "copy shader tables"; fi # Diagnostic build: DM_ENV_FILE=file → dm/env.txt in the bundle (main.m sets one environment variable per line at boot) if [ -n "${DM_ENV_FILE:-}" ]; then cp "$DM_ENV_FILE" Gen/dm/env.txt || die "copy DM_ENV_FILE"; t "env: $(grep -v '^#' "$DM_ENV_FILE" | tr '\n' ' ')"; fi -if [ "$GFXAPI" = vulkan ]; then cp -cR "$MSL_IOS/vk" Gen/msl/ || die "copy Vulkan shader library" -else cp -cR "$MSL_IOS/b-$V" Gen/msl/ || die "copy shader libraries"; fi +# The shader libraries go into the archive as DmIOS.app/msl/<folder> (main.m sets DM_MSL_DIR to it) +if [ "$GFXAPI" = vulkan ]; then MSLDIR=vk; else MSLDIR=b-$V; fi # Shared-font stand-ins: standard = macOS's Arial Unicode (absent on iOS), Nintendo extension = button icons generated by make_ext_font.py (needs fontTools; FONT_PY names a python that has it) mkdir -p Gen/dm/font && cp "${STD_FONT:-/System/Library/Fonts/Supplemental/Arial Unicode.ttf}" Gen/dm/font/standard.ttf || die "copy standard font (STD_FONT=a TTF)" ${FONT_PY:-python3} "$DM/tools/make_ext_font.py" Gen/dm/font/nintendo_ext.ttf > /dev/null || die "make_ext_font.py (pip install fonttools, or FONT_PY=python with fontTools)" @@ -71,16 +72,13 @@ xcodebuild -project DmIOS.xcodeproj -scheme DmIOS -configuration Release -destin APP=$DD/Build/Products/Release-iphoneos/DmIOS.app; [ -d "$APP" ] || APP=$(grep -o '/[^ ]*/Release-iphoneos/DmIOS\.app' "$O/build.log" | tail -1) [ -n "$APP" ] && [ -d "$APP" ] || die "no built app"; t "app: $APP" -t "assemble Payload (app + RomFS)" -rm -rf "$O/Payload" && mkdir -p "$O/Payload" && cp -cR "$APP" "$O/Payload/DmIOS.app" || die "copy app" -A=$O/Payload/DmIOS.app; rm -rf "$A/_CodeSignature" "$A/embedded.mobileprovision" -cp -cR "$ROMFS" "$A/romfs" || die "copy RomFS" -# Add-on content: the RomFS extracted from the player's DLC NSPs, one subdirectory per index (the app sets DM_AOC_DIR at launch; the host reports them as owned and mounts them from it) -if [ -n "$AOC" ] && [ -d "$AOC" ]; then cp -cR "$AOC" "$A/aoc" || die "copy add-on content"; t "add-on content: $(ls "$AOC" | tr '\n' ' ')"; fi - -t "zip (already-compressed game files stored, everything else deflated)" -rm -f "$O/out.ipa" -( cd "$O" && zip -qry -n "$(gp zip_store)" out.ipa Payload ) || die "zip" -mv "$O/out.ipa" "$OUT_IPA" || die "move ipa" -rm -rf "$O/Payload" +# Straight from the sources: the app without its signature, and without the folders an app built before pack_ipa.py still has (DerivedData keeps resources that left the project; +# msl/ is also deleted there, it took 1.5 GB of the home disk), then the shader libraries, the add-on content (the RomFS extracted from the player's DLC NSPs, one folder per index: +# the app sets DM_AOC_DIR at launch, the host reports them as owned and mounts them from it) and the RomFS +rm -rf "$APP/msl" "$O/Payload" "$O/out.ipa" # (the last two: what the copy-and-zip packing left behind) +P=Payload/DmIOS.app; MAPS=("$P=$APP" "$P/msl/$MSLDIR=$MSL_IOS/$MSLDIR") +if [ -n "$AOC" ] && [ -d "$AOC" ]; then MAPS+=("$P/aoc=$AOC"); t "add-on content: $(ls "$AOC" | tr '\n' ' ')"; fi +MAPS+=("$P/romfs=$ROMFS") +t "pack IPA (already-compressed game files stored, everything else deflated)" +python3 tools/pack_ipa.py "$OUT_IPA" --store "$(gp zip_store)" --skip $P/_CodeSignature --skip $P/embedded.mobileprovision --skip $P/msl --skip $P/aoc --skip $P/romfs "${MAPS[@]}" || die "pack_ipa" t "done: $OUT_IPA ($(du -h "$OUT_IPA" | cut -f1))" diff --git a/dm/ios/tools/pack_ipa.py b/dm/ios/tools/pack_ipa.py new file mode 100644 index 0000000..3aee7e0 --- /dev/null +++ b/dm/ios/tools/pack_ipa.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python3 +# Pack the IPA (a zip archive) straight from its source folders: the app Xcode built, the iOS shader libraries, the add-on content and the RomFS, each under its own path in the archive, +# with no Payload/ copy first. Replaces zip -qry -n with the same entries: folders, Unix modes (the executable's x bit), symlinks as links, DOS times, the UTF-8 flag for non-ASCII names; +# deflate level 6, a file whose deflate is not smaller is stored (zip's rule), and names ending in a --store suffix are always stored (the profile's zip_store, compared without regard to case). +# Speed (Info-ZIP managed about 100 MB/s on the T7): files up to BIG are read, and deflated, by a thread pool (zlib releases the GIL) while this thread writes them in order; +# bigger files are streamed here, their CRC patched into the local header afterwards (no data descriptors). Zip64 fields wherever a size, an offset or the entry count passes its 32 / 16-bit field. +# Usage: pack_ipa.py <out.ipa> [--store .a:.b] [--skip <archive path>]... <archive path>=<source folder>... +# --skip: an archive path left out of the walk of the folder that holds it (folders a stale app build still has, the signature); a mapping of its own still adds it +# Writes <out.ipa>.partial and renames it at the end; prints "pack: N entries, X bytes" first (the NSP script's progress total, X = the files' bytes) and a summary line last +import argparse, collections, concurrent.futures as cf, os, stat, struct, sys, time, zlib + +BIG = 256 << 20 # files above this are streamed by the writer; smaller ones go through the pool +WINDOW = 2 << 30 # input bytes in the pool at once +QUEUE = 4096 # entries in the pool at once (folders of tiny files) +CHUNK = 16 << 20 +L32, L16 = 0xFFFFFFFF, 0xFFFF +DIR_MODE = stat.S_IFDIR | 0o755 # a parent folder no source folder stands for (Payload/, msl/) +Entry = collections.namedtuple('Entry', 'name path st') # st None: such a parent folder + + +def entries(maps, skip): + """The archive's entries in order: for each mapping its parent folders (unless an earlier entry gave them), its root, then its tree depth first, sorted by name, + without the skipped paths and without other mappings' roots (each adds its own)""" + roots, out, seen = {a for a, _ in maps}, [], set() + def add(name, path, st): + if name in seen: sys.exit(f'[RED] pack_ipa: {name} twice in the archive') + seen.add(name); out.append(Entry(name, path, st)) + def walk(d, arc): + with os.scandir(d) as it: es = sorted(it, key=lambda e: e.name) + for e in es: + name = f'{arc}/{e.name}' + if name in skip or name in roots: continue + st = e.stat(follow_symlinks=False) + if stat.S_ISDIR(st.st_mode): add(name + '/', e.path, st); walk(e.path, name) + elif stat.S_ISREG(st.st_mode) or stat.S_ISLNK(st.st_mode): add(name, e.path, st) + else: sys.exit(f'[RED] pack_ipa: neither a file, a folder nor a symlink: {e.path}') + for arc, src in maps: + parts = arc.split('/') + for i in range(1, len(parts)): + p = '/'.join(parts[:i]) + '/' + if p not in seen: add(p, None, None) + add(arc + '/', src, os.stat(src)); walk(src, arc) + return out + + +def load(path, deflate): # in the pool: (method, CRC, the bytes to write) + with open(path, 'rb') as f: data = f.read() + crc = zlib.crc32(data) + if deflate and data: + c = zlib.compressobj(6, zlib.DEFLATED, -15); z = c.compress(data) + c.flush() + if len(z) < len(data): return 8, crc, z + return 0, crc, data + + +def dos_time(t): + y, mo, d, h, mi, s = time.localtime(t)[:6] + if y < 1980: y, mo, d, h, mi, s = 1980, 1, 1, 0, 0, 0 + return (h << 11) | (mi << 5) | (s // 2), ((y - 1980) << 9) | (mo << 5) | d + + +class Zip: + def __init__(self, path): + self.f, self.pos, self.cd, self.now = open(path, 'wb', buffering=1 << 20), 0, [], time.time() + self.stats = collections.Counter() + + def w(self, b): + self.f.write(b); self.pos += len(b) + + def local(self, e, method, crc, csize, usize, z64=False): # the local header; returns (its offset, name bytes, flags, time, date) + nb = e.name.encode('utf-8', 'surrogateescape'); fl = 0 if e.name.isascii() else 0x800 + t, d = dos_time(e.st.st_mtime if e.st else self.now) + ext = struct.pack('<HHQQ', 1, 16, usize, csize) if z64 else b'' + off = self.pos + self.w(struct.pack('<IHHHHHIIIHH', 0x04034B50, 45 if z64 else 20 if method == 8 else 10, fl, method, t, d, crc, L32 if z64 else csize, L32 if z64 else usize, len(nb), len(ext)) + nb + ext) + return off, nb, fl, t, d + + def central(self, e, h, method, crc, csize, usize): + off, nb, fl, t, d = h + z = [v for v in (usize, csize, off) if v >= L32] # zip64 extra: the fields that do not fit, in this order + ext = struct.pack(f'<HH{len(z)}Q', 1, 8 * len(z), *z) if z else b'' + mode = e.st.st_mode if e.st else DIR_MODE + self.cd.append(struct.pack('<IHHHHHHIIIHHHHHII', 0x02014B50, 0x031E, 45 if z else 20 if method == 8 else 10, fl, method, t, d, crc, min(csize, L32), min(usize, L32), + len(nb), len(ext), 0, 0, 0, (mode << 16) | (0x10 if stat.S_ISDIR(mode) else 0), min(off, L32)) + nb + ext) + + def put(self, e, method, crc, data, usize): # an entry whose bytes are at hand + h = self.local(e, method, crc, len(data), usize) + if data: self.w(data) + self.central(e, h, method, crc, len(data), usize) + self.stats['deflated' if method == 8 else 'stored'] += usize; self.stats['out'] += len(data) + + def other(self, e): # a folder or a symlink + if e.st and stat.S_ISLNK(e.st.st_mode): + t = os.readlink(e.path).encode('utf-8', 'surrogateescape'); self.put(e, 0, zlib.crc32(t), t, len(t)) + else: self.put(e, 0, 0, b'', 0) + + def stream(self, e, deflate): # a big file: the local header first, its CRC (and a deflated size) patched in afterwards + usize = e.st.st_size; z64 = usize >= L32 # a deflate that is not smaller is redone stored, so the deflated size stays below usize + h = self.local(e, 8 if deflate else 0, 0, 0 if deflate else usize, usize, z64) + crc, csize, c = 0, 0, zlib.compressobj(6, zlib.DEFLATED, -15) if deflate else None + with open(e.path, 'rb') as f: + while b := f.read(CHUNK): + crc = zlib.crc32(b, crc) + if c: b = c.compress(b) + self.w(b); csize += len(b) + if c: b = c.flush(); self.w(b); csize += len(b) + if csize != usize and not c: sys.exit(f'[RED] pack_ipa: {e.path} changed size while it was packed') + if c and csize >= usize: # not smaller: stored instead + self.f.seek(h[0]); self.f.truncate(); self.pos = h[0]; return self.stream(e, False) + self.f.seek(h[0] + 14); self.f.write(struct.pack('<I', crc) + (b'' if z64 else struct.pack('<II', csize, usize))) + if z64: self.f.seek(h[0] + 30 + len(h[1]) + 4); self.f.write(struct.pack('<QQ', usize, csize)) + self.f.seek(self.pos) + self.central(e, h, 8 if c else 0, crc, csize, usize) + self.stats['deflated' if c else 'stored'] += usize; self.stats['out'] += csize + + def close(self): # the central directory and the end records (zip64 ones when the count, the size or the offset does not fit) + cd_off, n = self.pos, len(self.cd); blob = b''.join(self.cd); self.w(blob) + if n >= L16 or cd_off >= L32 or len(blob) >= L32: + z = self.pos + self.w(struct.pack('<IQHHIIQQQQ', 0x06064B50, 44, 0x031E, 45, 0, 0, n, n, len(blob), cd_off)) + self.w(struct.pack('<IIQI', 0x07064B50, 0, z, 1)) + self.w(struct.pack('<IHHHHIIH', 0x06054B50, 0, 0, min(n, L16), min(n, L16), min(len(blob), L32), min(cd_off, L32), 0)) + self.f.close() + + +def main(): + ap = argparse.ArgumentParser(description='Pack an IPA (zip) from source folders mapped to archive paths, in parallel') + ap.add_argument('out'); ap.add_argument('maps', nargs='+', metavar='archive_path=folder') + ap.add_argument('--store', default='', help='suffixes stored without compressing, separated by ":"') + ap.add_argument('--skip', action='append', default=[], help='an archive path left out of the folder walk that would add it') + a = ap.parse_args() + maps = [] + for m in a.maps: + arc, sep, src = m.partition('=') + if not sep or not arc.strip('/') or not os.path.isdir(src): sys.exit(f'[RED] pack_ipa: not <archive path>=<folder>: {m}') + maps.append((arc.strip('/'), src)) + store = tuple(s.lower() for s in a.store.split(':') if s) + t0 = time.time() + es = entries(maps, {s.strip('/') for s in a.skip}) + total = sum(e.st.st_size for e in es if e.st and stat.S_ISREG(e.st.st_mode)) + print(f'pack: {len(es)} entries, {total} bytes', flush=True) + part = a.out + '.partial' + z = Zip(part) + try: + pend, inflight = collections.deque(), 0 + def one(): # write the oldest pending entry + e, fut = pend.popleft() + if fut is None: z.other(e); return 0 + method, crc, data = fut.result(); z.put(e, method, crc, data, e.st.st_size); return e.st.st_size + with cf.ThreadPoolExecutor(os.cpu_count()) as pool: + for e in es: + if not (e.st and stat.S_ISREG(e.st.st_mode)): pend.append((e, None)); continue # folders and symlinks keep their place in the queue + deflate = not e.name.lower().endswith(store) if store else True + if e.st.st_size > BIG: + while pend: inflight -= one() + z.stream(e, deflate); continue + while pend and (len(pend) >= QUEUE or inflight + e.st.st_size > WINDOW): inflight -= one() + pend.append((e, pool.submit(load, e.path, deflate))); inflight += e.st.st_size + while pend: inflight -= one() + z.close() + except BaseException: + z.f.close(); os.unlink(part); raise + os.replace(part, a.out) + s, dt = z.stats, time.time() - t0 + print(f'packed: {len(es)} entries, stored {s["stored"] / 2**30:.2f} GB, deflated {s["deflated"] / 2**30:.2f} GB, archive {os.path.getsize(a.out) / 2**30:.2f} GB ' + f'in {dt:.0f} s ({total / max(dt, 1e-3) / 2**20:.0f} MB/s)', flush=True) + + +if __name__ == '__main__': main() diff --git a/docs/development.md b/docs/development.md index d2482d7..61f9c74 100644 --- a/docs/development.md +++ b/docs/development.md @@ -25,8 +25,8 @@ python3 scripts/build_ipa_from_nsp.py ~/Games/MarioWonder -k ~/keys/prod.keys -o # pack Mario 1.0.0 instead of 1.2.1 (the folder still needs both NSPs: the shader tables cover both versions) python3 scripts/build_ipa_from_nsp.py ~/Games/MarioWonder -k ~/keys/prod.keys -p v0 -# keep the extracted game files (to run the game on the Mac, or to skip extraction next time) -python3 scripts/build_ipa_from_nsp.py ~/Games/MarioWonder -k ~/keys/prod.keys --keep-extracted +# delete the extracted game files after the IPA is built (they are kept by default, to skip extraction next time and to run the game on the Mac) +python3 scripts/build_ipa_from_nsp.py ~/Games/MarioWonder -k ~/keys/prod.keys --no-keep-extracted # a diagnostic build with environment switches baked in (one KEY=VALUE per line) python3 scripts/build_ipa_from_nsp.py ~/Games/MarioWonder -k ~/keys/prod.keys --env-file diag_env.txt @@ -41,7 +41,7 @@ python3 scripts/build_ipa_from_nsp.py ~/Games/NewGame -k ~/keys/prod.keys python3 scripts/build_ipa_from_nsp.py ~/Games/MarioGalaxy -k ~/keys/prod.keys --provisional --allow-damaged ``` -In Spyder, open the script, fill in the settings at the top (`NSP_DIR`, `PROD_KEYS`, and optionally `PACKAGE`, `OUT_IPA`, `APP_NAME`, `BUNDLE_ID`; for a game without a profile `HEAP_CHECK`, `HEAP_CHECK_SECS`, `PROVISIONAL`), and press F5. Empty paths are asked for at run time. +In Spyder, open the script, fill in the settings at the top (`NSP_DIR`, `PROD_KEYS`, and optionally `PACKAGE`, `OUT_IPA`, `APP_NAME`, `BUNDLE_ID`, `KEEP_EXTRACTED`; for a game without a profile `HEAP_CHECK`, `HEAP_CHECK_SECS`, `PROVISIONAL`), and press F5. Empty paths are asked for at run time. What the script does: 1. It finds the game profile from the title ID (the `[title ID]` in the file names, or the NSP's ticket; the update by its `[vN]` tag). @@ -51,18 +51,20 @@ What the script does: A Vulkan profile with `shader_vk_capture on` (Super Mario Galaxy) gets the same kind of run: the game boots in the Mac host with `DM_VK_CAPTURE` pointing at `<id>_build/extract/<version>/vk_capture` until every SPIR-V key in the profile's `vk_capture.tsv` is there (Galaxy: 3,089, about 7 s after launch), and gives up after 120 s. 5. It splits out the shaders and checks them against the profile's tables (a Vulkan profile: it translates the RomFS's SPIR-V and the capture with `vk_shaders.py` instead). 6. It builds the iOS shader libraries. -7. It calls `build_ipa.sh`. +7. It calls `build_ipa.sh`. Xcode builds the app, and `dm/ios/tools/pack_ipa.py` packs it with the shader libraries, the add-on content and the RomFS straight from where they are. It reads and deflates files on all cores and writes at about the T7's write speed (300 MB/s): Breath of the Wild's 16 GB IPA takes 57 s, Info-ZIP's copy and zip took 2 min 50 s. -While it runs, the console shows one line per stage: building nx_extract, extracting each version, the DLC, the rewrite, the run-time shader capture (only for such profiles), the shader split, the iOS shaders, the app build and packing the IPA. Each line shows `[k/n]`, the stage and its current step. Where the work can be counted, it adds a bar with done/total, percent, elapsed time and ETA. Counted work is bytes extracted, modules rewritten, shaders (cache check, translation, compile), files compiled by Xcode, and bytes copied and zipped into the IPA. Elsewhere a spinner shows the elapsed time. A total marked `~` is an estimate. A version's first extraction is estimated from the NSPs' size, and later extractions use the previous run's exact size. The zip is estimated from the previous IPA, or from the app's size before compression. In a terminal or in Spyder the line is redrawn in place. When the stage ends, it becomes a `✓` line with the elapsed time and a summary: +A repeat build skips work whose inputs have not changed. The extraction is kept (`extract/`). The rewrite, the shader split and the iOS (or Vulkan) shaders each write a key beside their output: a sha256 of their inputs (the game files, the tools, the profile, Xcode's and spirv-cross's versions). When the key matches, the stage line says `reused (inputs unchanged)` and the step does not run. Any change to a tool or the profile runs the step again; deleting the `.key` file forces it. + +While it runs, the console shows one line per stage: building nx_extract, extracting each version, the DLC, the rewrite, the run-time shader capture (only for such profiles), the shader split, the iOS shaders, the app build and packing the IPA. Each line shows `[k/n]`, the stage and its current step. Where the work can be counted, it adds a bar with done/total, percent, elapsed time and ETA. Counted work is bytes extracted, modules rewritten, shaders (cache check, translation, compile), files compiled by Xcode, and bytes packed into the IPA. Elsewhere a spinner shows the elapsed time. A total marked `~` is an estimate. A version's first extraction is estimated from the NSPs' size, and later extractions use the previous run's exact size. The IPA is estimated from the previous IPA, or from the files' size before compression. In a terminal or in Spyder the line is redrawn in place. When the stage ends, it becomes a `✓` line with the elapsed time and a summary: ``` [15:24:28] ✓ [2/7] extract v327680 (0:20): 9,779 files, 5.0 GB -⠸ [7/7] IPA pack · zip ████████▍░░░░░░░░░░░ 42% 1.9/~4.5 GB 0:30 · ETA 0:35 +⠸ [7/7] IPA pack · packing ████████▍░░░░░░░░░░░ 42% 1.9/~4.5 GB · 13,597 entries 0:05 · ETA 0:07 ``` When the output goes to a file or a pipe, the script prints a plain line instead, every `BEAT` seconds (30). `--progress bar|lines|off` (`PROGRESS` in Spyder) overrides that choice; `off` prints only each stage's start and end. The tools' own output goes only to the logs in `<id>_build/logs/`, one per step. When a step fails, the script prints the last 40 lines of that step's log and the log's path. -Everything it writes stays in the NSP folder: intermediate files and logs in `<id>_build/`, the IPA next to the NSPs. The extracted game files are deleted at the end unless you pass `--keep-extracted`. It exits with 0 when done and 1 on an error. The app's bundle ID is `org.nxnative.<base game title ID>` (for example `org.nxnative.0100a6301214e000`), and its version is the game's display version: the update's when an update is packed, else the base game's (Engage 2.0.0); `--app-name` / `--bundle-id` (or `APP_NAME` / `BUNDLE_ID` in Spyder) override the name and ID. +Everything it writes stays in the NSP folder: intermediate files and logs in `<id>_build/`, the IPA next to the NSPs. The extracted game files stay in `<id>_build/extract/` for the next build; `--no-keep-extracted` (`KEEP_EXTRACTED = False` in Spyder) deletes them at the end. It exits with 0 when done and 1 on an error. The app's bundle ID is `org.nxnative.<base game title ID>` (for example `org.nxnative.0100a6301214e000`), and its version is the game's display version: the update's when an update is packed, else the base game's (Engage 2.0.0); `--app-name` / `--bundle-id` (or `APP_NAME` / `BUNDLE_ID` in Spyder) override the name and ID. On the device, environment switches go in `Documents/env.txt` (one `KEY=VALUE` per line), and game files are also accepted at `Documents/romfs`. @@ -112,7 +114,7 @@ GAME=$G W=$W dm/ios/tools/build_ipa.sh [out.ipa] Notes on these steps: - `dmsh_build.py` compiles each shader's Metal source through a cache keyed by that source, the compile flags and the compiler (`dm/gfx/tools/dmsh_mtlcache.py`, in `<cache>/cas/metallib/`). A chain change that leaves a shader's MSL unchanged, such as a translator fix, costs that shader no Metal compile. Before building, the cache takes the libraries of the earlier `b-*/` folders beside its output (and beside `IOSOUT`), and of `DMSH_PREFILL=<Mac root>[=<iOS root>]` (several separated by `:`). It takes a folder only when its recorded flags are the current ones and a fresh compile of a few of its shaders reproduces their libraries byte for byte. The summary line `Metal cache:` shows reused, compiled and filled counts per target. -- `build_ipa.sh` takes the rewritten modules from `$W/build/patched_<package>` and the RomFS from `$W/data/<package>/RomFS`; `PATCHED` and `ROMFS` override them. +- `build_ipa.sh` takes the rewritten modules from `$W/build/patched_<package>` and the RomFS from `$W/data/<package>/RomFS`; `PATCHED` and `ROMFS` override them. The shader libraries, the RomFS and the add-on content never go through Xcode: `pack_ipa.py` packs them into the IPA beside the app Xcode built, as `DmIOS.app/msl/b-<chain version>`, `romfs` and `aoc`. - It also bundles a standard font (macOS's Arial Unicode, `STD_FONT=` overrides it) and the generated `nintendo_ext.ttf`. - The icon is the `icon.jpg` that `nx_extract` saves next to `RomFS/`; `DM_ICON=` points at another image. - `DM_ENV_FILE=` bakes environment switches into the app, and `DM_DISPLAY_NAME=` renames it. diff --git a/scripts/build_ipa_from_nsp.py b/scripts/build_ipa_from_nsp.py index 1decc82..516f914 100755 --- a/scripts/build_ipa_from_nsp.py +++ b/scripts/build_ipa_from_nsp.py @@ -8,11 +8,12 @@ # or in a terminal: python3 scripts/build_ipa_from_nsp.py <NSP folder> -k <prod.keys> [options] (values given on the command line override "Settings", -h lists every option; # without keyboard input a missing value is not asked for but reported as an error; exit code 0 on success, 1 on error, 130 when interrupted). Command-line options entered in Spyder's run configuration work the same way. # Uses only three things: this repository, a folder holding one game's NSPs (the base game plus the updates listed in the profile's shader_versions are required; DLC NSPs, when present, are packed too and unlock the add-on content already in the game), and prod.keys. -# Every intermediate file is under <id>_build/ in the NSP folder: extract/ (the extracted game files, deleted after the IPA is built by default), aoc/ (each DLC's RomFS by index), tools/ (nx_extract, and the Mac host for a run-time shader capture), patched_*/, shader/ (shader package and caches, kept so the next build is fast), logs/ (each step's full output), tmp/, ipa/. The finished IPA also goes to the NSP folder by default. +# Every intermediate file is under <id>_build/ in the NSP folder: extract/ (the extracted game files, kept for the next build unless KEEP_EXTRACTED / --no-keep-extracted), aoc/ (each DLC's RomFS by index), tools/ (nx_extract, and the Mac host for a run-time shader capture), patched_*/, shader/ (shader package and caches, kept so the next build is fast), logs/ (each step's full output), tmp/, ipa/. The finished IPA also goes to the NSP folder by default. +# The rewrite, the shader split and the iOS (or Vulkan) shaders are reused when all their inputs are as at their last successful run (the game files, the tools, the profile, Xcode and spirv-cross: fingerprint() keys beside their output). # The shader translator is the in-repo dm/gfx/tools/dmsh_xlate_bin (the build whose fingerprints match dmsh_chain.lock; a rebuilt dll has different fingerprints and dmsh_build refuses it). -# The Python running this file needs lz4, capstone and fonttools; also needed: Xcode, xcodegen, .NET SDK 10, zstd, spirv-cross. For Mario the first run takes about 35 minutes (mostly the shader libraries); later runs take about 10 (the shader libraries come from the cache). Key values are never printed. +# The Python running this file needs lz4, capstone and fonttools; also needed: Xcode, xcodegen, .NET SDK 10, zstd, spirv-cross. A first run takes 2 (Metroid Dread) to 10 minutes (Breath of the Wild: 5 of them the shader split), longer when shader libraries must be compiled; a repeat run with unchanged inputs takes about 20 s (Metroid Dread), mostly packing the IPA. Key values are never printed. # The console shows one line per stage ([k/n] stage, with a bar, done/total, percent, elapsed time and ETA where the work can be counted); the tools' own output goes only to logs/ (PROGRESS / --progress). -import collections, ctypes, getpass, os, re, shlex, shutil, struct, subprocess, sys, tempfile, threading, time, types, zlib +import collections, ctypes, getpass, hashlib, os, re, shlex, shutil, struct, subprocess, sys, tempfile, threading, time, types, zlib from pathlib import Path # ---- Settings ---- @@ -22,7 +23,7 @@ PACKAGE = '' # version packed into the IPA (empty = t OUT_IPA = '' # output IPA (empty = <the profile's ipa> in the NSP folder) APP_NAME = '' # app name on the home screen (empty = the game name in the base NSP) BUNDLE_ID = '' # the app's bundle ID (empty = org.nxnative.<base game title ID in lowercase>) -KEEP_EXTRACTED = False # keep the extracted game files after the IPA is built (saves extraction next time; as big as the game files) +KEEP_EXTRACTED = True # keep the extracted game files after the IPA is built (saves extraction next time; as big as the game files); False deletes them DM_ENV_FILE = '' # optional: environment switches for a diagnostic build (one KEY=VALUE per line) SURVEY_ONLY = False # a game with no profile in dm/games: only write the survey, no provisional profile and no IPA PROFILE_OUT = '' # where a provisional profile goes (empty = <NSP folder>/<title ID>_build/profile; the profile is <this>/<title ID>/) @@ -497,11 +498,60 @@ def write_profile(pdir, app, f): if k in G and G[k][1]: say(f' {k} {G[k][0]}: {G[k][1]}') +# ---- Stage reuse: a stage whose inputs are all as at its last successful run keeps its output ---- +def files_under(d): # (relative path, stat) of every file under d, sorted; [] when it does not exist + out, todo = [], [str(d)] + while todo: + try: it = os.scandir(todo.pop()) + except OSError: continue + with it: + for e in it: + if e.is_dir(follow_symlinks=False): todo.append(e.path) + else: out.append((os.path.relpath(e.path, d), e.stat(follow_symlinks=False))) + return sorted(out, key=lambda x: x[0]) + + +def fingerprint(*items, env=None): + """sha256 over a stage's inputs. Items: ('files', paths) = names and content (a missing file counts too); ('tree', folder) = relative path and size of every file under it + (a RomFS: check_build pins its version, so names and sizes stand for its content); ('stat', folder) = relative path, size and mtime of every file under it (the translator's binaries); + ('cmd', argv) = a tool's version output (run with env); anything else = its repr""" + h = hashlib.sha256() + for it in items: + kind, v = it if isinstance(it, tuple) and len(it) == 2 and it[0] in ('files', 'tree', 'stat', 'cmd') else ('repr', it) + h.update(f'\0{kind}\0'.encode()) + if kind == 'files': + for p in map(Path, v): + h.update(p.name.encode() + b'\0') + try: h.update(hashlib.sha256(p.read_bytes()).digest()) + except OSError: h.update(b'missing') + elif kind in ('tree', 'stat'): + for rel, st in files_under(v): h.update(f'{rel}\0{st.st_size}\0{st.st_mtime_ns if kind == "stat" else ""}\n'.encode()) + elif kind == 'cmd': + try: r = subprocess.run(v, env=env, capture_output=True, text=True, timeout=120); h.update((r.stdout + r.stderr).encode()) + except (OSError, subprocess.TimeoutExpired): h.update(b'missing') + else: h.update(repr(v).encode()) + return h.hexdigest() + + +def reused(keyfile, key): # the note the stage's last successful run wrote beside its key ('' when none), or None when the key differs or is missing + try: k, _, note = keyfile.read_text().partition('\n') + except OSError: return None + return note.strip() if k == key else None + + +def tool_files(repo): # what the shader stages run: dm/gfx/tools' scripts and the lock of the translator's fingerprints, the profile reader, the iOS compile convention, the NSO reader + return sorted((repo / 'dm/gfx/tools').glob('*.py')) + [repo / p for p in ('dm/gfx/tools/dmsh_chain.lock', 'dm/games/game.py', 'dm/ios/tools/ios_metal.py', 'dm/verify/nsoimg.py')] + + +def profile_files(g): # every file of the game profile (game.tsv and its tables) + return sorted(p for p in g.dir.iterdir() if p.is_file()) + + # ---- Steps ---- def check_prereqs(env, xlate): missing = [m for m in ('lz4.block', 'capstone', 'fontTools') if not _importable(m)] if missing: raise BuildError(f'the current Python ({sys.executable}) is missing modules: {", ".join(missing)} (pip install lz4 capstone fonttools)') - tools = {'xcodebuild': 'Xcode', 'xcodegen': 'brew install xcodegen', 'dotnet': '.NET SDK 10', 'zstd': 'brew install zstd', 'spirv-cross': 'brew install spirv-cross', 'zip': 'zip', 'make': 'Xcode'} + tools = {'xcodebuild': 'Xcode', 'xcodegen': 'brew install xcodegen', 'dotnet': '.NET SDK 10', 'zstd': 'brew install zstd', 'spirv-cross': 'brew install spirv-cross', 'make': 'Xcode'} gone = [f'{t} ({how})' for t, how in tools.items() if not shutil.which(t, path=env['PATH'])] if gone: raise BuildError('tools not found: ' + ', '.join(gone)) if not xlate.exists(): raise BuildError(f'the shader translator is missing from the repository: {xlate}') @@ -567,13 +617,38 @@ def check_build(d, name, g): # every module's build ID must be the one registe def rewrite(py, repo, exe_dir, out, env, logs, g): - tmp = out.with_name(out.name + '.partial'); rmtree(tmp); tmp.mkdir(parents=True) + """patch_nso.py on every code module at once (each writes its own <module>.* files, main also the boot list, all the same build_src.sha256 line), each with its log. + Reused when the modules, patch_nso.py, the boot lists, its switches and capstone are as at the last successful rewrite (Monster Hunter Rise's main alone takes 5.5 min)""" + import capstone mods = g.words('modules') - PROG.stage('rewrite') - for i, m in enumerate(mods): - PROG.set(step=m, done=i, total=len(mods), unit='modules') - run([py, repo / 'dm/tools/patch_nso.py', exe_dir / m, tmp, m], logs / f'rewrite_{m}.log', env=dict(env, DM_RULESET='native')) - swap_in(tmp, out) + PROG.stage('rewrite', step='checking the inputs') + kf = out.with_name(out.name + '.key') + key = fingerprint(('files', [repo / 'dm/tools/patch_nso.py'] + [exe_dir / m for m in mods] + sorted((repo / 'dm/data').glob('bootlist_*.tsv'))), mods, capstone.__version__, + {k: env.get(k) for k in ('DM_PATCH_FRONT', 'DM_PRIVATE_GATES')}) + if all((out / f'{m}.patched.bin').is_file() for m in mods) and reused(kf, key) is not None: + PROG.end(f'{", ".join(mods)}, reused (inputs unchanged)'); return + tmp = out.with_name(out.name + '.partial'); rmtree(tmp); tmp.mkdir(parents=True); logs.mkdir(parents=True, exist_ok=True) + PROG.set(step=', '.join(mods), done=0, total=len(mods), unit='modules') + procs, logf = {}, {} + try: + for m in mods: + logf[m] = open(logs / f'rewrite_{m}.log', 'w') + procs[m] = subprocess.Popen([str(py), str(repo / 'dm/tools/patch_nso.py'), str(exe_dir / m), str(tmp), m], env=dict(env, DM_RULESET='native'), stdout=logf[m], stderr=subprocess.STDOUT) + left = list(mods) + while left: + for m in [m for m in left if procs[m].poll() is not None]: + left.remove(m); PROG.set(done=len(mods) - len(left)) + if procs[m].returncode: + log = logs / f'rewrite_{m}.log'; logf[m].close() + PROG.end(ok=False) + PROG.say(f'---- last lines of {log} ----\n' + ''.join(collections.deque(open(log, errors='replace'), maxlen=40)).rstrip('\n')) + raise BuildError(f'patch_nso.py exit code {procs[m].returncode} on {m} (full output: {log})') + time.sleep(0.2) + finally: # an error or an interrupt stops the other rewrites + for p in procs.values(): + if p.poll() is None: p.terminate(); p.wait() + for f in logf.values(): f.close() + rmtree(kf); swap_in(tmp, out); kf.write_text(f'{key}\n') PROG.end(', '.join(mods)) @@ -797,9 +872,17 @@ def data_rows(b): # a table's data rows (without # comment lines: the header c def split_shaders(py, repo, roms, mains, pkg, env, logs, g, provisional=False): - tmp = pkg.with_name(pkg.name + '.partial'); rmtree(tmp); (tmp / 'tables').mkdir(parents=True) + """The shader split into pkg, checked against the profile's tables. Returns its key (None for a provisional profile or damaged files, which never reuse it): the package is reused + when the tools, the profile and each version's main, RomFS (paths and sizes) and run-time capture are as at the last successful split""" names = ('shader_map.tsv', 'shader_facts.tsv', 'shader_pairs.tsv') - PROG.stage('shader split') # a spinner: shader_split.py writes the package and the tables only at the end + PROG.stage('shader split', step='checking the inputs') # then a spinner: shader_split.py writes the package and the tables only at the end + kf = pkg.with_name(pkg.name + '.key') + key = None if provisional or ALLOW_DAMAGED else fingerprint(('files', tool_files(repo)), ('files', profile_files(g)), + *[x for r, m in zip(roms, mains) for x in (('files', [m]), ('tree', r), ('files', sorted((r.parent / 'glslc_capture').glob('*.glslc'))))]) + if key and pkg.is_dir() and (note := reused(kf, key)) is not None: + PROG.end(f'{note}, reused (inputs unchanged)'); return key + PROG.set(step='splitting') + tmp = pkg.with_name(pkg.name + '.partial'); rmtree(tmp); (tmp / 'tables').mkdir(parents=True) run([py, repo / 'dm/gfx/tools/shader_split.py', ','.join(map(str, roms)), ','.join(map(str, mains)), tmp / 'pkg'] + [tmp / 'tables' / n for n in names], logs / 'shader_split.log', env=env) m = re.search(r'shaders: (\d+) \(', (logs / 'shader_split.log').read_text(errors='replace')) if provisional: # a provisional profile takes the tables as they come @@ -808,8 +891,11 @@ def split_shaders(py, repo, roms, mains, pkg, env, logs, g, provisional=False): # Data rows byte-identical to the profile's = the same game files the tables were built from; if not, stop (the profile's tables are left alone). Only data rows are compared: the header of Engage's profile tables was written by an older shader_split diff = [n for n in names if data_rows((tmp / 'tables' / n).read_bytes()) != data_rows(g.table(n).read_bytes())] if diff: raise BuildError(f'the split shader tables differ from game profile {g.id}\'s: {", ".join(diff)} (game files not the versions the profile registers?)') - swap_in(tmp / 'pkg', pkg); rmtree(tmp) - PROG.end(f'{int(m.group(1)):,} shaders' if m else '') + rmtree(kf); swap_in(tmp / 'pkg', pkg); rmtree(tmp) + note = f'{int(m.group(1)):,} shaders' if m else '' + if key: kf.write_text(f'{key}\n{note}\n') + PROG.end(note) + return key class ShaderWatch: @@ -856,54 +942,79 @@ class ShaderWatch: 'note': f'{ext[".bin"]:,} not cached' if ext['.bin'] else ''} -def build_shaders(py, repo, sh, xlate, env, logs, g, provisional=False): +def toolchain(): # the compilers' versions, part of the shader stages' keys + return ('cmd', ['xcodebuild', '-version']), ('cmd', ['spirv-cross', '--revision']) + + +def build_shaders(py, repo, sh, xlate, env, logs, g, provisional=False, split_key=None): + """The iOS shader libraries (dmsh_build.py). Reused when the split's key (split_key; None never reuses) and the chain's tools, translator binaries, profile and compilers + are as at the last successful build: dmsh_build.py would only restore every shader from its cache (63,000 for Breath of the Wild: about 100 s)""" ios, log = sh / 'ios', logs / 'dmsh_build.log' PROG.stage('iOS shaders') if provisional: PROG.set(step='chain header') run([py, repo / 'dm/gfx/tools/dmsh_chain.py', '--header'], logs / 'dmsh_chain.log', env=env) # the profile's chain version header (dmsh_build checks it) + chain = re.search(r'#define DMSH_CHAIN "(.*)"', g.table('dmsh_chain.h').read_text()).group(1) + libs = ios / f'b-{chain}'; kf = libs.with_name(libs.name + '.key') + if split_key: + PROG.set(step='checking the inputs') + key = fingerprint(split_key, ('files', tool_files(repo)), ('stat', xlate.parent), ('files', profile_files(g)), *toolchain(), str(xlate), + {k: env.get(k) for k in ('MAIN', 'XLATE_DEBUG', 'XLATE_REDUCED')}, env=env) + if next(libs.glob('*_b.metallib'), None) and (note := reused(kf, key)) is not None: + PROG.end(f'{note}, reused (inputs unchanged)'); return ios + else: key = None + rmtree(kf) try: PROG.watch(ShaderWatch(g, ios, Path(env['WORK']))) except Exception: pass # progress never stops the build (a missing table is dmsh_build.py's to report) run([py, repo / 'dm/gfx/tools/dmsh_build.py', sh / 'pkg', sh / 'cache'], log, env=dict(env, XLATE=str(xlate), IOSOUT=str(ios))) PROG.watch(None) - chain = re.search(r'#define DMSH_CHAIN "(.*)"', g.table('dmsh_chain.h').read_text()).group(1) if provisional: PROG.set(step='c1 guard') run([py, repo / 'dm/gfx/tools/c1_guard.py', sh / 'cache' / f'b-{chain}', g.table('c1_full.tsv')], logs / 'c1_guard.log', env=env) # shaders whose c1 reads are not proven bounded - libs = ios / f'b-{chain}' n = sum(1 for _ in libs.glob('*_b.metallib')) if libs.is_dir() else 0 if n == 0: raise BuildError(f'no iOS shader libraries in {libs}') say(f'{n} iOS shader libraries ({libs})') t = log.read_text(errors='replace') m, c = re.search(r'shaders (\d+): ok (\d+)', t), re.search(r'shader cache: hit (\d+)', t) + note = '' if m: total, ok = int(m.group(1)), int(m.group(2)) - PROG.end(f'{ok:,} of {total:,} shaders ready' + (f' ({int(c.group(1)):,} from the cache)' if c else '') + (f'; {total - ok:,} failed, reasons at the end of {log}' if ok < total else '')) + failed = f'; {total - ok:,} failed, reasons at the end of {log}' if ok < total else '' + note = f'{ok:,} of {total:,} shaders ready{failed}' + PROG.end(f'{ok:,} of {total:,} shaders ready' + (f' ({int(c.group(1)):,} from the cache)' if c else '') + failed) else: PROG.end() + if key: kf.write_text(f'{key}\n{note}\n') return ios -def build_vk_shaders(py, repo, roots, sh, env, logs): +def build_vk_shaders(py, repo, roots, sh, env, logs, g, reuse=True): """A Vulkan game's SPIR-V shaders (roots: the RomFS, and a run-time capture folder) → one Metal library each for the Mac (sh/vk/) and the iPhone (sh/ios/vk/, what build_ipa.sh takes from MSL_IOS), - replacing the NVN split and chain build. The old output is deleted first, so a failed run leaves nothing for build_ipa.sh to pick up; vk_shaders.py's temporary files go to WORK (on the NSP folder's volume)""" - ios = sh / 'ios' - PROG.stage('Vulkan shaders') - rmtree(sh / 'vk'); rmtree(ios / 'vk') + replacing the NVN split and chain build. Reused when the roots (paths and sizes), the tools, the profile and the compilers are as at the last successful run (reuse False: never). + Otherwise the old output is deleted first, so a failed run leaves nothing for build_ipa.sh to pick up; vk_shaders.py's temporary files go to WORK (on the NSP folder's volume)""" + ios = sh / 'ios'; kf = ios / 'vk.key' + PROG.stage('Vulkan shaders', step='checking the inputs') + key = fingerprint(*[('tree', r) for r in roots], ('files', tool_files(repo)), ('files', profile_files(g)), *toolchain(), env=env) if reuse else None + if key and (ios / 'vk/vk_shaders.metallib').is_file() and (note := reused(kf, key)) is not None: + PROG.end(f'{note}, reused (inputs unchanged)'); return ios + PROG.set(step='translating') + rmtree(kf); rmtree(sh / 'vk'); rmtree(ios / 'vk') run([py, repo / 'dm/gfx/tools/vk_shaders.py', *roots, sh, '--ios', ios], logs / 'vk_shaders.log', env=dict(env, TMPDIR=env['WORK'])) if not (ios / 'vk/vk_shaders.metallib').is_file(): raise BuildError(f'no iOS Vulkan shader library in {ios / "vk"}') m = re.search(r'iphoneos: (\d+)', (logs / 'vk_shaders.log').read_text(errors='replace')) - PROG.end(f'{int(m.group(1)):,} shaders for the Mac and iOS' if m else '') + note = f'{int(m.group(1)):,} shaders for the Mac and iOS' if m else '' + if key: kf.write_text(f'{key}\n{note}\n') + PROG.end(note) return ios class IpaWatch: - """Progress of build_ipa.sh, from its step lines ("[time] generate Gen/ …", "xcodebuild …", "assemble Payload …", "zip …") and the files it writes: - the CompileC lines in xcodebuild's build.log (one per source file: Gen/src, Gen/main/main.c, Gen/embed.S and the app's Sources), the bytes copied into Payload/DmIOS.app/romfs, - and the bytes of the archive zip is writing (a zi* temporary file next to out.ipa, renamed at the end). Its two halves are the stages "app build" and "IPA pack" """ + """Progress of build_ipa.sh, from its step lines ("[time] generate Gen/ …", "xcodebuild …", "pack IPA …", then pack_ipa.py's "pack: N entries, X bytes") and the files it writes: + the CompileC lines in xcodebuild's build.log (one per source file: Gen/src, Gen/main/main.c, Gen/embed.S and the app's Sources), then the bytes of the archive pack_ipa.py is + writing (<IPA>.partial, renamed at the end). Its two halves are the stages "app build" and "IPA pack" """ XCODE = {'CompileC': 'compiling', 'Ld': 'linking', 'CpResource': 'copying resources', 'CompileAssetCatalogVariant': 'asset catalog', 'ProcessInfoPlistFile': 'Info.plist'} - def __init__(self, repo, w, romfs, out_ipa): - self.ios, self.o, self.romfs = repo / 'dm/ios', w / 'build/ios/pkg', romfs + def __init__(self, repo, w, out_ipa): + self.ios, self.o, self.part = repo / 'dm/ios', w / 'build/ios/pkg', out_ipa.with_name(out_ipa.name + '.partial') self.last = out_ipa.stat().st_size if out_ipa.is_file() else None # the last IPA's size: the expected size of the new archive self.state, self.t, self.off, self.compiled, self.what, self.total, self.most, self.lock = None, 0, 0, 0, '', None, 0, threading.Lock() @@ -914,12 +1025,13 @@ class IpaWatch: elif s.startswith('xcodebuild'): units = len(os.listdir(self.ios / 'Gen/src')) + 2 + sum(1 for p in (self.ios / 'Sources').iterdir() if p.suffix in ('.c', '.m')) # + Gen/main/main.c and Gen/embed.S self.state, self.t = 'xcode', time.time(); PROG.set(step='xcodebuild', done=0, total=units, unit='files', est=True) # an incremental build compiles fewer - elif s.startswith('assemble Payload'): + elif s.startswith('pack IPA'): try: self._poll() # the rest of build.log (xcodebuild writes it in big blocks) except OSError: pass PROG.end(f'{self.compiled:,} file{"" if self.compiled == 1 else "s"} compiled') - self.state, self.total, self.most = 'copy', None, 0; PROG.stage('IPA pack', poll=self.poll, step='copying the game files', unit='B') - elif s.startswith('zip'): self.state, self.total, self.most = 'zip', None, 0; PROG.set(step='zip', unit='B') + self.state, self.total, self.most = 'list', None, 0; PROG.stage('IPA pack', poll=self.poll, step='listing the files') + elif self.state == 'list' and (m := re.match(r'pack: (\d+) entries, (\d+) bytes', s)): # the files' bytes: the archive's size when nothing deflates, else the last IPA's size is closer + self.state, self.total = 'pack', self.last or int(m.group(2)); PROG.set(step='packing', unit='B', note=f'{int(m.group(1)):,} entries') def poll(self): # from the poll thread with self.lock: return self._poll() @@ -937,13 +1049,9 @@ class IpaWatch: if w == 'CompileC': self.compiled += 1 self.what = self.XCODE.get(w, self.what) r = {'done': self.compiled, 'note': self.what} - elif state == 'copy': - if self.total is None: self.total = tree_size(self.romfs)[1] - self.most = max(self.most, tree_size(self.o / 'Payload/DmIOS.app/romfs')[1]) # never backwards: the copy and the archive leave pkg/ at the end - r = {'done': self.most, 'total': self.total} - elif state == 'zip': - if self.total is None: self.total = self.last or tree_size(self.o / 'Payload')[1] - self.most = max([self.most] + [e.stat().st_size for e in os.scandir(self.o) if e.is_file() and e.name != 'build.log']) + elif state == 'pack': + try: self.most = max(self.most, self.part.stat().st_size) # never backwards: the archive is renamed at the end + except OSError: pass r = {'done': self.most, 'total': self.total, 'est': True} else: return None return r @@ -967,7 +1075,7 @@ def parse_args(argv): ap.add_argument('-o', '--out', help='output IPA (default: <the profile\'s ipa> in the NSP folder)') ap.add_argument('--app-name', help='app name on the home screen (default: the game name in the base NSP)') ap.add_argument('--bundle-id', help='the app\'s bundle ID (default: org.nxnative.<base game title ID in lowercase>)') - ap.add_argument('--keep-extracted', action='store_true', help='keep the extracted game files after the IPA is built') + ap.add_argument('--keep-extracted', action=argparse.BooleanOptionalAction, help='keep the extracted game files after the IPA is built, for the next build (default: the KEEP_EXTRACTED setting, on)') ap.add_argument('--env-file', help='environment switches for a diagnostic build (one KEY=VALUE per line)') ap.add_argument('--survey-only', action='store_true', help='a game with no profile: only the survey, no provisional profile and no IPA') ap.add_argument('--allow-damaged', action='store_true', help='a damaged NSP (files failing the NCA hash check): write them as read and go on instead of stopping') @@ -994,6 +1102,7 @@ def main(args=None): if not keys.is_file(): raise BuildError(f'no such file: {keys}') env = dict(os.environ, PATH=os.pathsep.join(TOOL_PATHS + [os.environ.get('PATH', '/usr/bin:/bin:/usr/sbin:/sbin')])) env.setdefault('USER', getpass.getuser()) # xcodegen fails without USER (Couldn't find current username) + env.setdefault('J', str(os.cpu_count() or 8)) # shader_split.py's and dmsh_build.py's workers (their default is 8) py = sys.executable app, fam = scan_nsps(nsp_dir) force = args.provisional or PROVISIONAL # a provisional profile even when dm/games has one (testing the guesses) @@ -1028,7 +1137,8 @@ def main(args=None): if shadowed and not (args.out or OUT_IPA): out_ipa = out_ipa.with_name(out_ipa.stem + '-provisional.ipa') # never over the IPA of the game's real profile say(f'output IPA: {out_ipa}') work = nsp_dir / f'{g.id}_build' - say(f'work folder (extracted game files in extract/, deleted after the IPA is built unless --keep-extracted): {work}') + keep = KEEP_EXTRACTED if args.keep_extracted is None else args.keep_extracted + say(f'work folder (extracted game files in extract/, {"kept for the next build" if keep else "deleted after the IPA is built"}): {work}') xlate = repo / 'dm/gfx/tools/dmsh_xlate_bin/dmsh_xlate' # DM_GAME picks the profile; DM_ROOT / WORK point at <id>_build: the tools' default paths and temporary directories all stay inside it env.update(DM_GAME=g.id, DM_ROOT=str(work), WORK=str(work / 'tmp')) @@ -1073,10 +1183,10 @@ def main(args=None): sh = work / 'shader' if gfx == 'vulkan': # the packed version's merged RomFS holds the SPIR-V files the game loads; a capture holds those it generates cap = capture_vk(repo, game, patched, work, env, logs, g) if vkcap else None - ios = build_vk_shaders(py, repo, [game / 'RomFS'] + ([cap] if cap else []), sh, env, logs) + ios = build_vk_shaders(py, repo, [game / 'RomFS'] + ([cap] if cap else []), sh, env, logs, g, reuse=not (provisional or ALLOW_DAMAGED)) else: - split_shaders(py, repo, [dirs[v] / 'RomFS' for v in need], [dirs[v] / 'ExeFS/main' for v in need], sh / 'pkg', env, logs, g, provisional) - ios = build_shaders(py, repo, sh, xlate, env, logs, g, provisional) + skey = split_shaders(py, repo, [dirs[v] / 'RomFS' for v in need], [dirs[v] / 'ExeFS/main' for v in need], sh / 'pkg', env, logs, g, provisional) + ios = build_shaders(py, repo, sh, xlate, env, logs, g, provisional, skey) if free_gb(out_ipa.parent) < 5: raise BuildError(f'less than 5 GB free on the volume of {out_ipa.parent}') say(f'building the IPA → {out_ipa}') @@ -1088,13 +1198,13 @@ def main(args=None): else: say('[RED] no game icon in the extraction (an older extraction did not save it: delete extract/ and run again); the home-screen icon will be blank') env_file = args.env_file or DM_ENV_FILE if env_file: ipa_env['DM_ENV_FILE'] = str(Path(os.path.expanduser(env_file)).resolve()) - watch = IpaWatch(repo, work / 'ipa', game / 'RomFS', out_ipa) - PROG.stage('app build', poll=watch.poll, step='starting') # build_ipa.sh's "assemble Payload" line starts the stage "IPA pack" + watch = IpaWatch(repo, work / 'ipa', out_ipa) + PROG.stage('app build', poll=watch.poll, step='starting') # build_ipa.sh's "pack IPA" line starts the stage "IPA pack" run(['bash', repo / 'dm/ios/tools/build_ipa.sh', out_ipa], logs / 'build_ipa.log', env=ipa_env, on_line=watch.line) if not out_ipa.is_file(): raise BuildError(f'no IPA at {out_ipa}') PROG.end(f'{out_ipa.name}, {amount(out_ipa.stat().st_size, "B")}') rmtree(work / 'tmp') - if not (args.keep_extracted or KEEP_EXTRACTED): + if not keep: say(f'deleting the extracted game files: {ex}'); rmtree(ex) say(f'done: {out_ipa} ({out_ipa.stat().st_size / 2**30:.1f} GB)') if provisional: