feat(scripts): add the unixnu build driver

A Python 3 driver with defconfig, fetch, tree, build, bootimg, status
and clean. Device repositories supply board.toml and configs/*_defconfig;
external sources come from feeds.toml and are fetched with shallow,
blobless, sparse git checkouts pinned in unixnu.lock. Modules can come
from local directories through --local, --local-<kind> and
--local-module. The boot image step packs BootShim, the firmware volume
and the device tree for a Mu-Silicium checkout given with --uefi.

.gitignore

@@ -42,3 +42,4 @@ __pycache__/
 *.pem
 *.key
 .env
+local.toml

feeds.toml

@@ -0,0 +1,44 @@
+# External sources of UniXNU. "commit" pins a source; "path" (optional) fetches only that
+# subdirectory with a shallow, blobless, sparse checkout. A device repository may add its own
+# feeds.toml, and a local = "DIR" entry (or the unixnu command line) points a source at a local tree.
+# These are Apple open source references and edk2; none of them is copied into a UniXNU repository.
+
+[[source]]
+name = "xnu"
+kind = "refer"
+git = "https://github.com/apple-oss-distributions/xnu.git"
+ref = "xnu-10063.141.1"
+commit = "d8b80295118ef25ac3a784134bcf95cd8e88109f"
+
+[[source]]
+name = "libplatform"
+kind = "refer"
+git = "https://github.com/apple-oss-distributions/libplatform.git"
+ref = "libplatform-316.100.10"
+commit = "a00a4cc36da2110578bcf3b8eeeeb93dcc7f4e11"
+
+[[source]]
+name = "libdispatch"
+kind = "refer"
+git = "https://github.com/apple-oss-distributions/libdispatch.git"
+ref = "libdispatch-1477.100.9"
+commit = "30fe6a7d945340ce69cef121132f049e810089a5"
+
+[[source]]
+name = "AvailabilityVersions"
+kind = "refer"
+git = "https://github.com/apple-oss-distributions/AvailabilityVersions.git"
+ref = "AvailabilityVersions-141.1"
+commit = "933f9360f1b70dfbda5548386a3d65e0358b4f7d"
+
+[[source]]
+name = "edk2"
+kind = "refer"
+git = "https://github.com/microsoft/mu_basecore.git"
+ref = "release/202608"
+commit = "cfb93724a06ff86ff5ba971413368a7809ee42a5"
+submodules = [
+  "MdePkg/Library/BaseFdtLib/libfdt",
+  "MdePkg/Library/MipiSysTLib/mipisyst",
+  "BaseTools/Source/C/BrotliCompress/brotli",
+]

scripts/check-unixnu.py

@@ -0,0 +1,58 @@
+#!/usr/bin/env python3
+"""Offline self-test of the unixnu build driver: defconfig parsing and module precedence."""
+import argparse
+import sys
+import tempfile
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from unixnu import cli, feeds
+from unixnu.config import Context, parse_config
+from unixnu.util import Error
+
+
+def expect(condition, message):
+    if not condition:
+        raise SystemExit(f'FAIL: {message}')
+
+
+def main():
+    config = parse_config('# c\nCONFIG_A=y\nCONFIG_B=n\n# CONFIG_C is not set\nCONFIG_D="x y"\nCONFIG_E=0x10\n')
+    expect(config == {'A': True, 'B': False, 'C': False, 'D': 'x y', 'E': 16}, 'defconfig values')
+    try:
+        parse_config('CONFIG_A=maybe\n')
+        expect(False, 'bad defconfig value accepted')
+    except Error:
+        pass
+    with tempfile.TemporaryDirectory() as name:
+        root = Path(name)
+        for path in ('dev/drivers/Both', 'dev/drivers/DevOnly', 'loc/drivers/Both', 'loc/drivers/LocOnly', 'kd/Both', 'one/Both'):
+            (root / path).mkdir(parents=True)
+        (root / 'dev/board.toml').write_text('[board]\nname = "t"\n')
+        parsed = cli.parser().parse_args(['status', '-D', str(root / 'dev'), '--out', str(root / 'out')])
+        ctx = Context(parsed)
+        found = feeds.resolve(ctx)
+        expect(found[('driver', 'Both')].origin == 'device', 'device beats base')
+        expect(('driver', 'ArmCpu') in found and found[('driver', 'ArmCpu')].origin == 'base', 'base modules found')
+        parsed = cli.parser().parse_args(['status', '-D', str(root / 'dev'), '--out', str(root / 'out'), '--local', str(root / 'loc')])
+        found = feeds.resolve(Context(parsed))
+        expect(found[('driver', 'Both')].origin == 'local (--local)' and ('driver', 'LocOnly') in found, '--local beats device')
+        parsed = cli.parser().parse_args(['status', '-D', str(root / 'dev'), '--out', str(root / 'out'), '--local', str(root / 'loc'),
+                                          '--local-drivers', str(root / 'kd')])
+        found = feeds.resolve(Context(parsed))
+        expect(found[('driver', 'Both')].origin == 'local (--local-drivers)', '--local-drivers beats --local')
+        parsed = cli.parser().parse_args(['status', '-D', str(root / 'dev'), '--out', str(root / 'out'), '--local', str(root / 'loc'),
+                                          '--local-drivers', str(root / 'kd'), '--local-module', f'Both={root / "one"}'])
+        found = feeds.resolve(Context(parsed))
+        expect(found[('driver', 'Both')].origin == 'local (--local-module)', '--local-module wins')
+        try:
+            feeds.resolve(Context(cli.parser().parse_args(['status', '-D', str(root / 'dev'), '--out', str(root / 'out'),
+                                                            '--local', str(root / 'missing')])))
+            expect(False, 'missing --local directory accepted')
+        except Error:
+            pass
+    print('PASS: unixnu defconfig parsing and module precedence')
+
+
+if __name__ == '__main__':
+    main()

scripts/unixnu/__init__.py

@@ -0,0 +1 @@
+"""UniXNU build driver: sources, build tree, kernel, package, firmware and boot image."""

scripts/unixnu/bootimg.py

@@ -0,0 +1,133 @@
+"""Android boot image: gzip(BootShim + UEFI FD) with the device tree appended, plus an optional ramdisk.
+
+Inputs are a Mu-Silicium checkout (BootShim sources, Resources/Configs/<device>.toml, mkbootimg.py),
+the built firmware volume and a compiled device tree. The image is meant for "fastboot boot" only.
+The result is verified by unpacking it again; the output is deterministic.
+"""
+import gzip
+import hashlib
+import json
+import shutil
+import struct
+import tempfile
+import zlib
+from pathlib import Path
+
+from .config import load_toml
+from .util import Error, log, run, sha256_file, tool
+
+DEFAULTS = {'header_version': 0, 'base': 0, 'kernel_offset': 0x8000, 'tags_offset': 0x1e00000, 'pagesize': 4096}
+MKBOOTIMG_FLAGS = ('base', 'kernel_offset', 'tags_offset', 'pagesize', 'header_version', 'ramdisk_offset', 'second_offset', 'cmdline')
+
+
+def compile_shim(uefi, base, size, requires_header, scratch):
+    clang, objcopy, nm = tool('clang'), tool('llvm-objcopy', 'brew install llvm'), tool('llvm-nm', 'brew install llvm')
+    source = uefi / 'BootShim/BootShim.S'
+    if not source.is_file():
+        raise Error(f'{source} missing: --uefi must be a Mu-Silicium checkout')
+    obj = scratch / 'shim.o'
+    run([clang, '--target=aarch64-none-elf', '-c', f'-DFD_BASE={base}', f'-DFD_SIZE={size}',
+         f'-DREQUIRES_KERNEL_HEADER={int(requires_header)}', source, '-o', obj], capture=True)
+    run([objcopy, '-O', 'binary', obj, scratch / 'shim.bin'], capture=True)
+    shim = (scratch / 'shim.bin').read_bytes()
+    symbols = run([nm, '-n', obj], capture=True)
+    offset = next((int(line.split()[0], 16) for line in symbols.splitlines() if line.endswith(' _Payload')), None)
+    if offset != len(shim):
+        raise Error('a literal pool or padding separates the _Payload label from the appended firmware volume')
+    if requires_header and (shim[56:60] != b'ARM\x64' or struct.unpack_from('<QQ', shim, 8) != (base, len(shim) + size)):
+        raise Error('the ARM64 image header does not describe the shim plus the firmware volume')
+    return shim
+
+
+def patch_dtb(dtb, nodes, scratch):
+    """Add device-tree nodes the stock bootloader expects (board.toml [[bootimg.dtb_node]])."""
+    if not nodes:
+        return dtb
+    fdtput, fdtget = tool('fdtput', 'brew install dtc'), tool('fdtget', 'brew install dtc')
+    work = scratch / 'device.dtb'
+    work.write_bytes(dtb)
+    for node in nodes:
+        run([fdtput, '-c', work, node['path']], capture=True)
+        for prop in node.get('props', []):
+            values = prop['value'] if isinstance(prop['value'], list) else [prop['value']]
+            run([fdtput, '-t', prop.get('type', 's'), work, node['path'], prop['name'], *[str(v) for v in values]], capture=True)
+            got = run([fdtget, '-t', prop.get('type', 's'), work, node['path'], prop['name']], capture=True).split()
+            kind = prop.get('type', 's')
+            if kind in ('x', 'i', 'u') and [int(v, 16 if kind == 'x' else 10) for v in got] != [int(str(v), 0) for v in values]:
+                raise Error(f'dtb node {node["path"]} property {prop["name"]} did not round-trip')
+    return work.read_bytes()
+
+
+def settings(ctx, uefi):
+    fw = ctx.board.get('firmware', {})
+    config = load_toml(uefi / 'Resources/Configs' / f'{fw["config"]}.toml') if fw.get('config') else {}
+    values = dict(DEFAULTS)
+    values.update(config.get('boot_image', {}))
+    values.update(ctx.board.get('bootimg', {}).get('image', {}))
+    fd = config.get('uefi_fd', {})
+    return values, fd.get('base'), fd.get('size'), config.get('boot_shim', {}).get('requires_kernel_header', True), config
+
+
+def build(ctx, uefi, fd, dtb, ramdisk=None, output=None):
+    log('bootimg')
+    image, base, size, requires_header, config = settings(ctx, uefi)
+    mkbootimg = uefi / 'Resources/Scripts/mkbootimg.py'
+    if not mkbootimg.is_file():
+        raise Error(f'{mkbootimg} missing: --uefi must be a Mu-Silicium checkout')
+    for path, what in ((fd, 'firmware volume (--fd)'), (dtb, 'device tree (--dtb)')):
+        if not path or not Path(path).is_file():
+            raise Error(f'{what} not found: {path}')
+    firmware = Path(fd).read_bytes()
+    if base is None or size is None:
+        raise Error('the Mu-Silicium device config has no [uefi_fd] base and size')
+    if len(firmware) != size:
+        raise Error(f'firmware volume is {len(firmware):#x} bytes, the device config says {size:#x}')
+    out = Path(output) if output else ctx.work / 'boot/boot.img'
+    out.parent.mkdir(parents=True, exist_ok=True)
+    with tempfile.TemporaryDirectory(prefix='unixnu-bootimg-') as scratch_name:
+        scratch = Path(scratch_name)
+        shim = compile_shim(uefi, base, size, requires_header, scratch)
+        tree = patch_dtb(Path(dtb).read_bytes(), ctx.board.get('bootimg', {}).get('dtb_node', []), scratch)
+        if tree[:4] != b'\xd0\x0d\xfe\xed' or struct.unpack_from('>I', tree, 4)[0] != len(tree):
+            raise Error('invalid compiled device tree')
+        combined = shim + firmware
+        kernel = gzip.compress(combined, mtime=0) + tree
+        kernel_path = scratch / 'Image.gz-dtb'
+        kernel_path.write_bytes(kernel)
+        if ramdisk:
+            ramdisk_path = Path(ramdisk)
+            if not ramdisk_path.is_file():
+                raise Error(f'ramdisk not found: {ramdisk_path}')
+        else:
+            ramdisk_path = scratch / 'ramdisk'
+            ramdisk_path.write_bytes(b'')
+        command = ['python3', mkbootimg, '--kernel', kernel_path, '--ramdisk', ramdisk_path, '-o', out]
+        for key in MKBOOTIMG_FLAGS:
+            if key in image:
+                command += [f'--{key}', str(image[key])]
+        run(command, capture=True)
+        report = verify(out, combined, tree, ramdisk_path, image)
+    report.update(image_sha256=sha256_file(out), image_bytes=out.stat().st_size, fd_sha256=sha256_file(fd),
+                  dtb_sha256=sha256_file(dtb), shim_bytes=len(shim), hardware_tested=False)
+    (out.parent / 'boot.json').write_text(json.dumps(report, indent=2) + '\n')
+    log(f'bootimg: {out} ({out.stat().st_size} bytes)')
+    return out
+
+
+def verify(path, combined, tree, ramdisk_path, image):
+    """Unpack the image and compare every part with its input."""
+    data = Path(path).read_bytes()
+    kernel_size, kernel_address, ramdisk_size, _, second_size, _, tags, page, version, _ = struct.unpack_from('<10I', data, 8)
+    if data[:8] != b'ANDROID!' or second_size or version != image['header_version'] or page != image['pagesize']:
+        raise Error('boot image header does not match the requested parameters')
+    if kernel_address != image['base'] + image['kernel_offset'] or tags != image['base'] + image['tags_offset']:
+        raise Error('boot image load addresses do not match the requested offsets')
+    kernel = data[page:page + kernel_size]
+    unpacker = zlib.decompressobj(16 + zlib.MAX_WBITS)
+    if unpacker.decompress(kernel) != combined or not unpacker.eof or unpacker.unused_data != tree:
+        raise Error('gzip, firmware and appended device tree do not round-trip')
+    start = page * (1 + -(-kernel_size // page))
+    if data[start:start + ramdisk_size] != Path(ramdisk_path).read_bytes():
+        raise Error('ramdisk bytes differ')
+    return {'kernel_sha256': hashlib.sha256(kernel).hexdigest(), 'ramdisk_bytes': ramdisk_size,
+            'header_version': version, 'kernel_address': hex(kernel_address), 'tags_address': hex(tags)}

scripts/unixnu/cli.py

@@ -0,0 +1,165 @@
+"""Command line of the UniXNU build driver."""
+import argparse
+import platform
+import shutil
+import sys
+
+from . import bootimg, feeds, firmware, kernel, package, tree
+from .config import LOCAL_KINDS, Context, parse_config
+from .util import Error, expand, git, log
+
+STEPS = ('kernel', 'package', 'firmware')
+
+
+def common():
+    p = argparse.ArgumentParser(add_help=False)
+    p.add_argument('-D', '--device', metavar='DIR', help='device repository (remembered; or UNIXNU_DEVICE)')
+    p.add_argument('--out', metavar='DIR', help='output root (default: out/ here, or UNIXNU_OUT)')
+    p.add_argument('-j', '--jobs', type=int, help='parallel jobs')
+    p.add_argument('--local', metavar='DIR', help='directory laid out like this repository; its drivers, kits, patches, scripts and package are used')
+    for kind in LOCAL_KINDS:
+        p.add_argument(f'--local-{kind}', metavar='DIR', help=f'use DIR as the {kind} directory (beats --local)')
+    p.add_argument('--local-module', action='append', metavar='[KIND:]NAME=PATH', help='use PATH for one module (beats everything)')
+    p.add_argument('--uefi', metavar='DIR', help='Mu-Silicium checkout for the device (uefi-sheng); or UNIXNU_UEFI')
+    p.add_argument('--edk2', metavar='DIR', help='Mu_Basecore checkout; or UNIXNU_EDK2')
+    p.add_argument('--kernelcache', metavar='FILE', help='original decompressed kernelcache; or UNIXNU_KERNELCACHE')
+    p.add_argument('--target', default='DEBUG', choices=('DEBUG', 'RELEASE'), help='EDK II build target')
+    return p
+
+
+def parser():
+    top = argparse.ArgumentParser(prog='unixnu', description='Build UniXNU, the XNU kernel and the boot image for a device repository.')
+    sub = top.add_subparsers(dest='command', required=True)
+    c = common()
+    d = sub.add_parser('defconfig', parents=[c], help='select configs/NAME in the device repository')
+    d.add_argument('name', nargs='?', help='defconfig name; omit to list')
+    f = sub.add_parser('fetch', parents=[c], help='shallow-fetch git sources and write unixnu.lock')
+    f.add_argument('names', nargs='*', help='sources to fetch (default: all)')
+    f.add_argument('--update', action='store_true', help='ignore the lock and move to the newest ref')
+    sub.add_parser('tree', parents=[c], help='assemble the build tree')
+    b = sub.add_parser('build', parents=[c], help='build kernel, package and firmware')
+    b.add_argument('steps', nargs='*', choices=STEPS + ('all', 'prelink'), default=['all'])
+    i = sub.add_parser('bootimg', parents=[c], help='pack the boot image from the firmware volume and a device tree')
+    i.add_argument('--fd', metavar='FILE', help='firmware volume (default: the build output under --uefi)')
+    i.add_argument('--dtb', metavar='FILE', help='compiled device tree; or UNIXNU_DTB')
+    i.add_argument('--ramdisk', metavar='FILE', help='ramdisk / payload image (default: empty)')
+    i.add_argument('-o', '--output', metavar='FILE', help='output (default: out/<device>/boot/boot.img)')
+    sub.add_parser('status', parents=[c], help='show the device, configuration and where each module comes from')
+    sub.add_parser('clean', parents=[c], help='remove the build output of the device (keeps fetched sources)')
+    return top
+
+
+def need_uefi(ctx):
+    uefi = ctx.option('uefi', 'UNIXNU_UEFI')
+    if not uefi:
+        raise Error('the firmware needs a Mu-Silicium checkout: --uefi DIR (or UNIXNU_UEFI)')
+    if not (uefi / 'Resources/Scripts/mkbootimg.py').is_file():
+        raise Error(f'{uefi} is not a Mu-Silicium checkout')
+    return uefi
+
+
+REFERENCES = {'kernel': ('xnu', 'libplatform', 'libdispatch', 'AvailabilityVersions'), 'package': ('edk2',), 'firmware': ('edk2',)}
+
+
+def prepare(ctx, steps=()):
+    """Resolve modules, fetch the git ones (references only for the steps that need them) and assemble the tree."""
+    modules = feeds.resolve(ctx)
+    wanted = {name for step in steps for name in REFERENCES.get(step, ())}
+    feeds.fetch(ctx, modules, only={m.name for m in modules.values() if m.origin == 'git' and (m.kind != 'refer' or m.name in wanted)})
+    tree.assemble(ctx, modules)
+    return modules
+
+
+def cmd_defconfig(ctx):
+    configs = ctx.device / 'configs'
+    names = sorted(p.name for p in configs.glob('*_defconfig')) if configs.is_dir() else []
+    if not ctx.args.name:
+        print('\n'.join(names) or 'no defconfigs')
+        return
+    source = configs / ctx.args.name
+    if not source.is_file():
+        raise Error(f'{source} not found; available: {", ".join(names) or "none"}')
+    parse_config(source.read_text())
+    ctx.work.mkdir(parents=True, exist_ok=True)
+    shutil.copyfile(source, ctx.config_path)
+    log(f'configuration {ctx.args.name} -> {ctx.config_path}')
+
+
+def cmd_fetch(ctx):
+    modules = feeds.resolve(ctx)
+    commits = feeds.fetch(ctx, modules, update=ctx.args.update, only=set(ctx.args.names) or None)
+    if commits:
+        feeds.lock_write(ctx, {**feeds.lock_read(ctx), **commits})
+        log(f'wrote {ctx.lock_path}')
+
+
+def cmd_build(ctx):
+    steps = ctx.args.steps
+    if 'all' in steps:
+        steps = list(STEPS)
+    ctx.need_config()
+    if 'firmware' in steps:
+        need_uefi(ctx)   # fail before fetching anything
+    modules = prepare(ctx, steps + (['kernel'] if 'prelink' in steps else []))
+    if 'kernel' in steps:
+        kernel.build(ctx)
+        kernel.prelink(ctx)
+    elif 'prelink' in steps:
+        kernel.prelink(ctx)
+    if 'package' in steps:
+        package.build(ctx, modules, ctx.args.target)
+    if 'firmware' in steps:
+        firmware.build(ctx, modules, need_uefi(ctx), package.edk2_dir(ctx, modules), ctx.args.target)
+
+
+def cmd_bootimg(ctx):
+    uefi = need_uefi(ctx)
+    fd = expand(ctx.args.fd) if ctx.args.fd else firmware.fd_path(ctx, uefi, ctx.args.target)
+    dtb = ctx.option('dtb', 'UNIXNU_DTB')
+    ramdisk = expand(ctx.args.ramdisk) if ctx.args.ramdisk else None
+    bootimg.build(ctx, uefi, fd, dtb, ramdisk, expand(ctx.args.output) if ctx.args.output else None)
+
+
+def cmd_status(ctx):
+    print(f'device   {ctx.device} ({ctx.name})')
+    print(f'output   {ctx.work}')
+    print(f'config   {ctx.config_path if ctx.config_path.is_file() else "none (run defconfig)"}')
+    modules = feeds.resolve(ctx)
+    locked = feeds.lock_read(ctx)
+    unpinned = []
+    for (kind, name), module in sorted(modules.items()):
+        extra = ''
+        if module.origin == 'git':
+            pinned = locked.get(name) or module.source.get('commit')
+            extra = f' pinned {pinned[:12]}' if pinned else ' NOT LOCKED'
+        elif module.local:
+            try:
+                extra = ' (dirty)' if git(module.path, 'status', '--porcelain', '--', '.').strip() else ' (clean)'
+            except Error:
+                extra = ' (not a git work tree)'
+            unpinned.append(f'{kind}:{name}')
+        print(f'{kind:8} {name:24} {module.origin}{extra}')
+    if unpinned:
+        print(f'\nnot reproducible: {len(unpinned)} local source(s) are not pinned by unixnu.lock')
+
+
+def cmd_clean(ctx):
+    for name in ('tree', 'logs', 'kernel', 'package', 'boot', 'edk2-conf'):
+        shutil.rmtree(ctx.work / name, ignore_errors=True)
+    log(f'cleaned {ctx.work}')
+
+
+def main(argv=None):
+    if sys.version_info < (3, 11):
+        sys.exit('unixnu needs Python 3.11 or newer')
+    if platform.system() != 'Darwin':
+        sys.exit('unixnu supports macOS hosts only')
+    args = parser().parse_args(argv)
+    try:
+        ctx = Context(args)
+        {'defconfig': cmd_defconfig, 'fetch': cmd_fetch, 'tree': lambda c: prepare(c), 'build': cmd_build,
+         'bootimg': cmd_bootimg, 'status': cmd_status, 'clean': cmd_clean}[args.command](ctx)
+    except Error as error:
+        sys.exit(f'unixnu: {error}')
+    except KeyboardInterrupt:
+        sys.exit(130)

scripts/unixnu/config.py

@@ -0,0 +1,107 @@
+"""Board, feeds and defconfig files, and the context every build step shares.
+
+A device repository supplies:
+  board.toml              board description (like Android BoardConfig.mk)
+  configs/<name>_defconfig  CONFIG_* switches (like an OpenWrt defconfig)
+  feeds.toml              optional extra sources
+  drivers/ kits/ patches/ scripts/ package/   its own modules, same layout as this repository
+"""
+import json
+import os
+import re
+import tomllib
+from pathlib import Path
+
+from .util import Error, expand
+
+ROOT = Path(__file__).resolve().parents[2]
+KINDS = {'driver': 'drivers', 'kit': 'kits', 'patch': 'patches', 'script': 'scripts', 'package': 'package', 'refer': 'refer'}
+LOCAL_KINDS = ('drivers', 'kits', 'patches', 'scripts', 'package')
+
+
+def load_toml(path):
+    path = Path(path)
+    try:
+        return tomllib.loads(path.read_text())
+    except FileNotFoundError:
+        raise Error(f'missing file: {path}') from None
+    except tomllib.TOMLDecodeError as error:
+        raise Error(f'{path}: {error}') from None
+
+
+def parse_config(text):
+    """Parse a defconfig: CONFIG_X=y, =n, ="text", =123 and "# CONFIG_X is not set"."""
+    values = {}
+    for number, line in enumerate(text.splitlines(), 1):
+        line = line.strip()
+        unset = re.fullmatch(r'#\s*CONFIG_(\w+) is not set', line)
+        if unset:
+            values[unset.group(1)] = False
+            continue
+        if not line or line.startswith('#'):
+            continue
+        match = re.fullmatch(r'CONFIG_(\w+)=(.*)', line)
+        if not match:
+            raise Error(f'defconfig line {number} is not CONFIG_NAME=value: {line}')
+        name, raw = match.groups()
+        if raw == 'y':
+            values[name] = True
+        elif raw == 'n':
+            values[name] = False
+        elif raw.startswith('"') and raw.endswith('"') and len(raw) >= 2:
+            values[name] = raw[1:-1]
+        elif re.fullmatch(r'-?(0x[0-9a-fA-F]+|\d+)', raw):
+            values[name] = int(raw, 0)
+        else:
+            raise Error(f'defconfig line {number}: value must be y, n, a number or a quoted string: {line}')
+    return values
+
+
+class Context:
+    """Everything the steps need: repositories, options, board, config and the output layout."""
+
+    def __init__(self, args):
+        self.root = ROOT
+        self.args = args
+        self.out_root = expand(args.out or os.environ.get('UNIXNU_OUT') or ROOT / 'out')
+        self.state_path = self.out_root / 'state.json'
+        state = self._load_state()
+        device = args.device or os.environ.get('UNIXNU_DEVICE') or state.get('device')
+        if not device:
+            raise Error('no device repository: pass -D DIR (or set UNIXNU_DEVICE); it is remembered afterwards')
+        self.device = expand(device)
+        if not (self.device / 'board.toml').is_file():
+            raise Error(f'{self.device} has no board.toml')
+        self.board = load_toml(self.device / 'board.toml')
+        self.name = self.board.get('board', {}).get('name')
+        if not self.name:
+            raise Error('board.toml needs [board] name')
+        self.work = self.out_root / self.name
+        self.tree = self.work / 'tree'
+        self.logs = self.work / 'logs'
+        self.feeds_dir = self.out_root / 'feeds'
+        self.config_path = self.work / '.config'
+        self.jobs = args.jobs or int(os.environ.get('JOBS', 0)) or os.cpu_count() or 2
+        self._save_state({'device': str(self.device)})
+        self.config = parse_config(self.config_path.read_text()) if self.config_path.is_file() else {}
+        self.lock_path = self.device / 'unixnu.lock'
+
+    def _load_state(self):
+        try:
+            return json.loads(self.state_path.read_text())
+        except (FileNotFoundError, ValueError):
+            return {}
+
+    def _save_state(self, state):
+        self.out_root.mkdir(parents=True, exist_ok=True)
+        self.state_path.write_text(json.dumps(state, indent=2) + '\n')
+
+    def need_config(self):
+        if not self.config_path.is_file():
+            raise Error(f'no configuration: run "unixnu defconfig NAME" first (see {self.device / "configs"})')
+        return self.config
+
+    def option(self, flag, env=None):
+        """A path given as a flag or, failing that, an environment variable."""
+        value = getattr(self.args, flag, None) or (os.environ.get(env) if env else None)
+        return expand(value) if value else None

scripts/unixnu/feeds.py

@@ -0,0 +1,215 @@
+"""Module sources: local directories, in-tree modules and shallow, sparse git checkouts.
+
+A module is a driver, kit, patch set or the package, identified by (kind, name). Candidates are
+taken in this order, first one that exists wins:
+
+  1. --local-module NAME=PATH
+  2. --local-<kind> DIR              (DIR/NAME)
+  3. --local DIR                     (DIR/<kind>/NAME, same layout as this repository)
+  4. a "local" entry in feeds.toml or the ignored local.toml of the device repository
+  5. the device repository's own directories
+  6. this repository's own directories
+  7. a git source from feeds.toml: shallow, blobless, sparse checkout of the module path
+
+Git sources never use submodules. Nothing fetched or linked is copied into either repository.
+"""
+import hashlib
+from pathlib import Path
+
+from .config import KINDS, LOCAL_KINDS, load_toml
+from .util import Error, expand, git, log, run
+
+
+class Module:
+    def __init__(self, kind, name, path=None, origin='', source=None):
+        self.kind, self.name, self.path, self.origin, self.source = kind, name, path, origin, source
+        self.commit = None
+
+    @property
+    def local(self):
+        return self.origin.startswith('local')
+
+    def __repr__(self):
+        return f'{self.kind}:{self.name} <{self.origin}> {self.path}'
+
+
+def read_sources(ctx):
+    """Entries of every feeds.toml (this repository, then the device repository) and the device local.toml."""
+    sources, local = {}, {}
+    for base in (ctx.root, ctx.device):
+        path = base / 'feeds.toml'
+        if path.is_file():
+            for entry in load_toml(path).get('source', []):
+                kind = entry.get('kind')
+                if kind not in KINDS or not entry.get('name'):
+                    raise Error(f'{path}: every [[source]] needs name and kind in {sorted(KINDS)}')
+                sources[(kind, entry['name'])] = entry
+    path = ctx.device / 'local.toml'
+    if path.is_file():
+        local = load_toml(path)
+    return sources, local
+
+
+def lock_read(ctx):
+    if not ctx.lock_path.is_file():
+        return {}
+    return {name: values['commit'] for name, values in load_toml(ctx.lock_path).get('source', {}).items()}
+
+
+def lock_write(ctx, commits):
+    lines = ['# Resolved git sources, written by "unixnu fetch". Local sources are never recorded.']
+    for name in sorted(commits):
+        lines += ['', f'[source."{name}"]', f'commit = "{commits[name]}"']
+    ctx.lock_path.write_text('\n'.join(lines) + '\n')
+
+
+def _children(directory):
+    directory = Path(directory)
+    return sorted(p.name for p in directory.iterdir() if p.is_dir() and not p.name.startswith('.')) if directory.is_dir() else []
+
+
+def _cli_locals(args):
+    """(per-module overrides, per-kind directories, local root) from the command line."""
+    modules = {}
+    for item in args.local_module or []:
+        if '=' not in item:
+            raise Error(f'--local-module needs NAME=PATH (or KIND:NAME=PATH): {item}')
+        key, path = item.split('=', 1)
+        kind, _, name = key.partition(':') if ':' in key else ('', '', key)
+        modules[(kind, name)] = expand(path)
+    kinds = {}
+    for kind in LOCAL_KINDS:
+        value = getattr(args, 'local_' + kind, None)
+        if value:
+            kinds[kind] = expand(value)
+    return modules, kinds, expand(args.local) if args.local else None
+
+
+def _need(path, what):
+    if not Path(path).is_dir():
+        raise Error(f'{what}: {path} is not a directory')
+    return Path(path)
+
+
+def resolve(ctx):
+    """Map (kind, name) to a Module. Git modules are resolved to a clone path later by fetch()."""
+    sources, local_file = read_sources(ctx)
+    cli_modules, cli_kinds, cli_root = _cli_locals(ctx.args)
+    if cli_root is not None:
+        _need(cli_root, '--local')
+    for (kind, name) in cli_modules:
+        if kind and kind not in KINDS:
+            raise Error(f'--local-module: unknown kind {kind}')
+    result = {}
+
+    def add(kind, name, path, origin, source=None):
+        result.setdefault((kind, name), Module(kind, name, path, origin, source))
+
+    plural = {kind: KINDS[kind] for kind in KINDS}
+    # 1: --local-module
+    for (kind, name), path in cli_modules.items():
+        _need(path, f'--local-module {name}')
+        targets = [kind] if kind else [k for k in KINDS if (k, name) in sources or any(
+            (base / plural[k] / name).is_dir() for base in (ctx.device, ctx.root))]
+        if not targets:
+            raise Error(f'--local-module {name}: unknown module; use KIND:NAME=PATH to add a new one')
+        if len(targets) > 1:
+            raise Error(f'--local-module {name} is ambiguous ({", ".join(targets)}); use KIND:NAME=PATH')
+        add(targets[0], name, path, 'local (--local-module)')
+    # 2 and 3: per-kind directories and the local root
+    for kind in KINDS:
+        directory = cli_kinds.get(plural[kind])
+        origin = f'local (--local-{plural[kind]})'
+        if directory is None and cli_root is not None and (cli_root / plural[kind]).is_dir():
+            directory, origin = cli_root / plural[kind], 'local (--local)'
+        if directory is None or kind in ('refer', 'script'):
+            continue
+        if kind == 'package':
+            add('package', 'main', _need(directory, '--local-package'), origin)
+            continue
+        for name in _children(_need(directory, f'--local-{plural[kind]}')):
+            add(kind, name, directory / name, origin)
+    # 4: local entries of the configuration files
+    for (kind, name), entry in sources.items():
+        if entry.get('local'):
+            add(kind, name, _need(expand(entry['local']), f'source {name}'), 'local (feeds)')
+    for kind, directory in local_file.get('local', {}).items():
+        if kind in LOCAL_KINDS and kind not in cli_kinds and kind != 'package':
+            for name in _children(_need(expand(directory), f'local.toml {kind}')):
+                add({v: k for k, v in KINDS.items()}[kind], name, expand(directory) / name, 'local (local.toml)')
+    # 5 and 6: the device repository, then this repository
+    for base, label in ((ctx.device, 'device'), (ctx.root, 'base')):
+        for kind, folder in KINDS.items():
+            if kind in ('refer', 'script'):
+                continue
+            if kind == 'package':
+                if (base / 'package').is_dir():
+                    add('package', 'main' if base == ctx.root else 'device', base / 'package', label)
+                continue
+            for name in _children(base / folder):
+                add(kind, name, base / folder / name, label)
+    # 7: git sources
+    for (kind, name), entry in sources.items():
+        if entry.get('git'):
+            add(kind, name, None, 'git', entry)
+    return result
+
+
+def _clone_dir(ctx, entry):
+    key = hashlib.sha1(f'{entry["git"]}\n{entry.get("ref", "")}'.encode()).hexdigest()[:8]
+    return ctx.feeds_dir / f'{Path(entry["git"]).stem}-{key}'
+
+
+def fetch(ctx, modules, update=False, only=None):
+    """Shallow, blobless, sparse checkout of git modules (all, or the named ones); returns {source name: commit}.
+
+    A clone already at the wanted commit is not contacted again."""
+    locked = {} if update else lock_read(ctx)
+    groups = {}
+    for module in modules.values():
+        if module.origin == 'git' and (only is None or module.name in only):
+            groups.setdefault(_clone_dir(ctx, module.source), []).append(module)
+    commits = {}
+    for directory, members in groups.items():
+        entry = members[0].source
+        paths = sorted({m.source['path'] for m in members if m.source.get('path')})
+        sparse = bool(paths) and all(m.source.get('path') for m in members)
+        target = next((locked[m.name] for m in members if m.name in locked), None) or entry.get('commit') or entry.get('ref') or 'HEAD'
+        current = None
+        if (directory / '.git').is_dir():
+            try:
+                current = git(directory, 'rev-parse', 'HEAD').strip()
+            except Error:
+                current = None
+        if current and current == target and (not sparse or all((directory / path).exists() for path in paths)):
+            _attach(members, directory, current, commits)
+            continue
+        log(f'fetch {entry["git"]} {target[:12]}' + (f' ({", ".join(paths)})' if sparse else ''))
+        if not (directory / '.git').is_dir():
+            directory.mkdir(parents=True, exist_ok=True)
+            git(directory, 'init', '-q')
+            git(directory, 'remote', 'add', 'origin', entry['git'])
+        if sparse:
+            git(directory, 'sparse-checkout', 'set', '--cone', *paths)
+            git(directory, 'config', 'remote.origin.promisor', 'true')
+            git(directory, 'config', 'remote.origin.partialclonefilter', 'blob:none')
+            run(['git', '-C', directory, 'fetch', '-q', '--depth', '1', '--filter=blob:none', 'origin', target])
+        else:
+            git(directory, 'sparse-checkout', 'disable')
+            run(['git', '-C', directory, 'fetch', '-q', '--depth', '1', 'origin', target])
+        git(directory, 'checkout', '-q', '--detach', 'FETCH_HEAD')
+        if entry.get('submodules'):
+            # Only for upstream trees that need them (edk2); UniXNU repositories never use submodules.
+            run(['git', '-C', directory, 'submodule', 'update', '--init', '--depth', '1', '--', *entry['submodules']])
+        _attach(members, directory, git(directory, 'rev-parse', 'HEAD').strip(), commits)
+    return commits
+
+
+def _attach(members, directory, commit, commits):
+    for module in members:
+        subpath = module.source.get('path', '')
+        module.path = directory / subpath if subpath else directory
+        if not module.path.exists():
+            raise Error(f'{module.source["git"]} at {commit[:12]} has no {subpath}')
+        module.commit = commit
+        commits[module.name] = commit

scripts/unixnu/firmware.py

@@ -0,0 +1,66 @@
+"""Device UEFI firmware: the EDK II build of a Mu-Silicium checkout (uefi-sheng), driven by board.toml [firmware]."""
+import os
+import shutil
+from pathlib import Path
+
+from .config import load_toml
+from .util import Error, git, log, run, tool
+
+
+def uefi_config(ctx, uefi):
+    fw = ctx.board.get('firmware')
+    if not fw:
+        raise Error('board.toml has no [firmware]')
+    return fw, load_toml(uefi / 'Resources/Configs' / f'{fw["config"]}.toml')
+
+
+def fd_path(ctx, uefi, target='DEBUG'):
+    fw = ctx.board['firmware']
+    return uefi / fw['fd'].format(target=target)
+
+
+def build(ctx, modules, uefi, edk2, target='DEBUG'):
+    log(f'firmware: {uefi.name}')
+    fw, config = uefi_config(ctx, uefi)
+    for name in ('clang', 'python3', 'make'):
+        tool(name)
+    base, size = config['uefi_fd']['base'], config['uefi_fd']['size']
+    for script in fw.get('preflight', []):
+        run(['python3', ctx.tree / script], cwd=ctx.tree, log_file=ctx.logs / 'firmware-preflight.log')
+    defines = {'FD_BASE': hex(base), 'FD_SIZE': hex(size), 'FD_BLOCKS': str(size // 0x1000)}
+    defines.update({key: str(value) for key, value in fw.get('defines', {}).items()})
+    fields = {'uefi': str(uefi), 'edk2': str(edk2), 'tree': str(ctx.tree)}
+    packages = ':'.join(item.format(**fields) for item in fw['packages'])
+    tools = edk2 / 'BaseTools'
+    if not (tools / 'Source/C/bin/GenFw').is_file():
+        log('firmware: building BaseTools')
+        run(['make', '-C', tools / 'Source/C', f'-j{ctx.jobs}'], log_file=ctx.logs / 'basetools.log')
+    conf = ctx.work / 'edk2-conf'
+    conf.mkdir(parents=True, exist_ok=True)
+    for name in ('target', 'tools_def', 'build_rule'):
+        if not (conf / f'{name}.txt').is_file():
+            shutil.copy(tools / 'Conf' / f'{name}.template', conf / f'{name}.txt')
+    env = dict(os.environ, WORKSPACE=str(uefi), EDK_TOOLS_PATH=str(tools), CONF_PATH=str(conf), PACKAGES_PATH=packages,
+               PYTHON_COMMAND='python3',
+               PATH=os.pathsep.join([str(tools / 'BinWrappers/PosixLike'), str(tools / 'Source/C/bin'), os.environ['PATH']]))
+    env.setdefault('CLANG_BIN', os.path.dirname(shutil.which('clang')) + '/')
+    applied = []
+    try:
+        # Board patches for the shared edk2 checkout: applied for the build, always reverted.
+        for patch in fw.get('patches', []):
+            path = uefi / patch
+            run(['git', '-C', edk2, 'apply', '--check', path])
+            run(['git', '-C', edk2, 'apply', path])
+            applied.append(path)
+        command = ['build', '-p', fw['dsc'], '-a', 'AARCH64', '-b', target, '-t', 'CLANGPDB', '-n', str(ctx.jobs)]
+        for key, value in defines.items():
+            command += ['-D', f'{key}={value}']
+        run(command, cwd=uefi, env=env, log_file=ctx.logs / 'firmware.log')
+    finally:
+        for path in reversed(applied):
+            git(edk2, 'apply', '--reverse', str(path))
+    fd = fd_path(ctx, uefi, target)
+    if not fd.is_file() or fd.stat().st_size != size:
+        raise Error(f'{fd} missing or not {size:#x} bytes')
+    log(f'firmware: {fd}')
+    return fd

scripts/unixnu/kernel.py

@@ -0,0 +1,59 @@
+"""XNU kernel: build from the patched Apple source, then prelink against the original kernelcache."""
+import os
+from pathlib import Path
+
+from .util import Error, expand, log, run, sha256_file, tool
+
+
+def xnu_env(ctx):
+    """CONFIG_XNU_* of the defconfig become XNU_* variables of the legacy kernel build script."""
+    env = {}
+    for key, value in ctx.need_config().items():
+        if not key.startswith('XNU_'):
+            continue
+        if isinstance(value, str):
+            value = value.replace('{tree}', str(ctx.tree))
+        env[key] = '1' if value is True else '0' if value is False else str(value)
+    return env
+
+
+def build(ctx):
+    log('kernel: XNU')
+    for name in ('git', 'make', 'clang', 'cmake', 'ninja', 'unifdef'):
+        tool(name, 'brew install ' + name)
+    script = ctx.tree / 'scripts/build-xnu-4k-baseline.sh'
+    if not script.is_file():
+        raise Error(f'{script} missing')
+    env = dict(os.environ)
+    env.update(xnu_env(ctx))
+    env['JOBS'] = str(ctx.jobs)
+    run(['bash', script], cwd=ctx.tree, env=env, log_file=ctx.logs / 'kernel.log')
+
+
+def prelink(ctx):
+    """Produce out/<device>/kernel/kernel.macho from the XNU build and the original kernelcache."""
+    log('kernel: prelink')
+    spec = ctx.board.get('kernel', {}).get('prelink')
+    if not spec:
+        raise Error('board.toml has no [kernel.prelink]')
+    cache = ctx.option('kernelcache', 'UNIXNU_KERNELCACHE') or (expand(ctx.tree / spec['kernelcache']) if spec.get('kernelcache') else None)
+    if not cache or not cache.is_file():
+        raise Error('the original decompressed kernelcache is an input you provide: --kernelcache FILE or UNIXNU_KERNELCACHE')
+    wanted = spec.get('kernelcache_sha256')
+    if wanted and sha256_file(cache) != wanted:
+        raise Error(f'{cache} does not match the kernelcache this board was qualified against (sha256 {wanted})')
+    kernel = ctx.tree / spec['kernel']
+    if not kernel.is_file():
+        raise Error(f'{kernel} not found; run "unixnu build kernel" first')
+    output = ctx.work / 'kernel/kernel.macho'
+    output.parent.mkdir(parents=True, exist_ok=True)
+    command = ['python3', ctx.tree / spec['tool'], '--kernel', kernel, '--prelinked', cache,
+               f'--slide={spec["slide"]}', '--output', output]
+    for root in spec.get('roots', []):
+        command += ['--root', root]
+    if spec.get('early_security_init'):
+        command += ['--early-security-init', spec['early_security_init']]
+    command += spec.get('extra_args', [])
+    run(command, cwd=ctx.tree, log_file=ctx.logs / 'prelink.log')
+    log(f'kernel: {output} sha256 {sha256_file(output)}')
+    return output

scripts/unixnu/package.py

@@ -0,0 +1,32 @@
+"""UniXNU itself: the EDK II build of package/ (CrabApple), output under out/<device>/package."""
+import os
+import shutil
+
+from .util import Error, expand, log, run, tool
+
+
+def edk2_dir(ctx, modules):
+    given = ctx.option('edk2', 'UNIXNU_EDK2')
+    if given:
+        path = given
+    elif ('refer', 'edk2') in modules and modules[('refer', 'edk2')].path:
+        path = modules[('refer', 'edk2')].path
+    else:
+        raise Error('no edk2 tree: pass --edk2 DIR (a Mu_Basecore checkout) or run "unixnu fetch" for the edk2 source')
+    if not (path / 'MdePkg/MdePkg.dec').is_file():
+        raise Error(f'{path} is not an edk2 or Mu_Basecore tree')
+    return path
+
+
+def build(ctx, modules, target='DEBUG', toolchain='CLANGPDB'):
+    log('package: UniXNU (CrabApple)')
+    edk2 = edk2_dir(ctx, modules)
+    tool('clang', 'brew install llvm')
+    env = dict(os.environ, CRABAPPLE_EDK2=str(edk2), UNIXNU_BUILD_DIR=str(ctx.work / 'package'), JOBS=str(ctx.jobs))
+    clang = shutil.which('clang')
+    if toolchain == 'CLANGPDB' and 'CLANG_BIN' not in env:
+        env['CLANG_BIN'] = os.path.dirname(clang) + '/'
+    (ctx.work / 'package').mkdir(parents=True, exist_ok=True)
+    run(['bash', ctx.tree / 'package/build.sh', '-b', target, '-t', toolchain], cwd=ctx.tree / 'package', env=env,
+        log_file=ctx.logs / 'package.log')
+    log(f'package: outputs in {ctx.work / "package/Build/Out"}')

scripts/unixnu/tree.py

@@ -0,0 +1,68 @@
+"""Assemble the build tree from the resolved modules.
+
+out/<device>/tree/ has the layout of this repository (drivers/ kits/ patches/ scripts/ package/)
+plus refer/, build/ and .cache/ for the scripts that expect them. Modules are symbolic links to
+their sources, so edits to a local module take effect on the next build; scripts are copied, since
+they locate the tree from their own real path. The device package appears as kits/<kit>.
+"""
+import shutil
+from pathlib import Path
+
+from .util import Error, log
+
+
+def _clear_links(directory):
+    if directory.is_dir():
+        for entry in directory.iterdir():
+            if entry.is_symlink():
+                entry.unlink()
+
+
+def _link(target, link):
+    link.parent.mkdir(parents=True, exist_ok=True)
+    if link.is_symlink() or link.exists():
+        link.unlink()
+    link.symlink_to(Path(target).resolve())
+
+
+def _copy_scripts(source, destination):
+    for path in sorted(Path(source).rglob('*')):
+        relative = path.relative_to(source)
+        if path.is_dir() or '__pycache__' in path.parts or relative.parts[0] == 'unixnu':
+            continue
+        target = destination / relative
+        if target.exists() and target.stat().st_size == path.stat().st_size and target.stat().st_mtime >= path.stat().st_mtime:
+            continue
+        target.parent.mkdir(parents=True, exist_ok=True)
+        shutil.copy2(path, target)
+
+
+def assemble(ctx, modules):
+    log(f'tree {ctx.tree}')
+    tree = ctx.tree
+    tree.mkdir(parents=True, exist_ok=True)
+    for folder in ('drivers', 'kits', 'patches', 'refer'):
+        _clear_links(tree / folder)
+        (tree / folder).mkdir(exist_ok=True)
+    for name in ('build', '.cache'):
+        (tree / name).mkdir(exist_ok=True)
+    kit = ctx.board.get('board', {}).get('kit', ctx.name)
+    for (kind, name), module in sorted(modules.items()):
+        if module.path is None:
+            if kind == 'refer':
+                continue   # fetched when a step needs it
+            raise Error(f'{kind} {name} has no source yet; run "unixnu fetch"')
+        if kind == 'package':
+            if name == 'main':
+                _link(module.path, tree / 'package')
+            else:
+                _link(module.path, tree / 'kits' / kit)
+        else:
+            _link(module.path, tree / {'driver': 'drivers', 'kit': 'kits', 'patch': 'patches', 'refer': 'refer'}[kind] / name)
+    if not (tree / 'package').is_symlink():
+        raise Error('no package module: this repository or --local-package must provide package/')
+    (tree / 'scripts').mkdir(exist_ok=True)
+    for base in (ctx.root, ctx.device):
+        if (base / 'scripts').is_dir():
+            _copy_scripts(base / 'scripts', tree / 'scripts')
+    return tree

scripts/unixnu/util.py

@@ -0,0 +1,67 @@
+"""Small helpers shared by the build driver."""
+import hashlib
+import os
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+
+
+class Error(Exception):
+    """A user-facing failure; the command line prints it without a traceback."""
+
+
+def log(message):
+    print(f'== {message}', flush=True)
+
+
+def expand(path):
+    return Path(os.path.expandvars(os.path.expanduser(str(path)))).resolve()
+
+
+def sha256_file(path):
+    digest = hashlib.sha256()
+    with Path(path).open('rb') as stream:
+        for chunk in iter(lambda: stream.read(1 << 20), b''):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+EXTRA_BIN = ('/opt/homebrew/opt/llvm/bin', '/opt/homebrew/opt/lld/bin', '/usr/local/opt/llvm/bin', '/usr/local/opt/lld/bin')
+
+
+def tool(name, hint=''):
+    """Find a tool on PATH, then in the usual Homebrew LLVM locations (keg-only, not on PATH)."""
+    found = shutil.which(name) or shutil.which(name, path=os.pathsep.join(EXTRA_BIN))
+    if not found:
+        raise Error(f'{name} not found on PATH' + (f' ({hint})' if hint else ''))
+    return found
+
+
+def run(command, *, cwd=None, env=None, log_file=None, capture=False):
+    """Run a command, echoing it. With log_file the output goes there and the tail is shown on failure."""
+    command = [str(part) for part in command]
+    if not capture:
+        print('+ ' + ' '.join(command), flush=True)
+    if capture:
+        result = subprocess.run(command, cwd=cwd, env=env, text=True, capture_output=True)
+        if result.returncode:
+            raise Error(f'{command[0]} failed ({result.returncode}): {result.stderr.strip() or result.stdout.strip()}')
+        return result.stdout
+    if log_file:
+        Path(log_file).parent.mkdir(parents=True, exist_ok=True)
+        with Path(log_file).open('w') as stream:
+            result = subprocess.run(command, cwd=cwd, env=env, stdout=stream, stderr=subprocess.STDOUT)
+        if result.returncode:
+            tail = Path(log_file).read_text(errors='replace').splitlines()[-30:]
+            print('\n'.join(tail), file=sys.stderr)
+            raise Error(f'{command[0]} failed ({result.returncode}); full log: {log_file}')
+        return ''
+    result = subprocess.run(command, cwd=cwd, env=env)
+    if result.returncode:
+        raise Error(f'{command[0]} failed ({result.returncode})')
+    return ''
+
+
+def git(repo, *args, capture=True):
+    return run(['git', '-C', repo, *args], capture=capture)

unixnu

@@ -0,0 +1,8 @@
+#!/usr/bin/env python3
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent / 'scripts'))
+from unixnu.cli import main
+
+main()