diff --git a/.gitignore b/.gitignore index c361bc4..564dc14 100644 --- a/.gitignore +++ b/.gitignore @@ -42,3 +42,4 @@ __pycache__/ *.pem *.key .env +local.toml diff --git a/feeds.toml b/feeds.toml new file mode 100644 index 0000000..b88d68a --- /dev/null +++ b/feeds.toml @@ -0,0 +1,44 @@ +# External sources of UniXNU. "commit" pins a source; "path" (optional) fetches only that +# subdirectory with a shallow, blobless, sparse checkout. A device repository may add its own +# feeds.toml, and a local = "DIR" entry (or the unixnu command line) points a source at a local tree. +# These are Apple open source references and edk2; none of them is copied into a UniXNU repository. + +[[source]] +name = "xnu" +kind = "refer" +git = "https://github.com/apple-oss-distributions/xnu.git" +ref = "xnu-10063.141.1" +commit = "d8b80295118ef25ac3a784134bcf95cd8e88109f" + +[[source]] +name = "libplatform" +kind = "refer" +git = "https://github.com/apple-oss-distributions/libplatform.git" +ref = "libplatform-316.100.10" +commit = "a00a4cc36da2110578bcf3b8eeeeb93dcc7f4e11" + +[[source]] +name = "libdispatch" +kind = "refer" +git = "https://github.com/apple-oss-distributions/libdispatch.git" +ref = "libdispatch-1477.100.9" +commit = "30fe6a7d945340ce69cef121132f049e810089a5" + +[[source]] +name = "AvailabilityVersions" +kind = "refer" +git = "https://github.com/apple-oss-distributions/AvailabilityVersions.git" +ref = "AvailabilityVersions-141.1" +commit = "933f9360f1b70dfbda5548386a3d65e0358b4f7d" + +[[source]] +name = "edk2" +kind = "refer" +git = "https://github.com/microsoft/mu_basecore.git" +ref = "release/202608" +commit = "cfb93724a06ff86ff5ba971413368a7809ee42a5" +submodules = [ + "MdePkg/Library/BaseFdtLib/libfdt", + "MdePkg/Library/MipiSysTLib/mipisyst", + "BaseTools/Source/C/BrotliCompress/brotli", +] diff --git a/scripts/check-unixnu.py b/scripts/check-unixnu.py new file mode 100644 index 0000000..ae92fa0 --- /dev/null +++ b/scripts/check-unixnu.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +"""Offline self-test of the unixnu build driver: defconfig parsing and module precedence.""" +import argparse +import sys +import tempfile +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from unixnu import cli, feeds +from unixnu.config import Context, parse_config +from unixnu.util import Error + + +def expect(condition, message): + if not condition: + raise SystemExit(f'FAIL: {message}') + + +def main(): + config = parse_config('# c\nCONFIG_A=y\nCONFIG_B=n\n# CONFIG_C is not set\nCONFIG_D="x y"\nCONFIG_E=0x10\n') + expect(config == {'A': True, 'B': False, 'C': False, 'D': 'x y', 'E': 16}, 'defconfig values') + try: + parse_config('CONFIG_A=maybe\n') + expect(False, 'bad defconfig value accepted') + except Error: + pass + with tempfile.TemporaryDirectory() as name: + root = Path(name) + for path in ('dev/drivers/Both', 'dev/drivers/DevOnly', 'loc/drivers/Both', 'loc/drivers/LocOnly', 'kd/Both', 'one/Both'): + (root / path).mkdir(parents=True) + (root / 'dev/board.toml').write_text('[board]\nname = "t"\n') + parsed = cli.parser().parse_args(['status', '-D', str(root / 'dev'), '--out', str(root / 'out')]) + ctx = Context(parsed) + found = feeds.resolve(ctx) + expect(found[('driver', 'Both')].origin == 'device', 'device beats base') + expect(('driver', 'ArmCpu') in found and found[('driver', 'ArmCpu')].origin == 'base', 'base modules found') + parsed = cli.parser().parse_args(['status', '-D', str(root / 'dev'), '--out', str(root / 'out'), '--local', str(root / 'loc')]) + found = feeds.resolve(Context(parsed)) + expect(found[('driver', 'Both')].origin == 'local (--local)' and ('driver', 'LocOnly') in found, '--local beats device') + parsed = cli.parser().parse_args(['status', '-D', str(root / 'dev'), '--out', str(root / 'out'), '--local', str(root / 'loc'), + '--local-drivers', str(root / 'kd')]) + found = feeds.resolve(Context(parsed)) + expect(found[('driver', 'Both')].origin == 'local (--local-drivers)', '--local-drivers beats --local') + parsed = cli.parser().parse_args(['status', '-D', str(root / 'dev'), '--out', str(root / 'out'), '--local', str(root / 'loc'), + '--local-drivers', str(root / 'kd'), '--local-module', f'Both={root / "one"}']) + found = feeds.resolve(Context(parsed)) + expect(found[('driver', 'Both')].origin == 'local (--local-module)', '--local-module wins') + try: + feeds.resolve(Context(cli.parser().parse_args(['status', '-D', str(root / 'dev'), '--out', str(root / 'out'), + '--local', str(root / 'missing')]))) + expect(False, 'missing --local directory accepted') + except Error: + pass + print('PASS: unixnu defconfig parsing and module precedence') + + +if __name__ == '__main__': + main() diff --git a/scripts/unixnu/__init__.py b/scripts/unixnu/__init__.py new file mode 100644 index 0000000..5d24fac --- /dev/null +++ b/scripts/unixnu/__init__.py @@ -0,0 +1 @@ +"""UniXNU build driver: sources, build tree, kernel, package, firmware and boot image.""" diff --git a/scripts/unixnu/bootimg.py b/scripts/unixnu/bootimg.py new file mode 100644 index 0000000..d2aa766 --- /dev/null +++ b/scripts/unixnu/bootimg.py @@ -0,0 +1,133 @@ +"""Android boot image: gzip(BootShim + UEFI FD) with the device tree appended, plus an optional ramdisk. + +Inputs are a Mu-Silicium checkout (BootShim sources, Resources/Configs/.toml, mkbootimg.py), +the built firmware volume and a compiled device tree. The image is meant for "fastboot boot" only. +The result is verified by unpacking it again; the output is deterministic. +""" +import gzip +import hashlib +import json +import shutil +import struct +import tempfile +import zlib +from pathlib import Path + +from .config import load_toml +from .util import Error, log, run, sha256_file, tool + +DEFAULTS = {'header_version': 0, 'base': 0, 'kernel_offset': 0x8000, 'tags_offset': 0x1e00000, 'pagesize': 4096} +MKBOOTIMG_FLAGS = ('base', 'kernel_offset', 'tags_offset', 'pagesize', 'header_version', 'ramdisk_offset', 'second_offset', 'cmdline') + + +def compile_shim(uefi, base, size, requires_header, scratch): + clang, objcopy, nm = tool('clang'), tool('llvm-objcopy', 'brew install llvm'), tool('llvm-nm', 'brew install llvm') + source = uefi / 'BootShim/BootShim.S' + if not source.is_file(): + raise Error(f'{source} missing: --uefi must be a Mu-Silicium checkout') + obj = scratch / 'shim.o' + run([clang, '--target=aarch64-none-elf', '-c', f'-DFD_BASE={base}', f'-DFD_SIZE={size}', + f'-DREQUIRES_KERNEL_HEADER={int(requires_header)}', source, '-o', obj], capture=True) + run([objcopy, '-O', 'binary', obj, scratch / 'shim.bin'], capture=True) + shim = (scratch / 'shim.bin').read_bytes() + symbols = run([nm, '-n', obj], capture=True) + offset = next((int(line.split()[0], 16) for line in symbols.splitlines() if line.endswith(' _Payload')), None) + if offset != len(shim): + raise Error('a literal pool or padding separates the _Payload label from the appended firmware volume') + if requires_header and (shim[56:60] != b'ARM\x64' or struct.unpack_from('I', tree, 4)[0] != len(tree): + raise Error('invalid compiled device tree') + combined = shim + firmware + kernel = gzip.compress(combined, mtime=0) + tree + kernel_path = scratch / 'Image.gz-dtb' + kernel_path.write_bytes(kernel) + if ramdisk: + ramdisk_path = Path(ramdisk) + if not ramdisk_path.is_file(): + raise Error(f'ramdisk not found: {ramdisk_path}') + else: + ramdisk_path = scratch / 'ramdisk' + ramdisk_path.write_bytes(b'') + command = ['python3', mkbootimg, '--kernel', kernel_path, '--ramdisk', ramdisk_path, '-o', out] + for key in MKBOOTIMG_FLAGS: + if key in image: + command += [f'--{key}', str(image[key])] + run(command, capture=True) + report = verify(out, combined, tree, ramdisk_path, image) + report.update(image_sha256=sha256_file(out), image_bytes=out.stat().st_size, fd_sha256=sha256_file(fd), + dtb_sha256=sha256_file(dtb), shim_bytes=len(shim), hardware_tested=False) + (out.parent / 'boot.json').write_text(json.dumps(report, indent=2) + '\n') + log(f'bootimg: {out} ({out.stat().st_size} bytes)') + return out + + +def verify(path, combined, tree, ramdisk_path, image): + """Unpack the image and compare every part with its input.""" + data = Path(path).read_bytes() + kernel_size, kernel_address, ramdisk_size, _, second_size, _, tags, page, version, _ = struct.unpack_from('<10I', data, 8) + if data[:8] != b'ANDROID!' or second_size or version != image['header_version'] or page != image['pagesize']: + raise Error('boot image header does not match the requested parameters') + if kernel_address != image['base'] + image['kernel_offset'] or tags != image['base'] + image['tags_offset']: + raise Error('boot image load addresses do not match the requested offsets') + kernel = data[page:page + kernel_size] + unpacker = zlib.decompressobj(16 + zlib.MAX_WBITS) + if unpacker.decompress(kernel) != combined or not unpacker.eof or unpacker.unused_data != tree: + raise Error('gzip, firmware and appended device tree do not round-trip') + start = page * (1 + -(-kernel_size // page)) + if data[start:start + ramdisk_size] != Path(ramdisk_path).read_bytes(): + raise Error('ramdisk bytes differ') + return {'kernel_sha256': hashlib.sha256(kernel).hexdigest(), 'ramdisk_bytes': ramdisk_size, + 'header_version': version, 'kernel_address': hex(kernel_address), 'tags_address': hex(tags)} diff --git a/scripts/unixnu/cli.py b/scripts/unixnu/cli.py new file mode 100644 index 0000000..9b4543d --- /dev/null +++ b/scripts/unixnu/cli.py @@ -0,0 +1,165 @@ +"""Command line of the UniXNU build driver.""" +import argparse +import platform +import shutil +import sys + +from . import bootimg, feeds, firmware, kernel, package, tree +from .config import LOCAL_KINDS, Context, parse_config +from .util import Error, expand, git, log + +STEPS = ('kernel', 'package', 'firmware') + + +def common(): + p = argparse.ArgumentParser(add_help=False) + p.add_argument('-D', '--device', metavar='DIR', help='device repository (remembered; or UNIXNU_DEVICE)') + p.add_argument('--out', metavar='DIR', help='output root (default: out/ here, or UNIXNU_OUT)') + p.add_argument('-j', '--jobs', type=int, help='parallel jobs') + p.add_argument('--local', metavar='DIR', help='directory laid out like this repository; its drivers, kits, patches, scripts and package are used') + for kind in LOCAL_KINDS: + p.add_argument(f'--local-{kind}', metavar='DIR', help=f'use DIR as the {kind} directory (beats --local)') + p.add_argument('--local-module', action='append', metavar='[KIND:]NAME=PATH', help='use PATH for one module (beats everything)') + p.add_argument('--uefi', metavar='DIR', help='Mu-Silicium checkout for the device (uefi-sheng); or UNIXNU_UEFI') + p.add_argument('--edk2', metavar='DIR', help='Mu_Basecore checkout; or UNIXNU_EDK2') + p.add_argument('--kernelcache', metavar='FILE', help='original decompressed kernelcache; or UNIXNU_KERNELCACHE') + p.add_argument('--target', default='DEBUG', choices=('DEBUG', 'RELEASE'), help='EDK II build target') + return p + + +def parser(): + top = argparse.ArgumentParser(prog='unixnu', description='Build UniXNU, the XNU kernel and the boot image for a device repository.') + sub = top.add_subparsers(dest='command', required=True) + c = common() + d = sub.add_parser('defconfig', parents=[c], help='select configs/NAME in the device repository') + d.add_argument('name', nargs='?', help='defconfig name; omit to list') + f = sub.add_parser('fetch', parents=[c], help='shallow-fetch git sources and write unixnu.lock') + f.add_argument('names', nargs='*', help='sources to fetch (default: all)') + f.add_argument('--update', action='store_true', help='ignore the lock and move to the newest ref') + sub.add_parser('tree', parents=[c], help='assemble the build tree') + b = sub.add_parser('build', parents=[c], help='build kernel, package and firmware') + b.add_argument('steps', nargs='*', choices=STEPS + ('all', 'prelink'), default=['all']) + i = sub.add_parser('bootimg', parents=[c], help='pack the boot image from the firmware volume and a device tree') + i.add_argument('--fd', metavar='FILE', help='firmware volume (default: the build output under --uefi)') + i.add_argument('--dtb', metavar='FILE', help='compiled device tree; or UNIXNU_DTB') + i.add_argument('--ramdisk', metavar='FILE', help='ramdisk / payload image (default: empty)') + i.add_argument('-o', '--output', metavar='FILE', help='output (default: out//boot/boot.img)') + sub.add_parser('status', parents=[c], help='show the device, configuration and where each module comes from') + sub.add_parser('clean', parents=[c], help='remove the build output of the device (keeps fetched sources)') + return top + + +def need_uefi(ctx): + uefi = ctx.option('uefi', 'UNIXNU_UEFI') + if not uefi: + raise Error('the firmware needs a Mu-Silicium checkout: --uefi DIR (or UNIXNU_UEFI)') + if not (uefi / 'Resources/Scripts/mkbootimg.py').is_file(): + raise Error(f'{uefi} is not a Mu-Silicium checkout') + return uefi + + +REFERENCES = {'kernel': ('xnu', 'libplatform', 'libdispatch', 'AvailabilityVersions'), 'package': ('edk2',), 'firmware': ('edk2',)} + + +def prepare(ctx, steps=()): + """Resolve modules, fetch the git ones (references only for the steps that need them) and assemble the tree.""" + modules = feeds.resolve(ctx) + wanted = {name for step in steps for name in REFERENCES.get(step, ())} + feeds.fetch(ctx, modules, only={m.name for m in modules.values() if m.origin == 'git' and (m.kind != 'refer' or m.name in wanted)}) + tree.assemble(ctx, modules) + return modules + + +def cmd_defconfig(ctx): + configs = ctx.device / 'configs' + names = sorted(p.name for p in configs.glob('*_defconfig')) if configs.is_dir() else [] + if not ctx.args.name: + print('\n'.join(names) or 'no defconfigs') + return + source = configs / ctx.args.name + if not source.is_file(): + raise Error(f'{source} not found; available: {", ".join(names) or "none"}') + parse_config(source.read_text()) + ctx.work.mkdir(parents=True, exist_ok=True) + shutil.copyfile(source, ctx.config_path) + log(f'configuration {ctx.args.name} -> {ctx.config_path}') + + +def cmd_fetch(ctx): + modules = feeds.resolve(ctx) + commits = feeds.fetch(ctx, modules, update=ctx.args.update, only=set(ctx.args.names) or None) + if commits: + feeds.lock_write(ctx, {**feeds.lock_read(ctx), **commits}) + log(f'wrote {ctx.lock_path}') + + +def cmd_build(ctx): + steps = ctx.args.steps + if 'all' in steps: + steps = list(STEPS) + ctx.need_config() + if 'firmware' in steps: + need_uefi(ctx) # fail before fetching anything + modules = prepare(ctx, steps + (['kernel'] if 'prelink' in steps else [])) + if 'kernel' in steps: + kernel.build(ctx) + kernel.prelink(ctx) + elif 'prelink' in steps: + kernel.prelink(ctx) + if 'package' in steps: + package.build(ctx, modules, ctx.args.target) + if 'firmware' in steps: + firmware.build(ctx, modules, need_uefi(ctx), package.edk2_dir(ctx, modules), ctx.args.target) + + +def cmd_bootimg(ctx): + uefi = need_uefi(ctx) + fd = expand(ctx.args.fd) if ctx.args.fd else firmware.fd_path(ctx, uefi, ctx.args.target) + dtb = ctx.option('dtb', 'UNIXNU_DTB') + ramdisk = expand(ctx.args.ramdisk) if ctx.args.ramdisk else None + bootimg.build(ctx, uefi, fd, dtb, ramdisk, expand(ctx.args.output) if ctx.args.output else None) + + +def cmd_status(ctx): + print(f'device {ctx.device} ({ctx.name})') + print(f'output {ctx.work}') + print(f'config {ctx.config_path if ctx.config_path.is_file() else "none (run defconfig)"}') + modules = feeds.resolve(ctx) + locked = feeds.lock_read(ctx) + unpinned = [] + for (kind, name), module in sorted(modules.items()): + extra = '' + if module.origin == 'git': + pinned = locked.get(name) or module.source.get('commit') + extra = f' pinned {pinned[:12]}' if pinned else ' NOT LOCKED' + elif module.local: + try: + extra = ' (dirty)' if git(module.path, 'status', '--porcelain', '--', '.').strip() else ' (clean)' + except Error: + extra = ' (not a git work tree)' + unpinned.append(f'{kind}:{name}') + print(f'{kind:8} {name:24} {module.origin}{extra}') + if unpinned: + print(f'\nnot reproducible: {len(unpinned)} local source(s) are not pinned by unixnu.lock') + + +def cmd_clean(ctx): + for name in ('tree', 'logs', 'kernel', 'package', 'boot', 'edk2-conf'): + shutil.rmtree(ctx.work / name, ignore_errors=True) + log(f'cleaned {ctx.work}') + + +def main(argv=None): + if sys.version_info < (3, 11): + sys.exit('unixnu needs Python 3.11 or newer') + if platform.system() != 'Darwin': + sys.exit('unixnu supports macOS hosts only') + args = parser().parse_args(argv) + try: + ctx = Context(args) + {'defconfig': cmd_defconfig, 'fetch': cmd_fetch, 'tree': lambda c: prepare(c), 'build': cmd_build, + 'bootimg': cmd_bootimg, 'status': cmd_status, 'clean': cmd_clean}[args.command](ctx) + except Error as error: + sys.exit(f'unixnu: {error}') + except KeyboardInterrupt: + sys.exit(130) diff --git a/scripts/unixnu/config.py b/scripts/unixnu/config.py new file mode 100644 index 0000000..a52fed4 --- /dev/null +++ b/scripts/unixnu/config.py @@ -0,0 +1,107 @@ +"""Board, feeds and defconfig files, and the context every build step shares. + +A device repository supplies: + board.toml board description (like Android BoardConfig.mk) + configs/_defconfig CONFIG_* switches (like an OpenWrt defconfig) + feeds.toml optional extra sources + drivers/ kits/ patches/ scripts/ package/ its own modules, same layout as this repository +""" +import json +import os +import re +import tomllib +from pathlib import Path + +from .util import Error, expand + +ROOT = Path(__file__).resolve().parents[2] +KINDS = {'driver': 'drivers', 'kit': 'kits', 'patch': 'patches', 'script': 'scripts', 'package': 'package', 'refer': 'refer'} +LOCAL_KINDS = ('drivers', 'kits', 'patches', 'scripts', 'package') + + +def load_toml(path): + path = Path(path) + try: + return tomllib.loads(path.read_text()) + except FileNotFoundError: + raise Error(f'missing file: {path}') from None + except tomllib.TOMLDecodeError as error: + raise Error(f'{path}: {error}') from None + + +def parse_config(text): + """Parse a defconfig: CONFIG_X=y, =n, ="text", =123 and "# CONFIG_X is not set".""" + values = {} + for number, line in enumerate(text.splitlines(), 1): + line = line.strip() + unset = re.fullmatch(r'#\s*CONFIG_(\w+) is not set', line) + if unset: + values[unset.group(1)] = False + continue + if not line or line.startswith('#'): + continue + match = re.fullmatch(r'CONFIG_(\w+)=(.*)', line) + if not match: + raise Error(f'defconfig line {number} is not CONFIG_NAME=value: {line}') + name, raw = match.groups() + if raw == 'y': + values[name] = True + elif raw == 'n': + values[name] = False + elif raw.startswith('"') and raw.endswith('"') and len(raw) >= 2: + values[name] = raw[1:-1] + elif re.fullmatch(r'-?(0x[0-9a-fA-F]+|\d+)', raw): + values[name] = int(raw, 0) + else: + raise Error(f'defconfig line {number}: value must be y, n, a number or a quoted string: {line}') + return values + + +class Context: + """Everything the steps need: repositories, options, board, config and the output layout.""" + + def __init__(self, args): + self.root = ROOT + self.args = args + self.out_root = expand(args.out or os.environ.get('UNIXNU_OUT') or ROOT / 'out') + self.state_path = self.out_root / 'state.json' + state = self._load_state() + device = args.device or os.environ.get('UNIXNU_DEVICE') or state.get('device') + if not device: + raise Error('no device repository: pass -D DIR (or set UNIXNU_DEVICE); it is remembered afterwards') + self.device = expand(device) + if not (self.device / 'board.toml').is_file(): + raise Error(f'{self.device} has no board.toml') + self.board = load_toml(self.device / 'board.toml') + self.name = self.board.get('board', {}).get('name') + if not self.name: + raise Error('board.toml needs [board] name') + self.work = self.out_root / self.name + self.tree = self.work / 'tree' + self.logs = self.work / 'logs' + self.feeds_dir = self.out_root / 'feeds' + self.config_path = self.work / '.config' + self.jobs = args.jobs or int(os.environ.get('JOBS', 0)) or os.cpu_count() or 2 + self._save_state({'device': str(self.device)}) + self.config = parse_config(self.config_path.read_text()) if self.config_path.is_file() else {} + self.lock_path = self.device / 'unixnu.lock' + + def _load_state(self): + try: + return json.loads(self.state_path.read_text()) + except (FileNotFoundError, ValueError): + return {} + + def _save_state(self, state): + self.out_root.mkdir(parents=True, exist_ok=True) + self.state_path.write_text(json.dumps(state, indent=2) + '\n') + + def need_config(self): + if not self.config_path.is_file(): + raise Error(f'no configuration: run "unixnu defconfig NAME" first (see {self.device / "configs"})') + return self.config + + def option(self, flag, env=None): + """A path given as a flag or, failing that, an environment variable.""" + value = getattr(self.args, flag, None) or (os.environ.get(env) if env else None) + return expand(value) if value else None diff --git a/scripts/unixnu/feeds.py b/scripts/unixnu/feeds.py new file mode 100644 index 0000000..a2d2869 --- /dev/null +++ b/scripts/unixnu/feeds.py @@ -0,0 +1,215 @@ +"""Module sources: local directories, in-tree modules and shallow, sparse git checkouts. + +A module is a driver, kit, patch set or the package, identified by (kind, name). Candidates are +taken in this order, first one that exists wins: + + 1. --local-module NAME=PATH + 2. --local- DIR (DIR/NAME) + 3. --local DIR (DIR//NAME, same layout as this repository) + 4. a "local" entry in feeds.toml or the ignored local.toml of the device repository + 5. the device repository's own directories + 6. this repository's own directories + 7. a git source from feeds.toml: shallow, blobless, sparse checkout of the module path + +Git sources never use submodules. Nothing fetched or linked is copied into either repository. +""" +import hashlib +from pathlib import Path + +from .config import KINDS, LOCAL_KINDS, load_toml +from .util import Error, expand, git, log, run + + +class Module: + def __init__(self, kind, name, path=None, origin='', source=None): + self.kind, self.name, self.path, self.origin, self.source = kind, name, path, origin, source + self.commit = None + + @property + def local(self): + return self.origin.startswith('local') + + def __repr__(self): + return f'{self.kind}:{self.name} <{self.origin}> {self.path}' + + +def read_sources(ctx): + """Entries of every feeds.toml (this repository, then the device repository) and the device local.toml.""" + sources, local = {}, {} + for base in (ctx.root, ctx.device): + path = base / 'feeds.toml' + if path.is_file(): + for entry in load_toml(path).get('source', []): + kind = entry.get('kind') + if kind not in KINDS or not entry.get('name'): + raise Error(f'{path}: every [[source]] needs name and kind in {sorted(KINDS)}') + sources[(kind, entry['name'])] = entry + path = ctx.device / 'local.toml' + if path.is_file(): + local = load_toml(path) + return sources, local + + +def lock_read(ctx): + if not ctx.lock_path.is_file(): + return {} + return {name: values['commit'] for name, values in load_toml(ctx.lock_path).get('source', {}).items()} + + +def lock_write(ctx, commits): + lines = ['# Resolved git sources, written by "unixnu fetch". Local sources are never recorded.'] + for name in sorted(commits): + lines += ['', f'[source."{name}"]', f'commit = "{commits[name]}"'] + ctx.lock_path.write_text('\n'.join(lines) + '\n') + + +def _children(directory): + directory = Path(directory) + return sorted(p.name for p in directory.iterdir() if p.is_dir() and not p.name.startswith('.')) if directory.is_dir() else [] + + +def _cli_locals(args): + """(per-module overrides, per-kind directories, local root) from the command line.""" + modules = {} + for item in args.local_module or []: + if '=' not in item: + raise Error(f'--local-module needs NAME=PATH (or KIND:NAME=PATH): {item}') + key, path = item.split('=', 1) + kind, _, name = key.partition(':') if ':' in key else ('', '', key) + modules[(kind, name)] = expand(path) + kinds = {} + for kind in LOCAL_KINDS: + value = getattr(args, 'local_' + kind, None) + if value: + kinds[kind] = expand(value) + return modules, kinds, expand(args.local) if args.local else None + + +def _need(path, what): + if not Path(path).is_dir(): + raise Error(f'{what}: {path} is not a directory') + return Path(path) + + +def resolve(ctx): + """Map (kind, name) to a Module. Git modules are resolved to a clone path later by fetch().""" + sources, local_file = read_sources(ctx) + cli_modules, cli_kinds, cli_root = _cli_locals(ctx.args) + if cli_root is not None: + _need(cli_root, '--local') + for (kind, name) in cli_modules: + if kind and kind not in KINDS: + raise Error(f'--local-module: unknown kind {kind}') + result = {} + + def add(kind, name, path, origin, source=None): + result.setdefault((kind, name), Module(kind, name, path, origin, source)) + + plural = {kind: KINDS[kind] for kind in KINDS} + # 1: --local-module + for (kind, name), path in cli_modules.items(): + _need(path, f'--local-module {name}') + targets = [kind] if kind else [k for k in KINDS if (k, name) in sources or any( + (base / plural[k] / name).is_dir() for base in (ctx.device, ctx.root))] + if not targets: + raise Error(f'--local-module {name}: unknown module; use KIND:NAME=PATH to add a new one') + if len(targets) > 1: + raise Error(f'--local-module {name} is ambiguous ({", ".join(targets)}); use KIND:NAME=PATH') + add(targets[0], name, path, 'local (--local-module)') + # 2 and 3: per-kind directories and the local root + for kind in KINDS: + directory = cli_kinds.get(plural[kind]) + origin = f'local (--local-{plural[kind]})' + if directory is None and cli_root is not None and (cli_root / plural[kind]).is_dir(): + directory, origin = cli_root / plural[kind], 'local (--local)' + if directory is None or kind in ('refer', 'script'): + continue + if kind == 'package': + add('package', 'main', _need(directory, '--local-package'), origin) + continue + for name in _children(_need(directory, f'--local-{plural[kind]}')): + add(kind, name, directory / name, origin) + # 4: local entries of the configuration files + for (kind, name), entry in sources.items(): + if entry.get('local'): + add(kind, name, _need(expand(entry['local']), f'source {name}'), 'local (feeds)') + for kind, directory in local_file.get('local', {}).items(): + if kind in LOCAL_KINDS and kind not in cli_kinds and kind != 'package': + for name in _children(_need(expand(directory), f'local.toml {kind}')): + add({v: k for k, v in KINDS.items()}[kind], name, expand(directory) / name, 'local (local.toml)') + # 5 and 6: the device repository, then this repository + for base, label in ((ctx.device, 'device'), (ctx.root, 'base')): + for kind, folder in KINDS.items(): + if kind in ('refer', 'script'): + continue + if kind == 'package': + if (base / 'package').is_dir(): + add('package', 'main' if base == ctx.root else 'device', base / 'package', label) + continue + for name in _children(base / folder): + add(kind, name, base / folder / name, label) + # 7: git sources + for (kind, name), entry in sources.items(): + if entry.get('git'): + add(kind, name, None, 'git', entry) + return result + + +def _clone_dir(ctx, entry): + key = hashlib.sha1(f'{entry["git"]}\n{entry.get("ref", "")}'.encode()).hexdigest()[:8] + return ctx.feeds_dir / f'{Path(entry["git"]).stem}-{key}' + + +def fetch(ctx, modules, update=False, only=None): + """Shallow, blobless, sparse checkout of git modules (all, or the named ones); returns {source name: commit}. + + A clone already at the wanted commit is not contacted again.""" + locked = {} if update else lock_read(ctx) + groups = {} + for module in modules.values(): + if module.origin == 'git' and (only is None or module.name in only): + groups.setdefault(_clone_dir(ctx, module.source), []).append(module) + commits = {} + for directory, members in groups.items(): + entry = members[0].source + paths = sorted({m.source['path'] for m in members if m.source.get('path')}) + sparse = bool(paths) and all(m.source.get('path') for m in members) + target = next((locked[m.name] for m in members if m.name in locked), None) or entry.get('commit') or entry.get('ref') or 'HEAD' + current = None + if (directory / '.git').is_dir(): + try: + current = git(directory, 'rev-parse', 'HEAD').strip() + except Error: + current = None + if current and current == target and (not sparse or all((directory / path).exists() for path in paths)): + _attach(members, directory, current, commits) + continue + log(f'fetch {entry["git"]} {target[:12]}' + (f' ({", ".join(paths)})' if sparse else '')) + if not (directory / '.git').is_dir(): + directory.mkdir(parents=True, exist_ok=True) + git(directory, 'init', '-q') + git(directory, 'remote', 'add', 'origin', entry['git']) + if sparse: + git(directory, 'sparse-checkout', 'set', '--cone', *paths) + git(directory, 'config', 'remote.origin.promisor', 'true') + git(directory, 'config', 'remote.origin.partialclonefilter', 'blob:none') + run(['git', '-C', directory, 'fetch', '-q', '--depth', '1', '--filter=blob:none', 'origin', target]) + else: + git(directory, 'sparse-checkout', 'disable') + run(['git', '-C', directory, 'fetch', '-q', '--depth', '1', 'origin', target]) + git(directory, 'checkout', '-q', '--detach', 'FETCH_HEAD') + if entry.get('submodules'): + # Only for upstream trees that need them (edk2); UniXNU repositories never use submodules. + run(['git', '-C', directory, 'submodule', 'update', '--init', '--depth', '1', '--', *entry['submodules']]) + _attach(members, directory, git(directory, 'rev-parse', 'HEAD').strip(), commits) + return commits + + +def _attach(members, directory, commit, commits): + for module in members: + subpath = module.source.get('path', '') + module.path = directory / subpath if subpath else directory + if not module.path.exists(): + raise Error(f'{module.source["git"]} at {commit[:12]} has no {subpath}') + module.commit = commit + commits[module.name] = commit diff --git a/scripts/unixnu/firmware.py b/scripts/unixnu/firmware.py new file mode 100644 index 0000000..299bcce --- /dev/null +++ b/scripts/unixnu/firmware.py @@ -0,0 +1,66 @@ +"""Device UEFI firmware: the EDK II build of a Mu-Silicium checkout (uefi-sheng), driven by board.toml [firmware].""" +import os +import shutil +from pathlib import Path + +from .config import load_toml +from .util import Error, git, log, run, tool + + +def uefi_config(ctx, uefi): + fw = ctx.board.get('firmware') + if not fw: + raise Error('board.toml has no [firmware]') + return fw, load_toml(uefi / 'Resources/Configs' / f'{fw["config"]}.toml') + + +def fd_path(ctx, uefi, target='DEBUG'): + fw = ctx.board['firmware'] + return uefi / fw['fd'].format(target=target) + + +def build(ctx, modules, uefi, edk2, target='DEBUG'): + log(f'firmware: {uefi.name}') + fw, config = uefi_config(ctx, uefi) + for name in ('clang', 'python3', 'make'): + tool(name) + base, size = config['uefi_fd']['base'], config['uefi_fd']['size'] + for script in fw.get('preflight', []): + run(['python3', ctx.tree / script], cwd=ctx.tree, log_file=ctx.logs / 'firmware-preflight.log') + defines = {'FD_BASE': hex(base), 'FD_SIZE': hex(size), 'FD_BLOCKS': str(size // 0x1000)} + defines.update({key: str(value) for key, value in fw.get('defines', {}).items()}) + fields = {'uefi': str(uefi), 'edk2': str(edk2), 'tree': str(ctx.tree)} + packages = ':'.join(item.format(**fields) for item in fw['packages']) + tools = edk2 / 'BaseTools' + if not (tools / 'Source/C/bin/GenFw').is_file(): + log('firmware: building BaseTools') + run(['make', '-C', tools / 'Source/C', f'-j{ctx.jobs}'], log_file=ctx.logs / 'basetools.log') + conf = ctx.work / 'edk2-conf' + conf.mkdir(parents=True, exist_ok=True) + for name in ('target', 'tools_def', 'build_rule'): + if not (conf / f'{name}.txt').is_file(): + shutil.copy(tools / 'Conf' / f'{name}.template', conf / f'{name}.txt') + env = dict(os.environ, WORKSPACE=str(uefi), EDK_TOOLS_PATH=str(tools), CONF_PATH=str(conf), PACKAGES_PATH=packages, + PYTHON_COMMAND='python3', + PATH=os.pathsep.join([str(tools / 'BinWrappers/PosixLike'), str(tools / 'Source/C/bin'), os.environ['PATH']])) + env.setdefault('CLANG_BIN', os.path.dirname(shutil.which('clang')) + '/') + applied = [] + try: + # Board patches for the shared edk2 checkout: applied for the build, always reverted. + for patch in fw.get('patches', []): + path = uefi / patch + run(['git', '-C', edk2, 'apply', '--check', path]) + run(['git', '-C', edk2, 'apply', path]) + applied.append(path) + command = ['build', '-p', fw['dsc'], '-a', 'AARCH64', '-b', target, '-t', 'CLANGPDB', '-n', str(ctx.jobs)] + for key, value in defines.items(): + command += ['-D', f'{key}={value}'] + run(command, cwd=uefi, env=env, log_file=ctx.logs / 'firmware.log') + finally: + for path in reversed(applied): + git(edk2, 'apply', '--reverse', str(path)) + fd = fd_path(ctx, uefi, target) + if not fd.is_file() or fd.stat().st_size != size: + raise Error(f'{fd} missing or not {size:#x} bytes') + log(f'firmware: {fd}') + return fd diff --git a/scripts/unixnu/kernel.py b/scripts/unixnu/kernel.py new file mode 100644 index 0000000..1bfc8d1 --- /dev/null +++ b/scripts/unixnu/kernel.py @@ -0,0 +1,59 @@ +"""XNU kernel: build from the patched Apple source, then prelink against the original kernelcache.""" +import os +from pathlib import Path + +from .util import Error, expand, log, run, sha256_file, tool + + +def xnu_env(ctx): + """CONFIG_XNU_* of the defconfig become XNU_* variables of the legacy kernel build script.""" + env = {} + for key, value in ctx.need_config().items(): + if not key.startswith('XNU_'): + continue + if isinstance(value, str): + value = value.replace('{tree}', str(ctx.tree)) + env[key] = '1' if value is True else '0' if value is False else str(value) + return env + + +def build(ctx): + log('kernel: XNU') + for name in ('git', 'make', 'clang', 'cmake', 'ninja', 'unifdef'): + tool(name, 'brew install ' + name) + script = ctx.tree / 'scripts/build-xnu-4k-baseline.sh' + if not script.is_file(): + raise Error(f'{script} missing') + env = dict(os.environ) + env.update(xnu_env(ctx)) + env['JOBS'] = str(ctx.jobs) + run(['bash', script], cwd=ctx.tree, env=env, log_file=ctx.logs / 'kernel.log') + + +def prelink(ctx): + """Produce out//kernel/kernel.macho from the XNU build and the original kernelcache.""" + log('kernel: prelink') + spec = ctx.board.get('kernel', {}).get('prelink') + if not spec: + raise Error('board.toml has no [kernel.prelink]') + cache = ctx.option('kernelcache', 'UNIXNU_KERNELCACHE') or (expand(ctx.tree / spec['kernelcache']) if spec.get('kernelcache') else None) + if not cache or not cache.is_file(): + raise Error('the original decompressed kernelcache is an input you provide: --kernelcache FILE or UNIXNU_KERNELCACHE') + wanted = spec.get('kernelcache_sha256') + if wanted and sha256_file(cache) != wanted: + raise Error(f'{cache} does not match the kernelcache this board was qualified against (sha256 {wanted})') + kernel = ctx.tree / spec['kernel'] + if not kernel.is_file(): + raise Error(f'{kernel} not found; run "unixnu build kernel" first') + output = ctx.work / 'kernel/kernel.macho' + output.parent.mkdir(parents=True, exist_ok=True) + command = ['python3', ctx.tree / spec['tool'], '--kernel', kernel, '--prelinked', cache, + f'--slide={spec["slide"]}', '--output', output] + for root in spec.get('roots', []): + command += ['--root', root] + if spec.get('early_security_init'): + command += ['--early-security-init', spec['early_security_init']] + command += spec.get('extra_args', []) + run(command, cwd=ctx.tree, log_file=ctx.logs / 'prelink.log') + log(f'kernel: {output} sha256 {sha256_file(output)}') + return output diff --git a/scripts/unixnu/package.py b/scripts/unixnu/package.py new file mode 100644 index 0000000..a87bad5 --- /dev/null +++ b/scripts/unixnu/package.py @@ -0,0 +1,32 @@ +"""UniXNU itself: the EDK II build of package/ (CrabApple), output under out//package.""" +import os +import shutil + +from .util import Error, expand, log, run, tool + + +def edk2_dir(ctx, modules): + given = ctx.option('edk2', 'UNIXNU_EDK2') + if given: + path = given + elif ('refer', 'edk2') in modules and modules[('refer', 'edk2')].path: + path = modules[('refer', 'edk2')].path + else: + raise Error('no edk2 tree: pass --edk2 DIR (a Mu_Basecore checkout) or run "unixnu fetch" for the edk2 source') + if not (path / 'MdePkg/MdePkg.dec').is_file(): + raise Error(f'{path} is not an edk2 or Mu_Basecore tree') + return path + + +def build(ctx, modules, target='DEBUG', toolchain='CLANGPDB'): + log('package: UniXNU (CrabApple)') + edk2 = edk2_dir(ctx, modules) + tool('clang', 'brew install llvm') + env = dict(os.environ, CRABAPPLE_EDK2=str(edk2), UNIXNU_BUILD_DIR=str(ctx.work / 'package'), JOBS=str(ctx.jobs)) + clang = shutil.which('clang') + if toolchain == 'CLANGPDB' and 'CLANG_BIN' not in env: + env['CLANG_BIN'] = os.path.dirname(clang) + '/' + (ctx.work / 'package').mkdir(parents=True, exist_ok=True) + run(['bash', ctx.tree / 'package/build.sh', '-b', target, '-t', toolchain], cwd=ctx.tree / 'package', env=env, + log_file=ctx.logs / 'package.log') + log(f'package: outputs in {ctx.work / "package/Build/Out"}') diff --git a/scripts/unixnu/tree.py b/scripts/unixnu/tree.py new file mode 100644 index 0000000..9ad5a17 --- /dev/null +++ b/scripts/unixnu/tree.py @@ -0,0 +1,68 @@ +"""Assemble the build tree from the resolved modules. + +out//tree/ has the layout of this repository (drivers/ kits/ patches/ scripts/ package/) +plus refer/, build/ and .cache/ for the scripts that expect them. Modules are symbolic links to +their sources, so edits to a local module take effect on the next build; scripts are copied, since +they locate the tree from their own real path. The device package appears as kits/. +""" +import shutil +from pathlib import Path + +from .util import Error, log + + +def _clear_links(directory): + if directory.is_dir(): + for entry in directory.iterdir(): + if entry.is_symlink(): + entry.unlink() + + +def _link(target, link): + link.parent.mkdir(parents=True, exist_ok=True) + if link.is_symlink() or link.exists(): + link.unlink() + link.symlink_to(Path(target).resolve()) + + +def _copy_scripts(source, destination): + for path in sorted(Path(source).rglob('*')): + relative = path.relative_to(source) + if path.is_dir() or '__pycache__' in path.parts or relative.parts[0] == 'unixnu': + continue + target = destination / relative + if target.exists() and target.stat().st_size == path.stat().st_size and target.stat().st_mtime >= path.stat().st_mtime: + continue + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(path, target) + + +def assemble(ctx, modules): + log(f'tree {ctx.tree}') + tree = ctx.tree + tree.mkdir(parents=True, exist_ok=True) + for folder in ('drivers', 'kits', 'patches', 'refer'): + _clear_links(tree / folder) + (tree / folder).mkdir(exist_ok=True) + for name in ('build', '.cache'): + (tree / name).mkdir(exist_ok=True) + kit = ctx.board.get('board', {}).get('kit', ctx.name) + for (kind, name), module in sorted(modules.items()): + if module.path is None: + if kind == 'refer': + continue # fetched when a step needs it + raise Error(f'{kind} {name} has no source yet; run "unixnu fetch"') + if kind == 'package': + if name == 'main': + _link(module.path, tree / 'package') + else: + _link(module.path, tree / 'kits' / kit) + else: + _link(module.path, tree / {'driver': 'drivers', 'kit': 'kits', 'patch': 'patches', 'refer': 'refer'}[kind] / name) + if not (tree / 'package').is_symlink(): + raise Error('no package module: this repository or --local-package must provide package/') + (tree / 'scripts').mkdir(exist_ok=True) + for base in (ctx.root, ctx.device): + if (base / 'scripts').is_dir(): + _copy_scripts(base / 'scripts', tree / 'scripts') + return tree diff --git a/scripts/unixnu/util.py b/scripts/unixnu/util.py new file mode 100644 index 0000000..becfb8c --- /dev/null +++ b/scripts/unixnu/util.py @@ -0,0 +1,67 @@ +"""Small helpers shared by the build driver.""" +import hashlib +import os +import shutil +import subprocess +import sys +from pathlib import Path + + +class Error(Exception): + """A user-facing failure; the command line prints it without a traceback.""" + + +def log(message): + print(f'== {message}', flush=True) + + +def expand(path): + return Path(os.path.expandvars(os.path.expanduser(str(path)))).resolve() + + +def sha256_file(path): + digest = hashlib.sha256() + with Path(path).open('rb') as stream: + for chunk in iter(lambda: stream.read(1 << 20), b''): + digest.update(chunk) + return digest.hexdigest() + + +EXTRA_BIN = ('/opt/homebrew/opt/llvm/bin', '/opt/homebrew/opt/lld/bin', '/usr/local/opt/llvm/bin', '/usr/local/opt/lld/bin') + + +def tool(name, hint=''): + """Find a tool on PATH, then in the usual Homebrew LLVM locations (keg-only, not on PATH).""" + found = shutil.which(name) or shutil.which(name, path=os.pathsep.join(EXTRA_BIN)) + if not found: + raise Error(f'{name} not found on PATH' + (f' ({hint})' if hint else '')) + return found + + +def run(command, *, cwd=None, env=None, log_file=None, capture=False): + """Run a command, echoing it. With log_file the output goes there and the tail is shown on failure.""" + command = [str(part) for part in command] + if not capture: + print('+ ' + ' '.join(command), flush=True) + if capture: + result = subprocess.run(command, cwd=cwd, env=env, text=True, capture_output=True) + if result.returncode: + raise Error(f'{command[0]} failed ({result.returncode}): {result.stderr.strip() or result.stdout.strip()}') + return result.stdout + if log_file: + Path(log_file).parent.mkdir(parents=True, exist_ok=True) + with Path(log_file).open('w') as stream: + result = subprocess.run(command, cwd=cwd, env=env, stdout=stream, stderr=subprocess.STDOUT) + if result.returncode: + tail = Path(log_file).read_text(errors='replace').splitlines()[-30:] + print('\n'.join(tail), file=sys.stderr) + raise Error(f'{command[0]} failed ({result.returncode}); full log: {log_file}') + return '' + result = subprocess.run(command, cwd=cwd, env=env) + if result.returncode: + raise Error(f'{command[0]} failed ({result.returncode})') + return '' + + +def git(repo, *args, capture=True): + return run(['git', '-C', repo, *args], capture=capture) diff --git a/unixnu b/unixnu new file mode 100755 index 0000000..24e7068 --- /dev/null +++ b/unixnu @@ -0,0 +1,8 @@ +#!/usr/bin/env python3 +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent / 'scripts')) +from unixnu.cli import main + +main()