build(kernel): run the kernel step on Linux
On a Linux host the driver sets the environment of the scripts/linux/setup-toolchain.py toolchain: the xcrun shim answers SDK queries with the build tree's fakeroot, clang targets plain arm64 for iOS 17, and CCC_OVERRIDE_OPTIONS turns off two warnings upstream clang has and Apple clang does not (the archive builders use -Werror and replay xnu's compile records, so only the clang driver sees them). - build-xnu-4k-baseline.sh runs scripts/linux/prepare-xnu.py: source rewrites the xnu sources that assume Apple's tools (SHA/AES assembler macros, makesyscalls.sh under mawk, kextsymboltool and /dev/null) and installs TargetConditionals.h into the fakeroot; libs builds libcc_kext.a from compiler-rt's cc_kext profile sources. xnu's make also gets the host compiler, mig and the machine sizes a Mac supplies. ditto becomes cp. - build-firehose.sh compiles libfirehose_kernel from its xcconfig. - build-corecrypto-14.py reads project.pbxproj with scripts/pbxproj.py (same JSON as plutil for all 13,112 objects). - The kernelcache is decoded with lzfse. The macOS path is unchanged apart from ditto. Built from the same sources and configuration on Linux and with Xcode 27.1, the kernels have the same global symbols, linked objects, archive symbols, 1,265 kalloc_type signatures, and prelink bindings and kexts. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
smeng9andClaude Opus 5.5 authored and rpone committed
1 parent
6dc20ebe08
commit
b0fe495af6
9 files changed
+481
-24
No files matched your search
@@ -9,12 +9,14 @@ import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import platform
|
||||
import subprocess
|
||||
import struct
|
||||
import sys
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'scripts'))
|
||||
import pbxproj
|
||||
from xnu_symbols import kernel_names
|
||||
SOURCE = ROOT / 'refer/corecrypto-2023'
|
||||
REVISION = '11fba65b12d8830630e248dfa8344caf8b065f26'
|
||||
@@ -34,8 +36,11 @@ if subprocess.check_output(['git', '-C', SOURCE, 'rev-parse', 'HEAD'], text=True
|
||||
if subprocess.check_output(['git', '-C', SOURCE, 'status', '--porcelain'], text=True):
|
||||
raise SystemExit('The crypto reference must stay immutable')
|
||||
OUT.mkdir(parents=True, exist_ok=True)
|
||||
subprocess.run(['plutil', '-convert', 'json', '-o', OUT / 'project.json',
|
||||
SOURCE / 'corecrypto.xcodeproj/project.pbxproj'], check=True)
|
||||
if platform.system() == 'Darwin':
|
||||
subprocess.run(['plutil', '-convert', 'json', '-o', OUT / 'project.json',
|
||||
SOURCE / 'corecrypto.xcodeproj/project.pbxproj'], check=True)
|
||||
else:
|
||||
(OUT / 'project.json').write_text(json.dumps(pbxproj.load(SOURCE / 'corecrypto.xcodeproj/project.pbxproj')))
|
||||
project = json.loads((OUT / 'project.json').read_text())
|
||||
objects = project['objects']
|
||||
parents = {child: key for key, value in objects.items() for child in value.get('children', [])}
|
||||
|
||||
@@ -21,12 +21,35 @@ s = '\n'.join(lines) + '\n'
|
||||
if p.read_text() != s:
|
||||
p.write_text(s)
|
||||
PY
|
||||
xcodebuild install -project .cache/libdispatch-work/libdispatch.xcodeproj \
|
||||
-target libfirehose_kernel -configuration Release -sdk iphoneos -jobs "${JOBS:-4}" \
|
||||
ARCHS=arm64 VALID_ARCHS=arm64 ONLY_ACTIVE_ARCH=NO IPHONEOS_DEPLOYMENT_TARGET=17.0 \
|
||||
CODE_SIGNING_ALLOWED=NO GCC_TREAT_WARNINGS_AS_ERRORS=NO \
|
||||
OBJROOT="$ROOT/build/firehose/obj" SYMROOT="$ROOT/build/firehose/sym" \
|
||||
DSTROOT="$ROOT/.cache/fakeroot" FAKEROOT_DIR="$ROOT/.cache/fakeroot"
|
||||
if [ "$(uname -s)" = Linux ]; then
|
||||
# No Xcode: build the target as its xcconfig describes it. libfirehose_kernel is one source file and one
|
||||
# private header (installed through unifdef with COPY_HEADERS_UNIFDEF_FLAGS); symbols stay external, as
|
||||
# Xcode's GCC_SYMBOLS_PRIVATE_EXTERN=NO leaves them.
|
||||
FAKEROOT=$ROOT/.cache/fakeroot
|
||||
WORK=.cache/libdispatch-work
|
||||
mkdir -p build/firehose/obj "$FAKEROOT/usr/local/lib/kernel" "$FAKEROOT/usr/local/include/kernel/os"
|
||||
xcrun --sdk iphoneos clang -target arm64-apple-ios17.0 -std=gnu11 -Os -g \
|
||||
-mkernel -nostdinc -Wno-packed \
|
||||
-DKERNEL=1 -DDISPATCH_USE_DTRACE=0 -DOS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY=1 \
|
||||
-DOS_ATOMIC_CONFIG_STARVATION_FREE_ONLY=0 \
|
||||
-I"$WORK" -I"$FAKEROOT/System/Library/Frameworks/Kernel.framework/PrivateHeaders" \
|
||||
-I"$FAKEROOT/System/Library/Frameworks/Kernel.framework/Headers" \
|
||||
-I"$FAKEROOT/usr/local/include/os" -I"$FAKEROOT/usr/local/include/firehose" \
|
||||
-I"$FAKEROOT/usr/include" -I"$FAKEROOT/usr/local/include" -I"$WORK/../xnu-work/EXTERNAL_HEADERS" \
|
||||
-c "$WORK/src/firehose/firehose_buffer.c" -o build/firehose/obj/firehose_buffer.o
|
||||
rm -f "$FAKEROOT/usr/local/lib/kernel/libfirehose_kernel.a"
|
||||
xcrun libtool -static -o "$FAKEROOT/usr/local/lib/kernel/libfirehose_kernel.a" build/firehose/obj/firehose_buffer.o
|
||||
unifdef -DKERNEL=1 -DOS_FIREHOSE_SPI=1 -DOS_VOUCHER_ACTIVITY_SPI_TYPES=1 -UOS_VOUCHER_ACTIVITY_SPI \
|
||||
-o "$FAKEROOT/usr/local/include/kernel/os/firehose_buffer_private.h" \
|
||||
"$WORK/os/firehose_buffer_private.h" || [ $? -eq 1 ]
|
||||
else
|
||||
xcodebuild install -project .cache/libdispatch-work/libdispatch.xcodeproj \
|
||||
-target libfirehose_kernel -configuration Release -sdk iphoneos -jobs "${JOBS:-4}" \
|
||||
ARCHS=arm64 VALID_ARCHS=arm64 ONLY_ACTIVE_ARCH=NO IPHONEOS_DEPLOYMENT_TARGET=17.0 \
|
||||
CODE_SIGNING_ALLOWED=NO GCC_TREAT_WARNINGS_AS_ERRORS=NO \
|
||||
OBJROOT="$ROOT/build/firehose/obj" SYMROOT="$ROOT/build/firehose/sym" \
|
||||
DSTROOT="$ROOT/.cache/fakeroot" FAKEROOT_DIR="$ROOT/.cache/fakeroot"
|
||||
fi
|
||||
if [ -f .cache/fakeroot/usr/local/lib/kernel/liblibfirehose_kernel.a ]; then
|
||||
cp .cache/fakeroot/usr/local/lib/kernel/liblibfirehose_kernel.a \
|
||||
.cache/fakeroot/usr/local/lib/kernel/libfirehose_kernel.a
|
||||
|
||||
@@ -49,6 +49,17 @@ for PATCH in "$ROOT"/patches/xnu/*.patch "$ROOT"/kits/NativePrelink/patches/*.pa
|
||||
done
|
||||
export XNU_AVAILABILITY_PL=$ROOT/.cache/fakeroot/usr/local/libexec/availability.pl
|
||||
python3 scripts/prepare-xnu-source.py
|
||||
# A Linux host builds with upstream clang and GNU tools (scripts/linux): rewrite the few sources that assume
|
||||
# Apple's, and pass xnu's make what a Mac provides itself (host compiler, mig, memory and CPU counts).
|
||||
XNU_HOST_MAKE_ARGS=()
|
||||
if [ "$(uname -s)" = Linux ]; then
|
||||
python3 scripts/linux/prepare-xnu.py source
|
||||
export PLATFORM=iPhoneOS HOST_CC="$CLT_PREFIX/bin/xnu-host-cc" MIGCOM="$CLT_PREFIX/libexec/migcom"
|
||||
XNU_HOST_MAKE_ARGS=(TIGHTBEAMC=tightbeamc-not-supported BUILD_DSYM=0
|
||||
MEMORY_SIZE="$(( $(getconf _PHYS_PAGES) * $(getconf PAGE_SIZE) ))"
|
||||
SYSCTL_HW_PHYSICALCPU="$(nproc)" SYSCTL_HW_LOGICALCPU="$(nproc)"
|
||||
LD_KERNEL_LIBS="$ROOT/.cache/fakeroot/usr/local/lib/kernel/libcc_kext.a")
|
||||
fi
|
||||
# The first pass of a new tree (XNU_BOOTSTRAP=1, set by the unixnu driver) only has to leave the compile records and
|
||||
# link.filelist for the archive builders that run after it, so it links without their archives.
|
||||
BOOTSTRAP=${XNU_BOOTSTRAP:-0}
|
||||
@@ -90,9 +101,13 @@ if [ ! -d refer/libplatform/.git ]; then
|
||||
fi
|
||||
export KERNEL_SDKROOT=$ROOT/.cache/fakeroot
|
||||
mkdir -p "$KERNEL_SDKROOT/usr/local/include/kernel" "$ROOT/build/xnu/obj/EXPORT_HDRS/libsa"
|
||||
ditto refer/libplatform/include "$KERNEL_SDKROOT/usr/local/include"
|
||||
ditto refer/libplatform/private "$KERNEL_SDKROOT/usr/local/include"
|
||||
ditto refer/libplatform/private/platform "$KERNEL_SDKROOT/System/Library/Frameworks/Kernel.framework/Versions/A/PrivateHeaders/platform"
|
||||
copy_tree() {
|
||||
mkdir -p "$2"
|
||||
cp -R "$1/." "$2/"
|
||||
}
|
||||
copy_tree refer/libplatform/include "$KERNEL_SDKROOT/usr/local/include"
|
||||
copy_tree refer/libplatform/private "$KERNEL_SDKROOT/usr/local/include"
|
||||
copy_tree refer/libplatform/private/platform "$KERNEL_SDKROOT/System/Library/Frameworks/Kernel.framework/Versions/A/PrivateHeaders/platform"
|
||||
if [ ! -d refer/libdispatch/.git ]; then
|
||||
git clone --depth 1 --branch "$LIBDISPATCH_TAG" https://github.com/apple-oss-distributions/libdispatch.git refer/libdispatch
|
||||
fi
|
||||
@@ -116,12 +131,16 @@ if [ ! -f "$KERNEL_SDKROOT/.xnu-$XNU_VERSION-headers" ]; then
|
||||
MACHINE_CONFIGS=BCM2837 KERNEL_CONFIGS=DEVELOPMENT \
|
||||
RC_DARWIN_KERNEL_VERSION="$XNU_DARWIN" RC_ProjectSourceVersion="$XNU_VERSION" \
|
||||
MAKEJOBS=-j"${JOBS:-4}" OBJROOT="$ROOT/build/xnu/obj" \
|
||||
SYMROOT="$ROOT/build/xnu/sym" DSTROOT="$KERNEL_SDKROOT"
|
||||
SYMROOT="$ROOT/build/xnu/sym" DSTROOT="$KERNEL_SDKROOT" \
|
||||
${XNU_HOST_MAKE_ARGS[@]+"${XNU_HOST_MAKE_ARGS[@]}"}
|
||||
touch "$KERNEL_SDKROOT/.xnu-$XNU_VERSION-headers"
|
||||
fi
|
||||
if [ ! -f "$KERNEL_SDKROOT/usr/local/lib/kernel/libfirehose_kernel.a" ]; then
|
||||
bash scripts/build-firehose.sh
|
||||
fi
|
||||
if [ "$(uname -s)" = Linux ]; then
|
||||
python3 scripts/linux/prepare-xnu.py libs
|
||||
fi
|
||||
# The baseline builds the kernel; `all` also checks the Apple kext export ABI.
|
||||
EXTRA_MAKE_ARGS=()
|
||||
ARM_DT_PLATFORM=${XNU_ARM_DT_PLATFORM:-0}
|
||||
@@ -779,7 +798,7 @@ make -C .cache/xnu-work "${XNU_BUILD_TARGET:-build}" SDKROOT="${XNU_SDKROOT:-iph
|
||||
DEPLOYMENT_LINKER_FLAGS="-Wl,-ios_version_min,17.0" \
|
||||
RC_DARWIN_KERNEL_VERSION="$XNU_DARWIN" RC_ProjectSourceVersion="$XNU_VERSION" \
|
||||
BUILD_LTO=0 BOUND_CHECKS=0 BUILD_WERROR="${XNU_BUILD_WERROR:-0}" MAKEJOBS=-j"${JOBS:-4}" \
|
||||
CFLAGS_EXTRA="$EXTRA_CFLAGS" "${EXTRA_MAKE_ARGS[@]}" \
|
||||
CFLAGS_EXTRA="$EXTRA_CFLAGS" "${EXTRA_MAKE_ARGS[@]}" ${XNU_HOST_MAKE_ARGS[@]+"${XNU_HOST_MAKE_ARGS[@]}"} \
|
||||
OBJROOT="$ROOT/build/xnu/ios-obj" SYMROOT="$ROOT/build/xnu/ios-sym" DSTROOT="$ROOT/build/xnu/ios-dst"
|
||||
if [ "${XNU_SOURCE_CORECRYPTO14:-0}" = 1 ] && [ "$CRYPTO_BOOTSTRAP_MODE" = source ]; then
|
||||
python3 scripts/seal-source-crypto-kernel.py \
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
/* <TargetConditionals.h> for the UniXNU kernel step on Linux (scripts/linux/prepare-xnu.py installs it).
|
||||
*
|
||||
* On a Mac this header comes from the iPhoneOS SDK's usr/include. xnu itself skips it under
|
||||
* XNU_KERNEL_PRIVATE; what includes it is code built against Kernel.framework from outside xnu
|
||||
* (libfirehose_kernel, the kext-style archives). The TARGET_OS_* values are derived from the
|
||||
* compiler's target triple, as the SDK header does, so an arm64-apple-ios17.0 compile sees iOS.
|
||||
* Clang predefines most TARGET_OS_* for Darwin targets itself; each one here applies only when
|
||||
* the compiler did not.
|
||||
*/
|
||||
#ifndef __TARGETCONDITIONALS__
|
||||
#define __TARGETCONDITIONALS__
|
||||
|
||||
#if !defined(__is_target_os)
|
||||
#error "TargetConditionals.h stand-in needs clang's __is_target_os"
|
||||
#endif
|
||||
|
||||
#ifndef TARGET_OS_MAC
|
||||
#define TARGET_OS_MAC 1
|
||||
#endif
|
||||
#ifndef TARGET_OS_OSX
|
||||
#define TARGET_OS_OSX (__is_target_os(macos))
|
||||
#endif
|
||||
#ifndef TARGET_OS_IOS
|
||||
#define TARGET_OS_IOS (__is_target_os(ios) && !__is_target_environment(macabi))
|
||||
#endif
|
||||
#ifndef TARGET_OS_TV
|
||||
#define TARGET_OS_TV (__is_target_os(tvos))
|
||||
#endif
|
||||
#ifndef TARGET_OS_WATCH
|
||||
#define TARGET_OS_WATCH (__is_target_os(watchos))
|
||||
#endif
|
||||
#ifndef TARGET_OS_VISION
|
||||
#define TARGET_OS_VISION (__is_target_os(xros))
|
||||
#endif
|
||||
#ifndef TARGET_OS_BRIDGE
|
||||
#define TARGET_OS_BRIDGE 0
|
||||
#endif
|
||||
#ifndef TARGET_OS_DRIVERKIT
|
||||
#define TARGET_OS_DRIVERKIT (__is_target_os(driverkit))
|
||||
#endif
|
||||
#ifndef TARGET_OS_MACCATALYST
|
||||
#define TARGET_OS_MACCATALYST (__is_target_environment(macabi))
|
||||
#endif
|
||||
#ifndef TARGET_OS_UIKITFORMAC
|
||||
#define TARGET_OS_UIKITFORMAC TARGET_OS_MACCATALYST
|
||||
#endif
|
||||
#ifndef TARGET_OS_IPHONE
|
||||
#define TARGET_OS_IPHONE (TARGET_OS_IOS || TARGET_OS_TV || TARGET_OS_WATCH || TARGET_OS_VISION || TARGET_OS_BRIDGE || TARGET_OS_MACCATALYST)
|
||||
#endif
|
||||
#ifndef TARGET_OS_SIMULATOR
|
||||
#define TARGET_OS_SIMULATOR (__is_target_environment(simulator))
|
||||
#endif
|
||||
#ifndef TARGET_OS_EMBEDDED
|
||||
#define TARGET_OS_EMBEDDED (TARGET_OS_IPHONE && !TARGET_OS_SIMULATOR && !TARGET_OS_MACCATALYST)
|
||||
#endif
|
||||
#ifndef TARGET_IPHONE_SIMULATOR
|
||||
#define TARGET_IPHONE_SIMULATOR TARGET_OS_SIMULATOR
|
||||
#endif
|
||||
#ifndef TARGET_OS_NANO
|
||||
#define TARGET_OS_NANO TARGET_OS_WATCH
|
||||
#endif
|
||||
#ifndef TARGET_OS_LINUX
|
||||
#define TARGET_OS_LINUX 0
|
||||
#endif
|
||||
#ifndef TARGET_OS_WINDOWS
|
||||
#define TARGET_OS_WINDOWS 0
|
||||
#endif
|
||||
#ifndef TARGET_OS_WIN32
|
||||
#define TARGET_OS_WIN32 0
|
||||
#endif
|
||||
#ifndef TARGET_OS_UNIX
|
||||
#define TARGET_OS_UNIX 0
|
||||
#endif
|
||||
#ifndef TARGET_OS_EXCLAVECORE
|
||||
#define TARGET_OS_EXCLAVECORE 0
|
||||
#endif
|
||||
#ifndef TARGET_OS_EXCLAVEKIT
|
||||
#define TARGET_OS_EXCLAVEKIT 0
|
||||
#endif
|
||||
|
||||
#if defined(__arm64__) || defined(__aarch64__)
|
||||
#ifndef TARGET_CPU_ARM64
|
||||
#define TARGET_CPU_ARM64 1
|
||||
#endif
|
||||
#else
|
||||
#ifndef TARGET_CPU_ARM64
|
||||
#define TARGET_CPU_ARM64 0
|
||||
#endif
|
||||
#endif
|
||||
#ifndef TARGET_CPU_ARM
|
||||
#define TARGET_CPU_ARM (defined(__arm__) && !TARGET_CPU_ARM64)
|
||||
#endif
|
||||
#ifndef TARGET_CPU_X86_64
|
||||
#define TARGET_CPU_X86_64 (defined(__x86_64__))
|
||||
#endif
|
||||
#ifndef TARGET_CPU_X86
|
||||
#define TARGET_CPU_X86 (defined(__i386__))
|
||||
#endif
|
||||
#ifndef TARGET_CPU_PPC
|
||||
#define TARGET_CPU_PPC 0
|
||||
#endif
|
||||
#ifndef TARGET_CPU_PPC64
|
||||
#define TARGET_CPU_PPC64 0
|
||||
#endif
|
||||
#ifndef TARGET_CPU_68K
|
||||
#define TARGET_CPU_68K 0
|
||||
#endif
|
||||
#ifndef TARGET_CPU_ALPHA
|
||||
#define TARGET_CPU_ALPHA 0
|
||||
#endif
|
||||
#ifndef TARGET_CPU_MIPS
|
||||
#define TARGET_CPU_MIPS 0
|
||||
#endif
|
||||
#ifndef TARGET_CPU_SPARC
|
||||
#define TARGET_CPU_SPARC 0
|
||||
#endif
|
||||
#ifndef TARGET_RT_LITTLE_ENDIAN
|
||||
#define TARGET_RT_LITTLE_ENDIAN (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
|
||||
#endif
|
||||
#ifndef TARGET_RT_BIG_ENDIAN
|
||||
#define TARGET_RT_BIG_ENDIAN (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
|
||||
#endif
|
||||
#ifndef TARGET_RT_64_BIT
|
||||
#define TARGET_RT_64_BIT (__LP64__)
|
||||
#endif
|
||||
#ifndef TARGET_RT_MAC_CFM
|
||||
#define TARGET_RT_MAC_CFM 0
|
||||
#endif
|
||||
#ifndef TARGET_RT_MAC_MACHO
|
||||
#define TARGET_RT_MAC_MACHO 1
|
||||
#endif
|
||||
#ifndef TARGET_ABI_USES_IOS_VALUES
|
||||
#define TARGET_ABI_USES_IOS_VALUES (!TARGET_CPU_X86_64 || (TARGET_OS_IPHONE && !TARGET_OS_MACCATALYST))
|
||||
#endif
|
||||
|
||||
#endif /* __TARGETCONDITIONALS__ */
|
||||
Executable
+140
@@ -0,0 +1,140 @@
|
||||
#!/usr/bin/env python3
|
||||
"""The Linux-only steps of scripts/build-xnu-4k-baseline.sh, run from the build tree.
|
||||
|
||||
prepare-xnu.py source after the xnu patches: rewrite the xnu sources that assume Apple's clang or BSD tools,
|
||||
and install TargetConditionals.h, which the iPhoneOS SDK provides, into the fakeroot.
|
||||
prepare-xnu.py libs after installhdrs: libcc_kext.a in the fakeroot. MakeInc.def links it as -lcc_kext; a Mac
|
||||
takes it from Apple clang.
|
||||
|
||||
Each rewrite checks whether it is still needed, so both modes can run on every build.
|
||||
"""
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
ROOT = HERE.parents[1]
|
||||
XNU = ROOT / '.cache/xnu-work'
|
||||
FAKEROOT = ROOT / '.cache/fakeroot'
|
||||
# llvm-project at the commit setup-toolchain.py builds clang from, and the Libc that macOS 14.6 ships with
|
||||
# xnu-10063.141.1 (distribution-macOS release.json).
|
||||
LLVM_COMMIT = '58c46bae21181fb4240cf964461bea738d96d50e'
|
||||
LIBC_COMMIT = '899a3b2d52d95d75e05fb286a5e64975ec3de757' # Libc-1592.100.35
|
||||
PROFILE_SOURCES = ('InstrProfiling', 'InstrProfilingBuffer', 'InstrProfilingPlatformDarwin', 'InstrProfilingWriter',
|
||||
'InstrProfilingInternal', 'InstrProfilingVersionVar')
|
||||
|
||||
ISA_APPLE = '#if defined(__clang__) && ((defined(__apple_build_version__) && __apple_build_version__ > 5010000))\n'
|
||||
ISA_ANY = '#if defined(__clang__) && (!defined(__apple_build_version__) || __apple_build_version__ > 5010000)\n'
|
||||
SEEK_OLD = ''' if (offset != lseek(fd, offset, SEEK_SET)) {
|
||||
err = kErrorDiskFull;
|
||||
} else {
|
||||
err = kErrorNone;
|
||||
}
|
||||
'''
|
||||
SEEK_NEW = ''' if (offset != lseek(fd, offset, SEEK_SET)) {
|
||||
struct stat st;
|
||||
/* Linux lseek() returns 0 on character devices such as the
|
||||
* /dev/null that check_all_exports writes to; not an error. */
|
||||
if (0 == fstat(fd, &st) && S_ISCHR(st.st_mode)) {
|
||||
return kErrorNone;
|
||||
}
|
||||
err = kErrorDiskFull;
|
||||
} else {
|
||||
err = kErrorNone;
|
||||
}
|
||||
'''
|
||||
|
||||
|
||||
def info(message):
|
||||
print(f'[info] {message}', file=sys.stderr)
|
||||
|
||||
|
||||
def rewrite_sources():
|
||||
# bsd/kern/makesyscalls.sh writes `\(sy_call_t *\)` inside an awk string. BSD awk and gawk drop the unknown
|
||||
# escape; Debian's default mawk keeps the backslash, and init_sysent.c does not compile. The escape is bogus
|
||||
# either way: remove it, and any generated file that carries it.
|
||||
script = XNU / 'bsd/kern/makesyscalls.sh'
|
||||
text = script.read_text()
|
||||
if '\\(sy_call_t *\\)' in text:
|
||||
script.write_text(text.replace('\\(sy_call_t *\\)', '(sy_call_t *)'))
|
||||
info('patched bsd/kern/makesyscalls.sh: dropped the \\( escapes mawk keeps')
|
||||
for generated in (ROOT / 'build/xnu').rglob('*.c') if (ROOT / 'build/xnu').is_dir() else ():
|
||||
if b'\\(sy_call_t' in generated.read_bytes():
|
||||
generated.unlink()
|
||||
info(f'removed mis-generated {generated}')
|
||||
|
||||
# arm64_isa_compatibility.h gives Apple clang newer than 5010000 the standard `v0.4s` SHA/AES assembler forms and
|
||||
# every other compiler the pre-2014 `q0, q1` forms, which no current assembler accepts. Upstream clang has no
|
||||
# __apple_build_version__ and is newer than 5010000.
|
||||
isa = XNU / 'osfmk/arm64/corecrypto/arm64_isa_compatibility.h'
|
||||
text = isa.read_text()
|
||||
if ISA_APPLE in text:
|
||||
isa.write_text(text.replace(ISA_APPLE, ISA_ANY))
|
||||
info('patched corecrypto/arm64_isa_compatibility.h: standard vector syntax for non-Apple clang')
|
||||
|
||||
# SETUP/kextsymboltool's seekFile() treats `offset != lseek()` as a full disk. check_all_exports runs it with
|
||||
# `-output /dev/null`, where Linux lseek() returns 0, so the export check failed after the kernel link. make
|
||||
# rebuilds the tool when its source changes.
|
||||
tool = XNU / 'SETUP/kextsymboltool/kextsymboltool.c'
|
||||
text = tool.read_text()
|
||||
if 'S_ISCHR' not in text:
|
||||
if text.count(SEEK_OLD) != 1:
|
||||
sys.exit(f'{tool}: seekFile() body not found')
|
||||
text = text.replace(SEEK_OLD, SEEK_NEW)
|
||||
if '#include <sys/stat.h>' not in text:
|
||||
text = text.replace('#include <sys/types.h>', '#include <sys/types.h>\n#include <sys/stat.h>', 1)
|
||||
tool.write_text(text)
|
||||
info('patched SETUP/kextsymboltool/kextsymboltool.c: lseek() on /dev/null is not a write error')
|
||||
|
||||
(FAKEROOT / 'usr/include').mkdir(parents=True, exist_ok=True)
|
||||
shutil.copyfile(HERE / 'TargetConditionals.h', FAKEROOT / 'usr/include/TargetConditionals.h')
|
||||
|
||||
|
||||
def fetch(directory, url, commit, *paths):
|
||||
"""A sparse, blobless checkout of paths at commit."""
|
||||
head = subprocess.run(['git', '-C', directory, 'rev-parse', 'HEAD'], capture_output=True, text=True)
|
||||
if head.returncode == 0 and head.stdout.strip() == commit:
|
||||
return
|
||||
shutil.rmtree(directory, ignore_errors=True)
|
||||
for command in (['init', '-q', directory], ['-C', directory, 'sparse-checkout', 'set', '--no-cone', *paths],
|
||||
['-C', directory, 'fetch', '-q', '--depth', '1', '--filter=blob:none', url, commit],
|
||||
['-C', directory, 'checkout', '-q', 'FETCH_HEAD']):
|
||||
subprocess.run(['git', *map(str, command)], check=True)
|
||||
|
||||
|
||||
def build_cc_kext():
|
||||
# The profile half of compiler-rt's cc_kext recipe (cmake/Modules/CompilerRTDarwinUtils.cmake: PROFILE_SOURCES,
|
||||
# -mkernel, KERNEL_USE). The builtins half is left out: the kernel link needs none of it. KERNEL_USE selects the
|
||||
# kernel headers (Kernel.framework); inttypes.h, stdio.h and gethostuuid.h come from Libc and Libsyscall, as they
|
||||
# do from an SDK's usr/include.
|
||||
out = FAKEROOT / 'usr/local/lib/kernel/libcc_kext.a'
|
||||
if out.is_file():
|
||||
return
|
||||
src = ROOT / '.cache/linux'
|
||||
fetch(src / 'llvm-project', 'https://github.com/llvm/llvm-project', LLVM_COMMIT,
|
||||
'compiler-rt/lib/profile', 'compiler-rt/include/profile')
|
||||
fetch(src / 'Libc', 'https://github.com/apple-oss-distributions/Libc.git', LIBC_COMMIT, 'include')
|
||||
rt = src / 'llvm-project/compiler-rt'
|
||||
objects = src / 'cc_kext'
|
||||
objects.mkdir(parents=True, exist_ok=True)
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
for name in PROFILE_SOURCES:
|
||||
subprocess.run(['xcrun', 'clang', '-target', 'arm64-apple-ios17.0', '-arch', 'arm64', '-isysroot', FAKEROOT,
|
||||
'-mkernel', '-DKERNEL_USE', '-O3', '-fPIC', '-w',
|
||||
'-I', rt / 'include', '-I', FAKEROOT / 'System/Library/Frameworks/Kernel.framework/Headers',
|
||||
'-idirafter', src / 'Libc/include', '-idirafter', XNU / 'libsyscall/wrappers',
|
||||
'-c', rt / f'lib/profile/{name}.c', '-o', objects / f'{name}.o'], check=True)
|
||||
subprocess.run(['xcrun', 'libtool', '-static', '-o', out, *(objects / f'{n}.o' for n in PROFILE_SOURCES)],
|
||||
check=True)
|
||||
info(f'libcc_kext.a: {out}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
mode = sys.argv[1] if len(sys.argv) > 1 else ''
|
||||
if mode == 'source':
|
||||
rewrite_sources()
|
||||
elif mode == 'libs':
|
||||
build_cc_kext()
|
||||
else:
|
||||
sys.exit('usage: prepare-xnu.py source|libs')
|
||||
@@ -0,0 +1,90 @@
|
||||
"""Read an Xcode project.pbxproj (an OpenStep property list) without plutil.
|
||||
|
||||
macOS `plutil -convert json` reads OpenStep plists; the plistlib-based plutil a Linux host has does not. Values come
|
||||
back as plutil's JSON has them: dictionaries, lists and strings.
|
||||
"""
|
||||
from pathlib import Path
|
||||
import re
|
||||
|
||||
BARE = re.compile(rb'[A-Za-z0-9_$+/:.\-]+')
|
||||
ESCAPES = {b'n': b'\n', b't': b'\t', b'r': b'\r', b'"': b'"', b'\\': b'\\'}
|
||||
|
||||
|
||||
def load(path):
|
||||
data = Path(path).read_bytes()
|
||||
at = 0
|
||||
|
||||
def skip():
|
||||
nonlocal at
|
||||
while at < len(data):
|
||||
if data[at:at + 1].isspace():
|
||||
at += 1
|
||||
elif data.startswith(b'//', at):
|
||||
end = data.find(b'\n', at)
|
||||
at = len(data) if end < 0 else end + 1
|
||||
elif data.startswith(b'/*', at):
|
||||
end = data.find(b'*/', at + 2)
|
||||
if end < 0:
|
||||
raise ValueError(f'{path}: unterminated comment')
|
||||
at = end + 2
|
||||
else:
|
||||
return
|
||||
|
||||
def expect(char):
|
||||
nonlocal at
|
||||
skip()
|
||||
if data[at:at + 1] != char:
|
||||
raise ValueError(f'{path}: expected {char.decode()} at offset {at}')
|
||||
at += 1
|
||||
|
||||
def value():
|
||||
nonlocal at
|
||||
skip()
|
||||
head = data[at:at + 1]
|
||||
if head == b'{':
|
||||
at += 1
|
||||
result = {}
|
||||
while True:
|
||||
skip()
|
||||
if data[at:at + 1] == b'}':
|
||||
at += 1
|
||||
return result
|
||||
key = value()
|
||||
expect(b'=')
|
||||
result[key] = value()
|
||||
expect(b';')
|
||||
if head == b'(':
|
||||
at += 1
|
||||
result = []
|
||||
while True:
|
||||
skip()
|
||||
if data[at:at + 1] == b')':
|
||||
at += 1
|
||||
return result
|
||||
result.append(value())
|
||||
skip()
|
||||
if data[at:at + 1] == b',':
|
||||
at += 1
|
||||
if head == b'"':
|
||||
at += 1
|
||||
out = bytearray()
|
||||
while True:
|
||||
char = data[at:at + 1]
|
||||
if not char:
|
||||
raise ValueError(f'{path}: unterminated string')
|
||||
at += 1
|
||||
if char == b'"':
|
||||
return out.decode()
|
||||
if char == b'\\':
|
||||
escape = data[at:at + 1]
|
||||
at += 1
|
||||
out += ESCAPES.get(escape, b'\\' + escape)
|
||||
else:
|
||||
out += char
|
||||
match = BARE.match(data, at)
|
||||
if not match:
|
||||
raise ValueError(f'{path}: unexpected {head!r} at offset {at}')
|
||||
at = match.end()
|
||||
return match.group().decode()
|
||||
|
||||
return value()
|
||||
@@ -203,8 +203,12 @@ def cmd_clean(ctx):
|
||||
def main(argv=None):
|
||||
if sys.version_info < (3, 11):
|
||||
sys.exit('unixnu needs Python 3.11 or newer')
|
||||
if platform.system() != 'Darwin':
|
||||
print('unixnu: developed and tested on macOS only; the kernel step needs Xcode and --skip-setup needs hdiutil', file=sys.stderr)
|
||||
if platform.system() == 'Linux':
|
||||
print('unixnu: on Linux the kernel step uses the toolchain of scripts/linux/setup-toolchain.py; '
|
||||
'userdata and --skip-setup need macOS hdiutil', file=sys.stderr)
|
||||
elif platform.system() != 'Darwin':
|
||||
print('unixnu: developed and tested on macOS and Linux only; the kernel step needs Xcode or the Linux toolchain, '
|
||||
'and --skip-setup needs hdiutil', file=sys.stderr)
|
||||
args = parser().parse_args(argv)
|
||||
try:
|
||||
ctx = Context(args)
|
||||
|
||||
+11
-3
@@ -10,6 +10,8 @@ chip_id, board_id, and the components the steps use.
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import platform
|
||||
import plistlib
|
||||
import struct
|
||||
import subprocess
|
||||
@@ -202,14 +204,20 @@ def kernelcache(ctx, source):
|
||||
pin = spec.get('kernelcache_sha256')
|
||||
if out.is_file() and (pin is None or sha256_file(out) == pin):
|
||||
return out
|
||||
tool('compression_tool', 'macOS /usr/bin/compression_tool decodes the LZFSE kernelcache')
|
||||
if platform.system() == 'Linux':
|
||||
# Apple's reference LZFSE tool, which scripts/linux/setup-toolchain.py installs.
|
||||
lzfse = Path(os.environ.get('UNIXNU_TOOLCHAIN', '/opt/unixnu-linux')) / 'bin/lzfse'
|
||||
decode = [str(lzfse) if lzfse.is_file() else tool('lzfse', 'run sudo scripts/linux/setup-toolchain.py'), '-decode']
|
||||
else:
|
||||
tool('compression_tool', 'macOS /usr/bin/compression_tool decodes the LZFSE kernelcache')
|
||||
decode = ['/usr/bin/compression_tool', '-decode', '-a', 'lzfse']
|
||||
source.cache.mkdir(parents=True, exist_ok=True)
|
||||
payload = im4p_payload(source.read('KernelCache'))
|
||||
packed, raw = source.cache / 'kernelcache.lzfse', source.cache / 'kernelcache.fat'
|
||||
packed.write_bytes(payload)
|
||||
result = subprocess.run(['/usr/bin/compression_tool', '-decode', '-a', 'lzfse', '-i', packed, '-o', raw], capture_output=True, text=True)
|
||||
result = subprocess.run([*decode, '-i', packed, '-o', raw], capture_output=True, text=True)
|
||||
if result.returncode:
|
||||
raise Error(f'compression_tool failed: {result.stderr.strip()}')
|
||||
raise Error(f'{Path(decode[0]).name} failed: {result.stderr.strip()}')
|
||||
fat = raw.read_bytes()
|
||||
if fat[:4] == b'\xca\xfe\xba\xbe':
|
||||
count = struct.unpack('>I', fat[4:8])[0]
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
"""XNU kernel: build from the patched Apple source, then prelink against the original kernelcache."""
|
||||
import os
|
||||
import platform
|
||||
import shlex
|
||||
from pathlib import Path
|
||||
|
||||
@@ -37,11 +38,37 @@ def version_env(modules):
|
||||
return env
|
||||
|
||||
|
||||
def toolchain_path():
|
||||
def toolchain_path(path=None):
|
||||
"""PATH with the Homebrew LLVM and LLD first, as the archive builders expect."""
|
||||
extra = [d for d in ('/opt/homebrew/opt/llvm/bin', '/opt/homebrew/opt/lld/bin', '/usr/local/opt/llvm/bin', '/usr/local/opt/lld/bin')
|
||||
if Path(d).is_dir()]
|
||||
return os.pathsep.join(extra + [os.environ.get('PATH', '')])
|
||||
return os.pathsep.join(extra + [path if path is not None else os.environ.get('PATH', '')])
|
||||
|
||||
|
||||
def linux_env(ctx):
|
||||
"""Environment of the kernel step on a Linux host, with the toolchain of scripts/linux/setup-toolchain.py.
|
||||
|
||||
The xcrun shim answers every --sdk query with the fakeroot this build makes. The target is plain arm64 for
|
||||
iOS 17, so the clang wrapper adds only the flag that keeps C++ static destructors in __mod_term_func.
|
||||
CCC_OVERRIDE_OPTIONS (read by the clang driver itself, so also by the archive builders that replay xnu's
|
||||
compile records) turns off two warnings that upstream clang has and Apple clang does not; the builders compile
|
||||
with -Werror.
|
||||
"""
|
||||
toolchain = Path(os.environ.get('UNIXNU_TOOLCHAIN', '/opt/unixnu-linux'))
|
||||
clt, xctoolchain = toolchain / 'CommandLineTools/usr', toolchain / 'XcodeDefault.xctoolchain/usr'
|
||||
for path in (toolchain / 'llvm/bin/clang', clt / 'bin/ld', clt / 'bin/xnu-host-cc', clt / 'libexec/migcom',
|
||||
xctoolchain / 'bin/iig', Path('/usr/bin/xcrun'), toolchain / 'bin/lzfse'):
|
||||
if not path.exists():
|
||||
raise Error(f'{path} missing; run "sudo scripts/linux/setup-toolchain.py {toolchain}"')
|
||||
fakeroot = ctx.tree / '.cache/fakeroot'
|
||||
return {'PATH': os.pathsep.join([str(clt / 'bin'), str(toolchain / 'bin'), os.environ.get('PATH', '')]),
|
||||
'LC_ALL': 'C', 'CLT_PREFIX': str(clt), 'XCTOOLCHAIN_PREFIX': str(xctoolchain),
|
||||
'XNU_TOOLCHAIN_BIN': f'{clt / "bin"} {xctoolchain / "bin"}',
|
||||
'UNIXNU_SDKROOT': str(fakeroot), 'UNIXNU_SDK_VERSION': '17.0',
|
||||
'UNIXNU_PLATFORM_PATH': str(fakeroot / 'Platforms/iPhoneOS.platform'),
|
||||
'CROSS_CLANG': str(toolchain / 'llvm/bin/clang'), 'DARWIN_TRIPLE': 'arm64-apple-ios17.0',
|
||||
'CROSS_EXTRA_FLAGS': '-mllvm -disable-atexit-based-global-dtor-lowering',
|
||||
'CCC_OVERRIDE_OPTIONS': '# +-Wno-missing-designated-field-initializers +-Wno-unused-but-set-global'}
|
||||
|
||||
|
||||
def build_archives(ctx, env):
|
||||
@@ -54,7 +81,7 @@ def build_archives(ctx, env):
|
||||
if not ctx.need_config().get('XNU_BUILD_ARCHIVES'):
|
||||
return
|
||||
entries = ctx.board.get('kernel', {}).get('archives', [])
|
||||
env = dict(env, PATH=toolchain_path())
|
||||
env = dict(env, PATH=toolchain_path(env.get('PATH')))
|
||||
for index, entry in enumerate(entries, 1):
|
||||
parts = shlex.split(entry)
|
||||
script = ctx.tree / 'scripts' / parts[0]
|
||||
@@ -70,13 +97,18 @@ def build_archives(ctx, env):
|
||||
|
||||
def build(ctx, modules):
|
||||
log('kernel: XNU')
|
||||
for name in ('git', 'make', 'clang', 'cmake', 'ninja', 'unifdef'):
|
||||
tool(name, 'brew install ' + name)
|
||||
tool('xcodebuild', 'the kernel step builds with the Xcode iphoneos SDK')
|
||||
script = ctx.tree / 'scripts/build-xnu-4k-baseline.sh'
|
||||
if not script.is_file():
|
||||
raise Error(f'{script} missing')
|
||||
env = dict(os.environ)
|
||||
if platform.system() == 'Linux':
|
||||
for name in ('git', 'make', 'cmake', 'ninja', 'unifdef'):
|
||||
tool(name, 'run sudo scripts/linux/setup-toolchain.py')
|
||||
env.update(linux_env(ctx))
|
||||
else:
|
||||
for name in ('git', 'make', 'clang', 'cmake', 'ninja', 'unifdef'):
|
||||
tool(name, 'brew install ' + name)
|
||||
tool('xcodebuild', 'the kernel step builds with the Xcode iphoneos SDK')
|
||||
env.update(xnu_env(ctx))
|
||||
env.update(version_env(modules))
|
||||
env['JOBS'] = str(ctx.jobs)
|
||||
|
||||
Reference in new issue
Block a user