commit dec4c7b5b64f35c8a95a828fb0f8375d91eae6a5 Author: bluseliu50 <265648257+bluseliu50@users.noreply.github.com> Date: Mon Oct 19 18:48:33 2026 +0800 Initial import diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..b373353 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,17 @@ +* text=auto eol=lf +*.sh text eol=lf +*.py text eol=lf +*.c text eol=lf +*.cpp text eol=lf +*.h text eol=lf +*.mm text eol=lf +*.rs text eol=lf +*.json text eol=lf +*.patch text eol=lf -whitespace +*.png binary +*.jpg binary +*.efi binary +*.dtb binary +*.bin binary +*.a binary +*.o binary diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c361bc4 --- /dev/null +++ b/.gitignore @@ -0,0 +1,44 @@ +# Build outputs and caches (re-derivable; nothing important lives here) +/build/ +/Build/ +/Conf/ +/out/ +/.cache/ +/refer/ +/.local*/ +/Mu_Basecore/ + +# Local development files +*.log +*.terminal +*.swp +*.swo +*~ +.DS_Store +.vscode/ +.idea/ +__pycache__/ +*.pyc +*.dSYM/ + +# Inputs, images and products that never belong in Git +*.ipsw +*.dmg +*.im4p +*.img +*.dbr +*.macho +*.o +*.a +*.dylib +*.efi +*.dtb +*.bc +*.spv +*.metallib +*.bin +*.mbn +*.fw +*.pem +*.key +.env diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..f63d2a6 --- /dev/null +++ b/LICENSE @@ -0,0 +1,51 @@ +Copyright (c) 2026, UniXNU contributors. All rights reserved. + +SPDX-License-Identifier: BSD-2-Clause-Patent + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +Subject to the terms and conditions of this license, each copyright holder +and contributor hereby grants to those receiving rights under this license +a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable +(except for failure to satisfy the conditions of this license) patent +license to make, have made, use, offer to sell, sell, import, and otherwise +transfer this software, where such license applies only to those patent +claims, already acquired or hereafter acquired, licensable by such copyright +holder or contributor that are necessarily infringed by: + +(a) their Contribution(s) (the licensed copyrights of copyright holders and + non-copyrightable additions of contributors, in source or binary form) + alone; or + +(b) combination of their Contribution(s) with the work of authorship to + which such Contribution(s) was added by such copyright holder or + contributor, if, at the time the Contribution is added, such addition + causes such combination to be necessarily infringed. The patent license + shall not apply to any other combinations which include the + Contribution. + +Except as expressly stated above, no rights or licenses from any copyright +holder or contributor is granted under this license, whether expressly, by +implication, estoppel or otherwise. + +DISCLAIMER + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..74d4744 --- /dev/null +++ b/README.md @@ -0,0 +1,11 @@ +# UniXNU + +An independent project to run Darwin/XNU userland on non-Apple ARM64 hardware through UEFI. + +Work in progress, released in a hurry. There is no documented build yet: the scripts are not generic, +and you will have to work out how to build it yourself. The development host is macOS. + +Device adaptations are separate projects: + +- [UniXNU-sheng](https://github.com/UniXNU/UniXNU-sheng): Xiaomi Pad 6S Pro (sheng) +- [uefi-sheng](https://github.com/UniXNU/uefi-sheng): UEFI firmware for sheng (Mu-Silicium fork) diff --git a/drivers/ArmCpu/ArmCpu.cpp b/drivers/ArmCpu/ArmCpu.cpp new file mode 100644 index 0000000..a1f7bc6 --- /dev/null +++ b/drivers/ArmCpu/ArmCpu.cpp @@ -0,0 +1,823 @@ +// SPDX-License-Identifier: BSD-2-Clause-Patent +#include +#include +#include +#include +#include +#include +#include + +// Exact private entry from Darwin 23 arm/rtclock.h. Its full data header is +// not exported to IOKit; the IRQ path calls the real kernel implementation. +extern "C" void rtclock_intr (unsigned int); +// Kernel-internal entries used for secondary CPU startup (Darwin 23 osfmk). +extern "C" vm_offset_t ml_static_vtop (vm_offset_t); +extern "C" char LowResetVectorBase[], LowResetVectorEnd[], ResetHandlerData[], const_boot_args[]; +extern "C" kern_return_t processor_start (processor_t); +extern "C" kern_return_t kernel_thread_start (void (*) (void *, wait_result_t), void *, thread_t *); +extern "C" void thread_deallocate (thread_t); +extern "C" thread_t current_thread (void); +extern "C" kern_return_t thread_terminate (thread_t); + +extern "C" { +volatile uint32_t armcpu_registered = 0; +volatile uint32_t armcpu_gic_ready = 0; +volatile uint64_t armcpu_timer_interrupts = 0; +volatile uint64_t armcpu_ipi_interrupts = 0; +volatile uint64_t armcpu_ipi_sent = 0; // SGIs the kernel asked ArmCpu::signalCPU to send +volatile uint64_t armcpu_ipi_selftest_sent = 0; +// Boot core's own GIC registers read back once after enabling (own frame, the same one just written): 0 IGROUPR0, 1 ISENABLER0, 2 GICD_CTLR, +// 3 GICD_TYPER, 4 IGRPMODR0, 5 ICC_PMR_EL1, 6 ICC_IGRPEN1_EL1, 7 ICC_SRE_EL1. +volatile uint32_t armcpu_gic_boot[8]; +volatile uint32_t armcpu_core_timer[8], armcpu_core_sgi[8]; // per logical core: architectural timer interrupts and SGI 0 taken +static bool gSpiIpi = false; // boot-arg armcpu-spi-ipi=1: IPIs are software-pended SPIs 1008+core routed to that core (ICC_SGI1R_EL1 generates nothing on this board) +static bool gSelfIpi = false; // boot-arg armcpu-self-ipi=1: every core sends itself SGI 0 from initCPU +volatile uint32_t armcpu_irq_phase = 0,armcpu_irq_id = 1023; +volatile uint32_t armcpu_quiesce_calls = 0,armcpu_halt_calls = 0; +// Secondary-CPU bring-up evidence (sheng-smp=1). Step values: 1 nub started, 2 registered, 3 start thread running, +// 4 PSCI CPU_ON issued, 5 entered initCPU on the new core, 6 GIC redistributor ready, 7 running. +volatile uint32_t armcpu_smp_requested = 0, armcpu_smp_running = 0, armcpu_smp_conduit = 0; +volatile uint32_t armcpu_smp_step[8], armcpu_smp_mpidr[8], armcpu_smp_seen_mpidr[8], armcpu_smp_fail[8], armcpu_boot_mpidr = 0; +// System registers seen on each secondary core at initCPU entry: CurrentEL, SCTLR_EL1, TCR_EL1, MAIR_EL1 (low words), MIDR_EL1. +volatile uint32_t armcpu_smp_sys[8][5], armcpu_boot_sys[5]; +volatile int32_t armcpu_smp_psci_rc[8]; +} + +#define ARMCPU_SMP_SLOTS 8 + +class ArmCpu : public IOCPU { + OSDeclareDefaultStructors (ArmCpu) + OSSharedPtr distributor; + OSSharedPtr redistributors; + IOCPUInterruptController *controller = nullptr; + const OSSymbol *cpuName = nullptr; + uintptr_t dist = 0; + uintptr_t redist = 0; + uint64_t affinity = 0; + uintptr_t redistBase = 0; + uint64_t redistSize = 0; + bool skipGic = false; + bool diagnosePower = false; + bool secondary = false; + uint32_t logicalId = 0; + bool mapGic (void); + bool initDistributor (void); + bool wakeRedistributor (bool local); + bool finishSecondary (void); + static void drainPending (void); + bool prepareGic (void); + void startSecondary (void); + static void startThread (void *argument, wait_result_t); + void enableGic (void); + void interrupt (void); + static void dispatch (void *target, void *, void *, int); +public: + IOService *probe (IOService *provider, SInt32 *score) override; + bool start (IOService *provider) override; + void initCPU (bool boot) override; + void quiesceCPU (void) override; + void haltCPU (void) override; + kern_return_t startCPU (vm_offset_t, vm_offset_t) override; + void signalCPU (IOCPU *target) override; + uint64_t signalTarget (void) const { return affinity; } + void enableCPUTimeBase (bool enabled) override; + const OSSymbol *getCPUName (void) override; +}; + +OSDefineMetaClassAndStructors (ArmCpu, IOCPU) + +// Redistributor ISENABLER0: SGIs 0..15 (only SGI 0 is used by the kernel; the others let the self-SGI probe tell which INTIDs the GIC accepts) and PPI 27. +static const uint32_t kSgiPpiEnable = 0xffffU | (1U << 27); + +static const uint32_t kIpiSpiBase = 1008; // INTIDs 1008..1015 stand for 'IPI to logical core 0..7' + +static uint32_t read32 (uintptr_t base, unsigned offset) +{ + return *reinterpret_cast(base + offset); +} + +static void write32 (uintptr_t base, unsigned offset, uint32_t value) +{ + *reinterpret_cast(base + offset) = value; +} + +// Distributor side of the software IPI of logical core `core`: group 1, edge-triggered, priority 0x80, routed to affinity `aff`, enabled. +static void spiIpiConfigure (uintptr_t dist, uint32_t core, uint64_t aff) +{ + const uint32_t id = kIpiSpiBase + core; + write32 (dist, 0x80 + (id / 32) * 4, read32 (dist, 0x80 + (id / 32) * 4) | (1U << (id % 32))); + write32 (dist, 0xc00 + (id / 16) * 4, read32 (dist, 0xc00 + (id / 16) * 4) | (2U << ((id % 16) * 2))); + write32 (dist, 0x400 + (id / 4) * 4, 0x80808080); + *reinterpret_cast(dist + 0x6000 + 8ULL * id) = aff & 0xff00ffffffULL; + __asm__ volatile ("dsb sy" ::: "memory"); + write32 (dist, 0x100 + (id / 32) * 4, 1U << (id % 32)); + __asm__ volatile ("dsb sy" ::: "memory"); +} + +static void spiIpiSend (uintptr_t dist, uint32_t core) +{ + const uint32_t id = kIpiSpiBase + core; + __asm__ volatile ("dsb ishst" ::: "memory"); + write32 (dist, 0x200 + (id / 32) * 4, 1U << (id % 32)); + __asm__ volatile ("dsb sy" ::: "memory"); +} + +static uint64_t counter (void) +{ + uint64_t value; + __asm__ volatile ("isb; mrs %0, cntvct_el0" : "=r" (value)); + return value; +} + +static bool waitClear (uintptr_t base, unsigned offset, uint32_t mask) +{ + uint64_t frequency; + __asm__ volatile ("mrs %0, cntfrq_el0" : "=r" (frequency)); + uint64_t started = counter (); + while ((read32 (base, offset) & mask) != 0) { + if (counter () - started > frequency / 10) { + return false; + } + } + return true; +} + +// GIC state shared by all CPU nubs. The boot CPU maps the distributor and redistributor frames; a secondary core +// only needs the virtual addresses and its own redistributor frame. +static OSSharedPtr gDistMap, gRedistMap; +static uintptr_t gDist = 0, gFrames = 0; +static uint64_t gFramesSize = 0; +static IOCPUInterruptController *gController = nullptr; +static volatile uint32_t gGicShared = 0; +static uint64_t gRedistStride = 0; // DT redistributor-stride; 0 = infer from GICR_TYPER.VLPIS +volatile uint32_t armcpu_gic_stride = 0, armcpu_gic_typer_probe[2]; +static bool gSmpWanted = false; +// Secondary nubs never wait for the boot CPU (IOSleep needs the timer that only the boot nub enables): they queue here and +// the boot nub, or a late secondary once the boot CPU is registered, finishes them. +static class ArmCpu *volatile gPending[ARMCPU_SMP_SLOTS]; +static volatile uint32_t gBootRegistered = 0; +static bool gRemoteGic = false; // sheng-smp-gic=1: the boot CPU prepares a new core's redistributor before CPU_ON + +// Shared peripheral interrupts (GICv3 SPIs, INTID 32..1019). A driver attaches a handler for one INTID; the line is set to +// Group 1 non-secure, priority 0xa0, the given trigger and routed to the boot CPU (GICD_IROUTER, affinity routing is on), +// then enabled. The handler runs in the primary interrupt context, before the EOI, and must clear a level source. +// An SPI without a handler is counted and disabled instead of being acknowledged forever. +#define ARMCPU_SPI_SLOTS 16 +struct ArmCpuSpi {uint32_t id; void (*handler) (void *); void *context; volatile uint64_t count;}; +static ArmCpuSpi gSpi[ARMCPU_SPI_SLOTS]; +static volatile uint32_t gSpiUsed = 0; +static uint64_t gBootAffinity = 0; +extern "C" { +volatile uint64_t armcpu_spi_unhandled = 0; +volatile uint32_t armcpu_spi_unhandled_id = 0; +} + +IOService *ArmCpu::probe (IOService *provider, SInt32 *score) +{ + OSData *type = OSDynamicCast (OSData, provider->getProperty ("device_type")); + OSData *reg = OSDynamicCast (OSData, provider->getProperty ("reg")); + uint32_t id; + if ((type == nullptr) || (type->getLength () != 4) || + (memcmp (type->getBytesNoCopy (), "cpu", 4) != 0) || + (reg == nullptr) || (reg->getLength () < sizeof (id))) { + return nullptr; + } + memcpy (&id, reg->getBytesNoCopy (), sizeof (id)); + if (ml_get_cpu_number (id) != ml_get_boot_cpu_number ()) { + unsigned smp = 0; + PE_parse_boot_argn ("sheng-smp", &smp, sizeof (smp)); + if (smp == 0 || ml_get_cpu_number (id) < 0 || ml_get_cpu_number (id) >= ARMCPU_SMP_SLOTS) { + return nullptr; // Secondary startup is opt-in (sheng-smp=1). + } + } + return IOCPU::probe (provider, score); +} + +bool ArmCpu::mapGic (void) +{ + IORegistryEntry *soc = IORegistryEntry::fromPath ("/arm-io", gIODTPlane); + IORegistryEntry *gic = IORegistryEntry::fromPath ("/arm-io/intc", gIODTPlane); + if ((soc == nullptr) || (gic == nullptr)) { + if (soc != nullptr) { soc->release (); } + if (gic != nullptr) { gic->release (); } + return false; + } + OSData *ranges = OSDynamicCast (OSData, soc->getProperty ("ranges")); + OSData *reg = OSDynamicCast (OSData, gic->getProperty ("reg")); + OSData *compatible = OSDynamicCast (OSData, gic->getProperty ("compatible")); + uint64_t bus[3]; + uint64_t regions[4]; + bool valid = (ranges != nullptr) && (ranges->getLength () == sizeof (bus)) && + (reg != nullptr) && (reg->getLength () == sizeof (regions)) && + (compatible != nullptr) && (compatible->getLength () == 11) && + (memcmp (compatible->getBytesNoCopy (), "arm,gic-v3", 11) == 0); + if (valid) { + memcpy (bus, ranges->getBytesNoCopy (), sizeof (bus)); + memcpy (regions, reg->getBytesNoCopy (), sizeof (regions)); + } + OSData *strideProperty = OSDynamicCast (OSData, gic->getProperty ("redistributor-stride")); + if (strideProperty != nullptr && strideProperty->getLength () == sizeof (uint64_t)) { + memcpy (&gRedistStride, strideProperty->getBytesNoCopy (), sizeof (uint64_t)); + if (gRedistStride != 0x20000 && gRedistStride != 0x30000 && gRedistStride != 0x40000) { gRedistStride = 0; } + } + armcpu_gic_stride = static_cast (gRedistStride); + soc->release (); + gic->release (); + if (!valid || (regions[1] < 0x10000) || (regions[3] < 0x20000)) { + return false; + } + for (unsigned index = 0; index < 4; index += 2) { + if ((regions[index] < bus[0]) || (regions[index] - bus[0] > bus[2]) || + (regions[index + 1] > bus[2] - (regions[index] - bus[0])) || + (bus[1] + regions[index] - bus[0] < bus[1])) { + return false; + } + } + OSSharedPtr distMemory (IOMemoryDescriptor::withPhysicalAddress ( + bus[1] + regions[0] - bus[0], regions[1], kIODirectionInOut), OSNoRetain); + OSSharedPtr rdMemory (IOMemoryDescriptor::withPhysicalAddress ( + bus[1] + regions[2] - bus[0], regions[3], kIODirectionInOut), OSNoRetain); + if (!distMemory || !rdMemory) { + return false; + } + distributor = OSSharedPtr (distMemory->map (kIOMapAnywhere | kIOMapInhibitCache), OSNoRetain); + redistributors = OSSharedPtr (rdMemory->map (kIOMapAnywhere | kIOMapInhibitCache), OSNoRetain); + if (!distributor || !redistributors) { + return false; + } + dist = static_cast (distributor->getVirtualAddress ()); + if (((read32 (dist, 0xffe8) >> 4) & 15) != 3) { + return false; + } + redistBase = static_cast (redistributors->getVirtualAddress ()); + redistSize = regions[3]; + gDistMap = distributor; + gRedistMap = redistributors; + gDist = dist; + gFrames = redistBase; + gFramesSize = redistSize; + return true; +} + +bool ArmCpu::initDistributor (void) +{ + uint32_t ctl = read32 (dist, 0); + write32 (dist, 0, ctl & ~3U); + if (!waitClear (dist, 0, 1U << 31)) { return false; } + write32 (dist, 0, (ctl & ~3U) | 16U); + if (!waitClear (dist, 0, 1U << 31)) { return false; } + unsigned interrupts = ((read32 (dist, 4) & 31) + 1) * 32; + for (unsigned id = 32; id < interrupts && id < 1020; id += 32) { + write32 (dist, 0x180 + id / 8, 0xffffffff); + } + return true; +} + +// Runs on the core whose redistributor is configured: the frame is selected by that core's own MPIDR affinity. +bool ArmCpu::wakeRedistributor (bool local) +{ + uintptr_t frames = redistBase; + // local: this core finds its own frame from MPIDR. Remote: the boot CPU prepares a not-yet-started core's frame + // from the DT affinity, so the new core never needs GIC MMIO of its own. + if (local) { __asm__ volatile ("mrs %0, mpidr_el1" : "=r" (affinity)); } + uint32_t target = static_cast ((affinity & 0xffffff) | ((affinity >> 8) & 0xff000000)); + redist = 0; + for (uint64_t offset = 0; offset + 0x20000 <= redistSize;) { + armcpu_gic_typer_probe[0] = static_cast (offset); + uint64_t typer = *reinterpret_cast (frames + offset + 8); + armcpu_gic_typer_probe[1] = static_cast (typer); + if (static_cast (typer >> 32) == target) { + redist = frames + offset; + break; + } + if ((typer & 16) != 0) { break; } + offset += gRedistStride != 0 ? gRedistStride : ((typer & 2) != 0 ? 0x40000 : 0x20000); + } + if (redist == 0) { return false; } + write32 (redist, 0x14, read32 (redist, 0x14) & ~2U); + if (!waitClear (redist, 0x14, 4)) { return false; } + uintptr_t sgi = redist + 0x10000; + write32 (sgi, 0x180, 0xffffffff); + write32 (sgi, 0x80, 0xffffffff); + write32 (sgi, 0xd00, 0); + write32 (sgi, 0x280, 0xffffffff); + for (unsigned offset = 0x400; offset < 0x420; offset += 4) { + write32 (sgi, offset, 0x80808080); + } + if (!local) { write32 (sgi, 0x100, kSgiPpiEnable); } + __asm__ volatile ("dsb sy" ::: "memory"); + return waitClear (redist, 0, 8); +} + +bool ArmCpu::prepareGic (void) +{ + if (!mapGic () || !initDistributor () || !wakeRedistributor (true)) { return false; } + gGicShared = 1; + return true; +} + +void ArmCpu::enableGic (void) +{ + uint64_t value; + __asm__ volatile ("mrs %0, icc_sre_el1" : "=r" (value)); + value |= 1; + __asm__ volatile ("msr icc_sre_el1, %0; isb" :: "r" (value) : "memory"); + __asm__ volatile ("mrs %0, icc_sre_el1" : "=r" (value)); + if ((value & 1) == 0) { panic ("ArmCpu: GICv3 system register interface unavailable"); } + __asm__ volatile ("mrs %0, icc_ctlr_el1" : "=r" (value)); + value &= ~2ULL; // EOImode 0: EOIR also deactivates the interrupt. + __asm__ volatile ("msr icc_ctlr_el1, %0" :: "r" (value) : "memory"); + value = 0xff; + __asm__ volatile ("msr icc_pmr_el1, %0" :: "r" (value) : "memory"); + value = 0; + __asm__ volatile ("msr icc_bpr1_el1, %0" :: "r" (value) : "memory"); + if (!secondary) { gBootAffinity = affinity & 0xff00ffffffULL; } + if (!secondary || !gRemoteGic) { + write32 (redist + 0x10000, 0x100, kSgiPpiEnable); + write32 (dist, 0, read32 (dist, 0) | 0x12U); + } + value = 1; + __asm__ volatile ("msr icc_igrpen1_el1, %0; dsb sy; isb" :: "r" (value) : "memory"); + if (!secondary) { + uint64_t pmr, ig, sre; + __asm__ volatile ("mrs %0, icc_pmr_el1" : "=r" (pmr)); + __asm__ volatile ("mrs %0, icc_igrpen1_el1" : "=r" (ig)); + __asm__ volatile ("mrs %0, icc_sre_el1" : "=r" (sre)); + armcpu_gic_boot[0] = read32 (redist + 0x10000, 0x80); + armcpu_gic_boot[1] = read32 (redist + 0x10000, 0x100); + armcpu_gic_boot[2] = read32 (dist, 0); + armcpu_gic_boot[3] = read32 (dist, 4); + armcpu_gic_boot[4] = read32 (redist + 0x10000, 0xd00); + armcpu_gic_boot[5] = static_cast (pmr); + armcpu_gic_boot[6] = static_cast (ig); + armcpu_gic_boot[7] = static_cast (sre); + } + armcpu_gic_ready = 1; +} + + +bool ArmCpu::start (IOService *provider) +{ + unsigned skip = 0; + PE_parse_boot_argn ("armcpu-skip-gic", &skip, sizeof (skip)); + skipGic = skip != 0; + unsigned selfIpi = 0; + PE_parse_boot_argn ("armcpu-self-ipi", &selfIpi, sizeof (selfIpi)); + gSelfIpi = selfIpi == 1; + unsigned spiIpi = 0; + PE_parse_boot_argn ("armcpu-spi-ipi", &spiIpi, sizeof (spiIpi)); + gSpiIpi = spiIpi == 1; + unsigned diagnostic = 0; + PE_parse_boot_argn("armcpu-diagnostics", &diagnostic, sizeof(diagnostic)); + diagnosePower=diagnostic==1; + uint32_t physical; + OSData *reg = OSDynamicCast (OSData, provider->getProperty ("reg")); + memcpy (&physical, reg->getBytesNoCopy (), sizeof (physical)); + logicalId = static_cast (ml_get_cpu_number (physical)); + secondary = logicalId != static_cast (ml_get_boot_cpu_number ()); + if (secondary) { + if (!IOCPU::start (provider)) { return false; } + armcpu_smp_step[logicalId] = 1; + affinity = physical; + armcpu_smp_mpidr[logicalId] = physical; + cpuName = OSSymbol::withCStringNoCopy ("ARM architectural secondary CPU"); + if (cpuName == nullptr) { return false; } + retain (); + __atomic_store_n (&gPending[logicalId], this, __ATOMIC_RELEASE); + if (gBootRegistered != 0) { drainPending (); } + return true; + } else { + uint64_t bootMpidr; + __asm__ volatile ("mrs %0, mpidr_el1" : "=r" (bootMpidr)); + armcpu_boot_mpidr = static_cast (bootMpidr); + if ((!skipGic && !prepareGic ()) || !IOCPU::start (provider)) { return false; } + } + cpuName = OSSymbol::withCStringNoCopy ("ARM architectural boot CPU"); + if (cpuName == nullptr) { return false; } + ml_processor_info_t info = {}; + info.cpu_id = this; + info.phys_id = physical; + info.log_id = logicalId; + info.cluster_id = static_cast (ml_get_cluster_number (physical)); + info.cluster_type = ml_get_boot_cluster_type (); + perfmon_interrupt_handler_func pmi = nullptr; + kern_return_t status = ml_processor_register (&info, &machProcessor, &ipi_handler, &pmi); + if (status != KERN_SUCCESS) { panic ("ArmCpu: CPU %u registration failed: %d", logicalId, status); } + setCPUNumber (info.log_id); + armcpu_registered = 1; + unsigned smp = 0; + PE_parse_boot_argn ("sheng-smp", &smp, sizeof (smp)); + gSmpWanted = smp != 0; + unsigned remote = 0; + PE_parse_boot_argn ("sheng-smp-gic", &remote, sizeof (remote)); + gRemoteGic = remote != 0; + armcpu_smp_requested = gSmpWanted ? 1 : 0; + unsigned cpuCount = static_cast (ml_get_max_cpu_number ()) + 1; + controller = new IOCPUInterruptController; + if ((controller == nullptr) || (controller->initCPUInterruptController (cpuCount, 1) != kIOReturnSuccess)) { + panic ("ArmCpu: CPU interrupt controller allocation failed"); + } + controller->registerCPUInterruptController (); + controller->enableCPUInterrupt (this); + if (controller->registerInterrupt (provider, logicalId, this, &ArmCpu::dispatch, nullptr) != kIOReturnSuccess) { + panic ("ArmCpu: CPU IRQ action registration failed"); + } + gController = controller; + initCPU (true); + registerService (); + __atomic_store_n (&gBootRegistered, 1U, __ATOMIC_RELEASE); + drainPending (); + return true; +} + +void ArmCpu::drainPending (void) +{ + for (unsigned id = 1; id < ARMCPU_SMP_SLOTS; id++) { + ArmCpu *cpu = __atomic_exchange_n (&gPending[id], static_cast (nullptr), __ATOMIC_ACQ_REL); + if (cpu != nullptr) { cpu->finishSecondary (); } + } +} + +bool ArmCpu::finishSecondary (void) +{ + dist = gDist; + redistBase = gFrames; + redistSize = gFramesSize; + if (gRemoteGic && !wakeRedistributor (false)) { armcpu_smp_fail[logicalId] = 3; return false; } + ml_processor_info_t info = {}; + info.cpu_id = this; + info.phys_id = static_cast (affinity); + info.log_id = logicalId; + info.cluster_id = static_cast (ml_get_cluster_number (static_cast (affinity))); + info.cluster_type = ml_get_boot_cluster_type (); + perfmon_interrupt_handler_func pmi = nullptr; + kern_return_t status = ml_processor_register (&info, &machProcessor, &ipi_handler, &pmi); + if (status != KERN_SUCCESS) { armcpu_smp_fail[logicalId] = 4; return false; } + setCPUNumber (info.log_id); + armcpu_smp_step[logicalId] = 2; + setCPUState (kIOCPUStateUninitalized); + registerService (); + startSecondary (); + return true; +} + +// PSCI conduit comes from the /psci node of the device tree ("smc" or "hvc"); no default is assumed. +static bool psciConduit (bool *hvc) +{ + IORegistryEntry *psci = IORegistryEntry::fromPath ("/psci", gIODTPlane); + if (psci == nullptr) { return false; } + OSData *method = OSDynamicCast (OSData, psci->getProperty ("method")); + bool ok = false; + if (method != nullptr && method->getLength () == 4 && memcmp (method->getBytesNoCopy (), "smc", 4) == 0) { + *hvc = false; ok = true; + } else if (method != nullptr && method->getLength () == 4 && memcmp (method->getBytesNoCopy (), "hvc", 4) == 0) { + *hvc = true; ok = true; + } + psci->release (); + return ok; +} + +static int64_t psciCall (bool hvc, uint64_t function, uint64_t a, uint64_t b, uint64_t c) +{ + register uint64_t x0 __asm__ ("x0") = function; + register uint64_t x1 __asm__ ("x1") = a; + register uint64_t x2 __asm__ ("x2") = b; + register uint64_t x3 __asm__ ("x3") = c; + if (hvc) { + __asm__ volatile ("hvc #0" : "+r" (x0) : "r" (x1), "r" (x2), "r" (x3) : "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "memory"); + } else { + __asm__ volatile ("smc #0" : "+r" (x0) : "r" (x1), "r" (x2), "r" (x3) : "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "memory"); + } + return static_cast (x0); +} + +void ArmCpu::startThread (void *argument, wait_result_t) +{ + ArmCpu *self = static_cast (argument); + armcpu_smp_step[self->logicalId] = 3; + // Blocks until the new core reaches the scheduler; a core that never arrives only stalls this helper thread. + processor_start (self->machProcessor); + thread_terminate (current_thread ()); +} + +void ArmCpu::startSecondary (void) +{ + thread_t thread; + if (kernel_thread_start (&ArmCpu::startThread, this, &thread) != KERN_SUCCESS) { + armcpu_smp_psci_rc[logicalId] = -1000; + return; + } + thread_deallocate (thread); +} + +void ArmCpu::initCPU (bool boot) +{ + if (secondary) { + if (!boot) { panic ("ArmCpu: secondary resume is unsupported"); } + armcpu_smp_step[logicalId] = 5; + uint64_t mpidr; + __asm__ volatile ("mrs %0, mpidr_el1" : "=r" (mpidr)); + armcpu_smp_seen_mpidr[logicalId] = static_cast (mpidr); + // A failure is recorded and the core parked (rather than a panic that hides the other cores' state). + if ((mpidr & 0xffff) != (affinity & 0xffff)) { armcpu_smp_fail[logicalId] = 1; for (;;) { __asm__ volatile ("wfe"); } } + if (!gRemoteGic && !wakeRedistributor (true)) { armcpu_smp_fail[logicalId] = 2; for (;;) { __asm__ volatile ("wfe"); } } + armcpu_smp_step[logicalId] = 6; + enableGic (); + if (gSpiIpi) { spiIpiConfigure (gDist, logicalId, affinity); } + // The shared IOCPUInterruptController blocks registrations until every core is enabled, so the secondary cores + // install their handler directly. ml_install_interrupt_handler() records it in the calling core's cpu_data. + ml_install_interrupt_handler (this, static_cast (logicalId), this, reinterpret_cast (&ArmCpu::dispatch), nullptr); + enableCPUTimeBase (true); + setCPUState (kIOCPUStateRunning); + armcpu_smp_step[logicalId] = 7; + armcpu_smp_running = armcpu_smp_running + 1; + // XNU clears SIGPdisabled for a core the first time that core takes an IPI, and cpu_signal() silently skips a core until then; Apple's + // CPU driver provokes it with a self-IPI from initCPU (osfmk/kern/machine.c, processor_up). + if (gSelfIpi) { signalCPU (this); } + return; + } + if (!boot) { panic ("ArmCpu: secondary/resume CPU startup is unsupported"); } + if (skipGic) { + // Explicit A/B hardware diagnostic: do not touch GIC MMIO or enable an + // interrupt source. CPU registration still happens; the timer is left off. + setCPUState (kIOCPUStateRunning); + return; + } + enableGic (); + if (gSpiIpi) { spiIpiConfigure (gDist, logicalId, affinity); } + enableCPUTimeBase (true); + setCPUState (kIOCPUStateRunning); + if (gSelfIpi) { signalCPU (this); } +} + +void ArmCpu::dispatch (void *target, void *, void *, int) +{ + static_cast (target)->interrupt (); +} + +void ArmCpu::interrupt (void) +{ + unsigned drained=0; + for (;;) { + armcpu_irq_phase=1; + uint64_t id; + __asm__ volatile ("mrs %0, icc_iar1_el1" : "=r" (id)); + armcpu_irq_id=static_cast(id & 0xffffff); + if ((id & 0xffffff) >= 1020) { armcpu_irq_phase=0;break; } + if(diagnosePower && ++drained>128)panic("ArmCpu: IRQ drain exceeds 128 id=%llx timer=%llu IPI=%llu",id,armcpu_timer_interrupts,armcpu_ipi_interrupts); + if ((id & 0xffffff) == 27) { + armcpu_timer_interrupts = armcpu_timer_interrupts + 1; + armcpu_core_timer[logicalId & 7] = armcpu_core_timer[logicalId & 7] + 1; + armcpu_irq_phase=2; + rtclock_intr (0); + armcpu_irq_phase=3; + } else if ((id & 0xffffff) == 0) { + armcpu_ipi_interrupts = armcpu_ipi_interrupts + 1; + armcpu_core_sgi[logicalId & 7] = armcpu_core_sgi[logicalId & 7] + 1; + armcpu_irq_phase=4; + if (ipi_handler != nullptr) { ipi_handler (); } + armcpu_irq_phase=5; + } else if (gSpiIpi && (id & 0xffffff) >= kIpiSpiBase && (id & 0xffffff) < kIpiSpiBase + 8) { + armcpu_ipi_interrupts = armcpu_ipi_interrupts + 1; + armcpu_core_sgi[logicalId & 7] = armcpu_core_sgi[logicalId & 7] + 1; + armcpu_irq_phase = 4; + if (ipi_handler != nullptr) { ipi_handler (); } + armcpu_irq_phase = 5; + } else { + const uint32_t intid = static_cast (id & 0xffffff); + ArmCpuSpi *spi = nullptr; + const uint32_t used = __atomic_load_n (&gSpiUsed, __ATOMIC_ACQUIRE); + for (uint32_t slot = 0; intid >= 32 && slot < used && spi == nullptr; slot++) { + if (gSpi[slot].id == intid) { spi = &gSpi[slot]; } + } + if (spi != nullptr) { + armcpu_irq_phase = 7; + spi->count = spi->count + 1; + spi->handler (spi->context); + } else if (intid >= 32 && gDist != 0) { + armcpu_spi_unhandled = armcpu_spi_unhandled + 1; + armcpu_spi_unhandled_id = intid; + write32 (gDist, 0x180 + (intid / 32) * 4, 1U << (intid % 32)); + } else { + panic ("ArmCpu: unexpected architectural IRQ %llu", id); + } + } + armcpu_irq_phase=6; + __asm__ volatile ("msr icc_eoir1_el1, %0; isb" :: "r" (id) : "memory"); + } +} + +void ArmCpu::signalCPU (IOCPU *target) +{ + ArmCpu *peer = OSDynamicCast (ArmCpu, target); + if (peer == nullptr || (peer->signalTarget () & 0xff) >= 16) { + panic ("ArmCpu: unsupported CPU signal target"); + } + uint64_t aff = peer->signalTarget (); + uint64_t sgi = ((aff & 0xff00000000ULL) << 16) | ((aff & 0xff0000) << 16) | + ((aff & 0xff00) << 8) | (1ULL << (aff & 0xff)); + armcpu_ipi_sent = armcpu_ipi_sent + 1; + if (gSpiIpi) { spiIpiSend (gDist, peer->logicalId & 7); return; } + __asm__ volatile ("dsb ishst; msr icc_sgi1r_el1, %0; isb" :: "r" (sgi) : "memory"); +} + +// Diagnostic: sends SGI 0 to the core whose MPIDR affinity is aff (the DT "reg" value, e.g. 0x100 for core 1) exactly as signalCPU +// does. The handler on the target runs the kernel's IPI callback with nothing pending, which is harmless. Returns nothing: the caller +// compares armcpu_ipi_interrupts before and after. +// Read-only snapshot of the GIC state that decides whether SGI 0 reaches the core with MPIDR affinity aff: its redistributor's SGI frame and +// the distributor control register. out: 0 IGROUPR0, 1 ISENABLER0, 2 ISPENDR0, 3 IGRPMODR0, 4 GICR_WAKER, 5 GICD_CTLR, 6 IPRIORITYR0, 7 found(1). +// armcpu_gic_diag_stage names the access in progress, so a read that never returns is identified from another thread: +// 0x100+frame walk, 2 IGROUPR0, 3 ISENABLER0, 4 ISPENDR0, 5 IGRPMODR0, 6 WAKER, 7 GICD_CTLR, 8 IPRIORITYR0, 9 done, 0xe not found. +volatile uint32_t armcpu_gic_diag_stage = 0; +extern "C" void armcpu_gic_diag (uint64_t aff, uint32_t *out) +{ + for (unsigned i = 0; i < 8; i++) { out[i] = 0xdeadbeef; } + if (gDist == 0 || gFrames == 0) { armcpu_gic_diag_stage = 0xe; return; } + const uint32_t target = static_cast ((aff & 0xffffff) | ((aff >> 8) & 0xff000000)); + for (uint64_t offset = 0; offset + 0x20000 <= gFramesSize;) { + armcpu_gic_diag_stage = 0x100 + static_cast (offset >> 16); + const uint64_t typer = *reinterpret_cast (gFrames + offset + 8); + if (static_cast (typer >> 32) == target) { + const uintptr_t redist = gFrames + offset, sgi = redist + 0x10000; + armcpu_gic_diag_stage = 2; out[0] = read32 (sgi, 0x80); + armcpu_gic_diag_stage = 3; out[1] = read32 (sgi, 0x100); + armcpu_gic_diag_stage = 4; out[2] = read32 (sgi, 0x200); + armcpu_gic_diag_stage = 5; out[3] = read32 (sgi, 0xd00); + armcpu_gic_diag_stage = 6; out[4] = read32 (redist, 0x14); + armcpu_gic_diag_stage = 7; out[5] = read32 (gDist, 0); + armcpu_gic_diag_stage = 8; out[6] = read32 (sgi, 0x400); + out[7] = 1; + armcpu_gic_diag_stage = 9; + return; + } + if ((typer & 16) != 0) { break; } + offset += gRedistStride != 0 ? gRedistStride : ((typer & 2) != 0 ? 0x40000 : 0x20000); + } + armcpu_gic_diag_stage = 0xe; +} + +// Does an SGI sent to the calling core itself become pending at its own CPU interface, and which INTIDs? Only system registers are used (no GIC +// MMIO). With IRQs masked, SGIs 0..15 are sent one at a time to this core's own MPIDR affinity through ICC_SGI1R_EL1 (Group 1) and, when that +// shows nothing, through ICC_SGI0R_EL1 (Group 0); ICC_HPPIR1_EL1 / ICC_HPPIR0_EL1 is polled and a pending SGI is acknowledged and ended here so the +// kernel never sees it. out[8], out[9]: CNTVCT ticks of one ICC_SGI1R_EL1 write and of one ICC_PMR_EL1 read (a trapped write costs far more). out: 0 MPIDR, 1 INTIDs pending as Group 1, 2 INTIDs pending as Group 0, 3 ICC_PMR, 4 ICC_IGRPEN1, 5 ICC_SRE, 6 ICC_CTLR, 7 ICC_RPR. +extern "C" void armcpu_sgi_selfprobe (uint64_t *out) +{ + uint64_t mpidr, daif, v; + __asm__ volatile ("mrs %0, mpidr_el1" : "=r" (mpidr)); + __asm__ volatile ("mrs %0, daif; msr daifset, #2; isb" : "=r" (daif) :: "memory"); + for (unsigned i = 0; i < 10; i++) { out[i] = 0; } + out[0] = mpidr; + __asm__ volatile ("mrs %0, icc_pmr_el1" : "=r" (v)); out[3] = v; + __asm__ volatile ("mrs %0, icc_igrpen1_el1" : "=r" (v)); out[4] = v; + __asm__ volatile ("mrs %0, icc_sre_el1" : "=r" (v)); out[5] = v; + __asm__ volatile ("mrs %0, icc_ctlr_el1" : "=r" (v)); out[6] = v; + __asm__ volatile ("mrs %0, icc_rpr_el1" : "=r" (v)); out[7] = v; + if ((mpidr & 0xff) < 16) { + const uint64_t base = ((mpidr & 0xff00000000ULL) << 16) | ((mpidr & 0xff0000) << 16) | ((mpidr & 0xff00) << 8) | (1ULL << (mpidr & 0xff)); + for (uint64_t intid = 0; intid < 16; intid++) { + const uint64_t sgi = base | (intid << 24); + uint64_t h = 1023, ta, tb; + __asm__ volatile ("dsb ishst; isb; mrs %0, cntvct_el0; isb" : "=r" (ta) :: "memory"); + __asm__ volatile ("msr icc_sgi1r_el1, %0; isb; mrs %1, cntvct_el0" : "=r" (tb) : "r" (sgi) : "memory"); + if (intid == 0) { out[8] = tb - ta; } + for (unsigned polls = 0; polls < 400; polls++) { + __asm__ volatile ("mrs %0, icc_hppir1_el1" : "=r" (h)); + if ((h & 0xffffff) == intid) { break; } + __asm__ volatile ("isb" ::: "memory"); + } + if ((h & 0xffffff) == intid) { + out[1] |= 1ULL << intid; + __asm__ volatile ("mrs %0, icc_iar1_el1" : "=r" (v)); + __asm__ volatile ("msr icc_eoir1_el1, %0; isb" :: "r" (v) : "memory"); + continue; + } + __asm__ volatile ("dsb ishst; msr icc_sgi0r_el1, %0; isb" :: "r" (sgi) : "memory"); + for (unsigned polls = 0; polls < 400; polls++) { + __asm__ volatile ("mrs %0, icc_hppir0_el1" : "=r" (h)); + if ((h & 0xffffff) == intid) { break; } + __asm__ volatile ("isb" ::: "memory"); + } + if ((h & 0xffffff) == intid) { + out[2] |= 1ULL << intid; + __asm__ volatile ("mrs %0, icc_iar0_el1" : "=r" (v)); + __asm__ volatile ("msr icc_eoir0_el1, %0; isb" :: "r" (v) : "memory"); + } + } + } + { + uint64_t ta, tb; + __asm__ volatile ("isb; mrs %0, cntvct_el0; isb; mrs %2, icc_pmr_el1; isb; mrs %1, cntvct_el0" : "=r" (ta), "=r" (tb), "=r" (v) :: "memory"); + out[9] = tb - ta; + } + __asm__ volatile ("msr daif, %0; isb" :: "r" (daif) : "memory"); +} + +extern "C" void armcpu_ipi_selftest (uint64_t aff) +{ + uint64_t sgi = ((aff & 0xff00000000ULL) << 16) | ((aff & 0xff0000) << 16) | ((aff & 0xff00) << 8) | (1ULL << (aff & 0xff)); + armcpu_ipi_selftest_sent = armcpu_ipi_selftest_sent + 1; + if (gSpiIpi) { spiIpiSend (gDist, static_cast ((aff >> 8) & 7)); return; } + __asm__ volatile ("dsb ishst; msr icc_sgi1r_el1, %0; isb" :: "r" (sgi) : "memory"); +} + +void ArmCpu::enableCPUTimeBase (bool enabled) +{ + uint64_t value = enabled ? 1 : 2; + __asm__ volatile ("msr cntv_ctl_el0, %0; isb" :: "r" (value) : "memory"); +} + +void ArmCpu::quiesceCPU (void) +{ + armcpu_quiesce_calls=armcpu_quiesce_calls+1; + if(diagnosePower)panic("ArmCpu: unsupported quiesce requested before timer/GIC disable caller=%p halt=%u timer=%llu phase=%u",__builtin_return_address(0),armcpu_halt_calls,armcpu_timer_interrupts,armcpu_irq_phase); + enableCPUTimeBase (false); + uint64_t value = 0; + __asm__ volatile ("msr icc_igrpen1_el1, %0; dsb sy; isb" :: "r" (value) : "memory"); +} + +void ArmCpu::haltCPU (void) +{ + armcpu_halt_calls=armcpu_halt_calls+1; + quiesceCPU (); + setCPUState (kIOCPUStateStopped); + __asm__ volatile ("msr daifset, #15" ::: "memory"); + for (;;) { __asm__ volatile ("wfi"); } +} + +// The new core runs with its MMU and caches off until start_cpu enables them, so it reads the reset vector, ResetHandlerData +// and the boot arguments from memory. Write those lines back to the point of coherency first: on this SoC the boot CPU's +// dirty cache lines are not guaranteed to be visible to a core with caches off (XNU only flushes the per-CPU data). +static void cleanToMemory (const void *start, size_t length) +{ + uint64_t ctr; + __asm__ volatile ("mrs %0, ctr_el0" : "=r" (ctr)); + uintptr_t line = static_cast (4) << ((ctr >> 16) & 15); + uintptr_t address = reinterpret_cast (start) & ~(line - 1), end = reinterpret_cast (start) + length; + for (; address < end; address += line) { __asm__ volatile ("dc civac, %0" :: "r" (address) : "memory"); } + __asm__ volatile ("dsb sy; isb" ::: "memory"); +} + +kern_return_t ArmCpu::startCPU (vm_offset_t, vm_offset_t) +{ + if (!secondary) { return KERN_NOT_SUPPORTED; } + bool hvc = false; + if (!psciConduit (&hvc)) { armcpu_smp_psci_rc[logicalId] = -1001; return KERN_NOT_SUPPORTED; } + armcpu_smp_conduit = hvc ? 2 : 1; + // PSCI 0.2 CPU_ON (SMC64 function id); the entry point is the kernel's physical reset vector page, whose reset + // code selects this core's cpu_data entry by MPIDR affinity. The context id is unused by that code. + uint64_t entry = static_cast (ml_static_vtop (reinterpret_cast (LowResetVectorBase))); + cleanToMemory (LowResetVectorBase, 0x10000); // reset vector, start_cpu and the code up to the MMU enable + cleanToMemory (ResetHandlerData, 0x1000); + cleanToMemory (const_boot_args, 0x1000); + armcpu_smp_step[logicalId] = 4; + int64_t result = psciCall (hvc, 0xc4000003ULL, affinity & 0xff00ffffffULL, entry, 0); + armcpu_smp_psci_rc[logicalId] = static_cast (result); + return result == 0 ? KERN_SUCCESS : KERN_FAILURE; +} + +const OSSymbol *ArmCpu::getCPUName (void) +{ + return cpuName; +} + +// Readback after configuration: bit 0 group 1, bit 1 enabled, bit 2 edge, bits 8..15 priority; IROUTER separately. +extern "C" IOReturn armcpu_spi_attach (uint32_t intid, bool edge, void (*handler) (void *), void *context, + uint32_t *readback, uint64_t *router) +{ + if (gDist == 0 || armcpu_gic_ready == 0 || handler == nullptr) { return kIOReturnNotReady; } + const unsigned interrupts = ((read32 (gDist, 4) & 31) + 1) * 32; + if (intid < 32 || intid >= 1020 || intid >= interrupts) { return kIOReturnBadArgument; } + const uint32_t used = __atomic_load_n (&gSpiUsed, __ATOMIC_ACQUIRE); + for (uint32_t slot = 0; slot < used; slot++) { + if (gSpi[slot].id == intid) { return kIOReturnExclusiveAccess; } + } + if (used >= ARMCPU_SPI_SLOTS) { return kIOReturnNoResources; } + const uint32_t word = (intid / 32) * 4, bit = 1U << (intid % 32); + write32 (gDist, 0x180 + word, bit); // ICENABLER while it is reconfigured + if (!waitClear (gDist, 0, 1U << 31)) { return kIOReturnTimeout; } + write32 (gDist, 0x80 + word, read32 (gDist, 0x80 + word) | bit); // IGROUPR: group 1 (RAZ/WI if secure owns it) + *reinterpret_cast (gDist + 0x400 + intid) = 0xa0; + const unsigned cfg = 0xc00 + (intid / 16) * 4, shift = (intid % 16) * 2 + 1; + write32 (gDist, cfg, (read32 (gDist, cfg) & ~(1U << shift)) | ((edge ? 1U : 0U) << shift)); + *reinterpret_cast (gDist + 0x6000 + 8 * intid) = gBootAffinity; + gSpi[used] = {intid, handler, context, 0}; + __atomic_store_n (&gSpiUsed, used + 1, __ATOMIC_RELEASE); + __asm__ volatile ("dsb sy" ::: "memory"); + write32 (gDist, 0x100 + word, bit); // ISENABLER + if (readback != nullptr) { + *readback = ((read32 (gDist, 0x80 + word) & bit) ? 1U : 0U) | ((read32 (gDist, 0x100 + word) & bit) ? 2U : 0U) | + (((read32 (gDist, cfg) >> shift) & 1U) << 2) | + (static_cast (*reinterpret_cast (gDist + 0x400 + intid)) << 8); + } + if (router != nullptr) { *router = *reinterpret_cast (gDist + 0x6000 + 8 * intid); } + return kIOReturnSuccess; +} + +extern "C" uint64_t armcpu_spi_count (uint32_t intid) +{ + const uint32_t used = __atomic_load_n (&gSpiUsed, __ATOMIC_ACQUIRE); + for (uint32_t slot = 0; slot < used; slot++) { + if (gSpi[slot].id == intid) { return gSpi[slot].count; } + } + return 0; +} diff --git a/drivers/ArmCpu/Info.plist b/drivers/ArmCpu/Info.plist new file mode 100644 index 0000000..9047b1d --- /dev/null +++ b/drivers/ArmCpu/Info.plist @@ -0,0 +1,15 @@ + + + + CFBundleIdentifierorg.blu-sharky.ArmCpu + CFBundleNameArmCpu + CFBundleExecutableArmCpu + CFBundlePackageTypeKEXT + CFBundleVersion0.1.0 + OSBundleRequiredRoot + IOKitPersonalitiesArchitectural boot CPU + IOClassArmCpu + IOProviderClassIOPlatformDevice + IOProbeScore2000 + + diff --git a/drivers/ArmCpu/LICENSE b/drivers/ArmCpu/LICENSE new file mode 100644 index 0000000..6110342 --- /dev/null +++ b/drivers/ArmCpu/LICENSE @@ -0,0 +1,51 @@ +Copyright (c) 2026, ArmCpu contributors. All rights reserved. + +SPDX-License-Identifier: BSD-2-Clause-Patent + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +Subject to the terms and conditions of this license, each copyright holder +and contributor hereby grants to those receiving rights under this license +a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable +(except for failure to satisfy the conditions of this license) patent +license to make, have made, use, offer to sell, sell, import, and otherwise +transfer this software, where such license applies only to those patent +claims, already acquired or hereafter acquired, licensable by such copyright +holder or contributor that are necessarily infringed by: + +(a) their Contribution(s) (the licensed copyrights of copyright holders and + non-copyrightable additions of contributors, in source or binary form) + alone; or + +(b) combination of their Contribution(s) with the work of authorship to + which such Contribution(s) was added by such copyright holder or + contributor, if, at the time the Contribution is added, such addition + causes such combination to be necessarily infringed. The patent license + shall not apply to any other combinations which include the + Contribution. + +Except as expressly stated above, no rights or licenses from any copyright +holder or contributor is granted under this license, whether expressly, by +implication, estoppel or otherwise. + +DISCLAIMER + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. diff --git a/drivers/ArmDTPlatform/ArmDTPlatform.cpp b/drivers/ArmDTPlatform/ArmDTPlatform.cpp new file mode 100644 index 0000000..fdafc43 --- /dev/null +++ b/drivers/ArmDTPlatform/ArmDTPlatform.cpp @@ -0,0 +1,210 @@ +// SPDX-License-Identifier: BSD-2-Clause-Patent +// +// Generic AArch64 Darwin platform expert for a supplied Apple-format device +// tree. It derives from XNU's real IODTPlatformExpert and relies on the +// platform's own DT root name ("device-tree") for matching. No board name, +// SoC address or vendor identity is compiled into this source. +// +// The DT must contain the nodes XNU's IODT platform path consumes: +// /chosen with a CHRP NVRAM proxy (nvram-bank-size, nvram-proxy-data) +// /options, /cpus, /arm-io and /defaults +// The NVRAM proxy is required so the real IODTNVRAM is published before any +// security initialization. This driver does not fabricate NVRAM or suppress +// a missing one; absent inputs make configure() fail. + +#include +#include +#include +#include +#include +#include + +// Observation counters describe actual published nubs, not started drivers. +extern "C" { +volatile uint32_t armdt_platform_configured = 0; +volatile uint32_t armdt_platform_child_count = 0; +volatile uint32_t armdt_platform_cpu_count = 0; +volatile uint32_t armdt_platform_bus_count = 0; +volatile uint32_t armdt_platform_defaults_count = 0; +volatile uint32_t armdt_platform_nvram_proxy = 0; +volatile uint32_t armdt_platform_device_count = 0; +} + +class ArmDTPlatform : public IODTPlatformExpert { + OSDeclareDefaultStructors (ArmDTPlatform) +public: + bool configure (IOService *provider) override; + const char *deleteList (void) override; + const char *excludeList (void) override; +private: + bool hasNVRAMProxy (IOService *provider); + bool createDeviceNubs (IOService *provider); +}; + +OSDefineMetaClassAndStructors (ArmDTPlatform, IODTPlatformExpert) + +// Serialized empty name lists: XNU's processTopLevel() treats a null pointer +// as "every entry" (remove every child), not as "nothing". An empty list +// genuinely selects no entry, so the supplied device tree is preserved. +const char *ArmDTPlatform::deleteList (void) +{ + return "()"; +} + +const char *ArmDTPlatform::excludeList (void) +{ + return "()"; +} + +bool ArmDTPlatform::hasNVRAMProxy (IOService *provider) +{ + IORegistryEntry *chosen = provider->childFromPath ("chosen", gIODTPlane); + if (chosen == nullptr) { + return false; + } + OSData *bankSize = OSDynamicCast (OSData, chosen->getProperty ("nvram-bank-size")); + OSData *proxyData = OSDynamicCast (OSData, chosen->getProperty ("nvram-proxy-data")); + bool valid = (bankSize != nullptr) && (bankSize->getLength () == sizeof (uint32_t)) && + (proxyData != nullptr) && (proxyData->getLength () != 0); + chosen->release (); + return valid; +} + +bool ArmDTPlatform::createDeviceNubs (IOService *provider) +{ + IORegistryEntry *bus = provider->childFromPath ("arm-io", gIODTPlane); + if (bus == nullptr) { + return false; + } + // IODTPlatformExpert's IOPlatformDevice nubs resolve resources through + // their provider's IOPlatformExpert::getNubResources. They must therefore + // be attached to this platform expert, not to the arm-io IOPlatformDevice + // (ArmVirtPlatform uses the same parent relationship). The device-tree + // parent remains arm-io, so reg/ranges resolution still follows the bus. + // Publish every real arm-io child, not only the first pattern match. The + // GIC lives here, and the UFS host/PHY nodes must be visible to their native + // drivers too. processTopLevel() still uses deleteList()/excludeList(). + OSIterator *iterator = bus->getChildIterator (gIODTPlane); + bus->release (); + if (iterator == nullptr) { + return false; + } + uint32_t count = 0; + IORegistryEntry *entry; + while ((entry = OSDynamicCast (IORegistryEntry, iterator->getNextObject ())) != nullptr) { + IOService *nub = createNub (entry); + if (nub == nullptr) { + continue; + } + if (!nub->attach (this)) { + nub->release (); + continue; + } + nub->registerService (); + nub->release (); + count++; + } + iterator->release (); + armdt_platform_device_count = count; + return true; +} + +static void platformStageReset (void) +{ + // PSCI SYSTEM_RESET via the same SMC conduit as the entry reset probe. + register uint64_t x0 __asm__ ("x0") = 0x84000009; + register uint64_t x1 __asm__ ("x1") = 0; + register uint64_t x2 __asm__ ("x2") = 0; + register uint64_t x3 __asm__ ("x3") = 0; + __asm__ volatile ("smc #0" : "+r" (x0) : "r" (x1), "r" (x2), "r" (x3) : "memory"); + for (;;) { __asm__ volatile ("wfe"); } +} + +bool ArmDTPlatform::configure (IOService *provider) +{ + IORegistryEntry *cpus = provider->childFromPath ("cpus", gIODTPlane); + IORegistryEntry *bus = provider->childFromPath ("arm-io", gIODTPlane); + IORegistryEntry *defaults = provider->childFromPath ("defaults", gIODTPlane); + IORegistryEntry *options = provider->childFromPath ("options", gIODTPlane); + bool structure = (cpus != nullptr) && (bus != nullptr) && + (defaults != nullptr) && (options != nullptr); + if (cpus != nullptr) { cpus->release (); } + if (bus != nullptr) { bus->release (); } + if (defaults != nullptr) { defaults->release (); } + if (options != nullptr) { options->release (); } + if (!structure) { + IOLog ("ArmDTPlatform: required /cpus, /arm-io, /defaults or /options node is absent\n"); + return false; + } + if (!hasNVRAMProxy (provider)) { + IOLog ("ArmDTPlatform: /chosen does not supply a usable CHRP NVRAM proxy\n"); + return false; + } + unsigned platformStage = 0; + PE_parse_boot_argn ("sheng-platform-stage", &platformStage, sizeof (platformStage)); + if (platformStage == 1) { + platformStageReset (); + } + + // The inherited implementation publishes real IODTNVRAM from /chosen, + // then attaches the CPU and top-level nubs. No identity or resource is + // invented here; failure is returned instead of replaced with a stub. + if (!IODTPlatformExpert::configure (provider)) { + return false; + } + if (platformStage == 2) { + platformStageReset (); + } + + OSIterator *iterator = getChildIterator (gIOServicePlane); + if (iterator == nullptr) { + return false; + } + uint32_t count = 0; + uint32_t cpuCount = 0; + uint32_t busCount = 0; + uint32_t defaultsCount = 0; + IORegistryEntry *child; + while ((child = OSDynamicCast (IORegistryEntry, iterator->getNextObject ())) != nullptr) { + count++; + OSData *type = OSDynamicCast (OSData, child->getProperty ("device_type")); + if ((type != nullptr) && (type->getLength () == 4) && + (memcmp (type->getBytesNoCopy (), "cpu", 4) == 0)) { + cpuCount++; + } + if (strcmp (child->getName (), "arm-io") == 0) { + busCount++; + } + if (strcmp (child->getName (), "defaults") == 0) { + defaultsCount++; + } + } + iterator->release (); + armdt_platform_child_count = count; + armdt_platform_cpu_count = cpuCount; + armdt_platform_bus_count = busCount; + armdt_platform_defaults_count = defaultsCount; + armdt_platform_nvram_proxy = 1; + armdt_platform_configured = 1; + // Publish the bus child devices (interrupt controller, VirtIO MMIO, ...) + // as real IOPlatformDevice nubs. No driver is started here. + if (!createDeviceNubs (provider)) { + return false; + } + if (platformStage == 3) { + platformStageReset (); + } + // Explicit hardware A/B diagnostic: reach the end of the real platform + // configuration and panic before publishing the service. The payload's + // panic-screen probe makes this observable without a serial console. + unsigned platformProbe = 0; + PE_parse_boot_argn ("sheng-platform-panic", &platformProbe, sizeof (platformProbe)); + if (platformProbe != 0 || platformStage == 4) { + platformStageReset (); + } + registerService (); + if (platformStage == 5) { + platformStageReset (); + } + return true; +} diff --git a/drivers/ArmDTPlatform/Info.plist b/drivers/ArmDTPlatform/Info.plist new file mode 100644 index 0000000..97281bd --- /dev/null +++ b/drivers/ArmDTPlatform/Info.plist @@ -0,0 +1,18 @@ + + + + CFBundleIdentifierorg.blu-sharky.ArmDTPlatform + CFBundleNameArmDTPlatform + CFBundleExecutableArmDTPlatform + CFBundlePackageTypeKEXT + CFBundleVersion0.1.0 + OSBundleRequiredRoot + IOKitPersonalities + DT platform root + IOClassArmDTPlatform + IOProviderClassIOPlatformExpertDevice + IONameMatchdevice-tree + IOProbeScore5000 + + + diff --git a/drivers/ArmDTPlatform/LICENSE b/drivers/ArmDTPlatform/LICENSE new file mode 100644 index 0000000..217b4df --- /dev/null +++ b/drivers/ArmDTPlatform/LICENSE @@ -0,0 +1,51 @@ +Copyright (c) 2026, ArmVirtPlatform contributors. All rights reserved. + +SPDX-License-Identifier: BSD-2-Clause-Patent + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +Subject to the terms and conditions of this license, each copyright holder +and contributor hereby grants to those receiving rights under this license +a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable +(except for failure to satisfy the conditions of this license) patent +license to make, have made, use, offer to sell, sell, import, and otherwise +transfer this software, where such license applies only to those patent +claims, already acquired or hereafter acquired, licensable by such copyright +holder or contributor that are necessarily infringed by: + +(a) their Contribution(s) (the licensed copyrights of copyright holders and + non-copyrightable additions of contributors, in source or binary form) + alone; or + +(b) combination of their Contribution(s) with the work of authorship to + which such Contribution(s) was added by such copyright holder or + contributor, if, at the time the Contribution is added, such addition + causes such combination to be necessarily infringed. The patent license + shall not apply to any other combinations which include the + Contribution. + +Except as expressly stated above, no rights or licenses from any copyright +holder or contributor is granted under this license, whether expressly, by +implication, estoppel or otherwise. + +DISCLAIMER + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. diff --git a/drivers/ArmVirtPlatform/ArmVirtPlatform.cpp b/drivers/ArmVirtPlatform/ArmVirtPlatform.cpp new file mode 100644 index 0000000..ec376dd --- /dev/null +++ b/drivers/ArmVirtPlatform/ArmVirtPlatform.cpp @@ -0,0 +1,143 @@ +// SPDX-License-Identifier: BSD-2-Clause-Patent +#include +#include +#include +#include +#include + +// Observation counters describe real attached nubs, not initialized drivers. +extern "C" { +volatile uint32_t armvirt_platform_configured = 0; +volatile uint32_t armvirt_platform_child_count = 0; +volatile uint32_t armvirt_platform_cpu_count = 0; +volatile uint32_t armvirt_platform_bus_count = 0; +volatile uint32_t armvirt_platform_defaults_count = 0; +} + +class ArmVirtPlatform : public IODTPlatformExpert { + OSDeclareDefaultStructors (ArmVirtPlatform) +public: + bool configure (IOService *provider) override; + const char *deleteList (void) override; + const char *excludeList (void) override; +private: + bool createVirtioNubs (IORegistryEntry *bus); +}; + +OSDefineMetaClassAndStructors (ArmVirtPlatform, IODTPlatformExpert) + +const char *ArmVirtPlatform::deleteList (void) +{ + return nullptr; +} + +const char *ArmVirtPlatform::excludeList (void) +{ + return nullptr; +} + +bool ArmVirtPlatform::createVirtioNubs (IORegistryEntry *bus) +{ + OSIterator *iterator = IODTFindMatchingEntries (bus, kIODTExclusive, nullptr); + if (iterator == nullptr) { return true; } + bool result = true; + IORegistryEntry *entry; + while ((entry = OSDynamicCast (IORegistryEntry, iterator->getNextObject ())) != nullptr) { + OSData *compatible = OSDynamicCast (OSData, entry->getProperty ("compatible")); + if ((compatible == nullptr) || (compatible->getLength () != 12) || + (memcmp (compatible->getBytesNoCopy (), "virtio,mmio", 12) != 0)) { continue; } + IOService *nub = createNub (entry); + if (nub == nullptr) { result = false; break; } + if (!nub->attach (this)) { nub->release (); result = false; break; } + nub->registerService (); + nub->release (); + } + iterator->release (); + return result; +} + +bool ArmVirtPlatform::configure (IOService *provider) +{ + // This platform has no Apple NVRAM controller. Enumerate its real DT nodes + // without calling the legacy IODT NVRAM initialization path. + if (!IOPlatformExpert::configure (provider)) { + return false; + } + IORegistryEntry *cpus = provider->childFromPath ("cpus", gIODTPlane); + IORegistryEntry *bus = provider->childFromPath ("arm-io", gIODTPlane); + if ((cpus == nullptr) || (bus == nullptr)) { + if (cpus != nullptr) { cpus->release (); } + if (bus != nullptr) { bus->release (); } + return false; + } + OSIterator *iterator = IODTFindMatchingEntries (cpus, kIODTExclusive, nullptr); + cpus->release (); + if (iterator == nullptr) { + bus->release (); + return false; + } + bool result = createNubs (this, iterator); // Consumes the iterator. + if (!result) { + bus->release (); + return false; + } + IOService *busNub = createNub (bus); + bool devicesCreated = (busNub != nullptr) && createVirtioNubs (busNub); + bus->release (); + if (!devicesCreated) { + if (busNub != nullptr) { busNub->release (); } + return false; + } + if (!busNub->attach (this)) { + busNub->release (); + return false; + } + busNub->registerService (); + busNub->release (); + IORegistryEntry *defaults = provider->childFromPath ("defaults", gIODTPlane); + if (defaults == nullptr) { + return false; + } + IOService *defaultsNub = createNub (defaults); + defaults->release (); + if (defaultsNub == nullptr) { + return false; + } + if (!defaultsNub->attach (this)) { + defaultsNub->release (); + return false; + } + defaultsNub->registerService (); + defaultsNub->release (); + iterator = getChildIterator (gIOServicePlane); + if (iterator == nullptr) { + return false; + } + uint32_t count = 0; + uint32_t cpuCount = 0; + uint32_t busCount = 0; + uint32_t defaultsCount = 0; + IORegistryEntry *child; + while ((child = OSDynamicCast (IORegistryEntry, iterator->getNextObject ())) != nullptr) { + count++; + OSData *type = OSDynamicCast (OSData, child->getProperty ("device_type")); + if ((type != nullptr) && (type->getLength () == 4) && + (memcmp (type->getBytesNoCopy (), "cpu", 4) == 0)) { + cpuCount++; + } + if (strcmp (child->getName (), "arm-io") == 0) { + busCount++; + } + if (strcmp (child->getName (), "defaults") == 0) { + defaultsCount++; + } + } + iterator->release (); + armvirt_platform_child_count = count; + armvirt_platform_cpu_count = cpuCount; + armvirt_platform_bus_count = busCount; + armvirt_platform_defaults_count = defaultsCount; + armvirt_platform_configured = 1; + registerService (); + return true; +} diff --git a/drivers/ArmVirtPlatform/Info.plist b/drivers/ArmVirtPlatform/Info.plist new file mode 100644 index 0000000..c7fcfdc --- /dev/null +++ b/drivers/ArmVirtPlatform/Info.plist @@ -0,0 +1,18 @@ + + + + CFBundleIdentifierorg.blu-sharky.ArmVirtPlatform + CFBundleNameArmVirtPlatform + CFBundleExecutableArmVirtPlatform + CFBundlePackageTypeKEXT + CFBundleVersion0.1.0 + OSBundleRequiredRoot + IOKitPersonalities + QEMU virt + IOClassArmVirtPlatform + IOProviderClassIOPlatformExpertDevice + IONameMatchlinux,dummy-virt + IOProbeScore10000 + + + diff --git a/drivers/ArmVirtPlatform/LICENSE b/drivers/ArmVirtPlatform/LICENSE new file mode 100644 index 0000000..217b4df --- /dev/null +++ b/drivers/ArmVirtPlatform/LICENSE @@ -0,0 +1,51 @@ +Copyright (c) 2026, ArmVirtPlatform contributors. All rights reserved. + +SPDX-License-Identifier: BSD-2-Clause-Patent + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +Subject to the terms and conditions of this license, each copyright holder +and contributor hereby grants to those receiving rights under this license +a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable +(except for failure to satisfy the conditions of this license) patent +license to make, have made, use, offer to sell, sell, import, and otherwise +transfer this software, where such license applies only to those patent +claims, already acquired or hereafter acquired, licensable by such copyright +holder or contributor that are necessarily infringed by: + +(a) their Contribution(s) (the licensed copyrights of copyright holders and + non-copyrightable additions of contributors, in source or binary form) + alone; or + +(b) combination of their Contribution(s) with the work of authorship to + which such Contribution(s) was added by such copyright holder or + contributor, if, at the time the Contribution is added, such addition + causes such combination to be necessarily infringed. The patent license + shall not apply to any other combinations which include the + Contribution. + +Except as expressly stated above, no rights or licenses from any copyright +holder or contributor is granted under this license, whether expressly, by +implication, estoppel or otherwise. + +DISCLAIMER + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. diff --git a/drivers/DarwinBoot/DarwinBootPkg/DarwinBoot.dsc b/drivers/DarwinBoot/DarwinBootPkg/DarwinBoot.dsc new file mode 100644 index 0000000..f437c4f --- /dev/null +++ b/drivers/DarwinBoot/DarwinBootPkg/DarwinBoot.dsc @@ -0,0 +1,34 @@ +## SPDX-License-Identifier: BSD-2-Clause-Patent +[Defines] + PLATFORM_NAME = DarwinBoot + PLATFORM_GUID = D94E8E62-812F-44AC-98B2-1BE8E1B2EF86 + PLATFORM_VERSION = 0.1 + DSC_SPECIFICATION = 0x0001001C + OUTPUT_DIRECTORY = build/DarwinBoot + SUPPORTED_ARCHITECTURES = AARCH64 + BUILD_TARGETS = DEBUG|RELEASE + SKUID_IDENTIFIER = DEFAULT + +!include MdePkg/MdeLibs.dsc.inc + +[LibraryClasses] + BaseLib|MdePkg/Library/BaseLib/BaseLib.inf + BaseMemoryLib|MdePkg/Library/BaseMemoryLib/BaseMemoryLib.inf + MemoryAllocationLib|MdePkg/Library/UefiMemoryAllocationLib/UefiMemoryAllocationLib.inf + PcdLib|MdePkg/Library/BasePcdLibNull/BasePcdLibNull.inf + PrintLib|MdePkg/Library/BasePrintLib/BasePrintLib.inf + UefiDriverEntryPoint|MdePkg/Library/UefiDriverEntryPoint/UefiDriverEntryPoint.inf + UefiBootServicesTableLib|MdePkg/Library/UefiBootServicesTableLib/UefiBootServicesTableLib.inf + UefiLib|MdePkg/Library/UefiLib/UefiLib.inf + UefiRuntimeServicesTableLib|MdePkg/Library/UefiRuntimeServicesTableLib/UefiRuntimeServicesTableLib.inf + DevicePathLib|MdePkg/Library/UefiDevicePathLib/UefiDevicePathLib.inf + DebugPrintErrorLevelLib|MdePkg/Library/BaseDebugPrintErrorLevelLib/BaseDebugPrintErrorLevelLib.inf + DebugLib|MdePkg/Library/UefiDebugLibStdErr/UefiDebugLibStdErr.inf + +[PcdsFixedAtBuild] + gEfiMdePkgTokenSpaceGuid.PcdDebugPropertyMask|0x07 + gEfiMdePkgTokenSpaceGuid.PcdDebugPrintErrorLevel|0x80000000 + +[Components] + DarwinBootPkg/Driver/DarwinBoot.inf + Test/Validation.inf diff --git a/drivers/DarwinBoot/DarwinBootPkg/Driver/BootArgs.h b/drivers/DarwinBoot/DarwinBootPkg/Driver/BootArgs.h new file mode 100644 index 0000000..aefcd90 --- /dev/null +++ b/drivers/DarwinBoot/DarwinBootPkg/Driver/BootArgs.h @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: BSD-2-Clause-Patent +#ifndef DARWIN_BOOT_ARGS_H_ +#define DARWIN_BOOT_ARGS_H_ + +#include + +// Darwin 23 pexpert/pexpert/arm64/boot.h ABI; all fields are little-endian. +typedef struct { + UINT16 Revision; + UINT16 Version; + UINT32 Padding; + UINT64 VirtualBase; + UINT64 PhysicalBase; + UINT64 MemorySize; + UINT64 TopOfKernelData; + UINT64 Video[6]; + UINT32 MachineType; + UINT32 TreePadding; + UINT64 DeviceTree; + UINT32 DeviceTreeLength; + CHAR8 CommandLine[1024]; + UINT32 FlagsPadding; + UINT64 BootFlags; + UINT64 MemorySizeActual; +} DARWIN_BOOT_ARGS; + +STATIC_ASSERT (OFFSET_OF (DARWIN_BOOT_ARGS, DeviceTree) == 0x60, "DT ABI"); +STATIC_ASSERT (OFFSET_OF (DARWIN_BOOT_ARGS, Video) == 0x28, "video ABI"); +STATIC_ASSERT (OFFSET_OF (DARWIN_BOOT_ARGS, CommandLine) == 0x6c, "command line ABI"); +STATIC_ASSERT (OFFSET_OF (DARWIN_BOOT_ARGS, MemorySizeActual) == 0x478, "memory ABI"); +STATIC_ASSERT (sizeof (DARWIN_BOOT_ARGS) == 0x480, "boot_args ABI"); + +#endif diff --git a/drivers/DarwinBoot/DarwinBootPkg/Driver/BootVideo.h b/drivers/DarwinBoot/DarwinBootPkg/Driver/BootVideo.h new file mode 100644 index 0000000..2bbb634 --- /dev/null +++ b/drivers/DarwinBoot/DarwinBootPkg/Driver/BootVideo.h @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: BSD-2-Clause-Patent +#ifndef DARWIN_BOOT_VIDEO_H_ +#define DARWIN_BOOT_VIDEO_H_ + +#include +#include +#include + +STATIC +EFI_STATUS +DarwinBootVideoFromGop (CONST EFI_GRAPHICS_OUTPUT_PROTOCOL *Gop, UINT64 Video[6]) +{ + EFI_GRAPHICS_OUTPUT_MODE_INFORMATION *Info; + UINT64 RowBytes; + UINT64 Bytes; + + ZeroMem (Video, 6 * sizeof (*Video)); + if ((Gop->Mode == NULL) || (Gop->Mode->Info == NULL) || + (Gop->Mode->SizeOfInfo < sizeof (*Info))) { return EFI_COMPROMISED_DATA; } + Info = Gop->Mode->Info; + if (Info->PixelFormat != PixelBlueGreenRedReserved8BitPerColor) { return EFI_UNSUPPORTED; } + RowBytes = (UINT64)Info->PixelsPerScanLine * 4; + if ((Info->HorizontalResolution == 0) || (Info->VerticalResolution == 0) || + (Info->PixelsPerScanLine < Info->HorizontalResolution) || + (Info->VerticalResolution > MAX_UINT64 / RowBytes)) { return EFI_COMPROMISED_DATA; } + Bytes = RowBytes * Info->VerticalResolution; + if ((Gop->Mode->FrameBufferBase == 0) || (Gop->Mode->FrameBufferBase & 3) || + (Gop->Mode->FrameBufferSize < Bytes) || + (Gop->Mode->FrameBufferBase > MAX_UINT64 - Gop->Mode->FrameBufferSize)) { return EFI_COMPROMISED_DATA; } + Video[0] = Gop->Mode->FrameBufferBase; + Video[1] = 1; + Video[2] = RowBytes; + Video[3] = Info->HorizontalResolution; + Video[4] = Info->VerticalResolution; + Video[5] = 32; + return EFI_SUCCESS; +} + +#endif diff --git a/drivers/DarwinBoot/DarwinBootPkg/Driver/DarwinBoot.c b/drivers/DarwinBoot/DarwinBootPkg/Driver/DarwinBoot.c new file mode 100644 index 0000000..06e23f7 --- /dev/null +++ b/drivers/DarwinBoot/DarwinBootPkg/Driver/DarwinBoot.c @@ -0,0 +1,691 @@ +// SPDX-License-Identifier: BSD-2-Clause-Patent +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "BootArgs.h" +#include "BootVideo.h" +#include "PackedRamdisk.h" +#include "WindowSize.h" + +#define WINDOW_SIZE SIZE_256MB +#define BLOCK_SIZE SIZE_2MB +#define MAX_SEGMENTS 64 + +typedef struct { + UINT64 Address; + UINT64 Size; + UINT64 FileOffset; + UINT64 FileSize; + UINT32 Protection; +} SEGMENT; + +typedef struct { + SEGMENT Segments[MAX_SEGMENTS]; + UINTN Count; + UINT64 Base; + UINT64 End; + UINT64 Entry; +} KERNEL; + +STATIC EFI_HANDLE mImage; +UINT64 EFIAPI DarwinCurrentEl (VOID); +VOID EFIAPI DarwinEnter (UINT64 Args, UINT64 Entry, UINT64 Memory, UINT64 Size, UINT64 VideoBase, UINT64 VideoBytes); + +#define READ_SYSREG(Name) ({ UINT64 Value_; __asm__ volatile ("mrs %0, " #Name : "=r" (Value_)); Value_; }) + +STATIC UINT32 U32 (CONST VOID *P) { return ReadUnaligned32 ((CONST UINT32 *)P); } +STATIC UINT64 U64 (CONST VOID *P) { return ReadUnaligned64 ((CONST UINT64 *)P); } + +STATIC +EFI_STATUS +ReadBootVideo (UINT64 Video[6]) +{ + EFI_GRAPHICS_OUTPUT_PROTOCOL *Gop; + EFI_STATUS Status; + + ZeroMem (Video, 6 * sizeof (*Video)); + Status = gBS->HandleProtocol (gST->ConsoleOutHandle, &gEfiGraphicsOutputProtocolGuid, (VOID **)&Gop); + if (EFI_ERROR (Status)) { + Status = gBS->LocateProtocol (&gEfiGraphicsOutputProtocolGuid, NULL, (VOID **)&Gop); + } + if (Status == EFI_NOT_FOUND) { return EFI_SUCCESS; } + if (EFI_ERROR (Status)) { return Status; } + return DarwinBootVideoFromGop (Gop, Video); +} + +STATIC +EFI_STATUS +ReadFile ( + EFI_FILE_PROTOCOL *Root, + CONST CHAR16 *Path, + UINTN Limit, + VOID **Data, + UINTN *Size + ) +{ + EFI_STATUS Status; + EFI_FILE_PROTOCOL *File; + EFI_FILE_INFO *Info; + UINTN InfoSize; + UINTN Actual; + + *Data = NULL; + *Size = 0; + Status = Root->Open (Root, &File, (CHAR16 *)Path, EFI_FILE_MODE_READ, 0); + if (EFI_ERROR (Status)) { return Status; } + Info = NULL; + InfoSize = 0; + Status = File->GetInfo (File, &gEfiFileInfoGuid, &InfoSize, NULL); + if (Status != EFI_BUFFER_TOO_SMALL) { goto Done; } + Info = AllocatePool (InfoSize); + if (Info == NULL) { Status = EFI_OUT_OF_RESOURCES; goto Done; } + Status = File->GetInfo (File, &gEfiFileInfoGuid, &InfoSize, Info); + if (EFI_ERROR (Status)) { goto Done; } + if ((Info->Attribute & EFI_FILE_DIRECTORY) || (Info->FileSize == 0) || (Info->FileSize > Limit)) { + Status = EFI_BAD_BUFFER_SIZE; + goto Done; + } + *Size = (UINTN)Info->FileSize; + *Data = AllocatePool (*Size); + if (*Data == NULL) { Status = EFI_OUT_OF_RESOURCES; goto Done; } + Actual = *Size; + Status = File->Read (File, &Actual, *Data); + if (!EFI_ERROR (Status) && (Actual != *Size)) { Status = EFI_DEVICE_ERROR; } +Done: + if (Info != NULL) { FreePool (Info); } + File->Close (File); + if (EFI_ERROR (Status) && (*Data != NULL)) { FreePool (*Data); *Data = NULL; } + return Status; +} + +STATIC +EFI_STATUS +ParseKernel (CONST UINT8 *Data, UINTN Size, KERNEL *Kernel) +{ + UINTN Offset; + UINTN End; + UINTN Index; + UINTN Other; + UINT32 Count; + UINT32 Command; + UINT32 Length; + SEGMENT *Segment; + BOOLEAN HasEntry; + BOOLEAN Executable; + + if ((Size < 32) || (U32 (Data) != 0xfeedfacf) || (U32 (Data + 4) != 0x100000c) || + (U32 (Data + 8) != 0) || (U32 (Data + 12) != 2)) { + return EFI_UNSUPPORTED; + } + if (U32 (Data + 20) > Size - 32) { return EFI_LOAD_ERROR; } + ZeroMem (Kernel, sizeof (*Kernel)); + Count = U32 (Data + 16); + End = 32 + U32 (Data + 20); + Offset = 32; + HasEntry = FALSE; + for (Index = 0; Index < Count; Index++) { + if (End - Offset < 8) { return EFI_LOAD_ERROR; } + Command = U32 (Data + Offset); + Length = U32 (Data + Offset + 4); + if ((Length < 8) || (Length & 7) || (Length > End - Offset)) { return EFI_LOAD_ERROR; } + if (Command == 0x19) { + if ((Length < 72) || (U32 (Data + Offset + 64) > (Length - 72) / 80) || + (Kernel->Count == MAX_SEGMENTS)) { return EFI_LOAD_ERROR; } + if (U64 (Data + Offset + 32) != 0) { + Segment = &Kernel->Segments[Kernel->Count++]; + Segment->Address = U64 (Data + Offset + 24); + Segment->Size = U64 (Data + Offset + 32); + Segment->FileOffset = U64 (Data + Offset + 40); + Segment->FileSize = U64 (Data + Offset + 48); + Segment->Protection = U32 (Data + Offset + 60); + if ((Segment->Size > WINDOW_SIZE) || (Segment->Address > MAX_UINT64 - Segment->Size) || + (Segment->FileSize > Segment->Size) || (Segment->FileOffset > Size) || + (Segment->FileSize > Size - Segment->FileOffset)) { return EFI_LOAD_ERROR; } + } + } else if (Command == 5) { + if (HasEntry || (Length != 288) || (U32 (Data + Offset + 8) != 6) || + (U32 (Data + Offset + 12) != 68)) { return EFI_LOAD_ERROR; } + Kernel->Entry = U64 (Data + Offset + 272); + HasEntry = TRUE; + } + Offset += Length; + } + if ((Offset != End) || !HasEntry || (Kernel->Count == 0)) { return EFI_LOAD_ERROR; } + Kernel->Base = MAX_UINT64; + Executable = FALSE; + for (Index = 0; Index < Kernel->Count; Index++) { + Segment = &Kernel->Segments[Index]; + Kernel->Base = MIN (Kernel->Base, Segment->Address); + Kernel->End = MAX (Kernel->End, Segment->Address + Segment->Size); + if ((Kernel->Entry >= Segment->Address) && (Kernel->Entry - Segment->Address < Segment->FileSize) && + ((Segment->Protection & 4) != 0)) { Executable = TRUE; } + for (Other = 0; Other < Index; Other++) { + if ((Segment->Address < Kernel->Segments[Other].Address + Kernel->Segments[Other].Size) && + (Kernel->Segments[Other].Address < Segment->Address + Segment->Size)) { return EFI_LOAD_ERROR; } + } + } + if (!Executable || (Kernel->Entry & 3) || (Kernel->Base & EFI_PAGE_MASK) || + (Kernel->Base > MAX_UINT64 - WINDOW_SIZE) || + (Kernel->End - Kernel->Base > WINDOW_SIZE - SIZE_2MB)) { return EFI_LOAD_ERROR; } + return EFI_SUCCESS; +} + +STATIC +BOOLEAN +AdtNode (CONST UINT8 *Data, UINTN Size, UINTN *Offset, UINTN Depth, UINTN ParentPath, UINTN *RamdiskOffset, UINTN *TrustCacheOffset, UINT64 *WindowSize, BOOLEAN *KeepPacked) +{ + UINT32 Properties; + UINT32 Children; + UINT32 Length; + UINTN Index; + UINTN Path; + UINTN Candidate; + UINTN TrustCandidate; + UINTN MemoryCandidate; + CONST UINT8 *Name; + UINTN NameLength; + + Name = NULL; + NameLength = 0; + Candidate = MAX_UINTN; + TrustCandidate = MAX_UINTN; + MemoryCandidate = MAX_UINTN; + if ((Depth > 64) || (Size - *Offset < 8)) { return FALSE; } + Properties = U32 (Data + *Offset); + Children = U32 (Data + *Offset + 4); + *Offset += 8; + if (Properties > (Size - *Offset) / 36) { return FALSE; } + for (Index = 0; Index < Properties; Index++) { + if (Size - *Offset < 36) { return FALSE; } + if (ScanMem8 (Data + *Offset, 32, 0) == NULL) { return FALSE; } + Length = U32 (Data + *Offset + 32); + *Offset += 36; + if (Length > Size - *Offset) { return FALSE; } + if (AsciiStrCmp ((CONST CHAR8 *)(Data + *Offset - 36), "name") == 0) { + if (Name != NULL) { return FALSE; } + Name = Data + *Offset; + NameLength = Length; + } + if ((AsciiStrCmp ((CONST CHAR8 *)(Data + *Offset - 36), "RAMDisk") == 0) || + (AsciiStrCmp ((CONST CHAR8 *)(Data + *Offset - 36), "PackedRAMDisk") == 0)) { + if (AsciiStrCmp ((CONST CHAR8 *)(Data + *Offset - 36), "PackedRAMDisk") == 0) { + if (*KeepPacked) { return FALSE; } + *KeepPacked = TRUE; + } + if ((Candidate != MAX_UINTN) || (Length != 16)) { return FALSE; } + Candidate = *Offset; + } + if (AsciiStrCmp ((CONST CHAR8 *)(Data + *Offset - 36), "TrustCache") == 0) { + if ((TrustCandidate != MAX_UINTN) || (Length != 16)) { return FALSE; } + TrustCandidate = *Offset; + } + if (AsciiStrCmp ((CONST CHAR8 *)(Data + *Offset - 36), "darwinboot-memory-size") == 0) { + if ((MemoryCandidate != MAX_UINTN) || (Length != 8)) { return FALSE; } + MemoryCandidate = *Offset; + } + *Offset += Length; + if (ALIGN_VALUE (*Offset, 4) > Size) { return FALSE; } + *Offset = ALIGN_VALUE (*Offset, 4); + } + Path = 3; + if (Depth == 0) { Path = 0; } + if ((Depth == 1) && (ParentPath == 0) && (NameLength == 7) && (CompareMem (Name, "chosen", 7) == 0)) { Path = 1; } + if ((Depth == 2) && (ParentPath == 1) && (NameLength == 11) && (CompareMem (Name, "memory-map", 11) == 0)) { Path = 2; } + if ((Path == 2) && (Candidate != MAX_UINTN)) { + if (*RamdiskOffset != MAX_UINTN) { return FALSE; } + *RamdiskOffset = Candidate; + } + if ((Path == 2) && (TrustCandidate != MAX_UINTN)) { + if (*TrustCacheOffset != MAX_UINTN) { return FALSE; } + *TrustCacheOffset = TrustCandidate; + } + if (MemoryCandidate != MAX_UINTN) { + if ((Path != 1) || (*WindowSize != 0)) { return FALSE; } + *WindowSize = U64 (Data + MemoryCandidate); + if (!DarwinWindowSizeValid (*WindowSize)) { return FALSE; } + } + if (Children > (Size - *Offset) / 8) { return FALSE; } + for (Index = 0; Index < Children; Index++) { + if (!AdtNode (Data, Size, Offset, Depth + 1, Path, RamdiskOffset, TrustCacheOffset, WindowSize, KeepPacked)) { return FALSE; } + } + return TRUE; +} + +STATIC +BOOLEAN +ValidTrustCaches (CONST UINT8 *Data, UINTN Size) +{ + UINTN Count; + UINTN Index; + UINTN Start; + UINTN End; + UINTN Entries; + UINTN Entry; + + if (Size < 8) { return FALSE; } + Count = U32 (Data); + if ((Count == 0) || (Count > 128) || (Count > (Size - 4) / 4)) { return FALSE; } + for (Index = 0; Index < Count; Index++) { + Start = U32 (Data + 4 + Index * 4); + End = (Index + 1 < Count) ? U32 (Data + 8 + Index * 4) : Size; + if ((Start < 4 + Count * 4) || (Start & 3) || (End > Size) || + (End < Start) || (End - Start < 24) || (U32 (Data + Start) != 2)) { return FALSE; } + Entries = U32 (Data + Start + 20); + if ((Entries > (End - Start - 24) / 24) || (End - Start != 24 + Entries * 24)) { return FALSE; } + for (Entry = 0; Entry < Entries; Entry++) { + CONST UINT8 *Hash = Data + Start + 24 + Entry * 24; + if ((Hash[20] < 1) || (Hash[20] > 4) || + ((Entry != 0) && (CompareMem (Hash - 24, Hash, 20) >= 0))) { return FALSE; } + } + } + return TRUE; +} + +STATIC +EFI_STATUS +GetMap (EFI_MEMORY_DESCRIPTOR **Map, UINTN *Capacity, UINTN *Size, UINTN *Key, UINTN *Stride) +{ + EFI_STATUS Status; + UINT32 Version; + UINTN Attempt; + + for (Attempt = 0; Attempt < 3; Attempt++) { + *Size = *Capacity; + Status = gBS->GetMemoryMap (Size, *Map, Key, Stride, &Version); + if (Status != EFI_BUFFER_TOO_SMALL) { return Status; } + if (*Map != NULL) { FreePool (*Map); } + *Capacity = *Size + 32 * sizeof (EFI_MEMORY_DESCRIPTOR); + *Map = AllocatePool (*Capacity); + if (*Map == NULL) { return EFI_OUT_OF_RESOURCES; } + } + return EFI_OUT_OF_RESOURCES; +} + +// End of the run of EfiConventionalMemory that begins at Start: the UEFI map often lists physically +// adjacent free ranges as separate descriptors, so extend over every descriptor that continues the run. +STATIC +UINT64 +RunEnd (EFI_MEMORY_DESCRIPTOR *Map, UINTN Size, UINTN Stride, UINT64 Start, UINT64 End) +{ + EFI_MEMORY_DESCRIPTOR *Descriptor; + UINTN Offset; + BOOLEAN Extended; + + do { + Extended = FALSE; + for (Offset = 0; Offset + Stride <= Size; Offset += Stride) { + Descriptor = (EFI_MEMORY_DESCRIPTOR *)((UINT8 *)Map + Offset); + if ((Descriptor->Type == EfiConventionalMemory) && (Descriptor->PhysicalStart == End) && + (Descriptor->NumberOfPages != 0) && (Descriptor->NumberOfPages < (MAX_UINT64 - End) / EFI_PAGE_SIZE)) { + End += EFI_PAGES_TO_SIZE (Descriptor->NumberOfPages); + Extended = TRUE; + } + } + } while (Extended); + (VOID)Start; + return End; +} + +// Large diagnostic text straight into the GOP framebuffer: the UEFI text console font is too small to +// photograph. 5x12 glyphs from DiagFont.h scaled 4x, white on a black box; wraps at the right edge. +#include "DiagFont.h" +STATIC CONST UINT64 mDiagFont[] = GLYPH_FONT; + +STATIC +VOID +DiagDraw (CONST CHAR16 *Text, UINTN Row) +{ + EFI_GRAPHICS_OUTPUT_PROTOCOL *Gop; + UINT64 Video[6]; + UINTN Scale = 4, X = 0, Y, Index, Gx, Gy, Dx, Dy, Cell, Line; + UINT32 *Fb; + UINT64 Glyph, Data; + CHAR16 Ch; + + if (EFI_ERROR (gBS->LocateProtocol (&gEfiGraphicsOutputProtocolGuid, NULL, (VOID **)&Gop)) || + EFI_ERROR (DarwinBootVideoFromGop (Gop, Video))) { return; } + Cell = 6 * Scale; + Line = 14 * Scale; + Y = Video[4] / 4 + Row * Line; + Fb = (UINT32 *)(UINTN)Video[0]; + for (Index = 0; Text[Index] != 0; Index++) { + Ch = Text[Index]; + if ((Ch < 32) || (Ch > 126)) { continue; } + if (X + Cell > Video[3]) { X = 0; Y += Line; } + if (Y + Line > Video[4]) { return; } + Glyph = mDiagFont[Ch - 32]; + for (Gy = 0; Gy < 14 * Scale; Gy++) { + for (Gx = 0; Gx < Cell; Gx++) { + Fb[(Y + Gy) * (Video[2] / 4) + X + Gx] = 0xFF000000; + } + } + for (Gy = 0; Gy < 12; Gy++) { + Data = (Gy < 6) ? ((Glyph >> 32) >> (Gy * 5)) : ((Glyph & 0xFFFFFFFFULL) >> ((Gy - 6) * 5)); + for (Gx = 0; Gx < 5; Gx++) { + if ((Data >> Gx) & 1) { + for (Dy = 0; Dy < Scale; Dy++) { + for (Dx = 0; Dx < Scale; Dx++) { + Fb[(Y + Gy * Scale + Dy) * (Video[2] / 4) + X + Gx * Scale + Dx] = 0xFFFFFFFF; + } + } + } + } + } + X += Cell; + } +} + +STATIC +EFI_STATUS +AllocateWindow (KERNEL *Kernel, UINT64 WindowSize, EFI_PHYSICAL_ADDRESS *Address) +{ + EFI_MEMORY_DESCRIPTOR *Map; + EFI_MEMORY_DESCRIPTOR *Descriptor; + EFI_STATUS Status; + UINTN Capacity; + UINTN Size; + UINTN Key; + UINTN Stride; + UINTN Offset; + UINT64 End; + UINT64 Candidate; + UINT64 BestStart; + UINT64 BestBytes; + UINT64 RunBytes; + + *Address = 0; + Map = NULL; + Capacity = 0; + BestStart = 0; + BestBytes = 0; + Status = GetMap (&Map, &Capacity, &Size, &Key, &Stride); + if (EFI_ERROR (Status)) { if (Map != NULL) { FreePool (Map); } return Status; } + Status = EFI_OUT_OF_RESOURCES; + for (Offset = 0; Offset + Stride <= Size; Offset += Stride) { + Descriptor = (EFI_MEMORY_DESCRIPTOR *)((UINT8 *)Map + Offset); + if ((Descriptor->Type != EfiConventionalMemory) || (Descriptor->NumberOfPages == 0)) { continue; } + End = RunEnd (Map, Size, Stride, Descriptor->PhysicalStart, + Descriptor->PhysicalStart + EFI_PAGES_TO_SIZE (Descriptor->NumberOfPages)); + RunBytes = End - Descriptor->PhysicalStart; + if (RunBytes > BestBytes) { BestBytes = RunBytes; BestStart = Descriptor->PhysicalStart; } + Candidate = ALIGN_VALUE (Descriptor->PhysicalStart, BLOCK_SIZE) + (Kernel->Base & (BLOCK_SIZE - 1)); + if ((Candidate < Descriptor->PhysicalStart) || (End < Candidate) || (End - Candidate < WindowSize)) { continue; } + Status = gBS->AllocatePages (AllocateAddress, EfiLoaderCode, EFI_SIZE_TO_PAGES (WindowSize), &Candidate); + if (!EFI_ERROR (Status)) { *Address = Candidate; break; } + } + if (EFI_ERROR (Status)) { + CHAR16 Line[96]; + UINTN Row = 1; + UnicodeSPrint (Line, sizeof (Line), L"NO WINDOW %lx BIG RUN %lx @ %lx", WindowSize, BestBytes, BestStart); + DiagDraw (Line, 0); + for (Offset = 0; Offset + Stride <= Size; Offset += Stride) { + Descriptor = (EFI_MEMORY_DESCRIPTOR *)((UINT8 *)Map + Offset); + if ((Descriptor->Type == EfiConventionalMemory) && (Descriptor->NumberOfPages >= 0x400) && (Row < 8)) { + UnicodeSPrint (Line, sizeof (Line), L"FREE %lx + %lx", Descriptor->PhysicalStart, EFI_PAGES_TO_SIZE (Descriptor->NumberOfPages)); + DiagDraw (Line, Row++); + } + } + + } + FreePool (Map); + return Status; +} + +STATIC +EFI_STATUS +EFIAPI +Boot ( + IN CRABAPPLE_XNU_LOADER_PROTOCOL *This, + IN CONST CRABAPPLE_XNU_BOOT_REQUEST *Request + ) +{ + EFI_STATUS Status; + EFI_SIMPLE_FILE_SYSTEM_PROTOCOL *Fs; + EFI_FILE_PROTOCOL *Root; + VOID *Image; + VOID *Tree; + VOID *TrustCaches; + UINTN ImageSize; + UINTN TreeSize; + UINTN TrustCacheSize; + CHAR16 *TreePath; + UINTN Length; + UINTN Index; + UINTN Offset; + KERNEL Kernel; + EFI_PHYSICAL_ADDRESS Address; + UINT64 ArgsAddress; + UINT64 TreeAddress; + UINT64 Used; + UINT64 WindowSize; + UINT64 RamdiskAddress; + UINTN RamdiskOffset; + UINTN RamdiskBytes; + UINTN RamdiskExpandedSize; + BOOLEAN RamdiskPacked; + BOOLEAN KeepPacked; + UINTN TrustCacheOffset; + UINT64 TrustCacheAddress; + DARWIN_BOOT_ARGS *Args; + EFI_MEMORY_DESCRIPTOR *Map; + UINTN Capacity; + UINTN MapSize; + UINTN Key; + UINTN Stride; + UINT32 Version; + UINTN Attempt; + CHAR8 CommandLine[1024]; + UINT64 Video[6]; + + (VOID)This; + if ((Request == NULL) || (Request->Volume == NULL) || (Request->KernelPath == NULL)) { return EFI_INVALID_PARAMETER; } + if ((Request->KextCount != 0) || (Request->DeviceTree != NULL) || (Request->DeviceTreeSize != 0) || + (DarwinCurrentEl () != 4)) { + Print (L"DARWINBOOT: unsupported inputs or exception level; firmware retained\n"); + return EFI_UNSUPPORTED; + } + if (!RamdiskLayout (Request->Ramdisk, Request->RamdiskSize, &RamdiskExpandedSize, &RamdiskPacked)) { + return EFI_BAD_BUFFER_SIZE; + } + ZeroMem (CommandLine, sizeof (CommandLine)); + if (Request->BootArgs != NULL) { + for (Index = 0; Request->BootArgs[Index] != 0; Index++) { + if ((Index >= sizeof (CommandLine) - 1) || (Request->BootArgs[Index] > 0x7f)) { return EFI_BAD_BUFFER_SIZE; } + CommandLine[Index] = (CHAR8)Request->BootArgs[Index]; + } + } + Root = NULL; + Image = NULL; + Tree = NULL; + TrustCaches = NULL; + TreePath = NULL; + Map = NULL; + Address = 0; + Capacity = 0; + Status = gBS->HandleProtocol (Request->Volume, &gEfiSimpleFileSystemProtocolGuid, (VOID **)&Fs); + if (EFI_ERROR (Status)) { return Status; } + Status = Fs->OpenVolume (Fs, &Root); + if (EFI_ERROR (Status)) { return Status; } + Status = ReadFile (Root, Request->KernelPath, SIZE_128MB, &Image, &ImageSize); + if (EFI_ERROR (Status)) { goto Fail; } + Status = ParseKernel (Image, ImageSize, &Kernel); + if (EFI_ERROR (Status)) { goto Fail; } + Length = StrSize (Request->KernelPath); + TreePath = AllocatePool (Length + 11 * sizeof (CHAR16)); + if (TreePath == NULL) { Status = EFI_OUT_OF_RESOURCES; goto Fail; } + CopyMem (TreePath, Request->KernelPath, Length); + StrCatS (TreePath, Length / sizeof (CHAR16) + 11, L".adt"); + Status = ReadFile (Root, TreePath, SIZE_16MB, &Tree, &TreeSize); + if (EFI_ERROR (Status)) { goto Fail; } + Offset = 0; + RamdiskOffset = MAX_UINTN; + TrustCacheOffset = MAX_UINTN; + WindowSize = 0; + KeepPacked = FALSE; + if (!AdtNode (Tree, TreeSize, &Offset, 0, 3, &RamdiskOffset, &TrustCacheOffset, &WindowSize, &KeepPacked) || (Offset != TreeSize)) { Status = EFI_COMPROMISED_DATA; goto Fail; } + if (KeepPacked && (!RamdiskPacked || (RamdiskOffset == MAX_UINTN))) { Status = EFI_COMPROMISED_DATA; goto Fail; } + if (WindowSize == 0) { WindowSize = WINDOW_SIZE; } + if (Kernel.Base > MAX_UINT64 - WindowSize) { Status = EFI_LOAD_ERROR; goto Fail; } + if (Request->Ramdisk != NULL) { + if ((RamdiskOffset == MAX_UINTN) || (U64 ((UINT8 *)Tree + RamdiskOffset) != 0) || + (U64 ((UINT8 *)Tree + RamdiskOffset + 8) != 0)) { Status = EFI_COMPROMISED_DATA; goto Fail; } + } else if (RamdiskOffset != MAX_UINTN) { + Status = EFI_UNSUPPORTED; + goto Fail; + } + CopyMem (TreePath, Request->KernelPath, Length); + StrCatS (TreePath, Length / sizeof (CHAR16) + 11, L".trustcache"); + Status = ReadFile (Root, TreePath, SIZE_16MB, &TrustCaches, &TrustCacheSize); + if (EFI_ERROR (Status) && (Status != EFI_NOT_FOUND)) { goto Fail; } + if (TrustCaches != NULL) { + if ((TrustCacheOffset == MAX_UINTN) || (U64 ((UINT8 *)Tree + TrustCacheOffset) != 0) || + (U64 ((UINT8 *)Tree + TrustCacheOffset + 8) != 0) || !ValidTrustCaches (TrustCaches, TrustCacheSize)) { + Status = EFI_COMPROMISED_DATA; + goto Fail; + } + } else if (TrustCacheOffset != MAX_UINTN) { + Status = EFI_COMPROMISED_DATA; + goto Fail; + } + Status = ReadBootVideo (Video); + if (EFI_ERROR (Status)) { goto Fail; } + Status = AllocateWindow (&Kernel, WindowSize, &Address); + if (EFI_ERROR (Status)) { goto Fail; } + ArgsAddress = ALIGN_VALUE (Address + Kernel.End - Kernel.Base, SIZE_16KB); + TreeAddress = ArgsAddress + EFI_PAGE_SIZE; + RamdiskAddress = ALIGN_VALUE (TreeAddress + TreeSize, EFI_PAGE_SIZE); + RamdiskBytes = ALIGN_VALUE (KeepPacked ? Request->RamdiskSize : RamdiskExpandedSize, EFI_PAGE_SIZE); + TrustCacheAddress = RamdiskAddress + RamdiskBytes; + Used = ALIGN_VALUE (TrustCacheAddress + TrustCacheSize, SIZE_16KB) - Address; + if (Used > WindowSize - BLOCK_SIZE) { Status = EFI_BAD_BUFFER_SIZE; goto Fail; } + ZeroMem ((VOID *)(UINTN)Address, (UINTN)Used); + for (Index = 0; Index < Kernel.Count; Index++) { + CopyMem ((VOID *)(UINTN)(Address + Kernel.Segments[Index].Address - Kernel.Base), + (UINT8 *)Image + Kernel.Segments[Index].FileOffset, (UINTN)Kernel.Segments[Index].FileSize); + } + Args = (DARWIN_BOOT_ARGS *)(UINTN)ArgsAddress; + Args->Revision = 2; + Args->Version = 2; + Args->VirtualBase = Kernel.Base; + Args->PhysicalBase = Address; + Args->MemorySize = WindowSize; + Args->MemorySizeActual = WindowSize; + Args->TopOfKernelData = Address + Used; + CopyMem (Args->Video, Video, sizeof (Video)); + Args->DeviceTree = Kernel.Base + TreeAddress - Address; + Args->DeviceTreeLength = (UINT32)TreeSize; + CopyMem (Args->CommandLine, CommandLine, sizeof (CommandLine)); + CopyMem ((VOID *)(UINTN)TreeAddress, Tree, TreeSize); + if (Request->Ramdisk != NULL) { + if (KeepPacked) { CopyMem ((VOID *)(UINTN)RamdiskAddress, Request->Ramdisk, Request->RamdiskSize); } + else { CopyRamdisk ((VOID *)(UINTN)RamdiskAddress, Request->Ramdisk, Request->RamdiskSize, RamdiskPacked); } + WriteUnaligned64 ((UINT64 *)(UINTN)(TreeAddress + RamdiskOffset), RamdiskAddress); + WriteUnaligned64 ((UINT64 *)(UINTN)(TreeAddress + RamdiskOffset + 8), KeepPacked ? Request->RamdiskSize : RamdiskBytes); + } + if (TrustCaches != NULL) { + CopyMem ((VOID *)(UINTN)TrustCacheAddress, TrustCaches, TrustCacheSize); + WriteUnaligned64 ((UINT64 *)(UINTN)(TreeAddress + TrustCacheOffset), TrustCacheAddress); + WriteUnaligned64 ((UINT64 *)(UINTN)(TreeAddress + TrustCacheOffset + 8), TrustCacheSize); + FreePool (TrustCaches); TrustCaches = NULL; + } + FreePool (Image); Image = NULL; + FreePool (Tree); Tree = NULL; + FreePool (TreePath); TreePath = NULL; + Root->Close (Root); Root = NULL; + Status = GetMap (&Map, &Capacity, &MapSize, &Key, &Stride); + if (EFI_ERROR (Status)) { goto Fail; } + Print (L"DARWINBOOT: ARM64 va=0x%lx pa=0x%lx args=0x%lx entry=0x%lx owned=0x%lx\n", + Kernel.Base, Address, ArgsAddress, Address + Kernel.Entry - Kernel.Base, (UINT64)WindowSize); + Print (L"DARWINBOOT: video base=0x%lx display=%lu row=%lu width=%lu height=%lu depth=%lu\n", + Video[0], Video[1], Video[2], Video[3], Video[4], Video[5]); + Print (L"DARWINBOOT: inputs validated; exiting boot services\n"); + + // + // The same facts in large type over the whole screen. The firmware's log + // draws far larger than the text console, and the form feed clears the + // screen first. This is what stays visible if the kernel never draws. + // + DEBUG ((DEBUG_ERROR, "\fDARWINBOOT: hand-off to the kernel\n\n")); + DEBUG ((DEBUG_ERROR, "kernel va 0x%lx pa 0x%lx\n", Kernel.Base, Address)); + DEBUG ((DEBUG_ERROR, "entry pa 0x%lx\n", Address + Kernel.Entry - Kernel.Base)); + DEBUG ((DEBUG_ERROR, "args pa 0x%lx window 0x%lx bytes\n", ArgsAddress, (UINT64)WindowSize)); + DEBUG ((DEBUG_ERROR, "video base 0x%lx %lux%lu row %lu bytes %lu bpp\n", Video[0], Video[3], Video[4], Video[2], Video[5])); + DEBUG ((DEBUG_ERROR, "entry words %08x %08x %08x %08x\n", + ReadUnaligned32 ((CONST UINT32 *)(UINTN)(Address + Kernel.Entry - Kernel.Base)), + ReadUnaligned32 ((CONST UINT32 *)(UINTN)(Address + Kernel.Entry - Kernel.Base + 4)), + ReadUnaligned32 ((CONST UINT32 *)(UINTN)(Address + Kernel.Entry - Kernel.Base + 8)), + ReadUnaligned32 ((CONST UINT32 *)(UINTN)(Address + Kernel.Entry - Kernel.Base + 12)))); + DEBUG ((DEBUG_ERROR, "CurrentEL %lu SCTLR_EL1 0x%lx TCR_EL1 0x%lx\n", DarwinCurrentEl () >> 2, READ_SYSREG (sctlr_el1), READ_SYSREG (tcr_el1))); + DEBUG ((DEBUG_ERROR, "ID_AA64MMFR0_EL1 0x%lx (PARange %lu, 4 KiB granule %a)\n", READ_SYSREG (id_aa64mmfr0_el1), + READ_SYSREG (id_aa64mmfr0_el1) & 0xF, (((READ_SYSREG (id_aa64mmfr0_el1) >> 28) & 0xF) == 0xF) ? "NOT supported" : "supported")); + DEBUG ((DEBUG_ERROR, "ID_AA64PFR0_EL1 0x%lx\n", READ_SYSREG (id_aa64pfr0_el1))); + DEBUG ((DEBUG_ERROR, "\nExiting boot services. A green bar across the top\nmeans the kernel entry code ran.\n")); + for (Attempt = 0; Attempt < 3; Attempt++) { + MapSize = Capacity; + Status = gBS->GetMemoryMap (&MapSize, Map, &Key, &Stride, &Version); + if (EFI_ERROR (Status)) { + if (Attempt == 0) { goto Fail; } + CpuDeadLoop (); + } + Status = gBS->ExitBootServices (mImage, Key); + if (!EFI_ERROR (Status)) { + DarwinEnter (ArgsAddress, Address + Kernel.Entry - Kernel.Base, Address, WindowSize, Video[0], Video[2] * Video[4]); + CpuDeadLoop (); + } + if (Status != EFI_INVALID_PARAMETER) { CpuDeadLoop (); } + } + CpuDeadLoop (); + return EFI_ABORTED; +Fail: + if (Root != NULL) { Root->Close (Root); } + if (Image != NULL) { FreePool (Image); } + if (Tree != NULL) { FreePool (Tree); } + if (TrustCaches != NULL) { FreePool (TrustCaches); } + if (TreePath != NULL) { FreePool (TreePath); } + if (Map != NULL) { FreePool (Map); } + if (Address != 0) { gBS->FreePages (Address, EFI_SIZE_TO_PAGES (WindowSize)); } + Print (L"DARWINBOOT: boot refused: %r; firmware retained\n", Status); + return Status; +} + +STATIC CRABAPPLE_XNU_LOADER_PROTOCOL mLoader = { CRABAPPLE_XNU_LOADER_REVISION, Boot }; + +STATIC +EFI_STATUS +EFIAPI +Unload (EFI_HANDLE ImageHandle) +{ + return gBS->UninstallMultipleProtocolInterfaces (ImageHandle, &gCrabAppleXnuLoaderProtocolGuid, &mLoader, NULL); +} + +EFI_STATUS +EFIAPI +DarwinBootEntry (EFI_HANDLE ImageHandle, EFI_SYSTEM_TABLE *SystemTable) +{ + EFI_LOADED_IMAGE_PROTOCOL *Loaded; + EFI_STATUS Status; + VOID *Existing; + + (VOID)SystemTable; + Status = gBS->LocateProtocol (&gCrabAppleXnuLoaderProtocolGuid, NULL, &Existing); + if (!EFI_ERROR (Status)) { return EFI_ALREADY_STARTED; } + Status = gBS->HandleProtocol (ImageHandle, &gEfiLoadedImageProtocolGuid, (VOID **)&Loaded); + if (EFI_ERROR (Status)) { return Status; } + mImage = ImageHandle; + Status = gBS->InstallMultipleProtocolInterfaces (&ImageHandle, &gCrabAppleXnuLoaderProtocolGuid, &mLoader, NULL); + if (!EFI_ERROR (Status)) { + Loaded->Unload = Unload; + Print (L"DARWINBOOT: loader v2 installed EL%u\n", (UINT32)(DarwinCurrentEl () >> 2)); + } + return Status; +} diff --git a/drivers/DarwinBoot/DarwinBootPkg/Driver/DarwinBoot.inf b/drivers/DarwinBoot/DarwinBootPkg/Driver/DarwinBoot.inf new file mode 100644 index 0000000..a59bc43 --- /dev/null +++ b/drivers/DarwinBoot/DarwinBootPkg/Driver/DarwinBoot.inf @@ -0,0 +1,39 @@ +## SPDX-License-Identifier: BSD-2-Clause-Patent +[Defines] + INF_VERSION = 0x00010019 + BASE_NAME = DarwinBoot + FILE_GUID = 1B64AF87-8415-4B57-9B16-680D70913B9F + MODULE_TYPE = UEFI_DRIVER + VERSION_STRING = 0.1 + ENTRY_POINT = DarwinBootEntry + +[Sources] + DarwinBoot.c + PackedRamdisk.h + BootArgs.h + BootVideo.h + DiagFont.h + Entry.S + +[Packages] + MdePkg/MdePkg.dec + CrabApplePkg/CrabApplePkg.dec + +[LibraryClasses] + BaseLib + BaseMemoryLib + DebugLib + MemoryAllocationLib + PrintLib + UefiBootServicesTableLib + UefiDriverEntryPoint + UefiLib + +[Protocols] + gCrabAppleXnuLoaderProtocolGuid ## PRODUCES + gEfiSimpleFileSystemProtocolGuid ## CONSUMES + gEfiLoadedImageProtocolGuid ## CONSUMES + gEfiGraphicsOutputProtocolGuid ## CONSUMES + +[Guids] + gEfiFileInfoGuid diff --git a/drivers/DarwinBoot/DarwinBootPkg/Driver/DiagFont.h b/drivers/DarwinBoot/DarwinBootPkg/Driver/DiagFont.h new file mode 100644 index 0000000..2ea15a2 --- /dev/null +++ b/drivers/DarwinBoot/DarwinBootPkg/Driver/DiagFont.h @@ -0,0 +1,92 @@ +/** + Copyright (C) 2008, The Android Open Source Project, All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + + THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED + WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS + BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +**/ + +#ifndef _FONT_H_ +#define _FONT_H_ + +// +// Font Dimension +// +#define FONT_WIDTH 5 +#define FONT_HEIGHT 16 + +// +// 5x12 Font +// +#define GLYPH_FONT \ +{ \ + 0x0000000000000000, 0x0842108000020084, \ + 0x0005294000000000, 0x15F528000000295F, \ + 0x1C52F88000023E94, 0x088556400004D542, \ + 0x04528800000B2725, 0x0002108000000000, \ + 0x0421108800821042, 0x1084108200221108, \ + 0x0957548000000000, 0x3E42000000000084, \ + 0x0000000000223000, 0x3E00000000000000, \ + 0x0000000000471000, 0x0884420000008442, \ + 0x2318A88000022A31, 0x08429880000F9084, \ + 0x1108C5C0000F8444, 0x1C4443E000074610, \ + 0x14A62100000423E9, 0x26D087E000074610, \ + 0x1E10C5C000074631, 0x088443E000010844, \ + 0x1D18C5C000074631, 0x3D18C5C000074610, \ + 0x08E2000000471000, 0x08E2000000223000, \ + 0x0222220000082082, 0x01F00000000003E0, \ + 0x2082082000008888, 0x1108C5C000020084, \ + 0x2B98C5C0000F05B5, 0x2318A8800008C63F, \ + 0x1D2949E00007CA52, 0x0210C5C000074421, \ + 0x252949E00007CA52, 0x1E1087E0000F8421, \ + 0x1E1087E000008421, 0x0210C5C000074639, \ + 0x3F18C6200008C631, 0x084211C000071084, \ + 0x1084238000032508, 0x0654C6200008C525, \ + 0x02108420000F8421, 0x2B5DC6200008C631, \ + 0x2B59CE200008C739, 0x2318C5C000074631, \ + 0x1F18C5E000008421, 0x2318C5C001075631, \ + 0x1F18C5E00008C525, 0x1C10C5C000074610, \ + 0x084213E000021084, 0x2318C62000074631, \ + 0x1518C6200002114A, 0x2B18C620000556B5, \ + 0x08A546200008C54A, 0x08A5462000021084, \ + 0x088443E0000F8442, 0x0421084E00E10842, \ + 0x0821042000084108, 0x1084210E00E42108, \ + 0x0008A88000000000, 0x0000000001F00000, \ + 0x0000010400000000, 0x20E00000000B663E, \ + 0x22F084200007C631, 0x22E0000000074421, \ + 0x23E84200000F4631, 0x22E000000007443F, \ + 0x1E21498000010842, 0x22E000001D187A31, \ + 0x26D084200008C631, 0x0860100000071084, \ + 0x10C020000C94A108, 0x0A9084200008A4A3, \ + 0x084210C000071084, 0x2AB000000008D6B5, \ + 0x26D000000008C631, 0x22E0000000074631, \ + 0x22F000000210BE31, 0x23E0000021087A31, \ + 0x26D0000000008421, 0x22E0000000074506, \ + 0x04F1080000064842, 0x23100000000B6631, \ + 0x2310000000022951, 0x23100000000556B5, \ + 0x151000000008A884, 0x231000001D185B31, \ + 0x11F00000000F8444, 0x0642109801821084, \ + 0x0842108000021084, 0x3042108300321084, \ + 0x0004D64000000000, 0x0000000000000000 \ +}; + +#endif /* _FONT_H_ */ diff --git a/drivers/DarwinBoot/DarwinBootPkg/Driver/Entry.S b/drivers/DarwinBoot/DarwinBootPkg/Driver/Entry.S new file mode 100644 index 0000000..890d4df --- /dev/null +++ b/drivers/DarwinBoot/DarwinBootPkg/Driver/Entry.S @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: BSD-2-Clause-Patent +#include + +ASM_FUNC(DarwinCurrentEl) + mrs x0, CurrentEL + ret + +// x0=boot_args PA, x1=entry PA, x2=owned PA, x3=owned size, x4=frame buffer +// PA (0 for none), x5=its size in bytes. EL1 only. +ASM_FUNC(DarwinEnter) + msr daifset, #15 + // Breadcrumb: a green bar (BGRA 0xFF00FF00) over the top 1/16 of the frame + // buffer shows that this code ran, whatever the kernel does afterwards. + cbz x4, 2f + lsr x9, x5, #4 + bic x9, x9, #7 + mov w10, #0xFF00 + movk w10, #0xFF00, lsl #16 + mov x11, x4 +3: + stp w10, w10, [x11], #8 + subs x9, x9, #8 + b.hi 3b + dsb sy +2: + add x5, x2, x3 + mrs x6, ctr_el0 + ubfx x6, x6, #16, #4 + mov x7, #4 + lsl x6, x7, x6 + sub x7, x6, #1 + bic x4, x2, x7 + dsb sy +1: + dc civac, x4 + add x4, x4, x6 + cmp x4, x5 + b.lo 1b + dsb sy + ic iallu + dsb sy + isb + mrs x4, sctlr_el1 + bic x4, x4, #1 + bic x4, x4, #4 + bic x4, x4, #0x1000 + msr sctlr_el1, x4 + isb + tlbi vmalle1 + dsb sy + isb + br x1 diff --git a/drivers/DarwinBoot/DarwinBootPkg/Driver/PackedRamdisk.h b/drivers/DarwinBoot/DarwinBootPkg/Driver/PackedRamdisk.h new file mode 100644 index 0000000..ee0d1e3 --- /dev/null +++ b/drivers/DarwinBoot/DarwinBootPkg/Driver/PackedRamdisk.h @@ -0,0 +1,58 @@ +// SPDX-License-Identifier: BSD-2-Clause-Patent +// Little-endian DBRDISK1: version, page size, expanded bytes, extent count; +// then sorted (destination offset, byte count) pairs and concatenated bytes. +// Omitted ranges are real zeroed RAM, not virtual/simulated disk storage. +#ifndef DARWIN_BOOT_PACKED_RAMDISK_H +#define DARWIN_BOOT_PACKED_RAMDISK_H + +STATIC +BOOLEAN +RamdiskLayout (CONST VOID *Input, UINTN InputSize, UINTN *Expanded, BOOLEAN *Packed) +{ + CONST UINT8 *Bytes = Input; + UINT64 Logical, Count, Table, Cursor, End, Index, Offset, Length; + *Expanded = InputSize; + *Packed = FALSE; + if ((InputSize > SIZE_256MB) || ((Input == NULL) != (InputSize == 0))) { return FALSE; } + if ((InputSize < 8) || (CompareMem (Input, "DBRDISK1", 8) != 0)) { return TRUE; } + if ((InputSize < 32) || (ReadUnaligned32 ((CONST UINT32 *)(Bytes + 8)) != 1) || + (ReadUnaligned32 ((CONST UINT32 *)(Bytes + 12)) != EFI_PAGE_SIZE)) { return FALSE; } + Logical = ReadUnaligned64 ((CONST UINT64 *)(Bytes + 16)); + Count = ReadUnaligned64 ((CONST UINT64 *)(Bytes + 24)); + if ((Logical == 0) || (Logical > SIZE_2GB) || (Logical & (EFI_PAGE_SIZE - 1)) || + (Count > 65536) || (Count > (InputSize - 32) / 16)) { return FALSE; } + Table = 32 + Count * 16; + Cursor = Table; + End = 0; + for (Index = 0; Index < Count; Index++) { + Offset = ReadUnaligned64 ((CONST UINT64 *)(Bytes + 32 + Index * 16)); + Length = ReadUnaligned64 ((CONST UINT64 *)(Bytes + 40 + Index * 16)); + if (!Length || ((Offset | Length) & (EFI_PAGE_SIZE - 1)) || (Offset < End) || + (Offset > Logical) || (Length > Logical - Offset) || (Length > InputSize - Cursor)) { return FALSE; } + End = Offset + Length; + Cursor += Length; + } + if (Cursor != InputSize) { return FALSE; } + *Expanded = (UINTN)Logical; + *Packed = TRUE; + return TRUE; +} + +STATIC +VOID +CopyRamdisk (VOID *Destination, CONST VOID *Input, UINTN InputSize, BOOLEAN Packed) +{ + CONST UINT8 *Bytes = Input; + UINT64 Count, Cursor, Index, Offset, Length; + // Caller has validated every range and zeroed the entire expanded allocation. + if (!Packed) { CopyMem (Destination, Input, InputSize); return; } + Count = ReadUnaligned64 ((CONST UINT64 *)(Bytes + 24)); + Cursor = 32 + Count * 16; + for (Index = 0; Index < Count; Index++) { + Offset = ReadUnaligned64 ((CONST UINT64 *)(Bytes + 32 + Index * 16)); + Length = ReadUnaligned64 ((CONST UINT64 *)(Bytes + 40 + Index * 16)); + CopyMem ((UINT8 *)Destination + Offset, Bytes + Cursor, (UINTN)Length); + Cursor += Length; + } +} +#endif diff --git a/drivers/DarwinBoot/DarwinBootPkg/Driver/WindowSize.h b/drivers/DarwinBoot/DarwinBootPkg/Driver/WindowSize.h new file mode 100644 index 0000000..055549a --- /dev/null +++ b/drivers/DarwinBoot/DarwinBootPkg/Driver/WindowSize.h @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: BSD-2-Clause-Patent +#ifndef DARWINBOOT_WINDOW_SIZE_H +#define DARWINBOOT_WINDOW_SIZE_H +// Allocation still requires one real conventional-memory run. This is a +// parser bound, never a declaration of available RAM or an allocation fallback. +static inline int DarwinWindowSizeValid(unsigned long long size) { + return size >= 0x10000000ULL && size <= 0x200000000ULL && !(size & 0x1fffffULL); +} +#endif diff --git a/drivers/DarwinBoot/LICENSE b/drivers/DarwinBoot/LICENSE new file mode 100644 index 0000000..4a71558 --- /dev/null +++ b/drivers/DarwinBoot/LICENSE @@ -0,0 +1,51 @@ +Copyright (c) 2026, DarwinBoot contributors. All rights reserved. + +SPDX-License-Identifier: BSD-2-Clause-Patent + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +Subject to the terms and conditions of this license, each copyright holder +and contributor hereby grants to those receiving rights under this license +a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable +(except for failure to satisfy the conditions of this license) patent +license to make, have made, use, offer to sell, sell, import, and otherwise +transfer this software, where such license applies only to those patent +claims, already acquired or hereafter acquired, licensable by such copyright +holder or contributor that are necessarily infringed by: + +(a) their Contribution(s) (the licensed copyrights of copyright holders and + non-copyrightable additions of contributors, in source or binary form) + alone; or + +(b) combination of their Contribution(s) with the work of authorship to + which such Contribution(s) was added by such copyright holder or + contributor, if, at the time the Contribution is added, such addition + causes such combination to be necessarily infringed. The patent license + shall not apply to any other combinations which include the + Contribution. + +Except as expressly stated above, no rights or licenses from any copyright +holder or contributor is granted under this license, whether expressly, by +implication, estoppel or otherwise. + +DISCLAIMER + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. diff --git a/drivers/DarwinBoot/Test/Kernel.S b/drivers/DarwinBoot/Test/Kernel.S new file mode 100644 index 0000000..8d11163 --- /dev/null +++ b/drivers/DarwinBoot/Test/Kernel.S @@ -0,0 +1,188 @@ +// SPDX-License-Identifier: BSD-2-Clause-Patent +// Only this synthetic QEMU payload uses PL011 MMIO after ExitBootServices. +#ifndef FIXTURE_WINDOW_SIZE +#define FIXTURE_WINDOW_SIZE 0x10000000 +#endif +#ifndef FIXTURE_ADT_EXTRA +#define FIXTURE_ADT_EXTRA 0 +#endif +.section .text, "ax" +.global _start +_start: + mrs x1, sctlr_el1 + mov x2, #0x1005 + tst x1, x2 + b.ne failed + ldr w1, [x0] + mov w2, #0x20002 + cmp w1, w2 + b.ne failed + ldr x3, [x0, #8] + ldr x4, =0xfffffff007004000 + cmp x3, x4 + b.ne failed + ldr x5, [x0, #16] + eor x6, x3, x5 + tst x6, #0x1fffff + b.ne failed + adr x6, _start + sub x6, x6, x5 + cmp x6, #0x1000 + b.ne failed + ldr x6, [x0, #24] + mov x7, #FIXTURE_WINDOW_SIZE + cmp x6, x7 + b.ne failed + ldr x6, [x0, #0x478] + cmp x6, x7 + b.ne failed + add x7, x5, x7 + ldr x6, [x0, #32] + cmp x6, x7 + b.hi failed + cmp x6, x0 + b.ls failed +#if FIXTURE_GOP + ldp x8, x9, [x0, #0x28] + cbz x8, failed + cmp x9, #1 + b.ne failed + ldp x8, x9, [x0, #0x38] + cbz x9, failed + cmp x8, x9, lsl #2 + b.lo failed + ldp x8, x9, [x0, #0x48] + cbz x8, failed + cmp x9, #32 + b.ne failed +#else + ldp x8, x9, [x0, #0x28] + orr x8, x8, x9 + cbnz x8, failed + ldp x8, x9, [x0, #0x38] + orr x8, x8, x9 + cbnz x8, failed + ldp x8, x9, [x0, #0x48] + orr x8, x8, x9 + cbnz x8, failed +#endif + ldr x6, [x0, #0x60] + sub x6, x6, x3 + add x6, x6, x5 + ldr w7, [x6] + cmp w7, #1 + b.ne failed + ldr w7, [x6, #4] + cmp w7, #1 + b.ne failed + ldr w7, [x6, #40] + cmp w7, #12 + b.ne failed + // Actual owned RAMDisk address/rounded size patched into the nested ADT. + add x14, x6, #200 + FIXTURE_ADT_EXTRA + ldp w9, w15, [x14] + orr x9, x9, x15, lsl #32 + ldp w10, w15, [x14, #8] + orr x10, x10, x15, lsl #32 + cmp x10, #0x2000 + b.ne failed + tst x9, #0xfff + b.ne failed + cmp x9, x5 + b.ls failed + add x10, x9, x10 + ldr x11, [x0, #32] + cmp x10, x11 + b.hi failed + ldr x10, [x9] + ldr x11, =0x4b5349444d415244 + cmp x10, x11 + b.ne failed + add x9, x9, #0x1000 + ldrb w10, [x9] + cmp w10, #0xa5 + b.ne failed + ldrb w10, [x9, #1] + cbnz w10, failed + ldrb w10, [x9, #4095] + cbnz w10, failed + // Independent owned TrustCache binding: exact 56 bytes and zero padding. + add x14, x6, #252 + FIXTURE_ADT_EXTRA + ldp w12, w15, [x14] + orr x12, x12, x15, lsl #32 + ldp w13, w15, [x14, #8] + orr x13, x13, x15, lsl #32 + cmp x13, #56 + b.ne failed + tst x12, #0xfff + b.ne failed + add x14, x9, #0x1000 + cmp x12, x14 + b.lo failed + add x14, x12, #0x1000 + ldr x15, [x0, #32] + cmp x14, x15 + b.hi failed + ldp w13, w14, [x12] + cmp w13, #1 + b.ne failed + cmp w14, #8 + b.ne failed + ldr w13, [x12, #8] + cmp w13, #2 + b.ne failed + ldr w13, [x12, #28] + cmp w13, #1 + b.ne failed + ldrb w13, [x12, #32] + cmp w13, #0x11 + b.ne failed + ldrb w13, [x12, #52] + cmp w13, #2 + b.ne failed + ldrb w13, [x12, #56] + cbnz w13, failed + ldrb w13, [x12, #4095] + cbnz w13, failed + add x6, x5, #0x4000 + mov x7, #0x200 +1: + ldr x8, [x6], #8 + cbnz x8, failed + subs x7, x7, #1 + b.ne 1b + adr x6, command_line + add x7, x0, #0x6c +2: + ldrb w8, [x6], #1 + ldrb w9, [x7], #1 + cmp w8, w9 + b.ne failed + cbnz w8, 2b + adr x0, passed + b puts +failed: + adr x0, failure +puts: + mov x1, #0x09000000 +3: + ldrb w2, [x0], #1 + cbz w2, done +4: + ldr w3, [x1, #0x18] + tbnz w3, #5, 4b + str w2, [x1] + b 3b +done: + wfe + b done +command_line: + .asciz "darwinboot.fixture=1" +passed: +#if FIXTURE_GOP + .asciz "DARWINBOOT-FIXTURE: actual GOP boot video after ExitBootServices PASS\n" +#else + .asciz "DARWINBOOT-FIXTURE: boot_args, RAMDisk, TrustCache, zero-fill and MMU-off entry PASS\n" +#endif +failure: + .asciz "DARWINBOOT-FIXTURE: FAIL\n" diff --git a/drivers/DarwinBoot/Test/Validation.c b/drivers/DarwinBoot/Test/Validation.c new file mode 100644 index 0000000..62effc9 --- /dev/null +++ b/drivers/DarwinBoot/Test/Validation.c @@ -0,0 +1,141 @@ +// SPDX-License-Identifier: BSD-2-Clause-Patent +#include +#include +#include +#include +#include +#include +#include +#include "../DarwinBootPkg/Driver/BootVideo.h" + +STATIC +EFI_STATUS +ValidateVideo (VOID) +{ + EFI_GRAPHICS_OUTPUT_PROTOCOL Gop; + EFI_GRAPHICS_OUTPUT_PROTOCOL_MODE Mode; + EFI_GRAPHICS_OUTPUT_MODE_INFORMATION Info; + EFI_STATUS Status; + EFI_STATUS Expected; + UINTN Index; + UINT64 Video[6]; + + ZeroMem (&Gop, sizeof (Gop)); + for (Index = 0; Index < 8; Index++) { + ZeroMem (&Info, sizeof (Info)); + ZeroMem (&Mode, sizeof (Mode)); + Info.HorizontalResolution = 8; + Info.VerticalResolution = 8; + Info.PixelsPerScanLine = 8; + Info.PixelFormat = PixelBlueGreenRedReserved8BitPerColor; + Mode.Info = &Info; + Mode.SizeOfInfo = sizeof (Info); + Mode.FrameBufferBase = 0x80000000; // Synthetic input; never accessed. + Mode.FrameBufferSize = 256; + Gop.Mode = &Mode; + Expected = EFI_COMPROMISED_DATA; + switch (Index) { + case 0: Gop.Mode = NULL; break; + case 1: Mode.Info = NULL; break; + case 2: Info.PixelFormat = PixelRedGreenBlueReserved8BitPerColor; Expected = EFI_UNSUPPORTED; break; + case 3: Info.PixelsPerScanLine = 7; break; + case 4: Info.VerticalResolution = 0; break; + case 5: Mode.FrameBufferBase = 0; break; + case 6: Mode.FrameBufferSize = 255; break; + case 7: Mode.FrameBufferBase = MAX_UINT64 - 15; break; + } + Status = DarwinBootVideoFromGop (&Gop, Video); + if (Status != Expected) { break; } + } + if (Index != 8) { return EFI_DEVICE_ERROR; } + Print (L"DARWINBOOT-VIDEO-VALIDATION: 8 malformed/unsupported GOP field sets refused PASS\n"); + return EFI_SUCCESS; +} + +EFI_STATUS +EFIAPI +ValidationEntry (EFI_HANDLE ImageHandle, EFI_SYSTEM_TABLE *SystemTable) +{ + EFI_STATUS Status; + EFI_STATUS Expected[] = { EFI_UNSUPPORTED, EFI_LOAD_ERROR, EFI_LOAD_ERROR, + EFI_LOAD_ERROR, EFI_COMPROMISED_DATA, EFI_NOT_FOUND }; + CHAR16 *Paths[] = { L"bad-subtype", L"bad-command", L"bad-overlap", + L"bad-entry", L"bad-adt", L"missing-adt" }; + CHAR16 Path[128]; + EFI_LOADED_IMAGE_PROTOCOL *Loaded; + CRABAPPLE_XNU_LOADER_PROTOCOL *Loader; + CRABAPPLE_XNU_BOOT_REQUEST Request; + UINTN Index; + UINT8 RamdiskByte = 0x5a; + CHAR16 *RamdiskPaths[] = { L"missing-ramdisk-adt", L"stale-ramdisk-adt", + L"bad-ramdisk-size", L"duplicate-ramdisk" }; + CHAR16 *TrustPaths[] = { L"missing-trustcache", L"orphan-trustcache", L"stale-trustcache", + L"duplicate-trustcache", L"bad-trustcache-version", L"bad-trustcache-count", + L"bad-trustcache-offset", L"bad-trustcache-order" }; + + CHAR16 *MemoryPaths[] = { L"bad-memory-size", L"small-memory-window", L"unaligned-memory-window", + L"large-memory-window", L"duplicate-memory-window", L"misplaced-memory-window" }; + + (VOID)SystemTable; + Status = gBS->HandleProtocol (ImageHandle, &gEfiLoadedImageProtocolGuid, (VOID **)&Loaded); + if (EFI_ERROR (Status)) { return Status; } + Status = gBS->LocateProtocol (&gCrabAppleXnuLoaderProtocolGuid, NULL, (VOID **)&Loader); + if (EFI_ERROR (Status)) { return Status; } + ZeroMem (&Request, sizeof (Request)); + Request.Volume = Loaded->DeviceHandle; + Request.KernelPath = L"\\EFI\\CrabApple\\Payloads\\kernel.macho"; + if (Loader->Boot (Loader, NULL) != EFI_INVALID_PARAMETER) { goto Failed; } + Request.KextCount = 1; + if (Loader->Boot (Loader, &Request) != EFI_UNSUPPORTED) { goto Failed; } + Request.KextCount = 0; + Request.DeviceTreeSize = 1; + if (Loader->Boot (Loader, &Request) != EFI_UNSUPPORTED) { goto Failed; } + Request.DeviceTreeSize = 0; + Request.RamdiskSize = 1; + if (Loader->Boot (Loader, &Request) != EFI_BAD_BUFFER_SIZE) { goto Failed; } + Request.RamdiskSize = 0; + Request.Ramdisk = &RamdiskByte; + if (Loader->Boot (Loader, &Request) != EFI_BAD_BUFFER_SIZE) { goto Failed; } + Request.RamdiskSize = SIZE_256MB + 1; + if (Loader->Boot (Loader, &Request) != EFI_BAD_BUFFER_SIZE) { goto Failed; } + Request.RamdiskSize = 1; + for (Index = 0; Index < ARRAY_SIZE (RamdiskPaths); Index++) { + UnicodeSPrint (Path, sizeof (Path), L"\\EFI\\CrabApple\\Payloads\\%s.macho", RamdiskPaths[Index]); + Request.KernelPath = Path; + if (Loader->Boot (Loader, &Request) != EFI_COMPROMISED_DATA) { goto Failed; } + } + Request.KernelPath = L"\\EFI\\CrabApple\\Payloads\\kernel.macho"; + Request.Ramdisk = NULL; + Request.RamdiskSize = 0; + if (Loader->Boot (Loader, &Request) != EFI_UNSUPPORTED) { goto Failed; } + Request.BootArgs = L"\x100"; + if (Loader->Boot (Loader, &Request) != EFI_BAD_BUFFER_SIZE) { goto Failed; } + Request.BootArgs = NULL; + for (Index = 0; Index < ARRAY_SIZE (TrustPaths); Index++) { + UnicodeSPrint (Path, sizeof (Path), L"\\EFI\\CrabApple\\Payloads\\%s.macho", TrustPaths[Index]); + Request.KernelPath = Path; + if (Loader->Boot (Loader, &Request) != EFI_COMPROMISED_DATA) { goto Failed; } + } + Request.KernelPath = L"\\EFI\\CrabApple\\Payloads\\unavailable-memory-window.macho"; + if (Loader->Boot (Loader, &Request) != EFI_OUT_OF_RESOURCES) { goto Failed; } + for (Index = 0; Index < ARRAY_SIZE (MemoryPaths); Index++) { + UnicodeSPrint (Path, sizeof (Path), L"\\EFI\\CrabApple\\Payloads\\%s.macho", MemoryPaths[Index]); + Request.KernelPath = Path; + if (Loader->Boot (Loader, &Request) != EFI_COMPROMISED_DATA) { goto Failed; } + } + for (Index = 0; Index < ARRAY_SIZE (Paths); Index++) { + UnicodeSPrint (Path, sizeof (Path), L"\\EFI\\CrabApple\\Payloads\\%s.macho", Paths[Index]); + Request.KernelPath = Path; + Status = Loader->Boot (Loader, &Request); + if (Status != Expected[Index]) { + Print (L"DARWINBOOT-VALIDATION: %s returned %r expected %r\n", Paths[Index], Status, Expected[Index]); + goto Failed; + } + } + Print (L"DARWINBOOT-VALIDATION: 33 rejected requests; firmware retained PASS\n"); + if (EFI_ERROR (ValidateVideo ())) { goto Failed; } + return EFI_SUCCESS; +Failed: + Print (L"DARWINBOOT-VALIDATION: FAIL\n"); + return EFI_DEVICE_ERROR; +} diff --git a/drivers/DarwinBoot/Test/Validation.inf b/drivers/DarwinBoot/Test/Validation.inf new file mode 100644 index 0000000..ddba6bc --- /dev/null +++ b/drivers/DarwinBoot/Test/Validation.inf @@ -0,0 +1,26 @@ +## SPDX-License-Identifier: BSD-2-Clause-Patent +[Defines] + INF_VERSION = 0x00010019 + BASE_NAME = DarwinBootValidation + FILE_GUID = D30DA217-ED5B-49F9-8CEF-DA7EB04F9221 + MODULE_TYPE = UEFI_DRIVER + VERSION_STRING = 0.1 + ENTRY_POINT = ValidationEntry + +[Sources] + Validation.c + +[Packages] + MdePkg/MdePkg.dec + CrabApplePkg/CrabApplePkg.dec + +[LibraryClasses] + BaseMemoryLib + PrintLib + UefiBootServicesTableLib + UefiDriverEntryPoint + UefiLib + +[Protocols] + gCrabAppleXnuLoaderProtocolGuid + gEfiLoadedImageProtocolGuid diff --git a/drivers/DarwinBoot/Test/make-fixtures.py b/drivers/DarwinBoot/Test/make-fixtures.py new file mode 100755 index 0000000..79d491a --- /dev/null +++ b/drivers/DarwinBoot/Test/make-fixtures.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 +"""Generate freely redistributable Mach-O/ADT fixtures; no Apple binaries.""" +import argparse +from pathlib import Path +import shutil +import struct +import subprocess + +BASE = 0xfffffff007004000 + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('output', type=Path) + parser.add_argument('--memory-size', type=lambda value: int(value, 0), choices=[0x10000000, 0x20000000]) + parser.add_argument('--gop', action='store_true') + args = parser.parse_args() + output = args.output.resolve() + output.mkdir(parents=True, exist_ok=True) + tools = {name: shutil.which(name) for name in ('clang', 'ld.lld', 'llvm-objcopy')} + if not all(tools.values()): + parser.error('LLVM and lld are required on PATH') + subprocess.run([tools['clang'], '--target=aarch64-none-elf', '-c', + f'-DFIXTURE_WINDOW_SIZE={args.memory_size or 0x10000000}', + f'-DFIXTURE_GOP={int(args.gop)}', + f'-DFIXTURE_ADT_EXTRA={44 if args.memory_size else 0}', + str(Path(__file__).with_name('Kernel.S')), '-o', str(output / 'kernel.o')], check=True) + subprocess.run([tools['ld.lld'], f'-Ttext={BASE + 0x1000:#x}', '-e', '_start', + str(output / 'kernel.o'), '-o', str(output / 'kernel.elf')], check=True) + subprocess.run([tools['llvm-objcopy'], '-O', 'binary', str(output / 'kernel.elf'), + str(output / 'kernel-code.bin')], check=True) + code = (output / 'kernel-code.bin').read_bytes() + if len(code) > 0x1000: + raise SystemExit('Fixture exceeds the synthetic TEXT segment') + + def segment(name, address, size, offset, length, protection): + return struct.pack(' MAX_INPUT: + raise ValueError('Packed input size out of bounds') + magic, version, page, logical, count = struct.unpack_from('<8sIIQQ', blob) + if (magic != b'DBRDISK1' or version != 1 or page != PAGE or + not 0 < logical <= MAX_LOGICAL or logical % PAGE or + count > 65536 or count > (len(blob) - 32) // 16): + raise ValueError('Invalid packed RAM disk header') + cursor = 32 + 16 * count + end = 0 + result = [] + for index in range(count): + offset, length = struct.unpack_from(' logical or length > logical - offset or length > len(blob) - cursor): + raise ValueError('Invalid packed RAM disk extent') + result.append((offset, memoryview(blob)[cursor:cursor + length])) + end = offset + length + cursor += length + if cursor != len(blob): + raise ValueError('Trailing packed RAM disk bytes') + return logical, result + + +def logical_hash(logical, extents): + digest = hashlib.sha256() + end = 0 + for offset, data in [*extents, (logical, b'')]: + while end < offset: + count = min(len(ZERO), offset - end) + digest.update(memoryview(ZERO)[:count]) + end += count + digest.update(data) + end += len(data) + return digest.hexdigest() + + +def inspect(path): + size = path.stat().st_size + if not 0 < size <= MAX_INPUT: + raise ValueError('Encoded RAM disk exceeds 256 MiB input bound') + blob = path.read_bytes() + encoded_hash = hashlib.sha256(blob).hexdigest() + if blob[:8] == b'DBRDISK1': + logical, extents = regions(blob) + header = bytes(PAGE) + if extents and extents[0][0] == 0: + header = bytes(extents[0][1][:PAGE]) + return dict(bytes=logical, sha256=logical_hash(logical, extents), + encoded_bytes=size, encoded_sha256=encoded_hash, packed=True), header + if size % PAGE: + raise ValueError('Raw RAM disk must be page aligned') + return dict(bytes=size, sha256=encoded_hash, packed=False), blob[:PAGE] + + +def pack(source, destination): + logical = source.stat().st_size + if not 0 < logical <= MAX_LOGICAL or logical % PAGE: + raise ValueError('Raw input must be page aligned and at most 2 GiB') + extents = [] + digest = hashlib.sha256() + total = 32 + with source.open('rb') as stream: + for base in range(0, logical, len(ZERO)): + chunk = stream.read(min(len(ZERO), logical - base)) + if len(chunk) != min(len(ZERO), logical - base): + raise ValueError('Raw input changed during packing') + digest.update(chunk) + for at in range(0, len(chunk), PAGE): + page = chunk[at:at + PAGE] + if not any(page): + continue + offset = base + at + if extents and extents[-1][0] + len(extents[-1][1]) == offset: + extents[-1][1].extend(page) + else: + extents.append((offset, bytearray(page))) + total += 16 + total += PAGE + if total > MAX_INPUT or len(extents) > 65536: + raise ValueError('Nonzero input exceeds packed format bound') + with destination.open('xb') as output: + output.write(struct.pack('<8sIIQQ', b'DBRDISK1', 1, PAGE, logical, len(extents))) + for offset, data in extents: + output.write(struct.pack(' +#include +#include +typedef uint8_t UINT8; typedef uint32_t UINT32; typedef uint64_t UINT64; +typedef size_t UINTN; typedef int BOOLEAN; +#define STATIC static +#define CONST const +#define VOID void +#define TRUE 1 +#define FALSE 0 +#define SIZE_256MB 0x10000000ULL +#define SIZE_2GB 0x80000000ULL +#define EFI_PAGE_SIZE 4096 +#define CompareMem memcmp +#define CopyMem memcpy +static uint32_t ReadUnaligned32(const uint32_t *p) {uint32_t v; memcpy(&v,p,4);return v;} +static uint64_t ReadUnaligned64(const uint64_t *p) {uint64_t v; memcpy(&v,p,8);return v;} +#include "PackedRamdisk.h" +int validate(void *p, size_t n, size_t *expanded) { + BOOLEAN packed; return RamdiskLayout(p,n,expanded,&packed); +} +int decode(void *out, size_t capacity, void *p, size_t n) { + BOOLEAN packed; size_t expanded; + if (!RamdiskLayout(p,n,&expanded,&packed) || expanded>capacity) return 0; + memset(out,0,expanded); CopyRamdisk(out,p,n,packed); return 1; +} +''') + library = work / 'parser.dylib' + subprocess.run(['clang', '-Wall', '-Wextra', '-Werror', '-shared', '-fPIC', + '-I', str(header.parent), str(source), '-o', str(library)], check=True) + lib = ctypes.CDLL(str(library)) + lib.validate.argtypes = [ctypes.c_void_p, ctypes.c_size_t, ctypes.POINTER(ctypes.c_size_t)] + lib.decode.argtypes = [ctypes.c_void_p, ctypes.c_size_t, ctypes.c_void_p, ctypes.c_size_t] + + def valid(blob): + size = ctypes.c_size_t() + result = lib.validate(ctypes.create_string_buffer(blob), len(blob), ctypes.byref(size)) + return result, size.value + + raw = b'head' + bytes(4092) + bytes(8192) + b'tail' + bytes(4092) + (work / 'raw').write_bytes(raw) + pack(work / 'raw', work / 'packed') + blob = (work / 'packed').read_bytes() + assert valid(blob) == (1, len(raw)) + out = ctypes.create_string_buffer(len(raw)) + assert lib.decode(out, len(raw), ctypes.create_string_buffer(blob), len(blob)) == 1 + assert out.raw == raw + assert inspect(work / 'packed')[0]['bytes'] == len(raw) + maximum = struct.pack('<8sIIQQ', b'DBRDISK1', 1, 4096, 2 * 1024**3, 0) + assert valid(maximum) == (1, 2 * 1024**3) + assert regions(maximum) == (2 * 1024**3, []) + invalid = [blob[:31], blob[:-1], blob + b'x'] + for offset, format_, value in [(8, ' +#include "../DarwinBootPkg/Driver/WindowSize.h" +int main(void) { + const unsigned long long yes[]={0x10000000ULL,0xe0000000ULL,0x100000000ULL,0x130000000ULL,0x200000000ULL}; + const unsigned long long no[]={0,0xfffffffULL,0x10000001ULL,0x130001000ULL,0x200200000ULL,~0ULL}; + for(unsigned i=0;i +#include +#include +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wextern-c-compat" +#include +#pragma clang diagnostic pop +#include +#include + +extern "C" { +volatile uint32_t native_aes_kat_status = 0; +volatile uint32_t native_aes_registered = 0; +volatile uint32_t native_aes_interface_status = 0; +volatile uint32_t native_aes_completed = 0; +volatile uint32_t native_aes_unsupported_key = 0; +} + +class NativeAES : public IOAESAccelerator { + OSDeclareDefaultStructors(NativeAES) +public: + bool start(IOService *) override; + IOWorkLoop *getWorkLoop() const override { return IOService::getWorkLoop(); } + IOReturn callPlatformFunction(const OSSymbol *name, bool wait, void *a, void *b, void *c, void *d) override { + return IOService::callPlatformFunction(name, wait, a, b, c, d); + } + const char *getAlgorithmName() override { return "AES-CBC"; } + bool isKeySupported(unsigned selector) override { return selector == 0; } + unsigned getSupportedOptions() override { return 1; } + IOReturn performAES(void *, void *, task_t, unsigned long long, IOAESIV *, + IOAESOperation, IOAESKeyData *, IOAESCompletion, void *) override; + IOReturn performAES(IOMemoryDescriptor *, IOMemoryDescriptor *, unsigned long long, + IOAESIV *, IOAESOperation, IOAESKeyData *, unsigned long long, + unsigned long long, IOAESCompletion, void *) override; +}; +OSDefineMetaClassAndStructors(NativeAES, IOAESAccelerator) + +// No padding, hidden device keys, or change to the registered corecrypto modes. +static int cbc(const unsigned char *input, unsigned char *output, size_t bytes, + unsigned char iv[16], const IOAESKeyData &key, IOAESOperation operation) +{ + union { aes_encrypt_ctx encrypt; aes_decrypt_ctx decrypt; } context; + unsigned char nextIV[16]; + int result; + if (operation == IOAESDecrypt) { + memcpy(nextIV, input + bytes - 16, 16); + result = aes_decrypt_key(key.bytes, key.bits / 8, &context.decrypt); + if (!result) result = aes_decrypt_cbc(input, iv, static_cast(bytes / 16), output, &context.decrypt); + } else { + result = aes_encrypt_key(key.bytes, key.bits / 8, &context.encrypt); + if (!result) result = aes_encrypt_cbc(input, iv, static_cast(bytes / 16), output, &context.encrypt); + if (!result) memcpy(nextIV, output + bytes - 16, 16); + } + if (!result) memcpy(iv, nextIV, 16); + cc_clear(sizeof(context), &context); + cc_clear(sizeof(nextIV), nextIV); + return result; +} + +// NIST SP800-38A F.2.1/F.2.2, four CBC blocks, then split/in-place decryption. +static bool knownAnswerTest(NativeAES *provider) +{ + const unsigned char input[64] = { + 0x6b,0xc1,0xbe,0xe2,0x2e,0x40,0x9f,0x96,0xe9,0x3d,0x7e,0x11,0x73,0x93,0x17,0x2a, + 0xae,0x2d,0x8a,0x57,0x1e,0x03,0xac,0x9c,0x9e,0xb7,0x6f,0xac,0x45,0xaf,0x8e,0x51, + 0x30,0xc8,0x1c,0x46,0xa3,0x5c,0xe4,0x11,0xe5,0xfb,0xc1,0x19,0x1a,0x0a,0x52,0xef, + 0xf6,0x9f,0x24,0x45,0xdf,0x4f,0x9b,0x17,0xad,0x2b,0x41,0x7b,0xe6,0x6c,0x37,0x10}; + const unsigned char expected[64] = { + 0x76,0x49,0xab,0xac,0x81,0x19,0xb2,0x46,0xce,0xe9,0x8e,0x9b,0x12,0xe9,0x19,0x7d, + 0x50,0x86,0xcb,0x9b,0x50,0x72,0x19,0xee,0x95,0xdb,0x11,0x3a,0x91,0x76,0x78,0xb2, + 0x73,0xbe,0xd6,0xb8,0xe3,0xc1,0x74,0x3b,0x71,0x16,0xe6,0x9e,0x22,0x22,0x95,0x16, + 0x3f,0xf1,0xca,0xa1,0x68,0x1f,0xac,0x09,0x12,0x0e,0xca,0x30,0x75,0x86,0xe1,0xa7}; + IOAESKeyData key = {128, {0x2b,0x7e,0x15,0x16,0x28,0xae,0xd2,0xa6, + 0xab,0xf7,0x15,0x88,0x09,0xcf,0x4f,0x3c}, 0}; + unsigned char iv[16], output[64]; + for (unsigned i = 0; i < 16; ++i) iv[i] = static_cast(i); + bool result = !cbc(input, output, 64, iv, key, IOAESEncrypt) && !memcmp(output, expected, 64) && + !memcmp(iv, expected + 48, 16); + for (unsigned i = 0; i < 16; ++i) iv[i] = static_cast(i); + result = result && !cbc(output, output, 16, iv, key, IOAESDecrypt) && + !cbc(output + 16, output + 16, 48, iv, key, IOAESDecrypt) && !memcmp(output, input, 64) && + !memcmp(iv, expected + 48, 16); + const IOAESKeyData wideKeys[2] = { + {192, {0x8e,0x73,0xb0,0xf7,0xda,0x0e,0x64,0x52,0xc8,0x10,0xf3,0x2b,0x80,0x90,0x79,0xe5, + 0x62,0xf8,0xea,0xd2,0x52,0x2c,0x6b,0x7b}, 0}, + {256, {0x60,0x3d,0xeb,0x10,0x15,0xca,0x71,0xbe,0x2b,0x73,0xae,0xf0,0x85,0x7d,0x77,0x81, + 0x1f,0x35,0x2c,0x07,0x3b,0x61,0x08,0xd7,0x2d,0x98,0x10,0xa3,0x09,0x14,0xdf,0xf4}, 0}}; + const unsigned char wideExpected[2][16] = { + {0x4f,0x02,0x1d,0xb2,0x43,0xbc,0x63,0x3d,0x71,0x78,0x18,0x3a,0x9f,0xa0,0x71,0xe8}, + {0xf5,0x8c,0x4c,0x04,0xd6,0xe5,0xf1,0xba,0x77,0x9e,0xab,0xfb,0x5f,0x7b,0xfb,0xd6}}; + for (unsigned n = 0; n < 2; ++n) { + for (unsigned i = 0; i < 16; ++i) iv[i] = static_cast(i); + result = result && !cbc(input, output, 16, iv, wideKeys[n], IOAESEncrypt) && !memcmp(output, wideExpected[n], 16); + for (unsigned i = 0; i < 16; ++i) iv[i] = static_cast(i); + result = result && !cbc(output, output, 16, iv, wideKeys[n], IOAESDecrypt) && !memcmp(output, input, 16); + } + IOBufferMemoryDescriptor *source = IOBufferMemoryDescriptor::withCapacity(128, kIODirectionInOut); + IOBufferMemoryDescriptor *destination = IOBufferMemoryDescriptor::withCapacity(128, kIODirectionInOut); + IOAESIV publicIV; + for (unsigned i = 0; i < 16; ++i) publicIV.bytes[i] = static_cast(i); + struct Completion { unsigned count; int status; } completion = {}; + auto callback = [](void *argument, int status) { + Completion *state = static_cast(argument); + state->count++; + state->status = status; + }; + bool interface = source && destination && source->writeBytes(16, input, 64) == 64 && + !provider->performAES(source, destination, 64, &publicIV, IOAESEncrypt, &key, 16, 32, callback, &completion) && + destination->readBytes(32, output, 64) == 64 && !memcmp(output, expected, 64) && + !memcmp(publicIV.bytes, expected + 48, 16) && completion.count == 1 && completion.status == 0; + key.selector = 2001; // Original hardware-key selector, deliberately unsupported. + interface = interface && provider->performAES(source, destination, 64, &publicIV, + IOAESEncrypt, &key, 16, 32, callback, &completion) == kIOReturnUnsupported && completion.count == 1 && + destination->readBytes(32, output, 64) == 64 && !memcmp(output, expected, 64) && + !memcmp(publicIV.bytes, expected + 48, 16) && !provider->isKeySupported(key.selector); + key.selector = 0; + interface = interface && provider->performAES(source, destination, 64, &publicIV, + IOAESEncrypt, &key, 16, 80, callback, &completion) == kIOReturnBadArgument && completion.count == 1; + for (unsigned i = 0; i < 16; ++i) publicIV.bytes[i] = static_cast(i); + interface = interface && !provider->performAES(const_cast(input), output, kernel_task, + 64, &publicIV, IOAESEncrypt, &key, nullptr, nullptr) && !memcmp(output, expected, 64); + for (unsigned i = 0; i < 16; ++i) publicIV.bytes[i] = static_cast(i); + interface = interface && !provider->performAES(output, output, kernel_task, 64, &publicIV, + IOAESDecrypt, &key, nullptr, nullptr) && !memcmp(output, input, 64) && !memcmp(publicIV.bytes, expected + 48, 16); + native_aes_interface_status = interface ? 1 : 2; + if (source) source->release(); + if (destination) destination->release(); + result = result && interface; + cc_clear(sizeof(key), &key); + cc_clear(sizeof(output), output); + return result; +} + +bool NativeAES::start(IOService *provider) +{ + if (!g_crypto_funcs || !g_crypto_funcs->ccaes_cbc_encrypt || !g_crypto_funcs->ccaes_cbc_decrypt || + !IOService::start(provider)) return false; + native_aes_kat_status = knownAnswerTest(this) ? 1 : 2; + if (native_aes_kat_status != 1) return false; + setProperty("NativeAESSoftwareKeysOnly", true); + native_aes_registered = 1; + registerService(); + return true; +} + +IOReturn NativeAES::performAES(IOMemoryDescriptor *input, IOMemoryDescriptor *output, + unsigned long long bytes, IOAESIV *iv, IOAESOperation operation, + IOAESKeyData *key, unsigned long long inputOffset, + unsigned long long outputOffset, IOAESCompletion callback, void *refcon) +{ + if (!input || !output || !key || !iv || !bytes || (bytes & 15) || + (key->bits != 128 && key->bits != 192 && key->bits != 256)) return kIOReturnBadArgument; + if (key->selector || (operation != IOAESEncrypt && operation != IOAESDecrypt)) { + __atomic_fetch_add(&native_aes_unsupported_key, 1, __ATOMIC_RELAXED); + return kIOReturnUnsupported; + } + if (inputOffset > input->getLength() || bytes > input->getLength() - inputOffset || + outputOffset > output->getLength() || bytes > output->getLength() - outputOffset) return kIOReturnBadArgument; + unsigned char *buffer = static_cast(IOMalloc(4096)); + if (!buffer) return kIOReturnNoMemory; + IOReturn result = input->prepare(); + if (result) { + IOFree(buffer, 4096); + return result; + } + result = output->prepare(); + if (result) { + input->complete(); + IOFree(buffer, 4096); + return result; + } + unsigned char chaining[16]; + memcpy(chaining, iv->bytes, 16); + for (unsigned long long done = 0; done < bytes; ) { + size_t count = static_cast(bytes - done > 4096 ? 4096 : bytes - done); + if (input->readBytes(inputOffset + done, buffer, count) != count || + cbc(buffer, buffer, count, chaining, *key, operation) || + output->writeBytes(outputOffset + done, buffer, count) != count) { + result = kIOReturnIOError; + break; + } + done += count; + } + if (!result) { + memcpy(iv->bytes, chaining, 16); + __atomic_fetch_add(&native_aes_completed, 1, __ATOMIC_RELAXED); + } + cc_clear(4096, buffer); + IOFree(buffer, 4096); + cc_clear(sizeof(chaining), chaining); + output->complete(); + input->complete(); + if (callback) { + callback(refcon, result); + return kIOReturnSuccess; + } + return result; +} + +IOReturn NativeAES::performAES(void *input, void *output, task_t task, unsigned long long bytes, + IOAESIV *iv, IOAESOperation operation, IOAESKeyData *key, + IOAESCompletion callback, void *refcon) +{ + if (!task || !input || !output || !bytes) return kIOReturnBadArgument; + IOMemoryDescriptor *source = IOMemoryDescriptor::withAddressRange( + reinterpret_cast(input), bytes, kIODirectionOut, task); + IOMemoryDescriptor *destination = IOMemoryDescriptor::withAddressRange( + reinterpret_cast(output), bytes, kIODirectionIn, task); + if (!source || !destination) { + if (source) source->release(); + if (destination) destination->release(); + return kIOReturnNoMemory; + } + IOReturn result = performAES(source, destination, bytes, iv, operation, key, 0, 0, callback, refcon); + source->release(); + destination->release(); + return result; +} diff --git a/drivers/NativeAES/OriginalAESABI.h b/drivers/NativeAES/OriginalAESABI.h new file mode 100644 index 0000000..e38d62b --- /dev/null +++ b/drivers/NativeAES/OriginalAESABI.h @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: BSD-2-Clause-Patent +#pragma once +#include +#include + +// Ordinary iOS ABI, qualified against the original component and live metadata. +class IOCryptoAccelerator : public IOService { + OSDeclareDefaultStructors(IOCryptoAccelerator) +}; +struct IOAESIV { unsigned char bytes[16]; }; +struct IOAESKeyData { + uint32_t bits; + unsigned char bytes[32]; + uint32_t selector; +}; +enum IOAESOperation : unsigned { IOAESEncrypt = 0, IOAESDecrypt = 1 }; +class IOAESAcceleratorCommand; +using IOAESCompletion = void (*)(void *, int); +class IOAESAccelerator : public IOCryptoAccelerator { + OSDeclareDefaultStructors(IOAESAccelerator) + unsigned char originalFields[0x98]; +public: + bool start(IOService *) override; + IOWorkLoop *getWorkLoop() const override; + IOReturn callPlatformFunction(const OSSymbol *, bool, void *, void *, void *, void *) override; + virtual const char *getAlgorithmName(); + virtual IOReturn performAES(void *, void *, task_t, unsigned long long, IOAESIV *, + IOAESOperation, IOAESKeyData *, IOAESCompletion, void *); + virtual IOReturn performAES(IOMemoryDescriptor *, IOMemoryDescriptor *, unsigned long long, + IOAESIV *, IOAESOperation, IOAESKeyData *, unsigned long long, + unsigned long long, IOAESCompletion, void *); + virtual bool isKeySupported(unsigned); + virtual unsigned getSupportedOptions(); + virtual IOReturn performAESQuantum(IOAESAcceleratorCommand *); + virtual void completeAESQuantum(IOAESAcceleratorCommand *); + virtual IOReturn performImmediateAES(IOAESAcceleratorCommand *); + virtual void disableHardwareKey(unsigned); + virtual void createSpecialKeys(); + virtual IOReturn _queueLocked(IOAESAcceleratorCommand *); + virtual void _startCommand(); + virtual void _completeAES(IOAESAcceleratorCommand *); +}; +static_assert(sizeof(IOService) == 0x88, "source IOService ABI changed"); +static_assert(sizeof(IOCryptoAccelerator) == 0x88, "original crypto parent ABI changed"); +static_assert(sizeof(IOAESAccelerator) == 0x120, "original AES instance ABI changed"); +static_assert(sizeof(IOAESIV) == 16 && __builtin_offsetof(IOAESKeyData, selector) == 36, + "original raw key/IV ABI changed"); diff --git a/drivers/NativeAdreno/Info.plist b/drivers/NativeAdreno/Info.plist new file mode 100644 index 0000000..c93bad2 --- /dev/null +++ b/drivers/NativeAdreno/Info.plist @@ -0,0 +1,23 @@ + + + + CFBundleIdentifierorg.blu-sharky.NativeAdreno + CFBundleNameNativeAdreno + CFBundleExecutableNativeAdreno + CFBundlePackageTypeKEXT + CFBundleVersion0.1.0 + OSBundleRequiredRoot + IOKitPersonalitiessheng SM8550 Adreno + IOClassNativeAdrenoGpuService + IOProviderClassIOPlatformDevice + IOProbeScore1000 + sheng SM8550 Adreno SMMU + IOClassNativeAdrenoSmmuService + IOProviderClassIOPlatformDevice + IOProbeScore1000 + sheng SM8550 Adreno GMU + IOClassNativeAdrenoGmuService + IOProviderClassIOPlatformDevice + IOProbeScore1000 + + diff --git a/drivers/NativeAdreno/LICENSE b/drivers/NativeAdreno/LICENSE new file mode 100644 index 0000000..8803ca3 --- /dev/null +++ b/drivers/NativeAdreno/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 NativeAdreno contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/drivers/NativeAdreno/NativeAdrenoA740HwInit.h b/drivers/NativeAdreno/NativeAdrenoA740HwInit.h new file mode 100644 index 0000000..e5cdfe4 --- /dev/null +++ b/drivers/NativeAdreno/NativeAdrenoA740HwInit.h @@ -0,0 +1,168 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef NATIVE_ADRENO_A740_HWINIT_H +#define NATIVE_ADRENO_A740_HWINIT_H +#include +/* Generated by scripts/derive-a740-hwinit.py from the pinned a6xx.xml/adreno_common.xml/a6xx_gmu.xml (A7XX + * variants) and a6xx_catalog.c (a740_hwcg, a730_protect). Dword register offsets. Do not edit by hand. */ +#define A740_GBIF_HALT 0x03c45 +#define A740_RBBM_GBIF_HALT 0x00016 +#define A740_RBBM_SECVID_TSB_CNTL 0x0f803 +#define A740_RBBM_SECVID_TSB_TRUSTED_BASE 0x0f800 +#define A740_RBBM_SECVID_TSB_TRUSTED_SIZE 0x0f802 +#define A740_RBBM_GBIF_CLIENT_QOS_CNTL 0x00011 +#define A740_UCHE_GBIF_GX_CONFIG 0x00e3a +#define A740_RBBM_PERFCTR_GPU_BUSY_MASKED 0x0050b +#define A740_UCHE_TRAP_BASE 0x00e09 +#define A740_UCHE_WRITE_THRU_BASE 0x00e07 +#define A740_UCHE_GMEM_RANGE_MIN 0x00e0b +#define A740_UCHE_GMEM_RANGE_MAX 0x00e0d +#define A740_UCHE_CACHE_WAYS 0x00e17 +#define A740_CP_AHB_CNTL 0x0098d +#define A740_RBBM_PERFCTR_CNTL 0x00500 +#define A740_RBBM_INTERFACE_HANG_INT_CNTL 0x0001f +#define A740_UCHE_CLIENT_PF 0x00e19 +#define A740_CP_PROTECT_CNTL 0x0084f +#define A740_UCHE_CMDQ_CONFIG 0x00e3c +#define A740_CP_APRIV_CNTL 0x00844 +#define A740_CP_BV_APRIV_CNTL 0x00ad0 +#define A740_CP_LPAC_APRIV_CNTL 0x00b31 +#define A740_RB_CMP_DBG_ECO_CNTL 0x08e28 +#define A740_RBBM_INT_0_MASK 0x00038 +#define A740_CP_SQE_INSTR_BASE 0x00830 +#define A740_CP_RB_BASE 0x00800 +#define A740_CP_RB_CNTL 0x00802 +#define A740_CP_RB_RPTR_ADDR 0x00804 +#define A740_CP_BV_RB_RPTR_ADDR 0x00a98 +#define A740_CP_SQE_CNTL 0x00808 +#define A740_CP_RB_RPTR 0x00806 +#define A740_CP_RB_WPTR 0x00807 +#define A740_RBBM_STATUS 0x00210 +#define A740_RBBM_INT_0_STATUS 0x00201 +#define A740_CP_HW_FAULT 0x00821 +#define A740_CP_INTERRUPT_STATUS 0x00823 +#define A740_RBBM_CLOCK_CNTL 0x000ae +#define A740_RBBM_SECVID_TRUST_CNTL 0x0f400 +#define A740_RBBM_STATUS3 0x00213 +#define A740_CP_SQE_STAT_ADDR 0x00908 +#define A740_CP_SQE_STAT_DATA 0x00909 +#define A740_RB_NC_MODE_CNTL 0x08e08 +#define A740_TPL1_NC_MODE_CNTL 0x0b604 +#define A740_SP_NC_MODE_CNTL 0x0ae02 +#define A740_CP_APERTURE_CNTL_HOST 0x00a00 +#define A740_GRAS_NC_MODE_CNTL 0x08602 +#define A740_UCHE_MODE_CNTL 0x00e01 +#define A740_RBBM_NC_MODE_CNTL 0x00534 +#define A740_RBBM_INT_CLEAR_CMD 0x00037 +#define A740_CP_ALWAYS_ON_COUNTER 0x00980 +#define A740_GPU_GMU_AO_GMU_CGC_MODE_CNTL 0x23b09 +#define A740_GPU_GMU_AO_GMU_CGC_DELAY_CNTL 0x23b0a +#define A740_GPU_GMU_AO_GMU_CGC_HYST_CNTL 0x23b0b +#define A740_CP_PROTECT 0x00850 +static const uint32_t NativeAdrenoA740Hwcg[][2]={ + {0x000b0,0x02222222}, /* RBBM_CLOCK_CNTL_SP0 */ + {0x000b4,0x22022222}, /* RBBM_CLOCK_CNTL2_SP0 */ + {0x000bc,0x003cf3cf}, /* RBBM_CLOCK_HYST_SP0 */ + {0x000b8,0x00000080}, /* RBBM_CLOCK_DELAY_SP0 */ + {0x000c0,0x22222220}, /* RBBM_CLOCK_CNTL_TP0 */ + {0x000c4,0x22222222}, /* RBBM_CLOCK_CNTL2_TP0 */ + {0x000c8,0x22222222}, /* RBBM_CLOCK_CNTL3_TP0 */ + {0x000cc,0x00222222}, /* RBBM_CLOCK_CNTL4_TP0 */ + {0x000e0,0x77777777}, /* RBBM_CLOCK_HYST_TP0 */ + {0x000e4,0x77777777}, /* RBBM_CLOCK_HYST2_TP0 */ + {0x000e8,0x77777777}, /* RBBM_CLOCK_HYST3_TP0 */ + {0x000ec,0x00077777}, /* RBBM_CLOCK_HYST4_TP0 */ + {0x000d0,0x11111111}, /* RBBM_CLOCK_DELAY_TP0 */ + {0x000d4,0x11111111}, /* RBBM_CLOCK_DELAY2_TP0 */ + {0x000d8,0x11111111}, /* RBBM_CLOCK_DELAY3_TP0 */ + {0x000dc,0x00011111}, /* RBBM_CLOCK_DELAY4_TP0 */ + {0x0010b,0x22222222}, /* RBBM_CLOCK_CNTL_UCHE */ + {0x0010c,0x00222222}, /* RBBM_CLOCK_CNTL2_UCHE */ + {0x00110,0x00000444}, /* RBBM_CLOCK_HYST_UCHE */ + {0x0010f,0x00000222}, /* RBBM_CLOCK_DELAY_UCHE */ + {0x000f0,0x22222222}, /* RBBM_CLOCK_CNTL_RB0 */ + {0x000f4,0x01002222}, /* RBBM_CLOCK_CNTL2_RB0 */ + {0x000f8,0x00002220}, /* RBBM_CLOCK_CNTL_CCU0 */ + {0x00100,0x44000f00}, /* RBBM_CLOCK_HYST_RB_CCU0 */ + {0x00104,0x25222022}, /* RBBM_CLOCK_CNTL_RAC */ + {0x00105,0x00555555}, /* RBBM_CLOCK_CNTL2_RAC */ + {0x00106,0x00000011}, /* RBBM_CLOCK_DELAY_RAC */ + {0x00107,0x00440044}, /* RBBM_CLOCK_HYST_RAC */ + {0x00108,0x04222222}, /* RBBM_CLOCK_CNTL_TSE_RAS_RBBM */ + {0x00286,0x00000222}, /* RBBM_CLOCK_MODE2_GRAS */ + {0x00285,0x00222222}, /* RBBM_CLOCK_MODE_BV_GRAS */ + {0x00114,0x02222223}, /* RBBM_CLOCK_MODE_GPC */ + {0x00111,0x00222222}, /* RBBM_CLOCK_MODE_VFD */ + {0x00288,0x00222222}, /* RBBM_CLOCK_MODE_BV_GPC */ + {0x00287,0x00002222}, /* RBBM_CLOCK_MODE_BV_VFD */ + {0x0010a,0x00000000}, /* RBBM_CLOCK_HYST_TSE_RAS_RBBM */ + {0x00116,0x04104004}, /* RBBM_CLOCK_HYST_GPC */ + {0x00113,0x00000000}, /* RBBM_CLOCK_HYST_VFD */ + {0x00109,0x00000000}, /* RBBM_CLOCK_DELAY_TSE_RAS_RBBM */ + {0x00115,0x00000200}, /* RBBM_CLOCK_DELAY_GPC */ + {0x00112,0x00000000}, /* RBBM_CLOCK_DELAY_VFD */ + {0x0011b,0x00002222}, /* RBBM_CLOCK_MODE_HLSQ */ + {0x0011c,0x00000000}, /* RBBM_CLOCK_DELAY_HLSQ */ + {0x0011d,0x00000000}, /* RBBM_CLOCK_HYST_HLSQ */ + {0x00284,0x55555552}, /* RBBM_CLOCK_MODE_BV_LRZ */ + {0x0012f,0x00000000}, /* RBBM_CLOCK_HYST2_VFD */ + {0x00260,0x00000222}, /* RBBM_CLOCK_MODE_CP */ + {0x000ae,0x8aa8aa82}, /* RBBM_CLOCK_CNTL */ + {0x00533,0x00000182}, /* RBBM_ISDB_CNT */ + {0x00044,0x00000000}, /* RBBM_RAC_THRESHOLD_CNT */ + {0x00042,0x00000000}, /* RBBM_SP_HYST_CNT */ + {0x00118,0x00000222}, /* RBBM_CLOCK_CNTL_GMU_GX */ + {0x00119,0x00000111}, /* RBBM_CLOCK_DELAY_GMU_GX */ + {0x0011a,0x00000555}, /* RBBM_CLOCK_HYST_GMU_GX */ +}; +#define NATIVE_ADRENO_A740_HWCG_COUNT 54 +/* {CP_PROTECT index, value} in a6xx_set_cp_protect order. */ +static const uint32_t NativeAdrenoA740Protect[][2]={ + { 0,0x13fc0000}, + { 1,0x0160050b}, + { 2,0x8000050e}, + { 3,0x80000510}, + { 4,0x80000534}, + { 5,0x027405fb}, + { 6,0x87a40699}, + { 7,0x802008a0}, + { 8,0x809008ab}, + { 9,0x800408de}, + {10,0x052c08e7}, + {11,0x81340900}, + {12,0x82c8098d}, + {13,0x86f80a41}, + {14,0x80040df0}, + {15,0x80000e01}, + {16,0x80200e07}, + {17,0x830c3c00}, + {18,0x7ffc3cc4}, + {19,0x873c8630}, + {20,0x80008e00}, + {21,0x80008e08}, + {22,0x807c8e50}, + {23,0x8a008e80}, + {24,0x876c9624}, + {25,0x80009e40}, + {26,0x80349e64}, + {27,0x861c9e78}, + {28,0x873ca630}, + {29,0x8000ae02}, + {30,0x803cae50}, + {31,0x800cae66}, + {32,0x800cae6f}, + {33,0x800cb604}, + {34,0xbffcec00}, + {35,0x7ffcfc00}, + {36,0x814d8400}, + {37,0x00118454}, + {38,0xfffd8459}, + {39,0xfffda459}, + {40,0xfffdc459}, + {41,0x910df400}, + {42,0x01edf844}, + {43,0x8001f860}, + {44,0x80a9f878}, + {47,0x8001f8c0}, +}; +#define NATIVE_ADRENO_A740_PROTECT_COUNT 46 +#endif diff --git a/drivers/NativeAdreno/NativeAdrenoABI.h b/drivers/NativeAdreno/NativeAdrenoABI.h new file mode 100644 index 0000000..3ca23cd --- /dev/null +++ b/drivers/NativeAdreno/NativeAdrenoABI.h @@ -0,0 +1,165 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef NATIVE_ADRENO_ABI_H +#define NATIVE_ADRENO_ABI_H +#include + +/* Versioned kernel/user-client wire ABI for the native Adreno driver. This is + * NOT the Metal, IOKit or Turnip ABI and it does not describe an ioctl. The + * IOKit service will copy exactly these fixed-size records across its external + * methods. All fields are little endian; both participants are ARM64. Every + * array is inside one fixed record so a caller cannot pass a length that + * disagrees with the storage. Unknown versions, methods and flags are refused + * rather than ignored. */ +enum { + NativeAdrenoAbiVersion = 1, + NativeAdrenoAbiMaxBuffers = 16, + NativeAdrenoAbiMaxBoBytesShift = 28, /* 256MiB per object */ + NativeAdrenoMethodQuery = 1, + NativeAdrenoMethodBoAlloc = 2, + NativeAdrenoMethodBoMap = 3, + NativeAdrenoMethodBoFree = 4, + NativeAdrenoMethodSubmit = 5, + NativeAdrenoMethodWaitFence = 6, + NativeAdrenoMethodGetTimestamp = 7, + NativeAdrenoMethodGetStatus = 8, + NativeAdrenoMethodSetMetadata = 9, + NativeAdrenoMethodGetMetadata = 10, + NativeAdrenoMethodCreateQueue = 11, + NativeAdrenoMethodCloseQueue = 12, + NativeAdrenoMethodBoCpuMap = 13, + NativeAdrenoMethodBoCpuUnmap = 14, + NativeAdrenoMethodGetDeviceInfo = 15, + NativeAdrenoMethodBoImport = 16, + NativeAdrenoAbiMetadataBytes = 64, + NativeAdrenoAbiMaxQueues = 8 +}; +struct NativeAdrenoAbiParams { + uint32_t abi_version; + uint32_t input_bits; /* GPU VA bits below 2^input_bits */ + uint32_t output_bits; /* physical address bits */ + uint32_t flags; /* NativeAdrenoParamsHostImport or zero */ + uint64_t max_bo_bytes; +}; +enum { + NativeAdrenoParamsHostImport = 1 /* BoImport pins the caller's own pages (an IOSurface's) as a BO */ +}; +struct NativeAdrenoAbiBoAlloc { + uint64_t bytes; /* 4KiB multiple, <= max_bo_bytes */ + uint32_t align_log2; /* 12..24 */ + uint32_t contiguous; /* 1 requires one raw physical span */ +}; +struct NativeAdrenoAbiBoAllocReply { + uint64_t handle; /* opaque to userspace, nonzero */ +}; +/* Pins [address, address + bytes) of the CALLING task and makes it a BO (reply: NativeAdrenoAbiBoAllocReply). The pages stay wired + * until BoFree; they keep the caller's own CPU mapping and cache mode, so the caller cleans its caches before GPU use and invalidates + * them after GPU writes (the GPU maps them non-cacheable). An imported BO has no CPU-map token. */ +struct NativeAdrenoAbiBoImport { + uint64_t address; /* 4KiB aligned */ + uint64_t bytes; /* 4KiB multiple, <= max_bo_bytes */ +}; +struct NativeAdrenoAbiBoMap { + uint64_t handle; + uint64_t iova; /* 4KiB aligned, inside the mounted aperture */ + uint32_t permissions; /* bit0 read, bit1 write; read required */ + uint32_t writeback; /* 0 normal non-coherent, 1 WB/WA */ + uint32_t shareability; /* 2 outer, 3 inner */ + uint32_t reserved; /* zero */ +}; +struct NativeAdrenoAbiBoMapReply { + uint64_t iova; +}; +struct NativeAdrenoAbiBoFree { + uint64_t handle; +}; +struct NativeAdrenoAbiSubmitBuffer { + uint64_t handle; + uint64_t iova; + uint64_t bytes; /* 4-byte multiple, dword count = bytes/4 */ +}; +struct NativeAdrenoAbiSubmit { + uint32_t buffer_count; /* 1..NativeAdrenoAbiMaxBuffers */ + uint32_t reserved; /* zero */ + uint64_t queue; /* open queue; 0 is created by open() */ + uint64_t wait_fence; /* zero or an earlier fence of this client */ + uint64_t wait_timeout_ns; /* used only when wait_fence is nonzero */ + NativeAdrenoAbiSubmitBuffer buffers[NativeAdrenoAbiMaxBuffers]; +}; +struct NativeAdrenoAbiMetadata { + uint64_t handle; /* object identity */ + uint32_t bytes; /* set: length; get: requested capacity */ + uint32_t reserved; /* zero */ + char text[NativeAdrenoAbiMetadataBytes]; /* raw bytes, not NUL terminated */ +}; +struct NativeAdrenoAbiSubmitReply { + uint64_t fence; /* nonzero CP timestamp value */ +}; +struct NativeAdrenoAbiWaitFence { + uint64_t fence; + uint64_t timeout_ns; /* 0 polls once */ +}; +struct NativeAdrenoAbiTimestamp { + uint64_t ticks; /* GPU always-on counter value */ +}; +struct NativeAdrenoAbiBoCpuMap { + uint64_t handle; +}; +struct NativeAdrenoAbiBoCpuMapReply { + uint32_t token; /* clientMemoryForType selector */ + uint32_t reserved; /* zero */ + uint64_t bytes; /* bytes the mapping covers */ +}; +struct NativeAdrenoAbiBoCpuUnmap { + uint64_t handle; + uint32_t token; /* exactly the token returned by map */ + uint32_t reserved; /* zero */ +}; +struct NativeAdrenoAbiQueue { + uint64_t queue; /* zero for create, an open id for close */ +}; +struct NativeAdrenoAbiQueueReply { + uint64_t queue; /* nonzero id of the created queue */ +}; +struct NativeAdrenoAbiStatus { + uint64_t faults; /* monotonic; change means context lost */ + uint64_t suspends; /* monotonic suspend/reset count */ +}; +/* Fixed facts of the mounted GPU that the Vulkan driver needs and cannot derive: all come from the board code, which takes + * them from the pinned sources (a6xx catalog, UBWC config, the programmed UCHE ranges), never from a guess. */ +enum { + NativeAdrenoDeviceClientIova = 1, /* flags: the client chooses BO addresses (BoMap iova) */ + NativeAdrenoDeviceCachedCoherent = 2 /* flags: unused until IO-coherent BO memory exists */ +}; +struct NativeAdrenoAbiDeviceInfo { + uint32_t chip_id; /* ADRENO_CHIPID, e.g. 0x43050a01 */ + uint32_t gmem_bytes; + uint64_t gmem_base; /* UCHE GMEM window start (GPU address) */ + uint64_t uche_trap_base; + uint64_t va_start; /* first GPU address the client may map */ + uint64_t va_size; + uint32_t highest_bank_bit; + uint32_t ubwc_swizzle; /* bit0 level 1, bit1 level 2, bit2 level 3 */ + uint32_t macrotile_mode; /* 0 four channel, 1 eight channel */ + uint32_t ubwc_version; /* encoder version, 4 for UBWC 4.0 */ + uint32_t flags; + uint32_t reserved; /* zero */ +}; +#if defined(__cplusplus) +static_assert(sizeof(NativeAdrenoAbiDeviceInfo) == 64, "ABI device info layout"); +static_assert(sizeof(NativeAdrenoAbiParams) == 24, "ABI params layout"); +static_assert(sizeof(NativeAdrenoAbiBoAlloc) == 16, "ABI alloc layout"); +static_assert(sizeof(NativeAdrenoAbiBoMap) == 32, "ABI map layout"); +static_assert(sizeof(NativeAdrenoAbiBoImport) == 16, "ABI import layout"); +static_assert(sizeof(NativeAdrenoAbiSubmitBuffer) == 24, "ABI buffer layout"); +static_assert(sizeof(NativeAdrenoAbiSubmit) == 416, "ABI submit layout"); +static_assert(sizeof(NativeAdrenoAbiBoCpuMap) == 8, "ABI cpu map layout"); +static_assert(sizeof(NativeAdrenoAbiBoCpuMapReply) == 16, "ABI cpu map reply layout"); +static_assert(sizeof(NativeAdrenoAbiBoCpuUnmap) == 16, "ABI cpu unmap layout"); +static_assert(sizeof(NativeAdrenoAbiQueue) == 8, "ABI queue layout"); +static_assert(sizeof(NativeAdrenoAbiQueueReply) == 8, "ABI queue reply layout"); +static_assert(sizeof(NativeAdrenoAbiMetadata) == 80, "ABI metadata layout"); +static_assert(sizeof(NativeAdrenoAbiWaitFence) == 16, "ABI wait layout"); +static_assert(sizeof(NativeAdrenoAbiTimestamp) == 8, "ABI timestamp layout"); +static_assert(sizeof(NativeAdrenoAbiStatus) == 16, "ABI status layout"); +#endif +#endif diff --git a/drivers/NativeAdreno/NativeAdrenoAccelerator.cpp b/drivers/NativeAdreno/NativeAdrenoAccelerator.cpp new file mode 100644 index 0000000..97a39ad --- /dev/null +++ b/drivers/NativeAdreno/NativeAdrenoAccelerator.cpp @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: MIT +// The registry-visible accelerator for the original Metal discovery (IOServiceMatching("IOAcceleratorES") -> +// MetalPluginName -> /System/Library/Extensions/.bundle). It derives from the original IOAcceleratorES base exactly as +// qualified in drivers/NativeMetal/OriginalAcceleratorABI.h: the original base has no operational overrides, so the only +// virtuals this class adds are newUserClient and the property reads below (diagnostic counting only). Its static initializer +// is staged out of __mod_init_func and run by the board probe after the original IOGPUFamily kext has loaded, because the +// parent metaclass does not exist before that. The class is named after the Apple accelerator class on purpose, so original +// sandbox profiles that name the service class accept the Adreno plugin's open; the Apple AGX kext is never loaded here. +#include "../NativeMetal/OriginalAcceleratorABI.h" +#include + +extern "C" IOReturn native_adreno_new_user_client(IOService *,task_t,void *,UInt32,OSDictionary *,IOUserClient **,unsigned); +extern "C" boolean_t PE_parse_boot_argn(const char *,void *,int); +extern "C" void native_adreno_note_property_read(const char *); + +class AGXAccelerator : public IOAcceleratorES { + OSDeclareDefaultStructors(AGXAccelerator) +public: + IOReturn newUserClient(task_t owningTask,void *securityID,UInt32 type,OSDictionary *properties, + IOUserClient **handler) APPLE_KEXT_OVERRIDE; + OSObject *copyProperty(const OSSymbol *key) const APPLE_KEXT_OVERRIDE; + OSObject *copyProperty(const OSString *key) const APPLE_KEXT_OVERRIDE; + OSObject *copyProperty(const char *key) const APPLE_KEXT_OVERRIDE; +}; +OSDefineMetaClassAndStructors(AGXAccelerator,IOAcceleratorES) + +IOReturn AGXAccelerator::newUserClient(task_t owningTask,void *securityID,UInt32 type,OSDictionary *properties, + IOUserClient **handler) +{ + unsigned limit=40; /* sheng-gpu-metal-opens: connections ever granted to the plugin (0 = unlimited) */ + PE_parse_boot_argn("sheng-gpu-metal-opens",&limit,sizeof(limit)); + return native_adreno_new_user_client(this,owningTask,securityID,type,properties,handler,limit); +} +OSObject *AGXAccelerator::copyProperty(const OSSymbol *key) const +{ + native_adreno_note_property_read(key?key->getCStringNoCopy():nullptr); + return IORegistryEntry::copyProperty(key); +} +OSObject *AGXAccelerator::copyProperty(const OSString *key) const +{ + native_adreno_note_property_read(key?key->getCStringNoCopy():nullptr); + return IORegistryEntry::copyProperty(key); +} +OSObject *AGXAccelerator::copyProperty(const char *key) const +{ + native_adreno_note_property_read(key); + return IORegistryEntry::copyProperty(key); +} +extern "C" IOService *native_adreno_accelerator_allocate(void) {return new AGXAccelerator;} +extern "C" unsigned native_adreno_accelerator_instance_size(void) {return sizeof(AGXAccelerator);} diff --git a/drivers/NativeAdreno/NativeAdrenoBoard.cpp b/drivers/NativeAdreno/NativeAdrenoBoard.cpp new file mode 100644 index 0000000..1f227fe --- /dev/null +++ b/drivers/NativeAdreno/NativeAdrenoBoard.cpp @@ -0,0 +1,57 @@ +/* SPDX-License-Identifier: MIT */ +#include "NativeAdrenoBoard.h" +enum { CellBytes = 4 }; +static uint32_t be32(const uint8_t *p) +{ + return (uint32_t(p[0])<<24)|(uint32_t(p[1])<<16)|(uint32_t(p[2])<<8)|uint32_t(p[3]); +} +IOReturn NativeAdrenoBoard::parseIommus(const uint8_t *cells,size_t length, + uint32_t cells_per_entry,uint16_t *stream_ids,size_t capacity,size_t *count) +{ + if(!count)return kIOReturnBadArgument; + *count=0; + if(!cells || !length || cells_per_entry<1 || cells_per_entry>4) + return kIOReturnBadArgument; + const size_t per_entry=(1+cells_per_entry)*(size_t)CellBytes; + if(length%per_entry)return kIOReturnBadArgument; + const size_t total=length/per_entry; + if(!stream_ids || total>capacity)return kIOReturnNoResources; + for(size_t index=0;index0xffffu)return kIOReturnBadArgument; + stream_ids[index]=static_cast(sid); + } + *count=total; + return kIOReturnSuccess; +} +IOReturn NativeAdrenoBoard::parseReg(const uint8_t *cells,size_t length, + uint32_t address_cells,uint32_t size_cells,NativeAdrenoWindow *windows, + size_t capacity,size_t *count) +{ + if(!count)return kIOReturnBadArgument; + *count=0; + if(!cells || !length || (address_cells<1 || address_cells>2) || + (size_cells<1 || size_cells>2))return kIOReturnBadArgument; + const size_t per_window=(address_cells+size_cells)*(size_t)CellBytes; + if(length%per_window)return kIOReturnBadArgument; + const size_t total=length/per_window; + if(total>capacity || !windows)return kIOReturnNoResources; + for(size_t index=0;index>32)return kIOReturnBadArgument; /* more than 64 bits */ + base=(base<<32)|be32(at+cell*CellBytes); + } + for(uint32_t cell=0;cell>32)return kIOReturnBadArgument; + bytes=(bytes<<32)|be32(at+(address_cells+cell)*CellBytes); + } + if(!bytes || base+bytes +#include +#include + +/* Device-tree memory-window parsing for an Adreno platform node. Cells are + * big-endian 32-bit as stored in the flattened device tree; the caller passes + * the raw property bytes and the node's own #address-cells/#size-cells. + * Only 1 or 2 cells per value are accepted, zero-length windows and any + * address/size overflow are refused, and the output count is the exact number + * of complete windows. This parses only; it does not map, claim, power or + * validate that the addresses belong to a GPU. */ +struct NativeAdrenoWindow { + uint64_t base; + uint64_t bytes; +}; +enum { NativeAdrenoMaxWindows = 8 }; +class NativeAdrenoBoard final { + NativeAdrenoBoard()=delete; +public: + /* Parse a device's `iommus` property. Each entry is one phandle cell + * followed by #iommu-cells specifier cells; the first specifier cell is the + * stream ID on this platform. Output keeps property order and never writes + * on failure. Only the SID is extracted, never a guessed context bank. */ + static IOReturn parseIommus(const uint8_t *cells,size_t length, + uint32_t cells_per_entry,uint16_t *stream_ids,size_t capacity,size_t *count); + static IOReturn parseReg(const uint8_t *cells,size_t length,uint32_t address_cells, + uint32_t size_cells,NativeAdrenoWindow *windows,size_t capacity,size_t *count); +}; +#endif diff --git a/drivers/NativeAdreno/NativeAdrenoClient.cpp b/drivers/NativeAdreno/NativeAdrenoClient.cpp new file mode 100644 index 0000000..3bf4449 --- /dev/null +++ b/drivers/NativeAdreno/NativeAdrenoClient.cpp @@ -0,0 +1,191 @@ +/* SPDX-License-Identifier: MIT */ +#include "NativeAdrenoClient.h" +#include +#include +IOReturn NativeAdrenoClient::open(const NativeAdrenoTransport *transport) +{ + if(open_ || transport_.call)return kIOReturnBusy; + if(!transport || !transport->call)return kIOReturnBadArgument; + transport_=*transport; + NativeAdrenoAbiParams reply{}; + const IOReturn status=transport_.call(transport_.cookie,NativeAdrenoMethodQuery, + nullptr,0,&reply,sizeof(reply)); + if(status) {transport_={};return status;} + if(reply.abi_version!=NativeAdrenoAbiVersion || reply.input_bits<32 || + reply.input_bits>48 || reply.output_bits<32 || reply.output_bits>48 || + !reply.max_bo_bytes || (reply.flags&~uint32_t(NativeAdrenoParamsHostImport))) {transport_={};return kIOReturnUnsupported;} + params_=reply;open_=true; + return kIOReturnSuccess; +} +IOReturn NativeAdrenoClient::close() +{ + if(!open_)return kIOReturnNotReady; + open_=false;transport_={};params_={}; + return kIOReturnSuccess; +} +IOReturn NativeAdrenoClient::query(NativeAdrenoAbiParams *params) +{ + if(!params)return kIOReturnBadArgument;*params={}; + if(!open_)return kIOReturnNotReady; + *params=params_; + return kIOReturnSuccess; +} +IOReturn NativeAdrenoClient::allocBo(uint64_t bytes,uint32_t align_log2,bool contiguous, + uint64_t *handle) +{ + if(!handle)return kIOReturnBadArgument;*handle=0; + if(!open_)return kIOReturnNotReady; + if(!bytes || (bytes&4095) || bytes>params_.max_bo_bytes || align_log2<12 || + align_log2>24)return kIOReturnBadArgument; + NativeAdrenoAbiBoAlloc request{bytes,align_log2,contiguous?1u:0u}; + NativeAdrenoAbiBoAllocReply reply{}; + const IOReturn status=transport_.call(transport_.cookie,NativeAdrenoMethodBoAlloc, + &request,sizeof(request),&reply,sizeof(reply)); + if(!status)*handle=reply.handle; + return status; +} +IOReturn NativeAdrenoClient::mapBo(uint64_t handle,uint64_t iova,uint32_t permissions, + uint32_t writeback,uint32_t shareability,uint64_t *mapped) +{ + if(!mapped)return kIOReturnBadArgument;*mapped=0; + if(!open_)return kIOReturnNotReady; + if(!handle || iova&4095)return kIOReturnBadArgument; + NativeAdrenoAbiBoMap request{handle,iova,permissions,writeback,shareability,0}; + NativeAdrenoAbiBoMapReply reply{}; + const IOReturn status=transport_.call(transport_.cookie,NativeAdrenoMethodBoMap, + &request,sizeof(request),&reply,sizeof(reply)); + if(!status)*mapped=reply.iova; + return status; +} +IOReturn NativeAdrenoClient::freeBo(uint64_t handle) +{ + if(!open_)return kIOReturnNotReady; + if(!handle)return kIOReturnBadArgument; + NativeAdrenoAbiBoFree request{handle}; + return transport_.call(transport_.cookie,NativeAdrenoMethodBoFree,&request, + sizeof(request),nullptr,0); +} +IOReturn NativeAdrenoClient::submit(const NativeAdrenoAbiSubmitBuffer *buffers,size_t count, + uint64_t wait_fence,uint64_t wait_timeout_ns,uint64_t *fence,uint64_t queue) +{ + if(!fence)return kIOReturnBadArgument;*fence=0; + if(!open_)return kIOReturnNotReady; + if(!buffers || !count || count>NativeAdrenoAbiMaxBuffers)return kIOReturnBadArgument; + NativeAdrenoAbiSubmit request{}; + request.buffer_count=(uint32_t)count; + request.queue=queue; + request.wait_fence=wait_fence; + request.wait_timeout_ns=wait_timeout_ns; + for(size_t i=0;iNativeAdrenoAbiMetadataBytes || (length && !bytes)) + return kIOReturnBadArgument; + NativeAdrenoAbiMetadata request{}; + request.handle=handle;request.bytes=length; + if(length)memcpy(request.text,bytes,length); + return transport_.call(transport_.cookie,NativeAdrenoMethodSetMetadata,&request, + sizeof(request),nullptr,0); +} +IOReturn NativeAdrenoClient::getMetadata(uint64_t handle,void *out,uint32_t capacity, + uint32_t *length) +{ + if(!length)return kIOReturnBadArgument;*length=0; + if(!open_)return kIOReturnNotReady; + if(!handle || capacity>NativeAdrenoAbiMetadataBytes || (capacity && !out)) + return kIOReturnBadArgument; + NativeAdrenoAbiMetadata request{}; + request.handle=handle;request.bytes=capacity; + NativeAdrenoAbiMetadata reply{}; + const IOReturn status=transport_.call(transport_.cookie,NativeAdrenoMethodGetMetadata, + &request,sizeof(request),&reply,sizeof(reply)); + if(status)return status; + if(reply.bytes>capacity)return kIOReturnBadArgument; + if(reply.bytes && out)memcpy(out,reply.text,reply.bytes); + *length=reply.bytes; + return kIOReturnSuccess; +} diff --git a/drivers/NativeAdreno/NativeAdrenoClient.h b/drivers/NativeAdreno/NativeAdrenoClient.h new file mode 100644 index 0000000..0645c92 --- /dev/null +++ b/drivers/NativeAdreno/NativeAdrenoClient.h @@ -0,0 +1,52 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef NATIVE_ADRENO_CLIENT_H +#define NATIVE_ADRENO_CLIENT_H +#include "NativeAdrenoABI.h" +#include +#include +#include + +/* Userspace side of the wire ABI. The transport is injected so the same client + * can drive an IOKit connection later and the kernel dispatch core during host + * qualification. Records are built and checked here at their exact fixed size; + * malformed counts are refused before any transport call. The client stores + * only the last reported parameters and never caches object identity. */ +struct NativeAdrenoTransport { + void *cookie; + IOReturn (*call)(void *cookie,uint32_t method,const void *input,size_t input_size, + void *output,size_t output_size); +}; +class NativeAdrenoClient final { + NativeAdrenoTransport transport_{}; + NativeAdrenoAbiParams params_{}; + bool open_=false; +public: + NativeAdrenoClient()=default; + ~NativeAdrenoClient()=default; + NativeAdrenoClient(const NativeAdrenoClient &)=delete; + NativeAdrenoClient &operator=(const NativeAdrenoClient &)=delete; + /* Queries the kernel and refuses a version or capability mismatch. */ + IOReturn open(const NativeAdrenoTransport *); + IOReturn close(); + IOReturn query(NativeAdrenoAbiParams *); + IOReturn allocBo(uint64_t bytes,uint32_t align_log2,bool contiguous,uint64_t *handle); + IOReturn mapBo(uint64_t handle,uint64_t iova,uint32_t permissions,uint32_t writeback, + uint32_t shareability,uint64_t *mapped); + IOReturn freeBo(uint64_t handle); + /* buffers are the caller's own array; the fixed record is assembled here. */ + IOReturn submit(const NativeAdrenoAbiSubmitBuffer *buffers,size_t count, + uint64_t wait_fence,uint64_t wait_timeout_ns,uint64_t *fence,uint64_t queue=0); + IOReturn waitFence(uint64_t fence,uint64_t timeout_ns); + IOReturn timestamp(uint64_t *ticks); + IOReturn status(uint64_t *faults,uint64_t *suspends); + IOReturn deviceInfo(NativeAdrenoAbiDeviceInfo *); + IOReturn createQueue(uint64_t *queue); + IOReturn cpuMap(uint64_t handle,uint32_t *token,uint64_t *bytes); + IOReturn cpuUnmap(uint64_t handle,uint32_t token); + IOReturn closeQueue(uint64_t queue); + IOReturn setMetadata(uint64_t handle,const void *,uint32_t bytes); + IOReturn getMetadata(uint64_t handle,void *out,uint32_t capacity,uint32_t *length); + bool isOpen() const {return open_;} + const NativeAdrenoAbiParams &info() const {return params_;} +}; +#endif diff --git a/drivers/NativeAdreno/NativeAdrenoCpInit.cpp b/drivers/NativeAdreno/NativeAdrenoCpInit.cpp new file mode 100644 index 0000000..23759d2 --- /dev/null +++ b/drivers/NativeAdreno/NativeAdrenoCpInit.cpp @@ -0,0 +1,62 @@ +/* SPDX-License-Identifier: MIT */ +#include "NativeAdrenoCpInit.h" +#include "AdrenoPM4.h" +#include + +/* Pinned definitions: CP_SQE_INSTR_BASE is the 64-bit register at dword offset + * 0x0830 for A6XX..A7XX (a6xx.xml). CP_THREAD_CONTROL is opcode 0x17 with + * CONCURRENT_BIN_DISABLE at bit 27, and CP_ME_INIT is opcode 0x48 with seven + * payload words (adreno_pm4.xml). The payload values are a7xx_cp_init in + * a6xx_gpu.c: mask BIT(0)|BIT(1)|BIT(3)|BIT(6)|BIT(8), three hardware + * contexts, 0x20000000 error detection, 0x00000002 operation mode, the + * power-up register list address and BIT(31) to read it. */ +enum : uint32_t { + CpSqeInstrBase = 0x0830, + CpThreadControlOpcode = 0x17, + CpMeInitOpcode = 0x48, + CpConcurrentBinDisable = 1u << 27, + CpMeInitWords = 8, + CpInitWords = 10 +}; +IOReturn NativeAdrenoCpInit::poison() +{ + poisoned_=true;return kIOReturnIOError; +} +uint32_t NativeAdrenoCpInit::meInitMask() +{ + return (1u<<0)|(1u<<1)|(1u<<3)|(1u<<6)|(1u<<8); +} +IOReturn NativeAdrenoCpInit::run(const NativeAdrenoRingIo *io,NativeAdrenoRing *ring, + const NativeAdrenoCpInitConfig *config,uint32_t *fence) +{ + if(!fence)return kIOReturnBadArgument;*fence=0; + if(done_ || poisoned_)return kIOReturnNotReady; + if(!io || !io->write32 || !ring || ring->poisoned())return kIOReturnBadArgument; + if(!config || !config->sqe_iova || (config->sqe_iova&4095) || + config->sqe_iova>=UINT64_C(1)<<48 || !config->pwrup_reglist_iova || + (config->pwrup_reglist_iova&3) || config->pwrup_reglist_iova>=UINT64_C(1)<<48) + return kIOReturnBadArgument; + if(config->sqe_base_programmed) { + uint32_t lo=0,hi=0; + if(!io->read32 || io->read32(io->cookie,CpSqeInstrBase,&lo) || io->read32(io->cookie,CpSqeInstrBase+1,&hi) || + (((uint64_t)hi<<32)|lo)!=config->sqe_iova)return poison(); + } else if(io->write32(io->cookie,CpSqeInstrBase,(uint32_t)config->sqe_iova) || + io->write32(io->cookie,CpSqeInstrBase+1,(uint32_t)(config->sqe_iova>>32))) + return poison(); + uint32_t words[CpInitWords]={0}; + if(AdrenoPM4Type7(CpThreadControlOpcode,1,&words[0])!=AdrenoPM4Success || + AdrenoPM4Type7(CpMeInitOpcode,7,&words[2])!=AdrenoPM4Success)return poison(); + words[1]=CpConcurrentBinDisable; + words[3]=meInitMask(); + words[4]=0x00000003; /* hardware contexts */ + words[5]=0x20000000; /* error detection */ + words[6]=0x00000002; /* operation mode */ + words[7]=(uint32_t)config->pwrup_reglist_iova; + words[8]=(uint32_t)(config->pwrup_reglist_iova>>32); + words[9]=1u<<31; /* read registers from the list */ + uint32_t value=0; + const IOReturn status=ring->submit(words,CpInitWords,&value); + if(status)return poison(); + done_=true;*fence=value; + return kIOReturnSuccess; +} diff --git a/drivers/NativeAdreno/NativeAdrenoCpInit.h b/drivers/NativeAdreno/NativeAdrenoCpInit.h new file mode 100644 index 0000000..7406ff9 --- /dev/null +++ b/drivers/NativeAdreno/NativeAdrenoCpInit.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef NATIVE_ADRENO_CPINIT_H +#define NATIVE_ADRENO_CPINIT_H +#include "NativeAdrenoRing.h" +struct NativeAdrenoCpInitConfig { + uint64_t sqe_iova; /* CP_SQE_INSTR_BASE, 4 KiB aligned, nonzero */ + uint64_t pwrup_reglist_iova; /* Power-up register list, 4-byte aligned */ + /* Nonzero when the caller already wrote CP_SQE_INSTR_BASE before CP_SQE_CNTL, as a6xx hw_init does: the base is + * then read back and compared instead of rewritten after the SQE started. */ + uint32_t sqe_base_programmed; +}; +/* One-shot A7XX command-processor start sequence, not a GPU power or reset + * driver. The caller must have completed GMU boot, powered GX/core clocks, + * released the CP reset, loaded and hash-verified the SQE microcode into the + * real IOVA given here, and hold exclusive MMIO. This programs + * CP_SQE_INSTR_BASE, then submits the pinned a7xx_cp_init sequence directly + * through the ring: CP_THREAD_CONTROL with concurrent binning disabled and + * CP_ME_INIT with the Linux mask, hardware-context count, operation mode and + * power-up register list. Completion is the ring timestamp fence, so a hung + * CP is reported as a timeout instead of a fabricated success. A second call + * cannot re-initialize the same object. */ +class NativeAdrenoCpInit final { + bool done_=false,poisoned_=false; + IOReturn poison(); +public: + NativeAdrenoCpInit()=default; + ~NativeAdrenoCpInit()=default; + NativeAdrenoCpInit(const NativeAdrenoCpInit &)=delete; + NativeAdrenoCpInit &operator=(const NativeAdrenoCpInit &)=delete; + IOReturn run(const NativeAdrenoRingIo *,NativeAdrenoRing *, + const NativeAdrenoCpInitConfig *,uint32_t *fence); + bool completed() const {return done_;} + bool poisoned() const {return poisoned_;} + static uint32_t meInitMask(); +}; +#endif diff --git a/drivers/NativeAdreno/NativeAdrenoCpu.cpp b/drivers/NativeAdreno/NativeAdrenoCpu.cpp new file mode 100644 index 0000000..66b6c22 --- /dev/null +++ b/drivers/NativeAdreno/NativeAdrenoCpu.cpp @@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: MIT */ +#include "NativeAdrenoMemory.h" +#include +#include +#include + +IOReturn NativeAdrenoQueryCpu4K(NativeAdrenoCpuState *out) +{ + if(!out)return kIOReturnBadArgument; + *out={}; + disable_preemption(); + __asm__ volatile("mrs %0, tcr_el1" : "=r"(out->tcr)); + __asm__ volatile("mrs %0, ttbr0_el1" : "=r"(out->ttbr0)); + __asm__ volatile("mrs %0, ttbr1_el1" : "=r"(out->ttbr1)); + __asm__ volatile("mrs %0, sctlr_el1" : "=r"(out->sctlr)); + out->kernel_page_bytes=page_size; + out->kernel_map_page_bytes=vm_map_page_size(kernel_map); + enable_preemption(); + bool native=PAGE_SIZE==4096 && out->kernel_page_bytes==4096 && out->kernel_map_page_bytes==4096 && + ((out->tcr>>14)&3)==0 && ((out->tcr>>30)&3)==2 && + (out->sctlr&1) && (out->ttbr1&UINT64_C(0x0000fffffffff000)); + return native?kIOReturnSuccess:kIOReturnUnsupported; +} +IOReturn NativeAdrenoMaintainOwnedPages(void *mapping,size_t bytes,NativeAdrenoCacheOperation op) +{ + /* Whole owned pages avoid IVAC corruption of a neighbour's dirty line. + * No Apple coherent-I/O shortcut is used for these noncoherent DMA paths. + * Pin this CPU while reading its CTR and issuing all cache operations. */ + if(!mapping || !bytes || (((uintptr_t)mapping|bytes)&4095) || + bytes>UINTPTR_MAX-(uintptr_t)mapping || op>NativeAdrenoInvalidate || op>16)&15); + if(line>4096) {enable_preemption();return kIOReturnUnsupported;} + uintptr_t address=(uintptr_t)mapping,end=address+bytes; + __asm__ volatile("dsb sy":::"memory"); + for(;address +IOReturn NativeAdrenoFirmware::validate(const uint8_t *image,uint64_t length, + const NativeAdrenoFirmwareRules *rules,NativeAdrenoFirmwareView *view) +{ + if(!view)return kIOReturnBadArgument; + *view={}; + if(!rules || !image || !length || length!=rules->bytes || + length>uint64_t(SIZE_MAX))return kIOReturnBadArgument; + uint8_t digest[32]; + NativeAdrenoSha256::hash(image,size_t(length),digest); + if(memcmp(digest,rules->sha256,32))return kIOReturnIOError; + view->bytes=image;view->length=size_t(length); + memcpy(view->sha256,digest,32); + return kIOReturnSuccess; +} diff --git a/drivers/NativeAdreno/NativeAdrenoFirmware.h b/drivers/NativeAdreno/NativeAdrenoFirmware.h new file mode 100644 index 0000000..6b989b6 --- /dev/null +++ b/drivers/NativeAdreno/NativeAdrenoFirmware.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef NATIVE_ADRENO_FIRMWARE_H +#define NATIVE_ADRENO_FIRMWARE_H +#include +#include +#include +struct NativeAdrenoFirmwareRules { + uint64_t bytes; + uint8_t sha256[32]; +}; +struct NativeAdrenoFirmwareView { + const uint8_t *bytes; + size_t length; + uint8_t sha256[32]; +}; +/* Validated view of an in-image firmware blob. The exact byte count is checked + * before any hashing, then SHA-256 must match. Refusal never returns a view and + * never copies the image; the caller keeps ownership and lifetime. This proves + * the bytes handed to the GMU loader are the pinned ones; it does not prove the + * payload is loadable or that a GMU exists. */ +class NativeAdrenoFirmware final { + NativeAdrenoFirmware()=delete; +public: + static IOReturn validate(const uint8_t *image,uint64_t length, + const NativeAdrenoFirmwareRules *rules,NativeAdrenoFirmwareView *view); +}; +#endif diff --git a/drivers/NativeAdreno/NativeAdrenoFirmwareDefaults.cpp b/drivers/NativeAdreno/NativeAdrenoFirmwareDefaults.cpp new file mode 100644 index 0000000..4a11d82 --- /dev/null +++ b/drivers/NativeAdreno/NativeAdrenoFirmwareDefaults.cpp @@ -0,0 +1,10 @@ +/* SPDX-License-Identifier: MIT */ +/* Empty weak defaults for the embedded firmware images. The kernel is linked statically, where an undefined + * weak reference is an error, so an image without the firmware archive still needs a definition. The + * archive's strong definitions replace these. A zero length is refused by NativeAdrenoFirmware::validate + * and gives no SQE upload, so nothing runs without real firmware. */ +#include +extern "C" __attribute__((weak)) const uint8_t native_adreno_firmware_image[1]={0}; +extern "C" __attribute__((weak)) const uint64_t native_adreno_firmware_image_size=0; +extern "C" __attribute__((weak)) const uint8_t native_adreno_sqe_firmware_image[1]={0}; +extern "C" __attribute__((weak)) const uint64_t native_adreno_sqe_firmware_image_size=0; diff --git a/drivers/NativeAdreno/NativeAdrenoGmuConfig.cpp b/drivers/NativeAdreno/NativeAdrenoGmuConfig.cpp new file mode 100644 index 0000000..ce6d5a0 --- /dev/null +++ b/drivers/NativeAdreno/NativeAdrenoGmuConfig.cpp @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: MIT */ +#include "NativeAdrenoGmuConfig.h" +#include +enum : uint64_t { + IcacheBytes=0xffc000, DummyBytes=0x4000, DebugBytes=0x7000, + LogBytes=0x4000, HfiBytes=0x4000 +}; +static bool matches(const NativeAdrenoMemory &memory,uint64_t bytes,uint64_t iova, + ShengGmuMemory *out) +{ + if(!memory.mapping() || memory.bytes()!=size_t(bytes) || memory.spanCount()!=1) + return false; + out->cpu=static_cast(memory.mapping()); + out->bytes=size_t(bytes); + out->iova=iova; + return true; +} +IOReturn NativeAdrenoAssembleGmuConfig(const NativeAdrenoFirmwareView *firmware, + const NativeAdrenoGmuAllocator *allocator,uint32_t chip_id,unsigned poll_budget, + ShengGmuConfig *config) +{ + if(!config)return kIOReturnBadArgument; + *config={}; + if(!firmware || !firmware->bytes || !firmware->length || !allocator || + !allocator->ready() || !chip_id || !poll_budget || poll_budget>1024) + return kIOReturnBadArgument; + const NativeAdrenoGmuMemoryLayout &layout=allocator->layout(); + ShengGmuConfig built{}; + if(!matches(allocator->icache(),IcacheBytes,layout.icache_iova,&built.icache) || + !matches(allocator->dummy(),DummyBytes,layout.dummy_iova,&built.dummy) || + !matches(allocator->debug(),DebugBytes,layout.debug_iova,&built.debug) || + !matches(allocator->log(),LogBytes,layout.log_iova,&built.log) || + !matches(allocator->hfi(),HfiBytes,layout.hfi_iova,&built.hfi)) + return kIOReturnBadArgument; + built.chip_id=chip_id; + built.firmware=firmware->bytes; + built.firmware_bytes=firmware->length; + built.poll_budget=poll_budget; + *config=built; + return kIOReturnSuccess; +} diff --git a/drivers/NativeAdreno/NativeAdrenoGmuConfig.h b/drivers/NativeAdreno/NativeAdrenoGmuConfig.h new file mode 100644 index 0000000..1e7a069 --- /dev/null +++ b/drivers/NativeAdreno/NativeAdrenoGmuConfig.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef NATIVE_ADRENO_GMU_CONFIG_H +#define NATIVE_ADRENO_GMU_CONFIG_H +#include "NativeAdrenoFirmware.h" +#include "NativeAdrenoGmuMemory.h" +#include "ShengGmu.h" + +/* Assemble the exact ChengGmuConfig the qualified GMU cold start consumes from + * a validated firmware view and the allocated payload layout. It enforces the + * qualified profile: firmware length must equal the payload's own image, the + * five payload sizes must be the startup sizes, and every payload must be + * allocated and IOVA-assigned. MMIO and time services stay caller-provided; + * this function proves nothing about the hardware and performs no access. */ +IOReturn NativeAdrenoAssembleGmuConfig(const NativeAdrenoFirmwareView *firmware, + const NativeAdrenoGmuAllocator *allocator,uint32_t chip_id,unsigned poll_budget, + ShengGmuConfig *config); +#endif diff --git a/drivers/NativeAdreno/NativeAdrenoGmuContext.cpp b/drivers/NativeAdreno/NativeAdrenoGmuContext.cpp new file mode 100644 index 0000000..eb41fd9 --- /dev/null +++ b/drivers/NativeAdreno/NativeAdrenoGmuContext.cpp @@ -0,0 +1,121 @@ +/* SPDX-License-Identifier: MIT */ +#include "NativeAdrenoGmuContext.h" +#include "ArmSmmu4KMmio.h" +#include + +static void readBack(const ArmSmmu4KIo *io,const ArmSmmu4KProgramResult *program,unsigned bank, + unsigned slot,NativeAdrenoGmuContextReport *r) +{ + /* Only registers the program step itself has already read; skipped if that step never ran. */ + if(!program->context_offset || !program->id1)return; + const size_t page=(program->id1&(1u<<31))?0x10000:0x1000; + io->read32(io->cookie,0x800+4*slot,&r->smr); + io->read32(io->cookie,0xc00+4*slot,&r->s2cr); + io->read32(io->cookie,page+4*bank,&r->cbar); + io->read32(io->cookie,size_t(program->context_offset),&r->sctlr); + io->read64(io->cookie,size_t(program->context_offset)+0x20,&r->ttbr0); +} +IOReturn NativeAdrenoGmuContext::bringUp(const ArmSmmu4KIo *io,unsigned bank,unsigned asid, + unsigned slot,uint16_t sid,void (*progress)(uint32_t),NativeAdrenoGmuContextReport *report) +{ + if(!report)return kIOReturnBadArgument; + *report={}; + if(used_)return kIOReturnBusy; + if(!io || bank>63 || !asid || asid>0xffff || slot>127)return kIOReturnBadArgument; + used_=true; + IOReturn status=kIOReturnSuccess; + auto step=[&](Step s) {report->step=s;if(progress)progress(s);}; + step(Allocate); + /* Stage-1 output size is ID2.OAS, as arm_smmu_init_domain_context uses smmu->ipa_size; tables and + * payloads must then lie below it. 40 bits is this driver's own allocation ceiling. */ + static const uint8_t oas_bits[16]={32,36,40,42,44,48}; + if(io->read32(io->cookie,0x28,&report->id2))status=kIOReturnIOError; + else { + const unsigned code=(report->id2>>4)&15; + report->output_bits=code<6?oas_bits[code]:0; + if(report->output_bits>40)report->output_bits=40; + if(!report->output_bits)status=kIOReturnUnsupported; + } + if(!status)status=gmu_.allocate(NativeAdrenoGmuDefaultSizes(TablePages*0x1000),report->output_bits); + if(!status) { + step(Prepare); + status=smmu_.prepare(&gmu_.tables(),0,0,GmuApertureBytes,NativeAdrenoSmmuContext::MaxLeaves, + bank,asid,1000,report->output_bits); + } + if(!status) { + step(Program); + status=smmu_.program(io); + const ArmSmmu4KProgramResult *p=smmu_.programResult(); + report->program_phase=p->phase;report->program_writes=p->writes; + report->program_readback_valid=p->readback_valid;report->program_enabled=p->enabled; + report->id1=p->id1;report->program_status=smmu_.programStatus(); + report->image=p->readback; + ArmSmmu4KContextConfig actual=*smmu_.configHandle(); + actual.id0=p->id0;actual.id2=p->id2; + ArmSmmu4KContextImage want; + if(p->phase>=3 && ArmSmmu4KEncodeContext(&actual,&want)==ArmSmmu4KSuccess) { + const ArmSmmu4KContextImage &got=p->readback; + const uint32_t sctlr_want=p->enabled?want.sctlr:0; + report->mismatch=(got.cbar!=want.cbar)|(got.cba2r!=want.cba2r)<<1|(got.tcr!=want.tcr)<<2| + (got.tcr2!=want.tcr2)<<3|(got.mair0!=want.mair0)<<4|(got.mair1!=want.mair1)<<5| + (got.ttbr0!=want.ttbr0)<<6|(got.ttbr1!=want.ttbr1)<<7|(got.sctlr!=sctlr_want)<<8; + } + } + if(!status) { + step(Attach); + const ArmSmmu4KStream route{slot,sid}; + status=smmu_.attach(&route,1); + const ArmSmmu4KStreamResult *s=smmu_.streamResult(); + report->route_status=smmu_.attachStatus(); + report->route_phase=s->phase;report->route_writes=s->writes;report->routes_verified=s->streams_verified; + } + if(!status) { + step(VmAttach); + NativeAdrenoVmRegion lower={&gmu_.tables(),smmu_.arenaHandle(),smmu_.stateHandle()}; + status=vm_.attach(io,smmu_.configHandle(),smmu_.streamsHandle(),&lower,nullptr,64,4000); + } + if(!status) { + step(Map); + status=gmu_.mapInto(&vm_); + report->spans=uint32_t(vm_.spanCount()); + } + if(!status) { + step(Ready); + ready_=true; + } + report->status=status; + if(report->step>=Program)readBack(io,smmu_.programResult(),bank,slot,report); + return status; +} +/* Lives for the rest of the boot: the context bank points at its tables. */ +static NativeAdrenoGmuContext *board_context; +extern "C" IOReturn NativeAdrenoGmuContextBringup(volatile uint8_t *smmu,size_t bytes, + unsigned bank,unsigned asid,unsigned slot,unsigned sid,void (*progress)(uint32_t), + NativeAdrenoGmuContextReport *report) +{ + static ArmSmmu4KMmio mmio; + static ArmSmmu4KIo io; + NativeAdrenoGmuContext *&context=board_context; + if(!report || context || !smmu || !bytes || sid>0xffff)return kIOReturnBadArgument; + mmio={reinterpret_cast(smmu),bytes}; + if(ArmSmmu4KBindMmio(&mmio,&io)!=ArmSmmu4KSuccess)return kIOReturnBadArgument; + context=new NativeAdrenoGmuContext; + if(!context)return kIOReturnNoMemory; + return context->bringUp(&io,bank,asid,slot,uint16_t(sid),progress,report); +} +extern "C" IOReturn NativeAdrenoGmuContextPayloads(NativeAdrenoGmuPayload out[5]) +{ + if(!out)return kIOReturnBadArgument; + for(unsigned i=0;i<5;++i)out[i]={}; + if(!board_context || !board_context->ready())return kIOReturnNotReady; + NativeAdrenoGmuAllocator &gmu=board_context->allocator(); + const NativeAdrenoGmuMemoryLayout &layout=gmu.layout(); + const NativeAdrenoMemory *objects[5]={&gmu.icache(),&gmu.dummy(),&gmu.debug(),&gmu.log(),&gmu.hfi()}; + const uint64_t iovas[5]={layout.icache_iova,layout.dummy_iova,layout.debug_iova,layout.log_iova,layout.hfi_iova}; + for(unsigned i=0;i<5;++i) { + const NativeAdrenoMemory *m=objects[i]; + if(m->spanCount()!=1 || m->spans()[0].bytes!=m->bytes())return kIOReturnUnsupported; + out[i]={m->spans()[0].physical,m->bytes(),iovas[i]}; + } + return kIOReturnSuccess; +} diff --git a/drivers/NativeAdreno/NativeAdrenoGmuContext.h b/drivers/NativeAdreno/NativeAdrenoGmuContext.h new file mode 100644 index 0000000..b6e4b12 --- /dev/null +++ b/drivers/NativeAdreno/NativeAdrenoGmuContext.h @@ -0,0 +1,61 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef NATIVE_ADRENO_GMU_CONTEXT_H +#define NATIVE_ADRENO_GMU_CONTEXT_H +#include "NativeAdrenoGmuMemory.h" +#include "NativeAdrenoSmmuContext.h" + +/* The GMU's own SMMU context, in the order a6xx_gmu_memory_probe and the arm-smmu + * attach build it: allocate the payloads at their fixed GMU addresses, prepare a + * native4K lower-region (TTBR0) table pool for the whole 0..2 GiB GMU space, + * program one context bank, route the GMU stream to it, attach the VM and map the + * payloads. Bank, ASID, SMR slot and stream ID are caller inputs. Every resource + * is retained on every outcome; there is no teardown. Caller keeps CX powered and + * the SMMU clocks running, and keeps the GMU halted (nothing is booted here). */ +struct NativeAdrenoGmuContextReport { + uint32_t step; /* last step started, NativeAdrenoGmuContext::Step */ + IOReturn status; /* of that step */ + uint32_t program_phase,program_writes,program_readback_valid,program_enabled; + uint32_t route_phase,route_writes,routes_verified; + uint32_t id1,id2; /* id2 is read before any allocation */ + uint32_t output_bits; /* from ID2.OAS, capped at 40 */ + uint32_t program_status,route_status; /* ArmSmmu4KStatus of program/attach */ + uint32_t sctlr,cbar,smr,s2cr; /* register readback after the last step reached */ + uint64_t ttbr0; + uint32_t spans; + /* The program step's last context readback and which fields differ from the encoded image: bit 0 CBAR, + * 1 CBA2R, 2 TCR, 3 TCR2, 4 MAIR0, 5 MAIR1, 6 TTBR0, 7 TTBR1, 8 SCTLR (SCTLR is 0 before enable). */ + ArmSmmu4KContextImage image; + uint32_t mismatch; +}; +class NativeAdrenoGmuContext final { + NativeAdrenoGmuAllocator gmu_; + NativeAdrenoSmmuContext smmu_; + NativeAdrenoVm vm_; + bool ready_=false,used_=false; +public: + /* Allocate also reads ID2 first and refuses an output size the kit cannot encode. */ + enum Step : uint32_t {Allocate=1,Prepare,Program,Attach,VmAttach,Map,Ready}; + /* Lower region: 1 L1 + 2 L2 + 1024 L3 pages for 2 GiB with a 32-bit input. */ + enum : size_t {TablePages=1027}; + NativeAdrenoGmuContext()=default; + NativeAdrenoGmuContext(const NativeAdrenoGmuContext &)=delete; + NativeAdrenoGmuContext &operator=(const NativeAdrenoGmuContext &)=delete; + /* One attempt per object. progress (may be null) is called before each step. */ + IOReturn bringUp(const ArmSmmu4KIo *io,unsigned bank,unsigned asid,unsigned slot, + uint16_t sid,void (*progress)(uint32_t step),NativeAdrenoGmuContextReport *report); + bool ready() const {return ready_;} + NativeAdrenoGmuAllocator &allocator() {return gmu_;} + NativeAdrenoVm &vm() {return vm_;} +}; +/* Board entry: binds the kit's direct MMIO adapter to an owned live mapping of the + * whole SMMU window and runs bringUp on a context that lives forever. */ +extern "C" IOReturn NativeAdrenoGmuContextBringup(volatile uint8_t *smmu,size_t bytes, + unsigned bank,unsigned asid,unsigned slot,unsigned sid,void (*progress)(uint32_t), + NativeAdrenoGmuContextReport *report); +/* The payloads of the context the board entry built, in ShengGmuConfig order (icache, dummy, debug, + * log, hfi): physical base, size and GMU address of each physically contiguous object. Only after + * that context reached Ready. The GMU maps them Normal non-cacheable (MAIR index 0), as Linux maps + * the GMU buffers without IOMMU_CACHE; the caller gives the CPU a matching non-cacheable view. */ +struct NativeAdrenoGmuPayload {uint64_t physical,bytes,iova;}; +extern "C" IOReturn NativeAdrenoGmuContextPayloads(NativeAdrenoGmuPayload out[5]); +#endif diff --git a/drivers/NativeAdreno/NativeAdrenoGmuMemory.cpp b/drivers/NativeAdreno/NativeAdrenoGmuMemory.cpp new file mode 100644 index 0000000..b7fc903 --- /dev/null +++ b/drivers/NativeAdreno/NativeAdrenoGmuMemory.cpp @@ -0,0 +1,92 @@ +/* SPDX-License-Identifier: MIT */ +#include "NativeAdrenoGmuMemory.h" +#include +enum : size_t { PageBytes=0x1000 }; +NativeAdrenoGmuSizes NativeAdrenoGmuDefaultSizes(size_t table_bytes) +{ + return {0xffc000,0x4000,0x7000,0x4000,0x4000,table_bytes}; +} +IOReturn NativeAdrenoGmuAllocator::allocate(const NativeAdrenoGmuSizes &sizes,unsigned address_bits) +{ + if(ready_ || icache_.bytes())return kIOReturnBusy; + const uint64_t requested[]={sizes.icache,sizes.dummy,sizes.debug,sizes.log, + sizes.hfi,sizes.tables}; + for(uint64_t value:requested) + if(!value || (value&(PageBytes-1)) || value>uint64_t(SIZE_MAX)) + return kIOReturnBadArgument; + /* The fixed addresses of a6xx_gmu.c at the pinned tree: icache at 0x4000 below the uncached range, dummy and + * debug at 0x60000000/0x60400000. Log and HFI have no fixed address there; drm_mm gives them the lowest free + * pages after the dummy, which is where they are put here. */ + if(sizes.icache>GmuUncachedBase-GmuIcacheIova || + sizes.dummy+sizes.log+sizes.hfi>GmuDebugIova-GmuUncachedBase || + sizes.debug>GmuApertureBytes-GmuDebugIova)return kIOReturnBadArgument; + struct Selection {NativeAdrenoMemory *memory;uint64_t bytes;uint64_t *iova;uint64_t at;}; + const Selection selections[]={ + {&icache_,sizes.icache,&layout_.icache_iova,GmuIcacheIova}, + {&dummy_,sizes.dummy,&layout_.dummy_iova,GmuUncachedBase}, + {&debug_,sizes.debug,&layout_.debug_iova,GmuDebugIova}, + {&log_,sizes.log,&layout_.log_iova,GmuUncachedBase+sizes.dummy}, + {&hfi_,sizes.hfi,&layout_.hfi_iova,GmuUncachedBase+sizes.dummy+sizes.log}, + {&tables_,sizes.tables,&layout_.tables_iova,0}}; + for(const Selection &selection:selections) { + const IOReturn status=selection.memory->allocate(static_cast(selection.bytes), + address_bits,true,static_cast(selection.bytes)); + if(status) { + (void)releaseAll(); + return status; + } + *selection.iova=selection.at; + } + layout_.tables_bytes=static_cast(sizes.tables); + layout_.end_iova=GmuDebugIova+sizes.debug; + ready_=true; + return kIOReturnSuccess; +} +IOReturn NativeAdrenoGmuAllocator::mapInto(NativeAdrenoVm *vm) +{ + if(!ready_ || !vm)return kIOReturnBadArgument; + for(uint64_t handle:handles_)if(handle)return kIOReturnBusy; + /* The table pool itself is never mapped: its physical pages are the + * translation storage and the VM refuses such an overlap. */ + NativeAdrenoMemory *objects[]={&icache_,&dummy_,&debug_,&log_,&hfi_}; + const uint64_t iovas[]={layout_.icache_iova,layout_.dummy_iova,layout_.debug_iova, + layout_.log_iova,layout_.hfi_iova}; + for(size_t index=0;index<5;++index) { + /* a6xx_gmu.c maps the icache MAP_PRIV and every buffer without NOEXEC; here only the icache, which + * the GMU executes from, is executable, and the data buffers stay execute-never. */ + const unsigned permissions=index?3u:15u; + const IOReturn status=vm->map(objects[index],0,iovas[index],permissions,0,2,&handles_[index]); + if(status) { + for(size_t previous=0;previousunmap(handles_[previous]);handles_[previous]=0;} + return status; + } + } + return kIOReturnSuccess; +} +IOReturn NativeAdrenoGmuAllocator::unmapFrom(NativeAdrenoVm *vm) +{ + if(!vm)return kIOReturnBadArgument; + IOReturn status=kIOReturnSuccess; + for(size_t index=5;index--;)if(handles_[index]) { + const IOReturn unmapped=vm->unmap(handles_[index]); + if(unmapped)status=unmapped;else handles_[index]=0; + } + return status; +} +IOReturn NativeAdrenoGmuAllocator::releaseAll() +{ + NativeAdrenoMemory *objects[]={&icache_,&dummy_,&debug_,&log_,&hfi_,&tables_}; + IOReturn status=kIOReturnSuccess; + for(NativeAdrenoMemory *object:objects) + if(object->bytes() && object->release()!=kIOReturnSuccess)status=kIOReturnIOError; + return status; +} +IOReturn NativeAdrenoGmuAllocator::release() +{ + if(!ready_)return kIOReturnNotReady; + const IOReturn status=releaseAll(); + if(status)return status; + ready_=false;layout_={}; + return kIOReturnSuccess; +} diff --git a/drivers/NativeAdreno/NativeAdrenoGmuMemory.h b/drivers/NativeAdreno/NativeAdrenoGmuMemory.h new file mode 100644 index 0000000..f7ffec2 --- /dev/null +++ b/drivers/NativeAdreno/NativeAdrenoGmuMemory.h @@ -0,0 +1,68 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef NATIVE_ADRENO_GMU_MEMORY_H +#define NATIVE_ADRENO_GMU_MEMORY_H +#include "NativeAdrenoMemory.h" +#include "NativeAdrenoVm.h" + +/* Backing objects for the GMU cold start, laid out at the fixed IOVAs of the + * GMU's own address space (0..2 GiB, the lower/TTBR0 region of its context + * bank, as a6xx_gmu_memory_probe creates it): instruction cache payload, dummy, + * debug and log buffers, HFI queues, plus the contiguous native4K table pool + * that will translate them (the pool itself has no IOVA). Sizes are the ones the qualified GMU startup uses. All + * objects are physically contiguous and page aligned; the allocator only + * assigns addresses and never maps them or touches the SMMU. Any failure + * releases what it allocated and leaves no layout. */ +enum : uint64_t { + GmuIcacheIova=0x4000, GmuUncachedBase=0x60000000, GmuDebugIova=0x60400000, + GmuApertureBytes=0x80000000 +}; +struct NativeAdrenoGmuSizes { + uint64_t icache,dummy,debug,log,hfi,tables; +}; +/* Exact sizes of the qualified startup; table pool is caller sized. */ +NativeAdrenoGmuSizes NativeAdrenoGmuDefaultSizes(size_t table_bytes); +struct NativeAdrenoGmuMemoryLayout { + uint64_t icache_iova,dummy_iova,debug_iova,log_iova,hfi_iova; + uint64_t tables_iova; + size_t tables_bytes; + uint64_t end_iova; /* first address after the whole reservation */ +}; +class NativeAdrenoGmuAllocator final { + NativeAdrenoMemory icache_,dummy_,debug_,log_,hfi_,tables_; + NativeAdrenoGmuMemoryLayout layout_{}; + uint64_t handles_[5]{}; + bool ready_=false; + IOReturn releaseAll(); +public: + NativeAdrenoGmuAllocator()=default; + ~NativeAdrenoGmuAllocator()=default; + NativeAdrenoGmuAllocator(const NativeAdrenoGmuAllocator &)=delete; + NativeAdrenoGmuAllocator &operator=(const NativeAdrenoGmuAllocator &)=delete; + /* Every size must be a nonzero 4KiB multiple and fit its fixed range. Every page lies below + * 2^address_bits: the SMMU output size (ID2.OAS) the context will use. */ + IOReturn allocate(const NativeAdrenoGmuSizes &sizes,unsigned address_bits=40); + /* Map every payload into the VM's lower region at its assigned IOVA with + * read/write access (icache also privileged and executable). The VM must already be attached with a lower arena + * covering the reservation. Handles are retained for unmapFrom. */ + IOReturn mapInto(NativeAdrenoVm *vm); + /* Reverse order unmap; refuses while a mapping has outstanding jobs. */ + IOReturn unmapFrom(NativeAdrenoVm *vm); + /* Only when every object has been released; on any refusal nothing is + * destroyed and the caller keeps the quarantined objects alive. */ + IOReturn release(); + bool ready() const {return ready_;} + const NativeAdrenoGmuMemoryLayout &layout() const {return layout_;} + NativeAdrenoMemory &icache() {return icache_;} + NativeAdrenoMemory &dummy() {return dummy_;} + NativeAdrenoMemory &debug() {return debug_;} + NativeAdrenoMemory &log() {return log_;} + NativeAdrenoMemory &hfi() {return hfi_;} + NativeAdrenoMemory &tables() {return tables_;} + const NativeAdrenoMemory &icache() const {return icache_;} + const NativeAdrenoMemory &dummy() const {return dummy_;} + const NativeAdrenoMemory &debug() const {return debug_;} + const NativeAdrenoMemory &log() const {return log_;} + const NativeAdrenoMemory &hfi() const {return hfi_;} + const NativeAdrenoMemory &tables() const {return tables_;} +}; +#endif diff --git a/drivers/NativeAdreno/NativeAdrenoGmuPlatform.cpp b/drivers/NativeAdreno/NativeAdrenoGmuPlatform.cpp new file mode 100644 index 0000000..7a26c85 --- /dev/null +++ b/drivers/NativeAdreno/NativeAdrenoGmuPlatform.cpp @@ -0,0 +1,63 @@ +/* SPDX-License-Identifier: MIT */ +#include "NativeAdrenoGmuPlatform.h" +#include + +int NativeAdrenoGmuPlatform::read32(void *cookie,unsigned window,size_t at,uint32_t *out) +{ + if(!cookie || !out || window>=ShengGmuWindows)return 1; + NativeAdrenoGmuPlatform *self=static_cast(cookie); + const NativeAdrenoGmuWindow &bounds=self->windows_[window]; + if(!bounds.base || (at&3) || at+4>bounds.bytes)return 1; + *out=*reinterpret_cast(bounds.base+at); + return 0; +} +int NativeAdrenoGmuPlatform::write32(void *cookie,unsigned window,size_t at,uint32_t value) +{ + if(!cookie || window>=ShengGmuWindows)return 1; + NativeAdrenoGmuPlatform *self=static_cast(cookie); + const NativeAdrenoGmuWindow &bounds=self->windows_[window]; + if(!bounds.base || (at&3) || at+4>bounds.bytes)return 1; + *reinterpret_cast(bounds.base+at)=value; + return 0; +} +int NativeAdrenoGmuPlatform::barrier(void *cookie) +{ + if(!cookie)return 1; + __atomic_thread_fence(__ATOMIC_SEQ_CST); + return 0; +} +IOReturn NativeAdrenoGmuPlatform::bind(const NativeAdrenoGmuWindow windows[ShengGmuWindows], + ShengGmuIo *io) +{ + if(!windows || !io)return kIOReturnBadArgument; + for(unsigned index=0;indexUINT32_MAX) + return kIOReturnBadArgument; + } + for(unsigned index=0;index(bytes),length,digest); + return 0; +} +IOReturn NativeAdrenoGmuPlatform::services(const NativeAdrenoGmuTime *time, + ShengGmuServices *services) +{ + if(!services)return kIOReturnBadArgument; + *services={}; + if(!time || !time->now_us || !time->wait_us)return kIOReturnBadArgument; + services->cookie=time->cookie; + services->sha256=sha256; + services->now_us=time->now_us; + services->wait_us=time->wait_us; + return kIOReturnSuccess; +} diff --git a/drivers/NativeAdreno/NativeAdrenoGmuPlatform.h b/drivers/NativeAdreno/NativeAdrenoGmuPlatform.h new file mode 100644 index 0000000..66cb13a --- /dev/null +++ b/drivers/NativeAdreno/NativeAdrenoGmuPlatform.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef NATIVE_ADRENO_GMU_PLATFORM_H +#define NATIVE_ADRENO_GMU_PLATFORM_H +#include +#include "NativeAdrenoSha256.h" +#include "ShengGmu.h" + +/* Bounds-checked MMIO adapter and time/hash services for the qualified GMU + * cold start. Windows are exactly the five the startup uses, already mapped + * and powered by the caller: GMU core, RSCC, PDC, the GPU register window and + * CX misc. Every access is 4-byte aligned and inside its window; a violation + * returns failure instead of touching another register. The adapter performs + * no mapping, clock, reset or power action, and the services object performs + * no I/O of its own beyond the caller-provided clock callbacks. */ +struct NativeAdrenoGmuWindow { + volatile uint8_t *base; + size_t bytes; +}; +struct NativeAdrenoGmuTime { + void *cookie; + int (*now_us)(void *,uint64_t *); + int (*wait_us)(void *,unsigned); +}; +class NativeAdrenoGmuPlatform final { + NativeAdrenoGmuWindow windows_[ShengGmuWindows]{}; + static int read32(void *,unsigned,size_t,uint32_t *); + static int write32(void *,unsigned,size_t,uint32_t); + static int barrier(void *); +public: + IOReturn bind(const NativeAdrenoGmuWindow windows[ShengGmuWindows],ShengGmuIo *io); + /* sha256 uses the kernel implementation; now/wait must be real. */ + static IOReturn services(const NativeAdrenoGmuTime *,ShengGmuServices *); +}; +#endif diff --git a/drivers/NativeAdreno/NativeAdrenoGmuService.cpp b/drivers/NativeAdreno/NativeAdrenoGmuService.cpp new file mode 100644 index 0000000..70d35ff --- /dev/null +++ b/drivers/NativeAdreno/NativeAdrenoGmuService.cpp @@ -0,0 +1,93 @@ +/* SPDX-License-Identifier: MIT */ +/* GMU platform-node driver: discovers the three GMU register windows, maps + * them read/write but performs no access, and publishes the bases so the GPU + * service can find them without resolving the DT phandle itself. It does not + * power CX/GX, start the CM3, load firmware or claim a working GMU; those steps + * belong to the GPU bring-up once board power exists. Publishing this node is + * safe because nothing is written and no interrupt is claimed. */ +#include +#include +#include +#include +#include "NativeAdrenoGmuService.h" + +OSDefineMetaClassAndStructors(NativeAdrenoGmuService,IOService) + + + +static bool advertisesGmu(IOService *provider) +{ + OSObject *property=provider?provider->getProperty("compatible"):nullptr; + OSData *data=OSDynamicCast(OSData,property); + if(!data || !data->getBytesNoCopy() || !data->getLength())return false; + const char *text=static_cast(data->getBytesNoCopy()); + const size_t length=data->getLength(); + static const char needle[]="qcom,adreno-gmu"; + for(size_t at=0;at+sizeof(needle)-1<=length;++at) { + size_t index=0; + while(indexgetProperty("reg")); + OSData *address=OSDynamicCast(OSData,provider->getProperty("#address-cells")); + OSData *size=OSDynamicCast(OSData,provider->getProperty("#size-cells")); + uint32_t address_cells=2,size_cells=2; + if(address && address->getLength()>=4) + address_cells=*static_cast(address->getBytesNoCopy()); + if(size && size->getLength()>=4) + size_cells=*static_cast(size->getBytesNoCopy()); + if(!reg || NativeAdrenoBoard::parseReg( + static_cast(reg->getBytesNoCopy()),reg->getLength(), + address_cells,size_cells,windows_,NativeAdrenoMaxWindows,&count_)!=kIOReturnSuccess) { + IOLog("NativeAdrenoGmu: reg windows are not usable\n"); + count_=0;IOService::stop(provider); + return false; + } + for(size_t index=0;index(index)); + if(!memory) { + IOLog("NativeAdrenoGmu: window %lu has no device memory\n", + static_cast(index)); + stop(provider); + return false; + } + maps_[index]=OSSharedPtr(memory->map(kIOMapAnywhere), + OSNoRetain); + if(!maps_[index]) { + IOLog("NativeAdrenoGmu: window %lu did not map\n", + static_cast(index)); + stop(provider); + return false; + } + } + /* Bases are properties for the GPU service; no phandle resolution needed. */ + OSData *bases=OSData::withCapacity(static_cast(count_*sizeof(uint64_t))); + if(!bases) {stop(provider);return false;} + for(size_t index=0;indexappendBytes(&base,sizeof(base)); + } + setProperty("gmu-window-bases",bases); + bases->release(); + IOLog("NativeAdrenoGmu: %lu windows mapped, no register access\n", + static_cast(count_)); + registerService(); + return true; +} +void NativeAdrenoGmuService::stop(IOService *provider) +{ + for(auto &map:maps_)map.reset(); + count_=0; + IOService::stop(provider); +} diff --git a/drivers/NativeAdreno/NativeAdrenoGmuService.h b/drivers/NativeAdreno/NativeAdrenoGmuService.h new file mode 100644 index 0000000..8dfa00e --- /dev/null +++ b/drivers/NativeAdreno/NativeAdrenoGmuService.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef NATIVE_ADRENO_GMU_SERVICE_H +#define NATIVE_ADRENO_GMU_SERVICE_H +#include +#include +#include "NativeAdrenoBoard.h" + +class NativeAdrenoGmuService : public IOService { + OSDeclareDefaultStructors(NativeAdrenoGmuService) + OSSharedPtr maps_[NativeAdrenoMaxWindows]; + NativeAdrenoWindow windows_[NativeAdrenoMaxWindows]{}; + size_t count_=0; +public: + IOService *probe(IOService *provider,SInt32 *score) APPLE_KEXT_OVERRIDE; + bool start(IOService *provider) APPLE_KEXT_OVERRIDE; + void stop(IOService *provider) APPLE_KEXT_OVERRIDE; + uint64_t windowBase(size_t index) const + {return index(maps_[index]->getVirtualAddress()):nullptr;} +}; + +#endif diff --git a/drivers/NativeAdreno/NativeAdrenoGpu.cpp b/drivers/NativeAdreno/NativeAdrenoGpu.cpp new file mode 100644 index 0000000..ba225cf --- /dev/null +++ b/drivers/NativeAdreno/NativeAdrenoGpu.cpp @@ -0,0 +1,66 @@ +/* SPDX-License-Identifier: MIT */ +#include "NativeAdrenoGpu.h" +#include +IOReturn NativeAdrenoGpu::fail(IOReturn status) +{ + failed_=true;ready_=false; + if(reserved_ && !quiesced_) { + quiesced_=board_.quiesce(board_.cookie)==kIOReturnSuccess; + } + return status; +} +IOReturn NativeAdrenoGpu::start(const NativeAdrenoGpuBoard *board, + const NativeAdrenoGpuConfig *config) +{ + if(started_)return kIOReturnBusy; + if(!board || !config || !board->reserve || !board->bootGmu || + !board->programSmmu || !board->quiesce)return kIOReturnBadArgument; + if(!config->mmio.write32 || !config->mmio.read32 || !config->mmio.barrier || + !config->cache.publish || !config->cache.acquire || !config->ring || + !config->fence || !config->sqe_iova || !config->pwrup_reglist_iova || + !config->clock.now_us || !config->start_timeout_us) + return kIOReturnBadArgument; + started_=true;board_=*board; + IOReturn status=board_.reserve(board_.cookie); + if(status) {failed_=true;return status;} + reserved_=true;steps_=1; + if((status=board_.bootGmu(board_.cookie)))return fail(status); + steps_=2; + if((status=board_.programSmmu(board_.cookie)))return fail(status); + steps_=3; + if((status=ring_.attach(&config->mmio,&config->cache,config->ring,config->ring_bytes, + config->ring_iova,config->fence,config->fence_bytes,config->fence_iova)))return fail(status); + steps_=4; + NativeAdrenoCpInitConfig cp{config->sqe_iova,config->pwrup_reglist_iova,0}; + if((status=cp_.run(&config->mmio,&ring_,&cp,&fence_)))return fail(status); + /* A started CP is not ready until its init fence is actually observed. */ + const NativeAdrenoRingFence started=ring_.wait(fence_,&config->clock, + config->start_timeout_us,nullptr); + if(started!=NativeAdrenoRingFenceComplete) + return fail(started==NativeAdrenoRingFenceTimeout?kIOReturnTimeout:kIOReturnIOError); + steps_=5; + if((status=irq_.attach(&config->mmio)))return fail(status); + if((status=irq_.programMask(nullptr)))return fail(status); + steps_=6; + if(config->time.read32) { + if((status=timer_.attach(&config->time,NativeAdrenoTimeGmu)))return fail(status); + timerTimeOk_=true; + } + ready_=true; + return kIOReturnSuccess; +} +IOReturn NativeAdrenoGpu::stop() +{ + if(!ready_)return kIOReturnNotReady; + if(ring_.poisoned() || irq_.poisoned() || cp_.poisoned())return kIOReturnBusy; + if(ring_.outstanding())return kIOReturnBusy; + IOReturn status=board_.quiesce(board_.cookie); + quiesced_=status==kIOReturnSuccess; + if(status) {failed_=true;ready_=false;return status;} + if(irq_.detach())return fail(kIOReturnIOError); + if(timerTimeOk_ && timer_.detach())return fail(kIOReturnIOError); + timerTimeOk_=false; + if(ring_.detach())return fail(kIOReturnIOError); + ready_=false; + return kIOReturnSuccess; +} diff --git a/drivers/NativeAdreno/NativeAdrenoGpu.h b/drivers/NativeAdreno/NativeAdrenoGpu.h new file mode 100644 index 0000000..62abf77 --- /dev/null +++ b/drivers/NativeAdreno/NativeAdrenoGpu.h @@ -0,0 +1,71 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef NATIVE_ADRENO_GPU_H +#define NATIVE_ADRENO_GPU_H +#include "NativeAdrenoCpInit.h" +#include "NativeAdrenoIrq.h" +#include "NativeAdrenoTimer.h" + +/* One GPU lifecycle attempt: board power/GMU/SMMU actions are injected, while + * ring programming, CP start and interrupt masking are the real components in + * this repository. The board operations own every resource that must exist + * before MMIO is touched; this object never powers, clocks or maps anything. + * Order is fixed: reserve, boot GMU, program the GPU SMMU context, attach the + * ring, start the CP and program the interrupt mask. A failure after a + * successful reserve calls quiesce exactly once, never retries and never + * reports readiness; components keep their own quarantine state. On success + * the ring, timer and IRQ objects are attached and owned here and may be handed + * to user clients. This is not a Metal device and does not prove hardware. */ +struct NativeAdrenoGpuBoard { + void *cookie; + IOReturn (*reserve)(void *); /* exclusive device, clocks, mappings */ + IOReturn (*bootGmu)(void *); /* GMU firmware/HFI startup */ + IOReturn (*programSmmu)(void *); /* streams + native4K context enabled */ + IOReturn (*quiesce)(void *); /* stop/fault masters after a failure */ +}; +struct NativeAdrenoGpuConfig { + NativeAdrenoRingIo mmio; /* powered GPU register window */ + NativeAdrenoRingCache cache; /* PoC publication for ring/fence */ + NativeAdrenoTimeIo time; /* optional GPU counter window */ + NativeAdrenoRingClock clock; /* required: bounds the CP start wait */ + uint32_t start_timeout_us; /* required: CP start fence timeout */ + void *ring; + size_t ring_bytes; + uint64_t ring_iova; + void *fence; + size_t fence_bytes; + uint64_t fence_iova; + uint64_t sqe_iova; /* verified SQE microcode IOVA */ + uint64_t pwrup_reglist_iova; +}; +enum { NativeAdrenoGpuSteps = 6 }; +class NativeAdrenoGpu final { + NativeAdrenoGpuBoard board_{}; + NativeAdrenoRing ring_; + NativeAdrenoCpInit cp_; + NativeAdrenoIrq irq_; + NativeAdrenoTimer timer_; + uint32_t steps_=0,fence_=0; + bool reserved_=false,ready_=false,failed_=false,quiesced_=false,started_=false; + IOReturn fail(IOReturn status); +public: + NativeAdrenoGpu()=default; + ~NativeAdrenoGpu()=default; + NativeAdrenoGpu(const NativeAdrenoGpu &)=delete; + NativeAdrenoGpu &operator=(const NativeAdrenoGpu &)=delete; + /* One attempt. Returns the failing step's error; readiness is only true + * when every step succeeded. Ring ownership transfers to this object. */ + IOReturn start(const NativeAdrenoGpuBoard *,const NativeAdrenoGpuConfig *); + /* Only a ready GPU may be stopped; requires every submission observed. */ + IOReturn stop(); + bool ready() const {return ready_;} + bool failed() const {return failed_;} + bool quiesced() const {return quiesced_;} + uint32_t stepsCompleted() const {return steps_;} + uint32_t startFence() const {return fence_;} + NativeAdrenoRing *ring() {return ready_?&ring_:nullptr;} + NativeAdrenoTimer *timer() {return ready_&&timerTimeOk_?&timer_:nullptr;} + NativeAdrenoIrq *interrupts() {return ready_?&irq_:nullptr;} +private: + bool timerTimeOk_=false; +}; +#endif diff --git a/drivers/NativeAdreno/NativeAdrenoGpuContext.cpp b/drivers/NativeAdreno/NativeAdrenoGpuContext.cpp new file mode 100644 index 0000000..75870b5 --- /dev/null +++ b/drivers/NativeAdreno/NativeAdrenoGpuContext.cpp @@ -0,0 +1,103 @@ +/* SPDX-License-Identifier: MIT */ +#include "NativeAdrenoGpuContext.h" +#include "ArmSmmu4KMmio.h" +#include + +static const uint64_t kIovas[GpuBufferCount]={GpuSqeIova,GpuRingIova,GpuMemptrsIova,GpuPwrupIova,GpuScratchIova}; +static const size_t kBytes[GpuBufferCount]={GpuSqeBytes,GpuRingBytes,GpuPageBytes,GpuPageBytes,GpuPageBytes}; +IOReturn NativeAdrenoGpuContext::bringUp(const ArmSmmu4KIo *io,unsigned bank,unsigned asid,const unsigned slots[2], + const uint16_t sids[2],void (*progress)(uint32_t),NativeAdrenoGmuContextReport *report) +{ + if(!report)return kIOReturnBadArgument; + *report={}; + if(used_)return kIOReturnBusy; + if(!io || !slots || !sids || bank>63 || !asid || asid>0xffff || slots[0]>127 || slots[1]>127 || + slots[0]==slots[1])return kIOReturnBadArgument; + used_=true; + IOReturn status=kIOReturnSuccess; + auto step=[&](Step s) {report->step=s;if(progress)progress(s);}; + step(Allocate); + static const uint8_t oas_bits[16]={32,36,40,42,44,48}; + if(io->read32(io->cookie,0x28,&report->id2))status=kIOReturnIOError; + else { + const unsigned code=(report->id2>>4)&15; + report->output_bits=code<6?oas_bits[code]:0; + if(report->output_bits>40)report->output_bits=40; + if(!report->output_bits)status=kIOReturnUnsupported; + } + if(!status)status=tables_.allocate(TablePages*0x1000,report->output_bits,true,TablePages*0x1000); + for(unsigned i=0;!status && ioutput_bits,true,kBytes[i]); + if(!status) { + step(Prepare); + status=smmu_.prepare(&tables_,0,0,GpuApertureBytes,NativeAdrenoSmmuContext::MaxLeaves, + bank,asid,1000,report->output_bits); + } + if(!status) { + step(Program); + status=smmu_.program(io); + const ArmSmmu4KProgramResult *p=smmu_.programResult(); + report->program_phase=p->phase;report->program_writes=p->writes; + report->program_readback_valid=p->readback_valid;report->program_enabled=p->enabled; + report->id1=p->id1;report->program_status=smmu_.programStatus(); + report->image=p->readback; + } + if(!status) { + step(Attach); + const ArmSmmu4KStream routes[2]={{slots[0],sids[0]},{slots[1],sids[1]}}; + status=smmu_.attach(routes,2); + const ArmSmmu4KStreamResult *s=smmu_.streamResult(); + report->route_status=smmu_.attachStatus(); + report->route_phase=s->phase;report->route_writes=s->writes;report->routes_verified=s->streams_verified; + } + if(!status) { + step(VmAttach); + NativeAdrenoVmRegion lower={&tables_,smmu_.arenaHandle(),smmu_.stateHandle()}; + status=vm_.attach(io,smmu_.configHandle(),smmu_.streamsHandle(),&lower,nullptr,16384,262144); + } + if(!status) { + step(Map); + /* adreno_fw_create_bo maps the SQE MSM_BO_GPU_READONLY without NOEXEC; every other kernel BO read/write. */ + for(unsigned i=0;!status && ispans=uint32_t(vm_.spanCount()); + } + if(!status) { + step(Ready); + ready_=true; + } + report->status=status; + return status; +} +static NativeAdrenoGpuContext *board_gpu_context; +extern "C" IOReturn NativeAdrenoGpuContextBringup(volatile uint8_t *smmu,size_t bytes,unsigned bank,unsigned asid, + unsigned slot0,unsigned sid0,unsigned slot1,unsigned sid1,void (*progress)(uint32_t), + NativeAdrenoGmuContextReport *report) +{ + static ArmSmmu4KMmio mmio; + static ArmSmmu4KIo io; + if(!report || board_gpu_context || !smmu || !bytes || sid0>0xffff || sid1>0xffff)return kIOReturnBadArgument; + mmio={reinterpret_cast(smmu),bytes}; + if(ArmSmmu4KBindMmio(&mmio,&io)!=ArmSmmu4KSuccess)return kIOReturnBadArgument; + board_gpu_context=new NativeAdrenoGpuContext; + if(!board_gpu_context)return kIOReturnNoMemory; + const unsigned slots[2]={slot0,slot1}; + const uint16_t sids[2]={uint16_t(sid0),uint16_t(sid1)}; + return board_gpu_context->bringUp(&io,bank,asid,slots,sids,progress,report); +} +extern "C" IOReturn NativeAdrenoGpuContextBuffers(NativeAdrenoGpuBuffer out[5]) +{ + if(!out)return kIOReturnBadArgument; + for(unsigned i=0;iready())return kIOReturnNotReady; + for(unsigned i=0;ibuffer(i); + if(m.spanCount()!=1 || m.spans()[0].bytes!=m.bytes())return kIOReturnUnsupported; + out[i]={m.spans()[0].physical,m.bytes(),kIovas[i]}; + } + return kIOReturnSuccess; +} +extern "C" NativeAdrenoVm *NativeAdrenoGpuContextVm() +{ + return board_gpu_context?board_gpu_context->vm():nullptr; +} diff --git a/drivers/NativeAdreno/NativeAdrenoGpuContext.h b/drivers/NativeAdreno/NativeAdrenoGpuContext.h new file mode 100644 index 0000000..2a0d5f3 --- /dev/null +++ b/drivers/NativeAdreno/NativeAdrenoGpuContext.h @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef NATIVE_ADRENO_GPU_CONTEXT_H +#define NATIVE_ADRENO_GPU_CONTEXT_H +#include "NativeAdrenoGmuContext.h" + +/* The GPU's own SMMU context, built like the GMU's (NativeAdrenoGmuContext): a native4K lower-region (TTBR0) table + * pool for a 0..2 GiB GPU space, one context bank, the GPU's two streams routed to it, then the kernel buffers the + * command processor needs at fixed GPU addresses below the A740 GMEM window (UCHE_GMEM_RANGE_MIN 16 MiB): + * sqe the SQE microcode (a740_sqe.fw without its first dword, as adreno_fw_create_bo copies it), R+X + * ring ring buffer 0, 32 KiB, aligned to its size, R+W + * memptrs msm_rbmemptrs layout: +0 rptr, +4 fence, +8 bv_rptr, +12 bv_fence, R+W + * pwrup the a7xx power-up register list (cpu_gpu_lock header + pairs), R+W + * scratch one page the first submissions may write, R+W + * All mappings are Normal non-cacheable (Linux maps kernel BOs write-combined) and outer shareable; the CPU must use + * a matching non-cacheable view. Bank, ASID, SMR slots and stream IDs are caller inputs. Nothing is torn down. */ +enum : uint64_t { + GpuSqeIova=0x100000, GpuRingIova=0x200000, GpuMemptrsIova=0x300000, GpuPwrupIova=0x301000, + GpuScratchIova=0x302000, GpuApertureBytes=0x80000000 +}; +enum : size_t {GpuSqeBytes=0x13000, GpuRingBytes=0x8000, GpuPageBytes=0x1000, GpuBufferCount=5}; +struct NativeAdrenoGpuBuffer {uint64_t physical,bytes,iova;}; +class NativeAdrenoGpuContext final { + NativeAdrenoMemory tables_,buffers_[GpuBufferCount]; + uint64_t handles_[GpuBufferCount]{}; + NativeAdrenoSmmuContext smmu_; + NativeAdrenoVm vm_; + bool ready_=false,used_=false; +public: + enum Step : uint32_t {Allocate=1,Prepare,Program,Attach,VmAttach,Map,Ready}; + enum : size_t {TablePages=1027}; + NativeAdrenoGpuContext()=default; + NativeAdrenoGpuContext(const NativeAdrenoGpuContext &)=delete; + NativeAdrenoGpuContext &operator=(const NativeAdrenoGpuContext &)=delete; + /* The report is the GMU context's: route counts cover both streams. */ + IOReturn bringUp(const ArmSmmu4KIo *io,unsigned bank,unsigned asid,const unsigned slots[2], + const uint16_t sids[2],void (*progress)(uint32_t step),NativeAdrenoGmuContextReport *report); + bool ready() const {return ready_;} + const NativeAdrenoMemory &buffer(unsigned index) const {return buffers_[index];} + NativeAdrenoVm *vm() {return ready_?&vm_:nullptr;} +}; +extern "C" IOReturn NativeAdrenoGpuContextBringup(volatile uint8_t *smmu,size_t bytes,unsigned bank,unsigned asid, + unsigned slot0,unsigned sid0,unsigned slot1,unsigned sid1,void (*progress)(uint32_t), + NativeAdrenoGmuContextReport *report); +/* sqe, ring, memptrs, pwrup, scratch: physical base, size and GPU address, only after Ready. */ +extern "C" IOReturn NativeAdrenoGpuContextBuffers(NativeAdrenoGpuBuffer out[5]); +/* The live VM of the ready GPU context (null before Ready), for the user-client core. */ +extern "C" NativeAdrenoVm *NativeAdrenoGpuContextVm(); +#endif diff --git a/drivers/NativeAdreno/NativeAdrenoIrq.cpp b/drivers/NativeAdreno/NativeAdrenoIrq.cpp new file mode 100644 index 0000000..96af912 --- /dev/null +++ b/drivers/NativeAdreno/NativeAdrenoIrq.cpp @@ -0,0 +1,124 @@ +/* SPDX-License-Identifier: MIT */ +#include "NativeAdrenoIrq.h" +#include + +/* Register offsets and bit positions are the pinned Linux definitions in + * registers/adreno/a6xx.xml: RBBM_INT_0_STATUS 0x0201, RBBM_INT_CLEAR_CMD + * 0x0037, RBBM_INT_0_MASK 0x0038, CP_INTERRUPT_STATUS 0x0823 and + * CP_PROTECT_STATUS 0x0824, plus the A7XX_INT_MASK enable set from + * a6xx_gpu.c. No bit is inferred beyond those sources. */ +enum : uint32_t { + IrqRbbmStatus = 0x0201, + IrqRbbmClear = 0x0037, + IrqRbbmMask = 0x0038, + IrqCpStatus = 0x0823, + IrqCpProtect = 0x0824, + IrqBitIdle = 1u << 0, + IrqBitAhb = 1u << 1, + IrqBitAtbAsync = 1u << 6, + IrqBitGpc = 1u << 7, + IrqBitCpSw = 1u << 8, + IrqBitCpHw = 1u << 9, + IrqBitCpCcuDepth = 1u << 10, + IrqBitCpCcuColor = 1u << 11, + IrqBitCpCcuResolve = 1u << 12, + IrqBitCpIb2 = 1u << 13, + IrqBitCpIb1 = 1u << 14, + IrqBitCpPm4 = 1u << 15, + IrqBitCpPm4Lpac = 1u << 16, + IrqBitCpRbDone = 1u << 17, + IrqBitCpWtDone = 1u << 18, + IrqBitCpFence = 1u << 20, + IrqBitCpFenceLpac = 1u << 21, + IrqBitAtbBus = 1u << 22, + IrqBitHang = 1u << 23, + IrqBitUcheOob = 1u << 24, + IrqBitUcheTrap = 1u << 25, + IrqBitTsbWrite = 1u << 28, + IrqBitFuse = 1u << 29, + IrqBitIsdbCpu = 1u << 30, + IrqBitIsdbDebug = 1u << 31 +}; +/* Every A7XX status bit that this decoder knows and may clear. */ +static constexpr uint32_t kKnown = IrqBitIdle | IrqBitAhb | IrqBitAtbAsync | IrqBitGpc | + IrqBitCpSw | IrqBitCpHw | IrqBitCpCcuDepth | IrqBitCpCcuColor | IrqBitCpCcuResolve | + IrqBitCpIb1 | IrqBitCpIb2 | IrqBitCpPm4 | IrqBitCpPm4Lpac | IrqBitCpRbDone | + IrqBitCpWtDone | IrqBitCpFence | IrqBitCpFenceLpac | IrqBitAtbBus | IrqBitHang | + IrqBitUcheOob | IrqBitUcheTrap | IrqBitTsbWrite | IrqBitFuse | IrqBitIsdbCpu | + IrqBitIsdbDebug; +/* A7XX_INT_MASK from the pinned a6xx_gpu.c. */ +static constexpr uint32_t kEnable = IrqBitAhb | IrqBitAtbAsync | IrqBitGpc | IrqBitCpSw | + IrqBitCpHw | IrqBitCpPm4 | IrqBitCpRbDone | IrqBitCpFence | IrqBitAtbBus | + IrqBitHang | IrqBitUcheOob | IrqBitUcheTrap | IrqBitTsbWrite | IrqBitFuse; +/* CP_INTERRUPT_STATUS error bits, including the A7XX LPAC/BV replicas. */ +static constexpr uint32_t kCpErrors = 0x3fff7u; +IOReturn NativeAdrenoIrq::poison() +{ + poisoned_=true;return kIOReturnIOError; +} +uint32_t NativeAdrenoIrq::mask() +{ + return kEnable; +} +uint32_t NativeAdrenoIrq::cpErrorMask() +{ + return kCpErrors; +} +IOReturn NativeAdrenoIrq::attach(const NativeAdrenoRingIo *io) +{ + if(attached_ || poisoned_)return kIOReturnBusy; + if(!io || !io->read32 || !io->write32 || !io->barrier)return kIOReturnBadArgument; + io_=*io;attached_=true;enabled_=0; + return kIOReturnSuccess; +} +IOReturn NativeAdrenoIrq::detach() +{ + if(poisoned_ || !attached_)return kIOReturnNotReady; + attached_=false;io_={};enabled_=0;return kIOReturnSuccess; +} +IOReturn NativeAdrenoIrq::programMask(uint32_t *programmed) +{ + if(!attached_ || poisoned_)return kIOReturnNotReady; + if(programmed)*programmed=0; + uint32_t readback=0; + if(io_.write32(io_.cookie,IrqRbbmMask,kEnable))return poison(); + if(io_.read32(io_.cookie,IrqRbbmMask,&readback))return poison(); + if(readback!=kEnable)return poison(); + enabled_=readback; + if(programmed)*programmed=readback; + return kIOReturnSuccess; +} +IOReturn NativeAdrenoIrq::handle(NativeAdrenoIrqResult *result) +{ + if(!result)return kIOReturnBadArgument; + *result={}; + if(!attached_ || poisoned_)return kIOReturnNotReady; + uint32_t status=0; + if(io_.read32(io_.cookie,IrqRbbmStatus,&status))return poison(); + /* Linux acknowledges the whole read status before decoding it. */ + if(io_.write32(io_.cookie,IrqRbbmClear,status))return poison(); + if(io_.barrier(io_.cookie))return poison(); + uint32_t flags=0,cp_status=0,protect=0; + if(status&IrqBitCpFence)flags|=NativeAdrenoIrqFence; + if(status&IrqBitCpSw)flags|=NativeAdrenoIrqPreempt; + if(status&(IrqBitIdle))flags|=NativeAdrenoIrqIdle; + if(status&(IrqBitHang))flags|=NativeAdrenoIrqHang; + if(status&(IrqBitUcheOob|IrqBitUcheTrap))flags|=NativeAdrenoIrqUche; + if(status&(IrqBitAhb|IrqBitGpc|IrqBitAtbAsync|IrqBitAtbBus))flags|=NativeAdrenoIrqBusError; + if(status&(IrqBitTsbWrite|IrqBitFuse))flags|=NativeAdrenoIrqSecurity; + if(status&IrqBitCpHw) { + if(io_.read32(io_.cookie,IrqCpStatus,&cp_status))return poison(); + if(cp_status&kCpErrors) { + if(io_.read32(io_.cookie,IrqCpProtect,&protect))return poison(); + flags|=NativeAdrenoIrqCpError; + } + } + const uint32_t unknown=status&~kKnown; + if(unknown)flags|=NativeAdrenoIrqUnknown; + if(flags&(NativeAdrenoIrqHang|NativeAdrenoIrqCpError|NativeAdrenoIrqUche| + NativeAdrenoIrqBusError|NativeAdrenoIrqSecurity|NativeAdrenoIrqUnknown)) + flags|=NativeAdrenoIrqFatal; + result->flags=flags;result->status=status;result->unknown=unknown; + result->cp_status=cp_status;result->protect_status=protect; + return kIOReturnSuccess; +} diff --git a/drivers/NativeAdreno/NativeAdrenoIrq.h b/drivers/NativeAdreno/NativeAdrenoIrq.h new file mode 100644 index 0000000..3640610 --- /dev/null +++ b/drivers/NativeAdreno/NativeAdrenoIrq.h @@ -0,0 +1,52 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef NATIVE_ADRENO_IRQ_H +#define NATIVE_ADRENO_IRQ_H +#include "NativeAdrenoRing.h" +#include + +enum NativeAdrenoIrqFlag { + NativeAdrenoIrqFence = 1u << 0, /* CP_CACHE_FLUSH_TS: retire candidates */ + NativeAdrenoIrqHang = 1u << 1, /* RBBM_HANG_DETECT */ + NativeAdrenoIrqCpError = 1u << 2, /* CP_HW_ERROR with a nonzero CP status */ + NativeAdrenoIrqUche = 1u << 3, /* UCHE out-of-bounds or trap */ + NativeAdrenoIrqBusError = 1u << 4, /* AHB/GPC/ATB overflow or async FIFO */ + NativeAdrenoIrqPreempt = 1u << 5, /* CP_SW */ + NativeAdrenoIrqIdle = 1u << 6, /* RBBM_GPU_IDLE */ + NativeAdrenoIrqSecurity = 1u << 7, /* SW fuse violation / TSB write error */ + NativeAdrenoIrqUnknown = 1u << 8, /* status outside the decoded mask */ + NativeAdrenoIrqFatal = 1u << 9 /* any class that cannot be ignored */ +}; +struct NativeAdrenoIrqResult { + uint32_t flags, status, unknown, cp_status, protect_status; +}; +/* A7XX RBBM/CP interrupt decode and acknowledgement, not an XNU interrupt + * source. IO read/write take dword register indices. The hardware interrupt + * must already be masked at the controller by the caller, and the caller must + * serialize handle() against every other MMIO user. Mask programming happens + * only in programMask(); handle() never changes the enabled set. A clear we + * cannot complete leaves the source armed, so it poisons the decoder and + * requires an independent reset rather than retrying with a changed mask. */ +class NativeAdrenoIrq final { + NativeAdrenoRingIo io_{}; + uint32_t enabled_=0; + bool attached_=false,poisoned_=false; + IOReturn poison(); +public: + NativeAdrenoIrq()=default; + ~NativeAdrenoIrq()=default; + NativeAdrenoIrq(const NativeAdrenoIrq &)=delete; + NativeAdrenoIrq &operator=(const NativeAdrenoIrq &)=delete; + IOReturn attach(const NativeAdrenoRingIo *); + IOReturn detach(); + /* The exact A7XX enable set: AHB, ATB async, GPC, CP_SW, CP_HW, PM4, + * RB_DONE_TS, CACHE_FLUSH_TS, ATB bus, hang, UCHE OOB/trap, TSB, fuse. */ + IOReturn programMask(uint32_t *programmed); + /* Reads, acknowledges, then decodes. Acknowledgement precedes decoding so + * an error reported here cannot be hidden by a later clear failure. */ + IOReturn handle(NativeAdrenoIrqResult *); + static uint32_t mask(); + static uint32_t cpErrorMask(); + bool poisoned() const {return poisoned_;} + uint32_t enabled() const {return enabled_;} +}; +#endif diff --git a/drivers/NativeAdreno/NativeAdrenoKernelTime.cpp b/drivers/NativeAdreno/NativeAdrenoKernelTime.cpp new file mode 100644 index 0000000..4a8d695 --- /dev/null +++ b/drivers/NativeAdreno/NativeAdrenoKernelTime.cpp @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: MIT */ +#include "NativeAdrenoKernelTime.h" +#include +#include +int NativeAdrenoKernelTime::now_us(void *cookie,uint64_t *value) +{ + if(cookie || !value)return 1; + uint64_t nanoseconds=0; + absolutetime_to_nanoseconds(mach_absolute_time(),&nanoseconds); + *value=nanoseconds/1000; + return 0; +} +int NativeAdrenoKernelTime::wait_us(void *cookie,unsigned microseconds) +{ + if(cookie)return 1; + IODelay(microseconds); + return 0; +} diff --git a/drivers/NativeAdreno/NativeAdrenoKernelTime.h b/drivers/NativeAdreno/NativeAdrenoKernelTime.h new file mode 100644 index 0000000..78641c2 --- /dev/null +++ b/drivers/NativeAdreno/NativeAdrenoKernelTime.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef NATIVE_ADRENO_KERNEL_TIME_H +#define NATIVE_ADRENO_KERNEL_TIME_H +#include "NativeAdrenoGmuPlatform.h" + +/* Real kernel time for the GMU startup: monotonic microseconds from + * mach_absolute_time and a bounded busy wait. This is the only production + * implementation of the time callbacks; tests inject their own. */ +struct NativeAdrenoKernelTime { + static int now_us(void *,uint64_t *); + static int wait_us(void *,unsigned); +}; +#endif diff --git a/drivers/NativeAdreno/NativeAdrenoMemory.cpp b/drivers/NativeAdreno/NativeAdrenoMemory.cpp new file mode 100644 index 0000000..6b46304 --- /dev/null +++ b/drivers/NativeAdreno/NativeAdrenoMemory.cpp @@ -0,0 +1,141 @@ +/* SPDX-License-Identifier: MIT */ +#include "NativeAdrenoMemory.h" +/* Where a call that never returns is stuck: the free/unmap/release path stores a stage number here, the board page shows it. */ +extern "C" {volatile uint32_t native_adreno_trace_stage;} +#include +#include +#include + +static bool outputWidth(unsigned bits) +{return bits==32 || bits==36 || bits==40 || bits==42 || bits==44 || bits==48;} +IOReturn NativeAdrenoMemory::allocate(size_t bytes,unsigned bits,bool contiguous,size_t budget) +{ + if(descriptor_ || spans_ || quarantined_)return kIOReturnBusy; + if(!bytes || (bytes&4095) || !outputWidth(bits) || !budget || budget>256u*1024u*1024u) + return kIOReturnBadArgument; + if(bytes>budget)return kIOReturnNoResources; + NativeAdrenoCpuState cpu{}; + IOReturn status=NativeAdrenoQueryCpu4K(&cpu);if(status!=kIOReturnSuccess)return status; + IOOptionBits options=static_cast(kIODirectionInOut)|static_cast(kIOMemoryMapperNone); + if(contiguous)options|=kIOMemoryPhysicallyContiguous; + uint64_t limit=UINT64_C(1)<prepare(); + if(status!=kIOReturnSuccess) {descriptor_->release();descriptor_=nullptr;return status;} + prepared_=true;bytes_=bytes;mapping_=descriptor_->getBytesNoCopy(); + status=kIOReturnUnsupported; + if(!mapping_ || ((uintptr_t)mapping_&4095) || bytes>UINTPTR_MAX-(uintptr_t)mapping_ || + descriptor_->getLength()!=bytes)goto failed; + span_capacity_=bytes/4096; + spans_=IONewData(NativeAdrenoPhysicalSpan,span_capacity_); + if(!spans_) {status=kIOReturnNoMemory;goto failed;} + for(size_t offset=0;offsetgetPhysicalSegment(offset,&length,kIOMemoryMapperNone); + if(!physical || (physical&4095) || length<4096 || physical>=limit || 4096>limit-physical) + {status=kIOReturnUnsupported;goto failed;} + if(contiguous && offset && physical!=spans_[0].physical+offset) + {status=kIOReturnUnsupported;goto failed;} + if(span_count_ && physical==spans_[span_count_-1].physical+spans_[span_count_-1].bytes) + spans_[span_count_-1].bytes+=4096; + else spans_[span_count_++]={offset,physical,4096}; + } + bzero(mapping_,bytes); + status=NativeAdrenoMaintainOwnedPages(mapping_,bytes,NativeAdrenoClean); + if(status!=kIOReturnSuccess)goto failed; + return kIOReturnSuccess; +failed: + /* release() deliberately retains descriptor/metadata if complete fails. + * Preserve the allocation's original failure instead of reporting cleanup + * as success. No object has been exposed to a GPU or userspace yet. */ + (void)release();return status; +} +IOReturn NativeAdrenoMemory::importTask(void *task,uint64_t address,size_t bytes,unsigned bits,size_t budget) +{ + if(descriptor_ || imported_ || spans_ || quarantined_)return kIOReturnBusy; + if(!task || !address || (address&4095) || !bytes || (bytes&4095) || address>UINT64_MAX-bytes || !outputWidth(bits) || + !budget || budget>256u*1024u*1024u)return kIOReturnBadArgument; + if(bytes>budget)return kIOReturnNoResources; + NativeAdrenoCpuState cpu{}; + IOReturn status=NativeAdrenoQueryCpu4K(&cpu);if(status!=kIOReturnSuccess)return status; + const uint64_t limit=UINT64_C(1)<(kIODirectionInOut)|static_cast(kIOMemoryMapperNone),static_cast(task)); + if(!imported_)return kIOReturnNoMemory; + status=imported_->prepare(); + if(status!=kIOReturnSuccess) {imported_->release();imported_=nullptr;return status;} + prepared_=true;bytes_=bytes; + status=kIOReturnUnsupported; + if(imported_->getLength()!=bytes)goto failed; + span_capacity_=bytes/4096; + spans_=IONewData(NativeAdrenoPhysicalSpan,span_capacity_); + if(!spans_) {status=kIOReturnNoMemory;goto failed;} + for(size_t offset=0;offsetgetPhysicalSegment(offset,&length,kIOMemoryMapperNone); + if(!physical || (physical&4095) || length<4096 || physical>=limit || 4096>limit-physical) + {status=kIOReturnUnsupported;goto failed;} + if(span_count_ && physical==spans_[span_count_-1].physical+spans_[span_count_-1].bytes) + spans_[span_count_-1].bytes+=4096; + else spans_[span_count_++]={offset,physical,4096}; + } + return kIOReturnSuccess; +failed: + (void)release();return status; +} +IOReturn NativeAdrenoMemory::release() +{ + native_adreno_trace_stage=30; + if(device_references_ || quarantined_)return kIOReturnBusy; + if(imported_) { + if(prepared_) { + IOReturn status=imported_->complete(); + if(status!=kIOReturnSuccess) {quarantined_=true;return status;} + prepared_=false; + } + imported_->release();imported_=nullptr; + } + if(descriptor_) { + if(prepared_) { + native_adreno_trace_stage=31; + IOReturn status=descriptor_->complete(); + native_adreno_trace_stage=32; + if(status!=kIOReturnSuccess) {quarantined_=true;return status;} + prepared_=false; + } + native_adreno_trace_stage=33; + descriptor_->release();descriptor_=nullptr; + native_adreno_trace_stage=34; + } + if(spans_)IODeleteData(spans_,NativeAdrenoPhysicalSpan,span_capacity_); + mapping_=nullptr;spans_=nullptr;bytes_=span_count_=span_capacity_=0; + return kIOReturnSuccess; +} +IOReturn NativeAdrenoMemory::holdForDevice() +{ + if(!backed() || !prepared_ || !span_count_ || quarantined_)return kIOReturnNotReady; + if(device_references_==UINT32_MAX)return kIOReturnNoResources; + ++device_references_;return kIOReturnSuccess; +} +IOReturn NativeAdrenoMemory::retireAfterQuiescence() +{ + if(!device_references_)return kIOReturnBadArgument; + if(quarantined_)return kIOReturnNotReady; + --device_references_;return kIOReturnSuccess; +} +IOReturn NativeAdrenoMemory::publishForDevice(size_t offset,size_t bytes,bool writes) +{ + if(!backed() || !prepared_ || quarantined_)return kIOReturnNotReady; + if(!bytes || ((offset|bytes)&4095) || offset>bytes_ || bytes>bytes_-offset)return kIOReturnBadArgument; + if(imported_)return kIOReturnSuccess; + return NativeAdrenoMaintainOwnedPages(static_cast(mapping_)+offset,bytes, + writes?NativeAdrenoCleanInvalidate:NativeAdrenoClean); +} +IOReturn NativeAdrenoMemory::acquireDeviceWrites(size_t offset,size_t bytes) +{ + if(!backed() || !prepared_ || quarantined_)return kIOReturnNotReady; + if(!bytes || ((offset|bytes)&4095) || offset>bytes_ || bytes>bytes_-offset)return kIOReturnBadArgument; + if(imported_)return kIOReturnSuccess; + return NativeAdrenoMaintainOwnedPages(static_cast(mapping_)+offset,bytes,NativeAdrenoInvalidate); +} diff --git a/drivers/NativeAdreno/NativeAdrenoMemory.h b/drivers/NativeAdreno/NativeAdrenoMemory.h new file mode 100644 index 0000000..3f38ed8 --- /dev/null +++ b/drivers/NativeAdreno/NativeAdrenoMemory.h @@ -0,0 +1,72 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef NATIVE_ADRENO_MEMORY_H +#define NATIVE_ADRENO_MEMORY_H +#include +#include +#include +class IOBufferMemoryDescriptor; +class IOMemoryDescriptor; +struct NativeAdrenoPhysicalSpan {uint64_t offset,physical,bytes;}; +struct NativeAdrenoCpuState { + uint64_t tcr,ttbr0,ttbr1,sctlr; + uint64_t kernel_page_bytes,kernel_map_page_bytes; +}; +IOReturn NativeAdrenoQueryCpu4K(NativeAdrenoCpuState *); +enum NativeAdrenoCacheOperation {NativeAdrenoClean,NativeAdrenoCleanInvalidate,NativeAdrenoInvalidate}; +IOReturn NativeAdrenoMaintainOwnedPages(void *,size_t,NativeAdrenoCacheOperation); + +/* Native XNU backing store, not a Metal device or user ABI. No IOMMU mapper, + * Linux fd/ioctl, hardware address, guessed PA or implicit destructor cleanup. + * Driver owns this object until release() succeeds; do not destroy/copy it + * with outstanding memory. Serialize every call under the queue/resource gate. + * Every published mapping/job must hold a device reference BEFORE publication, + * and retire it only after actual DMA quiescence and completed native TLB unmap. + * References are lifetime accounting, never fences or proof of quiescence. */ +class NativeAdrenoMemory final { + IOBufferMemoryDescriptor *descriptor_=nullptr; + IOMemoryDescriptor *imported_=nullptr; + void *mapping_=nullptr; + NativeAdrenoPhysicalSpan *spans_=nullptr; + size_t bytes_=0,span_count_=0,span_capacity_=0; + uint32_t device_references_=0; + bool prepared_=false,quarantined_=false; +public: + NativeAdrenoMemory()=default; + ~NativeAdrenoMemory()=default; + NativeAdrenoMemory(const NativeAdrenoMemory &)=delete; + NativeAdrenoMemory &operator=(const NativeAdrenoMemory &)=delete; + /* Explicit driver budget; bounded to 256MiB per backing object for now. + * Error may retain quarantined allocation state if complete() fails. + * Caller always keeps this object and checks quarantined(), even on error. + * CPU native4K state and kernel page ABI are checked before allocation. + * prepare(), raw unmapped physical segments, span/PA bounds and optional + * physical contiguity are verified before zeroing/PoC publication. */ + IOReturn allocate(size_t bytes,unsigned output_bits,bool contiguous,size_t byte_budget); + /* Pins bytes at address in task (a user task's own pages) instead of allocating; prepare() wires them until release(). + * There is no kernel mapping: the owner keeps its CPU mapping and does its own cache maintenance around GPU use, so + * publishForDevice/acquireDeviceWrites have nothing to do for this object. Spans and PA bounds are checked as for allocate. */ + IOReturn importTask(void *task,uint64_t address,size_t bytes,unsigned output_bits,size_t byte_budget); + bool imported() const {return imported_!=nullptr;} + bool backed() const {return descriptor_ || imported_;} + /* Refuses device references/quarantine; only clears ownership after real + * descriptor complete() succeeds. No automatic reset or resource recycling. */ + IOReturn release(); + IOReturn holdForDevice(); + IOReturn retireAfterQuiescence(); + void quarantine() {quarantined_=true;} + /* Requires real CPU/device exclusion and retained ownership. Before GPU + * writes, clean+invalidate; after genuine completion, invalidate only. + * CPU must not dirty these pages while the device owns access. */ + IOReturn publishForDevice(size_t offset,size_t bytes,bool device_writes); + IOReturn acquireDeviceWrites(size_t offset,size_t bytes); + bool quarantined() const {return quarantined_;} + uint32_t deviceReferences() const {return device_references_;} + size_t bytes() const {return bytes_;} + size_t spanCount() const {return span_count_;} + const NativeAdrenoPhysicalSpan *spans() const {return spans_;} + void *mapping() const {return mapping_;} + /* Borrowed descriptor, never release independently or expose table pools + * through clientMemoryForType. Sharing requires a separate owner/task ABI. */ + IOBufferMemoryDescriptor *descriptor() const {return descriptor_;} +}; +#endif diff --git a/drivers/NativeAdreno/NativeAdrenoPublish.cpp b/drivers/NativeAdreno/NativeAdrenoPublish.cpp new file mode 100644 index 0000000..6bd1bff --- /dev/null +++ b/drivers/NativeAdreno/NativeAdrenoPublish.cpp @@ -0,0 +1,504 @@ +/* SPDX-License-Identifier: MIT */ +/* IOKit surface for an already-running GPU context. The board code (kits/sheng) brings the GPU up, then hands this file + * the live VM, submit object, ring, clock and optional timer; one IOService is published and each IOServiceOpen creates a + * connection that mounts the NativeAdrenoUserClient core. Only one connection exists at a time: the core, the VM and the + * ring have one owner, and closing the connection waits for the CP, then unmaps and frees everything it created. + * Not yet done: per-client isolation (all objects share one GPU address space), entitlement/privilege checks on open, and + * a reset path. This is not the Metal or Turnip ABI; it carries NativeAdrenoABI.h unchanged. + * BO CPU mappings are handed out as uncached physical-range mappings to match the GPU's non-cacheable PTEs. + * The connection class is named AGXDeviceUserClient on purpose: the original sandbox profiles of the processes that use Metal + * allow IOServiceOpen for that user-client class name, and nothing else in this system can create one (the Apple AGX kext is + * never loaded without an Apple GPU), so the name only decides whether the Adreno Metal plugin may open its own driver. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include "NativeAdrenoUserClient.h" + +extern "C" void proc_selfname(char *, int); + +extern "C" { +/* What the processes that reached this driver did, for the boot diagnostic page. Plain counters and short strings. */ +struct NativeAdrenoMetalDiag { + uint32_t opens, busy, bad, closes, reports, property_reads, foreign_opens, foreign_type, name_reads, calls, last_selector, last_status, ext_entries, ext_selector, ext_status, ext_in, ext_out, tunnel_calls, probes, sel_bits[8], mb_maps, mb_calls, marks_total, marks[16], chk_macf, chk_res, chk_type, type_pass_lo, type_pass_hi, attach_status, inflight, inflight_since_calls, forced_unmaps; + char opener[3][17]; /* process names of the latest connections, newest first */ + char reader[3][17]; /* process names that read a registry property of the accelerator, newest first */ + char name_reader[17]; /* latest process that read MetalPluginName specifically */ + char starter[3][17]; /* processes that sent plugin marker 1 (the plugin's initWithAcceleratorPort: started), newest first */ + char report[64]; /* the plugin's own result line (SetMetadata text), latest */ + char report2[64]; /* the plugin's Turnip-phase line (text starting "T " or "W "), latest */ +} native_adreno_metal_diag; +} + +/* The open type the Adreno plugin passes to IOServiceOpen ('AG'); it lets the driver tell the plugin from the original IOGPU library + * (type 1 plus an API value) and from any other client of the accelerator. */ +enum : UInt32 {NativeAdrenoPluginOpenType=0x4147,NativeAdrenoTunnelMagic=0x4e544e4c /* 'NTNL' */}; + +/* Mailbox: a last-resort command channel for a sandbox that admits no external-method selector at all. It is one shared page (memory + * type NativeAdrenoMailboxType through IOConnectMapMemory64, which the selector filter does not cover) and a kernel poller that runs + * whatever the live connection writes into it. Layout (little endian dwords): +0 magic, +4 request seq, +8 response seq, +12 method, + * +16 status, +20 input bytes, +24 output bytes, input at +0x40 (max 0x700), output at +0x800 (max 0x700). */ +enum : UInt32 {NativeAdrenoMailboxType=0xC0DE,NativeAdrenoMailboxMagic=0x4d41494c /* 'MAIL' */}; +namespace { +IOBufferMemoryDescriptor *gMailbox=nullptr; +volatile uint32_t *gMailboxWords=nullptr; +volatile bool gPollerStarted=false; +} +class AGXDeviceUserClient; +static volatile bool gMailboxLive=false; /* a connection is mounted; set and cleared by it, read by the poller, which never touches the object */ +void nativeAdrenoMailboxPoller(void *,wait_result_t); +static IOReturn runCore(bool live,uint32_t method,const void *input,size_t input_size,void *output,size_t output_size); + +/* The two sandbox gates IOServiceOpen applies after newUserClient (XNU IOUserClient.cpp is_io_service_open_extended): the MACF open check on + * the client object and the sandbox's filter resolver. They are run here for the caller too, only to record which one refuses it. */ +extern "C" int mac_iokit_check_open(void *cred,void *client,unsigned type); +extern "C" void *kauth_cred_get(void); +extern const struct io_filter_callbacks *gIOUCFilterCallbacks; + +extern "C" volatile uint32_t native_adreno_trace_stage; +extern "C" { +/* One entry per IOServiceOpen of the accelerator (not markers): who, which type, what the two sandbox gates said. */ +struct NativeAdrenoOpenRecord {char name[20]; uint32_t type, macf, res, attach, foreign;}; +struct NativeAdrenoOpenRecord native_adreno_open_log[8]; +volatile uint32_t native_adreno_open_log_count; +} + +namespace { +void shiftName(char names[3][17]) { + memcpy(names[2], names[1], 17); + memcpy(names[1], names[0], 17); + proc_selfname(names[0], 17); +} +struct Published { + NativeAdrenoUserClient *core=nullptr; + NativeAdrenoVm *vm=nullptr; + NativeAdrenoSubmit *submit=nullptr; + NativeAdrenoRing *ring=nullptr; + NativeAdrenoRingClock clock{}; + const NativeAdrenoTimer *timer=nullptr; + const NativeAdrenoAbiDeviceInfo *info=nullptr; + IOLock *lock=nullptr; + IOService *service=nullptr; + bool connected=false; + uint32_t input_bits=0,output_bits=0; + uint32_t opened=0,closed=0; +} g; +} + +class NativeAdrenoGpuPublished : public IOService { + OSDeclareDefaultStructors(NativeAdrenoGpuPublished) +public: + IOReturn newUserClient(task_t owningTask,void *securityID,UInt32 type, + OSDictionary *properties,IOUserClient **handler) APPLE_KEXT_OVERRIDE; +}; +OSDefineMetaClassAndStructors(NativeAdrenoGpuPublished,IOService) + +class AGXDeviceUserClient : public IOUserClient { + OSDeclareDefaultStructors(AGXDeviceUserClient) + bool live_=false; + /* One user-view descriptor per BO token, reused by every clientMemoryForType call. IOConnectUnmapMemory64 asks clientMemoryForType for + * the descriptor again and unmaps only a mapping that belongs to THAT descriptor (kIOMapReference); a fresh descriptor per call made the + * unmap a silent no-op, the user PTEs outlived BoCpuUnmap, and BoFree handed pages with a live user mapping to the VM: + * pmap_assert_free panicked the freeing core and the panic screen parked it (the "BoFree hang", dead touch). */ + struct View {uint32_t token;IOMemoryDescriptor *descriptor;}; + enum {MaxViews=64}; + View views_[MaxViews]={}; + void dropViews(uint32_t token); + static IOReturn action(OSObject *target,void *reference,IOExternalMethodArguments *arguments); + IOReturn run(uint32_t method,const void *input,size_t input_size,void *output,size_t output_size); +public: + bool mount(); + IOReturn clientClose() APPLE_KEXT_OVERRIDE; + void free() APPLE_KEXT_OVERRIDE; + IOReturn clientMemoryForType(UInt32 type,IOOptionBits *options,IOMemoryDescriptor **memory) APPLE_KEXT_OVERRIDE; + IOReturn externalMethod(uint32_t selector,IOExternalMethodArguments *arguments, + IOExternalMethodDispatch *dispatch,OSObject *target,void *reference) APPLE_KEXT_OVERRIDE; +}; +OSDefineMetaClassAndStructors(AGXDeviceUserClient,IOUserClient) + +void nativeAdrenoMailboxPoller(void *,wait_result_t) +{ + uint32_t last=0; + for(;;) { + IOSleep(1); + if(!gMailboxLive || !gMailboxWords)continue; + const uint32_t request=gMailboxWords[1]; + if(request==last || request==gMailboxWords[2])continue; + last=request; + const uint32_t method=gMailboxWords[3],in_size=gMailboxWords[5],out_size=gMailboxWords[6]; + IOReturn status=kIOReturnBadArgument; + if(in_size<=0x700 && out_size<=0x700) { + uint8_t input[0x700],output[0x700]; + memcpy(input,const_cast(gMailboxWords)+0x10,in_size); + status=runCore(g.connected,method,input,in_size,output,out_size); + if(!status)memcpy(const_cast(gMailboxWords)+0x200,output,out_size); + } + native_adreno_metal_diag.mb_calls=native_adreno_metal_diag.mb_calls+1; + gMailboxWords[4]=uint32_t(status); + __asm__ volatile("dmb sy" ::: "memory"); + gMailboxWords[2]=request; + __asm__ volatile("dmb sy" ::: "memory"); + } +} + +bool AGXDeviceUserClient::mount() +{ + IOLockLock(g.lock); + bool ok=false; + if(!g.connected && g.core->open(NativeAdrenoAbiVersion,g.vm,g.submit,g.ring,g.input_bits,g.output_bits)==kIOReturnSuccess) { + if(g.timer)g.core->attachTimer(g.timer); + if(g.info)g.core->attachDeviceInfo(g.info); + g.connected=true;live_=true;++g.opened;ok=true; + if(!gMailbox) { + gMailbox=IOBufferMemoryDescriptor::inTaskWithOptions(kernel_task,kIODirectionInOut,4096,4096); + if(gMailbox) {gMailboxWords=static_cast(gMailbox->getBytesNoCopy());memset(const_cast(gMailboxWords),0,4096);} + } + if(gMailboxWords) { + memset(const_cast(gMailboxWords),0,4096); + gMailboxWords[0]=NativeAdrenoMailboxMagic; + if(!gPollerStarted) { + thread_t thread=nullptr; + if(kernel_thread_start(nativeAdrenoMailboxPoller,nullptr,&thread)==KERN_SUCCESS) {thread_deallocate(thread);gPollerStarted=true;} + } + } + gMailboxLive=true; + } + IOLockUnlock(g.lock); + return ok; +} +/* Removes the cached view(s) of token (0 = all) and tears down every user mapping of them, so the BO's pages are never freed while a user + * PTE still points at them, whether or not the client unmapped. */ +void AGXDeviceUserClient::dropViews(uint32_t token) +{ + for(;;) { + IOMemoryDescriptor *view=nullptr; + IOLockLock(g.lock); + for(View &v:views_)if(v.descriptor && (!token || v.token==token)) {view=v.descriptor;v.descriptor=nullptr;v.token=0;break;} + IOLockUnlock(g.lock); + if(!view)return; + /* The returned reference is the last one outside the descriptor's own list: dropping it frees the map, which unmaps the user range. */ + while(removeMappingForDescriptor(view))native_adreno_metal_diag.forced_unmaps=native_adreno_metal_diag.forced_unmaps+1; + view->release(); + } +} +IOReturn AGXDeviceUserClient::clientClose() +{ + dropViews(0); + IOLockLock(g.lock); + IOReturn status=kIOReturnSuccess; + if(live_) { + status=g.core->releaseAll(&g.clock,2000000); + /* A failed teardown leaves the objects live and the slot taken: nothing is forced or reused. */ + if(!status) {gMailboxLive=false;live_=false;g.connected=false;++g.closed;native_adreno_metal_diag.closes=native_adreno_metal_diag.closes+1;} + } + IOLockUnlock(g.lock); + if(!status)terminate(); + return status; +} +void AGXDeviceUserClient::free() +{ + dropViews(0); + IOUserClient::free(); +} +IOReturn AGXDeviceUserClient::clientMemoryForType(UInt32 type,IOOptionBits *options,IOMemoryDescriptor **memory) +{ + if(memory)*memory=nullptr; + if(!memory || !options)return kIOReturnBadArgument; + IOReturn status=kIOReturnNotFound; + if(type==NativeAdrenoMailboxType) { + if(live_ && gMailbox) { + gMailbox->retain(); + *memory=gMailbox; + *options=kIOMapAnywhere; /* ordinary cached memory: both sides are CPUs, so it is coherent */ + native_adreno_metal_diag.mb_maps=native_adreno_metal_diag.mb_maps+1; + return kIOReturnSuccess; + } + return kIOReturnNotReady; + } + IOLockLock(g.lock); + for(View &v:views_)if(v.descriptor && v.token==type) { + v.descriptor->retain();*memory=v.descriptor;*options=kIOMapAnywhere|kIOMapInhibitCache; + IOLockUnlock(g.lock); + return kIOReturnSuccess; + } + NativeAdrenoMemory *backing=live_?g.core->memoryForToken(type):nullptr; + if(backing) { + /* The BO's pages are cached in the kernel and the GPU's PTEs are non-cacheable, so the client needs an uncached + * view. A mapping of the buffer descriptor with a different cache mode is not reliable, so describe the same raw + * physical spans (the ones the SMMU maps) and ask for kIOMapInhibitCache, exactly how the board code maps the + * ring and the GMU windows. The descriptor does not retain the pages: the client must unmap before BoCpuUnmap/BoFree. */ + enum {MaxSpans=64}; + IOAddressRange ranges[MaxSpans]; + const size_t count=backing->spanCount(); + if(!count || count>MaxSpans)status=kIOReturnNoResources; + else { + for(size_t i=0;ispans()[i].physical,backing->spans()[i].bytes}; + IOMemoryDescriptor *descriptor=IOMemoryDescriptor::withAddressRanges(ranges,(UInt32)count, + static_cast(kIODirectionInOut)|static_cast(kIOMemoryTypePhysical64),nullptr); + if(!descriptor)status=kIOReturnNoMemory; + else { + for(View &v:views_)if(!v.descriptor) {v.token=type;v.descriptor=descriptor;descriptor->retain();break;} + *memory=descriptor; + *options=kIOMapAnywhere|kIOMapInhibitCache; + status=kIOReturnSuccess; + } + } + } + IOLockUnlock(g.lock); + return status; +} +static IOReturn runCore(bool live,uint32_t method,const void *input,size_t input_size,void *output,size_t output_size) +{ + if(method==NativeAdrenoMethodSetMetadata && input && input_size==sizeof(NativeAdrenoAbiMetadata)) { + const NativeAdrenoAbiMetadata *meta=static_cast(input); + /* The raw test's line and the Turnip phase's line ("T ..." result or "W ..." watchdog sample) are kept apart, so neither hides the other. */ + const bool turnip=meta->bytes>=2 && (meta->text[0]=='T' || meta->text[0]=='W') && meta->text[1]==' '; + char *line=turnip?native_adreno_metal_diag.report2:native_adreno_metal_diag.report; + uint32_t n=meta->bytesbytes:uint32_t(sizeof(native_adreno_metal_diag.report)-1); + memset(line,0,sizeof(native_adreno_metal_diag.report)); + memcpy(line,meta->text,n); + native_adreno_metal_diag.reports=native_adreno_metal_diag.reports+1; + } + /* A call that is still running when the boot page is photographed shows as inflight = its method number + 1 (0 = none), so a hang in the + * kernel is attributable to one ABI method. */ + native_adreno_metal_diag.inflight=method+1; + native_adreno_metal_diag.inflight_since_calls=native_adreno_metal_diag.calls; + IOLockLock(g.lock); + const IOReturn status=live?g.core->dispatch(method,input,input_size,output,output_size,&g.clock):kIOReturnNotReady; + IOLockUnlock(g.lock); + native_adreno_metal_diag.inflight=0; + native_adreno_metal_diag.calls=native_adreno_metal_diag.calls+1; + native_adreno_metal_diag.last_selector=method; + native_adreno_metal_diag.last_status=uint32_t(status); + return status; +} +IOReturn AGXDeviceUserClient::run(uint32_t method,const void *input,size_t input_size,void *output,size_t output_size) +{ + if(method==NativeAdrenoMethodBoCpuUnmap && input && input_size==sizeof(NativeAdrenoAbiBoCpuUnmap)) + dropViews(static_cast(input)->token); + if(method==NativeAdrenoMethodBoImport) { + /* The pages belong to the task making this call (an external method runs in the caller's context). */ + if(!input || input_size!=sizeof(NativeAdrenoAbiBoImport) || !output || output_size!=sizeof(NativeAdrenoAbiBoAllocReply)) + return kIOReturnBadArgument; + NativeAdrenoAbiBoImport request{}; + memcpy(&request,input,sizeof(request)); + NativeAdrenoAbiBoAllocReply reply{}; + native_adreno_metal_diag.inflight=method+1; + IOLockLock(g.lock); + const IOReturn status=live_?g.core->importBo(&request,current_task(),&reply):kIOReturnNotReady; + IOLockUnlock(g.lock); + native_adreno_metal_diag.inflight=0; + native_adreno_metal_diag.calls=native_adreno_metal_diag.calls+1; + native_adreno_metal_diag.last_selector=method; + native_adreno_metal_diag.last_status=uint32_t(status); + if(!status)memcpy(output,&reply,sizeof(reply)); + return status; + } + return runCore(live_,method,input,input_size,output,output_size); +} +IOReturn AGXDeviceUserClient::action(OSObject *target,void *,IOExternalMethodArguments *arguments) +{ + AGXDeviceUserClient *connection=OSDynamicCast(AGXDeviceUserClient,target); + if(!connection || !arguments)return kIOReturnBadArgument; + return connection->run(arguments->selector,arguments->structureInput,arguments->structureInputSize, + arguments->structureOutput,arguments->structureOutputSize); +} +IOReturn AGXDeviceUserClient::externalMethod(uint32_t selector,IOExternalMethodArguments *arguments, + IOExternalMethodDispatch *,OSObject *target,void *reference) +{ + /* One entry per ABI method; sizes are the exact wire records. */ + static const IOExternalMethodDispatch table[]={ + {&action,0,0,0,sizeof(NativeAdrenoAbiParams)}, + {&action,0,sizeof(NativeAdrenoAbiBoAlloc),0,sizeof(NativeAdrenoAbiBoAllocReply)}, + {&action,0,sizeof(NativeAdrenoAbiBoMap),0,sizeof(NativeAdrenoAbiBoMapReply)}, + {&action,0,sizeof(NativeAdrenoAbiBoFree),0,0}, + {&action,0,sizeof(NativeAdrenoAbiSubmit),0,sizeof(NativeAdrenoAbiSubmitReply)}, + {&action,0,sizeof(NativeAdrenoAbiWaitFence),0,0}, + {&action,0,0,0,sizeof(NativeAdrenoAbiTimestamp)}, + {&action,0,0,0,sizeof(NativeAdrenoAbiStatus)}, + {&action,0,sizeof(NativeAdrenoAbiMetadata),0,0}, + {&action,0,sizeof(NativeAdrenoAbiMetadata),0,sizeof(NativeAdrenoAbiMetadata)}, + {&action,0,sizeof(NativeAdrenoAbiQueue),0,sizeof(NativeAdrenoAbiQueueReply)}, + {&action,0,sizeof(NativeAdrenoAbiQueue),0,0}, + {&action,0,sizeof(NativeAdrenoAbiBoCpuMap),0,sizeof(NativeAdrenoAbiBoCpuMapReply)}, + {&action,0,sizeof(NativeAdrenoAbiBoCpuUnmap),0,0}, + {&action,0,0,0,sizeof(NativeAdrenoAbiDeviceInfo)}, + {&action,0,sizeof(NativeAdrenoAbiBoImport),0,sizeof(NativeAdrenoAbiBoAllocReply)}}; + native_adreno_metal_diag.ext_entries=native_adreno_metal_diag.ext_entries+1; + if(selector<256)native_adreno_metal_diag.sel_bits[selector>>5]|=1u<<(selector&31); + /* Tunnel. The sandbox resolves a filter policy for this class at open time and then admits only the selectors its profile lists, + * so which selector numbers reach this object is decided outside the driver. A tunnel call carries the real ABI method in its + * first eight bytes and may use ANY admitted selector: {magic, method, payload...} in, the method's exact output out. Method + * 0xffffffff is a probe that only proves the selector was admitted. */ + if(arguments && arguments->structureInput && arguments->structureInputSize>=8 && + static_cast(arguments->structureInput)[0]==NativeAdrenoTunnelMagic) { + const uint32_t *header=static_cast(arguments->structureInput); + if(header[1]==0xffffffffu) { + native_adreno_metal_diag.probes=native_adreno_metal_diag.probes+1; + return kIOReturnSuccess; + } + native_adreno_metal_diag.tunnel_calls=native_adreno_metal_diag.tunnel_calls+1; + const IOReturn tunneled=run(header[1],reinterpret_cast(arguments->structureInput)+8, + arguments->structureInputSize-8,arguments->structureOutput,arguments->structureOutputSize); + native_adreno_metal_diag.ext_status=uint32_t(tunneled); + return tunneled; + } + native_adreno_metal_diag.ext_selector=selector; + native_adreno_metal_diag.ext_in=arguments?arguments->structureInputSize:0xffff; + native_adreno_metal_diag.ext_out=arguments?arguments->structureOutputSize:0xffff; + if(selector<1 || selector>sizeof(table)/sizeof(table[0])) { + native_adreno_metal_diag.ext_status=uint32_t(kIOReturnUnsupported); + return kIOReturnUnsupported; + } + const IOReturn result=IOUserClient::externalMethod(selector,arguments,const_cast(&table[selector-1]), + target?target:this,reference); + native_adreno_metal_diag.ext_status=uint32_t(result); + return result; +} +/* limit > 0 caps the connections ever granted: a plugin that crashes its host process on every launch (the process respawns and + * loads it again) must not be able to keep a system process in a crash loop, so after the cap further opens are refused. */ +extern "C" IOReturn native_adreno_new_user_client(IOService *provider,task_t owningTask,void *securityID,UInt32 type, + OSDictionary *properties,IOUserClient **handler,unsigned limit) +{ + if(handler)*handler=nullptr; + if(!handler || !g.core)return kIOReturnNotReady; + /* Progress markers: open types 0x4147xxxx (xxxx nonzero) are not connections. The plugin sends one before and after each step so + * that a sandbox that admits neither selectors nor mapping, or a crash, still leaves a trace of how far it got. */ + if((type>>16)==NativeAdrenoPluginOpenType && (type&0xffff)) { + const uint32_t value=type&0xffff,slot=native_adreno_metal_diag.marks_total&15; + /* Text stream into report2, one character per marker: 0x7fff starts a line, 0x70cc appends character cc. It needs no connection, + * so the plugin's Turnip watchdog can report while Turnip itself holds the single connection slot. Not counted as markers. */ + if(value>=0x7000 && value<0x8000) { + /* 0x7ffe starts a line in report (GPU7) instead: the Metal bridge's result, after the raw test's line has been read. */ + static uint32_t length; + static char *line=native_adreno_metal_diag.report2; + static_assert(sizeof(native_adreno_metal_diag.report)==sizeof(native_adreno_metal_diag.report2),"stream lines"); + if(value==0x7fff || value==0x7ffe) { + line=value==0x7fff?native_adreno_metal_diag.report2:native_adreno_metal_diag.report; + memset(line,0,sizeof(native_adreno_metal_diag.report2));length=0; + } + else if(value<0x7100 && lengthinitWithTask(owningTask,securityID,type,properties)) { + connection->release(); + native_adreno_metal_diag.bad=native_adreno_metal_diag.bad+1; + return kIOReturnNoResources; + } + /* IOService::newUserClient attaches and starts what it creates (a client without a provider chain is also a reason for the sandbox to + * refuse). Then the two sandbox gates IOServiceOpen would apply afterwards are run here for every opener and logged with the + * process name and open type, so the boot page shows which processes the sandbox lets reach this class and with which type. */ + unsigned attach_state=0; + if(provider) { + const bool attached=connection->attach(provider); + attach_state=attached?1:2; + if(attached && !connection->start(provider)) { + connection->detach(provider); + attach_state=3; + } + } + native_adreno_metal_diag.attach_status=attach_state; + const uint32_t macf=uint32_t(mac_iokit_check_open(kauth_cred_get(),connection,type)); + uint32_t resolver=0xfeedfeed; + if(gIOUCFilterCallbacks && gIOUCFilterCallbacks->io_filter_resolver) { + io_filter_policy_t policy=0; + const IOReturn resolved=gIOUCFilterCallbacks->io_filter_resolver(owningTask,connection,type,&policy); + resolver=uint32_t(resolved); + if(resolved==kIOReturnSuccess && policy && gIOUCFilterCallbacks->io_filter_release)gIOUCFilterCallbacks->io_filter_release(policy); + } + native_adreno_metal_diag.chk_type=type; + native_adreno_metal_diag.chk_macf=macf; + native_adreno_metal_diag.chk_res=resolver; + { + NativeAdrenoOpenRecord &record=native_adreno_open_log[native_adreno_open_log_count&7]; + proc_selfname(record.name,sizeof(record.name)); + record.type=type;record.macf=macf;record.res=resolver;record.attach=attach_state;record.foreign=type!=NativeAdrenoPluginOpenType; + native_adreno_open_log_count=native_adreno_open_log_count+1; + } + /* Only the Adreno plugin's open (type NativeAdrenoPluginOpenType) is served. The original IOGPU library opens the accelerator with type 1 + * (and an API value in the upper bits) for an IOGPUDeviceUserClient, which this driver is not; refuse it cleanly. */ + if(type!=NativeAdrenoPluginOpenType) { + connection->release(); + native_adreno_metal_diag.foreign_opens=native_adreno_metal_diag.foreign_opens+1; + native_adreno_metal_diag.foreign_type=type; + return kIOReturnUnsupported; + } + if(macf || (resolver!=uint32_t(kIOReturnSuccess) && resolver!=uint32_t(kIOReturnUnsupported))) { + /* The sandbox refuses this process; IOServiceOpen would refuse it after us, so do not take the single connection slot. */ + connection->release(); + return macf?kIOReturnNotPermitted:IOReturn(resolver); + } + if(limit && native_adreno_metal_diag.opens>=limit) { + connection->release(); + native_adreno_metal_diag.busy=native_adreno_metal_diag.busy+1; + return kIOReturnBusy; + } + if(!connection->mount()) { + connection->release(); + native_adreno_metal_diag.busy=native_adreno_metal_diag.busy+1; + return kIOReturnBusy; + } + native_adreno_metal_diag.opens=native_adreno_metal_diag.opens+1; + *handler=connection; + return kIOReturnSuccess; +} +extern "C" void native_adreno_note_property_read(const char *key) +{ + if(key && !strcmp(key,"MetalPluginName")) { + proc_selfname(native_adreno_metal_diag.name_reader,17); + native_adreno_metal_diag.name_reads=native_adreno_metal_diag.name_reads+1; + } + shiftName(native_adreno_metal_diag.reader); + native_adreno_metal_diag.property_reads=native_adreno_metal_diag.property_reads+1; +} +IOReturn NativeAdrenoGpuPublished::newUserClient(task_t owningTask,void *securityID,UInt32 type, + OSDictionary *properties,IOUserClient **handler) +{ + return native_adreno_new_user_client(this,owningTask,securityID,type,properties,handler,0); +} + +/* Publishes the service once. Everything passed in must outlive the system (the board probe keeps them in statics) and + * the VM, submit object and ring must already be attached to the live GPU context. The core must not be open. */ +extern "C" IOReturn native_adreno_publish(NativeAdrenoVm *vm,NativeAdrenoSubmit *submit,NativeAdrenoRing *ring, + const NativeAdrenoRingClock *clock,const NativeAdrenoTimer *timer,const NativeAdrenoAbiDeviceInfo *info, + uint32_t input_bits,uint32_t output_bits) +{ + if(g.core || !vm || !submit || !ring || !clock || !clock->now_us)return kIOReturnBadArgument; + g.lock=IOLockAlloc(); + g.core=new NativeAdrenoUserClient; + NativeAdrenoGpuPublished *service=new NativeAdrenoGpuPublished; + if(!g.lock || !g.core || !service || !service->init() || !service->attach(IOService::getPlatform())) { + if(service)service->release(); + return kIOReturnNoMemory; + } + g.vm=vm;g.submit=submit;g.ring=ring;g.clock=*clock;g.timer=timer;g.info=info;g.input_bits=input_bits;g.output_bits=output_bits; + g.service=service; + service->setName("NativeAdrenoGpu"); + service->registerService(); + return kIOReturnSuccess; +} +extern "C" uint32_t native_adreno_publish_stats(uint32_t *closed) +{ + if(closed)*closed=g.closed; + return g.opened; +} +extern "C" IOService *native_adreno_published_service() +{ + return g.service; +} diff --git a/drivers/NativeAdreno/NativeAdrenoPwrup.cpp b/drivers/NativeAdreno/NativeAdrenoPwrup.cpp new file mode 100644 index 0000000..d604c14 --- /dev/null +++ b/drivers/NativeAdreno/NativeAdrenoPwrup.cpp @@ -0,0 +1,58 @@ +/* SPDX-License-Identifier: MIT */ +#include "NativeAdrenoPwrup.h" +#include +#include +#include "NativeAdrenoPwrupRegs.h" +enum : size_t { HeaderBytes=16, PairBytes=8, TripletBytes=12 }; +size_t NativeAdrenoPwrupList::requiredBytes(const NativeAdrenoPwrupCounts &counts) +{ + return HeaderBytes+size_t(counts.ifpc+counts.preemption)*PairBytes+ + size_t(counts.dynamic)*TripletBytes; +} +IOReturn NativeAdrenoPwrupList::buildA740(int (*read)(void *,uint32_t,uint32_t *), + void *cookie,uint8_t *out,size_t capacity,size_t *written) +{ + if(!read || !out || !written)return kIOReturnBadArgument; + uint32_t values[NATIVE_ADRENO_A740_PWRUP_COUNT]={}; + for(size_t index=0;index0xff || counts.preemption>0xff || counts.dynamic>0xffff) + return kIOReturnBadArgument; + const size_t needed=requiredBytes(counts); + if(capacity(counts.ifpc); + out[13]=static_cast(counts.preemption); + out[14]=static_cast(counts.dynamic&0xff); + out[15]=static_cast((counts.dynamic>>8)&0xff); + uint32_t *cursor=reinterpret_cast(out+HeaderBytes); + for(uint32_t i=0;i +#include +#include + +/* Pack the A7xx power-up register list consumed by CP_ME_INIT. The layout is + * the pinned `struct cpu_gpu_lock`: three u32 lock words, a union of either + * {u8 ifpc_len, u8 preemption_len, u16 dynamic_len} (A7xx) or two u16 fields + * (A6xx), then 64-bit register slots holding {reg, current value} pairs and + * 12-byte dynamic triplets {aperture, offset, value}. Only the A7xx form is + * packed here. Values must come from actual register reads at bring-up; this + * function never invents one, never reads MMIO and never clears the lock words + * silently. Capacity and count limits are checked before any write. */ +struct NativeAdrenoPwrupCounts { + uint32_t ifpc; + uint32_t preemption; + uint32_t dynamic; +}; +class NativeAdrenoPwrupList final { + NativeAdrenoPwrupList()=delete; +public: + static size_t requiredBytes(const NativeAdrenoPwrupCounts &); + /* regs/values are parallel arrays; dynamic triplets are {aperture,offset, + * value} in that order. Output is the exact byte image; *written receives + * the length on success. */ + /* Read the sixteen A740 preemption registers through the callback (raw + * 32-bit GPU register reads) and pack them with zero IFPC and dynamic + * counts. A read failure refuses and writes nothing. */ + static IOReturn buildA740(int (*read)(void *,uint32_t,uint32_t *),void *cookie, + uint8_t *out,size_t capacity,size_t *written); + static IOReturn pack(const NativeAdrenoPwrupCounts &, + const uint32_t *ifpc_regs,const uint32_t *ifpc_values, + const uint32_t *preemption_regs,const uint32_t *preemption_values, + const uint32_t *dynamic_triplets,uint8_t *out,size_t capacity,size_t *written); +}; +#endif diff --git a/drivers/NativeAdreno/NativeAdrenoPwrupRegs.h b/drivers/NativeAdreno/NativeAdrenoPwrupRegs.h new file mode 100644 index 0000000..570f464 --- /dev/null +++ b/drivers/NativeAdreno/NativeAdrenoPwrupRegs.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef NATIVE_ADRENO_PWRUP_REGS_H +#define NATIVE_ADRENO_PWRUP_REGS_H +#include +/* Generated by scripts/derive-a740-pwrup.py from the pinned + * a6xx_catalog.c list and the pinned XMLs, selecting only entries whose + * variants include A7XX. Do not edit by hand. */ +static const uint32_t NativeAdrenoA740PwrupRegs[]={ + 0x00000e09, + 0x00000e0a, + 0x00000e07, + 0x00000e08, + 0x00000e0b, + 0x00000e0c, + 0x00000e0d, + 0x00000e0e, + 0x00000e17, + 0x00000e01, + 0x00008e08, + 0x00008e28, + 0x00008e50, + 0x00000e3a, + 0x00000e19, + 0x0000b602, +}; +#define NATIVE_ADRENO_A740_PWRUP_COUNT \ + (sizeof(NativeAdrenoA740PwrupRegs)/sizeof(NativeAdrenoA740PwrupRegs[0])) +#endif diff --git a/drivers/NativeAdreno/NativeAdrenoRing.cpp b/drivers/NativeAdreno/NativeAdrenoRing.cpp new file mode 100644 index 0000000..4d37c28 --- /dev/null +++ b/drivers/NativeAdreno/NativeAdrenoRing.cpp @@ -0,0 +1,274 @@ +/* SPDX-License-Identifier: MIT */ +#include "NativeAdrenoRing.h" +#include "AdrenoPM4.h" +#include + +/* Values are taken from the pinned Linux source, not guessed: + * drivers/gpu/drm/msm/registers/adreno/a6xx.xml (register offsets), and + * registers/adreno/adreno_pm4.xml (opcode 0x46 CP_EVENT_WRITE7, event 0x31 + * CACHE_CLEAN, WRITE_SRC bits 20..22, WRITE_DST bit 24, WRITE_ENABLED bit 27, + * IRQ bit 31). CP_RB_CNTL BUFSZ bits 0..5 and BLKSZ bits 8..13 are the same + * fields MSM_GPU_RB_CNTL_DEFAULT programs. All are dword register indices. */ +enum : uint32_t { + RingBase = 0x0800, /* 64-bit CP_RB_BASE */ + RingControl = 0x0802, /* CP_RB_CNTL */ + RingRptrAddress = 0x0804, /* 64-bit CP_RB_RPTR_ADDR */ + RingRptr = 0x0806, + RingWptr = 0x0807, + RingScratch = 0x0885, /* CP_SCRATCH(2) for A6XX..A7XX */ + RingFenceOpcode = 0x46, /* CP_EVENT_WRITE7 on A7XX */ + RingFenceEvent = 0x31, /* CACHE_CLEAN */ + RingFenceEnable = 1u << 27, + RingFenceIrq = 1u << 31, + RingBlockBytes = 32, + /* A7XX framing, as a7xx_submit() in the pinned a6xx_gpu.c emits it: CP_THREAD_CONTROL (0x17; THREAD bits 0..1, + * BR 1, BV 2, BOTH 3; SYNC_THREADS bit 31), CP_SET_MARKER (0x65; 0x101 IFPC disable, 0x100 enable), CCU_INVALIDATE_DEPTH + * 0x18 / CCU_INVALIDATE_COLOR 0x19 and CP_WAIT_TIMESTAMP (0x14) from adreno_pm4.xml. User command streams synchronize + * the BR and BV threads themselves (Turnip waits on BV timestamps), so the IBs must run with both threads enabled. */ + RingThreadControl = 0x17, + RingThreadBr = 1, + RingThreadBv = 2, + RingThreadBoth = 3, + RingSyncThreads = 1u << 31, + RingSetMarker = 0x65, + RingWaitTimestamp = 0x14, + RingCcuInvalidateDepth = 0x18, + RingCcuInvalidateColor = 0x19, + /* What the SQE's CP_SMMU_TABLE_UPDATE handler does before it switches tables (a740_sqe.fw, qrisc-disasm): CACHE_FLUSH7 (0x32) + * and CACHE_INVALIDATE7 (0x33) with a wait for idle (CP_WAIT_FOR_IDLE 0x26). Linux gets these from that packet on every context + * switch; this ring never sends it, and event 0x31 is only CACHE_CLEAN on A7XX, so nothing else ever invalidated UCHE. */ + RingCacheFlush7 = 0x32, + RingCacheInvalidate7 = 0x33, + RingWaitForIdle = 0x26, + RingPrologueWords = 11, + /* a6xx_set_pagetable()'s A7XX context entry: CP_RESET_CONTEXT_STATE (0x1f; CLEAR_ON_CHIP_TS, CLEAR_RESOURCE_TABLE, + * CLEAR_BV_BR_COUNTER, RESET_GLOBAL_LOCAL_TS = bits 0..3) between thread syncs, LRZ_FLUSH_INVALIDATE (0x26), the + * RBBM_PERFCTR_SRAM_INIT_CMD (0x050e) write and CACHE_INVALIDATE (0x31), after a CP_WAIT_TIMESTAMP on the last fence, then the + * CP_WAIT_REG_MEM (0x3c, WRITE_EQ) on RBBM_PERFCTR_SRAM_INIT_STATUS (0x050f) bit 0 that Linux always emits after the SRAM init + * write. Without that wait the client's first IB started while the SRAM was still clearing; on sheng (int-064..067) the CP then + * stopped on the IB's first CP_THREAD_CONTROL, while the same IB retired under the Linux driver. */ + RingResetContextState = 0x1f, + RingResetAll = 0xf, + RingLrzFlushInvalidate = 0x26, + RingCacheInvalidate = 0x31, + RingPerfctrSramInit = 0x050e, + RingPerfctrSramInitStatus = 0x050f, + RingWaitRegMem = 0x3c, + RingWaitWriteEq = 3, + RingResetWords = 26, + RingFenceWords = 7, /* kernel bring-up submissions: TYPE4 scratch + TYPE7 event */ + RingFramedFenceWords = 31,/* user submissions: scratch, BV timestamp, BR wait, ring timestamp */ + RingBvFenceOffset = 12 /* msm_rbmemptrs.bv_fence; +8 is bv_rptr, the BV RPTR shadow hw init programs */ +}; +static uint32_t ilog2(size_t value) +{ + uint32_t result=0;while(value>1) {value>>=1;++result;}return result; +} +IOReturn NativeAdrenoRing::poison() +{ + poisoned_=true;return kIOReturnIOError; +} +IOReturn NativeAdrenoRing::readRptr(uint32_t *value) +{ + uint32_t rptr=0; + if(io_.read32(io_.cookie,RingRptr,&rptr))return poison(); + if(rptr>=ring_bytes_/4)return poison(); /* unusable read pointer */ + *value=rptr;rptr_shadow_=rptr;return kIOReturnSuccess; +} +/* Packs one TYPE7 packet into words; returns the next free index or 0 when the header cannot be built. */ +static size_t packet(uint32_t *words,size_t at,uint32_t opcode,const uint32_t *payload,uint32_t count) +{ + if(AdrenoPM4Type7(opcode,count,&words[at])!=AdrenoPM4Success)return 0; + for(uint32_t i=0;i>32),seqno_},all=RingResetAll,lrz=RingLrzFlushInvalidate, + invalidate=RingCacheInvalidate; + if(n)n=packet(words,n,RingWaitTimestamp,wait,4); + if(n)n=packet(words,n,RingThreadControl,&syncBoth,1); + if(n)n=packet(words,n,RingResetContextState,&all,1); + if(n)n=packet(words,n,RingThreadControl,&syncBoth,1); + if(n)n=packet(words,n,RingFenceOpcode,&lrz,1); + if(n)n=packet(words,n,RingThreadControl,&br,1); + if(n && AdrenoPM4Type4(RingPerfctrSramInit,1,&words[n])==AdrenoPM4Success) {words[n+1]=1;n+=2;} else n=0; + if(n)n=packet(words,n,RingFenceOpcode,&invalidate,1); + const uint32_t sram[6]={RingWaitWriteEq,RingPerfctrSramInitStatus,0,1,1,0}; + if(n)n=packet(words,n,RingWaitRegMem,sram,6); + } + const uint32_t flush=RingCacheFlush7,invalidate7=RingCacheInvalidate7; + if(n)n=packet(words,n,RingFenceOpcode,&flush,1); + if(n)n=packet(words,n,RingFenceOpcode,&invalidate7,1); + if(n)n=packet(words,n,RingWaitForIdle,nullptr,0); + if(n)n=packet(words,n,RingThreadControl,&both,1); + if(n)n=packet(words,n,RingSetMarker,&marker,1); + if(n!=count)return poison(); + for(uint32_t i=0;i>32);words[6]=seqno; + for(uint32_t i=0;i>32),seqno}; + const uint32_t bvWait[4]={0,(uint32_t)bv,(uint32_t)(bv>>32),seqno}; + const uint32_t fence[4]={RingFenceEvent|RingFenceEnable|RingFenceIrq,(uint32_t)at4,(uint32_t)(at4>>32),seqno}; + size_t n=packet(words,2,RingThreadControl,&br,1); + if(n)n=packet(words,n,RingFenceOpcode,&depth,1); + if(n)n=packet(words,n,RingFenceOpcode,&color,1); + if(n)n=packet(words,n,RingThreadControl,&bvThread,1); + if(n)n=packet(words,n,RingFenceOpcode,bvWrite,4); + if(n)n=packet(words,n,RingThreadControl,&br,1); + if(n)n=packet(words,n,RingWaitTimestamp,bvWait,4); + if(n)n=packet(words,n,RingFenceOpcode,fence,4); + if(n)n=packet(words,n,RingThreadControl,&both,1); + if(n)n=packet(words,n,RingSetMarker,&marker,1); + if(n!=RingFramedFenceWords)return poison(); + for(uint32_t i=0;iwrite32 || !io->read32 || !io->barrier || !cache || + !cache->publish || !cache->acquire || !ring || !fence || + ring_bytes<8192 || ring_bytes>(1u<<20) || (ring_bytes&(ring_bytes-1)) || + ((ring_iova|ring_bytes)&(ring_bytes-1)) || (ring_iova&4095) || + fence_bytes<64 || (fence_iova&7) || fence_bytes>=(1u<<24))return kIOReturnBadArgument; + /* No register may be touched before every argument is acceptable. */ + io_=*io;cache_=*cache;ring_=static_cast(ring); + ring_bytes_=ring_bytes;ring_iova_=ring_iova; + fence_=static_cast(fence);fence_bytes_=fence_bytes;fence_iova_=fence_iova; + for(size_t i=0;i>32); + const uint32_t rptr_lo=(uint32_t)fence_iova_,rptr_hi=(uint32_t)(fence_iova_>>32); + const uint32_t writes[]={RingBase,base_lo,RingBase+1,base_hi,RingControl,control, + RingRptrAddress,rptr_lo,RingRptrAddress+1,rptr_hi,RingWptr,0,RingScratch,0}; + for(size_t i=0;iSIZE_MAX/sizeof(uint32_t))return kIOReturnBadArgument; + const uint32_t size_words=(uint32_t)(ring_bytes_/4),mask=size_words-1; + const bool reset=framed && reset_pending_; + const uint32_t extra=framed?RingPrologueWords+(reset?RingResetWords:0)+RingFramedFenceWords:RingFenceWords; + if(word_count>size_words-extra-8)return kIOReturnNoResources; + uint32_t rptr=0;IOReturn status=readRptr(&rptr); + if(status)return status; + const uint32_t used=(next_-rptr)&mask; + if(word_count+extra+8>size_words-used)return kIOReturnNoResources; + uint32_t at=next_; + if(framed) { + status=emitPrologue(at,reset,&at); + if(status)return status; + } + for(size_t i=0;iseqno_)return NativeAdrenoRingFenceArgument; + uint64_t start=clock && clock->now_us?clock->now_us(clock->cookie):0; + for(;;) { + if(cache_.acquire(cache_.cookie,fence_,fence_bytes_)) {poison();return NativeAdrenoRingFencePoisoned;} + const uint32_t value=fence_[1]; + if(observed)*observed=value; + if(value>seqno_) {poison();return NativeAdrenoRingFencePoisoned;} + if(value>=fence)return NativeAdrenoRingFenceComplete; + if(!clock || !clock->now_us)return NativeAdrenoRingFenceTimeout; + if(clock->now_us(clock->cookie)-start>=timeout_us)return NativeAdrenoRingFenceTimeout; + if(clock->idle && clock->idle(clock->cookie))return NativeAdrenoRingFenceTimeout; + } +} +NativeAdrenoRingFence NativeAdrenoRing::wait(uint32_t fence,const NativeAdrenoRingClock *clock, + uint32_t timeout_us,uint32_t *observed) +{ + if(observed)*observed=0; + if(!attached_ || poisoned_)return NativeAdrenoRingFencePoisoned; + if(!fence || fence>seqno_)return NativeAdrenoRingFenceArgument; + uint64_t start=clock && clock->now_us?clock->now_us(clock->cookie):0; + for(;;) { + if(cache_.acquire(cache_.cookie,fence_,fence_bytes_)) {poison();return NativeAdrenoRingFencePoisoned;} + const uint32_t value=fence_[1]; + if(observed)*observed=value; + if(value>=fence) {if(value>seqno_) {poison();return NativeAdrenoRingFencePoisoned;}completed_=value; + return NativeAdrenoRingFenceComplete;} + if(!clock || !clock->now_us)return NativeAdrenoRingFenceTimeout; + if(clock->now_us(clock->cookie)-start>=timeout_us)return NativeAdrenoRingFenceTimeout; + if(clock->idle && clock->idle(clock->cookie))return NativeAdrenoRingFenceTimeout; + } +} diff --git a/drivers/NativeAdreno/NativeAdrenoRing.h b/drivers/NativeAdreno/NativeAdrenoRing.h new file mode 100644 index 0000000..b7065c4 --- /dev/null +++ b/drivers/NativeAdreno/NativeAdrenoRing.h @@ -0,0 +1,100 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef NATIVE_ADRENO_RING_H +#define NATIVE_ADRENO_RING_H +#include +#include +#include + +/* A7XX command processor ring and timestamp fence, not a scheduler. + * Register indices are dword offsets; the platform adds the MMIO base and + * multiplies by four. Caller must already own an exclusive, powered GPU with + * SQE microcode loaded (CP_SQE_INSTR_BASE), global reset released and all + * masters quiesced while attach programs the ring. Ring and fence CPU + * addresses must be actual writable mappings of the same IOVAs that were + * mapped into the active native-4K GPU context. No MMIO base, IRQ, power, + * GMU, context switching or per-client validation is implemented here. + * Every submit publishes ring memory before advancing WPTR and completes + * only when the CP timestamp value is observed at the fence address. + * A failed WPTR write or cache publication is uncertain: the ring is poisoned + * and every resource must be retained until an independently verified reset. */ +struct NativeAdrenoRingIo { + void *cookie; + /* Return zero only on a completed access. */ + int (*write32)(void *,uint32_t reg,uint32_t value); + int (*read32)(void *,uint32_t reg,uint32_t *value); + int (*barrier)(void *); +}; +struct NativeAdrenoRingCache { + void *cookie; + /* Clean owned ring bytes to PoC before the device reads them. */ + int (*publish)(void *,const volatile uint32_t *words,size_t bytes); + /* Invalidate the fence page before the CPU reads CP-written values. */ + int (*acquire)(void *,volatile uint32_t *words,size_t bytes); +}; +struct NativeAdrenoRingClock { + void *cookie; + uint64_t (*now_us)(void *); + /* Optional; may sleep. Never used before a successful submit. */ + int (*idle)(void *); +}; +enum NativeAdrenoRingFence { + NativeAdrenoRingFenceComplete = 0, + NativeAdrenoRingFenceTimeout, + NativeAdrenoRingFencePoisoned, + NativeAdrenoRingFenceArgument +}; +class NativeAdrenoRing final { + NativeAdrenoRingIo io_{}; + NativeAdrenoRingCache cache_{}; + volatile uint32_t *ring_=nullptr,*fence_=nullptr; + size_t ring_bytes_=0,fence_bytes_=0; + uint64_t ring_iova_=0,fence_iova_=0; + uint32_t next_=0,rptr_shadow_=0,seqno_=0,completed_=0,submissions_=0; + bool attached_=false,poisoned_=false,reset_pending_=false; + IOReturn poison(); + IOReturn readRptr(uint32_t *); + IOReturn emitPrologue(uint32_t at,bool reset,uint32_t *end); + IOReturn emitFence(uint32_t at,uint32_t seqno,uint32_t *end); + IOReturn emitFramedFence(uint32_t at,uint32_t seqno,uint32_t *end); + IOReturn submitWords(const uint32_t *words,size_t word_count,uint32_t *fence,bool framed); +public: + NativeAdrenoRing()=default; + ~NativeAdrenoRing()=default; + NativeAdrenoRing(const NativeAdrenoRing &)=delete; + NativeAdrenoRing &operator=(const NativeAdrenoRing &)=delete; + /* ring_bytes is a power of two in [8KiB,1MiB]; ring_iova must be aligned + * to ring_bytes. fence_bytes >=64 and fence_iova 8-aligned. The fence page + * layout is: +0 CP read-pointer shadow, +4 completed timestamp, +8 spare. + * All outputs are prevalidated before any register write. */ + IOReturn attach(const NativeAdrenoRingIo *,const NativeAdrenoRingCache *, + void *ring,size_t ring_bytes,uint64_t ring_iova, + void *fence,size_t fence_bytes,uint64_t fence_iova); + /* Programs nothing. Refuses while submissions are unobserved or poisoned, + * because only the caller can establish real GPU quiescence and unmap. */ + IOReturn detach(); + /* Emit an entire command stream followed by the timestamp fence and commit + * it with one WPTR update. Returns a nonzero fence value on success. The + * words are copied; the caller's buffer is never retained. */ + IOReturn submit(const uint32_t *words,size_t word_count,uint32_t *fence); + /* User command buffers: the A7XX BR/BV thread framing of a7xx_submit() around the words (CP init and the kernel's own + * bring-up packets keep the plain submit() above, as Linux sends them outside a7xx_submit()). */ + IOReturn submitFramed(const uint32_t *words,size_t word_count,uint32_t *fence); + /* The next framed submission starts with the A7XX context entry of a6xx_set_pagetable() (BR/BV counters, on-chip + * timestamps and resource table cleared between thread syncs); a new user client requests it, as Linux does per context. */ + void requestContextReset() {reset_pending_=true;} + /* Polls the CP-written timestamp after invalidating the fence page. Never + * writes registers and never advances the ring. Timeout is not poison. */ + NativeAdrenoRingFence wait(uint32_t fence,const NativeAdrenoRingClock *, + uint32_t timeout_us,uint32_t *observed); + /* Observe a previously issued fence without retiring or accounting for it. + * Used before a new submission waits on an earlier batch; only fence values + * this ring issued are accepted. Never writes a register. */ + NativeAdrenoRingFence observeFence(uint32_t fence,const NativeAdrenoRingClock *, + uint32_t timeout_us,uint32_t *observed); + bool poisoned() const {return poisoned_;} + uint32_t outstanding() const {return seqno_-completed_;} + uint32_t completedFence() const {return completed_;} + uint32_t submissions() const {return submissions_;} + uint32_t readPointer() const {return rptr_shadow_;} +}; +#endif diff --git a/drivers/NativeAdreno/NativeAdrenoService.cpp b/drivers/NativeAdreno/NativeAdrenoService.cpp new file mode 100644 index 0000000..b81393c --- /dev/null +++ b/drivers/NativeAdreno/NativeAdrenoService.cpp @@ -0,0 +1,618 @@ +/* SPDX-License-Identifier: MIT */ +/* IOKit surface for the native Adreno backend. This file is compiled against + * the real source-kernel headers; it does not run on any host. start() parses + * the provider's device-tree reg windows through NativeAdrenoBoard and maps the + * first window, then stops: board power, GMU boot and SMMU context programming + * are not implemented for Darwin yet, so the service refuses to publish rather + * than expose a device that cannot execute. newUserClient and the external + * method table are present because they are fixed by the core ABI; no client + * can be created while the GPU is not ready. clientMemoryForType has no + * descriptor to return until a client owns a leased BO, so it refuses. */ +#include +#include +#include +#include +#include +#include +#include "NativeAdrenoBoard.h" +#include "NativeAdrenoFirmware.h" +#include "NativeAdrenoGmuConfig.h" +#include "NativeAdrenoGmuContext.h" +#include "NativeAdrenoGmuPlatform.h" +#include "NativeAdrenoGmuService.h" +#include "NativeAdrenoSmmuContext.h" +#include "NativeAdrenoIrq.h" +#include "NativeAdrenoKernelTime.h" +#include "NativeAdrenoCpInit.h" +#include "NativeAdrenoPwrup.h" +#include "NativeAdrenoRing.h" +#include "ShengGmuCommandDb.h" +#include "NativeAdrenoSmmuService.h" +#include +#if __has_include("ShengGmuSource.h") +#include "ShengGmuSource.h" +#define NATIVE_ADRENO_HAVE_GMU_RULES 1 +#endif +#if __has_include("ShengGmuVotesSource.h") +#include "ShengGmuVotesSource.h" +#define NATIVE_ADRENO_HAVE_GMU_VOTES 1 +#endif +#if __has_include("ShengGmuCommandDbSource.h") +#include "ShengGmuCommandDbSource.h" +#define NATIVE_ADRENO_HAVE_COMMAND_DB 1 +#endif +extern "C" const uint8_t native_adreno_firmware_image[] __attribute__((weak)); +extern "C" const uint64_t native_adreno_firmware_image_size __attribute__((weak)); +extern "C" const uint8_t native_adreno_sqe_firmware_image[] __attribute__((weak)); +extern "C" const uint64_t native_adreno_sqe_firmware_image_size __attribute__((weak)); +#include "NativeAdrenoUserClient.h" + +class NativeAdrenoGpuClient; +static IOUserClient *createGpuClient(task_t,void *,UInt32,OSDictionary *,NativeAdrenoUserClient *); + +class NativeAdrenoGpuService : public IOService { + OSDeclareDefaultStructors(NativeAdrenoGpuService) + OSSharedPtr maps_[2]; + NativeAdrenoGmuContext gmu_ctx_; + NativeAdrenoGmuPlatform gmu_platform_; + ShengGmuIo gmu_io_{}; + ShengGmuDevice gmu_device_{}; + ShengGmuStartupResult gmu_startup_{}; + NativeAdrenoFirmwareView firmware_{}; + bool gmu_io_bound_=false,firmware_valid_=false,ready_=false; + /* Set only once a separate GPU-stream context exists; until then the CP path below is refused. */ + bool gpu_context_ready_=false; + NativeAdrenoMemory ring_mem_,fence_mem_,sqe_mem_,pwrup_mem_; + NativeAdrenoRing ring_; + uint64_t ring_map_=0,fence_map_=0,sqe_map_=0,pwrup_map_=0; + NativeAdrenoIrq irq_; + NativeAdrenoSubmit *execbuf_=nullptr; + NativeAdrenoUserClient *core_=nullptr; + bool core_ready_=false; + struct RingCache {NativeAdrenoMemory *ring;NativeAdrenoMemory *fence;}; + RingCache ring_cache_{}; + static int gpu_read32(void *,uint32_t,uint32_t *); + static int gpu_write32(void *,uint32_t,uint32_t); + static int gpu_barrier(void *); + static int ring_publish(void *,const volatile uint32_t *,size_t); + static int fence_acquire(void *,volatile uint32_t *,size_t); + static uint64_t ring_now(void *); +public: + IOService *probe(IOService *provider,SInt32 *score) APPLE_KEXT_OVERRIDE; + bool start(IOService *provider) APPLE_KEXT_OVERRIDE; + void stop(IOService *provider) APPLE_KEXT_OVERRIDE; + IOReturn newUserClient(task_t owningTask,void *securityID,UInt32 type, + OSDictionary *properties,IOUserClient **handler) APPLE_KEXT_OVERRIDE; +}; +OSDefineMetaClassAndStructors(NativeAdrenoGpuService,IOService) + +static bool advertisesAdreno(IOService *provider) +{ + OSObject *property=provider?provider->getProperty("compatible"):nullptr; + OSData *data=OSDynamicCast(OSData,property); + if(!data || !data->getBytesNoCopy() || !data->getLength())return false; + const char *text=static_cast(data->getBytesNoCopy()); + const size_t length=data->getLength(); + static const char needle[]="qcom,adreno"; + for(size_t at=0;at+sizeof(needle)-1<=length;++at) { + size_t index=0; + while(index(cookie); + if(!window || !value || !window->base || (at&3) || at+4>window->bytes)return 1; + *value=*reinterpret_cast(window->base+at); + return 0; +} +static int smmu_read64(void *cookie,size_t at,uint64_t *value) +{ + NativeAdrenoSmmuWindow *window=static_cast(cookie); + if(!window || !value || !window->base || (at&7) || at+8>window->bytes)return 1; + *value=*reinterpret_cast(window->base+at); + return 0; +} +static int smmu_write32(void *cookie,size_t at,uint32_t value) +{ + NativeAdrenoSmmuWindow *window=static_cast(cookie); + if(!window || !window->base || (at&3) || at+4>window->bytes)return 1; + *reinterpret_cast(window->base+at)=value; + return 0; +} +static int smmu_write64(void *cookie,size_t at,uint64_t value) +{ + NativeAdrenoSmmuWindow *window=static_cast(cookie); + if(!window || !window->base || (at&7) || at+8>window->bytes)return 1; + *reinterpret_cast(window->base+at)=value; + return 0; +} +static int smmu_barrier(void *cookie) +{ + if(!cookie)return 1; + __atomic_thread_fence(__ATOMIC_SEQ_CST); + return 0; +} +struct NativeAdrenoGpuWindow { + volatile uint8_t *base; + size_t bytes; +}; +int NativeAdrenoGpuService::gpu_read32(void *cookie,uint32_t reg,uint32_t *value) +{ + NativeAdrenoGpuWindow *window=static_cast(cookie); + const size_t at=size_t(reg)*4; + if(!window || !value || !window->base || at+4>window->bytes)return 1; + *value=*reinterpret_cast(window->base+at); + return 0; +} +int NativeAdrenoGpuService::gpu_write32(void *cookie,uint32_t reg,uint32_t value) +{ + NativeAdrenoGpuWindow *window=static_cast(cookie); + const size_t at=size_t(reg)*4; + if(!window || !window->base || at+4>window->bytes)return 1; + *reinterpret_cast(window->base+at)=value; + return 0; +} +int NativeAdrenoGpuService::gpu_barrier(void *cookie) +{ + if(!cookie)return 1; + __atomic_thread_fence(__ATOMIC_SEQ_CST); + return 0; +} +int NativeAdrenoGpuService::ring_publish(void *cookie,const volatile uint32_t *words,size_t bytes) +{ + NativeAdrenoGpuService::RingCache *cache=static_cast(cookie); + return cache && cache->ring && cache->ring->publishForDevice(0,bytes,false)==kIOReturnSuccess?0:1; +} +int NativeAdrenoGpuService::fence_acquire(void *cookie,volatile uint32_t *words,size_t bytes) +{ + NativeAdrenoGpuService::RingCache *cache=static_cast(cookie); + return cache && cache->fence && cache->fence->acquireDeviceWrites(0,bytes)==kIOReturnSuccess?0:1; +} +uint64_t NativeAdrenoGpuService::ring_now(void *) +{ + uint64_t microseconds=0; + NativeAdrenoKernelTime::now_us(nullptr,µseconds); + return microseconds; +} +IOService *NativeAdrenoGpuService::probe(IOService *provider,SInt32 *score) +{ + if(!advertisesAdreno(provider))return nullptr; + if(score)*score=1000; + return this; +} +bool NativeAdrenoGpuService::start(IOService *provider) +{ + if(!IOService::start(provider))return false; + OSData *reg=OSDynamicCast(OSData,provider->getProperty("reg")); + OSData *address=OSDynamicCast(OSData,provider->getProperty("#address-cells")); + OSData *size=OSDynamicCast(OSData,provider->getProperty("#size-cells")); + NativeAdrenoWindow windows[NativeAdrenoMaxWindows]{}; + size_t count=0; + uint32_t address_cells=2,size_cells=2; + if(address && address->getLength()>=4) + address_cells=*static_cast(address->getBytesNoCopy()); + if(size && size->getLength()>=4) + size_cells=*static_cast(size->getBytesNoCopy()); + if(!reg || NativeAdrenoBoard::parseReg( + static_cast(reg->getBytesNoCopy()),reg->getLength(), + address_cells,size_cells,windows,NativeAdrenoMaxWindows,&count)!=kIOReturnSuccess) { + IOLog("NativeAdreno: provider reg windows are not usable\n"); + IOService::stop(provider); + return false; + } + if(count<2) { + IOLog("NativeAdreno: GPU node needs the register and CX windows\n"); + IOService::stop(provider); + return false; + } + for(unsigned index=0;index<2;++index) { + IODeviceMemory *memory=getDeviceMemoryWithIndex(index); + if(!memory) { + IOLog("NativeAdreno: provider window %u has no device memory\n",index); + stop(provider); + return false; + } + maps_[index]=OSSharedPtr(memory->map(kIOMapAnywhere|kIOMapInhibitCache), + OSNoRetain); + if(!maps_[index]) { + IOLog("NativeAdreno: GPU window %u did not map\n",index); + stop(provider); + return false; + } + } + IOLog("NativeAdreno: %lu windows, GPU window at 0x%llx/0x%llx mapped\n", + static_cast(count), + static_cast(windows[0].base), + static_cast(windows[0].bytes)); + /* The GMU node publishes its bases so no DT phandle resolution is needed. + * Discovery does not imply readiness: bring-up still needs board power, + * firmware and the SMMU context before any register may be touched. */ + OSDictionary *matching=IOService::serviceMatching("NativeAdrenoGmuService"); + IOService *gmu=matching?IOService::copyMatchingService(matching):nullptr; + if(matching)matching->release(); + if(gmu) { + NativeAdrenoGmuService *node=OSDynamicCast(NativeAdrenoGmuService,gmu); + const unsigned published=node?static_cast(node->windowCount()):0; + IOLog("NativeAdreno: GMU node present with %u windows\n",published); + if(node && published>=3) { + NativeAdrenoGmuWindow gmu_windows[ShengGmuWindows]{}; + for(unsigned index=0;index<3;++index) { + gmu_windows[index].base=node->windowAddress(index); + gmu_windows[index].bytes=static_cast(node->windowBytes(index)); + } + gmu_windows[ShengGmuGpu].base= + reinterpret_cast(maps_[0]->getVirtualAddress()); + gmu_windows[ShengGmuGpu].bytes=static_cast(windows[0].bytes); + gmu_windows[ShengGmuCxMisc].base= + reinterpret_cast(maps_[1]->getVirtualAddress()); + gmu_windows[ShengGmuCxMisc].bytes=static_cast(windows[1].bytes); + if(gmu_platform_.bind(gmu_windows,&gmu_io_)==kIOReturnSuccess) { + gmu_io_bound_=true; + IOLog("NativeAdreno: GMU five-window I/O adapter bound\n"); + } else + IOLog("NativeAdreno: GMU windows are not usable\n"); + } + gmu->release(); + } else { + IOLog("NativeAdreno: GMU node not published yet\n"); + } + /* Stream IDs are this node's iommus; the specifier width is published by + * the SMMU node. Nothing is assumed when either is missing. */ + OSDictionary *smmu_match=IOService::serviceMatching("NativeAdrenoSmmuService"); + IOService *smmu=smmu_match?IOService::copyMatchingService(smmu_match):nullptr; + if(smmu_match)smmu_match->release(); + if(smmu) { + OSNumber *cells=OSDynamicCast(OSNumber,smmu->getProperty("iommu-cells")); + OSData *iommus=OSDynamicCast(OSData,provider->getProperty("iommus")); + uint16_t sids[8]{}; + size_t sid_count=0; + if(cells && iommus && NativeAdrenoBoard::parseIommus( + static_cast(iommus->getBytesNoCopy()),iommus->getLength(), + cells->unsigned32BitValue(),sids,8,&sid_count)==kIOReturnSuccess) + IOLog("NativeAdreno: %lu stream IDs from iommus, first %u\n", + static_cast(sid_count),sid_count?sids[0]:0); + else + IOLog("NativeAdreno: iommus are not usable\n"); + smmu->release(); + } else { + IOLog("NativeAdreno: SMMU node not published yet\n"); + } + /* Explicit opt-in (sheng-gpu-bringup=1). Without the bank/asid/sid/slot arguments the SMMU window is only read + * (identity and fault registers). With all four arguments the sequence below PROGRAMS the SMMU: it prepares a context + * bank, attaches the stream and maps the GMU payloads. Do not run it before the read-only power/clock observation + * (sheng-gpu-power, docs/gpu-hardware-plan.md) shows the GPU SMMU powered and writable: the apps SMMU on this board is + * hypervisor-managed (docs/native-ufs.md) and the adreno SMMU has not been shown to differ. */ + uint32_t bringup=0; + if(PE_parse_boot_argn("sheng-gpu-bringup",&bringup,sizeof(bringup)) && bringup) { + NativeAdrenoSmmuService *node=OSDynamicCast(NativeAdrenoSmmuService,smmu); + if(node && node->windowCount() && node->windowAddress(0)) { + volatile uint8_t *smmu_base=node->windowAddress(0); + const size_t smmu_bytes=static_cast(node->windowBytes(0)); + const uint32_t id0=*reinterpret_cast(smmu_base+0x20); + const uint32_t id1=*reinterpret_cast(smmu_base+0x24); + const uint32_t id2=*reinterpret_cast(smmu_base+0x28); + const uint32_t scr0=*reinterpret_cast(smmu_base+0x0); + const uint32_t fault=*reinterpret_cast(smmu_base+0x48); + IOLog("NativeAdreno: SMMU id0=%08x id1=%08x id2=%08x scr0=%08x fault=%08x window=%lu\n", + id0,id1,id2,scr0,fault,static_cast(smmu_bytes)); + uint32_t bank=0,asid=0,sid=0,slot=0; + const bool configured= + PE_parse_boot_argn("sheng-gpu-bank",&bank,sizeof(bank)) && + PE_parse_boot_argn("sheng-gpu-asid",&asid,sizeof(asid)) && + PE_parse_boot_argn("sheng-gpu-sid",&sid,sizeof(sid)) && + PE_parse_boot_argn("sheng-gpu-slot",&slot,sizeof(slot)); + if(!configured || bank>63 || !asid || asid>0xffff || sid>0xffff || slot>127) { + IOLog("NativeAdreno: bring-up needs bank/asid/sid/slot arguments\n"); + } else { + /* The GMU context at its fixed a6xx_gmu.c addresses (NativeAdrenoGmuContext). The ring, fence, + * SQE and power-up list are read by the CP through the GPU's own streams (iommus 0/1) and need + * the GPU context, which is not built yet; they are never placed in the GMU context. */ + const size_t ring_bytes=0x8000,fence_bytes=0x1000; + const uint64_t ring_iova=0,fence_iova=0,sqe_iova=0,pwrup_iova=0; + const uint64_t sqe_bytes=(reinterpret_cast(&native_adreno_sqe_firmware_image_size)&& + reinterpret_cast(native_adreno_sqe_firmware_image))? + ((native_adreno_sqe_firmware_image_size+0xfff)&~UINT64_C(0xfff)):0; + NativeAdrenoSmmuWindow window{smmu_base,smmu_bytes}; + ArmSmmu4KIo smmu_io{&window,smmu_bytes,smmu_read32,smmu_read64, + smmu_write32,smmu_write64,smmu_barrier}; + NativeAdrenoGmuContextReport report{}; + IOReturn status=gmu_ctx_.bringUp(&smmu_io,bank,asid,slot,uint16_t(sid),nullptr,&report); + NativeAdrenoGmuAllocator &gmu_=gmu_ctx_.allocator(); + NativeAdrenoVm &vm_=gmu_ctx_.vm(); + if(status) { + IOLog("NativeAdreno: GMU context stopped at step %u status 0x%x; resources retained\n", + report.step,status); + } else { + IOLog("NativeAdreno: GMU payloads mapped, bank=%u asid=%u sid=%u slot=%u ttbr0=0x%llx\n", + bank,asid,sid,slot,static_cast(report.ttbr0)); +#ifdef NATIVE_ADRENO_HAVE_GMU_RULES + if(reinterpret_cast(native_adreno_firmware_image) && + reinterpret_cast(&native_adreno_firmware_image_size)) { + NativeAdrenoFirmwareRules rules{}; + rules.bytes=SHENG_GMU_FIRMWARE_BYTES; + for(unsigned i=0;i<32;++i)rules.sha256[i]=sheng_gmu_firmware_sha256[i]; + const IOReturn checked=NativeAdrenoFirmware::validate( + native_adreno_firmware_image,native_adreno_firmware_image_size, + &rules,&firmware_); + if(checked) + IOLog("NativeAdreno: embedded GMU firmware failed validation 0x%x\n", + checked); + else { + firmware_valid_=true; + IOLog("NativeAdreno: embedded GMU firmware validated (%lu bytes)\n", + static_cast(firmware_.length)); + } + } else { + IOLog("NativeAdreno: no embedded GMU firmware in this image\n"); + } +#else + IOLog("NativeAdreno: GMU rules not generated for this build\n"); +#endif +#ifdef NATIVE_ADRENO_HAVE_COMMAND_DB + uint32_t version=0; + if(!firmware_valid_ || !gmu_io_bound_) { + IOLog("NativeAdreno: GMU boot needs firmware and bound windows\n"); + } else if(!PE_parse_boot_argn("sheng-gpu-cmddb-version",&version, + sizeof(version)) || !version) { + IOLog("NativeAdreno: GMU boot needs sheng-gpu-cmddb-version\n"); + } else { + IODeviceMemory *db=IODeviceMemory::withRange( + SHENG_GMU_COMMAND_DB_BASE,SHENG_GMU_COMMAND_DB_BYTES); + if(!db) { + IOLog("NativeAdreno: command-db reservation did not map\n"); + } else { + OSSharedPtr db_map(db->map( + kIOMapAnywhere|kIOMapInhibitCache),OSNoRetain); + db->release(); + if(!db_map) { + IOLog("NativeAdreno: command-db mapping failed\n"); + } else { + ShengGmuVoteInputs inputs{}; + const ShengGmuDbStatus parsed=ShengGmuCommandDbInputs( + reinterpret_cast(static_cast(db_map->getVirtualAddress())), + SHENG_GMU_COMMAND_DB_BYTES,version,&inputs); + if(parsed!=ShengGmuDbSuccess) { + IOLog("NativeAdreno: command-db refused (%d)\n",parsed); + } else { +#ifdef NATIVE_ADRENO_HAVE_GMU_VOTES + inputs.chip_id=SHENG_GMU_VOTES_CHIP_ID; + inputs.enable_acd=1;inputs.enable_ifpc=1; + AdrenoHfiStartupConfig startup{}; + ShengGmuConfig config{}; + NativeAdrenoGmuTime time{nullptr, + NativeAdrenoKernelTime::now_us, + NativeAdrenoKernelTime::wait_us}; + ShengGmuServices services{}; + if(ShengGmuBuildStartup(&inputs,&startup)!=AdrenoHfiStartupSuccess) + IOLog("NativeAdreno: startup config refused\n"); + else if(NativeAdrenoAssembleGmuConfig(&firmware_,&gmu_, + SHENG_GMU_VOTES_CHIP_ID,1000,&config)!=kIOReturnSuccess) + IOLog("NativeAdreno: GMU config assembly refused\n"); + else if(NativeAdrenoGmuPlatform::services(&time,&services)) + IOLog("NativeAdreno: GMU services refused\n"); + else { + ShengGmuStartupLimits limits{4,100,1000,10000}; + const ShengGmuStatus started=ShengGmuColdStart( + &gmu_device_,&config,&startup,&limits,&gmu_startup_); + IOLog("NativeAdreno: ColdStart status=%d fw_ready=%u hfi_ready=%u stage=%u\n", + started,gmu_startup_.load.firmware_ready_observed, + gmu_startup_.load.hfi_ready_observed, + gmu_startup_.protocol.stage); + if(started==ShengGmuSuccess && !gpu_context_ready_) { + IOLog("NativeAdreno: ring/SQE refused: the GPU context (iommus 0/1) is not built\n"); + } else if(started==ShengGmuSuccess) { + IOReturn ring_status=ring_mem_.allocate(ring_bytes,40,true,ring_bytes); + if(!ring_status)ring_status=fence_mem_.allocate(fence_bytes,40,true,fence_bytes); + if(!ring_status && sqe_bytes) + ring_status=sqe_mem_.allocate(size_t(sqe_bytes),40,true,size_t(sqe_bytes)); + if(!ring_status)ring_status=vm_.map(&ring_mem_,1,ring_iova,3,0,2,&ring_map_); + if(!ring_status)ring_status=vm_.map(&fence_mem_,1,fence_iova,3,0,2,&fence_map_); + if(!ring_status)ring_status=vm_.map(&sqe_mem_,1,sqe_iova,3,0,2,&sqe_map_); + if(ring_status) { + IOLog("NativeAdreno: ring/SQE setup stopped at 0x%x\n",ring_status); + } else { + NativeAdrenoGpuWindow gpu_window={ + reinterpret_cast(maps_[0]->getVirtualAddress()), + static_cast(windows[0].bytes)}; + NativeAdrenoRingIo ring_io{&gpu_window,gpu_write32,gpu_read32,gpu_barrier}; + ring_cache_={&ring_mem_,&fence_mem_}; + NativeAdrenoRingCache cache{&ring_cache_,ring_publish,fence_acquire}; + ring_status=ring_.attach(&ring_io,&cache,ring_mem_.mapping(), + ring_bytes,ring_iova,fence_mem_.mapping(),fence_bytes,fence_iova); + if(!ring_status && sqe_bytes) { + memcpy(sqe_mem_.mapping(),native_adreno_sqe_firmware_image, + size_t(native_adreno_sqe_firmware_image_size)); + ring_status=sqe_mem_.publishForDevice(0,size_t(sqe_bytes),false); + } + if(ring_status) + IOLog("NativeAdreno: ring attach stopped at 0x%x\n",ring_status); + else + IOLog("NativeAdreno: ring attached ring_iova=0x%llx fence_iova=0x%llx sqe_iova=0x%llx\n", + static_cast(ring_iova), + static_cast(fence_iova), + static_cast(sqe_iova)); + } + if(!ring_status) { + NativeAdrenoGpuWindow cp_window={ + reinterpret_cast(maps_[0]->getVirtualAddress()), + static_cast(windows[0].bytes)}; + NativeAdrenoRingIo cp_io{&cp_window,gpu_write32,gpu_read32,gpu_barrier}; + ring_status=pwrup_mem_.allocate(0x1000,40,true,0x1000); + if(!ring_status)ring_status=vm_.map(&pwrup_mem_,1, + pwrup_iova,3,0,2,&pwrup_map_); + size_t list_bytes=0; + if(!ring_status)ring_status=NativeAdrenoPwrupList::buildA740( + gpu_read32,&cp_window, + static_cast(pwrup_mem_.mapping()), + 0x1000,&list_bytes); + if(!ring_status)ring_status=pwrup_mem_.publishForDevice(0,0x1000,false); + if(!ring_status) { + NativeAdrenoCpInit cp; + NativeAdrenoCpInitConfig cfg{sqe_iova,pwrup_iova,0}; + uint32_t start_fence=0; + ring_status=cp.run(&cp_io,&ring_,&cfg,&start_fence); + if(!ring_status) { + NativeAdrenoRingClock clock{nullptr,ring_now,nullptr}; + ring_status=ring_.wait(start_fence,&clock,2000,nullptr) + ==NativeAdrenoRingFenceComplete? + kIOReturnSuccess:kIOReturnTimeout; + } + if(!ring_status) { + ring_status=irq_.attach(&cp_io); + if(!ring_status)ring_status=irq_.programMask(nullptr); + } + if(ring_status) + IOLog("NativeAdreno: CP start stopped at 0x%x\n",ring_status); + else + { + execbuf_=new NativeAdrenoSubmit; + core_=new NativeAdrenoUserClient; + if(execbuf_ && core_ && + execbuf_->attach(&vm_,&ring_)==kIOReturnSuccess && + core_->open(NativeAdrenoAbiVersion,&vm_,execbuf_, + &ring_,32,40)==kIOReturnSuccess) + core_ready_=true; + ready_=true; + IOLog("NativeAdreno: CP started, fence=%u, interrupts masked=%08x\n", + start_fence,NativeAdrenoIrq::mask()); + } + } + } + } + } +#else + IOLog("NativeAdreno: vote rules not generated for this build\n"); +#endif + } + } + } + } +#else + IOLog("NativeAdreno: command-db reservation not generated for this build\n"); +#endif + } + } + } + } + if(ready_) { + IOLog("NativeAdreno: GPU ready, publishing service\n"); + registerService(); + return true; + } + IOLog("NativeAdreno: refusing to publish without a started CP\n"); + IOService::stop(provider); + return false; +} +void NativeAdrenoGpuService::stop(IOService *provider) +{ + for(auto &map:maps_)map.reset(); + IOService::stop(provider); +} +IOReturn NativeAdrenoGpuService::newUserClient(task_t owningTask,void *securityID, + UInt32 type,OSDictionary *properties,IOUserClient **handler) +{ + if(handler)*handler=nullptr; + if(!core_ready_)return kIOReturnNotReady; + IOUserClient *client=createGpuClient(owningTask,securityID,type,properties,core_); + if(!client)return kIOReturnNoResources; + if(handler)*handler=client; + return kIOReturnSuccess; +} + +class NativeAdrenoGpuClient : public IOUserClient { + OSDeclareDefaultStructors(NativeAdrenoGpuClient) + NativeAdrenoUserClient *core_=nullptr; + task_t task_=TASK_NULL; + static IOReturn action(OSObject *target,void *reference, + IOExternalMethodArguments *arguments); +public: + void setCore(NativeAdrenoUserClient *core) {core_=core;} + bool initWithTask(task_t owningTask,void *securityID,UInt32 type, + OSDictionary *properties) APPLE_KEXT_OVERRIDE; + IOReturn clientClose() APPLE_KEXT_OVERRIDE; + IOReturn clientMemoryForType(UInt32 type,IOOptionBits *options, + IOMemoryDescriptor **memory) APPLE_KEXT_OVERRIDE; + IOReturn externalMethod(uint32_t selector,IOExternalMethodArguments *arguments, + IOExternalMethodDispatch *dispatch,OSObject *target, + void *reference) APPLE_KEXT_OVERRIDE; +}; +OSDefineMetaClassAndStructors(NativeAdrenoGpuClient,IOUserClient) + +bool NativeAdrenoGpuClient::initWithTask(task_t owningTask,void *securityID, + UInt32 type,OSDictionary *properties) +{ + if(!IOUserClient::initWithTask(owningTask,securityID,type,properties))return false; + task_=owningTask; + return true; +} +IOReturn NativeAdrenoGpuClient::clientClose() +{ + core_=nullptr;task_=TASK_NULL; + terminate(); + return kIOReturnSuccess; +} +IOReturn NativeAdrenoGpuClient::clientMemoryForType(UInt32 type,IOOptionBits *options, + IOMemoryDescriptor **memory) +{ + if(memory)*memory=nullptr; + if(options)*options=kIOMapAnywhere; + /* The type is a BO mapping token; no client can hold one before the GPU + * and its core client exist, and the descriptor handoff is not written. */ + return kIOReturnUnsupported; +} +IOReturn NativeAdrenoGpuClient::action(OSObject *target,void *reference, + IOExternalMethodArguments *arguments) +{ + NativeAdrenoGpuClient *client=OSDynamicCast(NativeAdrenoGpuClient,target); + if(!client || !arguments)return kIOReturnBadArgument; + if(!client->core_)return kIOReturnNotReady; + (void)reference; + /* Waits need a real time base, supplied by the service once bring-up exists. */ + return client->core_->dispatch(arguments->selector,arguments->structureInput, + arguments->structureInputSize,arguments->structureOutput, + arguments->structureOutputSize,nullptr); +} +IOReturn NativeAdrenoGpuClient::externalMethod(uint32_t selector, + IOExternalMethodArguments *arguments,IOExternalMethodDispatch *dispatch, + OSObject *target,void *reference) +{ + /* One entry per ABI method; sizes are the exact wire records. */ + static const IOExternalMethodDispatch table[]={ + {&action,0,0,0,sizeof(NativeAdrenoAbiParams)}, /* Query */ + {&action,0,sizeof(NativeAdrenoAbiBoAlloc),0,sizeof(NativeAdrenoAbiBoAllocReply)}, + {&action,0,sizeof(NativeAdrenoAbiBoMap),0,sizeof(NativeAdrenoAbiBoMapReply)}, + {&action,0,sizeof(NativeAdrenoAbiBoFree),0,0}, + {&action,0,sizeof(NativeAdrenoAbiSubmit),0,sizeof(NativeAdrenoAbiSubmitReply)}, + {&action,0,sizeof(NativeAdrenoAbiWaitFence),0,0}, + {&action,0,0,0,sizeof(NativeAdrenoAbiTimestamp)}, + {&action,0,0,0,sizeof(NativeAdrenoAbiStatus)}, + {&action,0,sizeof(NativeAdrenoAbiMetadata),0,0}, + {&action,0,sizeof(NativeAdrenoAbiMetadata),0,sizeof(NativeAdrenoAbiMetadata)}, + {&action,0,sizeof(NativeAdrenoAbiQueue),0,sizeof(NativeAdrenoAbiQueueReply)}, + {&action,0,sizeof(NativeAdrenoAbiQueue),0,0}, + {&action,0,sizeof(NativeAdrenoAbiBoCpuMap),0,sizeof(NativeAdrenoAbiBoCpuMapReply)}, + {&action,0,sizeof(NativeAdrenoAbiBoCpuUnmap),0,0}}; + if(selector<1 || selector>sizeof(table)/sizeof(table[0]))return kIOReturnUnsupported; + return IOUserClient::externalMethod(selector,arguments, + const_cast(&table[selector-1]),target?target:this,reference); +} + +static IOUserClient *createGpuClient(task_t owningTask,void *securityID,UInt32 type, + OSDictionary *properties,NativeAdrenoUserClient *core) +{ + NativeAdrenoGpuClient *client=new NativeAdrenoGpuClient; + if(!client)return nullptr; + if(!client->initWithTask(owningTask,securityID,type,properties)) { + client->release(); + return nullptr; + } + client->setCore(core); + return client; +} diff --git a/drivers/NativeAdreno/NativeAdrenoSha256.cpp b/drivers/NativeAdreno/NativeAdrenoSha256.cpp new file mode 100644 index 0000000..ecef8ad --- /dev/null +++ b/drivers/NativeAdreno/NativeAdrenoSha256.cpp @@ -0,0 +1,63 @@ +/* SPDX-License-Identifier: MIT */ +#include "NativeAdrenoSha256.h" +#include + +static const uint32_t K[64]={ + 0x428a2f98u,0x71374491u,0xb5c0fbcfu,0xe9b5dba5u,0x3956c25bu,0x59f111f1u, + 0x923f82a4u,0xab1c5ed5u,0xd807aa98u,0x12835b01u,0x243185beu,0x550c7dc3u, + 0x72be5d74u,0x80deb1feu,0x9bdc06a7u,0xc19bf174u,0xe49b69c1u,0xefbe4786u, + 0x0fc19dc6u,0x240ca1ccu,0x2de92c6fu,0x4a7484aau,0x5cb0a9dcu,0x76f988dau, + 0x983e5152u,0xa831c66du,0xb00327c8u,0xbf597fc7u,0xc6e00bf3u,0xd5a79147u, + 0x06ca6351u,0x14292967u,0x27b70a85u,0x2e1b2138u,0x4d2c6dfcu,0x53380d13u, + 0x650a7354u,0x766a0abbu,0x81c2c92eu,0x92722c85u,0xa2bfe8a1u,0xa81a664bu, + 0xc24b8b70u,0xc76c51a3u,0xd192e819u,0xd6990624u,0xf40e3585u,0x106aa070u, + 0x19a4c116u,0x1e376c08u,0x2748774cu,0x34b0bcb5u,0x391c0cb3u,0x4ed8aa4au, + 0x5b9cca4fu,0x682e6ff3u,0x748f82eeu,0x78a5636fu,0x84c87814u,0x8cc70208u, + 0x90befffau,0xa4506cebu,0xbef9a3f7u,0xc67178f2u}; +static inline uint32_t rotr(uint32_t value,unsigned count) +{return (value>>count)|(value<<(32-count));} +static void compress(uint32_t state[8],const uint8_t block[64]) +{ + uint32_t w[64]; + for(unsigned i=0;i<16;++i) + w[i]=(uint32_t(block[i*4])<<24)|(uint32_t(block[i*4+1])<<16)| + (uint32_t(block[i*4+2])<<8)|uint32_t(block[i*4+3]); + for(unsigned i=16;i<64;++i) { + const uint32_t s0=rotr(w[i-15],7)^rotr(w[i-15],18)^(w[i-15]>>3); + const uint32_t s1=rotr(w[i-2],17)^rotr(w[i-2],19)^(w[i-2]>>10); + w[i]=w[i-16]+s0+w[i-7]+s1; + } + uint32_t a=state[0],b=state[1],c=state[2],d=state[3]; + uint32_t e=state[4],f=state[5],g=state[6],h=state[7]; + for(unsigned i=0;i<64;++i) { + const uint32_t s1=rotr(e,6)^rotr(e,11)^rotr(e,25); + const uint32_t ch=(e&f)^(~e&g); + const uint32_t t1=h+s1+ch+K[i]+w[i]; + const uint32_t s0=rotr(a,2)^rotr(a,13)^rotr(a,22); + const uint32_t maj=(a&b)^(a&c)^(b&c); + const uint32_t t2=s0+maj; + h=g;g=f;f=e;e=d+t1;d=c;c=b;b=a;a=t1+t2; + } + state[0]+=a;state[1]+=b;state[2]+=c;state[3]+=d; + state[4]+=e;state[5]+=f;state[6]+=g;state[7]+=h; +} +void NativeAdrenoSha256::hash(const uint8_t *bytes,size_t length,uint8_t digest[32]) +{ + uint32_t state[8]={0x6a09e667u,0xbb67ae85u,0x3c6ef372u,0xa54ff53au, + 0x510e527fu,0x9b05688cu,0x1f83d9abu,0x5be0cd19u}; + size_t at=0; + while(length-at>=64) {compress(state,bytes+at);at+=64;} + uint8_t tail[64]; + const size_t rest=length-at; + memset(tail,0,sizeof(tail)); + if(rest)memcpy(tail,bytes+at,rest); + tail[rest]=0x80; + if(rest>=56) {compress(state,tail);memset(tail,0,sizeof(tail));} + const uint64_t bits=uint64_t(length)*8; + for(unsigned i=0;i<8;++i)tail[63-i]=uint8_t(bits>>(8*i)); + compress(state,tail); + for(unsigned i=0;i<8;++i) { + digest[i*4]=uint8_t(state[i]>>24);digest[i*4+1]=uint8_t(state[i]>>16); + digest[i*4+2]=uint8_t(state[i]>>8);digest[i*4+3]=uint8_t(state[i]); + } +} diff --git a/drivers/NativeAdreno/NativeAdrenoSha256.h b/drivers/NativeAdreno/NativeAdrenoSha256.h new file mode 100644 index 0000000..fab08e8 --- /dev/null +++ b/drivers/NativeAdreno/NativeAdrenoSha256.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef NATIVE_ADRENO_SHA256_H +#define NATIVE_ADRENO_SHA256_H +#include +#include +/* FIPS 180-4 SHA-256 for firmware and manifest validation inside the kernel. + * One-shot only, no allocation, no I/O. Inputs may be empty; a null pointer is + * accepted only for zero bytes. The digest is big-endian as specified. */ +class NativeAdrenoSha256 final { + NativeAdrenoSha256()=delete; +public: + static void hash(const uint8_t *bytes,size_t length,uint8_t digest[32]); +}; +#endif diff --git a/drivers/NativeAdreno/NativeAdrenoSmmuContext.cpp b/drivers/NativeAdreno/NativeAdrenoSmmuContext.cpp new file mode 100644 index 0000000..c9ae161 --- /dev/null +++ b/drivers/NativeAdreno/NativeAdrenoSmmuContext.cpp @@ -0,0 +1,59 @@ +/* SPDX-License-Identifier: MIT */ +#include "NativeAdrenoSmmuContext.h" +#include +IOReturn NativeAdrenoSmmuContext::prepare(NativeAdrenoMemory *tables,unsigned upper,uint64_t iova, + size_t bytes,size_t leaf_budget,unsigned bank,unsigned asid,unsigned poll_budget,unsigned output_bits) +{ + if(prepared_ || poisoned_)return kIOReturnBusy; + if(!tables || upper>1 || !bytes || (iova&4095) || !leaf_budget || leaf_budget>MaxLeaves || + bank>63 || asid>0xffff || !poll_budget || poll_budget>4096) + return kIOReturnBadArgument; + const IOReturn prepared=NativeAdrenoPrepareSmmuMemory(tables,32,output_bits,iova,bytes,upper,leaf_budget, + &arena_,&state_,&visibility_); + if(prepared!=kIOReturnSuccess)return prepared==kIOReturnNoResources?prepared:kIOReturnBadArgument; + /* The adreno SMMU node is dma-coherent at the pinned tree, so table walks are inner write-back. */ + config_=(ArmSmmu4KContextConfig){.lower=upper?nullptr:&arena_.config,.upper=upper?&arena_.config:nullptr, + .lower_tables=upper?nullptr:&state_.tables,.upper_tables=upper?&state_.tables:nullptr,.id0=0,.id2=0, + .asid=asid,.coherent_walk=1,.stall_on_fault=0,.continue_after_fault=1}; + tables_=tables;bank_=bank;poll_budget_=poll_budget;prepared_=true; + return kIOReturnSuccess; +} +IOReturn NativeAdrenoSmmuContext::program(const ArmSmmu4KIo *io) +{ + if(!prepared_ || programmed_ || poisoned_ || !io)return kIOReturnBadArgument; + const ArmSmmu4KStatus status=ArmSmmu4KProgramDisabledContext(io,bank_,&config_, + poll_budget_,&program_); + program_status_=status; + if(status!=ArmSmmu4KSuccess) {poisoned_=true;return kIOReturnIOError;} + io_=*io;programmed_=true; + return kIOReturnSuccess; +} +IOReturn NativeAdrenoSmmuContext::attach(const ArmSmmu4KStream *routes,size_t count) +{ + if(!programmed_ || attached_ || poisoned_ || !routes || !count || count>8) + return kIOReturnBadArgument; + const ArmSmmu4KStatus status=ArmSmmu4KAttachStreams(&io_,&streams_,routes,count, + bank_,&config_,&routes_); + attach_status_=status; + if(status!=ArmSmmu4KSuccess) {poisoned_=true;return kIOReturnIOError;} + attached_=true; + return kIOReturnSuccess; +} +IOReturn NativeAdrenoSmmuContext::observe() const +{ + if(!programmed_ || poisoned_)return kIOReturnBadArgument; + ArmSmmu4KProgramResult result{}; + return ArmSmmu4KObserveOwnedContext(&io_,bank_,&config_,&result)==ArmSmmu4KSuccess? + kIOReturnSuccess:kIOReturnIOError; +} +IOReturn NativeAdrenoSmmuContext::disable() +{ + if(!attached_ || poisoned_)return kIOReturnBadArgument; + ArmSmmu4KProgramResult result{}; + if(ArmSmmu4KDisableOwnedContext(&io_,bank_,&config_,poll_budget_,&result)!=ArmSmmu4KSuccess) { + poisoned_=true; + return kIOReturnIOError; + } + attached_=false;programmed_=false; + return kIOReturnSuccess; +} diff --git a/drivers/NativeAdreno/NativeAdrenoSmmuContext.h b/drivers/NativeAdreno/NativeAdrenoSmmuContext.h new file mode 100644 index 0000000..272402c --- /dev/null +++ b/drivers/NativeAdreno/NativeAdrenoSmmuContext.h @@ -0,0 +1,65 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef NATIVE_ADRENO_SMMU_CONTEXT_H +#define NATIVE_ADRENO_SMMU_CONTEXT_H +#include "NativeAdrenoSmmuMemory.h" +#include "ArmSmmu4KStreams.h" + +/* One single-region GPU SMMU context (lower/TTBR0, as the GMU's own 0..2 GiB + * space, or upper/TTBR1), composed from the qualified kits: + * prepare an inactive native4K table pool, program the context bank through the + * real register sequence, then attach the caller's streams. Every step is + * separate and can only run in order; a failure leaves the earlier state + * retained and marks this object unusable for further programming. The bank + * number, ASID and stream slots are required inputs and are never guessed. + * No power, clock, reset, fault-interrupt or global SMMU setup is performed. */ +class NativeAdrenoSmmuContext final { + NativeAdrenoMemory *tables_=nullptr; + ArmSmmu4KLiveArena arena_{}; + ArmSmmu4KLiveState state_{}; + ArmSmmu4KLiveVisibility visibility_{}; + ArmSmmu4KContextConfig config_{}; + ArmSmmu4KProgramResult program_{}; + ArmSmmu4KStreamState streams_{}; + ArmSmmu4KStreamResult routes_{}; + ArmSmmu4KIo io_{}; + unsigned bank_=0,poll_budget_=0; + ArmSmmu4KStatus program_status_=ArmSmmu4KSuccess,attach_status_=ArmSmmu4KSuccess; + bool prepared_=false,programmed_=false,attached_=false,poisoned_=false; +public: + NativeAdrenoSmmuContext()=default; + ~NativeAdrenoSmmuContext()=default; + NativeAdrenoSmmuContext(const NativeAdrenoSmmuContext &)=delete; + NativeAdrenoSmmuContext &operator=(const NativeAdrenoSmmuContext &)=delete; + /* A 2 GiB lower aperture, the GMU's, is 524288 leaves. */ + enum : size_t {MaxLeaves=size_t(1)<<19}; + /* iova/bytes describe the aperture in the chosen region; the pool must be contiguous. */ + /* output_bits: the context's output size, at most the SMMU's ID2.OAS (Linux uses OAS for stage 1). */ + IOReturn prepare(NativeAdrenoMemory *tables,unsigned upper,uint64_t iova,size_t bytes, + size_t leaf_budget,unsigned bank,unsigned asid,unsigned poll_budget,unsigned output_bits=40); + /* Reads actual IDs, programs and enables exactly this bank. */ + IOReturn program(const ArmSmmu4KIo *); + /* Attach the caller's exact stream routes to the programmed bank. */ + IOReturn attach(const ArmSmmu4KStream *,size_t count); + /* Read-only observation of the owned enabled context. */ + IOReturn observe() const; + /* Only after attach; leaves streams faulted by the caller's own recovery. */ + IOReturn disable(); + bool prepared() const {return prepared_;} + bool programmed() const {return programmed_;} + bool attached() const {return attached_;} + bool poisoned() const {return poisoned_;} + const ArmSmmu4KContextConfig *configuration() const {return &config_;} + const ArmSmmu4KLiveArena *arena() const {return &arena_;} + /* Handles for a NativeAdrenoVm that will map buffers through this bank. */ + ArmSmmu4KLiveArena *arenaHandle() {return &arena_;} + ArmSmmu4KLiveState *stateHandle() {return &state_;} + ArmSmmu4KContextConfig *configHandle() {return &config_;} + ArmSmmu4KStreamState *streamsHandle() {return &streams_;} + const ArmSmmu4KIo *ioHandle() const {return &io_;} + const ArmSmmu4KProgramResult *programResult() const {return &program_; } + const ArmSmmu4KStreamResult *streamResult() const {return &routes_;} + /* The kit's own status of the last program/attach, which IOReturn folds into one error. */ + ArmSmmu4KStatus programStatus() const {return program_status_;} + ArmSmmu4KStatus attachStatus() const {return attach_status_;} +}; +#endif diff --git a/drivers/NativeAdreno/NativeAdrenoSmmuMemory.cpp b/drivers/NativeAdreno/NativeAdrenoSmmuMemory.cpp new file mode 100644 index 0000000..01fe173 --- /dev/null +++ b/drivers/NativeAdreno/NativeAdrenoSmmuMemory.cpp @@ -0,0 +1,42 @@ +/* SPDX-License-Identifier: MIT */ +#include "NativeAdrenoSmmuMemory.h" +static int publishTable(void *cookie,volatile uint64_t *page,size_t bytes) +{ + auto *memory=static_cast(cookie); + if(!memory || !page || bytes!=4096 || ((uintptr_t)page&4095) || + (uintptr_t)page<(uintptr_t)memory->mapping())return 1; + uintptr_t offset=(uintptr_t)page-(uintptr_t)memory->mapping(); + if(offset>memory->bytes() || bytes>memory->bytes()-offset)return 1; + return memory->publishForDevice(offset,bytes,false)==kIOReturnSuccess?0:1; +} +IOReturn NativeAdrenoBindSmmuVisibility(NativeAdrenoMemory *memory,ArmSmmu4KLiveVisibility *visibility) +{ + if(!visibility)return kIOReturnBadArgument;*visibility={}; + if(!memory || memory->quarantined() || !memory->deviceReferences() || !memory->mapping() || + memory->spanCount()!=1 || memory->spans()[0].offset || + memory->spans()[0].bytes!=memory->bytes())return kIOReturnBadArgument; + *visibility={memory,publishTable};return kIOReturnSuccess; +} +IOReturn NativeAdrenoPrepareSmmuMemory(NativeAdrenoMemory *memory,unsigned input_bits, + unsigned output_bits,uint64_t iova,uint64_t bytes,unsigned upper,size_t budget, + ArmSmmu4KLiveArena *arena,ArmSmmu4KLiveState *state,ArmSmmu4KLiveVisibility *visibility) +{ + if(!memory || !arena || !state || !visibility)return kIOReturnBadArgument; + *visibility={}; + if(state->prepared || state->poisoned || memory->quarantined() || memory->deviceReferences()) + return kIOReturnBusy; + if(!memory->mapping() || memory->spanCount()!=1 || + memory->spans()[0].offset || memory->spans()[0].bytes!=memory->bytes() || + !memory->bytes() || (memory->bytes()&4095))return kIOReturnUnsupported; + *arena={{input_bits,output_bits,memory->spans()[0].physical,budget},iova,bytes,upper, + static_cast(memory->mapping()),memory->bytes()/4096}; + ArmSmmu4KStatus status=ArmSmmu4KLivePrepare(arena,state); + if(status!=ArmSmmu4KSuccess)return status==ArmSmmu4KCapacity?kIOReturnNoResources:kIOReturnBadArgument; + ArmSmmu4KLiveResult audit; + status=ArmSmmu4KLiveAudit(arena,state,nullptr,0,&audit); + if(status!=ArmSmmu4KSuccess) {state->poisoned=1;return kIOReturnIOError;} + IOReturn result=memory->publishForDevice(0,state->tables.table_pages*4096,false); + if(result==kIOReturnSuccess)result=memory->holdForDevice(); + if(result!=kIOReturnSuccess) {state->poisoned=1;return result;} + return NativeAdrenoBindSmmuVisibility(memory,visibility); +} diff --git a/drivers/NativeAdreno/NativeAdrenoSmmuMemory.h b/drivers/NativeAdreno/NativeAdrenoSmmuMemory.h new file mode 100644 index 0000000..6fb40d3 --- /dev/null +++ b/drivers/NativeAdreno/NativeAdrenoSmmuMemory.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef NATIVE_ADRENO_SMMU_MEMORY_H +#define NATIVE_ADRENO_SMMU_MEMORY_H +#include "NativeAdrenoMemory.h" +#include "ArmSmmu4KLive.h" +/* Bind the actual contiguous raw-PA backing allocation to an INACTIVE native + * arena, publish its used table pages and take a device-lifetime reference. + * Caller retains memory/arena/state/visibility under an exclusive resource + * gate; all outputs and the backing pool are disjoint. State starts zeroed. + * Never invoke on an active or foreign table pool. This does not program a + * context, route streams, allocate MMIO/IRQs or establish power/security. + * On success the memory reference remains held until real masters are stopped, + * streams faulted, context disabled and native TLB synchronization completes. + * On prepare/publication failure no context has been exposed; caller retains + * allocation and poisoned state and handles cleanup. No automatic replay. */ +IOReturn NativeAdrenoPrepareSmmuMemory(NativeAdrenoMemory *,unsigned input_bits, + unsigned output_bits,uint64_t iova,uint64_t bytes,unsigned upper_region, + size_t leaf_budget,ArmSmmu4KLiveArena *,ArmSmmu4KLiveState *,ArmSmmu4KLiveVisibility *); +/* Bind only the native PoC publisher to an already retained contiguous table + * allocation. No caller-supplied success callback or page-table copy. */ +IOReturn NativeAdrenoBindSmmuVisibility(NativeAdrenoMemory *,ArmSmmu4KLiveVisibility *); +#endif diff --git a/drivers/NativeAdreno/NativeAdrenoSmmuService.cpp b/drivers/NativeAdreno/NativeAdrenoSmmuService.cpp new file mode 100644 index 0000000..c6c7fdf --- /dev/null +++ b/drivers/NativeAdreno/NativeAdrenoSmmuService.cpp @@ -0,0 +1,95 @@ +/* SPDX-License-Identifier: MIT */ +/* GPU SMMU platform node: maps the SMMU register window and publishes its base + * so the GPU service does not resolve the DT phandle itself. No register is + * read or written, no context is allocated and no stream is attached; those + * steps remain with the GPU bring-up and its stream table. */ +#include +#include +#include +#include +#include "NativeAdrenoSmmuService.h" + +OSDefineMetaClassAndStructors(NativeAdrenoSmmuService,IOService) + + + +static bool advertisesSmmu(IOService *provider) +{ + OSObject *property=provider?provider->getProperty("compatible"):nullptr; + OSData *data=OSDynamicCast(OSData,property); + if(!data || !data->getBytesNoCopy() || !data->getLength())return false; + const char *text=static_cast(data->getBytesNoCopy()); + const size_t length=data->getLength(); + static const char needle[]="qcom,adreno-smmu"; + for(size_t at=0;at+sizeof(needle)-1<=length;++at) { + size_t index=0; + while(indexgetProperty("reg")); + OSData *address=OSDynamicCast(OSData,provider->getProperty("#address-cells")); + OSData *size=OSDynamicCast(OSData,provider->getProperty("#size-cells")); + uint32_t address_cells=2,size_cells=2; + if(address && address->getLength()>=4) + address_cells=*static_cast(address->getBytesNoCopy()); + if(size && size->getLength()>=4) + size_cells=*static_cast(size->getBytesNoCopy()); + if(!reg || NativeAdrenoBoard::parseReg( + static_cast(reg->getBytesNoCopy()),reg->getLength(), + address_cells,size_cells,windows_,NativeAdrenoMaxWindows,&count_)!=kIOReturnSuccess) { + IOLog("NativeAdrenoSmmu: reg windows are not usable\n"); + count_=0;IOService::stop(provider); + return false; + } + for(size_t index=0;index(index)); + if(!memory) { + IOLog("NativeAdrenoSmmu: window %lu has no device memory\n", + static_cast(index)); + stop(provider); + return false; + } + maps_[index]=OSSharedPtr(memory->map(kIOMapAnywhere|kIOMapInhibitCache), + OSNoRetain); + if(!maps_[index]) { + IOLog("NativeAdrenoSmmu: window %lu did not map\n", + static_cast(index)); + stop(provider); + return false; + } + } + OSData *bases=OSData::withCapacity(static_cast(count_*sizeof(uint64_t))); + if(!bases) {stop(provider);return false;} + for(size_t index=0;indexappendBytes(&base,sizeof(base)); + } + setProperty("smmu-window-bases",bases); + bases->release(); + OSData *cells=OSDynamicCast(OSData,provider->getProperty("#iommu-cells")); + if(cells && cells->getLength()>=4) { + const uint32_t value=*static_cast(cells->getBytesNoCopy()); + setProperty("iommu-cells",value,32); + } + IOLog("NativeAdrenoSmmu: %lu window mapped, no register access\n", + static_cast(count_)); + registerService(); + return true; +} +void NativeAdrenoSmmuService::stop(IOService *provider) +{ + for(auto &map:maps_)map.reset(); + count_=0; + IOService::stop(provider); +} diff --git a/drivers/NativeAdreno/NativeAdrenoSmmuService.h b/drivers/NativeAdreno/NativeAdrenoSmmuService.h new file mode 100644 index 0000000..bd0555b --- /dev/null +++ b/drivers/NativeAdreno/NativeAdrenoSmmuService.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef NATIVE_ADRENO_SMMU_SERVICE_H +#define NATIVE_ADRENO_SMMU_SERVICE_H +#include +#include +#include "NativeAdrenoBoard.h" + +class NativeAdrenoSmmuService : public IOService { + OSDeclareDefaultStructors(NativeAdrenoSmmuService) + OSSharedPtr maps_[NativeAdrenoMaxWindows]; + NativeAdrenoWindow windows_[NativeAdrenoMaxWindows]{}; + size_t count_=0; +public: + IOService *probe(IOService *provider,SInt32 *score) APPLE_KEXT_OVERRIDE; + bool start(IOService *provider) APPLE_KEXT_OVERRIDE; + void stop(IOService *provider) APPLE_KEXT_OVERRIDE; + uint64_t windowBase(size_t index) const + {return index(maps_[index]->getVirtualAddress()):nullptr;} +}; + +#endif diff --git a/drivers/NativeAdreno/NativeAdrenoStatus.h b/drivers/NativeAdreno/NativeAdrenoStatus.h new file mode 100644 index 0000000..fa69ac8 --- /dev/null +++ b/drivers/NativeAdreno/NativeAdrenoStatus.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef NATIVE_ADRENO_STATUS_H +#define NATIVE_ADRENO_STATUS_H +#include +#include +/* Monotonic fault and suspend counters for Turnip's device_check_status and + * device_get_suspend_count. Turnip compares the fault count against the value + * it saw last; a change means the context is lost. This object only counts + * what the driver tells it: the IRQ path records a fatal decode, the reset or + * power path records a suspend, and submission poisoning must be recorded by + * the caller that observed it. It performs no detection, reset or recovery and + * never clears a count, so a client cannot hide a fault by re-reading. */ +struct NativeAdrenoStatusCounts { + uint64_t faults; + uint64_t suspends; +}; +class NativeAdrenoStatus final { + uint64_t faults_=0,suspends_=0; +public: + void noteFault() {if(faults_!=UINT64_MAX)++faults_;} + void noteSuspend() {if(suspends_!=UINT64_MAX)++suspends_;} + uint64_t faults() const {return faults_;} + uint64_t suspends() const {return suspends_;} + NativeAdrenoStatusCounts counts() const {return {faults_,suspends_};} +}; +#endif diff --git a/drivers/NativeAdreno/NativeAdrenoSubmit.cpp b/drivers/NativeAdreno/NativeAdrenoSubmit.cpp new file mode 100644 index 0000000..5e25562 --- /dev/null +++ b/drivers/NativeAdreno/NativeAdrenoSubmit.cpp @@ -0,0 +1,197 @@ +/* SPDX-License-Identifier: MIT */ +#include "NativeAdrenoSubmit.h" +#include "AdrenoPM4.h" +#include + +/* CP_INDIRECT_BUFFER is opcode 0x3f with a 3-word payload: 64-bit GPU address + * and IB_SIZE in dwords (bits 0..19), exactly as the pinned adreno_pm4.xml + * domain states and Linux msm_gem_submit.c converts the userspace byte size. */ +enum : uint32_t {SubmitIndirectBuffer=0x3f,SubmitIbWords=3,SubmitMaxJobs=8, + SubmitMaxBuffers=16,SubmitMaxIbBytes=0xfffffu*4u}; +extern "C" {volatile uint32_t native_adreno_coherence[8];} +IOReturn NativeAdrenoSubmit::poison() +{ + poisoned_=true;return kIOReturnIOError; +} +NativeAdrenoSubmit::Job *NativeAdrenoSubmit::find(uint32_t fence) +{ + if(fence)for(auto &job:jobs_)if(job.active && job.fence==fence)return &job; + return nullptr; +} +NativeAdrenoSubmit::Job *NativeAdrenoSubmit::free() +{ + for(auto &job:jobs_)if(!job.active)return &job; + return nullptr; +} +IOReturn NativeAdrenoSubmit::attach(NativeAdrenoVm *vm,NativeAdrenoRing *ring) +{ + if(attached_ || poisoned_ || vm_ || ring_)return kIOReturnBusy; + if(!vm || !ring || vm->poisoned() || ring->poisoned())return kIOReturnBadArgument; + vm_=vm;ring_=ring;attached_=true;stats_={}; + return kIOReturnSuccess; +} +IOReturn NativeAdrenoSubmit::release(Job &job) +{ + IOReturn status=kIOReturnSuccess; + for(size_t i=0;iretireJobAfterCompletion(job.handles[i]); + if(retired)status=retired;else ++stats_.references_retired; + } + if(status)return poison(); + job={};return kIOReturnSuccess; +} +IOReturn NativeAdrenoSubmit::detach() +{ + if(poisoned_ || outstanding())return kIOReturnBusy; + if(!attached_)return kIOReturnNotReady; + attached_=false;vm_=nullptr;ring_=nullptr;return kIOReturnSuccess; +} +bool NativeAdrenoSubmit::queueOutstanding(uint32_t queue) const +{ + for(const auto &job:jobs_)if(job.active && job.queue==queue)return true; + return false; +} +size_t NativeAdrenoSubmit::outstanding() const +{ + size_t count=0;for(const auto &job:jobs_)if(job.active)++count;return count; +} +IOReturn NativeAdrenoSubmit::submit(const NativeAdrenoSubmitBuffer *buffers,size_t count, + uint32_t queue,uint32_t *fence) +{ + if(!fence)return kIOReturnBadArgument;*fence=0; + if(!attached_ || poisoned_)return kIOReturnNotReady; + if(!buffers || !count || count>SubmitMaxBuffers)return kIOReturnBadArgument; + /* Every buffer is validated against the live VM before any reference. */ + uint64_t handles[SubmitMaxBuffers];size_t handle_count=0; + uint32_t words[SubmitMaxBuffers*(1+SubmitIbWords)];size_t word_count=0; + for(size_t i=0;iSubmitMaxIbBytes) + return kIOReturnBadArgument; + uint64_t iova=0;size_t bytes=0;unsigned permissions=0; + IOReturn status=vm_->describe(buffer.handle,&iova,&bytes,&permissions); + if(status)return status; + if(!(permissions&ArmSmmu4KRead) || buffer.iovabytes || buffer.bytes>bytes-(buffer.iova-iova))return kIOReturnBadArgument; + uint32_t header=0; + if(AdrenoPM4Type7(SubmitIndirectBuffer,SubmitIbWords,&header)!=AdrenoPM4Success) + return poison(); + words[word_count++]=header; + words[word_count++]=(uint32_t)buffer.iova; + words[word_count++]=(uint32_t)(buffer.iova>>32); + words[word_count++]=(uint32_t)(buffer.bytes/4); + bool known=false; + for(size_t j=0;jholdJob(handles[i]); + if(status) { + while(i) {vm_->retireJobAfterCompletion(handles[--i]);++stats_.references_retired;} + return status; + } + ++stats_.references_held; + } + uint32_t value=0; + /* int-070: the first IB word as the kernel's cacheable alias and DRAM see it before any maintenance, then DRAM after the CPU + * caches over every mapped BO are cleaned and invalidated. The board shows the last submission's record on a stall. */ + uint32_t cached=~0u,before=~0u,ignored=0,after=~0u; + (void)vm_->sampleWord(buffers[0].iova,&cached,&before); + IOReturn status=vm_->publishMapped(); + if(!status) { + (void)vm_->sampleWord(buffers[0].iova,&ignored,&after); + native_adreno_coherence[0]=native_adreno_coherence[0]+1; + native_adreno_coherence[1]=cached;native_adreno_coherence[2]=before;native_adreno_coherence[3]=after; + if(cached!=before || before!=after) { + if(!native_adreno_coherence[4]) { + native_adreno_coherence[5]=cached;native_adreno_coherence[6]=before;native_adreno_coherence[7]=after; + } + native_adreno_coherence[4]=native_adreno_coherence[4]+1; + } + status=vm_->invalidateTlb(); + } + if(status) { + for(size_t i=0;iretireJobAfterCompletion(handles[i]);++stats_.references_retired;} + return status; + } + status=ring_->submitFramed(words,word_count,&value); + if(status) { + /* A poisoned ring may already have consumed the WPTR write, so the + * referenced buffers cannot be retired here. */ + if(ring_->poisoned()) { + *job=Job{};++stats_.poisoned_jobs; + for(size_t i=0;ihandles[i]=handles[i]; + job->count=handle_count;job->queue=queue;job->active=true; + return poison(); + } + while(handle_count) {vm_->retireJobAfterCompletion(handles[--handle_count]);++stats_.references_retired;} + return status; + } + for(size_t i=0;ihandles[i]=handles[i]; + job->count=handle_count;job->fence=value;job->queue=queue;job->active=true; + ++stats_.submissions; + if(outstanding()>stats_.peak_jobs)stats_.peak_jobs=outstanding(); + *fence=value; + return kIOReturnSuccess; +} +IOReturn NativeAdrenoSubmit::observeFence(uint32_t fence,const NativeAdrenoRingClock *clock, + uint32_t timeout_us) +{ + if(!attached_ || poisoned_)return kIOReturnNotReady; + if(!fence)return kIOReturnBadArgument; + const NativeAdrenoRingFence result=ring_->observeFence(fence,clock,timeout_us,nullptr); + if(result==NativeAdrenoRingFencePoisoned)return poison(); + if(result==NativeAdrenoRingFenceArgument)return kIOReturnBadArgument; + if(result==NativeAdrenoRingFenceTimeout)return kIOReturnTimeout; + return kIOReturnSuccess; +} +IOReturn NativeAdrenoSubmit::wait(uint32_t fence,const NativeAdrenoRingClock *clock, + uint32_t timeout_us) +{ + if(!attached_ || poisoned_)return kIOReturnNotReady; + Job *job=find(fence); + if(!job) { + const NativeAdrenoRingFence passed=ring_->observeFence(fence,clock,timeout_us,nullptr); + if(passed==NativeAdrenoRingFencePoisoned)return poison(); + if(passed==NativeAdrenoRingFenceArgument)return kIOReturnBadArgument; + if(passed==NativeAdrenoRingFenceTimeout) {++stats_.timeouts;return kIOReturnTimeout;} + return kIOReturnSuccess; + } + const NativeAdrenoRingFence result=ring_->wait(fence,clock,timeout_us,nullptr); + if(result==NativeAdrenoRingFencePoisoned)return poison(); + if(result==NativeAdrenoRingFenceArgument)return kIOReturnBadArgument; + if(result==NativeAdrenoRingFenceTimeout) {++stats_.timeouts;return kIOReturnTimeout;} + IOReturn status=release(*job); + if(status)return status; + ++stats_.completed; + return kIOReturnSuccess; +} +IOReturn NativeAdrenoSubmit::waitAll(const NativeAdrenoRingClock *clock,uint32_t timeout_us) +{ + if(!attached_ || poisoned_)return kIOReturnNotReady; + while(outstanding()) { + Job *oldest=nullptr; + for(auto &job:jobs_)if(job.active && (!oldest || job.fence-oldest->fence>0x80000000u))oldest=&job; + const IOReturn status=wait(oldest->fence,clock,timeout_us); + if(status)return status; + } + return kIOReturnSuccess; +} +IOReturn NativeAdrenoSubmit::retireCompleted() +{ + if(!attached_ || poisoned_)return kIOReturnNotReady; + for(auto &job:jobs_) { + if(!job.active)continue; + const NativeAdrenoRingFence state=ring_->observeFence(job.fence,nullptr,0,nullptr); + if(state==NativeAdrenoRingFencePoisoned)return poison(); + if(state!=NativeAdrenoRingFenceComplete)continue; + const IOReturn status=release(job); + if(status)return status; + ++stats_.completed; + } + return kIOReturnSuccess; +} diff --git a/drivers/NativeAdreno/NativeAdrenoSubmit.h b/drivers/NativeAdreno/NativeAdrenoSubmit.h new file mode 100644 index 0000000..3fcf4d3 --- /dev/null +++ b/drivers/NativeAdreno/NativeAdrenoSubmit.h @@ -0,0 +1,74 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef NATIVE_ADRENO_SUBMIT_H +#define NATIVE_ADRENO_SUBMIT_H +#include "NativeAdrenoRing.h" +#include "NativeAdrenoVm.h" + +struct NativeAdrenoSubmitBuffer { + uint64_t handle; + uint64_t iova; + size_t bytes; /* Complete indirect-buffer range in bytes. */ +}; +/* Coherence samples (int-070): [0] submissions sampled; [1..3] the last one's first IB word through the kernel's cacheable alias, + * through DRAM before and through DRAM after the CPU cache maintenance of every mapped BO (~0 when unreadable); [4] samples whose + * three values differ and [5..7] the first such sample. */ +extern "C" volatile uint32_t native_adreno_coherence[8]; +struct NativeAdrenoSubmitStats { + uint32_t submissions,completed,timeouts,poisoned_jobs; + size_t references_held,references_retired,peak_jobs; +}; +/* Kernel-side execbuf: turns validated indirect buffers into one ring + * submission and owns a device reference per distinct buffer until the + * timestamp fence is actually observed. This is not a userspace ABI and not a + * scheduler: at most eight jobs are outstanding, callers serialize every + * method, and the ring/VM must already be attached to the same live context. + * Buffers may be sub-ranges of a live VM mapping but never a different IOVA, + * unmapped handle, table pool or a range that includes unreadable bytes. + * A failed ring commit that poisons the ring keeps every reference and the + * submit object unusable; only an independent reset may reclaim them. */ +class NativeAdrenoSubmit final { + struct Job { + uint32_t fence=0; + size_t count=0; + uint64_t handles[16]{}; + uint32_t queue=0; + bool active=false; + } jobs_[8]; + NativeAdrenoVm *vm_=nullptr; + NativeAdrenoRing *ring_=nullptr; + NativeAdrenoSubmitStats stats_{}; + bool attached_=false,poisoned_=false; + Job *find(uint32_t fence); + Job *free(); + IOReturn release(Job &); + IOReturn poison(); +public: + NativeAdrenoSubmit()=default; + ~NativeAdrenoSubmit()=default; + NativeAdrenoSubmit(const NativeAdrenoSubmit &)=delete; + NativeAdrenoSubmit &operator=(const NativeAdrenoSubmit &)=delete; + IOReturn attach(NativeAdrenoVm *,NativeAdrenoRing *); + /* Refuses while jobs are outstanding or poisoned; the caller must have + * stopped the GPU and observed every fence before calling this. */ + IOReturn detach(); + /* queue is bookkeeping for one shared ring; it never reorders or isolates + * submissions and is validated by the caller's open queue set. */ + IOReturn submit(const NativeAdrenoSubmitBuffer *,size_t,uint32_t queue, + uint32_t *fence); + bool queueOutstanding(uint32_t queue) const; + /* Retires (without waiting) every job whose fence the CP has already passed, so completed work never pins buffers or + * job slots just because nobody called wait(). */ + IOReturn retireCompleted(); + /* Waits for every outstanding job in submission order and retires its references. */ + IOReturn waitAll(const NativeAdrenoRingClock *,uint32_t timeout_us); + /* Polls the CP timestamp; on completion retires every buffer reference. A fence whose job was already retired (by + * wait or retireCompleted) is checked against the ring and succeeds once the CP has passed it. */ + IOReturn wait(uint32_t fence,const NativeAdrenoRingClock *,uint32_t timeout_us); + /* Wait for an earlier fence without touching job references; a timeout + * leaves every job and reference exactly as they were. */ + IOReturn observeFence(uint32_t fence,const NativeAdrenoRingClock *,uint32_t timeout_us); + bool poisoned() const {return poisoned_;} + size_t outstanding() const; + NativeAdrenoSubmitStats stats() const {return stats_;} +}; +#endif diff --git a/drivers/NativeAdreno/NativeAdrenoTimer.cpp b/drivers/NativeAdreno/NativeAdrenoTimer.cpp new file mode 100644 index 0000000..0222415 --- /dev/null +++ b/drivers/NativeAdreno/NativeAdrenoTimer.cpp @@ -0,0 +1,55 @@ +/* SPDX-License-Identifier: MIT */ +#include "NativeAdrenoTimer.h" +#include + +/* Derivation, not a guess: + * pinned a6xx_gmu.xml GMU_ALWAYS_ON_COUNTER_L 0x1f888, _H 0x1f889 + * (dword offsets, variants A6XX-A7XX) + * pinned sm8550.dtsi gpu@3d00000, gmu@3d6a000 + * byte offset = (offset << 2) - (gmu_base - gpu_base) + * = (0x1f888 << 2) - 0x6a000 = 0x14220 + * pinned a6xx.xml CP_ALWAYS_ON_COUNTER 0x0980 (64-bit, A6XX-A7XX) + * read as low at 0x0980 and high at 0x0981, as Linux gpu_read64 does. */ +enum : uint32_t { + TimerGmuLow = 0x14220, + TimerGmuHigh = 0x14224, + TimerCpLow = 0x0980, + TimerCpHigh = 0x0981, + TimerRetries = 8 +}; +uint32_t NativeAdrenoTimer::gmuLowOffset(){return TimerGmuLow;} +uint32_t NativeAdrenoTimer::gmuHighOffset(){return TimerGmuHigh;} +uint32_t NativeAdrenoTimer::cpLowRegister(){return TimerCpLow;} +uint32_t NativeAdrenoTimer::cpHighRegister(){return TimerCpHigh;} +IOReturn NativeAdrenoTimer::attach(const NativeAdrenoTimeIo *io,uint32_t source) +{ + if(attached_)return kIOReturnBusy; + if(!io || !io->read32)return kIOReturnBadArgument; + if(source!=NativeAdrenoTimeGmu && source!=NativeAdrenoTimeCp)return kIOReturnBadArgument; + io_=*io;source_=source;attached_=true; + return kIOReturnSuccess; +} +IOReturn NativeAdrenoTimer::detach() +{ + if(!attached_)return kIOReturnNotReady; + attached_=false;io_={}; + return kIOReturnSuccess; +} +IOReturn NativeAdrenoTimer::read(uint64_t *ticks) const +{ + if(!ticks)return kIOReturnBadArgument;*ticks=0; + if(!attached_)return kIOReturnNotReady; + const uint32_t low=source_==NativeAdrenoTimeGmu?TimerGmuLow:TimerCpLow; + const uint32_t high=source_==NativeAdrenoTimeGmu?TimerGmuHigh:TimerCpHigh; + for(unsigned attempt=0;attempt +#include + +/* GPU always-on counter reader for the user-visible timestamp. Two sources are + * distinguished because their address spaces differ: + * - GMU window: byte offsets inside the already bound GMU window. The A740 + * offset is derived from the pinned a6xx_gmu.xml dword offset 0x1f888 and + * the SM8550 DT layout: (0x1f888 << 2) - (gmu 0x03d6a000 - gpu 0x03d00000) + * = 0x14220 for the low word and 0x14224 for the high word. + * - CP window: dword register indices in the GPU register window, from + * a6xx.xml CP_ALWAYS_ON_COUNTER at 0x0980 for A6XX..A7XX. + * The caller binds exactly one already powered window; this object performs no + * power, reset or window discovery. Reads retry while the high word changes, + * so a counter rollover cannot return a torn value; exceeding the retry bound + * is an error. A failed read reports an I/O error and never fabricates ticks. */ +enum NativeAdrenoTimeSource { NativeAdrenoTimeGmu = 0, NativeAdrenoTimeCp = 1 }; +struct NativeAdrenoTimeIo { + void *cookie; + int (*read32)(void *,uint32_t address,uint32_t *value); +}; +class NativeAdrenoTimer final { + NativeAdrenoTimeIo io_{}; + uint32_t source_=NativeAdrenoTimeGmu; + bool attached_=false; +public: + NativeAdrenoTimer()=default; + ~NativeAdrenoTimer()=default; + NativeAdrenoTimer(const NativeAdrenoTimer &)=delete; + NativeAdrenoTimer &operator=(const NativeAdrenoTimer &)=delete; + /* Source is a numeric ABI boundary value and is range-checked. */ + IOReturn attach(const NativeAdrenoTimeIo *,uint32_t source); + IOReturn detach(); + IOReturn read(uint64_t *ticks) const; + static uint32_t gmuLowOffset(); + static uint32_t gmuHighOffset(); + static uint32_t cpLowRegister(); + static uint32_t cpHighRegister(); +}; +#endif diff --git a/drivers/NativeAdreno/NativeAdrenoUserClient.cpp b/drivers/NativeAdreno/NativeAdrenoUserClient.cpp new file mode 100644 index 0000000..f5565dc --- /dev/null +++ b/drivers/NativeAdreno/NativeAdrenoUserClient.cpp @@ -0,0 +1,520 @@ +/* SPDX-License-Identifier: MIT */ +#include "NativeAdrenoUserClient.h" +extern "C" volatile uint32_t native_adreno_trace_stage; +#include +#include +NativeAdrenoUserClient::Entry *NativeAdrenoUserClient::find(uint64_t handle) +{ + if(handle)for(size_t i=0;imapped_state; +} +NativeAdrenoMemory *NativeAdrenoUserClient::memoryOf(uint64_t handle) +{ + Entry *entry=find(handle);return entry && !entry->unreleasable?&entry->memory:nullptr; +} +IOReturn NativeAdrenoUserClient::open(uint32_t abi_version,NativeAdrenoVm *vm, + NativeAdrenoSubmit *submit,NativeAdrenoRing *ring,uint32_t input_bits,uint32_t output_bits) +{ + if(open_ || vm_ || submit_)return kIOReturnBusy; + if(abi_version!=NativeAdrenoAbiVersion)return kIOReturnUnsupported; + if(!vm || !submit || !ring || input_bits<31 || input_bits>48 || + output_bits<32 || output_bits>48 || vm->poisoned() || submit->poisoned() || + ring->poisoned())return kIOReturnBadArgument; + entries_=IONewData(Entry,NativeAdrenoClientMaxObjects); + if(!entries_)return kIOReturnNoMemory; + memset(entries_,0,sizeof(Entry)*NativeAdrenoClientMaxObjects); + entry_count_=NativeAdrenoClientMaxObjects; + vm_=vm;submit_=submit;ring_=ring;input_bits_=input_bits;output_bits_=output_bits; + next_handle_=1;open_=true; + queues_[0]={0,true}; + /* Each client is a new GPU context for the CP: its first submission clears the BR/BV synchronization state. */ + ring_->requestContextReset(); + return kIOReturnSuccess; +} +IOReturn NativeAdrenoUserClient::close() +{ + if(!open_)return kIOReturnNotReady; + if(liveObjects() || submit_->outstanding())return kIOReturnBusy; + for(size_t i=1;iwaitAll(clock,timeout_us); + if(status)return status; + for(size_t index=0;indexentry_count_)return nullptr; + Entry &entry=entries_[token-1]; + return entry.used && !entry.unreleasable && entry.token==token && entry.mapping_leases?&entry.memory:nullptr; +} +IOReturn NativeAdrenoUserClient::query(NativeAdrenoAbiParams *params) +{ + if(!params)return kIOReturnBadArgument; + *params={}; + if(!open_)return kIOReturnNotReady; + params->abi_version=NativeAdrenoAbiVersion; + params->input_bits=input_bits_;params->output_bits=output_bits_; + params->flags=NativeAdrenoParamsHostImport;params->max_bo_bytes=UINT64_C(1)<handle=0; + if(!open_)return kIOReturnNotReady; + const uint64_t limit=UINT64_C(1)<bytes || (request->bytes&4095) || request->bytes>limit || + request->align_log2<12 || request->align_log2>24 || request->contiguous>1) + return kIOReturnBadArgument; + Entry *entry=free(); + if(!entry)return kIOReturnNoResources; + /* A scattered allocation costs one VM span per physical page (int-030: 16 MiB came back as 4096 single-page spans, the + * whole span table), so larger objects try one contiguous run first and fall back to scattered pages unless the + * caller demanded contiguity. */ + IOReturn status=kIOReturnUnsupported; + const bool prefer_contiguous=request->contiguous==0 && request->bytes>=NativeAdrenoClientContiguousFirstBytes; + if(prefer_contiguous) { + status=entry->memory.allocate((size_t)request->bytes,output_bits_,true,(size_t)request->bytes); + if(status && !entry->memory.quarantined() && entry->memory.release()==kIOReturnSuccess)status=kIOReturnUnsupported; + } + if(status) + status=entry->memory.allocate((size_t)request->bytes,output_bits_,request->contiguous!=0,(size_t)request->bytes); + if(status) { + /* A failed allocation can leave quarantine behind; never reuse it. */ + clear(*entry); + if(entry->memory.release()!=kIOReturnSuccess) { + entry->used=true;entry->unreleasable=true; + } + return status==kIOReturnUnsupported?kIOReturnUnsupported:status; + } + entry->used=true;entry->handle=next_handle_++;entry->token=(uint32_t)(entry-entries_)+1; + entry->mapped=0;entry->iova=0;entry->mapped_state=false; + reply->handle=entry->handle; + return kIOReturnSuccess; +} +IOReturn NativeAdrenoUserClient::importBo(const NativeAdrenoAbiBoImport *request,void *task, + NativeAdrenoAbiBoAllocReply *reply) +{ + if(!request || !reply)return kIOReturnBadArgument;reply->handle=0; + if(!open_)return kIOReturnNotReady; + const uint64_t limit=UINT64_C(1)<address || ((request->address|request->bytes)&4095) || !request->bytes || request->bytes>limit) + return kIOReturnBadArgument; + Entry *entry=free(); + if(!entry)return kIOReturnNoResources; + const IOReturn status=entry->memory.importTask(task,request->address,(size_t)request->bytes,output_bits_,(size_t)request->bytes); + if(status) { + clear(*entry); + if(entry->memory.release()!=kIOReturnSuccess) {entry->used=true;entry->unreleasable=true;} + return status; + } + entry->used=true;entry->handle=next_handle_++;entry->token=(uint32_t)(entry-entries_)+1; + entry->mapped=0;entry->iova=0;entry->mapped_state=false; + reply->handle=entry->handle; + return kIOReturnSuccess; +} +IOReturn NativeAdrenoUserClient::mapBo(const NativeAdrenoAbiBoMap *request, + NativeAdrenoAbiBoMapReply *reply) +{ + if(!request || !reply)return kIOReturnBadArgument;reply->iova=0; + if(!open_)return kIOReturnNotReady; + if(!(request->permissions&1) || (request->permissions&~3u) || request->writeback>1 || + (request->shareability!=2 && request->shareability!=3) || request->reserved || + (request->iova&4095))return kIOReturnBadArgument; + Entry *entry=find(request->handle); + if(!entry || entry->unreleasable)return kIOReturnBadArgument; + if(entry->mapped_state)return kIOReturnBusy; + uint64_t handle=0; + const IOReturn status=vm_->map(&entry->memory,0,request->iova,request->permissions, + request->writeback,request->shareability,&handle); + if(status)return status; + entry->mapped=handle;entry->iova=request->iova;entry->mapped_state=true; + reply->iova=request->iova; + return kIOReturnSuccess; +} +IOReturn NativeAdrenoUserClient::freeBo(const NativeAdrenoAbiBoFree *request) +{ + native_adreno_trace_stage=1; + if(!request)return kIOReturnBadArgument; + if(!open_)return kIOReturnNotReady; + Entry *entry=find(request->handle); + if(!entry)return kIOReturnBadArgument; + if(entry->unreleasable)return kIOReturnBusy; + if(entry->mapping_leases)return kIOReturnBusy; + native_adreno_trace_stage=2; + if(entry->mapped_state) { + /* Completed jobs must not pin the buffer: retire what the CP has passed before the unmap decides. */ + submit_->retireCompleted(); /* a poisoned execbuf refuses and the unmap below stays refused too */ + native_adreno_trace_stage=3; + const IOReturn unmap=vm_->unmap(entry->mapped); + native_adreno_trace_stage=4; + if(unmap)return unmap; + entry->mapped_state=false;entry->mapped=0;entry->iova=0; + } + const IOReturn release=entry->memory.release(); + if(release)return release; + clear(*entry); + return kIOReturnSuccess; +} +IOReturn NativeAdrenoUserClient::submit(const NativeAdrenoAbiSubmit *request, + NativeAdrenoAbiSubmitReply *reply,const NativeAdrenoRingClock *clock) +{ + if(!request || !reply)return kIOReturnBadArgument;reply->fence=0; + if(!open_)return kIOReturnNotReady; + if(!request->buffer_count || request->buffer_count>NativeAdrenoAbiMaxBuffers || + request->reserved)return kIOReturnBadArgument; + if(request->queue>=NativeAdrenoAbiMaxQueues || !queues_[request->queue].active) + return kIOReturnBadArgument; + if(request->wait_fence) { + if(!clock)return kIOReturnBadArgument; + const uint32_t wait_us=request->wait_timeout_ns==0?0: + (request->wait_timeout_ns>UINT64_C(0xffffffff)*1000u?0xffffffffu: + (uint32_t)(request->wait_timeout_ns/1000)); + const IOReturn waited=submit_->observeFence((uint32_t)request->wait_fence,clock,wait_us); + if(waited)return waited; + } + NativeAdrenoSubmitBuffer buffers[NativeAdrenoAbiMaxBuffers]; + for(uint32_t i=0;ibuffer_count;++i) { + const NativeAdrenoAbiSubmitBuffer &wire=request->buffers[i]; + if(!wire.bytes || (wire.bytes&3))return kIOReturnBadArgument; + Entry *entry=find(wire.handle); + if(!entry || !entry->mapped_state)return kIOReturnBadArgument; + buffers[i]={entry->mapped,wire.iova,(size_t)wire.bytes}; + } + uint32_t fence=0; + const IOReturn status=submit_->submit(buffers,request->buffer_count, + static_cast(request->queue),&fence); + if(status)return status; + reply->fence=fence; + return kIOReturnSuccess; +} +IOReturn NativeAdrenoUserClient::attachTimer(const NativeAdrenoTimer *timer) +{ + if(!open_)return kIOReturnNotReady; + if(!timer || timer_ )return kIOReturnBadArgument; + timer_=timer; + return kIOReturnSuccess; +} +IOReturn NativeAdrenoUserClient::timestamp(NativeAdrenoAbiTimestamp *reply) +{ + if(!reply)return kIOReturnBadArgument;*reply={}; + if(!open_)return kIOReturnNotReady; + if(!timer_)return kIOReturnUnsupported; + return timer_->read(&reply->ticks); +} +IOReturn NativeAdrenoUserClient::attachDeviceInfo(const NativeAdrenoAbiDeviceInfo *info) +{ + if(!open_)return kIOReturnNotReady; + if(!info || info_ || info->reserved || !info->chip_id || !info->va_size)return kIOReturnBadArgument; + info_=info; + return kIOReturnSuccess; +} +IOReturn NativeAdrenoUserClient::deviceInfo(NativeAdrenoAbiDeviceInfo *reply) +{ + if(!reply)return kIOReturnBadArgument;*reply={}; + if(!open_)return kIOReturnNotReady; + if(!info_)return kIOReturnUnsupported; + *reply=*info_; + return kIOReturnSuccess; +} +IOReturn NativeAdrenoUserClient::attachStatus(const NativeAdrenoStatus *status) +{ + if(!open_)return kIOReturnNotReady; + if(!status || status_)return kIOReturnBadArgument; + status_=status; + return kIOReturnSuccess; +} +IOReturn NativeAdrenoUserClient::status(NativeAdrenoAbiStatus *reply) +{ + if(!reply)return kIOReturnBadArgument;*reply={}; + if(!open_)return kIOReturnNotReady; + if(!status_)return kIOReturnUnsupported; + const NativeAdrenoStatusCounts counts=status_->counts(); + reply->faults=counts.faults;reply->suspends=counts.suspends; + return kIOReturnSuccess; +} +size_t NativeAdrenoUserClient::inputSize(uint32_t method) +{ + switch(method) { + case NativeAdrenoMethodQuery: + case NativeAdrenoMethodGetTimestamp: + case NativeAdrenoMethodGetStatus: + case NativeAdrenoMethodGetDeviceInfo:return 0; + case NativeAdrenoMethodBoAlloc:return sizeof(NativeAdrenoAbiBoAlloc); + case NativeAdrenoMethodBoMap:return sizeof(NativeAdrenoAbiBoMap); + case NativeAdrenoMethodBoImport:return sizeof(NativeAdrenoAbiBoImport); + case NativeAdrenoMethodBoFree:return sizeof(NativeAdrenoAbiBoFree); + case NativeAdrenoMethodSubmit:return sizeof(NativeAdrenoAbiSubmit); + case NativeAdrenoMethodSetMetadata: + case NativeAdrenoMethodGetMetadata:return sizeof(NativeAdrenoAbiMetadata); + case NativeAdrenoMethodCreateQueue: + case NativeAdrenoMethodCloseQueue:return sizeof(NativeAdrenoAbiQueue); + case NativeAdrenoMethodBoCpuMap:return sizeof(NativeAdrenoAbiBoCpuMap); + case NativeAdrenoMethodBoCpuUnmap:return sizeof(NativeAdrenoAbiBoCpuUnmap); + case NativeAdrenoMethodWaitFence:return sizeof(NativeAdrenoAbiWaitFence); + default:return SIZE_MAX; + } +} +size_t NativeAdrenoUserClient::outputSize(uint32_t method) +{ + switch(method) { + case NativeAdrenoMethodQuery:return sizeof(NativeAdrenoAbiParams); + case NativeAdrenoMethodBoAlloc:return sizeof(NativeAdrenoAbiBoAllocReply); + case NativeAdrenoMethodBoMap:return sizeof(NativeAdrenoAbiBoMapReply); + case NativeAdrenoMethodBoFree:return 0; + case NativeAdrenoMethodSubmit:return sizeof(NativeAdrenoAbiSubmitReply); + case NativeAdrenoMethodWaitFence:return 0; + case NativeAdrenoMethodGetTimestamp:return sizeof(NativeAdrenoAbiTimestamp); + case NativeAdrenoMethodGetStatus:return sizeof(NativeAdrenoAbiStatus); + case NativeAdrenoMethodGetDeviceInfo:return sizeof(NativeAdrenoAbiDeviceInfo); + case NativeAdrenoMethodGetMetadata:return sizeof(NativeAdrenoAbiMetadata); + case NativeAdrenoMethodSetMetadata:return 0; + case NativeAdrenoMethodCreateQueue:return sizeof(NativeAdrenoAbiQueueReply); + case NativeAdrenoMethodCloseQueue:return 0; + case NativeAdrenoMethodBoCpuMap:return sizeof(NativeAdrenoAbiBoCpuMapReply); + case NativeAdrenoMethodBoCpuUnmap:return 0; + case NativeAdrenoMethodBoImport:return sizeof(NativeAdrenoAbiBoAllocReply); + default:return SIZE_MAX; + } +} +IOReturn NativeAdrenoUserClient::dispatch(uint32_t method,const void *input, + size_t input_size,void *output,size_t output_size,const NativeAdrenoRingClock *clock) +{ + const size_t expected_in=inputSize(method),expected_out=outputSize(method); + if(expected_in==SIZE_MAX || expected_out==SIZE_MAX)return kIOReturnUnsupported; + if(input_size!=expected_in || output_size!=expected_out)return kIOReturnBadArgument; + if((expected_in && !input) || (expected_out && !output))return kIOReturnBadArgument; + IOReturn status=kIOReturnSuccess; + switch(method) { + case NativeAdrenoMethodQuery: { + NativeAdrenoAbiParams reply{}; + status=query(&reply); + if(!status)memcpy(output,&reply,sizeof(reply)); + break; + } + case NativeAdrenoMethodBoAlloc: { + NativeAdrenoAbiBoAlloc request{}; + memcpy(&request,input,sizeof(request)); + NativeAdrenoAbiBoAllocReply reply{}; + status=allocBo(&request,&reply); + if(!status)memcpy(output,&reply,sizeof(reply)); + break; + } + case NativeAdrenoMethodBoImport: + status=kIOReturnUnsupported; + break; + case NativeAdrenoMethodBoMap: { + NativeAdrenoAbiBoMap request{}; + memcpy(&request,input,sizeof(request)); + NativeAdrenoAbiBoMapReply reply{}; + status=mapBo(&request,&reply); + if(!status)memcpy(output,&reply,sizeof(reply)); + break; + } + case NativeAdrenoMethodBoFree: { + NativeAdrenoAbiBoFree request{}; + memcpy(&request,input,sizeof(request)); + status=freeBo(&request); + break; + } + case NativeAdrenoMethodSubmit: { + NativeAdrenoAbiSubmit request{}; + memcpy(&request,input,sizeof(request)); + NativeAdrenoAbiSubmitReply reply{}; + status=submit(&request,&reply,clock); + if(!status)memcpy(output,&reply,sizeof(reply)); + break; + } + case NativeAdrenoMethodWaitFence: { + NativeAdrenoAbiWaitFence request{}; + memcpy(&request,input,sizeof(request)); + status=waitFence(&request,clock); + break; + } + case NativeAdrenoMethodGetDeviceInfo: { + NativeAdrenoAbiDeviceInfo reply{}; + status=deviceInfo(&reply); + if(!status)memcpy(output,&reply,sizeof(reply)); + break; + } + case NativeAdrenoMethodGetTimestamp: { + NativeAdrenoAbiTimestamp reply{}; + status=timestamp(&reply); + if(!status)memcpy(output,&reply,sizeof(reply)); + break; + } + case NativeAdrenoMethodBoCpuMap: { + NativeAdrenoAbiBoCpuMap request{}; + memcpy(&request,input,sizeof(request)); + NativeAdrenoAbiBoCpuMapReply reply{}; + status=cpuMap(&request,&reply); + if(!status)memcpy(output,&reply,sizeof(reply)); + break; + } + case NativeAdrenoMethodBoCpuUnmap: { + NativeAdrenoAbiBoCpuUnmap request{}; + memcpy(&request,input,sizeof(request)); + status=cpuUnmap(&request); + break; + } + case NativeAdrenoMethodCreateQueue: { + NativeAdrenoAbiQueueReply reply{}; + status=createQueue(&reply); + if(!status)memcpy(output,&reply,sizeof(reply)); + break; + } + case NativeAdrenoMethodCloseQueue: { + NativeAdrenoAbiQueue request{}; + memcpy(&request,input,sizeof(request)); + status=closeQueue(&request); + break; + } + case NativeAdrenoMethodSetMetadata: { + NativeAdrenoAbiMetadata request{}; + memcpy(&request,input,sizeof(request)); + status=setMetadata(&request); + break; + } + case NativeAdrenoMethodGetMetadata: { + NativeAdrenoAbiMetadata request{}; + memcpy(&request,input,sizeof(request)); + NativeAdrenoAbiMetadata reply{}; + status=getMetadata(&request,&reply); + if(!status)memcpy(output,&reply,sizeof(reply)); + break; + } + case NativeAdrenoMethodGetStatus: { + NativeAdrenoAbiStatus reply{}; + status=this->status(&reply); + if(!status)memcpy(output,&reply,sizeof(reply)); + break; + } + default:return kIOReturnUnsupported; + } + return status; +} +IOReturn NativeAdrenoUserClient::cpuMap(const NativeAdrenoAbiBoCpuMap *request, + NativeAdrenoAbiBoCpuMapReply *reply) +{ + if(!request || !reply)return kIOReturnBadArgument;*reply={}; + if(!open_)return kIOReturnNotReady; + Entry *entry=find(request->handle); + if(!entry || entry->unreleasable)return kIOReturnBadArgument; + /* An imported BO is the caller's own memory, already mapped where it came from. */ + if(entry->memory.imported())return kIOReturnUnsupported; + if(entry->mapping_leases==UINT32_MAX)return kIOReturnNoResources; + ++entry->mapping_leases; + reply->token=entry->token;reply->bytes=entry->memory.bytes(); + return kIOReturnSuccess; +} +IOReturn NativeAdrenoUserClient::cpuUnmap(const NativeAdrenoAbiBoCpuUnmap *request) +{ + if(!request)return kIOReturnBadArgument; + if(!open_)return kIOReturnNotReady; + if(request->reserved)return kIOReturnBadArgument; + Entry *entry=find(request->handle); + if(!entry || entry->unreleasable || !entry->token)return kIOReturnBadArgument; + if(request->token!=entry->token)return kIOReturnBadArgument; + if(!entry->mapping_leases)return kIOReturnBusy; + --entry->mapping_leases; + return kIOReturnSuccess; +} +IOReturn NativeAdrenoUserClient::createQueue(NativeAdrenoAbiQueueReply *reply) +{ + if(!reply)return kIOReturnBadArgument; + if(!open_)return kIOReturnNotReady; + for(size_t i=1;i(i),true}; + reply->queue=i; + return kIOReturnSuccess; + } + return kIOReturnNoResources; +} +IOReturn NativeAdrenoUserClient::closeQueue(const NativeAdrenoAbiQueue *request) +{ + if(!request)return kIOReturnBadArgument; + if(!open_)return kIOReturnNotReady; + if(!request->queue || request->queue>=NativeAdrenoAbiMaxQueues)return kIOReturnBadArgument; + Queue &queue=queues_[request->queue]; + if(!queue.active)return kIOReturnBadArgument; + /* A queue must be drained by the caller before it disappears. */ + if(submit_->queueOutstanding(static_cast(request->queue))) + return kIOReturnBusy; + queue={}; + return kIOReturnSuccess; +} +IOReturn NativeAdrenoUserClient::setMetadata(const NativeAdrenoAbiMetadata *request) +{ + if(!request)return kIOReturnBadArgument; + if(!open_)return kIOReturnNotReady; + if(request->reserved || request->bytes>NativeAdrenoAbiMetadataBytes)return kIOReturnBadArgument; + Entry *entry=find(request->handle); + if(!entry || entry->unreleasable)return kIOReturnBadArgument; + if(request->bytes)memcpy(entry->metadata,request->text,request->bytes); + entry->metadata_bytes=request->bytes; + return kIOReturnSuccess; +} +IOReturn NativeAdrenoUserClient::getMetadata(const NativeAdrenoAbiMetadata *request, + NativeAdrenoAbiMetadata *reply) +{ + if(!request || !reply)return kIOReturnBadArgument; + /* Snapshot the request so an in/out alias is still well defined. */ + const uint64_t handle=request->handle; + const uint32_t asked=request->bytes; + const uint32_t reserved=request->reserved; + *reply={}; + if(!open_)return kIOReturnNotReady; + if(reserved || asked>NativeAdrenoAbiMetadataBytes)return kIOReturnBadArgument; + Entry *entry=find(handle); + if(!entry || entry->unreleasable)return kIOReturnBadArgument; + if(askedmetadata_bytes)return kIOReturnBadArgument; + reply->handle=entry->handle;reply->bytes=entry->metadata_bytes; + if(entry->metadata_bytes)memcpy(reply->text,entry->metadata,entry->metadata_bytes); + return kIOReturnSuccess; +} +IOReturn NativeAdrenoUserClient::waitFence(const NativeAdrenoAbiWaitFence *request, + const NativeAdrenoRingClock *clock) +{ + if(!request || !clock)return kIOReturnBadArgument; + if(!open_)return kIOReturnNotReady; + if(!request->fence)return kIOReturnBadArgument; + const uint32_t timeout_us=request->timeout_ns==0?0: + (request->timeout_ns>UINT64_C(0xffffffff)*1000u?0xffffffffu: + (uint32_t)(request->timeout_ns/1000)); + return submit_->wait((uint32_t)request->fence,clock,timeout_us); +} diff --git a/drivers/NativeAdreno/NativeAdrenoUserClient.h b/drivers/NativeAdreno/NativeAdrenoUserClient.h new file mode 100644 index 0000000..ab11b8d --- /dev/null +++ b/drivers/NativeAdreno/NativeAdrenoUserClient.h @@ -0,0 +1,104 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef NATIVE_ADRENO_USERCLIENT_H +#define NATIVE_ADRENO_USERCLIENT_H +#include "NativeAdrenoABI.h" +#include "NativeAdrenoSubmit.h" +#include "NativeAdrenoTimer.h" +#include "NativeAdrenoStatus.h" + +/* Kernel-side body of the Adreno user client, deliberately free of IOKit so it + * can be exercised on a host. The future IOService/IOUserClient wrapper only + * copies one fixed ABI record in and out and calls these methods; it must not + * add policy. Handles are this object's own identities, never VM, IOVA or + * pointer values. A mapping is retirable only through freeBo, which unmaps + * first and then releases the backing object; both steps must succeed. + * Up to NativeAdrenoClientMaxObjects live objects and the execbuf's own eight-job limit apply. */ +enum : size_t {NativeAdrenoClientMaxObjects=1024,NativeAdrenoClientContiguousFirstBytes=65536}; +class NativeAdrenoUserClient final { + struct Queue {uint32_t id=0;bool active=false;}; + struct Entry { + NativeAdrenoMemory memory; + uint64_t handle=0; + uint64_t mapped=0; + uint64_t iova=0; + bool used=false; + bool mapped_state=false; + bool unreleasable=false; + uint32_t metadata_bytes=0; + uint32_t token=0; + uint32_t mapping_leases=0; + char metadata[NativeAdrenoAbiMetadataBytes]{}; + }; + /* Allocated at open(), all-zero is the free state. */ + Entry *entries_=nullptr; + size_t entry_count_=0; + Queue queues_[NativeAdrenoAbiMaxQueues]{}; + NativeAdrenoVm *vm_=nullptr; + NativeAdrenoSubmit *submit_=nullptr; + NativeAdrenoRing *ring_=nullptr; + const NativeAdrenoTimer *timer_=nullptr; + const NativeAdrenoStatus *status_=nullptr; + const NativeAdrenoAbiDeviceInfo *info_=nullptr; + uint32_t input_bits_=0,output_bits_=0; + uint64_t next_handle_=1; + bool open_=false; + Entry *find(uint64_t); + Entry *free(); + static void clear(Entry &); +public: + NativeAdrenoUserClient()=default; + ~NativeAdrenoUserClient()=default; + NativeAdrenoUserClient(const NativeAdrenoUserClient &)=delete; + NativeAdrenoUserClient &operator=(const NativeAdrenoUserClient &)=delete; + /* Mounts one already-qualified context. input/output bits describe the + * active native4K arena and are reported, never guessed. */ + IOReturn open(uint32_t abi_version,NativeAdrenoVm *,NativeAdrenoSubmit *, + NativeAdrenoRing *,uint32_t input_bits,uint32_t output_bits); + /* Refuses while any object is live or any job is unobserved. */ + IOReturn close(); + /* Connection teardown: waits for every outstanding job, then unmaps and frees every object, closes every queue and + * closes the client. The first failure stops it and leaves what remains live; nothing is forced. */ + IOReturn releaseAll(const NativeAdrenoRingClock *,uint32_t timeout_us); + /* Backing object of a token returned by cpuMap while its lease is held (clientMemoryForType); null otherwise. */ + NativeAdrenoMemory *memoryForToken(uint32_t token); + IOReturn query(NativeAdrenoAbiParams *); + IOReturn allocBo(const NativeAdrenoAbiBoAlloc *,NativeAdrenoAbiBoAllocReply *); + IOReturn mapBo(const NativeAdrenoAbiBoMap *,NativeAdrenoAbiBoMapReply *); + /* BoImport: pins the caller's pages. Only the IOKit wrapper can call it, with the calling task; dispatch() refuses the method + * because it has no task (the kernel mailbox path cannot import). */ + IOReturn importBo(const NativeAdrenoAbiBoImport *,void *task,NativeAdrenoAbiBoAllocReply *); + /* Unmaps if still mapped, then releases the backing object. A failure at + * either step keeps the entry live and unmapped where possible. */ + IOReturn freeBo(const NativeAdrenoAbiBoFree *); + IOReturn submit(const NativeAdrenoAbiSubmit *,NativeAdrenoAbiSubmitReply *, + const NativeAdrenoRingClock *); + IOReturn cpuMap(const NativeAdrenoAbiBoCpuMap *,NativeAdrenoAbiBoCpuMapReply *); + IOReturn cpuUnmap(const NativeAdrenoAbiBoCpuUnmap *); + IOReturn createQueue(NativeAdrenoAbiQueueReply *); + IOReturn closeQueue(const NativeAdrenoAbiQueue *); + IOReturn setMetadata(const NativeAdrenoAbiMetadata *); + IOReturn getMetadata(const NativeAdrenoAbiMetadata *,NativeAdrenoAbiMetadata *); + IOReturn waitFence(const NativeAdrenoAbiWaitFence *,const NativeAdrenoRingClock *); + /* One external-method entry: exact record sizes are required, unknown + * methods are unsupported and no output byte is modified on failure. The + * IOKit wrapper should only copy these buffers and call this function. */ + IOReturn dispatch(uint32_t method,const void *input,size_t input_size, + void *output,size_t output_size,const NativeAdrenoRingClock *); + static size_t inputSize(uint32_t method); + static size_t outputSize(uint32_t method); + size_t liveObjects() const; + /* Optional GPU timestamp source; absent means the method stays unsupported. */ + IOReturn attachTimer(const NativeAdrenoTimer *); + IOReturn timestamp(NativeAdrenoAbiTimestamp *); + /* Optional counters; absent means the status method stays unsupported. */ + IOReturn attachStatus(const NativeAdrenoStatus *); + IOReturn status(NativeAdrenoAbiStatus *); + /* Optional constant device facts; absent means the method stays unsupported. */ + IOReturn attachDeviceInfo(const NativeAdrenoAbiDeviceInfo *); + IOReturn deviceInfo(NativeAdrenoAbiDeviceInfo *); + bool mapped(uint64_t handle); + /* Backing object of a live handle, for the IOKit wrapper's clientMemoryForType and for kernel self-tests; + * null for an unknown or unreleasable handle. The client keeps ownership. */ + NativeAdrenoMemory *memoryOf(uint64_t handle); +}; +#endif diff --git a/drivers/NativeAdreno/NativeAdrenoVm.cpp b/drivers/NativeAdreno/NativeAdrenoVm.cpp new file mode 100644 index 0000000..3411ce7 --- /dev/null +++ b/drivers/NativeAdreno/NativeAdrenoVm.cpp @@ -0,0 +1,262 @@ +/* SPDX-License-Identifier: MIT */ +#include "NativeAdrenoVm.h" +extern "C" volatile uint32_t native_adreno_trace_stage; +extern "C" {int (*native_adreno_dram_read32)(uint64_t,uint32_t *);} +#include +#include +static bool overlap(uint64_t a,uint64_t ab,uint64_t b,uint64_t bb) +{return aquarantined() && + r.tables->deviceReferences() && r.tables->spanCount()==1 && + r.tables->spans()[0].bytes==r.tables->bytes() && !r.tables->spans()[0].offset && + r.arena->mapping==r.tables->mapping() && r.arena->upper_region==upper && + r.arena->config.table_physical==r.tables->spans()[0].physical && + r.arena->capacity_pages==r.tables->bytes()/4096 && r.state->prepared && + !r.state->poisoned && !r.state->tables.leaf_pages; +} +ArmSmmu4KLiveRegion NativeAdrenoVm::region(unsigned upper) const +{ + return {regions_[upper].arena,regions_[upper].state, + ranges_+(upper?lower_count_:0),upper?upper_count_:lower_count_}; +} +ArmSmmu4KStatus NativeAdrenoVm::observe() const +{ + ArmSmmu4KStreamResult routes; + ArmSmmu4KStatus status=ArmSmmu4KObserveStreams(&io_,streams_,context_,&routes);if(status)return status; + ArmSmmu4KLiveRegion lower=region(0),upper=region(1);ArmSmmu4KLiveResult audit; + status=ArmSmmu4KLiveAuditContext(context_,lower.arena?&lower:nullptr,upper.arena?&upper:nullptr,&audit); + if(status)return status; + return ArmSmmu4KObserveStreams(&io_,streams_,context_,&routes); +} +IOReturn NativeAdrenoVm::fail(ArmSmmu4KStatus status) +{ + poisoned_=true;result_.operation=status;result_.complete=false; + for(size_t i=0;mappings_ && iquarantine(); + for(auto &r:regions_)if(r.arena) {r.tables->quarantine();r.state->poisoned=1;} + ArmSmmu4KStreamResult routes; + result_.quarantine=ArmSmmu4KFaultStreams(&io_,streams_,&routes); + return kIOReturnIOError; +} +IOReturn NativeAdrenoVm::attach(const ArmSmmu4KIo *io,ArmSmmu4KContextConfig *context, + ArmSmmu4KStreamState *streams,const NativeAdrenoVmRegion *lower,const NativeAdrenoVmRegion *upper, + size_t capacity,unsigned budget) +{ + if(attached_ || ranges_ || poisoned_)return kIOReturnBusy; + if(!io || !context || !streams || (!lower && !upper) || !capacity || capacity>65536 || + !budget || budget>1048576 || (lower && !validBinding(*lower,0)) || (upper && !validBinding(*upper,1))) + return kIOReturnBadArgument; + ranges_=IONewData(ArmSmmu4KRange,capacity);if(!ranges_)return kIOReturnNoMemory; + mappings_=IONewData(Mapping,NativeAdrenoVmMaxObjects); + if(!mappings_) {IODeleteData(ranges_,ArmSmmu4KRange,capacity);return kIOReturnNoMemory;} + memset(mappings_,0,sizeof(Mapping)*NativeAdrenoVmMaxObjects); + capacity_=capacity;io_=*io;context_=context;streams_=streams;budget_=budget; + if(lower)regions_[0]=*lower;if(upper)regions_[1]=*upper; + for(unsigned i=0;i<2;++i)if(regions_[i].arena) { + if(NativeAdrenoBindSmmuVisibility(regions_[i].tables,&visibility_[i])!=kIOReturnSuccess) { + IODeleteData(ranges_,ArmSmmu4KRange,capacity_);IODeleteData(mappings_,Mapping,NativeAdrenoVmMaxObjects); + regions_[0]={};regions_[1]={};return kIOReturnBadArgument; + } + } + attached_=true; + ArmSmmu4KStatus status=observe();if(status)return fail(status); + return kIOReturnSuccess; +} +IOReturn NativeAdrenoVm::detach() +{ + if(poisoned_ || objects_ || spanCount())return kIOReturnBusy; + if(!attached_)return kIOReturnNotReady; + IODeleteData(ranges_,ArmSmmu4KRange,capacity_);IODeleteData(mappings_,Mapping,NativeAdrenoVmMaxObjects); + attached_=false;regions_[0]={};regions_[1]={};context_=nullptr;streams_=nullptr;capacity_=0; + return kIOReturnSuccess; +} +IOReturn NativeAdrenoVm::findAddress(unsigned upper,size_t bytes,uint64_t alignment,uint64_t *out) const +{ + if(!out)return kIOReturnBadArgument;*out=0; + if(!attached_ || poisoned_)return kIOReturnNotReady; + if(upper>1 || !regions_[upper].arena || !bytes || (bytes&4095) || alignment<4096 || + (alignment&(alignment-1)))return kIOReturnBadArgument; + const ArmSmmu4KLiveArena *a=regions_[upper].arena; + uint64_t offset=0; + for(size_t i=upper?lower_count_:0;;++i) { + uint64_t rem=(a->iova+offset)&(alignment-1),padding=rem?alignment-rem:0; + if(padding>a->bytes-offset)return kIOReturnNoResources;offset+=padding; + if(bytes>a->bytes-offset)return kIOReturnNoResources; + if(i==(upper?spanCount():lower_count_) || + (ranges_[i].iova-a->iova>=offset && bytes<=ranges_[i].iova-a->iova-offset)) + {*out=a->iova+offset;return kIOReturnSuccess;} + uint64_t end=ranges_[i].iova-a->iova+ranges_[i].bytes; + if(end>offset)offset=end; + } +} +IOReturn NativeAdrenoVm::preflight(const NativeAdrenoMemory *memory,unsigned upper,uint64_t iova, + unsigned permissions,unsigned writeback,unsigned shareability) const +{ + if(!memory || upper>1 || !regions_[upper].arena || !memory->backed() || memory->quarantined() || + !memory->bytes() || !memory->spanCount() || !(permissions&ArmSmmu4KRead) || (permissions&~15u) || + writeback>1 || (shareability!=2 && shareability!=3))return kIOReturnBadArgument; + const ArmSmmu4KLiveArena *a=regions_[upper].arena; + if((iova&4095) || iovaiova || iova-a->iova>=a->bytes || memory->bytes()>a->bytes-(iova-a->iova)) + return kIOReturnBadArgument; + if(objects_>=NativeAdrenoVmMaxObjects || memory->spanCount()>capacity_-spanCount())return kIOReturnNoResources; + for(size_t i=0;ibytes()-1))return kIOReturnBusy; + } + if(a->config.output_bits<32 || a->config.output_bits>48)return kIOReturnBadArgument; + uint64_t offset=0,limit=UINT64_C(1)<config.output_bits; + for(size_t j=0;jspanCount();++j) { + const auto &span=memory->spans()[j]; + if(span.offset!=offset || !span.physical || !span.bytes || ((span.physical|span.bytes)&4095) || + span.bytes>memory->bytes()-offset || span.physical>=limit || span.bytes>limit-span.physical) + return kIOReturnBadArgument; + for(const auto &r:regions_)if(r.arena && overlap(span.physical,span.bytes, + r.arena->config.table_physical,r.arena->capacity_pages*UINT64_C(4096)))return kIOReturnBadArgument; + for(size_t i=0;ibytes()?kIOReturnSuccess:kIOReturnBadArgument; +} +NativeAdrenoVm::Mapping *NativeAdrenoVm::find(uint64_t handle) +{if(handle && mappings_)for(size_t i=0;iat;--i)ranges_[i]=ranges_[i-1];ranges_[at]=range; + if(upper)++upper_count_;else ++lower_count_; +} +void NativeAdrenoVm::erase(size_t at,unsigned upper) +{ + size_t count=spanCount();for(size_t i=at+1;iholdForDevice();if(checked)return checked; + Mapping *record=nullptr;for(size_t i=0;iquarantine();return fail(ArmSmmu4KStateMismatch);} + *record={memory,0,iova,memory->spanCount(),0,permissions,writeback,shareability,0};++objects_; + checked=memory->publishForDevice(0,memory->bytes(),permissions&ArmSmmu4KWrite); + if(checked)return fail(ArmSmmu4KIoError); + for(size_t i=0;ispans;++i) { + const auto &span=memory->spans()[i]; + ArmSmmu4KRange range={iova+span.offset,span.physical,span.bytes,permissions,writeback,shareability}; + ArmSmmu4KLiveRegion lower=region(0),high=region(1);ArmSmmu4KLiveResult update; + status=ArmSmmu4KLiveUpdateContext(&io_,streams_->bank,context_,lower.arena?&lower:nullptr, + high.arena?&high:nullptr,upper,&visibility_[upper],&range,1,budget_,&update); + if(status)return fail(status); + insert(range,upper);++record->mapped_spans;++result_.completed_spans; + } + status=observe();if(status)return fail(status); + record->handle=next_handle_++;*handle=record->handle;result_.complete=true;return kIOReturnSuccess; +} +IOReturn NativeAdrenoVm::unmap(uint64_t handle) +{ + native_adreno_trace_stage=10; + result_={};if(!attached_ || poisoned_)return kIOReturnNotReady; + Mapping *record=find(handle);if(!record)return kIOReturnBadArgument; + if(record->jobs)return kIOReturnBusy; + ArmSmmu4KStatus status=observe(); + native_adreno_trace_stage=11; + if(status)return fail(status); + unsigned upper=regions_[1].arena && record->iova>=regions_[1].arena->iova; + if(record->permissions&ArmSmmu4KWrite) { + if(record->memory->acquireDeviceWrites(0,record->memory->bytes())!=kIOReturnSuccess) + return fail(ArmSmmu4KIoError); + } + for(size_t i=0;ispans;++i) { + const auto &span=record->memory->spans()[i]; + ArmSmmu4KRange change={record->iova+span.offset,span.physical,span.bytes,record->permissions,record->writeback,record->shareability}; + size_t at=0;while(atbank,context_,lower.arena?&lower:nullptr, + high.arena?&high:nullptr,upper,&visibility_[upper],&change,0,budget_,&update); + native_adreno_trace_stage=13; + if(status)return fail(status); + erase(at,upper);--record->mapped_spans;++result_.completed_spans; + } + status=observe(); + native_adreno_trace_stage=14; + if(status)return fail(status); + if(record->memory->retireAfterQuiescence()!=kIOReturnSuccess)return fail(ArmSmmu4KStateMismatch); + *record={};--objects_;result_.complete=true;return kIOReturnSuccess; +} +IOReturn NativeAdrenoVm::describe(uint64_t handle,uint64_t *iova,size_t *bytes, + unsigned *permissions) const +{ + if(!iova || !bytes || !permissions)return kIOReturnBadArgument; + *iova=0;*bytes=0;*permissions=0; + if(!attached_ || poisoned_)return kIOReturnNotReady; + for(size_t m=0;mappings_ && mbytes();*permissions=mapping.permissions; + return mapping.mapped_spans==mapping.spans?kIOReturnSuccess:kIOReturnNotReady; + } + return kIOReturnBadArgument; +} +IOReturn NativeAdrenoVm::invalidateTlb() +{ + if(!attached_ || poisoned_)return kIOReturnNotReady; + ArmSmmu4KProgramResult r; + const ArmSmmu4KStatus status=ArmSmmu4KInvalidateAllOwnedContext(&io_,streams_->bank,context_,budget_,&r); + return status?fail(status):kIOReturnSuccess; +} +IOReturn NativeAdrenoVm::publishMapped() +{ + if(!attached_ || poisoned_)return kIOReturnNotReady; + for(size_t i=0;iimported())continue; + if(m.memory->publishForDevice(0,m.memory->bytes(),true))return fail(ArmSmmu4KIoError); + } + return kIOReturnSuccess; +} +IOReturn NativeAdrenoVm::sampleWord(uint64_t iova,uint32_t *cached,uint32_t *dram) const +{ + if(!cached || !dram)return kIOReturnBadArgument; + *cached=*dram=~0u; + if(!attached_ || poisoned_)return kIOReturnNotReady; + if(iova&3)return kIOReturnBadArgument; + for(size_t i=0;i=m.memory->bytes())continue; + const size_t offset=(size_t)(iova-m.iova); + if(m.memory->mapping())*cached=static_cast(m.memory->mapping())[offset/4]; + for(size_t j=0;jspanCount();++j) { + const auto &span=m.memory->spans()[j]; + if(offset=span.bytes)continue; + uint32_t value=0; + if(native_adreno_dram_read32 && !native_adreno_dram_read32(span.physical+(offset-span.offset),&value))*dram=value; + break; + } + return kIOReturnSuccess; + } + return kIOReturnBadArgument; +} +IOReturn NativeAdrenoVm::holdJob(uint64_t handle) +{ + if(!attached_ || poisoned_)return kIOReturnNotReady; + Mapping *record=find(handle);if(!record)return kIOReturnBadArgument; + if(record->jobs==UINT32_MAX)return kIOReturnNoResources; + IOReturn status=record->memory->holdForDevice();if(status)return status; + ++record->jobs;return kIOReturnSuccess; +} +IOReturn NativeAdrenoVm::retireJobAfterCompletion(uint64_t handle) +{ + if(!attached_ || poisoned_)return kIOReturnNotReady; + Mapping *record=find(handle);if(!record || !record->jobs)return kIOReturnBadArgument; + IOReturn status=record->memory->retireAfterQuiescence();if(status)return status; + --record->jobs;return kIOReturnSuccess; +} diff --git a/drivers/NativeAdreno/NativeAdrenoVm.h b/drivers/NativeAdreno/NativeAdrenoVm.h new file mode 100644 index 0000000..940c112 --- /dev/null +++ b/drivers/NativeAdreno/NativeAdrenoVm.h @@ -0,0 +1,95 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef NATIVE_ADRENO_VM_H +#define NATIVE_ADRENO_VM_H +#include "NativeAdrenoSmmuMemory.h" +#include "ArmSmmu4KStreams.h" +struct NativeAdrenoVmRegion { + NativeAdrenoMemory *tables; + ArmSmmu4KLiveArena *arena; + ArmSmmu4KLiveState *state; +}; +struct NativeAdrenoVmResult { + ArmSmmu4KStatus operation,quarantine; + size_t completed_spans; + bool complete; +}; +/* Kernel-owned BO mapping transactions, not a userspace ABI or GPU scheduler. + * Attach only under the actual bank/ASID/stream and powered-MMIO lease, with + * fault IRQ handling installed and empty native4K arenas already active. + * The real table allocations, context/streams and BO objects remain retained + * by the owning driver. No implicit destructor cleanup; detach must succeed. + * Serialize all methods, table updates and submission under the DRIVER gate. + * ALL masters must stay genuinely quiesced through each map/unmap transaction. + * Job counts are lifetime accounting; neither they nor these APIs are fences. + * Each active region is audited, including the unmodified peer. Partial failure + * keeps every old/new BO reference, poisons both arenas and attempts to FAULT + * only the owned streams. Uncertain resources require independent real reset. + * No VM may be advertised to apps until per-client isolation, submission + * validation/protection, genuine IRQ fences and reset are implemented. */ +enum : size_t {NativeAdrenoVmMaxObjects=1024}; +/* Board hook for sampleWord(): reads one aligned word of physical memory through a fresh uncached mapping; 0 on success. */ +extern "C" int (*native_adreno_dram_read32)(uint64_t physical,uint32_t *value); +class NativeAdrenoVm final { + struct Mapping { + NativeAdrenoMemory *memory=nullptr; + uint64_t handle=0,iova=0; + size_t spans=0,mapped_spans=0; + unsigned permissions=0,writeback=0,shareability=0,jobs=0; + }; + Mapping *mappings_=nullptr; + NativeAdrenoVmRegion regions_[2]{}; + ArmSmmu4KLiveVisibility visibility_[2]{}; + ArmSmmu4KIo io_{}; + ArmSmmu4KContextConfig *context_=nullptr; + ArmSmmu4KStreamState *streams_=nullptr; + ArmSmmu4KRange *ranges_=nullptr; + size_t capacity_=0,lower_count_=0,upper_count_=0,objects_=0; + unsigned budget_=0; + uint64_t next_handle_=1; + bool attached_=false,poisoned_=false; + NativeAdrenoVmResult result_{}; + ArmSmmu4KLiveRegion region(unsigned) const; + ArmSmmu4KStatus observe() const; + IOReturn fail(ArmSmmu4KStatus); + Mapping *find(uint64_t); + void insert(const ArmSmmu4KRange &,unsigned); + void erase(size_t,unsigned); + IOReturn preflight(const NativeAdrenoMemory *,unsigned,uint64_t,unsigned,unsigned,unsigned) const; +public: + NativeAdrenoVm()=default; + ~NativeAdrenoVm()=default; + NativeAdrenoVm(const NativeAdrenoVm &)=delete; + NativeAdrenoVm &operator=(const NativeAdrenoVm &)=delete; + IOReturn attach(const ArmSmmu4KIo *,ArmSmmu4KContextConfig *,ArmSmmu4KStreamState *, + const NativeAdrenoVmRegion *lower,const NativeAdrenoVmRegion *upper, + size_t max_spans,unsigned poll_budget); + /* Frees CPU metadata only, with no BOs remaining. Never releases roots, + * disables a bank or returns the platform lease; board lifecycle owns it. */ + IOReturn detach(); + IOReturn findAddress(unsigned upper,size_t bytes,uint64_t alignment,uint64_t *iova) const; + IOReturn map(NativeAdrenoMemory *,unsigned upper,uint64_t iova, + unsigned permissions,unsigned writeback,unsigned shareability,uint64_t *handle); + IOReturn unmap(uint64_t handle); + IOReturn holdJob(uint64_t handle); + /* TLBIALL for the context bank plus TLBSYNC: drops every cached translation of the bank whatever its ASID. Unmap already + * synchronizes by ASID; this is the stronger form the submit path uses before each submission (int-070: on sheng the CP read + * zeros from an IB whose table entry and DRAM were both correct, after the address had been mapped to other pages before). */ + IOReturn invalidateTlb(); + /* Cleans and invalidates the CPU caches over every mapped kernel-owned BO (whole objects; imported BOs are their owner's + * business), so no CPU cache line can hold data the GPU should see or hide what it wrote. Lines are physically tagged, so + * this also reaches lines a user mapping of the same pages left behind. int-070 runs it before every submission. */ + IOReturn publishMapped(); + /* Diagnostic, never a proof of coherence: the 32-bit word at a mapped GPU address as the kernel's cacheable alias reads it + * and, when the board installed native_adreno_dram_read32, as DRAM reads it through an uncached mapping (else ~0). */ + IOReturn sampleWord(uint64_t iova,uint32_t *cached,uint32_t *dram) const; + /* Read-only description of a live mapping; never pins or extends it. */ + IOReturn describe(uint64_t handle,uint64_t *iova,size_t *bytes,unsigned *permissions) const; + /* Only a scheduler's actual successful GPU completion may call this. + * A timeout, interrupt bit alone or CPU-written timestamp is insufficient. */ + IOReturn retireJobAfterCompletion(uint64_t handle); + bool poisoned() const {return poisoned_;} + size_t objectCount() const {return objects_;} + size_t spanCount() const {return lower_count_+upper_count_;} + NativeAdrenoVmResult lastResult() const {return result_;} +}; +#endif diff --git a/drivers/NativeAdreno/tests/XnuMemoryFixture.h b/drivers/NativeAdreno/tests/XnuMemoryFixture.h new file mode 100644 index 0000000..65ed4dc --- /dev/null +++ b/drivers/NativeAdreno/tests/XnuMemoryFixture.h @@ -0,0 +1,106 @@ +/* SPDX-License-Identifier: MIT */ +/* Deliberately small authored host doubles, not original XNU headers or ABI. */ +#ifndef NATIVE_ADRENO_XNU_MEMORY_FIXTURE_H +#define NATIVE_ADRENO_XNU_MEMORY_FIXTURE_H +#include +#include +#include +#include +#include +#include +using IOReturn=int; +using IOOptionBits=unsigned; +using IOByteCount=size_t; +using addr64_t=uint64_t; +constexpr IOReturn kIOReturnSuccess=0,kIOReturnBadArgument=1,kIOReturnBusy=2, + kIOReturnNoResources=3,kIOReturnNoMemory=4,kIOReturnUnsupported=5, + kIOReturnNotReady=6,kIOReturnIOError=7,kIOReturnTimeout=8; +constexpr unsigned kIODirectionInOut=3,kIOMemoryMapperNone=0x800,kIOMemoryPhysicallyContiguous=0x10; +inline void *kernel_task=reinterpret_cast(1); +using task_t=void *; +enum SegmentMode {Contiguous,Scattered,ZeroPhysical,UnalignedPhysical,ShortSegment,HighPhysical,Paired}; +class IOBufferMemoryDescriptor; +struct MemoryFixture { + unsigned created=0,prepared=0,completed=0,released=0,cache_calls=0,segment_calls=0; + unsigned allocation_fail_count=0; + bool allocation_fail=false,prepare_fail=false,complete_fail=false,metadata_fail=false; + bool null_mapping=false,unaligned_mapping=false,wrong_length=false; + unsigned cache_fail_call=0; + uint64_t next_physical=0x30000000; + unsigned imported=0,imports_live=0; + IOReturn cpu_status=kIOReturnSuccess; + SegmentMode mode=Contiguous; + std::vector live; + std::vector metadata; + std::vector cache_operations; +}; +inline MemoryFixture fixture; +inline void *fixtureMetadata(size_t bytes) { + if(fixture.metadata_fail)return nullptr; + void *p=std::calloc(1,bytes);if(p)fixture.metadata.push_back(p);return p; +} +inline void fixtureMetadataFree(void *p) { + auto at=std::find(fixture.metadata.begin(),fixture.metadata.end(),p); + if(at==fixture.metadata.end())std::abort();fixture.metadata.erase(at);std::free(p); +} +#define IONewData(type,count) static_cast(fixtureMetadata(sizeof(type)*(count))) +#define IODeleteData(ptr,type,count) do {fixtureMetadataFree(ptr);ptr=nullptr;}while(0) +class IOBufferMemoryDescriptor { +public: + void *raw=nullptr; + size_t bytes=0; + uint64_t mask=0; + uint64_t physical_base=0; + static IOBufferMemoryDescriptor *inTaskWithPhysicalMask(void *task,unsigned options,size_t bytes,uint64_t mask) { + if(task!=kernel_task || !(options&kIOMemoryMapperNone))std::abort(); + if(fixture.allocation_fail)return nullptr; + if(fixture.allocation_fail_count) {--fixture.allocation_fail_count;return nullptr;} + auto *d=new IOBufferMemoryDescriptor; + d->bytes=bytes;d->mask=mask;d->physical_base=fixture.next_physical; + fixture.next_physical+=2*bytes+0x10000; + if(posix_memalign(&d->raw,4096,bytes+4096))std::abort(); + std::memset(d->raw,0xa5,bytes+4096);++fixture.created;fixture.live.push_back(d);return d; + } + IOReturn prepare() {++fixture.prepared;return fixture.prepare_fail?kIOReturnIOError:kIOReturnSuccess;} + IOReturn complete() {++fixture.completed;return fixture.complete_fail?kIOReturnIOError:kIOReturnSuccess;} + void release() { + ++fixture.released;auto at=std::find(fixture.live.begin(),fixture.live.end(),this); + if(at==fixture.live.end())std::abort();fixture.live.erase(at);std::free(raw);delete this; + } + void *getBytesNoCopy() {return fixture.null_mapping?nullptr:static_cast(raw)+(fixture.unaligned_mapping?8:0);} + size_t getLength() {return bytes+(fixture.wrong_length?4096:0);} + uint64_t getPhysicalSegment(size_t offset,size_t *length,unsigned options) { + if((offset&4095) || offset>=bytes || options!=kIOMemoryMapperNone)std::abort(); + ++fixture.segment_calls;*length=bytes-offset; + uint64_t base=physical_base,physical=base+offset; + if(fixture.mode==Scattered)physical=base+2*offset; + if(fixture.mode==Paired)physical=base+offset+((offset/8192)*4096); + if(offset==4096) { + if(fixture.mode==ZeroPhysical)physical=0; + if(fixture.mode==UnalignedPhysical)++physical; + if(fixture.mode==ShortSegment)*length=4095; + if(fixture.mode==HighPhysical)physical=(mask|4095)+1; + } + return physical; + } +}; +/* A user range pinned with withAddressRange/prepare: pages at a fixed physical stride (every other page), so imports are scattered. */ +class IOMemoryDescriptor { +public: + uint64_t address=0,bytes=0,physical_base=0; + static IOMemoryDescriptor *withAddressRange(uint64_t address,uint64_t bytes,unsigned options,task_t task) { + if(!task || task==kernel_task || !(options&kIOMemoryMapperNone))std::abort(); + if(fixture.allocation_fail)return nullptr; + auto *d=new IOMemoryDescriptor;d->address=address;d->bytes=bytes;d->physical_base=fixture.next_physical; + fixture.next_physical+=2*bytes+0x10000;++fixture.imported;++fixture.imports_live;return d; + } + IOReturn prepare() {++fixture.prepared;return fixture.prepare_fail?kIOReturnIOError:kIOReturnSuccess;} + IOReturn complete() {++fixture.completed;return fixture.complete_fail?kIOReturnIOError:kIOReturnSuccess;} + void release() {--fixture.imports_live;delete this;} + uint64_t getLength() {return bytes;} + uint64_t getPhysicalSegment(size_t offset,size_t *length,unsigned options) { + if((offset&4095) || offset>=bytes || options!=kIOMemoryMapperNone)std::abort(); + ++fixture.segment_calls;*length=4096;return physical_base+2*offset; + } +}; +#endif diff --git a/drivers/NativeAdreno/tests/hostsim/host_gpu_service.cpp b/drivers/NativeAdreno/tests/hostsim/host_gpu_service.cpp new file mode 100644 index 0000000..4a0eeaf --- /dev/null +++ b/drivers/NativeAdreno/tests/hostsim/host_gpu_service.cpp @@ -0,0 +1,130 @@ +/* SPDX-License-Identifier: MIT */ +/* The kernel half of the NativeAdreno stack on the host: the real GPU context bring-up (software SMMU register peer), VM, ring, + * submit object and user-client core over the authored memory doubles, with a software "CP" that retires every submission at + * once. It runs no shader and touches no hardware; it exists so the Turnip darwin backend can be driven through a real + * Vulkan instance before the userspace side ever runs on the tablet. Narrow C interface for the IOKit shim translation unit. */ +#include "XnuMemoryFixture.h" +#include "NativeAdrenoGpuContext.h" +#include "NativeAdrenoUserClient.h" +#include "NativeAdrenoTimer.h" +#include +#include +#include + +IOReturn NativeAdrenoQueryCpu4K(NativeAdrenoCpuState *state) {*state={};return kIOReturnSuccess;} +IOReturn NativeAdrenoMaintainOwnedPages(void *,size_t,NativeAdrenoCacheOperation) {return kIOReturnSuccess;} + +namespace { +constexpr size_t kSmmuBytes=0x40000; +alignas(8) uint32_t mmio[kSmmuBytes/4]; +int sread32(void *,size_t at,uint32_t *v) {*v=mmio[at/4];return 0;} +int sread64(void *,size_t at,uint64_t *v) {std::memcpy(v,mmio+at/4,8);return 0;} +int swrite32(void *,size_t at,uint32_t v) { + if((at&0xfff)==0x58 && at>=0x20000) {mmio[at/4]&=~v;return 0;} /* context FSR is write-one-to-clear */ + mmio[at/4]=v;return 0; +} +int swrite64(void *,size_t at,uint64_t v) {std::memcpy(mmio+at/4,&v,8);return 0;} +int sbarrier(void *) {return 0;} + +/* Software CP: a write of the ring write pointer retires everything up to it, and the fence the ring reads is the number of + * submissions so far (one write pointer update per submission). */ +std::vector regs(0x10000); +volatile uint32_t *memptrs=nullptr; +unsigned submissions=0; +int cwrite(void *,uint32_t reg,uint32_t v) { + regs[reg]=v; + if(reg==0x0807) {regs[0x0806]=v;++submissions;if(memptrs)memptrs[1]=submissions;} + return 0; +} +int cread(void *,uint32_t reg,uint32_t *v) {*v=regs[reg];return 0;} +int cbarrier(void *) {return 0;} +int cpublish(void *,const volatile uint32_t *,size_t) {return 0;} +int cacquire(void *,volatile uint32_t *,size_t) {return 0;} +uint64_t cnow(void *) { + return uint64_t(std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()).count()); +} +int cidle(void *) {std::this_thread::sleep_for(std::chrono::microseconds(50));return 0;} +int timeread(void *,uint32_t address,uint32_t *value) { + static uint64_t ticks=1000; + if(address==0x0980)ticks+=1000; + *value=address==0x0980?uint32_t(ticks):(address==0x0981?uint32_t(ticks>>32):0);return 0; +} +const NativeAdrenoAbiDeviceInfo info={0x43050a01u,3u<<20,0x1000000ull,0x1fffffffff000ull,0x20000000ull,0x60000000ull, + 16,6,1,4,NativeAdrenoDeviceClientIova,0}; +const NativeAdrenoRingIo ringio{nullptr,cwrite,cread,cbarrier}; +const NativeAdrenoRingCache ringcache{nullptr,cpublish,cacquire}; +const NativeAdrenoRingClock clock_{nullptr,cnow,cidle}; +const NativeAdrenoTimeIo timeio{nullptr,timeread}; + +NativeAdrenoGpuContext *context; +NativeAdrenoRing ring; +NativeAdrenoSubmit execbuf; +NativeAdrenoUserClient client; +NativeAdrenoTimer timer; +bool built=false,opened=false; + +bool build() { + if(built)return true; + std::memset(mmio,0,sizeof(mmio)); + mmio[0]=0x002f0406;mmio[0x20/4]=0x4c017e18;mmio[0x24/4]=0x40000016;mmio[0x28/4]=0x1433; + for(unsigned i=0;i<24;++i)mmio[(0xc00+4*i)/4]=0x20000; + static const ArmSmmu4KIo io{nullptr,kSmmuBytes,sread32,sread64,swrite32,swrite64,sbarrier}; + context=new NativeAdrenoGpuContext; + NativeAdrenoGmuContextReport report; + const unsigned slots[2]={2,3}; + const uint16_t sids[2]={0,1}; + const IOReturn up=context->bringUp(&io,1,2,slots,sids,nullptr,&report); + if(up!=kIOReturnSuccess) {std::fprintf(stderr,"host gpu: context bring-up failed %d at step %u\n",up,report.step);return false;} + const NativeAdrenoMemory &ringmem=context->buffer(1),&memory=context->buffer(2); + memptrs=static_cast(memory.mapping()); + if(ring.attach(&ringio,&ringcache,ringmem.mapping(),ringmem.bytes(),GpuRingIova,memory.mapping(),memory.bytes(), + GpuMemptrsIova)!=kIOReturnSuccess) {std::fprintf(stderr,"host gpu: ring attach failed\n");return false;} + if(execbuf.attach(context->vm(),&ring)!=kIOReturnSuccess) {std::fprintf(stderr,"host gpu: submit attach failed\n");return false;} + if(timer.attach(&timeio,NativeAdrenoTimeCp)!=kIOReturnSuccess)return false; + /* Ring attach already wrote the write pointer once (the ring's own start-up packets); only submissions after this count as fences. */ + submissions=0;memptrs[1]=0; + built=true; + return true; +} +IOReturn translate(IOReturn status) {return status;} +} + +extern "C" { +/* Return codes are the fixture's IOReturn values; the shim translates them. */ +int host_gpu_open() { + if(!build() || opened)return kIOReturnBusy; + const IOReturn st=client.open(NativeAdrenoAbiVersion,context->vm(),&execbuf,&ring,31,40); + if(st) {std::fprintf(stderr,"host gpu: client open failed %d\n",st);return kIOReturnNotReady;} + client.attachTimer(&timer); + client.attachDeviceInfo(&info); + opened=true; + return kIOReturnSuccess; +} +int host_gpu_close() { + if(!opened)return kIOReturnNotReady; + const IOReturn status=client.releaseAll(&clock_,2000000); + if(!status)opened=false; + return translate(status); +} +int host_gpu_call(uint32_t method,const void *in,size_t in_size,void *out,size_t out_size) { + if(!opened)return kIOReturnNotReady; + /* As the IOKit wrapper does (NativeAdrenoPublish.cpp): an import is the calling task's pages, here a stand-in task. */ + if(method==NativeAdrenoMethodBoImport) { + if(!in || in_size!=sizeof(NativeAdrenoAbiBoImport) || !out || out_size!=sizeof(NativeAdrenoAbiBoAllocReply))return kIOReturnBadArgument; + NativeAdrenoAbiBoImport request{};std::memcpy(&request,in,sizeof(request)); + NativeAdrenoAbiBoAllocReply reply{}; + const IOReturn status=client.importBo(&request,reinterpret_cast(0x1000),&reply); + if(!status)std::memcpy(out,&reply,sizeof(reply)); + return translate(status); + } + return translate(client.dispatch(method,in,in_size,out,out_size,&clock_)); +} +int host_gpu_map(uint32_t token,void **address,uint64_t *bytes) { + NativeAdrenoMemory *memory=opened?client.memoryForToken(token):nullptr; + if(!memory)return kIOReturnBadArgument; + *address=memory->mapping();*bytes=memory->bytes();return kIOReturnSuccess; +} +unsigned host_gpu_submissions() {return submissions;} +unsigned host_gpu_live_objects() {return unsigned(client.liveObjects());} +} diff --git a/drivers/NativeAdreno/tests/hostsim/host_iokit_shim.cpp b/drivers/NativeAdreno/tests/hostsim/host_iokit_shim.cpp new file mode 100644 index 0000000..be24acf --- /dev/null +++ b/drivers/NativeAdreno/tests/hostsim/host_iokit_shim.cpp @@ -0,0 +1,104 @@ +/* SPDX-License-Identifier: MIT */ +/* IOKit entry points the Turnip darwin backend uses, answered by the host copy of the kernel stack. Inserted with DYLD_INSERT_LIBRARIES + * and redirected by dyld interposing. Never built for the tablet. */ +#include +#include +#include +#include +#include +#include +#include +static bool trace(){static int t=-1;if(t<0)t=getenv("HOSTSIM_TRACE")?1:0;return t;} + +extern "C" { +int host_gpu_open(); +int host_gpu_close(); +int host_gpu_call(uint32_t,const void *,size_t,void *,size_t); +int host_gpu_map(uint32_t,void **,uint64_t *); +} + +static kern_return_t translate(int fixture_status) { + switch(fixture_status) { + case 0:return KERN_SUCCESS; + case 1:return kIOReturnBadArgument; + case 2:return kIOReturnBusy; + case 3:return kIOReturnNoResources; + case 4:return kIOReturnNoMemory; + case 5:return kIOReturnUnsupported; + case 6:return kIOReturnNotReady; + case 7:return kIOReturnIOError; + case 8:return kIOReturnTimeout; + default:return kIOReturnError; + } +} + +/* Two-level-namespace binding of the Turnip library to the real IOKit is redirected with dyld interposing (the flat-namespace route no longer + * takes effect on current macOS). The service answers like the tablet: "IOAcceleratorES" needs open type 0x4147 and only the tunnel through one + * admitted selector (kAdmitted, deliberately not 0), "NativeAdrenoGpuPublished" is the unrestricted direct form. */ +static const uint32_t kAdmitted = 5, kTunnelMagic = 0x4e544e4c; +static uint32_t gOpenKind; /* 1 direct, 2 tunnel only */ +#define INTERPOSE(replacement, replacee) \ + __attribute__((used)) static const struct {const void *with; const void *what;} interpose_##replacee \ + __attribute__((section("__DATA,__interpose"))) = {reinterpret_cast(&replacement), reinterpret_cast(&replacee)} + +extern "C" { +static CFMutableDictionaryRef shim_IOServiceMatching(const char *name) { + if(trace())fprintf(stderr,"shim: IOServiceMatching(%s)\n",name); + if(!strcmp(name,"NativeAdrenoGpuPublished"))return reinterpret_cast(1); + if(!strcmp(name,"IOAcceleratorES"))return reinterpret_cast(2); + return nullptr; +} +static io_service_t shim_IOServiceGetMatchingService(mach_port_t,CFDictionaryRef matching) { + if(trace())fprintf(stderr,"shim: IOServiceGetMatchingService(%p)\n",(const void *)matching); + return static_cast(reinterpret_cast(matching)); +} +static kern_return_t shim_IOServiceOpen(io_service_t service,task_port_t,uint32_t type,io_connect_t *connect) { + if(trace())fprintf(stderr,"shim: IOServiceOpen(%u, 0x%x)\n",service,type); + if(service!=1 && service!=2)return kIOReturnBadArgument; + if(service==2 && type!=0x4147)return kIOReturnNotPermitted; + const kern_return_t kr=translate(host_gpu_open()); + if(trace())fprintf(stderr,"shim: open -> 0x%x\n",kr); + if(kr==KERN_SUCCESS) {*connect=7;gOpenKind=service;} + return kr; +} +static kern_return_t shim_IOServiceClose(io_connect_t) {return translate(host_gpu_close());} +static kern_return_t shim_IOObjectRelease(io_object_t) {return KERN_SUCCESS;} +static kern_return_t shim_IOConnectCallStructMethod(mach_port_t,uint32_t selector,const void *in,size_t in_size,void *out,size_t *out_size) { + if(gOpenKind==2) { + if(selector!=kAdmitted)return kIOReturnNotPermitted; + const uint32_t *header=static_cast(in); + if(!in || in_size<8 || header[0]!=kTunnelMagic)return kIOReturnBadArgument; + if(header[1]==0xffffffffu)return KERN_SUCCESS; + /* HOSTSIM_STALL=: the first BoAlloc (the first call vkCreateDevice makes) blocks that long, to exercise the plugin's watchdog. */ + static bool stalled=false; + if(header[1]==2 && !stalled && getenv("HOSTSIM_STALL")) {stalled=true;sleep((unsigned)atoi(getenv("HOSTSIM_STALL")));} + const int status=host_gpu_call(header[1],static_cast(in)+8,in_size-8,out,out_size?*out_size:0); + if(trace())fprintf(stderr,"shim: method %u first word 0x%llx -> %d\n",header[1],in_size>=16?(unsigned long long)*reinterpret_cast(static_cast(in)+8):0ull,status); + if(trace() && status)fprintf(stderr,"shim: method %u (in %zu, out %zu) -> fixture status %d\n",header[1],in_size-8,out_size?*out_size:0,status); + return translate(status); + } + return translate(host_gpu_call(selector,in,in_size,out,out_size?*out_size:0)); +} +static kern_return_t shim_IOConnectMapMemory64(io_connect_t,uint32_t type,task_port_t,mach_vm_address_t *address,mach_vm_size_t *size,IOOptionBits) { + void *mapping=nullptr;uint64_t bytes=0; + const kern_return_t kr=translate(host_gpu_map(type,&mapping,&bytes)); + if(kr==KERN_SUCCESS) {*address=reinterpret_cast(mapping);*size=bytes;} + return kr; +} +static kern_return_t shim_IOConnectUnmapMemory64(io_connect_t,uint32_t,task_port_t,mach_vm_address_t) {return KERN_SUCCESS;} +/* HOSTSIM_PAGE_SIZE=4096: Turnip sizes and places BOs with the page size (os_get_page_size, getpagesize), 16 KiB on this Mac and 4 KiB on the + * tablet, so a recording meant to match the tablet's GPU address layout reports the tablet's page size. */ +static long pageSize(){static long p=-1;if(p<0)p=getenv("HOSTSIM_PAGE_SIZE")?atol(getenv("HOSTSIM_PAGE_SIZE")):0;return p;} +static int shim_getpagesize(void) {return pageSize()?int(pageSize()):getpagesize();} +static long shim_sysconf(int name) {return name==_SC_PAGESIZE && pageSize()?pageSize():sysconf(name);} +} +INTERPOSE(shim_getpagesize, getpagesize); +INTERPOSE(shim_sysconf, sysconf); +INTERPOSE(shim_IOServiceMatching, IOServiceMatching); +INTERPOSE(shim_IOServiceGetMatchingService, IOServiceGetMatchingService); +INTERPOSE(shim_IOServiceOpen, IOServiceOpen); +INTERPOSE(shim_IOServiceClose, IOServiceClose); +INTERPOSE(shim_IOObjectRelease, IOObjectRelease); +INTERPOSE(shim_IOConnectCallStructMethod, IOConnectCallStructMethod); +INTERPOSE(shim_IOConnectMapMemory64, IOConnectMapMemory64); +INTERPOSE(shim_IOConnectUnmapMemory64, IOConnectUnmapMemory64); diff --git a/drivers/NativeAdreno/tests/hostsim/plugin_smoke.cpp b/drivers/NativeAdreno/tests/hostsim/plugin_smoke.cpp new file mode 100644 index 0000000..2814b21 --- /dev/null +++ b/drivers/NativeAdreno/tests/hostsim/plugin_smoke.cpp @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: MIT */ +/* Runs the plugin's own Turnip smoke sequence (drivers/NativeMetal/NativeTurnipSmoke.h, the code that ships in the Metal plugin) against + * the host copy of the kernel stack. The software CP executes nothing, so the fill pattern never appears: the expected outcome is that every + * stage up to and including the fence wait passes and the run stops at stage 10 (only the untouched half of the buffer, 512 words, matches). */ +#include "NativeTurnipSmoke.h" + +int main(int argc, char **argv) { + if (argc < 2) return 2; + unsigned marks[32], count = 0, samples = 0; + auto marker = [&](uint32_t value) { if (count < 32) marks[count++] = value; }; + // The shipping path: the sequence on a worker thread under the watchdog (HOSTSIM_STALL makes the shim block the first BoAlloc). + const turnip::Result r = turnip::runWatched(marker, [&](const char *text) { printf("watchdog: %s\n", text); ++samples; }, argv[1]); + printf("watchdog samples: %u\n", samples); + printf("markers:"); + for (unsigned i = 0; i < count; ++i) printf(" %x", marks[i]); + printf("\nstage %d result %d fill %d compute %d (step %d) render %d/%d (step %d) ok %d msg '%s'\n", r.stage, r.vkResult, r.verified, + r.computeWords, r.computeStage, r.renderTriangle, r.renderOther, r.renderStage, r.ok, turnip::gFirstError); + // The software CP executes nothing: every submission retires, nothing is written. Expected: no setup failure, the fill check sees only the + // untouched half (512), the compute check sees no shader writes (0), and the render read-back keeps the 0x5a pattern (4096 stray pixels). + return r.stage == 0 && r.verified == 512 && r.computeWords == 0 && r.computeStage == 0 && r.renderStage == 0 && r.renderTriangle == 0 && + r.renderOther == 4096 ? 0 : 1; +} diff --git a/drivers/NativeAdreno/tests/hostsim/vulkan_smoke.c b/drivers/NativeAdreno/tests/hostsim/vulkan_smoke.c new file mode 100644 index 0000000..8d05f55 --- /dev/null +++ b/drivers/NativeAdreno/tests/hostsim/vulkan_smoke.c @@ -0,0 +1,151 @@ +/* SPDX-License-Identifier: MIT */ +/* Drives the Turnip darwin backend through the Vulkan ICD entry point against the host copy of the kernel stack + * (host_gpu_service.cpp via host_iokit_shim.cpp). No shader runs; the software CP only retires submissions. */ +#include +#include +#include +#include +#include +#include + +typedef PFN_vkVoidFunction (*GetProc)(VkInstance, const char *); +typedef unsigned (*Counter)(void); +#define CHECK(call) do { VkResult r_ = (call); if (r_ != VK_SUCCESS) { fprintf(stderr, "%s:%d %s -> %d\n", __FILE__, __LINE__, #call, r_); return 1; } } while (0) +#define LOAD(var, name, inst) PFN_##name var = (PFN_##name)get(inst, #name); if (!var) { fprintf(stderr, "missing %s\n", #name); return 1; } + +int main(int argc, char **argv) +{ + void *lib = dlopen(argv[1], RTLD_NOW | RTLD_LOCAL); + if (!lib) { fprintf(stderr, "dlopen: %s\n", dlerror()); return 1; } + GetProc get = (GetProc)dlsym(lib, "vk_icdGetInstanceProcAddr"); + Counter submissions = (Counter)dlsym(RTLD_DEFAULT, "host_gpu_submissions"); + Counter live = (Counter)dlsym(RTLD_DEFAULT, "host_gpu_live_objects"); + if (!get || !submissions || !live) { fprintf(stderr, "missing entry points\n"); return 1; } + + LOAD(createInstance, vkCreateInstance, NULL); + VkApplicationInfo app = {.sType = VK_STRUCTURE_TYPE_APPLICATION_INFO, .pApplicationName = "smoke", .apiVersion = VK_API_VERSION_1_1}; + VkInstanceCreateInfo ici = {.sType = VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO, .pApplicationInfo = &app}; + VkInstance instance; + CHECK(createInstance(&ici, NULL, &instance)); + + LOAD(enumerate, vkEnumeratePhysicalDevices, instance); + uint32_t count = 0; + CHECK(enumerate(instance, &count, NULL)); + if (count != 1) { fprintf(stderr, "expected one physical device, got %u\n", count); return 1; } + VkPhysicalDevice phys; + CHECK(enumerate(instance, &count, &phys)); + + LOAD(getProps, vkGetPhysicalDeviceProperties, instance); + VkPhysicalDeviceProperties props; + getProps(phys, &props); + printf("device: %s (vendor 0x%x device 0x%x)\n", props.deviceName, props.vendorID, props.deviceID); + + LOAD(getMemProps, vkGetPhysicalDeviceMemoryProperties, instance); + VkPhysicalDeviceMemoryProperties mem; + getMemProps(phys, &mem); + int hostVisible = -1; + for (uint32_t i = 0; i < mem.memoryTypeCount; i++) { + VkMemoryPropertyFlags f = mem.memoryTypes[i].propertyFlags; + if ((f & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) && (f & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT)) { hostVisible = i; break; } + } + printf("memory types: %u, host visible+coherent type: %d\n", mem.memoryTypeCount, hostVisible); + if (hostVisible < 0) return 1; + + LOAD(getQueueFamilies, vkGetPhysicalDeviceQueueFamilyProperties, instance); + uint32_t families = 0; + getQueueFamilies(phys, &families, NULL); + VkQueueFamilyProperties fam[8]; + if (families > 8) families = 8; + getQueueFamilies(phys, &families, fam); + int gfx = -1; + for (uint32_t i = 0; i < families; i++) if (fam[i].queueFlags & VK_QUEUE_GRAPHICS_BIT) { gfx = i; break; } + if (gfx < 0) return 1; + + LOAD(createDevice, vkCreateDevice, instance); + float prio = 1.0f; + VkDeviceQueueCreateInfo qci = {.sType = VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO, .queueFamilyIndex = gfx, .queueCount = 1, .pQueuePriorities = &prio}; + VkDeviceCreateInfo dci = {.sType = VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO, .queueCreateInfoCount = 1, .pQueueCreateInfos = &qci}; + VkDevice device; + CHECK(createDevice(phys, &dci, NULL, &device)); + printf("device created, kernel objects live: %u, submissions: %u\n", live(), submissions()); + + PFN_vkGetDeviceProcAddr getDeviceProc = (PFN_vkGetDeviceProcAddr)get(instance, "vkGetDeviceProcAddr"); +#define DLOAD(var, name) PFN_##name var = (PFN_##name)getDeviceProc(device, #name); if (!var) { fprintf(stderr, "missing %s\n", #name); return 1; } + DLOAD(getQueue, vkGetDeviceQueue); + DLOAD(allocMem, vkAllocateMemory); + DLOAD(mapMem, vkMapMemory); + DLOAD(unmapMem, vkUnmapMemory); + DLOAD(freeMem, vkFreeMemory); + DLOAD(createBuffer, vkCreateBuffer); + DLOAD(destroyBuffer, vkDestroyBuffer); + DLOAD(getBufReq, vkGetBufferMemoryRequirements); + DLOAD(bindBuf, vkBindBufferMemory); + DLOAD(createPool, vkCreateCommandPool); + DLOAD(destroyPool, vkDestroyCommandPool); + DLOAD(allocCmd, vkAllocateCommandBuffers); + DLOAD(beginCmd, vkBeginCommandBuffer); + DLOAD(endCmd, vkEndCommandBuffer); + DLOAD(cmdFill, vkCmdFillBuffer); + DLOAD(createFence, vkCreateFence); + DLOAD(destroyFence, vkDestroyFence); + DLOAD(waitFences, vkWaitForFences); + DLOAD(submit, vkQueueSubmit); + DLOAD(waitIdle, vkDeviceWaitIdle); + DLOAD(destroyDevice, vkDestroyDevice); + + VkQueue queue; + getQueue(device, gfx, 0, &queue); + + /* Host-visible memory: allocate, map, write, read back through the kernel-provided mapping. */ + VkMemoryAllocateInfo mai = {.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO, .allocationSize = 1 << 20, .memoryTypeIndex = hostVisible}; + VkDeviceMemory memory; + CHECK(allocMem(device, &mai, NULL, &memory)); + volatile uint32_t *ptr; + CHECK(mapMem(device, memory, 0, VK_WHOLE_SIZE, 0, (void **)&ptr)); + for (unsigned i = 0; i < 1024; i++) ptr[i] = 0x1000 + i; + unsigned bad = 0; + for (unsigned i = 0; i < 1024; i++) bad += ptr[i] != 0x1000 + i; + printf("mapped 1 MiB through the kernel descriptor, mismatches: %u\n", bad); + if (bad) return 1; + + VkBufferCreateInfo bci = {.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .size = 1 << 20, + .usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT, .sharingMode = VK_SHARING_MODE_EXCLUSIVE}; + VkBuffer buffer; + CHECK(createBuffer(device, &bci, NULL, &buffer)); + VkMemoryRequirements req; + getBufReq(device, buffer, &req); + CHECK(bindBuf(device, buffer, memory, 0)); + + VkCommandPoolCreateInfo pci = {.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO, .queueFamilyIndex = gfx}; + VkCommandPool pool; + CHECK(createPool(device, &pci, NULL, &pool)); + VkCommandBufferAllocateInfo cai = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO, .commandPool = pool, + .level = VK_COMMAND_BUFFER_LEVEL_PRIMARY, .commandBufferCount = 1}; + VkCommandBuffer cmd; + CHECK(allocCmd(device, &cai, &cmd)); + VkCommandBufferBeginInfo cbi = {.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO}; + CHECK(beginCmd(cmd, &cbi)); + cmdFill(cmd, buffer, 0, 4096, 0xdeadbeef); + CHECK(endCmd(cmd)); + + VkFenceCreateInfo fci = {.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO}; + VkFence fence; + CHECK(createFence(device, &fci, NULL, &fence)); + VkSubmitInfo si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO, .commandBufferCount = 1, .pCommandBuffers = &cmd}; + unsigned before = submissions(); + CHECK(submit(queue, 1, &si, fence)); + CHECK(waitFences(device, 1, &fence, VK_TRUE, 5000000000ull)); + printf("fill-buffer submission retired: kernel submissions %u -> %u\n", before, submissions()); + if (submissions() <= before) return 1; + + CHECK(waitIdle(device)); + destroyFence(device, fence, NULL); + destroyPool(device, pool, NULL); + destroyBuffer(device, buffer, NULL); + unmapMem(device, memory); + freeMem(device, memory, NULL); + destroyDevice(device, NULL); + printf("device destroyed, kernel objects live: %u\n", live()); + printf("PASS\n"); + return live() == 0 ? 0 : 2; +} diff --git a/drivers/NativeAdreno/tests/test_board.cpp b/drivers/NativeAdreno/tests/test_board.cpp new file mode 100644 index 0000000..52f063d --- /dev/null +++ b/drivers/NativeAdreno/tests/test_board.cpp @@ -0,0 +1,98 @@ +/* SPDX-License-Identifier: MIT */ +/* Real parser over the exact SM8550 GPU/GMU reg cells; no device access. */ +#include "XnuMemoryFixture.h" +#include "NativeAdrenoBoard.h" +#include +#include +#define CHECK(x) do {if(!(x)){std::fprintf(stderr,"line %d: %s\n",__LINE__,#x);std::exit(1);}}while(0) +static void put(uint8_t *at,uint32_t value) +{at[0]=uint8_t(value>>24);at[1]=uint8_t(value>>16);at[2]=uint8_t(value>>8);at[3]=uint8_t(value);} +static size_t reg(uint8_t *out,const uint32_t (*cells)[4],size_t windows) +{ + for(size_t i=0;i, <&adreno_smmu 1 0>. */ + { + static const uint32_t cells[][3]={{0x1a,0,0},{0x1a,1,0}}; + uint8_t raw[32]; + for(size_t i=0;i<2;++i)for(size_t c=0;c<3;++c)put(raw+i*12+c*4,cells[i][c]); + uint16_t sids[4]{}; + size_t count=0; + CHECK(NativeAdrenoBoard::parseIommus(raw,24,2,sids,4,&count)==kIOReturnSuccess); + CHECK(count==2 && sids[0]==0 && sids[1]==1); + uint16_t untouched[2]={0xbeef,0xbeef}; + CHECK(NativeAdrenoBoard::parseIommus(raw,24,2,untouched,1,&count)==kIOReturnNoResources); + CHECK(untouched[0]==0xbeef); + CHECK(NativeAdrenoBoard::parseIommus(raw,23,2,sids,4,&count)==kIOReturnBadArgument); + CHECK(NativeAdrenoBoard::parseIommus(raw,24,0,sids,4,&count)==kIOReturnBadArgument); + CHECK(NativeAdrenoBoard::parseIommus(raw,24,5,sids,4,&count)==kIOReturnBadArgument); + CHECK(NativeAdrenoBoard::parseIommus(nullptr,24,2,sids,4,&count)==kIOReturnBadArgument); + CHECK(NativeAdrenoBoard::parseIommus(raw,24,2,nullptr,4,&count)==kIOReturnNoResources); + CHECK(NativeAdrenoBoard::parseIommus(raw,24,2,sids,4,nullptr)==kIOReturnBadArgument); + uint8_t unnamed[12];put(unnamed,0);put(unnamed+4,0);put(unnamed+8,0); + CHECK(NativeAdrenoBoard::parseIommus(unnamed,12,2,sids,4,&count)==kIOReturnBadArgument); + uint8_t wide[12];put(wide,0x1a);put(wide+4,0x10000);put(wide+8,0); + CHECK(NativeAdrenoBoard::parseIommus(wide,12,2,sids,4,&count)==kIOReturnBadArgument); + } + /* Refusals precede any output: nothing is written on failure. */ + { + static const uint32_t cells[][4]={{0,0x03d00000,0,0x40000},{0,0x03d9e000,0,0x1000}}; + uint8_t raw[64];const size_t bytes=reg(raw,cells,2); + size_t count=99; + std::memset(windows,0x5a,sizeof(windows)); + CHECK(NativeAdrenoBoard::parseReg(nullptr,bytes,2,2,windows,2,&count)==kIOReturnBadArgument); + CHECK(NativeAdrenoBoard::parseReg(raw,bytes,0,2,windows,2,&count)==kIOReturnBadArgument); + CHECK(NativeAdrenoBoard::parseReg(raw,bytes,3,2,windows,2,&count)==kIOReturnBadArgument); + CHECK(NativeAdrenoBoard::parseReg(raw,bytes,2,0,windows,2,&count)==kIOReturnBadArgument); + CHECK(NativeAdrenoBoard::parseReg(raw,bytes,2,2,windows,1,&count)==kIOReturnNoResources); + CHECK(NativeAdrenoBoard::parseReg(raw,bytes+1,2,2,windows,2,&count)==kIOReturnBadArgument); + CHECK(NativeAdrenoBoard::parseReg(raw,bytes,2,2,nullptr,2,&count)==kIOReturnNoResources); + CHECK(NativeAdrenoBoard::parseReg(raw,bytes,2,2,windows,2,nullptr)==kIOReturnBadArgument); + CHECK(windows[0].base==UINT64_C(0x5a5a5a5a5a5a5a5a)); + /* A zero size and an address wrap are refused. */ + static const uint32_t zero[][4]={{0,0x03d00000,0,0}}; + uint8_t one[16];reg(one,zero,1); + CHECK(NativeAdrenoBoard::parseReg(one,16,2,2,windows,1,&count)==kIOReturnBadArgument); + uint8_t wrap[16];put(wrap,0xffffffff);put(wrap+4,0xfffff000);put(wrap+8,0);put(wrap+12,0x2000); + CHECK(NativeAdrenoBoard::parseReg(wrap,16,2,2,windows,1,&count)==kIOReturnBadArgument); + CHECK(windows[0].base==UINT64_C(0x5a5a5a5a5a5a5a5a)); + } + std::printf("PASS: 2 real SM8550 reg sets, 2 alternate cell formats, the real GPU iommu specifiers and 19 malformed/limit refusals over the device-tree parser\n"); +} diff --git a/drivers/NativeAdreno/tests/test_client.cpp b/drivers/NativeAdreno/tests/test_client.cpp new file mode 100644 index 0000000..6da0bdb --- /dev/null +++ b/drivers/NativeAdreno/tests/test_client.cpp @@ -0,0 +1,251 @@ +/* SPDX-License-Identifier: MIT */ +/* Userspace client and kernel dispatch over software SMMU/CP peers. */ +#include "XnuMemoryFixture.h" +#include "NativeAdrenoClient.h" +#include "NativeAdrenoUserClient.h" +#include +#include +#include +#define CHECK(x) do {if(!(x)){std::fprintf(stderr,"line %d: %s\n",__LINE__,#x);std::exit(1);}}while(0) +#define SMMU_BYTES 0x40000 +#define CB 0x22000 +IOReturn NativeAdrenoQueryCpu4K(NativeAdrenoCpuState *out) +{CHECK(out);*out={};return kIOReturnSuccess;} +IOReturn NativeAdrenoMaintainOwnedPages(void *mapping,size_t bytes,NativeAdrenoCacheOperation op) +{ + CHECK(mapping && bytes && !(((uintptr_t)mapping|bytes)&4095)); + CHECK(op==NativeAdrenoClean || op==NativeAdrenoCleanInvalidate || op==NativeAdrenoInvalidate); + ++fixture.cache_calls; + return fixture.cache_calls==fixture.cache_fail_call?kIOReturnIOError:kIOReturnSuccess; +} +static uint32_t mmio[SMMU_BYTES/4]; +static struct {unsigned calls,fail_call,after;} mf; +static ArmSmmu4KIo smmuio; +static ArmSmmu4KStreamState streams; +static ArmSmmu4KStream routes[2]={{2,0},{4,1}}; +static ArmSmmu4KStreamResult route_result; +static ArmSmmu4KContextConfig context; +static ArmSmmu4KContextImage context_image; +static int mevent(void){return ++mf.calls==mf.fail_call;} +static int mread32(void *c,size_t at,uint32_t *v) +{CHECK(c==&mf && at%4==0 && at<=SMMU_BYTES-4);if(mevent())return 1;*v=mmio[at/4];return 0;} +static int mread64(void *c,size_t at,uint64_t *v) +{CHECK(c==&mf && at%8==0 && at<=SMMU_BYTES-8);if(mevent())return 1;std::memcpy(v,mmio+at/4,8);return 0;} +static int mwrite32(void *c,size_t at,uint32_t v) +{CHECK(c==&mf && at%4==0 && at<=SMMU_BYTES-4);int f=mevent();if(f && !mf.after)return 1;mmio[at/4]=v;return f;} +static int mwrite64(void *c,size_t at,uint64_t v){(void)c;(void)at;(void)v;CHECK(0);return 1;} +static int mbarrier(void *c){CHECK(c==&mf);return mevent();} +struct CpPeer { + std::vector regs,writes; + unsigned publish=0,acquire=0,barriers=0,fail_publish=0; + CpPeer():regs(0x10000,0){} +}; +static CpPeer cp; +static uint64_t clock_us=0; +static int cwrite(void *c,uint32_t reg,uint32_t v) +{CHECK(c==&cp && reg(cookie);CHECK(w && w->core);++w->calls; + size_t size=input_size; + if(w->corrupt_input && size)size-=1; + const IOReturn status=w->core->dispatch(method,input,size,output,output_size,w->clock); + if(!status && w->corrupt_version && method==NativeAdrenoMethodQuery && + output && output_size==sizeof(NativeAdrenoAbiParams)) + static_cast(output)->abi_version=NativeAdrenoAbiVersion+1; + return status; +} +static NativeAdrenoTransport transport(Wire *w){return NativeAdrenoTransport{w,call};} +int main() +{ + /* Full userspace round-trip through the kernel dispatch core. */ + { + Path p;CHECK(exposed(p)==kIOReturnSuccess); + NativeAdrenoStatus counters;CHECK(p.client.attachStatus(&counters)==kIOReturnSuccess); + NativeAdrenoTimer timer;CHECK(timer.attach(&timeio,NativeAdrenoTimeCp)==kIOReturnSuccess); + CHECK(p.client.attachTimer(&timer)==kIOReturnSuccess); + Wire wire{&p.client,&ringclock}; + NativeAdrenoTransport t=transport(&wire); + NativeAdrenoClient client; + CHECK(client.open(&t)==kIOReturnSuccess && client.isOpen()); + CHECK(client.open(&t)==kIOReturnBusy); + NativeAdrenoAbiParams info{}; + CHECK(client.query(&info)==kIOReturnSuccess && info.input_bits==32 && info.output_bits==40); + uint64_t handle=0,mapped=0; + CHECK(client.allocBo(8192,12,true,&handle)==kIOReturnSuccess && handle); + uint32_t token=0; + uint64_t mapbytes=0; + CHECK(client.cpuMap(handle,&token,&mapbytes)==kIOReturnSuccess && token && mapbytes==8192); + CHECK(client.freeBo(handle)==kIOReturnBusy); /* user mapping still leased */ + CHECK(client.cpuUnmap(handle,token)==kIOReturnSuccess); + CHECK(client.cpuUnmap(handle,token)==kIOReturnBusy); + CHECK(client.mapBo(handle,0x800000,3,0,2,&mapped)==kIOReturnSuccess && mapped==0x800000); + const char name[]="wire"; + CHECK(client.setMetadata(handle,name,4)==kIOReturnSuccess); + char read[8]{}; + uint32_t length=0; + CHECK(client.getMetadata(handle,read,8,&length)==kIOReturnSuccess); + CHECK(length==4 && !std::memcmp(read,"wire",4)); + NativeAdrenoAbiSubmitBuffer buffer{handle,0x800000,1024}; + uint64_t fence=0; + CHECK(client.submit(&buffer,1,0,0,&fence)==kIOReturnSuccess && fence==1); + fencemem[1]=1; + CHECK(client.waitFence(fence,0)==kIOReturnSuccess); + CHECK(client.submit(&buffer,1,fence,0,&fence)==kIOReturnSuccess && fence==2); + CHECK(client.waitFence(fence,0)==kIOReturnTimeout); + fencemem[1]=2; + CHECK(client.waitFence(fence,UINT64_C(1000000))==kIOReturnSuccess); + cp.regs[0x0980]=0x42;cp.regs[0x0981]=7; + uint64_t ticks=0; + CHECK(client.timestamp(&ticks)==kIOReturnSuccess && ticks==((UINT64_C(7)<<32)|0x42)); + counters.noteFault(); + uint64_t faults=0,suspends=0; + CHECK(client.status(&faults,&suspends)==kIOReturnSuccess && faults==1 && !suspends); + CHECK(client.waitFence(fence+5,0)==kIOReturnBadArgument); + CHECK(client.getMetadata(handle,read,2,&length)==kIOReturnBadArgument); + uint64_t queue=0; + CHECK(client.createQueue(&queue)==kIOReturnSuccess && queue==1); + NativeAdrenoAbiSubmitBuffer queued{handle,0x800000,256}; + uint64_t qfence=0; + CHECK(client.submit(&queued,1,0,0,&qfence,queue)==kIOReturnSuccess); + CHECK(client.closeQueue(queue)==kIOReturnBusy); + fencemem[1]=qfence; + CHECK(client.waitFence(qfence,0)==kIOReturnSuccess); + CHECK(client.closeQueue(queue)==kIOReturnSuccess); + CHECK(client.submit(&queued,1,0,0,&qfence,queue)==kIOReturnBadArgument); + CHECK(client.freeBo(handle)==kIOReturnSuccess); + CHECK(client.freeBo(handle)==kIOReturnBadArgument); + CHECK(client.close()==kIOReturnSuccess && !client.isOpen()); + CHECK(client.close()==kIOReturnNotReady); + CHECK(p.execbuf.outstanding()==0); + } + /* A transport that perturbs the version is refused at open. */ + { + Path p;CHECK(exposed(p)==kIOReturnSuccess); + Wire wire{&p.client,&ringclock};wire.corrupt_version=1; + NativeAdrenoTransport t=transport(&wire); + NativeAdrenoClient client; + CHECK(client.open(&t)==kIOReturnUnsupported && !client.isOpen()); + CHECK(client.query(nullptr)==kIOReturnBadArgument); + CHECK(client.close()==kIOReturnNotReady); + } + /* Altered record sizes reach the kernel and are refused without effects. */ + { + Path p;CHECK(exposed(p)==kIOReturnSuccess); + Wire wire{&p.client,&ringclock};wire.corrupt_input=1; + NativeAdrenoTransport t=transport(&wire); + NativeAdrenoClient client; + CHECK(client.open(&t)==kIOReturnSuccess); + uint64_t handle=0; + CHECK(client.allocBo(4096,12,false,&handle)==kIOReturnBadArgument && !handle); + unsigned calls=wire.calls; + CHECK(client.mapBo(1,0x800000,3,0,2,&handle)==kIOReturnBadArgument); + CHECK(wire.calls==calls+1); + CHECK(client.submit(nullptr,0,0,0,&handle)==kIOReturnBadArgument); + CHECK(client.setMetadata(0,nullptr,0)==kIOReturnBadArgument); + CHECK(client.close()==kIOReturnSuccess); + } + /* Client-side refusals never call the transport. */ + { + Path p;CHECK(exposed(p)==kIOReturnSuccess); + Wire wire{&p.client,&ringclock}; + NativeAdrenoTransport t=transport(&wire); + NativeAdrenoClient client; + uint64_t handle=0; + /* Before open every method reports not-ready, not a transport error. */ + CHECK(client.allocBo(4096,12,false,&handle)==kIOReturnNotReady); + CHECK(client.submit(nullptr,1,0,0,&handle)==kIOReturnNotReady); + CHECK(client.freeBo(0)==kIOReturnNotReady); + CHECK(client.setMetadata(1,nullptr,65)==kIOReturnNotReady); + CHECK(client.open(nullptr)==kIOReturnBadArgument); + NativeAdrenoTransport broken{&wire,nullptr}; + CHECK(client.open(&broken)==kIOReturnBadArgument); + CHECK(wire.calls==0); + CHECK(client.open(&t)==kIOReturnSuccess && client.isOpen()); + const unsigned calls=wire.calls; + /* Argument refusals are decided here and never reach the kernel. */ + CHECK(client.allocBo(4096,12,false,nullptr)==kIOReturnBadArgument); + CHECK(client.allocBo(4096,11,false,&handle)==kIOReturnBadArgument); + CHECK(client.allocBo((UINT64_C(1)<<28)+4096,12,false,&handle)==kIOReturnBadArgument); + CHECK(client.freeBo(0)==kIOReturnBadArgument); + CHECK(client.setMetadata(1,nullptr,65)==kIOReturnBadArgument); + uint32_t length=0; + CHECK(client.getMetadata(1,nullptr,65,&length)==kIOReturnBadArgument); + NativeAdrenoAbiSubmitBuffer many[NativeAdrenoAbiMaxBuffers+1]{}; + CHECK(client.submit(many,NativeAdrenoAbiMaxBuffers+1,0,0,&handle)==kIOReturnBadArgument); + CHECK(client.mapBo(0,0x800000,3,0,2,&handle)==kIOReturnBadArgument); + CHECK(wire.calls==calls); + CHECK(client.close()==kIOReturnSuccess); + } + std::printf("PASS: 1 userspace-to-kernel round trip including CPU mapping leases and queue lifecycle, 1 version mismatch, 1 altered-size rejection and 8 client-side refusals over software SMMU/CP peers\n"); +} diff --git a/drivers/NativeAdreno/tests/test_cpinit.cpp b/drivers/NativeAdreno/tests/test_cpinit.cpp new file mode 100644 index 0000000..4ad7fbb --- /dev/null +++ b/drivers/NativeAdreno/tests/test_cpinit.cpp @@ -0,0 +1,94 @@ +/* SPDX-License-Identifier: MIT */ +/* Real CP init sequence over a software CP register peer. No GPU start. */ +#include "XnuMemoryFixture.h" +#include "NativeAdrenoCpInit.h" +#include +#include +#include +#define CHECK(x) do {if(!(x)){std::fprintf(stderr,"line %d: %s\n",__LINE__,#x);std::exit(1);}}while(0) +struct Peer { + uint32_t regs[0x10000]{}; + std::vector writes; + unsigned publish=0,acquire=0,barriers=0,fail_write=0,fail_publish=0; +}; +static Peer peer; +static uint32_t ringmem[8192],fencemem[1024]; +static int write32(void *c,uint32_t reg,uint32_t v) +{ + CHECK(c==&peer && reg<0x10000); + if(peer.writes.size()+1==peer.fail_write)return 1; + peer.regs[reg]=v;peer.writes.push_back(reg);return 0; +} +static int read32(void *c,uint32_t reg,uint32_t *v) +{CHECK(c==&peer && v);*v=peer.regs[reg];return 0;} +static int barrier(void *c){CHECK(c==&peer);++peer.barriers;return 0;} +static int publish(void *c,const volatile uint32_t *w,size_t bytes) +{CHECK(c==&peer && w && bytes==32768);++peer.publish;return peer.publish==peer.fail_publish;} +static int acquire(void *c,volatile uint32_t *w,size_t bytes) +{CHECK(c==&peer && w && bytes==4096);++peer.acquire;return 0;} +static NativeAdrenoRingIo io{&peer,write32,read32,barrier}; +static NativeAdrenoRingCache cache{&peer,publish,acquire}; +static NativeAdrenoRingClock clock_{&peer,nullptr,nullptr}; +static void reset(void) +{ + peer=Peer{}; + std::memset(ringmem,0xee,sizeof(ringmem));std::memset(fencemem,0,sizeof(fencemem)); +} +int main() +{ + /* Exact SQE base and CP_ME_INIT sequence through the ring. */ + { + reset();NativeAdrenoRing ring;CHECK(ring.attach(&io,&cache,ringmem,sizeof(ringmem),0x100000,fencemem,sizeof(fencemem),0x108000)==kIOReturnSuccess); + NativeAdrenoCpInit init; + NativeAdrenoCpInitConfig config{0x200000,0x300000,0}; + uint32_t fence=0; + CHECK(init.run(&io,&ring,&config,&fence)==kIOReturnSuccess && fence==1 && init.completed()); + bool paired=false; + for(size_t i=0;i+1>16&0x7fu)==0x17 && (ringmem[0]&0x3fffu)==1); + CHECK(ringmem[1]==(1u<<27)); + CHECK((ringmem[2]>>16&0x7fu)==0x48 && (ringmem[2]&0x3fffu)==7); + CHECK(ringmem[3]==NativeAdrenoCpInit::meInitMask() && ringmem[3]==0x14b); + CHECK(ringmem[4]==3 && ringmem[5]==0x20000000 && ringmem[6]==2); + CHECK(ringmem[7]==0x300000 && ringmem[8]==0 && ringmem[9]==(1u<<31)); + CHECK(peer.regs[0x0807]==17); /* ten init words plus the seven-word fence */ + CHECK(init.run(&io,&ring,&config,&fence)==kIOReturnNotReady); + fencemem[1]=1; + CHECK(ring.wait(1,&clock_,1000,nullptr)==NativeAdrenoRingFenceComplete); + CHECK(ring.detach()==kIOReturnSuccess); + } + /* Refusals precede MMIO and ring writes. */ + { + reset();NativeAdrenoRing ring;CHECK(ring.attach(&io,&cache,ringmem,sizeof(ringmem),0x100000,fencemem,sizeof(fencemem),0x108000)==kIOReturnSuccess); + NativeAdrenoCpInit init;uint32_t fence=0; + const size_t attached_writes=peer.writes.size(); + NativeAdrenoCpInitConfig good{0x200000,0x300000,0},zero{0,0x300000,0}, + unaligned{0x200001,0x300000,0},high{1ull<<48,0x300000,0},list{0x200000,0x300002,0}; + CHECK(init.run(nullptr,&ring,&good,&fence)==kIOReturnBadArgument); + CHECK(init.run(&io,nullptr,&good,&fence)==kIOReturnBadArgument); + CHECK(init.run(&io,&ring,nullptr,&fence)==kIOReturnBadArgument); + CHECK(init.run(&io,&ring,&good,nullptr)==kIOReturnBadArgument); + CHECK(init.run(&io,&ring,&zero,&fence)==kIOReturnBadArgument); + CHECK(init.run(&io,&ring,&unaligned,&fence)==kIOReturnBadArgument); + CHECK(init.run(&io,&ring,&high,&fence)==kIOReturnBadArgument); + CHECK(init.run(&io,&ring,&list,&fence)==kIOReturnBadArgument); + CHECK(peer.writes.size()==attached_writes && peer.regs[0x0807]==0 && !init.completed()); + /* A failed SQE write poisons without submitting the ring. */ + peer.fail_write=peer.writes.size()+2; /* second SQE write */ + CHECK(init.run(&io,&ring,&good,&fence)==kIOReturnIOError && init.poisoned() && !fence); + CHECK(peer.regs[0x0807]==0 && peer.writes.size()==8); + CHECK(init.run(&io,&ring,&good,&fence)==kIOReturnNotReady); + CHECK(ring.detach()==kIOReturnSuccess); /* no submission was committed */ + } + /* A ring commit failure poisons the initializer instead of claiming start. */ + { + reset();NativeAdrenoRing ring;CHECK(ring.attach(&io,&cache,ringmem,sizeof(ringmem),0x100000,fencemem,sizeof(fencemem),0x108000)==kIOReturnSuccess); + NativeAdrenoCpInit init;NativeAdrenoCpInitConfig config{0x200000,0x300000,0};uint32_t fence=0; + peer.fail_publish=1; + CHECK(init.run(&io,&ring,&config,&fence)==kIOReturnIOError && init.poisoned()); + CHECK(ring.poisoned() && ring.detach()==kIOReturnBusy); + } + std::printf("PASS: 1 exact CP_ME_INIT sequence with SQE base, 8 pre-MMIO refusals, 1 SQE write failure and 1 ring commit failure over a software CP register peer\n"); +} diff --git a/drivers/NativeAdreno/tests/test_gmuconfig.cpp b/drivers/NativeAdreno/tests/test_gmuconfig.cpp new file mode 100644 index 0000000..e734fb7 --- /dev/null +++ b/drivers/NativeAdreno/tests/test_gmuconfig.cpp @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: MIT */ +/* Real assembler over host doubles; only structure fields are checked. */ +#include "XnuMemoryFixture.h" +#include "NativeAdrenoGmuConfig.h" +#include +#include +#define CHECK(x) do {if(!(x)){std::fprintf(stderr,"line %d: %s\n",__LINE__,#x);std::exit(1);}}while(0) +IOReturn NativeAdrenoQueryCpu4K(NativeAdrenoCpuState *state) +{CHECK(state);*state={};return kIOReturnSuccess;} +IOReturn NativeAdrenoMaintainOwnedPages(void *mapping,size_t bytes,NativeAdrenoCacheOperation op) +{CHECK(mapping && bytes && op<=NativeAdrenoInvalidate);return kIOReturnSuccess;} +int main() +{ + NativeAdrenoGmuAllocator allocator; + CHECK(allocator.allocate(NativeAdrenoGmuDefaultSizes(0x10000))==kIOReturnSuccess); + uint8_t image[16]={1,2,3}; + NativeAdrenoFirmwareView firmware{image,sizeof(image),{}}; + ShengGmuConfig config{}; + CHECK(NativeAdrenoAssembleGmuConfig(&firmware,&allocator,0x43050a01u,100,&config)==kIOReturnSuccess); + const NativeAdrenoGmuMemoryLayout &layout=allocator.layout(); + CHECK(config.chip_id==0x43050a01u && config.poll_budget==100); + CHECK(config.firmware==image && config.firmware_bytes==sizeof(image)); + CHECK(config.icache.cpu==static_cast(allocator.icache().mapping())); + CHECK(config.icache.bytes==0xffc000 && config.icache.iova==layout.icache_iova); + CHECK(config.dummy.bytes==0x4000 && config.dummy.iova==layout.dummy_iova); + CHECK(config.debug.bytes==0x7000 && config.debug.iova==layout.debug_iova); + CHECK(config.log.bytes==0x4000 && config.log.iova==layout.log_iova); + CHECK(config.hfi.bytes==0x4000 && config.hfi.iova==layout.hfi_iova); + /* Refusals leave no partially assembled config. */ + ShengGmuConfig empty{}; + CHECK(NativeAdrenoAssembleGmuConfig(nullptr,&allocator,1,100,&empty)==kIOReturnBadArgument); + CHECK(NativeAdrenoAssembleGmuConfig(&firmware,nullptr,1,100,&empty)==kIOReturnBadArgument); + CHECK(NativeAdrenoAssembleGmuConfig(&firmware,&allocator,0,100,&empty)==kIOReturnBadArgument); + CHECK(NativeAdrenoAssembleGmuConfig(&firmware,&allocator,1,0,&empty)==kIOReturnBadArgument); + CHECK(NativeAdrenoAssembleGmuConfig(&firmware,&allocator,1,1025,&empty)==kIOReturnBadArgument); + CHECK(NativeAdrenoAssembleGmuConfig(&firmware,&allocator,1,100,nullptr)==kIOReturnBadArgument); + NativeAdrenoFirmwareView missing{}; + CHECK(NativeAdrenoAssembleGmuConfig(&missing,&allocator,1,100,&empty)==kIOReturnBadArgument); + CHECK(empty.chip_id==0 && empty.firmware==nullptr); + /* A profile with non-startup payload sizes is refused, not assembled. */ + NativeAdrenoGmuAllocator small; + CHECK(small.allocate({0x4000,0x1000,0x2000,0x1000,0x1000,0x10000})==kIOReturnSuccess); + CHECK(NativeAdrenoAssembleGmuConfig(&firmware,&small,1,100,&empty)==kIOReturnBadArgument); + CHECK(empty.firmware==nullptr); + CHECK(small.release()==kIOReturnSuccess && allocator.release()==kIOReturnSuccess); + CHECK(fixture.live.empty()); + std::printf("PASS: 1 assembled GMU config, 7 argument refusals and 1 non-startup-profile refusal over host doubles\n"); +} diff --git a/drivers/NativeAdreno/tests/test_gmuctx.cpp b/drivers/NativeAdreno/tests/test_gmuctx.cpp new file mode 100644 index 0000000..b099d14 --- /dev/null +++ b/drivers/NativeAdreno/tests/test_gmuctx.cpp @@ -0,0 +1,162 @@ +/* SPDX-License-Identifier: MIT */ +/* The whole GMU context sequence from an unprogrammed SMMU over a software register peer: the real + * allocator, table pool, context-bank programming, stream attach, VM and payload mapping. ID0/ID1 are + * the values read on sheng (int-005); ID2 was not read there, so the peer's value is an assumption. */ +#include "XnuMemoryFixture.h" +#include "NativeAdrenoGmuContext.h" +#include +#include +#define CHECK(x) do {if(!(x)){std::fprintf(stderr,"line %d: %s\n",__LINE__,#x);std::exit(1);}}while(0) +IOReturn NativeAdrenoQueryCpu4K(NativeAdrenoCpuState *state) +{CHECK(state);*state={};return kIOReturnSuccess;} +IOReturn NativeAdrenoMaintainOwnedPages(void *mapping,size_t bytes,NativeAdrenoCacheOperation op) +{CHECK(mapping && bytes && op<=NativeAdrenoInvalidate);return kIOReturnSuccess;} +#define SMMU_BYTES 0x40000 +alignas(8) static uint32_t mmio[SMMU_BYTES/4]; +static struct {unsigned ignore_writes_from;size_t frozen;unsigned tbi;} peer; +static void reset() +{ + std::memset(mmio,0,sizeof(mmio));peer={}; + mmio[0]=0x002f0406;mmio[0x20/4]=0x4c017e18;mmio[0x24/4]=0x40000016;mmio[0x28/4]=0x1433; + for(unsigned i=0;i<24;++i)mmio[(0xc00+4*i)/4]=0x20000; /* every S2CR FAULT, every SMR invalid */ +} +static int read32(void *,size_t at,uint32_t *v){CHECK(at%4==0 && at<=SMMU_BYTES-4);*v=mmio[at/4];return 0;} +static int read64(void *,size_t at,uint64_t *v){CHECK(at%8==0 && at<=SMMU_BYTES-8);std::memcpy(v,mmio+at/4,8);return 0;} +static int write32(void *,size_t at,uint32_t v) +{ + CHECK(at%4==0 && at<=SMMU_BYTES-4); + if(peer.frozen && at==peer.frozen)return 0; /* accepted on the bus, not stored */ + if((at&0xfff)==0x58 && at>=0x20000) {mmio[at/4]&=~v;return 0;} /* context FSR is write-one-to-clear */ + if(peer.tbi && (at&0xfff)==0x10 && at>=0x20000)v|=0x60; /* int-008: TCR2 reads TBI0/TBI1 as 1 */ + if(peer.tbi && (at&0xfff)==0 && at>=0x20000)v&=~0x1000u; /* int-009: SCTLR.ASIDPNE reads 0 */ + mmio[at/4]=v;return 0; +} +static int write64(void *,size_t at,uint64_t v) +{CHECK(at%8==0 && at<=SMMU_BYTES-8);if(peer.frozen && at==peer.frozen)return 0;std::memcpy(mmio+at/4,&v,8);return 0;} +static int barrier(void *){return 0;} +static unsigned steps[16],step_count; +static void progress(uint32_t step){CHECK(step_count<16);steps[step_count++]=step;} +int main() +{ + const ArmSmmu4KIo io{&peer,SMMU_BYTES,read32,read64,write32,write64,barrier}; + const size_t cb=0x20000+1*0x1000; /* NUMPAGE 32 x 4 KiB global space, bank 1 */ + { + reset();step_count=0; + NativeAdrenoGmuContext context; + NativeAdrenoGmuContextReport report; + CHECK(context.bringUp(&io,1,1,2,5,progress,&report)==kIOReturnSuccess); + CHECK(report.step==NativeAdrenoGmuContext::Ready && report.status==kIOReturnSuccess && context.ready()); + CHECK(step_count==7); + for(unsigned i=0;i<7;++i)CHECK(steps[i]==i+1); + CHECK(report.program_enabled && report.program_readback_valid && report.routes_verified==1); + CHECK(report.mismatch==0); + CHECK(report.id1==0x40000016 && report.spans==5); + CHECK(report.id2==0x1433 && report.output_bits==40 && report.program_status==0 && report.route_status==0); + /* Route: S2CR translate to bank 1, then exact SMR for SID 5. */ + CHECK(report.s2cr==mmio[(0xc00+8)/4] && (report.s2cr&0xff)==1 && ((report.s2cr>>16)&3)==0); + CHECK(report.smr==((1u<<31)|5)); + CHECK(report.cbar==mmio[(0x1000+4)/4]); + CHECK(report.sctlr==mmio[cb/4] && (report.sctlr&1)); + /* TTBR0 holds the table pool and ASID 1; TTBR1 is unused. */ + CHECK(report.ttbr0==((uint64_t(1)<<48)|context.allocator().tables().spans()[0].physical)); + uint64_t ttbr1;std::memcpy(&ttbr1,mmio+(cb+0x28)/4,8); + CHECK((ttbr1&0x0000fffffffff000)==0); + CHECK(context.bringUp(&io,1,1,2,5,nullptr,&report)==kIOReturnBusy); /* one attempt per object */ + } + /* A S2CR that reads back other than FAULT (the apps SMMU stored BYPASS writes as FAULT; here the reverse + * case) is refused at attach, after the bank was programmed; the SMR stays invalid. */ + { + reset(); + mmio[(0xc00+8)/4]=0x10000; + NativeAdrenoGmuContext context; + NativeAdrenoGmuContextReport report; + CHECK(context.bringUp(&io,1,1,2,5,nullptr,&report)!=kIOReturnSuccess); + CHECK(report.step==NativeAdrenoGmuContext::Attach && report.program_enabled && !report.routes_verified); + CHECK(!(mmio[(0x800+8)/4]&(1u<<31)) && !context.ready()); + } + /* A context-bank write the peer ignores (a hypervisor that drops it) is caught by the program readback. */ + { + reset();peer.frozen=cb+0x30; /* TCR */ + NativeAdrenoGmuContext context; + NativeAdrenoGmuContextReport report; + CHECK(context.bringUp(&io,1,1,2,5,nullptr,&report)!=kIOReturnSuccess); + CHECK(report.step==NativeAdrenoGmuContext::Program && !report.program_enabled); + CHECK(report.mismatch==(1u<<2) && report.image.tcr==0); /* only TCR differs; the banner names it */ + CHECK(!(mmio[cb/4]&1) && !(mmio[(0x800+8)/4]&(1u<<31))); + } + /* An already enabled bank is refused before any write. */ + { + reset();mmio[cb/4]=1; + NativeAdrenoGmuContext context; + NativeAdrenoGmuContextReport report; + CHECK(context.bringUp(&io,1,1,2,5,nullptr,&report)!=kIOReturnSuccess); + CHECK(report.step==NativeAdrenoGmuContext::Program && report.program_writes==0); + } + /* A 36-bit OAS gives a 36-bit context (TCR2.PASize 1) and allocations below 2^36. */ + { + reset();mmio[0x28/4]=0x1413; /* OAS code 1 */ + NativeAdrenoGmuContext context; + NativeAdrenoGmuContextReport report; + CHECK(context.bringUp(&io,1,1,2,5,nullptr,&report)==kIOReturnSuccess && report.output_bits==36); + CHECK((mmio[(cb+0x10)/4]&7)==1); + CHECK(context.allocator().tables().spans()[0].physical+0x1000*NativeAdrenoGmuContext::TablePages<=(UINT64_C(1)<<36)); + } + /* An OAS code the kit cannot encode is refused before any allocation or write. */ + { + reset();mmio[0x28/4]=0x14f3; + NativeAdrenoGmuContext context; + NativeAdrenoGmuContextReport report; + CHECK(context.bringUp(&io,1,1,2,5,nullptr,&report)==kIOReturnUnsupported); + CHECK(report.step==NativeAdrenoGmuContext::Allocate && !context.allocator().ready() && !(mmio[cb/4]&1)); + } + /* No AArch64 4 KiB granule (ID2.PTFS_4K clear): the kit refuses in phase 1 with no write, the case int-006 + * may have hit; program_status names it. */ + { + reset();mmio[0x28/4]=0x0433; + NativeAdrenoGmuContext context; + NativeAdrenoGmuContextReport report; + CHECK(context.bringUp(&io,1,1,2,5,nullptr,&report)!=kIOReturnSuccess); + CHECK(report.step==NativeAdrenoGmuContext::Program && report.program_phase==1 && report.program_writes==0); + CHECK(report.program_status==ArmSmmu4KUnsupported); + } + /* The int-007..009 hardware as observed: ID2 0x5511 (OAS 36), TCR2 reading 0x38071 for 0x38011 and SCTLR + * reading 0x167 for 0x1167. */ + { + reset();mmio[0x28/4]=0x5511;peer.tbi=1; + NativeAdrenoGmuContext context; + NativeAdrenoGmuContextReport report; + CHECK(context.bringUp(&io,1,1,2,5,nullptr,&report)==kIOReturnSuccess && report.step==NativeAdrenoGmuContext::Ready); + CHECK(report.output_bits==36 && mmio[(cb+0x10)/4]==0x38071 && report.mismatch==((1u<<3)|(1u<<8))); + CHECK(mmio[cb/4]==0x167); + CHECK(report.smr==((1u<<31)|5) && (report.s2cr&0xff)==1 && (report.sctlr&1)); + } + /* Argument refusals touch nothing. */ + { + reset(); + NativeAdrenoGmuContext context; + NativeAdrenoGmuContextReport report; + CHECK(context.bringUp(nullptr,1,1,2,5,nullptr,&report)==kIOReturnBadArgument); + CHECK(context.bringUp(&io,64,1,2,5,nullptr,&report)==kIOReturnBadArgument); + CHECK(context.bringUp(&io,1,0,2,5,nullptr,&report)==kIOReturnBadArgument); + CHECK(context.bringUp(&io,1,1,128,5,nullptr,&report)==kIOReturnBadArgument); + CHECK(context.bringUp(&io,1,1,2,5,nullptr,nullptr)==kIOReturnBadArgument); + CHECK(report.step==0); + } + /* The board entry over the kit's direct MMIO adapter on a plain array. A plain array has no + * write-one-to-clear FSR, so the cleared fault status reads back set: the enable is refused as a + * fault and the bank left disabled. Then its single-use guard. */ + { + reset(); + NativeAdrenoGmuContextReport report; + CHECK(NativeAdrenoGmuContextBringup(reinterpret_cast(mmio),SMMU_BYTES,1,1,2,5, + nullptr,&report)!=kIOReturnSuccess); + CHECK(report.step==NativeAdrenoGmuContext::Program && !report.program_enabled && !(mmio[cb/4]&1)); + CHECK(NativeAdrenoGmuContextBringup(reinterpret_cast(mmio),SMMU_BYTES,1,1,2,5, + nullptr,&report)==kIOReturnBadArgument); + NativeAdrenoGmuPayload payloads[5]; /* that context never reached Ready */ + CHECK(NativeAdrenoGmuContextPayloads(payloads)==kIOReturnNotReady && !payloads[4].bytes); + CHECK(NativeAdrenoGmuContextPayloads(nullptr)==kIOReturnBadArgument); + } + std::printf("PASS: 1 complete GMU context bring-up with exact routes and TTBR0, 1 non-FAULT S2CR, 1 ignored " + "context write, 1 enabled-bank refusal, 1 36-bit OAS context, 1 sheng-observed ID2/TCR2 context, 2 ID2 refusals, 5 argument refusals and 1 board entry with a stuck fault status over a software SMMU peer\n"); +} diff --git a/drivers/NativeAdreno/tests/test_gmumap.cpp b/drivers/NativeAdreno/tests/test_gmumap.cpp new file mode 100644 index 0000000..d97e7b4 --- /dev/null +++ b/drivers/NativeAdreno/tests/test_gmumap.cpp @@ -0,0 +1,90 @@ +/* SPDX-License-Identifier: MIT */ +/* Real lower-region mapping of the GMU payloads at their fixed a6xx addresses over a software SMMU peer. */ +#include "XnuMemoryFixture.h" +#include "NativeAdrenoGmuMemory.h" +#include +#include +#define CHECK(x) do {if(!(x)){std::fprintf(stderr,"line %d: %s\n",__LINE__,#x);std::exit(1);}}while(0) +#define SMMU_BYTES 0x40000 +#define CB 0x22000 +IOReturn NativeAdrenoQueryCpu4K(NativeAdrenoCpuState *state) +{CHECK(state);*state={};return kIOReturnSuccess;} +IOReturn NativeAdrenoMaintainOwnedPages(void *mapping,size_t bytes,NativeAdrenoCacheOperation op) +{ + CHECK(mapping && bytes && op<=NativeAdrenoInvalidate); + ++fixture.cache_calls; + return fixture.cache_calls==fixture.cache_fail_call?kIOReturnIOError:kIOReturnSuccess; +} +static uint32_t mmio[SMMU_BYTES/4]; +static struct {unsigned calls,fail_call,after;} mf; +static ArmSmmu4KIo io; +static ArmSmmu4KStreamState streams; +static ArmSmmu4KStream routes[1]={{2,0}}; /* slot 0 holds an unrelated route */ +static ArmSmmu4KStreamResult route_result; +static ArmSmmu4KContextConfig context; +static ArmSmmu4KContextImage image; +static int event(void){return ++mf.calls==mf.fail_call;} +static int read32(void *c,size_t at,uint32_t *v) +{CHECK(c==&mf && at%4==0 && at<=SMMU_BYTES-4);if(event())return 1;*v=mmio[at/4];return 0;} +static int read64(void *c,size_t at,uint64_t *v) +{CHECK(c==&mf && at%8==0 && at<=SMMU_BYTES-8);if(event())return 1;std::memcpy(v,mmio+at/4,8);return 0;} +static int write32(void *c,size_t at,uint32_t v) +{CHECK(c==&mf && at%4==0 && at<=SMMU_BYTES-4);int f=event();if(f && !mf.after)return 1;mmio[at/4]=v;return f;} +static int write64(void *c,size_t at,uint64_t v){(void)c;(void)at;(void)v;CHECK(0);return 1;} +static int barrier(void *c){CHECK(c==&mf);return event();} +int main() +{ + /* A GMU-only context: lower region covering the whole 0..2 GiB GMU space, no upper region. */ + NativeAdrenoMemory tables; + ArmSmmu4KLiveArena arena; + ArmSmmu4KLiveState state{}; + ArmSmmu4KLiveVisibility visibility; + NativeAdrenoVm vm; + std::memset(mmio,0,sizeof(mmio));mf={};streams=(ArmSmmu4KStreamState){}; + io=(ArmSmmu4KIo){&mf,SMMU_BYTES,read32,read64,write32,write64,barrier}; + mmio[0x20/4]=(1u<<30)|(1u<<27)|(4u<<9)|8; + mmio[0x24/4]=0x40000010;mmio[0x28/4]=0x1433;mmio[0]=0x436; + for(unsigned i=0;i<8;++i)mmio[(0xc00+4*i)/4]=0x20000; + mmio[0x800/4]=1u<<31|9;mmio[0xc00/4]=3; + CHECK(tables.allocate(1027*4096,40,true,1027*4096)==kIOReturnSuccess); + CHECK(NativeAdrenoPrepareSmmuMemory(&tables,32,40,0,GmuApertureBytes,0,size_t(1)<<19, + &arena,&state,&visibility)==kIOReturnSuccess); + CHECK(state.tables.table_pages==1027); + context=(ArmSmmu4KContextConfig){.lower=&arena.config,.upper=nullptr, + .lower_tables=&state.tables,.upper_tables=nullptr,.id0=mmio[0x20/4], + .id2=mmio[0x28/4],.asid=1,.coherent_walk=1,.stall_on_fault=0, + .continue_after_fault=1}; + ArmSmmu4KContextConfig actual=context; + CHECK(ArmSmmu4KEncodeContext(&actual,&image)==ArmSmmu4KSuccess); + mmio[0x1008/4]=image.cbar;mmio[0x1808/4]=image.cba2r; + mmio[CB/4]=image.sctlr;mmio[(CB+0x10)/4]=image.tcr2;mmio[(CB+0x30)/4]=image.tcr; + std::memcpy(mmio+(CB+0x20)/4,&image.ttbr0,8); + std::memcpy(mmio+(CB+0x28)/4,&image.ttbr1,8); + mmio[(CB+0x38)/4]=image.mair0; + CHECK(ArmSmmu4KAttachStreams(&io,&streams,routes,1,2,&context,&route_result)==ArmSmmu4KSuccess); + NativeAdrenoVmRegion lower={&tables,&arena,&state}; + CHECK(vm.attach(&io,&context,&streams,&lower,nullptr,64,2000)==kIOReturnSuccess); + CHECK(vm.spanCount()==0 && vm.objectCount()==0); + + NativeAdrenoGmuSizes sizes{0x4000,0x1000,0x2000,0x1000,0x1000,0x10000}; + NativeAdrenoGmuAllocator allocator; + CHECK(allocator.mapInto(&vm)==kIOReturnBadArgument); /* not allocated yet */ + CHECK(allocator.allocate(sizes)==kIOReturnSuccess); + CHECK(allocator.layout().icache_iova==0x4000 && allocator.layout().dummy_iova==0x60000000); + CHECK(allocator.layout().log_iova==0x60001000 && allocator.layout().hfi_iova==0x60002000); + CHECK(allocator.layout().debug_iova==0x60400000 && allocator.layout().end_iova==0x60402000); + CHECK(allocator.mapInto(&vm)==kIOReturnSuccess); + CHECK(vm.objectCount()==5 && vm.spanCount()==5); + CHECK(allocator.mapInto(&vm)==kIOReturnBusy); + uint64_t iova=0; + CHECK(vm.findAddress(1,0x2000,0x2000,&iova)!=kIOReturnSuccess); /* no upper region */ + CHECK(allocator.unmapFrom(&vm)==kIOReturnSuccess); + CHECK(vm.objectCount()==0 && vm.spanCount()==0); + CHECK(allocator.unmapFrom(&vm)==kIOReturnSuccess); + CHECK(allocator.release()==kIOReturnSuccess); + CHECK(vm.detach()==kIOReturnSuccess); + CHECK(tables.retireAfterQuiescence()==kIOReturnSuccess); + CHECK(tables.release()==kIOReturnSuccess); + CHECK(fixture.live.empty() && fixture.metadata.empty()); + std::printf("PASS: 1 lower-region fixed-address GMU payload mapping and unmap, 1 pre-allocation refusal and 1 repeat-map refusal over a software SMMU peer\n"); +} diff --git a/drivers/NativeAdreno/tests/test_gmumem.cpp b/drivers/NativeAdreno/tests/test_gmumem.cpp new file mode 100644 index 0000000..75c70cf --- /dev/null +++ b/drivers/NativeAdreno/tests/test_gmumem.cpp @@ -0,0 +1,63 @@ +/* SPDX-License-Identifier: MIT */ +/* Real allocator/layout over host IOKit doubles. No mapping or GMU access. */ +#include "XnuMemoryFixture.h" +#include "NativeAdrenoGmuMemory.h" +#include +#include +#include +#define CHECK(x) do {if(!(x)){std::fprintf(stderr,"line %d: %s\n",__LINE__,#x);std::exit(1);}}while(0) +IOReturn NativeAdrenoQueryCpu4K(NativeAdrenoCpuState *state) +{CHECK(state);*state={};return kIOReturnSuccess;} +IOReturn NativeAdrenoMaintainOwnedPages(void *mapping,size_t bytes,NativeAdrenoCacheOperation op) +{ + CHECK(mapping && bytes && op<=NativeAdrenoInvalidate); + return kIOReturnSuccess; +} +int main() +{ + { + NativeAdrenoGmuAllocator allocator; + CHECK(allocator.allocate(NativeAdrenoGmuDefaultSizes(0x10000))==kIOReturnSuccess && allocator.ready()); + const NativeAdrenoGmuMemoryLayout &layout=allocator.layout(); + /* a6xx_gmu.c at the pinned tree: icache 0x4000, dummy 0x60000000, debug 0x60400000; log and hfi + * take the lowest free pages after the 16 KiB dummy. */ + CHECK(layout.icache_iova==0x4000); + CHECK(layout.dummy_iova==0x60000000); + CHECK(layout.debug_iova==0x60400000); + CHECK(layout.log_iova==0x60004000); + CHECK(layout.hfi_iova==0x60008000); + CHECK(layout.tables_iova==0); + CHECK(layout.tables_bytes==0x10000); + CHECK(layout.end_iova==0x60407000); + CHECK(allocator.icache().bytes()==0xffc000 && allocator.icache().spanCount()==1); + CHECK(allocator.hfi().bytes()==0x4000 && allocator.tables().spanCount()==1); + CHECK(allocator.allocate(NativeAdrenoGmuDefaultSizes(0x1000))==kIOReturnBusy); + CHECK(allocator.release()==kIOReturnSuccess && !allocator.ready()); + CHECK(fixture.live.empty()); + } + /* Bad sizes, payloads that leave their fixed range, and allocation failure leave nothing behind. */ + { + NativeAdrenoGmuAllocator allocator; + CHECK(allocator.allocate(NativeAdrenoGmuDefaultSizes(0))==kIOReturnBadArgument); + CHECK(allocator.allocate(NativeAdrenoGmuDefaultSizes(0x1001))==kIOReturnBadArgument); + NativeAdrenoGmuSizes big=NativeAdrenoGmuDefaultSizes(0x10000); + big.icache=0x60000000; /* would run into the uncached range */ + CHECK(allocator.allocate(big)==kIOReturnBadArgument); + big=NativeAdrenoGmuDefaultSizes(0x10000); + big.hfi=0x400000; /* dummy+log+hfi past 0x60400000 */ + CHECK(allocator.allocate(big)==kIOReturnBadArgument); + big=NativeAdrenoGmuDefaultSizes(0x10000); + big.debug=0x20000000; /* past the 2 GiB GMU space */ + CHECK(allocator.allocate(big)==kIOReturnBadArgument); + CHECK(!allocator.ready() && fixture.live.empty()); + } + { + fixture=MemoryFixture{}; + fixture.allocation_fail=true; + NativeAdrenoGmuAllocator allocator; + CHECK(allocator.allocate(NativeAdrenoGmuDefaultSizes(0x10000))==kIOReturnNoMemory); + CHECK(!allocator.ready() && fixture.live.empty()); + fixture.allocation_fail=false; + } + std::printf("PASS: 1 fixed a6xx GMU layout allocation and release, 5 size/range refusals, 1 allocation failure and 1 repeat-allocation refusal over host doubles\n"); +} diff --git a/drivers/NativeAdreno/tests/test_gmuplatform.cpp b/drivers/NativeAdreno/tests/test_gmuplatform.cpp new file mode 100644 index 0000000..e301a1e --- /dev/null +++ b/drivers/NativeAdreno/tests/test_gmuplatform.cpp @@ -0,0 +1,81 @@ +/* SPDX-License-Identifier: MIT */ +/* Real bounds-checked adapter over fake windows; no GMU is touched. */ +#include "XnuMemoryFixture.h" +#include "NativeAdrenoGmuPlatform.h" +#include +#include +#include +#define CHECK(x) do {if(!(x)){std::fprintf(stderr,"line %d: %s\n",__LINE__,#x);std::exit(1);}}while(0) +static uint32_t core[0x1000],rscc[0x400],pdc[0x400],gpu[0x1000],cx[0x400]; +static NativeAdrenoGmuWindow windows[ShengGmuWindows]={ + {reinterpret_cast(core),sizeof(core)}, + {reinterpret_cast(rscc),sizeof(rscc)}, + {reinterpret_cast(pdc),sizeof(pdc)}, + {reinterpret_cast(gpu),sizeof(gpu)}, + {reinterpret_cast(cx),sizeof(cx)}}; +static uint64_t fake_now=1000; +static unsigned waits=0; +static int now_us(void *cookie,uint64_t *value){CHECK(cookie==&fake_now && value);*value=fake_now;return 0;} +static int wait_us(void *cookie,unsigned us){CHECK(cookie==&fake_now);waits++;fake_now+=us;return 0;} +int main() +{ + NativeAdrenoGmuPlatform platform; + ShengGmuIo io{}; + CHECK(platform.bind(windows,&io)==kIOReturnSuccess); + CHECK(io.cookie==&platform && io.read32 && io.write32 && io.barrier); + CHECK(io.write32(io.cookie,ShengGmuCore,0x10,0xabcdef01)==0); + CHECK(core[4]==0xabcdef01); + uint32_t value=0; + CHECK(io.read32(io.cookie,ShengGmuCore,0x10,&value)==0 && value==0xabcdef01); + CHECK(io.write32(io.cookie,ShengGmuRscc,0x4,7)==0 && rscc[1]==7); + CHECK(io.write32(io.cookie,ShengGmuPdc,0x0,8)==0 && pdc[0]==8); + CHECK(io.write32(io.cookie,ShengGmuGpu,0x8,9)==0 && gpu[2]==9); + CHECK(io.write32(io.cookie,ShengGmuCxMisc,0xc,10)==0 && cx[3]==10); + CHECK(io.barrier(io.cookie)==0); + /* Out-of-window, misaligned and unknown windows fail without touching. */ + CHECK(io.read32(io.cookie,ShengGmuCore,sizeof(core),&value)==1); + CHECK(io.read32(io.cookie,ShengGmuCore,sizeof(core)-4,&value)==0); + CHECK(io.write32(io.cookie,ShengGmuCore,sizeof(core),1)==1); + CHECK(io.write32(io.cookie,ShengGmuCore,0x2,1)==1); + CHECK(io.write32(io.cookie,ShengGmuWindows,0,1)==1); + CHECK(io.read32(io.cookie,ShengGmuWindows,0,&value)==1); + CHECK(io.read32(nullptr,ShengGmuCore,0,&value)==1); + CHECK(io.read32(io.cookie,ShengGmuCore,0,nullptr)==1); + CHECK(io.write32(nullptr,ShengGmuCore,0,1)==1); + CHECK(io.barrier(nullptr)==1); + /* Binding refuses missing, empty or misaligned windows. */ + NativeAdrenoGmuWindow bad[ShengGmuWindows]; + std::memcpy(bad,windows,sizeof(bad)); + bad[2].base=nullptr; + CHECK(platform.bind(bad,&io)==kIOReturnBadArgument); + std::memcpy(bad,windows,sizeof(bad)); + bad[3].bytes=0; + CHECK(platform.bind(bad,&io)==kIOReturnBadArgument); + std::memcpy(bad,windows,sizeof(bad)); + bad[4].bytes=sizeof(cx)-2; + CHECK(platform.bind(bad,&io)==kIOReturnBadArgument); + CHECK(platform.bind(nullptr,&io)==kIOReturnBadArgument); + CHECK(platform.bind(windows,nullptr)==kIOReturnBadArgument); + /* Services: real SHA-256 plus injected clock callbacks. */ + NativeAdrenoGmuTime time{&fake_now,now_us,wait_us}; + ShengGmuServices services{}; + CHECK(NativeAdrenoGmuPlatform::services(&time,&services)==kIOReturnSuccess); + CHECK(services.cookie==&fake_now && services.now_us && services.wait_us); + uint64_t observed=0; + CHECK(services.now_us(services.cookie,&observed)==0 && observed==1000); + CHECK(services.wait_us(services.cookie,5)==0); + CHECK(waits==1 && fake_now==1005); + uint8_t digest[32]; + CHECK(services.sha256(services.cookie,"abc",3,digest)==0); + const uint8_t expected[32]={0xba,0x78,0x16,0xbf,0x8f,0x01,0xcf,0xea, + 0x41,0x41,0x40,0xde,0x5d,0xae,0x22,0x23,0xb0,0x03,0x61,0xa3, + 0x96,0x17,0x7a,0x9c,0xb4,0x10,0xff,0x61,0xf2,0x00,0x15,0xad}; + CHECK(!std::memcmp(digest,expected,32)); + CHECK(services.sha256(services.cookie,nullptr,3,digest)==1); + CHECK(services.sha256(services.cookie,"",0,digest)==1); + CHECK(services.sha256(services.cookie,"abc",3,nullptr)==1); + NativeAdrenoGmuTime bad_time{&fake_now,nullptr,wait_us}; + CHECK(NativeAdrenoGmuPlatform::services(&bad_time,&services)==kIOReturnBadArgument); + CHECK(NativeAdrenoGmuPlatform::services(&time,nullptr)==kIOReturnBadArgument); + std::printf("PASS: 1 five-window adapter with real accesses, 11 bounds/argument refusals, 1 service binding and 1 SHA-256 vector over fake windows\n"); +} diff --git a/drivers/NativeAdreno/tests/test_gpu.cpp b/drivers/NativeAdreno/tests/test_gpu.cpp new file mode 100644 index 0000000..d15f525 --- /dev/null +++ b/drivers/NativeAdreno/tests/test_gpu.cpp @@ -0,0 +1,174 @@ +/* SPDX-License-Identifier: MIT */ +/* Real ring/CP/IRQ/timer sequencing over software register peers; the board + * power/GMU/SMMU steps are fakes that only record their own calls. */ +#include "XnuMemoryFixture.h" +#include "NativeAdrenoGpu.h" +#include +#include +#include +#define CHECK(x) do {if(!(x)){std::fprintf(stderr,"line %d: %s\n",__LINE__,#x);std::exit(1);}}while(0) +struct Board { + std::vector order; + unsigned fail_call=0,quiesce_fail=0,quiesce_calls=0; + unsigned calls=0; +}; +static Board board; +static int step(const char *name) +{ + ++board.calls;board.order.push_back(name); + return board.calls==board.fail_call?kIOReturnIOError:0; +} +static int reserve(void *c){CHECK(c==&board);return step("reserve");} +static int bootGmu(void *c){CHECK(c==&board);return step("gmu");} +static int programSmmu(void *c){CHECK(c==&board);return step("smmu");} +static int quiesce(void *c) +{ + CHECK(c==&board);++board.quiesce_calls;board.order.push_back("quiesce"); + return board.quiesce_fail; +} +static NativeAdrenoGpuBoard ops{&board,reserve,bootGmu,programSmmu,quiesce}; +struct Registers { + std::vector values; + std::vector writes; + unsigned publish=0,acquire=0,barriers=0; + Registers():values(0x10000,0){} +}; +static Registers regs; +static uint32_t ringmem[8192],fencemem[1024]; +static int write32(void *c,uint32_t reg,uint32_t value) +{CHECK(c==®s && reg>16&0x7fu)==0x48 && ringmem[3]==0x14b); + CHECK(regs.values[0x0830]==0x300000 && regs.values[0x0038]==NativeAdrenoIrq::mask()); + CHECK(regs.values[0x0807]==17); + CHECK(gpu.start(&ops,&c)==kIOReturnBusy); + CHECK(gpu.stop()==kIOReturnSuccess && !gpu.ready()); + CHECK(board.quiesce_calls==1); /* only the explicit stop */ + CHECK(gpu.stop()==kIOReturnNotReady); + } + /* Each board step failure stops the sequence and quiesces exactly once. */ + { + for(unsigned failure=1;failure<=3;++failure) { + reset();board.fail_call=failure; + NativeAdrenoGpu gpu;const NativeAdrenoGpuConfig c=config(); + CHECK(gpu.start(&ops,&c)==kIOReturnIOError); + CHECK(gpu.failed() && !gpu.ready() && gpu.stepsCompleted()==failure-1); + CHECK(board.quiesce_calls==(failure==1?0:1)); + CHECK(board.order.size()==(failure==1?1:failure+1)); + CHECK(!gpu.ring() && !gpu.interrupts() && gpu.stop()==kIOReturnNotReady); + if(failure>1)CHECK(!std::strcmp(board.order.back(),"quiesce")); + CHECK(regs.writes.empty()); + } + } + /* Real component failures quarantine and never retry. */ + { + reset();board.fail_call=0;NativeAdrenoGpu gpu;NativeAdrenoGpuConfig c=config(); + c.ring_iova=0x201000; /* not ring-aligned */ + CHECK(gpu.start(&ops,&c)==kIOReturnBadArgument); + CHECK(gpu.failed() && gpu.quiesced() && board.quiesce_calls==1); + CHECK(!gpu.ready() && board.order.size()==4 && !std::strcmp(board.order[3],"quiesce")); + const NativeAdrenoGpuConfig again=config(); + CHECK(gpu.start(&ops,&again)==kIOReturnBusy); + } + { + reset();NativeAdrenoGpu gpu;NativeAdrenoGpuConfig c=config(); + c.sqe_iova=0x300001; /* CP init refuses an unaligned SQE base */ + CHECK(gpu.start(&ops,&c)==kIOReturnBadArgument); + CHECK(gpu.failed() && gpu.quiesced() && board.order.size()==4); + CHECK(regs.values[0x0800]==0x200000); /* ring was attached before failing */ + } + { + reset();board.quiesce_fail=1;NativeAdrenoGpu gpu;NativeAdrenoGpuConfig c=config(); + c.ring_bytes=4096; /* refused by the ring before CP start */ + CHECK(gpu.start(&ops,&c)==kIOReturnBadArgument); + CHECK(gpu.failed() && !gpu.quiesced() && board.quiesce_calls==1); + CHECK(!gpu.ready()); + CHECK(board.order.size()==4 && !std::strcmp(board.order[3],"quiesce")); + } + /* A CP start fence that never completes is a timeout, not readiness. */ + { + reset();start_pending=0;board.fail_call=0; + NativeAdrenoGpu gpu;const NativeAdrenoGpuConfig c=config(); + CHECK(gpu.start(&ops,&c)==kIOReturnTimeout); + CHECK(gpu.failed() && gpu.quiesced() && board.quiesce_calls==1); + CHECK(!gpu.ready() && gpu.startFence()==1); /* issued but never observed */ + CHECK(regs.values[0x0807]==17); /* ME_INIT was submitted */ + } + /* Argument refusals precede reserve and never touch the board. */ + { + reset();NativeAdrenoGpu gpu;const NativeAdrenoGpuConfig c=config(); + CHECK(gpu.start(nullptr,&c)==kIOReturnBadArgument); + CHECK(gpu.start(&ops,nullptr)==kIOReturnBadArgument); + NativeAdrenoGpuBoard broken=ops;broken.quiesce=nullptr; + CHECK(gpu.start(&broken,&c)==kIOReturnBadArgument); + NativeAdrenoGpuConfig bad=c;bad.cache.publish=nullptr; + CHECK(gpu.start(&ops,&bad)==kIOReturnBadArgument); + bad=c;bad.ring=nullptr; + CHECK(gpu.start(&ops,&bad)==kIOReturnBadArgument); + CHECK(board.calls==0 && board.quiesce_calls==0 && !gpu.failed()); + /* Validation failures do not consume the one attempt. */ + CHECK(gpu.start(&ops,&c)==kIOReturnSuccess && gpu.ready()); + } + /* Optional timer absent, stop refuses while a submission is outstanding. */ + { + reset();NativeAdrenoGpu gpu;NativeAdrenoGpuConfig c=config();c.time.read32=nullptr; + CHECK(gpu.start(&ops,&c)==kIOReturnSuccess && gpu.timer()==nullptr); + const uint32_t one=1;uint32_t fence=0; + CHECK(gpu.ring()->submit(&one,1,&fence)==kIOReturnSuccess); + CHECK(gpu.stop()==kIOReturnBusy); + fencemem[1]=fence; + CHECK(gpu.ring()->wait(fence,nullptr,0,nullptr)==NativeAdrenoRingFenceComplete); + CHECK(gpu.stop()==kIOReturnSuccess); + } + std::printf("PASS: 1 full lifecycle start/stop, 3 board-step failures, 3 real component failures and 6 argument refusals over software GPU registers and fake board power\n"); +} diff --git a/drivers/NativeAdreno/tests/test_irq.cpp b/drivers/NativeAdreno/tests/test_irq.cpp new file mode 100644 index 0000000..4b18a6e --- /dev/null +++ b/drivers/NativeAdreno/tests/test_irq.cpp @@ -0,0 +1,109 @@ +/* SPDX-License-Identifier: MIT */ +/* Real interrupt decode over a software register peer. No interrupt taken. */ +#include "XnuMemoryFixture.h" +#include "NativeAdrenoIrq.h" +#include +#define CHECK(x) do {if(!(x)){std::fprintf(stderr,"line %d: %s\n",__LINE__,#x);std::exit(1);}}while(0) +struct Peer { + uint32_t regs[0x10000]{}; + unsigned reads=0,writes=0,barriers=0,fail_read=0,fail_write=0,fail_barrier=0; + uint32_t cleared=0xffffffff,last_write_reg=0; +}; +static Peer peer; +static int write32(void *c,uint32_t reg,uint32_t v) +{ + CHECK(c==&peer && reg<0x10000);++peer.writes; + if(peer.writes==peer.fail_write)return 1; + peer.regs[reg]=v;peer.last_write_reg=reg; + if(reg==0x0037)peer.cleared=v; + return 0; +} +static int read32(void *c,uint32_t reg,uint32_t *v) +{ + CHECK(c==&peer && v && reg<0x10000);++peer.reads; + if(peer.reads==peer.fail_read)return 1; + *v=peer.regs[reg];return 0; +} +static int barrier(void *c){CHECK(c==&peer);++peer.barriers;return peer.barriers==peer.fail_barrier;} +static NativeAdrenoRingIo io{&peer,write32,read32,barrier}; +static void reset(void) +{ + peer=Peer{};peer.cleared=0xffffffff; + for(auto &v:peer.regs)v=0; +} +int main() +{ + /* Mask programming matches the pinned A7XX enable set exactly. */ + { + reset();NativeAdrenoIrq irq;CHECK(irq.attach(&io)==kIOReturnSuccess); + uint32_t programmed=0; + CHECK(irq.programMask(&programmed)==kIOReturnSuccess); + CHECK(programmed==NativeAdrenoIrq::mask()); + CHECK(programmed==((1u<<1)|(1u<<6)|(1u<<7)|(1u<<8)|(1u<<9)|(1u<<15)|(1u<<17)| + (1u<<20)|(1u<<22)|(1u<<23)|(1u<<24)|(1u<<25)|(1u<<28)|(1u<<29))); + CHECK(peer.regs[0x0038]==programmed && irq.enabled()==programmed); + CHECK(irq.detach()==kIOReturnSuccess && irq.detach()==kIOReturnNotReady); + } + /* Each class is decoded, acknowledged and classified. */ + { + reset();NativeAdrenoIrq irq;CHECK(irq.attach(&io)==kIOReturnSuccess); + NativeAdrenoIrqResult result; + peer.regs[0x0201]=1u<<20; + CHECK(irq.handle(&result)==kIOReturnSuccess); + CHECK(peer.cleared==(1u<<20) && peer.last_write_reg==0x0037); + CHECK(result.flags==NativeAdrenoIrqFence && !(result.flags&NativeAdrenoIrqFatal)); + peer.regs[0x0201]=1u<<0; + CHECK(irq.handle(&result)==kIOReturnSuccess && result.flags==NativeAdrenoIrqIdle); + peer.regs[0x0201]=1u<<8; + CHECK(irq.handle(&result)==kIOReturnSuccess && result.flags==NativeAdrenoIrqPreempt); + peer.regs[0x0201]=1u<<23; + CHECK(irq.handle(&result)==kIOReturnSuccess && + result.flags==(NativeAdrenoIrqHang|NativeAdrenoIrqFatal)); + peer.regs[0x0201]=1u<<24; + CHECK(irq.handle(&result)==kIOReturnSuccess && + result.flags==(NativeAdrenoIrqUche|NativeAdrenoIrqFatal)); + peer.regs[0x0201]=(1u<<1)|(1u<<7)|(1u<<22); + CHECK(irq.handle(&result)==kIOReturnSuccess && + result.flags==(NativeAdrenoIrqBusError|NativeAdrenoIrqFatal)); + peer.regs[0x0201]=(1u<<28)|(1u<<29); + CHECK(irq.handle(&result)==kIOReturnSuccess && + result.flags==(NativeAdrenoIrqSecurity|NativeAdrenoIrqFatal)); + /* CP_HW_ERROR without a CP status bit is not a CP fault class. */ + peer.regs[0x0201]=1u<<9;peer.regs[0x0823]=0; + CHECK(irq.handle(&result)==kIOReturnSuccess && !result.flags && !result.cp_status); + peer.regs[0x0201]=1u<<9;peer.regs[0x0823]=0xf7;peer.regs[0x0824]=0x00100003; + CHECK(irq.handle(&result)==kIOReturnSuccess); + CHECK(result.flags==(NativeAdrenoIrqCpError|NativeAdrenoIrqFatal)); + CHECK(result.cp_status==0xf7 && result.protect_status==0x00100003); + /* An unprogrammed source must never be silently ignored. */ + peer.regs[0x0201]=1u<<3; + CHECK(irq.handle(&result)==kIOReturnSuccess && + result.flags==(NativeAdrenoIrqUnknown|NativeAdrenoIrqFatal) && result.unknown==(1u<<3)); + peer.regs[0x0201]=0; + CHECK(irq.handle(&result)==kIOReturnSuccess && !result.flags && peer.cleared==0); + CHECK(irq.detach()==kIOReturnSuccess); + } + /* Failures poison because the source may still be armed. */ + { + reset();NativeAdrenoIrq irq;CHECK(irq.attach(&io)==kIOReturnSuccess); + peer.fail_read=1;CHECK(irq.programMask(nullptr)==kIOReturnIOError && irq.poisoned()); + CHECK(irq.detach()==kIOReturnNotReady); + reset();NativeAdrenoIrq second;CHECK(second.attach(&io)==kIOReturnSuccess); + peer.fail_write=1;CHECK(second.programMask(nullptr)==kIOReturnIOError && second.poisoned()); + reset();NativeAdrenoIrq third;CHECK(third.attach(&io)==kIOReturnSuccess); + peer.regs[0x0201]=1u<<23; + peer.fail_write=1; /* clear write failure after a successful status read */ + NativeAdrenoIrqResult result; + CHECK(third.handle(&result)==kIOReturnIOError && third.poisoned()); + reset();NativeAdrenoIrq fourth;CHECK(fourth.attach(&io)==kIOReturnSuccess); + peer.regs[0x0201]=1u<<9;peer.regs[0x0823]=1; + peer.fail_read=2; /* CP status read failure */ + CHECK(fourth.handle(&result)==kIOReturnIOError && fourth.poisoned()); + reset();NativeAdrenoIrq fifth;CHECK(fifth.attach(&io)==kIOReturnSuccess); + peer.fail_barrier=1; + CHECK(fifth.handle(&result)==kIOReturnIOError && fifth.poisoned()); + CHECK(fifth.detach()==kIOReturnNotReady); + CHECK(irq.detach()==kIOReturnNotReady && second.detach()==kIOReturnNotReady); + } + std::printf("PASS: 1 exact A7XX mask programming, 10 decoded interrupt/status cases, 5 poison-on-unacknowledged cases over a software register peer\n"); +} diff --git a/drivers/NativeAdreno/tests/test_memory.cpp b/drivers/NativeAdreno/tests/test_memory.cpp new file mode 100644 index 0000000..6a3c6d5 --- /dev/null +++ b/drivers/NativeAdreno/tests/test_memory.cpp @@ -0,0 +1,136 @@ +/* SPDX-License-Identifier: MIT */ +/* Real memory-management implementation linked to host-only IOKit doubles. + * Production CPU/cache assembly is separately cross-built, never run here. */ +#include "XnuMemoryFixture.h" +#include "NativeAdrenoMemory.h" +#include +#define CHECK(x) do {if(!(x)){std::fprintf(stderr,"line %d: %s\n",__LINE__,#x);std::exit(1);}}while(0) +IOReturn NativeAdrenoQueryCpu4K(NativeAdrenoCpuState *out) +{CHECK(out);*out={};return fixture.cpu_status;} +IOReturn NativeAdrenoMaintainOwnedPages(void *mapping,size_t bytes,NativeAdrenoCacheOperation op) +{ + CHECK(mapping && bytes && !(((uintptr_t)mapping|bytes)&4095)); + bool owned=false; + for(auto *d:fixture.live)if((uintptr_t)mapping>=(uintptr_t)d->raw && + (uintptr_t)mapping-(uintptr_t)d->raw<=d->bytes && + bytes<=d->bytes-((uintptr_t)mapping-(uintptr_t)d->raw))owned=true; + CHECK(owned);++fixture.cache_calls;fixture.cache_operations.push_back(op); + return fixture.cache_calls==fixture.cache_fail_call?kIOReturnIOError:kIOReturnSuccess; +} +static void reset() +{CHECK(fixture.live.empty() && fixture.metadata.empty());fixture=MemoryFixture{};} +/* Host fixture teardown only. This is not a production reset/release path. */ +static void discardFixtureAfterQuarantine() +{ + while(!fixture.live.empty())fixture.live.back()->release(); + while(!fixture.metadata.empty())fixtureMetadataFree(fixture.metadata.back()); + reset(); +} +static unsigned successes() +{ + unsigned profiles=0,outputs[]={32,36,40,42,44,48}; + for(unsigned output:outputs)for(unsigned mode:{Contiguous,Scattered,Paired}) { + reset();fixture.mode=static_cast(mode);NativeAdrenoMemory memory; + CHECK(memory.allocate(16384,output,mode==Contiguous,16384)==kIOReturnSuccess); + CHECK(memory.bytes()==16384 && memory.mapping() && memory.descriptor() && !memory.quarantined()); + size_t count=mode==Contiguous?1:mode==Scattered?4:2; + CHECK(memory.spanCount()==count && fixture.prepared==1 && fixture.segment_calls==4); + for(size_t i=0;i<16384;++i)CHECK(static_cast(memory.mapping())[i]==0); + uint64_t bytes=0; + for(size_t i=0;i=6 && failure<=9)fixture.mode=static_cast(ZeroPhysical+(failure-6)); + if(failure==10)fixture.mode=Scattered; + if(failure==11) {fixture.cache_fail_call=1;expected=kIOReturnIOError;} + CHECK(memory.allocate(16384,32,true,16384)==expected); + CHECK(!memory.descriptor() && !memory.spans() && !memory.quarantined() && fixture.live.empty()); + CHECK(fixture.released==fixture.created); + CHECK(fixture.completed==(failure<2?0:1));++controls; + } + for(unsigned cleanup_failure=0;cleanup_failure<3;++cleanup_failure) { + reset();NativeAdrenoMemory memory; + if(cleanup_failure==1)fixture.mode=ZeroPhysical; + if(cleanup_failure==2)fixture.cache_fail_call=1; + fixture.complete_fail=true; + IOReturn allocated=memory.allocate(16384,32,false,16384); + if(!cleanup_failure) { + CHECK(allocated==kIOReturnSuccess); + CHECK(memory.release()==kIOReturnIOError); + } else CHECK(allocated==(cleanup_failure==1?kIOReturnUnsupported:kIOReturnIOError)); + CHECK(memory.quarantined() && memory.descriptor() && memory.bytes()==16384); + unsigned completed=fixture.completed; + CHECK(memory.release()==kIOReturnBusy && fixture.completed==completed && !fixture.released); + CHECK(memory.holdForDevice()==kIOReturnNotReady); + CHECK(memory.publishForDevice(0,4096,false)==kIOReturnNotReady); + discardFixtureAfterQuarantine();++controls; + } + reset();NativeAdrenoMemory memory; + CHECK(memory.allocate(16384,32,false,16384)==kIOReturnSuccess); + CHECK(memory.holdForDevice()==kIOReturnSuccess);memory.quarantine(); + CHECK(memory.retireAfterQuiescence()==kIOReturnNotReady && memory.deviceReferences()==1); + CHECK(memory.release()==kIOReturnBusy && !fixture.completed && !fixture.released); + CHECK(memory.acquireDeviceWrites(0,4096)==kIOReturnNotReady); + discardFixtureAfterQuarantine();++controls; + return controls; +} +static unsigned arguments() +{ + reset();NativeAdrenoMemory memory;unsigned controls=0; + CHECK(memory.allocate(0,32,false,16384)==kIOReturnBadArgument);++controls; + CHECK(memory.allocate(4097,32,false,16384)==kIOReturnBadArgument);++controls; + CHECK(memory.allocate(4096,33,false,16384)==kIOReturnBadArgument);++controls; + CHECK(memory.allocate(4096,32,false,0)==kIOReturnBadArgument);++controls; + CHECK(memory.allocate(4096,32,false,256u*1024u*1024u+1)==kIOReturnBadArgument);++controls; + CHECK(memory.allocate(16384,32,false,4096)==kIOReturnNoResources);++controls; + CHECK(!fixture.created && !fixture.cache_calls); + fixture.cpu_status=kIOReturnUnsupported; + CHECK(memory.allocate(4096,32,false,4096)==kIOReturnUnsupported);++controls; + CHECK(!fixture.created);fixture.cpu_status=kIOReturnSuccess; + CHECK(memory.allocate(16384,32,false,16384)==kIOReturnSuccess); + CHECK(memory.allocate(16384,32,false,16384)==kIOReturnBusy);++controls; + for(auto range:std::vector>{{0,0},{1,4096},{0,4097},{16384,4096},{SIZE_MAX-4095,4096},{4096,SIZE_MAX-4095}}) { + CHECK(memory.publishForDevice(range.first,range.second,false)==kIOReturnBadArgument); + CHECK(memory.acquireDeviceWrites(range.first,range.second)==kIOReturnBadArgument);controls+=2; + } + CHECK(fixture.cache_calls==1);CHECK(memory.release()==kIOReturnSuccess); + return controls; +} +int main() +{ + auto profiles=successes(),failure_controls=failures(),argument_controls=arguments();reset(); + std::printf("PASS: %u allocation profiles, %u allocation/cache/lifetime failure controls, %u argument/CPU refusal controls\n",profiles,failure_controls,argument_controls); + std::puts("Host IOKit doubles only; no real kernel allocation, physical provenance, cache/DMA, SMMU, GPU or fence qualification"); +} diff --git a/drivers/NativeAdreno/tests/test_pwrup.cpp b/drivers/NativeAdreno/tests/test_pwrup.cpp new file mode 100644 index 0000000..758038d --- /dev/null +++ b/drivers/NativeAdreno/tests/test_pwrup.cpp @@ -0,0 +1,62 @@ +/* SPDX-License-Identifier: MIT */ +/* Exact power-up list layout over synthetic register pairs. */ +#include "XnuMemoryFixture.h" +#include "NativeAdrenoPwrup.h" +#include "NativeAdrenoPwrupRegs.h" +#include +#include +#define CHECK(x) do {if(!(x)){std::fprintf(stderr,"line %d: %s\n",__LINE__,#x);std::exit(1);}}while(0) +int main() +{ + const uint32_t ir[]={0x10,0x20},iv[]={0xaa,0xbb}; + const uint32_t pr[]={0x30},pv[]={0xcc}; + const uint32_t dt[]={0x1,0x40,0xdd,0x2,0x50,0xee}; + NativeAdrenoPwrupCounts counts{2,1,2}; + CHECK(NativeAdrenoPwrupList::requiredBytes(counts)==16+3*8+2*12); + uint8_t image[128]; + std::memset(image,0x5a,sizeof(image)); + size_t written=0; + CHECK(NativeAdrenoPwrupList::pack(counts,ir,iv,pr,pv,dt,image,sizeof(image),&written)==kIOReturnSuccess); + CHECK(written==16+24+24); + const uint32_t expected[]={0,0,0,0x00020102,0x10,0xaa,0x20,0xbb,0x30,0xcc,1,0x40,0xdd,2,0x50,0xee}; + CHECK(!std::memcmp(image,expected,sizeof(expected))); + CHECK(image[written]==0x5a); /* no write past the image */ + /* Capacity and count refusals leave everything untouched. */ + size_t before=0; + std::memset(image,0x5a,sizeof(image)); + CHECK(NativeAdrenoPwrupList::pack(counts,ir,iv,pr,pv,dt,image,written-1,&before)==kIOReturnNoResources); + CHECK(!before && image[0]==0x5a && image[sizeof(image)-1]==0x5a); + CHECK(NativeAdrenoPwrupList::pack(counts,nullptr,iv,pr,pv,dt,image,sizeof(image),&before)==kIOReturnBadArgument); + CHECK(NativeAdrenoPwrupList::pack(counts,ir,iv,pr,pv,nullptr,image,sizeof(image),&before)==kIOReturnBadArgument); + CHECK(NativeAdrenoPwrupList::pack(counts,ir,iv,pr,pv,dt,nullptr,sizeof(image),&before)==kIOReturnBadArgument); + CHECK(NativeAdrenoPwrupList::pack(counts,ir,iv,pr,pv,dt,image,sizeof(image),nullptr)==kIOReturnBadArgument); + NativeAdrenoPwrupCounts big{0x100,0,0}; + CHECK(NativeAdrenoPwrupList::pack(big,ir,iv,pr,pv,dt,image,sizeof(image),&before)==kIOReturnBadArgument); + NativeAdrenoPwrupCounts zero{0,0,0}; + CHECK(NativeAdrenoPwrupList::requiredBytes(zero)==16); + CHECK(NativeAdrenoPwrupList::pack(zero,nullptr,nullptr,nullptr,nullptr,nullptr,image,16,&before)==kIOReturnSuccess); + CHECK(before==16 && !image[12] && !image[13] && !image[14] && !image[15]); + /* A740 builder: sixteen live reads packed as the preemption list. */ + { + struct Reader {unsigned calls;unsigned fail_at;} reader{0,0}; + auto read=[](void *cookie,uint32_t reg,uint32_t *value)->int { + Reader *state=static_cast(cookie); + ++state->calls; + if(state->calls==state->fail_at)return 1; + *value=reg^0xf0f0;return 0; + }; + uint8_t built[256]; + size_t built_bytes=0; + CHECK(NativeAdrenoPwrupList::buildA740(read,&reader,built,sizeof(built),&built_bytes)==kIOReturnSuccess); + CHECK(reader.calls==16 && built_bytes==16+16*8); + CHECK(built[12]==0 && built[13]==16 && built[14]==0 && built[15]==0); + const uint32_t *slots=reinterpret_cast(built+16); + for(unsigned i=0;i<16;++i)CHECK(slots[i*2]==NativeAdrenoA740PwrupRegs[i] && + slots[i*2+1]==(NativeAdrenoA740PwrupRegs[i]^0xf0f0)); + Reader failing{0,9};size_t untouched=77; + CHECK(NativeAdrenoPwrupList::buildA740(read,&failing,built,sizeof(built),&untouched)==kIOReturnIOError); + CHECK(untouched==77); + CHECK(NativeAdrenoPwrupList::buildA740(nullptr,nullptr,built,sizeof(built),&untouched)==kIOReturnBadArgument); + } + std::printf("PASS: 1 exact power-up list image, 1 A740 live-read list, 1 capacity refusal and 5 argument/count refusals\n"); +} diff --git a/drivers/NativeAdreno/tests/test_ring.cpp b/drivers/NativeAdreno/tests/test_ring.cpp new file mode 100644 index 0000000..5bdd1d8 --- /dev/null +++ b/drivers/NativeAdreno/tests/test_ring.cpp @@ -0,0 +1,217 @@ +/* SPDX-License-Identifier: MIT */ +/* Real ring/fence code over a software CP register peer. No GPU access. */ +#include "XnuMemoryFixture.h" +#include "NativeAdrenoRing.h" +#include +#include +#include +#define CHECK(x) do {if(!(x)){std::fprintf(stderr,"line %d: %s\n",__LINE__,#x);std::exit(1);}}while(0) +struct Peer { + std::vector regs; + std::vector> writes; + unsigned read_calls=0,publish=0,acquire=0,barriers=0; + unsigned fail_write=0,fail_read=0,fail_barrier=0,fail_publish=0,fail_acquire=0; + bool log_order=false;std::vector order; + Peer():regs(0x10000,0){} +}; +static Peer peer; +static uint64_t clock_us=0; +static int write32(void *cookie,uint32_t reg,uint32_t value) +{ + CHECK(cookie==&peer && reg>28)==7u?(header>>16)&0x7fu:0xffu;}; + CHECK(op(ring[0])==0x17 && ring[1]==0x80000001u); + CHECK(op(ring[2])==0x46 && ring[3]==0x32 && op(ring[4])==0x46 && ring[5]==0x33 && op(ring[6])==0x26 && (ring[6]&0x3fffu)==0); + CHECK(op(ring[7])==0x17 && ring[8]==3 && op(ring[9])==0x65 && ring[10]==0x101); + for(unsigned i=0;i<4;++i)CHECK(ring[11+i]==words[i]); + CHECK((ring[15]>>8&0x3ffffu)==0x885 && (ring[15]&0x3fu)==1 && ring[16]==1); + CHECK(op(ring[17])==0x17 && ring[18]==1 && op(ring[19])==0x46 && ring[20]==0x18 && op(ring[21])==0x46 && ring[22]==0x19); + CHECK(op(ring[23])==0x17 && ring[24]==2); + CHECK(op(ring[25])==0x46 && ring[26]==(0x31u|(1u<<27)) && ring[27]==0x10800c && ring[28]==0 && ring[29]==1); /* BV timestamp */ + CHECK(op(ring[30])==0x17 && ring[31]==1); + CHECK(op(ring[32])==0x14 && ring[33]==0 && ring[34]==0x10800c && ring[35]==0 && ring[36]==1); /* BR waits for BV */ + CHECK(op(ring[37])==0x46 && ring[38]==(0x31u|(1u<<27)|(1u<<31))); + CHECK(ring[39]==0x108004 && ring[40]==0 && ring[41]==1); /* msm_rbmemptrs.fence, where wait() reads */ + CHECK(op(ring[42])==0x17 && ring[43]==3 && op(ring[44])==0x65 && ring[45]==0x100); + CHECK(peer.publish==1); + /* Cache publication precedes the barrier, which precedes WPTR. */ + CHECK(peer.order.size()==3 && !std::strcmp(peer.order[0],"publish") && + !std::strcmp(peer.order[1],"barrier") && !std::strcmp(peer.order[2],"write")); + uint32_t observed=0; + CHECK(r.wait(1,&clock_,5000,&observed)==NativeAdrenoRingFenceTimeout && !observed); + fence[1]=1; + CHECK(r.wait(1,&clock_,5000,&observed)==NativeAdrenoRingFenceComplete && + observed==1 && r.outstanding()==0 && r.completedFence()==1); + CHECK(r.wait(2,&clock_,5000,&observed)==NativeAdrenoRingFenceArgument); + CHECK(r.detach()==kIOReturnSuccess); + } + /* A requested context reset puts a6xx_set_pagetable()'s A7XX context entry into the next framed prologue, once. */ + { + reset();NativeAdrenoRing r;CHECK(attach(r)==kIOReturnSuccess); + const uint32_t words[1]={0x11111111}; + uint32_t end=0; + r.requestContextReset(); + CHECK(r.submitFramed(words,1,&end)==kIOReturnSuccess && end==1); + CHECK(peer.writes.back().first==0x0807 && peer.writes.back().second==11+26+1+31); + auto op=[](uint32_t header) {return (header>>28)==7u?(header>>16)&0x7fu:0xffu;}; + CHECK(op(ring[0])==0x17 && ring[1]==0x80000001u); + CHECK(op(ring[2])==0x14 && ring[3]==0 && ring[4]==0x108004 && ring[5]==0 && ring[6]==0); /* wait for the last fence */ + CHECK(op(ring[7])==0x17 && ring[8]==0x80000003u && op(ring[9])==0x1f && ring[10]==0xf); + CHECK(op(ring[11])==0x17 && ring[12]==0x80000003u && op(ring[13])==0x46 && ring[14]==0x26); + CHECK(op(ring[15])==0x17 && ring[16]==0x80000001u && (ring[17]>>8&0x3ffffu)==0x50e && ring[18]==1); + CHECK(op(ring[19])==0x46 && ring[20]==0x31); + CHECK(op(ring[21])==0x3c && ring[22]==3 && ring[23]==0x50f && ring[24]==0 && ring[25]==1 && ring[26]==1 && ring[27]==0); + CHECK(op(ring[28])==0x46 && ring[29]==0x32 && op(ring[30])==0x46 && ring[31]==0x33 && op(ring[32])==0x26); + CHECK(op(ring[33])==0x17 && ring[34]==3 && op(ring[35])==0x65 && ring[36]==0x101 && ring[37]==words[0]); + fence[1]=1; + CHECK(r.submitFramed(words,1,&end)==kIOReturnSuccess && end==2); + CHECK(peer.writes.back().second==69+11+1+31 && op(ring[69+2])==0x46 && op(ring[69+7])==0x17 && ring[69+8]==3); + } + /* Wrap-around emission keeps packets contiguous modulo the ring. */ + { + reset();NativeAdrenoRing r;CHECK(attach(r)==kIOReturnSuccess); + const uint32_t big[1]={0xabc}; + uint32_t end=0; + CHECK(r.submit(big,1,&end)==kIOReturnSuccess); + CHECK(r.submit(nullptr,0,&end)==kIOReturnBadArgument); + /* Re-attach is refused while the object is live. */ + CHECK(attach(r)==kIOReturnBusy); + CHECK(r.detach()==kIOReturnBusy); /* one unobserved submission */ + } + { + reset();NativeAdrenoRing r;CHECK(attach(r)==kIOReturnSuccess); + uint32_t words[sizeof(ring)/4-53]; + for(auto &w:words)w=0x5a5a5a5a; + uint32_t end=0; + CHECK(r.submitFramed(words,sizeof(words)/sizeof(words[0]),&end)==kIOReturnSuccess); + /* Consume the whole submission, then a second one wraps past the end. */ + CHECK(end==1 && peer.writes.back().second==8181); + peer.regs[0x0806]=8181; + CHECK(r.submitFramed(words,4,&end)==kIOReturnSuccess); + CHECK(peer.writes.back().first==0x0807 && peer.writes.back().second==35); + CHECK(ring[8181]>=0x70000000u && ring[8191]==0x101); + CHECK(ring[0]==words[0] && ring[1]==words[1] && ring[2]==words[2] && ring[3]==words[3]); + CHECK((ring[4]>>8&0x3ffffu)==0x885 && ring[5]==2); + CHECK(ring[28]==0x108004 && ring[30]==2 && ring[34]==0x100); + CHECK(r.detach()==kIOReturnBusy); /* unobserved submissions */ + } + /* Argument refusals never touch registers or memory. */ + { + reset();NativeAdrenoRing r; + CHECK(r.attach(nullptr,&cache,ring,sizeof(ring),0x100000,fence,sizeof(fence),0x108000)==kIOReturnBadArgument); + CHECK(r.attach(&io,nullptr,ring,sizeof(ring),0x100000,fence,sizeof(fence),0x108000)==kIOReturnBadArgument); + CHECK(r.attach(&io,&cache,nullptr,sizeof(ring),0x100000,fence,sizeof(fence),0x108000)==kIOReturnBadArgument); + CHECK(r.attach(&io,&cache,ring,4096,0x100000,fence,sizeof(fence),0x108000)==kIOReturnBadArgument); + CHECK(r.attach(&io,&cache,ring,16384+4096,0x100000,fence,sizeof(fence),0x108000)==kIOReturnBadArgument); + CHECK(r.attach(&io,&cache,ring,sizeof(ring),0x104000,fence,sizeof(fence),0x108000)==kIOReturnBadArgument); + CHECK(r.attach(&io,&cache,ring,sizeof(ring),0x100000,fence,32,0x108000)==kIOReturnBadArgument); + CHECK(r.attach(&io,&cache,ring,sizeof(ring),0x100000,fence,sizeof(fence),0x108004)==kIOReturnBadArgument); + CHECK(peer.writes.empty() && peer.read_calls==0); + CHECK(r.submit(nullptr,1,nullptr)==kIOReturnBadArgument); + uint32_t end=0;CHECK(r.submit(nullptr,1,&end)==kIOReturnNotReady); + CHECK(r.detach()==kIOReturnNotReady); + CHECK(attach(r)==kIOReturnSuccess); + const uint32_t one=1; + CHECK(r.submit(nullptr,1,&end)==kIOReturnBadArgument); + CHECK(r.submit(&one,0,&end)==kIOReturnBadArgument); + uint32_t huge[sizeof(ring)/4];CHECK(r.submit(huge,sizeof(huge)/sizeof(huge[0]),&end)==kIOReturnNoResources); + CHECK(r.submit(&one,1,&end)==kIOReturnSuccess && end==1); + CHECK(r.detach()==kIOReturnBusy); + fence[1]=1;uint32_t observed=0; + CHECK(r.wait(1,&clock_,1000,&observed)==NativeAdrenoRingFenceComplete); + CHECK(r.detach()==kIOReturnSuccess); + } + /* Every attach register-write failure leaves the ring unusable and owned. */ + { + unsigned cases=0; + for(unsigned failure=1;failure<=7;++failure) { + reset();NativeAdrenoRing r;peer.fail_write=failure; + CHECK(attach(r)==kIOReturnIOError && r.poisoned()); + CHECK(r.detach()==kIOReturnBusy); + uint32_t end=0;CHECK(r.submit(&failure,1,&end)==kIOReturnNotReady); + CHECK(peer.writes.size()+1==failure); + ++cases; + } + reset();NativeAdrenoRing broken;peer.fail_read=1; + CHECK(attach(broken)==kIOReturnIOError && broken.poisoned() && broken.detach()==kIOReturnBusy);++cases; + /* Publication, barrier and WPTR failures poison after partial commit. */ + for(unsigned kind=0;kind<3;++kind) { + reset();NativeAdrenoRing r;CHECK(attach(r)==kIOReturnSuccess); + if(kind==0)peer.fail_publish=1; + if(kind==1)peer.fail_barrier=1; + if(kind==2)peer.fail_write=peer.writes.size()+1; + const uint32_t one=0x1234;uint32_t end=0; + CHECK(r.submit(&one,1,&end)==kIOReturnIOError && !end && r.poisoned()); + CHECK(r.outstanding()==0 && r.detach()==kIOReturnBusy); + CHECK(fence[1]==0); /* never completed */ + ++cases; + } + reset();NativeAdrenoRing r;CHECK(attach(r)==kIOReturnSuccess); + const uint32_t one=1;uint32_t end=0; + CHECK(r.submit(&one,1,&end)==kIOReturnSuccess); + peer.fail_acquire=peer.acquire+1; + uint32_t observed=0; + CHECK(r.wait(1,&clock_,10,&observed)==NativeAdrenoRingFencePoisoned && r.poisoned());++cases; + std::printf("PASS: 1 exact ring programming and submission, 1 wrap submission, 1 refused re-attach, 7 write/1 read attach failures and 4 post-commit poison cases over a software CP register peer (%u failure cases)\n", + cases); + } +} diff --git a/drivers/NativeAdreno/tests/test_sha256.cpp b/drivers/NativeAdreno/tests/test_sha256.cpp new file mode 100644 index 0000000..8693ed1 --- /dev/null +++ b/drivers/NativeAdreno/tests/test_sha256.cpp @@ -0,0 +1,86 @@ +/* SPDX-License-Identifier: MIT */ +/* FIPS vectors plus the real pinned GMU firmware; host-side file read only. */ +#include "XnuMemoryFixture.h" +#include "NativeAdrenoFirmware.h" +#include "NativeAdrenoSha256.h" +#include +#include +#include +#include +#define CHECK(x) do {if(!(x)){std::fprintf(stderr,"line %d: %s\n",__LINE__,#x);std::exit(1);}}while(0) +static int hex(const char *text,uint8_t *out,size_t bytes) +{ + if(std::strlen(text)!=bytes*2)return 1; + for(size_t i=0;i(message), + message?std::strlen(message):0,digest); + CHECK(hex(wanted,expected,32)==0); + if(std::memcmp(digest,expected,32)) { + std::fprintf(stderr,"vector %s mismatch\n",label); + std::exit(1); + } +} +int main(int argc,char **argv) +{ + uint8_t digest[32]; + NativeAdrenoSha256::hash(nullptr,0,digest); + uint8_t empty[32]; + CHECK(hex("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",empty,32)==0); + CHECK(std::memcmp(digest,empty,32)==0); + expect("abc","abc","ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"); + expect("two-block","abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq", + "248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1"); + { + std::vector million(1000000,uint8_t('a')); + NativeAdrenoSha256::hash(million.data(),million.size(),digest); + CHECK(hex("cdc76e5c9914fb9281a1c7e284d73e67f1809a48a497200e046d39ccc7112cd0",empty,32)==0); + CHECK(std::memcmp(digest,empty,32)==0); + } + /* Real firmware: exact length and pinned digest, then refusals. */ + if(argc==3) { + FILE *file=std::fopen(argv[1],"rb"); + CHECK(file); + CHECK(std::fseek(file,0,SEEK_END)==0); + const long size=std::ftell(file); + CHECK(size>0); + CHECK(std::fseek(file,0,SEEK_SET)==0); + std::vector image(static_cast(size)); + CHECK(std::fread(image.data(),1,image.size(),file)==image.size()); + CHECK(std::fclose(file)==0); + NativeAdrenoFirmwareRules rules{}; + rules.bytes=image.size(); + CHECK(hex(argv[2],rules.sha256,32)==0); + NativeAdrenoFirmwareView view{}; + CHECK(NativeAdrenoFirmware::validate(image.data(),image.size(),&rules,&view)==kIOReturnSuccess); + CHECK(view.bytes==image.data() && view.length==image.size()); + CHECK(!std::memcmp(view.sha256,rules.sha256,32)); + NativeAdrenoFirmwareRules wrong_size=rules; + wrong_size.bytes=image.size()-1; + CHECK(NativeAdrenoFirmware::validate(image.data(),image.size(),&wrong_size,&view)==kIOReturnBadArgument); + CHECK(!view.bytes); + NativeAdrenoFirmwareRules wrong_hash=rules; + wrong_hash.sha256[0]^=1; + CHECK(NativeAdrenoFirmware::validate(image.data(),image.size(),&wrong_hash,&view)==kIOReturnIOError); + CHECK(!view.bytes && !view.length); + std::vector corrupt=image; + corrupt[corrupt.size()/2]^=1; + CHECK(NativeAdrenoFirmware::validate(corrupt.data(),corrupt.size(),&rules,&view)==kIOReturnIOError); + CHECK(NativeAdrenoFirmware::validate(nullptr,image.size(),&rules,&view)==kIOReturnBadArgument); + CHECK(NativeAdrenoFirmware::validate(image.data(),image.size(),nullptr,&view)==kIOReturnBadArgument); + CHECK(NativeAdrenoFirmware::validate(image.data(),image.size(),&rules,nullptr)==kIOReturnBadArgument); + CHECK(NativeAdrenoFirmware::validate(image.data(),0,&rules,&view)==kIOReturnBadArgument); + std::printf("PASS: 4 FIPS vectors and the pinned GMU firmware (%ld bytes) validated, 6 malformed/limit refusals\n", + static_cast(image.size())); + } else { + std::printf("PASS: 4 FIPS vectors; no firmware path supplied\n"); + } +} diff --git a/drivers/NativeAdreno/tests/test_smmu_memory.cpp b/drivers/NativeAdreno/tests/test_smmu_memory.cpp new file mode 100644 index 0000000..706777d --- /dev/null +++ b/drivers/NativeAdreno/tests/test_smmu_memory.cpp @@ -0,0 +1,54 @@ +/* SPDX-License-Identifier: MIT */ +/* Real XNU memory adapter plus real table builder/auditor; IOKit peers only. */ +#include "XnuMemoryFixture.h" +#include "NativeAdrenoSmmuMemory.h" +#include +#define CHECK(x) do {if(!(x)){std::fprintf(stderr,"line %d: %s\n",__LINE__,#x);std::exit(1);}}while(0) +IOReturn NativeAdrenoQueryCpu4K(NativeAdrenoCpuState *out) +{CHECK(out);*out={};return kIOReturnSuccess;} +IOReturn NativeAdrenoMaintainOwnedPages(void *mapping,size_t bytes,NativeAdrenoCacheOperation op) +{ + CHECK(mapping && bytes && !(((uintptr_t)mapping|bytes)&4095)); + CHECK(op==NativeAdrenoClean);++fixture.cache_calls; + return fixture.cache_calls==fixture.cache_fail_call?kIOReturnIOError:kIOReturnSuccess; +} +int main() +{ + unsigned profiles=0; + for(unsigned bits=32;bits<=48;++bits)for(unsigned upper=0;upper<2;++upper) { + NativeAdrenoMemory memory;ArmSmmu4KLiveArena arena;ArmSmmu4KLiveState state{}; + ArmSmmu4KLiveVisibility visibility; + CHECK(memory.allocate(8*4096,40,true,8*4096)==kIOReturnSuccess); + uint64_t va=(upper?UINT64_MAX-(UINT64_C(1)< +#define CHECK(x) do {if(!(x)){std::fprintf(stderr,"line %d: %s\n",__LINE__,#x);std::exit(1);}}while(0) +IOReturn NativeAdrenoQueryCpu4K(NativeAdrenoCpuState *state) +{CHECK(state);*state={};return kIOReturnSuccess;} +IOReturn NativeAdrenoMaintainOwnedPages(void *mapping,size_t bytes,NativeAdrenoCacheOperation op) +{CHECK(mapping && bytes && op<=NativeAdrenoInvalidate);return kIOReturnSuccess;} +int main() +{ + const uint64_t base=UINT64_C(0xffffffff00000000)+0x10000; + NativeAdrenoMemory tables; + CHECK(tables.allocate(32*4096,40,true,32*4096)==kIOReturnSuccess); + NativeAdrenoSmmuContext context; + CHECK(context.program(nullptr)==kIOReturnBadArgument); + CHECK(context.attach(nullptr,0)==kIOReturnBadArgument); + CHECK(context.observe()==kIOReturnBadArgument); + CHECK(context.disable()==kIOReturnBadArgument); + CHECK(context.prepare(&tables,1,base,0x4000,8,2,1,1000)==kIOReturnSuccess); + CHECK(context.prepared() && !context.programmed() && !context.attached()); + CHECK(context.configuration()->upper!=nullptr && context.configuration()->lower==nullptr); + CHECK(context.configuration()->asid==1 && context.configuration()->coherent_walk==1); + CHECK(context.prepare(&tables,1,base,0x4000,8,2,1,1000)==kIOReturnBusy); + CHECK(context.program(nullptr)==kIOReturnBadArgument); + CHECK(context.attach(nullptr,1)==kIOReturnBadArgument); + CHECK(context.observe()==kIOReturnBadArgument); + CHECK(context.disable()==kIOReturnBadArgument); + /* A refused pool or bad bank/ASID/budget never prepares. */ + NativeAdrenoSmmuContext bad; + CHECK(bad.prepare(nullptr,1,base,0x4000,8,2,1,1000)==kIOReturnBadArgument); + CHECK(bad.prepare(&tables,1,base+1,0x4000,8,2,1,1000)==kIOReturnBadArgument); + CHECK(bad.prepare(&tables,1,base,0,8,2,1,1000)==kIOReturnBadArgument); + CHECK(bad.prepare(&tables,1,base,0x4000,0,2,1,1000)==kIOReturnBadArgument); + CHECK(bad.prepare(&tables,1,base,0x4000,8,64,1,1000)==kIOReturnBadArgument); + CHECK(bad.prepare(&tables,1,base,0x4000,8,2,0x10000,1000)==kIOReturnBadArgument); + CHECK(bad.prepare(&tables,1,base,0x4000,8,2,1,0)==kIOReturnBadArgument); + CHECK(bad.prepare(&tables,2,base,0x4000,8,2,1,1000)==kIOReturnBadArgument); + CHECK(!bad.prepared()); + /* The GMU's lower region: TTBR0 only. */ + NativeAdrenoMemory low_tables; + CHECK(low_tables.allocate(32*4096,40,true,32*4096)==kIOReturnSuccess); + NativeAdrenoSmmuContext low; + CHECK(low.prepare(&low_tables,0,0x60000000,0x8000,8,1,1,1000)==kIOReturnSuccess); + CHECK(low.configuration()->lower!=nullptr && low.configuration()->upper==nullptr); + CHECK(low.configuration()->lower_tables!=nullptr && low.configuration()->upper_tables==nullptr); + CHECK(low_tables.retireAfterQuiescence()==kIOReturnSuccess); + CHECK(low_tables.release()==kIOReturnSuccess); + CHECK(context.configuration()->upper_tables!=nullptr); + CHECK(tables.retireAfterQuiescence()==kIOReturnSuccess); + CHECK(tables.release()==kIOReturnSuccess); + CHECK(fixture.live.empty() && fixture.metadata.empty()); + std::printf("PASS: 1 prepared upper and 1 prepared lower SMMU context, 6 step-order refusals and 8 prepare refusals over host doubles\n"); +} diff --git a/drivers/NativeAdreno/tests/test_submit.cpp b/drivers/NativeAdreno/tests/test_submit.cpp new file mode 100644 index 0000000..e8337d3 --- /dev/null +++ b/drivers/NativeAdreno/tests/test_submit.cpp @@ -0,0 +1,201 @@ +/* SPDX-License-Identifier: MIT */ +/* Real VM, ring and execbuf code over two software register peers. */ +#include "XnuMemoryFixture.h" +#include "NativeAdrenoSubmit.h" +#include +#include +#include +#define CHECK(x) do {if(!(x)){std::fprintf(stderr,"line %d: %s\n",__LINE__,#x);std::exit(1);}}while(0) +#define SMMU_BYTES 0x40000 +#define CB 0x22000 +IOReturn NativeAdrenoQueryCpu4K(NativeAdrenoCpuState *out) +{CHECK(out);*out={};return kIOReturnSuccess;} +IOReturn NativeAdrenoMaintainOwnedPages(void *mapping,size_t bytes,NativeAdrenoCacheOperation op) +{ + CHECK(mapping && bytes && !(((uintptr_t)mapping|bytes)&4095)); + CHECK(op==NativeAdrenoClean || op==NativeAdrenoCleanInvalidate || op==NativeAdrenoInvalidate); + ++fixture.cache_calls; + return fixture.cache_calls==fixture.cache_fail_call?kIOReturnIOError:kIOReturnSuccess; +} +/* SMMUv2 register peer. */ +static uint32_t mmio[SMMU_BYTES/4]; +static struct {unsigned calls,fail_call,after;} mf; +static ArmSmmu4KIo smmuio; +static ArmSmmu4KStreamState streams; +static ArmSmmu4KStream routes[2]={{2,0},{4,1}}; +static ArmSmmu4KStreamResult route_result; +static ArmSmmu4KContextConfig context; +static ArmSmmu4KContextImage context_image; +static int mevent(void){return ++mf.calls==mf.fail_call;} +static int mread32(void *c,size_t at,uint32_t *v) +{CHECK(c==&mf && at%4==0 && at<=SMMU_BYTES-4);if(mevent())return 1;*v=mmio[at/4];return 0;} +static int mread64(void *c,size_t at,uint64_t *v) +{CHECK(c==&mf && at%8==0 && at<=SMMU_BYTES-8);if(mevent())return 1;std::memcpy(v,mmio+at/4,8);return 0;} +static int mwrite32(void *c,size_t at,uint32_t v) +{CHECK(c==&mf && at%4==0 && at<=SMMU_BYTES-4);int f=mevent();if(f && !mf.after)return 1;mmio[at/4]=v;return f;} +static int mwrite64(void *c,size_t at,uint64_t v){(void)c;(void)at;(void)v;CHECK(0);return 1;} +static int mbarrier(void *c){CHECK(c==&mf);return mevent();} +/* CP register peer. */ +struct CpPeer { + std::vector regs; + std::vector writes; + unsigned publish=0,acquire=0,barriers=0,fail_publish=0,fail_acquire=0,fail_barrier=0; + CpPeer():regs(0x10000,0){} +}; +static CpPeer cp; +static uint64_t clock_us=0; +static int cwrite(void *c,uint32_t reg,uint32_t v) +{CHECK(c==&cp && reg dram_reads; +static NativeAdrenoRingIo ringio{&cp,cwrite,cread,cbarrier}; +static NativeAdrenoRingCache ringcache{&cp,cpublish,cacquire}; +static NativeAdrenoRingClock ringclock{&cp,cnow,cidle}; +static void smmuReset(void) +{ + std::memset(mmio,0,sizeof(mmio));mf={};streams=(ArmSmmu4KStreamState){}; + smmuio=(ArmSmmu4KIo){&mf,SMMU_BYTES,mread32,mread64,mwrite32,mwrite64,mbarrier}; + mmio[0x20/4]=(1u<<30)|(1u<<27)|(4u<<9)|8; + mmio[0x24/4]=0x40000010;mmio[0x28/4]=0x1433;mmio[0]=0x436; + for(unsigned i=0;i<8;++i)mmio[(0xc00+4*i)/4]=0x20000; + mmio[0x800/4]=1u<<31|9;mmio[0xc00/4]=3; +} +static int exposed(Path &p) +{ + smmuReset();cp=CpPeer{};clock_us=0; + std::memset(ringmem,0xef,sizeof(ringmem));std::memset(fencemem,0,sizeof(fencemem)); + if(p.tables.allocate(32*4096,40,true,32*4096))return kIOReturnNoMemory; + if(NativeAdrenoPrepareSmmuMemory(&p.tables,32,40,0x800000,65536,0,16,&p.arena,&p.state,&p.visibility)) + return kIOReturnBadArgument; + context=(ArmSmmu4KContextConfig){.lower=&p.arena.config,.lower_tables=&p.state.tables, + .id0=mmio[0x20/4],.id2=mmio[0x28/4],.asid=1,.coherent_walk=1, + .stall_on_fault=0,.continue_after_fault=1}; + ArmSmmu4KContextConfig actual=context; + CHECK(ArmSmmu4KEncodeContext(&actual,&context_image)==ArmSmmu4KSuccess); + mmio[0x1008/4]=context_image.cbar;mmio[0x1808/4]=context_image.cba2r; + mmio[CB/4]=context_image.sctlr;mmio[(CB+0x10)/4]=context_image.tcr2;mmio[(CB+0x30)/4]=context_image.tcr; + std::memcpy(mmio+(CB+0x20)/4,&context_image.ttbr0,8); + std::memcpy(mmio+(CB+0x28)/4,&context_image.ttbr1,8); + mmio[(CB+0x38)/4]=context_image.mair0; + if(ArmSmmu4KAttachStreams(&smmuio,&streams,routes,2,2,&context,&route_result))return kIOReturnBadArgument; + NativeAdrenoVmRegion lower={&p.tables,&p.arena,&p.state}; + if(p.vm.attach(&smmuio,&context,&streams,&lower,nullptr,32,1000))return kIOReturnBadArgument; + if(p.object.allocate(8192,40,false,8192))return kIOReturnNoMemory; + if(p.vm.map(&p.object,0,0x800000,3,0,2,&p.handle))return kIOReturnBadArgument; + if(p.ring.attach(&ringio,&ringcache,ringmem,sizeof(ringmem),0x100000,fencemem,sizeof(fencemem),0x108000)) + return kIOReturnBadArgument; + return p.submit.attach(&p.vm,&p.ring); +} +int main() +{ + /* One validated indirect-buffer submission with exact ring bytes. */ + { + Path p;CHECK(exposed(p)==kIOReturnSuccess); + NativeAdrenoSubmitBuffer buffer{p.handle,0x800000,4096}; + uint32_t fence=0; + /* Before the ring: clean+invalidate of the one mapped BO, a coherence sample of the IB's first word and TLBIALL. */ + static_cast(p.object.mapping())[0]=0x70970001u; + const unsigned cache_calls=fixture.cache_calls; + mmio[(CB+0x618)/4]=0xffffffffu; + CHECK(p.submit.submit(&buffer,1,0,&fence)==kIOReturnSuccess && fence==1); + CHECK(fixture.cache_calls==cache_calls+1 && mmio[(CB+0x618)/4]==0); + CHECK(native_adreno_coherence[0]==1 && native_adreno_coherence[1]==0x70970001u && + native_adreno_coherence[2]==~0u && native_adreno_coherence[3]==~0u); /* no board DRAM reader installed */ + CHECK(cp.writes.back()==0x0807 && cp.regs[0x0807]==11+4+31); + CHECK((ringmem[11]>>16&0x7fu)==0x3f && (ringmem[11]&0x3fffu)==3); + CHECK(ringmem[12]==0x800000 && ringmem[13]==0 && ringmem[14]==1024); + CHECK(p.object.deviceReferences()==2); /* mapping + job */ + CHECK(p.submit.outstanding()==1 && p.submit.stats().references_held==1); + CHECK(p.submit.detach()==kIOReturnBusy); + uint32_t observed=0; + CHECK(p.ring.wait(1,&ringclock,1000,&observed)==NativeAdrenoRingFenceTimeout); + CHECK(p.submit.wait(1,&ringclock,1000)==kIOReturnTimeout && p.submit.stats().timeouts==1); + fencemem[1]=1; + CHECK(p.submit.wait(1,&ringclock,1000)==kIOReturnSuccess); + CHECK(p.submit.outstanding()==0 && p.submit.stats().completed==1); + CHECK(p.object.deviceReferences()==1 && p.submit.stats().references_retired==1); + CHECK(p.submit.wait(1,&ringclock,1000)==kIOReturnSuccess); /* retired job: the ring says the CP passed it */ + CHECK(p.submit.wait(2,&ringclock,1000)==kIOReturnBadArgument); /* never submitted */ + /* With a board DRAM reader, the sample reads the IB's physical page (before and after the maintenance). */ + native_adreno_dram_read32=[](uint64_t physical,uint32_t *value) { + dram_reads.push_back(physical);*value=0xd00d0000u|uint32_t(dram_reads.size());return 0;}; + CHECK(p.submit.submit(&buffer,1,0,&fence)==kIOReturnSuccess && fence==2); + CHECK(dram_reads.size()==2 && dram_reads[0]==p.object.spans()[0].physical && dram_reads[1]==dram_reads[0]); + CHECK(native_adreno_coherence[0]==2 && native_adreno_coherence[2]==0xd00d0001u && native_adreno_coherence[3]==0xd00d0002u); + CHECK(native_adreno_coherence[4]==2 && native_adreno_coherence[5]==0x70970001u && native_adreno_coherence[6]==~0u); /* first differing sample kept */ + native_adreno_dram_read32=nullptr; + fencemem[1]=2; + CHECK(p.submit.wait(2,&ringclock,1000)==kIOReturnSuccess); + CHECK(p.submit.detach()==kIOReturnSuccess); + } + /* Refusals precede references, ring bytes and WPTR updates. */ + { + Path p;CHECK(exposed(p)==kIOReturnSuccess); + const uint32_t wptr=cp.regs[0x0807]; + uint32_t fence=0; + CHECK(p.submit.submit(nullptr,1,0,&fence)==kIOReturnBadArgument); + NativeAdrenoSubmitBuffer zero[1]={{p.handle,0x800000,0}}; + CHECK(p.submit.submit(zero,1,0,&fence)==kIOReturnBadArgument); + NativeAdrenoSubmitBuffer unaligned[1]={{p.handle,0x800000,4098}}; + CHECK(p.submit.submit(unaligned,1,0,&fence)==kIOReturnBadArgument); + NativeAdrenoSubmitBuffer wrong[1]={{p.handle,0x802000,4096}}; + CHECK(p.submit.submit(wrong,1,0,&fence)==kIOReturnBadArgument); + NativeAdrenoSubmitBuffer overrun[1]={{p.handle,0x800000,8192+4}}; + CHECK(p.submit.submit(overrun,1,0,&fence)==kIOReturnBadArgument); + NativeAdrenoSubmitBuffer unknown[1]={{p.handle+7,0x800000,4096}}; + CHECK(p.submit.submit(unknown,1,0,&fence)==kIOReturnBadArgument); + NativeAdrenoSubmitBuffer many[17]; + for(auto &b:many)b={p.handle,0x800000,4096}; + CHECK(p.submit.submit(many,17,0,&fence)==kIOReturnBadArgument); + CHECK(p.object.deviceReferences()==1 && cp.regs[0x0807]==wptr && p.submit.outstanding()==0); + /* Duplicate buffers share one device reference. */ + NativeAdrenoSubmitBuffer twice[2]={{p.handle,0x800000,4096},{p.handle,0x800000,4096}}; + CHECK(p.submit.submit(twice,2,0,&fence)==kIOReturnSuccess); + CHECK(p.object.deviceReferences()==2 && cp.regs[0x0807]==11+8+31); + fencemem[1]=1; + CHECK(p.submit.wait(1,&ringclock,100)==kIOReturnSuccess && p.object.deviceReferences()==1); + CHECK(p.submit.detach()==kIOReturnSuccess); + } + /* A commit that poisons the ring retains every reference and disables execbuf. */ + { + Path p;CHECK(exposed(p)==kIOReturnSuccess); + cp.fail_publish=cp.publish+1; + NativeAdrenoSubmitBuffer buffer{p.handle,0x800000,4096}; + uint32_t fence=0; + CHECK(p.submit.submit(&buffer,1,0,&fence)==kIOReturnIOError); + CHECK(!fence && p.submit.poisoned() && p.ring.poisoned()); + CHECK(p.submit.outstanding()==1 && p.submit.stats().poisoned_jobs==1); + CHECK(p.object.deviceReferences()==2); /* never retired */ + CHECK(p.submit.detach()==kIOReturnBusy && p.submit.wait(1,&ringclock,10)==kIOReturnNotReady); + CHECK(p.ring.detach()==kIOReturnBusy && p.vm.detach()==kIOReturnBusy); + CHECK(p.object.release()==kIOReturnBusy); + } + /* A refused attach leaves both peers untouched. */ + { + Path p;NativeAdrenoRing ring;NativeAdrenoSubmit submit; + CHECK(submit.attach(nullptr,&ring)==kIOReturnBadArgument); + CHECK(submit.attach(&p.vm,nullptr)==kIOReturnBadArgument); + CHECK(submit.detach()==kIOReturnNotReady); + CHECK(submit.submit(nullptr,1,0,nullptr)==kIOReturnBadArgument); + } + std::printf("PASS: 1 execbuf submission with exact IB packet and fence, 8 pre-reference refusals, 1 duplicate-buffer reference case and 1 poisoned-commit retention case over software SMMU/CP peers\n"); +} diff --git a/drivers/NativeAdreno/tests/test_timer.cpp b/drivers/NativeAdreno/tests/test_timer.cpp new file mode 100644 index 0000000..7f6c8e9 --- /dev/null +++ b/drivers/NativeAdreno/tests/test_timer.cpp @@ -0,0 +1,98 @@ +/* SPDX-License-Identifier: MIT */ +/* Real timestamp reader over a software counter peer. No device access. */ +#include "XnuMemoryFixture.h" +#include "NativeAdrenoTimer.h" +#include +#define CHECK(x) do {if(!(x)){std::fprintf(stderr,"line %d: %s\n",__LINE__,#x);std::exit(1);}}while(0) +struct Peer { + uint32_t low=0,high=0; + unsigned reads=0,fail_read=0,unstable=0; + uint32_t last_low_address=0,last_high_address=0; +}; +static Peer peer; +static int read32(void *cookie,uint32_t address,uint32_t *value) +{ + CHECK(cookie==&peer && value);++peer.reads; + if(peer.reads==peer.fail_read)return 1; + if(address==NativeAdrenoTimer::gmuLowOffset() || address==NativeAdrenoTimer::cpLowRegister()) { + peer.last_low_address=address;*value=peer.low;return 0; + } + if(address==NativeAdrenoTimer::gmuHighOffset() || address==NativeAdrenoTimer::cpHighRegister()) { + peer.last_high_address=address; + /* Emulate a high-word change between the bracketing reads. */ + *value=peer.high; + if(peer.unstable && peer.reads%3==1) {++peer.high;--peer.unstable;} + return 0; + } + CHECK(0);return 1; +} +static NativeAdrenoTimeIo io{&peer,read32}; +static unsigned unsettled_reads=0; +static int unsettled(void *cookie,uint32_t address,uint32_t *value) +{(void)cookie;(void)address;*value=++unsettled_reads;return 0;} +int main() +{ + /* The derived A740 GMU offsets are the pinned values. */ + { + CHECK(NativeAdrenoTimer::gmuLowOffset()==0x14220); + CHECK(NativeAdrenoTimer::gmuHighOffset()==0x14224); + CHECK(NativeAdrenoTimer::cpLowRegister()==0x0980); + CHECK(NativeAdrenoTimer::cpHighRegister()==0x0981); + CHECK(NativeAdrenoTimer::gmuHighOffset()==NativeAdrenoTimer::gmuLowOffset()+4); + } + /* A stable 64-bit counter reads exactly, through both address spaces. */ + { + peer=Peer{};peer.low=0x89abcdefu;peer.high=0x01234567u; + NativeAdrenoTimer timer; + CHECK(timer.attach(&io,NativeAdrenoTimeGmu)==kIOReturnSuccess); + uint64_t ticks=0; + CHECK(timer.read(&ticks)==kIOReturnSuccess); + CHECK(ticks==UINT64_C(0x0123456789abcdef)); + CHECK(peer.last_low_address==0x14220 && peer.last_high_address==0x14224); + CHECK(timer.detach()==kIOReturnSuccess && timer.detach()==kIOReturnNotReady); + peer.reads=0; + NativeAdrenoTimer cp; + CHECK(cp.attach(&io,NativeAdrenoTimeCp)==kIOReturnSuccess); + CHECK(cp.read(&ticks)==kIOReturnSuccess && ticks==UINT64_C(0x0123456789abcdef)); + CHECK(peer.last_low_address==0x0980 && peer.last_high_address==0x0981); + } + /* A high-word change is retried, then read consistently. */ + { + peer=Peer{};peer.low=0xfffffff0u;peer.high=7;peer.unstable=1; + NativeAdrenoTimer timer;CHECK(timer.attach(&io,NativeAdrenoTimeGmu)==kIOReturnSuccess); + uint64_t ticks=0; + CHECK(timer.read(&ticks)==kIOReturnSuccess); + CHECK(peer.high==8 && ticks==UINT64_C(0x00000008fffffff0)); + CHECK(peer.reads==6); /* two bracketing triples */ + } + /* Persistent instability and I/O failure are errors, not fabricated ticks. */ + { + peer=Peer{};peer.low=1;peer.high=0; + NativeAdrenoTimeIo missing{nullptr,nullptr}; + NativeAdrenoTimer unbound; + CHECK(unbound.attach(nullptr,NativeAdrenoTimeCp)==kIOReturnBadArgument); + CHECK(unbound.attach(&missing,NativeAdrenoTimeCp)==kIOReturnBadArgument); + CHECK(unbound.attach(&io,7)==kIOReturnBadArgument); + CHECK(unbound.attach(&io,0xffffffffu)==kIOReturnBadArgument); + CHECK(unbound.read(nullptr)==kIOReturnBadArgument); + NativeAdrenoTimer timer;CHECK(timer.attach(&io,NativeAdrenoTimeCp)==kIOReturnSuccess); + uint64_t ticks=0; + peer.fail_read=2; + CHECK(timer.read(&ticks)==kIOReturnIOError && !ticks); + peer.fail_read=0;peer.unstable=0; + CHECK(timer.read(&ticks)==kIOReturnSuccess); + CHECK(timer.read(nullptr)==kIOReturnBadArgument); + CHECK(timer.attach(&io,NativeAdrenoTimeCp)==kIOReturnBusy); + CHECK(timer.detach()==kIOReturnSuccess && timer.read(&ticks)==kIOReturnNotReady); + } + /* A torn loop that never settles is bounded and reported as a timeout. */ + { + NativeAdrenoTimeIo never{nullptr,unsettled}; + unsettled_reads=0; + NativeAdrenoTimer timer;CHECK(timer.attach(&never,NativeAdrenoTimeGmu)==kIOReturnSuccess); + uint64_t ticks=0; + CHECK(timer.read(&ticks)==kIOReturnTimeout && !ticks); + CHECK(unsettled_reads==3*8); + } + std::printf("PASS: 1 derived-offset check, 2 stable reads, 1 rollover retry, 1 I/O failure and 1 bounded torn-loop timeout over a software counter peer\n"); +} diff --git a/drivers/NativeAdreno/tests/test_userclient.cpp b/drivers/NativeAdreno/tests/test_userclient.cpp new file mode 100644 index 0000000..a99b91f --- /dev/null +++ b/drivers/NativeAdreno/tests/test_userclient.cpp @@ -0,0 +1,508 @@ +/* SPDX-License-Identifier: MIT */ +/* Real user-client body over software SMMU/CP register peers. No IOKit. */ +#include "XnuMemoryFixture.h" +#include "NativeAdrenoUserClient.h" +#include +#include +#include +#define CHECK(x) do {if(!(x)){std::fprintf(stderr,"line %d: %s\n",__LINE__,#x);std::exit(1);}}while(0) +#define SMMU_BYTES 0x40000 +#define CB 0x22000 +IOReturn NativeAdrenoQueryCpu4K(NativeAdrenoCpuState *out) +{CHECK(out);*out={};return kIOReturnSuccess;} +IOReturn NativeAdrenoMaintainOwnedPages(void *mapping,size_t bytes,NativeAdrenoCacheOperation op) +{ + CHECK(mapping && bytes && !(((uintptr_t)mapping|bytes)&4095)); + CHECK(op==NativeAdrenoClean || op==NativeAdrenoCleanInvalidate || op==NativeAdrenoInvalidate); + ++fixture.cache_calls; + return fixture.cache_calls==fixture.cache_fail_call?kIOReturnIOError:kIOReturnSuccess; +} +static uint32_t mmio[SMMU_BYTES/4]; +static struct {unsigned calls,fail_call,after;} mf; +static ArmSmmu4KIo smmuio; +static ArmSmmu4KStreamState streams; +static ArmSmmu4KStream routes[2]={{2,0},{4,1}}; +static ArmSmmu4KStreamResult route_result; +static ArmSmmu4KContextConfig context; +static ArmSmmu4KContextImage context_image; +static int mevent(void){return ++mf.calls==mf.fail_call;} +static int mread32(void *c,size_t at,uint32_t *v) +{CHECK(c==&mf && at%4==0 && at<=SMMU_BYTES-4);if(mevent())return 1;*v=mmio[at/4];return 0;} +static int mread64(void *c,size_t at,uint64_t *v) +{CHECK(c==&mf && at%8==0 && at<=SMMU_BYTES-8);if(mevent())return 1;std::memcpy(v,mmio+at/4,8);return 0;} +static int mwrite32(void *c,size_t at,uint32_t v) +{CHECK(c==&mf && at%4==0 && at<=SMMU_BYTES-4);int f=mevent();if(f && !mf.after)return 1;mmio[at/4]=v;return f;} +static int mwrite64(void *c,size_t at,uint64_t v){(void)c;(void)at;(void)v;CHECK(0);return 1;} +static int mbarrier(void *c){CHECK(c==&mf);return mevent();} +struct CpPeer { + std::vector regs,writes; + unsigned publish=0,acquire=0,barriers=0,fail_publish=0; + CpPeer():regs(0x10000,0){} +}; +static CpPeer cp; +static uint64_t clock_us=0; +static int cwrite(void *c,uint32_t reg,uint32_t v) +{CHECK(c==&cp && reg>16&0x7fu)==0x3f && (ringmem[9]>>16&0x7fu)==0x1f && (ringmem[21]>>16&0x7fu)==0x3c); + CHECK(ringmem[38]==0x800000 && ringmem[39]==0 && ringmem[40]==256); + CHECK(p.execbuf.stats().references_held==1); + NativeAdrenoAbiBoFree released{allocated.handle}; + CHECK(p.client.freeBo(&released)==kIOReturnBusy); /* job outstanding */ + NativeAdrenoAbiWaitFence wait{1,0}; + CHECK(p.client.waitFence(&wait,&ringclock)==kIOReturnTimeout); + CHECK(p.execbuf.outstanding()==1); + fencemem[1]=1; + wait.timeout_ns=UINT64_C(1000000); + CHECK(p.client.waitFence(&wait,&ringclock)==kIOReturnSuccess); + CHECK(p.execbuf.outstanding()==0); + NativeAdrenoAbiMetadata meta{}; + meta.handle=allocated.handle;meta.bytes=8; + std::memcpy(meta.text,"ring-buf",8); + CHECK(p.client.setMetadata(&meta)==kIOReturnSuccess); + NativeAdrenoAbiMetadata got{}; + got.handle=allocated.handle;got.bytes=16; + CHECK(p.client.getMetadata(&got,&got)==kIOReturnSuccess); + CHECK(got.bytes==8 && !std::memcmp(got.text,"ring-buf",8)); + got.bytes=4;CHECK(p.client.getMetadata(&got,&got)==kIOReturnBadArgument); + meta.bytes=65;CHECK(p.client.setMetadata(&meta)==kIOReturnBadArgument); + meta.bytes=8;meta.handle=allocated.handle+9; + CHECK(p.client.setMetadata(&meta)==kIOReturnBadArgument); + NativeAdrenoAbiSubmit second{}; + second.buffer_count=1;second.wait_fence=1; + second.buffers[0]={allocated.handle,0x800000,1024}; + CHECK(p.client.submit(&second,&reply,&ringclock)==kIOReturnSuccess && reply.fence==2); + const uint32_t wait_wptr=cp.regs[0x0807]; + NativeAdrenoAbiSubmit blocked{}; + blocked.buffer_count=1;blocked.wait_fence=2; + blocked.buffers[0]={allocated.handle,0x800000,1024}; + CHECK(p.client.submit(&blocked,&reply,&ringclock)==kIOReturnTimeout); + CHECK(cp.regs[0x0807]==wait_wptr && p.execbuf.outstanding()==1); + fencemem[1]=2; + wait.fence=2;wait.timeout_ns=UINT64_C(1000000); + CHECK(p.client.waitFence(&wait,&ringclock)==kIOReturnSuccess); + CHECK(p.client.freeBo(&released)==kIOReturnSuccess); + CHECK(p.client.liveObjects()==0 && !p.client.mapped(allocated.handle)); + CHECK(p.client.close()==kIOReturnSuccess); + } + /* Host import: a caller's pages become a BO with scattered spans, map into the GPU, refuse a CPU-map token and the task-less + * dispatch path, and unpin on free. */ + { + Path p;CHECK(exposed(p)==kIOReturnSuccess); + NativeAdrenoAbiParams params{}; + CHECK(p.client.query(¶ms)==kIOReturnSuccess && (params.flags&NativeAdrenoParamsHostImport)); + void *task=reinterpret_cast(0x77); + NativeAdrenoAbiBoAllocReply imported{}; + NativeAdrenoAbiBoImport bad{0x10000001,8192}; + CHECK(p.client.importBo(&bad,task,&imported)==kIOReturnBadArgument && !imported.handle); + bad={0x10000000,0};CHECK(p.client.importBo(&bad,task,&imported)==kIOReturnBadArgument); + NativeAdrenoAbiBoImport request{0x10000000,4*4096}; + CHECK(p.client.importBo(&request,nullptr,&imported)==kIOReturnBadArgument); + const unsigned before=fixture.imports_live; + CHECK(p.client.importBo(&request,task,&imported)==kIOReturnSuccess && imported.handle); + CHECK(fixture.imports_live==before+1); + NativeAdrenoMemory *memory=p.client.memoryOf(imported.handle); + CHECK(memory && memory->imported() && memory->spanCount()==4 && !memory->mapping()); + NativeAdrenoAbiBoCpuMap cpu{imported.handle}; + NativeAdrenoAbiBoCpuMapReply token{}; + CHECK(p.client.cpuMap(&cpu,&token)==kIOReturnUnsupported && !token.token); + NativeAdrenoAbiBoAllocReply untouched{}; + CHECK(p.client.dispatch(NativeAdrenoMethodBoImport,&request,sizeof(request),&untouched,sizeof(untouched),&ringclock)==kIOReturnUnsupported); + CHECK(!untouched.handle); + NativeAdrenoAbiBoMap map{imported.handle,0x808000,3,0,2,0}; + NativeAdrenoAbiBoMapReply mapped{}; + CHECK(p.client.mapBo(&map,&mapped)==kIOReturnSuccess && mapped.iova==0x808000); + NativeAdrenoAbiBoFree released{imported.handle}; + CHECK(p.client.freeBo(&released)==kIOReturnSuccess); + CHECK(fixture.imports_live==before && p.client.liveObjects()==0); + CHECK(p.client.close()==kIOReturnSuccess); + } + /* The external-method entry enforces exact record sizes and copies only + * on success. This is the entire surface the IOKit wrapper may expose. */ + { + Path p;CHECK(exposed(p)==kIOReturnSuccess); + NativeAdrenoAbiParams params{}; + CHECK(p.client.dispatch(NativeAdrenoMethodQuery,nullptr,0,¶ms,sizeof(params),&ringclock)==kIOReturnSuccess); + CHECK(params.abi_version==NativeAdrenoAbiVersion); + unsigned char untouched[sizeof(NativeAdrenoAbiParams)]; + std::memset(untouched,0x5a,sizeof(untouched)); + CHECK(p.client.dispatch(NativeAdrenoMethodQuery,nullptr,1,untouched,sizeof(untouched),&ringclock)==kIOReturnBadArgument); + CHECK(p.client.dispatch(NativeAdrenoMethodQuery,nullptr,0,untouched,sizeof(untouched)-1,&ringclock)==kIOReturnBadArgument); + CHECK(p.client.dispatch(NativeAdrenoMethodQuery,nullptr,0,nullptr,0,&ringclock)==kIOReturnBadArgument); + CHECK(p.client.dispatch(99,nullptr,0,nullptr,0,&ringclock)==kIOReturnUnsupported); + CHECK(untouched[0]==0x5a && untouched[sizeof(untouched)-1]==0x5a); + NativeAdrenoAbiBoAlloc alloc{8192,12,1}; + NativeAdrenoAbiBoAllocReply allocated{}; + CHECK(p.client.dispatch(NativeAdrenoMethodBoAlloc,&alloc,sizeof(alloc),&allocated,sizeof(allocated),&ringclock)==kIOReturnSuccess); + CHECK(allocated.handle); + NativeAdrenoAbiBoMap map{allocated.handle,0x800000,3,0,2,0}; + NativeAdrenoAbiBoMapReply mapped{}; + CHECK(p.client.dispatch(NativeAdrenoMethodBoMap,&map,sizeof(map),&mapped,sizeof(mapped),&ringclock)==kIOReturnSuccess); + NativeAdrenoAbiSubmit submit{}; + submit.buffer_count=1;submit.buffers[0]={allocated.handle,0x800000,2048}; + NativeAdrenoAbiSubmitReply reply{}; + CHECK(p.client.dispatch(NativeAdrenoMethodSubmit,&submit,sizeof(submit),&reply,sizeof(reply),&ringclock)==kIOReturnSuccess); + CHECK(reply.fence==1 && cp.regs[0x0807]==72); + NativeAdrenoAbiWaitFence wait{reply.fence,0}; + CHECK(p.client.dispatch(NativeAdrenoMethodWaitFence,&wait,sizeof(wait),nullptr,0,&ringclock)==kIOReturnTimeout); + fencemem[1]=1; + wait.timeout_ns=UINT64_C(1000000); + CHECK(p.client.dispatch(NativeAdrenoMethodWaitFence,&wait,sizeof(wait),nullptr,0,&ringclock)==kIOReturnSuccess); + NativeAdrenoAbiBoFree released{allocated.handle}; + CHECK(p.client.dispatch(NativeAdrenoMethodBoFree,&released,sizeof(released),nullptr,0,&ringclock)==kIOReturnSuccess); + CHECK(p.client.dispatch(NativeAdrenoMethodBoFree,&released,sizeof(released),nullptr,0,&ringclock)==kIOReturnBadArgument); + NativeAdrenoTimer timer;CHECK(timer.attach(&timeio,NativeAdrenoTimeCp)==kIOReturnSuccess); + CHECK(p.client.attachTimer(&timer)==kIOReturnSuccess); + cp.regs[0x0980]=0x00000011u;cp.regs[0x0981]=0x00000022u; + NativeAdrenoAbiTimestamp ticks{}; + CHECK(p.client.dispatch(NativeAdrenoMethodGetTimestamp,nullptr,0,&ticks,sizeof(ticks),&ringclock)==kIOReturnSuccess); + CHECK(ticks.ticks==UINT64_C(0x0000002200000011)); + NativeAdrenoStatus counters; + CHECK(p.client.attachStatus(&counters)==kIOReturnSuccess); + counters.noteFault(); + NativeAdrenoAbiStatus status{}; + CHECK(p.client.dispatch(NativeAdrenoMethodGetStatus,nullptr,0,&status,sizeof(status),&ringclock)==kIOReturnSuccess); + CHECK(status.faults==1 && status.suspends==0); + unsigned char small[4]{}; + CHECK(p.client.dispatch(NativeAdrenoMethodGetStatus,nullptr,0,small,sizeof(small),&ringclock)==kIOReturnBadArgument); + CHECK(p.client.close()==kIOReturnSuccess); + } + /* Refusals never reach a device reference, ring write or unmap. */ + { + Path p;CHECK(exposed(p)==kIOReturnSuccess); + const uint32_t wptr=cp.regs[0x0807]; + NativeAdrenoAbiBoAllocReply allocated{}; + NativeAdrenoAbiBoAlloc bad[]={{0,12,0},{4097,12,0},{8192,11,0}, + {8192,25,0},{8192,12,2},{0x10000001,12,0}}; + for(const auto &request:bad)CHECK(p.client.allocBo(&request,&allocated)==kIOReturnBadArgument); + CHECK(!allocated.handle); + NativeAdrenoAbiBoAlloc good{4096,12,0}; + CHECK(p.client.allocBo(&good,&allocated)==kIOReturnSuccess); + NativeAdrenoAbiBoMap wrong[]={{0,0x800000,3,0,2,0},{allocated.handle,0x800000,2,0,2,0}, + {allocated.handle,0x800001,3,0,2,0},{allocated.handle,0x800000,3,2,2,0}, + {allocated.handle,0x800000,3,0,1,0},{allocated.handle,0x800000,3,0,2,1}, + {allocated.handle+9,0x800000,3,0,2,0}}; + NativeAdrenoAbiBoMapReply mapped{}; + for(const auto &request:wrong)CHECK(p.client.mapBo(&request,&mapped)==kIOReturnBadArgument); + CHECK(!mapped.iova && !p.client.mapped(allocated.handle)); + NativeAdrenoAbiBoMap request{allocated.handle,0x800000,3,0,2,0}; + CHECK(p.client.mapBo(&request,&mapped)==kIOReturnSuccess); + CHECK(p.client.mapBo(&request,&mapped)==kIOReturnBusy); + NativeAdrenoAbiSubmit empty{}; + NativeAdrenoAbiSubmitReply reply{}; + CHECK(p.client.submit(&empty,&reply,&ringclock)==kIOReturnBadArgument); + empty.buffer_count=17;CHECK(p.client.submit(&empty,&reply,&ringclock)==kIOReturnBadArgument); + empty.buffer_count=1;empty.reserved=1;CHECK(p.client.submit(&empty,&reply,&ringclock)==kIOReturnBadArgument); + empty.reserved=0;empty.buffers[0]={allocated.handle+5,0x800000,1024}; + CHECK(p.client.submit(&empty,&reply,&ringclock)==kIOReturnBadArgument); + empty.buffers[0]={allocated.handle,0x800000,1026}; + CHECK(p.client.submit(&empty,&reply,&ringclock)==kIOReturnBadArgument); + CHECK(cp.regs[0x0807]==wptr && p.execbuf.outstanding()==0); + NativeAdrenoAbiWaitFence wait{0,0}; + CHECK(p.client.waitFence(&wait,&ringclock)==kIOReturnBadArgument); + wait.fence=7; + CHECK(p.client.waitFence(&wait,&ringclock)==kIOReturnBadArgument); + NativeAdrenoAbiBoFree unknown{allocated.handle+5}; + CHECK(p.client.freeBo(&unknown)==kIOReturnBadArgument); + CHECK(p.client.close()==kIOReturnBusy); /* object still live */ + NativeAdrenoAbiBoFree final_free{allocated.handle}; + CHECK(p.client.freeBo(&final_free)==kIOReturnSuccess); + CHECK(p.client.close()==kIOReturnSuccess); + CHECK(p.client.query(nullptr)==kIOReturnBadArgument); + } + /* Connection teardown: an in-flight job, a cpu-map lease and a second queue are all drained by releaseAll. */ + { + Path p;CHECK(exposed(p)==kIOReturnSuccess); + NativeAdrenoAbiBoAlloc alloc{4096,12,0}; + NativeAdrenoAbiBoAllocReply allocated{}; + CHECK(p.client.allocBo(&alloc,&allocated)==kIOReturnSuccess); + NativeAdrenoAbiBoMap map{allocated.handle,0x800000,3,0,2,0}; + NativeAdrenoAbiBoMapReply mapped{}; + CHECK(p.client.mapBo(&map,&mapped)==kIOReturnSuccess); + NativeAdrenoAbiBoCpuMap cm{allocated.handle}; + NativeAdrenoAbiBoCpuMapReply cr{}; + CHECK(p.client.cpuMap(&cm,&cr)==kIOReturnSuccess && cr.bytes==4096); + CHECK(p.client.memoryForToken(cr.token)==p.client.memoryOf(allocated.handle)); + CHECK(p.client.memoryForToken(0)==nullptr && p.client.memoryForToken(cr.token+1)==nullptr); + NativeAdrenoAbiQueueReply queue{}; + CHECK(p.client.createQueue(&queue)==kIOReturnSuccess); + NativeAdrenoAbiSubmit submit{}; + submit.buffer_count=1;submit.buffers[0]={allocated.handle,0x800000,1024}; + NativeAdrenoAbiSubmitReply reply{}; + CHECK(p.client.submit(&submit,&reply,&ringclock)==kIOReturnSuccess); + CHECK(p.client.releaseAll(&ringclock,1000)==kIOReturnTimeout); /* the CP has not retired the job */ + CHECK(p.client.liveObjects()==1); + fencemem[1]=reply.fence; + CHECK(p.client.releaseAll(nullptr,1000)==kIOReturnBadArgument); + CHECK(p.client.releaseAll(&ringclock,1000)==kIOReturnSuccess); + CHECK(p.client.liveObjects()==0 && p.client.memoryForToken(cr.token)==nullptr); + CHECK(p.client.query(nullptr)==kIOReturnBadArgument && p.client.close()==kIOReturnNotReady); + CHECK(p.client.open(NativeAdrenoAbiVersion,&p.vm,&p.execbuf,&p.ring,31,40)==kIOReturnSuccess); + } + /* More than the old 64-object limit: 300 unmapped objects live at once, then drained by releaseAll. */ + { + Path p;CHECK(exposed(p)==kIOReturnSuccess); + NativeAdrenoAbiBoAlloc alloc{4096,12,0}; + for(unsigned i=0;i<300;++i) { + NativeAdrenoAbiBoAllocReply allocated{}; + CHECK(p.client.allocBo(&alloc,&allocated)==kIOReturnSuccess && allocated.handle); + } + CHECK(p.client.liveObjects()==300); + CHECK(p.client.releaseAll(&ringclock,1000)==kIOReturnSuccess && p.client.liveObjects()==0); + } + /* Constant device facts: unsupported until attached, refused when malformed, then returned verbatim through dispatch. */ + { + Path p;CHECK(exposed(p)==kIOReturnSuccess); + NativeAdrenoAbiDeviceInfo got{}; + CHECK(p.client.deviceInfo(&got)==kIOReturnUnsupported); + NativeAdrenoAbiDeviceInfo bad{}; + CHECK(p.client.attachDeviceInfo(&bad)==kIOReturnBadArgument); + static const NativeAdrenoAbiDeviceInfo info={0x43050a01u,3u<<20,0x1000000ull,0x1fffffffff000ull,0x20000000ull, + 0x60000000ull,16,6,1,4,NativeAdrenoDeviceClientIova,0}; + CHECK(p.client.attachDeviceInfo(&info)==kIOReturnSuccess && p.client.attachDeviceInfo(&info)==kIOReturnBadArgument); + CHECK(p.client.dispatch(NativeAdrenoMethodGetDeviceInfo,nullptr,0,&got,sizeof(got),&ringclock)==kIOReturnSuccess); + CHECK(got.chip_id==0x43050a01u && got.va_start==0x20000000ull && got.highest_bank_bit==16 && got.ubwc_version==4); + CHECK(p.client.dispatch(NativeAdrenoMethodGetDeviceInfo,nullptr,0,&got,sizeof(got)-1,&ringclock)==kIOReturnBadArgument); + CHECK(p.client.close()==kIOReturnSuccess); + CHECK(p.client.deviceInfo(&got)==kIOReturnNotReady); + } + /* Completed work never pins a buffer: no wait() call, the CP only advances its fence, and BoFree still succeeds. */ + { + Path p;CHECK(exposed(p)==kIOReturnSuccess); + NativeAdrenoAbiBoAlloc alloc{4096,12,0}; + NativeAdrenoAbiBoAllocReply allocated{}; + CHECK(p.client.allocBo(&alloc,&allocated)==kIOReturnSuccess); + NativeAdrenoAbiBoMap map{allocated.handle,0x800000,3,0,2,0}; + NativeAdrenoAbiBoMapReply mapped{}; + CHECK(p.client.mapBo(&map,&mapped)==kIOReturnSuccess); + NativeAdrenoAbiSubmit submit{}; + submit.buffer_count=1;submit.buffers[0]={allocated.handle,0x800000,1024}; + NativeAdrenoAbiSubmitReply reply{}; + CHECK(p.client.submit(&submit,&reply,&ringclock)==kIOReturnSuccess); + NativeAdrenoAbiBoFree request{allocated.handle}; + CHECK(p.client.freeBo(&request)!=kIOReturnSuccess); /* the job is still in flight */ + fencemem[1]=reply.fence; + CHECK(p.client.freeBo(&request)==kIOReturnSuccess && p.client.liveObjects()==0); + NativeAdrenoAbiWaitFence wait{reply.fence,0}; + CHECK(p.client.waitFence(&wait,&ringclock)==kIOReturnSuccess); /* retired job, fence already passed */ + NativeAdrenoAbiWaitFence future{reply.fence+5,0}; + CHECK(p.client.waitFence(&future,&ringclock)==kIOReturnBadArgument); + CHECK(p.client.close()==kIOReturnSuccess); + } + /* Large objects try one contiguous run first and fall back to scattered pages when that fails (and a refused retry). */ + { + Path p;CHECK(exposed(p)==kIOReturnSuccess); + NativeAdrenoAbiBoAlloc big{1u<<20,12,0}; + NativeAdrenoAbiBoAllocReply allocated{}; + fixture.allocation_fail_count=1; + CHECK(p.client.allocBo(&big,&allocated)==kIOReturnSuccess && allocated.handle && fixture.allocation_fail_count==0); + NativeAdrenoAbiBoAllocReply second{}; + fixture.allocation_fail_count=2; /* contiguous and scattered both fail */ + CHECK(p.client.allocBo(&big,&second)==kIOReturnNoMemory && !second.handle && fixture.allocation_fail_count==0); + NativeAdrenoAbiBoAlloc demanded{1u<<20,12,1}; + fixture.allocation_fail_count=1; /* a demanded contiguous run is not retried scattered */ + CHECK(p.client.allocBo(&demanded,&second)==kIOReturnNoMemory); + CHECK(p.client.releaseAll(&ringclock,1000)==kIOReturnSuccess); + } + /* Version and mount refusals, then a commit that poisons everything. */ + { + Path p;CHECK(exposed(p)==kIOReturnSuccess); + NativeAdrenoUserClient client; + CHECK(client.open(NativeAdrenoAbiVersion+1,&p.vm,&p.execbuf,&p.ring,32,40)==kIOReturnUnsupported); + CHECK(client.open(NativeAdrenoAbiVersion,nullptr,&p.execbuf,&p.ring,32,40)==kIOReturnBadArgument); + CHECK(client.open(NativeAdrenoAbiVersion,&p.vm,nullptr,&p.ring,32,40)==kIOReturnBadArgument); + CHECK(client.open(NativeAdrenoAbiVersion,&p.vm,&p.execbuf,&p.ring,30,40)==kIOReturnBadArgument); + CHECK(client.close()==kIOReturnNotReady && client.query(nullptr)==kIOReturnBadArgument); + NativeAdrenoAbiBoAlloc alloc{4096,12,0}; + NativeAdrenoAbiBoAllocReply allocated{}; + CHECK(p.client.allocBo(&alloc,&allocated)==kIOReturnSuccess); + NativeAdrenoAbiBoMap map{allocated.handle,0x800000,3,0,2,0}; + NativeAdrenoAbiBoMapReply mapped{}; + CHECK(p.client.mapBo(&map,&mapped)==kIOReturnSuccess); + cp.fail_publish=cp.publish+1; + NativeAdrenoAbiSubmit submit{}; + submit.buffer_count=1;submit.buffers[0]={allocated.handle,0x800000,4096}; + NativeAdrenoAbiSubmitReply reply{}; + CHECK(p.client.submit(&submit,&reply,&ringclock)==kIOReturnIOError && !reply.fence); + CHECK(p.execbuf.poisoned() && p.execbuf.outstanding()==1); + NativeAdrenoAbiBoFree retained{allocated.handle}; + CHECK(p.client.freeBo(&retained)==kIOReturnBusy); + CHECK(p.client.close()==kIOReturnBusy); + } + /* CPU mapping leases keep the backing object alive for the user mapping. */ + { + Path p;CHECK(exposed(p)==kIOReturnSuccess); + NativeAdrenoAbiBoAlloc alloc{4096,12,0}; + NativeAdrenoAbiBoAllocReply allocated{}; + CHECK(p.client.allocBo(&alloc,&allocated)==kIOReturnSuccess); + NativeAdrenoAbiBoCpuMap map{allocated.handle}; + NativeAdrenoAbiBoCpuMapReply mapped{}; + CHECK(p.client.cpuMap(&map,&mapped)==kIOReturnSuccess); + CHECK(mapped.token && mapped.bytes==4096); + const uint32_t token=mapped.token; + CHECK(p.client.cpuMap(&map,&mapped)==kIOReturnSuccess && mapped.token==token); + NativeAdrenoAbiBoFree released{allocated.handle}; + CHECK(p.client.freeBo(&released)==kIOReturnBusy); + CHECK(p.client.close()==kIOReturnBusy); + NativeAdrenoAbiBoCpuUnmap wrong{allocated.handle,token+1,0}; + CHECK(p.client.cpuUnmap(&wrong)==kIOReturnBadArgument); + NativeAdrenoAbiBoCpuUnmap unmap{allocated.handle,token,0}; + CHECK(p.client.cpuUnmap(&unmap)==kIOReturnSuccess); + CHECK(p.client.freeBo(&released)==kIOReturnBusy); /* one lease left */ + CHECK(p.client.cpuUnmap(&unmap)==kIOReturnSuccess); + CHECK(p.client.cpuUnmap(&unmap)==kIOReturnBusy); + CHECK(p.client.freeBo(&released)==kIOReturnSuccess); + CHECK(p.client.cpuMap(&map,&mapped)==kIOReturnBadArgument); + CHECK(p.client.close()==kIOReturnSuccess); + } + /* Queue objects bind submissions and must be drained before close. */ + { + Path p;CHECK(exposed(p)==kIOReturnSuccess); + for(unsigned i=1;i +#define CHECK(x) do {if(!(x)){std::fprintf(stderr,"line %d: %s\n",__LINE__,#x);std::exit(1);}}while(0) +#define BYTES 0x40000 +#define CB 0x22000 +IOReturn NativeAdrenoQueryCpu4K(NativeAdrenoCpuState *out) +{CHECK(out);*out={};return kIOReturnSuccess;} +IOReturn NativeAdrenoMaintainOwnedPages(void *mapping,size_t bytes,NativeAdrenoCacheOperation op) +{ + CHECK(mapping && bytes && !(((uintptr_t)mapping|bytes)&4095)); + CHECK(op==NativeAdrenoClean || op==NativeAdrenoCleanInvalidate || op==NativeAdrenoInvalidate); + ++fixture.cache_calls; + return fixture.cache_calls==fixture.cache_fail_call?kIOReturnIOError:kIOReturnSuccess; +} +static uint32_t memory[BYTES/4]; +static struct {unsigned calls,fail_call,after;} f; +static ArmSmmu4KIo io; +static ArmSmmu4KStreamState streams; +static ArmSmmu4KStream routes[2]={{2,0},{4,1}}; +static ArmSmmu4KStreamResult route_result; +static ArmSmmu4KContextConfig context; +static ArmSmmu4KContextImage image; +static int event(void){return ++f.calls==f.fail_call;} +static int read32(void *cookie,size_t at,uint32_t *v) +{CHECK(cookie==&f && at%4==0 && at<=BYTES-4);if(event())return 1;*v=memory[at/4];return 0;} +static int read64(void *cookie,size_t at,uint64_t *v) +{CHECK(cookie==&f && at%8==0 && at<=BYTES-8);if(event())return 1;std::memcpy(v,memory+at/4,8);return 0;} +static int write32(void *cookie,size_t at,uint32_t v) +{ + CHECK(cookie==&f && at%4==0 && at<=BYTES-4); + int fail=event();if(fail && !f.after)return 1; + memory[at/4]=v;return fail; +} +static int write64(void *cookie,size_t at,uint64_t v){(void)at;(void)v;(void)cookie;CHECK(0);return 1;} +static int barrier(void *cookie){CHECK(cookie==&f);return event();} +struct Fixture { + NativeAdrenoMemory tables,object; + ArmSmmu4KLiveArena arena; + ArmSmmu4KLiveState state{}; + ArmSmmu4KLiveVisibility visibility; + NativeAdrenoVm vm; +}; +static void mmioReset(void) +{ + std::memset(memory,0,sizeof(memory));f={};streams=(ArmSmmu4KStreamState){}; + io=(ArmSmmu4KIo){&f,BYTES,read32,read64,write32,write64,barrier}; + memory[0x20/4]=(1u<<30)|(1u<<27)|(4u<<9)|8; + memory[0x24/4]=0x40000010;memory[0x28/4]=0x1433;memory[0]=0x436; + for(unsigned i=0;i<8;++i)memory[(0xc00+4*i)/4]=0x20000; + memory[0x800/4]=1u<<31|9;memory[0xc00/4]=3; +} +/* Real prepared arena, enabled actual context image and exact stream routes. */ +static IOReturn exposed(Fixture &s) +{ + mmioReset(); + IOReturn status=s.tables.allocate(32*4096,40,true,32*4096);if(status)return status; + IOReturn smmu=NativeAdrenoPrepareSmmuMemory(&s.tables,32,40,0x800000,65536,0,16, + &s.arena,&s.state,&s.visibility); + if(smmu)return smmu; + context=(ArmSmmu4KContextConfig){.lower=&s.arena.config,.lower_tables=&s.state.tables, + .id0=memory[0x20/4],.id2=memory[0x28/4],.asid=1,.coherent_walk=1, + .stall_on_fault=0,.continue_after_fault=1}; + ArmSmmu4KContextConfig actual=context; + CHECK(ArmSmmu4KEncodeContext(&actual,&image)==ArmSmmu4KSuccess); + memory[0x1008/4]=image.cbar;memory[0x1808/4]=image.cba2r; + memory[CB/4]=image.sctlr;memory[(CB+0x10)/4]=image.tcr2;memory[(CB+0x30)/4]=image.tcr; + std::memcpy(memory+(CB+0x20)/4,&image.ttbr0,8); + std::memcpy(memory+(CB+0x28)/4,&image.ttbr1,8); + memory[(CB+0x38)/4]=image.mair0; + if(ArmSmmu4KAttachStreams(&io,&streams,routes,2,2,&context,&route_result)!=ArmSmmu4KSuccess) + return kIOReturnBadArgument; + NativeAdrenoVmRegion lower={&s.tables,&s.arena,&s.state}; + return s.vm.attach(&io,&context,&streams,&lower,nullptr,32,1000); +} +static IOReturn object(Fixture &s,size_t pages) +{return s.object.allocate(pages*4096,40,false,pages*4096);} +int main() +{ + /* A real map/hold/unmap transaction across the actual code paths. */ + { + Fixture s; + CHECK(exposed(s)==kIOReturnSuccess); + CHECK(object(s,2)==kIOReturnSuccess); + uint64_t address=0,handle=0; + CHECK(s.vm.findAddress(0,8192,8192,&address)==kIOReturnSuccess); + CHECK(address==0x800000); + CHECK(s.vm.objectCount()==0 && s.vm.spanCount()==0); + CHECK(s.vm.map(&s.object,0,address,3,0,2,&handle)==kIOReturnSuccess); + CHECK(handle && s.vm.objectCount()==1 && s.vm.spanCount()==s.object.spanCount()); + CHECK(s.object.deviceReferences()==1); /* one mapping reference */ + CHECK(s.vm.findAddress(0,8192,8192,&address)==kIOReturnSuccess && address==0x802000); + CHECK(s.vm.holdJob(handle)==kIOReturnSuccess && s.object.deviceReferences()==2); + CHECK(s.vm.unmap(handle)==kIOReturnBusy); + CHECK(s.vm.retireJobAfterCompletion(handle)==kIOReturnSuccess); + CHECK(s.vm.unmap(handle)==kIOReturnSuccess); + CHECK(s.vm.objectCount()==0 && s.vm.spanCount()==0); + CHECK(s.object.deviceReferences()==0 && s.object.release()==kIOReturnSuccess); + CHECK(s.tables.release()==kIOReturnBusy); + CHECK(s.vm.detach()==kIOReturnSuccess); + CHECK(s.tables.retireAfterQuiescence()==kIOReturnSuccess); + CHECK(s.tables.release()==kIOReturnSuccess); + CHECK(fixture.live.empty() && fixture.metadata.empty()); + } + /* Refusals precede any ownership or table mutation. */ + { + Fixture s; + CHECK(exposed(s)==kIOReturnSuccess); + CHECK(object(s,2)==kIOReturnSuccess); + unsigned refs=s.object.deviceReferences(),spans=s.vm.spanCount(),reads=f.calls; + uint64_t handle=0; + CHECK(s.vm.map(&s.object,0,0x800001,3,0,2,&handle)==kIOReturnBadArgument); + CHECK(s.vm.map(&s.object,1,0x800000,3,0,2,&handle)==kIOReturnBadArgument); + CHECK(s.vm.map(&s.object,0,0xec0000,3,0,2,&handle)==kIOReturnBadArgument); + CHECK(s.vm.map(&s.object,0,0x800000,2,0,2,&handle)==kIOReturnBadArgument); + CHECK(s.vm.map(&s.object,0,0x800000,3,2,2,&handle)==kIOReturnBadArgument); + CHECK(s.vm.map(&s.object,0,0x800000,3,0,1,&handle)==kIOReturnBadArgument); + CHECK(s.vm.map(nullptr,0,0x800000,3,0,2,&handle)==kIOReturnBadArgument); + CHECK(!handle && refs==s.object.deviceReferences() && spans==s.vm.spanCount() && reads==f.calls); + CHECK(s.vm.map(&s.object,0,0x800000,3,0,2,&handle)==kIOReturnSuccess); + uint64_t mapped=handle; + /* A refused map clears only its own out-parameter. */ + CHECK(s.vm.map(&s.object,0,0x800000,3,0,2,&handle)==kIOReturnBusy && !handle); + CHECK(s.vm.unmap(mapped+100)==kIOReturnBadArgument); + CHECK(s.vm.holdJob(mapped+100)==kIOReturnBadArgument); + CHECK(s.vm.retireJobAfterCompletion(mapped)==kIOReturnBadArgument); + CHECK(s.vm.unmap(mapped)==kIOReturnSuccess); + CHECK(s.object.release()==kIOReturnSuccess); + CHECK(s.vm.detach()==kIOReturnSuccess); + CHECK(s.tables.retireAfterQuiescence()==kIOReturnSuccess && s.tables.release()==kIOReturnSuccess); + CHECK(fixture.live.empty() && fixture.metadata.empty()); + } + /* Post-hold CPU publication failure: poison, retain everything, refuse. */ + { + Fixture s; + CHECK(exposed(s)==kIOReturnSuccess); + CHECK(object(s,2)==kIOReturnSuccess); + fixture.cache_fail_call=fixture.cache_calls+1; /* next publication fails */ + uint64_t handle=0; + CHECK(s.vm.map(&s.object,0,0x800000,3,0,2,&handle)==kIOReturnIOError); + CHECK(!handle && s.vm.poisoned() && s.vm.spanCount()==0); /* new BO reference is retained */ + CHECK(s.object.quarantined() && s.object.deviceReferences()>=1 && s.tables.quarantined()); + CHECK(s.state.prepared && s.state.poisoned); + CHECK(s.vm.map(&s.object,0,0x800000,3,0,2,&handle)==kIOReturnNotReady); + CHECK(s.vm.unmap(0)==kIOReturnNotReady && s.vm.detach()==kIOReturnBusy); + CHECK(s.vm.attach(&io,&context,&streams,nullptr,nullptr,32,1000)==kIOReturnBusy); + CHECK(s.object.release()==kIOReturnBusy && s.tables.release()==kIOReturnBusy); + CHECK(!fixture.live.empty()); + } + std::printf("PASS: 1 native map/hold/unmap transaction, 13 pre-ownership refusals and 1 post-hold poison/retention case over a software SMMU register peer\n"); +} diff --git a/drivers/NativeGeniSpi/GeniSpi.c b/drivers/NativeGeniSpi/GeniSpi.c new file mode 100644 index 0000000..ec6705c --- /dev/null +++ b/drivers/NativeGeniSpi/GeniSpi.c @@ -0,0 +1,348 @@ +// SPDX-License-Identifier: BSD-2-Clause-Patent +#include "GeniSpi.h" + +static uint32_t rd(struct geni_spi *s, uint32_t reg) { return s->se.rd(s->se.ctx, reg); } +static void wr(struct geni_spi *s, uint32_t reg, uint32_t v) { s->se.wr(s->se.ctx, reg, v); } +static void rmw_set(struct geni_spi *s, uint32_t reg, uint32_t bits) { wr(s, reg, rd(s, reg) | bits); } + +static void irq_clear(struct geni_spi *s) { + wr(s, SE_GSI_EVENT_EN, 0); + wr(s, SE_GENI_M_IRQ_CLEAR, 0xffffffffu); + wr(s, SE_GENI_S_IRQ_CLEAR, 0xffffffffu); + wr(s, SE_DMA_TX_IRQ_CLR, 0xffffffffu); + wr(s, SE_DMA_RX_IRQ_CLR, 0xffffffffu); + wr(s, SE_IRQ_EN, 0xffffffffu); +} + +// FIFO words hold fifo_width_bits/8 packed bytes at 8 bits per word. +static unsigned bytes_per_word(const struct geni_spi *s) { return s->info.fifo_width_bits / 8; } + +int geni_spi_read_info(struct geni_spi *s) { + if (!s || !s->se.rd || !s->wrapper.rd) return GENI_ERR_PARAM; + uint32_t fw = rd(s, GENI_FW_REVISION_RO); + s->info.fw_revision = fw; + s->info.protocol = (fw >> GENI_FW_PROTO_SHIFT) & 0xff; + s->info.if_disable = rd(s, GENI_IF_DISABLE_RO); + s->info.fifo_disabled = (s->info.if_disable & GENI_FIFO_IF_DISABLE) != 0; + s->info.geni_status = rd(s, SE_GENI_STATUS); + s->info.hw_param0 = rd(s, SE_HW_PARAM_0); + s->info.qup_hw_version = s->wrapper.rd(s->wrapper.ctx, QUPV3_HW_VER_REG); + uint32_t major = s->info.qup_hw_version >> 28, minor = (s->info.qup_hw_version >> 16) & 0xfff; + uint32_t mask = ((major == 3 && minor >= 10) || major > 3) ? 0xff0000u : 0x3f0000u; + s->info.tx_fifo_depth = (s->info.hw_param0 & mask) >> 16; + s->info.fifo_width_bits = (s->info.hw_param0 >> 24) & 0x3f; + return 0; +} + +// Packing for 8-bit words, 4 per FIFO word, MSB first inside each byte; same values the vendor sequence programs. +static void config_packing(struct geni_spi *s, unsigned bpw, unsigned pack_words, int msb_first) { + uint32_t cfg[GENI_PACK_VECTORS] = {0}; + int idx_start = msb_first ? (int)bpw - 1 : 0, idx = idx_start; + int idx_delta = msb_first ? -8 : 8; + unsigned ceil_bpw = (bpw + 7) & ~7u; + unsigned iter = ceil_bpw * pack_words / 8; + int temp_bpw = (int)bpw; + if (iter == 0 || iter > GENI_PACK_VECTORS) return; + for (unsigned i = 0; i < iter; i++) { + int len = (temp_bpw < 8 ? temp_bpw : 8) - 1; + cfg[i] = ((uint32_t)idx << GENI_PACK_START_SHIFT) | ((uint32_t)(msb_first != 0) << GENI_PACK_DIR_SHIFT) | + ((uint32_t)len << GENI_PACK_LEN_SHIFT); + if (temp_bpw <= 8) { + idx = (int)(i + 1) * 8 + idx_start; + temp_bpw = (int)bpw; + } else { + idx += idx_delta; + temp_bpw -= 8; + } + } + cfg[iter - 1] |= GENI_PACK_STOP; + uint32_t cfg0 = cfg[0] | (cfg[1] << GENI_PACK_VECTOR_SHIFT); + uint32_t cfg1 = cfg[2] | (cfg[3] << GENI_PACK_VECTOR_SHIFT); + wr(s, SE_GENI_TX_PACKING_CFG0, cfg0); + wr(s, SE_GENI_TX_PACKING_CFG1, cfg1); + wr(s, SE_GENI_RX_PACKING_CFG0, cfg0); + wr(s, SE_GENI_RX_PACKING_CFG1, cfg1); + if (pack_words || bpw == 32) wr(s, SE_GENI_BYTE_GRAN, bpw / 16); +} + +static void set_word_len(struct geni_spi *s, unsigned bpw, int msb_first) { + unsigned pack = (s->info.fifo_width_bits % bpw) ? 1 : s->info.fifo_width_bits / bpw; + config_packing(s, bpw, pack, msb_first); + wr(s, SE_SPI_WORD_LEN, (bpw - GENI_WORD_LEN_MIN) & 0x3ff); +} + +int geni_spi_init(struct geni_spi *s) { + int r = geni_spi_read_info(s); + if (r) return r; + if (s->info.protocol != GENI_PROTO_SPI) return GENI_ERR_PROTO; + if (s->use_dma && (!s->dma.tx_virt || !s->dma.rx_virt || !s->dma.tx_phys || !s->dma.rx_phys || !s->dma.size)) + return GENI_ERR_PARAM; + if (s->info.fifo_disabled && !s->allow_fifo_disabled && !s->use_dma) return GENI_ERR_NOTSUP; // GPI DMA path not implemented + if (s->info.tx_fifo_depth < 4 || s->info.fifo_width_bits != 32) return GENI_ERR_NODEV; + // geni_se_init + irq_clear(s); + rmw_set(s, SE_GENI_CGC_CTRL, GENI_DEFAULT_CGC_EN); + rmw_set(s, SE_DMA_GENERAL_CFG, GENI_AHB_SEC_SLV_CGC_ON | GENI_DMA_AHB_SLV_CGC_ON | GENI_DMA_TX_CGC_ON | GENI_DMA_RX_CGC_ON); + wr(s, GENI_OUTPUT_CTRL, GENI_DEFAULT_IO_OUTPUT_CTRL); + wr(s, GENI_FORCE_DEFAULT_REG, GENI_FORCE_DEFAULT); + rmw_set(s, SE_IRQ_EN, GENI_IRQ_M | GENI_IRQ_S | GENI_IRQ_DMA_TX | GENI_IRQ_DMA_RX); + wr(s, SE_GENI_DMA_MODE_EN, rd(s, SE_GENI_DMA_MODE_EN) & ~GENI_DMA_MODE_EN); + wr(s, SE_GSI_EVENT_EN, 0); + wr(s, SE_GENI_RX_WATERMARK_REG, s->info.tx_fifo_depth - 3); + wr(s, SE_GENI_RX_RFR_WATERMARK_REG, s->info.tx_fifo_depth - 2); + rmw_set(s, SE_GENI_M_IRQ_EN, GENI_M_COMMON_IRQ_EN); + rmw_set(s, SE_GENI_S_IRQ_EN, GENI_S_COMMON_IRQ_EN); + s->tx_wm = 1; + uint32_t major = s->info.qup_hw_version >> 28, minor = (s->info.qup_hw_version >> 16) & 0xfff; + s->oversampling = (major == 1 && minor == 0) ? 2 : 1; + // FIFO mode selection + irq_clear(s); + rmw_set(s, SE_GENI_M_IRQ_EN, GENI_M_CMD_DONE | GENI_M_TX_FIFO_WATERMARK | GENI_M_RX_FIFO_WATERMARK | GENI_M_RX_FIFO_LAST); + wr(s, SE_GENI_DMA_MODE_EN, rd(s, SE_GENI_DMA_MODE_EN) & ~GENI_DMA_MODE_EN); + wr(s, SE_SPI_LOOPBACK, 0); + wr(s, SE_SPI_DEMUX_SEL, 0); + wr(s, SE_SPI_CPHA, 0); + wr(s, SE_SPI_CPOL, 0); + wr(s, SE_SPI_DEMUX_OUTPUT_INV, 0); + wr(s, SE_SPI_TRANS_CFG, rd(s, SE_SPI_TRANS_CFG) & ~GENI_CS_TOGGLE); // chip select is never driven manually + s->cur_bpw = 0; + s->cur_speed_hz = 0; + s->last_mode = 0; + s->last_cs = 0; + if (!s->poll_limit) s->poll_limit = 1000000; + if (s->use_dma) { + // geni_se_select_dma_mode: drop the FIFO interrupt sources and switch the engine to its DMA master. + irq_clear(s); + wr(s, SE_GENI_M_IRQ_EN, rd(s, SE_GENI_M_IRQ_EN) & ~(GENI_M_CMD_DONE | GENI_M_TX_FIFO_WATERMARK | GENI_M_RX_FIFO_WATERMARK | GENI_M_RX_FIFO_LAST)); + wr(s, SE_GENI_DMA_MODE_EN, rd(s, SE_GENI_DMA_MODE_EN) | GENI_DMA_MODE_EN); + } + if (s->poll_only == 1) wr(s, SE_IRQ_EN, 0); // status registers stay pollable; no unhandled interrupt reaches the GIC + return 0; +} + +int geni_spi_setup(struct geni_spi *s, unsigned mode, unsigned chip_select) { + if (chip_select > 15) return GENI_ERR_PARAM; + unsigned changed = s->last_mode ^ mode; + int cs_changed = ((unsigned)s->last_cs != chip_select); + s->last_cs = (int)chip_select; + s->last_mode = mode; + if (changed & GENI_SPI_LSB_FIRST) s->cur_bpw = 0; + if (changed & GENI_SPI_LOOP) wr(s, SE_SPI_LOOPBACK, (mode & GENI_SPI_LOOP) ? GENI_LOOPBACK_ENABLE : 0); + if (cs_changed) wr(s, SE_SPI_DEMUX_SEL, chip_select); + if (changed & GENI_SPI_CPHA) wr(s, SE_SPI_CPHA, (mode & GENI_SPI_CPHA) ? GENI_CPHA_BIT : 0); + if (changed & GENI_SPI_CPOL) wr(s, SE_SPI_CPOL, (mode & GENI_SPI_CPOL) ? GENI_CPOL_BIT : 0); + if ((changed & GENI_SPI_CS_HIGH) || (cs_changed && (mode & GENI_SPI_CS_HIGH))) + wr(s, SE_SPI_DEMUX_OUTPUT_INV, (mode & GENI_SPI_CS_HIGH) ? (1u << chip_select) : 0); + return 0; +} + +int geni_spi_set_clock(struct geni_spi *s, unsigned speed_hz, unsigned sclk_hz, unsigned clk_sel) { + if (!speed_hz || !sclk_hz || clk_sel > 7 || !s->oversampling) return GENI_ERR_PARAM; + unsigned long long denom = (unsigned long long)s->oversampling * speed_hz; + unsigned long long div = (sclk_hz + denom - 1) / denom; + if (div == 0 || div > GENI_CLK_DIV_MAX) return GENI_ERR_PARAM; + wr(s, SE_GENI_CLK_SEL, clk_sel); + wr(s, GENI_SER_M_CLK_CFG, ((uint32_t)div << GENI_CLK_DIV_SHIFT) | GENI_SER_CLK_EN); + s->cur_speed_hz = speed_hz; + return 0; +} + +struct xfer { + const uint8_t *tx; + uint8_t *rx; + size_t tx_rem, rx_rem, len; +}; + +static int handle_tx(struct geni_spi *s, struct xfer *x) { + unsigned bpw = bytes_per_word(s); + size_t max = (size_t)(s->info.tx_fifo_depth - s->tx_wm) * bpw; + if (x->tx_rem < max) max = x->tx_rem; + const uint8_t *p = x->tx + (x->len - x->tx_rem); + size_t i = 0; + while (i < max) { + uint32_t word = 0; + size_t n = max - i < bpw ? max - i : bpw; + for (size_t j = 0; j < n; j++) word |= (uint32_t)p[i++] << (8 * j); + wr(s, SE_GENI_TX_FIFOn, word); + } + x->tx_rem -= max; + if (!x->tx_rem) { + wr(s, SE_GENI_TX_WATERMARK_REG, 0); + return 0; + } + return 1; +} + +static void handle_rx(struct geni_spi *s, struct xfer *x) { + unsigned bpw = bytes_per_word(s); + uint32_t st = rd(s, SE_GENI_RX_FIFO_STATUS); + size_t bytes = (size_t)(st & GENI_RX_WC_MASK) * bpw; + if (st & GENI_RX_LAST) { + unsigned valid = (st >> GENI_RX_LAST_VALID_SHIFT) & GENI_RX_LAST_VALID_MASK; + if (valid && valid < 4) bytes -= bpw - valid; + } + if (!x->rx) { + for (size_t i = 0; i < (bytes + bpw - 1) / bpw; i++) (void)rd(s, SE_GENI_RX_FIFOn); + return; + } + if (x->rx_rem < bytes) bytes = x->rx_rem; + uint8_t *p = x->rx + (x->len - x->rx_rem); + size_t i = 0; + while (i < bytes) { + uint32_t word = rd(s, SE_GENI_RX_FIFOn); + size_t n = bytes - i < bpw ? bytes - i : bpw; + for (size_t j = 0; j < n; j++) p[i++] = (uint8_t)(word >> (8 * j)); + } + x->rx_rem -= bytes; +} + +static int wait_bit(struct geni_spi *s, uint32_t bit) { + for (unsigned n = 0; n < s->poll_limit; n++) { + uint32_t st = rd(s, SE_GENI_M_IRQ_STATUS); + if (st & bit) { + wr(s, SE_GENI_M_IRQ_CLEAR, st); + return 0; + } + if (s->se.delay_us) s->se.delay_us(s->se.ctx, 1); + } + return GENI_ERR_TIMEOUT; +} + +static void recover(struct geni_spi *s) { + wr(s, SE_GENI_TX_WATERMARK_REG, 0); + wr(s, SE_GENI_M_CMD_CTRL_REG, GENI_M_CMD_CANCEL); + if (wait_bit(s, GENI_M_CMD_CANCEL_DONE | GENI_M_CMD_DONE) == 0) return; + wr(s, SE_GENI_M_CMD_CTRL_REG, GENI_M_CMD_ABORT); + (void)wait_bit(s, GENI_M_CMD_ABORT_DONE | GENI_M_CMD_DONE); +} + +static int transfer_dma(struct geni_spi *s, const uint8_t *tx, uint8_t *rx, size_t len, int keep_cs) { + struct geni_dma *d = &s->dma; + if (len > d->size) return GENI_ERR_PARAM; + uint32_t cmd = 0; + if (tx) { + volatile uint8_t *tv = d->tx_virt; // volatile: the buffer may be an uncached device-type mapping, so no merged stores + for (size_t i = 0; i < len; i++) tv[i] = tx[i]; + if (d->clean) d->clean(d->ctx, d->tx_virt, len); + cmd |= GENI_M_CMD_TX_ONLY; + wr(s, SE_SPI_TX_TRANS_LEN, (uint32_t)len); + } + if (rx) { + volatile uint8_t *rv0 = d->rx_virt; + for (size_t i = 0; i < len; i++) rv0[i] = 0x5a; // canary: bytes the engine never wrote stay recognisable + if (d->clean_inval) d->clean_inval(d->ctx, d->rx_virt, len); + cmd |= GENI_M_CMD_RX_ONLY; + wr(s, SE_SPI_RX_TRANS_LEN, (uint32_t)len); + } + if (cmd == (GENI_M_CMD_TX_ONLY | GENI_M_CMD_RX_ONLY)) cmd = GENI_M_CMD_TX_RX; + wr(s, SE_GENI_M_IRQ_CLEAR, 0xffffffffu); + wr(s, SE_DMA_TX_IRQ_CLR, 0xffffffffu); + wr(s, SE_DMA_RX_IRQ_CLR, 0xffffffffu); + wr(s, SE_GENI_M_CMD0, (cmd << GENI_M_OPCODE_SHIFT) | (keep_cs ? GENI_M_PARAM_FRAGMENTATION : 0)); + // geni_se_rx_init_dma / geni_se_tx_init_dma: the LEN write starts each direction. + if (rx) { + wr(s, SE_DMA_RX_IRQ_EN_SET, GENI_DMA_DONE | GENI_DMA_EOT | GENI_DMA_AHB_ERR); + wr(s, SE_DMA_RX_PTR_L, (uint32_t)d->rx_phys); + wr(s, SE_DMA_RX_PTR_H, (uint32_t)(d->rx_phys >> 32)); + wr(s, SE_DMA_RX_ATTR, 0); + wr(s, SE_DMA_RX_LEN, (uint32_t)len); + } + if (tx) { + wr(s, SE_DMA_TX_IRQ_EN_SET, GENI_DMA_DONE | GENI_DMA_EOT | GENI_DMA_AHB_ERR); + wr(s, SE_DMA_TX_PTR_L, (uint32_t)d->tx_phys); + wr(s, SE_DMA_TX_PTR_H, (uint32_t)(d->tx_phys >> 32)); + wr(s, SE_DMA_TX_ATTR, GENI_DMA_TX_ATTR_EOT_BUF); + wr(s, SE_DMA_TX_LEN, (uint32_t)len); + } + int tx_done = !tx, rx_done = !rx, status = GENI_ERR_TIMEOUT; + for (unsigned n = 0; n < s->poll_limit; n++) { + uint32_t ts = tx ? rd(s, SE_DMA_TX_IRQ_STAT) : 0, rs = rx ? rd(s, SE_DMA_RX_IRQ_STAT) : 0; + uint32_t ms = rd(s, SE_GENI_M_IRQ_STATUS); + if (ts) wr(s, SE_DMA_TX_IRQ_CLR, ts); + if (rs) wr(s, SE_DMA_RX_IRQ_CLR, rs); + if (ms) wr(s, SE_GENI_M_IRQ_CLEAR, ms); + if ((ts & GENI_DMA_AHB_ERR) || (rs & GENI_DMA_AHB_ERR) || (ms & GENI_M_ERRORS)) { status = GENI_ERR_IO; break; } + if (ts & GENI_DMA_DONE) tx_done = 1; + if (rs & GENI_DMA_DONE) rx_done = 1; + if (tx_done && rx_done) { status = 0; break; } + if (s->se.delay_us) s->se.delay_us(s->se.ctx, 1); + } + s->last_tx_in = tx ? rd(s, SE_DMA_TX_LEN_IN) : 0; + s->last_rx_in = rx ? rd(s, SE_DMA_RX_LEN_IN) : 0; + if (status == 0) { + if (rx) { + if (d->inval) d->inval(d->ctx, d->rx_virt, len); + volatile uint8_t *rv = d->rx_virt; + for (size_t i = 0; i < len; i++) rx[i] = rv[i]; + } + return 0; + } + s->snap_dma_tx = rd(s, SE_DMA_TX_IRQ_STAT); + s->snap_dma_rx = rd(s, SE_DMA_RX_IRQ_STAT); + s->snap_dma_txin = rd(s, SE_DMA_TX_LEN_IN); + s->snap_dma_rxin = rd(s, SE_DMA_RX_LEN_IN); + s->snap_irq = rd(s, SE_GENI_M_IRQ_STATUS); + s->snap_status = rd(s, SE_GENI_STATUS); + s->snap_txfs = rd(s, SE_GENI_TX_FIFO_STATUS); + s->snap_rxfs = rd(s, SE_GENI_RX_FIFO_STATUS); + s->snap_valid = 1; + recover(s); + wr(s, SE_DMA_TX_FSM_RST, 1); + wr(s, SE_DMA_RX_FSM_RST, 1); + return status; +} + +int geni_spi_transfer(struct geni_spi *s, const uint8_t *tx, uint8_t *rx, size_t len, int keep_cs) { + if (!s || !len || len > GENI_TRANS_LEN_MASK || (!tx && !rx) || !s->cur_speed_hz) return GENI_ERR_PARAM; + if (s->cur_bpw != 8) { + set_word_len(s, 8, !(s->last_mode & GENI_SPI_LSB_FIRST)); + s->cur_bpw = 8; + } + if (s->use_dma) return transfer_dma(s, tx, rx, len, keep_cs); + struct xfer x = {tx, rx, tx ? len : 0, rx ? len : 0, len}; + uint32_t cmd = 0; + if (tx) { + cmd |= GENI_M_CMD_TX_ONLY; + wr(s, SE_SPI_TX_TRANS_LEN, (uint32_t)len); + } + if (rx) { + cmd |= GENI_M_CMD_RX_ONLY; + wr(s, SE_SPI_RX_TRANS_LEN, (uint32_t)len); + } + if (cmd == (GENI_M_CMD_TX_ONLY | GENI_M_CMD_RX_ONLY)) cmd = GENI_M_CMD_TX_RX; + wr(s, SE_GENI_M_IRQ_CLEAR, 0xffffffffu); + wr(s, SE_GENI_M_CMD0, (cmd << GENI_M_OPCODE_SHIFT) | (keep_cs ? GENI_M_PARAM_FRAGMENTATION : 0)); + if (tx && handle_tx(s, &x)) wr(s, SE_GENI_TX_WATERMARK_REG, s->tx_wm); + int status = GENI_ERR_TIMEOUT; + for (unsigned n = 0; n < s->poll_limit; n++) { + uint32_t st = rd(s, SE_GENI_M_IRQ_STATUS); + if (!st) { + if (s->se.delay_us) s->se.delay_us(s->se.ctx, 1); + continue; + } + wr(s, SE_GENI_M_IRQ_CLEAR, st); + if (st & GENI_M_ERRORS) { + status = GENI_ERR_IO; + break; + } + if (st & (GENI_M_RX_FIFO_WATERMARK | GENI_M_RX_FIFO_LAST)) handle_rx(s, &x); + if ((st & GENI_M_TX_FIFO_WATERMARK) && tx) handle_tx(s, &x); + if (st & GENI_M_CMD_DONE) { + if (rx && x.rx_rem) handle_rx(s, &x); + status = (x.tx_rem || x.rx_rem) ? GENI_ERR_IO : 0; + if (x.tx_rem) wr(s, SE_GENI_TX_WATERMARK_REG, 0); + return status; + } + } + s->snap_dma_tx = rd(s, SE_DMA_TX_IRQ_STAT); + s->snap_dma_rx = rd(s, SE_DMA_RX_IRQ_STAT); + s->snap_dma_txin = rd(s, SE_DMA_TX_LEN_IN); + s->snap_dma_rxin = rd(s, SE_DMA_RX_LEN_IN); + s->snap_irq = rd(s, SE_GENI_M_IRQ_STATUS); + s->snap_status = rd(s, SE_GENI_STATUS); + s->snap_txfs = rd(s, SE_GENI_TX_FIFO_STATUS); + s->snap_rxfs = rd(s, SE_GENI_RX_FIFO_STATUS); + s->snap_valid = 1; + recover(s); + return status; +} diff --git a/drivers/NativeGeniSpi/GeniSpi.h b/drivers/NativeGeniSpi/GeniSpi.h new file mode 100644 index 0000000..f2fdcc9 --- /dev/null +++ b/drivers/NativeGeniSpi/GeniSpi.h @@ -0,0 +1,223 @@ +// SPDX-License-Identifier: BSD-2-Clause-Patent +// Qualcomm GENI serial-engine SPI master, FIFO (PIO) mode. Register layout and sequences follow the study of the +// Linux spi-geni-qcom / qcom-geni-se drivers (no code copied). No board addresses, clocks or pins here. +#ifndef GENI_SPI_H +#define GENI_SPI_H +#include +#include + +// Serial-engine registers (offsets from the SE base). +#define GENI_FORCE_DEFAULT_REG 0x20 +#define GENI_OUTPUT_CTRL 0x24 +#define SE_GENI_CGC_CTRL 0x28 +#define SE_GENI_STATUS 0x40 +#define GENI_SER_M_CLK_CFG 0x48 +#define GENI_IF_DISABLE_RO 0x64 +#define GENI_FW_REVISION_RO 0x68 +#define SE_GENI_CLK_SEL 0x7c +#define SE_GENI_BYTE_GRAN 0x254 +#define SE_GENI_DMA_MODE_EN 0x258 +#define SE_GENI_TX_PACKING_CFG0 0x260 +#define SE_GENI_TX_PACKING_CFG1 0x264 +#define SE_GENI_RX_PACKING_CFG0 0x284 +#define SE_GENI_RX_PACKING_CFG1 0x288 +#define SE_SPI_CPHA 0x224 +#define SE_SPI_LOOPBACK 0x22c +#define SE_SPI_CPOL 0x230 +#define SE_SPI_DEMUX_OUTPUT_INV 0x24c +#define SE_SPI_DEMUX_SEL 0x250 +#define SE_SPI_TRANS_CFG 0x25c +#define SE_SPI_WORD_LEN 0x268 +#define SE_SPI_TX_TRANS_LEN 0x26c +#define SE_SPI_RX_TRANS_LEN 0x270 +#define SE_GENI_M_CMD0 0x600 +#define SE_GENI_M_CMD_CTRL_REG 0x604 +#define SE_GENI_M_IRQ_STATUS 0x610 +#define SE_GENI_M_IRQ_EN 0x614 +#define SE_GENI_M_IRQ_CLEAR 0x618 +#define SE_GENI_S_IRQ_EN 0x644 +#define SE_GENI_S_IRQ_CLEAR 0x648 +#define SE_DMA_TX_IRQ_CLR 0xc44 +#define SE_DMA_RX_IRQ_CLR 0xd44 +#define SE_GENI_TX_FIFOn 0x700 +#define SE_GENI_RX_FIFOn 0x780 +#define SE_GENI_TX_FIFO_STATUS 0x800 +#define SE_GENI_RX_FIFO_STATUS 0x804 +#define SE_GENI_TX_WATERMARK_REG 0x80c +#define SE_GENI_RX_WATERMARK_REG 0x810 +#define SE_GENI_RX_RFR_WATERMARK_REG 0x814 +#define SE_IRQ_EN 0xe1c +#define SE_HW_PARAM_0 0xe24 +#define SE_DMA_GENERAL_CFG 0xe30 +#define SE_GSI_EVENT_EN 0xe18 +// Serial-engine DMA registers (qcom-geni-se.c / geni-se.h of the pinned mainline tree). +#define SE_DMA_TX_PTR_L 0xc30 +#define SE_DMA_TX_PTR_H 0xc34 +#define SE_DMA_TX_ATTR 0xc38 +#define SE_DMA_TX_LEN 0xc3c +#define SE_DMA_TX_IRQ_STAT 0xc40 +#define SE_DMA_TX_IRQ_EN_SET 0xc4c +#define SE_DMA_TX_LEN_IN 0xc54 +#define SE_DMA_TX_FSM_RST 0xc58 +#define SE_DMA_RX_PTR_L 0xd30 +#define SE_DMA_RX_PTR_H 0xd34 +#define SE_DMA_RX_ATTR 0xd38 +#define SE_DMA_RX_LEN 0xd3c +#define SE_DMA_RX_IRQ_STAT 0xd40 +#define SE_DMA_RX_IRQ_EN_SET 0xd4c +#define SE_DMA_RX_LEN_IN 0xd54 +#define SE_DMA_RX_FSM_RST 0xd58 +// SE_DMA_{TX,RX}_IRQ_STAT / _EN_SET fields +#define GENI_DMA_DONE (1u<<0) +#define GENI_DMA_EOT (1u<<1) +#define GENI_DMA_AHB_ERR (1u<<2) // TX_SBE / RX_SBE +#define GENI_DMA_RESET_DONE (1u<<3) +#define GENI_DMA_TX_ATTR_EOT_BUF (1u<<0) +// Wrapper register (offset from the QUP wrapper base). +#define QUPV3_HW_VER_REG 0x4 + +#define GENI_PROTO_NONE 0 +#define GENI_PROTO_SPI 1 +#define GENI_PROTO_INVALID 255 + +#define GENI_M_CMD_TX_ONLY 1 +#define GENI_M_CMD_RX_ONLY 2 +#define GENI_M_CMD_TX_RX 3 // full duplex: spi-geni-qcom.c issues SPI_TX_ONLY|SPI_RX_ONLY (its SPI_TX_RX=7 is never used) +#define GENI_M_PARAM_FRAGMENTATION (1u<<2) + +// Register field values. +#define GENI_FORCE_DEFAULT (1u<<0) +#define GENI_SER_CLK_EN (1u<<0) +#define GENI_CLK_DIV_SHIFT 4 +#define GENI_CLK_DIV_MAX 0xfffu +#define GENI_FIFO_IF_DISABLE (1u<<0) +#define GENI_FW_PROTO_SHIFT 8 +#define GENI_DMA_MODE_EN (1u<<0) +#define GENI_DEFAULT_CGC_EN 0x7fu +#define GENI_DEFAULT_IO_OUTPUT_CTRL 0x7fu +#define GENI_M_OPCODE_SHIFT 27 +#define GENI_M_PARAMS_MASK 0x7ffffffu +#define GENI_M_CMD_CANCEL (1u<<2) +#define GENI_M_CMD_ABORT (1u<<1) +#define GENI_STATUS_M_ACTIVE (1u<<0) +#define GENI_TRANS_LEN_MASK 0xffffffu +#define GENI_WORD_LEN_MIN 4 +#define GENI_CS_TOGGLE (1u<<1) +#define GENI_CPHA_BIT (1u<<0) +#define GENI_CPOL_BIT (1u<<2) +#define GENI_LOOPBACK_ENABLE 1u +// SE_IRQ_EN +#define GENI_IRQ_DMA_RX (1u<<0) +#define GENI_IRQ_DMA_TX (1u<<1) +#define GENI_IRQ_M (1u<<2) +#define GENI_IRQ_S (1u<<3) +// SE_DMA_GENERAL_CFG +#define GENI_DMA_RX_CGC_ON (1u<<0) +#define GENI_DMA_TX_CGC_ON (1u<<1) +#define GENI_DMA_AHB_SLV_CGC_ON (1u<<2) +#define GENI_AHB_SEC_SLV_CGC_ON (1u<<3) +// M_IRQ bits +#define GENI_M_CMD_DONE (1u<<0) +#define GENI_M_CMD_OVERRUN (1u<<1) +#define GENI_M_ILLEGAL_CMD (1u<<2) +#define GENI_M_CMD_FAILURE (1u<<3) +#define GENI_M_CMD_CANCEL_DONE (1u<<4) +#define GENI_M_CMD_ABORT_DONE (1u<<5) +#define GENI_M_IO_DATA_DEASSERT (1u<<22) +#define GENI_M_IO_DATA_ASSERT (1u<<23) +#define GENI_M_RX_FIFO_RD_ERR (1u<<24) +#define GENI_M_RX_FIFO_WR_ERR (1u<<25) +#define GENI_M_RX_FIFO_WATERMARK (1u<<26) +#define GENI_M_RX_FIFO_LAST (1u<<27) +#define GENI_M_TX_FIFO_RD_ERR (1u<<28) +#define GENI_M_TX_FIFO_WR_ERR (1u<<29) +#define GENI_M_TX_FIFO_WATERMARK (1u<<30) +#define GENI_M_COMMON_IRQ_EN (0x7eu | GENI_M_IO_DATA_DEASSERT | GENI_M_IO_DATA_ASSERT | GENI_M_RX_FIFO_RD_ERR | \ + GENI_M_RX_FIFO_WR_ERR | GENI_M_TX_FIFO_RD_ERR | GENI_M_TX_FIFO_WR_ERR) +#define GENI_S_COMMON_IRQ_EN (0x3eu | 0x3e00u | (1u<<24) | (1u<<25)) +#define GENI_M_ERRORS (GENI_M_CMD_OVERRUN | GENI_M_ILLEGAL_CMD | GENI_M_CMD_FAILURE | GENI_M_RX_FIFO_RD_ERR | \ + GENI_M_RX_FIFO_WR_ERR | GENI_M_TX_FIFO_RD_ERR | GENI_M_TX_FIFO_WR_ERR) +// RX_FIFO_STATUS +#define GENI_RX_LAST (1u<<31) +#define GENI_RX_LAST_VALID_SHIFT 28 +#define GENI_RX_LAST_VALID_MASK 7u +#define GENI_RX_WC_MASK 0x1ffffffu +// Packing vector fields +#define GENI_PACK_START_SHIFT 5 +#define GENI_PACK_DIR_SHIFT 4 +#define GENI_PACK_LEN_SHIFT 1 +#define GENI_PACK_STOP (1u<<0) +#define GENI_PACK_VECTOR_SHIFT 10 +#define GENI_PACK_VECTORS 4 + +#define GENI_SPI_CPHA 0x01 +#define GENI_SPI_CPOL 0x02 +#define GENI_SPI_CS_HIGH 0x04 +#define GENI_SPI_LSB_FIRST 0x08 +#define GENI_SPI_LOOP 0x20 + +#define GENI_ERR_PARAM (-22) +#define GENI_ERR_PROTO (-71) +#define GENI_ERR_NODEV (-19) +#define GENI_ERR_TIMEOUT (-110) +#define GENI_ERR_IO (-5) +#define GENI_ERR_NOTSUP (-95) + +struct geni_io { + void *ctx; + uint32_t (*rd)(void *ctx, uint32_t reg); + void (*wr)(void *ctx, uint32_t reg, uint32_t value); + void (*delay_us)(void *ctx, unsigned us); +}; + +struct geni_spi_info { + uint32_t fw_revision, protocol, if_disable, hw_param0, geni_status; + uint32_t qup_hw_version; + unsigned tx_fifo_depth, fifo_width_bits; + int fifo_disabled; +}; + +struct geni_spi { + struct geni_io se; // serial engine registers + struct geni_io wrapper; // QUP wrapper registers (hardware version only) + struct geni_spi_info info; + unsigned oversampling, tx_wm; + unsigned cur_bpw; + unsigned cur_speed_hz; + uint32_t last_mode; + int last_cs; + unsigned poll_limit; // iterations of the completion poll before cancel/abort + int poll_only; // 1: switch the engine interrupt outputs off after init (completion polled); 2: leave them on + uint32_t snap_irq, snap_status, snap_txfs, snap_rxfs; // registers read just before a timed-out transfer is cancelled + uint32_t snap_dma_tx, snap_dma_rx, snap_dma_txin, snap_dma_rxin; + int snap_valid; + uint32_t last_tx_in, last_rx_in; // SE_DMA_{TX,RX}_LEN_IN after the latest DMA transfer: bytes the engine moved + // SE DMA mode (for engines whose FIFO interface is disabled and that cannot be re-enabled from the host): transfers go + // through two caller-provided buffers that the serial engine's DMA master can reach at tx_phys/rx_phys. The caller owns + // the translation (SMMU) and cache policy; the callbacks maintain the CPU caches around each transfer. + int use_dma; + struct geni_dma { + void *ctx; + void (*clean)(void *ctx, void *virt, size_t len); // write back CPU-dirty lines before the engine reads + void (*clean_inval)(void *ctx, void *virt, size_t len); // write back and drop before the engine writes + void (*inval)(void *ctx, void *virt, size_t len); // drop stale lines after the engine wrote + uint8_t *tx_virt, *rx_virt; + uint64_t tx_phys, rx_phys; + size_t size; + } dma; + int allow_fifo_disabled; // try FIFO mode even when FIFO_IF_DISABLE is set (the Linux fallback when no GPI channel exists) +}; + +// Read-only: protocol, FIFO geometry, FIFO-disable flag, version. Safe to run before any register write; only valid when +// the SE clocks are already running (the caller must know that). +int geni_spi_read_info(struct geni_spi *s); +// Select FIFO mode and program the controller defaults. Refuses engines that are not configured for SPI or whose FIFO +// interface is disabled (those need the GPI DMA path, which this driver does not implement). +int geni_spi_init(struct geni_spi *s); +int geni_spi_setup(struct geni_spi *s, unsigned mode, unsigned chip_select); +// Program the serial clock: divider from the SE source clock (sclk_hz) and the requested rate; clk_sel is the source index. +int geni_spi_set_clock(struct geni_spi *s, unsigned speed_hz, unsigned sclk_hz, unsigned clk_sel); +// With use_dma set the transfer runs through the SE DMA engine. +// One full-duplex (rx!=NULL, tx!=NULL), TX-only or RX-only transfer of len bytes (8 bits per word), polled. +int geni_spi_transfer(struct geni_spi *s, const uint8_t *tx, uint8_t *rx, size_t len, int keep_cs); +#endif diff --git a/drivers/NativeGeniSpi/LICENSE b/drivers/NativeGeniSpi/LICENSE new file mode 100644 index 0000000..cd608b2 --- /dev/null +++ b/drivers/NativeGeniSpi/LICENSE @@ -0,0 +1,34 @@ +Copyright (c) 2026, NativeGeniSpi contributors. All rights reserved. + +SPDX-License-Identifier: BSD-2-Clause-Patent + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the +following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following + disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided with the distribution. + +Subject to the terms and conditions of this license, each copyright holder and contributor hereby grants to those +receiving rights under this license a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable +(except for failure to satisfy the conditions of this license) patent license to make, have made, use, offer to sell, +sell, import, and otherwise transfer this software, where such license applies only to those patent claims, already +acquired or hereafter acquired, licensable by such copyright holder or contributor that are necessarily infringed by: +(a) their Contribution(s) (the licensed copyrights of copyright holders and non-copyrightable additions of +contributors, in source or binary form) alone; or (b) combination of their Contribution(s) with the work of +authorship to which such Contribution(s) was added by such copyright holder or contributor, if, at the time the +Contribution is added, such addition causes such combination to be necessarily infringed. The patent license shall not +apply to any other combinations which include the Contribution. Except as expressly stated above, no rights or +licenses from any copyright holder or contributor is granted under this license, whether expressly, by implication, +estoppel or otherwise. + +DISCLAIMER + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, +INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/drivers/NativeGeniSpi/Test/geni-spi.c b/drivers/NativeGeniSpi/Test/geni-spi.c new file mode 100644 index 0000000..7fd0b80 --- /dev/null +++ b/drivers/NativeGeniSpi/Test/geni-spi.c @@ -0,0 +1,294 @@ +// SPDX-License-Identifier: BSD-2-Clause-Patent +// Host test: GeniSpi.c against a register model of a GENI serial engine in FIFO mode. The model is derived from the +// documented register behaviour; it is not hardware. +#include "../GeniSpi.h" +#include +#include +#include + +#define FAIL(...) do { printf("FAIL %s:%d: ", __FILE__, __LINE__); printf(__VA_ARGS__); printf("\n"); exit(1); } while (0) +#define CHECK(c) do { if (!(c)) FAIL("%s", #c); } while (0) + +#define DEPTH 16 +struct sim { + uint32_t reg[0x1000 / 4]; + uint32_t wrap_ver, hw_param0, fw_rev, if_disable; + // DMA mode + uint32_t dma_tx_len, dma_rx_len, dma_tx_stat, dma_rx_stat; int dma_tx_set, dma_rx_set; + unsigned fsm_rst; + uint32_t txq[DEPTH + 8]; unsigned txn; + uint32_t rxq[4096]; unsigned rxh, rxt; // RX words + int active, tx_ops, rx_ops; + size_t tx_bytes_left, rx_bytes_left, tx_total, rx_total; + uint32_t cur_word; unsigned cur_bytes; // RX word being assembled + unsigned last_valid; + uint32_t irq; + int stuck, inject_error; + unsigned long writes_cancel, writes_abort; + uint8_t seen_tx[8192]; size_t seen_n; + uint32_t last_cmd0; + int max_txn; +}; + +static uint8_t slave(uint8_t in, size_t i) { return (uint8_t)((in ^ 0x5a) + (uint8_t)i); } + +static void finish_rx_word(struct sim *m) { m->rxq[m->rxt++ % 4096] = m->cur_word; m->cur_word = 0; m->cur_bytes = 0; } + +static void tick(struct sim *m) { + if (!m->active || m->stuck) return; + if (m->inject_error) { m->irq |= GENI_M_CMD_OVERRUN; return; } + // The slave shifts up to 8 bytes per poll. + for (int k = 0; k < 8; k++) { + size_t pos = m->tx_total ? m->tx_total - m->tx_bytes_left : m->rx_total - m->rx_bytes_left; + uint8_t in = 0; + if (m->tx_ops) { + if (m->tx_bytes_left == 0) break; + if (m->txn == 0) break; // underrun: wait for firmware + in = (uint8_t)(m->txq[0] >> 0); + // pop one byte from the packed word + uint8_t b = (uint8_t)(m->txq[0] & 0xff); + in = b; + m->txq[0] >>= 8; + m->tx_bytes_left--; + m->seen_tx[m->seen_n++] = b; + static unsigned used; // bytes consumed from txq[0] + used = (used + 1) % 4; + if (used == 0 || m->tx_bytes_left == 0) { + for (unsigned i = 1; i < m->txn; i++) m->txq[i - 1] = m->txq[i]; + m->txn--; + used = 0; + } + } else { + if (m->rx_bytes_left == 0) break; + } + if (m->rx_ops) { + if (m->rx_bytes_left == 0) break; + uint8_t out = slave(in, pos); + m->cur_word |= (uint32_t)out << (8 * m->cur_bytes); + m->cur_bytes++; + m->rx_bytes_left--; + if (m->cur_bytes == 4) finish_rx_word(m); + if (m->rx_bytes_left == 0 && m->cur_bytes) { m->last_valid = m->cur_bytes; finish_rx_word(m); m->irq |= GENI_M_RX_FIFO_LAST; } + else if (m->rx_bytes_left == 0) { m->last_valid = 0; m->irq |= GENI_M_RX_FIFO_LAST; } + } + } + unsigned rxwords = m->rxt - m->rxh; + if (rxwords >= m->reg[SE_GENI_RX_WATERMARK_REG / 4] && m->reg[SE_GENI_RX_WATERMARK_REG / 4]) m->irq |= GENI_M_RX_FIFO_WATERMARK; + if (m->tx_ops && m->reg[SE_GENI_TX_WATERMARK_REG / 4] && m->txn <= m->reg[SE_GENI_TX_WATERMARK_REG / 4]) + m->irq |= GENI_M_TX_FIFO_WATERMARK; + if ((!m->tx_ops || m->tx_bytes_left == 0) && (!m->rx_ops || m->rx_bytes_left == 0)) { + m->irq |= GENI_M_CMD_DONE; + m->active = 0; + } +} + +// DMA mode: when the command and every requested direction's LEN register have been written, the slave exchanges the bytes +// between the buffers the pointer registers name (the test uses host addresses as bus addresses). +static void dma_maybe_run(struct sim *m) { + if (!m->active || !(m->reg[SE_GENI_DMA_MODE_EN / 4] & 1)) return; + if ((m->tx_ops && !m->dma_tx_set) || (m->rx_ops && !m->dma_rx_set)) return; + if (m->stuck) return; + if (m->inject_error) { if (m->tx_ops) m->dma_tx_stat |= 4; if (m->rx_ops) m->dma_rx_stat |= 4; return; } + uint8_t *txp = (uint8_t *)(uintptr_t)(((uint64_t)m->reg[SE_DMA_TX_PTR_H / 4] << 32) | m->reg[SE_DMA_TX_PTR_L / 4]); + uint8_t *rxp = (uint8_t *)(uintptr_t)(((uint64_t)m->reg[SE_DMA_RX_PTR_H / 4] << 32) | m->reg[SE_DMA_RX_PTR_L / 4]); + size_t n = m->tx_ops ? m->dma_tx_len : m->dma_rx_len; + CHECK(!m->tx_ops || !m->rx_ops || m->dma_tx_len == m->dma_rx_len); + CHECK(!m->tx_ops || m->reg[SE_DMA_TX_ATTR / 4] == 1); + for (size_t i = 0; i < n; i++) { + uint8_t in = m->tx_ops ? txp[i] : 0; + if (m->tx_ops) m->seen_tx[m->seen_n++] = in; + if (m->rx_ops) rxp[i] = slave(in, i); + } + if (m->tx_ops) m->dma_tx_stat |= 1; + if (m->rx_ops) m->dma_rx_stat |= 1; + m->active = 0; +} + +static uint32_t se_rd(void *ctx, uint32_t r) { + struct sim *m = ctx; + switch (r) { + case GENI_FW_REVISION_RO: return m->fw_rev; + case GENI_IF_DISABLE_RO: return m->if_disable; + case SE_DMA_TX_IRQ_STAT: return m->dma_tx_stat; + case SE_DMA_RX_IRQ_STAT: return m->dma_rx_stat; + case SE_HW_PARAM_0: return m->hw_param0; + case SE_GENI_M_IRQ_STATUS: tick(m); return m->irq; + case SE_GENI_RX_FIFO_STATUS: { + unsigned words = m->rxt - m->rxh; + return words | ((m->irq & GENI_M_RX_FIFO_LAST) && words <= 1 ? GENI_RX_LAST | (m->last_valid << GENI_RX_LAST_VALID_SHIFT) : 0); + } + case SE_GENI_RX_FIFOn: return m->rxh < m->rxt ? m->rxq[m->rxh++ % 4096] : 0xdeadbeef; + default: return m->reg[r / 4]; + } +} + +static void se_wr(void *ctx, uint32_t r, uint32_t v) { + struct sim *m = ctx; + switch (r) { + case SE_GENI_M_IRQ_CLEAR: m->irq &= ~v; return; + case SE_DMA_TX_IRQ_CLR: m->dma_tx_stat &= ~v; return; + case SE_DMA_RX_IRQ_CLR: m->dma_rx_stat &= ~v; return; + case SE_DMA_TX_FSM_RST: case SE_DMA_RX_FSM_RST: m->fsm_rst++; return; + case SE_DMA_TX_LEN: m->dma_tx_len = v; m->dma_tx_set = 1; m->reg[r / 4] = v; dma_maybe_run(m); return; + case SE_DMA_RX_LEN: m->dma_rx_len = v; m->dma_rx_set = 1; m->reg[r / 4] = v; dma_maybe_run(m); return; + case SE_GENI_TX_FIFOn: + if (m->txn >= DEPTH) { m->irq |= GENI_M_TX_FIFO_WR_ERR; return; } + m->txq[m->txn++] = v; + if ((int)m->txn > m->max_txn) m->max_txn = (int)m->txn; + return; + case SE_GENI_M_CMD0: { + m->last_cmd0 = v; + uint32_t op = v >> GENI_M_OPCODE_SHIFT; + CHECK(op == 1 || op == 2 || op == GENI_M_CMD_TX_RX); + m->tx_ops = (op == 1 || op == GENI_M_CMD_TX_RX); + m->rx_ops = (op == 2 || op == GENI_M_CMD_TX_RX); + m->tx_total = m->tx_bytes_left = m->tx_ops ? m->reg[SE_SPI_TX_TRANS_LEN / 4] : 0; + m->rx_total = m->rx_bytes_left = m->rx_ops ? m->reg[SE_SPI_RX_TRANS_LEN / 4] : 0; + m->active = 1; m->cur_word = 0; m->cur_bytes = 0; m->rxh = m->rxt = 0; m->txn = 0; m->seen_n = 0; + m->dma_tx_set = m->dma_rx_set = 0; + return; + } + case SE_GENI_M_CMD_CTRL_REG: + if (v & GENI_M_CMD_CANCEL) { m->writes_cancel++; if (!m->stuck) m->irq |= GENI_M_CMD_CANCEL_DONE; } + if (v & GENI_M_CMD_ABORT) { m->writes_abort++; m->irq |= GENI_M_CMD_ABORT_DONE; m->active = 0; } + return; + } + m->reg[r / 4] = v; +} + +static uint32_t wr_rd(void *ctx, uint32_t r) { struct sim *m = ctx; CHECK(r == QUPV3_HW_VER_REG); return m->wrap_ver; } +static void wr_wr(void *ctx, uint32_t r, uint32_t v) { (void)ctx; (void)r; (void)v; FAIL("wrapper write"); } + +static void make(struct sim *m, struct geni_spi *s) { + memset(m, 0, sizeof *m); + m->wrap_ver = 0x40010000u; // major 4: 8-bit FIFO depth field + m->hw_param0 = (32u << 24) | ((uint32_t)DEPTH << 16); + m->fw_rev = GENI_PROTO_SPI << GENI_FW_PROTO_SHIFT; + memset(s, 0, sizeof *s); + s->se = (struct geni_io){m, se_rd, se_wr, 0}; + s->wrapper = (struct geni_io){m, wr_rd, wr_wr, 0}; + s->poll_limit = 2000; +} + +static struct { unsigned clean, clean_inval, inval; } cache; +static void clean(void *c, void *v, size_t l) { (void)c; (void)v; (void)l; cache.clean++; } +static void clean_inval(void *c, void *v, size_t l) { (void)c; (void)v; (void)l; cache.clean_inval++; } +static void inval(void *c, void *v, size_t l) { (void)c; (void)v; (void)l; cache.inval++; } + +static void fill(uint8_t *b, size_t n, unsigned seed) { for (size_t i = 0; i < n; i++) b[i] = (uint8_t)(seed + i * 7 + (i >> 3)); } + +int main(void) { + struct sim m; struct geni_spi s; + // protocol checks + make(&m, &s); m.fw_rev = 2u << GENI_FW_PROTO_SHIFT; CHECK(geni_spi_init(&s) == GENI_ERR_PROTO); + make(&m, &s); m.if_disable = 1; CHECK(geni_spi_init(&s) == GENI_ERR_NOTSUP); + make(&m, &s); m.if_disable = 1; s.allow_fifo_disabled = 1; CHECK(geni_spi_init(&s) == 0 && s.info.fifo_disabled); + make(&m, &s); m.hw_param0 = (16u << 24) | (16u << 16); CHECK(geni_spi_init(&s) == GENI_ERR_NODEV); + make(&m, &s); CHECK(geni_spi_init(&s) == 0); + CHECK(s.info.tx_fifo_depth == DEPTH && s.info.fifo_width_bits == 32 && s.oversampling == 1); + CHECK(m.reg[SE_GENI_RX_WATERMARK_REG / 4] == DEPTH - 3 && m.reg[SE_GENI_RX_RFR_WATERMARK_REG / 4] == DEPTH - 2); + CHECK(m.reg[SE_GENI_DMA_MODE_EN / 4] == 0 && (m.reg[SE_GENI_CGC_CTRL / 4] & 0x7f) == 0x7f); + make(&m, &s); s.poll_only = 1; CHECK(geni_spi_init(&s) == 0 && m.reg[SE_IRQ_EN / 4] == 0); + // old QUP version: 6-bit depth field, oversampling stays 1 except 1.0 + make(&m, &s); m.wrap_ver = 0x10000000u; m.hw_param0 = (32u << 24) | (0x3fu << 16) | (0xc0u << 16); CHECK(geni_spi_init(&s) == 0); + CHECK(s.info.tx_fifo_depth == 0x3f && s.oversampling == 2); + + make(&m, &s); CHECK(geni_spi_init(&s) == 0); + // clock: 100 MHz source, 19.2 MHz -> divider 6 + CHECK(geni_spi_set_clock(&s, 19200000, 100000000, 0) == 0); + CHECK(m.reg[GENI_SER_M_CLK_CFG / 4] == ((6u << 4) | 1) && m.reg[SE_GENI_CLK_SEL / 4] == 0); + CHECK(geni_spi_set_clock(&s, 1, 100000000, 0) == GENI_ERR_PARAM); + CHECK(geni_spi_set_clock(&s, 1000000, 100000000, 8) == GENI_ERR_PARAM); + CHECK(geni_spi_set_clock(&s, 19200000, 100000000, 0) == 0); + // mode 0, cs 0 then mode 3, cs 1 + CHECK(geni_spi_setup(&s, 0, 0) == 0); + CHECK(m.reg[SE_SPI_CPHA / 4] == 0 && m.reg[SE_SPI_CPOL / 4] == 0); + CHECK(geni_spi_setup(&s, GENI_SPI_CPHA | GENI_SPI_CPOL | GENI_SPI_CS_HIGH, 1) == 0); + CHECK(m.reg[SE_SPI_CPHA / 4] == 1 && m.reg[SE_SPI_CPOL / 4] == 4 && m.reg[SE_SPI_DEMUX_SEL / 4] == 1 && m.reg[SE_SPI_DEMUX_OUTPUT_INV / 4] == 2); + CHECK(geni_spi_setup(&s, 0, 0) == 0); + CHECK(m.reg[SE_SPI_CPHA / 4] == 0 && m.reg[SE_SPI_CPOL / 4] == 0 && m.reg[SE_SPI_DEMUX_OUTPUT_INV / 4] == 0); + CHECK(geni_spi_setup(&s, 0, 16) == GENI_ERR_PARAM); + + static uint8_t tx[8192], rx[8192]; + size_t lens[] = {1, 2, 3, 4, 5, 7, 8, 9, 15, 16, 17, 60, 63, 64, 65, 67, 100, 256, 1000, 5417 + 8}; + for (size_t li = 0; li < sizeof lens / sizeof *lens; li++) { + size_t n = lens[li]; + fill(tx, n, (unsigned)li); memset(rx, 0, sizeof rx); + int r = geni_spi_transfer(&s, tx, rx, n, 0); + if (r) FAIL("full duplex len %zu -> %d", n, r); + CHECK(m.last_cmd0 >> GENI_M_OPCODE_SHIFT == GENI_M_CMD_TX_RX); + CHECK((m.last_cmd0 & GENI_M_PARAMS_MASK) == 0); + CHECK(m.seen_n == n && memcmp(m.seen_tx, tx, n) == 0); + for (size_t i = 0; i < n; i++) if (rx[i] != slave(tx[i], i)) FAIL("rx mismatch len %zu at %zu", n, i); + CHECK(m.max_txn <= DEPTH); + CHECK(m.reg[SE_SPI_TX_TRANS_LEN / 4] == n && m.reg[SE_SPI_RX_TRANS_LEN / 4] == n); + } + // keep_cs sets FRAGMENTATION + fill(tx, 4, 9); CHECK(geni_spi_transfer(&s, tx, rx, 4, 1) == 0); + CHECK((m.last_cmd0 & GENI_M_PARAM_FRAGMENTATION) != 0); + // TX only + fill(tx, 129, 3); CHECK(geni_spi_transfer(&s, tx, 0, 129, 0) == 0); + CHECK(m.last_cmd0 >> GENI_M_OPCODE_SHIFT == 1 && m.seen_n == 129 && memcmp(m.seen_tx, tx, 129) == 0); + // RX only (slave answers with slave(0, i)) + memset(rx, 0, 70); CHECK(geni_spi_transfer(&s, 0, rx, 70, 0) == 0); + CHECK(m.last_cmd0 >> GENI_M_OPCODE_SHIFT == 2); + for (size_t i = 0; i < 70; i++) CHECK(rx[i] == slave(0, i)); + // packing registers for 8-bit words, MSB first, four per FIFO word + CHECK(m.reg[SE_GENI_TX_PACKING_CFG0 / 4] == 522494u && m.reg[SE_GENI_TX_PACKING_CFG1 / 4] == 1048318u); + CHECK(m.reg[SE_GENI_RX_PACKING_CFG0 / 4] == 522494u && m.reg[SE_GENI_RX_PACKING_CFG1 / 4] == 1048318u); + CHECK(m.reg[SE_SPI_WORD_LEN / 4] == 4 && m.reg[SE_GENI_BYTE_GRAN / 4] == 0); + // argument errors + CHECK(geni_spi_transfer(&s, tx, rx, 0, 0) == GENI_ERR_PARAM); + CHECK(geni_spi_transfer(&s, 0, 0, 4, 0) == GENI_ERR_PARAM); + // error IRQ + m.inject_error = 1; CHECK(geni_spi_transfer(&s, tx, rx, 8, 0) == GENI_ERR_IO); m.inject_error = 0; + // timeout -> cancel then abort + m.stuck = 1; m.writes_cancel = m.writes_abort = 0; + CHECK(geni_spi_transfer(&s, tx, rx, 8, 0) == GENI_ERR_TIMEOUT); + CHECK(s.snap_valid); + CHECK(m.writes_cancel == 1 && m.writes_abort == 1); + m.stuck = 0; + fill(tx, 12, 1); CHECK(geni_spi_transfer(&s, tx, rx, 12, 0) == 0); + // SE DMA mode (FIFO interface disabled by the firmware): buffers, cache callbacks, errors and timeout + { + static uint8_t dtx[4096], drx[4096]; + make(&m, &s); m.if_disable = 1; + CHECK(geni_spi_init(&s) == GENI_ERR_NOTSUP); // no DMA configured + make(&m, &s); m.if_disable = 1; s.use_dma = 1; + CHECK(geni_spi_init(&s) == GENI_ERR_PARAM); // buffers required + make(&m, &s); m.if_disable = 1; s.use_dma = 1; + s.dma = (struct geni_dma){0, clean, clean_inval, inval, dtx, drx, (uint64_t)(uintptr_t)dtx, (uint64_t)(uintptr_t)drx, sizeof dtx}; + CHECK(geni_spi_init(&s) == 0); + CHECK(m.reg[SE_GENI_DMA_MODE_EN / 4] == 1); + CHECK((m.reg[SE_GENI_M_IRQ_EN / 4] & (GENI_M_CMD_DONE | GENI_M_TX_FIFO_WATERMARK | GENI_M_RX_FIFO_WATERMARK | GENI_M_RX_FIFO_LAST)) == 0); + CHECK(geni_spi_set_clock(&s, 19200000, 100000000, 0) == 0 && geni_spi_setup(&s, 0, 0) == 0); + size_t dl[] = {1, 3, 4, 5, 63, 64, 100, 1000, 4096}; + for (size_t li = 0; li < sizeof dl / sizeof *dl; li++) { + size_t n = dl[li]; + fill(tx, n, (unsigned)li + 40); memset(rx, 0, sizeof rx); + int r = geni_spi_transfer(&s, tx, rx, n, li & 1); + if (r) FAIL("dma full duplex len %zu -> %d", n, r); + CHECK(m.last_cmd0 >> GENI_M_OPCODE_SHIFT == GENI_M_CMD_TX_RX); + CHECK(!!(m.last_cmd0 & GENI_M_PARAM_FRAGMENTATION) == (int)(li & 1)); + CHECK(m.seen_n == n && memcmp(m.seen_tx, tx, n) == 0); + for (size_t i = 0; i < n; i++) if (rx[i] != slave(tx[i], i)) FAIL("dma rx mismatch len %zu at %zu", n, i); + CHECK(m.dma_tx_stat == 0 && m.dma_rx_stat == 0); // status cleared after each transfer + } + CHECK(cache.clean == sizeof dl / sizeof *dl && cache.clean_inval == cache.clean && cache.inval == cache.clean); + // TX only and RX only + fill(tx, 33, 5); CHECK(geni_spi_transfer(&s, tx, 0, 33, 0) == 0 && m.last_cmd0 >> GENI_M_OPCODE_SHIFT == 1 && memcmp(m.seen_tx, tx, 33) == 0); + memset(rx, 0, 20); CHECK(geni_spi_transfer(&s, 0, rx, 20, 0) == 0 && m.last_cmd0 >> GENI_M_OPCODE_SHIFT == 2); + for (size_t i = 0; i < 20; i++) CHECK(rx[i] == slave(0, i)); + CHECK(geni_spi_transfer(&s, tx, rx, sizeof dtx + 1, 0) == GENI_ERR_PARAM); // larger than the DMA buffer + // errors: DMA bus error status, then a stuck engine (cancel/abort and FSM resets, snapshot kept) + m.inject_error = 1; CHECK(geni_spi_transfer(&s, tx, rx, 8, 0) == GENI_ERR_IO); m.inject_error = 0; + m.dma_tx_stat = m.dma_rx_stat = 0; + m.stuck = 1; m.writes_cancel = m.writes_abort = 0; m.fsm_rst = 0; s.snap_valid = 0; + CHECK(geni_spi_transfer(&s, tx, rx, 8, 0) == GENI_ERR_TIMEOUT && s.snap_valid); + CHECK(m.writes_cancel == 1 && m.writes_abort == 1 && m.fsm_rst == 2); + m.stuck = 0; + fill(tx, 12, 1); CHECK(geni_spi_transfer(&s, tx, rx, 12, 0) == 0); + } + printf("PASS: GeniSpi init, clock, mode, packing, PIO and SE DMA transfers (%zu PIO lengths) against the register model\n", sizeof lens / sizeof *lens); + return 0; +} diff --git a/drivers/NativeHidInput/LICENSE b/drivers/NativeHidInput/LICENSE new file mode 100644 index 0000000..72878ee --- /dev/null +++ b/drivers/NativeHidInput/LICENSE @@ -0,0 +1,51 @@ +Copyright (c) 2026, NativeHidInput contributors. All rights reserved. + +SPDX-License-Identifier: BSD-2-Clause-Patent + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +Subject to the terms and conditions of this license, each copyright holder +and contributor hereby grants to those receiving rights under this license +a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable +(except for failure to satisfy the conditions of this license) patent +license to make, have made, use, offer to sell, sell, import, and otherwise +transfer this software, where such license applies only to those patent +claims, already acquired or hereafter acquired, licensable by such copyright +holder or contributor that are necessarily infringed by: + +(a) their Contribution(s) (the licensed copyrights of copyright holders and + non-copyrightable additions of contributors, in source or binary form) + alone; or + +(b) combination of their Contribution(s) with the work of authorship to + which such Contribution(s) was added by such copyright holder or + contributor, if, at the time the Contribution is added, such addition + causes such combination to be necessarily infringed. The patent license + shall not apply to any other combinations which include the + Contribution. + +Except as expressly stated above, no rights or licenses from any copyright +holder or contributor is granted under this license, whether expressly, by +implication, estoppel or otherwise. + +DISCLAIMER + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. diff --git a/drivers/NativeHidInput/NativeHidInput.cpp b/drivers/NativeHidInput/NativeHidInput.cpp new file mode 100644 index 0000000..e0bbe9a --- /dev/null +++ b/drivers/NativeHidInput/NativeHidInput.cpp @@ -0,0 +1,192 @@ +// SPDX-License-Identifier: BSD-2-Clause-Patent +// Explicit offline keyboard/consumer input source. Uses the unchanged original +// HID parser and event dispatch; no user-client entitlement is substituted. +#include "OriginalHidDeviceABI.h" +#include +#include +#include +#include +#include +extern "C" void native_hid_input_observed(unsigned,int); +extern "C" void native_hid_live_checkpoint(IOService *); +extern "C" bool native_hid_input_requested(void); +// Optional touch digitizer fed by the board touch probe (kits/sheng/TouchProbe.cpp): finger set published with a seqlock. +extern "C" { +extern volatile uint32_t native_touch_seq, native_touch_count, native_touch_hid_reports; +extern volatile int native_touch_xs[12], native_touch_ys[12], native_touch_ids[12]; +} +// Keyboard Application collection ensures the original generic keyboard +// personality can match. Report 2 carries Consumer Menu, not a claimed result +// that the system accepted Home. Report 1 is an ordinary boot keyboard report. +static const unsigned char descriptor[]={ + 0x05,0x01,0x09,0x06,0xa1,0x01,0x85,0x01, + 0x05,0x07,0x19,0xe0,0x29,0xe7,0x15,0x00,0x25,0x01, + 0x75,0x01,0x95,0x08,0x81,0x02, + 0x75,0x08,0x95,0x01,0x81,0x03, + 0x19,0x00,0x29,0x65,0x15,0x00,0x25,0x65, + 0x75,0x08,0x95,0x06,0x81,0x00, + 0x85,0x02,0x05,0x0c,0x09,0x40,0x15,0x00,0x25,0x01, + 0x75,0x01,0x95,0x01,0x81,0x02, + 0x75,0x07,0x95,0x01,0x81,0x03,0xc0 +}; +// Report 3: ten-finger Touch Screen collection (Tip Switch, Untouch (set when the tip is up), Contact Identifier, X, Y per finger, then Contact Count). +// Coordinates are normalised to 0..32767 on both axes by the producer. +#define FINGER_DESC \ + 0x09,0x22,0xa1,0x02, 0x09,0x42,0x15,0x00,0x25,0x01,0x75,0x01,0x95,0x01,0x81,0x02, 0x09,0x47,0x81,0x02, 0x75,0x06,0x95,0x01,0x81,0x03, \ + 0x09,0x51,0x75,0x08,0x95,0x01,0x15,0x00,0x26,0xff,0x00,0x81,0x02, \ + 0x05,0x01,0x09,0x30,0x15,0x00,0x26,0xff,0x7f,0x75,0x10,0x95,0x01,0x81,0x02, 0x09,0x31,0x81,0x02, 0x05,0x0d, 0xc0, +static const unsigned char touchDescriptor[]={ + 0x05,0x0d,0x09,0x04,0xa1,0x01,0x85,0x03, + FINGER_DESC FINGER_DESC FINGER_DESC FINGER_DESC FINGER_DESC + FINGER_DESC FINGER_DESC FINGER_DESC FINGER_DESC FINGER_DESC + 0x09,0x54,0x75,0x08,0x95,0x01,0x15,0x00,0x26,0xff,0x00,0x81,0x02,0xc0 +}; +#define TOUCH_FINGERS 10 +class NativeHidInput : public IOHIDDevice { + OSDeclareDefaultStructors(NativeHidInput) + IOWorkLoop *inputLoop=nullptr; + IOTimerEventSource *timer=nullptr; + IOTimerEventSource *touchTimer=nullptr; + bool down=false; + bool touchEnabled=false; + // The digitizer is a separate service from the keyboard: backboardd matches digitizers by primary usage page/usage + // (Digitizer/Touch Screen) and the Built-In flag, which a combined keyboard-first descriptor would not provide. + bool touchOnly=false; + bool eventDriverFound=false; + uint32_t lastSeq=0; + // Fingers sent as down in the previous report; a vanished identifier gets one explicit tip-up report. + int prevIds[TOUCH_FINGERS]={-1,-1,-1,-1,-1,-1,-1,-1,-1,-1}; + int prevX[TOUCH_FINGERS]={},prevY[TOUCH_FINGERS]={}; + static unsigned transformCoordinate(int value,int maximum) { + if(value<0)value=0;if(value>=maximum)value=maximum-1; + return (unsigned)((long long)value*32767/(maximum-1)); + } + static void touchTick(OSObject *owner,IOTimerEventSource *source) { + NativeHidInput *self=OSDynamicCast(NativeHidInput,owner);if(!self)return; + source->setTimeoutMS(8); + if(!self->hasEventDriverCached())return; + uint32_t before=native_touch_seq; + if((before&1) || before==self->lastSeq)return; + __asm__ volatile("dmb ish":::"memory"); + unsigned count=native_touch_count;if(count>TOUCH_FINGERS)count=TOUCH_FINGERS; + int ids[TOUCH_FINGERS],xs[TOUCH_FINGERS],ys[TOUCH_FINGERS]; + for(unsigned i=0;ilastSeq=before; + // Sizes follow the axis maxima of the touch core: 30480 long, 20320 short, mapped to the same 0..32767 range. + unsigned char report[1+TOUCH_FINGERS*6+1]={3}; + unsigned slot=0; + for(unsigned i=0;i>8);f[4]=(unsigned char)y;f[5]=(unsigned char)(y>>8); + self->prevIds[slot]=ids[i];self->prevX[slot]=(int)x;self->prevY[slot]=(int)y; + } + unsigned downCount=slot; + // Identifiers that were down and are gone: tip up at the last known position. + for(unsigned j=downCount;jprevIds[j]<0)continue; + if(slotprevIds[j]&0xff); + f[2]=(unsigned char)self->prevX[j];f[3]=(unsigned char)(self->prevX[j]>>8); + f[4]=(unsigned char)self->prevY[j];f[5]=(unsigned char)(self->prevY[j]>>8); + ++slot; + } + self->prevIds[j]=-1; + } + if(!slot)return; + report[1+TOUCH_FINGERS*6]=(unsigned char)slot; + IOBufferMemoryDescriptor *buffer=IOBufferMemoryDescriptor::withBytes(report,sizeof(report),kIODirectionNone); + IOReturn result=buffer?self->handleReport(buffer,0,0):kIOReturnNoMemory; + OSSafeReleaseNULL(buffer); + native_hid_input_observed(6,result); + native_touch_hid_reports=native_touch_hid_reports+1; + } + bool hasEventDriverCached() { + if(!eventDriverFound)eventDriverFound=hasEventDriver(this,4); + return eventDriverFound; + } + static bool hasEventDriver(IORegistryEntry *parent,unsigned depth) { + if(!depth)return false; + OSIterator *it=parent->getChildIterator(gIOServicePlane);if(!it)return false; + bool found=false; + while(OSObject *child=it->getNextObject()) { + IORegistryEntry *entry=OSDynamicCast(IORegistryEntry,child); + if(entry && (entry->metaCast("IOHIDEventDriver") || hasEventDriver(entry,depth-1))) {found=true;break;} + } + it->release();return found; + } + static void tick(OSObject *owner,IOTimerEventSource *source) { + NativeHidInput *self=OSDynamicCast(NativeHidInput,owner);if(!self)return; + native_hid_live_checkpoint(self); + if(self->touchOnly){source->setTimeoutMS(1000);return;} + if(!self->down && !hasEventDriver(self,4)) { + native_hid_input_observed(3,0); + source->setTimeoutMS(1000); + return; + } + native_hid_input_observed(3,1); + if(!self->down && !native_hid_input_requested()) {source->setTimeoutMS(1000);return;} + unsigned char report[]={2,static_cast(self->down?0:1)}; + IOBufferMemoryDescriptor *buffer=IOBufferMemoryDescriptor::withBytes(report,sizeof(report),kIODirectionNone); + IOReturn result=buffer?self->handleReport(buffer,0,0):kIOReturnNoMemory; + OSSafeReleaseNULL(buffer); + native_hid_input_observed(self->down?5:4,result); + // Always attempt release, even if down returned an error: input must + // never be left pressed by an ambiguous report completion. + if(!self->down) {self->down=true;source->setTimeoutMS(100);} + else {self->down=false;source->setTimeoutMS(1000);} + } +public: + void setTouchOnly() {touchOnly=true;} + IOWorkLoop *getWorkLoop() const override {return inputLoop;} + OSString *newTransportString() const override {return OSString::withCString(touchOnly?"SPI":"Virtual");} + OSString *newProductString() const override {return OSString::withCString(touchOnly?"Offline touch digitizer":"Offline HID input probe");} + IOReturn newReportDescriptor(IOMemoryDescriptor **out) const override { + if(!out)return kIOReturnBadArgument; + unsigned char combined[sizeof(descriptor)+sizeof(touchDescriptor)]; + size_t size; + if(touchOnly){size=sizeof(touchDescriptor);memcpy(combined,touchDescriptor,size);} + else{ + size=sizeof(descriptor);memcpy(combined,descriptor,size); + // Default: one device, keyboard collection first, digitizer second (the arrangement whose event driver + // and user client attach on hardware); the separate digitizer device is the opt-in sheng-touch-split. + if(touchEnabled){memcpy(combined+size,touchDescriptor,sizeof(touchDescriptor));size+=sizeof(touchDescriptor);} + } + *out=IOBufferMemoryDescriptor::withBytes(combined,size,kIODirectionNone); + return *out?kIOReturnSuccess:kIOReturnNoMemory; + } + bool start(IOService *provider) override { + inputLoop=IOWorkLoop::workLoop();if(!inputLoop)return false; + // Request the normal generic HID event personality for this owned + // keyboard. This is a matching property, not a client entitlement. + if(!setProperty("HIDDefaultBehavior",true))return false; + unsigned touch=0; + touchEnabled=PE_parse_boot_argn("sheng-touch-hid",&touch,sizeof(touch)) && touch==1; + if(touchOnly||touchEnabled){setProperty("Built-In",true);setProperty("DisplayIntegrated",true);} + if(!IOHIDDevice::start(provider))return false; + timer=IOTimerEventSource::timerEventSource(this,tick); + if(!timer || inputLoop->addEventSource(timer)!=kIOReturnSuccess) { + OSSafeReleaseNULL(timer);IOHIDDevice::stop(provider);return false; + } + if(touchOnly||touchEnabled) { + touchTimer=IOTimerEventSource::timerEventSource(this,touchTick); + if(touchTimer && inputLoop->addEventSource(touchTimer)==kIOReturnSuccess)touchTimer->setTimeoutMS(1000); + else OSSafeReleaseNULL(touchTimer); + } + setProperty("OfflineInputProbe",true); + native_hid_input_observed(2,0); + timer->setTimeoutMS(1000);return true; + } + void stop(IOService *provider) override { + if(touchTimer){touchTimer->cancelTimeout();inputLoop->removeEventSource(touchTimer);OSSafeReleaseNULL(touchTimer);} + if(timer){timer->cancelTimeout();inputLoop->removeEventSource(timer);OSSafeReleaseNULL(timer);} + IOHIDDevice::stop(provider); + } + void free() override {OSSafeReleaseNULL(inputLoop);IOHIDDevice::free();} +}; +OSDefineMetaClassAndStructors(NativeHidInput,IOHIDDevice) +extern "C" IOService *native_hid_input_allocate(void) {return new NativeHidInput;} +extern "C" IOService *native_hid_touch_allocate(void) {NativeHidInput *device=new NativeHidInput;if(device)device->setTouchOnly();return device;} diff --git a/drivers/NativeKeyStoreEvents/LICENSE b/drivers/NativeKeyStoreEvents/LICENSE new file mode 100644 index 0000000..799f4cf --- /dev/null +++ b/drivers/NativeKeyStoreEvents/LICENSE @@ -0,0 +1,51 @@ +Copyright (c) 2026, NativeKeyStoreEvents contributors. All rights reserved. + +SPDX-License-Identifier: BSD-2-Clause-Patent + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +Subject to the terms and conditions of this license, each copyright holder +and contributor hereby grants to those receiving rights under this license +a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable +(except for failure to satisfy the conditions of this license) patent +license to make, have made, use, offer to sell, sell, import, and otherwise +transfer this software, where such license applies only to those patent +claims, already acquired or hereafter acquired, licensable by such copyright +holder or contributor that are necessarily infringed by: + +(a) their Contribution(s) (the licensed copyrights of copyright holders and + non-copyrightable additions of contributors, in source or binary form) + alone; or + +(b) combination of their Contribution(s) with the work of authorship to + which such Contribution(s) was added by such copyright holder or + contributor, if, at the time the Contribution is added, such addition + causes such combination to be necessarily infringed. The patent license + shall not apply to any other combinations which include the + Contribution. + +Except as expressly stated above, no rights or licenses from any copyright +holder or contributor is granted under this license, whether expressly, by +implication, estoppel or otherwise. + +DISCLAIMER + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. diff --git a/drivers/NativeKeyStoreEvents/NativeKeyStoreEvents.cpp b/drivers/NativeKeyStoreEvents/NativeKeyStoreEvents.cpp new file mode 100644 index 0000000..233d433 --- /dev/null +++ b/drivers/NativeKeyStoreEvents/NativeKeyStoreEvents.cpp @@ -0,0 +1,162 @@ +// SPDX-License-Identifier: BSD-2-Clause-Patent +// AppleKeyStore for an ephemeral offline fixture: a notification service (the public IOKit matching class used by +// MKBEventsRegister) and, only when sheng-aks=1 is set, a user client that answers the state queries of the unchanged +// libaks/MobileKeyBag truthfully for this environment: no keybag, no passcode, nothing locked. It provides no key, +// keybag, passcode or data-protection operation; every other selector fails with kIOReturnError exactly as an unreachable +// AppleKeyStore does. It protects nothing and must not be taken for secure storage. +// +// Answers (selectors and layouts read from the unchanged target's libaks/MobileKeyBag, see docs/touch-nt36532e.md; the set that is +// enough for the system daemons on iOS 16.7 was established on hardware by github.com/dipa-studio/ios-sweet, whose notes document it): +// 0, 4, 5, 9, 12, 13, 26, 27, 55, 101, 102 succeed, no outputs (open handshake, bag/assertion/notify bookkeeping) +// 14 aks_get_system scalar out: system keybag handle 0 (keybagd then needs no systembag.kb) +// 7 aks_get_lock_state(device) scalar out: bit 2 = unlocked since boot +// 17, 35 aks_get_(extended_)device_state struct out: DER SET{ SEQUENCE{ UTF8String key, INTEGER value } ... } +// keys ss sls sb sfa sgs sas sgpe srcd sms bh; ss = 4, sls = lock state (0 unlocked; 3 would be "no passcode"). +#include +#include +extern "C" void native_keyevents_open_refused(void); +extern "C" bool native_aks_state_enabled(void); +extern "C" uint32_t native_aks_lock_state(void); +extern "C" void native_aks_call(uint32_t selector, int32_t result); +extern "C" void native_aks_opened(void); + +namespace { +constexpr uint32_t kSelLockState = 7, kSelGetSystem = 14, kSelDeviceState = 17, kSelExtendedDeviceState = 35; +constexpr uint64_t kUnlockedSinceBoot = 4; +constexpr uint32_t kStateFlags = 4; +constexpr uint64_t kSystemHandle = 0; +constexpr IOReturn kAksError = (IOReturn)0xe00002bc; // what libaks reports itself when the connection is missing +constexpr IOReturn kAksUnsupported = (IOReturn)0xe00002c7; + +struct DerWriter { + uint8_t *p; size_t cap, n = 0; bool ok = true; + void put(uint8_t b) { if (n < cap) p[n++] = b; else ok = false; } + void len(size_t l) { if (l < 0x80) put((uint8_t)l); else { put(0x81); put((uint8_t)l); } } // entries stay below 256 bytes +}; +// Smallest two's-complement big-endian INTEGER content for a non-negative value. +size_t derIntegerContent(uint64_t v, uint8_t out[9]) { + uint8_t raw[8]; + for (int i = 0; i < 8; ++i) raw[i] = (uint8_t)(v >> (56 - 8 * i)); + size_t start = 0; + while (start < 7 && raw[start] == 0) ++start; + size_t n = 0; + if (raw[start] & 0x80) out[n++] = 0; + while (start < 8) out[n++] = raw[start++]; + return n; +} +size_t entrySize(const char *key, uint64_t value) { + uint8_t c[9]; size_t kl = 0; while (key[kl]) ++kl; + size_t body = (2 + kl) + (2 + derIntegerContent(value, c)); + return 2 + body; +} +} // namespace + +// DER SET of SEQUENCE{UTF8String, INTEGER}. Returns the length, or 0 when the buffer is too small. +extern "C" size_t native_aks_encode_state(uint8_t *out, size_t capacity, uint32_t lockState) { + struct Entry { const char *key; uint64_t value; }; + const Entry entries[] = {{"ss", kStateFlags}, {"sls", lockState}, {"sb", 0}, {"sfa", 0}, {"sgs", 0}, + {"sas", 0}, {"sgpe", 0}, {"srcd", 0}, {"sms", 0}, {"bh", 0}}; + size_t total = 0; + for (const Entry &e : entries) total += entrySize(e.key, e.value); + DerWriter w{out, capacity}; + w.put(0x31); w.len(total); + for (const Entry &e : entries) { + uint8_t c[9]; size_t kl = 0; while (e.key[kl]) ++kl; + size_t il = derIntegerContent(e.value, c); + w.put(0x30); w.len((2 + kl) + (2 + il)); + w.put(0x0c); w.put((uint8_t)kl); for (size_t i = 0; i < kl; ++i) w.put((uint8_t)e.key[i]); + w.put(0x02); w.put((uint8_t)il); for (size_t i = 0; i < il; ++i) w.put(c[i]); + } + return w.ok ? w.n : 0; +} + +class AppleKeyStoreUserClient final : public IOUserClient { + OSDeclareDefaultStructors(AppleKeyStoreUserClient) +public: + IOReturn clientClose() override { terminate(); return kIOReturnSuccess; } + IOReturn externalMethod(uint32_t selector, IOExternalMethodArguments *args, IOExternalMethodDispatch *, + OSObject *, void *) override { + IOReturn rc = kAksUnsupported; + if (!args) rc = kAksError; + else switch (selector) { + // Handshake right after IOServiceOpen (a failure makes libaks drop the connection and reopen on every call), and the + // bag/assertion/notification calls that need no answer in an environment without keybags. + case 0: case 4: case 5: case 9: case 12: case 13: case 26: case 27: case 55: case 101: case 102: + args->scalarOutputCount = 0; + rc = kIOReturnSuccess; + break; + case kSelGetSystem: + if (args->scalarOutputCount < 1 || !args->scalarOutput) { rc = kIOReturnBadArgument; break; } + args->scalarOutput[0] = kSystemHandle; + args->scalarOutputCount = 1; + rc = kIOReturnSuccess; + break; + case kSelLockState: + rc = (args->scalarInputCount == 1 && args->scalarOutputCount == 1 && args->scalarOutput) + ? kIOReturnSuccess : kIOReturnBadArgument; + if (rc == kIOReturnSuccess) args->scalarOutput[0] = kUnlockedSinceBoot; + break; + case kSelDeviceState: + case kSelExtendedDeviceState: { + if (args->scalarInputCount > 1 || args->scalarOutputCount != 0) { rc = kIOReturnBadArgument; break; } + uint8_t der[160]; + size_t n = native_aks_encode_state(der, sizeof der, native_aks_lock_state()); + if (!n) { rc = kIOReturnNoSpace; break; } + if (args->structureOutputDescriptor) { + if (n > args->structureOutputDescriptor->getLength()) { rc = kIOReturnNoSpace; break; } + args->structureOutputDescriptor->writeBytes(0, der, n); + args->structureOutputDescriptorSize = (uint32_t)n; + rc = kIOReturnSuccess; + } else if (args->structureOutput && n <= args->structureOutputSize) { + for (size_t k = 0; k < n; ++k) ((uint8_t *)args->structureOutput)[k] = der[k]; + args->structureOutputSize = (uint32_t)n; + rc = kIOReturnSuccess; + } else { + rc = kIOReturnNoSpace; + } + break; + } + default: break; + } + native_aks_call(selector, (int32_t)rc); + return rc; + } +}; +OSDefineMetaClassAndStructors(AppleKeyStoreUserClient, IOUserClient) + +class AppleKeyStore final : public IOService { + OSDeclareDefaultStructors(AppleKeyStore) + IOReturn open(task_t task, void *security, UInt32 type, OSDictionary *properties, IOUserClient **out) { + if (out) *out = nullptr; + if (!native_aks_state_enabled() || !out) { native_keyevents_open_refused(); return kIOReturnNotReady; } + AppleKeyStoreUserClient *client = new AppleKeyStoreUserClient; + if (!client) return kIOReturnNoMemory; + if (!client->initWithTask(task, security, type, properties) || !client->attach(this)) { + client->release(); return kIOReturnError; + } + if (!client->start(this)) { client->detach(this); client->release(); return kIOReturnError; } + native_aks_opened(); + *out = client; + return kIOReturnSuccess; + } +public: + bool start(IOService *provider) override { + if (!IOService::start(provider)) return false; + setName("Native keystore state service"); + if (!setProperty("NativeKeyStoreEvents", true) || + !setProperty("KeyOperationsAvailable", false)) { + IOService::stop(provider); + return false; + } + registerService(); + return true; + } + IOReturn newUserClient(task_t t, void *s, UInt32 type, OSDictionary *p, IOUserClient **out) override { + return open(t, s, type, p, out); + } + IOReturn newUserClient(task_t t, void *s, UInt32 type, IOUserClient **out) override { + return open(t, s, type, nullptr, out); + } +}; +OSDefineMetaClassAndStructors(AppleKeyStore, IOService) +extern "C" IOService *native_keyevents_allocate(void) { return new AppleKeyStore; } diff --git a/drivers/NativeKeyStoreEvents/test_closed_boundary.py b/drivers/NativeKeyStoreEvents/test_closed_boundary.py new file mode 100644 index 0000000..ac06d53 --- /dev/null +++ b/drivers/NativeKeyStoreEvents/test_closed_boundary.py @@ -0,0 +1,193 @@ +"""Compile the actual service against a narrow host lifecycle test double. + +Checks the open/answer boundaries, not real IOKit registration. Without sheng-aks every open is refused (closed boundary); +with it the user client answers the handshake/bookkeeping selectors, get_system (14), lock state (7) and device state (17, 35; decoded below the way libaks decodes it) and rejects the rest. +""" +from pathlib import Path +import subprocess +import tempfile + +root = Path(__file__).resolve().parent +with tempfile.TemporaryDirectory() as directory: + work = Path(directory) + (work / "IOKit").mkdir() + (work / "IOKit/IOService.h").write_text(r''' +#pragma once +#include +#include +using UInt32 = uint32_t; +using task_t = void *; +using IOReturn = int; +constexpr IOReturn kIOReturnSuccess = 0; +constexpr IOReturn kIOReturnNotReady = -7; +constexpr IOReturn kIOReturnBadArgument = -8; +constexpr IOReturn kIOReturnNoSpace = -9; +constexpr IOReturn kIOReturnNoMemory = -10; +constexpr IOReturn kIOReturnError = -11; +class OSDictionary {}; +class OSObject { public: virtual ~OSObject() = default; void release() { delete this; } }; +class IOMemoryDescriptor { +public: + uint8_t bytes[512]; size_t length = 512; + size_t getLength() const { return length; } + size_t writeBytes(size_t offset, const void *p, size_t n) { for (size_t i = 0; i < n; ++i) bytes[offset + i] = ((const uint8_t *)p)[i]; return n; } +}; +struct IOExternalMethodArguments { + const uint64_t *scalarInput; uint32_t scalarInputCount; uint64_t *scalarOutput; uint32_t scalarOutputCount; + void *structureOutput; uint32_t structureOutputSize; + IOMemoryDescriptor *structureOutputDescriptor = nullptr; uint32_t structureOutputDescriptorSize = 0; +}; +struct IOExternalMethodDispatch {}; +class IOService; +class IOUserClient : public OSObject { +public: + bool terminated = false, started = false; + virtual bool initWithTask(task_t, void *, UInt32, OSDictionary *) { return true; } + bool attach(IOService *) { return true; } + void detach(IOService *) {} + virtual bool start(IOService *) { started = true; return true; } + void terminate() { terminated = true; } + virtual IOReturn clientClose() { return kIOReturnSuccess; } + virtual IOReturn externalMethod(uint32_t, IOExternalMethodArguments *, IOExternalMethodDispatch * = nullptr, + OSObject * = nullptr, void * = nullptr) = 0; +}; +#define OSDeclareDefaultStructors(name) +#define OSDefineMetaClassAndStructors(name, base) +class IOService : public OSObject { +public: + bool baseOK = true; + unsigned properties = 0, failProperty = 0, publications = 0, stops = 0; + virtual bool start(IOService *) { return baseOK; } + virtual void stop(IOService *) { ++stops; } + void setName(const char *) {} + bool setProperty(const char *, bool) { return ++properties != failProperty; } + void registerService() { ++publications; } + virtual IOReturn newUserClient(task_t, void *, UInt32, OSDictionary *, IOUserClient **) = 0; + virtual IOReturn newUserClient(task_t, void *, UInt32, IOUserClient **) = 0; +}; +''') + (work / "IOKit/IOUserClient.h").write_text("#pragma once\n#include \n") + (work / "test.cpp").write_text(r''' +#include +#include +#include +#include +#include +static bool enabled; +static unsigned refused, opened, calls; +static uint32_t lockState; +extern "C" bool native_aks_state_enabled() { return enabled; } +extern "C" uint32_t native_aks_lock_state() { return lockState; } +extern "C" void native_keyevents_open_refused() { ++refused; } +extern "C" void native_aks_opened() { ++opened; } +extern "C" void native_aks_call(uint32_t, int32_t) { ++calls; } +#include "NativeKeyStoreEvents.cpp" + +// The decoder libaks uses (_der_dict_iterate/_der_get_number): SET { SEQUENCE { UTF8String key, INTEGER } ... }. +static bool decode(const uint8_t *p, size_t n, std::map &out) { + size_t i = 0; + auto len = [&](size_t &l) { if (i >= n) return false; uint8_t b = p[i++]; if (b < 0x80) { l = b; return true; } + if (b != 0x81 || i >= n) return false; l = p[i++]; return true; }; + if (n < 2 || p[i++] != 0x31) return false; + size_t total; if (!len(total) || i + total != n) return false; + while (i < n) { + if (p[i++] != 0x30) return false; + size_t sl; if (!len(sl) || i + sl > n) return false; + size_t end = i + sl; + if (p[i++] != 0x0c) return false; + size_t kl; if (!len(kl) || i + kl > end) return false; + std::string key((const char *)p + i, kl); i += kl; + if (p[i++] != 0x02) return false; + size_t il; if (!len(il) || i + il != end || il == 0 || il > 8) return false; + int64_t v = (int8_t)p[i]; for (size_t k = 1; k < il; ++k) v = (v << 8) | p[i + k]; + i = end; out[key] = v; + } + return true; +} +int main() { + auto *device = static_cast(native_keyevents_allocate()); + IOService *base = device; + assert(base->start(nullptr) && device->publications == 1); + IOUserClient sentinel2 = *(IOUserClient *)nullptr; +} +'''.replace(' IOUserClient sentinel2 = *(IOUserClient *)nullptr;\n', ''' + struct Probe : IOUserClient { IOReturn externalMethod(uint32_t, IOExternalMethodArguments *, IOExternalMethodDispatch *, OSObject *, void *) override { return 0; } }; + Probe sentinel; + // Closed boundary (default): every open is refused and clears the output. + for (UInt32 type : {0u, 1u, 7u, 0xffffffffu}) { + IOUserClient *out = &sentinel; + assert(base->newUserClient(nullptr, nullptr, type, nullptr, &out) == kIOReturnNotReady && !out); + out = &sentinel; + assert(base->newUserClient(nullptr, nullptr, type, &out) == kIOReturnNotReady && !out); + assert(base->newUserClient(nullptr, nullptr, type, nullptr, nullptr) == kIOReturnNotReady); + } + assert(refused == 12 && !opened); + // State mode: the client opens and answers only the state queries. + enabled = true; + IOUserClient *client = nullptr; + assert(base->newUserClient(nullptr, nullptr, 0, nullptr, &client) == kIOReturnSuccess && client && client->started); + assert(opened == 1); + uint64_t in[1] = {0}, outScalar[1] = {0}; + IOExternalMethodArguments a{in, 1, outScalar, 1, nullptr, 0}; + assert(client->externalMethod(7, &a) == kIOReturnSuccess && outScalar[0] == 4); + a.scalarInputCount = 2; assert(client->externalMethod(7, &a) == kIOReturnBadArgument); + // get_system hands out the system bag handle 0; handshake and bookkeeping selectors succeed without outputs. + outScalar[0] = 99; IOExternalMethodArguments sys{in, 0, outScalar, 1, nullptr, 0}; + assert(client->externalMethod(14, &sys) == kIOReturnSuccess && outScalar[0] == 0 && sys.scalarOutputCount == 1); + IOExternalMethodArguments nosys{in, 0, nullptr, 0, nullptr, 0}; + assert(client->externalMethod(14, &nosys) == kIOReturnBadArgument); + for (uint32_t sel : {0u, 4u, 5u, 9u, 12u, 13u, 26u, 27u, 55u, 101u, 102u}) { + IOExternalMethodArguments h{nullptr, 0, nullptr, 0, nullptr, 0}; + assert(client->externalMethod(sel, &h) == kIOReturnSuccess); + } + uint8_t buffer[0x1000]; + for (uint32_t want : {0u, 3u}) { + lockState = want; + for (uint32_t sel : {17u, 35u}) { + IOExternalMethodArguments s{in, 1, nullptr, 0, buffer, sizeof buffer}; + assert(client->externalMethod(sel, &s) == kIOReturnSuccess && s.structureOutputSize > 20 && s.structureOutputSize < 200); + std::map state; + assert(decode(buffer, s.structureOutputSize, state)); + assert(state.size() == 10 && state["sls"] == want && state["ss"] == 4 && state["bh"] == 0 && state.count("sgpe") && state.count("srcd")); + // Large outputs come through a descriptor. + IOMemoryDescriptor descriptor; + IOExternalMethodArguments d{in, 1, nullptr, 0, nullptr, 0, &descriptor, 0}; + assert(client->externalMethod(sel, &d) == kIOReturnSuccess && d.structureOutputDescriptorSize == s.structureOutputSize); + std::map viaDescriptor; + assert(decode(descriptor.bytes, d.structureOutputDescriptorSize, viaDescriptor) && viaDescriptor == state); + IOMemoryDescriptor tiny; tiny.length = 10; + IOExternalMethodArguments td{in, 1, nullptr, 0, nullptr, 0, &tiny, 0}; + assert(client->externalMethod(sel, &td) == kIOReturnNoSpace); + IOExternalMethodArguments none{nullptr, 0, nullptr, 0, buffer, sizeof buffer}; + assert(client->externalMethod(sel, &none) == kIOReturnSuccess); + IOExternalMethodArguments small{in, 1, nullptr, 0, buffer, 10}; + assert(client->externalMethod(sel, &small) == kIOReturnNoSpace); + IOExternalMethodArguments bad{in, 1, outScalar, 1, buffer, sizeof buffer}; + assert(client->externalMethod(sel, &bad) == kIOReturnBadArgument); + } + } + // Everything else is unsupported (keys, bags, passcodes are not provided); a missing argument block is an error. + for (uint32_t sel : {1u, 2u, 3u, 6u, 10u, 11u, 15u, 16u, 18u, 36u, 73u, 1000u}) { + IOExternalMethodArguments x{in, 1, outScalar, 1, buffer, sizeof buffer}; + assert(client->externalMethod(sel, &x) == (IOReturn)0xe00002c7); + } + assert(client->externalMethod(7, nullptr) == (IOReturn)0xe00002bc); + assert(client->clientClose() == kIOReturnSuccess && static_cast(client)->terminated); + assert(calls > 10); + client->release(); + delete device; + AppleKeyStore failed; + failed.baseOK = false; + assert(!failed.start(nullptr) && !failed.publications && !failed.properties); + for (unsigned which : {1u, 2u}) { + AppleKeyStore propertyFailure; + propertyFailure.failProperty = which; + assert(!propertyFailure.start(nullptr)); + assert(!propertyFailure.publications && propertyFailure.stops == 1); + } +''')) + subprocess.run(["c++", "-std=c++17", "-Wall", "-Wextra", "-Werror", + "-I", str(work), "-I", str(root), str(work / "test.cpp"), + "-o", str(work / "test")], check=True) + subprocess.run([str(work / "test")], check=True) +print("PASS: closed boundary by default; state mode answers the state/bookkeeping selectors and rejects every other selector") diff --git a/drivers/NativeMetal/HostMetalCommandServer.h b/drivers/NativeMetal/HostMetalCommandServer.h new file mode 100644 index 0000000..6c5502d --- /dev/null +++ b/drivers/NativeMetal/HostMetalCommandServer.h @@ -0,0 +1,4 @@ +// SPDX-License-Identifier: MIT +#import +#import +NSDictionary *ServeOfflineMetalCommands(const char *,id,NSArray> *,NSArray *,unsigned expectedSessions=0); diff --git a/drivers/NativeMetal/HostMetalCommandServer.mm b/drivers/NativeMetal/HostMetalCommandServer.mm new file mode 100644 index 0000000..bd7d548 --- /dev/null +++ b/drivers/NativeMetal/HostMetalCommandServer.mm @@ -0,0 +1,658 @@ +// SPDX-License-Identifier: MIT +// Actual host Metal objects and commands. Guest requests select operations; +// no pre-scripted draw, completion or texture content is returned. +#import "HostMetalCommandServer.h" +#import +#import +#import +#include "OfflineMetalWire.h" +#include "OfflineMetalFunctionWire.h" +#include "OfflineMetalQueueWire.h" +#include "OfflineMetalTextureWire.h" +#include "OfflineMetalDepthStencilWire.h" +#include "OfflineMetalVertexWire.h" +#include "OfflineMetalFenceWire.h" +#include "OfflineMetalCommandWire.h" +#include "OfflineMetalEventWire.h" +#include "OfflineMetalSharedEventWire.h" +#include "OfflineMetalSessionWire.h" +#include +#include +#include +#include +#include +#include +#include +#include +static bool exact(int fd,void *bytes,size_t size,bool sending) { + unsigned char *p=(unsigned char *)bytes; + while(size){ssize_t n=sending?write(fd,p,size):read(fd,p,size);if(n<0 && errno==EINTR)continue;if(n<=0)return false;p+=n;size-=n;}return true; +} +static NSString *hashBytes(const void *bytes,size_t size) { + if(size>UINT32_MAX)return nil;unsigned char hash[32];CC_SHA256(bytes,(CC_LONG)size,hash); + NSMutableString *s=[NSMutableString string];for(unsigned i=0;i<32;i++)[s appendFormat:@"%02x",hash[i]];return s; +} +static NSData *jsonBytes(id value) {return [NSJSONSerialization dataWithJSONObject:value options:0 error:nil];} +NSDictionary *ServeOfflineMetalCommands(const char *path,id device,NSArray> *libraries,NSArray *libraryRows,unsigned expectedSessions) { + if(expectedSessions>128)return nil; + if(strlen(path)>=sizeof(((sockaddr_un *)nullptr)->sun_path))return nil; + NSMutableDictionary *fenceProducers=[NSMutableDictionary dictionary],*eventProducers=[NSMutableDictionary dictionary]; + NSMutableArray *eventsCreated=[NSMutableArray array],*eventOperations=[NSMutableArray array],*sharedValues=[NSMutableArray array],*sharedLabels=[NSMutableArray array],*notificationJournal=[NSMutableArray array],*exportJournal=[NSMutableArray array]; + NSMutableDictionary *sharedExports=[NSMutableDictionary dictionary]; + NSMutableDictionary *sharedStates=[NSMutableDictionary dictionary]; + NSMutableDictionary *notifications=[NSMutableDictionary dictionary]; + NSLock *notificationLock=[NSLock new];MTLSharedEventListener *sharedListener=[[MTLSharedEventListener alloc] initWithDispatchQueue:dispatch_get_global_queue(QOS_CLASS_DEFAULT,0)]; + uint64_t nextExport=1,nextNotification=1;uint32_t controlRequest=1; + NSMutableArray *fencesCreated=[NSMutableArray array],*fenceOperations=[NSMutableArray array],*textureHazards=[NSMutableArray array]; + int listener=socket(AF_UNIX,SOCK_STREAM,0);if(listener<0)return nil; + sockaddr_un address={};address.sun_family=AF_UNIX;strcpy(address.sun_path,path); + if(bind(listener,(sockaddr *)&address,sizeof(address)) || listen(listener,1)){close(listener);return nil;} + fprintf(stderr,"OFFLINE_METAL_READY\n");fflush(stderr); + int fd=accept(listener,nullptr,nullptr);close(listener);if(fd<0)return nil; + timeval timeout={90,0};setsockopt(fd,SOL_SOCKET,SO_RCVTIMEO,&timeout,sizeof(timeout));setsockopt(fd,SOL_SOCKET,SO_SNDTIMEO,&timeout,sizeof(timeout)); + int yes=1;setsockopt(fd,SOL_SOCKET,SO_NOSIGPIPE,&yes,sizeof(yes)); + NSMutableDictionary *objects=[NSMutableDictionary dictionary];uint64_t next=1; + NSMutableDictionary *sessions=[NSMutableDictionary dictionary]; + NSMutableDictionary *sessionRequests=[NSMutableDictionary dictionary]; + uint64_t nextSession=1;unsigned openedSessions=0,closedSessions=0; + NSMutableArray *completed=[NSMutableArray array],*readbacks=[NSMutableArray array],*operations=[NSMutableArray array],*uploads=[NSMutableArray array]; + NSMutableArray *bufferWrites=[NSMutableArray array],*bufferReads=[NSMutableArray array],*bufferCopies=[NSMutableArray array]; + NSMutableArray *queuePriorities=[NSMutableArray array]; + NSMutableArray *enqueued=[NSMutableArray array]; + NSMutableDictionary *pendingEnqueues=[NSMutableDictionary dictionary]; + NSMutableArray *textureCopies=[NSMutableArray array]; + NSMutableArray *depthStates=[NSMutableArray array],*stencilReferences=[NSMutableArray array],*attachmentPasses=[NSMutableArray array]; + NSMutableDictionary *openEncoders=[NSMutableDictionary dictionary];NSMutableArray *abortedEncoders=[NSMutableArray array]; + NSMutableArray *specializations=[NSMutableArray array]; + NSMutableDictionary *asyncCommands=[NSMutableDictionary dictionary]; + NSMutableArray *submitted=[NSMutableArray array],*commandObservations=[NSMutableArray array],*failedCommands=[NSMutableArray array]; + NSMutableSet *recordedCompletions=[NSMutableSet set],*recordedFailures=[NSMutableSet set]; + auto observeCommand=[&](uint64_t handle,id command,NSString *reason)->OMCommandResult { + OMCommandResult value={(uint64_t)command.status,0,0}; + if(value.status==MTLCommandBufferStatusCompleted) { + value.start_ns=(uint64_t)(command.GPUStartTime*1e9);value.end_ns=(uint64_t)(command.GPUEndTime*1e9); + if(value.start_ns && value.end_ns>value.start_ns && ![recordedCompletions containsObject:@(handle)]) { + [recordedCompletions addObject:@(handle)];[completed addObject:@{@"command":@(handle),@"status":@(value.status),@"gpu_start_ns":@(value.start_ns),@"gpu_end_ns":@(value.end_ns),@"observed_by":reason}]; + } + } else if(value.status==MTLCommandBufferStatusError && ![recordedFailures containsObject:@(handle)]) { + [recordedFailures addObject:@(handle)];[failedCommands addObject:@{@"command":@(handle),@"status":@(value.status),@"actual_error":command.error.localizedDescription?:@""}]; + } + return value; + }; + auto finishSubmitted=[&](uint64_t selected,bool all,NSString *reason) { + for(NSNumber *handle in asyncCommands.allKeys) { + NSDictionary *row=asyncCommands[handle];if(!all && [row[@"session"] unsignedLongLongValue]!=selected)continue; + id command=row[@"command"];[command waitUntilCompleted]; + observeCommand(handle.unsignedLongLongValue,command,reason);[asyncCommands removeObjectForKey:handle]; + } + }; + auto abortEncoders=[&](uint64_t selected,bool all,NSString *reason) { + for(NSNumber *handle in openEncoders.allKeys) { + NSDictionary *row=openEncoders[handle];if(!all && [row[@"session"] unsignedLongLongValue]!=selected)continue; + id encoder=row[@"encoder"];id command=row[@"command_object"]; + uint64_t status=command.status;[encoder endEncoding]; + [abortedEncoders addObject:@{@"encoder":handle,@"command":row[@"command_handle"],@"session":row[@"session"], + @"actual_command_status":@(status),@"reason":reason,@"encoding_ended":@YES,@"gpu_submission_created_by_cleanup":@NO}]; + [openEncoders removeObjectForKey:handle]; + } + }; + auto hasOpenEncoder=[&](uint64_t command)->bool { + for(NSDictionary *row in openEncoders.allValues)if([row[@"command_handle"] unsignedLongLongValue]==command)return true;return false; + }; + auto cancelNotifications=[&](uint64_t selected,bool all) { + [notificationLock lock]; + for(NSNumber *key in notifications.allKeys) { + NSMutableDictionary *record=notifications[key];if(!all && [record[@"session"] unsignedLongLongValue]!=selected)continue; + record[@"cancelled"]=@YES;[notificationJournal addObject:@{@"notification":key,@"session":record[@"session"],@"event":record[@"event"],@"requested":record[@"requested"],@"actually_fired":record[@"ready"],@"actual_value":record[@"value"]}];[notifications removeObjectForKey:key]; + } + [notificationLock unlock]; + }; + NSMutableDictionary *renderExtents=[NSMutableDictionary dictionary];NSMutableArray *blendPipelines=[NSMutableArray array],*textureBindings=[NSMutableArray array],*vertexPipelines=[NSMutableArray array]; + uint32_t bootBytes=0;bool finished=false,healthy=true; + auto save=[&](id object)->uint64_t {if(!object)return 0;uint64_t key=next++;objects[@(key)]=object;return key;}; + for(uint32_t request=1;request<=8192 && healthy && !finished;request++) { + @autoreleasepool { + OMWireHeader header={}; + if(request==1) { + uint32_t prefix=0,count=0; + do {unsigned char byte=0;if(!exact(fd,&byte,1,false)){healthy=false;break;} + prefix=(prefix>>8)|((uint32_t)byte<<24);count++;if(count>65536){healthy=false;break;} + } while(count<4 || prefix!=OM_WIRE_MAGIC); + if(!healthy)break;bootBytes=count-4;header.magic=prefix; + if(!exact(fd,(unsigned char *)&header+4,sizeof(header)-4,false)){healthy=false;break;} + } else if(!exact(fd,&header,sizeof(header),false)){healthy=false;break;} + if(header.magic!=OM_WIRE_MAGIC || header.version!=(expectedSessions?OFFLINE_METAL_SESSION_VERSION:OFFLINE_METAL_VERSION) + || (!expectedSessions && header.request!=request) || header.bytes>4080){healthy=false;break;} + uint64_t session=0;bool validSession=true; + if(expectedSessions) { + if(!exact(fd,&session,sizeof(session),false)){healthy=false;break;} + if(header.operation==OM_SESSION_OPEN)validSession=!session && header.request==1 && !header.bytes; + else if(!session && header.operation==OM_SHARED_EXPORT_DROP){validSession=header.request==controlRequest;if(validSession)controlRequest++;} + else { + objects=sessions[@(session)];NSNumber *expected=sessionRequests[@(session)]; + validSession=objects && expected && header.request==expected.unsignedIntValue; + if(validSession)sessionRequests[@(session)]=@(header.request+1); + } + } + std::vector input(header.bytes); + if(!input.empty() && !exact(fd,input.data(),input.size(),false)){healthy=false;break;} + OMResult result={};NSData *body=nil;NSError *error=nil; + auto get=[&](uint64_t handle,Protocol *protocol)->id {id object=objects[@(handle)];return object && [object conformsToProtocol:protocol]?object:nil;}; + auto getEncoder=[&](uint64_t handle,Protocol *protocol)->id {return openEncoders[@(handle)]?get(handle,protocol):nil;}; +#define REQUEST(Type) if(input.size()!=sizeof(Type)){result.error=-2;break;} Type q={};memcpy(&q,input.data(),sizeof(q)) + if(!validSession)result.error=-2; + else @try { + switch(header.operation) { + case OM_SESSION_OPEN: + if(!expectedSessions || openedSessions>=expectedSessions){result.error=-2;break;} + result.handle=nextSession++;sessions[@(result.handle)]=[NSMutableDictionary dictionary]; + sessionRequests[@(result.handle)]=@1;openedSessions++;break; + case OM_SESSION_CLOSE: + if(!expectedSessions || !input.empty()){result.error=-2;break;} + // Remove only this real namespace's independent references. Command + // buffers retain their actual GPU dependencies according to Metal. + abortEncoders(session,false,@"session-close"); + finishSubmitted(session,false,@"session-close"); + cancelNotifications(session,false); + for(NSNumber *handle in objects.allKeys){[renderExtents removeObjectForKey:handle];[fenceProducers removeObjectForKey:handle];[eventProducers removeObjectForKey:handle];NSMutableDictionary *shared=sharedStates[handle];if([shared[@"producer"][@"session"] unsignedLongLongValue]==session)[shared removeObjectForKey:@"producer"];[sharedStates removeObjectForKey:handle];} + for(NSNumber *handle in pendingEnqueues.allKeys)if([pendingEnqueues[handle][@"session"] unsignedLongLongValue]==session)[pendingEnqueues removeObjectForKey:handle]; + [objects removeAllObjects];[sessions removeObjectForKey:@(session)];[sessionRequests removeObjectForKey:@(session)]; + closedSessions++;finished=openedSessions==expectedSessions && !sessions.count && !sharedExports.count;break; + case OM_DEVICE: + if(!input.empty()){result.error=-2;break;} + body=jsonBytes(@{@"name":device.name,@"registryID":@(device.registryID),@"maxBufferLength":@(MIN(device.maxBufferLength,(NSUInteger)OFFLINE_METAL_MAX_BUFFER)), + @"lowPower":@(device.lowPower),@"removable":@(device.removable),@"headless":@(device.headless), + @"maxThreads":@[@(device.maxThreadsPerThreadgroup.width),@(device.maxThreadsPerThreadgroup.height),@(device.maxThreadsPerThreadgroup.depth)]});break; + case OM_QUEUE: + if(!input.empty()){result.error=-2;break;}result.handle=save([device newCommandQueue]);if(!result.handle)result.error=-3;break; + case OM_QUEUE_PRIORITY_GET:case OM_QUEUE_PRIORITY_SET: { + REQUEST(OMQueuePriority);id queue=get(q.queue,@protocol(MTLCommandQueue)); + if(!queue || q.background>1 || (header.operation==OM_QUEUE_PRIORITY_GET && q.priority)){result.error=-2;break;} + bool writing=header.operation==OM_QUEUE_PRIORITY_SET; + if(writing && (q.priority>5 || (!q.background && q.priority==3))){result.error=-4;break;} + SEL getter=sel_registerName(q.background?"getBackgroundGPUPriority":"getGPUPriority"); + SEL setter=sel_registerName(q.background?"setBackgroundGPUPriority:":"setGPUPriority:"); + Method readMethod=class_getInstanceMethod(object_getClass(queue),getter),writeMethod=class_getInstanceMethod(object_getClass(queue),setter); + if(!readMethod || strcmp(method_getTypeEncoding(readMethod),"Q16@0:8") || + (writing && (!writeMethod || strcmp(method_getTypeEncoding(writeMethod),"B24@0:8Q16")))){result.error=-4;break;} + uint64_t before=((NSUInteger(*)(id,SEL))objc_msgSend)(queue,getter),value=before; + if(writing)value=((BOOL(*)(id,SEL,NSUInteger))objc_msgSend)(queue,setter,q.priority)?1:0; + uint64_t after=((NSUInteger(*)(id,SEL))objc_msgSend)(queue,getter); + body=[NSData dataWithBytes:&value length:sizeof(value)]; + [queuePriorities addObject:@{@"queue":@(q.queue),@"session":@(session),@"background":@(q.background),@"writing":@(writing), + @"requested":@(q.priority),@"before":@(before),@"after":@(after),@"result":@(value)}];break; + } + case OM_LIBRARY: { + REQUEST(OMLibrary);NSMutableString *hash=[NSMutableString string];for(unsigned i=0;i<32;i++)[hash appendFormat:@"%02x",q.sha256[i]]; + NSUInteger i=0;for(;i library=libraries[i];result.handle=save(library);body=jsonBytes(library.functionNames);break; + } + case OM_FUNCTION: { + OMFunction q={};OMSpecializedFunction extended={};bool specialized=input.size()!=sizeof(q); + if(!specialized)memcpy(&q,input.data(),sizeof(q)); + else { + if(input.size()OFFLINE_METAL_MAX_CONSTANTS || input.size()!=sizeof(extended)+extended.count*sizeof(OMFunctionConstant)){result.error=-2;break;} + } + id library=get(q.library,@protocol(MTLLibrary)); + if(!library || !memchr(q.name,0,sizeof(q.name))){result.error=-2;break;} + NSString *name=[NSString stringWithUTF8String:q.name];if(!name.length){result.error=-2;break;} + id function=nil; + if(specialized) { + MTLFunctionConstantValues *values=[MTLFunctionConstantValues new];NSMutableArray *rows=[NSMutableArray array];NSMutableSet *keys=[NSMutableSet set]; + for(uint64_t i=0;i1 || row.index>65535 || + (row.named && (!memchr(row.name,0,sizeof(row.name)) || row.index)) || (!row.named && row.name[0])){result.error=-2;break;} + NSString *constantName=row.named?[NSString stringWithUTF8String:row.name]:nil; + NSString *key=row.named?[NSString stringWithFormat:@"name:%@",constantName]:[NSString stringWithFormat:@"index:%llu",row.index]; + if((row.named && !constantName.length) || [keys containsObject:key]){result.error=-2;break;}[keys addObject:key]; + if(row.named)[values setConstantValue:row.data type:(MTLDataType)row.type withName:constantName]; + else [values setConstantValue:row.data type:(MTLDataType)row.type atIndex:row.index]; + NSMutableString *hex=[NSMutableString string];for(uint64_t j=0;jMTLHazardTrackingModeTracked){result.error=-2;break;} + bool attachment=q.format==MTLPixelFormatStencil8 || q.format==MTLPixelFormatDepth32Float || q.format==MTLPixelFormatDepth32Float_Stencil8; + if(q.type!=MTLTextureType2D || (!attachment && q.format!=MTLPixelFormatR32Float && q.format!=MTLPixelFormatRG16Float && q.format!=MTLPixelFormatBGRA8Unorm && q.format!=MTLPixelFormatRGBA8Unorm) || !q.width || !q.height || q.width>2048 || q.height>2048 + || q.depth!=1 || q.levels!=1 || q.samples!=1 || q.array!=1 || (q.storage!=MTLStorageModeShared && q.storage!=MTLStorageModePrivate) || q.cache!=MTLCPUCacheModeDefaultCache){result.error=-4;break;} + if(attachment && (q.storage!=MTLStorageModePrivate || q.usage!=MTLTextureUsageRenderTarget)){result.error=-4;break;} + MTLTextureDescriptor *d=[MTLTextureDescriptor texture2DDescriptorWithPixelFormat:(MTLPixelFormat)q.format width:q.width height:q.height mipmapped:NO]; + d.usage=q.usage;d.storageMode=(MTLStorageMode)q.storage;d.hazardTrackingMode=(MTLHazardTrackingMode)hazard; + id texture=[device newTextureWithDescriptor:d];if(!texture || (hazard && texture.hazardTrackingMode!=hazard)){result.error=-4;break;} + result.handle=save(texture); + if(header.operation==OM_TEXTURE_HAZARD){uint64_t actual=texture.hazardTrackingMode;body=[NSData dataWithBytes:&actual length:sizeof(actual)];[textureHazards addObject:@{@"texture":@(result.handle),@"session":@(session),@"requested":@(hazard),@"actual":@(actual)}];} + break; + } + case OM_PIPELINE:case OM_PIPELINE_BLEND:case OM_PIPELINE_ATTACHMENTS: { + OMPipeline q={};OMBlend blends[8]={};uint64_t depth=0,stencil=0;OMVertexPipeline vertexInput={};bool hasVertexInput=false; + if(header.operation==OM_PIPELINE) {if(input.size()!=sizeof(q)){result.error=-2;break;}memcpy(&q,input.data(),sizeof(q));} + else if(header.operation==OM_PIPELINE_BLEND){if(input.size()!=sizeof(OMBlendPipeline)){result.error=-2;break;}OMBlendPipeline ext={};memcpy(&ext,input.data(),sizeof(ext));q=ext.pipeline;memcpy(blends,ext.colors,sizeof(blends));} + else { + if(input.size()==sizeof(vertexInput)) {memcpy(&vertexInput,input.data(),sizeof(vertexInput));if(!OMVertexInputsValid(vertexInput)){result.error=-2;break;}hasVertexInput=true;} + else if(input.size()!=sizeof(OMAttachmentPipeline)){result.error=-2;break;} + OMAttachmentPipeline ext={};memcpy(&ext,input.data(),sizeof(ext));q=ext.colors.pipeline;memcpy(blends,ext.colors.colors,sizeof(blends));depth=ext.depthFormat;stencil=ext.stencilFormat; + } + id vertex=get(q.vertex,@protocol(MTLFunction)),fragment=get(q.fragment,@protocol(MTLFunction)); + if(!vertex || !fragment || q.samples!=1 || (depth && depth!=MTLPixelFormatDepth32Float && depth!=MTLPixelFormatDepth32Float_Stencil8) || + (stencil && stencil!=MTLPixelFormatStencil8 && stencil!=MTLPixelFormatDepth32Float_Stencil8)){result.error=-2;break;} + MTLRenderPipelineDescriptor *d=[MTLRenderPipelineDescriptor new];d.vertexFunction=vertex;d.fragmentFunction=fragment;d.rasterSampleCount=q.samples; + d.depthAttachmentPixelFormat=(MTLPixelFormat)depth;d.stencilAttachmentPixelFormat=(MTLPixelFormat)stencil; + NSMutableArray *actualAttributes=[NSMutableArray array],*actualLayouts=[NSMutableArray array]; + if(hasVertexInput) { + MTLVertexDescriptor *v=[MTLVertexDescriptor vertexDescriptor];bool used[31]={}; + for(unsigned i=0;i<31;i++) { + const auto &a=vertexInput.attributes[i];if(!a.format)continue; + v.attributes[i].format=(MTLVertexFormat)a.format;v.attributes[i].offset=a.offset;v.attributes[i].bufferIndex=a.buffer;used[a.buffer]=true; + [actualAttributes addObject:@{@"index":@(i),@"format":@(v.attributes[i].format),@"offset":@(v.attributes[i].offset),@"buffer":@(v.attributes[i].bufferIndex)}]; + } + for(unsigned i=0;i<31;i++)if(used[i]) { + const auto &l=vertexInput.layouts[i];v.layouts[i].stride=l.stride;v.layouts[i].stepFunction=(MTLVertexStepFunction)l.step;v.layouts[i].stepRate=l.rate; + [actualLayouts addObject:@{@"index":@(i),@"stride":@(v.layouts[i].stride),@"step":@(v.layouts[i].stepFunction),@"rate":@(v.layouts[i].stepRate)}]; + } + d.vertexDescriptor=v; + } + for(unsigned i=0;i<8;i++) { + d.colorAttachments[i].pixelFormat=(MTLPixelFormat)q.formats[i];d.colorAttachments[i].writeMask=(MTLColorWriteMask)q.masks[i]; + if(header.operation!=OM_PIPELINE) { + const OMBlend &b=blends[i];if(b.enabled>1 || b.sourceRGB>MTLBlendFactorOneMinusSource1Alpha || b.destinationRGB>MTLBlendFactorOneMinusSource1Alpha || + b.sourceAlpha>MTLBlendFactorOneMinusSource1Alpha || b.destinationAlpha>MTLBlendFactorOneMinusSource1Alpha || b.rgbOperation>MTLBlendOperationMax || b.alphaOperation>MTLBlendOperationMax){result.error=-2;break;} + MTLRenderPipelineColorAttachmentDescriptor *a=d.colorAttachments[i];a.blendingEnabled=b.enabled; + a.sourceRGBBlendFactor=(MTLBlendFactor)b.sourceRGB;a.destinationRGBBlendFactor=(MTLBlendFactor)b.destinationRGB;a.rgbBlendOperation=(MTLBlendOperation)b.rgbOperation; + a.sourceAlphaBlendFactor=(MTLBlendFactor)b.sourceAlpha;a.destinationAlphaBlendFactor=(MTLBlendFactor)b.destinationAlpha;a.alphaBlendOperation=(MTLBlendOperation)b.alphaOperation; + } + } + if(result.error)break;result.handle=save([device newRenderPipelineStateWithDescriptor:d error:&error]);if(!result.handle)result.error=-3; + if(result.handle && hasVertexInput)[vertexPipelines addObject:@{@"pipeline":@(result.handle),@"session":@(session),@"vertex":vertex.name,@"fragment":fragment.name,@"attributes":actualAttributes,@"layouts":actualLayouts}]; + if(result.handle && header.operation==OM_PIPELINE_BLEND)[blendPipelines addObject:@{@"pipeline":@(result.handle),@"vertex":vertex.name,@"fragment":fragment.name}];break; + } + case OM_FENCE: + if(!input.empty()){result.error=-2;break;} + result.handle=save([device newFence]);if(!result.handle){result.error=-4;break;} + [fencesCreated addObject:@{@"fence":@(result.handle),@"session":@(session)}];break; + case OM_RENDER_FENCE_UPDATE:case OM_RENDER_FENCE_WAIT:case OM_BLIT_FENCE_UPDATE:case OM_BLIT_FENCE_WAIT: { + REQUEST(OMFenceDependency); + bool render=header.operation==OM_RENDER_FENCE_UPDATE || header.operation==OM_RENDER_FENCE_WAIT; + bool update=header.operation==OM_RENDER_FENCE_UPDATE || header.operation==OM_BLIT_FENCE_UPDATE; + id encoder=getEncoder(q.encoder,render?@protocol(MTLRenderCommandEncoder):@protocol(MTLBlitCommandEncoder)); + id fence=get(q.fence,@protocol(MTLFence));NSDictionary *active=openEncoders[@(q.encoder)]; + if(!encoder || !fence || !active || (render?(!q.stages || (q.stages&~3ULL)):q.stages!=0)){result.error=-2;break;} + id command=active[@"command_object"];NSDictionary *producer=fenceProducers[@(q.fence)]; + if(!update) { + // Same-queue producers must be actually committed. Cross-queue fences + // stay limited to completed producers after a retained real pending- + // producer readback failure. Use actual GPU events for overlap across + // queues; CPU encoding metadata never represents a signaled fence. + id source=producer[@"command"];MTLCommandBufferStatus actual=source.status; + if(!producer || [producer[@"encoder"] unsignedLongLongValue]==q.encoder || openEncoders[producer[@"encoder"]] || + (source!=command && (actualMTLCommandBufferStatusCompleted || + (source.commandQueue!=command.commandQueue && actual!=MTLCommandBufferStatusCompleted)))){result.error=-4;break;} + } + if(render) { + if(update)[(id)encoder updateFence:fence afterStages:(MTLRenderStages)q.stages]; + else [(id)encoder waitForFence:fence beforeStages:(MTLRenderStages)q.stages]; + } else { + if(update)[(id)encoder updateFence:fence];else [(id)encoder waitForFence:fence]; + } + [fenceOperations addObject:@{@"fence":@(q.fence),@"encoder":@(q.encoder),@"session":@(session),@"command":active[@"command_handle"],@"stages":@(q.stages),@"render":@(render),@"update":@(update)}]; + if(update)fenceProducers[@(q.fence)]=@{@"encoder":@(q.encoder),@"command":command};break; + } + case OM_SHARED_EVENT: { + if(!input.empty()){result.error=-2;break;}id event=[device newSharedEvent];if(!event){result.error=-4;break;} + if(event.device && event.device.registryID!=device.registryID){result.error=-4;break;} + result.handle=save(event);sharedStates[@(result.handle)]=[NSMutableDictionary dictionary];[eventsCreated addObject:@{@"event":@(result.handle),@"session":@(session),@"shared":@YES,@"device_is_nil":@(event.device==nil)}]; + body=jsonBytes(@{@"device_is_nil":@(event.device==nil),@"device_is_owner":@(event.device && event.device.registryID==device.registryID),@"signaled_value":@(event.signaledValue),@"label":event.label?:[NSNull null]});break; + } + case OM_SHARED_VALUE_GET: { + REQUEST(OMHandle);id event=get(q.handle,@protocol(MTLSharedEvent));if(!event){result.error=-2;break;} + uint64_t value=event.signaledValue;body=[NSData dataWithBytes:&value length:sizeof(value)]; + [sharedValues addObject:@{@"event":@(q.handle),@"session":@(session),@"value":@(value),@"cpu_set":@NO}];break; + } + case OM_SHARED_VALUE_SET: { + REQUEST(OMSharedEventValue);id event=get(q.event,@protocol(MTLSharedEvent));if(!event || q.value event=get(q.event,@protocol(MTLSharedEvent)); + if(!event || q.present>1 || q.bytes>512 || input.size()!=sizeof(q)+q.bytes || (!q.present && q.bytes)){result.error=-2;break;} + NSString *label=q.present?[[NSString alloc] initWithBytes:input.data()+sizeof(q) length:q.bytes encoding:NSUTF8StringEncoding]:nil; + if(q.present && !label){result.error=-2;break;}event.label=label;body=jsonBytes(@{@"label":event.label?:[NSNull null]}); + [sharedLabels addObject:@{@"event":@(q.event),@"session":@(session),@"actual_label":event.label?:[NSNull null]}];break; + } + case OM_SHARED_NOTIFY: { + REQUEST(OMSharedEventValue);id event=get(q.event,@protocol(MTLSharedEvent));if(!event || !sharedListener){result.error=-2;break;} + uint64_t identifier=nextNotification++;NSMutableDictionary *record=[@{@"session":@(session),@"event":@(q.event),@"requested":@(q.value),@"ready":@NO,@"value":@0,@"cancelled":@NO,@"identity_match":@NO} mutableCopy]; + [notificationLock lock];notifications[@(identifier)]=record;[notificationLock unlock];__weak id expected=event; + [event notifyListener:sharedListener atValue:q.value block:^(id actual,uint64_t value) { + [notificationLock lock];if(![record[@"cancelled"] boolValue]){record[@"ready"]=@YES;record[@"value"]=@(value);record[@"identity_match"]=@(actual==expected);}[notificationLock unlock]; + }];result.handle=identifier;break; + } + case OM_SHARED_NOTIFICATION_POLL: { + REQUEST(OMHandle);[notificationLock lock];NSMutableDictionary *record=notifications[@(q.handle)]; + if(!record || [record[@"session"] unsignedLongLongValue]!=session){result.error=-2;[notificationLock unlock];break;} + OMSharedNotification value={};value.ready=[record[@"ready"] boolValue]; + if(value.ready){if(![record[@"identity_match"] boolValue] || [record[@"value"] unsignedLongLongValue]<[record[@"requested"] unsignedLongLongValue])result.error=-3;else {value.event=[record[@"event"] unsignedLongLongValue];value.value=[record[@"value"] unsignedLongLongValue];}} + [notificationLock unlock];if(!result.error)body=[NSData dataWithBytes:&value length:sizeof(value)];break; + } + case OM_SHARED_EXPORT: { + REQUEST(OMHandle);id event=get(q.handle,@protocol(MTLSharedEvent));if(!expectedSessions || !event || !sharedStates[@(q.handle)]){result.error=-2;break;} + MTLSharedEventHandle *handle=[event newSharedEventHandle];if(!handle){result.error=-4;break;}result.handle=nextExport++;sharedExports[@(result.handle)]=@{@"handle":handle,@"state":sharedStates[@(q.handle)]}; + Method trace=class_getInstanceMethod(object_getClass(handle),sel_registerName("labelTraceID"));BOOL supported=trace && !strcmp(method_getTypeEncoding(trace),"Q16@0:8"); + uint64_t traceID=supported?((uint64_t (*)(id,SEL))objc_msgSend)(handle,sel_registerName("labelTraceID")):0; + body=jsonBytes(@{@"label":handle.label?:[NSNull null],@"trace_id_supported":@(supported),@"trace_id":supported?@(traceID):[NSNull null]}); + [exportJournal addObject:@{@"operation":@"export",@"export":@(result.handle),@"event":@(q.handle),@"session":@(session)}];break; + } + case OM_SHARED_IMPORT: { + REQUEST(OMHandle);NSDictionary *exported=sharedExports[@(q.handle)];MTLSharedEventHandle *handle=exported[@"handle"];if(!session || !handle){result.error=-2;break;} + id event=[device newSharedEventWithHandle:handle];if(!event || (event.device && event.device.registryID!=device.registryID)){result.error=-4;break;}result.handle=save(event);sharedStates[@(result.handle)]=exported[@"state"]; + body=jsonBytes(@{@"device_is_nil":@(event.device==nil),@"device_is_owner":@(event.device && event.device.registryID==device.registryID),@"signaled_value":@(event.signaledValue),@"label":event.label?:[NSNull null]}); + [eventsCreated addObject:@{@"event":@(result.handle),@"session":@(session),@"shared":@YES,@"imported":@YES,@"device_is_nil":@(event.device==nil)}]; + [exportJournal addObject:@{@"operation":@"import",@"export":@(q.handle),@"event":@(result.handle),@"session":@(session)}];break; + } + case OM_SHARED_EXPORT_DROP: { + REQUEST(OMHandle);if(!expectedSessions || session || !sharedExports[@(q.handle)]){result.error=-2;break;}[sharedExports removeObjectForKey:@(q.handle)]; + [exportJournal addObject:@{@"operation":@"drop",@"export":@(q.handle),@"session":@0}];finished=openedSessions==expectedSessions && !sessions.count && !sharedExports.count;break; + } + case OM_EVENT: + if(!input.empty()){result.error=-2;break;} + result.handle=save([device newEvent]);if(!result.handle){result.error=-4;break;} + [eventsCreated addObject:@{@"event":@(result.handle),@"session":@(session)}];break; + case OM_EVENT_SIGNAL:case OM_EVENT_WAIT: { + REQUEST(OMEventDependency);id command=get(q.command,@protocol(MTLCommandBuffer));id event=get(q.event,@protocol(MTLEvent)); + if(!command || !event || !q.value || command.status>MTLCommandBufferStatusEnqueued || hasOpenEncoder(q.command)){result.error=-2;break;} + NSMutableDictionary *shared=sharedStates[@(q.event)];NSDictionary *producer=shared?shared[@"producer"]:eventProducers[@(q.event)];bool signal=header.operation==OM_EVENT_SIGNAL; + uint64_t previous=[producer[@"value"] unsignedLongLongValue],actualValue=shared?[(id)event signaledValue]:0; + uint64_t actualProducer=0; + if(signal) { + if(q.value<=previous || (shared && q.value<=actualValue)){result.error=-2;break;} + [command encodeSignalEvent:event value:q.value];NSDictionary *record=@{@"value":@(q.value),@"command":command,@"session":@(session)};if(shared)shared[@"producer"]=record;else eventProducers[@(q.event)]=record; + } else { + id source=producer[@"command"];actualProducer=source.status; + if(!(shared && actualValue>=q.value) && (!producer || q.value>previous || source==command || actualProducerMTLCommandBufferStatusCompleted)){result.error=-4;break;} + [command encodeWaitForEvent:event value:q.value]; + } + [eventOperations addObject:@{@"event":@(q.event),@"command":@(q.command),@"session":@(session),@"value":@(q.value),@"signal":@(signal),@"shared":@(shared!=nil),@"actual_signaled_value":@(actualValue),@"actual_producer_status":@(actualProducer)}];break; + } + case OM_COMMAND: { + REQUEST(OMHandle);id queue=get(q.handle,@protocol(MTLCommandQueue)); + if(!queue){result.error=-2;break;}result.handle=save([queue commandBuffer]);if(!result.handle)result.error=-3;break; + } + case OM_COMMAND_ENQUEUE: { + REQUEST(OMHandle);id command=get(q.handle,@protocol(MTLCommandBuffer)); + if(!command || command.status!=MTLCommandBufferStatusNotEnqueued){result.error=-2;break;} + // This synchronous transport currently supports one pending reservation + // per real queue. Refuse unsupported ordering before blocking the server + // behind a command whose predecessor cannot yet receive its commit. + for(NSDictionary *row in pendingEnqueues.allValues)if([row[@"command"] commandQueue]==command.commandQueue){result.error=-4;break;} + if(result.error)break;[command enqueue];uint64_t status=command.status; + if(status!=MTLCommandBufferStatusEnqueued){result.error=-3;break;} + pendingEnqueues[@(q.handle)]=@{@"command":command,@"session":@(session)}; + body=[NSData dataWithBytes:&status length:sizeof(status)]; + [enqueued addObject:@{@"command":@(q.handle),@"session":@(session),@"status":@(status),@"operation_request":@(request)}];break; + } + case OM_RENDER:case OM_RENDER_ATTACHMENTS: { + OMRender q={};OMDepthAttachment depth={};OMStencilAttachment stencil={}; + if(header.operation==OM_RENDER){if(input.size()!=sizeof(q)){result.error=-2;break;}memcpy(&q,input.data(),sizeof(q));} + else {if(input.size()!=sizeof(OMAttachmentRender)){result.error=-2;break;}OMAttachmentRender ext={};memcpy(&ext,input.data(),sizeof(ext));q=ext.colors;depth=ext.depth;stencil=ext.stencil;} + id command=get(q.command,@protocol(MTLCommandBuffer));if(!command || command.status>MTLCommandBufferStatusEnqueued || hasOpenEncoder(q.command)){result.error=-2;break;} + MTLRenderPassDescriptor *d=[MTLRenderPassDescriptor renderPassDescriptor];NSUInteger width=0,height=0; + for(unsigned i=0;i<8;i++)if(q.colors[i].texture) { + id texture=get(q.colors[i].texture,@protocol(MTLTexture));if(!texture || q.colors[i].load>MTLLoadActionClear || q.colors[i].store>MTLStoreActionStore || (width && (width!=texture.width || height!=texture.height))){result.error=-2;break;} + width=texture.width;height=texture.height; + d.colorAttachments[i].texture=texture;d.colorAttachments[i].loadAction=(MTLLoadAction)q.colors[i].load;d.colorAttachments[i].storeAction=(MTLStoreAction)q.colors[i].store; + const double *v=q.colors[i].clear;d.colorAttachments[i].clearColor=MTLClearColorMake(v[0],v[1],v[2],v[3]); + } + if(depth.texture) { + id texture=get(depth.texture,@protocol(MTLTexture)); + if(!texture || (texture.pixelFormat!=MTLPixelFormatDepth32Float && texture.pixelFormat!=MTLPixelFormatDepth32Float_Stencil8) || + depth.load>MTLLoadActionClear || depth.store>MTLStoreActionStore || !std::isfinite(depth.clear) || depth.clear<0 || depth.clear>1 || + (width && (width!=texture.width || height!=texture.height))){result.error=-2;break;} + width=texture.width;height=texture.height;d.depthAttachment.texture=texture;d.depthAttachment.loadAction=(MTLLoadAction)depth.load; + d.depthAttachment.storeAction=(MTLStoreAction)depth.store;d.depthAttachment.clearDepth=depth.clear; + } + if(stencil.texture) { + id texture=get(stencil.texture,@protocol(MTLTexture)); + if(!texture || (texture.pixelFormat!=MTLPixelFormatStencil8 && texture.pixelFormat!=MTLPixelFormatDepth32Float_Stencil8) || + stencil.load>MTLLoadActionClear || stencil.store>MTLStoreActionStore || stencil.clear>255 || + (width && (width!=texture.width || height!=texture.height))){result.error=-2;break;} + width=texture.width;height=texture.height;d.stencilAttachment.texture=texture;d.stencilAttachment.loadAction=(MTLLoadAction)stencil.load; + d.stencilAttachment.storeAction=(MTLStoreAction)stencil.store;d.stencilAttachment.clearStencil=(uint32_t)stencil.clear; + } + if(!width || !height){result.error=-2;break;} + if(result.error)break;result.handle=save([command renderCommandEncoderWithDescriptor:d]);if(!result.handle)result.error=-3; + else { + renderExtents[@(result.handle)]=@[@(width),@(height)]; + openEncoders[@(result.handle)]=@{@"encoder":objects[@(result.handle)],@"command_object":command,@"command_handle":@(q.command),@"session":@(session)}; + if(depth.texture || stencil.texture)[attachmentPasses addObject:@{@"command":@(q.command),@"encoder":@(result.handle),@"session":@(session), + @"depth":@(depth.texture),@"stencil":@(stencil.texture),@"depth_clear":@(depth.clear),@"stencil_clear":@(stencil.clear), + @"depth_load":@(depth.load),@"stencil_load":@(stencil.load),@"width":@(width),@"height":@(height)}]; + }break; + } + case OM_DEPTH_STENCIL_STATE: { + REQUEST(OMDepthStencil); + auto validFace=[](const OMStencilFace &f)->bool { + if(f.enabled>1)return false; + if(!f.enabled)return !(f.compare|f.failure|f.depthFailure|f.pass|f.readMask|f.writeMask); + return f.compare<=MTLCompareFunctionAlways && f.failure<=MTLStencilOperationDecrementWrap && + f.depthFailure<=MTLStencilOperationDecrementWrap && f.pass<=MTLStencilOperationDecrementWrap && f.readMask<=UINT32_MAX && f.writeMask<=UINT32_MAX; + }; + if(q.compare>MTLCompareFunctionAlways || q.write>1 || !validFace(q.front) || !validFace(q.back)){result.error=-2;break;} + auto face=[](const OMStencilFace &f)->MTLStencilDescriptor * { + if(!f.enabled)return nil;MTLStencilDescriptor *s=[MTLStencilDescriptor new];s.stencilCompareFunction=(MTLCompareFunction)f.compare; + s.stencilFailureOperation=(MTLStencilOperation)f.failure;s.depthFailureOperation=(MTLStencilOperation)f.depthFailure;s.depthStencilPassOperation=(MTLStencilOperation)f.pass; + s.readMask=(uint32_t)f.readMask;s.writeMask=(uint32_t)f.writeMask;return s; + }; + MTLDepthStencilDescriptor *d=[MTLDepthStencilDescriptor new];d.depthCompareFunction=(MTLCompareFunction)q.compare;d.depthWriteEnabled=q.write; + d.frontFaceStencil=face(q.front);d.backFaceStencil=face(q.back);result.handle=save([device newDepthStencilStateWithDescriptor:d]); + if(!result.handle){result.error=-3;break;} + [depthStates addObject:@{@"state":@(result.handle),@"session":@(session),@"depth_compare":@(q.compare),@"depth_write":@(q.write), + @"front_compare":@(q.front.compare),@"back_compare":@(q.back.compare),@"front_write_mask":@(q.front.writeMask),@"back_write_mask":@(q.back.writeMask)}];break; + } + case OM_SET_DEPTH_STENCIL: { + REQUEST(OMSetDepthStencil);id encoder=getEncoder(q.encoder,@protocol(MTLRenderCommandEncoder));id state=q.state?get(q.state,@protocol(MTLDepthStencilState)):nil; + if(!encoder || (q.state && !state)){result.error=-2;break;}[encoder setDepthStencilState:state];break; + } + case OM_STENCIL_REFERENCE: { + REQUEST(OMStencilReference);id encoder=getEncoder(q.encoder,@protocol(MTLRenderCommandEncoder)); + if(!encoder || q.front>UINT32_MAX || q.back>UINT32_MAX){result.error=-2;break;} + [encoder setStencilFrontReferenceValue:(uint32_t)q.front backReferenceValue:(uint32_t)q.back]; + [stencilReferences addObject:@{@"encoder":@(q.encoder),@"session":@(session),@"front":@(q.front),@"back":@(q.back)}];break; + } + case OM_SET_PIPELINE: { + REQUEST(OMSetPipeline);id encoder=getEncoder(q.encoder,@protocol(MTLRenderCommandEncoder));id pipeline=get(q.pipeline,@protocol(MTLRenderPipelineState)); + if(!encoder || !pipeline){result.error=-2;break;}[encoder setRenderPipelineState:pipeline];break; + } + case OM_SET_CULL: { + REQUEST(OMSetCull);id encoder=getEncoder(q.encoder,@protocol(MTLRenderCommandEncoder));if(!encoder || q.mode>MTLCullModeBack){result.error=-2;break;}[encoder setCullMode:(MTLCullMode)q.mode];break; + } + case OM_DRAW: { + REQUEST(OMDraw);id encoder=getEncoder(q.encoder,@protocol(MTLRenderCommandEncoder)); + if(!encoder || q.primitive>MTLPrimitiveTypeTriangleStrip || !q.count || q.count>65536 || q.start>65536 || !q.instances || q.instances>1024){result.error=-2;break;} + [encoder drawPrimitives:(MTLPrimitiveType)q.primitive vertexStart:q.start vertexCount:q.count instanceCount:q.instances];break; + } + case OM_END: { + REQUEST(OMHandle);id encoder=get(q.handle,@protocol(MTLCommandEncoder));if(!encoder || !openEncoders[@(q.handle)]){result.error=-2;break;}[encoder endEncoding];[openEncoders removeObjectForKey:@(q.handle)];break; + } + case OM_COMMIT:case OM_COMMAND_SUBMIT: { + REQUEST(OMHandle);id command=get(q.handle,@protocol(MTLCommandBuffer));if(!command || hasOpenEncoder(q.handle)){result.error=-2;break;} + if(command.status>MTLCommandBufferStatusEnqueued){result.error=-2;break;} + for(NSNumber *handle in pendingEnqueues)if(handle.unsignedLongLongValue!=q.handle && [pendingEnqueues[handle][@"command"] commandQueue]==command.commandQueue){result.error=-4;break;} + if(result.error)break;[command commit];[pendingEnqueues removeObjectForKey:@(q.handle)]; + if(header.operation==OM_COMMIT)[command waitUntilCompleted]; + else asyncCommands[@(q.handle)]=@{@"session":@(session),@"command":command}; + OMCommandResult value=observeCommand(q.handle,command,header.operation==OM_COMMIT?@"blocking-commit":@"nonblocking-submit");error=command.error; + if(value.statusMTLCommandBufferStatusCompleted || error || + (value.status==MTLCommandBufferStatusCompleted && (!value.start_ns || value.end_ns<=value.start_ns))){result.error=-3;break;} + if(header.operation==OM_COMMAND_SUBMIT)[submitted addObject:@{@"command":@(q.handle),@"session":@(session),@"actual_status":@(value.status),@"wait_until_completed_called":@NO}]; + body=[NSData dataWithBytes:&value length:sizeof(value)];break; + } + case OM_COMMAND_STATUS: { + REQUEST(OMHandle);id command=get(q.handle,@protocol(MTLCommandBuffer)); + if(!command || command.statusMTLCommandBufferStatusCompleted || error || (value.status==MTLCommandBufferStatusCompleted && (!value.start_ns || value.end_ns<=value.start_ns))){result.error=-3;break;} + body=[NSData dataWithBytes:&value length:sizeof(value)];break; + } + case OM_READ_TEXTURE: { + REQUEST(OMReadTexture);id texture=get(q.texture,@protocol(MTLTexture)); + if(!texture || texture.storageMode!=MTLStorageModeShared || q.level || !q.width || !q.height || q.x>texture.width || q.width>texture.width-q.x + || q.y>texture.height || q.height>texture.height-q.y || (texture.pixelFormat!=MTLPixelFormatR32Float && texture.pixelFormat!=MTLPixelFormatRG16Float && texture.pixelFormat!=MTLPixelFormatBGRA8Unorm && texture.pixelFormat!=MTLPixelFormatRGBA8Unorm) + || q.width>OFFLINE_METAL_MAX_REPLY/4/q.height){result.error=-2;break;} + NSMutableData *pixels=[NSMutableData dataWithLength:q.width*q.height*4]; + [texture getBytes:pixels.mutableBytes bytesPerRow:q.width*4 fromRegion:MTLRegionMake2D(q.x,q.y,q.width,q.height) mipmapLevel:0];body=pixels; + [readbacks addObject:@{@"texture":@(q.texture),@"bytes":@(pixels.length),@"sha256":hashBytes(pixels.bytes,pixels.length)}];break; + } + case OM_WRITE_TEXTURE: { + if(input.size() texture=get(q.texture,@protocol(MTLTexture)); + if(!texture || texture.storageMode!=MTLStorageModeShared || q.level || !q.width || !q.height || q.x>texture.width || q.width>texture.width-q.x + || q.y>texture.height || q.height>texture.height-q.y || (texture.pixelFormat!=MTLPixelFormatR32Float && texture.pixelFormat!=MTLPixelFormatRG16Float && texture.pixelFormat!=MTLPixelFormatBGRA8Unorm && texture.pixelFormat!=MTLPixelFormatRGBA8Unorm) + || q.width>SIZE_MAX/4 || q.stride!=q.width*4 || q.height>(input.size()-sizeof(q))/q.stride + || q.height*q.stride!=input.size()-sizeof(q)){result.error=-2;break;} + const void *pixels=input.data()+sizeof(q); + [texture replaceRegion:MTLRegionMake2D(q.x,q.y,q.width,q.height) mipmapLevel:0 withBytes:pixels bytesPerRow:q.stride]; + [uploads addObject:@{@"texture":@(q.texture),@"x":@(q.x),@"y":@(q.y),@"width":@(q.width),@"height":@(q.height), + @"bytes":@(q.stride*q.height),@"sha256":hashBytes(pixels,q.stride*q.height)}];break; + } + case OM_BUFFER: { + REQUEST(OMBuffer); + if(!q.bytes || q.bytes>OFFLINE_METAL_MAX_BUFFER || q.bytes>device.maxBufferLength || q.options){result.error=-4;break;} + result.handle=save([device newBufferWithLength:q.bytes options:MTLResourceStorageModeShared]);if(!result.handle)result.error=-3;break; + } + case OM_WRITE_BUFFER: { + if(input.size() buffer=get(q.buffer,@protocol(MTLBuffer)); + if(!buffer || !q.bytes || q.offset>buffer.length || q.bytes>buffer.length-q.offset || q.bytes!=input.size()-sizeof(q)){result.error=-2;break;} + memcpy((unsigned char *)buffer.contents+q.offset,input.data()+sizeof(q),q.bytes); + [bufferWrites addObject:@{@"buffer":@(q.buffer),@"offset":@(q.offset),@"bytes":@(q.bytes),@"sha256":hashBytes(input.data()+sizeof(q),q.bytes)}];break; + } + case OM_READ_BUFFER: { + REQUEST(OMBufferRegion);id buffer=get(q.buffer,@protocol(MTLBuffer)); + if(!buffer || !q.bytes || q.bytes>OFFLINE_METAL_MAX_REPLY || q.offset>buffer.length || q.bytes>buffer.length-q.offset){result.error=-2;break;} + body=[NSData dataWithBytes:(unsigned char *)buffer.contents+q.offset length:q.bytes]; + [bufferReads addObject:@{@"buffer":@(q.buffer),@"offset":@(q.offset),@"bytes":@(q.bytes),@"sha256":hashBytes(body.bytes,body.length)}];break; + } + case OM_VERTEX_BUFFER:case OM_FRAGMENT_BUFFER: { + REQUEST(OMSetBuffer);id encoder=getEncoder(q.encoder,@protocol(MTLRenderCommandEncoder)); + id buffer=q.buffer?get(q.buffer,@protocol(MTLBuffer)):nil; + if(!encoder || q.index>=31 || (q.buffer && !buffer) || (!buffer && q.offset) || (buffer && q.offset>=buffer.length)){result.error=-2;break;} + if(header.operation==OM_VERTEX_BUFFER)[encoder setVertexBuffer:buffer offset:q.offset atIndex:q.index]; + else [encoder setFragmentBuffer:buffer offset:q.offset atIndex:q.index];break; + } + case OM_DRAW_INDEXED: { + REQUEST(OMDrawIndexed);id encoder=getEncoder(q.encoder,@protocol(MTLRenderCommandEncoder));id buffer=get(q.buffer,@protocol(MTLBuffer)); + uint64_t stride=q.type==MTLIndexTypeUInt16?2:4; + if(!encoder || !buffer || q.primitive>MTLPrimitiveTypeTriangleStrip || q.type>MTLIndexTypeUInt32 || !q.count || + q.count>65536 || !q.instances || q.instances>1024 || q.offset%stride || q.offset>buffer.length || q.count>(buffer.length-q.offset)/stride){result.error=-2;break;} + [encoder drawIndexedPrimitives:(MTLPrimitiveType)q.primitive indexCount:q.count indexType:(MTLIndexType)q.type + indexBuffer:buffer indexBufferOffset:q.offset instanceCount:q.instances baseVertex:q.baseVertex baseInstance:q.baseInstance];break; + } + case OM_BLIT: { + REQUEST(OMHandle);id command=get(q.handle,@protocol(MTLCommandBuffer));if(!command || command.status>MTLCommandBufferStatusEnqueued || hasOpenEncoder(q.handle)){result.error=-2;break;} + result.handle=save([command blitCommandEncoder]);if(!result.handle)result.error=-3; + else openEncoders[@(result.handle)]=@{@"encoder":objects[@(result.handle)],@"command_object":command,@"command_handle":@(q.handle),@"session":@(session)};break; + } + case OM_COPY_BUFFER: { + REQUEST(OMCopyBuffer);id encoder=getEncoder(q.encoder,@protocol(MTLBlitCommandEncoder)); + id source=get(q.source,@protocol(MTLBuffer)),destination=get(q.destination,@protocol(MTLBuffer)); + if(!encoder || !source || !destination || !q.bytes || ((q.sourceOffset|q.destinationOffset|q.bytes)&3) || q.sourceOffset>source.length || q.bytes>source.length-q.sourceOffset || + q.destinationOffset>destination.length || q.bytes>destination.length-q.destinationOffset || + (source==destination && q.sourceOffset encoder=getEncoder(q.encoder,@protocol(MTLBlitCommandEncoder)); + id source=get(q.source,@protocol(MTLTexture)),destination=get(q.destination,@protocol(MTLTexture)); + if(!encoder || !source || !destination || source==destination || source.pixelFormat!=destination.pixelFormat || + source.textureType!=MTLTextureType2D || destination.textureType!=MTLTextureType2D || + q.sourceSlice || q.destinationSlice || q.sourceLevel || q.destinationLevel || q.sourceOrigin[2] || q.destinationOrigin[2] || q.sourceSize[2]!=1 || + !q.sourceSize[0] || !q.sourceSize[1] || q.sourceOrigin[0]>source.width || q.sourceSize[0]>source.width-q.sourceOrigin[0] || + q.sourceOrigin[1]>source.height || q.sourceSize[1]>source.height-q.sourceOrigin[1] || + q.destinationOrigin[0]>destination.width || q.sourceSize[0]>destination.width-q.destinationOrigin[0] || + q.destinationOrigin[1]>destination.height || q.sourceSize[1]>destination.height-q.destinationOrigin[1]){result.error=-2;break;} + if(source.pixelFormat!=MTLPixelFormatR32Float && source.pixelFormat!=MTLPixelFormatRG16Float && source.pixelFormat!=MTLPixelFormatRGBA8Unorm && source.pixelFormat!=MTLPixelFormatBGRA8Unorm){result.error=-4;break;} + [encoder copyFromTexture:source sourceSlice:0 sourceLevel:0 sourceOrigin:MTLOriginMake(q.sourceOrigin[0],q.sourceOrigin[1],0) + sourceSize:MTLSizeMake(q.sourceSize[0],q.sourceSize[1],1) toTexture:destination destinationSlice:0 destinationLevel:0 + destinationOrigin:MTLOriginMake(q.destinationOrigin[0],q.destinationOrigin[1],0)]; + [textureCopies addObject:@{@"source":@(q.source),@"destination":@(q.destination),@"session":@(session),@"format":@(source.pixelFormat), + @"source_storage":@(source.storageMode),@"destination_storage":@(destination.storageMode),@"width":@(q.sourceSize[0]),@"height":@(q.sourceSize[1])}];break; + } + case OM_VERTEX_TEXTURE:case OM_FRAGMENT_TEXTURE: { + REQUEST(OMSetTexture);id encoder=getEncoder(q.encoder,@protocol(MTLRenderCommandEncoder));id texture=q.texture?get(q.texture,@protocol(MTLTexture)):nil; + if(!encoder || q.index>=128 || (q.texture && !texture)){result.error=-2;break;} + if(header.operation==OM_VERTEX_TEXTURE)[encoder setVertexTexture:texture atIndex:q.index];else [encoder setFragmentTexture:texture atIndex:q.index]; + [textureBindings addObject:@{@"encoder":@(q.encoder),@"texture":@(q.texture),@"index":@(q.index),@"vertex":@(header.operation==OM_VERTEX_TEXTURE)}];break; + } + case OM_SCISSOR: { + REQUEST(OMScissor);id encoder=getEncoder(q.encoder,@protocol(MTLRenderCommandEncoder));NSArray *extent=renderExtents[@(q.encoder)]; + if(!encoder || !extent || !q.width || !q.height || q.x>[extent[0] unsignedLongLongValue] || q.y>[extent[1] unsignedLongLongValue] || + q.width>[extent[0] unsignedLongLongValue]-q.x || q.height>[extent[1] unsignedLongLongValue]-q.y){result.error=-2;break;} + [encoder setScissorRect:(MTLScissorRect){(NSUInteger)q.x,(NSUInteger)q.y,(NSUInteger)q.width,(NSUInteger)q.height}];break; + } + case OM_VIEWPORT: { + REQUEST(OMViewport);id encoder=getEncoder(q.encoder,@protocol(MTLRenderCommandEncoder)); + if(!encoder || !std::isfinite(q.x) || !std::isfinite(q.y) || !std::isfinite(q.width) || !std::isfinite(q.height) || + !std::isfinite(q.nearDepth) || !std::isfinite(q.farDepth) || q.width<=0 || q.height<=0 || q.nearDepth<0 || q.farDepth>1 || q.nearDepth>q.farDepth){result.error=-2;break;} + [encoder setViewport:(MTLViewport){q.x,q.y,q.width,q.height,q.nearDepth,q.farDepth}];break; + } + case OM_BLEND_COLOR: { + REQUEST(OMBlendColor);id encoder=getEncoder(q.encoder,@protocol(MTLRenderCommandEncoder)); + if(!encoder || !std::isfinite(q.red) || !std::isfinite(q.green) || !std::isfinite(q.blue) || !std::isfinite(q.alpha)){result.error=-2;break;} + [encoder setBlendColorRed:q.red green:q.green blue:q.blue alpha:q.alpha];break; + } + case OM_RELEASE: {REQUEST(OMHandle);if(!objects[@(q.handle)]){result.error=-2;break;}[objects removeObjectForKey:@(q.handle)];[renderExtents removeObjectForKey:@(q.handle)];[fenceProducers removeObjectForKey:@(q.handle)];[eventProducers removeObjectForKey:@(q.handle)];[sharedStates removeObjectForKey:@(q.handle)];break;} + case OM_FINISH:if(expectedSessions || !input.empty()){result.error=-2;break;}finishSubmitted(0,true,@"finish");finished=true;break; + default:result.error=-4;break; + } + } @catch(NSException *exception) {result.error=-5;body=[exception.reason dataUsingEncoding:NSUTF8StringEncoding];} +#undef REQUEST + if(error)body=[error.localizedDescription dataUsingEncoding:NSUTF8StringEncoding]; + if(body.length>OFFLINE_METAL_MAX_REPLY){result.error=-2;body=nil;} + if(result.error) { + NSString *reason=[[NSString alloc] initWithData:body?:[NSData data] encoding:NSUTF8StringEncoding]; + fprintf(stderr,"OFFLINE_METAL_ERROR request=%u operation=%u error=%d %s\n",request,header.operation,result.error, + reason?[[reason substringToIndex:MIN(reason.length,(NSUInteger)512)] UTF8String]:"");fflush(stderr); + } + [operations addObject:@{@"request":@(request),@"client_request":@(header.request),@"session":@(session),@"operation":@(header.operation),@"input_bytes":@(input.size()),@"error":@(result.error),@"handle":@(result.handle),@"result_bytes":@(body.length)}]; + header.operation|=OM_WIRE_RESPONSE;header.bytes=(uint32_t)(sizeof(result)+body.length); + healthy=exact(fd,&header,sizeof(header),true) && (!expectedSessions || exact(fd,&session,sizeof(session),true)) + && exact(fd,&result,sizeof(result),true) && (!body.length || exact(fd,(void *)body.bytes,body.length,true)); + } + } + close(fd);unlink(path); + // End abandoned encoding before releasing its real objects. Never commit + // interrupted buffers or invent a GPU completion to make teardown succeed. + abortEncoders(0,true,healthy?@"unclosed-at-finish":@"transport-lost"); + finishSubmitted(0,true,healthy?@"finish":@"transport-lost"); + cancelNotifications(0,true); + return @{@"host_device":device.name,@"operations":operations,@"completed_commands":completed,@"texture_readbacks":readbacks,@"texture_uploads":uploads, + @"transport_healthy":@(healthy),@"session_finished":@(finished),@"enqueued_commands":enqueued,@"pending_enqueues":@(pendingEnqueues.count), + @"aborted_encoders":abortedEncoders,@"open_encoders":@(openEncoders.count), + @"shared_event_values":sharedValues,@"shared_event_labels":sharedLabels,@"shared_event_notifications":notificationJournal,@"shared_event_exports":exportJournal,@"live_shared_exports":@(sharedExports.count), + @"events_created":eventsCreated,@"event_operations":eventOperations, + @"submitted_commands":submitted,@"command_observations":commandObservations,@"failed_commands":failedCommands,@"pending_submissions":@(asyncCommands.count), + @"function_specializations":specializations,@"fences_created":fencesCreated,@"fence_operations":fenceOperations,@"texture_hazard_modes":textureHazards, + @"buffer_uploads":bufferWrites,@"buffer_readbacks":bufferReads,@"buffer_copies":bufferCopies,@"queue_priorities":queuePriorities,@"texture_copies":textureCopies, + @"depth_stencil_states":depthStates,@"stencil_references":stencilReferences,@"attachment_passes":attachmentPasses, + @"blend_pipelines":blendPipelines,@"texture_bindings":textureBindings,@"vertex_pipelines":vertexPipelines, + @"boot_console_prefix_bytes":@(bootBytes),@"isolated_sessions":@(expectedSessions!=0),@"opened_sessions":@(openedSessions),@"closed_sessions":@(closedSessions), + @"cpu_pixel_renderer":@NO,@"guest_system_device_discovery":@NO, + @"adreno_execution":@NO,@"original_desktop":@NO}; +} diff --git a/drivers/NativeMetal/LICENSE b/drivers/NativeMetal/LICENSE new file mode 100644 index 0000000..424da21 --- /dev/null +++ b/drivers/NativeMetal/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 NativeMetal contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/drivers/NativeMetal/NativeAdrenoMetalPlugin.mm b/drivers/NativeMetal/NativeAdrenoMetalPlugin.mm new file mode 100644 index 0000000..07f8c4f --- /dev/null +++ b/drivers/NativeMetal/NativeAdrenoMetalPlugin.mm @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: MIT +// Adreno Metal plugin, stage 1: reached through the original discovery (IOAcceleratorES -> MetalPluginName -> +// NSBundle -> principal class -> initWithAcceleratorPort:), it runs the shared userspace GPU test (NativeAdrenoUserTest.h) from inside a +// real system process, under that process's sandbox, and then returns nil. No device is registered, so the original software +// rendering keeps running unchanged. +#import +#include +#include +#include +#include "NativeAdrenoUserTest.h" +#include "NativeMetalBridgePhase.h" + +@interface _MTLDevice : NSObject +@end +@interface NativeAdrenoMetalDevice : _MTLDevice +- (id)initWithAcceleratorPort:(unsigned)port; +@end + +@implementation NativeAdrenoMetalDevice +- (id)initWithAcceleratorPort:(unsigned)port { + self = [super init]; + if (!self || !port) return nil; + static bool attempted; + if (attempted) return nil; + attempted = true; + // The test runs on its own thread: a kernel call that never returns (int-042: BoFree) then stalls that thread only, not SpringBoard's + // main thread (the UI froze with the test running inline). + // The discovery code releases the accelerator port as soon as this returns nil, which is before the test thread has started: the first + // marker (and every later open) then went to a dead name and never reached the kernel (K0 in int-043..052, with the plugin file opened by + // SpringBoard). Keep one extra user reference on the port, and let the thread look the service up itself as well. + if (void *iokit = dlopen("/System/Library/Frameworks/IOKit.framework/IOKit", RTLD_NOW | RTLD_LOCAL)) { + if (auto retain = (int (*)(unsigned))dlsym(iokit, "IOObjectRetain")) retain(port); + } + struct Job {unsigned port; char name[32];}; + Job *job = new Job{port, {}}; + strlcpy(job->name, getprogname(), sizeof job->name); + pthread_t thread; + pthread_attr_t attributes; + pthread_attr_init(&attributes); + pthread_attr_setdetachstate(&attributes, PTHREAD_CREATE_DETACHED); + if (pthread_create(&thread, &attributes, [](void *arg) -> void * { + Job *j = static_cast(arg); + unsigned port = j->port; + if (void *iokit = dlopen("/System/Library/Frameworks/IOKit.framework/IOKit", RTLD_NOW | RTLD_LOCAL)) { + auto matching = (void *(*)(const char *))dlsym(iokit, "IOServiceMatching"); + auto getService = (unsigned (*)(unsigned, void *))dlsym(iokit, "IOServiceGetMatchingService"); + if (matching && getService) { + const unsigned own = getService(0, matching("IOAcceleratorES")); + if (own) port = own; + } + } + gAfterTurnip = bridgephase::run; + adrenoUserTest(port, j->name); + delete j; + return nullptr; + }, job) != 0) delete job; + pthread_attr_destroy(&attributes); + return nil; +} +@end diff --git a/drivers/NativeMetal/NativeAdrenoProbeDaemon.cpp b/drivers/NativeMetal/NativeAdrenoProbeDaemon.cpp new file mode 100644 index 0000000..a5f3b29 --- /dev/null +++ b/drivers/NativeMetal/NativeAdrenoProbeDaemon.cpp @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: MIT +// Adreno probe daemon: started by launchd from an owner job (a union view over /System/Library/LaunchDaemons, see the boot-time addon +// mount), so it runs outside every sandbox profile. It waits for the Adreno accelerator service to appear, runs the shared userspace GPU +// test once and exits. The result reaches the kernel through the driver's progress markers and SetMetadata. +#include +#include "NativeAdrenoUserTest.h" + +using MatchingFn = void *(*)(const char *); +using GetServiceFn = unsigned (*)(unsigned, void *); + +int main() { + void *iokit = dlopen("/System/Library/Frameworks/IOKit.framework/IOKit", RTLD_NOW | RTLD_LOCAL); + if (!iokit) return 1; + auto matching = (MatchingFn)dlsym(iokit, "IOServiceMatching"); + auto getService = (GetServiceFn)dlsym(iokit, "IOServiceGetMatchingService"); + if (!matching || !getService) return 2; + // The service is published a few seconds into boot (after the GPU is powered); poll for up to two minutes. + unsigned port = 0; + for (unsigned i = 0; i < 240 && !port; ++i) { + port = getService(0, matching("IOAcceleratorES")); + if (!port) usleep(500000); + } + if (!port) return 3; + return adrenoUserTest(port, "adrenoprobed") >= 0 ? 0 : 4; +} diff --git a/drivers/NativeMetal/NativeAdrenoUserTest.h b/drivers/NativeMetal/NativeAdrenoUserTest.h new file mode 100644 index 0000000..9249d49 --- /dev/null +++ b/drivers/NativeMetal/NativeAdrenoUserTest.h @@ -0,0 +1,620 @@ +// SPDX-License-Identifier: MIT +// The userspace GPU test shared by the Adreno Metal plugin (runs inside whichever system process the original Metal discovery loads it +// into, under that process's sandbox) and the launchd probe daemon (no sandbox profile). It proves the whole userspace path to the GPU: +// open the driver, move every ABI method through an admitted selector (or the mailbox page), map buffer objects write-combined or +// uncached, write an indirect buffer from userspace, submit it, wait for the fence, read the result back, and report one line to the +// kernel through SetMetadata. Progress markers are IOServiceOpen calls with type 0x4147, which the driver counts and refuses. +#pragma once +#include +#include +#include +#include +#include +#include +#include "NativeAdrenoABI.h" +#include "NativeTurnipSmoke.h" +#if __has_include("TurnipReplay.h") +#include "TurnipReplay.h" +#define NATIVE_TURNIP_REPLAY 1 +#endif +extern "C" { +#include "AdrenoPM4.h" +} + +namespace { +using IOServiceOpenFn = int (*)(unsigned, unsigned, uint32_t, unsigned *); +using IOServiceCloseFn = int (*)(unsigned); +using CallFn = int (*)(unsigned, uint32_t, const void *, size_t, void *, size_t *); +using MapFn = int (*)(unsigned, uint32_t, unsigned, uint64_t *, uint64_t *, uint32_t); +using UnmapFn = int (*)(unsigned, uint32_t, unsigned, uint64_t); + +// The sandbox admits only some external-method selector numbers for this user-client class, and which ones is decided by the +// profile, not by us. Every ABI method therefore travels through one admitted selector as {magic, method, payload}; the driver +// recognises the magic before its own selector table. Probing 0..255 finds an admitted selector without knowing the profile. +constexpr uint32_t kTunnelMagic = 0x4e544e4c, kProbe = 0xffffffffu; +struct Gpu { + unsigned conn = 0; + CallFn call = nullptr; + MapFn map = nullptr; + UnmapFn unmap = nullptr; + uint32_t selector = 0; + unsigned admitted = 0; + uint64_t lastTicks = 0; + unsigned port = 0; + IOServiceOpenFn openFn = nullptr; + IOServiceCloseFn closeFn = nullptr; + // Progress marker: an IOServiceOpen with type 0x4147. The driver counts it and refuses it, so it needs no selector, no + // mapping and no connection, and survives a sandbox that admits nothing else or a crash after it. + void mark(uint32_t value) { + unsigned c = 0; + if (openFn && port && openFn(port, mach_task_self(), 0x41470000u | (value & 0xffff), &c) == 0 && c && closeFn) closeFn(c); + } + volatile uint32_t *mailbox = nullptr; + uint32_t mailboxSeq = 0; + int mailboxMethod(uint32_t m, const void *in, size_t is, void *out, size_t os) { + if (!mailbox || is > 0x700 || os > 0x700) return -1; + if (is) memcpy(const_cast(mailbox) + 0x10, in, is); + mailbox[3] = m; + mailbox[5] = (uint32_t)is; + mailbox[6] = (uint32_t)os; + __sync_synchronize(); + const uint32_t seq = ++mailboxSeq; + mailbox[1] = seq; + __sync_synchronize(); + for (unsigned i = 0; i < 6000 && mailbox[2] != seq; ++i) usleep(500); // the kernel poller runs at about 1 kHz + if (mailbox[2] != seq) return -2; + __sync_synchronize(); + const int status = (int)mailbox[4]; + if (!status && os) memcpy(out, const_cast(mailbox) + 0x200, os); + return status; + } + int method(uint32_t m, const void *in, size_t is, void *out, size_t os) { + if (mailbox) return mailboxMethod(m, in, is, out, os); + uint8_t buffer[8 + sizeof(NativeAdrenoAbiSubmit)]; + if (is > sizeof(NativeAdrenoAbiSubmit)) return -1; + const uint32_t header[2] = {kTunnelMagic, m}; + memcpy(buffer, header, 8); + if (is) memcpy(buffer + 8, in, is); + size_t o = os; + return call(conn, selector, buffer, 8 + is, os ? out : nullptr, os ? &o : nullptr); + } + // No selector is admitted: use the mailbox page (IOConnectMapMemory64 is not covered by the selector filter). + bool openMailbox() { + uint64_t address = 0, length = 0; + if (map(conn, 0xC0DE, mach_task_self(), &address, &length, 1) != 0 || length < 4096) return false; + mailbox = reinterpret_cast(address); + return mailbox[0] == 0x4d41494c; + } + // Returns true when some selector was admitted; the first one becomes the tunnel selector. + bool probe() { + const uint32_t header[2] = {kTunnelMagic, kProbe}; + bool found = false; + for (uint32_t s = 0; s < 256; ++s) { + if (call(conn, s, header, sizeof header, nullptr, nullptr) == 0) { + if (!found) selector = s; + found = true; + ++admitted; + } + } + return found; + } +}; + +void report(Gpu &gpu, const char *process, int step, int error, int verified, int verifiedWc, uint64_t ticks) { + // SetMetadata with a bogus handle: the driver records the text before it validates the handle. + NativeAdrenoAbiMetadata meta = {}; + meta.handle = 1; + char text[NativeAdrenoAbiMetadataBytes] = {}; + snprintf(text, sizeof text, "%.9s s%d e%x v%d/%d t%llx a%u/%u", process, step, error, verified, verifiedWc, + (unsigned long long)ticks, gpu.mailbox ? 0xffffu : gpu.admitted, gpu.selector); + meta.bytes = (uint32_t)strlen(text); + memcpy(meta.text, text, meta.bytes); + gpu.method(NativeAdrenoMethodSetMetadata, &meta, sizeof meta, nullptr, 0); +} + +// Everything the Turnip backend does for a first submission, in the same order; returns the verified dword count or -1. +int runTest(Gpu &gpu, uint32_t mapFlags, int &step, int &error) { + auto check = [&](int status) { + if (status && !error) error = status; + if (!status) ++step; + gpu.mark(status ? 0x4000 | (status & 0xfff) : 0x3000 | (mapFlags == 0x401 ? 0x100 : 0) | step); + return status == 0; + }; + NativeAdrenoAbiParams params = {}; + if (!check(gpu.method(NativeAdrenoMethodQuery, nullptr, 0, ¶ms, sizeof params))) return -1; + NativeAdrenoAbiDeviceInfo info = {}; + if (!check(gpu.method(NativeAdrenoMethodGetDeviceInfo, nullptr, 0, &info, sizeof info))) return -1; + NativeAdrenoAbiBoAlloc alloc = {4096, 12, 0}; + NativeAdrenoAbiBoAllocReply ib = {}, target = {}; + if (!check(gpu.method(NativeAdrenoMethodBoAlloc, &alloc, sizeof alloc, &ib, sizeof ib)) || + !check(gpu.method(NativeAdrenoMethodBoAlloc, &alloc, sizeof alloc, &target, sizeof target))) return -1; + const uint64_t ibIova = info.va_start, tgtIova = info.va_start + 0x1000; + NativeAdrenoAbiBoMap mapIb = {ib.handle, ibIova, 1, 0, 2, 0}, mapTgt = {target.handle, tgtIova, 3, 0, 2, 0}; + NativeAdrenoAbiBoMapReply mr = {}; + if (!check(gpu.method(NativeAdrenoMethodBoMap, &mapIb, sizeof mapIb, &mr, sizeof mr)) || + !check(gpu.method(NativeAdrenoMethodBoMap, &mapTgt, sizeof mapTgt, &mr, sizeof mr))) return -1; + NativeAdrenoAbiBoCpuMap cmIb = {ib.handle}, cmTgt = {target.handle}; + NativeAdrenoAbiBoCpuMapReply crIb = {}, crTgt = {}; + if (!check(gpu.method(NativeAdrenoMethodBoCpuMap, &cmIb, sizeof cmIb, &crIb, sizeof crIb)) || + !check(gpu.method(NativeAdrenoMethodBoCpuMap, &cmTgt, sizeof cmTgt, &crTgt, sizeof crTgt))) return -1; + uint64_t ibAddr = 0, ibLen = 0, tgtAddr = 0, tgtLen = 0; + if (!check(gpu.map(gpu.conn, crIb.token, mach_task_self(), &ibAddr, &ibLen, mapFlags)) || + !check(gpu.map(gpu.conn, crTgt.token, mach_task_self(), &tgtAddr, &tgtLen, mapFlags))) return -1; + if (ibLen < 4096 || tgtLen < 4096) {error = 0xbad1; return -1;} + volatile uint32_t *iw = reinterpret_cast(ibAddr); + volatile uint32_t *tw = reinterpret_cast(tgtAddr); + for (unsigned i = 0; i < 1024; ++i) tw[i] = 0xdeadbeef; + const uint32_t values[3] = {0x0adc0de1u, 0xfeed0002u, 0x600d0003u}; + uint32_t words[12]; + for (unsigned i = 0; i < 3; ++i) { + if (AdrenoPM4Type7(0x3d, 3, &words[i * 4]) != AdrenoPM4Success) {error = 0xbad2; return -1;} + words[i * 4 + 1] = uint32_t(tgtIova + i * 4); + words[i * 4 + 2] = uint32_t(tgtIova >> 32); + words[i * 4 + 3] = values[i]; + } + for (unsigned i = 0; i < 12; ++i) iw[i] = words[i]; + __asm__ volatile("dsb sy" ::: "memory"); // the mapping is non-cacheable and the GPU is an outer-shareable observer + NativeAdrenoAbiSubmit submit = {}; + submit.buffer_count = 1; + submit.buffers[0] = {ib.handle, ibIova, sizeof words}; + NativeAdrenoAbiSubmitReply sr = {}; + if (!check(gpu.method(NativeAdrenoMethodSubmit, &submit, sizeof submit, &sr, sizeof sr))) return -1; + NativeAdrenoAbiWaitFence wait = {sr.fence, 200000000ull}; + if (!check(gpu.method(NativeAdrenoMethodWaitFence, &wait, sizeof wait, nullptr, 0))) return -1; + __asm__ volatile("dsb sy" ::: "memory"); + const int verified = (tw[0] == values[0]) + (tw[1] == values[1]) + (tw[2] == values[2]) + (tw[3] == 0xdeadbeef); + NativeAdrenoAbiTimestamp ts = {}; + uint64_t ticks = 0; + if (check(gpu.method(NativeAdrenoMethodGetTimestamp, nullptr, 0, &ts, sizeof ts))) ticks = ts.ticks; + gpu.unmap(gpu.conn, crIb.token, mach_task_self(), ibAddr); + gpu.unmap(gpu.conn, crTgt.token, mach_task_self(), tgtAddr); + NativeAdrenoAbiBoCpuUnmap ui = {ib.handle, crIb.token, 0}, ut = {target.handle, crTgt.token, 0}; + check(gpu.method(NativeAdrenoMethodBoCpuUnmap, &ui, sizeof ui, nullptr, 0)); + check(gpu.method(NativeAdrenoMethodBoCpuUnmap, &ut, sizeof ut, nullptr, 0)); + NativeAdrenoAbiBoFree fi = {ib.handle}, ft = {target.handle}; + check(gpu.method(NativeAdrenoMethodBoFree, &fi, sizeof fi, nullptr, 0)); + check(gpu.method(NativeAdrenoMethodBoFree, &ft, sizeof ft, nullptr, 0)); + gpu.lastTicks = ticks; + return verified; +} +} + +// A report line as a marker stream (0x7fff, then 0x70cc per character): works while another connection holds the driver's single slot. +static void streamText(Gpu &gpu, const char *text) { + gpu.mark(0x7fff); + for (unsigned i = 0; i < 63 && text[i]; ++i) gpu.mark(0x7000 | (uint8_t)text[i]); +} + +// Line into report (GPU7) instead of report2 (GPU8). +__attribute__((unused)) static void streamText7(Gpu &gpu, const char *text) { + gpu.mark(0x7ffe); + for (unsigned i = 0; i < 63 && text[i]; ++i) gpu.mark(0x7000 | (uint8_t)text[i]); +} + +// Remap probe (int-064): does the CP read the new pages after a GPU address is moved to other physical pages? int-063's W1 run (the +// replay IB on fresh buffers at the same addresses) stopped both CP threads in the SQE's unknown-opcode trap, i.e. on wrong data. A +// CP_MEM_TO_MEM from a remapped page into a result page reports what the CP sees instead of executing it, so no variant can stop the CP: +// v0 first mapping, v1 remap to a page allocated before the old one is freed (so the old page keeps its old pattern), +// v2 as v1 with a UCHE invalidate (CP_EVENT_WRITE7 0x31) and CP_WAIT_FOR_IDLE before the copy, +// v3 as v1 but remapped while the CP is busy spinning on a flag (CP_WAIT_REG_MEM), then released: GX stays up across the TLB invalidate. +// Each result: '1' the new pattern, 'o' the previous mapping's pattern, 'z' zero, 'd' result not written, 'g' other, 'x' setup failed, +// 't' fence timeout. Reported as "P" in front of the replay line. The addresses (0x5f000000..) are never used elsewhere. +struct ProbeBo { + uint64_t handle = 0, address = 0; + uint32_t token = 0; + volatile uint32_t *words = nullptr; +}; +static int probeAlloc(Gpu &gpu, ProbeBo &b, uint32_t fill) { + NativeAdrenoAbiBoAlloc alloc = {4096, 12, 0}; + NativeAdrenoAbiBoAllocReply reply = {}; + if (int e = gpu.method(NativeAdrenoMethodBoAlloc, &alloc, sizeof alloc, &reply, sizeof reply)) return e; + b.handle = reply.handle; + NativeAdrenoAbiBoCpuMap cm = {b.handle}; + NativeAdrenoAbiBoCpuMapReply cr = {}; + if (int e = gpu.method(NativeAdrenoMethodBoCpuMap, &cm, sizeof cm, &cr, sizeof cr)) return e; + b.token = cr.token; + uint64_t length = 0; + if (int e = gpu.map(gpu.conn, cr.token, mach_task_self(), &b.address, &length, 0x101)) return e; + if (length < 4096) return 0xbad5; + b.words = reinterpret_cast(b.address); + for (unsigned i = 0; i < 1024; ++i) b.words[i] = fill; + __asm__ volatile("dsb sy" ::: "memory"); + return 0; +} +static int probeMap(Gpu &gpu, const ProbeBo &b, uint64_t iova) { + NativeAdrenoAbiBoMap map = {b.handle, iova, 3, 0, 2, 0}; + NativeAdrenoAbiBoMapReply mr = {}; + return gpu.method(NativeAdrenoMethodBoMap, &map, sizeof map, &mr, sizeof mr); +} +static void probeFree(Gpu &gpu, ProbeBo &b) { + if (!b.handle) return; + if (b.address) { + gpu.unmap(gpu.conn, b.token, mach_task_self(), b.address); + NativeAdrenoAbiBoCpuUnmap cu = {b.handle, b.token, 0}; + gpu.method(NativeAdrenoMethodBoCpuUnmap, &cu, sizeof cu, nullptr, 0); + } + NativeAdrenoAbiBoFree f = {b.handle}; + gpu.method(NativeAdrenoMethodBoFree, &f, sizeof f, nullptr, 0); + b = ProbeBo(); +} +static char gProbe[6] = "P----"; +__attribute__((unused)) static void remapProbe(Gpu &gpu) { + constexpr uint64_t kSource = 0x5f000000ull, kResult = 0x5f010000ull, kIb = 0x5f020000ull, kFlag = 0x5f030000ull; + const uint32_t patterns[4] = {0x0a1a1a10u, 0x0b2b2b20u, 0x0c3c3c30u, 0x0d4d4d40u}; + ProbeBo ib, result, flag, source; + bool stuck = false; + if (probeAlloc(gpu, ib, 0) || probeAlloc(gpu, result, 0xdeadbeef) || probeAlloc(gpu, flag, 0) || + probeAlloc(gpu, source, patterns[0]) || probeMap(gpu, ib, kIb) || probeMap(gpu, result, kResult) || + probeMap(gpu, flag, kFlag) || probeMap(gpu, source, kSource)) { + for (unsigned v = 1; v < 5; ++v) gProbe[v] = 'x'; + probeFree(gpu, source); probeFree(gpu, flag); probeFree(gpu, result); probeFree(gpu, ib); + return; + } + for (unsigned v = 0; v < 4 && !stuck; ++v) { + gpu.mark(0x6d00 | v); + ProbeBo next; + if (v) { + // The new page exists before the old one is freed, so the two differ and the old one still holds the old pattern. + if (probeAlloc(gpu, next, patterns[v])) {gProbe[1 + v] = 'x'; probeFree(gpu, next); continue;} + } + uint32_t words[32], n = 0; + auto packet = [&](uint32_t opcode, std::initializer_list payload) { + if (AdrenoPM4Type7(opcode, uint32_t(payload.size()), &words[n]) != AdrenoPM4Success) return false; + ++n; + for (uint32_t w : payload) words[n++] = w; + return true; + }; + bool ok = true; + if (v == 3) ok = packet(0x3c, {3u | (1u << 4), uint32_t(kFlag), uint32_t(kFlag >> 32), 1u, 0xffffffffu, 16u}); // wait flag == 1 + if (v == 2) ok = packet(0x46, {0x31u}) && packet(0x26, {}); + ok = ok && packet(0x73, {0u, uint32_t(kResult), uint32_t(kResult >> 32), uint32_t(kSource), uint32_t(kSource >> 32)}); + if (!ok) {gProbe[1 + v] = 'x'; probeFree(gpu, next); continue;} + for (uint32_t i = 0; i < n; ++i) ib.words[i] = words[i]; + result.words[0] = 0xdeadbeef; + flag.words[0] = 0; + __asm__ volatile("dsb sy" ::: "memory"); + if (v && v != 3) { + probeFree(gpu, source); + if (probeMap(gpu, next, kSource)) {gProbe[1 + v] = 'x'; probeFree(gpu, next); break;} + source = next; + } + NativeAdrenoAbiSubmit submit = {}; + submit.buffer_count = 1; + submit.buffers[0] = {ib.handle, kIb, uint64_t(n) * 4}; + NativeAdrenoAbiSubmitReply sr = {}; + if (gpu.method(NativeAdrenoMethodSubmit, &submit, sizeof submit, &sr, sizeof sr)) {gProbe[1 + v] = 'x'; probeFree(gpu, next); continue;} + if (v == 3) { + usleep(20000); // the CP is spinning on the flag + probeFree(gpu, source); + if (probeMap(gpu, next, kSource)) {gProbe[1 + v] = 'x'; probeFree(gpu, next); stuck = true; break;} + source = next; + flag.words[0] = 1; + __asm__ volatile("dsb sy" ::: "memory"); + } + int waited = -1; + for (unsigned t = 0; t < 10 && waited; ++t) { + NativeAdrenoAbiWaitFence wait = {sr.fence, 200000000ull}; + waited = gpu.method(NativeAdrenoMethodWaitFence, &wait, sizeof wait, nullptr, 0); + } + if (waited) {gProbe[1 + v] = 't'; stuck = true; break;} + const uint32_t got = result.words[0], want = patterns[v]; + gProbe[1 + v] = got == want ? '1' : (v && got == patterns[v - 1]) ? 'o' : got == 0 ? 'z' : got == 0xdeadbeef ? 'd' : 'g'; + } + // A stuck CP may still reference the buffers. + if (!stuck) {probeFree(gpu, source); probeFree(gpu, flag); probeFree(gpu, result); probeFree(gpu, ib);} +} + +#ifdef NATIVE_TURNIP_REPLAY +// Prefix replay of Turnip's first submission (recorded on the host by scripts/gen-turnip-replay.py with the tablet's 4 KiB page size, so +// the GPU addresses are the ones Turnip itself uses on the tablet): its buffers are mapped at the recorded GPU addresses and growing +// prefixes of its IB, cut after each TYPE7 packet, are submitted one by one, each waited for up to 2 s. Reported on GPU7 as +// "B / t o c e", markers +// 0x6b00 | thread variant, then 0x6a00 | prefix. +struct Replay { + static constexpr unsigned kCount = sizeof turnipreplay::kBuffers / sizeof turnipreplay::kBuffers[0]; + uint64_t handles[kCount] = {}; + unsigned ibIndex = kCount; + bool hung = false; +}; +// Allocates and maps every recorded buffer and fills it through a CPU mapping with the given flags: 0x101 (Device, aligned volatile +// stores, as the prefix replay always did) or 0x401 (write combine filled with memcpy/memset, as Turnip fills its BOs). +static int replayLoad(Gpu &gpu, Replay &r, uint32_t mapFlags) { + using namespace turnipreplay; + for (unsigned i = 0; i < Replay::kCount; ++i) { + const Buffer &b = kBuffers[i]; + NativeAdrenoAbiBoAlloc alloc = {b.bytes, 12, 0}; + NativeAdrenoAbiBoAllocReply reply = {}; + if (int e = gpu.method(NativeAdrenoMethodBoAlloc, &alloc, sizeof alloc, &reply, sizeof reply)) return e; + r.handles[i] = reply.handle; + NativeAdrenoAbiBoMap map = {reply.handle, b.iova, 3, 0, 2, 0}; + NativeAdrenoAbiBoMapReply mr = {}; + if (int e = gpu.method(NativeAdrenoMethodBoMap, &map, sizeof map, &mr, sizeof mr)) return e; + NativeAdrenoAbiBoCpuMap cm = {reply.handle}; + NativeAdrenoAbiBoCpuMapReply cr = {}; + if (int e = gpu.method(NativeAdrenoMethodBoCpuMap, &cm, sizeof cm, &cr, sizeof cr)) return e; + uint64_t address = 0, length = 0; + if (int e = gpu.map(gpu.conn, cr.token, mach_task_self(), &address, &length, mapFlags)) return e; + if (length < b.bytes) return 0xbad3; + if (mapFlags == 0x401) { + memcpy(reinterpret_cast(address), b.data, b.words * 4); + memset(reinterpret_cast(address) + b.words * 4, 0, b.bytes - b.words * 4); + } else { + volatile uint32_t *w = reinterpret_cast(address); + for (uint64_t k = 0; k < b.bytes / 4; ++k) w[k] = k < b.words ? b.data[k] : 0; + } + __asm__ volatile("dsb sy" ::: "memory"); + gpu.unmap(gpu.conn, cr.token, mach_task_self(), address); + NativeAdrenoAbiBoCpuUnmap cu = {reply.handle, cr.token, 0}; + gpu.method(NativeAdrenoMethodBoCpuUnmap, &cu, sizeof cu, nullptr, 0); + if (b.iova <= kIbIova && kIbIova < b.iova + b.bytes) r.ibIndex = i; + } + return r.ibIndex == Replay::kCount ? 0xbad4 : 0; +} +// Submits the first dwords of the IB and waits up to 2 s; a timeout leaves the buffers referenced by the stuck CP. +static int replayRun(Gpu &gpu, Replay &r, uint32_t dwords) { + NativeAdrenoAbiSubmit submit = {}; + submit.buffer_count = 1; + submit.buffers[0] = {r.handles[r.ibIndex], turnipreplay::kIbIova, uint64_t(dwords) * 4}; + NativeAdrenoAbiSubmitReply sr = {}; + if (int e = gpu.method(NativeAdrenoMethodSubmit, &submit, sizeof submit, &sr, sizeof sr)) return e; + int waited = -1; + for (unsigned t = 0; t < 10 && waited; ++t) { + NativeAdrenoAbiWaitFence wait = {sr.fence, 200000000ull}; + waited = gpu.method(NativeAdrenoMethodWaitFence, &wait, sizeof wait, nullptr, 0); + } + if (waited) r.hung = true; + return waited; +} +// A stuck CP still references the buffers; they are only released when everything retired. +static void replayFree(Gpu &gpu, Replay &r) { + if (r.hung) return; + for (uint64_t &h : r.handles) + if (h) {NativeAdrenoAbiBoFree f = {h}; gpu.method(NativeAdrenoMethodBoFree, &f, sizeof f, nullptr, 0); h = 0;} +} +// One whole-IB control run on fresh buffers at the same GPU addresses: '1' retired, '0' stuck, 'x' setup failed. Marker 0x6c00 | tag. +static char replayControl(Gpu &gpu, uint32_t mapFlags, uint32_t tag, bool &hung) { + Replay r; + gpu.mark(0x6c00 | tag); + char result = 'x'; + if (!replayLoad(gpu, r, mapFlags)) result = replayRun(gpu, r, turnipreplay::kCuts[sizeof turnipreplay::kCuts / sizeof turnipreplay::kCuts[0] - 1]) ? (r.hung ? '0' : 'x') : '1'; + hung = r.hung; + replayFree(gpu, r); + return result; +} +static char gReplayControls[4] = "---"; // W1 W2 N, appended to the B line +static char gReplayLine[64]; +// Whole-IB runs before any prefix (int-065). int-064's remap probe read the new pages in every variant (P1111), yet the whole IB on fresh +// buffers after the prefix series stopped the CP again (W1). These runs separate "a fresh first run stops" (prefix runs had left GPU-written +// memory behind) from "a second whole run stops": E, r1 the whole IB on fresh buffers before anything else, r2 the same +// buffers again, r3 fresh buffers at the same addresses (W1's case); '1' retired, '0' stuck, 'x' setup failed, '-' not run. After r1 +// retires, "d" lists how many words of each buffer differ from the recording and "f" the first changed word outside the fill target, as +// @=, so the memory the GPU wrote is known. +static char gEarly[48] = "E---"; +__attribute__((unused)) static bool earlyRuns(Gpu &gpu) { + using namespace turnipreplay; + const uint32_t whole = kCuts[sizeof kCuts / sizeof kCuts[0] - 1]; + char diff[40] = ""; + Replay r; + gpu.mark(0x6e01); + if (replayLoad(gpu, r, 0x101)) {gEarly[1] = 'x'; replayFree(gpu, r); return true;} + gEarly[1] = replayRun(gpu, r, whole) ? (r.hung ? '0' : 'x') : '1'; + if (gEarly[1] == '1') { + unsigned counts[Replay::kCount] = {}, firstBuffer = 0, firstWord = 0; + uint32_t firstValue = 0; + bool found = false; + for (unsigned i = 0; i < Replay::kCount; ++i) { + const Buffer &b = kBuffers[i]; + NativeAdrenoAbiBoCpuMap cm = {r.handles[i]}; + NativeAdrenoAbiBoCpuMapReply cr = {}; + if (gpu.method(NativeAdrenoMethodBoCpuMap, &cm, sizeof cm, &cr, sizeof cr)) continue; + uint64_t address = 0, length = 0; + if (!gpu.map(gpu.conn, cr.token, mach_task_self(), &address, &length, 0x101)) { + const volatile uint32_t *w = reinterpret_cast(address); + for (uint64_t k = 0; k < b.bytes / 4 && k < length / 4; ++k) { + const uint32_t want = k < b.words ? b.data[k] : 0; + if (w[k] == want) continue; + ++counts[i]; + // The fill target is the recording's empty buffer; its change is expected. + if (!found && b.words) {found = true; firstBuffer = i; firstWord = unsigned(k); firstValue = w[k];} + } + gpu.unmap(gpu.conn, cr.token, mach_task_self(), address); + } + NativeAdrenoAbiBoCpuUnmap cu = {r.handles[i], cr.token, 0}; + gpu.method(NativeAdrenoMethodBoCpuUnmap, &cu, sizeof cu, nullptr, 0); + } + int at = snprintf(diff, sizeof diff, " d"); + for (unsigned i = 0; i < Replay::kCount && at < int(sizeof diff); ++i) + at += snprintf(diff + at, sizeof diff - at, "%s%x", i ? "," : "", counts[i]); + if (found && at < int(sizeof diff)) snprintf(diff + at, sizeof diff - at, " f%u@%x=%x", firstBuffer, firstWord, firstValue); + } + if (gEarly[1] == '1') {gpu.mark(0x6e02); gEarly[2] = replayRun(gpu, r, whole) ? (r.hung ? '0' : 'x') : '1';} + const bool stuck = r.hung; + replayFree(gpu, r); + if (!stuck && gEarly[2] == '1') { + Replay fresh; + gpu.mark(0x6e03); + gEarly[3] = replayLoad(gpu, fresh, 0x101) ? 'x' : replayRun(gpu, fresh, whole) ? (fresh.hung ? '0' : 'x') : '1'; + replayFree(gpu, fresh); + if (fresh.hung) gEarly[3] = '0'; + } + snprintf(gEarly + 4, sizeof gEarly - 4, "%s", diff); + streamText7(gpu, gEarly); + return gEarly[1] == '1' && gEarly[2] == '1' && gEarly[3] == '1'; +} +// Fresh-buffer prefix scan (int-066). int-065: the whole IB on fresh buffers, before any other replay, stopped the CP (E0--), while the +// prefix series on one set of buffers retires every prefix. Here every prefix runs on freshly loaded buffers, so memory written by +// earlier runs cannot help it; the first prefix that stops names the packet that needs that memory. When every prefix retires, CP state +// left by the earlier runs is what helped, and the whole IB then runs once more on fresh buffers (e). Reported on GPU7 as +// "F / o c e". +static bool freshScan(Gpu &gpu) { + using namespace turnipreplay; + constexpr unsigned kPrefixes = sizeof kCuts / sizeof kCuts[0]; + unsigned retired = 0; + bool stuck = false; + char whole = '-'; + for (unsigned p = 0; p < kPrefixes && !stuck; ++p) { + Replay r; + gpu.mark(0x6f80 | p); + int e = replayLoad(gpu, r, 0x101); + if (!e) e = replayRun(gpu, r, kCuts[p]); + stuck = r.hung; + replayFree(gpu, r); + if (e) break; + ++retired; + } + if (retired == kPrefixes) { + Replay r; + gpu.mark(0x6fa0); + whole = replayLoad(gpu, r, 0x101) ? 'x' : replayRun(gpu, r, kCuts[kPrefixes - 1]) ? (r.hung ? '0' : 'x') : '1'; + stuck = r.hung; + replayFree(gpu, r); + } + char text[64]; + snprintf(text, sizeof text, "F %u/%u o%x c%u e%c", retired, kPrefixes, retired < kPrefixes ? kCutOps[retired] : 0, + retired < kPrefixes ? kCuts[retired] : 0, whole); + streamText7(gpu, text); + return !stuck && whole == '1'; +} +static bool replayPhase(Gpu &gpu) { + using namespace turnipreplay; + constexpr unsigned kPrefixes = sizeof kCuts / sizeof kCuts[0]; + Replay r; + unsigned retired = 0, variants = 0; + int error = replayLoad(gpu, r, 0x101); + // The first packet is CP_THREAD_CONTROL(BOTH) and int-060 stopped on it alone. Before the prefixes, the same two dwords run with + // BR only (1) and BV only (2), so a hang names the thread: t. + auto patch = [&](uint32_t value) { + const Buffer &b = kBuffers[r.ibIndex]; + NativeAdrenoAbiBoCpuMap cm = {r.handles[r.ibIndex]}; + NativeAdrenoAbiBoCpuMapReply cr = {}; + if (int e = gpu.method(NativeAdrenoMethodBoCpuMap, &cm, sizeof cm, &cr, sizeof cr)) return e; + uint64_t address = 0, length = 0; + if (int e = gpu.map(gpu.conn, cr.token, mach_task_self(), &address, &length, 0x101)) return e; + reinterpret_cast(address)[(kIbIova - b.iova) / 4 + 1] = value; + __asm__ volatile("dsb sy" ::: "memory"); + gpu.unmap(gpu.conn, cr.token, mach_task_self(), address); + NativeAdrenoAbiBoCpuUnmap cu = {r.handles[r.ibIndex], cr.token, 0}; + gpu.method(NativeAdrenoMethodBoCpuUnmap, &cu, sizeof cu, nullptr, 0); + return 0; + }; + const uint32_t original = r.ibIndex < Replay::kCount ? kBuffers[r.ibIndex].data[(kIbIova - kBuffers[r.ibIndex].iova) / 4 + 1] : 0; + for (uint32_t thread = 1; thread <= 2 && !error; ++thread) { + gpu.mark(0x6b00 | thread); + if (!(error = patch(thread)) && !(error = replayRun(gpu, r, 2))) ++variants; + } + if (!error) error = patch(original); + for (unsigned p = 0; p < kPrefixes && !error; ++p) { + gpu.mark(0x6a00 | p); + if ((error = replayRun(gpu, r, kCuts[p]))) break; + ++retired; + } + replayFree(gpu, r); + // Turnip's own fill of this IB stopped in int-062 while this replay retired. The control runs separate the remaining differences, each + // on fresh buffers at the same addresses: W1 Device-mapped stores (address reuse after a free), W2 write-combine memcpy (how Turnip + // fills its BOs); the caller adds N, a second connection (a second context entry on a CP that already ran work). + bool stuck = r.hung; + if (retired == kPrefixes && !stuck) gReplayControls[0] = replayControl(gpu, 0x101, 1, stuck); + if (retired == kPrefixes && !stuck) gReplayControls[1] = replayControl(gpu, 0x401, 2, stuck); + snprintf(gReplayLine, sizeof gReplayLine, "%.4s B %u/%u t%u o%x c%u e%x ", gEarly, retired, kPrefixes, variants, + retired < kPrefixes ? kCutOps[retired] : 0, retired < kPrefixes ? kCuts[retired] : 0, unsigned(error)); + char text[64]; + snprintf(text, sizeof text, "%s%s", gReplayLine, gReplayControls); + streamText7(gpu, text); + return retired == kPrefixes && !stuck; +} +// N: the whole IB on a second connection, whose first submission carries the A7XX context entry again (Turnip's connection does that +// after the replay). Returns false when the CP stopped. +static bool replaySecondClient(Gpu &gpu) { + unsigned conn = 0; + if (gpu.openFn(gpu.port, mach_task_self(), 0x4147, &conn) != 0 || !conn) {gReplayControls[2] = 'o'; return true;} + const unsigned first = gpu.conn; + volatile uint32_t *mailbox = gpu.mailbox; + gpu.conn = conn; + gpu.mailbox = nullptr; + bool ok = !mailbox || gpu.openMailbox(); + bool stuck = false; + gReplayControls[2] = ok ? replayControl(gpu, 0x101, 3, stuck) : 'm'; + gpu.closeFn(conn); + gpu.conn = first; + gpu.mailbox = mailbox; + char text[64]; + snprintf(text, sizeof text, "%s%s", gReplayLine, gReplayControls); + streamText7(gpu, text); + return !stuck; +} +#endif + +// Run after a passing Turnip phase by the plugin (the Metal bridge self-test); null in the probe daemon. +static void (*gAfterTurnip)(Gpu &gpu, const char *turnipPath); + +// Turnip phase: only where libvulkan_freedreno.dylib sits next to this code (the plugin bundle), never in the probe daemon. +static void turnipPhase(Gpu &gpu) { + char path[1024]; + if (!turnip::libraryPath(path, sizeof path) || access(path, R_OK) != 0) return; + gpu.mark(0x6000); + auto marker = [&](uint32_t value) { gpu.mark(value); }; + const turnip::Result result = turnip::runWatched(marker, [&](const char *text) { streamText(gpu, text); }, path); + char text[NativeAdrenoAbiMetadataBytes] = {}; + snprintf(text, sizeof text, "T s%d r%d v%d c%d%s g%d/%d%s %.30s", result.stage, result.vkResult, result.verified, result.computeWords, + result.computeStage ? "!" : "", result.renderTriangle, result.renderOther, result.renderStage ? "!" : "", turnip::gFirstError); + streamText(gpu, text); + gpu.mark(result.ok ? 0x6fff : 0x6ffe); + // Also when results are wrong but every Vulkan step ran: one boot then shows both layers. + if (result.stage == 0 && result.vkResult == 0 && gAfterTurnip) gAfterTurnip(gpu, path); +} + +// Returns the GPU-verified dword count of the uncached run, or a negative number; every outcome is also reported to the kernel. +static int adrenoUserTest(unsigned port, const char *process) { + void *iokit = dlopen("/System/Library/Frameworks/IOKit.framework/IOKit", RTLD_NOW | RTLD_LOCAL); + if (!iokit) return -1; + auto open = (IOServiceOpenFn)dlsym(iokit, "IOServiceOpen"); + auto close = (IOServiceCloseFn)dlsym(iokit, "IOServiceClose"); + Gpu gpu; + gpu.call = (CallFn)dlsym(iokit, "IOConnectCallStructMethod"); + gpu.map = (MapFn)dlsym(iokit, "IOConnectMapMemory64"); + gpu.unmap = (UnmapFn)dlsym(iokit, "IOConnectUnmapMemory64"); + gpu.port = port; + gpu.openFn = open; + gpu.closeFn = close; + gpu.mark(1); + if (!open || !close || !gpu.call || !gpu.map || !gpu.unmap) return -1; + gpu.mark(2); + // 0x4147 ('AG') is the open type the driver serves; the original IOGPU library's type-1 opens are refused. + const int openResult = open(port, mach_task_self(), 0x4147, &gpu.conn); + if (openResult != 0 || !gpu.conn) { + // The exact kern_return_t, as two markers (high and low half), after the failure marker. + gpu.mark(0x4f01); + gpu.mark(((uint32_t)openResult >> 16) & 0xffff ? ((uint32_t)openResult >> 16) & 0xffff : 0xffff); + gpu.mark((uint32_t)openResult & 0xffff ? (uint32_t)openResult & 0xffff : 0xffff); + return -1; + } + gpu.mark(3); + const bool probed = gpu.probe(); + gpu.mark(0x1000 | (gpu.admitted & 0xfff)); + bool mailboxOk = false; + if (!probed) {mailboxOk = gpu.openMailbox(); gpu.mark(mailboxOk ? 0x2001 : 0x2000);} + if (!probed && !mailboxOk) {close(gpu.conn); return -2;} // neither channel works: the driver's E row shows what reached it + // mapFlags are what the CLIENT asks for: IOUserClient::mapClientMemory64 lets them override the cache mode the driver returns, so + // the cache bits must be set here. 0x101 = anywhere + inhibit cache (aligned dword accesses only), 0x401 = anywhere + write + // combine (Normal non-cacheable; what a Vulkan driver needs, since it uses memcpy and unaligned stores). + int step = 0, error = 0; + const int verified = runTest(gpu, 0x101, step, error); + int stepWc = 0, errorWc = 0; + const int verifiedWc = error ? -2 : runTest(gpu, 0x401, stepWc, errorWc); + report(gpu, process, step + stepWc, error ? error : errorWc, verified, verifiedWc, gpu.lastTicks); + gpu.mark(0x5000 | (verified & 0xff)); + bool replayed = true; +#ifdef NATIVE_TURNIP_REPLAY + // Plugin only (the Turnip library sits next to it): a stuck replay leaves the CP stopped, so Turnip is not started after it. + char turnipPath[1024]; + if (verified == 4 && verifiedWc == 4 && turnip::libraryPath(turnipPath, sizeof turnipPath) && access(turnipPath, R_OK) == 0) { + replayed = freshScan(gpu) && replayPhase(gpu); + } +#endif + close(gpu.conn); +#ifdef NATIVE_TURNIP_REPLAY + if (replayed && verified == 4 && verifiedWc == 4) replayed = replaySecondClient(gpu); +#endif + if (verified == 4 && verifiedWc == 4 && replayed) turnipPhase(gpu); + return verified; +} diff --git a/drivers/NativeMetal/NativeBridgeMetalFrontend.h b/drivers/NativeMetal/NativeBridgeMetalFrontend.h new file mode 100644 index 0000000..c584048 --- /dev/null +++ b/drivers/NativeMetal/NativeBridgeMetalFrontend.h @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: MIT +// Entry points of the native Metal frontend over the bridge (NativeBridgeMetalFrontend.mm). +#pragma once +#include +#include +#ifdef __OBJC__ +#import +extern "C" { +// Opens the process's one device: loader exports vkGetInstanceProcAddr (Turnip), cache is a writable shader-cache directory, registryID is the +// accelerator service's registry entry. missing (may be null) is told the class and selector of every unimplemented call. +id native_bridge_metal_open(const char *loader, const char *cache, uint64_t registryID, void (*missing)(const char *, const char *)); +// Bare AIR bitcode as a library (the bridge's fixtures; Metal itself accepts only metallibs). +id native_bridge_metal_library_with_air(const void *bytes, size_t length); +} +#endif diff --git a/drivers/NativeMetal/NativeBridgeMetalFrontend.mm b/drivers/NativeMetal/NativeBridgeMetalFrontend.mm new file mode 100644 index 0000000..7e3c88f --- /dev/null +++ b/drivers/NativeMetal/NativeBridgeMetalFrontend.mm @@ -0,0 +1,809 @@ +// SPDX-License-Identifier: MIT +// Native Metal API frontend over the bridge (bridge/NativeMetalObjects.h): the metal2vulkan runtime on Turnip, on the Adreno 740 through +// NativeAdreno. Same shape as the offline frontend: the device is a dynamically allocated subclass of the target's own _MTLDevice that +// only adds this file's implementations; every other object is an own class answering the Metal protocol methods it implements. +// Every result comes from the runtime and the GPU: a refused call returns nil/NO with the runtime's reason, an encoder call the runtime +// refuses fails its command buffer (status Error, handlers run) instead of being dropped. A selector this frontend does not implement is +// reported through the missing-selector hook before the usual exception, so a hardware run names the gap. +// Shared events work within this process (no cross-process handle yet). Not yet implemented here: heaps, argument buffers, indirect commands, drawables/presentation. +#import +#import +#import +#import +#include +#include +#include "bridge/NativeMetalObjects.h" +#include "NativeBridgeMetalFrontend.h" + +extern "C" void sys_dcache_flush(void *start, size_t length); + +static NmbDevice *gDevice; +static id gDeviceObject; +static uint64_t gRegistryID; +static NSString *gName; +static void (*gMissing)(const char *, const char *); + +static NSError *nbError(NSString *why) { + return [NSError errorWithDomain:@"org.blu-sharky.native-metal" code:-1 userInfo:@{NSLocalizedDescriptionKey:why?:@"refused"}]; +} +static NSString *lastError() {return [NSString stringWithUTF8String:nmb_last_error()];} + +@interface NBObject : NSObject +@property(atomic,copy) NSString *label; +@end +@implementation NBObject +- (id)device {return gDeviceObject;} +- (void)doesNotRecognizeSelector:(SEL)selector { + if(gMissing)gMissing(class_getName(object_getClass(self)),sel_getName(selector)); + [super doesNotRecognizeSelector:selector]; +} +@end + +// ---- resources ---- +@interface NBBuffer : NBObject +@property(nonatomic) NmbBuffer *handle; +@property(nonatomic) MTLResourceOptions options; +@property(nonatomic,copy) void (^deallocator)(void *,NSUInteger); +@property(nonatomic) void *wrapped; +@end +@interface NBTexture : NBObject +@property(nonatomic) NmbTexture *handle; +@property(nonatomic) NmbTextureDescriptor descriptor; +@property(nonatomic,strong) NBBuffer *buffer; +@property(nonatomic) IOSurfaceRef surface; +@property(nonatomic) NSUInteger offset,rowBytes; +@end +@implementation NBBuffer +- (void)dealloc { + const NSUInteger length=nmb_buffer_length(_handle);nmb_buffer_release(_handle); + if(_deallocator)_deallocator(_wrapped,length); +} +- (NSUInteger)length {return nmb_buffer_length(_handle);} +- (void *)contents {return nmb_buffer_contents(_handle);} +- (uint64_t)gpuAddress {return nmb_buffer_gpu_address(_handle);} +- (MTLResourceOptions)resourceOptions {return _options;} +- (MTLStorageMode)storageMode {return (MTLStorageMode)((_options&MTLResourceStorageModeMask)>>MTLResourceStorageModeShift);} +- (MTLCPUCacheMode)cpuCacheMode {return (MTLCPUCacheMode)((_options&MTLResourceCPUCacheModeMask)>>MTLResourceCPUCacheModeShift);} +- (MTLHazardTrackingMode)hazardTrackingMode {return (MTLHazardTrackingMode)((_options&MTLResourceHazardTrackingModeMask)>>MTLResourceHazardTrackingModeShift);} +- (void)didModifyRange:(NSRange)range {(void)range;} +@end +static bool textureDescriptor(MTLTextureDescriptor *d,NmbTextureDescriptor *out) { + if(!d)return false; + *out={(uint64_t)d.textureType,(uint64_t)d.pixelFormat,d.width,d.height,d.depth,d.mipmapLevelCount,d.sampleCount,d.arrayLength,(uint64_t)d.usage,(uint64_t)d.resourceOptions}; + return true; +} +@implementation NBTexture +- (void)dealloc {nmb_texture_release(_handle);if(_surface)CFRelease(_surface);} +- (MTLTextureType)textureType {return (MTLTextureType)_descriptor.type;} +- (MTLPixelFormat)pixelFormat {return (MTLPixelFormat)_descriptor.format;} +- (NSUInteger)width {return _descriptor.width;} +- (NSUInteger)height {return _descriptor.height;} +- (NSUInteger)depth {return _descriptor.depth;} +- (NSUInteger)mipmapLevelCount {return _descriptor.levels;} +- (NSUInteger)sampleCount {return _descriptor.samples;} +- (NSUInteger)arrayLength {return _descriptor.array_length;} +- (MTLTextureUsage)usage {return (MTLTextureUsage)_descriptor.usage;} +- (MTLResourceOptions)resourceOptions {return (MTLResourceOptions)_descriptor.options;} +- (MTLStorageMode)storageMode {return (MTLStorageMode)((_descriptor.options&MTLResourceStorageModeMask)>>MTLResourceStorageModeShift);} +- (MTLCPUCacheMode)cpuCacheMode {return (MTLCPUCacheMode)((_descriptor.options&MTLResourceCPUCacheModeMask)>>MTLResourceCPUCacheModeShift);} +- (BOOL)isFramebufferOnly {return NO;} +- (IOSurfaceRef)iosurface {return _surface;} +- (NSUInteger)iosurfacePlane {return 0;} +- (id)buffer {return _surface?nil:(id)_buffer;} +- (NSUInteger)bufferOffset {return _offset;} +- (NSUInteger)bufferBytesPerRow {return _rowBytes;} +- (void)replaceRegion:(MTLRegion)r mipmapLevel:(NSUInteger)level slice:(NSUInteger)slice withBytes:(const void *)bytes bytesPerRow:(NSUInteger)row bytesPerImage:(NSUInteger)image { + const uint64_t region[6]={r.origin.x,r.origin.y,r.origin.z,r.size.width,r.size.height,r.size.depth}; + const uint64_t length=(r.size.depth>1?image*(r.size.depth-1):0)+row*r.size.height; + if(!nmb_texture_replace(_handle,region,level,slice,bytes,row,image,length))NSLog(@"native-metal: replaceRegion refused: %@",lastError()); +} +- (void)replaceRegion:(MTLRegion)r mipmapLevel:(NSUInteger)level withBytes:(const void *)bytes bytesPerRow:(NSUInteger)row { + [self replaceRegion:r mipmapLevel:level slice:0 withBytes:bytes bytesPerRow:row bytesPerImage:row*r.size.height]; +} +- (void)getBytes:(void *)bytes bytesPerRow:(NSUInteger)row bytesPerImage:(NSUInteger)image fromRegion:(MTLRegion)r mipmapLevel:(NSUInteger)level slice:(NSUInteger)slice { + const uint64_t region[6]={r.origin.x,r.origin.y,r.origin.z,r.size.width,r.size.height,r.size.depth}; + const uint64_t length=(r.size.depth>1?image*(r.size.depth-1):0)+row*r.size.height; + if(!nmb_texture_get_bytes(_handle,bytes,row,image,length,region,level,slice))NSLog(@"native-metal: getBytes refused: %@",lastError()); +} +- (void)getBytes:(void *)bytes bytesPerRow:(NSUInteger)row fromRegion:(MTLRegion)r mipmapLevel:(NSUInteger)level { + [self getBytes:bytes bytesPerRow:row bytesPerImage:row*r.size.height fromRegion:r mipmapLevel:level slice:0]; +} +// The CPU's view of an imported surface or buffer: cleaned before the GPU reads it, invalidated after the GPU wrote it. +- (void)flushHostPages { + if(!_surface)return; + sys_dcache_flush(IOSurfaceGetBaseAddress(_surface),IOSurfaceGetAllocSize(_surface)); +} +@end +static NBTexture *textureOnBuffer(NBBuffer *buffer,MTLTextureDescriptor *d,NSUInteger offset,NSUInteger row,bool surface) { + NmbTextureDescriptor descriptor;if(!buffer || !textureDescriptor(d,&descriptor))return nil; + NmbTexture *handle=nmb_texture_new_on_buffer(buffer.handle,&descriptor,offset,row,surface); + if(!handle){NSLog(@"native-metal: texture over buffer refused: %@",lastError());return nil;} + NBTexture *texture=[NBTexture new];texture.handle=handle;nmb_texture_describe(handle,&descriptor);texture.descriptor=descriptor; + texture.buffer=buffer;texture.offset=offset;texture.rowBytes=row;return texture; +} +@implementation NBBuffer (Textures) +- (id)newTextureWithDescriptor:(MTLTextureDescriptor *)d offset:(NSUInteger)offset bytesPerRow:(NSUInteger)row { + return (id)textureOnBuffer(self,d,offset,row,false); +} +@end + +// ---- libraries, functions, states ---- +@interface NBLibrary : NBObject +@property(nonatomic) NmbLibrary *handle; +@end +@interface NBFunction : NBObject +@property(nonatomic) NmbFunction *handle; +@property(nonatomic,copy) NSString *name; +@property(nonatomic) MTLFunctionType functionType; +@end +@implementation NBFunction +- (void)dealloc {nmb_function_release(_handle);} +@end +// The target's own accessors of MTLFunctionConstantValues (as the offline frontend reads them); their ABI is checked before any call. +@interface MTLFunctionConstantValues (NBOriginalConstantAccess) +- (NSArray *)newIndexedConstantArray; +- (NSArray *)newNamedConstantArray; +@end +@protocol NBOriginalConstantValue +- (NSUInteger)dataType; +- (void *)data; +- (NSUInteger)index; +- (NSString *)name; +@end +static bool methodABI(id object,SEL selector,const char *encoding) { + Method method=class_getInstanceMethod(object_getClass(object),selector); + return method && !strcmp(method_getTypeEncoding(method),encoding); +} +// Bytes of a function constant of MTLDataType `type` (scalars and 2/4-wide vectors; 3-wide vectors are not qualified). +static uint64_t constantBytes(uint64_t type) { + switch(type) { + case 3:case 29:case 33:return 4; + case 4:case 30:case 34:return 8; + case 6:case 32:case 36:return 16; + case 16:case 37:case 41:return 2; + case 17:case 38:case 42:return 4; + case 19:case 40:case 44:return 8; + case 45:case 49:case 53:return 1; + case 46:case 50:case 54:return 2; + case 48:case 52:case 56:return 4; + case 81:case 85:return 8; + case 82:case 86:return 16; + case 84:case 88:return 32; + default:return 0; + } +} +@implementation NBLibrary +- (void)dealloc {nmb_library_release(_handle);} +- (id)newFunctionWithName:(NSString *)name constantValues:(MTLFunctionConstantValues *)values error:(NSError **)error { + if(!name || !values || !methodABI(values,@selector(newIndexedConstantArray),"@16@0:8") || !methodABI(values,@selector(newNamedConstantArray),"@16@0:8")) { + if(error)*error=nbError(@"unqualified function constant values");return nil; + } + MTLFunctionConstantValues *snapshot=[values copy]; + NSArray *indexed=[snapshot newIndexedConstantArray],*named=[snapshot newNamedConstantArray]; + std::vector constants; + for(unsigned group=0;group<2;group++)for(id value in (group?named:indexed)) { + if(!methodABI(value,@selector(dataType),"Q16@0:8") || !methodABI(value,@selector(data),"^v16@0:8") || + !methodABI(value,group?@selector(name):@selector(index),group?"@16@0:8":"Q16@0:8")) { + if(error)*error=nbError(@"unqualified constant value ABI");return nil; + } + id v=value; + const uint64_t bytes=constantBytes(v.dataType); + if(!bytes || !v.data){if(error)*error=nbError([NSString stringWithFormat:@"constant type %lu is not qualified",(unsigned long)v.dataType]);return nil;} + constants.push_back({v.dataType,group?0:v.index,group?v.name.UTF8String:nullptr,v.data,bytes}); + } + NmbFunction *handle=nmb_function_new_specialized(_handle,name.UTF8String,constants.data(),(uint32_t)constants.size()); + if(!handle){if(error)*error=nbError([NSString stringWithFormat:@"specialized function: %@",lastError()]);return nil;} + NBFunction *function=[NBFunction new];function.handle=handle;function.name=name;function.functionType=(MTLFunctionType)nmb_function_type(handle); + if(error)*error=nil;return (id)function; +} +- (void)newFunctionWithName:(NSString *)name constantValues:(MTLFunctionConstantValues *)values completionHandler:(void (^)(id,NSError *))handler { + NSError *error=nil;id function=[self newFunctionWithName:name constantValues:values error:&error]; + handler(function,error); +} +- (NSArray *)functionNames { + NSMutableArray *names=[NSMutableArray array]; + for(uint32_t i=0,n=nmb_library_function_count(_handle);i)newFunctionWithName:(NSString *)name { + NmbFunction *handle=name?nmb_function_new(_handle,name.UTF8String):nullptr;if(!handle)return nil; + NBFunction *function=[NBFunction new];function.handle=handle;function.name=name;function.functionType=(MTLFunctionType)nmb_function_type(handle); + return (id)function; +} +@end +@interface NBComputePipeline : NBObject +@property(nonatomic) NmbComputePipeline *handle; +@end +@implementation NBComputePipeline +- (void)dealloc {nmb_compute_pipeline_release(_handle);} +- (NSUInteger)maxTotalThreadsPerThreadgroup {return nmb_compute_pipeline_max_threads(_handle);} +- (NSUInteger)threadExecutionWidth {return nmb_compute_pipeline_execution_width(_handle);} +@end +@interface NBRenderPipeline : NBObject +@property(nonatomic) NmbRenderPipeline *handle; +@end +@implementation NBRenderPipeline +- (void)dealloc {nmb_render_pipeline_release(_handle);} +@end +@interface NBSampler : NBObject +@property(nonatomic) NmbSampler *handle; +@end +@implementation NBSampler +- (void)dealloc {nmb_sampler_release(_handle);} +@end +@interface NBDepthStencil : NBObject +@property(nonatomic) NmbDepthStencil *handle; +@end +@implementation NBDepthStencil +- (void)dealloc {nmb_depth_stencil_release(_handle);} +@end + +// ---- events and fences ---- +@interface NBEvent : NBObject +@property(nonatomic) NmbEvent *handle; +@end +@implementation NBEvent +- (void)dealloc {nmb_event_release(_handle);} +@end +@interface NBSharedEvent : NBEvent +@end +static void notified(void *context,uint64_t value); +@implementation NBSharedEvent +- (uint64_t)signaledValue {return nmb_shared_event_value(self.handle);} +- (void)setSignaledValue:(uint64_t)value {nmb_shared_event_set_value(self.handle,value);} +- (BOOL)waitUntilSignaledValue:(uint64_t)value timeoutMS:(uint64_t)milliseconds {return nmb_shared_event_wait(self.handle,value,milliseconds);} +// The block runs on the listener's dispatch queue (Metal's contract); the bridge's listener thread only forwards it there. +- (void)notifyListener:(MTLSharedEventListener *)listener atValue:(uint64_t)value block:(MTLSharedEventNotificationBlock)block { + if(!block)return; + dispatch_queue_t queue=listener.dispatchQueue?:dispatch_get_global_queue(QOS_CLASS_DEFAULT,0); + NSArray *job=@[self,queue,[block copy]]; + if(!nmb_shared_event_notify(self.handle,value,notified,(__bridge_retained void *)job)) { + (void)(__bridge_transfer NSArray *)(__bridge void *)job; + NSLog(@"native-metal: notifyListener refused: %@",lastError()); + } +} +// A handle for another process needs a kernel-backed event this bridge does not have yet: refused, never a handle that could not work. +- (MTLSharedEventHandle *)newSharedEventHandle { + if(gMissing)gMissing("NBSharedEvent","newSharedEventHandle (cross-process)"); + return nil; +} +@end +static void notified(void *context,uint64_t value) { + NSArray *job=(__bridge_transfer NSArray *)context; + id event=job[0];MTLSharedEventNotificationBlock block=job[2]; + dispatch_async(job[1],^{block(event,value);}); +} +@interface NBFence : NBObject +@property(nonatomic) NmbFence *handle; +@end +@implementation NBFence +- (void)dealloc {nmb_fence_release(_handle);} +@end + +// ---- command buffers and encoders ---- +@interface NBQueue : NBObject +@property(nonatomic) NmbQueue *handle; +@end +@interface NBCommandBuffer : NBObject +@property(nonatomic) NmbCommandBuffer *handle; +@property(nonatomic,strong) NBQueue *queue; +@property(atomic,copy) NSString *encodeFailure; +@property(atomic) BOOL committed,failedLocally; +@property(nonatomic,strong) NSMutableArray *scheduledHandlers,*completedHandlers; +@property(nonatomic,strong) NSMutableSet *surfaces; +@property(nonatomic,strong) dispatch_group_t handlers; +- (void)refuse:(NSString *)what; +@end +@interface NBEncoder : NBObject +@property(nonatomic) NmbEncoder *handle; +@property(nonatomic,strong) NBCommandBuffer *commandBuffer; +@end +@implementation NBEncoder +- (void)dealloc {if(_handle)nmb_encoder_discard(_handle);} +- (bool)check:(bool)ok what:(const char *)what { + if(!ok)[_commandBuffer refuse:[NSString stringWithFormat:@"%s: %@",what,lastError()]]; + return ok; +} +- (void)endEncoding { + NmbEncoder *handle=_handle;_handle=nullptr; + if(!handle){[_commandBuffer refuse:@"endEncoding twice"];return;} + [self check:nmb_encoder_end(handle) what:"endEncoding"]; +} +- (void)insertDebugSignpost:(NSString *)s {(void)s;} +- (void)pushDebugGroup:(NSString *)s {(void)s;} +- (void)popDebugGroup {} +// Stage 0 compute, 1 vertex, 2 fragment (the bridge's numbering). +- (void)buffer:(id)buffer offset:(NSUInteger)offset index:(NSUInteger)index stage:(uint32_t)stage { + if(buffer && ![(id)buffer isKindOfClass:[NBBuffer class]]){[_commandBuffer refuse:@"foreign buffer"];return;} + [self check:nmb_encoder_set_buffer(_handle,stage,buffer?[(NBBuffer *)buffer handle]:nullptr,offset,(uint32_t)index) what:"setBuffer"]; +} +- (void)texture:(id)texture index:(NSUInteger)index stage:(uint32_t)stage { + if(texture && ![(id)texture isKindOfClass:[NBTexture class]]){[_commandBuffer refuse:@"foreign texture"];return;} + if(texture && [(NBTexture *)texture surface])[_commandBuffer.surfaces addObject:texture]; + [self check:nmb_encoder_set_texture(_handle,stage,texture?[(NBTexture *)texture handle]:nullptr,(uint32_t)index) what:"setTexture"]; +} +- (void)sampler:(id)sampler index:(NSUInteger)index stage:(uint32_t)stage { + if(sampler && ![(id)sampler isKindOfClass:[NBSampler class]]){[_commandBuffer refuse:@"foreign sampler"];return;} + [self check:nmb_encoder_set_sampler(_handle,stage,sampler?[(NBSampler *)sampler handle]:nullptr,(uint32_t)index) what:"setSamplerState"]; +} +- (void)bytes:(const void *)bytes length:(NSUInteger)length index:(NSUInteger)index stage:(uint32_t)stage { + [self check:nmb_encoder_set_bytes(_handle,stage,bytes,length,(uint32_t)index) what:"setBytes"]; +} +- (void)fence:(id)fence stages:(uint64_t)stages update:(bool)update { + if(![(id)fence isKindOfClass:[NBFence class]]){[_commandBuffer refuse:@"foreign fence"];return;} + NmbFence *handle=[(NBFence *)fence handle]; + [self check:update?nmb_encoder_update_fence(_handle,handle,stages):nmb_encoder_wait_fence(_handle,handle,stages) what:update?"updateFence":"waitForFence"]; +} +- (void)updateFence:(id)fence {[self fence:fence stages:0 update:true];} +- (void)waitForFence:(id)fence {[self fence:fence stages:0 update:false];} +- (void)offset:(NSUInteger)offset index:(NSUInteger)index stage:(uint32_t)stage { + [self check:nmb_encoder_set_buffer_offset(_handle,stage,offset,(uint32_t)index) what:"setBufferOffset"]; +} +@end + +@interface NBComputeEncoder : NBEncoder +@end +@implementation NBComputeEncoder +- (MTLDispatchType)dispatchType {return MTLDispatchTypeSerial;} +- (void)setComputePipelineState:(id)state { + if(![(id)state isKindOfClass:[NBComputePipeline class]]){[self.commandBuffer refuse:@"foreign compute pipeline"];return;} + [self check:nmb_compute_set_pipeline(self.handle,[(NBComputePipeline *)state handle]) what:"setComputePipelineState"]; +} +- (void)setBuffer:(id)b offset:(NSUInteger)o atIndex:(NSUInteger)i {[self buffer:b offset:o index:i stage:0];} +- (void)setBufferOffset:(NSUInteger)o atIndex:(NSUInteger)i {[self offset:o index:i stage:0];} +- (void)setBuffers:(const id __unsafe_unretained [])b offsets:(const NSUInteger [])o withRange:(NSRange)r { + for(NSUInteger k=0;k)t atIndex:(NSUInteger)i {[self texture:t index:i stage:0];} +- (void)setTextures:(const id __unsafe_unretained [])t withRange:(NSRange)r { + for(NSUInteger k=0;k)s atIndex:(NSUInteger)i {[self sampler:s index:i stage:0];} +- (void)setThreadgroupMemoryLength:(NSUInteger)n atIndex:(NSUInteger)i { + [self check:nmb_compute_set_threadgroup_memory(self.handle,(uint32_t)n,(uint32_t)i) what:"setThreadgroupMemoryLength"]; +} +- (void)dispatch:(MTLSize)size group:(MTLSize)group threads:(bool)threads { + const uint64_t s[3]={size.width,size.height,size.depth},g[3]={group.width,group.height,group.depth}; + [self check:nmb_compute_dispatch(self.handle,s,g,threads) what:"dispatch"]; +} +- (void)dispatchThreads:(MTLSize)t threadsPerThreadgroup:(MTLSize)g {[self dispatch:t group:g threads:true];} +- (void)dispatchThreadgroups:(MTLSize)t threadsPerThreadgroup:(MTLSize)g {[self dispatch:t group:g threads:false];} +- (void)useResource:(id)r usage:(MTLResourceUsage)u {(void)r;(void)u;} +- (void)memoryBarrierWithScope:(MTLBarrierScope)s {(void)s;} +@end + +@interface NBRenderEncoder : NBEncoder +@end +@implementation NBRenderEncoder +- (void)setRenderPipelineState:(id)state { + if(![(id)state isKindOfClass:[NBRenderPipeline class]]){[self.commandBuffer refuse:@"foreign render pipeline"];return;} + [self check:nmb_render_set_pipeline(self.handle,[(NBRenderPipeline *)state handle]) what:"setRenderPipelineState"]; +} +- (void)setVertexBuffer:(id)b offset:(NSUInteger)o atIndex:(NSUInteger)i {[self buffer:b offset:o index:i stage:1];} +- (void)setFragmentBuffer:(id)b offset:(NSUInteger)o atIndex:(NSUInteger)i {[self buffer:b offset:o index:i stage:2];} +- (void)setVertexBufferOffset:(NSUInteger)o atIndex:(NSUInteger)i {[self offset:o index:i stage:1];} +- (void)setFragmentBufferOffset:(NSUInteger)o atIndex:(NSUInteger)i {[self offset:o index:i stage:2];} +- (void)setVertexBuffers:(const id __unsafe_unretained [])b offsets:(const NSUInteger [])o withRange:(NSRange)r { + for(NSUInteger k=0;k __unsafe_unretained [])b offsets:(const NSUInteger [])o withRange:(NSRange)r { + for(NSUInteger k=0;k)t atIndex:(NSUInteger)i {[self texture:t index:i stage:1];} +- (void)setFragmentTexture:(id)t atIndex:(NSUInteger)i {[self texture:t index:i stage:2];} +- (void)setFragmentTextures:(const id __unsafe_unretained [])t withRange:(NSRange)r { + for(NSUInteger k=0;k)s atIndex:(NSUInteger)i {[self sampler:s index:i stage:1];} +- (void)setFragmentSamplerState:(id)s atIndex:(NSUInteger)i {[self sampler:s index:i stage:2];} +- (void)setDepthStencilState:(id)state { + if(state && ![(id)state isKindOfClass:[NBDepthStencil class]]){[self.commandBuffer refuse:@"foreign depth/stencil state"];return;} + [self check:nmb_render_set_depth_stencil(self.handle,state?[(NBDepthStencil *)state handle]:nullptr) what:"setDepthStencilState"]; +} +- (void)setViewport:(MTLViewport)v { + const double values[6]={v.originX,v.originY,v.width,v.height,v.znear,v.zfar}; + [self check:nmb_render_set_viewport(self.handle,values) what:"setViewport"]; +} +- (void)setScissorRect:(MTLScissorRect)r { + const uint64_t values[4]={r.x,r.y,r.width,r.height}; + [self check:nmb_render_set_scissor(self.handle,values) what:"setScissorRect"]; +} +// The runtime takes cull mode, winding and fill mode together; this encoder keeps Metal's defaults until each is set. +- (void)raster { + NSNumber *cull=objc_getAssociatedObject(self,"cull"),*winding=objc_getAssociatedObject(self,"winding"),*fill=objc_getAssociatedObject(self,"fill"); + [self check:nmb_render_set_raster(self.handle,cull.unsignedLongLongValue,winding.unsignedLongLongValue,fill.unsignedLongLongValue) what:"raster state"]; +} +- (void)setCullMode:(MTLCullMode)m {objc_setAssociatedObject(self,"cull",@(m),OBJC_ASSOCIATION_RETAIN);[self raster];} +- (void)setFrontFacingWinding:(MTLWinding)w {objc_setAssociatedObject(self,"winding",@(w),OBJC_ASSOCIATION_RETAIN);[self raster];} +- (void)setTriangleFillMode:(MTLTriangleFillMode)f {objc_setAssociatedObject(self,"fill",@(f),OBJC_ASSOCIATION_RETAIN);[self raster];} +- (void)setStencilReferenceValue:(uint32_t)v {[self check:nmb_render_set_stencil_reference(self.handle,v,v) what:"setStencilReferenceValue"];} +- (void)setStencilFrontReferenceValue:(uint32_t)f backReferenceValue:(uint32_t)b { + [self check:nmb_render_set_stencil_reference(self.handle,f,b) what:"setStencilReferenceValues"]; +} +- (void)setBlendColorRed:(float)r green:(float)g blue:(float)b alpha:(float)a { + const float c[4]={r,g,b,a};[self check:nmb_render_set_blend_color(self.handle,c) what:"setBlendColor"]; +} +- (void)drawPrimitives:(MTLPrimitiveType)t vertexStart:(NSUInteger)s vertexCount:(NSUInteger)n instanceCount:(NSUInteger)i baseInstance:(NSUInteger)b { + [self check:nmb_render_draw(self.handle,t,(uint32_t)s,(uint32_t)n,(uint32_t)i,(uint32_t)b) what:"drawPrimitives"]; +} +- (void)drawPrimitives:(MTLPrimitiveType)t vertexStart:(NSUInteger)s vertexCount:(NSUInteger)n instanceCount:(NSUInteger)i { + [self drawPrimitives:t vertexStart:s vertexCount:n instanceCount:i baseInstance:0]; +} +- (void)drawPrimitives:(MTLPrimitiveType)t vertexStart:(NSUInteger)s vertexCount:(NSUInteger)n { + [self drawPrimitives:t vertexStart:s vertexCount:n instanceCount:1 baseInstance:0]; +} +- (void)drawIndexedPrimitives:(MTLPrimitiveType)t indexCount:(NSUInteger)n indexType:(MTLIndexType)k indexBuffer:(id)b indexBufferOffset:(NSUInteger)o + instanceCount:(NSUInteger)i baseVertex:(NSInteger)v baseInstance:(NSUInteger)bi { + if(![(id)b isKindOfClass:[NBBuffer class]]){[self.commandBuffer refuse:@"foreign index buffer"];return;} + [self check:nmb_render_draw_indexed(self.handle,t,(uint32_t)n,k,[(NBBuffer *)b handle],o,(uint32_t)i,(int32_t)v,(uint32_t)bi) what:"drawIndexedPrimitives"]; +} +- (void)drawIndexedPrimitives:(MTLPrimitiveType)t indexCount:(NSUInteger)n indexType:(MTLIndexType)k indexBuffer:(id)b indexBufferOffset:(NSUInteger)o + instanceCount:(NSUInteger)i { + [self drawIndexedPrimitives:t indexCount:n indexType:k indexBuffer:b indexBufferOffset:o instanceCount:i baseVertex:0 baseInstance:0]; +} +- (void)drawIndexedPrimitives:(MTLPrimitiveType)t indexCount:(NSUInteger)n indexType:(MTLIndexType)k indexBuffer:(id)b indexBufferOffset:(NSUInteger)o { + [self drawIndexedPrimitives:t indexCount:n indexType:k indexBuffer:b indexBufferOffset:o instanceCount:1 baseVertex:0 baseInstance:0]; +} +- (void)useResource:(id)r usage:(MTLResourceUsage)u stages:(MTLRenderStages)s {(void)r;(void)u;(void)s;} +- (void)updateFence:(id)fence afterStages:(MTLRenderStages)stages {[self fence:fence stages:stages update:true];} +- (void)waitForFence:(id)fence beforeStages:(MTLRenderStages)stages {[self fence:fence stages:stages update:false];} +@end + +@interface NBBlitEncoder : NBEncoder +@end +@implementation NBBlitEncoder +- (bool)buffers:(NSArray *)objects { + for(id o in objects)if(![o isKindOfClass:[NBBuffer class]]){[self.commandBuffer refuse:@"foreign blit buffer"];return false;} + return true; +} +- (bool)textures:(NSArray *)objects { + for(id o in objects) { + if(![o isKindOfClass:[NBTexture class]]){[self.commandBuffer refuse:@"foreign blit texture"];return false;} + if([(NBTexture *)o surface])[self.commandBuffer.surfaces addObject:o]; + } + return true; +} +- (void)copyFromBuffer:(id)s sourceOffset:(NSUInteger)so toBuffer:(id)d destinationOffset:(NSUInteger)dO size:(NSUInteger)n { + if(![self buffers:@[s,d]])return; + [self check:nmb_blit_copy_buffer(self.handle,[(NBBuffer *)s handle],so,[(NBBuffer *)d handle],dO,n) what:"copyFromBuffer:toBuffer:"]; +} +- (void)fillBuffer:(id)b range:(NSRange)r value:(uint8_t)v { + if(![self buffers:@[b]])return; + [self check:nmb_blit_fill_buffer(self.handle,[(NBBuffer *)b handle],r.location,r.location+r.length,v) what:"fillBuffer"]; +} +- (void)copyFromTexture:(id)s sourceSlice:(NSUInteger)ss sourceLevel:(NSUInteger)sl sourceOrigin:(MTLOrigin)so sourceSize:(MTLSize)z + toTexture:(id)d destinationSlice:(NSUInteger)ds destinationLevel:(NSUInteger)dl destinationOrigin:(MTLOrigin)dO { + if(![self textures:@[s,d]])return; + const uint64_t a[3]={so.x,so.y,so.z},size[3]={z.width,z.height,z.depth},b[3]={dO.x,dO.y,dO.z}; + [self check:nmb_blit_copy_texture(self.handle,[(NBTexture *)s handle],ss,sl,a,size,[(NBTexture *)d handle],ds,dl,b) what:"copyFromTexture:toTexture:"]; +} +- (void)copyFromTexture:(id)s toTexture:(id)d { + if(![self textures:@[s,d]])return; + const NSUInteger levels=MIN(s.mipmapLevelCount,d.mipmapLevelCount),slices=MIN(s.arrayLength,d.arrayLength); + for(NSUInteger slice=0;slice>level,(NSUInteger)1),MAX(s.height>>level,(NSUInteger)1),MAX(s.depth>>level,(NSUInteger)1)); + [self copyFromTexture:s sourceSlice:slice sourceLevel:level sourceOrigin:MTLOriginMake(0,0,0) sourceSize:size toTexture:d destinationSlice:slice + destinationLevel:level destinationOrigin:MTLOriginMake(0,0,0)]; + } +} +- (void)copyFromBuffer:(id)s sourceOffset:(NSUInteger)o sourceBytesPerRow:(NSUInteger)row sourceBytesPerImage:(NSUInteger)image + sourceSize:(MTLSize)z toTexture:(id)d destinationSlice:(NSUInteger)slice destinationLevel:(NSUInteger)level + destinationOrigin:(MTLOrigin)dO { + if(![self buffers:@[s]] || ![self textures:@[d]])return; + const uint64_t size[3]={z.width,z.height,z.depth},b[3]={dO.x,dO.y,dO.z}; + [self check:nmb_blit_buffer_to_texture(self.handle,[(NBBuffer *)s handle],o,row,image,size,[(NBTexture *)d handle],slice,level,b) what:"copyFromBuffer:toTexture:"]; +} +- (void)copyFromTexture:(id)s sourceSlice:(NSUInteger)slice sourceLevel:(NSUInteger)level sourceOrigin:(MTLOrigin)so sourceSize:(MTLSize)z + toBuffer:(id)d destinationOffset:(NSUInteger)o destinationBytesPerRow:(NSUInteger)row destinationBytesPerImage:(NSUInteger)image { + if(![self textures:@[s]] || ![self buffers:@[d]])return; + const uint64_t a[3]={so.x,so.y,so.z},size[3]={z.width,z.height,z.depth}; + [self check:nmb_blit_texture_to_buffer(self.handle,[(NBTexture *)s handle],slice,level,a,size,[(NBBuffer *)d handle],o,row,image) what:"copyFromTexture:toBuffer:"]; +} +- (void)generateMipmapsForTexture:(id)t { + if(![self textures:@[t]])return; + [self check:nmb_blit_generate_mipmaps(self.handle,[(NBTexture *)t handle]) what:"generateMipmapsForTexture"]; +} +- (void)synchronizeResource:(id)r {(void)r;} +@end + +static void completed(void *context); +@implementation NBCommandBuffer +- (instancetype)init { + if((self=[super init])){_scheduledHandlers=[NSMutableArray array];_completedHandlers=[NSMutableArray array];_surfaces=[NSMutableSet set];_handlers=dispatch_group_create();} + return self; +} +- (void)dealloc {nmb_command_buffer_release(_handle);} +- (id)commandQueue {return (id)_queue;} +- (BOOL)retainedReferences {return YES;} +- (void)refuse:(NSString *)what { + if(!self.encodeFailure){self.encodeFailure=what;NSLog(@"native-metal: command buffer refused: %@",what);} +} +- (MTLCommandBufferStatus)status { + if(self.failedLocally)return MTLCommandBufferStatusError; + return (MTLCommandBufferStatus)nmb_command_buffer_status(_handle); +} +- (NSError *)error { + if(self.failedLocally)return nbError(self.encodeFailure); + if(self.status!=MTLCommandBufferStatusError)return nil; + char text[512]={};nmb_command_buffer_error(_handle,text,sizeof text); + return nbError([NSString stringWithUTF8String:text]); +} +- (CFTimeInterval)GPUStartTime {return nmb_command_buffer_gpu_time(_handle,false);} +- (CFTimeInterval)GPUEndTime {return nmb_command_buffer_gpu_time(_handle,true);} +- (id)encoder:(NBEncoder *)encoder handle:(NmbEncoder *)handle what:(NSString *)what { + if(self.committed){[self refuse:[what stringByAppendingString:@" after commit"]];return nil;} + if(!handle){[self refuse:[NSString stringWithFormat:@"%@: %@",what,lastError()]];return nil;} + encoder.handle=handle;encoder.commandBuffer=self;return encoder; +} +- (id)computeCommandEncoder { + return [self encoder:[NBComputeEncoder new] handle:self.committed?nullptr:nmb_compute_encoder_new(_handle) what:@"computeCommandEncoder"]; +} +- (id)computeCommandEncoderWithDispatchType:(MTLDispatchType)type { + if(type!=MTLDispatchTypeSerial){[self refuse:@"concurrent compute dispatch"];return nil;} + return [self computeCommandEncoder]; +} +- (id)blitCommandEncoder { + return [self encoder:[NBBlitEncoder new] handle:self.committed?nullptr:nmb_blit_encoder_new(_handle) what:@"blitCommandEncoder"]; +} +- (id)renderCommandEncoderWithDescriptor:(MTLRenderPassDescriptor *)d { + NmbRenderPassDescriptor pass={}; + auto attachment=[&](MTLRenderPassAttachmentDescriptor *a,NmbPassAttachment &out)->bool { + if(!a.texture)return true; + if(![(id)a.texture isKindOfClass:[NBTexture class]])return false; + NBTexture *texture=(NBTexture *)a.texture;if(texture.surface)[_surfaces addObject:texture]; + out.texture=texture.handle;out.level=a.level;out.slice=a.slice;out.depth_plane=a.depthPlane;out.load=a.loadAction;out.store=a.storeAction; + return true; + }; + for(unsigned i=0;i<8;i++) { + MTLRenderPassColorAttachmentDescriptor *c=d.colorAttachments[i]; + if(!attachment(c,pass.colors[i])){[self refuse:@"foreign color attachment"];return nil;} + pass.colors[i].clear[0]=c.clearColor.red;pass.colors[i].clear[1]=c.clearColor.green;pass.colors[i].clear[2]=c.clearColor.blue;pass.colors[i].clear[3]=c.clearColor.alpha; + } + if(!attachment(d.depthAttachment,pass.depth) || !attachment(d.stencilAttachment,pass.stencil)){[self refuse:@"foreign depth/stencil attachment"];return nil;} + pass.depth.clear[0]=d.depthAttachment.clearDepth;pass.stencil.clear[0]=d.stencilAttachment.clearStencil; + pass.width=d.renderTargetWidth;pass.height=d.renderTargetHeight;pass.array_length=d.renderTargetArrayLength;pass.samples=d.defaultRasterSampleCount; + return [self encoder:[NBRenderEncoder new] handle:self.committed?nullptr:nmb_render_encoder_new(_handle,&pass) what:@"renderCommandEncoder"]; +} +- (void)addScheduledHandler:(MTLCommandBufferHandler)block {if(block)@synchronized(self){[_scheduledHandlers addObject:[block copy]];}} +- (void)addCompletedHandler:(MTLCommandBufferHandler)block {if(block)@synchronized(self){[_completedHandlers addObject:[block copy]];}} +- (void)enqueue {} +- (void)event:(id)event value:(uint64_t)value signal:(bool)signal { + if(![(id)event isKindOfClass:[NBEvent class]]){[self refuse:@"foreign event"];return;} + if(self.committed){[self refuse:@"event encoded after commit"];return;} + NmbEvent *handle=[(NBEvent *)event handle]; + if(!(signal?nmb_command_buffer_signal_event(_handle,handle,value):nmb_command_buffer_wait_event(_handle,handle,value))) + [self refuse:[NSString stringWithFormat:@"%s: %@",signal?"encodeSignalEvent":"encodeWaitForEvent",lastError()]]; +} +- (void)encodeSignalEvent:(id)event value:(uint64_t)value {[self event:event value:value signal:true];} +- (void)encodeWaitForEvent:(id)event value:(uint64_t)value {[self event:event value:value signal:false];} +// After completion (or a refusal): invalidate the CPU's view of imported surfaces, then scheduled handlers, then completed handlers, once. +- (void)finish { + for(NBTexture *texture in _surfaces)[texture flushHostPages]; + NSArray *scheduled,*completedHandlers; + @synchronized(self){scheduled=[_scheduledHandlers copy];completedHandlers=[_completedHandlers copy];[_scheduledHandlers removeAllObjects];[_completedHandlers removeAllObjects];} + for(MTLCommandBufferHandler block in scheduled)block((id)self); + for(MTLCommandBufferHandler block in completedHandlers)block((id)self); + dispatch_group_leave(_handlers); +} +- (void)commit { + if(self.committed){NSLog(@"native-metal: commit twice");return;} + self.committed=YES; + dispatch_group_enter(_handlers); + if(!self.encodeFailure) { + for(NBTexture *texture in _surfaces)[texture flushHostPages]; + if(nmb_command_buffer_on_completed(_handle,completed,(__bridge_retained void *)self) && nmb_command_buffer_commit(_handle))return; + [self refuse:[@"commit: " stringByAppendingString:lastError()]]; + } + self.failedLocally=YES; + dispatch_async(dispatch_get_global_queue(QOS_CLASS_DEFAULT,0),^{[self finish];}); +} +- (void)waitUntilCompleted { + if(!self.committed)return; + if(!self.failedLocally)nmb_command_buffer_wait(_handle,0); + dispatch_group_wait(_handlers,DISPATCH_TIME_FOREVER); +} +- (void)waitUntilScheduled {[self waitUntilCompleted];} +@end +// The runtime's completion: the command buffer's own reference, taken at commit, is given back here. +static void completed(void *context) { + NBCommandBuffer *commandBuffer=(__bridge_transfer NBCommandBuffer *)context; + [commandBuffer finish]; +} + +@implementation NBQueue +- (void)dealloc {nmb_queue_release(_handle);} +- (id)commandBuffer { + NmbCommandBuffer *handle=nmb_command_buffer_new(_handle); + if(!handle){NSLog(@"native-metal: commandBuffer refused: %@",lastError());return nil;} + NBCommandBuffer *commandBuffer=[NBCommandBuffer new];commandBuffer.handle=handle;commandBuffer.queue=self;return (id)commandBuffer; +} +- (id)commandBufferWithUnretainedReferences {return [self commandBuffer];} +- (id)commandBufferWithDescriptor:(MTLCommandBufferDescriptor *)d {(void)d;return [self commandBuffer];} +@end + +// ---- the device ---- +template static T failWith(NSError **error,NSString *what,T value) { + if(error)*error=nbError([NSString stringWithFormat:@"%@: %@",what,lastError()]); + return value; +} +@interface NBDeviceMethods : NSObject +@end +@implementation NBDeviceMethods +- (NSString *)name {return gName;} +- (uint64_t)registryID {return gRegistryID;} +- (NSUInteger)maxBufferLength {return nmb_device_max_buffer_length(gDevice);} +- (BOOL)hasUnifiedMemory {return YES;} +- (BOOL)isLowPower {return NO;} +- (BOOL)isRemovable {return NO;} +- (BOOL)isHeadless {return NO;} +// No GPU family or feature set is claimed until one is qualified; the base implementation reads Apple tables this device does not have. +- (BOOL)supportsFamily:(MTLGPUFamily)family {(void)family;return NO;} +- (BOOL)supportsFeatureSet:(MTLFeatureSet)set {(void)set;return NO;} +- (BOOL)supportsTextureSampleCount:(NSUInteger)count {return count==1;} +- (void)doesNotRecognizeSelector:(SEL)selector { + if(gMissing)gMissing("NativeBridgeMetalDevice",sel_getName(selector)); + [super doesNotRecognizeSelector:selector]; +} +- (id)newEvent { + NmbEvent *handle=nmb_event_new(gDevice,false);if(!handle)return nil; + NBEvent *event=[NBEvent new];event.handle=handle;return (id)event; +} +- (id)newSharedEvent { + NmbEvent *handle=nmb_event_new(gDevice,true);if(!handle)return nil; + NBSharedEvent *event=[NBSharedEvent new];event.handle=handle;return (id)event; +} +- (id)newFence { + NmbFence *handle=nmb_fence_new(gDevice);if(!handle)return nil; + NBFence *fence=[NBFence new];fence.handle=handle;return (id)fence; +} +- (id)newCommandQueue { + NmbQueue *handle=nmb_queue_new(gDevice);if(!handle)return nil; + NBQueue *queue=[NBQueue new];queue.handle=handle;return (id)queue; +} +- (id)newCommandQueueWithMaxCommandBufferCount:(NSUInteger)count {(void)count;return [(id)self newCommandQueue];} +- (id)newBufferWithLength:(NSUInteger)length options:(MTLResourceOptions)options { + NmbBuffer *handle=nmb_buffer_new(gDevice,length,options,nullptr); + if(!handle){NSLog(@"native-metal: buffer refused: %@",lastError());return nil;} + NBBuffer *buffer=[NBBuffer new];buffer.handle=handle;buffer.options=options;return (id)buffer; +} +- (id)newBufferWithBytes:(const void *)bytes length:(NSUInteger)length options:(MTLResourceOptions)options { + if(!bytes)return nil; + NmbBuffer *handle=nmb_buffer_new(gDevice,length,options,bytes); + if(!handle){NSLog(@"native-metal: buffer refused: %@",lastError());return nil;} + NBBuffer *buffer=[NBBuffer new];buffer.handle=handle;buffer.options=options;return (id)buffer; +} +- (id)newBufferWithBytesNoCopy:(void *)pointer length:(NSUInteger)length options:(MTLResourceOptions)options deallocator:(void (^)(void *,NSUInteger))deallocator { + NmbBuffer *handle=nmb_buffer_new_no_copy(gDevice,pointer,length,options); + if(!handle){NSLog(@"native-metal: no-copy buffer refused: %@",lastError());return nil;} + NBBuffer *buffer=[NBBuffer new];buffer.handle=handle;buffer.options=options; + buffer.wrapped=pointer;buffer.deallocator=deallocator; + return (id)buffer; +} +- (id)newTextureWithDescriptor:(MTLTextureDescriptor *)d { + NmbTextureDescriptor descriptor;if(!textureDescriptor(d,&descriptor))return nil; + NmbTexture *handle=nmb_texture_new(gDevice,&descriptor); + if(!handle){NSLog(@"native-metal: texture refused: %@",lastError());return nil;} + NBTexture *texture=[NBTexture new];texture.handle=handle;nmb_texture_describe(handle,&descriptor);texture.descriptor=descriptor;return (id)texture; +} +// The surface's pages are the texels: its mapping in this process is imported without a copy (pinned by the kernel, mapped into the GPU's +// 4 KiB context by Turnip), so every process that imports the surface renders and samples the same memory. +- (id)newTextureWithDescriptor:(MTLTextureDescriptor *)d iosurface:(IOSurfaceRef)surface plane:(NSUInteger)plane { + if(!surface || plane || IOSurfaceGetPlaneCount(surface))return nil; + void *base=IOSurfaceGetBaseAddress(surface);const size_t page=(size_t)getpagesize(),bytes=(IOSurfaceGetAllocSize(surface)+page-1)&~(page-1); + if(!base || ((uintptr_t)base&(page-1)))return nil; + NmbBuffer *handle=nmb_buffer_new_no_copy(gDevice,base,bytes,MTLResourceStorageModeShared); + if(!handle){NSLog(@"native-metal: surface import refused: %@",lastError());return nil;} + NBBuffer *buffer=[NBBuffer new];buffer.handle=handle;buffer.options=MTLResourceStorageModeShared; + NBTexture *texture=textureOnBuffer(buffer,d,0,IOSurfaceGetBytesPerRow(surface),true);if(!texture)return nil; + texture.surface=(IOSurfaceRef)CFRetain(surface);return (id)texture; +} +- (id)newLibraryWithData:(dispatch_data_t)data error:(NSError **)error { + const void *bytes=nullptr;size_t size=0;dispatch_data_t mapped=dispatch_data_create_map(data,&bytes,&size); + NmbLibrary *handle=nmb_library_new(gDevice,bytes,size,false);(void)mapped; + if(!handle)return failWith>(error,@"library",nil); + NBLibrary *library=[NBLibrary new];library.handle=handle;if(error)*error=nil;return (id)library; +} +- (id)newLibraryWithFile:(NSString *)path error:(NSError **)error { + NSData *bytes=path?[NSData dataWithContentsOfFile:path options:NSDataReadingMappedIfSafe error:error]:nil;if(!bytes)return nil; + NmbLibrary *handle=nmb_library_new(gDevice,bytes.bytes,bytes.length,false); + if(!handle)return failWith>(error,@"library",nil); + NBLibrary *library=[NBLibrary new];library.handle=handle;if(error)*error=nil;return (id)library; +} +- (id)newLibraryWithURL:(NSURL *)url error:(NSError **)error { + if(!url.isFileURL){if(error)*error=nbError(@"a file URL is required");return nil;} + return [(id)self newLibraryWithFile:url.path error:error]; +} +- (id)newDefaultLibraryWithBundle:(NSBundle *)bundle error:(NSError **)error { + NSURL *url=[bundle URLForResource:@"default" withExtension:@"metallib"]; + if(!url){if(error)*error=nbError(@"the bundle has no default.metallib");return nil;} + return [(id)self newLibraryWithURL:url error:error]; +} +- (id)newDefaultLibrary {return [(id)self newDefaultLibraryWithBundle:[NSBundle mainBundle] error:nil];} +- (id)newComputePipelineStateWithFunction:(id)function error:(NSError **)error { + if(![(id)function isKindOfClass:[NBFunction class]]){if(error)*error=nbError(@"foreign function");return nil;} + NmbComputePipeline *handle=nmb_compute_pipeline_new(gDevice,[(NBFunction *)function handle]); + if(!handle)return failWith>(error,@"compute pipeline",nil); + NBComputePipeline *state=[NBComputePipeline new];state.handle=handle;if(error)*error=nil;return (id)state; +} +- (id)newComputePipelineStateWithFunction:(id)function options:(MTLPipelineOption)options + reflection:(MTLAutoreleasedComputePipelineReflection *)reflection error:(NSError **)error { + (void)options;if(reflection)*reflection=nil; + return [(id)self newComputePipelineStateWithFunction:function error:error]; +} +- (id)newRenderPipelineStateWithDescriptor:(MTLRenderPipelineDescriptor *)d error:(NSError **)error { + if(![(id)d.vertexFunction isKindOfClass:[NBFunction class]] || (d.fragmentFunction && ![(id)d.fragmentFunction isKindOfClass:[NBFunction class]])) { + if(error)*error=nbError(@"foreign function");return nil; + } + auto *p=new NmbRenderPipelineDescriptor(); + p->vertex=[(NBFunction *)d.vertexFunction handle];p->fragment=d.fragmentFunction?[(NBFunction *)d.fragmentFunction handle]:nullptr; + for(unsigned i=0;i<8;i++) { + MTLRenderPipelineColorAttachmentDescriptor *a=d.colorAttachments[i]; + p->colors[i]={(uint64_t)a.pixelFormat,(uint64_t)a.writeMask,(uint64_t)a.blendingEnabled,(uint64_t)a.sourceRGBBlendFactor,(uint64_t)a.destinationRGBBlendFactor, + (uint64_t)a.rgbBlendOperation,(uint64_t)a.sourceAlphaBlendFactor,(uint64_t)a.destinationAlphaBlendFactor,(uint64_t)a.alphaBlendOperation}; + } + p->depth_format=d.depthAttachmentPixelFormat;p->stencil_format=d.stencilAttachmentPixelFormat;p->samples=d.rasterSampleCount; + p->topology=d.inputPrimitiveTopology;p->rasterization=d.rasterizationEnabled; + if(MTLVertexDescriptor *v=d.vertexDescriptor) { + for(unsigned i=0;i<31;i++) { + MTLVertexAttributeDescriptor *a=v.attributes[i];if(a.format==MTLVertexFormatInvalid)continue; + p->vertex_inputs=1;p->attributes[i]={(uint64_t)a.format,a.offset,a.bufferIndex}; + if(a.bufferIndex<31){MTLVertexBufferLayoutDescriptor *l=v.layouts[a.bufferIndex];p->layouts[a.bufferIndex]={l.stride,(uint64_t)l.stepFunction,l.stepRate};} + } + } + NmbRenderPipeline *handle=nmb_render_pipeline_new(gDevice,p);delete p; + if(!handle)return failWith>(error,@"render pipeline",nil); + NBRenderPipeline *state=[NBRenderPipeline new];state.handle=handle;if(error)*error=nil;return (id)state; +} +- (id)newRenderPipelineStateWithDescriptor:(MTLRenderPipelineDescriptor *)d options:(MTLPipelineOption)options + reflection:(MTLAutoreleasedRenderPipelineReflection *)reflection error:(NSError **)error { + (void)options;if(reflection)*reflection=nil; + return [(id)self newRenderPipelineStateWithDescriptor:d error:error]; +} +- (id)newSamplerStateWithDescriptor:(MTLSamplerDescriptor *)d { + NmbSamplerDescriptor s={(uint64_t)d.minFilter,(uint64_t)d.magFilter,(uint64_t)d.mipFilter,(uint64_t)d.sAddressMode,(uint64_t)d.tAddressMode,(uint64_t)d.rAddressMode, + (uint64_t)d.borderColor,(uint64_t)d.normalizedCoordinates,d.maxAnisotropy,(uint64_t)d.compareFunction,(uint64_t)d.lodAverage,d.lodMinClamp,d.lodMaxClamp}; + NmbSampler *handle=nmb_sampler_new(gDevice,&s); + if(!handle){NSLog(@"native-metal: sampler refused: %@",lastError());return nil;} + NBSampler *sampler=[NBSampler new];sampler.handle=handle;return (id)sampler; +} +- (id)newDepthStencilStateWithDescriptor:(MTLDepthStencilDescriptor *)d { + auto face=[](MTLStencilDescriptor *s)->NmbStencil { + return {(uint64_t)s.stencilCompareFunction,(uint64_t)s.stencilFailureOperation,(uint64_t)s.depthFailureOperation,(uint64_t)s.depthStencilPassOperation,s.readMask,s.writeMask}; + }; + NmbDepthStencilDescriptor s={(uint64_t)d.depthCompareFunction,(uint64_t)d.depthWriteEnabled,face(d.frontFaceStencil),face(d.backFaceStencil)}; + NmbDepthStencil *handle=nmb_depth_stencil_new(gDevice,&s); + if(!handle){NSLog(@"native-metal: depth/stencil state refused: %@",lastError());return nil;} + NBDepthStencil *state=[NBDepthStencil new];state.handle=handle;return (id)state; +} +@end + +extern "C" id native_bridge_metal_open(const char *loader,const char *cache,uint64_t registryID,void (*missing)(const char *,const char *)) { + if(gDeviceObject)return gDeviceObject; + gMissing=missing; + gDevice=nmb_device_open(loader,cache); + if(!gDevice){NSLog(@"native-metal: device refused: %s",nmb_last_error());return nil;} + char name[128]={};nmb_device_name(gDevice,name,sizeof name);gName=[NSString stringWithUTF8String:name];gRegistryID=registryID; + Class base=objc_getClass("_MTLDevice"); +#if TARGET_OS_IPHONE + // The target's base class ABI (iPadOS 17.7.11): the dynamic subclass adds no instance variables. + if(!base || class_getInstanceSize(base)!=536){NSLog(@"native-metal: unexpected _MTLDevice ABI");return nil;} +#endif + if(!base)base=[NSObject class]; + Class cls=objc_allocateClassPair(base,"NativeBridgeMetalDevice",0);if(!cls)return nil; + unsigned count=0;Method *methods=class_copyMethodList([NBDeviceMethods class],&count); + for(unsigned i=0;i ". +// After a passing self-test the worker runs the frontend checks (bridge/FrontendChecks.h) through the public Metal API on the native frontend +// (NativeBridgeMetalFrontend.mm) over the same Turnip: markers 0x6600 (device), 0x6601..0x660f, then 0x66f0 pass / 0x66f1 fail. Its verdict +// ends the same GPU7 line: "F0" (with the first unimplemented selector, if any), "F ", "Fo"/"Fx" + reason +// when the device or the fixtures were refused. +#pragma once +#include "bridge/NativeMetalBridge.h" +#include "bridge/NativeMetalObjects.h" +#include "bridge/FrontendChecks.h" +#include "NativeBridgeMetalFrontend.h" +#include +#include +#include +#include +#include + +namespace bridgephase { +struct Job { + const char *turnip; + char cache[512]; + NmbSelftest result; + uint32_t code; + frontendchecks::Report frontend; + bool frontendRan, frontendOpened; + char missing[64], error[48]; + volatile uint32_t last; + volatile bool done; +}; +static Gpu *gGpu; +static Job *gJob; +static void mark(uint32_t value) { + if (gJob) gJob->last = value; + if (gGpu) gGpu->mark(value); +} +static void missingSelector(const char *cls, const char *selector) { + if (gJob && !gJob->missing[0]) snprintf(gJob->missing, sizeof gJob->missing, "FM %.20s %.36s", cls, selector); +} +// The frontend checks on the native frontend over Turnip (QuartzCore is already translated by the self-test, so it is not repeated). +static void frontend(Job *job) { + mark(0x6600); + id device = native_bridge_metal_open(job->turnip, job->cache, 0, missingSelector); + // The runtime's reasons are per thread: keep this worker's for the report. + if (!device) {strlcpy(job->error, nmb_last_error(), sizeof job->error); return;} + job->frontendOpened = true; + size_t bytes = 0; + const void *air = nmb_fixture_air(&bytes); + id library = native_bridge_metal_library_with_air(air, bytes); + air = nmb_function_constant_air(&bytes); + id fcLibrary = native_bridge_metal_library_with_air(air, bytes); + if (!library || !fcLibrary) {strlcpy(job->error, nmb_last_error(), sizeof job->error); return;} + job->frontend.mark = mark; + job->frontendRan = true; + frontendchecks::run(device, library, fcLibrary, nil, &job->frontend); +} +static void *worker(void *arg) { + Job *job = static_cast(arg); + // The target's own compositor library, read in place (never copied anywhere). + const uint8_t *qc = nullptr; + size_t qcBytes = 0; + const int fd = open("/System/Library/Frameworks/QuartzCore.framework/default.metallib", O_RDONLY); + struct stat st = {}; + if (fd >= 0 && fstat(fd, &st) == 0 && st.st_size > 0) { + void *m = mmap(nullptr, (size_t)st.st_size, PROT_READ, MAP_PRIVATE, fd, 0); + if (m != MAP_FAILED) {qc = static_cast(m); qcBytes = (size_t)st.st_size;} + } + if (fd >= 0) close(fd); + job->code = nmb_selftest(job->turnip, job->cache, qc, qcBytes, mark, &job->result); + if (qc) munmap(const_cast(qc), qcBytes); + if (!job->code) frontend(job); + __atomic_store_n(&job->done, true, __ATOMIC_RELEASE); + return nullptr; +} +static void run(Gpu &gpu, const char *turnipPath) { + gpu.mark(0x6500); + Job *job = new Job{}; + job->turnip = turnipPath; + // A per-user cache directory of the host process (its sandbox allows it); the translated-shader cache goes in a subdirectory. + if (!confstr(_CS_DARWIN_USER_CACHE_DIR, job->cache, sizeof job->cache - 16)) strlcpy(job->cache, "/tmp/", sizeof job->cache); + strlcat(job->cache, "m2v", sizeof job->cache); + mkdir(job->cache, 0700); + gGpu = &gpu; + gJob = job; + pthread_attr_t attr; + pthread_attr_init(&attr); + pthread_attr_setstacksize(&attr, 8u << 20); + pthread_t thread; + const bool started = pthread_create(&thread, &attr, worker, job) == 0; + pthread_attr_destroy(&attr); + if (!started) {gpu.mark(0x65fd); return;} + static const unsigned checkpoints[] = {30, 90, 180}; + unsigned next = 0; + for (unsigned tenths = 0; !__atomic_load_n(&job->done, __ATOMIC_ACQUIRE); ++tenths) { + usleep(100000); + if (next < 3 && tenths == checkpoints[next] * 10) { + char text[64]; + snprintf(text, sizeof text, "MW %x %us", job->last, checkpoints[next]); + streamText7(gpu, text); + ++next; + } + } + pthread_join(thread, nullptr); + const NmbSelftest &r = job->result; + // One GPU7 line (the page keeps only the latest): the self-test's counts, then after a passing self-test the frontend's verdict. + char text[64]; + const int at = snprintf(text, sizeof text, "M s%x c%u r%u i%u q%u k%u/%u p%u/%u", r.stage, r.compute_bad, r.render_bad, r.import_bad, + r.qc_functions, r.qc_kernels_ok, r.qc_kernels_failed, r.qc_pairs_ok, r.qc_pairs_failed); + char *rest = text + (at > 0 && at < (int)sizeof text ? at : 0); + const size_t room = sizeof text - (size_t)(rest - text); + if (job->code) snprintf(rest, room, " %s", r.message); + else if (!job->frontendOpened) snprintf(rest, room, " Fo %s", job->error); + else if (!job->frontendRan) snprintf(rest, room, " Fx %s", job->error); + else if (job->frontend.failed) snprintf(rest, room, " F%u %s", job->frontend.failed, job->frontend.first); + else snprintf(rest, room, " F0%s%s", job->missing[0] ? " " : "", job->missing[0] ? job->missing + 3 : ""); + streamText7(gpu, text); + const bool ok = !job->code && !r.compute_bad && !r.render_bad && !r.import_bad; + gpu.mark(ok ? 0x65f0 : 0x65f1); + if (!job->code) gpu.mark(job->frontendRan && !job->frontend.failed ? 0x66f0 : 0x66f1); + gJob = nullptr; + gGpu = nullptr; + delete job; +} +} // namespace bridgephase diff --git a/drivers/NativeMetal/NativeOfflineAccelerator.cpp b/drivers/NativeMetal/NativeOfflineAccelerator.cpp new file mode 100644 index 0000000..11f1b9d --- /dev/null +++ b/drivers/NativeMetal/NativeOfflineAccelerator.cpp @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: MIT +// Unpublished original-base lifecycle qualification. No GPU service or device +// is attached/started/published, and no original client contract is claimed. +#include "OriginalAcceleratorABI.h" +class NativeOfflineAccelerator : public IOAcceleratorES { + OSDeclareDefaultStructors(NativeOfflineAccelerator) +}; +OSDefineMetaClassAndStructors(NativeOfflineAccelerator,IOAcceleratorES) +extern "C" IOService *native_offline_accelerator_allocate(void) {return new NativeOfflineAccelerator;} +extern "C" unsigned native_offline_accelerator_instance_size(void) {return sizeof(NativeOfflineAccelerator);} +extern "C" unsigned native_offline_accelerator_instances(void) {return NativeOfflineAccelerator::gMetaClass.getInstanceCount();} diff --git a/drivers/NativeMetal/NativeOfflineGPU.cpp b/drivers/NativeMetal/NativeOfflineGPU.cpp new file mode 100644 index 0000000..ae33bae --- /dev/null +++ b/drivers/NativeMetal/NativeOfflineGPU.cpp @@ -0,0 +1,312 @@ +// SPDX-License-Identifier: MIT +// Explicit offline GPU broker. No normal Metal/IOAccelerator publication. +// The kernel, not request bytes, selects each userclient's host namespace. +#include +#include +#include +#include +#include "NativeVirtioConsoleTransport.h" +#include "OfflineMetalSessionWire.h" +#include "OfflineMetalQueueWire.h" +#include "OfflineMetalTextureWire.h" +#include "OfflineMetalDepthStencilWire.h" +#include "OfflineMetalSharedEventWire.h" +#include +extern "C" { +volatile uint32_t native_gpu_broker_stage,native_gpu_broker_clients,native_gpu_broker_active,native_gpu_broker_closed; +volatile uint32_t native_gpu_broker_requests,native_gpu_broker_failures,native_gpu_broker_refusals; +volatile int native_gpu_broker_status=-1; +volatile uint64_t native_gpu_broker_reply_bytes; +volatile uint32_t native_gpu_broker_cpu_context_count; +volatile uint32_t native_gpu_shared_exports,native_gpu_shared_imports,native_gpu_shared_drops,native_gpu_shared_refusals,native_gpu_shared_tokens_live,native_gpu_shared_ports; +volatile uint64_t native_gpu_broker_cpu_contexts[8][8]; +} +class NativeOfflineGPUClient; +class NativeGPUSharedEventToken; +class NativeOfflineGPU : public IOService { + OSDeclareDefaultStructors(NativeOfflineGPU) + friend class NativeOfflineGPUClient; + friend class NativeGPUSharedEventToken; + IOService *transport=nullptr; + IOLock *gate=nullptr; + bool online=false; + uint32_t nextControlRequest=1; + IOReturn readExact(void *bytes,size_t size) { + auto p=static_cast(bytes); + while(size) { + size_t count=size>4096?4096:size; + IOReturn status=native_console_receive(transport,this,p,&count); + if(status!=kIOReturnSuccess)return status; + if(!count || count>size)return kIOReturnIOError; + p+=count;size-=count; + } + return kIOReturnSuccess; + } + // gate covers the complete actual request/reply, including every byte + // beyond a VirtIO descriptor. Userclients retain separate reply buffers. + IOReturn exchange(uint64_t session,const OMWireHeader &request,const void *input, + unsigned char **reply,size_t *length) { + *reply=nullptr;*length=0;if(!online)return kIOReturnNotReady; + OMSessionWireHeader header={request,session};header.wire.version=OFFLINE_METAL_SESSION_VERSION; + IOReturn status=native_console_send(transport,this,&header,sizeof(header)); + if(status==kIOReturnSuccess && request.bytes)status=native_console_send(transport,this,input,request.bytes); + OMSessionWireHeader response={};OMResult result={}; + if(status==kIOReturnSuccess)status=readExact(&response,sizeof(response)); + if(status==kIOReturnSuccess && (response.wire.magic!=OM_WIRE_MAGIC || response.wire.version!=OFFLINE_METAL_SESSION_VERSION || + response.wire.operation!=(request.operation|OM_WIRE_RESPONSE) || response.wire.request!=request.request || + response.session!=session || response.wire.bytesOFFLINE_METAL_MAX_REPLY))status=kIOReturnIOError; + if(status==kIOReturnSuccess)status=readExact(&result,sizeof(result)); + if(status==kIOReturnSuccess && result.reserved)status=kIOReturnIOError; + unsigned char *buffer=nullptr;size_t size=sizeof(OMWireHeader)+response.wire.bytes; + if(status==kIOReturnSuccess) { + buffer=static_cast(IOMallocData(size)); + if(!buffer)status=kIOReturnNoMemory; + } + if(status==kIOReturnSuccess) { + response.wire.version=OFFLINE_METAL_VERSION; + memcpy(buffer,&response.wire,sizeof(OMWireHeader));memcpy(buffer+sizeof(OMWireHeader),&result,sizeof(result)); + status=readExact(buffer+sizeof(OMWireHeader)+sizeof(result),response.wire.bytes-sizeof(result)); + } + native_gpu_broker_status=status; + if(status!=kIOReturnSuccess) { + if(buffer)IOFreeData(buffer,size); + // A partial stream cannot be reused for another client's reply. + online=false;native_gpu_broker_failures=native_gpu_broker_failures+1;return status; + } + native_gpu_broker_requests=native_gpu_broker_requests+1; + native_gpu_broker_reply_bytes=native_gpu_broker_reply_bytes+size; + if(result.error)native_gpu_broker_refusals=native_gpu_broker_refusals+1; + *reply=buffer;*length=size;return kIOReturnSuccess; + } + IOReturn createSession(task_t owner,uint64_t *session) { + *session=0;IOLockLock(gate); + uint64_t tcr=0,ttbr0=0,ttbr1=0,sctlr=0; + __asm__ volatile("mrs %0, tcr_el1" : "=r"(tcr)); + __asm__ volatile("mrs %0, ttbr0_el1" : "=r"(ttbr0)); + __asm__ volatile("mrs %0, ttbr1_el1" : "=r"(ttbr1)); + __asm__ volatile("mrs %0, sctlr_el1" : "=r"(sctlr)); + bool native4k=((tcr>>14)&3)==0 && ((tcr>>30)&3)==2 && (sctlr&1) && ttbr0 && ttbr1; + unsigned index=native_gpu_broker_cpu_context_count; + native_gpu_broker_cpu_context_count=index+1; + if(index<8) { + uint64_t values[8]={(uint64_t)proc_selfpid(),0,tcr,ttbr0,ttbr1,sctlr,(uintptr_t)owner,(uint64_t)native4k}; + for(unsigned n=0;n<8;n++)native_gpu_broker_cpu_contexts[index][n]=values[n]; + } + if(!native4k) { + native_gpu_broker_status=kIOReturnUnsupported;native_gpu_broker_failures=native_gpu_broker_failures+1; + IOLockUnlock(gate);return kIOReturnUnsupported; + } + OMWireHeader request={OM_WIRE_MAGIC,OFFLINE_METAL_VERSION,OM_SESSION_OPEN,0,1}; + unsigned char *reply=nullptr;size_t length=0;IOReturn status=exchange(0,request,nullptr,&reply,&length); + if(status==kIOReturnSuccess) { + OMResult result={};memcpy(&result,reply+sizeof(OMWireHeader),sizeof(result)); + if(result.error || !result.handle || length!=sizeof(OMWireHeader)+sizeof(result))status=kIOReturnNotOpen; + else { + *session=result.handle;native_gpu_broker_clients=native_gpu_broker_clients+1;native_gpu_broker_active=native_gpu_broker_active+1; + if(index<8)native_gpu_broker_cpu_contexts[index][1]=result.handle; + } + } + if(reply)IOFreeData(reply,length);IOLockUnlock(gate);return status; + } + void closeSession(uint64_t session,uint32_t number) { + IOLockLock(gate);OMWireHeader request={OM_WIRE_MAGIC,OFFLINE_METAL_VERSION,OM_SESSION_CLOSE,0,number}; + unsigned char *reply=nullptr;size_t length=0;IOReturn status=exchange(session,request,nullptr,&reply,&length); + if(status==kIOReturnSuccess) { + OMResult result={};memcpy(&result,reply+sizeof(OMWireHeader),sizeof(result)); + if(!result.error){native_gpu_broker_closed=native_gpu_broker_closed+1;native_gpu_broker_active=native_gpu_broker_active-1;} + } + if(reply)IOFreeData(reply,length);IOLockUnlock(gate); + } + void dropSharedExport(uint64_t exported) { + if(!exported)return;IOLockLock(gate); + OMWireHeader header={OM_WIRE_MAGIC,OFFLINE_METAL_VERSION,OM_SHARED_EXPORT_DROP,sizeof(OMHandle),nextControlRequest};OMHandle input={exported}; + unsigned char *reply=nullptr;size_t length=0;IOReturn status=exchange(0,header,&input,&reply,&length); + if(status==kIOReturnSuccess){nextControlRequest++;OMResult result={};memcpy(&result,reply+sizeof(OMWireHeader),sizeof(result));if(!result.error)__atomic_fetch_add(&native_gpu_shared_drops,1,__ATOMIC_RELAXED);} + if(reply)IOFreeData(reply,length);IOLockUnlock(gate); + } +public: + bool start(IOService *provider) override { + if(!IOService::start(provider))return false; + gate=IOLockAlloc();if(!gate){IOService::stop(provider);return false;} + if(!provider->open(this)){IOLockFree(gate);gate=nullptr;IOService::stop(provider);return false;} + transport=provider;online=true;native_gpu_broker_stage=2;return true; + } + void stop(IOService *provider) override { + if(gate){IOLockLock(gate);online=false;if(transport)transport->close(this);transport=nullptr;IOLockUnlock(gate);} + IOService::stop(provider); + } + void free() override { + if(transport){transport->close(this);transport=nullptr;} + if(gate){IOLockFree(gate);gate=nullptr;}IOService::free(); + } + IOReturn newUserClient(task_t,void *,UInt32,OSDictionary *,IOUserClient **) override; +}; +// A real kernel OSObject carried exclusively by an IKOT_IOKIT_IDENT Mach +// right. No user-supplied integer can stand in for this object's authority. +class NativeGPUSharedEventToken : public OSObject { + OSDeclareDefaultStructors(NativeGPUSharedEventToken) + friend class NativeOfflineGPUClient; + NativeOfflineGPU *controller=nullptr; + uint64_t exported=0; + bool initWithExport(NativeOfflineGPU *owner,uint64_t identifier) { + if(!owner || !identifier || !OSObject::init())return false; + controller=owner;controller->retain();exported=identifier; + __atomic_fetch_add(&native_gpu_shared_tokens_live,1,__ATOMIC_RELAXED);return true; + } + void free() override { + if(controller){uint64_t identifier=exported;exported=0;controller->dropSharedExport(identifier);controller->release();controller=nullptr;__atomic_fetch_sub(&native_gpu_shared_tokens_live,1,__ATOMIC_RELAXED);} + OSObject::free(); + } +}; +class NativeOfflineGPUClient : public IOUserClient { + OSDeclareDefaultStructors(NativeOfflineGPUClient) + NativeOfflineGPU *controller=nullptr; + task_t owner=nullptr; // The native IOUserClient base owns this task reference. + IOLock *gate=nullptr; + uint64_t session=0; + uint32_t nextRequest=1,nextHostRequest=1; + unsigned char *reply=nullptr;size_t replyLength=0,replyOffset=0; + void clearReply() {if(reply)IOFreeData(reply,replyLength);reply=nullptr;replyLength=replyOffset=0;} + void closeContext() { + if(gate)IOLockLock(gate); + clearReply();if(session && controller)controller->closeSession(session,nextHostRequest); + session=0;controller=nullptr;if(gate)IOLockUnlock(gate); + } + static IOReturn sendAction(OSObject *target,void *,IOExternalMethodArguments *args) { + auto self=OSDynamicCast(NativeOfflineGPUClient,target);if(!self || !self->gate)return kIOReturnNotOpen; + if(args->structureInputDescriptor)return kIOReturnUnsupported; + if(!args->structureInput || args->structureInputSizestructureInputSize>4096)return kIOReturnBadArgument; + OMWireHeader header={};memcpy(&header,args->structureInput,sizeof(header)); + IOLockLock(self->gate);IOReturn status=kIOReturnNotOpen; + if(self->session && self->controller) { + if(self->replyOffset!=self->replyLength)status=kIOReturnBusy; + else if(header.magic!=OM_WIRE_MAGIC || header.version!=OFFLINE_METAL_VERSION || header.request!=self->nextRequest || + header.bytes+sizeof(header)!=args->structureInputSize || header.bytes>4080 || + header.operationOM_SHARED_LABEL_SET || (header.operation==OM_FINISH && header.bytes))status=kIOReturnBadArgument; + else { + self->clearReply();uint16_t operation=header.operation;if(operation==OM_FINISH)header.operation=OM_SESSION_CLOSE; + IOLockLock(self->controller->gate); + OMWireHeader hostHeader=header;hostHeader.request=self->nextHostRequest; + status=self->controller->exchange(self->session,hostHeader,static_cast(args->structureInput)+sizeof(header),&self->reply,&self->replyLength); + if(status==kIOReturnSuccess) { + self->nextRequest++;self->nextHostRequest++; + // Preserve the real namespace-close result in the client's + // own framing; no host-wide Finish is forwarded. + header.operation=operation|OM_WIRE_RESPONSE;header.bytes=(uint32_t)(self->replyLength-sizeof(header)); + memcpy(self->reply,&header,sizeof(header));OMResult result={};memcpy(&result,self->reply+sizeof(header),sizeof(result)); + if(operation==OM_FINISH && !result.error) { + self->session=0;native_gpu_broker_closed=native_gpu_broker_closed+1;native_gpu_broker_active=native_gpu_broker_active-1; + } + } + IOLockUnlock(self->controller->gate); + } + } + IOLockUnlock(self->gate);return status; + } + static IOReturn receiveAction(OSObject *target,void *,IOExternalMethodArguments *args) { + auto self=OSDynamicCast(NativeOfflineGPUClient,target);if(!self || !self->gate)return kIOReturnNotOpen; + if(args->structureOutputDescriptor)return kIOReturnUnsupported; + if(!args->structureOutput || !args->structureOutputSize || args->structureOutputSize>4096)return kIOReturnBadArgument; + IOLockLock(self->gate);size_t available=self->replyLength-self->replyOffset; + size_t count=args->structureOutputSize;if(count>available)count=available; + if(count){memcpy(args->structureOutput,self->reply+self->replyOffset,count);self->replyOffset+=count;} + args->structureOutputSize=(uint32_t)count;IOLockUnlock(self->gate);return count?kIOReturnSuccess:kIOReturnNotReady; + } + IOReturn sharedExchange(uint16_t operation,uint64_t identifier,unsigned char **output,size_t *length) { + *output=nullptr;*length=0; + if(!session || !controller)return kIOReturnNotOpen; + if(replyOffset!=replyLength)return kIOReturnBusy; + OMWireHeader header={OM_WIRE_MAGIC,OFFLINE_METAL_VERSION,operation,sizeof(OMHandle),nextHostRequest};OMHandle input={identifier}; + IOLockLock(controller->gate);IOReturn status=controller->exchange(session,header,&input,output,length); + if(status==kIOReturnSuccess)nextHostRequest++;IOLockUnlock(controller->gate);return status; + } + static IOReturn exportSharedAction(OSObject *target,void *,IOExternalMethodArguments *args) { + auto self=OSDynamicCast(NativeOfflineGPUClient,target);if(!self || !self->gate)return kIOReturnNotOpen; + if(current_task()!=self->owner || args->structureOutputDescriptor || !args->structureOutput || !args->structureOutputSize || args->structureOutputSize>4096)return kIOReturnBadArgument; + args->scalarOutput[0]=0;IOLockLock(self->gate);unsigned char *bytes=nullptr;size_t length=0; + IOReturn status=self->sharedExchange(OM_SHARED_EXPORT,args->scalarInput[0],&bytes,&length);OMResult result={}; + if(status==kIOReturnSuccess) { + memcpy(&result,bytes+sizeof(OMWireHeader),sizeof(result));size_t prefix=sizeof(OMWireHeader)+sizeof(result); + if(result.error || !result.handle)status=kIOReturnBadArgument; + else { + __atomic_fetch_add(&native_gpu_shared_exports,1,__ATOMIC_RELAXED); + size_t count=length-prefix; + if(!count || count>args->structureOutputSize){self->controller->dropSharedExport(result.handle);status=kIOReturnNoSpace;} + else { + auto token=new NativeGPUSharedEventToken; + if(!token || !token->initWithExport(self->controller,result.handle)) { + OSSafeReleaseNULL(token);self->controller->dropSharedExport(result.handle);status=kIOReturnNoMemory; + } else { + // Kernel MACH_PORT_NULL is an ipc_port pointer, while + // this API returns a name in the owner's namespace. + mach_port_name_t name=0;status=IOUserClient::copyPortNameForObjectInTask(self->owner,token,&name); + if(status==kIOReturnSuccess && name && name!=MACH_PORT_DEAD) { + args->scalarOutput[0]=name;memcpy(args->structureOutput,bytes+prefix,count);args->structureOutputSize=(uint32_t)count; + __atomic_fetch_add(&native_gpu_shared_ports,1,__ATOMIC_RELAXED); + } else if(status==kIOReturnSuccess)status=kIOReturnNoResources; + token->release(); + } + } + } + } + if(bytes)IOFreeData(bytes,length);IOLockUnlock(self->gate); + if(status!=kIOReturnSuccess)__atomic_fetch_add(&native_gpu_shared_refusals,1,__ATOMIC_RELAXED);return status; + } + static IOReturn importSharedAction(OSObject *target,void *,IOExternalMethodArguments *args) { + auto self=OSDynamicCast(NativeOfflineGPUClient,target);if(!self || !self->gate)return kIOReturnNotOpen; + if(current_task()!=self->owner || args->scalarInput[0]>UINT32_MAX || args->structureOutputDescriptor || !args->structureOutput || !args->structureOutputSize || args->structureOutputSize>4096)return kIOReturnBadArgument; + args->scalarOutput[0]=0;OSObject *object=nullptr;IOReturn status=IOUserClient::copyObjectForPortNameInTask(self->owner,(mach_port_name_t)args->scalarInput[0],&object); + auto token=OSDynamicCast(NativeGPUSharedEventToken,object);unsigned char *bytes=nullptr;size_t length=0;IOLockLock(self->gate); + if(status!=kIOReturnSuccess || !token || token->controller!=self->controller || !token->exported)status=kIOReturnBadArgument; + else { + status=self->sharedExchange(OM_SHARED_IMPORT,token->exported,&bytes,&length); + if(status==kIOReturnSuccess) { + OMResult result={};memcpy(&result,bytes+sizeof(OMWireHeader),sizeof(result));size_t prefix=sizeof(OMWireHeader)+sizeof(result),count=length-prefix; + if(result.error || !result.handle)status=kIOReturnBadArgument; + else if(!count || count>args->structureOutputSize) { + // Retire the just-created independent event handle through + // the same real session without consuming a user sequence. + unsigned char *released=nullptr;size_t releasedLength=0; + (void)self->sharedExchange(OM_RELEASE,result.handle,&released,&releasedLength);if(released)IOFreeData(released,releasedLength);status=kIOReturnNoSpace; + } else {args->scalarOutput[0]=result.handle;memcpy(args->structureOutput,bytes+prefix,count);args->structureOutputSize=(uint32_t)count;__atomic_fetch_add(&native_gpu_shared_imports,1,__ATOMIC_RELAXED);} + } + } + if(bytes)IOFreeData(bytes,length);OSSafeReleaseNULL(object);IOLockUnlock(self->gate); + if(status!=kIOReturnSuccess)__atomic_fetch_add(&native_gpu_shared_refusals,1,__ATOMIC_RELAXED);return status; + } +public: + bool initWithTask(task_t task,void *security,UInt32 type,OSDictionary *properties) override { + if(type || clientHasPrivilege(security,kIOClientPrivilegeAdministrator)!=kIOReturnSuccess || + !IOUserClient::initWithTask(task,security,type,properties))return false; + owner=task;gate=IOLockAlloc();return gate!=nullptr; + } + bool start(IOService *provider) override { + controller=OSDynamicCast(NativeOfflineGPU,provider); + if(!controller || !IOUserClient::start(provider))return false; + if(controller->createSession(owner,&session)!=kIOReturnSuccess){controller=nullptr;IOUserClient::stop(provider);return false;} + return true; + } + void stop(IOService *provider) override {closeContext();IOUserClient::stop(provider);} + void free() override {closeContext();if(gate){IOLockFree(gate);gate=nullptr;}IOUserClient::free();} + IOReturn clientClose() override {terminate();return kIOReturnSuccess;} + IOReturn externalMethod(uint32_t selector,IOExternalMethodArguments *args,IOExternalMethodDispatch *,OSObject *,void *) override { + static const IOExternalMethodDispatch methods[]={{sendAction,0,kIOUCVariableStructureSize,0,0},{receiveAction,0,0,0,kIOUCVariableStructureSize}, + {exportSharedAction,1,0,1,kIOUCVariableStructureSize},{importSharedAction,1,0,1,kIOUCVariableStructureSize}}; + if(selector>=4)return kIOReturnUnsupported; + return IOUserClient::externalMethod(selector,args,const_cast(methods+selector),this,nullptr); + } +}; +OSDefineMetaClassAndStructors(NativeOfflineGPU,IOService) +OSDefineMetaClassAndStructors(NativeOfflineGPUClient,IOUserClient) +OSDefineMetaClassAndStructors(NativeGPUSharedEventToken,OSObject) +IOReturn NativeOfflineGPU::newUserClient(task_t task,void *security,UInt32 type,OSDictionary *properties,IOUserClient **out) { + if(!out)return kIOReturnBadArgument;*out=nullptr; + auto client=new NativeOfflineGPUClient;if(!client)return kIOReturnNoMemory; + bool attached=false; + if(!client->initWithTask(task,security,type,properties) || !(attached=client->attach(this)) || !client->start(this)) { + if(attached)client->detach(this);client->release();return kIOReturnNotOpen; + } + *out=client;return kIOReturnSuccess; +} +extern "C" IOService *native_offline_gpu_create(void) {return new NativeOfflineGPU;} diff --git a/drivers/NativeMetal/NativeTurnipSmoke.h b/drivers/NativeMetal/NativeTurnipSmoke.h new file mode 100644 index 0000000..8450ee7 --- /dev/null +++ b/drivers/NativeMetal/NativeTurnipSmoke.h @@ -0,0 +1,653 @@ +// SPDX-License-Identifier: MIT +// Turnip smoke test, run by the Adreno Metal plugin from inside the system process that loaded it (under that process's sandbox) once the raw +// userspace test (NativeAdrenoUserTest.h) has passed. It loads the real Mesa Turnip driver (libvulkan_freedreno.dylib, built with the native +// Darwin kernel backend and shipped next to the plugin binary) and drives it through the Vulkan ICD entry points without a loader: +// instance, physical device, device, queue, a host-visible buffer, one vkCmdFillBuffer, a fence, and a read-back of the fill pattern. +// Every stage leaves a progress marker (IOServiceOpen type 0x4147, see NativeAdrenoUserTest.h): 6001 library loaded, 6002 entry points, +// 6003 instance, 6004 physical device, 6005 device, 6007 buffer memory mapped, 6006 command buffer recorded, 6008 submitted, 6009 fence +// signalled, 600a fill pattern verified; then the shader tests: 6100 compute begins, 6101 pipeline, 6102 recorded, 6103 fence, 6104 all words +// right (61fS = compute step S failed); 6200 render begins, 6201 pipeline, 6202 recorded, 6203 fence, 6204 image right (62fS = render step S +// failed); 600b everything destroyed. A setup failure leaves 6fSS (stage SS) followed by 6eRR (RR = -VkResult). +// The outcome is a short text for the kernel's report row: +// "T s r v c g/ ". +#pragma once +#define VK_NO_PROTOTYPES 1 +#include +#include "NativeTurnipShaders.h" // generated: scripts/gen-turnip-smoke-shaders.py +#include +#include +#include +#include +#include +#include +#include +#include + +// Vulkan create-info structs are written as {sType} with every other member zero. +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wmissing-field-initializers" +namespace turnip { + +static char gFirstError[96]; + +static VKAPI_ATTR VkBool32 VKAPI_CALL onMessage(VkDebugUtilsMessageSeverityFlagBitsEXT severity, VkDebugUtilsMessageTypeFlagsEXT, + const VkDebugUtilsMessengerCallbackDataEXT *data, void *) { + if ((severity & VK_DEBUG_UTILS_MESSAGE_SEVERITY_ERROR_BIT_EXT) && !gFirstError[0] && data && data->pMessage) + snprintf(gFirstError, sizeof gFirstError, "%s", data->pMessage); + return VK_FALSE; +} + +struct Result { + int stage = 0; // first setup failure (0 = none): 1 library .. 9 fence wait + int vkResult = 0; + int verified = -1; // fill test: words of 1024 that hold the expected value (-1 = not run) + int computeWords = -1; // compute test: words of 1024 written by the shader with the expected value (-1 = not run) + int computeStage = 0; // first failing step of the compute test (0 = none) + int renderTriangle = -1, renderOther = -1; // render test: triangle-colour pixels, pixels that are neither colour (-1 = not run) + int renderStage = 0; + bool ok = false; +}; + +// Device-level entry points the tests use, resolved once through vkGetDeviceProcAddr. +#define TURNIP_DEVICE_FNS(X) \ + X(vkDestroyDevice) X(vkGetDeviceQueue) X(vkCreateCommandPool) X(vkDestroyCommandPool) X(vkAllocateCommandBuffers) \ + X(vkResetCommandBuffer) X(vkBeginCommandBuffer) X(vkEndCommandBuffer) X(vkCmdFillBuffer) X(vkCreateBuffer) X(vkDestroyBuffer) \ + X(vkGetBufferMemoryRequirements) X(vkAllocateMemory) X(vkFreeMemory) X(vkBindBufferMemory) X(vkMapMemory) X(vkUnmapMemory) \ + X(vkCreateFence) X(vkDestroyFence) X(vkQueueSubmit) X(vkWaitForFences) X(vkDeviceWaitIdle) X(vkCreateShaderModule) \ + X(vkDestroyShaderModule) X(vkCreateDescriptorSetLayout) X(vkDestroyDescriptorSetLayout) X(vkCreatePipelineLayout) \ + X(vkDestroyPipelineLayout) X(vkCreateComputePipelines) X(vkCreateGraphicsPipelines) X(vkDestroyPipeline) \ + X(vkCreateDescriptorPool) X(vkDestroyDescriptorPool) X(vkAllocateDescriptorSets) X(vkUpdateDescriptorSets) \ + X(vkCmdBindPipeline) X(vkCmdBindDescriptorSets) X(vkCmdPushConstants) X(vkCmdDispatch) X(vkCmdPipelineBarrier) \ + X(vkCreateImage) X(vkDestroyImage) X(vkGetImageMemoryRequirements) X(vkBindImageMemory) X(vkCreateImageView) \ + X(vkDestroyImageView) X(vkCreateRenderPass) X(vkDestroyRenderPass) X(vkCreateFramebuffer) X(vkDestroyFramebuffer) \ + X(vkCmdBeginRenderPass) X(vkCmdEndRenderPass) X(vkCmdDraw) X(vkCmdCopyImageToBuffer) + +struct Device { + VkDevice device = nullptr; + VkQueue queue = nullptr; + VkCommandPool pool = nullptr; + VkCommandBuffer cmd = nullptr; + VkPhysicalDeviceMemoryProperties memory = {}; +#define TURNIP_DECLARE(name) PFN_##name name = nullptr; + TURNIP_DEVICE_FNS(TURNIP_DECLARE) +#undef TURNIP_DECLARE + + uint32_t memoryType(uint32_t bits, VkMemoryPropertyFlags flags) const { + for (uint32_t i = 0; i < memory.memoryTypeCount; ++i) + if ((bits & (1u << i)) && (memory.memoryTypes[i].propertyFlags & flags) == flags) return i; + return UINT32_MAX; + } + // Records into the one command buffer, submits it with a fresh fence and waits up to five seconds. + VkResult submitAndWait() { + VkFenceCreateInfo fci = {VK_STRUCTURE_TYPE_FENCE_CREATE_INFO}; + VkFence fence = nullptr; + VkResult r = vkCreateFence(device, &fci, nullptr, &fence); + if (r != VK_SUCCESS) return r; + VkSubmitInfo si = {VK_STRUCTURE_TYPE_SUBMIT_INFO}; + si.commandBufferCount = 1; + si.pCommandBuffers = &cmd; + r = vkQueueSubmit(queue, 1, &si, fence); + if (r == VK_SUCCESS) r = vkWaitForFences(device, 1, &fence, VK_TRUE, 5000000000ull); + vkDestroyFence(device, fence, nullptr); + return r; + } + VkResult begin() { + vkResetCommandBuffer(cmd, 0); + VkCommandBufferBeginInfo bi = {VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO}; + bi.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; + return vkBeginCommandBuffer(cmd, &bi); + } +}; + +// A host-visible buffer with its memory, mapped for the CPU. +struct HostBuffer { + VkBuffer buffer = nullptr; + VkDeviceMemory memory = nullptr; + volatile uint32_t *words = nullptr; + VkResult create(Device &d, VkDeviceSize bytes, VkBufferUsageFlags usage) { + VkBufferCreateInfo bci = {VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO}; + bci.size = bytes; + bci.usage = usage; + bci.sharingMode = VK_SHARING_MODE_EXCLUSIVE; + VkResult r = d.vkCreateBuffer(d.device, &bci, nullptr, &buffer); + if (r != VK_SUCCESS) return r; + VkMemoryRequirements req = {}; + d.vkGetBufferMemoryRequirements(d.device, buffer, &req); + const uint32_t type = d.memoryType(req.memoryTypeBits, VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT); + if (type == UINT32_MAX) return VK_ERROR_FEATURE_NOT_PRESENT; + VkMemoryAllocateInfo mai = {VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO}; + mai.allocationSize = req.size; + mai.memoryTypeIndex = type; + r = d.vkAllocateMemory(d.device, &mai, nullptr, &memory); + if (r == VK_SUCCESS) r = d.vkBindBufferMemory(d.device, buffer, memory, 0); + void *mapped = nullptr; + if (r == VK_SUCCESS) r = d.vkMapMemory(d.device, memory, 0, bytes, 0, &mapped); + words = static_cast(mapped); + return r != VK_SUCCESS ? r : (words ? VK_SUCCESS : VK_ERROR_MEMORY_MAP_FAILED); + } + void destroy(Device &d) { + if (words) d.vkUnmapMemory(d.device, memory); + if (buffer) d.vkDestroyBuffer(d.device, buffer, nullptr); + if (memory) d.vkFreeMemory(d.device, memory, nullptr); + *this = HostBuffer(); + } +}; + +static VkShaderModule shaderModule(Device &d, const uint32_t *words, size_t bytes) { + VkShaderModuleCreateInfo ci = {VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO}; + ci.codeSize = bytes; + ci.pCode = words; + VkShaderModule module = nullptr; + return d.vkCreateShaderModule(d.device, &ci, nullptr, &module) == VK_SUCCESS ? module : nullptr; +} + +static void hostBarrier(Device &d, VkPipelineStageFlags from, VkAccessFlags access) { + VkMemoryBarrier barrier = {VK_STRUCTURE_TYPE_MEMORY_BARRIER}; + barrier.srcAccessMask = access; + barrier.dstAccessMask = VK_ACCESS_HOST_READ_BIT; + d.vkCmdPipelineBarrier(d.cmd, from, VK_PIPELINE_STAGE_HOST_BIT, 0, 1, &barrier, 0, nullptr, 0, nullptr); +} + +// Fill: vkCmdFillBuffer of the first half of a 4 KiB buffer (Turnip implements it with its own internal blit). Returns verified words or < 0. +template +static int fillTest(Device &d, Marker &mark, int &failStage, VkResult &failResult) { + constexpr VkDeviceSize kBytes = 4096; + constexpr uint32_t kPattern = 0xc0ffee42u; + HostBuffer b; + VkResult r = b.create(d, kBytes, VK_BUFFER_USAGE_TRANSFER_DST_BIT); + if (r != VK_SUCCESS) {b.destroy(d); failStage = 7; failResult = r; return -1;} + for (unsigned i = 0; i < kBytes / 4; ++i) b.words[i] = 0xdeadbeef; + mark(0x6007); + if ((r = d.begin()) != VK_SUCCESS) {b.destroy(d); failStage = 6; failResult = r; return -1;} + d.vkCmdFillBuffer(d.cmd, b.buffer, 0, kBytes / 2, kPattern); // the first half only: the second half must keep 0xdeadbeef + hostBarrier(d, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_TRANSFER_WRITE_BIT); + if ((r = d.vkEndCommandBuffer(d.cmd)) != VK_SUCCESS) {b.destroy(d); failStage = 6; failResult = r; return -1;} + mark(0x6006); + mark(0x6008); + if ((r = d.submitAndWait()) != VK_SUCCESS) {b.destroy(d); failStage = 9; failResult = r; return -1;} + mark(0x6009); + int good = 0; + for (unsigned i = 0; i < kBytes / 8; ++i) good += b.words[i] == kPattern; + for (unsigned i = kBytes / 8; i < kBytes / 4; ++i) good += b.words[i] == 0xdeadbeefu; + if (good == int(kBytes / 4)) mark(0x600a); + b.destroy(d); + return good; +} + +// Compute: 16 workgroups of 64 invocations each write v[i] = i * 3 + 0x1000 (push constants) into a storage buffer. +template +static int computeTest(Device &d, Marker &mark, int &stage) { + constexpr uint32_t kWords = 1024, kMul = 3, kAdd = 0x1000; + mark(0x6100); + HostBuffer b; + VkShaderModule module = nullptr; + VkDescriptorSetLayout setLayout = nullptr; + VkPipelineLayout layout = nullptr; + VkPipeline pipeline = nullptr; + VkDescriptorPool pool = nullptr; + int result = -1; + auto fail = [&](int s) {stage = s; mark(0x61f0 | s);}; + do { + if (b.create(d, kWords * 4, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT) != VK_SUCCESS) {fail(1); break;} + for (uint32_t i = 0; i < kWords; ++i) b.words[i] = 0xdeadbeef; + if (!(module = shaderModule(d, kSpv_smoke_fill_comp, sizeof kSpv_smoke_fill_comp))) {fail(2); break;} + VkDescriptorSetLayoutBinding binding = {0, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1, VK_SHADER_STAGE_COMPUTE_BIT, nullptr}; + VkDescriptorSetLayoutCreateInfo dsl = {VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO}; + dsl.bindingCount = 1; + dsl.pBindings = &binding; + if (d.vkCreateDescriptorSetLayout(d.device, &dsl, nullptr, &setLayout) != VK_SUCCESS) {fail(3); break;} + VkPushConstantRange push = {VK_SHADER_STAGE_COMPUTE_BIT, 0, 8}; + VkPipelineLayoutCreateInfo pl = {VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO}; + pl.setLayoutCount = 1; + pl.pSetLayouts = &setLayout; + pl.pushConstantRangeCount = 1; + pl.pPushConstantRanges = &push; + if (d.vkCreatePipelineLayout(d.device, &pl, nullptr, &layout) != VK_SUCCESS) {fail(4); break;} + VkComputePipelineCreateInfo cp = {VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO}; + cp.stage = {VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, nullptr, 0, VK_SHADER_STAGE_COMPUTE_BIT, module, "main", nullptr}; + cp.layout = layout; + if (d.vkCreateComputePipelines(d.device, nullptr, 1, &cp, nullptr, &pipeline) != VK_SUCCESS) {fail(5); break;} + mark(0x6101); + VkDescriptorPoolSize size = {VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1}; + VkDescriptorPoolCreateInfo dp = {VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO}; + dp.maxSets = 1; + dp.poolSizeCount = 1; + dp.pPoolSizes = &size; + if (d.vkCreateDescriptorPool(d.device, &dp, nullptr, &pool) != VK_SUCCESS) {fail(6); break;} + VkDescriptorSetAllocateInfo da = {VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO}; + da.descriptorPool = pool; + da.descriptorSetCount = 1; + da.pSetLayouts = &setLayout; + VkDescriptorSet set = nullptr; + if (d.vkAllocateDescriptorSets(d.device, &da, &set) != VK_SUCCESS) {fail(6); break;} + VkDescriptorBufferInfo info = {b.buffer, 0, kWords * 4}; + VkWriteDescriptorSet write = {VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET}; + write.dstSet = set; + write.descriptorCount = 1; + write.descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER; + write.pBufferInfo = &info; + d.vkUpdateDescriptorSets(d.device, 1, &write, 0, nullptr); + if (d.begin() != VK_SUCCESS) {fail(7); break;} + const uint32_t constants[2] = {kMul, kAdd}; + d.vkCmdBindPipeline(d.cmd, VK_PIPELINE_BIND_POINT_COMPUTE, pipeline); + d.vkCmdBindDescriptorSets(d.cmd, VK_PIPELINE_BIND_POINT_COMPUTE, layout, 0, 1, &set, 0, nullptr); + d.vkCmdPushConstants(d.cmd, layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof constants, constants); + d.vkCmdDispatch(d.cmd, kWords / 64, 1, 1); + hostBarrier(d, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_SHADER_WRITE_BIT); + if (d.vkEndCommandBuffer(d.cmd) != VK_SUCCESS) {fail(7); break;} + mark(0x6102); + if (d.submitAndWait() != VK_SUCCESS) {fail(8); break;} + mark(0x6103); + int good = 0; + for (uint32_t i = 0; i < kWords; ++i) good += b.words[i] == i * kMul + kAdd; + result = good; + if (good == int(kWords)) mark(0x6104); + } while (false); + if (pipeline) d.vkDestroyPipeline(d.device, pipeline, nullptr); + if (pool) d.vkDestroyDescriptorPool(d.device, pool, nullptr); + if (layout) d.vkDestroyPipelineLayout(d.device, layout, nullptr); + if (setLayout) d.vkDestroyDescriptorSetLayout(d.device, setLayout, nullptr); + if (module) d.vkDestroyShaderModule(d.device, module, nullptr); + b.destroy(d); + return result; +} + +// Render: a 64x64 RGBA8 colour attachment cleared to blue, one triangle drawn in orange (push constant), copied to a host buffer. +// The triangle covers 512 pixels in exact arithmetic; every other pixel must keep the clear colour. +template +static void renderTest(Device &d, Marker &mark, int &stage, int &triangle, int &other) { + constexpr uint32_t kSize = 64; + mark(0x6200); + HostBuffer b; + VkImage image = nullptr; + VkDeviceMemory imageMemory = nullptr; + VkImageView view = nullptr; + VkRenderPass pass = nullptr; + VkFramebuffer framebuffer = nullptr; + VkShaderModule vs = nullptr, fs = nullptr; + VkPipelineLayout layout = nullptr; + VkPipeline pipeline = nullptr; + auto fail = [&](int s) {stage = s; mark(0x62f0 | s);}; + do { + if (b.create(d, kSize * kSize * 4, VK_BUFFER_USAGE_TRANSFER_DST_BIT) != VK_SUCCESS) {fail(1); break;} + for (uint32_t i = 0; i < kSize * kSize; ++i) b.words[i] = 0x5a5a5a5a; + VkImageCreateInfo ic = {VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO}; + ic.imageType = VK_IMAGE_TYPE_2D; + ic.format = VK_FORMAT_R8G8B8A8_UNORM; + ic.extent = {kSize, kSize, 1}; + ic.mipLevels = 1; + ic.arrayLayers = 1; + ic.samples = VK_SAMPLE_COUNT_1_BIT; + ic.tiling = VK_IMAGE_TILING_OPTIMAL; + ic.usage = VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT; + ic.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; + if (d.vkCreateImage(d.device, &ic, nullptr, &image) != VK_SUCCESS) {fail(2); break;} + VkMemoryRequirements req = {}; + d.vkGetImageMemoryRequirements(d.device, image, &req); + uint32_t type = d.memoryType(req.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); + if (type == UINT32_MAX) type = d.memoryType(req.memoryTypeBits, 0); + VkMemoryAllocateInfo mai = {VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO}; + mai.allocationSize = req.size; + mai.memoryTypeIndex = type; + if (type == UINT32_MAX || d.vkAllocateMemory(d.device, &mai, nullptr, &imageMemory) != VK_SUCCESS || + d.vkBindImageMemory(d.device, image, imageMemory, 0) != VK_SUCCESS) {fail(2); break;} + VkImageViewCreateInfo vc = {VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO}; + vc.image = image; + vc.viewType = VK_IMAGE_VIEW_TYPE_2D; + vc.format = ic.format; + vc.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}; + if (d.vkCreateImageView(d.device, &vc, nullptr, &view) != VK_SUCCESS) {fail(3); break;} + VkAttachmentDescription attachment = {0, ic.format, VK_SAMPLE_COUNT_1_BIT, VK_ATTACHMENT_LOAD_OP_CLEAR, VK_ATTACHMENT_STORE_OP_STORE, + VK_ATTACHMENT_LOAD_OP_DONT_CARE, VK_ATTACHMENT_STORE_OP_DONT_CARE, VK_IMAGE_LAYOUT_UNDEFINED, + VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL}; + VkAttachmentReference ref = {0, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL}; + VkSubpassDescription subpass = {}; + subpass.pipelineBindPoint = VK_PIPELINE_BIND_POINT_GRAPHICS; + subpass.colorAttachmentCount = 1; + subpass.pColorAttachments = &ref; + VkSubpassDependency after = {0, VK_SUBPASS_EXTERNAL, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, + VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT, 0}; + VkRenderPassCreateInfo rp = {VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO}; + rp.attachmentCount = 1; + rp.pAttachments = &attachment; + rp.subpassCount = 1; + rp.pSubpasses = &subpass; + rp.dependencyCount = 1; + rp.pDependencies = &after; + if (d.vkCreateRenderPass(d.device, &rp, nullptr, &pass) != VK_SUCCESS) {fail(4); break;} + VkFramebufferCreateInfo fb = {VK_STRUCTURE_TYPE_FRAMEBUFFER_CREATE_INFO}; + fb.renderPass = pass; + fb.attachmentCount = 1; + fb.pAttachments = &view; + fb.width = kSize; + fb.height = kSize; + fb.layers = 1; + if (d.vkCreateFramebuffer(d.device, &fb, nullptr, &framebuffer) != VK_SUCCESS) {fail(4); break;} + vs = shaderModule(d, kSpv_smoke_triangle_vert, sizeof kSpv_smoke_triangle_vert); + fs = shaderModule(d, kSpv_smoke_triangle_frag, sizeof kSpv_smoke_triangle_frag); + if (!vs || !fs) {fail(5); break;} + VkPushConstantRange push = {VK_SHADER_STAGE_FRAGMENT_BIT, 0, 16}; + VkPipelineLayoutCreateInfo pl = {VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO}; + pl.pushConstantRangeCount = 1; + pl.pPushConstantRanges = &push; + if (d.vkCreatePipelineLayout(d.device, &pl, nullptr, &layout) != VK_SUCCESS) {fail(6); break;} + VkPipelineShaderStageCreateInfo stages[2] = { + {VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, nullptr, 0, VK_SHADER_STAGE_VERTEX_BIT, vs, "main", nullptr}, + {VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, nullptr, 0, VK_SHADER_STAGE_FRAGMENT_BIT, fs, "main", nullptr}}; + VkPipelineVertexInputStateCreateInfo vi = {VK_STRUCTURE_TYPE_PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO}; + VkPipelineInputAssemblyStateCreateInfo ia = {VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO}; + ia.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST; + VkViewport viewport = {0, 0, float(kSize), float(kSize), 0, 1}; + VkRect2D scissor = {{0, 0}, {kSize, kSize}}; + VkPipelineViewportStateCreateInfo vp = {VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO}; + vp.viewportCount = 1; + vp.pViewports = &viewport; + vp.scissorCount = 1; + vp.pScissors = &scissor; + VkPipelineRasterizationStateCreateInfo rs = {VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO}; + rs.polygonMode = VK_POLYGON_MODE_FILL; + rs.cullMode = VK_CULL_MODE_NONE; + rs.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE; + rs.lineWidth = 1.0f; + VkPipelineMultisampleStateCreateInfo ms = {VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO}; + ms.rasterizationSamples = VK_SAMPLE_COUNT_1_BIT; + VkPipelineColorBlendAttachmentState blend = {}; + blend.colorWriteMask = 0xf; + VkPipelineColorBlendStateCreateInfo cb = {VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO}; + cb.attachmentCount = 1; + cb.pAttachments = &blend; + VkGraphicsPipelineCreateInfo gp = {VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO}; + gp.stageCount = 2; + gp.pStages = stages; + gp.pVertexInputState = &vi; + gp.pInputAssemblyState = &ia; + gp.pViewportState = &vp; + gp.pRasterizationState = &rs; + gp.pMultisampleState = &ms; + gp.pColorBlendState = &cb; + gp.layout = layout; + gp.renderPass = pass; + if (d.vkCreateGraphicsPipelines(d.device, nullptr, 1, &gp, nullptr, &pipeline) != VK_SUCCESS) {fail(7); break;} + mark(0x6201); + if (d.begin() != VK_SUCCESS) {fail(8); break;} + VkClearValue clear = {}; + clear.color.float32[2] = 1.0f; + clear.color.float32[3] = 1.0f; + VkRenderPassBeginInfo rb = {VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO}; + rb.renderPass = pass; + rb.framebuffer = framebuffer; + rb.renderArea = scissor; + rb.clearValueCount = 1; + rb.pClearValues = &clear; + d.vkCmdBeginRenderPass(d.cmd, &rb, VK_SUBPASS_CONTENTS_INLINE); + d.vkCmdBindPipeline(d.cmd, VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline); + const float color[4] = {1.0f, 0.5f, 0.0f, 1.0f}; + d.vkCmdPushConstants(d.cmd, layout, VK_SHADER_STAGE_FRAGMENT_BIT, 0, sizeof color, color); + d.vkCmdDraw(d.cmd, 3, 1, 0, 0); + d.vkCmdEndRenderPass(d.cmd); + VkBufferImageCopy copy = {}; + copy.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}; + copy.imageExtent = {kSize, kSize, 1}; + d.vkCmdCopyImageToBuffer(d.cmd, image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, b.buffer, 1, ©); + hostBarrier(d, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_TRANSFER_WRITE_BIT); + if (d.vkEndCommandBuffer(d.cmd) != VK_SUCCESS) {fail(8); break;} + mark(0x6202); + if (d.submitAndWait() != VK_SUCCESS) {fail(9); break;} + mark(0x6203); + // RGBA8 little-endian words: clear = (0, 0, 255, 255), triangle = (255, 127 or 128, 0, 255). + int tri = 0, rest = 0; + for (uint32_t i = 0; i < kSize * kSize; ++i) { + const uint32_t px = b.words[i]; + if (px == 0xffff0000u) continue; + if (px == 0xff0080ffu || px == 0xff007fffu) ++tri; + else ++rest; + } + triangle = tri; + other = rest; + if (rest == 0 && tri >= 480 && tri <= 544) mark(0x6204); + } while (false); + if (pipeline) d.vkDestroyPipeline(d.device, pipeline, nullptr); + if (layout) d.vkDestroyPipelineLayout(d.device, layout, nullptr); + if (vs) d.vkDestroyShaderModule(d.device, vs, nullptr); + if (fs) d.vkDestroyShaderModule(d.device, fs, nullptr); + if (framebuffer) d.vkDestroyFramebuffer(d.device, framebuffer, nullptr); + if (pass) d.vkDestroyRenderPass(d.device, pass, nullptr); + if (view) d.vkDestroyImageView(d.device, view, nullptr); + if (image) d.vkDestroyImage(d.device, image, nullptr); + if (imageMemory) d.vkFreeMemory(d.device, imageMemory, nullptr); + b.destroy(d); +} + +template +static Result run(Marker &&mark, const char *libraryPath) { + Result out; + auto fail = [&](int stage, int result) { + out.stage = stage; + out.vkResult = result; + mark(0x6f00 | stage); + mark(0x6e00 | ((-result) & 0xff)); + return out; + }; + // The Darwin backend reaches the driver the same way the plugin does: the AGX accelerator service with open type 0x4147, ABI methods through + // an admitted selector. Nothing to configure; set here only so the choice is visible. The shader disk cache is off: this runs inside a + // system process whose home directory is not ours to fill. + setenv("TU_DARWIN_SERVICE", "IOAcceleratorES", 0); + setenv("MESA_SHADER_CACHE_DISABLE", "true", 0); + // Concurrent binning (BV running ahead of BR) is a performance feature; its barriers make BR wait for BV timestamps. int-058's + // first fill hung with the ring stopped inside the IB (no fault), before the kernel framed IBs for both threads. Keep it off + // until a run with the framing has passed. + setenv("TU_DEBUG", "nocb", 0); + + void *lib = dlopen(libraryPath, RTLD_NOW | RTLD_LOCAL); + if (!lib) return fail(1, -1); + mark(0x6001); + auto gipa = reinterpret_cast(dlsym(lib, "vk_icdGetInstanceProcAddr")); + if (!gipa) return fail(2, -2); + auto createInstance = reinterpret_cast(gipa(nullptr, "vkCreateInstance")); + if (!createInstance) return fail(2, -3); + mark(0x6002); + + const char *extensions[] = {VK_EXT_DEBUG_UTILS_EXTENSION_NAME}; + VkDebugUtilsMessengerCreateInfoEXT messenger = {VK_STRUCTURE_TYPE_DEBUG_UTILS_MESSENGER_CREATE_INFO_EXT}; + messenger.messageSeverity = VK_DEBUG_UTILS_MESSAGE_SEVERITY_ERROR_BIT_EXT; + messenger.messageType = VK_DEBUG_UTILS_MESSAGE_TYPE_GENERAL_BIT_EXT | VK_DEBUG_UTILS_MESSAGE_TYPE_VALIDATION_BIT_EXT; + messenger.pfnUserCallback = onMessage; + VkApplicationInfo app = {VK_STRUCTURE_TYPE_APPLICATION_INFO}; + app.pApplicationName = "NativeAdrenoMetal"; + app.apiVersion = VK_API_VERSION_1_1; + VkInstanceCreateInfo ici = {VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO}; + ici.pNext = &messenger; + ici.pApplicationInfo = &app; + ici.enabledExtensionCount = 1; + ici.ppEnabledExtensionNames = extensions; + VkInstance instance = nullptr; + VkResult r = createInstance(&ici, nullptr, &instance); + if (r != VK_SUCCESS) return fail(3, r); + mark(0x6003); + +#define INSTANCE_FN(name) auto name = reinterpret_cast(gipa(instance, #name)); if (!name) return fail(3, -4) + INSTANCE_FN(vkEnumeratePhysicalDevices); + INSTANCE_FN(vkGetPhysicalDeviceMemoryProperties); + INSTANCE_FN(vkGetPhysicalDeviceQueueFamilyProperties); + INSTANCE_FN(vkCreateDevice); + INSTANCE_FN(vkGetDeviceProcAddr); + INSTANCE_FN(vkDestroyInstance); +#undef INSTANCE_FN + + uint32_t count = 1; + VkPhysicalDevice physical = nullptr; + r = vkEnumeratePhysicalDevices(instance, &count, &physical); + if ((r != VK_SUCCESS && r != VK_INCOMPLETE) || !count || !physical) return fail(4, r == VK_SUCCESS ? -5 : r); + mark(0x6004); + + uint32_t families = 0; + vkGetPhysicalDeviceQueueFamilyProperties(physical, &families, nullptr); + if (!families) return fail(4, -6); + VkQueueFamilyProperties family[8] = {}; + if (families > 8) families = 8; + vkGetPhysicalDeviceQueueFamilyProperties(physical, &families, family); + uint32_t queueFamily = families; + for (uint32_t i = 0; i < families && queueFamily == families; ++i) + if (family[i].queueFlags & VK_QUEUE_GRAPHICS_BIT) queueFamily = i; + if (queueFamily == families) return fail(4, -7); + + float priority = 1.0f; + VkDeviceQueueCreateInfo qci = {VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO}; + qci.queueFamilyIndex = queueFamily; + qci.queueCount = 1; + qci.pQueuePriorities = &priority; + VkDeviceCreateInfo dci = {VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO}; + dci.queueCreateInfoCount = 1; + dci.pQueueCreateInfos = &qci; + Device d; + r = vkCreateDevice(physical, &dci, nullptr, &d.device); + if (r != VK_SUCCESS) return fail(5, r); + mark(0x6005); +#define TURNIP_RESOLVE(name) d.name = reinterpret_cast(vkGetDeviceProcAddr(d.device, #name)); if (!d.name) return fail(5, -8); + TURNIP_DEVICE_FNS(TURNIP_RESOLVE) +#undef TURNIP_RESOLVE + vkGetPhysicalDeviceMemoryProperties(physical, &d.memory); + d.vkGetDeviceQueue(d.device, queueFamily, 0, &d.queue); + VkCommandPoolCreateInfo pci = {VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO}; + pci.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT; + pci.queueFamilyIndex = queueFamily; + r = d.vkCreateCommandPool(d.device, &pci, nullptr, &d.pool); + if (r != VK_SUCCESS) return fail(6, r); + VkCommandBufferAllocateInfo cai = {VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO}; + cai.commandPool = d.pool; + cai.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY; + cai.commandBufferCount = 1; + r = d.vkAllocateCommandBuffers(d.device, &cai, &d.cmd); + if (r != VK_SUCCESS) return fail(6, r); + + // The three tests are independent: a failed or unverified one does not stop the next, unless the GPU stopped retiring work (fence timeout). + int failStage = 0; + VkResult failResult = VK_SUCCESS; + out.verified = fillTest(d, mark, failStage, failResult); + if (failStage) { + fail(failStage, failResult); + } + if (failStage != 9) { + out.computeWords = computeTest(d, mark, out.computeStage); + if (out.computeStage != 8) renderTest(d, mark, out.renderStage, out.renderTriangle, out.renderOther); + } + + d.vkDeviceWaitIdle(d.device); + d.vkDestroyCommandPool(d.device, d.pool, nullptr); + d.vkDestroyDevice(d.device, nullptr); + vkDestroyInstance(instance, nullptr); + mark(0x600b); + out.ok = !failStage && out.verified == 1024 && out.computeWords == 1024 && out.renderOther == 0 && out.renderTriangle >= 480 && + out.renderTriangle <= 544; + return out; +} + +// Watchdog. run() executes on a worker thread; if it has not returned after 15, 45 and 90 seconds, the calling thread samples the worker and +// sends "W s u/k over 300 ms, then the stack": the PC, LR and frame-pointer return addresses, each +// as (v = libvulkan_freedreno.dylib; offsets resolve against the unstripped build). A worker that is +// running in the kernel (system time only, or no time at all: a core parked by the panic screen) is not suspended, because thread_suspend +// would wait for it forever; its line ends in "kern". +struct Watched { + const char *libraryPath; + void (*mark)(void *, uint32_t); + void *markContext; + volatile uint32_t lastMark; + volatile int done; + Result result; +}; + +static void *watchedWorker(void *argument) { + Watched *w = static_cast(argument); + w->result = run([w](uint32_t value) { w->lastMark = value; w->mark(w->markContext, value); }, w->libraryPath); + __atomic_store_n(&w->done, 1, __ATOMIC_RELEASE); + return nullptr; +} + +static size_t frameText(char *out, size_t size, uint64_t address) { + Dl_info info = {}; + if (!address || !dladdr(reinterpret_cast(address), &info) || !info.dli_fbase) + return (size_t)snprintf(out, size, " ?%llx", (unsigned long long)address); + const char *name = info.dli_fname ? strrchr(info.dli_fname, '/') : nullptr; + name = name ? name + 1 : (info.dli_fname ? info.dli_fname : "?"); + const char tag = !strncmp(name, "libvulkan", 9) ? 'v' : name[0]; + return (size_t)snprintf(out, size, " %c%llx", tag, (unsigned long long)(address - reinterpret_cast(info.dli_fbase))); +} + +static void sample(Watched &w, pthread_t worker, char *text, size_t size) { + const mach_port_t thread = pthread_mach_thread_np(worker); + thread_basic_info_data_t a = {}, b = {}; + mach_msg_type_number_t count = THREAD_BASIC_INFO_COUNT; + thread_info(thread, THREAD_BASIC_INFO, reinterpret_cast(&a), &count); + usleep(300000); + count = THREAD_BASIC_INFO_COUNT; + thread_info(thread, THREAD_BASIC_INFO, reinterpret_cast(&b), &count); + auto us = [](time_value_t t) { return int64_t(t.seconds) * 1000000 + t.microseconds; }; + const int64_t du = us(b.user_time) - us(a.user_time), dk = us(b.system_time) - us(a.system_time); + size_t at = (size_t)snprintf(text, size, "W %x s%d u%lld/k%lld", unsigned(w.lastMark), b.run_state, (long long)du, (long long)dk); + if (b.run_state == TH_STATE_RUNNING && du <= 0) { + snprintf(text + at, size - at, " kern"); + return; + } + if (thread_suspend(thread) != KERN_SUCCESS) {snprintf(text + at, size - at, " nosusp"); return;} + arm_thread_state64_t state = {}; + count = ARM_THREAD_STATE64_COUNT; + if (thread_get_state(thread, ARM_THREAD_STATE64, reinterpret_cast(&state), &count) == KERN_SUCCESS) { + const uint64_t pc = arm_thread_state64_get_pc(state), lr = arm_thread_state64_get_lr(state); + uint64_t fp = arm_thread_state64_get_fp(state); + at += frameText(text + at, size - at, pc); + if (at < size) at += frameText(text + at, size - at, lr); + const uint64_t low = reinterpret_cast(pthread_get_stackaddr_np(worker)) - pthread_get_stacksize_np(worker); + const uint64_t high = reinterpret_cast(pthread_get_stackaddr_np(worker)); + for (unsigned i = 0; i < 8 && at + 4 < size && fp >= low && fp + 16 <= high && !(fp & 7); ++i) { + const uint64_t next = reinterpret_cast(fp)[0], ret = reinterpret_cast(fp)[1]; + if (ret != lr || i) at += frameText(text + at, size - at, ret); + if (next <= fp) break; + fp = next; + } + } + thread_resume(thread); +} + +template +static Result runWatched(Marker &mark, Reporter &&report, const char *libraryPath) { + Watched *w = new Watched{}; + w->libraryPath = libraryPath; + w->markContext = &mark; + w->mark = [](void *context, uint32_t value) { (*static_cast(context))(value); }; + pthread_attr_t attr; + pthread_attr_init(&attr); + pthread_attr_setstacksize(&attr, 8u << 20); // Mesa's NIR/ir3 passes recurse; the default 512 KiB secondary stack is small + pthread_t worker; + const bool started = pthread_create(&worker, &attr, watchedWorker, w) == 0; + pthread_attr_destroy(&attr); + if (!started) {Result r; r.stage = 0; r.vkResult = -12; return r;} + static const unsigned checkpoints[] = {15, 45, 90}; + unsigned next = 0; + for (unsigned tenths = 0; !__atomic_load_n(&w->done, __ATOMIC_ACQUIRE); ++tenths) { + usleep(100000); + if (next < 3 && tenths == checkpoints[next] * 10) { + char text[64] = {}; + sample(*w, worker, text, sizeof text); + report(text); + ++next; + } + } + pthread_join(worker, nullptr); + Result result = w->result; + delete w; + return result; +} + +// Path of libvulkan_freedreno.dylib: next to the image this code is part of (the plugin binary inside the bundle). +static bool libraryPath(char *path, size_t size) { + Dl_info info = {}; + if (!dladdr(reinterpret_cast(&libraryPath), &info) || !info.dli_fname) return false; + snprintf(path, size, "%s", info.dli_fname); + char *slash = strrchr(path, '/'); + if (!slash) return false; + snprintf(slash + 1, size - size_t(slash + 1 - path), "libvulkan_freedreno.dylib"); + return true; +} + +} // namespace turnip +#pragma clang diagnostic pop diff --git a/drivers/NativeMetal/OfflineMetalCommandWire.h b/drivers/NativeMetal/OfflineMetalCommandWire.h new file mode 100644 index 0000000..a0f4c12 --- /dev/null +++ b/drivers/NativeMetal/OfflineMetalCommandWire.h @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: MIT +// Actual asynchronous host submission and status observation. Native integration +// must separately qualify resource visibility, lifetime and callback delivery. +#pragma once +#include "OfflineMetalFenceWire.h" +enum { OM_COMMAND_SUBMIT=0x12e,OM_COMMAND_STATUS=0x12f }; +static_assert(OM_COMMAND_SUBMIT==OM_TEXTURE_HAZARD+1,"Command extension ordering"); +// Reply is OMCommandResult: non-completed states have unavailable (zero) times. +// Successful completion requires actual status 4 and genuine nonzero GPU times. diff --git a/drivers/NativeMetal/OfflineMetalDepthStencilWire.h b/drivers/NativeMetal/OfflineMetalDepthStencilWire.h new file mode 100644 index 0000000..20ec4f4 --- /dev/null +++ b/drivers/NativeMetal/OfflineMetalDepthStencilWire.h @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: MIT +// Explicit offline depth/stencil resources, state and render attachments. +#pragma once +#include "OfflineMetalTextureWire.h" +enum { OM_DEPTH_STENCIL_STATE=0x123,OM_SET_DEPTH_STENCIL=0x124, + OM_STENCIL_REFERENCE=0x125,OM_PIPELINE_ATTACHMENTS=0x126,OM_RENDER_ATTACHMENTS=0x127 }; +static_assert(OM_DEPTH_STENCIL_STATE==OM_COPY_TEXTURE+1,"Depth/stencil extension follows texture copy"); +typedef struct {uint64_t enabled,compare,failure,depthFailure,pass,readMask,writeMask;} OMStencilFace; +typedef struct {uint64_t compare,write;OMStencilFace front,back;} OMDepthStencil; +typedef struct {uint64_t encoder,state;} OMSetDepthStencil; +typedef struct {uint64_t encoder,front,back;} OMStencilReference; +typedef struct {OMBlendPipeline colors;uint64_t depthFormat,stencilFormat;} OMAttachmentPipeline; +typedef struct {uint64_t texture,load,store;double clear;} OMDepthAttachment; +typedef struct {uint64_t texture,load,store,clear;} OMStencilAttachment; +typedef struct {OMRender colors;OMDepthAttachment depth;OMStencilAttachment stencil;} OMAttachmentRender; +static_assert(sizeof(OMDepthStencil)==128 && sizeof(OMAttachmentRender)==520,"Bounded attachment/state ABI"); diff --git a/drivers/NativeMetal/OfflineMetalEventWire.h b/drivers/NativeMetal/OfflineMetalEventWire.h new file mode 100644 index 0000000..3f2d277 --- /dev/null +++ b/drivers/NativeMetal/OfflineMetalEventWire.h @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: MIT +// Actual GPU timeline events for cross-queue dependencies; CPU bookkeeping +// never substitutes for executing a GPU signal/wait or reports a signaled value. +#pragma once +#include "OfflineMetalCommandWire.h" +enum { OM_EVENT=0x130,OM_EVENT_SIGNAL,OM_EVENT_WAIT }; +static_assert(OM_EVENT==OM_COMMAND_STATUS+1,"Event extension ordering"); +typedef struct {uint64_t command,event,value;} OMEventDependency; +static_assert(sizeof(OMEventDependency)==24,"Bounded real GPU event dependency"); diff --git a/drivers/NativeMetal/OfflineMetalFenceWire.h b/drivers/NativeMetal/OfflineMetalFenceWire.h new file mode 100644 index 0000000..a533fcd --- /dev/null +++ b/drivers/NativeMetal/OfflineMetalFenceWire.h @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: MIT +// Explicit offline actual GPU fence dependencies; no emulated completion. +#pragma once +#include "OfflineMetalDepthStencilWire.h" +enum { OM_FENCE=0x128,OM_RENDER_FENCE_UPDATE,OM_RENDER_FENCE_WAIT, + OM_BLIT_FENCE_UPDATE,OM_BLIT_FENCE_WAIT,OM_TEXTURE_HAZARD }; +static_assert(OM_FENCE==OM_RENDER_ATTACHMENTS+1,"Fence extension ordering"); +typedef struct {uint64_t encoder,fence,stages;} OMFenceDependency; +typedef struct {OMTexture texture;uint64_t hazard;} OMHazardTexture; +static_assert(sizeof(OMHazardTexture)==96,"Bounded actual texture hazard mode"); +static_assert(sizeof(OMFenceDependency)==24,"Bounded real fence dependency"); diff --git a/drivers/NativeMetal/OfflineMetalFrontend.mm b/drivers/NativeMetal/OfflineMetalFrontend.mm new file mode 100644 index 0000000..55fab6b --- /dev/null +++ b/drivers/NativeMetal/OfflineMetalFrontend.mm @@ -0,0 +1,1001 @@ +// SPDX-License-Identifier: MIT +// Explicit offline native Metal API frontend. Limited supported operations; +// never installed or advertised as a complete system/hardware GPU driver. +#import +#import +#import +#import +#import +#import +#include +#include +#include "OfflineMetalWire.h" +#include "OfflineMetalFunctionWire.h" +#include "OfflineMetalQueueWire.h" +#include "OfflineMetalTextureWire.h" +#include "OfflineMetalDepthStencilWire.h" +#include "OfflineMetalVertexWire.h" +#include "OfflineMetalFenceWire.h" +#include "OfflineMetalSharedEventWire.h" +#include +#include +#include +#include +using IOMethod=int (*)(unsigned,unsigned,const uint64_t *,unsigned,const void *,size_t,uint64_t *,unsigned *,void *,size_t *); +static IOMethod ioMethod; +static unsigned connection; +static uint32_t sequence; +static NSRecursiveLock *transaction; +static NSDictionary *deviceInfo; +static id remoteDevice; +static std::atomic live{false},asyncEnabled{false},closing{false}; +static NSLock *pendingLock; +static NSMutableSet *pendingCommands; +static NSMutableSet *pendingNotifications; +static NSException *notificationFailure; +static uint64_t notificationsRegistered,notificationsDelivered,notificationsCancelled,notificationsFailed; +static void fail(NSString *reason) { @throw [NSException exceptionWithName:@"OfflineMetalTransport" reason:reason userInfo:nil]; } +static void receiveBytes(void *bytes,size_t size) { + unsigned char *p=(unsigned char *)bytes; + while(size){size_t count=MIN(size,(size_t)4096);int status=ioMethod(connection,1,nullptr,0,nullptr,0,nullptr,nullptr,p,&count); + if(status || !count || count>size)fail(@"Actual console receive failed");p+=count;size-=count;} +} +static NSData *call(uint16_t operation,const void *input,size_t size,OMResult *result) { + [transaction lock]; + @try { + if(!live || size>4080)fail(@"Invalid live transport request"); + OMWireHeader header={OM_WIRE_MAGIC,OFFLINE_METAL_VERSION,operation,(uint32_t)size,sequence+1}; + unsigned char bytes[4096];memcpy(bytes,&header,sizeof(header));if(size)memcpy(bytes+sizeof(header),input,size); + int sendStatus=ioMethod(connection,0,nullptr,0,bytes,sizeof(header)+size,nullptr,nullptr,nullptr,nullptr); + if(sendStatus)fail([NSString stringWithFormat:@"Actual transport send failed: %08x",(unsigned)sendStatus]); + sequence++; + OMWireHeader reply={};receiveBytes(&reply,sizeof(reply)); + if(reply.magic!=OM_WIRE_MAGIC || reply.version!=OFFLINE_METAL_VERSION || reply.operation!=(operation|OM_WIRE_RESPONSE) + || reply.request!=sequence || reply.bytesOFFLINE_METAL_MAX_REPLY)fail(@"Invalid actual Metal reply"); + receiveBytes(result,sizeof(*result));if(result->reserved)fail(@"Unknown reply flags"); + NSMutableData *body=[NSMutableData dataWithLength:reply.bytes-sizeof(*result)];if(body.length)receiveBytes(body.mutableBytes,body.length); + return body; + } @finally {[transaction unlock];} +} +static NSData *checked(uint16_t operation,const void *input,size_t size,OMResult *result) { + NSData *body=call(operation,input,size,result);if(result->error)fail([[NSString alloc] initWithData:body encoding:NSUTF8StringEncoding]?:@"Unsupported or failed actual Metal operation");return body; +} +static NSError *operationError(OMResult result,NSData *body) { + NSString *why=[[NSString alloc] initWithData:body encoding:NSUTF8StringEncoding]?:@"Actual offline backend refused this operation"; + return [NSError errorWithDomain:@"org.blu-sharky.offline-metal" code:result.error userInfo:@{NSLocalizedDescriptionKey:why}]; +} +@interface OMObject : NSObject +@property(readonly) id device; +@property(nonatomic) uint64_t handle; +@property(atomic,copy) NSString *label; +@end +@implementation OMObject +- (id)device {return remoteDevice;} +- (void)dealloc { + // Actual host Metal objects retain resources required by their recorded + // command buffers. Drop exactly this proxy's independent table reference. + // Session teardown releases any remaining references after real commits. + [transaction lock]; + @try {if(live && !closing && _handle){OMHandle request={_handle};OMResult result={};checked(OM_RELEASE,&request,sizeof(request),&result);}} + @finally {[transaction unlock];} +} +@end +@interface OMFenceObject : OMObject +@end +@implementation OMFenceObject +@end +@interface OMEventObject : OMObject +@end +@implementation OMEventObject +@end +// These selectors are read by the unchanged target handle constructor. The +// eventPort is a real Mach capability created by the native kernel broker. +@interface MTLSharedEventHandle (OMOriginalSharedHandle) +- (instancetype)initWithSharedEvent:(id)event; +- (mach_port_t)eventPort; +- (uint64_t)labelTraceID; +@end +static NSDictionary *sharedMetadata(NSData *body) { + NSDictionary *info=[NSJSONSerialization JSONObjectWithData:body options:0 error:nil]; + if(![info isKindOfClass:[NSDictionary class]] || ![info[@"device_is_nil"] isKindOfClass:[NSNumber class]] || + ![info[@"device_is_owner"] isKindOfClass:[NSNumber class]] || + ([info[@"device_is_nil"] boolValue]==[info[@"device_is_owner"] boolValue]) || + ![info[@"signaled_value"] isKindOfClass:[NSNumber class]] || + (info[@"label"]!=[NSNull null] && ![info[@"label"] isKindOfClass:[NSString class]]))fail(@"Invalid actual shared-event metadata"); + return info; +} +enum {OMNotificationPending,OMNotificationQueued,OMNotificationRunning,OMNotificationFinished,OMNotificationCancelled}; +@interface OMSharedNotificationState : NSObject +@property(nonatomic) uint64_t handle,value; +@property(nonatomic,strong) id event; +@property(nonatomic,strong) MTLSharedEventListener *listener; +@property(nonatomic,strong) dispatch_queue_t queue; +@property(nonatomic,strong) dispatch_group_t callbackGroup; +@property(nonatomic,copy) MTLSharedEventNotificationBlock block; +// All state transitions and real callback thread identities use pendingLock. +@property(nonatomic) unsigned phase; +@property(nonatomic) pthread_t callbackThread; +- (void)start; +@end +@implementation OMSharedNotificationState +- (void)start { + dispatch_async(dispatch_get_global_queue(QOS_CLASS_USER_INITIATED,0),^{@autoreleasepool { + @try { + struct timespec start={},now={};if(clock_gettime(CLOCK_MONOTONIC,&start))fail(@"Actual listener clock unavailable"); + for(;;) { + [pendingLock lock];BOOL cancelled=self.phase==OMNotificationCancelled;[pendingLock unlock];if(cancelled)return; + OMSharedNotification reply={}; + [transaction lock];@try { + if(!live || closing)return; + OMHandle input={self.handle};OMResult result={};NSData *body=checked(OM_SHARED_NOTIFICATION_POLL,&input,sizeof(input),&result); + if(body.length!=sizeof(reply))fail(@"Invalid actual listener reply");memcpy(&reply,body.bytes,sizeof(reply)); + } @finally {[transaction unlock];} + if(reply.ready) { + if(reply.ready!=1 || reply.event!=[(OMObject *)self.event handle] || reply.value=60)fail(@"Bounded actual listener observation timed out");usleep(1000); + } + } @catch(NSException *exception) { + [pendingLock lock];if(self.phase!=OMNotificationCancelled){if(!notificationFailure)notificationFailure=exception;notificationsFailed++;self.phase=OMNotificationFinished;[pendingNotifications removeObject:self];}[pendingLock unlock]; + } + }}); +} +@end +@interface OMSharedEventObject : OMEventObject { + mach_port_t _exportPort; + uint64_t _traceID; +} +@property(nonatomic) BOOL deviceIsNil; +- (mach_port_t)eventPort; +- (uint64_t)labelTraceID; +@end +@implementation OMSharedEventObject +- (id)device {return self.deviceIsNil?nil:remoteDevice;} +- (void)dealloc {if(_exportPort)mach_port_deallocate(mach_task_self(),_exportPort);} +- (uint64_t)signaledValue { + OMHandle input={self.handle};OMResult result={};NSData *body=checked(OM_SHARED_VALUE_GET,&input,sizeof(input),&result); + if(body.length!=8)fail(@"Invalid actual event counter");uint64_t value=0;memcpy(&value,body.bytes,8);return value; +} +- (void)setSignaledValue:(uint64_t)value { + OMSharedEventValue input={self.handle,value};OMResult result={};NSData *body=checked(OM_SHARED_VALUE_SET,&input,sizeof(input),&result); + if(body.length!=8)fail(@"Missing actual event counter write result"); +} +- (void)setLabel:(NSString *)label { + if(label && ![label isKindOfClass:[NSString class]])fail(@"Invalid shared-event label"); + NSData *text=[label dataUsingEncoding:NSUTF8StringEncoding];if(text.length>512)fail(@"Excessive shared-event label"); + OMSharedLabel input={self.handle,(uint64_t)(label!=nil),text.length};NSMutableData *data=[NSMutableData dataWithBytes:&input length:sizeof(input)];if(text)[data appendData:text]; + [transaction lock];@try { + OMResult result={};NSData *body=checked(OM_SHARED_LABEL_SET,data.bytes,data.length,&result);NSDictionary *info=[NSJSONSerialization JSONObjectWithData:body options:0 error:nil]; + id actual=info[@"label"];if(actual!=[NSNull null] && ![actual isKindOfClass:[NSString class]])fail(@"Missing actual label result"); + [super setLabel:actual==[NSNull null]?nil:actual]; + if(_exportPort){mach_port_deallocate(mach_task_self(),_exportPort);_exportPort=0;_traceID=0;} + } @finally {[transaction unlock];} +} +- (mach_port_t)eventPort { + [transaction lock];@try { + if(!live || closing)fail(@"Shared export on a closing context");if(_exportPort)return _exportPort; + uint64_t output=0,input=self.handle;unsigned count=1;unsigned char bytes[4096];size_t size=sizeof(bytes); + int status=ioMethod(connection,2,&input,1,nullptr,0,&output,&count,bytes,&size); + if(status || count!=1 || !output || output>UINT32_MAX || output==MACH_PORT_DEAD)fail(@"Native shared-event capability export refused"); + NSDictionary *info=[NSJSONSerialization JSONObjectWithData:[NSData dataWithBytes:bytes length:size] options:0 error:nil]; + if(![info[@"trace_id_supported"] boolValue] || ![info[@"trace_id"] isKindOfClass:[NSNumber class]]){mach_port_deallocate(mach_task_self(),(mach_port_t)output);fail(@"Unqualified actual shared-event trace ABI");} + _exportPort=(mach_port_t)output;_traceID=[info[@"trace_id"] unsignedLongLongValue];return _exportPort; + } @finally {[transaction unlock];} +} +- (uint64_t)labelTraceID {(void)self.eventPort;return _traceID;} +- (MTLSharedEventHandle *)newSharedEventHandle { + Class cls=NSClassFromString(@"MTLSharedEventHandle");if(!cls || class_getInstanceSize(cls)!=16)fail(@"Wrong original shared-handle ABI"); + return [[cls alloc] initWithSharedEvent:self]; +} +- (void)notifyListener:(MTLSharedEventListener *)listener atValue:(uint64_t)value block:(MTLSharedEventNotificationBlock)block { + Class cls=NSClassFromString(@"MTLSharedEventListenerInternal");Method getter=class_getInstanceMethod(cls,@selector(dispatchQueue)); + if(!listener || !block || object_getClass(listener)!=cls || class_getInstanceSize(cls)!=24 || !getter || strcmp(method_getTypeEncoding(getter),"@16@0:8"))fail(@"Unqualified original shared listener ABI"); + dispatch_queue_t queue=listener.dispatchQueue;if(!queue)fail(@"Missing actual original listener queue"); + [transaction lock];@try { + if(!live || closing)fail(@"Listener registration on a closing context");OMSharedEventValue input={self.handle,value};OMResult result={};checked(OM_SHARED_NOTIFY,&input,sizeof(input),&result);if(!result.handle)fail(@"Missing actual listener registration"); + OMSharedNotificationState *state=[OMSharedNotificationState new];state.handle=result.handle;state.value=value;state.event=self;state.listener=listener;state.queue=queue;state.block=block;state.callbackGroup=dispatch_group_create(); + [pendingLock lock];if(closing){[pendingLock unlock];fail(@"Listener registration raced context finish");}[pendingNotifications addObject:state];notificationsRegistered++;[pendingLock unlock];[state start]; + } @finally {[transaction unlock];} +} +- (BOOL)waitUntilSignaledValue:(uint64_t)value timeoutMS:(uint64_t)milliseconds { + struct timespec start={},now={};if(clock_gettime(CLOCK_MONOTONIC,&start))fail(@"Actual monotonic clock unavailable"); + for(;;){if(self.signaledValue>=value)return YES;if(clock_gettime(CLOCK_MONOTONIC,&now))fail(@"Actual monotonic clock unavailable"); + uint64_t elapsed=(uint64_t)(now.tv_sec-start.tv_sec)*1000; + if(now.tv_nsec>=start.tv_nsec)elapsed+=(now.tv_nsec-start.tv_nsec)/1000000; + else elapsed-=(start.tv_nsec-now.tv_nsec+999999)/1000000; + if(elapsed>=milliseconds)return NO;usleep(1000);} +} +@end +@interface OMFunctionObject : OMObject +@property(nonatomic,strong) NSString *name; +@property(nonatomic) MTLFunctionType functionType; +@end +@implementation OMFunctionObject +@end +@interface MTLFunctionConstantValues (OMOriginalConstantAccess) +- (NSArray *)newIndexedConstantArray; +- (NSArray *)newNamedConstantArray; +@end +@protocol OMOriginalConstantValue +- (NSUInteger)dataType; +- (void *)data; +- (NSUInteger)index; +- (NSString *)name; +@end +static bool methodABI(id object,SEL selector,const char *encoding) { + Method method=class_getInstanceMethod(object_getClass(object),selector); + return method && !strcmp(method_getTypeEncoding(method),encoding); +} +static NSError *constantError(NSString *reason) { + return [NSError errorWithDomain:@"org.blu-sharky.offline-metal" code:-4 userInfo:@{NSLocalizedDescriptionKey:reason}]; +} +@interface OMLibraryObject : OMObject +@property(nonatomic,strong) NSArray *functionNames; +- (id)newFunctionWithName:(NSString *)name; +- (id)newFunctionWithName:(NSString *)name constantValues:(MTLFunctionConstantValues *)values error:(NSError **)error; +@end +@implementation OMLibraryObject +- (id)newFunctionWithName:(NSString *)name { + if(!name || [name lengthOfBytesUsingEncoding:NSUTF8StringEncoding]>=128)return nil; + OMFunction request={};request.library=self.handle;strcpy(request.name,name.UTF8String); + OMResult result={};NSData *body=call(OM_FUNCTION,&request,sizeof(request),&result);if(result.error)return nil; + NSDictionary *info=[NSJSONSerialization JSONObjectWithData:body options:0 error:nil];if(![info isKindOfClass:[NSDictionary class]])fail(@"Actual function metadata missing"); + OMFunctionObject *object=[OMFunctionObject new];object.handle=result.handle;object.name=info[@"name"];object.functionType=(MTLFunctionType)[info[@"type"] unsignedIntegerValue]; + return (id)object; +} +- (id)newFunctionWithName:(NSString *)name constantValues:(MTLFunctionConstantValues *)values error:(NSError **)error { + auto refuse=[&](NSString *why)->id {if(error)*error=constantError(why);return nil;}; + if(!name.length || [name lengthOfBytesUsingEncoding:NSUTF8StringEncoding]>=128 || + object_getClass(values)!=NSClassFromString(@"MTLFunctionConstantValuesInternal") || + !methodABI(values,@selector(newIndexedConstantArray),"@16@0:8") || !methodABI(values,@selector(newNamedConstantArray),"@16@0:8")) + return refuse(@"Unqualified actual function-constant object or name"); + // Read through the original target's real accessors. Never replace setters, + // interpret opaque storage, or substitute unspecialized shader functions. + MTLFunctionConstantValues *snapshot=[values copy]; + NSArray *indexed=[snapshot newIndexedConstantArray],*named=[snapshot newNamedConstantArray]; + if(![indexed isKindOfClass:[NSArray class]] || ![named isKindOfClass:[NSArray class]] || indexed.count+named.count>OFFLINE_METAL_MAX_CONSTANTS) + return refuse(@"Unqualified or excessive original constants"); + OMSpecializedFunction request={};request.function.library=self.handle;strcpy(request.function.name,name.UTF8String);request.count=indexed.count+named.count; + NSMutableData *packet=[NSMutableData dataWithBytes:&request length:sizeof(request)]; + for(unsigned group=0;group<2;group++)for(id value in (group?named:indexed)) { + if(object_getClass(value)!=NSClassFromString(group?@"MTLNamedConstantValue":@"MTLIndexedConstantValue") || + !methodABI(value,@selector(dataType),"Q16@0:8") || !methodABI(value,@selector(data),"^v16@0:8") || + !methodABI(value,group?@selector(name):@selector(index),group?"@16@0:8":"Q16@0:8")) + return refuse(@"Unqualified actual constant descriptor ABI"); + id descriptor=value; + OMFunctionConstant row={};row.type=[descriptor dataType];row.bytes=OMFunctionConstantBytes(row.type);row.named=group; + const void *data=[descriptor data];if(!row.bytes || !data)return refuse(@"Unsupported original constant type or missing actual value"); + if(group) { + NSString *key=[descriptor name];if(![key isKindOfClass:[NSString class]] || !key.length || [key lengthOfBytesUsingEncoding:NSUTF8StringEncoding]>=128){ + return refuse(@"Invalid actual named constant");}strcpy(row.name,key.UTF8String); + } else {row.index=[descriptor index];if(row.index>65535)return refuse(@"Original constant index exceeds qualified range");} + memcpy(row.data,data,row.bytes);[packet appendBytes:&row length:sizeof(row)]; + } + OMResult result={};NSData *body=call(OM_FUNCTION,packet.bytes,packet.length,&result); + if(result.error){if(error)*error=operationError(result,body);return nil;} + NSDictionary *info=[NSJSONSerialization JSONObjectWithData:body options:0 error:nil];if(![info isKindOfClass:[NSDictionary class]])fail(@"Actual specialized function metadata missing"); + OMFunctionObject *object=[OMFunctionObject new];object.handle=result.handle;object.name=info[@"name"];object.functionType=(MTLFunctionType)[info[@"type"] unsignedIntegerValue]; + if(error)*error=nil;return (id)object; +} +@end +@interface OMTextureObject : OMObject +@property(nonatomic) OMTexture descriptor; +@property(nonatomic) MTLHazardTrackingMode actualHazardMode; +@property(nonatomic) IOSurfaceRef surface; +@property(nonatomic) uint32_t surfaceSeed; +@property(nonatomic) BOOL surfaceSeedValid; +- (void)uploadSurface; +- (void)downloadSurface; +- (void)getBytes:(void *)pixels bytesPerRow:(NSUInteger)stride fromRegion:(MTLRegion)region mipmapLevel:(NSUInteger)level; +@end +@implementation OMTextureObject +- (void)dealloc {if(_surface)CFRelease(_surface);} +- (IOSurfaceRef)iosurface {return self.surface;} +- (NSUInteger)iosurfacePlane {return 0;} +- (void)uploadSurface { + @synchronized(self) { + if(!self.surface)return;uint32_t seed=0; + if(IOSurfaceLock(self.surface,kIOSurfaceLockReadOnly,&seed))fail(@"Actual IOSurface read lock failed"); + @try { + if(self.surfaceSeedValid && seed==self.surfaceSeed)return; + const unsigned char *base=(const unsigned char *)IOSurfaceGetBaseAddress(self.surface);size_t row=IOSurfaceGetBytesPerRow(self.surface); + if(!base)fail(@"Missing actual IOSurface mapping"); + for(uint64_t y=0;ySIZE_MAX/4 + || strideSIZE_MAX/region.size.height)fail(@"Unsupported texture readback layout"); + OMReadTexture request={self.handle,region.origin.x,region.origin.y,region.size.width,region.size.height,level};OMResult result={}; + NSData *body=checked(OM_READ_TEXTURE,&request,sizeof(request),&result); + if(region.size.width>SIZE_MAX/4/region.size.height || body.length!=region.size.width*region.size.height*4)fail(@"Actual pixel extent differs"); + for(NSUInteger row=0;rowself.data.length || range.length>self.data.length-range.location)fail(@"Invalid native buffer modification range");} +- (void)upload { + for(NSUInteger offset=0;offset +- (id)texture; +@end +@interface OMCommandObject : OMObject +@property(nonatomic,strong) id commandQueue; +@property(atomic) OMCommandResult actualResult; +@property(nonatomic) BOOL useAsync; +@property(atomic) BOOL submissionStarted; +@property(atomic) BOOL resourcesVisible; +@property(atomic) pthread_t callbackThread; +@property(atomic,strong) NSException *asynchronousFailure; +@property(atomic,strong) NSException *deliveryFailure; +- (void)commitAsync; +@property(nonatomic) MTLCommandBufferStatus actualEnqueueStatus; +@property(nonatomic,strong) NSMutableArray *completedHandlers; +@property(nonatomic,strong) NSMutableArray *scheduledHandlers; +@property(atomic,strong) dispatch_group_t callbackGroup; +@property(atomic,strong) dispatch_group_t scheduledGroup; +@property(atomic,strong) dispatch_group_t visibilityGroup; +@property(atomic) BOOL scheduledObserved; +@property(atomic) MTLCommandBufferStatus actualScheduledStatus; +@property(nonatomic,strong) NSMutableArray *surfaceTargets; +@property(nonatomic,strong) NSMutableArray *buffers; +@property(nonatomic,strong) NSMutableArray *textures; +@property(nonatomic,strong) NSMutableArray> *presentations; +@property(nonatomic,strong) NSMutableArray *fences; +@property(nonatomic,strong) NSMutableArray *events; +- (void)trackFence:(id)fence; +- (void)trackBuffer:(OMBufferObject *)buffer; +- (void)trackTexture:(OMTextureObject *)texture; +- (void)trackSurfaceTarget:(OMTextureObject *)texture; +- (id)renderCommandEncoderWithDescriptor:(MTLRenderPassDescriptor *)descriptor; +- (id)blitCommandEncoder; +- (void)commit; +- (void)waitUntilCompleted; +@end +@interface OMEncoderObject : OMObject +@property(nonatomic,strong) OMCommandObject *command; +@property(nonatomic,strong) NSMutableDictionary *vertexBuffers; +@property(nonatomic,strong) NSMutableDictionary *fragmentBuffers; +@end +@implementation OMEncoderObject +- (void)updateFence:(id)fence afterStages:(MTLRenderStages)stages { + if(![(id)fence isKindOfClass:[OMFenceObject class]] || fence.device!=remoteDevice)fail(@"Foreign render fence"); + OMFenceDependency request={self.handle,[(OMObject *)fence handle],(uint64_t)stages};OMResult result={};checked(OM_RENDER_FENCE_UPDATE,&request,sizeof(request),&result);[self.command trackFence:fence]; +} +- (void)waitForFence:(id)fence beforeStages:(MTLRenderStages)stages { + if(![(id)fence isKindOfClass:[OMFenceObject class]] || fence.device!=remoteDevice)fail(@"Foreign render fence"); + OMFenceDependency request={self.handle,[(OMObject *)fence handle],(uint64_t)stages};OMResult result={};checked(OM_RENDER_FENCE_WAIT,&request,sizeof(request),&result);[self.command trackFence:fence]; +} +- (void)setDepthStencilState:(id)state { + if(state && ![(id)state isKindOfClass:[OMDepthStateObject class]])fail(@"Foreign depth/stencil state"); + OMSetDepthStencil request={self.handle,state?[(OMObject *)state handle]:0};OMResult result={};checked(OM_SET_DEPTH_STENCIL,&request,sizeof(request),&result); +} +- (void)setStencilFrontReferenceValue:(uint32_t)front backReferenceValue:(uint32_t)back { + OMStencilReference request={self.handle,front,back};OMResult result={};checked(OM_STENCIL_REFERENCE,&request,sizeof(request),&result); +} +- (void)setStencilReferenceValue:(uint32_t)value {[self setStencilFrontReferenceValue:value backReferenceValue:value];} +- (void)setRenderPipelineState:(id)pipeline { + if(![(id)pipeline isKindOfClass:[OMPipelineObject class]])fail(@"Foreign pipeline object"); + OMSetPipeline request={self.handle,[(OMObject *)pipeline handle]};OMResult result={};checked(OM_SET_PIPELINE,&request,sizeof(request),&result); +} +- (void)setCullMode:(MTLCullMode)mode {OMSetCull request={self.handle,(uint64_t)mode};OMResult result={};checked(OM_SET_CULL,&request,sizeof(request),&result);} +- (void)setViewport:(MTLViewport)viewport { + OMViewport request={self.handle,viewport.originX,viewport.originY,viewport.width,viewport.height,viewport.znear,viewport.zfar};OMResult result={};checked(OM_VIEWPORT,&request,sizeof(request),&result); +} +- (void)setScissorRect:(MTLScissorRect)rect {OMScissor request={self.handle,rect.x,rect.y,rect.width,rect.height};OMResult result={};checked(OM_SCISSOR,&request,sizeof(request),&result);} +- (void)setBlendColorRed:(float)red green:(float)green blue:(float)blue alpha:(float)alpha { + OMBlendColor request={self.handle,red,green,blue,alpha};OMResult result={};checked(OM_BLEND_COLOR,&request,sizeof(request),&result); +} +- (void)setTexture:(id)texture index:(NSUInteger)index vertex:(BOOL)vertex { + if(texture && ![(id)texture isKindOfClass:[OMTextureObject class]])fail(@"Foreign texture object"); + OMSetTexture request={self.handle,texture?[(OMObject *)texture handle]:0,index};OMResult result={}; + checked(vertex?OM_VERTEX_TEXTURE:OM_FRAGMENT_TEXTURE,&request,sizeof(request),&result); + if(texture)[self.command trackTexture:(OMTextureObject *)texture]; +} +- (void)setVertexTexture:(id)texture atIndex:(NSUInteger)index {[self setTexture:texture index:index vertex:YES];} +- (void)setFragmentTexture:(id)texture atIndex:(NSUInteger)index {[self setTexture:texture index:index vertex:NO];} +- (void)drawPrimitives:(MTLPrimitiveType)type vertexStart:(NSUInteger)start vertexCount:(NSUInteger)count { + OMDraw request={self.handle,(uint64_t)type,start,count,1};OMResult result={};checked(OM_DRAW,&request,sizeof(request),&result); +} +- (void)setBuffer:(id)buffer offset:(NSUInteger)offset index:(NSUInteger)index vertex:(BOOL)vertex { + if(index>=31 || (buffer && ![(id)buffer isKindOfClass:[OMBufferObject class]]))fail(@"Foreign buffer object or invalid binding index"); + OMSetBuffer request={self.handle,buffer?[(OMObject *)buffer handle]:0,offset,index};OMResult result={}; + checked(vertex?OM_VERTEX_BUFFER:OM_FRAGMENT_BUFFER,&request,sizeof(request),&result); + if(vertex && !self.vertexBuffers)self.vertexBuffers=[NSMutableDictionary dictionary]; + if(!vertex && !self.fragmentBuffers)self.fragmentBuffers=[NSMutableDictionary dictionary]; + NSMutableDictionary *bindings=vertex?self.vertexBuffers:self.fragmentBuffers; + if(buffer)bindings[@(index)]=buffer;else [bindings removeObjectForKey:@(index)]; + if(buffer)[self.command trackBuffer:(OMBufferObject *)buffer]; +} +- (void)setVertexBuffer:(id)buffer offset:(NSUInteger)offset atIndex:(NSUInteger)index {[self setBuffer:buffer offset:offset index:index vertex:YES];} +- (void)setFragmentBuffer:(id)buffer offset:(NSUInteger)offset atIndex:(NSUInteger)index {[self setBuffer:buffer offset:offset index:index vertex:NO];} +- (void)setVertexBufferOffset:(NSUInteger)offset atIndex:(NSUInteger)index { + OMBufferObject *buffer=self.vertexBuffers[@(index)];if(!buffer)fail(@"No actual vertex buffer bound at this index"); + [self setVertexBuffer:(id)buffer offset:offset atIndex:index]; +} +- (void)setFragmentBufferOffset:(NSUInteger)offset atIndex:(NSUInteger)index { + OMBufferObject *buffer=self.fragmentBuffers[@(index)];if(!buffer)fail(@"No actual fragment buffer bound at this index"); + [self setFragmentBuffer:(id)buffer offset:offset atIndex:index]; +} +- (void)setVertexBytes:(const void *)bytes length:(NSUInteger)length atIndex:(NSUInteger)index { + id buffer=[remoteDevice newBufferWithBytes:bytes length:length options:MTLResourceStorageModeShared]; + if(!buffer)fail(@"Invalid vertex-byte allocation");[self setVertexBuffer:buffer offset:0 atIndex:index]; +} +- (void)setFragmentBytes:(const void *)bytes length:(NSUInteger)length atIndex:(NSUInteger)index { + id buffer=[remoteDevice newBufferWithBytes:bytes length:length options:MTLResourceStorageModeShared]; + if(!buffer)fail(@"Invalid fragment-byte allocation");[self setFragmentBuffer:buffer offset:0 atIndex:index]; +} +- (void)drawIndexedPrimitives:(MTLPrimitiveType)type indexCount:(NSUInteger)count indexType:(MTLIndexType)indexType indexBuffer:(id)buffer indexBufferOffset:(NSUInteger)offset instanceCount:(NSUInteger)instances baseVertex:(NSInteger)baseVertex baseInstance:(NSUInteger)baseInstance { + if(![(id)buffer isKindOfClass:[OMBufferObject class]])fail(@"Foreign index buffer"); + OMDrawIndexed request={self.handle,(uint64_t)type,count,(uint64_t)indexType,[(OMObject *)buffer handle],offset,instances,baseVertex,baseInstance};OMResult result={}; + checked(OM_DRAW_INDEXED,&request,sizeof(request),&result);[self.command trackBuffer:(OMBufferObject *)buffer]; +} +- (void)drawIndexedPrimitives:(MTLPrimitiveType)type indexCount:(NSUInteger)count indexType:(MTLIndexType)indexType indexBuffer:(id)buffer indexBufferOffset:(NSUInteger)offset { + [self drawIndexedPrimitives:type indexCount:count indexType:indexType indexBuffer:buffer indexBufferOffset:offset instanceCount:1 baseVertex:0 baseInstance:0]; +} +- (void)drawIndexedPrimitives:(MTLPrimitiveType)type indexCount:(NSUInteger)count indexType:(MTLIndexType)indexType indexBuffer:(id)buffer indexBufferOffset:(NSUInteger)offset instanceCount:(NSUInteger)instances { + [self drawIndexedPrimitives:type indexCount:count indexType:indexType indexBuffer:buffer indexBufferOffset:offset instanceCount:instances baseVertex:0 baseInstance:0]; +} +- (void)endEncoding {OMHandle request={self.handle};OMResult result={};checked(OM_END,&request,sizeof(request),&result);} +@end +@interface OMBlitEncoderObject : OMEncoderObject +@end +@implementation OMBlitEncoderObject +- (void)updateFence:(id)fence { + if(![(id)fence isKindOfClass:[OMFenceObject class]] || fence.device!=remoteDevice)fail(@"Foreign blit fence"); + OMFenceDependency request={self.handle,[(OMObject *)fence handle],0};OMResult result={};checked(OM_BLIT_FENCE_UPDATE,&request,sizeof(request),&result);[self.command trackFence:fence]; +} +- (void)waitForFence:(id)fence { + if(![(id)fence isKindOfClass:[OMFenceObject class]] || fence.device!=remoteDevice)fail(@"Foreign blit fence"); + OMFenceDependency request={self.handle,[(OMObject *)fence handle],0};OMResult result={};checked(OM_BLIT_FENCE_WAIT,&request,sizeof(request),&result);[self.command trackFence:fence]; +} +- (void)copyFromTexture:(id)source sourceSlice:(NSUInteger)sourceSlice sourceLevel:(NSUInteger)sourceLevel sourceOrigin:(MTLOrigin)sourceOrigin sourceSize:(MTLSize)sourceSize + toTexture:(id)destination destinationSlice:(NSUInteger)destinationSlice destinationLevel:(NSUInteger)destinationLevel destinationOrigin:(MTLOrigin)destinationOrigin { + if(![(id)source isKindOfClass:[OMTextureObject class]] || ![(id)destination isKindOfClass:[OMTextureObject class]])fail(@"Foreign blit texture"); + OMCopyTexture request={self.handle,[(OMObject *)source handle],[(OMObject *)destination handle],sourceSlice,sourceLevel, + {sourceOrigin.x,sourceOrigin.y,sourceOrigin.z},{sourceSize.width,sourceSize.height,sourceSize.depth},destinationSlice,destinationLevel, + {destinationOrigin.x,destinationOrigin.y,destinationOrigin.z}}; + OMResult result={};checked(OM_COPY_TEXTURE,&request,sizeof(request),&result); + [self.command trackTexture:(OMTextureObject *)source];[self.command trackSurfaceTarget:(OMTextureObject *)destination]; +} +- (void)copyFromTexture:(id)source toTexture:(id)destination { + [self copyFromTexture:source sourceSlice:0 sourceLevel:0 sourceOrigin:MTLOriginMake(0,0,0) sourceSize:MTLSizeMake(source.width,source.height,source.depth) + toTexture:destination destinationSlice:0 destinationLevel:0 destinationOrigin:MTLOriginMake(0,0,0)]; +} +- (void)copyFromBuffer:(id)source sourceOffset:(NSUInteger)sourceOffset toBuffer:(id)destination destinationOffset:(NSUInteger)destinationOffset size:(NSUInteger)size { + if(![(id)source isKindOfClass:[OMBufferObject class]] || ![(id)destination isKindOfClass:[OMBufferObject class]])fail(@"Foreign blit buffer"); + OMCopyBuffer request={self.handle,[(OMObject *)source handle],sourceOffset,[(OMObject *)destination handle],destinationOffset,size};OMResult result={}; + checked(OM_COPY_BUFFER,&request,sizeof(request),&result); + [self.command trackBuffer:(OMBufferObject *)source];[self.command trackBuffer:(OMBufferObject *)destination]; +} +@end +@implementation OMCommandObject +- (void)encodeSignalEvent:(id)event value:(uint64_t)value { + if(self.submissionStarted || self.actualResult.status || ![(id)event isKindOfClass:[OMEventObject class]])fail(@"Unqualified event or submitted command"); + OMEventDependency request={self.handle,[(OMObject *)event handle],value};OMResult result={};checked(OM_EVENT_SIGNAL,&request,sizeof(request),&result); + if(!self.events)self.events=[NSMutableArray array];if([self.events indexOfObjectIdenticalTo:event]==NSNotFound)[self.events addObject:(OMEventObject *)event]; +} +- (void)encodeWaitForEvent:(id)event value:(uint64_t)value { + if(self.submissionStarted || self.actualResult.status || ![(id)event isKindOfClass:[OMEventObject class]])fail(@"Unqualified event or submitted command"); + OMEventDependency request={self.handle,[(OMObject *)event handle],value};OMResult result={};checked(OM_EVENT_WAIT,&request,sizeof(request),&result); + if(!self.events)self.events=[NSMutableArray array];if([self.events indexOfObjectIdenticalTo:event]==NSNotFound)[self.events addObject:(OMEventObject *)event]; +} +- (void)commitAsync { + if(self.submissionStarted || self.actualResult.status)fail(@"Command already submitted"); + dispatch_group_t group=dispatch_group_create();dispatch_group_enter(group);self.callbackGroup=group; + self.scheduledGroup=dispatch_group_create();self.visibilityGroup=dispatch_group_create();dispatch_group_enter(self.scheduledGroup);dispatch_group_enter(self.visibilityGroup); + [pendingLock lock]; + if(closing || !live){[pendingLock unlock];dispatch_group_leave(self.scheduledGroup);dispatch_group_leave(self.visibilityGroup);dispatch_group_leave(group);fail(@"Context is closing");} + self.submissionStarted=YES;[pendingCommands addObject:self];[pendingLock unlock]; + @try { + for(OMBufferObject *buffer in self.buffers)[buffer upload]; + for(OMTextureObject *texture in self.textures)[texture uploadSurface]; + OMHandle request={self.handle};OMResult result={};NSData *body=checked(OM_COMMAND_SUBMIT,&request,sizeof(request),&result); + if(body.length!=sizeof(OMCommandResult))fail(@"Missing actual asynchronous submission");OMCommandResult initial={};memcpy(&initial,body.bytes,sizeof(initial)); + if(initial.statusMTLCommandBufferStatusCompleted || + (initial.status=MTLCommandBufferStatusScheduled){self.actualScheduledStatus=(MTLCommandBufferStatus)initial.status;self.scheduledObserved=YES;dispatch_group_leave(self.scheduledGroup);} + NSArray *scheduled=[self.scheduledHandlers copy],*completed=[self.completedHandlers copy],*presentations=[self.presentations copy]; + self.scheduledHandlers=nil;self.completedHandlers=nil;self.presentations=nil; + dispatch_async(dispatch_get_global_queue(QOS_CLASS_DEFAULT,0),^{@autoreleasepool { + @try { + OMCommandResult value=initial;unsigned polls=0; + while(value.status!=MTLCommandBufferStatusCompleted) { + if(++polls>4096)fail(@"Actual GPU completion observation exceeded its bound"); + OMResult observed={};NSData *reply=checked(OM_COMMAND_STATUS,&request,sizeof(request),&observed); + if(reply.length!=sizeof(value))fail(@"Missing actual GPU status");memcpy(&value,reply.bytes,sizeof(value)); + if(value.statusMTLCommandBufferStatusCompleted || + (value.status=MTLCommandBufferStatusScheduled && !self.scheduledObserved){self.actualScheduledStatus=(MTLCommandBufferStatus)value.status;self.scheduledObserved=YES;dispatch_group_leave(self.scheduledGroup);} + if(value.status!=MTLCommandBufferStatusCompleted){self.actualResult=value;usleep(1000);} + } + if(!value.start_ns || value.end_ns<=value.start_ns)fail(@"Missing actual completed GPU timing"); + for(OMTextureObject *texture in self.surfaceTargets)[texture downloadSurface]; + for(OMBufferObject *buffer in self.buffers)[buffer download]; + self.actualResult=value;self.resourcesVisible=YES; + @try {for(id drawable in presentations)[drawable present];} + @catch(NSException *exception){self.deliveryFailure=exception;} + } @catch(NSException *exception){self.asynchronousFailure=exception;} + if(!self.scheduledObserved){self.scheduledObserved=YES;dispatch_group_leave(self.scheduledGroup);} + dispatch_group_leave(self.visibilityGroup); + // Callbacks are deliberately late: real GPU completion and CPU visibility + // precede success delivery. A transport/GPU failure has a real error and + // never delivers a fabricated successful command or display timestamp. + self.callbackThread=pthread_self(); + @try { + if(self.resourcesVisible)for(MTLCommandBufferHandler handler in scheduled)handler((id)self); + for(MTLCommandBufferHandler handler in completed)handler((id)self); + } @catch(NSException *exception){self.deliveryFailure=exception;} + @finally { + self.callbackThread=nullptr;[pendingLock lock];[pendingCommands removeObject:self];[pendingLock unlock];dispatch_group_leave(group); + } + }}); + } @catch(NSException *exception) { + self.asynchronousFailure=exception;if(!self.scheduledObserved){self.scheduledObserved=YES;dispatch_group_leave(self.scheduledGroup);}dispatch_group_leave(self.visibilityGroup);[pendingLock lock];[pendingCommands removeObject:self];[pendingLock unlock];dispatch_group_leave(group);@throw; + } +} +- (void)trackFence:(id)fence { + if(!self.fences)self.fences=[NSMutableArray array];if([self.fences indexOfObjectIdenticalTo:fence]==NSNotFound)[self.fences addObject:(OMFenceObject *)fence]; +} +- (void)presentDrawable:(id)drawable { + if(self.submissionStarted || self.actualResult.status || !drawable || ![(id)drawable conformsToProtocol:@protocol(MTLDrawable)] || + !methodABI(drawable,@selector(texture),"@16@0:8") || !methodABI(drawable,@selector(present),"v16@0:8"))fail(@"Unqualified drawable or already completed command"); + id texture=[(id)drawable texture]; + if(![(id)texture isKindOfClass:[OMTextureObject class]] || texture.device!=remoteDevice || !texture.iosurface) + fail(@"Drawable requires this device's actual IOSurface texture"); + if(!self.presentations)self.presentations=[NSMutableArray array]; + if(self.presentations.count>=8 || [self.presentations indexOfObjectIdenticalTo:drawable]!=NSNotFound)fail(@"Duplicate or excessive actual drawable presentation"); + [self trackSurfaceTarget:(OMTextureObject *)texture];[self.presentations addObject:drawable]; +} +- (void)addCompletedHandler:(MTLCommandBufferHandler)handler { + if(!handler || self.submissionStarted || self.actualResult.status)fail(@"Completion handler requires a valid uncommitted command"); + if(!self.completedHandlers)self.completedHandlers=[NSMutableArray array];[self.completedHandlers addObject:[handler copy]]; +} +- (void)addScheduledHandler:(MTLCommandBufferHandler)handler { + if(!handler || self.submissionStarted || self.actualResult.status)fail(@"Scheduled handler requires a valid uncommitted command"); + if(!self.scheduledHandlers)self.scheduledHandlers=[NSMutableArray array];[self.scheduledHandlers addObject:[handler copy]]; +} +- (void)enqueue { + if(self.submissionStarted || self.actualResult.status || self.actualEnqueueStatus)fail(@"Command already enqueued or submitted"); + OMHandle request={self.handle};OMResult result={};NSData *body=checked(OM_COMMAND_ENQUEUE,&request,sizeof(request),&result); + if(body.length!=sizeof(uint64_t))fail(@"Missing actual enqueue status");uint64_t status=0;memcpy(&status,body.bytes,sizeof(status)); + if(status!=MTLCommandBufferStatusEnqueued)fail(@"Actual GPU command was not enqueued");self.actualEnqueueStatus=(MTLCommandBufferStatus)status; +} +- (void)trackBuffer:(OMBufferObject *)buffer { + if(!self.buffers)self.buffers=[NSMutableArray array];if(![self.buffers containsObject:buffer])[self.buffers addObject:buffer]; +} +- (void)trackTexture:(OMTextureObject *)texture { + if(!self.textures)self.textures=[NSMutableArray array];if(![self.textures containsObject:texture])[self.textures addObject:texture]; +} +- (void)trackSurfaceTarget:(OMTextureObject *)texture { + [self trackTexture:texture]; + if(texture.surface){if(!self.surfaceTargets)self.surfaceTargets=[NSMutableArray array];if(![self.surfaceTargets containsObject:texture])[self.surfaceTargets addObject:texture];} +} +- (id)renderCommandEncoderWithDescriptor:(MTLRenderPassDescriptor *)descriptor { + if(!descriptor)fail(@"Missing render pass descriptor"); + OMAttachmentRender extended={};OMRender &request=extended.colors;request.command=self.handle; + for(unsigned i=0;i<8;i++) { + MTLRenderPassColorAttachmentDescriptor *a=descriptor.colorAttachments[i];if(!a.texture)continue; + if(![(id)a.texture isKindOfClass:[OMTextureObject class]] || a.level || a.slice || a.depthPlane || a.resolveTexture)fail(@"Unsupported or foreign render attachment"); + request.colors[i].texture=[(OMObject *)a.texture handle];request.colors[i].load=a.loadAction;request.colors[i].store=a.storeAction; + OMTextureObject *texture=(OMTextureObject *)a.texture; + [self trackSurfaceTarget:texture]; + if(texture.surface)[texture uploadSurface]; + MTLClearColor color=a.clearColor;request.colors[i].clear[0]=color.red;request.colors[i].clear[1]=color.green;request.colors[i].clear[2]=color.blue;request.colors[i].clear[3]=color.alpha; + } + MTLRenderPassDepthAttachmentDescriptor *depth=descriptor.depthAttachment; + if(depth.texture) { + if(![(id)depth.texture isKindOfClass:[OMTextureObject class]] || depth.level || depth.slice || depth.depthPlane || depth.resolveTexture)fail(@"Unsupported or foreign depth attachment"); + extended.depth={[(OMObject *)depth.texture handle],(uint64_t)depth.loadAction,(uint64_t)depth.storeAction,depth.clearDepth};[self trackTexture:(OMTextureObject *)depth.texture]; + } + MTLRenderPassStencilAttachmentDescriptor *stencil=descriptor.stencilAttachment; + if(stencil.texture) { + if(![(id)stencil.texture isKindOfClass:[OMTextureObject class]] || stencil.level || stencil.slice || stencil.depthPlane || stencil.resolveTexture)fail(@"Unsupported or foreign stencil attachment"); + extended.stencil={[(OMObject *)stencil.texture handle],(uint64_t)stencil.loadAction,(uint64_t)stencil.storeAction,stencil.clearStencil};[self trackTexture:(OMTextureObject *)stencil.texture]; + } + bool attachments=depth.texture || stencil.texture; + OMResult result={};checked(attachments?OM_RENDER_ATTACHMENTS:OM_RENDER,attachments?(const void *)&extended:(const void *)&request,attachments?sizeof(extended):sizeof(request),&result); + OMEncoderObject *object=[OMEncoderObject new];object.handle=result.handle;object.command=self;return (id)object; +} +- (id)blitCommandEncoder { + OMHandle request={self.handle};OMResult result={};checked(OM_BLIT,&request,sizeof(request),&result); + OMBlitEncoderObject *object=[OMBlitEncoderObject new];object.handle=result.handle;object.command=self;return (id)object; +} +- (void)commit { + if(self.useAsync){[self commitAsync];return;} + if(self.actualResult.status)fail(@"Command already submitted"); + for(OMBufferObject *buffer in self.buffers)[buffer upload]; + for(OMTextureObject *texture in self.textures)[texture uploadSurface]; + OMHandle request={self.handle};OMResult result={};NSData *body=checked(OM_COMMIT,&request,sizeof(request),&result); + if(body.length!=sizeof(OMCommandResult))fail(@"Missing actual command result");OMCommandResult value={};memcpy(&value,body.bytes,sizeof(value)); + if(value.status!=MTLCommandBufferStatusCompleted || !value.start_ns || value.end_ns<=value.start_ns)fail(@"Actual GPU command did not complete"); + for(OMTextureObject *texture in self.surfaceTargets)[texture downloadSurface]; + for(OMBufferObject *buffer in self.buffers)[buffer download]; + self.actualResult=value;self.resourcesVisible=YES; + // Present the real original drawable only after actual GPU completion and + // visibility in its original IOSurface. Its real framework owns presentation + // scheduling and timestamps; GPU completion is never a fabricated display + // completion. Preserve actual GPU status/callbacks even if presentation fails. + NSArray> *presentations=self.presentations;self.presentations=nil; + NSException *presentationFailure=nil; + @try {for(id drawable in presentations)[drawable present];} + @catch(NSException *exception){presentationFailure=exception;} + // This transport is synchronous: submission, execution and surface/buffer + // readback have all actually finished. Deliver each registered block exactly + // once on a real dispatch worker, with the real completed command. Completion + // waits also wait for that delivery, without invoking callbacks under the + // original renderer's encoding/commit call stack. + // Scheduled callbacks are deliberately late here; no asynchronous timing or + // fabricated Scheduled/Completed state is advertised. + NSArray *scheduled=self.scheduledHandlers,*completed=self.completedHandlers; + self.scheduledHandlers=nil;self.completedHandlers=nil; + dispatch_group_t group=nil; + if(scheduled.count || completed.count){group=dispatch_group_create();dispatch_group_enter(group);self.callbackGroup=group;} + if(group)dispatch_async(dispatch_get_global_queue(QOS_CLASS_DEFAULT,0),^{ + self.callbackThread=pthread_self(); + @try { + for(MTLCommandBufferHandler handler in scheduled)handler((id)self); + for(MTLCommandBufferHandler handler in completed)handler((id)self); + } @finally {self.callbackThread=nullptr;dispatch_group_leave(group);} + }); + if(presentationFailure)@throw presentationFailure; +} +- (void)waitUntilScheduled { + if(self.useAsync) { + if(!self.submissionStarted || !self.scheduledGroup)fail(@"Command has not actually been submitted"); + if(dispatch_group_wait(self.scheduledGroup,dispatch_time(DISPATCH_TIME_NOW,15*NSEC_PER_SEC)))fail(@"Actual scheduling observation timed out"); + if(self.asynchronousFailure)@throw self.asynchronousFailure; + if(self.actualScheduledStatus=MTLCommandBufferStatusCommitted && self.actualResult.status<=MTLCommandBufferStatusCompleted; + [self waitUntilScheduled];return self.actualResult.status==MTLCommandBufferStatusCompleted; +} +- (void)waitUntilCompleted { + if(self.useAsync) { + if(!self.submissionStarted || !self.callbackGroup)fail(@"Command has not actually been submitted"); + bool ownCallback=self.callbackThread && pthread_equal(self.callbackThread,pthread_self()); + if(!ownCallback && dispatch_group_wait(self.callbackGroup,dispatch_time(DISPATCH_TIME_NOW,15*NSEC_PER_SEC)))fail(@"Actual asynchronous completion/visibility/callback wait timed out"); + if(self.asynchronousFailure)@throw self.asynchronousFailure;if(self.deliveryFailure)@throw self.deliveryFailure; + if(!self.resourcesVisible || self.actualResult.status!=MTLCommandBufferStatusCompleted)fail(@"Actual GPU resources are not completed and visible");return; + } + if(self.actualResult.status!=MTLCommandBufferStatusCompleted)fail(@"Command has not actually completed"); + if(self.callbackGroup && !(self.callbackThread && pthread_equal(self.callbackThread,pthread_self())) && dispatch_group_wait(self.callbackGroup,dispatch_time(DISPATCH_TIME_NOW,10*NSEC_PER_SEC)))fail(@"Actual completion-handler delivery timed out"); +} +- (MTLCommandBufferStatus)status { + if(self.asynchronousFailure)return MTLCommandBufferStatusError; + OMCommandResult value=self.actualResult; + if(self.useAsync && value.status==MTLCommandBufferStatusCompleted && !self.resourcesVisible){if(dispatch_group_wait(self.visibilityGroup,dispatch_time(DISPATCH_TIME_NOW,15*NSEC_PER_SEC)))fail(@"Actual resource visibility timed out");if(self.asynchronousFailure)return MTLCommandBufferStatusError;value=self.actualResult;} + return value.status?(MTLCommandBufferStatus)value.status:self.actualEnqueueStatus; +} +- (NSError *)error { + if(self.asynchronousFailure)return [NSError errorWithDomain:@"org.blu-sharky.offline-metal.async" code:-1 userInfo:@{NSLocalizedDescriptionKey:self.asynchronousFailure.reason?:@"Actual asynchronous transport/GPU failure"}]; + return nil; +} +- (CFTimeInterval)GPUStartTime {return self.actualResult.start_ns/1e9;} +- (CFTimeInterval)GPUEndTime {return self.actualResult.end_ns/1e9;} +@end +@interface OMQueueObject : OMObject +@end +@implementation OMQueueObject +- (NSUInteger)readPriority:(BOOL)background { + OMQueuePriority request={self.handle,(uint64_t)background,0};OMResult result={};NSData *body=checked(OM_QUEUE_PRIORITY_GET,&request,sizeof(request),&result); + if(body.length!=sizeof(uint64_t))fail(@"Missing actual queue priority");uint64_t value=0;memcpy(&value,body.bytes,sizeof(value));return value; +} +- (BOOL)writePriority:(NSUInteger)priority background:(BOOL)background { + // Exact original MTLIOAccelCommandQueue validation: 0..5, except 3 in foreground. + if(priority>5 || (!background && priority==3))return NO; + OMQueuePriority request={self.handle,(uint64_t)background,priority};OMResult result={};NSData *body=call(OM_QUEUE_PRIORITY_SET,&request,sizeof(request),&result); + if(result.error)return NO;if(body.length!=sizeof(uint64_t))fail(@"Missing actual queue priority-change result");uint64_t value=0;memcpy(&value,body.bytes,sizeof(value)); + if(value>1)fail(@"Invalid actual priority-change result");return value!=0; +} +- (NSUInteger)getGPUPriority {return [self readPriority:NO];} +- (NSUInteger)getBackgroundGPUPriority {return [self readPriority:YES];} +- (BOOL)setGPUPriority:(NSUInteger)priority {return [self writePriority:priority background:NO];} +- (BOOL)setBackgroundGPUPriority:(NSUInteger)priority {return [self writePriority:priority background:YES];} +- (id)commandBuffer { + OMHandle request={self.handle};OMResult result={};checked(OM_COMMAND,&request,sizeof(request),&result); + OMCommandObject *object=[OMCommandObject new];object.handle=result.handle;object.commandQueue=(id)self;object.useAsync=asyncEnabled;return (id)object; +} +@end +// This model contributes only own implementations to a dynamically allocated +// real _MTLDevice subclass. No original method implementation is overwritten. +@interface OMDeviceMethods : NSObject +@end +@implementation OMDeviceMethods +- (NSString *)name {return deviceInfo[@"name"];} +- (uint64_t)registryID {return [deviceInfo[@"registryID"] unsignedLongLongValue];} +- (NSUInteger)maxBufferLength {return [deviceInfo[@"maxBufferLength"] unsignedIntegerValue];} +- (BOOL)isLowPower {return [deviceInfo[@"lowPower"] boolValue];} +- (BOOL)isRemovable {return [deviceInfo[@"removable"] boolValue];} +- (BOOL)isHeadless {return [deviceInfo[@"headless"] boolValue];} +// This partial offline device does not implement any complete GPU family or +// feature set. The original base implementation needs Apple capability tables +// which this independent device neither owns nor populates. +- (BOOL)supportsFamily:(MTLGPUFamily)family {(void)family;return NO;} +- (BOOL)supportsFeatureSet:(MTLFeatureSet)featureSet {(void)featureSet;return NO;} +- (id)newEvent { + OMResult result={};call(OM_EVENT,nullptr,0,&result);if(result.error)return nil; + if(!result.handle)fail(@"Missing actual GPU event identity");OMEventObject *object=[OMEventObject new];object.handle=result.handle;return (id)object; +} +- (id)newSharedEvent { + OMResult result={};NSData *body=call(OM_SHARED_EVENT,nullptr,0,&result);if(result.error)return nil; + if(!result.handle)fail(@"Missing actual shared-event identity");NSDictionary *info=sharedMetadata(body); + OMSharedEventObject *object=[OMSharedEventObject new];object.handle=result.handle;object.deviceIsNil=[info[@"device_is_nil"] boolValue]; + // Initialize without writing metadata back to the actual event. + ((void (*)(id,SEL,id))class_getMethodImplementation([OMObject class],@selector(setLabel:)))(object,@selector(setLabel:),info[@"label"]==[NSNull null]?nil:info[@"label"]); + return object; +} +- (id)newSharedEventWithHandle:(MTLSharedEventHandle *)handle { + Class cls=NSClassFromString(@"MTLSharedEventHandle");if(!handle || object_getClass(handle)!=cls || class_getInstanceSize(cls)!=16)return nil; + mach_port_t port=handle.eventPort;if(!MACH_PORT_VALID(port))return nil; + [transaction lock];@try { + if(!live || closing)return nil;uint64_t input=port,output=0;unsigned count=1;unsigned char bytes[4096];size_t size=sizeof(bytes); + int status=ioMethod(connection,3,&input,1,nullptr,0,&output,&count,bytes,&size);if(status)return nil; + if(count!=1 || !output)fail(@"Missing actual imported shared event");NSDictionary *info=sharedMetadata([NSData dataWithBytes:bytes length:size]); + OMSharedEventObject *object=[OMSharedEventObject new];object.handle=output;object.deviceIsNil=[info[@"device_is_nil"] boolValue]; + ((void (*)(id,SEL,id))class_getMethodImplementation([OMObject class],@selector(setLabel:)))(object,@selector(setLabel:),info[@"label"]==[NSNull null]?nil:info[@"label"]); + return object; + } @finally {[transaction unlock];} +} +- (id)newFence { + OMResult result={};call(OM_FENCE,nullptr,0,&result);if(result.error)return nil; + if(!result.handle)fail(@"Missing actual GPU fence identity");OMFenceObject *object=[OMFenceObject new];object.handle=result.handle;return (id)object; +} +- (id)newCommandQueue { + OMResult result={};checked(OM_QUEUE,nullptr,0,&result);OMQueueObject *object=[OMQueueObject new];object.handle=result.handle;return (id)object; +} +- (id)newBufferWithLength:(NSUInteger)length options:(MTLResourceOptions)options { + if(!length || length>MIN((NSUInteger)OFFLINE_METAL_MAX_BUFFER,self.maxBufferLength) || options)return nil; + OMBuffer request={length,(uint64_t)options};OMResult result={};call(OM_BUFFER,&request,sizeof(request),&result);if(result.error)return nil; + OMBufferObject *object=[OMBufferObject new];object.handle=result.handle;object.data=[NSMutableData dataWithLength:length];return (id)object; +} +- (id)newBufferWithBytes:(const void *)bytes length:(NSUInteger)length options:(MTLResourceOptions)options { + if(!bytes)return nil;OMBufferObject *object=(OMBufferObject *)[self newBufferWithLength:length options:options];if(!object)return nil; + memcpy(object.data.mutableBytes,bytes,length);return (id)object; +} +- (id)newPipelineLibraryWithFilePath:(NSString *)path error:(NSError **)error { + // This optional original directory contains Apple-family pipeline entries. + // No backend qualification exists for that private archive format. Refuse + // explicitly so the unchanged caller can take its real AIR-library path, + // rather than invoking the Apple base implementation with absent internals. + (void)path; + if(error){*error=[NSError errorWithDomain:@"org.blu-sharky.offline-metal" code:-4 + userInfo:@{NSLocalizedDescriptionKey:@"Apple-family pipeline libraries are not qualified for this independent device; use portable AIR input"}];} + return nil; +} +- (id)newLibraryWithFile:(NSString *)path error:(NSError **)error { + NSError *failure=nil;NSData *bytes=path?[NSData dataWithContentsOfFile:path options:NSDataReadingMappedIfSafe error:&failure]:nil; + if(!bytes){if(error)*error=failure?:[NSError errorWithDomain:@"org.blu-sharky.offline-metal" code:-2 userInfo:@{NSLocalizedDescriptionKey:@"A readable actual library file is required"}];return nil;} + dispatch_data_t data=dispatch_data_create(bytes.bytes,bytes.length,dispatch_get_global_queue(QOS_CLASS_DEFAULT,0),^{(void)bytes.length;}); + return [(id)self newLibraryWithData:data error:error]; +} +- (id)newLibraryWithURL:(NSURL *)url error:(NSError **)error { + if(!url.isFileURL){if(error)*error=[NSError errorWithDomain:@"org.blu-sharky.offline-metal" code:-2 userInfo:@{NSLocalizedDescriptionKey:@"An actual local library file URL is required"}];return nil;} + return [(id)self newLibraryWithFile:url.path error:error]; +} +- (id)newDefaultLibraryWithBundle:(NSBundle *)bundle error:(NSError **)error { + NSURL *url=[bundle URLForResource:@"default" withExtension:@"metallib"]; + if(!url){if(error)*error=[NSError errorWithDomain:@"org.blu-sharky.offline-metal" code:-2 userInfo:@{NSLocalizedDescriptionKey:@"The actual bundle has no default Metal library"}];return nil;} + return [(id)self newLibraryWithURL:url error:error]; +} +- (id)newLibraryWithData:(dispatch_data_t)data error:(NSError **)error { + const void *bytes=nullptr;size_t size=0;dispatch_data_t mapped=dispatch_data_create_map(data,&bytes,&size);(void)mapped; + if(!bytes || !size || size>64*1024*1024){if(error)*error=[NSError errorWithDomain:@"org.blu-sharky.offline-metal" code:-2 userInfo:nil];return nil;} + OMLibrary request={};CC_SHA256(bytes,(CC_LONG)size,request.sha256);OMResult result={};NSData *body=call(OM_LIBRARY,&request,sizeof(request),&result); + if(result.error){if(error)*error=operationError(result,body);return nil;} + NSArray *names=[NSJSONSerialization JSONObjectWithData:body options:0 error:nil];if(![names isKindOfClass:[NSArray class]])fail(@"Missing actual library metadata"); + OMLibraryObject *object=[OMLibraryObject new];object.handle=result.handle;object.functionNames=names;if(error)*error=nil;return (id)object; +} +- (id)newTextureWithDescriptor:(MTLTextureDescriptor *)d { + if(!d || d.hazardTrackingMode>MTLHazardTrackingModeTracked || !d.allowGPUOptimizedContents)return nil; + OMTexture request={(uint64_t)d.textureType,(uint64_t)d.pixelFormat,d.width,d.height,d.depth,d.mipmapLevelCount,d.sampleCount,d.arrayLength,(uint64_t)d.usage,(uint64_t)d.storageMode,(uint64_t)d.cpuCacheMode}; + OMHazardTexture extended={request,(uint64_t)d.hazardTrackingMode};OMResult result={};NSData *body=call(OM_TEXTURE_HAZARD,&extended,sizeof(extended),&result);if(result.error)return nil; + if(body.length!=sizeof(uint64_t))fail(@"Missing actual texture hazard mode");uint64_t actual=0;memcpy(&actual,body.bytes,sizeof(actual)); + if(actual>MTLHazardTrackingModeTracked || (extended.hazard && actual!=extended.hazard))fail(@"Actual texture hazard mode differs"); + OMTextureObject *object=[OMTextureObject new];object.handle=result.handle;object.descriptor=request;object.actualHazardMode=(MTLHazardTrackingMode)actual;return (id)object; +} +- (id)newDepthStencilStateWithDescriptor:(MTLDepthStencilDescriptor *)d { + if(!d)return nil; + auto face=[](MTLStencilDescriptor *s)->OMStencilFace { + if(!s)return {};return {1,(uint64_t)s.stencilCompareFunction,(uint64_t)s.stencilFailureOperation,(uint64_t)s.depthFailureOperation, + (uint64_t)s.depthStencilPassOperation,s.readMask,s.writeMask}; + }; + OMDepthStencil request={(uint64_t)d.depthCompareFunction,(uint64_t)d.depthWriteEnabled,face(d.frontFaceStencil),face(d.backFaceStencil)}; + OMResult result={};call(OM_DEPTH_STENCIL_STATE,&request,sizeof(request),&result);if(result.error)return nil; + OMDepthStateObject *object=[OMDepthStateObject new];object.handle=result.handle;return (id)object; +} +- (id)newTextureWithDescriptor:(MTLTextureDescriptor *)d iosurface:(IOSurfaceRef)surface plane:(NSUInteger)plane { + if(!surface || CFGetTypeID(surface)!=IOSurfaceGetTypeID() || !d || plane || IOSurfaceGetPlaneCount(surface) || d.storageMode!=MTLStorageModeShared)return nil; + bool format=((d.pixelFormat==MTLPixelFormatR32Float && IOSurfaceGetPixelFormat(surface)==kCVPixelFormatType_OneComponent32Float) + || (d.pixelFormat==MTLPixelFormatRGBA8Unorm && IOSurfaceGetPixelFormat(surface)==kCVPixelFormatType_32RGBA) + || (d.pixelFormat==MTLPixelFormatBGRA8Unorm && IOSurfaceGetPixelFormat(surface)==kCVPixelFormatType_32BGRA)); + if(!format + || !d.width || !d.height || d.width!=IOSurfaceGetWidth(surface) || d.height!=IOSurfaceGetHeight(surface) + || d.width>SIZE_MAX/4 || IOSurfaceGetBytesPerElement(surface)!=4 || IOSurfaceGetBytesPerRow(surface)SIZE_MAX/d.height || IOSurfaceGetAllocSize(surface))object; +} +- (id)newRenderPipelineStateWithDescriptor:(MTLRenderPipelineDescriptor *)d error:(NSError **)error { + static_assert(MTLVertexFormatFloat4==31 && MTLVertexFormatHalf4==27 && MTLVertexStepFunctionPerInstance==2,"Actual target SDK vertex enum ABI"); + bool vertexInputs=false;OMVertexPipeline vertexPipeline={}; + // Read real original descriptors. Inactive attributes/layouts carry zeroes; + // static stepping is transported without modifying any original descriptor. + if(d.vertexDescriptor)for(unsigned i=0;i<31;i++) { + MTLVertexAttributeDescriptor *a=d.vertexDescriptor.attributes[i];if(a.format==MTLVertexFormatInvalid)continue; + vertexInputs=true;vertexPipeline.attributes[i]={(uint64_t)a.format,a.offset,a.bufferIndex}; + if(a.bufferIndex<31) { + MTLVertexBufferLayoutDescriptor *l=d.vertexDescriptor.layouts[a.bufferIndex]; + vertexPipeline.layouts[a.bufferIndex]={l.stride,(uint64_t)l.stepFunction,l.stepRate}; + } + } + if(![(id)d.vertexFunction isKindOfClass:[OMFunctionObject class]] || ![(id)d.fragmentFunction isKindOfClass:[OMFunctionObject class]] + || (vertexInputs && !OMVertexInputsValid(vertexPipeline)) || !d.rasterizationEnabled || d.alphaToCoverageEnabled || d.alphaToOneEnabled) { + if(error)*error=[NSError errorWithDomain:@"org.blu-sharky.offline-metal" code:-4 userInfo:nil];return nil; + } + OMAttachmentPipeline attachments={};OMBlendPipeline &extended=attachments.colors;OMPipeline &request=extended.pipeline;bool blending=false; + attachments.depthFormat=d.depthAttachmentPixelFormat;attachments.stencilFormat=d.stencilAttachmentPixelFormat; + request.vertex=[(OMObject *)d.vertexFunction handle];request.fragment=[(OMObject *)d.fragmentFunction handle];request.samples=d.rasterSampleCount; + for(unsigned i=0;i<8;i++) { + MTLRenderPipelineColorAttachmentDescriptor *a=d.colorAttachments[i];request.formats[i]=a.pixelFormat;request.masks[i]=a.writeMask; + blending|=a.blendingEnabled;extended.colors[i]={(uint64_t)a.blendingEnabled,(uint64_t)a.sourceRGBBlendFactor,(uint64_t)a.destinationRGBBlendFactor, + (uint64_t)a.rgbBlendOperation,(uint64_t)a.sourceAlphaBlendFactor,(uint64_t)a.destinationAlphaBlendFactor,(uint64_t)a.alphaBlendOperation}; + } + bool attached=attachments.depthFormat || attachments.stencilFormat; + vertexPipeline.pipeline=attachments; + OMResult result={};NSData *body=call((vertexInputs || attached)?OM_PIPELINE_ATTACHMENTS:blending?OM_PIPELINE_BLEND:OM_PIPELINE, + vertexInputs?(const void *)&vertexPipeline:attached?(const void *)&attachments:blending?(const void *)&extended:(const void *)&request, + vertexInputs?sizeof(vertexPipeline):attached?sizeof(attachments):blending?sizeof(extended):sizeof(request),&result);if(result.error){if(error)*error=operationError(result,body);return nil;} + OMPipelineObject *object=[OMPipelineObject new];object.handle=result.handle;return (id)object; +} +@end +extern "C" id offline_metal_frontend_open(void); +extern "C" id offline_metal_frontend_open_service(const char *); +extern "C" void offline_metal_frontend_finish(void); +id offline_metal_frontend_open(void) { + return offline_metal_frontend_open_service("NativeVirtioConsole"); +} +id offline_metal_frontend_open_service(const char *serviceName) { + if(!serviceName || (strcmp(serviceName,"NativeVirtioConsole") && strcmp(serviceName,"NativeOfflineGPU")))return nil; + if(live)return nil;void *iokit=dlopen("/System/Library/Frameworks/IOKit.framework/IOKit",RTLD_NOW|RTLD_LOCAL);if(!iokit)return nil; + auto matching=(void *(*)(const char *))dlsym(iokit,"IOServiceMatching");auto services=(int (*)(unsigned,void *,unsigned *))dlsym(iokit,"IOServiceGetMatchingServices"); + auto next=(unsigned (*)(unsigned))dlsym(iokit,"IOIteratorNext");auto release=(int (*)(unsigned))dlsym(iokit,"IOObjectRelease");auto open=(int (*)(unsigned,unsigned,unsigned,unsigned *))dlsym(iokit,"IOServiceOpen"); + ioMethod=(IOMethod)dlsym(iokit,"IOConnectCallMethod");if(!matching || !services || !next || !release || !open || !ioMethod)return nil; + unsigned iterator=0;if(services(0,matching(serviceName),&iterator) || !iterator)return nil;unsigned service=next(iterator); + unsigned extra=next(iterator);int openStatus=-1; + if(service && !extra)openStatus=open(service,mach_task_self(),0,&connection); + if(extra)release(extra);if(service)release(service);release(iterator);if(openStatus)return nil; + transaction=[NSRecursiveLock new];pendingLock=[NSLock new];pendingCommands=[NSMutableSet set];pendingNotifications=[NSMutableSet set];notificationFailure=nil;asyncEnabled=false;closing=false;live=true;OMResult result={};NSData *body=checked(OM_DEVICE,nullptr,0,&result); + deviceInfo=[NSJSONSerialization JSONObjectWithData:body options:0 error:nil];if(![deviceInfo isKindOfClass:[NSDictionary class]])fail(@"Actual device metadata missing"); + Class base=objc_getClass("_MTLDevice");if(!base || class_getInstanceSize(base)!=536)fail(@"Wrong original Metal base ABI"); + Class cls=objc_allocateClassPair(base,"NativeOfflineMetalDevice",0);if(!cls)fail(@"Own device class allocation failed"); + unsigned count=0;Method *methods=class_copyMethodList([OMDeviceMethods class],&count); + for(unsigned i=0;i=1 && format<=12)return 2+(format-1)%3; + if(format>=13 && format<=24)return 2*(2+(format-13)%3); + if(format>=25 && format<=27)return 2*(format-23); + if(format>=28 && format<=39)return 4*(1+(format-28)%4); + if(format>=40 && format<=42)return 4; + if(format>=45 && format<=48)return 1; + if(format>=49 && format<=53)return 2; + if(format==54 || format==55)return 4; + return 0; +} +static inline bool OMVertexInputsValid(const OMVertexPipeline &p) { + bool used[31]={};unsigned count=0; + for(const auto &a:p.attributes) { + if(!a.format){if(a.offset || a.buffer)return false;continue;} + unsigned bytes=OMVertexFormatBytes(a.format); + if(!bytes || a.buffer>=31 || a.offset>4096-bytes)return false; + used[a.buffer]=true;count++; + } + if(!count)return false; + for(unsigned i=0;i<31;i++) { + const auto &l=p.layouts[i]; + if(!used[i]){if(l.stride || l.step || l.rate)return false;continue;} + if(l.stride>OFFLINE_METAL_MAX_VERTEX_STRIDE || l.step>2 || l.rate>65535 || (l.step && (!l.rate || !l.stride)))return false; + } + return true; +} diff --git a/drivers/NativeMetal/OfflineMetalWire.h b/drivers/NativeMetal/OfflineMetalWire.h new file mode 100644 index 0000000..90f644e --- /dev/null +++ b/drivers/NativeMetal/OfflineMetalWire.h @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: MIT +// Offline Metal command transport. No deployment GPU or page-table ABI. +#pragma once +#include +enum { OM_WIRE_MAGIC=0x5550474e, OM_WIRE_RESPONSE=0x8000 }; +typedef struct {uint32_t magic;uint16_t version,operation;uint32_t bytes,request;} OMWireHeader; +static_assert(sizeof(OMWireHeader)==16,"Native Metal wire header"); +enum { OFFLINE_METAL_VERSION=2, OFFLINE_METAL_MAX_REPLY=16*1024*1024, + OM_DEVICE=0x100,OM_QUEUE,OM_LIBRARY,OM_FUNCTION,OM_TEXTURE,OM_PIPELINE, + OM_COMMAND,OM_RENDER,OM_SET_PIPELINE,OM_SET_CULL,OM_DRAW,OM_END, + OM_COMMIT,OM_READ_TEXTURE,OM_RELEASE,OM_FINISH,OM_WRITE_TEXTURE, + OM_BUFFER,OM_WRITE_BUFFER,OM_READ_BUFFER,OM_VERTEX_BUFFER,OM_FRAGMENT_BUFFER, + OM_DRAW_INDEXED,OM_BLIT,OM_COPY_BUFFER,OM_PIPELINE_BLEND,OM_VERTEX_TEXTURE,OM_FRAGMENT_TEXTURE, + OM_SCISSOR,OM_VIEWPORT,OM_BLEND_COLOR }; +enum { OFFLINE_METAL_MAX_BUFFER=64*1024*1024 }; +typedef struct {int32_t error;uint32_t reserved;uint64_t handle;} OMResult; +typedef struct {uint64_t handle;} OMHandle; +typedef struct {uint8_t sha256[32];} OMLibrary; +typedef struct {uint64_t library;char name[128];} OMFunction; +typedef struct {uint64_t type,format,width,height,depth,levels,samples,array,usage,storage,cache;} OMTexture; +typedef struct {uint64_t vertex,fragment,samples,formats[8],masks[8];} OMPipeline; +typedef struct {uint64_t texture,load,store;double clear[4];} OMColor; +typedef struct {uint64_t command;OMColor colors[8];} OMRender; +typedef struct {uint64_t encoder,pipeline;} OMSetPipeline; +typedef struct {uint64_t encoder,mode;} OMSetCull; +typedef struct {uint64_t encoder,primitive,start,count,instances;} OMDraw; +typedef struct {uint64_t status,start_ns,end_ns;} OMCommandResult; +typedef struct {uint64_t texture,x,y,width,height,level;} OMReadTexture; +typedef struct {uint64_t texture,x,y,width,height,level,stride;} OMWriteTexture; +typedef struct {uint64_t bytes,options;} OMBuffer; +typedef struct {uint64_t buffer,offset,bytes;} OMBufferRegion; +typedef struct {uint64_t encoder,buffer,offset,index;} OMSetBuffer; +typedef struct {uint64_t encoder,primitive,count,type,buffer,offset,instances;int64_t baseVertex;uint64_t baseInstance;} OMDrawIndexed; +typedef struct {uint64_t encoder,source,sourceOffset,destination,destinationOffset,bytes;} OMCopyBuffer; +typedef struct {uint64_t enabled,sourceRGB,destinationRGB,rgbOperation,sourceAlpha,destinationAlpha,alphaOperation;} OMBlend; +typedef struct {OMPipeline pipeline;OMBlend colors[8];} OMBlendPipeline; +typedef struct {uint64_t encoder,texture,index;} OMSetTexture; +typedef struct {uint64_t encoder,x,y,width,height;} OMScissor; +typedef struct {uint64_t encoder;double x,y,width,height,nearDepth,farDepth;} OMViewport; +typedef struct {uint64_t encoder;float red,green,blue,alpha;} OMBlendColor; diff --git a/drivers/NativeMetal/OriginalAcceleratorABI.h b/drivers/NativeMetal/OriginalAcceleratorABI.h new file mode 100644 index 0000000..a052b09 --- /dev/null +++ b/drivers/NativeMetal/OriginalAcceleratorABI.h @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: MIT +#pragma once +// Exact supplied iOS17 base, independently audited against its complete +// original vtable and protected constructor/destructor bodies. No extra fields +// or operational overrides beyond IOService. This declares a real ancestry; +// no original class, method, vtable or metaclass is replaced. +#include +class IOAcceleratorES : public IOService { + OSDeclareAbstractStructors(IOAcceleratorES) +}; +static_assert(sizeof(IOService)==136 && sizeof(IOAcceleratorES)==136,"Qualified original accelerator base extent"); diff --git a/drivers/NativeMetal/OriginalGPUProviderABI.h b/drivers/NativeMetal/OriginalGPUProviderABI.h new file mode 100644 index 0000000..78ecf5b --- /dev/null +++ b/drivers/NativeMetal/OriginalGPUProviderABI.h @@ -0,0 +1,74 @@ +// SPDX-License-Identifier: MIT +#pragma once +// Unpublished ABI instrument only. The native service prefix and opaque storage +// match the exact original abstract base. Non-pure tail entries are bound to +// their ORIGINAL entry addresses, with no C++ caller supplied for those opaque +// source signatures. Required tail probes capture their actual register frame; +// descriptor slot170 may return NULL only for an explicit missing-resource +// control. No unknown signature, capability or successful +// operational implementation is claimed. This header is not a system driver API. +#include "OriginalAcceleratorABI.h" +class IOMemoryDescriptor; +class IOGPU : public IOAcceleratorES { + OSDeclareAbstractStructors(IOGPU) + unsigned char originalPrivateState[600]; +protected: + void free() override; +public: + IOReturn configureReport(IOReportChannelList *,IOReportConfigureAction,void *,void *) override; + IOReturn updateReport(IOReportChannelList *,IOReportUpdateAction,void *,void *) override; + bool start(IOService *) override; + void stop(IOService *) override; + IOWorkLoop *getWorkLoop() const override; + using IOAcceleratorES::newUserClient; + IOReturn newUserClient(task_t,void *,UInt32,IOUserClient **) override; +protected: + virtual void abiSlot168(); + virtual void abiSlot169(); + virtual IOMemoryDescriptor *abiSlot170(UInt32 *) = 0; + virtual void abiSlot171(); + virtual void abiSlot172(); + virtual void abiSlot173(); + virtual void abiSlot174() = 0; + virtual void abiSlot175(); + virtual void abiSlot176(); + virtual void abiSlot177(); + virtual void abiSlot178(); + virtual void abiSlot179(); + virtual void abiSlot180(); + virtual void abiSlot181(); + virtual void abiSlot182(); + virtual void abiSlot183(); + virtual void abiSlot184(); + virtual void abiSlot185(); + virtual void abiSlot186(); + virtual void abiSlot187(); + virtual void abiSlot188(); + virtual void abiSlot189() = 0; + virtual void abiSlot190(); + virtual void abiSlot191(); + virtual void abiSlot192(void *) = 0; + virtual void abiSlot193(); + virtual void abiSlot194(); + virtual bool abiSlot195(IOService *) = 0; + virtual void abiSlot196() = 0; + virtual void abiSlot197() = 0; + virtual void abiSlot198(); + virtual void abiSlot199(); + virtual void abiSlot200() = 0; + virtual void abiSlot201(); + virtual void abiSlot202() = 0; + virtual void abiSlot203(); + virtual void abiSlot204(); + virtual void abiSlot205() = 0; + virtual void abiSlot206(); + virtual void abiSlot207(); + virtual void abiSlot208(); + virtual void abiSlot209(); + virtual void abiSlot210(); + virtual void abiSlot211(); + virtual void abiSlot212(); + virtual void abiSlot213(); + virtual void abiSlot214(); +}; +static_assert(sizeof(IOGPU)==736,"Exact original provider storage extent"); diff --git a/drivers/NativeMetal/bridge/FrontendChecks.h b/drivers/NativeMetal/bridge/FrontendChecks.h new file mode 100644 index 0000000..a3d4e3d --- /dev/null +++ b/drivers/NativeMetal/bridge/FrontendChecks.h @@ -0,0 +1,391 @@ +// SPDX-License-Identifier: MIT +// The native Metal frontend's qualification through the public Metal API only, shared by the host test (frontend_host.mm, over MoltenVK) and +// the plugin (NativeMetalBridgePhase.h, over Turnip on the Adreno 740). Every check reads back what the GPU wrote. Steps (markers 0x6601..): +// 1 compute; 2 render to a private texture with a blit readback; 3 render into a real IOSurface read by the CPU, 4 sampling it into a +// second surface (the composition shape); 5 a compute kernel reading and writing textures; 6 completion handler order; 7 an encoder refusal +// failing its command buffer; 8 the target's QuartzCore library (when a path is given); 9 function constants (when that fixture is given); +// 10 a shared event ordering two queues (GPU wait, GPU signal, CPU signal, listener notification) and a fence between two encoders. +#pragma once +#import +#import +#import +#include +#include +#include +#include + +namespace frontendchecks { +struct Report { + unsigned failed, last; + char first[96]; + unsigned qcFunctions, qcKernels, qcKernelsOk, qcPairs, qcPairsOk; + void (*mark)(uint32_t); + bool hostSimulation; // the host simulation's IOKit shim hides IOSurface (steps 3-4 skipped) and Metal's listener (none is made) +}; +static Report *gReport; +__attribute__((format(printf, 2, 3))) static void fail(Report *r, const char *format, ...) { + char text[160]; + va_list args; + va_start(args, format); + vsnprintf(text, sizeof text, format, args); + va_end(args); + std::printf("FAIL %s\n", text); + if (!r->failed++) snprintf(r->first, sizeof r->first, "%x %s", r->last, text); +} +static void step(Report *r, uint32_t marker) { + r->last = marker; + if (r->mark) r->mark(marker); +} +#define CHECK(c, ...) do { if (!(c)) fail(gReport, __VA_ARGS__); } while (0) +static bool completed(id cb, const char *what) { + [cb waitUntilCompleted]; + if (cb.status != MTLCommandBufferStatusCompleted) { + fail(gReport, "%s: status %lu %s", what, (unsigned long)cb.status, cb.error.localizedDescription.UTF8String ?: ""); + return false; + } + return true; +} +static bool finish(id cb, const char *what) { + [cb commit]; + return completed(cb, what); +} +// The fixture's picture: top-left quarter (255, 51, 0, 255), the rest cleared blue; `bgra` swaps red and blue in memory. +static unsigned wrongTexels(const uint8_t *bytes, size_t row, unsigned side, bool bgra = false) { + unsigned bad = 0; + for (unsigned y = 0; y < side; ++y) + for (unsigned x = 0; x < side; ++x) { + const bool inside = x < side / 2 && y < side / 2; + uint8_t e[4] = {inside ? (uint8_t)255 : (uint8_t)0, inside ? (uint8_t)51 : (uint8_t)0, inside ? (uint8_t)0 : (uint8_t)255, 255}; + if (bgra) std::swap(e[0], e[2]); + if (memcmp(bytes + y * row + x * 4, e, 4)) ++bad; + } + return bad; +} +static IOSurfaceRef surface(unsigned side, OSType format) { + NSDictionary *p = @{(id)kIOSurfaceWidth: @(side), (id)kIOSurfaceHeight: @(side), (id)kIOSurfaceBytesPerElement: @4, + (id)kIOSurfacePixelFormat: @(format)}; + return IOSurfaceCreate((__bridge CFDictionaryRef)p); +} +static id pipeline(id device, id library, NSString *fragment, MTLPixelFormat format) { + MTLRenderPipelineDescriptor *d = [MTLRenderPipelineDescriptor new]; + d.vertexFunction = [library newFunctionWithName:@"colored_vertex"]; + d.fragmentFunction = [library newFunctionWithName:fragment]; + d.colorAttachments[0].pixelFormat = format; + NSError *error = nil; + id state = [device newRenderPipelineStateWithDescriptor:d error:&error]; + CHECK(state, "render pipeline %s: %s", fragment.UTF8String, error.localizedDescription.UTF8String); + return state; +} +// Draws the fixture quad (vertex positions/colors at buffers 0/1) into `target`, cleared blue first. +static void drawQuad(id cb, id state, id target, id sampled, + const float (*positions)[2], unsigned count) { + MTLRenderPassDescriptor *pass = [MTLRenderPassDescriptor renderPassDescriptor]; + pass.colorAttachments[0].texture = target; + pass.colorAttachments[0].loadAction = MTLLoadActionClear; + pass.colorAttachments[0].storeAction = MTLStoreActionStore; + pass.colorAttachments[0].clearColor = MTLClearColorMake(0, 0, 1, 1); + id e = [cb renderCommandEncoderWithDescriptor:pass]; + [e setRenderPipelineState:state]; + [e setVertexBytes:positions length:sizeof(float) * 2 * count atIndex:0]; + std::vector colors; + for (unsigned i = 0; i < count; ++i) colors.insert(colors.end(), {1.0f, 0.2f, 0.0f, 1.0f}); + [e setVertexBytes:colors.data() length:colors.size() * sizeof(float) atIndex:1]; + if (sampled) [e setFragmentTexture:sampled atIndex:0]; + [e drawPrimitives:MTLPrimitiveTypeTriangle vertexStart:0 vertexCount:count]; + [e endEncoding]; +} +static const float kQuarter[6][2] = {{-1, 1}, {0, 1}, {-1, 0}, {0, 1}, {0, 0}, {-1, 0}}; +static const float kFull[6][2] = {{-1, 1}, {1, 1}, {-1, -1}, {1, 1}, {1, -1}, {-1, -1}}; + +// Runs every step on `device` (the frontend's); `library` is the bridge's runtime_examples fixture, `fcLibrary` (may be nil) the +// function-constant fixture, `qcPath` (may be nil) the target's QuartzCore default.metallib. +static void run(id device, id library, id fcLibrary, NSString *qcPath, Report *r) { + gReport = r; + @autoreleasepool { + CHECK([library.functionNames containsObject:@"sampled_fragment"], "function names"); + CHECK(![library newFunctionWithName:@"no_such_function"], "a missing function must be nil"); + id queue = [device newCommandQueue]; + CHECK(queue, "command queue"); + if (!queue) return; + + step(r, 0x6601); + // 1. compute: out[i] = 3 * in[i] (the fixture's scale_and_offset with scale 2) + { + NSError *error = nil; + id f = [library newFunctionWithName:@"scale_and_offset"]; + CHECK(f.functionType == MTLFunctionTypeKernel, "kernel type %lu", (unsigned long)f.functionType); + id state = [device newComputePipelineStateWithFunction:f error:&error]; + CHECK(state, "compute pipeline: %s", error.localizedDescription.UTF8String); + std::vector in(1000); + for (unsigned i = 0; i < in.size(); ++i) in[i] = (float)i; + id input = [device newBufferWithBytes:in.data() length:in.size() * 4 options:MTLResourceStorageModeShared]; + id output = [device newBufferWithLength:in.size() * 4 options:MTLResourceStorageModeShared]; + const float scale = 2; + id cb = [queue commandBuffer]; + id e = [cb computeCommandEncoder]; + [e setComputePipelineState:state]; + [e setBuffer:input offset:0 atIndex:0]; + [e setBuffer:output offset:0 atIndex:1]; + [e setBytes:&scale length:4 atIndex:2]; + [e dispatchThreads:MTLSizeMake(in.size(), 1, 1) threadsPerThreadgroup:MTLSizeMake(64, 1, 1)]; + [e endEncoding]; + if (finish(cb, "compute")) { + const float *v = (const float *)output.contents; + unsigned bad = 0; + for (unsigned i = 0; i < in.size(); ++i) bad += v[i] != 3.0f * i; + CHECK(!bad, "compute: %u wrong outputs", bad); + } + } + id colored = pipeline(device, library, @"colored_fragment", MTLPixelFormatRGBA8Unorm); + step(r, 0x6602); + // 2. render into a private texture, blit to a buffer + { + MTLTextureDescriptor *d = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatRGBA8Unorm width:8 height:8 mipmapped:NO]; + d.usage = MTLTextureUsageRenderTarget; + d.storageMode = MTLStorageModePrivate; + id target = [device newTextureWithDescriptor:d]; + CHECK(target && target.width == 8 && target.pixelFormat == MTLPixelFormatRGBA8Unorm, "private texture"); + id readback = [device newBufferWithLength:8 * 8 * 4 options:MTLResourceStorageModeShared]; + id cb = [queue commandBuffer]; + drawQuad(cb, colored, target, nil, kQuarter, 6); + id b = [cb blitCommandEncoder]; + [b copyFromTexture:target sourceSlice:0 sourceLevel:0 sourceOrigin:MTLOriginMake(0, 0, 0) sourceSize:MTLSizeMake(8, 8, 1) + toBuffer:readback destinationOffset:0 destinationBytesPerRow:32 destinationBytesPerImage:256]; + [b endEncoding]; + if (finish(cb, "render")) CHECK(!wrongTexels((const uint8_t *)readback.contents, 32, 8), "render: wrong texels"); + } + step(r, 0x6603); + // 3. render into an IOSurface (its pages imported, no copy), then 4. sample it into a second surface + IOSurfaceRef layer = r->hostSimulation ? nullptr : surface(64, 'RGBA'), screen = r->hostSimulation ? nullptr : surface(64, 'RGBA'); + CHECK(r->hostSimulation || (layer && screen), "IOSurfaceCreate"); + if (layer && screen) { + MTLTextureDescriptor *d = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatRGBA8Unorm width:64 height:64 mipmapped:NO]; + d.usage = MTLTextureUsageRenderTarget | MTLTextureUsageShaderRead; + d.storageMode = MTLStorageModeShared; + id layerTexture = [device newTextureWithDescriptor:d iosurface:layer plane:0]; + id screenTexture = [device newTextureWithDescriptor:d iosurface:screen plane:0]; + CHECK(layerTexture && screenTexture && layerTexture.iosurface == layer, "IOSurface textures"); + id cb = [queue commandBuffer]; + drawQuad(cb, colored, layerTexture, nil, kQuarter, 6); + if (finish(cb, "surface render")) { + IOSurfaceLock(layer, kIOSurfaceLockReadOnly, nullptr); + CHECK(!wrongTexels((const uint8_t *)IOSurfaceGetBaseAddress(layer), IOSurfaceGetBytesPerRow(layer), 64), "surface render: wrong texels"); + IOSurfaceUnlock(layer, kIOSurfaceLockReadOnly, nullptr); + } + step(r, 0x6604); + // sampled_fragment samples its texture at position * 1/8 (normalized), so an 8x8 viewport of the screen reads the layer whole + id sampled = pipeline(device, library, @"sampled_fragment", MTLPixelFormatRGBA8Unorm); + cb = [queue commandBuffer]; + MTLRenderPassDescriptor *pass = [MTLRenderPassDescriptor renderPassDescriptor]; + pass.colorAttachments[0].texture = screenTexture; + pass.colorAttachments[0].loadAction = MTLLoadActionClear; + pass.colorAttachments[0].storeAction = MTLStoreActionStore; + pass.colorAttachments[0].clearColor = MTLClearColorMake(0, 1, 0, 1); + id e = [cb renderCommandEncoderWithDescriptor:pass]; + [e setRenderPipelineState:sampled]; + [e setViewport:(MTLViewport){0, 0, 8, 8, 0, 1}]; + [e setVertexBytes:kFull length:sizeof kFull atIndex:0]; + float colors[6][4] = {}; + [e setVertexBytes:colors length:sizeof colors atIndex:1]; + [e setFragmentTexture:layerTexture atIndex:0]; + [e drawPrimitives:MTLPrimitiveTypeTriangle vertexStart:0 vertexCount:6]; + [e endEncoding]; + if (finish(cb, "composition")) { + IOSurfaceLock(screen, kIOSurfaceLockReadOnly, nullptr); + const uint8_t *p = (const uint8_t *)IOSurfaceGetBaseAddress(screen); + std::vector corner(8 * 8 * 4); + for (unsigned y = 0; y < 8; ++y) memcpy(&corner[y * 32], p + y * IOSurfaceGetBytesPerRow(screen), 32); + const unsigned bad = wrongTexels(corner.data(), 32, 8); + const uint8_t *outside = p + 20 * IOSurfaceGetBytesPerRow(screen) + 20 * 4; + CHECK(!bad, "composition: %u wrong texels in the sampled corner", bad); + CHECK(outside[0] == 0 && outside[1] == 255 && outside[2] == 0, "composition: the rest keeps its clear color"); + IOSurfaceUnlock(screen, kIOSurfaceLockReadOnly, nullptr); + } + } + step(r, 0x6605); + // 5. compute over textures: invert_texels writes 1 - rgb of its input texel + { + MTLTextureDescriptor *d = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatRGBA8Unorm width:16 height:16 mipmapped:NO]; + d.usage = MTLTextureUsageShaderRead | MTLTextureUsageShaderWrite; + id src = [device newTextureWithDescriptor:d], dst = [device newTextureWithDescriptor:d]; + std::vector texels(16 * 16 * 4); + for (size_t i = 0; i < texels.size(); ++i) texels[i] = (uint8_t)(i * 7); + [src replaceRegion:MTLRegionMake2D(0, 0, 16, 16) mipmapLevel:0 withBytes:texels.data() bytesPerRow:64]; + NSError *error = nil; + id state = [device newComputePipelineStateWithFunction:[library newFunctionWithName:@"invert_texels"] error:&error]; + CHECK(state, "invert pipeline: %s", error.localizedDescription.UTF8String); + id cb = [queue commandBuffer]; + id e = [cb computeCommandEncoder]; + [e setComputePipelineState:state]; + [e setTexture:src atIndex:0]; + [e setTexture:dst atIndex:1]; + [e dispatchThreads:MTLSizeMake(16, 16, 1) threadsPerThreadgroup:MTLSizeMake(8, 8, 1)]; + [e endEncoding]; + if (finish(cb, "texture compute")) { + std::vector out(texels.size()); + [dst getBytes:out.data() bytesPerRow:64 fromRegion:MTLRegionMake2D(0, 0, 16, 16) mipmapLevel:0]; + unsigned bad = 0; + for (size_t i = 0; i < out.size(); ++i) { + const int expected = i % 4 == 3 ? texels[i] : 255 - texels[i]; + bad += abs(out[i] - expected) > 1; + } + CHECK(!bad, "texture compute: %u wrong channels", bad); + } + } + step(r, 0x6606); + // 6. handlers: scheduled before completed, each once, after the GPU finished + { + __block int order = 0, scheduled = -1, done = -1; + id cb = [queue commandBuffer]; + id b = [cb blitCommandEncoder]; + id buffer = [device newBufferWithLength:4096 options:MTLResourceStorageModeShared]; + [b fillBuffer:buffer range:NSMakeRange(0, 4096) value:0xa5]; + [b endEncoding]; + [cb addScheduledHandler:^(id) {scheduled = order++;}]; + __block MTLCommandBufferStatus seen = MTLCommandBufferStatusNotEnqueued; + [cb addCompletedHandler:^(id c) {seen = c.status; done = order++;}]; + if (finish(cb, "handlers")) { + CHECK(scheduled == 0 && done == 1 && order == 2, "handler order %d %d", scheduled, done); + CHECK(seen == MTLCommandBufferStatusCompleted, "status inside the completed handler"); + const uint8_t *p = (const uint8_t *)buffer.contents; + CHECK(p[0] == 0xa5 && p[4095] == 0xa5, "fill"); + } + } + step(r, 0x6607); + // 7. a refused encoder call fails its command buffer (no silent drop), and the handlers still run + { + id cb = [queue commandBuffer]; + id e = nil; + MTLTextureDescriptor *d = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatRGBA8Unorm width:8 height:8 mipmapped:NO]; + d.usage = MTLTextureUsageRenderTarget; + MTLRenderPassDescriptor *pass = [MTLRenderPassDescriptor renderPassDescriptor]; + pass.colorAttachments[0].texture = [device newTextureWithDescriptor:d]; + e = [cb renderCommandEncoderWithDescriptor:pass]; + [e drawPrimitives:MTLPrimitiveTypeTriangle vertexStart:0 vertexCount:3]; // no pipeline set + [e endEncoding]; + __block bool ran = false; + [cb addCompletedHandler:^(id) {ran = true;}]; + [cb commit]; + [cb waitUntilCompleted]; + CHECK(cb.status == MTLCommandBufferStatusError && cb.error, "a draw without a pipeline must fail the command buffer"); + CHECK(ran, "the completed handler runs for a failed command buffer"); + } + step(r, 0x6609); + // 9. function constants (newFunctionWithName:constantValues:error:): the runtime's fixture kernel_function_constant_gate_fold, whose + // out[24] is in * 1000 * mode for mode 1..3 and in otherwise. + if (fcLibrary) { + for (int mode : {0, 2, 3}) { + MTLFunctionConstantValues *values = [MTLFunctionConstantValues new]; + [values setConstantValue:&mode type:MTLDataTypeInt atIndex:0]; + NSError *error = nil; + id f = [fcLibrary newFunctionWithName:@"fcfold" constantValues:values error:&error]; + CHECK(f, "specialized function: %s", error.localizedDescription.UTF8String); + id state = f ? [device newComputePipelineStateWithFunction:f error:&error] : nil; + CHECK(state, "specialized pipeline: %s", error.localizedDescription.UTF8String); + if (!state) continue; + const float in = 2; + id input = [device newBufferWithBytes:&in length:4 options:MTLResourceStorageModeShared]; + id output = [device newBufferWithLength:32 * 4 options:MTLResourceStorageModeShared]; + id cb = [queue commandBuffer]; + id e = [cb computeCommandEncoder]; + [e setComputePipelineState:state]; + [e setBuffer:input offset:0 atIndex:0]; + [e setBuffer:output offset:0 atIndex:1]; + [e dispatchThreads:MTLSizeMake(1, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)]; + [e endEncoding]; + if (finish(cb, "specialized compute")) { + const float got = ((const float *)output.contents)[24], expected = mode ? in * 1000 * mode : in; + CHECK(got == expected, "mode %d: out[24] %g, expected %g", mode, got, expected); + } + } + } + step(r, 0x6608); + // 8. the target's own QuartzCore library through newLibraryWithURL: (a universal metallib; its AIR64 slice): every kernel into a + // compute pipeline and every _vert_

/ _frag_

pair into a BGRA8 render pipeline. Refusals are counted, not hidden. + if (qcPath) { + NSError *error = nil; + id qc = [device newLibraryWithURL:[NSURL fileURLWithPath:qcPath] error:&error]; + CHECK(qc, "QuartzCore library: %s", error.localizedDescription.UTF8String); + unsigned kernels = 0, kernelsOk = 0, pairs = 0, pairsOk = 0; + NSArray *names = qc.functionNames; + for (NSString *name in names) { + id f = [qc newFunctionWithName:name]; + if (f.functionType == MTLFunctionTypeKernel) { + ++kernels; + kernelsOk += [device newComputePipelineStateWithFunction:f error:nil] != nil; + } else if (f.functionType == MTLFunctionTypeVertex) { + NSString *fragment = [name stringByReplacingOccurrencesOfString:@"_vert_" withString:@"_frag_"]; + if ([fragment isEqualToString:name] || ![names containsObject:fragment]) continue; + ++pairs; + MTLRenderPipelineDescriptor *d = [MTLRenderPipelineDescriptor new]; + d.vertexFunction = f; + d.fragmentFunction = [qc newFunctionWithName:fragment]; + d.colorAttachments[0].pixelFormat = MTLPixelFormatBGRA8Unorm; + pairsOk += [device newRenderPipelineStateWithDescriptor:d error:nil] != nil; + } + } + CHECK(names.count && kernelsOk + pairsOk, "QuartzCore: nothing translated"); + r->qcFunctions = (unsigned)names.count; r->qcKernels = kernels; r->qcKernelsOk = kernelsOk; r->qcPairs = pairs; r->qcPairsOk = pairsOk; + } + step(r, 0x660a); + // 10. A second queue's command buffer waits for value 1, then copies; the first queue fills and signals 1. The waiting one is committed + // first, so its copy only sees the fill if the GPU wait held. A third waits for 3, which only the CPU signals. Then a fence orders two + // blit encoders of one command buffer, and the listener is notified at 3. + { + id event = [device newSharedEvent]; + id fence = [device newFence]; + id other = [device newCommandQueue]; + CHECK(event && fence && other, "event, fence, second queue"); + id a = [device newBufferWithLength:4096 options:MTLResourceStorageModeShared]; + id b = [device newBufferWithLength:4096 options:MTLResourceStorageModeShared]; + id c = [device newBufferWithLength:4096 options:MTLResourceStorageModeShared]; + if (event && fence && other) { + id waiting = [other commandBuffer]; + [waiting encodeWaitForEvent:event value:1]; + id copy = [waiting blitCommandEncoder]; + [copy copyFromBuffer:a sourceOffset:0 toBuffer:b destinationOffset:0 size:4096]; + [copy endEncoding]; + [waiting commit]; + id signalling = [queue commandBuffer]; + id fill = [signalling blitCommandEncoder]; + [fill fillBuffer:a range:NSMakeRange(0, 4096) value:0x3c]; + [fill endEncoding]; + [signalling encodeSignalEvent:event value:1]; + id cpu = [other commandBuffer]; + [cpu encodeWaitForEvent:event value:3]; + id second = [cpu blitCommandEncoder]; + [second fillBuffer:c range:NSMakeRange(0, 64) value:0x77]; + [second updateFence:fence]; + [second endEncoding]; + id third = [cpu blitCommandEncoder]; + [third waitForFence:fence]; + [third copyFromBuffer:c sourceOffset:0 toBuffer:c destinationOffset:2048 size:64]; + [third endEncoding]; + [cpu commit]; + if (finish(signalling, "event signal") && completed(waiting, "event wait")) { + CHECK(((const uint8_t *)b.contents)[4095] == 0x3c, "the GPU wait did not order the copy after the fill"); + CHECK(event.signaledValue == 1, "signaled value %llu", (unsigned long long)event.signaledValue); + } + CHECK(cpu.status != MTLCommandBufferStatusCompleted, "a wait for 3 completed before 3 was signaled"); + __block uint64_t heard = 0; + dispatch_semaphore_t notified = dispatch_semaphore_create(0); + MTLSharedEventListener *listener = r->hostSimulation ? nil : [[MTLSharedEventListener alloc] init]; + [event notifyListener:listener atValue:3 block:^(id, uint64_t value) {heard = value; dispatch_semaphore_signal(notified);}]; + event.signaledValue = 3; + if (completed(cpu, "CPU-signaled wait")) { + const uint8_t *p = (const uint8_t *)c.contents; + CHECK(p[2048] == 0x77 && p[2111] == 0x77, "the fence did not order the copy after the fill"); + } + CHECK(!dispatch_semaphore_wait(notified, dispatch_time(DISPATCH_TIME_NOW, 5 * NSEC_PER_SEC)) && heard >= 3, + "listener notification %llu", (unsigned long long)heard); + CHECK([event waitUntilSignaledValue:3 timeoutMS:10], "waitUntilSignaledValue"); + } + } + if (layer) CFRelease(layer); + if (screen) CFRelease(screen); + } + step(r, 0x660f); + gReport = nullptr; +} +#undef CHECK +} // namespace frontendchecks diff --git a/drivers/NativeMetal/bridge/NativeMetalBridge.h b/drivers/NativeMetal/bridge/NativeMetalBridge.h new file mode 100644 index 0000000..7d6bbf0 --- /dev/null +++ b/drivers/NativeMetal/bridge/NativeMetalBridge.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: MIT */ +/* C ABI of the native Metal bridge (NativeMetalBridge.rs): the metal2vulkan runtime over Turnip. */ +#ifndef NATIVE_METAL_BRIDGE_H +#define NATIVE_METAL_BRIDGE_H +#include +#include +#ifdef __cplusplus +extern "C" { +#endif +struct NmbSelftest { + uint32_t stage; /* 0 when every step ran, else the failing step's marker (0x6501..0x6506) */ + uint32_t compute_bad; /* wrong outputs of 1000 */ + uint32_t render_bad; /* wrong texels of 64 (8x8) */ + uint32_t import_bad; /* wrong texels of 4096 (64x64 over imported host pages) */ + uint32_t qc_functions; + uint32_t qc_kernels_ok, qc_kernels_failed; + uint32_t qc_pairs_ok, qc_pairs_failed; + uint32_t reserved; + char message[96]; +}; +/* Markers: 0x6501 device, 0x6502 fixture library, 0x6503 compute, 0x6504 render, 0x6505 import, 0x6506 QuartzCore, 0x650f done. */ +uint32_t nmb_selftest(const char *loader, const char *cache_dir, const uint8_t *quartzcore, size_t quartzcore_bytes, + void (*mark)(uint32_t), struct NmbSelftest *out); +#ifdef __cplusplus +} +#endif +#endif diff --git a/drivers/NativeMetal/bridge/NativeMetalBridge.rs b/drivers/NativeMetal/bridge/NativeMetalBridge.rs new file mode 100644 index 0000000..40a753f --- /dev/null +++ b/drivers/NativeMetal/bridge/NativeMetalBridge.rs @@ -0,0 +1,363 @@ +// SPDX-License-Identifier: MIT +//! Native Metal bridge: the metal2vulkan runtime (the Metal API on Vulkan) over Turnip, behind a C ABI the Objective-C +//! Metal frontend and the plugin's self-test call. Turnip is opened as the runtime's Vulkan "loader" (it exports +//! vkGetInstanceProcAddr; iPadOS has no Khronos loader). The pinned runtime is GPL-2.0-or-later OR LGPL-3.0-or-later and is +//! linked unmodified from its own source; nothing of it is copied here. +//! +//! The self-test runs real Metal-level work and checks the GPU's results: a compute kernel, a render pass, a render into +//! host pages imported without a copy (the IOSurface path), and translation of the target's own QuartzCore library into +//! A740 pipelines. Each step reports a marker before it starts so a hang names the step. + +#[path = "NativeMetalObjects.rs"] +pub mod objects; + +use metal2vulkan::metal::{PixelFormat, PrimitiveType, TextureUsage}; +use metal2vulkan::pipeline::{ColorAttachmentDescriptor, RenderPipelineDescriptor, StageSource}; +use metal2vulkan_runtime::{ + BlitOption, ClearColor, CommandBuffer, CommandBufferStatus, Device, DeviceOptions, HostMemory, LoadAction, MetalLevel, + Origin, PipelineOptions, RenderPassDescriptor, ResourceOptions, Size, StoreAction, TextureDescriptor, +}; +use std::ffi::{c_char, c_void, CStr}; +use std::path::PathBuf; +use std::ptr::NonNull; +use std::time::Duration; + +/// The runtime's own example library (validation/fixtures/public/examples/runtime_examples.ll), assembled to AIR at build time. +static FIXTURE_AIR: &[u8] = include_bytes!(env!("NMB_FIXTURE_AIR")); + +unsafe extern "C" { + /// libkern (libSystem): clean and invalidate the data cache over a range, from user space. + fn sys_dcache_flush(start: *mut c_void, len: usize); +} + +/// Result of nmb_selftest; mirrors NativeMetalBridge.h. +#[repr(C)] +pub struct NmbSelftest { + /// 0 when every step ran; otherwise the marker of the step that failed. + pub stage: u32, + pub compute_bad: u32, + pub render_bad: u32, + pub import_bad: u32, + pub qc_functions: u32, + pub qc_kernels_ok: u32, + pub qc_kernels_failed: u32, + pub qc_pairs_ok: u32, + pub qc_pairs_failed: u32, + pub reserved: u32, + pub message: [u8; 96], +} + +type Mark = Option; + +struct Run<'a> { + out: &'a mut NmbSelftest, + mark: Mark, +} + +impl Run<'_> { + fn step(&mut self, marker: u32) { + if let Some(mark) = self.mark { + // SAFETY: the caller's callback, valid for the call. + unsafe { mark(marker) }; + } + } + fn fail(&mut self, marker: u32, why: impl std::fmt::Display) -> u32 { + self.out.stage = marker; + let text = format!("{why}"); + let n = text.len().min(self.out.message.len() - 1); + self.out.message[..n].copy_from_slice(&text.as_bytes()[..n]); + self.out.message[n] = 0; + marker + } +} + +const WAIT: Duration = Duration::from_secs(10); + +fn finish(command_buffer: &CommandBuffer) -> Result<(), String> { + command_buffer.commit().map_err(|e| format!("commit: {e:?}"))?; + command_buffer.wait_until_completed_for(WAIT).map_err(|_| "no completion in 10 s".to_string())?; + match command_buffer.status() { + CommandBufferStatus::Completed => Ok(()), + status => Err(format!("{status:?} {:?}", command_buffer.error())), + } +} + +fn open(loader: &CStr, cache: &CStr) -> Result { + let mut options = DeviceOptions::default(); + options.loader = Some(PathBuf::from(loader.to_string_lossy().into_owned())); + // No macOS release names this host; answer as the newest measured Metal. + options.metal_level = Some(MetalLevel::NEWEST); + options.shader_cache = metal2vulkan_runtime::ShaderCacheOptions::in_directory(cache.to_string_lossy().into_owned()); + Device::system_default_with(&options).map_err(|e| format!("device: {e:?}")) +} + +fn compute(device: &Device, library: &metal2vulkan_runtime::Library) -> Result { + const COUNT: usize = 1000; + let function = library.new_function_with_name("scale_and_offset").map_err(|e| format!("{e:?}"))?; + let state = device + .new_compute_pipeline_state_with_function(&function, &PipelineOptions::default()) + .map_err(|e| format!("pipeline: {e:?}"))?; + let input: Vec = (0..COUNT).flat_map(|i| (i as f32).to_le_bytes()).collect(); + let input = device.new_buffer_with_bytes(&input, ResourceOptions::shared()).map_err(|e| format!("{e:?}"))?; + let output = device.new_buffer((COUNT * 4) as u64, ResourceOptions::shared()).map_err(|e| format!("{e:?}"))?; + let scale = device.new_buffer_with_bytes(&2.0f32.to_le_bytes(), ResourceOptions::shared()).map_err(|e| format!("{e:?}"))?; + let queue = device.new_command_queue().map_err(|e| format!("{e:?}"))?; + let command_buffer = queue.command_buffer().map_err(|e| format!("{e:?}"))?; + let mut encoder = command_buffer.compute_command_encoder().map_err(|e| format!("{e:?}"))?; + encoder.set_compute_pipeline_state(&state).map_err(|e| format!("{e:?}"))?; + encoder.set_buffer(Some(&input), 0, 0).map_err(|e| format!("{e:?}"))?; + encoder.set_buffer(Some(&output), 0, 1).map_err(|e| format!("{e:?}"))?; + encoder.set_buffer(Some(&scale), 0, 2).map_err(|e| format!("{e:?}"))?; + encoder.dispatch_threads(Size::new(COUNT as u64, 1, 1), Size::new(64, 1, 1)).map_err(|e| format!("{e:?}"))?; + encoder.end_encoding().map_err(|e| format!("{e:?}"))?; + finish(&command_buffer)?; + let contents = output.contents().ok_or("no contents")?; + // SAFETY: the buffer holds COUNT floats and no command buffer writes it now. + let values = unsafe { std::slice::from_raw_parts(contents.as_ptr() as *const f32, COUNT) }; + Ok(values.iter().enumerate().filter(|(i, v)| **v != 3.0 * *i as f32).count() as u32) +} + +/// Draws the fixture's top-left-quarter quad into `target` (cleared blue) and returns the encoded command buffer's result. +fn draw_quad(device: &Device, library: &metal2vulkan_runtime::Library, target: &metal2vulkan_runtime::Texture, + readback: Option<(&metal2vulkan_runtime::Buffer, u64, u64)>) -> Result<(), String> { + let vertex = library.new_function_with_name("colored_vertex").map_err(|e| format!("{e:?}"))?; + let fragment = library.new_function_with_name("colored_fragment").map_err(|e| format!("{e:?}"))?; + let mut descriptor = RenderPipelineDescriptor::new(StageSource::Function(vertex.function_descriptor())); + descriptor.fragment = Some(StageSource::Function(fragment.function_descriptor())); + descriptor.color_attachments[0] = Some(ColorAttachmentDescriptor::new(PixelFormat::RGBA8Unorm)); + let state = device.new_render_pipeline_state(&descriptor, &PipelineOptions::default()).map_err(|e| format!("pipeline: {e:?}"))?; + let positions: [[f32; 2]; 6] = [[-1.0, 1.0], [0.0, 1.0], [-1.0, 0.0], [0.0, 1.0], [0.0, 0.0], [-1.0, 0.0]]; + let positions: Vec = positions.iter().flatten().flat_map(|v| v.to_le_bytes()).collect(); + let colors: Vec = (0..6).flat_map(|_| [1.0f32, 0.2, 0.0, 1.0]).flat_map(|v| v.to_le_bytes()).collect(); + let positions = device.new_buffer_with_bytes(&positions, ResourceOptions::shared()).map_err(|e| format!("{e:?}"))?; + let colors = device.new_buffer_with_bytes(&colors, ResourceOptions::shared()).map_err(|e| format!("{e:?}"))?; + let mut pass = RenderPassDescriptor::new(); + let color = &mut pass.color_attachments[0]; + color.texture = Some(target.clone()); + color.load_action = LoadAction::Clear; + color.store_action = StoreAction::Store; + color.clear_color = ClearColor::new(0.0, 0.0, 1.0, 1.0); + let queue = device.new_command_queue().map_err(|e| format!("{e:?}"))?; + let command_buffer = queue.command_buffer().map_err(|e| format!("{e:?}"))?; + let mut encoder = command_buffer.render_command_encoder(&pass).map_err(|e| format!("{e:?}"))?; + encoder.set_render_pipeline_state(&state).map_err(|e| format!("{e:?}"))?; + encoder.set_vertex_buffer(Some(&positions), 0, 0).map_err(|e| format!("{e:?}"))?; + encoder.set_vertex_buffer(Some(&colors), 0, 1).map_err(|e| format!("{e:?}"))?; + encoder.draw_primitives(PrimitiveType::Triangle, 0, 6, 1, 0).map_err(|e| format!("{e:?}"))?; + encoder.end_encoding().map_err(|e| format!("{e:?}"))?; + if let Some((buffer, row, side)) = readback { + let mut blit = command_buffer.blit_command_encoder().map_err(|e| format!("{e:?}"))?; + blit.copy_from_texture_to_buffer(target, 0, 0, Origin::new(0, 0, 0), Size::new(side, side, 1), buffer, 0, row, + row * side, BlitOption::None).map_err(|e| format!("{e:?}"))?; + blit.end_encoding().map_err(|e| format!("{e:?}"))?; + } + finish(&command_buffer) +} + +/// Texels off the expected picture: the top-left quarter drawn (255, 51, 0, 255), the rest cleared blue. +fn wrong_texels(bytes: &[u8], row: usize, side: usize) -> u32 { + let mut bad = 0; + for y in 0..side { + for x in 0..side { + let at = y * row + x * 4; + let inside = x < side / 2 && y < side / 2; + let expected: [u8; 4] = if inside { [255, 51, 0, 255] } else { [0, 0, 255, 255] }; + if bytes[at..at + 4] != expected { + bad += 1; + } + } + } + bad +} + +fn render(device: &Device, library: &metal2vulkan_runtime::Library) -> Result { + const SIDE: u64 = 8; + let mut texture = TextureDescriptor::texture_2d(PixelFormat::RGBA8Unorm, SIDE, SIDE, false); + texture.usage = TextureUsage::RENDER_TARGET; + texture.resource_options = ResourceOptions::private(); + let target = device.new_texture(&texture).map_err(|e| format!("{e:?}"))?; + let readback = device.new_buffer(SIDE * 4 * SIDE, ResourceOptions::shared()).map_err(|e| format!("{e:?}"))?; + draw_quad(device, library, &target, Some((&readback, SIDE * 4, SIDE)))?; + let contents = readback.contents().ok_or("no contents")?; + // SAFETY: SIDE rows of SIDE * 4 bytes, no longer written. + let bytes = unsafe { std::slice::from_raw_parts(contents.as_ptr(), (SIDE * 4 * SIDE) as usize) }; + Ok(wrong_texels(bytes, (SIDE * 4) as usize, SIDE as usize)) +} + +/// The IOSurface path without an IOSurface: page-aligned host pages imported without a copy (the kernel pins them, Turnip maps +/// them into the GPU), a linear render target over them, CPU caches cleaned before and invalidated after the GPU's use. +fn import(device: &Device, library: &metal2vulkan_runtime::Library) -> Result { + const SIDE: u64 = 64; + const ROW: u64 = 256; + let bytes = (ROW * SIDE) as usize; + // SAFETY: an anonymous private mapping of whole pages, unmapped below after every GPU use completed. + let pages = unsafe { + libc::mmap(std::ptr::null_mut(), bytes, libc::PROT_READ | libc::PROT_WRITE, libc::MAP_PRIVATE | libc::MAP_ANON, -1, 0) + }; + if pages == libc::MAP_FAILED { + return Err("mmap".into()); + } + let result = (|| { + // SAFETY: the pages were just mapped; nothing else uses them. + unsafe { std::ptr::write_bytes(pages as *mut u8, 0x5a, bytes) }; + // SAFETY: the mapping is writable, page aligned and outlives the buffer (dropped before the unmap). + let memory = unsafe { HostMemory::new(NonNull::new(pages as *mut u8).ok_or("null")?) }; + // SAFETY: as above, for `bytes` bytes. + let buffer = unsafe { device.new_buffer_with_bytes_no_copy(memory, bytes as u64, ResourceOptions::shared(), None) } + .map_err(|e| format!("no-copy buffer: {e:?}"))?; + let mut texture = TextureDescriptor::texture_2d(PixelFormat::RGBA8Unorm, SIDE, SIDE, false); + texture.usage = TextureUsage::RENDER_TARGET | TextureUsage::SHADER_READ; + texture.resource_options = ResourceOptions::shared(); + let target = buffer.new_texture(&texture, 0, ROW).map_err(|e| format!("texture over pages: {e:?}"))?; + // SAFETY: the range is this mapping. + unsafe { sys_dcache_flush(pages, bytes) }; + draw_quad(device, library, &target, None)?; + // SAFETY: the GPU finished writing; drop any line the CPU fetched meanwhile. + unsafe { sys_dcache_flush(pages, bytes) }; + // SAFETY: the pages hold `bytes` bytes and no command buffer writes them now. + let view = unsafe { std::slice::from_raw_parts(pages as *const u8, bytes) }; + let bad = wrong_texels(view, ROW as usize, SIDE as usize); + drop(target); + drop(buffer); + Ok(bad) + })(); + // SAFETY: the buffer and texture over the pages are gone. + unsafe { libc::munmap(pages, bytes) }; + result +} + +/// The AIR slice of a universal metallib (a fat container, big endian, of an AIR64 metallib and an Apple-GPU binary, as the target's +/// frameworks ship them), as Metal itself picks it; a thin metallib is returned as it is. +pub fn air_slice(bytes: &[u8]) -> Option<&[u8]> { + const AIR64: u32 = 0x0100_0017; + let word = |at: usize| bytes.get(at..at + 4).map(|w| u32::from_be_bytes([w[0], w[1], w[2], w[3]])); + if word(0)? != 0xcafe_babe { + return Some(bytes); + } + let count = word(4)? as usize; + for i in 0..count.min(16) { + let at = 8 + 20 * i; + if word(at)? == AIR64 { + let (offset, size) = (word(at + 8)? as usize, word(at + 12)? as usize); + return bytes.get(offset..offset.checked_add(size)?); + } + } + None +} + +/// Translates the target's own QuartzCore library: every kernel into a compute pipeline, and the vertex/fragment pairs named +/// alike into render pipelines over a BGRA8 attachment. Counts only; refusals (function constants, tile functions) are expected. +fn quartzcore(run: &mut Run, device: &Device, bytes: &[u8]) -> Result<(), String> { + let bytes = air_slice(bytes).ok_or("QC library: no AIR64 slice")?; + let library = device.new_library_with_data(bytes).map_err(|e| format!("QC library: {e:?}"))?; + let names = library.function_names(); + let list = std::env::var_os("NMB_LIST").is_some(); + run.out.qc_functions = names.len() as u32; + let mut vertices = Vec::new(); + let mut fragments = Vec::new(); + for name in &names { + let Ok(function) = library.new_function_with_name(name) else { continue }; + if list { + eprintln!("fn {name} {:?}", function.function_type()); + } + match function.function_type() { + Ok(metal2vulkan::metal::FunctionType::Kernel) => { + match device.new_compute_pipeline_state_with_function(&function, &PipelineOptions::default()) { + Ok(_) => run.out.qc_kernels_ok += 1, + Err(e) => { + run.out.qc_kernels_failed += 1; + if list { + eprintln!("kernel {name}: {e:?}"); + } + } + } + } + Ok(metal2vulkan::metal::FunctionType::Vertex) => vertices.push((function, name.clone())), + Ok(metal2vulkan::metal::FunctionType::Fragment) => fragments.push((function, name.clone())), + _ => {} + } + } + // The library's own pairs: `_vert_` with `_frag_`. + for (vertex, vname) in &vertices { + let Some(fname) = [vname.replace("_vert_", "_frag_"), vname.replacen("_vert", "_frag", 1)] + .into_iter() + .find(|candidate| fragments.iter().any(|(_, n)| n == candidate)) + else { + continue; + }; + let fragment = &fragments.iter().find(|(_, n)| *n == fname).unwrap().0; + let mut descriptor = RenderPipelineDescriptor::new(StageSource::Function(vertex.function_descriptor())); + descriptor.fragment = Some(StageSource::Function(fragment.function_descriptor())); + descriptor.color_attachments[0] = Some(ColorAttachmentDescriptor::new(PixelFormat::BGRA8Unorm)); + match device.new_render_pipeline_state(&descriptor, &PipelineOptions::default()) { + Ok(_) => run.out.qc_pairs_ok += 1, + Err(e) => { + run.out.qc_pairs_failed += 1; + if list { + eprintln!("pair {vname} {fname}: {e:?}"); + } + } + } + } + Ok(()) +} + +fn selftest(run: &mut Run, loader: &CStr, cache: &CStr, qc: &[u8]) -> u32 { + run.step(0x6501); + let device = match open(loader, cache) { + Ok(device) => device, + Err(why) => return run.fail(0x6501, why), + }; + run.step(0x6502); + let library = match metal2vulkan::library::Library::from_air(FIXTURE_AIR) { + Ok(library) => device.new_library_with_compile_library(library), + Err(why) => return run.fail(0x6502, format!("{why:?}")), + }; + run.step(0x6503); + match compute(&device, &library) { + Ok(bad) => run.out.compute_bad = bad, + Err(why) => return run.fail(0x6503, why), + } + run.step(0x6504); + match render(&device, &library) { + Ok(bad) => run.out.render_bad = bad, + Err(why) => return run.fail(0x6504, why), + } + run.step(0x6505); + match import(&device, &library) { + Ok(bad) => run.out.import_bad = bad, + Err(why) => return run.fail(0x6505, why), + } + if !qc.is_empty() { + run.step(0x6506); + if let Err(why) = quartzcore(run, &device, qc) { + return run.fail(0x6506, why); + } + } + run.step(0x650f); + 0 +} + +/// Runs the self-test; returns 0 or the marker of the failed step (also in out->stage, with out->message). +/// +/// # Safety +/// `loader` and `cache` are C strings; `qc` points at `qc_len` bytes (or is null with 0); `out` is writable. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn nmb_selftest(loader: *const c_char, cache: *const c_char, qc: *const u8, qc_len: usize, mark: Mark, + out: *mut NmbSelftest) -> u32 { + if loader.is_null() || cache.is_null() || out.is_null() { + return 0x65ff; + } + // SAFETY: the caller's contract. + let out = unsafe { &mut *out }; + // SAFETY: as above. + let (loader, cache) = unsafe { (CStr::from_ptr(loader), CStr::from_ptr(cache)) }; + let qc: &[u8] = if qc.is_null() || qc_len == 0 { &[] } else { unsafe { std::slice::from_raw_parts(qc, qc_len) } }; + // SAFETY: plain integers and bytes. + *out = unsafe { std::mem::zeroed() }; + let mut run = Run { out, mark }; + match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| selftest(&mut run, loader, cache, qc))) { + Ok(code) => code, + Err(_) => run.fail(0x65fe, "panic"), + } +} diff --git a/drivers/NativeMetal/bridge/NativeMetalObjects.h b/drivers/NativeMetal/bridge/NativeMetalObjects.h new file mode 100644 index 0000000..6cf7327 --- /dev/null +++ b/drivers/NativeMetal/bridge/NativeMetalObjects.h @@ -0,0 +1,192 @@ +/* SPDX-License-Identifier: MIT */ +/* C ABI of the bridge's Metal objects (NativeMetalObjects.rs): the metal2vulkan runtime's device, resources, states, queues, command buffers + * and encoders as opaque handles, for the Objective-C frontend (NativeBridgeMetalFrontend.mm). Enumerations and option words are Metal's raw + * values. A call that fails returns 0/NULL and leaves its reason in nmb_last_error() (per thread). Every handle is released once with its + * release call; an encoder is released by its end call (or by nmb_encoder_discard when it is never ended). */ +#ifndef NATIVE_METAL_OBJECTS_H +#define NATIVE_METAL_OBJECTS_H +#include +#include +#include +#ifdef __cplusplus +extern "C" { +#endif +typedef struct NmbDevice NmbDevice; +typedef struct NmbBuffer NmbBuffer; +typedef struct NmbTexture NmbTexture; +typedef struct NmbLibrary NmbLibrary; +typedef struct NmbFunction NmbFunction; +typedef struct NmbComputePipeline NmbComputePipeline; +typedef struct NmbRenderPipeline NmbRenderPipeline; +typedef struct NmbSampler NmbSampler; +typedef struct NmbDepthStencil NmbDepthStencil; +typedef struct NmbQueue NmbQueue; +typedef struct NmbCommandBuffer NmbCommandBuffer; +typedef struct NmbEncoder NmbEncoder; +typedef struct NmbEvent NmbEvent; +typedef struct NmbFence NmbFence; + +const char *nmb_last_error(void); +/* The bridge's embedded AIR fixtures: the runtime's own runtime_examples and kernel_function_constant_gate_fold. */ +const void *nmb_fixture_air(size_t *length); +const void *nmb_function_constant_air(size_t *length); + +/* loader: a library exporting vkGetInstanceProcAddr (Turnip on the tablet). cache: a writable directory for translated shaders. */ +NmbDevice *nmb_device_open(const char *loader, const char *cache); +void nmb_device_release(NmbDevice *); +size_t nmb_device_name(const NmbDevice *, char *out, size_t size); +uint64_t nmb_device_max_buffer_length(const NmbDevice *); + +NmbBuffer *nmb_buffer_new(NmbDevice *, uint64_t length, uint64_t options, const void *bytes /* NULL: zeroed */); +/* Wraps page-aligned memory the caller keeps mapped until the buffer is released (an IOSurface's base, host pages). */ +NmbBuffer *nmb_buffer_new_no_copy(NmbDevice *, void *pointer, uint64_t length, uint64_t options); +void nmb_buffer_release(NmbBuffer *); +void *nmb_buffer_contents(const NmbBuffer *); +uint64_t nmb_buffer_length(const NmbBuffer *); +uint64_t nmb_buffer_gpu_address(const NmbBuffer *); + +struct NmbTextureDescriptor { + uint64_t type, format, width, height, depth, levels, samples, array_length, usage, options; +}; +NmbTexture *nmb_texture_new(NmbDevice *, const struct NmbTextureDescriptor *); +/* A linear texture over a buffer's bytes; iosurface marks the buffer as standing for a surface without planes (Metal's IOSurface rules). */ +NmbTexture *nmb_texture_new_on_buffer(NmbBuffer *, const struct NmbTextureDescriptor *, uint64_t offset, uint64_t bytes_per_row, bool iosurface); +void nmb_texture_release(NmbTexture *); +void nmb_texture_describe(const NmbTexture *, struct NmbTextureDescriptor *out); +bool nmb_texture_replace(NmbTexture *, const uint64_t region[6], uint64_t level, uint64_t slice, const void *bytes, uint64_t bytes_per_row, + uint64_t bytes_per_image, uint64_t length); +bool nmb_texture_get_bytes(const NmbTexture *, void *bytes, uint64_t bytes_per_row, uint64_t bytes_per_image, uint64_t length, + const uint64_t region[6], uint64_t level, uint64_t slice); + +/* A metallib (a universal one gives its AIR64 slice), or with air set bare AIR bitcode. */ +NmbLibrary *nmb_library_new(NmbDevice *, const void *bytes, size_t length, bool air); +void nmb_library_release(NmbLibrary *); +uint32_t nmb_library_function_count(const NmbLibrary *); +size_t nmb_library_function_name(const NmbLibrary *, uint32_t index, char *out, size_t size); +NmbFunction *nmb_function_new(NmbLibrary *, const char *name); +/* A function specialized with constant values (newFunctionWithName:constantValues:error:): type is Metal's MTLDataType, name is null for an + * indexed value; bytes holds the value's bytes. */ +struct NmbConstant {uint64_t type, index; const char *name; const void *bytes; uint64_t length;}; +NmbFunction *nmb_function_new_specialized(NmbLibrary *, const char *name, const struct NmbConstant *constants, uint32_t count); +void nmb_function_release(NmbFunction *); +/* Metal's MTLFunctionType (1 vertex, 2 fragment, 3 kernel, ...). */ +uint64_t nmb_function_type(const NmbFunction *); + +NmbComputePipeline *nmb_compute_pipeline_new(NmbDevice *, NmbFunction *); +void nmb_compute_pipeline_release(NmbComputePipeline *); +uint64_t nmb_compute_pipeline_max_threads(const NmbComputePipeline *); +uint64_t nmb_compute_pipeline_execution_width(const NmbComputePipeline *); + +struct NmbColorAttachment { + uint64_t format, write_mask, blending, source_rgb, destination_rgb, rgb_operation, source_alpha, destination_alpha, alpha_operation; +}; +struct NmbVertexAttribute {uint64_t format, offset, buffer;}; +struct NmbVertexLayout {uint64_t stride, step_function, step_rate;}; +struct NmbRenderPipelineDescriptor { + NmbFunction *vertex, *fragment; + struct NmbColorAttachment colors[8]; + uint64_t depth_format, stencil_format, samples, topology, rasterization; + uint64_t vertex_inputs; /* 0: no vertex descriptor */ + struct NmbVertexAttribute attributes[31]; + struct NmbVertexLayout layouts[31]; +}; +NmbRenderPipeline *nmb_render_pipeline_new(NmbDevice *, const struct NmbRenderPipelineDescriptor *); +void nmb_render_pipeline_release(NmbRenderPipeline *); + +struct NmbSamplerDescriptor { + uint64_t min_filter, mag_filter, mip_filter, s_address, t_address, r_address, border_color, normalized, max_anisotropy, compare, lod_average; + float lod_min, lod_max; +}; +NmbSampler *nmb_sampler_new(NmbDevice *, const struct NmbSamplerDescriptor *); +void nmb_sampler_release(NmbSampler *); + +struct NmbStencil {uint64_t compare, failure, depth_failure, pass, read_mask, write_mask;}; +struct NmbDepthStencilDescriptor {uint64_t compare, write; struct NmbStencil front, back;}; +NmbDepthStencil *nmb_depth_stencil_new(NmbDevice *, const struct NmbDepthStencilDescriptor *); +void nmb_depth_stencil_release(NmbDepthStencil *); + +NmbQueue *nmb_queue_new(NmbDevice *); +void nmb_queue_release(NmbQueue *); +NmbCommandBuffer *nmb_command_buffer_new(NmbQueue *); +void nmb_command_buffer_release(NmbCommandBuffer *); +/* The handler runs once, on a runtime thread, after the command buffer completed or failed. */ +bool nmb_command_buffer_on_completed(NmbCommandBuffer *, void (*handler)(void *context), void *context); +bool nmb_command_buffer_commit(NmbCommandBuffer *); +/* 0 completed in time, 1 timed out (timeout_ms 0 waits without a bound). */ +int nmb_command_buffer_wait(NmbCommandBuffer *, uint64_t timeout_ms); +/* Metal's MTLCommandBufferStatus (0 not enqueued .. 4 completed, 5 error). */ +uint64_t nmb_command_buffer_status(const NmbCommandBuffer *); +size_t nmb_command_buffer_error(const NmbCommandBuffer *, char *out, size_t size); +double nmb_command_buffer_gpu_time(const NmbCommandBuffer *, bool end); + +/* Events: an MTLEvent (shared false) or an MTLSharedEvent within this process. */ +NmbEvent *nmb_event_new(NmbDevice *, bool shared); +void nmb_event_release(NmbEvent *); +uint64_t nmb_shared_event_value(const NmbEvent *); +bool nmb_shared_event_set_value(NmbEvent *, uint64_t value); +/* true once the value is reached within timeout_ms */ +bool nmb_shared_event_wait(NmbEvent *, uint64_t value, uint64_t timeout_ms); +/* handler(context, value) runs once on the bridge's listener thread when the value is reached (at once if it already is). */ +bool nmb_shared_event_notify(NmbEvent *, uint64_t value, void (*handler)(void *context, uint64_t value), void *context); +bool nmb_command_buffer_signal_event(NmbCommandBuffer *, NmbEvent *, uint64_t value); +bool nmb_command_buffer_wait_event(NmbCommandBuffer *, NmbEvent *, uint64_t value); +NmbFence *nmb_fence_new(NmbDevice *); +void nmb_fence_release(NmbFence *); +/* stages: Metal's MTLRenderStages for a render encoder, ignored otherwise */ +bool nmb_encoder_update_fence(NmbEncoder *, NmbFence *, uint64_t stages); +bool nmb_encoder_wait_fence(NmbEncoder *, NmbFence *, uint64_t stages); + +NmbEncoder *nmb_compute_encoder_new(NmbCommandBuffer *); +NmbEncoder *nmb_blit_encoder_new(NmbCommandBuffer *); +struct NmbPassAttachment { + NmbTexture *texture; + uint64_t level, slice, depth_plane, load, store; + double clear[4]; /* color, or clear[0] depth / stencil */ +}; +struct NmbRenderPassDescriptor { + struct NmbPassAttachment colors[8], depth, stencil; + uint64_t width, height, array_length, samples; +}; +NmbEncoder *nmb_render_encoder_new(NmbCommandBuffer *, const struct NmbRenderPassDescriptor *); +bool nmb_encoder_end(NmbEncoder *); +void nmb_encoder_discard(NmbEncoder *); + +/* stage: 0 compute, 1 vertex, 2 fragment */ +bool nmb_encoder_set_buffer(NmbEncoder *, uint32_t stage, NmbBuffer *, uint64_t offset, uint32_t index); +bool nmb_encoder_set_buffer_offset(NmbEncoder *, uint32_t stage, uint64_t offset, uint32_t index); +bool nmb_encoder_set_bytes(NmbEncoder *, uint32_t stage, const void *bytes, size_t length, uint32_t index); +bool nmb_encoder_set_texture(NmbEncoder *, uint32_t stage, NmbTexture *, uint32_t index); +bool nmb_encoder_set_sampler(NmbEncoder *, uint32_t stage, NmbSampler *, uint32_t index); + +bool nmb_compute_set_pipeline(NmbEncoder *, NmbComputePipeline *); +bool nmb_compute_set_threadgroup_memory(NmbEncoder *, uint32_t length, uint32_t index); +bool nmb_compute_dispatch(NmbEncoder *, const uint64_t size[3], const uint64_t per_group[3], bool threads); + +bool nmb_render_set_pipeline(NmbEncoder *, NmbRenderPipeline *); +bool nmb_render_set_depth_stencil(NmbEncoder *, NmbDepthStencil *); +bool nmb_render_set_viewport(NmbEncoder *, const double viewport[6]); +bool nmb_render_set_scissor(NmbEncoder *, const uint64_t rect[4]); +bool nmb_render_set_raster(NmbEncoder *, uint64_t cull_mode, uint64_t winding, uint64_t fill_mode); +bool nmb_render_set_stencil_reference(NmbEncoder *, uint32_t front, uint32_t back); +bool nmb_render_set_blend_color(NmbEncoder *, const float color[4]); +bool nmb_render_draw(NmbEncoder *, uint64_t primitive, uint32_t start, uint32_t count, uint32_t instances, uint32_t base_instance); +bool nmb_render_draw_indexed(NmbEncoder *, uint64_t primitive, uint32_t count, uint64_t index_type, NmbBuffer *indices, uint64_t offset, + uint32_t instances, int32_t base_vertex, uint32_t base_instance); + +bool nmb_blit_copy_buffer(NmbEncoder *, NmbBuffer *source, uint64_t source_offset, NmbBuffer *destination, uint64_t destination_offset, + uint64_t size); +bool nmb_blit_fill_buffer(NmbEncoder *, NmbBuffer *, uint64_t start, uint64_t end, uint8_t value); +/* origins and sizes are x,y,z / width,height,depth */ +bool nmb_blit_copy_texture(NmbEncoder *, NmbTexture *source, uint64_t source_slice, uint64_t source_level, const uint64_t source_origin[3], + const uint64_t size[3], NmbTexture *destination, uint64_t destination_slice, uint64_t destination_level, + const uint64_t destination_origin[3]); +bool nmb_blit_buffer_to_texture(NmbEncoder *, NmbBuffer *source, uint64_t offset, uint64_t bytes_per_row, uint64_t bytes_per_image, + const uint64_t size[3], NmbTexture *destination, uint64_t slice, uint64_t level, const uint64_t origin[3]); +bool nmb_blit_texture_to_buffer(NmbEncoder *, NmbTexture *source, uint64_t slice, uint64_t level, const uint64_t origin[3], + const uint64_t size[3], NmbBuffer *destination, uint64_t offset, uint64_t bytes_per_row, + uint64_t bytes_per_image); +bool nmb_blit_generate_mipmaps(NmbEncoder *, NmbTexture *); +#ifdef __cplusplus +} +#endif +#endif diff --git a/drivers/NativeMetal/bridge/NativeMetalObjects.rs b/drivers/NativeMetal/bridge/NativeMetalObjects.rs new file mode 100644 index 0000000..aef44c3 --- /dev/null +++ b/drivers/NativeMetal/bridge/NativeMetalObjects.rs @@ -0,0 +1,1340 @@ +// SPDX-License-Identifier: MIT +//! The bridge's Metal objects behind a C ABI (NativeMetalObjects.h): the metal2vulkan runtime's device, resources, states, queues, +//! command buffers and encoders as boxed handles for the Objective-C frontend. Raw Metal enumerations go through the runtime's own +//! checked conversions; a value it does not know fails the call (never a substitute). Failures leave their reason in a per-thread +//! string, and no call unwinds into C. + +use metal2vulkan::bind::SamplerDescriptor; +use metal2vulkan::metal::state::{DepthStencilDescriptor, StencilDescriptor}; +use metal2vulkan::library::{FunctionConstantValue, FunctionConstantValues}; +use metal2vulkan::metal::{ + BlendFactor, BlendOperation, ColorWriteMask, CompareFunction, CullMode, DataType, IndexType, PixelFormat, PrimitiveTopologyClass, + PrimitiveType, RenderStages, SamplerAddressMode, SamplerBorderColor, SamplerMinMagFilter, SamplerMipFilter, StencilOperation, TextureType, + TextureUsage, TriangleFillMode, VertexFormat, VertexStepFunction, Winding, +}; +use metal2vulkan::pipeline::{ + ColorAttachmentDescriptor, RenderPipelineDescriptor, StageSource, VertexAttributeDescriptor, VertexBufferLayoutDescriptor, + VertexDescriptor, +}; +use metal2vulkan_runtime::{ + BlitCommandEncoder, BlitOption, Buffer, ClearColor, CommandBuffer, CommandQueue, ComputeCommandEncoder, + ComputePipelineState, DepthStencilState, Device, DeviceOptions, Event, EventRef, Fence, Function, HostMemory, IoSurfacePlane, Library, LoadAction, + MetalLevel, Origin, PipelineOptions, Region, RenderCommandEncoder, RenderPassDescriptor, RenderPipelineState, ResourceOptions, + SamplerState, ScissorRect, SharedEvent, SharedEventListener, Size, StoreAction, Texture, TextureDescriptor, Viewport, +}; +use std::cell::RefCell; +use std::ffi::{c_char, c_void, CStr}; +use std::path::PathBuf; +use std::ptr::NonNull; +use std::time::Duration; + +thread_local! { + static LAST: RefCell> = const { RefCell::new(Vec::new()) }; +} + +fn set_error(why: String) { + LAST.with(|last| { + let mut last = last.borrow_mut(); + last.clear(); + last.extend(why.bytes().filter(|b| *b != 0)); + last.push(0); + }); +} + +/// Runs `body`, turning an error or a panic into the per-thread reason and `fallback`. +fn guard(fallback: T, body: impl FnOnce() -> Result) -> T { + match std::panic::catch_unwind(std::panic::AssertUnwindSafe(body)) { + Ok(Ok(value)) => value, + Ok(Err(why)) => { + set_error(why); + fallback + } + Err(_) => { + set_error("panic in the Metal runtime".into()); + fallback + } + } +} + +fn boxed(value: T) -> *mut T { + Box::into_raw(Box::new(value)) +} + +/// The object a handle points at. +fn get<'a, T>(handle: *const T) -> Result<&'a T, String> { + // SAFETY: handles come from `boxed` and live until their release call (the header's contract). + unsafe { handle.as_ref() }.ok_or_else(|| "null handle".to_string()) +} + +fn get_mut<'a, T>(handle: *mut T) -> Result<&'a mut T, String> { + // SAFETY: as in `get`; an encoder is used by one thread at a time (Metal's rule). + unsafe { handle.as_mut() }.ok_or_else(|| "null handle".to_string()) +} + +fn release(handle: *mut T) { + if !handle.is_null() { + // SAFETY: from `boxed`, released once. + drop(unsafe { Box::from_raw(handle) }); + } +} + +fn copy_out(text: &str, out: *mut c_char, size: usize) -> usize { + if !out.is_null() && size > 0 { + let n = text.len().min(size - 1); + // SAFETY: the caller's buffer holds `size` bytes. + unsafe { + std::ptr::copy_nonoverlapping(text.as_ptr(), out as *mut u8, n); + *out.add(n) = 0; + } + } + text.len() +} + +fn raw>(value: u64, what: &str) -> Result +where + T::Error: std::fmt::Debug, +{ + u32::try_from(value).ok().and_then(|v| T::try_from(v).ok()).ok_or_else(|| format!("{what} {value} is not a Metal value")) +} + +fn e(what: &'static str) -> impl FnOnce(E) -> String { + move |error| format!("{what}: {error:?}") +} + +fn options(value: u64) -> Result { + ResourceOptions::from_raw(value).map_err(e("resource options")) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_last_error() -> *const c_char { + LAST.with(|last| { + let last = last.borrow(); + if last.is_empty() { c"".as_ptr() } else { last.as_ptr() as *const c_char } + }) +} + +/// The runtime's kernel_function_constant_gate_fold fixture, assembled to AIR at build time. +static FUNCTION_CONSTANT_AIR: &[u8] = include_bytes!(env!("NMB_FC_AIR")); + +/// # Safety +/// `length` is writable. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn nmb_fixture_air(length: *mut usize) -> *const c_void { + // SAFETY: the caller's contract. + if let Some(length) = unsafe { length.as_mut() } { + *length = crate::FIXTURE_AIR.len(); + } + crate::FIXTURE_AIR.as_ptr() as *const c_void +} + +/// # Safety +/// `length` is writable. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn nmb_function_constant_air(length: *mut usize) -> *const c_void { + // SAFETY: the caller's contract. + if let Some(length) = unsafe { length.as_mut() } { + *length = FUNCTION_CONSTANT_AIR.len(); + } + FUNCTION_CONSTANT_AIR.as_ptr() as *const c_void +} + +// ---- device ---- + +pub struct NmbDevice(Device); + +/// # Safety +/// `loader` and `cache` are C strings. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn nmb_device_open(loader: *const c_char, cache: *const c_char) -> *mut NmbDevice { + guard(std::ptr::null_mut(), || { + if loader.is_null() || cache.is_null() { + return Err("loader and cache are required".into()); + } + // SAFETY: the caller's contract. + let (loader, cache) = unsafe { (CStr::from_ptr(loader), CStr::from_ptr(cache)) }; + let mut options = DeviceOptions::default(); + options.loader = Some(PathBuf::from(loader.to_string_lossy().into_owned())); + options.metal_level = Some(MetalLevel::NEWEST); + options.shader_cache = metal2vulkan_runtime::ShaderCacheOptions::in_directory(cache.to_string_lossy().into_owned()); + Ok(boxed(NmbDevice(Device::system_default_with(&options).map_err(e("device"))?))) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_device_release(device: *mut NmbDevice) { + release(device) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_device_name(device: *const NmbDevice, out: *mut c_char, size: usize) -> usize { + guard(0, || Ok(copy_out(get(device)?.0.name(), out, size))) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_device_max_buffer_length(device: *const NmbDevice) -> u64 { + guard(0, || Ok(get(device)?.0.properties().max_buffer_length)) +} + +// ---- buffers ---- + +pub struct NmbBuffer(Buffer); + +/// # Safety +/// `bytes` is null or holds `length` bytes. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn nmb_buffer_new(device: *mut NmbDevice, length: u64, options_word: u64, bytes: *const c_void) -> *mut NmbBuffer { + guard(std::ptr::null_mut(), || { + let device = &get(device)?.0; + let options = options(options_word)?; + let buffer = if bytes.is_null() { + device.new_buffer(length, options) + } else { + // SAFETY: the caller's contract. + let bytes = unsafe { std::slice::from_raw_parts(bytes as *const u8, length as usize) }; + device.new_buffer_with_bytes(bytes, options) + }; + Ok(boxed(NmbBuffer(buffer.map_err(e("buffer"))?))) + }) +} + +/// # Safety +/// `pointer` is page aligned, holds `length` bytes and stays mapped until the buffer is released. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn nmb_buffer_new_no_copy(device: *mut NmbDevice, pointer: *mut c_void, length: u64, options_word: u64) + -> *mut NmbBuffer { + guard(std::ptr::null_mut(), || { + let device = &get(device)?.0; + // SAFETY: the caller's contract. + let memory = unsafe { HostMemory::new(NonNull::new(pointer as *mut u8).ok_or("null memory")?) }; + // SAFETY: as above. + let buffer = unsafe { device.new_buffer_with_bytes_no_copy(memory, length, options(options_word)?, None) }; + Ok(boxed(NmbBuffer(buffer.map_err(e("no-copy buffer"))?))) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_buffer_release(buffer: *mut NmbBuffer) { + release(buffer) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_buffer_contents(buffer: *const NmbBuffer) -> *mut c_void { + guard(std::ptr::null_mut(), || Ok(get(buffer)?.0.contents().map_or(std::ptr::null_mut(), |p| p.as_ptr() as *mut c_void))) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_buffer_length(buffer: *const NmbBuffer) -> u64 { + guard(0, || Ok(get(buffer)?.0.length())) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_buffer_gpu_address(buffer: *const NmbBuffer) -> u64 { + guard(0, || Ok(get(buffer)?.0.gpu_address().0)) +} + +// ---- textures ---- + +#[repr(C)] +pub struct NmbTextureDescriptor { + pub kind: u64, + pub format: u64, + pub width: u64, + pub height: u64, + pub depth: u64, + pub levels: u64, + pub samples: u64, + pub array_length: u64, + pub usage: u64, + pub options: u64, +} + +pub struct NmbTexture(Texture); + +fn texture_descriptor(d: *const NmbTextureDescriptor) -> Result { + let d = get(d)?; + let mut out = TextureDescriptor::default(); + out.texture_type = raw::(d.kind, "texture type")?; + out.pixel_format = raw::(d.format, "pixel format")?; + out.width = d.width; + out.height = d.height; + out.depth = d.depth; + out.mipmap_level_count = d.levels; + out.sample_count = d.samples; + out.array_length = d.array_length; + out.usage = raw::(d.usage, "texture usage")?; + out.resource_options = options(d.options)?; + Ok(out) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_texture_new(device: *mut NmbDevice, descriptor: *const NmbTextureDescriptor) -> *mut NmbTexture { + guard(std::ptr::null_mut(), || { + let descriptor = texture_descriptor(descriptor)?; + Ok(boxed(NmbTexture(get(device)?.0.new_texture(&descriptor).map_err(e("texture"))?))) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_texture_new_on_buffer(buffer: *mut NmbBuffer, descriptor: *const NmbTextureDescriptor, offset: u64, + bytes_per_row: u64, iosurface: bool) -> *mut NmbTexture { + guard(std::ptr::null_mut(), || { + let descriptor = texture_descriptor(descriptor)?; + let buffer = &get(buffer)?.0; + let texture = if iosurface { + buffer.new_texture_with_iosurface_plane(&descriptor, IoSurfacePlane::new(offset, bytes_per_row, false)) + } else { + buffer.new_texture(&descriptor, offset, bytes_per_row) + }; + Ok(boxed(NmbTexture(texture.map_err(e("texture over buffer"))?))) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_texture_release(texture: *mut NmbTexture) { + release(texture) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_texture_describe(texture: *const NmbTexture, out: *mut NmbTextureDescriptor) { + guard((), || { + let t = &get(texture)?.0; + let out = get_mut(out)?; + *out = NmbTextureDescriptor { + kind: t.texture_type().as_raw() as u64, + format: t.pixel_format().as_raw() as u64, + width: t.width(), + height: t.height(), + depth: t.depth(), + levels: t.mipmap_level_count(), + samples: t.sample_count(), + array_length: t.array_length(), + usage: t.usage().as_raw() as u64, + options: t.resource_options().raw(), + }; + Ok(()) + }) +} + +fn region(r: *const u64) -> Result { + if r.is_null() { + return Err("null region".into()); + } + // SAFETY: the header's six words. + let r = unsafe { std::slice::from_raw_parts(r, 6) }; + Ok(Region::new_3d(r[0], r[1], r[2], r[3], r[4], r[5])) +} + +fn triple(v: *const u64) -> Result<[u64; 3], String> { + if v.is_null() { + return Err("null origin or size".into()); + } + // SAFETY: the header's three words. + let v = unsafe { std::slice::from_raw_parts(v, 3) }; + Ok([v[0], v[1], v[2]]) +} + +/// # Safety +/// `bytes` holds `length` bytes. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn nmb_texture_replace(texture: *mut NmbTexture, region_words: *const u64, level: u64, slice: u64, + bytes: *const c_void, bytes_per_row: u64, bytes_per_image: u64, length: u64) -> bool { + guard(false, || { + if bytes.is_null() { + return Err("null bytes".into()); + } + // SAFETY: the caller's contract. + let bytes = unsafe { std::slice::from_raw_parts(bytes as *const u8, length as usize) }; + get(texture)?.0.replace_region(region(region_words)?, level, slice, bytes, bytes_per_row, bytes_per_image) + .map_err(e("replace region"))?; + Ok(true) + }) +} + +/// # Safety +/// `bytes` holds `length` writable bytes. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn nmb_texture_get_bytes(texture: *const NmbTexture, bytes: *mut c_void, bytes_per_row: u64, bytes_per_image: u64, + length: u64, region_words: *const u64, level: u64, slice: u64) -> bool { + guard(false, || { + if bytes.is_null() { + return Err("null bytes".into()); + } + // SAFETY: the caller's contract. + let bytes = unsafe { std::slice::from_raw_parts_mut(bytes as *mut u8, length as usize) }; + get(texture)?.0.get_bytes(bytes, bytes_per_row, bytes_per_image, region(region_words)?, level, slice) + .map_err(e("get bytes"))?; + Ok(true) + }) +} + +// ---- libraries and functions ---- + +pub struct NmbLibrary(Library, Vec); +pub struct NmbFunction(Function); + +/// # Safety +/// `bytes` holds `length` bytes. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn nmb_library_new(device: *mut NmbDevice, bytes: *const c_void, length: usize, air: bool) -> *mut NmbLibrary { + guard(std::ptr::null_mut(), || { + let device = &get(device)?.0; + if bytes.is_null() || length == 0 { + return Err("no library bytes".into()); + } + // SAFETY: the caller's contract. + let bytes = unsafe { std::slice::from_raw_parts(bytes as *const u8, length) }; + let library = if air { + device.new_library_with_compile_library(metal2vulkan::library::Library::from_air(bytes).map_err(e("AIR"))?) + } else { + device.new_library_with_data(crate::air_slice(bytes).ok_or("metallib without an AIR64 slice")?).map_err(e("metallib"))? + }; + let names = library.function_names(); + Ok(boxed(NmbLibrary(library, names))) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_library_release(library: *mut NmbLibrary) { + release(library) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_library_function_count(library: *const NmbLibrary) -> u32 { + guard(0, || Ok(get(library)?.1.len() as u32)) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_library_function_name(library: *const NmbLibrary, index: u32, out: *mut c_char, size: usize) -> usize { + guard(0, || { + let names = &get(library)?.1; + Ok(copy_out(names.get(index as usize).ok_or("function index")?, out, size)) + }) +} + +/// # Safety +/// `name` is a C string. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn nmb_function_new(library: *mut NmbLibrary, name: *const c_char) -> *mut NmbFunction { + guard(std::ptr::null_mut(), || { + if name.is_null() { + return Err("null function name".into()); + } + // SAFETY: the caller's contract. + let name = unsafe { CStr::from_ptr(name) }.to_str().map_err(e("function name"))?; + Ok(boxed(NmbFunction(get(library)?.0.new_function_with_name(name).map_err(e("function"))?))) + }) +} + +#[repr(C)] +pub struct NmbConstant { + pub kind: u64, + pub index: u64, + pub name: *const c_char, + pub bytes: *const c_void, + pub length: u64, +} + +/// # Safety +/// `name` is a C string; `constants` holds `count` entries whose names are C strings (or null) and whose bytes hold `length` bytes. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn nmb_function_new_specialized(library: *mut NmbLibrary, name: *const c_char, constants: *const NmbConstant, + count: u32) -> *mut NmbFunction { + guard(std::ptr::null_mut(), || { + if name.is_null() || (constants.is_null() && count > 0) { + return Err("null function name or constants".into()); + } + // SAFETY: the caller's contract. + let name = unsafe { CStr::from_ptr(name) }.to_str().map_err(e("function name"))?; + let constants = if count == 0 { &[][..] } else { unsafe { std::slice::from_raw_parts(constants, count as usize) } }; + let mut values = FunctionConstantValues::defaults(); + for c in constants { + if c.bytes.is_null() { + return Err("constant without bytes".into()); + } + // SAFETY: the caller's contract. + let bytes = unsafe { std::slice::from_raw_parts(c.bytes as *const u8, c.length as usize) }; + let value = FunctionConstantValue::new(raw::(c.kind, "constant type")?, bytes.to_vec()); + if c.name.is_null() { + values.by_index.insert(u32::try_from(c.index).map_err(e("constant index"))?, value); + } else { + // SAFETY: the caller's contract. + let key = unsafe { CStr::from_ptr(c.name) }.to_str().map_err(e("constant name"))?; + values.by_name.insert(key.to_string(), value); + } + } + let function = get(library)?.0.new_function_with_name_constant_values(name, &values).map_err(e("specialized function"))?; + Ok(boxed(NmbFunction(function))) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_function_release(function: *mut NmbFunction) { + release(function) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_function_type(function: *const NmbFunction) -> u64 { + guard(0, || Ok(get(function)?.0.function_type().map_err(e("function type"))?.as_raw() as u64)) +} + +// ---- pipelines and states ---- + +pub struct NmbComputePipeline(ComputePipelineState); +pub struct NmbRenderPipeline(RenderPipelineState); +pub struct NmbSampler(SamplerState); +pub struct NmbDepthStencil(DepthStencilState); + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_compute_pipeline_new(device: *mut NmbDevice, function: *mut NmbFunction) -> *mut NmbComputePipeline { + guard(std::ptr::null_mut(), || { + let state = get(device)?.0.new_compute_pipeline_state_with_function(&get(function)?.0, &PipelineOptions::default()); + Ok(boxed(NmbComputePipeline(state.map_err(e("compute pipeline"))?))) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_compute_pipeline_release(pipeline: *mut NmbComputePipeline) { + release(pipeline) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_compute_pipeline_max_threads(pipeline: *const NmbComputePipeline) -> u64 { + guard(0, || Ok(get(pipeline)?.0.max_total_threads_per_threadgroup() as u64)) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_compute_pipeline_execution_width(pipeline: *const NmbComputePipeline) -> u64 { + guard(0, || Ok(get(pipeline)?.0.thread_execution_width() as u64)) +} + +#[repr(C)] +pub struct NmbColorAttachment { + pub format: u64, + pub write_mask: u64, + pub blending: u64, + pub source_rgb: u64, + pub destination_rgb: u64, + pub rgb_operation: u64, + pub source_alpha: u64, + pub destination_alpha: u64, + pub alpha_operation: u64, +} + +#[repr(C)] +pub struct NmbVertexAttribute { + pub format: u64, + pub offset: u64, + pub buffer: u64, +} + +#[repr(C)] +pub struct NmbVertexLayout { + pub stride: u64, + pub step_function: u64, + pub step_rate: u64, +} + +#[repr(C)] +pub struct NmbRenderPipelineDescriptor { + pub vertex: *mut NmbFunction, + pub fragment: *mut NmbFunction, + pub colors: [NmbColorAttachment; 8], + pub depth_format: u64, + pub stencil_format: u64, + pub samples: u64, + pub topology: u64, + pub rasterization: u64, + pub vertex_inputs: u64, + pub attributes: [NmbVertexAttribute; 31], + pub layouts: [NmbVertexLayout; 31], +} + +fn optional_format(value: u64) -> Result, String> { + if value == 0 { Ok(None) } else { raw::(value, "pixel format").map(Some) } +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_render_pipeline_new(device: *mut NmbDevice, descriptor: *const NmbRenderPipelineDescriptor) -> *mut NmbRenderPipeline { + guard(std::ptr::null_mut(), || { + let device = &get(device)?.0; + let d = get(descriptor)?; + let mut out = RenderPipelineDescriptor::new(StageSource::Function(get(d.vertex)?.0.function_descriptor())); + if !d.fragment.is_null() { + out.fragment = Some(StageSource::Function(get(d.fragment)?.0.function_descriptor())); + } + for (slot, c) in out.color_attachments.iter_mut().zip(d.colors.iter()) { + if c.format == 0 { + continue; + } + let mut a = ColorAttachmentDescriptor::new(raw::(c.format, "color format")?); + a.write_mask = raw::(c.write_mask, "write mask")?; + a.blending_enabled = c.blending != 0; + a.source_rgb_blend_factor = raw::(c.source_rgb, "blend factor")?; + a.destination_rgb_blend_factor = raw::(c.destination_rgb, "blend factor")?; + a.rgb_blend_operation = raw::(c.rgb_operation, "blend operation")?; + a.source_alpha_blend_factor = raw::(c.source_alpha, "blend factor")?; + a.destination_alpha_blend_factor = raw::(c.destination_alpha, "blend factor")?; + a.alpha_blend_operation = raw::(c.alpha_operation, "blend operation")?; + *slot = Some(a); + } + out.depth_format = optional_format(d.depth_format)?; + out.stencil_format = optional_format(d.stencil_format)?; + out.raster_sample_count = d.samples.max(1) as u32; + out.input_primitive_topology = raw::(d.topology, "topology")?; + out.rasterization_enabled = d.rasterization != 0; + if d.vertex_inputs != 0 { + let mut inputs = VertexDescriptor::new(); + for (i, a) in d.attributes.iter().enumerate() { + if a.format == 0 { + continue; + } + inputs.attributes.insert(i as u32, VertexAttributeDescriptor::new(raw::(a.format, "vertex format")?, + a.offset as u32, a.buffer as u32)); + if let Some(l) = d.layouts.get(a.buffer as usize) { + let mut layout = VertexBufferLayoutDescriptor::new(Some(l.stride as u32)); + layout.step_function = raw::(l.step_function, "step function")?; + layout.step_rate = l.step_rate as u32; + inputs.layouts.insert(a.buffer as u32, layout); + } + } + out.vertex_descriptor = Some(inputs); + } + let state = device.new_render_pipeline_state(&out, &PipelineOptions::default()).map_err(e("render pipeline"))?; + Ok(boxed(NmbRenderPipeline(state))) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_render_pipeline_release(pipeline: *mut NmbRenderPipeline) { + release(pipeline) +} + +#[repr(C)] +pub struct NmbSamplerDescriptor { + pub min_filter: u64, + pub mag_filter: u64, + pub mip_filter: u64, + pub s_address: u64, + pub t_address: u64, + pub r_address: u64, + pub border_color: u64, + pub normalized: u64, + pub max_anisotropy: u64, + pub compare: u64, + pub lod_average: u64, + pub lod_min: f32, + pub lod_max: f32, +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_sampler_new(device: *mut NmbDevice, descriptor: *const NmbSamplerDescriptor) -> *mut NmbSampler { + guard(std::ptr::null_mut(), || { + let d = get(descriptor)?; + let mut out = SamplerDescriptor::new(); + out.min_filter = raw::(d.min_filter, "min filter")?; + out.mag_filter = raw::(d.mag_filter, "mag filter")?; + out.mip_filter = raw::(d.mip_filter, "mip filter")?; + out.s_address_mode = raw::(d.s_address, "address mode")?; + out.t_address_mode = raw::(d.t_address, "address mode")?; + out.r_address_mode = raw::(d.r_address, "address mode")?; + out.border_color = raw::(d.border_color, "border color")?; + out.normalized_coordinates = d.normalized != 0; + out.max_anisotropy = d.max_anisotropy.max(1) as u32; + out.compare_function = raw::(d.compare, "compare function")?; + out.lod_average = d.lod_average != 0; + out.lod_min_clamp = d.lod_min; + out.lod_max_clamp = d.lod_max; + Ok(boxed(NmbSampler(get(device)?.0.new_sampler_state(&out).map_err(e("sampler"))?))) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_sampler_release(sampler: *mut NmbSampler) { + release(sampler) +} + +#[repr(C)] +pub struct NmbStencil { + pub compare: u64, + pub failure: u64, + pub depth_failure: u64, + pub pass: u64, + pub read_mask: u64, + pub write_mask: u64, +} + +#[repr(C)] +pub struct NmbDepthStencilDescriptor { + pub compare: u64, + pub write: u64, + pub front: NmbStencil, + pub back: NmbStencil, +} + +fn stencil(s: &NmbStencil) -> Result { + let mut out = StencilDescriptor::default(); + out.stencil_compare_function = raw::(s.compare, "stencil compare")?; + out.stencil_failure_operation = raw::(s.failure, "stencil operation")?; + out.depth_failure_operation = raw::(s.depth_failure, "stencil operation")?; + out.depth_stencil_pass_operation = raw::(s.pass, "stencil operation")?; + out.read_mask = s.read_mask as u32; + out.write_mask = s.write_mask as u32; + Ok(out) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_depth_stencil_new(device: *mut NmbDevice, descriptor: *const NmbDepthStencilDescriptor) -> *mut NmbDepthStencil { + guard(std::ptr::null_mut(), || { + let d = get(descriptor)?; + let mut out = DepthStencilDescriptor::default(); + out.depth_compare_function = raw::(d.compare, "depth compare")?; + out.depth_write_enabled = d.write != 0; + out.front_face_stencil = stencil(&d.front)?; + out.back_face_stencil = stencil(&d.back)?; + Ok(boxed(NmbDepthStencil(get(device)?.0.new_depth_stencil_state(&out).map_err(e("depth/stencil state"))?))) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_depth_stencil_release(state: *mut NmbDepthStencil) { + release(state) +} + +// ---- queues and command buffers ---- + +pub struct NmbQueue(CommandQueue); +pub struct NmbCommandBuffer(CommandBuffer); + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_queue_new(device: *mut NmbDevice) -> *mut NmbQueue { + guard(std::ptr::null_mut(), || Ok(boxed(NmbQueue(get(device)?.0.new_command_queue().map_err(e("queue"))?)))) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_queue_release(queue: *mut NmbQueue) { + release(queue) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_command_buffer_new(queue: *mut NmbQueue) -> *mut NmbCommandBuffer { + guard(std::ptr::null_mut(), || Ok(boxed(NmbCommandBuffer(get(queue)?.0.command_buffer().map_err(e("command buffer"))?)))) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_command_buffer_release(command_buffer: *mut NmbCommandBuffer) { + release(command_buffer) +} + +struct Context(*mut c_void); +// SAFETY: the context is the caller's, handed back on the runtime's thread as the header states. +unsafe impl Send for Context {} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_command_buffer_on_completed(command_buffer: *mut NmbCommandBuffer, handler: Option, + context: *mut c_void) -> bool { + guard(false, || { + let handler = handler.ok_or("null handler")?; + let context = Context(context); + get(command_buffer)?.0.add_completed_handler(Box::new(move |_| { + let context = context; + handler(context.0) + })).map_err(e("completed handler"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_command_buffer_commit(command_buffer: *mut NmbCommandBuffer) -> bool { + guard(false, || { + get(command_buffer)?.0.commit().map_err(e("commit"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_command_buffer_wait(command_buffer: *mut NmbCommandBuffer, timeout_ms: u64) -> i32 { + guard(-1, || { + let command_buffer = &get(command_buffer)?.0; + if timeout_ms == 0 { + command_buffer.wait_until_completed().map_err(e("wait"))?; + return Ok(0); + } + Ok(if command_buffer.wait_until_completed_for(Duration::from_millis(timeout_ms)).is_ok() { 0 } else { 1 }) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_command_buffer_status(command_buffer: *const NmbCommandBuffer) -> u64 { + guard(5, || { + Ok(get(command_buffer)?.0.status().raw()) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_command_buffer_error(command_buffer: *const NmbCommandBuffer, out: *mut c_char, size: usize) -> usize { + guard(0, || Ok(get(command_buffer)?.0.error_description().map_or(0, |text| copy_out(&text, out, size)))) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_command_buffer_gpu_time(command_buffer: *const NmbCommandBuffer, end: bool) -> f64 { + guard(0.0, || { + let command_buffer = &get(command_buffer)?.0; + let time = if end { command_buffer.gpu_end_time() } else { command_buffer.gpu_start_time() }; + time.map_err(e("GPU time")) + }) +} + +// ---- events and fences ---- + +pub enum NmbEvent { + Event(Event), + Shared(SharedEvent), +} +pub struct NmbFence(Fence); + +impl NmbEvent { + fn as_ref(&self) -> EventRef<'_> { + match self { + NmbEvent::Event(e) => EventRef::Event(e), + NmbEvent::Shared(s) => EventRef::Shared(s), + } + } +} + +fn shared<'a>(event: *const NmbEvent) -> Result<&'a SharedEvent, String> { + match get(event)? { + NmbEvent::Shared(s) => Ok(s), + NmbEvent::Event(_) => Err("not a shared event".into()), + } +} + +/// One listener thread for every notification of this process (Metal's listener only names the queue a block runs on; the frontend +/// forwards to it). +fn listener() -> Result<&'static SharedEventListener, String> { + static LISTENER: std::sync::OnceLock> = std::sync::OnceLock::new(); + LISTENER.get_or_init(|| SharedEventListener::new().map_err(e("listener"))).as_ref().map_err(|why| why.clone()) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_event_new(device: *mut NmbDevice, is_shared: bool) -> *mut NmbEvent { + guard(std::ptr::null_mut(), || { + let device = &get(device)?.0; + Ok(boxed(if is_shared { + NmbEvent::Shared(device.new_shared_event().map_err(e("shared event"))?) + } else { + NmbEvent::Event(device.new_event().map_err(e("event"))?) + })) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_event_release(event: *mut NmbEvent) { + release(event) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_shared_event_value(event: *const NmbEvent) -> u64 { + guard(0, || Ok(shared(event)?.signaled_value())) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_shared_event_set_value(event: *mut NmbEvent, value: u64) -> bool { + guard(false, || { + shared(event)?.set_signaled_value(value); + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_shared_event_wait(event: *mut NmbEvent, value: u64, timeout_ms: u64) -> bool { + guard(false, || Ok(shared(event)?.wait_until_signaled_value(value, Duration::from_millis(timeout_ms)))) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_shared_event_notify(event: *mut NmbEvent, value: u64, handler: Option, + context: *mut c_void) -> bool { + guard(false, || { + let handler = handler.ok_or("null handler")?; + let context = Context(context); + shared(event)?.notify_listener(listener()?, value, Box::new(move |_, reached| { + let context = context; + handler(context.0, reached) + })); + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_command_buffer_signal_event(command_buffer: *mut NmbCommandBuffer, event: *mut NmbEvent, value: u64) -> bool { + guard(false, || { + get(command_buffer)?.0.encode_signal_event(get(event)?.as_ref(), value).map_err(e("signal event"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_command_buffer_wait_event(command_buffer: *mut NmbCommandBuffer, event: *mut NmbEvent, value: u64) -> bool { + guard(false, || { + get(command_buffer)?.0.encode_wait_for_event(get(event)?.as_ref(), value).map_err(e("wait for event"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_fence_new(device: *mut NmbDevice) -> *mut NmbFence { + guard(std::ptr::null_mut(), || Ok(boxed(NmbFence(get(device)?.0.new_fence())))) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_fence_release(fence: *mut NmbFence) { + release(fence) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_encoder_update_fence(encoder: *mut NmbEncoder, fence: *mut NmbFence, stages: u64) -> bool { + guard(false, || { + let fence = &get(fence)?.0; + match get_mut(encoder)? { + NmbEncoder::Compute(c) => c.update_fence(fence), + NmbEncoder::Blit(b) => b.update_fence(fence), + NmbEncoder::Render(r) => r.update_fence(fence, raw::(stages, "render stages")?), + } + .map_err(e("update fence"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_encoder_wait_fence(encoder: *mut NmbEncoder, fence: *mut NmbFence, stages: u64) -> bool { + guard(false, || { + let fence = &get(fence)?.0; + match get_mut(encoder)? { + NmbEncoder::Compute(c) => c.wait_for_fence(fence), + NmbEncoder::Blit(b) => b.wait_for_fence(fence), + NmbEncoder::Render(r) => r.wait_for_fence(fence, raw::(stages, "render stages")?), + } + .map_err(e("wait for fence"))?; + Ok(true) + }) +} + +// ---- encoders ---- + +pub enum NmbEncoder { + Compute(ComputeCommandEncoder), + Render(RenderCommandEncoder), + Blit(BlitCommandEncoder), +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_compute_encoder_new(command_buffer: *mut NmbCommandBuffer) -> *mut NmbEncoder { + guard(std::ptr::null_mut(), || { + Ok(boxed(NmbEncoder::Compute(get(command_buffer)?.0.compute_command_encoder().map_err(e("compute encoder"))?))) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_blit_encoder_new(command_buffer: *mut NmbCommandBuffer) -> *mut NmbEncoder { + guard(std::ptr::null_mut(), || Ok(boxed(NmbEncoder::Blit(get(command_buffer)?.0.blit_command_encoder().map_err(e("blit encoder"))?)))) +} + +#[repr(C)] +pub struct NmbPassAttachment { + pub texture: *mut NmbTexture, + pub level: u64, + pub slice: u64, + pub depth_plane: u64, + pub load: u64, + pub store: u64, + pub clear: [f64; 4], +} + +#[repr(C)] +pub struct NmbRenderPassDescriptor { + pub colors: [NmbPassAttachment; 8], + pub depth: NmbPassAttachment, + pub stencil: NmbPassAttachment, + pub width: u64, + pub height: u64, + pub array_length: u64, + pub samples: u64, +} + +fn texture_of(a: &NmbPassAttachment) -> Result, String> { + if a.texture.is_null() { Ok(None) } else { Ok(Some(get(a.texture)?.0.clone())) } +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_render_encoder_new(command_buffer: *mut NmbCommandBuffer, descriptor: *const NmbRenderPassDescriptor) + -> *mut NmbEncoder { + guard(std::ptr::null_mut(), || { + let d = get(descriptor)?; + let mut pass = RenderPassDescriptor::new(); + for (slot, a) in pass.color_attachments.iter_mut().zip(d.colors.iter()) { + slot.texture = texture_of(a)?; + slot.level = a.level; + slot.slice = a.slice; + slot.depth_plane = a.depth_plane; + slot.load_action = LoadAction::from_raw(a.load).map_err(e("load action"))?; + slot.store_action = StoreAction::from_raw(a.store).map_err(e("store action"))?; + slot.clear_color = ClearColor::new(a.clear[0], a.clear[1], a.clear[2], a.clear[3]); + } + let depth = &mut pass.depth_attachment; + depth.texture = texture_of(&d.depth)?; + depth.level = d.depth.level; + depth.slice = d.depth.slice; + depth.depth_plane = d.depth.depth_plane; + depth.load_action = LoadAction::from_raw(d.depth.load).map_err(e("load action"))?; + depth.store_action = StoreAction::from_raw(d.depth.store).map_err(e("store action"))?; + depth.clear_depth = d.depth.clear[0]; + let stencil = &mut pass.stencil_attachment; + stencil.texture = texture_of(&d.stencil)?; + stencil.level = d.stencil.level; + stencil.slice = d.stencil.slice; + stencil.depth_plane = d.stencil.depth_plane; + stencil.load_action = LoadAction::from_raw(d.stencil.load).map_err(e("load action"))?; + stencil.store_action = StoreAction::from_raw(d.stencil.store).map_err(e("store action"))?; + stencil.clear_stencil = d.stencil.clear[0] as u32; + pass.render_target_width = d.width; + pass.render_target_height = d.height; + pass.render_target_array_length = d.array_length; + pass.default_raster_sample_count = d.samples; + let encoder = get(command_buffer)?.0.render_command_encoder(&pass).map_err(e("render encoder"))?; + Ok(boxed(NmbEncoder::Render(encoder))) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_encoder_end(encoder: *mut NmbEncoder) -> bool { + guard(false, || { + if encoder.is_null() { + return Err("null encoder".into()); + } + // SAFETY: from `boxed`; ending releases it. + let encoder = unsafe { Box::from_raw(encoder) }; + match *encoder { + NmbEncoder::Compute(c) => c.end_encoding(), + NmbEncoder::Render(r) => r.end_encoding(), + NmbEncoder::Blit(b) => b.end_encoding(), + } + .map_err(e("end encoding"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_encoder_discard(encoder: *mut NmbEncoder) { + release(encoder) +} + +fn compute<'a>(encoder: *mut NmbEncoder) -> Result<&'a mut ComputeCommandEncoder, String> { + match get_mut(encoder)? { + NmbEncoder::Compute(c) => Ok(c), + _ => Err("not a compute encoder".into()), + } +} + +fn render<'a>(encoder: *mut NmbEncoder) -> Result<&'a mut RenderCommandEncoder, String> { + match get_mut(encoder)? { + NmbEncoder::Render(r) => Ok(r), + _ => Err("not a render encoder".into()), + } +} + +fn blit<'a>(encoder: *mut NmbEncoder) -> Result<&'a mut BlitCommandEncoder, String> { + match get_mut(encoder)? { + NmbEncoder::Blit(b) => Ok(b), + _ => Err("not a blit encoder".into()), + } +} + +fn optional<'a, T>(handle: *const T) -> Result, String> { + if handle.is_null() { Ok(None) } else { get(handle).map(Some) } +} + +const COMPUTE: u32 = 0; +const VERTEX: u32 = 1; +const FRAGMENT: u32 = 2; + +fn bad_stage(stage: u32) -> String { + format!("stage {stage} does not fit this encoder") +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_encoder_set_buffer(encoder: *mut NmbEncoder, stage: u32, buffer: *mut NmbBuffer, offset: u64, index: u32) -> bool { + guard(false, || { + let buffer = optional(buffer)?.map(|b| &b.0); + match (get_mut(encoder)?, stage) { + (NmbEncoder::Compute(c), COMPUTE) => c.set_buffer(buffer, offset, index), + (NmbEncoder::Render(r), VERTEX) => r.set_vertex_buffer(buffer, offset, index), + (NmbEncoder::Render(r), FRAGMENT) => r.set_fragment_buffer(buffer, offset, index), + _ => return Err(bad_stage(stage)), + } + .map_err(e("set buffer"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_encoder_set_buffer_offset(encoder: *mut NmbEncoder, stage: u32, offset: u64, index: u32) -> bool { + guard(false, || { + match (get_mut(encoder)?, stage) { + (NmbEncoder::Compute(c), COMPUTE) => c.set_buffer_offset(offset, index), + (NmbEncoder::Render(r), VERTEX) => r.set_vertex_buffer_offset(offset, index), + (NmbEncoder::Render(r), FRAGMENT) => r.set_fragment_buffer_offset(offset, index), + _ => return Err(bad_stage(stage)), + } + .map_err(e("set buffer offset"))?; + Ok(true) + }) +} + +/// # Safety +/// `bytes` holds `length` bytes. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn nmb_encoder_set_bytes(encoder: *mut NmbEncoder, stage: u32, bytes: *const c_void, length: usize, index: u32) + -> bool { + guard(false, || { + if bytes.is_null() { + return Err("null bytes".into()); + } + // SAFETY: the caller's contract. + let bytes = unsafe { std::slice::from_raw_parts(bytes as *const u8, length) }; + match (get_mut(encoder)?, stage) { + (NmbEncoder::Compute(c), COMPUTE) => c.set_bytes(bytes, index), + (NmbEncoder::Render(r), VERTEX) => r.set_vertex_bytes(bytes, index), + (NmbEncoder::Render(r), FRAGMENT) => r.set_fragment_bytes(bytes, index), + _ => return Err(bad_stage(stage)), + } + .map_err(e("set bytes"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_encoder_set_texture(encoder: *mut NmbEncoder, stage: u32, texture: *mut NmbTexture, index: u32) -> bool { + guard(false, || { + let texture = optional(texture)?.map(|t| &t.0); + match (get_mut(encoder)?, stage) { + (NmbEncoder::Compute(c), COMPUTE) => c.set_texture(texture, index), + (NmbEncoder::Render(r), VERTEX) => r.set_vertex_texture(texture, index), + (NmbEncoder::Render(r), FRAGMENT) => r.set_fragment_texture(texture, index), + _ => return Err(bad_stage(stage)), + } + .map_err(e("set texture"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_encoder_set_sampler(encoder: *mut NmbEncoder, stage: u32, sampler: *mut NmbSampler, index: u32) -> bool { + guard(false, || { + let sampler = optional(sampler)?.map(|s| &s.0); + match (get_mut(encoder)?, stage) { + (NmbEncoder::Compute(c), COMPUTE) => c.set_sampler_state(sampler, index), + (NmbEncoder::Render(r), VERTEX) => r.set_vertex_sampler_state(sampler, index), + (NmbEncoder::Render(r), FRAGMENT) => r.set_fragment_sampler_state(sampler, index), + _ => return Err(bad_stage(stage)), + } + .map_err(e("set sampler"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_compute_set_pipeline(encoder: *mut NmbEncoder, pipeline: *mut NmbComputePipeline) -> bool { + guard(false, || { + compute(encoder)?.set_compute_pipeline_state(&get(pipeline)?.0).map_err(e("compute pipeline"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_compute_set_threadgroup_memory(encoder: *mut NmbEncoder, length: u32, index: u32) -> bool { + guard(false, || { + compute(encoder)?.set_threadgroup_memory_length(length, index).map_err(e("threadgroup memory"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_compute_dispatch(encoder: *mut NmbEncoder, size: *const u64, per_group: *const u64, threads: bool) -> bool { + guard(false, || { + let (s, g) = (triple(size)?, triple(per_group)?); + let (s, g) = (Size::new(s[0], s[1], s[2]), Size::new(g[0], g[1], g[2])); + let c = compute(encoder)?; + if threads { c.dispatch_threads(s, g) } else { c.dispatch_threadgroups(s, g) }.map_err(e("dispatch"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_render_set_pipeline(encoder: *mut NmbEncoder, pipeline: *mut NmbRenderPipeline) -> bool { + guard(false, || { + render(encoder)?.set_render_pipeline_state(&get(pipeline)?.0).map_err(e("render pipeline"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_render_set_depth_stencil(encoder: *mut NmbEncoder, state: *mut NmbDepthStencil) -> bool { + guard(false, || { + render(encoder)?.set_depth_stencil_state(optional(state)?.map(|s| &s.0)).map_err(e("depth/stencil state"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_render_set_viewport(encoder: *mut NmbEncoder, viewport: *const f64) -> bool { + guard(false, || { + if viewport.is_null() { + return Err("null viewport".into()); + } + // SAFETY: the header's six values. + let v = unsafe { std::slice::from_raw_parts(viewport, 6) }; + render(encoder)?.set_viewport(Viewport::new(v[0], v[1], v[2], v[3], v[4], v[5])).map_err(e("viewport"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_render_set_scissor(encoder: *mut NmbEncoder, rect: *const u64) -> bool { + guard(false, || { + if rect.is_null() { + return Err("null scissor".into()); + } + // SAFETY: the header's four values. + let r = unsafe { std::slice::from_raw_parts(rect, 4) }; + render(encoder)?.set_scissor_rect(ScissorRect::new(r[0], r[1], r[2], r[3])).map_err(e("scissor"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_render_set_raster(encoder: *mut NmbEncoder, cull_mode: u64, winding: u64, fill_mode: u64) -> bool { + guard(false, || { + let r = render(encoder)?; + r.set_cull_mode(raw::(cull_mode, "cull mode")?); + r.set_front_facing_winding(raw::(winding, "winding")?); + r.set_triangle_fill_mode(raw::(fill_mode, "fill mode")?).map_err(e("fill mode"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_render_set_stencil_reference(encoder: *mut NmbEncoder, front: u32, back: u32) -> bool { + guard(false, || { + render(encoder)?.set_stencil_reference_values(front, back); + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_render_set_blend_color(encoder: *mut NmbEncoder, color: *const f32) -> bool { + guard(false, || { + if color.is_null() { + return Err("null color".into()); + } + // SAFETY: the header's four values. + let c = unsafe { std::slice::from_raw_parts(color, 4) }; + render(encoder)?.set_blend_color(c[0], c[1], c[2], c[3]); + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_render_draw(encoder: *mut NmbEncoder, primitive: u64, start: u32, count: u32, instances: u32, base_instance: u32) + -> bool { + guard(false, || { + render(encoder)?.draw_primitives(raw::(primitive, "primitive")?, start, count, instances, base_instance) + .map_err(e("draw"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +#[allow(clippy::too_many_arguments)] +pub extern "C" fn nmb_render_draw_indexed(encoder: *mut NmbEncoder, primitive: u64, count: u32, index_type: u64, indices: *mut NmbBuffer, + offset: u64, instances: u32, base_vertex: i32, base_instance: u32) -> bool { + guard(false, || { + render(encoder)?.draw_indexed_primitives(raw::(primitive, "primitive")?, count, + raw::(index_type, "index type")?, &get(indices)?.0, offset, instances, + base_vertex, base_instance).map_err(e("indexed draw"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_blit_copy_buffer(encoder: *mut NmbEncoder, source: *mut NmbBuffer, source_offset: u64, destination: *mut NmbBuffer, + destination_offset: u64, size: u64) -> bool { + guard(false, || { + blit(encoder)?.copy_from_buffer_to_buffer(&get(source)?.0, source_offset, &get(destination)?.0, destination_offset, size) + .map_err(e("buffer copy"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_blit_fill_buffer(encoder: *mut NmbEncoder, buffer: *mut NmbBuffer, start: u64, end: u64, value: u8) -> bool { + guard(false, || { + blit(encoder)?.fill_buffer(&get(buffer)?.0, start..end, value).map_err(e("fill"))?; + Ok(true) + }) +} + +fn origin(v: *const u64) -> Result { + let v = triple(v)?; + Ok(Origin::new(v[0], v[1], v[2])) +} + +fn size(v: *const u64) -> Result { + let v = triple(v)?; + Ok(Size::new(v[0], v[1], v[2])) +} + +#[unsafe(no_mangle)] +#[allow(clippy::too_many_arguments)] +pub extern "C" fn nmb_blit_copy_texture(encoder: *mut NmbEncoder, source: *mut NmbTexture, source_slice: u64, source_level: u64, + source_origin: *const u64, extent: *const u64, destination: *mut NmbTexture, + destination_slice: u64, destination_level: u64, destination_origin: *const u64) -> bool { + guard(false, || { + blit(encoder)?.copy_from_texture_to_texture(&get(source)?.0, source_slice, source_level, origin(source_origin)?, size(extent)?, + &get(destination)?.0, destination_slice, destination_level, + origin(destination_origin)?).map_err(e("texture copy"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +#[allow(clippy::too_many_arguments)] +pub extern "C" fn nmb_blit_buffer_to_texture(encoder: *mut NmbEncoder, source: *mut NmbBuffer, offset: u64, bytes_per_row: u64, + bytes_per_image: u64, extent: *const u64, destination: *mut NmbTexture, slice: u64, + level: u64, destination_origin: *const u64) -> bool { + guard(false, || { + blit(encoder)?.copy_from_buffer_to_texture(&get(source)?.0, offset, bytes_per_row, bytes_per_image, size(extent)?, + &get(destination)?.0, slice, level, origin(destination_origin)?, BlitOption::None) + .map_err(e("buffer to texture"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +#[allow(clippy::too_many_arguments)] +pub extern "C" fn nmb_blit_texture_to_buffer(encoder: *mut NmbEncoder, source: *mut NmbTexture, slice: u64, level: u64, + source_origin: *const u64, extent: *const u64, destination: *mut NmbBuffer, offset: u64, + bytes_per_row: u64, bytes_per_image: u64) -> bool { + guard(false, || { + blit(encoder)?.copy_from_texture_to_buffer(&get(source)?.0, slice, level, origin(source_origin)?, size(extent)?, + &get(destination)?.0, offset, bytes_per_row, bytes_per_image, BlitOption::None) + .map_err(e("texture to buffer"))?; + Ok(true) + }) +} + +#[unsafe(no_mangle)] +pub extern "C" fn nmb_blit_generate_mipmaps(encoder: *mut NmbEncoder, texture: *mut NmbTexture) -> bool { + guard(false, || { + blit(encoder)?.generate_mipmaps(&get(texture)?.0).map_err(e("mipmaps"))?; + Ok(true) + }) +} diff --git a/drivers/NativeMetal/bridge/bridge_smoke.cpp b/drivers/NativeMetal/bridge/bridge_smoke.cpp new file mode 100644 index 0000000..b89220c --- /dev/null +++ b/drivers/NativeMetal/bridge/bridge_smoke.cpp @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: MIT +// Host run of the bridge self-test over the host Turnip and the host kernel stack (dyld-interposed IOKit shim). The software CP executes +// nothing, so result counts are not expected to be zero here; the run proves every step builds, submits and retires. +#include "NativeMetalBridge.h" +#include +#include +#include +static void mark(uint32_t value) {std::printf("mark %x\n",value);std::fflush(stdout);} +int main(int argc,char **argv) { + if(argc<3)return 2; + std::vector qc; + if(argc>3) { + FILE *f=std::fopen(argv[3],"rb");if(!f)return 3; + std::fseek(f,0,SEEK_END);qc.resize(size_t(std::ftell(f)));std::fseek(f,0,SEEK_SET); + if(std::fread(qc.data(),1,qc.size(),f)!=qc.size())return 4;std::fclose(f); + } + NmbSelftest r{}; + const uint32_t code=nmb_selftest(argv[1],argv[2],qc.data(),qc.size(),mark,&r); + std::printf("code %x stage %x compute_bad %u render_bad %u import_bad %u qc %u kernels %u/%u pairs %u/%u msg '%s'\n",code,r.stage, + r.compute_bad,r.render_bad,r.import_bad,r.qc_functions,r.qc_kernels_ok,r.qc_kernels_failed,r.qc_pairs_ok,r.qc_pairs_failed,r.message); + return code?1:0; +} diff --git a/drivers/NativeMetal/bridge/frontend_host.mm b/drivers/NativeMetal/bridge/frontend_host.mm new file mode 100644 index 0000000..6865cea --- /dev/null +++ b/drivers/NativeMetal/bridge/frontend_host.mm @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: MIT +// Host run of the frontend checks (FrontendChecks.h) over the bridge and a host Vulkan driver as the loader (MoltenVK on the Mac: an offline +// instrument; on the tablet the plugin runs the same checks over Turnip). Prints one PASS line or the failures. +#include "FrontendChecks.h" +#include +#include "../NativeBridgeMetalFrontend.h" +#include "NativeMetalObjects.h" + +static std::vector missing; +static void onMissing(const char *cls, const char *sel) {missing.push_back(std::string(cls) + " " + sel);} + +// argv: loader, shader cache, [QuartzCore default.metallib or ""] +int main(int argc, char **argv) { + @autoreleasepool { + if (argc < 3) return 2; + id device = native_bridge_metal_open(argv[1], argv[2], 0x6e6d62, onMissing); + if (!device) {std::printf("FAIL device\n"); return 1;} + size_t bytes = 0; + const void *air = nmb_fixture_air(&bytes); + id library = native_bridge_metal_library_with_air(air, bytes); + air = nmb_function_constant_air(&bytes); + id fcLibrary = native_bridge_metal_library_with_air(air, bytes); + if (!library || !fcLibrary) {std::printf("FAIL fixture libraries\n"); return 1;} + frontendchecks::Report r = {}; + frontendchecks::run(device, library, fcLibrary, argc > 3 && argv[3][0] ? @(argv[3]) : nil, &r); + if (r.qcFunctions) std::printf("QuartzCore: %u functions, kernels %u/%u, pairs %u/%u\n", r.qcFunctions, r.qcKernelsOk, r.qcKernels, r.qcPairsOk, r.qcPairs); + for (auto &m : missing) std::printf("missing selector: %s\n", m.c_str()); + if (r.failed) {std::printf("FAILED %u checks, first: %s\n", r.failed, r.first); return 1;} + std::printf("PASS: native Metal frontend over the bridge on %s: compute, render, IOSurface render, surface sampling, texture compute, " + "handlers, refusal, function constants, events and fences\n", device.name.UTF8String); + return 0; + } +} diff --git a/drivers/NativeMetal/bridge/plugin_sequence_host.mm b/drivers/NativeMetal/bridge/plugin_sequence_host.mm new file mode 100644 index 0000000..6dafabe --- /dev/null +++ b/drivers/NativeMetal/bridge/plugin_sequence_host.mm @@ -0,0 +1,30 @@ +// SPDX-License-Identifier: MIT +// The plugin's bridge phase (NativeMetalBridgePhase.h) in order over the host simulation: the bridge self-test, then the native frontend on a +// second device over the same Turnip and the frontend checks. The software CP executes nothing, so results are wrong by design; this +// proves every step builds, submits and retires over the host kernel stack and that the second device opens after the first is gone. +// IOSurface steps are skipped (the simulation's IOKit shim hides that service). Prints "frontend failed first <..> last ". +#include "bridge/FrontendChecks.h" +#include "bridge/NativeMetalBridge.h" +#include "bridge/NativeMetalObjects.h" +#include "NativeBridgeMetalFrontend.h" +static void mark(uint32_t v) {std::printf("mark %x\n", v); + std::fflush(stdout);} +static void missing(const char *c, const char *s) {std::printf("missing %s %s\n", c, s);} +int main(int, char **argv) { + NmbSelftest r{}; + uint32_t code = nmb_selftest(argv[1], argv[2], nullptr, 0, mark, &r); + std::printf("selftest code %x stage %x msg %s\n", code, r.stage, r.message); + id device = native_bridge_metal_open(argv[1], argv[2], 0, missing); + if (!device) {std::printf("open failed: %s\n", nmb_last_error()); return 1;} + size_t n = 0; + const void *a = nmb_fixture_air(&n); + id lib = native_bridge_metal_library_with_air(a, n); + a = nmb_function_constant_air(&n); + id fc = native_bridge_metal_library_with_air(a, n); + frontendchecks::Report rep = {}; + rep.mark = mark; + rep.hostSimulation = true; + frontendchecks::run(device, lib, fc, nil, &rep); + std::printf("frontend failed %u first %s last %x\n", rep.failed, rep.first, rep.last); + return 0; +} diff --git a/drivers/NativeMetal/shaders/smoke_fill.comp b/drivers/NativeMetal/shaders/smoke_fill.comp new file mode 100644 index 0000000..42f392a --- /dev/null +++ b/drivers/NativeMetal/shaders/smoke_fill.comp @@ -0,0 +1,9 @@ +#version 450 +// Shader-execution check: every invocation writes a value derived from its global index and two push constants. +layout(local_size_x = 64) in; +layout(std430, set = 0, binding = 0) buffer Out { uint v[]; } o; +layout(push_constant) uniform P { uint mul; uint add; } p; +void main() { + uint i = gl_GlobalInvocationID.x; + o.v[i] = i * p.mul + p.add; +} diff --git a/drivers/NativeMetal/shaders/smoke_triangle.frag b/drivers/NativeMetal/shaders/smoke_triangle.frag new file mode 100644 index 0000000..908bb17 --- /dev/null +++ b/drivers/NativeMetal/shaders/smoke_triangle.frag @@ -0,0 +1,4 @@ +#version 450 +layout(location = 0) out vec4 color; +layout(push_constant) uniform P { vec4 color; } p; +void main() { color = p.color; } diff --git a/drivers/NativeMetal/shaders/smoke_triangle.vert b/drivers/NativeMetal/shaders/smoke_triangle.vert new file mode 100644 index 0000000..9aa6974 --- /dev/null +++ b/drivers/NativeMetal/shaders/smoke_triangle.vert @@ -0,0 +1,6 @@ +#version 450 +// Rasterisation check: one triangle from the vertex index, no vertex buffers. +void main() { + const vec2 p[3] = vec2[](vec2(-0.5, -0.5), vec2(0.5, -0.5), vec2(0.0, 0.5)); + gl_Position = vec4(p[gl_VertexIndex], 0.0, 1.0); +} diff --git a/drivers/NativeMetal/tools/MetalVulkanProbe.rs b/drivers/NativeMetal/tools/MetalVulkanProbe.rs new file mode 100644 index 0000000..11321fa --- /dev/null +++ b/drivers/NativeMetal/tools/MetalVulkanProbe.rs @@ -0,0 +1,143 @@ +// SPDX-License-Identifier: BSD-2-Clause-Patent +// Offline consumer of a pinned translator's public API; never opens a GPU. +use metal2vulkan::device::{Device, DeviceDescription, DevicePolicy, SamplingPolicy}; +use metal2vulkan::library::{FunctionStage, Library}; +use metal2vulkan::metal::PixelFormat; +use metal2vulkan::pipeline::{ + ColorAttachmentDescriptor, ComputePipelineDescriptor, FunctionDescriptor, + RenderPipelineDescriptor, StageModule, StageSource, VertexDescriptor, +}; +use metal2vulkan::Budget; +use serde_json::{json, Value}; +use std::{collections::BTreeMap, env, fs, process}; + +fn stage(module: &StageModule) -> Value { + json!({ + "stage": format!("{:?}", module.stage), + "entry_point": module.entry_point, + "spirv": &*module.spirv, + "precision_notes": module.precision_notes.iter().map(|n| format!("{n:?}")).collect::>(), + "required_subgroup_size": module.required_subgroup_size, + "specialization": format!("{:?}", module.specialization), + "constant_blob_bytes": module.constant_blob.as_ref().map_or(0, |b| b.len()), + }) +} + +fn run(args: &[String]) -> Result> { + if args.len() == 2 && args[0] == "--list" { + let library = Library::parse(&fs::read(&args[1])?)?; + return Ok(json!({"functions": library.functions().map(|f| json!({ + "name": f.name, "stage": format!("{:?}", f.stage) + })).collect::>(), "gpu_execution": false})); + } + let mut args = args; + let mut vertex_descriptor = None; + let mut color_attachments = None; + while args.first().is_some_and(|v| v.starts_with("--")) { + let path = args.get(1).ok_or("missing descriptor path")?; + match args[0].as_str() { + "--vertex-descriptor" if vertex_descriptor.is_none() => { + vertex_descriptor = Some(serde_json::from_slice::(&fs::read( + path, + )?)?); + } + "--color-attachments" if color_attachments.is_none() => { + let colors = + serde_json::from_slice::>(&fs::read(path)?)?; + if colors.is_empty() || colors.keys().any(|index| *index >= 8) { + return Err("color attachments require indices in 0..8".into()); + } + color_attachments = Some(colors); + } + _ => return Err("unknown or duplicate descriptor option".into()), + } + args = &args[2..]; + } + if args.len() != 3 && args.len() != 5 { + return Err( + "expected --list INPUT, or PROFILE INPUT FUNCTION [FRAGMENT_INPUT FUNCTION]".into(), + ); + } + let description = DeviceDescription::from_khronos_json(&fs::read_to_string(&args[0])?, None)?; + let mut policy = DevicePolicy::default().exact_answers(); + // exact_answers deliberately keeps sampling unchanged in this upstream revision. + policy.sampling = SamplingPolicy::Exact; + let device = Device::new(description, policy)?; + let library = Library::parse(&fs::read(&args[1])?)?; + let function = library.function(&args[2])?; + let budget = Budget::standard(); + let (stages, layout, interface, dependencies, requirements) = match function.info().stage { + FunctionStage::Kernel if args.len() == 3 => { + if vertex_descriptor.is_some() || color_attachments.is_some() { + return Err("compute pipeline does not take render descriptors".into()); + } + let descriptor = ComputePipelineDescriptor::new(StageSource::Function( + FunctionDescriptor::new(function), + )); + let p = device.compile_compute_pipeline(&descriptor, &budget)?; + ( + vec![stage(&p.stage)], + format!("{:?}", p.layout), + format!("{:?}", p.interface), + format!("{:?}", p.dependencies), + format!("{:?}", p.requirements), + ) + } + FunctionStage::Vertex => { + let mut descriptor = RenderPipelineDescriptor::new(StageSource::Function( + FunctionDescriptor::new(function), + )); + descriptor.vertex_descriptor = vertex_descriptor; + if args.len() == 5 { + let fragment_library = Library::parse(&fs::read(&args[3])?)?; + let fragment = fragment_library.function(&args[4])?; + if fragment.info().stage != FunctionStage::Fragment { + return Err("fragment input is not a fragment function".into()); + } + descriptor.fragment = + Some(StageSource::Function(FunctionDescriptor::new(fragment))); + let colors = color_attachments + .as_ref() + .ok_or("fragment translation requires explicit color attachments")?; + for (&index, &format) in colors { + descriptor.color_attachments[index] = + Some(ColorAttachmentDescriptor::new(format)); + } + } else if color_attachments.is_some() { + return Err("color attachments require a fragment function in this probe".into()); + } + let p = device.compile_render_pipeline(&descriptor, &budget)?; + ( + p.stages.iter().map(stage).collect(), + format!("{:?}", p.layout), + format!("{:?}", p.interface), + format!("{:?}", p.dependencies), + format!("{:?}", p.requirements), + ) + } + _ => { + return Err( + "stage needs a matching pipeline descriptor; standalone fragment refused".into(), + ) + } + }; + Ok(json!({ + "stages": stages, "layout": layout, "interface": interface, + "dependencies": dependencies, "requirements": requirements, + "policy": "exact_answers+exact_sampling", + "validation_envs": device.profile().spirv_validation().target_envs, + "validation_relaxations": device.profile().spirv_validation().relaxations, + "profile_is_hardware_observation": false, + "gpu_execution": false, "metal_device_published": false, + })) +} + +fn main() { + match run(&env::args().skip(1).collect::>()) { + Ok(value) => println!("{value}"), + Err(error) => { + eprintln!("REFUSED: {error}"); + process::exit(2); + } + } +} diff --git a/drivers/NativeMetal/tools/metal-vulkan-probe.lock b/drivers/NativeMetal/tools/metal-vulkan-probe.lock new file mode 100644 index 0000000..3800b8d --- /dev/null +++ b/drivers/NativeMetal/tools/metal-vulkan-probe.lock @@ -0,0 +1,208 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "libc" +version = "0.2.191" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "metal2vulkan" +version = "0.2.0" +dependencies = [ + "metal2vulkan-air", + "serde", + "serde_json", + "sha2", +] + +[[package]] +name = "metal2vulkan-air" +version = "0.2.0" +dependencies = [ + "serde", + "sha2", +] + +[[package]] +name = "native-metal-vulkan-probe" +version = "0.1.0" +dependencies = [ + "metal2vulkan", + "serde_json", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.154" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "syn" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/drivers/NativeMetal/tools/probe-coverage-attachment.json b/drivers/NativeMetal/tools/probe-coverage-attachment.json new file mode 100644 index 0000000..d75ed15 --- /dev/null +++ b/drivers/NativeMetal/tools/probe-coverage-attachment.json @@ -0,0 +1 @@ +{"2": "R32Float"} diff --git a/drivers/NativeMetal/tools/probe-vertex-layout.json b/drivers/NativeMetal/tools/probe-vertex-layout.json new file mode 100644 index 0000000..72deaf8 --- /dev/null +++ b/drivers/NativeMetal/tools/probe-vertex-layout.json @@ -0,0 +1,10 @@ +{ + "layouts": { + "0": {"stride": 48, "step_function": "PerVertex", "step_rate": 1} + }, + "attributes": { + "0": {"format": "Float4", "offset": 0, "buffer_index": 0}, + "1": {"format": "Float2", "offset": 16, "buffer_index": 0}, + "3": {"format": "Float4", "offset": 32, "buffer_index": 0} + } +} diff --git a/drivers/NativeMobileDisplay/LICENSE b/drivers/NativeMobileDisplay/LICENSE new file mode 100644 index 0000000..217b4df --- /dev/null +++ b/drivers/NativeMobileDisplay/LICENSE @@ -0,0 +1,51 @@ +Copyright (c) 2026, ArmVirtPlatform contributors. All rights reserved. + +SPDX-License-Identifier: BSD-2-Clause-Patent + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +Subject to the terms and conditions of this license, each copyright holder +and contributor hereby grants to those receiving rights under this license +a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable +(except for failure to satisfy the conditions of this license) patent +license to make, have made, use, offer to sell, sell, import, and otherwise +transfer this software, where such license applies only to those patent +claims, already acquired or hereafter acquired, licensable by such copyright +holder or contributor that are necessarily infringed by: + +(a) their Contribution(s) (the licensed copyrights of copyright holders and + non-copyrightable additions of contributors, in source or binary form) + alone; or + +(b) combination of their Contribution(s) with the work of authorship to + which such Contribution(s) was added by such copyright holder or + contributor, if, at the time the Contribution is added, such addition + causes such combination to be necessarily infringed. The patent license + shall not apply to any other combinations which include the + Contribution. + +Except as expressly stated above, no rights or licenses from any copyright +holder or contributor is granted under this license, whether expressly, by +implication, estoppel or otherwise. + +DISCLAIMER + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. diff --git a/drivers/NativeMobileDisplay/NativeGOPPresenter.cpp b/drivers/NativeMobileDisplay/NativeGOPPresenter.cpp new file mode 100644 index 0000000..63ea825 --- /dev/null +++ b/drivers/NativeMobileDisplay/NativeGOPPresenter.cpp @@ -0,0 +1,143 @@ +// SPDX-License-Identifier: BSD-2-Clause-Patent +// Transfer completed GPU pixels to actual firmware scanout. No CPU rendering, +// fabricated refresh event, Metal discovery or original display publication. +#include +#include +#include +#include +#include +#include +#include "NativeGOPWire.h" +extern "C" { +volatile uint32_t native_gop_ready,native_gop_failures; +volatile uint64_t native_gop_transfers,native_gop_bytes,native_gop_physical,native_gop_virtual; +IOService *native_gop_presenter_create(void); +} +class NativeGOPClient; +class NativeGOPPresenter : public IOService { + OSDeclareDefaultStructors(NativeGOPPresenter) + friend class NativeGOPClient; + IOMemoryMap *mapping=nullptr; + IOLock *gate=nullptr; + IOService *owner=nullptr; + NGPMode mode={}; + bool online=false; + IOReturn transfer(task_t,const NGPPresent &,NGPTransfer *); +public: + bool start(IOService *) override; + void stop(IOService *) override; + void free() override; + bool handleOpen(IOService *client,IOOptionBits,void *) override {if(owner)return false;owner=client;return true;} + void handleClose(IOService *client,IOOptionBits) override {if(owner==client)owner=nullptr;} + bool handleIsOpen(const IOService *client) const override {return client?owner==client:owner!=nullptr;} + IOReturn newUserClient(task_t,void *,UInt32,OSDictionary *,IOUserClient **) override; +}; +class NativeGOPClient : public IOUserClient { + OSDeclareDefaultStructors(NativeGOPClient) + NativeGOPPresenter *controller=nullptr; + task_t task=TASK_NULL; + bool opened=false; + static IOReturn modeAction(OSObject *,void *,IOExternalMethodArguments *); + static IOReturn presentAction(OSObject *,void *,IOExternalMethodArguments *); +public: + bool initWithTask(task_t,void *,UInt32,OSDictionary *) override; + bool start(IOService *) override; + void stop(IOService *) override; + void free() override; + IOReturn clientClose() override {terminate();return kIOReturnSuccess;} + IOReturn externalMethod(uint32_t,IOExternalMethodArguments *,IOExternalMethodDispatch *,OSObject *,void *) override; + IOReturn clientMemoryForType(UInt32,IOOptionBits *,IOMemoryDescriptor **) override; +}; +OSDefineMetaClassAndStructors(NativeGOPPresenter,IOService) +OSDefineMetaClassAndStructors(NativeGOPClient,IOUserClient) +IOService *native_gop_presenter_create(void) {return new NativeGOPPresenter;} +bool NativeGOPPresenter::start(IOService *provider) { + if(!IOService::start(provider))return false; + PE_Video video={}; + if(PE_current_console(&video) || !video.v_baseAddr || !video.v_width || !video.v_height || + video.v_width>16384 || video.v_height>16384 || video.v_depth!=32 || video.v_rotate || + video.v_rowBytesUINT64_MAX/video.v_height)return false; + mapping=provider->mapDeviceMemoryWithIndex(0,kIOMapInhibitCache); + if(!mapping || !mapping->getVirtualAddress() || mapping->getPhysicalAddress()!=video.v_baseAddr || + mapping->getLength()!=video.v_rowBytes*video.v_height)return false; + gate=IOLockAlloc();if(!gate)return false; + mode={NGP_VERSION,NGP_BGRX8888,video.v_width,video.v_height,video.v_rowBytes,mapping->getLength()}; + native_gop_physical=mapping->getPhysicalAddress();native_gop_virtual=mapping->getVirtualAddress(); + online=true;native_gop_ready=1;registerService();return true; +} +void NativeGOPPresenter::stop(IOService *provider) { + if(gate)IOLockLock(gate);online=false;native_gop_ready=0;if(gate)IOLockUnlock(gate); + IOService::stop(provider); +} +void NativeGOPPresenter::free() { + OSSafeReleaseNULL(mapping);if(gate){IOLockFree(gate);gate=nullptr;}IOService::free(); +} +IOReturn NativeGOPPresenter::transfer(task_t task,const NGPPresent &q,NGPTransfer *result) { + if(!q.address || q.bytes!=mode.bytes || q.width!=mode.width || q.height!=mode.height || + q.stride!=mode.stride || q.format!=mode.format || q.address>UINT64_MAX-q.bytes)return kIOReturnBadArgument; + IOMemoryDescriptor *input=IOMemoryDescriptor::withAddressRange(q.address,q.bytes,kIODirectionIn,task); + if(!input)return kIOReturnNoMemory; + IOReturn status=input->prepare(kIODirectionIn); + if(status==kIOReturnSuccess) { + IOLockLock(gate); + if(!online)status=kIOReturnNotReady; + else if(input->readBytes(0,reinterpret_cast(mapping->getVirtualAddress()),q.bytes)!=q.bytes)status=kIOReturnUnderrun; + else { + __asm__ volatile("dsb oshst" ::: "memory"); + native_gop_transfers=native_gop_transfers+1;result->sequence=native_gop_transfers; + result->bytes=q.bytes;native_gop_bytes=native_gop_bytes+q.bytes; + } + IOLockUnlock(gate); + IOReturn completion=input->complete(kIODirectionIn);if(status==kIOReturnSuccess)status=completion; + } + input->release();if(status!=kIOReturnSuccess)__atomic_fetch_add(&native_gop_failures,1,__ATOMIC_RELAXED);return status; +} +IOReturn NativeGOPPresenter::newUserClient(task_t task,void *security,UInt32 type,OSDictionary *properties,IOUserClient **out) { + if(!out || type)return kIOReturnBadArgument; + *out=nullptr;NativeGOPClient *client=new NativeGOPClient;if(!client)return kIOReturnNoMemory; + bool attached=false; + if(!client->initWithTask(task,security,type,properties) || !(attached=client->attach(this)) || !client->start(this)) { + if(attached)client->detach(this);client->release();return kIOReturnNotOpen; + } + *out=client;return kIOReturnSuccess; +} +bool NativeGOPClient::initWithTask(task_t ownerTask,void *security,UInt32 type,OSDictionary *properties) { + if(!ownerTask || type || clientHasPrivilege(security,kIOClientPrivilegeAdministrator)!=kIOReturnSuccess || + !IOUserClient::initWithTask(ownerTask,security,type,properties))return false; + task=ownerTask;task_reference(task);return true; +} +bool NativeGOPClient::start(IOService *provider) { + controller=OSDynamicCast(NativeGOPPresenter,provider); + if(!controller || !IOUserClient::start(provider))return false; + opened=controller->open(this);return opened; +} +void NativeGOPClient::stop(IOService *provider) { + if(opened){controller->close(this);opened=false;}controller=nullptr;IOUserClient::stop(provider); +} +void NativeGOPClient::free() {if(task){task_deallocate(task);task=TASK_NULL;}IOUserClient::free();} +IOReturn NativeGOPClient::modeAction(OSObject *target,void *,IOExternalMethodArguments *a) { + NativeGOPClient *client=OSDynamicCast(NativeGOPClient,target); + if(!client || !client->opened || !client->controller)return kIOReturnNotOpen; + if(a->structureOutputDescriptor || !a->structureOutput)return kIOReturnUnsupported; + memcpy(a->structureOutput,&client->controller->mode,sizeof(NGPMode));return kIOReturnSuccess; +} +IOReturn NativeGOPClient::presentAction(OSObject *target,void *,IOExternalMethodArguments *a) { + NativeGOPClient *client=OSDynamicCast(NativeGOPClient,target); + if(!client || !client->opened || !client->controller)return kIOReturnNotOpen; + if(a->structureInputDescriptor || a->structureOutputDescriptor || !a->structureInput || !a->structureOutput)return kIOReturnUnsupported; + NGPPresent q={};NGPTransfer result={};memcpy(&q,a->structureInput,sizeof(q)); + IOReturn status=client->controller->transfer(client->task,q,&result); + if(status==kIOReturnSuccess)memcpy(a->structureOutput,&result,sizeof(result));return status; +} +IOReturn NativeGOPClient::externalMethod(uint32_t selector,IOExternalMethodArguments *a,IOExternalMethodDispatch *,OSObject *,void *) { + static const IOExternalMethodDispatch methods[]={{modeAction,0,0,0,sizeof(NGPMode)}, + {presentAction,0,sizeof(NGPPresent),0,sizeof(NGPTransfer)}}; + if(selector>=2)return kIOReturnUnsupported; + return IOUserClient::externalMethod(selector,a,const_cast(methods+selector),this,nullptr); +} +IOReturn NativeGOPClient::clientMemoryForType(UInt32 type,IOOptionBits *options,IOMemoryDescriptor **memory) { + if(type || !options || !memory)return kIOReturnBadArgument; + *memory=nullptr;if(!opened || !controller || !controller->online)return kIOReturnNotReady; + *memory=controller->mapping->getMemoryDescriptor();if(!*memory)return kIOReturnNoMemory; + (*memory)->retain();*options|=kIOMapReadOnly;return kIOReturnSuccess; +} diff --git a/drivers/NativeMobileDisplay/NativeGOPWire.h b/drivers/NativeMobileDisplay/NativeGOPWire.h new file mode 100644 index 0000000..e7a7668 --- /dev/null +++ b/drivers/NativeMobileDisplay/NativeGOPWire.h @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: BSD-2-Clause-Patent +// Actual firmware scanout transfer, not a compositor or hardware vblank ABI. +#pragma once +#include +enum { NGP_BGRX8888=1,NGP_VERSION=1 }; +struct NGPMode {uint32_t version,format;uint64_t width,height,stride,bytes;}; +struct NGPPresent {uint64_t address,bytes;uint32_t width,height,stride,format;}; +struct NGPTransfer {uint64_t sequence,bytes;}; +static_assert(sizeof(NGPMode)==40 && sizeof(NGPPresent)==32 && sizeof(NGPTransfer)==16,"GOP transfer ABI"); diff --git a/drivers/NativeMobileDisplay/NativeLegacyDisplay.cpp b/drivers/NativeMobileDisplay/NativeLegacyDisplay.cpp new file mode 100644 index 0000000..8813f45 --- /dev/null +++ b/drivers/NativeMobileDisplay/NativeLegacyDisplay.cpp @@ -0,0 +1,323 @@ +#include "NativeLegacyRequest.h" +// SPDX-License-Identifier: BSD-2-Clause-Patent +// Reuse the unchanged original generic framebuffer implementation. Unknown +// backend leaves stop; this class is not yet a published system display. +#include "OriginalLegacyFramebufferABI.h" +#include +#include +#include +#include +#include +#include +extern "C" __attribute__((noreturn)) void native_legacy_backend_guard(unsigned,void *,void *); +extern "C" IOService *native_legacy_display_allocate(void); +extern "C" bool native_legacy_gop_mapping_observed(IOService *,IOMemoryMap *,uint32_t,uint32_t); +extern "C" void native_legacy_registration_observed(IOService *); +extern "C" void native_legacy_client_enter(task_t,uint32_t); +extern "C" void native_legacy_client_observed(IOReturn,IOUserClient *); +extern "C" void native_legacy_pm_observed(unsigned,IOReturn,unsigned long); +extern "C" IOMemoryDescriptor *native_legacy_surface_descriptor(OSObject *); +extern "C" uint64_t native_legacy_surface_width(OSObject *),native_legacy_surface_height(OSObject *); +extern "C" uint64_t native_legacy_surface_stride(OSObject *),native_legacy_surface_allocation(OSObject *); +extern "C" uint32_t native_legacy_surface_format(OSObject *),native_legacy_surface_planes(OSObject *); +extern "C" void native_legacy_frame_observed(unsigned kind,uint64_t count,uint64_t minx,uint64_t miny,uint64_t maxx,uint64_t maxy); +extern "C" void native_legacy_frame_colors_observed(const uint32_t *colors,const uint64_t *counts,uint64_t other); +extern "C" void native_legacy_swap_validation_observed(bool,unsigned,const void *,const void *); +extern "C" bool native_legacy_prepare_scanout(void); +extern "C" void native_legacy_transfer_observed(IOReturn,const void *,uint64_t); +extern "C" void native_legacy_copy_timing_observed(uint64_t ticks,uint64_t bytes); +extern "C" void native_legacy_retirement_observed(IOService *); +class NativeLegacyDisplay : public IOMobileFramebufferLegacy { + OSDeclareDefaultStructors(NativeLegacyDisplay) + IOMemoryMap *bootMapping=nullptr; + uint32_t scanBudget=32; + void *shadow=nullptr;uint64_t shadowBytes=0;bool shadowValid=false; + bool primary=false; + bool logicalPM=false; + bool transferred=false; + const OriginalLegacySwapRequest *validatedRequest=nullptr; +public: + void free() override {if(logicalPM){PMstop();logicalPM=false;}primary=false;OSSafeReleaseNULL(bootMapping);IOMobileFramebufferLegacy::free();} + void stop(IOService *provider) override { + transferred=false;validatedRequest=nullptr; + if(logicalPM){PMstop();logicalPM=false;} + primary=false;IOMobileFramebufferLegacy::stop(provider);OSSafeReleaseNULL(bootMapping); + } + bool start(IOService *provider) override { + if(!IOMobileFramebufferLegacy::start(provider))return false; + unsigned enabled=0;PE_parse_boot_argn("offline-legacy-pm",&enabled,sizeof(enabled)); + if(!enabled)return true; + unsigned phase=0;PE_parse_boot_argn("offline-legacy-phase",&phase,sizeof(phase)); + if(enabled!=1 || phase!=4 || !provider || !bootMapping || !primary) { + native_legacy_pm_observed(1,kIOReturnNotReady,0);IOMobileFramebufferLegacy::stop(provider);return false; + } + // Actual IOKit PM lifecycle for logical scanout scheduling only. GOP + // provides no panel power switch: neither state advertises physical + // power output, power control or a hardware power transition. + static IOPMPowerState states[2]={}; + states[0].version=states[1].version=1; + states[1].capabilityFlags=kIOPMDeviceUsable; + PMinit();logicalPM=initialized; + if(!logicalPM){native_legacy_pm_observed(1,kIOReturnNotReady,0);IOMobileFramebufferLegacy::stop(provider);return false;} + provider->joinPMtree(this); + IOReturn status=registerPowerDriver(this,states,2);native_legacy_pm_observed(1,status,0); + if(status==kIOReturnSuccess){status=changePowerStateToPriv(1);native_legacy_pm_observed(2,status,1);} + if(status!=kIOReturnSuccess){PMstop();logicalPM=false;IOMobileFramebufferLegacy::stop(provider);return false;} + return true; + } + IOReturn setPowerState(unsigned long ordinal,IOService *device) override { + if(!logicalPM)return IOMobileFramebufferLegacy::setPowerState(ordinal,device); + if(ordinal>1 || !bootMapping){native_legacy_pm_observed(3,kIOReturnUnsupported,ordinal);return kIOReturnUnsupported;} + native_legacy_pm_observed(3,kIOReturnSuccess,ordinal); + IOReturn status=IOMobileFramebufferLegacy::setPowerState(ordinal,device); + native_legacy_pm_observed(4,status,ordinal);return status; + } + // The explicit offline system-client experiment publishes only the real + // GOP mode. Unknown backend calls retain their stop-with-caller guards. + void registerService(IOOptionBits options=0) override { + unsigned enabled=0;PE_parse_boot_argn("offline-legacy-register",&enabled,sizeof(enabled)); + if(enabled!=1)native_legacy_backend_guard(84,this,__builtin_return_address(0)); + IOService::registerService(options);native_legacy_registration_observed(this); + } + bool isMainDisplay() override {return primary;} + void kickSwapQueue() override { + unsigned enabled=0;PE_parse_boot_argn("offline-legacy-transfer",&enabled,sizeof(enabled)); + if(enabled!=1 || transferred || !logicalPM || !bootMapping || !primary)return; + const OriginalLegacySwapRequest *request=pendingSwapRequest(); + if(!request || request!=validatedRequest)return; + // This bounded first-frame transfer preserves the original queue's + // resource ownership. Separately gated retirement follows the copy; + // neither path signals a hardware fence, announces a vblank or + // claims general producer synchronization. + // The owned GPU instrument completes its work before submission. + // The separate original software-renderer experiment uses completed + // synchronous CPU surfaces, with no GPU or display fence fabricated. + OSObject *surface=nullptr; + memcpy(&surface,reinterpret_cast(request)+2624,sizeof(surface)); + if(!surface) { + // Validated blank swap (all layers disabled over black): solid-fill the actual GOP. + PE_Video blank={}; + uint32_t bw=0,bh=0;getDisplayDimensions(&bw,&bh); + uint64_t blankLength=(uint64_t)bw*bh*4; + if(PE_current_console(&blank) || blank.v_baseAddr!=bootMapping->getPhysicalAddress() || + blank.v_width!=bw || blank.v_height!=bh || blank.v_depth!=32 || blank.v_rotate || + !native_legacy_surface_bounds(bw,bh,blank.v_rowBytes,bootMapping->getLength(),bootMapping->getLength())) { + native_legacy_transfer_observed(kIOReturnNotReady,request,0);return; + } + if(!native_legacy_prepare_scanout()) {native_legacy_transfer_observed(kIOReturnNotReady,request,0);return;} + if(!native_legacy_fill_pixels(reinterpret_cast(bootMapping->getVirtualAddress()), + bootMapping->getLength(),blank.v_rowBytes,bw,bh,0xff000000U)) { + native_legacy_transfer_observed(kIOReturnBadArgument,request,0);return; + } + __asm__ volatile("dsb sy" ::: "memory"); + shadowValid=false; + transferred=true; + native_legacy_frame_observed(1,0,0,0,0,0); + {const uint32_t colors[4]={0xff000000U,0,0,0};const uint64_t counts[4]={(uint64_t)bw*bh,0,0,0};native_legacy_frame_colors_observed(colors,counts,0);} + native_legacy_transfer_observed(kIOReturnSuccess,request,blankLength); + finishTransfer(request); + return; + } + if(!surface->metaCast("IOSurface"))return; + IOMemoryDescriptor *memory=native_legacy_surface_descriptor(surface); + PE_Video video={}; + uint32_t width=0,height=0;getDisplayDimensions(&width,&height); + uint64_t length=(uint64_t)width*height*4; + uint64_t sourceStride=native_legacy_surface_stride(surface); + uint64_t allocation=native_legacy_surface_allocation(surface); + if(!memory || !memory->metaCast("IOMemoryDescriptor") || !native_legacy_surface_bounds(width,height,sourceStride,allocation,memory->getLength()) || + native_legacy_surface_width(surface)!=width || native_legacy_surface_height(surface)!=height || + native_legacy_surface_format(surface)!=0x42475241 || native_legacy_surface_planes(surface) || + PE_current_console(&video) || video.v_baseAddr!=bootMapping->getPhysicalAddress() || + video.v_width!=width || video.v_height!=height || video.v_depth!=32 || video.v_rotate || + !native_legacy_surface_bounds(width,height,video.v_rowBytes,bootMapping->getLength(),bootMapping->getLength())) { + native_legacy_transfer_observed(kIOReturnNotReady,request,0);return; + } + IOMemoryMap *source=memory->map(kIOMapAnywhere|kIOMapReadOnly); + if(!source || !source->getVirtualAddress() || source->getLength()(bootMapping->getVirtualAddress()), + bootMapping->getLength(),video.v_rowBytes,reinterpret_cast(source->getVirtualAddress()), + source->getLength(),sourceStride,width,height)) { + OSSafeReleaseNULL(source);native_legacy_transfer_observed(kIOReturnBadArgument,request,0);return; + } + // The two content scans read every pixel again; they only describe the first frames (the probe keeps 32). + if(scanBudget) { + --scanBudget; + uint64_t scan[5]; + if(native_legacy_frame_scan(reinterpret_cast(source->getVirtualAddress()),source->getLength(),sourceStride,width,height,scan)) + native_legacy_frame_observed(0,scan[0],scan[1],scan[2],scan[3],scan[4]); + uint32_t colors[4];uint64_t counts[4],other=0; + if(native_legacy_frame_colors(reinterpret_cast(source->getVirtualAddress()),source->getLength(),sourceStride,width,height,colors,counts,&other)) + native_legacy_frame_colors_observed(colors,counts,other); + } + if(!native_legacy_prepare_scanout()) { + OSSafeReleaseNULL(source);native_legacy_transfer_observed(kIOReturnNotReady,request,0);return; + } + uint64_t copyStart;__asm__ volatile("isb\n\tmrs %0, cntpct_el0":"=r"(copyStart)); + uint64_t stored=UINT64_MAX; + { + // Store only what changed since the previous frame (the scanout memory is slow device memory). + uint64_t need=(uint64_t)width*height*4; + // Differential copy is opt-in (sheng-fb-diff=1): on hardware it reads the whole source every frame and was not faster + // than the plain copy (143-285 ms per swap either way), and the first boots with it were followed by an unresponsive + // lock screen that disappeared with the plain copy (int-005). + static int diffOff=-1; + if(diffOff<0){unsigned v=0;PE_parse_boot_argn("sheng-fb-diff",&v,sizeof v);diffOff=(v!=1);} + if(!diffOff && !shadow && need<=(64ull<<20)){shadow=IOMallocAligned((vm_size_t)need,16384);if(shadow){shadowBytes=need;shadowValid=false;}} + if(!diffOff && shadow && shadowBytes>=need) + stored=native_legacy_copy_diff(reinterpret_cast(bootMapping->getVirtualAddress()), + bootMapping->getLength(),video.v_rowBytes,reinterpret_cast(source->getVirtualAddress()), + source->getLength(),sourceStride,width,height,shadow,shadowValid); + if(stored!=UINT64_MAX)shadowValid=true; + } + if(stored==UINT64_MAX) { + shadowValid=false; + if(!native_legacy_copy_pixels(reinterpret_cast(bootMapping->getVirtualAddress()), + bootMapping->getLength(),video.v_rowBytes,reinterpret_cast(source->getVirtualAddress()), + source->getLength(),sourceStride,width,height)) { + OSSafeReleaseNULL(source);native_legacy_transfer_observed(kIOReturnBadArgument,request,0);return; + } + stored=length; + } + __asm__ volatile("dsb sy" ::: "memory"); + {uint64_t copyEnd;__asm__ volatile("isb\n\tmrs %0, cntpct_el0":"=r"(copyEnd));native_legacy_copy_timing_observed(copyEnd-copyStart,stored);} + OSSafeReleaseNULL(source);transferred=true; + native_legacy_transfer_observed(kIOReturnSuccess,request,length); + finishTransfer(request); + } + // Separately gated retirement after a real scanout store, shared by copied and blank frames. + void finishTransfer(const OriginalLegacySwapRequest *request) { + unsigned retire=0,cpuPresent=0; + PE_parse_boot_argn("offline-legacy-retire",&retire,sizeof(retire)); + PE_parse_boot_argn("offline-legacy-cpu-present",&cpuPresent,sizeof(cpuPresent)); + if(retire==1 || cpuPresent==1) { + // Only after the real copy and store barrier: select the actual + // pending head and let the unchanged family release it and wake + // waiters. This reports memory scanout completion, not a panel + // vblank or hardware fence. Producer completion must precede + // submission; this does not implement asynchronous GPU sync. + if(!activateOnlyPendingSwap(request))return; + validatedRequest=nullptr; + IOMobileFramebufferLegacy::completeCurrentSwap(); + native_legacy_retirement_observed(this); + // The unchanged family has retired this generation and released + // its current reference. Permit the next independently validated + // single pending frame. Keep the latch set while completion can + // reenter this method, and whenever retirement remains disabled. + transferred=false; + } + } + IOReturn collectCompletionReports(uint32_t,void *,uint32_t *count,uint32_t) override { + // The original completion helper passes null optional outputs and + // ignores this IOReturn. GOP exposes no hardware completion telemetry. + if(count)*count=0; + return kIOReturnUnsupported; + } + bool validateSwap(const OriginalLegacySwapRequest *request) override { + // Qualified original request fields are packed; use memcpy for each + // access. Accept only one full-frame, untransformed, nonplanar BGRA + // surface with real original backing memory. No generic capability + // or success is supplied for other layer/format/geometry contracts. + auto bytes=reinterpret_cast(request); + auto word=[&](unsigned offset){uint32_t value=0;memcpy(&value,bytes+offset,4);return value;}; + OSObject *surface=nullptr;unsigned reason=1;bool accepted=false; + unsigned enabled=0;PE_parse_boot_argn("offline-legacy-validate",&enabled,sizeof(enabled)); + if(enabled==1 && logicalPM && primary && bootMapping && request) { + reason=2;OSObject *surfaces[4]={};memcpy(surfaces,bytes+2624,sizeof(surfaces));surface=surfaces[0]; + uint32_t width=0,height=0;getDisplayDimensions(&width,&height); + if(surface && surface->metaCast("IOSurface") && !surfaces[1] && !surfaces[2] && !surfaces[3] && + native_legacy_single_layer_state(word(792),word(796),word(800))) { + reason=3;bool rectangles=true; + const unsigned rectOffsets[]={17,81}; + for(unsigned offset: rectOffsets) { + uint32_t rect[4]={};memcpy(rect,bytes+offset,sizeof(rect)); + rectangles=rectangles && rect[0]==0 && rect[1]==0 && rect[2]==width && rect[3]==height; + } + bool options=true;for(unsigned offset=804;offset<=812;offset+=4)options=options && word(offset)==0; + if(width && height && rectangles && options && word(816)==width && word(820)==height) { + reason=4; + IOMemoryDescriptor *memory=native_legacy_surface_descriptor(surface); + if(native_legacy_surface_width(surface)==width && native_legacy_surface_height(surface)==height && + native_legacy_surface_format(surface)==0x42475241 && !native_legacy_surface_planes(surface) && + memory && memory->metaCast("IOMemoryDescriptor") && + native_legacy_surface_bounds(width,height,native_legacy_surface_stride(surface), + native_legacy_surface_allocation(surface),memory->getLength())) { + reason=0;accepted=true; + } + } + } + } + if(!accepted && reason==2 && !surface && enabled==1) { + // No surface in any slot and every updated layer disabled over black: present a blank screen. + OSObject *none[4]={};memcpy(none,bytes+2624,sizeof(none)); + uint32_t width=0,height=0;getDisplayDimensions(&width,&height); + bool options=true;for(unsigned offset=804;offset<=812;offset+=4)options=options && word(offset)==0; + if(!none[0] && !none[1] && !none[2] && !none[3] && width && height && options && + word(816)==width && word(820)==height && native_legacy_blank_state(word(792),word(796),word(800))) { + reason=5;accepted=true; + } + } + validatedRequest=accepted?request:nullptr; + native_legacy_swap_validation_observed(accepted,reason,request,surface);return accepted; + } + // The original slot-188 caller passes a uint32_t output and consumes an + // IOReturn. Its implementation assumes an AppleARMIODevice provider. + // Query our actual generic provider instead; GOP supplies no dot pitch. + IOReturn getDotPitch(uint32_t *pitch) override { + if(!pitch)return kIOReturnBadArgument; + *pitch=0;IOService *provider=getProvider();if(!provider)return kIOReturnNotReady; + OSObject *property=provider->getProperty("dot-pitch"); + if(!property)return kIOReturnNotFound; + OSData *data=OSDynamicCast(OSData,property); + if(!data || data->getLength()!=sizeof(*pitch))return kIOReturnBadArgument; + memcpy(pitch,data->getBytesNoCopy(),sizeof(*pitch));return kIOReturnSuccess; + } + // Selector 74 enumerates optional frame-info trace keys, not display + // modes. This backend implements no trace-key subscription contract. + IOReturn supportedFrameInfo(uint32_t,uint32_t *count,char *key,uint32_t *mask) override { + if(!count || !key || !mask)return kIOReturnBadArgument; + *count=0;*mask=0;bzero(key,64);return kIOReturnUnsupported; + } + IOReturn newUserClient(task_t task,void *security,UInt32 type,IOUserClient **client) override { + native_legacy_client_enter(task,type); + IOReturn status=IOMobileFramebufferLegacy::newUserClient(task,security,type,client); + native_legacy_client_observed(status,status==kIOReturnSuccess && client?*client:nullptr);return status; + } + bool startHardware() override { + unsigned enabled=0; + PE_parse_boot_argn("offline-legacy-gop",&enabled,sizeof(enabled)); + if(enabled!=1)return false; + PE_Video video={};IOService *provider=getProvider(); + if(!provider || PE_current_console(&video) || !video.v_baseAddr || + !video.v_width || !video.v_height || video.v_depth!=32 || video.v_rotate || + video.v_width>16384 || video.v_height>16384 || video.v_rowBytesUINT64_MAX/video.v_height)return false; + IOMemoryMap *mapping=provider->mapDeviceMemoryWithIndex(0,kIOMapInhibitCache); + if(!mapping)return false; + bool valid=mapping->getVirtualAddress() && mapping->getPhysicalAddress()==video.v_baseAddr && + mapping->getLength()==video.v_rowBytes*video.v_height; + if(valid) { + installBootDimensions((uint32_t)video.v_width,(uint32_t)video.v_height); + uint32_t width=0,height=0;getDisplayDimensions(&width,&height); + valid=width==video.v_width && height==video.v_height; + if(valid) { + OSSafeReleaseNULL(bootMapping);bootMapping=mapping;mapping=nullptr;primary=video.v_display==1; + valid=native_legacy_gop_mapping_observed(this,bootMapping,width,height); + } + } + OSSafeReleaseNULL(mapping);return valid; + } +#define LEAF(n) __attribute__((noreturn,noinline)) void opaque##n() override { native_legacy_backend_guard(n,this,__builtin_return_address(0)); } + LEAF(211) + LEAF(245) + LEAF(246) + LEAF(306) + LEAF(307) + LEAF(338) + LEAF(345) +#undef LEAF +}; +OSDefineMetaClassAndStructors(NativeLegacyDisplay,IOMobileFramebufferLegacy) +IOService *native_legacy_display_allocate(void) {return new NativeLegacyDisplay;} diff --git a/drivers/NativeMobileDisplay/NativeLegacyRequest.h b/drivers/NativeMobileDisplay/NativeLegacyRequest.h new file mode 100644 index 0000000..c5a1e1b --- /dev/null +++ b/drivers/NativeMobileDisplay/NativeLegacyRequest.h @@ -0,0 +1,181 @@ +// SPDX-License-Identifier: BSD-2-Clause-Patent +#pragma once +#include +// Original SwapSetLayer sets the update bit and clears the disable bit for +// a nonnull surface; a null surface sets both. SwapSetBackgroundColor sets +// bit 31 in both masks and writes opaque RGB. These are masks, not counts. +// This backend only copies one complete premultiplied BGRA layer over black. +// Other layers, unknown bits and nonblack backgrounds require composition. +static inline int native_legacy_single_layer_state(uint32_t updated, + uint32_t disabled, uint32_t background) { + const uint32_t known = 0x8000000fU; + if (((updated | disabled) & ~known) || (disabled & ~updated) || + (updated & ~disabled) != 1U) return 0; + if (updated & 0x80000000U) + return (disabled & 0x80000000U) && background == 0xff000000U; + return background == 0; +} + +// A swap that disables every updated layer over the black background asks the +// display to show nothing. It carries no surface; presenting it is a solid fill, +// not composition. Bit 31 is the background pseudo-layer, as above. +static inline int native_legacy_blank_state(uint32_t updated, + uint32_t disabled, uint32_t background) { + const uint32_t known = 0x8000000fU; + if (((updated | disabled) & ~known) || !updated || updated != disabled) return 0; + if (updated & 0x80000000U) return background == 0xff000000U; + return background == 0; +} + +// IOSurface rows and allocation have independent alignment. Copy visible +// pixels only, after bounding every row in the real memory descriptor. +static inline int native_legacy_surface_bounds(uint32_t width,uint32_t height, + uint64_t stride,uint64_t allocation,uint64_t descriptorBytes) { + if (!width || !height || width>16384 || height>16384 || + stride<(uint64_t)width*4 || stride>UINT64_MAX/height) return 0; + return allocation>=stride*height && descriptorBytes>=allocation; +} + +// The GOP destination is device memory: use aligned scalar pixel stores, +// never a general memcpy that may issue unaligned or wider device accesses. +// The source is normal mapped IOSurface memory and may have padded rows. +static inline int native_legacy_copy_layout(volatile void *destination, + uint64_t destinationBytes, uint64_t destinationStride, const void *source, + uint64_t sourceBytes, uint64_t sourceStride, uint32_t width, uint32_t height) { + uintptr_t dst=(uintptr_t)destination, src=(uintptr_t)source; + if (!dst || !src || (dst&3) || (destinationStride&3) || + !native_legacy_surface_bounds(width,height,destinationStride,destinationBytes,destinationBytes) || + !native_legacy_surface_bounds(width,height,sourceStride,sourceBytes,sourceBytes) || + destinationBytes>UINTPTR_MAX || sourceBytes>UINTPTR_MAX || + dst>UINTPTR_MAX-destinationBytes || src>UINTPTR_MAX-sourceBytes) return 0; + return 1; +} +static inline int native_legacy_fill_pixels(volatile void *destination, + uint64_t destinationBytes, uint64_t destinationStride, + uint32_t width, uint32_t height, uint32_t value) { + uintptr_t dst=(uintptr_t)destination; + if (!dst || (dst&3) || (destinationStride&3) || + !native_legacy_surface_bounds(width,height,destinationStride,destinationBytes,destinationBytes) || + destinationBytes>UINTPTR_MAX || dst>UINTPTR_MAX-destinationBytes) return 0; + for (uint32_t row=0;rowUINTPTR_MAX || src>UINTPTR_MAX-sourceBytes) return 0; + for (uint32_t row=0;rowout[3]) out[3]=column; + if (row>out[4]) out[4]=row; + } + } + } + return 1; +} +// Observation only: first-fit table of the first four distinct pixel values (with alpha) and their +// counts; every further distinct value is counted in *other. +static inline int native_legacy_frame_colors(const void *source, uint64_t sourceBytes, + uint64_t sourceStride, uint32_t width, uint32_t height, + uint32_t colors[4], uint64_t counts[4], uint64_t *other) { + for (unsigned i=0;i<4;++i){colors[i]=0;counts[i]=0;} + *other=0; + uintptr_t src=(uintptr_t)source; + if (!src || !native_legacy_surface_bounds(width,height,sourceStride,sourceBytes,sourceBytes) || + sourceBytes>UINTPTR_MAX || src>UINTPTR_MAX-sourceBytes) return 0; + unsigned used=0; + for (uint32_t row=0;row +#include +#include +// Prototype only. No registry publication or display operation is enabled. +class NativeMobileDisplay : public IOMobileFramebuffer { + OSDeclareDefaultStructors(NativeMobileDisplay) + PE_Video video = {}; +public: + bool init(OSDictionary *properties = nullptr) override { + if (!IOService::init(properties)) return false; + if (PE_current_console(&video) || !video.v_baseAddr || !video.v_width || + !video.v_height || video.v_depth != 32 || video.v_rotate || + video.v_width > 16384 || video.v_height > 16384 || + video.v_rowBytes < video.v_width * 4 || + video.v_rowBytes > UINT64_MAX / video.v_height) return false; + return true; + } + IOReturn beforeExternalMethod() override { + return video.v_width && video.v_height ? kIOReturnSuccess : kIOReturnNotReady; + } + void getDisplayDimensions(uint32_t *width,uint32_t *height) override { + if (!width || !height) panic("NativeMobileDisplay: invalid dimensions query"); + *width = (uint32_t)video.v_width; *height = (uint32_t)video.v_height; + } + __attribute__((noreturn)) void unclassified169() override { panic("NativeMobileDisplay: unclassified ABI entry 169"); } + __attribute__((noreturn)) void unclassified170() override { panic("NativeMobileDisplay: unclassified ABI entry 170"); } + __attribute__((noreturn)) void unclassified168() override { panic("NativeMobileDisplay: unclassified ABI entry 168"); } + __attribute__((noreturn)) void unclassified171() override { panic("NativeMobileDisplay: unclassified ABI entry 171"); } + __attribute__((noreturn)) void unclassified172() override { panic("NativeMobileDisplay: unclassified ABI entry 172"); } + __attribute__((noreturn)) void unclassified173() override { panic("NativeMobileDisplay: unclassified ABI entry 173"); } + __attribute__((noreturn)) void unclassified174() override { panic("NativeMobileDisplay: unclassified ABI entry 174"); } + __attribute__((noreturn)) void unclassified175() override { panic("NativeMobileDisplay: unclassified ABI entry 175"); } + __attribute__((noreturn)) void unclassified176() override { panic("NativeMobileDisplay: unclassified ABI entry 176"); } + __attribute__((noreturn)) void unclassified177() override { panic("NativeMobileDisplay: unclassified ABI entry 177"); } + __attribute__((noreturn)) void unclassified178() override { panic("NativeMobileDisplay: unclassified ABI entry 178"); } + __attribute__((noreturn)) void unclassified179() override { panic("NativeMobileDisplay: unclassified ABI entry 179"); } + __attribute__((noreturn)) void unclassified180() override { panic("NativeMobileDisplay: unclassified ABI entry 180"); } + __attribute__((noreturn)) void unclassified181() override { panic("NativeMobileDisplay: unclassified ABI entry 181"); } + __attribute__((noreturn)) void unclassified183() override { panic("NativeMobileDisplay: unclassified ABI entry 183"); } + __attribute__((noreturn)) void unclassified184() override { panic("NativeMobileDisplay: unclassified ABI entry 184"); } + __attribute__((noreturn)) void unclassified185() override { panic("NativeMobileDisplay: unclassified ABI entry 185"); } + __attribute__((noreturn)) void unclassified186() override { panic("NativeMobileDisplay: unclassified ABI entry 186"); } + __attribute__((noreturn)) void unclassified187() override { panic("NativeMobileDisplay: unclassified ABI entry 187"); } + __attribute__((noreturn)) void unclassified188() override { panic("NativeMobileDisplay: unclassified ABI entry 188"); } + __attribute__((noreturn)) void unclassified189() override { panic("NativeMobileDisplay: unclassified ABI entry 189"); } + __attribute__((noreturn)) void unclassified190() override { panic("NativeMobileDisplay: unclassified ABI entry 190"); } + __attribute__((noreturn)) void unclassified191() override { panic("NativeMobileDisplay: unclassified ABI entry 191"); } + __attribute__((noreturn)) void unclassified192() override { panic("NativeMobileDisplay: unclassified ABI entry 192"); } + IOReturn releaseClientState(IOUserClient *client) override { + // Original client stop calls this with itself and checks IOReturn. + // Only metadata queries are implemented: this backend has allocated + // no client buffers, queues, notification hooks or surfaces to free. + if (!client || client->getProvider() != this) return kIOReturnBadArgument; + return kIOReturnSuccess; + } + __attribute__((noreturn)) void unclassified194() override { panic("NativeMobileDisplay: unclassified ABI entry 194"); } + __attribute__((noreturn)) void unclassified195() override { panic("NativeMobileDisplay: unclassified ABI entry 195"); } + __attribute__((noreturn)) void unclassified196() override { panic("NativeMobileDisplay: unclassified ABI entry 196"); } + __attribute__((noreturn)) void unclassified197() override { panic("NativeMobileDisplay: unclassified ABI entry 197"); } + __attribute__((noreturn)) void unclassified198() override { panic("NativeMobileDisplay: unclassified ABI entry 198"); } + __attribute__((noreturn)) void unclassified199() override { panic("NativeMobileDisplay: unclassified ABI entry 199"); } + __attribute__((noreturn)) void unclassified200() override { panic("NativeMobileDisplay: unclassified ABI entry 200"); } + __attribute__((noreturn)) void unclassified201() override { panic("NativeMobileDisplay: unclassified ABI entry 201"); } + __attribute__((noreturn)) void unclassified202() override { panic("NativeMobileDisplay: unclassified ABI entry 202"); } + __attribute__((noreturn)) void unclassified203() override { panic("NativeMobileDisplay: unclassified ABI entry 203"); } + __attribute__((noreturn)) void unclassified204() override { panic("NativeMobileDisplay: unclassified ABI entry 204"); } + __attribute__((noreturn)) void unclassified205() override { panic("NativeMobileDisplay: unclassified ABI entry 205"); } + __attribute__((noreturn)) void unclassified206() override { panic("NativeMobileDisplay: unclassified ABI entry 206"); } + __attribute__((noreturn)) void unclassified207() override { panic("NativeMobileDisplay: unclassified ABI entry 207"); } + __attribute__((noreturn)) void unclassified208() override { panic("NativeMobileDisplay: unclassified ABI entry 208"); } + __attribute__((noreturn)) void unclassified209() override { panic("NativeMobileDisplay: unclassified ABI entry 209"); } + __attribute__((noreturn)) void unclassified210() override { panic("NativeMobileDisplay: unclassified ABI entry 210"); } + __attribute__((noreturn)) void unclassified211() override { panic("NativeMobileDisplay: unclassified ABI entry 211"); } + __attribute__((noreturn)) void unclassified212() override { panic("NativeMobileDisplay: unclassified ABI entry 212"); } + __attribute__((noreturn)) void unclassified213() override { panic("NativeMobileDisplay: unclassified ABI entry 213"); } + __attribute__((noreturn)) void unclassified214() override { panic("NativeMobileDisplay: unclassified ABI entry 214"); } + __attribute__((noreturn)) void unclassified215() override { panic("NativeMobileDisplay: unclassified ABI entry 215"); } + __attribute__((noreturn)) void unclassified216() override { panic("NativeMobileDisplay: unclassified ABI entry 216"); } + __attribute__((noreturn)) void unclassified217() override { panic("NativeMobileDisplay: unclassified ABI entry 217"); } + __attribute__((noreturn)) void unclassified218() override { panic("NativeMobileDisplay: unclassified ABI entry 218"); } + __attribute__((noreturn)) void unclassified219() override { panic("NativeMobileDisplay: unclassified ABI entry 219"); } + __attribute__((noreturn)) void unclassified220() override { panic("NativeMobileDisplay: unclassified ABI entry 220"); } + __attribute__((noreturn)) void unclassified221() override { panic("NativeMobileDisplay: unclassified ABI entry 221"); } + __attribute__((noreturn)) void unclassified222() override { panic("NativeMobileDisplay: unclassified ABI entry 222"); } + __attribute__((noreturn)) void unclassified223() override { panic("NativeMobileDisplay: unclassified ABI entry 223"); } + __attribute__((noreturn)) void unclassified224() override { panic("NativeMobileDisplay: unclassified ABI entry 224"); } + __attribute__((noreturn)) void unclassified225() override { panic("NativeMobileDisplay: unclassified ABI entry 225"); } + __attribute__((noreturn)) void unclassified226() override { panic("NativeMobileDisplay: unclassified ABI entry 226"); } + __attribute__((noreturn)) void unclassified227() override { panic("NativeMobileDisplay: unclassified ABI entry 227"); } + __attribute__((noreturn)) void unclassified228() override { panic("NativeMobileDisplay: unclassified ABI entry 228"); } + __attribute__((noreturn)) void unclassified229() override { panic("NativeMobileDisplay: unclassified ABI entry 229"); } + __attribute__((noreturn)) void unclassified230() override { panic("NativeMobileDisplay: unclassified ABI entry 230"); } + __attribute__((noreturn)) void unclassified231() override { panic("NativeMobileDisplay: unclassified ABI entry 231"); } + __attribute__((noreturn)) void unclassified232() override { panic("NativeMobileDisplay: unclassified ABI entry 232"); } + __attribute__((noreturn)) void unclassified233() override { panic("NativeMobileDisplay: unclassified ABI entry 233"); } + __attribute__((noreturn)) void unclassified234() override { panic("NativeMobileDisplay: unclassified ABI entry 234"); } + __attribute__((noreturn)) void unclassified235() override { panic("NativeMobileDisplay: unclassified ABI entry 235"); } + __attribute__((noreturn)) void unclassified236() override { panic("NativeMobileDisplay: unclassified ABI entry 236"); } + __attribute__((noreturn)) void unclassified237() override { panic("NativeMobileDisplay: unclassified ABI entry 237"); } + __attribute__((noreturn)) void unclassified238() override { panic("NativeMobileDisplay: unclassified ABI entry 238"); } + __attribute__((noreturn)) void unclassified239() override { panic("NativeMobileDisplay: unclassified ABI entry 239"); } + __attribute__((noreturn)) void unclassified240() override { panic("NativeMobileDisplay: unclassified ABI entry 240"); } + __attribute__((noreturn)) void unclassified241() override { panic("NativeMobileDisplay: unclassified ABI entry 241"); } + __attribute__((noreturn)) void unclassified242() override { panic("NativeMobileDisplay: unclassified ABI entry 242"); } + __attribute__((noreturn)) void unclassified243() override { panic("NativeMobileDisplay: unclassified ABI entry 243"); } + __attribute__((noreturn)) void unclassified244() override { panic("NativeMobileDisplay: unclassified ABI entry 244"); } + __attribute__((noreturn)) void unclassified245() override { panic("NativeMobileDisplay: unclassified ABI entry 245"); } + __attribute__((noreturn)) void unclassified246() override { panic("NativeMobileDisplay: unclassified ABI entry 246"); } + __attribute__((noreturn)) void unclassified247() override { panic("NativeMobileDisplay: unclassified ABI entry 247"); } + __attribute__((noreturn)) void unclassified248() override { panic("NativeMobileDisplay: unclassified ABI entry 248"); } + __attribute__((noreturn)) void unclassified250() override { panic("NativeMobileDisplay: unclassified ABI entry 250"); } + __attribute__((noreturn)) void unclassified251() override { panic("NativeMobileDisplay: unclassified ABI entry 251"); } + __attribute__((noreturn)) void unclassified252() override { panic("NativeMobileDisplay: unclassified ABI entry 252"); } + __attribute__((noreturn)) void unclassified253() override { panic("NativeMobileDisplay: unclassified ABI entry 253"); } + __attribute__((noreturn)) void unclassified254() override { panic("NativeMobileDisplay: unclassified ABI entry 254"); } + __attribute__((noreturn)) void unclassified255() override { panic("NativeMobileDisplay: unclassified ABI entry 255"); } + __attribute__((noreturn)) void unclassified256() override { panic("NativeMobileDisplay: unclassified ABI entry 256"); } + __attribute__((noreturn)) void unclassified257() override { panic("NativeMobileDisplay: unclassified ABI entry 257"); } + __attribute__((noreturn)) void unclassified258() override { panic("NativeMobileDisplay: unclassified ABI entry 258"); } + __attribute__((noreturn)) void unclassified259() override { panic("NativeMobileDisplay: unclassified ABI entry 259"); } + __attribute__((noreturn)) void unclassified260() override { panic("NativeMobileDisplay: unclassified ABI entry 260"); } + __attribute__((noreturn)) void unclassified261() override { panic("NativeMobileDisplay: unclassified ABI entry 261"); } +}; +OSDefineMetaClassAndStructors(NativeMobileDisplay,IOMobileFramebuffer) + +extern "C" IOService *native_mobile_display_create(void); +extern "C" IOService *native_mobile_display_create(void) { + // Caller must load the original parent and register this real metaclass + // through the normal kernel/kext C++ initialization machinery first. + const OSSymbol *name = OSSymbol::withCString("NativeMobileDisplay"); + if (!name) return nullptr; + const OSMetaClass *meta = OSMetaClass::getMetaClassWithName(name); + name->release(); + if (meta != &NativeMobileDisplay::gMetaClass) return nullptr; + NativeMobileDisplay *display = new NativeMobileDisplay; + if (display && !display->init()) { display->release(); return nullptr; } + return display; +} diff --git a/drivers/NativeMobileDisplay/OriginalMobileFramebufferABI.h b/drivers/NativeMobileDisplay/OriginalMobileFramebufferABI.h new file mode 100644 index 0000000..71c6a00 --- /dev/null +++ b/drivers/NativeMobileDisplay/OriginalMobileFramebufferABI.h @@ -0,0 +1,107 @@ +// SPDX-License-Identifier: BSD-2-Clause-Patent +#pragma once +// Experimental binary dispatch layout for the exact supplied iOS17 family. +// Unclassified entries have no usable prototype or return contract. Derived +// implementations must fail without returning; never advertise them as supported. +#include +#include +class IOMobileFramebuffer : public IOService { + OSDeclareAbstractStructors(IOMobileFramebuffer) + uint64_t familyPrivate; +public: + virtual __attribute__((noreturn)) void unclassified168() = 0; + virtual __attribute__((noreturn)) void unclassified169() = 0; + virtual __attribute__((noreturn)) void unclassified170() = 0; + virtual __attribute__((noreturn)) void unclassified171() = 0; + virtual __attribute__((noreturn)) void unclassified172() = 0; + virtual __attribute__((noreturn)) void unclassified173() = 0; + virtual __attribute__((noreturn)) void unclassified174() = 0; + virtual __attribute__((noreturn)) void unclassified175() = 0; + virtual __attribute__((noreturn)) void unclassified176() = 0; + virtual __attribute__((noreturn)) void unclassified177() = 0; + virtual __attribute__((noreturn)) void unclassified178() = 0; + virtual __attribute__((noreturn)) void unclassified179() = 0; + virtual __attribute__((noreturn)) void unclassified180() = 0; + virtual __attribute__((noreturn)) void unclassified181() = 0; + virtual IOReturn beforeExternalMethod() = 0; + virtual __attribute__((noreturn)) void unclassified183() = 0; + virtual __attribute__((noreturn)) void unclassified184() = 0; + virtual __attribute__((noreturn)) void unclassified185() = 0; + virtual __attribute__((noreturn)) void unclassified186() = 0; + virtual __attribute__((noreturn)) void unclassified187() = 0; + virtual __attribute__((noreturn)) void unclassified188() = 0; + virtual __attribute__((noreturn)) void unclassified189() = 0; + virtual __attribute__((noreturn)) void unclassified190() = 0; + virtual __attribute__((noreturn)) void unclassified191() = 0; + virtual __attribute__((noreturn)) void unclassified192() = 0; + virtual IOReturn releaseClientState(IOUserClient *) = 0; + virtual __attribute__((noreturn)) void unclassified194() = 0; + virtual __attribute__((noreturn)) void unclassified195() = 0; + virtual __attribute__((noreturn)) void unclassified196() = 0; + virtual __attribute__((noreturn)) void unclassified197() = 0; + virtual __attribute__((noreturn)) void unclassified198() = 0; + virtual __attribute__((noreturn)) void unclassified199() = 0; + virtual __attribute__((noreturn)) void unclassified200() = 0; + virtual __attribute__((noreturn)) void unclassified201() = 0; + virtual __attribute__((noreturn)) void unclassified202() = 0; + virtual __attribute__((noreturn)) void unclassified203() = 0; + virtual __attribute__((noreturn)) void unclassified204() = 0; + virtual __attribute__((noreturn)) void unclassified205() = 0; + virtual __attribute__((noreturn)) void unclassified206() = 0; + virtual __attribute__((noreturn)) void unclassified207() = 0; + virtual __attribute__((noreturn)) void unclassified208() = 0; + virtual __attribute__((noreturn)) void unclassified209() = 0; + virtual __attribute__((noreturn)) void unclassified210() = 0; + virtual __attribute__((noreturn)) void unclassified211() = 0; + virtual __attribute__((noreturn)) void unclassified212() = 0; + virtual __attribute__((noreturn)) void unclassified213() = 0; + virtual __attribute__((noreturn)) void unclassified214() = 0; + virtual __attribute__((noreturn)) void unclassified215() = 0; + virtual __attribute__((noreturn)) void unclassified216() = 0; + virtual __attribute__((noreturn)) void unclassified217() = 0; + virtual __attribute__((noreturn)) void unclassified218() = 0; + virtual __attribute__((noreturn)) void unclassified219() = 0; + virtual __attribute__((noreturn)) void unclassified220() = 0; + virtual __attribute__((noreturn)) void unclassified221() = 0; + virtual __attribute__((noreturn)) void unclassified222() = 0; + virtual __attribute__((noreturn)) void unclassified223() = 0; + virtual __attribute__((noreturn)) void unclassified224() = 0; + virtual __attribute__((noreturn)) void unclassified225() = 0; + virtual __attribute__((noreturn)) void unclassified226() = 0; + virtual __attribute__((noreturn)) void unclassified227() = 0; + virtual __attribute__((noreturn)) void unclassified228() = 0; + virtual __attribute__((noreturn)) void unclassified229() = 0; + virtual __attribute__((noreturn)) void unclassified230() = 0; + virtual __attribute__((noreturn)) void unclassified231() = 0; + virtual __attribute__((noreturn)) void unclassified232() = 0; + virtual __attribute__((noreturn)) void unclassified233() = 0; + virtual __attribute__((noreturn)) void unclassified234() = 0; + virtual __attribute__((noreturn)) void unclassified235() = 0; + virtual __attribute__((noreturn)) void unclassified236() = 0; + virtual __attribute__((noreturn)) void unclassified237() = 0; + virtual __attribute__((noreturn)) void unclassified238() = 0; + virtual __attribute__((noreturn)) void unclassified239() = 0; + virtual __attribute__((noreturn)) void unclassified240() = 0; + virtual __attribute__((noreturn)) void unclassified241() = 0; + virtual __attribute__((noreturn)) void unclassified242() = 0; + virtual __attribute__((noreturn)) void unclassified243() = 0; + virtual __attribute__((noreturn)) void unclassified244() = 0; + virtual __attribute__((noreturn)) void unclassified245() = 0; + virtual __attribute__((noreturn)) void unclassified246() = 0; + virtual __attribute__((noreturn)) void unclassified247() = 0; + virtual __attribute__((noreturn)) void unclassified248() = 0; + virtual void getDisplayDimensions(uint32_t *, uint32_t *) = 0; + virtual __attribute__((noreturn)) void unclassified250() = 0; + virtual __attribute__((noreturn)) void unclassified251() = 0; + virtual __attribute__((noreturn)) void unclassified252() = 0; + virtual __attribute__((noreturn)) void unclassified253() = 0; + virtual __attribute__((noreturn)) void unclassified254() = 0; + virtual __attribute__((noreturn)) void unclassified255() = 0; + virtual __attribute__((noreturn)) void unclassified256() = 0; + virtual __attribute__((noreturn)) void unclassified257() = 0; + virtual __attribute__((noreturn)) void unclassified258() = 0; + virtual __attribute__((noreturn)) void unclassified259() = 0; + virtual __attribute__((noreturn)) void unclassified260() = 0; + virtual __attribute__((noreturn)) void unclassified261() = 0; +}; +static_assert(sizeof(IOService)==136 && sizeof(IOMobileFramebuffer)==144); diff --git a/drivers/NativeMobileDisplay/Test/request-state.c b/drivers/NativeMobileDisplay/Test/request-state.c new file mode 100644 index 0000000..18eb405 --- /dev/null +++ b/drivers/NativeMobileDisplay/Test/request-state.c @@ -0,0 +1,95 @@ +// SPDX-License-Identifier: BSD-2-Clause-Patent +#include "../NativeLegacyRequest.h" +#include +#include +int main(void) { + assert(native_legacy_single_layer_state(1,0,0)); + assert(native_legacy_single_layer_state(0x80000007U,0x80000006U,0xff000000U)); + // Exhaust the four-layer masks: exactly layer zero may remain active. + for(uint32_t updated=0;updated<16;updated++) + for(uint32_t disabled=0;disabled<16;disabled++) { + int expected=!(disabled & ~updated) && (updated & ~disabled)==1; + assert(native_legacy_single_layer_state(updated,disabled,0)==expected); + assert(native_legacy_single_layer_state(updated|0x80000000U, + disabled|0x80000000U,0xff000000U)==expected); + } + assert(!native_legacy_single_layer_state(0x80000001U,0,0xff000000U)); + assert(!native_legacy_single_layer_state(0x80000001U,0x80000000U,0xffff0000U)); + assert(!native_legacy_single_layer_state(1,0,0xff000000U)); + assert(!native_legacy_single_layer_state(0x101,0x100,0)); + assert(!native_legacy_single_layer_state(0x80000001U,0x80000000U,0)); + // Blank swaps: every updated layer disabled over black, nothing enabled. + assert(native_legacy_blank_state(0x80000007U,0x80000007U,0xff000000U)); + assert(native_legacy_blank_state(0x80000000U,0x80000000U,0xff000000U)); + assert(native_legacy_blank_state(3,3,0)); + assert(!native_legacy_blank_state(0,0,0)); + assert(!native_legacy_blank_state(0x80000007U,0x80000006U,0xff000000U)); + assert(!native_legacy_blank_state(0x80000007U,0x80000007U,0)); + assert(!native_legacy_blank_state(3,3,0xff000000U)); + assert(!native_legacy_blank_state(0x80000007U,0x80000007U,0xffff0000U)); + assert(!native_legacy_blank_state(0x100,0x100,0)); + for(uint32_t updated=0;updated<16;updated++) + for(uint32_t disabled=0;disabled<16;disabled++) + assert(!native_legacy_blank_state(updated,disabled,0) || (updated && updated==disabled)); + { + uint32_t frame[24];memset(frame,0xcc,sizeof(frame)); + assert(native_legacy_fill_pixels(frame,64,16,3,4,0xff000000U)); + for(unsigned row=0;row<4;++row) { + for(unsigned column=0;column<3;++column)assert(frame[row*4+column]==0xff000000U); + assert(frame[row*4+3]==0xcccccccc); + } + for(unsigned i=16;i<24;++i)assert(frame[i]==0xcccccccc); + uint32_t before[24];memcpy(before,frame,sizeof(frame)); + assert(!native_legacy_fill_pixels((unsigned char *)frame+1,64,16,3,4,0)); + assert(!native_legacy_fill_pixels(frame,63,16,3,4,0)); + assert(!native_legacy_fill_pixels(frame,64,14,3,4,0)); + assert(!native_legacy_fill_pixels(frame,64,16,0,4,0)); + assert(!memcmp(before,frame,sizeof(frame))); + } + { + unsigned char image[4*4*4];memset(image,0,sizeof(image));uint64_t scan[5]; + assert(native_legacy_frame_scan(image,sizeof(image),16,4,4,scan)&&!scan[0]&&scan[1]==UINT64_MAX&&scan[2]==UINT64_MAX&&!scan[3]&&!scan[4]); + image[3]=0xff;image[3*16+2*4+3]=0xff; /* alpha alone is black */ + assert(native_legacy_frame_scan(image,sizeof(image),16,4,4,scan)&&!scan[0]); + image[1*16+2*4+1]=1;image[3*16+0*4]=7;image[2*16+3*4+2]=9; + assert(native_legacy_frame_scan(image,sizeof(image),16,4,4,scan)&&scan[0]==3&&scan[1]==0&&scan[2]==1&&scan[3]==3&&scan[4]==3); + assert(!native_legacy_frame_scan(image,sizeof(image)-1,16,4,4,scan)&&!scan[0]); + assert(!native_legacy_frame_scan(NULL,sizeof(image),16,4,4,scan)); + } + { + unsigned char image[4*4*4];uint32_t colors[4];uint64_t counts[4],other=99; + for(unsigned i=0;i<4*4;++i){image[i*4]=1;image[i*4+1]=2;image[i*4+2]=3;image[i*4+3]=0xff;} + assert(native_legacy_frame_colors(image,sizeof(image),16,4,4,colors,counts,&other)&&colors[0]==0xff030201U&&counts[0]==16&&!counts[1]&&!other); + for(unsigned i=0;i<6;++i)image[i*4]=(unsigned char)(10+i); /* 6 distinct values + 10 original */ + assert(native_legacy_frame_colors(image,sizeof(image),16,4,4,colors,counts,&other)); + assert(counts[0]+counts[1]+counts[2]+counts[3]+other==16&&counts[0]==1&&counts[3]==1&&other==2+10); + assert(!native_legacy_frame_colors(image,sizeof(image)-1,16,4,4,colors,counts,&other)&&!counts[0]&&!other); + } + assert(native_legacy_surface_bounds(800,600,3328,1998848,1998848)); + assert(native_legacy_surface_bounds(800,600,3200,1920000,1920000)); + assert(!native_legacy_surface_bounds(800,600,3199,1920000,1920000)); + assert(!native_legacy_surface_bounds(800,600,3328,1996799,1998848)); + assert(!native_legacy_surface_bounds(800,600,3328,1998848,1998847)); + assert(!native_legacy_surface_bounds(800,600,UINT64_MAX,UINT64_MAX,UINT64_MAX)); + assert(!native_legacy_surface_bounds(0,600,3328,1998848,1998848)); + for(unsigned offset=0;offset<8;++offset) { + unsigned char storage[80]; uint32_t target[20]; + for(unsigned i=0;i +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "SparseDisk.h" +#include "PersistOverlay.h" + +extern "C" { +extern unsigned long gPhysBase, gPhysSize; +int native_ram_block_volume_device(const char *, dev_t *, char *, size_t); +volatile uint32_t native_ram_block_ready = 0; +volatile uint64_t native_ram_block_bytes = 0; +volatile uint64_t native_ram_block_reads = 0; +volatile uint64_t native_ram_block_writes = 0; +volatile uint64_t native_ram_block_failures = 0; +volatile uint64_t native_ram_block_resident_bytes = 0; +volatile uint32_t native_ram_block_sparse = 0; +uint8_t native_ram_block_seed_sha256[32]; +// Optional persistence of the changed pages through a board-provided store window (see PersistOverlay.h). All weak: a build +// without the provider keeps the volatile disk. +unsigned native_ufs_store_mode(void) __attribute__((weak_import)); +int native_ufs_store_open(uint64_t, uint64_t *, const char **) __attribute__((weak_import)); +int native_ufs_store_read(uint64_t, uint32_t, void *) __attribute__((weak_import)); +int native_ufs_store_write(uint64_t, uint32_t, const void *) __attribute__((weak_import)); +int native_ufs_store_sync(void) __attribute__((weak_import)); +unsigned native_ufs_store_gpt_check(void) __attribute__((weak_import)); +// Diagnostics. state: 0 off, 1 blank (read-only probe), 2 resumed, 3 fresh, 4 refused/failed, 5 degraded after an I/O error, +// 6 resumed read-only (probe). header: persist::checkHeader result. +volatile uint32_t native_data_persist_state = 0, native_data_persist_header = 9, native_data_persist_gpt = 0xff, native_data_persist_gpt_after = 0xff; +volatile uint64_t native_data_persist_pages = 0, native_data_persist_loaded = 0, native_data_persist_writes = 0, + native_data_persist_syncs = 0, native_data_persist_errors = 0, native_data_persist_first_lba = 0, + native_data_persist_present = 0; +} + +class NativeRamBlock : public IOBlockStorageDevice { + OSDeclareDefaultStructors(NativeRamBlock) + IOLock *gate = nullptr; + uint8_t *memory = nullptr; + uint64_t capacity = 0; + bool online = false; + RamSparseDisk sparse = {}; + uint8_t seedHash[32] = {}; + persist::Overlay overlay; + uint8_t *overlayBits = nullptr, *overlayDirty = nullptr; + bool storeAttached = false; + uint64_t lastSync = 0; + struct StoreOps { + bool read(uint64_t b, uint32_t n, void *d) { return native_ufs_store_read(b, n, d); } + bool write(uint64_t b, uint32_t n, const void *s) { return native_ufs_store_write(b, n, s); } + bool sync() { return native_ufs_store_sync(); } + }; + void persistFailed() { overlay.active = false; native_data_persist_state = 5; __atomic_fetch_add(&native_data_persist_errors, 1, __ATOMIC_RELAXED); } + void persistSync() { + if (!overlay.active || overlay.readOnly) return; + StoreOps ops; + if (!overlay.sync(ops)) { persistFailed(); return; } + __atomic_fetch_add(&native_data_persist_syncs, 1, __ATOMIC_RELAXED); + native_data_persist_present = overlay.present; + lastSync = mach_absolute_time(); + } +public: + // Called once, before the Data volume is mounted (native_ram_block_volume_seed): open the store window, put the pages that + // earlier boots changed back into the sparse disk, and from then on write changed pages through. The seed identity + // (seedHash) is not affected. Any refusal or error leaves the disk volatile, exactly as without persistence. + void attachStore() { + if (!gate) return; + IOLockLock(gate); + if (storeAttached) { IOLockUnlock(gate); return; } + storeAttached = true; + unsigned mode = &native_ufs_store_mode ? native_ufs_store_mode() : 0; + if (mode && sparse.pages && &native_ufs_store_open && &native_ufs_store_read && &native_ufs_store_write && + &native_ufs_store_sync && &native_ufs_store_gpt_check) { + uint64_t pages = sparse.bytes / 4096, first = 0; + const char *why = ""; + native_data_persist_pages = pages; + native_data_persist_gpt = native_ufs_store_gpt_check(); + if (!native_ufs_store_open(persist::windowBlocks(pages), &first, &why)) { + native_data_persist_state = 4; + IOLog("NativeRamBlock: persistent store refused: %s\n", why); + } else { + native_data_persist_first_lba = first; + uint64_t bb = persist::bitmapBlocks(pages); + overlayBits = static_cast(IOMallocZero(bb * persist::kBlock)); + overlayDirty = static_cast(IOMallocZero(bb)); + StoreOps ops; + persist::Open result = overlayBits && overlayDirty ? overlay.open(ops, pages, seedHash, mode == 1, overlayBits, overlayDirty) + : persist::Open::Failed; + native_data_persist_header = static_cast(overlay.header); + if (result == persist::Open::Failed) { + native_data_persist_state = 4; + overlay.active = false; + } else if (result == persist::Open::Blank) { + native_data_persist_state = 1; + } else if (result == persist::Open::Fresh) { + native_data_persist_state = 3; + } else { + struct Loader { + NativeRamBlock *self; + bool operator()(uint64_t i, const uint8_t *page) { + if (i >= self->sparse.bytes / 4096) return false; + if (!self->sparse.pages[i]) { + uint8_t *fresh = static_cast(IOMallocZero(4096)); + if (!fresh) return false; + self->sparse.pages[i] = fresh; + self->sparse.resident += 4096; + } + memcpy(self->sparse.pages[i], page, 4096); + __atomic_fetch_add(&native_data_persist_loaded, 1, __ATOMIC_RELAXED); + return true; + } + } load = { this }; + if (overlay.forEachPresent(ops, load)) { + native_data_persist_state = mode == 1 ? 6 : 2; + native_ram_block_resident_bytes = sparse.resident; + } else { + // A partly restored disk would be worse than a volatile one, but the pages already copied are consistent + // overlay content; stop writing back and let the system decide what it can mount. + native_data_persist_state = 4; + overlay.active = false; + } + } + native_data_persist_present = overlay.present; + native_data_persist_gpt_after = native_ufs_store_gpt_check(); + lastSync = mach_absolute_time(); + } + } + IOLockUnlock(gate); + } + + bool available() const { return online && (memory || sparse.pages) && capacity; } + bool seedInfo(uint64_t *base,uint64_t *bytes,uint8_t hash[32]) { + if(!gate || !base || !bytes || !hash)return false; + IOLockLock(gate);bool ready=available(); + if(ready){*base=sparse.pages?0:reinterpret_cast(memory);*bytes=capacity*512;memcpy(hash,seedHash,32);} + IOLockUnlock(gate);return ready; + } + bool start(IOService *) override; + void stop(IOService *) override; + void free() override; + IOReturn doEjectMedia() override { return kIOReturnUnsupported; } + IOReturn doFormatMedia(UInt64) override { return kIOReturnUnsupported; } + UInt32 doGetFormatCapacities(UInt64 *, UInt32) const override { return 0; } + char *getVendorString() override { return const_cast("Native"); } + char *getProductString() override { return const_cast("Owned RAM block storage"); } + char *getRevisionString() override { return const_cast("1.0"); } + char *getAdditionalDeviceInfoString() override { return const_cast("volatile storage; no persistent trust claim"); } + IOReturn reportBlockSize(UInt64 *value) override { *value = 512; return kIOReturnSuccess; } + IOReturn reportEjectability(bool *value) override { *value = false; return kIOReturnSuccess; } + IOReturn reportMaxValidBlock(UInt64 *value) override { *value = capacity - 1; return kIOReturnSuccess; } + IOReturn reportMediaState(bool *present, bool *changed) override { + *present = online; if (changed) *changed = false; return kIOReturnSuccess; + } + IOReturn reportRemovability(bool *value) override { *value = false; return kIOReturnSuccess; } + IOReturn reportWriteProtection(bool *value) override { *value = false; return kIOReturnSuccess; } + IOReturn doSynchronizeCache() override { + if (!online) return kIOReturnNotReady; + if (gate) { IOLockLock(gate); persistSync(); IOLockUnlock(gate); } + return kIOReturnSuccess; + } + IOReturn doSynchronize(UInt64, UInt64, IOOptionBits) override { return doSynchronizeCache(); } + IOReturn getWriteCacheState(bool *value) override { *value = false; return kIOReturnSuccess; } + IOReturn setWriteCacheState(bool) override { return kIOReturnUnsupported; } + IOReturn doAsyncReadWrite(IOMemoryDescriptor *, UInt64, UInt64, IOStorageAttributes *, IOStorageCompletion *) override; +}; +OSDefineMetaClassAndStructors(NativeRamBlock, IOBlockStorageDevice) + +bool NativeRamBlock::start(IOService *provider) +{ + if (!IOBlockStorageDevice::start(provider)) return false; + IORegistryEntry *map = IORegistryEntry::fromPath("/chosen/memory-map", gIODTPlane); + if (!map) return false; + OSData *packed = OSDynamicCast(OSData, map->getProperty("PackedRAMDisk")); + OSData *range = OSDynamicCast(OSData, map->getProperty("RAMDisk")); + if(packed && range){map->release();return false;} + if(packed)range=packed; + bool packedInput=packed!=nullptr; + uint64_t physical = 0, bytes = 0; + if (range && range->getLength() == 16) { + memcpy(&physical, range->getBytesNoCopy(), 8); + memcpy(&bytes, static_cast(range->getBytesNoCopy()) + 8, 8); + } + map->release(); + // Boot memory belongs to the loader/kernel; never allocate or free it here. + if (!bytes || bytes > 0x80000000 || (physical & 4095) || (!packedInput && (bytes & 4095)) || + physical < gPhysBase || bytes > gPhysSize || physical - gPhysBase > gPhysSize - bytes) return false; + memory = reinterpret_cast(ml_static_ptovirt(physical)); + if (!memory || !(gate = IOLockAlloc())) return false; + uint64_t logical=bytes; + if(packedInput){ + if(!ramPackedLayout(memory,bytes,&logical))return false; + sparse.bytes=logical; + sparse.pages=static_cast(IOMallocZero((logical/4096)*sizeof(uint8_t *))); + struct ImportOps {uint8_t *allocate(){return static_cast(IOMallocZero(4096));}} ops; + if(!ramSparseImport(sparse,memory,bytes,ops))return false; + } + capacity = logical / 512; + SHA256_CTX context; + SHA256_Init(&context); + if(sparse.pages){ + static const uint8_t zero[4096]={}; + for(uint64_t i=0;iaction || !count || block >= capacity || + count > capacity - block || buffer->getLength() < count * 512 || + (buffer->getDirection() != kIODirectionIn && buffer->getDirection() != kIODirectionOut)) return kIOReturnBadArgument; + if (!gate) return kIOReturnNotReady; + const uint64_t bytes = count * 512; + IOLockLock(gate); + if (!online) { IOLockUnlock(gate); return kIOReturnNotReady; } + uint64_t actual;bool noMemory=false; + if(sparse.pages){ + struct Operations { + IOMemoryDescriptor *buffer; + uint8_t *allocate(){return static_cast(IOMallocZero(4096));} + uint64_t read(uint64_t at,void *p,uint64_t n){return buffer->readBytes(at,p,n);} + uint64_t copy(uint64_t at,const void *p,uint64_t n){return buffer->writeBytes(at,p,n);} + } ops={buffer}; + bool write=buffer->getDirection()==kIODirectionOut; + actual=ramSparseIO(sparse,write,block*512,bytes,ops,noMemory); + if(write && actual==bytes && overlay.active && !overlay.readOnly){ + StoreOps store; + uint64_t firstPage=block*512/4096,lastPage=(block*512+bytes-1)/4096; + for(uint64_t i=firstPage;i<=lastPage && overlay.active;++i){ + if(!sparse.pages[i] || !overlay.writePage(store,i,sparse.pages[i]))persistFailed(); + else __atomic_fetch_add(&native_data_persist_writes,1,__ATOMIC_RELAXED); + } + uint64_t interval=0;clock_interval_to_absolutetime_interval(2,NSEC_PER_SEC,&interval); + if(overlay.anyDirty && mach_absolute_time()-lastSync>interval)persistSync(); + } + if(write)__atomic_fetch_add(&native_ram_block_writes,1,__ATOMIC_RELAXED); + else __atomic_fetch_add(&native_ram_block_reads,1,__ATOMIC_RELAXED); + native_ram_block_resident_bytes=sparse.resident; + }else if (buffer->getDirection() == kIODirectionIn) { + actual = buffer->writeBytes(0, memory + block * 512, bytes); + __atomic_fetch_add(&native_ram_block_reads, 1, __ATOMIC_RELAXED); + } else { + actual = buffer->readBytes(0, memory + block * 512, bytes); + __atomic_fetch_add(&native_ram_block_writes, 1, __ATOMIC_RELAXED); + } + IOReturn status = actual == bytes ? kIOReturnSuccess : noMemory ? kIOReturnNoMemory : kIOReturnIOError; + if (status) __atomic_fetch_add(&native_ram_block_failures, 1, __ATOMIC_RELAXED); + IOLockUnlock(gate); + completion->action(completion->target, completion->parameter, status, actual); + return kIOReturnSuccess; +} + +void NativeRamBlock::stop(IOService *provider) +{ + if (gate) { IOLockLock(gate); online = false; IOLockUnlock(gate); } + IOBlockStorageDevice::stop(provider); +} + +void NativeRamBlock::free() +{ + if (overlayBits) { IOFree(overlayBits, persist::bitmapBlocks(sparse.bytes / 4096) * persist::kBlock); overlayBits = nullptr; } + if (overlayDirty) { IOFree(overlayDirty, persist::bitmapBlocks(sparse.bytes / 4096)); overlayDirty = nullptr; } + if (gate) { IOLockFree(gate); gate = nullptr; } + if(sparse.pages){ + for(uint64_t i=0;isetObject("UUID", wanted) && matching->setObject("IOPropertyMatch", properties); + wanted->release(); properties->release(); + OSIterator *iterator = ready ? IOService::getMatchingServices(matching) : nullptr; + matching->release(); + if (!iterator) return 0; + bool selected = false; + unsigned matches = 0; + dev_t candidate = NODEV; + char candidatePath[64] = {}; + uint64_t base=0,bytes=0;uint8_t hash[32]={}; + while (OSObject *object = iterator->getNextObject()) { + IOService *media = OSDynamicCast(IOService, object); + if (!media) continue; + if (++matches != 1) { selected = false; break; } + OSString *identity = OSDynamicCast(OSString, media->getProperty("UUID")); + OSString *name = OSDynamicCast(OSString, media->getProperty(kIOBSDNameKey)); + OSNumber *mjr = OSDynamicCast(OSNumber, media->getProperty(kIOBSDMajorKey)); + OSNumber *mnr = OSDynamicCast(OSNumber, media->getProperty(kIOBSDMinorKey)); + if (!identity || strcmp(identity->getCStringNoCopy(), uuid) || !name || !mjr || !mnr || + !name->getLength() || name->getLength() >= sizeof(candidatePath) - 6 || name->getLength() >= size - 6 || + mjr->unsigned64BitValue() > 255 || mnr->unsigned64BitValue() > 0xffffff) continue; + unsigned depth = 0; + for (IOService *provider = media; provider && depth++ < 64; provider = provider->getProvider()) { + NativeRamBlock *ram = OSDynamicCast(NativeRamBlock, provider); + if (!ram) continue; + ram->attachStore(); + if (!ram->seedInfo(&base,&bytes,hash)) continue; + candidate = static_cast(makedev(mjr->unsigned32BitValue(), mnr->unsigned32BitValue())); + snprintf(candidatePath, sizeof(candidatePath), "/dev/%s", name->getCStringNoCopy()); + selected = true; + break; + } + } + iterator->release(); + if (!selected || matches != 1) return 0; + *device = candidate; + strlcpy(path, candidatePath, size); + if(seedBase)*seedBase=base; + if(seedBytes)*seedBytes=bytes; + if(seedDigest)memcpy(seedDigest,hash,32); + return 1; +} + +extern "C" int native_ram_block_volume_device(const char *uuid,dev_t *device,char *path,size_t size) { + return selectVolumeSeed(uuid,device,path,size,nullptr,nullptr,nullptr); +} +extern "C" int native_ram_block_volume_seed(const char *uuid,dev_t *device,char *path,size_t size, + uint64_t *base,uint64_t *bytes,uint8_t digest[32]) { + if(!base || !bytes || !digest)return 0; + *base=0;*bytes=0;memset(digest,0,32); + return selectVolumeSeed(uuid,device,path,size,base,bytes,digest); +} diff --git a/drivers/NativeRamBlock/PersistOverlay.h b/drivers/NativeRamBlock/PersistOverlay.h new file mode 100644 index 0000000..4e5678f --- /dev/null +++ b/drivers/NativeRamBlock/PersistOverlay.h @@ -0,0 +1,143 @@ +// SPDX-License-Identifier: BSD-2-Clause-Patent +#pragma once +// Persistent overlay over a seeded sparse disk. The seed (the packed RAM disk the loader provides) stays the immutable base; the +// pages a running system changes are stored in a window of 4096-byte blocks on some other medium and put back on the next boot. +// +// Window layout (block = 4096 bytes, all integers little endian): +// block 0 header: "SHNGDATA", version 1, header bytes 64, page count, SHA-256 of the seed, generation (32 bit), crc32 +// blocks 1 .. B presence bitmap, one bit per page of the disk, B = ceil(pages / 32768) +// blocks B+1 .. B+pages page i at B+1+i, valid only when its bit is set +// A page is written before its bit is set; the bit blocks are written by sync(), after the pages were made durable, and the +// medium is flushed again afterwards, so a bit never promises a page that did not reach the medium before it. +// A header whose page count or seed digest differs from the current seed is stale and is replaced (the overlay starts empty): an +// overlay is only ever applied on top of the exact seed it was made against. +// +// Ops: bool read(uint64_t block, uint32_t count, void *dst); bool write(uint64_t block, uint32_t count, const void *src); +// bool sync(); (blocks are relative to the window) +#include +#include + +namespace persist { +constexpr uint32_t kBlock = 4096; +constexpr uint32_t kHeaderBytes = 64; + +inline uint32_t crc32 (const uint8_t *data, size_t bytes) +{ + uint32_t crc = 0xffffffffU; + for (size_t i = 0; i < bytes; ++i) { + crc ^= data[i]; + for (unsigned bit = 0; bit < 8; ++bit) crc = (crc >> 1) ^ (0xedb88320U & (0U - (crc & 1))); + } + return ~crc; +} +inline void put32 (uint8_t *p, uint32_t v) { for (int i = 0; i < 4; ++i) p[i] = uint8_t (v >> (8 * i)); } +inline void put64 (uint8_t *p, uint64_t v) { for (int i = 0; i < 8; ++i) p[i] = uint8_t (v >> (8 * i)); } +inline uint32_t get32 (const uint8_t *p) { uint32_t v = 0; for (int i = 3; i >= 0; --i) v = (v << 8) | p[i]; return v; } +inline uint64_t get64 (const uint8_t *p) { uint64_t v = 0; for (int i = 7; i >= 0; --i) v = (v << 8) | p[i]; return v; } + +inline uint64_t bitmapBlocks (uint64_t pages) { return (pages + 8ULL * kBlock - 1) / (8ULL * kBlock); } +inline uint64_t windowBlocks (uint64_t pages) { return 1 + bitmapBlocks (pages) + pages; } +inline uint64_t pageBlock (uint64_t pages, uint64_t index) { return 1 + bitmapBlocks (pages) + index; } + +inline void encodeHeader (uint8_t block[kBlock], uint64_t pages, const uint8_t seed[32], uint64_t generation) +{ + memset (block, 0, kBlock); + memcpy (block, "SHNGDATA", 8); + put32 (block + 8, 1); + put32 (block + 12, kHeaderBytes); + put64 (block + 16, pages); + memcpy (block + 24, seed, 32); + put32 (block + 56, uint32_t (generation)); + // The CRC covers the first 60 bytes (the generation included); bytes 60..63 hold it. + put32 (block + 60, crc32 (block, 60)); +} +// 0 valid and for this seed; 1 blank or foreign (no magic); 2 damaged (CRC, version, size); 3 valid but for another seed/size. +inline int checkHeader (const uint8_t block[kBlock], uint64_t pages, const uint8_t seed[32], uint64_t *generation) +{ + if (memcmp (block, "SHNGDATA", 8)) return 1; + if (get32 (block + 8) != 1 || get32 (block + 12) != kHeaderBytes || get32 (block + 60) != crc32 (block, 60)) return 2; + if (generation) *generation = get32 (block + 56); + if (get64 (block + 16) != pages || memcmp (block + 24, seed, 32)) return 3; + return 0; +} + +enum class Open { Resumed, Fresh, Blank, Failed }; + +struct Overlay { + uint64_t pages = 0, generation = 0, present = 0; + uint8_t *bitmap = nullptr; // bitmapBlocks(pages) * kBlock bytes, owned by the caller + uint8_t *dirty = nullptr; // bitmapBlocks(pages) bytes, owned by the caller + bool readOnly = true, anyDirty = false, active = false; + int header = -1; // checkHeader result at open + + bool has (uint64_t i) const { return bitmap[i >> 3] >> (i & 7) & 1; } + + // `readOnly`: never write; a stale or blank window is reported (Blank) and left alone. Otherwise it is initialised: bitmap + // first, header last, so a tear leaves no valid header. + template Open open (Ops &ops, uint64_t pageCount, const uint8_t seed[32], bool ro, uint8_t *bits, uint8_t *dirtyFlags) + { + pages = pageCount; bitmap = bits; dirty = dirtyFlags; readOnly = ro; anyDirty = false; present = 0; active = false; + uint64_t bb = bitmapBlocks (pages); + uint8_t block[kBlock]; + if (!pages || !ops.read (0, 1, block)) return Open::Failed; + header = checkHeader (block, pages, seed, &generation); + if (header == 0) { + for (uint64_t b = 0; b < bb; ++b) if (!ops.read (1 + b, 1, bitmap + b * kBlock)) return Open::Failed; + memset (dirty, 0, bb); + // Bits past the last page must be clear, or the loader would read blocks outside the window. + for (uint64_t i = pages; i < bb * 8ULL * kBlock; ++i) if (has (i)) return Open::Failed; + for (uint64_t i = 0; i < pages; ++i) present += has (i); + active = true; + return Open::Resumed; + } + memset (bitmap, 0, bb * kBlock); + memset (dirty, 0, bb); + if (ro) return Open::Blank; + uint8_t zero[kBlock] = {}; + for (uint64_t b = 0; b < bb; ++b) if (!ops.write (1 + b, 1, zero)) return Open::Failed; + if (!ops.sync ()) return Open::Failed; + generation = (header == 3 || header == 2) ? generation + 1 : 1; + encodeHeader (block, pages, seed, generation); + if (!ops.write (0, 1, block) || !ops.sync ()) return Open::Failed; + active = true; + return Open::Fresh; + } + + // Calls load(index, page) for every present page, in order. Stops at the first failure. + template bool forEachPresent (Ops &ops, Load &load) + { + uint8_t block[kBlock]; + for (uint64_t i = 0; i < pages; ++i) { + if (!has (i)) continue; + if (!ops.read (pageBlock (pages, i), 1, block) || !load (i, block)) return false; + } + return true; + } + + template bool writePage (Ops &ops, uint64_t index, const void *page) + { + if (readOnly || !active || index >= pages) return false; + if (!ops.write (pageBlock (pages, index), 1, page)) return false; + if (!has (index)) { bitmap[index >> 3] |= uint8_t (1 << (index & 7)); ++present; } + dirty[index / (8ULL * kBlock)] = 1; + anyDirty = true; + return true; + } + + template bool sync (Ops &ops) + { + if (readOnly || !active) return false; + if (!anyDirty) return true; + if (!ops.sync ()) return false; // pages durable before any bit that names them + uint64_t bb = bitmapBlocks (pages); + for (uint64_t b = 0; b < bb; ++b) { + if (!dirty[b]) continue; + if (!ops.write (1 + b, 1, bitmap + b * kBlock)) return false; + } + if (!ops.sync ()) return false; + memset (dirty, 0, bb); + anyDirty = false; + return true; + } +}; +} // namespace persist diff --git a/drivers/NativeRamBlock/SparseDisk.h b/drivers/NativeRamBlock/SparseDisk.h new file mode 100644 index 0000000..e563b1a --- /dev/null +++ b/drivers/NativeRamBlock/SparseDisk.h @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: BSD-2-Clause-Patent +#pragma once +#include +#include +#include +struct RamSparseDisk { + uint8_t **pages; + uint64_t bytes, resident; +}; +static uint32_t ramU32(const uint8_t *p){uint32_t v;memcpy(&v,p,4);return v;} +static uint64_t ramU64(const uint8_t *p){uint64_t v;memcpy(&v,p,8);return v;} +static bool ramPackedLayout(const uint8_t *p,uint64_t n,uint64_t *logical) { + if(!p || n<32 || n>0x10000000 || memcmp(p,"DBRDISK1",8) || ramU32(p+8)!=1 || ramU32(p+12)!=4096)return false; + uint64_t size=ramU64(p+16),count=ramU64(p+24),end=0; + if(!size || size>0x80000000 || (size&4095) || count>65536 || count>(n-32)/16)return false; + uint64_t cursor=32+count*16; + for(uint64_t i=0;isize || len>size-at || len>n-cursor)return false; + end=at+len;cursor+=len; + } + if(cursor!=n)return false; + *logical=size;return true; +} +template static bool ramSparseImport(RamSparseDisk &d,const uint8_t *p,uint64_t n,Ops &ops) { + uint64_t size=0;if(!d.pages || !ramPackedLayout(p,n,&size) || d.bytes!=size || d.resident)return false; + uint64_t count=ramU64(p+24),cursor=32+count*16; + for(uint64_t i=0;i static uint64_t ramSparseIO(RamSparseDisk &d,bool write,uint64_t offset,uint64_t length,Ops &ops,bool &noMemory) { + static const uint8_t zero[4096]={};noMemory=false; + if(!d.pages || !length || offset>d.bytes || length>d.bytes-offset)return 0; + uint64_t done=0; + while(donelength-done)n=length-done; + uint8_t *page=d.pages[index]; + if(write && !page){ + page=ops.allocate();if(!page){noMemory=true;break;} + d.pages[index]=page;d.resident+=4096; + } + uint64_t actual=write?ops.read(done,page+skip,n):ops.copy(done,(page?page:zero)+skip,n); + if(actual>n)break; + done+=actual;if(actual!=n)break; + } + return done; +} diff --git a/drivers/NativeRamBlock/Test/packed-hash.cpp b/drivers/NativeRamBlock/Test/packed-hash.cpp new file mode 100644 index 0000000..82d1a41 --- /dev/null +++ b/drivers/NativeRamBlock/Test/packed-hash.cpp @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: BSD-2-Clause-Patent +// Host-only complete logical-seed check without materializing a dense disk. +#include "../SparseDisk.h" +#include +#include +#include +#include +#include +int main(int argc,char **argv){ + assert(argc==2);FILE *f=fopen(argv[1],"rb");assert(f);assert(fseek(f,0,SEEK_END)==0);long n=ftell(f);assert(n>=32&&n<=0x10000000);rewind(f); + std::vector input(n);assert(fread(input.data(),1,n,f)==static_cast(n));fclose(f); + uint64_t logical=0;assert(ramPackedLayout(input.data(),input.size(),&logical)); + RamSparseDisk d={static_cast(calloc(logical/4096,sizeof(uint8_t*))),logical,0}; + struct Ops{uint8_t *allocate(){return static_cast(calloc(1,4096));}} op; + assert(ramSparseImport(d,input.data(),input.size(),op)); + CC_SHA256_CTX context;CC_SHA256_Init(&context);const uint8_t zero[4096]={};uint8_t digest[32]; + for(uint64_t i=0;i +#include +#include +#include "../PersistOverlay.h" +using namespace persist; +struct Medium { + std::vector durable, cache; // cache holds writes not yet flushed + std::vector>> pending; + unsigned writes = 0, syncs = 0, failWriteAt = ~0u, failSyncAt = ~0u, reads = 0; + explicit Medium(uint64_t blocks) : durable(blocks * kBlock, 0xA5) {} // never-written medium holds junk, not zeros + bool read(uint64_t b, uint32_t n, void *d) { + ++reads; + for (uint32_t i = 0; i < n; ++i) { std::vector v(durable.begin() + (b + i) * kBlock, durable.begin() + (b + i + 1) * kBlock); + for (auto &p : pending) if (p.first == b + i) v = p.second; memcpy((uint8_t *)d + i * kBlock, v.data(), kBlock); } + return true; } + bool write(uint64_t b, uint32_t n, const void *s) { + if (writes++ == failWriteAt) return false; + for (uint32_t i = 0; i < n; ++i) pending.push_back({b + i, std::vector((const uint8_t *)s + i * kBlock, (const uint8_t *)s + (i + 1) * kBlock)}); + return true; } + bool sync() { if (syncs++ == failSyncAt) return false; + for (auto &p : pending) memcpy(&durable[p.first * kBlock], p.second.data(), kBlock); + pending.clear(); return true; } + void powerLoss() { pending.clear(); } // unflushed writes vanish +}; +static std::vector pageOf(uint8_t v) { return std::vector(kBlock, v); } +int main() { + const uint64_t pages = 65536 + 7; // forces two bitmap blocks and a ragged last byte + uint8_t seedA[32], seedB[32]; memset(seedA, 1, 32); memset(seedB, 2, 32); + assert(bitmapBlocks(pages) == 3); + assert(bitmapBlocks(32768) == 1 && bitmapBlocks(32769) == 2 && bitmapBlocks(524288) == 16); + assert(windowBlocks(524288) == 1 + 16 + 524288); + Medium m(windowBlocks(pages)); + std::vector bits(bitmapBlocks(pages) * kBlock), dirty(bitmapBlocks(pages)); + // Blank medium: read-only open must not write and reports Blank; read-write initialises it. + { Overlay o; assert(o.open(m, pages, seedA, true, bits.data(), dirty.data()) == Open::Blank && m.writes == 0 && !o.active && o.header == 1); } + Overlay o; assert(o.open(m, pages, seedA, false, bits.data(), dirty.data()) == Open::Fresh && o.active && o.present == 0 && o.generation == 1); + // Write pages, sync, reopen: resumed with exactly those pages. + assert(o.writePage(m, 0, pageOf(0x11).data()) && o.writePage(m, 40000, pageOf(0x22).data()) && o.writePage(m, pages - 1, pageOf(0x33).data())); + assert(!o.writePage(m, pages, pageOf(0).data())); + assert(o.sync(m)); + { Medium copy = m; Overlay r; std::vector b2(bits.size()), d2(dirty.size()); + assert(r.open(copy, pages, seedA, true, b2.data(), d2.data()) == Open::Resumed && r.present == 3 && r.has(0) && r.has(40000) && r.has(pages - 1) && !r.has(1)); + unsigned seen = 0; struct L { unsigned *seen; bool operator()(uint64_t i, const uint8_t *p) { ++*seen; return p[0] == (i == 0 ? 0x11 : i == 40000 ? 0x22 : 0x33) && p[4095] == p[0]; } } load{&seen}; + assert(r.forEachPresent(copy, load) && seen == 3); + assert(!r.writePage(copy, 1, pageOf(9).data()) && !r.sync(copy)); // read-only really is read-only + } + // A different seed (the seed was rebuilt) makes the overlay stale: it is replaced, not applied. + { Medium copy = m; Overlay r; std::vector b2(bits.size()), d2(dirty.size()); + assert(r.open(copy, pages, seedB, true, b2.data(), d2.data()) == Open::Blank && r.header == 3 && r.present == 0); + assert(r.open(copy, pages, seedB, false, b2.data(), d2.data()) == Open::Fresh && r.generation == 2 && r.present == 0); } + // Power loss before sync: pages written after the last sync are gone, and no bit promises them. + assert(o.writePage(m, 5, pageOf(0x44).data())); m.powerLoss(); + { Overlay r; std::vector b2(bits.size()), d2(dirty.size()); + assert(r.open(m, pages, seedA, true, b2.data(), d2.data()) == Open::Resumed && !r.has(5) && r.present == 3); } + // Torn sync: pages flushed, bitmap write fails: the next open sees the old bitmap (page 5 absent), never a bit without its page. + { Medium t = m; Overlay w; std::vector b2(bits.size()), d2(dirty.size()); + assert(w.open(t, pages, seedA, false, b2.data(), d2.data()) == Open::Resumed); + assert(w.writePage(t, 6, pageOf(0x55).data())); t.failWriteAt = t.writes; // the data write already happened; the bitmap write that sync() issues next fails + assert(!w.sync(t)); t.powerLoss(); + Overlay r; std::vector b3(bits.size()), d3(dirty.size()); + assert(r.open(t, pages, seedA, true, b3.data(), d3.data()) == Open::Resumed && !r.has(6)); } + // Damaged header (flip a bit): reported as damaged, read-write open reinitialises with a higher generation. + { Medium t = m; t.durable[20] ^= 1; Overlay r; std::vector b2(bits.size()), d2(dirty.size()); + assert(r.open(t, pages, seedA, true, b2.data(), d2.data()) == Open::Blank && r.header == 2); + assert(r.open(t, pages, seedA, false, b2.data(), d2.data()) == Open::Fresh && r.present == 0); } + // Bits beyond the last page are corruption. + { Medium t = m; Overlay r; std::vector b2(bits.size()), d2(dirty.size()); + t.durable[1 * kBlock + 2 * kBlock + 100] |= 0x80; // inside bitmap block 3 (last), page index >= pages + assert(r.open(t, pages, seedA, true, b2.data(), d2.data()) == Open::Failed); } + // Failed initialisation leaves no valid header. + { Medium t(windowBlocks(pages)); t.failSyncAt = 0; Overlay r; std::vector b2(bits.size()), d2(dirty.size()); + assert(r.open(t, pages, seedA, false, b2.data(), d2.data()) == Open::Failed); t.powerLoss(); + Overlay q; assert(q.open(t, pages, seedA, true, b2.data(), d2.data()) == Open::Blank && q.header == 1); } + puts("PASS: overlay header, bitmap, ordering, stale seed, torn writes, power loss"); +} diff --git a/drivers/NativeRamBlock/Test/sparse.cpp b/drivers/NativeRamBlock/Test/sparse.cpp new file mode 100644 index 0000000..1a05d00 --- /dev/null +++ b/drivers/NativeRamBlock/Test/sparse.cpp @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: BSD-2-Clause-Patent +#include "../SparseDisk.h" +#include +#include +#include +#include +struct Ops { + std::vector buffer;long budget=-1;uint64_t shortAt=UINT64_MAX; + uint8_t *allocate(){if(budget==0)return nullptr;if(budget>0)--budget;return static_cast(calloc(1,4096));} + uint64_t read(uint64_t at,void *p,uint64_t n){if(at>=shortAt)return 0;if(n>shortAt-at)n=shortAt-at;memcpy(p,buffer.data()+at,n);return n;} + uint64_t copy(uint64_t at,const void *p,uint64_t n){if(at>=shortAt)return 0;if(n>shortAt-at)n=shortAt-at;memcpy(buffer.data()+at,p,n);return n;} +}; +static void put32(std::vector&b,size_t at,uint32_t v){memcpy(b.data()+at,&v,4);} +static void put64(std::vector&b,size_t at,uint64_t v){memcpy(b.data()+at,&v,8);} +static void cleanup(RamSparseDisk &d){for(uint64_t i=0;i packed(48+4096); + memcpy(packed.data(),"DBRDISK1",8);put32(packed,8,1);put32(packed,12,4096);put64(packed,16,size);put64(packed,24,1);put64(packed,32,4096*7);put64(packed,40,4096);packed[48+9]=0xa5; + uint64_t logical=0;assert(ramPackedLayout(packed.data(),packed.size(),&logical)&&logical==size); + for(unsigned which=0;which<9;++which){auto bad=packed; + if(which==0)bad[0]=0; + if(which==1)put32(bad,8,2); + if(which==2)put32(bad,12,16384); + if(which==3)put64(bad,16,0x80001000); + if(which==4)put64(bad,24,65537); + if(which==5)put64(bad,32,1); + if(which==6)put64(bad,40,0); + if(which==7)bad.pop_back(); + if(which==8)bad.push_back(0); + assert(!ramPackedLayout(bad.data(),bad.size(),&logical)); + } + Ops op;RamSparseDisk d={static_cast(calloc(size/4096,sizeof(uint8_t*))),size,0}; + assert(ramSparseImport(d,packed.data(),packed.size(),op)&&d.resident==4096); + std::vector flat(size);flat[7*4096+9]=0xa5;uint64_t random=42; + for(unsigned i=0;i<4000;++i){random=random*6364136223846793005ULL+1;uint64_t at=random%size; + uint64_t n=1+((random>>32)%9000);if(n>size-at)n=size-at; + op.buffer.assign(n,0);bool write=(i%3)!=0; + if(write)for(size_t j=0;j(i+j); + bool oom=false;assert(ramSparseIO(d,write,at,n,op,oom)==n&&!oom); + if(write)memcpy(flat.data()+at,op.buffer.data(),n);else assert(!memcmp(flat.data()+at,op.buffer.data(),n)); + } + op.buffer.assign(size,0);bool oom=false;assert(ramSparseIO(d,false,0,size,op,oom)==size && op.buffer==flat); + assert(ramSparseIO(d,false,UINT64_MAX,1,op,oom)==0);assert(ramSparseIO(d,true,size,1,op,oom)==0);cleanup(d); + d={static_cast(calloc(2,sizeof(uint8_t*))),8192,0};op.buffer.assign(8192,0x37);op.budget=1; + assert(ramSparseIO(d,true,0,8192,op,oom)==4096&&oom&&d.resident==4096&&!d.pages[1]); + op.buffer.assign(8192,0);op.budget=-1;assert(ramSparseIO(d,false,0,8192,op,oom)==8192&&!oom); + for(size_t i=0;i<8192;++i)assert(op.buffer[i]==(i<4096?0x37:0)); + op.shortAt=100;op.buffer.assign(4096,0x55);assert(ramSparseIO(d,true,0,4096,op,oom)==100&&!oom); + assert(d.pages[0][99]==0x55&&d.pages[0][100]==0x37);cleanup(d); + d={static_cast(calloc(size/4096,sizeof(uint8_t*))),size,0};op.budget=0; + assert(!ramSparseImport(d,packed.data(),packed.size(),op)&&d.resident==0);cleanup(d); + puts("PASS: actual sparse parser/import/I/O, 4000 differential operations, overflow/malformed/short-copy/allocation failures"); +} diff --git a/drivers/NativeUfs/GccUfs.h b/drivers/NativeUfs/GccUfs.h new file mode 100644 index 0000000..5d11663 --- /dev/null +++ b/drivers/NativeUfs/GccUfs.h @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: BSD-2-Clause-Patent +// SM8550 GCC registers used by UFS, transcribed as facts from the public +// Linux gcc-sm8550.c clock driver. No write is performed by NativeUfs yet. +#ifndef NATIVE_UFS_GCC_UFS_H +#define NATIVE_UFS_GCC_UFS_H + +// Physical windows from the sheng board DT (clock-controller@100000, ufshc@1d84000). +#define GCC_BASE 0x00100000ULL +#define UFS_HC_BASE 0x01d84000ULL +#define UFS_HC_SIZE 0x3000ULL + +#define GCC_UFS_PHY_GDSCR 0x77004 +#define GCC_UFS_MEM_PHY_GDSCR 0x9e000 + +#define GCC_UFS_PHY_AXI_ENABLE 0x77018 +#define GCC_UFS_PHY_AHB_ENABLE 0x77024 +#define GCC_UFS_PHY_TX_SYMBOL0_ENABLE 0x77028 +#define GCC_UFS_PHY_RX_SYMBOL0_ENABLE 0x7702c +#define GCC_UFS_PHY_UNIPRO_CORE_ENABLE 0x77068 +#define GCC_UFS_PHY_ICE_CORE_ENABLE 0x77074 +#define GCC_UFS_PHY_PHY_AUX_ENABLE 0x770b0 +#define GCC_UFS_PHY_RX_SYMBOL1_ENABLE 0x770cc +// bus_aggr_clk in the board DT: gcc_aggre_ufs_phy_axi_clk, halt/enable 0x770e4, HW clock gating bit1 +#define GCC_AGGRE_UFS_PHY_AXI_ENABLE 0x770e4 + +#define GCC_UFS_BRANCH_ENABLE_BIT 0x1U + +// gcc_ufs_phy_ahb_clk and gcc_aggre_ufs_phy_axi_clk set hwcg_bit=1 in gcc-sm8550.c, so CLK_OFF=1 on +// an idle branch is normal (hardware 2026-10-17 read ahb=80000003). +// The reset and RPMh/TCSR/QMP-PHY programming are not represented here: their +// semantics are not qualified and NativeUfs must not write them speculatively. +#endif diff --git a/drivers/NativeUfs/Info.plist b/drivers/NativeUfs/Info.plist new file mode 100644 index 0000000..2e36595 --- /dev/null +++ b/drivers/NativeUfs/Info.plist @@ -0,0 +1,15 @@ + + + + CFBundleIdentifierorg.blu-sharky.NativeUfs + CFBundleNameNativeUfs + CFBundleExecutableNativeUfs + CFBundlePackageTypeKEXT + CFBundleVersion0.1.0 + OSBundleRequiredRoot + IOKitPersonalitiessheng SM8550 UFSHCI + IOClassNativeUfs + IOProviderClassIOPlatformDevice + IOProbeScore1000 + + diff --git a/drivers/NativeUfs/LICENSE b/drivers/NativeUfs/LICENSE new file mode 100644 index 0000000..52a23a0 --- /dev/null +++ b/drivers/NativeUfs/LICENSE @@ -0,0 +1,25 @@ +BSD 2-Clause License + +Copyright (c) 2026, blu-sharky +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/drivers/NativeUfs/NativeUfs.cpp b/drivers/NativeUfs/NativeUfs.cpp new file mode 100644 index 0000000..9e1f22f --- /dev/null +++ b/drivers/NativeUfs/NativeUfs.cpp @@ -0,0 +1,1862 @@ +// SPDX-License-Identifier: BSD-2-Clause-Patent +// Native sheng SM8550 UFS bring-up. Default-off: no MMIO is touched unless +// native-ufs-probe=1 is present. This is a register probe, not a storage driver. +#include +#include +#include +#include +#include +#include +#include +#if defined(NATIVE_VERIFIED_BLOCK) && NATIVE_VERIFIED_BLOCK +#include "verified_root.h" +#include "VerifiedBlockPolicy.h" +extern "C" { +extern const struct vr_crypto vr_xnu_crypto; +extern const uint8_t verified_root_key[]; +} +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "GccUfs.h" +#include "RootDiagnostics.h" +#include "TransferPoll.h" +#include "SmmuTakeover.h" + +static UfsRootDiagnostics gRootDiagnostic; +static bool gRootDiagnosticEnabled; +static void rootDiagnosticFailure (const char *reason); + +extern "C" { +// Optional passive observer; absent in ordinary builds. It may run while a +// window lock is held, so its contract forbids blocking or transport access. +void boot_progress_poll (void) __attribute__((weak)); +volatile uint32_t native_ufs_probe_calls = 0; +volatile uint32_t native_ufs_matched = 0; +volatile uint32_t native_ufs_probe_requested = 0; +volatile uint32_t native_ufs_mapped = 0; +volatile uint32_t native_ufs_status = 0; +// Root start: 1 entered, 2 host up, 3 layout found; the last soft-failure reason (a static string), for the boot page. +volatile uint32_t native_ufs_root_step = 0; +const char *volatile native_ufs_root_failure = nullptr; +volatile uint32_t native_ufs_cap = 0; +volatile uint32_t native_ufs_version = 0; +volatile uint32_t native_ufs_hcs = 0; +volatile uint32_t native_ufs_hce = 0; +volatile uint32_t native_ufs_utrl_ready = 0; +volatile uint32_t native_ufs_uic_status = 0; +volatile uint32_t native_ufs_uic_result = 0; +} + +class NativeUfs : public IOService { + OSDeclareDefaultStructors (NativeUfs) + OSSharedPtr fMap; + OSSharedPtr fCmdDesc; + void inspect (void); + void adoptAndRead (unsigned level); + void dumpFirmwareList (void); + void inspectSmmu (void); + void inspectContextBanks (void); + bool startRoot (IOService *provider); + static bool waitSet (uintptr_t base, unsigned offset, uint32_t mask, unsigned loops); +public: + IOService *probe (IOService *provider, SInt32 *score) override; + bool start (IOService *provider) override; + void stop (IOService *provider) override; +}; + +OSDefineMetaClassAndStructors (NativeUfs, IOService) + +static uint32_t read32 (uintptr_t base, unsigned offset) +{ + return *reinterpret_cast (base + offset); +} + +static void write32 (uintptr_t base, unsigned offset, uint32_t value) +{ + *reinterpret_cast (base + offset) = value; +} + +static bool contains (const char *data, size_t length, const char *needle) +{ + size_t wanted = 0; + while (needle[wanted]) { + wanted++; + } + for (size_t at = 0; at + wanted <= length; at++) { + size_t index = 0; + while (index < wanted && data[at + index] == needle[index]) { + index++; + } + if (index == wanted) { + return true; + } + } + return false; +} + +IOService *NativeUfs::probe (IOService *provider, SInt32 *score) +{ + native_ufs_probe_calls = native_ufs_probe_calls + 1; + if (OSDynamicCast (IOPlatformExpertDevice, provider)) { + // Second personality (KernelConfigTables: IOMatchCategory NativeUfsRoot): attaches to the + // root device, which starts before the platform expert publishes IOResources, so the + // hardware probes run before the original AMFI (whose start panics without a PMGR). + native_ufs_matched = 1; + if (score) { + *score = 500; + } + return this; + } + OSObject *property = provider->getProperty ("compatible"); + OSData *data = OSDynamicCast (OSData, property); + OSString *string = OSDynamicCast (OSString, property); + bool match = false; + if (data) { + const char *text = static_cast (data->getBytesNoCopy ()); + match = text && contains (text, data->getLength (), "qcom,sm8550-ufshc"); + } else if (string) { + const char *text = string->getCStringNoCopy (); + match = text && contains (text, string->getLength (), "qcom,sm8550-ufshc"); + } + if (!match) { + return nullptr; + } + native_ufs_matched = 1; + if (score) { + *score = 1000; + } + return this; +} + +bool NativeUfs::waitSet (uintptr_t base, unsigned offset, uint32_t mask, unsigned loops) +{ + for (unsigned index = 0; index < loops; index++) { + if ((read32 (base, offset) & mask) == mask) { + return true; + } + IODelay (1); + } + return false; +} + +// Read-only inspection (native-ufs-probe=2): reports what firmware left behind. +// The physical windows are the sheng board DT's own GCC (0x100000) and UFS host +// (0x1d84000, 0x3000) ranges. The controller is read only if the GCC says its +// GDSC is powered and the AXI/AHB branches are running; nothing is written. +static uintptr_t mapPhys (uint64_t phys, uint64_t length, OSSharedPtr &keep) +{ + IODeviceMemory *memory = IODeviceMemory::withRange (phys, length); + if (!memory) { + return 0; + } + keep = OSSharedPtr (memory->map (kIOMapAnywhere | kIOMapInhibitCache), OSNoRetain); + memory->release (); + return keep ? static_cast (keep->getVirtualAddress ()) : 0; +} + +void NativeUfs::inspect (void) +{ + OSSharedPtr gccMap, memPhyMap, hcMap; + uintptr_t gcc = mapPhys (GCC_BASE + 0x77000, 0x1000, gccMap); + uintptr_t memPhy = mapPhys (GCC_BASE + 0x9e000, 0x1000, memPhyMap); + if (!gcc || !memPhy) { + panic ("NATIVE_UFS inspect: GCC map failed gcc=%p memphy=%p", (void *) gcc, (void *) memPhy); + } + uint32_t gdsc = read32 (gcc, GCC_UFS_PHY_GDSCR - 0x77000); + uint32_t axi = read32 (gcc, GCC_UFS_PHY_AXI_ENABLE - 0x77000); + uint32_t ahb = read32 (gcc, GCC_UFS_PHY_AHB_ENABLE - 0x77000); + uint32_t unipro = read32 (gcc, GCC_UFS_PHY_UNIPRO_CORE_ENABLE - 0x77000); + uint32_t txs = read32 (gcc, GCC_UFS_PHY_TX_SYMBOL0_ENABLE - 0x77000); + uint32_t rxs = read32 (gcc, GCC_UFS_PHY_RX_SYMBOL0_ENABLE - 0x77000); + uint32_t aux = read32 (gcc, GCC_UFS_PHY_PHY_AUX_ENABLE - 0x77000); + uint32_t memGdsc = read32 (memPhy, 0); + // GDSCR bit31 PWR_ON; CBCR bit31 CLK_OFF. + bool powered = (gdsc & 0x80000000U) != 0; + // Hardware 2026-10-17: AXI running, AHB enabled but CLK_OFF (bit31). With bit1 (HW gating) + // set the branch wakes on access, so enable bits, not CLK_OFF, gate the read-only probe. + bool busClocks = !(axi & 0x80000000U) && (ahb & 0x1U) && (axi & 0x1U); + uint32_t cap = 0, ver = 0, hcs = 0, hce = 0, utrlba = 0, is = 0; + unsigned readHc = 0; + if (powered && busClocks) { + uintptr_t hc = mapPhys (UFS_HC_BASE, UFS_HC_SIZE, hcMap); + if (hc) { + cap = read32 (hc, 0x00); + ver = read32 (hc, 0x08); + is = read32 (hc, 0x20); + hcs = read32 (hc, 0x30); + hce = read32 (hc, 0x34); + utrlba = read32 (hc, 0x50); + readHc = 1; + } + } + panic ("NATIVE_UFS inspect\n" + "gdsc=%08x memphygdsc=%08x\n" + "axi=%08x ahb=%08x unipro=%08x\n" + "txsym=%08x rxsym=%08x aux=%08x\n" + "hc_read=%u cap=%08x ver=%08x\n" + "is=%08x hcs=%08x hce=%08x utrlba=%08x", + gdsc, memGdsc, axi, ahb, unipro, txs, rxs, aux, readHc, cap, ver, is, hcs, hce, utrlba); +} + +// Adopt the controller the firmware left running (native-ufs-probe=3) and issue +// read-only SCSI commands through one UTP transfer request slot. Nothing is written +// to the storage. The controller is not reset and the link is not restarted; the +// firmware's request list is stopped (refused if a request is outstanding) and replaced +// by ours. Interrupts stay masked and completion is polled. +namespace { +struct UfsResult { + uint32_t ocs; // UTRD overall command status; 0xf = never completed + uint32_t response; // UPIU response byte + uint32_t scsi; // SCSI status byte + uint32_t sense; // sense key<<16 | ASC<<8 | ASCQ from the response UPIU's autosense data + uint32_t senseLen; // sense data length reported by the device + uint32_t seg; // UPIU data segment length +}; + +static void be32 (uint8_t *at, uint32_t value) +{ + at[0] = value >> 24; at[1] = value >> 16; at[2] = value >> 8; at[3] = value; +} + +static uint32_t getBe32 (const uint8_t *at) +{ + return (uint32_t (at[0]) << 24) | (uint32_t (at[1]) << 16) | (uint32_t (at[2]) << 8) | at[3]; +} +} + +static void cacheClean (const void *start, size_t length) +{ + uintptr_t address = reinterpret_cast (start) & ~uintptr_t (63); + uintptr_t end = reinterpret_cast (start) + length; + for (; address < end; address += 64) { + __asm__ volatile ("dc cvac, %0" :: "r" (address) : "memory"); + } + __asm__ volatile ("dsb sy" ::: "memory"); +} + +static void cacheCleanInvalidate (const void *start, size_t length) +{ + uintptr_t address = reinterpret_cast (start) & ~uintptr_t (63); + uintptr_t end = reinterpret_cast (start) + length; + for (; address < end; address += 64) { + __asm__ volatile ("dc civac, %0" :: "r" (address) : "memory"); + } + __asm__ volatile ("dsb sy" ::: "memory"); +} + +// The region is mapped as device memory: only aligned element-sized stores are allowed +// (bzero's block zeroing faulted on hardware, 2026-10-17). +static void zero32 (void *start, size_t length) +{ + volatile uint32_t *word = static_cast (start); + for (size_t index = 0; index < length / 4; index++) { + word[index] = 0; + } +} + + +// Give the UFS stream a pass-through mapping the way Linux does on this firmware. +// Hardware 2026-10-17: after hand-off apps_smmu has sCR0 = 0x002f0406 (USFCFG=1, unmatched +// streams fault), no stream-match entry for the UFS host (ID 0x60, `iommus = <&apps_smmu 0x60 +// 0x0>` in the board DT), a TYPE=BYPASS S2CR reads back as FAULT, and sCR0 writes are ignored. +// Linux (drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c, qcom_smmu_cfg_probe + qcom_smmu_write_s2cr, +// "bypass_quirk") meets exactly this: it reserves the LAST context bank (num_context_banks-1), +// writes SCTLR=0 (stage 1 disabled) and CBAR.TYPE=S1_TRANS_S2_BYPASS there, and maps a bypass +// stream as S2CR TYPE=TRANSLATE with CBNDX = that bank. S2CR is written before SMR +// (arm_smmu_write_sme). ARM IHI 0062D.c table 3-1 additionally requires VA64=1 +// for inputs above 32 bits even with SCTLR.M=0. Storage DMA path only; this +// is physical pass-through, not a page-table or Adreno 4 KiB qualification. +static void smmuBypassStream (uint16_t stream, OSSharedPtr &keep, uint32_t *report, + uint64_t dma, uint64_t bytes) +{ + uintptr_t smmu = mapPhys (0x15000000ULL, 0xd0000, keep); + if (!smmu) { + panic ("NATIVE_UFS smmu: map failed"); + } + UfsSmmuPlan plan = planUfsSmmu (stream, dma, bytes, [smmu](unsigned offset) { return read32 (smmu, offset); }); + gRootDiagnostic.smmuId1 = read32 (smmu, 0x24); + gRootDiagnostic.smmuId2 = read32 (smmu, 0x28); + gRootDiagnostic.smmuSlot = plan.slot; + if (plan.error) { + gRootDiagnostic.stage = "SMMU planning"; + gRootDiagnostic.result = static_cast (kIOReturnNotReady); + gRootDiagnostic.smr = plan.smr; gRootDiagnostic.s2cr = plan.s2cr; + rootDiagnosticFailure (plan.error); + panic ("NATIVE_UFS smmu: %s slot=%u smr=%x s2cr=%x", plan.error, plan.slot, plan.smr, plan.s2cr); + } + unsigned bank = plan.bank, index = plan.slot; + uintptr_t cb = smmu + (plan.pages + bank) * 4096; + const char *failure = configureUfsSmmu (plan, stream, + [smmu](unsigned offset) { return read32 (smmu, offset); }, + [smmu](unsigned offset, uint32_t value) { + write32 (smmu, offset, value); + __asm__ volatile ("dsb sy" ::: "memory"); + }); + uint32_t smr = read32 (smmu, 0x800 + 4 * index), s2cr = read32 (smmu, 0xc00 + 4 * index); + uint32_t cbar = read32 (smmu, 0x1000 + 4 * bank), sctlr = read32 (cb, 0x0); + gRootDiagnostic.smr = smr; gRootDiagnostic.s2cr = s2cr; + gRootDiagnostic.cbar = cbar; gRootDiagnostic.sctlr = sctlr; + gRootDiagnostic.smmuId2 = read32 (smmu, 0x28); + gRootDiagnostic.cba2r = read32 (smmu, 0x1800 + 4 * bank); + if (failure) { + gRootDiagnostic.stage = "SMMU readback"; + gRootDiagnostic.result = static_cast (kIOReturnNotReady); + rootDiagnosticFailure (failure); + panic ("NATIVE_UFS smmu: %s idx=%u bank=%u cba2r=%x", failure, index, bank, + gRootDiagnostic.cba2r); + } + report[0] = index | (bank << 16); + report[1] = cbar; + report[2] = sctlr; +} + +void NativeUfs::adoptAndRead (unsigned level) +{ + OSSharedPtr hcMap; + uintptr_t hc = mapPhys (UFS_HC_BASE, UFS_HC_SIZE, hcMap); + if (!hc) { + panic ("NATIVE_UFS adopt: controller map failed"); + } + // The controller's own request list is the one region its DMA is known to reach + // (hardware 2026-10-17: a kernel-allocated buffer at 0xf080f000 reset the system when the + // first command was issued, and nothing below 0xc8000000 is allocatable). The firmware + // zeroed it; use 16 KiB from UTRLBA as ordinary cached memory (the DT marks the host + // dma-coherent). Test instrument only: the kernel may also own these pages. + uint64_t fwList = (uint64_t (read32 (hc, 0x54)) << 32) | read32 (hc, 0x50); + if (!fwList || (fwList & 0xfff)) { + panic ("NATIVE_UFS adopt: no usable firmware list base=%llx", fwList); + } + IOMemoryDescriptor *region = IOMemoryDescriptor::withPhysicalAddress (fwList, 16384, kIODirectionInOut); + OSSharedPtr regionMap; + if (region) { + regionMap = OSSharedPtr (region->map (kIOMapAnywhere), OSNoRetain); + region->release (); + } + if (!regionMap) { + panic ("NATIVE_UFS adopt: firmware list map failed base=%llx", fwList); + } + uint8_t *virt = reinterpret_cast (regionMap->getVirtualAddress ()); + uint64_t phys = fwList; + zero32 (virt, 16384); + cacheClean (virt, 16384); + // Layout: [0,4096) request list (slot 0 used), [4096,5120) command + response UPIU + + // PRDT, [8192,16384) data. + const uint32_t ucdOffset = 4096, rspOffset = 512, prdtOffset = 1024, dataOffset = 8192; + uint32_t hcsBefore = read32 (hc, 0x30); + uint32_t doorbellBefore = read32 (hc, 0x58); + if (doorbellBefore != 0 || read32 (hc, 0x78) != 0) { + panic ("NATIVE_UFS adopt: firmware request outstanding doorbell=%08x hcs=%08x", doorbellBefore, hcsBefore); + } + OSSharedPtr smmuMap; + uint32_t smmuReport[3] = { 0, 0, 0 }; + write32 (hc, 0x24, 0); // mask interrupts + write32 (hc, 0x60, 0); // stop the firmware's request list + __asm__ volatile ("dsb sy" ::: "memory"); + unsigned stopLoops = 10000; + while ((read32 (hc, 0x60) & 1) && stopLoops--) IODelay (10); + if (read32 (hc, 0x60) & 1) panic ("NATIVE_UFS adopt: request list did not stop"); + smmuBypassStream (0x60, smmuMap, smmuReport, phys, 16384); + write32 (hc, 0x50, static_cast (phys)); + write32 (hc, 0x54, static_cast (phys >> 32)); + write32 (hc, 0x20, 0xffffffffU); // clear stale status + write32 (hc, 0x60, 1); + if (!waitSet (hc, 0x30, 0x2, 100000)) { + panic ("NATIVE_UFS adopt: request list not ready hcs=%08x", read32 (hc, 0x30)); + } + if (level == 3) { + panic ("NATIVE_UFS adopt: list replaced\n" + "phys=%llx hcs=%08x->%08x\n" + "is=%08x db=%08x rsr=%08x ba=%08x bau=%08x ie=%08x", + phys, hcsBefore, read32 (hc, 0x30), read32 (hc, 0x20), read32 (hc, 0x58), + read32 (hc, 0x60), read32 (hc, 0x50), read32 (hc, 0x54), read32 (hc, 0x24)); + } + + auto command = [&] (uint8_t lun, const uint8_t *cdb, unsigned cdbLength, bool toHost, uint32_t length) -> UfsResult { + uint8_t *utrd = virt; + uint8_t *ucd = virt + ucdOffset; + zero32 (utrd, 32); + zero32 (ucd, 1024 + 16); + zero32 (virt + dataOffset, 8192); + ucd[0] = 0x01; // COMMAND UPIU + ucd[1] = toHost ? 0x40 : 0x00; // flags: read + ucd[2] = lun; + ucd[3] = 0; // task tag + be32 (ucd + 12, length); // expected data transfer length + for (unsigned index = 0; index < cdbLength; index++) { + ucd[16 + index] = cdb[index]; + } + uint64_t ucdPhys = phys + ucdOffset; + uint32_t direction = length ? (toHost ? 2U : 1U) : 0U; + *reinterpret_cast (utrd + 0) = 0x10000000U | (direction << 25); + *reinterpret_cast (utrd + 8) = 0x0fU; // OCS: not completed + *reinterpret_cast (utrd + 16) = static_cast (ucdPhys); + *reinterpret_cast (utrd + 20) = static_cast (ucdPhys >> 32); + *reinterpret_cast (utrd + 24) = (512U >> 2) | ((rspOffset >> 2) << 16); // response length | offset (dwords) + uint32_t prdEntries = length ? 1U : 0U; + *reinterpret_cast (utrd + 28) = prdEntries | ((prdtOffset >> 2) << 16); + if (length) { + uint64_t dataPhys = phys + dataOffset; + uint8_t *prd = ucd + prdtOffset; + *reinterpret_cast (prd + 0) = static_cast (dataPhys); + *reinterpret_cast (prd + 4) = static_cast (dataPhys >> 32); + *reinterpret_cast (prd + 12) = length - 1; + } + cacheClean (virt, 16384); + write32 (hc, 0x20, 0xffffffffU); + write32 (hc, 0x58, 1); // ring slot 0 + UfsResult result = { 0xf, 0xff, 0xff, 0, 0, 0 }; + bool done = false; + for (unsigned loops = 0; loops < 100000; loops++) { + if (!(read32 (hc, 0x58) & 1)) { + done = true; + break; + } + IODelay (1); + } + cacheCleanInvalidate (virt, 16384); + if (!done) { + return result; + } + result.ocs = *reinterpret_cast (utrd + 8) & 0xff; + const uint8_t *rsp = ucd + rspOffset; + result.response = rsp[6]; + result.scsi = rsp[7]; + // Response UPIU: 12-byte header, residual count, 16 reserved bytes, then the data segment + // (2-byte sense length + sense data) at offset 32. + result.seg = (uint32_t (rsp[10]) << 8) | rsp[11]; + result.senseLen = (uint32_t (rsp[32]) << 8) | rsp[33]; + result.sense = ((rsp[36] & 0xfU) << 16) | (uint32_t (rsp[46]) << 8) | rsp[47]; + return result; + }; + + // NOP OUT (kind 0) and Query read requests (kind 1: opcode 0x05 READ FLAG, 0x03 READ ATTRIBUTE). + // Read-only: no data segment, no state change on the device. Returns the UPIU response byte and + // the 32-bit value field (attributes) or flag byte via *value; ocs 0xf means no completion. + auto query = [&] (unsigned kind, uint8_t opcode, uint8_t idn, uint32_t *value, uint32_t *respByte) -> uint32_t { + uint8_t *utrd = virt; + uint8_t *ucd = virt + ucdOffset; + zero32 (utrd, 32); + zero32 (ucd, 1024 + 16); + if (kind == 0) { + ucd[0] = 0x00; // NOP OUT + } else { + ucd[0] = 0x16; // QUERY REQUEST + ucd[5] = 0x01; // query function (header dword 1, byte 1): standard read request + ucd[12] = opcode; + ucd[13] = idn; + } + ucd[3] = 1; // task tag + uint64_t ucdPhys = phys + ucdOffset; + *reinterpret_cast (utrd + 0) = 0x10000000U; + *reinterpret_cast (utrd + 8) = 0x0fU; + *reinterpret_cast (utrd + 16) = static_cast (ucdPhys); + *reinterpret_cast (utrd + 20) = static_cast (ucdPhys >> 32); + *reinterpret_cast (utrd + 24) = (512U >> 2) | ((rspOffset >> 2) << 16); + *reinterpret_cast (utrd + 28) = (prdtOffset >> 2) << 16; + cacheClean (virt, 16384); + write32 (hc, 0x20, 0xffffffffU); + write32 (hc, 0x58, 1); + bool done = false; + for (unsigned loops = 0; loops < 100000; loops++) { + if (!(read32 (hc, 0x58) & 1)) { + done = true; + break; + } + IODelay (1); + } + cacheCleanInvalidate (virt, 16384); + if (!done) { + return 0xf; + } + const uint8_t *rsp = ucd + rspOffset; + *respByte = (uint32_t (rsp[0]) << 8) | rsp[6]; // response type << 8 | query/NOP response + *value = getBe32 (rsp + 20); + return *reinterpret_cast (utrd + 8) & 0xff; + }; + // Device state before waking it. Hardware 2026-10-17: NOP OUT answers, fDeviceInit = 0 (done), + // bCurrentPowerMode = 0x33 (UFS-PowerDown) and every LU returns NOT READY. Linux resumes the + // device with START STOP UNIT (power condition ACTIVE) to the UFS Device well-known LU 0xD0 + // (ufshcd_set_dev_pwr_mode); no user data is touched. + uint32_t powerBefore = 0, powerAfter = 0, queryResp = 0, ignoredValue = 0; + uint32_t wakeStatus = 0xee; + if (level == 5 || level == 9) { + query (1, 0x03, 0x02, &powerBefore, &queryResp); // READ ATTRIBUTE bCurrentPowerMode + uint8_t ssu[6] = { 0x1b, 0, 0, 0, 0x10, 0 }; // START STOP UNIT, power condition ACTIVE + UfsResult w = command (0xd0, ssu, 6, false, 0); + wakeStatus = (w.ocs << 16) | (w.response << 8) | w.scsi; + for (unsigned settle = 0; settle < 50; settle++) { + IODelay (1000); + } + query (1, 0x03, 0x02, &powerAfter, &queryResp); + (void) ignoredValue; + } + if (level == 4) { + uint8_t tur0[6] = { 0, 0, 0, 0, 0, 0 }; + UfsResult t = command (0, tur0, 6, false, 0); + const uint8_t *rsp0 = virt + ucdOffset + rspOffset; + panic ("NATIVE_UFS adopt: TEST UNIT READY lun0\n" + "ocs=%x resp=%02x scsi=%02x\n" + "phys=%llx hcs=%08x->%08x smmu idx/bank=%08x cbar=%08x\n" + "is=%08x db=%08x rsr=%08x\n" + "rsp=%08x %08x", + t.ocs, t.response, t.scsi, phys, hcsBefore, read32 (hc, 0x30), smmuReport[0], smmuReport[1], + read32 (hc, 0x20), read32 (hc, 0x58), read32 (hc, 0x60), getBe32 (rsp0), getBe32 (rsp0 + 4)); + } + uint32_t hcsAfter = 0; + const unsigned maxLun = 6; + uint32_t status[maxLun], last[maxLun], blockSize[maxLun], senseOf[maxLun]; + for (unsigned lun = 0; lun < maxLun; lun++) { + status[lun] = 0xee; last[lun] = 0; blockSize[lun] = 0; senseOf[lun] = 0; + uint8_t tur[6] = { 0x00, 0, 0, 0, 0, 0 }; + UfsResult r = { 0xf, 0xff, 0xff, 0, 0, 0 }; + for (unsigned attempt = 0; attempt < 3; attempt++) { + r = command (static_cast (lun), tur, 6, false, 0); + if (lun == 0 && attempt == 0 && r.ocs == 0xf) { + // Never completed: stop here and report instead of waiting out every command. + const uint8_t *rsp = virt + ucdOffset + rspOffset; + panic ("NATIVE_UFS adopt: first command did not complete\n" + "phys=%llx hcs=%08x->%08x\n" + "is=%08x db=%08x rsr=%08x ba=%08x\n" + "utrd2=%08x rsp=%08x %08x", + phys, hcsBefore, read32 (hc, 0x30), + read32 (hc, 0x20), read32 (hc, 0x58), read32 (hc, 0x60), read32 (hc, 0x50), + *reinterpret_cast (virt + 8), getBe32 (rsp), getBe32 (rsp + 4)); + } + if (r.ocs == 0 && r.scsi == 0) { + break; + } + } + uint8_t cap[10] = { 0x25, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; + r = command (static_cast (lun), cap, 10, true, 8); + status[lun] = (r.ocs << 16) | (r.response << 8) | r.scsi; + senseOf[lun] = r.sense; + if (r.ocs == 0 && r.scsi == 0) { + last[lun] = getBe32 (virt + dataOffset); + blockSize[lun] = getBe32 (virt + dataOffset + 4); + } + } + if (level == 9) { + // Read-only GPT survey of LUN 0 (4096-byte blocks): header fields and CRCs for before/after + // comparison, then every non-empty entry among the first 64. Nothing is written. + if (blockSize[0] != 4096) { + panic ("NATIVE_UFS gpt: unexpected LUN0 block size %08x", blockSize[0]); + } + uint8_t rd[10] = { 0x28, 0, 0, 0, 0, 1, 0, 0, 1, 0 }; + UfsResult h = command (0, rd, 10, true, 4096); + if (h.ocs != 0 || h.scsi != 0) { + panic ("NATIVE_UFS gpt: header read failed ocs=%x resp=%x scsi=%x", h.ocs, h.response, h.scsi); + } + const uint8_t *hdr = virt + dataOffset; + auto le32 = [] (const uint8_t *at) -> uint32_t { + return uint32_t (at[0]) | (uint32_t (at[1]) << 8) | (uint32_t (at[2]) << 16) | (uint32_t (at[3]) << 24); + }; + auto le64 = [&] (const uint8_t *at) -> uint64_t { + return uint64_t (le32 (at)) | (uint64_t (le32 (at + 4)) << 32); + }; + uint32_t sig0 = le32 (hdr), sig1 = le32 (hdr + 4), hdrCrc = le32 (hdr + 16), entCrc = le32 (hdr + 88); + uint64_t curLba = le64 (hdr + 24), bakLba = le64 (hdr + 32), firstUsable = le64 (hdr + 40), lastUsable = le64 (hdr + 48); + uint64_t entLba = le64 (hdr + 72); + uint32_t entCount = le32 (hdr + 80), entSize = le32 (hdr + 84); + uint32_t guid0 = le32 (hdr + 56), guid1 = le32 (hdr + 60), guid2 = le32 (hdr + 64), guid3 = le32 (hdr + 68); + if (sig0 != 0x20494645U || sig1 != 0x54524150U || entSize != 128 || entLba > 64) { + panic ("NATIVE_UFS gpt: unexpected header sig=%08x %08x esz=%u elba=%llx", sig0, sig1, entSize, entLba); + } + uint32_t shown = entCount > 64 ? 64 : entCount; + uint32_t blocks = (shown * 128 + 4095) / 4096; + uint8_t rd2[10] = { 0x28, 0, 0, 0, 0, uint8_t (entLba), 0, 0, uint8_t (blocks), 0 }; + UfsResult e = command (0, rd2, 10, true, blocks * 4096); + if (e.ocs != 0 || e.scsi != 0) { + panic ("NATIVE_UFS gpt: entries read failed ocs=%x scsi=%x", e.ocs, e.scsi); + } + const uint8_t *ents = virt + dataOffset; + static char text[3400]; + int used = snprintf (text, sizeof (text), + "NATIVE_UFS gpt LUN0 hdr crc=%08x ents crc=%08x cnt=%u\n" + "cur=%llx backup=%llx usable=%llx..%llx elba=%llx\n" + "disk guid %08x %08x %08x %08x last=%08x\n", + hdrCrc, entCrc, entCount, curLba, bakLba, firstUsable, lastUsable, entLba, + guid0, guid1, guid2, guid3, last[0]); + unsigned column = 0, listed = 0; + for (uint32_t index = 0; index < shown && used < int (sizeof (text)) - 80; index++) { + const uint8_t *entry = ents + index * 128; + bool empty = true; + for (unsigned b = 0; b < 16; b++) { + if (entry[b]) { empty = false; break; } + } + if (empty) { + continue; + } + char name[14]; + for (unsigned c = 0; c < 13; c++) { + uint8_t lo = entry[56 + 2 * c], hi = entry[57 + 2 * c]; + name[c] = (hi == 0 && lo >= 32 && lo < 127) ? char (lo) : (lo ? '?' : 0); + if (!name[c]) { name[c] = 0; break; } + } + name[13] = 0; + used += snprintf (text + used, sizeof (text) - used, "%u:%s %llx-%llx%s", index, name, + le64 (entry + 32), le64 (entry + 40), (++column % 3) ? " " : "\n"); + listed++; + } + snprintf (text + used, sizeof (text) - used, "\nlisted=%u", listed); + panic ("%s", text); + } + // INQUIRY of LUN 0: vendor/product bytes 8..23. + uint8_t inquiry[6] = { 0x12, 0, 0, 0, 36, 0 }; + UfsResult inq = command (0, inquiry, 6, true, 36); + uint32_t inqWord[4] = { 0, 0, 0, 0 }; + for (unsigned index = 0; index < 4; index++) { + inqWord[index] = getBe32 (virt + dataOffset + 8 + index * 4); + } + // LBA 1 of LUN 0 holds the GPT header when the block size is 4096 (or 512). + uint32_t gptStatus = 0xee, sig0 = 0, sig1 = 0; + if (blockSize[0] >= 512 && blockSize[0] <= 4096) { + uint8_t read10[10] = { 0x28, 0, 0, 0, 0, 1, 0, 0, 1, 0 }; + UfsResult g = command (0, read10, 10, true, blockSize[0]); + gptStatus = (g.ocs << 16) | (g.response << 8) | g.scsi; + if (g.ocs == 0 && g.scsi == 0) { + sig0 = *reinterpret_cast (virt + dataOffset); + sig1 = *reinterpret_cast (virt + dataOffset + 4); + } + } + hcsAfter = read32 (hc, 0x30); + panic ("NATIVE_UFS adopt+read\n" + "hcs %08x->%08x pwr %08x->%08x ssu=%06x\n" + "L0 st=%06x last=%08x bs=%08x sns=%06x\n" + "L1 st=%06x last=%08x bs=%08x sns=%06x\n" + "L2 st=%06x last=%08x bs=%08x sns=%06x\n" + "L3 st=%06x last=%08x bs=%08x sns=%06x\n" + "L4 st=%06x last=%08x bs=%08x sns=%06x\n" + "L5 st=%06x last=%08x bs=%08x sns=%06x\n" + "inq st=%06x %08x %08x %08x %08x\n" + "inq sns=%06x slen=%x seg=%x\n" + "rsp0 %08x %08x %08x %08x\n" + "rsp8 %08x %08x %08x %08x\n" + "gpt st=%06x sig=%08x %08x", + hcsBefore, hcsAfter, powerBefore, powerAfter, wakeStatus, + status[0], last[0], blockSize[0], senseOf[0], status[1], last[1], blockSize[1], senseOf[1], + status[2], last[2], blockSize[2], senseOf[2], status[3], last[3], blockSize[3], senseOf[3], + status[4], last[4], blockSize[4], senseOf[4], status[5], last[5], blockSize[5], senseOf[5], + (inq.ocs << 16) | (inq.response << 8) | inq.scsi, inqWord[0], inqWord[1], inqWord[2], inqWord[3], + inq.sense, inq.senseLen, inq.seg, + getBe32 (virt + ucdOffset + rspOffset), getBe32 (virt + ucdOffset + rspOffset + 4), + getBe32 (virt + ucdOffset + rspOffset + 8), getBe32 (virt + ucdOffset + rspOffset + 12), + getBe32 (virt + ucdOffset + rspOffset + 32), getBe32 (virt + ucdOffset + rspOffset + 36), + getBe32 (virt + ucdOffset + rspOffset + 40), getBe32 (virt + ucdOffset + rspOffset + 44), + gptStatus, sig0, sig1); +} + +// Read-only dump of the request list the firmware left (native-ufs-probe=6): shows where +// the firmware placed its descriptors, to compare with an allocation the controller's DMA +// is known to reach. Nothing is written and no register is touched except reads. +static uint32_t dumpWord (const volatile uint32_t *base, unsigned index) +{ + return base[index]; +} + +void NativeUfs::dumpFirmwareList (void) +{ + OSSharedPtr hcMap, listMap, ucdMap; + uintptr_t hc = mapPhys (UFS_HC_BASE, UFS_HC_SIZE, hcMap); + if (!hc) { + panic ("NATIVE_UFS dump: controller map failed"); + } + uint64_t listPhys = (uint64_t (read32 (hc, 0x54)) << 32) | read32 (hc, 0x50); + uint32_t db = read32 (hc, 0x58), rsr = read32 (hc, 0x60), hcs = read32 (hc, 0x30); + IOMemoryDescriptor *listDesc = IOMemoryDescriptor::withPhysicalAddress (listPhys, 4096, kIODirectionIn); + if (!listDesc) { + panic ("NATIVE_UFS dump: list descriptor failed phys=%llx", listPhys); + } + listMap = OSSharedPtr (listDesc->map (kIOMapAnywhere), OSNoRetain); + listDesc->release (); + if (!listMap) { + panic ("NATIVE_UFS dump: list map failed phys=%llx", listPhys); + } + const volatile uint32_t *utrd = reinterpret_cast (listMap->getVirtualAddress ()); + uint64_t ucdPhys = (uint64_t (dumpWord (utrd, 5)) << 32) | dumpWord (utrd, 4); + uint32_t ucd0 = 0, ucd1 = 0, ucd2 = 0, ucd3 = 0; + if (ucdPhys) { + IOMemoryDescriptor *ucdDesc = IOMemoryDescriptor::withPhysicalAddress (ucdPhys, 4096, kIODirectionIn); + if (ucdDesc) { + ucdMap = OSSharedPtr (ucdDesc->map (kIOMapAnywhere), OSNoRetain); + ucdDesc->release (); + if (ucdMap) { + const volatile uint32_t *ucd = reinterpret_cast (ucdMap->getVirtualAddress ()); + ucd0 = dumpWord (ucd, 0); ucd1 = dumpWord (ucd, 1); ucd2 = dumpWord (ucd, 2); ucd3 = dumpWord (ucd, 3); + } + } + } + panic ("NATIVE_UFS dump: firmware list\n" + "list=%llx hcs=%08x db=%08x rsr=%08x\n" + "utrd0 %08x %08x %08x %08x\n" + "utrd0 %08x %08x %08x %08x\n" + "utrd1 %08x %08x %08x %08x\n" + "ucd=%llx %08x %08x %08x %08x", + listPhys, hcs, db, rsr, + dumpWord (utrd, 0), dumpWord (utrd, 1), dumpWord (utrd, 2), dumpWord (utrd, 3), + dumpWord (utrd, 4), dumpWord (utrd, 5), dumpWord (utrd, 6), dumpWord (utrd, 7), + dumpWord (utrd, 8), dumpWord (utrd, 9), dumpWord (utrd, 10), dumpWord (utrd, 11), + ucdPhys, ucd0, ucd1, ucd2, ucd3); +} + +// Read-only dump of the apps SMMU (apps_smmu@15000000) global register space and the stream +// match entry for the UFS stream ID 0x60 (native-ufs-probe=7). SMMUv2 layout: sCR0 0x0, +// IDR0 0x20, sGFSR 0x48, SMR(n) 0x800+4n, S2CR(n) 0xc00+4n. Nothing is written. +void NativeUfs::inspectSmmu (void) +{ + OSSharedPtr map; + uintptr_t smmu = mapPhys (0x15000000ULL, 0x1000, map); + if (!smmu) { + panic ("NATIVE_UFS smmu: map failed"); + } + uint32_t scr0 = read32 (smmu, 0x00), idr0 = read32 (smmu, 0x20), idr1 = read32 (smmu, 0x24); + uint32_t gfsr = read32 (smmu, 0x48); + unsigned count = idr0 & 0xff; + if (count > 128) { + count = 128; + } + unsigned match = 0xffff, valid = 0, typeCount[4] = { 0, 0, 0, 0 }; + uint32_t matchSmr = 0, matchS2cr = 0; + for (unsigned index = 0; index < count; index++) { + uint32_t smr = read32 (smmu, 0x800 + 4 * index); + uint32_t s2cr = read32 (smmu, 0xc00 + 4 * index); + if (smr & 0x80000000U) { + valid++; + typeCount[(s2cr >> 16) & 3]++; + uint32_t mask = (smr >> 16) & 0x7fff; + if (match == 0xffff && (((smr ^ 0x60U) & 0x7fffU) & ~mask) == 0) { + match = index; + matchSmr = smr; + matchS2cr = s2cr; + } + } + } + panic ("NATIVE_UFS smmu\n" + "scr0=%08x idr0=%08x idr1=%08x gfsr=%08x\n" + "nsmr=%u valid=%u types t0=%u t1=%u t2=%u\n" + "sid0x60 match=%x smr=%08x s2cr=%08x", + scr0, idr0, idr1, gfsr, count, valid, typeCount[0], typeCount[1], typeCount[2], + match, matchSmr, matchS2cr); +} + +// Read-only context-bank survey (native-ufs-probe=8): CBAR/CBA2R in gr1 and SCTLR/TTBR0 in the +// context banks, to find a bank the hypervisor leaves to us. Nothing is written. +void NativeUfs::inspectContextBanks (void) +{ + OSSharedPtr map; + uintptr_t smmu = mapPhys (0x15000000ULL, 0xd0000, map); + if (!smmu) { + panic ("NATIVE_UFS cb: map failed"); + } + uint32_t idr1 = read32 (smmu, 0x24); + unsigned numCb = idr1 & 0xff; + unsigned globalPages = 1U << (((idr1 >> 28) & 7) + 1); + if (globalPages > 128 || numCb > 128 || (globalPages + numCb) * 0x1000 > 0xd0000) { + panic ("NATIVE_UFS cb: unexpected geometry idr1=%08x", idr1); + } + unsigned cbarZero = 0, sctlrZero = 0, mSet = 0, bothZero = 0; + uint32_t firstZero = 0xffff; + for (unsigned index = 0; index < numCb; index++) { + uint32_t cbar = read32 (smmu, 0x1000 + 4 * index); + uint32_t sctlr = read32 (smmu + (globalPages + index) * 0x1000, 0x0); + cbarZero += cbar == 0; + sctlrZero += sctlr == 0; + mSet += (sctlr & 1) != 0; + if (cbar == 0 && sctlr == 0) { + bothZero++; + if (firstZero == 0xffff) { + firstZero = index; + } + } + } + uintptr_t cb0 = smmu + globalPages * 0x1000, cb1 = cb0 + 0x1000, cbLast = cb0 + (numCb - 1) * 0x1000; + panic ("NATIVE_UFS cb survey\n" + "ncb=%u pages=%u cbar0=%u sctlr0=%u msset=%u both0=%u first0=%x\n" + "cbar[0]=%08x cbar[1]=%08x cbar[%u]=%08x\n" + "sctlr[0]=%08x sctlr[1]=%08x sctlr[last]=%08x\n" + "cba2r[0]=%08x cba2r[1]=%08x", + numCb, globalPages, cbarZero, sctlrZero, mSet, bothZero, firstZero, + read32 (smmu, 0x1000), read32 (smmu, 0x1004), numCb - 1, read32 (smmu, 0x1000 + 4 * (numCb - 1)), + read32 (cb0, 0), read32 (cb1, 0), read32 (cbLast, 0), + read32 (smmu, 0x1800), read32 (smmu, 0x1804)); +} + +// ====================================================================================== +// Root media: the live controller as a read-only block device. +// +// One shared host (a single UTP request slot, polled; interrupts stay masked) serves every +// block device. The DMA region is ordinary cached, physically contiguous kernel memory with +// explicit cache maintenance around each command (the stream is mapped through the reserved +// apps_smmu bypass context bank by smmuBypassStream, so physical addresses are used as-is). +// Nothing here writes to the storage: only INQUIRY-class and READ(10) commands are built, plus +// the single START STOP UNIT that wakes the device from the power-down the firmware left. +// ====================================================================================== +namespace { +#include "RootTransportLayout.h" + +struct UfsHost { + OSSharedPtr hcMap, smmuMap; + uintptr_t hc; + IOBufferMemoryDescriptor *dma; + uint8_t *virt; + uint64_t phys; + IOLock *lock; + bool ready; + bool poisoned = false; + uintptr_t smmu = 0, cb = 0; +}; +UfsHost gHost = { OSSharedPtr (), OSSharedPtr (), 0, nullptr, nullptr, 0, nullptr, false }; + +static IOReturn recordTransport (IOReturn result, uint32_t ocs) +{ + gRootDiagnostic.result = static_cast (result); + gRootDiagnostic.ocs = ocs; + if (result != kIOReturnSuccess && gRootDiagnosticEnabled) { + // Read only the already mapped HC's status/registers used by this driver. + gRootDiagnostic.hcs = read32 (gHost.hc, 0x30); + gRootDiagnostic.hce = read32 (gHost.hc, 0x34); + gRootDiagnostic.doorbell = read32 (gHost.hc, 0x58); + gRootDiagnostic.interrupt = read32 (gHost.hc, 0x20); + gRootDiagnostic.list = uint64_t (read32 (gHost.hc, 0x50)) | + (uint64_t (read32 (gHost.hc, 0x54)) << 32); + gRootDiagnostic.runStop = read32 (gHost.hc, 0x60); + gRootDiagnostic.ahit = read32 (gHost.hc, 0x18); + for (unsigned i = 0; i < 5; ++i) gRootDiagnostic.uic[i] = read32 (gHost.hc, 0x38 + 4 * i); + if (gHost.smmu && gHost.cb) { + gRootDiagnostic.gfsr = read32 (gHost.smmu, 0x48); + gRootDiagnostic.gsyn = read32 (gHost.smmu, 0x54); + gRootDiagnostic.fsr = read32 (gHost.cb, 0x58); + gRootDiagnostic.fsyn = read32 (gHost.cb, 0x68); + gRootDiagnostic.far = uint64_t (read32 (gHost.cb, 0x60)) | + (uint64_t (read32 (gHost.cb, 0x64)) << 32); + } + } + return result; +} + +static void cacheCivac (const void *start, size_t length) +{ + uintptr_t address = reinterpret_cast (start) & ~uintptr_t (63); + uintptr_t end = reinterpret_cast (start) + length; + for (; address < end; address += 64) { + __asm__ volatile ("dc civac, %0" :: "r" (address) : "memory"); + } + __asm__ volatile ("dsb sy" ::: "memory"); +} + +// Device-owned cache lines must not be written back over DMA results. All +// callers use complete 64-byte lines from the dedicated page-aligned buffer. +static void cacheInvalidate (const void *start, size_t length) +{ + __asm__ volatile ("dsb sy" ::: "memory"); + uintptr_t address = reinterpret_cast (start); + for (uintptr_t end = address + length; address < end; address += 64) { + __asm__ volatile ("dc ivac, %0" :: "r" (address) : "memory"); + } + __asm__ volatile ("dsb sy" ::: "memory"); +} + +static void hostWrite (unsigned offset, uint32_t value) +{ + __asm__ volatile ("dsb sy" ::: "memory"); + write32 (gHost.hc, offset, value); + __asm__ volatile ("dsb sy" ::: "memory"); +} + +#include "RootQueue.h" + +#include "RootCommands.h" + +// Bring the live controller up for reading: map, SMMU pass-through for the UFS stream, take over +// the request list, wake the device from UFS-PowerDown and wait for LUN 0 to be ready. +// `why` receives a short reason on failure. +static bool hostBringUp (const char **why) +{ + if (gHost.poisoned) { + *why = "transport quarantined"; + return false; + } + if (gHost.ready) { + return true; + } + gRootDiagnostic.stage = "controller map"; + gHost.hc = mapPhys (UFS_HC_BASE, UFS_HC_SIZE, gHost.hcMap); + if (!gHost.hc) { + *why = "controller map"; + return false; + } + if (gRootDiagnosticEnabled) { + gRootDiagnostic.cap = read32 (gHost.hc, 0x00); + gRootDiagnostic.version = read32 (gHost.hc, 0x08); + gRootDiagnostic.hcs = read32 (gHost.hc, 0x30); + gRootDiagnostic.hce = read32 (gHost.hc, 0x34); + } + gRootDiagnostic.stage = "DMA allocation"; + gHost.dma = IOBufferMemoryDescriptor::inTaskWithOptions ( + kernel_task, static_cast (kIODirectionInOut) | static_cast (kIOMemoryPhysicallyContiguous) | + static_cast (kIOMemoryMapperNone), + kDmaBytes, 4096); + gHost.lock = IOLockAlloc (); + if (!gHost.dma || !gHost.lock) { + *why = "dma buffer"; + return false; + } + gHost.virt = static_cast (gHost.dma->getBytesNoCopy ()); + IOByteCount contiguous = 0; + gHost.phys = gHost.dma->getPhysicalSegment (0, &contiguous, kIOMemoryMapperNone); + gRootDiagnostic.dma = gHost.phys; + gRootDiagnostic.contiguous = contiguous; + if (!gHost.virt || !gHost.phys || (gHost.phys & 4095) || contiguous < kDmaBytes || + (!(read32 (gHost.hc, 0) & 0x01000000) && gHost.phys + kDmaBytes > 0x100000000ULL)) { + *why = "DMA address/contiguity"; + return false; + } + bzero (gHost.virt, kDmaBytes); + cacheCivac (gHost.virt, kDmaBytes); + uint32_t report[3] = { 0, 0, 0 }; + gRootDiagnostic.stage = "SMMU and request list"; + if (!hostStopQueue (why)) return false; + smmuBypassStream (0x60, gHost.smmuMap, report, gHost.phys, kDmaBytes); + gHost.smmu = gHost.smmuMap->getVirtualAddress (); + uint32_t idr1 = read32 (gHost.smmu, 0x24); + gHost.cb = gHost.smmu + ((1U << (((idr1 >> 28) & 7) + 1)) + (report[0] >> 16)) * 4096; + if (gRootDiagnosticEnabled) { + gRootDiagnostic.gfsrBefore = read32 (gHost.smmu, 0x48); + gRootDiagnostic.fsrBefore = read32 (gHost.cb, 0x58); + gRootDiagnostic.smmuId2 = read32 (gHost.smmu, 0x28); + gRootDiagnostic.cba2r = read32 (gHost.smmu, 0x1800 + 4 * (report[0] >> 16)); + } + if (!hostStartQueue (why)) return false; + gRootDiagnostic.stage = "NOP OUT"; + // NOP OUT proves the transport before touching the SCSI layer. + if (hostNop () != kIOReturnSuccess) { + gHost.ready = false; + *why = "NOP OUT"; + return false; + } + uint32_t power = 0; + gRootDiagnostic.stage = "power mode query"; + if (hostQueryRead (0x03, 0x02, &power) != kIOReturnSuccess) { + gHost.ready = false; + *why = "power mode query"; + return false; + } + gRootDiagnostic.power = power; + if ((power & 0xff) != 0x11) { + gRootDiagnostic.stage = "device wake"; + // START STOP UNIT, power condition ACTIVE, to the UFS Device well-known LU (as Linux resumes). + uint8_t ssu[6] = { 0x1b, 0, 0, 0, 0x10, 0 }; + uint32_t scsi = 0; + if (hostScsi (0xd0, ssu, 6, false, 0, 500000, &scsi) != kIOReturnSuccess) { + gHost.ready = false; + *why = "wake (START STOP UNIT)"; + return false; + } + IOSleep (50); + } + gRootDiagnostic.stage = "LUN ready"; + for (unsigned attempt = 0; attempt < 20; attempt++) { + uint8_t tur[6] = { 0, 0, 0, 0, 0, 0 }; + uint32_t scsi = 0; + if (hostScsi (0, tur, 6, false, 0, 200000, &scsi) == kIOReturnSuccess) { + gHost.ready = true; + return true; + } + if (gHost.poisoned) break; + IOSleep (10); + } + gHost.ready = false; + *why = "LUN 0 not ready"; + return false; +} +} + +namespace { +static uint32_t be32w (const uint8_t *at) +{ + return (uint32_t (at[0]) << 24) | (uint32_t (at[1]) << 16) | (uint32_t (at[2]) << 8) | at[3]; +} + +#include "RootLayout.h" +#include "RootWindowRead.h" +#include "RootPublication.h" + +} + +// ---- Persistent Data store window (sheng-data-persist) ---------------------------------------------------------- +// The only part of this driver that ever writes. The consumer (NativeRamBlock's overlay) asks for a window of the given +// size at the END of the userdata partition. Mode 1 (boot argument sheng-data-persist=1) only reads: it proves the window and +// lets the consumer inspect its header. Mode 2 additionally opens the write window. A window is refused unless it lies inside +// the partition, clear of the flashed root image plus a 4 MiB guard, and is at most a quarter of the partition. +namespace { +struct StoreGeometry { + bool valid; + uint64_t first, last, imageBytes, entriesLba; + uint32_t headerCrc, entriesCrc, entryCount; +}; +StoreGeometry gStoreGeometry; +uint64_t gStoreRegionFirst, gStoreRegionBlocks; + +static unsigned storeMode (void) +{ + static int cached = -1; + if (cached < 0) { + uint32_t value = 0; + if (!PE_parse_boot_argn ("sheng-data-persist", &value, sizeof (value)) || value > 2) value = 0; + cached = static_cast (value); + } + return static_cast (cached); +} +} + +extern "C" unsigned native_ufs_store_mode (void) { return storeMode (); } + +// Returns 1 and the first absolute 4096-byte LBA of the window, 0 on refusal (reason in *why, may be null). +extern "C" int native_ufs_store_open (uint64_t blocks, uint64_t *firstLba, const char **why) +{ + const char *reason = ""; + int ok = 0; + if (!storeMode ()) reason = "off"; + else if (!gStoreGeometry.valid || !gHost.ready) reason = "no root layout"; + else if (gStoreRegionBlocks) reason = "already open"; + else { + uint64_t partition = gStoreGeometry.last - gStoreGeometry.first + 1; + uint64_t imageBlocks = (gStoreGeometry.imageBytes + 4095) / 4096; + if (!blocks || blocks > partition / 4) reason = "size"; + else if (partition - blocks < imageBlocks + 1024) reason = "overlaps the root image"; + else { + uint64_t first = gStoreGeometry.last + 1 - blocks; + if (storeMode () == 2 && !hostStoreOpen (first, blocks)) reason = "window refused"; + else { + gStoreRegionFirst = first; + gStoreRegionBlocks = blocks; + if (firstLba) *firstLba = first; + ok = 1; + } + } + } + if (why) *why = reason; + return ok; +} + +extern "C" int native_ufs_store_read (uint64_t rel, uint32_t blocks, void *dst) +{ + if (!gStoreRegionBlocks || !dst || !blocks || rel >= gStoreRegionBlocks || blocks > gStoreRegionBlocks - rel) return 0; + while (blocks) { + uint32_t chunk = blocks > kDataMax / 4096 ? static_cast (kDataMax / 4096) : blocks; + if (hostRead4k (0, gStoreRegionFirst + rel, chunk, dst) != kIOReturnSuccess) return 0; + dst = static_cast (dst) + chunk * 4096ULL; + rel += chunk; + blocks -= chunk; + } + return 1; +} + +extern "C" int native_ufs_store_write (uint64_t rel, uint32_t blocks, const void *src) +{ + if (storeMode () != 2 || !gStoreRegionBlocks || !src || !blocks || rel >= gStoreRegionBlocks || + blocks > gStoreRegionBlocks - rel) return 0; + while (blocks) { + uint32_t chunk = blocks > kDataMax / 4096 ? static_cast (kDataMax / 4096) : blocks; + if (hostWrite4k (0, gStoreRegionFirst + rel, chunk, src) != kIOReturnSuccess) return 0; + src = static_cast (src) + chunk * 4096ULL; + rel += chunk; + blocks -= chunk; + } + return 1; +} + +extern "C" int native_ufs_store_sync (void) +{ + return storeMode () == 2 && gStoreRegionBlocks && hostSyncCache () == kIOReturnSuccess; +} + +// Re-read the GPT header and entry array and compare their CRCs with what was recorded at root start. 0 = unchanged. +extern "C" unsigned native_ufs_store_gpt_check (void) +{ + if (!gStoreGeometry.valid || !gHost.ready) return 1; + static uint8_t block[4096]; + if (hostRead4k (0, 1, 1, block) != kIOReturnSuccess) return 2; + uint32_t headerBytes = le32 (block + 12); + if (headerBytes < 92 || headerBytes > 4096 || rootCrc32 (block, headerBytes, true) != le32 (block + 16) || + le32 (block + 16) != gStoreGeometry.headerCrc) return 3; + uint32_t blocks = (gStoreGeometry.entryCount * 128 + 4095) / 4096; + static uint8_t entries[4096 * 4]; + if (blocks > 4 || hostRead4k (0, gStoreGeometry.entriesLba, blocks, entries) != kIOReturnSuccess) return 4; + return rootCrc32 (entries, gStoreGeometry.entryCount * 128) == gStoreGeometry.entriesCrc ? 0 : 5; +} + +// One read-only window of the root image as a block device: 512-byte sectors on top of 4096-byte +// UFS blocks, so the unchanged dmg disk images (GPT with 512-byte sectors) parse as they do on +// the virtio path. +class NativeUfsBlock : public IOBlockStorageDevice { + OSDeclareDefaultStructors (NativeUfsBlock) + uint64_t absolute = 0; // byte offset of the window on LUN 0 + uint64_t bytes = 0; + char label[32] = {}; + uint8_t *bounce = nullptr; + IOLock *windowLock = nullptr; + bool online = false; +#if defined(NATIVE_VERIFIED_BLOCK) && NATIVE_VERIFIED_BLOCK + struct vr_view verified = {}; + vm_offset_t verifiedStorage = 0; + vm_size_t verifiedStorageBytes = 0; + bool configureVerification (); +#endif +public: +#if defined(NATIVE_VERIFIED_BLOCK) && NATIVE_VERIFIED_BLOCK + bool authenticatesVolume (const char *uuid, uint8_t *manifestDigest = nullptr) const; + bool signedVolumeUuid (char out[37]) const; +#endif + const char *windowLabel () const { return label; } + bool configure (const char *name, uint64_t absoluteOffset, uint64_t size); + bool start (IOService *provider) override; + void free () override; + IOReturn doEjectMedia () override { return kIOReturnUnsupported; } + IOReturn doFormatMedia (UInt64) override { return kIOReturnNotWritable; } + UInt32 doGetFormatCapacities (UInt64 *, UInt32) const override { return 0; } + char *getVendorString () override { return const_cast ("Native"); } + char *getProductString () override { return label; } + char *getRevisionString () override { return const_cast ("1.0"); } + char *getAdditionalDeviceInfoString () override { return const_cast ("UFS LUN 0 userdata window, read-only"); } + IOReturn reportBlockSize (UInt64 *value) override { *value = 512; return kIOReturnSuccess; } + IOReturn reportEjectability (bool *value) override { *value = false; return kIOReturnSuccess; } + IOReturn reportMaxValidBlock (UInt64 *value) override { *value = bytes / 512 - 1; return kIOReturnSuccess; } + IOReturn reportMediaState (bool *present, bool *changed) override { + *present = online; if (changed) *changed = false; return kIOReturnSuccess; + } + IOReturn reportRemovability (bool *value) override { *value = false; return kIOReturnSuccess; } + IOReturn reportWriteProtection (bool *value) override { *value = true; return kIOReturnSuccess; } + IOReturn doSynchronizeCache () override { return online ? kIOReturnSuccess : kIOReturnNotReady; } + IOReturn doSynchronize (UInt64, UInt64, IOOptionBits) override { return doSynchronizeCache (); } + IOReturn getWriteCacheState (bool *value) override { *value = false; return kIOReturnSuccess; } + IOReturn setWriteCacheState (bool) override { return kIOReturnUnsupported; } + IOReturn doAsyncReadWrite (IOMemoryDescriptor *, UInt64, UInt64, IOStorageAttributes *, IOStorageCompletion *) override; +}; +OSDefineMetaClassAndStructors (NativeUfsBlock, IOBlockStorageDevice) + +extern "C" { +volatile uint32_t native_ufs_blocks_published = 0; +volatile uint64_t native_ufs_block_reads = 0; +volatile uint64_t native_ufs_block_failures = 0; +} + +static char gRootDiagnosticText[2500]; +static char gRootMediaText[1000]; +// Passive diagnostic reader: cached software values only. It must remain safe +// when called from an IRQ while the transport is busy: no locks, allocations, +// MMIO reads, queue resets, or changes to the command/trust state. Fields can +// represent adjacent updates; this is explicitly not an atomic HW snapshot. +extern "C" void native_ufs_progress_text (char *out, size_t size) +{ + UfsRootDiagnostics sample = gRootDiagnostic; + sample.published = native_ufs_blocks_published; + sample.reads = native_ufs_block_reads; + sample.failures = native_ufs_block_failures; + formatUfsRootDiagnostics (out, size, sample, "cached values; not a new HW fault sample", "UFS CACHED SNAPSHOT"); +} + +static void rootDiagnosticFailure (const char *reason) +{ + // Kept even when the panic diagnostic is off, so the boot page can say why nothing was published. + native_ufs_root_failure = reason; + if (!gRootDiagnosticEnabled) return; + gRootDiagnostic.published = native_ufs_blocks_published; + gRootDiagnostic.reads = native_ufs_block_reads; + gRootDiagnostic.failures = native_ufs_block_failures; + formatUfsRootDiagnostics (gRootDiagnosticText, sizeof (gRootDiagnosticText), gRootDiagnostic, reason); + strlcat (gRootDiagnosticText, gRootMediaText, sizeof (gRootDiagnosticText)); + panic ("%s", gRootDiagnosticText); +} + +bool NativeUfsBlock::configure (const char *name, uint64_t absoluteOffset, uint64_t size) +{ + if (!name || !ufsWindowGeometry (absoluteOffset, size)) { + return false; + } + strlcpy (label, name, sizeof (label)); + absolute = absoluteOffset; + bytes = size; + windowLock = IOLockAlloc (); + if (!windowLock) return false; + bounce = static_cast (IOMalloc (kDataMax)); + if (!bounce) { + return false; + } +#if defined(NATIVE_VERIFIED_BLOCK) && NATIVE_VERIFIED_BLOCK + // Reads are served only after each 4096-byte integrity block verifies against the owner-signed + // policy; without a matching policy the device is never published. + return configureVerification (); +#else + return true; +#endif +} + +#if defined(NATIVE_VERIFIED_BLOCK) && NATIVE_VERIFIED_BLOCK +volatile uint64_t native_ufs_verified_blocks = 0; +// Time spent hashing/verifying and reading from the UFS transport, in generic-timer ticks (diagnostic only). +volatile uint64_t native_ufs_verify_ticks = 0, native_ufs_read_ticks = 0; +static inline uint64_t timerTicks (void) {uint64_t v; __asm__ volatile ("isb\n\tmrs %0, cntpct_el0" : "=r" (v)); return v;} +volatile uint32_t native_ufs_integrity_failures = 0; + +bool NativeUfsBlock::configureVerification () +{ + gRootDiagnostic.stage = "policy selection"; + gRootDiagnostic.windowBytes = bytes; + gRootDiagnostic.policyCount = verified_root_policy_count; + gRootDiagnostic.verify = 0xffffffff; + // Select the one policy whose signed image size equals this window; two matches are refused. + const uint8_t *manifest = nullptr; + uint64_t manifestBytes = 0; + if (!verified_root_policy_count || verified_root_policy_count > 16) { + gRootDiagnostic.stage = "policy count invalid"; + return false; + } + for (uint64_t index = 0; index < verified_root_policy_count; ++index) { + const NativeVerifiedBlockPolicy &policy = verified_root_policies[index]; + if (!policy.manifest || policy.manifestBytes < VR_HEADER_BYTES + VR_SIGNATURE_BYTES) { + gRootDiagnostic.stage = "policy envelope invalid"; + return false; + } + uint64_t imageBytes = 0; + for (unsigned i = 0; i < 8; ++i) { + imageBytes |= static_cast (policy.manifest[24 + i]) << (8 * i); + } + if (imageBytes != bytes) { + continue; + } + if (manifest) { + gRootDiagnostic.stage = "duplicate capacity-matching policy"; + return false; + } + manifest = policy.manifest; + manifestBytes = policy.manifestBytes; + } + if (!manifest) { + gRootDiagnostic.stage = "no capacity-matching policy"; + return false; + } + gRootDiagnostic.stage = "policy storage"; + vm_size_t tableBytes = round_page (static_cast (manifestBytes)); + if (tableBytes < manifestBytes || tableBytes > UINT64_MAX - 4096) { + gRootDiagnostic.stage = "policy size overflow"; + return false; + } + verifiedStorageBytes = tableBytes + 4096; + gRootDiagnostic.stage = "policy allocation"; + if (kmem_alloc (kernel_map, &verifiedStorage, verifiedStorageBytes, + static_cast (KMA_KOBJECT | KMA_DATA), VM_KERN_MEMORY_SECURITY) != KERN_SUCCESS) { + return false; + } + memcpy (reinterpret_cast (verifiedStorage), manifest, manifestBytes); + uint8_t *key = reinterpret_cast (verifiedStorage + tableBytes); + memcpy (key, verified_root_key, VR_KEY_BYTES); + gRootDiagnostic.stage = "policy read-only protection"; + if (vm_map_protect (kernel_map, verifiedStorage, verifiedStorage + verifiedStorageBytes, + VM_PROT_READ, TRUE) != KERN_SUCCESS) { + return false; + } + gRootDiagnostic.stage = "owner signature and hash table"; + int status = vr_open (reinterpret_cast (verifiedStorage), manifestBytes, key, 1, + &vr_xnu_crypto, &verified); + gRootDiagnostic.verify = static_cast (status); + return status == VR_OK && verified.image_bytes == bytes; +} + +bool NativeUfsBlock::authenticatesVolume (const char *uuid, uint8_t *manifestDigest) const +{ + if (!uuid || !online || !verifiedStorage || !verified.hashes) { + return false; + } + static const char hex[] = "0123456789ABCDEF"; + char expected[37]; + unsigned at = 0; + for (unsigned i = 0; i < 16; ++i) { + if (i == 4 || i == 6 || i == 8 || i == 10) { + expected[at++] = '-'; + } + expected[at++] = hex[verified.volume_uuid[i] >> 4]; + expected[at++] = hex[verified.volume_uuid[i] & 15]; + } + expected[at] = 0; + if (strcmp (uuid, expected)) { + return false; + } + if (manifestDigest) { + // Hash only the authenticated envelope, excluding allocation padding and the key. + const uint64_t envelope = VR_HEADER_BYTES + VR_SIGNATURE_BYTES + verified.block_count * 32; + SHA384_CTX context; + SHA384_Init (&context); + SHA384_Update (&context, reinterpret_cast (verifiedStorage), envelope); + SHA384_Final (manifestDigest, &context); + } + return true; +} + +bool NativeUfsBlock::signedVolumeUuid (char out[37]) const +{ + if (!verifiedStorage || !verified.hashes) { + return false; + } + static const char hex[] = "0123456789ABCDEF"; + unsigned at = 0; + for (unsigned i = 0; i < 16; ++i) { + if (i == 4 || i == 6 || i == 8 || i == 10) { + out[at++] = '-'; + } + out[at++] = hex[verified.volume_uuid[i] >> 4]; + out[at++] = hex[verified.volume_uuid[i] & 15]; + } + out[at] = 0; + return true; +} + +// Called from the root authorization query in NativeVirtioBlock: -1 when `provider` is not a +// UFS window, otherwise 1 if its signed volume UUID equals `uuid`, else 0. +extern "C" int native_ufs_block_authenticates (IOService *provider, const char *uuid, uint8_t *digest) +{ + NativeUfsBlock *block = OSDynamicCast (NativeUfsBlock, provider); + if (!block) { + return -1; + } + return block->authenticatesVolume (uuid, digest) ? 1 : 0; +} +#endif + +bool NativeUfsBlock::start (IOService *provider) +{ + if (!bounce || !windowLock || !IOBlockStorageDevice::start (provider)) { + return false; + } + online = true; + registerService (); + return true; +} + +IOReturn NativeUfsBlock::doAsyncReadWrite (IOMemoryDescriptor *buffer, UInt64 block, UInt64 count, + IOStorageAttributes *, IOStorageCompletion *completion) +{ + if (!buffer || !completion || !completion->action || !count || block >= bytes / 512 || + count > bytes / 512 - block || buffer->getLength () < count * 512) { + return kIOReturnBadArgument; + } + if (buffer->getDirection () != kIODirectionIn) { + return kIOReturnNotWritable; + } + if (!online) { + return kIOReturnNotReady; + } + struct Operations { + IOLock *mutex; + IOMemoryDescriptor *buffer; +#if defined(NATIVE_VERIFIED_BLOCK) && NATIVE_VERIFIED_BLOCK + const vr_view *view; +#endif + void lock () { IOLockLock (mutex); } + void unlock () { IOLockUnlock (mutex); } + IOReturn read (uint64_t lba, uint32_t blocks, void *out) { + uint64_t t0 = timerTicks (); + IOReturn r = hostRead4k (0, lba, blocks, out); + __atomic_fetch_add (&native_ufs_read_ticks, timerTicks () - t0, __ATOMIC_RELAXED); + return r; + } + IOReturn verify (uint64_t index, const void *data, size_t length) { +#if defined(NATIVE_VERIFIED_BLOCK) && NATIVE_VERIFIED_BLOCK + uint64_t t0 = timerTicks (); + int status = vr_verify_block (view, index, data, length, &vr_xnu_crypto); + __atomic_fetch_add (&native_ufs_verify_ticks, timerTicks () - t0, __ATOMIC_RELAXED); + if (status != VR_OK) { + gRootDiagnostic.stage = "read block integrity"; + gRootDiagnostic.verify = static_cast (status); + __atomic_fetch_add (&native_ufs_integrity_failures, 1, __ATOMIC_RELAXED); + return kIOReturnNotPrivileged; + } + __atomic_fetch_add (&native_ufs_verified_blocks, 1, __ATOMIC_RELAXED); + if (boot_progress_poll) boot_progress_poll (); +#else + (void) index; (void) data; (void) length; +#endif + return kIOReturnSuccess; + } + uint64_t copy (uint64_t offset, const void *data, uint64_t length) { + uint64_t copied = buffer->writeBytes (offset, data, length); + if (copied == length) __atomic_fetch_add (&native_ufs_block_reads, 1, __ATOMIC_RELAXED); + return copied; + } + } operations = { windowLock, buffer +#if defined(NATIVE_VERIFIED_BLOCK) && NATIVE_VERIFIED_BLOCK + , &verified +#endif + }; + UfsWindowResult outcome = ufsReadWindow (operations, absolute, bytes, block, count, bounce); + IOReturn result = outcome.status; + uint64_t done = outcome.done; + if (result != kIOReturnSuccess) { + __atomic_fetch_add (&native_ufs_block_failures, 1, __ATOMIC_RELAXED); + gRootDiagnostic.result = static_cast (result); + rootDiagnosticFailure ("root window read failed"); + } + completion->action (completion->target, completion->parameter, result, done); + return kIOReturnSuccess; +} + +void NativeUfsBlock::free () +{ +#if defined(NATIVE_VERIFIED_BLOCK) && NATIVE_VERIFIED_BLOCK + if (verifiedStorage) { + kmem_free (kernel_map, verifiedStorage, verifiedStorageBytes); + verifiedStorage = 0; + } +#endif + if (bounce) { + IOFree (bounce, kDataMax); + bounce = nullptr; + } + if (windowLock) { + IOLockFree (windowLock); + windowLock = nullptr; + } + IOBlockStorageDevice::free (); +} + +// Ordered publication of the root windows. The original owner-trust code resolves the SystemOS +// volume by UUID immediately after the root is mounted, so SystemOS must already have published its +// volume media by then; the OS window (whose volume is the root being waited for) goes last. +static NativeUfsBlock *gPendingDevice[4]; +static unsigned gPending; +static bool gPendingAttached[4]; +static char gPendingUuid[4][37]; +static UfsRootPlan gPublicationPlan; +static NativeUfs *gPendingProvider; + +static bool mediaWithUuidExists (const char *uuid, NativeUfsBlock *window) +{ + OSDictionary *matching = IOService::serviceMatching ("IOMedia"); + OSDictionary *properties = OSDictionary::withCapacity (1); + OSString *wanted = OSString::withCString (uuid); + bool found = false; + if (matching && properties && wanted && properties->setObject ("UUID", wanted) && + matching->setObject ("IOPropertyMatch", properties)) { + OSIterator *iterator = IOService::getMatchingServices (matching); + if (iterator) { + for (unsigned matches = 0; matches < 64 && !found; ++matches) { + IORegistryEntry *media = OSDynamicCast (IORegistryEntry, iterator->getNextObject ()); + if (!media) break; + IORegistryIterator *parents = IORegistryIterator::iterateOver ( + media, gIOServicePlane, kIORegistryIterateRecursively | kIORegistryIterateParents); + if (parents) { + found = ufsMediaHasAncestor (*parents, window); + parents->release (); + } + } + iterator->release (); + } + } + OSSafeReleaseNULL (wanted); + OSSafeReleaseNULL (properties); + OSSafeReleaseNULL (matching); + return found; +} + +static void captureMissingMedia (const char *uuid) +{ + int used = snprintf (gRootMediaText, sizeof (gRootMediaText), "Waiting for signed UUID %s\n", uuid); + OSDictionary *matching = IOService::serviceMatching ("IOMedia"); + OSIterator *iterator = matching ? IOService::getMatchingServices (matching) : nullptr; + unsigned count = 0; + while (iterator && used < int (sizeof (gRootMediaText)) - 180 && count++ < 8) { + IOService *media = OSDynamicCast (IOService, iterator->getNextObject ()); + if (!media) break; + OSString *id = OSDynamicCast (OSString, media->getProperty ("UUID")); + OSString *bsd = OSDynamicCast (OSString, media->getProperty (kIOBSDNameKey)); + OSNumber *size = OSDynamicCast (OSNumber, media->getProperty ("Size")); + used += snprintf (gRootMediaText + used, sizeof (gRootMediaText) - used, + "media %.24s bytes=%llx uuid=%.36s\n", + bsd ? bsd->getCStringNoCopy () : "-", size ? size->unsigned64BitValue () : 0ULL, + id ? id->getCStringNoCopy () : "-"); + } + OSSafeReleaseNULL (iterator); + OSSafeReleaseNULL (matching); +} + +// Used only for objects that were never started/published. Started objects +// retain their owner references even on a later failure; clients can have I/O. +static void discardPendingWindow (unsigned index) +{ + NativeUfsBlock *device = gPendingDevice[index]; + if (!device) return; + if (gPendingAttached[index]) device->detach (gPendingProvider); + gPendingAttached[index] = false; + gPendingDevice[index] = nullptr; + device->release (); +} + +static void publishThread (void *, wait_result_t) +{ + struct Operations { + bool start (unsigned index) { + NativeUfsBlock *device = gPendingDevice[index]; + gRootDiagnostic.stage = "root window service start"; + gRootDiagnostic.publicationPhase = static_cast (UfsRootPhase::Start); + gRootDiagnostic.publicationWindow = index; + if (!device->start (gPendingProvider)) return false; + native_ufs_blocks_published = native_ufs_blocks_published + 1; + IOLog ("NativeUfs: published %s\n", device->windowLabel ()); + return true; + } + bool wait (unsigned index) { + NativeUfsBlock *device = gPendingDevice[index]; + const char *uuid = gPendingUuid[index]; + gRootDiagnostic.stage = "media publication"; + gRootDiagnostic.publicationPhase = static_cast (UfsRootPhase::Media); + gRootDiagnostic.publicationWindow = index; + bool ready = false; + // Diagnostic mode affects reporting only, never the dependency gate. + for (unsigned tick = 0; tick <= 300; ++tick) { + ready = mediaWithUuidExists (uuid, device); + if (ready || tick == 300) break; + IOSleep (100); + } + IOLog ("NativeUfs: %s volume %s %s\n", device->windowLabel (), uuid, ready ? "ready" : "not ready"); + if (!ready) captureMissingMedia (uuid); + return ready; + } + void discard (unsigned index) { discardPendingWindow (index); } + } operations; + UfsRootPublicationResult result = ufsPublishRootWindows (operations, gPublicationPlan); + if (result.phase != UfsRootPhase::Complete) { + gRootDiagnostic.result = result.phase == UfsRootPhase::Start ? kIOReturnIOError : kIOReturnTimeout; + const char *reason = result.phase == UfsRootPhase::Start ? "root window service start failed" : + (!strcmp (gPendingDevice[result.index]->windowLabel (), "SystemOS") ? + "SystemOS signed volume not found" : "signed root window volume not found"); + IOLog ("NativeUfs: root publisher stopped at window %u; published=%u\n", result.index, result.published); + // Start failures have already released their unpublished object; do not + // dereference it here. The media-failure object is deliberately retained. + rootDiagnosticFailure (reason); + } + gPendingProvider->release (); // Balanced with successful thread dispatch. +} + +// Memory snapshot (native-ufs-probe=13): root mode as usual, then, after a delay, the kernel's own +// page counters on the panic screen. Observation only. +extern "C" { +extern unsigned int vm_page_free_count, vm_page_active_count, vm_page_inactive_count, + vm_page_wire_count, vm_page_speculative_count, vm_page_purgeable_count; +extern uint32_t vm_page_pages; +extern unsigned int memorystatus_available_pages; +extern int nprocs; +} + +static void statsThread (void *argument, wait_result_t) +{ + unsigned delay = static_cast (reinterpret_cast (argument)); + IOSleep (delay * 1000); + panic ("NATIVE_UFS memory snapshot after %u s\n" + "pages total=%u free=%u active=%u inactive=%u wired=%u spec=%u purgeable=%u\n" + "available=%u max_mem=%llx procs=%d blocks=%llu\n", + delay, vm_page_pages, vm_page_free_count, vm_page_active_count, vm_page_inactive_count, + vm_page_wire_count, vm_page_speculative_count, vm_page_purgeable_count, + memorystatus_available_pages, max_mem, nprocs, native_ufs_block_reads); +} + +// Storage discovery report (native-ufs-probe=12): a separate thread waits for the original +// IOStorageFamily to publish IOMedia for the root windows, then shows what exists on the screen. +// It must not run inside the matching thread that is starting NativeUfs. +static char gMediaReport[3000]; + +static void mediaReportThread (void *, wait_result_t) +{ + unsigned count = 0; + for (unsigned tick = 0; tick < 200; tick++) { // up to 20 s + OSDictionary *matching = IOService::serviceMatching ("IOMedia"); + OSIterator *iterator = matching ? IOService::getMatchingServices (matching) : nullptr; + count = 0; + if (iterator) { + while (iterator->getNextObject ()) { + count++; + } + iterator->release (); + } + if (matching) { + matching->release (); + } + if (count >= 4) { + break; + } + IOSleep (100); + } + IOSleep (500); + int used = snprintf (gMediaReport, sizeof (gMediaReport), + "NATIVE_UFS media report: published=%u reads=%llu failures=%llu media=%u\n", + native_ufs_blocks_published, native_ufs_block_reads, native_ufs_block_failures, count); + OSDictionary *matching = IOService::serviceMatching ("IOMedia"); + OSIterator *iterator = matching ? IOService::getMatchingServices (matching) : nullptr; + while (iterator && used < int (sizeof (gMediaReport)) - 200) { + IOService *media = OSDynamicCast (IOService, iterator->getNextObject ()); + if (!media) { + break; + } + OSString *uuid = OSDynamicCast (OSString, media->getProperty ("UUID")); + OSString *bsd = OSDynamicCast (OSString, media->getProperty (kIOBSDNameKey)); + OSString *content = OSDynamicCast (OSString, media->getProperty ("Content")); + OSNumber *size = OSDynamicCast (OSNumber, media->getProperty ("Size")); + used += snprintf (gMediaReport + used, sizeof (gMediaReport) - used, "%s %s size=%llx uuid=%s\n", + bsd ? bsd->getCStringNoCopy () : "-", content ? content->getCStringNoCopy () : "-", + size ? size->unsigned64BitValue () : 0ULL, uuid ? uuid->getCStringNoCopy () : "-"); + } + if (iterator) { + iterator->release (); + } + if (matching) { + matching->release (); + } + panic ("%s", gMediaReport); +} + +bool NativeUfs::start (IOService *provider) +{ + unsigned probe = 0, rootMode = 0; + PE_parse_boot_argn ("native-ufs-probe", &probe, sizeof (probe)); + PE_parse_boot_argn ("native-ufs-root", &rootMode, sizeof (rootMode)); + native_ufs_probe_requested = probe != 0; + // Default: match for observability only, never touch the controller. Hardware work runs once, + // from the root-device personality; the ufshc nub match stays observation-only. + if ((!probe && !rootMode) || !OSDynamicCast (IOPlatformExpertDevice, provider)) { + return false; + } + unsigned diagnostic = 0; + PE_parse_boot_argn ("native-ufs-diagnostics", &diagnostic, sizeof (diagnostic)); + gRootDiagnosticEnabled = diagnostic == 1; + if (probe == 2) { + inspect (); + } + if (probe == 8) { + inspectContextBanks (); + } + if (probe == 7) { + inspectSmmu (); + } + if (probe == 6) { + dumpFirmwareList (); + } + if ((probe >= 3 && probe <= 5) || probe == 9) { + adoptAndRead (probe); + } + if (probe == 10) { + // Read-only check before anything is flashed: LUN 0 GPT -> "userdata" -> its first block. + const char *why = ""; + if (!hostBringUp (&why)) { + panic ("NATIVE_UFS root check: bring-up failed: %s", why); + } + RootLayout *layout = static_cast (IOMalloc (sizeof (RootLayout))); + if (!layout || !findRootLayout (layout, &why)) { + panic ("NATIVE_UFS root check: %s", why); + } + panic ("NATIVE_UFS root check\n" + "userdata first=%llx last=%llx\n" + "block0 magic %08x %08x ext4magic=%04x ext4blocks=%u\n" + "manifest status=%u windows=%u", + layout->userdataFirst, layout->userdataLast, layout->magic0, layout->magic1, + layout->ext4Magic, layout->ext4Blocks, layout->manifestStatus, layout->windows); + } + if (probe == 11) { + // After the root image is flashed: manifest, then each window's own GPT (sector 1 of the + // attached dmg disk image carries "EFI PART"). Read-only. + const char *why = ""; + if (!hostBringUp (&why)) { + panic ("NATIVE_UFS root windows: bring-up failed: %s", why); + } + RootLayout *layout = static_cast (IOMalloc (sizeof (RootLayout))); + if (!layout || !findRootLayout (layout, &why)) { + panic ("NATIVE_UFS root windows: %s", why); + } + uint32_t sig[4][2] = {}; + uint32_t seen[4] = {}; + for (uint32_t index = 0; index < layout->windows && index < 4; index++) { + uint64_t absolute = layout->userdataFirst * 4096ULL + layout->window[index].offset; + if (hostRead4k (0, absolute / 4096, 1, gScratch) == kIOReturnSuccess) { + seen[index] = 1; + sig[index][0] = le32 (gScratch + 512); + sig[index][1] = le32 (gScratch + 516); + } + } + panic ("NATIVE_UFS root windows\n" + "userdata first=%llx last=%llx manifest status=%u windows=%u\n" + "w0 %s off=%llx size=%llx gpt=%08x %08x read=%u\n" + "w1 %s off=%llx size=%llx gpt=%08x %08x read=%u\n" + "w0 sha %08x %08x %08x %08x\n" + "w1 sha %08x %08x %08x %08x", + layout->userdataFirst, layout->userdataLast, layout->manifestStatus, layout->windows, + layout->window[0].name, layout->window[0].offset, layout->window[0].size, sig[0][0], sig[0][1], seen[0], + layout->window[1].name, layout->window[1].offset, layout->window[1].size, sig[1][0], sig[1][1], seen[1], + be32w (layout->window[0].sha256), be32w (layout->window[0].sha256 + 4), be32w (layout->window[0].sha256 + 8), be32w (layout->window[0].sha256 + 12), + be32w (layout->window[1].sha256), be32w (layout->window[1].sha256 + 4), be32w (layout->window[1].sha256 + 8), be32w (layout->window[1].sha256 + 12)); + } + if (probe == 14) { + // Immediate memory snapshot at root-device start, before the userspace that triggers jetsam. + panic ("NATIVE_UFS early memory snapshot\n" + "pages total=%u free=%u active=%u inactive=%u wired=%u spec=%u purgeable=%u\n" + "available=%u max_mem=%llx procs=%d", + vm_page_pages, vm_page_free_count, vm_page_active_count, vm_page_inactive_count, + vm_page_wire_count, vm_page_speculative_count, vm_page_purgeable_count, + memorystatus_available_pages, max_mem, nprocs); + } + if (probe == 13) { + unsigned delay = 12; + PE_parse_boot_argn ("native-ufs-stats-delay", &delay, sizeof (delay)); + if (!startRoot (provider)) { + panic ("NATIVE_UFS memory snapshot: root start failed"); + } + thread_t thread = nullptr; + if (kernel_thread_start (statsThread, reinterpret_cast (static_cast (delay)), &thread) != KERN_SUCCESS) { + panic ("NATIVE_UFS memory snapshot: cannot start the thread"); + } + thread_deallocate (thread); + return true; + } + if (probe == 12) { + bool published = startRoot (provider); + if (!published) { + panic ("NATIVE_UFS media report: root start failed (blocks=%u)", native_ufs_blocks_published); + } + thread_t thread = nullptr; + if (kernel_thread_start (mediaReportThread, nullptr, &thread) != KERN_SUCCESS) { + panic ("NATIVE_UFS media report: cannot start the report thread"); + } + thread_deallocate (thread); + return true; + } + if (rootMode && !probe) { + return startRoot (provider); + } + return false; +} + +// Boot the root image's windows as block devices. Soft-fails (logs, publishes nothing). +bool NativeUfs::startRoot (IOService *provider) +{ + const char *why = ""; + native_ufs_root_step = 1; + if (!hostBringUp (&why)) { + rootDiagnosticFailure (why); + IOLog ("NativeUfs: root bring-up failed: %s\n", why); + return false; + } + native_ufs_root_step = 2; + RootLayout *layout = static_cast (IOMalloc (sizeof (RootLayout))); + if (!layout || !findRootLayout (layout, &why)) { + rootDiagnosticFailure (layout ? why : "root layout allocation failed"); + IOLog ("NativeUfs: root layout failed: %s\n", why); + if (layout) IOFree (layout, sizeof (RootLayout)); + return false; + } + native_ufs_root_step = 3; + if (layout->manifestStatus != 0) { + rootDiagnosticFailure ("root manifest rejected"); + IOLog ("NativeUfs: no valid root manifest in userdata (status %u, magic %08x %08x)\n", + layout->manifestStatus, layout->magic0, layout->magic1); + IOFree (layout, sizeof (RootLayout)); + return false; + } + gStoreGeometry = { true, layout->userdataFirst, layout->userdataLast, layout->imageBytes, layout->gptEntriesLba, + layout->gptHeaderCrc, layout->gptEntriesCrc, layout->gptEntryCount }; + const char *names[4] = {}; + for (unsigned i = 0; i < layout->windows; ++i) names[i] = layout->window[i].name; + if (!ufsBuildRootPlan (names, layout->windows, gPublicationPlan)) { + rootDiagnosticFailure ("root windows require distinct OS and SystemOS"); + IOFree (layout, sizeof (RootLayout)); + return false; + } + if (!IOService::start (provider)) { + rootDiagnosticFailure ("root IOService start failed"); + IOFree (layout, sizeof (RootLayout)); + return false; + } + // Prepare all windows or publish none. Diagnostics never decide policy. + gPending = 0; + gPendingProvider = this; + struct Operations { + NativeUfs *owner; + RootLayout *layout; + bool prepare (unsigned index) { + gRootDiagnostic.stage = "root window allocation/configuration"; + gRootDiagnostic.publicationPhase = static_cast (UfsRootPhase::Prepare); + gRootDiagnostic.publicationWindow = index; + gRootDiagnostic.windowBytes = layout->window[index].size; + NativeUfsBlock *device = OSTypeAlloc (NativeUfsBlock); + gPendingDevice[index] = device; + gPendingAttached[index] = false; + gPendingUuid[index][0] = 0; + uint64_t absolute = layout->userdataFirst * 4096ULL + layout->window[index].offset; + if (!device || !device->init (static_cast (nullptr)) || + !device->configure (layout->window[index].name, absolute, layout->window[index].size)) return false; + gRootDiagnostic.stage = "root window attachment"; + if (!device->attach (owner)) return false; + gPendingAttached[index] = true; + ++gPending; + gRootDiagnostic.prepared = gPending; + return true; + } + bool validate () { + gRootDiagnostic.stage = "signed window identities"; + gRootDiagnostic.publicationPhase = static_cast (UfsRootPhase::Identity); +#if defined(NATIVE_VERIFIED_BLOCK) && NATIVE_VERIFIED_BLOCK + for (unsigned i = 0; i < gPending; ++i) { + gRootDiagnostic.publicationWindow = i; + if (!gPendingDevice[i]->signedVolumeUuid (gPendingUuid[i])) return false; + for (unsigned j = 0; j < i; ++j) if (!strcmp (gPendingUuid[i], gPendingUuid[j])) return false; + } + return true; +#else + return false; // Root publication requires owner verification in every build. +#endif + } + bool schedule () { + gRootDiagnostic.stage = "root publisher thread"; + gRootDiagnostic.publicationPhase = static_cast (UfsRootPhase::Schedule); + gRootDiagnostic.publicationWindow = 0xffffffff; + thread_t thread = nullptr; + owner->retain (); + if (kernel_thread_start (publishThread, nullptr, &thread) != KERN_SUCCESS) { + owner->release (); + return false; + } + thread_deallocate (thread); + return true; + } + void discard (unsigned index) { discardPendingWindow (index); } + } operations = { this, layout }; + UfsRootPublicationResult result = ufsPrepareRootWindows (operations, layout->windows); + IOFree (layout, sizeof (RootLayout)); + if (result.phase != UfsRootPhase::Complete) { + gPending = 0; + gRootDiagnostic.result = result.phase == UfsRootPhase::Identity ? kIOReturnNotPrivileged : + result.phase == UfsRootPhase::Schedule ? kIOReturnNoResources : kIOReturnError; + IOLog ("NativeUfs: root preparation failed phase=%u window=%u; nothing published\n", + static_cast (result.phase), result.index); + IOService::stop (provider); + rootDiagnosticFailure (result.phase == UfsRootPhase::Schedule ? "root publisher thread creation failed" : + result.phase == UfsRootPhase::Identity ? "signed window identities rejected" : + "root window preparation failed"); + return false; + } + return true; +} + +void NativeUfs::stop (IOService *provider) +{ + fCmdDesc.reset (); + fMap.reset (); + IOService::stop (provider); +} diff --git a/drivers/NativeUfs/RootCommands.h b/drivers/NativeUfs/RootCommands.h new file mode 100644 index 0000000..ade4d5c --- /dev/null +++ b/drivers/NativeUfs/RootCommands.h @@ -0,0 +1,304 @@ +// SPDX-License-Identifier: BSD-2-Clause-Patent +// Shared production command path. Included inside the root transport namespace. +// The host harness replaces only MMIO, DMA synchronization, clock and lock primitives. +struct HardwareTransferPort { + uint64_t deadline; + uint32_t doorbell () { return read32 (gHost.hc, 0x58); } + uint32_t interrupt () { return read32 (gHost.hc, 0x20); } + uint32_t completedOcs () { + cacheInvalidate (gHost.virt, 64); + return *reinterpret_cast (gHost.virt + 8) & 0xff; + } + bool expired () { return mach_absolute_time () >= deadline; } + void delay () { IODelay (2); } +}; + +static void putBe32 (uint8_t *at, uint32_t value) +{ + at[0] = static_cast (value >> 24); + at[1] = static_cast (value >> 16); + at[2] = static_cast (value >> 8); + at[3] = static_cast (value); +} + +static uint32_t loadBe32 (const uint8_t *at) +{ + return (uint32_t (at[0]) << 24) | (uint32_t (at[1]) << 16) | (uint32_t (at[2]) << 8) | at[3]; +} + +// Issue the UPIU already placed at gHost.virt + kUcdOffset through request slot 0. `direction` is +// the UTRD data direction (0 none, 1 from host, 2 to host). Direction 1 is reachable only through hostWrite4k, which +// confines it to the persistent store window. Returns the overall command status +// (0 = completed) in *ocs. Timeout/fatal status quarantines the allocation. +static IOReturn hostSubmit (uint32_t direction, uint32_t dataBytes, uint32_t timeoutUs, uint32_t *ocs) +{ + *ocs = 15; + if (!timeoutUs || dataBytes > kDataMax || (dataBytes & 3) || + (dataBytes ? (direction != 1 && direction != 2) : direction != 0)) return kIOReturnBadArgument; + if (gHost.poisoned) return kIOReturnNotReady; + if (read32 (gHost.hc, 0x58) != 0) { + gHost.poisoned = true; + gHost.ready = false; + return recordTransport (kIOReturnBusy, *ocs); + } + if (read32 (gHost.hc, 0x20) & kUfsFatalStatus) { + gHost.poisoned = true; + gHost.ready = false; + return recordTransport (kIOReturnIOError, *ocs); + } + gRootDiagnostic.command++; + gRootDiagnostic.type = gHost.virt[kUcdOffset]; + gRootDiagnostic.lun = gHost.virt[kUcdOffset + 2]; + gRootDiagnostic.opcode = gHost.virt[kUcdOffset + (gRootDiagnostic.type == 1 ? 16 : 12)]; + gRootDiagnostic.scsi = 0xffff; + gRootDiagnostic.response = 0; + gRootDiagnostic.protocol = 0; + gRootDiagnostic.residual = 0; + gRootDiagnostic.sense = 0; + uint8_t *utrd = gHost.virt; + uint64_t ucdPhys = gHost.phys + kUcdOffset; + uint32_t *words = reinterpret_cast (utrd); + for (unsigned index = 0; index < 8; index++) { + words[index] = 0; + } + words[0] = 0x10000000U | (direction << 25); + words[2] = 0x0fU; + words[4] = static_cast (ucdPhys); + words[5] = static_cast (ucdPhys >> 32); + words[6] = (512U >> 2) | ((kRspOffset >> 2) << 16); + words[7] = (dataBytes ? 1U : 0U) | ((kPrdtOffset >> 2) << 16); + if (dataBytes) { + uint64_t dataPhys = gHost.phys + kDataOffset; + uint32_t *prd = reinterpret_cast (gHost.virt + kUcdOffset + kPrdtOffset); + prd[0] = static_cast (dataPhys); + prd[1] = static_cast (dataPhys >> 32); + prd[2] = 0; + prd[3] = dataBytes - 1; + } + cacheCivac (gHost.virt, kDataOffset + dataBytes); + hostWrite (0x20, 1); // Acknowledge completion only; preserve error evidence. + HardwareTransferPort port; + clock_interval_to_deadline (timeoutUs, kMicrosecondScale, &port.deadline); + hostWrite (0x58, 1); + UfsTransferPoll state; + UfsTransferEnd end = pollUfsTransfer (port, state); + gRootDiagnostic.polls = state.polls; + gRootDiagnostic.dbSeen = state.doorbellSeen; + gRootDiagnostic.isSeen = state.interruptSeen; + *ocs = state.ocs; + if (end != UfsTransferEnd::Completed) { + // No cancel/reset/retry: the controller may still own the buffer, even + // with DB=0. Keep it allocated and prohibit all subsequent CPU writes. + gHost.poisoned = true; + gHost.ready = false; + return recordTransport (end == UfsTransferEnd::Timeout ? kIOReturnTimeout : kIOReturnIOError, *ocs); + } + cacheInvalidate (gHost.virt, kDataOffset + dataBytes); + gRootDiagnostic.response = loadBe32 (gHost.virt + kUcdOffset + kRspOffset); + return recordTransport (*ocs == 0 ? kIOReturnSuccess : kIOReturnIOError, *ocs); +} + +static bool hostClearUcd (void) +{ + if (gHost.poisoned) return false; + // Check before touching the command/response area, not just before ringing. + if (read32 (gHost.hc, 0x58) != 0) { + gHost.poisoned = true; + gHost.ready = false; + recordTransport (kIOReturnBusy, 15); + return false; + } + if (read32 (gHost.hc, 0x20) & kUfsFatalStatus) { + gHost.poisoned = true; + gHost.ready = false; + recordTransport (kIOReturnIOError, 15); + return false; + } + uint32_t *words = reinterpret_cast (gHost.virt + kUcdOffset); + for (unsigned index = 0; index < (kPrdtOffset + 16) / 4; index++) { + words[index] = 0; + } + return true; +} + +static IOReturn hostProtocolError (uint32_t reason) +{ + gRootDiagnostic.protocol = reason; + gHost.ready = false; + gHost.poisoned = true; + return recordTransport (kIOReturnIOError, gRootDiagnostic.ocs); +} + +// Validate a completed response before interpreting zero-filled status bytes. +// Protocol codes: 1 type, 2 task tag, 3 LUN, 4 length/EHS, 5 query echo, +// 6 short/overrun transfer, 7 nonzero NOP result. +static IOReturn hostResponseHeader (uint8_t type, uint8_t lun, uint8_t tag) +{ + const uint8_t *rsp = gHost.virt + kUcdOffset + kRspOffset; + if (rsp[0] != type) return hostProtocolError (1); + if (rsp[3] != tag) return hostProtocolError (2); + if (rsp[2] != lun) return hostProtocolError (3); + unsigned length = (unsigned (rsp[10]) << 8) | rsp[11]; + if (rsp[8] || length > 512 - 32) return hostProtocolError (4); + return kIOReturnSuccess; +} + +static IOReturn hostNop (void) +{ + if (!hostClearUcd ()) return kIOReturnNotReady; + gHost.virt[kUcdOffset + 3] = 1; + uint32_t ocs = 15; + IOReturn result = hostSubmit (0, 0, 200000, &ocs); + if (result != kIOReturnSuccess) return result; + result = hostResponseHeader (0x20, 0, 1); + if (result != kIOReturnSuccess) return result; + const uint8_t *rsp = gHost.virt + kUcdOffset + kRspOffset; + if (rsp[10] || rsp[11]) return hostProtocolError (4); + if (rsp[6] || rsp[7]) return hostProtocolError (7); + return kIOReturnSuccess; +} + +// Run a SCSI command. direction 0: no data, 1: host -> device from the DMA data area (already filled by the caller), 2: device +// -> host into the DMA data area. Callers other than hostWrite4k never pass direction 1. +static IOReturn hostScsiDir (uint8_t lun, const uint8_t *cdb, unsigned cdbLength, unsigned direction, uint32_t dataBytes, + uint32_t timeoutUs, uint32_t *scsiStatus) +{ + *scsiStatus = 0xffff; + if (!cdb || !cdbLength || cdbLength > 16 || dataBytes > kDataMax || direction > 2 || + (dataBytes & 3) || (dataBytes ? !direction : direction)) return kIOReturnBadArgument; + if (!hostClearUcd ()) return kIOReturnNotReady; + uint8_t *ucd = gHost.virt + kUcdOffset; + ucd[0] = 0x01; // COMMAND UPIU + ucd[1] = direction == 2 ? 0x40 : direction == 1 ? 0x20 : 0x00; // R: data in, W: data out + ucd[2] = lun; + putBe32 (ucd + 12, dataBytes); + for (unsigned index = 0; index < cdbLength; index++) { + ucd[16 + index] = cdb[index]; + } + uint32_t ocs = 0xf; + IOReturn status = hostSubmit (direction ? direction : 0U, dataBytes, timeoutUs, &ocs); + if (status != kIOReturnSuccess) return status; + status = hostResponseHeader (0x21, lun, 0); + if (status != kIOReturnSuccess) return status; + const uint8_t *rsp = ucd + kRspOffset; + *scsiStatus = (uint32_t (rsp[6]) << 8) | rsp[7]; + gRootDiagnostic.scsi = *scsiStatus; + gRootDiagnostic.residual = loadBe32 (rsp + 12); + unsigned length = (unsigned (rsp[10]) << 8) | rsp[11]; + if (length) { + unsigned senseLength = (unsigned (rsp[32]) << 8) | rsp[33]; + if (length < 2 || senseLength > length - 2) return hostProtocolError (4); + const uint8_t *sense = rsp + 34; + if (senseLength >= 14 && ((sense[0] & 0x7f) == 0x70 || (sense[0] & 0x7f) == 0x71)) + gRootDiagnostic.sense = (uint32_t (sense[2] & 15) << 16) | (uint32_t (sense[12]) << 8) | sense[13]; + else if (senseLength >= 4 && ((sense[0] & 0x7f) == 0x72 || (sense[0] & 0x7f) == 0x73)) + gRootDiagnostic.sense = (uint32_t (sense[1] & 15) << 16) | (uint32_t (sense[2]) << 8) | sense[3]; + } + if (!*scsiStatus && ((rsp[1] & 0x60) || gRootDiagnostic.residual)) return hostProtocolError (6); + IOReturn result = *scsiStatus == 0 ? kIOReturnSuccess : kIOReturnNotReady; + gRootDiagnostic.result = static_cast (result); + return result; +} + +// The original read-only entry point: data, when present, always flows device -> host. +static IOReturn hostScsi (uint8_t lun, const uint8_t *cdb, unsigned cdbLength, bool toHost, uint32_t dataBytes, + uint32_t timeoutUs, uint32_t *scsiStatus) +{ + if (dataBytes && !toHost) { *scsiStatus = 0xffff; return kIOReturnBadArgument; } + return hostScsiDir (lun, cdb, cdbLength, dataBytes ? 2U : 0U, dataBytes, timeoutUs, scsiStatus); +} + +// Query read request (opcode 0x03 READ ATTRIBUTE / 0x05 READ FLAG). Returns the value field. +static IOReturn hostQueryRead (uint8_t opcode, uint8_t idn, uint32_t *value) +{ + if (!value || (opcode != 3 && opcode != 5)) return kIOReturnBadArgument; + if (!hostClearUcd ()) return kIOReturnNotReady; + uint8_t *ucd = gHost.virt + kUcdOffset; + ucd[0] = 0x16; // QUERY REQUEST + ucd[3] = 1; + ucd[5] = 0x01; // standard read request (header dword 1, byte 1) + ucd[12] = opcode; + ucd[13] = idn; + uint32_t ocs = 0xf; + IOReturn status = hostSubmit (0, 0, 200000, &ocs); + if (status != kIOReturnSuccess) { + return status; + } + status = hostResponseHeader (0x36, 0, 1); + if (status != kIOReturnSuccess) return status; + const uint8_t *rsp = ucd + kRspOffset; + if (rsp[10] || rsp[11]) return hostProtocolError (4); + if (rsp[12] != opcode || rsp[13] != idn || rsp[14] || rsp[15]) return hostProtocolError (5); + if (rsp[6] != 0) { + gRootDiagnostic.result = static_cast (kIOReturnError); + return kIOReturnError; + } + *value = loadBe32 (rsp + 20); + return kIOReturnSuccess; +} + +// Read `blocks` (<= kDataMax / 4096) 4096-byte blocks of LUN `lun` starting at `lba` into dst. +static IOReturn hostRead4k (uint8_t lun, uint64_t lba, uint32_t blocks, void *dst) +{ + if (!gHost.ready || !dst || !blocks || blocks > kDataMax / 4096 || lba > 0xffffffffULL - (blocks - 1)) { + return kIOReturnBadArgument; + } + IOLockLock (gHost.lock); + gRootDiagnostic.lba = lba; + uint8_t cdb[10] = { 0x28, 0, uint8_t (lba >> 24), uint8_t (lba >> 16), uint8_t (lba >> 8), uint8_t (lba), + 0, uint8_t (blocks >> 8), uint8_t (blocks), 0 }; + uint32_t scsi = 0; + IOReturn status = hostScsi (lun, cdb, 10, true, blocks * 4096U, 1000000, &scsi); + if (status == kIOReturnSuccess) { + memcpy (dst, gHost.virt + kDataOffset, blocks * 4096U); + } + IOLockUnlock (gHost.lock); + return status; +} + +// ---- Persistent store window ------------------------------------------------------------------------------------- +// The only place this driver writes. One window of 4096-byte LBAs on LUN 0 is opened once, by the store integration, after +// it has proved that the window lies inside the userdata partition and clear of the root image; every write is checked +// against it again here, so nothing outside it can be written even if a caller is wrong. +static bool gStoreOpen = false; +static uint64_t gStoreFirst = 0, gStoreBlocks = 0; + +static bool hostStoreOpen (uint64_t first, uint64_t blocks) +{ + if (gStoreOpen || !blocks || first > 0xffffffffULL || blocks > 0xffffffffULL - first) return false; + gStoreFirst = first; + gStoreBlocks = blocks; + gStoreOpen = true; + return true; +} + +// WRITE(10) of `blocks` 4096-byte blocks from src to lba on LUN 0; the range must lie inside the store window. +static IOReturn hostWrite4k (uint8_t lun, uint64_t lba, uint32_t blocks, const void *src) +{ + if (!gHost.ready || !src || !blocks || blocks > kDataMax / 4096 || lun != 0 || !gStoreOpen || + lba < gStoreFirst || lba - gStoreFirst >= gStoreBlocks || blocks > gStoreBlocks - (lba - gStoreFirst)) { + return kIOReturnBadArgument; + } + IOLockLock (gHost.lock); + gRootDiagnostic.lba = lba; + memcpy (gHost.virt + kDataOffset, src, blocks * 4096U); + uint8_t cdb[10] = { 0x2a, 0, uint8_t (lba >> 24), uint8_t (lba >> 16), uint8_t (lba >> 8), uint8_t (lba), + 0, uint8_t (blocks >> 8), uint8_t (blocks), 0 }; + uint32_t scsi = 0; + IOReturn status = hostScsiDir (0, cdb, 10, 1, blocks * 4096U, 2000000, &scsi); + IOLockUnlock (gHost.lock); + return status; +} + +// SYNCHRONIZE CACHE(10) for the whole LUN: everything written so far is on the medium when this returns. +static IOReturn hostSyncCache (void) +{ + if (!gHost.ready || !gStoreOpen) return kIOReturnNotReady; + IOLockLock (gHost.lock); + uint8_t cdb[10] = { 0x35 }; + uint32_t scsi = 0; + IOReturn status = hostScsiDir (0, cdb, 10, 0, 0, 10000000, &scsi); + IOLockUnlock (gHost.lock); + return status; +} + diff --git a/drivers/NativeUfs/RootDiagnostics.h b/drivers/NativeUfs/RootDiagnostics.h new file mode 100644 index 0000000..f8188dc --- /dev/null +++ b/drivers/NativeUfs/RootDiagnostics.h @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: BSD-2-Clause-Patent +// Passive snapshot formatter; caller supplies integer types and snprintf. +// No hardware access or decision about trust, transport or publication here. +struct UfsRootDiagnostics { + const char *stage = "not started"; + uint64_t dma = 0, list = 0, lba = 0, first = 0, last = 0; + uint64_t windowBytes = 0, policyCount = 0, reads = 0, failures = 0; + uint32_t cap = 0, version = 0, hcs = 0, hce = 0, doorbell = 0, interrupt = 0; + uint32_t smr = 0, s2cr = 0, cbar = 0, sctlr = 0; + uint32_t type = 0, opcode = 0, lun = 0, command = 0, ocs = 0xff, scsi = 0xffff; + uint32_t response = 0, result = 0, power = 0, manifest = 0xffffffff; + uint32_t magic0 = 0, magic1 = 0, windows = 0, prepared = 0, published = 0; + uint32_t verify = 0xffffffff; + uint32_t protocol = 0, residual = 0, sense = 0; + uint64_t far = 0, contiguous = 0; + uint32_t runStop = 0, ahit = 0, polls = 0, dbSeen = 0, isSeen = 0, isBefore = 0; + uint32_t gfsrBefore = 0, gfsr = 0, gsyn = 0, fsrBefore = 0, fsr = 0, fsyn = 0; + uint32_t smmuId2 = 0, cba2r = 0, uic[5] = {}; + uint32_t smmuId1 = 0, smmuSlot = 0xffffffff; + uint32_t publicationPhase = 0, publicationWindow = 0xffffffff; +}; + +static inline int formatUfsRootDiagnostics(char *buffer, size_t capacity, + const UfsRootDiagnostics &d, const char *reason, + const char *title = "UFS ROOT FAILURE") +{ + return snprintf(buffer, capacity, + "%s: %s\nstage=%s result=%08x\n" + "HC cap=%08x ver=%08x hcs=%08x hce=%08x db=%08x is=%08x\n" + "DMA=%llx UTRL=%llx SMMU smr=%08x s2cr=%08x cbar=%08x sctlr=%08x\n" + "CMD n=%u type=%02x op=%02x lun=%02x ocs=%02x rsp=%08x scsi=%04x power=%02x\n" + "PROTOCOL reject=%u residual=%u sense=%06x SMMU slot=%u id1=%x\n" + "QUEUE rsr=%x ahit=%x polls=%u dbSeen=%x isSeen=%x isBefore=%x contiguous=%llx\n" + "FAULT global=%x->%x syn=%x cb=%x->%x syn=%x far=%llx id2=%x cba2r=%x\n" + "UIC pa=%08x dl=%08x net=%08x tr=%08x dme=%08x\n" + "GPT/read lba=%llx userdata=%llx..%llx magic=%08x %08x manifest=%u\n" + "WINDOWS found=%u prepared=%u published=%u bytes=%llx policies=%llu verify=%08x\n" + "IO reads=%llu failures=%llu pubPhase=%u window=%u\n", + title, reason, d.stage, d.result, d.cap, d.version, d.hcs, d.hce, d.doorbell, d.interrupt, + (unsigned long long)d.dma, (unsigned long long)d.list, d.smr, d.s2cr, d.cbar, d.sctlr, + d.command, d.type, d.opcode, d.lun, d.ocs, d.response, d.scsi, d.power, + d.protocol, d.residual, d.sense, d.smmuSlot, d.smmuId1, + d.runStop, d.ahit, d.polls, d.dbSeen, d.isSeen, d.isBefore, (unsigned long long)d.contiguous, + d.gfsrBefore, d.gfsr, d.gsyn, d.fsrBefore, d.fsr, d.fsyn, (unsigned long long)d.far, + d.smmuId2, d.cba2r, d.uic[0], d.uic[1], d.uic[2], d.uic[3], d.uic[4], + (unsigned long long)d.lba, (unsigned long long)d.first, (unsigned long long)d.last, + d.magic0, d.magic1, d.manifest, d.windows, d.prepared, d.published, + (unsigned long long)d.windowBytes, (unsigned long long)d.policyCount, d.verify, + (unsigned long long)d.reads, (unsigned long long)d.failures, d.publicationPhase, d.publicationWindow); +} diff --git a/drivers/NativeUfs/RootLayout.h b/drivers/NativeUfs/RootLayout.h new file mode 100644 index 0000000..4a596c8 --- /dev/null +++ b/drivers/NativeUfs/RootLayout.h @@ -0,0 +1,219 @@ +// SPDX-License-Identifier: BSD-2-Clause-Patent +#pragma once +// Shared production parser. Requires hostRead4k, SHA256, diagnostics and C memory primitives. +// ---- LUN 0 GPT and the root-image manifest ------------------------------------------------- +struct RootWindow { + char name[32]; + uint64_t offset; // bytes from the start of the userdata partition + uint64_t size; // bytes + uint8_t sha256[32]; +}; + +struct RootLayout { + uint64_t userdataFirst, userdataLast; // 4096-byte LBAs on LUN 0 + uint64_t imageBytes; // size of the flashed root image from the manifest (0 when none) + uint64_t gptEntriesLba; // GPT facts recorded for the persistent store's before/after comparison + uint32_t gptHeaderCrc, gptEntriesCrc, gptEntryCount; + uint32_t windows; + RootWindow window[4]; + uint32_t manifestStatus; // 0 ok, 1 no manifest magic, 2 bad version/count, 3 hash, 4 bounds, 5 names, 6 overlap, 7 attributes + uint32_t magic0, magic1; // first 8 bytes of the partition's first block + uint32_t ext4Magic; // 16 bits at byte 1080 of the partition (0xef53 on Linux ext4) + uint32_t ext4Blocks; // s_blocks_count_lo at byte 1028 +}; + +static uint8_t gScratch[kDataMax]; + +static uint32_t le32 (const uint8_t *at) +{ + return uint32_t (at[0]) | (uint32_t (at[1]) << 8) | (uint32_t (at[2]) << 16) | (uint32_t (at[3]) << 24); +} + +static uint64_t le64 (const uint8_t *at) +{ + return uint64_t (le32 (at)) | (uint64_t (le32 (at + 4)) << 32); +} + +// GPT uses the reflected IEEE CRC32 polynomial. The header CRC field is zero +// during its own checksum; array CRC covers exactly count * entrySize bytes. +static uint32_t rootCrc32 (const uint8_t *data, size_t bytes, bool header = false) +{ + uint32_t crc = 0xffffffffU; + for (size_t i = 0; i < bytes; ++i) { + crc ^= header && i >= 16 && i < 20 ? 0 : data[i]; + for (unsigned bit = 0; bit < 8; ++bit) + crc = (crc >> 1) ^ (0xedb88320U & (0U - (crc & 1))); + } + return ~crc; +} + +static bool rootEntryUsed (const uint8_t *entry) +{ + uint8_t used = 0; + for (unsigned i = 0; i < 16; ++i) used |= entry[i]; + return used != 0; +} + +// Find the partition named "userdata" in LUN 0's primary GPT (4096-byte sectors), then read the +// manifest at its first block. Read-only. Returns false only if the GPT itself is unusable. +static bool findRootLayout (RootLayout *layout, const char **why) +{ + gRootDiagnostic.stage = "GPT header"; + bzero (layout, sizeof (*layout)); + if (hostRead4k (0, 1, 1, gScratch) != kIOReturnSuccess) { + *why = "GPT header read"; + return false; + } + if (le32 (gScratch) != 0x20494645U || le32 (gScratch + 4) != 0x54524150U || le32 (gScratch + 84) != 128) { + *why = "GPT header"; + return false; + } + uint32_t headerBytes = le32 (gScratch + 12); + if (le32 (gScratch + 8) != 0x00010000 || headerBytes < 92 || headerBytes > 4096 || + le32 (gScratch + 20) || le64 (gScratch + 24) != 1 || + rootCrc32 (gScratch, headerBytes, true) != le32 (gScratch + 16)) { + *why = "GPT header CRC/revision"; + return false; + } + uint64_t backup = le64 (gScratch + 32), first = le64 (gScratch + 40), last = le64 (gScratch + 48); + uint32_t entriesCrc = le32 (gScratch + 88); + uint64_t entriesLba = le64 (gScratch + 72); + uint32_t count = le32 (gScratch + 80); + layout->gptHeaderCrc = le32 (gScratch + 16); + layout->gptEntriesCrc = entriesCrc; + layout->gptEntriesLba = entriesLba; + layout->gptEntryCount = count; + if (entriesLba < 2 || entriesLba > 64 || !count || count > 128) { + *why = "GPT geometry"; + return false; + } + uint32_t blocks = (count * 128 + 4095) / 4096; + // This path uses READ(10), so reject geometry beyond its actual LBA range. + if (first < entriesLba + blocks || last < first || backup <= last || backup > UINT32_MAX || + backup - last <= blocks) { + *why = "GPT usable range"; + return false; + } + gRootDiagnostic.stage = "GPT entries"; + if (hostRead4k (0, entriesLba, blocks, gScratch) != kIOReturnSuccess) { + *why = "GPT entries read"; + return false; + } + if (rootCrc32 (gScratch, count * 128) != entriesCrc) { + *why = "GPT entries CRC"; + return false; + } + static const char wanted[] = "userdata"; + bool found = false; + for (uint32_t index = 0; index < count; index++) { + const uint8_t *entry = gScratch + index * 128; + if (!rootEntryUsed (entry)) continue; + uint64_t entryFirst = le64 (entry + 32), entryLast = le64 (entry + 40); + if (entryFirst < first || entryLast < entryFirst || entryLast > last) { + *why = "GPT partition bounds"; + return false; + } + for (uint32_t prior = 0; prior < index; ++prior) { + const uint8_t *other = gScratch + prior * 128; + if (rootEntryUsed (other) && entryFirst <= le64 (other + 40) && le64 (other + 32) <= entryLast) { + *why = "GPT partition overlap"; + return false; + } + } + bool match = true; + for (unsigned c = 0; c < sizeof (wanted); c++) { + if (entry[56 + 2 * c] != static_cast (wanted[c]) || entry[57 + 2 * c] != 0) { + match = false; + break; + } + } + if (match) { + if (found) { + *why = "duplicate userdata partition"; + return false; + } + layout->userdataFirst = le64 (entry + 32); + layout->userdataLast = le64 (entry + 40); + found = true; + } + } + if (!found || layout->userdataLast < layout->userdataFirst) { + *why = "no userdata partition"; + return false; + } + gRootDiagnostic.first = layout->userdataFirst; + gRootDiagnostic.last = layout->userdataLast; + gRootDiagnostic.stage = "root manifest"; + // First block of the partition: manifest if the root image was flashed, ext4 otherwise. + if (hostRead4k (0, layout->userdataFirst, 1, gScratch) != kIOReturnSuccess) { + *why = "userdata first block read"; + return false; + } + layout->magic0 = le32 (gScratch); + layout->magic1 = le32 (gScratch + 4); + layout->ext4Magic = uint32_t (gScratch[1080]) | (uint32_t (gScratch[1081]) << 8); + layout->ext4Blocks = le32 (gScratch + 1028); + layout->manifestStatus = 1; + if (layout->magic0 == 0x474e4853U && layout->magic1 == 0x544f4f52U) { // "SHNGROOT" + uint32_t version = le32 (gScratch + 8), n = le32 (gScratch + 12); + uint64_t imageBytes = le64 (gScratch + 16); + layout->manifestStatus = 2; + if (version == 1 && n && n <= 4) { + SHA256_CTX context; + uint8_t digest[32]; + SHA256_Init (&context); + SHA256_Update (&context, gScratch, 4064); + SHA256_Final (digest, &context); + layout->manifestStatus = 3; + if (memcmp (digest, gScratch + 4064, 32) == 0) { + uint64_t partitionBytes = (layout->userdataLast - layout->userdataFirst + 1) * 4096ULL; + layout->manifestStatus = 4; + bool bounds = imageBytes >= 4096 && !(imageBytes & 4095) && imageBytes <= partitionBytes; + for (uint32_t index = 0; index < n && bounds; index++) { + const uint8_t *entry = gScratch + 24 + 128 * index; + uint64_t offset = le64 (entry + 32), size = le64 (entry + 40); + if ((offset & 4095) || offset < 4096 || (size & 511) || !size || offset >= imageBytes || + size > imageBytes - offset || ((size + 4095) & ~4095ULL) > imageBytes - offset) { + bounds = false; + break; + } + if (le32 (entry + 48) != 512 || le32 (entry + 52) != 1) { + layout->manifestStatus = 7; bounds = false; break; + } + unsigned length = 0; + while (length < 32 && entry[length] >= 0x20 && entry[length] <= 0x7e) ++length; + if (!length || length == 32 || entry[length] != 0) { + layout->manifestStatus = 5; bounds = false; break; + } + for (uint32_t prior = 0; prior < index; ++prior) { + const RootWindow &other = layout->window[prior]; + if (!strcmp (reinterpret_cast (entry), other.name)) { + layout->manifestStatus = 5; bounds = false; break; + } + // Include the final sector padding that a physical 4K read fetches. + if (offset < other.offset + ((other.size + 4095) & ~4095ULL) && + other.offset < offset + ((size + 4095) & ~4095ULL)) { + layout->manifestStatus = 6; bounds = false; break; + } + } + if (!bounds) break; + memcpy (layout->window[index].name, entry, 32); + layout->window[index].name[31] = 0; + layout->window[index].offset = offset; + layout->window[index].size = size; + memcpy (layout->window[index].sha256, entry + 56, 32); + } + if (bounds) { + layout->windows = n; + layout->imageBytes = imageBytes; + layout->manifestStatus = 0; + } + } + } + } + gRootDiagnostic.magic0 = layout->magic0; + gRootDiagnostic.magic1 = layout->magic1; + gRootDiagnostic.manifest = layout->manifestStatus; + gRootDiagnostic.windows = layout->windows; + return true; +} diff --git a/drivers/NativeUfs/RootPublication.h b/drivers/NativeUfs/RootPublication.h new file mode 100644 index 0000000..e6e6c3f --- /dev/null +++ b/drivers/NativeUfs/RootPublication.h @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: BSD-2-Clause-Patent +#pragma once +// Shared production lifecycle decisions; adapters own IOKit objects and waits. +struct UfsRootPlan { unsigned count = 0, order[4] = {}; }; +static bool ufsBuildRootPlan (const char *const *names, unsigned count, UfsRootPlan &plan) +{ + plan = {}; + if (!names || count < 2 || count > 4) return false; + unsigned system = count, os = count; + for (unsigned i = 0; i < count; ++i) { + if (!names[i] || !names[i][0]) return false; + for (unsigned j = 0; j < i; ++j) if (!strcmp (names[i], names[j])) return false; + if (!strcmp (names[i], "SystemOS")) system = i; + if (!strcmp (names[i], "OS")) os = i; + } + if (system == count || os == count) return false; + plan.order[0] = system; + unsigned position = 1; + for (unsigned i = 0; i < count; ++i) if (i != system && i != os) plan.order[position++] = i; + plan.order[position] = os; + plan.count = count; + return true; +} +enum class UfsRootPhase { Complete, Prepare, Identity, Schedule, Start, Media }; +struct UfsRootPublicationResult { + UfsRootPhase phase; + unsigned index; + unsigned published; +}; +template static void ufsDiscardPrepared (Ops &ops, unsigned count) +{ + while (count) ops.discard (--count); +} +template +static UfsRootPublicationResult ufsPrepareRootWindows (Ops &ops, unsigned count) +{ + if (count < 2 || count > 4) return { UfsRootPhase::Prepare, 0, 0 }; + for (unsigned i = 0; i < count; ++i) { + if (!ops.prepare (i)) { + // Include a partially allocated/initialized current object. + ufsDiscardPrepared (ops, i + 1); + return { UfsRootPhase::Prepare, i, 0 }; + } + } + if (!ops.validate ()) { + ufsDiscardPrepared (ops, count); + return { UfsRootPhase::Identity, count, 0 }; + } + if (!ops.schedule ()) { + ufsDiscardPrepared (ops, count); + return { UfsRootPhase::Schedule, count, 0 }; + } + return { UfsRootPhase::Complete, count, 0 }; +} +template +static UfsRootPublicationResult ufsPublishRootWindows (Ops &ops, const UfsRootPlan &plan) +{ + unsigned published = 0; + for (unsigned position = 0; position < plan.count; ++position) { + unsigned index = plan.order[position]; + bool started = ops.start (index); + if (!started || !ops.wait (index)) { + // Keep already-started objects alive: their asynchronous clients may + // still own I/O. Only discard windows that have never been published. + unsigned first = position + (started ? 1 : 0); + for (unsigned p = plan.count; p > first; --p) ops.discard (plan.order[p - 1]); + return { started ? UfsRootPhase::Media : UfsRootPhase::Start, index, published + (started ? 1U : 0U) }; + } + ++published; + } + return { UfsRootPhase::Complete, plan.count, published }; +} +// A same-UUID IOMedia is ready only when its parent traversal reaches the +// actual verified window. An invalid or overlong iterator fails closed. +template +static bool ufsMediaHasAncestor (Iterator &iterator, Entry *window) +{ + for (unsigned steps = 0; steps < 256 && iterator.isValid (); ++steps) { + auto *entry = iterator.getNextObject (); + if (!entry) return false; + if (entry == window) return iterator.isValid (); + } + return false; +} diff --git a/drivers/NativeUfs/RootQueue.h b/drivers/NativeUfs/RootQueue.h new file mode 100644 index 0000000..989b956 --- /dev/null +++ b/drivers/NativeUfs/RootQueue.h @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: BSD-2-Clause-Patent +// Root queue takeover; shared with the host fault-injection harness. +static bool hostWaitRegister (unsigned offset, uint32_t mask, uint32_t value) +{ + uint64_t deadline; + clock_interval_to_deadline (100, kMillisecondScale, &deadline); + do { + if ((read32 (gHost.hc, offset) & mask) == value) return true; + IODelay (10); + } while (mach_absolute_time () < deadline); + return false; +} + +static bool hostStopQueue (const char **why) +{ + // Confirm the firmware has no active transfer/task before changing its + // request list or the storage stream's SMMU routing. + if (!(read32 (gHost.hc, 0x34) & 1) || (read32 (gHost.hc, 0x30) & 0x0f) != 0x0f) { + recordTransport (kIOReturnNotReady, 15); + *why = "controller not ready for takeover"; + return false; + } + if (read32 (gHost.hc, 0x58) || read32 (gHost.hc, 0x78)) { + recordTransport (kIOReturnBusy, 15); + *why = "firmware request outstanding"; + return false; + } + if (read32 (gHost.hc, 0x20) & kUfsFatalStatus) { + recordTransport (kIOReturnIOError, 15); + *why = "firmware controller error"; + return false; + } + hostWrite (0x24, 0); + hostWrite (0x60, 0); + if (!hostWaitRegister (0x60, 1, 0)) { + recordTransport (kIOReturnTimeout, 15); + *why = "request list stop"; + return false; + } + return true; +} + +static bool hostStartQueue (const char **why) +{ + hostWrite (0x50, static_cast (gHost.phys)); + hostWrite (0x54, static_cast (gHost.phys >> 32)); + if (read32 (gHost.hc, 0x50) != uint32_t (gHost.phys) || + read32 (gHost.hc, 0x54) != uint32_t (gHost.phys >> 32)) { + recordTransport (kIOReturnIOError, 15); + *why = "request list address"; + return false; + } + gRootDiagnostic.isBefore = read32 (gHost.hc, 0x20); + hostWrite (0x20, gRootDiagnostic.isBefore); + hostWrite (0x60, 1); + if (!hostWaitRegister (0x60, 1, 1) || !hostWaitRegister (0x30, 3, 3)) { + recordTransport (kIOReturnTimeout, 15); + *why = "request list ready"; + return false; + } + return true; +} diff --git a/drivers/NativeUfs/RootTransportLayout.h b/drivers/NativeUfs/RootTransportLayout.h new file mode 100644 index 0000000..a6245c6 --- /dev/null +++ b/drivers/NativeUfs/RootTransportLayout.h @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: BSD-2-Clause-Patent +enum : uint32_t { + kUcdOffset = 4096, + kRspOffset = 512, + kPrdtOffset = 1024, + kDataOffset = 8192, + kDataMax = 131072, + kDmaBytes = kDataOffset + kDataMax, +}; diff --git a/drivers/NativeUfs/RootWindowRead.h b/drivers/NativeUfs/RootWindowRead.h new file mode 100644 index 0000000..6e92707 --- /dev/null +++ b/drivers/NativeUfs/RootWindowRead.h @@ -0,0 +1,54 @@ +// SPDX-License-Identifier: BSD-2-Clause-Patent +#pragma once +// Shared production window reader. Ops owns synchronization, block I/O, +// integrity verification and copying to the caller. No callback runs under +// this guard: the IOKit wrapper completes the request after this returns. +static bool ufsWindowGeometry (uint64_t absolute, uint64_t bytes) +{ + constexpr uint64_t limit = (uint64_t (UINT32_MAX) + 1) * 4096; + return bytes && !(bytes & 511) && !(absolute & 4095) && absolute < limit && bytes <= limit - absolute; +} +struct UfsWindowResult { IOReturn status; uint64_t done; }; +template struct UfsWindowGuard { + Ops &ops; + explicit UfsWindowGuard (Ops &value) : ops(value) { ops.lock (); } + ~UfsWindowGuard () { ops.unlock (); } + UfsWindowGuard (const UfsWindowGuard &) = delete; + UfsWindowGuard &operator= (const UfsWindowGuard &) = delete; +}; +template +static UfsWindowResult ufsReadWindow (Ops &ops, uint64_t absolute, uint64_t bytes, + uint64_t block, uint64_t count, uint8_t *bounce) +{ + if (!bounce || !ufsWindowGeometry (absolute, bytes) || !count || block >= bytes / 512 || count > bytes / 512 - block) + return { kIOReturnBadArgument, 0 }; + UfsWindowGuard guard (ops); + uint64_t start = absolute + block * 512, remaining = count * 512; + UfsWindowResult result = { kIOReturnSuccess, 0 }; + while (remaining) { + uint64_t lba = (start + result.done) / 4096, skip = (start + result.done) % 4096; + uint64_t blocks = (skip + remaining + 4095) / 4096; + if (blocks > kDataMax / 4096) blocks = kDataMax / 4096; + result.status = ops.read (lba, static_cast (blocks), bounce); + if (result.status != kIOReturnSuccess) break; + // The window lock protects this same snapshot through verify AND copy. + // The transport lock alone protects only the DMA-to-bounce copy. + for (uint64_t index = 0; index < blocks; ++index) { + uint64_t windowBlock = lba + index - absolute / 4096; + uint64_t offset = windowBlock * 4096; + if (offset >= bytes) { result.status = kIOReturnBadArgument; break; } + uint64_t length = bytes - offset < 4096 ? bytes - offset : 4096; + result.status = ops.verify (windowBlock, bounce + index * 4096, static_cast (length)); + if (result.status != kIOReturnSuccess) break; + } + if (result.status != kIOReturnSuccess) break; + uint64_t usable = blocks * 4096 - skip; + if (usable > remaining) usable = remaining; + uint64_t copied = ops.copy (result.done, bounce + skip, usable); + if (copied > usable) { result.status = kIOReturnIOError; break; } + result.done += copied; + if (copied != usable) { result.status = kIOReturnIOError; break; } + remaining -= usable; + } + return result; +} diff --git a/drivers/NativeUfs/SmmuTakeover.h b/drivers/NativeUfs/SmmuTakeover.h new file mode 100644 index 0000000..79ae904 --- /dev/null +++ b/drivers/NativeUfs/SmmuTakeover.h @@ -0,0 +1,101 @@ +// SPDX-License-Identifier: BSD-2-Clause-Patent +// Plan before any SMMU write. Interface register facts: ARM SMMUv2. +struct UfsSmmuPlan { + unsigned slot = 0xffffffffU, bank = 0, pages = 0; + uint32_t smr = 0, s2cr = 0; + bool reuse = false; + const char *error = nullptr; +}; +// ARM IHI 0062D.c sections 1.6 and 3.7: VA64=0 still rejects inputs above +// 32 bits with SCTLR.M=0. With VA64=1 the bypass range is bounded by all +// three hardware widths, not TCR2.PASize (which only limits translation). +static unsigned ufsSmmuAddressBits(unsigned code) +{ + switch (code) { + case 0: return 32; case 1: return 36; case 2: return 40; + case 3: return 42; case 4: return 44; case 5: return 48; + default: return 0; + } +} +template +static UfsSmmuPlan planUfsSmmu(uint16_t stream, uint64_t dma, uint64_t bytes, Read read) +{ + UfsSmmuPlan p; + uint32_t idr0 = read(0x20), idr2 = read(0x28); + unsigned count = idr0 & 255; + if (!(idr0 & (1U << 30)) || !(idr2 & (1U << 12))) { + p.error = "SMMU AArch64 stage 1 unsupported"; return p; + } + for (unsigned shift = 0; shift <= 8; shift += 4) { + unsigned bits = ufsSmmuAddressBits((idr2 >> shift) & 15); + if (!bits || !bytes || dma >= (1ULL << bits) || bytes > (1ULL << bits) - dma) { + p.error = "DMA exceeds SMMU address width"; return p; + } + } + uint32_t idr1 = read(0x24); + unsigned banks = idr1 & 255; + p.pages = 1U << (((idr1 >> 28) & 7) + 1); + if ((idr1 >> 31) || !count || p.pages > 128 || banks < 2 || banks > 128 || + (p.pages + banks) * 4096 > 0xd0000) { + p.error = "unsupported SMMU geometry"; return p; + } + if (count > 128) count = 128; // Firmware's reliably accessible groups only. + p.bank = banks - 1; + unsigned matches = 0; + for (unsigned i = 0; i < count; ++i) { + uint32_t smr = read(0x800 + 4 * i); + if (!(smr & 0x80000000U)) { + if (!p.reuse) p.slot = i; + continue; + } + uint32_t s2cr = read(0xc00 + 4 * i); + uint32_t mask = (smr >> 16) & 0x7fff; + bool match = ((smr ^ stream) & 0xffff & ~mask) == 0; + bool ourBank = ((s2cr >> 16) & 3) == 0 && (s2cr & 255) == p.bank; + if (match) { + ++matches; + if (matches > 1 || mask || !ourBank) { + p.slot = i; p.smr = smr; p.s2cr = s2cr; + p.error = "conflicting UFS stream mapping"; return p; + } + p.reuse = true; p.slot = i; + } else if (ourBank) { + p.slot = i; p.smr = smr; p.s2cr = s2cr; + p.error = "reserved context bank already used"; return p; + } + } + uint32_t sctlr = read((p.pages + p.bank) * 4096); + if ((sctlr & 1) || (p.reuse && (sctlr || read(0x1000 + 4 * p.bank) != 0x10000 || + (read(0x1800 + 4 * p.bank) & ~1U)))) { + p.error = "reserved context bank not bypass"; return p; + } + if (p.slot == 0xffffffffU) p.error = "no free stream-match group"; + return p; +} + +// The caller must first quiesce the UFS queue. The planner refuses other +// streams sharing this bank and enabled translation; no live context is +// disabled here. Write callbacks must complete an MMIO barrier before return. +template +static const char *configureUfsSmmu(const UfsSmmuPlan &p, uint16_t stream, Read read, Write write) +{ + if (p.error) return p.error; + unsigned cb = (p.pages + p.bank) * 4096; + if (!p.reuse) write(cb, 0); + if (read(cb) != 0) return "SMMU bypass disable readback"; + // Set VA64 before exposing a newly allocated stream. Repair an exclusive + // existing 32-bit bypass too; do not silently reuse the faulty state. + if (read(0x1800 + 4 * p.bank) != 1) write(0x1800 + 4 * p.bank, 1); + if (!p.reuse) write(0x1000 + 4 * p.bank, 0x10000); + if (read(0x1800 + 4 * p.bank) != 1 || read(0x1000 + 4 * p.bank) != 0x10000 || read(cb) != 0) + return "SMMU VA64 bypass readback"; + if (!p.reuse) { + write(0xc00 + 4 * p.slot, p.bank); + if (read(0xc00 + 4 * p.slot) != p.bank) return "SMMU stream context readback"; + write(0x800 + 4 * p.slot, 0x80000000U | stream); + } + if (read(0x800 + 4 * p.slot) != (0x80000000U | stream) || + read(0xc00 + 4 * p.slot) != p.bank) + return "SMMU stream mapping readback"; + return nullptr; +} diff --git a/drivers/NativeUfs/TransferPoll.h b/drivers/NativeUfs/TransferPoll.h new file mode 100644 index 0000000..2f56e30 --- /dev/null +++ b/drivers/NativeUfs/TransferPoll.h @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: BSD-2-Clause-Patent +// Single-slot completion policy. Port supplies ordered MMIO, invalidate-only +// descriptor reads, a monotonic deadline and bounded pacing. No resubmission. +enum class UfsTransferEnd { Completed, Fatal, Timeout }; +struct UfsTransferPoll { + uint32_t polls = 0, doorbellSeen = 0, interruptSeen = 0, ocs = 15; +}; +static constexpr uint32_t kUfsFatalStatus = 0x71880U; + +template +static UfsTransferEnd pollUfsTransfer(Port &port, UfsTransferPoll &state) +{ + for (;;) { + ++state.polls; + uint32_t doorbell = port.doorbell(); + state.doorbellSeen |= doorbell; + state.interruptSeen |= port.interrupt(); + if (state.interruptSeen & kUfsFatalStatus) return UfsTransferEnd::Fatal; + if (!(doorbell & 1)) { + state.ocs = port.completedOcs(); + // A zero doorbell alone does not prove that this command was accepted. + // Also allow a completion so fast that software never observes DB=1. + if (state.ocs != 15) return UfsTransferEnd::Completed; + } + if (port.expired()) return UfsTransferEnd::Timeout; + port.delay(); + } +} diff --git a/drivers/NativeUfs/tests/RootCommands.cpp b/drivers/NativeUfs/tests/RootCommands.cpp new file mode 100644 index 0000000..b5c0a7d --- /dev/null +++ b/drivers/NativeUfs/tests/RootCommands.cpp @@ -0,0 +1,124 @@ +// SPDX-License-Identifier: BSD-2-Clause-Patent +// Real production commands with a deterministic DMA/HC fault fixture. +#include "RootCommandsBridge.cpp" +#include +#include +#include +enum class Fault { None, Empty, Type, Tag, Lun, Query, Residual, Underflow, + Oversize, SenseLength, CheckCondition, Timeout, Fatal, Stop, Start, HighBase }; +static Fault fault; +static std::array device; +static std::array registers; +static uint64_t now; +static unsigned writes, releases; +static bool released; + +static uint64_t io(unsigned op,uint64_t address,uint64_t value,void *bytes) +{ + if (op==0) return registers.at(address/4); + if (op==1) { + ++writes; + if (address==0x20) registers[address/4]&=~uint32_t(value); + else if (!(address==0x60 && ((fault==Fault::Stop && !value) || (fault==Fault::Start && value))) && + !(address==0x54 && fault==Fault::HighBase)) registers.at(address/4)=uint32_t(value); + if (address==0x58) { + assert(released); released=false; + if (fault==Fault::Fatal) { registers[0x20/4]=0x20000; return 0; } + registers[0x58/4]=0; + if (fault==Fault::Timeout) return 0; + uint8_t *cmd=device.data()+kUcdOffset, *rsp=cmd+kRspOffset; + std::fill(rsp,rsp+512,0); + memset(device.data()+kDataOffset,0x7b,kDataMax); + *reinterpret_cast(device.data()+8)=0; + if (fault==Fault::Empty) return 0; + memcpy(rsp,cmd,32); rsp[0]=cmd[0]+0x20; rsp[1]=0; + if (cmd[0]==1) putBe32(rsp+12,0); + if (cmd[0]==0x16) putBe32(rsp+20,0x11); + if (fault==Fault::Type) rsp[0]=0x3f; + if (fault==Fault::Tag) ++rsp[3]; + if (fault==Fault::Lun) ++rsp[2]; + if (fault==Fault::Query) ++rsp[13]; + if (fault==Fault::Residual) putBe32(rsp+12,4096); + if (fault==Fault::Underflow) rsp[1]=0x20; + if (fault==Fault::Oversize) rsp[10]=0xff; + if (fault==Fault::SenseLength) { rsp[11]=2; rsp[33]=18; } + if (fault==Fault::CheckCondition) { + rsp[7]=2; rsp[11]=20; rsp[33]=18; rsp[34]=0x70; + rsp[36]=6; rsp[46]=0x29; rsp[47]=0; + } + } + } + if (op==2 || op==3) { + assert(address>=gHost.phys && address-gHost.phys+value<=device.size()); + auto at=device.data()+(address-gHost.phys); + if (op==2) { ++releases; memcpy(at,bytes,value); released=true; } + else memcpy(bytes,at,value); + } + if (op==4) return now; + if (op==5) now+=value; + return 0; +} +static void reset(Fault f=Fault::None) +{ + fault=f; device.fill(0); registers.fill(0); now=0; writes=releases=0; released=false; + registers[0x34/4]=1; registers[0x30/4]=15; registers[0x60/4]=1; + ufs_test_reset(io,0x8cb5e9000); +} +static void assertQuarantined() +{ + assert(ufs_test_poisoned()); + std::array old; memcpy(old.data(),memory,sizeof(memory)); + unsigned oldWrites=writes, oldReleases=releases; + uint8_t cdb[6]={}; uint32_t scsi=0, value=0xabcdef; + assert(ufs_test_nop()!=0); + assert(ufs_test_query(3,2,&value)!=0 && value==0xabcdef); + assert(ufs_test_scsi(0,cdb,6,false,0,&scsi)!=0); + assert(writes==oldWrites && releases==oldReleases); + assert(!memcmp(old.data(),memory,sizeof(memory))); +} +int main() +{ + reset(); assert(ufs_test_nop()==0); + uint32_t value=0; assert(ufs_test_query(3,2,&value)==0 && value==0x11); + std::array out; + assert(ufs_test_read(0,1,32,out.data())==0); + assert(std::all_of(out.begin(),out.end(),[](uint8_t x){return x==0x7b;})); + assert(*reinterpret_cast(device.data()+20)==8); + assert(*reinterpret_cast(device.data()+4096+1024+4)==8); + assert(ufs_test_read(0,0xffffffffULL,1,out.data())==0); // Last READ(10) LBA is valid. + auto oldWrites=writes; + assert(ufs_test_read(0,0xffffffffULL,2,out.data())==kIOReturnBadArgument && writes==oldWrites); + for (Fault f: {Fault::Empty,Fault::Type,Fault::Tag,Fault::Lun,Fault::Residual, + Fault::Underflow,Fault::Oversize,Fault::SenseLength,Fault::Timeout,Fault::Fatal}) { + reset(f); out.fill(0xa5); + assert(ufs_test_read(0,1,1,out.data())!=0); + assert(std::all_of(out.begin(),out.end(),[](uint8_t x){return x==0xa5;})); + assertQuarantined(); + } + reset(Fault::Query); value=0xabcdef; + assert(ufs_test_query(3,2,&value)!=0 && value==0xabcdef); assertQuarantined(); + reset(Fault::CheckCondition); out.fill(0xa5); + assert(ufs_test_read(0,1,1,out.data())==kIOReturnNotReady); + assert(!gHost.poisoned && gRootDiagnostic.sense==0x062900); + assert(out[0]==0xa5); + fault=Fault::None; assert(ufs_test_read(0,1,1,out.data())==0); + reset(); uint8_t cdb[17]={}; uint32_t scsi; + assert(ufs_test_scsi(0,cdb,17,false,0,&scsi)==kIOReturnBadArgument); + assert(ufs_test_scsi(0,cdb,6,true,kDataMax+4,&scsi)==kIOReturnBadArgument); + assert(ufs_test_scsi(0,cdb,6,false,4096,&scsi)==kIOReturnBadArgument); // No writes to media. + assert(writes==0 && releases==0); + for (unsigned offset: {0x58U,0x78U}) { + reset(); registers[offset/4]=1; + assert(!ufs_test_stop_queue() && writes==0); + } + reset(); registers[0x34/4]=0; assert(!ufs_test_stop_queue() && writes==0); + reset(); registers[0x20/4]=0x20000; + assert(!ufs_test_stop_queue() && writes==0); + assert(ufs_test_nop()!=0 && writes==0 && releases==0); assertQuarantined(); + reset(Fault::Stop); assert(!ufs_test_stop_queue() && now>=100000); + reset(Fault::Start); assert(ufs_test_stop_queue()); assert(!ufs_test_start_queue()); + reset(Fault::HighBase); assert(ufs_test_stop_queue()); assert(!ufs_test_start_queue()); + reset(); assert(ufs_test_stop_queue() && ufs_test_start_queue()); + assert(registers[0x50/4]==0xcb5e9000 && registers[0x54/4]==8); + puts("PASS: actual command/queue path; malformed DMA responses, bounds, short reads, sense, quarantine, queue refusal and full-width DMA"); +} diff --git a/drivers/NativeUfs/tests/RootCommandsArm64.S b/drivers/NativeUfs/tests/RootCommandsArm64.S new file mode 100644 index 0000000..a49ff27 --- /dev/null +++ b/drivers/NativeUfs/tests/RootCommandsArm64.S @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: BSD-2-Clause-Patent +// Offline QEMU virt fixture only; these are not sheng boot tables. +.section .text.entry,"ax" +.global _start +_start: + msr daifset, #15 + ldr x0, =stack_end + mov sp, x0 + ldr x0, =vectors + msr vbar_el1, x0 + ldr x0, =__bss_start + ldr x1, =__bss_end +1: + cmp x0, x1 + b.hs 2f + str xzr, [x0], #8 + b 1b +2: + bl guest_main + b . + +.balign 2048 +vectors: +.rept 16 + bl guest_exception + b . + .balign 128 +.endr + +.global enable_translation +enable_translation: + msr ttbr0_el1, x0 + msr ttbr1_el1, xzr + // Attr0: normal WBWA; Attr1: device nGnRnE. 4 KiB TG0, 38-bit + // VA, caller-selected shareability, WBWA walks, 48-bit PA, TTBR1 disabled. + mov x0, #0xff + msr mair_el1, x0 + msr tcr_el1, x1 + dsb sy + isb + tlbi vmalle1 + dsb sy + isb + mrs x0, sctlr_el1 + orr x0, x0, #1 + orr x0, x0, #4 + orr x0, x0, #0x1000 + msr sctlr_el1, x0 + isb + ret + +.section .bss,"aw",%nobits +.balign 16 +stack: + .skip 65536 +stack_end: diff --git a/drivers/NativeUfs/tests/RootCommandsArm64.cpp b/drivers/NativeUfs/tests/RootCommandsArm64.cpp new file mode 100644 index 0000000..12db0cf --- /dev/null +++ b/drivers/NativeUfs/tests/RootCommandsArm64.cpp @@ -0,0 +1,161 @@ +// SPDX-License-Identifier: BSD-2-Clause-Patent +// Freestanding EL1 execution of production UFS code. QEMU virt addresses only. +#include +#include +extern "C" int snprintf(char *, size_t, const char *, ...); +extern "C" void *memcpy(void *out, const void *in, size_t n) { + auto d=static_cast(out); auto s=static_cast(in); + for (size_t i=0;i(0x09000000)=*s; +} +static void number(const char *name,uint64_t value) { + puts(name); puts("=0x"); + for (int shift=60;shift>=0;shift-=4) { + char c[2]={"0123456789abcdef"[(value>>shift)&15],0}; puts(c); + } + puts("\n"); +} +[[noreturn]] static void finish(unsigned status) { + uint64_t args[2]={0x20026,status}; + register uint64_t x0 asm("x0")=0x20; + register uint64_t x1 asm("x1")=reinterpret_cast(args); + asm volatile("hlt #0xf000"::"r"(x0),"r"(x1):"memory"); + for (;;) asm volatile("wfe"); +} +static void check(bool okay,const char *label) { + if (okay) return; + puts("FAIL "); puts(label); puts("\n"); + number("result",gRootDiagnostic.result); number("ocs",gRootDiagnostic.ocs); + number("polls",gRootDiagnostic.polls); number("dbSeen",gRootDiagnostic.dbSeen); + number("isSeen",gRootDiagnostic.isSeen); finish(1); +} +extern "C" [[noreturn]] void guest_exception() { + puts("FAIL exception\n"); uint64_t v; + asm volatile("mrs %0, esr_el1":"=r"(v)); number("ESR",v); + asm volatile("mrs %0, far_el1":"=r"(v)); number("FAR",v); + asm volatile("mrs %0, elr_el1":"=r"(v)); number("ELR",v); + finish(1); +} +alignas(4096) static uint64_t l1[512],l2[512],l3[512]; +alignas(4096) static uint8_t output[kDataMax]; +extern "C" void enable_translation(uint64_t,uint64_t); +extern "C" void guest_main() { + uint64_t el; asm volatile("mrs %0, CurrentEL":"=r"(el)); check(el==4,"EL1"); + // Device block for UART/PCI/UFS, cached identity map for fixture code, + // stack and tables. DMA uses only 4 KiB leaves, with an unmapped guard. + l1[0]=(uint64_t(3)<<53)|0x405; + l1[1]=reinterpret_cast(l2)|3; + constexpr uint64_t share=TEST_SHAREABILITY; + static_assert(share==2 || share==3,"Test outer or inner sharing"); + constexpr uint64_t tcr=(0x50080351aULL & ~0x3000ULL)|(share<<12); + for (unsigned i=0;i<64;++i) l2[i]=(0x40000000ULL+i*0x200000ULL)|0x401|(share<<8); + l2[128]=reinterpret_cast(l3)|3; + constexpr uint64_t physical=TEST_DMA_PHYS; + for (unsigned i=0;i(l1),tcr); + uint64_t state; + asm volatile("mrs %0, tcr_el1":"=r"(state)); number("TCR",state); + check(state==tcr,"TCR 4K"); + asm volatile("mrs %0, sctlr_el1":"=r"(state)); number("SCTLR",state); + check((state&0x1005)==0x1005,"MMU and caches enabled"); + asm volatile("mrs %0, ttbr0_el1":"=r"(state)); number("TTBR0",state); + check(state==reinterpret_cast(l1),"active table"); + asm volatile("mrs %0, mair_el1":"=r"(state)); number("MAIR",state); + asm volatile("mrs %0, ctr_el0":"=r"(state)); number("CTR",state); + number("DMA_VA",0x50000000); number("DMA_PA",physical); number("DMA_PTE",l3[0]); + // AT checks the active hardware translation, not merely table construction. + asm volatile("at s1e1r, %0; isb"::"r"(uint64_t(0x50000000)):"memory"); + asm volatile("mrs %0, par_el1":"=r"(state)); number("DMA_PAR",state); + check(!(state&1) && (state&0x0000fffffffff000ULL)==physical,"VA to PA"); + asm volatile("at s1e1r, %0; isb"::"r"(uint64_t(0x50000000+kDmaBytes)):"memory"); + asm volatile("mrs %0, par_el1":"=r"(state)); check(state&1,"4K guard page"); + + gHost={reinterpret_cast(0x50000000),physical,true,false,0x10000000,nullptr}; + for (unsigned i=0;i(pci+4)=6; + asm volatile("dsb sy":::"memory"); + number("CAP",read32(gHost.hc,0)); check(read32(gHost.hc,0)&0x01000000,"64-bit DMA"); + hostWrite(0x34,1); hostWrite(0x90,0x16); // Fixture-only HCE/link startup. + check(hostWaitRegister(0x30,15,15),"fixture link startup"); + const char *why=nullptr; + check(hostStopQueue(&why),"queue stop"); +#if TEST_BAD_DMA == 1 + gHost.phys=uint32_t(physical); +#elif TEST_BAD_DMA == 2 + gHost.phys=reinterpret_cast(gHost.virt); +#endif + check(hostStartQueue(&why),"queue start"); + auto result=hostNop(); +#if TEST_BAD_DMA + check(result==kIOReturnTimeout && gHost.poisoned,"bad DMA not quarantined on timeout"); + number("result",result); number("ocs",gRootDiagnostic.ocs); + number("polls",gRootDiagnostic.polls); number("dbSeen",gRootDiagnostic.dbSeen); + number("isSeen",gRootDiagnostic.isSeen); + check(hostNop()==kIOReturnNotReady && gRootDiagnostic.command==1,"quarantine resubmitted"); + puts("PASS rejected bad DMA without resubmission\n"); finish(0); +#endif + check(result==0,"NOP"); puts("PASS NOP\n"); + check(*reinterpret_cast(gHost.virt+20)==uint32_t((physical+kUcdOffset)>>32),"UTRD upper PA"); + uint32_t power=0; check(hostQueryRead(3,2,&power)==0,"power query"); number("power",power); + uint8_t tur[6]={}; uint32_t scsi; + for (unsigned i=0;i<3;++i) { + result=hostScsi(0,tur,6,false,0,200000,&scsi); + if (!result) break; + check(result==kIOReturnNotReady && (scsi==2 || scsi==0x102),"TUR check condition"); + } + check(result==0,"TUR ready"); + const unsigned blockCounts[]={1,32}; + for (unsigned blocks: blockCounts) { + check(hostRead4k(0,8,blocks,output)==0,"READ10"); + for (unsigned i=0;i>8)*31)^0xa5),"read payload"); + check(*reinterpret_cast(gHost.virt+kUcdOffset+kPrdtOffset+4)==uint32_t((physical+kDataOffset)>>32),"PRDT upper PA"); + } + puts("PASS READ4K READ128K\n"); + for (unsigned i=0;i<4096;++i) output[i]=0xa5; + check(hostRead4k(0,0x10000,1,output)!=0,"bad LBA accepted"); + for (unsigned i=0;i<4096;++i) check(output[i]==0xa5,"bad LBA copied stale data"); + puts("PASS bad-LBA-no-copy\nPASS ALL\n"); finish(0); +} diff --git a/drivers/NativeUfs/tests/RootCommandsBridge.cpp b/drivers/NativeUfs/tests/RootCommandsBridge.cpp new file mode 100644 index 0000000..46945f3 --- /dev/null +++ b/drivers/NativeUfs/tests/RootCommandsBridge.cpp @@ -0,0 +1,67 @@ +// SPDX-License-Identifier: BSD-2-Clause-Patent +// Compile the SAME command implementation as the kernel. Test-only callbacks +// replace physical I/O and synchronization, not descriptor or response logic. +#include +#include +#include +#include +#include "../RootDiagnostics.h" +#include "../TransferPoll.h" +using IOReturn = uint32_t; +constexpr IOReturn kIOReturnSuccess=0, kIOReturnError=0xe00002bc, + kIOReturnBadArgument=0xe00002c2, kIOReturnIOError=0xe00002ca, + kIOReturnBusy=0xe00002d5, kIOReturnTimeout=0xe00002d6, kIOReturnNotReady=0xe00002d8; +#include "../RootTransportLayout.h" +constexpr unsigned kMicrosecondScale=1; +constexpr unsigned kMillisecondScale=1000; +static UfsRootDiagnostics gRootDiagnostic; +struct Host { uint8_t *virt; uint64_t phys; bool ready, poisoned; uintptr_t hc; void *lock; }; +static Host gHost; +alignas(4096) static uint8_t memory[kDmaBytes]; +using Callback = uint64_t (*)(unsigned operation, uint64_t address, uint64_t value, void *bytes); +static Callback callback; +// 0: MMIO read; 1: ordered MMIO write; 2: release DMA; 3: acquire DMA; +// 4: monotonic microseconds; 5: delay microseconds. Callbacks must not throw. +static uint32_t read32(uintptr_t, unsigned offset) { return uint32_t(callback(0,offset,0,nullptr)); } +static void hostWrite(unsigned offset, uint32_t value) { callback(1,offset,value,nullptr); } +static void cacheCivac(const void *start, size_t length) { + callback(2,gHost.phys+uint64_t(static_cast(start)-memory),length,const_cast(start)); +} +static void cacheInvalidate(const void *start, size_t length) { + callback(3,gHost.phys+uint64_t(static_cast(start)-memory),length,const_cast(start)); +} +static void clock_interval_to_deadline(uint32_t interval, unsigned scale, uint64_t *deadline) { + *deadline=callback(4,0,0,nullptr)+uint64_t(interval)*scale; +} +static uint64_t mach_absolute_time() { return callback(4,0,0,nullptr); } +static void IODelay(uint32_t us) { callback(5,0,us,nullptr); } +static void IOLockLock(void *) {} +static void IOLockUnlock(void *) {} +static IOReturn recordTransport(IOReturn result,uint32_t ocs) { + gRootDiagnostic.result=result; gRootDiagnostic.ocs=ocs; return result; +} +#include "../RootCommands.h" +#include "../RootQueue.h" +extern "C" { +void ufs_test_reset(Callback cb, uint64_t physical) { + callback=cb; memset(memory,0,sizeof(memory)); + gHost={memory,physical,true,false,0,nullptr}; gRootDiagnostic=UfsRootDiagnostics{}; + gStoreOpen=false; gStoreFirst=gStoreBlocks=0; +} +uint8_t *ufs_test_memory() { return memory; } +bool ufs_test_poisoned() { return gHost.poisoned; } +bool ufs_test_stop_queue() { const char *why=nullptr; return hostStopQueue(&why); } +bool ufs_test_start_queue() { const char *why=nullptr; return hostStartQueue(&why); } +uint32_t ufs_test_query(uint8_t op,uint8_t idn,uint32_t *value) { return hostQueryRead(op,idn,value); } +uint32_t ufs_test_scsi(uint8_t lun,const uint8_t *cdb,unsigned length,bool read,uint32_t bytes,uint32_t *scsi) { + return hostScsi(lun,cdb,length,read,bytes,200000,scsi); +} +uint32_t ufs_test_read(uint8_t lun,uint64_t lba,uint32_t blocks,void *out) { return hostRead4k(lun,lba,blocks,out); } +uint32_t ufs_test_store_open(uint64_t first,uint64_t blocks) { return hostStoreOpen(first,blocks) ? 1 : 0; } +uint32_t ufs_test_write(uint8_t lun,uint64_t lba,uint32_t blocks,const void *src) { return hostWrite4k(lun,lba,blocks,src); } +uint32_t ufs_test_sync() { return hostSyncCache(); } +uint32_t ufs_test_nop() { + return hostNop(); +} +int ufs_test_report(char *out,size_t size) { return formatUfsRootDiagnostics(out,size,gRootDiagnostic,"host harness"); } +} diff --git a/drivers/NativeUfs/tests/RootDiagnostics.cpp b/drivers/NativeUfs/tests/RootDiagnostics.cpp new file mode 100644 index 0000000..126624a --- /dev/null +++ b/drivers/NativeUfs/tests/RootDiagnostics.cpp @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: BSD-2-Clause-Patent +#include +#include +#include +#include +#include +#include "../RootDiagnostics.h" + +int main(int argc, char **) +{ + UfsRootDiagnostics d; + d.stage = "NOP OUT"; + d.dma = 0x8ca774000ULL; + d.list = 0x8ca774000ULL; + d.result = 0xe00002d6; + d.ocs = 15; + d.smr = 0x80000060; + d.s2cr = 127; + d.cbar = 0x10000; + d.command = 1; + d.far = 0x8cb5e9000ULL; + d.contiguous = 139264; + d.polls = 100001; + d.uic[4] = 0x80000001; + d.protocol = 6; + d.residual = 4096; + d.sense = 0x062900; + d.publicationPhase = 5; + d.publicationWindow = 1; + char full[2500]; + int length = formatUfsRootDiagnostics(full, sizeof(full), d, "transport timeout fixture"); + assert(length > 0 && length < 2000 && size_t(length) == strlen(full)); + assert(strstr(full, "DMA=8ca774000 UTRL=8ca774000")); + assert(strstr(full, "result=e00002d6")); + assert(strstr(full, "ocs=0f")); + assert(strstr(full, "verify=ffffffff")); // Not sampled must not look successful. + assert(strstr(full, "far=8cb5e9000")); + assert(strstr(full, "polls=100001")); + assert(strstr(full, "dme=80000001")); + assert(strstr(full, "reject=6 residual=4096 sense=062900")); + assert(strstr(full, "pubPhase=5 window=1")); + char passive[2500]; + formatUfsRootDiagnostics(passive,sizeof(passive),d,"cached values; not a new HW fault sample","UFS CACHED SNAPSHOT"); + assert(strstr(passive,"UFS CACHED SNAPSHOT:") == passive && !strstr(passive,"UFS ROOT FAILURE")); + assert(strstr(passive,"far=8cb5e9000")); + // Every truncation boundary, including zero and a one-byte destination. + for (size_t size = 0; size <= sizeof(full); ++size) { + unsigned char storage[2502]; + memset(storage, 0xa5, sizeof(storage)); + int result = formatUfsRootDiagnostics(reinterpret_cast(storage + 1), size, d, + "transport timeout fixture"); + assert(result == length && storage[0] == 0xa5 && storage[size + 1] == 0xa5); + if (size) assert(storage[1 + (size <= size_t(length) ? size - 1 : size_t(length))] == 0); + } + if (argc > 1) { + // Maximum-width numeric fields plus all eight bounded media entries. + // The stage pointer is the only noninteger field; preserve it. + memset(reinterpret_cast(&d) + offsetof(UfsRootDiagnostics,dma), 0xff, + sizeof(d) - offsetof(UfsRootDiagnostics,dma)); + d.stage = "media publication"; + int used = formatUfsRootDiagnostics(full,sizeof(full),d,"SystemOS signed volume not found"); + used += snprintf(full+used,sizeof(full)-used,"Waiting for signed UUID FFFFFFFF-FFFF-FFFF-FFFF-FFFFFFFFFFFF\n"); + for (unsigned i=0;i<8;++i) { + assert(used < int(sizeof(full))-110); + used += snprintf(full+used,sizeof(full)-used, + "media abcdefghijklmnopqrstuvwx bytes=ffffffffffffffff uuid=FFFFFFFF-FFFF-FFFF-FFFF-FFFFFFFFFFFF\n"); + } + assert(used < int(sizeof(full)) && size_t(used)==strlen(full)); + } + fputs(full, stdout); +} diff --git a/drivers/NativeUfs/tests/RootLayout.cpp b/drivers/NativeUfs/tests/RootLayout.cpp new file mode 100644 index 0000000..fde4ba0 --- /dev/null +++ b/drivers/NativeUfs/tests/RootLayout.cpp @@ -0,0 +1,107 @@ +// SPDX-License-Identifier: BSD-2-Clause-Patent +#define OPENSSL_SUPPRESS_DEPRECATED +#include +#include +#include +#include +#include +#include +#include +#include "../RootDiagnostics.h" +using IOReturn=uint32_t; +constexpr IOReturn kIOReturnSuccess=0; +constexpr unsigned kDataMax=131072; +static UfsRootDiagnostics gRootDiagnostic; +static uint8_t header[4096],entries[16384],manifest[4096]; +static uint64_t userdata=8; +static unsigned reads; +static bool failRead; +static IOReturn hostRead4k(uint8_t,uint64_t lba,uint32_t blocks,void *out) { + ++reads; if(failRead) return 1; + if(lba==1 && blocks==1) memcpy(out,header,4096); + else if(lba==2 && blocks<=4) memcpy(out,entries,blocks*4096); + else if(lba==userdata && blocks==1) memcpy(out,manifest,4096); + else { assert(false); } + return 0; +} +#include "../RootLayout.h" +static void put32(uint8_t *p,uint32_t v) { for(unsigned i=0;i<4;++i)p[i]=uint8_t(v>>(i*8)); } +static void put64(uint8_t *p,uint64_t v) { for(unsigned i=0;i<8;++i)p[i]=uint8_t(v>>(i*8)); } +static void checksums() { + SHA256(manifest,4064,manifest+4064); + put32(header+88,rootCrc32(entries,32*128)); + put32(header+16,rootCrc32(header,92,true)); +} +static void fixture() { + memset(header,0,sizeof(header)); memset(entries,0,sizeof(entries)); memset(manifest,0,sizeof(manifest)); + reads=0; failRead=false; userdata=8; + memcpy(header,"EFI PART",8); put32(header+8,0x10000);put32(header+12,92); + put64(header+24,1);put64(header+32,511);put64(header+40,6);put64(header+48,506); + put64(header+72,2);put32(header+80,32);put32(header+84,128); + entries[0]=1; entries[16]=2;put64(entries+32,8);put64(entries+40,506); + for(unsigned i=0;i<8;++i)entries[56+2*i]="userdata"[i]; + memcpy(manifest,"SHNGROOT",8);put32(manifest+8,1);put32(manifest+12,2);put64(manifest+16,65536); + for(unsigned i=0;i<2;++i) { + uint8_t *e=manifest+24+i*128;strcpy(reinterpret_cast(e),i?"SystemOS":"OS"); + put64(e+32,4096+i*16384);put64(e+40,8192+512);put32(e+48,512);put32(e+52,1); + } + checksums(); +} +static void expect(const std::function &change,bool gpt,uint32_t status,bool rehash=true) { + fixture();change();if(rehash)checksums();RootLayout layout;const char *why=nullptr; + bool result=findRootLayout(&layout,&why); + assert(result==gpt);if(gpt)assert(layout.manifestStatus==status); + if(!result || status)assert(layout.windows==0); +} +int main(int argc,char **argv) { + assert(rootCrc32(reinterpret_cast("123456789"),9)==0xcbf43926); + expect([]{},true,0); + expect([]{header[16]^=1;},false,0,false); + expect([]{entries[16]^=1;},false,0,false); + expect([]{put32(header+12,4097);},false,0); + expect([]{put64(header+24,2);},false,0); + expect([]{put32(header+20,1);},false,0); + expect([]{put32(header+80,129);},false,0); + expect([]{put64(header+32,UINT64_MAX);},false,0); + expect([]{put64(header+40,2);},false,0); + expect([]{put64(header+48,511);},false,0); + expect([]{put64(entries+32,1);},false,0); + expect([]{put64(entries+40,UINT64_MAX);},false,0); + expect([]{entries[0]=0;},false,0); + expect([]{memcpy(entries+128,entries,128);},false,0); + expect([]{memcpy(entries+128,entries,128);put64(entries+40,100);put64(entries+128+32,101);},false,0); + expect([]{failRead=true;},false,0); + expect([]{manifest[0]^=1;},true,1); + expect([]{put32(manifest+12,5);},true,2); + expect([]{manifest[4064]^=1;},true,3,false); + expect([]{put64(manifest+16,UINT64_MAX);},true,4); + expect([]{put64(manifest+24+32,0);},true,4); + expect([]{put64(manifest+24+40,UINT64_MAX-511);},true,4); + expect([]{put64(manifest+24+40,512+1);},true,4); + expect([]{put64(manifest+24+128+32,8192);},true,6); + expect([]{strcpy(reinterpret_cast(manifest+24+128),"OS");},true,5); + expect([]{memset(manifest+24,'A',32);},true,5); + expect([]{manifest[24]='\n';},true,5); + expect([]{manifest[24]=0;},true,5); + expect([]{put32(manifest+24+48,4096);},true,7); + expect([]{put32(manifest+24+52,0);},true,7); + // A final physical 4K read may include padding, but cannot enter another window. + expect([]{put64(manifest+24+40,4096+512);put64(manifest+24+128+32,8192);},true,6); + // Bounded mutation pass checks parser memory safety, not semantic correctness. + for(unsigned i=0;i<2048;++i) { + fixture(); unsigned at=(i*313)%4064;manifest[at]^=uint8_t(1+(i%255));checksums(); + RootLayout layout;const char *why=nullptr;(void)findRootLayout(&layout,&why); + } + if(argc==2) { + fixture();std::ifstream f(argv[1],std::ios::binary);f.read(reinterpret_cast(manifest),4096);assert(f.gcount()==4096); + userdata=0x2f3840;put64(entries+32,userdata);put64(entries+40,0x3d0abfa); + put64(header+32,0x3d0abff);put64(header+48,0x3d0abfa); + // Only the synthetic GPT is rechecksummed. Preserve the actual on-host manifest hash. + put32(header+88,rootCrc32(entries,4096));put32(header+16,rootCrc32(header,92,true)); + RootLayout layout;const char *why=nullptr; + assert(findRootLayout(&layout,&why) && layout.manifestStatus==0 && layout.windows==2); + printf("PASS: existing root image's 4096-byte manifest, OS=%llu SystemOS=%llu; synthetic GPT uses recorded geometry, no full-image rehash\n", + (unsigned long long)layout.window[0].size,(unsigned long long)layout.window[1].size); + } + puts("PASS: GPT CRC/geometry/partition faults, manifest hashes/names/attributes/overlap/overflow, 2048 bounded mutations"); +} diff --git a/drivers/NativeUfs/tests/RootPathBridge.cpp b/drivers/NativeUfs/tests/RootPathBridge.cpp new file mode 100644 index 0000000..5a42c68 --- /dev/null +++ b/drivers/NativeUfs/tests/RootPathBridge.cpp @@ -0,0 +1,55 @@ +// SPDX-License-Identifier: BSD-2-Clause-Patent +// Same production layout/window/command code; OpenSSL replaces kernel crypto, +// qtest replaces hardware primitives. No fake verifier or signature bypass. +#define OPENSSL_SUPPRESS_DEPRECATED +#include +#include +#include +#include "verified_root.h" +#include "RootCommandsBridge.cpp" +constexpr IOReturn kIOReturnNotPrivileged=0xe00002c1; +#include "../RootLayout.h" +#include "../RootWindowRead.h" +extern "C" int vr_host_open(const uint8_t *,size_t,const uint8_t *,uint64_t,vr_view *); +extern "C" int vr_host_block(const vr_view *,uint64_t,const void *,size_t); +static RootLayout layout; +static std::vector envelope; +static vr_view verified; +static std::mutex windowLock; +static uint8_t bounce[kDataMax]; +static unsigned selected; +extern "C" { +uint32_t ufs_path_layout() { + const char *why=nullptr; + return findRootLayout(&layout,&why) ? layout.manifestStatus : 0x100; +} +uint64_t ufs_path_window(unsigned index,unsigned field) { + if(index>=layout.windows) return 0; + return field==0 ? layout.window[index].offset : layout.window[index].size; +} +uint32_t ufs_path_open(unsigned index,const uint8_t *manifest,size_t bytes,const uint8_t *key) { + verified={}; envelope.clear(); + if(layout.manifestStatus || index>=layout.windows) return 0x100; + envelope.assign(manifest,manifest+bytes); + int result=vr_host_open(envelope.data(),envelope.size(),key,1,&verified); + if(result || verified.image_bytes!=layout.window[index].size) { verified={}; return result?result:0x101; } + selected=index; return 0; +} +uint32_t ufs_path_read(uint64_t block,uint64_t count,uint8_t *out,uint64_t capacity,uint64_t *done) { + *done=0; + if(!verified.hashes || count>capacity/512) return kIOReturnBadArgument; + struct Operations { + uint8_t *out; + void lock() { windowLock.lock(); } + void unlock() { windowLock.unlock(); } + IOReturn read(uint64_t lba,uint32_t blocks,void *data) { return hostRead4k(0,lba,blocks,data); } + IOReturn verify(uint64_t index,const void *data,size_t length) { + return vr_host_block(&verified,index,data,length)==VR_OK?kIOReturnSuccess:kIOReturnNotPrivileged; + } + uint64_t copy(uint64_t offset,const void *data,uint64_t length) { memcpy(out+offset,data,length); return length; } + } ops{out}; + auto result=ufsReadWindow(ops,layout.userdataFirst*4096+layout.window[selected].offset, + layout.window[selected].size,block,count,bounce); + *done=result.done; return result.status; +} +} diff --git a/drivers/NativeUfs/tests/RootPublication.cpp b/drivers/NativeUfs/tests/RootPublication.cpp new file mode 100644 index 0000000..611d3a9 --- /dev/null +++ b/drivers/NativeUfs/tests/RootPublication.cpp @@ -0,0 +1,86 @@ +// SPDX-License-Identifier: BSD-2-Clause-Patent +#include +#include +#include +#include +#include "../RootPublication.h" +struct Operations { + bool allocated[4]={}, attached[4]={}, started[4]={}; + int failPrepare=-1, failStep=-1, failStart=-1, failWait=-1; + bool valid=true, scheduleOk=true; + unsigned schedules=0; + std::vector startOrder, waitOrder, discarded; + bool prepare(unsigned index) { + // Distinct failures before allocation, after allocation, and after attach. + if(int(index)==failPrepare && failStep==0) return false; + allocated[index]=true; + if(int(index)==failPrepare && failStep==1) return false; + attached[index]=true; + return !(int(index)==failPrepare && failStep==2); + } + bool validate() { return valid; } + bool schedule() { ++schedules; return scheduleOk; } + void discard(unsigned index) { + assert(!started[index]); + allocated[index]=attached[index]=false;discarded.push_back(index); + } + bool start(unsigned index) { + assert(allocated[index] && attached[index] && !started[index]); + startOrder.push_back(index); + if(int(index)==failStart) return false; + started[index]=true;return true; + } + bool wait(unsigned index) { assert(started[index]);waitOrder.push_back(index);return int(index)!=failWait; } + bool empty() const { for(bool x:allocated)if(x)return false;return true; } +}; +struct Iterator { + std::vector entries; + size_t position=0; + bool valid=true, invalidateOnNext=false; + bool isValid() {return valid;} + int *getNextObject() {if(invalidateOnNext)valid=false;return position{2,1,3,0}) && ops.startOrder==ops.waitOrder && ops.discarded.empty());} + // No diagnostic flag enters the coordinator: reporting cannot relax gates. + for(unsigned position=0;position<4;++position)for(bool waitFailure:{false,true}) { + Operations ops;assert(ufsPrepareRootWindows(ops,4).phase==UfsRootPhase::Complete); + unsigned index=plan.order[position];if(waitFailure)ops.failWait=index;else ops.failStart=index; + auto r=ufsPublishRootWindows(ops,plan); + assert(r.index==index && r.phase==(waitFailure?UfsRootPhase::Media:UfsRootPhase::Start)); + assert(ops.startOrder.size()==position+1 && r.published==position+unsigned(waitFailure)); + for(unsigned p=0;p<4;++p) { + unsigned i=plan.order[p];bool keep=p +#include +#include +#include +#include +#include +#include +#include +using IOReturn = uint32_t; +constexpr IOReturn kIOReturnSuccess=0, kIOReturnBadArgument=1, kIOReturnIOError=2, kIOReturnNotPrivileged=3; +constexpr uint64_t kDataMax=131072; +#include "../RootWindowRead.h" +struct Shared { + std::mutex window, control; + std::condition_variable changed; + unsigned attempts=0; + bool concurrent=false; + uint8_t bounce[kDataMax]; +}; +struct Ops { + Shared &shared; + std::vector disk, output; + bool held=false, parked=false; + unsigned reads=0, copies=0, checks=0, badRead=0, badVerify=0; + uint64_t shortCopy=UINT64_MAX; + Ops(Shared &s, unsigned blocks=70) : shared(s), disk(blocks*4096), output(disk.size(),0xa5) { + for (size_t i=0;i g(shared.control); ++shared.attempts; shared.changed.notify_all(); } + shared.window.lock(); held=true; + } + void unlock() { assert(held); held=false; shared.window.unlock(); } + IOReturn read(uint64_t lba,uint32_t blocks,void *out) { + assert(held && blocks<=32); ++reads; + if(reads==badRead) return kIOReturnIOError; + assert(lba*4096+blocks*4096<=disk.size()); + memcpy(out,disk.data()+lba*4096,blocks*4096); return 0; + } + IOReturn verify(uint64_t index,const void *data,size_t bytes) { + assert(held); ++checks; + if(shared.concurrent && !parked) { + parked=true; + std::unique_lock guard(shared.control); + shared.changed.wait(guard,[&]{return shared.attempts>=2;}); + } + if(checks==badVerify) return kIOReturnNotPrivileged; + assert(index*4096+bytes<=disk.size()); + assert(!memcmp(data,disk.data()+index*4096,bytes)); return 0; + } + uint64_t copy(uint64_t offset,const void *data,uint64_t bytes) { + assert(held); ++copies; + uint64_t n=bytes guard(s.control); s.changed.wait(guard,[&]{return s.attempts==1;}); } + std::thread two([&]{ b=ufsReadWindow(second,0,second.disk.size(),8,8,s.bounce); }); + one.join(); two.join(); + assert(a.status==0 && b.status==0); + assert(!memcmp(first.output.data(),first.disk.data(),4096)); + assert(!memcmp(second.output.data(),second.disk.data()+4096,4096)); + // The wrapper can invoke a reentrant completion now: guard has released. + assert(s.window.try_lock()); s.window.unlock(); + puts("PASS: window geometry, 512/4K/128K boundaries, tail, no unverified copy, partial progress, concurrent snapshot lock, completion outside lock"); +} diff --git a/drivers/NativeUfs/tests/SmmuTakeover.cpp b/drivers/NativeUfs/tests/SmmuTakeover.cpp new file mode 100644 index 0000000..2ca43ad --- /dev/null +++ b/drivers/NativeUfs/tests/SmmuTakeover.cpp @@ -0,0 +1,81 @@ +// SPDX-License-Identifier: BSD-2-Clause-Patent +#include +#include +#include +#include +#include +#include +#include "../SmmuTakeover.h" +int main() +{ + std::array r{}; + auto reset=[&] {r.fill(0); r[0x20/4]=0x40000080; r[0x24/4]=0x60000046; r[0x28/4]=0x5111;}; + auto plan=[&] {return planUfsSmmu(0x60,0x8cb5e9000ULL,0x22000,[&](unsigned at){return r.at(at/4);});}; + reset(); auto p=plan(); assert(!p.error && !p.reuse && p.slot==127 && p.bank==69); + r[(0x800+4*110)/4]=0x80000060; r[(0xc00+4*110)/4]=69; + r[(0x1000+4*69)/4]=0x10000; + p=plan(); assert(!p.error && p.reuse && p.slot==110); + r[(0x800+4*111)/4]=0x80000060; r[(0xc00+4*111)/4]=69; assert(plan().error); + reset(); r[0x800/4]=0x80000061; r[0xc00/4]=69; assert(plan().error); + reset(); r[0x800/4]=0x80000060; r[0xc00/4]=1; assert(plan().error); + reset(); r[0x800/4]=0x80010060; r[0xc00/4]=69; assert(plan().error); + reset(); r[0x800/4]=0x80008060; r[0xc00/4]=1; assert(!plan().error); // Distinct 16-bit SID. + reset(); r[(128+69)*4096/4]=1; assert(plan().error); + reset(); r[0x24/4]|=0x80000000; assert(plan().error); // Never guess 64K register geometry. + reset(); for(unsigned i=0;i<128;++i) {r[(0x800+4*i)/4]=0x80000100+i;r[(0xc00+4*i)/4]=1;} + assert(plan().error); + // The reported DMA range fits every 36-bit IDR2 limit, but the previous + // CBA2R=0 setup faults on its first byte (ARM IHI0062D.c table 3-1). + reset(); p=plan(); assert(!p.error && p.pages==128); + auto read=[&](unsigned at){return r.at(at/4);}; + std::vector> writes; + unsigned drop=0xffffffffU; + auto write=[&](unsigned at,uint32_t value) { + writes.emplace_back(at,value); + if(at!=drop) r.at(at/4)=value; + if(at==0x800+4*p.slot) { + assert(read(0x1800+4*p.bank)==1); + assert(read(0x1000+4*p.bank)==0x10000); + assert(read((p.pages+p.bank)*4096)==0); + } + }; + auto configure=[&]{return configureUfsSmmu(p,0x60,read,write);}; + auto inputFault=[&](uint64_t address) { + // Independent bypass-only architecture rule, not a hardware emulator. + return !(read(0x1800+4*p.bank)&1) && (address>>32); + }; + assert(inputFault(0x8cb5e9000ULL)); + assert(!inputFault(0xffffffffULL) && inputFault(0x100000000ULL)); + assert(!configure() && !inputFault(0x8cb5e9000ULL)); + assert(!inputFault(0x8cb60afffULL)); // Final byte of the same allocation. + assert(writes.size()==5); + p=plan(); writes.clear(); assert(p.reuse && !configure() && writes.empty()); + r[(0x1800+4*p.bank)/4]=0; + assert(!configure() && writes.size()==1 && writes[0].first==0x1800+4*p.bank); + // A refused context-format write must stop before stream publication. + for(unsigned offset : {0x1800+4*69, 0x1000+4*69, 0xc00+4*127, 0x800+4*127}) { + reset(); p=plan(); writes.clear(); drop=offset; + assert(configure()); + if(offset!=0x800+4*127) assert(read(0x800+4*127)==0); + } + reset(); p=plan(); r[(128+69)*4096/4]=2; drop=(128+69)*4096; + assert(configure()); // An ignored disable write cannot publish a stream. + assert(read(0x800+4*p.slot)==0); + // Full extent, overflow and each independent hardware address width. + for(unsigned shift : {0U,4U,8U}) { + reset(); r[0x28/4] &= ~(15U<{0x8cb5e9000ULL,0}, + {0xfffffffffULL,2}, {0xfffffffffffff000ULL,0x22000}, {0x1000000000ULL,1}}) + assert(planUfsSmmu(0x60,range.first,range.second,read).error); + assert(!planUfsSmmu(0x60,0xfffffffffULL,1,read).error); + r[0x28/4]&=~(1U<<12); assert(plan().error); + reset(); r[0x20/4]&=~(1U<<30); assert(plan().error); + // Reused context with an unexpected VMID/attribute must not be repaired. + reset(); p=plan(); drop=0xffffffffU; assert(!configure()); + r[(0x1800+4*p.bank)/4]=0x10001; p=plan(); assert(p.error); + writes.clear(); assert(configure() && writes.empty()); + puts("PASS: SMMU geometry/conflicts, VA64 high-DMA regression, full-range width limits, ordered configuration, reuse/repair and blocked-write refusal"); +} diff --git a/drivers/NativeUfs/tests/TransferPoll.cpp b/drivers/NativeUfs/tests/TransferPoll.cpp new file mode 100644 index 0000000..3005110 --- /dev/null +++ b/drivers/NativeUfs/tests/TransferPoll.cpp @@ -0,0 +1,58 @@ +// SPDX-License-Identifier: BSD-2-Clause-Patent +#include +#include +#include +#include +#include "../TransferPoll.h" + +struct Sample { uint32_t db, ocs, is; }; +struct FakePort { + std::vector samples; + unsigned index = 0, reads = 0; + uint32_t doorbell() { return samples[index].db; } + uint32_t interrupt() { return samples[index].is; } + uint32_t completedOcs() { + assert(!(samples[index].db & 1)); // Never inspect a descriptor still owned by HC. + ++reads; + return samples[index].ocs; + } + bool expired() { return index + 1 == samples.size(); } + void delay() { assert(!expired()); ++index; } +}; + +int main() +{ + // Exact IMG_2971 observation must not be treated as a completed command. + FakePort noAcceptance{{{0, 15, 0}, {0, 15, 0}, {0, 15, 0}}}; + UfsTransferPoll a; + assert(pollUfsTransfer(noAcceptance, a) == UfsTransferEnd::Timeout); + assert(a.polls == 3 && a.ocs == 15 && a.doorbellSeen == 0); + + FakePort delayed{{{0, 15, 0}, {1, 15, 0}, {0, 15, 0}, {0, 0, 1}}}; + UfsTransferPoll b; + assert(pollUfsTransfer(delayed, b) == UfsTransferEnd::Completed); + assert(b.polls == 4 && b.ocs == 0 && b.doorbellSeen == 1 && delayed.reads == 3); + + FakePort immediate{{{0, 0, 0}}}; + UfsTransferPoll c; + assert(pollUfsTransfer(immediate, c) == UfsTransferEnd::Completed); + assert(c.doorbellSeen == 0); // Seeing DB=1 must not be mandatory. + + FakePort failedCommand{{{1, 15, 0}, {0, 4, 1}}}; + UfsTransferPoll d; + assert(pollUfsTransfer(failedCommand, d) == UfsTransferEnd::Completed && d.ocs == 4); + + FakePort stuck{{{1, 15, 0}, {1, 15, 0}}}; + UfsTransferPoll e; + assert(pollUfsTransfer(stuck, e) == UfsTransferEnd::Timeout && stuck.reads == 0); + + for (unsigned bit = 0; bit < 32; ++bit) { + uint32_t mask = uint32_t(1) << bit; + if (!(mask & kUfsFatalStatus)) continue; + FakePort fatal{{{0, 0, mask}}}; + UfsTransferPoll f; + assert(pollUfsTransfer(fatal, f) == UfsTransferEnd::Fatal); + assert(fatal.reads == 0 && f.interruptSeen == mask); // Fatal wins over OCS=0. + } + puts("PASS: unaccepted, delayed, immediate, failed, stuck and fatal UFS completions"); +} diff --git a/drivers/NativeUfs/tests/check_arm64_root_commands.py b/drivers/NativeUfs/tests/check_arm64_root_commands.py new file mode 100644 index 0000000..decfc1b --- /dev/null +++ b/drivers/NativeUfs/tests/check_arm64_root_commands.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +"""Bounded EL1 TCG execution of production commands and cache/MMIO primitives. + +Only a synthetic disk and QEMU virt addresses. No device access. QEMU does +not model SM8550 cache coherency, its SMMU or its bus/power/timing behavior. +Generated files and raw logs belong in an ignored output directory. +""" +import argparse +import hashlib +import json +from pathlib import Path +import re +import shutil +import subprocess + + +def extract(text, name): + matches=list(re.finditer(r'^static (?:uint32_t|void) '+name+r' \(',text,re.M)) + assert len(matches)==1, f'ambiguous/missing production function {name}' + start=matches[0].start(); brace=text.index('{',start); depth=1; end=brace+1 + while depth: + depth+=(text[end]=='{')-(text[end]=='}'); end+=1 + return text[start:end]+'\n' + + +def main(): + parser=argparse.ArgumentParser(description=__doc__) + parser.add_argument('--output',type=Path,required=True) + parser.add_argument('--qemu',default='qemu-system-aarch64') + parser.add_argument('--clang',default='clang++') + parser.add_argument('--linker',default='ld.lld') + args=parser.parse_args() + out=args.output.resolve(); out.mkdir(parents=True,exist_ok=True) + if shutil.disk_usage(out).free < 5*1024**3+512*1024**2: + raise SystemExit('Keep 5 GiB free plus fixture headroom') + source=Path(__file__).resolve().parent + production=source.parent/'NativeUfs.cpp' + extracted='\n'.join(extract(production.read_text(),name) for name in + ('read32','write32','cacheCivac','cacheInvalidate','hostWrite')) + (out/'RootHardwareExtracted.h').write_text(extracted) + (out/'link.ld').write_text('''ENTRY(_start) +SECTIONS { + . = 0x40200000; + .text : { *(.text.entry) *(.text*) } + .rodata : { *(.rodata*) } + .data : { *(.data*) } + . = ALIGN(4096); + __bss_start = .; + .bss : { *(.bss*) *(COMMON) } + . = ALIGN(8); __bss_end = .; + ASSERT(. < 0x40400000, "fixture exceeds identity mapping budget") +} +''') + (out/'blank-firmware.bin').write_bytes(bytes(4096)) + disk=bytearray(2*1024*1024) + disk[8*4096:8*4096+131072]=bytes(((i*17+(i>>8)*31)^0xa5)&255 for i in range(131072)) + (out/'disk.raw').write_bytes(disk) + compiler=shutil.which(args.clang); linker=shutil.which(args.linker); qemu=shutil.which(args.qemu) + assert compiler and linker and qemu,'Missing cross compiler, linker or QEMU' + common=[compiler,'--target=aarch64-none-elf','-O2','-ffreestanding','-fno-builtin','-nostdinc++', + '-fno-exceptions','-fno-rtti','-fno-stack-protector','-mgeneral-regs-only', + '-fno-strict-aliasing','-Wall','-Wextra','-Werror','-Wno-unused-function', + '-I',str(out)] + subprocess.run([compiler,'--target=aarch64-none-elf','-c',str(source/'RootCommandsArm64.S'), + '-o',str(out/'entry.o')],check=True) + results=[] + for name,physical,bad_dma,sharing in [('low',0x40010000,0,2),('high',0x8cb5e9000,0,2), + ('high-inner-shareable',0x8cb5e9000,0,3), + ('truncated-high',0x8cb5e9000,1,2),('virtual-as-physical',0x8cb5e9000,2,2)]: + directory=out/name; directory.mkdir(exist_ok=True) + compile_command=common+[f'-DTEST_DMA_PHYS={physical:#x}ULL'] + compile_command+=[f'-DTEST_BAD_DMA={bad_dma}',f'-DTEST_SHAREABILITY={sharing}'] + compile_command+=['-c',str(source/'RootCommandsArm64.cpp'),'-o',str(directory/'guest.o')] + subprocess.run(compile_command,check=True) + subprocess.run([linker,'-T',str(out/'link.ld'),str(out/'entry.o'),str(directory/'guest.o'), + '-o',str(directory/'guest.elf')],check=True) + command=[qemu,'-machine','virt,gic-version=3,highmem-ecam=off,acpi=on,virtualization=off,secure=off', + '-cpu','cortex-a710','-accel','tcg','-m','128M,slots=1,maxmem=64G', + '-bios',str(out/'blank-firmware.bin'), + '-object','memory-backend-ram,id=hi,size=256M', + '-device','pc-dimm,memdev=hi,addr=0x8c0000000', + '-device','ufs,id=ufs0,bus=pcie.0,addr=1.0', + '-drive',f'if=none,id=disk,format=raw,readonly=on,file={out / "disk.raw"}', + '-device','ufs-lu,bus=ufs0,drive=disk,lun=0', + '-device',f'loader,file={directory / "guest.elf"},cpu-num=0', + '-nodefaults','-display','none','-serial','stdio','-monitor','none','-no-reboot', + '-semihosting-config','enable=on,target=native'] + (directory/'commands.json').write_text(json.dumps({'compile':compile_command,'qemu':command},indent=2)+'\n') + try: + result=subprocess.run(command,capture_output=True,timeout=15) + log=result.stdout.decode(errors='replace'); errors=result.stderr.decode(errors='replace') + except subprocess.TimeoutExpired as error: + (directory/'serial.log').write_bytes(error.stdout or b'') + (directory/'qemu.log').write_bytes(error.stderr or b'') + raise + (directory/'serial.log').write_text(log); (directory/'qemu.log').write_text(errors) + print(name+'\n'+log,flush=True) + assert result.returncode==0 and 'FAIL' not in log,(name,result.returncode,errors) + assert ('PASS rejected bad DMA without resubmission' if bad_dma else 'PASS ALL') in log,name + results.append({'case':name,'dma_pa':hex(physical),'shareability':sharing,'passed':True, + 'registers':dict(re.findall(r'^(\w+)=(0x[0-9a-f]+)$',log,re.M))}) + files=[production,source/'RootCommandsArm64.cpp',source/'RootCommandsArm64.S',Path(__file__).resolve()] + files += [source.parent/name for name in ('RootCommands.h','RootQueue.h','TransferPoll.h', + 'RootTransportLayout.h','RootDiagnostics.h')] + report={'results':results,'qemu':subprocess.check_output([qemu,'--version'],text=True).splitlines()[0], + 'compiler':subprocess.check_output([compiler,'--version'],text=True).splitlines()[0], + 'source_revision':subprocess.check_output(['git','-C',str(source),'rev-parse','HEAD'],text=True).strip(), + 'source_sha256':{str(f.relative_to(source.parent)):hashlib.sha256(f.read_bytes()).hexdigest() for f in files}, + 'extracted_primitives_sha256':hashlib.sha256(extracted.encode()).hexdigest(), + 'scope':'EL1 TCG, active native 4K tables, cached VA != DMA PA, production ARM64 commands/DC/barriers; ' + 'not XNU allocation/locks, real cache coherency, SM8550 SMMU/bus/timing or hardware acceptance'} + (out/'report.json').write_text(json.dumps(report,indent=2)+'\n') + + +if __name__=='__main__': + main() diff --git a/drivers/NativeUfs/tests/check_legacy_descriptors.py b/drivers/NativeUfs/tests/check_legacy_descriptors.py new file mode 100644 index 0000000..654934e --- /dev/null +++ b/drivers/NativeUfs/tests/check_legacy_descriptors.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +"""Compare actual adoptAndRead builders with current root descriptors offline. + +Extract only the historical builder prefixes, stopping before cache/MMIO. +They are comparison oracles, never a fallback to commandeer firmware RAM. +""" +import argparse +import hashlib +import json +from pathlib import Path +import subprocess + + +def main(): + parser=argparse.ArgumentParser(description=__doc__) + parser.add_argument('--output',required=True,type=Path) + parser.add_argument('--compiler',default='clang++') + args=parser.parse_args() + source=Path(__file__).resolve().parent + out=args.output.resolve(); out.mkdir(parents=True,exist_ok=True) + production=(source.parent/'NativeUfs.cpp').read_text() + legacy=production[production.index('void NativeUfs::adoptAndRead ('):] + prefixes=[] + for name in ('query','command'): + start=legacy.index('auto '+name+' = [&]') + end=legacy.index(' cacheClean (virt, 16384);',start) + prefixes.append(legacy[start:end]+('return 0;' if name=='query' else 'return UfsResult{};')+'\n};\n') + fixture=r''' +#include "RootCommandsBridge.cpp" +#include +static std::array sent; +static uint64_t ticks; +static uint64_t io(unsigned op,uint64_t address,uint64_t,void *) { + if (op==1 && address==0x58) { + memcpy(sent.data(),memory,kDmaBytes); + *reinterpret_cast(memory+8)=0; + uint8_t *c=memory+kUcdOffset,*r=c+kRspOffset; + memcpy(r,c,32); r[0]=c[0]+0x20; + } + if (op==4) return ++ticks; + return 0; +} +static void zero32(void *p,unsigned length) { memset(p,0,length); } +static void be32(uint8_t *p,uint32_t value) { putBe32(p,value); } +struct UfsResult {}; +static unsigned cases; +static void same(const uint8_t *old) { + assert(!memcmp(sent.data(),old,32)); + assert(!memcmp(sent.data()+kUcdOffset,old+kUcdOffset,kPrdtOffset+16)); + ++cases; +} +int main() { + alignas(4096) uint8_t old[kDmaBytes]; + uint8_t *virt=old; + const uint32_t ucdOffset=4096,rspOffset=512,prdtOffset=1024,dataOffset=8192; + for (uint64_t phys: {0x40010000ULL,0x8cb5e9000ULL}) { + // PREFIXES + uint32_t ignored=0; + memset(old,0,sizeof(old)); ufs_test_reset(io,phys); + (void)query(0,0,0,&ignored,&ignored); (void)ufs_test_nop(); same(old); + for (uint8_t opcode: {3,5}) { + memset(old,0,sizeof(old)); ufs_test_reset(io,phys); + (void)query(1,opcode,2,&ignored,&ignored); + (void)ufs_test_query(opcode,2,&ignored); same(old); + } + for (uint8_t lun: {0,0xd0}) { + uint8_t cdb[10]={0x28,0,0,0,0,8,0,0,1,0}; + memset(old,0,sizeof(old)); ufs_test_reset(io,phys); + (void)command(lun,cdb,10,true,4096); + (void)ufs_test_scsi(lun,cdb,10,true,4096,&ignored); same(old); + cdb[0]=0x1b; cdb[3]=0x10; + memset(old,0,sizeof(old)); ufs_test_reset(io,phys); + (void)command(lun,cdb,6,false,0); + (void)ufs_test_scsi(lun,cdb,6,false,0,&ignored); same(old); + } + } + assert(cases==14); printf("PASS %u legacy/current descriptor comparisons\n",cases); +} +'''.replace('// PREFIXES','\n'.join(prefixes)) + (out/'compare.cpp').write_text(fixture) + subprocess.run([args.compiler,'-std=c++17','-O2','-fsanitize=address,undefined', + '-fno-omit-frame-pointer','-I',str(source),str(out/'compare.cpp'), + '-o',str(out/'compare')],check=True) + log=subprocess.check_output([str(out/'compare')],text=True,timeout=10) + (out/'result.log').write_text(log); print(log,end='') + report={'cases':14,'passed':True,'scope':'UTRD 32 bytes and UCD/response/PRDT 1040 bytes before doorbell; ' + 'does not compare allocation, coherence, queue takeover or controller behavior', + 'source_sha256':{name:hashlib.sha256((source.parent/name).read_bytes()).hexdigest() + for name in ('NativeUfs.cpp','RootCommands.h','RootTransportLayout.h','tests/RootCommandsBridge.cpp')}, + 'extracted_prefix_sha256':[hashlib.sha256(p.encode()).hexdigest() for p in prefixes]} + (out/'report.json').write_text(json.dumps(report,indent=2)+'\n') + + +if __name__=='__main__': + main() diff --git a/drivers/NativeUfs/tests/check_qemu_root_commands.py b/drivers/NativeUfs/tests/check_qemu_root_commands.py new file mode 100644 index 0000000..3658580 --- /dev/null +++ b/drivers/NativeUfs/tests/check_qemu_root_commands.py @@ -0,0 +1,202 @@ +#!/usr/bin/env python3 +"""Execute the production root commands against QEMU's generic UFS model. + +No USB, sheng MMIO, target OS, SMMU or guest kernel. Qtest supplies PCI/MMIO +and guest RAM; the compiled production code constructs every UTRD/UPIU/PRDT. +Only a tiny synthetic read-only disk is attached. Output must be outside Git. +""" +import argparse +import ctypes as C +import hashlib +import json +from pathlib import Path +import shutil +import socket +import subprocess +import tempfile +import time + +p = argparse.ArgumentParser(description=__doc__) +p.add_argument('--output', required=True, type=Path) +p.add_argument('--qemu', default='qemu-system-aarch64') +p.add_argument('--verified-root', type=Path, help='VerifiedRoot checkout (default: kits/VerifiedRoot)') +p.add_argument('--openssl-prefix', type=Path, help='OpenSSL 3 prefix (default: brew --prefix openssl@3)') +a = p.parse_args() +out = a.output.resolve(); out.mkdir(parents=True, exist_ok=True) +source = Path(__file__).resolve().parent +import root_path_fixture +verified_root = (a.verified_root or source.parents[2] / 'kits/VerifiedRoot').resolve() +openssl = a.openssl_prefix or Path(subprocess.check_output(['brew','--prefix','openssl@3'],text=True).strip()) +library = root_path_fixture.build_library(source,out,verified_root,openssl) +fixture = root_path_fixture.prepare(out,verified_root) +lib = C.CDLL(str(library)) +Callback = C.CFUNCTYPE(C.c_uint64, C.c_uint, C.c_uint64, C.c_uint64, C.c_void_p) +lib.ufs_test_reset.argtypes = [Callback, C.c_uint64] +lib.ufs_test_nop.restype = C.c_uint32 +lib.ufs_test_query.argtypes = [C.c_uint8, C.c_uint8, C.POINTER(C.c_uint32)] +lib.ufs_test_query.restype = C.c_uint32 +lib.ufs_test_scsi.argtypes = [C.c_uint8,C.c_void_p,C.c_uint,C.c_bool,C.c_uint32,C.POINTER(C.c_uint32)] +lib.ufs_test_scsi.restype = C.c_uint32 +lib.ufs_test_read.argtypes = [C.c_uint8,C.c_uint64,C.c_uint32,C.c_void_p] +lib.ufs_test_read.restype = C.c_uint32 +lib.ufs_test_poisoned.restype = C.c_bool +lib.ufs_test_store_open.argtypes = [C.c_uint64,C.c_uint64] +lib.ufs_test_store_open.restype = C.c_uint32 +lib.ufs_test_write.argtypes = [C.c_uint8,C.c_uint64,C.c_uint32,C.c_void_p] +lib.ufs_test_write.restype = C.c_uint32 +lib.ufs_test_sync.restype = C.c_uint32 +lib.ufs_test_report.argtypes = [C.c_void_p,C.c_size_t] +lib.ufs_test_stop_queue.restype = C.c_bool +lib.ufs_test_start_queue.restype = C.c_bool + +# The stopped CPU never executes this fixture firmware. Its presence enables +# virt's ACPI memory device, allowing 256 MiB at the observed high DMA address +# without reserving tens of GiB of host RAM. +(out / 'blank-firmware.bin').write_bytes(bytes(4096)) +pattern = fixture['pattern'] +# The write test needs a writable LU; the read-only tests never write, so they run on the same image. Original kept for comparison. +shutil.copyfile(out / 'disk.raw', out / 'disk-rw.raw') +qemu = shutil.which(a.qemu) +assert qemu, 'QEMU unavailable' +version = subprocess.check_output([qemu, '--version'], text=True).splitlines()[0] +results = [] +with tempfile.TemporaryDirectory(prefix='ufs-qtest-') as td: + sock = Path(td) / 'io.sock' + command = [qemu, '-machine', 'virt,gic-version=3,highmem-ecam=off,acpi=on', + '-cpu', 'max', '-accel', 'tcg', '-m', '128M,slots=1,maxmem=64G', + '-bios', str(out / 'blank-firmware.bin'), + '-object', 'memory-backend-ram,id=hi,size=256M', + '-device', 'pc-dimm,memdev=hi,addr=0x8c0000000', + '-device', 'ufs,id=ufs0,bus=pcie.0,addr=1.0', + '-drive', f'if=none,id=disk,format=raw,file={out / "disk-rw.raw"}', + '-device', 'ufs-lu,bus=ufs0,drive=disk,lun=0', + '-nodefaults', '-display', 'none', '-S', + '-qtest', f'unix:{sock},server=on,wait=off', '-qtest-log', '/dev/null'] + with (out / 'qemu.log').open('w') as log: + process = subprocess.Popen(command, stdout=log, stderr=log) + try: + deadline = time.monotonic() + 5 + while not sock.exists(): + assert process.poll() is None, (out / 'qemu.log').read_text() + assert time.monotonic() < deadline, 'Qtest socket startup timeout' + time.sleep(.01) + connection = socket.socket(socket.AF_UNIX) + connection.settimeout(3); connection.connect(str(sock)) + channel = connection.makefile('rwb', buffering=0) + + def qtest(command): + connection.sendall((command + '\n').encode()) + while True: + response = channel.readline().decode().strip() + assert response, (out / 'qemu.log').read_text() + if response.startswith('IRQ '): continue + assert response.startswith('OK'), (command, response) + return response[2:].strip() + + def read32(address): return int(qtest(f'readl {address:#x}'), 0) + def write32(address, value): qtest(f'writel {address:#x} {value:#x}') + + # QEMU virt fixture ECAM, slot 1; never a claimed sheng address. + pci = 0x3f008000; hc = 0x10000000 + pci_id = read32(pci) + assert pci_id not in (0, 0xffffffff), 'No UFS PCI device' + write32(pci + 0x10, hc) + write32(pci + 0x14, 0) + qtest(f'writew {pci + 4:#x} 0x6') + assert read32(hc) & 0x01000000, 'Model lacks 64-bit DMA' + write32(hc + 0x34, 1) + write32(hc + 0x90, 0x16) # Fixture link startup, not production takeover. + assert read32(hc + 0x30) & 3 == 3 + cap, ver = read32(hc), read32(hc + 8) + + errors = [] + def io(operation, address, value, ptr): + try: + if errors: return 0xffffffffffffffff + if operation == 0: return read32(hc + address) + if operation == 1: write32(hc + address, value) + if operation == 2: + data = C.string_at(ptr, value) + for at in range(0,value,4096): + part = data[at:at+4096] + qtest(f'write {address+at:#x} {len(part):#x} 0x{part.hex()}') + if operation == 3: + data = b''.join(bytes.fromhex(qtest(f'read {address+at:#x} {min(4096,value-at):#x}').removeprefix('0x')) + for at in range(0,value,4096)) + assert len(data) == value + C.memmove(ptr, data, value) + if operation == 4: return time.monotonic_ns() // 1000 + if operation == 5: time.sleep(value / 1000000) + return 0 + except BaseException as e: + errors.append(repr(e)); return 0xffffffffffffffff + callback = Callback(io) + + for physical in [0x40010000, 0x8cb5e9000]: + lib.ufs_test_reset(callback, physical) + assert lib.ufs_test_stop_queue(), 'Production queue stop failed' + assert lib.ufs_test_start_queue(), 'Production queue start failed' + nop = lib.ufs_test_nop() + if nop: + text = C.create_string_buffer(2500); lib.ufs_test_report(text,2500) + print(text.value.decode(), flush=True) + print('IO errors', errors, flush=True) + assert not errors, errors + print('UTRD', qtest(f'read {physical:#x} 0x20'), 'RSP', + qtest(f'read {physical+4096+512:#x} 0x20'), flush=True) + assert nop == 0, 'NOP failed' + value = C.c_uint32() + assert lib.ufs_test_query(3, 2, C.byref(value)) == 0, 'Power query failed' + # New QEMU LUs may first return UNIT ATTENTION; allow only a + # bounded valid CHECK CONDITION before ordinary readiness. + tur = (C.c_uint8 * 6)(); status = C.c_uint32() + for attempt in range(3): + result = lib.ufs_test_scsi(0, tur, 6, False, 0, C.byref(status)) + if result == 0: break + assert result == 0xe00002d8 and status.value in (2,0x102), (hex(result),hex(status.value),errors) + assert result == 0, 'TUR did not become ready' + for blocks in (1, 32): + buffer = (C.c_uint8 * (blocks * 4096))() + assert lib.ufs_test_read(0, 384, blocks, buffer) == 0, 'READ(10) failed' + assert bytes(buffer) == pattern[:len(buffer)], 'DMA payload mismatch' + path_result = root_path_fixture.check(lib,fixture) + rejected = (C.c_uint8 * 4096)(*([0xa5] * 4096)) + assert lib.ufs_test_read(0, 0x10000, 1, rejected) != 0, 'Invalid LBA accepted' + assert bytes(rejected) == bytes([0xa5]) * 4096, 'Error path copied stale payload' + assert not errors, errors + # Persistent-store window: nothing is writable until it is opened; writes outside it are refused before any I/O. + block = (C.c_uint8 * 4096)(*([0x5a] * 4096)) + assert lib.ufs_test_write(0, 448, 1, block) == 0xe00002c2, 'Write accepted before the window was opened' + assert lib.ufs_test_sync() == 0xe00002d8, 'Sync accepted before the window was opened' + assert lib.ufs_test_store_open(448, 32) == 1 and lib.ufs_test_store_open(448, 32) == 0, 'Window must open exactly once' + for lba, blocks in ((447, 1), (479, 2), (480, 1), (0, 1), (8, 1)): + assert lib.ufs_test_write(0, lba, blocks, (C.c_uint8 * (4096 * blocks))()) == 0xe00002c2, f'Write outside the window accepted at {lba}' + assert lib.ufs_test_write(1, 448, 1, block) == 0xe00002c2, 'Write to another LUN accepted' + wpattern = bytes((i * 7 + physical // 4096 + 3) & 255 for i in range(4096 * 5)) + assert lib.ufs_test_write(0, 449, 5, wpattern) == 0, 'WRITE(10) failed' + assert lib.ufs_test_sync() == 0, 'SYNCHRONIZE CACHE failed' + back = (C.c_uint8 * (4096 * 5))() + assert lib.ufs_test_read(0, 449, 5, back) == 0 and bytes(back) == wpattern, 'Write did not read back' + assert lib.ufs_test_write(0, 479, 1, block) == 0, 'Last block of the window refused' + assert lib.ufs_test_write(0, 448, 1, block) == 0, 'First block of the window refused' + assert lib.ufs_test_sync() == 0 + assert not lib.ufs_test_poisoned() and not errors, errors + results.append({'dma': hex(physical), 'nop': 'pass', 'power_query': value.value, 'store_window': 'pass', + 'root_path': path_result, 'read_4k': 'pass', 'read_128k': 'pass', 'bad_lba_no_copy': 'pass'}) + finally: + process.terminate() + try: process.wait(timeout=5) + except subprocess.TimeoutExpired: process.kill(); process.wait(timeout=5) +original = (out / 'disk.raw').read_bytes(); after = (out / 'disk-rw.raw').read_bytes() +assert len(original) == len(after) +changed = [i for i in range(len(original) // 4096) if original[i*4096:(i+1)*4096] != after[i*4096:(i+1)*4096]] +assert changed and all(448 <= i < 480 for i in changed), f'Blocks outside the window changed: {changed}' +assert after[479*4096:480*4096] == bytes([0x5a]) * 4096 and after[448*4096:449*4096] == bytes([0x5a]) * 4096 +results.append({'image_after': 'only blocks %d..%d of the window changed' % (changed[0], changed[-1])}) +report = {'qemu': version, 'pci_id': hex(pci_id), 'cap': hex(cap), 'version': hex(ver), + 'results': results, 'source_sha256': {str(f.relative_to(source.parent)): hashlib.sha256(f.read_bytes()).hexdigest() + for f in [source.parent / 'RootDiagnostics.h', source.parent / 'RootCommands.h', source.parent / 'RootQueue.h', source.parent / 'RootTransportLayout.h', source.parent / 'TransferPoll.h', source / 'RootCommandsBridge.cpp', source / 'RootPathBridge.cpp', source / 'root_path_fixture.py', source.parent / 'RootLayout.h', source.parent / 'RootWindowRead.h']}, + 'verified_root_sha256': {name: hashlib.sha256((verified_root/name).read_bytes()).hexdigest() for name in ('src/verified_root.c','tests/openssl_provider.c','include/verified_root.h')}, + 'scope': 'Production layout/window/command code, actual portable VerifiedRoot with OpenSSL / generic QEMU UFS DMA; not XNU, CPU cache, SM8550, SMMU or hardware timing'} +(out / 'report.json').write_text(json.dumps(report, indent=2) + '\n') +print(json.dumps(report, indent=2)) diff --git a/drivers/NativeUfs/tests/check_root_path_host.py b/drivers/NativeUfs/tests/check_root_path_host.py new file mode 100644 index 0000000..be777fb --- /dev/null +++ b/drivers/NativeUfs/tests/check_root_path_host.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: BSD-2-Clause-Patent +"""Bounded ASan/UBSan checks of production root layout and window reads.""" +import argparse +from pathlib import Path +import subprocess + +parser=argparse.ArgumentParser(description=__doc__) +parser.add_argument('--output',required=True,type=Path) +parser.add_argument('--openssl-prefix',type=Path) +parser.add_argument('--root-image',type=Path,help='Optional existing image: read only its first 4096 bytes') +args=parser.parse_args() +source=Path(__file__).resolve().parent +out=args.output.resolve();out.mkdir(parents=True,exist_ok=True) +openssl=args.openssl_prefix or Path(subprocess.check_output(['brew','--prefix','openssl@3'],text=True).strip()) +for name in ('RootLayout','RootWindowRead','RootDiagnostics','RootPublication'): + binary=out/name + command=['clang++','-std=c++17','-Wall','-Wextra','-Werror','-fsanitize=address,undefined','-g',str(source/(name+'.cpp')),'-o',str(binary)] + if name=='RootLayout':command+=['-I',str(openssl/'include'),'-L',str(openssl/'lib'),'-lcrypto'] + subprocess.run(command,check=True,timeout=60) + command=[str(binary)] + if name=='RootLayout' and args.root_image:command.append(str(args.root_image.resolve())) + with (out/(name+'.log')).open('w') as log:subprocess.run(command,check=True,timeout=20,stdout=log,stderr=subprocess.STDOUT) + print('PASS:',name) +with (out/'media-diagnostics.txt').open('w') as log: + subprocess.run([str(out/'RootDiagnostics'),'--media'],check=True,timeout=20,stdout=log,stderr=subprocess.STDOUT) +print('PASS: maximum-width diagnostics plus eight media entries prepared for actual renderer') diff --git a/drivers/NativeUfs/tests/root_path_fixture.py b/drivers/NativeUfs/tests/root_path_fixture.py new file mode 100644 index 0000000..a7038a0 --- /dev/null +++ b/drivers/NativeUfs/tests/root_path_fixture.py @@ -0,0 +1,114 @@ +# SPDX-License-Identifier: BSD-2-Clause-Patent +"""Small synthetic signed windows for the independent QEMU UFS test. + +Fixture keys stay in the caller's ignored output directory. No target image, +user key, USB, guest CPU execution or root-acceptance claim is involved. +""" +import ctypes as C +import hashlib +import importlib.util +import io +import os +from pathlib import Path +import struct +import subprocess +import uuid +import zlib + + +def build_library(source, out, verified_root, openssl): + objects = [] + for name in ('src/verified_root.c', 'tests/openssl_provider.c'): + target = out / (Path(name).stem + '.o') + subprocess.run(['clang', '-std=c11', '-Wall', '-Wextra', '-Werror', '-fPIC', + '-I', str(verified_root / 'include'), '-I', str(openssl / 'include'), + '-c', str(verified_root / name), '-o', str(target)], check=True) + objects.append(str(target)) + library = out / 'root-path.dylib' + subprocess.run(['clang++', '-std=c++17', '-Wall', '-Wextra', '-Werror', '-dynamiclib', + '-I', str(verified_root / 'include'), '-I', str(openssl / 'include'), + str(source / 'RootPathBridge.cpp'), *objects, '-L', str(openssl / 'lib'), '-lcrypto', + '-o', str(library)], check=True) + return library + + +def prepare(out, verified_root): + spec = importlib.util.spec_from_file_location('ufs_fixture_manifest', verified_root / 'tools/manifest.py') + signer = importlib.util.module_from_spec(spec); spec.loader.exec_module(signer) + keys = out / 'fixture-keys'; keys.mkdir(mode=0o700, exist_ok=True); os.chmod(keys,0o700) + private, public = keys / 'owner.pem', keys / 'owner-public.pem' + if not private.exists(): + fd = os.open(private,os.O_WRONLY|os.O_CREAT|os.O_EXCL,0o600); os.close(fd) + subprocess.run(['openssl','genpkey','-algorithm','RSA','-pkeyopt','rsa_keygen_bits:3072', + '-out',str(private)],check=True,capture_output=True) + os.chmod(private,0o600) + subprocess.run(['openssl','pkey','-in',str(private),'-pubout','-out',str(public)],check=True,capture_output=True) + os_data = bytes((i*29+i//4096)&255 for i in range(33*4096+512)) + system_data = bytes((i*13+i//4096)&255 for i in range(4096+512)) + corrupt = bytearray(os_data); corrupt[33*4096] ^= 1 + swapped = os_data[4096:8192] + os_data[:4096] + os_data[8192:] + windows = [('OS',os_data),('SystemOS',system_data),('Corrupt',bytes(corrupt)),('Swapped',swapped)] + header = bytearray(4096); header[:8]=b'SHNGROOT'; struct.pack_into(' +#include +#include +#include +#include +#include +#include +#include + +#if defined(NATIVE_VERIFIED_BLOCK) && NATIVE_VERIFIED_BLOCK +#include "verified_root.h" +#if defined(NATIVE_VERIFIED_BLOCK_MULTI_POLICY) && NATIVE_VERIFIED_BLOCK_MULTI_POLICY +#include "VerifiedBlockPolicy.h" +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +extern "C" { +extern const struct vr_crypto vr_xnu_crypto; +extern const uint8_t verified_root_manifest[], verified_root_key[]; +extern const uint64_t verified_root_manifest_bytes; +} +#endif + +extern "C" { +volatile uint32_t native_virtio_ready = 0; +#if defined(NATIVE_VERIFIED_BLOCK) && NATIVE_VERIFIED_BLOCK +int native_verified_block_root_authorized(dev_t device); +// Provided by NativeUfs when linked: -1 not a UFS window, else 1/0 for the signed volume UUID. +int native_ufs_block_authenticates(IOService *, const char *, uint8_t *) __attribute__((weak)); +int native_verified_block_volume_device(const char *, dev_t *, char *, size_t); +volatile uint32_t native_verified_block_root_device = 0; +volatile uint32_t native_verified_block_root_media_matches = 0; +volatile uint32_t native_verified_block_root_provider_depth = 0; +volatile uint32_t native_verified_block_root_uuid_match = 0; +volatile uint32_t native_verified_block_wrong_uuid_refused = 0; +char native_verified_block_root_uuid[37] = {}; +volatile uint32_t native_virtio_manifest_ready = 0; +volatile uint32_t native_virtio_verified_devices = 0; +volatile uint32_t native_virtio_ambiguous_policies = 0; +volatile uint64_t native_virtio_verified_blocks = 0; +volatile uint32_t native_virtio_integrity_failures = 0; +volatile uintptr_t native_virtio_verified_manifest_pointer = 0; +volatile uintptr_t native_virtio_verified_key_pointer = 0; +#endif +volatile uint32_t native_virtio_read_requests = 0; +volatile uint32_t native_virtio_read_failures = 0; +volatile uint64_t native_virtio_sectors = 0; +volatile uint64_t native_virtio_mmio_physical = 0; +unsigned char native_virtio_first_sector_sha256[32] = {}; +__attribute__((noinline)) void native_virtio_read_checkpoint(void *device) { + __asm__ volatile("" : : "r"(device) : "memory"); +} +} + +// VirtIO 1.2 MMIO, modern split queue. All numbers here are protocol offsets. +namespace { +constexpr unsigned queueSize = 8; +constexpr unsigned transferSize = 65536; +struct Descriptor { uint64_t address; uint32_t length; uint16_t flags; uint16_t next; }; +struct Request { uint32_t operation; uint32_t reserved; uint64_t sector; }; +struct Used { uint32_t id; uint32_t length; }; +static_assert(sizeof(Descriptor) == 16 && sizeof(Request) == 16 && sizeof(Used) == 8, "VirtIO wire layout"); +inline void dmaBarrier() { __asm__ volatile("dmb osh" ::: "memory"); } +} + +class NativeVirtioBlock : public IOBlockStorageDevice { + OSDeclareDefaultStructors(NativeVirtioBlock) + IOMemoryMap *mapping = nullptr; + IOBufferMemoryDescriptor *queueMemory = nullptr; + IOBufferMemoryDescriptor *transferMemory = nullptr; + IOLock *gate = nullptr; + uintptr_t registers = 0; + uint64_t queuePhysical = 0; + uint64_t transferPhysical = 0; + uint64_t capacity = 0; + uint16_t availableIndex = 0; + uint16_t usedIndex = 0; +#if defined(NATIVE_VERIFIED_BLOCK) && NATIVE_VERIFIED_BLOCK + struct vr_view verified = {}; + vm_offset_t verifiedStorage = 0; + vm_size_t verifiedStorageBytes = 0; + unsigned char *verifiedSnapshot = nullptr; + bool configureVerification(); + IOReturn readRawSectors(uint64_t, uint32_t); +#endif + const void *readBuffer() const { +#if defined(NATIVE_VERIFIED_BLOCK) && NATIVE_VERIFIED_BLOCK + return verifiedSnapshot; +#else + return transferMemory->getBytesNoCopy(); +#endif + } + bool online = false; + bool ownsDevice = false; + uint32_t read32(unsigned offset) const { + return *reinterpret_cast(registers + offset); + } + void write32(unsigned offset, uint32_t value) { + *reinterpret_cast(registers + offset) = value; + __asm__ volatile("dsb oshst" ::: "memory"); + } + void writeAddress(unsigned offset, uint64_t value) { + write32(offset, static_cast(value)); + write32(offset + 4, static_cast(value >> 32)); + } + bool reset(); + bool configureQueue(); + IOReturn readSectors(uint64_t sector, uint32_t bytes); +public: +#if defined(NATIVE_VERIFIED_BLOCK) && NATIVE_VERIFIED_BLOCK + bool authenticatesVolume(const char *, uint8_t * = nullptr) const; +#endif + IOService *probe(IOService *, SInt32 *) override; + bool start(IOService *) override; + void stop(IOService *) override; + void free() override; + IOReturn doEjectMedia() override { return kIOReturnUnsupported; } + IOReturn doFormatMedia(UInt64) override { return kIOReturnNotWritable; } + UInt32 doGetFormatCapacities(UInt64 *, UInt32) const override { return 0; } + char *getVendorString() override { return const_cast("VirtIO"); } + char *getProductString() override { return const_cast("VirtIO read-only block"); } + char *getRevisionString() override { return const_cast("1.0"); } + char *getAdditionalDeviceInfoString() override { return const_cast("modern MMIO, coherent DMA"); } + IOReturn reportBlockSize(UInt64 *value) override { *value = 512; return kIOReturnSuccess; } + IOReturn reportEjectability(bool *value) override { *value = false; return kIOReturnSuccess; } + IOReturn reportMaxValidBlock(UInt64 *value) override { *value = capacity - 1; return kIOReturnSuccess; } + IOReturn reportMediaState(bool *present, bool *changed) override { + *present = online; if (changed) *changed = false; return kIOReturnSuccess; + } + IOReturn reportRemovability(bool *value) override { *value = false; return kIOReturnSuccess; } + IOReturn reportWriteProtection(bool *value) override { *value = true; return kIOReturnSuccess; } + IOReturn doSynchronizeCache() override { return online ? kIOReturnSuccess : kIOReturnNotReady; } + IOReturn doSynchronize(UInt64, UInt64, IOOptionBits) override { return doSynchronizeCache(); } + IOReturn getWriteCacheState(bool *value) override { *value = false; return kIOReturnSuccess; } + IOReturn setWriteCacheState(bool) override { return kIOReturnUnsupported; } + IOReturn doAsyncReadWrite(IOMemoryDescriptor *, UInt64, UInt64, IOStorageAttributes *, IOStorageCompletion *) override; +}; +OSDefineMetaClassAndStructors(NativeVirtioBlock, IOBlockStorageDevice) + +IOService *NativeVirtioBlock::probe(IOService *provider, SInt32 *score) +{ + OSData *compatible = OSDynamicCast(OSData, provider->getProperty("compatible")); + if (!compatible || compatible->getLength() != 12 || + memcmp(compatible->getBytesNoCopy(), "virtio,mmio", 12)) return nullptr; + return IOBlockStorageDevice::probe(provider, score); +} + +bool NativeVirtioBlock::reset() +{ + if (!registers || !ownsDevice) return true; + online = false; + write32(0x70, 0); + uint64_t deadline; + clock_interval_to_deadline(1000, kMillisecondScale, &deadline); + while (read32(0x70)) { + if (mach_absolute_time() >= deadline) return false; + IODelay(1); + } + return true; +} + +bool NativeVirtioBlock::configureQueue() +{ + if (!reset()) return false; + write32(0x70, 1); + write32(0x70, 3); + write32(0x14, 0); + uint32_t lowFeatures = read32(0x10); + write32(0x14, 1); + uint32_t highFeatures = read32(0x10); + // A read-only backend and VIRTIO_F_VERSION_1 are mandatory for this driver. + if (!(lowFeatures & (1U << 5)) || !(highFeatures & 1)) return false; + write32(0x24, 0); + write32(0x20, 1U << 5); + write32(0x24, 1); + write32(0x20, 1); + write32(0x70, 11); + if (!(read32(0x70) & 8)) return false; + write32(0x30, 0); + if (read32(0x44) || read32(0x34) < queueSize) return false; + queueMemory = IOBufferMemoryDescriptor::inTaskWithOptions(kernel_task, + static_cast(kIODirectionInOut) | static_cast(kIOMemoryPhysicallyContiguous), 4096, 4096); + transferMemory = IOBufferMemoryDescriptor::inTaskWithOptions(kernel_task, + static_cast(kIODirectionInOut) | static_cast(kIOMemoryPhysicallyContiguous), transferSize, 4096); + if (!queueMemory || !transferMemory) return false; + IOByteCount queueLength = 0, transferLength = 0; + queuePhysical = queueMemory->getPhysicalSegment(0, &queueLength); + transferPhysical = transferMemory->getPhysicalSegment(0, &transferLength); + if (!queuePhysical || !transferPhysical || queueLength < 4096 || transferLength < transferSize) return false; + bzero(queueMemory->getBytesNoCopy(), 4096); + // Suppress used-buffer interrupts: this bounded initial transport polls. + *reinterpret_cast(static_cast(queueMemory->getBytesNoCopy()) + 128) = 1; + availableIndex = usedIndex = 0; + dmaBarrier(); + write32(0x38, queueSize); + writeAddress(0x80, queuePhysical); + writeAddress(0x90, queuePhysical + 128); + writeAddress(0xa0, queuePhysical + 256); + write32(0x44, 1); + // Read 64-bit capacity through stable, aligned 32-bit configuration accesses. + bool stable = false; + for (unsigned attempt = 0; attempt < 8; ++attempt) { + uint32_t generation = read32(0xfc); + uint64_t low = read32(0x100), high = read32(0x104); + if (generation == read32(0xfc)) { + capacity = low | (high << 32); + stable = true; + break; + } + } + if (!stable || !capacity || capacity > UINT64_MAX / 512) return false; + write32(0x70, 15); + online = (read32(0x70) & 15) == 15; + return online; +} + +#if defined(NATIVE_VERIFIED_BLOCK) && NATIVE_VERIFIED_BLOCK +bool NativeVirtioBlock::authenticatesVolume(const char *uuid, uint8_t *manifestDigest) const +{ + if (!uuid || !online || !verifiedStorage || !verifiedSnapshot || !verified.hashes) return false; + static const char hex[] = "0123456789ABCDEF"; + char expected[37]; + unsigned at = 0; + for (unsigned i = 0; i < 16; ++i) { + if (i == 4 || i == 6 || i == 8 || i == 10) expected[at++] = '-'; + expected[at++] = hex[verified.volume_uuid[i] >> 4]; + expected[at++] = hex[verified.volume_uuid[i] & 15]; + } + expected[at] = 0; + if (strcmp(uuid, expected)) return false; + if (manifestDigest) { + // Hash only the authenticated envelope, excluding allocation padding and key. + const uint64_t bytes = VR_HEADER_BYTES + VR_SIGNATURE_BYTES + verified.block_count * 32; + SHA384_CTX context; + SHA384_Init(&context); + SHA384_Update(&context, reinterpret_cast(verifiedStorage), bytes); + SHA384_Final(manifestDigest, &context); + } + return true; +} + +static int verifiedVolumeQuery(dev_t device, const char *expectedUUID, uint8_t *manifestDigest) +{ + native_verified_block_root_device = static_cast(device); + native_verified_block_root_media_matches = 0; + native_verified_block_root_provider_depth = 0; + native_verified_block_root_uuid_match = 0; + native_verified_block_wrong_uuid_refused = 0; + bzero(native_verified_block_root_uuid, sizeof(native_verified_block_root_uuid)); + OSDictionary *matching = IOService::serviceMatching("IOMedia"); + OSDictionary *properties = OSDictionary::withCapacity(2); + OSNumber *mjr = OSNumber::withNumber(static_cast(major(static_cast(device))), 32); + OSNumber *mnr = OSNumber::withNumber(static_cast(minor(static_cast(device))), 32); + if (!matching || !properties || !mjr || !mnr) { + OSSafeReleaseNULL(mnr); OSSafeReleaseNULL(mjr); OSSafeReleaseNULL(properties); OSSafeReleaseNULL(matching); + return 0; + } + bool ready = properties->setObject(kIOBSDMajorKey, mjr) && properties->setObject(kIOBSDMinorKey, mnr) && + matching->setObject("IOPropertyMatch", properties); + mnr->release(); mjr->release(); properties->release(); + OSIterator *iterator = ready ? IOService::getMatchingServices(matching) : nullptr; + matching->release(); + if (!iterator) return 0; + bool authenticated = false; + unsigned matches = 0, depth = 0; + while (OSObject *object = iterator->getNextObject()) { + IOService *media = OSDynamicCast(IOService, object); + if (!media) continue; + if (++matches != 1) { authenticated = false; break; } + OSString *uuid = OSDynamicCast(OSString, media->getProperty("UUID")); + if (!uuid || uuid->getLength() != 36 || + (expectedUUID && strcmp(expectedUUID, uuid->getCStringNoCopy()))) continue; + strlcpy(native_verified_block_root_uuid, uuid->getCStringNoCopy(), sizeof(native_verified_block_root_uuid)); + for (IOService *provider = media; provider && depth < 64; + provider = provider->getProvider()) { + ++depth; + NativeVirtioBlock *block = OSDynamicCast(NativeVirtioBlock, provider); + if (!block) { + if (native_ufs_block_authenticates) { + int ufs = native_ufs_block_authenticates(provider, uuid->getCStringNoCopy(), manifestDigest); + if (ufs >= 0) { + authenticated = ufs == 1; + native_verified_block_root_uuid_match = authenticated; + native_verified_block_wrong_uuid_refused = 1; + break; + } + } + continue; + } + authenticated = block->authenticatesVolume(uuid->getCStringNoCopy(), manifestDigest); + native_verified_block_root_uuid_match = authenticated; + char wrong[37]; + strlcpy(wrong, uuid->getCStringNoCopy(), sizeof(wrong)); + wrong[0] ^= 1; // Local negative identity only; actual registry/media is unchanged. + native_verified_block_wrong_uuid_refused = !block->authenticatesVolume(wrong); + break; + } + } + iterator->release(); + native_verified_block_root_media_matches = matches; + native_verified_block_root_provider_depth = depth; + if (!authenticated || matches != 1) { + if (manifestDigest) bzero(manifestDigest, SHA384_DIGEST_LENGTH); + return 0; + } + return 1; +} + +extern "C" int native_verified_block_root_authorized(dev_t device) +{ + return verifiedVolumeQuery(device, nullptr, nullptr); +} + +extern "C" int native_verified_block_manifest_sha384(dev_t device, const char *uuid, uint8_t digest[48]) +{ + if (!digest) return 0; + bzero(digest, SHA384_DIGEST_LENGTH); + if (!uuid || strlen(uuid) != 36) return 0; + return verifiedVolumeQuery(device, uuid, digest); +} + +extern "C" int native_verified_block_volume_device(const char *uuid, dev_t *device, char *path, size_t size) +{ + if (!uuid || !device || !path || size < 6 || strlen(uuid) != 36) return 0; + *device = NODEV; + path[0] = 0; + OSDictionary *matching = IOService::serviceMatching("IOMedia"); + OSDictionary *properties = OSDictionary::withCapacity(1); + OSString *wanted = OSString::withCString(uuid); + if (!matching || !properties || !wanted) { + OSSafeReleaseNULL(wanted); OSSafeReleaseNULL(properties); OSSafeReleaseNULL(matching); + return 0; + } + bool ready = properties->setObject("UUID", wanted) && matching->setObject("IOPropertyMatch", properties); + wanted->release(); properties->release(); + OSIterator *iterator = ready ? IOService::getMatchingServices(matching) : nullptr; + matching->release(); + if (!iterator) return 0; + unsigned matches = 0; + bool authenticated = false; + dev_t candidate = NODEV; + char candidatePath[64] = {}; + while (OSObject *object = iterator->getNextObject()) { + IOService *media = OSDynamicCast(IOService, object); + if (!media) continue; + if (++matches != 1) { authenticated = false; break; } + OSString *identity = OSDynamicCast(OSString, media->getProperty("UUID")); + OSString *name = OSDynamicCast(OSString, media->getProperty(kIOBSDNameKey)); + OSNumber *mjr = OSDynamicCast(OSNumber, media->getProperty(kIOBSDMajorKey)); + OSNumber *mnr = OSDynamicCast(OSNumber, media->getProperty(kIOBSDMinorKey)); + if (!identity || strcmp(identity->getCStringNoCopy(), uuid) || !name || !mjr || !mnr || + !name->getLength() || name->getLength() >= sizeof(candidatePath) - 6 || name->getLength() >= size - 6 || + mjr->unsigned64BitValue() > 255 || mnr->unsigned64BitValue() > 0xffffff) continue; + candidate = static_cast(makedev(mjr->unsigned32BitValue(), mnr->unsigned32BitValue())); + if (!native_verified_block_root_authorized(candidate)) continue; + snprintf(candidatePath, sizeof(candidatePath), "/dev/%s", name->getCStringNoCopy()); + authenticated = true; + } + iterator->release(); + if (!authenticated || matches != 1) return 0; + *device = candidate; + strlcpy(path, candidatePath, size); + return 1; +} + +bool NativeVirtioBlock::configureVerification() +{ + if (capacity > UINT64_MAX / 512) return false; + const uint8_t *manifest = verified_root_manifest; + uint64_t manifestBytes = verified_root_manifest_bytes; +#if defined(NATIVE_VERIFIED_BLOCK_MULTI_POLICY) && NATIVE_VERIFIED_BLOCK_MULTI_POLICY + manifest = nullptr; + manifestBytes = 0; + if (!verified_root_policy_count || verified_root_policy_count > 16) return false; + for (uint64_t index = 0; index < verified_root_policy_count; ++index) { + const auto &policy = verified_root_policies[index]; + if (!policy.manifest || policy.manifestBytes < VR_HEADER_BYTES + VR_SIGNATURE_BYTES) return false; + uint64_t bytes = 0; + for (unsigned i = 0; i < 8; ++i) bytes |= static_cast(policy.manifest[24 + i]) << (8 * i); + if (bytes != capacity * 512) continue; + if (manifest) { + __atomic_fetch_add(&native_virtio_ambiguous_policies, 1, __ATOMIC_RELAXED); + return false; + } + manifest = policy.manifest; + manifestBytes = policy.manifestBytes; + } + if (!manifest) return false; +#endif + vm_size_t tableBytes = round_page(static_cast(manifestBytes)); + if (tableBytes < manifestBytes || tableBytes > UINT64_MAX - 4096) return false; + verifiedStorageBytes = tableBytes + 4096; + if (kmem_alloc(kernel_map, &verifiedStorage, verifiedStorageBytes, + static_cast(KMA_KOBJECT | KMA_DATA), VM_KERN_MEMORY_SECURITY) != KERN_SUCCESS) return false; + memcpy(reinterpret_cast(verifiedStorage), manifest, manifestBytes); + auto *key = reinterpret_cast(verifiedStorage + tableBytes); + memcpy(key, verified_root_key, VR_KEY_BYTES); + if (vm_map_protect(kernel_map, verifiedStorage, verifiedStorage + verifiedStorageBytes, + VM_PROT_READ, TRUE) != KERN_SUCCESS) return false; + if (vr_open(reinterpret_cast(verifiedStorage), manifestBytes, + key, 1, &vr_xnu_crypto, &verified) != VR_OK || verified.image_bytes != capacity * 512) return false; + verifiedSnapshot = static_cast(IOMalloc(transferSize)); + if (!verifiedSnapshot) return false; + native_virtio_verified_manifest_pointer = verifiedStorage; + native_virtio_verified_key_pointer = reinterpret_cast(key); + native_virtio_manifest_ready = 1; + return true; +} + +IOReturn NativeVirtioBlock::readSectors(uint64_t sector, uint32_t bytes) +{ + if (!verifiedSnapshot || !online || !bytes || bytes > transferSize - VR_BLOCK_BYTES || + (bytes & 511) || sector >= capacity || bytes / 512 > capacity - sector) return kIOReturnBadArgument; + uint64_t begin = sector * 512, end = begin + bytes; + uint64_t alignedBegin = begin & ~static_cast(VR_BLOCK_BYTES - 1); + uint64_t padding = (VR_BLOCK_BYTES - (end & (VR_BLOCK_BYTES - 1))) & (VR_BLOCK_BYTES - 1); + if (padding > capacity * 512 - end) padding = capacity * 512 - end; + uint32_t span = static_cast(end + padding - alignedBegin); + if (span > transferSize) return kIOReturnBadArgument; + IOReturn status = readRawSectors(alignedBegin / 512, span); + if (status) return status; + // Snapshot completed DMA before hashing; only this CPU-owned copy is exposed. + memcpy(verifiedSnapshot, transferMemory->getBytesNoCopy(), span); +#if defined(NATIVE_VERIFIED_BLOCK_INJECT_FAULT) && NATIVE_VERIFIED_BLOCK_INJECT_FAULT + verifiedSnapshot[0] ^= 1; // Isolated negative input, original disk/signature untouched. +#endif + for (uint32_t offset = 0; offset < span; offset += VR_BLOCK_BYTES) { + uint32_t length = span - offset < VR_BLOCK_BYTES ? span - offset : VR_BLOCK_BYTES; + if (vr_verify_block(&verified, alignedBegin / VR_BLOCK_BYTES + offset / VR_BLOCK_BYTES, + verifiedSnapshot + offset, length, &vr_xnu_crypto) != VR_OK) { + __atomic_fetch_add(&native_virtio_integrity_failures, 1, __ATOMIC_RELAXED); + if (!reset()) panic("NativeVirtioBlock: unverified DMA could not be quiesced"); + return kIOReturnNotPrivileged; + } + __atomic_fetch_add(&native_virtio_verified_blocks, 1, __ATOMIC_RELAXED); + } + memmove(verifiedSnapshot, verifiedSnapshot + (begin - alignedBegin), bytes); + return kIOReturnSuccess; +} +IOReturn NativeVirtioBlock::readRawSectors(uint64_t sector, uint32_t bytes) +#else +IOReturn NativeVirtioBlock::readSectors(uint64_t sector, uint32_t bytes) +#endif +{ + if (!online || !bytes || bytes > transferSize || (bytes & 511) || + sector >= capacity || bytes / 512 > capacity - sector) return kIOReturnBadArgument; + auto *queue = static_cast(queueMemory->getBytesNoCopy()); + auto *descriptors = reinterpret_cast(queue); + auto *request = reinterpret_cast(queue + 512); + volatile uint8_t *status = queue + 528; + *request = {0, 0, sector}; + *status = 0xff; + descriptors[0] = {queuePhysical + 512, sizeof(Request), 1, 1}; + descriptors[1] = {transferPhysical, bytes, 3, 2}; + descriptors[2] = {queuePhysical + 528, 1, 2, 0}; + auto *available = reinterpret_cast(queue + 128); + available[2 + availableIndex % queueSize] = 0; + dmaBarrier(); + ++availableIndex; + available[1] = availableIndex; + dmaBarrier(); + write32(0x50, 0); + auto *used = reinterpret_cast(queue + 256); + uint64_t deadline; + clock_interval_to_deadline(1000, kMillisecondScale, &deadline); + while (used[1] == usedIndex) { + if (mach_absolute_time() >= deadline) { + reset(); + __atomic_fetch_add(&native_virtio_read_failures, 1, __ATOMIC_RELAXED); + return kIOReturnTimeout; + } + dmaBarrier(); + IODelay(1); + } + dmaBarrier(); + auto *element = reinterpret_cast(queue + 260) + usedIndex % queueSize; + bool valid = used[1] == static_cast(usedIndex + 1) && element->id == 0 && + element->length >= bytes + 1 && *status == 0; + ++usedIndex; + uint32_t interrupts = read32(0x60) & 3; + if (interrupts) write32(0x64, interrupts); + if (!valid) { + reset(); + __atomic_fetch_add(&native_virtio_read_failures, 1, __ATOMIC_RELAXED); + return kIOReturnIOError; + } + __atomic_fetch_add(&native_virtio_read_requests, 1, __ATOMIC_RELAXED); + return kIOReturnSuccess; +} + +bool NativeVirtioBlock::start(IOService *provider) +{ + if (!IOBlockStorageDevice::start(provider) || !provider->getProperty("dma-coherent") || + provider->getResources() != kIOReturnSuccess) return false; + mapping = provider->mapDeviceMemoryWithIndex(0, kIOMapAnywhere | kIOMapInhibitCache); + if (!mapping || mapping->getLength() < 0x108) return false; + registers = static_cast(mapping->getVirtualAddress()); + if (read32(0x00) != 0x74726976 || read32(0x04) != 2 || read32(0x08) != 2) return false; + ownsDevice = true; + gate = IOLockAlloc(); + if (!gate || !configureQueue()) return false; +#if defined(NATIVE_VERIFIED_BLOCK) && NATIVE_VERIFIED_BLOCK + if (!configureVerification()) return false; +#endif + if (readSectors(0, 512)) return false; + SHA256_CTX hash; + SHA256_Init(&hash); + SHA256_Update(&hash, readBuffer(), 512); + SHA256_Final(native_virtio_first_sector_sha256, &hash); + native_virtio_sectors = capacity; + native_virtio_mmio_physical = mapping->getPhysicalAddress(); + native_virtio_ready = 1; +#if defined(NATIVE_VERIFIED_BLOCK) && NATIVE_VERIFIED_BLOCK + __atomic_fetch_add(&native_virtio_verified_devices, 1, __ATOMIC_RELAXED); +#endif + native_virtio_read_checkpoint(this); + registerService(); + return true; +} + +IOReturn NativeVirtioBlock::doAsyncReadWrite(IOMemoryDescriptor *buffer, UInt64 block, UInt64 count, + IOStorageAttributes *, IOStorageCompletion *completion) +{ + if (!buffer || !completion || !completion->action || !count || block >= capacity || + count > capacity - block || count > UINT64_MAX / 512 || buffer->getLength() < count * 512) return kIOReturnBadArgument; + if (buffer->getDirection() != kIODirectionIn) return kIOReturnNotWritable; + uint64_t actual = 0; + IOReturn result = kIOReturnNotReady; + IOLockLock(gate); + while (online && actual < count * 512) { +#if defined(NATIVE_VERIFIED_BLOCK) && NATIVE_VERIFIED_BLOCK + constexpr unsigned chunkSize = transferSize - VR_BLOCK_BYTES; +#else + constexpr unsigned chunkSize = transferSize; +#endif + uint32_t bytes = static_cast(count * 512 - actual > chunkSize ? chunkSize : count * 512 - actual); + result = readSectors(block + actual / 512, bytes); + if (result) break; + if (buffer->writeBytes(actual, readBuffer(), bytes) != bytes) { + result = kIOReturnIOError; + break; + } + actual += bytes; + } + IOLockUnlock(gate); + completion->action(completion->target, completion->parameter, result, actual); + return kIOReturnSuccess; +} + +void NativeVirtioBlock::stop(IOService *provider) +{ + if (!reset()) panic("NativeVirtioBlock: device did not acknowledge DMA reset"); + IOBlockStorageDevice::stop(provider); +} + +void NativeVirtioBlock::free() +{ + if (registers && !reset()) panic("NativeVirtioBlock: refusing to free active DMA buffers"); +#if defined(NATIVE_VERIFIED_BLOCK) && NATIVE_VERIFIED_BLOCK + if (verifiedSnapshot) { IOFree(verifiedSnapshot, transferSize); verifiedSnapshot = nullptr; } + if (verifiedStorage) { kmem_free(kernel_map, verifiedStorage, verifiedStorageBytes); verifiedStorage = 0; } +#endif + if (queueMemory) { queueMemory->release(); queueMemory = nullptr; } + if (transferMemory) { transferMemory->release(); transferMemory = nullptr; } + if (mapping) { mapping->release(); mapping = nullptr; } + if (gate) { IOLockFree(gate); gate = nullptr; } + IOBlockStorageDevice::free(); +} diff --git a/drivers/NativeVirtioBlock/VerifiedBlockPolicy.h b/drivers/NativeVirtioBlock/VerifiedBlockPolicy.h new file mode 100644 index 0000000..1ce9c7e --- /dev/null +++ b/drivers/NativeVirtioBlock/VerifiedBlockPolicy.h @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: BSD-2-Clause-Patent +#ifndef NATIVE_VERIFIED_BLOCK_POLICY_H +#define NATIVE_VERIFIED_BLOCK_POLICY_H +#include +struct NativeVerifiedBlockPolicy { + const uint8_t *manifest; + uint64_t manifestBytes; +}; +static_assert(sizeof(NativeVerifiedBlockPolicy) == 16 && alignof(NativeVerifiedBlockPolicy) == 8); +extern "C" { +extern const NativeVerifiedBlockPolicy verified_root_policies[]; +extern const uint64_t verified_root_policy_count; +} +#endif diff --git a/drivers/NativeVirtioConsole/LICENSE b/drivers/NativeVirtioConsole/LICENSE new file mode 100644 index 0000000..217b4df --- /dev/null +++ b/drivers/NativeVirtioConsole/LICENSE @@ -0,0 +1,51 @@ +Copyright (c) 2026, ArmVirtPlatform contributors. All rights reserved. + +SPDX-License-Identifier: BSD-2-Clause-Patent + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +Subject to the terms and conditions of this license, each copyright holder +and contributor hereby grants to those receiving rights under this license +a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable +(except for failure to satisfy the conditions of this license) patent +license to make, have made, use, offer to sell, sell, import, and otherwise +transfer this software, where such license applies only to those patent +claims, already acquired or hereafter acquired, licensable by such copyright +holder or contributor that are necessarily infringed by: + +(a) their Contribution(s) (the licensed copyrights of copyright holders and + non-copyrightable additions of contributors, in source or binary form) + alone; or + +(b) combination of their Contribution(s) with the work of authorship to + which such Contribution(s) was added by such copyright holder or + contributor, if, at the time the Contribution is added, such addition + causes such combination to be necessarily infringed. The patent license + shall not apply to any other combinations which include the + Contribution. + +Except as expressly stated above, no rights or licenses from any copyright +holder or contributor is granted under this license, whether expressly, by +implication, estoppel or otherwise. + +DISCLAIMER + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. diff --git a/drivers/NativeVirtioConsole/NativeVirtioConsole.cpp b/drivers/NativeVirtioConsole/NativeVirtioConsole.cpp new file mode 100644 index 0000000..323cb0c --- /dev/null +++ b/drivers/NativeVirtioConsole/NativeVirtioConsole.cpp @@ -0,0 +1,271 @@ +// SPDX-License-Identifier: BSD-2-Clause-Patent +// Modern VirtIO console port 0. OASIS VirtIO 1.2 sections 4.2 and 5.3. +// Binary transport only: this does not implement a GPU, compositor or display. +#include +#include +#include +#include +#include +#include +#include +#include "NativeVirtioConsoleTransport.h" +namespace { +constexpr unsigned queueSize=8, bufferSize=4096; +struct Descriptor { uint64_t address;uint32_t length;uint16_t flags,next; }; +struct Used { uint32_t id,length; }; +static_assert(sizeof(Descriptor)==16 && sizeof(Used)==8,"Actual VirtIO wire sizes"); +inline void dmaBarrier() { __asm__ volatile("dmb osh" ::: "memory"); } +struct Queue { + IOBufferMemoryDescriptor *ring=nullptr,*buffer=nullptr; + uint64_t physical=0,dataPhysical=0; + uint16_t available=0,used=0; + unsigned bytes=0,offset=0; + bool pending=false; + unsigned char *memory() const {return static_cast(ring->getBytesNoCopy());} + void *data() const {return buffer->getBytesNoCopy();} +}; +} +extern "C" { +volatile uint32_t native_console_ready; +volatile uint32_t native_console_tx_requests,native_console_rx_requests,native_console_failures; +volatile uint64_t native_console_tx_bytes,native_console_rx_bytes,native_console_mmio; +} +class NativeVirtioConsoleClient; +class NativeVirtioConsole : public IOService { + OSDeclareDefaultStructors(NativeVirtioConsole) + friend class NativeVirtioConsoleClient; + friend IOReturn native_console_send(IOService *,IOService *,const void *,size_t); + friend IOReturn native_console_receive(IOService *,IOService *,void *,size_t *); + IOMemoryMap *mapping=nullptr; + IOLock *gate=nullptr; + IOService *owner=nullptr; + Queue queues[2]; + uintptr_t registers=0; + bool online=false,ownsDevice=false; + uint32_t read32(unsigned n) const {return *reinterpret_cast(registers+n);} + void write32(unsigned n,uint32_t v) { + *reinterpret_cast(registers+n)=v; + __asm__ volatile("dsb oshst" ::: "memory"); + } + void address(unsigned n,uint64_t v) {write32(n,(uint32_t)v);write32(n+4,(uint32_t)(v>>32));} + bool reset(); + bool configure(); + bool configureQueue(unsigned); + void post(unsigned,unsigned); + IOReturn complete(unsigned,unsigned *); + void cleanup(); + IOReturn send(const void *,size_t); + IOReturn receive(void *,size_t *); +public: + IOService *probe(IOService *,SInt32 *) override; + bool start(IOService *) override; + void stop(IOService *) override; + void free() override; + bool handleOpen(IOService *,IOOptionBits,void *) override; + void handleClose(IOService *,IOOptionBits) override; + bool handleIsOpen(const IOService *) const override; + IOReturn newUserClient(task_t,void *,UInt32,OSDictionary *,IOUserClient **) override; +}; +class NativeVirtioConsoleClient : public IOUserClient { + OSDeclareDefaultStructors(NativeVirtioConsoleClient) + NativeVirtioConsole *controller=nullptr; + bool opened=false; + static IOReturn sendAction(OSObject *,void *,IOExternalMethodArguments *); + static IOReturn receiveAction(OSObject *,void *,IOExternalMethodArguments *); +public: + bool initWithTask(task_t,void *,UInt32,OSDictionary *) override; + bool start(IOService *) override; + void stop(IOService *) override; + IOReturn clientClose() override; + IOReturn externalMethod(uint32_t,IOExternalMethodArguments *,IOExternalMethodDispatch *,OSObject *,void *) override; +}; +OSDefineMetaClassAndStructors(NativeVirtioConsole,IOService) +OSDefineMetaClassAndStructors(NativeVirtioConsoleClient,IOUserClient) + +extern "C" IOReturn native_console_send(IOService *provider,IOService *owner,const void *bytes,size_t size) { + NativeVirtioConsole *console=OSDynamicCast(NativeVirtioConsole,provider); + return console && owner && console->isOpen(owner)?console->send(bytes,size):kIOReturnNotOpen; +} +extern "C" IOReturn native_console_receive(IOService *provider,IOService *owner,void *bytes,size_t *size) { + NativeVirtioConsole *console=OSDynamicCast(NativeVirtioConsole,provider); + return console && owner && console->isOpen(owner)?console->receive(bytes,size):kIOReturnNotOpen; +} + +IOService *NativeVirtioConsole::probe(IOService *provider,SInt32 *score) { + OSData *compatible=OSDynamicCast(OSData,provider->getProperty("compatible")); + if(!compatible || compatible->getLength()!=12 || memcmp(compatible->getBytesNoCopy(),"virtio,mmio",12))return nullptr; + IOMemoryMap *view=provider->mapDeviceMemoryWithIndex(0); + if(!view)return nullptr; + uintptr_t base=view->getVirtualAddress(); + bool matches=view->getLength()>=0x100 && + *reinterpret_cast(base)==0x74726976 && + *reinterpret_cast(base+4)==2 && + *reinterpret_cast(base+8)==3; + view->release(); + return matches?IOService::probe(provider,score):nullptr; +} +bool NativeVirtioConsole::reset() { + online=false;if(ownsDevice)native_console_ready=0; + if(!registers || !ownsDevice)return true; + write32(0x70,0);uint64_t deadline; + clock_interval_to_deadline(1000,kMillisecondScale,&deadline); + while(read32(0x70)) {if(mach_absolute_time()>=deadline)return false;IODelay(1);} + for(unsigned i=0;i<2;i++)queues[i].pending=false; + return true; +} +bool NativeVirtioConsole::configureQueue(unsigned id) { + Queue &q=queues[id];write32(0x30,id); + if(read32(0x44) || read32(0x34)(kIODirectionInOut)|static_cast(kIOMemoryPhysicallyContiguous); + q.ring=IOBufferMemoryDescriptor::inTaskWithOptions(kernel_task,options,4096,4096); + q.buffer=IOBufferMemoryDescriptor::inTaskWithOptions(kernel_task,options,bufferSize,4096); + if(!q.ring || !q.buffer)return false; + IOByteCount rn=0,bn=0;q.physical=q.ring->getPhysicalSegment(0,&rn);q.dataPhysical=q.buffer->getPhysicalSegment(0,&bn); + if(!q.physical || !q.dataPhysical || rn<4096 || bn(q.memory()+128)=1; // NO_INTERRUPT; bounded polling. + dmaBarrier();write32(0x38,queueSize);address(0x80,q.physical);address(0x90,q.physical+128);address(0xa0,q.physical+256);write32(0x44,1); + return true; +} +bool NativeVirtioConsole::configure() { + if(!reset())return false; + write32(0x70,1);write32(0x70,3); + write32(0x14,1);if(!(read32(0x10)&1))return false; + // Port zero only. No MULTIPORT, event-index, indirect descriptors or DMA remapping. + write32(0x24,0);write32(0x20,0);write32(0x24,1);write32(0x20,1); + write32(0x70,11);if((read32(0x70)&(8|64|128))!=8)return false; + if(!configureQueue(0) || !configureQueue(1))return false; + write32(0x70,15);online=(read32(0x70)&(15|64|128))==15; + if(online)post(0,bufferSize); + return online; +} +void NativeVirtioConsole::post(unsigned id,unsigned bytes) { + Queue &q=queues[id];Descriptor *d=reinterpret_cast(q.memory()); + d[0]={q.dataPhysical,bytes,(uint16_t)(id==0?2:0),0}; + volatile uint16_t *available=reinterpret_cast(q.memory()+128); + available[2+(q.available%queueSize)]=0; + dmaBarrier();available[1]=++q.available;dmaBarrier();q.pending=true;write32(0x50,id); +} +IOReturn NativeVirtioConsole::complete(unsigned id,unsigned *length) { + Queue &q=queues[id];if(!q.pending)return kIOReturnNotReady; + volatile uint16_t *used=reinterpret_cast(q.memory()+256); + uint64_t deadline;clock_interval_to_deadline(2000,kMillisecondScale,&deadline); + while(used[1]==q.used) { + if(read32(0x70)&(64|128)) {online=false;return kIOReturnIOError;} + if(mach_absolute_time()>=deadline)return kIOReturnNotResponding; + IODelay(50); + } + dmaBarrier();if((uint16_t)(used[1]-q.used)!=1) {online=false;return kIOReturnIOError;} + volatile Used *entries=reinterpret_cast(q.memory()+260); + uint32_t index=entries[q.used%queueSize].id,n=entries[q.used%queueSize].length; + if(index || n>bufferSize) {online=false;return kIOReturnIOError;} + ++q.used;q.pending=false;*length=n; + uint32_t irq=read32(0x60);if(irq&3)write32(0x64,irq&3); + return kIOReturnSuccess; +} +IOReturn NativeVirtioConsole::send(const void *data,size_t bytes) { + if(!data || !bytes || bytes>bufferSize)return kIOReturnBadArgument; + IOLockLock(gate);IOReturn status=kIOReturnNotReady; + Queue &q=queues[1]; + if(online) { + unsigned consumed=0; + // A timed-out descriptor is still device-owned. Never overwrite it. + if(q.pending)status=complete(1,&consumed);else status=kIOReturnSuccess; + if(status==kIOReturnSuccess) { + memcpy(q.data(),data,bytes);post(1,(unsigned)bytes); + status=complete(1,&consumed); + if(status==kIOReturnSuccess) {native_console_tx_requests=native_console_tx_requests+1;native_console_tx_bytes+=bytes;} + } + } + if(status!=kIOReturnSuccess)native_console_failures=native_console_failures+1; + IOLockUnlock(gate);return status; +} +IOReturn NativeVirtioConsole::receive(void *data,size_t *bytes) { + if(!data || !bytes || !*bytes || *bytes>bufferSize)return kIOReturnBadArgument; + size_t capacity=*bytes;*bytes=0;IOLockLock(gate);IOReturn status=kIOReturnNotReady; + Queue &q=queues[0]; + if(online) { + status=kIOReturnSuccess; + if(q.pending) { + unsigned n=0;status=complete(0,&n); + if(status==kIOReturnSuccess) {q.bytes=n;q.offset=0;} + } + if(status==kIOReturnSuccess) { + unsigned left=q.bytes-q.offset,n=left(q.data())+q.offset,n);q.offset+=n;*bytes=n; + native_console_rx_requests=native_console_rx_requests+1;native_console_rx_bytes+=n; + if(q.offset==q.bytes) {q.offset=q.bytes=0;post(0,bufferSize);} + } + } + if(status!=kIOReturnSuccess)native_console_failures=native_console_failures+1; + IOLockUnlock(gate);return status; +} +bool NativeVirtioConsole::start(IOService *provider) { + uint64_t tcr,sctlr;__asm__ volatile("mrs %0,tcr_el1":"=r"(tcr));__asm__ volatile("mrs %0,sctlr_el1":"=r"(sctlr)); + if(((tcr>>14)&3)!=0 || ((tcr>>30)&3)!=2 || !(sctlr&1) || !IOService::start(provider))return false; + gate=IOLockAlloc();mapping=provider->mapDeviceMemoryWithIndex(0); + if(!gate || !mapping || mapping->getLength()<0x100)return false; + registers=mapping->getVirtualAddress(); + if(read32(0)!=0x74726976 || read32(4)!=2 || read32(8)!=3)return false; + ownsDevice=true;if(!configure())return false; + native_console_mmio=mapping->getPhysicalAddress();native_console_ready=1; + registerService();return true; +} +void NativeVirtioConsole::cleanup() { + if(gate)IOLockLock(gate); + bool stopped=reset(); + if(stopped) { + for(unsigned i=0;i<2;i++) {OSSafeReleaseNULL(queues[i].ring);OSSafeReleaseNULL(queues[i].buffer);} + OSSafeReleaseNULL(mapping);registers=0;ownsDevice=false; + } + // If reset fails, preserve device-reachable allocations instead of freeing + // storage while the device may still DMA. This path requires recovery. + if(gate)IOLockUnlock(gate); +} +void NativeVirtioConsole::stop(IOService *provider) {cleanup();IOService::stop(provider);} +void NativeVirtioConsole::free() {cleanup();if(gate){IOLockFree(gate);gate=nullptr;}IOService::free();} +bool NativeVirtioConsole::handleOpen(IOService *client,IOOptionBits,void *) {if(owner)return owner==client;owner=client;return true;} +void NativeVirtioConsole::handleClose(IOService *client,IOOptionBits) {if(owner==client)owner=nullptr;} +bool NativeVirtioConsole::handleIsOpen(const IOService *client) const {return client?owner==client:owner!=nullptr;} +IOReturn NativeVirtioConsole::newUserClient(task_t task,void *security,UInt32 type,OSDictionary *properties,IOUserClient **out) { + if(!out || type || !online)return kIOReturnBadArgument;*out=nullptr; + NativeVirtioConsoleClient *client=new NativeVirtioConsoleClient;if(!client)return kIOReturnNoMemory; + bool attached=false; + if(!client->initWithTask(task,security,type,properties) || !(attached=client->attach(this)) || !client->start(this)) { + if(attached)client->detach(this);client->release();return kIOReturnNotOpen; + } + *out=client;return kIOReturnSuccess; +} +bool NativeVirtioConsoleClient::initWithTask(task_t task,void *security,UInt32 type,OSDictionary *properties) { + if(type || clientHasPrivilege(security,kIOClientPrivilegeAdministrator)!=kIOReturnSuccess)return false; + return IOUserClient::initWithTask(task,security,type,properties); +} +bool NativeVirtioConsoleClient::start(IOService *provider) { + controller=OSDynamicCast(NativeVirtioConsole,provider); + if(!controller || !IOUserClient::start(provider))return false; + opened=controller->open(this);return opened; +} +void NativeVirtioConsoleClient::stop(IOService *provider) { + if(opened){controller->close(this);opened=false;}controller=nullptr;IOUserClient::stop(provider); +} +IOReturn NativeVirtioConsoleClient::clientClose() {terminate();return kIOReturnSuccess;} +IOReturn NativeVirtioConsoleClient::sendAction(OSObject *target,void *,IOExternalMethodArguments *args) { + NativeVirtioConsoleClient *client=OSDynamicCast(NativeVirtioConsoleClient,target); + if(!client || !client->opened || !client->controller)return kIOReturnNotOpen; + if(args->structureInputDescriptor)return kIOReturnUnsupported; + return client->controller->send(args->structureInput,args->structureInputSize); +} +IOReturn NativeVirtioConsoleClient::receiveAction(OSObject *target,void *,IOExternalMethodArguments *args) { + NativeVirtioConsoleClient *client=OSDynamicCast(NativeVirtioConsoleClient,target); + if(!client || !client->opened || !client->controller)return kIOReturnNotOpen; + if(args->structureOutputDescriptor)return kIOReturnUnsupported; + size_t size=args->structureOutputSize; + IOReturn status=client->controller->receive(args->structureOutput,&size);args->structureOutputSize=(uint32_t)size;return status; +} +IOReturn NativeVirtioConsoleClient::externalMethod(uint32_t selector,IOExternalMethodArguments *args,IOExternalMethodDispatch *,OSObject *,void *) { + static const IOExternalMethodDispatch methods[]={ + {sendAction,0,kIOUCVariableStructureSize,0,0}, + {receiveAction,0,0,0,kIOUCVariableStructureSize}}; + if(selector>=2)return kIOReturnUnsupported; + return IOUserClient::externalMethod(selector,args,const_cast(methods+selector),this,nullptr); +} diff --git a/drivers/NativeVirtioConsole/NativeVirtioConsoleTransport.h b/drivers/NativeVirtioConsole/NativeVirtioConsoleTransport.h new file mode 100644 index 0000000..5bf7314 --- /dev/null +++ b/drivers/NativeVirtioConsole/NativeVirtioConsoleTransport.h @@ -0,0 +1,6 @@ +// SPDX-License-Identifier: BSD-2-Clause-Patent +// Generic exclusive-owner kernel byte transport. No message/GPU semantics. +#pragma once +#include +extern "C" IOReturn native_console_send(IOService *,IOService *,const void *,size_t); +extern "C" IOReturn native_console_receive(IOService *,IOService *,void *,size_t *); diff --git a/kits/AdrenoFirmware/AdrenoFirmware.c b/kits/AdrenoFirmware/AdrenoFirmware.c new file mode 100644 index 0000000..e776c80 --- /dev/null +++ b/kits/AdrenoFirmware/AdrenoFirmware.c @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: BSD-2-Clause-Patent */ +#include "AdrenoFirmware.h" + +static uint32_t le32(const uint8_t *p) +{ + return (uint32_t)p[0] | (uint32_t)p[1] << 8 | + (uint32_t)p[2] << 16 | (uint32_t)p[3] << 24; +} + +AdrenoFirmwareResult AdrenoFirmwarePlanGmu( + const uint8_t *firmware, size_t bytes, + const AdrenoFirmwareRegion *regions, size_t region_count, + AdrenoFirmwareBlock *blocks, size_t capacity, size_t *count) +{ + if (!count) return AdrenoFirmwareInvalid; + *count = 0; + if (!firmware || !regions || !blocks || !region_count || region_count > 64) + return AdrenoFirmwareInvalid; + if (bytes < 16) return AdrenoFirmwareTruncated; + for (size_t i = 0; i < region_count; ++i) { + const AdrenoFirmwareRegion *a = regions + i; + if (!a->bytes || (a->address & 3) || (a->bytes & 3) || + a->bytes > UINT64_C(0x100000000) - a->address || a->identifier == UINT32_MAX) + return AdrenoFirmwareInvalid; + for (size_t j = 0; j < i; ++j) { + const AdrenoFirmwareRegion *b = regions + j; + if (a->identifier == b->identifier || + ((uint64_t)a->address < (uint64_t)b->address + b->bytes && + (uint64_t)b->address < (uint64_t)a->address + a->bytes)) + return AdrenoFirmwareInvalid; + } + } + for (unsigned pass = 0; pass < 2; ++pass) { + size_t offset = 0, n = 0; + while (offset < bytes) { + if (bytes - offset < 16) return AdrenoFirmwareTruncated; + uint32_t address = le32(firmware + offset); + uint32_t length = le32(firmware + offset + 4); + uint32_t identifier = UINT32_MAX; + if (length > bytes - offset - 16) return AdrenoFirmwareTruncated; + if (length) { + if ((address & 3) || (length & 3) || + length > UINT64_C(0x100000000) - address) + return AdrenoFirmwareInvalid; + for (size_t i = 0; i < region_count; ++i) { + if (address >= regions[i].address && length <= regions[i].bytes && + (uint64_t)address - regions[i].address <= regions[i].bytes - length) { + identifier = regions[i].identifier; + break; + } + } + if (identifier == UINT32_MAX) return AdrenoFirmwareUnmapped; + } + if (n == capacity) return AdrenoFirmwareNoSpace; + if (pass) { + blocks[n].address = address; + blocks[n].bytes = length; + blocks[n].type = le32(firmware + offset + 8); + blocks[n].value = le32(firmware + offset + 12); + blocks[n].payload_offset = offset + 16; + blocks[n].region_identifier = identifier; + } + n++; + offset += 16 + (size_t)length; + } + if (pass) *count = n; + } + return AdrenoFirmwareSuccess; +} diff --git a/kits/AdrenoFirmware/AdrenoFirmware.h b/kits/AdrenoFirmware/AdrenoFirmware.h new file mode 100644 index 0000000..9588fa7 --- /dev/null +++ b/kits/AdrenoFirmware/AdrenoFirmware.h @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: BSD-2-Clause-Patent */ +#ifndef ADRENO_FIRMWARE_H +#define ADRENO_FIRMWARE_H +#include +#include + +typedef struct { + uint32_t address; + uint64_t bytes; + uint32_t identifier; +} AdrenoFirmwareRegion; + +typedef struct { + uint32_t address, bytes, type, value; + size_t payload_offset; + uint32_t region_identifier; +} AdrenoFirmwareBlock; + +typedef enum { + AdrenoFirmwareSuccess, + AdrenoFirmwareInvalid, + AdrenoFirmwareTruncated, + AdrenoFirmwareUnmapped, + AdrenoFirmwareNoSpace +} AdrenoFirmwareResult; + +/* No output entries are changed on failure; *count is then zero. + * Zero-size metadata stays opaque, with region_identifier UINT32_MAX. + * Success validates structure/ranges, not firmware authenticity or execution. + */ +#ifdef __cplusplus +extern "C" { +#endif +AdrenoFirmwareResult AdrenoFirmwarePlanGmu( + const uint8_t *firmware, size_t bytes, + const AdrenoFirmwareRegion *regions, size_t region_count, + AdrenoFirmwareBlock *blocks, size_t capacity, size_t *count); +#ifdef __cplusplus +} +#endif +#endif diff --git a/kits/AdrenoFirmware/AdrenoHfi.c b/kits/AdrenoFirmware/AdrenoHfi.c new file mode 100644 index 0000000..45f1557 --- /dev/null +++ b/kits/AdrenoFirmware/AdrenoHfi.c @@ -0,0 +1,115 @@ +/* SPDX-License-Identifier: BSD-2-Clause-Patent */ +/* HFI wire facts from the source pinned in scripts/data/adreno-inputs.json. No Linux runtime. */ +#include "AdrenoHfi.h" +#if !defined(__aarch64__) || __BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__ +#error Modern GMU HFI transport requires little-endian AArch64 +#endif +#define WORDS 1024u +static void acquire(void){__asm__ volatile("dmb oshld":::"memory");} +static void publish(void){__asm__ volatile("dsb oshst":::"memory");} +static void complete(void){__asm__ volatile("dmb osh":::"memory");} +static AdrenoHfiStatus poison(AdrenoHfi *h,AdrenoHfiStatus status) +{h->poisoned=1;return status;} +static AdrenoHfiStatus valid(AdrenoHfi *h,unsigned q,volatile uint32_t **header, + volatile uint32_t **data,uint32_t *read,uint32_t *write) +{ + if(!h || !h->shared || !h->notify || !h->iova || (h->iova&4095) || + ((uintptr_t)h->shared&4095))return AdrenoHfiInvalid; + if(h->poisoned)return AdrenoHfiPoisoned; + volatile uint32_t *m=h->shared;acquire(); + if(m[0]!=0 || m[1]!=168 || m[2]!=6 || m[3]!=12 || m[4]!=3 || m[5]!=3) + return poison(h,AdrenoHfiCorrupt); + volatile uint32_t *v=m+6+12*q; + if(v[0]!=1 || v[1]!=h->iova+(q+1)*4096 || v[2]!=(0xa00|q) || + v[3]!=WORDS || v[4]!=0)return poison(h,AdrenoHfiCorrupt); + *read=v[10];*write=v[11];acquire(); + if(*read>=WORDS || *write>=WORDS || (*read&3) || (*write&3))return poison(h,AdrenoHfiCorrupt); + *header=v;*data=m+(q+1)*WORDS;return AdrenoHfiSuccess; +} +AdrenoHfiStatus AdrenoHfiInitializeStopped(AdrenoHfi *h,volatile uint32_t *shared, + size_t bytes,uint64_t iova,void *cookie,int (*notify)(void *)) +{ + if(!h || !shared || !notify || bytes!=16384 || ((uintptr_t)shared&4095) || + (uintptr_t)shared>UINTPTR_MAX-16383 || !iova || (iova&4095) || iova>UINT32_MAX-16383) + return AdrenoHfiInvalid; + for(unsigned i=0;i<4096;++i)shared[i]=0; + shared[1]=168;shared[2]=6;shared[3]=12;shared[4]=3;shared[5]=3; + for(unsigned q=0;q<3;++q) { + volatile uint32_t *v=shared+6+12*q; + v[0]=1;v[1]=(uint32_t)iova+(q+1)*4096;v[2]=0xa00|q;v[3]=WORDS; + v[6]=1;v[7]=1;v[8]=1; + } + publish(); + *h=(AdrenoHfi){shared,(uint32_t)iova,0,0,0,cookie,notify,0,0,0,0}; + return AdrenoHfiSuccess; +} +AdrenoHfiStatus AdrenoHfiSend(AdrenoHfi *h,unsigned id,const uint32_t *payload, + size_t words,uint32_t *wire_header) +{ + if(!wire_header || id>255 || words>254 || (words && !payload))return AdrenoHfiInvalid; + volatile uint32_t *v,*data;uint32_t read,write; + AdrenoHfiStatus status=valid(h,0,&v,&data,&read,&write); + if(status!=AdrenoHfiSuccess)return status; + if(h->pending_header)return AdrenoHfiBusy; + unsigned count=(unsigned)words+1,padded=(count+3)&~3u; + if(((read-write-1)&(WORDS-1))sequence+1)%4095; + uint32_t header=(sequence<<20)|(count<<8)|id; + data[write]=header; + for(unsigned i=0;ipending_header=header;h->sequence=sequence;*wire_header=header; + if(h->notify(h->cookie))return poison(h,AdrenoHfiNotifyError); + return AdrenoHfiSuccess; +} +AdrenoHfiStatus AdrenoHfiPoll(AdrenoHfi *h,uint32_t *payload,size_t capacity, + size_t *payload_words,uint32_t *error) +{ + if(!payload || capacity<16 || !payload_words || !error)return AdrenoHfiInvalid; + *payload_words=0;*error=0; + volatile uint32_t *v,*data;uint32_t read,write; + AdrenoHfiStatus status=valid(h,1,&v,&data,&read,&write); + if(status!=AdrenoHfiSuccess)return status; + if(!h->pending_header)return AdrenoHfiInvalid; + /* A ring holds at most WORDS/4 packets; each pass consumes one. */ + for(unsigned pass=0;pass>8)&255; + unsigned available=(write-read)&(WORDS-1),padded=(count+3)&~3u; + if(!count || count>19 || padded>available)return poison(h,AdrenoHfiCorrupt); + for(unsigned i=0;i=3 && (packet[1]>>20)==(h->pending_header>>20); + if(!match) { + if(count<2)return poison(h,AdrenoHfiCorrupt); + if(!h->skipped++) {h->skipped_header=header;h->skipped_ret=packet[1];h->skipped_code=count>2?packet[2]:0;} + read=(read+padded)&(WORDS-1);v[10]=read;publish(); + continue; + } + if(packet[2]) {*error=packet[2];return poison(h,AdrenoHfiFirmwareError);} + /* Copy only the checked matching response, never padding. This is not + * cryptographic authentication of firmware or its response. */ + for(unsigned i=3;ipending_header=0;*payload_words=count-3;return AdrenoHfiSuccess; + } + return poison(h,AdrenoHfiCorrupt); +} +AdrenoHfiStatus AdrenoHfiExchange(AdrenoHfi *h,unsigned id,const uint32_t *input, + size_t words,unsigned budget,int (*wait_once)(void *),uint32_t *output, + size_t capacity,size_t *count,uint32_t *error) +{ + if(!budget || budget>1048576 || !output || capacity<16 || !count || !error) + return AdrenoHfiInvalid; + *count=0;*error=0;uint32_t header; + AdrenoHfiStatus status=AdrenoHfiSend(h,id,input,words,&header); + if(status!=AdrenoHfiSuccess)return status; + for(unsigned i=0;icookie))return poison(h,AdrenoHfiWaitError); + } + return poison(h,AdrenoHfiTimeout); +} diff --git a/kits/AdrenoFirmware/AdrenoHfi.h b/kits/AdrenoFirmware/AdrenoHfi.h new file mode 100644 index 0000000..15986b5 --- /dev/null +++ b/kits/AdrenoFirmware/AdrenoHfi.h @@ -0,0 +1,60 @@ +/* SPDX-License-Identifier: BSD-2-Clause-Patent */ +#ifndef ADRENO_HFI_H +#define ADRENO_HFI_H +#include +#include +#ifdef __cplusplus +extern "C" { +#endif +typedef enum { + AdrenoHfiSuccess,AdrenoHfiInvalid,AdrenoHfiBusy,AdrenoHfiNoSpace, + AdrenoHfiEmpty,AdrenoHfiCorrupt,AdrenoHfiUnexpected,AdrenoHfiFirmwareError, + AdrenoHfiNotifyError,AdrenoHfiWaitError,AdrenoHfiTimeout,AdrenoHfiPoisoned +} AdrenoHfiStatus; +typedef struct { + volatile uint32_t *shared; + uint32_t iova,sequence,pending_header,poisoned; + void *cookie; + int (*notify)(void *); /* real HOST2GMU interrupt write, 0=success */ + /* Diagnostics of skipped response packets (a6xx_hfi_wait_for_ack skips firmware error messages and replies to + * another sequence number): their count and the first one's header, return header and code. */ + uint32_t skipped,skipped_header,skipped_ret,skipped_code; +} AdrenoHfi; +/* Modern (nonlegacy) GMU HFI only. Caller owns an exclusive, 16KiB extent + * mapped with four native4K pages, coherent for CPU and GMU, and a valid 32bit + * GMU IOVA. Coherence, ownership and active mappings are platform obligations, + * not established by this API. The GMU MUST be stopped and unable to access + * this extent during initialization. No reset of a running queue is allowed. + * AArch64 little-endian; real outer-shareable barriers, no fake ACKs. + * All calls on an instance are serialized by the caller. Shared memory, + * object and message/output storage must be disjoint and remain owned. + * Never free/reuse shared storage until an external real stop+TLB drain is + * proved. Poison/timeout/notify failure do not establish safe release. */ +AdrenoHfiStatus AdrenoHfiInitializeStopped(AdrenoHfi *,volatile uint32_t *, + size_t shared_bytes,uint64_t iova,void *cookie,int (*notify)(void *)); +/* payload excludes the wire header. One command may be outstanding. A failed + * notify after publishing is indeterminate: poisons the session, no retry. + * NoSpace/Busy refuse before publishing. No firmware readiness is inferred. */ +AdrenoHfiStatus AdrenoHfiSend(AdrenoHfi *,unsigned id,const uint32_t *payload, + size_t payload_words,uint32_t *wire_header); +/* Poll the response queue. As a6xx_hfi_wait_for_ack: the reply whose return + * header carries the pending command's sequence number completes it (error=0 + * succeeds, otherwise FirmwareError); F2H error messages and replies to other + * sequence numbers are consumed, counted and skipped. payload_words reports + * actual returned words; output remains untouched on failure. Capacity must be + * >=16, the protocol's maximum ACK payload. Empty keeps the pending command. + * Malformed queues or packets poison the session; no success or queue reset is + * synthesized. Debug queue not drained. */ +AdrenoHfiStatus AdrenoHfiPoll(AdrenoHfi *,uint32_t *payload,size_t capacity, + size_t *payload_words,uint32_t *firmware_error); +/* Serialized command + bounded response polling. Poll budget limits software + * iterations, not hardware elapsed microseconds. Optional wait callback must + * return promptly; an error poisons the session. Timeout also poisons it. + * This transport alone performs no power/firmware/performance-table startup. */ +AdrenoHfiStatus AdrenoHfiExchange(AdrenoHfi *,unsigned id,const uint32_t *,size_t, + unsigned poll_budget,int (*wait_once)(void *),uint32_t *,size_t, + size_t *,uint32_t *firmware_error); +#ifdef __cplusplus +} +#endif +#endif diff --git a/kits/AdrenoFirmware/AdrenoHfiStartup.c b/kits/AdrenoFirmware/AdrenoHfiStartup.c new file mode 100644 index 0000000..a136768 --- /dev/null +++ b/kits/AdrenoFirmware/AdrenoHfiStartup.c @@ -0,0 +1,157 @@ +/* SPDX-License-Identifier: BSD-2-Clause-Patent */ +/* Independent serialization from the pinned modern HFI wire contract. */ +#include "AdrenoHfiStartup.h" +static int zero(const uint32_t *v,size_t n) +{for(size_t i=0;imax || v[0].hz)return 0; + for(unsigned i=0;iUINT32_MAX || (i && v[i].hz<=v[i-1].hz))return 0; + for(unsigned i=n;imax)return 0; + for(unsigned i=0;igpu,c->gpu_count,16) || !levels(c->gmu,c->gmu_count,4) || + !c->bandwidth_count || c->bandwidth_count>16 || c->enable_ifpc>1 || + !addresses(c->cnoc_addresses,c->cnoc_count,6) || + !addresses(c->ddr_addresses,c->ddr_count,8))return AdrenoHfiStartupInvalid; + if(c->cnoc_wait_mask>>c->cnoc_count || c->ddr_wait_mask>>c->ddr_count)return AdrenoHfiStartupInvalid; + for(unsigned i=0;icnoc_count;++i) + for(unsigned j=0;jddr_count;++j)if(c->cnoc_addresses[i]==c->ddr_addresses[j])return AdrenoHfiStartupInvalid; + for(unsigned i=0;i<2;++i)if(!zero(c->cnoc_data[i]+c->cnoc_count,6-c->cnoc_count))return AdrenoHfiStartupInvalid; + for(unsigned i=0;i<16;++i) + if(!zero(c->ddr_data[i]+(ibandwidth_count?c->ddr_count:0), + 8-(ibandwidth_count?c->ddr_count:0)))return AdrenoHfiStartupInvalid; + if(!c->acd_enable_mask) { + if(c->acd_version || c->acd_stride || c->acd_count || !zero(c->acd_data,32))return AdrenoHfiStartupInvalid; + } else { + if(c->acd_version!=1 || c->acd_stride!=1 || (c->acd_enable_mask&1) || + c->acd_enable_mask>>c->gpu_count)return AdrenoHfiStartupInvalid; + unsigned count=0; + for(uint32_t bits=c->acd_enable_mask;bits;bits>>=1)count+=bits&1; + if(c->acd_count!=count || !zero(c->acd_data+count,32-count))return AdrenoHfiStartupInvalid; + } + return AdrenoHfiStartupSuccess; +} +AdrenoHfiStartupStatus AdrenoHfiStartupEncode(const AdrenoHfiStartupConfig *c, + AdrenoHfiStartupStage stage,uint32_t *id,uint32_t *p,size_t capacity,size_t *count) +{ + if(!count)return AdrenoHfiStartupInvalid; + *count=0; + if(!id || !p || capacity<159 || AdrenoHfiStartupValidate(c)!=AdrenoHfiStartupSuccess || + stageAdrenoHfiStageStart || + ((stage==AdrenoHfiStageAcdEnable || stage==AdrenoHfiStageAcdTable) && !c->acd_enable_mask) || + (stage==AdrenoHfiStageIfpc && !c->enable_ifpc))return AdrenoHfiStartupInvalid; + size_t n=0;unsigned message=0; + switch(stage) { + case AdrenoHfiStagePerformance: + message=4;n=58; + for(size_t i=0;igpu_count;p[1]=c->gmu_count; + for(unsigned i=0;igpu_count;++i) { + p[2+3*i]=c->gpu[i].vote;p[3+3*i]=UINT32_MAX;p[4+3*i]=(uint32_t)(c->gpu[i].hz/1000); + } + for(unsigned i=0;igmu_count;++i) { + p[50+2*i]=c->gmu[i].vote;p[51+2*i]=(uint32_t)(c->gmu[i].hz/1000); + } + break; + case AdrenoHfiStageBandwidth: + message=3;n=159; + p[0]=c->bandwidth_count;p[1]=c->cnoc_count;p[2]=c->ddr_count; + p[3]=c->cnoc_wait_mask;p[4]=c->ddr_wait_mask; + for(unsigned i=0;i<6;++i)p[5+i]=c->cnoc_addresses[i]; + for(unsigned i=0;i<12;++i)p[11+i]=c->cnoc_data[i/6][i%6]; + for(unsigned i=0;i<8;++i)p[23+i]=c->ddr_addresses[i]; + for(unsigned i=0;i<128;++i)p[31+i]=c->ddr_data[i/8][i%8]; + break; + case AdrenoHfiStageAcdEnable:message=11;n=3;p[0]=12;p[1]=1;p[2]=0;break; + case AdrenoHfiStageAcdTable: + message=7;n=36;p[0]=c->acd_version;p[1]=c->acd_enable_mask;p[2]=c->acd_stride;p[3]=c->acd_count; + for(unsigned i=0;i<32;++i)p[4+i]=c->acd_data[i]; + break; + case AdrenoHfiStageIfpc:message=11;n=3;p[0]=9;p[1]=1;p[2]=0x1680;break; + case AdrenoHfiStageCoreStart:message=14;n=1;p[0]=0;break; + case AdrenoHfiStageStart:message=10;n=0;break; + default:return AdrenoHfiStartupInvalid; + } + *id=message;*count=n;return AdrenoHfiStartupSuccess; +} +AdrenoHfiStartupStatus AdrenoHfiStartupValidateServices(const AdrenoHfiStartupServices *s) +{ + if(!s || !s->now_us || !s->wait_us || !s->check_health || + !s->polls_per_command || s->polls_per_command>1048576 || + !s->wait_interval_us || s->wait_interval_us>1000000 || + !s->command_timeout_us || s->command_timeout_us>10000000 || + s->total_timeout_uscommand_timeout_us || s->total_timeout_us>60000000) + return AdrenoHfiStartupInvalid; + return AdrenoHfiStartupSuccess; +} +static AdrenoHfiStartupStatus clock_check(const AdrenoHfiStartupServices *s, + uint64_t begin,uint64_t command,uint64_t *last,AdrenoHfiStartupResult *r) +{ + uint64_t now; + if(s->now_us(s->cookie,&now) || now<*last)return AdrenoHfiStartupClockError; + *last=now;r->elapsed_us=now-begin; + if(now-begin>=s->total_timeout_us || now-command>=s->command_timeout_us)return AdrenoHfiStartupTimeout; + return AdrenoHfiStartupSuccess; +} +AdrenoHfiStartupStatus AdrenoHfiStartupRun(AdrenoHfiStartup *state,AdrenoHfi *h, + const AdrenoHfiStartupConfig *c,const AdrenoHfiStartupServices *s,AdrenoHfiStartupResult *r) +{ + if(!r)return AdrenoHfiStartupInvalid; + *r=(AdrenoHfiStartupResult){0};r->resources_must_retain=1; + if(!state || !h || AdrenoHfiStartupValidateServices(s)!=AdrenoHfiStartupSuccess || + AdrenoHfiStartupValidate(c)!=AdrenoHfiStartupSuccess)return AdrenoHfiStartupInvalid; + if(state->poisoned || h->poisoned)return AdrenoHfiStartupPoisoned; + if(state->attempted || state->completed || h->pending_header)return AdrenoHfiStartupBusy; + uint64_t begin,last,command; + if(s->now_us(s->cookie,&begin))return AdrenoHfiStartupClockError; + last=begin; + if(s->check_health(s->cookie))return AdrenoHfiStartupPlatformFault; + state->attempted=1; + AdrenoHfiStartupStatus status=AdrenoHfiStartupSuccess; + uint32_t payload[159],output[16],id;size_t words,returned; + for(unsigned step=AdrenoHfiStagePerformance;step<=AdrenoHfiStageStart;++step) { + if((step==AdrenoHfiStageAcdEnable || step==AdrenoHfiStageAcdTable) && !c->acd_enable_mask)continue; + if(step==AdrenoHfiStageIfpc && !c->enable_ifpc)continue; + r->stage=(AdrenoHfiStartupStage)step;command=last; + status=clock_check(s,begin,command,&last,r);if(status)goto failed; + if(s->check_health(s->cookie)) {status=AdrenoHfiStartupPlatformFault;goto failed;} + status=AdrenoHfiStartupEncode(c,r->stage,&id,payload,159,&words);if(status)goto failed; + r->wire_header=0; + r->transport_status=AdrenoHfiSend(h,id,payload,words,&r->wire_header); + if(r->wire_header)++r->published_commands; + if(r->transport_status!=AdrenoHfiSuccess) {status=AdrenoHfiStartupTransportError;goto failed;} + unsigned acknowledged=0; + for(unsigned i=0;ipolls_per_command;++i) { + status=clock_check(s,begin,command,&last,r);if(status)goto failed; + if(s->check_health(s->cookie)) {status=AdrenoHfiStartupPlatformFault;goto failed;} + ++r->poll_reads; + r->transport_status=AdrenoHfiPoll(h,output,16,&returned,&r->firmware_error); + if(r->transport_status==AdrenoHfiSuccess) { + status=clock_check(s,begin,command,&last,r);if(status)goto failed; + if(s->check_health(s->cookie)) {status=AdrenoHfiStartupPlatformFault;goto failed;} + r->acknowledged_stages|=1u<transport_status!=AdrenoHfiEmpty) {status=AdrenoHfiStartupTransportError;goto failed;} + if(i+1polls_per_command && s->wait_us(s->cookie,s->wait_interval_us)) { + status=AdrenoHfiStartupWaitError;goto failed; + } + } + if(!acknowledged) {status=AdrenoHfiStartupTimeout;goto failed;} + } + state->completed=1;r->stage=AdrenoHfiStageComplete;return AdrenoHfiStartupSuccess; +failed: + state->poisoned=1;h->poisoned=1;return status; +} diff --git a/kits/AdrenoFirmware/AdrenoHfiStartup.h b/kits/AdrenoFirmware/AdrenoHfiStartup.h new file mode 100644 index 0000000..0496c6f --- /dev/null +++ b/kits/AdrenoFirmware/AdrenoHfiStartup.h @@ -0,0 +1,75 @@ +/* SPDX-License-Identifier: BSD-2-Clause-Patent */ +#ifndef ADRENO_HFI_STARTUP_H +#define ADRENO_HFI_STARTUP_H +#include "AdrenoHfi.h" +#ifdef __cplusplus +extern "C" { +#endif +/* Modern A6xx/A7xx wire profile. Legacy and A8xx table formats are excluded. */ +typedef struct {uint64_t hz;uint32_t vote;} AdrenoHfiPerformanceLevel; +typedef struct { + unsigned gpu_count,gmu_count; + AdrenoHfiPerformanceLevel gpu[16],gmu[4]; + unsigned bandwidth_count,cnoc_count,ddr_count; + uint32_t cnoc_wait_mask,ddr_wait_mask; + uint32_t cnoc_addresses[6],cnoc_data[2][6]; + uint32_t ddr_addresses[8],ddr_data[16][8]; + uint32_t acd_version,acd_enable_mask,acd_stride,acd_count,acd_data[32]; + unsigned enable_ifpc; +} AdrenoHfiStartupConfig; +typedef enum { + AdrenoHfiStageNone,AdrenoHfiStagePerformance,AdrenoHfiStageBandwidth, + AdrenoHfiStageAcdEnable,AdrenoHfiStageAcdTable,AdrenoHfiStageIfpc, + AdrenoHfiStageCoreStart,AdrenoHfiStageStart,AdrenoHfiStageComplete +} AdrenoHfiStartupStage; +typedef enum { + AdrenoHfiStartupSuccess,AdrenoHfiStartupInvalid,AdrenoHfiStartupBusy, + AdrenoHfiStartupPoisoned,AdrenoHfiStartupClockError,AdrenoHfiStartupWaitError, + AdrenoHfiStartupTimeout,AdrenoHfiStartupPlatformFault,AdrenoHfiStartupTransportError +} AdrenoHfiStartupStatus; +typedef struct { + void *cookie; + int (*now_us)(void *,uint64_t *); + int (*wait_us)(void *,unsigned); + int (*check_health)(void *); /* real platform fault/ownership checks */ + unsigned polls_per_command,wait_interval_us; + uint64_t command_timeout_us,total_timeout_us; +} AdrenoHfiStartupServices; +typedef struct {unsigned attempted,poisoned,completed;} AdrenoHfiStartup; +typedef struct { + AdrenoHfiStartupStage stage; + AdrenoHfiStatus transport_status; + uint32_t firmware_error,wire_header,acknowledged_stages; + unsigned published_commands,poll_reads,resources_must_retain; + uint64_t elapsed_us; +} AdrenoHfiStartupResult; +/* Validate every input before any queue publication. Frequencies include the + * zero/off entry, are strictly ascending, exactly representable in wire kHz. + * Votes and command-db addresses are trusted platform inputs, never guessed + * by this library. ACD entries are packed for each enabled GPU level; AOSS + * notification and IFPC power prerequisites must already be established. + * Unused fixed-capacity slots must be zero. This prevents silently ignoring + * unsupported or malformed configuration. No input is authenticated here. */ +AdrenoHfiStartupStatus AdrenoHfiStartupValidate(const AdrenoHfiStartupConfig *); +AdrenoHfiStartupStatus AdrenoHfiStartupValidateServices(const AdrenoHfiStartupServices *); +/* Payload excludes header. All output storage must be disjoint from inputs. + * Encodes one selected stage; unused optional stages refuse. Failure leaves + * payload/header outputs untouched except word_count=0. Capacity >=159. */ +AdrenoHfiStartupStatus AdrenoHfiStartupEncode(const AdrenoHfiStartupConfig *, + AdrenoHfiStartupStage,uint32_t *id,uint32_t *payload,size_t capacity,size_t *word_count); +/* Single attempt, serialized, caller-owned zero-initialized state. Requires + * a real ready GMU, exclusive coherent HFI allocation and active native4K GMU + * mappings. Only matching actual ACKs advance the startup sequence. All + * commands and the whole sequence have monotonic-time AND iteration bounds. + * Callback operations must themselves be bounded. On any attempted-sequence + * failure the transport is poisoned; no replay, synthetic ACK or implicit + * stop occurs. Even success does not authorize resource release or prove GPU + * execution, power/coherence/SMMU readiness, shader submission or a desktop. + * Input, state, services, transport, result and shared queues must remain + * disjoint and pinned; config remains immutable throughout the sequence. */ +AdrenoHfiStartupStatus AdrenoHfiStartupRun(AdrenoHfiStartup *,AdrenoHfi *, + const AdrenoHfiStartupConfig *,const AdrenoHfiStartupServices *,AdrenoHfiStartupResult *); +#ifdef __cplusplus +} +#endif +#endif diff --git a/kits/AdrenoFirmware/AdrenoHfiVotes.c b/kits/AdrenoFirmware/AdrenoHfiVotes.c new file mode 100644 index 0000000..f3ce145 --- /dev/null +++ b/kits/AdrenoFirmware/AdrenoHfiVotes.c @@ -0,0 +1,89 @@ +/* SPDX-License-Identifier: BSD-2-Clause-Patent */ +#include "AdrenoHfiVotes.h" +static uint16_t le16(const uint8_t *p){return (uint16_t)(p[0]|(uint16_t)p[1]<<8);} +static uint32_t le32(const uint8_t *p){return le16(p)|(uint32_t)le16(p+2)<<16;} +static int rail(const uint8_t *p,size_t bytes) +{ + if(!p || !bytes || (bytes&1) || bytes>512 || le16(p))return 0; + unsigned padding=0;uint16_t last=0; + for(size_t i=2;i16 || capacityUINT16_MAX || (i && levels[i]=levels[i]) {s=j;break;} + if(value)s=j; + } + result[i]=(uint32_t)le16(primary+2*p)<<16 | (uint32_t)s<<8 | (uint32_t)p; + } + for(size_t i=0;iunit=le32(p);b->width=le16(p+4);b->vcd=p[6];return AdrenoHfiStartupSuccess; +} +static int address(uint32_t a){return a && !(a&3);} +AdrenoHfiStartupStatus AdrenoHfiBuildBandwidth(const uint32_t *kbps,size_t count, + const AdrenoHfiBcm *bcms,size_t n,uint32_t cn0,AdrenoHfiStartupConfig *c) +{ + if(!c || !kbps || !bcms || !count || count>16 || kbps[0] || !n || n>3 || !address(cn0)) + return AdrenoHfiStartupInvalid; + for(size_t i=1;iaddress) || b->address==cn0 || b->fixed>1 || b->perfmode>0x3fff || + (!b->fixed && (!b->unit || !b->width || !b->buswidth || b->perfmode || b->perfmode_kbps))) + return AdrenoHfiStartupInvalid; + for(size_t k=0;kaddress==bcms[k].address)return AdrenoHfiStartupInvalid; + } + /* All input validation is finished before changing caller storage. */ + c->bandwidth_count=(unsigned)count;c->cnoc_count=1;c->ddr_count=(unsigned)n; + c->cnoc_wait_mask=1;c->ddr_wait_mask=0; + for(size_t i=0;i<6;++i) { + c->cnoc_addresses[i]=i?0:cn0;c->cnoc_data[0][i]=i?0:0x40000000; + c->cnoc_data[1][i]=i?0:0x60000001; + } + for(size_t j=0;j<8;++j) { + c->ddr_addresses[j]=jddr_wait_mask|=1u<fixed) { + if(b->perfmode_kbps && kbps[i]>=b->perfmode_kbps)data|=b->perfmode; + } else { + uint64_t peak=(uint64_t)kbps[i]*b->width/b->buswidth; + peak=peak*1000/b->unit; + uint32_t vote=peak>0x3fff?0x3fff:peak?(uint32_t)peak:1; + data|=vote<<14|vote; + } + } + } + c->ddr_data[i][j]=data; + } + } + return AdrenoHfiStartupSuccess; +} diff --git a/kits/AdrenoFirmware/AdrenoHfiVotes.h b/kits/AdrenoFirmware/AdrenoHfiVotes.h new file mode 100644 index 0000000..48081ff --- /dev/null +++ b/kits/AdrenoFirmware/AdrenoHfiVotes.h @@ -0,0 +1,42 @@ +/* SPDX-License-Identifier: BSD-2-Clause-Patent */ +#ifndef ADRENO_HFI_VOTES_H +#define ADRENO_HFI_VOTES_H +#include "AdrenoHfiStartup.h" +#ifdef __cplusplus +extern "C" { +#endif +/* CPU-side generation only, no RPMh transaction or power transition. + * Input levels are from exact enabled OPPs (including off=0), not frequencies + * or voltages. Rail byte spans are original little-endian cmd-db aux data. + * Caller selects actual primary/secondary rails (including GMxC discovery). + * Rail profile: off at entry zero, ascending active levels (duplicates allowed), + * optional zero padding at the end. Other profiles explicitly refuse. + * Supports <=256 rail entries, <=16 OPPs; no truncation or guessed fallback. + * Output remains unchanged on failure. Inputs/outputs must be disjoint and + * immutable during calls. Successful arithmetic does not authenticate inputs. */ +AdrenoHfiStartupStatus AdrenoHfiArcVotes(const uint32_t *levels,size_t level_count, + const uint8_t *primary,size_t primary_bytes,const uint8_t *secondary,size_t secondary_bytes, + uint32_t *votes,size_t capacity); +typedef struct { + uint32_t address,unit; + uint16_t width,buswidth; + uint8_t vcd,fixed; + uint16_t perfmode; + uint32_t perfmode_kbps; +} AdrenoHfiBcm; +/* Decode exactly eight bytes of BCM aux data; address/buswidth/fixed/perfmode + * are separately supplied source/platform policy and are preserved. */ +AdrenoHfiStartupStatus AdrenoHfiBcmAux(AdrenoHfiBcm *,const uint8_t *,size_t); +/* A7xx BCM arithmetic and modern HFI bandwidth layout. Off=0, then strictly + * ascending KB/s OPP bandwidths, <=16. Up to three platform-ordered BCMs; + * CN0 address must come from command-db. No source fallback address exists. + * Unit/width come from BCM aux; buswidth/fixed/perfmode from target catalog. + * Fills only bandwidth-related config fields, preserving performance/features. + * Failure leaves the entire config unchanged. Success requires no HFI/device. + * Does not validate actual rail/BCM ownership, limits, speed bin or power. */ +AdrenoHfiStartupStatus AdrenoHfiBuildBandwidth(const uint32_t *kbps,size_t level_count, + const AdrenoHfiBcm *bcms,size_t bcm_count,uint32_t cn0_address,AdrenoHfiStartupConfig *); +#ifdef __cplusplus +} +#endif +#endif diff --git a/kits/AdrenoFirmware/LICENSE b/kits/AdrenoFirmware/LICENSE new file mode 100644 index 0000000..217b4df --- /dev/null +++ b/kits/AdrenoFirmware/LICENSE @@ -0,0 +1,51 @@ +Copyright (c) 2026, ArmVirtPlatform contributors. All rights reserved. + +SPDX-License-Identifier: BSD-2-Clause-Patent + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +Subject to the terms and conditions of this license, each copyright holder +and contributor hereby grants to those receiving rights under this license +a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable +(except for failure to satisfy the conditions of this license) patent +license to make, have made, use, offer to sell, sell, import, and otherwise +transfer this software, where such license applies only to those patent +claims, already acquired or hereafter acquired, licensable by such copyright +holder or contributor that are necessarily infringed by: + +(a) their Contribution(s) (the licensed copyrights of copyright holders and + non-copyrightable additions of contributors, in source or binary form) + alone; or + +(b) combination of their Contribution(s) with the work of authorship to + which such Contribution(s) was added by such copyright holder or + contributor, if, at the time the Contribution is added, such addition + causes such combination to be necessarily infringed. The patent license + shall not apply to any other combinations which include the + Contribution. + +Except as expressly stated above, no rights or licenses from any copyright +holder or contributor is granted under this license, whether expressly, by +implication, estoppel or otherwise. + +DISCLAIMER + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. diff --git a/kits/AdrenoFirmware/test_gmu.c b/kits/AdrenoFirmware/test_gmu.c new file mode 100644 index 0000000..610deff --- /dev/null +++ b/kits/AdrenoFirmware/test_gmu.c @@ -0,0 +1,73 @@ +/* SPDX-License-Identifier: BSD-2-Clause-Patent */ +#include "AdrenoFirmware.h" +#include +#include +#include +#include + +static const AdrenoFirmwareRegion regions[] = { + {0, 0x4000, 1}, {0x4000, 0xffc000, 2}, {0x10004000, 0x4000, 3} +}; + +static void refusal(const uint8_t *data, size_t bytes, + const AdrenoFirmwareRegion *ranges, size_t range_count, + size_t capacity, AdrenoFirmwareResult expected) +{ + AdrenoFirmwareBlock blocks[32], before[32]; + memset(blocks, 0xa5, sizeof(blocks)); + memcpy(before, blocks, sizeof(blocks)); + size_t count = 123; + AdrenoFirmwareResult result = AdrenoFirmwarePlanGmu(data, bytes, ranges, range_count, blocks, capacity, &count); + assert(result != AdrenoFirmwareSuccess); + if (expected != AdrenoFirmwareSuccess) assert(result == expected); + assert(count == 0 && !memcmp(blocks, before, sizeof(blocks))); +} + +int main(int argc, char **argv) +{ + assert(argc == 2); + FILE *file = fopen(argv[1], "rb"); + assert(file && !fseek(file, 0, SEEK_END)); + long length = ftell(file); + assert(length == 66844 && !fseek(file, 0, SEEK_SET)); + uint8_t *data = malloc((size_t)length); + assert(data && fread(data, 1, (size_t)length, file) == (size_t)length); + fclose(file); + AdrenoFirmwareBlock blocks[32]; + size_t count = 0; + assert(AdrenoFirmwarePlanGmu(data, length, regions, 3, blocks, 32, &count) == AdrenoFirmwareSuccess && count == 15); + unsigned payloads = 0; + size_t payload_bytes = 0; + for (size_t i = 0; i < count; ++i) { + payloads += blocks[i].bytes != 0; + payload_bytes += blocks[i].bytes; + if (!blocks[i].bytes) assert(blocks[i].region_identifier == UINT32_MAX); + } + assert(payloads == 4 && payload_bytes == 66604); + assert(blocks[11].address == 0 && blocks[11].bytes == 0x1dc && blocks[11].region_identifier == 1); + assert(blocks[12].address == 0x4000 && blocks[12].bytes == 0x10210 && blocks[12].region_identifier == 2); + assert(blocks[13].address == 0x10007f98 && blocks[13].bytes == 0x28 && blocks[13].region_identifier == 3); + assert(blocks[14].address == 0x10007fdc && blocks[14].bytes == 0x18 && blocks[14].region_identifier == 3); + unsigned valid_prefixes = 0; + for (size_t bytes = 0; bytes < (size_t)length; ++bytes) { + count = 0; + AdrenoFirmwareResult result = AdrenoFirmwarePlanGmu(data, bytes, regions, 3, blocks, 32, &count); + if (result == AdrenoFirmwareSuccess) valid_prefixes++; + else refusal(data, bytes, regions, 3, 32, AdrenoFirmwareSuccess); + } + /* Ending on a complete record is structurally valid, not authentic firmware. */ + assert(valid_prefixes == 14); + refusal(data, length, regions, 3, 14, AdrenoFirmwareNoSpace); + refusal(data, length, regions, 2, 32, AdrenoFirmwareUnmapped); + AdrenoFirmwareRegion bad[3]; + memcpy(bad, regions, sizeof(bad)); bad[1].address = 0x3ffc; + refusal(data, length, bad, 3, 32, AdrenoFirmwareInvalid); + memcpy(bad, regions, sizeof(bad)); bad[1].identifier = 1; + refusal(data, length, bad, 3, 32, AdrenoFirmwareInvalid); + memcpy(bad, regions, sizeof(bad)); bad[2].bytes -= 0x10; + refusal(data, length, bad, 3, 32, AdrenoFirmwareUnmapped); + data[0xb4] = 0xff; data[0xb5] = 0xff; data[0xb6] = 0xff; data[0xb7] = 0xff; + refusal(data, length, regions, 3, 32, AdrenoFirmwareTruncated); + free(data); + puts("PASS: actual 15-record GMU plan, all 66844 prefixes and six explicit refusal controls; no partial output"); +} diff --git a/kits/AdrenoFirmware/test_hfi.c b/kits/AdrenoFirmware/test_hfi.c new file mode 100644 index 0000000..2e1e9ea --- /dev/null +++ b/kits/AdrenoFirmware/test_hfi.c @@ -0,0 +1,155 @@ +/* SPDX-License-Identifier: BSD-2-Clause-Patent */ +/* All peers below are software fixtures; no firmware/device is executed. */ +#include "AdrenoHfi.h" +#include +#include +#include +#define CHECK(x) do {if(!(x)){fprintf(stderr,"line %d: %s\n",__LINE__,#x);exit(1);}}while(0) +static _Alignas(4096) uint32_t shared[4096],before[4096]; +static AdrenoHfi h;static unsigned notified,notify_fail,auto_reply,waits,reply_at; +static uint32_t input[254],output[16]; +#ifdef SOURCE_ORACLE +static _Alignas(4096) uint32_t reference[4096]; +extern void source_initialize(uint32_t *,uint32_t); +extern int source_write(uint32_t *,uint32_t,uint32_t *,uint32_t); +#endif +static void response(unsigned count,unsigned id,uint32_t ret,uint32_t error) +{ + uint32_t *v=shared+18;unsigned at=v[11]; + CHECK(count && count<=255); + for(unsigned i=0;i1)shared[2048+((at+1)&1023)]=ret; + if(count>2)shared[2048+((at+2)&1023)]=error; + v[11]=(at+((count+3)&~3u))&1023; +} +static int notify(void *cookie) +{ + CHECK(cookie==&h);++notified; + if(auto_reply)response(4,126,h.pending_header,0); + return notify_fail; +} +static int wait_once(void *cookie) +{CHECK(cookie==&h);if(++waits==reply_at)response(4,126,h.pending_header,0);return 0;} +static int failed_wait(void *cookie) +{CHECK(cookie==&h);++waits;return 1;} +static void reset(void) +{ + notified=notify_fail=auto_reply=waits=reply_at=0; + memset(&h,0,sizeof(h));memset(shared,0xa5,sizeof(shared)); + CHECK(AdrenoHfiInitializeStopped(&h,shared,sizeof(shared),0x60000000,&h,notify)==AdrenoHfiSuccess); + CHECK(shared[1]==168 && shared[2]==6 && shared[3]==12 && shared[4]==3 && shared[5]==3); + for(unsigned q=0;q<3;++q) { + uint32_t *v=shared+6+12*q; + CHECK(v[0]==1 && v[1]==0x60000000+(q+1)*4096 && v[2]==(0xa00|q) && v[3]==1024); + CHECK(v[4]==0 && v[5]==0 && v[6]==1 && v[7]==1 && v[8]==1 && v[9]==0 && !v[10] && !v[11]); + } +#ifdef SOURCE_ORACLE + memset(reference,0,sizeof(reference));source_initialize(reference,h.iova); + CHECK(!memcmp(shared,reference,sizeof(shared))); +#endif +} +static void untouched_output(void) +{for(unsigned i=0;i<16;++i)CHECK(output[i]==0x5a5a5a5a);} +int main(void) +{ + for(unsigned i=0;i<254;++i)input[i]=0xabcdef00+i; + size_t count;uint32_t error,header;unsigned cases=0; + for(unsigned wrap=0;wrap<2;++wrap)for(unsigned words=0;words<255;++words) { + reset();unsigned at=wrap?1020:0;shared[16]=shared[17]=at; + CHECK(AdrenoHfiSend(&h,14,input,words,&header)==AdrenoHfiSuccess && notified==1); + CHECK(header==((1u<<20)|((words+1)<<8)|14) && h.pending_header==header); + CHECK(shared[1024+at]==header); + for(unsigned i=0;i +#include +#include +#define CHECK(x) do {if(!(x)){fprintf(stderr,"line %d: %s\n",__LINE__,#x);exit(1);}}while(0) +static uint32_t *shared; +static AdrenoHfi hfi; +static AdrenoHfiStartup state; +static AdrenoHfiStartupConfig config; +static AdrenoHfiStartupServices services; +static AdrenoHfiStartupResult result; +static struct { + uint64_t time,jump; + unsigned count,waits,health_calls,now_calls; + unsigned fail_at,mode,clock_fail,clock_reverse,health_fail,wait_fail,clock_stall; + unsigned ids[8],words[8];uint32_t messages[8][159]; +} peer; +#ifdef SOURCE_ORACLE +extern int source_startup(const AdrenoHfiStartupConfig *,unsigned,unsigned *,unsigned *,unsigned *,uint32_t [8][159]); +#endif +static void response(uint32_t command) +{ + volatile uint32_t *q=shared+18;unsigned at=q[11]; + shared[2048+at]=(3u<<8)|126u; + shared[2048+((at+1)&1023)]=command; + shared[2048+((at+2)&1023)]=(peer.fail_at==peer.count && peer.mode==1)?37:0; + shared[2048+((at+3)&1023)]=0xfafafafa; + q[11]=(at+4)&1023; +} +static int notify(void *cookie) +{ + CHECK(cookie==&peer && peer.count<8); + unsigned at=shared[16];uint32_t header=shared[1024+at];unsigned n=(header>>8)&255; + unsigned index=peer.count++;peer.ids[index]=header&255;peer.words[index]=n-1; + for(unsigned i=1;i=3))response(header); + peer.time+=peer.jump; + return 0; +} +static int now(void *cookie,uint64_t *value) +{ + CHECK(cookie==&peer);++peer.now_calls; + if(peer.clock_fail && peer.now_calls==peer.clock_fail)return 1; + if(peer.clock_reverse && peer.now_calls>=peer.clock_reverse)--peer.time; + *value=peer.time;return 0; +} +static int wait_us(void *cookie,unsigned us) +{ + CHECK(cookie==&peer && us==10);++peer.waits; + if(peer.wait_fail)return 1; + if(!peer.clock_stall)peer.time+=us; + return 0; +} +static int health(void *cookie) +{CHECK(cookie==&peer);return ++peer.health_calls==peer.health_fail;} +static void setup(unsigned acd,unsigned ifpc) +{ + memset(&peer,0,sizeof(peer));peer.time=1000; + CHECK(AdrenoHfiInitializeStopped(&hfi,shared,16384,0x61004000,&peer,notify)==AdrenoHfiSuccess); + state=(AdrenoHfiStartup){0};config=(AdrenoHfiStartupConfig){0}; + config.gpu_count=4;config.gmu_count=3; + for(unsigned i=0;i<4;++i)config.gpu[i]=(AdrenoHfiPerformanceLevel){i*100000000ULL,i*3}; + for(unsigned i=0;i<3;++i)config.gmu[i]=(AdrenoHfiPerformanceLevel){i*200000000ULL,i*4}; + config.bandwidth_count=3;config.cnoc_count=2;config.ddr_count=3; + config.cnoc_wait_mask=3;config.ddr_wait_mask=7; + for(unsigned i=0;i<2;++i) { + config.cnoc_addresses[i]=0x1000+4*i; // deliberate software fixture addresses, not board facts + config.cnoc_data[0][i]=0x40000000;config.cnoc_data[1][i]=0x60000001; + } + for(unsigned i=0;i<3;++i) { + config.ddr_addresses[i]=0x2000+4*i; + for(unsigned j=0;j<3;++j)config.ddr_data[j][i]=0x40000000+i+j; + } + if(acd) {config.acd_version=1;config.acd_enable_mask=10;config.acd_stride=1;config.acd_count=2;config.acd_data[0]=41;config.acd_data[1]=99;} + config.enable_ifpc=ifpc; + services=(AdrenoHfiStartupServices){&peer,now,wait_us,health,5,10,100,500}; +} +static AdrenoHfiStartupStatus run(void) +{return AdrenoHfiStartupRun(&state,&hfi,&config,&services,&result);} +static void refused(void) +{ + uint32_t before[4096];memcpy(before,shared,sizeof(before)); + CHECK(run()==AdrenoHfiStartupInvalid && !state.attempted && !peer.count); + CHECK(!memcmp(before,shared,sizeof(before)) && result.resources_must_retain); +} +int main(void) +{ + CHECK(!posix_memalign((void **)&shared,4096,16384)); + unsigned cases=0; + for(unsigned acd=0;acd<2;++acd)for(unsigned ifpc=0;ifpc<2;++ifpc) { + setup(acd,ifpc);CHECK(run()==AdrenoHfiStartupSuccess); + unsigned count=4+2*acd+ifpc;CHECK(peer.count==count && state.completed && !state.poisoned); + CHECK(result.stage==AdrenoHfiStageComplete && result.published_commands==count && result.resources_must_retain); +#ifdef SOURCE_ORACLE + unsigned expected_count=0,ids[8]={0},words[8]={0};uint32_t messages[8][159]={{0}}; + CHECK(!source_startup(&config,0,&expected_count,ids,words,messages)); + CHECK(expected_count==count); + for(unsigned i=0;i +#include +#include +#define CHECK(x) do {if(!(x)){fprintf(stderr,"line %d: %s\n",__LINE__,#x);exit(1);}}while(0) +extern int source_arc(const uint32_t *,unsigned,const uint8_t *,size_t,const uint8_t *,size_t,uint32_t *); +extern int source_bandwidth(const uint32_t *,unsigned,const AdrenoHfiBcm *,unsigned,uint32_t,uint32_t [159]); +static uint32_t seed=0x740; +static uint32_t random32(void){seed^=seed<<13;seed^=seed>>17;seed^=seed<<5;return seed;} +static void put16(uint8_t *p,unsigned n){p[0]=(uint8_t)n;p[1]=(uint8_t)(n>>8);} +static void arc_tests(void) +{ + uint8_t primary[513]={0},secondary[513]={0};uint32_t levels[16]={0},votes[16],expected[16]; + /* Unaligned original aux data, duplicated levels, trailing zero padding, + * secondary fallback and maximum representable 8-bit rail index. */ + for(unsigned i=1;i<256;++i)put16(primary+1+2*i,i*257); + put16(secondary+3,128);put16(secondary+5,500); + for(unsigned run=0;run<1000;++run) { + unsigned count=1+random32()%16;levels[0]=0; + for(unsigned i=1;i UINTPTR_MAX - pointer; +} +static int overlaps(uintptr_t a, size_t an, uintptr_t b, size_t bn) +{ + return an && bn && a < b + bn && b < a + an; +} +struct AdrenoConstantResult AdrenoA740Constants( + const struct AdrenoConstantConfig *cfg, const struct AdrenoConstantSpan *spans, size_t count, + const uint8_t *source, size_t source_bytes, uint32_t *output, size_t capacity) +{ + if (!cfg || count > ADRENO_CONSTANT_MAX_SPANS || (count && (!spans || !source || !output)) || + source_bytes > 16384 || capacity > SIZE_MAX / sizeof(*output) || + (count && ((uintptr_t)output % _Alignof(uint32_t))) || + wraps((uintptr_t)cfg, sizeof(*cfg)) || wraps((uintptr_t)spans, count * sizeof(*spans)) || + wraps((uintptr_t)source, source_bytes) || wraps((uintptr_t)output, capacity * sizeof(*output))) + return fail(AdrenoConstantArgument); + if ((cfg->chip_id != ADRENO_A740_CHIP_ID && cfg->chip_id != ADRENO_A740_FALLBACK_CHIP_ID) || cfg->stage > AdrenoConstantCompute || + cfg->constant_bytes > ADRENO_CONSTANT_MAX_BYTES || cfg->constant_bytes % 16) + return fail(AdrenoConstantProfile); + size_t words = 0; + for (size_t i = 0; i < count; ++i) { + const struct AdrenoConstantSpan *s = &spans[i]; + if (!s->bytes || s->bytes % 16 || s->destination_byte % 16 || + s->source_byte > source_bytes || s->bytes > source_bytes - s->source_byte || + s->destination_byte > cfg->constant_bytes || s->bytes > cfg->constant_bytes - s->destination_byte) + return fail(AdrenoConstantRange); + for (size_t j = 0; j < i; ++j) { + const struct AdrenoConstantSpan *p = &spans[j]; + if (overlaps(s->destination_byte, s->bytes, p->destination_byte, p->bytes)) + return fail(AdrenoConstantRange); + } + words += 4 + s->bytes / 4; + } + if (words > capacity) + return fail(AdrenoConstantCapacity); + size_t bytes = words * sizeof(*output); + uintptr_t dest = (uintptr_t)output; + if (overlaps(dest, bytes, (uintptr_t)cfg, sizeof(*cfg)) || + overlaps(dest, bytes, (uintptr_t)spans, count * sizeof(*spans)) || + overlaps(dest, bytes, (uintptr_t)source, source_bytes)) + return fail(AdrenoConstantAlias); + size_t at = 0; + uint32_t opcode = cfg->stage <= AdrenoConstantGeometry ? 0x32 : 0x34; + uint32_t block = 8 + cfg->stage; + for (size_t i = 0; i < count; ++i) { + const struct AdrenoConstantSpan *s = &spans[i]; + uint32_t num_unit = s->bytes / 16; + uint32_t header = 0; + (void)AdrenoPM4Type7(opcode, 3 + s->bytes / 4, &header); + output[at++] = header; + /* XML: DST_OFF[13:0], TYPE[15:14]=CONSTANTS(1), + * SRC[17:16]=DIRECT(0), BLOCK[21:18], UNIT[31:22]. */ + output[at++] = s->destination_byte / 16 | (1u << 14) | (block << 18) | (num_unit << 22); + output[at++] = 0; /* direct source: no external GPU address */ + output[at++] = 0; + for (uint32_t j = 0; j < s->bytes; j += 4) { + const uint8_t *p = source + s->source_byte + j; + output[at++] = (uint32_t)p[0] | ((uint32_t)p[1] << 8) | + ((uint32_t)p[2] << 16) | ((uint32_t)p[3] << 24); + } + } + return (struct AdrenoConstantResult){AdrenoConstantOK, (uint32_t)words, (uint32_t)count}; +} diff --git a/kits/AdrenoPM4/AdrenoConstants.h b/kits/AdrenoPM4/AdrenoConstants.h new file mode 100644 index 0000000..afdd15b --- /dev/null +++ b/kits/AdrenoPM4/AdrenoConstants.h @@ -0,0 +1,52 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef ADRENO_CONSTANTS_H +#define ADRENO_CONSTANTS_H +#include +#include +#ifdef __cplusplus +extern "C" { +#endif +#define ADRENO_A740_CHIP_ID UINT64_C(0x43050a01) +#define ADRENO_A740_FALLBACK_CHIP_ID UINT64_C(0xffff43050a01) +#define ADRENO_CONSTANT_MAX_SPANS 64u +#define ADRENO_CONSTANT_MAX_BYTES 8192u +/* Values are this API's stage identities, not an external compiler enum. */ +enum AdrenoConstantStage { + AdrenoConstantVertex, AdrenoConstantTessControl, AdrenoConstantTessEvaluation, + AdrenoConstantGeometry, AdrenoConstantFragment, AdrenoConstantCompute +}; +struct AdrenoConstantConfig { + uint64_t chip_id; /* Complete identity; never implicitly truncate UAPI IDs. */ + uint32_t stage, constant_bytes; +}; +struct AdrenoConstantSpan { + uint32_t source_byte, destination_byte, bytes; +}; +enum AdrenoConstantStatus { + AdrenoConstantOK, AdrenoConstantArgument, AdrenoConstantProfile, + AdrenoConstantRange, AdrenoConstantCapacity, AdrenoConstantAlias +}; +struct AdrenoConstantResult { + uint32_t status, words, packets; +}; +/* Build inline CP_LOAD_STATE6_GEOM/FRAG constant uploads for exact A740. + * Accept the two explicitly catalogued base/no-speedbin identities only. + * Only explicit spans are copied; undeclared constant allocations are untouched. + * Offsets/sizes are bytes, destination and sizes must be vec4-aligned. Sources + * are little-endian bytes and need no host alignment. Output uses target-native + * logical uint32 words; output storage must be aligned and actually accessible. + * Failure preserves every output byte and returns zero words/packets. Metadata, + * source and output must not overlap, wrap, or change during the call. + * Caller must own readable metadata/source lifetimes. No user-copy validation, + * untrusted command acceptance, cache barriers, indirect GPU addresses, MMIO, + * submission, fence completion, shader/pixel execution or rendering fallback. + * Runtime shader/context authorization and pipeline constant sharing are still + * the driver's responsibility; successful encoding does not authorize submission. + */ +struct AdrenoConstantResult AdrenoA740Constants( + const struct AdrenoConstantConfig *, const struct AdrenoConstantSpan *, size_t span_count, + const uint8_t *source, size_t source_bytes, uint32_t *output, size_t output_words); +#ifdef __cplusplus +} +#endif +#endif diff --git a/kits/AdrenoPM4/AdrenoPM4.c b/kits/AdrenoPM4/AdrenoPM4.c new file mode 100644 index 0000000..12b2936 --- /dev/null +++ b/kits/AdrenoPM4/AdrenoPM4.c @@ -0,0 +1,87 @@ +/* SPDX-License-Identifier: MIT */ +#include "AdrenoPM4.h" + +/* Add one bit to make the field's total population odd. */ +static uint32_t odd_bit(uint32_t field) +{ + uint32_t parity = 1; + while (field) { + parity ^= 1; + field &= field - 1; + } + return parity; +} + +AdrenoPM4Status AdrenoPM4Type4(uint32_t reg, uint32_t count, uint32_t *header) +{ + if (!header || reg > 0x3ffff || !count || count > 127 || + count - 1 > 0x3ffff - reg) + return AdrenoPM4InvalidArgument; + *header = 0x40000000u | count | (odd_bit(count) << 7) | + (reg << 8) | (odd_bit(reg) << 27); + return AdrenoPM4Success; +} + +AdrenoPM4Status AdrenoPM4Type7(uint32_t opcode, uint32_t count, uint32_t *header) +{ + if (!header || opcode > 127 || count > 0x3fff) + return AdrenoPM4InvalidArgument; + *header = 0x70000000u | count | (odd_bit(count) << 15) | + (opcode << 16) | (odd_bit(opcode) << 23); + return AdrenoPM4Success; +} + +static AdrenoPM4Status decode(uint32_t word, AdrenoPM4Packet *packet) +{ + uint32_t rebuilt; + AdrenoPM4Status status; + packet->type = word >> 28; + if (packet->type == 4) { + packet->identifier = (word >> 8) & 0x3ffff; + packet->payload_words = word & 127; + status = AdrenoPM4Type4(packet->identifier, packet->payload_words, &rebuilt); + } else if (packet->type == 7) { + packet->identifier = (word >> 16) & 127; + packet->payload_words = word & 0x3fff; + status = AdrenoPM4Type7(packet->identifier, packet->payload_words, &rebuilt); + } else { + return AdrenoPM4InvalidHeader; + } + if (status != AdrenoPM4Success || rebuilt != word) + return AdrenoPM4InvalidHeader; + return AdrenoPM4Success; +} + +AdrenoPM4Status AdrenoPM4Plan(const uint32_t *words, size_t word_count, + AdrenoPM4Packet *output, size_t capacity, size_t *count) +{ + size_t at = 0, packets = 0; + AdrenoPM4Packet packet; + if (!count) + return AdrenoPM4InvalidArgument; + *count = 0; + if ((word_count && !words) || (capacity && !output) || + word_count > SIZE_MAX / sizeof(*words) || capacity > SIZE_MAX / sizeof(*output)) + return AdrenoPM4InvalidArgument; + while (at < word_count) { + AdrenoPM4Status status = decode(words[at], &packet); + if (status != AdrenoPM4Success) + return status; + /* Subtraction keeps both bounds checking and the next offset safe. */ + if (packet.payload_words > word_count - at - 1) + return AdrenoPM4Truncated; + at += 1 + packet.payload_words; + ++packets; + } + if (packets > capacity) + return AdrenoPM4Capacity; + at = packets = 0; + while (at < word_count) { + (void)decode(words[at], &packet); /* Immutable input, already checked. */ + packet.header_word = at; + output[packets++] = packet; + at += 1 + packet.payload_words; + } + *count = packets; + return AdrenoPM4Success; +} diff --git a/kits/AdrenoPM4/AdrenoPM4.h b/kits/AdrenoPM4/AdrenoPM4.h new file mode 100644 index 0000000..ceaefa2 --- /dev/null +++ b/kits/AdrenoPM4/AdrenoPM4.h @@ -0,0 +1,43 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef ADRENO_PM4_H +#define ADRENO_PM4_H +#include +#include +#ifdef __cplusplus +extern "C" { +#endif + +typedef enum { + AdrenoPM4Success = 0, + AdrenoPM4InvalidArgument, + AdrenoPM4InvalidHeader, + AdrenoPM4Truncated, + AdrenoPM4Capacity +} AdrenoPM4Status; + +typedef struct { + size_t header_word; + uint32_t type; /* 4 (register sequence) or 7 (opcode). */ + uint32_t identifier; /* Register index or opcode; NOT authorization. */ + uint32_t payload_words; +} AdrenoPM4Packet; + +/* Supported emission profile: 18-bit registers, 1..127 values; 7-bit + * opcodes, 0..16383 values. No silent masking. Failure preserves *header. */ +AdrenoPM4Status AdrenoPM4Type4(uint32_t reg, uint32_t count, uint32_t *header); +AdrenoPM4Status AdrenoPM4Type7(uint32_t opcode, uint32_t count, uint32_t *header); + +/* Structural decoding only. A successful plan DOES NOT authorize register + * writes, addresses, opcodes, indirect buffers or hardware submission. + * Input is an immutable array of logical 32-bit words (little-endian on + * the target). All extents must be real caller-owned accessible storage. + * Input, output and count must not alias; input must remain immutable. + * Validate the whole input and capacity before writing any packet record. + * On failure, *count=0 and output is untouched. Empty input succeeds. + * No allocation, recursion, payload interpretation, MMIO or completion. */ +AdrenoPM4Status AdrenoPM4Plan(const uint32_t *words, size_t word_count, + AdrenoPM4Packet *output, size_t capacity, size_t *count); +#ifdef __cplusplus +} +#endif +#endif diff --git a/kits/AdrenoPM4/AdrenoRaster.c b/kits/AdrenoPM4/AdrenoRaster.c new file mode 100644 index 0000000..00c99c0 --- /dev/null +++ b/kits/AdrenoPM4/AdrenoRaster.c @@ -0,0 +1,58 @@ +/* SPDX-License-Identifier: MIT */ +#include "AdrenoRaster.h" +#include "AdrenoPM4.h" +static struct AdrenoRasterResult fail(uint32_t s) { + return (struct AdrenoRasterResult){s,0,0}; +} +static int wraps(uintptr_t p,size_t n) { return n>UINTPTR_MAX-p; } +static int overlap(uintptr_t a,size_t n,uintptr_t b,size_t m) { + return n && m && aSIZE_MAX/4 || wraps((uintptr_t)output,capacity*4)) return fail(AdrenoRasterArgument); + if((s->chip_id!=ADRENO_A740_CHIP_ID && s->chip_id!=ADRENO_A740_FALLBACK_CHIP_ID) || + (s->flags&~ADRENO_RASTER_FLAGS) || s->reserved || s->polygon_mode<1 || s->polygon_mode>3 || + s->point_size_q4>65472 || s->line_width_q1>255 || + !finite32(s->offset_scale_bits) || !finite32(s->offset_units_bits) || + !finite32(s->offset_clamp_bits)) return fail(AdrenoRasterProfile); + uint32_t f=s->flags; + int clamp=(f&(AdrenoRasterClipNear|AdrenoRasterClipFar))!= + (AdrenoRasterClipNear|AdrenoRasterClipFar); + uint32_t words=clamp?24:60; + if(capacitypoint_size_q4,point_max=s->point_size_q4; + if(f&AdrenoRasterPointSizePerVertex) { + point_min=(f&(AdrenoRasterPointQuad|AdrenoRasterPointSmooth|AdrenoRasterMultisample))?0:16; + point_max=65472; + } + uint32_t cl=0xa0u|((f&AdrenoRasterClipNear)?0:2u)| + ((f&AdrenoRasterClipFar)?0:4u)|((f&AdrenoRasterHalfZ)?64u:0); + uint32_t su=((f&AdrenoRasterCullFront)?1u:0)|((f&AdrenoRasterCullBack)?2u:0)| + ((f&AdrenoRasterFrontCCW)?0:4u)|(s->line_width_q1<<3)| + ((f&AdrenoRasterOffsetTriangles)?0x800u:0)|((f&AdrenoRasterMultisample)?0x2000u:0); + uint32_t pc=((f&AdrenoRasterPrimitiveRestart)?1u:0)|((f&AdrenoRasterFlatFirst)?0:2u); + uint32_t *out=output; + reg32(&out,0x8000,cl);reg32(&out,0x8090,su); + reg32(&out,0x8091,point_min|(point_max<<16));reg32(&out,0x8092,s->point_size_q4); + reg32(&out,0x8095,s->offset_scale_bits);reg32(&out,0x8096,s->offset_units_bits); + reg32(&out,0x8097,s->offset_clamp_bits); + reg32(&out,0x9b00,pc);reg32(&out,0x9109,pc); + reg32(&out,0x9108,s->polygon_mode);reg32(&out,0x9809,s->polygon_mode); + reg32(&out,0x9307,s->polygon_mode); + if(!clamp) { + uint32_t ranges[32]; + for(unsigned i=0;i<16;i++) { ranges[2*i]=0;ranges[2*i+1]=UINT32_C(0x3f800000); } + packet(&out,0x8070,ranges,32);packet(&out,0x88c0,ranges,2); + } + return (struct AdrenoRasterResult){AdrenoRasterOK,(uint32_t)(out-output),clamp?12u:46u}; +} diff --git a/kits/AdrenoPM4/AdrenoRaster.h b/kits/AdrenoPM4/AdrenoRaster.h new file mode 100644 index 0000000..57ac5ec --- /dev/null +++ b/kits/AdrenoPM4/AdrenoRaster.h @@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef ADRENO_RASTER_H +#define ADRENO_RASTER_H +#include "AdrenoConstants.h" +#ifdef __cplusplus +extern "C" { +#endif +enum AdrenoRasterFlags { + AdrenoRasterClipNear=1u, AdrenoRasterClipFar=2u, + AdrenoRasterHalfZ=4u, AdrenoRasterCullFront=8u, + AdrenoRasterCullBack=16u, AdrenoRasterFrontCCW=32u, + AdrenoRasterOffsetTriangles=64u, AdrenoRasterMultisample=128u, + AdrenoRasterPrimitiveRestart=256u, AdrenoRasterFlatFirst=512u, + AdrenoRasterPointSizePerVertex=1024u, AdrenoRasterPointQuad=2048u, + AdrenoRasterPointSmooth=4096u +}; +#define ADRENO_RASTER_FLAGS 8191u +#define ADRENO_RASTER_MAX_WORDS 60u +/* polygon_mode: 1 points, 2 lines, 3 triangles. Fixed-point sizes are + * explicit hardware units: point_size_q4 / 16 pixels, line_width_q1 / 2 + * pixels. The caller performs any API float quantization. Offset values + * are exact finite binary32 representations. No CPU FP operation is used. */ +struct AdrenoRasterState { + uint64_t chip_id; + uint32_t flags, polygon_mode, point_size_q4, line_width_q1; + uint32_t offset_scale_bits, offset_units_bits, offset_clamp_bits, reserved; +}; +enum AdrenoRasterStatus { + AdrenoRasterOK, AdrenoRasterArgument, AdrenoRasterProfile, + AdrenoRasterCapacity, AdrenoRasterAlias +}; +struct AdrenoRasterResult { uint32_t status, words, registers; }; +/* Encode the complete pinned fd6 A740 rasterizer-state register set for + * these explicit fields. Dynamic viewport state, fragment/depth/blend/ + * sample/primitive-draw/input/resource/cache state and live GPU ownership + * are separate requirements. In particular, when either depth clip flag + * is disabled, the viewport path MUST program matching GRAS/RB Z clamps. + * Point-size exports must agree with the separately bound shader/VPC. + * No allocation, MMIO, submission or rendering. Failure preserves output. */ +struct AdrenoRasterResult AdrenoA740Raster(const struct AdrenoRasterState *, + uint32_t *output, size_t output_words); +#ifdef __cplusplus +} +#endif +#endif diff --git a/kits/AdrenoPM4/AdrenoShader.c b/kits/AdrenoPM4/AdrenoShader.c new file mode 100644 index 0000000..e31d3c9 --- /dev/null +++ b/kits/AdrenoPM4/AdrenoShader.c @@ -0,0 +1,83 @@ +/* SPDX-License-Identifier: MIT */ +#include "AdrenoShader.h" +#include "AdrenoPM4.h" +static struct AdrenoShaderResult fail(uint32_t status) +{ + return (struct AdrenoShaderResult){status, 0, 0}; +} +static int wraps(uintptr_t p, size_t n) { return n > UINTPTR_MAX - p; } +static int overlaps(uintptr_t a, size_t n, uintptr_t b, size_t m) +{ + return a < b + m && b < a + n; +} +static void reg32(uint32_t **out, uint32_t reg, uint32_t value) +{ + uint32_t header = 0; + (void)AdrenoPM4Type4(reg, 1, &header); + *(*out)++ = header; + *(*out)++ = value; +} +static void reg64(uint32_t **out, uint32_t reg, uint64_t value) +{ + uint32_t header = 0; + (void)AdrenoPM4Type4(reg, 2, &header); + *(*out)++ = header; + *(*out)++ = (uint32_t)value; + *(*out)++ = (uint32_t)(value >> 32); +} +struct AdrenoShaderResult AdrenoA740ShaderBind(const struct AdrenoShaderState *s, + const struct AdrenoShaderMapping *m, uint32_t *output, size_t capacity) +{ + if (!s || !m || !output || (uintptr_t)output % _Alignof(uint32_t) || + capacity > SIZE_MAX / 4 || wraps((uintptr_t)s, sizeof(*s)) || + wraps((uintptr_t)m, sizeof(*m)) || wraps((uintptr_t)output, capacity * 4)) + return fail(AdrenoShaderArgument); + if ((s->chip_id != ADRENO_A740_CHIP_ID && s->chip_id != ADRENO_A740_FALLBACK_CHIP_ID) || + (s->stage != AdrenoConstantVertex && s->stage != AdrenoConstantFragment) || + !s->binary_bytes || s->binary_bytes > 1024 * 1024 || s->binary_bytes % 128 || + !s->instruction_groups || s->instruction_groups > s->binary_bytes / 128 || + s->max_full_register < -1 || s->max_full_register > 62 || s->max_half_register != -1 || + s->branch_stack_hw > 126 || s->branch_stack_hw % 2 || s->merged_registers > 1 || + s->early_preamble > 1 || s->double_threadsize > 1 || s->need_full_quad > 1 || s->need_pixel_lod > 1 || + s->input_words > 128 || s->private_bytes || s->constant_data_bytes || + (s->stage == AdrenoConstantVertex && (s->need_full_quad || s->need_pixel_lod))) + return fail(AdrenoShaderProfile); + if (!m->context || !m->base || m->base >= (UINT64_C(1) << 48) || + !m->bytes || m->bytes > (UINT64_C(1) << 48) - m->base || + (m->base & 4095) || (m->bytes & 4095) || m->page_shift != 12 || m->readonly != 1 || + (m->shader_offset & 127) || m->shader_offset > m->bytes || s->binary_bytes > m->bytes - m->shader_offset) + return fail(AdrenoShaderMappingError); + if (capacity < ADRENO_SHADER_BIND_WORDS) + return fail(AdrenoShaderCapacity); + if (overlaps((uintptr_t)output, ADRENO_SHADER_BIND_WORDS * 4, (uintptr_t)s, sizeof(*s)) || + overlaps((uintptr_t)output, ADRENO_SHADER_BIND_WORDS * 4, (uintptr_t)m, sizeof(*m))) + return fail(AdrenoShaderAlias); + uint32_t *out = output; + /* Pinned A7xx GL path: shared constants off, constant demotion on. + * TPL1 CLAMP_DISABLE is A8xx-only and must not leak into A740. */ + reg32(&out, 0xab00, 5); /* SP_MODE_CNTL */ + reg32(&out, 0xb309, 0xa2); /* TPL1_MODE_CNTL */ + uint32_t control = ((uint32_t)(s->max_full_register + 1) << 7) | (s->branch_stack_hw << 13); + uint32_t ctrl, instr, pc, base, param, pvt_base, pvt_size, stack, vgs; + if (s->stage == AdrenoConstantVertex) { + control |= (s->merged_registers << 20) | (s->early_preamble << 21); + ctrl=0xa800; instr=0xa824; pc=0xa81b; base=0xa81c; param=0xa81e; + pvt_base=0xa81f; pvt_size=0xa821; stack=0xa825; vgs=0xa82d; + } else { + control |= (s->double_threadsize << 20) | ((s->input_words != 0) << 22) | + (s->need_full_quad << 23) | (1u << 24) | (s->need_pixel_lod << 26) | + (s->early_preamble << 28) | (s->merged_registers << 31); + ctrl=0xa980; instr=0xab05; pc=0xa982; base=0xa983; param=0xa985; + pvt_base=0xa986; pvt_size=0xa988; stack=0xa9a9; vgs=0xaa01; + } + reg32(&out, ctrl, control); + reg32(&out, instr, s->instruction_groups); + reg32(&out, pc, 0); + reg64(&out, base, m->base + m->shader_offset); + reg32(&out, param, 0); + reg64(&out, pvt_base, 0); + reg32(&out, pvt_size, 0); + reg32(&out, stack, 0); + reg32(&out, vgs, 0); + return (struct AdrenoShaderResult){AdrenoShaderOK, ADRENO_SHADER_BIND_WORDS, 13}; +} diff --git a/kits/AdrenoPM4/AdrenoShader.h b/kits/AdrenoPM4/AdrenoShader.h new file mode 100644 index 0000000..48f2bf9 --- /dev/null +++ b/kits/AdrenoPM4/AdrenoShader.h @@ -0,0 +1,43 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef ADRENO_SHADER_H +#define ADRENO_SHADER_H +#include "AdrenoConstants.h" +#ifdef __cplusplus +extern "C" { +#endif +#define ADRENO_SHADER_BIND_WORDS 24u +struct AdrenoShaderState { + uint64_t chip_id; + uint32_t stage, binary_bytes, instruction_groups; + int32_t max_full_register, max_half_register; + uint32_t branch_stack_hw, merged_registers, early_preamble; + uint32_t double_threadsize, input_words, need_full_quad, need_pixel_lod; + uint32_t private_bytes, constant_data_bytes; +}; +/* A view into a caller-owned native-4K readonly shader mapping. This record + * describes ownership; it does not verify or activate an SMMU context. */ +struct AdrenoShaderMapping { + uint64_t context, base, bytes, shader_offset; + uint32_t page_shift, readonly; +}; +enum AdrenoShaderStatus { + AdrenoShaderOK, AdrenoShaderArgument, AdrenoShaderProfile, + AdrenoShaderMappingError, AdrenoShaderCapacity, AdrenoShaderAlias +}; +struct AdrenoShaderResult { uint32_t status, words, registers; }; +/* Encode the A740 fd6_emit_shader register subset for real vertex/fragment + * variants without private memory/constant-data blobs. All actual compiler + * binding fields must be supplied; do not infer them from code size or names. + * Both full catalog identities are checked. No A6xx instruction-preload packet + * is emitted. Logical shader extent must fit its declared 4K resource view. + * Before submission the driver must qualify/own that live native-4K context, + * immutable shader bytes, complete pipeline/cache/barrier state and lifetime. + * This function does not accept user command streams or authorize submission. + * Failure preserves output, including capacity/metadata alias and pointer wrap. + * No allocation, MMIO, CPU rendering, simulated mappings or completion. */ +struct AdrenoShaderResult AdrenoA740ShaderBind(const struct AdrenoShaderState *, + const struct AdrenoShaderMapping *, uint32_t *output, size_t output_words); +#ifdef __cplusplus +} +#endif +#endif diff --git a/kits/AdrenoPM4/AdrenoVPC.c b/kits/AdrenoPM4/AdrenoVPC.c new file mode 100644 index 0000000..997d3b3 --- /dev/null +++ b/kits/AdrenoPM4/AdrenoVPC.c @@ -0,0 +1,83 @@ +/* SPDX-License-Identifier: MIT */ +#include "AdrenoVPC.h" +#include "AdrenoPM4.h" +static struct AdrenoVPCResult failure(uint32_t s) { return (struct AdrenoVPCResult){s,0,0,0,0,0}; } +static int wraps(uintptr_t p,size_t n) { return n>UINTPTR_MAX-p; } +static int overlap(uintptr_t a,size_t n,uintptr_t b,size_t m) { return n && m && a>=1;}return n; } +static int same(const uint8_t *a,size_t n,const uint8_t *b,size_t m) { + if(n!=m) return 0; + for(size_t i=0;i256 || wraps((uintptr_t)p,n)) return AdrenoVPCArgument; + if(overlap((uintptr_t)p,n,output,bytes)) return AdrenoVPCAlias; + for(size_t i=0;i126) return AdrenoVPCIdentity; + if(position(p,n)) return 0; + if(n<=11 || !same(p,10,(const uint8_t *)"generated(",10) || p[n-1]!=')') return AdrenoVPCIdentity; + return 0; +} +static void packet(uint32_t **out,uint32_t reg,const uint32_t *values,uint32_t n) { + uint32_t header=0;(void)AdrenoPM4Type4(reg,n,&header);*(*out)++=header; + for(uint32_t i=0;iSIZE_MAX/4 || wraps((uintptr_t)output,capacity*4)) return failure(AdrenoVPCArgument); + if((s->chip_id!=ADRENO_A740_CHIP_ID && s->chip_id!=ADRENO_A740_FALLBACK_CHIP_ID) || s->reserved || s->vertex_max_register<0 || s->vertex_max_register>62 || !s->output_count || s->output_count>ADRENO_VPC_MAX_OUTPUTS || s->input_count>ADRENO_VPC_MAX_INPUTS) return failure(AdrenoVPCProfile); + if(!s->outputs || (s->input_count && !s->inputs) || wraps((uintptr_t)s->outputs,s->output_count*sizeof(*s->outputs)) || wraps((uintptr_t)s->inputs,s->input_count*sizeof(*s->inputs))) return failure(AdrenoVPCArgument); + uint32_t entries=(uint32_t)s->input_count+1,nreg=(entries+1)/2,ndest=(entries+3)/4,words=40+nreg+ndest; + if(capacityoutputs,s->output_count*sizeof(*s->outputs)) || overlap(dst,bytes,(uintptr_t)s->inputs,s->input_count*sizeof(*s->inputs))) return failure(AdrenoVPCAlias); + unsigned pos=ADRENO_VPC_MAX_OUTPUTS,limit=((unsigned)s->vertex_max_register+1)*4; + for(size_t i=0;ioutput_count;i++) { + const struct AdrenoVPCOutput *o=s->outputs+i; + uint32_t error=name_check(o->identity,o->identity_bytes,dst,bytes);if(error) return failure(error); + if(!o->components || o->components>4 || o->bit_size!=32 || o->half || o->view || o->aliased_components || o->register_id>=limit || o->components>limit-o->register_id) return failure(AdrenoVPCProfile); + if(position(o->identity,o->identity_bytes)) { if(pos!=ADRENO_VPC_MAX_OUTPUTS || o->components!=4) return failure(AdrenoVPCIdentity);pos=(unsigned)i; } + for(size_t j=0;jidentity,o->identity_bytes,s->outputs[j].identity,s->outputs[j].identity_bytes)) return failure(AdrenoVPCIdentity); + } + if(pos==ADRENO_VPC_MAX_OUTPUTS) return failure(AdrenoVPCIdentity); + unsigned order[ADRENO_VPC_MAX_INPUTS],matched[ADRENO_VPC_MAX_INPUTS],max_loc=0;uint32_t mask[4]={0}; + for(size_t i=0;iinput_count;i++) { + const struct AdrenoVPCInput *in=s->inputs+i; + uint32_t error=name_check(in->identity,in->identity_bytes,dst,bytes);if(error) return failure(error); + unsigned width=last_bit(in->component_mask); + if(position(in->identity,in->identity_bytes) || !in->components || in->components>4 || in->bit_size!=32 || in->interpolation || !width || width>in->components || in->input_location>124 || width>124-in->input_location) return failure(AdrenoVPCProfile); + for(size_t j=0;jinputs+j;unsigned pw=last_bit(p->component_mask); + if(same(in->identity,in->identity_bytes,p->identity,p->identity_bytes)) return failure(AdrenoVPCIdentity); + if(overlap(in->input_location,width,p->input_location,pw)) return failure(AdrenoVPCProfile); + } + unsigned found=ADRENO_VPC_MAX_OUTPUTS; + for(size_t j=0;joutput_count;j++) if(same(in->identity,in->identity_bytes,s->outputs[j].identity,s->outputs[j].identity_bytes)) found=(unsigned)j; + if(found==ADRENO_VPC_MAX_OUTPUTS) return failure(AdrenoVPCMissingVarying); + if(s->outputs[found].components!=in->components) return failure(AdrenoVPCProfile); + matched[i]=found;order[i]=(unsigned)i; + for(unsigned j=0;jinput_location+j)/32]|=UINT32_C(1)<<((in->input_location+j)%32); + if(in->input_location+width>max_loc) max_loc=in->input_location+width; + } + for(size_t i=1;iinput_count;i++) { + unsigned v=order[i];size_t j=i; + while(j && s->inputs[order[j-1]].input_location>s->inputs[v].input_location) {order[j]=order[j-1];j--;} + order[j]=v; + } + uint32_t outputs[8]={0},destinations[4]={0},disabled[4],zeros[16]={0}; + for(unsigned i=0;iinput_count) {unsigned k=order[i];reg=s->outputs[matched[k]].register_id;comp=s->inputs[k].component_mask;loc=s->inputs[k].input_location;} + else {reg=s->outputs[pos].register_id;comp=15;loc=max_loc;} + outputs[i/2]|=(reg|(comp<<8))<<((i%2)*16);destinations[i/4]|=loc<<((i%4)*8); + } + for(unsigned i=0;i<4;i++) disabled[i]=~mask[i]; + uint32_t *out=output; + packet(&out,0x9212,disabled,4);packet(&out,0xa803,outputs,nreg);packet(&out,0xa813,destinations,ndest); + reg32(&out,0x9301,(max_loc+4)|(max_loc<<8)|UINT32_C(0xff0000)); + reg32(&out,0x9101,0xffff00);reg32(&out,0x9311,0xffff00);reg32(&out,0x8001,0); + reg32(&out,0xa802,entries);reg32(&out,0x9104,0xffffff);reg32(&out,0x9314,0xffffff);reg32(&out,0x809b,0); + packet(&out,0x9200,zeros,16); + return (struct AdrenoVPCResult){AdrenoVPCOK,(uint32_t)(out-output),words-12,(uint32_t)s->input_count,max_loc+4,max_loc}; +} diff --git a/kits/AdrenoPM4/AdrenoVPC.h b/kits/AdrenoPM4/AdrenoVPC.h new file mode 100644 index 0000000..41a4cc9 --- /dev/null +++ b/kits/AdrenoPM4/AdrenoVPC.h @@ -0,0 +1,50 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef ADRENO_VPC_H +#define ADRENO_VPC_H +#include "AdrenoConstants.h" +#ifdef __cplusplus +extern "C" { +#endif +#define ADRENO_VPC_MAX_OUTPUTS 16u +#define ADRENO_VPC_MAX_INPUTS 15u +#define ADRENO_VPC_MAX_WORDS 52u +struct AdrenoVPCOutput { + const uint8_t *identity; + size_t identity_bytes; + uint32_t register_id, components, bit_size, half, view, aliased_components; +}; +struct AdrenoVPCInput { + const uint8_t *identity; + size_t identity_bytes; + uint32_t components, bit_size, component_mask, input_location, interpolation; +}; +struct AdrenoVPCState { + uint64_t chip_id; + int32_t vertex_max_register; + uint32_t reserved; + const struct AdrenoVPCOutput *outputs; + size_t output_count; + const struct AdrenoVPCInput *inputs; + size_t input_count; +}; +enum AdrenoVPCStatus { + AdrenoVPCOK, AdrenoVPCArgument, AdrenoVPCProfile, AdrenoVPCIdentity, + AdrenoVPCMissingVarying, AdrenoVPCCapacity, AdrenoVPCAlias +}; +struct AdrenoVPCResult { + uint32_t status, words, registers, varying_count, stride, position_location; +}; +/* Link exact semantic identities to actual IR3 registers/inloc, then encode + * the A740 VS->VPC linkage subset. Requires float32 smooth-center inputs, + * actual compiler output/register metadata and one float4 air.position. + * Other builtins, alias/half outputs and tessellation/geometry/streamout are + * outside this profile. Missing/mismatched varyings refuse; no zero synthesis. + * Output is transactional, including metadata/name aliases. Immutable readable + * metadata/backing lifetime is the caller's responsibility. No allocation, + * MMIO, mappings, submission or completion; the driver still needs the complete + * primitive/raster/resource/cache state and a live owned native-4K GPU context. */ +struct AdrenoVPCResult AdrenoA740VaryingMap(const struct AdrenoVPCState *, uint32_t *, size_t); +#ifdef __cplusplus +} +#endif +#endif diff --git a/kits/AdrenoPM4/LICENSE b/kits/AdrenoPM4/LICENSE new file mode 100644 index 0000000..9bcaf67 --- /dev/null +++ b/kits/AdrenoPM4/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 AdrenoPM4 contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/kits/AdrenoPM4/test_constants.c b/kits/AdrenoPM4/test_constants.c new file mode 100644 index 0000000..78ddd1b --- /dev/null +++ b/kits/AdrenoPM4/test_constants.c @@ -0,0 +1,143 @@ +/* SPDX-License-Identifier: MIT */ +#include "AdrenoConstants.h" +#include "AdrenoPM4.h" +#include +#include +#include +#include +#include +#ifdef MESA_REFERENCE +#include "freedreno_pm4.h" +#endif +#define WORDS 2304 +static uint32_t output[WORDS], before[WORDS]; +static unsigned checks, refusals; +static struct AdrenoConstantConfig config = {ADRENO_A740_CHIP_ID, AdrenoConstantVertex, 8192}; +static uint32_t reference_state(unsigned stage, unsigned destination, unsigned units) +{ +#ifdef MESA_REFERENCE + const enum a6xx_state_block blocks[] = {SB6_VS_SHADER, SB6_HS_SHADER, SB6_DS_SHADER, + SB6_GS_SHADER, SB6_FS_SHADER, SB6_CS_SHADER}; + return CP_LOAD_STATE6_0_DST_OFF(destination) | CP_LOAD_STATE6_0_STATE_TYPE(ST6_CONSTANTS) | + CP_LOAD_STATE6_0_STATE_SRC(SS6_DIRECT) | CP_LOAD_STATE6_0_STATE_BLOCK(blocks[stage]) | + CP_LOAD_STATE6_0_NUM_UNIT(units); +#else + const uint32_t blocks[] = {0x200000, 0x240000, 0x280000, 0x2c0000, 0x300000, 0x340000}; + return destination | 0x4000 | blocks[stage] | units * 0x400000; +#endif +} +static void reset(void) +{ + memset(output, 0x6b, sizeof(output)); + memcpy(before, output, sizeof(output)); +} +static void rejected(const struct AdrenoConstantConfig *cfg, const struct AdrenoConstantSpan *spans, + size_t count, const uint8_t *data, size_t bytes, uint32_t *dst, size_t capacity, unsigned expected) +{ + memcpy(before, output, sizeof(output)); + struct AdrenoConstantResult r = AdrenoA740Constants(cfg, spans, count, data, bytes, dst, capacity); + assert(r.status == expected && !r.words && !r.packets); + assert(!memcmp(output, before, sizeof(output))); + ++refusals; +} +static void positive(const struct AdrenoConstantSpan *spans, size_t count, const uint8_t *data, size_t bytes) +{ + reset(); + struct AdrenoConstantResult r = AdrenoA740Constants(&config, spans, count, data, bytes, output, WORDS); + assert(r.status == AdrenoConstantOK && r.packets == count); + AdrenoPM4Packet packets[64]; + size_t packet_count = 0; + assert(AdrenoPM4Plan(output, r.words, packets, 64, &packet_count) == AdrenoPM4Success && packet_count == count); + size_t at = 0; + for (size_t i = 0; i < count; ++i) { + const struct AdrenoConstantSpan *s = &spans[i]; + unsigned opcode = config.stage < 4 ? 0x32 : 0x34; +#ifdef MESA_REFERENCE + assert(output[at] == pm4_pkt7_hdr(opcode, 3 + s->bytes / 4)); +#endif + assert(packets[i].header_word == at && packets[i].identifier == opcode && packets[i].type == 7 && + packets[i].payload_words == 3 + s->bytes / 4); + assert(output[at+1] == reference_state(config.stage, s->destination_byte / 16, s->bytes / 16)); + assert(!output[at+2] && !output[at+3]); + assert(!memcmp(output + at + 4, data + s->source_byte, s->bytes)); + at += 4 + s->bytes / 4; + } + assert(r.words == at && !memcmp(output + at, before + at, sizeof(output) - at * 4)); + ++checks; +} +int main(void) +{ + long page = sysconf(_SC_PAGESIZE); + assert(page >= 4096); + uint8_t *mapping = mmap(NULL, (size_t)page*3, PROT_READ | PROT_WRITE, MAP_ANON | MAP_PRIVATE, -1, 0); + assert(mapping != MAP_FAILED && !mprotect(mapping+2*page, (size_t)page, PROT_NONE)); + uint8_t *data = mapping + 2*page - 8192; + for (unsigned i = 0; i < 8192; ++i) data[i] = (uint8_t)(i * 7 + 3); + struct AdrenoConstantSpan one = {0, 0, 16}; + for (unsigned stage = 0; stage < 6; ++stage) { + config.stage = stage; + for (unsigned off = 0; off < 512; ++off) { + one.destination_byte = off*16; + const unsigned units[] = {1, 512-off, 4}; + for (unsigned i = 0; i < 3; ++i) if (units[i] <= 512-off) { + one.bytes = units[i]*16; + one.source_byte = 8192-one.bytes; + positive(&one, 1, data, 8192); + } + } + } + config.stage = 0; + config.chip_id = ADRENO_A740_FALLBACK_CHIP_ID; + one = (struct AdrenoConstantSpan){0, 0, 16}; + positive(&one, 1, data, 8192); + config.chip_id = ADRENO_A740_CHIP_ID; + struct AdrenoConstantSpan spans[64]; + for (unsigned i = 0; i < 64; ++i) spans[i] = (struct AdrenoConstantSpan){i*16, i*32, 16}; + positive(spans, 64, data, 8192); + /* An unaligned byte source is legal and reads exactly through its logical end. */ + one = (struct AdrenoConstantSpan){0, 0, 16}; + positive(&one, 1, data+1, 16); + positive(NULL, 0, NULL, 0); + reset(); + rejected(NULL, &one, 1, data, 8192, output, WORDS, AdrenoConstantArgument); + rejected(&config, NULL, 1, data, 8192, output, WORDS, AdrenoConstantArgument); + rejected(&config, &one, 65, data, 8192, output, WORDS, AdrenoConstantArgument); + rejected(&config, &one, 1, NULL, 8192, output, WORDS, AdrenoConstantArgument); + rejected(&config, &one, 1, data, 8192, NULL, WORDS, AdrenoConstantArgument); + rejected(&config, &one, 1, data, 16385, output, WORDS, AdrenoConstantArgument); + rejected(&config, &one, 1, data, 8192, output, SIZE_MAX, AdrenoConstantArgument); + rejected(&config, &one, 1, data, 8192, (uint32_t *)((uint8_t *)output+1), WORDS-1, AdrenoConstantArgument); + rejected(&config, &one, 1, (void *)(UINTPTR_MAX-3), 16, output, WORDS, AdrenoConstantArgument); + rejected(&config, &one, 1, data, 8192, (void *)(UINTPTR_MAX-3), 8, AdrenoConstantArgument); + rejected((void *)(UINTPTR_MAX-3), &one, 1, data, 8192, output, WORDS, AdrenoConstantArgument); + rejected(&config, (void *)(UINTPTR_MAX-3), 1, data, 8192, output, WORDS, AdrenoConstantArgument); + struct AdrenoConstantConfig bad = config; + bad.chip_id ^= 1; + rejected(&bad, &one, 1, data, 8192, output, WORDS, AdrenoConstantProfile); + bad=config;bad.chip_id=UINT64_C(0xffff000043050a01); + rejected(&bad, &one, 1, data, 8192, output, WORDS, AdrenoConstantProfile); + bad=config;bad.stage=6; + rejected(&bad, &one, 1, data, 8192, output, WORDS, AdrenoConstantProfile); + bad=config;bad.constant_bytes=8208; + rejected(&bad, &one, 1, data, 8192, output, WORDS, AdrenoConstantProfile); + bad=config;bad.constant_bytes=17; + rejected(&bad, &one, 1, data, 8192, output, WORDS, AdrenoConstantProfile); + rejected(&config, &one, 1, data, 8192, output, 7, AdrenoConstantCapacity); + rejected(&config, &one, 1, (const uint8_t *)output, 16, output, WORDS, AdrenoConstantAlias); + memcpy(output, &config, sizeof(config)); + rejected((const void *)output, &one, 1, data, 8192, output, WORDS, AdrenoConstantAlias); + memcpy(output, &one, sizeof(one)); + rejected(&config, (const void *)output, 1, data, 8192, output, WORDS, AdrenoConstantAlias); + const struct AdrenoConstantSpan invalid[] = {{0,0,0},{0,0,15},{0,1,16},{8192,0,16}, + {0,8192,16},{UINT32_MAX,0,16},{0,UINT32_MAX-15,16},{0,0,UINT32_MAX-15}}; + reset(); + for (unsigned i = 0; i < sizeof(invalid)/sizeof(invalid[0]); ++i) + rejected(&config, &invalid[i], 1, data, 8192, output, WORDS, AdrenoConstantRange); + struct AdrenoConstantSpan late[] = {{0,0,16},{16,0,16}}; + rejected(&config, late, 2, data, 8192, output, WORDS, AdrenoConstantRange); + late[1].destination_byte=8192; + rejected(&config, late, 2, data, 8192, output, WORDS, AdrenoConstantRange); + assert(!munmap(mapping, (size_t)page*3)); + printf("PASS: %u direct upload cases, six stages/all 512 offsets, maximum extents, guard and %u transactional refusals\n", checks, refusals); + return 0; +} diff --git a/kits/AdrenoPM4/test_pm4.c b/kits/AdrenoPM4/test_pm4.c new file mode 100644 index 0000000..f7005b7 --- /dev/null +++ b/kits/AdrenoPM4/test_pm4.c @@ -0,0 +1,101 @@ +/* SPDX-License-Identifier: MIT */ +#include "AdrenoPM4.h" +#include +#include +#include +#define CHECK(c) do { if (!(c)) { fprintf(stderr, "line %d: %s\n", __LINE__, #c); exit(1); } } while (0) + +static unsigned population(uint32_t word) +{ + unsigned sum = 0; + for (unsigned bit = 0; bit < 32; ++bit) + sum += (word >> bit) & 1; + return sum; +} +static void refusal(const uint32_t *words, size_t length, size_t capacity, AdrenoPM4Status expected) +{ + AdrenoPM4Packet output[8], before[8]; + size_t count = 99; + memset(output, 0xa5, sizeof(output)); + memcpy(before, output, sizeof(output)); + CHECK(AdrenoPM4Plan(words, length, output, capacity, &count) == expected); + CHECK(!count && !memcmp(output, before, sizeof(output))); +} +int main(void) +{ + uint32_t header, words[0x4000] = {0}; + AdrenoPM4Packet packet[4]; + size_t count; + unsigned flips = 0; + CHECK(AdrenoPM4Type7(0x26, 0, &header) == AdrenoPM4Success && header == 0x70268000); + CHECK(AdrenoPM4Type7(0x10, 1, &header) == AdrenoPM4Success && header == 0x70100001); + CHECK(AdrenoPM4Type7(0x3d, 3, &header) == AdrenoPM4Success && header == 0x703d8003); + CHECK(AdrenoPM4Type4(0, 1, &header) == AdrenoPM4Success && header == 0x48000001); + /* Entire type-7 field domain: check independent population invariants, + * exact field preservation, reserved bits and real plan boundaries. */ + for (uint32_t opcode = 0; opcode <= 127; ++opcode) { + for (uint32_t n = 0; n <= 0x3fff; ++n) { + CHECK(AdrenoPM4Type7(opcode, n, &header) == AdrenoPM4Success); + CHECK((header & 0xff004000) == 0x70000000); + CHECK(((header >> 16) & 127) == opcode && (header & 0x3fff) == n); + CHECK(population(header & 0xffff) % 2 == 1); + CHECK(population((header >> 16) & 255) % 2 == 1); + words[0] = header; + CHECK(AdrenoPM4Plan(words, n + 1, packet, 1, &count) == AdrenoPM4Success); + CHECK(count == 1 && packet[0].type == 7 && packet[0].identifier == opcode && packet[0].payload_words == n); + if (n == 0 || n == 0x3fff) { + for (unsigned bit = 0; bit < 32; ++bit) { + words[0] = header ^ (1u << bit); + refusal(words, n + 1, 1, AdrenoPM4InvalidHeader); + ++flips; + } + } + } + } + /* All supported register indices and all count values. */ + for (uint32_t reg = 0; reg <= 0x3ffff; ++reg) { + CHECK(AdrenoPM4Type4(reg, 1, &header) == AdrenoPM4Success); + CHECK(((header >> 8) & 0x3ffff) == reg && (header & 127) == 1); + CHECK(population(header & 255) % 2 == 1 && population((header >> 8) & 0xfffff) % 2 == 1); + words[0] = header; + CHECK(AdrenoPM4Plan(words, 2, packet, 1, &count) == AdrenoPM4Success); + CHECK(count == 1 && packet[0].type == 4 && packet[0].identifier == reg); + } + for (uint32_t n = 1; n <= 127; ++n) { + CHECK(AdrenoPM4Type4(0, n, &header) == AdrenoPM4Success); + words[0] = header; + CHECK(AdrenoPM4Plan(words, n + 1, packet, 1, &count) == AdrenoPM4Success); + CHECK(packet[0].payload_words == n); + for (unsigned bit = 0; bit < 32; ++bit) { + words[0] = header ^ (1u << bit); + refusal(words, n + 1, 1, AdrenoPM4InvalidHeader); + ++flips; + } + } + /* Mixed packets, payload containing header-like data and transactional + * late failures: payload must never be interpreted as packet headers. */ + uint32_t mixed[] = {0x70100001, 0xffffffff, 0x48000001, 0x70000000, 0x70268000}; + CHECK(AdrenoPM4Plan(mixed, 5, packet, 3, &count) == AdrenoPM4Success && count == 3); + CHECK(packet[0].header_word == 0 && packet[1].header_word == 2 && packet[2].header_word == 4); + refusal(mixed, 5, 2, AdrenoPM4Capacity); + refusal(mixed, 3, 3, AdrenoPM4Truncated); + mixed[4] ^= 1u << 15; + refusal(mixed, 5, 3, AdrenoPM4InvalidHeader); + refusal(NULL, 1, 1, AdrenoPM4InvalidArgument); + refusal(words, SIZE_MAX, 1, AdrenoPM4InvalidArgument); + CHECK(AdrenoPM4Plan(NULL, 0, NULL, 0, &count) == AdrenoPM4Success && !count); + CHECK(AdrenoPM4Plan(words, 1, NULL, 1, &count) == AdrenoPM4InvalidArgument && !count); + CHECK(AdrenoPM4Plan(NULL, 0, NULL, 0, NULL) == AdrenoPM4InvalidArgument); + header = 0xa5a5a5a5; + CHECK(AdrenoPM4Type7(128, 1, &header) == AdrenoPM4InvalidArgument); + CHECK(AdrenoPM4Type7(0, 0x4000, &header) == AdrenoPM4InvalidArgument); + CHECK(AdrenoPM4Type4(0x40000, 1, &header) == AdrenoPM4InvalidArgument); + CHECK(AdrenoPM4Type4(0x3ffff, 2, &header) == AdrenoPM4InvalidArgument); + CHECK(AdrenoPM4Type4(0, 0, &header) == AdrenoPM4InvalidArgument); + CHECK(AdrenoPM4Type4(0, 128, &header) == AdrenoPM4InvalidArgument); + CHECK(header == 0xa5a5a5a5); + CHECK(AdrenoPM4Type7(0, 0, NULL) == AdrenoPM4InvalidArgument); + CHECK(AdrenoPM4Type4(0, 1, NULL) == AdrenoPM4InvalidArgument); + printf("PASS: 2097152 type-7 pairs, 262144 registers, 127 counts, %u single-bit corruptions; transactional stream refusals\n", flips); + return 0; +} diff --git a/kits/AdrenoPM4/test_raster.c b/kits/AdrenoPM4/test_raster.c new file mode 100644 index 0000000..9d81d7e --- /dev/null +++ b/kits/AdrenoPM4/test_raster.c @@ -0,0 +1,90 @@ +/* SPDX-License-Identifier: MIT */ +#include "AdrenoRaster.h" +#include "AdrenoPM4.h" +#include +#include +#include +#ifdef MESA_REFERENCE +size_t AdrenoRasterReference(const struct AdrenoRasterState *,uint32_t *,uint32_t *,size_t); +#endif +static unsigned positives,refusals; +static uint32_t out[80],saved[80]; +static struct AdrenoRasterState state; +static void fixture(void) { + state=(struct AdrenoRasterState){ADRENO_A740_CHIP_ID, + AdrenoRasterClipNear|AdrenoRasterClipFar|AdrenoRasterHalfZ|AdrenoRasterFrontCCW, + 3,16,2,0,0,0,0}; +} +static void reset(void) { memset(out,0x6b,sizeof out);memcpy(saved,out,sizeof out); } +static void positive_capacity(size_t capacity) { + reset();struct AdrenoRasterResult r=AdrenoA740Raster(&state,out,capacity); + int clamp=(state.flags&3)!=3; + assert(!r.status && r.words==(clamp?24u:60u) && r.registers==(clamp?12u:46u)); + assert(!memcmp(out+r.words,saved+r.words,sizeof out-r.words*4)); + AdrenoPM4Packet packets[16];size_t count=0; + assert(!AdrenoPM4Plan(out,r.words,packets,16,&count) && count==(clamp?12u:14u)); +#ifdef MESA_REFERENCE + uint32_t regs[64],values[64];size_t nr=AdrenoRasterReference(&state,regs,values,64),at=0; + assert(nr==r.registers); + for(size_t i=0;i +#include +#include +#ifdef MESA_REFERENCE +#include "util/u_math.h" +#include "adreno_common.xml.h" +#include "adreno_pm4.xml.h" +#include "a6xx_enums.xml.h" +#include "a7xx_enums.xml.h" +#include "a8xx_enums.xml.h" +#include "a6xx.xml.h" +#include "freedreno_pm4.h" +#endif +static uint32_t out[128], before[128]; +static unsigned positives, refusals; +static struct AdrenoShaderState state = {ADRENO_A740_CHIP_ID, AdrenoConstantVertex, 128, 1, + 0, -1, 0, 1, 0, 0, 0, 0, 0, 0, 0}; +static struct AdrenoShaderMapping mapping = {1, UINT64_C(0x100000000), 4096, 0, 12, 1}; +static void reset(void) { memset(out, 0x6b, sizeof(out)); memcpy(before, out, sizeof(out)); } +static void positive(void) +{ + reset(); + struct AdrenoShaderResult r = AdrenoA740ShaderBind(&state, &mapping, out, 128); + assert(!r.status && r.words == 24 && r.registers == 13); + AdrenoPM4Packet packets[12];size_t count=0; + assert(!AdrenoPM4Plan(out, r.words, packets, 12, &count) && count == 11); + uint32_t regs[13], values[13];unsigned n=0; + for (size_t i=0;itype==4 && p->payload_words>=1 && p->payload_words<=2); +#ifdef MESA_REFERENCE + assert(out[p->header_word]==pm4_pkt4_hdr(p->identifier,p->payload_words)); +#endif + for (unsigned j=0;jpayload_words;j++) {regs[n]=p->identifier+j;values[n++]=out[p->header_word+1+j];} + } + assert(n==13); + assert(regs[0]==0xab00 && values[0]==5 && regs[1]==0xb309 && values[1]==0xa2); + unsigned fragment=state.stage==AdrenoConstantFragment; + const uint32_t vs[]={0xa800,0xa824,0xa81b,0xa81c,0xa81d,0xa81e,0xa81f,0xa820,0xa821,0xa825,0xa82d}; + const uint32_t fs[]={0xa980,0xab05,0xa982,0xa983,0xa984,0xa985,0xa986,0xa987,0xa988,0xa9a9,0xaa01}; + assert(!memcmp(regs+2,fragment?fs:vs,sizeof(vs))); + uint32_t control; +#ifdef MESA_REFERENCE + assert(values[0]==(A6XX_SP_MODE_CNTL_CONSTANT_DEMOTION_ENABLE | A6XX_SP_MODE_CNTL_ISAMMODE(ISAMMODE_GL))); + assert(values[1]==(A6XX_TPL1_MODE_CNTL_ISAMMODE(ISAMMODE_GL) | + A6XX_TPL1_MODE_CNTL_TEXCOORDROUNDMODE(COORD_TRUNCATE) | + A6XX_TPL1_MODE_CNTL_ARRAYCOORDROUNDMODE(COORD_TRUNCATE) | + A6XX_TPL1_MODE_CNTL_NEARESTMIPSNAP(CLAMP_ROUND_TRUNCATE) | A6XX_TPL1_MODE_CNTL_DESTDATATYPEOVERRIDE)); + if (fragment) { + assert(regs[2]==REG_A6XX_SP_PS_CNTL_0 && regs[3]==REG_A6XX_SP_PS_INSTR_SIZE && + regs[5]==REG_A6XX_SP_PS_BASE && regs[12]==REG_A7XX_SP_PS_VGS_CNTL); + control=A6XX_SP_PS_CNTL_0_FULLREGFOOTPRINT(state.max_full_register+1) | + A6XX_SP_PS_CNTL_0_BRANCHSTACK(state.branch_stack_hw) | + A6XX_SP_PS_CNTL_0_THREADSIZE(state.double_threadsize?THREAD128:THREAD64) | + A6XX_SP_PS_CNTL_0_INOUTREGOVERLAP; + if (state.input_words) control|=A6XX_SP_PS_CNTL_0_VARYING; + if (state.need_full_quad) control|=A6XX_SP_PS_CNTL_0_LODPIXMASK; + if (state.need_pixel_lod) control|=A6XX_SP_PS_CNTL_0_PIXLODENABLE; + if (state.merged_registers) control|=A6XX_SP_PS_CNTL_0_MERGEDREGS; + if (state.early_preamble) control|=A6XX_SP_PS_CNTL_0_EARLYPREAMBLE; + } else { + assert(regs[2]==REG_A6XX_SP_VS_CNTL_0 && regs[3]==REG_A6XX_SP_VS_INSTR_SIZE && + regs[5]==REG_A6XX_SP_VS_BASE && regs[12]==REG_A7XX_SP_VS_VGS_CNTL); + control=A6XX_SP_VS_CNTL_0_FULLREGFOOTPRINT(state.max_full_register+1) | + A6XX_SP_VS_CNTL_0_BRANCHSTACK(state.branch_stack_hw); + if (state.merged_registers) control|=A6XX_SP_VS_CNTL_0_MERGEDREGS; + if (state.early_preamble) control|=A6XX_SP_VS_CNTL_0_EARLYPREAMBLE; + } +#else + control=(uint32_t)(state.max_full_register+1)<<7 | state.branch_stack_hw<<13; + if (fragment) control|=state.double_threadsize<<20 | (state.input_words!=0)<<22 | state.need_full_quad<<23 | + 1u<<24 | state.need_pixel_lod<<26 | state.early_preamble<<28 | state.merged_registers<<31; + else control|=state.merged_registers<<20 | state.early_preamble<<21; +#endif + assert(values[2]==control && values[3]==state.instruction_groups && !values[4]); + uint64_t address=mapping.base+mapping.shader_offset; + assert(values[5]==(uint32_t)address && values[6]==address>>32); + for (unsigned i=7;i<13;i++) assert(!values[i]); + assert(!memcmp(out+24,before+24,sizeof(out)-24*4)); + ++positives; +} +static void rejected(const struct AdrenoShaderState *s,const struct AdrenoShaderMapping *m, + uint32_t *destination,size_t capacity,unsigned expected) +{ + memcpy(before,out,sizeof(out)); + struct AdrenoShaderResult r=AdrenoA740ShaderBind(s,m,destination,capacity); + assert(r.status==expected && !r.words && !r.registers && !memcmp(out,before,sizeof(out))); + ++refusals; +} +int main(void) +{ + for (unsigned profile=0;profile<2;profile++) { + state.chip_id=profile?ADRENO_A740_FALLBACK_CHIP_ID:ADRENO_A740_CHIP_ID; + for (unsigned stage=0;stage<2;stage++) { + state.stage=stage?AdrenoConstantFragment:AdrenoConstantVertex; + state.merged_registers=1;state.early_preamble=0;state.double_threadsize=0; + state.input_words=0;state.need_full_quad=0;state.need_pixel_lod=0; + for (int full=-1;full<63;full++) for (unsigned stack=0;stack<=126;stack+=2) { + state.max_full_register=full;state.branch_stack_hw=stack;positive(); + } + state.max_full_register=0;state.branch_stack_hw=0; + for (unsigned flags=0;flags<64;flags++) { + state.merged_registers=flags&1;state.early_preamble=flags>>1&1;state.double_threadsize=flags>>2&1; + state.input_words=(flags>>3&1)?128:0; + state.need_full_quad=stage?(flags>>4&1):0;state.need_pixel_lod=stage?(flags>>5&1):0; + positive(); + } + } + } + state=(struct AdrenoShaderState){ADRENO_A740_CHIP_ID,AdrenoConstantVertex,128,1,0,-1,0,1,0,0,0,0,0,0,0}; + state.binary_bytes=384;state.instruction_groups=2;positive(); + state.binary_bytes=128;state.instruction_groups=1; + mapping.shader_offset=3968;positive(); + mapping.shader_offset=0;mapping.base=(UINT64_C(1)<<48)-4096;positive(); + mapping.base=UINT64_C(0x100000000);mapping.bytes=1024*1024; + state.binary_bytes=1024*1024;state.instruction_groups=8192;positive(); + state.binary_bytes=128;state.instruction_groups=1;mapping.bytes=4096; + reset(); + rejected(NULL,&mapping,out,128,AdrenoShaderArgument); + rejected(&state,NULL,out,128,AdrenoShaderArgument); + rejected(&state,&mapping,NULL,128,AdrenoShaderArgument); + rejected(&state,&mapping,(void *)((uint8_t *)out+1),127,AdrenoShaderArgument); + rejected(&state,&mapping,out,SIZE_MAX,AdrenoShaderArgument); + rejected((void *)(UINTPTR_MAX-3),&mapping,out,128,AdrenoShaderArgument); + rejected(&state,(void *)(UINTPTR_MAX-3),out,128,AdrenoShaderArgument); + rejected(&state,&mapping,(void *)(UINTPTR_MAX-3),24,AdrenoShaderArgument); + rejected(&state,&mapping,out,23,AdrenoShaderCapacity); + memcpy(out,&state,sizeof(state));rejected((void *)out,&mapping,out,128,AdrenoShaderAlias); + memcpy(out,&mapping,sizeof(mapping));rejected(&state,(void *)out,out,128,AdrenoShaderAlias); + for (unsigned i=0;i<19;i++) { + struct AdrenoShaderState bad=state; + switch(i) { + case 0:bad.chip_id^=UINT64_C(1)<<63;break; + case 1:bad.stage=AdrenoConstantCompute;break; + case 2:bad.binary_bytes=0;break; + case 3:bad.binary_bytes=127;break; + case 4:bad.binary_bytes=1024*1024+128;break; + case 5:bad.instruction_groups=0;break; + case 6:bad.max_full_register=63;break; + case 7:bad.max_full_register=-2;break; + case 8:bad.max_half_register=0;break; + case 9:bad.branch_stack_hw=1;break; + case 10:bad.branch_stack_hw=128;break; + case 11:bad.merged_registers=2;break; + case 12:bad.early_preamble=2;break; + case 13:bad.double_threadsize=2;break; + case 14:bad.input_words=129;break; + case 15:bad.need_full_quad=1;break; + case 16:bad.private_bytes=16;break; + case 17:bad.constant_data_bytes=16;break; + default:bad.instruction_groups=2;break; + } + rejected(&bad,&mapping,out,128,AdrenoShaderProfile); + } + for (unsigned i=0;i<12;i++) { + struct AdrenoShaderMapping bad=mapping; + switch(i) { + case 0:bad.context=0;break; + case 1:bad.base=0;break; + case 2:bad.base=UINT64_C(1)<<48;break; + case 3:bad.bytes=0;break; + case 4:bad.bytes=UINT64_MAX;break; + case 5:bad.base++;break; + case 6:bad.bytes--;break; + case 7:bad.page_shift=14;break; + case 8:bad.readonly=0;break; + case 9:bad.shader_offset=1;break; + case 10:bad.shader_offset=8192;break; + default:bad.shader_offset=4096;break; + } + rejected(&state,&bad,out,128,AdrenoShaderMappingError); + } + printf("PASS: %u shader register cases, both full identities and full-register/branch fields; %u transactional refusals, native-4K view required\n",positives,refusals); + return 0; +} diff --git a/kits/AdrenoPM4/test_vpc.c b/kits/AdrenoPM4/test_vpc.c new file mode 100644 index 0000000..1e8db7e --- /dev/null +++ b/kits/AdrenoPM4/test_vpc.c @@ -0,0 +1,127 @@ +/* SPDX-License-Identifier: MIT */ +#include "AdrenoVPC.h" +#include "AdrenoPM4.h" +#include +#include +#include +#ifdef MESA_REFERENCE +#include "util/u_math.h" +#include "ir3_shader.h" +#include "adreno_common.xml.h" +#include "adreno_pm4.xml.h" +#include "a6xx_enums.xml.h" +#include "a7xx_enums.xml.h" +#include "a8xx_enums.xml.h" +#include "a6xx.xml.h" +#include "freedreno_pm4.h" +#endif +static uint32_t out[128],saved[128]; +static unsigned positives,refusals; +static uint8_t names[15][32]; +static struct AdrenoVPCOutput outputs[16]; +static struct AdrenoVPCInput inputs[15]; +static struct AdrenoVPCState state={ADRENO_A740_CHIP_ID,62,0,outputs,2,inputs,1}; +static void reset(void) {memset(out,0x6b,sizeof out);memcpy(saved,out,sizeof out);} +static int equal(const uint8_t *a,size_t an,const uint8_t *b,size_t bn) {return an==bn && !memcmp(a,b,an);} +static void fixture(unsigned n) { + state=(struct AdrenoVPCState){ADRENO_A740_CHIP_ID,62,0,outputs,n+1,inputs,n}; + outputs[0]=(struct AdrenoVPCOutput){(const uint8_t *)"air.position",12,0,4,32,0,0,0}; + for(unsigned i=0;itype==4); +#ifdef MESA_REFERENCE + assert(out[p->header_word]==pm4_pkt4_hdr(p->identifier,p->payload_words)); +#endif + for(unsigned j=0;jpayload_words;j++){regs[nr]=p->identifier+j;values[nr++]=out[p->header_word+1+j];} + } + assert(nr==r.registers); +#ifdef MESA_REFERENCE + static struct ir3_shader_variant vs,fs; + vs.type=MESA_SHADER_VERTEX;fs.type=MESA_SHADER_FRAGMENT; + vs.outputs_count=state.output_count;fs.inputs_count=state.input_count;fs.total_in=124; + for(unsigned i=0;iidentity,input->identity_bytes,outputs[j].identity,outputs[j].identity_bytes))match=j; + assert(match<16);fs.inputs[i].slot=VARYING_SLOT_VAR0+match;fs.inputs[i].compmask=input->component_mask;fs.inputs[i].inloc=input->input_location;fs.inputs[i].bary=true; + } + struct ir3_shader_linkage link={0};ir3_link_shaders(&link,&vs,&fs,true); + uint32_t disabled[4];for(unsigned i=0;i<4;i++)disabled[i]=~link.varmask[i]; + unsigned position_location=link.max_loc;ir3_link_add(&link,VARYING_SLOT_POS,outputs[pos].register_id,15,link.max_loc); + assert(link.cnt==state.input_count+1 && link.max_loc==r.stride && position_location==r.position_location); + uint32_t packed[8]={0},dest[4]={0}; + for(unsigned i=0;i +#ifdef __cplusplus +extern "C" { +#endif +struct air_nir; +struct nir_def; +enum air_op { + AIR_EQ, + AIR_UGT, + AIR_OR, + AIR_AND, + AIR_SELECT, + AIR_FADD, + AIR_FSUB, + AIR_FMUL, + AIR_FDIV, + AIR_FMA, + AIR_FMA_FINITE_NSZ, + AIR_FSAT +}; +enum air_fp_flag { AIR_NNAN = 1, AIR_NINF = 2, AIR_NSZ = 4, AIR_ARCP = 8 }; +enum air_stage { AIR_VERTEX, AIR_FRAGMENT }; +struct air_nir *air_nir_create_stage(uint64_t chip, const char *name, enum air_stage); +struct air_nir *air_nir_create(uint64_t chip, const char *name); +struct nir_def *air_nir_bits(struct air_nir *, uint32_t bits, unsigned width); +struct nir_def *air_nir_vertex(struct air_nir *, unsigned width); +int air_nir_fragment_input(struct air_nir *, unsigned location, unsigned components, struct nir_def **); +int air_nir_attribute(struct air_nir *, unsigned location, unsigned components, struct nir_def **); +struct nir_def *air_nir_load(struct air_nir *, unsigned binding, unsigned offset); +struct nir_def *air_nir_op(struct air_nir *, enum air_op, unsigned fp_flags, struct nir_def *, + struct nir_def *, struct nir_def *); +int air_nir_store(struct air_nir *, unsigned slot, unsigned count, struct nir_def **); +int air_nir_color(struct air_nir *, unsigned slot, unsigned target, unsigned count, struct nir_def **); +int air_nir_test_eval(struct air_nir *, uint32_t vertex, const uint8_t *data, unsigned size, + float *out); +struct air_nir_test_buffer { + const uint8_t *data; + unsigned size; +}; +int air_nir_test_eval_resources(struct air_nir *, uint32_t vertex, const uint32_t (*attributes)[4], + const struct air_nir_test_buffer buffers[16], float *out); +int air_nir_lower_integer_ops(struct air_nir *); +int air_nir_compile(struct air_nir *, const char *prefix); +void air_nir_destroy(struct air_nir *); +#ifdef __cplusplus +} +#endif +#endif diff --git a/kits/AirNir/BackendABI.h b/kits/AirNir/BackendABI.h new file mode 100644 index 0000000..be743cc --- /dev/null +++ b/kits/AirNir/BackendABI.h @@ -0,0 +1,143 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef AIR_BACKEND_ABI_H +#define AIR_BACKEND_ABI_H +#include "ConstantUploads.h" +#include "ir3/ir3_shader.h" +#include + +static void air_json_string(FILE *f, const char *s) { + fputc('"', f); + for (const unsigned char *p = (const unsigned char *)s; *p; p++) { + if (*p == '"' || *p == '\\') + fprintf(f, "\\%c", *p); + else if (*p < 32) + fprintf(f, "\\u%04x", *p); + else + fputc(*p, f); + } + fputc('"', f); +} +static const char *air_allocation_name(unsigned i) { + const char *names[] = {"push_consts", + "dynamic_descriptor_offsets", + "inline_uniform_addresses", + "driver_params", + "ubo_ranges", + "preamble", + "global", + "ubo_pointers", + "image_dimensions", + "transform_feedback", + "primitive_params", + "primitive_map", + "bindless_base_addresses"}; + return i < sizeof(names) / sizeof(names[0]) ? names[i] : "unknown"; +} +static int air_export_backend_abi(FILE *f, const struct ir3_shader_variant *v, + const struct air_constant_read *reads, unsigned count) { + const struct ir3_const_state *cs = ir3_const_state(v); + const struct ir3_compiler *c = v->compiler; + if ((v->type != MESA_SHADER_VERTEX && v->type != MESA_SHADER_FRAGMENT) || v->inputs_count > 34 || v->outputs_count > 34 || + cs->ubo_state.num_enabled > AIR_CONSTANT_MAX_RANGES || count > AIR_CONSTANT_MAX_READS || + v->imm_state.size > AIR_CONSTANT_MAX_BYTES / 4 || + (v->imm_state.size && !v->imm_state.values) || v->constlen > AIR_CONSTANT_MAX_BYTES / 16) + return -1; + fprintf(f, + "{\n\"format_version\":1,\"stage\":\"%s\",\"generation\":%u,\"gpu_name\":", + v->type == MESA_SHADER_VERTEX ? "vertex" : "fragment", c->gen); + air_json_string(f, fd_dev_name(c->dev_id)); + fprintf(f, + ",\"chip_id\":\"0x%" PRIx64 "\",\"binary_bytes\":%u,\"code_dwords\":%u," + "\"binary_alignment_bytes\":%u,\"instruction_groups\":%u,\"constant_bytes\":%u," + "\"upload_alignment_bytes\":%u,\"max_full_register\":%d,\"max_half_register\":%d," + "\"branch_stack\":%u,\"private_bytes_per_fiber\":%u,\"private_per_wave\":%s," + "\"subgroup_size\":%u,\"double_threadsize\":%s,\"early_preamble\":%s," + "\"main_instructions\":%u,\"preamble_instructions\":%u," + "\"constant_data_offset\":%u,\"constant_data_bytes\":%u," + "\"app_ubo_slots\":%u,\"remaining_ubo_slots\":%u,\"driver_parameter_words\":%u," + "\"driver_params_ubo\":%d,\"primitive_params_ubo\":%d,\"primitive_map_ubo\":%d," + "\"push_ubo_with_preamble\":%s,\"load_shader_consts_via_preamble\":%s," + "\"input_attribute_slots\":%u,\"output_words\":%u,\"writes_position\":%s," + "\"runtime_binding_qualified\":false,\"gpu_executed\":false,\n", + c->dev_id->chip_id, v->info.size, v->info.sizedwords, c->instr_align * 8, v->instrlen, + v->constlen * 16, c->const_upload_unit * 16, v->info.max_reg, v->info.max_half_reg, + v->branchstack, v->pvtmem_size, v->pvtmem_per_wave ? "true" : "false", + v->info.subgroup_size, v->info.double_threadsize ? "true" : "false", + v->early_preamble ? "true" : "false", v->info.instrs_count, + v->info.preamble_instrs_count, v->info.constant_data_offset, v->constant_data_size, + cs->num_app_ubos, cs->num_ubos, cs->num_driver_params, cs->driver_params_ubo.idx, + cs->primitive_param_ubo.idx, cs->primitive_map_ubo.idx, + c->options.push_ubo_with_preamble ? "true" : "false", + c->info->props.load_shader_consts_via_preamble ? "true" : "false", v->attr_in, + v->output_size, v->writes_pos ? "true" : "false"); + fprintf(f, + "\"binding_state\":{\"branch_stack_hw\":%u,\"merged_registers\":%s," + "\"input_words\":%u,\"need_full_quad\":%s,\"need_pixel_lod\":%s},\n", + ir3_shader_branchstack_hw(v), v->mergedregs ? "true" : "false", v->total_in, + v->need_full_quad ? "true" : "false", v->need_pixlod ? "true" : "false"); + fprintf( + f, + "\"constant_limits_vec4\":{\"pipeline\":%u,\"geometry\":%u,\"fragment\":%u,\"safe\":%u},\n", + c->max_const_pipeline, c->max_const_geom, c->max_const_frag, c->max_const_safe); + fprintf(f, "\"allocations\":["); + for (unsigned i = 0; i < IR3_CONST_ALLOC_MAX; i++) { + if (i) + fputc(',', f); + const struct ir3_const_allocation *a = &cs->allocs.consts[i]; + fprintf(f, + "{\"type\":%u,\"name\":\"%s\",\"begin\":%u,\"bytes\":%u,\"reserved_vec4\":%u," + "\"reserved_alignment_vec4\":%u}", + i, air_allocation_name(i), a->offset_vec4 * 16, a->size_vec4 * 16, + a->reserved_size_vec4, a->reserved_align_vec4); + } + fprintf(f, "],\n\"ubo_ranges\":["); + for (unsigned i = 0; i < cs->ubo_state.num_enabled; i++) { + if (i) + fputc(',', f); + const struct ir3_ubo_range *r = &cs->ubo_state.range[i]; + fprintf(f, + "{\"binding\":%u,\"source_begin\":%u,\"source_end\":%u,\"destination_begin\":%u," + "\"bindless\":%s,\"global\":%s,\"descriptor_set\":%u}", + r->ubo.block, r->start, r->end, r->offset, r->ubo.bindless ? "true" : "false", + r->ubo.global ? "true" : "false", r->ubo.bindless_base); + } + fprintf(f, "],\n\"source_reads\":["); + for (unsigned i = 0; i < count; i++) { + if (i) + fputc(',', f); + fprintf(f, "{\"binding\":%u,\"offset\":%u,\"bytes\":%u}", reads[i].binding, reads[i].offset, + reads[i].bytes); + } + fprintf(f, + "],\n\"immediates\":{\"destination_begin\":%u,\"logical_words\":%u,\"padded_words\":%u," + "\"cpu_upload\":%s,\"values\":[", + cs->allocs.max_const_offset_vec4 * 16, v->imm_state.count, v->imm_state.size, + c->info->props.load_shader_consts_via_preamble ? "false" : "true"); + for (unsigned i = 0; i < v->imm_state.size; i++) { + if (i) + fputc(',', f); + fprintf(f, "%u", v->imm_state.values[i]); + } + fprintf(f, "]},\n\"inputs\":["); + for (unsigned i = 0; i < v->inputs_count; i++) { + if (i) + fputc(',', f); + fprintf(f, + "{\"slot\":%u,\"register_id\":%u,\"component_mask\":%u,\"inloc\":%u,\"system_" + "value\":%s,\"half\":%s}", + v->inputs[i].slot, v->inputs[i].regid, v->inputs[i].compmask, v->inputs[i].inloc, + v->inputs[i].sysval ? "true" : "false", v->inputs[i].half ? "true" : "false"); + } + fprintf(f, "],\n\"outputs\":["); + for (unsigned i = 0; i < v->outputs_count; i++) { + if (i) + fputc(',', f); + fprintf( + f, "{\"slot\":%u,\"register_id\":%u,\"view\":%u,\"aliased_components\":%u,\"half\":%s}", + v->outputs[i].slot, v->outputs[i].regid, v->outputs[i].view, + v->outputs[i].aliased_components, v->outputs[i].half ? "true" : "false"); + } + fprintf(f, "]\n}\n"); + return ferror(f) ? -1 : 0; +} +#endif diff --git a/kits/AirNir/ConstantUploads.c b/kits/AirNir/ConstantUploads.c new file mode 100644 index 0000000..41021fe --- /dev/null +++ b/kits/AirNir/ConstantUploads.c @@ -0,0 +1,128 @@ +/* SPDX-License-Identifier: MIT */ +#include "ConstantUploads.h" + +static int inside(size_t begin, size_t bytes, size_t limit) { + return begin <= limit && bytes <= limit - begin; +} +static int overlap(uintptr_t a, size_t an, uintptr_t b, size_t bn) { + if (!an || !bn) + return 0; + if (an > UINTPTR_MAX - a || bn > UINTPTR_MAX - b) + return 1; + return a < b + bn && b < a + an; +} +static struct air_constant_result failure(uint32_t status) { + return (struct air_constant_result){status, 0, 0}; +} +struct air_constant_result +air_pack_constants(const struct air_constant_layout *l, + const struct air_constant_binding bindings[AIR_CONSTANT_MAX_BINDINGS], + uint8_t *dst, size_t dst_bytes) { + if (!l || !bindings || !dst) + return failure(AIR_CONSTANT_ARGUMENT); + if (!l->constant_bytes || l->constant_bytes > AIR_CONSTANT_MAX_BYTES || + l->constant_bytes % 16 || l->range_count > AIR_CONSTANT_MAX_RANGES || + l->read_count > AIR_CONSTANT_MAX_READS || (l->range_count && !l->ranges) || + (l->read_count && !l->reads) || l->immediate_words > AIR_CONSTANT_MAX_BYTES / 4 || + (l->immediate_words && !l->immediates)) + return failure(AIR_CONSTANT_ARGUMENT); + if (dst_bytes < l->constant_bytes) + return failure(AIR_CONSTANT_CAPACITY); + uint32_t align = l->upload_alignment; + if (align < 16 || align > 256 || (align & (align - 1)) || + !inside(l->ubo_begin, l->ubo_bytes, l->constant_bytes) || l->ubo_begin % 16 || + l->ubo_bytes % 16 || + !inside(l->immediate_begin, (size_t)l->immediate_words * 4, l->constant_bytes) || + (l->immediate_words && (l->immediate_begin % 16 || l->immediate_words % 4))) + return failure(AIR_CONSTANT_LAYOUT); + if (overlap(l->ubo_begin, l->ubo_bytes, l->immediate_begin, (size_t)l->immediate_words * 4)) + return failure(AIR_CONSTANT_LAYOUT); + /* Metadata cannot change underneath a copy because it aliases the output. */ + uintptr_t output = (uintptr_t)dst; + if (overlap(output, l->constant_bytes, (uintptr_t)l, sizeof(*l)) || + overlap(output, l->constant_bytes, (uintptr_t)bindings, + sizeof(*bindings) * AIR_CONSTANT_MAX_BINDINGS) || + overlap(output, l->constant_bytes, (uintptr_t)l->ranges, + l->range_count * sizeof(*l->ranges)) || + overlap(output, l->constant_bytes, (uintptr_t)l->reads, + l->read_count * sizeof(*l->reads)) || + overlap(output, l->constant_bytes, (uintptr_t)l->immediates, + (size_t)l->immediate_words * 4)) + return failure(AIR_CONSTANT_ALIAS); + uint32_t written = l->immediate_words * 4, padded = 0; + for (size_t i = 0; i < l->range_count; i++) { + const struct air_constant_range *r = &l->ranges[i]; + if (r->binding >= AIR_CONSTANT_MAX_BINDINGS || r->source_begin >= r->source_end || + r->source_begin % align || r->source_end % align || r->destination_begin % align) + return failure(AIR_CONSTANT_LAYOUT); + uint32_t bytes = r->source_end - r->source_begin; + if (r->destination_begin < l->ubo_begin || + !inside(r->destination_begin - l->ubo_begin, bytes, l->ubo_bytes)) + return failure(AIR_CONSTANT_LAYOUT); + if (overlap(r->destination_begin, bytes, l->immediate_begin, + (size_t)l->immediate_words * 4)) + return failure(AIR_CONSTANT_LAYOUT); + for (size_t j = 0; j < i; j++) { + const struct air_constant_range *p = &l->ranges[j]; + if (overlap(r->destination_begin, bytes, p->destination_begin, + p->source_end - p->source_begin)) + return failure(AIR_CONSTANT_LAYOUT); + } + const struct air_constant_binding *b = &bindings[r->binding]; + if (!b->data || b->bytes < b->minimum_bytes || !b->minimum_bytes || + overlap(output, l->constant_bytes, (uintptr_t)b->data, b->bytes)) + return failure(b->data && b->bytes >= b->minimum_bytes && b->minimum_bytes + ? AIR_CONSTANT_ALIAS + : AIR_CONSTANT_BINDING); + int used = 0; + for (size_t j = 0; j < l->read_count; j++) { + const struct air_constant_read *read = &l->reads[j]; + if (read->binding == r->binding && read->offset >= r->source_begin && + inside(read->offset - r->source_begin, read->bytes, bytes)) + used = 1; + } + if (!used) + return failure(AIR_CONSTANT_READ); + size_t available = r->source_begin < b->bytes ? b->bytes - r->source_begin : 0; + if (available > bytes) + available = bytes; + padded += bytes - (uint32_t)available; + written += bytes; + } + for (size_t i = 0; i < l->read_count; i++) { + const struct air_constant_read *read = &l->reads[i]; + if (read->binding >= AIR_CONSTANT_MAX_BINDINGS || !read->bytes || read->bytes % 4 || + read->offset % 4) + return failure(AIR_CONSTANT_READ); + const struct air_constant_binding *b = &bindings[read->binding]; + if (!b->data || b->bytes < b->minimum_bytes || !inside(read->offset, read->bytes, b->bytes)) + return failure(AIR_CONSTANT_READ); + int covered = 0; + for (size_t j = 0; j < l->range_count; j++) { + const struct air_constant_range *r = &l->ranges[j]; + if (r->binding == read->binding && read->offset >= r->source_begin && + inside(read->offset - r->source_begin, read->bytes, + r->source_end - r->source_begin)) + covered = 1; + } + if (!covered) + return failure(AIR_CONSTANT_READ); + } + /* No condition below can fail. Padding is transport alignment only, and + * no recorded shader read may reference a byte that would be padded. */ + for (size_t i = 0; i < l->range_count; i++) { + const struct air_constant_range *r = &l->ranges[i]; + const struct air_constant_binding *b = &bindings[r->binding]; + uint32_t bytes = r->source_end - r->source_begin; + for (uint32_t j = 0; j < bytes; j++) { + size_t offset = (size_t)r->source_begin + j; + dst[r->destination_begin + j] = offset < b->bytes ? b->data[offset] : 0; + } + } + for (uint32_t i = 0; i < l->immediate_words; i++) { + uint32_t word = l->immediates[i]; + for (unsigned byte = 0; byte < 4; byte++) + dst[l->immediate_begin + i * 4 + byte] = (uint8_t)(word >> (byte * 8)); + } + return (struct air_constant_result){AIR_CONSTANT_OK, written, padded}; +} diff --git a/kits/AirNir/ConstantUploads.h b/kits/AirNir/ConstantUploads.h new file mode 100644 index 0000000..5ddc013 --- /dev/null +++ b/kits/AirNir/ConstantUploads.h @@ -0,0 +1,56 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef AIR_CONSTANT_UPLOADS_H +#define AIR_CONSTANT_UPLOADS_H +#include +#include +#ifdef __cplusplus +extern "C" { +#endif +#define AIR_CONSTANT_MAX_BINDINGS 16u +#define AIR_CONSTANT_MAX_RANGES 32u +#define AIR_CONSTANT_MAX_READS 4096u +#define AIR_CONSTANT_MAX_BYTES 16384u +struct air_constant_read { + uint32_t binding, offset, bytes; +}; +struct air_constant_range { + uint32_t binding, source_begin, source_end, destination_begin; +}; +struct air_constant_binding { + const uint8_t *data; + size_t bytes, minimum_bytes; +}; +struct air_constant_layout { + uint32_t constant_bytes, upload_alignment, ubo_begin, ubo_bytes; + const struct air_constant_range *ranges; + size_t range_count; + const struct air_constant_read *reads; + size_t read_count; + uint32_t immediate_begin, immediate_words; + const uint32_t *immediates; +}; +enum air_constant_status { + AIR_CONSTANT_OK, + AIR_CONSTANT_ARGUMENT, + AIR_CONSTANT_CAPACITY, + AIR_CONSTANT_LAYOUT, + AIR_CONSTANT_BINDING, + AIR_CONSTANT_READ, + AIR_CONSTANT_ALIAS +}; +struct air_constant_result { + uint32_t status, written_bytes, padded_bytes; +}; +/* Trusted readable metadata/backing lifetimes are the caller's responsibility. + * Validation precedes every write. Failure leaves destination unchanged and + * returns zero counts. Only declared ranges/immediates are written; internal + * preamble, primitive and driver-parameter allocations remain untouched. + * No allocation, GPU submission, memory translation or pixel computation. */ +struct air_constant_result +air_pack_constants(const struct air_constant_layout *, + const struct air_constant_binding bindings[AIR_CONSTANT_MAX_BINDINGS], + uint8_t *destination, size_t destination_bytes); +#ifdef __cplusplus +} +#endif +#endif diff --git a/kits/AirNir/Frontend.cpp b/kits/AirNir/Frontend.cpp new file mode 100644 index 0000000..0dfdc25 --- /dev/null +++ b/kits/AirNir/Frontend.cpp @@ -0,0 +1,880 @@ +// SPDX-License-Identifier: MIT +// Strict straight-line vertex/fragment AIR frontend. Unsupported constructs fail. +#include "AirNir.h" +#include "Library.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +using namespace llvm; +[[noreturn]] static void fail(const Twine &why) { + errs() << "AIR refused: " << why << '\n'; + std::exit(4); +} +static StringRef str(Metadata *m) { + auto *s = dyn_cast_or_null(m); + if (!s) + fail("metadata string required"); + return s->getString(); +} +static uint64_t integer(Metadata *m) { + auto *v = mdconst::dyn_extract_or_null(m); + if (!v || v->getBitWidth() > 64) + fail("metadata integer required"); + return v->getZExtValue(); +} +static MDNode *node(Metadata *m) { + auto *n = dyn_cast_or_null(m); + if (!n) + fail("metadata node required"); + return n; +} +static Metadata *at(MDNode *n, unsigned index) { + if (index >= n->getNumOperands()) + fail("metadata operand missing"); + return n->getOperand(index); +} +static int key(MDNode *n, StringRef name) { + for (unsigned i = 0; i < n->getNumOperands(); i++) + if (auto *s = dyn_cast_or_null(n->getOperand(i)); s && s->getString() == name) + return i; + return -1; +} +static uint64_t after(MDNode *n, StringRef name) { + int i = key(n, name); + if (i < 0 || unsigned(i + 1) >= n->getNumOperands()) + fail("missing metadata " + name); + return integer(n->getOperand(i + 1)); +} +static unsigned width(Type *t) { + if (t->isFloatTy()) + return 32; + if (t->isIntegerTy(1)) + return 1; + if (t->isIntegerTy(16) || t->isIntegerTy(32)) + return 32; + fail("unsupported scalar type"); +} +static unsigned lanes(Type *t) { + if (auto *v = dyn_cast(t)) { + if (v->getNumElements() > 4) + fail("vector wider than four"); + width(v->getElementType()); + return v->getNumElements(); + } + if (auto *s = dyn_cast(t)) { + unsigned result = 0; + for (Type *e : s->elements()) + result += lanes(e); + if (result > 64) + fail("aggregate too wide"); + return result; + } + width(t); + return 1; +} +struct Pointer { + unsigned binding, size; + uint64_t offset; +}; +using ValueLanes = std::vector; +class Lower { + air_nir *s; + const DataLayout &dl; + std::map values; + std::map pointers; + + public: + Lower(air_nir *shader, const DataLayout &layout) : s(shader), dl(layout) {} + ValueLanes get(Value *v) { + auto found = values.find(v); + if (found != values.end()) + return found->second; + unsigned n = lanes(v->getType()); + ValueLanes out(n, nullptr); + if (isa(v) || isa(v)) + return out; + if (auto *f = dyn_cast(v); f && v->getType()->isFloatTy()) + out[0] = air_nir_bits(s, f->getValueAPF().bitcastToAPInt().getZExtValue(), 32); + else if (auto *i = dyn_cast(v); i && v->getType()->isIntegerTy()) + out[0] = air_nir_bits(s, i->getZExtValue(), width(i->getType())); + else if (auto *c = dyn_cast(v); c && v->getType()->isVectorTy()) { + for (unsigned j = 0; j < n; j++) + out[j] = get(c->getAggregateElement(j))[0]; + } else if (auto *c = dyn_cast(v); c && v->getType()->isStructTy()) { + out.clear(); + for (unsigned j = 0; j < cast(v->getType())->getNumElements(); j++) { + auto data = get(c->getAggregateElement(j)); + out.insert(out.end(), data.begin(), data.end()); + } + } else + fail("unlowered value or unsupported constant"); + values[v] = out; + return out; + } + nir_def *defined(nir_def *v) { + if (!v) + fail("undefined lane reaches arithmetic or output"); + return v; + } + void buffer(Argument *a, unsigned binding, unsigned size) { pointers[a] = {binding, size, 0}; } + void vertex(Argument *a) { + values[a] = {air_nir_vertex(s, a->getType()->getIntegerBitWidth())}; + } + void attribute(Argument *a, unsigned location, unsigned count) { + ValueLanes data(count); + if (air_nir_attribute(s, location, count, data.data())) + fail("invalid vertex attribute"); + values[a] = data; + } + void fragment_input(Argument *a, unsigned location, unsigned count) { + ValueLanes data(count); + if (air_nir_fragment_input(s, location, count, data.data())) fail("invalid fragment input"); + values[a] = data; + } + void lower(Instruction &i) { + if (auto *g = dyn_cast(&i)) { + auto base = pointers.find(g->getPointerOperand()); + if (base == pointers.end()) + fail("GEP is not based on a declared buffer"); + APInt offset(64, 0); + if (!g->accumulateConstantOffset(dl, offset) || offset.isNegative() || + offset.getZExtValue() > base->second.size) + fail("nonconstant, negative or out-of-range GEP"); + auto p = base->second; + p.offset += offset.getZExtValue(); + if (p.offset > p.size) + fail("GEP outside buffer"); + pointers[&i] = p; + return; + } + if (auto *l = dyn_cast(&i)) { + auto p = pointers.find(l->getPointerOperand()); + Type *t = l->getType(); + auto *vt = dyn_cast(t); + bool floatType = t->isFloatTy() || (vt && vt->getElementType()->isFloatTy()); + unsigned count = vt ? vt->getNumElements() : 1; + unsigned bytes = count * 4; + if (p == pointers.end() || !floatType || count < 1 || count > 4 || l->isVolatile() || + l->isAtomic() || p->second.size < bytes || + p->second.offset > p->second.size - bytes || p->second.offset % 4 || + l->getAlign().value() < 4) + fail("unsupported or out-of-range buffer load"); + ValueLanes data; + for (unsigned k = 0; k < count; k++) + data.push_back(air_nir_load(s, p->second.binding, p->second.offset + k * 4)); + values[&i] = data; + return; + } + if (auto *e = dyn_cast(&i)) { + auto v = get(e->getOperand(0)); + auto *index = dyn_cast(e->getOperand(2)); + if (!index || index->getZExtValue() >= v.size()) + fail("dynamic or out-of-range insert"); + v[index->getZExtValue()] = get(e->getOperand(1))[0]; + values[&i] = v; + return; + } + if (auto *v = dyn_cast(&i)) { + auto a = get(v->getOperand(0)), b = get(v->getOperand(1)); + a.insert(a.end(), b.begin(), b.end()); + ValueLanes out; + for (int index : v->getShuffleMask()) + out.push_back(index < 0 ? nullptr : a.at(index)); + values[&i] = out; + return; + } + if (auto *v = dyn_cast(&i)) { + auto *type = dyn_cast(v->getType()); + if (!type || v->getNumIndices() != 1) + fail("nested aggregate insertion"); + unsigned index = *v->idx_begin(), offset = 0; + for (unsigned j = 0; j < index; j++) + offset += lanes(type->getElementType(j)); + auto out = get(v->getAggregateOperand()), in = get(v->getInsertedValueOperand()); + std::copy(in.begin(), in.end(), out.begin() + offset); + values[&i] = out; + return; + } + if (auto *z = dyn_cast(&i)) { + if (!z->getSrcTy()->isIntegerTy(16) || !z->getDestTy()->isIntegerTy(32)) + fail("unsupported zero extension"); + values[&i] = get(z->getOperand(0)); + return; + } + enum air_op op; + unsigned count = 2; + if (auto *cmp = dyn_cast(&i)) { + if (cmp->getPredicate() == CmpInst::ICMP_EQ) + op = AIR_EQ; + else if (cmp->getPredicate() == CmpInst::ICMP_UGT) + op = AIR_UGT; + else + fail("unsupported integer comparison"); + } else if (isa(i)) { + op = AIR_SELECT; + count = 3; + } else if (auto *c = dyn_cast(&i)) { + auto *f = c->getCalledFunction(); + if (f && f->getName() == "air.fast_saturate.f32" && c->arg_size() == 1 && + c->getType()->isFloatTy() && c->getArgOperand(0)->getType()->isFloatTy() && + c->hasNoNaNs() && c->hasNoInfs() && c->hasNoSignedZeros()) { + op = AIR_FSAT; + count = 1; + } else { + if (!f || f->getName() != "air.fma.v2f32" || c->arg_size() != 3 || + !c->getType()->isVectorTy() || + cast(c->getType())->getNumElements() != 2 || + !cast(c->getType())->getElementType()->isFloatTy()) + fail("unsupported AIR call"); + op = c->hasNoNaNs() && c->hasNoInfs() && c->hasNoSignedZeros() ? AIR_FMA_FINITE_NSZ : AIR_FMA; + count = 3; + } + } else if (i.getOpcode() == Instruction::Or && i.getType()->isIntegerTy(1)) + op = AIR_OR; + else if (i.getOpcode() == Instruction::And && + (i.getType()->isIntegerTy(16) || i.getType()->isIntegerTy(32))) + op = AIR_AND; + else if (i.getOpcode() == Instruction::FAdd) + op = AIR_FADD; + else if (i.getOpcode() == Instruction::FSub) + op = AIR_FSUB; + else if (i.getOpcode() == Instruction::FMul) + op = AIR_FMUL; + else if (i.getOpcode() == Instruction::FDiv) { + if (!i.hasAllowReciprocal()) + fail("strict division lowering is not implemented"); + op = AIR_FDIV; + } else + fail(Twine("unsupported instruction ") + i.getOpcodeName()); + unsigned flags = 0; + if (auto *fp = dyn_cast(&i)) { + if (fp->hasNoNaNs()) + flags |= AIR_NNAN; + if (fp->hasNoInfs()) + flags |= AIR_NINF; + if (fp->hasNoSignedZeros()) + flags |= AIR_NSZ; + if (fp->hasAllowReciprocal()) + flags |= AIR_ARCP; + } + unsigned n = lanes(i.getType()); + ValueLanes args[3]; + for (unsigned j = 0; j < count; j++) + args[j] = get(i.getOperand(j)); + ValueLanes out; + for (unsigned j = 0; j < n; j++) { + nir_def *in[3] = {nullptr, nullptr, nullptr}; + for (unsigned k = 0; k < count; k++) { + if (args[k].size() != n && !(op == AIR_SELECT && k == 0 && args[k].size() == 1)) + fail("operand shape mismatch"); + in[k] = defined(args[k][args[k].size() == 1 ? 0 : j]); + } + out.push_back(air_nir_op(s, op, flags, in[0], in[1], in[2])); + } + values[&i] = out; + } + void outputs(Value *v, Type *type, MDNode *metadata, bool fragment) { + auto data = get(v); + if (data.size() != lanes(type)) + fail("return aggregate shape"); + unsigned offset = 0, slot = 0; + std::vector elements; + if (auto *st = dyn_cast(type)) + elements.assign(st->element_begin(), st->element_end()); + else + elements.push_back(type); + for (Type *e : elements) { + unsigned n = lanes(e); + for (unsigned j = 0; j < n; j++) + defined(data[offset + j]); + unsigned target = fragment ? integer(at(node(at(metadata, slot)), 1)) : slot; + if (fragment ? air_nir_color(s, slot, target, n, data.data() + offset) + : air_nir_store(s, slot, n, data.data() + offset)) + fail("invalid output"); + slot++; + offset += n; + } + } +}; +static Function *validate(Module &m, MDNode *&outputs, MDNode *&inputs, bool &fragment) { + if (m.getTargetTriple().str() != "air64-apple-ios17.7.0" || + !m.getDataLayout().isLittleEndian() || m.getDataLayout().getPointerSizeInBits(2) != 64) + fail("unsupported triple or data layout"); + if (verifyModule(m, &errs())) + fail("LLVM verification failed"); + auto *version = m.getNamedMetadata("air.version"); + if (!version || version->getNumOperands() != 1) + fail("AIR version missing"); + auto *v = version->getOperand(0); + if (v->getNumOperands() != 3 || integer(at(v, 0)) != 2 || integer(at(v, 1)) != 6 || + integer(at(v, 2)) != 0) + fail("unsupported AIR version"); + auto *options = m.getNamedMetadata("air.compile_options"); + bool denormDisable = false, fastMath = false; + if (options) { + for (MDNode *option : options->operands()) { + if (option->getNumOperands() != 1) + fail("compile option shape"); + StringRef name = str(at(option, 0)); + denormDisable |= name == "air.compile.denorms_disable"; + fastMath |= name == "air.compile.fast_math_enable"; + } + } + if (!denormDisable || !fastMath) + fail("only explicit fast-math and denorm-disabled AIR is qualified"); + for (GlobalVariable &g : m.globals()) { + auto *t = dyn_cast(g.getValueType()); + if (g.getName() != "llvm.global_ctors" || !t || t->getNumElements() != 0 || + !g.hasInitializer() || !g.getInitializer()->isNullValue()) + fail("globals or function-constant initialization unsupported"); + } + for (const char *kind : {"air.kernel", "air.visible"}) + if (m.getNamedMetadata(kind)) + fail("unsupported shader stage"); + auto *vertex = m.getNamedMetadata("air.vertex"); + auto *frag = m.getNamedMetadata("air.fragment"); + if (bool(vertex) == bool(frag)) + fail("exactly one vertex or fragment stage required"); + fragment = frag != nullptr; + auto *stage = fragment ? frag : vertex; + if (stage->getNumOperands() != 1) + fail("exactly one entry required"); + auto *entry = stage->getOperand(0); + if (entry->getNumOperands() != 3) + fail("entry metadata shape"); + auto *vm = dyn_cast(at(entry, 0)); + auto *f = vm ? dyn_cast(vm->getValue()) : nullptr; + if (!f || f->isDeclaration() || f->size() != 1 || f->isVarArg()) + fail("single-block function required"); + for (Function &other : m) + if (!other.isDeclaration() && &other != f) + fail("multiple function bodies"); + outputs = node(at(entry, 1)); + inputs = node(at(entry, 2)); + std::vector elements; + if (auto *ret = dyn_cast(f->getReturnType())) + elements.assign(ret->element_begin(), ret->element_end()); + else + elements.push_back(f->getReturnType()); + if (elements.empty() || elements.size() > 16 || elements.size() != outputs->getNumOperands()) + fail("return metadata mismatch"); + std::map targets; + for (unsigned j = 0; j < elements.size(); j++) { + if (fragment) { + auto *meta = node(at(outputs, j)); + Type *type = elements[j]; + auto *vector = dyn_cast(type); + if (elements.size() > 8 || (!type->isFloatTy() && (!vector || + !vector->getElementType()->isFloatTy() || vector->getNumElements() > 4)) || + str(at(meta, 0)) != "air.render_target" || meta->getNumOperands() < 3) + fail("unsupported fragment output"); + uint64_t target = integer(at(meta, 1)); + if (target >= 8 || integer(at(meta, 2)) != 0 || targets.count(target)) + fail("fragment target range, dual source or duplicate"); + targets[target] = true; + continue; + } + auto *v = dyn_cast(elements[j]); + auto *meta = node(at(outputs, j)); + if (!v || !v->getElementType()->isFloatTy() || v->getNumElements() > 4 || + str(at(meta, 0)) != (j ? "air.vertex_output" : "air.position") || + (!j && v->getNumElements() != 4)) + fail("unsupported vertex output"); + if (j && (meta->getNumOperands() < 2 || !str(at(meta, 1)).starts_with("generated("))) + fail("varying identity missing"); + } + if (inputs->getNumOperands() != f->arg_size()) + fail("input metadata mismatch"); + if (!isa(f->front().getTerminator())) + fail("return required"); + return f; +} +struct Input { + unsigned argument, index, components, size; + std::string kind, bound_source; + bool unused; + std::string identity = ""; + unsigned alignment = 0; +}; +static bool argument_offset(Value *v, Argument *a, const DataLayout &dl, uint64_t &offset) { + if (v == a) { + offset = 0; + return true; + } + auto *g = dyn_cast(v); + if (!g || !argument_offset(g->getPointerOperand(), a, dl, offset)) + return false; + APInt delta(64, 0); + if (!g->accumulateConstantOffset(dl, delta) || delta.isNegative() || + delta.getZExtValue() > 65536 || offset > 65536 - delta.getZExtValue()) + fail("unqualified static buffer extent"); + offset += delta.getZExtValue(); + return true; +} +static uint64_t static_extent(Argument *a) { + uint64_t extent = 0; + const DataLayout &dl = a->getParent()->getParent()->getDataLayout(); + for (Instruction &i : instructions(a->getParent())) + if (auto *l = dyn_cast(&i)) { + uint64_t offset; + if (!argument_offset(l->getPointerOperand(), a, dl, offset)) + continue; + TypeSize bytes = dl.getTypeStoreSize(l->getType()); + if (bytes.isScalable() || bytes.getFixedValue() > 65536 || + offset > 65536 - bytes.getFixedValue()) + fail("unqualified static buffer extent"); + extent = std::max(extent, offset + bytes.getFixedValue()); + } + return extent; +} +static uint64_t static_alignment(Argument *a) { + uint64_t alignment = a->getParamAlign().valueOrOne().value(); + const DataLayout &dl = a->getParent()->getParent()->getDataLayout(); + for (Instruction &i : instructions(a->getParent())) + if (auto *l = dyn_cast(&i)) { + uint64_t offset; + if (!argument_offset(l->getPointerOperand(), a, dl, offset)) + continue; + uint64_t required = l->getAlign().value(); + if (offset % required) + fail("unsupported buffer base alignment residue"); + alignment = std::max(alignment, required); + } + return alignment; +} +static std::vector parse_inputs(Function *f, MDNode *inputs, bool fragment) { + std::vector schema; + std::map bindings, locations; + unsigned vertices = 0, attributes = 0, varyings = 0; + std::map identities; + for (unsigned j = 0; j < f->arg_size(); j++) { + auto *a = f->getArg(j); + auto *meta = node(at(inputs, j)); + if (meta->getNumOperands() < 2 || integer(at(meta, 0)) != j) + fail("input index"); + StringRef kind = str(at(meta, 1)); + Input in = {j, 0, 0, 0, kind.str(), "", key(meta, "air.arg_unused") >= 0}; + if (in.unused && !a->use_empty()) + fail("unused argument metadata contradicts body"); + if (!fragment && kind == "air.vertex_id") { + if (!(a->getType()->isIntegerTy(16) || a->getType()->isIntegerTy(32)) || ++vertices > 1) + fail("vertex ID type/count"); + in.components = 1; + } else if (!fragment && kind == "air.vertex_input") { + auto *type = dyn_cast(a->getType()); + if (!type || !type->getElementType()->isFloatTy() || type->getNumElements() > 4) + fail("unsupported vertex attribute type"); + uint64_t location = after(meta, "air.location_index"); + int loc = key(meta, "air.location_index"); + if (location >= 16 || integer(at(meta, loc + 2)) != 1 || locations.count(location)) + fail("vertex attribute location/range overlap"); + locations[location] = true; + in.index = location; + in.components = type->getNumElements(); + attributes++; + } else if (fragment && kind == "air.position") { + auto *type = dyn_cast(a->getType()); + if (!in.unused || !type || !type->getElementType()->isFloatTy() || type->getNumElements() != 4 || + key(meta, "air.center") < 0 || key(meta, "air.no_perspective") < 0 || + key(meta, "air.perspective") >= 0 || key(meta, "air.centroid") >= 0 || key(meta, "air.sample") >= 0) + fail("only unused center fragment position is qualified"); + in.components = 4; + } else if (fragment && kind == "air.fragment_input") { + Type *type = a->getType(); + if (!(type->isFloatTy() || isa(type)) || !type->getScalarType()->isFloatTy() || + lanes(type) > 4 || varyings >= 16 || meta->getNumOperands() < 3 || + key(meta, "air.center") < 0 || key(meta, "air.perspective") < 0 || + key(meta, "air.no_perspective") >= 0 || key(meta, "air.centroid") >= 0 || + key(meta, "air.sample") >= 0 || key(meta, "air.flat") >= 0) + fail("unsupported fragment interpolation/type"); + in.identity = str(at(meta, 2)).str(); + if (!StringRef(in.identity).starts_with("generated(") || !StringRef(in.identity).ends_with(")") || in.identity.size() <= 11 || identities.count(in.identity)) + fail("fragment varying identity missing or duplicate"); + identities[in.identity] = true; + in.index = varyings++; + in.components = lanes(type); + } else if (kind == "air.buffer") { + uint64_t size, binding = after(meta, "air.location_index"); + if (key(meta, "air.buffer_size") >= 0) { + size = after(meta, "air.buffer_size"); + in.bound_source = "air.buffer_size"; + } else { + size = std::max(after(meta, "air.arg_type_size"), static_extent(a)); + size = std::max(size, a->getDereferenceableBytes()); + in.bound_source = "type-and-static-access-span"; + } + int loc = key(meta, "air.location_index"); + uint64_t alignment = static_alignment(a); + if (key(meta, "air.arg_type_align_size") >= 0) + alignment = std::max(alignment, after(meta, "air.arg_type_align_size")); + if (!alignment || (alignment & (alignment - 1)) || alignment > 4096) + fail("unsupported constant buffer alignment"); + in.alignment = alignment; + if (!a->getType()->isPointerTy() || a->getType()->getPointerAddressSpace() != 2 || + after(meta, "air.address_space") != 2 || key(meta, "air.read") < 0 || + key(meta, "air.write") >= 0 || (!in.unused && !a->onlyReadsMemory()) || !size || + size > 65536 || binding >= 16 || bindings.count(binding) || + integer(at(meta, loc + 2)) != 1) + fail("constant buffer contract"); + bindings[binding] = true; + in.index = binding; + in.size = size; + } else + fail("unsupported input kind"); + schema.push_back(in); + } + if (!fragment && !vertices && !attributes) + fail("vertex ID or attributes required"); + return schema; +} +static void interface_file(Function *f, MDNode *outputs, const std::vector &schema, + StringRef prefix, bool fragment) { + json::Array ins, outs; + for (const Input &in : schema) { + json::Object item{{"argument", in.argument}, {"kind", in.kind}, {"index", in.index}, + {"components", in.components}, {"minimum_bytes", in.size}, {"minimum_alignment", in.alignment}, + {"bound_source", in.bound_source}, {"unused", in.unused}}; + if (in.kind == "air.fragment_input") { + item["identity"] = in.identity; + item["interpolation"] = "perspective"; + item["sampling"] = "center"; + } + ins.push_back(std::move(item)); + } + Type *r = f->getReturnType(); + auto *s = dyn_cast(r); + for (unsigned i = 0; i < outputs->getNumOperands(); i++) { + Type *type = s ? s->getElementType(i) : r; + auto *meta = node(at(outputs, i)); + json::Object output{{"slot", i}, {"components", lanes(type)}, + {"identity", fragment ? "air.render_target" : i ? str(at(meta, 1)).str() : "air.position"}}; + if (fragment) { + output["target"] = integer(at(meta, 1)); + output["blend_index"] = integer(at(meta, 2)); + } + outs.push_back(std::move(output)); + } + std::error_code ec; + raw_fd_ostream out(prefix.str() + ".interface.json", ec); + if (ec) + fail("interface output open"); + out << formatv("{0:2}", json::Value(json::Object{{"function", f->getName().str()}, + {"stage", fragment ? "fragment" : "vertex"}, + {"inputs", std::move(ins)}, + {"outputs", std::move(outs)}, + {"runtime_resource_layout_qualified", false}})) + << "\n"; +} +static void oracle(Module &m, Function *f, StringRef path) { + // Test oracle only: retarget a clone of the original IR and replace the + // explicitly supported AIR FMA with LLVM's fused operation. + for (Instruction &i : instructions(f)) + if (auto *c = dyn_cast(&i)) { + auto *callee = c->getCalledFunction(); + if (!callee || callee->getName() != "air.fma.v2f32") + fail("oracle unknown call"); + c->setCalledFunction( + Intrinsic::getOrInsertDeclaration(&m, Intrinsic::fma, {c->getType()})); + } + auto &ctx = m.getContext(); + IRBuilder<> b(ctx); + Type *ptr = PointerType::getUnqual(ctx), *i16 = Type::getInt16Ty(ctx); + bool vertexOnly = f->arg_size() == 1 && f->getArg(0)->getType() == i16; + if (!vertexOnly && (f->arg_size() != 2 || !f->getArg(0)->getType()->isPointerTy() || + f->getArg(1)->getType() != i16)) + fail("oracle ABI requires optional buffer0 and ushort vertex"); + auto *wrapper = Function::Create(FunctionType::get(b.getVoidTy(), {ptr, i16, ptr}, false), + GlobalValue::ExternalLinkage, "air_test_oracle", m); + b.SetInsertPoint(BasicBlock::Create(ctx, "entry", wrapper)); + Value *r = + vertexOnly + ? b.CreateCall(f, {wrapper->getArg(1)}) + : b.CreateCall(f, {b.CreateAddrSpaceCast(wrapper->getArg(0), f->getArg(0)->getType()), + wrapper->getArg(1)}); + auto *rt = dyn_cast(f->getReturnType()); + unsigned slots = rt ? rt->getNumElements() : 1; + for (unsigned j = 0; j < slots; j++) { + Value *vector = rt ? b.CreateExtractValue(r, j) : r; + unsigned n = cast(vector->getType())->getNumElements(); + for (unsigned k = 0; k < n; k++) + b.CreateStore(b.CreateExtractElement(vector, k), + b.CreateGEP(b.getFloatTy(), wrapper->getArg(2), b.getInt32(j * 4 + k))); + } + b.CreateRetVoid(); + m.setTargetTriple(Triple(sys::getDefaultTargetTriple())); + if (verifyModule(m, &errs())) + fail("oracle LLVM verification"); + std::error_code ec; + raw_fd_ostream stream(path, ec); + if (ec) + fail("oracle output open"); + WriteBitcodeToFile(m, stream); +} +static void resource_oracle(Module &m, Function *f, const std::vector &schema, + StringRef path) { + // Host numerical reference only. Clamp finite values through LLVM's + // min/max intrinsics; interpolation itself is supplied, never rasterized. + std::vector calls; + for (Instruction &i : instructions(f)) + if (auto *c = dyn_cast(&i)) calls.push_back(c); + for (CallInst *c : calls) { + auto *callee = c->getCalledFunction(); + if (callee && callee->getName() == "air.fma.v2f32") { + c->setCalledFunction(Intrinsic::getOrInsertDeclaration(&m, Intrinsic::fma, {c->getType()})); + } else if (callee && callee->getName() == "air.fast_saturate.f32" && c->arg_size() == 1 && + c->getType()->isFloatTy() && c->hasNoNaNs() && c->hasNoInfs() && c->hasNoSignedZeros()) { + IRBuilder<> builder(c); + Value *low = builder.CreateBinaryIntrinsic(Intrinsic::maxnum, c->getArgOperand(0), ConstantFP::get(c->getType(), 0.0)); + Value *high = builder.CreateBinaryIntrinsic(Intrinsic::minnum, low, ConstantFP::get(c->getType(), 1.0)); + c->replaceAllUsesWith(high); + c->eraseFromParent(); + } else fail("resource oracle unknown call"); + } + // Preserve the frontend's separate-rounding policy in the host reference. + for (Instruction &i : instructions(f)) + if (isa(i)) i.copyFastMathFlags(FastMathFlags()); + for (const char *name : {"unsafe-fp-math", "approx-func-fp-math", "no-infs-fp-math", + "no-nans-fp-math", "no-signed-zeros-fp-math"}) + f->removeFnAttr(name); + auto &ctx = m.getContext(); + IRBuilder<> b(ctx); + Type *ptr = PointerType::getUnqual(ctx); + auto *wrapper = + Function::Create(FunctionType::get(b.getVoidTy(), {ptr, ptr, b.getInt32Ty(), ptr}, false), + GlobalValue::ExternalLinkage, "air_test_resource_oracle", m); + b.SetInsertPoint(BasicBlock::Create(ctx, "entry", wrapper)); + std::vector args; + for (const Input &in : schema) { + Type *type = f->getArg(in.argument)->getType(); + Value *value; + if (in.kind == "air.vertex_id") + value = b.CreateIntCast(wrapper->getArg(2), type, false); + else if (in.kind == "air.position") + value = Constant::getNullValue(type); // Validated unused float4 only. + else if (in.kind == "air.buffer") { + Value *address = b.CreateGEP(ptr, wrapper->getArg(1), b.getInt32(in.index)); + value = b.CreateAddrSpaceCast(b.CreateLoad(ptr, address), type); + } else { + value = PoisonValue::get(type); + for (unsigned k = 0; k < in.components; k++) { + Value *address = + b.CreateGEP(b.getFloatTy(), wrapper->getArg(0), b.getInt32(in.index * 4 + k)); + Value *component = b.CreateLoad(b.getFloatTy(), address); + value = type->isFloatTy() ? component : b.CreateInsertElement(value, component, k); + } + } + args.push_back(value); + } + Value *r = b.CreateCall(f, args); + auto *rt = dyn_cast(f->getReturnType()); + unsigned slots = rt ? rt->getNumElements() : 1; + for (unsigned j = 0; j < slots; j++) { + Value *vector = rt ? b.CreateExtractValue(r, j) : r; + unsigned count = lanes(vector->getType()); + for (unsigned k = 0; k < count; k++) + b.CreateStore(vector->getType()->isVectorTy() ? b.CreateExtractElement(vector, k) : vector, + b.CreateGEP(b.getFloatTy(), wrapper->getArg(3), b.getInt32(j * 4 + k))); + } + b.CreateRetVoid(); + m.setTargetTriple(Triple(sys::getDefaultTargetTriple())); + if (verifyModule(m, &errs())) + fail("resource oracle LLVM verification"); + std::error_code ec; + raw_fd_ostream out(path, ec); + if (ec) + fail("resource oracle output open"); + WriteBitcodeToFile(m, out); +} +static int library_sha256(const uint8_t *data, size_t bytes, uint8_t out[32], void *) { + auto digest = SHA256::hash(ArrayRef(data, bytes)); + for (unsigned i = 0; i < 32; i++) out[i] = digest[i]; + return 0; +} +int main(int argc, char **argv) { + bool library_input = argc == 6 && !strcmp(argv[4], "--compile-library"); + if (argc != 5 && argc != 7 && !library_input) { + errs() << "usage: air_nir INPUT.bc CHIP_ID OUTPUT_PREFIX --compile [--test-evaluate " + "CASES.bin]\n" + " air_nir INPUT.metallib CHIP_ID OUTPUT_PREFIX --compile-library INDEX\n" + " air_nir INPUT.bc CHIP_ID OUTPUT.bc --test-oracle\n" + " air_nir INPUT.bc CHIP_ID OUTPUT_PREFIX --compile --test-resources " + "CASES.bin\n" + " air_nir INPUT.bc CHIP_ID OUTPUT.bc --test-resource-oracle\n"; + return 2; + } + LLVMContext ctx; + auto buffer = MemoryBuffer::getFile(argv[1]); + if (!buffer) + fail("input read"); + if ((*buffer)->getBufferSize() > (library_input ? AIR_LIBRARY_MAX_BYTES : 16u * 1024u * 1024u)) + fail("input too large"); + MemoryBufferRef bitcode = (*buffer)->getMemBufferRef(); + struct air_library_function library_function = {}; + if (library_input) { + if (!argv[5][0]) fail("library function index"); + unsigned index = 0; + for (const char *p = argv[5]; *p; ++p) { + if (*p < '0' || *p > '9' || index > AIR_LIBRARY_MAX_FUNCTIONS / 10) + fail("library function index"); + index = index * 10 + unsigned(*p - '0'); + if (index >= AIR_LIBRARY_MAX_FUNCTIONS) fail("library function index"); + } + struct air_library_view view; + uint32_t error = air_library_open(reinterpret_cast(bitcode.getBufferStart()), + bitcode.getBufferSize(), library_sha256, nullptr, &view); + if (!error) error = air_library_get(&view, index, &library_function); + if (error) fail("library container status " + Twine(error)); + if (library_function.bitcode_bytes > 16u * 1024u * 1024u) fail("module too large"); + bitcode = MemoryBufferRef(StringRef(reinterpret_cast(library_function.bitcode), + library_function.bitcode_bytes), bitcode.getBufferIdentifier()); + } + auto parsed = parseBitcodeFile(bitcode, ctx); + if (!parsed) { + consumeError(parsed.takeError()); + fail("bitcode parse"); + } + auto m = std::move(*parsed); + MDNode *outputs, *inputs; + bool fragment; + Function *f = validate(*m, outputs, inputs, fragment); + if (library_input && (f->getName() != StringRef(reinterpret_cast(library_function.name), + library_function.name_bytes) || library_function.stage != unsigned(fragment))) + fail("library record contradicts actual AIR entry"); + auto schema = parse_inputs(f, inputs, fragment); + if (!strcmp(argv[4], "--test-oracle") && argc == 5) { + oracle(*m, f, argv[3]); + return 0; + } + if (!strcmp(argv[4], "--test-resource-oracle") && argc == 5) { + resource_oracle(*m, f, schema, argv[3]); + return 0; + } + bool resources = argc == 7 && !strcmp(argv[5], "--test-resources"); + bool testInt32 = argc == 7 && (!strcmp(argv[5], "--test-evaluate-int32") || resources); + if ((!library_input && strcmp(argv[4], "--compile")) || + (argc == 7 && strcmp(argv[5], "--test-evaluate") && !testInt32)) + return 2; + char *end = nullptr; + uint64_t chip = strtoull(argv[2], &end, 0); + if (!end || *end || !chip) + fail("chip ID"); + air_nir *s = air_nir_create_stage(chip, f->getName().str().c_str(), fragment ? AIR_FRAGMENT : AIR_VERTEX); + if (!s) + fail("target or mandatory asm roundtrip"); + Lower lower(s, m->getDataLayout()); + for (const Input &in : schema) { + auto *a = f->getArg(in.argument); + if (in.unused) + continue; + if (in.kind == "air.position") + continue; // Validated unused builtin: no fabricated position input. + if (in.kind == "air.vertex_id") + lower.vertex(a); + else if (in.kind == "air.fragment_input") + lower.fragment_input(a, in.index, in.components); + else if (in.kind == "air.vertex_input") + lower.attribute(a, in.index, in.components); + else + lower.buffer(a, in.index, in.size); + } + for (Instruction &i : f->front()) { + if (auto *r = dyn_cast(&i)) + lower.outputs(r->getReturnValue(), f->getReturnType(), outputs, fragment); + else + lower.lower(i); + } + interface_file(f, outputs, schema, argv[3], fragment); + if (testInt32 && air_nir_lower_integer_ops(s)) + fail("int64 arithmetic did not lower completely"); + if (resources) { + std::ifstream input(argv[6], std::ios::binary); + std::ofstream output(std::string(argv[3]) + ".eval", std::ios::binary); + if (!input || !output) + fail("resource test IO"); + unsigned count = 0; + for (;;) { + uint32_t vertex; + input.read(reinterpret_cast(&vertex), 4); + if (!input.gcount() && input.eof()) + break; + if (input.gcount() != 4 || ++count > 65536) + fail("resource case framing/bound"); + uint32_t attributes[16][4]; + input.read(reinterpret_cast(attributes), sizeof(attributes)); + if (input.gcount() != sizeof(attributes)) + fail("resource attribute framing"); + std::vector data[16]; + struct air_nir_test_buffer buffers[16] = {}; + for (const Input &in : schema) + if (in.kind == "air.buffer") { + data[in.index].resize(in.size); + input.read(reinterpret_cast(data[in.index].data()), in.size); + if (unsigned(input.gcount()) != in.size) + fail("resource buffer framing"); + buffers[in.index] = {data[in.index].data(), in.size}; + } + float out[64] = {}; + if (air_nir_test_eval_resources(s, vertex, attributes, buffers, out)) + fail("resource NIR evaluation"); + output.write(reinterpret_cast(out), + outputs->getNumOperands() * 4 * sizeof(float)); + } + if (!count || !output) + fail("resource evaluation output"); + } else if (argc == 7) { + unsigned size = 0, buffers = 0; + for (const Input &in : schema) { + if (in.kind == "air.vertex_input") + fail("old test ABI has no attributes"); + if (in.kind == "air.buffer") { + if (++buffers > 1 || in.index != 0) + fail("old test ABI requires optional binding0"); + size = in.size; + } + } + std::ifstream input(argv[6], std::ios::binary); + std::ofstream output(std::string(argv[3]) + ".eval", std::ios::binary); + if (!input || !output) + fail("test IO"); + unsigned count = 0; + for (;;) { + uint32_t vertex; + input.read(reinterpret_cast(&vertex), 4); + if (input.gcount() == 0 && input.eof()) + break; + if (input.gcount() != 4 || ++count > 65536) + fail("test case framing/bound"); + std::vector data(size); + if (size) + input.read(reinterpret_cast(data.data()), size); + float out[64] = {}; + if ((size && unsigned(input.gcount()) != size) || + air_nir_test_eval(s, vertex, data.data(), size, out)) + fail("NIR constant evaluation"); + output.write(reinterpret_cast(out), + outputs->getNumOperands() * 4 * sizeof(float)); + } + if (!count || !output) + fail("test evaluation output"); + } + int result = air_nir_compile(s, argv[3]); + air_nir_destroy(s); + return result ? 5 : 0; +} diff --git a/kits/AirNir/FusedF32.h b/kits/AirNir/FusedF32.h new file mode 100644 index 0000000..88cf88c --- /dev/null +++ b/kits/AirNir/FusedF32.h @@ -0,0 +1,137 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Derived from Berkeley SoftFloat Release 3e, s_mulAddF32.c, + * s_roundPackToF32.c, s_normSubnormalF32Sig.c and shift/jam primitives. + * Copyright 2011-2018 The Regents of the University of California. + * All rights reserved. Full notice/conditions/disclaimer: licenses/SoftFloat-3e.txt. + * + * NIR integer instruction construction, not a host floating-point operation. + * Round-to-nearest, ties-to-even, canonical quiet NaNs, no exception flags. + * 64-bit integer intermediates are subsequently lowered to native GPU 32-bit + * integer operations by NIR. No floating-point multiply or unfused MAD is used. + */ +#ifndef AIR_FUSED_F32_H +#define AIR_FUSED_F32_H +#include "nir/nir_builder.h" + +/* Unlike the upstream primitive this helper also defines distance == 0. + * Saturation prevents modulo-width GPU shifts from losing the sticky bit. */ +static nir_def *air_shift_jam(nir_builder *b, nir_def *v, nir_def *distance) { + unsigned bits = v->bit_size; + nir_def *small = nir_ult_imm(b, distance, bits); + nir_def *d = nir_umin(b, distance, nir_imm_int(b, bits - 1)); + nir_def *shifted = nir_ushr(b, v, d); + nir_def *lost = nir_ine(b, nir_ishl(b, shifted, d), v); + nir_def *jammed = nir_ior(b, shifted, nir_b2iN(b, lost, bits)); + return nir_bcsel(b, small, jammed, nir_b2iN(b, nir_ine_imm(b, v, 0), bits)); +} +struct air_sig_exp { + nir_def *sig, *exp; +}; +static struct air_sig_exp air_normalize_f32(nir_builder *b, nir_def *ui) { + nir_def *exp = nir_iand_imm(b, nir_ushr_imm(b, ui, 23), 255); + nir_def *frac = nir_iand_imm(b, ui, 0x7fffff); + nir_def *shift = nir_isub(b, nir_imm_int(b, 23), nir_ufind_msb(b, frac)); + nir_def *subnormal = nir_ieq_imm(b, exp, 0); + return (struct air_sig_exp){ + nir_bcsel(b, subnormal, nir_ishl(b, frac, shift), frac), + nir_bcsel(b, subnormal, nir_isub(b, nir_imm_int(b, 1), shift), exp)}; +} +static nir_def *air_round_f32(nir_builder *b, nir_def *sign, nir_def *exp, nir_def *sig) { + nir_def *negative_exp = nir_ilt_imm(b, exp, 0); + nir_def *overflow = + nir_ior(b, nir_ilt(b, nir_imm_int(b, 253), exp), + nir_iand(b, nir_ieq_imm(b, exp, 253), + nir_uge(b, nir_iadd_imm(b, sig, 0x40), nir_imm_int(b, 0x80000000)))); + sig = nir_bcsel(b, negative_exp, air_shift_jam(b, sig, nir_ineg(b, exp)), sig); + exp = nir_bcsel(b, negative_exp, nir_imm_int(b, 0), exp); + nir_def *tie = nir_ieq_imm(b, nir_iand_imm(b, sig, 127), 64); + sig = nir_ushr_imm(b, nir_iadd_imm(b, sig, 64), 7); + sig = nir_iand(b, sig, nir_inot(b, nir_b2i32(b, tie))); + exp = nir_bcsel(b, nir_ieq_imm(b, sig, 0), nir_imm_int(b, 0), exp); + /* Addition is intentional: the explicit hidden bit carries into exponent. */ + nir_def *packed = nir_iadd(b, nir_iadd(b, sign, nir_ishl_imm(b, exp, 23)), sig); + return nir_bcsel(b, overflow, nir_ior_imm(b, sign, 0x7f800000), packed); +} +static nir_def *air_fused_f32(nir_builder *b, nir_def *a, nir_def *v, nir_def *c) { + struct air_sig_exp na = air_normalize_f32(b, a), nb = air_normalize_f32(b, v); + struct air_sig_exp nc = air_normalize_f32(b, c); + nir_def *sign_p = nir_iand_imm(b, nir_ixor(b, a, v), 0x80000000); + nir_def *sign_c = nir_iand_imm(b, c, 0x80000000); + nir_def *exp_p = nir_iadd_imm(b, nir_iadd(b, na.exp, nb.exp), -126); + nir_def *ma = nir_ishl_imm(b, nir_ior_imm(b, na.sig, 0x800000), 7); + nir_def *mb = nir_ishl_imm(b, nir_ior_imm(b, nb.sig, 0x800000), 7); + nir_def *prod = nir_imul(b, nir_u2u64(b, ma), nir_u2u64(b, mb)); + nir_def *short_product = nir_ult(b, prod, nir_imm_int64(b, UINT64_C(0x2000000000000000))); + prod = nir_bcsel(b, short_product, nir_ishl_imm(b, prod, 1), prod); + exp_p = nir_isub(b, exp_p, nir_b2i32(b, short_product)); + nir_def *mc = nir_ishl_imm(b, nir_ior_imm(b, nc.sig, 0x800000), 6); + nir_def *c64 = nir_ishl_imm(b, nir_u2u64(b, mc), 32); + nir_def *diff = nir_isub(b, exp_p, nc.exp); + nir_def *negative = nir_ilt_imm(b, diff, 0); + nir_def *positive = nir_ilt(b, nir_imm_int(b, 0), diff); + nir_def *max_exp = nir_bcsel(b, negative, nc.exp, exp_p); + + /* Same-sign addition, preserving all product bits until final rounding. */ + nir_def *sum_c = nir_iadd( + b, mc, nir_u2u32(b, air_shift_jam(b, prod, nir_isub(b, nir_imm_int(b, 32), diff)))); + nir_def *sum_p = nir_u2u32( + b, air_shift_jam(b, nir_iadd(b, prod, air_shift_jam(b, c64, diff)), nir_imm_int(b, 32))); + nir_def *sum = nir_bcsel(b, positive, sum_p, sum_c); + nir_def *small_sum = nir_ult_imm(b, sum, 0x40000000); + nir_def *sum_exp = nir_isub(b, max_exp, nir_b2i32(b, small_sum)); + sum = nir_bcsel(b, small_sum, nir_ishl_imm(b, sum, 1), sum); + + /* Opposite-sign subtraction and cancellation normalization. */ + nir_def *equal_difference = nir_isub(b, prod, c64); + nir_def *equal_negative = nir_ilt_imm(b, equal_difference, 0); + nir_def *equal_magnitude = + nir_bcsel(b, equal_negative, nir_ineg(b, equal_difference), equal_difference); + nir_def *magnitude = nir_bcsel( + b, negative, nir_isub(b, c64, air_shift_jam(b, prod, nir_ineg(b, diff))), + nir_bcsel(b, positive, nir_isub(b, prod, air_shift_jam(b, c64, diff)), equal_magnitude)); + nir_def *sub_sign = nir_bcsel( + b, negative, sign_c, + nir_bcsel(b, positive, sign_p, + nir_ixor(b, sign_p, nir_ishl_imm(b, nir_b2i32(b, equal_negative), 31)))); + nir_def *shift = nir_isub(b, nir_imm_int(b, 62), nir_ufind_msb(b, magnitude)); + nir_def *sub_exp = nir_isub(b, max_exp, shift); + nir_def *remaining = nir_iadd_imm(b, shift, -32); + nir_def *sub_sig = nir_bcsel(b, nir_ilt_imm(b, remaining, 0), + nir_u2u32(b, air_shift_jam(b, magnitude, nir_ineg(b, remaining))), + nir_ishl(b, nir_u2u32(b, magnitude), remaining)); + nir_def *same_sign = nir_ieq(b, sign_p, sign_c); + nir_def *result = air_round_f32(b, nir_bcsel(b, same_sign, sign_p, sub_sign), + nir_bcsel(b, same_sign, sum_exp, sub_exp), + nir_bcsel(b, same_sign, sum, sub_sig)); + result = nir_bcsel(b, nir_iand(b, nir_inot(b, same_sign), nir_ieq_imm(b, magnitude, 0)), + nir_imm_int(b, 0), result); + /* A zero addend bypasses the fictitious normalized significand for zero. */ + nir_def *zero_c = nir_ieq_imm(b, nir_iand_imm(b, c, 0x7fffffff), 0); + nir_def *only_product = air_round_f32(b, sign_p, nir_iadd_imm(b, exp_p, -1), + nir_u2u32(b, air_shift_jam(b, prod, nir_imm_int(b, 31)))); + result = nir_bcsel(b, zero_c, only_product, result); + nir_def *zero_a = nir_ieq_imm(b, nir_iand_imm(b, a, 0x7fffffff), 0); + nir_def *zero_b = nir_ieq_imm(b, nir_iand_imm(b, v, 0x7fffffff), 0); + result = nir_bcsel(b, nir_ior(b, zero_a, zero_b), + nir_bcsel(b, zero_c, nir_iand(b, sign_p, sign_c), c), result); + + /* Canonical quiet NaN; exception flags and NaN-payload compatibility + * are not implemented or claimed by this compiler path. */ + nir_def *ea = nir_iand_imm(b, nir_ushr_imm(b, a, 23), 255); + nir_def *eb = nir_iand_imm(b, nir_ushr_imm(b, v, 23), 255); + nir_def *ec = nir_iand_imm(b, nir_ushr_imm(b, c, 23), 255); + nir_def *special_a = nir_ieq_imm(b, ea, 255), *special_b = nir_ieq_imm(b, eb, 255); + nir_def *special_c = nir_ieq_imm(b, ec, 255); + nir_def *nan_a = nir_iand(b, special_a, nir_ine_imm(b, nir_iand_imm(b, a, 0x7fffff), 0)); + nir_def *nan_b = nir_iand(b, special_b, nir_ine_imm(b, nir_iand_imm(b, v, 0x7fffff), 0)); + nir_def *nan_c = nir_iand(b, special_c, nir_ine_imm(b, nir_iand_imm(b, c, 0x7fffff), 0)); + nir_def *inf_p = nir_ior(b, special_a, special_b); + nir_def *invalid = nir_iand( + b, inf_p, + nir_ior(b, nir_ior(b, zero_a, zero_b), nir_iand(b, special_c, nir_ine(b, sign_p, sign_c)))); + result = nir_bcsel(b, special_c, c, result); + result = nir_bcsel(b, inf_p, nir_ior_imm(b, sign_p, 0x7f800000), result); + return nir_bcsel(b, nir_ior(b, invalid, nir_ior(b, nan_a, nir_ior(b, nan_b, nan_c))), + nir_imm_int(b, 0x7fc00000), result); +} +#endif diff --git a/kits/AirNir/LICENSE b/kits/AirNir/LICENSE new file mode 100644 index 0000000..9b9bc8a --- /dev/null +++ b/kits/AirNir/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 AirNir contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/kits/AirNir/Library.c b/kits/AirNir/Library.c new file mode 100644 index 0000000..50cb68f --- /dev/null +++ b/kits/AirNir/Library.c @@ -0,0 +1,141 @@ +/* SPDX-License-Identifier: MIT */ +#include "Library.h" +static uint32_t le32(const uint8_t *p) { + return (uint32_t)p[0] | (uint32_t)p[1]<<8 | (uint32_t)p[2]<<16 | (uint32_t)p[3]<<24; +} +static uint32_t be32(const uint8_t *p) { + return (uint32_t)p[3] | (uint32_t)p[2]<<8 | (uint32_t)p[1]<<16 | (uint32_t)p[0]<<24; +} +static uint64_t le64(const uint8_t *p) { return le32(p) | (uint64_t)le32(p+4)<<32; } +static int inside(size_t at, size_t n, size_t limit) { return at<=limit && n<=limit-at; } +static int overlap(uintptr_t a, size_t an, uintptr_t b, size_t bn) { + if (an>UINTPTR_MAX-a || bn>UINTPTR_MAX-b) return 1; + return an && bn && a=88 && equal(p,magic,16); +} +static uint32_t select_slice(const uint8_t *p, size_t n, struct air_library_view *v) { + size_t offset=0,size=n; + if (n>=8 && be32(p)==0xcafebabe) { + uint32_t count=be32(p+4),selected=0; + if (!count || count>16 || !inside(8,(size_t)count*20,n)) return AIR_LIBRARY_FORMAT; + size_t previous=8+(size_t)count*20; + for (uint32_t i=0;i20 || at%((size_t)1<size || bytes>size-at || at<88) return AIR_LIBRARY_BOUNDS; + section[i][0]=(size_t)at;section[i][1]=(size_t)bytes; + if (i && section[i-1][0]+section[i-1][1]>at) return AIR_LIBRARY_BOUNDS; + } + size_t begin=section[0][0],bytes=section[0][1]; + if (!inside(begin,4,size) || !inside(begin+4,bytes,size)) return AIR_LIBRARY_BOUNDS; + size_t end=begin+4+bytes; + if (!inside(end,48,size) || end+48!=section[1][0]) return AIR_LIBRARY_FORMAT; + const uint8_t *trailer=s+end; + static const uint8_t rl[6]={'R','L','S','T',16,0}, uuid[6]={'U','U','I','D',16,0}; + if (!equal(trailer,rl,6) || !equal(trailer+22,uuid,6) || le32(trailer+44)!=0x54444e45) return AIR_LIBRARY_FORMAT; + uint64_t reflection=le64(trailer+6),length=le64(trailer+14); + if (reflection!=section[3][0]+section[3][1] || reflection>size || length!=size-reflection) return AIR_LIBRARY_BOUNDS; + uint32_t count=le32(s+begin); + if (!count || count>AIR_LIBRARY_MAX_FUNCTIONS || (size_t)count>bytes/8) return AIR_LIBRARY_FORMAT; + *v=(struct air_library_view){p,n,offset,size,begin+4,bytes,count,0}; + return AIR_LIBRARY_OK; +} +static uint32_t record(const struct air_library_view *v, size_t *cursor, uint32_t index, struct air_library_function *out) { + const uint8_t *s=v->data+v->slice_offset; + size_t end=v->functions_offset+v->functions_bytes,at=*cursor; + if (!inside(at,4,end)) return AIR_LIBRARY_BOUNDS; + size_t size=le32(s+at); + if (size<8 || !inside(at,size,end)) return AIR_LIBRARY_BOUNDS; + end=at+size;at+=4; + const uint8_t *name=0,*type=0,*hash=0,*offsets=0,*module_size=0; + size_t name_bytes=0;uint32_t seen[16];unsigned tags=0;int terminated=0; + while (at1024 || name[name_bytes-1]) return AIR_LIBRARY_FORMAT; + for (size_t i=0;i+1126) return AIR_LIBRARY_FORMAT; + if (*type!=0 && *type!=1 && *type!=2 && *type!=4) return AIR_LIBRARY_FORMAT; + uint64_t public=le64(offsets),private=le64(offsets+8),module=le64(offsets+16),bytes=le64(module_size); + uint64_t public_bytes=le64(s+48),private_bytes=le64(s+64),code_begin=le64(s+72),code_bytes=le64(s+80); + if (public>=public_bytes || private>=private_bytes || module>code_bytes || bytes<20 || bytes>code_bytes-module) return AIR_LIBRARY_BOUNDS; + const uint8_t *wrapper=s+(size_t)(code_begin+module); + if (le32(wrapper)!=0x0b17c0de || le32(wrapper+4)!=0 || le32(wrapper+16)!=0xffffffff) return AIR_LIBRARY_FORMAT; + size_t bc=le32(wrapper+8),bc_bytes=le32(wrapper+12); + if (bc<20 || bc_bytes<=4 || !inside(bc,bc_bytes,(size_t)bytes)) return AIR_LIBRARY_BOUNDS; + if (le32(wrapper+bc)!=0xdec04342) return AIR_LIBRARY_FORMAT; + *out=(struct air_library_function){name,wrapper+bc,wrapper,hash,name_bytes-1,bc_bytes,(size_t)bytes,index,*type}; + *cursor=end; + return AIR_LIBRARY_OK; +} +uint32_t air_library_open(const uint8_t *p, size_t n, air_library_sha256 hash, void *context, struct air_library_view *out) { + if (!p || !out || !hash || !n || n>AIR_LIBRARY_MAX_BYTES || n>UINTPTR_MAX-(uintptr_t)p) return AIR_LIBRARY_ARGUMENT; + if (overlap((uintptr_t)p,n,(uintptr_t)out,sizeof(*out))) return AIR_LIBRARY_ALIAS; + struct air_library_view v;uint32_t error=select_slice(p,n,&v); + if (error) return error; + size_t cursor=v.functions_offset;const uint8_t *previous=0;size_t previous_bytes=0;const uint8_t *previous_hash=0; + for (uint32_t i=0;idata || !v->bytes || v->bytes>AIR_LIBRARY_MAX_BYTES || v->bytes>UINTPTR_MAX-(uintptr_t)v->data) return AIR_LIBRARY_ARGUMENT; + if (overlap((uintptr_t)v,sizeof(*v),(uintptr_t)out,sizeof(*out)) || overlap((uintptr_t)v->data,v->bytes,(uintptr_t)out,sizeof(*out))) return AIR_LIBRARY_ALIAS; + struct air_library_view actual;uint32_t error=select_slice(v->data,v->bytes,&actual); + if (error) return error; + if (actual.slice_offset!=v->slice_offset || actual.slice_bytes!=v->slice_bytes || actual.functions_offset!=v->functions_offset || actual.functions_bytes!=v->functions_bytes || actual.count!=v->count || v->reserved) return AIR_LIBRARY_FORMAT; + if (index>=v->count) return AIR_LIBRARY_INDEX; + size_t cursor=v->functions_offset; + for (uint32_t i=0;i<=index;i++) { + struct air_library_function f;error=record(v,&cursor,i,&f); + if (error) return error; + if (i==index) { *out=f;return AIR_LIBRARY_OK; } + } + return AIR_LIBRARY_INDEX; +} diff --git a/kits/AirNir/Library.h b/kits/AirNir/Library.h new file mode 100644 index 0000000..98abe08 --- /dev/null +++ b/kits/AirNir/Library.h @@ -0,0 +1,43 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef AIR_LIBRARY_H +#define AIR_LIBRARY_H +#include +#include +#ifdef __cplusplus +extern "C" { +#endif +#define AIR_LIBRARY_MAX_BYTES (64u * 1024u * 1024u) +#define AIR_LIBRARY_MAX_FUNCTIONS 4096u +enum air_library_status { + AIR_LIBRARY_OK, AIR_LIBRARY_ARGUMENT, AIR_LIBRARY_FORMAT, + AIR_LIBRARY_BOUNDS, AIR_LIBRARY_HASH, AIR_LIBRARY_INDEX, AIR_LIBRARY_ALIAS +}; +/* The callback must hash all supplied bytes, write exactly 32 digest bytes and + * return zero on success. It is trusted code, not a library-supplied callback. */ +typedef int (*air_library_sha256)(const uint8_t *, size_t, uint8_t[32], void *); +struct air_library_view { + const uint8_t *data; + size_t bytes, slice_offset, slice_bytes; + size_t functions_offset, functions_bytes; + uint32_t count, reserved; +}; +struct air_library_function { + const uint8_t *name, *bitcode, *wrapper, *wrapper_sha256; + size_t name_bytes, bitcode_bytes, wrapper_bytes; + uint32_t index, stage; +}; +/* The input, callback and output must be valid accessible storage. Opening + * checks every record and complete module hash before publishing a view. + * Only the observed AIR-bearing MTLB profile is accepted. Fat selection uses + * the actual container profile, never a host/Apple-GPU ISA assumption. + * A view and its backing must remain unchanged/alive for get(); callers must + * own or pin immutable bytes. This is structural/integrity validation, not + * AMFI, a signature trust decision or permission to execute a shader. + * Both calls leave outputs unchanged on error. No allocation or copying of + * library contents, hardware mappings, GPU work or pixel computation. */ +uint32_t air_library_open(const uint8_t *, size_t, air_library_sha256, void *, struct air_library_view *); +uint32_t air_library_get(const struct air_library_view *, uint32_t, struct air_library_function *); +#ifdef __cplusplus +} +#endif +#endif diff --git a/kits/AirNir/MesaBackend.c b/kits/AirNir/MesaBackend.c new file mode 100644 index 0000000..492e966 --- /dev/null +++ b/kits/AirNir/MesaBackend.c @@ -0,0 +1,439 @@ +/* SPDX-License-Identifier: MIT */ +#include "AirNir.h" +#include "BackendABI.h" +#include "FusedF32.h" +#include "compiler/glsl_types.h" +#include "freedreno_dev_info.h" +#include "ir3/ir3_compiler.h" +#include "ir3/ir3_shader.h" +#include "nir/nir_builder.h" +#include +#include +#include + +struct air_nir { + struct ir3_compiler *compiler; + struct fd_dev_id dev_id; + struct air_constant_read reads[AIR_CONSTANT_MAX_READS]; + unsigned read_count; + nir_builder b; + unsigned outputs; + bool allow_reciprocal; + unsigned reciprocal_lowerings; +}; +struct air_nir *air_nir_create_stage(uint64_t chip, const char *name, enum air_stage stage) { + if (stage != AIR_VERTEX && stage != AIR_FRAGMENT) + return NULL; + const struct fd_dev_id id = {.chip_id = chip}; + const struct fd_dev_info *info = fd_dev_info_raw(&id); + if (!info || fd_dev_gen(&id) != 7) + return NULL; + struct air_nir *s = calloc(1, sizeof(*s)); + if (!s) + return NULL; + s->dev_id = id; + const struct ir3_compiler_options options = { + .disable_cache = true, .bindless_fb_read_descriptor = -1, .bindless_fb_read_slot = -1}; + s->compiler = ir3_compiler_create(NULL, &s->dev_id, info, &options); + if (!s->compiler || !(ir3_shader_debug & IR3_DBG_ASM_ROUNDTRIP)) { + if (s->compiler) + ir3_compiler_destroy(s->compiler); + free(s); + return NULL; + } + glsl_type_singleton_init_or_ref(); + s->b = nir_builder_init_simple_shader(stage == AIR_VERTEX ? MESA_SHADER_VERTEX : MESA_SHADER_FRAGMENT, ir3_get_compiler_options(s->compiler), + "%s", name); + s->b.shader->info.internal = false; + return s; +} +struct air_nir *air_nir_create(uint64_t chip, const char *name) { + return air_nir_create_stage(chip, name, AIR_VERTEX); +} +struct nir_def *air_nir_bits(struct air_nir *s, uint32_t bits, unsigned width) { + return nir_imm_intN_t(&s->b, bits, width); +} +struct nir_def *air_nir_vertex(struct air_nir *s, unsigned width) { + nir_def *v = nir_load_vertex_id(&s->b); + return width == 16 ? nir_iand_imm(&s->b, v, 0xffff) : v; +} +int air_nir_attribute(struct air_nir *s, unsigned location, unsigned components, nir_def **out) { + if (location >= 16 || !components || components > 4) + return -1; + nir_def *v = nir_load_input(&s->b, components, 32, nir_imm_int(&s->b, 0), .base = location, + .component = 0, .dest_type = nir_type_float32, + .io_semantics = {.location = VERT_ATTRIB_GENERIC0 + location}); + for (unsigned i = 0; i < components; i++) + out[i] = nir_channel(&s->b, v, i); + s->b.shader->num_inputs = MAX2(s->b.shader->num_inputs, location + 1); + return 0; +} +int air_nir_fragment_input(struct air_nir *s, unsigned location, unsigned components, nir_def **out) { + if (s->b.shader->info.stage != MESA_SHADER_FRAGMENT || location >= 16 || !components || components > 4) + return -1; + nir_def *bary = nir_load_barycentric_pixel(&s->b, 32, .interp_mode = INTERP_MODE_SMOOTH); + nir_def *v = nir_load_interpolated_input(&s->b, components, 32, bary, nir_imm_int(&s->b, 0), + .base = location, .component = 0, .dest_type = nir_type_float32, + .io_semantics = {.location = VARYING_SLOT_VAR0 + location, .num_slots = 1}); + for (unsigned i = 0; i < components; i++) out[i] = nir_channel(&s->b, v, i); + s->b.shader->num_inputs = MAX2(s->b.shader->num_inputs, location + 1); + return 0; +} +struct nir_def *air_nir_load(struct air_nir *s, unsigned binding, unsigned offset) { + s->b.shader->info.num_ubos = MAX2(s->b.shader->info.num_ubos, binding + 1); + return nir_load_ubo(&s->b, 1, 32, nir_imm_int(&s->b, binding), nir_imm_int(&s->b, offset), + .align_mul = 4, .align_offset = 0, .range_base = offset, .range = 4); +} +/* Exact for a coefficient in {+0,+1} under the source call's explicit + * nnan/ninf/nsz contract. The product introduces no rounding. The zero + * product may return the addend directly because zero sign is immaterial. + * This does not weaken arbitrary-coefficient FMA or infer flags globally. */ +static nir_def *exact_finite_add(nir_builder *b, nir_def *value, nir_def *addend) { + unsigned saved = b->fp_math_ctrl; + /* Source flags permit discarding special-value/zero-sign distinctions, + * but this identity does not authorize another rounding or reassociation. */ + b->fp_math_ctrl = nir_fp_exact; + nir_def *sum = nir_fadd(b, value, addend); + b->fp_math_ctrl = saved; + return sum; +} +static nir_def *finite_binary_coefficient(nir_builder *b, nir_def *coef, nir_def *value, + nir_def *addend) { + if (coef->bit_size != 32 || coef->num_components != 1) + return NULL; + nir_src source = nir_src_for_ssa(coef); + if (nir_src_is_const(source)) { + uint32_t bits = nir_src_as_uint(source); + if (!(bits & 0x7fffffff)) + return addend; + if (bits == 0x3f800000) + return exact_finite_add(b, value, addend); + return NULL; + } + if (nir_def_instr(coef)->type != nir_instr_type_alu) + return NULL; + nir_alu_instr *alu = nir_instr_as_alu(nir_def_instr(coef)); + if (alu->op != nir_op_bcsel) + return NULL; + for (unsigned i = 1; i < 3; i++) + if (!nir_src_is_const(alu->src[i].src) || alu->src[i].src.ssa->num_components != 1 || + alu->src[i].swizzle[0] != 0) + return NULL; + uint32_t yes = nir_src_as_uint(alu->src[1].src), no = nir_src_as_uint(alu->src[2].src); + bool yes_one = yes == 0x3f800000, no_one = no == 0x3f800000; + if ((!yes_one && (yes & 0x7fffffff)) || (!no_one && (no & 0x7fffffff))) + return NULL; + nir_def *sum = exact_finite_add(b, value, addend); + return nir_bcsel(b, nir_ssa_for_alu_src(b, alu, 0), yes_one ? sum : addend, + no_one ? sum : addend); +} +struct nir_def *air_nir_op(struct air_nir *s, enum air_op op, unsigned flags, nir_def *a, + nir_def *b, nir_def *c) { + /* Conservatively preserve every source rounding step. Special-value + * permissions are taken from this instruction, never module defaults. */ + s->b.fp_math_ctrl = nir_fp_exact; + if (!(flags & AIR_NNAN)) + s->b.fp_math_ctrl |= nir_fp_preserve_nan; + if (!(flags & AIR_NINF)) + s->b.fp_math_ctrl |= nir_fp_preserve_inf; + if (!(flags & AIR_NSZ)) + s->b.fp_math_ctrl |= nir_fp_preserve_signed_zero; + switch (op) { + case AIR_EQ: + return nir_ieq(&s->b, a, b); + case AIR_UGT: + return nir_ult(&s->b, b, a); + case AIR_OR: + return nir_ior(&s->b, a, b); + case AIR_AND: + return nir_iand(&s->b, a, b); + case AIR_SELECT: + return nir_bcsel(&s->b, a, b, c); + case AIR_FADD: + return nir_fadd(&s->b, a, b); + case AIR_FSUB: + return nir_fsub(&s->b, a, b); + case AIR_FMUL: + return nir_fmul(&s->b, a, b); + case AIR_FDIV: + if (!(flags & AIR_ARCP)) + return NULL; + s->allow_reciprocal = true; + return nir_fdiv(&s->b, a, b); + case AIR_FMA_FINITE_NSZ: { + if ((flags & (AIR_NNAN | AIR_NINF | AIR_NSZ)) != (AIR_NNAN | AIR_NINF | AIR_NSZ)) + return air_fused_f32(&s->b, a, b, c); + nir_def *special = finite_binary_coefficient(&s->b, a, b, c); + if (!special) + special = finite_binary_coefficient(&s->b, b, a, c); + if (special) + return special; + return air_fused_f32(&s->b, a, b, c); + } + case AIR_FSAT: + if ((flags & (AIR_NNAN | AIR_NINF | AIR_NSZ)) != (AIR_NNAN | AIR_NINF | AIR_NSZ)) return NULL; + return nir_fsat(&s->b, a); + case AIR_FMA: + return air_fused_f32(&s->b, a, b, c); + } + abort(); +} +int air_nir_store(struct air_nir *s, unsigned slot, unsigned count, nir_def **values) { + if (s->b.shader->info.stage != MESA_SHADER_VERTEX || slot >= 16 || count < 1 || count > 4) + return -1; + nir_def *vector = nir_vec(&s->b, values, count); + nir_store_output( + &s->b, vector, nir_imm_int(&s->b, 0), .base = slot, .write_mask = (1u << count) - 1, + .src_type = nir_type_float32, + .io_semantics = {.location = slot ? VARYING_SLOT_VAR0 + slot - 1 : VARYING_SLOT_POS}); + s->outputs = MAX2(s->outputs, slot + 1); + s->b.shader->num_outputs = s->outputs; + return 0; +} +int air_nir_color(struct air_nir *s, unsigned slot, unsigned target, unsigned count, nir_def **values) { + if (s->b.shader->info.stage != MESA_SHADER_FRAGMENT || slot >= 8 || target >= 8 || count < 1 || count > 4) + return -1; + nir_def *vector = nir_vec(&s->b, values, count); + nir_store_output(&s->b, vector, nir_imm_int(&s->b, 0), .base = slot, + .write_mask = (1u << count) - 1, .src_type = nir_type_float32, + .io_semantics = {.location = FRAG_RESULT_DATA0 + target}); + s->outputs = MAX2(s->outputs, slot + 1); + s->b.shader->num_outputs = s->outputs; + return 0; +} +/* Host test only: substitute inputs in a clone and use NIR's constant folder. + * This is neither the deployment path nor a CPU rendering implementation. */ +int air_nir_test_eval_resources(struct air_nir *s, uint32_t vertex, const uint32_t (*attributes)[4], + const struct air_nir_test_buffer buffers[16], float *out) { + nir_shader *clone = nir_shader_clone(NULL, s->b.shader); + nir_function_impl *impl = nir_shader_get_entrypoint(clone); + nir_builder b = nir_builder_create(impl); + int result = -1; + nir_foreach_block(block, impl) { + nir_foreach_instr_safe(instr, block) { + if (instr->type != nir_instr_type_intrinsic) + continue; + nir_intrinsic_instr *in = nir_instr_as_intrinsic(instr); + b.cursor = nir_before_instr(instr); + nir_def *value; + if (in->intrinsic == nir_intrinsic_load_vertex_id) { + value = nir_imm_int(&b, vertex); + } else if (in->intrinsic == nir_intrinsic_load_ubo) { + if (!nir_src_is_const(in->src[0]) || nir_src_as_uint(in->src[0]) >= 16 || + !nir_src_is_const(in->src[1])) + goto done; + unsigned binding = nir_src_as_uint(in->src[0]); + const uint8_t *data = buffers[binding].data; + unsigned size = buffers[binding].size; + unsigned offset = nir_src_as_uint(in->src[1]); + if (!data || size < 4 || offset > size - 4) + goto done; + uint32_t bits; + memcpy(&bits, data + offset, 4); + value = nir_imm_int(&b, bits); + } else if (in->intrinsic == nir_intrinsic_load_barycentric_pixel) { + if (!attributes || clone->info.stage != MESA_SHADER_FRAGMENT || + nir_intrinsic_interp_mode(in) != INTERP_MODE_SMOOTH) goto done; + // Test only: the supplied fragment values are already interpolated. + // The barycentric source is not evaluated as rasterization. + value = nir_imm_vec2(&b, 0, 0); + } else if (in->intrinsic == nir_intrinsic_load_input || + in->intrinsic == nir_intrinsic_load_interpolated_input) { + unsigned slot = nir_intrinsic_base(in), component = nir_intrinsic_component(in); + if (!attributes || slot >= 16 || component + in->num_components > 4 || + !nir_src_is_const(in->src[in->intrinsic == nir_intrinsic_load_interpolated_input ? 1 : 0]) || + nir_src_as_uint(in->src[in->intrinsic == nir_intrinsic_load_interpolated_input ? 1 : 0])) + goto done; + nir_const_value words[4] = {{0}}; + for (unsigned i = 0; i < in->num_components; i++) + words[i].u32 = attributes[slot][component + i]; + value = nir_build_imm(&b, in->num_components, 32, words); + } else if (in->intrinsic == nir_intrinsic_store_output) + continue; + else + goto done; + nir_def_rewrite_uses(&in->def, value); + nir_instr_remove(instr); + } + } + for (unsigned i = 0; i < 16; i++) { + bool progress = nir_opt_constant_folding(clone); + progress |= nir_opt_dce(clone); + if (!progress) + break; + if (i == 15) + goto done; + } + unsigned seen = 0; + nir_foreach_block(block, impl) { + nir_foreach_instr(instr, block) { + if (instr->type != nir_instr_type_intrinsic) + continue; + nir_intrinsic_instr *in = nir_instr_as_intrinsic(instr); + if (in->intrinsic != nir_intrinsic_store_output || !nir_src_is_const(in->src[0])) + goto done; + unsigned slot = nir_intrinsic_base(in); + if (slot >= s->outputs || (seen & (1u << slot))) + goto done; + seen |= 1u << slot; + nir_const_value *values = nir_src_as_const_value(in->src[0]); + for (unsigned i = 0; i < in->num_components; i++) + out[slot * 4 + i] = values[i].f32; + } + } + if (seen == (1u << s->outputs) - 1) + result = 0; +done: + ralloc_free(clone); + return result; +} +int air_nir_test_eval(struct air_nir *s, uint32_t vertex, const uint8_t *data, unsigned size, + float *out) { + struct air_nir_test_buffer buffers[16] = {{0}}; + buffers[0] = (struct air_nir_test_buffer){data, size}; + return air_nir_test_eval_resources(s, vertex, NULL, buffers, out); +} +int air_nir_lower_integer_ops(struct air_nir *s) { + for (unsigned i = 0; i < 8; i++) { + bool lowered = nir_lower_int64(s->b.shader); + nir_opt_copy_prop(s->b.shader); + nir_opt_constant_folding(s->b.shader); + nir_opt_dce(s->b.shader); + if (!lowered) + break; + if (i == 7) + return -1; + } + /* Do not run general floating-point algebraic optimizations here: this + * stage qualifies integer lowering independently of reciprocal lowering. + * 64-bit pack/unpack wires may remain until standard backend cleanup. */ + if (nir_lower_int64(s->b.shader)) + return -1; + nir_validate_shader(s->b.shader, "integer FMA after real NIR int64 lowering"); + return 0; +} +static int write_file(const char *prefix, const char *suffix, const void *data, size_t size) { + char path[4096]; + int length = snprintf(path, sizeof(path), "%s.%s", prefix, suffix); + if (length < 0 || (size_t)length >= sizeof(path)) + return -1; + FILE *f = fopen(path, "wb"); + if (!f) + return -1; + bool ok = fwrite(data, 1, size, f) == size; + return fclose(f) || !ok ? -1 : 0; +} +static int dump_nir(struct air_nir *s, const char *prefix, const char *suffix) { + char *text = NULL; + size_t length = 0; + FILE *f = open_memstream(&text, &length); + if (!f) + return -1; + nir_print_shader(s->b.shader, f); + int status = fclose(f); + if (!status) + status = write_file(prefix, suffix, text, length); + free(text); + return status ? -1 : 0; +} + +static bool lower_permitted_div(nir_builder *b, nir_alu_instr *alu, void *data) { + struct air_nir *s = data; + if (alu->op != nir_op_fdiv || !s->allow_reciprocal) + return false; + b->cursor = nir_before_instr(&alu->instr); + b->fp_math_ctrl = alu->fp_math_ctrl; + nir_def *x = nir_ssa_for_alu_src(b, alu, 0), *y = nir_ssa_for_alu_src(b, alu, 1); + nir_def *result = nir_fmul(b, x, nir_frcp(b, y)); + nir_def_rewrite_uses(&alu->def, result); + nir_instr_remove(&alu->instr); + s->reciprocal_lowerings++; + return true; +} +static int collect_constant_reads(struct air_nir *s) { + s->read_count = 0; + nir_foreach_block(block, nir_shader_get_entrypoint(s->b.shader)) { + nir_foreach_instr(instr, block) { + if (instr->type != nir_instr_type_intrinsic) + continue; + nir_intrinsic_instr *in = nir_instr_as_intrinsic(instr); + if (in->intrinsic != nir_intrinsic_load_ubo) + continue; + if (!nir_src_is_const(in->src[0]) || !nir_src_is_const(in->src[1]) || + nir_src_as_uint(in->src[0]) >= AIR_CONSTANT_MAX_BINDINGS || + s->read_count >= AIR_CONSTANT_MAX_READS || in->def.bit_size != 32) + return -1; + s->reads[s->read_count++] = (struct air_constant_read){ + nir_src_as_uint(in->src[0]), nir_src_as_uint(in->src[1]), in->num_components * 4}; + } + } + return 0; +} +static int write_backend_abi(struct air_nir *s, const char *prefix, + const struct ir3_shader_variant *v) { + if (s->compiler->dev_id != &s->dev_id) + return -1; + char *text = NULL; + size_t size = 0; + FILE *f = open_memstream(&text, &size); + if (!f) + return -1; + int status = air_export_backend_abi(f, v, s->reads, s->read_count); + if (fclose(f)) + status = -1; + if (!status) + status = write_file(prefix, "ir3.json", text, size); + free(text); + return status; +} +int air_nir_compile(struct air_nir *s, const char *prefix) { + if (air_nir_lower_integer_ops(s)) + return -2; + nir_shader_gather_info(s->b.shader, nir_shader_get_entrypoint(s->b.shader)); + nir_validate_shader(s->b.shader, "original AIR lowered before backend float legalization"); + if (dump_nir(s, prefix, "nir")) + return -1; + nir_shader_alu_pass(s->b.shader, lower_permitted_div, nir_metadata_control_flow, s); + nir_validate_shader(s->b.shader, "explicit source-permitted reciprocal lowering"); + if (dump_nir(s, prefix, "backend.nir")) + return -1; + /* Strong FMA must not silently become the backend's unfused MAD. */ + nir_foreach_block(block, nir_shader_get_entrypoint(s->b.shader)) { + nir_foreach_instr(instr, block) { + if (instr->type == nir_instr_type_alu && (nir_instr_as_alu(instr)->op == nir_op_ffma || + nir_instr_as_alu(instr)->op == nir_op_fdiv)) { + fprintf(stderr, "AIR backend refused: unlegalized fused FMA or division\n"); + return -2; + } + } + } + if (collect_constant_reads(s)) + return -2; + const struct ir3_shader_options options = {.api_wavesize = IR3_SINGLE_OR_DOUBLE, + .real_wavesize = IR3_SINGLE_OR_DOUBLE}; + struct ir3_shader *shader = ir3_shader_from_nir(s->compiler, s->b.shader, &options); + s->b.shader = NULL; /* consumed by IR3 */ + const struct ir3_shader_key key = {0}; + struct ir3_shader_variant *v = ir3_shader_get_variant(shader, &key, false, true, NULL, NULL); + int result = -1; + if (v && v->bin && v->info.sizedwords && v->info.size >= v->info.sizedwords * 4 && + v->disasm_info.disasm && !write_backend_abi(s, prefix, v) && + !write_file(prefix, "bin", v->bin, v->info.size) && + !write_file(prefix, "asm", v->disasm_info.disasm, strlen(v->disasm_info.disasm))) { + printf("{\"binary_bytes\":%u,\"code_dwords\":%u,\"asm_roundtrip\":true,\"gpu_executed\":" + "false,\"reciprocal_lowerings\":%u}\n", + v->info.size, v->info.sizedwords, s->reciprocal_lowerings); + result = 0; + } + ir3_shader_destroy(shader); + return result; +} +void air_nir_destroy(struct air_nir *s) { + if (!s) + return; + if (s->b.shader) + ralloc_free(s->b.shader); + glsl_type_singleton_decref(); + ir3_compiler_destroy(s->compiler); + free(s); +} diff --git a/kits/AirNir/licenses/SoftFloat-3e.txt b/kits/AirNir/licenses/SoftFloat-3e.txt new file mode 100644 index 0000000..b5690fa --- /dev/null +++ b/kits/AirNir/licenses/SoftFloat-3e.txt @@ -0,0 +1,37 @@ + +License for Berkeley SoftFloat Release 3e + +John R. Hauser +2018 January 20 + +The following applies to the whole of SoftFloat Release 3e as well as to +each source file individually. + +Copyright 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018 The Regents of the +University of California. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions, and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + diff --git a/kits/AirNir/tests/check_coefficients.py b/kits/AirNir/tests/check_coefficients.py new file mode 100644 index 0000000..e7915e5 --- /dev/null +++ b/kits/AirNir/tests/check_coefficients.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +"""Qualify exact 0/1-coefficient FMA simplification and every required flag.""" +import argparse +import ctypes +import hashlib +import json +import os +from pathlib import Path +import random +import struct +import subprocess + +p = argparse.ArgumentParser(description=__doc__) +p.add_argument('--compiler', type=Path, required=True) +p.add_argument('--llvm-bindir', type=Path, required=True) +p.add_argument('--out', type=Path, required=True) +p.add_argument('--chip', required=True) +a = p.parse_args() +a.out.mkdir(parents=True, exist_ok=True) +env = {k: v for k, v in os.environ.items() if not k.startswith(('IR3_', 'FD_MESA_', 'NIR_', 'MESA_SHADER_'))} +env['IR3_SHADER_DEBUG'] = 'asmroundtrip' + + +def run(cmd): + r = subprocess.run([str(x) for x in cmd], env=env, capture_output=True, text=True, timeout=30) + if r.returncode: + raise RuntimeError(f'Status {r.returncode}: {r.stderr[-3000:]}') + return r.stdout + + +def sha(path): + return hashlib.sha256(path.read_bytes()).hexdigest() + + +original = Path(__file__).with_name('fma_vertex.ll').read_text() +original = original.replace(' %atmp =', ' %iszero = icmp eq i16 %vertex, 0\n %coefficient = select i1 %iszero, float 0.0, float 1.0\n %atmp =') +original = original.replace('float %a0, i64 0', 'float %coefficient, i64 0').replace('float %a1, i64 1', 'float %coefficient, i64 1') +rng = random.Random(20261018) +cases = [] +vertices = [0, 1, 65535, 65536, 65537, 0xffffffff] +for i in range(4096): + vertex = vertices[i % len(vertices)] + values = [rng.uniform(-32, 32) for _ in range(6)] + if i < 64: + values[2:] = [0.0, -0.0, -0.0, 0.0] + cases.append((vertex, values)) +case_path = a.out / 'coefficient.cases' +case_path.write_bytes(b''.join(struct.pack(' 0x7f800000 + + +native_fma = ctypes.CDLL(None).fmaf +native_fma.argtypes = [ctypes.c_float] * 3 +native_fma.restype = ctypes.c_float +# A known cancellation example must distinguish a fused result from two +# separately rounded operations. Also test finite overflow cancellation. +known = [(0x3f800001, 0x3f7ffffe, 0xbf800000, 0xa8800000), + (0x7f7fffff, 0x40000000, 0xff7fffff, 0x7f7fffff)] +for x, y, z, expected in known: + got = bits(native_fma(f32(x), f32(y), f32(z))) + unfused = bits(f32(bits(f32(x) * f32(y))) + f32(z)) + if got != expected or unfused == expected: + raise RuntimeError('Host reference did not pass fused/unfused distinction') + +special = [0, 1, 2, 0x003fffff, 0x007fffff, 0x00800000, 0x00800001, + 0x3eaaaaab, 0x3f000000, 0x3f7fffff, 0x3f800000, 0x3f800001, + 0x40000000, 0x7f7fffff, 0x7f800000, 0x7fc00000] +special += [v | 0x80000000 for v in special] +triples = list(itertools.product(special, repeat=3)) +counts = {'special_cartesian': len(triples)} +rng = random.Random(20261017) +triples += [tuple(rng.getrandbits(32) for _ in range(3)) for _ in range(32768)] +counts['random_raw_bits'] = 32768 +for _ in range(32768): + x, y = [(rng.randrange(60, 191) << 23) | rng.getrandbits(23) | (rng.getrandbits(1) << 31) for _ in range(2)] + z = bits(f32(x) * f32(y)) ^ 0x80000000 + triples.append((x, y, z)) +counts['product_cancellation'] = 32768 +for i in range(32768): + exp = rng.randrange(25, 254) + y = (exp << 23) | rng.getrandbits(23) + z = ((exp - 24) << 23) + (i % 3 - 1) + z |= rng.getrandbits(1) << 31 + triples.append((0x3f800000, y, z)) +counts['halfway_and_neighbors'] = 32768 +offset = counts['special_cartesian'] +triples[offset:offset+2] = [(x, y, z) for x, y, z, _ in known] +counts['random_raw_bits'] -= 2 +counts['directed_fused_controls'] = 2 +if len(triples) != 131072: + raise RuntimeError('Fixed bounded test inventory differs') +case_file = a.out / 'fma.cases' +records = [] +for first, second in zip(triples[::2], triples[1::2]): + records.append(struct.pack('<7I', 0, first[0], second[0], first[1], second[1], first[2], second[2])) +case_file.write_bytes(b''.join(records)) +reference = [bits(native_fma(*(f32(x) for x in t))) for t in triples] +source = Path(__file__).with_name('fma_vertex.ll') +bc = a.out / 'fma.bc' +run([a.llvm_bindir / 'llvm-as', source, '-o', bc]) +reports = [] +for mode in ('--test-evaluate', '--test-evaluate-int32'): + name = 'wide-intermediates' if mode == '--test-evaluate' else 'lowered-int32' + prefix = a.out / name + for suffix in ('.bin', '.asm', '.nir', '.eval'): + prefix.with_suffix(suffix).unlink(missing_ok=True) + start = time.monotonic() + meta = json.loads(run([a.compiler, bc, a.chip, prefix, '--compile', mode, case_file])) + elapsed = time.monotonic() - start + raw = prefix.with_suffix('.eval').read_bytes() + if len(raw) != len(records) * 16: + raise RuntimeError('FMA test record output length differs') + for i, expected in enumerate(reference): + got = struct.unpack_from(' %a, %b\n %f = fadd {modifier}<2 x float> %product, %c')) + run([a.llvm_bindir / 'llvm-as', ll, '-o', bc]) + meta = json.loads(run([a.compiler, bc, a.chip, prefix, '--compile', '--test-evaluate-int32', case_path])) + raw = prefix.with_suffix('.eval').read_bytes() + for i, values in enumerate(cases): + got = struct.unpack_from('<4f', raw, i * 16) + expected = [f32(f32(values[j] * values[2+j]) + values[4+j]) for j in range(2)] + if got[:2] != tuple(expected): + raise RuntimeError('Source fmul/fadd changed separate rounding') + nir = prefix.with_suffix('.nir').read_text() + math = [l for l in nir.splitlines() if '= fadd ' in l or '= fmul ' in l] + if not math or any('// exact' not in l for l in math): + raise RuntimeError('Separate rounding was not protected in actual NIR') + if not flags and any('preserve:sz,inf,nan' not in l for l in math): + raise RuntimeError('Strict source lost special-value preservation') + if flags and any('preserve:' in l for l in math): + raise RuntimeError('Explicit special-value source flags were not propagated') + assembly = prefix.with_suffix('.asm').read_text() + if 'mad.f32' in assembly or meta['reciprocal_lowerings']: + raise RuntimeError('Unrequested contraction/reciprocal appeared') + reports.append(dict(name=tag, numeric_cases=len(cases), separate_rounding=True, + exact_nir_controls=True, binary_bytes=meta['binary_bytes'], asm_roundtrip=meta['asm_roundtrip'])) +# Allowed division must remain in pre-backend NIR and be legalized separately. +for flags, accepted in [('arcp', True), ('', False)]: + tag = 'permitted-div' if accepted else 'strict-div' + ll, bc, prefix = a.out / (tag + '.ll'), a.out / (tag + '.bc'), a.out / tag + ll.write_text(source.replace(call, ' %f = fdiv ' + (flags + ' ' if flags else '') + '<2 x float> %a, %b')) + run([a.llvm_bindir / 'llvm-as', ll, '-o', bc]) + output = run([a.compiler, bc, a.chip, prefix, '--compile'], code=0 if accepted else 4) + if accepted: + meta = json.loads(output) + pre = prefix.with_suffix('.nir').read_text() + post = prefix.with_suffix('.backend.nir').read_text() + if '= fdiv ' not in pre or '= fdiv ' in post or '= frcp ' not in post or meta['reciprocal_lowerings'] != 2: + raise RuntimeError('Division legalization did not match source arcp permission') +reports.append(dict(permitted_division_legalized=True, strict_division_refused=True)) +(a.out / 'report.json').write_text(json.dumps(dict(reports=reports, generated_isa_executed=False), indent=2) + '\n') +print('PASS: per-instruction special-value controls, exact rounding and source-gated reciprocal lowering') diff --git a/kits/AirNir/tests/constant_cpp_client.cpp b/kits/AirNir/tests/constant_cpp_client.cpp new file mode 100644 index 0000000..eaa336b --- /dev/null +++ b/kits/AirNir/tests/constant_cpp_client.cpp @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: MIT +// Cross-build witness for the native C API's IOKit/C++ call boundary. +#include "../ConstantUploads.h" +extern "C" air_constant_result air_constant_cpp_client(const air_constant_layout *layout, + const air_constant_binding *bindings, + uint8_t *destination, size_t bytes) { + return air_pack_constants(layout, bindings, destination, bytes); +} diff --git a/kits/AirNir/tests/fma_vertex.ll b/kits/AirNir/tests/fma_vertex.ll new file mode 100644 index 0000000..70aaa01 --- /dev/null +++ b/kits/AirNir/tests/fma_vertex.ll @@ -0,0 +1,40 @@ +; SPDX-License-Identifier: MIT +; Authored numerical test input, not an extracted target component. +source_filename = "air_fma_numerical_test" +target datalayout = "e-p:64:64-i16:16-i64:64-f32:32-n8:16:32" +target triple = "air64-apple-ios17.7.0" +define <4 x float> @fma_numerical_test(ptr addrspace(2) readonly align 4 dereferenceable(24) %buffer, i16 %vertex) { + %a0 = load float, ptr addrspace(2) %buffer, align 4 + %pa1 = getelementptr float, ptr addrspace(2) %buffer, i64 1 + %a1 = load float, ptr addrspace(2) %pa1, align 4 + %pb0 = getelementptr float, ptr addrspace(2) %buffer, i64 2 + %b0 = load float, ptr addrspace(2) %pb0, align 4 + %pb1 = getelementptr float, ptr addrspace(2) %buffer, i64 3 + %b1 = load float, ptr addrspace(2) %pb1, align 4 + %pc0 = getelementptr float, ptr addrspace(2) %buffer, i64 4 + %c0 = load float, ptr addrspace(2) %pc0, align 4 + %pc1 = getelementptr float, ptr addrspace(2) %buffer, i64 5 + %c1 = load float, ptr addrspace(2) %pc1, align 4 + %atmp = insertelement <2 x float> poison, float %a0, i64 0 + %a = insertelement <2 x float> %atmp, float %a1, i64 1 + %btmp = insertelement <2 x float> poison, float %b0, i64 0 + %b = insertelement <2 x float> %btmp, float %b1, i64 1 + %ctmp = insertelement <2 x float> poison, float %c0, i64 0 + %c = insertelement <2 x float> %ctmp, float %c1, i64 1 + %f = call <2 x float> @air.fma.v2f32(<2 x float> %a, <2 x float> %b, <2 x float> %c) + %result = shufflevector <2 x float> %f, <2 x float> , <4 x i32> + ret <4 x float> %result +} +declare <2 x float> @air.fma.v2f32(<2 x float>, <2 x float>, <2 x float>) +!air.version = !{!0} +!air.compile_options = !{!1, !2} +!air.vertex = !{!3} +!0 = !{i32 2, i32 6, i32 0} +!1 = !{!"air.compile.fast_math_enable"} +!2 = !{!"air.compile.denorms_disable"} +!3 = !{ptr @fma_numerical_test, !4, !6} +!4 = !{!5} +!5 = !{!"air.position", !"air.arg_type_name", !"float4"} +!6 = !{!7, !8} +!7 = !{i32 0, !"air.buffer", !"air.buffer_size", i32 24, !"air.location_index", i32 0, i32 1, !"air.read", !"air.address_space", i32 2} +!8 = !{i32 1, !"air.vertex_id"} diff --git a/kits/AirNir/tests/test_constant_uploads.c b/kits/AirNir/tests/test_constant_uploads.c new file mode 100644 index 0000000..297c681 --- /dev/null +++ b/kits/AirNir/tests/test_constant_uploads.c @@ -0,0 +1,165 @@ +/* SPDX-License-Identifier: MIT */ +#include "../ConstantUploads.h" +#include +#include +#include +#include +#include +#ifndef MAP_ANONYMOUS +#define MAP_ANONYMOUS MAP_ANON +#endif +static unsigned cases; +static void refuse(struct air_constant_layout *l, struct air_constant_binding *bindings, + uint8_t *dst, size_t bytes, unsigned status) { + uint8_t before[256]; + memcpy(before, dst, 256); + struct air_constant_result r = air_pack_constants(l, bindings, dst, bytes); + assert(r.status == status && r.written_bytes == 0 && r.padded_bytes == 0); + assert(!memcmp(before, dst, 256)); + cases++; +} +int main(void) { + size_t page = (size_t)sysconf(_SC_PAGESIZE); + uint8_t *guard = + mmap(NULL, page * 4, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + assert(guard != MAP_FAILED); + assert(!mprotect(guard + page, page, PROT_NONE)); + assert(!mprotect(guard + page * 3, page, PROT_NONE)); + uint8_t *a = guard + page - 8, *b = guard + page * 3 - 80; + for (unsigned i = 0; i < 8; i++) + a[i] = (uint8_t)(0x10 + i); + for (unsigned i = 0; i < 80; i++) + b[i] = (uint8_t)(0x40 + i); + uint8_t *dst = mmap(NULL, page, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + assert(dst != MAP_FAILED); + memset(dst, 0x6b, 256); + struct air_constant_binding bindings[16] = {{0}}; + bindings[3] = (struct air_constant_binding){a, 8, 8}; + bindings[7] = (struct air_constant_binding){b, 80, 80}; + struct air_constant_range ranges[] = {{3, 0, 64, 64}, {7, 64, 128, 128}}; + struct air_constant_read reads[] = {{3, 0, 8}, {7, 64, 16}}; + uint32_t imms[] = {0x11223344, 0x89abcdef, 0, 0xffffffff}; + struct air_constant_layout original = {256, 64, 64, 128, ranges, 2, reads, 2, 208, 4, imms}, + l = original; + struct air_constant_result r = air_pack_constants(&l, bindings, dst, 256); + assert(!r.status && r.written_bytes == 144 && r.padded_bytes == 104); + for (unsigned i = 0; i < 256; i++) { + uint8_t expected = 0x6b; + if (i >= 64 && i < 128) + expected = i < 72 ? a[i - 64] : 0; + if (i >= 128 && i < 192) + expected = i < 144 ? b[i - 64] : 0; + if (i >= 208 && i < 224) + expected = (uint8_t)(imms[(i - 208) / 4] >> (((i - 208) % 4) * 8)); + assert(dst[i] == expected); + } + cases++; + l = original; + l.range_count = 0; + l.read_count = 0; + r = air_pack_constants(&l, bindings, dst, 256); + assert(!r.status && r.written_bytes == 16); + cases++; + l = original; + refuse(&l, bindings, dst, 255, AIR_CONSTANT_CAPACITY); + l.constant_bytes = AIR_CONSTANT_MAX_BYTES + 16; + refuse(&l, bindings, dst, 256, AIR_CONSTANT_ARGUMENT); + l = original; + l.upload_alignment = 0; + refuse(&l, bindings, dst, 256, AIR_CONSTANT_LAYOUT); + l = original; + l.range_count = 33; + refuse(&l, bindings, dst, 256, AIR_CONSTANT_ARGUMENT); + l = original; + l.read_count = 4097; + refuse(&l, bindings, dst, 256, AIR_CONSTANT_ARGUMENT); + l = original; + l.ranges = NULL; + refuse(&l, bindings, dst, 256, AIR_CONSTANT_ARGUMENT); + l = original; + l.reads = NULL; + refuse(&l, bindings, dst, 256, AIR_CONSTANT_ARGUMENT); + l = original; + l.immediates = NULL; + refuse(&l, bindings, dst, 256, AIR_CONSTANT_ARGUMENT); + l = original; + l.immediate_begin = 128; + refuse(&l, bindings, dst, 256, AIR_CONSTANT_LAYOUT); + l = original; + l.immediate_begin = 209; + refuse(&l, bindings, dst, 256, AIR_CONSTANT_LAYOUT); + l = original; + l.immediate_words = 3; + refuse(&l, bindings, dst, 256, AIR_CONSTANT_LAYOUT); + l = original; + l.immediate_words = 4097; + refuse(&l, bindings, dst, 256, AIR_CONSTANT_ARGUMENT); + l = original; + l.immediate_begin = 256; + refuse(&l, bindings, dst, 256, AIR_CONSTANT_LAYOUT); + l = original; + l.ubo_begin = 256; + refuse(&l, bindings, dst, 256, AIR_CONSTANT_LAYOUT); + l = original; + l.ubo_bytes = 127; + refuse(&l, bindings, dst, 256, AIR_CONSTANT_LAYOUT); + l = original; + bindings[3].bytes = 7; + refuse(&l, bindings, dst, 256, AIR_CONSTANT_BINDING); + bindings[3].bytes = 8; + bindings[3].data = NULL; + refuse(&l, bindings, dst, 256, AIR_CONSTANT_BINDING); + bindings[3].data = a; + bindings[3].bytes = 4; + bindings[3].minimum_bytes = 4; + refuse(&l, bindings, dst, 256, AIR_CONSTANT_READ); + bindings[3] = (struct air_constant_binding){a, 8, 8}; + ranges[0].binding = 16; + refuse(&l, bindings, dst, 256, AIR_CONSTANT_LAYOUT); + ranges[0].binding = 3; + ranges[0].source_begin = 4; + refuse(&l, bindings, dst, 256, AIR_CONSTANT_LAYOUT); + ranges[0].source_begin = 0; + ranges[0].source_end = 0; + refuse(&l, bindings, dst, 256, AIR_CONSTANT_LAYOUT); + ranges[0].source_end = 64; + ranges[0].destination_begin = 0; + refuse(&l, bindings, dst, 256, AIR_CONSTANT_LAYOUT); + ranges[0].destination_begin = 64; + ranges[1].destination_begin = 64; + refuse(&l, bindings, dst, 256, AIR_CONSTANT_LAYOUT); + ranges[1].destination_begin = 128; + reads[0].binding = 16; + refuse(&l, bindings, dst, 256, AIR_CONSTANT_READ); + reads[0].binding = 3; + reads[0].bytes = 0; + refuse(&l, bindings, dst, 256, AIR_CONSTANT_READ); + reads[0].bytes = 8; + reads[0].offset = 1; + refuse(&l, bindings, dst, 256, AIR_CONSTANT_READ); + reads[0].offset = 0; + l.read_count = 1; + refuse(&l, bindings, dst, 256, AIR_CONSTANT_READ); + l = original; + bindings[3].data = dst + 64; + refuse(&l, bindings, dst, 256, AIR_CONSTANT_ALIAS); + bindings[3].data = a; + l.immediates = (uint32_t *)(void *)(dst + 32); + refuse(&l, bindings, dst, 256, AIR_CONSTANT_ALIAS); + l = original; + bindings[3].data = (const uint8_t *)(UINTPTR_MAX - 3); + refuse(&l, bindings, dst, 256, AIR_CONSTANT_ALIAS); + bindings[3].data = a; + uint8_t before[256]; + memcpy(before, dst, 256); + memcpy(dst, ranges, sizeof(ranges)); + l.ranges = (void *)dst; + refuse(&l, bindings, dst, 256, AIR_CONSTANT_ALIAS); + memcpy(dst, before, 256); + l = original; + assert(!munmap(guard, page * 4)); + assert(!munmap(dst, page)); + printf("PASS: %u guarded constant upload cases; short bindings end at inaccessible pages\n", + cases); + return 0; +} diff --git a/kits/AirNir/tests/test_library.c b/kits/AirNir/tests/test_library.c new file mode 100644 index 0000000..19084b8 --- /dev/null +++ b/kits/AirNir/tests/test_library.c @@ -0,0 +1,97 @@ +/* SPDX-License-Identifier: MIT */ +#include "Library.h" +#include +#include +#include +#include +#include +#include +#include +static unsigned positive,negative,hash_calls; +static void put32(uint8_t *p,uint32_t v) { for(unsigned i=0;i<4;i++) p[i]=v>>(i*8); } +static void put64(uint8_t *p,uint64_t v) { put32(p,v);put32(p+4,v>>32); } +static void big32(uint8_t *p,uint32_t v) { for(unsigned i=0;i<4;i++) p[3-i]=v>>(i*8); } +static int digest(const uint8_t *p,size_t n,uint8_t out[32],void *ctx) { + hash_calls++;if(ctx) return 1; + assert(n<=UINT32_MAX);return CC_SHA256(p,(CC_LONG)n,out)?0:1; +} +static size_t tag(uint8_t *p,size_t at,const char *name,const uint8_t *value,size_t n) { + memcpy(p+at,name,4);p[at+4]=n;p[at+5]=n>>8;memcpy(p+at+6,value,n);return at+6+n; +} +static size_t fixture(uint8_t *p) { + memset(p,0,512); + const uint8_t magic[]={0x4d,0x54,0x4c,0x42,1,0,2,0,7,0,0,0x82,0x11,0,7,0};memcpy(p,magic,16); + uint8_t type=0,hash[32],offsets[24]={0},module_size[8]={0}; + put32(p+256,0x0b17c0de);put32(p+264,20);put32(p+268,12);put32(p+272,0xffffffff);put32(p+276,0xdec04342); + digest(p+256,32,hash,0);put64(module_size,32); + size_t at=96;at=tag(p,at,"NAME",(const uint8_t *)"main",5);at=tag(p,at,"TYPE",&type,1);at=tag(p,at,"HASH",hash,32); + at=tag(p,at,"OFFT",offsets,24);at=tag(p,at,"MDSZ",module_size,8);memcpy(p+at,"ENDT",4);at+=4;assert(at==200); + put32(p+88,1);put32(p+92,108); + put64(p+16,288);put64(p+24,88);put64(p+32,108);put64(p+40,248);put64(p+48,4);put64(p+56,252);put64(p+64,4);put64(p+72,256);put64(p+80,32); + memcpy(p+200,"RLST\x10\0",6);put64(p+206,288);memcpy(p+222,"UUID\x10\0",6);memcpy(p+244,"ENDT",4); + return 288; +} +static void refuse(const uint8_t *p,size_t n,air_library_sha256 hash,void *ctx) { + struct air_library_view v,saved;memset(&v,0xa5,sizeof v);saved=v; + uint32_t status=air_library_open(p,n,hash,ctx,&v);if(!status)fprintf(stderr,"Unexpected acceptance at negative %u\n",negative);assert(status && status<=AIR_LIBRARY_ALIAS && !memcmp(&v,&saved,sizeof v));negative++; +} +static void accept(const uint8_t *p,size_t n) { + struct air_library_view v;struct air_library_function f; + assert(!air_library_open(p,n,digest,0,&v) && v.count==1); + assert(!air_library_get(&v,0,&f) && f.name_bytes==4 && !memcmp(f.name,"main",4) && f.stage==0 && f.bitcode_bytes==12 && f.wrapper_bytes==32); + assert(f.bitcode>=p && f.bitcode+f.bitcode_bytes<=p+n);positive++; + struct air_library_function saved;memset(&f,0xa5,sizeof f);saved=f; + assert(air_library_get(&v,1,&f)==AIR_LIBRARY_INDEX && !memcmp(&f,&saved,sizeof f));negative++; + assert(air_library_get(&v,UINT32_MAX,&f)==AIR_LIBRARY_INDEX && !memcmp(&f,&saved,sizeof f));negative++; + v.reserved=1;assert(air_library_get(&v,0,&f)==AIR_LIBRARY_FORMAT && !memcmp(&f,&saved,sizeof f));negative++; +} +int main(int argc,char **argv) { + uint8_t good[512],p[1024];size_t n=fixture(good);accept(good,n); + for(size_t i=0;i>24;} + struct air_library_view x,saved;memset(&x,0xa5,sizeof x);saved=x; + uint32_t status=air_library_open(p,n,digest,0,&x); + if(status) { assert(status<=AIR_LIBRARY_ALIAS && !memcmp(&x,&saved,sizeof x));negative++; } + else { struct air_library_function f;assert(!air_library_get(&x,0,&f));positive++; } + } + for(int arg=1;arg0 && length<=AIR_LIBRARY_MAX_BYTES && !fseek(file,0,SEEK_SET)); + size_t pages=((size_t)length+(size_t)page-1)/(size_t)page*(size_t)page; + uint8_t *map=mmap(0,pages+2*(size_t)page,PROT_NONE,MAP_PRIVATE|MAP_ANON,-1,0);assert(map!=MAP_FAILED && !mprotect(map+page,pages,PROT_READ|PROT_WRITE)); + uint8_t *bytes=map+page+pages-(size_t)length;assert(fread(bytes,1,(size_t)length,file)==(size_t)length && !fclose(file));assert(!mprotect(map+page,pages,PROT_READ)); + struct air_library_view original;assert(!air_library_open(bytes,(size_t)length,digest,0,&original)); + for(uint32_t i=0;iroot_physical = result->mair = 0; + result->table_pages = result->leaf_pages = result->start_level = 0; + result->upper_region = 0; + if (!config || !ranges || !range_count || !tables || !capacity || + capacity > SIZE_MAX / PAGE_BYTES || range_count > SIZE_MAX / sizeof(*ranges) || + config->input_bits < 32 || config->input_bits > 48 || + !(config->output_bits == 32 || config->output_bits == 36 || config->output_bits == 40 || + config->output_bits == 42 || config->output_bits == 44 || config->output_bits == 48) || + !config->leaf_budget || config->leaf_budget > (SIZE_MAX - 1) / 3 || + ((uintptr_t)tables & 7)) return ArmSmmu4KArgument; + if (range_count > config->leaf_budget) return ArmSmmu4KBudget; + uint64_t input_limit = UINT64_C(1) << config->input_bits; + uint64_t output_limit = UINT64_C(1) << config->output_bits; + uint64_t pool_bytes = (uint64_t)capacity * PAGE_BYTES; + uint64_t pool = config->table_physical; + if ((pool & 4095) || pool >= output_limit || pool_bytes > output_limit - pool) + return ArmSmmu4KBadRange; + size_t leaves = 0; + uint64_t previous_end = 0; + for (size_t i = 0; i < range_count; ++i) { + const ArmSmmu4KRange *r = &ranges[i]; + if (!r->bytes || ((r->iova | r->physical | r->bytes) & 4095) || + (!(r->permissions & ArmSmmu4KRead) || (r->permissions & ~15u)) || r->writeback > 1 || (r->shareability != 2 && r->shareability != 3) || + r->physical >= output_limit || r->bytes > output_limit - r->physical) + return ArmSmmu4KBadRange; + if (upper ? (r->iova < UINT64_MAX-input_limit+1 || r->bytes-1 > UINT64_MAX-r->iova) : + (r->iova >= input_limit || r->bytes > input_limit-r->iova)) + return ArmSmmu4KBadRange; + if (i && r->iova <= previous_end) return ArmSmmu4KOverlap; + previous_end = r->iova + (r->bytes-1); + if (r->physical < pool + pool_bytes && pool < r->physical + r->bytes) + return ArmSmmu4KOverlap; + for (size_t j = 0; j < i; ++j) { + if ((r->writeback != ranges[j].writeback || r->shareability != ranges[j].shareability || ((r->permissions ^ ranges[j].permissions) & ArmSmmu4KPrivileged)) && r->physical < ranges[j].physical + ranges[j].bytes && + ranges[j].physical < r->physical + r->bytes) return ArmSmmu4KOverlap; + } + uint64_t pages = r->bytes / PAGE_BYTES; + if (pages > config->leaf_budget - leaves) return ArmSmmu4KBudget; + leaves += (size_t)pages; + } + /* Each range has at least one leaf, so the validated leaf budget also + * bounds the range/alias loops. Sorted IOVAs make prefix counting exact. */ + unsigned levels = (config->input_bits - 12 + 8) / 9; + unsigned start = 4 - levels; + size_t required = 1; + uint64_t last[3] = {UINT64_MAX, UINT64_MAX, UINT64_MAX}; + for (size_t i = 0; i < range_count; ++i) { + for (uint64_t offset = 0; offset < ranges[i].bytes; offset += PAGE_BYTES) { + uint64_t va = ranges[i].iova + offset; + for (unsigned level = start; level < 3; ++level) { + uint64_t prefix = va >> (39 - 9 * level); + if (prefix != last[level]) { ++required; last[level] = prefix; } + } + } + } + if (required > capacity) return ArmSmmu4KCapacity; + for (size_t i = 0; i < required * 512; ++i) tables[i] = 0; + size_t next = 1; + for (size_t i = 0; i < range_count; ++i) { + const ArmSmmu4KRange *r = &ranges[i]; + uint64_t attributes = (1u << 11) | (1u << 10) | 3; + if (!(r->permissions & ArmSmmu4KExecute)) attributes |= UINT64_C(3) << 53; + if (!(r->permissions & ArmSmmu4KPrivileged)) attributes |= 1u << 6; + if (!(r->permissions & ArmSmmu4KWrite)) attributes |= 1u << 7; + attributes |= (uint64_t)r->shareability << 8; + if (r->writeback) attributes |= 1u << 2; + for (uint64_t offset = 0; offset < r->bytes; offset += PAGE_BYTES) { + uint64_t va = r->iova + offset; + size_t table = 0; + for (unsigned level = start; level < 3; ++level) { + unsigned shift = 39-9*level; + unsigned width = level==start ? config->input_bits-shift : 9; + size_t slot = table * 512 + ((va >> shift) & ((1u<> 12) & 511)] = (r->physical + offset) | attributes; + } + } + result->root_physical = pool; + result->mair = 0xff44; + result->table_pages = required; + result->leaf_pages = leaves; + result->start_level = start; + result->upper_region = upper; + return ArmSmmu4KSuccess; +} + +ArmSmmu4KStatus ArmSmmu4KBuild(const ArmSmmu4KConfig *config, + const ArmSmmu4KRange *ranges, size_t range_count, uint64_t *tables, + size_t capacity, ArmSmmu4KResult *result) +{ + return build_region(config,ranges,range_count,tables,capacity,result,0); +} + +ArmSmmu4KStatus ArmSmmu4KBuildUpper(const ArmSmmu4KConfig *config, + const ArmSmmu4KRange *ranges, size_t range_count, uint64_t *tables, + size_t capacity, ArmSmmu4KResult *result) +{ + return build_region(config,ranges,range_count,tables,capacity,result,1); +} diff --git a/kits/ArmSmmu4K/ArmSmmu4K.h b/kits/ArmSmmu4K/ArmSmmu4K.h new file mode 100644 index 0000000..1b09a29 --- /dev/null +++ b/kits/ArmSmmu4K/ArmSmmu4K.h @@ -0,0 +1,63 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef ARM_SMMU_4K_H +#define ARM_SMMU_4K_H +#include +#include +#ifdef __cplusplus +extern "C" { +#endif + +typedef enum { + ArmSmmu4KSuccess = 0, ArmSmmu4KArgument, ArmSmmu4KBadRange, + ArmSmmu4KOverlap, ArmSmmu4KCapacity, ArmSmmu4KBudget, + ArmSmmu4KUnsupported, ArmSmmu4KStateMismatch, + ArmSmmu4KIoError, ArmSmmu4KTimeout, ArmSmmu4KBusy, ArmSmmu4KFault +} ArmSmmu4KStatus; +typedef struct { + unsigned input_bits; /* 32..48, low-address TTBR0 profile. */ + unsigned output_bits; /* 32,36,40,42,44,48. */ + uint64_t table_physical; /* Caller-owned contiguous pool, aligned 4K. */ + size_t leaf_budget; /* Explicit upper bound on mapping work. */ +} ArmSmmu4KConfig; +enum {ArmSmmu4KRead=1,ArmSmmu4KWrite=2,ArmSmmu4KExecute=4,ArmSmmu4KPrivileged=8}; +typedef struct { + uint64_t iova, physical, bytes; + unsigned permissions; /* Read required; optional Write/Execute/Privileged. */ + unsigned writeback; /* 0=normal NC, 1=normal WB/WA. */ + unsigned shareability; /* 2=outer, 3=inner; explicit, no default. */ +} ArmSmmu4KRange; +typedef struct { + uint64_t root_physical; + uint64_t mair; /* NC at index 0, WB/WA at index 1. */ + size_t table_pages, leaf_pages; + unsigned start_level; + unsigned upper_region; /* 0=Build TTBR0, 1=BuildUpper TTBR1. */ +} ArmSmmu4KResult; + +/* Build inactive ARM LPAE stage-1 tables with ONLY 4K level-3 leaves. + * Sorted disjoint low IOVA ranges; no sign extension or input conversion. + * No resource may map the table pool. Conflicting cache/shareability/privilege + * aliases refuse. Default 1/3 retains unprivileged PXN/UXN behavior. Execute + * and Privileged explicitly describe DEVICE-stage1 accesses (e.g. firmware), + * never CPU permissions or authorization to execute unauthenticated images. + * All validation/capacity checks precede table writes. Failure zeros result + * and leaves tables untouched. Success writes only result.table_pages pages. + * Caller guarantees real accessible, disjoint storage, immutable inputs, + * native-endian uint64_t table storage aligned to at least 8 bytes, actual + * physical ownership and matching CPU attributes/cache maintenance. + * NO MMIO, allocation, barriers, TLB operations or activation. This does NOT + * prove hardware support, coherent walks or a live isolated GPU context. */ +ArmSmmu4KStatus ArmSmmu4KBuild(const ArmSmmu4KConfig *config, + const ArmSmmu4KRange *ranges, size_t range_count, uint64_t *tables, + size_t table_capacity_pages, ArmSmmu4KResult *result); +/* Same native 4K format for TTBR1's upper, sign-extended address region. + * IOVAs must already be in [2^64-2^input_bits,2^64); truncated low addresses + * refuse. Uses the architectural partial-root index width. No lower-region + * remapping, page-size conversion, MMIO or translation activation occurs. */ +ArmSmmu4KStatus ArmSmmu4KBuildUpper(const ArmSmmu4KConfig *config, + const ArmSmmu4KRange *ranges, size_t range_count, uint64_t *tables, + size_t table_capacity_pages, ArmSmmu4KResult *result); +#ifdef __cplusplus +} +#endif +#endif diff --git a/kits/ArmSmmu4K/ArmSmmu4KAudit.c b/kits/ArmSmmu4K/ArmSmmu4KAudit.c new file mode 100644 index 0000000..9e4e9a5 --- /dev/null +++ b/kits/ArmSmmu4K/ArmSmmu4KAudit.c @@ -0,0 +1,172 @@ +/* SPDX-License-Identifier: MIT */ +#include "ArmSmmu4KAudit.h" +#define PAGE UINT64_C(4096) +#define ADDRESS UINT64_C(0x0000fffffffff000) +typedef struct { + const ArmSmmu4KTableIo *io; + const ArmSmmu4KConfig *c; + const ArmSmmu4KResult *built; + const ArmSmmu4KAuditRegion *a; + ArmSmmu4KTableAuditResult *r; + size_t range; + uint64_t offset; +} Walk; +static int intersects(uint64_t a,uint64_t an,uint64_t b,uint64_t bn) +{return awriteback!=b->writeback || a->shareability!=b->shareability || ((a->permissions^b->permissions)&ArmSmmu4KPrivileged);} +static ArmSmmu4KStatus validate(const ArmSmmu4KConfig *c,const ArmSmmu4KResult *b, + const ArmSmmu4KAuditRegion *a,uint64_t root) +{ + if(!c || !b || !a || !a->ranges || !a->range_count || !a->visited || + !a->capacity_pages || a->capacity_pages>SIZE_MAX/4096 || + a->visited_bytescapacity_pages || !a->read_budget || + c->input_bits<32 || c->input_bits>48 || + !(c->output_bits==32 || c->output_bits==36 || c->output_bits==40 || + c->output_bits==42 || c->output_bits==44 || c->output_bits==48) || + !c->leaf_budget || c->leaf_budget>(SIZE_MAX-1)/3 || + a->range_count>SIZE_MAX/sizeof(*a->ranges))return ArmSmmu4KArgument; + if(a->range_count>c->leaf_budget)return ArmSmmu4KBudget; + uint64_t limit=UINT64_C(1)<output_bits,span=UINT64_C(1)<input_bits; + uint64_t pool_bytes=(uint64_t)a->capacity_pages*PAGE; + if((c->table_physical&4095) || c->table_physical>=limit || pool_bytes>limit-c->table_physical) + return ArmSmmu4KBadRange; + if(root!=c->table_physical || b->root_physical!=root || b->mair!=0xff44 || + !b->table_pages || b->table_pages>a->capacity_pages || !b->leaf_pages || + b->start_level!=4-(c->input_bits-12+8)/9 || b->upper_region>1) + return ArmSmmu4KStateMismatch; + if(b->table_pages>a->read_budget/512)return ArmSmmu4KBudget; + size_t leaves=0;uint64_t end=0; + for(size_t i=0;irange_count;++i) { + const ArmSmmu4KRange *r=&a->ranges[i]; + if(!r->bytes || ((r->iova|r->physical|r->bytes)&4095) || + !(r->permissions&ArmSmmu4KRead) || (r->permissions&~15u) || + r->writeback>1 || (r->shareability!=2 && r->shareability!=3) || + r->physical>=limit || r->bytes>limit-r->physical || + (b->upper_region?(r->iovabytes-1>UINT64_MAX-r->iova): + (r->iova>=span || r->bytes>span-r->iova)))return ArmSmmu4KBadRange; + if(i && r->iova<=end)return ArmSmmu4KOverlap; + end=r->iova+r->bytes-1; + if(intersects(r->physical,r->bytes,c->table_physical,pool_bytes))return ArmSmmu4KOverlap; + for(size_t j=0;jranges[j]) && + intersects(r->physical,r->bytes,a->ranges[j].physical,a->ranges[j].bytes))return ArmSmmu4KOverlap; + uint64_t pages=r->bytes/PAGE; + if(pages>c->leaf_budget-leaves)return ArmSmmu4KBudget; + leaves+=(size_t)pages; + } + return leaves==b->leaf_pages?ArmSmmu4KSuccess:ArmSmmu4KStateMismatch; +} +static ArmSmmu4KStatus walk(Walk *w,uint64_t table,unsigned level,uint64_t base) +{ + if((table&4095) || tablec->table_physical || + (table-w->c->table_physical)/PAGE>=w->a->capacity_pages)return ArmSmmu4KBadRange; + size_t page=(size_t)((table-w->c->table_physical)/PAGE); + if(w->a->visited[page])return ArmSmmu4KOverlap; + if(w->r->table_pages>=w->built->table_pages)return ArmSmmu4KStateMismatch; + w->a->visited[page]=1;++w->r->table_pages; + size_t initial_leaves=w->r->leaf_pages; + unsigned shift=39-9*level; + unsigned width=level==w->built->start_level?w->c->input_bits-shift:9; + for(unsigned i=0;i<512;++i) { + w->r->descriptor_physical=table+8*i;w->r->descriptor=0; + w->r->level=level;w->r->iova=base|((uint64_t)i<r->expected_descriptor=0; + if(w->r->descriptor_reads==w->a->read_budget)return ArmSmmu4KBudget; + ++w->r->descriptor_reads; + uint64_t d; + if(w->io->read_descriptor(w->io->cookie,table+8*i,&d))return ArmSmmu4KIoError; + w->r->descriptor=d; + if(!d)continue; + if(i>=(1u<range==w->a->range_count)return ArmSmmu4KStateMismatch; + const ArmSmmu4KRange *r=&w->a->ranges[w->range]; + uint64_t expected=(r->physical+w->offset)|3|(1u<<11)|(1u<<10)|((uint64_t)r->shareability<<8); + if(!(r->permissions&ArmSmmu4KExecute))expected|=UINT64_C(3)<<53; + if(!(r->permissions&ArmSmmu4KPrivileged))expected|=1u<<6; + if(!(r->permissions&ArmSmmu4KWrite))expected|=1u<<7; + if(r->writeback)expected|=1u<<2; + w->r->expected_descriptor=expected; + if(w->r->iova!=r->iova+w->offset || d!=expected)return ArmSmmu4KStateMismatch; + ++w->r->leaf_pages;w->offset+=PAGE; + if(w->offset==r->bytes){++w->range;w->offset=0;} + } + } + /* Empty intermediate tables are not part of the declared resource graph. */ + return w->r->leaf_pages>initial_leaves?ArmSmmu4KSuccess:ArmSmmu4KStateMismatch; +} +ArmSmmu4KStatus ArmSmmu4KAuditTables(const ArmSmmu4KTableIo *io, + const ArmSmmu4KConfig *c,const ArmSmmu4KResult *b,const ArmSmmu4KAuditRegion *a, + uint64_t root,ArmSmmu4KTableAuditResult *r) +{ + if(!r)return ArmSmmu4KArgument;*r=(ArmSmmu4KTableAuditResult){0}; + if(!io || !io->read_descriptor)return ArmSmmu4KArgument; + ArmSmmu4KStatus status=validate(c,b,a,root);if(status)return status; + for(size_t i=0;icapacity_pages;++i)a->visited[i]=0; + Walk w={io,c,b,a,r,0,0}; + uint64_t base=b->upper_region?UINT64_MAX-(UINT64_C(1)<input_bits)+1:0; + status=walk(&w,root,b->start_level,base);if(status)return status; + if(r->table_pages!=b->table_pages || r->leaf_pages!=b->leaf_pages || w.range!=a->range_count) + return ArmSmmu4KStateMismatch; + r->complete=1;return ArmSmmu4KSuccess; +} +static ArmSmmu4KStatus pair_validate(const ArmSmmu4KContextConfig *c,const ArmSmmu4KAuditConfig *a) +{ + if(!c || !a || !a->io.read_descriptor || (!c->lower && !c->upper) || + !!c->lower!=!!c->lower_tables || !!c->upper!=!!c->upper_tables || + !!c->lower!=!!a->lower || !!c->upper!=!!a->upper)return ArmSmmu4KArgument; + const ArmSmmu4KConfig *cs[2]={c->lower,c->upper}; + const ArmSmmu4KResult *bs[2]={c->lower_tables,c->upper_tables}; + const ArmSmmu4KAuditRegion *as[2]={a->lower,a->upper}; + for(unsigned i=0;i<2;++i)if(cs[i]) { + ArmSmmu4KStatus status=validate(cs[i],bs[i],as[i],bs[i]->root_physical);if(status)return status; + if(bs[i]->upper_region!=i)return ArmSmmu4KStateMismatch; + } + if(c->lower && c->upper) { + if(c->lower->output_bits!=c->upper->output_bits)return ArmSmmu4KStateMismatch; + if(intersects(c->lower->table_physical,a->lower->capacity_pages*PAGE, + c->upper->table_physical,a->upper->capacity_pages*PAGE))return ArmSmmu4KOverlap; + for(unsigned side=0;side<2;++side)for(size_t i=0;irange_count;++i) { + const ArmSmmu4KRange *r=&as[side]->ranges[i]; + if(intersects(r->physical,r->bytes,cs[!side]->table_physical,as[!side]->capacity_pages*PAGE))return ArmSmmu4KOverlap; + } + for(size_t i=0;ilower->range_count;++i)for(size_t j=0;jupper->range_count;++j) { + const ArmSmmu4KRange *x=&a->lower->ranges[i],*y=&a->upper->ranges[j]; + if(incompatible(x,y) && intersects(x->physical,x->bytes,y->physical,y->bytes))return ArmSmmu4KOverlap; + } + } + return ArmSmmu4KSuccess; +} +static ArmSmmu4KStatus regions(const ArmSmmu4KContextConfig *c,const ArmSmmu4KAuditConfig *a, + uint64_t lower,uint64_t upper,ArmSmmu4KAuditResult *r) +{ + ArmSmmu4KStatus status; + if(c->lower){r->phase=2;status=ArmSmmu4KAuditTables(&a->io,c->lower,c->lower_tables,a->lower,lower,&r->lower);if(status)return status;} + if(c->upper){r->phase=3;status=ArmSmmu4KAuditTables(&a->io,c->upper,c->upper_tables,a->upper,upper,&r->upper);if(status)return status;} + return ArmSmmu4KSuccess; +} +ArmSmmu4KStatus ArmSmmu4KAuditDeclaredContext(const ArmSmmu4KContextConfig *c, + const ArmSmmu4KAuditConfig *a,ArmSmmu4KAuditResult *r) +{ + if(!r)return ArmSmmu4KArgument;*r=(ArmSmmu4KAuditResult){0}; + ArmSmmu4KStatus status=pair_validate(c,a);if(status)return status; + status=regions(c,a,c->lower?c->lower_tables->root_physical:0,c->upper?c->upper_tables->root_physical:0,r); + if(!status){r->phase=4;r->complete=1;}return status; +} +ArmSmmu4KStatus ArmSmmu4KObserveAuditedContext(const ArmSmmu4KIo *io,unsigned bank, + const ArmSmmu4KContextConfig *c,const ArmSmmu4KAuditConfig *a,ArmSmmu4KAuditResult *r) +{ + if(!r)return ArmSmmu4KArgument;*r=(ArmSmmu4KAuditResult){0}; + ArmSmmu4KStatus status=pair_validate(c,a);if(status)return status; + r->phase=1;status=ArmSmmu4KObserveOwnedContext(io,bank,c,&r->before);if(status)return status; + status=regions(c,a,r->before.readback.ttbr0&ADDRESS,r->before.readback.ttbr1&ADDRESS,r);if(status)return status; + r->phase=5;status=ArmSmmu4KObserveOwnedContext(io,bank,c,&r->after);if(status)return status; + /* Each complete image independently matched the same immutable expected + * context, including capabilities, ASID, roots, MAIR and granules. */ + if(r->before.id0!=r->after.id0 || r->before.id1!=r->after.id1 || r->before.id2!=r->after.id2) + return ArmSmmu4KStateMismatch; + r->phase=6;r->complete=1;return ArmSmmu4KSuccess; +} diff --git a/kits/ArmSmmu4K/ArmSmmu4KAudit.h b/kits/ArmSmmu4K/ArmSmmu4KAudit.h new file mode 100644 index 0000000..147ad65 --- /dev/null +++ b/kits/ArmSmmu4K/ArmSmmu4KAudit.h @@ -0,0 +1,64 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef ARM_SMMU_4K_AUDIT_H +#define ARM_SMMU_4K_AUDIT_H +#include "ArmSmmu4KProgram.h" +#ifdef __cplusplus +extern "C" { +#endif +typedef struct { + void *cookie; + /* Read the actual descriptor at an owned physical address. No generated, + * cached reference-table or translated/shadow-table substitute. Return 0 + * on success. Platform supplies a real coherent CPU mapping and ownership. */ + int (*read_descriptor)(void *,uint64_t physical,uint64_t *); +} ArmSmmu4KTableIo; +typedef struct { + const ArmSmmu4KRange *ranges; + size_t range_count,capacity_pages,read_budget; + /* Scratch: one byte per pool page, reset by each audit. Disjoint from all + * inputs, outputs, mapped resources, physical tables and other scratch. */ + unsigned char *visited; + size_t visited_bytes; +} ArmSmmu4KAuditRegion; +typedef struct { + ArmSmmu4KTableIo io; + const ArmSmmu4KAuditRegion *lower,*upper; /* exactly the enabled regions */ +} ArmSmmu4KAuditConfig; +typedef struct { + size_t descriptor_reads,table_pages,leaf_pages; + uint64_t descriptor_physical,descriptor,iova,expected_descriptor; + unsigned level,complete; +} ArmSmmu4KTableAuditResult; +typedef struct { + ArmSmmu4KTableAuditResult lower,upper; + ArmSmmu4KProgramResult before,after; + unsigned phase,complete; +} ArmSmmu4KAuditResult; +/* Walk the physical graph, including every unused slot, with a strict read + * budget and bounded depth (four levels). Reject blocks, cycles, aliases of + * table pages, extra/missing mappings, permissions/attributes differing from + * declared resources, foreign PAs and nonzero reserved descriptor bits. + * No allocation, descriptor writes, rebuilding, conversion or MMIO. Each + * expected resource must appear exactly once, in ascending native IOVA order. + * Caller holds immutable tables/ranges/results and exclusive leases throughout; + * all masters are quiesced. Scratch and physical callback are trusted kernel + * boundaries. Declared bounds do NOT establish physical ownership/provenance. */ +ArmSmmu4KStatus ArmSmmu4KAuditTables(const ArmSmmu4KTableIo *, + const ArmSmmu4KConfig *,const ArmSmmu4KResult *,const ArmSmmu4KAuditRegion *, + uint64_t root_physical,ArmSmmu4KTableAuditResult *); +/* Pre-activation audit of BOTH declared physical regions; also rejects pool + * overlap and resource access to either entire pool, including spare pages. + * This is not an observation of an active context. */ +ArmSmmu4KStatus ArmSmmu4KAuditDeclaredContext(const ArmSmmu4KContextConfig *, + const ArmSmmu4KAuditConfig *,ArmSmmu4KAuditResult *); +/* Read actual context before AND after the physical walks; use its observed + * TTBR addresses as traversal roots. Require exact enabled 4K state and no + * faults at both observations. No writes, activation or hardware timing proof. + * Exclusive ownership/quiescence and coherent immutable tables are mandatory; + * two observations cannot detect an unowned writer changing and restoring state. */ +ArmSmmu4KStatus ArmSmmu4KObserveAuditedContext(const ArmSmmu4KIo *,unsigned bank, + const ArmSmmu4KContextConfig *,const ArmSmmu4KAuditConfig *,ArmSmmu4KAuditResult *); +#ifdef __cplusplus +} +#endif +#endif diff --git a/kits/ArmSmmu4K/ArmSmmu4KContext.c b/kits/ArmSmmu4K/ArmSmmu4KContext.c new file mode 100644 index 0000000..2cc334e --- /dev/null +++ b/kits/ArmSmmu4K/ArmSmmu4KContext.c @@ -0,0 +1,84 @@ +/* SPDX-License-Identifier: MIT */ +#include "ArmSmmu4KContext.h" +static int width_code(unsigned bits) +{ + switch(bits) {case 32:return 0;case 36:return 1;case 40:return 2; + case 42:return 3;case 44:return 4;case 48:return 5;default:return -1;} +} +static unsigned width_bits(unsigned code) +{ + switch(code) {case 0:return 32;case 1:return 36;case 2:return 40; + case 3:return 42;case 4:return 44;case 5:return 48;default:return 0;} +} +static int region_valid(const ArmSmmu4KConfig *c,const ArmSmmu4KResult *r,unsigned upper) +{ + if(!c || !r)return !c && !r; + if(c->input_bits<32 || c->input_bits>48 || width_code(c->output_bits)<0 || + (c->table_physical&4095) || r->root_physical!=c->table_physical || + r->mair!=0xff44 || r->upper_region!=upper || !r->table_pages || + r->leaf_pages>c->leaf_budget || r->table_pages>UINT64_MAX/4096 || + r->start_level!=4-(c->input_bits-12+8)/9)return 0; + uint64_t limit=UINT64_C(1)<output_bits; + return r->root_physicaltable_pages*4096<=limit-r->root_physical; +} +ArmSmmu4KStatus ArmSmmu4KEncodeContext(const ArmSmmu4KContextConfig *c, + ArmSmmu4KContextImage *out) +{ + if(!out)return ArmSmmu4KArgument; + out->cbar=out->cba2r=out->tcr=out->tcr2=out->mair0=out->mair1=out->sctlr=0; + out->reserved_zero=0; + out->ttbr0=out->ttbr1=0; + if(!c || (!c->lower && !c->upper) || c->asid>65535 || c->coherent_walk>1 || + c->stall_on_fault>1 || c->continue_after_fault>1 || + !region_valid(c->lower,c->lower_tables,0) || !region_valid(c->upper,c->upper_tables,1))return ArmSmmu4KArgument; + unsigned va=width_bits((c->id2>>8)&15),pa=width_bits((c->id2>>4)&15); + if(!(c->id0&(UINT32_C(1)<<30)) || !(c->id2&(1u<<12)) || !va || !pa)return ArmSmmu4KUnsupported; + const ArmSmmu4KConfig *basis=c->lower?c->lower:c->upper; + if(basis->output_bits>pa || (c->lower && c->lower->input_bits>va) || + (c->upper && (c->upper->input_bits>va || c->upper->output_bits!=basis->output_bits)))return ArmSmmu4KUnsupported; + if(c->lower && c->upper) { + const ArmSmmu4KResult *a=c->lower_tables,*b=c->upper_tables; + if(a->root_physicalroot_physical+(uint64_t)b->table_pages*4096 && + b->root_physicalroot_physical+(uint64_t)a->table_pages*4096)return ArmSmmu4KOverlap; + } + ArmSmmu4KContextImage v; + v.cbar=0x1f300; /* Stage1 translation, stage2 bypass; weakest bypass attrs. */ + v.cba2r=1; /* AArch64. No stage2 VMID or guest translation. */ + v.tcr=UINT32_C(2)<<30; /* TG1=4K, TG0=4K. */ + unsigned walk=c->coherent_walk?0x3500:0x2000; + if(c->lower)v.tcr|=walk|(64-c->lower->input_bits); + else v.tcr|=1u<<7; /* EPD0 */ + if(c->upper)v.tcr|=(walk|(64-c->upper->input_bits))<<16; + else v.tcr|=1u<<23; /* EPD1 */ + v.tcr2=(7u<<15)|(1u<<4)|(unsigned)width_code(basis->output_bits); + v.ttbr0=(uint64_t)c->asid<<48;v.ttbr1=v.ttbr0; + if(c->lower)v.ttbr0|=c->lower_tables->root_physical; + if(c->upper)v.ttbr1|=c->upper_tables->root_physical; + v.mair0=0xff44;v.mair1=0; + v.sctlr=0x1067|(c->stall_on_fault<<7)|(c->continue_after_fault<<8); + v.reserved_zero=0; + *out=v; + return ArmSmmu4KSuccess; +} +ArmSmmu4KStatus ArmSmmu4KValidateContextReadback(const ArmSmmu4KContextConfig *c, + const ArmSmmu4KContextImage *observed) +{ + if(!observed)return ArmSmmu4KArgument; + ArmSmmu4KContextImage expected; + ArmSmmu4KStatus status=ArmSmmu4KEncodeContext(c,&expected); + if(status!=ArmSmmu4KSuccess)return status; + /* TCR2.TBI0/TBI1 (bits 5/6, top-byte ignore) are written 0 but may read 1: the sheng Adreno MMU-500 reads + * 0x38071 for 0x38011 (int-008). Linux neither sets nor reads them back. With a 48-bit upstream bus the top + * byte of a device address is zero, so ignoring it changes no translation. Only 0 -> 1 is accepted here. */ + const uint32_t tbi_set=observed->tcr2&~expected.tcr2&UINT32_C(0x60); + /* SCTLR.ASIDPNE (bit 12) is written 1 but may read 0: the same SMMU reads 0x167 for 0x1167 (int-009). Linux + * writes it and never reads it back. A shared ASID namespace is safe because every bank's ASID is the + * caller's exclusive lease. Only 1 -> 0 is accepted. */ + const uint32_t asidpne_clear=expected.sctlr&~observed->sctlr&UINT32_C(0x1000); + if(expected.cbar!=observed->cbar || expected.cba2r!=observed->cba2r || + expected.tcr!=observed->tcr || ((expected.tcr2^observed->tcr2)&~tbi_set) || + expected.mair0!=observed->mair0 || expected.mair1!=observed->mair1 || + ((expected.sctlr^observed->sctlr)&~asidpne_clear) || observed->reserved_zero || expected.ttbr0!=observed->ttbr0 || + expected.ttbr1!=observed->ttbr1)return ArmSmmu4KStateMismatch; + return ArmSmmu4KSuccess; +} diff --git a/kits/ArmSmmu4K/ArmSmmu4KContext.h b/kits/ArmSmmu4K/ArmSmmu4KContext.h new file mode 100644 index 0000000..3c2faa4 --- /dev/null +++ b/kits/ArmSmmu4K/ArmSmmu4KContext.h @@ -0,0 +1,43 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef ARM_SMMU_4K_CONTEXT_H +#define ARM_SMMU_4K_CONTEXT_H +#include "ArmSmmu4K.h" +#ifdef __cplusplus +extern "C" { +#endif +typedef struct { + /* A config/result pair from Build, BuildUpper or LivePrepare; NULL pair disables it. + * A live arena may have zero mapped leaves while retaining its table topology. + * Both enabled regions need distinct physical pools and equal OAS/MAIR. + * The trusted caller retains actual pool ownership and immutable results. */ + const ArmSmmu4KConfig *lower, *upper; + const ArmSmmu4KResult *lower_tables, *upper_tables; + uint32_t id0, id2; /* Raw SMMUv2 identification register values. */ + unsigned asid; /* 0..65535, ownership managed by the caller. */ + unsigned coherent_walk; /* Explicit: 1=inner WBWA, 0=outer NC. */ + unsigned stall_on_fault, continue_after_fault; /* Each 0 or 1. */ +} ArmSmmu4KContextConfig; +typedef struct { + uint32_t cbar, cba2r, tcr, tcr2, mair0, mair1, sctlr, reserved_zero; + uint64_t ttbr0, ttbr1; +} ArmSmmu4KContextImage; + +/* Encode AArch64 stage1, stage2-bypass native4K SMMUv2 context registers. + * Capability/width/result/refusal checks precede publishing an image. Failure + * zeros it. Both TG0/TG1 select 4K even for a disabled region. No MMIO, power, + * stream/context assignment, cache barriers, TLB operations or activation. + * ID inputs are not proof that the caller read actual hardware. Inputs, + * their pointed-to configs/results, and output must be disjoint and immutable + * throughout the call; this is a trusted kernel integration boundary. */ +ArmSmmu4KStatus ArmSmmu4KEncodeContext(const ArmSmmu4KContextConfig *, + ArmSmmu4KContextImage *); +/* Compare all supplied readback fields with the freshly encoded expected + * image. Caller must read real hardware and independently walk its actual + * TTBR tables, verify ownership/coherency and handle faults/TLB lifetime. + * Matching caller-supplied values alone never qualify an active GPU mapping. */ +ArmSmmu4KStatus ArmSmmu4KValidateContextReadback(const ArmSmmu4KContextConfig *, + const ArmSmmu4KContextImage *); +#ifdef __cplusplus +} +#endif +#endif diff --git a/kits/ArmSmmu4K/ArmSmmu4KLive.c b/kits/ArmSmmu4K/ArmSmmu4KLive.c new file mode 100644 index 0000000..4ca15a8 --- /dev/null +++ b/kits/ArmSmmu4K/ArmSmmu4KLive.c @@ -0,0 +1,346 @@ +/* SPDX-License-Identifier: MIT */ +#include "ArmSmmu4KLive.h" +#define PAGE UINT64_C(4096) +#define PA_MASK UINT64_C(0x0000fffffffff000) + +static int output_width(unsigned bits) +{return bits==32 || bits==36 || bits==40 || bits==42 || bits==44 || bits==48;} +static ArmSmmu4KStatus layout(const ArmSmmu4KLiveArena *a,ArmSmmu4KResult *out) +{ + if(!a || !a->mapping || ((uintptr_t)a->mapping&4095) || a->upper_region>1 || + a->config.input_bits<32 || a->config.input_bits>48 || + !output_width(a->config.output_bits) || !a->config.leaf_budget || + !a->capacity_pages || a->capacity_pages>SIZE_MAX/4096 || + a->capacity_pages*4096>UINTPTR_MAX-(uintptr_t)a->mapping)return ArmSmmu4KArgument; + uint64_t limit=UINT64_C(1)<config.input_bits; + uint64_t physical_limit=UINT64_C(1)<config.output_bits; + if(!a->bytes || ((a->iova|a->bytes|a->config.table_physical)&4095) || + (a->upper_region ? a->iovaiova>=limit) || + a->bytes>limit-(a->iova&(limit-1)) || + a->config.table_physical>=physical_limit || + a->capacity_pages*PAGE>physical_limit-a->config.table_physical)return ArmSmmu4KBadRange; + if(a->bytes/PAGE>a->config.leaf_budget)return ArmSmmu4KBudget; + unsigned start=4-(a->config.input_bits-12+8)/9; + uint64_t low=a->iova&(limit-1),high=low+a->bytes-1; + size_t pages=1; + for(unsigned level=start;level<3;++level) { + unsigned shift=39-9*level; + uint64_t extra=(high>>shift)-(low>>shift)+1; + if(extra>SIZE_MAX-pages)return ArmSmmu4KCapacity; + pages+=(size_t)extra; + } + if(pages>a->capacity_pages)return ArmSmmu4KCapacity; + *out=(ArmSmmu4KResult){a->config.table_physical,0xff44,pages,0,start,a->upper_region}; + return ArmSmmu4KSuccess; +} +static uint64_t descriptor(const ArmSmmu4KRange *r,uint64_t iova) +{ + uint64_t attrs=(1u<<11)|(1u<<10)|3u|((uint64_t)r->shareability<<8); + if(!(r->permissions&ArmSmmu4KExecute))attrs|=UINT64_C(3)<<53; + if(!(r->permissions&ArmSmmu4KPrivileged))attrs|=1u<<6; + if(!(r->permissions&ArmSmmu4KWrite))attrs|=1u<<7; + if(r->writeback)attrs|=1u<<2; + return (r->physical+(iova-r->iova))|attrs; +} +static uint64_t expected(const ArmSmmu4KRange *ranges,size_t count,uint64_t iova) +{ + size_t low=0,high=count; + while(lowiova)high=mid; + else if(iova-r->iova>=r->bytes)low=mid+1; + else return descriptor(r,iova); + } + return 0; +} +static int contains(const ArmSmmu4KRange *r,uint64_t iova) +{return r && iova>=r->iova && iova-r->iovabytes;} +static ArmSmmu4KStatus validate_range(const ArmSmmu4KLiveArena *a,const ArmSmmu4KRange *r) +{ + uint64_t physical_limit=UINT64_C(1)<config.output_bits; + uint64_t pool=a->config.table_physical,pool_bytes=a->capacity_pages*PAGE; + if(!r || !r->bytes || ((r->iova|r->physical|r->bytes)&4095) || + !(r->permissions&ArmSmmu4KRead) || (r->permissions&~15u) || + r->writeback>1 || (r->shareability!=2 && r->shareability!=3) || + r->physical>=physical_limit || r->bytes>physical_limit-r->physical || + r->iovaiova || r->iova-a->iova>=a->bytes || + r->bytes>a->bytes-(r->iova-a->iova))return ArmSmmu4KBadRange; + if(r->physicalphysical+r->bytes)return ArmSmmu4KOverlap; + return ArmSmmu4KSuccess; +} +static int conflicting_alias(const ArmSmmu4KRange *a,const ArmSmmu4KRange *b) +{ + return (a->writeback!=b->writeback || a->shareability!=b->shareability || + ((a->permissions^b->permissions)&ArmSmmu4KPrivileged)) && + a->physicalphysical+b->bytes && b->physicalphysical+a->bytes; +} +static ArmSmmu4KStatus validate_ranges(const ArmSmmu4KLiveArena *a, + const ArmSmmu4KRange *ranges,size_t count,size_t *leaves) +{ + *leaves=0; + if((count && !ranges) || count>a->config.leaf_budget || + count>SIZE_MAX/sizeof(*ranges))return ArmSmmu4KArgument; + for(size_t i=0;ia->config.leaf_budget-*leaves)return ArmSmmu4KBudget; + *leaves+=pages; + } + return ArmSmmu4KSuccess; +} +/* Traverse the actual fixed topology, never generate/copy a reference table. + * A fixed owned aperture reserves empty intermediate tables; every other slot + * must be zero. Resource expectations are metadata, not translated page tables. */ +static ArmSmmu4KStatus walk(const ArmSmmu4KLiveArena *a,const ArmSmmu4KResult *shape, + unsigned level,size_t page,uint64_t prefix,size_t *next,unsigned build, + const ArmSmmu4KRange *ranges,size_t count,const ArmSmmu4KRange *change, + unsigned map,ArmSmmu4KLiveResult *r) +{ + uint64_t limit=UINT64_C(1)<config.input_bits; + uint64_t low=a->iova&(limit-1),high=low+a->bytes-1; + unsigned shift=39-9*level; + unsigned width=level==shape->start_level?a->config.input_bits-shift:9; + for(unsigned slot=0;slot<512;++slot) { + uint64_t begin=prefix+((uint64_t)slot<=shape->table_pages)return ArmSmmu4KStateMismatch; + value=(a->config.table_physical+child*PAGE)|3u; + } else if(level==3 && reserved && !build) { + uint64_t va=a->upper_region ? begin|~(limit-1) : begin; + value=contains(change,va)?(map?descriptor(change,va):0):expected(ranges,count,va); + } + size_t index=page*512+slot; + if(build)a->mapping[index]=value; + else {++r->descriptor_reads;if(a->mapping[index]!=value)return ArmSmmu4KStateMismatch;} + if(level<3 && reserved) { + ArmSmmu4KStatus status=walk(a,shape,level+1,child,begin,next,build,ranges,count,change,map,r); + if(status!=ArmSmmu4KSuccess)return status; + } + } + return ArmSmmu4KSuccess; +} +static ArmSmmu4KStatus verify(const ArmSmmu4KLiveArena *a,const ArmSmmu4KResult *shape, + const ArmSmmu4KRange *ranges,size_t count,const ArmSmmu4KRange *change, + unsigned map,ArmSmmu4KLiveResult *r) +{ + size_t next=1; + ArmSmmu4KStatus status=walk(a,shape,shape->start_level,0,0,&next,0,ranges,count,change,map,r); + return status==ArmSmmu4KSuccess && next!=shape->table_pages?ArmSmmu4KStateMismatch:status; +} +ArmSmmu4KStatus ArmSmmu4KLivePrepare(const ArmSmmu4KLiveArena *a,ArmSmmu4KLiveState *state) +{ + if(!state || state->prepared || state->poisoned)return ArmSmmu4KArgument; + ArmSmmu4KResult shape; + ArmSmmu4KStatus status=layout(a,&shape); + if(status!=ArmSmmu4KSuccess)return status; + size_t next=1; + status=walk(a,&shape,shape.start_level,0,0,&next,1,0,0,0,0,0); + if(status!=ArmSmmu4KSuccess || next!=shape.table_pages) {state->poisoned=1;return ArmSmmu4KStateMismatch;} + state->tables=shape;state->generation=0;state->prepared=1; + return ArmSmmu4KSuccess; +} +static int same_config(const ArmSmmu4KConfig *a,const ArmSmmu4KConfig *b) +{ + return a && b && a->input_bits==b->input_bits && a->output_bits==b->output_bits && + a->table_physical==b->table_physical && a->leaf_budget==b->leaf_budget; +} +ArmSmmu4KStatus ArmSmmu4KLiveAudit(const ArmSmmu4KLiveArena *a,const ArmSmmu4KLiveState *state, + const ArmSmmu4KRange *ranges,size_t count,ArmSmmu4KLiveResult *r) +{ + if(!r)return ArmSmmu4KArgument; + *r=(ArmSmmu4KLiveResult){0};r->must_retain_resources=1; + if(!state || !state->prepared || state->poisoned)return ArmSmmu4KArgument; + ArmSmmu4KResult shape; + ArmSmmu4KStatus status=layout(a,&shape); + if(status!=ArmSmmu4KSuccess)return status; + if(state->tables.root_physical!=shape.root_physical || state->tables.mair!=shape.mair || + state->tables.table_pages!=shape.table_pages || state->tables.start_level!=shape.start_level || + state->tables.upper_region!=shape.upper_region)return ArmSmmu4KArgument; + size_t leaves=0; + status=validate_ranges(a,ranges,count,&leaves);if(status!=ArmSmmu4KSuccess)return status; + if(leaves!=state->tables.leaf_pages)return ArmSmmu4KArgument; + status=verify(a,&shape,ranges,count,0,0,r); + r->complete=status==ArmSmmu4KSuccess;return status; +} +static ArmSmmu4KStatus leaf_index(const ArmSmmu4KLiveArena *a, + const ArmSmmu4KResult *shape,uint64_t va,size_t *index,ArmSmmu4KLiveResult *r) +{ + size_t page=0; + for(unsigned level=shape->start_level;level<3;++level) { + unsigned shift=39-9*level,width=level==shape->start_level?a->config.input_bits-shift:9; + uint64_t entry=a->mapping[page*512+((va>>shift)&((1u<descriptor_reads; + uint64_t physical=entry&PA_MASK; + if(entry!=(physical|3u) || physicalconfig.table_physical || + physical-a->config.table_physical>=shape->table_pages*PAGE)return ArmSmmu4KStateMismatch; + page=(size_t)((physical-a->config.table_physical)/PAGE); + } + *index=page*512+((va>>12)&511); + return ArmSmmu4KSuccess; +} +static int overlaps(uint64_t a,uint64_t abytes,uint64_t b,uint64_t bbytes) +{return aphysical,r->bytes,a->config.table_physical,a->capacity_pages*PAGE);} +static ArmSmmu4KStatus region_metadata(const ArmSmmu4KLiveRegion *region, + const ArmSmmu4KConfig *config,const ArmSmmu4KResult *root,unsigned upper) +{ + if(!region)return (!config && !root)?ArmSmmu4KSuccess:ArmSmmu4KArgument; + if(!region->arena || !region->state || region->arena->upper_region!=upper || + root!=®ion->state->tables || !same_config(config,®ion->arena->config) || + !region->state->prepared || region->state->poisoned)return ArmSmmu4KArgument; + ArmSmmu4KResult shape; + ArmSmmu4KStatus status=layout(region->arena,&shape);if(status)return status; + const ArmSmmu4KResult *r=®ion->state->tables; + if(r->root_physical!=shape.root_physical || r->mair!=shape.mair || r->table_pages!=shape.table_pages || + r->start_level!=shape.start_level || r->upper_region!=shape.upper_region)return ArmSmmu4KArgument; + size_t leaves=0; + status=validate_ranges(region->arena,region->ranges,region->count,&leaves);if(status)return status; + return leaves==r->leaf_pages?ArmSmmu4KSuccess:ArmSmmu4KArgument; +} +static ArmSmmu4KStatus context_metadata(const ArmSmmu4KContextConfig *context, + const ArmSmmu4KLiveRegion *lower,const ArmSmmu4KLiveRegion *upper) +{ + if(!context || (!lower && !upper))return ArmSmmu4KArgument; + ArmSmmu4KStatus status=region_metadata(lower,context->lower,context->lower_tables,0);if(status)return status; + status=region_metadata(upper,context->upper,context->upper_tables,1);if(status)return status; + if(lower && upper) { + const ArmSmmu4KLiveArena *a=lower->arena,*b=upper->arena; + if(lower->state==upper->state || a->config.output_bits!=b->config.output_bits || + overlaps(a->config.table_physical,a->capacity_pages*PAGE,b->config.table_physical,b->capacity_pages*PAGE) || + overlaps((uintptr_t)a->mapping,a->capacity_pages*PAGE,(uintptr_t)b->mapping,b->capacity_pages*PAGE)) + return ArmSmmu4KOverlap; + for(size_t i=0;icount;++i) { + if(touches_pool(&lower->ranges[i],b))return ArmSmmu4KOverlap; + for(size_t j=0;jcount;++j) + if(conflicting_alias(&lower->ranges[i],&upper->ranges[j]))return ArmSmmu4KOverlap; + } + for(size_t i=0;icount;++i)if(touches_pool(&upper->ranges[i],a))return ArmSmmu4KOverlap; + } + return ArmSmmu4KSuccess; +} +ArmSmmu4KStatus ArmSmmu4KLiveAuditContext(const ArmSmmu4KContextConfig *context, + const ArmSmmu4KLiveRegion *lower,const ArmSmmu4KLiveRegion *upper,ArmSmmu4KLiveResult *r) +{ + if(!r)return ArmSmmu4KArgument; + *r=(ArmSmmu4KLiveResult){0};r->must_retain_resources=1; + ArmSmmu4KStatus status=context_metadata(context,lower,upper);if(status)return status; + const ArmSmmu4KLiveRegion *regions[]={lower,upper}; + for(unsigned i=0;i<2;++i)if(regions[i]) { + const ArmSmmu4KLiveRegion *v=regions[i]; + status=verify(v->arena,&v->state->tables,v->ranges,v->count,0,0,r);if(status)return status; + } + r->complete=1;return ArmSmmu4KSuccess; +} +static ArmSmmu4KStatus update(const ArmSmmu4KLiveArena *a,ArmSmmu4KLiveState *state, + const ArmSmmu4KIo *io,unsigned bank,const ArmSmmu4KContextConfig *context, + const ArmSmmu4KLiveVisibility *visibility,const ArmSmmu4KRange *before,size_t count, + const ArmSmmu4KLiveRegion *peer,const ArmSmmu4KRange *change,unsigned map,unsigned budget,ArmSmmu4KLiveResult *r) +{ + if(!r)return ArmSmmu4KArgument; + *r=(ArmSmmu4KLiveResult){0};r->must_retain_resources=1; + if(!state || !state->prepared || state->poisoned || !context || + !visibility || !visibility->publish_table_page || map>1 || + !budget || budget>1048576 || state->generation==UINT64_MAX)return ArmSmmu4KArgument; + ArmSmmu4KResult shape; + ArmSmmu4KStatus status=layout(a,&shape); + if(status!=ArmSmmu4KSuccess)return status; + const ArmSmmu4KResult *root=a->upper_region?context->upper_tables:context->lower_tables; + if((!peer && (a->upper_region?(context->lower || context->lower_tables):(context->upper || context->upper_tables))) || + !same_config(a->upper_region?context->upper:context->lower,&a->config) || + root!=&state->tables || state->tables.root_physical!=shape.root_physical || + state->tables.mair!=shape.mair || state->tables.table_pages!=shape.table_pages || + state->tables.start_level!=shape.start_level || state->tables.upper_region!=shape.upper_region) + return ArmSmmu4KArgument; + size_t leaves=0; + status=validate_ranges(a,before,count,&leaves);if(status!=ArmSmmu4KSuccess)return status; + if(leaves!=state->tables.leaf_pages)return ArmSmmu4KArgument; + status=validate_range(a,change);if(status!=ArmSmmu4KSuccess)return status; + if(peer) { + ArmSmmu4KLiveRegion target={a,state,before,count}; + status=context_metadata(context,a->upper_region?peer:&target,a->upper_region?&target:peer); + if(status)return status; + if(touches_pool(change,peer->arena))return ArmSmmu4KOverlap; + if(map)for(size_t i=0;icount;++i) + if(conflicting_alias(change,&peer->ranges[i]))return ArmSmmu4KOverlap; + } + size_t changed=(size_t)(change->bytes/PAGE); + if(map && changed>a->config.leaf_budget-leaves)return ArmSmmu4KBudget; + if(map)for(size_t i=0;ibytes;offset+=PAGE) { + uint64_t va=change->iova+offset,old=expected(before,count,va); + if(map?old!=0:old!=descriptor(change,va))return ArmSmmu4KStateMismatch; + } + status=ArmSmmu4KObserveOwnedContext(io,bank,context,&r->synchronization); + if(status!=ArmSmmu4KSuccess)goto poisoned; + status=verify(a,&shape,before,count,0,0,r);if(status!=ArmSmmu4KSuccess)goto poisoned; + if(peer) { + status=verify(peer->arena,&peer->state->tables,peer->ranges,peer->count,0,0,r); + if(status)goto poisoned; + } + ArmSmmu4KProgramResult initial=r->synchronization; + status=ArmSmmu4KObserveOwnedContext(io,bank,context,&r->synchronization); + if(status!=ArmSmmu4KSuccess)goto poisoned; + if(initial.id0!=r->synchronization.id0 || initial.id1!=r->synchronization.id1 || + initial.id2!=r->synchronization.id2 || initial.context_offset!=r->synchronization.context_offset) + {status=ArmSmmu4KStateMismatch;goto poisoned;} + /* All validation precedes the first store. Caller keeps every master + * quiesced and retains both old/new object references through this call. */ + for(uint64_t offset=0;offsetbytes;offset+=PAGE) { + uint64_t va=change->iova+offset;size_t index; + status=leaf_index(a,&shape,va,&index,r);if(status!=ArmSmmu4KSuccess)goto poisoned; + a->mapping[index]=map?descriptor(change,va):0;++r->descriptor_writes; + } + size_t last_page=SIZE_MAX; + for(uint64_t offset=0;offsetbytes;offset+=PAGE) { + size_t index; + status=leaf_index(a,&shape,change->iova+offset,&index,r);if(status!=ArmSmmu4KSuccess)goto poisoned; + size_t page=index/512; + if(page!=last_page) { + if(visibility->publish_table_page(visibility->cookie,a->mapping+page*512,4096)) {status=ArmSmmu4KIoError;goto poisoned;} + ++r->published_pages;last_page=page; + } + } + status=ArmSmmu4KSynchronizeOwnedContext(io,bank,context,budget,&r->synchronization); + if(status!=ArmSmmu4KSuccess)goto poisoned; + status=verify(a,&shape,before,count,change,map,r);if(status!=ArmSmmu4KSuccess)goto poisoned; + if(peer) { + status=verify(peer->arena,&peer->state->tables,peer->ranges,peer->count,0,0,r); + if(status)goto poisoned; + } + ArmSmmu4KProgramResult after; + status=ArmSmmu4KObserveOwnedContext(io,bank,context,&after);if(status!=ArmSmmu4KSuccess)goto poisoned; + if(after.id0!=r->synchronization.id0 || after.id1!=r->synchronization.id1 || + after.id2!=r->synchronization.id2 || after.context_offset!=r->synchronization.context_offset) + {status=ArmSmmu4KStateMismatch;goto poisoned;} + state->tables.leaf_pages=map?leaves+changed:leaves-changed;++state->generation; + r->complete=1;r->translation_removed=!map;r->must_retain_resources=0; + return ArmSmmu4KSuccess; +poisoned: + state->poisoned=1;if(peer)peer->state->poisoned=1;return status; +} +ArmSmmu4KStatus ArmSmmu4KLiveUpdate(const ArmSmmu4KLiveArena *a,ArmSmmu4KLiveState *state, + const ArmSmmu4KIo *io,unsigned bank,const ArmSmmu4KContextConfig *context, + const ArmSmmu4KLiveVisibility *visibility,const ArmSmmu4KRange *before,size_t count, + const ArmSmmu4KRange *change,unsigned map,unsigned budget,ArmSmmu4KLiveResult *r) +{return update(a,state,io,bank,context,visibility,before,count,0,change,map,budget,r);} +ArmSmmu4KStatus ArmSmmu4KLiveUpdateContext(const ArmSmmu4KIo *io,unsigned bank, + const ArmSmmu4KContextConfig *context,const ArmSmmu4KLiveRegion *lower, + const ArmSmmu4KLiveRegion *upper,unsigned change_upper,const ArmSmmu4KLiveVisibility *visibility, + const ArmSmmu4KRange *change,unsigned map,unsigned budget,ArmSmmu4KLiveResult *r) +{ + if(!r)return ArmSmmu4KArgument; + *r=(ArmSmmu4KLiveResult){0};r->must_retain_resources=1; + if(change_upper>1)return ArmSmmu4KArgument; + ArmSmmu4KStatus status=context_metadata(context,lower,upper);if(status)return status; + const ArmSmmu4KLiveRegion *target=change_upper?upper:lower,*peer=change_upper?lower:upper; + if(!target)return ArmSmmu4KArgument; + return update(target->arena,target->state,io,bank,context,visibility,target->ranges,target->count, + peer,change,map,budget,r); +} diff --git a/kits/ArmSmmu4K/ArmSmmu4KLive.h b/kits/ArmSmmu4K/ArmSmmu4KLive.h new file mode 100644 index 0000000..b1298bd --- /dev/null +++ b/kits/ArmSmmu4K/ArmSmmu4KLive.h @@ -0,0 +1,95 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef ARM_SMMU_4K_LIVE_H +#define ARM_SMMU_4K_LIVE_H +#include "ArmSmmu4KProgram.h" +#ifdef __cplusplus +extern "C" { +#endif +typedef struct { + ArmSmmu4KConfig config; + uint64_t iova,bytes; /* Bounded native VA aperture, no conversion. */ + unsigned upper_region; + volatile uint64_t *mapping; /* Real writable CPU mapping of owned pool. */ + size_t capacity_pages; +} ArmSmmu4KLiveArena; +typedef struct { + ArmSmmu4KResult tables; + uint64_t generation; + unsigned prepared,poisoned; +} ArmSmmu4KLiveState; +typedef struct { + void *cookie; + /* Clean/publish the actual mapped table page for device walks. Required + * even for coherent walks (where the platform may use ordering alone). + * Return zero only after visibility is established. No fake success. */ + int (*publish_table_page)(void *,volatile uint64_t *,size_t bytes); +} ArmSmmu4KLiveVisibility; +typedef struct { + const ArmSmmu4KLiveArena *arena; + ArmSmmu4KLiveState *state; + const ArmSmmu4KRange *ranges; + size_t count; +} ArmSmmu4KLiveRegion; +typedef struct { + size_t descriptor_reads,descriptor_writes,published_pages; + unsigned complete,translation_removed,must_retain_resources; + ArmSmmu4KProgramResult synchronization; +} ArmSmmu4KLiveResult; +/* Prepare an INACTIVE empty native4K arena. Preallocate only real intermediate + * table pages; all leaf slots remain invalid. No MMIO, TLB, allocator or cache + * operations. The explicit leaf budget bounds aperture size and traversal. + * state must be zero initialized and never reset while a context can access + * the pool. Capacity errors do not write tables. Caller must publish all used + * pages before enabling a context with &state->tables. This is not activation. + * The pool uses a fixed owned topology; no shadow/rebuilt translation exists. */ +ArmSmmu4KStatus ArmSmmu4KLivePrepare(const ArmSmmu4KLiveArena *,ArmSmmu4KLiveState *); +/* Read-only exact graph check, including reserved empty intermediate tables. + * Use before context activation and while masters are quiesced. This neither + * observes a context/streams nor clears poison or authorizes resource release. */ +ArmSmmu4KStatus ArmSmmu4KLiveAudit(const ArmSmmu4KLiveArena *,const ArmSmmu4KLiveState *, + const ArmSmmu4KRange *ranges,size_t count,ArmSmmu4KLiveResult *); +/* Incrementally map (map=1) an entirely invalid range, or unmap (map=0) an + * exactly matching mapped range. Replacement requires unmap then map; valid + * descriptors are never overwritten with different valid descriptors. + * before describes ALL currently mapped resources, sorted by native IOVA. + * Exact physical graph checks precede/follow changes to the actual pool. + * No copies/reference table graphs, block mappings or table retirement. + * + * Requires a SINGLE enabled region rooted in this arena, an exclusive powered + * bank/ASID/stream lease, installed fault handling, and ALL masters quiesced + * until successful completion. Caller establishes/retains actual CPU->PA + * ownership, attributes, object references, data-cache ordering and queue + * serialization. Metadata, arrays, state/results and table storage are disjoint + * and immutable during each call except the explicit outputs/table writes. + * These conditions are platform responsibilities, not caller boolean proofs. + * On any uncertain context/table/cache/TLB failure the state is poisoned and + * all old/new resources and tables must remain owned until an independently + * verified quiesce/fault/disable/reset. No automatic rollback/replay occurs. + * Successful unmap removes translation only; it does not generate a GPU fence + * or establish completion of previous DMA. No hardware ownership is inferred + * from declared addresses. The software tests are not hardware qualification. */ +ArmSmmu4KStatus ArmSmmu4KLiveUpdate(const ArmSmmu4KLiveArena *,ArmSmmu4KLiveState *, + const ArmSmmu4KIo *,unsigned bank,const ArmSmmu4KContextConfig *, + const ArmSmmu4KLiveVisibility *,const ArmSmmu4KRange *before,size_t count, + const ArmSmmu4KRange *change,unsigned map,unsigned poll_budget,ArmSmmu4KLiveResult *); +/* Full enabled-context variant: supply exactly its enabled lower/upper regions. + * Both actual pool graphs are audited; cross-pool resource access, pool CPU/PA + * aliases and inconsistent cross-region physical aliases refuse before writes. + * Context roots/configs must match the actual supplied arena/state pairs. + * AuditContext is read-only and does not observe hardware or release anything. + * UpdateContext edits only the selected region, publishes its changed pages, + * synchronizes the actual shared ASID and rechecks BOTH graphs. Any uncertainty + * poisons both states and retains all resources. All ownership/quiescence/cache + * preconditions of LiveUpdate apply to the entire context. No context switch, + * TTBR change, copied/converted/shadow mapping or GPU fence is implemented. */ +ArmSmmu4KStatus ArmSmmu4KLiveAuditContext(const ArmSmmu4KContextConfig *, + const ArmSmmu4KLiveRegion *lower,const ArmSmmu4KLiveRegion *upper,ArmSmmu4KLiveResult *); +ArmSmmu4KStatus ArmSmmu4KLiveUpdateContext(const ArmSmmu4KIo *,unsigned bank, + const ArmSmmu4KContextConfig *,const ArmSmmu4KLiveRegion *lower, + const ArmSmmu4KLiveRegion *upper,unsigned change_upper, + const ArmSmmu4KLiveVisibility *,const ArmSmmu4KRange *change,unsigned map, + unsigned poll_budget,ArmSmmu4KLiveResult *); +#ifdef __cplusplus +} +#endif +#endif diff --git a/kits/ArmSmmu4K/ArmSmmu4KMmio.c b/kits/ArmSmmu4K/ArmSmmu4KMmio.c new file mode 100644 index 0000000..df974ba --- /dev/null +++ b/kits/ArmSmmu4K/ArmSmmu4KMmio.c @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: MIT */ +#include "ArmSmmu4KMmio.h" +#if !defined(__aarch64__) || __BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__ +#error Direct SMMUv2 MMIO binding requires little-endian AArch64 +#endif +static int fits(const ArmSmmu4KMmio *m,size_t at,size_t bytes) +{return m && m->mapping && m->bytes && m->bytes-1<=UINTPTR_MAX-(uintptr_t)m->mapping && !((uintptr_t)m->mapping&7) && !(at&(bytes-1)) && at<=m->bytes && bytes<=m->bytes-at;} +static int read32(void *cookie,size_t at,uint32_t *out) +{ArmSmmu4KMmio *m=cookie;if(!out || !fits(m,at,4))return 1;*out=*(volatile uint32_t *)(m->mapping+at);return 0;} +static int read64(void *cookie,size_t at,uint64_t *out) +{ArmSmmu4KMmio *m=cookie;if(!out || !fits(m,at,8))return 1;*out=*(volatile uint64_t *)(m->mapping+at);return 0;} +static int write32(void *cookie,size_t at,uint32_t value) +{ArmSmmu4KMmio *m=cookie;if(!fits(m,at,4))return 1;*(volatile uint32_t *)(m->mapping+at)=value;return 0;} +static int write64(void *cookie,size_t at,uint64_t value) +{ArmSmmu4KMmio *m=cookie;if(!fits(m,at,8))return 1;*(volatile uint64_t *)(m->mapping+at)=value;return 0;} +static int barrier(void *cookie) +{(void)cookie;__asm__ volatile("dsb sy":::"memory");return 0;} +ArmSmmu4KStatus ArmSmmu4KBindMmio(ArmSmmu4KMmio *mapping,ArmSmmu4KIo *out) +{ + if(!out)return ArmSmmu4KArgument; + *out=(ArmSmmu4KIo){0}; + if(!mapping || !mapping->mapping || ((uintptr_t)mapping->mapping&7) || mapping->bytes<0x2c || mapping->bytes-1>UINTPTR_MAX-(uintptr_t)mapping->mapping)return ArmSmmu4KArgument; + out->cookie=mapping;out->register_bytes=mapping->bytes;out->read32=read32;out->read64=read64; + out->write32=write32;out->write64=write64;out->barrier=barrier; + return ArmSmmu4KSuccess; +} diff --git a/kits/ArmSmmu4K/ArmSmmu4KMmio.h b/kits/ArmSmmu4K/ArmSmmu4KMmio.h new file mode 100644 index 0000000..52a2d37 --- /dev/null +++ b/kits/ArmSmmu4K/ArmSmmu4KMmio.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef ARM_SMMU_4K_MMIO_H +#define ARM_SMMU_4K_MMIO_H +#include "ArmSmmu4KProgram.h" +#ifdef __cplusplus +extern "C" { +#endif +typedef struct {volatile unsigned char *mapping;size_t bytes;} ArmSmmu4KMmio; +/* AArch64 little-endian direct volatile MMIO adapter, with dsb sy barriers. + * Caller supplies an owned live Device mapping. This allocates/maps nothing, + * powers nothing and cannot recover a synchronous external bus abort. All + * inputs/output must be disjoint and retained for the I/O object's lifetime. + * Never bind arbitrary physical/virtual addresses or ordinary user buffers + * as a hardware mapping. Software tests are explicitly separate fixtures. */ +ArmSmmu4KStatus ArmSmmu4KBindMmio(ArmSmmu4KMmio *,ArmSmmu4KIo *); +#ifdef __cplusplus +} +#endif +#endif diff --git a/kits/ArmSmmu4K/ArmSmmu4KProgram.c b/kits/ArmSmmu4K/ArmSmmu4KProgram.c new file mode 100644 index 0000000..9da9c95 --- /dev/null +++ b/kits/ArmSmmu4K/ArmSmmu4KProgram.c @@ -0,0 +1,183 @@ +/* SPDX-License-Identifier: MIT */ +#include "ArmSmmu4KProgram.h" +#define FSR_FAULT UINT32_C(0xc00001fe) +static int read_image(const ArmSmmu4KIo *io,size_t gr1,size_t cb,unsigned bank, + ArmSmmu4KContextImage *v) +{ + v->reserved_zero=0; + return io->read32(io->cookie,gr1+bank*4,&v->cbar) || + io->read32(io->cookie,gr1+0x800+bank*4,&v->cba2r) || + io->read32(io->cookie,cb+0x30,&v->tcr) || + io->read32(io->cookie,cb+0x10,&v->tcr2) || + io->read64(io->cookie,cb+0x20,&v->ttbr0) || + io->read64(io->cookie,cb+0x28,&v->ttbr1) || + io->read32(io->cookie,cb+0x38,&v->mair0) || + io->read32(io->cookie,cb+0x3c,&v->mair1) || + io->read32(io->cookie,cb,&v->sctlr); +} +static int write32(const ArmSmmu4KIo *io,size_t at,uint32_t value,ArmSmmu4KProgramResult *r) +{ + ++r->writes;return io->write32(io->cookie,at,value); +} +static int write64(const ArmSmmu4KIo *io,size_t at,uint64_t value,ArmSmmu4KProgramResult *r) +{ + ++r->writes;return io->write64(io->cookie,at,value); +} +/* TLBIASID (+0x610, value = ASID) or, with all set, TLBIALL (+0x618, value ignored), then TLBSYNC/TLBSTATUS. */ +static ArmSmmu4KStatus sync_tlb(const ArmSmmu4KIo *io,size_t cb,unsigned asid,unsigned all, + unsigned budget,ArmSmmu4KProgramResult *r) +{ + r->tlb_synchronized=0; + if(io->barrier(io->cookie) || write32(io,cb+(all?0x618:0x610),all?0:asid,r) || + write32(io,cb+0x7f0,UINT32_MAX,r) || io->barrier(io->cookie))return ArmSmmu4KIoError; + for(unsigned i=0;ipoll_reads; + if(io->read32(io->cookie,cb+0x7f4,&status))return ArmSmmu4KIoError; + if(!(status&1)) { + /* Order the completed device observation before callers recycle + * normal-memory tables or publish subsequent context work. */ + if(io->barrier(io->cookie))return ArmSmmu4KIoError; + r->tlb_synchronized=1;return ArmSmmu4KSuccess; + } + } + return ArmSmmu4KTimeout; +} +static ArmSmmu4KStatus sync_asid(const ArmSmmu4KIo *io,size_t cb,unsigned asid, + unsigned budget,ArmSmmu4KProgramResult *r) +{ + return sync_tlb(io,cb,asid,0,budget,r); +} +static ArmSmmu4KStatus disable_sync(const ArmSmmu4KIo *io,size_t cb,unsigned asid, + unsigned budget,ArmSmmu4KProgramResult *r) +{ + r->must_retain_tables=1;r->disabled_verified=0; + uint32_t value; + if(write32(io,cb,0,r) || io->barrier(io->cookie) || io->read32(io->cookie,cb,&value))return ArmSmmu4KIoError; + if(value)return ArmSmmu4KStateMismatch; + r->disabled_verified=1;r->enabled=0; + ArmSmmu4KStatus status=sync_asid(io,cb,asid,budget,r); + if(status==ArmSmmu4KSuccess)r->must_retain_tables=0; + return status; +} +static ArmSmmu4KStatus prepare(const ArmSmmu4KIo *io,unsigned bank, + const ArmSmmu4KContextConfig *config,unsigned budget,ArmSmmu4KProgramResult *r, + ArmSmmu4KContextConfig *actual,ArmSmmu4KContextImage *image,size_t *gr1) +{ + if(!r)return ArmSmmu4KArgument; + *r=(ArmSmmu4KProgramResult){0}; + r->must_retain_tables=1; + if(!io || !config || !io->read32 || !io->read64 || !io->write32 || !io->write64 || + !io->barrier || io->register_bytes<0x2c || !budget || budget>1048576 || bank>255)return ArmSmmu4KArgument; + r->phase=1; + if(io->read32(io->cookie,0x20,&r->id0) || io->read32(io->cookie,0x24,&r->id1) || + io->read32(io->cookie,0x28,&r->id2))return ArmSmmu4KIoError; + unsigned shift=(r->id1>>31)?16:12; + size_t pages=(size_t)1<<(((r->id1>>28)&7)+1); + size_t stride=(size_t)1<id1&255,s2=(r->id1>>16)&255; + if(!banks || banks>pages || s2>banks || bank=banks || + io->register_bytes<2*pages*stride)return ArmSmmu4KUnsupported; + *gr1=stride;r->context_offset=(pages+bank)*stride; + if(r->context_offset>io->register_bytes || io->register_bytes-r->context_offset<0x7f8 || + *gr1+0x804+bank*4>io->register_bytes)return ArmSmmu4KBadRange; + *actual=*config;actual->id0=r->id0;actual->id2=r->id2; + return ArmSmmu4KEncodeContext(actual,image); +} +ArmSmmu4KStatus ArmSmmu4KProgramDisabledContext(const ArmSmmu4KIo *io,unsigned bank, + const ArmSmmu4KContextConfig *config,unsigned budget,ArmSmmu4KProgramResult *r) +{ + ArmSmmu4KContextConfig actual;ArmSmmu4KContextImage image;size_t gr1; + ArmSmmu4KStatus status=prepare(io,bank,config,budget,r,&actual,&image,&gr1); + if(status!=ArmSmmu4KSuccess)return status; + size_t cb=(size_t)r->context_offset;uint32_t sctlr; + if(io->read32(io->cookie,cb,&sctlr))return ArmSmmu4KIoError; + if(sctlr&1){r->enabled=1;return ArmSmmu4KBusy;} + r->must_retain_tables=1;r->phase=2; + /* A disabled owned bank only. TCR2/TCR precede all TTBR accesses. */ + if(write32(io,cb,0,r) || io->barrier(io->cookie) || + write32(io,gr1+0x800+bank*4,image.cba2r,r) || write32(io,gr1+bank*4,image.cbar,r) || + write32(io,cb+0x10,image.tcr2,r) || write32(io,cb+0x30,image.tcr,r) || + write64(io,cb+0x20,image.ttbr0,r) || write64(io,cb+0x28,image.ttbr1,r) || + write32(io,cb+0x38,image.mair0,r) || write32(io,cb+0x3c,image.mair1,r) || + write32(io,cb+0x58,UINT32_MAX,r)) {status=ArmSmmu4KIoError;goto failed;} + status=sync_asid(io,cb,actual.asid,budget,r);if(status!=ArmSmmu4KSuccess)goto failed; + r->phase=3; + if(read_image(io,gr1,cb,bank,&r->readback)) {status=ArmSmmu4KIoError;goto failed;} + r->readback_valid=1; + /* Verify configuration while M is still zero; never enable a bad granule. */ + sctlr=r->readback.sctlr;r->readback.sctlr=image.sctlr; + status=ArmSmmu4KValidateContextReadback(&actual,&r->readback);r->readback.sctlr=sctlr; + if(status!=ArmSmmu4KSuccess || sctlr) {status=ArmSmmu4KStateMismatch;goto failed;} + r->readback_valid=0; + if(io->barrier(io->cookie) || write32(io,cb,image.sctlr,r) || io->barrier(io->cookie)) {status=ArmSmmu4KIoError;goto failed;} + r->phase=4; + if(read_image(io,gr1,cb,bank,&r->readback)) {status=ArmSmmu4KIoError;goto failed;} + r->readback_valid=1;r->enabled=r->readback.sctlr&1; + if(io->read32(io->cookie,cb+0x58,&r->fault_status)) {status=ArmSmmu4KIoError;goto failed;} + status=ArmSmmu4KValidateContextReadback(&actual,&r->readback);if(status!=ArmSmmu4KSuccess)goto failed; + if(r->fault_status&FSR_FAULT) {status=ArmSmmu4KFault;goto failed;} + r->phase=5;r->enabled=1;return ArmSmmu4KSuccess; +failed: + /* Even a reported failed write may have reached hardware. On uncertainty + * retain all tables/leases; never claim safe recycling after timeout. */ + r->readback_valid=0;(void)disable_sync(io,cb,actual.asid,budget,r); + r->phase=6;return status; +} +ArmSmmu4KStatus ArmSmmu4KDisableOwnedContext(const ArmSmmu4KIo *io,unsigned bank, + const ArmSmmu4KContextConfig *config,unsigned budget,ArmSmmu4KProgramResult *r) +{ + ArmSmmu4KContextConfig actual;ArmSmmu4KContextImage image;size_t gr1; + ArmSmmu4KStatus status=prepare(io,bank,config,budget,r,&actual,&image,&gr1); + if(status!=ArmSmmu4KSuccess)return status; + size_t cb=(size_t)r->context_offset;r->must_retain_tables=1; + if(read_image(io,gr1,cb,bank,&r->readback))return ArmSmmu4KIoError; + r->readback_valid=1;uint32_t sctlr=r->readback.sctlr; + /* An enabled bank must match the owned image (ValidateContextReadback, with its documented read-only bits); a + * disabled one is compared with SCTLR substituted. */ + if(sctlr)status=ArmSmmu4KValidateContextReadback(&actual,&r->readback); + else {r->readback.sctlr=image.sctlr;status=ArmSmmu4KValidateContextReadback(&actual,&r->readback);r->readback.sctlr=sctlr;} + if(status!=ArmSmmu4KSuccess)return status; /* Another context: do not touch. */ + r->readback_valid=0;status=disable_sync(io,cb,actual.asid,budget,r);r->phase=status==ArmSmmu4KSuccess?7:6; + return status; +} + +ArmSmmu4KStatus ArmSmmu4KObserveOwnedContext(const ArmSmmu4KIo *io,unsigned bank, + const ArmSmmu4KContextConfig *config,ArmSmmu4KProgramResult *r) +{ + ArmSmmu4KContextConfig actual;ArmSmmu4KContextImage image;size_t gr1; + ArmSmmu4KStatus status=prepare(io,bank,config,1,r,&actual,&image,&gr1); + if(status!=ArmSmmu4KSuccess)return status; + if(read_image(io,gr1,(size_t)r->context_offset,bank,&r->readback))return ArmSmmu4KIoError; + r->readback_valid=1;r->enabled=r->readback.sctlr&1; + status=ArmSmmu4KValidateContextReadback(&actual,&r->readback); + if(status!=ArmSmmu4KSuccess)return status; + if(io->read32(io->cookie,(size_t)r->context_offset+0x58,&r->fault_status) || io->barrier(io->cookie))return ArmSmmu4KIoError; + if(r->fault_status&FSR_FAULT)return ArmSmmu4KFault; + r->phase=8;return ArmSmmu4KSuccess; +} + +static ArmSmmu4KStatus synchronize(const ArmSmmu4KIo *io,unsigned bank, + const ArmSmmu4KContextConfig *config,unsigned all,unsigned budget,ArmSmmu4KProgramResult *r) +{ + if(!r)return ArmSmmu4KArgument; + *r=(ArmSmmu4KProgramResult){0};r->must_retain_tables=1; + if(!budget || budget>1048576)return ArmSmmu4KArgument; + ArmSmmu4KStatus status=ArmSmmu4KObserveOwnedContext(io,bank,config,r); + if(status!=ArmSmmu4KSuccess)return status; + status=sync_tlb(io,(size_t)r->context_offset,config->asid,all,budget,r); + if(status!=ArmSmmu4KSuccess)return status; + ArmSmmu4KProgramResult after; + status=ArmSmmu4KObserveOwnedContext(io,bank,config,&after); + r->readback=after.readback;r->readback_valid=after.readback_valid; + r->fault_status=after.fault_status;r->enabled=after.enabled; + if(status!=ArmSmmu4KSuccess)return status; + if(r->id0!=after.id0 || r->id1!=after.id1 || r->id2!=after.id2 || + r->context_offset!=after.context_offset)return ArmSmmu4KStateMismatch; + r->phase=9;return ArmSmmu4KSuccess; +} +ArmSmmu4KStatus ArmSmmu4KSynchronizeOwnedContext(const ArmSmmu4KIo *io,unsigned bank, + const ArmSmmu4KContextConfig *config,unsigned budget,ArmSmmu4KProgramResult *r) +{return synchronize(io,bank,config,0,budget,r);} +ArmSmmu4KStatus ArmSmmu4KInvalidateAllOwnedContext(const ArmSmmu4KIo *io,unsigned bank, + const ArmSmmu4KContextConfig *config,unsigned budget,ArmSmmu4KProgramResult *r) +{return synchronize(io,bank,config,1,budget,r);} diff --git a/kits/ArmSmmu4K/ArmSmmu4KProgram.h b/kits/ArmSmmu4K/ArmSmmu4KProgram.h new file mode 100644 index 0000000..77babf5 --- /dev/null +++ b/kits/ArmSmmu4K/ArmSmmu4KProgram.h @@ -0,0 +1,68 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef ARM_SMMU_4K_PROGRAM_H +#define ARM_SMMU_4K_PROGRAM_H +#include "ArmSmmu4KContext.h" +#ifdef __cplusplus +extern "C" { +#endif +/* Byte offsets are relative to one real, owned SMMUv2 MMIO mapping. All I/O + * callbacks return 0 on success. read/write64 must perform actual aligned + * 64-bit TTBR accesses; barrier orders table visibility and device writes. + * A barrier is NOT cache cleaning. No callback may silently fake success. */ +typedef struct { + void *cookie; + size_t register_bytes; + int (*read32)(void *,size_t,uint32_t *); + int (*read64)(void *,size_t,uint64_t *); + int (*write32)(void *,size_t,uint32_t); + int (*write64)(void *,size_t,uint64_t); + int (*barrier)(void *); +} ArmSmmu4KIo; +typedef struct { + uint64_t context_offset; + uint32_t id0,id1,id2,phase,writes,poll_reads,fault_status; + uint32_t readback_valid,enabled,disabled_verified,tlb_synchronized,must_retain_tables; + ArmSmmu4KContextImage readback; +} ArmSmmu4KProgramResult; + +/* REAL register programming primitive, not a device-discovery/ownership layer. + * Caller MUST hold an exclusive bank/ASID/stream lease, keep the device powered + * and clocked, quiesce/fault its streams, install its fault interrupt handler, + * establish platform ACTLR/security requirements, and clean/publish real owned + * native4K tables before calling. Every argument/result must be disjoint. + * Reads actual ID0/ID1/ID2 through I/O and overrides config's supplied IDs. + * Refuses enabled banks before any write. Enables SCTLR only after register + * readback and bounded ASID invalidation/synchronization. No global reset, + * stream reassignment or other bank is touched. On failure attempts disable + * and sync; must_retain_tables stays set unless both are confirmed. Success + * also retains table ownership while the installed context is active. + * This does not walk tables or establish actual GPU execution/correctness. */ +ArmSmmu4KStatus ArmSmmu4KProgramDisabledContext(const ArmSmmu4KIo *,unsigned bank, + const ArmSmmu4KContextConfig *,unsigned poll_budget,ArmSmmu4KProgramResult *); +/* Read-only observation of actual IDs and all owned context registers. Requires + * exact enabled native4K image and no pending context fault. Does not walk the + * physical tables or establish ownership. Readback is a momentary observation; + * callers retain resources, quiescence/exclusive lease and fault handling. */ +ArmSmmu4KStatus ArmSmmu4KObserveOwnedContext(const ArmSmmu4KIo *,unsigned bank, + const ArmSmmu4KContextConfig *,ArmSmmu4KProgramResult *); +/* Publish already cleaned descriptor changes with a real ASID invalidation + * and bounded context TLBSYNC. Observe the complete enabled native4K image and + * faults before and after. Caller exclusively owns this bank/ASID, serializes + * all page-table updates and keeps every master quiesced. A barrier is not a + * cache clean. No SCTLR/TTBR/stream writes or recovery/reset are performed. + * Any failure retains all mappings/resources; success is not a GPU fence. */ +ArmSmmu4KStatus ArmSmmu4KSynchronizeOwnedContext(const ArmSmmu4KIo *,unsigned bank, + const ArmSmmu4KContextConfig *,unsigned poll_budget,ArmSmmu4KProgramResult *); +/* As ArmSmmu4KSynchronizeOwnedContext, but with TLBIALL for the bank instead of TLBIASID: every cached translation of the bank goes, + * whatever ASID it was tagged with. */ +ArmSmmu4KStatus ArmSmmu4KInvalidateAllOwnedContext(const ArmSmmu4KIo *,unsigned bank, + const ArmSmmu4KContextConfig *,unsigned poll_budget,ArmSmmu4KProgramResult *); +/* Disable only if actual stage/format/roots/ASID/attributes still match the + * expected owned context. Retain bank, mappings and tables on any uncertainty. + * Does not release the caller's lease or power/stream/interrupt resources. */ +ArmSmmu4KStatus ArmSmmu4KDisableOwnedContext(const ArmSmmu4KIo *,unsigned bank, + const ArmSmmu4KContextConfig *,unsigned poll_budget,ArmSmmu4KProgramResult *); +#ifdef __cplusplus +} +#endif +#endif diff --git a/kits/ArmSmmu4K/ArmSmmu4KStreams.c b/kits/ArmSmmu4K/ArmSmmu4KStreams.c new file mode 100644 index 0000000..1c08253 --- /dev/null +++ b/kits/ArmSmmu4K/ArmSmmu4KStreams.c @@ -0,0 +1,140 @@ +/* SPDX-License-Identifier: MIT */ +#include "ArmSmmu4KStreams.h" +#define FAULT (UINT32_C(2)<<16) +static int rd(const ArmSmmu4KIo *io,size_t at,uint32_t *v,ArmSmmu4KStreamResult *r) +{++r->reads;return io->read32(io->cookie,at,v);} +static int wr(const ArmSmmu4KIo *io,size_t at,uint32_t v,ArmSmmu4KStreamResult *r) +{++r->writes;return io->write32(io->cookie,at,v);} +static unsigned exids(uint32_t id){return (id>>8)&1;} +static uint32_t valid_smr(uint32_t id){return exids(id)?0:UINT32_C(1)<<31;} +static uint32_t valid_s2cr(uint32_t id){return exids(id)?1u<<10:0;} +static int valid(uint32_t id,uint32_t smr,uint32_t s2cr) +{return exids(id)?!!(s2cr&(1u<<10)):!!(smr&(UINT32_C(1)<<31));} +static ArmSmmu4KStatus global(const ArmSmmu4KIo *io,ArmSmmu4KStreamResult *r) +{ + if(!io || !io->read32 || !io->write32 || !io->barrier || io->register_bytes<0xc04)return ArmSmmu4KArgument; + if(rd(io,0x20,&r->id0,r) || rd(io,0,&r->scr0,r) || rd(io,0x48,&r->global_fault,r))return ArmSmmu4KIoError; + if(!(r->id0&(1u<<27)) || !(r->id0&255) || 0xc00u+4*(r->id0&255)>io->register_bytes)return ArmSmmu4KUnsupported; + /* Real global enable (CLIENTPD clear), global fault reporting (GFRE/GFIE) and unmatched-stream fault + * policy (USFCFG). Configuration-access fault reporting (GCFGFRE/GCFGFIE) is not required: Linux sets + * it only in its own global reset, and firmware that owns sCR0 may leave it clear (an Adreno SMMU read + * 0x002f0406). It reports bad register accesses, it does not change translation or bypass. */ + if((r->scr0&0x407)!=0x406 || !!(r->scr0&8)!=exids(r->id0))return ArmSmmu4KStateMismatch; + return r->global_fault?ArmSmmu4KFault:ArmSmmu4KSuccess; +} +static ArmSmmu4KStatus scan(const ArmSmmu4KIo *io,const ArmSmmu4KStream *s,unsigned n, + unsigned bank,unsigned attached,ArmSmmu4KStreamResult *r) +{ + unsigned slots=r->id0&255;uint32_t sidmask=exids(r->id0)?0xffffu:(1u<<((r->id0>>9)&15))-1; + for(unsigned i=0;i=slots || (s[i].sid&~sidmask))return ArmSmmu4KArgument; + for(unsigned j=0;jstreams_verified=0; + for(unsigned slot=0;slotid0,smr,s2cr); + if(attached) { + if(smr!=(valid_smr(r->id0)|s[own].sid) || s2cr!=(bank|valid_s2cr(r->id0)))return ArmSmmu4KStateMismatch; + } else if((s2cr&~valid_s2cr(r->id0))!=FAULT || + (live && smr!=(valid_smr(r->id0)|s[own].sid)))return ArmSmmu4KBusy; + ++r->streams_verified; + } else if(valid(r->id0,smr,s2cr)) { + if(!(s2cr&(3u<<16)) && (s2cr&255)==bank)return ArmSmmu4KBusy; + uint32_t mask=(smr>>16)&(exids(r->id0)?0xffffu:0x7fffu),id=smr&0xffff; + for(unsigned i=0;imust_retain_resources=1; + if(!s || !n || n>8 || bank>255)return ArmSmmu4KArgument; + ArmSmmu4KStatus status=global(io,r);if(status)return status; + return scan(io,s,n,bank,0,r); +} +static int quarantine(const ArmSmmu4KIo *io,const ArmSmmu4KStreamState *state,unsigned count,ArmSmmu4KStreamResult *r) +{ + r->streams_faulted=0;int failed=0; + for(unsigned i=0;istreams[i].slot;uint32_t value; + /* First stop translation. Only after confirmed FAULT may we restore + * the exact owned SID and make EXIDS matching valid again. This also + * handles an indeterminate/failed SMR publication without activating + * an old or corrupted SID from an initially invalid slot. */ + size_t smr=0x800+4*state->streams[i].slot; + uint32_t match=valid_smr(state->id0)|state->streams[i].sid; + if(wr(io,at,FAULT,r) || io->barrier(io->cookie) || rd(io,at,&value,r) || value!=FAULT || + wr(io,smr,match,r) || io->barrier(io->cookie) || rd(io,smr,&value,r) || value!=match || + wr(io,at,FAULT|valid_s2cr(state->id0),r) || io->barrier(io->cookie) || + rd(io,at,&value,r) || value!=(FAULT|valid_s2cr(state->id0)))failed=1; + else ++r->streams_faulted; + } + return failed; +} +static ArmSmmu4KStatus observe(const ArmSmmu4KIo *io,const ArmSmmu4KStreamState *s, + const ArmSmmu4KContextConfig *c,ArmSmmu4KStreamResult *r) +{ + ArmSmmu4KStatus status=global(io,r);if(status)return status; + if(r->id0!=s->id0 || r->scr0!=s->scr0)return ArmSmmu4KStateMismatch; + status=ArmSmmu4KObserveOwnedContext(io,s->bank,c,&r->context);if(status)return status; + status=scan(io,s->streams,s->count,s->bank,1,r);if(status)return status; + return io->barrier(io->cookie)?ArmSmmu4KIoError:ArmSmmu4KSuccess; +} +ArmSmmu4KStatus ArmSmmu4KObserveStreams(const ArmSmmu4KIo *io,const ArmSmmu4KStreamState *s, + const ArmSmmu4KContextConfig *c,ArmSmmu4KStreamResult *r) +{ + if(!r)return ArmSmmu4KArgument; + *r=(ArmSmmu4KStreamResult){0};r->must_retain_resources=1; + if(!s || !s->attached || s->poisoned || !s->count || s->count>8)return ArmSmmu4KArgument; + return observe(io,s,c,r); +} +ArmSmmu4KStatus ArmSmmu4KAttachStreams(const ArmSmmu4KIo *io,ArmSmmu4KStreamState *state, + const ArmSmmu4KStream *s,unsigned n,unsigned bank,const ArmSmmu4KContextConfig *c,ArmSmmu4KStreamResult *r) +{ + if(!r)return ArmSmmu4KArgument; + *r=(ArmSmmu4KStreamResult){0};r->must_retain_resources=1; + if(!state || !s || !n || n>8 || bank>255)return ArmSmmu4KArgument; + if(state->attempted || state->attached || state->poisoned)return ArmSmmu4KBusy; + r->phase=1;ArmSmmu4KStatus status=global(io,r);if(status)return status; + status=ArmSmmu4KObserveOwnedContext(io,bank,c,&r->context);if(status)return status; + status=scan(io,s,n,bank,0,r);if(status)return status; + state->attempted=1;state->id0=r->id0;state->scr0=r->scr0;state->count=n;state->bank=bank; + for(unsigned i=0;istreams[i]=s[i]; + unsigned touched=0;r->phase=2; + for(unsigned i=0;ibarrier(io->cookie) || rd(io,s2cr,&value,r)) {status=ArmSmmu4KIoError;goto failed;} + if(value!=bank){status=ArmSmmu4KStateMismatch;goto failed;} + if(wr(io,smr,valid_smr(r->id0)|s[i].sid,r) || io->barrier(io->cookie) || rd(io,smr,&value,r)) {status=ArmSmmu4KIoError;goto failed;} + if(value!=(valid_smr(r->id0)|s[i].sid)){status=ArmSmmu4KStateMismatch;goto failed;} + if(exids(r->id0) && (wr(io,s2cr,bank|valid_s2cr(r->id0),r) || io->barrier(io->cookie))) {status=ArmSmmu4KIoError;goto failed;} + } + r->phase=3;status=observe(io,state,c,r);if(status)goto failed; + state->attached=1;r->phase=4;return ArmSmmu4KSuccess; +failed: + state->poisoned=1;state->attached=0;r->phase=5; + (void)quarantine(io,state,touched,r);return status; +} +ArmSmmu4KStatus ArmSmmu4KFaultStreams(const ArmSmmu4KIo *io,ArmSmmu4KStreamState *s,ArmSmmu4KStreamResult *r) +{ + if(!r)return ArmSmmu4KArgument; + *r=(ArmSmmu4KStreamResult){0};r->must_retain_resources=1; + if(!s || !s->attached || s->poisoned || !s->count || s->count>8)return ArmSmmu4KArgument; + ArmSmmu4KStatus status=global(io,r);if(status)return status; + if(r->id0!=s->id0 || r->scr0!=s->scr0)return ArmSmmu4KStateMismatch; + status=scan(io,s->streams,s->count,s->bank,1,r);if(status)return status; + r->phase=6; + if(quarantine(io,s,s->count,r)){s->poisoned=1;s->attached=0;return ArmSmmu4KIoError;} + s->attached=0;r->phase=7;return ArmSmmu4KSuccess; +} diff --git a/kits/ArmSmmu4K/ArmSmmu4KStreams.h b/kits/ArmSmmu4K/ArmSmmu4KStreams.h new file mode 100644 index 0000000..6e6d3ce --- /dev/null +++ b/kits/ArmSmmu4K/ArmSmmu4KStreams.h @@ -0,0 +1,56 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef ARM_SMMU_4K_STREAMS_H +#define ARM_SMMU_4K_STREAMS_H +#include "ArmSmmu4KProgram.h" +#ifdef __cplusplus +extern "C" { +#endif +typedef struct {unsigned slot;uint16_t sid;} ArmSmmu4KStream; +typedef struct { + unsigned attempted,attached,poisoned,count,bank; + uint32_t id0,scr0; + ArmSmmu4KStream streams[8]; +} ArmSmmu4KStreamState; +typedef struct { + unsigned phase,reads,writes,streams_verified,streams_faulted,must_retain_resources; + uint32_t id0,scr0,global_fault; + ArmSmmu4KProgramResult context; +} ArmSmmu4KStreamResult; +/* Read-only preflight of global fault policy, target faulting slots and all + * conflicting live routes, including unrelated SIDs using the selected bank. + * Intended before programming a disabled context under the same global lease. */ +ArmSmmu4KStatus ArmSmmu4KCheckFaultedStreams(const ArmSmmu4KIo *,const ArmSmmu4KStream *, + unsigned count,unsigned bank,ArmSmmu4KStreamResult *); +/* Native SMMUv2 exact stream matching (SMS), both legacy and EXIDS layouts. + * Caller owns the listed slots, bank/ASID, mappings and a global configuration + * lock. All client masters MUST remain quiesced until successful attachment. + * Requires live powered MMIO and installed context/global fault handlers, + * active native4K context/tables with verified physical ownership/coherence, + * and global fault-on-unmatched (no bypass). Global setup/quirks and context + * creation belong to the platform; no global reset or mask probing occurs. + * All target S2CRs must already fault; valid target SMRs must be exact matches. + * Every other active SMR is checked for overlapping requested SIDs. No masked + * attachment, bypass, privilege override, implicit slot selection or context + * reassignment is supported. Reads actual ID0 and enabled context registers. + * Inputs/state/result must be disjoint; state is zero-initialized and remains + * exclusively owned. Configuration/tables remain immutable while attached. + * Writes S2CR before publishing SMR; EXIDS keeps EXIDVALID clear until the + * exact SMR has been read back. Success rereads actual context/stream state. + * Any failure after a mutation attempt faults only the leased touched slots, + * poisons state and forbids replay. Resources are retained on EVERY outcome. + * Successful software I/O fixtures are not proof of active hardware tables. */ +ArmSmmu4KStatus ArmSmmu4KAttachStreams(const ArmSmmu4KIo *,ArmSmmu4KStreamState *, + const ArmSmmu4KStream *,unsigned count,unsigned bank,const ArmSmmu4KContextConfig *,ArmSmmu4KStreamResult *); +/* Read-only, checks all routes/conflicts/global fault policy and complete + * enabled native4K context image. No fault clear or recovery is synthesized. */ +ArmSmmu4KStatus ArmSmmu4KObserveStreams(const ArmSmmu4KIo *,const ArmSmmu4KStreamState *, + const ArmSmmu4KContextConfig *,ArmSmmu4KStreamResult *); +/* Require exact owned active routes before changing them to FAULT. SMRs stay + * valid, so those SIDs remain blocked. Does not drain transactions/TLBs, stop + * devices, disable context, release ownership or authorize freeing tables. + * Failure poisons state; external real reset/recovery is required. */ +ArmSmmu4KStatus ArmSmmu4KFaultStreams(const ArmSmmu4KIo *,ArmSmmu4KStreamState *,ArmSmmu4KStreamResult *); +#ifdef __cplusplus +} +#endif +#endif diff --git a/kits/ArmSmmu4K/ArmSmmu4KTableMemory.c b/kits/ArmSmmu4K/ArmSmmu4KTableMemory.c new file mode 100644 index 0000000..c21faa9 --- /dev/null +++ b/kits/ArmSmmu4K/ArmSmmu4KTableMemory.c @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: MIT */ +#include "ArmSmmu4KTableMemory.h" +static int read_descriptor(void *cookie,uint64_t physical,uint64_t *value) +{ + const ArmSmmu4KTableMemory *memory=cookie; + if(!value || (physical&7))return 1; + for(unsigned i=0;icount;++i) { + const ArmSmmu4KTableMapping *p=&memory->pools[i]; + if(physical>=p->physical && physical-p->physical<(uint64_t)p->pages*4096) { + *value=p->mapping[(physical-p->physical)/8];return 0; + } + } + return 1; +} +ArmSmmu4KStatus ArmSmmu4KBindTableMemory(const ArmSmmu4KTableMemory *memory,ArmSmmu4KTableIo *io) +{ + if(!io)return ArmSmmu4KArgument;*io=(ArmSmmu4KTableIo){0}; + if(!memory || !memory->count || memory->count>2)return ArmSmmu4KArgument; + for(unsigned i=0;icount;++i) { + const ArmSmmu4KTableMapping *p=&memory->pools[i];uintptr_t cpu=(uintptr_t)p->mapping; + if(!cpu || (cpu&4095) || (p->physical&4095) || !p->pages || p->pages>SIZE_MAX/4096 || + p->physical>=(UINT64_C(1)<<48) || p->pages*UINT64_C(4096)>(UINT64_C(1)<<48)-p->physical || + p->pages*4096>UINTPTR_MAX-cpu)return ArmSmmu4KBadRange; + for(unsigned j=0;jpools[j]; + if((p->physicalphysical+q->pages*UINT64_C(4096) && q->physicalphysical+p->pages*UINT64_C(4096)) || + (cpu<(uintptr_t)q->mapping+q->pages*4096 && (uintptr_t)q->mappingpages*4096))return ArmSmmu4KOverlap; + } + } + io->cookie=(void *)memory;io->read_descriptor=read_descriptor;return ArmSmmu4KSuccess; +} diff --git a/kits/ArmSmmu4K/ArmSmmu4KTableMemory.h b/kits/ArmSmmu4K/ArmSmmu4KTableMemory.h new file mode 100644 index 0000000..da4f92a --- /dev/null +++ b/kits/ArmSmmu4K/ArmSmmu4KTableMemory.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef ARM_SMMU_4K_TABLE_MEMORY_H +#define ARM_SMMU_4K_TABLE_MEMORY_H +#include "ArmSmmu4KAudit.h" +#ifdef __cplusplus +extern "C" { +#endif +typedef struct { + const volatile uint64_t *mapping; + uint64_t physical; + size_t pages; +} ArmSmmu4KTableMapping; +typedef struct { + unsigned count; + ArmSmmu4KTableMapping pools[2]; +} ArmSmmu4KTableMemory; +/* Concrete aligned volatile 64-bit descriptor reads through already existing + * CPU mappings of one/two physically contiguous owned table pools. Validates + * CPU/physical bounds and aliasing; it cannot create or establish the claimed + * VA->PA relation, ownership, cache attributes or visibility. Platform must + * obtain and retain those mappings from its real allocator, clean tables for + * device visibility and order publication before activation/audit. Caller + * retains immutable memory metadata and tables under an exclusive lease. + * No fallback tables, copying/rebuilding, MMIO, cache clean, allocation or + * hardware probe. A matching software buffer alone is not hardware evidence. + * On bind failure io is cleared. Inputs/output/storage must be disjoint. */ +ArmSmmu4KStatus ArmSmmu4KBindTableMemory(const ArmSmmu4KTableMemory *,ArmSmmu4KTableIo *); +#ifdef __cplusplus +} +#endif +#endif diff --git a/kits/ArmSmmu4K/LICENSE b/kits/ArmSmmu4K/LICENSE new file mode 100644 index 0000000..f5c4149 --- /dev/null +++ b/kits/ArmSmmu4K/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 ArmSmmu4K contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/kits/ArmSmmu4K/test_audit.c b/kits/ArmSmmu4K/test_audit.c new file mode 100644 index 0000000..5f2d85f --- /dev/null +++ b/kits/ArmSmmu4K/test_audit.c @@ -0,0 +1,104 @@ +/* SPDX-License-Identifier: MIT */ +/* Synthetic owned physical memory only; no hardware observation. */ +#include "ArmSmmu4KTableMemory.h" +#include +#include +#include +#define CHECK(x) do {if(!(x)){fprintf(stderr,"line %d: %s\n",__LINE__,#x);exit(1);}}while(0) +#define CAP 16 +_Alignas(4096) static uint64_t tables[CAP*512]; +static unsigned char visited[CAP]; +static ArmSmmu4KConfig c; +static ArmSmmu4KResult built; +static ArmSmmu4KRange ranges[3]; +static ArmSmmu4KAuditRegion a; +static ArmSmmu4KTableAuditResult result; +static size_t calls,fail_at; +static int read_descriptor(void *cookie,uint64_t pa,uint64_t *out) +{ + CHECK(cookie==tables && !(pa&7) && pa>=c.table_physical && pa-c.table_physicalPA. */ + setup(40,40,1,15); + ArmSmmu4KTableMemory memory={1,{{tables,c.table_physical,CAP},{0}}};ArmSmmu4KTableIo direct; + CHECK(!ArmSmmu4KBindTableMemory(&memory,&direct)); + CHECK(!ArmSmmu4KAuditTables(&direct,&c,&built,&a,c.table_physical,&result)); + uint64_t value; + CHECK(direct.read_descriptor(direct.cookie,c.table_physical-8,&value)); + CHECK(direct.read_descriptor(direct.cookie,c.table_physical+CAP*4096,&value)); + CHECK(direct.read_descriptor(direct.cookie,c.table_physical+1,&value)); + CHECK(direct.read_descriptor(direct.cookie,c.table_physical,NULL)); + memory.count=2;memory.pools[1]=memory.pools[0]; + CHECK(ArmSmmu4KBindTableMemory(&memory,&direct)==ArmSmmu4KOverlap && !direct.read_descriptor); + memory.count=1;memory.pools[0].mapping=tables+1; + CHECK(ArmSmmu4KBindTableMemory(&memory,&direct)==ArmSmmu4KBadRange && !direct.read_descriptor); + memory.pools[0].mapping=(const volatile uint64_t *)(UINTPTR_MAX-4095); + CHECK(ArmSmmu4KBindTableMemory(&memory,&direct)==ArmSmmu4KBadRange); + memory.pools[0].mapping=tables;memory.pools[0].physical=(UINT64_C(1)<<48)-4096; + CHECK(ArmSmmu4KBindTableMemory(&memory,&direct)==ArmSmmu4KBadRange); + printf("PASS: %u physical-graph profiles, %u descriptor mutations, %u read failures; upper final page, relocated tables, cycles, full-pool and cross-region isolation; synthetic physical memory only\n",profiles,mutations,failures); +} diff --git a/kits/ArmSmmu4K/test_context.c b/kits/ArmSmmu4K/test_context.c new file mode 100644 index 0000000..2fa8f37 --- /dev/null +++ b/kits/ArmSmmu4K/test_context.c @@ -0,0 +1,146 @@ +/* SPDX-License-Identifier: MIT */ +#include "ArmSmmu4KContext.h" +#include +#include +#include +#include +#define CHECK(c) do {if(!(c)){fprintf(stderr,"line %d: %s\n",__LINE__,#c);exit(1);}}while(0) +#define PAGES 32 +static uint64_t low[PAGES*512],high[PAGES*512],before[PAGES*512]; +/* Built separately from the pinned original arm-smmu.h inline helpers. */ +extern uint32_t source_tcr(unsigned,unsigned,unsigned); +extern uint32_t source_tcr2(unsigned); +static uint64_t walk(const uint64_t *tables,const ArmSmmu4KConfig *c, + const ArmSmmu4KResult *r,uint64_t va) +{ + unsigned level=4-(c->input_bits-12+8)/9;size_t page=0; + for(;level<=3;++level) { + unsigned shift=39-9*level; + unsigned bits=level==r->start_level?c->input_bits-shift:9; + uint64_t entry=tables[page*512+((va>>shift)&((1u<=r->root_physical && physical-r->root_physicaltable_pages*4096); + page=(physical-r->root_physical)/4096; + } + exit(1); +} +static void upper_tables(void) +{ + for(unsigned bits=32;bits<=48;++bits) { + ArmSmmu4KConfig c={bits,48,0x11000000,64}; + uint64_t base=UINT64_MAX-(UINT64_C(1)<>53)&3)==3 && ((entry>>2)&7)==ranges[i].writeback); + CHECK(((entry>>8)&3)==ranges[i].shareability && ((entry>>10)&3)==3); + CHECK(((entry>>6)&3)==(ranges[i].permissions==1?3:1));++leaves; + } + CHECK(leaves==r.leaf_pages && !walk(high,&c,&r,base+4096)); + CHECK(!memcmp(high+r.table_pages*512,before+r.table_pages*512,(PAGES-r.table_pages)*4096)); + memcpy(before,high,sizeof(high)); + ArmSmmu4KRange bad=ranges[count-1];bad.bytes=8192; + CHECK(ArmSmmu4KBuildUpper(&c,&bad,1,high,PAGES,&r)==ArmSmmu4KBadRange); + CHECK(!memcmp(high,before,sizeof(high)) && !r.table_pages); + bad=ranges[0];bad.iova=0; + CHECK(ArmSmmu4KBuildUpper(&c,&bad,1,high,PAGES,&r)==ArmSmmu4KBadRange); + bad.iova=base-4096; + CHECK(ArmSmmu4KBuildUpper(&c,&bad,1,high,PAGES,&r)==ArmSmmu4KBadRange); + bad=ranges[0]; + CHECK(ArmSmmu4KBuild(&c,&bad,1,high,PAGES,&r)==ArmSmmu4KBadRange); + CHECK(!memcmp(high,before,sizeof(high))); + } +} +static void refusal(ArmSmmu4KContextConfig c,ArmSmmu4KStatus status) +{ + ArmSmmu4KContextImage image;memset(&image,0xa5,sizeof(image)); + CHECK(ArmSmmu4KEncodeContext(&c,&image)==status); + unsigned char zero[sizeof(image)]={0};CHECK(!memcmp(&image,zero,sizeof(image))); +} +int main(void) +{ + upper_tables();unsigned widths[]={32,36,40,42,44,48};size_t images=0,mismatches=0,tbi_accepted=0; + for(unsigned input=32;input<=48;++input)for(unsigned output=0;output<6;++output) { + ArmSmmu4KConfig a={input,widths[output],0x10000000,16},b=a;b.table_physical=0x11000000; + ArmSmmu4KRange ra={0x1000,0x20000000,4096,3,1,3},rb=ra;rb.iova=UINT64_MAX-4095; + ArmSmmu4KResult ar,br; + CHECK(ArmSmmu4KBuild(&a,&ra,1,low,PAGES,&ar)==ArmSmmu4KSuccess); + CHECK(ArmSmmu4KBuildUpper(&b,&rb,1,high,PAGES,&br)==ArmSmmu4KSuccess); + for(unsigned mode=1;mode<=3;++mode)for(unsigned coherent=0;coherent<2;++coherent) + for(unsigned fault=0;fault<4;++fault)for(unsigned ai=0;ai<3;++ai) { + unsigned asids[]={0,1,65535}; + ArmSmmu4KContextConfig c={mode&1?&a:NULL,mode&2?&b:NULL, + mode&1?&ar:NULL,mode&2?&br:NULL,1u<<30,0x1555,asids[ai],coherent,fault&1,fault>>1}; + ArmSmmu4KContextImage v; + CHECK(ArmSmmu4KEncodeContext(&c,&v)==ArmSmmu4KSuccess); + uint32_t expected=UINT32_C(2)<<30; + if(mode&1)expected|=source_tcr(input,0,coherent); + if(mode&2)expected|=source_tcr(input,1,coherent); + if(mode==3)expected&=~((1u<<7)|(1u<<23)); + CHECK(v.tcr==expected && v.tcr2==(source_tcr2(output)|(1u<<4))); + CHECK(v.cbar==0x1f300 && v.cba2r==1 && v.mair0==0xff44 && !v.mair1 && !v.reserved_zero); + CHECK(v.sctlr==(0x1067|((fault&1)<<7)|((fault>>1)<<8))); + CHECK(v.ttbr0==((uint64_t)c.asid<<48|(mode&1?a.table_physical:0))); + CHECK(v.ttbr1==((uint64_t)c.asid<<48|(mode&2?b.table_physical:0))); + CHECK(ArmSmmu4KValidateContextReadback(&c,&v)==ArmSmmu4KSuccess);++images; + /* Every stored bit, including each granule, ASID, root address, + * enable/stage/format/cache/attribute field, must match. */ + /* Except TCR2.TBI0/TBI1 reading 1 where 0 was written (int-008 hardware), accepted alone or together, + * and SCTLR.ASIDPNE reading 0 where 1 was written (int-009 hardware). */ + for(size_t byte=0;byte32 || widths[output]>32)refusal(bad,ArmSmmu4KUnsupported); + uint64_t saved=br.root_physical;br.root_physical++;refusal(c,ArmSmmu4KArgument);br.root_physical=saved; + saved=br.mair;br.mair=0;refusal(c,ArmSmmu4KArgument);br.mair=saved; + unsigned level=br.start_level;br.start_level=3;refusal(c,ArmSmmu4KArgument);br.start_level=level; + br.upper_region=0;refusal(c,ArmSmmu4KArgument);br.upper_region=1; + saved=b.table_physical;b.table_physical=a.table_physical;br.root_physical=b.table_physical; + refusal(c,ArmSmmu4KOverlap);b.table_physical=saved;br.root_physical=saved; + } + ArmSmmu4KContextConfig none={0};refusal(none,ArmSmmu4KArgument); + CHECK(ArmSmmu4KEncodeContext(NULL,NULL)==ArmSmmu4KArgument); + CHECK(ArmSmmu4KValidateContextReadback(NULL,NULL)==ArmSmmu4KArgument); + printf("PASS: native4K TTBR1 high/partial-root/upper-bound tables; %zu context images versus pinned original helpers; %zu one-bit readback refusals; %zu TCR2.TBI/SCTLR.ASIDPNE readback tolerances\n",images,mismatches,tbi_accepted); +} diff --git a/kits/ArmSmmu4K/test_live.c b/kits/ArmSmmu4K/test_live.c new file mode 100644 index 0000000..5514b15 --- /dev/null +++ b/kits/ArmSmmu4K/test_live.c @@ -0,0 +1,362 @@ +/* SPDX-License-Identifier: MIT */ +/* Host memory and register fixtures only; no physical device is accessed. */ +#include "ArmSmmu4KLive.h" +#include +#include +#include +#define CHECK(x) do {if(!(x)){fprintf(stderr,"line %d: %s\n",__LINE__,#x);exit(1);}}while(0) +#define POOL_PAGES 16 +#define MMIO_BYTES 0x40000 +extern uint64_t source_permission_pte(unsigned,unsigned); +static _Alignas(4096) uint64_t tables[POOL_PAGES*512],snapshot[POOL_PAGES*512]; +static _Alignas(4096) uint64_t peer_tables[POOL_PAGES*512]; +static ArmSmmu4KLiveArena peer_arena; +static ArmSmmu4KLiveState peer_state; +static unsigned pair_mode; +typedef struct { + uint64_t registers[MMIO_BYTES/8]; + size_t calls,fail_call,publication_calls,fail_publication,tlbi_calls,sync_calls; + size_t writes,barriers,cb; + unsigned after_write,timeout,dirty,require_published,mutate_on_publish; + unsigned synced,clean_observed; +} Fixture; +static Fixture f; +static ArmSmmu4KLiveArena arena; +static ArmSmmu4KLiveState state; +static ArmSmmu4KContextConfig context; +static ArmSmmu4KIo io; +static ArmSmmu4KLiveVisibility visibility; +static ArmSmmu4KLiveResult result; +static uint32_t get32(size_t at) +{uint32_t value;memcpy(&value,(char *)f.registers+at,4);return value;} +static uint64_t get64(size_t at) +{uint64_t value;memcpy(&value,(char *)f.registers+at,8);return value;} +static void put32(size_t at,uint32_t value){memcpy((char *)f.registers+at,&value,4);} +static void put64(size_t at,uint64_t value){memcpy((char *)f.registers+at,&value,8);} +static int call(void *cookie,size_t at,size_t bytes) +{CHECK(cookie==&f && !(at&(bytes-1)) && at<=MMIO_BYTES-bytes);return ++f.calls==f.fail_call;} +static int read32(void *cookie,size_t at,uint32_t *value) +{ + if(call(cookie,at,4))return 1; + *value=at==f.cb+0x7f4 && f.timeout?1:get32(at); + if(at==f.cb+0x7f4 && !(*value&1))f.synced=1; + return 0; +} +static int read64(void *cookie,size_t at,uint64_t *value) +{if(call(cookie,at,8))return 1;*value=get64(at);return 0;} +static int write32(void *cookie,size_t at,uint32_t value) +{ + int fail=call(cookie,at,4);++f.writes; + if(fail && !f.after_write)return 1; + if(f.require_published) { + /* Live updates must not disable translation or change another bank, + * roots, granules, stream routes or interrupt/fault registers. */ + CHECK(at==f.cb+0x610 || at==f.cb+0x7f0); + CHECK(f.publication_calls && f.clean_observed && !f.dirty); + } + if(at==f.cb+0x610) {CHECK(value==context.asid);++f.tlbi_calls;f.synced=0;} + if(at==f.cb+0x7f0) {CHECK(f.tlbi_calls && value==UINT32_MAX);++f.sync_calls;} + if(at==f.cb+0x58)put32(at,get32(at)&~value); + else put32(at,value); + return fail; +} +static int write64(void *cookie,size_t at,uint64_t value) +{ + int fail=call(cookie,at,8);++f.writes; + CHECK(!f.require_published); + if(fail && !f.after_write)return 1;put64(at,value);return fail; +} +static int barrier(void *cookie) +{CHECK(cookie==&f);++f.barriers;return ++f.calls==f.fail_call;} +static int publish(void *cookie,volatile uint64_t *page,size_t bytes) +{ + CHECK(cookie==&f && bytes==4096 && !((uintptr_t)page&4095)); + uintptr_t at=(uintptr_t)page; + CHECK((at>=(uintptr_t)tables && at-(uintptr_t)tables=(uintptr_t)peer_tables && at-(uintptr_t)peer_tablestables.start_level; + for(;;) { + unsigned shift=12+9*(3-level),width=level==s->tables.start_level?a->config.input_bits-shift:9; + uint64_t entry=pool[at+((va>>shift)&((1u<=a->config.table_physical && physical-a->config.table_physicaltables.table_pages*4096); + at=(size_t)((physical-a->config.table_physical)/8);++level; + } +} +static uint64_t pte(uint64_t va) {return pte_from(&arena,&state,tables,va);} +static ArmSmmu4KStatus update(const ArmSmmu4KRange *before,size_t count,const ArmSmmu4KRange *change,unsigned map) +{return ArmSmmu4KLiveUpdate(&arena,&state,&io,0,&context,&visibility,before,count,change,map,4,&result);} +static void complete(unsigned removed,size_t pages) +{ + CHECK(result.complete && result.translation_removed==removed && !result.must_retain_resources); + CHECK(!state.poisoned && state.tables.leaf_pages==pages && result.synchronization.tlb_synchronized); + CHECK(f.tlbi_calls==1 && f.sync_calls==1 && f.synced && f.barriers>=4); + CHECK(result.descriptor_reads>=state.tables.table_pages*1024); +} +static unsigned profiles(void) +{ + unsigned count=0,outputs[]={32,36,40,42,44,48}; + for(unsigned bits=32;bits<=48;++bits)for(unsigned upper=0;upper<2;++upper) + for(unsigned output=0;output<6;++output)for(unsigned permissions=1;permissions<16;permissions+=2) + for(unsigned cache=0;cache<2;++cache) { + uint64_t boundary=bits>=40?UINT64_C(1)<<39:UINT64_C(1)<<30; + setup(bits,upper,outputs[output],cache,boundary-4096); + ArmSmmu4KRange r=resource(2,permissions,cache); + CHECK(update(0,0,&r,1)==ArmSmmu4KSuccess);complete(0,2); + CHECK(result.descriptor_writes==2 && result.published_pages==2 && state.generation==1); + for(unsigned i=0;i<2;++i)CHECK(pte(r.iova+4096*i)==((r.physical+4096*i)|source_permission_pte(permissions,cache)|3)); + CHECK(!pte(r.iova+8192)); + reset_counters();CHECK(update(&r,1,&r,0)==ArmSmmu4KSuccess);complete(1,0); + CHECK(!pte(r.iova) && !pte(r.iova+4096) && state.generation==2); + /* Reuse after a completed native invalidation, with different PA. */ + r.physical+=0x10000;reset_counters(); + CHECK(update(0,0,&r,1)==ArmSmmu4KSuccess);complete(0,2); + CHECK(pte(r.iova)==(r.physical|source_permission_pte(permissions,cache)|3)); + ++count; + } + return count; +} +static unsigned failures(void) +{ + setup(48,1,48,0,0x1ff000);ArmSmmu4KRange r=resource(2,3,1); + CHECK(update(0,0,&r,1)==ArmSmmu4KSuccess);size_t calls=f.calls; + unsigned count=0; + for(unsigned unmap=0;unmap<2;++unmap)for(unsigned after=0;after<2;++after) + for(size_t failed=1;failed<=calls;++failed) { + setup(48,1,48,0,0x1ff000);r=resource(2,3,1); + if(unmap) {CHECK(update(0,0,&r,1)==ArmSmmu4KSuccess);reset_counters();} + f.fail_call=failed;f.after_write=after; + CHECK(update(unmap?&r:0,unmap?1:0,&r,!unmap)==ArmSmmu4KIoError); + CHECK(state.poisoned && result.must_retain_resources && !result.complete && !result.translation_removed); + size_t old_calls=f.calls,old_writes=f.writes; + CHECK(update(unmap?&r:0,unmap?1:0,&r,!unmap)==ArmSmmu4KArgument); + CHECK(f.calls==old_calls && f.writes==old_writes);++count; + } + for(unsigned unmap=0;unmap<2;++unmap)for(unsigned fail=1;fail<=2;++fail) { + setup(48,1,48,0,0x1ff000);r=resource(2,3,1); + if(unmap) {CHECK(update(0,0,&r,1)==ArmSmmu4KSuccess);reset_counters();} + f.fail_publication=fail; + CHECK(update(unmap?&r:0,unmap?1:0,&r,!unmap)==ArmSmmu4KIoError); + CHECK(state.poisoned && result.must_retain_resources && !result.complete && !f.writes);++count; + } + for(unsigned unmap=0;unmap<2;++unmap) { + setup(48,1,48,0,0x1ff000);r=resource(2,3,1); + if(unmap) {CHECK(update(0,0,&r,1)==ArmSmmu4KSuccess);reset_counters();} + f.timeout=1; + CHECK(update(unmap?&r:0,unmap?1:0,&r,!unmap)==ArmSmmu4KTimeout); + CHECK(state.poisoned && !result.complete && result.must_retain_resources); + CHECK(result.synchronization.poll_reads==4 && !result.synchronization.tlb_synchronized);++count; + } + for(unsigned mutation=1;mutation<=3;++mutation) { + setup(48,1,48,0,0x1ff000);r=resource(2,3,1);f.mutate_on_publish=mutation; + ArmSmmu4KStatus status=update(0,0,&r,1); + CHECK(status==(mutation==3?ArmSmmu4KFault:ArmSmmu4KStateMismatch)); + CHECK(state.poisoned && result.must_retain_resources && !result.complete);++count; + } + return count; +} +static unsigned mutations(void) +{ + setup(48,1,48,0,0x1ff000);size_t slots=state.tables.table_pages*512; + memcpy(snapshot,tables,sizeof(tables));ArmSmmu4KRange r=resource(2,3,1); + unsigned count=0; + for(size_t index=0;index +#include +#define CHECK(x) do {if(!(x)){fprintf(stderr,"line %d: %s\n",__LINE__,#x);exit(1);}}while(0) +extern uint64_t source_permission_pte(unsigned,unsigned); +static uint64_t tables[8*512]; +static uint64_t walk(const ArmSmmu4KConfig *c,const ArmSmmu4KResult *r,uint64_t va) +{ + size_t page=0; + for(unsigned level=r->start_level;level<=3;++level) { + unsigned width=level==r->start_level?c->input_bits-12-9*(3-level):9; + size_t index=(va>>(12+9*(3-level)))&((1u<=r->root_physical);page=(physical-r->root_physical)/4096;CHECK(pagetable_pages); + } + CHECK(0);return 0; +} +int main(void) +{ + unsigned count=0; + for(unsigned bits=32;bits<=48;++bits)for(unsigned upper=0;upper<2;++upper) + for(unsigned permission=1;permission<16;permission+=2)for(unsigned cache=0;cache<2;++cache) { + ArmSmmu4KConfig c={bits,40,0x10000000,2};ArmSmmu4KResult r; + uint64_t iova=upper?UINT64_MAX-(UINT64_C(1)< +#include +#include +#define CHECK(c) do {if(!(c)){fprintf(stderr,"line %d: %s\n",__LINE__,#c);exit(1);}}while(0) +#define BYTES 0x40000 +#ifdef SOURCE_ORACLE +extern uint32_t source_register(unsigned); +static void source_definitions(void) +{ + const uint32_t expected[]={0x20,0x24,0x28,0,0x10,0x20,0x28,0x30,0x38,0x3c, + 0x58,0x610,0x7f0,0x7f4,0xc00001fe,0x800,UINT32_MAX}; + for(unsigned i=0;i>14&3)==0 && (r.readback.tcr>>30&3)==2); + CHECK(first(2,f.gr1+0x800+f.bank*4)=f.cb && atfirst(2,f.cb+0x7f0)); + } + successful_calls=f.calls; + /* Synchronize uses TLBIASID with the ASID; InvalidateAll uses TLBIALL; both sync and change nothing else. */ + f.event_count=f.calls=0; + CHECK(ArmSmmu4KSynchronizeOwnedContext(&io,f.bank,&config,4,&r)==ArmSmmu4KSuccess && r.tlb_synchronized); + CHECK(f.events[first(2,f.cb+0x610)].value==0xabcd && first(2,f.cb+0x610) +#include +#include +#define CHECK(x) do {if(!(x)){fprintf(stderr,"line %d: %s\n",__LINE__,#x);exit(1);}}while(0) +#define BYTES 0x40000 +#define CB 0x22000 +static uint32_t memory[BYTES/4]; +static struct {unsigned calls,fail_call,after,writes,events,corrupt,cleanup_fail;size_t at[1024];uint32_t value[1024];} f; +static ArmSmmu4KIo io; +static ArmSmmu4KStreamState state; +static ArmSmmu4KStream routes[2]={{2,0},{4,1}}; +static ArmSmmu4KStreamResult result; +static uint64_t tables[2048]; +static ArmSmmu4KConfig lower={32,40,0x10000000,1}; +static ArmSmmu4KResult built; +static ArmSmmu4KContextConfig config; +#ifdef SOURCE_ORACLE +extern void source_stream(unsigned,unsigned,unsigned,unsigned,unsigned,uint32_t *,uint32_t *); +extern uint32_t source_stream_register(unsigned); +#endif +static int event(void){return ++f.calls==f.fail_call;} +static int read32(void *cookie,size_t at,uint32_t *v) +{CHECK(cookie==&f && at%4==0 && at<=BYTES-4);if(event())return 1;*v=memory[at/4];return 0;} +static int read64(void *cookie,size_t at,uint64_t *v) +{CHECK(cookie==&f && at%8==0 && at<=BYTES-8);if(event())return 1;memcpy(v,memory+at/4,8);return 0;} +static int write32(void *cookie,size_t at,uint32_t v) +{ + CHECK(cookie==&f && at%4==0 && at<=BYTES-4 && f.events<1024); + ++f.writes;f.at[f.events]=at;f.value[f.events++]=v; + int fail=event();if(fail && !f.after)return 1; + if(f.cleanup_fail && v==0x20000)return 1; + memory[at/4]=v; + if(f.corrupt && at==0x808)memory[at/4]^=2; + return fail; +} +static int write64(void *cookie,size_t at,uint64_t v) +{(void)cookie;(void)at;(void)v;CHECK(0);return 1;} +static int barrier(void *cookie){CHECK(cookie==&f);return event();} +static void reset(unsigned exid) +{ + memset(memory,0,sizeof(memory));memset(&f,0,sizeof(f));state=(ArmSmmu4KStreamState){0}; + io=(ArmSmmu4KIo){&f,BYTES,read32,read64,write32,write64,barrier}; + memory[0x20/4]=(1u<<30)|(1u<<27)|(4u<<9)|8|(exid<<8); + memory[0x24/4]=0x40000010;memory[0x28/4]=0x1433;memory[0]=0x436|(exid<<3); + config=(ArmSmmu4KContextConfig){.lower=&lower,.lower_tables=&built,.asid=1,.coherent_walk=1,.continue_after_fault=1}; + ArmSmmu4KContextConfig actual=config;actual.id0=memory[0x20/4];actual.id2=memory[0x28/4]; + ArmSmmu4KContextImage image;CHECK(!ArmSmmu4KEncodeContext(&actual,&image)); + memory[0x1008/4]=image.cbar;memory[0x1808/4]=image.cba2r; + memory[CB/4]=image.sctlr;memory[(CB+0x10)/4]=image.tcr2;memory[(CB+0x30)/4]=image.tcr; + memcpy(memory+(CB+0x20)/4,&image.ttbr0,8);memcpy(memory+(CB+0x28)/4,&image.ttbr1,8); + memory[(CB+0x38)/4]=image.mair0; + for(unsigned i=0;i<8;++i)memory[(0xc00+4*i)/4]=0x20000; + /* An unrelated live route, never to be altered by attachment/recovery. */ + memory[0x800/4]=(exid?0:1u<<31)|9;memory[0xc00/4]=(exid?1u<<10:0)|3; +} +static ArmSmmu4KStatus attach(void) +{return ArmSmmu4KAttachStreams(&io,&state,routes,2,2,&config,&result);} +static void isolated(unsigned exid) +{ + CHECK(memory[0x800/4]==((exid?0:1u<<31)|9) && memory[0xc00/4]==((exid?1u<<10:0)|3)); + for(unsigned i=0;i=0xc00)CHECK(((f.value[i]>>16)&3)!=1); /* Never BYPASS. */ + } +} +int main(void) +{ + ArmSmmu4KRange range={0,0x20000000,4096,3,0,2}; + CHECK(!ArmSmmu4KBuild(&lower,&range,1,tables,4,&built)); +#ifdef SOURCE_ORACLE + const uint32_t offsets[]={0,0x20,0x48,0x808,0xc08,0x80000000,0x400,0x400}; + for(unsigned i=0;i +#include +#include +#define CHECK(c) do { if (!(c)) { fprintf(stderr,"line %d: %s\n",__LINE__,#c);exit(1); } } while (0) +#define CAPACITY 1024 +static uint64_t tables[CAPACITY * 512], before[CAPACITY * 512]; +static unsigned visited[CAPACITY]; +static uint64_t walk_at(const uint64_t *base, const ArmSmmu4KResult *result, uint64_t va) +{ + size_t page = 0; + for (unsigned level = result->start_level; level <= 3; ++level) { + uint64_t entry = base[page * 512 + ((va >> (39 - 9 * level)) & 511)]; + if (!entry) return 0; + CHECK((entry & 3) == 3); /* No block leaves anywhere. */ + if (level == 3) return entry; + uint64_t address = entry & UINT64_C(0xfffffffff000); + CHECK(address >= result->root_physical); + page = (address - result->root_physical) >> 12; + CHECK(page < result->table_pages); + } + exit(1); +} +static size_t graph(size_t page, unsigned level, const ArmSmmu4KResult *result, size_t *leaves) +{ + CHECK(page < result->table_pages && !visited[page]); + visited[page] = 1; + size_t count = 1; + for (size_t slot = 0; slot < 512; ++slot) { + uint64_t entry = tables[page * 512 + slot]; + if (!entry) continue; + CHECK((entry & 3) == 3); + if (level == 3) { ++*leaves; continue; } + CHECK(!(entry & ~UINT64_C(0xfffffffff003))); + CHECK((entry & UINT64_C(0xfffffffff000)) >= result->root_physical); + size_t child = ((entry & UINT64_C(0xfffffffff000)) - result->root_physical) >> 12; + count += graph(child, level + 1, result, leaves); + } + return count; +} +static void verify(const ArmSmmu4KConfig *config, const ArmSmmu4KRange *ranges, size_t n, const ArmSmmu4KResult *result) +{ + CHECK(result->root_physical == config->table_physical && result->mair == 0xff44 && !result->upper_region); + size_t leaves = 0; + memset(visited, 0, sizeof(visited)); + CHECK(graph(0, result->start_level, result, &leaves) == result->table_pages); + CHECK(leaves == result->leaf_pages); + size_t checked = 0; + for (size_t i = 0; i < n; ++i) { + for (uint64_t off = 0; off < ranges[i].bytes; off += 4096) { + uint64_t entry = walk_at(tables,result, ranges[i].iova + off); + CHECK((entry & UINT64_C(0xfffffffff000)) == ranges[i].physical + off); + CHECK(((entry >> 53) & 3) == (ranges[i].permissions & ArmSmmu4KExecute ? 0 : 3)); + CHECK(((entry >> 10) & 1) && ((entry >> 11) & 1)); + CHECK(((entry >> 6) & 1) == !(ranges[i].permissions & ArmSmmu4KPrivileged)); + CHECK(((entry >> 7) & 1) == !(ranges[i].permissions & ArmSmmu4KWrite)); + CHECK(((entry >> 2) & 7) == ranges[i].writeback); + CHECK(((entry >> 8) & 3) == ranges[i].shareability); + ++checked; + } + if (i + 1 < n && ranges[i].iova + ranges[i].bytes < ranges[i + 1].iova) + CHECK(!walk_at(tables,result, ranges[i].iova + ranges[i].bytes)); + } + CHECK(checked == result->leaf_pages); + /* Successful building touches only the actually used table pages. */ + CHECK(!memcmp(tables + result->table_pages * 512, before + result->table_pages * 512, + (CAPACITY - result->table_pages) * 4096)); +} +static void refusal(ArmSmmu4KConfig c, const ArmSmmu4KRange *r, size_t n, size_t capacity, ArmSmmu4KStatus status) +{ + ArmSmmu4KResult result; + memset(tables, 0xa5, sizeof(tables));memcpy(before,tables,sizeof(tables)); + memset(&result,0xa5,sizeof(result)); + CHECK(ArmSmmu4KBuild(&c,r,n,tables,capacity,&result) == status); + CHECK(!result.root_physical && !result.mair && !result.table_pages && !result.leaf_pages && !result.start_level && !result.upper_region); + CHECK(!memcmp(tables,before,sizeof(tables))); +} +static void success(ArmSmmu4KConfig c, const ArmSmmu4KRange *r, size_t n) +{ + ArmSmmu4KResult result; + memset(tables,0xa5,sizeof(tables));memcpy(before,tables,sizeof(tables)); + CHECK(ArmSmmu4KBuild(&c,r,n,tables,CAPACITY,&result) == ArmSmmu4KSuccess); + verify(&c,r,n,&result); +} +int main(void) +{ + ArmSmmu4KConfig c = {48,48,0x10000000,4096}; + ArmSmmu4KRange r[] = { + {0,0x20000000,0x3000,1,0,2}, {0x1ff000,0x21000000,0x3000,3,1,3}, + {0x3ffff000,0x22000000,0x3000,1,1,3}, {UINT64_C(0x7ffffff000),0x23000000,0x3000,3,0,2}, + {UINT64_C(0xfffffffff000),UINT64_C(0xfffffffff000),0x1000,1,0,2} + }; + success(c,r,5); + refusal(c,r,5,1,ArmSmmu4KCapacity); + c.leaf_budget = 12;refusal(c,r,5,CAPACITY,ArmSmmu4KBudget);c.leaf_budget=4096; + ArmSmmu4KRange bad[5];memcpy(bad,r,sizeof(r)); + bad[4].bytes=8192;refusal(c,bad,5,CAPACITY,ArmSmmu4KBadRange); + memcpy(bad,r,sizeof(r));bad[4].physical += 4096;refusal(c,bad,5,CAPACITY,ArmSmmu4KBadRange); + memcpy(bad,r,sizeof(r));bad[3].iova=0;refusal(c,bad,5,CAPACITY,ArmSmmu4KOverlap); + memcpy(bad,r,sizeof(r));bad[3].physical=c.table_physical;refusal(c,bad,5,CAPACITY,ArmSmmu4KOverlap); + memcpy(bad,r,sizeof(r));bad[3].physical=r[1].physical;refusal(c,bad,5,CAPACITY,ArmSmmu4KOverlap); + bad[3].writeback=1;bad[3].shareability=3;success(c,bad,5); /* Consistent physical aliases allowed. */ + memcpy(bad,r,sizeof(r));bad[3].permissions=2;refusal(c,bad,5,CAPACITY,ArmSmmu4KBadRange); + memcpy(bad,r,sizeof(r));bad[3].iova++;refusal(c,bad,5,CAPACITY,ArmSmmu4KBadRange); + memcpy(bad,r,sizeof(r));bad[3].writeback=2;refusal(c,bad,5,CAPACITY,ArmSmmu4KBadRange); + memcpy(bad,r,sizeof(r));bad[3].shareability=1;refusal(c,bad,5,CAPACITY,ArmSmmu4KBadRange); + memcpy(bad,r,sizeof(r));bad[3].physical=r[1].physical;bad[3].writeback=1; + refusal(c,bad,5,CAPACITY,ArmSmmu4KOverlap); /* Same MAIR, different SH. */ + c.table_physical++;refusal(c,r,5,CAPACITY,ArmSmmu4KBadRange);c.table_physical--; + c.input_bits=49;refusal(c,r,5,CAPACITY,ArmSmmu4KArgument);c.input_bits=48; + c.output_bits=52;refusal(c,r,5,CAPACITY,ArmSmmu4KArgument);c.output_bits=48; + ArmSmmu4KRange many[256]; + for (unsigned bits=32;bits<=48;++bits) { + c.input_bits=bits; + uint64_t limit=UINT64_C(1)<> 5) & 0x7ffff) << 2 | ((word >> 29) & 3), 21) + page = bool(word & 0x80000000) + return ((pc & ~4095) + (immediate << 12), True) if page else (pc + immediate, False) + if word & 0x7c000000 == 0x14000000: + return pc + (signed(word & 0x3ffffff, 26) << 2), False + if word & 0xff000010 == 0x54000000 or word & 0x7e000000 == 0x34000000 or word & 0x3b000000 == 0x18000000: + return pc + (signed((word >> 5) & 0x7ffff, 19) << 2), False + if word & 0x7e000000 == 0x36000000: + return pc + (signed((word >> 5) & 0x3fff, 14) << 2), False + return None + + +def instruction_inventory(original_data, parent, modules): + extents = [s for module in modules.values() for s in module['segments'] if s['size'] and s['name'] != '__LINKEDIT'] + literal_ranges, instruction_count = [], 0 + for identifier, module in modules.items(): + for segment in module['segments']: + if segment['name'] != '__TEXT_EXEC' or not segment['filesize']: + continue + at = backed_offset(parent, segment['va'], segment['filesize'], len(original_data)) + blob = original_data[at:at + segment['filesize']] + literals = [] + # Complete identity is required: a prefix alone never excludes code. + for prefix, size, digest, prefix_offset in ( + (bytes.fromhex('982f8a42'), 256, SHA256_K_DIGEST, 0), + (bytes.fromhex('0001000100010001'), 528, VECTOR_REPEAT_DIGEST, 64), + (bytes(range(32, 24, -1)), 40, ADLER32_VECTOR_DIGEST, 0), + *((bytes.fromhex(prefix), size, digest, 0) for prefix, size, digest in MATH_LITERAL_POOLS)): + search = 0 + while (search := blob.find(prefix, search)) >= 0: + start = search - prefix_offset + if start >= 0 and start % 4 == 0 and hashlib.sha256(blob[start:start + size]).hexdigest() == digest: + literals.append((start, start + size)) + literal_ranges.append({'module': identifier, 'va': segment['va'] + start, + 'size': size, 'sha256': digest}) + search += 4 + for offset in range(0, len(blob) - 3, 4): + if any(start <= offset < end for start, end in literals): + continue + word, = struct.unpack_from(' size: + raise ValueError('Coalesced segment must have native 4K alignment') + if any(s['size'] and va < s['va'] + s['size'] and s['va'] < va + size + for s in [*native['segments'], *new_segments]): + raise ValueError('Coalesced segment collides with existing mapping') + placeholder = next((s for s in native['segments'] if s['name'] == name), None) + if placeholder is None or placeholder['size'] or len(placeholder['sections']) != 1: + raise ValueError(f'Expected one empty native {name} segment/section') + fileoff = align(len(output)) + output.extend(bytes(fileoff - len(output))) + output.extend(blob) + output.extend(bytes(size - len(blob))) + command = placeholder['command_offset'] + struct.pack_into('<4Q2I', output, command + 24, va, size, fileoff, size, protection, protection) + section = placeholder['sections'][0]['command_offset'] + struct.pack_into(' len(blob): + raise ValueError('Component is outside original coalesced range') + fileoff = backed_offset(parent, segment['va'], segment['filesize'], len(original_data)) + blob[begin:begin + segment['filesize']] = original_data[fileoff:fileoff + segment['filesize']] + if coalesced_name == '__PRELINK_TEXT': + for identifier in report['kernel_interfaces']: + entry = entries[identifier] + va, length = entry['_PrelinkExecutableLoadAddr'], entry['_PrelinkExecutableSize'] + begin = va - source['va'] + at = backed_offset(parent, va, length, len(original_data)) + if begin < 0 or begin + length > len(blob): + raise ValueError('Kernel interface outside readonly prelink range') + blob[begin:begin + length] = original_data[at:at + length] + append_segment(coalesced_name, source['va'] + slide, blob, source['size'], source['initprot']) + + def new_offset(address, length): + return backed_offset(new_segments, address, length, len(output)) + + for binding in report['bindings']: + struct.pack_into('', b'/>') + b'\0' + info_va = align(max(s['va'] + s['size'] for s in native['segments'] if s['size'])) + append_segment('__PRELINK_INFO', info_va, xml, align(len(xml)), 3) + # Expand the existing command in verified Mach-O header padding; mapped + # payload offsets and all original kernel/component instruction bytes stay put. + commands = [] + for command, at, length in native['commands']: + blob = bytearray(output[at:at + length]) + if command == 0x19 and blob[8:24].rstrip(b'\0') == b'__PRELINK_TEXT': + struct.pack_into(' first_payload: + raise ValueError('Insufficient native Mach-O header padding for sparse index section') + struct.pack_into(' len(data): + raise ValueError('Mach-O header outside image') + magic, cpu, subtype, kind, count, size = struct.unpack_from('<6I', data, at) + if (magic, cpu, subtype) != (0xfeedfacf, 0x100000c, 0): + raise ValueError('Only ordinary little-endian ARM64 is supported') + end = at + 32 + size + if end > len(data): + raise ValueError('Load commands outside image') + offset = at + 32 + commands, segments = [], [] + for _ in range(count): + if offset + 8 > end: + raise ValueError('Truncated load command') + command, length = struct.unpack_from(' end: + raise ValueError('Invalid load command extent') + commands.append((command, offset, length)) + if command == 0x19: + if length < 72: + raise ValueError('Truncated segment') + name, va, size, fileoff, filesize, maxprot, initprot, nsects, flags = struct.unpack_from('<16s4Q4I', data, offset + 8) + if length != 72 + 80 * nsects or filesize > size or va + size > 1 << 64: + raise ValueError('Invalid segment extent') + segments.append({'name': name.rstrip(b'\0').decode(), 'va': va, 'size': size, + 'fileoff': fileoff, 'filesize': filesize, 'command_offset': offset, + 'maxprot': maxprot, 'initprot': initprot, 'sections': []}) + for index in range(nsects): + section_at = offset + 72 + index * 80 + section_name, _, address, extent, section_offset = struct.unpack_from('<16s16sQQI', data, section_at) + if extent and not va <= address <= address + extent <= va + size: + raise ValueError('Section outside segment') + segments[-1]['sections'].append({'name': section_name.rstrip(b'\0').decode(), + 'va': address, 'size': extent, 'fileoff': section_offset, 'command_offset': section_at}) + offset += length + if offset != end: + raise ValueError('Load command size mismatch') + return {'kind': kind, 'segments': segments, 'commands': commands} + + +def backed_offset(segments, address, length, data_length): + matches = [s['fileoff'] + address - s['va'] for s in segments + if s['va'] <= address and address + length <= s['va'] + s['filesize']] + if len(matches) != 1 or matches[0] + length > data_length: + raise ValueError(f'Ambiguous or unbacked address {address:#x}+{length:#x}') + return matches[0] + + +def exports(data, image): + result = {} + tables = [(offset, length) for command, offset, length in image['commands'] if command == 2] + if len(tables) != 1 or tables[0][1] != 24: + raise ValueError('Expected one symbol table') + symoff, count, stroff, strsize = struct.unpack_from('<4I', data, tables[0][0] + 8) + if symoff + count * 16 > len(data) or stroff + strsize > len(data): + raise ValueError('Symbol table outside image') + strings = data[stroff:stroff + strsize] + for index in range(count): + strx, kind, _, _, address = struct.unpack_from('= strsize or end < 0: + raise ValueError('Unterminated symbol name') + name = strings[strx:end].decode() + result.setdefault(address, []).append(name) + return result + + +def unserialize_xml(blob): + refs = {} + + def read(element): + if 'IDREF' in element.attrib: + return refs[element.attrib['IDREF']] + tag = element.tag + if tag == 'dict': + children = list(element) + if len(children) % 2: + raise ValueError('Odd XML dictionary') + value = {} + for index in range(0, len(children), 2): + key = read(children[index]) + if key in value: + raise ValueError('Duplicate XML dictionary key') + value[key] = read(children[index + 1]) + elif tag == 'array': + value = [read(child) for child in element] + elif tag in ('key', 'string'): + value = element.text or '' + elif tag == 'integer': + value = int(element.text or '0', 0) + elif tag == 'data': + value = base64.b64decode(''.join((element.text or '').split()), validate=True) + elif tag in ('true', 'false'): + value = tag == 'true' + else: + raise ValueError(f'Unsupported prelink XML element {tag}') + if 'ID' in element.attrib: + identifier = element.attrib['ID'] + if identifier in refs: + raise ValueError('Duplicate XML object ID') + refs[identifier] = value + return value + + return read(ET.fromstring(blob.split(b'\0', 1)[0])) + + +def unique_kernel_binding(address, original_exports, native_by_name): + names = original_exports.get(address, []) + choices = {new for name in names for new in native_by_name.get(name, set())} + if len(choices) != 1: + raise ValueError(f'Kernel target {address:#x} has {len(choices)} native address candidates: {names}') + return next(iter(choices)) + + +def plan(native_data, original_data, roots, slide): + if slide % 4096: + raise ValueError('Rebasing must preserve native 4K alignment') + native, original = macho(native_data), macho(original_data) + if native['kind'] != 2 or original['kind'] != 2: + raise ValueError('Expected MH_EXECUTE kernel images') + parent = original['segments'] + info = next(s for s in parent if s['name'] == '__PRELINK_INFO') + at = backed_offset(parent, info['va'], info['filesize'], len(original_data)) + metadata = unserialize_xml(original_data[at:at + info['filesize']]) + entries = {} + for entry in metadata['_PrelinkInfoDictionary']: + identifier = entry['CFBundleIdentifier'] + if identifier in entries: + raise ValueError('Duplicate prelink identifier') + entries[identifier] = entry + selected, interfaces = set(), set() + pending = list(roots) + while pending: + identifier = pending.pop() + if identifier in selected or identifier in interfaces: + continue + if identifier not in entries: + raise ValueError(f'Missing declared dependency {identifier}') + entry = entries[identifier] + if entry.get('OSKernelResource') is True: + interfaces.add(identifier) + continue + selected.add(identifier) + pending.extend(entry.get('OSBundleLibraries') or {}) + owned, modules = [], {} + for identifier, entry in entries.items(): + if entry.get('OSKernelResource') is True: + continue + va = entry.get('_PrelinkExecutableLoadAddr') + if va is None or not entry.get('_PrelinkExecutableSize'): + continue + header = backed_offset(parent, va, 32, len(original_data)) + module = macho(original_data, header) + if module['kind'] != 11: + raise ValueError('Expected MH_KEXT_BUNDLE component') + modules[identifier] = module + for segment in module['segments']: + if segment['size'] and segment['name'] != '__LINKEDIT': + owned.append((segment['va'], segment['va'] + segment['size'], identifier, segment)) + codeless = selected - modules.keys() + for identifier in codeless: + entry = entries[identifier] + executable_keys = ('CFBundleExecutable', '_PrelinkExecutableLoadAddr', + '_PrelinkExecutableSourceAddr', '_PrelinkExecutableSize', '_PrelinkKmodInfo') + if any(key in entry for key in executable_keys) or not isinstance(entry.get('IOKitPersonalities'), dict) or not entry['IOKitPersonalities']: + raise ValueError('Codeless component must contain personalities and no executable fields: ' + identifier) + owned.sort(key=lambda item: item[:3]) + for left, right in zip(owned, owned[1:]): + if left[1] > right[0]: + raise ValueError(f'Overlapping component extents: {left[2]} and {right[2]}') + starts = [item[0] for item in owned] + + def owner(address): + index = bisect_right(starts, address) - 1 + return owned[index] if index >= 0 and address < owned[index][1] else None + + native_symbols = {} + for address, names in exports(native_data, native).items(): + for name in names: + native_symbols.setdefault(name, set()).add(address) + original_symbols = exports(original_data, original) + packed = metadata['_PrelinkLinkKASLROffsets'] + count, = struct.unpack_from(' + #include ++ ++#if CONFIG_MACF ++extern "C" int mac_late; ++extern "C" __attribute__((used)) const uint32_t native_prelink_security_init_version = 1; ++#endif + #include + + #include +@@ -8446,6 +8451,43 @@ + result = OSRuntimeInitializeCPP(this); + if (result == KERN_SUCCESS) { + result = startfunc(kmod_info, kmodStartData); ++ } ++ ++ /* An explicitly linked security initializer must run with its real ++ * dependencies and constructors ready, before MAC policy lockdown. ++ * It executes original component code; no policy result is supplied. ++ */ ++ { ++ OSObject *earlyInitProperty = getPropertyForHostArch("_NativePrelinkEarlySecurityInit"); ++ if (result == KERN_SUCCESS && earlyInitProperty != NULL) { ++ #if CONFIG_MACF ++ OSNumber *earlyInit = OSDynamicCast(OSNumber, earlyInitProperty); ++ uintptr_t target = 0; ++ bool executable = false; ++ if (!earlyInit || !isPrelinked() || mac_late || ++ getPropertyForHostArch("AppleSecurityExtension") != kOSBooleanTrue || ++ __builtin_add_overflow((uintptr_t)earlyInit->unsigned64BitValue(), ++ vm_kernel_slide, &target) || (target & 3)) { ++ panic("NativePrelink: invalid early security initializer or phase"); ++ } ++ kernel_mach_header_t *header = (kernel_mach_header_t *)kmod_info->address; ++ for (kernel_segment_command_t *seg = firstsegfromheader(header); ++ seg != NULL; seg = nextsegfromheader(header, seg)) { ++ if (seg->initprot == (VM_PROT_READ | VM_PROT_EXECUTE) && ++ !(seg->maxprot & VM_PROT_WRITE) && seg->vmsize >= 4 && ++ target >= seg->vmaddr && target - seg->vmaddr <= seg->vmsize - 4) { ++ executable = true; ++ break; ++ } ++ } ++ if (!executable) { ++ panic("NativePrelink: early security initializer outside module code"); ++ } ++ ((void (*)(void))target)(); ++ #else ++ panic("NativePrelink: early security initializer requires MAC framework"); ++ #endif ++ } + } + + flags.starting = 0; diff --git a/kits/NativePrelink/patches/xnu-10063.141.1-sparse-index.patch b/kits/NativePrelink/patches/xnu-10063.141.1-sparse-index.patch new file mode 100644 index 0000000..3fe5270 --- /dev/null +++ b/kits/NativePrelink/patches/xnu-10063.141.1-sparse-index.patch @@ -0,0 +1,99 @@ +diff --git a/osfmk/kern/kalloc.c b/osfmk/kern/kalloc.c +index dbddc01..ada897e 100644 +--- a/osfmk/kern/kalloc.c ++++ b/osfmk/kern/kalloc.c +@@ -950,6 +950,8 @@ kalloc_type_view_copy( + } + } + ++__attribute__((used)) const uint32_t kalloc_sparse_prelink_index_version = 1; ++ + __startup_func + static uint64_t + kalloc_type_view_parse(const kalloc_type_variant_t type) +@@ -1032,10 +1034,46 @@ kalloc_type_view_parse(const kalloc_type_variant_t type) + cur = prelink_sect->addr; + end = prelink_sect->addr + prelink_sect->size; + +- while (cur < end) { ++ /* A sparse native prelink supplies its actual component headers. ++ * Keep the original contiguous KCGEN format when no index exists. ++ */ ++ kernel_section_t *macho_list = getsectbynamefromheader( ++ xnu_mh, kPrelinkTextSegment, "__macho_list"); ++ uint64_t list_index = 0, list_count = 0; ++ uintptr_t previous_header = 0; ++ if (macho_list) { ++ kernel_segment_command_t *prelink_segment = ++ getsegbynamefromheader(xnu_mh, kPrelinkTextSegment); ++ if (!macho_list->size || macho_list->size % sizeof(uintptr_t) || ++ macho_list->addr % sizeof(uintptr_t) || ++ macho_list->size > prelink_segment->vmsize || ++ macho_list->addr < prelink_segment->vmaddr || ++ macho_list->addr - prelink_segment->vmaddr > ++ prelink_segment->vmsize - macho_list->size) { ++ panic("kalloc_type_view_parse: invalid sparse prelink index"); ++ } ++ list_count = macho_list->size / sizeof(uintptr_t); ++ } ++ ++ while (macho_list ? list_index < list_count : cur < end) { ++ if (macho_list) { ++ uintptr_t unslid = ((const uintptr_t *)macho_list->addr)[list_index++]; ++ if ((list_index > 1 && unslid <= previous_header) || ++ __builtin_add_overflow(unslid, vm_kernel_slide, &cur) || ++ cur % sizeof(uintptr_t) || cur < prelink_sect->addr || ++ cur > end || end - cur < sizeof(kernel_mach_header_t)) { ++ panic("kalloc_type_view_parse: sparse header outside prelink text"); ++ } ++ previous_header = unslid; ++ } + uint64_t kext_text_sz = 0; + kernel_mach_header_t *kext_mh = (kernel_mach_header_t *) cur; + ++ if (macho_list && (kext_mh->magic != MH_MAGIC_64 || ++ kext_mh->sizeofcmds > end - cur - sizeof(*kext_mh) || ++ kext_mh->ncmds > kext_mh->sizeofcmds / sizeof(struct load_command))) { ++ panic("kalloc_type_view_parse: malformed sparse component header"); ++ } + if (kext_mh->magic == 0) { + /* + * Assert that we have processed all kexts and all that is left +@@ -1058,6 +1096,15 @@ kalloc_type_view_parse(const kalloc_type_variant_t type) + + for (uint32_t i = 0; i < kext_mh->ncmds && (vm_offset_t)lc < end; + i++, lc = (struct load_command *)((vm_offset_t)lc + lc->cmdsize)) { ++ if (macho_list) { ++ vm_offset_t header_end = cur + sizeof(*kext_mh) + kext_mh->sizeofcmds; ++ if ((vm_offset_t)lc > header_end || ++ header_end - (vm_offset_t)lc < sizeof(*lc) || ++ lc->cmdsize < sizeof(*lc) || lc->cmdsize % sizeof(uintptr_t) || ++ lc->cmdsize > header_end - (vm_offset_t)lc) { ++ panic("kalloc_type_view_parse: malformed sparse load command"); ++ } ++ } + if (lc->cmd == LC_SEGMENT_SPLIT_INFO) { + isSplitKext = true; + continue; +@@ -1067,6 +1114,10 @@ kalloc_type_view_parse(const kalloc_type_variant_t type) + + kernel_segment_command_t *seg_cmd = + (struct segment_command_64 *)(vm_offset_t)lc; ++ if (macho_list && (lc->cmdsize < sizeof(*seg_cmd) || ++ seg_cmd->nsects > (lc->cmdsize - sizeof(*seg_cmd)) / sizeof(kernel_section_t))) { ++ panic("kalloc_type_view_parse: malformed sparse segment"); ++ } + /* + * Parse kalloc_type section + */ +@@ -1100,7 +1151,9 @@ kalloc_type_view_parse(const kalloc_type_variant_t type) + break; + } + +- cur += ((kext_text_sz + (KEXT_ALIGN_BYTES - 1)) & (~KEXT_ALIGN_MASK)); ++ if (!macho_list) { ++ cur += ((kext_text_sz + (KEXT_ALIGN_BYTES - 1)) & (~KEXT_ALIGN_MASK)); ++ } + } + } else { + /* diff --git a/kits/NativePrelink/test_codeless.py b/kits/NativePrelink/test_codeless.py new file mode 100644 index 0000000..d38c5fb --- /dev/null +++ b/kits/NativePrelink/test_codeless.py @@ -0,0 +1,53 @@ +"""Preserve genuine codeless matching metadata without pretending it is code.""" +import copy +import unittest +import xml.etree.ElementTree as ET +from native_prelink import macho,unserialize_xml,plan +from emit_prelink import emit +from test_emit_prelink import images +from test_native_prelink import xml_value + +class CodelessTests(unittest.TestCase): + def fixture(self, change=None): + native,original=images();raw=bytearray(original) + info=next(s for s in macho(raw)['segments'] if s['name']=='__PRELINK_INFO') + at,size=info['fileoff'],info['filesize'] + metadata=unserialize_xml(raw[at:at+size]) + entry={'CFBundleIdentifier':'test.matching','OSBundleLibraries':{'test.component':'1'}, + 'IOKitPersonalities':{'Actual Match':{'CFBundleIdentifier':'test.component', + 'IOClass':'TestProvider','IOProviderClass':'IOResources','IOResourceMatch':'IOBSD'}}} + if change:change(entry) + metadata['_PrelinkInfoDictionary'].append(entry) + data=ET.tostring(xml_value(metadata))+b'\0';self.assertLessEqual(len(data),size) + raw[at:at+size]=data+bytes(size-len(data)) + return native,bytes(raw),entry + + def test_preserve_metadata_and_dependency_code(self): + native,original,entry=self.fixture() + output,report=emit(native,original,['test.matching'],0x4000) + self.assertEqual(report['components'],['test.component']) + self.assertEqual(report['codeless_components'],['test.matching']) + self.assertNotIn('test.matching',report['module_entrypoints']) + info=next(s for s in macho(output)['segments'] if s['name']=='__PRELINK_INFO') + metadata=unserialize_xml(output[info['fileoff']:info['fileoff']+info['filesize']]) + actual=next(e for e in metadata['_PrelinkInfoDictionary'] if e['CFBundleIdentifier']=='test.matching') + self.assertEqual(actual,entry) + self.assertIn('test.component',report['original_executable_sha256']) + + def test_refuse_disguised_or_empty_component(self): + changes=[lambda e:e.pop('IOKitPersonalities'),lambda e:e.update(IOKitPersonalities={}), + lambda e:e.update(IOKitPersonalities=[])] + for key in ('CFBundleExecutable','_PrelinkExecutableLoadAddr','_PrelinkExecutableSourceAddr', + '_PrelinkExecutableSize','_PrelinkKmodInfo'): + changes.append(lambda e,key=key:e.update({key:0})) + for change in changes: + native,original,_=self.fixture(change) + with self.assertRaisesRegex(ValueError,'Codeless component'): + plan(native,original,['test.matching'],0x4000) + + def test_refuse_missing_declared_dependency(self): + native,original,_=self.fixture(lambda e:e['OSBundleLibraries'].update({'absent':'1'})) + with self.assertRaisesRegex(ValueError,'Missing declared dependency'): + plan(native,original,['test.matching'],0x4000) + +if __name__=='__main__':unittest.main() diff --git a/kits/NativePrelink/test_emit_prelink.py b/kits/NativePrelink/test_emit_prelink.py new file mode 100644 index 0000000..4d0c694 --- /dev/null +++ b/kits/NativePrelink/test_emit_prelink.py @@ -0,0 +1,207 @@ +#!/usr/bin/env python3 +"""Exercise candidate emission using wholly synthetic component images.""" +import hashlib +import struct +import unittest +import xml.etree.ElementTree as ET + +from emit_prelink import ADLER32_VECTOR_DIGEST, VECTOR_REPEAT_DIGEST, emit +from native_prelink import backed_offset, macho, unserialize_xml +from test_native_prelink import header, segment, xml_value + + +def section_segment(name, va, size, off, length, section_name, section_va, section_size, section_off): + command = bytearray(segment(name, va, size, off, length)) + struct.pack_into(' & fixture', 'TestFlag': True, 'EmptyArray': [], + 'AppleSecurityExtension': True}], + '_PrelinkLinkKASLROffsets': struct.pack('<5I', 4, 0x2000, 0x2008, 0x30b4, 0x30bc)} + blob = ET.tostring(xml_value(metadata)) + b'\0'; original[0x5000:0x5000 + len(blob)] = blob + return bytes(native), bytes(original) + + +class EmitTests(unittest.TestCase): + def test_explicit_early_security_initializer_metadata(self): + native, original = images() + output, report = emit(native, original, ['test.component'], 0x4000, {'test.component': 0x3000}) + self.assertEqual(report['early_security_initializers'], {'test.component': 0x7000}) + info = next(s for s in macho(output)['segments'] if s['name'] == '__PRELINK_INFO') + entry = unserialize_xml(output[info['fileoff']:info['fileoff'] + info['filesize']])['_PrelinkInfoDictionary'][0] + self.assertEqual(entry['_NativePrelinkEarlySecurityInit'], 0x7000) + at = backed_offset(macho(output)['segments'], 0x7000, 0x1000, len(output)) + self.assertEqual(output[at:at + 0x1000], original[0x2000:0x3000]) + + def test_refuse_early_security_initializer_without_source_abi(self): + native, original = images() + native = native.replace(b'_native_prelink_security_init_version', b'_absent_prelink_security_init_version') + with self.assertRaisesRegex(ValueError, 'lacks early security initializer ABI'): + emit(native, original, ['test.component'], 0x4000, {'test.component': 0x3000}) + native, original = images(); native = bytearray(native) + struct.pack_into('', xml) + self.assertIn(b'', xml) + metadata = unserialize_xml(xml) + entry = metadata['_PrelinkInfoDictionary'][0] + self.assertEqual((entry['_PrelinkExecutableLoadAddr'], entry['_PrelinkKmodInfo']), (0x6000, 0x9000)) + self.assertFalse(report['runtime_qualified']) + self.assertEqual(report['sparse_index']['headers'], [0x6000]) + text = next(s for s in segments if s['name'] == '__PRELINK_TEXT') + index = next(s for s in text['sections'] if s['name'] == '__macho_list') + self.assertEqual(struct.unpack_from('same'), ['same', 'same']) + with self.assertRaisesRegex(ValueError, 'Duplicate XML dictionary'): + unserialize_xml(b'xx') + + +if __name__ == '__main__': + unittest.main() diff --git a/kits/VerifiedRoot/LICENSE b/kits/VerifiedRoot/LICENSE new file mode 100644 index 0000000..f3bdfd0 --- /dev/null +++ b/kits/VerifiedRoot/LICENSE @@ -0,0 +1,51 @@ +Copyright (c) 2026, VerifiedRoot contributors. All rights reserved. + +SPDX-License-Identifier: BSD-2-Clause-Patent + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +Subject to the terms and conditions of this license, each copyright holder +and contributor hereby grants to those receiving rights under this license +a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable +(except for failure to satisfy the conditions of this license) patent +license to make, have made, use, offer to sell, sell, import, and otherwise +transfer this software, where such license applies only to those patent +claims, already acquired or hereafter acquired, licensable by such copyright +holder or contributor that are necessarily infringed by: + +(a) their Contribution(s) (the licensed copyrights of copyright holders and + non-copyrightable additions of contributors, in source or binary form) + alone; or + +(b) combination of their Contribution(s) with the work of authorship to + which such Contribution(s) was added by such copyright holder or + contributor, if, at the time the Contribution is added, such addition + causes such combination to be necessarily infringed. The patent license + shall not apply to any other combinations which include the + Contribution. + +Except as expressly stated above, no rights or licenses from any copyright +holder or contributor is granted under this license, whether expressly, by +implication, estoppel or otherwise. + +DISCLAIMER + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. diff --git a/kits/VerifiedRoot/adapters/xnu_crypto.c b/kits/VerifiedRoot/adapters/xnu_crypto.c new file mode 100644 index 0000000..eb8facb --- /dev/null +++ b/kits/VerifiedRoot/adapters/xnu_crypto.c @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: BSD-2-Clause-Patent */ +#include "verified_root.h" +#include +#include +#include + +static int hash(const void *data, size_t n, uint8_t out[32]) { + SHA256_CTX context; + SHA256_Init(&context); + SHA256_Update(&context, data, n); + SHA256_Final(out, &context); + return 1; +} +static int verify(const uint8_t key[VR_KEY_BYTES], const uint8_t digest[32], const uint8_t signature[VR_SIGNATURE_BYTES]) { + rsa_pub_ctx public_key = {0}; + bool valid = false; + int error = rsa_make_pub(&public_key, 3, key + 392, 384, key + 8); + if (error) return 0; + error = rsa_verify_pkcs1v15(&public_key, ccoid_sha256, 32, digest, + VR_SIGNATURE_BYTES, signature, &valid); + return error == 0 && valid; +} +const struct vr_crypto vr_xnu_crypto = {hash, verify}; diff --git a/kits/VerifiedRoot/include/verified_root.h b/kits/VerifiedRoot/include/verified_root.h new file mode 100644 index 0000000..66c3959 --- /dev/null +++ b/kits/VerifiedRoot/include/verified_root.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: BSD-2-Clause-Patent */ +#ifndef VERIFIED_ROOT_H +#define VERIFIED_ROOT_H +#include +#include +#ifdef __cplusplus +extern "C" { +#endif +#define VR_HEADER_BYTES 160u +#define VR_SIGNATURE_BYTES 384u +#define VR_KEY_BYTES 395u +#define VR_BLOCK_BYTES 4096u + +enum vr_status { VR_OK, VR_FORMAT, VR_KEY, VR_SIGNATURE, VR_HASH_TABLE, VR_BLOCK }; +struct vr_crypto { + int (*sha256)(const void *, size_t, uint8_t[32]); + int (*rsa3072_sha256)(const uint8_t[VR_KEY_BYTES], const uint8_t[32], const uint8_t[VR_SIGNATURE_BYTES]); +}; +struct vr_view { + const uint8_t *hashes; + uint64_t image_bytes, block_count, generation; + uint8_t volume_uuid[16], image_sha256[32]; +}; +/* Trusted key is supplied independently of the untrusted envelope. Both buffers + * must remain immutable for the lifetime of the returned authenticated view. */ +int vr_open(const uint8_t *, size_t, const uint8_t[VR_KEY_BYTES], uint64_t, + const struct vr_crypto *, struct vr_view *); +int vr_verify_block(const struct vr_view *, uint64_t, const void *, size_t, + const struct vr_crypto *); +#ifdef __cplusplus +} +#endif +#endif diff --git a/kits/VerifiedRoot/src/verified_root.c b/kits/VerifiedRoot/src/verified_root.c new file mode 100644 index 0000000..55c3e5a --- /dev/null +++ b/kits/VerifiedRoot/src/verified_root.c @@ -0,0 +1,56 @@ +/* SPDX-License-Identifier: BSD-2-Clause-Patent */ +#include "verified_root.h" + +static uint64_t le(const uint8_t *p, unsigned n) { + uint64_t v = 0; + for (unsigned i = 0; i < n; ++i) v |= (uint64_t)p[i] << (8 * i); + return v; +} +static int same(const uint8_t *a, const uint8_t *b, size_t n) { + uint8_t diff = 0; + for (size_t i = 0; i < n; ++i) diff |= a[i] ^ b[i]; + return diff == 0; +} +int vr_open(const uint8_t *p, size_t n, const uint8_t key[VR_KEY_BYTES], uint64_t minimum_generation, + const struct vr_crypto *crypto, struct vr_view *out) { + static const uint8_t magic[8] = {'V','R','O','O','T','0','1',0}; + static const uint8_t key_magic[8] = {'V','R','O','O','T','K','E','Y'}; + uint8_t digest[32]; + uint64_t bytes, count, generation; + if (!out) return VR_FORMAT; + *out = (struct vr_view){0}; + if (!p || !key || !crypto || !crypto->sha256 || !crypto->rsa3072_sha256 || + n < VR_HEADER_BYTES + VR_SIGNATURE_BYTES || !same(p, magic, 8) || + le(p + 8, 4) != 1 || le(p + 12, 4) != VR_HEADER_BYTES || + le(p + 16, 4) != VR_BLOCK_BYTES || le(p + 20, 4) != 0 || !minimum_generation) return VR_FORMAT; + bytes = le(p + 24, 8); + count = le(p + 32, 8); + generation = le(p + 152, 8); + if (!bytes || count != bytes / VR_BLOCK_BYTES + (bytes % VR_BLOCK_BYTES != 0) || + count > (SIZE_MAX - VR_HEADER_BYTES - VR_SIGNATURE_BYTES) / 32 || + n != VR_HEADER_BYTES + VR_SIGNATURE_BYTES + (size_t)count * 32 || generation < minimum_generation) return VR_FORMAT; + if (!same(key, key_magic, 8) || !(key[8] & 0x80) || !(key[391] & 1) || + key[392] != 1 || key[393] != 0 || key[394] != 1 || + !crypto->sha256(key, VR_KEY_BYTES, digest) || !same(digest, p + 120, 32)) return VR_KEY; + if (!crypto->sha256(p, VR_HEADER_BYTES, digest) || + !crypto->rsa3072_sha256(key, digest, p + VR_HEADER_BYTES)) return VR_SIGNATURE; + const uint8_t *hashes = p + VR_HEADER_BYTES + VR_SIGNATURE_BYTES; + if (!crypto->sha256(hashes, (size_t)count * 32, digest) || !same(digest, p + 88, 32)) return VR_HASH_TABLE; + out->hashes = hashes; + out->image_bytes = bytes; + out->block_count = count; + out->generation = generation; + for (unsigned i = 0; i < 16; ++i) out->volume_uuid[i] = p[40 + i]; + for (unsigned i = 0; i < 32; ++i) out->image_sha256[i] = p[56 + i]; + return VR_OK; +} +int vr_verify_block(const struct vr_view *view, uint64_t index, const void *data, size_t n, + const struct vr_crypto *crypto) { + uint8_t digest[32]; + if (!view || !view->hashes || !data || !crypto || !crypto->sha256 || index >= view->block_count) return VR_BLOCK; + uint64_t remaining = view->image_bytes - index * VR_BLOCK_BYTES; + size_t expected = remaining < VR_BLOCK_BYTES ? (size_t)remaining : VR_BLOCK_BYTES; + if (n != expected || !crypto->sha256(data, n, digest) || + !same(digest, view->hashes + index * 32, 32)) return VR_BLOCK; + return VR_OK; +} diff --git a/kits/VerifiedRoot/tests/check.py b/kits/VerifiedRoot/tests/check.py new file mode 100644 index 0000000..275d81a --- /dev/null +++ b/kits/VerifiedRoot/tests/check.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +"""Exercise actual C parsing, real RSA verification and block integrity.""" +import argparse +import ctypes +import importlib.util +import io +import json +import os +from pathlib import Path +import subprocess +import uuid + +ROOT = Path(__file__).resolve().parents[1] +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument('--output', type=Path, required=True) +parser.add_argument('--openssl-prefix', type=Path, required=True) +args = parser.parse_args() +out = args.output.resolve() +out.mkdir(parents=True, exist_ok=True) +keys = out / 'keys' +keys.mkdir(mode=0o700, exist_ok=True) +os.chmod(keys, 0o700) +for name in ('owner', 'other'): + private, public = keys / (name + '.pem'), keys / (name + '-public.pem') + if not private.exists(): + fd = os.open(private, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600) + os.close(fd) + subprocess.run(['openssl', 'genpkey', '-algorithm', 'RSA', '-pkeyopt', 'rsa_keygen_bits:3072', + '-out', private], check=True, capture_output=True) + os.chmod(private, 0o600) + subprocess.run(['openssl', 'pkey', '-in', private, '-pubout', '-out', public], check=True, capture_output=True) +spec = importlib.util.spec_from_file_location('manifest', ROOT / 'tools/manifest.py') +manifest = importlib.util.module_from_spec(spec) +spec.loader.exec_module(manifest) +data = bytes((i * 23 + i // 4096) & 255 for i in range(4096 * 3 + 512)) +record = manifest.make(io.BytesIO(data), len(data), keys / 'owner.pem', keys / 'owner-public.pem', + uuid.UUID('01e286f1-f4dd-4b2f-99d9-4973337fc788'), 7, out / 'signed') +library = out / 'verified-root.dylib' +subprocess.run(['xcrun', 'clang', '-std=c11', '-Wall', '-Wextra', '-Werror', '-dynamiclib', + '-I', ROOT / 'include', '-I', args.openssl_prefix / 'include', ROOT / 'src/verified_root.c', + ROOT / 'tests/openssl_provider.c', '-L', args.openssl_prefix / 'lib', '-lcrypto', '-o', library], check=True) +sdk = subprocess.check_output(['xcrun', '--sdk', 'iphoneos', '--show-sdk-path'], text=True).strip() +subprocess.run(['xcrun', 'clang', '-target', 'arm64-apple-ios17.0', '-isysroot', sdk, + '-std=c11', '-ffreestanding', '-Wall', '-Wextra', '-Werror', '-I', ROOT / 'include', + '-c', ROOT / 'src/verified_root.c', '-o', out / 'verified-root-arm64.o'], check=True) + + +class View(ctypes.Structure): + _fields_ = [('hashes', ctypes.c_void_p), ('image_bytes', ctypes.c_uint64), ('block_count', ctypes.c_uint64), + ('generation', ctypes.c_uint64), ('volume_uuid', ctypes.c_uint8 * 16), ('image_sha256', ctypes.c_uint8 * 32)] + + +native = ctypes.CDLL(str(library)) +native.vr_host_open.argtypes = [ctypes.c_void_p, ctypes.c_size_t, ctypes.c_void_p, ctypes.c_uint64, ctypes.POINTER(View)] +native.vr_host_open.restype = ctypes.c_int +native.vr_host_block.argtypes = [ctypes.POINTER(View), ctypes.c_uint64, ctypes.c_void_p, ctypes.c_size_t] +native.vr_host_block.restype = ctypes.c_int +envelope = (out / 'signed/manifest.bin').read_bytes() +key = (out / 'signed/trusted-key.bin').read_bytes() +other_key = manifest.public_anchor(keys / 'other-public.pem') +assert len(key) == 395 +buffers = [] +checks = [] + + +def opened(blob, anchor=key, generation=7, expected=0): + encoded, trusted, view = ctypes.create_string_buffer(blob), ctypes.create_string_buffer(anchor), View() + result = native.vr_host_open(encoded, len(blob), trusted, generation, ctypes.byref(view)) + assert result == expected, (result, expected) + if result: + assert not view.hashes and view.image_bytes == 0 + buffers.extend((encoded, trusted)) + return view + + +view = opened(envelope) +assert view.image_bytes == len(data) and view.block_count == 4 and view.generation == 7 +assert bytes(view.volume_uuid) == uuid.UUID(record['volume_uuid']).bytes +checks.append('real RSA3072-SHA256 verification and signed metadata') +for index in range(4): + block = data[index * 4096:(index + 1) * 4096] + assert native.vr_host_block(ctypes.byref(view), index, block, len(block)) == 0 +checks.append('all original blocks including the exact final 512-byte tail') +for index in range(4): + block = bytearray(data[index * 4096:(index + 1) * 4096]) + block[0] ^= 1 + assert native.vr_host_block(ctypes.byref(view), index, bytes(block), len(block)) == 5 +checks.append('one-byte mutation refused in every actual block') +assert native.vr_host_block(ctypes.byref(view), 1, data[:4096], 4096) == 5 +assert native.vr_host_block(ctypes.byref(view), 4, data[:4096], 4096) == 5 +assert native.vr_host_block(ctypes.byref(view), 0, data[:512], 512) == 5 +checks.append('block substitution, outside index and partial-block reads refused') +for offset, expected in ((40, 3), (160, 3), (544, 4)): + changed = bytearray(envelope) + changed[offset] ^= 1 + opened(bytes(changed), expected=expected) +checks.append('signed UUID/header, RSA signature and hash-table mutation refused') +opened(envelope, anchor=other_key, expected=2) +opened(envelope, generation=8, expected=1) +opened(envelope, generation=0, expected=1) +opened(envelope[:-1], expected=1) +opened(envelope + b'\0', expected=1) +checks.append('wrong trusted anchor, rollback generation, truncation and trailing bytes refused') +changed = bytearray(envelope) +changed[120:152] = __import__('hashlib').sha256(other_key).digest() +opened(bytes(changed), anchor=other_key, expected=3) +checks.append('matching attacker-selected key identifier cannot replace the actual RSA signature') +report = dict(real_c_parser=True, real_openssl_signature_verification=True, native_arm64_ios17_cross_compile=True, + checks=checks, qemu_tested=False, boot_root_authenticated=False, hardware_tested=False) +(out / 'checks.json').write_text(json.dumps(report, indent=2) + '\n') +print('PASS:', len(checks), 'groups using real RSA signatures and actual C block verification; host only') diff --git a/kits/VerifiedRoot/tests/openssl_provider.c b/kits/VerifiedRoot/tests/openssl_provider.c new file mode 100644 index 0000000..447ec7f --- /dev/null +++ b/kits/VerifiedRoot/tests/openssl_provider.c @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: BSD-2-Clause-Patent */ +#include "verified_root.h" +#include +#include +#include +#include + +static int hash(const void *data, size_t n, uint8_t out[32]) { + unsigned size = 0; + return EVP_Digest(data, n, out, &size, EVP_sha256(), NULL) == 1 && size == 32; +} +static int verify(const uint8_t key[VR_KEY_BYTES], const uint8_t digest[32], const uint8_t sig[VR_SIGNATURE_BYTES]) { + int ok = 0; + BIGNUM *modulus = BN_bin2bn(key + 8, 384, NULL); + BIGNUM *exponent = BN_bin2bn(key + 392, 3, NULL); + OSSL_PARAM_BLD *builder = OSSL_PARAM_BLD_new(); + OSSL_PARAM *params = NULL; + EVP_PKEY_CTX *make = NULL, *check = NULL; + EVP_PKEY *public_key = NULL; + if (!modulus || !exponent || !builder || + OSSL_PARAM_BLD_push_BN(builder, OSSL_PKEY_PARAM_RSA_N, modulus) != 1 || + OSSL_PARAM_BLD_push_BN(builder, OSSL_PKEY_PARAM_RSA_E, exponent) != 1) goto done; + params = OSSL_PARAM_BLD_to_param(builder); + make = EVP_PKEY_CTX_new_from_name(NULL, "RSA", NULL); + if (!params || !make || EVP_PKEY_fromdata_init(make) != 1 || + EVP_PKEY_fromdata(make, &public_key, EVP_PKEY_PUBLIC_KEY, params) != 1) goto done; + check = EVP_PKEY_CTX_new(public_key, NULL); + if (!check || EVP_PKEY_verify_init(check) != 1 || + EVP_PKEY_CTX_set_rsa_padding(check, RSA_PKCS1_PADDING) <= 0 || + EVP_PKEY_CTX_set_signature_md(check, EVP_sha256()) <= 0) goto done; + ok = EVP_PKEY_verify(check, sig, VR_SIGNATURE_BYTES, digest, 32) == 1; +done: + EVP_PKEY_CTX_free(check); + EVP_PKEY_free(public_key); + EVP_PKEY_CTX_free(make); + OSSL_PARAM_free(params); + OSSL_PARAM_BLD_free(builder); + BN_free(exponent); + BN_free(modulus); + return ok; +} +static const struct vr_crypto provider = {hash, verify}; +int vr_host_open(const uint8_t *p, size_t n, const uint8_t *key, uint64_t generation, struct vr_view *view) { + return vr_open(p, n, key, generation, &provider, view); +} +int vr_host_block(const struct vr_view *view, uint64_t index, const void *data, size_t n) { + return vr_verify_block(view, index, data, n, &provider); +} diff --git a/kits/VerifiedRoot/tools/manifest.py b/kits/VerifiedRoot/tools/manifest.py new file mode 100644 index 0000000..da122d2 --- /dev/null +++ b/kits/VerifiedRoot/tools/manifest.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +"""Sign a complete original image's ordered block hashes with an owner key.""" +import argparse +import hashlib +import json +from pathlib import Path +import re +import struct +import subprocess +import uuid + + +def public_anchor(public_key): + text = subprocess.check_output(['openssl', 'rsa', '-pubin', '-in', str(public_key), '-text', '-noout'], text=True) + if not re.search(r'(?:Exponent|exponent): 65537\b', text): + raise ValueError('Require exponent 65537') + line = subprocess.check_output(['openssl', 'rsa', '-pubin', '-in', str(public_key), '-modulus', '-noout'], text=True) + modulus = bytes.fromhex(line.strip().split('=', 1)[1]) + if len(modulus) != 384 or not modulus[0] & 128 or not modulus[-1] & 1: + raise ValueError('Require an actual 3072-bit RSA public key') + return b'VROOTKEY' + modulus + b'\x01\x00\x01' + + +def make(stream, image_bytes, key, public_key, volume_uuid, generation, output): + if image_bytes <= 0 or generation <= 0: + raise ValueError('Require a nonempty image and a positive generation') + output.mkdir(parents=True, exist_ok=True) + anchor = public_anchor(public_key) + table = output / 'hashes.bin' + image_hash, table_hash = hashlib.sha256(), hashlib.sha256() + count = 0 + with table.open('wb') as hashes: + remaining = image_bytes + while remaining: + data = stream.read(min(4096, remaining)) + if len(data) != min(4096, remaining): + raise ValueError('Short image stream') + digest = hashlib.sha256(data).digest() + image_hash.update(data) + table_hash.update(digest) + hashes.write(digest) + remaining -= len(data) + count += 1 + if stream.read(1): + raise ValueError('Image stream exceeds its actual declared size') + header = struct.pack('<8s4I2Q16s32s32s32sQ', b'VROOT01\0', 1, 160, 4096, 0, + image_bytes, count, uuid.UUID(str(volume_uuid)).bytes, + image_hash.digest(), table_hash.digest(), hashlib.sha256(anchor).digest(), generation) + (output / 'header.bin').write_bytes(header) + (output / 'trusted-key.bin').write_bytes(anchor) + signature = output / 'signature.bin' + subprocess.run(['openssl', 'dgst', '-sha256', '-sign', str(key), '-out', str(signature), str(output / 'header.bin')], check=True) + if len(signature.read_bytes()) != 384: + raise ValueError('Unexpected RSA signature size') + subprocess.run(['openssl', 'dgst', '-sha256', '-verify', str(public_key), '-signature', str(signature), + str(output / 'header.bin')], check=True, capture_output=True) + with (output / 'manifest.bin').open('wb') as envelope: + envelope.write(header) + envelope.write(signature.read_bytes()) + with table.open('rb') as hashes: + for data in iter(lambda: hashes.read(1024 * 1024), b''): + envelope.write(data) + record = dict(version=1, block_bytes=4096, image_bytes=image_bytes, block_count=count, + volume_uuid=str(volume_uuid).upper(), generation=generation, + image_sha256=image_hash.hexdigest(), hash_table_sha256=table_hash.hexdigest(), + trusted_key_sha256=hashlib.sha256(anchor).hexdigest(), + signature_algorithm='RSA3072-PKCS1v1.5-SHA256', signature_verified_by_openssl=True, + image_modified=False, hardware_tested=False, boot_root_authenticated=False) + (output / 'manifest.json').write_text(json.dumps(record, indent=2) + '\n') + return record + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--image', type=Path, required=True) + parser.add_argument('--private-key', type=Path, required=True) + parser.add_argument('--public-key', type=Path, required=True) + parser.add_argument('--volume-uuid', type=uuid.UUID, required=True) + parser.add_argument('--generation', type=int, required=True) + parser.add_argument('--output', type=Path, required=True) + args = parser.parse_args() + if not args.image.is_file(): + parser.error('CLI image input must be an existing regular file') + with args.image.open('rb') as stream: + result = make(stream, args.image.stat().st_size, args.private_key, args.public_key, + args.volume_uuid, args.generation, args.output) + print(json.dumps(result, indent=2)) diff --git a/kits/VerifiedRoot/tools/verify_image.py b/kits/VerifiedRoot/tools/verify_image.py new file mode 100644 index 0000000..2d147d3 --- /dev/null +++ b/kits/VerifiedRoot/tools/verify_image.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +"""Verify real signatures and every image block with the compiled C provider.""" +import argparse +import ctypes +import hashlib +import json +from pathlib import Path +import uuid + +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument('--library', type=Path, required=True) +parser.add_argument('--manifest', type=Path, required=True) +parser.add_argument('--trusted-key', type=Path, required=True) +parser.add_argument('--image', type=Path, required=True) +parser.add_argument('--volume-uuid', type=uuid.UUID, required=True) +parser.add_argument('--minimum-generation', type=int, required=True) +parser.add_argument('--output', type=Path, required=True) +args = parser.parse_args() + + +class View(ctypes.Structure): + _fields_ = [('hashes', ctypes.c_void_p), ('image_bytes', ctypes.c_uint64), ('block_count', ctypes.c_uint64), + ('generation', ctypes.c_uint64), ('volume_uuid', ctypes.c_uint8 * 16), ('image_sha256', ctypes.c_uint8 * 32)] + + +native = ctypes.CDLL(str(args.library.resolve())) +native.vr_host_open.argtypes = [ctypes.c_void_p, ctypes.c_size_t, ctypes.c_void_p, ctypes.c_uint64, ctypes.POINTER(View)] +native.vr_host_block.argtypes = [ctypes.POINTER(View), ctypes.c_uint64, ctypes.c_void_p, ctypes.c_size_t] +encoded = ctypes.create_string_buffer(args.manifest.read_bytes()) +anchor_bytes = args.trusted_key.read_bytes() +if len(anchor_bytes) != 395: + raise SystemExit('Require the explicit complete trusted public anchor') +anchor = ctypes.create_string_buffer(anchor_bytes) +view = View() +status = native.vr_host_open(encoded, len(encoded) - 1, anchor, args.minimum_generation, ctypes.byref(view)) +if status or bytes(view.volume_uuid) != args.volume_uuid.bytes: + raise SystemExit('Real manifest authentication or authorized volume identity failed') +if not args.image.is_file() or args.image.stat().st_size != view.image_bytes: + raise SystemExit('Require the actual exact-size original regular image') +digest = hashlib.sha256() +with args.image.open('rb') as stream: + for index in range(view.block_count): + block = stream.read(min(4096, view.image_bytes - index * 4096)) + if native.vr_host_block(ctypes.byref(view), index, block, len(block)) != 0: + raise SystemExit('Actual image block authentication failed at index ' + str(index)) + digest.update(block) + if stream.read(1): + raise SystemExit('The image grew beyond the signed bounds') +if digest.digest() != bytes(view.image_sha256): + raise SystemExit('Complete image digest differs from the genuinely signed metadata') +report = dict(image_bytes=view.image_bytes, blocks_verified=view.block_count, + image_sha256=digest.hexdigest(), volume_uuid=str(args.volume_uuid).upper(), + generation=view.generation, trusted_key_sha256=hashlib.sha256(anchor_bytes).hexdigest(), + real_c_signature_and_block_verification=True, original_image_modified=False, + host_only=True, boot_root_authenticated=False, qemu_tested=False, hardware_tested=False) +args.output.write_text(json.dumps(report, indent=2) + '\n') +print('PASS: real RSA signature and all', view.block_count, 'original blocks verified by C; host only') diff --git a/package/CrabApplePkg/Application/CrabApple/CrabApple.c b/package/CrabApplePkg/Application/CrabApple/CrabApple.c new file mode 100644 index 0000000..3d3c45a --- /dev/null +++ b/package/CrabApplePkg/Application/CrabApple/CrabApple.c @@ -0,0 +1,50 @@ +/** @file + UniXNU UEFI application entry point. + + Copyright (c) 2026, UniXNU contributors. All rights reserved. + SPDX-License-Identifier: BSD-2-Clause-Patent +**/ + +#include +#include +#include +#include +#include + +// +// Optional data of the boot option CrabAppleDxe registers. Keep in sync with +// CrabAppleDxe.c. +// +STATIC CONST CHAR16 mFirmwareMarker[] = L"crabapple:firmware"; + +EFI_STATUS +EFIAPI +CrabAppleApplicationEntry ( + IN EFI_HANDLE ImageHandle, + IN EFI_SYSTEM_TABLE *SystemTable + ) +{ + EFI_LOADED_IMAGE_PROTOCOL *Loaded; + CRABAPPLE_HOST Host; + EFI_STATUS Status; + + Host = CrabAppleHostApplication; + if (!EFI_ERROR (gBS->HandleProtocol (ImageHandle, &gEfiLoadedImageProtocolGuid, (VOID **)&Loaded)) && + (Loaded->LoadOptions != NULL) && (Loaded->LoadOptionsSize >= sizeof (mFirmwareMarker)) && + (CompareMem (Loaded->LoadOptions, mFirmwareMarker, sizeof (mFirmwareMarker)) == 0)) + { + Host = CrabAppleHostFirmware; + } + + Status = CrabAppleRun (ImageHandle, Host); + + // + // "Return to firmware" from a firmware boot option: report failure so BDS + // moves on to its next option instead of restarting UniXNU. + // + if ((Host == CrabAppleHostFirmware) && !EFI_ERROR (Status)) { + return EFI_ABORTED; + } + + return Status; +} diff --git a/package/CrabApplePkg/Application/CrabApple/CrabApple.inf b/package/CrabApplePkg/Application/CrabApple/CrabApple.inf new file mode 100644 index 0000000..e60fdae --- /dev/null +++ b/package/CrabApplePkg/Application/CrabApple/CrabApple.inf @@ -0,0 +1,31 @@ +## @file +# UniXNU as a UEFI application (CrabApple.efi): runs on any standard +# UEFI implementation for the target architecture. +# +# Copyright (c) 2026, UniXNU contributors. All rights reserved. +# SPDX-License-Identifier: BSD-2-Clause-Patent +## + +[Defines] + INF_VERSION = 0x00010019 + BASE_NAME = CrabApple + FILE_GUID = 9B665204-A133-47A2-9396-9E2B370A4E25 + MODULE_TYPE = UEFI_APPLICATION + VERSION_STRING = 0.1 + ENTRY_POINT = CrabAppleApplicationEntry + +[Sources] + CrabApple.c + +[Packages] + MdePkg/MdePkg.dec + CrabApplePkg/CrabApplePkg.dec + +[LibraryClasses] + BaseMemoryLib + CrabAppleLib + UefiApplicationEntryPoint + UefiBootServicesTableLib + +[Protocols] + gEfiLoadedImageProtocolGuid ## CONSUMES diff --git a/package/CrabApplePkg/CrabApplePkg.dec b/package/CrabApplePkg/CrabApplePkg.dec new file mode 100644 index 0000000..443e798 --- /dev/null +++ b/package/CrabApplePkg/CrabApplePkg.dec @@ -0,0 +1,47 @@ +## @file +# UniXNU: a device-agnostic UEFI boot manager. +# +# Copyright (c) 2026, UniXNU contributors. All rights reserved. +# SPDX-License-Identifier: BSD-2-Clause-Patent +## + +[Defines] + DEC_SPECIFICATION = 0x00010019 + PACKAGE_NAME = CrabApplePkg + PACKAGE_GUID = E2C7EB96-33AF-4BCD-BA70-26A6BAB3719D + PACKAGE_VERSION = 0.1 + +[Includes] + Include + +[LibraryClasses] + ## @libraryclass The boot manager core shared by the application and the + # DXE driver builds. + CrabAppleLib|Include/Library/CrabAppleLib.h + +[Guids] + gCrabApplePkgTokenSpaceGuid = { 0x13c7e6d7, 0x043c, 0x475b, { 0xba, 0x77, 0x0e, 0x79, 0xd5, 0xad, 0x88, 0x98 } } + + ## Vendor namespace for UEFI variables that UniXNU itself defines. + gCrabAppleVariableGuid = { 0xf50548ac, 0xb469, 0x47c6, { 0x89, 0xea, 0xc1, 0x73, 0x8b, 0x81, 0x21, 0x65 } } + + ## Vendor media node of block devices published by [[block_slice]] + # (Include/Guid/CrabAppleBlockSlice.h). + gCrabAppleBlockSliceGuid = { 0x63e1d76a, 0x715f, 0x4b8c, { 0x85, 0x0a, 0xe1, 0x58, 0xd1, 0x27, 0x0f, 0x9f } } + +[Protocols] + ## Optional loader for `type = "xnu"` entries. UniXNU only defines the + # interface; it never ships an implementation (see Docs/Boundaries.md). + gCrabAppleXnuLoaderProtocolGuid = { 0x0f0a492e, 0x5a6a, 0x49f6, { 0xa1, 0xbf, 0x74, 0x3c, 0x87, 0xab, 0x1c, 0xfe } } + +[PcdsFixedAtBuild] + ## DEBUG() level used for UniXNU's own log lines. Platforms whose debug + # output shares the screen with the menu can lower it to DEBUG_VERBOSE. + gCrabApplePkgTokenSpaceGuid.PcdCrabAppleDebugLevel|0x00000040|UINT32|0x00000001 + + ## Path of the configuration file on every scanned volume. + gCrabApplePkgTokenSpaceGuid.PcdCrabAppleConfigPath|L"\\EFI\\CrabApple\\crabapple.toml"|VOID*|0x00000002 + + ## FV file name (GUID) of the UniXNU application that CrabAppleDxe + # registers as a boot option. Defaults to CrabApple.inf's FILE_GUID. + gCrabApplePkgTokenSpaceGuid.PcdCrabAppleApplicationFile|{ GUID("9B665204-A133-47A2-9396-9E2B370A4E25") }|VOID*|0x00000003 diff --git a/package/CrabApplePkg/CrabApplePkg.dsc b/package/CrabApplePkg/CrabApplePkg.dsc new file mode 100644 index 0000000..8eeb47f --- /dev/null +++ b/package/CrabApplePkg/CrabApplePkg.dsc @@ -0,0 +1,58 @@ +## @file +# Standalone build of UniXNU: the application, the firmware registrar and +# the smoke-test payloads. Platforms that embed UniXNU in their own +# firmware list the INFs in their DSC/FDF instead (see README). +# +# Copyright (c) 2026, UniXNU contributors. All rights reserved. +# SPDX-License-Identifier: BSD-2-Clause-Patent +## + +[Defines] + PLATFORM_NAME = CrabApple + PLATFORM_GUID = 1C1F7C2E-7A84-4F7B-9C55-2B7E4C0F5A31 + PLATFORM_VERSION = 0.1 + DSC_SPECIFICATION = 0x0001001C + OUTPUT_DIRECTORY = Build/CrabApple + SUPPORTED_ARCHITECTURES = AARCH64 + BUILD_TARGETS = DEBUG|RELEASE + SKUID_IDENTIFIER = DEFAULT + +!include MdePkg/MdeLibs.dsc.inc + +[LibraryClasses] + BaseLib|MdePkg/Library/BaseLib/BaseLib.inf + BaseMemoryLib|MdePkg/Library/BaseMemoryLib/BaseMemoryLib.inf + DevicePathLib|MdePkg/Library/UefiDevicePathLib/UefiDevicePathLib.inf + FdtLib|MdePkg/Library/BaseFdtLib/BaseFdtLib.inf + MemoryAllocationLib|MdePkg/Library/UefiMemoryAllocationLib/UefiMemoryAllocationLib.inf + PcdLib|MdePkg/Library/BasePcdLibNull/BasePcdLibNull.inf + PrintLib|MdePkg/Library/BasePrintLib/BasePrintLib.inf + UefiApplicationEntryPoint|MdePkg/Library/UefiApplicationEntryPoint/UefiApplicationEntryPoint.inf + UefiBootServicesTableLib|MdePkg/Library/UefiBootServicesTableLib/UefiBootServicesTableLib.inf + UefiDriverEntryPoint|MdePkg/Library/UefiDriverEntryPoint/UefiDriverEntryPoint.inf + UefiLib|MdePkg/Library/UefiLib/UefiLib.inf + UefiRuntimeServicesTableLib|MdePkg/Library/UefiRuntimeServicesTableLib/UefiRuntimeServicesTableLib.inf + DebugPrintErrorLevelLib|MdePkg/Library/BaseDebugPrintErrorLevelLib/BaseDebugPrintErrorLevelLib.inf + CrabAppleLib|CrabApplePkg/Library/CrabAppleLib/CrabAppleLib.inf + +!if $(TARGET) == RELEASE + DebugLib|MdePkg/Library/BaseDebugLibNull/BaseDebugLibNull.inf +!else + # + # Errors and ASSERTs only, on StdErr; UniXNU's own log lines go to + # EFI_SERIAL_IO_PROTOCOL when present (see CaLog()). + # + DebugLib|MdePkg/Library/UefiDebugLibStdErr/UefiDebugLibStdErr.inf +!endif + +[PcdsFixedAtBuild] + gEfiMdePkgTokenSpaceGuid.PcdDebugPrintErrorLevel|0x80000000 + gEfiMdePkgTokenSpaceGuid.PcdDebugPropertyMask|0x07 + +[Components] + CrabApplePkg/Application/CrabApple/CrabApple.inf + CrabApplePkg/Driver/CrabAppleDxe/CrabAppleDxe.inf + CrabApplePkg/Test/HelloPayload/HelloPayload.inf + CrabApplePkg/Test/SampleDriver/SampleDriver.inf + CrabApplePkg/Test/XnuTest/XnuTest.inf + diff --git a/package/CrabApplePkg/Driver/CrabAppleDxe/CrabAppleDxe.c b/package/CrabApplePkg/Driver/CrabAppleDxe/CrabAppleDxe.c new file mode 100644 index 0000000..5c2cfbc --- /dev/null +++ b/package/CrabApplePkg/Driver/CrabAppleDxe/CrabAppleDxe.c @@ -0,0 +1,237 @@ +/** @file + CrabAppleDxe: registers the UniXNU application in this firmware volume + as the first boot option. + + Only standard UEFI variables are used (Boot####, BootOrder). On platforms + whose variable store is RAM-backed the option is simply re-created on every + boot. + + Copyright (c) 2026, UniXNU contributors. All rights reserved. + SPDX-License-Identifier: BSD-2-Clause-Patent +**/ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define OPTION_ATTRIBUTES (EFI_VARIABLE_NON_VOLATILE | EFI_VARIABLE_BOOTSERVICE_ACCESS | EFI_VARIABLE_RUNTIME_ACCESS) + +// +// Optional data of the boot option; the application treats it as "started +// by firmware". Keep in sync with CrabApple.c. +// +STATIC CONST CHAR16 mFirmwareMarker[] = L"crabapple:firmware"; +STATIC CONST CHAR16 mDescription[] = L"UniXNU"; + +STATIC +BOOLEAN +FvHasFile ( + IN EFI_HANDLE Handle, + IN CONST EFI_GUID *File + ) +{ + EFI_FIRMWARE_VOLUME2_PROTOCOL *Fv; + EFI_FV_FILETYPE Type; + EFI_FV_FILE_ATTRIBUTES Attributes; + UINT32 AuthStatus; + UINTN Size; + + if (EFI_ERROR (gBS->HandleProtocol (Handle, &gEfiFirmwareVolume2ProtocolGuid, (VOID **)&Fv))) { + return FALSE; + } + + return !EFI_ERROR (Fv->ReadFile (Fv, File, NULL, &Size, &Type, &Attributes, &AuthStatus)); +} + +// +// Device path of the application file: our own FV first, then any FV. +// +STATIC +EFI_DEVICE_PATH_PROTOCOL * +FindApplication ( + IN EFI_HANDLE ImageHandle + ) +{ + EFI_LOADED_IMAGE_PROTOCOL *Loaded; + CONST EFI_GUID *File; + EFI_HANDLE Fv; + EFI_HANDLE *Handles; + UINTN Count; + UINTN Index; + MEDIA_FW_VOL_FILEPATH_DEVICE_PATH Node; + + File = (CONST EFI_GUID *)PcdGetPtr (PcdCrabAppleApplicationFile); + Fv = NULL; + if (!EFI_ERROR (gBS->HandleProtocol (ImageHandle, &gEfiLoadedImageProtocolGuid, (VOID **)&Loaded)) && + FvHasFile (Loaded->DeviceHandle, File)) + { + Fv = Loaded->DeviceHandle; + } + + if ((Fv == NULL) && + !EFI_ERROR (gBS->LocateHandleBuffer (ByProtocol, &gEfiFirmwareVolume2ProtocolGuid, NULL, &Count, &Handles))) + { + for (Index = 0; Index < Count && Fv == NULL; Index++) { + if (FvHasFile (Handles[Index], File)) { + Fv = Handles[Index]; + } + } + + FreePool (Handles); + } + + if ((Fv == NULL) || (DevicePathFromHandle (Fv) == NULL)) { + return NULL; + } + + EfiInitializeFwVolDevicepathNode (&Node, File); + return AppendDevicePathNode (DevicePathFromHandle (Fv), (EFI_DEVICE_PATH_PROTOCOL *)&Node); +} + +// +// Serialize an EFI_LOAD_OPTION (UEFI spec 3.1.3). +// +STATIC +VOID * +BuildLoadOption ( + IN EFI_DEVICE_PATH_PROTOCOL *Path, + OUT UINTN *Size + ) +{ + UINTN PathSize; + UINT8 *Option; + UINT8 *P; + + PathSize = GetDevicePathSize (Path); + *Size = sizeof (UINT32) + sizeof (UINT16) + sizeof (mDescription) + PathSize + sizeof (mFirmwareMarker); + Option = AllocateZeroPool (*Size); + if (Option == NULL) { + return NULL; + } + + P = Option; + WriteUnaligned32 ((UINT32 *)P, LOAD_OPTION_ACTIVE); + P += sizeof (UINT32); + WriteUnaligned16 ((UINT16 *)P, (UINT16)PathSize); + P += sizeof (UINT16); + CopyMem (P, mDescription, sizeof (mDescription)); + P += sizeof (mDescription); + CopyMem (P, Path, PathSize); + P += PathSize; + CopyMem (P, mFirmwareMarker, sizeof (mFirmwareMarker)); + return Option; +} + +EFI_STATUS +EFIAPI +CrabAppleDxeEntry ( + IN EFI_HANDLE ImageHandle, + IN EFI_SYSTEM_TABLE *SystemTable + ) +{ + EFI_STATUS Status; + EFI_DEVICE_PATH_PROTOCOL *Path; + VOID *Option; + UINTN OptionSize; + UINT16 *Order; + UINTN OrderSize; + UINT16 *NewOrder; + UINTN Index; + UINTN Out; + UINT16 Number; + CHAR16 Name[sizeof ("Boot####")]; + VOID *Existing; + UINTN ExistingSize; + BOOLEAN Found; + + Path = FindApplication (ImageHandle); + if (Path == NULL) { + DEBUG ((DEBUG_ERROR, "CrabAppleDxe: application %g not found in any FV\n", PcdGetPtr (PcdCrabAppleApplicationFile))); + return EFI_NOT_FOUND; + } + + Option = BuildLoadOption (Path, &OptionSize); + FreePool (Path); + if (Option == NULL) { + return EFI_OUT_OF_RESOURCES; + } + + Order = NULL; + OrderSize = 0; + GetEfiGlobalVariable2 (EFI_BOOT_ORDER_VARIABLE_NAME, (VOID **)&Order, &OrderSize); + + // + // Reuse an identical Boot#### if one exists, otherwise take the lowest + // free number. + // + Found = FALSE; + for (Index = 0; Index < OrderSize / sizeof (UINT16) && !Found; Index++) { + UnicodeSPrint (Name, sizeof (Name), L"Boot%04x", Order[Index]); + if (!EFI_ERROR (GetEfiGlobalVariable2 (Name, &Existing, &ExistingSize))) { + Found = (ExistingSize == OptionSize) && (CompareMem (Existing, Option, OptionSize) == 0); + if (Found) { + Number = Order[Index]; + } + + FreePool (Existing); + } + } + + if (!Found) { + for (Number = 0; Number < 0xFFFF; Number++) { + UnicodeSPrint (Name, sizeof (Name), L"Boot%04x", Number); + if (EFI_ERROR (GetEfiGlobalVariable2 (Name, &Existing, &ExistingSize))) { + break; + } + + FreePool (Existing); + } + + Status = gRT->SetVariable (Name, &gEfiGlobalVariableGuid, OPTION_ATTRIBUTES, OptionSize, Option); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "CrabAppleDxe: cannot write %s: %r\n", Name, Status)); + FreePool (Option); + return Status; + } + } + + FreePool (Option); + + // + // BootOrder = Number, then everything else in its old order. + // + NewOrder = AllocatePool (OrderSize + sizeof (UINT16)); + if (NewOrder == NULL) { + return EFI_OUT_OF_RESOURCES; + } + + NewOrder[0] = Number; + Out = 1; + for (Index = 0; Index < OrderSize / sizeof (UINT16); Index++) { + if (Order[Index] != Number) { + NewOrder[Out++] = Order[Index]; + } + } + + Status = gRT->SetVariable (EFI_BOOT_ORDER_VARIABLE_NAME, &gEfiGlobalVariableGuid, OPTION_ATTRIBUTES, Out * sizeof (UINT16), NewOrder); + DEBUG ((DEBUG_INFO, "CrabAppleDxe: Boot%04x first in BootOrder: %r\n", Number, Status)); + + FreePool (NewOrder); + if (Order != NULL) { + FreePool (Order); + } + + return Status; +} diff --git a/package/CrabApplePkg/Driver/CrabAppleDxe/CrabAppleDxe.inf b/package/CrabApplePkg/Driver/CrabAppleDxe/CrabAppleDxe.inf new file mode 100644 index 0000000..c8f22bc --- /dev/null +++ b/package/CrabApplePkg/Driver/CrabAppleDxe/CrabAppleDxe.inf @@ -0,0 +1,55 @@ +## @file +# Firmware integration of UniXNU: a registrar DXE driver. +# +# Put this driver and CrabApplePkg/Application/CrabApple/CrabApple.inf in the +# same firmware volume. When dispatched, the driver registers a Boot#### +# option for the UniXNU application file in that FV and moves it to the +# front of BootOrder, so the platform BDS starts UniXNU like any other +# boot manager (BDS only boots EFI_APPLICATION images). The option carries +# "crabapple:firmware" as optional data, which tells the application it was +# started by firmware. +# +# Copyright (c) 2026, UniXNU contributors. All rights reserved. +# SPDX-License-Identifier: BSD-2-Clause-Patent +## + +[Defines] + INF_VERSION = 0x00010019 + BASE_NAME = CrabAppleDxe + FILE_GUID = 29B950CF-586D-49EA-9D71-F88645CCDAF6 + MODULE_TYPE = DXE_DRIVER + VERSION_STRING = 0.1 + ENTRY_POINT = CrabAppleDxeEntry + +[Sources] + CrabAppleDxe.c + +[Packages] + MdePkg/MdePkg.dec + CrabApplePkg/CrabApplePkg.dec + +[LibraryClasses] + BaseLib + BaseMemoryLib + DebugLib + DevicePathLib + MemoryAllocationLib + PcdLib + PrintLib + UefiBootServicesTableLib + UefiDriverEntryPoint + UefiLib + UefiRuntimeServicesTableLib + +[Guids] + gEfiGlobalVariableGuid ## SOMETIMES_PRODUCES ## Variable:L"Boot####" L"BootOrder" + +[Protocols] + gEfiFirmwareVolume2ProtocolGuid ## CONSUMES + gEfiLoadedImageProtocolGuid ## CONSUMES + +[Pcd] + gCrabApplePkgTokenSpaceGuid.PcdCrabAppleApplicationFile + +[Depex] + gEfiVariableArchProtocolGuid AND gEfiVariableWriteArchProtocolGuid diff --git a/package/CrabApplePkg/Include/Guid/CrabAppleBlockSlice.h b/package/CrabApplePkg/Include/Guid/CrabAppleBlockSlice.h new file mode 100644 index 0000000..344fe80 --- /dev/null +++ b/package/CrabApplePkg/Include/Guid/CrabAppleBlockSlice.h @@ -0,0 +1,31 @@ +/** @file + Device path node of block devices published by `[[block_slice]]`. + + A slice handle carries EFI_BLOCK_IO_PROTOCOL and a device path made of the + source partition's path followed by this vendor media node. + + Copyright (c) 2026, UniXNU contributors. All rights reserved. + SPDX-License-Identifier: BSD-2-Clause-Patent +**/ + +#ifndef CRABAPPLE_BLOCK_SLICE_H_ +#define CRABAPPLE_BLOCK_SLICE_H_ + +#include +#include + +#define CRABAPPLE_BLOCK_SLICE_GUID \ + { 0x63e1d76a, 0x715f, 0x4b8c, { 0x85, 0x0a, 0xe1, 0x58, 0xd1, 0x27, 0x0f, 0x9f } } + +#pragma pack (1) +typedef struct { + VENDOR_DEVICE_PATH Header; ///< MEDIA_DEVICE_PATH / MEDIA_VENDOR_DP, CRABAPPLE_BLOCK_SLICE_GUID. + UINT64 StartingLba; ///< First block of the slice on the source partition. + UINT64 NumberOfBlocks; + CHAR16 Name[36]; ///< `name` from the configuration, NUL padded. +} CRABAPPLE_BLOCK_SLICE_DEVICE_PATH; +#pragma pack () + +extern EFI_GUID gCrabAppleBlockSliceGuid; + +#endif diff --git a/package/CrabApplePkg/Include/Library/CrabAppleLib.h b/package/CrabApplePkg/Include/Library/CrabAppleLib.h new file mode 100644 index 0000000..914b73a --- /dev/null +++ b/package/CrabApplePkg/Include/Library/CrabAppleLib.h @@ -0,0 +1,47 @@ +/** @file + UniXNU boot manager core. + + Both the UEFI application and the DXE driver builds are thin wrappers around + CrabAppleRun(); everything else lives in CrabAppleLib. + + Copyright (c) 2026, UniXNU contributors. All rights reserved. + SPDX-License-Identifier: BSD-2-Clause-Patent +**/ + +#ifndef CRABAPPLE_LIB_H_ +#define CRABAPPLE_LIB_H_ + +#include + +typedef enum { + /// + /// Started as an ordinary UEFI application (from a disk, the shell, a + /// firmware boot option...). + /// + CrabAppleHostApplication, + /// + /// Started by the platform BDS through the boot option the DXE driver + /// registered for itself. + /// + CrabAppleHostFirmware +} CRABAPPLE_HOST; + +/** + Run the boot manager: locate and parse the configuration, inject drivers, + show the menu and start the selected entry. + + @param[in] ImageHandle Image handle of the running UniXNU image. + @param[in] Host How this image was started. + + @retval EFI_SUCCESS The user chose to leave UniXNU. + @retval EFI_ABORTED Nothing could be booted. + @retval other Error from the boot path. +**/ +EFI_STATUS +EFIAPI +CrabAppleRun ( + IN EFI_HANDLE ImageHandle, + IN CRABAPPLE_HOST Host + ); + +#endif diff --git a/package/CrabApplePkg/Include/Protocol/CrabAppleXnuLoader.h b/package/CrabApplePkg/Include/Protocol/CrabAppleXnuLoader.h new file mode 100644 index 0000000..3ecabc9 --- /dev/null +++ b/package/CrabApplePkg/Include/Protocol/CrabAppleXnuLoader.h @@ -0,0 +1,74 @@ +/** @file + Interface for an external loader of `type = "xnu"` entries. + + UniXNU defines this interface so that the boot path for Darwin-style + payloads has a fixed, documented seam. UniXNU does not implement it and + ships no implementation: anything that would have to defeat a vendor's + signature, personalization or hardware-identity checks belongs outside this + project. See Docs/Boundaries.md. + + Copyright (c) 2026, UniXNU contributors. All rights reserved. + SPDX-License-Identifier: BSD-2-Clause-Patent +**/ + +#ifndef CRABAPPLE_XNU_LOADER_H_ +#define CRABAPPLE_XNU_LOADER_H_ + +#include + +#define CRABAPPLE_XNU_LOADER_PROTOCOL_GUID \ + { 0x0f0a492e, 0x5a6a, 0x49f6, { 0xa1, 0xbf, 0x74, 0x3c, 0x87, 0xab, 0x1c, 0xfe } } + +#define CRABAPPLE_XNU_LOADER_REVISION_1 0x00010000 +#define CRABAPPLE_XNU_LOADER_REVISION 0x00020000 + +typedef struct _CRABAPPLE_XNU_LOADER_PROTOCOL CRABAPPLE_XNU_LOADER_PROTOCOL; + +/// Unmodified bundle files, in configured dependency order. This is staging, +/// not proof of signature validity or successful kernel loading. The loader +/// must validate metadata, dependencies, ABI and its vendor's boot policy. +typedef struct { + CONST CHAR16 *BundlePath; + CONST CHAR16 *PlistPath; ///< Absolute path on the source volume. + CONST CHAR16 *ExecutablePath; ///< NULL for a codeless kext. + CONST VOID *InfoPlist; + UINTN InfoPlistSize; + CONST VOID *Executable; ///< NULL for a codeless kext. + UINTN ExecutableSize; +} CRABAPPLE_XNU_KEXT; + +/// +/// Everything UniXNU resolved for the entry. Buffers belong to UniXNU +/// and stay valid for the duration of the Boot() call. +/// +typedef struct { + EFI_HANDLE Volume; ///< Volume the entry's paths refer to. + CONST CHAR16 *KernelPath; ///< Entry `path`. + CONST CHAR16 *BootArgs; ///< Entry `args` (may be NULL). + VOID *DeviceTree; ///< Patched FDT if the entry has `dtb`, else NULL. + UINTN DeviceTreeSize; + VOID *Ramdisk; ///< Entry `initrd` contents, or NULL. + UINTN RamdiskSize; + // Revision 2 tail. Revision 1 loaders receive only requests with no kexts. + CONST CRABAPPLE_XNU_KEXT *Kexts; + UINTN KextCount; +} CRABAPPLE_XNU_BOOT_REQUEST; + +/** + Boot a Darwin-style payload. Returns only on failure. +**/ +typedef +EFI_STATUS +(EFIAPI *CRABAPPLE_XNU_BOOT)( + IN CRABAPPLE_XNU_LOADER_PROTOCOL *This, + IN CONST CRABAPPLE_XNU_BOOT_REQUEST *Request + ); + +struct _CRABAPPLE_XNU_LOADER_PROTOCOL { + UINT32 Revision; + CRABAPPLE_XNU_BOOT Boot; +}; + +extern EFI_GUID gCrabAppleXnuLoaderProtocolGuid; + +#endif diff --git a/package/CrabApplePkg/Library/CrabAppleLib/Boot.c b/package/CrabApplePkg/Library/CrabAppleLib/Boot.c new file mode 100644 index 0000000..9fc8c59 --- /dev/null +++ b/package/CrabApplePkg/Library/CrabAppleLib/Boot.c @@ -0,0 +1,223 @@ +/** @file + Entry execution. + + Copyright (c) 2026, UniXNU contributors. All rights reserved. + SPDX-License-Identifier: BSD-2-Clause-Patent +**/ + +#include "Internal.h" + +#include +#include + +STATIC +EFI_STATUS +RunAction ( + IN CA_ENTRY *Entry + ) +{ + switch (Entry->Action) { + case CaActionReset: + CaLog ("action: reset"); + gRT->ResetSystem (EfiResetCold, EFI_SUCCESS, 0, NULL); + break; + case CaActionShutdown: + CaLog ("action: shutdown"); + gRT->ResetSystem (EfiResetShutdown, EFI_SUCCESS, 0, NULL); + break; + case CaActionExit: + CaLog ("action: exit"); + return EFI_SUCCESS; + default: + break; + } + + return EFI_UNSUPPORTED; +} + +// +// Seam for Darwin-style payloads; see Docs/Boundaries.md. +// +STATIC +EFI_STATUS +RunXnu ( + IN CA_CONFIG *Config, + IN CA_ENTRY *Entry, + IN VOID *Fdt, + IN UINTN FdtSize, + IN VOID *Ramdisk, + IN UINTN RamdiskSize + ) +{ + EFI_STATUS Status; + CRABAPPLE_XNU_LOADER_PROTOCOL *Loader; + CRABAPPLE_XNU_BOOT_REQUEST Request; + CRABAPPLE_XNU_KEXT *Kexts; + UINTN KextCount; + + Status = gBS->LocateProtocol (&gCrabAppleXnuLoaderProtocolGuid, NULL, (VOID **)&Loader); + if (EFI_ERROR (Status)) { + CaLog ("xnu: no loader protocol installed; UniXNU does not provide one"); + return EFI_UNSUPPORTED; + } + + if (((Loader->Revision != CRABAPPLE_XNU_LOADER_REVISION_1) && + (Loader->Revision != CRABAPPLE_XNU_LOADER_REVISION)) || (Loader->Boot == NULL)) + { + CaLog ("xnu: unsupported loader revision or missing Boot callback"); + return EFI_UNSUPPORTED; + } + + Status = CaStageKexts (Config, Entry, &Kexts, &KextCount); + if (EFI_ERROR (Status)) { + return Status; + } + + if ((KextCount != 0) && (Loader->Revision == CRABAPPLE_XNU_LOADER_REVISION_1)) { + CaLog ("xnu: loader revision 1 cannot accept kexts"); + CaFreeKexts (Kexts, KextCount); + return EFI_UNSUPPORTED; + } + + ZeroMem (&Request, sizeof (Request)); + Request.Volume = Entry->Volume; + Request.KernelPath = Entry->Path; + Request.BootArgs = Entry->Args; + Request.DeviceTree = Fdt; + Request.DeviceTreeSize = FdtSize; + Request.Ramdisk = Ramdisk; + Request.RamdiskSize = RamdiskSize; + Request.Kexts = Kexts; + Request.KextCount = KextCount; + CaLog ("xnu: handing off %u staged kext(s) to loader", (UINT32)KextCount); + Status = Loader->Boot (Loader, &Request); + CaFreeKexts (Kexts, KextCount); + return Status; +} + +EFI_STATUS +CaBootEntry ( + IN EFI_HANDLE ImageHandle, + IN CA_CONFIG *Config, + IN CA_ENTRY *Entry + ) +{ + EFI_STATUS Status; + VOID *Fdt; + UINTN FdtSize; + VOID *Initrd; + UINTN InitrdSize; + EFI_HANDLE InitrdHandle; + EFI_DEVICE_PATH_PROTOCOL *Path; + EFI_HANDLE Image; + EFI_LOADED_IMAGE_PROTOCOL *Loaded; + UINTN ExitDataSize; + CHAR16 *ExitData; + VOID *FirmwareFdt; + + CaLog ("boot: entry '%a' (%s)", (Entry->Id != NULL) ? Entry->Id : "?", Entry->Title); + + if (Entry->KextConfigError) { + CaLog ("kext: invalid entry configuration; boot refused"); + return EFI_INVALID_PARAMETER; + } + + if (Entry->Type == CaEntryAction) { + return RunAction (Entry); + } + + Fdt = NULL; + FdtSize = 0; + Initrd = NULL; + InitrdSize = 0; + InitrdHandle = NULL; + + CaSetEntryVariables (Entry); + + if (EFI_ERROR (EfiGetSystemConfigurationTable (&gFdtTableGuid, &FirmwareFdt))) { + FirmwareFdt = NULL; + } + + Status = CaPrepareDeviceTree (Config, Entry, &Fdt, &FdtSize); + if (EFI_ERROR (Status)) { + return Status; + } + + if (Entry->Initrd != NULL) { + Status = CaReadFile (Entry->Volume, Entry->Initrd, &Initrd, &InitrdSize); + if (EFI_ERROR (Status)) { + CaLog ("boot: cannot read initrd %s: %r", Entry->Initrd, Status); + goto Done; + } + + if (Entry->Type != CaEntryXnu) { + Status = CaInstallInitrd (Initrd, InitrdSize, &InitrdHandle); + if (EFI_ERROR (Status)) { + CaLog ("boot: cannot publish initrd: %r", Status); + goto Done; + } + + CaLog ("boot: initrd %s, %u bytes on LoadFile2 initrd media", Entry->Initrd, (UINT32)InitrdSize); + } + } + + if (Entry->Type == CaEntryXnu) { + Status = RunXnu (Config, Entry, Fdt, FdtSize, Initrd, InitrdSize); + goto Done; + } + + // + // efi and linux: a Linux kernel with the EFI stub is an EFI image whose + // command line is its LoadOptions; initrd and DTB are published above. + // + Path = CaResolveImagePath (Entry->Volume, Entry->Path); + if (Path == NULL) { + CaLog ("boot: cannot resolve %s", Entry->Path); + Status = EFI_NOT_FOUND; + goto Done; + } + + Image = NULL; + Status = gBS->LoadImage (FALSE, ImageHandle, Path, NULL, 0, &Image); + FreePool (Path); + if (EFI_ERROR (Status)) { + if ((Status == EFI_SECURITY_VIOLATION) && (Image != NULL)) { + gBS->UnloadImage (Image); + } + + CaLog ("boot: LoadImage %s: %r", Entry->Path, Status); + goto Done; + } + + if ((Entry->Args != NULL) && + !EFI_ERROR (gBS->HandleProtocol (Image, &gEfiLoadedImageProtocolGuid, (VOID **)&Loaded))) + { + Loaded->LoadOptions = Entry->Args; + Loaded->LoadOptionsSize = (UINT32)StrSize (Entry->Args); + } + + CaLog ("boot: starting %s", Entry->Path); + ExitData = NULL; + ExitDataSize = 0; + Status = gBS->StartImage (Image, &ExitDataSize, &ExitData); + CaLog ("boot: %s returned %r", Entry->Path, Status); + if (ExitData != NULL) { + FreePool (ExitData); + } + +Done: + // + // The payload came back: put the firmware's own tree back so the next + // entry starts from the same state. + // + if (Fdt != NULL) { + gBS->InstallConfigurationTable (&gFdtTableGuid, FirmwareFdt); + } + + CaUninstallInitrd (InitrdHandle); + if (Initrd != NULL) { + FreePool (Initrd); + } + + return Status; +} diff --git a/package/CrabApplePkg/Library/CrabAppleLib/ClangdAutoGen.h b/package/CrabApplePkg/Library/CrabAppleLib/ClangdAutoGen.h new file mode 100644 index 0000000..2525d53 --- /dev/null +++ b/package/CrabApplePkg/Library/CrabAppleLib/ClangdAutoGen.h @@ -0,0 +1,12 @@ +/** @file + Editor-only stand-in for the build's AutoGen.h (referenced from + compile_flags.txt so clangd can parse sources outside an edk2 build). + Not used by the real build. + + Copyright (c) 2026, UniXNU contributors. All rights reserved. + SPDX-License-Identifier: BSD-2-Clause-Patent +**/ +#ifndef CLANGD_AUTOGEN_H_ +#define CLANGD_AUTOGEN_H_ +#define MDE_CPU_AARCH64 +#endif diff --git a/package/CrabApplePkg/Library/CrabAppleLib/Config.c b/package/CrabApplePkg/Library/CrabAppleLib/Config.c new file mode 100644 index 0000000..651b2d9 --- /dev/null +++ b/package/CrabApplePkg/Library/CrabAppleLib/Config.c @@ -0,0 +1,1570 @@ +/** @file + Configuration discovery and the TOML -> CA_CONFIG mapping, including kit + bundles. The schema is documented in Docs/Configuration.md and + Docs/Kits.md; keep them in sync with this file. + + Copyright (c) 2026, UniXNU contributors. All rights reserved. + SPDX-License-Identifier: BSD-2-Clause-Patent +**/ + +#include "Internal.h" + +#include + +// +// UniXNU targets AArch64 UEFI only. +// +#if !defined (MDE_CPU_AARCH64) +#error "UniXNU supports AARCH64 only" +#endif + +#define CA_ARCH_NAME "AARCH64" +#define CA_FALLBACK_LOADER L"\\EFI\\BOOT\\BOOTAA64.EFI" + +#define CA_CONFIG_DIR L"\\EFI\\CrabApple" +#define CA_KITS_DIR "Kits" +#define CA_KIT_MANIFEST L"Kit.toml" + +// +// Used when no configuration file is found. Parsed with the same code path +// as a file; removable-media loaders found on other volumes are added in +// front of the action entries. With no loader, the default is "Return to +// firmware": the least surprising thing for an unattended machine, since +// the platform's boot manager then carries on as if UniXNU were absent. +// +STATIC CONST CHAR8 mBuiltinConfig[] = + "version = 1\n" + "[boot]\n" + "timeout = 10\n" + "rotate = 2\n" + "[ui]\n" + "title = \"UniXNU\"\n" + "[[entry]]\n" + "id = \"exit\"\n" + "title = \"Return to firmware\"\n" + "type = \"action\"\n" + "action = \"exit\"\n" + "[[entry]]\n" + "id = \"reset\"\n" + "title = \"Restart\"\n" + "type = \"action\"\n" + "action = \"reset\"\n" + "[[entry]]\n" + "id = \"shutdown\"\n" + "title = \"Power off\"\n" + "type = \"action\"\n" + "action = \"shutdown\"\n"; + +STATIC +VOID +ConfigError ( + IN CA_CONFIG *Config, + IN CONST CHAR8 *Format, + ... + ) +{ + VA_LIST Args; + CHAR8 Message[160]; + + VA_START (Args, Format); + AsciiVSPrint (Message, sizeof (Message), Format, Args); + VA_END (Args); + + CaLog ("config: %a", Message); + if (Config->Error[0] == '\0') { + AsciiStrCpyS (Config->Error, sizeof (Config->Error), Message); + } +} + +// +// Grow *Array (of ElementSize items, *Count used) by one zeroed element. +// +STATIC +VOID * +GrowArray ( + IN OUT VOID **Array, + IN OUT UINTN *Count, + IN UINTN ElementSize + ) +{ + VOID *New; + + New = ReallocatePool (*Count * ElementSize, (*Count + 1) * ElementSize, *Array); + if (New == NULL) { + return NULL; + } + + *Array = New; + ZeroMem ((UINT8 *)New + *Count * ElementSize, ElementSize); + (*Count)++; + return (UINT8 *)New + (*Count - 1) * ElementSize; +} + +STATIC +CHAR8 * +AsciiDup ( + IN CONST CHAR8 *String + ) +{ + return (String == NULL) ? NULL : AllocateCopyPool (AsciiStrSize (String), String); +} + +// +// Config/kit relative path -> absolute volume path. "fv:" is kept as +// "\fv:" and resolved at load time. +// +STATIC +CHAR16 * +MakePath ( + IN CONST CHAR16 *BaseDir, + IN CONST CHAR8 *Path + ) +{ + if (Path == NULL) { + return NULL; + } + + if (AsciiStrnCmp (Path, "fv:", 3) == 0) { + return CaNormalizePath (NULL, Path); + } + + return CaNormalizePath (BaseDir, Path); +} + +// --------------------------------------------------------------------------- +// Value encoding +// --------------------------------------------------------------------------- + +STATIC +VOID +PutBe32 ( + IN UINT8 *Out, + IN UINT32 Value + ) +{ + Out[0] = (UINT8)(Value >> 24); + Out[1] = (UINT8)(Value >> 16); + Out[2] = (UINT8)(Value >> 8); + Out[3] = (UINT8)Value; +} + +STATIC +BOOLEAN +ParseHexBytes ( + IN CONST CHAR8 *Text, + OUT UINT8 **Bytes, + OUT UINTN *Size + ) +{ + UINTN Len; + UINTN Index; + UINTN Count; + UINT8 *Out; + INTN High; + UINT8 V; + CHAR8 C; + + if ((Text[0] == '0') && ((Text[1] == 'x') || (Text[1] == 'X'))) { + Text += 2; + } + + Len = AsciiStrLen (Text); + Out = AllocateZeroPool (Len / 2 + 1); + if (Out == NULL) { + return FALSE; + } + + Count = 0; + High = -1; + for (Index = 0; Index < Len; Index++) { + C = Text[Index]; + if ((C == ' ') || (C == ':') || (C == '-') || (C == '_')) { + continue; + } + + if ((C >= '0') && (C <= '9')) { + V = (UINT8)(C - '0'); + } else if ((C >= 'a') && (C <= 'f')) { + V = (UINT8)(C - 'a' + 10); + } else if ((C >= 'A') && (C <= 'F')) { + V = (UINT8)(C - 'A' + 10); + } else { + FreePool (Out); + return FALSE; + } + + if (High < 0) { + High = V; + } else { + Out[Count++] = (UINT8)((High << 4) | V); + High = -1; + } + } + + if (High >= 0) { + FreePool (Out); + return FALSE; + } + + *Bytes = Out; + *Size = Count; + return TRUE; +} + +// +// Encode a TOML value as FDT property bytes. Type may be NULL (inferred). +// +STATIC +BOOLEAN +EncodeFdtValue ( + IN TOML_NODE *Value, + IN CONST CHAR8 *Type, + OUT VOID **Bytes, + OUT UINTN *Size + ) +{ + TOML_NODE *Item; + UINTN Count; + UINTN Total; + UINT8 *Out; + UINTN O; + BOOLEAN Wide; + + *Bytes = NULL; + *Size = 0; + + if (Value == NULL) { + return (Type != NULL && AsciiStrCmp (Type, "empty") == 0); + } + + if (Type == NULL) { + switch (Value->Type) { + case TomlString: + Type = "string"; + break; + case TomlInteger: + Type = ((UINT64)Value->Integer > MAX_UINT32) ? "u64" : "u32"; + break; + case TomlBoolean: + if (!Value->Boolean) { + return FALSE; + } + + Type = "empty"; + break; + case TomlArray: + Type = (Value->Child != NULL && Value->Child->Type == TomlString) ? "stringlist" : "u32"; + break; + default: + return FALSE; + } + } + + if (AsciiStrCmp (Type, "empty") == 0) { + return TRUE; + } + + if ((AsciiStrCmp (Type, "string") == 0) && (Value->Type == TomlString)) { + *Size = AsciiStrSize (Value->String); + *Bytes = AllocateCopyPool (*Size, Value->String); + return *Bytes != NULL; + } + + if (AsciiStrCmp (Type, "stringlist") == 0) { + if (Value->Type == TomlString) { + return EncodeFdtValue (Value, "string", Bytes, Size); + } + + Total = 0; + for (Item = Value->Child; Item != NULL; Item = Item->Next) { + if (Item->Type != TomlString) { + return FALSE; + } + + Total += AsciiStrSize (Item->String); + } + + Out = AllocateZeroPool (Total + 1); + if (Out == NULL) { + return FALSE; + } + + O = 0; + for (Item = Value->Child; Item != NULL; Item = Item->Next) { + CopyMem (Out + O, Item->String, AsciiStrSize (Item->String)); + O += AsciiStrSize (Item->String); + } + + *Bytes = Out; + *Size = Total; + return TRUE; + } + + if ((AsciiStrCmp (Type, "u32") == 0) || (AsciiStrCmp (Type, "u64") == 0)) { + Wide = (Type[1] == '6'); + Count = (Value->Type == TomlArray) ? TomlCount (Value) : 1; + Out = AllocateZeroPool (Count * (Wide ? 8 : 4) + 1); + if (Out == NULL) { + return FALSE; + } + + O = 0; + Item = (Value->Type == TomlArray) ? Value->Child : Value; + for ( ; Item != NULL; Item = (Value->Type == TomlArray) ? Item->Next : NULL) { + if (Item->Type != TomlInteger) { + FreePool (Out); + return FALSE; + } + + if (Wide) { + PutBe32 (Out + O, (UINT32)RShiftU64 ((UINT64)Item->Integer, 32)); + O += 4; + } + + PutBe32 (Out + O, (UINT32)Item->Integer); + O += 4; + } + + *Bytes = Out; + *Size = O; + return TRUE; + } + + if (AsciiStrCmp (Type, "bytes") == 0) { + if (Value->Type == TomlString) { + return ParseHexBytes (Value->String, (UINT8 **)Bytes, Size); + } + + if (Value->Type != TomlArray) { + return FALSE; + } + + Count = TomlCount (Value); + Out = AllocateZeroPool (Count + 1); + if (Out == NULL) { + return FALSE; + } + + O = 0; + for (Item = Value->Child; Item != NULL; Item = Item->Next) { + if ((Item->Type != TomlInteger) || ((UINT64)Item->Integer > 0xFF)) { + FreePool (Out); + return FALSE; + } + + Out[O++] = (UINT8)Item->Integer; + } + + *Bytes = Out; + *Size = O; + return TRUE; + } + + return FALSE; +} + +// +// Encode a TOML value as UEFI variable data (little endian). +// +STATIC +BOOLEAN +EncodeVariableValue ( + IN TOML_NODE *Value, + IN CONST CHAR8 *Type, + OUT VOID **Bytes, + OUT UINTN *Size + ) +{ + CHAR16 *Wide; + + if (Value == NULL) { + return FALSE; + } + + if (Type == NULL) { + Type = (Value->Type == TomlString) ? "ascii" : ((Value->Type == TomlInteger) ? "u32" : "bytes"); + } + + if ((AsciiStrCmp (Type, "ascii") == 0) && (Value->Type == TomlString)) { + *Size = AsciiStrLen (Value->String); + *Bytes = AllocateCopyPool (*Size + 1, Value->String); + return *Bytes != NULL; + } + + if ((AsciiStrCmp (Type, "ucs2") == 0) && (Value->Type == TomlString)) { + Wide = CaAsciiToUnicode (Value->String); + if (Wide == NULL) { + return FALSE; + } + + *Bytes = Wide; + *Size = StrSize (Wide); + return TRUE; + } + + if ((Value->Type == TomlInteger) && + ((AsciiStrCmp (Type, "u8") == 0) || (AsciiStrCmp (Type, "u16") == 0) || + (AsciiStrCmp (Type, "u32") == 0) || (AsciiStrCmp (Type, "u64") == 0))) + { + *Size = (Type[1] == '8') ? 1 : ((Type[1] == '1') ? 2 : ((Type[1] == '3') ? 4 : 8)); + *Bytes = AllocateZeroPool (8); + if (*Bytes == NULL) { + return FALSE; + } + + CopyMem (*Bytes, &Value->Integer, *Size); + return TRUE; + } + + if (AsciiStrCmp (Type, "bytes") == 0) { + // + // Same rules as FDT bytes; byte order does not apply. + // + return EncodeFdtValue (Value, "bytes", Bytes, Size); + } + + return FALSE; +} + +// --------------------------------------------------------------------------- +// Sections shared by the config file and kit manifests +// --------------------------------------------------------------------------- + +STATIC +BOOLEAN +ParseDtOp ( + IN CONST CHAR8 *Op, + OUT CA_DT_OP *Out + ) +{ + if (AsciiStrCmp (Op, "set") == 0) { + *Out = CaDtSet; + } else if (AsciiStrCmp (Op, "delete-prop") == 0) { + *Out = CaDtDeleteProp; + } else if (AsciiStrCmp (Op, "add-node") == 0) { + *Out = CaDtAddNode; + } else if (AsciiStrCmp (Op, "delete-node") == 0) { + *Out = CaDtDeleteNode; + } else if (AsciiStrCmp (Op, "overlay") == 0) { + *Out = CaDtOverlay; + } else { + return FALSE; + } + + return TRUE; +} + +// +// Append one [[dt_patch]] table to (*Patches, *Count). +// +STATIC +VOID +AddDtPatch ( + IN CA_CONFIG *Config, + IN TOML_NODE *Table, + IN EFI_HANDLE Volume, + IN CONST CHAR16 *BaseDir, + IN CONST CHAR8 *Origin, + IN OUT CA_DT_PATCH **Patches, + IN OUT UINTN *Count + ) +{ + CA_DT_PATCH Patch; + CA_DT_PATCH *Slot; + CONST CHAR8 *OpName; + CONST CHAR8 *File; + CHAR16 *FilePath; + EFI_STATUS Status; + + ZeroMem (&Patch, sizeof (Patch)); + OpName = TomlGetString (Table, "op", "set"); + if (!ParseDtOp (OpName, &Patch.Op)) { + ConfigError (Config, "%a: unknown dt_patch op '%a' (line %u)", Origin, OpName, (UINT32)Table->Line); + return; + } + + Patch.Node = AsciiDup (TomlGetString (Table, "node", NULL)); + Patch.Prop = AsciiDup (TomlGetString (Table, "prop", NULL)); + Patch.Origin = AsciiDup (Origin); + + switch (Patch.Op) { + case CaDtOverlay: + File = TomlGetString (Table, "file", NULL); + if (File == NULL) { + ConfigError (Config, "%a: overlay needs 'file' (line %u)", Origin, (UINT32)Table->Line); + return; + } + + FilePath = MakePath (BaseDir, File); + Status = CaReadFile (Volume, FilePath, &Patch.Value, &Patch.ValueSize); + if (EFI_ERROR (Status)) { + ConfigError (Config, "%a: cannot read overlay %s: %r", Origin, FilePath, Status); + FreePool (FilePath); + return; + } + + FreePool (FilePath); + break; + + case CaDtSet: + if ((Patch.Node == NULL) || (Patch.Prop == NULL) || + !EncodeFdtValue (TomlGet (Table, "value"), TomlGetString (Table, "type", NULL), &Patch.Value, &Patch.ValueSize)) + { + ConfigError (Config, "%a: dt_patch set needs node, prop and a valid value (line %u)", Origin, (UINT32)Table->Line); + return; + } + + break; + + case CaDtDeleteProp: + if ((Patch.Node == NULL) || (Patch.Prop == NULL)) { + ConfigError (Config, "%a: delete-prop needs node and prop (line %u)", Origin, (UINT32)Table->Line); + return; + } + + break; + + default: + if (Patch.Node == NULL) { + ConfigError (Config, "%a: dt_patch needs node (line %u)", Origin, (UINT32)Table->Line); + return; + } + + break; + } + + Slot = GrowArray ((VOID **)Patches, Count, sizeof (CA_DT_PATCH)); + if (Slot != NULL) { + CopyMem (Slot, &Patch, sizeof (Patch)); + } +} + +STATIC +VOID +AddDriver ( + IN CA_CONFIG *Config, + IN TOML_NODE *Table, + IN EFI_HANDLE Volume, + IN CONST CHAR16 *BaseDir, + IN CONST CHAR8 *Origin, + IN CONST CHAR8 *PathKey + ) +{ + CA_DRIVER *Driver; + CONST CHAR8 *Path; + + if (!TomlGetBoolean (Table, "enabled", TRUE)) { + return; + } + + Path = TomlGetString (Table, PathKey, NULL); + if (Path == NULL) { + ConfigError (Config, "%a: driver needs '%a' (line %u)", Origin, PathKey, (UINT32)Table->Line); + return; + } + + Driver = GrowArray ((VOID **)&Config->Drivers, &Config->DriverCount, sizeof (CA_DRIVER)); + if (Driver == NULL) { + return; + } + + Driver->Volume = Volume; + Driver->File = MakePath (BaseDir, Path); + Driver->Args = CaAsciiToUnicode (TomlGetString (Table, "args", NULL)); + Driver->Connect = TomlGetBoolean (Table, "connect", TRUE); + Driver->Origin = AsciiDup (Origin); +} + +STATIC +VOID +AddAcpi ( + IN CA_CONFIG *Config, + IN EFI_HANDLE Volume, + IN CONST CHAR16 *BaseDir, + IN CONST CHAR8 *Path, + IN CONST CHAR8 *Origin + ) +{ + CA_ACPI_TABLE *Table; + + Table = GrowArray ((VOID **)&Config->AcpiTables, &Config->AcpiCount, sizeof (CA_ACPI_TABLE)); + if (Table != NULL) { + Table->Volume = Volume; + Table->File = MakePath (BaseDir, Path); + Table->Origin = AsciiDup (Origin); + } +} + +// --------------------------------------------------------------------------- +// Kits +// --------------------------------------------------------------------------- + +STATIC +BOOLEAN +BundleRelativePath ( + IN CONST CHAR8 *Path + ) +{ + CONST CHAR8 *Part; + CONST CHAR8 *End; + UINTN Length; + + Part = Path; + while (*Part != '\0') { + End = Part; + while ((*End != '\0') && (*End != '/') && (*End != '\\')) { + if (((UINT8)*End < 0x20) || ((UINT8)*End >= 0x7F) || (*End == ':')) { + return FALSE; + } + + End++; + } + + Length = (UINTN)(End - Part); + if ((Length == 0) || ((Length == 1) && (Part[0] == '.')) || + ((Length == 2) && (Part[0] == '.') && (Part[1] == '.'))) + { + return FALSE; + } + + if (*End == '\0') { + return TRUE; + } + + Part = End + 1; + } + + return FALSE; +} + +STATIC +VOID +AddKext ( + IN CA_CONFIG *Config, + IN TOML_NODE *Table, + IN EFI_HANDLE Volume, + IN CONST CHAR16 *BaseDir, + IN CONST CHAR8 *Origin, + IN BOOLEAN FromKit, + IN OUT CA_KEXT **Kexts, + IN OUT UINTN *Count, + IN OUT BOOLEAN *Error + ) +{ + CONST CHAR8 *Bundle; + CONST CHAR8 *Plist; + CONST CHAR8 *Executable; + CONST CHAR8 *EntryId; + TOML_NODE *Value; + CA_KEXT *Kext; + UINTN Index; + CONST CHAR8 *StringKeys[] = { "bundle", "plist", "executable", "entry" }; + + Value = TomlGet (Table, "enabled"); + if ((Value != NULL) && (Value->Type != TomlBoolean)) { + ConfigError (Config, "%a: kext enabled must be boolean", Origin); + *Error = TRUE; + return; + } + + if (!TomlGetBoolean (Table, "enabled", TRUE)) { + return; + } + + for (Index = 0; Index < ARRAY_SIZE (StringKeys); Index++) { + Value = TomlGet (Table, StringKeys[Index]); + if ((Value != NULL) && (Value->Type != TomlString)) { + ConfigError (Config, "%a: kext %a must be a string", Origin, StringKeys[Index]); + *Error = TRUE; + return; + } + } + + Bundle = TomlGetString (Table, "bundle", NULL); + Plist = TomlGetString (Table, "plist", "Contents/Info.plist"); + Executable = TomlGetString (Table, "executable", ""); + EntryId = TomlGetString (Table, "entry", NULL); + if ((Bundle == NULL) || (AsciiStrLen (Bundle) < 6) || + (AsciiStrCmp (Bundle + AsciiStrLen (Bundle) - 5, ".kext") != 0) || + !BundleRelativePath (Bundle + ((Bundle[0] == '/') || (Bundle[0] == '\\'))) || + !BundleRelativePath (Plist) || + ((Executable[0] != '\0') && !BundleRelativePath (Executable)) || + (FromKit && ((EntryId == NULL) || (EntryId[0] == '\0'))) || + (!FromKit && (EntryId != NULL))) + { + ConfigError (Config, "%a: kext needs a .kext bundle, relative file paths and valid entry scope", Origin); + *Error = TRUE; + return; + } + + Kext = GrowArray ((VOID **)Kexts, Count, sizeof (CA_KEXT)); + if (Kext == NULL) { + *Error = TRUE; + return; + } + + Kext->Volume = Volume; + Kext->EntryId = AsciiDup (EntryId); + Kext->BundlePath = MakePath (BaseDir, Bundle); + if (Kext->BundlePath != NULL) { + Kext->PlistPath = MakePath (Kext->BundlePath, Plist); + if (Executable[0] != '\0') { + Kext->ExecutablePath = MakePath (Kext->BundlePath, Executable); + } + } + + Kext->Invalid = (Kext->BundlePath == NULL) || (Kext->PlistPath == NULL) || + (FromKit && (Kext->EntryId == NULL)) || + ((Executable[0] != '\0') && (Kext->ExecutablePath == NULL)); +} + +STATIC +BOOLEAN +AnyStringMatches ( + IN TOML_NODE *Value, + IN BOOLEAN (*Predicate)(IN CONST CHAR8 *Candidate) + ) +{ + TOML_NODE *Item; + + if (Value->Type == TomlString) { + return Predicate (Value->String); + } + + if (Value->Type != TomlArray) { + return FALSE; + } + + for (Item = Value->Child; Item != NULL; Item = Item->Next) { + if ((Item->Type == TomlString) && Predicate (Item->String)) { + return TRUE; + } + } + + return FALSE; +} + +STATIC +BOOLEAN +MatchCompatible ( + IN CONST CHAR8 *Candidate + ) +{ + CONST CHAR8 *List; + UINTN Length; + UINTN Offset; + + List = CaFirmwareCompatible (&Length); + if (List == NULL) { + return FALSE; + } + + for (Offset = 0; Offset < Length; Offset += AsciiStrSize (List + Offset)) { + if (AsciiStrCmp (List + Offset, Candidate) == 0) { + return TRUE; + } + } + + return FALSE; +} + +STATIC +BOOLEAN +MatchVendor ( + IN CONST CHAR8 *Candidate + ) +{ + CHAR16 *Wide; + BOOLEAN Found; + + if (gST->FirmwareVendor == NULL) { + return FALSE; + } + + Wide = CaAsciiToUnicode (Candidate); + if (Wide == NULL) { + return FALSE; + } + + Found = (StrStr (gST->FirmwareVendor, Wide) != NULL); + FreePool (Wide); + return Found; +} + +STATIC +BOOLEAN +MatchProtocol ( + IN CONST CHAR8 *Candidate + ) +{ + EFI_GUID Guid; + VOID *Interface; + + if (RETURN_ERROR (AsciiStrToGuid (Candidate, &Guid))) { + return FALSE; + } + + return !EFI_ERROR (gBS->LocateProtocol (&Guid, NULL, &Interface)); +} + +// +// All present [match] keys must be satisfied; within a key, any listed value. +// +STATIC +BOOLEAN +KitMatches ( + IN TOML_NODE *Manifest, + OUT CHAR8 *Reason, + IN UINTN ReasonSize + ) +{ + TOML_NODE *Match; + TOML_NODE *Value; + TOML_NODE *Item; + + Match = TomlGet (Manifest, "match"); + if (Match == NULL) { + return TRUE; + } + + Value = TomlGet (Match, "arch"); + if (Value != NULL) { + BOOLEAN Ok = FALSE; + if (Value->Type == TomlString) { + Ok = (AsciiStrCmp (Value->String, CA_ARCH_NAME) == 0); + } else { + for (Item = Value->Child; Item != NULL; Item = Item->Next) { + Ok = Ok || (Item->Type == TomlString && AsciiStrCmp (Item->String, CA_ARCH_NAME) == 0); + } + } + + if (!Ok) { + AsciiStrCpyS (Reason, ReasonSize, "arch"); + return FALSE; + } + } + + Value = TomlGet (Match, "fdt_compatible"); + if ((Value != NULL) && !AnyStringMatches (Value, MatchCompatible)) { + AsciiStrCpyS (Reason, ReasonSize, "fdt_compatible"); + return FALSE; + } + + Value = TomlGet (Match, "firmware_vendor"); + if ((Value != NULL) && !AnyStringMatches (Value, MatchVendor)) { + AsciiStrCpyS (Reason, ReasonSize, "firmware_vendor"); + return FALSE; + } + + // + // requires_protocol: every GUID must be present. + // + Value = TomlGet (Match, "requires_protocol"); + if (Value != NULL) { + if (Value->Type == TomlString) { + if (!MatchProtocol (Value->String)) { + AsciiStrCpyS (Reason, ReasonSize, "requires_protocol"); + return FALSE; + } + } else { + for (Item = Value->Child; Item != NULL; Item = Item->Next) { + if ((Item->Type != TomlString) || !MatchProtocol (Item->String)) { + AsciiStrCpyS (Reason, ReasonSize, "requires_protocol"); + return FALSE; + } + } + } + } + + return TRUE; +} + +// +// Load \Kit.toml and merge its contents into Config. +// +STATIC +VOID +LoadKit ( + IN CA_CONFIG *Config, + IN EFI_HANDLE Volume, + IN CONST CHAR16 *KitDir + ) +{ + EFI_STATUS Status; + CHAR16 *ManifestPath; + CHAR8 *Text; + UINTN Size; + TOML_NODE *Manifest; + TOML_NODE *Kit; + TOML_NODE *List; + TOML_NODE *Item; + CHAR8 Error[96]; + CHAR8 Origin[80]; + CHAR8 Reason[32]; + CONST CHAR8 *Name; + + ManifestPath = AllocateZeroPool (StrSize (KitDir) + StrSize (CA_KIT_MANIFEST)); + if (ManifestPath == NULL) { + return; + } + + StrCpyS (ManifestPath, StrSize (KitDir) / sizeof (CHAR16) + StrLen (CA_KIT_MANIFEST) + 1, KitDir); + StrCatS (ManifestPath, StrSize (KitDir) / sizeof (CHAR16) + StrLen (CA_KIT_MANIFEST) + 1, L"\\"); + StrCatS (ManifestPath, StrSize (KitDir) / sizeof (CHAR16) + StrLen (CA_KIT_MANIFEST) + 1, CA_KIT_MANIFEST); + + Status = CaReadFile (Volume, ManifestPath, (VOID **)&Text, &Size); + if (EFI_ERROR (Status)) { + ConfigError (Config, "kit %s: no Kit.toml (%r)", KitDir, Status); + FreePool (ManifestPath); + Config->KitsSkipped++; + return; + } + + FreePool (ManifestPath); + Status = TomlParse (Text, Size, &Manifest, Error, sizeof (Error)); + FreePool (Text); + if (EFI_ERROR (Status)) { + ConfigError (Config, "kit %s: %a", KitDir, Error); + Config->KitsSkipped++; + return; + } + + Kit = TomlGet (Manifest, "kit"); + Name = TomlGetString (Kit, "name", NULL); + if ((Name == NULL) || (TomlGetInteger (Manifest, "schema", 0) != CA_SCHEMA_VERSION)) { + ConfigError (Config, "kit %s: needs [kit] name and schema = %d", KitDir, CA_SCHEMA_VERSION); + TomlFree (Manifest); + Config->KitsSkipped++; + return; + } + + AsciiSPrint (Origin, sizeof (Origin), "kit %a", Name); + if (!KitMatches (Manifest, Reason, sizeof (Reason))) { + CaLog ("kit %a %a: not loaded, [match] %a not satisfied", Name, TomlGetString (Kit, "version", "?"), Reason); + TomlFree (Manifest); + Config->KitsSkipped++; + return; + } + + List = TomlGet (Manifest, "driver"); + for (Item = (List != NULL) ? List->Child : NULL; Item != NULL; Item = Item->Next) { + AddDriver (Config, Item, Volume, KitDir, Origin, "file"); + } + + List = TomlGet (Manifest, "overlay"); + for (Item = (List != NULL) ? List->Child : NULL; Item != NULL; Item = Item->Next) { + CA_DT_PATCH *Patch; + CHAR16 *Path; + CONST CHAR8 *File; + + File = TomlGetString (Item, "file", NULL); + if (File == NULL) { + ConfigError (Config, "%a: [[overlay]] needs 'file'", Origin); + continue; + } + + Path = MakePath (KitDir, File); + Patch = GrowArray ((VOID **)&Config->Patches, &Config->PatchCount, sizeof (CA_DT_PATCH)); + if ((Path == NULL) || (Patch == NULL)) { + continue; + } + + Patch->Op = CaDtOverlay; + Patch->Origin = AsciiDup (Origin); + Status = CaReadFile (Volume, Path, &Patch->Value, &Patch->ValueSize); + if (EFI_ERROR (Status)) { + ConfigError (Config, "%a: cannot read %s: %r", Origin, Path, Status); + Config->PatchCount--; + } + + FreePool (Path); + } + + List = TomlGet (Manifest, "dt_patch"); + for (Item = (List != NULL) ? List->Child : NULL; Item != NULL; Item = Item->Next) { + AddDtPatch (Config, Item, Volume, KitDir, Origin, &Config->Patches, &Config->PatchCount); + } + + List = TomlGet (Manifest, "acpi"); + for (Item = (List != NULL) ? List->Child : NULL; Item != NULL; Item = Item->Next) { + if (TomlGetString (Item, "file", NULL) != NULL) { + AddAcpi (Config, Volume, KitDir, TomlGetString (Item, "file", NULL), Origin); + } + } + + List = TomlGet (Manifest, "kext"); + if ((List != NULL) && ((List->Type != TomlArray) || !List->TableArray)) { + ConfigError (Config, "%a: kext must be an array of tables", Origin); + Config->KextConfigError = TRUE; + } else { + for (Item = (List != NULL) ? List->Child : NULL; Item != NULL; Item = Item->Next) { + AddKext (Config, Item, Volume, KitDir, Origin, TRUE, + &Config->Kexts, &Config->KextCount, &Config->KextConfigError); + } + } + + CaLog ("kit %a %a: loaded", Name, TomlGetString (Kit, "version", "?")); + Config->KitsLoaded++; + TomlFree (Manifest); +} + +STATIC +INTN +EFIAPI +CompareNames ( + IN CONST VOID *A, + IN CONST VOID *B + ) +{ + return CaStriCmp (*(CHAR16 **)A, *(CHAR16 **)B); +} + +STATIC +VOID +LoadKits ( + IN CA_CONFIG *Config, + IN TOML_NODE *Root + ) +{ + TOML_NODE *List; + TOML_NODE *Item; + TOML_NODE *Kits; + CHAR16 *Dir; + CHAR16 **Names; + UINTN Count; + UINTN Index; + UINTN Len; + CHAR16 *KitDir; + + // + // Explicit [[kit]] entries first, in file order. + // + List = TomlGet (Root, "kit"); + for (Item = (List != NULL) ? List->Child : NULL; Item != NULL; Item = Item->Next) { + if (!TomlGetBoolean (Item, "enabled", TRUE) || (TomlGetString (Item, "path", NULL) == NULL)) { + continue; + } + + KitDir = MakePath (CA_CONFIG_DIR, TomlGetString (Item, "path", NULL)); + if (KitDir != NULL) { + LoadKit (Config, Config->Volume, KitDir); + FreePool (KitDir); + } + } + + // + // Then every *.kit directory under [kits] directory, sorted by name. + // + Kits = TomlGet (Root, "kits"); + if (!TomlGetBoolean (Kits, "autoload", FALSE)) { + return; + } + + Dir = MakePath (CA_CONFIG_DIR, TomlGetString (Kits, "directory", CA_KITS_DIR)); + if ((Dir == NULL) || EFI_ERROR (CaListDirectory (Config->Volume, Dir, &Names, &Count))) { + if (Dir != NULL) { + FreePool (Dir); + } + + return; + } + + if (Count > 1) { + CHAR16 *Swap; + QuickSort (Names, Count, sizeof (CHAR16 *), CompareNames, &Swap); + } + + for (Index = 0; Index < Count; Index++) { + Len = StrLen (Names[Index]); + if ((Len > 4) && (CaStriCmp (Names[Index] + Len - 4, L".kit") == 0)) { + UINTN Chars = StrLen (Dir) + Len + 2; + KitDir = AllocateZeroPool (Chars * sizeof (CHAR16)); + if (KitDir != NULL) { + UnicodeSPrint (KitDir, Chars * sizeof (CHAR16), L"%s\\%s", Dir, Names[Index]); + LoadKit (Config, Config->Volume, KitDir); + FreePool (KitDir); + } + } + + FreePool (Names[Index]); + } + + FreePool (Names); + FreePool (Dir); +} + +// --------------------------------------------------------------------------- +// Entries +// --------------------------------------------------------------------------- + +STATIC +VOID +AddEntry ( + IN CA_CONFIG *Config, + IN TOML_NODE *Table + ) +{ + CA_ENTRY *Entry; + CONST CHAR8 *Type; + CONST CHAR8 *Action; + CONST CHAR8 *Path; + TOML_NODE *List; + TOML_NODE *Item; + CHAR8 Origin[64]; + CA_DT_PATCH *Patch; + + if (!TomlGetBoolean (Table, "enabled", TRUE)) { + return; + } + + Entry = GrowArray ((VOID **)&Config->Entries, &Config->EntryCount, sizeof (CA_ENTRY)); + if (Entry == NULL) { + return; + } + + Entry->Id = AsciiDup (TomlGetString (Table, "id", NULL)); + Entry->Title = CaAsciiToUnicode (TomlGetString (Table, "title", TomlGetString (Table, "id", "Untitled"))); + Entry->Volume = Config->Volume; + AsciiSPrint (Origin, sizeof (Origin), "entry %a", (Entry->Id != NULL) ? Entry->Id : "?"); + + Type = TomlGetString (Table, "type", "efi"); + if (AsciiStrCmp (Type, "efi") == 0) { + Entry->Type = CaEntryEfi; + } else if (AsciiStrCmp (Type, "linux") == 0) { + Entry->Type = CaEntryLinux; + } else if (AsciiStrCmp (Type, "xnu") == 0) { + Entry->Type = CaEntryXnu; + } else if (AsciiStrCmp (Type, "action") == 0) { + Entry->Type = CaEntryAction; + Action = TomlGetString (Table, "action", ""); + if (AsciiStrCmp (Action, "reset") == 0) { + Entry->Action = CaActionReset; + } else if (AsciiStrCmp (Action, "shutdown") == 0) { + Entry->Action = CaActionShutdown; + } else if (AsciiStrCmp (Action, "exit") == 0) { + Entry->Action = CaActionExit; + } else { + ConfigError (Config, "%a: unknown action '%a'", Origin, Action); + } + } else { + ConfigError (Config, "%a: unknown type '%a'", Origin, Type); + } + + Path = TomlGetString (Table, "path", NULL); + if ((Entry->Type != CaEntryAction) && (Path == NULL)) { + ConfigError (Config, "%a: needs 'path'", Origin); + } + + Entry->Path = MakePath (CA_CONFIG_DIR, Path); + Entry->Args = CaAsciiToUnicode (TomlGetString (Table, "args", NULL)); + Entry->Initrd = MakePath (CA_CONFIG_DIR, TomlGetString (Table, "initrd", NULL)); + List = TomlGet (Table, "kext"); + if (List != NULL) { + if ((Entry->Type != CaEntryXnu) || (List->Type != TomlArray) || !List->TableArray) { + ConfigError (Config, "%a: [[entry.kext]] requires type xnu and an array of tables", Origin); + Entry->KextConfigError = TRUE; + } else { + for (Item = List->Child; Item != NULL; Item = Item->Next) { + AddKext (Config, Item, Config->Volume, CA_CONFIG_DIR, Origin, FALSE, + &Entry->Kexts, &Entry->KextCount, &Entry->KextConfigError); + } + } + } + + if (TomlGetString (Table, "dtb", NULL) != NULL) { + if (AsciiStrCmp (TomlGetString (Table, "dtb", NULL), "firmware") == 0) { + Entry->Dtb = AllocateCopyPool (sizeof (L"firmware"), L"firmware"); + } else { + Entry->Dtb = MakePath (CA_CONFIG_DIR, TomlGetString (Table, "dtb", NULL)); + } + } + + List = TomlGet (Table, "dt_patch"); + for (Item = (List != NULL) ? List->Child : NULL; Item != NULL; Item = Item->Next) { + AddDtPatch (Config, Item, Config->Volume, CA_CONFIG_DIR, Origin, &Entry->Patches, &Entry->PatchCount); + } + + // + // [entry.chosen]: each key becomes a /chosen property, applied last. + // + List = TomlGet (Table, "chosen"); + for (Item = (List != NULL && List->Type == TomlTable) ? List->Child : NULL; Item != NULL; Item = Item->Next) { + VOID *Bytes; + UINTN Size; + + if (!EncodeFdtValue (Item, NULL, &Bytes, &Size)) { + ConfigError (Config, "%a: bad chosen value '%a'", Origin, Item->Key); + continue; + } + + Patch = GrowArray ((VOID **)&Entry->Patches, &Entry->PatchCount, sizeof (CA_DT_PATCH)); + if (Patch != NULL) { + Patch->Op = CaDtSet; + Patch->Node = AsciiDup ("/chosen"); + Patch->Prop = AsciiDup (Item->Key); + Patch->Value = Bytes; + Patch->ValueSize = Size; + Patch->Origin = AsciiDup (Origin); + } + } + + List = TomlGet (Table, "variable"); + for (Item = (List != NULL) ? List->Child : NULL; Item != NULL; Item = Item->Next) { + CA_VARIABLE Var; + CA_VARIABLE *Slot; + + ZeroMem (&Var, sizeof (Var)); + if ((TomlGetString (Item, "guid", NULL) == NULL) || (TomlGetString (Item, "name", NULL) == NULL) || + RETURN_ERROR (AsciiStrToGuid (TomlGetString (Item, "guid", NULL), &Var.Guid)) || + !EncodeVariableValue (TomlGet (Item, "value"), TomlGetString (Item, "type", NULL), &Var.Data, &Var.DataSize)) + { + ConfigError (Config, "%a: variable needs guid, name and a valid value (line %u)", Origin, (UINT32)Item->Line); + continue; + } + + Var.Name = CaAsciiToUnicode (TomlGetString (Item, "name", NULL)); + Var.Attributes = EFI_VARIABLE_BOOTSERVICE_ACCESS | EFI_VARIABLE_RUNTIME_ACCESS; + if (!TomlGetBoolean (Item, "volatile", TRUE)) { + Var.Attributes |= EFI_VARIABLE_NON_VOLATILE; + } + + Slot = GrowArray ((VOID **)&Entry->Variables, &Entry->VariableCount, sizeof (CA_VARIABLE)); + if (Slot != NULL) { + CopyMem (Slot, &Var, sizeof (Var)); + } + } +} + +// +// Add an entry for the removable-media loader of every other volume and +// return how many were found. +// +STATIC +UINTN +AddFallbackLoaders ( + IN CA_CONFIG *Config, + IN EFI_HANDLE ImageHandle + ) +{ + EFI_STATUS Status; + EFI_HANDLE *Handles; + UINTN Count; + UINTN Index; + EFI_LOADED_IMAGE_PROTOCOL *Self; + CHAR16 *SelfPath; + CA_ENTRY *Entry; + CA_ENTRY Saved[3]; + UINTN Actions; + UINTN Found; + CHAR16 Title[48]; + + Self = NULL; + SelfPath = NULL; + if (!EFI_ERROR (gBS->HandleProtocol (ImageHandle, &gEfiLoadedImageProtocolGuid, (VOID **)&Self)) && + (Self->FilePath != NULL)) + { + SelfPath = ConvertDevicePathToText (Self->FilePath, FALSE, FALSE); + } + + Status = gBS->LocateHandleBuffer (ByProtocol, &gEfiSimpleFileSystemProtocolGuid, NULL, &Count, &Handles); + if (EFI_ERROR (Status)) { + Count = 0; + Handles = NULL; + } + + // + // The built-in action entries stay at the end of the list. + // + Actions = MIN (Config->EntryCount, ARRAY_SIZE (Saved)); + CopyMem (Saved, Config->Entries + Config->EntryCount - Actions, Actions * sizeof (CA_ENTRY)); + Config->EntryCount -= Actions; + + Found = 0; + for (Index = 0; Index < Count; Index++) { + if (!CaFileExists (Handles[Index], CA_FALLBACK_LOADER)) { + continue; + } + + if ((Self != NULL) && (Handles[Index] == Self->DeviceHandle) && (SelfPath != NULL) && + (CaStriCmp (SelfPath, CA_FALLBACK_LOADER) == 0)) + { + continue; + } + + Entry = GrowArray ((VOID **)&Config->Entries, &Config->EntryCount, sizeof (CA_ENTRY)); + if (Entry == NULL) { + break; + } + + Found++; + UnicodeSPrint (Title, sizeof (Title), L"EFI loader on volume %u", (UINT32)Found); + Entry->Id = AsciiDup ("removable"); + Entry->Title = AllocateCopyPool (StrSize (Title), Title); + Entry->Type = CaEntryEfi; + Entry->Volume = Handles[Index]; + Entry->Path = AllocateCopyPool (sizeof (CA_FALLBACK_LOADER), CA_FALLBACK_LOADER); + } + + for (Index = 0; Index < Actions; Index++) { + Entry = GrowArray ((VOID **)&Config->Entries, &Config->EntryCount, sizeof (CA_ENTRY)); + if (Entry != NULL) { + CopyMem (Entry, &Saved[Index], sizeof (CA_ENTRY)); + } + } + + if (Handles != NULL) { + FreePool (Handles); + } + + if (SelfPath != NULL) { + FreePool (SelfPath); + } + + return Found; +} + +// --------------------------------------------------------------------------- +// Top level +// --------------------------------------------------------------------------- + +STATIC +VOID +ApplyRoot ( + IN CA_CONFIG *Config, + IN TOML_NODE *Root + ) +{ + TOML_NODE *Boot; + TOML_NODE *Ui; + TOML_NODE *List; + TOML_NODE *Item; + TOML_NODE *Acpi; + TOML_NODE *Default; + CONST CHAR8 *Console; + UINTN Index; + + if (TomlGetInteger (Root, "version", 0) != CA_SCHEMA_VERSION) { + ConfigError (Config, "expected 'version = %d'", CA_SCHEMA_VERSION); + } + + Boot = TomlGet (Root, "boot"); + Config->Timeout = (INTN)TomlGetInteger (Boot, "timeout", 5); + Config->RotateSeconds = (UINTN)MAX (0, TomlGetInteger (Boot, "rotate", 0)); + Config->FallbackTimeout = (UINTN)MAX (1, TomlGetInteger (Boot, "fallback_timeout", 10)); + Config->ConnectAll = TomlGetBoolean (Boot, "connect_all", TRUE); + + Ui = TomlGet (Root, "ui"); + Config->Title = CaAsciiToUnicode (TomlGetString (Ui, "title", "UniXNU")); + Config->Scale = (UINTN)MAX (0, TomlGetInteger (Ui, "scale", 0)); + Config->ShowLogo = TomlGetBoolean (Ui, "logo", TRUE); + Console = TomlGetString (Ui, "console", "auto"); + if (AsciiStrCmp (Console, "graphics") == 0) { + Config->Console = CaConsoleGraphics; + } else if (AsciiStrCmp (Console, "text") == 0) { + Config->Console = CaConsoleText; + } else if (AsciiStrCmp (Console, "none") == 0) { + Config->Console = CaConsoleNone; + } else { + Config->Console = CaConsoleAuto; + } + + List = TomlGet (Root, "driver"); + for (Item = (List != NULL) ? List->Child : NULL; Item != NULL; Item = Item->Next) { + AddDriver (Config, Item, Config->Volume, CA_CONFIG_DIR, "config", "path"); + } + + List = TomlGet (Root, "dt_patch"); + for (Item = (List != NULL) ? List->Child : NULL; Item != NULL; Item = Item->Next) { + AddDtPatch (Config, Item, Config->Volume, CA_CONFIG_DIR, "config", &Config->Patches, &Config->PatchCount); + } + + List = TomlGet (Root, "block_slice"); + for (Item = (List != NULL) ? List->Child : NULL; Item != NULL; Item = Item->Next) { + CA_BLOCK_SLICE *Slice; + CONST CHAR8 *Partition; + INT64 Offset; + INT64 Length; + + if (!TomlGetBoolean (Item, "enabled", TRUE)) { + continue; + } + + Partition = TomlGetString (Item, "partition", NULL); + Offset = TomlGetInteger (Item, "offset", 0); + Length = TomlGetInteger (Item, "length", 0); + if ((Partition == NULL) || (Offset < 0) || (Length < 0)) { + ConfigError (Config, "block_slice needs 'partition' and non-negative offset/length (line %u)", (UINT32)Item->Line); + continue; + } + + Slice = GrowArray ((VOID **)&Config->Slices, &Config->SliceCount, sizeof (CA_BLOCK_SLICE)); + if (Slice != NULL) { + Slice->Partition = CaAsciiToUnicode (Partition); + Slice->Name = CaAsciiToUnicode (TomlGetString (Item, "name", Partition)); + Slice->Offset = (UINT64)Offset; + Slice->Length = (UINT64)Length; + Slice->Writable = TomlGetBoolean (Item, "writable", FALSE); + } + } + + Acpi = TomlGet (TomlGet (Root, "acpi"), "add"); + for (Item = (Acpi != NULL) ? Acpi->Child : NULL; Item != NULL; Item = Item->Next) { + if (Item->Type == TomlString) { + AddAcpi (Config, Config->Volume, CA_CONFIG_DIR, Item->String, "config"); + } + } + + // + // Kits after the config's own drivers and patches, so a config can stage + // prerequisites (for example a file system driver) ahead of them. + // + LoadKits (Config, Root); + + List = TomlGet (Root, "entry"); + for (Item = (List != NULL) ? List->Child : NULL; Item != NULL; Item = Item->Next) { + AddEntry (Config, Item); + } + + Default = TomlGet (Boot, "default"); + Config->DefaultIndex = 0; + if ((Default != NULL) && (Default->Type == TomlString)) { + for (Index = 0; Index < Config->EntryCount; Index++) { + if ((Config->Entries[Index].Id != NULL) && (AsciiStrCmp (Config->Entries[Index].Id, Default->String) == 0)) { + Config->DefaultIndex = Index; + break; + } + } + + if (Index == Config->EntryCount) { + ConfigError (Config, "default entry '%a' not found", Default->String); + } + } else if ((Default != NULL) && (Default->Type == TomlInteger) && ((UINTN)Default->Integer < Config->EntryCount)) { + Config->DefaultIndex = (UINTN)Default->Integer; + } +} + +STATIC +BOOLEAN +TryVolume ( + IN CA_CONFIG *Config, + IN EFI_HANDLE Volume + ) +{ + EFI_STATUS Status; + CHAR8 *Text; + UINTN Size; + TOML_NODE *Root; + CHAR8 Error[96]; + + Status = CaReadFile (Volume, (CONST CHAR16 *)PcdGetPtr (PcdCrabAppleConfigPath), (VOID **)&Text, &Size); + if (EFI_ERROR (Status)) { + return FALSE; + } + + Config->Volume = Volume; + Status = TomlParse (Text, Size, &Root, Error, sizeof (Error)); + FreePool (Text); + if (EFI_ERROR (Status)) { + ConfigError (Config, "crabapple.toml %a", Error); + return FALSE; + } + + CaLog ("config: loaded %s", (CONST CHAR16 *)PcdGetPtr (PcdCrabAppleConfigPath)); + ApplyRoot (Config, Root); + TomlFree (Root); + return TRUE; +} + +VOID +CaConfigLoad ( + IN EFI_HANDLE ImageHandle, + OUT CA_CONFIG *Config + ) +{ + EFI_STATUS Status; + EFI_LOADED_IMAGE_PROTOCOL *Self; + EFI_HANDLE *Handles; + UINTN Count; + UINTN Index; + BOOLEAN Loaded; + TOML_NODE *Root; + CHAR8 Error[96]; + CHAR16 Label[64]; + + ZeroMem (Config, sizeof (*Config)); + Loaded = FALSE; + + // + // The volume UniXNU was started from wins; then every other volume. + // + if (!EFI_ERROR (gBS->HandleProtocol (ImageHandle, &gEfiLoadedImageProtocolGuid, (VOID **)&Self))) { + Loaded = TryVolume (Config, Self->DeviceHandle); + } + + if (!Loaded && (Config->Error[0] == '\0')) { + Status = gBS->LocateHandleBuffer (ByProtocol, &gEfiSimpleFileSystemProtocolGuid, NULL, &Count, &Handles); + if (!EFI_ERROR (Status)) { + for (Index = 0; Index < Count && !Loaded && Config->Error[0] == '\0'; Index++) { + Loaded = TryVolume (Config, Handles[Index]); + } + + FreePool (Handles); + } + } + + if (Loaded) { + UnicodeSPrint (Label, sizeof (Label), L"config: %s", (CONST CHAR16 *)PcdGetPtr (PcdCrabAppleConfigPath)); + } else { + // + // Built-in fallback. A parse error of a real file is kept in Error so the + // menu can show it. + // + CHAR8 Saved[sizeof (Config->Error)]; + + CopyMem (Saved, Config->Error, sizeof (Saved)); + ZeroMem (Config, sizeof (*Config)); + CopyMem (Config->Error, Saved, sizeof (Saved)); + + Status = TomlParse (mBuiltinConfig, sizeof (mBuiltinConfig) - 1, &Root, Error, sizeof (Error)); + ASSERT_EFI_ERROR (Status); + if (!EFI_ERROR (Status)) { + ApplyRoot (Config, Root); + TomlFree (Root); + } + + if (AddFallbackLoaders (Config, ImageHandle) == 0) { + // + // Only the actions are left. Say so instead of showing a menu that + // looks complete. + // + AsciiStrCpyS (Config->Notice, sizeof (Config->Notice), "no boot entry found"); + CaLog ("config: no boot entry found"); + } + + Config->DefaultIndex = 0; + UnicodeSPrint (Label, sizeof (Label), L"built-in configuration"); + CaLog ("config: none found, using built-in configuration"); + } + + Config->SourceLabel = AllocateCopyPool (StrSize (Label), Label); + + // + // A configuration without entries would leave nothing to show. + // + if (Config->EntryCount == 0) { + CA_ENTRY *Entry; + + ConfigError (Config, "no entries defined"); + Entry = GrowArray ((VOID **)&Config->Entries, &Config->EntryCount, sizeof (CA_ENTRY)); + if (Entry != NULL) { + Entry->Id = AsciiDup ("exit"); + Entry->Title = AllocateCopyPool (sizeof (L"Return to firmware"), L"Return to firmware"); + Entry->Type = CaEntryAction; + Entry->Action = CaActionExit; + } + } + + if (Config->DefaultIndex >= Config->EntryCount) { + Config->DefaultIndex = 0; + } +} diff --git a/package/CrabApplePkg/Library/CrabAppleLib/CrabAppleLib.inf b/package/CrabApplePkg/Library/CrabAppleLib/CrabAppleLib.inf new file mode 100644 index 0000000..a5cb721 --- /dev/null +++ b/package/CrabApplePkg/Library/CrabAppleLib/CrabAppleLib.inf @@ -0,0 +1,70 @@ +## @file +# UniXNU boot manager core, shared by the application and the DXE driver. +# +# Copyright (c) 2026, UniXNU contributors. All rights reserved. +# SPDX-License-Identifier: BSD-2-Clause-Patent +## + +[Defines] + INF_VERSION = 0x00010019 + BASE_NAME = CrabAppleLib + FILE_GUID = 2EA3E316-7EC4-4867-9FEA-ABE1D4E9DBA6 + MODULE_TYPE = UEFI_DRIVER + VERSION_STRING = 0.1 + LIBRARY_CLASS = CrabAppleLib|UEFI_APPLICATION UEFI_DRIVER DXE_DRIVER + +[Sources] + Internal.h + Boot.c + Config.c + Font.c + Inject.c + Kext.c + Logo.c + Main.c + Menu.c + Slice.c + Toml.c + Ui.c + Volume.c + +[Packages] + MdePkg/MdePkg.dec + CrabApplePkg/CrabApplePkg.dec + +[LibraryClasses] + BaseLib + BaseMemoryLib + DebugLib + DevicePathLib + FdtLib + MemoryAllocationLib + PcdLib + PrintLib + UefiBootServicesTableLib + UefiLib + UefiRuntimeServicesTableLib + +[Guids] + gEfiFileInfoGuid ## CONSUMES + gFdtTableGuid ## SOMETIMES_CONSUMES ## SOMETIMES_PRODUCES + gLinuxEfiInitrdMediaGuid ## SOMETIMES_PRODUCES + gCrabAppleBlockSliceGuid ## SOMETIMES_PRODUCES + +[Protocols] + gEfiAcpiTableProtocolGuid ## SOMETIMES_CONSUMES + gEfiBlockIoProtocolGuid ## SOMETIMES_CONSUMES ## SOMETIMES_PRODUCES + gEfiPartitionInfoProtocolGuid ## SOMETIMES_CONSUMES + gEfiDevicePathProtocolGuid ## CONSUMES + gEfiFirmwareVolume2ProtocolGuid ## SOMETIMES_CONSUMES + gEfiGraphicsOutputProtocolGuid ## SOMETIMES_CONSUMES + gEfiLoadedImageProtocolGuid ## CONSUMES + gEfiLoadFile2ProtocolGuid ## SOMETIMES_PRODUCES + gEfiSerialIoProtocolGuid ## SOMETIMES_CONSUMES + gEfiSimpleFileSystemProtocolGuid ## SOMETIMES_CONSUMES + gEfiSimpleTextInProtocolGuid ## SOMETIMES_CONSUMES + gCrabAppleXnuLoaderProtocolGuid ## SOMETIMES_CONSUMES + +[Pcd] + gCrabApplePkgTokenSpaceGuid.PcdCrabAppleDebugLevel + gCrabApplePkgTokenSpaceGuid.PcdCrabAppleConfigPath diff --git a/package/CrabApplePkg/Library/CrabAppleLib/Font.c b/package/CrabApplePkg/Library/CrabAppleLib/Font.c new file mode 100644 index 0000000..a338c1b --- /dev/null +++ b/package/CrabApplePkg/Library/CrabAppleLib/Font.c @@ -0,0 +1,110 @@ +/** @file + 8x19 bitmap font for printable ASCII (0x20..0x7E). + + Glyph data taken from the EDK II GraphicsConsoleDxe narrow font + (MdeModulePkg/Universal/Console/GraphicsConsoleDxe/LaffStd.c, + Copyright (c) 2006 - 2008, Intel Corporation. All rights reserved.), + BSD-2-Clause-Patent. Generated; do not edit by hand. + + SPDX-License-Identifier: BSD-2-Clause-Patent +**/ + +#include "Internal.h" + +CONST UINT8 gCaFontGlyphs[95][CA_GLYPH_HEIGHT] = { + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, // ' ' + { 0x00, 0x00, 0x00, 0x18, 0x3C, 0x3C, 0x3C, 0x18, 0x18, 0x18, 0x18, 0x18, 0x00, 0x18, 0x18, 0x00, 0x00, 0x00, 0x00 }, // '!' + { 0x00, 0x00, 0x00, 0x6C, 0x6C, 0x6C, 0x28, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, // '"' + { 0x00, 0x00, 0x00, 0x00, 0x6C, 0x6C, 0x6C, 0xFE, 0x6C, 0x6C, 0x6C, 0xFE, 0x6C, 0x6C, 0x6C, 0x00, 0x00, 0x00, 0x00 }, // '#' + { 0x00, 0x00, 0x18, 0x18, 0x7C, 0xC6, 0xC6, 0x60, 0x38, 0x0C, 0x06, 0xC6, 0xC6, 0x7C, 0x18, 0x18, 0x00, 0x00, 0x00 }, // '$' + { 0x00, 0x00, 0x00, 0xC6, 0xC6, 0x0C, 0x0C, 0x18, 0x18, 0x30, 0x30, 0x60, 0x60, 0xC6, 0xC6, 0x00, 0x00, 0x00, 0x00 }, // '%' + { 0x00, 0x00, 0x00, 0x78, 0xCC, 0xCC, 0xCC, 0x78, 0x76, 0xDC, 0xCC, 0xCC, 0xCC, 0xCC, 0x76, 0x00, 0x00, 0x00, 0x00 }, // '&' + { 0x00, 0x00, 0x18, 0x18, 0x18, 0x30, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, // "'" + { 0x00, 0x00, 0x00, 0x06, 0x0C, 0x0C, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x0C, 0x0C, 0x06, 0x00, 0x00, 0x00, 0x00 }, // '(' + { 0x00, 0x00, 0x00, 0xC0, 0x60, 0x60, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x60, 0x60, 0xC0, 0x00, 0x00, 0x00, 0x00 }, // ')' + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6C, 0x38, 0xFE, 0x38, 0x6C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, // '*' + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x18, 0x7E, 0x18, 0x18, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, // '+' + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x18, 0x18, 0x30, 0x00, 0x00, 0x00, 0x00 }, // ',' + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7E, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, // '-' + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x18, 0x00, 0x00, 0x00, 0x00, 0x00 }, // '.' + { 0x00, 0x00, 0x00, 0x06, 0x06, 0x0C, 0x0C, 0x18, 0x18, 0x30, 0x30, 0x60, 0x60, 0xC0, 0xC0, 0x00, 0x00, 0x00, 0x00 }, // '/' + { 0x00, 0x00, 0x00, 0x38, 0x6C, 0xC6, 0xC6, 0xC6, 0xD6, 0xD6, 0xC6, 0xC6, 0xC6, 0x6C, 0x38, 0x00, 0x00, 0x00, 0x00 }, // '0' + { 0x00, 0x00, 0x00, 0x18, 0x38, 0x78, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x7E, 0x00, 0x00, 0x00, 0x00 }, // '1' + { 0x00, 0x00, 0x00, 0x7C, 0xC6, 0x06, 0x06, 0x06, 0x0C, 0x18, 0x30, 0x60, 0xC0, 0xC2, 0xFE, 0x00, 0x00, 0x00, 0x00 }, // '2' + { 0x00, 0x00, 0x00, 0x7C, 0xC6, 0x06, 0x06, 0x06, 0x3C, 0x06, 0x06, 0x06, 0x06, 0xC6, 0x7C, 0x00, 0x00, 0x00, 0x00 }, // '3' + { 0x00, 0x00, 0x00, 0x1C, 0x1C, 0x3C, 0x3C, 0x6C, 0x6C, 0xCC, 0xFE, 0x0C, 0x0C, 0x0C, 0x1E, 0x00, 0x00, 0x00, 0x00 }, // '4' + { 0x00, 0x00, 0x00, 0xFE, 0xC0, 0xC0, 0xC0, 0xC0, 0xFC, 0x06, 0x06, 0x06, 0x06, 0xC6, 0x7C, 0x00, 0x00, 0x00, 0x00 }, // '5' + { 0x00, 0x00, 0x00, 0x3C, 0x60, 0xC0, 0xC0, 0xC0, 0xFC, 0xC6, 0xC6, 0xC6, 0xC6, 0xC6, 0x7C, 0x00, 0x00, 0x00, 0x00 }, // '6' + { 0x00, 0x00, 0x00, 0xFE, 0xC6, 0x06, 0x06, 0x06, 0x0C, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x00, 0x00, 0x00, 0x00 }, // '7' + { 0x00, 0x00, 0x00, 0x7C, 0xC6, 0xC6, 0xC6, 0xC6, 0x7C, 0xC6, 0xC6, 0xC6, 0xC6, 0xC6, 0x7C, 0x00, 0x00, 0x00, 0x00 }, // '8' + { 0x00, 0x00, 0x00, 0x7C, 0xC6, 0xC6, 0xC6, 0xC6, 0x7E, 0x06, 0x06, 0x06, 0x06, 0x0C, 0x78, 0x00, 0x00, 0x00, 0x00 }, // '9' + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x18, 0x00, 0x00, 0x00, 0x18, 0x18, 0x00, 0x00, 0x00, 0x00, 0x00 }, // ':' + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x18, 0x00, 0x00, 0x00, 0x18, 0x18, 0x30, 0x00, 0x00, 0x00, 0x00 }, // ';' + { 0x00, 0x00, 0x00, 0x00, 0x06, 0x0C, 0x18, 0x30, 0x60, 0xC0, 0x60, 0x30, 0x18, 0x0C, 0x06, 0x00, 0x00, 0x00, 0x00 }, // '<' + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xFE, 0x00, 0x00, 0xFE, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, // '=' + { 0x00, 0x00, 0x00, 0x00, 0xC0, 0x60, 0x30, 0x18, 0x0C, 0x06, 0x0C, 0x18, 0x30, 0x60, 0xC0, 0x00, 0x00, 0x00, 0x00 }, // '>' + { 0x00, 0x00, 0x00, 0x7C, 0xC6, 0xC6, 0x0C, 0x0C, 0x18, 0x18, 0x18, 0x00, 0x00, 0x18, 0x18, 0x00, 0x00, 0x00, 0x00 }, // '?' + { 0x00, 0x00, 0x00, 0x00, 0x7C, 0xC6, 0xC6, 0xC6, 0xDE, 0xDE, 0xDE, 0xDC, 0xC0, 0xC0, 0x7E, 0x00, 0x00, 0x00, 0x00 }, // '@' + { 0x00, 0x00, 0x00, 0x10, 0x38, 0x6C, 0xC6, 0xC6, 0xC6, 0xFE, 0xC6, 0xC6, 0xC6, 0xC6, 0xC6, 0x00, 0x00, 0x00, 0x00 }, // 'A' + { 0x00, 0x00, 0x00, 0xFC, 0x66, 0x66, 0x66, 0x66, 0x7C, 0x66, 0x66, 0x66, 0x66, 0x66, 0xFC, 0x00, 0x00, 0x00, 0x00 }, // 'B' + { 0x00, 0x00, 0x00, 0x3C, 0x66, 0xC2, 0xC0, 0xC0, 0xC0, 0xC0, 0xC0, 0xC0, 0xC2, 0x66, 0x3C, 0x00, 0x00, 0x00, 0x00 }, // 'C' + { 0x00, 0x00, 0x00, 0xF8, 0x6C, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x6C, 0xF8, 0x00, 0x00, 0x00, 0x00 }, // 'D' + { 0x00, 0x00, 0x00, 0xFE, 0x66, 0x62, 0x60, 0x68, 0x78, 0x68, 0x60, 0x60, 0x62, 0x66, 0xFE, 0x00, 0x00, 0x00, 0x00 }, // 'E' + { 0x00, 0x00, 0x00, 0xFE, 0x66, 0x62, 0x60, 0x64, 0x7C, 0x64, 0x60, 0x60, 0x60, 0x60, 0xF0, 0x00, 0x00, 0x00, 0x00 }, // 'F' + { 0x00, 0x00, 0x00, 0x3C, 0x66, 0xC2, 0xC0, 0xC0, 0xC0, 0xDE, 0xC6, 0xC6, 0xC6, 0x66, 0x3C, 0x00, 0x00, 0x00, 0x00 }, // 'G' + { 0x00, 0x00, 0x00, 0xC6, 0xC6, 0xC6, 0xC6, 0xC6, 0xFE, 0xC6, 0xC6, 0xC6, 0xC6, 0xC6, 0xC6, 0x00, 0x00, 0x00, 0x00 }, // 'H' + { 0x00, 0x00, 0x00, 0xFC, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0xFC, 0x00, 0x00, 0x00, 0x00 }, // 'I' + { 0x00, 0x00, 0x00, 0x1E, 0x0C, 0x0C, 0x0C, 0x0C, 0x0C, 0x0C, 0x0C, 0x0C, 0xCC, 0xCC, 0x78, 0x00, 0x00, 0x00, 0x00 }, // 'J' + { 0x00, 0x00, 0x00, 0xE6, 0x66, 0x6C, 0x6C, 0x78, 0x70, 0x78, 0x6C, 0x6C, 0x66, 0x66, 0xE6, 0x00, 0x00, 0x00, 0x00 }, // 'K' + { 0x00, 0x00, 0x00, 0xF0, 0x60, 0x60, 0x60, 0x60, 0x60, 0x60, 0x60, 0x60, 0x62, 0x66, 0xFE, 0x00, 0x00, 0x00, 0x00 }, // 'L' + { 0x00, 0x00, 0x00, 0xC6, 0xEE, 0xEE, 0xFE, 0xFE, 0xD6, 0xC6, 0xC6, 0xC6, 0xC6, 0xC6, 0xC6, 0x00, 0x00, 0x00, 0x00 }, // 'M' + { 0x00, 0x00, 0x00, 0xC6, 0xE6, 0xF6, 0xF6, 0xF6, 0xDE, 0xCE, 0xCE, 0xC6, 0xC6, 0xC6, 0xC6, 0x00, 0x00, 0x00, 0x00 }, // 'N' + { 0x00, 0x00, 0x00, 0x7C, 0xC6, 0xC6, 0xC6, 0xC6, 0xC6, 0xC6, 0xC6, 0xC6, 0xC6, 0xC6, 0x7C, 0x00, 0x00, 0x00, 0x00 }, // 'O' + { 0x00, 0x00, 0x00, 0xFC, 0x66, 0x66, 0x66, 0x66, 0x66, 0x7C, 0x60, 0x60, 0x60, 0x60, 0xF0, 0x00, 0x00, 0x00, 0x00 }, // 'P' + { 0x00, 0x00, 0x00, 0x7C, 0xC6, 0xC6, 0xC6, 0xC6, 0xC6, 0xC6, 0xC6, 0xC6, 0xD6, 0xD6, 0x7C, 0x1C, 0x0E, 0x00, 0x00 }, // 'Q' + { 0x00, 0x00, 0x00, 0xFC, 0x66, 0x66, 0x66, 0x66, 0x7C, 0x78, 0x6C, 0x6C, 0x66, 0x66, 0xE6, 0x00, 0x00, 0x00, 0x00 }, // 'R' + { 0x00, 0x00, 0x00, 0x7C, 0xC6, 0xC6, 0xC6, 0x60, 0x38, 0x0C, 0x06, 0x06, 0xC6, 0xC6, 0x7C, 0x00, 0x00, 0x00, 0x00 }, // 'S' + { 0x00, 0x00, 0x00, 0xFC, 0xFC, 0xB4, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x78, 0x00, 0x00, 0x00, 0x00 }, // 'T' + { 0x00, 0x00, 0x00, 0xC6, 0xC6, 0xC6, 0xC6, 0xC6, 0xC6, 0xC6, 0xC6, 0xC6, 0xC6, 0xC6, 0x7C, 0x00, 0x00, 0x00, 0x00 }, // 'U' + { 0x00, 0x00, 0x00, 0xC6, 0xC6, 0xC6, 0xC6, 0xC6, 0xC6, 0xC6, 0xC6, 0xC6, 0x6C, 0x38, 0x10, 0x00, 0x00, 0x00, 0x00 }, // 'V' + { 0x00, 0x00, 0x00, 0xC6, 0xC6, 0xC6, 0xC6, 0xC6, 0xC6, 0xD6, 0xD6, 0xD6, 0xFE, 0x6C, 0x6C, 0x00, 0x00, 0x00, 0x00 }, // 'W' + { 0x00, 0x00, 0x00, 0xC6, 0xC6, 0xC6, 0x6C, 0x6C, 0x38, 0x6C, 0x6C, 0xC6, 0xC6, 0xC6, 0xC6, 0x00, 0x00, 0x00, 0x00 }, // 'X' + { 0x00, 0x00, 0x00, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0x78, 0x30, 0x30, 0x30, 0x30, 0x30, 0x78, 0x00, 0x00, 0x00, 0x00 }, // 'Y' + { 0x00, 0x00, 0x00, 0xFE, 0xC6, 0x86, 0x0C, 0x0C, 0x18, 0x30, 0x60, 0xC0, 0xC2, 0xC6, 0xFE, 0x00, 0x00, 0x00, 0x00 }, // 'Z' + { 0x00, 0x00, 0x00, 0x1E, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x1E, 0x00, 0x00, 0x00, 0x00 }, // '[' + { 0x00, 0x00, 0x00, 0xC0, 0xC0, 0x60, 0x60, 0x30, 0x30, 0x18, 0x18, 0x0C, 0x0C, 0x06, 0x06, 0x00, 0x00, 0x00, 0x00 }, // '\\' + { 0x00, 0x00, 0x00, 0xF0, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0xF0, 0x00, 0x00, 0x00, 0x00 }, // ']' + { 0x00, 0x00, 0x10, 0x38, 0x6C, 0xC6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, // '^' + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xFE, 0x00, 0x00, 0x00, 0x00 }, // '_' + { 0x00, 0x30, 0x30, 0x18, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, // '`' + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x78, 0x0C, 0x0C, 0x7C, 0xCC, 0xCC, 0xCC, 0x76, 0x00, 0x00, 0x00, 0x00 }, // 'a' + { 0x00, 0x00, 0x00, 0xE0, 0x60, 0x60, 0x60, 0x7C, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x7C, 0x00, 0x00, 0x00, 0x00 }, // 'b' + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7C, 0xC6, 0xC0, 0xC0, 0xC0, 0xC0, 0xC6, 0x7C, 0x00, 0x00, 0x00, 0x00 }, // 'c' + { 0x00, 0x00, 0x00, 0x1C, 0x0C, 0x0C, 0x0C, 0x3C, 0x6C, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0x7E, 0x00, 0x00, 0x00, 0x00 }, // 'd' + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7C, 0xC6, 0xC6, 0xFE, 0xC0, 0xC0, 0xC6, 0x7C, 0x00, 0x00, 0x00, 0x00 }, // 'e' + { 0x00, 0x00, 0x00, 0x1E, 0x33, 0x30, 0x30, 0x30, 0x78, 0x30, 0x30, 0x30, 0x30, 0x30, 0x78, 0x00, 0x00, 0x00, 0x00 }, // 'f' + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x76, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0x7C, 0x0C, 0xCC, 0x78, 0x00 }, // 'g' + { 0x00, 0x00, 0x00, 0xE0, 0x60, 0x60, 0x60, 0x7C, 0x76, 0x66, 0x66, 0x66, 0x66, 0x66, 0xE6, 0x00, 0x00, 0x00, 0x00 }, // 'h' + { 0x00, 0x00, 0x00, 0x00, 0x18, 0x18, 0x00, 0x38, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x3C, 0x00, 0x00, 0x00, 0x00 }, // 'i' + { 0x00, 0x00, 0x00, 0x0C, 0x0C, 0x0C, 0x00, 0x1C, 0x0C, 0x0C, 0x0C, 0x0C, 0x0C, 0x0C, 0x0C, 0x0C, 0x6C, 0x38, 0x00 }, // 'j' + { 0x00, 0x00, 0x00, 0xE0, 0x60, 0x60, 0x66, 0x6C, 0x78, 0x70, 0x78, 0x6C, 0x6C, 0x66, 0xE6, 0x00, 0x00, 0x00, 0x00 }, // 'k' + { 0x00, 0x00, 0x00, 0x38, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x3C, 0x00, 0x00, 0x00, 0x00 }, // 'l' + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xEC, 0xEE, 0xFE, 0xD6, 0xD6, 0xD6, 0xD6, 0xD6, 0x00, 0x00, 0x00, 0x00 }, // 'm' + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xDC, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x00, 0x00, 0x00, 0x00 }, // 'n' + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7C, 0xC6, 0xC6, 0xC6, 0xC6, 0xC6, 0xC6, 0x7C, 0x00, 0x00, 0x00, 0x00 }, // 'o' + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xDC, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x7C, 0x60, 0x60, 0xF0, 0x00 }, // 'p' + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x76, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0x7C, 0x0C, 0x0C, 0x1E, 0x00 }, // 'q' + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xDC, 0x66, 0x60, 0x60, 0x60, 0x60, 0x60, 0xF0, 0x00, 0x00, 0x00, 0x00 }, // 'r' + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7C, 0xC6, 0xC0, 0x7C, 0x06, 0x06, 0xC6, 0x7C, 0x00, 0x00, 0x00, 0x00 }, // 's' + { 0x00, 0x00, 0x00, 0x10, 0x30, 0x30, 0x30, 0xFC, 0x30, 0x30, 0x30, 0x30, 0x30, 0x36, 0x1C, 0x00, 0x00, 0x00, 0x00 }, // 't' + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0x76, 0x00, 0x00, 0x00, 0x00 }, // 'u' + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0xCC, 0x78, 0x30, 0x00, 0x00, 0x00, 0x00 }, // 'v' + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xC6, 0xC6, 0xC6, 0xD6, 0xD6, 0xFE, 0xEE, 0x6C, 0x00, 0x00, 0x00, 0x00 }, // 'w' + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xC6, 0x6C, 0x38, 0x38, 0x6C, 0x6C, 0xC6, 0xC6, 0x00, 0x00, 0x00, 0x00 }, // 'x' + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xC6, 0xC6, 0xC6, 0xC6, 0xC6, 0xC6, 0xC6, 0x7E, 0x06, 0x0C, 0xF8, 0x00 }, // 'y' + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xFE, 0x86, 0x0C, 0x18, 0x30, 0x60, 0xC0, 0xFE, 0x00, 0x00, 0x00, 0x00 }, // 'z' + { 0x00, 0x00, 0x00, 0x0E, 0x18, 0x18, 0x18, 0x18, 0x30, 0x18, 0x18, 0x18, 0x18, 0x18, 0x0E, 0x00, 0x00, 0x00, 0x00 }, // '{' + { 0x00, 0x00, 0x00, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x00, 0x00, 0x00, 0x00 }, // '|' + { 0x00, 0x00, 0x00, 0xE0, 0x30, 0x30, 0x30, 0x30, 0x18, 0x30, 0x30, 0x30, 0x30, 0x30, 0xE0, 0x00, 0x00, 0x00, 0x00 }, // '}' + { 0x00, 0x00, 0x00, 0x76, 0xDC, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, // '~' +}; diff --git a/package/CrabApplePkg/Library/CrabAppleLib/Inject.c b/package/CrabApplePkg/Library/CrabAppleLib/Inject.c new file mode 100644 index 0000000..ac12557 --- /dev/null +++ b/package/CrabApplePkg/Library/CrabAppleLib/Inject.c @@ -0,0 +1,597 @@ +/** @file + Injection engine: driver loading, ACPI table installation, device tree + patching, Linux initrd media and entry variables. + + Copyright (c) 2026, UniXNU contributors. All rights reserved. + SPDX-License-Identifier: BSD-2-Clause-Patent +**/ + +#include "Internal.h" + +#include +#include +#include +#include +#include +#include + +// +// Room added to every DTB copy for properties and nodes UniXNU adds. +// +#define CA_FDT_SLACK SIZE_64KB + +// --------------------------------------------------------------------------- +// Drivers +// --------------------------------------------------------------------------- + +VOID +CaConnectAll ( + VOID + ) +{ + EFI_STATUS Status; + EFI_HANDLE *Handles; + UINTN Count; + UINTN Index; + + Status = gBS->LocateHandleBuffer (AllHandles, NULL, NULL, &Count, &Handles); + if (EFI_ERROR (Status)) { + return; + } + + for (Index = 0; Index < Count; Index++) { + gBS->ConnectController (Handles[Index], NULL, NULL, TRUE); + } + + FreePool (Handles); +} + +// +// The menu shows only how many drivers failed. Keep the first reason where it +// can be read, as on a machine without a serial console the log is out of reach. +// +STATIC +VOID +NoteDriverFailure ( + IN CA_CONFIG *Config, + IN CONST CHAR16 *File, + IN CONST CHAR8 *What, + IN EFI_STATUS Status + ) +{ + if (Config->Error[0] == '\0') { + AsciiSPrint (Config->Error, sizeof (Config->Error), "driver %s: %a: %r", File, What, Status); + } +} + +VOID +CaInjectDrivers ( + IN EFI_HANDLE ImageHandle, + IN CA_CONFIG *Config, + OUT CA_INJECT_STATS *Stats + ) +{ + EFI_STATUS Status; + UINTN Index; + CA_DRIVER *Driver; + EFI_DEVICE_PATH_PROTOCOL *Path; + EFI_HANDLE Image; + EFI_LOADED_IMAGE_PROTOCOL *Loaded; + BOOLEAN NeedConnect; + + NeedConnect = FALSE; + for (Index = 0; Index < Config->DriverCount; Index++) { + Driver = &Config->Drivers[Index]; + Path = CaResolveImagePath (Driver->Volume, Driver->File); + if (Path == NULL) { + CaLog ("driver %s (%a): cannot resolve path", Driver->File, Driver->Origin); + NoteDriverFailure (Config, Driver->File, "cannot resolve path", EFI_NOT_FOUND); + Stats->DriversFailed++; + continue; + } + + Status = gBS->LoadImage (FALSE, ImageHandle, Path, NULL, 0, &Image); + FreePool (Path); + if (EFI_ERROR (Status)) { + // + // LoadImage may hand back a handle on security failures; drop it. + // + if ((Status == EFI_SECURITY_VIOLATION) && (Image != NULL)) { + gBS->UnloadImage (Image); + } + + CaLog ("driver %s (%a): load failed: %r", Driver->File, Driver->Origin, Status); + NoteDriverFailure (Config, Driver->File, "load failed", Status); + Stats->DriversFailed++; + continue; + } + + if ((Driver->Args != NULL) && + !EFI_ERROR (gBS->HandleProtocol (Image, &gEfiLoadedImageProtocolGuid, (VOID **)&Loaded))) + { + Loaded->LoadOptions = Driver->Args; + Loaded->LoadOptionsSize = (UINT32)StrSize (Driver->Args); + } + + Status = gBS->StartImage (Image, NULL, NULL); + if (EFI_ERROR (Status)) { + CaLog ("driver %s (%a): start failed: %r", Driver->File, Driver->Origin, Status); + NoteDriverFailure (Config, Driver->File, "start failed", Status); + Stats->DriversFailed++; + continue; + } + + CaLog ("driver %s (%a): started", Driver->File, Driver->Origin); + Stats->DriversLoaded++; + NeedConnect = NeedConnect || Driver->Connect; + } + + // + // UEFI driver-model drivers only bind on ConnectController. + // + if (NeedConnect && Config->ConnectAll) { + CaConnectAll (); + } +} + +// --------------------------------------------------------------------------- +// ACPI +// --------------------------------------------------------------------------- + +VOID +CaInjectAcpi ( + IN CA_CONFIG *Config, + OUT CA_INJECT_STATS *Stats + ) +{ + EFI_STATUS Status; + EFI_ACPI_TABLE_PROTOCOL *AcpiTable; + UINTN Index; + VOID *Data; + UINTN Size; + UINTN Key; + + if (Config->AcpiCount == 0) { + return; + } + + Status = gBS->LocateProtocol (&gEfiAcpiTableProtocolGuid, NULL, (VOID **)&AcpiTable); + if (EFI_ERROR (Status)) { + CaLog ("acpi: no EFI_ACPI_TABLE_PROTOCOL, %u table(s) skipped", (UINT32)Config->AcpiCount); + Stats->AcpiFailed += Config->AcpiCount; + return; + } + + for (Index = 0; Index < Config->AcpiCount; Index++) { + Status = CaReadFile (Config->AcpiTables[Index].Volume, Config->AcpiTables[Index].File, &Data, &Size); + if (!EFI_ERROR (Status)) { + if ((Size < sizeof (EFI_ACPI_DESCRIPTION_HEADER)) || + (((EFI_ACPI_DESCRIPTION_HEADER *)Data)->Length != Size)) + { + Status = EFI_VOLUME_CORRUPTED; + } else { + Status = AcpiTable->InstallAcpiTable (AcpiTable, Data, Size, &Key); + } + + FreePool (Data); + } + + CaLog ("acpi %s (%a): %r", Config->AcpiTables[Index].File, Config->AcpiTables[Index].Origin, Status); + if (EFI_ERROR (Status)) { + Stats->AcpiFailed++; + } else { + Stats->AcpiInstalled++; + } + } +} + +// --------------------------------------------------------------------------- +// Device tree +// --------------------------------------------------------------------------- + +CONST CHAR8 * +CaFirmwareCompatible ( + OUT UINTN *Length + ) +{ + VOID *Fdt; + CONST CHAR8 *Value; + INT32 Len; + + if (EFI_ERROR (EfiGetSystemConfigurationTable (&gFdtTableGuid, &Fdt)) || (FdtCheckHeader (Fdt) != 0)) { + return NULL; + } + + Value = FdtGetProp (Fdt, 0, "compatible", &Len); + if ((Value == NULL) || (Len <= 0)) { + return NULL; + } + + *Length = (UINTN)Len; + return Value; +} + +// +// Offset of Path, creating missing nodes along the way. +// +STATIC +INT32 +EnsureNode ( + IN VOID *Fdt, + IN CONST CHAR8 *Path + ) +{ + INT32 Offset; + INT32 Parent; + CONST CHAR8 *Start; + CONST CHAR8 *End; + + Offset = FdtPathOffset (Fdt, Path); + if (Offset >= 0) { + return Offset; + } + + Parent = 0; + Start = Path; + while (*Start != '\0') { + while (*Start == '/') { + Start++; + } + + if (*Start == '\0') { + break; + } + + End = Start; + while (*End != '\0' && *End != '/') { + End++; + } + + Offset = FdtSubnodeOffsetNameLen (Fdt, Parent, Start, (INT32)(End - Start)); + if (Offset < 0) { + CHAR8 Name[128]; + UINTN Len = MIN ((UINTN)(End - Start), sizeof (Name) - 1); + + CopyMem (Name, Start, Len); + Name[Len] = '\0'; + Offset = FdtAddSubnode (Fdt, Parent, Name); + if (Offset < 0) { + return Offset; + } + } + + Parent = Offset; + Start = End; + } + + return Parent; +} + +EFI_STATUS +CaApplyDtPatches ( + IN OUT VOID *Fdt, + IN CA_DT_PATCH *Patches, + IN UINTN Count + ) +{ + UINTN Index; + CA_DT_PATCH *Patch; + INT32 Offset; + INT32 Result; + VOID *Overlay; + UINTN Failed; + + Failed = 0; + for (Index = 0; Index < Count; Index++) { + Patch = &Patches[Index]; + Result = 0; + switch (Patch->Op) { + case CaDtSet: + Offset = EnsureNode (Fdt, Patch->Node); + Result = (Offset < 0) ? Offset : FdtSetProp (Fdt, Offset, Patch->Prop, Patch->Value, (UINT32)Patch->ValueSize); + break; + + case CaDtDeleteProp: + Offset = FdtPathOffset (Fdt, Patch->Node); + Result = (Offset < 0) ? Offset : FdtDelProp (Fdt, Offset, Patch->Prop); + break; + + case CaDtAddNode: + Result = EnsureNode (Fdt, Patch->Node); + Result = (Result < 0) ? Result : 0; + break; + + case CaDtDeleteNode: + Offset = FdtPathOffset (Fdt, Patch->Node); + Result = (Offset < 0) ? Offset : FdtDelNode (Fdt, Offset); + break; + + case CaDtOverlay: + // + // fdt_overlay_apply() consumes the overlay; work on a copy so the + // entry can be booted again. + // + Overlay = AllocateCopyPool (Patch->ValueSize, Patch->Value); + if (Overlay == NULL) { + Result = -1; + break; + } + + Result = FdtOverlayApply (Fdt, Overlay); + FreePool (Overlay); + break; + } + + if (Result < 0) { + Failed++; + CaLog ( + "dt: %a: op %u on %a%a%a failed: %a", + Patch->Origin, + (UINT32)Patch->Op, + (Patch->Node != NULL) ? Patch->Node : "(overlay)", + (Patch->Prop != NULL) ? ":" : "", + (Patch->Prop != NULL) ? Patch->Prop : "", + FdtStrerror (Result) + ); + } + } + + return (Failed == 0) ? EFI_SUCCESS : EFI_PROTOCOL_ERROR; +} + +EFI_STATUS +CaPrepareDeviceTree ( + IN CA_CONFIG *Config, + IN CA_ENTRY *Entry, + OUT VOID **Fdt OPTIONAL, + OUT UINTN *FdtSize OPTIONAL + ) +{ + EFI_STATUS Status; + VOID *Base; + UINTN BaseSize; + BOOLEAN FreeBase; + UINTN Size; + UINTN Index; + EFI_PHYSICAL_ADDRESS Address; + VOID *Tree; + + if (Entry->Dtb == NULL) { + return EFI_SUCCESS; + } + + FreeBase = FALSE; + if (StrCmp (Entry->Dtb, L"firmware") == 0) { + Status = EfiGetSystemConfigurationTable (&gFdtTableGuid, &Base); + if (EFI_ERROR (Status) || (FdtCheckHeader (Base) != 0)) { + CaLog ("dt: entry wants the firmware DTB but none is installed"); + return EFI_NOT_FOUND; + } + + BaseSize = Fdt32ToCpu (((FDT_HEADER *)Base)->TotalSize); + } else { + Status = CaReadFile (Entry->Volume, Entry->Dtb, &Base, &BaseSize); + if (EFI_ERROR (Status)) { + CaLog ("dt: cannot read %s: %r", Entry->Dtb, Status); + return Status; + } + + FreeBase = TRUE; + if (FdtCheckHeader (Base) != 0) { + CaLog ("dt: %s is not a flattened device tree", Entry->Dtb); + FreePool (Base); + return EFI_VOLUME_CORRUPTED; + } + } + + // + // Overlays need room for their contents in the base tree. + // + Size = BaseSize + CA_FDT_SLACK; + for (Index = 0; Index < Config->PatchCount; Index++) { + Size += Config->Patches[Index].ValueSize; + } + + for (Index = 0; Index < Entry->PatchCount; Index++) { + Size += Entry->Patches[Index].ValueSize; + } + + // + // ACPI reclaim memory survives ExitBootServices(), as the DTB table must. + // + Status = gBS->AllocatePages (AllocateAnyPages, EfiACPIReclaimMemory, EFI_SIZE_TO_PAGES (Size), &Address); + if (EFI_ERROR (Status)) { + if (FreeBase) { + FreePool (Base); + } + + return Status; + } + + Tree = (VOID *)(UINTN)Address; + if (FdtOpenInto (Base, Tree, (INT32)Size) != 0) { + Status = EFI_VOLUME_CORRUPTED; + } + + if (FreeBase) { + FreePool (Base); + } + + if (!EFI_ERROR (Status)) { + // + // Global (config + kits) first, then the entry's own, /chosen last. + // + CaApplyDtPatches (Tree, Config->Patches, Config->PatchCount); + CaApplyDtPatches (Tree, Entry->Patches, Entry->PatchCount); + FdtPack (Tree); + Status = gBS->InstallConfigurationTable (&gFdtTableGuid, Tree); + } + + if (EFI_ERROR (Status)) { + gBS->FreePages (Address, EFI_SIZE_TO_PAGES (Size)); + return Status; + } + + CaLog ( + "dt: installed %u bytes, %u global + %u entry patch(es)", + Fdt32ToCpu (((FDT_HEADER *)Tree)->TotalSize), + (UINT32)Config->PatchCount, + (UINT32)Entry->PatchCount + ); + + if (Fdt != NULL) { + *Fdt = Tree; + } + + if (FdtSize != NULL) { + *FdtSize = Fdt32ToCpu (((FDT_HEADER *)Tree)->TotalSize); + } + + return EFI_SUCCESS; +} + +// --------------------------------------------------------------------------- +// Linux initrd media (LINUX_EFI_INITRD_MEDIA_GUID + EFI_LOAD_FILE2_PROTOCOL) +// --------------------------------------------------------------------------- + +#pragma pack (1) +typedef struct { + VENDOR_DEVICE_PATH VenMedia; + EFI_DEVICE_PATH_PROTOCOL End; +} INITRD_DEVICE_PATH; +#pragma pack () + +typedef struct { + EFI_LOAD_FILE2_PROTOCOL LoadFile2; + VOID *Data; + UINTN Size; +} INITRD_CONTEXT; + +STATIC CONST INITRD_DEVICE_PATH mInitrdDevicePath = { + { + { MEDIA_DEVICE_PATH, MEDIA_VENDOR_DP, { sizeof (VENDOR_DEVICE_PATH), 0 } + }, + LINUX_EFI_INITRD_MEDIA_GUID + }, + { + END_DEVICE_PATH_TYPE, END_ENTIRE_DEVICE_PATH_SUBTYPE, { sizeof (EFI_DEVICE_PATH_PROTOCOL), 0 } + } +}; + +STATIC INITRD_CONTEXT mInitrd; + +STATIC +EFI_STATUS +EFIAPI +InitrdLoadFile2 ( + IN EFI_LOAD_FILE2_PROTOCOL *This, + IN EFI_DEVICE_PATH_PROTOCOL *FilePath, + IN BOOLEAN BootPolicy, + IN OUT UINTN *BufferSize, + IN VOID *Buffer OPTIONAL + ) +{ + INITRD_CONTEXT *Context; + + Context = BASE_CR (This, INITRD_CONTEXT, LoadFile2); + if (BootPolicy) { + return EFI_UNSUPPORTED; + } + + if ((BufferSize == NULL) || !IsDevicePathValid (FilePath, 0)) { + return EFI_INVALID_PARAMETER; + } + + if ((FilePath->Type != END_DEVICE_PATH_TYPE) || (FilePath->SubType != END_ENTIRE_DEVICE_PATH_SUBTYPE)) { + return EFI_NOT_FOUND; + } + + if ((Buffer == NULL) || (*BufferSize < Context->Size)) { + *BufferSize = Context->Size; + return EFI_BUFFER_TOO_SMALL; + } + + CopyMem (Buffer, Context->Data, Context->Size); + *BufferSize = Context->Size; + return EFI_SUCCESS; +} + +EFI_STATUS +CaInstallInitrd ( + IN VOID *Data, + IN UINTN Size, + OUT EFI_HANDLE *Handle + ) +{ + EFI_STATUS Status; + EFI_DEVICE_PATH_PROTOCOL *Path; + EFI_HANDLE Existing; + + // + // Only one provider may exist; refuse to shadow somebody else's. + // + Path = (EFI_DEVICE_PATH_PROTOCOL *)&mInitrdDevicePath; + Status = gBS->LocateDevicePath (&gEfiLoadFile2ProtocolGuid, &Path, &Existing); + if (!EFI_ERROR (Status) && IsDevicePathEnd (Path)) { + return EFI_ALREADY_STARTED; + } + + mInitrd.LoadFile2.LoadFile = InitrdLoadFile2; + mInitrd.Data = Data; + mInitrd.Size = Size; + *Handle = NULL; + return gBS->InstallMultipleProtocolInterfaces ( + Handle, + &gEfiDevicePathProtocolGuid, + &mInitrdDevicePath, + &gEfiLoadFile2ProtocolGuid, + &mInitrd.LoadFile2, + NULL + ); +} + +VOID +CaUninstallInitrd ( + IN EFI_HANDLE Handle + ) +{ + if (Handle == NULL) { + return; + } + + gBS->UninstallMultipleProtocolInterfaces ( + Handle, + &gEfiDevicePathProtocolGuid, + &mInitrdDevicePath, + &gEfiLoadFile2ProtocolGuid, + &mInitrd.LoadFile2, + NULL + ); +} + +// --------------------------------------------------------------------------- +// Variables +// --------------------------------------------------------------------------- + +EFI_STATUS +CaSetEntryVariables ( + IN CA_ENTRY *Entry + ) +{ + EFI_STATUS Status; + EFI_STATUS Result; + UINTN Index; + CA_VARIABLE *Var; + + Result = EFI_SUCCESS; + for (Index = 0; Index < Entry->VariableCount; Index++) { + Var = &Entry->Variables[Index]; + Status = gRT->SetVariable (Var->Name, &Var->Guid, Var->Attributes, Var->DataSize, Var->Data); + CaLog ("variable %g:%s (%u bytes): %r", &Var->Guid, Var->Name, (UINT32)Var->DataSize, Status); + if (EFI_ERROR (Status)) { + Result = Status; + } + } + + return Result; +} diff --git a/package/CrabApplePkg/Library/CrabAppleLib/Internal.h b/package/CrabApplePkg/Library/CrabAppleLib/Internal.h new file mode 100644 index 0000000..d6e44b7 --- /dev/null +++ b/package/CrabApplePkg/Library/CrabAppleLib/Internal.h @@ -0,0 +1,548 @@ +/** @file + CrabAppleLib internal definitions. + + Copyright (c) 2026, UniXNU contributors. All rights reserved. + SPDX-License-Identifier: BSD-2-Clause-Patent +**/ + +#ifndef CRABAPPLE_INTERNAL_H_ +#define CRABAPPLE_INTERNAL_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define CA_VERSION_STRING "0.1.0" + +// +// Schema version understood by this build (`version = N` in the config, +// `schema = N` in a kit manifest). +// +#define CA_SCHEMA_VERSION 1 + +// --------------------------------------------------------------------------- +// Logging. Every line goes to DEBUG() and, when the platform exposes one, to +// the first EFI_SERIAL_IO_PROTOCOL instance. Lines are prefixed "CRABAPPLE: " +// so test harnesses can grep for them. +// --------------------------------------------------------------------------- + +VOID +CaLogInit ( + VOID + ); + +VOID +EFIAPI +CaLog ( + IN CONST CHAR8 *Format, + ... + ); + +// --------------------------------------------------------------------------- +// TOML subset (Toml.c) +// --------------------------------------------------------------------------- + +typedef enum { + TomlString, + TomlInteger, + TomlBoolean, + TomlArray, + TomlTable +} TOML_TYPE; + +typedef struct TOML_NODE TOML_NODE; + +struct TOML_NODE { + TOML_TYPE Type; + CHAR8 *Key; ///< Member name inside a table; NULL for array items. + UINTN Line; ///< Source line, for diagnostics. + CHAR8 *String; + INT64 Integer; + BOOLEAN Boolean; + BOOLEAN Defined; ///< Table was opened by an explicit [header]. + BOOLEAN TableArray; ///< Array created by [[header]]. + TOML_NODE *Child; ///< First member / item. + TOML_NODE *LastChild; + TOML_NODE *Next; +}; + +/** + Parse TOML text. On failure *Root is NULL and ErrorBuffer holds a message. +**/ +EFI_STATUS +TomlParse ( + IN CONST CHAR8 *Text, + IN UINTN Length, + OUT TOML_NODE **Root, + OUT CHAR8 *ErrorBuffer, + IN UINTN ErrorBufferSize + ); + +VOID +TomlFree ( + IN TOML_NODE *Node + ); + +TOML_NODE * +TomlGet ( + IN TOML_NODE *Table, + IN CONST CHAR8 *Key + ); + +CONST CHAR8 * +TomlGetString ( + IN TOML_NODE *Table, + IN CONST CHAR8 *Key, + IN CONST CHAR8 *Default + ); + +INT64 +TomlGetInteger ( + IN TOML_NODE *Table, + IN CONST CHAR8 *Key, + IN INT64 Default + ); + +BOOLEAN +TomlGetBoolean ( + IN TOML_NODE *Table, + IN CONST CHAR8 *Key, + IN BOOLEAN Default + ); + +UINTN +TomlCount ( + IN TOML_NODE *Array + ); + +// --------------------------------------------------------------------------- +// Configuration model (Config.c) +// --------------------------------------------------------------------------- + +typedef enum { + CaDtSet, + CaDtDeleteProp, + CaDtAddNode, + CaDtDeleteNode, + CaDtOverlay +} CA_DT_OP; + +typedef struct { + CA_DT_OP Op; + CHAR8 *Node; ///< Absolute node path. + CHAR8 *Prop; ///< Property name (set / delete-prop). + VOID *Value; ///< Encoded property bytes (set), overlay blob (overlay). + UINTN ValueSize; + CHAR8 *Origin; ///< "config", "entry " or "kit ", for logs. +} CA_DT_PATCH; + +typedef struct { + EFI_HANDLE Volume; ///< Volume holding File. + CHAR16 *File; + CHAR16 *Args; + BOOLEAN Connect; + CHAR8 *Origin; +} CA_DRIVER; + +typedef struct { + EFI_HANDLE Volume; + CHAR16 *File; + CHAR8 *Origin; +} CA_ACPI_TABLE; + +typedef struct { + EFI_HANDLE Volume; + CHAR8 *EntryId; ///< Required for kit kexts; NULL for entry kexts. + CHAR16 *BundlePath; + CHAR16 *PlistPath; + CHAR16 *ExecutablePath; + BOOLEAN Invalid; +} CA_KEXT; + +typedef struct { + EFI_GUID Guid; + CHAR16 *Name; + VOID *Data; + UINTN DataSize; + UINT32 Attributes; +} CA_VARIABLE; + +typedef struct { + CHAR16 *Partition; ///< GPT partition name. + CHAR16 *Name; ///< Label, also stored in the device path. + UINT64 Offset; ///< Bytes from the start of the partition. + UINT64 Length; ///< Bytes; 0 = to the end of the partition. + BOOLEAN Writable; +} CA_BLOCK_SLICE; + +typedef enum { + CaEntryEfi, + CaEntryLinux, + CaEntryAction, + CaEntryXnu +} CA_ENTRY_TYPE; + +typedef enum { + CaActionNone, + CaActionReset, + CaActionShutdown, + CaActionExit +} CA_ACTION; + +typedef struct { + CHAR8 *Id; + CHAR16 *Title; + CA_ENTRY_TYPE Type; + CA_ACTION Action; + EFI_HANDLE Volume; ///< Volume for Path/Dtb/Initrd. + CHAR16 *Path; ///< "\\EFI\\x.efi" or "fv:". + CHAR16 *Args; + CHAR16 *Dtb; ///< NULL, L"firmware" or a file path. + CHAR16 *Initrd; + CA_DT_PATCH *Patches; ///< Entry patches, chosen properties last. + UINTN PatchCount; + CA_VARIABLE *Variables; + UINTN VariableCount; + CA_KEXT *Kexts; + UINTN KextCount; + BOOLEAN KextConfigError; +} CA_ENTRY; + +typedef enum { + CaConsoleAuto, + CaConsoleGraphics, + CaConsoleText, + CaConsoleNone +} CA_CONSOLE; + +typedef struct { + // + // [boot] + // + INTN Timeout; ///< Seconds; <0 waits for input; 0 boots at once. + UINTN DefaultIndex; + UINTN RotateSeconds; ///< Idle highlight rotation step, 0 = off. + UINTN FallbackTimeout; ///< Used when Timeout < 0 and no input exists. + BOOLEAN ConnectAll; ///< ConnectController all handles after drivers. + // + // [ui] + // + CA_CONSOLE Console; + CHAR16 *Title; + UINTN Scale; ///< 0 = automatic. + BOOLEAN ShowLogo; + // + // Injection, after kits are merged in. + // + CA_DRIVER *Drivers; + UINTN DriverCount; + CA_DT_PATCH *Patches; ///< Global patches (config + kits). + UINTN PatchCount; + CA_ACPI_TABLE *AcpiTables; + UINTN AcpiCount; + CA_BLOCK_SLICE *Slices; + UINTN SliceCount; + CA_KEXT *Kexts; ///< Kit kexts, selected by entry id. + UINTN KextCount; + BOOLEAN KextConfigError; + CA_ENTRY *Entries; + UINTN EntryCount; + // + // Provenance and diagnostics shown on screen. + // + EFI_HANDLE Volume; ///< Volume the config came from (NULL = built-in). + CHAR16 *SourceLabel; + CHAR8 Error[160]; ///< First config/kit error, empty if none. + CHAR8 Notice[96]; ///< Note for the menu that is not an error, empty if none. + UINTN KitsLoaded; + UINTN KitsSkipped; +} CA_CONFIG; + +/** + Find a config file on the boot volume or any other volume and parse it; + fall back to the built-in configuration. Never fails. +**/ +VOID +CaConfigLoad ( + IN EFI_HANDLE ImageHandle, + OUT CA_CONFIG *Config + ); + +// --------------------------------------------------------------------------- +// Volumes and files (Volume.c) +// --------------------------------------------------------------------------- + +EFI_STATUS +CaReadFile ( + IN EFI_HANDLE Volume, + IN CONST CHAR16 *Path, + OUT VOID **Buffer, + OUT UINTN *Size + ); + +BOOLEAN +CaFileExists ( + IN EFI_HANDLE Volume, + IN CONST CHAR16 *Path + ); + +/** + List the names of the entries of a directory. Caller frees each name and + the array. +**/ +EFI_STATUS +CaListDirectory ( + IN EFI_HANDLE Volume, + IN CONST CHAR16 *Path, + OUT CHAR16 ***Names, + OUT UINTN *Count + ); + +/** + Turn "a/b", "\\a\\b" or "/a/b" into a newly allocated "\\a\\b". When Base is + given and Path is relative, the result is Base\\Path. +**/ +CHAR16 * +CaNormalizePath ( + IN CONST CHAR16 *Base OPTIONAL, + IN CONST CHAR8 *Path + ); + +/** + ASCII case-insensitive CHAR16 comparison (FAT names, fallback paths). +**/ +INTN +CaStriCmp ( + IN CONST CHAR16 *A, + IN CONST CHAR16 *B + ); + +CHAR16 * +CaAsciiToUnicode ( + IN CONST CHAR8 *String + ); + +/** + Build the device path for "\\file" on Volume or "fv:" in any FV. +**/ +EFI_DEVICE_PATH_PROTOCOL * +CaResolveImagePath ( + IN EFI_HANDLE Volume, + IN CONST CHAR16 *Path + ); + +// --------------------------------------------------------------------------- +// Injection (Inject.c) +// --------------------------------------------------------------------------- + +typedef struct { + UINTN DriversLoaded; + UINTN DriversFailed; + UINTN AcpiInstalled; + UINTN AcpiFailed; +} CA_INJECT_STATS; + +/** + ConnectController() every handle, recursively. +**/ +VOID +CaConnectAll ( + VOID + ); + +VOID +CaInjectDrivers ( + IN EFI_HANDLE ImageHandle, + IN CA_CONFIG *Config, + OUT CA_INJECT_STATS *Stats + ); + +VOID +CaInjectAcpi ( + IN CA_CONFIG *Config, + OUT CA_INJECT_STATS *Stats + ); + +EFI_STATUS +CaStageKexts ( + IN CA_CONFIG *Config, + IN CA_ENTRY *Entry, + OUT CRABAPPLE_XNU_KEXT **Kexts, + OUT UINTN *Count + ); + +VOID +CaFreeKexts ( + IN CRABAPPLE_XNU_KEXT *Kexts OPTIONAL, + IN UINTN Count + ); + +/** + Apply DT patches to a writable FDT that has room to grow. +**/ +EFI_STATUS +CaApplyDtPatches ( + IN OUT VOID *Fdt, + IN CA_DT_PATCH *Patches, + IN UINTN Count + ); + +/** + Build the entry's DTB (base + global patches + entry patches) and install it + as the EFI_DTB_TABLE configuration table. No-op when the entry has no dtb. +**/ +EFI_STATUS +CaPrepareDeviceTree ( + IN CA_CONFIG *Config, + IN CA_ENTRY *Entry, + OUT VOID **Fdt OPTIONAL, + OUT UINTN *FdtSize OPTIONAL + ); + +EFI_STATUS +CaInstallInitrd ( + IN VOID *Data, + IN UINTN Size, + OUT EFI_HANDLE *Handle + ); + +VOID +CaUninstallInitrd ( + IN EFI_HANDLE Handle + ); + +EFI_STATUS +CaSetEntryVariables ( + IN CA_ENTRY *Entry + ); + +/** + Return the root "compatible" list of the firmware's DTB table, or NULL. +**/ +CONST CHAR8 * +CaFirmwareCompatible ( + OUT UINTN *Length + ); + +// --------------------------------------------------------------------------- +// Block slices (Slice.c) +// --------------------------------------------------------------------------- + +/** + Publish every configured [[block_slice]] as a block device. +**/ +VOID +CaPublishSlices ( + IN CA_CONFIG *Config + ); + +// --------------------------------------------------------------------------- +// Boot (Boot.c) +// --------------------------------------------------------------------------- + +EFI_STATUS +CaBootEntry ( + IN EFI_HANDLE ImageHandle, + IN CA_CONFIG *Config, + IN CA_ENTRY *Entry + ); + +// --------------------------------------------------------------------------- +// User interface (Ui.c, Font.c, Logo.c) +// --------------------------------------------------------------------------- + +#define CA_GLYPH_WIDTH 8 +#define CA_GLYPH_HEIGHT 19 + +extern CONST UINT8 gCaFontGlyphs[95][CA_GLYPH_HEIGHT]; + +typedef enum { + CaUiGraphics, + CaUiText, + CaUiHeadless +} CA_UI_MODE; + +typedef struct { + CA_UI_MODE Mode; + EFI_GRAPHICS_OUTPUT_PROTOCOL *Gop; + UINT32 Width; + UINT32 Height; + UINTN Scale; + UINTN MenuTop; + UINTN RowHeight; + UINTN MenuLeft; + UINTN MenuWidth; + UINTN FooterTop; + UINTN VisibleRows; + UINTN TextColumns; ///< Text mode only. + UINTN TextRows; + CHAR16 Info[96]; ///< Injection summary line. + CHAR16 Notice[96]; ///< Result of the last boot attempt. +} CA_UI; + +VOID +CaUiInit ( + IN CA_CONFIG *Config, + OUT CA_UI *Ui + ); + +VOID +CaUiDrawFrame ( + IN CA_UI *Ui, + IN CA_CONFIG *Config + ); + +VOID +CaUiDrawEntry ( + IN CA_UI *Ui, + IN CA_CONFIG *Config, + IN UINTN Index, + IN BOOLEAN Selected + ); + +VOID +CaUiDrawStatus ( + IN CA_UI *Ui, + IN UINTN Line, + IN CONST CHAR16 *Text + ); + +VOID +CaLogoRender ( + OUT EFI_GRAPHICS_OUTPUT_BLT_PIXEL *Buffer, + IN UINTN Size, + IN EFI_GRAPHICS_OUTPUT_BLT_PIXEL Background + ); + +// --------------------------------------------------------------------------- +// Menu (Menu.c) +// --------------------------------------------------------------------------- + +/** + Show the menu and return the chosen entry index. + + @param[in] Countdown FALSE disables the timeout (after a failed boot). +**/ +UINTN +CaMenuRun ( + IN CA_UI *Ui, + IN CA_CONFIG *Config, + IN BOOLEAN Countdown + ); + +#endif diff --git a/package/CrabApplePkg/Library/CrabAppleLib/Kext.c b/package/CrabApplePkg/Library/CrabAppleLib/Kext.c new file mode 100644 index 0000000..aece484 --- /dev/null +++ b/package/CrabApplePkg/Library/CrabAppleLib/Kext.c @@ -0,0 +1,215 @@ +/** @file + Read-only kext staging for external XNU loaders. + + Copyright (c) 2026, UniXNU contributors. All rights reserved. + SPDX-License-Identifier: BSD-2-Clause-Patent +**/ + +#include "Internal.h" + +STATIC +BOOLEAN +KextImageValid ( + IN CONST VOID *Data, + IN UINTN Size + ) +{ + CONST UINT8 *Bytes; + UINT32 Commands; + UINT32 CommandSize; + UINT32 Subtype; + UINTN Offset; + UINTN End; + UINTN Index; + + Bytes = Data; + if ((Size < 32) || (ReadUnaligned32 ((CONST UINT32 *)Bytes) != 0xFEEDFACF) || + (ReadUnaligned32 ((CONST UINT32 *)(Bytes + 4)) != 0x0100000C) || + (ReadUnaligned32 ((CONST UINT32 *)(Bytes + 12)) != 11)) + { + return FALSE; + } + + // Accept thin arm64/arm64e bundles. A loader must enforce its kernel ABI. + Subtype = ReadUnaligned32 ((CONST UINT32 *)(Bytes + 8)) & 0x00FFFFFF; + if ((Subtype != 0) && (Subtype != 2)) { + return FALSE; + } + + Commands = ReadUnaligned32 ((CONST UINT32 *)(Bytes + 16)); + CommandSize = ReadUnaligned32 ((CONST UINT32 *)(Bytes + 20)); + if ((CommandSize > Size - 32) || (Commands > CommandSize / 8)) { + return FALSE; + } + + End = 32 + CommandSize; + Offset = 32; + for (Index = 0; Index < Commands; Index++) { + if (End - Offset < 8) { + return FALSE; + } + + CommandSize = ReadUnaligned32 ((CONST UINT32 *)(Bytes + Offset + 4)); + if ((CommandSize < 8) || ((CommandSize & 7) != 0) || (CommandSize > End - Offset)) { + return FALSE; + } + + Offset += CommandSize; + } + + return Offset == End; +} + +VOID +CaFreeKexts ( + IN CRABAPPLE_XNU_KEXT *Kexts OPTIONAL, + IN UINTN Count + ) +{ + UINTN Index; + + if (Kexts == NULL) { + return; + } + + for (Index = 0; Index < Count; Index++) { + if (Kexts[Index].InfoPlist != NULL) { + FreePool ((VOID *)Kexts[Index].InfoPlist); + } + + if (Kexts[Index].Executable != NULL) { + FreePool ((VOID *)Kexts[Index].Executable); + } + } + + FreePool (Kexts); +} + +STATIC +BOOLEAN +KitKextMatches ( + IN CA_KEXT *Kext, + IN CA_ENTRY *Entry + ) +{ + return (Kext->EntryId != NULL) && (Entry->Id != NULL) && + (AsciiStrCmp (Kext->EntryId, Entry->Id) == 0); +} + +EFI_STATUS +CaStageKexts ( + IN CA_CONFIG *Config, + IN CA_ENTRY *Entry, + OUT CRABAPPLE_XNU_KEXT **Kexts, + OUT UINTN *Count + ) +{ + EFI_STATUS Status; + UINTN Total; + UINTN Index; + UINTN Used; + UINTN Previous; + CA_KEXT *Source; + CRABAPPLE_XNU_KEXT *Out; + VOID *Data; + UINTN Size; + + *Kexts = NULL; + *Count = 0; + if (Config->KextConfigError || Entry->KextConfigError) { + CaLog ("kext: invalid configuration; boot refused"); + return EFI_INVALID_PARAMETER; + } + + Total = Entry->KextCount; + for (Index = 0; Index < Config->KextCount; Index++) { + if (Config->Kexts[Index].Invalid || KitKextMatches (&Config->Kexts[Index], Entry)) { + Total++; + } + } + + if (Total == 0) { + return EFI_SUCCESS; + } + + Out = AllocateZeroPool (Total * sizeof (*Out)); + if (Out == NULL) { + return EFI_OUT_OF_RESOURCES; + } + + Used = 0; + Status = EFI_SUCCESS; + for (Index = 0; Index < Config->KextCount + Entry->KextCount; Index++) { + if (Index < Config->KextCount) { + Source = &Config->Kexts[Index]; + if (!Source->Invalid && !KitKextMatches (Source, Entry)) { + continue; + } + } else { + Source = &Entry->Kexts[Index - Config->KextCount]; + } + + if (Source->Invalid) { + Status = EFI_OUT_OF_RESOURCES; + break; + } + + for (Previous = 0; Previous < Used; Previous++) { + if (CaStriCmp (Out[Previous].BundlePath, Source->BundlePath) == 0) { + CaLog ("kext %s: duplicate bundle; boot refused", Source->BundlePath); + Status = EFI_INVALID_PARAMETER; + break; + } + } + + if (EFI_ERROR (Status)) { + break; + } + + Out[Used].BundlePath = Source->BundlePath; + Out[Used].PlistPath = Source->PlistPath; + Out[Used].ExecutablePath = Source->ExecutablePath; + Status = CaReadFile (Source->Volume, Source->PlistPath, &Data, &Size); + if (EFI_ERROR (Status)) { + CaLog ("kext %s: cannot read plist: %r", Source->BundlePath, Status); + break; + } + + Out[Used].InfoPlist = Data; + Out[Used].InfoPlistSize = Size; + if (Size == 0) { + CaLog ("kext %s: empty plist; boot refused", Source->BundlePath); + Status = EFI_LOAD_ERROR; + break; + } + + if (Source->ExecutablePath != NULL) { + Status = CaReadFile (Source->Volume, Source->ExecutablePath, &Data, &Size); + if (EFI_ERROR (Status)) { + CaLog ("kext %s: cannot read executable: %r", Source->BundlePath, Status); + break; + } + + Out[Used].Executable = Data; + Out[Used].ExecutableSize = Size; + if (!KextImageValid (Data, Size)) { + CaLog ("kext %s: invalid AArch64 MH_KEXT_BUNDLE; boot refused", Source->BundlePath); + Status = EFI_LOAD_ERROR; + break; + } + } + + CaLog ("kext %s: staged (%u plist, %u executable bytes)", Source->BundlePath, + (UINT32)Out[Used].InfoPlistSize, (UINT32)Out[Used].ExecutableSize); + Used++; + } + + if (EFI_ERROR (Status)) { + CaFreeKexts (Out, Total); + return Status; + } + + *Kexts = Out; + *Count = Used; + return EFI_SUCCESS; +} diff --git a/package/CrabApplePkg/Library/CrabAppleLib/Logo.c b/package/CrabApplePkg/Library/CrabAppleLib/Logo.c new file mode 100644 index 0000000..d1a3a4e --- /dev/null +++ b/package/CrabApplePkg/Library/CrabAppleLib/Logo.c @@ -0,0 +1,182 @@ +/** @file + UniXNU logo: the vector strokes of Docs/Images/unixnu-logo.svg (flattened + into LogoShape.h by Scripts/gen-logo.py), rasterized at any size with + integer math (no floating point in firmware) and 4x4 supersampling. + + Each stroke is a round-capped, round-joined polyline. Every segment marks + the subsamples within its half width in a coverage bitmap, so overlapping + segments never darken or brighten twice; the bitmap is then averaged into + pixels blended from the background to the foreground. + + Copyright (c) 2026, UniXNU contributors. All rights reserved. + SPDX-License-Identifier: BSD-2-Clause-Patent +**/ + +#include "Internal.h" +#include "LogoShape.h" + +#define SS 4 +#define FIX 16 + +STATIC CONST EFI_GRAPHICS_OUTPUT_BLT_PIXEL mInk = { 0xFF, 0xFF, 0xFF, 0 }; + +// +// Whether (X,Y) lies within HalfWidth of segment (X0,Y0)-(X1,Y1), all in +// the same fixed-point units. A zero-length segment is a dot. +// +STATIC +BOOLEAN +NearSegment ( + IN INT64 X, + IN INT64 Y, + IN INT64 X0, + IN INT64 Y0, + IN INT64 X1, + IN INT64 Y1, + IN INT64 HalfWidth + ) +{ + INT64 Dx = X1 - X0; + INT64 Dy = Y1 - Y0; + INT64 Len = Dx * Dx + Dy * Dy; + INT64 T; + INT64 Px; + INT64 Py; + + if (Len == 0) { + Px = X - X0; + Py = Y - Y0; + return Px * Px + Py * Py <= HalfWidth * HalfWidth; + } + + T = (X - X0) * Dx + (Y - Y0) * Dy; + if (T < 0) { + T = 0; + } else if (T > Len) { + T = Len; + } + + // + // Closest point = P0 + d * T / Len. + // + Px = X0 - X + DivS64x64Remainder (Dx * T, Len, NULL); + Py = Y0 - Y + DivS64x64Remainder (Dy * T, Len, NULL); + return Px * Px + Py * Py <= HalfWidth * HalfWidth; +} + +// +// Mark every subsample of the Sub x Sub bitmap within HalfWidth of the +// segment. Coordinates are fixed point, FIX units per subsample. +// +STATIC +VOID +MarkSegment ( + IN OUT UINT8 *Mask, + IN INT64 Sub, + IN INT64 X0, + IN INT64 Y0, + IN INT64 X1, + IN INT64 Y1, + IN INT64 HalfWidth + ) +{ + INT64 Left; + INT64 Right; + INT64 Top; + INT64 Bottom; + INT64 Sx; + INT64 Sy; + INT64 Bit; + + Left = MAX (0, (MIN (X0, X1) - HalfWidth) / FIX); + Right = MIN (Sub - 1, (MAX (X0, X1) + HalfWidth) / FIX); + Top = MAX (0, (MIN (Y0, Y1) - HalfWidth) / FIX); + Bottom = MIN (Sub - 1, (MAX (Y0, Y1) + HalfWidth) / FIX); + for (Sy = Top; Sy <= Bottom; Sy++) { + for (Sx = Left; Sx <= Right; Sx++) { + if (NearSegment (Sx * FIX + FIX / 2, Sy * FIX + FIX / 2, X0, Y0, X1, Y1, HalfWidth)) { + Bit = Sy * Sub + Sx; + Mask[Bit / 8] |= (UINT8)(1 << (Bit % 8)); + } + } + } +} + +VOID +CaLogoRender ( + OUT EFI_GRAPHICS_OUTPUT_BLT_PIXEL *Buffer, + IN UINTN Size, + IN EFI_GRAPHICS_OUTPUT_BLT_PIXEL Background + ) +{ + INT64 Sub; + INT64 Unit; + UINT8 *Mask; + UINTN Stroke; + UINTN Index; + UINTN Px; + UINTN Py; + UINTN Sx; + UINTN Sy; + UINTN Bit; + UINT32 Cover; + INT64 HalfWidth; + CONST INT16 *P; + CONST INT16 *Q; + EFI_GRAPHICS_OUTPUT_BLT_PIXEL *Out; + + for (Index = 0; Index < Size * Size; Index++) { + Buffer[Index] = Background; + } + + Sub = (INT64)(Size * SS); + Unit = Sub * FIX; + Mask = AllocateZeroPool ((UINTN)((Sub * Sub + 7) / 8)); + if (Mask == NULL) { + return; + } + + for (Stroke = 0; Stroke < ARRAY_SIZE (mLogoStrokes); Stroke++) { + HalfWidth = MAX (FIX / 2, mLogoStrokes[Stroke][2] * Unit / LOGO_DESIGN); + for (Index = 0; Index < mLogoStrokes[Stroke][1]; Index++) { + P = mLogoPoints[mLogoStrokes[Stroke][0] + Index]; + Q = mLogoPoints[mLogoStrokes[Stroke][0] + MIN (Index + 1, mLogoStrokes[Stroke][1] - 1u)]; + if ((Index > 0) && (P == Q)) { + break; + } + + MarkSegment ( + Mask, + Sub, + P[0] * Unit / LOGO_DESIGN, + P[1] * Unit / LOGO_DESIGN, + Q[0] * Unit / LOGO_DESIGN, + Q[1] * Unit / LOGO_DESIGN, + HalfWidth + ); + } + } + + for (Py = 0; Py < Size; Py++) { + for (Px = 0; Px < Size; Px++) { + Cover = 0; + for (Sy = 0; Sy < SS; Sy++) { + for (Sx = 0; Sx < SS; Sx++) { + Bit = (Py * SS + Sy) * (UINTN)Sub + Px * SS + Sx; + Cover += (Mask[Bit / 8] >> (Bit % 8)) & 1; + } + } + + if (Cover == 0) { + continue; + } + + Out = &Buffer[Py * Size + Px]; + Out->Red = (UINT8)((Background.Red * (SS * SS - Cover) + mInk.Red * Cover) / (SS * SS)); + Out->Green = (UINT8)((Background.Green * (SS * SS - Cover) + mInk.Green * Cover) / (SS * SS)); + Out->Blue = (UINT8)((Background.Blue * (SS * SS - Cover) + mInk.Blue * Cover) / (SS * SS)); + } + } + + FreePool (Mask); +} diff --git a/package/CrabApplePkg/Library/CrabAppleLib/LogoShape.h b/package/CrabApplePkg/Library/CrabAppleLib/LogoShape.h new file mode 100644 index 0000000..141dd87 --- /dev/null +++ b/package/CrabApplePkg/Library/CrabAppleLib/LogoShape.h @@ -0,0 +1,74 @@ +/** @file + UniXNU logo strokes, generated by Scripts/gen-logo.py from the vector + description that also produces Docs/Images/unixnu-logo.svg. Do not edit. + + Copyright (c) 2026, UniXNU contributors. All rights reserved. + SPDX-License-Identifier: BSD-2-Clause-Patent +**/ + +#ifndef LOGO_SHAPE_H_ +#define LOGO_SHAPE_H_ + +#define LOGO_DESIGN 1000 + +// +// Points of every stroke, in strokes order. +// +STATIC CONST INT16 mLogoPoints[][2] = { + { 515, 224 }, { 507, 221 }, { 499, 218 }, { 491, 216 }, { 484, 213 }, { 476, 211 }, { 469, 208 }, { 461, 206 }, { 454, 204 }, { 447, 202 }, { 439, 201 }, { 432, 199 }, { 425, 198 }, { 417, 197 }, { 410, 196 }, { 402, 195 }, { 394, 194 }, { 386, 194 }, { 377, 194 }, { 369, 194 }, { 360, 194 }, { 348, 195 }, { 337, 197 }, { 326, 200 }, { 316, 204 }, { 306, 209 }, { 297, 215 }, { 288, 222 }, { 280, 229 }, { 272, 238 }, { 265, 247 }, { 259, 256 }, { 253, 267 }, { 247, 278 }, { 242, 290 }, { 238, 302 }, { 235, 315 }, { 232, 329 }, { 229, 343 }, { 227, 357 }, { 226, 372 }, { 225, 388 }, { 226, 403 }, { 226, 419 }, { 228, 435 }, { 230, 450 }, { 232, 466 }, { 235, 481 }, { 239, 496 }, { 243, 511 }, { 248, 525 }, { 254, 539 }, { 259, 553 }, { 265, 567 }, { 272, 580 }, { 279, 593 }, { 287, 606 }, { 294, 618 }, { 303, 630 }, { 311, 641 }, { 320, 652 }, { 326, 658 }, { 333, 664 }, { 339, 669 }, { 346, 674 }, { 353, 678 }, { 360, 682 }, { 368, 685 }, { 375, 687 }, { 383, 690 }, { 390, 691 }, { 397, 692 }, { 405, 693 }, { 412, 693 }, { 420, 693 }, { 427, 693 }, { 434, 692 }, { 441, 691 }, { 447, 690 }, { 454, 688 }, { 460, 686 }, { 465, 685 }, { 470, 683 }, { 475, 682 }, { 480, 681 }, { 486, 680 }, { 491, 680 }, { 497, 679 }, { 503, 679 }, { 509, 679 }, { 515, 678 }, { 521, 679 }, { 527, 679 }, { 533, 679 }, { 539, 680 }, { 544, 680 }, { 550, 681 }, { 555, 682 }, { 560, 683 }, { 565, 685 }, { 570, 686 }, { 576, 688 }, { 582, 690 }, { 589, 691 }, { 596, 692 }, { 602, 693 }, { 609, 693 }, { 616, 693 }, { 623, 693 }, { 630, 692 }, { 637, 691 }, { 644, 690 }, { 651, 687 }, { 657, 685 }, { 664, 682 }, { 671, 678 }, { 677, 674 }, { 683, 669 }, { 689, 664 }, { 695, 658 }, { 700, 652 }, { 704, 646 }, { 708, 640 }, { 712, 634 }, { 716, 628 }, { 720, 622 }, { 724, 615 }, { 728, 608 }, { 732, 602 }, { 736, 595 }, { 739, 588 }, { 743, 581 }, { 746, 574 }, { 749, 567 }, { 753, 560 }, { 756, 553 }, { 759, 546 }, { 762, 539 }, { 765, 532 }, { 767, 525 }, { 770, 518 }, { 761, 514 }, { 753, 509 }, { 746, 504 }, { 738, 499 }, { 732, 493 }, { 725, 487 }, { 719, 481 }, { 713, 474 }, { 708, 467 }, { 704, 460 }, { 699, 452 }, { 695, 444 }, { 692, 436 }, { 689, 428 }, { 687, 420 }, { 685, 411 }, { 683, 403 }, { 682, 394 }, { 682, 385 }, { 682, 376 }, { 682, 368 }, { 683, 361 }, { 685, 354 }, { 686, 346 }, { 688, 340 }, { 690, 333 }, { 693, 326 }, { 696, 320 }, { 699, 314 }, { 702, 308 }, { 706, 303 }, { 711, 297 }, { 715, 292 }, { 720, 287 }, { 725, 282 }, { 731, 277 }, { 737, 273 }, { 743, 268 }, { 749, 264 }, { 756, 260 }, { 751, 254 }, { 745, 249 }, { 739, 244 }, { 733, 239 }, { 726, 234 }, { 720, 230 }, { 713, 226 }, { 705, 222 }, { 698, 218 }, { 690, 215 }, { 682, 212 }, { 674, 210 }, { 665, 207 }, { 657, 206 }, { 648, 204 }, { 639, 203 }, { 630, 202 }, { 621, 202 }, { 611, 202 }, { 602, 202 }, { 597, 202 }, { 592, 203 }, { 587, 204 }, { 582, 204 }, { 577, 205 }, { 572, 206 }, { 568, 207 }, { 564, 208 }, { 559, 209 }, { 555, 210 }, { 551, 212 }, { 547, 213 }, { 543, 214 }, { 539, 215 }, { 535, 217 }, { 531, 218 }, { 527, 220 }, { 523, 221 }, { 519, 223 }, { 515, 224 }, + { 501, 188 }, { 501, 182 }, { 502, 176 }, { 503, 171 }, { 504, 165 }, { 506, 160 }, { 508, 154 }, { 510, 149 }, { 512, 144 }, { 514, 139 }, { 517, 134 }, { 520, 130 }, { 523, 125 }, { 526, 120 }, { 530, 116 }, { 533, 112 }, { 537, 107 }, { 541, 103 }, { 545, 98 }, { 550, 94 }, { 554, 90 }, { 528, 64 }, { 648, 48 }, { 636, 152 }, { 609, 126 }, { 605, 130 }, { 601, 135 }, { 597, 139 }, { 593, 143 }, { 589, 148 }, { 585, 151 }, { 580, 155 }, { 575, 159 }, { 571, 163 }, { 566, 166 }, { 560, 169 }, { 555, 172 }, { 549, 175 }, { 543, 178 }, { 537, 180 }, { 530, 182 }, { 523, 184 }, { 516, 186 }, { 509, 187 }, { 501, 188 }, + { 455, 363 }, { 545, 363 }, { 548, 363 }, { 551, 363 }, { 553, 364 }, { 556, 365 }, { 558, 366 }, { 561, 367 }, { 563, 368 }, { 565, 370 }, { 567, 371 }, { 569, 373 }, { 571, 375 }, { 572, 377 }, { 574, 379 }, { 575, 381 }, { 576, 384 }, { 577, 386 }, { 578, 389 }, { 579, 391 }, { 579, 394 }, { 579, 397 }, { 579, 487 }, { 579, 490 }, { 579, 493 }, { 578, 495 }, { 577, 498 }, { 576, 500 }, { 575, 503 }, { 574, 505 }, { 572, 507 }, { 571, 509 }, { 569, 511 }, { 567, 513 }, { 565, 514 }, { 563, 516 }, { 561, 517 }, { 558, 518 }, { 556, 519 }, { 553, 520 }, { 551, 521 }, { 548, 521 }, { 545, 521 }, { 455, 521 }, { 452, 521 }, { 449, 521 }, { 447, 520 }, { 444, 519 }, { 442, 518 }, { 439, 517 }, { 437, 516 }, { 435, 514 }, { 433, 513 }, { 431, 511 }, { 429, 509 }, { 428, 507 }, { 426, 505 }, { 425, 503 }, { 424, 500 }, { 423, 498 }, { 422, 495 }, { 421, 493 }, { 421, 490 }, { 421, 487 }, { 421, 397 }, { 421, 394 }, { 421, 391 }, { 422, 389 }, { 423, 386 }, { 424, 384 }, { 425, 381 }, { 426, 379 }, { 428, 377 }, { 429, 375 }, { 431, 373 }, { 433, 371 }, { 435, 370 }, { 437, 368 }, { 439, 367 }, { 442, 366 }, { 444, 365 }, { 447, 364 }, { 449, 363 }, { 452, 363 }, { 455, 363 }, + { 456, 320 }, { 456, 352 }, + { 456, 532 }, { 456, 564 }, + { 500, 320 }, { 500, 352 }, + { 500, 532 }, { 500, 564 }, + { 544, 320 }, { 544, 352 }, + { 544, 532 }, { 544, 564 }, + { 378, 398 }, { 410, 398 }, + { 590, 398 }, { 622, 398 }, + { 378, 442 }, { 410, 442 }, + { 590, 442 }, { 622, 442 }, + { 378, 486 }, { 410, 486 }, + { 590, 486 }, { 622, 486 }, + { 119, 804 }, { 119, 887 }, { 119, 891 }, { 119, 895 }, { 120, 898 }, { 121, 901 }, { 122, 904 }, { 123, 907 }, { 125, 910 }, { 127, 913 }, { 129, 915 }, { 131, 917 }, { 133, 919 }, { 136, 921 }, { 139, 922 }, { 142, 923 }, { 145, 924 }, { 149, 925 }, { 153, 926 }, { 157, 927 }, { 161, 927 }, { 166, 927 }, { 171, 927 }, { 175, 927 }, { 179, 926 }, { 183, 925 }, { 187, 924 }, { 190, 923 }, { 193, 922 }, { 196, 921 }, { 199, 919 }, { 201, 917 }, { 203, 915 }, { 205, 913 }, { 207, 910 }, { 209, 907 }, { 210, 904 }, { 211, 901 }, { 212, 898 }, { 213, 895 }, { 213, 891 }, { 213, 887 }, { 213, 804 }, + { 285, 927 }, { 285, 864 }, + { 285, 890 }, { 286, 887 }, { 286, 885 }, { 287, 883 }, { 288, 881 }, { 289, 879 }, { 290, 877 }, { 292, 875 }, { 293, 873 }, { 295, 872 }, { 297, 870 }, { 299, 869 }, { 301, 868 }, { 303, 867 }, { 305, 866 }, { 308, 866 }, { 310, 865 }, { 313, 865 }, { 316, 864 }, { 319, 864 }, { 322, 864 }, { 325, 864 }, { 328, 864 }, { 331, 865 }, { 334, 865 }, { 337, 866 }, { 339, 867 }, { 342, 868 }, { 344, 869 }, { 346, 871 }, { 348, 872 }, { 349, 874 }, { 351, 876 }, { 352, 878 }, { 353, 880 }, { 354, 883 }, { 355, 885 }, { 355, 888 }, { 356, 891 }, { 356, 894 }, { 356, 897 }, { 356, 927 }, + { 419, 864 }, { 419, 927 }, + { 419, 793 }, { 419, 793 }, + { 463, 789 }, { 580, 940 }, + { 580, 789 }, { 463, 940 }, + { 635, 940 }, { 635, 789 }, { 733, 940 }, { 733, 789 }, + { 783, 789 }, { 783, 887 }, { 783, 892 }, { 784, 897 }, { 784, 902 }, { 785, 906 }, { 787, 910 }, { 788, 914 }, { 790, 918 }, { 792, 921 }, { 795, 924 }, { 797, 927 }, { 800, 929 }, { 804, 932 }, { 807, 934 }, { 811, 935 }, { 815, 937 }, { 819, 938 }, { 824, 939 }, { 829, 939 }, { 834, 940 }, { 840, 940 }, { 846, 940 }, { 851, 939 }, { 856, 939 }, { 861, 938 }, { 865, 937 }, { 869, 935 }, { 873, 934 }, { 876, 932 }, { 880, 929 }, { 883, 927 }, { 885, 924 }, { 888, 921 }, { 890, 918 }, { 892, 914 }, { 893, 910 }, { 895, 906 }, { 896, 902 }, { 896, 897 }, { 897, 892 }, { 897, 887 }, { 897, 789 }, +}; + +// +// First point, point count and half width of each round-capped stroke. +// +STATIC CONST UINT16 mLogoStrokes[][3] = { + { 0, 221, 12 }, + { 221, 45, 11 }, + { 266, 85, 11 }, + { 351, 2, 8 }, + { 353, 2, 8 }, + { 355, 2, 8 }, + { 357, 2, 8 }, + { 359, 2, 8 }, + { 361, 2, 8 }, + { 363, 2, 8 }, + { 365, 2, 8 }, + { 367, 2, 8 }, + { 369, 2, 8 }, + { 371, 2, 8 }, + { 373, 2, 8 }, + { 375, 43, 22 }, + { 418, 2, 22 }, + { 420, 42, 22 }, + { 462, 2, 22 }, + { 464, 2, 25 }, + { 466, 2, 7 }, + { 468, 2, 7 }, + { 470, 4, 7 }, + { 474, 43, 7 }, +}; + +#endif diff --git a/package/CrabApplePkg/Library/CrabAppleLib/Main.c b/package/CrabApplePkg/Library/CrabAppleLib/Main.c new file mode 100644 index 0000000..41c3026 --- /dev/null +++ b/package/CrabApplePkg/Library/CrabAppleLib/Main.c @@ -0,0 +1,135 @@ +/** @file + CrabAppleRun() and logging. + + Copyright (c) 2026, UniXNU contributors. All rights reserved. + SPDX-License-Identifier: BSD-2-Clause-Patent +**/ + +#include "Internal.h" + +#include + +STATIC EFI_SERIAL_IO_PROTOCOL *mSerial; + +VOID +CaLogInit ( + VOID + ) +{ + if (EFI_ERROR (gBS->LocateProtocol (&gEfiSerialIoProtocolGuid, NULL, (VOID **)&mSerial))) { + mSerial = NULL; + } +} + +VOID +EFIAPI +CaLog ( + IN CONST CHAR8 *Format, + ... + ) +{ + VA_LIST Args; + CHAR8 Line[256]; + UINTN Prefix; + UINTN Size; + + Prefix = AsciiSPrint (Line, sizeof (Line), "CRABAPPLE: "); + VA_START (Args, Format); + AsciiVSPrint (Line + Prefix, sizeof (Line) - Prefix - 2, Format, Args); + VA_END (Args); + AsciiStrCatS (Line, sizeof (Line), "\r\n"); + + DEBUG ((PcdGet32 (PcdCrabAppleDebugLevel), "%a", Line)); + if (mSerial != NULL) { + Size = AsciiStrLen (Line); + mSerial->Write (mSerial, &Size, Line); + } +} + +EFI_STATUS +EFIAPI +CrabAppleRun ( + IN EFI_HANDLE ImageHandle, + IN CRABAPPLE_HOST Host + ) +{ + EFI_STATUS Status; + CA_CONFIG Config; + CA_UI Ui; + CA_INJECT_STATS Stats; + UINTN Index; + BOOLEAN Countdown; + CA_ENTRY *Entry; + + // + // A boot manager waits for people; the 5 minute BDS watchdog does not. + // + gBS->SetWatchdogTimer (0, 0, 0, NULL); + CaLogInit (); + CaLog ( + "UniXNU %a starting as %a, firmware %s rev 0x%x", + CA_VERSION_STRING, + (Host == CrabAppleHostFirmware) ? "firmware boot option" : "application", + (gST->FirmwareVendor != NULL) ? gST->FirmwareVendor : L"?", + gST->FirmwareRevision + ); + + // + // Started straight from BDS, storage may not be connected yet. + // + if (Host == CrabAppleHostFirmware) { + CaConnectAll (); + } + + CaConfigLoad (ImageHandle, &Config); + CaLog ( + "config: %u entries, %u drivers, %u dt patches, %u acpi tables, %u slices, %u kits loaded, %u skipped", + (UINT32)Config.EntryCount, + (UINT32)Config.DriverCount, + (UINT32)Config.PatchCount, + (UINT32)Config.AcpiCount, + (UINT32)Config.SliceCount, + (UINT32)Config.KitsLoaded, + (UINT32)Config.KitsSkipped + ); + + CaPublishSlices (&Config); + + // + // Drivers before the UI: an injected driver may provide the display or + // the input device the menu needs. + // + ZeroMem (&Stats, sizeof (Stats)); + CaInjectDrivers (ImageHandle, &Config, &Stats); + CaInjectAcpi (&Config, &Stats); + + CaUiInit (&Config, &Ui); + UnicodeSPrint ( + Ui.Info, + sizeof (Ui.Info), + L"drivers %u loaded / %u failed kits %u acpi %u", + (UINT32)Stats.DriversLoaded, + (UINT32)Stats.DriversFailed, + (UINT32)Config.KitsLoaded, + (UINT32)Stats.AcpiInstalled + ); + + Countdown = TRUE; + for ( ; ;) { + Index = CaMenuRun (&Ui, &Config, Countdown); + Entry = &Config.Entries[Index]; + Status = CaBootEntry (ImageHandle, &Config, Entry); + if ((Entry->Type == CaEntryAction) && (Entry->Action == CaActionExit)) { + return EFI_SUCCESS; + } + + // + // The payload returned or failed: back to the menu, without the + // countdown so a broken default cannot loop on a machine with input. + // + UnicodeSPrint (Ui.Notice, sizeof (Ui.Notice), L"'%s' returned: %r", Entry->Title, Status); + CaLog ("menu: entry %u returned %r", (UINT32)Index, Status); + gBS->SetWatchdogTimer (0, 0, 0, NULL); + Countdown = FALSE; + } +} diff --git a/package/CrabApplePkg/Library/CrabAppleLib/Menu.c b/package/CrabApplePkg/Library/CrabAppleLib/Menu.c new file mode 100644 index 0000000..ba33e63 --- /dev/null +++ b/package/CrabApplePkg/Library/CrabAppleLib/Menu.c @@ -0,0 +1,248 @@ +/** @file + Menu loop: input, countdown and idle rotation. + + Degradation rules (see README): + - No input device: the timeout always runs (fallback_timeout replaces a + negative timeout) and the highlight rotates through the entries. + - Any key press stops both the countdown and the rotation. + - The countdown always boots the default entry; rotation is display only. + + Copyright (c) 2026, UniXNU contributors. All rights reserved. + SPDX-License-Identifier: BSD-2-Clause-Patent +**/ + +#include "Internal.h" + +#define TICK_MS 100 +#define DEFAULT_ROTATE_SEC 2 + +#ifndef SCAN_SUSPEND +#define SCAN_SUSPEND 0x0102 +#endif + +// +// Physical input = a SimpleTextIn instance that sits on a device path; the +// ConSplitter's virtual handle has none. +// +STATIC +UINTN +CountInputDevices ( + VOID + ) +{ + EFI_STATUS Status; + EFI_HANDLE *Handles; + UINTN Count; + UINTN Index; + UINTN Physical; + VOID *Path; + + if (gST->ConIn == NULL) { + return 0; + } + + Status = gBS->LocateHandleBuffer (ByProtocol, &gEfiSimpleTextInProtocolGuid, NULL, &Count, &Handles); + if (EFI_ERROR (Status)) { + return 0; + } + + Physical = 0; + for (Index = 0; Index < Count; Index++) { + if (!EFI_ERROR (gBS->HandleProtocol (Handles[Index], &gEfiDevicePathProtocolGuid, &Path))) { + Physical++; + } + } + + FreePool (Handles); + return Physical; +} + +STATIC +VOID +DrawCountdown ( + IN CA_UI *Ui, + IN CA_CONFIG *Config, + IN INTN RemainingMs, + IN BOOLEAN HasInput + ) +{ + CHAR16 Line[128]; + + if (RemainingMs >= 0) { + UnicodeSPrint ( + Line, + sizeof (Line), + L"Booting %s in %u s", + Config->Entries[Config->DefaultIndex].Title, + (UINT32)((RemainingMs + 999) / 1000) + ); + } else { + UnicodeSPrint (Line, sizeof (Line), L"Select an entry"); + } + + CaUiDrawStatus (Ui, 0, Line); +} + +UINTN +CaMenuRun ( + IN CA_UI *Ui, + IN CA_CONFIG *Config, + IN BOOLEAN Countdown + ) +{ + EFI_STATUS Status; + EFI_EVENT Timer; + EFI_EVENT Events[2]; + UINTN EventCount; + UINTN Signaled; + EFI_INPUT_KEY Key; + UINTN Selected; + UINTN Previous; + UINTN Index; + BOOLEAN HasInput; + BOOLEAN UserActive; + INTN RemainingMs; + UINTN RotateMs; + UINTN RotateAccum; + INTN LastShownSecond; + UINTN Count; + + Count = Config->EntryCount; + Selected = Config->DefaultIndex; + HasInput = (CountInputDevices () > 0); + + if (!Countdown) { + RemainingMs = HasInput ? -1 : (INTN)Config->FallbackTimeout * 1000; + } else if (Config->Timeout < 0) { + RemainingMs = HasInput ? -1 : (INTN)Config->FallbackTimeout * 1000; + } else { + RemainingMs = Config->Timeout * 1000; + } + + RotateMs = Config->RotateSeconds * 1000; + if (!HasInput && (RotateMs == 0)) { + RotateMs = DEFAULT_ROTATE_SEC * 1000; + } + + CaUiDrawFrame (Ui, Config); + for (Index = 0; Index < Count; Index++) { + CaUiDrawEntry (Ui, Config, Index, Index == Selected); + } + + CaUiDrawStatus ( + Ui, + 1, + HasInput ? L"Up/Down: select Enter: boot 1-9: boot entry Esc: stop timer" + : L"No input device: the default entry boots when the timer ends" + ); + if (Ui->Notice[0] != L'\0') { + CaUiDrawStatus (Ui, 2, Ui->Notice); + } + + CaLog ( + "menu: shown %u entries, default %u '%a', timeout %d ms, input %a, rotate %u ms", + (UINT32)Count, + (UINT32)Config->DefaultIndex, + (Config->Entries[Config->DefaultIndex].Id != NULL) ? Config->Entries[Config->DefaultIndex].Id : "?", + (INT32)RemainingMs, + HasInput ? "yes" : "no", + (UINT32)RotateMs + ); + + if (RemainingMs == 0) { + CaLog ("menu: timeout 0, booting default"); + return Config->DefaultIndex; + } + + Status = gBS->CreateEvent (EVT_TIMER, TPL_CALLBACK, NULL, NULL, &Timer); + if (EFI_ERROR (Status)) { + return Config->DefaultIndex; + } + + gBS->SetTimer (Timer, TimerPeriodic, EFI_TIMER_PERIOD_MILLISECONDS (TICK_MS)); + + EventCount = 0; + Events[EventCount++] = Timer; + if (gST->ConIn != NULL) { + Events[EventCount++] = gST->ConIn->WaitForKey; + } + + UserActive = FALSE; + RotateAccum = 0; + LastShownSecond = -2; + for ( ; ;) { + INTN Second = (RemainingMs < 0) ? -1 : (RemainingMs + 999) / 1000; + + if (Second != LastShownSecond) { + DrawCountdown (Ui, Config, RemainingMs, HasInput); + LastShownSecond = Second; + } + + Status = gBS->WaitForEvent (EventCount, Events, &Signaled); + if (EFI_ERROR (Status)) { + break; + } + + Previous = Selected; + if (Signaled == 0) { + if (RemainingMs > 0) { + RemainingMs -= TICK_MS; + if (RemainingMs <= 0) { + CaLog ("menu: timeout, booting default %u", (UINT32)Config->DefaultIndex); + Selected = Config->DefaultIndex; + break; + } + } + + if (!UserActive && (RotateMs > 0) && (Count > 1)) { + RotateAccum += TICK_MS; + if (RotateAccum >= RotateMs) { + RotateAccum = 0; + Selected = (Selected + 1) % Count; + } + } + } else { + if (EFI_ERROR (gST->ConIn->ReadKeyStroke (gST->ConIn, &Key))) { + continue; + } + + if (!UserActive) { + // + // The first key takes over: no more countdown or rotation, and the + // highlight returns to the default before the key applies. + // + UserActive = TRUE; + RemainingMs = -1; + Selected = Config->DefaultIndex; + } + + if ((Key.ScanCode == SCAN_UP) || (Key.UnicodeChar == L'k')) { + Selected = (Selected + Count - 1) % Count; + } else if ((Key.ScanCode == SCAN_DOWN) || (Key.UnicodeChar == L'j')) { + Selected = (Selected + 1) % Count; + } else if (Key.ScanCode == SCAN_HOME) { + Selected = 0; + } else if (Key.ScanCode == SCAN_END) { + Selected = Count - 1; + } else if ((Key.UnicodeChar >= L'1') && (Key.UnicodeChar <= L'9') && ((UINTN)(Key.UnicodeChar - L'1') < Count)) { + Selected = Key.UnicodeChar - L'1'; + CaLog ("menu: key %c, booting %u", Key.UnicodeChar, (UINT32)Selected); + break; + } else if ((Key.UnicodeChar == CHAR_CARRIAGE_RETURN) || (Key.UnicodeChar == CHAR_LINEFEED) || + (Key.UnicodeChar == L' ') || (Key.ScanCode == SCAN_RIGHT) || (Key.ScanCode == SCAN_SUSPEND)) + { + CaLog ("menu: enter, booting %u", (UINT32)Selected); + break; + } + } + + if (Selected != Previous) { + CaUiDrawEntry (Ui, Config, Previous, FALSE); + CaUiDrawEntry (Ui, Config, Selected, TRUE); + } + } + + gBS->SetTimer (Timer, TimerCancel, 0); + gBS->CloseEvent (Timer); + return Selected; +} diff --git a/package/CrabApplePkg/Library/CrabAppleLib/Slice.c b/package/CrabApplePkg/Library/CrabAppleLib/Slice.c new file mode 100644 index 0000000..8a6c502 --- /dev/null +++ b/package/CrabApplePkg/Library/CrabAppleLib/Slice.c @@ -0,0 +1,321 @@ +/** @file + Block slices: publish a byte range of a GPT partition as its own block + device (EFI_BLOCK_IO_PROTOCOL), located by partition name through + EFI_PARTITION_INFO_PROTOCOL. + + Typical use: a partition that starts with a small FAT volume (found by the + firmware's FAT driver directly, since a FAT boot sector carries its own + size) and whose remainder should reach the payload as a separate disk. + + Copyright (c) 2026, UniXNU contributors. All rights reserved. + SPDX-License-Identifier: BSD-2-Clause-Patent +**/ + +#include "Internal.h" + +#include +#include +#include + +#define SLICE_SIGNATURE SIGNATURE_32 ('C', 'A', 'S', 'L') + +typedef struct { + UINT32 Signature; + EFI_BLOCK_IO_PROTOCOL BlockIo; + EFI_BLOCK_IO_MEDIA Media; + EFI_BLOCK_IO_PROTOCOL *Parent; + EFI_LBA Start; + EFI_DEVICE_PATH_PROTOCOL *DevicePath; + EFI_HANDLE Handle; +} SLICE_DEVICE; + +#define SLICE_FROM_BLOCK_IO(a) CR (a, SLICE_DEVICE, BlockIo, SLICE_SIGNATURE) + +STATIC +EFI_STATUS +EFIAPI +SliceReset ( + IN EFI_BLOCK_IO_PROTOCOL *This, + IN BOOLEAN ExtendedVerification + ) +{ + return EFI_SUCCESS; +} + +STATIC +EFI_STATUS +CheckRange ( + IN SLICE_DEVICE *Slice, + IN UINT32 MediaId, + IN EFI_LBA Lba, + IN UINTN BufferSize, + IN VOID *Buffer + ) +{ + UINTN Blocks; + + if (MediaId != Slice->Media.MediaId) { + return EFI_MEDIA_CHANGED; + } + + if (Buffer == NULL) { + return EFI_INVALID_PARAMETER; + } + + if ((BufferSize % Slice->Media.BlockSize) != 0) { + return EFI_BAD_BUFFER_SIZE; + } + + Blocks = BufferSize / Slice->Media.BlockSize; + if ((Lba > Slice->Media.LastBlock) || (Blocks > Slice->Media.LastBlock - Lba + 1)) { + return EFI_INVALID_PARAMETER; + } + + return EFI_SUCCESS; +} + +STATIC +EFI_STATUS +EFIAPI +SliceReadBlocks ( + IN EFI_BLOCK_IO_PROTOCOL *This, + IN UINT32 MediaId, + IN EFI_LBA Lba, + IN UINTN BufferSize, + OUT VOID *Buffer + ) +{ + SLICE_DEVICE *Slice; + EFI_STATUS Status; + + Slice = SLICE_FROM_BLOCK_IO (This); + if (BufferSize == 0) { + return EFI_SUCCESS; + } + + Status = CheckRange (Slice, MediaId, Lba, BufferSize, Buffer); + if (EFI_ERROR (Status)) { + return Status; + } + + return Slice->Parent->ReadBlocks (Slice->Parent, Slice->Parent->Media->MediaId, Slice->Start + Lba, BufferSize, Buffer); +} + +STATIC +EFI_STATUS +EFIAPI +SliceWriteBlocks ( + IN EFI_BLOCK_IO_PROTOCOL *This, + IN UINT32 MediaId, + IN EFI_LBA Lba, + IN UINTN BufferSize, + IN VOID *Buffer + ) +{ + SLICE_DEVICE *Slice; + EFI_STATUS Status; + + Slice = SLICE_FROM_BLOCK_IO (This); + if (Slice->Media.ReadOnly) { + return EFI_WRITE_PROTECTED; + } + + if (BufferSize == 0) { + return EFI_SUCCESS; + } + + Status = CheckRange (Slice, MediaId, Lba, BufferSize, Buffer); + if (EFI_ERROR (Status)) { + return Status; + } + + return Slice->Parent->WriteBlocks (Slice->Parent, Slice->Parent->Media->MediaId, Slice->Start + Lba, BufferSize, Buffer); +} + +STATIC +EFI_STATUS +EFIAPI +SliceFlushBlocks ( + IN EFI_BLOCK_IO_PROTOCOL *This + ) +{ + SLICE_DEVICE *Slice; + + Slice = SLICE_FROM_BLOCK_IO (This); + return Slice->Media.ReadOnly ? EFI_SUCCESS : Slice->Parent->FlushBlocks (Slice->Parent); +} + +// +// The handle of the GPT partition whose name is Name, or NULL. +// +STATIC +EFI_HANDLE +FindPartition ( + IN CONST CHAR16 *Name + ) +{ + EFI_STATUS Status; + EFI_HANDLE *Handles; + UINTN Count; + UINTN Index; + EFI_PARTITION_INFO_PROTOCOL *Info; + EFI_HANDLE Found; + CHAR16 Entry[37]; + + Status = gBS->LocateHandleBuffer (ByProtocol, &gEfiPartitionInfoProtocolGuid, NULL, &Count, &Handles); + if (EFI_ERROR (Status)) { + return NULL; + } + + Found = NULL; + for (Index = 0; Index < Count && Found == NULL; Index++) { + Status = gBS->HandleProtocol (Handles[Index], &gEfiPartitionInfoProtocolGuid, (VOID **)&Info); + if (EFI_ERROR (Status) || (Info->Type != PARTITION_TYPE_GPT)) { + continue; + } + + // + // The GPT name field is not NUL-terminated when all 36 characters are used. + // + CopyMem (Entry, Info->Info.Gpt.PartitionName, sizeof (Info->Info.Gpt.PartitionName)); + Entry[36] = L'\0'; + if (StrCmp (Entry, Name) == 0) { + Found = Handles[Index]; + } + } + + FreePool (Handles); + return Found; +} + +STATIC +EFI_STATUS +PublishSlice ( + IN CA_BLOCK_SLICE *Config + ) +{ + EFI_STATUS Status; + EFI_HANDLE Partition; + EFI_BLOCK_IO_PROTOCOL *Parent; + SLICE_DEVICE *Slice; + UINT64 TotalBlocks; + UINT64 StartBlock; + UINT64 Blocks; + CRABAPPLE_BLOCK_SLICE_DEVICE_PATH Node; + UINTN NameLen; + + Partition = FindPartition (Config->Partition); + if (Partition == NULL) { + return EFI_NOT_FOUND; + } + + Status = gBS->HandleProtocol (Partition, &gEfiBlockIoProtocolGuid, (VOID **)&Parent); + if (EFI_ERROR (Status)) { + return Status; + } + + // + // Byte offsets must fall on block boundaries of the partition. + // + if (((Config->Offset % Parent->Media->BlockSize) != 0) || ((Config->Length % Parent->Media->BlockSize) != 0)) { + return EFI_INVALID_PARAMETER; + } + + TotalBlocks = Parent->Media->LastBlock + 1; + StartBlock = DivU64x32 (Config->Offset, Parent->Media->BlockSize); + if (StartBlock >= TotalBlocks) { + return EFI_INVALID_PARAMETER; + } + + Blocks = (Config->Length == 0) ? TotalBlocks - StartBlock : DivU64x32 (Config->Length, Parent->Media->BlockSize); + if (Blocks > TotalBlocks - StartBlock) { + return EFI_INVALID_PARAMETER; + } + + Slice = AllocateZeroPool (sizeof (SLICE_DEVICE)); + if (Slice == NULL) { + return EFI_OUT_OF_RESOURCES; + } + + Slice->Signature = SLICE_SIGNATURE; + Slice->Parent = Parent; + Slice->Start = StartBlock; + CopyMem (&Slice->Media, Parent->Media, sizeof (EFI_BLOCK_IO_MEDIA)); + Slice->Media.LastBlock = Blocks - 1; + Slice->Media.ReadOnly = !Config->Writable || Parent->Media->ReadOnly; + // + // Present the slice as a whole disk: the payload may put its own + // partition table or container on it. + // + Slice->Media.LogicalPartition = FALSE; + Slice->Media.MediaPresent = TRUE; + + Slice->BlockIo.Revision = EFI_BLOCK_IO_PROTOCOL_REVISION; + Slice->BlockIo.Media = &Slice->Media; + Slice->BlockIo.Reset = SliceReset; + Slice->BlockIo.ReadBlocks = SliceReadBlocks; + Slice->BlockIo.WriteBlocks = SliceWriteBlocks; + Slice->BlockIo.FlushBlocks = SliceFlushBlocks; + + ZeroMem (&Node, sizeof (Node)); + Node.Header.Header.Type = MEDIA_DEVICE_PATH; + Node.Header.Header.SubType = MEDIA_VENDOR_DP; + SetDevicePathNodeLength (&Node.Header.Header, sizeof (Node)); + CopyGuid (&Node.Header.Guid, &gCrabAppleBlockSliceGuid); + Node.StartingLba = StartBlock; + Node.NumberOfBlocks = Blocks; + NameLen = MIN (StrLen (Config->Name), ARRAY_SIZE (Node.Name) - 1); + CopyMem (Node.Name, Config->Name, NameLen * sizeof (CHAR16)); + + Slice->DevicePath = AppendDevicePathNode (DevicePathFromHandle (Partition), (EFI_DEVICE_PATH_PROTOCOL *)&Node); + if (Slice->DevicePath == NULL) { + FreePool (Slice); + return EFI_OUT_OF_RESOURCES; + } + + Status = gBS->InstallMultipleProtocolInterfaces ( + &Slice->Handle, + &gEfiDevicePathProtocolGuid, + Slice->DevicePath, + &gEfiBlockIoProtocolGuid, + &Slice->BlockIo, + NULL + ); + if (EFI_ERROR (Status)) { + FreePool (Slice->DevicePath); + FreePool (Slice); + return Status; + } + + CaLog ( + "slice %s: partition '%s' blocks %lu+%lu (%u-byte), %a", + Config->Name, + Config->Partition, + StartBlock, + Blocks, + Parent->Media->BlockSize, + Slice->Media.ReadOnly ? "read-only" : "writable" + ); + + // + // Let partition and file system drivers look at it, as for any new disk. + // + gBS->ConnectController (Slice->Handle, NULL, NULL, TRUE); + return EFI_SUCCESS; +} + +VOID +CaPublishSlices ( + IN CA_CONFIG *Config + ) +{ + UINTN Index; + EFI_STATUS Status; + + for (Index = 0; Index < Config->SliceCount; Index++) { + Status = PublishSlice (&Config->Slices[Index]); + if (EFI_ERROR (Status)) { + CaLog ("slice %s: partition '%s': %r", Config->Slices[Index].Name, Config->Slices[Index].Partition, Status); + } + } +} diff --git a/package/CrabApplePkg/Library/CrabAppleLib/Toml.c b/package/CrabApplePkg/Library/CrabAppleLib/Toml.c new file mode 100644 index 0000000..2934582 --- /dev/null +++ b/package/CrabApplePkg/Library/CrabAppleLib/Toml.c @@ -0,0 +1,924 @@ +/** @file + A small TOML subset parser. + + Supported: comments, bare/quoted/dotted keys, [table], [[array.of.tables]], + basic and literal strings, integers (decimal, 0x, 0o, 0b, '_' separators), + booleans, arrays (multi-line, nested, trailing comma) and inline tables. + Not supported: floats, dates, multi-line strings. Unsupported syntax is a + parse error, never silently ignored. + + Copyright (c) 2026, UniXNU contributors. All rights reserved. + SPDX-License-Identifier: BSD-2-Clause-Patent +**/ + +#include "Internal.h" + +typedef struct { + CONST CHAR8 *Text; + UINTN Length; + UINTN Pos; + UINTN Line; + CHAR8 *Error; + UINTN ErrorSize; + BOOLEAN Failed; +} TOML_PARSER; + +STATIC +VOID +Fail ( + IN TOML_PARSER *P, + IN CONST CHAR8 *Message + ) +{ + if (!P->Failed) { + AsciiSPrint (P->Error, P->ErrorSize, "line %u: %a", (UINT32)P->Line, Message); + P->Failed = TRUE; + } +} + +STATIC +CHAR8 +Peek ( + IN TOML_PARSER *P + ) +{ + return (P->Pos < P->Length) ? P->Text[P->Pos] : '\0'; +} + +STATIC +CHAR8 +Advance ( + IN TOML_PARSER *P + ) +{ + CHAR8 C; + + if (P->Pos >= P->Length) { + return '\0'; + } + + C = P->Text[P->Pos++]; + if (C == '\n') { + P->Line++; + } + + return C; +} + +STATIC +VOID +SkipSpaces ( + IN TOML_PARSER *P + ) +{ + while (Peek (P) == ' ' || Peek (P) == '\t') { + Advance (P); + } +} + +STATIC +VOID +SkipComment ( + IN TOML_PARSER *P + ) +{ + if (Peek (P) == '#') { + while (P->Pos < P->Length && Peek (P) != '\n') { + Advance (P); + } + } +} + +// +// Skip whitespace, newlines and comments (inside arrays). +// +STATIC +VOID +SkipBlank ( + IN TOML_PARSER *P + ) +{ + for ( ; ;) { + CHAR8 C = Peek (P); + + if ((C == ' ') || (C == '\t') || (C == '\r') || (C == '\n')) { + Advance (P); + } else if (C == '#') { + SkipComment (P); + } else { + return; + } + } +} + +// +// After a value or header only spaces and a comment may follow on the line. +// +STATIC +VOID +ExpectLineEnd ( + IN TOML_PARSER *P + ) +{ + SkipSpaces (P); + SkipComment (P); + if (Peek (P) == '\r') { + Advance (P); + } + + if (Peek (P) == '\n') { + Advance (P); + } else if (P->Pos < P->Length) { + Fail (P, "unexpected text after value"); + } +} + +STATIC +TOML_NODE * +NewNode ( + IN TOML_PARSER *P, + IN TOML_TYPE Type + ) +{ + TOML_NODE *Node; + + Node = AllocateZeroPool (sizeof (TOML_NODE)); + if (Node == NULL) { + Fail (P, "out of memory"); + return NULL; + } + + Node->Type = Type; + Node->Line = P->Line; + return Node; +} + +STATIC +VOID +AppendChild ( + IN TOML_NODE *Parent, + IN TOML_NODE *Child + ) +{ + if (Parent->LastChild == NULL) { + Parent->Child = Child; + } else { + Parent->LastChild->Next = Child; + } + + Parent->LastChild = Child; +} + +VOID +TomlFree ( + IN TOML_NODE *Node + ) +{ + TOML_NODE *Child; + TOML_NODE *Next; + + if (Node == NULL) { + return; + } + + for (Child = Node->Child; Child != NULL; Child = Next) { + Next = Child->Next; + TomlFree (Child); + } + + if (Node->Key != NULL) { + FreePool (Node->Key); + } + + if (Node->String != NULL) { + FreePool (Node->String); + } + + FreePool (Node); +} + +TOML_NODE * +TomlGet ( + IN TOML_NODE *Table, + IN CONST CHAR8 *Key + ) +{ + TOML_NODE *Child; + + if ((Table == NULL) || (Table->Type != TomlTable)) { + return NULL; + } + + for (Child = Table->Child; Child != NULL; Child = Child->Next) { + if ((Child->Key != NULL) && (AsciiStrCmp (Child->Key, Key) == 0)) { + return Child; + } + } + + return NULL; +} + +CONST CHAR8 * +TomlGetString ( + IN TOML_NODE *Table, + IN CONST CHAR8 *Key, + IN CONST CHAR8 *Default + ) +{ + TOML_NODE *Node; + + Node = TomlGet (Table, Key); + return (Node != NULL && Node->Type == TomlString) ? Node->String : Default; +} + +INT64 +TomlGetInteger ( + IN TOML_NODE *Table, + IN CONST CHAR8 *Key, + IN INT64 Default + ) +{ + TOML_NODE *Node; + + Node = TomlGet (Table, Key); + return (Node != NULL && Node->Type == TomlInteger) ? Node->Integer : Default; +} + +BOOLEAN +TomlGetBoolean ( + IN TOML_NODE *Table, + IN CONST CHAR8 *Key, + IN BOOLEAN Default + ) +{ + TOML_NODE *Node; + + Node = TomlGet (Table, Key); + return (Node != NULL && Node->Type == TomlBoolean) ? Node->Boolean : Default; +} + +UINTN +TomlCount ( + IN TOML_NODE *Array + ) +{ + UINTN Count; + TOML_NODE *Child; + + if ((Array == NULL) || (Array->Type != TomlArray)) { + return 0; + } + + Count = 0; + for (Child = Array->Child; Child != NULL; Child = Child->Next) { + Count++; + } + + return Count; +} + +STATIC +BOOLEAN +IsBareKeyChar ( + IN CHAR8 C + ) +{ + return (C >= 'A' && C <= 'Z') || (C >= 'a' && C <= 'z') || + (C >= '0' && C <= '9') || C == '_' || C == '-'; +} + +STATIC +UINT8 +HexValue ( + IN CHAR8 C + ) +{ + if ((C >= '0') && (C <= '9')) { + return (UINT8)(C - '0'); + } + + if ((C >= 'a') && (C <= 'f')) { + return (UINT8)(C - 'a' + 10); + } + + if ((C >= 'A') && (C <= 'F')) { + return (UINT8)(C - 'A' + 10); + } + + return 0xFF; +} + +// +// Parse "..." or '...' at the cursor into a new NUL-terminated buffer. +// Non-ASCII \u escapes become '?'; raw UTF-8 bytes pass through. +// +STATIC +CHAR8 * +ParseString ( + IN TOML_PARSER *P + ) +{ + CHAR8 Quote; + CHAR8 *Out; + UINTN Len; + UINTN Cap; + CHAR8 C; + UINTN Index; + UINT32 Code; + UINTN Digits; + + Quote = Advance (P); + Cap = 32; + Len = 0; + Out = AllocatePool (Cap); + if (Out == NULL) { + Fail (P, "out of memory"); + return NULL; + } + + for ( ; ;) { + C = Peek (P); + if ((C == '\0') || (C == '\n')) { + Fail (P, "unterminated string"); + FreePool (Out); + return NULL; + } + + Advance (P); + if (C == Quote) { + break; + } + + if ((C == '\\') && (Quote == '"')) { + C = Advance (P); + switch (C) { + case 'n': C = '\n'; + break; + case 't': C = '\t'; + break; + case 'r': C = '\r'; + break; + case '\\': C = '\\'; + break; + case '"': C = '"'; + break; + case 'u': + case 'U': + Digits = (C == 'u') ? 4 : 8; + Code = 0; + for (Index = 0; Index < Digits; Index++) { + UINT8 V = HexValue (Advance (P)); + if (V == 0xFF) { + Fail (P, "bad \\u escape"); + FreePool (Out); + return NULL; + } + + Code = (Code << 4) | V; + } + + C = (Code < 0x80) ? (CHAR8)Code : '?'; + break; + default: + Fail (P, "unknown escape in string"); + FreePool (Out); + return NULL; + } + } + + if (Len + 2 > Cap) { + Out = ReallocatePool (Cap, Cap * 2, Out); + Cap = Cap * 2; + if (Out == NULL) { + Fail (P, "out of memory"); + return NULL; + } + } + + Out[Len++] = C; + } + + Out[Len] = '\0'; + return Out; +} + +STATIC +CHAR8 * +ParseKeyPart ( + IN TOML_PARSER *P + ) +{ + UINTN Start; + CHAR8 *Key; + + SkipSpaces (P); + if ((Peek (P) == '"') || (Peek (P) == '\'')) { + return ParseString (P); + } + + Start = P->Pos; + while (IsBareKeyChar (Peek (P))) { + Advance (P); + } + + if (P->Pos == Start) { + Fail (P, "expected a key"); + return NULL; + } + + Key = AllocateZeroPool (P->Pos - Start + 1); + if (Key == NULL) { + Fail (P, "out of memory"); + return NULL; + } + + CopyMem (Key, P->Text + Start, P->Pos - Start); + return Key; +} + +// +// Find or create the table member Key of Table. A member that is an array of +// tables resolves to its last element (TOML semantics for [a.b] after [[a]]). +// +STATIC +TOML_NODE * +DescendTable ( + IN TOML_PARSER *P, + IN TOML_NODE *Table, + IN CHAR8 *Key + ) +{ + TOML_NODE *Node; + + Node = TomlGet (Table, Key); + if (Node == NULL) { + Node = NewNode (P, TomlTable); + if (Node == NULL) { + FreePool (Key); + return NULL; + } + + Node->Key = Key; + AppendChild (Table, Node); + return Node; + } + + FreePool (Key); + if (Node->Type == TomlTable) { + return Node; + } + + if ((Node->Type == TomlArray) && Node->TableArray && (Node->LastChild != NULL)) { + return Node->LastChild; + } + + Fail (P, "key already holds a value"); + return NULL; +} + +STATIC +TOML_NODE * +ParseValue ( + IN TOML_PARSER *P + ); + +STATIC +BOOLEAN +ParseKeyValue ( + IN TOML_PARSER *P, + IN TOML_NODE *Table + ); + +STATIC +TOML_NODE * +ParseInteger ( + IN TOML_PARSER *P + ) +{ + TOML_NODE *Node; + BOOLEAN Negative; + UINT64 Value; + UINTN Base; + UINTN Digits; + UINT8 D; + CHAR8 C; + + Negative = FALSE; + if ((Peek (P) == '+') || (Peek (P) == '-')) { + Negative = (Advance (P) == '-'); + } + + Base = 10; + if ((Peek (P) == '0') && (P->Pos + 1 < P->Length)) { + C = P->Text[P->Pos + 1]; + if ((C == 'x') || (C == 'o') || (C == 'b')) { + Base = (C == 'x') ? 16 : ((C == 'o') ? 8 : 2); + Advance (P); + Advance (P); + } + } + + Value = 0; + Digits = 0; + for ( ; ;) { + C = Peek (P); + if ((C == '_') && (Digits > 0)) { + Advance (P); + continue; + } + + D = HexValue (C); + if ((D == 0xFF) || (D >= Base)) { + break; + } + + Value = Value * Base + D; + Digits++; + Advance (P); + } + + if ((Digits == 0) || (Peek (P) == '.') || (Peek (P) == 'e') || (Peek (P) == 'E')) { + Fail (P, "invalid number (floats are not supported)"); + return NULL; + } + + Node = NewNode (P, TomlInteger); + if (Node != NULL) { + Node->Integer = Negative ? -(INT64)Value : (INT64)Value; + } + + return Node; +} + +STATIC +TOML_NODE * +ParseArray ( + IN TOML_PARSER *P + ) +{ + TOML_NODE *Array; + TOML_NODE *Item; + + Advance (P); + Array = NewNode (P, TomlArray); + if (Array == NULL) { + return NULL; + } + + for ( ; ;) { + SkipBlank (P); + if (Peek (P) == ']') { + Advance (P); + return Array; + } + + Item = ParseValue (P); + if (Item == NULL) { + TomlFree (Array); + return NULL; + } + + AppendChild (Array, Item); + SkipBlank (P); + if (Peek (P) == ',') { + Advance (P); + } else if (Peek (P) != ']') { + Fail (P, "expected ',' or ']' in array"); + TomlFree (Array); + return NULL; + } + } +} + +STATIC +TOML_NODE * +ParseInlineTable ( + IN TOML_PARSER *P + ) +{ + TOML_NODE *Table; + + Advance (P); + Table = NewNode (P, TomlTable); + if (Table == NULL) { + return NULL; + } + + Table->Defined = TRUE; + SkipSpaces (P); + if (Peek (P) == '}') { + Advance (P); + return Table; + } + + for ( ; ;) { + if (!ParseKeyValue (P, Table)) { + TomlFree (Table); + return NULL; + } + + SkipSpaces (P); + if (Peek (P) == ',') { + Advance (P); + continue; + } + + if (Peek (P) == '}') { + Advance (P); + return Table; + } + + Fail (P, "expected ',' or '}' in inline table"); + TomlFree (Table); + return NULL; + } +} + +STATIC +TOML_NODE * +ParseValue ( + IN TOML_PARSER *P + ) +{ + TOML_NODE *Node; + CHAR8 C; + + SkipSpaces (P); + C = Peek (P); + if ((C == '"') || (C == '\'')) { + if ((P->Pos + 2 < P->Length) && (P->Text[P->Pos + 1] == C) && (P->Text[P->Pos + 2] == C)) { + Fail (P, "multi-line strings are not supported"); + return NULL; + } + + Node = NewNode (P, TomlString); + if (Node != NULL) { + Node->String = ParseString (P); + if (Node->String == NULL) { + FreePool (Node); + return NULL; + } + } + + return Node; + } + + if (C == '[') { + return ParseArray (P); + } + + if (C == '{') { + return ParseInlineTable (P); + } + + if ((P->Length - P->Pos >= 4) && (AsciiStrnCmp (P->Text + P->Pos, "true", 4) == 0)) { + P->Pos += 4; + Node = NewNode (P, TomlBoolean); + if (Node != NULL) { + Node->Boolean = TRUE; + } + + return Node; + } + + if ((P->Length - P->Pos >= 5) && (AsciiStrnCmp (P->Text + P->Pos, "false", 5) == 0)) { + P->Pos += 5; + return NewNode (P, TomlBoolean); + } + + if (((C >= '0') && (C <= '9')) || (C == '+') || (C == '-')) { + return ParseInteger (P); + } + + Fail (P, "expected a value"); + return NULL; +} + +// +// key = value, with dotted keys creating intermediate tables. +// +STATIC +BOOLEAN +ParseKeyValue ( + IN TOML_PARSER *P, + IN TOML_NODE *Table + ) +{ + CHAR8 *Key; + TOML_NODE *Value; + + Key = ParseKeyPart (P); + if (Key == NULL) { + return FALSE; + } + + SkipSpaces (P); + while (Peek (P) == '.') { + Advance (P); + Table = DescendTable (P, Table, Key); + if (Table == NULL) { + return FALSE; + } + + Key = ParseKeyPart (P); + if (Key == NULL) { + return FALSE; + } + + SkipSpaces (P); + } + + if (Peek (P) != '=') { + Fail (P, "expected '='"); + FreePool (Key); + return FALSE; + } + + Advance (P); + if (TomlGet (Table, Key) != NULL) { + Fail (P, "duplicate key"); + FreePool (Key); + return FALSE; + } + + Value = ParseValue (P); + if (Value == NULL) { + FreePool (Key); + return FALSE; + } + + Value->Key = Key; + AppendChild (Table, Value); + return TRUE; +} + +// +// [a.b.c] or [[a.b.c]]; returns the table that following keys go into. +// +STATIC +TOML_NODE * +ParseHeader ( + IN TOML_PARSER *P, + IN TOML_NODE *Root + ) +{ + BOOLEAN IsArray; + TOML_NODE *Table; + TOML_NODE *Node; + CHAR8 *Key; + + Advance (P); + IsArray = (Peek (P) == '['); + if (IsArray) { + Advance (P); + } + + Table = Root; + for ( ; ;) { + Key = ParseKeyPart (P); + if (Key == NULL) { + return NULL; + } + + SkipSpaces (P); + if (Peek (P) == '.') { + Advance (P); + Table = DescendTable (P, Table, Key); + if (Table == NULL) { + return NULL; + } + + continue; + } + + break; + } + + if (Peek (P) != ']') { + Fail (P, "expected ']'"); + FreePool (Key); + return NULL; + } + + Advance (P); + if (IsArray) { + if (Peek (P) != ']') { + Fail (P, "expected ']]'"); + FreePool (Key); + return NULL; + } + + Advance (P); + } + + ExpectLineEnd (P); + if (P->Failed) { + FreePool (Key); + return NULL; + } + + Node = TomlGet (Table, Key); + if (IsArray) { + if (Node == NULL) { + Node = NewNode (P, TomlArray); + if (Node == NULL) { + FreePool (Key); + return NULL; + } + + Node->Key = Key; + Node->TableArray = TRUE; + AppendChild (Table, Node); + } else { + FreePool (Key); + if ((Node->Type != TomlArray) || !Node->TableArray) { + Fail (P, "[[header]] reuses a non-array key"); + return NULL; + } + } + + Table = NewNode (P, TomlTable); + if (Table == NULL) { + return NULL; + } + + Table->Defined = TRUE; + AppendChild (Node, Table); + return Table; + } + + if (Node == NULL) { + Node = NewNode (P, TomlTable); + if (Node == NULL) { + FreePool (Key); + return NULL; + } + + Node->Key = Key; + AppendChild (Table, Node); + } else { + FreePool (Key); + if ((Node->Type != TomlTable) || Node->Defined) { + Fail (P, "table defined twice"); + return NULL; + } + } + + Node->Defined = TRUE; + return Node; +} + +EFI_STATUS +TomlParse ( + IN CONST CHAR8 *Text, + IN UINTN Length, + OUT TOML_NODE **Root, + OUT CHAR8 *ErrorBuffer, + IN UINTN ErrorBufferSize + ) +{ + TOML_PARSER P; + TOML_NODE *Current; + + ZeroMem (&P, sizeof (P)); + P.Text = Text; + P.Length = Length; + P.Line = 1; + P.Error = ErrorBuffer; + P.ErrorSize = ErrorBufferSize; + ErrorBuffer[0] = '\0'; + + // + // Skip a UTF-8 byte order mark. + // + if ((Length >= 3) && ((UINT8)Text[0] == 0xEF) && ((UINT8)Text[1] == 0xBB) && ((UINT8)Text[2] == 0xBF)) { + P.Pos = 3; + } + + *Root = NewNode (&P, TomlTable); + if (*Root == NULL) { + return EFI_OUT_OF_RESOURCES; + } + + Current = *Root; + while (!P.Failed && P.Pos < P.Length) { + SkipBlank (&P); + if (P.Pos >= P.Length) { + break; + } + + if (Peek (&P) == '[') { + Current = ParseHeader (&P, *Root); + } else if (ParseKeyValue (&P, Current)) { + ExpectLineEnd (&P); + } + } + + if (P.Failed) { + TomlFree (*Root); + *Root = NULL; + return EFI_INVALID_PARAMETER; + } + + return EFI_SUCCESS; +} diff --git a/package/CrabApplePkg/Library/CrabAppleLib/Ui.c b/package/CrabApplePkg/Library/CrabAppleLib/Ui.c new file mode 100644 index 0000000..2ea325f --- /dev/null +++ b/package/CrabApplePkg/Library/CrabAppleLib/Ui.c @@ -0,0 +1,446 @@ +/** @file + Menu rendering on EFI_GRAPHICS_OUTPUT_PROTOCOL, with a text console + fallback and a headless mode (log only). + + Graphics mode never prints through ConOut, so firmware consoles that mirror + ConOut onto the same framebuffer cannot scribble over the menu. + + Copyright (c) 2026, UniXNU contributors. All rights reserved. + SPDX-License-Identifier: BSD-2-Clause-Patent +**/ + +#include "Internal.h" + +#define PIXEL(r, g, b) { (b), (g), (r), 0 } + +STATIC CONST EFI_GRAPHICS_OUTPUT_BLT_PIXEL mBackground = PIXEL (0x0B, 0x0C, 0x0F); +STATIC CONST EFI_GRAPHICS_OUTPUT_BLT_PIXEL mText = PIXEL (0xF2, 0xF3, 0xF5); +STATIC CONST EFI_GRAPHICS_OUTPUT_BLT_PIXEL mDim = PIXEL (0x7C, 0x83, 0x8D); +STATIC CONST EFI_GRAPHICS_OUTPUT_BLT_PIXEL mAccent = PIXEL (0x2A, 0x5B, 0xD7); +STATIC CONST EFI_GRAPHICS_OUTPUT_BLT_PIXEL mRow = PIXEL (0x17, 0x1A, 0x20); +STATIC CONST EFI_GRAPHICS_OUTPUT_BLT_PIXEL mError = PIXEL (0xFF, 0x8A, 0x80); +STATIC CONST EFI_GRAPHICS_OUTPUT_BLT_PIXEL mRule = PIXEL (0x26, 0x2A, 0x31); +STATIC CONST EFI_GRAPHICS_OUTPUT_BLT_PIXEL mMarker = PIXEL (0xFF, 0xFF, 0xFF); + +// +// Rows of the status area (CaUiDrawStatus Line numbers). +// +#define STATUS_LINES 3 + +STATIC +VOID +FillRect ( + IN CA_UI *Ui, + IN UINTN X, + IN UINTN Y, + IN UINTN W, + IN UINTN H, + IN EFI_GRAPHICS_OUTPUT_BLT_PIXEL Color + ) +{ + if ((X >= Ui->Width) || (Y >= Ui->Height)) { + return; + } + + W = MIN (W, Ui->Width - X); + H = MIN (H, Ui->Height - Y); + Ui->Gop->Blt (Ui->Gop, &Color, EfiBltVideoFill, 0, 0, X, Y, W, H, 0); +} + +// +// Draw Text with its top-left at (X, Y) on a Background box. Returns the +// width in pixels. Characters outside printable ASCII render as '?'. +// +STATIC +UINTN +DrawText ( + IN CA_UI *Ui, + IN UINTN X, + IN UINTN Y, + IN CONST CHAR16 *Text, + IN UINTN Scale, + IN EFI_GRAPHICS_OUTPUT_BLT_PIXEL Foreground, + IN EFI_GRAPHICS_OUTPUT_BLT_PIXEL Background, + IN UINTN MaxWidth + ) +{ + UINTN Len; + UINTN Cw; + UINTN Ch; + UINTN W; + EFI_GRAPHICS_OUTPUT_BLT_PIXEL *Buffer; + UINTN Index; + UINTN Row; + UINTN Col; + UINTN Sy; + UINTN Sx; + CHAR16 C; + CONST UINT8 *Glyph; + UINTN Base; + + Len = StrLen (Text); + Cw = CA_GLYPH_WIDTH * Scale; + Ch = CA_GLYPH_HEIGHT * Scale; + if ((MaxWidth > 0) && (Len * Cw > MaxWidth)) { + Len = MaxWidth / Cw; + } + + if ((Len == 0) || (X >= Ui->Width) || (Y + Ch > Ui->Height)) { + return 0; + } + + Len = MIN (Len, (Ui->Width - X) / Cw); + W = Len * Cw; + if (W == 0) { + return 0; + } + + Buffer = AllocatePool (W * Ch * sizeof (EFI_GRAPHICS_OUTPUT_BLT_PIXEL)); + if (Buffer == NULL) { + return 0; + } + + for (Index = 0; Index < W * Ch; Index++) { + Buffer[Index] = Background; + } + + for (Index = 0; Index < Len; Index++) { + C = Text[Index]; + Glyph = gCaFontGlyphs[((C >= 0x20) && (C <= 0x7E)) ? (C - 0x20) : ('?' - 0x20)]; + for (Row = 0; Row < CA_GLYPH_HEIGHT; Row++) { + for (Col = 0; Col < CA_GLYPH_WIDTH; Col++) { + if ((Glyph[Row] & (0x80 >> Col)) == 0) { + continue; + } + + Base = (Row * Scale) * W + Index * Cw + Col * Scale; + for (Sy = 0; Sy < Scale; Sy++) { + for (Sx = 0; Sx < Scale; Sx++) { + Buffer[Base + Sy * W + Sx] = Foreground; + } + } + } + } + } + + Ui->Gop->Blt (Ui->Gop, Buffer, EfiBltBufferToVideo, 0, 0, X, Y, W, Ch, 0); + FreePool (Buffer); + return W; +} + +STATIC +VOID +DrawCentered ( + IN CA_UI *Ui, + IN UINTN Y, + IN CONST CHAR16 *Text, + IN UINTN Scale, + IN EFI_GRAPHICS_OUTPUT_BLT_PIXEL Foreground + ) +{ + UINTN W; + + W = MIN (StrLen (Text) * CA_GLYPH_WIDTH * Scale, Ui->Width); + DrawText (Ui, (Ui->Width - W) / 2, Y, Text, Scale, Foreground, mBackground, Ui->Width); +} + +// --------------------------------------------------------------------------- +// Text console +// --------------------------------------------------------------------------- + +STATIC +VOID +TextPrintAt ( + IN CA_UI *Ui, + IN UINTN Row, + IN UINTN Attribute, + IN CONST CHAR16 *Text + ) +{ + EFI_SIMPLE_TEXT_OUTPUT_PROTOCOL *Out; + CHAR16 Line[160]; + UINTN Width; + UINTN Len; + + Out = gST->ConOut; + if ((Out == NULL) || (Row >= Ui->TextRows)) { + return; + } + + // + // Pad to the full width (minus the last column, which would scroll). + // + Width = MIN (Ui->TextColumns - 1, ARRAY_SIZE (Line) - 1); + Len = MIN (StrLen (Text), Width); + CopyMem (Line, Text, Len * sizeof (CHAR16)); + while (Len < Width) { + Line[Len++] = L' '; + } + + Line[Len] = L'\0'; + Out->SetAttribute (Out, Attribute); + Out->SetCursorPosition (Out, 0, Row); + Out->OutputString (Out, Line); + Out->SetAttribute (Out, EFI_LIGHTGRAY | EFI_BACKGROUND_BLACK); +} + +// --------------------------------------------------------------------------- +// Public +// --------------------------------------------------------------------------- + +VOID +CaUiInit ( + IN CA_CONFIG *Config, + OUT CA_UI *Ui + ) +{ + EFI_STATUS Status; + + ZeroMem (Ui, sizeof (*Ui)); + Ui->Mode = CaUiHeadless; + + if ((Config->Console == CaConsoleAuto) || (Config->Console == CaConsoleGraphics)) { + Status = EFI_NOT_FOUND; + if (gST->ConsoleOutHandle != NULL) { + Status = gBS->HandleProtocol (gST->ConsoleOutHandle, &gEfiGraphicsOutputProtocolGuid, (VOID **)&Ui->Gop); + } + + if (EFI_ERROR (Status)) { + Status = gBS->LocateProtocol (&gEfiGraphicsOutputProtocolGuid, NULL, (VOID **)&Ui->Gop); + } + + if (!EFI_ERROR (Status) && (Ui->Gop->Mode != NULL) && (Ui->Gop->Mode->Info != NULL)) { + Ui->Mode = CaUiGraphics; + Ui->Width = Ui->Gop->Mode->Info->HorizontalResolution; + Ui->Height = Ui->Gop->Mode->Info->VerticalResolution; + Ui->Scale = (Config->Scale > 0) ? Config->Scale : MAX (1, Ui->Height / 540); + } + } + + if ((Ui->Mode == CaUiHeadless) && (Config->Console != CaConsoleNone) && (gST->ConOut != NULL)) { + if (!EFI_ERROR (gST->ConOut->QueryMode (gST->ConOut, gST->ConOut->Mode->Mode, &Ui->TextColumns, &Ui->TextRows))) { + Ui->Mode = CaUiText; + } + } + + if (Ui->Mode == CaUiGraphics) { + CaLog ("ui: graphics %ux%u, scale %u", Ui->Width, Ui->Height, (UINT32)Ui->Scale); + } else if (Ui->Mode == CaUiText) { + CaLog ("ui: text console %ux%u", (UINT32)Ui->TextColumns, (UINT32)Ui->TextRows); + } else { + CaLog ("ui: headless (no GOP, no text console)"); + } +} + +VOID +CaUiDrawFrame ( + IN CA_UI *Ui, + IN CA_CONFIG *Config + ) +{ + UINTN Logo; + UINTN Y; + UINTN Cw; + UINTN Ch; + UINTN Index; + UINTN Longest; + UINTN MaxRows; + EFI_GRAPHICS_OUTPUT_BLT_PIXEL *Buffer; + CHAR16 Line[128]; + + if (Ui->Mode == CaUiText) { + gST->ConOut->EnableCursor (gST->ConOut, FALSE); + gST->ConOut->ClearScreen (gST->ConOut); + UnicodeSPrint (Line, sizeof (Line), L"%s (UniXNU %a)", Config->Title, CA_VERSION_STRING); + TextPrintAt (Ui, 0, EFI_WHITE | EFI_BACKGROUND_BLACK, Line); + TextPrintAt (Ui, 1, EFI_DARKGRAY | EFI_BACKGROUND_BLACK, Config->SourceLabel); + Ui->MenuTop = 3; + Ui->VisibleRows = MIN (Config->EntryCount, Ui->TextRows > 8 ? Ui->TextRows - 8 : 1); + Ui->FooterTop = Ui->TextRows - STATUS_LINES - 2; + if (Config->Error[0] != '\0') { + UnicodeSPrint (Line, sizeof (Line), L"config error: %a", Config->Error); + TextPrintAt (Ui, Ui->FooterTop + STATUS_LINES, EFI_LIGHTRED | EFI_BACKGROUND_BLACK, Line); + } else if (Config->Notice[0] != '\0') { + UnicodeSPrint (Line, sizeof (Line), L"%a", Config->Notice); + TextPrintAt (Ui, Ui->FooterTop + STATUS_LINES, EFI_YELLOW | EFI_BACKGROUND_BLACK, Line); + } + + if (Ui->Info[0] != L'\0') { + TextPrintAt (Ui, Ui->FooterTop + STATUS_LINES + 1, EFI_DARKGRAY | EFI_BACKGROUND_BLACK, Ui->Info); + } + + return; + } + + if (Ui->Mode != CaUiGraphics) { + return; + } + + Cw = CA_GLYPH_WIDTH * Ui->Scale; + Ch = CA_GLYPH_HEIGHT * Ui->Scale; + FillRect (Ui, 0, 0, Ui->Width, Ui->Height, mBackground); + + // + // Logo (it carries the UniXNU wordmark), title and provenance. + // + Y = Ui->Height / 18; + if (Config->ShowLogo) { + Logo = MAX (96, MIN (Ui->Width, Ui->Height) * 3 / 10); + Buffer = AllocatePool (Logo * Logo * sizeof (EFI_GRAPHICS_OUTPUT_BLT_PIXEL)); + if (Buffer != NULL) { + CaLogoRender (Buffer, Logo, mBackground); + Ui->Gop->Blt (Ui->Gop, Buffer, EfiBltBufferToVideo, 0, 0, (Ui->Width - Logo) / 2, Y, Logo, Logo, 0); + FreePool (Buffer); + } + + Y += Logo + Ch / 2; + } + + DrawCentered (Ui, Y, Config->Title, Ui->Scale * 2, mText); + Y += Ch * 2 + Ch / 2; + UnicodeSPrint (Line, sizeof (Line), L"UniXNU %a | %s", CA_VERSION_STRING, Config->SourceLabel); + DrawCentered (Ui, Y, Line, Ui->Scale, mDim); + Y += Ch + Ch / 2; + + // + // Menu box: wide enough for the longest title plus the type tag. + // + Longest = 24; + for (Index = 0; Index < Config->EntryCount; Index++) { + Longest = MAX (Longest, StrLen (Config->Entries[Index].Title) + 12); + } + + Ui->RowHeight = Ch + Ch / 2; + Ui->MenuWidth = MIN (Longest * Cw, Ui->Width - 2 * Cw); + Ui->MenuLeft = (Ui->Width - Ui->MenuWidth) / 2; + FillRect (Ui, Ui->MenuLeft, Y, Ui->MenuWidth, MAX (1, Ui->Scale / 2), mRule); + Y += Ch; + Ui->MenuTop = Y; + Ui->FooterTop = Ui->Height - (STATUS_LINES + 2) * (Ch + Ch / 3); + MaxRows = (Ui->FooterTop > Y + Ui->RowHeight) ? (Ui->FooterTop - Y) / Ui->RowHeight : 1; + Ui->VisibleRows = MIN (Config->EntryCount, MaxRows); + + if (Config->Error[0] != '\0') { + UnicodeSPrint (Line, sizeof (Line), L"config error: %a", Config->Error); + DrawCentered (Ui, Ui->FooterTop + STATUS_LINES * (Ch + Ch / 3), Line, Ui->Scale, mError); + } else if (Config->Notice[0] != '\0') { + UnicodeSPrint (Line, sizeof (Line), L"%a", Config->Notice); + DrawCentered (Ui, Ui->FooterTop + STATUS_LINES * (Ch + Ch / 3), Line, Ui->Scale, mError); + } + + if (Ui->Info[0] != L'\0') { + DrawCentered (Ui, Ui->FooterTop + (STATUS_LINES + 1) * (Ch + Ch / 3), Ui->Info, Ui->Scale, mDim); + } +} + +STATIC +CONST CHAR16 * +TypeTag ( + IN CA_ENTRY *Entry + ) +{ + switch (Entry->Type) { + case CaEntryLinux: + return L"linux"; + case CaEntryXnu: + return L"xnu"; + case CaEntryAction: + return L"action"; + default: + return L"efi"; + } +} + +VOID +CaUiDrawEntry ( + IN CA_UI *Ui, + IN CA_CONFIG *Config, + IN UINTN Index, + IN BOOLEAN Selected + ) +{ + CA_ENTRY *Entry; + UINTN Y; + UINTN Pad; + UINTN Cw; + UINTN Ch; + EFI_GRAPHICS_OUTPUT_BLT_PIXEL Bg; + CONST CHAR16 *Tag; + CHAR16 Line[160]; + + if (Index >= Ui->VisibleRows) { + return; + } + + Entry = &Config->Entries[Index]; + if (Ui->Mode == CaUiText) { + UnicodeSPrint (Line, sizeof (Line), L" %c %u. %s [%s]", Selected ? L'>' : L' ', (UINT32)Index + 1, Entry->Title, TypeTag (Entry)); + TextPrintAt ( + Ui, + Ui->MenuTop + Index, + Selected ? (EFI_WHITE | EFI_BACKGROUND_RED) : (EFI_LIGHTGRAY | EFI_BACKGROUND_BLACK), + Line + ); + return; + } + + if (Ui->Mode != CaUiGraphics) { + return; + } + + Cw = CA_GLYPH_WIDTH * Ui->Scale; + Ch = CA_GLYPH_HEIGHT * Ui->Scale; + Y = Ui->MenuTop + Index * Ui->RowHeight; + Pad = (Ui->RowHeight - Ch) / 2; + Bg = Selected ? mAccent : mRow; + Tag = TypeTag (Entry); + + // + // Leave a one-scale-unit gap between rows. + // + FillRect (Ui, Ui->MenuLeft, Y, Ui->MenuWidth, Ui->RowHeight - Ui->Scale, Bg); + if (Selected) { + FillRect (Ui, Ui->MenuLeft, Y, Ui->Scale * 3, Ui->RowHeight - Ui->Scale, mMarker); + } + UnicodeSPrint (Line, sizeof (Line), L"%u %s", (UINT32)Index + 1, Entry->Title); + DrawText (Ui, Ui->MenuLeft + Cw, Y + Pad, Line, Ui->Scale, mText, Bg, Ui->MenuWidth - (StrLen (Tag) + 3) * Cw); + DrawText ( + Ui, + Ui->MenuLeft + Ui->MenuWidth - (StrLen (Tag) + 1) * Cw, + Y + Pad, + Tag, + Ui->Scale, + Selected ? mText : mDim, + Bg, + 0 + ); +} + +VOID +CaUiDrawStatus ( + IN CA_UI *Ui, + IN UINTN Line, + IN CONST CHAR16 *Text + ) +{ + UINTN Ch; + UINTN Y; + + if (Line >= STATUS_LINES) { + return; + } + + if (Ui->Mode == CaUiText) { + TextPrintAt (Ui, Ui->FooterTop + Line, EFI_LIGHTGRAY | EFI_BACKGROUND_BLACK, Text); + return; + } + + if (Ui->Mode != CaUiGraphics) { + return; + } + + Ch = CA_GLYPH_HEIGHT * Ui->Scale; + Y = Ui->FooterTop + Line * (Ch + Ch / 3); + FillRect (Ui, 0, Y, Ui->Width, Ch, mBackground); + DrawCentered (Ui, Y, Text, Ui->Scale, (Line == 0) ? mText : mDim); +} diff --git a/package/CrabApplePkg/Library/CrabAppleLib/Volume.c b/package/CrabApplePkg/Library/CrabAppleLib/Volume.c new file mode 100644 index 0000000..a8324f0 --- /dev/null +++ b/package/CrabApplePkg/Library/CrabAppleLib/Volume.c @@ -0,0 +1,429 @@ +/** @file + File access through EFI_SIMPLE_FILE_SYSTEM_PROTOCOL and image path + resolution. Read-only by design: UniXNU never opens a file for writing. + + Copyright (c) 2026, UniXNU contributors. All rights reserved. + SPDX-License-Identifier: BSD-2-Clause-Patent +**/ + +#include +#include "Internal.h" + +#include +#include + +CHAR16 * +CaAsciiToUnicode ( + IN CONST CHAR8 *String + ) +{ + CHAR16 *Out; + UINTN Len; + UINTN Index; + + if (String == NULL) { + return NULL; + } + + Len = AsciiStrLen (String); + Out = AllocatePool ((Len + 1) * sizeof (CHAR16)); + if (Out == NULL) { + return NULL; + } + + for (Index = 0; Index < Len; Index++) { + // + // UTF-8 continuation/lead bytes cannot be shown by the built-in font. + // + Out[Index] = ((UINT8)String[Index] < 0x80) ? (CHAR16)String[Index] : L'?'; + } + + Out[Len] = L'\0'; + return Out; +} + +INTN +CaStriCmp ( + IN CONST CHAR16 *A, + IN CONST CHAR16 *B + ) +{ + CHAR16 Ca; + CHAR16 Cb; + + do { + Ca = *A++; + Cb = *B++; + if ((Ca >= L'a') && (Ca <= L'z')) { + Ca = (CHAR16)(Ca - L'a' + L'A'); + } + + if ((Cb >= L'a') && (Cb <= L'z')) { + Cb = (CHAR16)(Cb - L'a' + L'A'); + } + } while (Ca != L'\0' && Ca == Cb); + + return (INTN)Ca - (INTN)Cb; +} + +CHAR16 * +CaNormalizePath ( + IN CONST CHAR16 *Base OPTIONAL, + IN CONST CHAR8 *Path + ) +{ + UINTN BaseLen; + UINTN PathLen; + UINTN Size; + CHAR16 *Out; + UINTN O; + UINTN Index; + BOOLEAN Absolute; + CHAR8 C; + + if (Path == NULL) { + return NULL; + } + + Absolute = (Path[0] == '\\' || Path[0] == '/'); + BaseLen = (Base != NULL && !Absolute) ? StrLen (Base) : 0; + PathLen = AsciiStrLen (Path); + Size = (BaseLen + PathLen + 3) * sizeof (CHAR16); + Out = AllocateZeroPool (Size); + if (Out == NULL) { + return NULL; + } + + O = 0; + if (BaseLen > 0) { + CopyMem (Out, Base, BaseLen * sizeof (CHAR16)); + O = BaseLen; + if (Out[O - 1] != L'\\') { + Out[O++] = L'\\'; + } + } else { + Out[O++] = L'\\'; + } + + for (Index = 0; Index < PathLen; Index++) { + C = (Path[Index] == '/') ? '\\' : Path[Index]; + // + // Collapse separators so "a//b" and a leading separator do not double up. + // + if ((C == '\\') && (O > 0) && (Out[O - 1] == L'\\')) { + continue; + } + + Out[O++] = ((UINT8)C < 0x80) ? (CHAR16)C : L'_'; + } + + Out[O] = L'\0'; + return Out; +} + +STATIC +EFI_STATUS +OpenRoot ( + IN EFI_HANDLE Volume, + OUT EFI_FILE_PROTOCOL **Root + ) +{ + EFI_STATUS Status; + EFI_SIMPLE_FILE_SYSTEM_PROTOCOL *Fs; + + if (Volume == NULL) { + return EFI_NOT_FOUND; + } + + Status = gBS->HandleProtocol (Volume, &gEfiSimpleFileSystemProtocolGuid, (VOID **)&Fs); + if (EFI_ERROR (Status)) { + return Status; + } + + return Fs->OpenVolume (Fs, Root); +} + +STATIC +EFI_STATUS +OpenPath ( + IN EFI_HANDLE Volume, + IN CONST CHAR16 *Path, + OUT EFI_FILE_PROTOCOL **File + ) +{ + EFI_STATUS Status; + EFI_FILE_PROTOCOL *Root; + + Status = OpenRoot (Volume, &Root); + if (EFI_ERROR (Status)) { + return Status; + } + + Status = Root->Open (Root, File, (CHAR16 *)Path, EFI_FILE_MODE_READ, 0); + Root->Close (Root); + return Status; +} + +STATIC +EFI_FILE_INFO * +GetInfo ( + IN EFI_FILE_PROTOCOL *File + ) +{ + EFI_STATUS Status; + UINTN Size; + EFI_FILE_INFO *Info; + + Size = 0; + Status = File->GetInfo (File, &gEfiFileInfoGuid, &Size, NULL); + if (Status != EFI_BUFFER_TOO_SMALL) { + return NULL; + } + + Info = AllocatePool (Size); + if (Info == NULL) { + return NULL; + } + + Status = File->GetInfo (File, &gEfiFileInfoGuid, &Size, Info); + if (EFI_ERROR (Status)) { + FreePool (Info); + return NULL; + } + + return Info; +} + +EFI_STATUS +CaReadFile ( + IN EFI_HANDLE Volume, + IN CONST CHAR16 *Path, + OUT VOID **Buffer, + OUT UINTN *Size + ) +{ + EFI_STATUS Status; + EFI_FILE_PROTOCOL *File; + EFI_FILE_INFO *Info; + UINTN ReadSize; + VOID *Data; + + *Buffer = NULL; + *Size = 0; + + Status = OpenPath (Volume, Path, &File); + if (EFI_ERROR (Status)) { + return Status; + } + + Info = GetInfo (File); + if (Info == NULL) { + File->Close (File); + return EFI_DEVICE_ERROR; + } + + if ((Info->Attribute & EFI_FILE_DIRECTORY) != 0) { + FreePool (Info); + File->Close (File); + return EFI_NOT_FOUND; + } + + ReadSize = (UINTN)Info->FileSize; + FreePool (Info); + + // + // One spare zero byte so text files can be treated as C strings. + // + Data = AllocateZeroPool (ReadSize + 1); + if (Data == NULL) { + File->Close (File); + return EFI_OUT_OF_RESOURCES; + } + + Status = File->Read (File, &ReadSize, Data); + File->Close (File); + if (EFI_ERROR (Status)) { + FreePool (Data); + return Status; + } + + *Buffer = Data; + *Size = ReadSize; + return EFI_SUCCESS; +} + +BOOLEAN +CaFileExists ( + IN EFI_HANDLE Volume, + IN CONST CHAR16 *Path + ) +{ + EFI_FILE_PROTOCOL *File; + + if (EFI_ERROR (OpenPath (Volume, Path, &File))) { + return FALSE; + } + + File->Close (File); + return TRUE; +} + +EFI_STATUS +CaListDirectory ( + IN EFI_HANDLE Volume, + IN CONST CHAR16 *Path, + OUT CHAR16 ***Names, + OUT UINTN *Count + ) +{ + EFI_STATUS Status; + EFI_FILE_PROTOCOL *Dir; + UINT8 Buffer[SIZE_OF_EFI_FILE_INFO + 512]; + EFI_FILE_INFO *Info; + UINTN Size; + CHAR16 **List; + UINTN Used; + UINTN Cap; + + *Names = NULL; + *Count = 0; + + Status = OpenPath (Volume, Path, &Dir); + if (EFI_ERROR (Status)) { + return Status; + } + + Cap = 16; + Used = 0; + List = AllocateZeroPool (Cap * sizeof (CHAR16 *)); + if (List == NULL) { + Dir->Close (Dir); + return EFI_OUT_OF_RESOURCES; + } + + Info = (EFI_FILE_INFO *)Buffer; + for ( ; ;) { + Size = sizeof (Buffer); + Status = Dir->Read (Dir, &Size, Info); + if (EFI_ERROR (Status) || (Size == 0)) { + break; + } + + if ((StrCmp (Info->FileName, L".") == 0) || (StrCmp (Info->FileName, L"..") == 0)) { + continue; + } + + if (Used == Cap) { + List = ReallocatePool (Cap * sizeof (CHAR16 *), Cap * 2 * sizeof (CHAR16 *), List); + Cap *= 2; + if (List == NULL) { + Dir->Close (Dir); + return EFI_OUT_OF_RESOURCES; + } + } + + List[Used++] = AllocateCopyPool (StrSize (Info->FileName), Info->FileName); + } + + Dir->Close (Dir); + *Names = List; + *Count = Used; + return EFI_SUCCESS; +} + +STATIC +BOOLEAN +ParseGuid16 ( + IN CONST CHAR16 *Text, + OUT EFI_GUID *Guid + ) +{ + CHAR8 Ascii[37]; + UINTN Index; + + if (StrLen (Text) != 36) { + return FALSE; + } + + for (Index = 0; Index < 36; Index++) { + Ascii[Index] = (CHAR8)Text[Index]; + } + + Ascii[36] = '\0'; + return !RETURN_ERROR (AsciiStrToGuid (Ascii, Guid)); +} + +// +// "fv:": the first firmware volume containing that file. +// +STATIC +EFI_DEVICE_PATH_PROTOCOL * +ResolveFvFile ( + IN CONST CHAR16 *GuidText + ) +{ + EFI_STATUS Status; + EFI_GUID FileGuid; + EFI_HANDLE *Handles; + UINTN Count; + UINTN Index; + EFI_FIRMWARE_VOLUME2_PROTOCOL *Fv; + EFI_FV_FILETYPE Type; + EFI_FV_FILE_ATTRIBUTES Attributes; + UINT32 AuthStatus; + UINTN Size; + MEDIA_FW_VOL_FILEPATH_DEVICE_PATH Node; + EFI_DEVICE_PATH_PROTOCOL *Result; + + if (!ParseGuid16 (GuidText, &FileGuid)) { + return NULL; + } + + Status = gBS->LocateHandleBuffer (ByProtocol, &gEfiFirmwareVolume2ProtocolGuid, NULL, &Count, &Handles); + if (EFI_ERROR (Status)) { + return NULL; + } + + Result = NULL; + for (Index = 0; Index < Count && Result == NULL; Index++) { + Status = gBS->HandleProtocol (Handles[Index], &gEfiFirmwareVolume2ProtocolGuid, (VOID **)&Fv); + if (EFI_ERROR (Status)) { + continue; + } + + Status = Fv->ReadFile (Fv, &FileGuid, NULL, &Size, &Type, &Attributes, &AuthStatus); + if (EFI_ERROR (Status)) { + continue; + } + + EfiInitializeFwVolDevicepathNode (&Node, &FileGuid); + Result = AppendDevicePathNode (DevicePathFromHandle (Handles[Index]), (EFI_DEVICE_PATH_PROTOCOL *)&Node); + } + + FreePool (Handles); + return Result; +} + +EFI_DEVICE_PATH_PROTOCOL * +CaResolveImagePath ( + IN EFI_HANDLE Volume, + IN CONST CHAR16 *Path + ) +{ + if (Path == NULL) { + return NULL; + } + + // + // Normalized paths start with a separator; "fv:" survives as "\fv:". + // + if (StrnCmp (Path, L"\\fv:", 4) == 0) { + return ResolveFvFile (Path + 4); + } + + if (Volume == NULL) { + return NULL; + } + + return FileDevicePath (Volume, Path); +} diff --git a/package/CrabApplePkg/Test/CrabAppleTest.dec b/package/CrabApplePkg/Test/CrabAppleTest.dec new file mode 100644 index 0000000..03668d1 --- /dev/null +++ b/package/CrabApplePkg/Test/CrabAppleTest.dec @@ -0,0 +1,18 @@ +## @file +# Test-only declarations (QEMU smoke payloads). Not needed by platforms. +# +# Copyright (c) 2026, UniXNU contributors. All rights reserved. +# SPDX-License-Identifier: BSD-2-Clause-Patent +## + +[Defines] + DEC_SPECIFICATION = 0x00010019 + PACKAGE_NAME = CrabAppleTest + PACKAGE_GUID = 7736FB07-995C-4602-B3D1-65302C7E2438 + PACKAGE_VERSION = 0.1 + +[Includes] + Include + +[Protocols] + gCrabAppleSampleProtocolGuid = { 0xaaf1116f, 0x7b72, 0x468c, { 0xa2, 0x9e, 0x36, 0x31, 0x73, 0x2f, 0x2b, 0x84 } } diff --git a/package/CrabApplePkg/Test/HelloPayload/HelloPayload.c b/package/CrabApplePkg/Test/HelloPayload/HelloPayload.c new file mode 100644 index 0000000..8aa3681 --- /dev/null +++ b/package/CrabApplePkg/Test/HelloPayload/HelloPayload.c @@ -0,0 +1,337 @@ +/** @file + Smoke-test payload for UniXNU. + + Load options are "": "expect=a,b,c" selects which hand-over checks + must pass (dtb, overlay, driver, initrd, var, slice), "pause" keeps the + report on screen for 15 s before returning, "shutdown" powers the machine + off at the end so an emulator run terminates. Every line it prints starts + with "CRABAPPLE-HELLO:" for log matching. + + Copyright (c) 2026, UniXNU contributors. All rights reserved. + SPDX-License-Identifier: BSD-2-Clause-Patent +**/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +extern EFI_GUID gCrabAppleVariableGuid; + +#pragma pack (1) +typedef struct { + VENDOR_DEVICE_PATH VenMedia; + EFI_DEVICE_PATH_PROTOCOL End; +} INITRD_DEVICE_PATH; +#pragma pack () + +STATIC CONST INITRD_DEVICE_PATH mInitrdPath = { + { + { MEDIA_DEVICE_PATH, MEDIA_VENDOR_DP, { sizeof (VENDOR_DEVICE_PATH), 0 } + }, + LINUX_EFI_INITRD_MEDIA_GUID + }, + { END_DEVICE_PATH_TYPE, END_ENTIRE_DEVICE_PATH_SUBTYPE, { sizeof (EFI_DEVICE_PATH_PROTOCOL), 0 } + } +}; + +STATIC UINTN mPassed; +STATIC UINTN mFailed; + +STATIC +VOID +Report ( + IN CONST CHAR16 *Check, + IN BOOLEAN Ok, + IN CONST CHAR16 *Detail + ) +{ + Print (L"CRABAPPLE-HELLO: check %s %s (%s)\n", Check, Ok ? L"PASS" : L"FAIL", Detail); + if (Ok) { + mPassed++; + } else { + mFailed++; + } +} + +STATIC +CONST CHAR8 * +FdtString ( + IN VOID *Fdt, + IN CONST CHAR8 *Node, + IN CONST CHAR8 *Prop + ) +{ + INT32 Offset; + INT32 Len; + + Offset = FdtPathOffset (Fdt, Node); + if (Offset < 0) { + return NULL; + } + + return FdtGetProp (Fdt, Offset, Prop, &Len); +} + +STATIC +VOID +CheckDtb ( + IN BOOLEAN WantOverlay + ) +{ + VOID *Fdt; + CONST CHAR8 *Bootargs; + CONST CHAR8 *Marker; + CONST CHAR8 *Overlay; + CHAR16 Detail[200]; + + if (EFI_ERROR (EfiGetSystemConfigurationTable (&gFdtTableGuid, &Fdt)) || (FdtCheckHeader (Fdt) != 0)) { + Report (L"dtb", FALSE, L"no EFI_DTB_TABLE"); + if (WantOverlay) { + Report (L"overlay", FALSE, L"no EFI_DTB_TABLE"); + } + + return; + } + + Bootargs = FdtString (Fdt, "/chosen", "bootargs"); + Marker = FdtString (Fdt, "/crabapple-test", "marker"); + UnicodeSPrint ( + Detail, + sizeof (Detail), + L"bootargs='%a' marker='%a'", + (Bootargs != NULL) ? Bootargs : "(none)", + (Marker != NULL) ? Marker : "(none)" + ); + Report (L"dtb", Bootargs != NULL && Marker != NULL && AsciiStrCmp (Marker, "patched") == 0, Detail); + + if (WantOverlay) { + Overlay = FdtString (Fdt, "/crabapple-overlay", "status"); + UnicodeSPrint (Detail, sizeof (Detail), L"/crabapple-overlay status='%a'", (Overlay != NULL) ? Overlay : "(none)"); + Report (L"overlay", Overlay != NULL && AsciiStrCmp (Overlay, "applied") == 0, Detail); + } +} + +STATIC +VOID +CheckDriver ( + VOID + ) +{ + CRABAPPLE_SAMPLE_PROTOCOL *Sample; + CHAR16 Detail[160]; + + if (EFI_ERROR (gBS->LocateProtocol (&gCrabAppleSampleProtocolGuid, NULL, (VOID **)&Sample)) || + (Sample->Signature != CRABAPPLE_SAMPLE_SIGNATURE)) + { + Report (L"driver", FALSE, L"sample protocol missing"); + return; + } + + UnicodeSPrint (Detail, sizeof (Detail), L"sample driver args='%s'", (Sample->LoadOptions != NULL) ? Sample->LoadOptions : L"(none)"); + Report (L"driver", TRUE, Detail); +} + +STATIC +VOID +CheckInitrd ( + VOID + ) +{ + EFI_STATUS Status; + EFI_DEVICE_PATH_PROTOCOL *Path; + EFI_HANDLE Handle; + EFI_LOAD_FILE2_PROTOCOL *LoadFile2; + UINTN Size; + CHAR8 *Data; + CHAR16 Detail[120]; + UINTN Index; + + Path = (EFI_DEVICE_PATH_PROTOCOL *)&mInitrdPath; + Status = gBS->LocateDevicePath (&gEfiLoadFile2ProtocolGuid, &Path, &Handle); + if (EFI_ERROR (Status)) { + Report (L"initrd", FALSE, L"no initrd media"); + return; + } + + gBS->HandleProtocol (Handle, &gEfiLoadFile2ProtocolGuid, (VOID **)&LoadFile2); + Size = 0; + Status = LoadFile2->LoadFile (LoadFile2, Path, FALSE, &Size, NULL); + if ((Status != EFI_BUFFER_TOO_SMALL) || (Size == 0)) { + Report (L"initrd", FALSE, L"size probe failed"); + return; + } + + Data = AllocateZeroPool (Size + 1); + Status = LoadFile2->LoadFile (LoadFile2, Path, FALSE, &Size, Data); + for (Index = 0; Index < Size; Index++) { + if ((Data[Index] < 0x20) || (Data[Index] > 0x7E)) { + Data[Index] = '\0'; + break; + } + } + + UnicodeSPrint (Detail, sizeof (Detail), L"%u bytes, '%.40a'", (UINT32)Size, Data); + Report (L"initrd", !EFI_ERROR (Status), Detail); + FreePool (Data); +} + +STATIC +VOID +CheckVariable ( + VOID + ) +{ + EFI_STATUS Status; + CHAR8 Value[64]; + UINTN Size; + UINT32 Attributes; + CHAR16 Detail[120]; + + ZeroMem (Value, sizeof (Value)); + Size = sizeof (Value) - 1; + Status = gRT->GetVariable (L"CrabAppleTest", &gCrabAppleVariableGuid, &Attributes, &Size, Value); + UnicodeSPrint (Detail, sizeof (Detail), L"CrabAppleTest='%a' attr 0x%x: %r", Value, Attributes, Status); + Report (L"var", !EFI_ERROR (Status), Detail); +} + +// +// A [[block_slice]] device whose first block starts with the tail marker. +// +STATIC +VOID +CheckSlice ( + VOID + ) +{ + EFI_STATUS Status; + EFI_HANDLE *Handles; + UINTN Count; + UINTN Index; + EFI_DEVICE_PATH_PROTOCOL *Node; + CRABAPPLE_BLOCK_SLICE_DEVICE_PATH *Slice; + EFI_BLOCK_IO_PROTOCOL *BlockIo; + CHAR8 *Block; + CHAR16 Detail[160]; + BOOLEAN Found; + + Found = FALSE; + Status = gBS->LocateHandleBuffer (ByProtocol, &gEfiBlockIoProtocolGuid, NULL, &Count, &Handles); + if (EFI_ERROR (Status)) { + Count = 0; + } + + for (Index = 0; Index < Count && !Found; Index++) { + if (EFI_ERROR (gBS->HandleProtocol (Handles[Index], &gEfiDevicePathProtocolGuid, (VOID **)&Node))) { + continue; + } + + while (!IsDevicePathEnd (NextDevicePathNode (Node))) { + Node = NextDevicePathNode (Node); + } + + Slice = (CRABAPPLE_BLOCK_SLICE_DEVICE_PATH *)Node; + if ((DevicePathType (Node) != MEDIA_DEVICE_PATH) || (DevicePathSubType (Node) != MEDIA_VENDOR_DP) || + !CompareGuid (&Slice->Header.Guid, &gCrabAppleBlockSliceGuid)) + { + continue; + } + + Found = TRUE; + gBS->HandleProtocol (Handles[Index], &gEfiBlockIoProtocolGuid, (VOID **)&BlockIo); + Block = AllocateZeroPool (BlockIo->Media->BlockSize + 1); + Status = BlockIo->ReadBlocks (BlockIo, BlockIo->Media->MediaId, 0, BlockIo->Media->BlockSize, Block); + Block[MIN (BlockIo->Media->BlockSize, 40)] = '\0'; + UnicodeSPrint ( + Detail, + sizeof (Detail), + L"'%s' %lu blocks %a, block 0 '%a': %r", + Slice->Name, + BlockIo->Media->LastBlock + 1, + BlockIo->Media->ReadOnly ? "ro" : "rw", + Block, + Status + ); + Report (L"slice", !EFI_ERROR (Status) && AsciiStrnCmp (Block, "CRABAPPLE-TAIL", 14) == 0, Detail); + FreePool (Block); + } + + if (!Found) { + Report (L"slice", FALSE, L"no block_slice device"); + } + + if (Count > 0) { + FreePool (Handles); + } +} + +EFI_STATUS +EFIAPI +HelloPayloadEntry ( + IN EFI_HANDLE ImageHandle, + IN EFI_SYSTEM_TABLE *SystemTable + ) +{ + EFI_LOADED_IMAGE_PROTOCOL *Loaded; + CONST CHAR16 *Args; + + Args = L""; + if (!EFI_ERROR (gBS->HandleProtocol (ImageHandle, &gEfiLoadedImageProtocolGuid, (VOID **)&Loaded)) && + (Loaded->LoadOptions != NULL) && (Loaded->LoadOptionsSize >= sizeof (CHAR16))) + { + Args = Loaded->LoadOptions; + } + + if (gST->ConOut != NULL) { + gST->ConOut->ClearScreen (gST->ConOut); + } + + Print (L"CRABAPPLE-HELLO: Hello from the UniXNU test payload\n"); + Print (L"CRABAPPLE-HELLO: load options '%s'\n", Args); + + if (StrStr (Args, L"dtb") != NULL) { + CheckDtb (StrStr (Args, L"overlay") != NULL); + } + + if (StrStr (Args, L"driver") != NULL) { + CheckDriver (); + } + + if (StrStr (Args, L"initrd") != NULL) { + CheckInitrd (); + } + + if (StrStr (Args, L"var") != NULL) { + CheckVariable (); + } + + if (StrStr (Args, L"slice") != NULL) { + CheckSlice (); + } + + Print (L"CRABAPPLE-HELLO: RESULT %s %u passed %u failed\n", (mFailed == 0) ? L"PASS" : L"FAIL", (UINT32)mPassed, (UINT32)mFailed); + + if (StrStr (Args, L"pause") != NULL) { + gBS->Stall (15 * 1000 * 1000); + } + + if (StrStr (Args, L"shutdown") != NULL) { + gBS->Stall (500000); + gRT->ResetSystem (EfiResetShutdown, EFI_SUCCESS, 0, NULL); + } + + return (mFailed == 0) ? EFI_SUCCESS : EFI_ABORTED; +} diff --git a/package/CrabApplePkg/Test/HelloPayload/HelloPayload.inf b/package/CrabApplePkg/Test/HelloPayload/HelloPayload.inf new file mode 100644 index 0000000..8171f68 --- /dev/null +++ b/package/CrabApplePkg/Test/HelloPayload/HelloPayload.inf @@ -0,0 +1,48 @@ +## @file +# Smoke-test payload: reports what UniXNU handed over (load options, +# patched DTB, injected driver, initrd media, variables) on ConOut. +# +# Copyright (c) 2026, UniXNU contributors. All rights reserved. +# SPDX-License-Identifier: BSD-2-Clause-Patent +## + +[Defines] + INF_VERSION = 0x00010019 + BASE_NAME = HelloPayload + FILE_GUID = 3E16C4EE-F825-4B27-B7BC-E46041D641B5 + MODULE_TYPE = UEFI_APPLICATION + VERSION_STRING = 0.1 + ENTRY_POINT = HelloPayloadEntry + +[Sources] + HelloPayload.c + +[Packages] + MdePkg/MdePkg.dec + CrabApplePkg/CrabApplePkg.dec + CrabApplePkg/Test/CrabAppleTest.dec + +[LibraryClasses] + BaseLib + BaseMemoryLib + DevicePathLib + FdtLib + MemoryAllocationLib + PrintLib + UefiApplicationEntryPoint + UefiBootServicesTableLib + UefiLib + UefiRuntimeServicesTableLib + +[Guids] + gCrabAppleBlockSliceGuid ## CONSUMES + gCrabAppleVariableGuid ## CONSUMES + gFdtTableGuid ## CONSUMES + gLinuxEfiInitrdMediaGuid ## CONSUMES + +[Protocols] + gCrabAppleSampleProtocolGuid ## CONSUMES + gEfiBlockIoProtocolGuid ## CONSUMES + gEfiDevicePathProtocolGuid ## CONSUMES + gEfiLoadFile2ProtocolGuid ## CONSUMES + gEfiLoadedImageProtocolGuid ## CONSUMES diff --git a/package/CrabApplePkg/Test/Include/Protocol/CrabAppleSample.h b/package/CrabApplePkg/Test/Include/Protocol/CrabAppleSample.h new file mode 100644 index 0000000..f3c5c70 --- /dev/null +++ b/package/CrabApplePkg/Test/Include/Protocol/CrabAppleSample.h @@ -0,0 +1,24 @@ +/** @file + Protocol installed by the test SampleDriver so the HelloPayload can prove + that driver injection happened. + + Copyright (c) 2026, UniXNU contributors. All rights reserved. + SPDX-License-Identifier: BSD-2-Clause-Patent +**/ + +#ifndef CRABAPPLE_SAMPLE_PROTOCOL_H_ +#define CRABAPPLE_SAMPLE_PROTOCOL_H_ + +#define CRABAPPLE_SAMPLE_PROTOCOL_GUID \ + { 0xaaf1116f, 0x7b72, 0x468c, { 0xa2, 0x9e, 0x36, 0x31, 0x73, 0x2f, 0x2b, 0x84 } } + +#define CRABAPPLE_SAMPLE_SIGNATURE SIGNATURE_32 ('C', 'A', 'S', 'D') + +typedef struct { + UINT32 Signature; + CHAR16 *LoadOptions; ///< What the driver was started with (may be NULL). +} CRABAPPLE_SAMPLE_PROTOCOL; + +extern EFI_GUID gCrabAppleSampleProtocolGuid; + +#endif diff --git a/package/CrabApplePkg/Test/Qemu/empty/README.txt b/package/CrabApplePkg/Test/Qemu/empty/README.txt new file mode 100644 index 0000000..8b974b9 --- /dev/null +++ b/package/CrabApplePkg/Test/Qemu/empty/README.txt @@ -0,0 +1,2 @@ +Intentionally empty ESP for the degraded smoke scenario: UniXNU finds no +configuration file and falls back to its built-in configuration. diff --git a/package/CrabApplePkg/Test/Qemu/esp/EFI/CrabApple/DeviceTree/test.dts b/package/CrabApplePkg/Test/Qemu/esp/EFI/CrabApple/DeviceTree/test.dts new file mode 100644 index 0000000..3547a49 --- /dev/null +++ b/package/CrabApplePkg/Test/Qemu/esp/EFI/CrabApple/DeviceTree/test.dts @@ -0,0 +1,16 @@ +// Minimal tree the smoke test patches; not a description of real hardware. +/dts-v1/; + +/ { + compatible = "crabapple,smoke-test"; + model = "CrabApple smoke-test tree"; + #address-cells = <2>; + #size-cells = <2>; + + chosen { + }; + + crabapple-test { + marker = "original"; + }; +}; diff --git a/package/CrabApplePkg/Test/Qemu/esp/EFI/CrabApple/Kits/NeverMatch.kit/Kit.toml b/package/CrabApplePkg/Test/Qemu/esp/EFI/CrabApple/Kits/NeverMatch.kit/Kit.toml new file mode 100644 index 0000000..63ec383 --- /dev/null +++ b/package/CrabApplePkg/Test/Qemu/esp/EFI/CrabApple/Kits/NeverMatch.kit/Kit.toml @@ -0,0 +1,13 @@ +# Kit that must be skipped: its [match] names a board that does not exist. +schema = 1 + +[kit] +name = "NeverMatch" +version = "0.0.1" +description = "Negative test for [match]." + +[match] +fdt_compatible = "crabapple,no-such-board" + +[[driver]] +file = "Missing.efi" diff --git a/package/CrabApplePkg/Test/Qemu/esp/EFI/CrabApple/Kits/SampleDriver.kit/Kit.toml b/package/CrabApplePkg/Test/Qemu/esp/EFI/CrabApple/Kits/SampleDriver.kit/Kit.toml new file mode 100644 index 0000000..0f3431a --- /dev/null +++ b/package/CrabApplePkg/Test/Qemu/esp/EFI/CrabApple/Kits/SampleDriver.kit/Kit.toml @@ -0,0 +1,17 @@ +# Kit manifest: the smoke-test driver plus a DT overlay. +schema = 1 + +[kit] +name = "SampleDriver" +version = "1.0.0" +description = "Injection self-test: installs CRABAPPLE_SAMPLE_PROTOCOL and adds /crabapple-overlay." + +[match] +arch = "AARCH64" + +[[driver]] +file = "SampleDriver.efi" +args = "from-kit" + +[[overlay]] +file = "overlay.dtbo" diff --git a/package/CrabApplePkg/Test/Qemu/esp/EFI/CrabApple/Kits/SampleDriver.kit/overlay.dtso b/package/CrabApplePkg/Test/Qemu/esp/EFI/CrabApple/Kits/SampleDriver.kit/overlay.dtso new file mode 100644 index 0000000..d38e1fe --- /dev/null +++ b/package/CrabApplePkg/Test/Qemu/esp/EFI/CrabApple/Kits/SampleDriver.kit/overlay.dtso @@ -0,0 +1,9 @@ +// Overlay applied to every entry DTB while the SampleDriver kit is loaded. +/dts-v1/; +/plugin/; + +&{/} { + crabapple-overlay { + status = "applied"; + }; +}; diff --git a/package/CrabApplePkg/Test/Qemu/esp/EFI/CrabApple/crabapple.toml b/package/CrabApplePkg/Test/Qemu/esp/EFI/CrabApple/crabapple.toml new file mode 100644 index 0000000..3b1e5b5 --- /dev/null +++ b/package/CrabApplePkg/Test/Qemu/esp/EFI/CrabApple/crabapple.toml @@ -0,0 +1,57 @@ +# UniXNU smoke-test configuration (QEMU virt + AAVMF). +# Exercises: kits (match + skip), driver injection, global and per-entry DT +# patches, a DT overlay, /chosen, initrd media, entry variables, timeout. +version = 1 + +[boot] +timeout = 3 +default = "hello" +rotate = 1 + +[ui] +title = "CrabApple" + +[kits] +autoload = true + +# Applied to every entry that has a dtb, before the entry's own patches. +[[dt_patch]] +op = "set" +node = "/crabapple-test" +prop = "marker" +value = "patched" + +[[entry]] +id = "hello" +title = "Hello payload (full injection)" +type = "efi" +path = "Payloads/HelloPayload.efi" +args = "expect dtb overlay driver initrd var shutdown" +dtb = "DeviceTree/test.dtb" +initrd = "initrd.txt" + +[entry.chosen] +bootargs = "console=ttyAMA0 crabapple.test=1" + +[[entry.variable]] +guid = "F50548AC-B469-47C6-89EA-C1738B812165" +name = "CrabAppleTest" +value = "hello-variable" + +[[entry]] +id = "hello-plain" +title = "Hello payload (plain)" +path = "Payloads/HelloPayload.efi" +args = "plain shutdown" + +[[entry]] +id = "reset" +title = "Restart" +type = "action" +action = "reset" + +[[entry]] +id = "shutdown" +title = "Power off" +type = "action" +action = "shutdown" diff --git a/package/CrabApplePkg/Test/Qemu/esp/EFI/CrabApple/initrd.txt b/package/CrabApplePkg/Test/Qemu/esp/EFI/CrabApple/initrd.txt new file mode 100644 index 0000000..dad8b33 --- /dev/null +++ b/package/CrabApplePkg/Test/Qemu/esp/EFI/CrabApple/initrd.txt @@ -0,0 +1 @@ +CRABAPPLE-INITRD-OK this text stands in for an initramfs diff --git a/package/CrabApplePkg/Test/Qemu/userdata/EFI/CrabApple/crabapple.toml b/package/CrabApplePkg/Test/Qemu/userdata/EFI/CrabApple/crabapple.toml new file mode 100644 index 0000000..08889bd --- /dev/null +++ b/package/CrabApplePkg/Test/Qemu/userdata/EFI/CrabApple/crabapple.toml @@ -0,0 +1,19 @@ +# Smoke scenario 4: this volume sits at the start of a GPT partition named +# "userdata"; everything after the first 64 MiB is published as a separate +# block device and checked by the payload. +version = 1 + +[boot] +timeout = 1 +default = "hello" + +[[block_slice]] +partition = "userdata" +name = "passthrough" +offset = 67_108_864 # 64 MiB, where the FAT volume ends + +[[entry]] +id = "hello" +title = "Hello payload (block slice)" +path = "Payloads/HelloPayload.efi" +args = "expect slice shutdown" diff --git a/package/CrabApplePkg/Test/SampleDriver/SampleDriver.c b/package/CrabApplePkg/Test/SampleDriver/SampleDriver.c new file mode 100644 index 0000000..c7fb0f6 --- /dev/null +++ b/package/CrabApplePkg/Test/SampleDriver/SampleDriver.c @@ -0,0 +1,35 @@ +/** @file + Test driver for UniXNU injection: installs CRABAPPLE_SAMPLE_PROTOCOL + carrying the load options it was started with. + + Copyright (c) 2026, UniXNU contributors. All rights reserved. + SPDX-License-Identifier: BSD-2-Clause-Patent +**/ + +#include +#include +#include +#include +#include + +STATIC CRABAPPLE_SAMPLE_PROTOCOL mSample = { CRABAPPLE_SAMPLE_SIGNATURE, NULL }; + +EFI_STATUS +EFIAPI +SampleDriverEntry ( + IN EFI_HANDLE ImageHandle, + IN EFI_SYSTEM_TABLE *SystemTable + ) +{ + EFI_LOADED_IMAGE_PROTOCOL *Loaded; + EFI_HANDLE Handle; + + if (!EFI_ERROR (gBS->HandleProtocol (ImageHandle, &gEfiLoadedImageProtocolGuid, (VOID **)&Loaded)) && + (Loaded->LoadOptions != NULL) && (Loaded->LoadOptionsSize >= sizeof (CHAR16))) + { + mSample.LoadOptions = AllocateCopyPool (Loaded->LoadOptionsSize, Loaded->LoadOptions); + } + + Handle = NULL; + return gBS->InstallMultipleProtocolInterfaces (&Handle, &gCrabAppleSampleProtocolGuid, &mSample, NULL); +} diff --git a/package/CrabApplePkg/Test/SampleDriver/SampleDriver.inf b/package/CrabApplePkg/Test/SampleDriver/SampleDriver.inf new file mode 100644 index 0000000..d28a5cd --- /dev/null +++ b/package/CrabApplePkg/Test/SampleDriver/SampleDriver.inf @@ -0,0 +1,30 @@ +## @file +# Test driver for UniXNU injection: installs CRABAPPLE_SAMPLE_PROTOCOL. +# +# Copyright (c) 2026, UniXNU contributors. All rights reserved. +# SPDX-License-Identifier: BSD-2-Clause-Patent +## + +[Defines] + INF_VERSION = 0x00010019 + BASE_NAME = SampleDriver + FILE_GUID = E11737A2-C846-41C1-850A-594DB97C34C9 + MODULE_TYPE = UEFI_DRIVER + VERSION_STRING = 0.1 + ENTRY_POINT = SampleDriverEntry + +[Sources] + SampleDriver.c + +[Packages] + MdePkg/MdePkg.dec + CrabApplePkg/Test/CrabAppleTest.dec + +[LibraryClasses] + MemoryAllocationLib + UefiBootServicesTableLib + UefiDriverEntryPoint + +[Protocols] + gCrabAppleSampleProtocolGuid ## PRODUCES + gEfiLoadedImageProtocolGuid ## CONSUMES diff --git a/package/CrabApplePkg/Test/XnuTest/XnuTest.c b/package/CrabApplePkg/Test/XnuTest/XnuTest.c new file mode 100644 index 0000000..56cd5af --- /dev/null +++ b/package/CrabApplePkg/Test/XnuTest/XnuTest.c @@ -0,0 +1,106 @@ +/** @file + Synthetic XNU protocol receiver. Never executes a kernel or a kext. + + Copyright (c) 2026, UniXNU contributors. All rights reserved. + SPDX-License-Identifier: BSD-2-Clause-Patent +**/ + +#include +#include +#include +#include +#include +#include +#include +#include + +STATIC BOOLEAN mLegacy; +STATIC UINTN mCalls; + +STATIC +EFI_STATUS +EFIAPI +TestBoot ( + IN CRABAPPLE_XNU_LOADER_PROTOCOL *This, + IN CONST CRABAPPLE_XNU_BOOT_REQUEST *Request + ) +{ + UINTN Index; + UINTN Length; + BOOLEAN Ok; + CHAR8 Plist[256]; + CONST CHAR8 *Names[] = { "Kit", "Codeless", "Code" }; + CONST CHAR16 *Suffixes[] = { L"Kit.kext", L"Codeless.kext", L"Code.kext" }; + CONST CHAR8 Marker[] = "TEST-KEXT-CODE"; + + Ok = (Request->Volume != NULL) && (Request->KernelPath != NULL) && + (StrStr (Request->KernelPath, L"synthetic-kernel") != NULL) && + (Request->BootArgs != NULL) && (StrCmp (Request->BootArgs, L"test handoff") == 0); + if (!Ok) { + Print (L"CRABAPPLE-XNU-TEST: bad base request path '%s' args '%s'\n", + Request->KernelPath, Request->BootArgs); + } + if (!mLegacy) { + Ok = Ok && (Request->KextCount == 3) && (Request->Kexts != NULL); + for (Index = 0; Ok && (Index < Request->KextCount); Index++) { + CONST CRABAPPLE_XNU_KEXT *Kext = &Request->Kexts[Index]; + + Length = AsciiSPrint (Plist, sizeof (Plist), + "CFBundleIdentifier" + "org.crabapple.test.%a", Names[Index]); + Plist[Length++] = '\n'; + Plist[Length] = '\0'; + Ok = (Kext->BundlePath != NULL) && (StrStr (Kext->BundlePath, Suffixes[Index]) != NULL) && + (Kext->PlistPath != NULL) && (Kext->InfoPlist != NULL) && + (Kext->InfoPlistSize == AsciiStrLen (Plist)) && + (CompareMem (Kext->InfoPlist, Plist, Kext->InfoPlistSize) == 0); + if (Index == 1) { + Ok = Ok && (Kext->Executable == NULL) && (Kext->ExecutableSize == 0) && + (Kext->ExecutablePath == NULL); + } else { + Ok = Ok && (Kext->ExecutablePath != NULL) && (Kext->Executable != NULL) && + (Kext->ExecutableSize == 56 + sizeof (Marker)) && + (CompareMem ((CONST UINT8 *)Kext->Executable + 56, Marker, sizeof (Marker)) == 0); + } + + if (!Ok) { + Print (L"CRABAPPLE-XNU-TEST: bad bundle %u plist %u expected %u code %u\n", + (UINT32)Index, (UINT32)Kext->InfoPlistSize, + (UINT32)AsciiStrLen (Plist), (UINT32)Kext->ExecutableSize); + } + } + } + + Print (L"CRABAPPLE-XNU-TEST: %s handoff %s call %u\n", + mLegacy ? L"v1" : L"v2", Ok ? L"PASS" : L"FAIL", (UINT32)++mCalls); + return EFI_ABORTED; +} + +STATIC CRABAPPLE_XNU_LOADER_PROTOCOL mLoader = { CRABAPPLE_XNU_LOADER_REVISION, TestBoot }; + +EFI_STATUS +EFIAPI +XnuTestEntry ( + IN EFI_HANDLE ImageHandle, + IN EFI_SYSTEM_TABLE *SystemTable + ) +{ + EFI_LOADED_IMAGE_PROTOCOL *Loaded; + EFI_HANDLE Handle; + + if (!EFI_ERROR (gBS->HandleProtocol (ImageHandle, &gEfiLoadedImageProtocolGuid, (VOID **)&Loaded)) && + (Loaded->LoadOptions != NULL)) + { + mLegacy = (StrCmp (Loaded->LoadOptions, L"v1") == 0); + if (mLegacy) { + mLoader.Revision = CRABAPPLE_XNU_LOADER_REVISION_1; + } else if (StrCmp (Loaded->LoadOptions, L"unknown") == 0) { + mLoader.Revision = 0x00030000; + } else if (StrCmp (Loaded->LoadOptions, L"nullboot") == 0) { + mLoader.Boot = NULL; + } + } + + Handle = NULL; + return gBS->InstallMultipleProtocolInterfaces (&Handle, &gCrabAppleXnuLoaderProtocolGuid, &mLoader, NULL); +} diff --git a/package/CrabApplePkg/Test/XnuTest/XnuTest.inf b/package/CrabApplePkg/Test/XnuTest/XnuTest.inf new file mode 100644 index 0000000..a2dd249 --- /dev/null +++ b/package/CrabApplePkg/Test/XnuTest/XnuTest.inf @@ -0,0 +1,31 @@ +## @file +# Synthetic XNU loader protocol smoke-test receiver. +# SPDX-License-Identifier: BSD-2-Clause-Patent +## + +[Defines] + INF_VERSION = 0x00010019 + BASE_NAME = XnuTest + FILE_GUID = 90311252-9F79-46D5-83CA-42D6738A5D0C + MODULE_TYPE = UEFI_DRIVER + VERSION_STRING = 0.1 + ENTRY_POINT = XnuTestEntry + +[Sources] + XnuTest.c + +[Packages] + MdePkg/MdePkg.dec + CrabApplePkg/CrabApplePkg.dec + +[LibraryClasses] + BaseLib + BaseMemoryLib + PrintLib + UefiBootServicesTableLib + UefiDriverEntryPoint + UefiLib + +[Protocols] + gCrabAppleXnuLoaderProtocolGuid ## PRODUCES + gEfiLoadedImageProtocolGuid ## CONSUMES diff --git a/package/LICENSE b/package/LICENSE new file mode 100644 index 0000000..f63d2a6 --- /dev/null +++ b/package/LICENSE @@ -0,0 +1,51 @@ +Copyright (c) 2026, UniXNU contributors. All rights reserved. + +SPDX-License-Identifier: BSD-2-Clause-Patent + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +Subject to the terms and conditions of this license, each copyright holder +and contributor hereby grants to those receiving rights under this license +a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable +(except for failure to satisfy the conditions of this license) patent +license to make, have made, use, offer to sell, sell, import, and otherwise +transfer this software, where such license applies only to those patent +claims, already acquired or hereafter acquired, licensable by such copyright +holder or contributor that are necessarily infringed by: + +(a) their Contribution(s) (the licensed copyrights of copyright holders and + non-copyrightable additions of contributors, in source or binary form) + alone; or + +(b) combination of their Contribution(s) with the work of authorship to + which such Contribution(s) was added by such copyright holder or + contributor, if, at the time the Contribution is added, such addition + causes such combination to be necessarily infringed. The patent license + shall not apply to any other combinations which include the + Contribution. + +Except as expressly stated above, no rights or licenses from any copyright +holder or contributor is granted under this license, whether expressly, by +implication, estoppel or otherwise. + +DISCLAIMER + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. diff --git a/package/Scripts/gen-logo.py b/package/Scripts/gen-logo.py new file mode 100755 index 0000000..43a14fb --- /dev/null +++ b/package/Scripts/gen-logo.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026, UniXNU contributors. All rights reserved. +# SPDX-License-Identifier: BSD-2-Clause-Patent +"""Generate the UniXNU logo from one vector description. + +The logo is white strokes on black: an apple outline with a bite, an +arrow-shaped leaf, a chip in the middle and the "UniXNU" wordmark ("Uni" +heavy, "XNU" light). Every shape is a stroked path of lines and Bezier +curves with round joins and caps. + +Outputs: + Docs/Images/unixnu-logo.svg the vector logo + CrabApplePkg/Library/CrabAppleLib/LogoShape.h the same strokes, flattened + to integer polylines for the + firmware renderer (Logo.c) +""" +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + +# Design space: a 1000-unit square (the artwork's 1280 grid, cropped by ORIGIN). +DESIGN = 1000 +ORIGIN = (140, 148) + +# (stroke width, path). Paths: ('M', x, y), ('L', x, y), ('C', x1, y1, x2, y2, x, y), ('Q', x1, y1, x, y), ('Z',). +APPLE = [('M', 655, 372), + ('C', 600, 352, 560, 340, 500, 342), + ('C', 420, 345, 372, 420, 366, 520), + ('C', 360, 625, 400, 730, 460, 800), + ('C', 500, 845, 560, 848, 600, 834), + ('C', 630, 824, 680, 824, 710, 834), + ('C', 750, 848, 805, 845, 840, 800), + ('C', 868, 762, 893, 712, 910, 666), + ('C', 852, 640, 820, 584, 822, 524), + ('C', 824, 472, 850, 434, 896, 408), + ('C', 862, 370, 805, 346, 742, 350), + ('C', 706, 353, 682, 362, 655, 372), + ('Z',)] +LEAF = [('M', 641, 336), + ('C', 643, 296, 664, 266, 694, 238), + ('L', 668, 212), + ('L', 788, 196), + ('L', 776, 300), + ('L', 749, 274), + ('C', 724, 304, 694, 330, 641, 336), + ('Z',)] + + +def rounded_rect(x0, y0, x1, y1, r): + k = 0.5523 * r + return [('M', x0 + r, y0), ('L', x1 - r, y0), ('C', x1 - r + k, y0, x1, y0 + r - k, x1, y0 + r), + ('L', x1, y1 - r), ('C', x1, y1 - r + k, x1 - r + k, y1, x1 - r, y1), + ('L', x0 + r, y1), ('C', x0 + r - k, y1, x0, y1 - r + k, x0, y1 - r), + ('L', x0, y0 + r), ('C', x0, y0 + r - k, x0 + r - k, y0, x0 + r, y0), ('Z',)] + + +CHIP = rounded_rect(561, 511, 719, 669, 34) +PINS = [] +for at in (596, 640, 684): + PINS += [[('M', at, 468), ('L', at, 500)], [('M', at, 680), ('L', at, 712)]] +for at in (546, 590, 634): + PINS += [[('M', 518, at), ('L', 550, at)], [('M', 730, at), ('L', 762, at)]] + +HEAVY = [ # "Uni" + [('M', 259, 952), ('L', 259, 1035), ('Q', 259, 1075, 306, 1075), ('Q', 353, 1075, 353, 1035), ('L', 353, 952)], + [('M', 425, 1075), ('L', 425, 1012)], + [('M', 425, 1038), ('Q', 430, 1012, 462, 1012), ('Q', 496, 1012, 496, 1045), ('L', 496, 1075)], + [('M', 559, 1012), ('L', 559, 1075)], + [('M', 559, 941), ('L', 559, 941)], +] +LIGHT = [ # "XNU" + [('M', 603, 937), ('L', 720, 1088)], + [('M', 720, 937), ('L', 603, 1088)], + [('M', 775, 1088), ('L', 775, 937), ('L', 873, 1088), ('L', 873, 937)], + [('M', 923, 937), ('L', 923, 1035), ('Q', 923, 1088, 980, 1088), ('Q', 1037, 1088, 1037, 1035), ('L', 1037, 937)], +] + +STROKES = ([(24, APPLE), (22, LEAF), (22, CHIP)] + [(16, p) for p in PINS] + [(44, p) for p in HEAVY[:4]] + [(50, HEAVY[4])] + + [(14, p) for p in LIGHT]) + + +def flatten(path, steps=20): + points, start, cur = [], None, None + for seg in path: + op = seg[0] + if op == 'M': + cur = start = seg[1:] + points.append(cur) + elif op == 'L': + cur = seg[1:] + points.append(cur) + elif op == 'Z': + if cur != start: + points.append(start) + else: + ctrl = seg[1:] + for i in range(1, steps + 1): + t, u = i / steps, 1 - i / steps + if op == 'C': + x = u**3 * cur[0] + 3 * u * u * t * ctrl[0] + 3 * u * t * t * ctrl[2] + t**3 * ctrl[4] + y = u**3 * cur[1] + 3 * u * u * t * ctrl[1] + 3 * u * t * t * ctrl[3] + t**3 * ctrl[5] + else: + x = u * u * cur[0] + 2 * u * t * ctrl[0] + t * t * ctrl[2] + y = u * u * cur[1] + 2 * u * t * ctrl[1] + t * t * ctrl[3] + points.append((x, y)) + cur = ctrl[-2:] + return [(round(x - ORIGIN[0]), round(y - ORIGIN[1])) for x, y in points] + + +def svg_path(path): + out = [] + for seg in path: + op, values = seg[0], seg[1:] + moved = [v - ORIGIN[i % 2] for i, v in enumerate(values)] + out.append(op + ' '.join('%g' % v for v in moved)) + return ''.join(out) + + +def main(): + lines = ['' % (DESIGN, DESIGN, DESIGN, DESIGN), + ' UniXNU', + ' ' % (DESIGN, DESIGN), + ' '] + for width, path in STROKES: + lines.append(' ' % (width, svg_path(path))) + lines += [' ', '', ''] + (ROOT / 'Docs/Images/unixnu-logo.svg').write_text('\n'.join(lines)) + + header = ['/** @file', + ' UniXNU logo strokes, generated by Scripts/gen-logo.py from the vector', + ' description that also produces Docs/Images/unixnu-logo.svg. Do not edit.', + '', + ' Copyright (c) 2026, UniXNU contributors. All rights reserved.', + ' SPDX-License-Identifier: BSD-2-Clause-Patent', + '**/', + '', + '#ifndef LOGO_SHAPE_H_', + '#define LOGO_SHAPE_H_', + '', + '#define LOGO_DESIGN %d' % DESIGN, + '', + '//', + '// Points of every stroke, in strokes order.', + '//', + 'STATIC CONST INT16 mLogoPoints[][2] = {'] + strokes = [] + at = 0 + for width, path in STROKES: + points = flatten(path) + header.append(' ' + ', '.join('{ %d, %d }' % p for p in points) + ',') + strokes.append((at, len(points), width // 2)) + at += len(points) + header += ['};', '', + '//', + '// First point, point count and half width of each round-capped stroke.', + '//', + 'STATIC CONST UINT16 mLogoStrokes[][3] = {'] + header += [' { %d, %d, %d },' % s for s in strokes] + header += ['};', '', '#endif', ''] + (ROOT / 'CrabApplePkg/Library/CrabAppleLib/LogoShape.h').write_text('\n'.join(header)) + print('logo: %d strokes, %d points' % (len(strokes), at)) + + +main() diff --git a/package/Scripts/kext-fixtures.py b/package/Scripts/kext-fixtures.py new file mode 100644 index 0000000..1e3862d --- /dev/null +++ b/package/Scripts/kext-fixtures.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +"""Generate synthetic kext staging smoke fixtures. No Apple files are used.""" + +# SPDX-License-Identifier: BSD-2-Clause-Patent + +import argparse +import shutil +import struct +from pathlib import Path + + +def bundle(root: Path, name: str, code: bool = True) -> Path: + path = root / f"{name}.kext" / "Contents" + path.mkdir(parents=True, exist_ok=True) + (path / "Info.plist").write_text( + 'CFBundleIdentifier' + f'org.crabapple.test.{name}\n' + ) + executable = path / "MacOS" / "Test" + if code: + executable.parent.mkdir(exist_ok=True) + executable.write_bytes( + struct.pack("<8I", 0xFEEDFACF, 0x0100000C, 2, 11, 1, 24, 0, 0) + + struct.pack(" None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("out", type=Path) + parser.add_argument("binaries", type=Path) + args = parser.parse_args() + variants = ( + "v2", "v1", "v1-empty", "missing-loader", "unknown", "nullboot", + "missing-file", "bad-image", "truncated-command", "empty-plist", + "duplicate", "bad-path", "bad-type", "wrong-entry", + ) + for variant in variants: + esp = args.out / variant + ca = esp / "EFI" / "CrabApple" + ca.mkdir(parents=True, exist_ok=True) + boot = esp / "EFI" / "BOOT" + boot.mkdir(parents=True, exist_ok=True) + shutil.copy2(args.binaries / "CrabApple.efi", boot / "BOOTAA64.EFI") + shutil.copy2(args.binaries / "XnuTest.efi", ca / "XnuTest.efi") + code = bundle(ca / "Kexts", "Code") + bundle(ca / "Kexts", "Codeless", False) + kit = ca / "Kits" / "Test.kit" + bundle(kit, "Kit") + # This missing bundle must stay scoped to an unselected entry. + (kit / "Kit.toml").write_text( + 'schema = 1\n[kit]\nname = "KextTest"\nversion = "1.0"\n' + '[[kext]]\nentry = "other"\nbundle = "Absent.kext"\n' + '[[kext]]\nentry = "xnu"\nbundle = "Kit.kext"\n' + 'executable = "Contents/MacOS/Test"\n' + ) + mode = variant if variant in {"v1", "unknown", "nullboot"} else "v2" + if variant == "v1-empty": + mode = "v1" + config = 'version = 1\n[boot]\ntimeout = 0\n' + if variant != "v1-empty": + config += '[[kit]]\npath = "Kits/Test.kit"\n' + if variant != "missing-loader": + config += f'[[driver]]\npath = "XnuTest.efi"\nargs = "{mode}"\nconnect = false\n' + entry_type = "efi" if variant == "wrong-entry" else "xnu" + config += ( + f'[[entry]]\nid = "xnu"\ntype = "{entry_type}"\n' + 'title = "Synthetic XNU request"\npath = "synthetic-kernel"\n' + 'args = "test handoff"\n' + ) + if variant != "v1-empty": + config += ( + '[[entry.kext]]\nenabled = false\nbundle = "Absent.kext"\n' + '[[entry.kext]]\nbundle = "Kexts/Codeless.kext"\n' + '[[entry.kext]]\nbundle = "Kexts/Code.kext"\n' + 'executable = "Contents/MacOS/Test"\n' + ) + if variant == "missing-file": + code.unlink() + elif variant == "bad-image": + data = bytearray(code.read_bytes()) + struct.pack_into(" the rest of the disk: + [0, --fat) FAT32 volume populated from --tree + [--fat, ...) raw; its first bytes are --marker + +This models a device partition whose head carries the UniXNU payload +volume and whose tail is handed to the payload as a separate disk through +[[block_slice]]. Needs mkfs.vfat or mformat; files are copied with mcopy +when available, otherwise with the pyfatfs module (run this script with an +interpreter that has it). +""" + +import argparse +import shutil +import struct +import subprocess +import sys +import tempfile +import uuid +import zlib +from pathlib import Path + +MIB = 1024 * 1024 + + +def write_gpt(path: Path, size: int, misc: int, name: str) -> int: + sector = 512 + last = size * MIB // sector - 1 + start = 2048 + misc * MIB // sector + entries = bytearray(128 * 128) + linux_data = uuid.UUID("0fc63daf-8483-4772-8e79-3d69d8477de4").bytes_le + for index, (first, end, label) in enumerate([ + (2048, start - 1, "misc"), (start, last - 33, name) + ]): + encoded = label.encode("utf-16-le") + if len(encoded) > 72: + raise ValueError("GPT partition name exceeds 36 UTF-16 units") + struct.pack_into("<16s16sQQQ72s", entries, index * 128, + linux_data, uuid.uuid4().bytes_le, first, end, 0, encoded) + disk_guid = uuid.uuid4().bytes_le + + def header(current, alternate, table): + data = bytearray(sector) + struct.pack_into("<8sIIIIQQQQ16sQIII", data, 0, + b"EFI PART", 0x10000, 92, 0, 0, current, alternate, + 34, last - 33, disk_guid, table, 128, 128, + zlib.crc32(entries) & 0xFFFFFFFF) + struct.pack_into(" None: + for path in sorted(tree.rglob("*")): + rel = "::/" + path.relative_to(tree).as_posix() + if path.is_dir(): + subprocess.run(["mmd", "-i", str(image), rel], check=True) + else: + subprocess.run(["mcopy", "-i", str(image), str(path), rel], check=True) + + +def populate_pyfatfs(image: Path, tree: Path) -> None: + import warnings + + warnings.filterwarnings("ignore") + from pyfatfs.PyFatFS import PyFatFS + + fat = PyFatFS(str(image)) + try: + for path in sorted(tree.rglob("*")): + rel = "/" + path.relative_to(tree).as_posix() + if path.is_dir(): + fat.makedirs(rel, recreate=True) + else: + with fat.openbin(rel, "wb") as out: + out.write(path.read_bytes()) + finally: + fat.close() + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--out", type=Path, required=True) + parser.add_argument("--tree", type=Path, required=True, help="files for the FAT volume") + parser.add_argument("--size", type=int, default=256, help="disk size, MiB") + parser.add_argument("--misc", type=int, default=8, help="first partition size, MiB") + parser.add_argument("--fat", type=int, default=64, help="FAT volume size, MiB (>= 33)") + parser.add_argument("--name", default="userdata", help="GPT name of the second partition") + parser.add_argument("--marker", default="CRABAPPLE-TAIL-MARKER", help="text at the start of the tail") + args = parser.parse_args() + + if args.fat < 33 or args.misc + args.fat + 2 >= args.size: + sys.exit("error: sizes do not fit (FAT32 needs >= 33 MiB)") + + args.out.parent.mkdir(parents=True, exist_ok=True) + with open(args.out, "wb") as f: + f.truncate(args.size * MIB) + + start = write_gpt(args.out, args.size, args.misc, args.name) + + with tempfile.TemporaryDirectory() as tmp: + fat = Path(tmp) / "fat.img" + if shutil.which("mkfs.vfat"): + subprocess.run(["mkfs.vfat", "-F", "32", "-n", "PAYLOAD", "-C", str(fat), str(args.fat * 1024)], + check=True, capture_output=True) + else: + with fat.open("wb") as image: + image.truncate(args.fat * MIB) + subprocess.run(["mformat", "-i", str(fat), "-F", "-v", "PAYLOAD", "::"], + check=True, capture_output=True) + if shutil.which("mcopy") and shutil.which("mmd"): + populate_mtools(fat, args.tree) + else: + populate_pyfatfs(fat, args.tree) + data = fat.read_bytes() + + with open(args.out, "r+b") as disk: + disk.seek(start) + disk.write(data) + disk.seek(start + args.fat * MIB) + disk.write(args.marker.encode()) + + print(f"{args.out}: partition '{args.name}' at {start} bytes, FAT {args.fat} MiB, tail marker at +{args.fat} MiB") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/package/Scripts/qemu-run.py b/package/Scripts/qemu-run.py new file mode 100755 index 0000000..52ad4c8 --- /dev/null +++ b/package/Scripts/qemu-run.py @@ -0,0 +1,228 @@ +#!/usr/bin/env python3 +"""Boot an ESP directory in QEMU (AArch64 virt + AAVMF) and check the log. + +The ESP is exposed read-only through QEMU's vvfat driver, so nothing on the +host is modified. The serial console is captured to a log; when UniXNU +logs "menu: shown", a screenshot is taken over QMP and optional keys are +sent to the emulated USB keyboard. Each --expect is a regular expression +that must match the (ANSI-stripped) serial log. Exit status 0 = all matched. +""" + +import argparse +import json +import os +import re +import shutil +import socket +import struct +import subprocess +import sys +import tempfile +import time +import zlib +from pathlib import Path + +FIRMWARE_CANDIDATES = [ + ("/usr/share/edk2/aarch64/QEMU_EFI.fd", "/usr/share/edk2/aarch64/QEMU_VARS.fd"), + ("/usr/share/AAVMF/AAVMF_CODE.fd", "/usr/share/AAVMF/AAVMF_VARS.fd"), + ("/usr/share/qemu-efi-aarch64/QEMU_EFI.fd", None), + ("/usr/share/edk2/aarch64/QEMU_CODE.fd", "/usr/share/edk2/aarch64/QEMU_VARS.fd"), +] + +ANSI = re.compile(r"\x1b\[[0-9;?]*[A-Za-z]|\x1b[()][A-Za-z0-9]") + + +def find_firmware(code): + if code: + return Path(code), None + qemu = shutil.which("qemu-system-aarch64") + if qemu: + share = Path(qemu).resolve().parent.parent / "share" / "qemu" + firmware = share / "edk2-aarch64-code.fd" + variables = share / "edk2-arm-vars.fd" + if firmware.is_file(): + return firmware, (variables if variables.is_file() else None) + for c, v in FIRMWARE_CANDIDATES: + if Path(c).is_file(): + return Path(c), (Path(v) if v and Path(v).is_file() else None) + sys.exit("error: no AAVMF firmware found; pass --firmware") + + +class Qmp: + def __init__(self, path, timeout=10.0): + deadline = time.time() + timeout + while True: + try: + self.sock = socket.socket(socket.AF_UNIX) + self.sock.connect(path) + break + except OSError: + if time.time() > deadline: + raise + time.sleep(0.1) + self.file = self.sock.makefile("rw") + self.file.readline() + self.command("qmp_capabilities") + + def command(self, name, **arguments): + msg = {"execute": name} + if arguments: + msg["arguments"] = arguments + self.file.write(json.dumps(msg) + "\n") + self.file.flush() + while True: + reply = json.loads(self.file.readline()) + if "return" in reply or "error" in reply: + return reply + + +def ppm_to_png(ppm: Path, png: Path) -> None: + data = ppm.read_bytes() + parts = data.split(maxsplit=4) + if parts[0] != b"P6": + raise ValueError("not a binary PPM") + width, height = int(parts[1]), int(parts[2]) + pixels = parts[4][: width * height * 3] + raw = b"".join(b"\x00" + pixels[y * width * 3:(y + 1) * width * 3] for y in range(height)) + + def chunk(tag, body): + return struct.pack(">I", len(body)) + tag + body + struct.pack(">I", zlib.crc32(tag + body) & 0xFFFFFFFF) + + png.write_bytes( + b"\x89PNG\r\n\x1a\n" + + chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)) + + chunk(b"IDAT", zlib.compress(raw, 9)) + + chunk(b"IEND", b"") + ) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("esp", type=Path, help="ESP directory") + parser.add_argument("--firmware", help="AAVMF code image (default: first found)") + parser.add_argument("--log", type=Path, required=True, help="serial log output") + parser.add_argument("--screenshot", type=Path, help="PNG of the menu") + parser.add_argument("--keys", default="", help="comma-separated QEMU qcodes sent after the menu shows") + parser.add_argument("--expect", action="append", default=[], help="regex that must match the log") + parser.add_argument("--reject", action="append", default=[], help="regex that must not match the log") + parser.add_argument("--no-gop", action="store_true", help="no display device (text console only)") + parser.add_argument("--disk", action="append", default=[], type=Path, help="extra raw disk image (read-only)") + parser.add_argument("--timeout", type=float, default=90.0) + parser.add_argument("--cpus", type=int, default=2, help="number of virtual CPUs (default: 2)") + parser.add_argument("--finish-on-expect", action="store_true", + help="finish two seconds after all expected patterns match") + parser.add_argument("--menu-marker", default=r"CRABAPPLE: menu: shown") + args = parser.parse_args() + if args.cpus < 1: + parser.error("--cpus must be positive") + if args.finish_on_expect and not args.expect: + parser.error("--finish-on-expect requires --expect") + + qemu = shutil.which("qemu-system-aarch64") + if not qemu: + sys.exit("error: qemu-system-aarch64 not found") + code, vars_template = find_firmware(args.firmware) + + work = Path(tempfile.mkdtemp(prefix="crabapple-qemu-")) + qmp_path = work / "qmp.sock" + args.log.parent.mkdir(parents=True, exist_ok=True) + if args.log.exists(): + args.log.unlink() + + cmd = [ + qemu, "-M", "virt", "-cpu", "cortex-a72", "-m", "1024", "-smp", str(args.cpus), + "-drive", f"if=pflash,format=raw,readonly=on,file={code}", + ] + if vars_template: + vars_copy = work / "vars.fd" + shutil.copy(vars_template, vars_copy) + cmd += ["-drive", f"if=pflash,format=raw,file={vars_copy}"] + cmd += [ + "-drive", f"if=none,id=esp,format=raw,readonly=on,file=fat:{args.esp.resolve()}", + "-device", "virtio-blk-pci,drive=esp", + ] + for index, disk in enumerate(args.disk): + cmd += [ + "-drive", f"if=none,id=disk{index},format=raw,readonly=on,file={disk.resolve()}", + "-device", f"virtio-blk-pci,drive=disk{index}", + ] + if not args.no_gop: + cmd += ["-device", "ramfb"] + cmd += [ + "-device", "qemu-xhci", "-device", "usb-kbd", + "-nic", "none", "-display", "none", "-no-reboot", + "-serial", f"file:{args.log}", + "-qmp", f"unix:{qmp_path},server=on,wait=off", + ] + print("qemu:", " ".join(cmd)) + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + qmp = None + menu_seen = False + expectations_since = None + expectations_complete = False + deadline = time.time() + args.timeout + try: + qmp = Qmp(str(qmp_path)) + while proc.poll() is None and time.time() < deadline: + text = args.log.read_text(errors="replace") if args.log.exists() else "" + if args.finish_on_expect: + clean_text = ANSI.sub("", text) + if all(re.search(pattern, clean_text) for pattern in args.expect): + if expectations_since is None: + expectations_since = time.time() + elif time.time() - expectations_since >= 2.0: + expectations_complete = True + break + if not menu_seen and re.search(args.menu_marker, text): + menu_seen = True + time.sleep(0.6) + if args.screenshot: + ppm = work / "screen.ppm" + reply = qmp.command("screendump", filename=str(ppm)) + if "error" in reply: + print("screendump failed:", reply["error"]) + else: + args.screenshot.parent.mkdir(parents=True, exist_ok=True) + ppm_to_png(ppm, args.screenshot) + print("screenshot:", args.screenshot) + for key in filter(None, args.keys.split(",")): + qmp.command("send-key", keys=[{"type": "qcode", "data": key.strip()}]) + time.sleep(0.3) + time.sleep(0.2) + finally: + if proc.poll() is None: + print("expectations complete: stopping QEMU" if expectations_complete else "timeout: stopping QEMU") + if qmp: + try: + qmp.command("quit") + except OSError: + pass + try: + proc.wait(5) + except subprocess.TimeoutExpired: + proc.kill() + out = proc.stdout.read().decode(errors="replace") if proc.stdout else "" + if out.strip(): + print("qemu output:", out.strip()) + shutil.rmtree(work, ignore_errors=True) + + log = ANSI.sub("", args.log.read_text(errors="replace")) if args.log.exists() else "" + clean = args.log.with_suffix(".clean.log") + clean.write_text(log) + + failed = 0 + for pattern in args.expect: + ok = re.search(pattern, log) is not None + failed += not ok + print(f"{'PASS' if ok else 'FAIL'} expect {pattern}") + for pattern in args.reject: + ok = re.search(pattern, log) is None + failed += not ok + print(f"{'PASS' if ok else 'FAIL'} reject {pattern}") + print(f"log: {clean}") + print("SMOKE", "PASS" if failed == 0 else f"FAIL ({failed})") + return 0 if failed == 0 else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/package/Scripts/smoke.sh b/package/Scripts/smoke.sh new file mode 100755 index 0000000..ace85ff --- /dev/null +++ b/package/Scripts/smoke.sh @@ -0,0 +1,114 @@ +#!/usr/bin/env bash +# QEMU smoke test: build, stage the test ESP, boot in AArch64 QEMU. +# 1. no input: the countdown boots the default entry, which checks every +# hand-over (DTB patches + overlay, /chosen, injected kit driver, initrd +# media, variable) and powers off; +# 2. keyboard: Down + Enter picks the second entry; +# 3. degraded: no display device and no config file, so the built-in +# configuration runs on the text console and its default (Return to +# firmware) fires when the timer ends; +# 4. block slice: UniXNU on a bare ESP finds its config on a FAT volume +# at the start of a GPT partition named "userdata" and publishes the +# rest of that partition as its own block device (needs mtools, or a +# python with pyfatfs in FAT_PYTHON; skipped otherwise). +# Logs and screenshots go to Build/Smoke/. +set -euo pipefail + +ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +TARGET=${TARGET:-DEBUG} +OUT=$ROOT/Build/Out/AARCH64-$TARGET +SMOKE=$ROOT/Build/Smoke +ESP=$SMOKE/esp +KEXT_TIMEOUT=${KEXT_TIMEOUT:-60} + +"$ROOT/build.sh" -b "$TARGET" >/dev/null +mkdir -p "$SMOKE" + +python3 "$ROOT/Scripts/stage-esp.py" -o "$ESP" "$ROOT/CrabApplePkg/Test/Qemu/esp" \ + --put "$OUT/CrabApple.efi=EFI/BOOT/BOOTAA64.EFI" \ + --put "$OUT/HelloPayload.efi=EFI/CrabApple/Payloads/HelloPayload.efi" \ + --put "$OUT/SampleDriver.efi=EFI/CrabApple/Kits/SampleDriver.kit/SampleDriver.efi" + +echo "== scenario 1: countdown boots the default entry" +python3 "$ROOT/Scripts/qemu-run.py" "$ESP" \ + --log "$SMOKE/default.log" --screenshot "$SMOKE/menu.png" \ + --expect 'CRABAPPLE: UniXNU [0-9.]+ starting as application' \ + --expect 'CRABAPPLE: config: loaded' \ + --expect 'CRABAPPLE: kit SampleDriver 1\.0\.0: loaded' \ + --expect 'CRABAPPLE: kit NeverMatch 0\.0\.1: not loaded' \ + --expect 'CRABAPPLE: driver .*SampleDriver\.efi \(kit SampleDriver\): started' \ + --expect 'CRABAPPLE: ui: graphics' \ + --expect 'CRABAPPLE: menu: shown 4 entries, default 0' \ + --expect 'CRABAPPLE: menu: timeout, booting default 0' \ + --expect 'CRABAPPLE: dt: installed' \ + --expect "CRABAPPLE-HELLO: check dtb PASS \(bootargs='console=ttyAMA0 crabapple.test=1' marker='patched'\)" \ + --expect 'CRABAPPLE-HELLO: check overlay PASS' \ + --expect "CRABAPPLE-HELLO: check driver PASS \(sample driver args='from-kit'\)" \ + --expect 'CRABAPPLE-HELLO: check initrd PASS .*CRABAPPLE-INITRD-OK' \ + --expect "CRABAPPLE-HELLO: check var PASS \(CrabAppleTest='hello-variable'" \ + --expect 'CRABAPPLE-HELLO: RESULT PASS 5 passed 0 failed' \ + --reject 'CRABAPPLE: config: .*(error|needs|unknown)' \ + --reject 'ASSERT' + +echo "== scenario 2: keyboard selects the second entry" +python3 "$ROOT/Scripts/qemu-run.py" "$ESP" \ + --log "$SMOKE/keys.log" --keys down,ret \ + --expect 'CRABAPPLE: menu: enter, booting 1' \ + --expect "CRABAPPLE-HELLO: load options 'plain shutdown'" \ + --expect 'CRABAPPLE-HELLO: RESULT PASS 0 passed 0 failed' \ + --reject 'CRABAPPLE: menu: timeout' \ + --reject 'ASSERT' + +echo "== scenario 3: no GOP, no config file" +python3 "$ROOT/Scripts/stage-esp.py" -o "$SMOKE/esp-bare" "$ROOT/CrabApplePkg/Test/Qemu/empty" \ + --put "$OUT/CrabApple.efi=EFI/BOOT/BOOTAA64.EFI" +python3 "$ROOT/Scripts/qemu-run.py" "$SMOKE/esp-bare" --no-gop --timeout 40 \ + --log "$SMOKE/degraded.log" \ + --expect 'CRABAPPLE: config: none found, using built-in configuration' \ + --expect 'CRABAPPLE: config: no boot entry found' \ + --expect 'CRABAPPLE: ui: text console' \ + --expect "CRABAPPLE: menu: shown 3 entries, default 0 'exit'" \ + --expect 'CRABAPPLE: menu: timeout, booting default 0' \ + --expect 'CRABAPPLE: action: exit' \ + --reject 'ASSERT' + +echo "== scenario 4: config on a partition head, tail as a block slice" +FAT_PYTHON=${FAT_PYTHON:-python3} +if command -v mcopy >/dev/null || "$FAT_PYTHON" -W ignore -c 'import pyfatfs.PyFatFS' 2>/dev/null; then + python3 "$ROOT/Scripts/stage-esp.py" -o "$SMOKE/userdata-tree" "$ROOT/CrabApplePkg/Test/Qemu/userdata" \ + --put "$OUT/HelloPayload.efi=EFI/CrabApple/Payloads/HelloPayload.efi" >/dev/null + "$FAT_PYTHON" "$ROOT/Scripts/mk-gpt-disk.py" --out "$SMOKE/gpt-disk.img" --tree "$SMOKE/userdata-tree" \ + --size 256 --fat 64 --name userdata + python3 "$ROOT/Scripts/qemu-run.py" "$SMOKE/esp-bare" --disk "$SMOKE/gpt-disk.img" \ + --log "$SMOKE/slice.log" \ + --expect 'CRABAPPLE: config: loaded' \ + --expect "CRABAPPLE: slice passthrough: partition 'userdata' blocks 131072\\+[0-9]+ \\(512-byte\\), read-only" \ + --expect "CRABAPPLE-HELLO: check slice PASS \\('passthrough' [0-9]+ blocks ro, block 0 'CRABAPPLE-TAIL-MARKER'" \ + --expect 'CRABAPPLE-HELLO: RESULT PASS 1 passed 0 failed' \ + --reject 'ASSERT' +else + echo " skipped: needs mtools or FAT_PYTHON with pyfatfs" +fi + +echo "== scenario 5: synthetic kext staging and XNU protocol compatibility" +python3 "$ROOT/Scripts/kext-fixtures.py" "$SMOKE/kext" "$OUT" +for variant in v2 v1 v1-empty missing-loader unknown nullboot missing-file bad-image truncated-command empty-plist duplicate bad-path bad-type wrong-entry; do + expected='CRABAPPLE: kext: invalid entry configuration; boot refused' + reject='CRABAPPLE-XNU-TEST: .* handoff' + case "$variant" in + v2) expected='CRABAPPLE-XNU-TEST: v2 handoff PASS call 1'; reject='CRABAPPLE-XNU-TEST: .* FAIL' ;; + v1) expected='CRABAPPLE: xnu: loader revision 1 cannot accept kexts' ;; + v1-empty) expected='CRABAPPLE-XNU-TEST: v1 handoff PASS call 1'; reject='CRABAPPLE-XNU-TEST: .* FAIL' ;; + missing-loader) expected='CRABAPPLE: xnu: no loader protocol installed' ;; + unknown|nullboot) expected='CRABAPPLE: xnu: unsupported loader revision or missing Boot callback' ;; + missing-file) expected='CRABAPPLE: kext .*Code.kext: cannot read executable' ;; + bad-image|truncated-command) expected='CRABAPPLE: kext .*Code.kext: invalid AArch64 MH_KEXT_BUNDLE; boot refused' ;; + empty-plist) expected='CRABAPPLE: kext .*Code.kext: empty plist; boot refused' ;; + duplicate) expected='CRABAPPLE: kext .*code.kext: duplicate bundle; boot refused' ;; + esac + python3 "$ROOT/Scripts/qemu-run.py" "$SMOKE/kext/$variant" --timeout "$KEXT_TIMEOUT" \ + --cpus 1 --finish-on-expect \ + --log "$SMOKE/kext-$variant.log" --expect "$expected" --reject "$reject" --reject 'ASSERT' +done + +echo "== smoke test passed; logs and menu.png in $SMOKE" diff --git a/package/Scripts/stage-esp.py b/package/Scripts/stage-esp.py new file mode 100755 index 0000000..65a5a45 --- /dev/null +++ b/package/Scripts/stage-esp.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +"""Assemble an ESP directory for UniXNU. + +Copies one or more source trees into OUT (later trees win), compiles every +*.dts to *.dtb and *.dtso to *.dtbo with dtc (sources are not copied), and +places extra files with --put SRC=DEST (DEST relative to OUT). + +Example: + stage-esp.py -o out/esp CrabApplePkg/Test/Qemu/esp \\ + --put Build/Out/AARCH64-DEBUG/CrabApple.efi=EFI/BOOT/BOOTAA64.EFI +""" + +import argparse +import shutil +import subprocess +import sys +from pathlib import Path + +DT_SOURCES = {".dts": ".dtb", ".dtso": ".dtbo"} + + +def copy_tree(src: Path, out: Path) -> None: + for path in sorted(src.rglob("*")): + rel = path.relative_to(src) + dest = out / rel + if path.is_dir(): + dest.mkdir(parents=True, exist_ok=True) + continue + dest.parent.mkdir(parents=True, exist_ok=True) + if path.suffix in DT_SOURCES: + target = dest.with_suffix(DT_SOURCES[path.suffix]) + subprocess.run( + ["dtc", "-q", "-I", "dts", "-O", "dtb", "-o", str(target), str(path)], + check=True, + ) + else: + shutil.copy2(path, dest) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("trees", nargs="+", type=Path, help="source trees, copied in order") + parser.add_argument("-o", "--out", type=Path, required=True, help="ESP directory to (re)create") + parser.add_argument("--put", action="append", default=[], metavar="SRC=DEST", help="extra file to place") + args = parser.parse_args() + + if args.out.exists(): + shutil.rmtree(args.out) + args.out.mkdir(parents=True) + + for tree in args.trees: + if not tree.is_dir(): + print(f"error: {tree} is not a directory", file=sys.stderr) + return 1 + copy_tree(tree, args.out) + + for item in args.put: + src, _, dest = item.partition("=") + if not dest: + print(f"error: --put needs SRC=DEST, got {item!r}", file=sys.stderr) + return 1 + target = args.out / dest + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(src, target) + + for path in sorted(args.out.rglob("*")): + if path.is_file(): + print(f" {path.relative_to(args.out)} ({path.stat().st_size} bytes)") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/package/build.sh b/package/build.sh new file mode 100755 index 0000000..ffba998 --- /dev/null +++ b/package/build.sh @@ -0,0 +1,94 @@ +#!/usr/bin/env bash +# Build UniXNU with the EDK II build system. +# +# ./build.sh [-b DEBUG|RELEASE] [-t CLANGPDB|GCC] +# +# UniXNU targets AArch64 UEFI only. +# +# The edk2 base defaults to a Mu_Basecore checkout in this directory (not tracked); +# CRABAPPLE_EDK2 may point at any other edk2 or Project Mu checkout that has MdePkg +# with BaseFdtLib. +# Results are copied to Build/Out/-/. +set -euo pipefail + +ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +EDK2=${CRABAPPLE_EDK2:-$ROOT/Mu_Basecore} +ARCH=AARCH64 +TARGET=DEBUG +TOOLCHAIN=CLANGPDB +JOBS=${JOBS:-$(getconf _NPROCESSORS_ONLN 2>/dev/null || echo 2)} + +usage() { + sed -n '2,10p' "$0" | sed 's/^# \{0,1\}//' + exit "${1:-0}" +} + +while getopts "b:t:h" opt; do + case $opt in + b) TARGET=$OPTARG ;; + t) TOOLCHAIN=$OPTARG ;; + h) usage 0 ;; + *) usage 1 ;; + esac +done + +if [ ! -f "$EDK2/MdePkg/MdePkg.dec" ]; then + echo "error: no edk2 tree at $EDK2 (clone https://github.com/microsoft/mu_basecore, branch release/202608, into $ROOT/Mu_Basecore or set CRABAPPLE_EDK2)" >&2 + exit 1 +fi + +if [ ! -f "$EDK2/MdePkg/Library/BaseFdtLib/libfdt/libfdt/fdt.c" ] || [ ! -d "$EDK2/MdePkg/Library/MipiSysTLib/mipisyst/library/include" ]; then + echo "error: MdePkg submodules are missing; run in $EDK2:" >&2 + echo " git submodule update --init --depth 1 MdePkg/Library/BaseFdtLib/libfdt MdePkg/Library/MipiSysTLib/mipisyst BaseTools/Source/C/BrotliCompress/brotli" >&2 + exit 1 +fi + +# BaseTools host utilities (GenFw, GenFv, ...), built once from source. +if [ ! -x "$EDK2/BaseTools/Source/C/bin/GenFw" ]; then + echo "== building BaseTools C utilities" + make -C "$EDK2/BaseTools/Source/C" -j"$JOBS" >/dev/null +fi + +export WORKSPACE=$ROOT +export PACKAGES_PATH=$ROOT:$EDK2 +export EDK_TOOLS_PATH=$EDK2/BaseTools +export CONF_PATH=$ROOT/Build/Conf +export PYTHON_COMMAND=${PYTHON_COMMAND:-python3} +export PATH=$EDK_TOOLS_PATH/BinWrappers/PosixLike:$EDK_TOOLS_PATH/Source/C/bin:$PATH + +case $TOOLCHAIN in + CLANGPDB) + if [ -z "${CLANG_BIN:-}" ]; then + CLANG=$(command -v clang) || { echo "error: clang not found" >&2; exit 1; } + CLANG_BIN=$(dirname "$CLANG")/ + fi + export CLANG_BIN + ;; + GCC|GCC5) + if [ "$(uname -m)" != aarch64 ]; then + export GCC_AARCH64_PREFIX=${GCC_AARCH64_PREFIX:-aarch64-linux-gnu-} + export GCC5_AARCH64_PREFIX=$GCC_AARCH64_PREFIX + fi + ;; +esac + +mkdir -p "$CONF_PATH" +for f in target tools_def build_rule; do + if [ ! -f "$CONF_PATH/$f.txt" ] || [ "$EDK_TOOLS_PATH/Conf/$f.template" -nt "$CONF_PATH/$f.txt" ]; then + cp "$EDK_TOOLS_PATH/Conf/$f.template" "$CONF_PATH/$f.txt" + fi +done + +echo "== edk2: $EDK2" +echo "== build $ARCH $TARGET $TOOLCHAIN" +build -p CrabApplePkg/CrabApplePkg.dsc -a "$ARCH" -b "$TARGET" -t "$TOOLCHAIN" -n "$JOBS" + +OUT=$ROOT/Build/Out/$ARCH-$TARGET +mkdir -p "$OUT" +BIN=$ROOT/Build/CrabApple/${TARGET}_$TOOLCHAIN/$ARCH +for m in CrabApple CrabAppleDxe HelloPayload SampleDriver XnuTest; do + cp "$BIN/$m.efi" "$OUT/" +done + +echo "== outputs in $OUT" +ls -l "$OUT" diff --git a/package/compile_flags.txt b/package/compile_flags.txt new file mode 100644 index 0000000..e2ab0fa --- /dev/null +++ b/package/compile_flags.txt @@ -0,0 +1,10 @@ +--target=aarch64-unknown-windows +-ffreestanding +-fshort-wchar +-include +CrabApplePkg/Library/CrabAppleLib/ClangdAutoGen.h +-IMu_Basecore/MdePkg/Include +-IMu_Basecore/MdePkg/Include/AArch64 +-IMu_Basecore/MdeModulePkg/Include +-ICrabApplePkg/Include +-ICrabApplePkg/Test/Include diff --git a/patches/libpthread/explicit-kernel-address.patch b/patches/libpthread/explicit-kernel-address.patch new file mode 100644 index 0000000..bde4cb6 --- /dev/null +++ b/patches/libpthread/explicit-kernel-address.patch @@ -0,0 +1,120 @@ +diff --git a/kern/kern_support.c b/kern/kern_support.c +index bdb04ab..8596903 100644 +--- a/kern/kern_support.c ++++ b/kern/kern_support.c +@@ -902,7 +902,7 @@ workq_kevent(proc_t p, struct workq_thread_addrs *th_addrs, + + // squash any errors into just empty output + if (ret != 0 || *kevent_count_out == -1) { +- *kevent_list_out = NULL; ++ *kevent_list_out = 0; + *kevent_count_out = 0; + return ret; + } +@@ -943,7 +943,7 @@ workq_setup_thread(proc_t p, thread_t th, vm_map_t map, user_addr_t stackaddr, + { + struct workq_thread_addrs th_addrs; + bool first_use = (setup_flags & WQ_SETUP_FIRST_USE); +- user_addr_t kevent_list = NULL; ++ user_addr_t kevent_list = 0; + int kevent_count = 0; + + workq_thread_get_addrs(map, stackaddr, &th_addrs); +@@ -976,7 +976,7 @@ workq_setup_thread(proc_t p, thread_t th, vm_map_t map, user_addr_t stackaddr, + kevent_count = WORKQ_EXIT_THREAD_NKEVENT; + } else if (upcall_flags & WQ_FLAG_THREAD_KEVENT) { + unsigned int flags = KEVENT_FLAG_STACK_DATA | KEVENT_FLAG_IMMEDIATE; +- workq_kevent(p, &th_addrs, NULL, 0, flags, &kevent_list, &kevent_count); ++ workq_kevent(p, &th_addrs, 0, 0, flags, &kevent_list, &kevent_count); + } + + workq_set_register_state(p, th, &th_addrs, kport, +@@ -996,7 +996,7 @@ workq_handle_stack_events(proc_t p, thread_t th, vm_map_t map, + user_addr_t events, int nevents, int upcall_flags) + { + struct workq_thread_addrs th_addrs; +- user_addr_t kevent_list = NULL; ++ user_addr_t kevent_list = 0; + int kevent_count = 0, error; + __assert_only kern_return_t kr; + +diff --git a/kern/kern_synch.c b/kern/kern_synch.c +index 97343a3..841473f 100644 +--- a/kern/kern_synch.c ++++ b/kern/kern_synch.c +@@ -559,7 +559,7 @@ redrive: + // we have no way of knowing who it is. When it arrives, the lock + // path will update the turnstile owner and return it to userspace. + old_owner = _kwq_clear_owner(kwq); +- pthread_kern->psynch_wait_update_owner(kwq, THREAD_NULL, ++ pthread_kern->psynch_wait_update_owner((uintptr_t)kwq, THREAD_NULL, + &kwq->kw_turnstile); + PTHREAD_TRACE(psynch_mutex_kwqprepost, kwq->kw_addr, + kwq->kw_prepost.lseq, count, 0); +@@ -623,7 +623,7 @@ redrive: + 0); + } + old_owner = _kwq_clear_owner(kwq); +- pthread_kern->psynch_wait_update_owner(kwq, THREAD_NULL, ++ pthread_kern->psynch_wait_update_owner((uintptr_t)kwq, THREAD_NULL, + &kwq->kw_turnstile); + } + } +@@ -682,7 +682,7 @@ again: + + if (_kwq_handle_interrupted_wakeup(kwq, KWQ_INTR_WRITE, lseq, retval)) { + old_owner = _kwq_set_owner(kwq, current_thread(), 0); +- pthread_kern->psynch_wait_update_owner(kwq, kwq->kw_owner, ++ pthread_kern->psynch_wait_update_owner((uintptr_t)kwq, kwq->kw_owner, + &kwq->kw_turnstile); + ksyn_wqunlock(kwq); + goto out; +@@ -719,7 +719,7 @@ again: + kwq->kw_prepost.lseq, kwq->kw_prepost.count, 1); + + old_owner = _kwq_set_owner(kwq, current_thread(), 0); +- pthread_kern->psynch_wait_update_owner(kwq, kwq->kw_owner, ++ pthread_kern->psynch_wait_update_owner((uintptr_t)kwq, kwq->kw_owner, + &kwq->kw_turnstile); + + ksyn_wqunlock(kwq); +@@ -805,7 +805,7 @@ psynch_mtxcontinue(void *parameter, wait_result_t result) + } + } + +- pthread_kern->psynch_wait_complete(kwq, &kwq->kw_turnstile); ++ pthread_kern->psynch_wait_complete((uintptr_t)kwq, &kwq->kw_turnstile); + + ksyn_wqunlock(kwq); + pthread_kern->psynch_wait_cleanup(); +@@ -2009,7 +2009,7 @@ ksyn_signal_thread(ksyn_wait_queue_t kwq, kwq_queue_type_t kqi, + tstore = &kwq->kw_turnstile; + } + +- ret = pthread_kern->psynch_wait_wakeup(kwq, kwe, tstore); ++ ret = pthread_kern->psynch_wait_wakeup((uintptr_t)kwq, kwe, tstore); + + if (ret != KERN_SUCCESS && ret != KERN_NOT_WAITING) { + panic("ksyn_signal: panic waking up thread %x\n", ret); +@@ -2053,7 +2053,7 @@ ksyn_signal(ksyn_wait_queue_t kwq, kwq_queue_type_t kqi, + tstore = &kwq->kw_turnstile; + } + +- ret = pthread_kern->psynch_wait_wakeup(kwq, kwe, tstore); ++ ret = pthread_kern->psynch_wait_wakeup((uintptr_t)kwq, kwe, tstore); + + if (ret != KERN_SUCCESS && ret != KERN_NOT_WAITING) { + panic("ksyn_signal: panic waking up thread %x\n", ret); +diff --git a/kern/kern_trace.h b/kern/kern_trace.h +index ce87532..bf0f90d 100644 +--- a/kern/kern_trace.h ++++ b/kern/kern_trace.h +@@ -62,7 +62,7 @@ static __unused void* + VM_UNSLIDE(void* ptr) + { + vm_offset_t unslid_ptr; +- vm_kernel_unslide_or_perm_external(ptr, &unslid_ptr); ++ vm_kernel_unslide_or_perm_external((vm_offset_t)ptr, &unslid_ptr); + return (void*)unslid_ptr; + } + diff --git a/patches/mesa/freedreno-tools-nir.patch b/patches/mesa/freedreno-tools-nir.patch new file mode 100644 index 0000000..30c183a --- /dev/null +++ b/patches/mesa/freedreno-tools-nir.patch @@ -0,0 +1,12 @@ +diff --git a/meson.build b/meson.build +index 3aed8ca..6a2a30c 100644 +--- a/meson.build ++++ b/meson.build +@@ -985,6 +985,7 @@ if with_clc + endif + + with_gfx_compute = [ ++ with_tools.contains('freedreno'), + with_any_opengl, + with_any_vk, + with_clc, diff --git a/patches/mesa/ir3-darwin-build-id.patch b/patches/mesa/ir3-darwin-build-id.patch new file mode 100644 index 0000000..a6f31c7 --- /dev/null +++ b/patches/mesa/ir3-darwin-build-id.patch @@ -0,0 +1,14 @@ +diff --git a/src/freedreno/ir3/ir3_disk_cache.c b/src/freedreno/ir3/ir3_disk_cache.c +index 3b732d3..8946939 100644 +--- a/src/freedreno/ir3/ir3_disk_cache.c ++++ b/src/freedreno/ir3/ir3_disk_cache.c +@@ -37,7 +37,8 @@ ir3_disk_cache_init(struct ir3_compiler *compiler) + const struct build_id_note *note = + build_id_find_nhdr_for_addr(ir3_disk_cache_init); + unsigned build_id_len = build_id_length(note); +- assert(note && build_id_len == BUILD_ID_EXPECTED_HASH_LENGTH); /* sha1 */ ++ /* Mach-O identifies a build with its 16-byte LC_UUID; ELF uses SHA-1. */ ++ assert(note && build_id_len == (DETECT_OS_APPLE ? 16 : BUILD_ID_EXPECTED_HASH_LENGTH)); + + const uint8_t *id_sha1 = build_id_data(note); + assert(id_sha1); diff --git a/patches/mesa/ir3-portable-asm-roundtrip.patch b/patches/mesa/ir3-portable-asm-roundtrip.patch new file mode 100644 index 0000000..0ecaba9 --- /dev/null +++ b/patches/mesa/ir3-portable-asm-roundtrip.patch @@ -0,0 +1,36 @@ +diff --git a/src/freedreno/ir3/ir3_shader.c b/src/freedreno/ir3/ir3_shader.c +index 6b8419b..cfe684a 100644 +--- a/src/freedreno/ir3/ir3_shader.c ++++ b/src/freedreno/ir3/ir3_shader.c +@@ -352,17 +352,27 @@ create_roundtrip_variant(struct ir3_shader *shader, struct ir3_shader_variant *v + char *disasm_data = NULL; + size_t disasm_size = 0; + FILE *disasm_stream = open_memstream(&disasm_data, &disasm_size); ++ if (!disasm_stream) ++ goto fail; + ir3_shader_disasm(v, v->bin, disasm_stream); +- fflush(disasm_stream); ++ if (fclose(disasm_stream)) { ++ free(disasm_data); ++ goto fail; ++ } + + struct ir3_kernel_info info; + memset(&info, 0, sizeof(info)); + info.numwg = INVALID_REG; + +- fseek(disasm_stream, 0, SEEK_SET); +- rt_v->ir = ir3_parse(rt_v, &info, disasm_stream); ++ /* open_memstream is a write stream; read the completed bytes separately. */ ++ FILE *input = fmemopen(disasm_data, disasm_size, "r"); ++ if (!input) { ++ free(disasm_data); ++ goto fail; ++ } ++ rt_v->ir = ir3_parse(rt_v, &info, input); + +- fclose(disasm_stream); ++ fclose(input); + free(disasm_data); + + if (!rt_v->ir) { diff --git a/patches/mesa/offline-adreno-compiler-probe.patch b/patches/mesa/offline-adreno-compiler-probe.patch new file mode 100644 index 0000000..e9a65ba --- /dev/null +++ b/patches/mesa/offline-adreno-compiler-probe.patch @@ -0,0 +1,52 @@ +diff --git a/meson.options b/meson.options +index 678bc67..fc7e9cb 100644 +--- a/meson.options ++++ b/meson.options +@@ -899,3 +899,11 @@ option( + value : false, + description : 'Build panfrost Rust components.' + ) ++ ++option('offline-adreno-probe', type : 'string', value : '', ++ description : 'C source for an explicitly requested offline compiler fixture') ++ ++option('offline-air-nir-dir', type : 'string', value : '', ++ description : 'Optional external AIR frontend source directory for offline tests') ++option('offline-air-nir-llvm-config', type : 'string', value : '', ++ description : 'Explicit LLVM config executable for the offline AIR frontend') +diff --git a/src/freedreno/ir3/meson.build b/src/freedreno/ir3/meson.build +index c6258e5..ab6a9c0 100644 +--- a/src/freedreno/ir3/meson.build ++++ b/src/freedreno/ir3/meson.build +@@ -173,3 +173,31 @@ test('ir3_delay_test', + ), + suite: ['freedreno'], + ) ++ ++if get_option('offline-adreno-probe') != '' ++ executable( ++ 'adreno_compiler_probe', ++ get_option('offline-adreno-probe'), ++ link_with : libfreedreno_ir3, ++ dependencies : [idep_libir3decode, idep_mesautil, idep_nir], ++ include_directories : [inc_freedreno, inc_include, inc_src], ++ build_by_default : false, ++ ) ++endif ++ ++if get_option('offline-air-nir-dir') != '' ++ air_llvm_config = find_program(get_option('offline-air-nir-llvm-config'), required : true) ++ air_cpp_args = run_command(air_llvm_config, '--cxxflags', check : true).stdout().strip().split() ++ air_link_args = run_command(air_llvm_config, '--ldflags', '--libs', 'core', 'bitreader', 'bitwriter', 'analysis', '--system-libs', check : true).stdout().strip().split() ++ air_dir = get_option('offline-air-nir-dir') ++ executable( ++ 'air_nir', ++ [air_dir / 'Frontend.cpp', air_dir / 'MesaBackend.c', air_dir / 'Library.c'], ++ cpp_args : air_cpp_args, ++ link_args : air_link_args, ++ link_with : libfreedreno_ir3, ++ dependencies : [idep_libir3decode, idep_mesautil, idep_nir], ++ include_directories : [inc_freedreno, inc_include, inc_src], ++ build_by_default : false, ++ ) ++endif diff --git a/patches/mesa/turnip-darwin-kmd.patch b/patches/mesa/turnip-darwin-kmd.patch new file mode 100644 index 0000000..30649ac --- /dev/null +++ b/patches/mesa/turnip-darwin-kmd.patch @@ -0,0 +1,1283 @@ +diff --git a/meson.options b/meson.options +index fc7e9cb..680d08f 100644 +--- a/meson.options ++++ b/meson.options +@@ -222,11 +222,18 @@ option( + 'freedreno-kmds', + type : 'array', + value : ['msm'], +- choices : ['msm', 'kgsl', 'virtio', 'wsl'], ++ choices : ['msm', 'kgsl', 'virtio', 'wsl', 'darwin'], + description : 'List of kernel-mode drivers to enable for freedreno ' + + 'gallium and vulkan driver', + ) + ++option( ++ 'tu-darwin-abi-dir', ++ type : 'string', ++ value : '', ++ description : 'Directory holding NativeAdrenoABI.h for the Turnip darwin kernel backend (freedreno-kmds=darwin)', ++) ++ + option( + 'amdgpu-virtio', + type : 'boolean', +diff --git a/src/freedreno/vulkan/meson.build b/src/freedreno/vulkan/meson.build +index d6c8a05..762ecc8 100644 +--- a/src/freedreno/vulkan/meson.build ++++ b/src/freedreno/vulkan/meson.build +@@ -100,6 +100,7 @@ libtu_includes = [ + ] + + tu_link_with = [] ++tu_link_args = [] + tu_deps = [] + tu_flags = [sse2_args] + +@@ -115,6 +116,10 @@ if with_platform_wayland + tu_wsi = true + endif + ++if with_platform_macos ++ tu_wsi = true ++endif ++ + if system_has_kms_drm and not with_platform_android + tu_wsi = true + endif +@@ -132,6 +137,15 @@ if freedreno_kmds.contains('kgsl') + libtu_files += files('tu_knl_kgsl.cc') + endif + ++if freedreno_kmds.contains('darwin') ++ tu_flags += '-DTU_HAS_DARWIN' ++ libtu_files += files('tu_knl_darwin.cc') ++ libtu_includes += include_directories(get_option('tu-darwin-abi-dir'), is_system : false) ++ tu_deps += dependency('appleframeworks', modules : ['IOKit', 'CoreGraphics', 'Foundation', 'QuartzCore']) ++ # The wsi layer's device entrypoint table references weak symbols that nothing defines; ld64 needs them allowed. ++ tu_link_args += ['-Wl,-undefined,dynamic_lookup'] ++endif ++ + if freedreno_kmds.contains('msm') + tu_flags += '-DTU_HAS_MSM' + libtu_files += files('tu_knl_drm_msm.cc', 'tu_knl_drm.cc') +@@ -225,7 +239,7 @@ libvulkan_freedreno = shared_library( + c_args : [no_override_init_args, tu_flags, soong_compat_c_args], + cpp_args : [tu_cpp_args, tu_flags, soong_compat_cpp_args], + gnu_symbol_visibility : 'hidden', +- link_args : [vulkan_icd_link_args, ld_args_bsymbolic, ld_args_gc_sections, ld_args_build_id], ++ link_args : [vulkan_icd_link_args, ld_args_bsymbolic, ld_args_gc_sections, ld_args_build_id, tu_link_args], + link_depends : vulkan_icd_link_depends, + install : true, + ) +diff --git a/src/freedreno/vulkan/tu_device.cc b/src/freedreno/vulkan/tu_device.cc +index 82efb07..3b512f8 100644 +--- a/src/freedreno/vulkan/tu_device.cc ++++ b/src/freedreno/vulkan/tu_device.cc +@@ -327,6 +327,7 @@ get_device_extensions(const struct tu_physical_device *device, + .EXT_extended_dynamic_state2 = true, + .EXT_extended_dynamic_state3 = true, + .EXT_external_memory_dma_buf = true, ++ .EXT_external_memory_host = device->has_host_import, + .EXT_filter_cubic = device->info->props.has_tex_filter_cubic, + .EXT_fragment_density_map = true, + .EXT_fragment_density_map_offset = true, +@@ -1163,6 +1164,8 @@ tu_get_properties(struct tu_physical_device *pdevice, + : pdevice->info->props.max_storage_buffer_range_bytes; + props->maxPushConstantsSize = MAX_PUSH_CONSTANTS_SIZE; + props->maxMemoryAllocationCount = UINT32_MAX; ++ /* VK_EXT_external_memory_host: the kernel pins whole 4 KiB pages. */ ++ props->minImportedHostPointerAlignment = 4096; + props->maxSamplerAllocationCount = 64 * 1024; + props->bufferImageGranularity = 64; /* A cache line */ + props->sparseAddressSpaceSize = pdevice->va_size; +@@ -3161,6 +3164,9 @@ tu_CreateDevice(VkPhysicalDevice physicalDevice, + "pthread condattr init"); + goto fail_timeline_cond; + } ++#ifndef __APPLE__ ++ /* Darwin has no pthread_condattr_setclock: its condition variables always use the realtime clock, and the darwin ++ * backend converts monotonic deadlines itself. */ + if (pthread_condattr_setclock(&condattr, CLOCK_MONOTONIC) != 0) { + pthread_condattr_destroy(&condattr); + result = vk_startup_errorf(physical_device->instance, +@@ -3168,6 +3174,7 @@ tu_CreateDevice(VkPhysicalDevice physicalDevice, + "pthread condattr clock setup"); + goto fail_timeline_cond; + } ++#endif + if (pthread_cond_init(&device->timeline_cond, &condattr) != 0) { + pthread_condattr_destroy(&condattr); + result = vk_startup_errorf(physical_device->instance, +@@ -3485,6 +3492,17 @@ vk_icdGetInstanceProcAddr(VkInstance instance, const char *pName) + return tu_GetInstanceProcAddr(instance, pName); + } + ++#ifdef __APPLE__ ++/* iPadOS has no Khronos loader: a consumer that opens a loader by path (the metal2vulkan runtime behind the native Metal device) opens this ++ * library and looks up the loader's name. Turnip is the only driver there, so the loader's entry is the driver's own. */ ++PUBLIC ++VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL ++vkGetInstanceProcAddr(VkInstance instance, const char *pName) ++{ ++ return tu_GetInstanceProcAddr(instance, pName); ++} ++#endif ++ + static VkResult + tu_add_to_heap(struct tu_device *dev, struct tu_bo *bo) + { +@@ -3684,7 +3702,18 @@ tu_AllocateMemory(VkDevice _device, + const VkImportMemoryFdInfoKHR *fd_info = + vk_find_struct_const(pAllocateInfo->pNext, IMPORT_MEMORY_FD_INFO_KHR); + +- if (fd_info && fd_info->handleType) { ++ const VkImportMemoryHostPointerInfoEXT *host_info = ++ vk_find_struct_const(pAllocateInfo->pNext, IMPORT_MEMORY_HOST_POINTER_INFO_EXT); ++ ++ if (host_info && host_info->handleType) { ++ if (host_info->handleType != VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT || ++ ((uintptr_t) host_info->pHostPointer | pAllocateInfo->allocationSize) & 4095) { ++ result = VK_ERROR_INVALID_EXTERNAL_HANDLE; ++ } else { ++ result = tu_bo_init_host(device, &mem->bo, host_info->pHostPointer, ++ pAllocateInfo->allocationSize); ++ } ++ } else if (fd_info && fd_info->handleType) { + assert(fd_info->handleType == + VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT || + fd_info->handleType == +@@ -3808,6 +3837,22 @@ tu_AllocateMemory(VkDevice _device, + return VK_SUCCESS; + } + ++VKAPI_ATTR VkResult VKAPI_CALL ++tu_GetMemoryHostPointerPropertiesEXT(VkDevice _device, ++ VkExternalMemoryHandleTypeFlagBits handleType, ++ const void *pHostPointer, ++ VkMemoryHostPointerPropertiesEXT *pMemoryHostPointerProperties) ++{ ++ VK_FROM_HANDLE(tu_device, device, _device); ++ if (handleType != VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT || ++ !device->physical_device->has_host_import || ((uintptr_t) pHostPointer & 4095)) ++ return vk_error(device, VK_ERROR_INVALID_EXTERNAL_HANDLE); ++ /* Imported pages keep the owner's CPU mapping; they are offered as the plain (host-coherent) type. On Darwin the importer cleans ++ * and invalidates its own CPU caches around GPU use (NativeAdrenoAbiBoImport), as an IOSurface's owner does. */ ++ pMemoryHostPointerProperties->memoryTypeBits = 1u; ++ return VK_SUCCESS; ++} ++ + VkResult + tu_allocate_lazy_memory(struct tu_device *dev, + struct tu_device_memory *mem) +diff --git a/src/freedreno/vulkan/tu_knl.cc b/src/freedreno/vulkan/tu_knl.cc +index ab27798..9c545c6 100644 +--- a/src/freedreno/vulkan/tu_knl.cc ++++ b/src/freedreno/vulkan/tu_knl.cc +@@ -85,6 +85,19 @@ tu_bo_init_new_explicit_iova(struct tu_device *dev, + return VK_SUCCESS; + } + ++VkResult ++tu_bo_init_host(struct tu_device *dev, struct tu_bo **bo, void *pointer, uint64_t size) ++{ ++ if (!dev->instance->knl->bo_init_host) ++ return vk_error(dev, VK_ERROR_INVALID_EXTERNAL_HANDLE); ++ VkResult result = dev->instance->knl->bo_init_host(dev, bo, pointer, size); ++ if (result != VK_SUCCESS) ++ return result; ++ if (!(*bo)->unique_id) ++ (*bo)->unique_id = (*bo)->gem_handle; ++ return VK_SUCCESS; ++} ++ + VkResult + tu_bo_init_dmabuf(struct tu_device *dev, + struct tu_bo **bo, +@@ -362,6 +375,14 @@ tu_enumerate_devices(struct vk_instance *vk_instance) + mesa_logi("Found compatible device '%s'.", path); + + return result; ++#elif defined(TU_HAS_DARWIN) ++ struct tu_instance *instance = ++ container_of(vk_instance, struct tu_instance, vk); ++ ++ VkResult result = tu_knl_darwin_load(instance); ++ if (result == VK_SUCCESS && TU_DEBUG(STARTUP)) ++ mesa_logi("Found the NativeAdreno kernel client."); ++ return result; + #else + return VK_ERROR_INCOMPATIBLE_DRIVER; + #endif +diff --git a/src/freedreno/vulkan/tu_knl.h b/src/freedreno/vulkan/tu_knl.h +index 4c2bf6a..a8ccd76 100644 +--- a/src/freedreno/vulkan/tu_knl.h ++++ b/src/freedreno/vulkan/tu_knl.h +@@ -58,6 +58,13 @@ struct tu_bo { + int shared_fd; + #endif + ++#ifdef TU_HAS_DARWIN ++ /* The NativeAdreno kernel client's buffer handle (64 bit, gem_handle is its low half) and the token that selects its ++ * CPU mapping (IOConnectMapMemory64 memory type). */ ++ uint64_t darwin_handle; ++ uint32_t darwin_token; ++#endif ++ + bool implicit_sync : 1; + bool never_unmap : 1; + bool cached_non_coherent : 1; +@@ -140,6 +147,9 @@ struct tu_knl { + const char *name); + VkResult (*bo_init_dmabuf)(struct tu_device *dev, struct tu_bo **out_bo, + uint64_t size, enum tu_bo_alloc_flags flags, int prime_fd); ++ /* Optional (VK_EXT_external_memory_host): a BO over the caller's own pages at pointer. */ ++ VkResult (*bo_init_host)(struct tu_device *dev, struct tu_bo **out_bo, ++ void *pointer, uint64_t size); + int (*bo_export_dmabuf)(struct tu_device *dev, struct tu_bo *bo); + VkResult (*bo_alloc_lazy)(struct tu_device *dev, struct tu_bo *bo); + VkResult (*bo_map)(struct tu_device *dev, struct tu_bo *bo, void *placed_addr); +@@ -221,6 +231,9 @@ tu_bo_init_dmabuf(struct tu_device *dev, + int + tu_bo_export_dmabuf(struct tu_device *dev, struct tu_bo *bo); + ++VkResult ++tu_bo_init_host(struct tu_device *dev, struct tu_bo **bo, void *pointer, uint64_t size); ++ + void + tu_bo_finish(struct tu_device *dev, struct tu_bo *bo); + +@@ -262,6 +275,7 @@ void tu_sparse_vma_finish(struct tu_device *device, + struct tu_sparse_vma *vma); + + VkResult tu_knl_kgsl_load(struct tu_instance *instance, int fd); ++VkResult tu_knl_darwin_load(struct tu_instance *instance); + + struct _drmVersion; + VkResult tu_knl_drm_msm_load(struct tu_instance *instance, +diff --git a/src/freedreno/vulkan/tu_knl_darwin.cc b/src/freedreno/vulkan/tu_knl_darwin.cc +new file mode 100644 +index 0000000..8601aaa +--- /dev/null ++++ b/src/freedreno/vulkan/tu_knl_darwin.cc +@@ -0,0 +1,967 @@ ++/* ++ * Native Darwin kernel backend for Turnip: talks to the NativeAdreno IOKit user client (NativeAdrenoABI.h, version 1). ++ * Modelled on tu_knl_kgsl.cc (no libdrm) and tu_knl_drm_msm.cc (the client chooses GPU addresses). ++ * SPDX-License-Identifier: MIT ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++ ++#include ++#include ++ ++#include "NativeAdrenoABI.h" ++ ++#include "util/os_time.h" ++#include "util/timespec.h" ++#include "util/u_debug.h" ++#include "util/u_vector.h" ++#include "vk_util.h" ++ ++#include "tu_cmd_buffer.h" ++#include "tu_cs.h" ++#include "tu_device.h" ++#include "tu_dynamic_rendering.h" ++#include "tu_knl.h" ++#include "tu_queue.h" ++#include "tu_rmv.h" ++ ++/* 'NTNL': first word of a tunnelled call, as served by the driver (NativeAdrenoPublish.cpp). */ ++#define DW_TUNNEL_MAGIC 0x4e544e4cu ++ ++/* One GPU per process: the kernel service has a single connection slot, so the connection and the fence bookkeeping are ++ * process state rather than per tu_device state. */ ++static struct { ++ io_connect_t conn; ++ bool open; ++ pthread_mutex_t lock; ++ /* Fences of the submissions still possibly in flight, oldest first (the kernel keeps at most eight jobs). */ ++ uint32_t inflight[8]; ++ unsigned inflight_count; ++ /* Transport. On the tablet a sandboxed process (the Metal plugin inside SpringBoard) reaches the driver through the AGX accelerator ++ * service with open type 0x4147, and the sandbox profile decides which external-method selector numbers are admitted, so every ABI ++ * method travels as {magic, method, payload} through one admitted selector (the same tunnel the plugin's own test uses). Direct ++ * selectors (selector = ABI method) stay available for an unrestricted process that opens NativeAdrenoGpuPublished. */ ++ bool tunnel; ++ uint32_t selector; ++ uint32_t last_fence; ++ NativeAdrenoAbiDeviceInfo info; ++ NativeAdrenoAbiParams params; ++} dw = { .lock = PTHREAD_MUTEX_INITIALIZER }; ++ ++static kern_return_t ++dw_call(uint32_t method, const void *in, size_t in_size, void *out, ++ size_t out_size) ++{ ++ size_t out_bytes = out_size; ++ if (!dw.tunnel) ++ return IOConnectCallStructMethod(dw.conn, method, in, in_size, out_size ? out : NULL, ++ out_size ? &out_bytes : NULL); ++ uint8_t buffer[8 + sizeof(NativeAdrenoAbiSubmit)]; ++ if (in_size > sizeof(NativeAdrenoAbiSubmit)) ++ return kIOReturnBadArgument; ++ const uint32_t header[2] = {DW_TUNNEL_MAGIC, method}; ++ memcpy(buffer, header, sizeof(header)); ++ if (in_size) ++ memcpy(buffer + sizeof(header), in, in_size); ++ return IOConnectCallStructMethod(dw.conn, dw.selector, buffer, sizeof(header) + in_size, out_size ? out : NULL, ++ out_size ? &out_bytes : NULL); ++} ++ ++/* Finds an admitted selector: the driver answers a tunnel probe (method 0xffffffff) with success for every selector the sandbox lets through. */ ++static bool ++dw_probe_selector(void) ++{ ++ const uint32_t header[2] = {DW_TUNNEL_MAGIC, 0xffffffffu}; ++ for (uint32_t s = 0; s < 256; s++) { ++ if (IOConnectCallStructMethod(dw.conn, s, header, sizeof(header), NULL, NULL) == KERN_SUCCESS) { ++ dw.selector = s; ++ return true; ++ } ++ } ++ return false; ++} ++ ++static VkResult ++dw_open_connection(struct tu_instance *instance) ++{ ++ /* TU_DARWIN_SERVICE names the IOKit service (default: the AGX accelerator the original Metal discovery finds), TU_DARWIN_OPEN_TYPE its open ++ * type (default 0x4147, the type the driver serves to the plugin). NativeAdrenoGpuPublished with type 0 is the unsandboxed direct form. */ ++ const char *service_name = getenv("TU_DARWIN_SERVICE"); ++ if (!service_name || !*service_name) ++ service_name = "IOAcceleratorES"; ++ const char *type_text = getenv("TU_DARWIN_OPEN_TYPE"); ++ uint32_t open_type = type_text && *type_text ? (uint32_t)strtoul(type_text, NULL, 0) : 0x4147u; ++ const bool direct = !strcmp(service_name, "NativeAdrenoGpuPublished"); ++ if (direct && !(type_text && *type_text)) ++ open_type = 0; ++ io_service_t service = IOServiceGetMatchingService( ++ kIOMainPortDefault, IOServiceMatching(service_name)); ++ if (!service) ++ return VK_ERROR_INCOMPATIBLE_DRIVER; ++ ++ kern_return_t kr = IOServiceOpen(service, mach_task_self(), open_type, &dw.conn); ++ IOObjectRelease(service); ++ if (kr != KERN_SUCCESS) { ++ return vk_startup_errorf(instance, VK_ERROR_INITIALIZATION_FAILED, ++ "IOServiceOpen(%s, 0x%x) failed: 0x%x", service_name, open_type, kr); ++ } ++ dw.tunnel = !direct; ++ if (dw.tunnel && !dw_probe_selector()) { ++ IOServiceClose(dw.conn); ++ dw.conn = 0; ++ return vk_startup_errorf(instance, VK_ERROR_INITIALIZATION_FAILED, "no external-method selector is admitted for this process"); ++ } ++ ++ kr = dw_call(NativeAdrenoMethodQuery, NULL, 0, &dw.params, sizeof(dw.params)); ++ if (kr == KERN_SUCCESS && dw.params.abi_version != NativeAdrenoAbiVersion) ++ kr = kIOReturnUnsupported; ++ if (kr == KERN_SUCCESS) ++ kr = dw_call(NativeAdrenoMethodGetDeviceInfo, NULL, 0, &dw.info, sizeof(dw.info)); ++ if (kr != KERN_SUCCESS) { ++ IOServiceClose(dw.conn); ++ dw.conn = 0; ++ return vk_startup_errorf(instance, VK_ERROR_INITIALIZATION_FAILED, ++ "NativeAdreno query failed: 0x%x", kr); ++ } ++ dw.open = true; ++ dw.inflight_count = 0; ++ dw.last_fence = 0; ++ return VK_SUCCESS; ++} ++ ++static void ++dw_close_connection(void) ++{ ++ if (dw.open) { ++ IOServiceClose(dw.conn); ++ dw.conn = 0; ++ dw.open = false; ++ } ++} ++ ++static VkResult ++dw_status_to_vk(struct tu_device *dev, kern_return_t kr, const char *what) ++{ ++ if (kr == KERN_SUCCESS) ++ return VK_SUCCESS; ++ if (kr == kIOReturnTimeout) ++ return VK_TIMEOUT; ++ return vk_device_set_lost(&dev->vk, "%s failed: 0x%x", what, kr); ++} ++ ++/* Waits for a fence in short kernel waits so the connection lock is never held for long. */ ++static VkResult ++dw_wait_fence_locked_free(struct tu_device *dev, uint32_t fence, uint64_t abs_timeout_ns) ++{ ++ for (;;) { ++ uint64_t now = os_time_get_nano(); ++ uint64_t slice = 1000000; /* 1 ms */ ++ if (abs_timeout_ns != UINT64_MAX && abs_timeout_ns <= now) ++ slice = 0; ++ NativeAdrenoAbiWaitFence req = {.fence = fence, .timeout_ns = slice}; ++ kern_return_t kr = dw_call(NativeAdrenoMethodWaitFence, &req, sizeof(req), NULL, 0); ++ if (kr == KERN_SUCCESS) ++ return VK_SUCCESS; ++ if (kr != kIOReturnTimeout) ++ return dw_status_to_vk(dev, kr, "wait fence"); ++ if (slice == 0) ++ return VK_TIMEOUT; ++ if (abs_timeout_ns != UINT64_MAX && os_time_get_nano() >= abs_timeout_ns) ++ return VK_TIMEOUT; ++ } ++} ++ ++static VkResult ++darwin_queue_wait_fence(struct tu_queue *queue, uint32_t fence, uint64_t timeout_ns) ++{ ++ uint64_t abs = timeout_ns == UINT64_MAX ? UINT64_MAX : os_time_get_nano() + timeout_ns; ++ return dw_wait_fence_locked_free(queue->device, fence, abs); ++} ++ ++static int ++darwin_submitqueue_new(struct tu_device *dev, struct tu_queue *queue) ++{ ++ /* The kernel client has queue 0 from open(); every submission rides the one shared ring. */ ++ queue->msm_queue_id = 0; ++ return 0; ++} ++ ++static void ++darwin_submitqueue_close(struct tu_device *dev, struct tu_queue *queue) ++{ ++} ++ ++/* ---- GPU address space: the client picks addresses out of [va_start, va_start + va_size). ---- */ ++ ++static VkResult ++darwin_allocate_iova(struct tu_device *dev, uint64_t size, uint64_t client_iova, ++ enum tu_bo_alloc_flags flags, uint64_t *iova) ++{ ++ *iova = 0; ++ mtx_lock(&dev->vma_mutex); ++ if (flags & TU_BO_ALLOC_REPLAYABLE) { ++ if (client_iova) { ++ if (util_vma_heap_alloc_addr(&dev->vma, client_iova, size)) ++ *iova = client_iova; ++ else { ++ mtx_unlock(&dev->vma_mutex); ++ return VK_ERROR_INVALID_OPAQUE_CAPTURE_ADDRESS; ++ } ++ } else { ++ dev->vma.alloc_high = true; ++ *iova = util_vma_heap_alloc(&dev->vma, size, os_page_size); ++ } ++ } else { ++ dev->vma.alloc_high = false; ++ *iova = util_vma_heap_alloc(&dev->vma, size, os_page_size); ++ } ++ mtx_unlock(&dev->vma_mutex); ++ return *iova ? VK_SUCCESS : VK_ERROR_OUT_OF_DEVICE_MEMORY; ++} ++ ++static void ++darwin_free_iova(struct tu_device *dev, uint64_t iova, uint64_t size) ++{ ++ mtx_lock(&dev->vma_mutex); ++ util_vma_heap_free(&dev->vma, iova, size); ++ mtx_unlock(&dev->vma_mutex); ++} ++ ++/* ---- buffer objects ---- */ ++ ++static VkResult ++darwin_bo_init(struct tu_device *dev, struct vk_object_base *base, ++ struct tu_bo **out_bo, uint64_t size, uint64_t client_iova, ++ VkMemoryPropertyFlags mem_property, ++ enum tu_bo_alloc_flags flags, struct tu_sparse_vma *lazy_vma, ++ const char *name) ++{ ++ MESA_TRACE_FUNC(); ++ if (lazy_vma) ++ return vk_error(dev, VK_ERROR_FEATURE_NOT_PRESENT); ++ ++ uint64_t iova; ++ VkResult result = darwin_allocate_iova(dev, size, client_iova, flags, &iova); ++ if (result != VK_SUCCESS) ++ return result; ++ ++ NativeAdrenoAbiBoAlloc alloc = {.bytes = size, .align_log2 = 12, .contiguous = 0}; ++ NativeAdrenoAbiBoAllocReply reply = {}; ++ kern_return_t kr = dw_call(NativeAdrenoMethodBoAlloc, &alloc, sizeof(alloc), &reply, sizeof(reply)); ++ if (kr != KERN_SUCCESS) { ++ darwin_free_iova(dev, iova, size); ++ return vk_errorf(dev, VK_ERROR_OUT_OF_DEVICE_MEMORY, "BoAlloc failed: 0x%x", kr); ++ } ++ ++ NativeAdrenoAbiBoMap map = { ++ .handle = reply.handle, ++ .iova = iova, ++ .permissions = (flags & TU_BO_ALLOC_GPU_READ_ONLY) ? 1u : 3u, ++ .writeback = 0, ++ .shareability = 2, ++ }; ++ NativeAdrenoAbiBoMapReply mapped = {}; ++ kr = dw_call(NativeAdrenoMethodBoMap, &map, sizeof(map), &mapped, sizeof(mapped)); ++ if (kr != KERN_SUCCESS) { ++ NativeAdrenoAbiBoFree release = {.handle = reply.handle}; ++ dw_call(NativeAdrenoMethodBoFree, &release, sizeof(release), NULL, 0); ++ darwin_free_iova(dev, iova, size); ++ return vk_errorf(dev, VK_ERROR_OUT_OF_DEVICE_MEMORY, "BoMap failed: 0x%x", kr); ++ } ++ ++ uint32_t gem_handle = (uint32_t)reply.handle; ++ struct tu_bo *bo = tu_device_lookup_bo(dev, gem_handle); ++ assert(bo && bo->gem_handle == 0); ++ ++ name = tu_debug_bos_add(dev, size, name); ++ ++ *bo = (struct tu_bo) { ++ .gem_handle = gem_handle, ++ .size = size, ++ .iova = iova, ++ .name = name, ++ .refcnt = 1, ++ .implicit_sync = !!(flags & TU_BO_ALLOC_IMPLICIT_SYNC), ++ .base = base, ++ .unique_id = reply.handle, ++ }; ++ bo->darwin_handle = reply.handle; ++ ++ tu_dump_bo_init(dev, bo); ++ TU_RMV(bo_allocate, dev, bo); ++ ++ *out_bo = bo; ++ if (flags & TU_BO_ALLOC_INTERNAL_RESOURCE) { ++ TU_RMV(internal_resource_create, dev, bo); ++ TU_RMV(resource_name, dev, bo, name); ++ } ++ return VK_SUCCESS; ++} ++ ++/* VK_EXT_external_memory_host: the kernel pins the caller's pages (an IOSurface's) as a BO, which is then mapped into the GPU like any ++ * other. The CPU keeps the caller's own mapping, so there is no CPU-map token and the BO is "mapped" at the caller's pointer. */ ++static VkResult ++darwin_bo_init_host(struct tu_device *dev, struct tu_bo **out_bo, void *pointer, uint64_t size) ++{ ++ if (!(dw.params.flags & NativeAdrenoParamsHostImport)) ++ return vk_error(dev, VK_ERROR_INVALID_EXTERNAL_HANDLE); ++ uint64_t iova; ++ VkResult result = darwin_allocate_iova(dev, size, 0, TU_BO_ALLOC_NO_FLAGS, &iova); ++ if (result != VK_SUCCESS) ++ return result; ++ ++ NativeAdrenoAbiBoImport import = {.address = (uint64_t)(uintptr_t)pointer, .bytes = size}; ++ NativeAdrenoAbiBoAllocReply reply = {}; ++ kern_return_t kr = dw_call(NativeAdrenoMethodBoImport, &import, sizeof(import), &reply, sizeof(reply)); ++ if (kr != KERN_SUCCESS) { ++ darwin_free_iova(dev, iova, size); ++ return vk_errorf(dev, VK_ERROR_INVALID_EXTERNAL_HANDLE, "BoImport failed: 0x%x", kr); ++ } ++ NativeAdrenoAbiBoMap map = {.handle = reply.handle, .iova = iova, .permissions = 3u, .writeback = 0, .shareability = 2}; ++ NativeAdrenoAbiBoMapReply mapped = {}; ++ kr = dw_call(NativeAdrenoMethodBoMap, &map, sizeof(map), &mapped, sizeof(mapped)); ++ if (kr != KERN_SUCCESS) { ++ NativeAdrenoAbiBoFree release = {.handle = reply.handle}; ++ dw_call(NativeAdrenoMethodBoFree, &release, sizeof(release), NULL, 0); ++ darwin_free_iova(dev, iova, size); ++ return vk_errorf(dev, VK_ERROR_OUT_OF_DEVICE_MEMORY, "BoMap of an import failed: 0x%x", kr); ++ } ++ ++ uint32_t gem_handle = (uint32_t)reply.handle; ++ struct tu_bo *bo = tu_device_lookup_bo(dev, gem_handle); ++ assert(bo && bo->gem_handle == 0); ++ *bo = (struct tu_bo) { ++ .gem_handle = gem_handle, ++ .size = size, ++ .iova = iova, ++ .name = tu_debug_bos_add(dev, size, "host import"), ++ .refcnt = 1, ++ .unique_id = reply.handle, ++ }; ++ bo->darwin_handle = reply.handle; ++ bo->darwin_token = 0; ++ bo->map = pointer; ++ bo->never_unmap = true; ++ tu_dump_bo_init(dev, bo); ++ TU_RMV(bo_allocate, dev, bo); ++ *out_bo = bo; ++ return VK_SUCCESS; ++} ++ ++static VkResult ++darwin_bo_init_dmabuf(struct tu_device *dev, struct tu_bo **out_bo, ++ uint64_t size, enum tu_bo_alloc_flags flags, int prime_fd) ++{ ++ /* Darwin has no dmabuf; the later equivalent is an IOSurface import. */ ++ return vk_error(dev, VK_ERROR_FEATURE_NOT_PRESENT); ++} ++ ++static int ++darwin_bo_export_dmabuf(struct tu_device *dev, struct tu_bo *bo) ++{ ++ return -1; ++} ++ ++static VkResult ++darwin_bo_map(struct tu_device *dev, struct tu_bo *bo, void *placed_addr) ++{ ++ NativeAdrenoAbiBoCpuMap req = {.handle = bo->darwin_handle}; ++ NativeAdrenoAbiBoCpuMapReply reply = {}; ++ kern_return_t kr = dw_call(NativeAdrenoMethodBoCpuMap, &req, sizeof(req), &reply, sizeof(reply)); ++ if (kr != KERN_SUCCESS) ++ return vk_errorf(dev, VK_ERROR_MEMORY_MAP_FAILED, "BoCpuMap failed: 0x%x", kr); ++ ++ mach_vm_address_t address = (mach_vm_address_t)(uintptr_t)placed_addr; ++ mach_vm_size_t length = 0; ++ /* IOUserClient::mapClientMemory64 takes the cache mode from THESE flags, not from the descriptor the driver returns, so a plain ++ * kIOMapAnywhere would give a cached mapping of memory the GPU reads non-cacheable. Write combine is Normal non-cacheable: the ++ * GPU sees CPU writes after a barrier, and memcpy/unaligned stores stay legal (device-memory mappings fault on those). */ ++ uint32_t options = (placed_addr ? 0 : kIOMapAnywhere) | kIOMapWriteCombineCache; ++ kr = IOConnectMapMemory64(dw.conn, reply.token, mach_task_self(), &address, &length, options); ++ if (kr != KERN_SUCCESS || !address || length < bo->size) { ++ NativeAdrenoAbiBoCpuUnmap undo = {.handle = bo->darwin_handle, .token = reply.token}; ++ dw_call(NativeAdrenoMethodBoCpuUnmap, &undo, sizeof(undo), NULL, 0); ++ return vk_errorf(dev, VK_ERROR_MEMORY_MAP_FAILED, "IOConnectMapMemory64 failed: 0x%x", kr); ++ } ++ ++ bo->map = (void *)(uintptr_t)address; ++ bo->darwin_token = reply.token; ++ /* The mapping belongs to the kernel descriptor; tu_bo_unmap() must not munmap() it. It goes away in bo_finish. */ ++ bo->never_unmap = true; ++ TU_RMV(bo_map, dev, bo); ++ return VK_SUCCESS; ++} ++ ++static void ++darwin_bo_allow_dump(struct tu_device *dev, struct tu_bo *bo) ++{ ++} ++ ++static void ++darwin_bo_set_metadata(struct tu_device *dev, struct tu_bo *bo, void *metadata, ++ uint32_t metadata_size) ++{ ++ if (metadata_size > NativeAdrenoAbiMetadataBytes) ++ return; ++ NativeAdrenoAbiMetadata req = {.handle = bo->darwin_handle, .bytes = metadata_size}; ++ memcpy(req.text, metadata, metadata_size); ++ dw_call(NativeAdrenoMethodSetMetadata, &req, sizeof(req), NULL, 0); ++} ++ ++static int ++darwin_bo_get_metadata(struct tu_device *dev, struct tu_bo *bo, void *metadata, ++ uint32_t metadata_size) ++{ ++ NativeAdrenoAbiMetadata req = {.handle = bo->darwin_handle, ++ .bytes = MIN2(metadata_size, NativeAdrenoAbiMetadataBytes)}; ++ NativeAdrenoAbiMetadata reply = {}; ++ if (dw_call(NativeAdrenoMethodGetMetadata, &req, sizeof(req), &reply, sizeof(reply)) != KERN_SUCCESS) ++ return -1; ++ memcpy(metadata, reply.text, MIN2(reply.bytes, metadata_size)); ++ return 0; ++} ++ ++/* Waits until every submission made so far has retired, so a freed buffer cannot still be in flight. The kernel also ++ * retires completed jobs itself; this only covers the window before the GPU passes the last fence. */ ++static void ++darwin_wait_idle(struct tu_device *dev) ++{ ++ pthread_mutex_lock(&dw.lock); ++ uint32_t last = dw.last_fence; ++ pthread_mutex_unlock(&dw.lock); ++ if (last) ++ dw_wait_fence_locked_free(dev, last, os_time_get_nano() + 5000000000ull); ++} ++ ++static void ++darwin_bo_finish(struct tu_device *dev, struct tu_bo *bo) ++{ ++ assert(bo->gem_handle); ++ ++ u_rwlock_rdlock(&dev->dma_bo_lock); ++ ++ if (!p_atomic_dec_zero(&bo->refcnt)) { ++ u_rwlock_rdunlock(&dev->dma_bo_lock); ++ return; ++ } ++ ++ tu_debug_bos_del(dev, bo); ++ tu_dump_bo_del(dev, bo); ++ ++ /* An import has no token: its mapping is the owner's. */ ++ if (bo->map && bo->darwin_token) { ++ TU_RMV(bo_unmap, dev, bo); ++ IOConnectUnmapMemory64(dw.conn, bo->darwin_token, mach_task_self(), ++ (mach_vm_address_t)(uintptr_t)bo->map); ++ NativeAdrenoAbiBoCpuUnmap undo = {.handle = bo->darwin_handle, .token = bo->darwin_token}; ++ dw_call(NativeAdrenoMethodBoCpuUnmap, &undo, sizeof(undo), NULL, 0); ++ } ++ ++ TU_RMV(bo_destroy, dev, bo); ++ ++ darwin_wait_idle(dev); ++ ++ NativeAdrenoAbiBoFree release = {.handle = bo->darwin_handle}; ++ kern_return_t kr = dw_call(NativeAdrenoMethodBoFree, &release, sizeof(release), NULL, 0); ++ if (kr == kIOReturnBusy) { ++ /* A job still references it; give the CP a moment and retry once. */ ++ darwin_wait_idle(dev); ++ kr = dw_call(NativeAdrenoMethodBoFree, &release, sizeof(release), NULL, 0); ++ } ++ if (kr != KERN_SUCCESS) ++ mesa_loge("BoFree failed: 0x%x (the address range is leaked)", kr); ++ else ++ darwin_free_iova(dev, bo->iova, bo->size); ++ ++ /* The BO structs live in a sparse array: zero the entry to mark it free. */ ++ memset(bo, 0, sizeof(*bo)); ++ ++ u_rwlock_rdunlock(&dev->dma_bo_lock); ++} ++ ++/* ---- submissions ---- */ ++ ++struct tu_darwin_queue_submit { ++ struct util_dynarray commands; /* NativeAdrenoAbiSubmitBuffer */ ++}; ++ ++static void * ++darwin_submit_create(struct tu_device *device) ++{ ++ return vk_zalloc(&device->vk.alloc, sizeof(struct tu_darwin_queue_submit), 8, ++ VK_SYSTEM_ALLOCATION_SCOPE_DEVICE); ++} ++ ++static void ++darwin_submit_finish(struct tu_device *device, void *_submit) ++{ ++ struct tu_darwin_queue_submit *submit = (struct tu_darwin_queue_submit *)_submit; ++ util_dynarray_fini(&submit->commands); ++ vk_free(&device->vk.alloc, submit); ++} ++ ++static void ++darwin_submit_add_entries(struct tu_device *device, void *_submit, ++ struct tu_cs_entry *entries, unsigned num_entries) ++{ ++ struct tu_darwin_queue_submit *submit = (struct tu_darwin_queue_submit *)_submit; ++ NativeAdrenoAbiSubmitBuffer *cmds = (NativeAdrenoAbiSubmitBuffer *) ++ util_dynarray_grow(&submit->commands, NativeAdrenoAbiSubmitBuffer, num_entries); ++ for (unsigned i = 0; i < num_entries; i++) { ++ cmds[i] = (NativeAdrenoAbiSubmitBuffer) { ++ .handle = entries[i].bo->darwin_handle, ++ .iova = entries[i].bo->iova + entries[i].offset, ++ .bytes = entries[i].size, ++ }; ++ } ++} ++ ++static void ++darwin_submit_add_bind(struct tu_device *device, void *_submit, ++ struct tu_sparse_vma *vma, uint64_t vma_offset, ++ struct tu_bo *bo, uint64_t bo_offset, uint64_t size) ++{ ++ UNREACHABLE("sparse binding is not supported by the NativeAdreno kernel client"); ++} ++ ++/* ---- synchronization: a sync is either signaled, unsignaled, or pending on one kernel fence ---- */ ++ ++enum darwin_sync_state { ++ DARWIN_SYNC_UNSIGNALED, ++ DARWIN_SYNC_SIGNALED, ++ DARWIN_SYNC_FENCE, ++}; ++ ++struct vk_darwin_sync { ++ struct vk_sync vk; ++ enum darwin_sync_state state; ++ uint32_t fence; ++}; ++ ++static VkResult ++vk_darwin_sync_init(struct vk_device *device, struct vk_sync *sync, uint64_t initial_value) ++{ ++ struct vk_darwin_sync *s = container_of(sync, struct vk_darwin_sync, vk); ++ s->state = initial_value ? DARWIN_SYNC_SIGNALED : DARWIN_SYNC_UNSIGNALED; ++ s->fence = 0; ++ return VK_SUCCESS; ++} ++ ++static void ++vk_darwin_sync_finish(struct vk_device *device, struct vk_sync *sync) ++{ ++} ++ ++static VkResult ++vk_darwin_sync_reset(struct vk_device *device, struct vk_sync *sync) ++{ ++ struct vk_darwin_sync *s = container_of(sync, struct vk_darwin_sync, vk); ++ s->state = DARWIN_SYNC_UNSIGNALED; ++ s->fence = 0; ++ return VK_SUCCESS; ++} ++ ++static VkResult ++vk_darwin_sync_move(struct vk_device *device, struct vk_sync *dst, struct vk_sync *src) ++{ ++ struct vk_darwin_sync *d = container_of(dst, struct vk_darwin_sync, vk); ++ struct vk_darwin_sync *s = container_of(src, struct vk_darwin_sync, vk); ++ d->state = s->state; ++ d->fence = s->fence; ++ s->state = DARWIN_SYNC_UNSIGNALED; ++ s->fence = 0; ++ return VK_SUCCESS; ++} ++ ++static VkResult ++darwin_sync_wait(struct tu_device *device, struct vk_darwin_sync *s, uint64_t abs_timeout_ns) ++{ ++ if (s->state == DARWIN_SYNC_UNSIGNALED) { ++ /* Wait for the sync to resolve to a submission first (no wait-before-signal support in the kernel). */ ++ if (abs_timeout_ns == 0) ++ return VK_TIMEOUT; ++ ++ pthread_mutex_lock(&device->submit_mutex); ++ /* os_time_get_nano() is monotonic but Darwin condition variables wait on the realtime clock. */ ++ struct timespec abstime; ++ { ++ uint64_t now = os_time_get_nano(); ++ uint64_t relative = abs_timeout_ns > now ? abs_timeout_ns - now : 0; ++ clock_gettime(CLOCK_REALTIME, &abstime); ++ uint64_t nsec = (uint64_t)abstime.tv_nsec + relative % 1000000000ull; ++ abstime.tv_sec += relative / 1000000000ull + nsec / 1000000000ull; ++ abstime.tv_nsec = nsec % 1000000000ull; ++ } ++ while (s->state == DARWIN_SYNC_UNSIGNALED) { ++ int ret; ++ if (abs_timeout_ns == UINT64_MAX) ++ ret = pthread_cond_wait(&device->timeline_cond, &device->submit_mutex); ++ else ++ ret = pthread_cond_timedwait(&device->timeline_cond, &device->submit_mutex, &abstime); ++ if (ret != 0) { ++ assert(ret == ETIMEDOUT); ++ pthread_mutex_unlock(&device->submit_mutex); ++ return VK_TIMEOUT; ++ } ++ } ++ pthread_mutex_unlock(&device->submit_mutex); ++ } ++ ++ switch (s->state) { ++ case DARWIN_SYNC_SIGNALED: ++ return VK_SUCCESS; ++ case DARWIN_SYNC_FENCE: ++ return dw_wait_fence_locked_free(device, s->fence, abs_timeout_ns); ++ default: ++ return VK_TIMEOUT; ++ } ++} ++ ++static VkResult ++vk_darwin_sync_wait(struct vk_device *_device, struct vk_sync *sync, uint64_t wait_value, ++ enum vk_sync_wait_flags wait_flags, uint64_t abs_timeout_ns) ++{ ++ struct tu_device *device = container_of(_device, struct tu_device, vk); ++ struct vk_darwin_sync *s = container_of(sync, struct vk_darwin_sync, vk); ++ ++ if (wait_flags & VK_SYNC_WAIT_PENDING) ++ return VK_SUCCESS; ++ ++ return darwin_sync_wait(device, s, abs_timeout_ns); ++} ++ ++static VkResult ++vk_darwin_sync_wait_many(struct vk_device *_device, uint32_t wait_count, ++ const struct vk_sync_wait *waits, ++ enum vk_sync_wait_flags wait_flags, uint64_t abs_timeout_ns) ++{ ++ struct tu_device *device = container_of(_device, struct tu_device, vk); ++ ++ if (wait_flags & VK_SYNC_WAIT_PENDING) ++ return VK_SUCCESS; ++ ++ if (wait_flags & VK_SYNC_WAIT_ANY) { ++ /* Poll every sync in turn until one is done or the deadline passes. */ ++ for (;;) { ++ for (uint32_t i = 0; i < wait_count; i++) { ++ struct vk_darwin_sync *s = container_of(waits[i].sync, struct vk_darwin_sync, vk); ++ if (darwin_sync_wait(device, s, 0) == VK_SUCCESS || ++ (s->state == DARWIN_SYNC_FENCE && ++ dw_wait_fence_locked_free(device, s->fence, 0) == VK_SUCCESS)) ++ return VK_SUCCESS; ++ } ++ if (abs_timeout_ns != UINT64_MAX && os_time_get_nano() >= abs_timeout_ns) ++ return VK_TIMEOUT; ++ usleep(200); ++ } ++ } ++ ++ for (uint32_t i = 0; i < wait_count; i++) { ++ struct vk_darwin_sync *s = container_of(waits[i].sync, struct vk_darwin_sync, vk); ++ VkResult result = darwin_sync_wait(device, s, abs_timeout_ns); ++ if (result != VK_SUCCESS) ++ return result; ++ } ++ return VK_SUCCESS; ++} ++ ++const struct vk_sync_type vk_darwin_sync_type = { ++ .size = sizeof(struct vk_darwin_sync), ++ .features = (enum vk_sync_features) ++ (VK_SYNC_FEATURE_BINARY | ++ VK_SYNC_FEATURE_GPU_WAIT | ++ VK_SYNC_FEATURE_GPU_MULTI_WAIT | ++ VK_SYNC_FEATURE_CPU_WAIT | ++ VK_SYNC_FEATURE_CPU_RESET | ++ VK_SYNC_FEATURE_WAIT_ANY | ++ VK_SYNC_FEATURE_WAIT_PENDING), ++ .init = vk_darwin_sync_init, ++ .finish = vk_darwin_sync_finish, ++ .reset = vk_darwin_sync_reset, ++ .move = vk_darwin_sync_move, ++ .wait = vk_darwin_sync_wait, ++ .wait_many = vk_darwin_sync_wait_many, ++}; ++ ++static void ++darwin_note_fence(uint32_t fence) ++{ ++ if (dw.inflight_count == 8) { ++ memmove(dw.inflight, dw.inflight + 1, 7 * sizeof(dw.inflight[0])); ++ dw.inflight_count = 7; ++ } ++ dw.inflight[dw.inflight_count++] = fence; ++ dw.last_fence = fence; ++} ++ ++/* One kernel submission of up to NativeAdrenoAbiMaxBuffers indirect buffers; waits for the oldest job when all eight kernel job slots ++ * are taken. */ ++static VkResult ++darwin_submit_chunk(struct tu_queue *queue, const NativeAdrenoAbiSubmitBuffer *entries, ++ unsigned count, uint32_t *fence) ++{ ++ struct tu_device *dev = queue->device; ++ NativeAdrenoAbiSubmit req = {}; ++ req.buffer_count = count; ++ req.queue = 0; ++ for (unsigned i = 0; i < count; i++) ++ req.buffers[i] = entries[i]; ++ ++ for (unsigned attempt = 0; attempt < 64; attempt++) { ++ NativeAdrenoAbiSubmitReply reply = {}; ++ kern_return_t kr = dw_call(NativeAdrenoMethodSubmit, &req, sizeof(req), &reply, sizeof(reply)); ++ if (kr == KERN_SUCCESS) { ++ *fence = (uint32_t)reply.fence; ++ return VK_SUCCESS; ++ } ++ if (kr != kIOReturnBusy) ++ return vk_device_set_lost(&dev->vk, "submit failed: 0x%x", kr); ++ ++ pthread_mutex_lock(&dw.lock); ++ uint32_t oldest = dw.inflight_count ? dw.inflight[0] : 0; ++ pthread_mutex_unlock(&dw.lock); ++ if (!oldest) ++ usleep(100); ++ else if (dw_wait_fence_locked_free(dev, oldest, os_time_get_nano() + 5000000000ull) != VK_SUCCESS) ++ return vk_device_set_lost(&dev->vk, "submit: GPU did not retire a job"); ++ } ++ return vk_device_set_lost(&dev->vk, "submit: no free job slot"); ++} ++ ++static VkResult ++darwin_queue_submit(struct tu_queue *queue, void *_submit, ++ struct vk_sync_wait *waits, uint32_t wait_count, ++ struct vk_sync_signal *signals, uint32_t signal_count, ++ struct tu_u_trace_submission_data *u_trace_submission_data) ++{ ++ struct tu_darwin_queue_submit *submit = (struct tu_darwin_queue_submit *)_submit; ++ struct tu_device *dev = queue->device; ++ ++ /* The kernel client has one ring and no GPU-side waits: wait on the CPU before submitting. */ ++ for (uint32_t i = 0; i < wait_count; i++) { ++ struct vk_darwin_sync *s = container_of(waits[i].sync, struct vk_darwin_sync, vk); ++ VkResult result = darwin_sync_wait(dev, s, UINT64_MAX); ++ if (result != VK_SUCCESS) ++ return vk_device_set_lost(&dev->vk, "wait before submit failed"); ++ } ++ ++ uint32_t fence = 0; ++ const unsigned total = util_dynarray_num_elements(&submit->commands, NativeAdrenoAbiSubmitBuffer); ++ if (total) { ++ const NativeAdrenoAbiSubmitBuffer *all = ++ util_dynarray_element(&submit->commands, NativeAdrenoAbiSubmitBuffer, 0); ++ for (unsigned at = 0; at < total; at += NativeAdrenoAbiMaxBuffers) { ++ unsigned count = MIN2(total - at, (unsigned)NativeAdrenoAbiMaxBuffers); ++ uint32_t chunk_fence = 0; ++ VkResult result = darwin_submit_chunk(queue, all + at, count, &chunk_fence); ++ if (result != VK_SUCCESS) ++ return result; ++ pthread_mutex_lock(&dw.lock); ++ darwin_note_fence(chunk_fence); ++ pthread_mutex_unlock(&dw.lock); ++ fence = chunk_fence; ++ } ++ p_atomic_set(&queue->fence, fence); ++ } else { ++ /* Nothing to run: the signals resolve with the last earlier submission. */ ++ fence = p_atomic_read(&queue->fence) > 0 ? (uint32_t)p_atomic_read(&queue->fence) : 0; ++ } ++ ++ for (uint32_t i = 0; i < signal_count; i++) { ++ struct vk_darwin_sync *s = container_of(signals[i].sync, struct vk_darwin_sync, vk); ++ s->fence = fence; ++ s->state = fence ? DARWIN_SYNC_FENCE : DARWIN_SYNC_SIGNALED; ++ } ++ ++ if (u_trace_submission_data) ++ u_trace_submission_data->gpu_ts_offset = 0; ++ return VK_SUCCESS; ++} ++ ++/* ---- device ---- */ ++ ++static VkResult ++darwin_device_init(struct tu_device *dev) ++{ ++ dev->fd = -1; ++ /* The enumeration probe closed its connection; take the single slot for this device. */ ++ if (!dw.open) { ++ VkResult result = dw_open_connection(dev->physical_device->instance); ++ if (result != VK_SUCCESS) ++ return result; ++ } ++ return VK_SUCCESS; ++} ++ ++static void ++darwin_device_finish(struct tu_device *dev) ++{ ++ /* Closing the connection makes the kernel drain the GPU and free every object this client created. */ ++ dw_close_connection(); ++} ++ ++static int ++darwin_device_get_gpu_timestamp(struct tu_device *dev, uint64_t *ts) ++{ ++ NativeAdrenoAbiTimestamp reply = {}; ++ if (dw_call(NativeAdrenoMethodGetTimestamp, NULL, 0, &reply, sizeof(reply)) != KERN_SUCCESS) ++ return -1; ++ *ts = reply.ticks; ++ return 0; ++} ++ ++static int ++darwin_device_get_suspend_count(struct tu_device *dev, uint64_t *suspend_count) ++{ ++ NativeAdrenoAbiStatus reply = {}; ++ if (dw_call(NativeAdrenoMethodGetStatus, NULL, 0, &reply, sizeof(reply)) != KERN_SUCCESS) ++ *suspend_count = 0; ++ else ++ *suspend_count = reply.suspends; ++ return 0; ++} ++ ++static VkResult ++darwin_device_check_status(struct tu_device *device) ++{ ++ NativeAdrenoAbiStatus reply = {}; ++ kern_return_t kr = dw_call(NativeAdrenoMethodGetStatus, NULL, 0, &reply, sizeof(reply)); ++ if (kr == kIOReturnUnsupported) ++ return VK_SUCCESS; /* counters not attached in this kernel */ ++ if (kr != KERN_SUCCESS) ++ return vk_device_set_lost(&device->vk, "error getting GPU status: 0x%x", kr); ++ if (reply.faults != device->fault_count) { ++ device->fault_count = reply.faults; ++ return vk_device_set_lost(&device->vk, "GPU faulted or hung"); ++ } ++ return VK_SUCCESS; ++} ++ ++static VkResult ++darwin_sparse_vma_init(struct tu_device *dev, struct vk_object_base *base, ++ struct tu_sparse_vma *out_vma, uint64_t *out_iova, ++ enum tu_sparse_vma_flags flags, uint64_t size, uint64_t client_iova) ++{ ++ return vk_error(dev, VK_ERROR_FEATURE_NOT_PRESENT); ++} ++ ++static void ++darwin_sparse_vma_finish(struct tu_device *dev, struct tu_sparse_vma *vma) ++{ ++} ++ ++static const struct tu_knl darwin_knl_funcs = { ++ .name = "darwin", ++ ++ .device_init = darwin_device_init, ++ .device_finish = darwin_device_finish, ++ .device_get_gpu_timestamp = darwin_device_get_gpu_timestamp, ++ .device_get_suspend_count = darwin_device_get_suspend_count, ++ .device_check_status = darwin_device_check_status, ++ .submitqueue_new = darwin_submitqueue_new, ++ .submitqueue_close = darwin_submitqueue_close, ++ .bo_init = darwin_bo_init, ++ .bo_init_dmabuf = darwin_bo_init_dmabuf, ++ .bo_init_host = darwin_bo_init_host, ++ .bo_export_dmabuf = darwin_bo_export_dmabuf, ++ .bo_map = darwin_bo_map, ++ .bo_allow_dump = darwin_bo_allow_dump, ++ .bo_finish = darwin_bo_finish, ++ .bo_set_metadata = darwin_bo_set_metadata, ++ .bo_get_metadata = darwin_bo_get_metadata, ++ .submit_create = darwin_submit_create, ++ .submit_finish = darwin_submit_finish, ++ .submit_add_entries = darwin_submit_add_entries, ++ .submit_add_bind = darwin_submit_add_bind, ++ .queue_submit = darwin_queue_submit, ++ .queue_wait_fence = darwin_queue_wait_fence, ++ .sparse_vma_init = darwin_sparse_vma_init, ++ .sparse_vma_finish = darwin_sparse_vma_finish, ++}; ++ ++VkResult ++tu_knl_darwin_load(struct tu_instance *instance) ++{ ++ if (instance->vk.enabled_extensions.KHR_display) { ++ return vk_errorf(instance, VK_ERROR_INITIALIZATION_FAILED, "I can't KHR_display"); ++ } ++ ++ VkResult result = dw_open_connection(instance); ++ if (result != VK_SUCCESS) { ++ if (result != VK_ERROR_INCOMPATIBLE_DRIVER) ++ mesa_loge("NativeAdreno: could not open the kernel client (%d)", result); ++ return result; ++ } ++ ++ struct tu_physical_device *device = (struct tu_physical_device *) ++ vk_zalloc(&instance->vk.alloc, sizeof(*device), 8, VK_SYSTEM_ALLOCATION_SCOPE_INSTANCE); ++ if (!device) { ++ dw_close_connection(); ++ return vk_error(instance, VK_ERROR_OUT_OF_HOST_MEMORY); ++ } ++ ++ const NativeAdrenoAbiDeviceInfo *info = &dw.info; ++ ++ device->instance = instance; ++ device->local_fd = -1; ++ device->master_fd = -1; ++ device->kgsl_dma_fd = -1; ++ snprintf(device->fd_path, sizeof(device->fd_path), "NativeAdrenoGpu"); ++ ++ device->dev_id.chip_id = info->chip_id; ++ device->dev_id.gpu_id = ((info->chip_id >> 24) & 0xff) * 100 + ++ ((info->chip_id >> 16) & 0xff) * 10 + ++ ((info->chip_id >> 8) & 0xff); ++ /* The chip id decodes to the Adreno family only for kgsl-style ids; the freedreno device table matches this chip by ++ * chip id (0x43050a01) alone. */ ++ if (info->chip_id == 0x43050a01) ++ device->dev_id.gpu_id = 740; ++ device->gmem_size = debug_get_num_option("TU_GMEM", info->gmem_bytes); ++ device->gmem_base = info->gmem_base; ++ device->uche_trap_base = info->uche_trap_base; ++ ++ device->has_set_iova = !!(info->flags & NativeAdrenoDeviceClientIova); ++ device->va_start = info->va_start; ++ device->va_size = info->va_size; ++ device->has_lazy_bos = false; ++ device->has_sparse = false; ++ device->has_sparse_prr = false; ++ device->has_raytracing = false; ++ device->has_preemption = false; ++ device->is_perf_cntr_selectable = false; ++ device->has_cached_coherent_memory = !!(info->flags & NativeAdrenoDeviceCachedCoherent); ++ device->has_host_import = !!(dw.params.flags & NativeAdrenoParamsHostImport); ++ device->submitqueue_priority_count = 1; ++ ++ device->ubwc_config.highest_bank_bit = info->highest_bank_bit; ++ device->ubwc_config.bank_swizzle_levels = info->ubwc_swizzle; ++ device->ubwc_config.macrotile_mode = ++ info->macrotile_mode ? FDL_MACROTILE_8_CHANNEL : FDL_MACROTILE_4_CHANNEL; ++ ++ device->timeline_type = vk_sync_timeline_get_type(&vk_darwin_sync_type); ++ device->sync_types[0] = &vk_darwin_sync_type; ++ device->sync_types[1] = &device->timeline_type.sync; ++ device->sync_types[2] = NULL; ++ ++ device->heap.size = tu_get_system_heap_size(device); ++ device->heap.used = 0u; ++ device->heap.flags = VK_MEMORY_HEAP_DEVICE_LOCAL_BIT; ++ ++ instance->knl = &darwin_knl_funcs; ++ ++ result = tu_physical_device_init(device, instance); ++ /* The enumeration probe only needed the facts; the device takes the slot again in device_init. */ ++ dw_close_connection(); ++ if (result != VK_SUCCESS) { ++ mesa_loge("NativeAdreno: tu_physical_device_init failed (%d)", result); ++ vk_free(&instance->vk.alloc, device); ++ return result; ++ } ++ ++ list_addtail(&device->vk.link, &instance->vk.physical_devices.list); ++ return VK_SUCCESS; ++} +diff --git a/src/util/cache_ops_aarch64.c b/src/util/cache_ops_aarch64.c +index d56ab8a..2ab772b 100644 +--- a/src/util/cache_ops_aarch64.c ++++ b/src/util/cache_ops_aarch64.c +@@ -29,9 +29,16 @@ + static uint32_t + get_ctr_el0(void) + { ++#if defined(__APPLE__) ++ /* Darwin does not let EL0 read CTR_EL0 (the instruction raises SIGILL, which would take down whichever system process hosts the ++ * driver). The Adreno 740 hosts are Cortex cores with 64-byte data cache lines: CWG = DminLine = 4 (16 words). A line size that is ++ * too small only costs extra cache maintenance operations; one that is too large would skip lines. */ ++ return (4u << 24) | (4u << 16); ++#else + uint32_t ctr_el0; + __asm("mrs\t%x0, ctr_el0" : "=r"(ctr_el0)); + return ctr_el0; ++#endif + } + + static uint32_t +diff --git a/src/vulkan/wsi/wsi_common_metal_layer.m b/src/vulkan/wsi/wsi_common_metal_layer.m +index 67c3e7f..9fbb042 100644 +--- a/src/vulkan/wsi/wsi_common_metal_layer.m ++++ b/src/vulkan/wsi/wsi_common_metal_layer.m +@@ -4,6 +4,7 @@ + * SPDX-License-Identifier: MIT + */ + ++#include + #include "wsi_common_metal_layer.h" + + #import +@@ -137,7 +138,9 @@ + wsi_metal_layer_set_immediate(const CAMetalLayer *metal_layer, + bool enable_immediate) + { ++#if TARGET_OS_OSX + metal_layer.displaySyncEnabled = !enable_immediate; ++#endif /* iOS layers are always display synchronised */ + } + + VkResult +@@ -169,7 +172,9 @@ + metal_layer.maximumDrawableCount = image_count; + metal_layer.drawableSize = (CGSize){.width = width, .height = height}; + metal_layer.opaque = enable_opaque; ++#if TARGET_OS_OSX + metal_layer.displaySyncEnabled = !enable_immediate; ++#endif + metal_layer.pixelFormat = metal_format; + + metal_layer.colorspace = cg_color_space; diff --git a/patches/xnu/apple-only-cpu-tunables.patch b/patches/xnu/apple-only-cpu-tunables.patch new file mode 100644 index 0000000..9470a24 --- /dev/null +++ b/patches/xnu/apple-only-cpu-tunables.patch @@ -0,0 +1,28 @@ +diff --git a/osfmk/arm64/start.s b/osfmk/arm64/start.s +index d4909c7..0c01b79 100644 +--- a/osfmk/arm64/start.s ++++ b/osfmk/arm64/start.s +@@ -33,7 +33,9 @@ + #include + #include + #include ++#if defined(APPLE_ARM64_ARCH_FAMILY) + #include ++#endif + #include + + #if __ARM_KERNEL_PROTECT__ +@@ -783,10 +785,13 @@ common_start: + // Initialization common to all non-virtual Apple targets + #endif // APPLE_ARM64_ARCH_FAMILY + ++ ++#if defined(APPLE_ARM64_ARCH_FAMILY) + // Read MIDR before start of per-SoC tunables + mrs x12, MIDR_EL1 + + APPLY_TUNABLES x12, x13, x14 ++#endif + + #if HAS_CLUSTER && !NO_CPU_OVRD + // Unmask external IRQs if we're restarting from non-retention WFI diff --git a/patches/xnu/architectural-timer-entry.patch b/patches/xnu/architectural-timer-entry.patch new file mode 100644 index 0000000..635ed0d --- /dev/null +++ b/patches/xnu/architectural-timer-entry.patch @@ -0,0 +1,11 @@ +--- a/osfmk/arm64/start.s ++++ b/osfmk/arm64/start.s +@@ -697,7 +697,7 @@ + dsb ish + + +-#if defined(BCM2837) ++#if defined(BCM2837) && (!defined(XNU_NATIVE_ARCH_TIMER) || !XNU_NATIVE_ARCH_TIMER) + // Setup timer interrupt routing; must be done before MMU is enabled + mrs x15, MPIDR_EL1 // Load MPIDR to get CPU number + and x15, x15, #0xFF // CPU number is in MPIDR Affinity Level 0 diff --git a/patches/xnu/armvirt-platform-personality.patch b/patches/xnu/armvirt-platform-personality.patch new file mode 100644 index 0000000..fefd299 --- /dev/null +++ b/patches/xnu/armvirt-platform-personality.patch @@ -0,0 +1,28 @@ +diff --git a/iokit/KernelConfigTables.cpp b/iokit/KernelConfigTables.cpp +index 559a9db..080332e 100644 +--- a/iokit/KernelConfigTables.cpp ++++ b/iokit/KernelConfigTables.cpp +@@ -35,6 +35,23 @@ + const char * gIOKernelConfigTables = + "(" + " {" ++ " 'IOClass' = ArmCpu;" ++ " 'IOProviderClass' = IOPlatformDevice;" ++ " 'IOProbeScore' = 2000:32;" ++ " }," ++ " {" ++ " 'IOClass' = ArmVirtPlatform;" ++ " 'IOProviderClass' = IOPlatformExpertDevice;" ++ " 'IONameMatch' = 'linux,dummy-virt';" ++ " 'IOProbeScore' = 10000:32;" ++ " }," ++ " {" ++ " 'IOClass' = ArmDTPlatform;" ++ " 'IOProviderClass' = IOPlatformExpertDevice;" ++ " 'IONameMatch' = 'device-tree';" ++ " 'IOProbeScore' = 5000:32;" ++ " }," ++ " {" + " 'IOClass' = IOPanicPlatform;" + " 'IOProviderClass' = IOPlatformExpertDevice;" + " 'IOProbeScore' = 0:32;" diff --git a/patches/xnu/cxx-flag-dependencies.patch b/patches/xnu/cxx-flag-dependencies.patch new file mode 100644 index 0000000..08adcde --- /dev/null +++ b/patches/xnu/cxx-flag-dependencies.patch @@ -0,0 +1,104 @@ +diff --git a/bsd/conf/Makefile.template b/bsd/conf/Makefile.template +index 1e2c5d1..882c1c9 100644 +--- a/bsd/conf/Makefile.template ++++ b/bsd/conf/Makefile.template +@@ -544,7 +544,7 @@ $(COBJS): .CFLAGS + $(_v)$(REPLACECONTENTS) $@ $(KCC) $(CFLAGS) $(INCFLAGS) + $(CXXOBJS): .CXXFLAGS + .CXXFLAGS: ALWAYS +- $(_v)$(REPLACECONTENTS) $@ $(KC++) $(CXXFLAGS) $(INCFLAGS) ++ $(_v)$(REPLACECONTENTS) $@ $(KC++) $(CFLAGS) $(CXXFLAGS) $(INCFLAGS) + $(SOBJS): .SFLAGS + .SFLAGS: ALWAYS + $(_v)$(REPLACECONTENTS) $@ $(S_KCC) $(SFLAGS) $(INCFLAGS) +diff --git a/iokit/conf/Makefile.template b/iokit/conf/Makefile.template +index 2b3b974..c300580 100644 +--- a/iokit/conf/Makefile.template ++++ b/iokit/conf/Makefile.template +@@ -209,7 +209,7 @@ $(COBJS): .CFLAGS + $(_v)$(REPLACECONTENTS) $@ $(KCC) $(CFLAGS) $(INCFLAGS) + $(CXXOBJS): .CXXFLAGS + .CXXFLAGS: ALWAYS +- $(_v)$(REPLACECONTENTS) $@ $(KC++) $(CXXFLAGS) $(INCFLAGS) ++ $(_v)$(REPLACECONTENTS) $@ $(KC++) $(CFLAGS) $(CXXFLAGS) $(INCFLAGS) + $(SOBJS): .SFLAGS + .SFLAGS: ALWAYS + $(_v)$(REPLACECONTENTS) $@ $(S_KCC) $(SFLAGS) $(INCFLAGS) +diff --git a/libkern/conf/Makefile.template b/libkern/conf/Makefile.template +index f0c8a72..c382184 100644 +--- a/libkern/conf/Makefile.template ++++ b/libkern/conf/Makefile.template +@@ -144,7 +144,7 @@ $(COBJS): .CFLAGS + $(_v)$(REPLACECONTENTS) $@ $(KCC) $(CFLAGS) $(INCFLAGS) + $(CXXOBJS): .CXXFLAGS + .CXXFLAGS: ALWAYS +- $(_v)$(REPLACECONTENTS) $@ $(KC++) $(CXXFLAGS) $(INCFLAGS) ++ $(_v)$(REPLACECONTENTS) $@ $(KC++) $(CFLAGS) $(CXXFLAGS) $(INCFLAGS) + $(SOBJS): .SFLAGS + .SFLAGS: ALWAYS + $(_v)$(REPLACECONTENTS) $@ $(S_KCC) $(SFLAGS) $(INCFLAGS) +diff --git a/libsa/conf/Makefile.template b/libsa/conf/Makefile.template +index b20de6a..5a21d5d 100644 +--- a/libsa/conf/Makefile.template ++++ b/libsa/conf/Makefile.template +@@ -59,7 +59,7 @@ $(COBJS): .CFLAGS + $(_v)$(REPLACECONTENTS) $@ $(KCC) $(CFLAGS) $(INCFLAGS) + $(CXXOBJS): .CXXFLAGS + .CXXFLAGS: ALWAYS +- $(_v)$(REPLACECONTENTS) $@ $(KC++) $(CXXFLAGS) $(INCFLAGS) ++ $(_v)$(REPLACECONTENTS) $@ $(KC++) $(CFLAGS) $(CXXFLAGS) $(INCFLAGS) + $(SOBJS): .SFLAGS + .SFLAGS: ALWAYS + $(_v)$(REPLACECONTENTS) $@ $(S_KCC) $(SFLAGS) $(INCFLAGS) +diff --git a/osfmk/conf/Makefile.template b/osfmk/conf/Makefile.template +index 5126a45..4d7c60b 100644 +--- a/osfmk/conf/Makefile.template ++++ b/osfmk/conf/Makefile.template +@@ -389,7 +389,7 @@ $(COBJS): .CFLAGS + $(_v)$(REPLACECONTENTS) $@ $(KCC) $(CFLAGS) $(INCFLAGS) + $(CXXOBJS): .CXXFLAGS + .CXXFLAGS: ALWAYS +- $(_v)$(REPLACECONTENTS) $@ $(KC++) $(CXXFLAGS) $(INCFLAGS) ++ $(_v)$(REPLACECONTENTS) $@ $(KC++) $(CFLAGS) $(CXXFLAGS) $(INCFLAGS) + $(SOBJS): .SFLAGS + .SFLAGS: ALWAYS + $(_v)$(REPLACECONTENTS) $@ $(S_KCC) $(SFLAGS) $(INCFLAGS) +diff --git a/pexpert/conf/Makefile.template b/pexpert/conf/Makefile.template +index 2d65891..30b79ba 100644 +--- a/pexpert/conf/Makefile.template ++++ b/pexpert/conf/Makefile.template +@@ -78,7 +78,7 @@ $(COBJS): .CFLAGS + $(_v)$(REPLACECONTENTS) $@ $(KCC) $(CFLAGS) $(INCFLAGS) + $(CXXOBJS): .CXXFLAGS + .CXXFLAGS: ALWAYS +- $(_v)$(REPLACECONTENTS) $@ $(KC++) $(CXXFLAGS) $(INCFLAGS) ++ $(_v)$(REPLACECONTENTS) $@ $(KC++) $(CFLAGS) $(CXXFLAGS) $(INCFLAGS) + $(SOBJS): .SFLAGS + .SFLAGS: ALWAYS + $(_v)$(REPLACECONTENTS) $@ $(S_KCC) $(SFLAGS) $(INCFLAGS) +diff --git a/san/conf/Makefile.template b/san/conf/Makefile.template +index 349746a..0902e06 100644 +--- a/san/conf/Makefile.template ++++ b/san/conf/Makefile.template +@@ -55,7 +55,7 @@ $(COBJS): .CFLAGS + $(_v)$(REPLACECONTENTS) $@ $(KCC) $(CFLAGS) $(INCFLAGS) + $(CXXOBJS): .CXXFLAGS + .CXXFLAGS: ALWAYS +- $(_v)$(REPLACECONTENTS) $@ $(KC++) $(CXXFLAGS) $(INCFLAGS) ++ $(_v)$(REPLACECONTENTS) $@ $(KC++) $(CFLAGS) $(CXXFLAGS) $(INCFLAGS) + $(SOBJS): .SFLAGS + .SFLAGS: ALWAYS + $(_v)$(REPLACECONTENTS) $@ $(S_KCC) $(SFLAGS) $(INCFLAGS) +diff --git a/security/conf/Makefile.template b/security/conf/Makefile.template +index 67b6106..308d638 100644 +--- a/security/conf/Makefile.template ++++ b/security/conf/Makefile.template +@@ -81,7 +81,7 @@ $(COBJS): .CFLAGS + $(_v)$(REPLACECONTENTS) $@ $(KCC) $(CFLAGS) $(INCFLAGS) + $(CXXOBJS): .CXXFLAGS + .CXXFLAGS: ALWAYS +- $(_v)$(REPLACECONTENTS) $@ $(KC++) $(CXXFLAGS) $(INCFLAGS) ++ $(_v)$(REPLACECONTENTS) $@ $(KC++) $(CFLAGS) $(CXXFLAGS) $(INCFLAGS) + $(SOBJS): .SFLAGS + .SFLAGS: ALWAYS + $(_v)$(REPLACECONTENTS) $@ $(S_KCC) $(SFLAGS) $(INCFLAGS) diff --git a/patches/xnu/disabled-deferred-reclaim.patch b/patches/xnu/disabled-deferred-reclaim.patch new file mode 100644 index 0000000..3b1ae40 --- /dev/null +++ b/patches/xnu/disabled-deferred-reclaim.patch @@ -0,0 +1,11 @@ +diff --git a/osfmk/vm/vm_user.c b/osfmk/vm/vm_user.c +--- a/osfmk/vm/vm_user.c ++++ b/osfmk/vm/vm_user.c +@@ -4503,7 +4503,6 @@ + (void) task; + (void) address; + (void) size; +- (void) indices; + return KERN_NOT_SUPPORTED; + #endif /* CONFIG_DEFERRED_RECLAIM */ + } diff --git a/patches/xnu/external-archive-dependencies.patch b/patches/xnu/external-archive-dependencies.patch new file mode 100644 index 0000000..04cc0e0 --- /dev/null +++ b/patches/xnu/external-archive-dependencies.patch @@ -0,0 +1,14 @@ +diff --git a/makedefs/MakeInc.kernel b/makedefs/MakeInc.kernel +index e181436..f3532bd 100644 +--- a/makedefs/MakeInc.kernel ++++ b/makedefs/MakeInc.kernel +@@ -187,6 +187,9 @@ $(TARGET)/lib$(KERNEL_FILE_NAME).a: $(addprefix $(TARGET)/,$(foreach component,$ + $(_v)$(LN) $(call function_convert_build_config_to_objdir,$(CURRENT_BUILD_CONFIG))/lib$(KERNEL_FILE_NAME).a $(OBJROOT)/lib$(KERNEL_FILE_NAME).a + endif + ++# Explicit external source archives participate in incremental relinking. ++$(TARGET)/$(KERNEL_FILE_NAME).unstripped.noctf: $(XNU_EXTERNAL_KERNEL_ARCHIVES) ++ + $(TARGET)/$(KERNEL_FILE_NAME).unstripped.noctf: $(addprefix $(TARGET)/,$(foreach component,$(COMPONENT_LIST),$(component)/$(CURRENT_KERNEL_CONFIG)/$(component).filelist)) lastkerneldataconst.o lastkernelconstructor.o nonlto.o version.o $(LDFILES_KERNEL_ONLY) .LDFLAGS $(filter %/MakeInc.kernel,$(MAKEFILE_LIST)) + ifeq ($(PRE_LTO),1) + @$(LOG_LTO) "$(@F)" diff --git a/patches/xnu/kernel-sdk-root.patch b/patches/xnu/kernel-sdk-root.patch new file mode 100644 index 0000000..81826e4 --- /dev/null +++ b/patches/xnu/kernel-sdk-root.patch @@ -0,0 +1,25 @@ +diff --git a/makedefs/MakeInc.def b/makedefs/MakeInc.def +index c66add1..3e1121d 100644 +--- a/makedefs/MakeInc.def ++++ b/makedefs/MakeInc.def +@@ -667,7 +667,8 @@ LDFLAGS_KERNEL_GEN = \ + -Wl,-headerpad,152 + + # LDFLAGS_KERNEL_SDK = -L$(SDKROOT)/usr/local/lib/kernel -lfirehose_kernel +-LDFLAGS_KERNEL_SDK = -L$(SDKROOT)/usr/local/lib/kernel -L$(SDKROOT)/usr/local/lib/kernel/platform ++export KERNEL_SDKROOT ?= $(SDKROOT) ++LDFLAGS_KERNEL_SDK = -L$(KERNEL_SDKROOT)/usr/local/lib/kernel -L$(KERNEL_SDKROOT)/usr/local/lib/kernel/platform + + LDFLAGS_KERNEL_RELEASE = + LDFLAGS_KERNEL_DEVELOPMENT = +@@ -911,8 +912,8 @@ INCFLAGS_IMPORT = $(patsubst %, -I$(OBJROOT)/EXPORT_HDRS/%, $(COMPONENT_IMPORT_L + INCFLAGS_EXTERN = -I$(SRCROOT)/EXTERNAL_HEADERS + INCFLAGS_GEN = -I$(SRCROOT)/$(COMPONENT) -I$(OBJROOT)/EXPORT_HDRS/$(COMPONENT) + INCFLAGS_LOCAL = -I. +-INCFLAGS_SDK = -I$(SDKROOT)/usr/local/include/kernel +-INCFLAGS_PLATFORM = -I$(SDKROOT)/$(KPINCDIR)/platform ++INCFLAGS_SDK = -I$(KERNEL_SDKROOT)/usr/local/include/kernel -I$(KERNEL_SDKROOT)/usr/local/include ++INCFLAGS_PLATFORM = -I$(KERNEL_SDKROOT)/$(KPINCDIR)/platform + + INCFLAGS = $(INCFLAGS_LOCAL) $(INCFLAGS_GEN) $(INCFLAGS_IMPORT) $(INCFLAGS_EXTERN) $(INCFLAGS_MAKEFILE) $(INCFLAGS_SDK) $(INCFLAGS_PLATFORM) + diff --git a/patches/xnu/local-availability.patch b/patches/xnu/local-availability.patch new file mode 100644 index 0000000..4d1708a --- /dev/null +++ b/patches/xnu/local-availability.patch @@ -0,0 +1,12 @@ +diff --git a/bsd/sys/make_symbol_aliasing.sh b/bsd/sys/make_symbol_aliasing.sh +--- a/bsd/sys/make_symbol_aliasing.sh ++++ b/bsd/sys/make_symbol_aliasing.sh +@@ -34,7 +34,7 @@ + SDKROOT="$1" + OUTPUT="$2" + +-AVAILABILITY_PL="${SDKROOT}/${DRIVERKITROOT}/usr/local/libexec/availability.pl" ++AVAILABILITY_PL="${XNU_AVAILABILITY_PL:-${SDKROOT}/${DRIVERKITROOT}/usr/local/libexec/availability.pl}" + + if [ ! -x "${AVAILABILITY_PL}" ] ; then + echo "Unable to locate ${AVAILABILITY_PL} (or not executable)" >&2 diff --git a/patches/xnu/mockfs-intermediate-lookup.patch b/patches/xnu/mockfs-intermediate-lookup.patch new file mode 100644 index 0000000..c9b33e9 --- /dev/null +++ b/patches/xnu/mockfs-intermediate-lookup.patch @@ -0,0 +1,11 @@ +--- a/bsd/miscfs/mockfs/mockfs_vnops.c ++++ b/bsd/miscfs/mockfs/mockfs_vnops.c +@@ -108,7 +108,7 @@ + fsnode = (mockfs_fsnode_t) dvp->v_data; + target_fsnode = NULL; + +- if ((op == LOOKUP) && (fsnode->type == MOCKFS_ROOT)) { ++ if ((op == LOOKUP || !(cnp->cn_flags & ISLASTCN)) && (fsnode->type == MOCKFS_ROOT)) { + /* + * Okay, we're looking in the root directory, so we aren't necessarily + * going to fail. What are we looking for? diff --git a/patches/xnu/mockfs-memory-device-width.patch b/patches/xnu/mockfs-memory-device-width.patch new file mode 100644 index 0000000..cef7849 --- /dev/null +++ b/patches/xnu/mockfs-memory-device-width.patch @@ -0,0 +1,18 @@ +diff --git a/bsd/miscfs/mockfs/mockfs_vfsops.c b/bsd/miscfs/mockfs/mockfs_vfsops.c +index 325bb06..bea8e6e 100644 +--- a/bsd/miscfs/mockfs/mockfs_vfsops.c ++++ b/bsd/miscfs/mockfs/mockfs_vfsops.c +@@ -99,9 +99,11 @@ mockfs_mountroot(mount_t mp, vnode_t rvp, __unused vfs_context_t ctx) + + if (!VNOP_IOCTL(rvp, DKIOCGETMEMDEVINFO, (caddr_t)&memdev_info, 0, NULL)) { + /* +- * For the moment, we won't try to optimize when mi_phys is true. ++ * The legacy ioctl exposes a 32-bit base page. A 64-bit kernel ++ * virtual address cannot be represented; use the real device I/O ++ * path instead. Physical memory is not supported by this optimization. + */ +- if (!mockfs_mount_data->mockfs_physical_memory) { ++ if (!memdev_info.mi_phys && (sizeof(uintptr_t) == sizeof(uint32_t))) { + mockfs_mount_data->mockfs_memory_backed = memdev_info.mi_mdev; + mockfs_mount_data->mockfs_physical_memory = memdev_info.mi_phys; + mockfs_mount_data->mockfs_memdev_base = memdev_info.mi_base; diff --git a/patches/xnu/mockfs-statfs.patch b/patches/xnu/mockfs-statfs.patch new file mode 100644 index 0000000..6a5eef6 --- /dev/null +++ b/patches/xnu/mockfs-statfs.patch @@ -0,0 +1,37 @@ +--- a/bsd/miscfs/mockfs/mockfs_vfsops.c ++++ b/bsd/miscfs/mockfs/mockfs_vfsops.c +@@ -286,6 +286,27 @@ + return 0; + } + ++static int ++mockfs_getattr(mount_t mp, struct vfs_attr *attributes, __unused vfs_context_t ctx) ++{ ++ mockfs_fsnode_t root = ((mockfs_mount_t)mp->mnt_data)->mockfs_root; ++ uint64_t nodes = 1 + (root->child_a != NULL) + (root->child_b != NULL) + (root->child_c != NULL); ++ uint64_t bytes = mp->mnt_devvp->v_specinfo->si_devsize; ++ uint64_t blocks = bytes / 4096 + ((bytes & 4095) != 0); ++ VFSATTR_RETURN(attributes, f_bsize, 4096); ++ VFSATTR_RETURN(attributes, f_iosize, 4096); ++ VFSATTR_RETURN(attributes, f_blocks, blocks); ++ VFSATTR_RETURN(attributes, f_bused, blocks); ++ VFSATTR_RETURN(attributes, f_bfree, 0); ++ VFSATTR_RETURN(attributes, f_bavail, 0); ++ VFSATTR_RETURN(attributes, f_files, nodes); ++ VFSATTR_RETURN(attributes, f_ffree, 0); ++ VFSATTR_RETURN(attributes, f_objcount, nodes); ++ VFSATTR_RETURN(attributes, f_maxobjcount, nodes); ++ VFSATTR_RETURN(attributes, f_fssubtype, 0); ++ return 0; ++} ++ + int + mockfs_init(__unused struct vfsconf * vfsc) + { +@@ -296,5 +317,6 @@ + .vfs_unmount = mockfs_unmount, + .vfs_root = mockfs_root, + .vfs_sync = mockfs_sync, ++ .vfs_getattr = mockfs_getattr, + .vfs_init = mockfs_init, + }; diff --git a/patches/xnu/mockfs-test-file.patch b/patches/xnu/mockfs-test-file.patch new file mode 100644 index 0000000..8204567 --- /dev/null +++ b/patches/xnu/mockfs-test-file.patch @@ -0,0 +1,271 @@ +--- a/bsd/miscfs/mockfs/mockfs_fsnode.h ++++ b/bsd/miscfs/mockfs/mockfs_fsnode.h +@@ -46,7 +46,8 @@ + enum mockfs_fsnode_type { + MOCKFS_ROOT, + MOCKFS_DEV, +- MOCKFS_FILE ++ MOCKFS_FILE, ++ MOCKFS_TEST_FILE + }; + + /* +@@ -58,6 +59,7 @@ + * VREG access). + */ + struct mockfs_fsnode { ++ uint64_t backing_offset; /* Offline two-file image only. */ + uint64_t size; /* Bytes of data; 0 unless type is MOCKFS_FILE */ + uint8_t type; /* Serves as a unique identifier for now */ + mount_t mnt; /* The mount that this node belongs to */ +@@ -66,6 +68,7 @@ + /* TODO: Replace child_a/child_b with something more flexible */ + struct mockfs_fsnode * child_a; /* TEMPORARY */ + struct mockfs_fsnode * child_b; /* TEMPORARY */ ++ struct mockfs_fsnode * child_c; /* Offline test input. */ + }; + + typedef struct mockfs_fsnode * mockfs_fsnode_t; +--- a/bsd/miscfs/mockfs/mockfs_fsnode.c ++++ b/bsd/miscfs/mockfs/mockfs_fsnode.c +@@ -74,6 +74,7 @@ + case MOCKFS_DEV: + break; + case MOCKFS_FILE: ++ case MOCKFS_TEST_FILE: + /* + * For a regular file, size is meaningful, but it will always be equal to the + * size of the backing device. +@@ -145,6 +146,12 @@ + } + } + ++ if (fsnp->child_c) { ++ if ((rvalue = mockfs_fsnode_destroy(fsnp->child_c))) { ++ panic("mockfs: failed to destroy test node"); ++ } ++ } ++ + /* + * We need to orphan this node before we destroy it. + */ +@@ -202,6 +209,9 @@ + } else if (!parent->child_b) { + parent->child_b = child; + child->parent = parent; ++ } else if (!parent->child_c) { ++ parent->child_c = child; ++ child->parent = parent; + } else { + rvalue = ENOMEM; + } +@@ -243,6 +253,9 @@ + } else if (parent->child_b == fsnp) { + parent->child_b = NULL; + fsnp->parent = NULL; ++ } else if (parent->child_c == fsnp) { ++ parent->child_c = NULL; ++ fsnp->parent = NULL; + } else { + panic("mockfs_fsnode_orphan insanity, fsnp->parent != parent->child; fsnp = %p (in case gdb is screwing with you)", fsnp); + } +@@ -278,6 +291,8 @@ + *child = parent->child_a; + } else if ((parent->child_b) && (parent->child_b->type == type)) { + *child = parent->child_b; ++ } else if ((parent->child_c) && (parent->child_c->type == type)) { ++ *child = parent->child_c; + } else { + rvalue = ENOENT; + } +@@ -322,7 +337,7 @@ + * We need to create the vnode; this will be unpleasant. + */ + vnfs_param.vnfs_mp = fsnp->mnt; +- vnfs_param.vnfs_vtype = (fsnp->type == MOCKFS_FILE) ? VREG : VDIR; ++ vnfs_param.vnfs_vtype = ((fsnp->type == MOCKFS_FILE) || (fsnp->type == MOCKFS_TEST_FILE)) ? VREG : VDIR; + vnfs_param.vnfs_str = "mockfs"; + vnfs_param.vnfs_dvp = (fsnp->type == MOCKFS_ROOT) ? NULL : fsnp->parent->vp; + vnfs_param.vnfs_fsnode = fsnp; +--- a/bsd/miscfs/mockfs/mockfs_vnops.c ++++ b/bsd/miscfs/mockfs/mockfs_vnops.c +@@ -29,6 +29,7 @@ + #include + #include + #include ++#include + #include + #include + #include +@@ -46,6 +47,11 @@ + * each function does in generic terms. + */ + int mockfs_lookup(struct vnop_lookup_args * ap); ++static int mockfs_readonly_open(struct vnop_open_args *ap) ++{ ++ return (ap->a_mode & FWRITE) ? EROFS : 0; ++} ++ + int mockfs_getattr(struct vnop_getattr_args * ap); + int mockfs_read(struct vnop_read_args * ap); + int mockfs_strategy(struct vnop_strategy_args * ap); +@@ -117,10 +123,15 @@ + * accidentally commit a change to the init_process pathname. We map from name to node type + * here, as mockfs doesn't current use names; just unique types. + */ +- if (!strncmp(cnp->cn_nameptr, "sbin", 5)) { ++ if (!strncmp(cnp->cn_nameptr, "sbin", 5) || ++ !strncmp(cnp->cn_nameptr, "usr", 4) || !strncmp(cnp->cn_nameptr, "lib", 4)) { + target_fsnode = fsnode; + } else if (!strncmp(cnp->cn_nameptr, "dev", 4)) { + mockfs_fsnode_child_by_type(fsnode, MOCKFS_DEV, &target_fsnode); ++ } else if (!strncmp(cnp->cn_nameptr, "dyld", 5)) { ++ mockfs_fsnode_child_by_type(fsnode, MOCKFS_TEST_FILE, &target_fsnode); ++ } else if (!strncmp(cnp->cn_nameptr, "probe.bin", 10)) { ++ mockfs_fsnode_child_by_type(fsnode, MOCKFS_TEST_FILE, &target_fsnode); + } else if (!strncmp(cnp->cn_nameptr, "launchd", 8)) { + mockfs_fsnode_child_by_type(fsnode, MOCKFS_FILE, &target_fsnode); + } else { +@@ -392,8 +403,12 @@ + * This also means that their request should always be contiguous, so the run calculation is easy! + */ + if (vp->v_type == VREG) { +- *bpn = foffset / blksize; +- *run = fsnode->size - foffset; ++ if (foffset < 0 || (uint64_t)foffset > fsnode->size) { ++ return EINVAL; ++ } ++ *bpn = (fsnode->backing_offset + foffset) / blksize; ++ /* File metadata is exact; block I/O may include allocated zero padding. */ ++ *run = ((fsnode->size + 4095) & ~4095ULL) - foffset; + + if (ap->a_size > *run) { + /* We've been asked for more data than the backing device can provide; we're done. */ +@@ -411,9 +426,9 @@ + { .opve_op = &vnop_default_desc, .opve_impl = (VOPFUNC) vn_default_error }, /* default */ + { .opve_op = &vnop_lookup_desc, .opve_impl = (VOPFUNC) mockfs_lookup }, /* lookup */ + { .opve_op = &vnop_create_desc, .opve_impl = (VOPFUNC) err_create },/* create */ +- { .opve_op = &vnop_open_desc, .opve_impl = (VOPFUNC) err_open }, /* open */ ++ { .opve_op = &vnop_open_desc, .opve_impl = (VOPFUNC) mockfs_readonly_open }, /* open */ + { .opve_op = &vnop_mknod_desc, .opve_impl = (VOPFUNC) err_mknod }, /* mknod */ +- { .opve_op = &vnop_close_desc, .opve_impl = (VOPFUNC) err_close }, /* close */ ++ { .opve_op = &vnop_close_desc, .opve_impl = (VOPFUNC) nop_close }, /* close */ + { .opve_op = &vnop_access_desc, .opve_impl = (VOPFUNC) err_access }, /* access */ + { .opve_op = &vnop_getattr_desc, .opve_impl = (VOPFUNC) mockfs_getattr }, /* getattr */ + { .opve_op = &vnop_setattr_desc, .opve_impl = (VOPFUNC) err_setattr }, /* setattr */ +@@ -421,7 +436,7 @@ + { .opve_op = &vnop_write_desc, .opve_impl = (VOPFUNC) err_write }, /* write */ + { .opve_op = &vnop_ioctl_desc, .opve_impl = (VOPFUNC) err_ioctl }, /* ioctl */ + { .opve_op = &vnop_select_desc, .opve_impl = (VOPFUNC) err_select }, /* select */ +- { .opve_op = &vnop_mmap_desc, .opve_impl = (VOPFUNC) err_mmap }, /* mmap */ ++ { .opve_op = &vnop_mmap_desc, .opve_impl = (VOPFUNC) nop_mmap }, /* mmap */ + { .opve_op = &vnop_fsync_desc, .opve_impl = (VOPFUNC) nop_fsync }, /* fsync */ + { .opve_op = &vnop_remove_desc, .opve_impl = (VOPFUNC) err_remove }, /* remove */ + { .opve_op = &vnop_link_desc, .opve_impl = (VOPFUNC) err_link }, /* link */ +--- a/bsd/miscfs/mockfs/mockfs_vfsops.c ++++ b/bsd/miscfs/mockfs/mockfs_vfsops.c +@@ -34,12 +34,23 @@ + #include + #include + #include ++#include ++#include + #include + #include + #include + #include + + static LCK_GRP_DECLARE(mockfs_mtx_grp, "mockfs-mutex"); ++ ++/* Bounded offline input container, not a deployment filesystem or trust claim. ++ * Two unmodified files: /sbin/launchd and /probe.bin; all offsets native 4K. ++ */ ++struct mockfs_test_header { ++ char magic[8]; ++ uint32_t version, count; ++ uint64_t launch_offset, launch_size, test_offset, test_size; ++}; + + int mockfs_mountroot(mount_t mp, vnode_t rvp, __unused vfs_context_t ctx); + +@@ -68,6 +79,10 @@ + mockfs_fsnode_t root_fsnode = NULL; + mockfs_fsnode_t dev_fsnode = NULL; + mockfs_fsnode_t file_fsnode = NULL; ++ mockfs_fsnode_t test_fsnode = NULL; ++ struct mockfs_test_header header = {0}; ++ buf_t header_buf = NULL; ++ boolean_t test_image = FALSE; + mockfs_mount_t mockfs_mount_data = NULL; + dk_memdev_info_t memdev_info; + +@@ -113,6 +128,42 @@ + + lck_mtx_init(&mockfs_mount_data->mockfs_mnt_mtx, &mockfs_mtx_grp, LCK_ATTR_NULL); + ++ mp->mnt_data = (typeof(mp->mnt_data))mockfs_mount_data; ++ ++ /* Legacy single-Mach-O images retain their original mapping. */ ++ rvalue = buf_meta_bread(rvp, 0, 4096, vfs_context_ucred(ctx), &header_buf); ++ if (!rvalue) { ++ memcpy(&header, (void *)buf_dataptr(header_buf), sizeof(header)); ++ } ++ if (header_buf) { ++ buf_brelse(header_buf); ++ } ++ if (rvalue) { ++ goto done; ++ } ++ if (!memcmp(header.magic, "N4KROOT", 8)) { ++ uint64_t bytes = rvp->v_specinfo->si_devsize; ++ if (header.version != 1 || header.count != 2 || bytes > 32 * 1024 * 1024 || ++ header.launch_offset != 4096 || !header.launch_size || !header.test_size || ++ (header.test_offset & 4095) || header.test_offset < 8192 || header.test_offset > bytes || ++ header.launch_size > header.test_offset - header.launch_offset || ++ header.test_size > bytes - header.test_offset) { ++ rvalue = EINVAL; ++ goto done; ++ } ++ rvalue = mockfs_fsnode_create(mp, MOCKFS_TEST_FILE, &test_fsnode); ++ if (rvalue) { ++ goto done; ++ } ++ file_fsnode->backing_offset = header.launch_offset; ++ file_fsnode->size = header.launch_size; ++ test_fsnode->backing_offset = header.test_offset; ++ test_fsnode->size = header.test_size; ++ mockfs_mount_data->mockfs_memory_backed = FALSE; ++ test_image = TRUE; ++ } ++ ++ + /* + * All of the needed nodes/structures have been set up; now we just need to establish the relationships + * between the various mockfs nodes. +@@ -125,11 +176,18 @@ + goto done; + } + ++ if (test_image && (rvalue = mockfs_fsnode_adopt(root_fsnode, test_fsnode))) { ++ goto done; ++ } ++ + mockfs_mount_data->mockfs_root = root_fsnode; + mp->mnt_data = (typeof(mp->mnt_data))mockfs_mount_data; + + done: + if (rvalue) { ++ if (test_fsnode) { ++ mockfs_fsnode_destroy(test_fsnode); ++ } + if (file_fsnode) { + mockfs_fsnode_destroy(file_fsnode); + } +@@ -142,6 +200,7 @@ + if (mockfs_mount_data) { + lck_mtx_destroy(&mockfs_mount_data->mockfs_mnt_mtx, &mockfs_mtx_grp); + kfree_type(struct mockfs_mount, mockfs_mount_data); ++ mp->mnt_data = NULL; + } + } + diff --git a/patches/xnu/mockfs-typed-allocation.patch b/patches/xnu/mockfs-typed-allocation.patch new file mode 100644 index 0000000..cc494ff --- /dev/null +++ b/patches/xnu/mockfs-typed-allocation.patch @@ -0,0 +1,22 @@ +diff --git a/bsd/miscfs/mockfs/mockfs_fsnode.c b/bsd/miscfs/mockfs/mockfs_fsnode.c +index e000126..a470fc8 100644 +--- a/bsd/miscfs/mockfs/mockfs_fsnode.c ++++ b/bsd/miscfs/mockfs/mockfs_fsnode.c +@@ -85,7 +85,7 @@ mockfs_fsnode_create(mount_t mp, uint8_t type, mockfs_fsnode_t * fsnpp) + goto done; + } + +- *fsnpp = kalloc_type(mockfs_fsnode_t, Z_WAITOK | Z_ZERO | Z_NOFAIL); ++ *fsnpp = kalloc_type(struct mockfs_fsnode, Z_WAITOK | Z_ZERO | Z_NOFAIL); + (*fsnpp)->size = new_size; + (*fsnpp)->type = type; + (*fsnpp)->mnt = mp; +@@ -154,7 +154,7 @@ mockfs_fsnode_destroy(mockfs_fsnode_t fsnp) + } + } + +- kfree_type(mockfs_fsnode_t, fsnp); ++ kfree_type(struct mockfs_fsnode, fsnp); + done: + return rvalue; + } diff --git a/patches/xnu/native-4k-bootstrap-capacity.patch b/patches/xnu/native-4k-bootstrap-capacity.patch new file mode 100644 index 0000000..a047677 --- /dev/null +++ b/patches/xnu/native-4k-bootstrap-capacity.patch @@ -0,0 +1,61 @@ +--- a/osfmk/arm/pmap/pmap.h ++++ b/osfmk/arm/pmap/pmap.h +@@ -115,6 +115,11 @@ + * Upper limit on how many pages can be consumed by bootstrap page tables + */ + #define BOOTSTRAP_TABLE_SIZE (ARM_PGBYTES * 34) ++#elif !defined(APPLE_ARM64_ARCH_FAMILY) && !__ARM_16K_PG__ ++/* Native 4 KiB: two roots plus up to seven L2 pages per 6 GiB mapping. ++ * The physical and virtual bases may each straddle an extra 1 GiB region. ++ */ ++#define BOOTSTRAP_TABLE_SIZE (ARM_PGBYTES * 16) + #else /* defined(ARM_LARGE_MEMORY) */ + #define BOOTSTRAP_TABLE_SIZE (ARM_PGBYTES * 8) + #endif /* defined(ARM_LARGE_MEMORY) */ +--- a/osfmk/arm64/start.s ++++ b/osfmk/arm64/start.s +@@ -426,6 +426,10 @@ + + /* allocate an L2 table */ + 3: add $4, $4, PGBYTES ++ /* Refuse before a new table can overwrite adjacent kernel globals. */ ++ add $6, x25, #(BOOTSTRAP_TABLE_BYTES >> 12), lsl #12 ++ cmp $4, $6 ++ b.hs Lbootstrap_tables_exhausted + + /* create_l1_table_entry(virt_base, L1 table, L2 table, scratch1, scratch2, scratch3) */ + create_l1_table_entry $0, $3, $4, $6, $7, $8 +@@ -528,8 +532,8 @@ + // Invalidate all entries in the bootstrap page tables + mov x0, #(ARM_TTE_EMPTY) // Load invalid entry template + mov x1, x25 // Start at V=P pagetable root +- mov x2, #(TTE_PGENTRIES) // Load number of entries per page +- lsl x2, x2, #2 // Shift by 2 for num entries on 4 pages ++ mov x2, #(BOOTSTRAP_TABLE_BYTES / ARM_PGBYTES) ++ lsl x2, x2, #(ARM_PGSHIFT - TTE_SHIFT) // Clear the whole reserved pool + + Linvalidate_bootstrap: // do { + str x0, [x1], #(1 << TTE_SHIFT) // Invalidate and advance +@@ -613,6 +617,12 @@ + lsr x5, x24, #(ARM_TT_L2_SHIFT) + create_bootstrap_mapping x14, x15, x5, x3, x2, x9, x10, x11, x12, x13 + ++ b Lbootstrap_tables_ready ++Lbootstrap_tables_exhausted: ++ // MMU is still off. Stop with the offending cursor/end preserved for ++ // the early debugger rather than corrupting memory or forging a map. ++ b Lbootstrap_tables_exhausted ++Lbootstrap_tables_ready: + /* Ensure TTEs are visible */ + dsb ish + +--- a/osfmk/arm64/genassym.c ++++ b/osfmk/arm64/genassym.c +@@ -270,6 +270,7 @@ + + + DECLARE("PGBYTES", ARM_PGBYTES); ++ DECLARE("BOOTSTRAP_TABLE_BYTES", BOOTSTRAP_TABLE_SIZE); + DECLARE("PGSHIFT", ARM_PGSHIFT); + + DECLARE("VM_MIN_KERNEL_ADDRESS", VM_MIN_KERNEL_ADDRESS); diff --git a/patches/xnu/native-4k-signature-hash-cache.patch b/patches/xnu/native-4k-signature-hash-cache.patch new file mode 100644 index 0000000..9e17f53 --- /dev/null +++ b/patches/xnu/native-4k-signature-hash-cache.patch @@ -0,0 +1,257 @@ +--- a/bsd/sys/ubc_internal.h ++++ b/bsd/sys/ubc_internal.h +@@ -158,6 +158,13 @@ + void *XNU_PTRAUTH_SIGNED_PTR("cs_blob.csb_csm_obj") csb_csm_obj; + bool csb_csm_managed; + #endif ++ ++ /* Native 4K authenticators derived only from fully verified original ++ * 16K blocks. Append fields to preserve the existing binary prefix. ++ */ ++ const uint8_t *csb_native4k_hashes; ++ vm_size_t csb_native4k_hashes_size; ++ vm_size_t csb_native4k_hash_count; + }; + + /* +--- a/bsd/kern/ubc_subr.c ++++ b/bsd/kern/ubc_subr.c +@@ -69,6 +69,7 @@ + #include + #include + #include ++#include + #include /* last */ + + #include +@@ -108,6 +109,9 @@ + + static boolean_t ubc_cs_supports_multilevel_hash(struct cs_blob *blob); + static kern_return_t ubc_cs_convert_to_multilevel_hash(struct cs_blob *blob); ++#if defined(XNU_NATIVE_CS_HASH_CACHE) && XNU_NATIVE_CS_HASH_CACHE ++static int cs_native4k_prepare(vnode_t vp, struct cs_blob *blob); ++#endif + + ZONE_DEFINE_TYPE(ubc_info_zone, "ubc_info zone", struct ubc_info, + ZC_ZFREE_CLEARMEM); +@@ -371,7 +375,11 @@ + if (ntohl(cd->magic) != CSMAGIC_CODEDIRECTORY) { + return EBADEXEC; + } +- if ((cd->pageSize != PAGE_SHIFT_4K) && (cd->pageSize != PAGE_SHIFT)) { ++ if ((cd->pageSize != PAGE_SHIFT_4K) && (cd->pageSize != PAGE_SHIFT) ++#if defined(XNU_NATIVE_CS_HASH_CACHE) && XNU_NATIVE_CS_HASH_CACHE ++ && !(PAGE_SHIFT == 12 && cd->pageSize == 14) ++#endif ++ ) { + printf("disallowing unsupported code signature page shift: %u\n", cd->pageSize); + return EBADEXEC; + } +@@ -3925,6 +3933,13 @@ + static void + cs_blob_cleanup(struct cs_blob *blob) + { ++ if (blob->csb_native4k_hashes != NULL) { ++ kmem_free(kernel_map, (vm_offset_t)blob->csb_native4k_hashes, ++ blob->csb_native4k_hashes_size); ++ blob->csb_native4k_hashes = NULL; ++ blob->csb_native4k_hashes_size = 0; ++ blob->csb_native4k_hash_count = 0; ++ } + if (blob->csb_entitlements != NULL) { + amfi->OSEntitlements_invalidate(blob->csb_entitlements); + osobject_release(blob->csb_entitlements); +@@ -4858,6 +4873,14 @@ + error = EINVAL; + goto out; + } ++ ++#if defined(XNU_NATIVE_CS_HASH_CACHE) && XNU_NATIVE_CS_HASH_CACHE ++ /* Authenticate before publishing the signed object or taking its locks. */ ++ error = cs_native4k_prepare(vp, &tmp_blob); ++ if (error != 0) { ++ goto out; ++ } ++#endif + + /* + * The vnode_lock, linked list traversal, and marking of the memory object as +@@ -5726,6 +5749,121 @@ + + unsigned long cs_validate_page_no_hash = 0; + unsigned long cs_validate_page_bad_hash = 0; ++ ++#if defined(XNU_NATIVE_CS_HASH_CACHE) && XNU_NATIVE_CS_HASH_CACHE ++__attribute__((used)) const uint32_t cs_native4k_hash_cache_version = 1; ++unsigned long cs_native4k_full_blocks = 0; ++unsigned long cs_native4k_page_checks = 0; ++unsigned long cs_native4k_ready_blobs = 0; ++ ++/* The original signed 16K hash authenticates the whole immutable input ++ * buffer first. Only then derive native-page digests from that same buffer. ++ * Publish nothing until every original block has passed, and make the ++ * derived array readonly before its owning RO cs_blob can become visible. ++ * VM fault validation consequently performs no vnode I/O under object locks. ++ */ ++__attribute__((noinline)) ++static int ++cs_native4k_prepare(vnode_t vp, struct cs_blob *blob) ++{ ++ const CS_CodeDirectory *cd = blob->csb_cd; ++ const struct cs_hash *md = blob->csb_hashtype; ++ vm_offset_t storage = 0; ++ vm_size_t allocation = 0, count = 0; ++ uint8_t *buffer = NULL; ++ uint64_t limit, offset; ++ int error = EBADEXEC; ++ kern_return_t kr; ++ ++ if (blob->csb_hash_pageshift <= PAGE_SHIFT) { ++ return 0; ++ } ++ if (PAGE_SHIFT != 12 || blob->csb_hash_pageshift != 14 || cd == NULL || md == NULL || ++ md->cs_size == 0 || md->cs_size > CS_HASH_MAX_SIZE || ++ md->cs_digest_size > CS_HASH_MAX_SIZE || blob->csb_base_offset < 0 || ++ (blob->csb_base_offset & PAGE_MASK) || blob->csb_hash_firstlevel_pageshift || ++ ((ntohl(cd->version) >= CS_SUPPORTSSCATTER) && ntohl(cd->scatterOffset))) { ++ return ENOTSUP; ++ } ++ limit = ntohl(cd->codeLimit); ++ if (!limit || ((ntohl(cd->version) >= CS_SUPPORTSCODELIMIT64) && ++ ntohll(cd->codeLimit64) && ntohll(cd->codeLimit64) != limit) || ++ ntohl(cd->nCodeSlots) != ((limit + 16383) >> 14)) { ++ return ENOTSUP; ++ } ++ count = (limit + PAGE_MASK) >> PAGE_SHIFT; ++ if (os_mul_overflow(count, (vm_size_t)md->cs_size, &allocation) || ++ round_page_overflow(allocation, &allocation)) { ++ return EOVERFLOW; ++ } ++ kr = kmem_alloc(kernel_map, &storage, allocation, ++ KMA_KOBJECT | KMA_DATA | KMA_ZERO, VM_KERN_MEMORY_SECURITY); ++ if (kr != KERN_SUCCESS) { ++ return ENOMEM; ++ } ++ buffer = kalloc_data(16384, Z_WAITOK); ++ for (offset = 0; offset < limit; offset += 16384) { ++ union cs_hash_union ctx; ++ uint8_t digest[CS_HASH_MAX_SIZE]; ++ const uint8_t *expected; ++ const char *lower = blob->csb_mem_kaddr; ++ const char *upper = lower + blob->csb_mem_size; ++ size_t length = MIN((uint64_t)16384, limit - offset); ++ int resid = 0; ++ off_t position; ++ if (os_add_overflow(blob->csb_base_offset, (off_t)offset, &position)) { ++ error = EOVERFLOW; ++ goto out; ++ } ++ error = vn_rdwr(UIO_READ, vp, (caddr_t)buffer, (int)length, position, ++ UIO_SYSSPACE, 0, vfs_context_ucred(vfs_context_current()), ++ &resid, current_proc()); ++ if (error || resid) { ++ error = error ? error : EIO; ++ goto out; ++ } ++ expected = hashes(cd, (uint32_t)(offset >> 14), md->cs_size, lower, upper); ++ if (expected == NULL) { ++ error = EBADEXEC; ++ goto out; ++ } ++ md->cs_init(&ctx); ++ md->cs_update(&ctx, buffer, length); ++ md->cs_final(digest, &ctx); ++ if (bcmp(expected, digest, md->cs_size)) { ++ os_atomic_inc(&cs_validate_page_bad_hash, relaxed); ++ error = EBADEXEC; ++ goto out; ++ } ++ os_atomic_inc(&cs_native4k_full_blocks, relaxed); ++ for (size_t sub = 0; sub < length; sub += PAGE_SIZE) { ++ size_t bytes = MIN((size_t)PAGE_SIZE, length - sub); ++ vm_size_t index = (offset + sub) >> PAGE_SHIFT; ++ md->cs_init(&ctx); ++ md->cs_update(&ctx, buffer + sub, bytes); ++ md->cs_final(digest, &ctx); ++ memcpy((void *)(storage + index * md->cs_size), digest, md->cs_size); ++ } ++ } ++ kr = vm_map_protect(kernel_map, storage, storage + allocation, VM_PROT_READ, TRUE); ++ if (kr != KERN_SUCCESS) { ++ error = EPERM; ++ goto out; ++ } ++ blob->csb_native4k_hashes = (const uint8_t *)storage; ++ blob->csb_native4k_hashes_size = allocation; ++ blob->csb_native4k_hash_count = count; ++ storage = 0; ++ os_atomic_inc(&cs_native4k_ready_blobs, relaxed); ++ error = 0; ++out: ++ kfree_data(buffer, 16384); ++ if (storage) { ++ kmem_free(kernel_map, storage, allocation); ++ } ++ return error; ++} ++#endif + static boolean_t + cs_validate_hash( + struct cs_blob *blobs, +@@ -5749,6 +5887,7 @@ + off_t codeLimit = 0; + const char *lower_bound, *upper_bound; + vm_offset_t kaddr, blob_addr; ++ boolean_t native4k_cached = FALSE; + + /* retrieve the expected hash */ + found_hash = FALSE; +@@ -5784,15 +5923,31 @@ + if (hashtype->cs_digest_size > sizeof(actual_hash)) { + panic("hash size too large"); + } +- if (offset & ((1U << blob->csb_hash_pageshift) - 1)) { +- panic("offset not aligned to cshash boundary"); ++ codeLimit = ntohl(cd->codeLimit); ++#if defined(XNU_NATIVE_CS_HASH_CACHE) && XNU_NATIVE_CS_HASH_CACHE ++ if (PAGE_SHIFT == 12 && blob->csb_hash_pageshift == 14) { ++ vm_size_t index = (vm_size_t)offset >> PAGE_SHIFT; ++ if ((offset & PAGE_MASK) || offset >= codeLimit || ++ blob->csb_native4k_hashes == NULL || ++ index >= blob->csb_native4k_hash_count || ++ hashtype->cs_size > blob->csb_native4k_hashes_size || ++ index > (blob->csb_native4k_hashes_size - hashtype->cs_size) / hashtype->cs_size) { ++ *bytes_processed = PAGE_SIZE; ++ *tainted = CS_VALIDATE_TAINTED; ++ return FALSE; ++ } ++ hash = blob->csb_native4k_hashes + index * hashtype->cs_size; ++ native4k_cached = TRUE; ++ os_atomic_inc(&cs_native4k_page_checks, relaxed); ++ } else ++#endif ++ { ++ if (offset & ((1U << blob->csb_hash_pageshift) - 1)) { ++ panic("offset not aligned to cshash boundary"); ++ } ++ hash = hashes(cd, (uint32_t)(offset >> blob->csb_hash_pageshift), ++ hashtype->cs_size, lower_bound, upper_bound); + } +- +- codeLimit = ntohl(cd->codeLimit); +- +- hash = hashes(cd, (uint32_t)(offset >> blob->csb_hash_pageshift), +- hashtype->cs_size, +- lower_bound, upper_bound); + if (hash != NULL) { + bcopy(hash, expected_hash, hashtype->cs_size); + found_hash = TRUE; +@@ -5822,7 +5977,7 @@ + } else { + *tainted = 0; + +- size = (1U << blob->csb_hash_pageshift); ++ size = native4k_cached ? PAGE_SIZE : (1U << blob->csb_hash_pageshift); + *bytes_processed = size; + + const uint32_t *asha1, *esha1; diff --git a/patches/xnu/native-4k-user-abi.patch b/patches/xnu/native-4k-user-abi.patch new file mode 100644 index 0000000..2d66c48 --- /dev/null +++ b/patches/xnu/native-4k-user-abi.patch @@ -0,0 +1,85 @@ +diff --git a/bsd/kern/mach_loader.c b/bsd/kern/mach_loader.c +index 4435c16..665536f 100644 +--- a/bsd/kern/mach_loader.c ++++ b/bsd/kern/mach_loader.c +@@ -511,8 +511,12 @@ load_machfile( + + #if defined(__arm64__) + if (result->is_64bit_addr) { ++#if !defined(APPLE_ARM64_ARCH_FAMILY) && !__ARM_16K_PG__ ++ vm_map_set_page_shift(map, FOURK_PAGE_SHIFT); ++#else + /* enforce 16KB alignment of VM map entries */ + vm_map_set_page_shift(map, SIXTEENK_PAGE_SHIFT); ++#endif + } else { + vm_map_set_page_shift(map, page_shift_user32); + } +diff --git a/osfmk/arm/arm_init.c b/osfmk/arm/arm_init.c +index 8fe43b6..a81d6c4 100644 +--- a/osfmk/arm/arm_init.c ++++ b/osfmk/arm/arm_init.c +@@ -399,6 +399,10 @@ arm_init( + + + { ++ /* Native non-Apple 4K builds expose the hardware page size at any RAM size. */ ++#if !defined(APPLE_ARM64_ARCH_FAMILY) && !__ARM_16K_PG__ ++ PAGE_SHIFT_CONST = ARM_PGSHIFT; ++#else + /* + * Select the advertised kernel page size. + */ +@@ -417,8 +421,14 @@ arm_init( + PAGE_SHIFT_CONST = ARM_PGSHIFT; + } + ++#endif ++ ++#if !defined(APPLE_ARM64_ARCH_FAMILY) && !__ARM_16K_PG__ ++ page_shift_user32 = ARM_PGSHIFT; ++#else + /* 32-bit apps always see 16KB page size */ + page_shift_user32 = PAGE_MAX_SHIFT; ++#endif + #ifdef APPLETYPHOON + if (PE_parse_boot_argn("cpus_defeatures", &cpus_defeatures, sizeof(cpus_defeatures))) { + if ((cpus_defeatures & 0xF) != 0) { +diff --git a/osfmk/arm/commpage/commpage.c b/osfmk/arm/commpage/commpage.c +index 896a7c8..315ecea 100644 +--- a/osfmk/arm/commpage/commpage.c ++++ b/osfmk/arm/commpage/commpage.c +@@ -185,8 +185,13 @@ commpage_populate(void) + #if __arm64__ + *((uint8_t*)(_COMM_PAGE_USER_PAGE_SHIFT_32_LEGACY + _COMM_PAGE_RW_OFFSET)) = (uint8_t) page_shift_user32; + *((uint8_t*)(_COMM_PAGE_USER_PAGE_SHIFT_32 + _COMM_PAGE_RO_OFFSET)) = (uint8_t) page_shift_user32; ++#if !defined(APPLE_ARM64_ARCH_FAMILY) && !__ARM_16K_PG__ ++ *((uint8_t*)(_COMM_PAGE_USER_PAGE_SHIFT_64_LEGACY + _COMM_PAGE_RW_OFFSET)) = (uint8_t) FOURK_PAGE_SHIFT; ++ *((uint8_t*)(_COMM_PAGE_USER_PAGE_SHIFT_64 + _COMM_PAGE_RO_OFFSET)) = (uint8_t) FOURK_PAGE_SHIFT; ++#else + *((uint8_t*)(_COMM_PAGE_USER_PAGE_SHIFT_64_LEGACY + _COMM_PAGE_RW_OFFSET)) = (uint8_t) SIXTEENK_PAGE_SHIFT; + *((uint8_t*)(_COMM_PAGE_USER_PAGE_SHIFT_64 + _COMM_PAGE_RO_OFFSET)) = (uint8_t) SIXTEENK_PAGE_SHIFT; ++#endif + #endif /* __arm64__ */ + + commpage_update_timebase(); +diff --git a/osfmk/vm/vm_shared_region.c b/osfmk/vm/vm_shared_region.c +index 0bb45c5..1ab7232 100644 +--- a/osfmk/vm/vm_shared_region.c ++++ b/osfmk/vm/vm_shared_region.c +@@ -778,11 +778,15 @@ vm_shared_region_create( + sub_map = vm_map_create_options(pmap_nested, 0, + (vm_map_offset_t)size, VM_MAP_CREATE_PAGEABLE); + ++#if !defined(APPLE_ARM64_ARCH_FAMILY) && !__ARM_16K_PG__ ++ vm_map_set_page_shift(sub_map, FOURK_PAGE_SHIFT); ++#else + if (is_64bit || + page_shift_user32 == SIXTEENK_PAGE_SHIFT) { + /* enforce 16KB alignment of VM map entries */ + vm_map_set_page_shift(sub_map, SIXTEENK_PAGE_SHIFT); + } ++#endif + #if __ARM_MIXED_PAGE_SIZE__ + if (cputype == CPU_TYPE_ARM64 && + target_page_shift == FOURK_PAGE_SHIFT) { diff --git a/patches/xnu/native-aes-personality.patch b/patches/xnu/native-aes-personality.patch new file mode 100644 index 0000000..b6c3737 --- /dev/null +++ b/patches/xnu/native-aes-personality.patch @@ -0,0 +1,52 @@ +diff --git a/iokit/KernelConfigTables.cpp b/iokit/KernelConfigTables.cpp +--- a/iokit/KernelConfigTables.cpp ++++ b/iokit/KernelConfigTables.cpp +@@ -61,4 +61,48 @@ + " 'IOProviderClass' = IOService;" + " 'IOExclaveProxy' = 1:32;" + " }" ++#if defined(XNU_NATIVE_AES) && XNU_NATIVE_AES ++ " ,{" ++ " 'IOClass' = NativeAES;" ++ " 'IOProviderClass' = IOResources;" ++ " 'IOResourceMatch' = IOBSD;" ++ " 'IOProbeScore' = 1000:32;" ++ " }" ++#endif ++#if defined(XNU_NATIVE_VIRTIO_BLOCK) && XNU_NATIVE_VIRTIO_BLOCK ++ " ,{" ++ " 'IOClass' = NativeVirtioBlock;" ++ " 'IOProviderClass' = IOPlatformDevice;" ++ " 'IOProbeScore' = 1000:32;" ++ " }" ++#endif ++#if defined(XNU_NATIVE_VIRTIO_CONSOLE) && XNU_NATIVE_VIRTIO_CONSOLE ++ " ,{" ++ " 'IOClass' = NativeVirtioConsole;" ++ " 'IOProviderClass' = IOPlatformDevice;" ++ " 'IOProbeScore' = 1000:32;" ++ " }" ++#endif ++#if defined(XNU_NATIVE_RAM_BLOCK) && XNU_NATIVE_RAM_BLOCK ++ " ,{" ++ " 'IOClass' = NativeRamBlock;" ++ " 'IOMatchCategory' = NativeRamBlock;" ++ " 'IOProviderClass' = IOResources;" ++ " 'IOResourceMatch' = IOBSD;" ++ " 'IOProbeScore' = 1000:32;" ++ " }" ++#endif ++#if defined(XNU_NATIVE_UFS) && XNU_NATIVE_UFS ++ " ,{" ++ " 'IOClass' = NativeUfs;" ++ " 'IOProviderClass' = IOPlatformDevice;" ++ " 'IOProbeScore' = 1000:32;" ++ " }" ++ " ,{" ++ " 'IOClass' = NativeUfs;" ++ " 'IOMatchCategory' = NativeUfsRoot;" ++ " 'IOProviderClass' = IOPlatformExpertDevice;" ++ " 'IOProbeScore' = 500:32;" ++ " }" ++#endif + ")"; diff --git a/patches/xnu/native-arm-pan.patch b/patches/xnu/native-arm-pan.patch new file mode 100644 index 0000000..cda8457 --- /dev/null +++ b/patches/xnu/native-arm-pan.patch @@ -0,0 +1,14 @@ +--- a/pexpert/pexpert/arm64/BCM2837.h ++++ b/pexpert/pexpert/arm64/BCM2837.h +@@ -18,6 +18,11 @@ + #define __ARM_COHERENT_CACHE__ 1 + #define __ARM_DEBUG__ 7 + #define __ARM64_PMAP_SUBPAGE_L1__ 1 ++ ++/* Explicit native ARMv8.1+ bring-up; the original A53 target has no PAN. */ ++#if defined(XNU_NATIVE_PAN) && XNU_NATIVE_PAN ++#define __ARM_PAN_AVAILABLE__ 1 ++#endif + + #ifndef ASSEMBLER + diff --git a/patches/xnu/native-dt-panic-region.patch b/patches/xnu/native-dt-panic-region.patch new file mode 100644 index 0000000..7ba1a04 --- /dev/null +++ b/patches/xnu/native-dt-panic-region.patch @@ -0,0 +1,42 @@ +--- a/pexpert/arm/pe_init.c ++++ b/pexpert/arm/pe_init.c +@@ -102,7 +102,7 @@ + static void + check_for_panic_log(void) + { +-#ifdef PLATFORM_PANIC_LOG_PADDR ++#if defined(PLATFORM_PANIC_LOG_PADDR) && !(defined(XNU_DT_PANIC_REGION) && XNU_DT_PANIC_REGION) + gPanicBase = ml_io_map_wcomb(PLATFORM_PANIC_LOG_PADDR, PLATFORM_PANIC_LOG_SIZE); + panic_text_len = PLATFORM_PANIC_LOG_SIZE - sizeof(struct embedded_panic_header); + gPanicSize = PLATFORM_PANIC_LOG_SIZE; +@@ -132,7 +132,10 @@ + return; + } + +- if (kSuccess != SecureDTGetProperty(entry, "reg", (void const **)®_prop, &size)) { ++ if (kSuccess != SecureDTGetProperty(entry, "reg", (void const **)®_prop, &size) || ++ size != 2 * sizeof(*reg_prop) || !reg_prop[0] || ++ reg_prop[1] < sizeof(struct embedded_panic_header) || ++ reg_prop[0] > UINTPTR_MAX - reg_prop[1]) { + return; + } + +@@ -140,11 +143,17 @@ + return; + } + +- if (kSuccess != SecureDTGetProperty(chosen, "embedded-panic-log-size", (void const **) &panic_region_length, &size)) { ++ if (kSuccess != SecureDTGetProperty(chosen, "embedded-panic-log-size", (void const **) &panic_region_length, &size) || ++ size != sizeof(*panic_region_length) || ++ panic_region_length[0] < sizeof(struct embedded_panic_header) || ++ panic_region_length[0] > reg_prop[1]) { + return; + } + + gPanicBase = ml_io_map_wcomb(reg_prop[0], panic_region_length[0]); ++ if (!gPanicBase) { ++ return; ++ } + + /* Deduct the size of the panic header from the panic region size */ + panic_text_len = panic_region_length[0] - sizeof(struct embedded_panic_header); diff --git a/patches/xnu/native-priv-work-interval.patch b/patches/xnu/native-priv-work-interval.patch new file mode 100644 index 0000000..c1feffa --- /dev/null +++ b/patches/xnu/native-priv-work-interval.patch @@ -0,0 +1,21 @@ +diff --git a/bsd/kern/kern_priv.c b/bsd/kern/kern_priv.c +index 88adf2d..05e7679 100644 +--- a/bsd/kern/kern_priv.c ++++ b/bsd/kern/kern_priv.c +@@ -82,6 +82,16 @@ priv_check_cred(kauth_cred_t cred, int priv, int flags) + #endif + int error; + ++ /* ++ * CoreAnimation creates its render-server work interval as an ordinary user, and ++ * UIKit processes (SpringBoard, apps) abort with "Failed to create work interval" ++ * without it. This board loads no Sandbox/AMFI privilege policy that would grant it, ++ * and the policy set that is present denies it, so it is decided before the policies. ++ */ ++ if (priv == PRIV_WORK_INTERVAL) { ++ return 0; ++ } ++ + /* + * We first evaluate policies that may deny the granting of + * privilege unilaterally. diff --git a/patches/xnu/native-smp-secondary.patch b/patches/xnu/native-smp-secondary.patch new file mode 100644 index 0000000..a757f29 --- /dev/null +++ b/patches/xnu/native-smp-secondary.patch @@ -0,0 +1,24 @@ +--- a/pexpert/pexpert/arm64/board_config.h ++++ b/pexpert/pexpert/arm64/board_config.h +@@ -134,7 +134,7 @@ + #include + + #define MAX_L2_CLINE 6 +-#define MAX_CPUS 4 ++#define MAX_CPUS 8 /* sheng: SM8550 has eight cores */ + #define MAX_CPU_CLUSTERS 1 + + #define CORE_NCTRS 8 /* Placeholder; KPC is not enabled for this target */ +--- a/osfmk/arm64/start.s ++++ b/osfmk/arm64/start.s +@@ -145,8 +145,8 @@ + adrp x19, EXT(ResetHandlerData)@page // Get address of the reset handler data + add x19, x19, EXT(ResetHandlerData)@pageoff + mrs x15, MPIDR_EL1 // Load MPIDR to get CPU number +-#if HAS_CLUSTER +- and x0, x15, #0xFFFF // CPU number in Affinity0, cluster ID in Affinity1 ++#if HAS_CLUSTER || defined(BCM2837) ++ and x0, x15, #0xFFFF // CPU number in Affinity0, cluster ID in Affinity1 (SM8550 numbers cores in Affinity1) + #else + and x0, x15, #0xFF // CPU number is in MPIDR Affinity Level 0 + #endif diff --git a/patches/xnu/native-trust-cache-runtime.patch b/patches/xnu/native-trust-cache-runtime.patch new file mode 100644 index 0000000..1d088a2 --- /dev/null +++ b/patches/xnu/native-trust-cache-runtime.patch @@ -0,0 +1,11 @@ +--- a/osfmk/kern/startup.c ++++ b/osfmk/kern/startup.c +@@ -751,7 +751,7 @@ + csm_initialize_provisioning_profiles(); + #endif + +-#ifndef BCM2837 ++#if !defined(BCM2837) || (defined(XNU_NATIVE_TRUST_CACHE) && XNU_NATIVE_TRUST_CACHE) + kernel_bootstrap_log("trust_cache_init"); + + /* Initialize the runtime for the trust cache interface */ diff --git a/patches/xnu/native-union-getattrlistbulk.patch b/patches/xnu/native-union-getattrlistbulk.patch new file mode 100644 index 0000000..64f3c19 --- /dev/null +++ b/patches/xnu/native-union-getattrlistbulk.patch @@ -0,0 +1,65 @@ +--- a/bsd/vfs/vfs_attrlist.c ++++ b/bsd/vfs/vfs_attrlist.c +@@ -4191,6 +4191,9 @@ + + FV_LOCK(fvdata); + ++#if CONFIG_UNION_MOUNTS ++unionread: ++#endif + /* + * getattrlistbulk(2) maintains its offset in fv_offset. However + * if the offset in the file glob is set (or reset) to 0, the directory +@@ -4282,6 +4285,52 @@ + &count, &eofflag, ctx); + ut->uu_flag &= ~UT_KERN_RAGE_VNODES; + } ++ ++#if CONFIG_UNION_MOUNTS ++ /* Match getdirentriesattr: consume the upper directory before moving ++ * this open descriptor to the corresponding lower union directory. ++ * Snapshot enumeration must stay on the requested snapshot directory. ++ * Recheck lower-directory access before publishing the new vnode. ++ */ ++ if (!error && !count && eofflag && ++ !(uap->options & FSOPT_LIST_SNAPSHOT) && ++ (dvp->v_mount->mnt_flag & MNT_UNION)) { ++ vnode_t uvp; ++ int union_error = lookup_traverse_union(dvp, &uvp, ctx); ++ if (!union_error) { ++ if (uvp->v_type != VDIR) { ++ union_error = ENOTDIR; ++ } ++#if CONFIG_MACF ++ if (!union_error) { ++ union_error = mac_vnode_check_readdir(ctx, uvp); ++ } ++#endif ++ if (!union_error) { ++ union_error = vnode_authorize(uvp, NULL, action, ctx); ++ } ++ if (!union_error) { ++ union_error = vnode_ref_ext(uvp, fp->fp_glob->fg_flag & O_EVTONLY, 0); ++ } ++ if (!union_error) { ++ fp_set_data(fp, uvp); ++ fp->fp_glob->fg_offset = 0; ++ vnode_rele_internal(dvp, fp->fp_glob->fg_flag & O_EVTONLY, 0, 0); ++ vnode_put(dvp); ++ dvp = uvp; ++ /* The existing zero-offset path discards upper EOF and ++ * buffered dirents before constructing the lower UIO. */ ++ goto unionread; ++ } ++ vnode_put(uvp); ++ } ++ /* No corresponding lower directory is ordinary end-of-directory; ++ * other failures must not silently hide lower entries. */ ++ if (union_error != ENOENT) { ++ error = union_error; ++ } ++ } ++#endif + + if (count) { + fvdata->fv_offset = uio_offset(auio); diff --git a/patches/xnu/native-work-interval-soft-fail.patch b/patches/xnu/native-work-interval-soft-fail.patch new file mode 100644 index 0000000..ba28cf8 --- /dev/null +++ b/patches/xnu/native-work-interval-soft-fail.patch @@ -0,0 +1,23 @@ +diff --git a/osfmk/kern/work_interval.c b/osfmk/kern/work_interval.c +index 96e985d..46ef2b3 100644 +--- a/osfmk/kern/work_interval.c ++++ b/osfmk/kern/work_interval.c +@@ -1115,7 +1115,7 @@ kern_work_interval_create(thread_t thread, + return KERN_FAILURE; + } + if (!task_is_app(creating_task)) { +-#if XNU_TARGET_OS_OSX ++#if XNU_TARGET_OS_OSX || 1 /* sheng: UIKit aborts on a refused CA_CLIENT interval; soft-fail it as macOS does */ + /* + * Soft-fail the case of a non-app pretending to be an + * app, by allowing it to press the buttons, but they're +@@ -1131,7 +1131,8 @@ kern_work_interval_create(thread_t thread, + #endif /* XNU_TARGET_OS_OSX */ + } + if (task_set_ca_client_wi(creating_task, true) == false) { +- return KERN_FAILURE; ++ /* sheng: a second CA_CLIENT interval in a task is accepted but not connected, instead of aborting the app */ ++ create_flags |= WORK_INTERVAL_FLAG_IGNORED; + } + } + diff --git a/patches/xnu/native-work-interval-trace.patch b/patches/xnu/native-work-interval-trace.patch new file mode 100644 index 0000000..f855f6e --- /dev/null +++ b/patches/xnu/native-work-interval-trace.patch @@ -0,0 +1,43 @@ +diff --git a/bsd/kern/sys_work_interval.c b/bsd/kern/sys_work_interval.c +index b544bf3..662bb00 100644 +--- a/bsd/kern/sys_work_interval.c ++++ b/bsd/kern/sys_work_interval.c +@@ -38,6 +38,22 @@ + + #include + ++#include ++#include ++ ++/* Board bring-up diagnostic: the last work-interval CREATE2 results, read by the boot-progress banner. */ ++char native_wi_log[4][100]; ++volatile unsigned native_wi_count; ++static void ++native_wi_note(const char *what, int error, int kret, uint32_t flags) ++{ ++ char name[20]; ++ proc_selfname(name, sizeof(name)); ++ unsigned slot = native_wi_count++ % 4; ++ snprintf(native_wi_log[slot], sizeof(native_wi_log[slot]), "WI#%u p%d %s %s err=%d kret=%d flags=%x app=%d", ++ native_wi_count, proc_selfpid(), name, what, error, kret, flags, (int)task_is_app(current_task())); ++} ++ + int + work_interval_ctl(__unused proc_t p, struct work_interval_ctl_args *uap, + __unused int32_t *retval) +@@ -69,6 +85,7 @@ work_interval_ctl(__unused proc_t p, struct work_interval_ctl_args *uap, + } + + if ((error = priv_check_cred(kauth_cred_get(), PRIV_WORK_INTERVAL, 0)) != 0) { ++ native_wi_note("priv", error, 0, create_params.wicp_create_flags); + return error; + } + +@@ -79,6 +96,7 @@ work_interval_ctl(__unused proc_t p, struct work_interval_ctl_args *uap, + }; + + kret = kern_work_interval_create(current_thread(), &create_args); ++ native_wi_note("create", 0, kret, create_params.wicp_create_flags); + + /* thread already has a work interval */ + if (kret == KERN_FAILURE) { diff --git a/patches/xnu/non-apple-cache-monitor.patch b/patches/xnu/non-apple-cache-monitor.patch new file mode 100644 index 0000000..781f1d0 --- /dev/null +++ b/patches/xnu/non-apple-cache-monitor.patch @@ -0,0 +1,29 @@ +diff --git a/osfmk/arm64/arm_vm_init.c b/osfmk/arm64/arm_vm_init.c +index 985370e..a688021 100644 +--- a/osfmk/arm64/arm_vm_init.c ++++ b/osfmk/arm64/arm_vm_init.c +@@ -2150,7 +2150,9 @@ arm_vm_init(uint64_t memory_size, boot_args * args) + arm_vm_physmap_init(args); + set_mmu_ttb_alternate(cpu_ttep & TTBR_BADDR_MASK); + ++#if XNU_MONITOR + ml_enable_monitor(); ++#endif + + set_mmu_ttb(invalid_ttep & TTBR_BADDR_MASK); + +diff --git a/osfmk/arm64/caches_asm.s b/osfmk/arm64/caches_asm.s +index f2c2a00..385f3ab 100644 +--- a/osfmk/arm64/caches_asm.s ++++ b/osfmk/arm64/caches_asm.s +@@ -276,6 +276,10 @@ L_cpudr_loop: + */ + .text + .align 2 ++#if !defined(APPLE_ARM64_ARCH_FAMILY) ++ .globl EXT(CleanPoC_DcacheRegion_Force_nopreempt_nohid) ++LEXT(CleanPoC_DcacheRegion_Force_nopreempt_nohid) ++#endif + LEXT(CleanPoC_DcacheRegion_internal) + #define CLINE_FLUSH_STRIDE MMU_CLINE + diff --git a/patches/xnu/non-apple-iommu-bootstrap.patch b/patches/xnu/non-apple-iommu-bootstrap.patch new file mode 100644 index 0000000..8bac3ee --- /dev/null +++ b/patches/xnu/non-apple-iommu-bootstrap.patch @@ -0,0 +1,79 @@ +diff --git a/osfmk/arm/pmap/pmap.c b/osfmk/arm/pmap/pmap.c +index 7db00b6..9f9d7bf 100644 +--- a/osfmk/arm/pmap/pmap.c ++++ b/osfmk/arm/pmap/pmap.c +@@ -84,8 +84,12 @@ + + #include + #include ++#if XNU_MONITOR_ANS2_SART + #include ++#endif ++#if XNU_MONITOR_UAT_PPL + #include ++#endif + + #if defined(KERNEL_INTEGRITY_KTRR) || defined(KERNEL_INTEGRITY_CTRR) + #include +@@ -883,6 +887,8 @@ PMAP_SUPPORT_PROTOTYPES( + vm_prot_t allow_mode, + int options), ARM_FORCE_FAST_FAULT_INDEX); + ++MARK_AS_PMAP_TEXT static void pmap_phys_write_disable(vm_address_t va); ++ + MARK_AS_PMAP_TEXT static boolean_t + arm_force_fast_fault_with_flush_range( + ppnum_t ppnum, +@@ -2267,13 +2273,17 @@ pmap_bootstrap( + /** + * Bootstrap any necessary UAT data structures and values needed from the device tree. + */ ++#if XNU_MONITOR_UAT_PPL + uat_bootstrap(); ++#endif + + + /** + * Bootstrap any necessary SART data structures and values needed from the device tree. + */ ++#if XNU_MONITOR_ANS2_SART + sart_bootstrap(); ++#endif + + /** + * Don't make any assumptions about the alignment of avail_start before this +@@ -4809,13 +4819,13 @@ pmap_page_protect_options_with_flush_range( + if (pvh_ptep_is_iommu(pte_p)) { + #if XNU_MONITOR + if (__improbable(pvh_flags & PVH_FLAG_LOCKDOWN_MASK)) { +- panic("pmap_page_protect: ppnum 0x%x locked down, cannot be owned by iommu %p, pve_p=%p", +- ppnum, ptep_get_iommu(pte_p), pve_p); ++ panic("pmap_page_protect: ppnum 0x%x locked down, cannot be owned by iommu ptep %p, pve_p=%p", ++ ppnum, pte_p, pve_p); + } + #endif + if (remove && (options & PMAP_OPTIONS_COMPRESSOR)) { +- panic("pmap_page_protect: attempt to compress ppnum 0x%x owned by iommu %p, pve_p=%p", +- ppnum, ptep_get_iommu(pte_p), pve_p); ++ panic("pmap_page_protect: attempt to compress ppnum 0x%x owned by iommu ptep %p, pve_p=%p", ++ ppnum, pte_p, pve_p); + } + goto protect_skip_pve_pass1; + } +@@ -12026,6 +12036,7 @@ pmap_unpin_kernel_pages(vm_offset_t kva __unused, size_t nbytes __unused) + #endif /* !XNU_MONITOR */ + + ++#if PMAP_CS_PPL_MONITOR || PMAP_CS_INCLUDE_CODE_SIGNING + MARK_AS_PMAP_TEXT static inline void + pmap_cs_lockdown_pages(vm_address_t kva, vm_size_t size, bool ppl_writable) + { +@@ -12046,6 +12057,8 @@ pmap_cs_unlockdown_pages(vm_address_t kva, vm_size_t size, bool ppl_writable) + #endif + } + ++#endif /* PMAP_CS_PPL_MONITOR || PMAP_CS_INCLUDE_CODE_SIGNING */ ++ + /** + * Perform basic validation checks on the destination only and + * corresponding offset/sizes prior to writing to a read only allocation. diff --git a/patches/xnu/non-apple-pmap-field.patch b/patches/xnu/non-apple-pmap-field.patch new file mode 100644 index 0000000..2fb23fb --- /dev/null +++ b/patches/xnu/non-apple-pmap-field.patch @@ -0,0 +1,12 @@ +diff --git a/osfmk/arm/pmap/pmap.h b/osfmk/arm/pmap/pmap.h +--- a/osfmk/arm/pmap/pmap.h ++++ b/osfmk/arm/pmap/pmap.h +@@ -473,7 +473,7 @@ + #if HAS_APPLE_PAC + bool disable_jop; + #else +- bool reserved10; ++ bool reserved13; + #endif /* HAS_APPLE_PAC */ + + bool reserved11; diff --git a/patches/xnu/null-root-media.patch b/patches/xnu/null-root-media.patch new file mode 100644 index 0000000..0af8163 --- /dev/null +++ b/patches/xnu/null-root-media.patch @@ -0,0 +1,27 @@ +diff --git a/iokit/Kernel/IOService.cpp b/iokit/Kernel/IOService.cpp +index 4903e7d..c22edff 100644 +--- a/iokit/Kernel/IOService.cpp ++++ b/iokit/Kernel/IOService.cpp +@@ -5257,14 +5257,16 @@ IOService::setRootMedia(IOService * root) + const OSMetaClass * ioblockstoragedriverClass; + bool unhide; + +- ioblockstoragedriverClass = OSMetaClass::getMetaClassWithName(gIOBlockStorageDriverKey); +- assert(ioblockstoragedriverClass); ++ if (root) { ++ ioblockstoragedriverClass = OSMetaClass::getMetaClassWithName(gIOBlockStorageDriverKey); ++ assert(ioblockstoragedriverClass); + +- while (root) { +- if (root->metaCast(ioblockstoragedriverClass)) { +- break; ++ while (root) { ++ if (root->metaCast(ioblockstoragedriverClass)) { ++ break; ++ } ++ root = root->getProvider(); + } +- root = root->getProvider(); + } + + LOCKWRITENOTIFY(); diff --git a/patches/xnu/offline-component-load-probes.patch b/patches/xnu/offline-component-load-probes.patch new file mode 100644 index 0000000..94bf523 --- /dev/null +++ b/patches/xnu/offline-component-load-probes.patch @@ -0,0 +1,188 @@ +--- a/bsd/kern/bsd_init.c ++++ b/bsd/kern/bsd_init.c +@@ -478,5 +478,88 @@ + * of the tro_ucred field so that the uthread structure can be + * used like any other. + */ ++#if DEVELOPMENT || DEBUG ++extern kern_return_t OSKextLoadKextWithIdentifier(const char *identifier); ++volatile int native_apfs_load_status = -1; ++volatile uint32_t native_apfs_registered = 0; ++volatile uintptr_t native_apfs_vfsops = 0; ++ ++__attribute__((noinline)) void ++native_apfs_probe_checkpoint(void) ++{ ++ __asm__ volatile("" ::: "memory"); ++} ++ ++static void ++native_apfs_load_probe(void) ++{ ++ int enabled = 0; ++ if (!PE_parse_boot_argn("offline-apfs-load", &enabled, sizeof(enabled)) || !enabled) { ++ return; ++ } ++ native_apfs_load_status = OSKextLoadKextWithIdentifier("com.apple.filesystems.apfs"); ++ mount_list_lock(); ++ for (struct vfstable *entry = vfsconf; entry; entry = entry->vfc_next) { ++ if (!strncmp(entry->vfc_name, "apfs", MFSNAMELEN)) { ++ native_apfs_registered = 1; ++ native_apfs_vfsops = (uintptr_t)entry->vfc_vfsops; ++ break; ++ } ++ } ++ mount_list_unlock(); ++ native_apfs_probe_checkpoint(); ++} ++ ++volatile int native_iogpu_load_status = -1; ++ ++__attribute__((noinline)) void ++native_iogpu_probe_checkpoint(void) ++{ ++ __asm__ volatile("" ::: "memory"); ++} ++ ++static void ++native_iogpu_load_probe(void) ++{ ++ int enabled = 0; ++ if (!PE_parse_boot_argn("offline-iogpu-load", &enabled, sizeof(enabled)) || !enabled) { ++ return; ++ } ++ native_iogpu_load_status = OSKextLoadKextWithIdentifier("com.apple.iokit.IOGPUFamily"); ++ native_iogpu_probe_checkpoint(); ++} ++ ++volatile int native_mobile_graphics_load_status = -1; ++ ++__attribute__((noinline)) void ++native_mobile_graphics_probe_checkpoint(void) ++{ ++ __asm__ volatile("" ::: "memory"); ++} ++ ++static void ++native_mobile_graphics_load_probe(void) ++{ ++ int enabled = 0; ++ if (!PE_parse_boot_argn("offline-mobile-graphics-load", &enabled, sizeof(enabled)) || !enabled) { ++ return; ++ } ++ native_mobile_graphics_load_status = OSKextLoadKextWithIdentifier("com.apple.iokit.IOMobileGraphicsFamily"); ++ native_mobile_graphics_probe_checkpoint(); ++ int legacy_register = 0; ++ if (native_mobile_graphics_load_status == 0 && ++ PE_parse_boot_argn("offline-legacy-register", &legacy_register, sizeof(legacy_register)) && legacy_register) { ++ extern void native_legacy_display_probe(void); ++ native_legacy_display_probe(); ++ } ++ int client_probe = 0; ++ if (native_mobile_graphics_load_status == 0 && ++ PE_parse_boot_argn("offline-mobile-graphics-client", &client_probe, sizeof(client_probe)) && client_probe) { ++ extern void native_mobile_graphics_client_probe(void); ++ native_mobile_graphics_client_probe(); ++ } ++} ++#endif ++ + void + bsd_init(void) +@@ -698,6 +774,27 @@ + /* Initialize the file systems. */ + bsd_init_kprintf("calling vfsinit\n"); + vfsinit(); ++#if DEVELOPMENT || DEBUG ++ native_apfs_load_probe(); ++ native_mobile_graphics_load_probe(); ++ native_iogpu_load_probe(); ++#if defined(XNU_ADRENO_FIRMWARE_PROBE) && XNU_ADRENO_FIRMWARE_PROBE ++ extern void native_adreno_firmware_probe(void); ++ native_adreno_firmware_probe(); ++#endif ++#if defined(XNU_ADRENO_PM4_PROBE) && XNU_ADRENO_PM4_PROBE ++ extern void native_adreno_pm4_probe(void); ++ native_adreno_pm4_probe(); ++#endif ++#if defined(XNU_SMMU4K_PROBE) && XNU_SMMU4K_PROBE ++ extern void native_smmu4k_probe(void); ++ native_smmu4k_probe(); ++#endif ++#if defined(XNU_CONSTANT_UPLOADS_PROBE) && XNU_CONSTANT_UPLOADS_PROBE ++ extern void native_constants_probe(void); ++ native_constants_probe(); ++#endif ++#endif + + #if CONFIG_PROC_UUID_POLICY + /* Initial proc_uuid_policy subsystem */ +--- a/iokit/Kernel/IOUserClient.cpp ++++ b/iokit/Kernel/IOUserClient.cpp +@@ -6693,3 +6693,66 @@ + OSMetaClassDefineReservedUnused(IOUserClient2022, 1); + OSMetaClassDefineReservedUnused(IOUserClient2022, 2); + OSMetaClassDefineReservedUnused(IOUserClient2022, 3); ++ ++#if DEVELOPMENT || DEBUG ++extern "C" { ++volatile uint32_t native_mobile_graphics_abstract_null_mask; ++volatile uint64_t native_mobile_graphics_client_object; ++volatile uint64_t native_mobile_graphics_client_vptr; ++volatile uint64_t native_mobile_graphics_client_meta; ++volatile uint32_t native_mobile_graphics_client_size; ++volatile int native_mobile_graphics_client_init = -1; ++volatile uint32_t native_mobile_graphics_client_results[3] = {~0U, ~0U, ~0U}; ++volatile int native_mobile_graphics_client_released; ++ ++__attribute__((noinline)) void ++native_mobile_graphics_client_checkpoint(void) ++{ ++ __asm__ volatile("" ::: "memory"); ++} ++ ++void ++native_mobile_graphics_client_probe(void) ++{ ++ const char *abstract_names[] = {"IOMobileFramebuffer", "IOMobileFramebufferService", "IOMobileFramebufferLegacy"}; ++ for (unsigned i = 0; i < 3; i++) { ++ OSObject *object = OSMetaClass::allocClassWithName(abstract_names[i]); ++ if (!object) { ++ native_mobile_graphics_abstract_null_mask |= 1U << i; ++ } else { ++ object->release(); ++ } ++ } ++ OSObject *object = OSMetaClass::allocClassWithName("IOMobileFramebufferUserClient"); ++ IOUserClient *client = OSDynamicCast(IOUserClient, object); ++ if (client) { ++ native_mobile_graphics_client_object = (uintptr_t)client; ++ native_mobile_graphics_client_vptr = *(const uintptr_t *)client; ++ native_mobile_graphics_client_meta = (uintptr_t)client->getMetaClass(); ++ native_mobile_graphics_client_size = client->getMetaClass()->getClassSize(); ++ native_mobile_graphics_client_init = client->init(); ++ if (native_mobile_graphics_client_init) { ++ IOExternalMethodArguments arguments = {}; ++ const uint32_t selectors[] = {90, 0x80000000U, ~0U}; ++ for (unsigned i = 0; i < 3; i++) { ++ arguments.selector = selectors[i]; ++ native_mobile_graphics_client_results[i] = client->externalMethod(selectors[i], &arguments, NULL, NULL, NULL); ++ } ++ } ++ } ++ if (object) { ++ object->release(); ++ native_mobile_graphics_client_released = 1; ++ } ++ native_mobile_graphics_client_checkpoint(); ++#if defined(XNU_NATIVE_MOBILE_DISPLAY) && XNU_NATIVE_MOBILE_DISPLAY ++ extern void native_mobile_display_provider_probe(void); ++ native_mobile_display_provider_probe(); ++#endif ++#if defined(XNU_NATIVE_LEGACY_DISPLAY) && XNU_NATIVE_LEGACY_DISPLAY ++ extern void native_legacy_display_probe(void); ++ native_legacy_display_probe(); ++#endif ++} ++} ++#endif diff --git a/patches/xnu/offline-legacy-start.patch b/patches/xnu/offline-legacy-start.patch new file mode 100644 index 0000000..55d0d98 --- /dev/null +++ b/patches/xnu/offline-legacy-start.patch @@ -0,0 +1,19 @@ +--- a/bsd/kern/bsd_init.c ++++ b/bsd/kern/bsd_init.c +@@ -1282,7 +1282,15 @@ bsd_autoconf(void) + } + } + +- return IOKitBSDInit(); ++ kern_return_t result = IOKitBSDInit(); ++#if (DEVELOPMENT || DEBUG) && defined(XNU_NATIVE_LEGACY_DISPLAY) && XNU_NATIVE_LEGACY_DISPLAY ++ /* The original IOSurface personality requires the genuine IOBSD resource. ++ * The default-off framebuffer instrument may start only after publication. ++ */ ++ extern void native_legacy_display_late_probe(void); ++ native_legacy_display_late_probe(); ++#endif ++ return result; + } + + diff --git a/patches/xnu/offline-native-gpu-broker.patch b/patches/xnu/offline-native-gpu-broker.patch new file mode 100644 index 0000000..1c991eb --- /dev/null +++ b/patches/xnu/offline-native-gpu-broker.patch @@ -0,0 +1,13 @@ +--- a/bsd/kern/bsd_init.c ++++ b/bsd/kern/bsd_init.c +@@ -912,6 +912,10 @@ + #if CONFIG_DTRACE + dtrace_postinit(); + #endif ++#if (DEVELOPMENT || DEBUG) && defined(XNU_NATIVE_OFFLINE_GPU) && XNU_NATIVE_OFFLINE_GPU ++ extern void native_offline_gpu_publish(void); ++ native_offline_gpu_publish(); ++#endif + + /* + * We attach the loopback interface *way* down here to ensure diff --git a/patches/xnu/offline-owner-boot-manifest.patch b/patches/xnu/offline-owner-boot-manifest.patch new file mode 100644 index 0000000..589054d --- /dev/null +++ b/patches/xnu/offline-owner-boot-manifest.patch @@ -0,0 +1,72 @@ +--- a/iokit/bsddev/IOKitBSDInit.cpp ++++ b/iokit/bsddev/IOKitBSDInit.cpp +@@ -566,6 +566,69 @@ + return result; + } + ++#if defined(XNU_OWNER_BOOT_MANIFEST) && XNU_OWNER_BOOT_MANIFEST ++#if !DEVELOPMENT && !DEBUG ++#error Owner image-set metadata is an offline development instrument ++#endif ++#include ++extern "C" int native_verified_block_manifest_sha384(dev_t, const char *, uint8_t[48]); ++extern "C" int native_owner_publish_boot_manifest(dev_t, const char *, dev_t, const char *); ++extern "C" { ++volatile int native_owner_boot_manifest_status = -1; ++uint8_t native_owner_boot_manifest_hash[48]; ++} ++ ++extern "C" int ++native_owner_publish_boot_manifest(dev_t rootDevice, const char *rootUUID, ++ dev_t cacheDevice, const char *cacheUUID) ++{ ++ static const char domain[] = "VerifiedRoot.OwnerBootSet.v1"; ++ uint8_t rootDigest[48], cacheDigest[48], digest[48]; ++ uint8_t rootRole = 1, cacheRole = 2; ++ if (rootDevice == cacheDevice || ++ !native_verified_block_manifest_sha384(rootDevice, rootUUID, rootDigest) || ++ !native_verified_block_manifest_sha384(cacheDevice, cacheUUID, cacheDigest)) { ++ native_owner_boot_manifest_status = 1; ++ return 0; ++ } ++ SHA384_CTX context; ++ SHA384_Init(&context); ++ SHA384_Update(&context, domain, sizeof(domain)); ++ SHA384_Update(&context, &rootRole, sizeof(rootRole)); ++ SHA384_Update(&context, rootDigest, sizeof(rootDigest)); ++ SHA384_Update(&context, &cacheRole, sizeof(cacheRole)); ++ SHA384_Update(&context, cacheDigest, sizeof(cacheDigest)); ++ SHA384_Final(digest, &context); ++ IORegistryEntry *chosen = IORegistryEntry::fromPath("/chosen", gIODTPlane); ++ if (!chosen) { native_owner_boot_manifest_status = 2; return 0; } ++ // Never replace an existing firmware/Apple identity with an owner identity. ++ if (chosen->getProperty("boot-manifest-hash") || chosen->getProperty("owner-boot-manifest-domain")) { ++ chosen->release(); ++ native_owner_boot_manifest_status = 3; ++ return 0; ++ } ++ OSData *value = OSData::withBytes(digest, sizeof(digest)); ++ OSData *tag = OSData::withBytes(domain, sizeof(domain)); ++ bool tagged = tag && chosen->setProperty("owner-boot-manifest-domain", tag); ++ bool published = tagged && value && chosen->setProperty("boot-manifest-hash", value); ++ uint8_t actual[48]; ++ size_t actualSize = sizeof(actual); ++ bool result = published && IOGetBootManifestHash(reinterpret_cast(actual), &actualSize) && ++ actualSize == sizeof(digest) && !memcmp(actual, digest, sizeof(digest)); ++ if (!result) { ++ if (published) chosen->removeProperty("boot-manifest-hash"); ++ if (tagged) chosen->removeProperty("owner-boot-manifest-domain"); ++ } else { ++ memcpy(native_owner_boot_manifest_hash, actual, sizeof(actual)); ++ } ++ OSSafeReleaseNULL(tag); ++ OSSafeReleaseNULL(value); ++ chosen->release(); ++ native_owner_boot_manifest_status = result ? 0 : 4; ++ return result; ++} ++#endif ++ + /* + * Set NVRAM to boot into the right flavor of Recovery, + * optionally passing a UUID of a volume that failed to boot. diff --git a/patches/xnu/offline-owner-cache-carrier.patch b/patches/xnu/offline-owner-cache-carrier.patch new file mode 100644 index 0000000..dacc709 --- /dev/null +++ b/patches/xnu/offline-owner-cache-carrier.patch @@ -0,0 +1,108 @@ +--- a/bsd/vm/vm_unix.c ++++ b/bsd/vm/vm_unix.c +@@ -2225,6 +2225,65 @@ + struct _sr_file_mappings *sr_file_mappings, + struct vm_shared_region *shared_region); + ++#if defined(XNU_OWNER_CACHE_CARRIER) && XNU_OWNER_CACHE_CARRIER && !CONFIG_CSR ++#if !DEVELOPMENT && !DEBUG ++#error The independent owner cache carrier is a development instrument ++#endif ++extern int native_verified_block_root_authorized(dev_t); ++extern int native_verified_block_volume_device(const char *, dev_t *, char *, size_t); ++unsigned long native_owner_cache_carrier_accepts = 0; ++unsigned long native_owner_cache_carrier_rejects = 0; ++volatile dev_t native_owner_cache_carrier_root_device = NODEV; ++volatile dev_t native_owner_cache_carrier_file_device = NODEV; ++ ++static boolean_t ++native_owner_shared_cache_carrier(vnode_t process_root, vnode_t file) ++{ ++ const char *root_uuid = "04769D26-2CA0-4825-B443-E2E5F88300F2"; ++#if defined(XNU_OWNER_CACHE_CARRIER_WRONG_UUID) && XNU_OWNER_CACHE_CARRIER_WRONG_UUID ++ const char *cache_uuid = "9846FE78-2910-4297-AF97-D59BB6F8FC16"; ++#else ++ const char *cache_uuid = "8846FE78-2910-4297-AF97-D59BB6F8FC16"; ++#endif ++ const char *directory = "/private/preboot/System/Library/Caches/com.apple.dyld"; ++ vnode_t expected_parent = NULLVP, device_vnode; ++ dev_t root_device = NODEV, cache_device = NODEV, expected = NODEV; ++ char device_path[64]; ++ boolean_t authorized = FALSE; ++ if (process_root != rootvnode || !file || vnode_vtype(file) != VREG) goto out; ++ mount_t root_mount = vnode_mount(process_root), cache_mount = vnode_mount(file); ++ if (!root_mount || !cache_mount || root_mount == cache_mount || ++ !(vfs_flags(root_mount) & MNT_RDONLY) || !(vfs_flags(cache_mount) & MNT_RDONLY) || ++ strcmp(vfs_statfs(root_mount)->f_fstypename, "apfs") || ++ strcmp(vfs_statfs(cache_mount)->f_fstypename, "apfs")) goto out; ++ device_vnode = vfs_devvp(root_mount); ++ if (device_vnode) { ++ root_device = vnode_specrdev(device_vnode); ++ vnode_put(device_vnode); ++ } ++ device_vnode = vfs_devvp(cache_mount); ++ if (device_vnode) { ++ cache_device = vnode_specrdev(device_vnode); ++ vnode_put(device_vnode); ++ } ++ if (root_device == NODEV || cache_device == NODEV || ++ !native_verified_block_volume_device(root_uuid, &expected, device_path, sizeof(device_path)) || ++ expected != root_device || ++ !native_verified_block_volume_device(cache_uuid, &expected, device_path, sizeof(device_path)) || ++ expected != cache_device) goto out; ++ if (vnode_lookup(directory, 0, &expected_parent, vfs_context_current())) goto out; ++ authorized = vnode_vtype(expected_parent) == VDIR && vnode_mount(expected_parent) == cache_mount && ++ vnode_parent(file) == expected_parent && native_verified_block_root_authorized(root_device); ++out: ++ if (expected_parent) vnode_put(expected_parent); ++ native_owner_cache_carrier_root_device = root_device; ++ native_owner_cache_carrier_file_device = cache_device; ++ if (authorized) os_atomic_inc(&native_owner_cache_carrier_accepts, relaxed); ++ else os_atomic_inc(&native_owner_cache_carrier_rejects, relaxed); ++ return authorized; ++} ++#endif ++ + /* + * Setup part of _shared_region_map_and_slide(). + * It had to be broken out of _shared_region_map_and_slide() to +@@ -2557,7 +2616,13 @@ + * is either on the root volume or in the preboot cryptex volume. + */ + assert(rdir_vp != NULL); ++#if defined(XNU_OWNER_CACHE_CARRIER) && XNU_OWNER_CACHE_CARRIER ++ boolean_t owner_carrier = FALSE; ++#endif + if (srfmp->vp->v_mount != rdir_vp->v_mount) { ++#if defined(XNU_OWNER_CACHE_CARRIER) && XNU_OWNER_CACHE_CARRIER ++ owner_carrier = native_owner_shared_cache_carrier(rdir_vp, srfmp->vp); ++#endif + vnode_t preboot_vp = NULL; + #if XNU_TARGET_OS_OSX + #define PREBOOT_CRYPTEX_PATH "/System/Volumes/Preboot/Cryptexes" +@@ -2565,7 +2630,11 @@ + #define PREBOOT_CRYPTEX_PATH "/private/preboot/Cryptexes" + #endif + error = vnode_lookup(PREBOOT_CRYPTEX_PATH, 0, &preboot_vp, vfs_context_current()); +- if (error || srfmp->vp->v_mount != preboot_vp->v_mount) { ++ if ((error || srfmp->vp->v_mount != preboot_vp->v_mount) ++#if defined(XNU_OWNER_CACHE_CARRIER) && XNU_OWNER_CACHE_CARRIER ++ && !owner_carrier ++#endif ++ ) { + SHARED_REGION_TRACE_ERROR( + ("shared_region: %p [%d(%s)] map(%p:'%s'): " + "not on process' root volume nor preboot volume\n", +@@ -2584,7 +2653,11 @@ + } + #endif /* CONFIG_CSR */ + +- if (scdir_enforce) { ++ if (scdir_enforce ++#if defined(XNU_OWNER_CACHE_CARRIER) && XNU_OWNER_CACHE_CARRIER && !CONFIG_CSR ++ && !owner_carrier ++#endif ++ ) { + char **expected_scdir_path = is_driverkit ? driverkit_scdir_path : scdir_path; + struct vnode *scdir_vp = NULL; + for (expected_scdir_path = is_driverkit ? driverkit_scdir_path : scdir_path; diff --git a/patches/xnu/offline-owner-root-binding-audit.patch b/patches/xnu/offline-owner-root-binding-audit.patch new file mode 100644 index 0000000..abb99c7 --- /dev/null +++ b/patches/xnu/offline-owner-root-binding-audit.patch @@ -0,0 +1,78 @@ +--- a/bsd/kern/bsd_init.c ++++ b/bsd/kern/bsd_init.c +@@ -987,9 +987,41 @@ + if (require_rootauth) { + /* enforce sealedness */ + int autherr = VNOP_IOCTL(rootvnode, FSIOC_KERNEL_ROOTAUTH, NULL, 0, vfs_context_kernel()); +- if (autherr) { ++ int owner_authorized = 0; ++#if defined(XNU_OWNER_ROOT_AUDIT) && XNU_OWNER_ROOT_AUDIT ++ { ++ extern int native_verified_block_root_authorized(dev_t); ++ extern volatile int native_owner_root_apple_autherr, native_owner_root_decision, native_owner_root_readonly; ++ extern volatile dev_t native_owner_root_mount_device, native_owner_root_selected_device; ++ extern void native_owner_root_checkpoint(int, int, dev_t); ++ mount_t root_mount = vnode_mount(rootvnode); ++ vnode_t device_vnode = vfs_devvp(root_mount); ++ dev_t device = device_vnode ? vnode_specrdev(device_vnode) : NODEV; ++ int readonly = (vfs_flags(root_mount) & MNT_RDONLY) != 0; ++ int owner = readonly && device_vnode && native_verified_block_root_authorized(device); ++ if (device_vnode) vnode_put(device_vnode); ++ native_owner_root_mount_device = device; ++ native_owner_root_selected_device = rootdev; ++ native_owner_root_apple_autherr = autherr; ++ native_owner_root_decision = owner; ++ native_owner_root_readonly = readonly; ++#if defined(XNU_OWNER_ROOT_TRUST) && XNU_OWNER_ROOT_TRUST ++ owner_authorized = owner; ++#endif ++ native_owner_root_checkpoint(autherr, owner, device); ++ } ++#endif ++ if (autherr && !owner_authorized) { + panic("rootvp not authenticated after mounting"); + } ++#if defined(XNU_OWNER_ROOT_TRUST) && XNU_OWNER_ROOT_TRUST ++ { ++ extern volatile int native_owner_root_accepted; ++ extern void native_owner_root_accepted_checkpoint(int, int); ++ native_owner_root_accepted = owner_authorized; ++ native_owner_root_accepted_checkpoint(autherr, owner_authorized); ++ } ++#endif + } + } + +@@ -1114,6 +1146,32 @@ + + bsd_init_kprintf("done\n"); + } ++ ++#if defined(XNU_OWNER_ROOT_AUDIT) && XNU_OWNER_ROOT_AUDIT ++#if defined(XNU_OWNER_ROOT_TRUST) && XNU_OWNER_ROOT_TRUST ++volatile int native_owner_root_trust_enabled = 1; ++#else ++volatile int native_owner_root_trust_enabled = 0; ++#endif ++volatile int native_owner_root_accepted = 0; ++volatile int native_owner_root_apple_autherr = -1; ++volatile int native_owner_root_decision = -1; ++volatile int native_owner_root_readonly = -1; ++volatile dev_t native_owner_root_mount_device = NODEV; ++volatile dev_t native_owner_root_selected_device = NODEV; ++void native_owner_root_accepted_checkpoint(int, int); ++__attribute__((noinline)) void ++native_owner_root_accepted_checkpoint(int apple, int owner) ++{ ++ __asm__ volatile("" : : "r"(apple), "r"(owner) : "memory"); ++} ++void native_owner_root_checkpoint(int, int, dev_t); ++__attribute__((noinline)) void ++native_owner_root_checkpoint(int apple, int owner, dev_t device) ++{ ++ __asm__ volatile("" : : "r"(apple), "r"(owner), "r"(device) : "memory"); ++} ++#endif + + void + bsdinit_task(void) diff --git a/patches/xnu/offline-owner-root-crypto-probe.patch b/patches/xnu/offline-owner-root-crypto-probe.patch new file mode 100644 index 0000000..175f169 --- /dev/null +++ b/patches/xnu/offline-owner-root-crypto-probe.patch @@ -0,0 +1,18 @@ +--- a/bsd/kern/bsd_init.c ++++ b/bsd/kern/bsd_init.c +@@ -834,6 +834,15 @@ + + bsd_init_kprintf("calling sysctl_mib_init\n"); + sysctl_mib_init(); ++#if defined(XNU_VERIFIED_ROOT_PROBE) && XNU_VERIFIED_ROOT_PROBE ++ { ++ extern void native_verified_root_probe(void); ++ int enabled = 0; ++ if (PE_parse_boot_argn("offline-owner-root-check", &enabled, sizeof(enabled)) && enabled) { ++ native_verified_root_probe(); ++ } ++ } ++#endif + + bsd_init_kprintf("calling bsd_autoconf\n"); + bsd_autoconf(); diff --git a/patches/xnu/offline-owner-systemos-mount.patch b/patches/xnu/offline-owner-systemos-mount.patch new file mode 100644 index 0000000..a55938c --- /dev/null +++ b/patches/xnu/offline-owner-systemos-mount.patch @@ -0,0 +1,317 @@ +--- a/bsd/kern/kern_exec.c ++++ b/bsd/kern/kern_exec.c +@@ -183,7 +183,7 @@ + + #include "kern_exec_internal.h" + +-#include ++#include + + #include + +@@ -7013,8 +7013,252 @@ + * word width of the kernel. Once the self-exec completes, the + * initproc might be different. + */ ++#if defined(XNU_OWNER_CACHE_MOUNT) && XNU_OWNER_CACHE_MOUNT ++#if !DEVELOPMENT && !DEBUG ++#error The offline owner cache mount is a development instrument ++#endif ++extern int native_verified_block_volume_device(const char *, dev_t *, char *, size_t); ++#if defined(XNU_OWNER_BOOT_MANIFEST) && XNU_OWNER_BOOT_MANIFEST ++extern int native_owner_publish_boot_manifest(dev_t, const char *, dev_t, const char *); ++#endif ++#if defined(XNU_OWNER_CACHE_STARTUP) && XNU_OWNER_CACHE_STARTUP ++extern void commpage_update_dyld_flags(uint64_t); ++volatile uint64_t native_owner_cache_dyld_flags = 0; ++#endif ++extern int native_verified_block_root_authorized(dev_t); ++volatile int native_owner_cache_stage = 0; ++volatile int native_owner_cache_status = -1; ++volatile int native_owner_cache_mount_status = -1; ++volatile int native_owner_cache_readonly = 0; ++volatile int native_owner_cache_authorized = 0; ++volatile int native_owner_cache_root_recheck = 0; ++volatile uint64_t native_owner_cache_actual_mount_flags = 0; ++volatile uintptr_t native_owner_cache_original_root = 0; ++volatile uintptr_t native_owner_cache_current_root = 0; ++volatile int native_owner_cache_native_union = 0; ++volatile dev_t native_owner_cache_selected_device = NODEV; ++volatile dev_t native_owner_cache_mounted_device = NODEV; ++char native_owner_cache_device_path[64]; ++void native_owner_cache_checkpoint(int, int); ++__attribute__((noinline)) void ++native_owner_cache_checkpoint(int status, int authorized) ++{ ++ __asm__ volatile("" : : "r"(status), "r"(authorized) : "memory"); ++} ++ + static int +-load_init_program_at_path(proc_t p, user_addr_t scratch_addr, const char* path) ++native_owner_cache_prepare(proc_t p, user_addr_t scratch, user_addr_t *envp) ++{ ++ const char *volume = "8846FE78-2910-4297-AF97-D59BB6F8FC16"; ++#if defined(XNU_OWNER_UNION_SYSTEMOS) && XNU_OWNER_UNION_SYSTEMOS ++#if !CONFIG_UNION_MOUNTS ++#error The owner composition requires the original native union-mount implementation ++#endif ++ const char *mountpoint = "/"; ++#else ++ const char *mountpoint = "/private/preboot"; ++#endif ++#if !defined(XNU_OWNER_UNION_SYSTEMOS) || !XNU_OWNER_UNION_SYSTEMOS ++ const char *environment = "DYLD_SHARED_CACHE_DIR=/private/preboot/System/Library/Caches/com.apple.dyld"; ++#endif ++ vnode_t original_os_root = rootvnode; ++ native_owner_cache_original_root = (uintptr_t)original_os_root; ++ mount_t root_mount = vnode_mount(original_os_root); ++ vnode_t device_vnode = vfs_devvp(root_mount); ++ dev_t root_device = device_vnode ? vnode_specrdev(device_vnode) : NODEV; ++ dev_t selected = NODEV; ++ vnode_t mounted_root = NULLVP; ++ int error = EAUTH, root_authorized = 0, cache_authorized = 0; ++ int32_t result = 0; ++ user64_addr_t spec = scratch + 0x160; ++ char device_path[64] = {0}; ++ uint8_t mount_data[0x150] = {0}; ++ uint64_t mount_flags = MNT_RDONLY; ++ uint16_t mount_mode = 1; ++ uint32_t anonymous_owner = 99; ++#if !defined(XNU_OWNER_UNION_SYSTEMOS) || !XNU_OWNER_UNION_SYSTEMOS ++#if defined(XNU_OWNER_CRYPTEX_FILES) && XNU_OWNER_CRYPTEX_FILES ++ const char *fallback = "DYLD_FALLBACK_LIBRARY_PATH=/private/preboot/usr/lib"; ++ user64_addr_t environment_vector[3] = {scratch + 0x400, scratch + 0x600, USER_ADDR_NULL}; ++#else ++ user64_addr_t environment_vector[2] = {scratch + 0x400, USER_ADDR_NULL}; ++#endif ++#endif ++ struct mount_args request = {.type = scratch + 0x100, .path = scratch + 0x120, ++ .flags = MNT_RDONLY, .data = scratch + 0x200}; ++ request.flags = (int)mount_flags; ++#if defined(XNU_OWNER_UNION_SYSTEMOS) && XNU_OWNER_UNION_SYSTEMOS ++ request.flags |= MNT_UNION; ++#endif ++ if (device_vnode) vnode_put(device_vnode); ++ root_authorized = (vfs_flags(root_mount) & MNT_RDONLY) && root_device != NODEV && ++ native_verified_block_root_authorized(root_device); ++ if (!root_authorized) goto done; ++ native_owner_cache_stage = 1; ++ if (!native_verified_block_volume_device(volume, &selected, device_path, ++ sizeof(device_path))) goto done; ++ strlcpy(native_owner_cache_device_path, device_path, sizeof(native_owner_cache_device_path)); ++ native_owner_cache_selected_device = selected; ++ native_owner_cache_stage = 2; ++ /* Exact user-mount payload already qualified against this original APFS. */ ++ memcpy(mount_data, &spec, sizeof(spec)); ++ memcpy(mount_data + 8, &mount_flags, sizeof(mount_flags)); ++ memcpy(mount_data + 16, &mount_mode, sizeof(mount_mode)); ++ memcpy(mount_data + 0x144, &anonymous_owner, sizeof(anonymous_owner)); ++ memcpy(mount_data + 0x148, &anonymous_owner, sizeof(anonymous_owner)); ++ if ((error = copyout("apfs", request.type, 5)) != 0 || ++ (error = copyout(mountpoint, request.path, strlen(mountpoint) + 1)) != 0 || ++ (error = copyout(device_path, spec, strlen(device_path) + 1)) != 0 || ++ (error = copyout(mount_data, request.data, sizeof(mount_data))) != 0) goto done; ++ native_owner_cache_stage = 3; ++ error = mount(p, &request, &result); ++ native_owner_cache_mount_status = error; ++ if (error) goto done; ++ native_owner_cache_stage = 4; ++ error = vnode_lookup(mountpoint, 0, &mounted_root, vfs_context_current()); ++ if (error) goto done; ++ mount_t cache_mount = vnode_mount(mounted_root); ++ device_vnode = vfs_devvp(cache_mount); ++ dev_t actual = device_vnode ? vnode_specrdev(device_vnode) : NODEV; ++ if (device_vnode) vnode_put(device_vnode); ++ native_owner_cache_mounted_device = actual; ++ int readonly = (vfs_flags(cache_mount) & MNT_RDONLY) != 0; ++ native_owner_cache_readonly = readonly; ++ native_owner_cache_actual_mount_flags = vfs_flags(cache_mount); ++ native_owner_cache_current_root = (uintptr_t)rootvnode; ++ int native_union = (vfs_flags(cache_mount) & MNT_UNION) && ++ vnode_mountedhere(original_os_root) == cache_mount && rootvnode == mounted_root; ++ native_owner_cache_native_union = native_union; ++ cache_authorized = readonly && actual == selected && ++ !strcmp(vfs_statfs(cache_mount)->f_fstypename, "apfs") && native_verified_block_root_authorized(actual); ++#if defined(XNU_OWNER_UNION_SYSTEMOS) && XNU_OWNER_UNION_SYSTEMOS ++ cache_authorized = cache_authorized && native_union; ++#endif ++ if (!cache_authorized) { error = EAUTH; goto done; } ++ native_owner_cache_stage = 5; ++#if !defined(XNU_OWNER_UNION_SYSTEMOS) || !XNU_OWNER_UNION_SYSTEMOS ++ if ((error = copyout(environment, environment_vector[0], strlen(environment) + 1)) != 0 || ++ (error = copyout(environment_vector, scratch + 0x500, sizeof(environment_vector))) != 0) goto done; ++#if defined(XNU_OWNER_CRYPTEX_FILES) && XNU_OWNER_CRYPTEX_FILES ++ if ((error = copyout(fallback, scratch + 0x600, strlen(fallback) + 1)) != 0) goto done; ++#endif ++ *envp = scratch + 0x500; ++#else ++ *envp = USER_ADDR_NULL; ++#endif ++ native_owner_cache_stage = 6; ++done: ++ if (mounted_root) vnode_put(mounted_root); ++ int root_recheck = root_authorized && native_verified_block_root_authorized(root_device); ++ native_owner_cache_root_recheck = root_recheck; ++ if (!root_recheck) error = EAUTH; ++#if defined(XNU_OWNER_BOOT_MANIFEST) && XNU_OWNER_BOOT_MANIFEST ++ if (!error && cache_authorized && root_recheck && ++ !native_owner_publish_boot_manifest(root_device, "04769D26-2CA0-4825-B443-E2E5F88300F2", ++ selected, volume)) error = EAUTH; ++#endif ++#if defined(XNU_OWNER_CACHE_STARTUP) && XNU_OWNER_CACHE_STARTUP ++ if (!error && cache_authorized && root_recheck) { ++ /* Original dyld bit 3 selects no ignition; the owner chain and actual ++ * readonly mounts above supply image authentication in this mode. */ ++ dyld_flags |= UINT64_C(8); ++ commpage_update_dyld_flags(dyld_flags); ++ native_owner_cache_dyld_flags = dyld_flags; ++ } ++#endif ++ native_owner_cache_authorized = cache_authorized; ++ native_owner_cache_status = error; ++ native_owner_cache_checkpoint(error, cache_authorized); ++ return error; ++} ++#endif ++ ++#if defined(XNU_OWNER_DATA_MOUNT) && XNU_OWNER_DATA_MOUNT ++#if !defined(XNU_OWNER_BOOT_MANIFEST) || !XNU_OWNER_BOOT_MANIFEST ++#error Scratch Data mounting requires the authenticated original OS startup path ++#endif ++#if defined(XNU_NATIVE_RAM_BLOCK) && XNU_NATIVE_RAM_BLOCK ++#ifndef XNU_OWNER_DATA_VOLUME_UUID ++#error Supply the observed scratch volume UUID for the IOKit backend ++#endif ++extern int native_ram_block_volume_seed(const char *, dev_t *, char *, size_t, uint64_t *, uint64_t *, uint8_t *); ++#endif ++extern dev_t mdevlookup(int); ++extern int mdevgetrange(int, uint64_t *, uint64_t *); ++volatile int native_owner_data_status = -1; ++volatile int native_owner_data_stage = 0; ++volatile int native_owner_data_readonly = -1; ++volatile dev_t native_owner_data_device = NODEV; ++uint64_t native_owner_data_seed_base, native_owner_data_seed_size; ++uint8_t native_owner_data_seed_sha256[32]; ++char native_owner_data_device_path[64]; ++ ++static int ++native_owner_data_prepare(proc_t p, user_addr_t scratch) ++{ ++ uint64_t base = 0, size = 0; ++ dev_t device = NODEV; ++ int error = EINVAL; ++ vnode_t data_root = NULLVP; ++ char device_path[64] = "/dev/md0"; ++#if defined(XNU_NATIVE_RAM_BLOCK) && XNU_NATIVE_RAM_BLOCK ++ /* The typed provider supplies logical seed identity for both dense and ++ * sparse media. A sparse disk has no contiguous logical-byte address. */ ++ if (!native_ram_block_volume_seed(XNU_OWNER_DATA_VOLUME_UUID, &device, device_path, ++ sizeof(device_path), &base, &size, native_owner_data_seed_sha256)) { error = ENOENT; goto done; } ++ if (!size || size > UINT64_C(0x80000000) || (size & 4095)) goto done; ++#else ++ device = mdevlookup(0); ++ if (device == NODEV || mdevgetrange(0, &base, &size) || base < VM_MIN_KERNEL_ADDRESS || ++ !size || size > UINT64_C(0x80000000) || (base & 4095) || (size & 4095)) goto done; ++ SHA256_CTX context; ++ SHA256_Init(&context); ++ SHA256_Update(&context, (const void *)base, size); ++ SHA256_Final(native_owner_data_seed_sha256, &context); ++#endif ++ native_owner_data_seed_base = base; ++ native_owner_data_seed_size = size; ++ native_owner_data_stage = 1; ++ strlcpy(native_owner_data_device_path, device_path, sizeof(native_owner_data_device_path)); ++ uint8_t payload[0x150] = {0}; ++ user64_addr_t spec = scratch + 0x160; ++ uint16_t mode = 1; ++ uint32_t anonymous_owner = 99; ++ memcpy(payload, &spec, sizeof(spec)); ++ memcpy(payload + 16, &mode, sizeof(mode)); ++ memcpy(payload + 0x144, &anonymous_owner, sizeof(anonymous_owner)); ++ memcpy(payload + 0x148, &anonymous_owner, sizeof(anonymous_owner)); ++ struct mount_args request = {.type = scratch + 0x100, .path = scratch + 0x120, ++ .flags = 0, .data = scratch + 0x200}; ++ int32_t result = 0; ++ if ((error = copyout("apfs", request.type, 5)) || ++ (error = copyout("/private/var", request.path, 13)) || ++ (error = copyout(device_path, spec, strlen(device_path) + 1)) || ++ (error = copyout(payload, request.data, sizeof(payload)))) goto done; ++ native_owner_data_stage = 2; ++ error = mount(p, &request, &result); ++ if (error) goto done; ++ native_owner_data_stage = 3; ++ error = vnode_lookup("/private/var", 0, &data_root, vfs_context_current()); ++ if (error) goto done; ++ mount_t data_mount = vnode_mount(data_root); ++ vnode_t device_node = vfs_devvp(data_mount); ++ dev_t actual = device_node ? vnode_specrdev(device_node) : NODEV; ++ if (device_node) vnode_put(device_node); ++ native_owner_data_device = actual; ++ native_owner_data_readonly = (vfs_flags(data_mount) & MNT_RDONLY) != 0; ++ if (actual != device || (vfs_flags(data_mount) & MNT_RDONLY) || ++ strcmp(vfs_statfs(data_mount)->f_fstypename, "apfs")) error = EINVAL; ++ if (!error) native_owner_data_stage = 4; ++done: ++ if (data_root) vnode_put(data_root); ++ native_owner_data_status = error; ++ return error; ++} ++#endif ++ ++static int ++load_init_program_at_path(proc_t p, user_addr_t scratch_addr, const char* path, user_addr_t envp) + { + int retval[2]; + int error; +@@ -7086,7 +7325,7 @@ + */ + init_exec_args.fname = argv0; + init_exec_args.argp = scratch_addr; +- init_exec_args.envp = USER_ADDR_NULL; ++ init_exec_args.envp = envp; + + /* + * So that init task is set with uid,gid 0 token +@@ -7148,6 +7387,15 @@ + mach_vm_size_t map_page_size = vm_map_page_size(map); + + (void) mach_vm_allocate_kernel(map, &scratch_addr, map_page_size, VM_FLAGS_ANYWHERE, VM_KERN_MEMORY_NONE); ++ user_addr_t init_envp = USER_ADDR_NULL; ++#if defined(XNU_OWNER_CACHE_MOUNT) && XNU_OWNER_CACHE_MOUNT ++ int cache_error = native_owner_cache_prepare(p, (user_addr_t)scratch_addr, &init_envp); ++ if (cache_error) panic("offline owner SystemOS mount failed: %d", cache_error); ++#endif ++#if defined(XNU_OWNER_DATA_MOUNT) && XNU_OWNER_DATA_MOUNT ++ int data_error = native_owner_data_prepare(p, (user_addr_t)scratch_addr); ++ if (data_error) panic("offline scratch Data mount failed: %d", data_error); ++#endif + #if CONFIG_MEMORYSTATUS + (void) memorystatus_init_at_boot_snapshot(); + #endif /* CONFIG_MEMORYSTATUS */ +@@ -7162,7 +7410,7 @@ + + if (is_release_suffix) { + printf("load_init_program: attempting to load /sbin/launchd\n"); +- error = load_init_program_at_path(p, (user_addr_t)scratch_addr, "/sbin/launchd"); ++ error = load_init_program_at_path(p, (user_addr_t)scratch_addr, "/sbin/launchd", init_envp); + if (!error) { + return; + } +@@ -7173,7 +7421,7 @@ + strlcat(launchd_path, launchd_suffix, sizeof(launchd_path)); + + printf("load_init_program: attempting to load %s\n", launchd_path); +- error = load_init_program_at_path(p, (user_addr_t)scratch_addr, launchd_path); ++ error = load_init_program_at_path(p, (user_addr_t)scratch_addr, launchd_path, init_envp); + if (!error) { + return; + } else if (error != ENOENT) { +@@ -7186,7 +7434,7 @@ + error = ENOENT; + for (i = 0; i < sizeof(init_programs) / sizeof(init_programs[0]); i++) { + printf("load_init_program: attempting to load %s\n", init_programs[i]); +- error = load_init_program_at_path(p, (user_addr_t)scratch_addr, init_programs[i]); ++ error = load_init_program_at_path(p, (user_addr_t)scratch_addr, init_programs[i], init_envp); + if (!error) { + return; + } else if (error != ENOENT) { diff --git a/patches/xnu/offline-owner-z-air-init.patch b/patches/xnu/offline-owner-z-air-init.patch new file mode 100644 index 0000000..0c82dbd --- /dev/null +++ b/patches/xnu/offline-owner-z-air-init.patch @@ -0,0 +1,20 @@ +--- a/bsd/kern/kern_exec.c ++++ b/bsd/kern/kern_exec.c +@@ -7400,6 +7400,17 @@ + (void) memorystatus_init_at_boot_snapshot(); + #endif /* CONFIG_MEMORYSTATUS */ + ++#if (DEBUG || DEVELOPMENT) && defined(XNU_NATIVE_AIR_INIT) && XNU_NATIVE_AIR_INIT ++ int offline_air = 0; ++ if (PE_parse_boot_argn("offline-air-init", &offline_air, sizeof(offline_air)) && offline_air) { ++ if (offline_air != 1) panic("offline AIR init invalid selector"); ++ error = load_init_program_at_path(p, (user_addr_t)scratch_addr, ++ "/private/var/tmp/air-native", init_envp); ++ if (error) panic("offline signed AIR instrument exec failed: %d", error); ++ return; ++ } ++#endif ++ + #if DEBUG || DEVELOPMENT + /* Check for boot-arg suffix first */ + char launchd_suffix[64]; diff --git a/patches/xnu/offline-pgz-alignment-probe.patch b/patches/xnu/offline-pgz-alignment-probe.patch new file mode 100644 index 0000000..0bff398 --- /dev/null +++ b/patches/xnu/offline-pgz-alignment-probe.patch @@ -0,0 +1,94 @@ +--- a/osfmk/kern/zalloc.c ++++ b/osfmk/kern/zalloc.c +@@ -5629,6 +5629,91 @@ + } + } + STARTUP(EARLY_BOOT, STARTUP_RANK_MIDDLE, pgz_init); ++ ++#if (DEVELOPMENT || DEBUG) && defined(XNU_ZONE_PGZ_ALIGNMENT_PROBE) && XNU_ZONE_PGZ_ALIGNMENT_PROBE ++volatile uint32_t native_zone_pgz_probe_case; ++volatile uint32_t native_zone_pgz_probe_stage; ++volatile uint32_t native_zone_pgz_probe_zone; ++volatile uint32_t native_zone_pgz_probe_expected_zone; ++volatile uint32_t native_zone_pgz_probe_metadata; ++volatile uintptr_t native_zone_pgz_probe_backing; ++volatile uintptr_t native_zone_pgz_probe_alias; ++volatile uintptr_t native_zone_pgz_probe_argument; ++ ++__attribute__((noinline)) void ++native_zone_pgz_probe_checkpoint(void) ++{ ++ __asm__ volatile ("" ::: "memory"); ++} ++ ++__startup_func static void ++native_zone_pgz_alignment_probe(void) ++{ ++ uint32_t mode = 0; ++ void *objects[128] = { 0 }; ++ void *backing = NULL; ++ zone_t other = ZONE_NULL; ++ unsigned count = 0; ++ ++ if (!PE_parse_boot_argn("offline-zone-pgz-case", &mode, sizeof(mode)) || !mode) return; ++ if (mode > 4 || !zone_info.zi_pgz_meta || pgz_slot_avail <= 0) { ++ panic("offline PGZ alignment fixture unavailable"); ++ } ++ zone_t zone = zone_create("native.pgz.alignment", 192, ZC_NOPGZ | ZC_DESTRUCTIBLE); ++ native_zone_pgz_probe_case = mode; ++ native_zone_pgz_probe_zone = zone_index(zone); ++ for (; count < 128;) { ++ void *obj = zalloc_flags(zone, Z_WAITOK | Z_ZERO); ++ objects[count++] = obj; ++ vm_offset_t offset = ((vm_offset_t)obj & PAGE_MASK) - zone_elem_inner_offs(zone); ++ if (zone_meta_from_addr((vm_offset_t)obj)->zm_chunk_len == ZM_SECONDARY_PAGE && ++ !Z_FAST_ALIGNED(offset, zone->z_align_magic)) { ++ backing = obj; ++ break; ++ } ++ } ++ if (!backing || pgz_owned((vm_offset_t)backing)) panic("offline PGZ fixture lacks secondary-page element"); ++ zone_id_require_aligned(zone_index(zone), backing); ++ native_zone_pgz_probe_stage = 1; ++ native_zone_pgz_probe_backing = (uintptr_t)backing; ++ *(volatile uint64_t *)backing = 0x50475a344b50524fULL; ++ vm_offset_t alias = pgz_protect(zone, (vm_offset_t)backing, __builtin_frame_address(0)); ++ if (alias == (vm_offset_t)backing || *(volatile uint64_t *)alias != *(volatile uint64_t *)backing) { ++ panic("offline PGZ fixture did not protect actual backing"); ++ } ++ native_zone_pgz_probe_alias = alias; ++ native_zone_pgz_probe_metadata = zone_meta_from_addr(alias)->zm_bits; ++ native_zone_pgz_probe_stage = 2; ++ zone_id_t expected = zone_index(zone); ++ vm_offset_t argument = alias; ++ if (mode == 2) argument++; ++ if (mode == 3) { ++ other = zone_create("native.pgz.other", 192, ZC_NOPGZ | ZC_DESTRUCTIBLE); ++ expected = zone_index(other); ++ } ++ if (mode == 4) { ++ if (pgz_unprotect(alias, __builtin_frame_address(0)) != (vm_offset_t)backing) panic("offline PGZ fixture backing changed"); ++ zfree(zone, objects[count - 1]); ++ objects[count - 1] = NULL; ++ native_zone_pgz_probe_stage = 3; ++ } ++ native_zone_pgz_probe_expected_zone = expected; ++ native_zone_pgz_probe_argument = argument; ++ zone_id_require_aligned(expected, (void *)argument); ++ if (mode != 1) panic("offline PGZ negative case unexpectedly returned"); ++ *(volatile uint64_t *)alias ^= 1; ++ if (*(volatile uint64_t *)backing != 0x50475a344b50524eULL || ++ pgz_unprotect(alias, __builtin_frame_address(0)) != (vm_offset_t)backing) { ++ panic("offline PGZ backing identity or write failed"); ++ } ++ for (unsigned i = 0; i < count; i++) zfree(zone, objects[i]); ++ if (other) zdestroy(other); ++ zdestroy(zone); ++ native_zone_pgz_probe_stage = 4; ++ native_zone_pgz_probe_checkpoint(); ++} ++STARTUP(EARLY_BOOT, STARTUP_RANK_LAST, native_zone_pgz_alignment_probe); ++#endif + + static void + panic_display_pgz_bt(bool has_syms, uint32_t slot, bool free) diff --git a/patches/xnu/pgz-aligned-validation.patch b/patches/xnu/pgz-aligned-validation.patch new file mode 100644 index 0000000..1b290e1 --- /dev/null +++ b/patches/xnu/pgz-aligned-validation.patch @@ -0,0 +1,15 @@ +--- a/osfmk/kern/zalloc.c ++++ b/osfmk/kern/zalloc.c +@@ -1474,10 +1474,10 @@ zone_id_require_aligned(zone_id_t zid, void *addr) + zone_t zone = zone_by_id(zid); + vm_offset_t elem, offs; + +- elem = (vm_offset_t)addr; ++ elem = pgz_decode((vm_offset_t)addr, 1); + offs = (elem & PAGE_MASK) - zone_elem_inner_offs(zone); + +- if (from_zone_map(addr, 1)) { ++ if (from_zone_map(elem, 1)) { + struct zone_page_metadata *meta; + + meta = zone_meta_from_addr(elem); diff --git a/patches/xnu/public-entitlements-header.patch b/patches/xnu/public-entitlements-header.patch new file mode 100644 index 0000000..6fcdbce --- /dev/null +++ b/patches/xnu/public-entitlements-header.patch @@ -0,0 +1,42 @@ +diff --git a/EXTERNAL_HEADERS/CoreEntitlements/CoreEntitlements.h b/EXTERNAL_HEADERS/CoreEntitlements/CoreEntitlements.h +index 60e7e8f..5831a6c 100644 +--- a/EXTERNAL_HEADERS/CoreEntitlements/CoreEntitlements.h ++++ b/EXTERNAL_HEADERS/CoreEntitlements/CoreEntitlements.h +@@ -5,6 +5,11 @@ + #ifndef CORE_ENTITLEMENTS_H + #define CORE_ENTITLEMENTS_H + ++#define kCSWebBrowserNetworkEntitlement "com.apple.developer.web-browser-engine.networking" ++#define kCSWebBrowserWebContentEntitlement "com.apple.developer.web-browser-engine.webcontent" ++#define kCSWebBrowserGPUEntitlement "com.apple.developer.web-browser-engine.rendering" ++#define kCSWebBrowserHostEntitlement "com.apple.developer.web-browser-engine.host" ++ + #ifdef __cplusplus + extern "C" { + #endif +diff --git a/bsd/kern/kern_exec.c b/bsd/kern/kern_exec.c +index 8bba078..7bfca65 100644 +--- a/bsd/kern/kern_exec.c ++++ b/bsd/kern/kern_exec.c +@@ -183,7 +183,7 @@ + + #include "kern_exec_internal.h" + +-#include ++#include + + #include + +diff --git a/bsd/net/necp_client.c b/bsd/net/necp_client.c +index 74562d8..c5d1f29 100644 +--- a/bsd/net/necp_client.c ++++ b/bsd/net/necp_client.c +@@ -68,7 +68,7 @@ + + #include + +-#include ++#include + + #if SKYWALK + #include diff --git a/patches/xnu/published-freezer-condition.patch b/patches/xnu/published-freezer-condition.patch new file mode 100644 index 0000000..048d0f9 --- /dev/null +++ b/patches/xnu/published-freezer-condition.patch @@ -0,0 +1,12 @@ +diff --git a/bsd/kern/kern_memorystatus_freeze.c b/bsd/kern/kern_memorystatus_freeze.c +--- a/bsd/kern/kern_memorystatus_freeze.c ++++ b/bsd/kern/kern_memorystatus_freeze.c +@@ -133,7 +133,7 @@ + #define FREEZE_APPS_IDLE_DELAY_MULTIPLIER_DEFAULT FREEZE_APPS_IDLE_DELAY_MULTIPLIER_FAST + unsigned int memorystatus_freeze_apps_idle_delay_multiplier = FREEZE_APPS_IDLE_DELAY_MULTIPLIER_DEFAULT; + +-#if (XNU_TARGET_OS_IOS && || XNU_TARGET_OS_WATCH ++#if XNU_TARGET_OS_IOS || XNU_TARGET_OS_WATCH + #define FREEZE_ENABLED_DEFAULT true + #else + #define FREEZE_ENABLED_DEFAULT false diff --git a/patches/xnu/sheng-platform-power.patch b/patches/xnu/sheng-platform-power.patch new file mode 100644 index 0000000..99f2694 --- /dev/null +++ b/patches/xnu/sheng-platform-power.patch @@ -0,0 +1,15 @@ +diff --git a/bsd/kern/bsd_init.c b/bsd/kern/bsd_init.c +--- a/bsd/kern/bsd_init.c ++++ b/bsd/kern/bsd_init.c +@@ -688,6 +688,11 @@ + bsd_init_kprintf("calling bsd_bufferinit\n"); + bsd_bufferinit(); + ++#if defined(XNU_SHENG_PMGR) && XNU_SHENG_PMGR ++ extern void sheng_pmgr_run(void); ++ sheng_pmgr_run(); ++#endif ++ + /* + * Initialize the calendar. + */ diff --git a/patches/xnu/source-corecrypto14-bootstrap.patch b/patches/xnu/source-corecrypto14-bootstrap.patch new file mode 100644 index 0000000..df9a6c8 --- /dev/null +++ b/patches/xnu/source-corecrypto14-bootstrap.patch @@ -0,0 +1,40 @@ +diff --git a/osfmk/prng/prng_random.c b/osfmk/prng/prng_random.c +index c52f0ab..412c167 100644 +--- a/osfmk/prng/prng_random.c ++++ b/osfmk/prng/prng_random.c +@@ -420,3 +420,35 @@ random_bool_gen_bits(struct bool_gen * bg, unsigned int * buffer, unsigned int c + } + return rbits; + } ++ ++/* Internal source-provider verification; no replacement callbacks or skipped POST. */ ++#if defined(XNU_SOURCE_CORECRYPTO14) ++#include ++#include ++#include ++ ++extern struct mach_header_64 _mh_execute_header; ++extern kern_return_t corecrypto_kext_start(kmod_info_t *, void *); ++SECURITY_READ_ONLY_LATE(kern_return_t) source_corecrypto14_status = KERN_FAILURE; ++ ++static void ++source_corecrypto14_bootstrap(void) ++{ ++ static kmod_info_t info = { ++ .info_version = KMOD_INFO_VERSION, ++ .name = "source-corecrypto14-evaluation", ++ .version = "14.0", ++ }; ++ uint32_t mode = 0; ++ ++ if (PE_parse_boot_argn("fips_mode", &mode, sizeof(mode)) && mode != 0) { ++ panic("source corecrypto verification requires complete default POST"); ++ } ++ info.address = (vm_address_t)&_mh_execute_header; ++ source_corecrypto14_status = corecrypto_kext_start(&info, NULL); ++ if (source_corecrypto14_status != KERN_SUCCESS) { ++ panic("source corecrypto initialization failed: %d", source_corecrypto14_status); ++ } ++} ++STARTUP(EARLY_BOOT, STARTUP_RANK_FIRST, source_corecrypto14_bootstrap); ++#endif diff --git a/patches/xnu/source-machine-layer.patch b/patches/xnu/source-machine-layer.patch new file mode 100644 index 0000000..8c419b5 --- /dev/null +++ b/patches/xnu/source-machine-layer.patch @@ -0,0 +1,56 @@ +diff --git a/config/MASTER b/config/MASTER +index cbe6e15..abf9fe9 100644 +--- a/config/MASTER ++++ b/config/MASTER +@@ -332,6 +332,8 @@ options CONFIG_EXCLAVES # + # Compile XNU with PPL support. + options CONFIG_PMAP_PPL # + ++options CONFIG_APPLE_AMCC # ++ + options NOS_ARM_ASM # + options NOS_ARM_PMAP # + +diff --git a/config/MASTER.arm64.bcm2837 b/config/MASTER.arm64.bcm2837 +index 77b9dfe..a185cf4 100644 +--- a/config/MASTER.arm64.bcm2837 ++++ b/config/MASTER.arm64.bcm2837 +@@ -16,7 +16,7 @@ + # Standard Apple OS Configurations: + # -------- ----- -- --------------- + # +-# KERNEL_BASE = [ arm64 config_pmap_ppl xsmall msgb_small config_embedded config_requires_u32_munging ARM_EXTRAS_BASE ] ++# KERNEL_BASE = [ arm64 config_pmap_ppl nos_arm_asm nos_arm_pmap xsmall msgb_small config_embedded config_requires_u32_munging ARM_EXTRAS_BASE ] + # KERNEL_RELEASE = [ KERNEL_BASE ] + # KERNEL_DEV = [ KERNEL_BASE development mach_assert config_xnupost proc_ref_debug config_ext_paniclog ] + # KERNEL_DEBUG = [ KERNEL_BASE debug mach_assert config_xnupost config_waitq_stats config_workloop_debug config_ext_paniclog ] +@@ -50,8 +50,8 @@ + # LIBKERN_DEBUG = [ LIBKERN_BASE iotracking ] + # PERF_DBG_BASE = [ mach_kdp config_serial_kdp kperf ] + # PERF_DBG_RELEASE=[ PERF_DBG_BASE ist_kdebug ] +-# PERF_DBG_DEV = [ PERF_DBG_BASE config_dtrace zleaks kdp_interactive_debugging alternate_debugger sched_hygiene_debug config_iotrace] +-# PERF_DBG_DEBUG = [ PERF_DBG_BASE config_dtrace zleaks kdp_interactive_debugging alternate_debugger sched_hygiene_debug config_iotrace] ++# PERF_DBG_DEV = [ PERF_DBG_BASE config_dtrace zleaks kdp_interactive_debugging alternate_debugger config_iotrace] ++# PERF_DBG_DEBUG = [ PERF_DBG_BASE config_dtrace zleaks kdp_interactive_debugging alternate_debugger config_iotrace] + # MACH_BASE = [ mach config_ecc_logging vc_progress_white importance_inheritance config_atm config_coalitions config_iosched config_library_validation config_sysdiagnose config_telemetry config_mach_bridge_recv_time config_quiesce_counter config_service_port_info ] + # MACH_RELEASE = [ MACH_BASE config_skip_precise_user_kernel_time debugger_for_zone_info ] + # MACH_DEV = [ MACH_BASE config_io_accounting importance_trace ] +diff --git a/osfmk/conf/files.arm64 b/osfmk/conf/files.arm64 +index 528625b..0ee31c6 100644 +--- a/osfmk/conf/files.arm64 ++++ b/osfmk/conf/files.arm64 +@@ -34,10 +34,10 @@ osfmk/arm64/WKdmCompress_4k.s standard + osfmk/arm64/WKdmData.s standard + osfmk/arm64/lz4_decode_arm64.s standard + osfmk/arm64/lz4_encode_arm64.s standard +-osfmk/arm64/amcc_rorgn_ppl.c optional nos_arm_asm config_pmap_ppl +-osfmk/arm64/amcc_rorgn_ppl_amcc.c optional nos_arm_asm config_pmap_ppl +-osfmk/arm64/amcc_rorgn_ppl_ctrr3.c optional nos_arm_asm config_pmap_ppl +-osfmk/arm64/amcc_rorgn_common.c optional nos_arm_asm ++osfmk/arm64/amcc_rorgn_ppl.c optional nos_arm_asm config_pmap_ppl apple_amcc ++osfmk/arm64/amcc_rorgn_ppl_amcc.c optional nos_arm_asm config_pmap_ppl apple_amcc ++osfmk/arm64/amcc_rorgn_ppl_ctrr3.c optional nos_arm_asm config_pmap_ppl apple_amcc ++osfmk/arm64/amcc_rorgn_common.c optional nos_arm_asm apple_amcc + osfmk/arm64/bcopy.s standard + osfmk/arm64/bzero.s standard + osfmk/arm/caches.c standard diff --git a/patches/xnu/source-pthread-bootstrap.patch b/patches/xnu/source-pthread-bootstrap.patch new file mode 100644 index 0000000..4b4429d --- /dev/null +++ b/patches/xnu/source-pthread-bootstrap.patch @@ -0,0 +1,29 @@ +diff --git a/bsd/kern/source_pthread_bootstrap.c b/bsd/kern/source_pthread_bootstrap.c +new file mode 100644 +index 0000000..86a48e9 +--- /dev/null ++++ b/bsd/kern/source_pthread_bootstrap.c +@@ -0,0 +1,23 @@ ++/* SPDX-License-Identifier: BSD-2-Clause-Patent */ ++#include ++#include ++#include ++#include ++ ++extern kern_return_t pthread_start(kmod_info_t *, void *); ++SECURITY_READ_ONLY_LATE(kern_return_t) source_pthread_status = KERN_FAILURE; ++ ++static void ++source_pthread_bootstrap(void) ++{ ++ static kmod_info_t info = { ++ .info_version = KMOD_INFO_VERSION, ++ .name = "source-libpthread", ++ .version = "519.120.4", ++ }; ++ source_pthread_status = pthread_start(&info, NULL); ++ if (source_pthread_status != KERN_SUCCESS) { ++ panic("source libpthread registration failed: %d", source_pthread_status); ++ } ++} ++STARTUP(EARLY_BOOT, STARTUP_RANK_SECOND, source_pthread_bootstrap); diff --git a/patches/xnu/zzx-pmap-sync-64bit-pa.patch b/patches/xnu/zzx-pmap-sync-64bit-pa.patch new file mode 100644 index 0000000..12f1154 --- /dev/null +++ b/patches/xnu/zzx-pmap-sync-64bit-pa.patch @@ -0,0 +1,30 @@ +diff --git a/osfmk/arm/pmap/pmap.c b/osfmk/arm/pmap/pmap.c +index 9f9d7bf..7b8ccd4 100644 +--- a/osfmk/arm/pmap/pmap.c ++++ b/osfmk/arm/pmap/pmap.c +@@ -7063,7 +7063,9 @@ void + pmap_sync_page_attributes_phys( + ppnum_t pp) + { +- flush_dcache((vm_offset_t) (pp << PAGE_SHIFT), PAGE_SIZE, TRUE); ++ /* ptoa() widens before the shift: pp << PAGE_SHIFT is a 32-bit shift and drops the high bits of physical ++ * addresses at or above 4 GiB (all of DRAM on boards whose memory starts there), which phystokv() then rejects. */ ++ flush_dcache((vm_offset_t) ptoa(pp), PAGE_SIZE, TRUE); + } + + #if CONFIG_COREDUMP +diff --git a/osfmk/arm64/sptm/pmap/pmap.c b/osfmk/arm64/sptm/pmap/pmap.c +index 5e8d9cf..c79e02d 100644 +--- a/osfmk/arm64/sptm/pmap/pmap.c ++++ b/osfmk/arm64/sptm/pmap/pmap.c +@@ -6445,7 +6445,9 @@ void + pmap_sync_page_attributes_phys( + ppnum_t pp) + { +- flush_dcache((vm_offset_t) (pp << PAGE_SHIFT), PAGE_SIZE, TRUE); ++ /* ptoa() widens before the shift: pp << PAGE_SHIFT is a 32-bit shift and drops the high bits of physical ++ * addresses at or above 4 GiB (all of DRAM on boards whose memory starts there), which phystokv() then rejects. */ ++ flush_dcache((vm_offset_t) ptoa(pp), PAGE_SIZE, TRUE); + } + + #if CONFIG_COREDUMP diff --git a/patches/xnu/zzy-native-gpu-addon-mount.patch b/patches/xnu/zzy-native-gpu-addon-mount.patch new file mode 100644 index 0000000..342b733 --- /dev/null +++ b/patches/xnu/zzy-native-gpu-addon-mount.patch @@ -0,0 +1,184 @@ +diff --git a/bsd/kern/kern_exec.c b/bsd/kern/kern_exec.c +index 35c58d2..c0300ef 100644 +--- a/bsd/kern/kern_exec.c ++++ b/bsd/kern/kern_exec.c +@@ -7193,6 +7193,82 @@ uint64_t native_owner_data_seed_base, native_owner_data_seed_size; + uint8_t native_owner_data_seed_sha256[32]; + char native_owner_data_device_path[64]; + ++/* Boot-time read-only union views of owner-provided GPU addon content over sealed system directories, mounted after the Data mount and ++ * before launchd starts: the Adreno Metal plugin bundle over /System/Library/Extensions (the original Metal discovery loads ++ * /System/Library/Extensions/.bundle) and an owner launchd job over /System/Library/LaunchDaemons (a process outside ++ * every sandbox profile). Only with boot-arg sheng-gpu-addon=1. Never panics: a missing source directory or a refused mount only ++ * records its status. The backing directories are owner Data (not an authenticated immutable volume); code in them still has to ++ * pass AMFI through the loaded trust cache. */ ++extern volatile int native_bindfs_boot_mount; ++volatile int native_gpu_addon_stage = 0; ++volatile int native_gpu_addon_status = -1; ++volatile int native_gpu_addon2_stage = 0; ++volatile int native_gpu_addon2_status = -1; ++static void ++native_gpu_addon_mount(proc_t p, user_addr_t scratch, user_addr_t offset, const char *source, size_t source_size, ++ const char *point, size_t point_size, const char *visible, const char *own, volatile int *stage, volatile int *status) ++{ ++ vnode_t probe = NULLVP; ++ int error; ++ *stage = 1; ++ error = vnode_lookup(source, 0, &probe, vfs_context_current()); ++ if (error) { ++ *status = error; ++ return; ++ } ++ vnode_put(probe); ++ probe = NULLVP; ++ *stage = 2; ++ struct mount_args request = {.type = scratch + offset, .path = scratch + offset + 0x20, ++ .flags = MNT_RDONLY | MNT_UNION, .data = scratch + offset + 0x100}; ++ int32_t result = 0; ++ if ((error = copyout("bindfs", request.type, 7)) || (error = copyout(point, request.path, point_size)) || ++ (error = copyout(source, request.data, source_size))) { ++ *status = error; ++ return; ++ } ++ native_bindfs_boot_mount = 1; ++ error = mount(p, &request, &result); ++ native_bindfs_boot_mount = 0; ++ if (error) { ++ *status = error; ++ return; ++ } ++ *stage = 3; ++ /* The union must still show a sealed lower entry next to the owner's. */ ++ error = vnode_lookup(visible, 0, &probe, vfs_context_current()); ++ if (!error) { ++ vnode_put(probe); ++ probe = NULLVP; ++ *stage = 4; ++ /* ... and the owner's own entry must be visible through it (stage 5). */ ++ error = vnode_lookup(own, 0, &probe, vfs_context_current()); ++ if (!error) { ++ vnode_put(probe); ++ probe = NULLVP; ++ *stage = 5; ++ } ++ } ++ *status = error; ++} ++static void ++native_gpu_addon_prepare(proc_t p, user_addr_t scratch) ++{ ++ uint32_t enabled = 0; ++ if (!PE_parse_boot_argn("sheng-gpu-addon", &enabled, sizeof(enabled)) || enabled != 1) { ++ return; ++ } ++ static const char extensions_source[] = "/private/var/ngpu/addons", extensions_point[] = "/System/Library/Extensions"; ++ static const char daemons_source[] = "/private/var/ngpu/daemons", daemons_point[] = "/System/Library/LaunchDaemons"; ++ native_gpu_addon_mount(p, scratch, 0x400, extensions_source, sizeof(extensions_source), extensions_point, ++ sizeof(extensions_point), "/System/Library/Extensions/AGXMetalA10.bundle", ++ "/System/Library/Extensions/NativeAdrenoMetal.bundle/Info.plist", &native_gpu_addon_stage, ++ &native_gpu_addon_status); ++ native_gpu_addon_mount(p, scratch, 0x600, daemons_source, sizeof(daemons_source), daemons_point, sizeof(daemons_point), ++ "/System/Library/LaunchDaemons/com.apple.backboardd.plist", ++ "/System/Library/LaunchDaemons/org.blu-sharky.adrenoprobed.plist", &native_gpu_addon2_stage, &native_gpu_addon2_status); ++} ++ + static int + native_owner_data_prepare(proc_t p, user_addr_t scratch) + { +@@ -7250,6 +7326,7 @@ native_owner_data_prepare(proc_t p, user_addr_t scratch) + if (actual != device || (vfs_flags(data_mount) & MNT_RDONLY) || + strcmp(vfs_statfs(data_mount)->f_fstypename, "apfs")) error = EINVAL; + if (!error) native_owner_data_stage = 4; ++ if (!error) native_gpu_addon_prepare(p, scratch); + done: + if (data_root) vnode_put(data_root); + native_owner_data_status = error; +diff --git a/bsd/miscfs/bindfs/bind_vfsops.c b/bsd/miscfs/bindfs/bind_vfsops.c +index 412f9ec..4466a64 100644 +--- a/bsd/miscfs/bindfs/bind_vfsops.c ++++ b/bsd/miscfs/bindfs/bind_vfsops.c +@@ -78,6 +78,8 @@ + + #include "bindfs.h" + ++volatile int native_bindfs_boot_mount = 0; ++ + #define BINDFS_ENTITLEMENT "com.apple.private.bindfs-allow" + + #define SIZEOF_MEMBER(type, member) (sizeof(((type *)0)->member)) +@@ -139,8 +141,9 @@ bindfs_mount(struct mount * mp, __unused vnode_t devvp, user_addr_t user_data, v + return ENOTSUP; + } + +- /* check entitlement */ +- if (!IOCurrentTaskHasEntitlement(BINDFS_ENTITLEMENT)) { ++ /* check entitlement; the boot-time addon mount (kern_exec.c native_gpu_addon_prepare) runs before any ++ * entitled process exists and sets this flag around its own mount() call only */ ++ if (!native_bindfs_boot_mount && !IOCurrentTaskHasEntitlement(BINDFS_ENTITLEMENT)) { + return EPERM; + } + +@@ -230,6 +233,9 @@ bindfs_mount(struct mount * mp, __unused vnode_t devvp, user_addr_t user_data, v + sp->f_flags = flags; + + xmp->bindm_flags = BINDM_CASEINSENSITIVE; /* default to case insensitive */ ++ if (native_bindfs_boot_mount) { ++ xmp->bindm_flags |= BINDM_FORCE_ROOT; ++ } + + error = bindfs_vfs_getlowerattr(vnode_mount(lowerrootvp), &vfa, ctx); + if (error == 0) { +diff --git a/bsd/miscfs/bindfs/bind_vnops.c b/bsd/miscfs/bindfs/bind_vnops.c +index 8609009..0c659c7 100644 +--- a/bsd/miscfs/bindfs/bind_vnops.c ++++ b/bsd/miscfs/bindfs/bind_vnops.c +@@ -105,6 +105,14 @@ bindfs_getattr(struct vnop_getattr_args * args) + vnode_put(lowervp); + + if (error == 0) { ++ if (MOUNTTOBINDMOUNT(vnode_mount(args->a_vp))->bindm_flags & BINDM_FORCE_ROOT) { ++ if (VATTR_IS_ACTIVE(args->a_vap, va_uid)) { ++ VATTR_RETURN(args->a_vap, va_uid, 0); ++ } ++ if (VATTR_IS_ACTIVE(args->a_vap, va_gid)) { ++ VATTR_RETURN(args->a_vap, va_gid, 0); ++ } ++ } + if (VATTR_IS_ACTIVE(args->a_vap, va_fsid)) { + /* fix up fsid so it doesn't say the underlying fs*/ + VATTR_RETURN(args->a_vap, va_fsid, vfs_statfs(vnode_mount(args->a_vp))->f_fsid.val[0]); +diff --git a/bsd/miscfs/bindfs/bindfs.h b/bsd/miscfs/bindfs/bindfs.h +index c3bebc8..4555530 100644 +--- a/bsd/miscfs/bindfs/bindfs.h ++++ b/bsd/miscfs/bindfs/bindfs.h +@@ -81,6 +81,9 @@ + //#define BINDFS_DEBUG 0 + + #define BINDM_CACHE 0x0001 ++/* Owner boot overlay (native_bindfs_boot_mount): report every file as owned by root:wheel. The backing directory is owner data made ++ * without root; launchd and the loader expect system-directory content to be root owned. Mode bits are left as they are. */ ++#define BINDM_FORCE_ROOT 0x80000000ULL + #define BINDM_CASEINSENSITIVE 0x0000000000000002 + + typedef int (*vop_t)(void *); +diff --git a/osfmk/kern/startup.c b/osfmk/kern/startup.c +index dfa5da2..e9f129e 100644 +--- a/osfmk/kern/startup.c ++++ b/osfmk/kern/startup.c +@@ -470,10 +470,12 @@ kernel_bootstrap(void) + if (PE_parse_boot_argn("-show_pointers", &namep, sizeof(namep))) { + doprnt_hide_pointers = FALSE; + } +- if (PE_parse_boot_argn("-no_slto_panic", &namep, sizeof(namep))) { +- extern boolean_t spinlock_timeout_panic; +- spinlock_timeout_panic = FALSE; +- } ++ } ++ /* Owner device: honour -no_slto_panic without a debugger-enabled device policy. A thread that waits for an IPI acknowledgement ++ * (restartable ranges) then keeps waiting instead of panicking the whole machine; the other cores keep running. */ ++ if (PE_parse_boot_argn("-no_slto_panic", &namep, sizeof(namep))) { ++ extern boolean_t spinlock_timeout_panic; ++ spinlock_timeout_panic = FALSE; + } + + kernel_bootstrap_log("console_init"); diff --git a/patches/xnu/zzz-boot-progress.patch b/patches/xnu/zzz-boot-progress.patch new file mode 100644 index 0000000..4a188fb --- /dev/null +++ b/patches/xnu/zzz-boot-progress.patch @@ -0,0 +1,349 @@ +--- a/osfmk/arm/arm_init.c ++++ b/osfmk/arm/arm_init.c +@@ -1,3 +1,4 @@ ++#include + /* + * Copyright (c) 2007-2021 Apple Inc. All rights reserved. + * +@@ -369,6 +370,7 @@ + #endif /* defined(HAS_APPLE_PAC) */ + + PE_init_platform(FALSE, args); /* Get platform expert set up */ ++ boot_progress_init(); + + #if __arm64__ + configure_timer_apple_regs(); +@@ -529,7 +531,9 @@ + __builtin_arm_wsr("pan", 1); + #endif /* __ARM_PAN_AVAILABLE__ */ + ++ boot_progress_stage("arm_vm_init enter"); + arm_vm_init(xmaxmem, args); ++ boot_progress_stage("arm_vm_init returned"); + + if (debug_boot_arg) { + patch_low_glo(); +--- a/osfmk/kern/startup.c ++++ b/osfmk/kern/startup.c +@@ -1,3 +1,4 @@ ++#include + /* + * Copyright (c) 2000-2020 Apple Inc. All rights reserved. + * +@@ -221,6 +222,7 @@ + static inline void + kernel_bootstrap_log(const char *message) + { ++ boot_progress_stage(message); + if ((startup_debug & STARTUP_DEBUG_VERBOSE) && + startup_phase >= STARTUP_SUB_KPRINTF) { + kprintf("kernel_bootstrap: %s\n", message); +@@ -231,6 +233,7 @@ + static inline void + kernel_bootstrap_thread_log(const char *message) + { ++ boot_progress_stage(message); + if ((startup_debug & STARTUP_DEBUG_VERBOSE) && + startup_phase >= STARTUP_SUB_KPRINTF) { + kprintf("kernel_bootstrap_thread: %s\n", message); +@@ -649,6 +652,7 @@ + */ + kernel_bootstrap_thread_log("clock_service_create"); + clock_service_create(); ++ boot_progress_start(); + + /* + * Create the device service. +--- a/osfmk/console/video_console.c ++++ b/osfmk/console/video_console.c +@@ -1,3 +1,4 @@ ++#include + /* + * Copyright (c) 2000-2020 Apple Inc. All rights reserved. + * +@@ -1230,6 +1231,7 @@ + void + vcputc_options(char c, __unused bool poll) + { ++ if (boot_progress_active()) { boot_progress_putc(c); return; } + if (gc_initialized && gc_enabled) { + VCPUTC_LOCK_LOCK(); + if (gc_enabled) { +@@ -2896,6 +2898,9 @@ + vinfo = new_vinfo; + } + ++ boot_progress_bind(vinfo.v_baseaddr, vinfo.v_width, vinfo.v_height, ++ vinfo.v_rowbytes, vinfo.v_depth); ++ + // If we changed the virtual address, remove the old mapping + if (newVideoVirt != 0) { + if (lastVideoVirt && lastVideoMapSize) { /* Was the framebuffer mapped before? */ +@@ -3048,6 +3053,10 @@ + } + #endif + } ++ ++ /* Console initialization/acquisition may clear the existing mapping. */ ++ boot_progress_bind(vinfo.v_baseaddr, vinfo.v_width, vinfo.v_height, ++ vinfo.v_rowbytes, vinfo.v_depth); + } + + void vcattach(void); /* XXX gcc 4 warning cleanup */ +--- a/osfmk/console/serial_console.c ++++ b/osfmk/console/serial_console.c +@@ -1,3 +1,4 @@ ++#include + /* + * Copyright (c) 2000-2020 Apple, Inc. All rights reserved. + * +@@ -251,6 +252,7 @@ + static inline void + _cnputc(char c, bool poll) + { ++ if (boot_progress_active()) { boot_progress_putc(c); return; } + bool in_debugger = (kernel_debugger_entry_count > 0); + const uint32_t idx = get_cons_ops_index(); + +--- a/pexpert/arm/pe_kprintf.c ++++ b/pexpert/arm/pe_kprintf.c +@@ -1,3 +1,4 @@ ++#include + /* + * Copyright (c) 2000-2019 Apple Inc. All rights reserved. + */ +@@ -58,6 +59,7 @@ + static void + PE_init_kprintf(void) + { ++ if (boot_progress_active()) { PE_kputc = boot_progress_putc; return; } + if (serial_init()) { + PE_kputc = serial_putc_crlf; + } else { +--- a/bsd/kern/bsd_init.c ++++ b/bsd/kern/bsd_init.c +@@ -1,3 +1,4 @@ ++#include + /* + * Copyright (c) 2000-2021 Apple Inc. All rights reserved. + * +@@ -564,6 +565,7 @@ + void + bsd_init(void) + { ++ boot_progress_stage("bsd_init enter"); + struct uthread *ut; + vnode_t init_rootvnode = NULLVP; + struct proc_ro_data kernproc_ro_data = { +@@ -583,7 +585,7 @@ + #if DEBUG_BSDINIT + #define bsd_init_kprintf(x, ...) kprintf("bsd_init: " x, ## __VA_ARGS__) + #else +-#define bsd_init_kprintf(x, ...) ++#define bsd_init_kprintf(x, ...) boot_progress_stage("bsd_init: " x) + #endif + + throttle_init(); +@@ -773,7 +775,9 @@ + + #if defined(XNU_SHENG_PMGR) && XNU_SHENG_PMGR + extern void sheng_pmgr_run(void); ++ boot_progress_stage("original PMGR nub start"); + sheng_pmgr_run(); ++ boot_progress_stage("original PMGR nub returned"); + #endif + + /* +@@ -1010,7 +1014,9 @@ + #endif + + bsd_init_kprintf("vfs_mountroot\n"); ++ boot_progress_stage("vfs_mountroot enter"); + if (0 == (err = vfs_mountroot())) { ++ boot_progress_stage("vfs_mountroot mounted; authentication pending"); + break; + } + rootdevice[0] = '\0'; +@@ -1097,6 +1103,7 @@ + extern void native_owner_root_accepted_checkpoint(int, int); + native_owner_root_accepted = owner_authorized; + native_owner_root_accepted_checkpoint(autherr, owner_authorized); ++ boot_progress_stage("root authentication decision recorded"); + } + #endif + } +--- a/bsd/kern/kern_exec.c ++++ b/bsd/kern/kern_exec.c +@@ -1,3 +1,4 @@ ++#include + /* + * Copyright (c) 2000-2020 Apple Inc. All rights reserved. + * +@@ -1343,7 +1344,9 @@ + /* + * Actually load the image file we previously decided to load. + */ ++ boot_progress_process_stage(proc_getpid(p),p->p_name,"exec load_machfile enter"); + lret = load_machfile(imgp, mach_header, thread, &map, &load_result); ++ boot_progress_process_stage(proc_getpid(p),p->p_name,"exec load_machfile returned"); + if (lret != LOAD_SUCCESS) { + error = load_return_to_errno(lret); + +@@ -7380,6 +7383,7 @@ + void + load_init_program(proc_t p) + { ++ boot_progress_stage("load_init_program enter"); + uint32_t i; + int error; + vm_map_t map = current_map(); +@@ -7389,11 +7393,22 @@ + (void) mach_vm_allocate_kernel(map, &scratch_addr, map_page_size, VM_FLAGS_ANYWHERE, VM_KERN_MEMORY_NONE); + user_addr_t init_envp = USER_ADDR_NULL; + #if defined(XNU_OWNER_CACHE_MOUNT) && XNU_OWNER_CACHE_MOUNT ++ boot_progress_stage("SystemOS mount enter"); + int cache_error = native_owner_cache_prepare(p, (user_addr_t)scratch_addr, &init_envp); ++ boot_progress_stage("SystemOS mount returned"); + if (cache_error) panic("offline owner SystemOS mount failed: %d", cache_error); + #endif + #if defined(XNU_OWNER_DATA_MOUNT) && XNU_OWNER_DATA_MOUNT ++ boot_progress_stage("Data mount enter"); + int data_error = native_owner_data_prepare(p, (user_addr_t)scratch_addr); ++ boot_progress_stage("Data mount returned"); + if (data_error) panic("offline scratch Data mount failed: %d", data_error); ++#if defined(XNU_BOOT_PROGRESS) && XNU_BOOT_PROGRESS ++ extern int sheng_boot_watchdog_policy(void *, uint64_t); ++ boot_progress_stage("temporary watchdog launch policy enter"); ++ int policy_error=sheng_boot_watchdog_policy(p, (uint64_t)scratch_addr); ++ if (policy_error) panic("sheng watchdog launch policy failed: %d", policy_error); ++ boot_progress_stage("temporary watchdog launch policy ready"); ++#endif + #endif + #if CONFIG_MEMORYSTATUS +@@ -7891,6 +7899,7 @@ + * approve of exec, kill and return immediately. + */ + if (imgp->ip_mac_return != 0) { ++ boot_progress_process_stage(proc_getpid(p),p->p_name,"exec MAC policy REFUSED"); + KERNEL_DEBUG_CONSTANT(BSDDBG_CODE(DBG_BSD_PROC, BSD_PROC_EXITREASON_CREATE) | DBG_FUNC_NONE, + proc_getpid(p), OS_REASON_EXEC, EXEC_EXIT_REASON_SECURITY_POLICY, 0, 0); + signature_failure_reason = os_reason_create(OS_REASON_EXEC, EXEC_EXIT_REASON_SECURITY_POLICY); +@@ -8053,7 +8062,9 @@ + * state. + */ + ++ boot_progress_process_stage(proc_getpid(p),p->p_name,"exec process signature enter"); + error = proc_process_signature(p, &signature_failure_reason); ++ boot_progress_process_stage(proc_getpid(p),p->p_name,error ? "exec process signature REFUSED" : "exec process signature accepted"); + } + + if (0 == error) { +--- a/osfmk/arm64/sleh.c ++++ b/osfmk/arm64/sleh.c +@@ -1,3 +1,4 @@ ++#include + /* + * Copyright (c) 2012-2023 Apple Inc. All rights reserved. + * +@@ -1230,6 +1231,7 @@ + mach_exception_data_type_t codes[2] = {EXC_ARM_BREAKPOINT}; + mach_msg_type_number_t numcodes = 2; + ++ boot_progress_user_breakpoint(state, esr); + #if __has_feature(ptrauth_calls) + if (ESR_EC(esr) == ESR_EC_BRK_AARCH64 && + brk_comment_is_ptrauth(ISS_BRK_COMMENT(esr))) { +@@ -1671,6 +1673,7 @@ + #endif /* __has_feature(ptrauth_calls) */ + ++ boot_progress_user_abort(state, esr, fault_addr); + if (user_fault_in_self_restrict_mode(thread) && + task_is_jit_exception_fatal(get_threadtask(thread))) { + extern int exit_with_jit_exception(proc_t p); + exit_with_jit_exception(current_proc()); +@@ -2111,6 +2112,7 @@ + + + sleh_interrupt_handler_prologue(state, DBG_INTR_TYPE_OTHER); ++ boot_progress_irq(get_saved_state_pc(state), get_saved_state_cpsr(state), get_saved_state_sp(state)); + + #if USE_APPLEARMSMP + PE_handle_ext_interrupt(); +--- /dev/null ++++ b/osfmk/kern/boot_progress.h +@@ -0,0 +1,34 @@ ++// SPDX-License-Identifier: MIT ++#ifndef XNU_BOOT_PROGRESS_H ++#define XNU_BOOT_PROGRESS_H ++#include ++#if defined(XNU_BOOT_PROGRESS) && XNU_BOOT_PROGRESS ++void boot_progress_init(void); ++void boot_progress_start(void); ++int boot_progress_active(void); ++void boot_progress_bind(uintptr_t, unsigned, unsigned, unsigned, unsigned); ++void boot_progress_putc(char); ++void boot_progress_stage(const char *); ++void boot_progress_irq(uint64_t, uint64_t, uint64_t); ++void boot_progress_poll(void); ++void boot_progress_user_breakpoint(void *, unsigned); ++void boot_progress_user_abort(void *, unsigned, uint64_t); ++void boot_progress_critical_exit(int, char *, unsigned long); ++void boot_progress_process_stage(int, const char *, const char *); ++void boot_progress_exit(int, const char *, int, unsigned, uint64_t); ++#else ++#define boot_progress_init() ((void)0) ++#define boot_progress_start() ((void)0) ++#define boot_progress_active() 0 ++#define boot_progress_bind(...) ((void)0) ++static inline void boot_progress_putc(char c) { (void)c; } ++#define boot_progress_stage(s) ((void)0) ++#define boot_progress_irq(...) ((void)0) ++#define boot_progress_poll() ((void)0) ++#define boot_progress_user_breakpoint(...) ((void)0) ++#define boot_progress_user_abort(...) ((void)0) ++#define boot_progress_critical_exit(...) ((void)0) ++#define boot_progress_process_stage(...) ((void)0) ++#define boot_progress_exit(...) ((void)0) ++#endif ++#endif +--- a/osfmk/kern/Makefile ++++ b/osfmk/kern/Makefile +@@ -114,6 +114,7 @@ + + + XNU_ONLY_EXPORTS = \ ++ boot_progress.h \ + arcade.h \ + btlog.h \ + cpc.h \ +--- a/bsd/kern/kern_exit.c ++++ b/bsd/kern/kern_exit.c +@@ -1,3 +1,4 @@ ++#include + /* + * Copyright (c) 2000-2016 Apple Inc. All rights reserved. + * +@@ -1852,3 +1853,7 @@ + const char *prefix_str; ++#if defined(XNU_BOOT_PROGRESS) && XNU_BOOT_PROGRESS ++ char prefix_str_buf[1024]; ++#else + char prefix_str_buf[128]; ++#endif + +@@ -1864,3 +1869,4 @@ + snprintf(prefix_str_buf, sizeof(prefix_str_buf), "%s[%d] exited", p->p_name, proc_getpid(p)); ++ boot_progress_critical_exit(proc_getpid(p), prefix_str_buf, sizeof(prefix_str_buf)); + prefix_str = prefix_str_buf; + } +@@ -1893,6 +1899,10 @@ + task_t task = proc_task(p); + + ++ boot_progress_exit(proc_getpid(p),p->p_name,rv, ++ p->p_exit_reason ? p->p_exit_reason->osr_namespace : 0, ++ p->p_exit_reason ? p->p_exit_reason->osr_code : 0); ++ + if (p->p_crash_behavior != 0 || p == initproc) { + proc_handle_critical_exit(p, rv); + } diff --git a/scripts/add-gpu-plugin-data.py b/scripts/add-gpu-plugin-data.py new file mode 100644 index 0000000..12a067f --- /dev/null +++ b/scripts/add-gpu-plugin-data.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +"""Stage the owner-built Adreno Metal plugin into a copy of a packed Data seed. + + add-gpu-plugin-data.py BASE.dbr OUTPUT.dbr + +Adds /private/var/ngpu/addons/NativeAdrenoMetal.bundle, /private/var/ngpu/bin/adrenoprobed and /private/var/ngpu/daemons/*.plist (from build/adreno-metal-plugin, see build-adreno-metal-plugin.py). The +kernel's boot-time addon mount (sheng-gpu-addon=1) presents that directory as a read-only union view over +/System/Library/Extensions, which is where the original Metal discovery looks for MetalPluginName. BASE is not modified and the +volume UUID is unchanged. Nothing is written to Git or to the OS inputs.""" +import json +import os +import plistlib +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'drivers/DarwinBoot/Test')) +import packed_ramdisk + +BUNDLE = ROOT / 'build/adreno-metal-plugin/NativeAdrenoMetal.bundle' + + +def main(): + if len(sys.argv) != 3: + sys.exit(__doc__) + base, output = Path(sys.argv[1]).resolve(), Path(sys.argv[2]).resolve() + if output.exists(): + sys.exit('Refuse to overwrite ' + str(output)) + if not (BUNDLE / 'NativeAdrenoMetal').is_file(): + sys.exit('Build the plugin first: python3 scripts/build-adreno-metal-plugin.py') + blob = base.read_bytes() + logical, extents = packed_ramdisk.regions(blob) + with tempfile.TemporaryDirectory(prefix='data-seed-', dir=str(ROOT / 'build')) as td: + raw = Path(td) / 'seed.raw' + with raw.open('wb') as f: + f.truncate(logical) + for offset, data in extents: + f.seek(offset) + f.write(data) + mount = Path(td) / 'mount' + mount.mkdir() + attached = plistlib.loads(subprocess.check_output( + ['hdiutil', 'attach', '-imagekey', 'diskimage-class=CRawDiskImage', '-nobrowse', '-owners', 'off', + '-mountpoint', str(mount), '-plist', str(raw)])) + device = attached['system-entities'][0]['dev-entry'] + try: + if not (mount / 'mobile').is_dir(): + sys.exit('The Data seed has no /mobile; wrong base image?') + target = mount / 'ngpu/addons/NativeAdrenoMetal.bundle' + if target.exists(): + shutil.rmtree(target) + target.parent.mkdir(parents=True) + for directory in (mount / 'ngpu', mount / 'ngpu/addons'): + os.chmod(directory, 0o755) + shutil.copytree(BUNDLE, target, symlinks=False) + daemon_dir, bin_dir = mount / 'ngpu/daemons', mount / 'ngpu/bin' + for directory in (daemon_dir, bin_dir): + if directory.exists(): + shutil.rmtree(directory) + directory.mkdir(parents=True) + os.chmod(directory, 0o755) + shutil.copyfile(BUNDLE.parent / 'adrenoprobed', bin_dir / 'adrenoprobed') + os.chmod(bin_dir / 'adrenoprobed', 0o755) + for plist in sorted(BUNDLE.parent.glob('*.plist')): + if plist.name.endswith('-entitlements.plist'): + continue # build input, not a launchd job + shutil.copyfile(plist, daemon_dir / plist.name) + os.chmod(daemon_dir / plist.name, 0o644) + for path in [target, *target.rglob('*')]: + os.chmod(path, 0o755 if path.is_dir() or path.name in ('NativeAdrenoMetal', 'libvulkan_freedreno.dylib') else 0o644) + print('staged', target.relative_to(mount), sorted(p.name for p in target.iterdir())) + finally: + subprocess.run(['hdiutil', 'detach', device], check=True) + if raw.read_bytes()[32:36] != b'NXSB': + sys.exit('The edited image is no APFS container any more') + metadata = packed_ramdisk.pack(raw, output) + report = dict(base_sha256=packed_ramdisk.inspect(base)[0]['sha256'], output=metadata, + plugin=json.loads((ROOT / 'build/adreno-metal-plugin/trust.json').read_text())) + output.with_suffix('.json').write_text(json.dumps(report, indent=2) + '\n') + print(json.dumps(report, indent=2)) + + +if __name__ == '__main__': + main() diff --git a/scripts/analyze-corecrypto-edges.py b/scripts/analyze-corecrypto-edges.py new file mode 100755 index 0000000..42d6068 --- /dev/null +++ b/scripts/analyze-corecrypto-edges.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +"""Compare visible prelinked corecrypto branch/pointer edges with baseline KPIs. + +This is a partial static survey. It cannot recover stripped relocations, +prove ABI compatibility, or make the extracted bytes loadable. +""" +import argparse +import json +from pathlib import Path +import struct +import subprocess + +ROOT = Path(__file__).resolve().parents[1] +stock = ROOT / '.cache/firmware/kernelcache.macho/kernelcache.release.ipad7.decompressed' +baseline = ROOT / 'build/xnu/ios-obj/DEVELOPMENT_ARM64_BCM2837/mach.development.bcm2837.unstripped.noctf' +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument('--bundle', default='com.apple.kec.corecrypto') +args = parser.parse_args() +folder = ROOT / 'build/target-kexts' / args.bundle +metadata = json.loads((folder / 'segments.json').read_text()) + + +def symbols(path): + result = {} + output = subprocess.check_output(['xcrun', 'nm', '-g', '-n', str(path)], text=True) + for line in output.splitlines(): + parts = line.split() + if len(parts) == 3 and parts[1].upper() != 'U': + result.setdefault(int(parts[0], 16), []).append(parts[2]) + return result + + +old_symbols = symbols(stock) +new_symbols = {name for names in symbols(baseline).values() for name in names} +segments = metadata['segments'] + + +def owned(address): + return any(segment['va'] <= address < segment['va'] + segment['size'] for segment in segments) + + +edges = [] +unresolved_branches = [] +for segment in segments: + data = (folder / f'{segment["name"]}.bin').read_bytes() + if segment['name'] == '__TEXT_EXEC': + for offset in range(0, len(data) - 3, 4): + instruction = struct.unpack_from('> 9) + imports.append(reference[offset:reference.index(b'\0', offset)].decode()) +# All pointer-bearing reference segments must use the qualified kernel format. +segment_count = struct.unpack_from(' vtable) +offset = np.backed_offset(segments, vtable, end - vtable, len(reference)) +reference_slots = [] +for slot in range((end - vtable) // 8 - 2): + pointer = struct.unpack_from('> 62 & 1) + target = pointer & (0xffffffff if pointer >> 63 else (1 << 43) - 1) + names = [imports[pointer & 0xffff]] if bind else by_address.get(target, []) + reference_slots.append(dict(reference_slot=slot, names=names, bind=bind, target=target)) +require(len(reference_slots) == 286 and not reference_slots[-1]['names'], 'Unexpected reference table boundary') +require(not any(s['va'] <= reference_slots[-1]['target'] < s['va'] + s['filesize'] + for s in segments if s['name'] == '__TEXT_EXEC'), 'Trailing reference word is executable') +require(all(r['names'] for r in reference_slots[:-1]), 'Unannotated reference method') +padding = [r for r in reference_slots[:-1] if any('_RESERVED' in n for n in r['names'])] +require(len(padding) == 104, 'Unexpected macOS reserved-slot count') +header = (ROOT / '.cache/xnu-work/libkern/libkern/c++/OSMetaClass.h').read_text() +require(re.search(r'#if !XNU_TARGET_OS_OSX\s+#define APPLE_KEXT_VTABLE_PADDING\s+0', header), 'Source iOS padding policy changed') +slots = [r for r in reference_slots[:-1] if r not in padding] +require(len(slots) == 181, 'Unexpected iOS method count') + +candidate = args.candidate.read_bytes() +plan = json.loads(args.candidate.with_suffix('.json').read_text()) +candidate_sha = hashlib.sha256(candidate).hexdigest() +require(candidate_sha == plan['candidate_sha256'], 'Candidate digest mismatch') +require(plan['original_sha256'] == 'f18b4df865405f2a287b84977d7aca745e056d656d9fc9df1662970ce2969de7', 'Unexpected target kernel provenance') +native = np.macho(candidate) # Deliberately retains the ordinary ARM64-only gate. +exports = np.exports(candidate, native) +module = [s for s in plan['extents'] if s['module'] == 'com.apple.iokit.IOCryptoAcceleratorFamily'] +text = next(s for s in module if s['segment'] == '__TEXT_EXEC') +const = next(s for s in module if s['segment'] == '__DATA_CONST') + + +def read(va, size): + at = np.backed_offset(native['segments'], va, size, len(candidate)) + return candidate[at:at + size] + + +def returned_address(va): + a, add, ret = struct.unpack('<3I', read(va, 12)) + if a & 0x9f00001f != 0x90000000 or add & 0xffc003ff != 0x91000000 or ret != 0xd65f03c0: + return None + immediate = ((a >> 5 & 0x7ffff) << 2) | (a >> 29 & 3) + if immediate & 0x100000: + immediate -= 0x200000 + return (va & ~4095) + immediate * 4096 + (add >> 10 & 4095) + + +algorithm_slot = next(i for i, r in enumerate(slots) if '__ZN16IOAESAccelerator16getAlgorithmNameEv' in r['names']) +data = read(const['va'], const['size']) +matches = [] +for offset in range(0, len(data) - (len(slots) + 2) * 8 + 1, 8): + if data[offset:offset + 16] != bytes(16): + continue + pointer = struct.unpack_from('= 40: + return bytes.fromhex(value) + for value in snapshot['indirect_memory'].values(): + if int(value['address'], 16) == address: + return bytes.fromhex(value['bytes']) + raise SystemExit(f'Missing actual guest metadata at {address:#x}') + + +aes_data, service_data = observed(meta), observed(service_meta) +crypto_meta = struct.unpack_from('> 14 & 3, tcr >> 30 & 3) == (0, 2), 'Live translation is not native 4K on both halves') +require(int(registers['SCTLR_EL1'], 16) & 1 and all(int(registers[r], 16) for r in ('TTBR0_EL1', 'TTBR1_EL1')), 'Inactive translation roots') +result = dict(reference_annotation_sha256=ref_sha, reference_executed_or_linked=False, + candidate_sha256=candidate_sha, original_sha256=plan['original_sha256'], + macos_padding_slots=104, ordinary_ios_method_slots=181, inherited_kernel_matches=inherited, + vtable=hex(table), algorithm='AES-CBC', algorithm_slot=algorithm_slot, methods=rows, + compiled_metaclass_offsets=layout, live_metaclasses=dict(IOAESAccelerator=hex(meta), + IOCryptoAccelerator=hex(crypto_meta), IOService=hex(service_meta)), + live_instance_sizes=dict(zip(('IOAESAccelerator', 'IOCryptoAccelerator', 'IOService'), sizes)), + live_native_4k=True, live_registers=registers, request_structure_qualified=False, + aes_operations_tested=False, service_registered=False, hardware_observation=False) +args.output.parent.mkdir(parents=True, exist_ok=True) +args.output.write_text(json.dumps(result, indent=2) + '\n') +print('PASS: 181 ordinary iOS slots, 162 inherited matches; live AES/parent/service sizes 0x120/0x88/0x88') +print('Reference-only ARM64e annotations and native 4K metadata; this class audit does not qualify private requests or provider operations.') diff --git a/scripts/audit-amfi-pmgr-contract.py b/scripts/audit-amfi-pmgr-contract.py new file mode 100644 index 0000000..12eb8a8 --- /dev/null +++ b/scripts/audit-amfi-pmgr-contract.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python3 +"""Audit the unchanged target AMFI -> PMGR boundary; no hardware access.""" +import argparse +import hashlib +import json +from pathlib import Path +import struct +import sys + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'kits/NativePrelink')) +from native_prelink import backed_offset, exports, macho + + +def require(ok, message): + if not ok: + raise SystemExit(message) + + +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument('--candidate', type=Path, default=ROOT / + 'build/native-prelink/native-owner-ufs-rootauth3-candidate.macho') +args = parser.parse_args() +if not args.candidate.exists(): + # Build products are pruned: a later candidate built from the same original kernelcache carries the same original + # AppleMobileFileIntegrity/ApplePMGR instructions; the instruction contracts below are checked against it. + later = sorted((ROOT / 'build/native-prelink').glob('native-owner-ufs-cpu-desktop-*-candidate.macho'), key=lambda f: f.stat().st_mtime) + require(later, 'No reference candidate available') + args.candidate = later[-1] +data = args.candidate.read_bytes() +plan = json.loads(args.candidate.with_suffix('.json').read_text()) +meta = macho(data) +symbols = exports(data, meta) + + +def read(va, size): + at = backed_offset(meta['segments'], va, size, len(data)) + return data[at:at + size] + + +def words(va, expected): + require(struct.unpack('<' + 'I' * len(expected), read(va, 4 * len(expected))) == + expected, f'Instruction contract changed at {va:#x}') + + +def pointer(va): + return struct.unpack('> 5) & 0x7ffff) << 2 | ((first >> 29) & 3) + if imm & (1 << 20): + imm -= 1 << 21 + return (pc & ~4095) + (imm << 12) + ((second >> 10) & 4095) + + +base_vptr = constructor_vptr(read, 0xfffffff0049f0058) +require(base_vptr == 0xfffffff005ce26d8 and + struct.unpack('<2Q', read(base_vptr + 320 * 8, 16)) == + (0xfffffff004a20fbc, 0xfffffff004a20fc4), 'Base provider slots differ') +words(0xfffffff004a20fbc, (0x12800000, 0xd65f03c0, 0xd65f03c0)) + +# The target-specific driver is inspected in the original supplied cache; it +# is not in this native candidate. Do not infer hardware semantics from its name. +original = (ROOT / '.cache/firmware/kernelcache.macho/kernelcache.release.ipad7.decompressed').read_bytes() +original_meta = macho(original) + + +def original_read(va, size): + at = backed_offset(original_meta['segments'], va, size, len(original)) + return original[at:at + size] + + +header = backed_offset(original_meta['segments'], 0xfffffff00584f680, 32, len(original)) +target = macho(original, header) +target_hashes = { + '__TEXT': 'bd4be83b7c24b18d4796be180e8bdaf96a2a4670407e85101164c991cd9d408c', + '__TEXT_EXEC': '59d36c869db750dcde3ec1ab87d26619beef7c12d929d674e0b3d38357ad5b22', + '__DATA_CONST': '429c542da926dfa6376ba1dc6e622f2a3cb6afa3c0bfade43d4954c2c63a2a07', +} +for name, digest in target_hashes.items(): + segment, = [s for s in target['segments'] if s['name'] == name] + require(hashlib.sha256(original_read(segment['va'], segment['filesize'])).hexdigest() == digest, + f'Target AppleT8011PMGR {name} differs') +require(original_read(0xfffffff005852125, 15) == b'AppleT8011PMGR\0', 'Target class name differs') +target_vptr = constructor_vptr(original_read, 0xfffffff006a55fb0) +require(target_vptr == 0xfffffff006f9da90, 'Target constructor vptr differs') +target_slots = struct.unpack('<2Q', original_read(target_vptr + 320 * 8, 16)) +require(target_slots == (0xfffffff006a577e8, 0xfffffff006a577f0), 'Target PMGR slots differ') +require(struct.unpack('<2I', original_read(target_slots[0], 8)) == (0x12800000, 0xd65f03c0) and + struct.unpack('> 5 & 0x7ffff) << 2) | (a >> 29 & 3) + if delta & 0x100000: + delta -= 0x200000 + got = (target & ~4095) + delta * 4096 + (load >> 10 & 4095) * 8 + if got in bindings: + destination = bindings[got]['new_target'] + calls.append(dict(site=hex(slid + offset), target=hex(destination), names=exports.get(destination, []))) +wait = [c for c in calls if '_lck_mtx_gate_wait' in c['names']] +if len(wait) != 1: + raise SystemExit('Cannot establish the real source-kernel gate-wait binding') +result = dict(original_sha256=hashlib.sha256(original).hexdigest(), candidate_sha256=plan['candidate_sha256'], + original_handler=hex(handler), candidate_handler=hex(slid), oid=hex(oids[0]), + compiled_oid_offsets={k: layout[k] for k in ('oid_name', 'oid_handler')}, + original_handler_prefix_sha256=hashlib.sha256(prefix).hexdigest(), kernel_calls=calls, + personality=entry['IOKitPersonalities']['AppleImage4'], + code_or_security_result_modified=False, source_analysis_only=True, handler_return_observed=False) +args.output.parent.mkdir(parents=True, exist_ok=True) +args.output.write_text(json.dumps(result, indent=2) + '\n') +print('PASS: actual original sysctl object/handler, compiled ABI, unchanged code and genuine gate-wait binding') +print(f'Handler {slid:#x}; gate wait {wait[0]["target"]}; this is static analysis, not service startup') diff --git a/scripts/audit-metal-api-surface.py b/scripts/audit-metal-api-surface.py new file mode 100644 index 0000000..b2fc037 --- /dev/null +++ b/scripts/audit-metal-api-surface.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python3 +"""Which Metal API does the unchanged target's graphics stack actually send? (static, read-only) + +The native Adreno Metal plugin has to supply a real _MTLDevice and the objects hanging off it. How much of Metal that means depends +on the clients, so this measures it: the selectors declared by the Metal protocols (from the iPhoneOS SDK headers, via the clang AST) +against the selector references (__objc_selrefs) of the compositor and its neighbours in the target's unchanged dyld shared cache. +A selector reference proves the image can send that message, not that it does on the paths the system UI takes; the list is an upper +bound of the surface that compositing needs and a lower bound of what any client may call. + +Output: build/gpu-analysis/metal-api-surface.json (ignored) and a summary on stdout. The SDK is newer than iPadOS 17.7.11; selectors the +SDK declares but the target never references simply do not appear, and selectors only the target knows are listed as 'unmatched'.""" +import json +import mmap +from pathlib import Path +import re +import struct +import subprocess +import sys +import tempfile + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'kits/NativePrelink')) +from native_prelink import macho + +PROTOCOLS = ['MTLDevice', 'MTLCommandQueue', 'MTLCommandBuffer', 'MTLCommandEncoder', 'MTLRenderCommandEncoder', + 'MTLBlitCommandEncoder', 'MTLComputeCommandEncoder', 'MTLParallelRenderCommandEncoder', 'MTLResource', 'MTLTexture', + 'MTLBuffer', 'MTLLibrary', 'MTLFunction', 'MTLRenderPipelineState', 'MTLComputePipelineState', 'MTLSamplerState', + 'MTLDepthStencilState', 'MTLEvent', 'MTLSharedEvent', 'MTLFence', 'MTLHeap', 'MTLDrawable', 'MTLCaptureScope', + 'MTLArgumentEncoder', 'MTLIndirectCommandBuffer', 'MTLCounterSampleBuffer', 'MTLBinaryArchive'] +IMAGES = ['/System/Library/Frameworks/QuartzCore.framework/QuartzCore', '/System/Library/Frameworks/Metal.framework/Metal', + '/System/Library/Frameworks/MetalKit.framework/MetalKit', '/System/Library/PrivateFrameworks/UIKitCore.framework/UIKitCore', + '/System/Library/Frameworks/CoreImage.framework/CoreImage', '/System/Library/PrivateFrameworks/IOMobileFramebuffer.framework/IOMobileFramebuffer'] + + +def sdk_selectors(): + sdk = subprocess.check_output(['xcrun', '--sdk', 'iphoneos', '--show-sdk-path'], text=True).strip() + result = {} + with tempfile.TemporaryDirectory() as td: + source = Path(td) / 'm.m' + source.write_text('#import \n#import \n') + for name in PROTOCOLS: + out = subprocess.run(['xcrun', 'clang', '-fsyntax-only', '-fobjc-arc', '-isysroot', sdk, '-target', 'arm64-apple-ios17.0', + '-Xclang', '-ast-dump=json', '-Xclang', '-ast-dump-filter', '-Xclang', name, str(source)], + capture_output=True, text=True).stdout + selectors = set() + decoder, at = json.JSONDecoder(), 0 + while True: + at = out.find('{', at) + if at < 0: + break + node, at = decoder.raw_decode(out, at) + if node.get('kind') != 'ObjCProtocolDecl' or node.get('name') != name: + continue + for child in node.get('inner', []): + if child.get('kind') == 'ObjCMethodDecl': + selectors.add(child['name']) + elif child.get('kind') == 'ObjCPropertyDecl': + prop = child['name'] + selectors.add(prop) + if not child.get('readonly') and 'readonly' not in json.dumps(child.get('inner', [])): + selectors.add('set' + prop[0].upper() + prop[1:] + ':') + result[name] = sorted(selectors) + return result + + +def cache(): + directory = ROOT / '.cache/firmware/target-volumes/systemos-mount/System/Library/Caches/com.apple.dyld' + files, mappings, slides = [], [], [] + for path in sorted(directory.glob('dyld_shared_cache_arm64*')): + if path.suffix == '.symbols': + continue + stream = path.open('rb') + data = mmap.mmap(stream.fileno(), 0, access=mmap.ACCESS_READ) + files.append((stream, data)) + if data[:16].rstrip(b'\0') != b'dyld_v1 arm64': + raise SystemExit('Require the original ordinary ARM64 cache') + at, count = struct.unpack_from('<2I', data, 16) + for i in range(count): + va, size, off, _, _ = struct.unpack_from('<3Q2I', data, at + 32 * i) + mappings.append((va, va + size, off, data)) + at, count = struct.unpack_from('<2I', data, 312) + for i in range(count): + va, size, _, off, length, _, _, _ = struct.unpack_from('<6Q2I', data, at + 56 * i) + if length: + mask, base = struct.unpack_from('<2Q', data, off + 24) + slides.append((va, va + size, mask, base)) + main = files[0][1] + if main[88:104].hex() != '590d839a82d230f1a17d0cc053420f73': + raise SystemExit('Not the exact target cache') + + def read(va, size): + for start, end, off, data in mappings: + if start <= va and va + size <= end: + return data[off + va - start:off + va - start + size] + raise SystemExit('Unbacked cache extent ' + hex(va)) + + def pointer(va): + raw, = struct.unpack('>5)&0x7ffff)*4+((a>>29)&3) + if imm&(1<<20):imm-=1<<21 + return (pc&~4095)+(imm<<12)+((b>>10)&4095)+16 +def branch(pc): + w=struct.unpack(' Service -> Framebuffer constructor chain') +base=struct.unpack('<262Q',read(base_vptr,262*8));service=struct.unpack('<262Q',read(service_vptr,262*8)) +require([i for i,(a,b) in enumerate(zip(base,service)) if a!=b]==[0,1,7],'Service adds no generic operational overrides') +legacy=struct.unpack('<358Q',read(legacy_vptr,358*8)) +require(all(legacy) and read(legacy_vptr+358*8,24)==bytes(24),'Exact nonnull method extent and next aligned vtable boundary') +pure=[i for i,va in enumerate(legacy) if va==names['___cxa_pure_virtual']] +require(pure==[211,213,245,246,306,307,337,338,345],'Actual remaining abstract slots') +changed=[i for i,(a,b) in enumerate(zip(base,legacy)) if a!=b] +require(len(changed)==103 and [i for i in changed if i<168]==[0,1,7,14,21,44,79,85,86,87,111,141,155], + 'Actual original generic implementation overrides') +require(names['__ZN9IOService13newUserClientEP4taskPvjPP12IOUserClient']==base[141], + 'Actual newUserClient override occupies the native four-argument ABI slot') +for i,va in enumerate(legacy): + if i>=168 and i not in pure:require(text['va']<=va> 5) & 0x7ffff) << 2 | ((first >> 29) & 3) + if imm & (1 << 20): + imm -= 1 << 21 + return (pc & ~4095) + (imm << 12) + ((second >> 10) & 4095) + + +require(hashlib.sha256(read(text['va'], text['size'])).hexdigest() == + plan['original_executable_sha256']['com.apple.iokit.IOMobileGraphicsFamily'], 'Original executable bytes changed') +obj = ROOT / 'build/xnu/ios-obj/DEVELOPMENT_ARM64_BCM2837/iokit/DEVELOPMENT/IOUserClient.cpo' +layouts = {} +for name, size, members in ( + ('IOExternalMethodDispatch', 24, dict(function=0, checkScalarInputCount=8, + checkStructureInputSize=12, checkScalarOutputCount=16, checkStructureOutputSize=20)), + ('IOExternalMethodArguments', 248, dict(scalarInput=32, scalarInputCount=40, + structureInput=48, structureInputSize=56, scalarOutput=72, + scalarOutputCount=80, structureOutput=88, structureOutputSize=96))): + dwarf = subprocess.check_output(['xcrun', 'dwarfdump', '--name=' + name, '--show-children', obj], text=True) + found = re.search(r'DW_AT_name\s+\("' + name + r'"\)[\s\S]{0,150}?DW_AT_byte_size\s+\((0x[0-9a-f]+)\)', dwarf) + require(found and int(found[1], 16) == size, 'Actual compiler structure size differs') + for member, offset in members.items(): + found = re.search(r'DW_AT_name\s+\("' + member + r'"\)[\s\S]*?DW_AT_data_member_location\s+\((0x[0-9a-f]+)\)', dwarf) + require(found and int(found[1], 16) == offset, 'Actual compiler field differs: ' + member) + layouts[name] = dict(bytes=size, fields=members) + +# The concrete factory's original constructor supplies this object's vptr. +vptr = address_pair(0xfffffff005776358, 8) + 16 +require(word(0xfffffff005776360) == 0x91004108 and vptr == 0xfffffff005f34dd0, + 'Original userclient constructor vptr differs') +compiled = obj.read_bytes() +compiled_image = macho(compiled) +sym_at = next(at for cmd, at, _ in compiled_image['commands'] if cmd == 2) +symoff, count, stroff, _ = struct.unpack_from('<4I', compiled, sym_at + 8) +compiled_symbols = [] +for i in range(count): + name, _, _, _, value = struct.unpack_from('> 27 & 1 and bits >> 28 == 0 and bits >> 25 & 3 == 3: + relocations[section['va'] + offset] = compiled_symbols[bits & 0xffffff][0] +slots = 0 +while compiled_base + 16 + slots * 8 in relocations: + slots += 1 +base = by_name['__ZTV12IOUserClient'] +require(slots == 186 and read(vptr - 16, 16) == bytes(16), 'Compiler-qualified native iOS IOUserClient table shape differs') +overrides = [] +for slot in range(slots): + parent, = struct.unpack('\n#include \n#include \n' + '#define PAGE_SIZE_FOR_SR_SLIDE ' + granule + '\n' + definition + '\n' + 'typedef struct vm_shared_region_slide_info_entry_v2 *vm_shared_region_slide_info_entry_v2_t;\n') +bridge = ''' +int probe_rebase(uint8_t *page, uint16_t start, uint32_t slide, void *info) { + return rebase_chain_64(page, start, slide, info); +} +unsigned probe_header_size(void) { return sizeof(struct vm_shared_region_slide_info_entry_v2); } +''' +unit = out / 'original-rebase.c' +unit.write_text(prefix + function + bridge) +library = out / 'original-rebase.dylib' +subprocess.run(['xcrun', 'clang', '-std=c11', '-Wall', '-Wextra', '-Werror', '-dynamiclib', + '-arch', 'arm64', unit, '-o', library], check=True) +native = ctypes.CDLL(str(library)) +native.probe_rebase.argtypes = [ctypes.c_void_p, ctypes.c_uint16, ctypes.c_uint32, ctypes.c_void_p] +native.probe_rebase.restype = ctypes.c_int +if native.probe_header_size() != struct.calcsize('<6I2Q'): + raise SystemExit('Compiled source metadata layout disagrees with actual encoded header') +audit = json.loads((ROOT / 'build/target-userspace/dyld-cache-audit.json').read_text()) +directory = ROOT / '.cache/firmware/target-volumes/systemos-mount/System/Library/Caches/com.apple.dyld' +records = [] +slide = 0x4000 + + +def independent_chain(page, start, mask, value_add): + shift = (mask & -mask).bit_length() - 1 - 2 + position = start + delta = 1 + changes = 0 + while delta and position <= 4088: + value = struct.unpack_from('> shift + value &= ~mask & ((1 << 64) - 1) + if value: + value = (value + value_add + slide) & ((1 << 64) - 1) + struct.pack_into(' 4088: + raise SystemExit('Actual source chain would escape a native 4K page') + return changes + + +for row in audit['cache_files']: + for mapping in row['mappings']: + if not mapping.get('slide_size'): + continue + with (directory / row['file']).open('rb') as stream: + stream.seek(mapping['slide_offset']) + metadata = stream.read(mapping['slide_size']) + version, page_size, starts_at, starts_count, extras_at, extras_count, mask, value_add = struct.unpack_from('<6I2Q', metadata) + if version != 2 or page_size != 4096 or starts_count * page_size != mapping['size'] or not mask >> 32: + raise SystemExit('Target ordinary ARM64 data mapping is not native-4K v2 metadata') + if starts_at + starts_count * 2 > len(metadata) or extras_at + extras_count * 2 > len(metadata): + raise SystemExit('Actual slide tables exceed their declared bounds') + starts = struct.unpack_from('<' + 'H' * starts_count, metadata, starts_at) + extras = struct.unpack_from('<' + 'H' * extras_count, metadata, extras_at) + active = [i for i, value in enumerate(starts) if value != 0x4000] + samples = sorted({active[i * (len(active) - 1) // 7] for i in range(8)}) + info = ctypes.create_string_buffer(metadata) + page_records = [] + for index in samples: + stream.seek(mapping['fileoff'] + index * 4096) + original = stream.read(4096) + if len(original) != 4096: + raise SystemExit('Cannot read the exact original 4K data page') + entry = starts[index] + if entry & 0x8000: + chain_starts = [] + extra_index = entry & 0x3fff + while True: + value = extras[extra_index] + chain_starts.append((value & 0x3fff) * 4) + extra_index += 1 + if value & 0x8000: + break + else: + chain_starts = [entry * 4] + expected = bytearray(original) + guarded = ctypes.create_string_buffer(b'G' * 32 + original + b'Z' * 32) + changes = 0 + for chain in chain_starts: + changes += independent_chain(expected, chain, mask, value_add) + if native.probe_rebase(ctypes.addressof(guarded) + 32, chain, slide, info) != 0: + raise SystemExit('Original Darwin 23 rebasing rejected this actual target page') + actual = guarded.raw[32:4128] + if actual != expected or guarded.raw[:32] != b'G' * 32 or guarded.raw[4128:4160] != b'Z' * 32: + raise SystemExit('Original source result differs or writes outside its 4K page') + if native.probe_rebase(ctypes.addressof(guarded) + 32, 4095, slide, info) != 5: + raise SystemExit('Original out-of-range chain refusal is not preserved') + page_records.append(dict(index=index, chains=len(chain_starts), rebased_words=changes, + original_sha256=hashlib.sha256(original).hexdigest(), + result_sha256=hashlib.sha256(actual).hexdigest())) + records.append(dict(file=row['file'], mapping_address=mapping['address'], mapping_bytes=mapping['size'], + slide_version=version, slide_page_bytes=page_size, + metadata_sha256=hashlib.sha256(metadata).hexdigest(), + sampled_pages=page_records)) +if len(records) != 4 or sum(len(r['sampled_pages']) for r in records) != 32: + raise SystemExit('Expected the four actual v2 mappings and 32 bounded real-page samples') +report = dict(source_revision=revision, source_function_sha256=hashlib.sha256(function.encode()).hexdigest(), + compiled_library_sha256=hashlib.sha256(library.read_bytes()).hexdigest(), + host_execution_only=True, hardware_translation_inferred_from_host_test=False, + original_source_algorithm_modified=False, metadata_or_cache_rewritten=False, + slide_amount=slide, native_slide_granule=4096, actual_mappings=records, + independent_results_match=True, page_guards_unchanged=True, + out_of_range_chain_refused=True, qemu_shared_region_pager_observed=False, + original_full_dyld_cache_loaded=False, hardware_tested=False) +(out / 'audit.json').write_text(json.dumps(report, indent=2) + '\n') +print('PASS: unchanged Darwin 23 64-bit rebase logic matches 32 real target 4K pages across all four v2 slide mappings') +print('Host execution/source audit only: full cache-loader, QEMU pager faults and hardware still need qualification') diff --git a/scripts/audit-storage-interface.py b/scripts/audit-storage-interface.py new file mode 100644 index 0000000..b26b81d --- /dev/null +++ b/scripts/audit-storage-interface.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python3 +"""Qualify original ordinary-ARM64 block-storage ABI using public Darwin 23 source.""" +import hashlib +import json +from pathlib import Path +import re +import shutil +import struct +import subprocess +import sys + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'kits/NativePrelink')) +import native_prelink as np +source = ROOT / 'refer/IOStorageFamily' +revision = subprocess.check_output(['git', '-C', source, 'rev-parse', 'HEAD'], text=True).strip() +if revision != 'e45fbe30a7879e0b16e26dddb78538f1b00e0bff': + raise SystemExit('Unexpected public Darwin 23 IOStorageFamily source') +out = ROOT / 'build/storage-abi' +headers = out / 'include/IOKit/storage' +headers.mkdir(parents=True, exist_ok=True) +for path in source.glob('*.h'): + shutil.copyfile(path, headers / path.name) +# Only this platform-selection header is missing from the source-kernel SDK. +# The reference object is an annotation instrument and is never linked/executed. +sdk = Path(subprocess.check_output(['xcrun', '--sdk', 'macosx', '--show-sdk-path'], text=True).strip()) +conditions = sdk / 'usr/include/TargetConditionals.h' +shutil.copyfile(conditions, out / 'include/TargetConditionals.h') +record = json.loads((ROOT / 'build/xnu/ios-obj/DEVELOPMENT_ARM64_BCM2837/iokit/DEVELOPMENT/IOPlatformExpert.cpo.json').read_text().strip().rstrip(',')) +obj_path = out / 'IOBlockStorageDevice-reference.o' +arguments = record['arguments'] +command, index = [], 0 +while index < len(arguments): + value = arguments[index] + if value == record['file']: + command.append(str(source / 'IOBlockStorageDevice.cpp')) + elif value == '-o': + command += ['-o', str(obj_path)] + index += 1 + elif value in ('-MF', '-MJ'): + index += 1 + elif value == '-Xclang' and arguments[index + 1].startswith('-W'): + index += 1 + elif not value.startswith('-W') and value not in ('-MD', '-MP'): + command.append(value) + index += 1 +command += ['-I', str(out / 'include'), '-I', str(ROOT / '.cache/xnu-work/osfmk'), + '-Wall', '-Wextra', '-Werror', '-Wno-unused-parameter', '-Wno-deprecated-declarations', + '-Wno-c99-designator', '-Wno-reorder-init-list', '-Wno-unnecessary-virtual-specifier', + '-Wno-deprecated-enum-enum-conversion'] +(out / 'compile-command.json').write_text(json.dumps(command, indent=2) + '\n') +r = subprocess.run(command, cwd=record['directory'], capture_output=True, text=True) +(out / 'compile.log').write_text(r.stdout + r.stderr) +if r.returncode: + raise SystemExit((r.stdout + r.stderr)[-3500:]) + + +def require(value, message): + if not value: + raise SystemExit(message) + + +obj = obj_path.read_bytes() +om = np.macho(obj) +require(om['kind'] == 1, 'Expected an ordinary ARM64 reference object') +dwarf = subprocess.check_output(['xcrun', 'dwarfdump', '--name=IOBlockStorageDevice', '--show-children', obj_path], text=True) +class_size = re.search(r'DW_AT_name\s+\("IOBlockStorageDevice"\)[\s\S]{0,200}?DW_AT_byte_size\s+\((0x[0-9a-f]+)\)', dwarf) +require(class_size and int(class_size[1], 16) == 0x90, 'Actual compiled public class size changed') +version_command = next(at for command, at, size in om['commands'] if command == 0x32) +require(struct.unpack_from(' vtable) +relocations = {} +for segment in om['segments']: + for section in segment['sections']: + if section['va'] <= vtable < section['va'] + section['size']: + ro, rc = struct.unpack_from('> 27 & 1 and word >> 28 == 0 and word >> 25 & 3 == 3: + relocations[section['va'] + address] = symbols[word & 0xffffff]['name'] +methods = [] +for at in range(vtable + 16, end, 8): + if at not in relocations: + break + methods.append(relocations[at]) +require(len(methods) == 190 and methods.count('___cxa_pure_virtual') == 14, 'Public iOS class dispatch shape changed') +kernel_path = ROOT / 'build/native-prelink/native-apfs-load-positive.macho' +kernel = kernel_path.read_bytes() +plan = json.loads(kernel_path.with_suffix('.json').read_text()) +require(hashlib.sha256(kernel).hexdigest() == plan['candidate_sha256'] and + plan['original_sha256'] == 'f18b4df865405f2a287b84977d7aca745e056d656d9fc9df1662970ce2969de7', 'Unexpected original candidate provenance') +native = np.macho(kernel) +exports = np.exports(kernel, native) +by_name = {name: va for va, names in exports.items() for name in names} +extents = [e for e in plan['extents'] if e['module'] == 'com.apple.iokit.IOStorageFamily'] +const = next(e for e in extents if e['segment'] == '__DATA_CONST') +text = next(e for e in extents if e['segment'] == '__TEXT_EXEC') + + +def read(va, length): + at = np.backed_offset(native['segments'], va, length, len(kernel)) + return kernel[at:at + length] + + +require(hashlib.sha256(read(text['va'], text['size'])).hexdigest() == + plan['original_executable_sha256']['com.apple.iokit.IOStorageFamily'], 'Original storage executable code changed') +blob = read(const['va'], const['size']) +found = [] +for at in range(0, len(blob) - (len(methods) + 2) * 8 + 1, 8): + if blob[at:at + 16] != bytes(16): + continue + pointers = struct.unpack_from('<' + 'Q' * len(methods), blob, at + 16) + if all(pointers[i] == by_name[name] for i, name in enumerate(methods) if name in by_name): + found.append((const['va'] + at, pointers)) +require(len(found) == 1, 'Cannot uniquely identify the original block-storage class dispatch') +table, pointers = found[0] +matched = sum(name in by_name for name in methods) +require(matched == 176, 'Unexpected original/native kernel binding count') +for i, name in enumerate(methods): + if name not in by_name: + require(text['va'] <= pointers[i] < text['va'] + text['size'], 'Owned storage method outside original text') + + +def relative_branch(address, instruction): + immediate = instruction & 0x3ffffff + if immediate & 0x2000000: + immediate -= 0x4000000 + return address + immediate * 4 + + +def thunk_names(address): + a, load, branch = struct.unpack('<3I', read(address, 12)) + require(a & 0x9f00001f == 0x90000010 and load & 0xffc003ff == 0xf9400210 and branch == 0xd61f0200, + 'Unqualified original native import thunk') + immediate = ((a >> 5 & 0x7ffff) << 2) | (a >> 29 & 3) + if immediate & 0x100000: + immediate -= 0x200000 + slot = (address & ~4095) + immediate * 4096 + (load >> 10 & 4095) * 8 + target = struct.unpack('> 5 & 0x7ffff) << 2) | (words[3] >> 29 & 3) + if immediate & 0x100000: + immediate -= 0x200000 + stored = ((address + 12) & ~4095) + immediate * 4096 + (words[4] >> 10 & 4095) + if stored == table + 16: + constructors.append((address, relative_branch(address + 8, words[2]))) +require(len(constructors) == 1 and '__ZN9IOServiceC2EPK11OSMetaClass' in thunk_names(constructors[0][1]), + 'Original protected block constructor does not forward to actual source IOService') +destructor = pointers[0] - 4 +word = struct.unpack('> 5 & 0x7ffff) << 2) | (a >> 29 & 3) +if immediate & 0x100000: + immediate -= 0x200000 +meta = (function & ~4095) + immediate * 4096 + (add >> 10 & 4095) +service_meta = by_name['__ZN9IOService10gMetaClassE'] +d = json.loads((ROOT / 'build/darwinboot-storage-native-metadata/gdb-stop.json').read_text()) +require(d['kernel_sha256'] == plan['candidate_sha256'] and not d['bounded_timeout_interrupted'], 'Wrong or incomplete live storage metadata observation') +actual = bytes.fromhex(d['memory'][f'{meta:x}:50']) +service = bytes.fromhex(d['memory'][f'{service_meta:x}:50']) +# The companion AES audit derives these member offsets from actual OSMetaClass DWARF. +layout = json.loads((ROOT / 'build/image4-ignition/aes-interface-audit.json').read_text())['compiled_metaclass_offsets'] +require(layout == dict(superClassLink=16, className=24, classSize=32), 'Compiled metaclass ABI changed') +require(struct.unpack_from('> 14 & 3 == 0 and r['TCR_EL1'] >> 30 & 3 == 2 and + r['SCTLR_EL1'] & 1 and r['TTBR0_EL1'] and r['TTBR1_EL1'], 'Live translation is not native 4K') +result = dict(public_source_revision=revision, source_sha256=hashlib.sha256((source / 'IOBlockStorageDevice.cpp').read_bytes()).hexdigest(), + reference_object_sha256=hashlib.sha256(obj).hexdigest(), reference_object_executed_or_linked=False, + platform_header_sdk_version=subprocess.check_output(['xcrun', '--sdk', 'macosx', '--show-sdk-version'], text=True).strip(), + platform_header_sha256=hashlib.sha256(conditions.read_bytes()).hexdigest(), + candidate_sha256=plan['candidate_sha256'], original_executable_code_unchanged=True, + method_slots=190, kernel_bindings=matched, pure_virtual_slots=14, inherited_kernel_methods=162, + vtable=hex(table), original_metaclass=hex(meta), source_ioservice_metaclass=hex(service_meta), + original_protected_constructor=hex(constructors[0][0]), original_base_destructor=hex(destructor), + abstract_complete_destructors_trap=True, + live_instance_sizes=dict(IOBlockStorageDevice=0x90, IOService=0x88), live_native_4k=True, + compiled_public_class_size=int(class_size[1], 16), + methods=[dict(slot=i, reference_name=name, candidate_target=hex(pointers[i]), kernel_names=exports.get(pointers[i], [])) + for i, name in enumerate(methods)], disk_read_tested=False, transport_implemented=False, hardware_tested=False) +(out / 'interface-audit.json').write_text(json.dumps(result, indent=2) + '\n') +print('PASS: 190 ordinary iOS storage slots; 176 kernel bindings (162 inherited + 14 pure virtual)') +print('Live original block/service sizes 0x90/0x88 and native 4K. Annotation object is never linked/executed; block I/O remains unqualified.') diff --git a/scripts/audit-target-accelerator-base.py b/scripts/audit-target-accelerator-base.py new file mode 100644 index 0000000..55f624a --- /dev/null +++ b/scripts/audit-target-accelerator-base.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +"""Derive the exact original IOAcceleratorES base ABI without publishing a device.""" +import hashlib,json,struct,sys +from pathlib import Path +ROOT=Path(__file__).resolve().parents[1];sys.path.insert(0,str(ROOT/'kits/NativePrelink')) +from native_prelink import macho,backed_offset,exports +def require(v,s): + if not v:raise SystemExit(s) +def sha(p):return hashlib.sha256(p.read_bytes()).hexdigest() +p=ROOT/'build/native-prelink/native-owner-renderbox-presentation-candidate.macho';plan=json.loads(p.with_suffix('.json').read_text());raw=p.read_bytes();im=macho(raw) +require(sha(p)==plan['candidate_sha256'],'Exact immutable candidate') +def read(a,n): + at=backed_offset(im['segments'],a,n,len(raw));return raw[at:at+n] +def adrp(pc,word): + imm=((word>>5)&0x7ffff)*4+((word>>29)&3) + if imm&(1<<20):imm-=1<<21 + return (pc&~4095)+(imm<<12) +def branch(pc): + w=struct.unpack('>10)&4095)*8,8))[0] +names={n:a for a,group in exports(raw,im).items() for n in group} +runtime_path=ROOT/'build/darwinboot-owner-hvf-iogpu-gpu-class/gdb-stop.json';runtime=json.loads(runtime_path.read_text());parent=bytes.fromhex(runtime['entry_regions']['2:0:80']['bytes']);meta=int(runtime['function_entry_arguments'][2],16) +require(runtime['function_return_stack_matched'] and meta==0xfffffff006aeb688 and struct.unpack_from('>10)&4095)==meta:getters.append(pc) +require(len(getters)==1,'Unique exact original getMetaClass body') +base=struct.unpack('<168Q',read(names['__ZTV9IOService']+16,168*8));tables=[] +for e in es: + if e['segment']=='__TEXT_EXEC':continue + data=read(e['va'],e['size']);at=0 + while True: + i=data.find(struct.pack('>10)&4095)!=vp:continue + start=pc-12;w=struct.unpack('<8I',read(start,32)) + require(w[0]==0xa9bf7bfd and w[1]==0x910003fd and w[6]==0xa8c17bfd and w[7]==0xd65f03c0 and + imported(branch(start+8))==names['__ZN9IOServiceC2EPK11OSMetaClass'],'Exact constructor only calls genuine IOService C2 and installs its own vptr') + constructors.append(start) +require(len(constructors)==2 and imported(branch(v[0]))==names['__ZN9IOServiceD2Ev'],'Both identical constructor aliases and real non-deleting destructor chain') +report=dict(candidate_sha256=plan['candidate_sha256'],original_iogpu_text_sha256=hashlib.sha256(code).hexdigest(),runtime_parent_evidence_sha256=sha(runtime_path), + metaclass=hex(meta),instance_bytes=136,vptr=hex(vp),virtual_entries=168,operational_overrides=[],metadata_overrides=[0,1,7], + protected_compatible_constructor_aliases=[hex(a) for a in constructors],nondeleting_destructor=hex(v[0]),get_meta_class=hex(getters[0]), + original_entry_sha256={hex(a):hashlib.sha256(read(a,32)).hexdigest() for a in (*constructors,v[0],getters[0])}, + native_iogpu_instance_created=False,system_device_published=False,system_metal=False,original_desktop=False) +(ROOT/'build/gpu-analysis/ioaccelerator-base-abi.json').write_text(json.dumps(report,indent=2)+'\n') +print('PASS: exact original IOAcceleratorES is a 136-byte, 168-entry IOService base with no operational overrides; real C2/D2 chains derived') +print('Static/runtime-parent metadata evidence only: no accelerator instance, normal Metal device or desktop created') diff --git a/scripts/audit-target-air-inputs.py b/scripts/audit-target-air-inputs.py new file mode 100644 index 0000000..5008679 --- /dev/null +++ b/scripts/audit-target-air-inputs.py @@ -0,0 +1,205 @@ +#!/usr/bin/env python3 +"""Audit exact original UI AIR inputs; decode only, never compile/execute a GPU shader.""" +import concurrent.futures +from collections import Counter +import hashlib +import json +from pathlib import Path +import re +import struct +import subprocess +import time + +ROOT = Path(__file__).resolve().parents[1] +BASE = ROOT / '.cache/firmware/target-volumes/os-mount/System/Library' +OUT = ROOT / 'build/gpu-analysis/air' +LLVM_ROOT = Path('/opt/homebrew/opt/llvm') +LLVM_DIS = LLVM_ROOT / 'bin/llvm-dis' +PINS = { + 'Frameworks/QuartzCore.framework/default.metallib': + '606f913adf520420f9b4bb666efb782a68c830f082f1e559e17b20d0d13adbc9', + 'PrivateFrameworks/RenderBox.framework/default.metallib': + 'd76883f626b1527c36fb0940cf27dcade503a55cf2fa05c8f7a247a40ccebec4', +} + + +def require(value, message): + if not value: + raise ValueError(message) + + +def extent(data, start, length): + require(0 <= start <= len(data) and 0 <= length <= len(data) - start, 'Original input extent outside file') + return data[start:start + length] + + +def slices(data): + if data[:4] != b'\xca\xfe\xba\xbe': + return [(dict(offset=0, bytes=len(data)), data)] + n, = struct.unpack_from('>I', data, 4) + require(n == 2, 'Exact target QuartzCore has two bounded universal slices') + records = [] + for i in range(n): + cpu, subtype, offset, size, alignment = struct.unpack('>5I', extent(data, 8 + 20 * i, 20)) + require(offset >= 8 + 20 * n and alignment <= 16 and not offset % (1 << alignment), 'Bounded original universal slice alignment') + records.append((dict(cpu=hex(cpu), subtype=hex(subtype), offset=offset, bytes=size), extent(data, offset, size))) + require(records[0][0]['offset'] + records[0][0]['bytes'] <= records[1][0]['offset'], 'Disjoint original universal slices') + return records + + +def entries(data): + require(data[:16] == bytes.fromhex('4d544c42010002000700008211000700') and + struct.unpack_from('= 20 and bc_size > 4 and cpu == 0xffffffff, + 'Supported original LLVM bitcode wrapper') + bc = extent(wrapper, offset, bc_size) + require(bc[:4] == b'BC\xc0\xde', 'Actual LLVM bitcode signature, not a coincidental text string') + records.append(dict(index=index, name=name[:-1].decode('ascii'), type=tags[b'TYPE'][0], + module_offset=bitcode, module_bytes=size, wrapper_sha256=tags[b'HASH'].hex(), + bitcode_sha256=hashlib.sha256(bc).hexdigest(), bitcode=bc)) + cursor = end + require(cursor == at + 4 + length and cursor + 48 == sections[1][0], + 'Exact original function bytes: declared size excludes the four-byte record count') + require(data[cursor:cursor + 6] == b'RLST\x10\x00' and data[cursor + 22:cursor + 28] == b'UUID\x10\x00' and + data[cursor + 44:cursor + 48] == b'ENDT', 'Exact original reflection/UUID trailer profile') + reflection, reflection_size = struct.unpack_from('<2Q', data, cursor + 6) + require(reflection == sections[3][0] + sections[3][1] and reflection + reflection_size == len(data), + 'Original trailing reflection extent, not an unbounded bitcode scan') + intervals = sorted(set((r['module_offset'], r['module_offset'] + r['module_bytes']) for r in records)) + require(all(a[1] <= b[0] for a, b in zip(intervals, intervals[1:])), 'Original module extents do not partially overlap') + return records + + +OUT.mkdir(parents=True, exist_ok=True) +version = subprocess.check_output([LLVM_DIS, '--version'], text=True).strip() +require('LLVM version 23.1.3' in version, 'Qualify this audit with the actual inspected host LLVM version') +inputs, jobs = [], [] +for file, expected in PINS.items(): + data = (BASE / file).read_bytes() + require(hashlib.sha256(data).hexdigest() == expected, 'Exact unchanged target library: ' + file) + library = Path(file).parent.stem + views = slices(data) + # The observed QuartzCore AIR-bearing slice is the first. Its classification + # comes from real decoded triples, not assumptions about numeric CPU types. + record, selected = views[0] + modules = entries(selected) + inputs.append(dict(file=file, sha256=expected, bytes=len(data), slices=[r for r, _ in views], + selected_slice=record, functions=len(modules))) + for module in modules: + module['library'] = library + jobs.append(module) + +# Verify the original triple in-process through LLVM's module API, without +# creating a target machine or overriding AIR with another architecture. +verifier_source = OUT / 'verify-air.c' +verifier_source.write_text(r'''#include +#include +#include +#include +#include +int main(int argc, char **argv) { + if (argc != 2) return 2; + LLVMMemoryBufferRef buffer = NULL; + LLVMModuleRef module = NULL; + char *message = NULL; + LLVMContextRef context = LLVMContextCreate(); + int status = 1; + if (LLVMCreateMemoryBufferWithContentsOfFile(argv[1], &buffer, &message)) goto done; + if (LLVMParseBitcodeInContext2(context, buffer, &module)) goto done; + if (strcmp(LLVMGetTarget(module), "air64-apple-ios17.7.0")) goto done; + if (LLVMVerifyModule(module, LLVMReturnStatusAction, &message)) goto done; + if (strcmp(LLVMGetTarget(module), "air64-apple-ios17.7.0")) goto done; + status = 0; +done: + if (message) { fputs(message, stderr); LLVMDisposeMessage(message); } + if (module) LLVMDisposeModule(module); + if (buffer) LLVMDisposeMemoryBuffer(buffer); + LLVMContextDispose(context); + return status; +} +''') +verifier = OUT / 'verify-air' +subprocess.run(['xcrun', 'clang', '-std=c11', '-O2', '-Wall', '-Wextra', '-Werror', + '-I', LLVM_ROOT / 'include', verifier_source, '-L', LLVM_ROOT / 'lib', + '-lLLVM', '-Wl,-rpath,' + str(LLVM_ROOT / 'lib'), '-o', verifier], check=True, timeout=30) + +deadline = time.monotonic() + 60 + + +def decode(module): + require(time.monotonic() < deadline, 'Bounded whole-library decoding deadline') + bc = module.pop('bitcode') + prefix = OUT / module['library'] / ('module-' + str(module['index'])) + prefix.parent.mkdir(parents=True, exist_ok=True) + prefix.with_suffix('.bc').write_bytes(bc) + result = subprocess.run([LLVM_DIS, prefix.with_suffix('.bc'), '-o', prefix.with_suffix('.ll')], + text=True, capture_output=True, timeout=8) + require(result.returncode == 0, 'Actual LLVM bitcode decoding failed: ' + module['library'] + '/' + module['name'] + ': ' + result.stderr[:200]) + verified = subprocess.run([verifier, prefix.with_suffix('.bc')], text=True, capture_output=True, timeout=8) + require(verified.returncode == 0, 'LLVM original-triple module verification failed: ' + module['library'] + '/' + module['name'] + ': ' + verified.stderr[:200]) + text = prefix.with_suffix('.ll').read_text() + triple = re.search(r'^target triple = "([^"]+)"$', text, re.M) + require(triple and triple[1] == 'air64-apple-ios17.7.0', 'Actual original AIR target triple') + stages = [stage for stage in ('vertex', 'fragment', 'kernel', 'visible') if re.search(r'^!air\.' + stage + r' = ', text, re.M)] + require(len(stages) == 1, 'One actual original function-kind metadata record: ' + module['library'] + '/' + module['name']) + # This target's TYPE interpretation is checked against decoded AIR metadata. + require(module['type'] == {'vertex': 0, 'fragment': 1, 'kernel': 2, 'visible': 4}[stages[0]], 'Actual TYPE agrees with original AIR stage metadata') + definitions = re.findall(r'^define .*?@(?:"([^"]+)"|([^\s(]+))\(', text, re.M) + require(module['name'] in {a or b for a, b in definitions}, 'Original exported function is defined by its own decoded module') + module.update(triple=triple[1], stage=stages[0], definitions=len(definitions), llvm_module_verified=True, + air_intrinsics=sorted(set(re.findall(r'@(?:"(air\.[^"]+)"|(air\.[^\s(]+))\(', text)))) + module['air_intrinsics'] = sorted({a or b for a, b in module['air_intrinsics']}) + return module + + +with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool: + modules = list(pool.map(decode, jobs)) +report = dict(inputs=inputs, llvm_dis_version=version, + llvm_dis_sha256=hashlib.sha256(LLVM_DIS.read_bytes()).hexdigest(), + modules=modules, stages=dict(Counter(m['stage'] for m in modules)), + intrinsics=sorted({n for m in modules for n in m['air_intrinsics']}), + original_files_modified=False, host_bitcode_decode_only=True, + llvm_original_triple_module_verification=True, + llvm_library_sha256=hashlib.sha256((LLVM_ROOT / 'lib/libLLVM.dylib').read_bytes()).hexdigest(), + apple_bundle_signature_runtime_acceptance_qualified=False, + adreno_code_generated=False, shader_execution_qualified=False, metal_api_qualified=False) +(OUT / 'target-inputs.json').write_text(json.dumps(report, indent=2) + '\n') +print('PASS:', len(modules), 'original UI AIR modules decoded/verified with real triples/function-kind/export metadata and complete module hashes') +print('Stages:', report['stages'], '; distinct AIR intrinsics:', len(report['intrinsics'])) +print('Host input qualification only: no Adreno shader generation/execution, Metal API or UI qualification') diff --git a/scripts/audit-target-cache-leaf.py b/scripts/audit-target-cache-leaf.py new file mode 100644 index 0000000..4c6cd80 --- /dev/null +++ b/scripts/audit-target-cache-leaf.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +"""Locate the original cache's public getuid stub without rewriting any input.""" +import hashlib +import json +from pathlib import Path +import struct +import subprocess + +ROOT = Path(__file__).resolve().parents[1] +source = ROOT / 'refer/dyld' +revision = subprocess.check_output(['git', '-C', source, 'rev-parse', 'HEAD'], text=True).strip() +if revision != 'a571176e8e00c47e95b95e3156820ebec0cbd5e6': + raise SystemExit('Expected the pinned Darwin 23 dyld format source') +out = ROOT / 'build/cache-leaf-host' +out.mkdir(parents=True, exist_ok=True) +unit = out / 'offsets.cpp' +unit.write_text('''#include +#include +#include +#include +#include "dyld_cache_format.h" +int main() { printf("%zu %zu\\n", offsetof(dyld_cache_header, imagesOffset), sizeof(dyld_cache_image_info)); } +''') +subprocess.run(['xcrun', 'clang++', '-std=c++17', '-Wall', '-Wextra', '-Werror', '-I', + source / 'cache-builder', unit, '-o', out / 'offsets'], check=True) +images_field, image_size = map(int, subprocess.check_output([out / 'offsets'], text=True).split()) +if image_size != 32: + raise SystemExit('Unexpected original image table ABI') +directory = ROOT / '.cache/firmware/target-volumes/systemos-mount/System/Library/Caches/com.apple.dyld' +main = (directory / 'dyld_shared_cache_arm64').read_bytes() +audit = json.loads((ROOT / 'build/target-userspace/dyld-cache-audit.json').read_text()) + + +def read(va, size): + for row in audit['cache_files']: + for mapping in row['mappings']: + start = int(mapping['address'], 16) + if start <= va and va + size <= start + mapping['size']: + offset = mapping['fileoff'] + va - start + with (directory / row['file']).open('rb') as stream: + stream.seek(offset) + data = stream.read(size) + if len(data) != size: + raise SystemExit('Short original cache read') + return data, row['file'], offset + raise SystemExit('Address is outside actual target mappings') + + +at, count = struct.unpack_from(' 4096: + raise SystemExit('The original getuid success path is not self-contained in one native page') +path = directory / filename +with path.open('rb') as stream: + cache_header = stream.read(56) + sig_offset, sig_size = struct.unpack_from('I', signature, 8)[0]): + _, offset = struct.unpack_from('>II', signature, 12 + index * 8) + magic, length = struct.unpack_from('>II', signature, offset) + if magic == 0xfade0c02: + cd = signature[offset:offset + length] + code_directories.append((hashlib.sha256(cd).digest()[:20].hex(), cd[39])) +expected = next(r for r in audit['cache_files'] if r['file'] == filename)['code_directories'] +if len(code_directories) != 1 or code_directories != [(expected[0]['cdhash'], 14)]: + raise SystemExit('Expected the actual unchanged 16K-signed original cache file') +digest = hashlib.sha256() +with path.open('rb') as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b''): + digest.update(chunk) +report = dict(source_revision=revision, library='/usr/lib/system/libsystem_kernel.dylib', + image_address=hex(image), public_export='_getuid', original_address=hex(target), + file=filename, file_bytes=path.stat().st_size, file_sha256=digest.hexdigest(), + file_offset=file_offset, native_page_offset=file_offset & ~4095, + within_page_offset=file_offset & 4095, function_bytes=code.hex(), + original_page_sha256=hashlib.sha256(native_page).hexdigest(), + success_return_offset=36, syscall_number=24, signature_offset=sig_offset, + signature_size=sig_size, original_cdhash=code_directories[0][0], signing_block_bytes=16384, + original_input_modified=False, host_source_analysis_only=True, runtime_executed=False) +(out / 'getuid.json').write_text(json.dumps(report, indent=2) + '\n') +print('PASS: actual public getuid export located in original 16K-signed cache; runtime still unqualified') +print(filename, hex(file_offset), digest.hexdigest()) diff --git a/scripts/audit-target-display-query.py b/scripts/audit-target-display-query.py new file mode 100644 index 0000000..dfc93f5 --- /dev/null +++ b/scripts/audit-target-display-query.py @@ -0,0 +1,259 @@ +#!/usr/bin/env python3 +"""Bind an offline display-query observation plan to unchanged target images.""" +import hashlib +import argparse +import json +import mmap +from pathlib import Path +import re +import plistlib +import struct +import subprocess + +ROOT = Path(__file__).resolve().parents[1] +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument('--discovery', action='store_true') +args = parser.parse_args() +cache = ROOT / '.cache/firmware/target-volumes/systemos-mount/System/Library/Caches/com.apple.dyld' +launchd = ROOT / '.cache/firmware/target-volumes/os-mount/sbin/launchd' +if hashlib.sha256(launchd.read_bytes()).hexdigest() != '8cec9a3c44a443b08e977ccc5d577850316ad428c575dc01f09b018dbd5b6a25': + raise SystemExit('Require the unchanged supplied launchd') +if subprocess.check_output(['git', '-C', ROOT / 'refer/dyld', 'rev-parse', 'HEAD'], text=True).strip() != 'a571176e8e00c47e95b95e3156820ebec0cbd5e6': + raise SystemExit('Require pinned dyld-1165.3 cache format') +work = ROOT / 'build/display-analysis' +work.mkdir(parents=True, exist_ok=True) +offsets = work / ('discovery-cache-offsets' if args.discovery else 'query-cache-offsets') +source = offsets.with_suffix('.cpp') +source.write_text('''#include +#include +#include "dyld_cache_format.h" +int main(){printf("%zu %zu %zu %zu %zu %zu %zu %zu %zu %zu\\n", +offsetof(dyld_cache_header,imagesTextOffset),offsetof(dyld_cache_header,localSymbolsOffset), +offsetof(dyld_cache_header,sharedRegionStart),sizeof(dyld_cache_image_text_info), +sizeof(dyld_cache_local_symbols_info),sizeof(dyld_cache_local_symbols_entry_64), +sizeof(dyld_cache_local_symbols_entry),offsetof(dyld_cache_header,symbolFileUUID), +offsetof(dyld_cache_header,mappingWithSlideOffset),sizeof(dyld_cache_mapping_and_slide_info));} +''') +subprocess.run(['xcrun', 'clang++', '-std=c++17', '-Wall', '-Wextra', '-Werror', '-I', + ROOT / 'refer/dyld/cache-builder', source, '-o', offsets], check=True) +expected_offsets = [136, 72, 224, 32, 24, 16, 12, 400, 312, 56] +if list(map(int, subprocess.check_output([offsets], text=True).split())) != expected_offsets: + raise SystemExit('Require the existing compiler-derived cache ABI qualification') +files, mappings, slide_mappings = [], [], [] +for path in sorted(cache.glob('dyld_shared_cache_arm64*')): + if path.suffix == '.symbols': + continue + stream = path.open('rb') + data = mmap.mmap(stream.fileno(), 0, access=mmap.ACCESS_READ) + files.append((path, stream, data)) + if data[:16].rstrip(b'\0') != b'dyld_v1 arm64': + raise SystemExit('Require original ARM64 cache mappings') + at, count = struct.unpack_from('<2I', data, 16) + for i in range(count): + va, size, off, _, prot = struct.unpack_from('<3Q2I', data, at + i * 32) + mappings.append((va, va + size, off, data, path.name, prot)) + at, count = struct.unpack_from('<2I', data, 312) + for i in range(count): + va, size, _, off, length, _, _, _ = struct.unpack_from('<6Q2I', data, at + i * 56) + if length and struct.unpack_from(' +#include +#include +#include +#include "dyld_cache_format.h" +int main() { + printf("%zu %zu %zu %zu %zu %zu\\n", offsetof(dyld_cache_header, codeSignatureOffset), offsetof(dyld_cache_header, mappingWithSlideOffset), offsetof(dyld_cache_header, imagesCount), sizeof(dyld_cache_mapping_and_slide_info), offsetof(dyld_cache_header, sharedRegionStart), offsetof(dyld_cache_header, sharedRegionSize)); +} +''') +subprocess.run(['xcrun', 'clang++', '-std=c++17', '-Wall', '-Wextra', '-Werror', '-I', + SOURCE / 'cache-builder', cpp, '-o', out / 'cache-offsets'], check=True) +cs_offset, mapping_offset, images_offset, mapping_size, region_start_offset, region_size_offset = map( + int, subprocess.check_output([out / 'cache-offsets'], text=True).split()) +cache_dir = ROOT / '.cache/firmware/target-volumes/systemos-mount/System/Library/Caches/com.apple.dyld' +files = sorted(path for path in cache_dir.glob('dyld_shared_cache_arm64*') if path.suffix != '.symbols') +if not files: + raise SystemExit('Mount the manifest-selected SystemOS volume read-only first') +results = [] +first_block = None +for path in files: + with path.open('rb') as stream: + header = stream.read(512) + if header[:16].rstrip(b'\0') != b'dyld_v1 arm64': + raise SystemExit(f'{path.name}: unexpected shared-cache architecture') + sigoff, sigsize = struct.unpack_from('3I', signature) + if magic != 0xfade0cc0 or length > len(signature): + raise SystemExit('Invalid cache signature container') + directories = [] + for i in range(count): + slot, at = struct.unpack_from('>II', signature, 12 + i * 8) + magic, length = struct.unpack_from('>II', signature, at) + if magic != 0xfade0c02: + continue + cd = signature[at:at + length] + version, flags, hashoff, identoff, special, slots, limit = struct.unpack_from('>7I', cd, 8) + hashsize, hashtype, platform, shift = struct.unpack_from('4B', cd, 36) + if version >= 0x20300: + limit = struct.unpack_from('>Q', cd, 56)[0] or limit + if hashtype != 2 or hashsize != 32 or shift not in (12, 14): + raise SystemExit('Unexpected original cache hash algorithm or block size') + blocksize = 1 << shift + if slots != (limit + blocksize - 1) // blocksize: + raise SystemExit('Unexpected cache slot count') + stream.seek(0) + for page in range(slots): + block = stream.read(min(blocksize, limit - page * blocksize)) + digest = hashlib.sha256() + # Hash the unchanged complete signing block through 4K pieces. + for start in range(0, len(block), 4096): + digest.update(block[start:start + 4096]) + expected = cd[hashoff + page * 32:hashoff + (page + 1) * 32] + if digest.digest() != expected: + raise SystemExit(f'{path.name}: original block {page} signature mismatch') + if first_block is None and len(block) == 16384: + first_block = (block, expected) + directories.append({'cdhash': hashlib.sha256(cd).hexdigest()[:40], + 'hash_block_bytes': blocksize, 'original_code_slots_verified': slots}) + if not directories: + raise SystemExit('Missing original cache CodeDirectory') + results.append({'file': path.name, 'mappings': mappings, 'code_directories': directories}) +if first_block is None: + raise SystemExit('Expected an original 16K signature block for negative checks') +block, expected = first_block +for quarter in range(4): + corrupt = bytearray(block) + corrupt[quarter * 4096] ^= 1 + digest = hashlib.sha256() + for start in range(0, len(corrupt), 4096): + digest.update(corrupt[start:start + 4096]) + if digest.digest() == expected: + raise SystemExit('Corruption in a native 4K piece was not detected') +report = {'cache_files': results, 'files_checked': len(results), + 'all_mappings_directly_native_4k_aligned': True, + 'original_complete_signature_blocks_verified': True, 'corruption_in_each_4k_quarter_rejected': True, + 'code_directory_or_image_rewritten': False, 'kernel_large_hash_support_implemented': False, + 'native_4k_userspace_execution_observed': False} +(out / 'dyld-cache-audit.json').write_text(json.dumps(report, indent=2) + '\n') +print(f'PASS: {len(results)} unchanged ARM64 cache files, native 4K mapping alignment, original complete signature hashes') +print('PASS: corrupted bytes in each of four native 4K pieces invalidate the original 16K signature block') +print('Kernel fault-path integration for larger signature blocks remains required') diff --git a/scripts/audit-target-iogpu-client-contract.py b/scripts/audit-target-iogpu-client-contract.py new file mode 100755 index 0000000..a8c028f --- /dev/null +++ b/scripts/audit-target-iogpu-client-contract.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +"""Derive original client factory, mapping and provider/device dependencies.""" +import hashlib,json,runpy,struct +from pathlib import Path +ROOT=Path(__file__).resolve().parents[1] +provider=runpy.run_path(str(ROOT/'scripts/audit-target-iogpu-start-contract.py')) +client=runpy.run_path(str(ROOT/'scripts/audit-target-iogpu-userclient.py')) +n=provider['n'];read,require=n['read'],n['require'];values=provider['abi']['inherited_entries'];uc=client['values'] +def words(address,size):return struct.unpack('<'+'I'*(size//4),read(address,size)) +def entry(address,size):return dict(address=hex(address),bytes=size,sha256=hashlib.sha256(read(address,size)).hexdigest()) +new_client=values[141];factory=words(new_client,372) +require(factory[10:17]==(0xaa0403f4,0xaa0303f7,0xf900009f,0x52800028,0xb9017e68,0x12003c68,0x7100851f),'Actual output x4 cleared, saved open type x3 and low16 type selector') +require(factory[18]==0x7100051f and factory[20:24]==(0xaa0103f6,0xf9400268,0xf9431d08,0xaa1303e0),'Actual type1 factory dispatches original slot199 with real this') +require(factory[26:35]==(0xaa0003f5,0x53107ef7,0xf9400008,0xf9401108,0xd63f0100,0xaa1503e0,0xd2800001,0xaa1603e2,0xaa1703e3),'Actual high16 open type passed as scalar x3, real owning task x2 and NULL properties x1') +initialize=n['branch'](new_client+35*4);init=words(initialize,148) +require(init[4:7]==(0xaa0303f5,0xaa0203f4,0xaa0003f3) and init[23:25]==(0xf9000114,0xf9009675),'Actual custom initializer saves task and scalar subtype, not an output-pointer argument') +require(factory[55:61]==(0xf94002a8,0xf941b108,0xaa1503e0,0xaa1303e1,0xd63f0100,0x340002c0) and factory[61:69]==(0xf94002a8,0xf9415908,0xaa1503e0,0xaa1303e1,0xd63f0100,0x34000160,0x52800000,0xf9000295),'Actual original start must succeed before output object is published to caller') +require(factory[6:9]==(0x528057d8,0x72bc0018,0x11001300) and factory[77:88]==(0xf94002a8,0xf941b508,0xaa1503e0,0xaa1303e1,0xd63f0100,0x11002f18,0xf94002a8,0xf9401508,0xaa1503e0,0xd63f0100,0xaa1803e0),'Actual failed original start detaches/releases client and returns original 0xe00002c9') + +initialize_device=uc[186];body=words(initialize_device,436) +require(body[6:12]==(0xf9407c00,0x91046261,0xf9400008,0xf942a908,0xd63f0100,0xf9008660),'First required slot170 gets client+280 output and returns actual descriptor stored at+264') +require(body[13:16]==(0xf9400008,0xf9401108,0xd63f0100),'Actual returned shared descriptor is retained') +require(body[16:24]==(0xf9408660,0xf9406e68,0xf9400101,0xd2800002,0x52820023,0x72a08003,0xd2800004,0xd2800005),'Actual owner task and native createMappingInTask options 0x04001001 with zero address/offset/length') +mapping=n['imported'](n['branch'](initialize_device+24*4)) +require(mapping==n['names']['__ZN18IOMemoryDescriptor19createMappingInTaskEP4taskyjyy'],'Actual native IOMemoryDescriptor mapping import') +require(body[25:35]==(0xf9008a60,0xb4000660,0xf9407e60,0xf9406e68,0xf9400101,0xf9409662,0xf9400008,0xf942c508,0xd63f0100,0xf9009260),'Mapping stored+272; slot189 receives real owning task and loaded subtype scalar; device stored+288') +require(body[12]==0xb40006a0 and body[76]==0x14000018 and body[100:104]==(0xaa1303e0,0x9400000b,0x52800014,0xaa1403e0) and n['branch'](initialize_device+404)==uc[187],'Actual NULL descriptor follows original error/cleanup path and returns false') +first=provider['values'][170];first_words=struct.unpack('<3I',provider['original_read'](first,12)) +require(first_words==(0xb900003f,0xf9421800,0xd65f03c0),'Actual concrete slot170 writes uint32 output and returns descriptor from own+1072') +second=provider['values'][189];second_words=struct.unpack('<36I',provider['original_read'](second,144)) +require(second_words[4:10]==(0xaa0103f4,0xaa0003f5,0xf9400008,0xf9440508,0xd63f0100,0xb40002e0),'Actual concrete slot189 allocates via own slot257, not an invented device') +require(second_words[10:18]==(0xaa0003f3,0xf9400008,0xf9405508,0xaa1503e1,0xaa1403e2,0xd2800003,0xd63f0100,0x34000220),'Actual concrete device slot21 receives original provider/task/zero; failed init refuses') +# Pin the exact original cleanup, including guarded release of actual mapping +# and descriptor. A missing backend may refuse; an empty buffer is not a device. +cleanup=uc[187];clean=words(cleanup,112) +require(clean[12:25]==(0xf9408a60,0xb4000080,0xf9400008,0xf9401508,0xd63f0100,0xf9008a7f,0xf9408660,0xb4000080,0xf9400008,0xf9401508,0xd63f0100,0xf900867f,0x3904027f),'Actual cleanup guards/releases mapping+272 and descriptor+264, then clears active flag') +report=dict(original_kernel_sha256=provider['n']['plan']['original_sha256'],original_iogpu_text_sha256=provider['n']['plan']['original_executable_sha256']['com.apple.iokit.IOGPUFamily'], + new_user_client=entry(new_client,372),custom_client_initialize=entry(initialize,148),initialize_device=entry(initialize_device,436),cleanup=entry(cleanup,112), + device_factory_open_type_low16=1,device_factory_open_subtype_shift=16,client_subtype_offset=296,shared_descriptor_slot=170,shared_descriptor_out_offset=280,shared_descriptor_offset=264, + mapping_import='__ZN18IOMemoryDescriptor19createMappingInTaskEP4taskyjyy',mapping_options=0x04001001,mapping_offset=272,device_factory_slot=189,device_offset=288, + first_required_callback_caller_lr=hex(initialize_device+44),absent_descriptor_factory_error=0xe00002c9-(1<<32),concrete_shared_descriptor_field=1072,concrete_device_allocator_slot=257,concrete_device_initializer_slot=21, + source_analysis_only=True,source_signatures_complete=False,original_client_started=False,backend_resources_implemented=False,normal_device_discovery=False,original_desktop=False,adreno_execution=False) +(ROOT/'build/gpu-analysis/original-iogpu-client-contract.json').write_text(json.dumps(report,indent=2)+'\n') +print('PASS: original type1 factory/custom task+subtype initialization, shared descriptor/map, required device object and guarded cleanup derived') +print('No successful client start or empty-buffer/device substitute is qualified') diff --git a/scripts/audit-target-iogpu-provider.py b/scripts/audit-target-iogpu-provider.py new file mode 100644 index 0000000..0ed7d76 --- /dev/null +++ b/scripts/audit-target-iogpu-provider.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +"""Derive the genuine original abstract IOGPU provider and required dispatch ABI.""" +import hashlib,json,runpy,struct +from pathlib import Path +ROOT=Path(__file__).resolve().parents[1] +n=runpy.run_path(str(ROOT/'scripts/audit-target-accelerator-base.py'));read,require,adrp=n['read'],n['require'],n['adrp'];code,text=n['code'],n['text'] +runtime_path=ROOT/'build/darwinboot-owner-hvf-iogpu-gpu-class/gdb-stop.json';runtime=json.loads(runtime_path.read_text());args=[int(v,16) for v in runtime['function_entry_arguments']];meta=args[0] +require(runtime['function_return_stack_matched'] and runtime['entry_kernel_bytes_match']['bytes']==b'IOGPU\0'.hex() and args[3]==736 and args[2]==int(n['meta']),'Previously observed actual 736-byte IOGPU and original accelerator parent') +getters=[] +for i in range(0,len(code)-12,4): + pc=text['va']+i;a,b,c=struct.unpack_from('<3I',code,i) + if a&0x9f00001f==0x90000000 and b&0xffc003ff==0x91000000 and c==0xd65f03c0 and adrp(pc,a)+((b>>10)&4095)==meta:getters.append(pc) +require(len(getters)==1,'Unique actual original IOGPU getMetaClass') +base=struct.unpack('<168Q',read(n['vp'],168*8));tables=[] +for e in n['es']: + if e['segment']=='__TEXT_EXEC':continue + data=read(e['va'],e['size']);at=0 + while True: + pos=data.find(struct.pack('>10)&4095)!=vp:continue + start=pc-12;w=struct.unpack('<8I',read(start,32)) + if w[:2]!=(0xa9bf7bfd,0x910003fd) or w[-2:]!=(0xa8c17bfd,0xd65f03c0):continue + require(n['imported'](n['branch'](start+8))==n['names']['__ZN9IOServiceC2EPK11OSMetaClass'],'Original protected constructor inlines empty accelerator base and calls genuine native service C2');constructors.append(start) +require(len(constructors)==1,'Unique original abstract protected constructor') +protected_destructor=constructors[0]+32 +require(n['imported'](n['branch'](protected_destructor))==n['names']['__ZN9IOServiceD2Ev'],'Actual protected base D2 delegates to genuine native service D2') +require(read(values[0],8)==struct.pack('<2I',0xd4200020,0xd4200020) and values[1]==values[0]+4,'Original abstract D1/D0 are genuine BRK traps, not usable protected D2 aliases') +reference_meta_vp=n['names']['__ZTVN9IOService9MetaClassE']+16;reference_alloc=n['names']['__ZNK9IOService9MetaClass5allocEv'];reference_table=struct.unpack('<32Q',read(reference_meta_vp,256));slots=[i for i,a in enumerate(reference_table) if a==reference_alloc] +require(slots==[13],'Actual native metaclass allocation virtual position') +pc=int(runtime['pc'],16);a,b,c=struct.unpack('<3I',read(pc,12));require(a&0x9f00001f==0x90000008 and b&0xffc003ff==0x91000108 and c==0xf9000008,'Observed original metaclass constructor return installs its actual own table') +meta_vp=adrp(pc,a)+((b>>10)&4095);allocate=struct.unpack('>10)&4095 + if any(v==0xd63f0100 for v in words[i+2:min(i+8,len(words))]):calls.append(dict(pc=start+4*i,slot=slot)) +first=next(v for v in calls if v['slot'] in pure);at=(first['pc']-start)//4 +require(first['slot']==195 and tuple(words[at+2:at+5])==(0xaa1303e0,0xaa1403e1,0xd63f0100) and words[at+5]&0xff00001f==0x36000000,'First required pure hook gets genuine this/provider and its actual Boolean result is checked before proceeding') +report=dict(reference_candidate_sha256=n['plan']['candidate_sha256'],original_iogpu_text_sha256=n['plan']['original_executable_sha256']['com.apple.iokit.IOGPUFamily'],runtime_class_evidence_sha256=hashlib.sha256(runtime_path.read_bytes()).hexdigest(),metaclass=meta,instance_bytes=736,parent_bytes=136,vptr=vp,virtual_entries=215,pure_virtual_slots=pure,operational_overrides=[14,81,82,86,87,111,141],inherited_entries=list(values),protected_constructor=constructors[0],protected_destructor=protected_destructor,abstract_destructor_traps=list(values[:2]),metaclass_vptr=meta_vp,abstract_factory=allocate,original_factory_returns_null=True,start=start,start_bytes=len(body),start_sha256=hashlib.sha256(body).hexdigest(),self_virtual_dispatches=calls,first_required_dispatch=first,first_required_register_abi=dict(this='x0',provider='x1',result='w0 bit 0'),opaque_tail_source_signatures_qualified=False,metadata_only=True,provider_instance_qualified=False,normal_system_authorization=False,original_desktop=False,adreno_execution=False) +(ROOT/'build/gpu-analysis/original-iogpu-provider-abi.json').write_text(json.dumps(report,indent=2)+'\n') +print('PASS: actual abstract 736-byte/215-entry IOGPU, ten pure callbacks, genuine NULL factory and first required this/provider Boolean hook 195') +print('Private tail source signatures, provider construction/start and ordinary system authorization remain unqualified') diff --git a/scripts/audit-target-iogpu-start-contract.py b/scripts/audit-target-iogpu-start-contract.py new file mode 100644 index 0000000..e1ea346 --- /dev/null +++ b/scripts/audit-target-iogpu-start-contract.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python3 +"""Pin original provider startup dependencies; do not run the Apple GPU driver.""" +import hashlib,json,runpy,struct +from pathlib import Path + +ROOT=Path(__file__).resolve().parents[1] +provider=runpy.run_path(str(ROOT/'scripts/audit-target-iogpu-provider.py'));n=provider['n'] +read,require,adrp=n['read'],n['require'],n['adrp'] +abi=json.loads((ROOT/'build/gpu-analysis/original-iogpu-provider-abi.json').read_text()) +start=abi['start'];body=read(start,abi['start_bytes']);words=struct.unpack('<'+'I'*(len(body)//4),body) +require(hashlib.sha256(body).hexdigest()==abi['start_sha256'],'Exact whole original start body') + +def branch(pc,word): + require(word&0x7c000000==0x14000000,'Actual direct B/BL') + imm=word&0x3ffffff + return pc+4*(imm-(1<<26) if imm&(1<<25) else imm) + +def target(pc,word): + dest=branch(pc,word);a,b,c=struct.unpack('<3I',read(dest,12)) + if a&0x9f00001f==0x90000010 and b&0xffc003ff==0xf9400210 and c==0xd61f0200:return n['imported'](dest) + return dest + +def string(pc,a,b): + require(a&0x9f000000==0x90000000 and b&0xffc00000==0x91000000 and (b>>5)&31==a&31,'Actual ADRP/ADD string reference') + return read(adrp(pc,a)+((b>>10)&4095),160).split(b'\0')[0].decode() + +direct=[];strings={};metas={} +for i,w in enumerate(words): + pc=start+i*4 + if w&0xfc000000==0x94000000: + dest=target(pc,w);direct.append(dict(pc=pc,target=dest,names=sorted(name for name,a in n['names'].items() if a==dest))) + if i+1==len(words) or w&0x9f000000!=0x90000000:continue + following=words[i+1] + if following&0xffc00000==0x91000000 and (following>>5)&31==w&31: + try:value=string(pc,w,following) + except (ValueError,UnicodeError):continue + if value and all(c.isprintable() for c in value):strings[pc]=value + if following&0xffc00000==0xf9400000 and (following>>5)&31==w&31: + value=struct.unpack('>10)&4095)*8,8))[0] + names=sorted(name for name,a in n['names'].items() if a==value) + if names:metas[pc]=names + +model=[pc for pc,v in strings.items() if v=='model'] +require(len(model)==1,'Unique provider model property reference') +at=(model[0]-start)//4 +require(words[at-2:at]==(0xf9400008,0xf940a508) and words[at+2]==0xd63f0100,'Actual provider copyProperty virtual slot 41') +require(metas[start+4*(at+4)]==['__ZN6OSData9metaClassE'],'Actual model value must pass original OSData metaclass cast') +casts=[d for d in direct if '__ZN15OSMetaClassBase12safeMetaCastEPKS_PK11OSMetaClass' in d['names']] +require(any(d['pc']==start+4*(at+7) for d in casts),'Actual model OSData cast is executed') +compare=next(i for i in range(at+6,at+20) if words[i]==0x713ff81f) +conditional=words[compare+1];require(conditional&0xff00001f==0x54000008,'Actual unsigned higher-than-4094 branch') +imm=(conditional>>5)&0x7ffff;clamp=compare+1+(imm-(1<<19) if imm&(1<<18) else imm) +require(words[clamp]==0x5281fff7,'Oversized model data is clamped to 4095 before its NUL byte') + +surface=[pc for pc,v in strings.items() if v=='IOCoreSurfaceRoot'] +require(len(surface)==1,'Unique actual IOSurface root service name') +at=(surface[0]-start)//4 +require(words[at+2]==0xd2800001 and target(start+4*(at+3),words[at+3])==n['names']['__ZN9IOService12nameMatchingEPKcP12OSDictionary'],'Actual nameMatching with NULL dictionary') +require(words[at+5]==0x92800001 and target(start+4*(at+6),words[at+6])==n['names']['__ZN9IOService22waitForMatchingServiceEP12OSDictionaryy'],'Actual original IOSurface wait uses UINT64_MAX nanoseconds') + +# The later configuration callback receives the actual private output at +488. +dispatch=next(d for d in abi['self_virtual_dispatches'] if d['slot']==192) +at=(dispatch['pc']-start)//4 +require(words[at-1]==0x9107a276 and words[at+2:at+5]==(0xaa1303e0,0xaa1603e1,0xd63f0100),'Actual output x1=this+488, no guessed private source type') +require(any('__ZN10IOWorkLoop8workLoopEv' in d['names'] for d in direct),'Original startup also allocates an actual workloop') + +# Inspect only this supplied image's concrete Apple implementation to qualify +# the abstract callback layout. It will never be loaded as an Adreno backend. +from native_prelink import macho,backed_offset,unserialize_xml,exports +kernel=ROOT/'.cache/firmware/kernelcache.macho/kernelcache.release.ipad7.decompressed' +raw=kernel.read_bytes();require(hashlib.sha256(raw).hexdigest()==n['plan']['original_sha256'],'Exact supplied original kernelcache') +image=macho(raw);segments=image['segments'] +def original_read(address,size): + offset=backed_offset(segments,address,size,len(raw));return raw[offset:offset+size] +info=next(s for s in segments if s['name']=='__PRELINK_INFO') +entries={e['CFBundleIdentifier']:e for e in unserialize_xml(original_read(info['va'],info['filesize']))['_PrelinkInfoDictionary']} +entry=entries['com.apple.AGXG9G'];offset=backed_offset(segments,entry['_PrelinkExecutableLoadAddr'],32,len(raw));module=macho(raw,offset) +text=next(s for s in module['segments'] if s['name']=='__TEXT_EXEC');code=original_read(text['va'],text['filesize']) +original_names={name:a for a,group in exports(raw,image).items() for name in group} +constructor=original_names['__ZN11OSMetaClassC2EPKcPKS_j'] + +def original_target(pc,word): + dest=branch(pc,word);a,b,c=struct.unpack('<3I',original_read(dest,12)) + require(a&0x9f00001f==0x90000010 and b&0xffc003ff==0xf9400210 and c==0xd61f0200,'Original class initializer calls a genuine import stub') + return struct.unpack('>10)&4095)*8,8))[0] + +classes=[] +for i in range(8,len(code)-24,4): + pc=text['va']+i;a,b,c,d,e,f=struct.unpack_from('<6I',code,i) + if a&0x9f00001f!=0x90000001 or b&0xffc003ff!=0x91000021 or c&0x9f00001f!=0x90000002 or e&0xffe0001f!=0x52800003 or f&0xfc000000!=0x94000000:continue + if d&0xffc003ff not in (0x91000042,0xf9400042):continue + if original_target(pc+20,f)!=constructor:continue + x,y=struct.unpack_from('<2I',code,i-8) + require(x&0x9f00001f==0x90000000 and y&0xffc003ff==0x91000000,'Original metadata object is directly derived') + meta=adrp(pc-8,x)+((y>>10)&4095);name=original_read(adrp(pc,a)+((b>>10)&4095),128).split(b'\0')[0].decode() + parent=adrp(pc+8,c)+((d>>10)&4095)*(8 if d&0xffc00000==0xf9400000 else 1) + if d&0xffc00000==0xf9400000:parent=struct.unpack('>5)&65535,initializer=pc-8)) +name=entry['IOKitPersonalities']['AGXG9G_A0']['IOClass'];found=[c for c in classes if c['name']==name] +require(len(found)==1,'Unique original target personality/concrete class initializer') +concrete=found[0];chain=[concrete];by_meta={c['metaclass']:c for c in classes} +while chain[-1]['parent'] in by_meta:chain.append(by_meta[chain[-1]['parent']]) +require(chain[-1]['parent']==abi['metaclass']-n['plan']['slide'],'Actual concrete class ancestry reaches original IOGPU') +getters=[] +for i in range(0,len(code)-12,4): + pc=text['va']+i;a,b,c=struct.unpack_from('<3I',code,i) + if a&0x9f00001f==0x90000000 and b&0xffc003ff==0x91000000 and c==0xd65f03c0 and adrp(pc,a)+((b>>10)&4095)==concrete['metaclass']:getters.append(pc) +require(len(getters)==1,'Unique real concrete getMetaClass') +tables=[] +for segment in module['segments']: + if segment['name'] in ('__TEXT_EXEC','__LINKEDIT'):continue + data=original_read(segment['va'],segment['filesize']);at=0 + while True: + pos=data.find(struct.pack('=215:tables.append((vp,values[:count])) +require(len(tables)==1,'Unique genuine concrete provider table') +vp,values=tables[0] +hooks={} +for slot in abi['pure_virtual_slots']: + address=values[slot] + require(text['va']<=addressconfig);config_body=original_read(config,end-config) +require(struct.pack('first)-first) +calls=[branch(first+i,w) for i in range(0,len(wrapper),4) if (w:=struct.unpack_from('>5)&31!=a&31:continue + try:value=original_read(adrp(pc,a)+((b>>10)&4095),220).split(b'\0')[0].decode() + except (ValueError,UnicodeError):continue + if value and all(c.isprintable() for c in value):configuration_strings[pc]=value +signature='virtual bool AGXAccelerator::configureDevice(IOService *)' +require(signature in configuration_strings.values(),'Exact supplied concrete implementation names the required Boolean callback') +require({'interrupts','interrupts-valid','RGXRegs','FenderRegs'}<=set(configuration_strings.values()),'Original configureDevice includes real interrupt and hardware mapping configuration') + +initializers=[d for d in direct if model[0]>10)&4095)==meta:getters.append(pc) +require(len(getters)==1,'Unique original userclient getMetaClass') +baseva=n['names']['__ZTV12IOUserClient']+16;base=struct.unpack('<256Q',read(baseva,2048));base_count=next(i for i,v in enumerate(base) if not v) +require(base_count==186,'Actual native reference IOUserClient vtable extent') +tables=[] +for e in n['es']: + if e['segment']=='__TEXT_EXEC':continue + raw=read(e['va'],e['size']);at=0 + while True: + pos=raw.find(struct.pack('>10)&4095)!=vp:continue + start=pc-12;words=struct.unpack('<8I',read(start,32)) + if words[0:2]!=(0xa9bf7bfd,0x910003fd) or words[6:8]!=(0xa8c17bfd,0xd65f03c0):continue + require(n['imported'](n['branch'](start+8))==n['names']['__ZN12IOUserClientC2EPK11OSMetaClass'],'Real protected constructor calls genuine native IOUserClient C2') + constructors.append(start) +require(len(constructors)==2 and n['imported'](n['branch'](values[0]))==n['names']['__ZN12IOUserClientD2Ev'],'Two original protected constructors and genuine non-deleting destructor chain') +free=read(values[14],256);words=struct.unpack('<64I',free);tail=next(i for i,v in enumerate(words) if v==0xd61f0020);free=free[:4*(tail+1)] +guards=[] +for i in range(tail): + a,b=words[i:i+2] + if a&0xffc003ff==0xf9400260 and b&0xff00001f==0xb4000000:guards.append(((a>>10)&4095)*8) + # The first load uses x0 rather than the preserved x19. + if a&0xffc003ff==0xf9400000 and b&0xff00001f==0xb4000000:guards.append(((a>>10)&4095)*8) + if a&0xffc003ff==0xf9400268 and b&0xff00001f==0xb4000008:guards.append(((a>>10)&4095)*8) +require(guards==[288,272,264,216,240],'Original free genuinely guards every uninitialized private pointer before cleanup') +start=read(values[86],108);sw=struct.unpack('<27I',start) +require(sw[10]&0x9f00001f==0x90000008 and sw[11]&0xffc003ff==0x91000108 and sw[12]==0xf9400101,'Original start loads its genuine provider metaclass') +provider_meta=struct.unpack('>10)&4095),8))[0] +gpu_runtime=json.loads((ROOT/'build/darwinboot-owner-hvf-iogpu-gpu-class/gdb-stop.json').read_text());require(provider_meta==int(gpu_runtime['function_entry_arguments'][0],16),'Original start requires the actually observed IOGPU provider class') +require(sw[15]==0xf9007e60 and sw[16]&0xff00001f==0xb4000000 and sw[18]==0xf942e908,'Actual provider cast must succeed before original extra virtual initialization at slot 186') +report=dict(reference_candidate_sha256=plan['candidate_sha256'],current_candidate_sha256=current_plan['candidate_sha256'],original_iogpu_text_sha256=plan['original_executable_sha256']['com.apple.iokit.IOGPUFamily'],runtime_class_evidence_sha256=hashlib.sha256(runtime_path.read_bytes()).hexdigest(),metaclass=hex(meta),instance_bytes=304,parent_instance_bytes=216,vptr=hex(vp),virtual_entries=189,operational_overrides=[14,86,87,168,169,172,185],extra_virtual_entries=[186,187,188],constructors=[hex(v) for v in constructors],destructor=hex(values[0]),free=hex(values[14]),free_bytes=len(free),zero_private_pointer_guards=guards,start=hex(values[86]),required_provider_metaclass=hex(provider_meta),entry_sha256={name:hashlib.sha256(raw).hexdigest() for name,raw in [('free',free),('start',start),*(('constructor_'+str(i),read(va,32)) for i,va in enumerate(constructors))]},metadata_only=True,native_original_client_instance_qualified=False,original_authorization_qualified=False,system_metal=False,original_desktop=False) +(ROOT/'build/gpu-analysis/original-iogpu-userclient-abi.json').write_text(json.dumps(report,indent=2)+'\n') +print('PASS: genuine original 304-byte/189-entry IOGPU userclient C2/D2/free and mandatory real IOGPU provider; no runtime instance or authorization claimed') diff --git a/scripts/audit-target-kext-bindings.py b/scripts/audit-target-kext-bindings.py new file mode 100644 index 0000000..abb8f22 --- /dev/null +++ b/scripts/audit-target-kext-bindings.py @@ -0,0 +1,194 @@ +#!/usr/bin/env python3 +"""Audit exact prelink slide sites against a source-built kernel, without loading. + +The upstream KASLR list identifies absolute pointers, not original link symbols +or a complete relocation recipe. PC-relative references and ABI compatibility +remain separate requirements. No target bytes or signatures are changed. +""" +import argparse +from collections import Counter +import hashlib +import json +from pathlib import Path +import struct +import subprocess + +ROOT = Path(__file__).resolve().parents[1] +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument('--bundle', default='com.apple.driver.AppleMobileFileIntegrity') +args = parser.parse_args() +folder = ROOT / 'build/target-kexts' / args.bundle +metadata = json.loads((folder / 'segments.json').read_text()) +stock = ROOT / '.cache/firmware/kernelcache.macho/kernelcache.release.ipad7.decompressed' +baseline = ROOT / 'build/xnu/ios-obj/DEVELOPMENT_ARM64_BCM2837/mach.development.bcm2837.unstripped.noctf' +data = stock.read_bytes() +if hashlib.sha256(data).hexdigest() != 'f18b4df865405f2a287b84977d7aca745e056d656d9fc9df1662970ce2969de7': + raise SystemExit('Wrong target kernelcache') + + +def segments(at): + magic, cpu, subtype, _, count, size = struct.unpack_from('<6I', data, at) + if (magic, cpu, subtype) != (0xfeedfacf, 0x100000c, 0): + raise ValueError('Expected supplied ordinary ARM64 Mach-O') + offset = at + 32 + result = [] + for _ in range(count): + command, length = struct.unpack_from(' at + 32 + size or at + 32 + size > len(data): + raise ValueError(f'Malformed load command extent at {at:#x}: command={command:#x}, offset={offset:#x}, length={length:#x}, count={count}, size={size:#x}') + if command == 0x19: + name, va, vmsize, fileoff, filesize = struct.unpack_from('<16s4Q', data, offset + 8) + result.append({'name': name.rstrip(b'\0').decode(), 'va': va, 'size': vmsize, + 'fileoff': fileoff, 'filesize': filesize}) + offset += length + if offset != at + 32 + size: + raise ValueError('Load command size mismatch') + return result + + +parent = segments(0) + + +def file_offset(va, length): + matches = [s['fileoff'] + va - s['va'] for s in parent + if s['va'] <= va and va + length <= s['va'] + s['filesize']] + if len(matches) != 1 or matches[0] + length > len(data): + raise ValueError(f'Ambiguous/unbacked cache extent {va:#x}+{length:#x}') + return matches[0] + + +owners = [] +interface_records = [] +for entry in json.loads((ROOT / 'build/target-kexts/inventory.json').read_text()): + if entry['CFBundleIdentifier'].startswith('com.apple.kpi.'): + # These packed KPI symbol-set records contain no executable mappings. + interface_records.append(entry['CFBundleIdentifier']) + continue + va = entry.get('_PrelinkExecutableLoadAddr') + if va is None or not entry.get('_PrelinkExecutableSize'): + continue + try: + module_segments = segments(file_offset(va, 32)) + except ValueError as error: + raise ValueError(f'{entry["CFBundleIdentifier"]}: {error}') from error + for segment in module_segments: + if segment['size'] and segment['name'] != '__LINKEDIT': + owners.append((segment, entry['CFBundleIdentifier'])) + + +def ownership(va): + return sorted({name for s, name in owners if s['va'] <= va < s['va'] + s['size']}) + + +def symbols(path): + result = {} + for line in subprocess.check_output(['xcrun', 'nm', '-g', '-n', path], text=True).splitlines(): + parts = line.split() + if len(parts) == 3 and parts[1].upper() != 'U': + result.setdefault(int(parts[0], 16), []).append(parts[2]) + return result + + +old_symbols = symbols(stock) +new_symbols = {} +for address, names in symbols(baseline).items(): + for name in names: + new_symbols.setdefault(name, set()).add(address) +relocations = (ROOT / 'build/target-kexts/prelink-kaslr.bin').read_bytes() +count, = struct.unpack_from('> 5) & 0x7ffff) << 2 | ((word >> 29) & 3), 21) + page = bool(word & 0x80000000) + target = (pc & ~4095) + (immediate << 12) if page else pc + immediate + kind = 'adrp-page' if page else 'adr' + elif word & 0x7c000000 == 0x14000000: + target = pc + (signed(word & 0x3ffffff, 26) << 2) + kind = 'b-or-bl' + elif word & 0xff000010 == 0x54000000: + target = pc + (signed((word >> 5) & 0x7ffff, 19) << 2) + kind = 'conditional-branch' + elif word & 0x7e000000 == 0x34000000: + target = pc + (signed((word >> 5) & 0x7ffff, 19) << 2) + kind = 'compare-branch' + elif word & 0x7e000000 == 0x36000000: + target = pc + (signed((word >> 5) & 0x3fff, 14) << 2) + kind = 'test-bit-branch' + elif word & 0x3b000000 == 0x18000000: + target = pc + (signed((word >> 5) & 0x7ffff, 19) << 2) + kind = 'literal-load-or-prefetch' + else: + continue + if kind == 'adrp-page': + internal = any((s['va'] & ~4095) <= target < ((s['va'] + s['size'] + 4095) & ~4095) + for s in metadata['segments']) + else: + internal = any(s['va'] <= target < s['va'] + s['size'] for s in metadata['segments']) + pc_relative.append({'site': hex(pc), 'target': hex(target), 'kind': kind, + 'within_module_extent_or_adrp_page': internal}) + + +summary = dict(Counter(record['kind'] for record in records)) +report = {'bundle_id': args.bundle, 'stock_sha256': hashlib.sha256(data).hexdigest(), + 'baseline_sha256': hashlib.sha256(baseline.read_bytes()).hexdigest(), + 'kaslr_table_sha256': hashlib.sha256(relocations).hexdigest(), + 'excluded_kernel_kpi_symbol_sets': interface_records, + 'total_cache_pointer_sites': count, 'module_pointer_sites': len(records), + 'summary': summary, 'pointers': records, + 'pc_relative_encoding_counts': dict(Counter(record['kind'] for record in pc_relative)), + 'pc_relative_encodings_outside_module': [r for r in pc_relative if not r['within_module_extent_or_adrp_page']], + 'pc_relative_consumer_dataflow_proven': False, + 'complete_relocation_recipe': False, 'abi_compatibility_proven': False, + 'pc_relative_references_audited': False, 'target_bytes_modified': False} +(folder / 'exact-pointer-audit.json').write_text(json.dumps(report, indent=2) + '\n') +print(json.dumps({key: report[key] for key in ('bundle_id', 'module_pointer_sites', 'summary')}, indent=2)) +print('PC-relative encoding counts:', report['pc_relative_encoding_counts']) +print('Encodings outside module extents/pages:', len(report['pc_relative_encodings_outside_module'])) +print('Static pointer inventory only: no module loaded, no ABI/PC-relative binding proof') diff --git a/scripts/audit-target-libm-inline-data.py b/scripts/audit-target-libm-inline-data.py new file mode 100644 index 0000000..276bb16 --- /dev/null +++ b/scripts/audit-target-libm-inline-data.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +"""Qualify unchanged inline math pools and actual-image prelink refusal controls.""" +import hashlib +import json +import math +from pathlib import Path +import struct +import sys + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'kits/NativePrelink')) +from native_prelink import backed_offset, macho, unserialize_xml +from emit_prelink import instruction_inventory, pc_relative_target, MATH_LITERAL_POOLS + +source = ROOT / '.cache/firmware/kernelcache.macho/kernelcache.release.ipad7.decompressed' +data = source.read_bytes() +expected = 'f18b4df865405f2a287b84977d7aca745e056d656d9fc9df1662970ce2969de7' +if hashlib.sha256(data).hexdigest() != expected: + raise SystemExit('Require the unchanged supplied original kernelcache') +parent = macho(data)['segments'] +info = next(s for s in parent if s['name'] == '__PRELINK_INFO') +at = backed_offset(parent, info['va'], info['filesize'], len(data)) +entries = {e['CFBundleIdentifier']: e for e in unserialize_xml(data[at:at + info['filesize']])['_PrelinkInfoDictionary']} +plan = json.loads((ROOT / 'build/display-analysis/mobile-graphics-plan.json').read_text()) +modules = {name: macho(data, backed_offset(parent, entries[name]['_PrelinkExecutableLoadAddr'], 32, len(data))) + for name in plan['components']} +module = modules['com.apple.kec.Libm'] +text = next(s for s in module['segments'] if s['name'] == '__TEXT_EXEC') +at = backed_offset(parent, text['va'], text['filesize'], len(data)) +blob = data[at:at + text['filesize']] +ranges = [(0x350, 0xbc0), (0xf60, 0x1440), (0x1884, 0x2110), (0x21e8, 0x2628), (0x280c, 0x28e0), (0x3010, 0x3048)] +records = [] +for (start, end), (prefix, size, digest) in zip(ranges, MATH_LITERAL_POOLS): + pool = blob[start:end] + if len(pool) != size or not pool.startswith(bytes.fromhex(prefix)) or hashlib.sha256(pool).hexdigest() != digest: + raise SystemExit('Complete original math-pool identity differs') + previous = blob[start - 4:start] + if previous != bytes.fromhex('c0035fd6') or blob[end:end + 4] != bytes.fromhex('fd7bbfa9'): + raise SystemExit('Require original RET/data/next-function boundaries') + references, incoming_branches = [], [] + for offset in range(0, len(blob) - 3, 4): + if any(a <= offset < b for a, b in ranges): + continue + word, = struct.unpack_from(' 1 or any(a >= b for a, b in zip(highs, highs[1:])): + raise SystemExit('Original 128-row power-of-two lookup highs differ from independent arithmetic') +positive = instruction_inventory(data, parent, modules) +identified = [r for r in positive['known_inline_data'] if r['module'] == 'com.apple.kec.Libm'] +if len(identified) != 6: + raise SystemExit('Actual-image positive must identify exactly six original math pools') +controls = [] +for start, end in ranges: + for offset in (start, end - 1): + changed = bytearray(data) + changed[at + offset] ^= 1 + try: + instruction_inventory(changed, parent, modules) + except ValueError as error: + if 'External PC-relative encoding' not in str(error): + raise + controls.append(dict(offset=offset, refusal=str(error))) + else: + raise SystemExit('A modified known pool must not be excluded by its prefix') +changed = bytearray(data) +struct.pack_into('\n#include \n#include "dyld_cache_format.h"\nint main(){printf("%zu\\n",offsetof(dyld_cache_header,objcOptsOffset));}\n') +subprocess.run(['xcrun','clang++','-I',ROOT/'refer/dyld/cache-builder',source,'-o',exe],check=True,timeout=30) +require(subprocess.check_output([exe],text=True).strip()=='464','Compiler-derived ObjC optimization-header field') +opt=region+struct.unpack_from('>16 + address=entry+offset # Pinned MachOAnalyzer: final list is the class list. + flags,count=struct.unpack('=start and size<=1024*1024 and name,'Bounded actual class instance layout') + return dict(address=address,name=name,superclass=pointer(address+8),instance_start=start,instance_size=size, + methods=methods(pointer(ro+32))) +base=cls(symbol('_OBJC_CLASS_$__MTLDevice'));meta=cls(pointer(base['address'])) +require(base['name']=='_MTLDevice' and base['instance_size']==536 and len(base['methods'])==575,'Exact original base-device metadata') +# Record actual ancestry; the bundle principal's name alone does not establish +# which runtime device class will be published. Never inherit an AGX class. +family=[];address=symbol('_OBJC_CLASS_$_AGXA10FamilyDevice') +while address: + require(len(family)<16 and address not in {x['address'] for x in family},'Bounded original class ancestry') + item=cls(address);family.append(item);address=item['superclass'] +require(any(x['name']=='_MTLDevice' for x in family),'Actual Apple device ancestry reaches Metal base') +principal=cls(symbol('_OBJC_CLASS_$_AGXPrincipalDevice')) +require(cls(principal['superclass'])['name']=='NSObject','Original principal class has NSObject ancestry') +functions={name:symbol(name) for name in ('_MTLCreateSystemDefaultDevice','_MTLRegisterDevices')} +pins={str(p.relative_to(ROOT/'refer/dyld')):hashlib.sha256(p.read_bytes()).hexdigest() for p in + (ROOT/'refer/dyld/common/ObjCVisitor.h',ROOT/'refer/dyld/common/ObjCVisitor.cpp',ROOT/'refer/dyld/common/DyldSharedCache.h',ROOT/'refer/dyld/common/DyldSharedCache.cpp',ROOT/'refer/dyld/common/MachOAnalyzer.cpp',ROOT/'refer/dyld/common/OptimizerObjC.h')} +report=dict(cache_uuid=main[88:104].hex(),dyld_revision=subprocess.check_output(['git','-C',ROOT/'refer/dyld','rev-parse','HEAD'],text=True).strip(), + layout_source_sha256=pins,selector_base=selector_base,device=base,metaclass=meta, + principal_class={k:v for k,v in principal.items() if k!='methods'}, + apple_device_ancestry=[{k:v for k,v in x.items() if k!='methods'} for x in family],functions=functions, + runtime_observed=False,native_adreno_device=False) +(OUT/'metal-device-abi.json').write_text(json.dumps(report,indent=2)+'\n') +print('PASS: original _MTLDevice has',len(base['methods']),'own methods,',len(meta['methods']),'class methods and',base['instance_size'],'instance bytes') +print('Actual Apple family ancestry:', ' -> '.join(c['name'] for c in family)) diff --git a/scripts/audit-target-metal-discovery.py b/scripts/audit-target-metal-discovery.py new file mode 100644 index 0000000..69cc138 --- /dev/null +++ b/scripts/audit-target-metal-discovery.py @@ -0,0 +1,243 @@ +#!/usr/bin/env python3 +"""Qualify the unchanged target's actual Metal service/plugin discovery contract.""" +import hashlib +import json +import mmap +from pathlib import Path +import plistlib +import struct +import sys + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'kits/NativePrelink')) +from native_prelink import backed_offset, macho, unserialize_xml + + +def require(value, message): + if not value: + raise SystemExit(message) + + +directory = ROOT / '.cache/firmware/target-volumes/systemos-mount/System/Library/Caches/com.apple.dyld' +files, mappings, slides = [], [], [] +for path in sorted(directory.glob('dyld_shared_cache_arm64*')): + if path.suffix == '.symbols': + continue + stream = path.open('rb') + data = mmap.mmap(stream.fileno(), 0, access=mmap.ACCESS_READ) + files.append((path.name, stream, data)) + require(data[:16].rstrip(b'\0') == b'dyld_v1 arm64', 'Require original ordinary ARM64 caches') + at, count = struct.unpack_from('<2I', data, 16) + for i in range(count): + va, size, off, _, _ = struct.unpack_from('<3Q2I', data, at + 32 * i) + mappings.append((va, va + size, off, data)) + at, count = struct.unpack_from('<2I', data, 312) + for i in range(count): + va, size, _, off, length, _, _, _ = struct.unpack_from('<6Q2I', data, at + 56 * i) + if length: + require(struct.unpack_from('> 5) & 0x7ffff) << 2 | ((first >> 29) & 3) + if immediate & (1 << 20): + immediate -= 1 << 21 + return (pc & ~4095) + (immediate << 12) + ((second >> 10) & 4095) + + +def branch(pc, linked=True): + word, = struct.unpack('> 5) & 0x7ffff) << 2 | ((first >> 29) & 3) +if immediate & (1 << 20): + immediate -= 1 << 21 +class_reference = ((plugin + 0xf4) & ~4095) + (immediate << 12) + ((second >> 10) & 4095) * 8 +require(pointer(class_reference) == symbol('_OBJC_CLASS_$__MTLDevice'), 'Actual required subclass is _MTLDevice') +device = symbol('_IOGPUDeviceCreateWithAPIProperty') +require(resolved_call(device + 0x4c) == symbol('_IOServiceOpen') and + read(device + 0x30, 12)[:8] == bytes.fromhex('28008052483c1033'), 'Original IOGPU open uses base type 1 and the supplied API value in its upper 16 bits') +wrapper = symbol('_IOGPUDeviceCreate') +require(read(wrapper, 8) == bytes.fromhex('010080d2020080d2') and branch(wrapper + 8, False) == device, 'Default IOGPU wrapper preserves null API-property / zero API value') + +kernel = (ROOT / '.cache/firmware/kernelcache.macho/kernelcache.release.ipad7.decompressed').read_bytes() +require(hashlib.sha256(kernel).hexdigest() == 'f18b4df865405f2a287b84977d7aca745e056d656d9fc9df1662970ce2969de7', 'Require unchanged original kernelcache') +segments = macho(kernel)['segments'] +info = next(s for s in segments if s['name'] == '__PRELINK_INFO') +at = backed_offset(segments, info['va'], info['filesize'], len(kernel)) +entries = {e['CFBundleIdentifier']: e for e in unserialize_xml(kernel[at:at + info['filesize']])['_PrelinkInfoDictionary']} +agx = entries['com.apple.AGXG9G']['IOKitPersonalities']['AGXG9G_A0'] +require(agx['MetalPluginName'] == 'AGXMetalA10' and agx['IONameMatch'] == ['gpu,t8011'], 'Original Apple GPU personality identity, not an Adreno match') +plist = ROOT / '.cache/firmware/target-volumes/os-mount/System/Library/Extensions/AGXMetalA10.bundle/Info.plist' +bundle = plistlib.loads(plist.read_bytes()) +require(bundle['CFBundleIdentifier'] == 'com.apple.AGXMetalA10' and bundle['CFBundleExecutable'] == 'AGXMetalA10' and + bundle['NSPrincipalClass'] == 'AGXPrincipalDevice' and bundle['CFBundleVersion'] == '282.14', 'Original target plugin metadata') +out = ROOT / 'build/gpu-analysis' +out.mkdir(parents=True, exist_ok=True) +(out / 'metal-discovery.json').write_text(json.dumps(dict( + cache_uuid=main[88:104].hex(), local_symbols_uuid=main[400:416].hex(), images=list(images.values()), + service_match='IOAcceleratorES', plugin_properties=list(properties.values()), + plugin_info_sha256=hashlib.sha256(plist.read_bytes()).hexdigest(), original_principal_class=bundle['NSPrincipalClass'], + functions={name: dict(va=hex(va), inspected_bytes=n, sha256=hashlib.sha256(read(va, n)).hexdigest()) + for name, va, n in [('register', register, 0xf0), ('plugin', plugin, 0x130), ('iogpu_open', device, 0x78)]}, + iogpu_base_open_type=1, iogpu_api_shift=16, native_metal_plugin_required=True, + static_only=True, runtime_device_created=False, original_instructions_modified=False, + original_principal_class_runtime_acceptance_qualified=False, adreno_metal_qualified=False), indent=2) + '\n') +print('PASS: actual target IOAcceleratorES matching, Metal bundle/property/class checks and IOGPU open-type contract') +print('Static discovery evidence only: no native Adreno Metal plugin, runtime device or GPU execution is qualified') diff --git a/scripts/audit-target-metal-shared-events.py b/scripts/audit-target-metal-shared-events.py new file mode 100644 index 0000000..d01c608 --- /dev/null +++ b/scripts/audit-target-metal-shared-events.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +"""Derive the supplied target's real shared-event handle/port ABI, read-only.""" +import hashlib,json,runpy,struct +from pathlib import Path +ROOT=Path(__file__).resolve().parents[1] +n=runpy.run_path(str(ROOT/'scripts/audit-target-metal-device.py')) +read,pointer,cls,symbol,require=n['read'],n['pointer'],n['cls'],n['symbol'],n['require'] +handle=cls(symbol('_OBJC_CLASS_$_MTLSharedEventHandle'));listener=cls(symbol('_OBJC_CLASS_$_MTLSharedEventListener')) +require(handle['instance_size']==16 and handle['instance_start']==8 and cls(handle['superclass'])['name']=='NSObject','Actual original 16-byte handle and NSObject base') +methods={m['selector']:m for m in handle['methods']} +for name,encoding in {'eventPort':'I16@0:8','label':'@16@0:8','labelTraceID':'Q16@0:8','initWithSharedEvent:':'@24@0:8@16','encodeWithCoder:':'v24@0:8@16','initWithCoder:':'@24@0:8@16'}.items():require(methods[name]['encoding']==encoding,'Exact original handle selector ABI: '+name) +meta=cls(pointer(handle['address']));secure=next(m for m in meta['methods'] if m['selector']=='supportsSecureCoding') +require(secure['encoding']=='B16@0:8' and read(secure['implementation'],8)==struct.pack('<2I',0x52800020,0xd65f03c0),'Original secure coding implementation genuinely returns true') +for name,second in [('eventPort',0xb9400100),('label',0xf9400500),('labelTraceID',0xf9400900)]:require(read(methods[name]['implementation'],12)==struct.pack('<3I',0xf9400408,second,0xd65f03c0),'Actual 24-byte private-data getter: '+name) +constructor=methods['initWithSharedEvent:']['implementation'];code=read(constructor,204) +require(struct.unpack_from('<2I',code,0x38)==(0x52800300,0x52800021),'Actual constructor allocates 24 bytes of zeroed private data') +def adrp(pc,word): + value=((word>>5)&0x7ffff)*4+((word>>29)&3) + if value&(1<<20):value-=1<<21 + return (pc&~4095)+(value<<12) +def branch(pc): + word=struct.unpack('>10)&4095)*8));calls.append(dict(call=constructor+offset,stub=target,selector=name)) +require([r['selector'] for r in calls]==['eventPort','label','copy','labelTraceID'],'Original constructor borrows the real port, copies the label and reads its typed trace getter') +require(struct.unpack_from('<2I',code,0x70)==(0x52800002,0x52800023),'Actual send-right kind zero and reference increment one') +stub=branch(constructor+0x78);a,b,c=struct.unpack('<3I',read(stub,12));require(a&0x9f00001f==0x90000010 and b&0xffc003ff==0x91000210 and c==0xd61f0200,'Original optimized direct import stub') +ref_function=adrp(stub,a)+((b>>10)&4095) +# Prove the imported public function through the original libsystem export trie. +main=n['main'];at,count=struct.unpack_from('<2Q',main,136);image_va=None +for i in range(count): + uuid,va,_,name=struct.unpack_from('<16sQ2I',main,at+32*i) + if main[name:main.find(b'\0',name)]==b'/usr/lib/system/libsystem_kernel.dylib':image_va=va;break +require(image_va is not None,'Original libsystem kernel image') +header=read(image_va,32);blob=header+read(image_va+32,struct.unpack_from('>10)&4095)*4 +queue_ivar_offset=struct.unpack('>10)&4095)*8;coder=cls(pointer(reference)) + require(coder['name']=='NSXPCCoder','Original handle explicitly requires the real NSXPCCoder class') + selectors=[] + for offset in range(0,length,4): + word=struct.unpack_from('>10)&4095)*8))) + expected=['raise:format:','encodeXPCObject:forKey:','encodeObject:forKey:'] if name=='encodeWithCoder:' else ['raise:format:','decodeXPCObjectOfType:forKey:','decodeObjectOfClass:forKey:'] + require(selectors==expected,'Original NSXPC-only object/port coding selectors: '+name+' '+repr(selectors)) + coding_methods[name]=dict(implementation=start,bytes=length,sha256=hashlib.sha256(raw).hexdigest(),required_coder_class=coder['name'],selectors=selectors) +report=dict(coding_methods=coding_methods,cache_uuid=main[88:104].hex(),handle=handle,listener=listener,concrete_listener=concrete,listener_queue_ivar_offset=queue_ivar_offset,private_bytes=24,constructor_sha256=hashlib.sha256(code).hexdigest(),constructor_calls=calls,mach_port_mod_refs=hex(ref_function),actual_send_ref_increment=1,metadata_only=True,native_mach_export_qualified=False,original_desktop=False,adreno_execution=False) +(ROOT/'build/gpu-analysis/shared-event-handle-contract.json').write_text(json.dumps(report,indent=2)+'\n') +print('PASS: exact original secure handle, typed port/label/trace ABI and real Mach send-right retain; native export/import remains unqualified') diff --git a/scripts/audit-target-renderbox-device.py b/scripts/audit-target-renderbox-device.py new file mode 100644 index 0000000..ea06eaa --- /dev/null +++ b/scripts/audit-target-renderbox-device.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +"""Read the exact original RenderBox class/export ABI; no runtime qualification.""" +import hashlib +import json +from pathlib import Path +import runpy +import struct +ROOT=Path(__file__).resolve().parents[1] +n=runpy.run_path(str(ROOT/'scripts/audit-target-metal-device.py')) +read,cls,main,require=n['read'],n['cls'],n['main'],n['require'] +a=n['n'];macho,uleb=a['macho'],a['uleb'] +at,count=struct.unpack_from('<2Q',main,136);matches=[] +for i in range(count): + uuid,va,_,path=struct.unpack_from('<16sQ2I',main,at+i*32) + name=main[path:main.find(b'\0',path)].decode() + if name.endswith('/RenderBox.framework/RenderBox'):matches.append((uuid,va,name)) +require(len(matches)==1,'One original RenderBox image') +uuid,va,name=matches[0];size=struct.unpack(' len(data): + raise SystemExit('DER element extends beyond the actual input') + return tag, data[at:end], end + + +with zipfile.ZipFile(ROOT / 'refer/iPad_Pro_HFR_17.7.11_21H461_Restore.ipsw') as archive: + manifest = plistlib.loads(archive.read('BuildManifest.plist')) + identities = [i for i in manifest['BuildIdentities'] if i['Info']['DeviceClass'] == 'j120ap'] + if manifest['ProductBuildVersion'] != '21H461' or len(identities) != 2: + raise SystemExit('Expected the supplied 2017 target build and both actual restore identities') + records = [] + for name in ('SystemVolume', 'Cryptex1,SystemVolume', 'Cryptex1,AppVolume'): + component = identities[0]['Manifest'][name] + path = component['Info']['Path'] + data = archive.read(path) + tag, sequence, end = tlv(data, 0) + if tag != 0x30 or end != len(data): + raise SystemExit('Expected one complete actual DER sequence') + fields, cursor = [], 0 + while cursor < len(sequence): + kind, value, cursor = tlv(sequence, cursor) + fields.append((kind, value)) + if [t for t, _ in fields] != [0x16, 0x16, 0x16, 4] or fields[0][1] != b'IM4P': + raise SystemExit('Expected actual IM4P fields, without fabricating an Image4 manifest') + digest = hashlib.sha384(data).digest() + if any(i['Manifest'][name]['Digest'] != digest or + i['Manifest'][name]['Info']['Path'] != path or + i['Manifest'][name]['Info'].get('Personalize') is not True for i in identities): + raise SystemExit('Root-hash input disagrees with the actual target restore manifests') + records.append(dict(component=name, member=path, bytes=len(data), + sha256=hashlib.sha256(data).hexdigest(), sha384=digest.hex(), + image4_payload_type=fields[1][1].decode(), im4p_version=fields[2][1].decode(), + payload_bytes=len(fields[3][1]), + payload_sha256=hashlib.sha256(fields[3][1]).hexdigest(), + both_manifest_digests_match=True, personalize_required=True, + personalized_im4m_present=False, signature_authentication_qualified=False)) +report = dict(target='iPad7,1', board='j120ap', build='21H461', root_hash_inputs=records, + build_manifest_digest_check_is_not_signature_authentication=True, + root_authentication_solved=False, hardware_tested=False) +out = ROOT / 'build/target-root-hash-inputs.json' +out.write_text(json.dumps(report, indent=2) + '\n') +print('PASS: three real 229-byte IM4P root-hash inputs match both original target manifest digests') +print('All require personalization; these payloads alone do not supply a personalized IM4M or authenticate a boot root') diff --git a/scripts/audit-target-sandbox-client-class.py b/scripts/audit-target-sandbox-client-class.py new file mode 100755 index 0000000..5914daf --- /dev/null +++ b/scripts/audit-target-sandbox-client-class.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +"""Derive the supplied original Sandbox I/O Kit hook and genuine class walk.""" +import hashlib,json,re,runpy,struct,subprocess +from pathlib import Path +ROOT=Path(__file__).resolve().parents[1] +n=runpy.run_path(str(ROOT/'scripts/audit-target-accelerator-base.py')) +read,require=n['read'],n['require'] +es=[e for e in n['plan']['extents'] if e['module']=='com.apple.security.sandbox'] +text=next(e for e in es if e['segment']=='__TEXT_EXEC');code=read(text['va'],text['size']) +digest=hashlib.sha256(code).hexdigest() +require(digest==n['plan']['original_executable_sha256']['com.apple.security.sandbox'],'Exact complete supplied original Sandbox executable') +obj=ROOT/'build/xnu/ios-obj/DEVELOPMENT_ARM64_BCM2837/security/DEVELOPMENT/mac_vfs.o' +layouts={} +for typename,fields in (('mac_policy_conf',('mpc_name','mpc_fullname','mpc_ops')),('mac_policy_ops',('mpo_iokit_check_open','mpo_iokit_check_open_service'))): + dwarf=subprocess.check_output(['xcrun','dwarfdump','--name='+typename,'--show-children',obj],text=True) + layouts[typename]={} + for field in fields: + match=re.search(r'DW_AT_name\s+\("'+field+r'"\)[\s\S]*?DW_AT_data_member_location\s+\((0x[0-9a-f]+)\)',dwarf) + require(match,'Actual compiled MAC policy field '+field);layouts[typename][field]=int(match[1],16) +def q(a):return struct.unpack('=0:strings.append(e['va']+at);at+=1 +confs=[];fields=layouts['mac_policy_conf'] +for e in es: + if e['segment'] not in ('__DATA_CONST','__DATA'):continue + data=read(e['va'],e['size']) + for name in strings: + at=0 + while (at:=data.find(struct.pack('=0: + address=e['va']+at-fields['mpc_name'];at+=1 + if address%8:continue + full=q(address+fields['mpc_fullname']) + try:fullname=read(full,len(b'Seatbelt sandbox policy\0')) + except ValueError:continue + if fullname==b'Seatbelt sandbox policy\0':confs.append(address) +require(len(confs)==1,'Unique actual original Sandbox policy conf') +conf=confs[0];ops=q(conf+fields['mpc_ops']);hooks={} +for name,offset in layouts['mac_policy_ops'].items(): + address=q(ops+offset);require(text['va']<=address=text['va'],'Actual client x1 -> filter context +216, operation 63 -> original evaluator') +def calls(name): + target=n['names'][name];found=[] + for at in range(0,len(code)-12,4): + a,b,c=struct.unpack_from('<3I',code,at) + if a&0x9f00001f==0x90000010 and b&0xffc003ff==0xf9400210 and c==0xd61f0200 and n['imported'](text['va']+at)==target: + stub=text['va']+at + for off in range(0,len(code),4): + word=struct.unpack_from('>5)&0x7ffff;dest=start+136+4*(imm-(1<<19) if imm&(1<<18) else imm) + require(dest==call-48,'Superclass loop returns to real current-class name call') + walks.append(dict(address=hex(start),bytes=176,sha256=hashlib.sha256(read(start,176)).hexdigest(),get_class_name_call=hex(call-44),get_super_class_call=hex(call),back_edge=hex(dest))) +require(len(walks)==1,'Unique actual exact/prefix class-filter walk with genuine superclass traversal') +report=dict(original_sandbox_text_sha256=digest,reference_candidate_sha256=n['plan']['candidate_sha256'],compiled_policy_layout_object_sha256=hashlib.sha256(obj.read_bytes()).hexdigest(),layouts=layouts, + policy_conf=hex(conf),policy_ops=hex(ops),hooks={name:hex(a) for name,a in hooks.items()},client_filter_context_offset=216,open_policy_operation=63,class_walk=walks[0], + source_analysis_only=True,system_app_authorization_qualified=False,system_metal=False,original_desktop=False,adreno_execution=False) +(ROOT/'build/gpu-analysis/original-sandbox-client-class.json').write_text(json.dumps(report,indent=2)+'\n') +print('PASS: unchanged original Sandbox policy hook and real getMetaClass/getClassName/getSuperClass loop derived; actual signed-policy behavior requires runtime controls') diff --git a/scripts/audit-target-userspace.py b/scripts/audit-target-userspace.py new file mode 100644 index 0000000..3fe0731 --- /dev/null +++ b/scripts/audit-target-userspace.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +"""Audit unchanged target launchd/dyld; this does not demonstrate userspace execution.""" +import argparse +import hashlib +import json +from pathlib import Path +import struct + +ROOT = Path(__file__).resolve().parents[1] +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument('--volume', type=Path, default=ROOT / '.cache/firmware/target-volumes/os-mount') +args = parser.parse_args() +trustcache = ROOT / '.cache/firmware/target-volumes/static-trustcache.bin' +trust = trustcache.read_bytes() +version, trust_entries = struct.unpack_from(' 32 + commands_size: + raise SystemExit('Invalid Mach-O load command') + if command == 0x19: + segname = data[offset + 8:offset + 24].split(b'\0')[0].decode() + va, vmsize, fileoff, filesize = struct.unpack_from('<4Q', data, offset + 24) + if (va % 4096 or vmsize % 4096 or fileoff % 4096 + or fileoff + filesize > len(data) or filesize > vmsize): + raise SystemExit(f'{name}: segment {segname} is not directly 4K mappable') + segments.append({'name': segname, 'va': hex(va), 'vmsize': vmsize, + 'fileoff': fileoff, 'filesize': filesize}) + elif command == 0x32: + platform, minimum, sdk = struct.unpack_from('<3I', data, offset + 8) + if platform != 2 or minimum >> 16 != 17: + raise SystemExit('Unexpected target userspace platform/deployment') + elif command == 0x1d: + sigoff, sigsize = struct.unpack_from('3I', signature) + if smagic != 0xfade0cc0 or slength > len(signature): + raise SystemExit('Unexpected signature superblob') + directories = [] + for index in range(entries): + slot, start = struct.unpack_from('>2I', signature, 12 + index * 8) + magic, length = struct.unpack_from('>2I', signature, start) + if magic != 0xfade0c02: + continue + directory = signature[start:start + length] + version, flags, hashoff, identoff, special, codeslots, limit = struct.unpack_from('>7I', directory, 8) + hashsize, hashtype, platform, pageshift = struct.unpack_from('4B', directory, 36) + if version >= 0x20300: + limit64 = struct.unpack_from('>Q', directory, 56)[0] + if limit64: + limit = limit64 + if hashtype != 2 or hashsize != 32 or pageshift != 12 or limit > len(data): + raise SystemExit('Expected the original target SHA-256 / 4K CodeDirectory') + if codeslots != (limit + 4095) // 4096: + raise SystemExit('Unexpected target code slot count') + for page in range(codeslots): + actual = hashlib.sha256(data[page * 4096:min((page + 1) * 4096, limit)]).digest() + expected = directory[hashoff + page * hashsize:hashoff + (page + 1) * hashsize] + if actual != expected: + raise SystemExit(f'{name}: original code page {page} hash mismatch') + cdhash = hashlib.sha256(directory).digest()[:20] + matches = [i for i in range(trust_entries) if trust[24 + i * 24:44 + i * 24] == cdhash] + if len(matches) != 1: + raise SystemExit('Target CodeDirectory is absent or ambiguous in its original trustcache') + directories.append({'slot': slot, 'original_static_trustcache_index': matches[0], 'identifier': directory[identoff:].split(b'\0')[0].decode(), + 'flags': hex(flags), 'hash_page_bytes': 4096, + 'code_slots_verified': codeslots, 'cdhash': hashlib.sha256(directory).hexdigest()[:40]}) + if not directories: + raise SystemExit('No verified original CodeDirectory') + results.append({'path': name, 'sha256': hashlib.sha256(data).hexdigest(), 'arm64_subtype': 0, + 'minimum_ios': f'{minimum >> 16}.{minimum >> 8 & 255}', + 'segments_directly_4k_mappable': True, 'segments': segments, + 'code_directories': directories}) +report = {'target': 'iPad7,1 / 17.7.11 / 21H461', 'files': results, + 'code_page_hashes_verified': True, 'original_static_trustcache_sha256': hashlib.sha256(trust).hexdigest(), 'apple_trust_chain_verified': False, + 'native_4k_userspace_execution_observed': False, 'ipados_boot_demonstrated': False} +out = ROOT / 'build/target-userspace' +out.mkdir(parents=True, exist_ok=True) +(out / 'audit.json').write_text(json.dumps(report, indent=2) + '\n') +print('PASS: unchanged target launchd/dyld ordinary ARM64, direct 4K segment layout and all original code-page hashes') +print('Apple trust-chain integration and actual userspace execution remain unverified') diff --git a/scripts/audit-target-xpc-endpoints.py b/scripts/audit-target-xpc-endpoints.py new file mode 100644 index 0000000..c6589ca --- /dev/null +++ b/scripts/audit-target-xpc-endpoints.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +"""Read exact original Foundation/libxpc endpoint ABI without extracting images.""" +import hashlib,json,runpy,struct +from pathlib import Path +ROOT=Path(__file__).resolve().parents[1] +n=runpy.run_path(str(ROOT/'scripts/audit-target-metal-device.py')) +read,require=n['read'],n['require'];main=n['main'];at,count=struct.unpack_from('<2Q',main,136) +wanted={'/System/Library/Frameworks/Foundation.framework/Foundation','/usr/lib/system/libxpc.dylib'};images={};exports={} +for i in range(count): + uuid,va,_,name=struct.unpack_from('<16sQ2I',main,at+32*i);end=main.find(b'\0',name);require(end>=name,'Bounded original image name');path=main[name:end].decode() + if path not in wanted:continue + header=read(va,32);blob=header+read(va+32,struct.unpack_from('= 8 and offset + length <= 32 + size <= len(data) + if command == 0x19: + va, _, fileoff, filesize = struct.unpack_from('<4Q', data, offset + 24) + if va <= address and address + 8 <= va + filesize: + tcr = struct.unpack_from('> 14) & 3 == 0 and (tcr >> 30) & 3 == 2, hex(tcr) +report = {'sha256': hashlib.sha256(data).hexdigest(), 'cpu_subtype': subtype, + 'deployment': 'iOS 17.0', 'entry_va': hex(entry), + 'initial_tcr': hex(tcr), 'tg0_bytes': 4096, 'tg1_bytes': 4096, + 'hardware_boot_demonstrated': False, 'os_boot_demonstrated': False} +output = Path(__file__).resolve().parents[1] / 'build/xnu/kernel-audit.json' +output.parent.mkdir(parents=True, exist_ok=True) +output.write_text(json.dumps(report, indent=2) + '\n') +print(json.dumps(report, indent=2)) diff --git a/scripts/audit-xnu-objects.py b/scripts/audit-xnu-objects.py new file mode 100755 index 0000000..c7f952b --- /dev/null +++ b/scripts/audit-xnu-objects.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 +"""Audit the actual XNU target objects, independent of stale compiler JSON.""" +import json +from pathlib import Path +import struct + +ROOT = Path(__file__).resolve().parents[1] +TARGET = ROOT / 'build/xnu/ios-obj/DEVELOPMENT_ARM64_BCM2837' +objects = [] +failures = [] +non_macho = [] +for path in sorted(TARGET.rglob('*')): + if path.suffix not in ('.o', '.cpo') or not path.is_file(): + continue + data = path.read_bytes() + name = str(path.relative_to(TARGET)) + if data[:4] != b'\xcf\xfa\xed\xfe': + non_macho.append(name) + if path.name != 'genassym.o': + failures.append(f'{name}: unexpected non-Mach-O object') + continue + _, cpu, subtype, kind, count, size = struct.unpack_from('<6I', data) + if (cpu, subtype, kind) != (0x100000c, 0, 1): + failures.append(f'{name}: expected ordinary ARM64 MH_OBJECT') + offset = 32 + versions = [] + for _ in range(count): + if offset + 8 > 32 + size or 32 + size > len(data): + raise ValueError(f'{name}: truncated load commands') + cmd, length = struct.unpack_from(' 32 + size: + raise ValueError(f'{name}: invalid load command') + if cmd == 0x32: + platform, minimum = struct.unpack_from(' NIR -> IR3 host qualification; no GPU device access.""" +import concurrent.futures +import ctypes +import hashlib +import json +import os +from pathlib import Path +import random +import re +import urllib.request +import shutil +import struct +import subprocess +import sys +import time + +ROOT = Path(__file__).resolve().parents[1] +OUT = ROOT / 'build/air-nir' +MESA = ROOT / 'build/mesa-ir3' +KIT = ROOT / 'kits/AirNir' +CHIP = '0x43050a01' # board source/catalog identity; no hardware read +llvm_config = os.environ.get('LLVM_CONFIG') or shutil.which('llvm-config') +if not llvm_config and shutil.which('brew'): + prefix = subprocess.check_output(['brew', '--prefix', 'llvm'], text=True, timeout=10).strip() + llvm_config = str(Path(prefix) / 'bin/llvm-config') +if not llvm_config: + raise SystemExit('Set LLVM_CONFIG to the qualified LLVM 23.1.3 config executable') +LLVM = Path(llvm_config) +ENV = {k: v for k, v in os.environ.items() if not k.startswith(('IR3_', 'FD_MESA_', 'NIR_', 'MESA_SHADER_'))} +ENV['PATH'] = str(ROOT / '.cache/mesa-tools/bin') + ':' + str(ROOT / '.cache/mesa-venv/bin') + ':' + ENV['PATH'] +ENV['IR3_SHADER_DEBUG'] = 'asmroundtrip' + + +def run(command, *, timeout=30, ok=0, env=ENV): + p = subprocess.run([str(x) for x in command], env=env, capture_output=True, text=True, timeout=timeout) + if p.returncode != ok: + raise RuntimeError(f'Expected status {ok}, got {p.returncode}: {command[0]}\n{p.stdout[-2000:]}\n{p.stderr[-3000:]}') + return p + + +def sha(path): + return hashlib.sha256(path.read_bytes()).hexdigest() + + +if shutil.disk_usage(ROOT).free < 27 * 1024**3: + raise SystemExit('Require 7 GiB free to preserve the 5-GiB floor') +if run([LLVM, '--version']).stdout.strip() != '23.1.3': + raise SystemExit('This frontend is qualified only with LLVM 23.1.3') +llvm_bin = Path(run([LLVM, '--bindir']).stdout.strip()) +pins = json.loads((ROOT / 'scripts/data/adreno-compiler-inputs.json').read_text()) +reference = pins['integer_fma_reference'] +reference_root = ROOT / '.cache/softfloat-source' +for name, expected in reference['files'].items(): + path = reference_root / name + if not path.exists(): + path.parent.mkdir(parents=True, exist_ok=True) + url = reference['repository'].replace('github.com', 'raw.githubusercontent.com') + '/' + reference['commit'] + '/' + name + with urllib.request.urlopen(url, timeout=20) as stream: + path.write_bytes(stream.read(1024 * 1024)) + if sha(path) != expected: + raise RuntimeError('Pinned SoftFloat algorithm reference differs: ' + name) +if sha(KIT / 'licenses/SoftFloat-3e.txt') != reference['files']['COPYING.txt']: + raise RuntimeError('Full upstream SoftFloat license must be retained') +# The base builder verifies every Mesa source change against recorded patches, +# enables full NIR, disables runtime render drivers, and qualifies real IR3. +run([sys.executable, ROOT / 'scripts/build-adreno-compiler-host.py', '--probe'], timeout=600) +run([ROOT / '.cache/mesa-venv/bin/meson', 'configure', MESA, + '-Doffline-air-nir-dir=' + str(KIT), '-Doffline-air-nir-llvm-config=' + str(LLVM)]) +build = run(['ninja', '-C', MESA, '-j4', 'src/freedreno/ir3/air_nir'], timeout=180) +(ROOT / '.cache/air-nir-build.log').write_text(build.stdout + build.stderr) +exe = MESA / 'src/freedreno/ir3/air_nir' +OUT.mkdir(parents=True, exist_ok=True) +audit = json.loads((ROOT / 'build/gpu-analysis/air/target-inputs.json').read_text()) +modules = {(m['library'], m['index']): m for m in audit['modules']} +if len(modules) != 240: + raise SystemExit('The expected original library inventory must contain exactly 240 modules') + + +def source(library, index): + path = ROOT / f'build/gpu-analysis/air/{library}/module-{index}.bc' + if sha(path) != modules[(library, index)]['bitcode_sha256']: + raise RuntimeError('Original AIR input differs from audited library member') + return path + + +results = [] +fixtures = [('QuartzCore', 111, 0, 4, True), ('RenderBox', 0, 84, 6, True), + ('RenderBox', 2, 56, 6, True), ('RenderBox', 26, 28, 4, True), + ('RenderBox', 28, 60, 6, True), ('RenderBox', 45, 56, 6, True), + ('RenderBox', 65, 88, 6, True), ('RenderBox', 71, 56, 6, True)] +for library, index, size, channel_count, compiled in fixtures: + src = source(library, index) + prefix = OUT / f'{library}-{index}' + # Clean this known prefix so old products cannot satisfy a new refusal. + for suffix in ('.bin', '.asm', '.nir', '.eval'): + prefix.with_suffix(suffix).unlink(missing_ok=True) + rng = random.Random(20261016) + cases = [] + count = 65536 if size == 0 else 4096 + for i in range(count): + vertex = i if size == 0 else ([0, 1, 2, 3, 4, 65535, 65536, 65537, 0xffffffff][i] if i < 9 else rng.getrandbits(32)) + values = [rng.uniform(-32, 32) for _ in range(size // 4)] + if values: + values[6] = rng.uniform(0, 100) # finite, nonsingular depth/(depth+1) + cases.append((vertex, struct.pack('<' + 'f' * len(values), *values))) + case_path = prefix.with_suffix('.cases') + case_path.write_bytes(b''.join(struct.pack(' 1024 * 1024 or + not metadata['asm_roundtrip'] or metadata['gpu_executed']): + raise RuntimeError('Original vertex backend result differs') + if library == 'QuartzCore' and (binary_size != 128 or sha(prefix.with_suffix('.bin')) != + '183870069979673265622eb249a9a540422c650464b247f0ec248ce2555480ad'): + raise RuntimeError('Previously qualified original AIR generated code differs') + assembly = prefix.with_suffix('.asm').read_text() + for token in ('SYSTEM_VALUE_VERTEX_ID', 'VARYING_SLOT_POS'): + if token not in assembly: + raise RuntimeError('Original AIR generated code lacks expected data flow') + else: + raise RuntimeError('All eight selected originals must compile') + # Independent original-IR oracle: clone original module, adapt only host + # entry ABI/triple and replace the named AIR FMA with LLVM fused FMA. + oracle_bc = Path(str(prefix) + '.oracle.bc') + oracle_lib = Path(str(prefix) + '.dylib') + run([exe, src, CHIP, oracle_bc, '--test-oracle']) + run([llvm_bin / 'clang', '-dynamiclib', '-O0', oracle_bc, '-o', oracle_lib]) + library_handle = ctypes.CDLL(str(oracle_lib)) + oracle = library_handle.air_test_oracle + oracle.argtypes = [ctypes.c_void_p, ctypes.c_uint16, ctypes.POINTER(ctypes.c_float)] + oracle.restype = None + actual = prefix.with_suffix('.eval').read_bytes() + stride = ((channel_count + 3) // 4) * 16 + if len(actual) != count * stride: + raise RuntimeError('Missing/extra numeric test records') + channels = range(channel_count) + for i, (vertex, data) in enumerate(cases): + output = (ctypes.c_float * 8)() + buffer = ctypes.create_string_buffer(data or b'\0') + oracle(buffer, vertex & 65535, output) + got = struct.unpack_from('<' + 'f' * (stride // 4), actual, i * stride) + # Exact equality on these finite test values, including fused FMA + # cancellation behavior. This does not evaluate generated Adreno ISA. + if any(struct.pack(' 30: + raise RuntimeError('Coverage exceeded the bounded 30-second host test window') +successes = [(r['library'], r['index']) for r in coverage_results if r['exit_code'] == 0] +frontend_only = [(r['library'], r['index']) for r in coverage_results if r['exit_code'] == 5] +if frontend_only: + raise RuntimeError('Frontend-only coverage or refusal reason differs') +expected_successes = sorted([(library,index) for library,index,*_ in fixtures] + + [(r['library'],r['index']) for r in resource_report['originals']+fragment_report['originals']+fragment_input_report['originals']]) +if successes != expected_successes: + raise RuntimeError('Original library compiler coverage changed; qualify every new success') + +# Mutate ignored copies of the original IR for meaningful refusal controls; +# original target inputs and reference files stay unchanged. +qc = source('QuartzCore', 111).with_suffix('.ll').read_text() +rb = source('RenderBox', 0).with_suffix('.ll').read_text() +mutations = { + 'wrong-triple': qc.replace('air64-apple-ios17.7.0', 'arm64-apple-ios17.7.0'), + 'strict-float-options': qc.replace('air.compile.denorms_disable', 'air.compile.denorms_enable'), + 'strict-division': rb.replace('fdiv fast float', 'fdiv float'), + 'wrong-air-version': qc.replace('!10 = !{i32 2, i32 6, i32 0}', '!10 = !{i32 2, i32 7, i32 0}'), + 'missing-output-metadata': qc.replace('!17 = !{!"air.position", !"air.arg_type_name", !"float4"}', '!17 = !{}'), + 'unsupported-op': qc.replace('%3 = and i32 %2, 1', '%3 = xor i32 %2, 1'), + 'undefined-output': qc.replace('ret <4 x float> %10', 'ret <4 x float> poison'), + 'wrong-argument-kind': qc.replace('!"air.vertex_id"', '!"air.instance_id"'), + 'dynamic-gep': rb.replace('i64 0, i64 0\n %4 = load', 'i64 0, i16 %1\n %4 = load', 1), + 'out-of-range-gep': rb.replace('i64 0, i64 0\n %4 = load', 'i64 0, i64 100\n %4 = load', 1), + 'volatile-load': rb.replace('%4 = load float', '%4 = load volatile float', 1), + 'unknown-intrinsic': rb.replace('air.fma.v2f32', 'air.unknown.v2f32'), +} +negative = [] +for name, text in mutations.items(): + if text in (qc, rb): + raise RuntimeError('Refusal mutation did not change its source: ' + name) + ll, bc = OUT / (name + '.ll'), OUT / (name + '.bc') + ll.write_text(text) + run([llvm_bin / 'llvm-as', ll, '-o', bc]) + prefix = OUT / ('refused-' + name) + prefix.with_suffix('.bin').unlink(missing_ok=True) + p = run([exe, bc, CHIP, prefix, '--compile'], ok=4) + if prefix.with_suffix('.bin').exists() or 'AIR refused:' not in p.stderr: + raise RuntimeError('Mutation did not refuse before machine-code export: ' + name) + negative.append(dict(name=name, exit_code=4, reason=p.stderr.strip())) +for name, chip, env in [('unknown-chip', '0xdeadbeef', ENV), ('missing-roundtrip', CHIP, dict(ENV, IR3_SHADER_DEBUG=''))]: + prefix = OUT / ('refused-' + name) + prefix.with_suffix('.bin').unlink(missing_ok=True) + p = run([exe, source('QuartzCore', 111), chip, prefix, '--compile'], ok=4, env=env) + if prefix.with_suffix('.bin').exists(): + raise RuntimeError('Target/check refusal exported code') + negative.append(dict(name=name, exit_code=4, reason=p.stderr.strip())) +truncated = OUT / 'truncated.bc' +truncated.write_bytes(source('QuartzCore', 111).read_bytes()[:64]) +p = run([exe, truncated, CHIP, OUT / 'refused-truncated', '--compile'], ok=4) +negative.append(dict(name='truncated-bitcode', exit_code=4, reason=p.stderr.strip())) +run([sys.executable, KIT / 'tests/check_fma.py', '--compiler', exe, + '--llvm-bindir', llvm_bin, '--out', OUT / 'fma', '--chip', CHIP], timeout=60) +fma_report = json.loads((OUT / 'fma/report.json').read_text()) +run([sys.executable, KIT / 'tests/check_coefficients.py', '--compiler', exe, + '--llvm-bindir', llvm_bin, '--out', OUT / 'coefficients', '--chip', CHIP], timeout=30) +coefficient_report = json.loads((OUT / 'coefficients/report.json').read_text()) +run([sys.executable, KIT / 'tests/check_math_flags.py', '--compiler', exe, + '--llvm-bindir', llvm_bin, '--out', OUT / 'math-flags', '--chip', CHIP], timeout=30) +math_flags_report = json.loads((OUT / 'math-flags/report.json').read_text()) +run([sys.executable, ROOT/'scripts/check-adreno-constant-uploads.py', '--compiler', exe], timeout=60) +constant_upload_report=json.loads((OUT/'constant-uploads/report.json').read_text()) +compiled_hashes={(r['library'],r['index']):r['binary_sha256'] for r in results+resource_report['originals']} +packed_hashes={(r['library'],r['index']):r['binary_sha256'] for r in constant_upload_report['original_layouts']} +if compiled_hashes != packed_hashes: + raise RuntimeError('Shader binaries changed between numerical qualification and ABI packing') +report = dict(llvm_version='23.1.3', llvm_config_sha256=sha(LLVM), executable_sha256=sha(exe), + kit_sources={p.name: sha(p) for p in sorted(KIT.glob('*')) if p.suffix in ('.c', '.cpp', '.h')}, + mesa_commit=json.loads((ROOT / 'scripts/data/adreno-compiler-inputs.json').read_text())['mesa']['commit'], + chip_id=CHIP, chip_identity_source='pinned board DTS + Mesa catalog; hardware not read', + original_cases=results, coverage=coverage_results, refusal_controls=negative, + integer_fma_reference=reference, fused_f32_checks=fma_report, + coefficient_fma_checks=coefficient_report, per_instruction_math_checks=math_flags_report, + original_resource_checks=resource_report, original_fragment_checks=fragment_report, + original_fragment_input_checks=fragment_input_report, + constant_upload_checks=constant_upload_report, + original_modules=240, compiled_modules=len(successes), frontend_only_modules=len(frontend_only), + host_only=True, generated_isa_executed=False, ios_compiler_crossbuilt=False, + metal_pipeline_linked=False, complete_library_support=False, + test_oracle_air_fma_semantics='Explicit fused-FMA interpretation; AIR implementation equivalence remains unqualified') +(OUT / 'report.json').write_text(json.dumps(report, indent=2) + '\n') +if shutil.disk_usage(ROOT).free < 5 * 1024**3: + raise RuntimeError('Disk floor violated') +print('PASS: 240 originals classified, 15 refusal controls; 22 original shaders compiled with verified vertex/resource/fragment outputs and center-perspective inputs + real roundtrip') +print('No GPU execution, complete AIR library support, native iOS compiler or Metal/UI claim') diff --git a/scripts/build-air-nir-ios.py b/scripts/build-air-nir-ios.py new file mode 100644 index 0000000..cf1ecb1 --- /dev/null +++ b/scripts/build-air-nir-ios.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +"""Cross-build the actual AIR/NIR/IR3 compiler for ordinary ARM64/iOS17.""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import shutil +import shlex +import subprocess +import time +ROOT=Path(__file__).resolve().parents[1] +OUT=ROOT/'build/mesa-ir3-ios';TOOLS=ROOT/'build/air-nir-ios';LLVM=ROOT/'build/llvm-ios' +SOURCE=ROOT/'.cache/mesa-work';LLVMSOURCE=ROOT/'.cache/llvm-ios-source' +p=argparse.ArgumentParser(description=__doc__);p.add_argument('--configure-only',action='store_true');a=p.parse_args() +TOOLS.mkdir(parents=True,exist_ok=True) +def read(cmd):return subprocess.check_output([str(v) for v in cmd],text=True,timeout=30).strip() +def run(cmd,log,timeout): + with (TOOLS/log).open('w') as f: + process=subprocess.Popen([str(v) for v in cmd],stdout=f,stderr=subprocess.STDOUT,env=env) + start=time.monotonic() + while process.poll() is None: + if shutil.disk_usage(ROOT).free<5*1024**3 or time.monotonic()-start>timeout: + process.terminate() + try:process.wait(timeout=10) + except subprocess.TimeoutExpired:process.kill();process.wait() + raise SystemExit('Native compiler stopped at disk/time bound; see '+log) + time.sleep(1) + if process.returncode:raise SystemExit('Native compiler failed; see '+log) +if shutil.disk_usage(ROOT).free<28*1024**3:raise SystemExit('Require 28 GiB free before native compiler build') +pins=json.loads((ROOT/'scripts/data/adreno-compiler-inputs.json').read_text()) +if read(['git','-C',SOURCE,'rev-parse','HEAD'])!=pins['mesa']['commit']:raise SystemExit('Pinned Mesa source differs') +for patch in pins['patches']: + subprocess.run(['git','-C',SOURCE,'apply','--reverse','--check',ROOT/patch],check=True,capture_output=True) +sdk=read(['xcrun','--sdk','iphoneos','--show-sdk-path']) +clang=read(['xcrun','--sdk','iphoneos','--find','clang']);clangxx=read(['xcrun','--sdk','iphoneos','--find','clang++']) +config=shutil.which('llvm-config') +if not config and shutil.which('brew'):config=str(Path(read(['brew','--prefix','llvm']))/'bin/llvm-config') +if not config or read([config,'--version'])!='23.1.3':raise SystemExit('Need the same host LLVM config for component graph') +libs=shlex.split(read([config,'--link-static','--libs','core','bitreader','bitwriter','analysis','targetparser'])) +wrapper=TOOLS/'llvm-config-ios' +cxxflags=['-I'+str(LLVMSOURCE/'llvm/include'),'-I'+str(LLVM/'include'),'-std=c++17','-stdlib=libc++','-fno-exceptions', + '-D__STDC_CONSTANT_MACROS','-D__STDC_FORMAT_MACROS','-D__STDC_LIMIT_MACROS'] +wrapper.write_text('#!/usr/bin/env python3\nimport sys\n'+ + 'if sys.argv[1:]==["--version"]: print("23.1.3")\n'+ + 'elif "--cxxflags" in sys.argv: print('+repr(' '.join(cxxflags))+')\n'+ + 'elif "--libs" in sys.argv: print('+repr(' '.join(['-L'+str(LLVM/'lib')]+libs))+')\n'+ + 'else: raise SystemExit("Unsupported native component config request")\n') +wrapper.chmod(0o755) +cross=TOOLS/'ios.cross' +flags=['-target','arm64-apple-ios17.0','-isysroot',sdk,'-fstack-protector-strong'] +cross.write_text('[binaries]\nc = '+repr(clang)+'\ncpp = '+repr(clangxx)+'\nar = '+repr(read(['xcrun','--sdk','iphoneos','--find','ar']))+'\n'+ + 'objc = '+repr(clang)+'\nstrip = '+repr(read(['xcrun','--sdk','iphoneos','--find','strip']))+'\npkg-config = \'/usr/bin/false\'\n'+ + '[host_machine]\nsystem = \'darwin\'\ncpu_family = \'aarch64\'\ncpu = \'arm64\'\nendian = \'little\'\n'+ + '[properties]\nneeds_exe_wrapper = true\n'+ + '[built-in options]\nc_args = '+repr(flags)+'\ncpp_args = '+repr(flags)+'\nobjc_args = '+repr(flags)+'\nobjc_link_args = '+repr(flags)+'\nc_link_args = '+repr(flags)+'\ncpp_link_args = '+repr(flags)+'\n') +env={k:v for k,v in os.environ.items() if not k.startswith(('IR3_','FD_MESA_','NIR_','MESA_SHADER_'))} +env.update(PATH=str(ROOT/'.cache/mesa-tools/bin')+':'+str(ROOT/'.cache/mesa-venv/bin')+':'+env['PATH'], + CC=clang,CXX=clangxx,CFLAGS='',CXXFLAGS='',CPPFLAGS='',LDFLAGS='') +options=['-Dgallium-drivers=','-Dvulkan-drivers=','-Dplatforms=','-Dglx=disabled','-Degl=disabled', + '-Dgbm=disabled','-Dopengl=false','-Dgles1=disabled','-Dgles2=disabled','-Dllvm=disabled', + '-Dshared-glapi=disabled','-Dxmlconfig=disabled','-Dshader-cache=disabled','-Dtools=freedreno', + '-Dbuild-tests=false','-Dvideo-codecs=','-Dlibunwind=disabled','-Dlmsensors=disabled', + '-Dvalgrind=disabled','-Dexpat=disabled','-Dzstd=disabled','-Dzlib=disabled', + '-Dbuildtype=debugoptimized','-Db_ndebug=false','-Doffline-air-nir-dir='+str(ROOT/'kits/AirNir'), + '-Doffline-air-nir-llvm-config='+str(wrapper)] +meson=ROOT/'.cache/mesa-venv/bin/meson' +if not (OUT/'build.ninja').exists():run([meson,'setup',OUT,SOURCE,'--cross-file',cross,'--wrap-mode=nodownload',*options],'configure.log',180) +else:run([meson,'configure',OUT,*options],'configure.log',60) +if a.configure_only: + print('PASS: native AIR/NIR/IR3 cross configuration; build and runtime remain unqualified') + raise SystemExit(0) +report=json.loads((LLVM/'build-report.json').read_text()) +if report['llvm_revision']!='0d261d1ca552c95a8f007e061c787ac7132fbcbc':raise SystemExit('Native LLVM identity differs') +for name,digest in report['archives'].items(): + if hashlib.sha256((LLVM/'lib'/name).read_bytes()).hexdigest()!=digest['sha256']:raise SystemExit('Native LLVM archive differs') +run(['ninja','-C',OUT,'-j4','src/freedreno/ir3/air_nir'],'build.log',600) +exe=OUT/'src/freedreno/ir3/air_nir' +(TOOLS/'build-report.json').write_text(json.dumps(dict(llvm_revision=report['llvm_revision'],mesa_revision=pins['mesa']['commit'], + target='arm64-apple-ios17.0',sdk_version=read(['xcrun','--sdk','iphoneos','--show-sdk-version']), + executable_bytes=exe.stat().st_size,executable_sha256=hashlib.sha256(exe.read_bytes()).hexdigest(), + kit_sources={p.name:hashlib.sha256(p.read_bytes()).hexdigest() for p in sorted((ROOT/'kits/AirNir').glob('*')) if p.suffix in ('.h','.c','.cpp')}, + target_runtime_executed=False,generated_shader_executed=False,metal_device=False),indent=2)+'\n') +print('PASS: actual AIR/NIR/IR3 compiler cross-built for ARM64/iOS17; runtime/Metal not qualified') diff --git a/scripts/build-corecrypto-14.py b/scripts/build-corecrypto-14.py new file mode 100755 index 0000000..710e1da --- /dev/null +++ b/scripts/build-corecrypto-14.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python3 +"""Compile the pinned 2023 crypto provider internally for ABI verification. + +Apple source, generated headers and objects remain ignored. This does not +register the provider, produce a certified module or demonstrate a kernel boot. +""" +import concurrent.futures +import argparse +import hashlib +import json +from pathlib import Path +import subprocess +import struct +import sys + +ROOT = Path(__file__).resolve().parents[1] +SOURCE = ROOT / 'refer/corecrypto-2023' +REVISION = '11fba65b12d8830630e248dfa8344caf8b065f26' +OUT = ROOT / 'build/corecrypto-2023' +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument('--namespace', action='store_true', help='Isolate provider globals from XNU crypto wrappers') +args = parser.parse_args() + + +def names(path, mode): + text = subprocess.check_output(['xcrun', 'nm', '-g', mode, '-j', path], text=True) + return {line.strip() for line in text.splitlines() if line.startswith('_')} + + +if subprocess.check_output(['git', '-C', SOURCE, 'rev-parse', 'HEAD'], text=True).strip() != REVISION: + raise SystemExit('Unexpected corecrypto source revision') +if subprocess.check_output(['git', '-C', SOURCE, 'status', '--porcelain'], text=True): + raise SystemExit('The crypto reference must stay immutable') +OUT.mkdir(parents=True, exist_ok=True) +subprocess.run(['plutil', '-convert', 'json', '-o', OUT / 'project.json', + SOURCE / 'corecrypto.xcodeproj/project.pbxproj'], check=True) +project = json.loads((OUT / 'project.json').read_text()) +objects = project['objects'] +parents = {child: key for key, value in objects.items() for child in value.get('children', [])} + + +def resolve(key): + entry = objects[key] + path = entry.get('path', '') + if entry.get('sourceTree') == 'SOURCE_ROOT' or key not in parents: + return SOURCE / path + if entry.get('sourceTree') != '': + raise ValueError(f'Unsupported source tree: {entry}') + return resolve(parents[key]) / path + + +target = next(value for value in objects.values() if value.get('isa') == 'PBXNativeTarget' + and value.get('name') == 'corecrypto_kext') +phase = next(objects[key] for key in target['buildPhases'] if objects[key]['isa'] == 'PBXSourcesBuildPhase') +sources = [resolve(objects[key]['fileRef']) for key in phase['files'] + if objects[objects[key]['fileRef']].get('path', '').endswith('.c')] +for source in sources: + if not source.is_file() or not source.resolve().is_relative_to(SOURCE): + raise SystemExit(f'Missing or unexpected source path: {source}') +include = OUT / 'include/corecrypto' +include.mkdir(parents=True, exist_ok=True) +for header in SOURCE.glob('*/corecrypto/*.h'): + link = include / header.name + if link.exists() and link.resolve() != header.resolve(): + raise SystemExit(f'Conflicting header: {header.name}') + if not link.exists(): + link.symlink_to(header) +sdk = subprocess.check_output(['xcrun', '--sdk', 'iphoneos', '--show-sdk-path'], text=True).strip() +framework = ROOT / '.cache/fakeroot/System/Library/Frameworks/Kernel.framework/Versions/A' +command = ['xcrun', '--sdk', 'iphoneos', 'clang', '-target', 'arm64-apple-ios17.0', '-isysroot', sdk, + '-mkernel', '-ffreestanding', '-fno-builtin', '-fno-stack-check', '-std=gnu99', '-fblocks', '-O2', + '-fno-vectorize', '-fno-slp-vectorize', '-mno-implicit-float', + '-DKERNEL=1', '-DKERNEL_PRIVATE=1', '-DBUILDKERNEL=1', '-DCC_USE_ASM=0', '-DNDEBUG=1', + '-DCC_DISABLE_RSAKEYGEN=1', '-Wno-deprecated-declarations', '-Wno-incompatible-library-redeclaration', + '-I' + str(include.parent), '-I' + str(framework / 'PrivateHeaders'), '-I' + str(framework / 'Headers')] +for folder in sorted({h.parent for h in SOURCE.rglob('*.h') if 'test' not in str(h.relative_to(SOURCE)).lower()}): + command.extend(['-I', str(folder)]) +kernel = ROOT / 'build/xnu/ios-obj/DEVELOPMENT_ARM64_BCM2837/mach.development.bcm2837.unstripped.noctf' +variant = '-namespaced' if args.namespace else '' +if args.namespace: + plain = OUT / 'libcorecrypto_kernel_noasm.a' + if not plain.is_file(): + raise SystemExit('Build the plain provider once before creating its namespace') + collisions = sorted(names(plain, '-U') & names(kernel, '-U')) + if any(not name[1:].isidentifier() for name in collisions): + raise SystemExit('A colliding symbol cannot be namespaced with a C identifier') + header = OUT / 'namespace.h' + header.write_text(''.join(f'#define {name[1:]} cc14_{name[1:]}\n' for name in collisions)) + command.extend(['-include', str(header)]) +(OUT / f'compile-command{variant}.json').write_text(json.dumps(command, indent=2) + '\n') +object_dir = OUT / f'objects{variant}' +object_dir.mkdir(exist_ok=True) + + +def compile_one(source): + relative = str(source.relative_to(SOURCE)) + name = hashlib.sha256(relative.encode()).hexdigest()[:12] + '-' + source.stem + destination = object_dir / (name + '.o') + result = subprocess.run([*command, '-c', source, '-o', destination], stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, text=True) + (object_dir / (name + '.log')).write_text(result.stdout) + return {'source': relative, 'object': destination.name, 'exit_code': result.returncode, + 'diagnostic': result.stdout[-2500:] if result.returncode else None} + + +with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor: + results = list(executor.map(compile_one, sources)) +failed = [result for result in results if result['exit_code']] +report = {'source_url': 'https://github.com/ONLYA/corecrypto-2023', 'source_revision': REVISION, + 'provenance': 'Third-party source snapshot; not authenticated against an Apple archive', + 'module_version': '14.0', 'target': 'arm64-apple-ios17.0', 'assembly_enabled': False, + 'compiled_sources': len(results), 'failures': len(failed), 'objects': results, + 'kernel_provider_registered': False, 'crypto_selftests_passed': False} +(OUT / f'build-report{variant}.json').write_text(json.dumps(report, indent=2) + '\n') +print(f'{len(results)} kernel C sources; {len(failed)} failures', flush=True) +for result in failed[:8]: + print(result['source'], result['diagnostic'], flush=True) +if failed: + sys.exit(1) +for result in results: + data = (object_dir / result['object']).read_bytes() + magic, cpu, subtype, kind, count, size = struct.unpack_from('<6I', data) + if (magic, cpu, subtype, kind) != (0xfeedfacf, 0x100000c, 0, 1): + raise SystemExit(f'Wrong object architecture: {result["object"]}') + offset = 32 + deployment = None + for _ in range(count): + cmd, length = struct.unpack_from('=5.2*1024**3 +names='SOURCE_CORECRYPTO14 SOURCE_PTHREAD ARM_DT_PLATFORM ARMCPU NATIVE_UFS NATIVE_AES NATIVE_VIRTIO_BLOCK NATIVE_RAM_BLOCK TRUST_CACHE_RUNTIME SOURCE_MOCKFS NATIVE_CS_HASH_CACHE VERIFIED_ROOT_PROBE OWNER_ROOT_AUDIT OWNER_ROOT_TRUST SHENG_PMGR OWNER_CACHE_MOUNT OWNER_CACHE_STARTUP OWNER_BOOT_MANIFEST OWNER_DATA_MOUNT OWNER_UNION_SYSTEMOS DT_PANIC_REGION ZONE_PGZ_ALIGNMENT_PROBE NATIVE_ARCH_TIMER NATIVE_PAN BOOT_PROGRESS NATIVE_LEGACY_DISPLAY NATIVE_ADRENO'.split() +updates={'XNU_'+n:'1' for n in names} +updates.update(XNU_CRYPTO_BOOTSTRAP='prelinked',XNU_OWNER_DATA_VOLUME_UUID='18AB5557-DC2D-469B-8758-855E60F5B0DF', + XNU_NATIVE_VIRTIO_BLOCK_ARCHIVE=str(root/'build/native-virtio-block-verified-bundle/libNativeVirtioBlock.a'), + XNU_VERIFIED_ROOT_ARCHIVE=str(root/'build/verified-root-os-xnu-bundle/libVerifiedRootProbe.a'), + XNU_EXTRA_LD_ARCHIVES='',XNU_NATIVE_MOBILE_DISPLAY='0',XNU_ARMVIRT_PLATFORM='0',JOBS='4') +(root/'build/cpu-desktop/kernel-environment.json').write_text(json.dumps(updates,indent=2)+'\n') +env=dict(os.environ);env.update(updates) +with (root/'build/cpu-desktop/kernel-build.log').open('w') as f: + result=subprocess.run(['bash','scripts/build-xnu-4k-baseline.sh'],env=env,stdout=f,stderr=subprocess.STDOUT,timeout=900) +print('kernel build exit',result.returncode,'free GiB',round(shutil.disk_usage(root).free/1024**3,2)) +if result.returncode:raise SystemExit('KERNEL BUILD FAILED: see build/cpu-desktop/kernel-build.log') diff --git a/scripts/build-crabapple.sh b/scripts/build-crabapple.sh new file mode 100755 index 0000000..1336e8c --- /dev/null +++ b/scripts/build-crabapple.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +if [ "$(uname -s)" = Darwin ]; then + LLVM_PREFIX=${LLVM_PREFIX:-$(brew --prefix llvm)} + LLD_PREFIX=${LLD_PREFIX:-$(brew --prefix lld)} + TOOLS=$ROOT/.cache/toolchain + mkdir -p "$TOOLS" + for tool in clang llvm-lib llvm-rc; do + ln -sf "$LLVM_PREFIX/bin/$tool" "$TOOLS/$tool" + done + ln -sf "$LLD_PREFIX/bin/lld-link" "$TOOLS/lld-link" + export CLANG_BIN=$TOOLS/ +fi +export JOBS=${JOBS:-4} +exec "$ROOT/crabapple/build.sh" "$@" diff --git a/scripts/build-darwinboot.sh b/scripts/build-darwinboot.sh new file mode 100755 index 0000000..f27dfa2 --- /dev/null +++ b/scripts/build-darwinboot.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +export WORKSPACE=$ROOT +export PACKAGES_PATH=$ROOT/drivers/DarwinBoot:$ROOT/crabapple:$ROOT/crabapple/Mu_Basecore +export EDK_TOOLS_PATH=$ROOT/crabapple/Mu_Basecore/BaseTools +export CONF_PATH=$ROOT/crabapple/Build/Conf +export CLANG_BIN=${CLANG_BIN:-$ROOT/.cache/toolchain/} +export PYTHON_COMMAND=python3 +export PATH=$EDK_TOOLS_PATH/BinWrappers/PosixLike:$EDK_TOOLS_PATH/Source/C/bin:$PATH +cd "$ROOT" +build -p DarwinBootPkg/DarwinBoot.dsc -a AARCH64 -b "${TARGET:-DEBUG}" -t CLANGPDB -n "${JOBS:-4}" diff --git a/scripts/build-firehose.sh b/scripts/build-firehose.sh new file mode 100755 index 0000000..c47301d --- /dev/null +++ b/scripts/build-firehose.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +cd "$ROOT" +if [ ! -d .cache/libdispatch-work/.git ]; then + git clone --shared refer/libdispatch .cache/libdispatch-work +fi +python3 - <<'PY' +from pathlib import Path +p = Path('.cache/libdispatch-work/xcodeconfig/libfirehose_kernel.xcconfig') +s = p.read_text() +s = s.replace('$(SDKROOT)/System/Library/Frameworks/Kernel.framework', + '$(FAKEROOT_DIR)/System/Library/Frameworks/Kernel.framework') +s = s.replace('$(SDKROOT)/usr/local/include', '$(FAKEROOT_DIR)/usr/local/include') +lines = s.splitlines() +for index, line in enumerate(lines): + if line.startswith('HEADER_SEARCH_PATHS =') and '$(SDKROOT)/usr/include' not in line: + lines[index] += (' $(SDKROOT)/usr/include $(FAKEROOT_DIR)/usr/local/include' + ' $(PROJECT_DIR)/../xnu-work/EXTERNAL_HEADERS') +s = '\n'.join(lines) + '\n' +if p.read_text() != s: + p.write_text(s) +PY +xcodebuild install -project .cache/libdispatch-work/libdispatch.xcodeproj \ + -target libfirehose_kernel -configuration Release -sdk iphoneos -jobs "${JOBS:-4}" \ + ARCHS=arm64 VALID_ARCHS=arm64 ONLY_ACTIVE_ARCH=NO IPHONEOS_DEPLOYMENT_TARGET=17.0 \ + CODE_SIGNING_ALLOWED=NO GCC_TREAT_WARNINGS_AS_ERRORS=NO \ + OBJROOT="$ROOT/build/firehose/obj" SYMROOT="$ROOT/build/firehose/sym" \ + DSTROOT="$ROOT/.cache/fakeroot" FAKEROOT_DIR="$ROOT/.cache/fakeroot" +if [ -f .cache/fakeroot/usr/local/lib/kernel/liblibfirehose_kernel.a ]; then + cp .cache/fakeroot/usr/local/lib/kernel/liblibfirehose_kernel.a \ + .cache/fakeroot/usr/local/lib/kernel/libfirehose_kernel.a +fi +xcrun lipo -info .cache/fakeroot/usr/local/lib/kernel/libfirehose_kernel.a diff --git a/scripts/build-libpthread-kernel.py b/scripts/build-libpthread-kernel.py new file mode 100644 index 0000000..96ccde2 --- /dev/null +++ b/scripts/build-libpthread-kernel.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 +"""Build the matching Darwin 23 pthread kernel component and its real registration.""" +import json +from pathlib import Path +import struct +import subprocess + +ROOT = Path(__file__).resolve().parents[1] +SOURCE = ROOT / 'refer/libpthread' +REVISION = 'c0e60fb5ecfc54cf379c261d3f9572cc22ded891' +if subprocess.check_output(['git', '-C', SOURCE, 'rev-parse', 'HEAD'], text=True).strip() != REVISION: + raise SystemExit('Unexpected libpthread source revision') +if subprocess.check_output(['git', '-C', SOURCE, 'status', '--porcelain'], text=True): + raise SystemExit('The libpthread reference must remain immutable') +WORK = ROOT / '.cache/libpthread-work' +if not WORK.exists(): + subprocess.run(['git', 'clone', '--shared', SOURCE, WORK], check=True) +for patch in sorted((ROOT / 'patches/libpthread').glob('*.patch')): + if subprocess.run(['git', '-C', WORK, 'apply', '--reverse', '--check', patch], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL).returncode: + subprocess.run(['git', '-C', WORK, 'apply', patch], check=True) +SOURCE = WORK +OUT = ROOT / 'build/libpthread-kernel' +OUT.mkdir(parents=True, exist_ok=True) +recipe = ROOT / 'build/xnu/ios-obj/DEVELOPMENT_ARM64_BCM2837/osfmk/DEVELOPMENT/prng_random.o.json' +record = json.loads(recipe.read_text().strip().rstrip(',')) +sdk = subprocess.check_output(['xcrun', '--sdk', 'iphoneos', '--show-sdk-path'], text=True).strip() + + +def compile_one(source, destination, namespace=None): + if source.name == 'source_pthread_bootstrap.c': + command = [] + arguments = record['arguments'] + index = 0 + while index < len(arguments): + value = arguments[index] + if value == record['file']: + command.append(str(source)) + elif value == '-o': + command.extend(['-o', str(destination)]) + index += 1 + else: + command.append(value) + index += 1 + else: + framework = ROOT / '.cache/fakeroot/System/Library/Frameworks/Kernel.framework/Versions/A' + command = ['xcrun', '--sdk', 'iphoneos', 'clang', '-target', 'arm64-apple-ios17.0', + '-isysroot', sdk, '-mkernel', '-ffreestanding', '-fno-builtin', '-fno-stack-check', + '-std=gnu11', '-fblocks', '-O2', '-mno-implicit-float', + '-DKERNEL=1', '-DKERNEL_PRIVATE=1', '-DXNU_KERNEL_PRIVATE=1', + '-DMACH_KERNEL_PRIVATE=1', '-DMACH_ASSERT=1', '-DDEBUG=1', + '-D__PTHREAD_EXPOSE_INTERNALS__=1', '-DABSOLUTETIME_SCALAR_TYPE=1', '-DNEEDS_SCHED_CALL_T=1', + '-I', str(SOURCE / 'private'), '-I', str(SOURCE / 'include'), '-I', str(SOURCE), + '-I', str(framework / 'PrivateHeaders'), '-I', str(framework / 'Headers'), + '-Wno-deprecated-declarations', '-c', str(source), '-o', str(destination)] + if namespace: + command += ['-include', str(namespace)] + result = subprocess.run(command, cwd=record['directory'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + destination.with_suffix('.log').write_text(result.stdout) + if result.returncode: + raise SystemExit(result.stdout[-4500:]) + data = destination.read_bytes() + if struct.unpack_from('<4I', data) != (0xfeedfacf, 0x100000c, 0, 1): + raise SystemExit('Unexpected pthread object architecture') + offset = 32 + deployment = None + for _ in range(struct.unpack_from('timeout: + process.terminate() + try:process.wait(timeout=10) + except subprocess.TimeoutExpired:process.kill();process.wait() + raise SystemExit('Native dependency build stopped at disk/time bound; see '+log) + time.sleep(1) + if process.returncode:raise SystemExit('Native dependency build failed; see '+log) + +if shutil.disk_usage(ROOT).free<28*1024**3:raise SystemExit('Require 28 GiB free before native dependency build') +if not (SOURCE/'.git').exists(): + run(['git','clone','--filter=blob:none','--depth','1','--single-branch','--branch','llvmorg-23.1.3', + '--sparse','https://github.com/llvm/llvm-project.git',SOURCE],'fetch.log',120) +if read(['git','-C',SOURCE,'rev-parse','HEAD'])!=REV:raise SystemExit('Pinned LLVM release differs') +if read(['git','-C',SOURCE,'status','--porcelain']):raise SystemExit('LLVM source must match recorded patches') +# LLVM23 uses only libc's common-utility headers; this does not build or +# replace the target system libc. Omit LLVM tests/docs/examples entirely. +patterns='/llvm/*\n!/llvm/test/\n!/llvm/unittests/\n!/llvm/benchmarks/\n!/llvm/examples/\n!/llvm/docs/\n/cmake/\n/third-party/\n/libc/\n' +current=(SOURCE/'.git/info/sparse-checkout').read_text() if (SOURCE/'.git/info/sparse-checkout').exists() else '' +if current!=patterns: + subprocess.run(['git','-C',SOURCE,'sparse-checkout','set','--no-cone','--stdin'],input=patterns,text=True,check=True,timeout=120) +config=shutil.which('llvm-config') +if not config and shutil.which('brew'):config=str(Path(read(['brew','--prefix','llvm']))/'bin/llvm-config') +if not config or read([config,'--version'])!=VERSION:raise SystemExit('Matching native host TableGen/config required') +tblgen=Path(read([config,'--bindir']))/'llvm-tblgen' +if not tblgen.is_file():raise SystemExit('Missing qualified host TableGen') +sdk=read(['xcrun','--sdk','iphoneos','--show-sdk-path']) +clang=read(['xcrun','--sdk','iphoneos','--find','clang']);clangxx=read(['xcrun','--sdk','iphoneos','--find','clang++']) +args=['cmake','-G','Ninja','-S',SOURCE/'llvm','-B',OUT, + '-DCMAKE_SYSTEM_NAME=iOS','-DCMAKE_BUILD_TYPE=Release','-DCMAKE_OSX_ARCHITECTURES=arm64', + '-DCMAKE_OSX_DEPLOYMENT_TARGET=17.0','-DCMAKE_OSX_SYSROOT='+sdk, + '-DCMAKE_C_COMPILER='+clang,'-DCMAKE_CXX_COMPILER='+clangxx, + '-DCMAKE_C_COMPILER_TARGET=arm64-apple-ios17.0','-DCMAKE_CXX_COMPILER_TARGET=arm64-apple-ios17.0', + '-DCMAKE_C_FLAGS=-fstack-protector-strong','-DCMAKE_CXX_FLAGS=-fstack-protector-strong', + '-DLLVM_DEFAULT_TARGET_TRIPLE=arm64-apple-ios17.0','-DLLVM_TARGETS_TO_BUILD=', + '-DLLVM_TABLEGEN='+str(tblgen),'-DLLVM_ENABLE_ASSERTIONS=ON', + '-DBUILD_SHARED_LIBS=OFF','-DLLVM_BUILD_LLVM_DYLIB=OFF','-DLLVM_LINK_LLVM_DYLIB=OFF', + '-DLLVM_ENABLE_PROJECTS=','-DLLVM_ENABLE_RUNTIMES='] +for name in ['LLVM_INCLUDE_TESTS','LLVM_BUILD_TESTS','LLVM_INCLUDE_TOOLS','LLVM_BUILD_TOOLS', + 'LLVM_BUILD_UTILS','LLVM_INCLUDE_UTILS','LLVM_INCLUDE_EXAMPLES','LLVM_INCLUDE_BENCHMARKS','LLVM_INCLUDE_DOCS', + 'LLVM_ENABLE_ZLIB','LLVM_ENABLE_ZSTD','LLVM_ENABLE_LIBXML2','LLVM_ENABLE_TERMINFO','LLVM_ENABLE_LIBEDIT','LLVM_ENABLE_LIBPFM']: + args+=['-D'+name+'=OFF'] +run(args,'configure.log',120) +run(['cmake','--build',OUT,'--parallel','4','--target','LLVMBitReader','LLVMBitWriter','LLVMTargetParser'],'build.log',600) +archives=sorted((OUT/'lib').glob('libLLVM*.a')) +if not archives:raise SystemExit('No target archives produced') +report=dict(llvm_revision=REV,llvm_tree=read(['git','-C',SOURCE,'rev-parse','HEAD^{tree}']),llvm_version=VERSION, + host_tblgen_sha256=sha(tblgen),sdk_version=read(['xcrun','--sdk','iphoneos','--show-sdk-version']), + target='arm64-apple-ios17.0',native_compiler_runtime_executed=False, + archives={p.name:dict(bytes=p.stat().st_size,sha256=sha(p)) for p in archives}) +(OUT/'build-report.json').write_text(json.dumps(report,indent=2)+'\n') +print('PASS: LLVM23 AIR-reader/IR dependencies built for ordinary ARM64/iOS17; target runtime not executed') diff --git a/scripts/build-metal-vulkan-probe.py b/scripts/build-metal-vulkan-probe.py new file mode 100644 index 0000000..6bf4b2f --- /dev/null +++ b/scripts/build-metal-vulkan-probe.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +"""Build an offline exact-policy AIR/SPIR-V probe, with no Vulkan loader or GPU.""" +import argparse +import hashlib +import json +from pathlib import Path +import shutil +import subprocess + +ROOT = Path(__file__).resolve().parents[1] +PIN = json.loads((ROOT / 'scripts/data/gpu-metal-vulkan-inputs.json').read_text())['translator']['commit'] + + +def digest(path): + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--translator-source', type=Path, + default=ROOT / '.cache/gpu-reference/metal2vulkan') + parser.add_argument('--output', type=Path, default=ROOT / 'build/gpu-001/compiler') + args = parser.parse_args() + source, output = args.translator_source.resolve(), args.output.resolve() + (output / 'build-report.json').unlink(missing_ok=True) + if shutil.disk_usage(ROOT).free < 7 * 1024**3: + raise SystemExit('Preserve the 5 GiB floor and 2 GiB build allowance') + revision = subprocess.check_output(['git', '-C', str(source), 'rev-parse', 'HEAD'], text=True).strip() + if revision != PIN: + raise SystemExit('Translator revision differs from the qualified source pin') + if subprocess.check_output(['git', '-C', str(source), 'diff', 'HEAD', '--'], text=True): + raise SystemExit('Translator source must be unmodified') + toolchain = subprocess.check_output(['rustc', '+1.98.0', '--version'], text=True).strip() + if not toolchain.startswith('rustc 1.98.0 '): + raise SystemExit('This probe requires Rust 1.98.0') + output.mkdir(parents=True, exist_ok=True) + manifest = output / 'Cargo.toml' + # Only generated build files contain local dependency paths. + manifest.write_text('\n'.join([ + '[package]', 'name = "native-metal-vulkan-probe"', 'version = "0.1.0"', + 'edition = "2021"', 'rust-version = "1.98"', '[workspace]', + '[[bin]]', 'name = "native-metal-vulkan-probe"', + 'path = ' + json.dumps(str(ROOT / 'drivers/NativeMetal/tools/MetalVulkanProbe.rs')), + '[dependencies]', + 'metal2vulkan = { path = ' + json.dumps(str(source / 'crates/metal2vulkan')) + ', features = ["serde"] }', + 'serde_json = "=1.0.154"', '[profile.release]', 'incremental = true', + '', + ])) + shutil.copyfile(ROOT / 'drivers/NativeMetal/tools/metal-vulkan-probe.lock', output / 'Cargo.lock') + target = output / 'target' + command = ['cargo', '+1.98.0', 'build', '--release', '--locked', '--manifest-path', str(manifest), + '--target-dir', str(target), '-j4'] + with (output / 'build.log').open('w') as log: + result = subprocess.run(command, stdout=log, stderr=subprocess.STDOUT, timeout=600) + if result.returncode: + raise SystemExit((output / 'build.log').read_text()[-6000:]) + binary = target / 'release/native-metal-vulkan-probe' + report = dict(translator_revision=revision, rustc=toolchain, + source_sha256=digest(ROOT / 'drivers/NativeMetal/tools/MetalVulkanProbe.rs'), + cargo_lock_sha256=digest(output / 'Cargo.lock'), binary_sha256=digest(binary), + policy='exact_answers+exact_sampling', gpu_execution=False, + hardware_capabilities_observed=False, binary=str(binary)) + (output / 'build-report.json').write_text(json.dumps(report, indent=2) + '\n') + print(json.dumps(report, indent=2)) + + +if __name__ == '__main__': + main() diff --git a/scripts/build-mockfs-fixture.py b/scripts/build-mockfs-fixture.py new file mode 100644 index 0000000..fd8574b --- /dev/null +++ b/scripts/build-mockfs-fixture.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +"""Compile XNU's minimal executable-root filesystem for offline tests.""" +import json +from pathlib import Path +import struct +import subprocess + +ROOT = Path(__file__).resolve().parents[1] +recipe = ROOT / 'build/xnu/ios-obj/DEVELOPMENT_ARM64_BCM2837/bsd/DEVELOPMENT/vfs_conf.o.json' +record = json.loads(recipe.read_text().strip().rstrip(',')) +out = ROOT / 'build/mockfs-fixture' +out.mkdir(parents=True, exist_ok=True) +objects = [] +for name in ('mockfs_fsnode', 'mockfs_vfsops', 'mockfs_vnops'): + source = ROOT / '.cache/xnu-work/bsd/miscfs/mockfs' / (name + '.c') + destination = out / (name + '.o') + arguments = record['arguments'] + command = [] + index = 0 + while index < len(arguments): + value = arguments[index] + if value == record['file']: + command.append(str(source)) + elif value == '-o': + command.extend(['-o', str(destination)]) + index += 1 + elif value in ('-MF', '-MJ'): + index += 1 + elif value not in ('-MD', '-MP') and not value.startswith(('-DMOCKFS=', '-UMOCKFS')): + command.append(value) + index += 1 + command += ['-DMOCKFS=1'] + result = subprocess.run(command, cwd=record['directory'], capture_output=True, text=True) + (out / (name + '.log')).write_text(result.stdout + result.stderr) + if result.returncode: + raise SystemExit(result.stderr[-4500:]) + data = destination.read_bytes() + if struct.unpack_from('<4I', data) != (0xfeedfacf, 0x100000c, 0, 1): + raise SystemExit('Mockfs requires ordinary ARM64 output') + offset = 32 + deployment = None + for _ in range(struct.unpack_from('> 27 & 1 and word >> 28 == 0 and word >> 25 & 3 == 3: + relocations[section['va'] + address] = object_symbols[word & 0xffffff][0] +overrides = {0, 1, 7, 86, 111, 117, 168, 169, 170, 171, 172} +for row in audit['methods']: + names = row['reference_names'] + if row['slot'] in overrides: + names = [n.replace('16IOAESAccelerator', '9NativeAES') for n in names] + if relocations.get(own_vtable + 16 + row['slot'] * 8) not in names: + raise SystemExit(f'Compiled native subclass dispatch mismatch at slot {row["slot"]}') +if own_vtable + 16 + len(audit['methods']) * 8 in relocations: + raise SystemExit('Unexpected additional native subclass method') +kernel = (ROOT / 'build/native-prelink/native-defaults-candidate.macho').read_bytes() +if hashlib.sha256(kernel).hexdigest() != audit['candidate_sha256'] or not audit['live_native_4k']: + raise SystemExit('First qualify the original AES interface and live metadata') +image = np.macho(kernel) +plan = json.loads((ROOT / 'build/native-prelink/native-defaults-candidate.json').read_text()) +text = next(e for e in plan['extents'] if e['module'] == 'com.apple.iokit.IOCryptoAcceleratorFamily' and e['segment'] == '__TEXT_EXEC') + + +def read(va, length): + at = np.backed_offset(image['segments'], va, length, len(kernel)) + return kernel[at:at + length] + + +def branch(va): + word = struct.unpack('> 5 & 0x7ffff) << 2) | (a[3] >> 29 & 3) + if immediate & 0x100000: + immediate -= 0x200000 + address = ((va + 12) & ~4095) + immediate * 4096 + (a[4] >> 10 & 4095) + if address == vptr: + constructors.append(va) +if len(constructors) != 2: + raise SystemExit('Cannot uniquely qualify the two original protected constructors') +# Identical x0/x1 forwarding, same BL destination and same vptr store. +for left, right in zip(struct.unpack('<8I', read(constructors[0], 32)), + struct.unpack('<8I', read(constructors[1], 32))): + if left != right and left & 0xfc000000 != 0x94000000: + raise SystemExit('Original constructor forms differ beyond their relative call') +calls = [] +for va in constructors: + word = struct.unpack(' +#include +class IOHIDDevice : public IOService { + OSDeclareAbstractStructors(IOHIDDevice) + unsigned char originalPrivate[0xd0-sizeof(IOService)]; +public: + using IOService::setProperty; + IOReturn Dispatch(IORPC) override; + void free() override; + bool init(OSDictionary *properties=nullptr) override; + bool setProperty(const OSSymbol *,OSObject *) override; + bool willTerminate(IOService *,IOOptionBits) override; + IOReturn configureReport(IOReportChannelList *,IOReportConfigureAction,void *,void *) override; + IOReturn updateReport(IOReportChannelList *,IOReportUpdateAction,void *,void *) override; + bool start(IOService *) override; + void stop(IOService *) override; + bool handleOpen(IOService *,IOOptionBits,void *) override; + void handleClose(IOService *,IOOptionBits) override; + bool handleIsOpen(const IOService *) const override; + bool matchPropertyTable(OSDictionary *,SInt32 *) override; + IOReturn message(UInt32,IOService *,void *) override; + IOReturn newUserClient(task_t,void *,UInt32,OSDictionary *,IOUserClient **) override; +''' +for i in range(168,207): + if i==171:header+=' virtual OSString *newTransportString() const;\n' + elif i==173:header+=' virtual OSString *newProductString() const;\n' + elif i==180:header+=' virtual IOReturn newReportDescriptor(IOMemoryDescriptor **) const=0;\n' + elif i==181:header+=' virtual IOReturn handleReport(IOMemoryDescriptor *,unsigned,IOOptionBits);\n' + else:header+=f' virtual __attribute__((deprecated("Unqualified prototype: never call from source"))) void opaque{i}();\n' +header+='};\nstatic_assert(sizeof(IOHIDDevice)==0xd0,"Original device instance size");\n' +(OUT/'OriginalHidDeviceABI.h').write_text(header) +print(run(['python3',ROOT/'scripts/build-arm-driver.py','NativeHidInput'],text=True),end='') +obj=OUT/'NativeHidInput.o';raw=obj.read_bytes();om=macho(raw) +st=next(p for c,p,_ in om['commands'] if c==2);sy,count,strings,_=struct.unpack_from('<4I',raw,st+8);symbols=[] +for i in range(count): + ix,kind,section,_,va=struct.unpack_from('>27&1 and bits>>28==0 and bits>>25&3==3:relocations[sec['va']+offset]=symbols[bits&0xffffff][0] +compiled=[relocations.get(vt+16+i*8) for i in range(207)] +require(all(compiled) and vt+16+207*8 not in relocations,'Compiled exact 207-slot vtable') +bindings={'__ZN11IOHIDDeviceC2EPK11OSMetaClass':0xfffffff004ad94f8,'__ZN11IOHIDDeviceD2Ev':0xfffffff004ad9518,'__ZN11IOHIDDevice10gMetaClassE':0xfffffff006ae6fa0,'__ZTV11IOHIDDevice':vptr-16} +owned=[] +for i,name in enumerate(compiled): + if name.startswith(('__ZN14NativeHidInput','__ZNK14NativeHidInput')):owned.append(i) + elif '11IOHIDDevice' in name:bindings[name]=methods[i] + else:require(i<168 and names.get(name)==methods[i],'Actual inherited native method '+str(i)) +require(owned==[0,1,7,14,86,87,111,171,173,180],'Only qualified HID methods overridden') +# This kernel compiler resolves qualified super calls through the original vtable. +undefined={n for n,_,k in symbols if k==1 and '11IOHIDDevice' in n} +require(undefined==set(bindings),'All and only actual original HID imports bound') +asm=OUT/'original-bindings.S';asm.write_text(''.join(f'.globl {n}\n.set {n}, {a:#x}\n' for n,a in sorted(bindings.items()))) +sdk=run(['xcrun','--sdk','iphoneos','--show-sdk-path'],text=True).strip() +run(['xcrun','clang','-target','arm64-apple-ios17.0','-isysroot',sdk,'-c',asm,'-o',OUT/'hid-original-bindings.o']) +llvm=Path(run(['brew','--prefix','llvm'],text=True).strip())/'bin';late=OUT/'NativeHidInput.late.o' +run([llvm/'llvm-objcopy','--remove-section=__DATA,__mod_init_func','--globalize-symbol=__GLOBAL__sub_I_NativeHidInput.cpp',obj,late]) +def code(data,image): + s=next(s for g in image['segments'] for s in g['sections'] if s['name']=='__text');return data[s['fileoff']:s['fileoff']+s['size']] +require(code(raw,om)==code(late.read_bytes(),macho(late.read_bytes())),'Owned code unchanged by deferred metaclass registration') +(OUT/'abi-audit.json').write_text(json.dumps({'reference_candidate_sha256':plan['candidate_sha256'],'original_text_sha256':plan['original_executable_sha256']['com.apple.iokit.IOHIDFamily'],'original_slots':207,'original_instance_size':208,'owned_slots':owned,'bindings':bindings,'runtime_qualified':False,'source_sha256':hashlib.sha256((ROOT/'drivers/NativeHidInput/NativeHidInput.cpp').read_bytes()).hexdigest()},indent=2)+'\n') +print('PASS: original HID instruction identity, base imports, call boundaries and exact owned vtable') diff --git a/scripts/build-native-metal-bridge.py b/scripts/build-native-metal-bridge.py new file mode 100644 index 0000000..8dda667 --- /dev/null +++ b/scripts/build-native-metal-bridge.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +"""Build the native Metal bridge (drivers/NativeMetal/bridge/NativeMetalBridge.rs): the pinned metal2vulkan runtime behind a C ABI, as a static library. + +--target ios (default) cross-builds for arm64-apple-ios17.0, --target host for the macOS host simulation. The runtime's own example library +(runtime_examples.ll) and its function-constant fixture are assembled to AIR with llvm-as and embedded for the self-tests. Only generated build files hold local paths; nothing here +is committed. Output: build/native-metal-bridge//libnative_metal_bridge.a""" +import argparse +import json +import os +from pathlib import Path +import shutil +import subprocess + +ROOT = Path(__file__).resolve().parents[1] +PIN = json.loads((ROOT / 'scripts/data/gpu-metal-vulkan-inputs.json').read_text())['translator']['commit'] +SOURCE = ROOT / '.cache/gpu-reference/metal2vulkan' +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument('--target', choices=['ios', 'host'], default='ios') +args = parser.parse_args() +if shutil.disk_usage(ROOT).free < 7 * 1024**3: + raise SystemExit('Preserve the 5 GiB floor and 2 GiB build allowance') +if subprocess.check_output(['git', '-C', str(SOURCE), 'rev-parse', 'HEAD'], text=True).strip() != PIN: + raise SystemExit('Translator revision differs from the qualified source pin') +if subprocess.check_output(['git', '-C', str(SOURCE), 'diff', 'HEAD', '--'], text=True): + raise SystemExit('Translator source must be unmodified') +out = ROOT / 'build/native-metal-bridge' +out.mkdir(parents=True, exist_ok=True) +llvm_as = next((p for p in ['/opt/homebrew/opt/llvm/bin/llvm-as', shutil.which('llvm-as') or ''] if p and Path(p).exists()), None) +if not llvm_as: + raise SystemExit('llvm-as (LLVM 21 or newer) is required to assemble the fixture') +air = out / 'runtime_examples.air' +subprocess.run([llvm_as, '-o', air, SOURCE / 'crates/metal2vulkan-validation/fixtures/public/examples/runtime_examples.ll'], check=True) +fc_air = out / 'function_constant.air' +subprocess.run([llvm_as, '-o', fc_air, SOURCE / 'crates/metal2vulkan-validation/fixtures/public/kernel_function_constant_gate_fold.ll'], check=True) +manifest = out / 'Cargo.toml' +manifest.write_text('\n'.join([ + '[package]', 'name = "native-metal-bridge"', 'version = "0.1.0"', 'edition = "2024"', 'rust-version = "1.98"', '[workspace]', + '[lib]', 'crate-type = ["staticlib"]', 'path = ' + json.dumps(str(ROOT / 'drivers/NativeMetal/bridge/NativeMetalBridge.rs')), + '[dependencies]', + 'metal2vulkan = { path = ' + json.dumps(str(SOURCE / 'crates/metal2vulkan')) + ', features = ["ash"] }', + 'metal2vulkan-runtime = { path = ' + json.dumps(str(SOURCE / 'crates/metal2vulkan-runtime')) + ' }', + 'libc = "0.2"', + '[profile.release]', 'panic = "unwind"', 'debug = 1', '', +])) +triple = 'aarch64-apple-ios' if args.target == 'ios' else 'aarch64-apple-darwin' +env = dict(os.environ, NMB_FIXTURE_AIR=str(air), NMB_FC_AIR=str(fc_air), IPHONEOS_DEPLOYMENT_TARGET='17.0', MACOSX_DEPLOYMENT_TARGET='14.0') +target_dir = ROOT / 'build/m2v-target' +log = out / ('build-%s.log' % args.target) +with log.open('w') as stream: + result = subprocess.run(['cargo', '+1.98.0', 'build', '--release', '--manifest-path', manifest, '--target', triple, + '--target-dir', target_dir], env=env, stdout=stream, stderr=subprocess.STDOUT, timeout=1800) +if result.returncode: + raise SystemExit(log.read_text()[-6000:]) +library = target_dir / triple / 'release/libnative_metal_bridge.a' +final = out / args.target / 'libnative_metal_bridge.a' +final.parent.mkdir(exist_ok=True) +shutil.copyfile(library, final) +print('PASS: native Metal bridge static library for %s (%d bytes); translator %s' % (args.target, final.stat().st_size, PIN[:12])) diff --git a/scripts/build-native-user-fixture.py b/scripts/build-native-user-fixture.py new file mode 100644 index 0000000..73d9899 --- /dev/null +++ b/scripts/build-native-user-fixture.py @@ -0,0 +1,249 @@ +#!/usr/bin/env python3 +"""Build an ordinary ARM64 static Darwin process for native-4K EL0/SVC tests.""" +import json +import hashlib +from pathlib import Path +import re +import subprocess +import struct +import uuid +import plistlib +import zipfile +import argparse + +ROOT = Path(__file__).resolve().parents[1] +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument('--output', type=Path, default=ROOT / 'build/native-user-fixture') +parser.add_argument('--signing-page-size', type=int, choices=[4096, 16384], default=4096, + help='CodeDirectory integrity block size; hardware/VM page size stays native 4K') +parser.add_argument('--probe-file', type=Path, + help='Unchanged target dyld_shared_cache_arm64.02 for real file-signature/mmap tests') +parser.add_argument('--probe-cache-getuid', type=Path, help='Exact original cache file identified by audit-target-cache-leaf.py') +parser.add_argument('--apfs-device', help='Actual observed /dev/disk... volume for an offline read-only data mount') +parser.add_argument('--apfs-mount-mode', type=int, choices=(0, 1), default=1, + help='Original API mode: 1 ordinary data mount; 0 negative user-root request control') +parser.add_argument('--apfs-launchd-devices', nargs=2, metavar=('OS_DEVICE', 'CACHE_DEVICE'), + help='Actual observed volume devices for a read-only APFS/chroot/original-dyld component test') +parser.add_argument('--stop-before-apfs-exec', action='store_true', help='Qualify data mounts and chroot before original exec') +parser.add_argument('--fp-context',action='store_true',help='Bounded all-32-SIMD-register syscall and blocking thread-switch preservation instrument') +args = parser.parse_args() +if args.fp_context and args.signing_page_size!=4096:parser.error('FP context instrument requires its own native-4K CodeDirectory') +if args.fp_context and (args.probe_file or args.probe_cache_getuid or args.apfs_device or args.apfs_launchd_devices):parser.error('FP context instrument is separate from file/APFS probes') +if args.probe_cache_getuid and (not args.apfs_device or args.probe_file or args.apfs_mount_mode != 1): + parser.error('The original cache-code probe requires one real ordinary APFS data mount') +if args.apfs_launchd_devices and (args.probe_file or args.probe_cache_getuid or args.apfs_device or + any(not re.fullmatch(r'/dev/disk[0-9]+(?:s[0-9]+)+', d) for d in args.apfs_launchd_devices)): + parser.error('Use two observed APFS device names without the separate file probe') +if args.stop_before_apfs_exec and not args.apfs_launchd_devices: + parser.error('--stop-before-apfs-exec requires the two-volume component fixture') +if args.apfs_device and (not (args.probe_file or args.probe_cache_getuid) or not re.fullmatch(r'/dev/disk[0-9]+(?:s[0-9]+)+', args.apfs_device)): + parser.error('The APFS data test needs the original cache probe and an actual volume device name') +out = args.output.resolve() +out.mkdir(parents=True, exist_ok=True) +sdk = subprocess.check_output(['xcrun', '--sdk', 'iphoneos', '--show-sdk-path'], text=True).strip() +probe = None +cache_leaf = None +probe_flags = ['-DNATIVE_FP_CONTEXT_PROBE=1'] if args.fp_context else [] +if args.apfs_launchd_devices: + probe_flags += ['-DNATIVE_APFS_LAUNCHD_PROBE=1', + '-DNATIVE_APFS_OS_DEVICE="' + args.apfs_launchd_devices[0] + '"', + '-DNATIVE_APFS_CACHE_DEVICE="' + args.apfs_launchd_devices[1] + '"'] + if args.stop_before_apfs_exec: + probe_flags += ['-DNATIVE_APFS_STOP_BEFORE_EXEC=1'] +if args.apfs_device: + probe_flags += ['-DNATIVE_APFS_FILE_PROBE=1', '-DNATIVE_APFS_DEVICE="' + args.apfs_device + '"', + f'-DNATIVE_APFS_MOUNT_MODE={args.apfs_mount_mode}'] +if args.probe_cache_getuid: + cache_leaf = json.loads((ROOT / 'build/cache-leaf-host/getuid.json').read_text()) + digest = hashlib.sha256() + with args.probe_cache_getuid.open('rb') as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b''): + digest.update(chunk) + if args.probe_cache_getuid.name != cache_leaf['file'] or digest.hexdigest() != cache_leaf['file_sha256']: + raise SystemExit('Require the exact unchanged original cache file from the export audit') + probe_flags += ['-DNATIVE_CACHE_LEAF_PROBE=1', + '-DNATIVE_CACHE_LEAF_PATH="/dev/apfs-offline/System/Library/Caches/com.apple.dyld/' + cache_leaf['file'] + '"', + f'-DNATIVE_CACHE_LEAF_PAGE_OFFSET={cache_leaf["native_page_offset"]}', + f'-DNATIVE_CACHE_LEAF_ENTRY_OFFSET={cache_leaf["within_page_offset"]}', + f'-DNATIVE_CACHE_LEAF_SIGNATURE_OFFSET={cache_leaf["signature_offset"]}', + f'-DNATIVE_CACHE_LEAF_SIGNATURE_SIZE={cache_leaf["signature_size"]}'] +if args.probe_file: + probe = args.probe_file.read_bytes() + if hashlib.sha256(probe).hexdigest() != '3d923c404fe519184a99042b60a4cb3d37247dcb8b070242ed3e670d4d523d04': + raise SystemExit('Expected the exact supplied 21H461 arm64.02 cache file') + sigoff, sigsize = struct.unpack_from('I', signature, 8)[0] + directories = [] + for i in range(count): + _, at = struct.unpack_from('>II', signature, 12 + i * 8) + magic, length = struct.unpack_from('>II', signature, at) + if magic == 0xfade0c02: + cd = signature[at:at + length] + directories.append(cd) + if len(directories) != 1 or directories[0][39] != 14: + raise SystemExit('Expected the target original 16K CodeDirectory') + cd = directories[0] + limit = struct.unpack_from('>I', cd, 32)[0] + hashoff = struct.unpack_from('>I', cd, 16)[0] + if limit != 32768 or any(hashlib.sha256(probe[o:o + 16384]).digest() != + cd[hashoff + (o >> 14) * 32:hashoff + ((o >> 14) + 1) * 32] + for o in range(0, limit, 16384)): + raise SystemExit('Original complete signing block verification failed') + probe_flags += ['-DNATIVE_FILE_PROBE=1', f'-DNATIVE_PROBE_LIMIT={limit}', + f'-DNATIVE_PROBE_SIGNATURE_OFFSET={sigoff}', f'-DNATIVE_PROBE_SIGNATURE_SIZE={sigsize}'] +subprocess.run(['xcrun', 'clang', '-target', 'arm64-apple-ios17.0', '-isysroot', sdk, + *probe_flags, '-c', ROOT / 'scripts/qemu-entry/native-user.S', '-o', out / 'user.o'], check=True) +subprocess.run(['xcrun', 'ld', '-arch', 'arm64', '-static', '-e', '_start', + '-platform_version', 'ios', '17.0', '17.0', out / 'user.o', + '-o', out / 'launchd-static.macho'], check=True) +path = out / 'launchd.macho' +subprocess.run(['xcrun', 'vtool', '-set-build-version', 'ios', '17.0', '17.0', + '-replace', '-output', path, out / 'launchd-static.macho'], check=True) +subprocess.run(['codesign', '--force', '--sign', '-', '--pagesize', str(args.signing_page_size), + '--identifier', 'org.blu-sharky.native4k-fixture', path], check=True) +subprocess.run(['codesign', '--verify', '--strict', path], check=True) +details = subprocess.run(['codesign', '--display', '--verbose=4', path], + capture_output=True, text=True, check=True).stderr +match = re.search(r'^CDHash=([0-9a-f]{40})$', details, re.M) +if not match: + raise SystemExit('Cannot identify the signed own fixture CodeDirectory') +page = re.search(r'^Page size=(\d+)$', details, re.M) +if not page or int(page[1]) != args.signing_page_size: + raise SystemExit('CodeDirectory does not identify the requested integrity block size') +cdhash = bytes.fromhex(match[1]) +identifier = uuid.uuid5(uuid.NAMESPACE_URL, 'org.blu-sharky.native4k-fixture:' + cdhash.hex()) +module = struct.pack(' len(data): + raise SystemExit('Trust-cache DER exceeds the original input') + return tag, data[at:at + count], at + count + + with zipfile.ZipFile(ROOT / 'refer/iPad_Pro_HFR_17.7.11_21H461_Restore.ipsw') as archive: + manifest = plistlib.loads(archive.read('BuildManifest.plist')) + identities = [i for i in manifest['BuildIdentities'] if i['Info']['DeviceClass'] == 'j120ap'] + if manifest['ProductBuildVersion'] != '21H461' or len(identities) != 2: + raise SystemExit('Expected both original iPad7,1 restore identities') + entry = identities[0]['Manifest']['Cryptex1,SystemTrustCache'] + original_im4p = archive.read(entry['Info']['Path']) + if any(i['Manifest']['Cryptex1,SystemTrustCache']['Digest'] != hashlib.sha384(original_im4p).digest() + for i in identities): + raise SystemExit('Original System trust-cache manifest digest mismatch') + tag, sequence, end = der_value(original_im4p, 0) + fields, cursor = [], 0 + while cursor < len(sequence): + kind, value, cursor = der_value(sequence, cursor) + fields.append((kind, value)) + if tag != 0x30 or end != len(original_im4p) or [t for t, _ in fields] != [0x16, 0x16, 0x16, 4] or fields[0][1] != b'IM4P': + raise SystemExit('Expected the genuine unpersonalized trust-cache IM4P') + system_module = fields[3][1] + static_module = (ROOT / '.cache/firmware/target-volumes/static-trustcache.bin').read_bytes() + if hashlib.sha256(static_module).hexdigest() != 'da88460e6e513438d5307e5c526f76497a74754476a24da29f239f0c90120fc2': + raise SystemExit('Original static trust-cache identity mismatch') + cache_audit = json.loads((ROOT / 'build/target-userspace/dyld-cache-audit.json').read_text()) + trusted = {system_module[i:i + 20].hex() for i in range(24, len(system_module), 24)} + cache_hashes = {c['cdhash'] for f in cache_audit['cache_files'] for c in f['code_directories']} + if len(cache_hashes) != 53 or cache_hashes - trusted: + raise SystemExit('The unchanged original System module must cover all 53 audited cache files') + modules = [module, static_module, system_module] + for data in modules: + if struct.unpack_from('> 27 & 1 and word >> 28 == 0 and word >> 25 & 3 == 3: + relocations[section['va'] + address] = symbols[word & 0xffffff][0] +compiled = [relocations.get(vtable + 16 + i * 8) for i in range(190)] +if not all(compiled) or vtable + 16 + 190 * 8 in relocations or '___cxa_pure_virtual' in compiled: + raise SystemExit('Concrete block class has missing/additional/pure virtual methods') +all_names = list(dict.fromkeys(compiled + [r['reference_name'] for r in audit['methods']])) +demangled = subprocess.check_output(['xcrun', 'c++filt'], input='\n'.join(n[1:] for n in all_names) + '\n', text=True).splitlines() +readable = dict(zip(all_names, demangled)) +overrides = 0 +for row, actual in zip(audit['methods'], compiled): + expected = row['reference_name'] + own = actual.startswith(('__ZN' + mangled, '__ZNK' + mangled)) + if expected == '___cxa_pure_virtual': + if not own: + raise SystemExit('A required transport operation is not implemented') + elif actual != expected: + if not own or (row['slot'] not in (0, 1) and readable[actual].split('::', 1)[1] != readable[expected].split('::', 1)[1]): + raise SystemExit('Concrete subclass dispatch mismatch at slot ' + str(row['slot'])) + overrides += own +kernel_path = ROOT / 'build/native-prelink/native-apfs-load-positive.macho' +# The binding values come from the recorded audit; the candidate only proves the audit's identity. Build products are pruned, so a +# missing candidate is accepted (the audit JSON is the pinned record) while a present one must still match. +if kernel_path.exists() and hashlib.sha256(kernel_path.read_bytes()).hexdigest() != audit['candidate_sha256']: + raise SystemExit('Original binding input identity changed') +possible = {r['reference_name']: int(r['candidate_target'], 16) for r in audit['methods'] if not r['kernel_names']} +possible.update({'__ZN20IOBlockStorageDeviceC2EPK11OSMetaClass': int(audit['original_protected_constructor'], 16), + '__ZN20IOBlockStorageDeviceD2Ev': int(audit['original_base_destructor'], 16), + '__ZN20IOBlockStorageDevice10gMetaClassE': int(audit['original_metaclass'], 16), + '__ZTV20IOBlockStorageDevice': int(audit['vtable'], 16)}) +undefined = [name for name, value, kind in symbols if kind == 1 and ('IOBlockStorageDevice' in name)] +if any(name not in possible for name in undefined): + raise SystemExit('Unqualified original storage import') +# Abstract D1/D0 are BRK traps. Only actual base D2 is appropriate for this subclass. +if any(re.search(r'IOBlockStorageDeviceD[01]Ev$', n) for n in undefined): + raise SystemExit('Concrete driver must not invoke abstract complete/deleting destruction') +bindings = {name: possible[name] for name in undefined} +assembly = out / 'original-bindings.S' +assembly.write_text(''.join(f'.globl {name}\n.set {name}, {value:#x}\n' for name, value in sorted(bindings.items()))) +subprocess.run(['xcrun', 'clang', '-target', 'arm64-apple-ios17.0', '-Werror', '-Wno-incompatible-sysroot', + '-c', assembly, '-o', out / 'original-bindings.o'], check=True) +archive = out / ('lib' + component + '.a') +archive.unlink(missing_ok=True) +subprocess.run(['xcrun', 'ar', 'rcs', archive, obj_path, out / 'original-bindings.o'], check=True) +(out / 'bindings.json').write_text(json.dumps(dict(original_candidate_sha256=audit['candidate_sha256'], + compiled_object_sha256=hashlib.sha256(obj).hexdigest(), virtual_methods_checked=190, + concrete_overrides=overrides, unresolved_original_imports=0, + bindings={name: hex(value) for name, value in bindings.items()}, original_code_modified=False, + actual_io_qualified=False), indent=2) + '\n') +print(f'PASS: 190 concrete slots, {overrides} overrides and {len(bindings)} genuine original storage imports') +print('Actual device I/O and filesystem integration remain unqualified by this compile/ABI audit.') diff --git a/scripts/build-original-system-trust.py b/scripts/build-original-system-trust.py new file mode 100644 index 0000000..3345236 --- /dev/null +++ b/scripts/build-original-system-trust.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +"""Bundle only the two unchanged original target trust-cache modules.""" +import hashlib +import json +from pathlib import Path +import plistlib +import struct +import zipfile + +ROOT = Path(__file__).resolve().parents[1] + + +def der(data, at): + if at + 2 > len(data): + raise ValueError('Truncated DER') + tag, size = data[at:at + 2] + at += 2 + if size & 128: + count = size & 127 + if not 0 < count <= 4 or at + count > len(data): + raise ValueError('Invalid DER length') + size = int.from_bytes(data[at:at + count], 'big') + at += count + if at + size > len(data): + raise ValueError('DER exceeds original input') + return tag, data[at:at + size], at + size + + +with zipfile.ZipFile(ROOT / 'refer/iPad_Pro_HFR_17.7.11_21H461_Restore.ipsw') as archive: + manifest = plistlib.loads(archive.read('BuildManifest.plist')) + identities = [i for i in manifest['BuildIdentities'] if i['Info']['DeviceClass'] == 'j120ap'] + if manifest['ProductBuildVersion'] != '21H461' or len(identities) != 2: + raise SystemExit('Require supplied ordinary-ARM64 iPad7,1 target') + entry = identities[0]['Manifest']['Cryptex1,SystemTrustCache'] + original = archive.read(entry['Info']['Path']) + if any(i['Manifest']['Cryptex1,SystemTrustCache']['Digest'] != hashlib.sha384(original).digest() for i in identities): + raise SystemExit('Both original manifest digests must match') +tag, sequence, end = der(original, 0) +fields, cursor = [], 0 +while cursor < len(sequence): + kind, data, cursor = der(sequence, cursor) + fields.append((kind, data)) +if tag != 0x30 or end != len(original) or [kind for kind, _ in fields] != [0x16, 0x16, 0x16, 4] or fields[0][1] != b'IM4P': + raise SystemExit('Require the original System trust-cache IM4P') +system = fields[3][1] +static = (ROOT / '.cache/firmware/target-volumes/static-trustcache.bin').read_bytes() +expected = ('da88460e6e513438d5307e5c526f76497a74754476a24da29f239f0c90120fc2', + 'd121ba89167e03aafd00fa687c30cd8ebc4ccafd10c1408adb2ba3de12403651') +for module, digest in zip((static, system), expected): + if (hashlib.sha256(module).hexdigest() != digest or struct.unpack_from('&2 + exit 1 +fi +if [ ! -d .cache/xnu-work/.git ]; then + git clone --shared refer/xnu .cache/xnu-work +fi +if [ ! -x .cache/fakeroot/usr/local/libexec/availability.pl ]; then + if [ ! -d refer/AvailabilityVersions/.git ]; then + git clone --depth 1 --branch AvailabilityVersions-141.1 https://github.com/apple-oss-distributions/AvailabilityVersions.git refer/AvailabilityVersions + fi + mkdir -p .cache/fakeroot + make -C refer/AvailabilityVersions install -j"${JOBS:-4}" \ + OBJROOT="$ROOT/build/availability" SYMROOT="$ROOT/build/availability" \ + DSTROOT="$ROOT/.cache/fakeroot" CMAKE="$(command -v cmake)" NINJA="$(command -v ninja)" +fi +# The optional observation hooks overlap existing root/exec patch context. +# Remove them first so earlier independent patches retain their idempotent checks. +# The same holds for the boot-time GPU addon mount (zzy), which edits the owner Data mount function inside the systemos patch. +# Reverse newest first; the loop below re-applies them in name order. +for LATE_PATCH in "$ROOT/patches/xnu/zzz-boot-progress.patch" "$ROOT/patches/xnu/zzy-native-gpu-addon-mount.patch"; do + if git -C .cache/xnu-work apply --reverse --check "$LATE_PATCH" >/dev/null 2>&1; then + git -C .cache/xnu-work apply --reverse "$LATE_PATCH" + fi +done +for PATCH in "$ROOT"/patches/xnu/*.patch "$ROOT"/kits/NativePrelink/patches/*.patch; do + if ! git -C .cache/xnu-work apply --reverse --check "$PATCH" >/dev/null 2>&1; then + git -C .cache/xnu-work apply --check "$PATCH" + git -C .cache/xnu-work apply "$PATCH" + fi +done +export XNU_AVAILABILITY_PL=$ROOT/.cache/fakeroot/usr/local/libexec/availability.pl +python3 scripts/prepare-xnu-source.py +BOOT_PROGRESS=${XNU_BOOT_PROGRESS:-0} +if [ "$BOOT_PROGRESS" != 0 ] && [ "$BOOT_PROGRESS" != 1 ]; then + echo 'XNU_BOOT_PROGRESS must be 0 or 1' >&2 + exit 1 +fi +if [ "$BOOT_PROGRESS" = 1 ]; then + if [ "${XNU_NATIVE_UFS:-0}" != 1 ] || [ "${XNU_OWNER_ROOT_AUDIT:-0}" != 1 ] || \ + [ "${XNU_NATIVE_VIRTIO_BLOCK:-0}" != 1 ] || [ "${XNU_SOURCE_CORECRYPTO14:-0}" != 1 ]; then + echo 'Boot progress requires the audited UFS/owner-root, Virtio and source-corecrypto build' >&2 + exit 1 + fi + python3 scripts/build-boot-progress.py +fi +python3 - "$BOOT_PROGRESS" <<'PY' +from pathlib import Path +import hashlib, sys +mode = sys.argv[1] +out = Path('build/xnu/ios-obj/DEVELOPMENT_ARM64_BCM2837') +stamp = out / '.boot-progress-mode' +identity = mode + ':' + (hashlib.sha256(Path('build/boot-progress/libBootProgress.a').read_bytes()).hexdigest() if mode == '1' else 'disabled') +if not stamp.exists() or stamp.read_text() != identity: + for directory, names in [('osfmk', ['arm_init','startup','video_console','serial_console','sleh']), + ('bsd', ['bsd_init','kern_exec','kern_exit']), ('pexpert', ['pe_kprintf'])]: + for name in names: + (out / directory / 'DEVELOPMENT' / (name + '.o')).unlink(missing_ok=True) + (out / 'mach.development.bcm2837.unstripped.noctf').unlink(missing_ok=True) + out.mkdir(parents=True, exist_ok=True) + stamp.write_text(identity) +PY +if [ ! -d refer/libplatform/.git ]; then + git clone --depth 1 --branch libplatform-316.100.10 https://github.com/apple-oss-distributions/libplatform.git refer/libplatform +fi +export KERNEL_SDKROOT=$ROOT/.cache/fakeroot +mkdir -p "$KERNEL_SDKROOT/usr/local/include/kernel" "$ROOT/build/xnu/obj/EXPORT_HDRS/libsa" +ditto refer/libplatform/include "$KERNEL_SDKROOT/usr/local/include" +ditto refer/libplatform/private "$KERNEL_SDKROOT/usr/local/include" +ditto refer/libplatform/private/platform "$KERNEL_SDKROOT/System/Library/Frameworks/Kernel.framework/Versions/A/PrivateHeaders/platform" +if [ ! -d refer/libdispatch/.git ]; then + git clone --depth 1 --branch libdispatch-1477.100.9 https://github.com/apple-oss-distributions/libdispatch.git refer/libdispatch +fi +python3 - <<'PY' +from pathlib import Path +import subprocess +output = Path('.cache/fakeroot/usr/local/include/os') +output.mkdir(parents=True, exist_ok=True) +for header in Path('refer/libdispatch/os').glob('*.h'): + result = subprocess.run(['unifdef', '-DKERNEL=1', '-DOS_FIREHOSE_SPI=1', + '-DOS_VOUCHER_ACTIVITY_SPI_TYPES=1', + '-UOS_VOUCHER_ACTIVITY_SPI', str(header)], capture_output=True) + if result.returncode not in (0, 1): + raise SystemExit(result.stderr.decode()) + target = output / header.name + if not target.exists() or target.read_bytes() != result.stdout: + target.write_bytes(result.stdout) +PY +if [ ! -f "$KERNEL_SDKROOT/.xnu-10063.141.1-headers" ]; then + make -C .cache/xnu-work installhdrs SDKROOT=macosx ARCH_CONFIGS=ARM64 \ + MACHINE_CONFIGS=BCM2837 KERNEL_CONFIGS=DEVELOPMENT \ + RC_DARWIN_KERNEL_VERSION=23.6.0 RC_ProjectSourceVersion=10063.141.1 \ + MAKEJOBS=-j"${JOBS:-4}" OBJROOT="$ROOT/build/xnu/obj" \ + SYMROOT="$ROOT/build/xnu/sym" DSTROOT="$KERNEL_SDKROOT" + touch "$KERNEL_SDKROOT/.xnu-10063.141.1-headers" +fi +if [ ! -f "$KERNEL_SDKROOT/usr/local/lib/kernel/libfirehose_kernel.a" ]; then + bash scripts/build-firehose.sh +fi +# The baseline builds the kernel; `all` also checks the Apple kext export ABI. +EXTRA_MAKE_ARGS=() +ARM_DT_PLATFORM=${XNU_ARM_DT_PLATFORM:-0} +if [ "$ARM_DT_PLATFORM" != 0 ] && [ "$ARM_DT_PLATFORM" != 1 ]; then + echo 'XNU_ARM_DT_PLATFORM must be 0 or 1' >&2 + exit 1 +fi +if [ "${XNU_ARMCPU:-0}" = 1 ] && { [ "${XNU_SOURCE_CORECRYPTO14:-0}" != 1 ] || \ + { [ "${XNU_ARMVIRT_PLATFORM:-0}" != 1 ] && [ "$ARM_DT_PLATFORM" != 1 ]; }; }; then + echo 'ArmCpu bring-up requires the real source provider and a real platform driver' >&2 + exit 1 +fi +if [ "${XNU_SOURCE_PTHREAD:-0}" = 1 ] && [ "${XNU_SOURCE_CORECRYPTO14:-0}" != 1 ]; then + echo 'The current pthread source-link recipe requires XNU_SOURCE_CORECRYPTO14=1' >&2 + exit 1 +fi +CRYPTO_BOOTSTRAP_MODE=${XNU_CRYPTO_BOOTSTRAP:-source} +if [ "$CRYPTO_BOOTSTRAP_MODE" != source ] && [ "$CRYPTO_BOOTSTRAP_MODE" != prelinked ]; then + echo 'XNU_CRYPTO_BOOTSTRAP must be source or prelinked' >&2 + exit 1 +fi +CRYPTO_OBJECT_DIR=$ROOT/build/xnu/ios-obj/DEVELOPMENT_ARM64_BCM2837/osfmk/DEVELOPMENT +if [ -d "$CRYPTO_OBJECT_DIR" ]; then + if [ "$(cat "$CRYPTO_OBJECT_DIR/.crypto-bootstrap-mode" 2>/dev/null || echo source)" != "$CRYPTO_BOOTSTRAP_MODE" ]; then + rm -f "$CRYPTO_OBJECT_DIR/prng_random.o" + fi + printf '%s\n' "$CRYPTO_BOOTSTRAP_MODE" > "$CRYPTO_OBJECT_DIR/.crypto-bootstrap-mode" +fi +if [ "$CRYPTO_BOOTSTRAP_MODE" = prelinked ]; then + # The original component must initialize through real OSKext startup. + # Only this source-provider entry is omitted; no POST or missing-provider + # failure is suppressed, and no replacement crypto callbacks are supplied. + EXTRA_MAKE_ARGS+=("prng_random.o_CFLAGS_ADD=-UXNU_SOURCE_CORECRYPTO14") +else + EXTRA_MAKE_ARGS+=("prng_random.o_CFLAGS_ADD=") +fi +if [ "${XNU_NATIVE_AES:-0}" = 1 ]; then + if [ "$CRYPTO_BOOTSTRAP_MODE" != prelinked ] || [ "${XNU_SOURCE_CORECRYPTO14:-0}" != 1 ]; then + echo 'NativeAES currently requires the qualified original AES/crypto prelink closure' >&2 + exit 1 + fi + PERSONALITY_CFLAGS=-DXNU_NATIVE_AES=1 +else + PERSONALITY_CFLAGS= +fi +if [ "${XNU_NATIVE_VIRTIO_BLOCK:-0}" = 1 ]; then + if [ "${XNU_NATIVE_AES:-0}" != 1 ]; then + echo 'NativeVirtioBlock currently requires the qualified NativeAES/original-storage prelink baseline' >&2 + exit 1 + fi + PERSONALITY_CFLAGS="$PERSONALITY_CFLAGS -DXNU_NATIVE_VIRTIO_BLOCK=1" +fi +NATIVE_RAM_BLOCK=${XNU_NATIVE_RAM_BLOCK:-0} +if [ "$NATIVE_RAM_BLOCK" != 0 ] && [ "$NATIVE_RAM_BLOCK" != 1 ]; then + echo 'XNU_NATIVE_RAM_BLOCK must be 0 or 1' >&2 + exit 1 +fi +if [ "$NATIVE_RAM_BLOCK" = 1 ]; then + if [ "${XNU_NATIVE_AES:-0}" != 1 ]; then + echo 'NativeRamBlock requires the qualified original storage/AES prelink closure' >&2 + exit 1 + fi + PERSONALITY_CFLAGS="$PERSONALITY_CFLAGS -DXNU_NATIVE_RAM_BLOCK=1" +fi +if [ "${XNU_NATIVE_UFS:-0}" = 1 ]; then + PERSONALITY_CFLAGS="$PERSONALITY_CFLAGS -DXNU_NATIVE_UFS=1" +fi +NATIVE_VIRTIO_CONSOLE=${XNU_NATIVE_VIRTIO_CONSOLE:-0} +if [ "$NATIVE_VIRTIO_CONSOLE" != 0 ] && [ "$NATIVE_VIRTIO_CONSOLE" != 1 ]; then + echo 'XNU_NATIVE_VIRTIO_CONSOLE must be 0 or 1' >&2 + exit 1 +fi +if [ "$NATIVE_VIRTIO_CONSOLE" = 1 ]; then + PERSONALITY_CFLAGS="$PERSONALITY_CFLAGS -DXNU_NATIVE_VIRTIO_CONSOLE=1" +fi +EXTRA_MAKE_ARGS+=("KernelConfigTables.cpo_CFLAGS_ADD=$PERSONALITY_CFLAGS") +TRUST_CACHE_MODE=${XNU_TRUST_CACHE_RUNTIME:-0} +if [ "$TRUST_CACHE_MODE" != 0 ] && [ "$TRUST_CACHE_MODE" != 1 ]; then + echo 'XNU_TRUST_CACHE_RUNTIME must be 0 or 1' >&2 + exit 1 +fi +if [ -d "$CRYPTO_OBJECT_DIR" ]; then + if [ "$(cat "$CRYPTO_OBJECT_DIR/.trust-cache-mode" 2>/dev/null || echo 0)" != "$TRUST_CACHE_MODE" ]; then + rm -f "$CRYPTO_OBJECT_DIR/startup.o" + fi + printf '%s\n' "$TRUST_CACHE_MODE" > "$CRYPTO_OBJECT_DIR/.trust-cache-mode" +fi +EXTRA_MAKE_ARGS+=("startup.o_CFLAGS_ADD=-DXNU_NATIVE_TRUST_CACHE=$TRUST_CACHE_MODE") +MOCKFS_MODE=${XNU_SOURCE_MOCKFS:-0} +if [ "$MOCKFS_MODE" != 0 ] && [ "$MOCKFS_MODE" != 1 ]; then + echo 'XNU_SOURCE_MOCKFS must be 0 or 1' >&2 + exit 1 +fi +if [ "$MOCKFS_MODE" = 1 ] && { [ "${XNU_ARMCPU:-0}" != 1 ] || [ "${XNU_SOURCE_PTHREAD:-0}" != 1 ]; }; then + echo 'The offline executable-root fixture requires the real CPU/pthread components' >&2 + exit 1 +fi +MOCKFS_OBJECT_DIR=$ROOT/build/xnu/ios-obj/DEVELOPMENT_ARM64_BCM2837/bsd/DEVELOPMENT +if [ -d "$MOCKFS_OBJECT_DIR" ]; then + if [ "$(cat "$MOCKFS_OBJECT_DIR/.source-mockfs-mode" 2>/dev/null || echo 0)" != "$MOCKFS_MODE" ]; then + rm -f "$MOCKFS_OBJECT_DIR/vfs_conf.o" + fi + printf '%s\n' "$MOCKFS_MODE" > "$MOCKFS_OBJECT_DIR/.source-mockfs-mode" +fi +EXTRA_MAKE_ARGS+=("vfs_conf.o_CFLAGS_ADD=-DMOCKFS=$MOCKFS_MODE") +CS_HASH_CACHE_MODE=${XNU_NATIVE_CS_HASH_CACHE:-0} +if [ "$CS_HASH_CACHE_MODE" != 0 ] && [ "$CS_HASH_CACHE_MODE" != 1 ]; then + echo 'XNU_NATIVE_CS_HASH_CACHE must be 0 or 1' >&2 + exit 1 +fi +if [ -d "$MOCKFS_OBJECT_DIR" ]; then + if [ "$(cat "$MOCKFS_OBJECT_DIR/.native-cs-hash-cache-mode" 2>/dev/null || echo 0)" != "$CS_HASH_CACHE_MODE" ]; then + rm -f "$MOCKFS_OBJECT_DIR/ubc_subr.o" + fi + printf '%s\n' "$CS_HASH_CACHE_MODE" > "$MOCKFS_OBJECT_DIR/.native-cs-hash-cache-mode" +fi +EXTRA_MAKE_ARGS+=("ubc_subr.o_CFLAGS_ADD=-DXNU_NATIVE_CS_HASH_CACHE=$CS_HASH_CACHE_MODE") +EXTRA_CFLAGS=${XNU_EXTRA_CFLAGS:--Wno-reserved-identifier -Wno-pre-c11-compat -Wno-missing-designated-field-initializers -Wno-suggest-override -Wno-unnecessary-virtual-specifier -Wno-suggest-destructor-override -Wno-ms-bitfield-padding} +OWNER_ROOT_PROBE=${XNU_VERIFIED_ROOT_PROBE:-0} +if [ "$OWNER_ROOT_PROBE" != 0 ] && [ "$OWNER_ROOT_PROBE" != 1 ]; then + echo 'XNU_VERIFIED_ROOT_PROBE must be 0 or 1' >&2 + exit 1 +fi +if [ "$OWNER_ROOT_PROBE" = 1 ] && { [ "${XNU_SOURCE_CORECRYPTO14:-0}" != 1 ] || [ "${XNU_CRYPTO_BOOTSTRAP:-source}" != prelinked ]; }; then + echo 'The owner crypto probe requires the qualified original prelinked crypto provider' >&2 + exit 1 +fi +OWNER_ROOT_AUDIT=${XNU_OWNER_ROOT_AUDIT:-0} +if [ "$OWNER_ROOT_AUDIT" != 0 ] && [ "$OWNER_ROOT_AUDIT" != 1 ]; then + echo 'XNU_OWNER_ROOT_AUDIT must be 0 or 1' >&2 + exit 1 +fi +if [ "$OWNER_ROOT_AUDIT" = 1 ] && [ "$OWNER_ROOT_PROBE" != 1 ]; then + echo 'The root-binding audit requires real owner-verified block crypto' >&2 + exit 1 +fi +OWNER_ROOT_TRUST=${XNU_OWNER_ROOT_TRUST:-0} +if [ "$OWNER_ROOT_TRUST" != 0 ] && [ "$OWNER_ROOT_TRUST" != 1 ]; then + echo 'XNU_OWNER_ROOT_TRUST must be 0 or 1' >&2 + exit 1 +fi +if [ "$OWNER_ROOT_TRUST" = 1 ] && { [ "$OWNER_ROOT_AUDIT" != 1 ] || [ "${XNU_NATIVE_VIRTIO_BLOCK:-0}" != 1 ]; }; then + echo 'Owner root acceptance requires actual mounted-media binding and verified block storage' >&2 + exit 1 +fi +ADRENO_FIRMWARE_PROBE=${XNU_ADRENO_FIRMWARE_PROBE:-0} +if [ "$ADRENO_FIRMWARE_PROBE" != 0 ] && [ "$ADRENO_FIRMWARE_PROBE" != 1 ]; then + echo 'XNU_ADRENO_FIRMWARE_PROBE must be 0 or 1' >&2 + exit 1 +fi +if [ "$ADRENO_FIRMWARE_PROBE" = 1 ] && [ "${XNU_SOURCE_CORECRYPTO14:-0}" != 1 ]; then + echo 'The offline Adreno planner probe requires the qualified crypto build' >&2 + exit 1 +fi +ADRENO_PM4_PROBE=${XNU_ADRENO_PM4_PROBE:-0} +if [ "$ADRENO_PM4_PROBE" != 0 ] && [ "$ADRENO_PM4_PROBE" != 1 ]; then + echo 'XNU_ADRENO_PM4_PROBE must be 0 or 1' >&2 + exit 1 +fi +if [ "$ADRENO_PM4_PROBE" = 1 ] && [ "${XNU_SOURCE_CORECRYPTO14:-0}" != 1 ]; then + echo 'The offline PM4 fixture requires the qualified native kernel configuration' >&2 + exit 1 +fi +SMMU4K_PROBE=${XNU_SMMU4K_PROBE:-0} +if [ "$SMMU4K_PROBE" != 0 ] && [ "$SMMU4K_PROBE" != 1 ]; then + echo 'XNU_SMMU4K_PROBE must be 0 or 1' >&2 + exit 1 +fi +if [ "$SMMU4K_PROBE" = 1 ] && [ "${XNU_SOURCE_CORECRYPTO14:-0}" != 1 ]; then + echo 'The offline native-table fixture requires the qualified kernel configuration' >&2 + exit 1 +fi +CONSTANT_UPLOADS_PROBE=${XNU_CONSTANT_UPLOADS_PROBE:-0} +if [ "$CONSTANT_UPLOADS_PROBE" != 0 ] && [ "$CONSTANT_UPLOADS_PROBE" != 1 ]; then + echo 'XNU_CONSTANT_UPLOADS_PROBE must be 0 or 1' >&2 + exit 1 +fi +if [ "$CONSTANT_UPLOADS_PROBE" = 1 ] && [ "${XNU_SOURCE_CORECRYPTO14:-0}" != 1 ]; then + echo 'The offline constant upload fixture requires the qualified kernel configuration' >&2 + exit 1 +fi +NATIVE_OFFLINE_GPU=${XNU_NATIVE_OFFLINE_GPU:-0} +if [ "$NATIVE_OFFLINE_GPU" != 0 ] && [ "$NATIVE_OFFLINE_GPU" != 1 ]; then + echo 'XNU_NATIVE_OFFLINE_GPU must be 0 or 1' >&2 + exit 1 +fi +if [ "$NATIVE_OFFLINE_GPU" = 1 ] && [ "$NATIVE_VIRTIO_CONSOLE" != 1 ]; then + echo 'The private offline GPU broker requires the actual byte transport' >&2 + exit 1 +fi +SHENG_PMGR=${XNU_SHENG_PMGR:-0} +if [ "$SHENG_PMGR" != 0 ] && [ "$SHENG_PMGR" != 1 ]; then + echo 'XNU_SHENG_PMGR must be 0 or 1' >&2 + exit 1 +fi +if [ "$SHENG_PMGR" = 1 ] && [ "${XNU_SOURCE_CORECRYPTO14:-0}" != 1 ]; then + echo 'The sheng PMGR instrument requires the original-component kernel configuration' >&2 + exit 1 +fi +SHENG_PMGR_STAMP=$ROOT/build/xnu/ios-obj/DEVELOPMENT_ARM64_BCM2837/.sheng-pmgr-mode +if [ "$(cat "$SHENG_PMGR_STAMP" 2>/dev/null || echo 0)" != "$SHENG_PMGR" ]; then + rm -f "$ROOT/build/xnu/ios-obj/DEVELOPMENT_ARM64_BCM2837/bsd/DEVELOPMENT/bsd_init.o" + printf '%s\n' "$SHENG_PMGR" > "$SHENG_PMGR_STAMP" +fi +EXTRA_MAKE_ARGS+=("bsd_init.o_CFLAGS_ADD=-DXNU_SHENG_PMGR=$SHENG_PMGR -DXNU_VERIFIED_ROOT_PROBE=$OWNER_ROOT_PROBE -DXNU_OWNER_ROOT_AUDIT=$OWNER_ROOT_AUDIT -DXNU_OWNER_ROOT_TRUST=$OWNER_ROOT_TRUST -DXNU_ADRENO_FIRMWARE_PROBE=$ADRENO_FIRMWARE_PROBE -DXNU_ADRENO_PM4_PROBE=$ADRENO_PM4_PROBE -DXNU_SMMU4K_PROBE=$SMMU4K_PROBE -DXNU_CONSTANT_UPLOADS_PROBE=$CONSTANT_UPLOADS_PROBE -DXNU_NATIVE_LEGACY_DISPLAY=${XNU_NATIVE_LEGACY_DISPLAY:-0} -DXNU_NATIVE_OFFLINE_GPU=$NATIVE_OFFLINE_GPU") +NATIVE_MOBILE_DISPLAY=${XNU_NATIVE_MOBILE_DISPLAY:-0} +if [ "$NATIVE_MOBILE_DISPLAY" != 0 ] && [ "$NATIVE_MOBILE_DISPLAY" != 1 ]; then + echo 'XNU_NATIVE_MOBILE_DISPLAY must be 0 or 1' >&2 + exit 1 +fi +NATIVE_LEGACY_DISPLAY=${XNU_NATIVE_LEGACY_DISPLAY:-0} +if [ "$NATIVE_LEGACY_DISPLAY" != 0 ] && [ "$NATIVE_LEGACY_DISPLAY" != 1 ]; then + echo 'XNU_NATIVE_LEGACY_DISPLAY must be 0 or 1' >&2 + exit 1 +fi +EXTRA_MAKE_ARGS+=("IOUserClient.cpo_CFLAGS_ADD=-DXNU_NATIVE_MOBILE_DISPLAY=$NATIVE_MOBILE_DISPLAY -DXNU_NATIVE_LEGACY_DISPLAY=$NATIVE_LEGACY_DISPLAY") +MOBILE_DISPLAY_STAMP=$ROOT/build/xnu/ios-obj/DEVELOPMENT_ARM64_BCM2837/.native-mobile-display-mode +if [ "$(cat "$MOBILE_DISPLAY_STAMP" 2>/dev/null || echo 0)" != "$NATIVE_MOBILE_DISPLAY:$NATIVE_LEGACY_DISPLAY" ]; then + rm -f "$ROOT/build/xnu/ios-obj/DEVELOPMENT_ARM64_BCM2837/iokit/DEVELOPMENT/IOUserClient.cpo" + printf '%s\n' "$NATIVE_MOBILE_DISPLAY:$NATIVE_LEGACY_DISPLAY" > "$MOBILE_DISPLAY_STAMP" +fi +OWNER_ROOT_STAMP=$ROOT/build/xnu/ios-obj/DEVELOPMENT_ARM64_BCM2837/.owner-root-probe-mode +if [ "$(cat "$OWNER_ROOT_STAMP" 2>/dev/null || echo 0:0:0:0:0:0)" != "$OWNER_ROOT_PROBE:$OWNER_ROOT_AUDIT:$OWNER_ROOT_TRUST:$ADRENO_FIRMWARE_PROBE:$ADRENO_PM4_PROBE:$SMMU4K_PROBE:$CONSTANT_UPLOADS_PROBE:$NATIVE_LEGACY_DISPLAY:$NATIVE_OFFLINE_GPU" ]; then + rm -f "$ROOT/build/xnu/ios-obj/DEVELOPMENT_ARM64_BCM2837/bsd/DEVELOPMENT/bsd_init.o" + printf '%s\n' "$OWNER_ROOT_PROBE:$OWNER_ROOT_AUDIT:$OWNER_ROOT_TRUST:$ADRENO_FIRMWARE_PROBE:$ADRENO_PM4_PROBE:$SMMU4K_PROBE:$CONSTANT_UPLOADS_PROBE:$NATIVE_LEGACY_DISPLAY:$NATIVE_OFFLINE_GPU" > "$OWNER_ROOT_STAMP" +fi +OWNER_CACHE_MOUNT=${XNU_OWNER_CACHE_MOUNT:-0} +if [ "$OWNER_CACHE_MOUNT" != 0 ] && [ "$OWNER_CACHE_MOUNT" != 1 ]; then + echo 'XNU_OWNER_CACHE_MOUNT must be 0 or 1' >&2 + exit 1 +fi +if [ "$OWNER_CACHE_MOUNT" = 1 ] && [ "$OWNER_ROOT_TRUST" != 1 ]; then + echo 'The offline SystemOS mount requires real independent owner root enforcement' >&2 + exit 1 +fi +OWNER_CACHE_STARTUP=${XNU_OWNER_CACHE_STARTUP:-0} +if [ "$OWNER_CACHE_STARTUP" != 0 ] && [ "$OWNER_CACHE_STARTUP" != 1 ]; then + echo 'XNU_OWNER_CACHE_STARTUP must be 0 or 1' >&2 + exit 1 +fi +if [ "$OWNER_CACHE_STARTUP" = 1 ] && [ "$OWNER_CACHE_MOUNT" != 1 ]; then + echo 'Owner dyld startup selection requires authenticated actual cache mounting' >&2 + exit 1 +fi +OWNER_BOOT_MANIFEST=${XNU_OWNER_BOOT_MANIFEST:-0} +if [ "$OWNER_BOOT_MANIFEST" != 0 ] && [ "$OWNER_BOOT_MANIFEST" != 1 ]; then + echo 'XNU_OWNER_BOOT_MANIFEST must be 0 or 1' >&2 + exit 1 +fi +if [ "$OWNER_BOOT_MANIFEST" = 1 ] && [ "$OWNER_CACHE_STARTUP" != 1 ]; then + echo 'Owner boot metadata requires real authenticated owner root/cache startup' >&2 + exit 1 +fi +EXTRA_MAKE_ARGS+=("IOKitBSDInit.cpo_CFLAGS_ADD=-DXNU_OWNER_BOOT_MANIFEST=$OWNER_BOOT_MANIFEST") +OWNER_MANIFEST_STAMP=$ROOT/build/xnu/ios-obj/DEVELOPMENT_ARM64_BCM2837/.owner-boot-manifest-mode +if [ "$(cat "$OWNER_MANIFEST_STAMP" 2>/dev/null || echo 0)" != "$OWNER_BOOT_MANIFEST" ]; then + rm -f "$ROOT/build/xnu/ios-obj/DEVELOPMENT_ARM64_BCM2837/iokit/DEVELOPMENT/IOKitBSDInit.cpo" + rm -f "$ROOT/build/xnu/ios-obj/DEVELOPMENT_ARM64_BCM2837/bsd/DEVELOPMENT/kern_exec.o" + printf '%s\n' "$OWNER_BOOT_MANIFEST" > "$OWNER_MANIFEST_STAMP" +fi +OWNER_DATA_MOUNT=${XNU_OWNER_DATA_MOUNT:-0} +if [ "$OWNER_DATA_MOUNT" != 0 ] && [ "$OWNER_DATA_MOUNT" != 1 ]; then + echo 'XNU_OWNER_DATA_MOUNT must be 0 or 1' >&2 + exit 1 +fi +if [ "$OWNER_DATA_MOUNT" = 1 ] && [ "$OWNER_BOOT_MANIFEST" != 1 ]; then + echo 'Scratch Data requires authenticated original OS/owner metadata startup' >&2 + exit 1 +fi +OWNER_DATA_HEADER=$ROOT/build/xnu/owner-data-volume.h +python3 - "$OWNER_DATA_HEADER" "$OWNER_DATA_MOUNT" "$NATIVE_RAM_BLOCK" "${XNU_OWNER_DATA_VOLUME_UUID:-}" <<'PY_DATA_UUID' +from pathlib import Path +import sys +import uuid +path = Path(sys.argv[1]) +text = '/* Scratch Data IOKit selection disabled. */\n' +if sys.argv[2:4] == ['1', '1']: + try: + value = str(uuid.UUID(sys.argv[4])).upper() + except ValueError: + raise SystemExit('Set XNU_OWNER_DATA_VOLUME_UUID to the actual observed scratch APFS volume UUID') + text = '#define XNU_OWNER_DATA_VOLUME_UUID "' + value + '"\n' +path.parent.mkdir(parents=True, exist_ok=True) +if not path.exists() or path.read_text() != text: + path.write_text(text) +PY_DATA_UUID +OWNER_DATA_STAMP=$ROOT/build/xnu/ios-obj/DEVELOPMENT_ARM64_BCM2837/.owner-data-mount-mode +if [ "$(cat "$OWNER_DATA_STAMP" 2>/dev/null || echo 0)" != "$OWNER_DATA_MOUNT:$NATIVE_RAM_BLOCK" ]; then + rm -f "$ROOT/build/xnu/ios-obj/DEVELOPMENT_ARM64_BCM2837/bsd/DEVELOPMENT/kern_exec.o" + printf '%s\n' "$OWNER_DATA_MOUNT:$NATIVE_RAM_BLOCK" > "$OWNER_DATA_STAMP" +fi +OWNER_CRYPTEX_FILES=${XNU_OWNER_CRYPTEX_FILES:-0} +if [ "$OWNER_CRYPTEX_FILES" != 0 ] && [ "$OWNER_CRYPTEX_FILES" != 1 ]; then + echo 'XNU_OWNER_CRYPTEX_FILES must be 0 or 1' >&2 + exit 1 +fi +if [ "$OWNER_CRYPTEX_FILES" = 1 ] && [ "$OWNER_CACHE_STARTUP" != 1 ]; then + echo 'Owner Cryptex file lookup requires authenticated mounted SystemOS startup' >&2 + exit 1 +fi +OWNER_CRYPTEX_STAMP=$ROOT/build/xnu/ios-obj/DEVELOPMENT_ARM64_BCM2837/.owner-cryptex-files-mode +if [ "$(cat "$OWNER_CRYPTEX_STAMP" 2>/dev/null || echo 0)" != "$OWNER_CRYPTEX_FILES" ]; then + rm -f "$ROOT/build/xnu/ios-obj/DEVELOPMENT_ARM64_BCM2837/bsd/DEVELOPMENT/kern_exec.o" + printf '%s\n' "$OWNER_CRYPTEX_FILES" > "$OWNER_CRYPTEX_STAMP" +fi +OWNER_UNION_SYSTEMOS=${XNU_OWNER_UNION_SYSTEMOS:-0} +if [ "$OWNER_UNION_SYSTEMOS" != 0 ] && [ "$OWNER_UNION_SYSTEMOS" != 1 ]; then + echo 'XNU_OWNER_UNION_SYSTEMOS must be 0 or 1' >&2 + exit 1 +fi +if [ "$OWNER_UNION_SYSTEMOS" = 1 ] && { [ "$OWNER_CACHE_STARTUP" != 1 ] || [ "$OWNER_CRYPTEX_FILES" != 0 ]; }; then + echo 'Owner union composition requires authenticated startup and no fallback-path experiment' >&2 + exit 1 +fi +OWNER_UNION_STAMP=$ROOT/build/xnu/ios-obj/DEVELOPMENT_ARM64_BCM2837/.owner-union-systemos-mode +if [ "$(cat "$OWNER_UNION_STAMP" 2>/dev/null || echo 0)" != "$OWNER_UNION_SYSTEMOS" ]; then + rm -f "$ROOT/build/xnu/ios-obj/DEVELOPMENT_ARM64_BCM2837/bsd/DEVELOPMENT/kern_exec.o" + printf '%s\n' "$OWNER_UNION_SYSTEMOS" > "$OWNER_UNION_STAMP" +fi +NATIVE_AIR_INIT=${XNU_NATIVE_AIR_INIT:-0} +if [ "$NATIVE_AIR_INIT" != 0 ] && [ "$NATIVE_AIR_INIT" != 1 ]; then + echo 'XNU_NATIVE_AIR_INIT must be 0 or 1' >&2 + exit 1 +fi +if [ "$NATIVE_AIR_INIT" = 1 ] && { [ "$OWNER_CACHE_MOUNT" != 1 ] || [ "$OWNER_DATA_MOUNT" != 1 ] || [ "$OWNER_ROOT_TRUST" != 1 ]; }; then + echo 'Offline AIR init requires the actual owner root, SystemOS and scratch Data path' >&2 + exit 1 +fi +NATIVE_AIR_STAMP=$ROOT/build/xnu/ios-obj/DEVELOPMENT_ARM64_BCM2837/.native-air-init-mode +if [ "$(cat "$NATIVE_AIR_STAMP" 2>/dev/null || echo 0)" != "$NATIVE_AIR_INIT" ]; then + rm -f "$ROOT/build/xnu/ios-obj/DEVELOPMENT_ARM64_BCM2837/bsd/DEVELOPMENT/kern_exec.o" + printf '%s\n' "$NATIVE_AIR_INIT" > "$NATIVE_AIR_STAMP" +fi +EXTRA_MAKE_ARGS+=("kern_exec.o_CFLAGS_ADD=-DXNU_NATIVE_AIR_INIT=$NATIVE_AIR_INIT -DXNU_OWNER_CACHE_MOUNT=$OWNER_CACHE_MOUNT -DXNU_OWNER_CACHE_STARTUP=$OWNER_CACHE_STARTUP -DXNU_OWNER_BOOT_MANIFEST=$OWNER_BOOT_MANIFEST -DXNU_OWNER_DATA_MOUNT=$OWNER_DATA_MOUNT -DXNU_NATIVE_RAM_BLOCK=$NATIVE_RAM_BLOCK -DXNU_OWNER_CRYPTEX_FILES=$OWNER_CRYPTEX_FILES -DXNU_OWNER_UNION_SYSTEMOS=$OWNER_UNION_SYSTEMOS -include $OWNER_DATA_HEADER") +OWNER_CACHE_STAMP=$ROOT/build/xnu/ios-obj/DEVELOPMENT_ARM64_BCM2837/.owner-cache-mount-mode +if [ "$(cat "$OWNER_CACHE_STAMP" 2>/dev/null || echo 0:0)" != "$OWNER_CACHE_MOUNT:$OWNER_CACHE_STARTUP" ]; then + rm -f "$ROOT/build/xnu/ios-obj/DEVELOPMENT_ARM64_BCM2837/bsd/DEVELOPMENT/kern_exec.o" + printf '%s\n' "$OWNER_CACHE_MOUNT:$OWNER_CACHE_STARTUP" > "$OWNER_CACHE_STAMP" +fi +OWNER_CACHE_CARRIER=${XNU_OWNER_CACHE_CARRIER:-0} +OWNER_CACHE_CARRIER_WRONG_UUID=${XNU_OWNER_CACHE_CARRIER_WRONG_UUID:-0} +if { [ "$OWNER_CACHE_CARRIER" != 0 ] && [ "$OWNER_CACHE_CARRIER" != 1 ]; } || + { [ "$OWNER_CACHE_CARRIER_WRONG_UUID" != 0 ] && [ "$OWNER_CACHE_CARRIER_WRONG_UUID" != 1 ]; }; then + echo 'Owner cache carrier selectors must be 0 or 1' >&2 + exit 1 +fi +if { [ "$OWNER_CACHE_CARRIER" = 1 ] && [ "$OWNER_CACHE_STARTUP" != 1 ]; } || + { [ "$OWNER_CACHE_CARRIER_WRONG_UUID" = 1 ] && [ "$OWNER_CACHE_CARRIER" != 1 ]; }; then + echo 'Owner carrier policy requires authenticated owner startup; wrong UUID is an isolated negative control' >&2 + exit 1 +fi +EXTRA_MAKE_ARGS+=("vm_unix.o_CFLAGS_ADD=-DXNU_OWNER_CACHE_CARRIER=$OWNER_CACHE_CARRIER -DXNU_OWNER_CACHE_CARRIER_WRONG_UUID=$OWNER_CACHE_CARRIER_WRONG_UUID") +OWNER_CARRIER_STAMP=$ROOT/build/xnu/ios-obj/DEVELOPMENT_ARM64_BCM2837/.owner-cache-carrier-mode +if [ "$(cat "$OWNER_CARRIER_STAMP" 2>/dev/null || echo 0:0)" != "$OWNER_CACHE_CARRIER:$OWNER_CACHE_CARRIER_WRONG_UUID" ]; then + rm -f "$ROOT/build/xnu/ios-obj/DEVELOPMENT_ARM64_BCM2837/bsd/DEVELOPMENT/vm_unix.o" + printf '%s\n' "$OWNER_CACHE_CARRIER:$OWNER_CACHE_CARRIER_WRONG_UUID" > "$OWNER_CARRIER_STAMP" +fi +DT_PANIC_REGION=${XNU_DT_PANIC_REGION:-0} +if [ "$DT_PANIC_REGION" != 0 ] && [ "$DT_PANIC_REGION" != 1 ]; then + echo 'XNU_DT_PANIC_REGION must be 0 or 1' >&2 + exit 1 +fi +EXTRA_MAKE_ARGS+=("pe_init.o_CFLAGS_ADD=-DXNU_DT_PANIC_REGION=$DT_PANIC_REGION") +DT_PANIC_STAMP=$ROOT/build/xnu/ios-obj/DEVELOPMENT_ARM64_BCM2837/.dt-panic-region-mode +if [ "$(cat "$DT_PANIC_STAMP" 2>/dev/null || echo 0)" != "$DT_PANIC_REGION" ]; then + rm -f "$ROOT/build/xnu/ios-obj/DEVELOPMENT_ARM64_BCM2837/pexpert/DEVELOPMENT/pe_init.o" + printf '%s\n' "$DT_PANIC_REGION" > "$DT_PANIC_STAMP" +fi +ZONE_PGZ_PROBE=${XNU_ZONE_PGZ_ALIGNMENT_PROBE:-0} +if [ "$ZONE_PGZ_PROBE" != 0 ] && [ "$ZONE_PGZ_PROBE" != 1 ]; then + echo 'XNU_ZONE_PGZ_ALIGNMENT_PROBE must be 0 or 1' >&2 + exit 1 +fi +EXTRA_MAKE_ARGS+=("zalloc.o_CFLAGS_ADD=-DXNU_ZONE_PGZ_ALIGNMENT_PROBE=$ZONE_PGZ_PROBE") +ZONE_PGZ_STAMP=$ROOT/build/xnu/ios-obj/DEVELOPMENT_ARM64_BCM2837/.zone-pgz-probe-mode +if [ "$(cat "$ZONE_PGZ_STAMP" 2>/dev/null || echo 0)" != "$ZONE_PGZ_PROBE" ]; then + rm -f "$ROOT/build/xnu/ios-obj/DEVELOPMENT_ARM64_BCM2837/osfmk/DEVELOPMENT/zalloc.o" + printf '%s\n' "$ZONE_PGZ_PROBE" > "$ZONE_PGZ_STAMP" +fi +NATIVE_ARCH_TIMER=${XNU_NATIVE_ARCH_TIMER:-${XNU_ARMCPU:-0}} +if [ "$NATIVE_ARCH_TIMER" != 0 ] && [ "$NATIVE_ARCH_TIMER" != 1 ]; then + echo 'XNU_NATIVE_ARCH_TIMER must be 0 or 1' >&2 + exit 1 +fi +if [ "$NATIVE_ARCH_TIMER" = 1 ] && [ "${XNU_ARMCPU:-0}" != 1 ]; then + echo 'Architectural timer entry requires the actual ArmCpu interrupt provider' >&2 + exit 1 +fi +EXTRA_MAKE_ARGS+=("start.o_SFLAGS_ADD=-DXNU_NATIVE_ARCH_TIMER=$NATIVE_ARCH_TIMER") +NATIVE_TIMER_STAMP=$ROOT/build/xnu/ios-obj/DEVELOPMENT_ARM64_BCM2837/.native-arch-timer-mode +if [ "$(cat "$NATIVE_TIMER_STAMP" 2>/dev/null || echo 0)" != "$NATIVE_ARCH_TIMER" ]; then + rm -f "$ROOT/build/xnu/ios-obj/DEVELOPMENT_ARM64_BCM2837/osfmk/DEVELOPMENT/start.o" + printf '%s\n' "$NATIVE_ARCH_TIMER" > "$NATIVE_TIMER_STAMP" +fi +NATIVE_PAN=${XNU_NATIVE_PAN:-0} +if [ "$NATIVE_PAN" != 0 ] && [ "$NATIVE_PAN" != 1 ]; then + echo 'XNU_NATIVE_PAN must be 0 or 1' >&2 + exit 1 +fi +EXTRA_CFLAGS="$EXTRA_CFLAGS -DXNU_NATIVE_PAN=$NATIVE_PAN" +if [ "$NATIVE_PAN" = 1 ]; then + EXTRA_MAKE_ARGS+=("copyio.o_CFLAGS_ADD=-march=armv8.1-a") + EXTRA_MAKE_ARGS+=("pcb.o_CFLAGS_ADD=-march=armv8.1-a") + EXTRA_MAKE_ARGS+=("platform_tests.o_CFLAGS_ADD=-march=armv8.1-a") +else + EXTRA_MAKE_ARGS+=("copyio.o_CFLAGS_ADD=") + EXTRA_MAKE_ARGS+=("pcb.o_CFLAGS_ADD=") + EXTRA_MAKE_ARGS+=("platform_tests.o_CFLAGS_ADD=") +fi +if [ "${XNU_SOURCE_CORECRYPTO14:-0}" = 1 ]; then + CRYPTO_ARCHIVE=$ROOT/build/corecrypto-2023/libcorecrypto_kernel_noasm-namespaced.a + test -f "$CRYPTO_ARCHIVE" + EXTRA_CFLAGS="$EXTRA_CFLAGS -DXNU_SOURCE_CORECRYPTO14=1" + KERNEL_EXTRA_ARCHIVES="-L$KERNEL_SDKROOT/usr/local/lib/kernel -lfirehose_kernel $CRYPTO_ARCHIVE" + KERNEL_ARCHIVE_DEPENDENCIES=$CRYPTO_ARCHIVE + if [ "$ADRENO_FIRMWARE_PROBE" = 1 ]; then + ADRENO_PROBE_ARCHIVE=$ROOT/build/adreno-firmware-probe/libAdrenoFirmwareProbe.a + test -f "$ADRENO_PROBE_ARCHIVE" + KERNEL_EXTRA_ARCHIVES="$KERNEL_EXTRA_ARCHIVES -Wl,-force_load,$ADRENO_PROBE_ARCHIVE" + KERNEL_ARCHIVE_DEPENDENCIES="$KERNEL_ARCHIVE_DEPENDENCIES $ADRENO_PROBE_ARCHIVE" + fi + if [ "$ADRENO_PM4_PROBE" = 1 ]; then + PM4_PROBE_ARCHIVE=$ROOT/build/adreno-pm4-probe/libAdrenoPM4Probe.a + test -f "$PM4_PROBE_ARCHIVE" + KERNEL_EXTRA_ARCHIVES="$KERNEL_EXTRA_ARCHIVES -Wl,-force_load,$PM4_PROBE_ARCHIVE" + KERNEL_ARCHIVE_DEPENDENCIES="$KERNEL_ARCHIVE_DEPENDENCIES $PM4_PROBE_ARCHIVE" + fi + if [ "$SMMU4K_PROBE" = 1 ]; then + SMMU4K_PROBE_ARCHIVE=$ROOT/build/smmu4k-probe/libArmSmmu4KProbe.a + test -f "$SMMU4K_PROBE_ARCHIVE" + KERNEL_EXTRA_ARCHIVES="$KERNEL_EXTRA_ARCHIVES -Wl,-force_load,$SMMU4K_PROBE_ARCHIVE" + KERNEL_ARCHIVE_DEPENDENCIES="$KERNEL_ARCHIVE_DEPENDENCIES $SMMU4K_PROBE_ARCHIVE" + fi + if [ "$CONSTANT_UPLOADS_PROBE" = 1 ]; then + CONSTANT_UPLOADS_ARCHIVE=$ROOT/build/constant-uploads-probe/libConstantUploadsProbe.a + test -f "$CONSTANT_UPLOADS_ARCHIVE" + KERNEL_EXTRA_ARCHIVES="$KERNEL_EXTRA_ARCHIVES -Wl,-force_load,$CONSTANT_UPLOADS_ARCHIVE" + KERNEL_ARCHIVE_DEPENDENCIES="$KERNEL_ARCHIVE_DEPENDENCIES $CONSTANT_UPLOADS_ARCHIVE" + fi + if [ "${XNU_ARMVIRT_PLATFORM:-0}" = 1 ]; then + ARMVIRT_ARCHIVE=$ROOT/build/armvirt-platform/libArmVirtPlatform.a + test -f "$ARMVIRT_ARCHIVE" + KERNEL_EXTRA_ARCHIVES="$KERNEL_EXTRA_ARCHIVES -Wl,-force_load,$ARMVIRT_ARCHIVE" + KERNEL_ARCHIVE_DEPENDENCIES="$KERNEL_ARCHIVE_DEPENDENCIES $ARMVIRT_ARCHIVE" + fi + if [ "$ARM_DT_PLATFORM" = 1 ]; then + ARM_DT_ARCHIVE=$ROOT/build/armdt-platform/libArmDTPlatform.a + test -f "$ARM_DT_ARCHIVE" + KERNEL_EXTRA_ARCHIVES="$KERNEL_EXTRA_ARCHIVES -Wl,-force_load,$ARM_DT_ARCHIVE" + KERNEL_ARCHIVE_DEPENDENCIES="$KERNEL_ARCHIVE_DEPENDENCIES $ARM_DT_ARCHIVE" + fi + if [ "${XNU_ARMCPU:-0}" = 1 ]; then + ARMCPU_ARCHIVE=$ROOT/build/armcpu/libArmCpu.a + test -f "$ARMCPU_ARCHIVE" + KERNEL_EXTRA_ARCHIVES="$KERNEL_EXTRA_ARCHIVES -Wl,-force_load,$ARMCPU_ARCHIVE" + KERNEL_ARCHIVE_DEPENDENCIES="$KERNEL_ARCHIVE_DEPENDENCIES $ARMCPU_ARCHIVE" + fi + if [ "${XNU_NATIVE_UFS:-0}" = 1 ]; then + UFS_ARCHIVE=$ROOT/build/native-ufs/libNativeUfs.a + test -f "$UFS_ARCHIVE" + KERNEL_EXTRA_ARCHIVES="$KERNEL_EXTRA_ARCHIVES -Wl,-force_load,$UFS_ARCHIVE" + KERNEL_ARCHIVE_DEPENDENCIES="$KERNEL_ARCHIVE_DEPENDENCIES $UFS_ARCHIVE" + fi + if [ "${XNU_NATIVE_ADRENO:-0}" = 1 ]; then + # NativeAdreno plus its kits (scripts/build-native-adreno-kits.py). No personality is installed: the + # board probe calls NativeAdrenoGmuContextBringup directly once it has powered the GPU CX domain. + ADRENO_ARCHIVE=$ROOT/build/native-adreno/libNativeAdreno.a + test -f "$ADRENO_ARCHIVE" + KERNEL_EXTRA_ARCHIVES="$KERNEL_EXTRA_ARCHIVES -Wl,-force_load,$ADRENO_ARCHIVE" + KERNEL_ARCHIVE_DEPENDENCIES="$KERNEL_ARCHIVE_DEPENDENCIES $ADRENO_ARCHIVE" + fi + if [ "$NATIVE_MOBILE_DISPLAY" = 1 ]; then + MOBILE_DISPLAY_ARCHIVE=$ROOT/build/native-mobile-display/libNativeMobileDisplay.late.a + test -f "$MOBILE_DISPLAY_ARCHIVE" + python3 - "$MOBILE_DISPLAY_ARCHIVE" <<'PY_MOBILE_DISPLAY' +import hashlib, json, sys +from pathlib import Path +archive = Path(sys.argv[1]) +report = json.loads(archive.with_name('build-report.json').read_text()) +if hashlib.sha256(archive.read_bytes()).hexdigest() != report['archive_sha256']: + raise SystemExit('Rebuild and qualify the exact own display archive before linking') +PY_MOBILE_DISPLAY + KERNEL_EXTRA_ARCHIVES="$KERNEL_EXTRA_ARCHIVES -Wl,-force_load,$MOBILE_DISPLAY_ARCHIVE" + KERNEL_ARCHIVE_DEPENDENCIES="$KERNEL_ARCHIVE_DEPENDENCIES $MOBILE_DISPLAY_ARCHIVE" + fi + if [ "$NATIVE_LEGACY_DISPLAY" = 1 ]; then + LEGACY_DISPLAY_ARCHIVE=$ROOT/build/native-legacy-display/libNativeLegacyDisplay.late.a + test -f "$LEGACY_DISPLAY_ARCHIVE" + python3 - "$LEGACY_DISPLAY_ARCHIVE" "$ROOT" <<'PY_LEGACY_DISPLAY' +import hashlib, json, sys +from pathlib import Path +archive=Path(sys.argv[1]);root=Path(sys.argv[2]);report=json.loads(archive.with_name('build-report.json').read_text()) +if hashlib.sha256(archive.read_bytes()).hexdigest()!=report['archive_sha256']: + raise SystemExit('Rebuild and qualify the exact owned Legacy archive before linking') +for name,digest in report['source_sha256'].items(): + if hashlib.sha256((root/name).read_bytes()).hexdigest()!=digest: + raise SystemExit('Rebuild the owned Legacy archive after changing '+name) +PY_LEGACY_DISPLAY + KERNEL_EXTRA_ARCHIVES="$KERNEL_EXTRA_ARCHIVES -Wl,-force_load,$LEGACY_DISPLAY_ARCHIVE" + KERNEL_ARCHIVE_DEPENDENCIES="$KERNEL_ARCHIVE_DEPENDENCIES $LEGACY_DISPLAY_ARCHIVE" + fi + if [ "${XNU_NATIVE_AES:-0}" = 1 ]; then + AES_ARCHIVE=${XNU_NATIVE_AES_ARCHIVE:-$ROOT/build/native-aes/libNativeAES.a} + test -f "$AES_ARCHIVE" + KERNEL_EXTRA_ARCHIVES="$KERNEL_EXTRA_ARCHIVES -Wl,-force_load,$AES_ARCHIVE" + KERNEL_ARCHIVE_DEPENDENCIES="$KERNEL_ARCHIVE_DEPENDENCIES $AES_ARCHIVE" + fi + if [ "${XNU_NATIVE_VIRTIO_BLOCK:-0}" = 1 ]; then + VIRTIO_ARCHIVE=${XNU_NATIVE_VIRTIO_BLOCK_ARCHIVE:-$ROOT/build/native-virtio-block/libNativeVirtioBlock.a} + test -f "$VIRTIO_ARCHIVE" + KERNEL_EXTRA_ARCHIVES="$KERNEL_EXTRA_ARCHIVES -Wl,-force_load,$VIRTIO_ARCHIVE" + KERNEL_ARCHIVE_DEPENDENCIES="$KERNEL_ARCHIVE_DEPENDENCIES $VIRTIO_ARCHIVE" + fi + if [ "$NATIVE_OFFLINE_GPU" = 1 ]; then + GPU_BROKER_ARCHIVE=$ROOT/build/native-offline-gpu/libNativeOfflineGPU.late.a + test -f "$GPU_BROKER_ARCHIVE" + python3 - "$GPU_BROKER_ARCHIVE" "$ROOT" <<'PY_GPU_BROKER' +import hashlib,json,sys +from pathlib import Path +archive=Path(sys.argv[1]);root=Path(sys.argv[2]);report=json.loads(archive.with_name('build-report.json').read_text()) +if hashlib.sha256(archive.read_bytes()).hexdigest()!=report['archive_sha256']: + raise SystemExit('Rebuild the exact private GPU broker archive') +for name,digest in report['source_sha256'].items(): + if hashlib.sha256((root/name).read_bytes()).hexdigest()!=digest: + raise SystemExit('Rebuild the GPU broker after changing '+name) +if hashlib.sha256((root/'build/native-virtio-console/libNativeVirtioConsole.a').read_bytes()).hexdigest()!=report['console_archive_sha256']: + raise SystemExit('Rebuild the GPU broker against the actual byte-transport archive') +PY_GPU_BROKER + KERNEL_EXTRA_ARCHIVES="$KERNEL_EXTRA_ARCHIVES -Wl,-force_load,$GPU_BROKER_ARCHIVE" + KERNEL_ARCHIVE_DEPENDENCIES="$KERNEL_ARCHIVE_DEPENDENCIES $GPU_BROKER_ARCHIVE" + fi + if [ "$NATIVE_VIRTIO_CONSOLE" = 1 ]; then + CONSOLE_ARCHIVE=$ROOT/build/native-virtio-console/libNativeVirtioConsole.a + test -f "$CONSOLE_ARCHIVE" + KERNEL_EXTRA_ARCHIVES="$KERNEL_EXTRA_ARCHIVES -Wl,-force_load,$CONSOLE_ARCHIVE" + KERNEL_ARCHIVE_DEPENDENCIES="$KERNEL_ARCHIVE_DEPENDENCIES $CONSOLE_ARCHIVE" + fi + if [ "$NATIVE_RAM_BLOCK" = 1 ]; then + RAM_BLOCK_ARCHIVE=$ROOT/build/native-ram-block/libNativeRamBlock.a + test -f "$RAM_BLOCK_ARCHIVE" + if [ "${XNU_NATIVE_VIRTIO_BLOCK:-0}" = 1 ]; then + # Both transports import the same original class. Keep one exact alias + # definition; link the real RAM transport object, not duplicate aliases. + python3 - "$VIRTIO_ARCHIVE" "$RAM_BLOCK_ARCHIVE" <<'PY_BINDINGS' +import json +from pathlib import Path +import sys +import hashlib +records = [json.loads(Path(p).with_name('bindings.json').read_text()) for p in sys.argv[1:]] +for archive, record in zip(sys.argv[1:], records): + path = Path(archive) + obj = path.with_name(path.stem.removeprefix('lib') + '.o') + if hashlib.sha256(obj.read_bytes()).hexdigest() != record['compiled_object_sha256']: + raise SystemExit('Re-audit the actual compiled storage object before linking') +first, second = [record['bindings'] for record in records] +if any(first.get(name) != value for name, value in second.items()): + raise SystemExit('RAM transport imports must match the already linked original storage bindings') +PY_BINDINGS + RAM_BLOCK_OBJECT=$ROOT/build/native-ram-block/NativeRamBlock.o + test -f "$RAM_BLOCK_OBJECT" + KERNEL_EXTRA_ARCHIVES="$KERNEL_EXTRA_ARCHIVES $RAM_BLOCK_OBJECT" + else + KERNEL_EXTRA_ARCHIVES="$KERNEL_EXTRA_ARCHIVES -Wl,-force_load,$RAM_BLOCK_ARCHIVE" + fi + KERNEL_ARCHIVE_DEPENDENCIES="$KERNEL_ARCHIVE_DEPENDENCIES $RAM_BLOCK_ARCHIVE" + fi + if [ "${XNU_SOURCE_PTHREAD:-0}" = 1 ]; then + PTHREAD_ARCHIVE=$ROOT/build/libpthread-kernel/libpthread_kernel.a + PTHREAD_BOOTSTRAP=$ROOT/build/libpthread-kernel/source_pthread_bootstrap.o + test -f "$PTHREAD_ARCHIVE" + test -f "$PTHREAD_BOOTSTRAP" + KERNEL_EXTRA_ARCHIVES="$KERNEL_EXTRA_ARCHIVES -Wl,-force_load,$PTHREAD_ARCHIVE" + KERNEL_ARCHIVE_DEPENDENCIES="$KERNEL_ARCHIVE_DEPENDENCIES $PTHREAD_ARCHIVE" + fi + if [ "$OWNER_ROOT_PROBE" = 1 ]; then + VERIFIED_ROOT_ARCHIVE=${XNU_VERIFIED_ROOT_ARCHIVE:-$ROOT/build/verified-root-xnu/libVerifiedRootProbe.a} + test -f "$VERIFIED_ROOT_ARCHIVE" + KERNEL_EXTRA_ARCHIVES="$KERNEL_EXTRA_ARCHIVES -Wl,-force_load,$VERIFIED_ROOT_ARCHIVE" + KERNEL_ARCHIVE_DEPENDENCIES="$KERNEL_ARCHIVE_DEPENDENCIES $VERIFIED_ROOT_ARCHIVE" + fi + if [ "$MOCKFS_MODE" = 1 ]; then + MOCKFS_ARCHIVE=$ROOT/build/mockfs-fixture/libmockfs_fixture.a + test -f "$MOCKFS_ARCHIVE" + KERNEL_EXTRA_ARCHIVES="$KERNEL_EXTRA_ARCHIVES -Wl,-force_load,$MOCKFS_ARCHIVE" + KERNEL_ARCHIVE_DEPENDENCIES="$KERNEL_ARCHIVE_DEPENDENCIES $MOCKFS_ARCHIVE" + fi + if [ "$SHENG_PMGR" = 1 ]; then + PMGR_ARCHIVE=$ROOT/build/sheng-platform-power/libShengPlatformPower.a + test -f "$PMGR_ARCHIVE" + KERNEL_EXTRA_ARCHIVES="$KERNEL_EXTRA_ARCHIVES -Wl,-force_load,$PMGR_ARCHIVE" + KERNEL_ARCHIVE_DEPENDENCIES="$KERNEL_ARCHIVE_DEPENDENCIES $PMGR_ARCHIVE" + fi + # Optional extra link inputs (e.g. a stub for an unselected offline probe hook). + if [ "$BOOT_PROGRESS" = 1 ]; then + KERNEL_EXTRA_ARCHIVES="$KERNEL_EXTRA_ARCHIVES -Wl,-force_load,$ROOT/build/boot-progress/libBootProgress.a" + KERNEL_ARCHIVE_DEPENDENCIES="$KERNEL_ARCHIVE_DEPENDENCIES $ROOT/build/boot-progress/libBootProgress.a" + fi + KERNEL_EXTRA_ARCHIVES="$KERNEL_EXTRA_ARCHIVES ${XNU_EXTRA_LD_ARCHIVES:-}" + EXTRA_MAKE_ARGS+=("LD_KERNEL_ARCHIVES=$KERNEL_EXTRA_ARCHIVES") + EXTRA_MAKE_ARGS+=("XNU_EXTERNAL_KERNEL_ARCHIVES=$KERNEL_ARCHIVE_DEPENDENCIES") +fi +python3 - "${XNU_NATIVE_AES:-0}" "${XNU_NATIVE_AES_ARCHIVE:-$ROOT/build/native-aes/libNativeAES.a}" \ + "${XNU_NATIVE_VIRTIO_BLOCK:-0}" "${XNU_NATIVE_VIRTIO_BLOCK_ARCHIVE:-$ROOT/build/native-virtio-block/libNativeVirtioBlock.a}" \ + "$OWNER_ROOT_PROBE" "${XNU_VERIFIED_ROOT_ARCHIVE:-$ROOT/build/verified-root-xnu/libVerifiedRootProbe.a}" \ + "$NATIVE_RAM_BLOCK" "$ROOT/build/native-ram-block/libNativeRamBlock.a" \ + "$ADRENO_FIRMWARE_PROBE" "$ROOT/build/adreno-firmware-probe/libAdrenoFirmwareProbe.a" \ + "$ADRENO_PM4_PROBE" "$ROOT/build/adreno-pm4-probe/libAdrenoPM4Probe.a" \ + "$SMMU4K_PROBE" "$ROOT/build/smmu4k-probe/libArmSmmu4KProbe.a" \ + "$CONSTANT_UPLOADS_PROBE" "$ROOT/build/constant-uploads-probe/libConstantUploadsProbe.a" \ + "$NATIVE_VIRTIO_CONSOLE" "$ROOT/build/native-virtio-console/libNativeVirtioConsole.a" \ + "$NATIVE_OFFLINE_GPU" "$ROOT/build/native-offline-gpu/libNativeOfflineGPU.late.a" \ + "$SHENG_PMGR" "$ROOT/build/sheng-platform-power/libShengPlatformPower.a" \ + "${XNU_NATIVE_ADRENO:-0}" "$ROOT/build/native-adreno/libNativeAdreno.a" <<'PY' +import hashlib +from pathlib import Path +import sys +directory = Path('build/xnu/ios-obj/DEVELOPMENT_ARM64_BCM2837') +stamp = directory / '.native-aes-link-input' +identity = '|'.join(mode + ':' + (hashlib.sha256(Path(archive).read_bytes()).hexdigest() if mode == '1' else 'disabled') + for mode, archive in zip(sys.argv[1::2], sys.argv[2::2])) +if not stamp.exists() or stamp.read_text() != identity: + # Make does not track selection of an older alternative external archive. + # Rebuild the conditional personality and relink when this input changes. + for relative in ('mach.development.bcm2837.unstripped.noctf', + 'iokit/DEVELOPMENT/KernelConfigTables.cpo'): + (directory / relative).unlink(missing_ok=True) + directory.mkdir(parents=True, exist_ok=True) + stamp.write_text(identity) +PY +EXTRA_CFLAGS="$EXTRA_CFLAGS -DXNU_BOOT_PROGRESS=$BOOT_PROGRESS" +make -C .cache/xnu-work "${XNU_BUILD_TARGET:-build}" SDKROOT="${XNU_SDKROOT:-iphoneos}" ARCH_CONFIGS=ARM64 \ + MACHINE_CONFIGS=BCM2837 KERNEL_CONFIGS=DEVELOPMENT \ + ARCH_STRING_FOR_CURRENT_MACHINE_CONFIG=arm64 LARGE_MEMORY_DEFINE=-UARM_LARGE_MEMORY \ + ARM64_LINK_ADDR=0xfffffff007004000 ARM64_PLKSEG_ADDR=0xfffffff004004000 \ + DEPLOYMENT_TARGET_FLAGS="-miphoneos-version-min=17.0 -DXNU_TARGET_OS_IOS" \ + DEPLOYMENT_LINKER_FLAGS="-Wl,-ios_version_min,17.0" \ + RC_DARWIN_KERNEL_VERSION=23.6.0 RC_ProjectSourceVersion=10063.141.1 \ + BUILD_LTO=0 BOUND_CHECKS=0 BUILD_WERROR="${XNU_BUILD_WERROR:-0}" MAKEJOBS=-j"${JOBS:-4}" \ + CFLAGS_EXTRA="$EXTRA_CFLAGS" "${EXTRA_MAKE_ARGS[@]}" \ + OBJROOT="$ROOT/build/xnu/ios-obj" SYMROOT="$ROOT/build/xnu/ios-sym" DSTROOT="$ROOT/build/xnu/ios-dst" +if [ "${XNU_SOURCE_CORECRYPTO14:-0}" = 1 ] && [ "$CRYPTO_BOOTSTRAP_MODE" = source ]; then + python3 scripts/seal-source-crypto-kernel.py \ + build/xnu/ios-obj/DEVELOPMENT_ARM64_BCM2837/mach.development.bcm2837.unstripped.noctf +fi diff --git a/scripts/check-accelerator-abi.py b/scripts/check-accelerator-abi.py new file mode 100644 index 0000000..e17f3f8 --- /dev/null +++ b/scripts/check-accelerator-abi.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 +"""Qualify actual original-base construction/lifetime and two real unpublished controls.""" +import hashlib,json,re,struct,subprocess,sys +from pathlib import Path +ROOT=Path(__file__).resolve().parents[1];sys.path.insert(0,str(ROOT/'kits/NativePrelink')) +from native_prelink import macho,backed_offset,exports +def require(v,s): + if not v:raise SystemExit(s) +def sha(p):return hashlib.sha256(p.read_bytes()).hexdigest() +def load(p):return json.loads(p.read_text()) +base=ROOT/'build/native-prelink/native-owner-accelerator-abi-candidate';plan=load(base.with_suffix('.json'));raw=base.with_suffix('.macho').read_bytes();im=macho(raw) +require(hashlib.sha256(raw).hexdigest()==plan['candidate_sha256'] and sha(ROOT/'build/xnu/ios-obj/DEVELOPMENT_ARM64_BCM2837/mach.development.bcm2837.unstripped.noctf')==plan['native_sha256'],'Exact current native and immutable candidate') +old=load(ROOT/'build/native-prelink/native-owner-offline-gpu-broker-cpu-candidate.json') +require(plan['components']==old['components'] and plan['original_executable_sha256']==old['original_executable_sha256'] and plan['early_security_initializers']==old['early_security_initializers'],'Same original component/security initialization closure') +for module,digest in plan['original_executable_sha256'].items(): + e=next(e for e in plan['extents'] if e['module']==module and e['segment']=='__TEXT_EXEC');at=backed_offset(im['segments'],e['va'],e['size'],len(raw)) + require(hashlib.sha256(raw[at:at+e['size']]).hexdigest()==digest,'Complete unchanged original executable: '+module) +abi=load(ROOT/'build/gpu-analysis/ioaccelerator-base-abi.json');build=load(ROOT/'build/native-offline-accelerator/build-report.json');broker=load(ROOT/'build/native-offline-gpu/build-report.json') +for report in (build,broker): + for name,digest in report['source_sha256'].items():require(sha(ROOT/name)==digest,'Exact actual compiled source: '+name) +require(broker['accelerator_abi_probe'] and sha(ROOT/'build/native-offline-gpu/libNativeOfflineGPU.late.a')==broker['archive_sha256'] and + sha(ROOT/'build/native-offline-accelerator/libNativeOfflineAccelerator.late.a')==build['archive_sha256'] and + build['original_abi_sha256']==sha(ROOT/'build/gpu-analysis/ioaccelerator-base-abi.json') and build['virtual_entries']==168 and build['owned_entries']==[0,1,7], + 'Actual explicitly enabled component, inherited table and exactly qualified original constructor/destructor/meta imports') +names={n:v for v,group in exports(raw,im).items() for n in group};vtable=names['__ZTV24NativeOfflineAccelerator']+16 +records=[] +for case in ('positive','missing-base','default-off'): + folder=ROOT/'build'/('darwinboot-owner-hvf-accelerator-abi'+('' if case=='positive' else '-'+case));path=folder/'gdb-stop.json';d=load(path);state=d['native_accelerator'] + require(d['kernel_sha256']==plan['candidate_sha256'] and not d['bounded_timeout_interrupted'] and + d['stop_symbol']==('_native_gpu_broker_checkpoint' if case=='default-off' else '_native_accelerator_checkpoint') and d['native_4k_mmu_observed'] and + d['original_amfi_table_owner_observed'] and d['actual_signature_failures']==dict(_cs_validate_page_bad_hash=0,_cs_validate_page_no_hash=0), + 'Actual bounded native CPU-4K/original AMFI kernel observation, not a userspace/desktop checkpoint') + require(not d['owner_root_binding_audit']['boot_root_accepted'],'These actual early observations precede root acceptance; never report them as a root/userspace boot') + require(d['native_iogpu_load_status']==(-1 if case=='missing-base' else 0),'Actual original IOGPU component load/default-off state') + if case=='positive': + require(state['stage']==3 and not state['status'] and state['allocated']==state['initialized']==state['parent_match']==state['released']==1 and + not state['unrelated_match'] and not state['attached'] and state['size']==136 and state['instances_before']==state['instances_after']==0 and state['instances_live']==1 and + state['object'] and state['vptr']==vtable and state['parent']==state['superclass']==int(abi['metaclass'],16), + 'Real original C2-backed object initializes, matches its actual original parent, stays outside registry planes and releases its only live instance') + else: + require(state['stage']==(1 if case=='missing-base' else 0) and state['status']==(-536870160 if case=='missing-base' else -1) and + all(not value for name,value in state.items() if name not in ('stage','status')), + 'Actual missing original parent refuses before allocation; default-off constructs no object or device') + b=d['native_gpu_broker'];require(b['stage']==3 and not b['status'] and all(not b[name] for name in ('clients','active','closed','requests','failures','refusals','reply_bytes')) and not b['actual_cpu_contexts'], + 'Only the unchanged private service startup occurs; no GPU clients, submissions or protocol success is invented') + sink=load(folder/'console-sink.json');require(not sink['errors'] and sink['protocol_responses_sent']==0 and not sink['gpu_execution'] and not sink['system_device_published'],'Read-only console prefix sink supplies zero GPU replies') + records.append(dict(case=case,report_sha256=sha(path),actual_class_state=state,original_root_accepted=False,userspace_executed=False)) +(ROOT/'build/native-offline-accelerator/abi-checkpoint.json').write_text(json.dumps(dict(candidate_sha256=plan['candidate_sha256'],native_sha256=plan['native_sha256'],observations=records, + actual_original_base_lifecycle=True,actual_instance_counts=[0,1,0],actual_native_cpu_4k=True,original_instructions_modified=False, + system_device_published=False,system_metal=False,original_desktop=False,adreno_execution=False,active_native_adreno_4k=False),indent=2)+'\n') +print('PASS: real original IOAcceleratorES constructor/ancestry/lifetime (0 -> 1 -> 0), actual missing-parent refusal and default-off; no device attached/published') +print('Early native CPU 4K/kernel ABI evidence only, before root acceptance/userland; normal Metal, desktop and Adreno remain open') diff --git a/scripts/check-adreno-constant-commands.py b/scripts/check-adreno-constant-commands.py new file mode 100644 index 0000000..592c874 --- /dev/null +++ b/scripts/check-adreno-constant-commands.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python3 +"""Check actual AIR/IR3 parameter images through native A740 direct PM4 encoding.""" +import ctypes as C +import hashlib +import json +import os +from pathlib import Path +import struct +import subprocess +import sys +import xml.etree.ElementTree as ET +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'kits/NativePrelink')) +from native_prelink import macho +OUT = ROOT / 'build/adreno-constant-commands' +OUT.mkdir(parents=True, exist_ok=True) +KIT = ROOT / 'kits/AdrenoPM4' +MESA = ROOT / '.cache/mesa-work' +REV = '85fda3b36405bcaea8bffc64aa4f2651dc90770d' + + +def run(cmd, **kw): + return subprocess.check_output([str(v) for v in cmd], timeout=30, **kw) + + +def sha(raw): + return hashlib.sha256(raw).hexdigest() + + +if run(['git', '-C', MESA, 'rev-parse', 'HEAD'], text=True).strip() != REV: + raise SystemExit('Pinned Mesa revision differs') +paths = ['src/gallium/drivers/freedreno/a6xx/fd6_const.cc', + 'src/gallium/drivers/freedreno/a6xx/fd6_emit.h', + 'src/gallium/drivers/freedreno/ir3/ir3_const.h', + 'src/freedreno/registers/adreno/adreno_pm4.xml', + 'src/freedreno/common/freedreno_pm4.h'] +documents = {p: run(['git', '-C', MESA, 'show', REV + ':' + p]) for p in paths} +manifest = json.loads((ROOT / 'scripts/data/adreno-constant-inputs.json').read_text()) +if manifest['mesa_revision'] != REV or manifest['files'] != {p: sha(raw) for p, raw in documents.items()}: + raise SystemExit('Exact reference source hashes differ') +for p in paths[3:]: + if (MESA / p).read_bytes() != documents[p]: + raise SystemExit('Actual encoder/XML reference working source differs') +xml = ET.fromstring(documents[paths[3]]) +ns = {'r': xml.tag.split('}')[0].lstrip('{')} +domain = xml.find("r:domain[@name='CP_LOAD_STATE6']", ns) +fields = {v.attrib['name']: (int(v.attrib['low']), int(v.attrib['high'])) + for v in domain.find("r:reg32[@name='0']", ns)} +enums = {v.attrib['name']: int(v.attrib['value'], 0) for enum in xml.findall('.//r:enum', ns) for v in enum.findall('r:value', ns) if 'value' in v.attrib} +# Build the unit oracle against the actual Mesa-generated XML/header functions. +test = OUT / 'test-constants' +run(['xcrun', 'clang', '-std=c11', '-O2', '-Wall', '-Wextra', '-Werror', + '-fsanitize=address,undefined', '-DMESA_REFERENCE', + '-I', ROOT / 'build/mesa-ir3/src/freedreno/registers/adreno', + '-I', MESA / 'src/freedreno/common', KIT / 'AdrenoPM4.c', KIT / 'AdrenoConstants.c', + KIT / 'test_constants.c', '-o', test]) +unit = run([test], text=True).strip() +if not unit.startswith('PASS: 9202 direct upload cases,'): + raise SystemExit('Native unit inventory changed') +print(unit) +objects = [] +native_imports = {} +for name in ('AdrenoPM4', 'AdrenoConstants'): + obj = OUT / (name + '.ios.o') + run(['xcrun', 'clang', '-target', 'arm64-apple-ios17.0', '-std=c11', '-O2', '-Wall', '-Wextra', '-Werror', + '-Wno-incompatible-sysroot', '-ffreestanding', '-fstack-protector-strong', '-c', KIT / (name + '.c'), '-o', obj]) + parsed = macho(obj.read_bytes()) + version = next(at for cmd, at, _ in parsed['commands'] if cmd == 0x32) + if parsed['kind'] != 1 or struct.unpack_from('> 16 & 127, header & 0x3fff + if header >> 28 != 7 or opcode != enums['CP_LOAD_STATE6_GEOM'] or payload != 3 + size//4 or low or high: + raise SystemExit('Unexpected command/opcode/address') + for field, expected in [('DST_OFF', offset//16), ('STATE_TYPE', enums['ST6_CONSTANTS']), ('STATE_SRC', enums['SS6_DIRECT']), ('STATE_BLOCK', enums['SB6_VS_SHADER']), ('NUM_UNIT', size//16)]: + lo, hi = fields[field] + if (state >> lo) & ((1 << (hi-lo+1))-1) != expected: raise SystemExit('XML field mismatch: '+field) + restored[offset:offset+size] = encoded[at+16:at+16+size]; at += (1+payload)*4 + if restored != bytes(image) or at != len(encoded) or bytes(output)[at:] != b'\x6b'*(len(output)*4-at): + raise SystemExit('Payload, undeclared allocation or output tail differs') + (OUT / f'{library}-{index}.pm4').write_bytes(encoded) + results.append(dict(library=library, index=index, abi_sha256=old['abi_sha256'], binary_sha256=old['binary_sha256'], + command_sha256=sha(encoded), command_words=r.a, packets=r.b, transferred_bytes=packed.a, + only_explicit_intervals=True, indirect_gpu_addresses=False, full_chip_identity_profiles=2, gpu_execution=False)) +report = dict(mesa_revision=REV, source_documents=manifest['files'], unit=unit, layouts=results, + source_sha256={str(p.relative_to(ROOT)): sha(p.read_bytes()) for p in (KIT/'AdrenoPM4.c', KIT/'AdrenoConstants.c', KIT/'AdrenoConstants.h', ROOT/'kits/AirNir/ConstantUploads.c')}, + native_imports=native_imports, native_objects={p.name: sha(p.read_bytes()) for p in objects}, native_archive_sha256=sha(archive.read_bytes()), + native_cpp_sha256=sha(cpp_obj.read_bytes()), ordinary_arm64_ios17=True, + actual_submission=False, gpu_execution=False, metal_device=False) +(OUT / 'report.json').write_text(json.dumps(report, indent=2)+'\n') +print('PASS: 14 actual IR3 parameter images through PM4/XML decode, no unowned intervals uploaded; native ARM64/iOS17 C/C++ archive') diff --git a/scripts/check-adreno-constant-uploads.py b/scripts/check-adreno-constant-uploads.py new file mode 100755 index 0000000..158fd45 --- /dev/null +++ b/scripts/check-adreno-constant-uploads.py @@ -0,0 +1,169 @@ +#!/usr/bin/env python3 +"""Qualify actual IR3 constant/I/O ABI and native bounded constant packing.""" +import argparse +import ctypes as C +import hashlib +import json +import mmap +import os +from pathlib import Path +import re +import struct +import subprocess + +p=argparse.ArgumentParser(description=__doc__) +p.add_argument('--compiler',type=Path,required=True) +a=p.parse_args() +ROOT=Path(__file__).resolve().parents[1] +KIT=ROOT/'kits/AirNir' +OUT=ROOT/'build/air-nir/constant-uploads' +OUT.mkdir(parents=True,exist_ok=True) +env={k:v for k,v in os.environ.items() if not k.startswith(('IR3_','FD_MESA_','NIR_','MESA_SHADER_'))} +env['IR3_SHADER_DEBUG']='asmroundtrip' + + +def run(command,timeout=30): + r=subprocess.run([str(x) for x in command],env=env,capture_output=True,text=True,timeout=timeout) + if r.returncode:raise RuntimeError(f'Status {r.returncode}: {r.stderr[-2000:]}') + return r.stdout.strip() + + +def sha(path):return hashlib.sha256(path.read_bytes()).hexdigest() + + +# Sanitized synthetic controls include actual unreadable pages after 8/80 bytes. +test=OUT/'guarded-test' +run(['/usr/bin/clang','-std=c11','-Wall','-Wextra','-Werror','-O1','-g','-fsanitize=address,undefined', + KIT/'ConstantUploads.c',KIT/'tests/test_constant_uploads.c','-o',test]) +unit=run([test]) +if not unit.startswith('PASS: 33 guarded constant upload cases;'):raise RuntimeError('Guard test inventory differs') +shared=OUT/'ConstantUploads.dylib' +run(['/usr/bin/clang','-std=c11','-Wall','-Wextra','-Werror','-O2','-dynamiclib',KIT/'ConstantUploads.c','-o',shared]) +# Native ordinary ARM64/iOS 17 cross-build, no LLVM/NIR or host-only imports. +obj=OUT/'ConstantUploads.ios.o' +clang=run(['xcrun','--sdk','iphoneos','--find','clang']) +run([clang,'-target','arm64-apple-ios17.0','-std=c11','-O2','-Wall','-Wextra','-Werror', + '-ffreestanding','-fstack-protector-strong','-c',KIT/'ConstantUploads.c','-o',obj]) +data=obj.read_bytes() +magic,cpu,subtype,kind,ncmds,cmdbytes,flags,reserved=struct.unpack_from('<8I',data) +if (magic,cpu,subtype,kind)!=(0xfeedfacf,0x100000c,0,1):raise RuntimeError('Native target Mach-O header differs') +platforms=[];offset=32 +for _ in range(ncmds): + cmd,size=struct.unpack_from('len(data):raise RuntimeError('Invalid target object load command') + if cmd==0x32:platforms.append(struct.unpack_from('len(binary) or + abi['binary_alignment_bytes']!=128 or len(binary)%128 or abi['upload_alignment_bytes']!=64 or + abi['constant_data_bytes'] or abi['private_bytes_per_fiber'] or + abi['remaining_ubo_slots'] or abi['push_ubo_with_preamble'] or abi['load_shader_consts_via_preamble']): + raise RuntimeError('Target ABI capability/configuration differs; do not assume newer A7xx behavior') + if abi['driver_parameter_words'] or any(abi[k]>=0 for k in ('driver_params_ubo','primitive_params_ubo','primitive_map_ubo')): + raise RuntimeError('New driver-owned UBO/parameter requirement needs independent qualification') + allocations={v['name']:v for v in abi['allocations']} + printed={name:(int(begin)*16,int(words)*16) for name,begin,words in re.findall(r'^; (\w+)\s+c(\d+)\.x \((\d+) vec4\)$',assembly,re.M)} + printed={('primitive_params' if name=='primitive_param' else name):value for name,value in printed.items()} + if printed!={v['name']:(v['begin'],v['bytes']) for v in abi['allocations'] if v['bytes']}: + raise RuntimeError('Actual struct allocation units differ from independent disassembly') + imm=abi['immediates'] + shown=[] + for reg,values in re.findall(r'^@const\(c(\d+)\.x\)\s+(.+)$',assembly,re.M): + if int(reg)*16 != imm['destination_begin']+len(shown)*4:raise RuntimeError('Immediate destination differs') + shown += [int(value,16) for value in re.findall(r'0x[0-9a-f]+',values)] + if shown!=imm['values'] or len(shown)!=imm['padded_words'] or imm['logical_words']>len(shown) or not imm['cpu_upload']: + raise RuntimeError('Omitted/corrupt immediate pool') + ranges=(Range*len(abi['ubo_ranges']))(*(Range(r['binding'],r['source_begin'],r['source_end'],r['destination_begin']) for r in abi['ubo_ranges'])) + reads=(Read*len(abi['source_reads']))(*(Read(r['binding'],r['offset'],r['bytes']) for r in abi['source_reads'])) + for r in abi['ubo_ranges']: + if r['bindless'] or r['global']:raise RuntimeError('Unsupported resource promotion kind') + immediates=(C.c_uint32*len(shown))(*shown) + alloc=allocations['ubo_ranges'] + layout=Layout(abi['constant_bytes'],abi['upload_alignment_bytes'],alloc['begin'],alloc['bytes'],ranges,len(ranges),reads,len(reads),imm['destination_begin'],len(shown),immediates) + bindings=(Binding*16)();backings=[];original_bytes={} + for item in source['inputs']: + if item['kind']!='air.buffer':continue + size=item['minimum_bytes'];slot=item['index'] + if size>page:raise RuntimeError('Guarded fixture currently supports one page per actual source binding') + memory=mmap.mmap(-1,page*2,flags=mmap.MAP_PRIVATE|mmap.MAP_ANONYMOUS,prot=mmap.PROT_READ|mmap.PROT_WRITE) + base=C.addressof(C.c_char.from_buffer(memory));address=base+page-size + if address%item['minimum_alignment']:raise RuntimeError('Original source alignment not met') + contents=bytes((slot*29+i*7)&255 for i in range(size)) + C.memmove(address,contents,size) + if libc.mprotect(base+page,page,0):raise RuntimeError('Guard page protection failed') + bindings[slot]=Binding(address,size,size);original_bytes[slot]=contents;backings.append(memory) + image=(C.c_uint8*abi['constant_bytes'])(*([0x6b]*abi['constant_bytes'])) + result=pack(C.byref(layout),bindings,image,len(image)) + if result.status:raise RuntimeError(f'Actual compiler layout refused: {prefix.name}, status {result.status}') + expected=bytearray([0x6b]*len(image));written=set();padded=0 + for r in abi['ubo_ranges']: + data=original_bytes[r['binding']] + for offset in range(r['source_begin'],r['source_end']): + dest=r['destination_begin']+offset-r['source_begin'] + expected[dest]=data[offset] if offset=len(data));written.add(dest) + for i,value in enumerate(shown): + dest=imm['destination_begin']+i*4;expected[dest:dest+4]=struct.pack('> 14 & 3 == 0 and r['TCR_EL1'] >> 30 & 3 == 2 and + r['SCTLR_EL1'] & 1 and r['TTBR0_EL1'] and r['TTBR1_EL1'] and + d['original_crypto_initialized_observed'] and d['original_crypto_table_owner_observed'], 'Bounded genuine kernel execution and native-4K hardware state') + state = d['native_adreno_firmware'] + require(state['input_bytes'] == 66844 and not state['gpu_execution_qualified'] and not state['firmware_authenticity_qualified'], 'Do not promote a CPU-side planner/hash check into GPU execution or firmware authenticity') + capture, span = state['input_capture'], state['input_pages'] + va = int(span['address'], 16) + at = backed_offset(macho(image)['segments'], va, 66844, len(image)) + require(hashlib.sha256(image[at:at + 66844]).hexdigest() == firmware_hash and + int(capture['pointer'], 16) == va and capture['bytes'] == image[at:at + 32].hex() and + span['bytes'] == 66844 and [int(p['virtual_address'], 16) for p in span['pages']] == + list(range(va & ~4095, (va + 66844 + 4095) & ~4095, 4096)), 'Actual complete firmware extent and every covered hardware-page sample') + require(len(span['pages']) == 17 and all(p['leaf_bytes'] in (4096, 2097152) and p['ap'] == 2 and p['pxn'] and p['uxn'] + for p in span['pages']), 'Entire firmware input must use native-4K-granule readonly/NX mappings; 2-MiB block leaves are recorded, not relabeled 4K leaves') + if label == 'native': + require(d['stop_symbol'] == '_native_adreno_firmware_checkpoint' and state['requested'] and + state['status'] == 0 and state['records'] == 15 and state['payload_bytes'] == 66604 and + state['refusal_mask'] == 15 and state['output_preserved'] == 1 and state['sha256'] == firmware_hash, + 'Actual complete hash/planning plus truncation/capacity/missing-range/owned-copy hash refusals') + else: + require(not state['requested'] and state['status'] == -1 and state['records'] == state['payload_bytes'] == + state['refusal_mask'] == state['output_preserved'] == 0 and state['sha256'] == '00' * 32 and + d['original_launchd_main_executed'] and d['followed_main_entry']['first_instruction_completed'] and + int(d['pstate'], 16) & 31 == 0 and d['direct_user_commpage_shifts'] == {'fffff4025:1': 12, 'fffff4037:1': 12} and + d['owner_root_binding_audit']['boot_root_accepted'] and d['owner_root_binding_audit']['_native_owner_root_apple_autherr'] == 80 and + d['original_amfi_table_owner_observed'] and d['actual_signature_failures'] == + dict(_cs_validate_page_bad_hash=0, _cs_validate_page_no_hash=0) and + d['native_virtio_integrity']['_native_virtio_integrity_failures'] == 0, + 'Default-off must preserve original launchd, actual owner-root binding, original signatures and native user 4K') +print('PASS: native-XNU actual GMU plan/hash and four real refusal controls; all input mappings readonly/NX with 4K hardware granule') +print('PASS: probe remains default-off through original launchd main; actual Adreno execution/Metal remain unqualified') diff --git a/scripts/check-adreno-firmware.py b/scripts/check-adreno-firmware.py new file mode 100644 index 0000000..4a05b55 --- /dev/null +++ b/scripts/check-adreno-firmware.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +"""Run bounded real-firmware host controls and cross-build the native GMU planner.""" +import hashlib +import json +from pathlib import Path +import struct +import subprocess +import sys + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'kits/NativePrelink')) +from native_prelink import macho + +subprocess.run([sys.executable, ROOT / 'scripts/fetch-adreno-inputs.py'], check=True) +directory = ROOT / 'kits/AdrenoFirmware' +out = ROOT / 'build/adreno-firmware' +out.mkdir(parents=True, exist_ok=True) +firmware = ROOT / '.cache/firmware/sheng-gpu/qcom/gmu_gen70200.bin' +test = out / 'test-gmu' +subprocess.run(['xcrun', 'clang', '-std=c11', '-O2', '-Wall', '-Wextra', '-Werror', + '-fsanitize=address,undefined', directory / 'AdrenoFirmware.c', directory / 'test_gmu.c', '-o', test], check=True) +result = subprocess.run([test, firmware], capture_output=True, text=True, timeout=30, check=True) +print(result.stdout, end='') +obj = out / 'AdrenoFirmware.o' +subprocess.run(['xcrun', 'clang', '-target', 'arm64-apple-ios17.0', '-std=c11', '-O2', + '-Wall', '-Wextra', '-Werror', '-Wno-incompatible-sysroot', '-ffreestanding', '-c', + directory / 'AdrenoFirmware.c', '-o', obj], check=True) +data = obj.read_bytes() +image = macho(data) +version = next(at for cmd, at, _ in image['commands'] if cmd == 0x32) +if image['kind'] != 1 or struct.unpack_from(' +#include +#include +#include "AdrenoHfiStartup.h" +typedef uint8_t u8;typedef uint16_t u16;typedef uint32_t u32;typedef uint64_t u64;typedef int32_t s32; +typedef int spinlock_t;typedef int atomic_t; +#define __packed __attribute__((packed)) +#define container_of(p,t,m) ((t *)((char *)(p)-offsetof(t,m))) +#define DRM_DEV_ERROR(...) ((void)0) +'''+header+''' +#define GMU_IDLE_STATE_IFPC 3 +struct adreno_gpu {int unused;}; +struct a6xx_gmu { + int legacy,nr_gpu_freqs,nr_gmu_freqs,idle_level;void *dev; + u64 gpu_freqs[16],gmu_freqs[4];u32 gx_arc_votes[16],cx_arc_votes[4]; + struct a6xx_hfi_acd_table acd_table;struct a6xx_hfi_msg_bw_table bw; +}; +struct a6xx_gpu {struct adreno_gpu base;struct a6xx_gmu gmu;}; +static int adreno_is_a8xx(const struct adreno_gpu *gpu){(void)gpu;return 0;} +static int a8xx_hfi_send_perf_table(struct a6xx_gmu *gmu){(void)gmu;return -1;} +static int a6xx_hfi_start_v1(struct a6xx_gmu *gmu,int boot){(void)gmu;(void)boot;return -1;} +static unsigned *sink_count,*sink_ids,*sink_words,fail_at;static u32 (*sink_data)[159]; +static int a6xx_hfi_send_msg(struct a6xx_gmu *g,int id,void *msg,u32 bytes,u32 *output,u32 capacity) { + (void)g;(void)output;(void)capacity;unsigned i=(*sink_count)++; + if(i>=8 || bytes<4 || bytes>640 || bytes%4)return -1; + sink_ids[i]=(unsigned)id;sink_words[i]=bytes/4-1;memcpy(sink_data[i],(u32 *)msg+1,bytes-4); + return fail_at==i+1?-1:0; +} +static int a6xx_hfi_send_bw_table(struct a6xx_gmu *g) { + return a6xx_hfi_send_msg(g,HFI_H2F_MSG_BW_TABLE,&g->bw,sizeof(g->bw),NULL,0); +} +#define IFPC_LONG_HYST 0x1680 +''' +if '#define GMU_IDLE_STATE_IFPC 3' not in (cache/names[2]).read_text() or '#define IFPC_LONG_HYST 0x1680' not in source:raise SystemExit('Pinned IFPC policy changed') +functions=['static int a6xx_hfi_send_perf_table(','static int a6xx_hfi_feature_ctrl_msg(', + 'static int a6xx_hfi_enable_ifpc(','static int a6xx_hfi_enable_acd(', + 'static int a6xx_hfi_send_start(','static int a6xx_hfi_send_core_fw_start(','int a6xx_hfi_start('] +wrapper=''' +int source_startup(const AdrenoHfiStartupConfig *c,unsigned fail,unsigned *count,unsigned *ids,unsigned *words,u32 data[8][159]) { + struct a6xx_gpu gpu={0};struct a6xx_gmu *g=&gpu.gmu; + g->nr_gpu_freqs=c->gpu_count;g->nr_gmu_freqs=c->gmu_count; + for(unsigned i=0;igpu_count;++i){g->gpu_freqs[i]=c->gpu[i].hz;g->gx_arc_votes[i]=c->gpu[i].vote;} + for(unsigned i=0;igmu_count;++i){g->gmu_freqs[i]=c->gmu[i].hz;g->cx_arc_votes[i]=c->gmu[i].vote;} + g->bw.bw_level_num=c->bandwidth_count;g->bw.cnoc_cmds_num=c->cnoc_count;g->bw.ddr_cmds_num=c->ddr_count; + g->bw.cnoc_wait_bitmask=c->cnoc_wait_mask;g->bw.ddr_wait_bitmask=c->ddr_wait_mask; + memcpy(g->bw.cnoc_cmds_addrs,c->cnoc_addresses,sizeof(c->cnoc_addresses));memcpy(g->bw.cnoc_cmds_data,c->cnoc_data,sizeof(c->cnoc_data)); + memcpy(g->bw.ddr_cmds_addrs,c->ddr_addresses,sizeof(c->ddr_addresses));memcpy(g->bw.ddr_cmds_data,c->ddr_data,sizeof(c->ddr_data)); + g->acd_table.version=c->acd_version;g->acd_table.enable_by_level=c->acd_enable_mask;g->acd_table.stride=c->acd_stride;g->acd_table.num_levels=c->acd_count; + memcpy(g->acd_table.data,c->acd_data,sizeof(c->acd_data));g->idle_level=c->enable_ifpc?GMU_IDLE_STATE_IFPC:0; + *count=0;sink_count=count;sink_ids=ids;sink_words=words;sink_data=data;fail_at=fail; + return a6xx_hfi_start(g,0); +} +''' +oracle=out/'original-startup-oracle.c';oracle.write_text(prelude+'\n'.join(function(f) for f in functions)+wrapper) +sources=[kit/'AdrenoHfi.c',kit/'AdrenoHfiStartup.c',kit/'test_startup.c'] +test=out/'test-startup' +subprocess.run(['xcrun','clang','-std=gnu11','-O2','-Wall','-Wextra','-Werror','-fsanitize=address,undefined','-DSOURCE_ORACLE','-I',kit,*sources,oracle,'-o',test],check=True,timeout=60) +r=subprocess.run([test],check=True,capture_output=True,text=True,timeout=30);print(r.stdout,end='');(out/'host.log').write_text(r.stdout) +flags=['-target','arm64-apple-ios17.0','-O2','-Wall','-Wextra','-Werror','-Wno-incompatible-sysroot','-ffreestanding','-mgeneral-regs-only'] +objects=[] +for source in sources[:2]: + obj=out/(source.stem+'.o');subprocess.run(['xcrun','clang',*flags,'-std=c11','-c',source,'-o',obj],check=True,timeout=30) + raw=obj.read_bytes() + if struct.unpack_from('<4I',raw)!=(0xfeedfacf,0x100000c,0,1):raise SystemExit('Not ordinary ARM64 object') + at=32;ios17=False + for _ in range(struct.unpack_from(' +#include +#include +#include +#include "AdrenoHfiVotes.h" +typedef uint8_t u8;typedef uint16_t u16;typedef uint32_t u32;typedef uint64_t u64;typedef int32_t s32; +typedef uint16_t __le16;typedef uint32_t __le32;typedef int spinlock_t;typedef int atomic_t; +#define __packed __attribute__((packed)) +#define BIT(n) (1u<<(n)) +#define GENMASK(h,l) ((UINT32_MAX>>(31-(h)))& (UINT32_MAX<<(l))) +static u32 u32_encode_bits(u32 value,u32 mask){return (value<<__builtin_ctz(mask))&mask;} +#define IS_ERR(p) ((p)==NULL) +#define PTR_ERR(p) (-1) +#define EINVAL 22 +#define DRM_DEV_ERROR(...) ((void)0) +#define dev_err(...) ((void)0) +#define le16_to_cpu(x) (x) +#define le32_to_cpu(x) (x) +#define do_div(n,d) ((n)/=(d)) +#define clamp(v,lo,hi) ((v)<(lo)?(lo):(v)>(hi)?(hi):(v)) +#define GMU_MAX_BCMS 3 +'''+tcs+'\n'+(cache/names[3]).read_text()+'\n'+bcm+'\n'+db+''' +struct device {int unused;};struct adreno_gpu {int unused;}; +struct a6xx_info {const struct a6xx_bcm *bcms;}; +struct a6xx_gmu {struct device *dev;unsigned nr_gpu_bws;u32 gpu_bw_table[16],gpu_ib_votes[16][3];}; +static int adreno_is_a7xx(const struct adreno_gpu *g){(void)g;return 1;} +static int adreno_is_a8xx(const struct adreno_gpu *g){(void)g;return 0;} +static const u32 *opp_levels;static u16 rails[2][256];static size_t rail_sizes[2]; +static struct bcm_db aux[3];static u32 addrs[3],cn0; +static const void *cmd_db_read_aux_data(const char *name,size_t *count) { + if(name[0]=='p' || name[0]=='s'){unsigned i=name[0]=='s';*count=rail_sizes[i];return rails[i];} + unsigned i=(unsigned)(name[0]-'0');*count=sizeof(aux[i]);return aux+i; +} +static u32 cmd_db_read_addr(const char *name){return name[0]=='C'?cn0:addrs[name[0]-'0'];} +static unsigned a6xx_gmu_get_arc_level(struct device *dev,unsigned long freq){(void)dev;return opp_levels[freq];} +''' +wrapper=''' +int source_arc(const u32 *levels,unsigned count,const u8 *p,size_t pn,const u8 *s,size_t sn,u32 *votes) { + unsigned long freq[16];opp_levels=levels;rail_sizes[0]=pn;rail_sizes[1]=sn; + memcpy(rails[0],p,pn);memcpy(rails[1],s,sn);for(unsigned i=0;i +#include +#include +typedef uint32_t u32;typedef uint64_t u64;typedef int32_t s32;typedef uint8_t u8; +#define __packed __attribute__((packed)) +#define HFI_MAX_QUEUES 3 +#define HFI_HISTORY_SZ 8 +#define SZ_4K 4096 +#define ARRAY_SIZE(a) (sizeof(a)/sizeof((a)[0])) +#define CIRC_SPACE(h,t,s) (((t)-(h)-1)&((s)-1)) +#define READ_ONCE(v) (v) /* single-thread software oracle only */ +#define WRITE_ONCE(v,x) ((v)=(x)) +#define spin_lock_init(v) ((void)(v)) +#define spin_lock(v) ((void)(v)) +#define spin_unlock(v) ((void)(v)) +#define atomic_set(v,x) (*(v)=(x)) +#define dma_mb() __sync_synchronize() +#define WARN_ON(x) ((void)(x)) +#define REG_A6XX_GMU_HOST2GMU_INTR_SET 0 +#define gmu_write(g,r,v) ((void)(g),(void)(r),(void)(v)) +''' +standins=''' +struct a6xx_hfi_queue {struct a6xx_hfi_queue_header *header;u32 *data;int lock,seqnum;s32 history[8];u8 history_idx;}; +struct a6xx_gmu_bo {void *virt;u64 iova;}; +struct a6xx_gmu {struct a6xx_gmu_bo hfi;struct a6xx_hfi_queue queues[3];int legacy;}; +''' +wrappers=''' +void source_initialize(u32 *shared,u32 iova) { + struct a6xx_gmu g={0};g.hfi.virt=shared;g.hfi.iova=iova;a6xx_hfi_init(&g); +} +int source_write(u32 *shared,u32 iova,u32 *message,u32 count) { + struct a6xx_gmu g={0};g.hfi.virt=shared;g.hfi.iova=iova; + g.queues[0].header=(void *)(shared+6);g.queues[0].data=shared+1024; + return a6xx_hfi_queue_write(&g,&g.queues[0],message,count); +} +''' +oracle=out/'original-hfi-oracle.c' +oracle.write_text(prelude+'\n'.join(wire)+standins+'\n'.join([ + function('static int a6xx_hfi_queue_write('),function('static void a6xx_hfi_queue_init('),function('void a6xx_hfi_init(')])+wrappers) +test=out/'test-hfi' +subprocess.run(['xcrun','clang','-std=gnu11','-O2','-Wall','-Wextra','-Werror','-fsanitize=address,undefined','-DSOURCE_ORACLE',kit/'AdrenoHfi.c',kit/'test_hfi.c',oracle,'-o',test],check=True,timeout=60) +result=subprocess.run([test],check=True,capture_output=True,text=True,timeout=30);print(result.stdout,end='') +if '510 modern HFI wrapped command/response cases, 4100 sequence exchanges' not in result.stdout:raise SystemExit('Actual bounded coverage differs') +flags=['-target','arm64-apple-ios17.0','-O2','-Wall','-Wextra','-Werror','-Wno-incompatible-sysroot','-ffreestanding','-mgeneral-regs-only'] +obj=out/'AdrenoHfi.o';subprocess.run(['xcrun','clang',*flags,'-std=c11','-c',kit/'AdrenoHfi.c','-o',obj],check=True,timeout=30) +raw=obj.read_bytes();info=macho(raw);version=next(at for command,at,_ in info['commands'] if command==0x32) +if struct.unpack_from('<4I',raw)!=(0xfeedfacf,0x100000c,0,1) or struct.unpack_from('> 14 & 3 == 0 and r['TCR_EL1'] >> 30 & 3 == 2 and + r['SCTLR_EL1'] & 1 and r['TTBR0_EL1'] and r['TTBR1_EL1'] and + d['original_crypto_initialized_observed'] and d['original_crypto_table_owner_observed'], + 'Bounded actual kernel execution with active native-4K hardware translation') + state = d['native_adreno_pm4'] + require(not state['gpu_execution_qualified'] and not state['submission_authorized'], 'Do not promote structural planning into real GPU submission') + if label == 'native': + require(d['stop_symbol'] == '_native_adreno_pm4_checkpoint' and state['requested'] and state['status'] == 0 and + state['packets'] == 3 and state['refusal_mask'] == 63 and state['bit_refusals'] == 64 and + state['input_preserved'] == state['output_preserved'] == 1, + 'Actual packet plans, six refusal classes, 64 bit corruptions and unchanged buffers') + else: + require(not state['requested'] and state['status'] == -1 and + all(state[name] == 0 for name in ('packets', 'refusal_mask', 'output_preserved', 'bit_refusals', 'input_preserved')) and + d['original_launchd_main_executed'] and d['followed_main_entry']['first_instruction_completed'] and + int(d['pstate'], 16) & 31 == 0 and d['direct_user_commpage_shifts'] == {'fffff4025:1': 12, 'fffff4037:1': 12} and + d['owner_root_binding_audit']['boot_root_accepted'] and d['owner_root_binding_audit']['_native_owner_root_apple_autherr'] == 80 and + d['original_amfi_table_owner_observed'] and d['actual_signature_failures'] == + dict(_cs_validate_page_bad_hash=0, _cs_validate_page_no_hash=0) and + d['native_virtio_integrity']['_native_virtio_integrity_failures'] == 0, + 'Default-off preserves original EL0 launchd, native user 4K, real owner binding and original signing enforcement') +print('PASS: native-XNU PM4 mixed plans, six refusal classes, 64 header corruptions and unchanged input/output') +print('PASS: default-off original launchd, native user 4K and signatures; GPU execution and Metal remain unqualified') diff --git a/scripts/check-adreno-pm4.py b/scripts/check-adreno-pm4.py new file mode 100644 index 0000000..7fd4f04 --- /dev/null +++ b/scripts/check-adreno-pm4.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +"""Qualify native PM4 structure handling; this is not a GPU execution test.""" +import hashlib +import json +from pathlib import Path +import struct +import subprocess +import sys + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'kits/NativePrelink')) +from native_prelink import macho + +subprocess.run([sys.executable, ROOT / 'scripts/fetch-adreno-inputs.py'], check=True) +kit = ROOT / 'kits/AdrenoPM4' +out = ROOT / 'build/adreno-pm4' +out.mkdir(parents=True, exist_ok=True) +test = out / 'test-pm4' +subprocess.run(['xcrun', 'clang', '-std=c11', '-O2', '-Wall', '-Wextra', '-Werror', + '-fsanitize=address,undefined', kit / 'AdrenoPM4.c', kit / 'test_pm4.c', '-o', test], check=True) +result = subprocess.run([test], capture_output=True, text=True, timeout=30, check=True) +print(result.stdout, end='') +obj = out / 'AdrenoPM4.o' +subprocess.run(['xcrun', 'clang', '-target', 'arm64-apple-ios17.0', '-std=c11', '-O2', + '-Wall', '-Wextra', '-Werror', '-Wno-incompatible-sysroot', '-ffreestanding', '-c', + kit / 'AdrenoPM4.c', '-o', obj], check=True) +data = obj.read_bytes() +image = macho(data) +version = next(at for cmd, at, _ in image['commands'] if cmd == 0x32) +if image['kind'] != 1 or struct.unpack_from('');end=raster.index('\nFD_GENX(__fd6_setup_rasterizer_stateobj);',start) +actual_function=raster[start:end]+'\n' +reference=OUT/'reference.cpp' +reference.write_text('''#include +#include +#include +#include "pipe/p_state.h" +#include "util/half_float.h" +#include "util/u_math.h" +#include "AdrenoRaster.h" +#include "adreno_common.xml.h" +#include "adreno_pm4.xml.h" +#include "a6xx_enums.xml.h" +#include "a7xx_enums.xml.h" +#include "a8xx_enums.xml.h" +#include "a6xx.xml.h" +struct fd_reg_pair {uint32_t reg;uint64_t value;void *bo;bool is_address;uint32_t bo_offset,bo_shift,bo_low;}; +#define __bo_type void * +#include "a6xx-pack.xml.h" +// Only the transport is replaced: collect actual register/value pairs. +struct fd_ringbuffer {unsigned count;uint32_t reg[64],value[64];}; +struct ref_props {bool is_a702,has_legacy_pipeline_shading_rate;}; +struct ref_info {ref_props props;};struct ref_screen {ref_info *info;}; +struct fd_context {fd_ringbuffer *pipe;ref_screen *screen;}; +struct fd_crb {fd_ringbuffer *ring; + fd_crb(fd_ringbuffer *r,unsigned) : ring(r) {ring->count=0;} + fd_crb &add(fd_reg_pair pair) {assert(ring->count<64 && !pair.bo && !pair.is_address && pair.value<=UINT32_MAX);unsigned n=ring->count++;ring->reg[n]=pair.reg;ring->value[n]=(uint32_t)pair.value;return *this;} + operator fd_ringbuffer *() {return ring;} +}; +#define assert_dt +'''+function(sources['src/gallium/auxiliary/util/u_inlines.h'],'util_get_min_point_size')+ + function(sources['src/gallium/drivers/freedreno/freedreno_state.h'],'fd_rast_depth_clamp_enabled')+actual_function+''' +static float raw_float(uint32_t bits) {float value;memcpy(&value,&bits,4);return value;} +extern "C" size_t AdrenoRasterReference(const AdrenoRasterState *,uint32_t *,uint32_t *,size_t); +extern "C" size_t AdrenoRasterReference(const AdrenoRasterState *s,uint32_t *regs,uint32_t *values,size_t capacity) { + pipe_rasterizer_state cso={};uint32_t f=s->flags; + cso.depth_clip_near=f&AdrenoRasterClipNear;cso.depth_clip_far=(f&AdrenoRasterClipFar)!=0; + cso.clip_halfz=(f&AdrenoRasterHalfZ)!=0;cso.cull_face=((f&AdrenoRasterCullFront)?PIPE_FACE_FRONT:0)|((f&AdrenoRasterCullBack)?PIPE_FACE_BACK:0); + cso.front_ccw=(f&AdrenoRasterFrontCCW)!=0;cso.offset_tri=(f&AdrenoRasterOffsetTriangles)!=0; + cso.multisample=(f&AdrenoRasterMultisample)!=0;cso.flatshade_first=(f&AdrenoRasterFlatFirst)!=0; + cso.point_size_per_vertex=(f&AdrenoRasterPointSizePerVertex)!=0; + cso.point_quad_rasterization=(f&AdrenoRasterPointQuad)!=0;cso.point_smooth=(f&AdrenoRasterPointSmooth)!=0; + cso.fill_front=s->polygon_mode==1?PIPE_POLYGON_MODE_POINT:s->polygon_mode==2?PIPE_POLYGON_MODE_LINE:PIPE_POLYGON_MODE_FILL; + cso.point_size=s->point_size_q4/16.0f;cso.line_width=s->line_width_q1/2.0f; + cso.offset_scale=raw_float(s->offset_scale_bits);cso.offset_units=raw_float(s->offset_units_bits);cso.offset_clamp=raw_float(s->offset_clamp_bits); + fd_ringbuffer ring={};fd_context ctx={&ring,nullptr}; + assert(__fd6_setup_rasterizer_stateobj(&ctx,&cso,(f&AdrenoRasterPrimitiveRestart)!=0)==&ring); + assert(ring.count<=capacity);memcpy(regs,ring.reg,ring.count*4);memcpy(values,ring.value,ring.count*4);return ring.count; +} +''') +build=ROOT/'build/mesa-ir3' +commands=run(['ninja','-C',build,'-t','commands','src/freedreno/ir3/air_nir'],text=True).splitlines() +base=next(shlex.split(x) for x in commands if shlex.split(x)[-1]==str(ROOT/'kits/AirNir/MesaBackend.c')) +for flag in ('-MQ','-MF'): + i=base.index(flag);del base[i:i+2] +base=[x for x in base if x not in ('-MD',)] +base[1:1]=['-DMESA_REFERENCE','-DFD_NO_DEPRECATED_PACK','-fsanitize=address,undefined','-I'+str(KIT),'-I'+str(OUT),'-I'+str(MESA/'src/gallium/include')] +objects=[] +for name,path in (('test',KIT/'test_raster.c'),('raster',KIT/'AdrenoRaster.c'),('pm4',KIT/'AdrenoPM4.c'),('reference',reference)): + cmd=base.copy();obj=OUT/(name+'.o');cmd[cmd.index('-o')+1]=str(obj);cmd[-1]=str(path) + if name=='reference': + cmd[0]=str(Path(cmd[0]).with_name('clang++'));cmd=[x for x in cmd if not x.startswith('-std=')];cmd[1:1]=['-std=c++20','-Wno-c99-designator','-Wno-c++11-narrowing','-Wno-missing-field-initializers'] + run(cmd,cwd=build);objects.append(obj) +exe=OUT/'test-reference';run(['xcrun','clang++','-fsanitize=address,undefined',*objects,'-o',exe]) +unit=run([exe],text=True).strip();print(unit) +require(unit=='PASS: 114916 raster profiles, 41 transactional refusals','Exact source-reference suite inventory') +import sys +import struct +sys.path.insert(0,str(ROOT/'kits/NativePrelink')) +from native_prelink import macho +sdk=run(['xcrun','--sdk','iphoneos','--show-sdk-path'],text=True).strip() +objects=[] +for name in ('AdrenoPM4','AdrenoConstants','AdrenoShader','AdrenoVPC','AdrenoRaster'): + obj=OUT/(name+'.ios.o') + run(['xcrun','clang','-target','arm64-apple-ios17.0','-isysroot',sdk,'-std=c11','-O2','-Wall','-Wextra','-Werror','-ffreestanding','-fstack-protector-strong','-c',KIT/(name+'.c'),'-o',obj]) + raw=obj.read_bytes();parsed=macho(raw) + require(parsed['kind']==1 and [struct.unpack_from('\n' + 'static_assert(sizeof(AdrenoConstantConfig)==16 && offsetof(AdrenoConstantConfig,stage)==8);\n' + 'static_assert(sizeof(AdrenoShaderState)==64 && sizeof(AdrenoShaderMapping)==40);\n' + 'extern "C" unsigned client(const AdrenoShaderState *s,const AdrenoShaderMapping *m,const AdrenoConstantConfig *c,const AdrenoConstantSpan *p,const unsigned char *b,unsigned *o){auto a=AdrenoA740ShaderBind(s,m,o,24);auto t=AdrenoA740Constants(c,p,1,b,16,o+24,8);return a.status|t.status;}\n') +cpp_obj=OUT/'client.ios.o' +run(['xcrun','clang++','-target','arm64-apple-ios17.0','-std=c++17','-O2','-Wall','-Wextra','-Werror','-Wno-incompatible-sysroot', + '-ffreestanding','-fno-exceptions','-fno-rtti','-I',KIT,'-c',cpp,'-o',cpp_obj]) +if run(['xcrun','nm','-u',cpp_obj],text=True).split()!=['_AdrenoA740Constants','_AdrenoA740ShaderBind']:raise SystemExit('Native shader C++ ABI differs') +# Link the C++ consumer and all command helpers into a real target-native +# dylib. One archive avoids duplicated PM4 objects under whole-archive loading. +sdk=run(['xcrun','--sdk','iphoneos','--show-sdk-path'],text=True).strip() +native=OUT/'AdrenoCommands.ios.dylib' +run(['xcrun','clang','-target','arm64-apple-ios17.0','-isysroot',sdk,'-dynamiclib',cpp_obj,archive, + '-Wl,-install_name,@rpath/AdrenoCommands.ios.dylib','-Wl,-undefined,error','-o',native]) +parsed_native=macho(native.read_bytes()) +version=next(at for cmd,at,_ in parsed_native['commands'] if cmd==0x32) +if parsed_native['kind']!=6 or struct.unpack_from('>8&0x3ffff + if header>>28!=4 or count not in (1,2):raise SystemExit('Unexpected A6xx preload/opcode packet') + for i in range(count): + if reg+i in decoded:raise SystemExit('Duplicate shader register') + decoded[reg+i]=output[at+1+i] + at+=1+count + stage='PS' if isfrag else 'VS';ctrl='SP_'+stage+'_CNTL_0' + control=field(ctrl,'FULLREGFOOTPRINT',abi['max_full_register']+1)|field(ctrl,'HALFREGFOOTPRINT',abi['max_half_register']+1)|field(ctrl,'BRANCHSTACK',b['branch_stack_hw']) + for name,value in [('MERGEDREGS',b['merged_registers']),('EARLYPREAMBLE',abi['early_preamble'])]:control|=field(ctrl,name,int(value)) + if isfrag: + for name,value in [('THREADSIZE',int(abi['double_threadsize'])),('VARYING',int(b['input_words']!=0)),('LODPIXMASK',int(b['need_full_quad'])),('INOUTREGOVERLAP',1),('PIXLODENABLE',int(b['need_pixel_lod']))]:control|=field(ctrl,name,value) + # Other required register values follow the exact pinned no-private-memory + # fd6_emit_shader path. Pipeline/config/linkage state is separate work. + expected={offset('SP_MODE_CNTL'):field('SP_MODE_CNTL','CONSTANT_DEMOTION_ENABLE',1)|field('SP_MODE_CNTL','ISAMMODE',enums['ISAMMODE_GL']), + offset('TPL1_MODE_CNTL'):field('TPL1_MODE_CNTL','ISAMMODE',enums['ISAMMODE_GL'])|field('TPL1_MODE_CNTL','NEARESTMIPSNAP',enums['CLAMP_ROUND_TRUNCATE'])|field('TPL1_MODE_CNTL','DESTDATATYPEOVERRIDE',1), + offset(ctrl):control,offset('SP_'+stage+'_INSTR_SIZE'):abi['instruction_groups'],offset('SP_'+stage+'_PROGRAM_COUNTER_OFFSET'):0, + offset('SP_'+stage+'_BASE'):view.base&0xffffffff,offset('SP_'+stage+'_BASE')+1:view.base>>32, + offset('SP_'+stage+'_PVT_MEM_PARAM'):0,offset('SP_'+stage+'_PVT_MEM_BASE'):0,offset('SP_'+stage+'_PVT_MEM_BASE')+1:0, + offset('SP_'+stage+'_PVT_MEM_SIZE'):0,offset('SP_'+stage+'_PVT_MEM_STACK_OFFSET'):0,offset('SP_'+stage+'_VGS_CNTL'):0} + if decoded!=expected or at!=24 or bytes(output)[96:]!=b'\x6b'*(len(output)*4-96):raise SystemExit('Actual shader state differs from XML/source oracle') + encoded=bytes(output)[:96];(OUT/f'{library}-{index}.pm4').write_bytes(encoded) + results.append(dict(library=library,index=index,stage=abi['stage'],binary_sha256=sha(binary),abi_sha256=sha(prefix.with_suffix('.ir3.json').read_bytes()), + command_sha256=sha(encoded),command_words=24,registers=13,input_words=b['input_words'],native_page_shift=12,synthetic_inactive_resource_view=True)) +(OUT/'report.json').write_text(json.dumps(dict(mesa_revision=REV,source_documents=manifest['files'],unit=unit_result,original_shaders=results, + source_sha256={p.name:sha(p.read_bytes()) for p in [KIT/"AdrenoPM4.c",KIT/"AdrenoShader.c",KIT/"AdrenoShader.h",KIT/"test_shader.c",ROOT/"kits/AirNir/BackendABI.h"]}, + native_objects={p.name:sha(p.read_bytes()) for p in objects},native_imports=imports,native_archive_sha256=sha(archive.read_bytes()), + native_cpp_sha256=sha(cpp_obj.read_bytes()),native_linked_module_sha256=sha(native.read_bytes()), + native_linked_imports=native_imports,ordinary_arm64_ios17=True, + shader_mapping_or_smmu_activated=False,submission_authorized=False,shader_executed=False,metal_pipeline_or_ui=False),indent=2)+'\n') +print('PASS: 22 actual IR3 vertex/fragment binding states, complete XML register map and native ARM64/iOS17 C/C++ archive + real linked module; no A6xx preload or GPU execution claim') diff --git a/scripts/check-air-fragment-inputs.py b/scripts/check-air-fragment-inputs.py new file mode 100644 index 0000000..6e5b4d7 --- /dev/null +++ b/scripts/check-air-fragment-inputs.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +"""Qualify original smooth-center fragment inputs and post-interpolation arithmetic.""" +import argparse +import ctypes as C +import hashlib +import json +import os +from pathlib import Path +import random +import re +import struct +import subprocess +ROOT=Path(__file__).resolve().parents[1] +p=argparse.ArgumentParser(description=__doc__);p.add_argument('--compiler',type=Path,required=True);p.add_argument('--llvm-bindir',type=Path,required=True);a=p.parse_args() +OUT=ROOT/'build/air-nir/fragment-inputs';OUT.mkdir(parents=True,exist_ok=True) +env={k:v for k,v in os.environ.items() if not k.startswith(('IR3_','FD_MESA_','NIR_','MESA_SHADER_'))};env['IR3_SHADER_DEBUG']='asmroundtrip' +def run(cmd,status=0): + r=subprocess.run([str(v) for v in cmd],capture_output=True,text=True,env=env,timeout=20) + if r.returncode!=status:raise RuntimeError(f'Expected {status}, got {r.returncode}: '+r.stderr[-1500:]) + return r +def require(value,why): + if not value:raise RuntimeError(why) +def sha(path):return hashlib.sha256(path.read_bytes()).hexdigest() +reports=[] +for index in (65,67,75,153,155): + src=ROOT/f'build/gpu-analysis/air/QuartzCore/module-{index}.bc';prefix=OUT/f'QuartzCore-{index}' + probe=run([a.compiler,src,'0xffff43050a01',prefix,'--compile']);require(json.loads(probe.stdout)['asm_roundtrip'],'Actual assembler roundtrip') + interface=json.loads(prefix.with_suffix('.interface.json').read_text());abi=json.loads(prefix.with_suffix('.ir3.json').read_text()) + inputs=[i for i in interface['inputs'] if i['kind']=='air.fragment_input'];unused=[i for i in interface['inputs'] if i['kind']=='air.position'] + require(len(unused)==1 and unused[0]['unused'] and unused[0]['components']==4,'Only unused position is accepted') + text=src.with_suffix('.ll').read_text();identities=re.findall(r'!"air.fragment_input", !"([^"]+)"',text) + require([i['identity'] for i in inputs]==identities and [i['index'] for i in inputs]==list(range(len(inputs))) and all(i['interpolation']=='perspective' and i['sampling']=='center' for i in inputs),'Exact original varying identities, dense slots and qualifiers') + actual={i['slot']:i for i in abi['inputs'] if not i['system_value']} + require(set(actual)=={32+i['index'] for i in inputs} and all(actual[32+i['index']]['component_mask']==(1<>23)&255==255:bits^=1<<23 + words.append(bits) + values.append(words);frames.append(struct.pack(' %0, i64 0\n %7 = tail call fast float @air.fast_saturate.f32(float %p)'), + 'declared-used-position':original.replace(', !"air.arg_unused"',''), + 'half-input':original.replace('float %2, float %3','half %2, float %3').replace('@air.fast_saturate.f32(float %2)','@air.fast_saturate.f32(float 5.000000e-01)'), +} +for name,text in mutations.items(): + require(text!=original,'Mutation must change input '+name) + ll,bc,prefix=OUT/(name+'.ll'),OUT/(name+'.bc'),OUT/name;ll.write_text(text);run([a.llvm_bindir/'llvm-as',ll,'-o',bc]);prefix.with_suffix('.bin').unlink(missing_ok=True) + run([a.compiler,bc,'0xffff43050a01',prefix,'--compile'],4);require(not prefix.with_suffix('.bin').exists(),'Refusal exported GPU code') +(OUT/'report.json').write_text(json.dumps(dict(originals=reports,refusals=len(mutations),gpu_execution=False,hardware_interpolation=False,cpu_rasterizer=False),indent=2)+'\n') +print('PASS:',len(mutations),'interpolation, identity, position and per-instruction math refusals; no GPU interpolation claim') diff --git a/scripts/check-air-fragment-outputs.py b/scripts/check-air-fragment-outputs.py new file mode 100644 index 0000000..8a5ec4d --- /dev/null +++ b/scripts/check-air-fragment-outputs.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +"""Qualify unchanged QuartzCore fragment colors, sparse targets and MRT identity.""" +import argparse +import ctypes as C +import hashlib +import json +import os +from pathlib import Path +import re +import struct +import subprocess +p=argparse.ArgumentParser(description=__doc__) +p.add_argument('--compiler', type=Path, required=True) +p.add_argument('--llvm-bindir', type=Path, required=True) +a=p.parse_args() +ROOT=Path(__file__).resolve().parents[1] +OUT=ROOT/'build/air-nir/fragments';OUT.mkdir(parents=True,exist_ok=True) +env=dict(os.environ,IR3_SHADER_DEBUG='asmroundtrip') + +def run(cmd, status=0): + r=subprocess.run([str(v) for v in cmd],env=env,capture_output=True,text=True,timeout=15) + if r.returncode!=status:raise RuntimeError(f'Expected {status}, got {r.returncode}: {r.stderr[-1200:]}') + return r + +def sha(path):return hashlib.sha256(path.read_bytes()).hexdigest() + +def evaluate(src,prefix,expected_targets): + # Evaluate exact original/module-copy IR and NIR independently; no rasterizer. + interface=prefix.with_suffix('.interface.json');abi=prefix.with_suffix('.ir3.json') + case=prefix.with_suffix('.cases');case.write_bytes(bytes(4+16*4*4)) + r=run([a.compiler,src,'0x43050a01',prefix,'--compile','--test-resources',case]) + if not json.loads(r.stdout)['asm_roundtrip']:raise RuntimeError('Missing actual encoding roundtrip') + schema=json.loads(interface.read_text());device=json.loads(abi.read_text()) + if schema['stage']!='fragment' or schema['inputs'] or device['stage']!='fragment' or device['writes_position']: + raise RuntimeError('Actual fragment stage/interface differs') + if [(v['target'],v['components']) for v in schema['outputs']]!=expected_targets: + raise RuntimeError('Logical output identity/width differs') + # NIR store location must follow the source target, not the dense return index. + nir=prefix.with_suffix('.nir').read_text();asm=prefix.with_suffix('.asm').read_text() + for target,components in expected_targets: + if f'FRAG_RESULT_DATA{target}' not in asm or f'VARYING_SLOT_POS' in nir: + raise RuntimeError('Actual backend target mapping differs') + actual={v['slot']:v for v in device['outputs']} + if set(actual)!={4+target for target,_ in expected_targets}: + raise RuntimeError('IR3 allocated output targets differ from source') + for target,components in expected_targets: + # IR3 allocated mask may contain only requested components. + if actual[4+target]['register_id']==252:raise RuntimeError('No actual color output register') + oracle=prefix.with_suffix('.oracle.bc') + run([a.compiler,src,'0x43050a01',oracle,'--test-resource-oracle']) + dylib=prefix.with_suffix('.dylib') + run([a.llvm_bindir/'clang','-O2','-dynamiclib',oracle,'-o',dylib]) + lib=C.CDLL(str(dylib));fn=lib.air_test_resource_oracle + fn.argtypes=[C.c_void_p,C.c_void_p,C.c_uint32,C.POINTER(C.c_float)] + expected=(C.c_float*(len(expected_targets)*4))();fn(None,None,0,expected) + if prefix.with_suffix('.eval').read_bytes()!=bytes(expected):raise RuntimeError('Original IR versus NIR bitwise color mismatch') + return dict(binary_sha256=sha(prefix.with_suffix('.bin')),abi_sha256=sha(abi),binary_bytes=device['binary_bytes'], + instructions=device['main_instructions'],targets=expected_targets,original_ir_nir_match=True,gpu_executed=False) + +audit=json.loads((ROOT/'build/gpu-analysis/air/target-inputs.json').read_text()) +originals=[] +for index,targets in [(69,[(2,1)]),(73,[(0,4),(1,4)]),(157,[(2,1)])]: + src=ROOT/f'build/gpu-analysis/air/QuartzCore/module-{index}.bc' + entry=next(v for v in audit['modules'] if v['library']=='QuartzCore' and v['index']==index) + if sha(src)!=entry['bitcode_sha256']:raise RuntimeError('Supplied original input changed') + result=evaluate(src,OUT/f'QuartzCore-{index}',targets) + originals.append(dict(library='QuartzCore',index=index,source_sha256=sha(src),**result)) +# Authored variants preserve target triple/version/options and mutate only ignored +# copies. They prevent all-zero original clears from hiding channel/target swaps. +scalar=(ROOT/'build/gpu-analysis/air/QuartzCore/module-69.ll').read_text() +mrt=(ROOT/'build/gpu-analysis/air/QuartzCore/module-73.ll').read_text() +synthetic=[] +for target in range(8): + text=scalar.replace('ret <{ float }> zeroinitializer','ret <{ float }> <{ float -1.250000e+00 }>') + text=text.replace('!"air.render_target", i32 2, i32 0','!"air.render_target", i32 '+str(target)+', i32 0') + ll=OUT/f'target-{target}.ll';ll.write_text(text);bc=ll.with_suffix('.bc') + run([a.llvm_bindir/'llvm-as',ll,'-o',bc]);synthetic.append(evaluate(bc,OUT/f'target-{target}',[(target,1)])) +text=mrt.replace('ret <{ <4 x float>, <4 x float> }> zeroinitializer', + 'ret <{ <4 x float>, <4 x float> }> <{ <4 x float> , <4 x float> }>') +ll=OUT/'mrt-distinct.ll';ll.write_text(text);bc=ll.with_suffix('.bc');run([a.llvm_bindir/'llvm-as',ll,'-o',bc]) +synthetic.append(evaluate(bc,OUT/'mrt-distinct',[(0,4),(1,4)])) +mutations={ + 'target-out-of-range':scalar.replace('!"air.render_target", i32 2, i32 0','!"air.render_target", i32 8, i32 0'), + 'dual-source':scalar.replace('!"air.render_target", i32 2, i32 0','!"air.render_target", i32 2, i32 1'), + 'duplicate-target':mrt.replace('!"air.render_target", i32 1, i32 0','!"air.render_target", i32 0, i32 0'), + 'depth-kind':scalar.replace('!"air.render_target"','!"air.depth"'), + 'undefined-scalar':scalar.replace('ret <{ float }> zeroinitializer','ret <{ float }> poison'), + 'undefined-vector':mrt.replace('ret <{ <4 x float>, <4 x float> }> zeroinitializer','ret <{ <4 x float>, <4 x float> }> poison'), + 'half-output':mrt.replace('float','half'), + 'mixed-stage':scalar.replace('!air.fragment = !{!15}','!air.fragment = !{!15}\n!air.vertex = !{!15}'), + 'missing-stage':scalar.replace('!air.fragment = !{!15}',''), + 'wrong-stage':scalar.replace('!air.fragment = !{!15}','!air.vertex = !{!15}'), +} +refusals=[] +for name,text in mutations.items(): + if text in (scalar,mrt):raise RuntimeError('No-op negative mutation') + ll=OUT/(name+'.ll');ll.write_text(text);bc=ll.with_suffix('.bc');run([a.llvm_bindir/'llvm-as',ll,'-o',bc]) + r=run([a.compiler,bc,'0x43050a01',OUT/name,'--compile'],status=4) + if not r.stderr.startswith('AIR refused:'):raise RuntimeError('Expected explicit frontend refusal') + refusals.append(dict(name=name,reason=r.stderr.strip())) +(OUT/'report.json').write_text(json.dumps(dict(originals=originals,authored_color_controls=synthetic, + refusal_controls=refusals,original_inventory_unchanged=True,actual_rasterization=False, + fragment_interpolation=False,texture_sampling=False,metal_pipeline_linked=False),indent=2)+'\n') +print('PASS: three original scalar/MRT fragment shaders, all eight sparse targets and distinct MRT channels, actual roundtrip/IR3 targets and original-IR/NIR bits; ten refusals') diff --git a/scripts/check-air-library-compiler.py b/scripts/check-air-library-compiler.py new file mode 100644 index 0000000..c1a1d77 --- /dev/null +++ b/scripts/check-air-library-compiler.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +"""Compile directly from unchanged target metallibs and test entry/integrity refusals.""" +import hashlib +import json +import os +from pathlib import Path +import struct +import subprocess +ROOT=Path(__file__).resolve().parents[1];OUT=ROOT/'build/air-library/compiler' +OUT.mkdir(parents=True,exist_ok=True);exe=ROOT/'build/mesa-ir3/src/freedreno/ir3/air_nir' +manifest=json.loads((ROOT/'scripts/data/native-air-diagnostic-inputs.json').read_text());audit=json.loads((ROOT/'build/gpu-analysis/air/target-inputs.json').read_text()) +paths={Path(x['file']).parent.stem:ROOT/'.cache/firmware/target-volumes/os-mount/System/Library'/x['file'] for x in audit['inputs']} +env={k:v for k,v in os.environ.items() if not k.startswith(('IR3_','FD_MESA_','NIR_','MESA_SHADER_'))};env['IR3_SHADER_DEBUG']='asmroundtrip' +def run(path,index,prefix,expected=0): + p=subprocess.run([str(exe),str(path),manifest['chip_id'],str(prefix),'--compile-library',str(index)],capture_output=True,text=True,env=env,timeout=15) + if p.returncode!=expected:raise SystemExit(f'Unexpected compiler result {p.returncode}: '+p.stderr[-2000:]) + if expected and prefix.with_suffix('.bin').exists():raise SystemExit('Refused library exported GPU code') + return p +reports=[] +for case in manifest['cases']: + prefix=OUT/case['name'];p=run(paths[case['library']],case['index'],prefix) + for suffix,key in (('.bin','binary_sha256'),('.ir3.json','abi_sha256'),('.interface.json','interface_sha256')): + if hashlib.sha256(prefix.with_suffix(suffix).read_bytes()).hexdigest()!=case[key]:raise SystemExit('Direct library compilation changed '+case['name']+suffix) + meta=json.loads(p.stdout.strip().splitlines()[-1]) + if not meta['asm_roundtrip'] or meta['gpu_executed']:raise SystemExit('Actual compiler/assembler evidence required') + reports.append(dict(name=case['name'],binary_sha256=case['binary_sha256'],complete_abi_and_interface_exact=True)) + print('PASS:',case['name'],'direct unchanged metallib -> exact A740 binary/ABI/interface',flush=True) +# Independent bounded record walk obtains the exact original NAME/TYPE/HASH +# fields. Metadata contradictions must fail after container hash validation. +raw=paths['QuartzCore'].read_bytes();slice_offset=struct.unpack_from('>I',raw,16)[0];start=struct.unpack_from('AIR_MAX: return 1 + digest=hashlib.sha256(c.string_at(data,n)).digest();c.memmove(out,digest,32);return 0 +AIR_MAX=64*1024*1024 +lib=c.CDLL(str(OUT/'library-host.dylib'));lib.air_library_open.argtypes=[c.c_void_p,c.c_size_t,Hash,c.c_void_p,c.POINTER(View)];lib.air_library_open.restype=c.c_uint32 +lib.air_library_get.argtypes=[c.POINTER(View),c.c_uint32,c.POINTER(Function)];lib.air_library_get.restype=c.c_uint32 +reference=json.loads((ROOT/'build/gpu-analysis/air/target-inputs.json').read_text());reports=[];functions=[] +for item in reference['inputs']: + path=ROOT/'.cache/firmware/target-volumes/os-mount/System/Library'/item['file'];raw=path.read_bytes();require(sha(path)==item['sha256'],'Exact unchanged original metallib') + backing=c.create_string_buffer(raw);base=c.addressof(backing);view=View() + require(lib.air_library_open(base,len(raw),hash_bytes,None,c.byref(view))==0,'Actual original container validates') + expected=[x for x in reference['modules'] if x['library']==Path(item['file']).parent.stem] + require(view.count==len(expected)==item['functions'] and view.slice_offset==item['selected_slice']['offset'] and view.slice_bytes==item['selected_slice']['bytes'],'Actual whole-library slice and function inventory') + for module in expected: + f=Function();require(lib.air_library_get(c.byref(view),module['index'],c.byref(f))==0,'Every actual function lookup') + for ptr,n in ((f.name,f.name_bytes+1),(f.bitcode,f.bitcode_bytes),(f.wrapper,f.wrapper_bytes),(f.wrapper_sha256,32)):require(base<=ptr and ptr+n<=base+len(raw),'Every returned pointer is within the actual original backing') + require(c.string_at(f.name,f.name_bytes).decode()==module['name'] and f.index==module['index'] and f.stage==module['type'],'Actual index/name/stage match independent AIR audit') + bc=hashlib.sha256(c.string_at(f.bitcode,f.bitcode_bytes)).hexdigest();wrapper=hashlib.sha256(c.string_at(f.wrapper,f.wrapper_bytes)).hexdigest() + require(bc==module['bitcode_sha256'] and wrapper==module['wrapper_sha256']==c.string_at(f.wrapper_sha256,32).hex(),'Every complete original bitcode/wrapper/HASH byte identity') + functions.append(dict(library=module['library'],index=f.index,stage=f.stage,bitcode_sha256=bc,wrapper_sha256=wrapper)) + reports.append(dict(library=Path(item['file']).parent.stem,sha256=item['sha256'],count=view.count,slice_offset=view.slice_offset,slice_bytes=view.slice_bytes)) + print('PASS:',reports[-1]['library'],view.count,'actual original module names/stages/complete hashes') +sdk=subprocess.check_output(['xcrun','--sdk','iphoneos','--show-sdk-path'],text=True).strip() +run(['xcrun','clang','-target','arm64-apple-ios17.0','-isysroot',sdk,'-std=c11','-O2','-Wall','-Wextra','-Werror','-fstack-protector-strong','-c',KIT/'Library.c','-o',OUT/'Library.ios.o']) +sys.path.insert(0,str(ROOT/'kits/NativePrelink'));from native_prelink import macho +raw=(OUT/'Library.ios.o').read_bytes();parsed=macho(raw);import struct +require(parsed['kind']==1 and [struct.unpack_from('> 14 & 3 == 0 and r['TCR_EL1'] >> 30 & 3 == 2 and + r['SCTLR_EL1'] & 1 and r['TTBR0_EL1'] and r['TTBR1_EL1'], 'Active native 4K is required') + require(d['virtio_block_input']['read_only'] and d['virtio_extra_block_input']['read_only'], + 'Both real inputs must be read-only') + require(d['virtio_block_input']['source_sha256'] == + '3f99b859c9728379110b66657c7b15285bcb1925ad8b5726542e2cdcf8b1028d', 'Original OS identity differs') + require(d['virtio_extra_block_input']['first_sector_sha256'] == + '42b1178767bb27611cd1836cdd5779c48bfca8a3e719ec06dc4c8ef78d5e6387', 'Original raw container sector differs') + + +discovery = report('darwinboot-two-volume-discovery') +mounts = report('darwinboot-apfs-component-mounts') +refusal = report('darwinboot-original-apfs-cache-loader') +for d in (discovery, mounts, refusal): + native(d) + require(d['kernel_sha256'] == '8db7506164cb7772c6af3e8bd21b4a9ee3a951c5c611550deae923ed1558628f', + 'These checkpoints require the exact unchanged candidate') +for d in (discovery, mounts): + properties = {} + for e in d['registry_string_events']: + properties.setdefault(e['entry'], {})[e['key']] = e['value'] + names = {e['UUID']: e['BSD Name'] for e in properties.values() if 'UUID' in e and 'BSD Name' in e} + require(names.get('04769D26-2CA0-4825-B443-E2E5F88300F2') == 'disk3s1' and + names.get('8846FE78-2910-4297-AF97-D59BB6F8FC16') == 'disk2s1', + 'Use the actual same-entry UUID/BSD publication, not enumeration guesses') + require(d['native_user_4k_abi_observed'], 'The own process must retain its actual 4K ABI') +r = mounts['apfs_launchd_component'] +require(r['stage'] == 5 and all(r[k] == 0 for k in ('mkdir', 'os_mount', 'cache_mount', 'chroot', 'chdir')) and + all(r[k] >> 29 & 1 == 0 for k in ('mkdir_nzcv', 'os_mount_nzcv', 'cache_mount_nzcv', 'chroot_nzcv', 'chdir_nzcv')), + 'All five actual syscalls must succeed') +require(not refusal['bounded_timeout_interrupted'] and not refusal.get('followed_user_entry') and + not refusal['original_dyld_cache_loader_returned'] and not refusal['root_authentication_solved'], + 'The actual AMFI refusal must remain a refusal before original dyld') +require('AMFI: No PMGR?' in refusal['argument_memory']['x0'], 'Preserve the actual original AMFI panic') +require(refusal['kernel_observations']['_native_virtio_read_requests'] == 305 and + refusal['kernel_observations']['_native_virtio_read_failures'] == 0 and + refusal['kernel_observations']['_cs_validate_page_bad_hash'] == 0 and + refusal['kernel_observations']['_cs_validate_page_no_hash'] == 0, 'Real storage/signature state differs') +require([e['number'] for e in refusal['syscall_events']][-5:] == [167, 167, 61, 12, 59], + 'The real data mounts/chroot/chdir/exec sequence is required') +plan = json.loads((ROOT / 'build/native-prelink/native-apfs-data-candidate.json').read_text()) +kernel = (ROOT / 'build/native-prelink/native-apfs-data-candidate.macho').read_bytes() +require(hashlib.sha256(kernel).hexdigest() == refusal['kernel_sha256'], 'Candidate identity mismatch') +meta = np.macho(kernel) +original = (ROOT / '.cache/firmware/kernelcache.macho/kernelcache.release.ipad7.decompressed').read_bytes() +old = np.macho(original) +require(hashlib.sha256(original).hexdigest() == 'f18b4df865405f2a287b84977d7aca745e056d656d9fc9df1662970ce2969de7', + 'Use the actual ordinary-ARM64 supplied kernel') +for extent in plan['extents']: + if extent['segment'] != '__TEXT_EXEC': + continue + va, size = extent['va'], extent['size'] + newat = np.backed_offset(meta['segments'], va, size, len(kernel)) + oldat = np.backed_offset(old['segments'], va - plan['slide'], size, len(original)) + require(kernel[newat:newat + size] == original[oldat:oldat + size], 'Original executable bytes changed') +lr = int(refusal['x30'], 16) +require(any(e['module'] == 'com.apple.driver.AppleMobileFileIntegrity' and e['segment'] == '__TEXT_EXEC' and + e['va'] <= lr < e['va'] + e['size'] for e in plan['extents']), 'Panic caller must be genuine original AMFI') +print('PASS: actual UUID discovery, both read-only APFS mounts/chroot and genuine original AMFI PMGR refusal') +print('Original dyld/cache loading, production root authentication and hardware remain unqualified') diff --git a/scripts/check-arm-smmu-4k.py b/scripts/check-arm-smmu-4k.py new file mode 100644 index 0000000..aaec709 --- /dev/null +++ b/scripts/check-arm-smmu-4k.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +"""Qualify inactive native 4K SMMU tables; this is not a GPU execution test.""" +import hashlib +import json +from pathlib import Path +import struct +import subprocess +import sys + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'kits/NativePrelink')) +from native_prelink import macho + +subprocess.run([sys.executable, ROOT / 'scripts/fetch-adreno-inputs.py'], check=True) +kit = ROOT / 'kits/ArmSmmu4K' +out = ROOT / 'build/arm-smmu-4k' +out.mkdir(parents=True, exist_ok=True) +test = out / 'test-tables' +subprocess.run(['xcrun', 'clang', '-std=c11', '-O2', '-Wall', '-Wextra', '-Werror', + '-fsanitize=address,undefined', kit / 'ArmSmmu4K.c', kit / 'test_tables.c', '-o', test], check=True) +result = subprocess.run([test], capture_output=True, text=True, timeout=30, check=True) +print(result.stdout, end='') +obj = out / 'ArmSmmu4K.o' +subprocess.run(['xcrun', 'clang', '-target', 'arm64-apple-ios17.0', '-std=c11', '-O2', + '-Wall', '-Wextra', '-Werror', '-Wno-incompatible-sysroot', '-ffreestanding', '-c', + kit / 'ArmSmmu4K.c', '-o', obj], check=True) +data = obj.read_bytes() +image = macho(data) +version = next(at for cmd, at, _ in image['commands'] if cmd == 0x32) +if image['kind'] != 1 or struct.unpack_from(' +#include +#include +#include +#include +struct Failure{const char *format;}; +[[noreturn]] static void panic(const char *f,...){throw Failure{f};} +static std::vector pending,events;static size_t cursor; +volatile uint32_t armcpu_irq_phase,armcpu_irq_id,armcpu_quiesce_calls,armcpu_halt_calls; +volatile uint64_t armcpu_timer_interrupts,armcpu_ipi_interrupts; +static bool failClock; +static uint64_t ack(){assert(cursor({1,127,2,100}));assert(armcpu_irq_phase==0); + events.clear();cursor=0;pending.assign(128,27);pending.push_back(1023);cpu.interrupt();assert(events.size()==256&&armcpu_irq_phase==0); + events.clear();cursor=0;pending.assign(130,27);bool failed=false; + try{cpu.interrupt();}catch(Failure){failed=true;}assert(failed&&events.size()==256&&cursor==129); + events.clear();cursor=0;pending={99};failed=false;try{cpu.interrupt();}catch(Failure){failed=true;}assert(failed&&events.empty()); + events.clear();cursor=0;pending={27};failClock=true;failed=false;try{cpu.interrupt();}catch(Failure){failed=true;}assert(failed&&events==std::vector({1})&&armcpu_irq_phase==2);failClock=false; + events.clear();failed=false;try{cpu.quiesceCPU();}catch(Failure){failed=true;}assert(failed&&events.empty()); + failed=false;try{cpu.haltCPU();}catch(Failure){failed=true;}assert(failed&&events.empty()&&armcpu_halt_calls==1); + cpu.diagnosePower=false;cpu.quiesceCPU();assert(events==std::vector({202,300})); + puts("PASS: actual CPU method bodies, timer/IPI/EOI order, bounded flood, unexpected IRQ, callback failure, quiesce guards and legacy disable order; hardware I/O substituted"); +} +''' +(out/'test.cpp').write_text(prefix+body+main) +subprocess.run(['clang++','-std=c++17','-O1','-g','-Wall','-Wextra','-Werror','-Wno-unused-function','-fsanitize=address,undefined',str(out/'test.cpp'),'-o',str(out/'test')],check=True,timeout=60) +r=subprocess.run([str(out/'test')],check=True,capture_output=True,text=True,timeout=30) +(out/'report.json').write_text(json.dumps({'method_bodies_sha256':hashlib.sha256(original.encode()).hexdigest(),'passed':True,'privileged_io_substituted':True,'hardware_observed':False},indent=2)+'\n');print(r.stdout,end='') diff --git a/scripts/check-boot-progress-code.py b/scripts/check-boot-progress-code.py new file mode 100644 index 0000000..65d175d --- /dev/null +++ b/scripts/check-boot-progress-code.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +"""Inspect real ARM64 instructions; compile all hook callers with hooks disabled.""" +import hashlib +import json +from pathlib import Path +import re +import shutil +import subprocess + +root=Path(__file__).resolve().parents[1] +out=root/'build/ufs-root-diagnostics/progress/code' +out.mkdir(parents=True,exist_ok=True) +(out/'report.json').unlink(missing_ok=True) +objdump=shutil.which('llvm-objdump') or '/opt/homebrew/opt/llvm/bin/llvm-objdump' +nm=shutil.which('llvm-nm') or '/opt/homebrew/opt/llvm/bin/llvm-nm' +objects=[root/'build/boot-progress/BootProgress.o',root/'build/boot-progress/BootFaultProbe.o',root/'build/boot-progress/BootWatchdogPolicy.o',root/'build/native-ufs/NativeUfs.o',root/'build/native-legacy-display/BootUiTraceProbe.o'] +for obj in objects: + text=subprocess.check_output([objdump,'-d',str(obj)],text=True,timeout=30) + operands='\n'.join(re.findall(r'^[ \t]*[0-9a-f]+:[ \t]+[0-9a-f]{8}[ \t]+\w+[ \t]+([^\n]+)',text,re.M)) + assert not re.search(r'\b(?:[qds][0-9]+|v[0-9]+(?:\.[0-9a-z]+)?)\b',operands), 'Unexpected FP/SIMD in IRQ-reachable diagnostic' + assert not re.search(r'\bmsr\s+(?:TCR_EL1|TTBR[01]_EL1|SCTLR_EL1)',text), 'Observer object cannot directly modify translation' + (out/(obj.stem+'.asm')).write_text(text) + if obj.stem=='BootProgress': + assert len(re.findall(r'\bmrs\s+x\d+, CNTPCT_EL0',text))>=4, 'Actual counter reads are required, not just source asm strings' + assert re.search(r'\bmrs\s+x\d+, CNTFRQ_EL0',text) + assert re.search(r'msr\s+DAIFSet, #0xf\s*\n.*dsb\s+sy',text), 'Freeze must mask IRQs AND complete framebuffer stores' + assert 'wfe' in text +base=root/'build/xnu/ios-obj/DEVELOPMENT_ARM64_BCM2837' +checked=[] +for area,names in [('osfmk',['arm_init','startup','video_console','serial_console','sleh']), + ('bsd',['bsd_init','kern_exec','kern_exit']),('pexpert',['pe_kprintf'])]: + for name in names: + r=json.loads((base/area/'DEVELOPMENT'/(name+'.o.json')).read_text().strip().rstrip(',')) + cmd=[];a=r['arguments'];i=0;obj=out/(name+'.off.o') + while ipageSize != PAGE_SHIFT_4K\).*?\n\t\}', body, re.S) +if not match: + raise SystemExit('Cannot isolate the actual CodeDirectory page-size gate') +out = ROOT / 'build/target-userspace' +out.mkdir(parents=True, exist_ok=True) +program = out / 'codesign-granule-gate.c' +program.write_text('''#include +#include +#include +#include +#define PAGE_SHIFT_4K 12 +static int native_kernel_shift = NATIVE_TEST_PAGE_SHIFT; +#define PAGE_SHIFT native_kernel_shift +struct directory_probe { uint8_t pageSize; }; +int main(int argc, char **argv) { + if (argc != 2) return 2; + const struct directory_probe value = { .pageSize = (uint8_t)strtoul(argv[1], NULL, 10) }; + const struct directory_probe *cd = &value; +''' + match[0] + '\n return 0;\n}\n') +observed = [] +for cache_mode in (0, 1): + for kernel_shift in (12, 14): + binary = out / f'codesign-granule-{kernel_shift}-cache-{cache_mode}' + subprocess.run(['xcrun', 'clang', '-std=c11', '-Wall', '-Wextra', '-Werror', + f'-DNATIVE_TEST_PAGE_SHIFT={kernel_shift}', f'-DXNU_NATIVE_CS_HASH_CACHE={cache_mode}', + program, '-o', binary], check=True) + for signature_shift in (12, 14): + result = subprocess.run([binary, str(signature_shift)], capture_output=True, text=True) + accepted = result.returncode == 0 + expected = signature_shift == 12 or kernel_shift == 14 or cache_mode == 1 + if accepted != expected: + raise SystemExit('Unexpected behavior in the actual XNU page-size gate') + observed.append({'kernel_page_shift': kernel_shift, 'signature_hash_shift': signature_shift, + 'native_integrity_cache_enabled': bool(cache_mode), + 'accepted_by_actual_source_gate': accepted, 'exit_status': result.returncode}) +(out / 'codesign-granule-gate.json').write_text(json.dumps({ + 'source': 'Darwin 23 ubc_subr.c / cs_validate_codedirectory page-size gate only', + 'host_probe': True, 'complete_kernel_signature_validator_exercised': False, 'cases': observed}, indent=2) + '\n') +print('PASS: eight compiled actual gate cases distinguish native integrity-cache opt-in and original refusal behavior') +print('Gate acceptance is not complete integrity validation; no hardware page tables are changed by this host probe') diff --git a/scripts/check-constant-uploads-checkpoints.py b/scripts/check-constant-uploads-checkpoints.py new file mode 100644 index 0000000..d6a0447 --- /dev/null +++ b/scripts/check-constant-uploads-checkpoints.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 +"""Qualify the real native parameter helper, guards, refusals and default-off path.""" +import hashlib +import json +from pathlib import Path +import struct +import subprocess +import sys +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'kits/NativePrelink')) +from native_prelink import macho, exports, backed_offset + + +def require(value, message): + if not value: + raise SystemExit(message) + + +def sha(data): + return hashlib.sha256(data).hexdigest() + + +source_hash = 'f162b0a2c4fc0f45760a0fcaaf940d3039d6bb26310b79a2810ed16bf5c7e571' +candidate_hash = 'ac7c4b2ef5f427392b7a314939112e66151f815594ee5651eff8076a2d8bb0a1' +candidate = ROOT / 'build/native-prelink/native-owner-constants-outer-candidate.macho' +plan = json.loads(candidate.with_suffix('.json').read_text()) +image = candidate.read_bytes() +require(plan['native_sha256'] == source_hash and plan['candidate_sha256'] == candidate_hash == sha(image) and + plan['original_sha256'] == 'f18b4df865405f2a287b84977d7aca745e056d656d9fc9df1662970ce2969de7' and + len(plan['components']) == 29 and not plan['images_modified'] and not plan['source_inputs_modified'], + 'Exact native build/candidate and unchanged supplied target components') +build_dir = ROOT / 'build/constant-uploads-probe' +build = json.loads((build_dir / 'build-report.json').read_text()) +require(build['kit_revision'] == '2805ae10bf64af7e3517660189b2a0d5073febe6' and + not build['gpu_execution'] and not build['submission_authorized'], 'Exact parameter implementation and evidence limits') +for name, digest in build['source_sha256'].items(): + require(sha((ROOT / name).read_bytes()) == digest, 'Native fixture/kit source identity') +require(sha((ROOT / 'kits/AirNir/ConstantUploads.h').read_bytes()) == build['header_sha256'], 'Exact C ABI header') +require(sha((build_dir / 'libConstantUploadsProbe.a').read_bytes()) == build['archive_sha256'], 'Current archive identity') +for name, digest in build['object_sha256'].items(): + raw = (build_dir / name).read_bytes() + parsed = macho(raw) + version = next(at for cmd, at, _ in parsed['commands'] if cmd == 0x32) + require(sha(raw) == digest and parsed['kind'] == 1 and struct.unpack_from('> 14) & 3 == 0 and (r['TCR_EL1'] >> 30) & 3 == 2 and + r['SCTLR_EL1'] & 1 and r['TTBR0_EL1'] and r['TTBR1_EL1'] and + d['original_crypto_initialized_observed'] and d['original_crypto_table_owner_observed'], + 'Bounded actual XNU with active native-4K CPU translation') + state = d['native_constant_uploads'] + require(not state['gpu_execution_qualified'], 'Parameter copies are not GPU execution') + if label != 'default-off': + require(state['requested'] and state['status'] == 0 and state['refusal_mask'] == 1023 and + state['output_preserved'] == 1 and state['written_bytes'] == 144 and state['padded_bytes'] == 104, + 'Real positive and ten transactional native refusals') + else: + require(not state['requested'] and state['status'] == -1 and + all(state[k] == 0 for k in ('refusal_mask', 'output_preserved', 'written_bytes', 'padded_bytes', 'allocations_released')) and + set(state['physical']) == {'0x0'}, 'Default-off must perform no diagnostic work/allocation') + if label != 'native': + require(all(int(state[k], 16) == 0 for k in ('a_pointer', 'b_pointer', 'output_pointer')) and + state['allocations_released'] == (label == 'enabled-main') and + d['original_launchd_main_executed'] and d['followed_main_entry']['first_instruction_completed'] and + int(d['pstate'], 16) & 31 == 0 and + d['direct_user_commpage_shifts'] == {'fffff4025:1': 12, 'fffff4037:1': 12} and + d['owner_root_binding_audit']['boot_root_accepted'] and + d['owner_root_binding_audit']['_native_owner_root_apple_autherr'] == 80 and + d['original_amfi_table_owner_observed'] and d['actual_signature_failures'] == + dict(_cs_validate_page_bad_hash=0, _cs_validate_page_no_hash=0) and + d['native_virtio_integrity']['_native_virtio_integrity_failures'] == 0, + 'Allocation release before real original EL0 launchd; user-4K and signature/root checks remain intact') + reports[label] = d +native = reports['native'] +state = native['native_constant_uploads'] +require(native['stop_symbol'] == '_native_constants_checkpoint' and not state['allocations_released'], 'Live allocated checkpoint') +r = {k: int(v, 16) for k, v in native['system_registers'].items()} +mask = 0x0000fffffffff000 +for i, label in enumerate(('a', 'b', 'output')): + allocation = state[label] + span = allocation['cpu_mappings'] + va = int(state[label + '_pointer'], 16) + physical = [int(v, 16) for v in state['physical'][2*i:2*i+2]] + require(va and not va & 4095 and physical[0] and physical[1] == 0 and span['address'] == hex(va) and + span['bytes'] == 8192 and len(span['pages']) == 2, 'Actual page plus adjacent real guard') + for j, page in enumerate(span['pages']): + pointer = va + j * 4096 + require(int(page['virtual_address'], 16) == pointer, 'Exact allocation and adjacent guard addresses') + bits = 64 - ((r['TCR_EL1'] >> 16) & 63) + start = 4 - ((bits - 12 + 8) // 9) + table = r['TTBR1_EL1'] & mask + walk = page['walk'] + require(len(walk) == 4 - start, 'Complete independent actual TTBR1 walk') + for level, step in zip(range(start, 4), walk): + descriptor = int(step['descriptor'], 16) + address = table + ((((pointer & ((1 << bits) - 1)) >> (39 - 9 * level)) & 511) * 8) + require(step['level'] == level and int(step['descriptor_pa'], 16) == address, + 'Descriptor follows the active native TTBR and 4K indices') + if j and level == 3: + require(not descriptor & 1 and page['mapped'] is False, 'Guard has no valid hardware leaf') + else: + require(descriptor & 3 == 3, 'Native table/page descriptor; no block/conversion/shadow tables') + table = descriptor & mask + if not j: + require(table == physical[0] and page['leaf_level'] == 3 and page['leaf_bytes'] == 4096 and + int(page['physical_address'], 16) == physical[0] and page['ap'] == 0 and page['pxn'] and page['uxn'] and + page['accessed'] and page['shareability'] == 2, 'Actual real 4K RW/NX source/output page') + capture = allocation['capture'] + require(int(capture['pointer'], 16) == int(capture['address'], 16) == va, 'Actual allocation byte capture') + raw = bytes.fromhex(capture['bytes']) + expected = bytearray(b'\x39' * 4096 if label == 'a' else b'\x49' * 4096 if label == 'b' else b'\x6b' * 4096) + if label == 'a': expected[-8:] = bytes(range(1, 9)) + elif label == 'b': expected[-80:] = bytes(range(17, 97)) + else: + expected[3840+64:3840+128] = bytes(range(1, 9)) + bytes(56) + expected[3840+128:3840+192] = bytes(range(81, 97)) + bytes(48) + expected[3840+208:3840+224] = struct.pack('<4I', 0x11223344, 0x89abcdef, 0, 0xffffffff) + require(raw == expected, 'Complete sources unchanged and exact parameters/padding with all other bytes untouched') +(build_dir / 'native-checkpoint.json').write_text(json.dumps(dict( + kit_revision=build['kit_revision'], source_sha256=source_hash, candidate_sha256=candidate_hash, + helper_instruction_sha256=sha(code), helper_instruction_bytes=len(code), + refusals=10, written_bytes=144, padded_bytes=104, + actual_cpu_leaf_bytes=4096, actual_guard_pages=3, allocations_released_before_original_main=True, + default_off_original_main=True, gpu_execution=False, hardware_observed=False), indent=2) + '\n') +print('PASS: same native helper instructions, actual 4K pages/guards, exact parameter bytes and ten transactional refusals') +print('PASS: enabled allocation release and default-off original launchd/signatures/user-4K; no GPU/Metal claim') diff --git a/scripts/check-darwinboot-window-parser.py b/scripts/check-darwinboot-window-parser.py new file mode 100644 index 0000000..04b239c --- /dev/null +++ b/scripts/check-darwinboot-window-parser.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +"""Exercise the actual loader ADT parser on the host; no UEFI allocation claim.""" +import hashlib +import importlib.util +import json +from pathlib import Path +import struct +import subprocess +ROOT=Path(__file__).resolve().parents[1] +out=ROOT/'build/cpu-desktop/window-parser';out.mkdir(parents=True,exist_ok=True) +p=ROOT/'drivers/DarwinBoot/DarwinBootPkg/Driver/DarwinBoot.c' +s=p.read_text();start=s.index('STATIC\nBOOLEAN\nAdtNode (');end=s.index('\nSTATIC\nBOOLEAN\nValidTrustCaches',start) +spec=importlib.util.spec_from_file_location('adt',ROOT/'scripts/make-sheng-adt.py') +m=importlib.util.module_from_spec(spec);spec.loader.exec_module(m) +prelude='''#include +#include +#include +#include +#include +#include "WindowSize.h" +typedef uint8_t UINT8;typedef uint32_t UINT32;typedef uint64_t UINT64; +typedef size_t UINTN;typedef int BOOLEAN;typedef char CHAR8; +#define STATIC static +#define CONST const +#define TRUE 1 +#define FALSE 0 +#define MAX_UINTN SIZE_MAX +#define ALIGN_VALUE(a,b) (((a)+(b)-1)&~((b)-1)) +#define AsciiStrCmp strcmp +#define CompareMem memcmp +static const void *ScanMem8(const void *p,size_t n,int v){return memchr(p,v,n);} +static UINT32 U32(const void *p){UINT32 v;memcpy(&v,p,4);return v;} +static UINT64 U64(const void *p){UINT64 v;memcpy(&v,p,8);return v;} +''' +code=prelude+s[start:end]+'\nint main(void){\n' +checks=[] +for value in [0,0xfffffff,0x10000000,0xe0000000,0x100000000,0x130000000,0x200000000,0x200200000,0x130001000,0xffffffffffffffff]: + for variant in ['chosen','duplicate','misplaced']: + props=[('darwinboot-memory-size',struct.pack(' boot_args -> native-4K XNU PE video handoff.""" +import hashlib +import json +from pathlib import Path +import re +import runpy + +ROOT = Path(__file__).resolve().parents[1] +base = runpy.run_path(str(ROOT / 'scripts/check-native-arch-timer-checkpoints.py')) +require = base['require'] +elf = '54e699b71025e66b97033c4836d746ae8dd5b48963001a62e587be30b3a477e9' +require(hashlib.sha256((ROOT / 'build/DarwinBoot/DEBUG_CLANGPDB/AARCH64/DarwinBoot.efi').read_bytes()).hexdigest() == elf, + 'Exact tested loader binary') +for name, present in (('gop-native-launchd-main', True), ('video-headless-native-main', False)): + d = json.loads((ROOT / ('build/darwinboot-owner-hvf-' + name) / 'gdb-stop.json').read_text()) + v = d['actual_boot_video'] + expected = dict(physical_base=2088370176, display=1, row_bytes=3200, width=800, height=600, depth=32) if present else { + k: 0 for k in ('physical_base', 'display', 'row_bytes', 'width', 'height', 'depth')} + require(d['kernel_sha256'] == base['expected'] and d['stop_symbol'] == 'original-launchd-main' and + d['original_launchd_main_executed'] and d['followed_main_entry']['first_instruction_completed'] and + d['native_4k_mmu_observed'] and d['direct_user_commpage_shifts'] == {'fffff4025:1': 12, 'fffff4037:1': 12} and + v['qemu_ramfb_attached'] == present and v['darwinboot_driver_sha256'] == elf and + v['boot_args'] == expected and v['pe_video'] == dict(zip( + ('v_baseAddr', 'v_rowBytes', 'v_width', 'v_height', 'v_depth', 'v_display'), + (expected['physical_base'], expected['row_bytes'], expected['width'], expected['height'], expected['depth'], expected['display']))), + 'Actual firmware resource must propagate through actual boot args and XNU PE state; headless stays zero') + layout = v['compiled_layout'] + require(layout['state_size'] == 176 and layout['video'] == 8 and layout['boot_args'] == 160 and + layout['object_sha256'] == 'ba3cadd9293005dd74c033b91839d00a6c658044cf7db5650d3e505dc22eb365' and + d['actual_signature_failures'] == dict(_cs_validate_page_bad_hash=0, _cs_validate_page_no_hash=0) and + d['owner_boot_manifest']['status'] == 0 and d['owner_root_binding_audit']['boot_root_accepted'] and + not v['native_mobile_framebuffer_qualified'] and not d['functional_ipados_boot_demonstrated'], + 'Compiler-derived layout, actual code/root integrity and honest display scope') +for directory, text in (('darwinboot-test', 'boot_args, RAMDisk, TrustCache, zero-fill and MMU-off entry'), + ('darwinboot-test-gop', 'actual GOP boot video after ExitBootServices')): + log = (ROOT / 'build' / directory / 'qemu.clean.log').read_text() + require('DARWINBOOT-FIXTURE: ' + text + ' PASS' in log and + '33 rejected requests; firmware retained PASS' in log and + '8 malformed/unsupported GOP field sets refused PASS' in log and 'ASSERT' not in log, + 'Bounded loader fixture and same-parser invalid field controls must pass') +for mode in ('debug', 'release'): + log = (ROOT / ('build/darwinboot-video-' + mode + '-build.log')).read_text() + require('-Werror' in log and '- Done -' in log and not re.search(r'error:|error [A-Z0-9]+:', log), + 'Both actual warning-as-error builds must complete') +print('PASS: actual GOP 800x600/BGRX32 and headless zeros propagate into native-4K XNU PE state') +print('PASS: original launchd main, DEBUG/RELEASE builds and eight malformed-field refusals; mobile framebuffer/UI remain open') diff --git a/scripts/check-host-metal-async.py b/scripts/check-host-metal-async.py new file mode 100644 index 0000000..fd04a5c --- /dev/null +++ b/scripts/check-host-metal-async.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +"""Execute genuine asynchronous GPU submission/status and completion-safe teardown.""" +import argparse,hashlib,json,os,re,socket,struct,subprocess,tempfile,time +from pathlib import Path +p=argparse.ArgumentParser(description=__doc__);p.add_argument('--disconnect-control',action='store_true');p.add_argument('--api-validation',action='store_true');a=p.parse_args() +ROOT=Path(__file__).resolve().parents[1];OUT=ROOT/('build/host-metal-async-disconnect' if a.disconnect_control else 'build/host-metal-async');OUT.mkdir(exist_ok=True) +def require(v,s): + if not v:raise SystemExit(s) +def sha(p):return hashlib.sha256(p.read_bytes()).hexdigest() +def words(*v):return struct.pack('<'+'Q'*len(v),*v) +exe=ROOT/'build/host-metal/host-metal-originals';build=json.loads((exe.parent/'qualification.json').read_text()) +require(sha(exe)==build['process_sha256'],'Exact real host executable') +for rel,digest in build['backend_sources'].items():require(sha(ROOT/rel)==digest,'Exact backend source: '+rel) +base=ROOT/'.cache/firmware/target-volumes/os-mount/System/Library' +paths=[base/'Frameworks/QuartzCore.framework/default.metallib',base/'PrivateFrameworks/RenderBox.framework/default.metallib'] +require([sha(p) for p in paths]==['606f913adf520420f9b4bb666efb782a68c830f082f1e559e17b20d0d13adbc9','d76883f626b1527c36fb0940cf27dcade503a55cf2fa05c8f7a247a40ccebec4'],'Exact unchanged original libraries') +requests={};observed=[] +with tempfile.TemporaryDirectory(prefix='metal-fences-',dir='/tmp') as directory: + path=Path(directory)/'gpu.sock' + with (OUT/'host-report.json').open('w') as output,(OUT/'host-stderr.log').open('w') as errors: + process=subprocess.Popen([str(exe),*map(str,paths),'--metal-session-socket',str(path)],stdout=output,stderr=errors,env={**os.environ,**({'MTL_DEBUG_LAYER':'1'} if a.api_validation else {})}) + try: + deadline=time.monotonic()+15 + while not path.exists():require(process.poll() is None and time.monotonic()=value,'Actual Metal listener event identity and counter');return actual + require(eventID==actual==0 and time.monotonic()> 14 & 3 == 0 and r['TCR_EL1'] >> 30 & 3 == 2 and r['SCTLR_EL1'] & 1 and + r['TTBR0_EL1'] and r['TTBR1_EL1'] and r['CNTFRQ_EL0'] == 24000000 and + d['kernel_page_globals'] == dict(_PAGE_SHIFT_CONST=12, _page_shift=12, _page_shift_user32=12, _page_size=4096), 'Actual native-4K hardware mappings and runtime globals') +require(d['original_amfi_table_owner_observed'] and d['original_crypto_initialized_observed'] and + d['original_crypto_table_owner_observed'] and d['owner_root_binding_audit']['boot_root_accepted'] and + d['owner_root_binding_audit']['_native_owner_root_apple_autherr'] == 80, 'Real original AMFI/crypto and owner root enforcement remain active') +state = d['owner_cache_mount'] +require(state['_native_owner_cache_stage'] == 6 and state['_native_owner_cache_status'] == 0 and + state['_native_owner_cache_mount_status'] == 0 and state['_native_owner_cache_readonly'] == 1 and + state['_native_owner_cache_authorized'] == state['_native_owner_cache_root_recheck'] == 1 and + state['_native_owner_cache_selected_device'] == state['_native_owner_cache_mounted_device'] == 0x01000004 and + state['actual_device_path'] == '/dev/disk2s1', 'Actual authenticated readonly SystemOS mount') +require(d['actual_panic_region'] == dict(_gPanicBase=0, _gPanicSize=0, _panic_info=0), 'No advertised panic region means no mapping or fabricated log') +require(d['dyld_startup_flags'] == dict(_dyld_flags=8, _native_owner_cache_dyld_flags=8, actual_commpage_flags=8) and + d['native_signature_hash_cache'] == dict(_cs_native4k_hash_cache_version=1, _cs_native4k_full_blocks=61441, + _cs_native4k_page_checks=76, _cs_native4k_ready_blobs=19), 'Retain original code authentication and native-4K page checks') +require(d['native_virtio_globals']['_native_virtio_read_requests'] == 18537 and + d['native_virtio_globals']['_native_virtio_read_failures'] == 0 and + d['native_virtio_integrity']['_native_virtio_verified_blocks'] == 251564 and + d['native_virtio_integrity']['_native_virtio_integrity_failures'] == 0 and + d['native_virtio_integrity']['_native_virtio_verified_devices'] == 2, 'Actual original media I/O remains authenticated') +require(d['stop_symbol'] == '_panic' and d['argument_memory']['x0'].startswith('memorystatus_jetsam_thread: no victim!') and + struct.unpack_from('> 14 & 3 == 0 and r['TCR_EL1'] >> 30 & 3 == 2 and + r['SCTLR_EL1'] & 1 and r['TTBR0_EL1'] and r['TTBR1_EL1'], 'actual native 4K hardware') + return d + + +positive = read_report('darwinboot-defaults-configure-strict') +negative = read_report('darwinboot-defaults-unmatched-strict') +require(positive['original_crypto_initialized_observed'] and negative['original_crypto_initialized_observed'], + 'genuine original crypto before platform matching') +require(not positive['bounded_timeout_interrupted'] and int(positive['x0'], 16) == 1 and + positive['pc'] == positive['function_return_target'], 'actual configure returned true') +g = positive['armvirt_globals'] +require(all(g[n] == 1 for n in ('_armvirt_platform_configured', '_armvirt_platform_cpu_count', + '_armvirt_platform_bus_count', '_armvirt_platform_defaults_count')), 'actual required DT nubs') +require(g['_armvirt_platform_child_count'] == 5, 'actual child count, not an assumed device count') +require(not negative['bounded_timeout_interrupted'] and not any(negative['armvirt_globals'].values()) and + 'Unable to find driver for this platform' in negative['argument_memory']['x0'], 'original platform refusal') + +service = read_report('darwinboot-target-dyld-defaults-services') +require(service['kernel_observations']['_armvirt_platform_defaults_count'] == 1 and + service['kernel_observations']['_armcpu_timer_interrupts'] > 0, 'enumerated node and genuine interrupts') +queries = [q for q in service['cstring_events'] if q['value'] == 'IOAESAccelerator'] +require(len(queries) == 1 and not service['cstring_log_exhausted'], 'bounded actual AES service request') +query = queries[0] +require(any(e['module'] == 'com.apple.security.AppleImage4' and e['segment'] == '__TEXT_EXEC' and + e['va'] <= int(query['caller'], 16) < e['va'] + e['size'] for e in plan['extents']), + 'actual request caller is the original Image4 module') +at = np.backed_offset(native['segments'], int(query['argument'], 16), 17, len(kernel)) +require(kernel[at:at + 17] == b'IOAESAccelerator\0', 'original request string') +last = service['syscall_events'][-1] +require(last['number'] == 202 and struct.unpack('<4I', bytes.fromhex(last['sysctl_mib_bytes'])) == + (101, 100, 118, 100) and service['bounded_timeout_interrupted'], 'measured ignition wait, not prepare success') + +wait = read_report('darwinboot-image4-aes-wait-entry') +exports = np.exports(kernel, native) +require('__ZN9IOService22waitForMatchingServiceEP12OSDictionaryy' in exports[int(wait['pc'], 16)], + 'actual source waitForMatchingService entry') +require(not wait['bounded_timeout_interrupted'] and int(wait['x1'], 16) == (1 << 64) - 1 and + int(wait['x30'], 16) == int(query['caller'], 16) + 12, 'original Image4 requested unbounded service wait') +require(any(int(t['pc'], 16) == int(query['caller'], 16) - 4 for t in wait['trace_events']), + 'same original serviceMatching call precedes the actual wait') +require(audit['candidate_sha256'] == plan['candidate_sha256'] and audit['compiled_oid_offsets'] == + dict(oid_name=40, oid_handler=48) and not audit['code_or_security_result_modified'], 'qualified original handler analysis') +require(any('_lck_mtx_gate_wait' in c['names'] for c in audit['kernel_calls']), 'real source gate-wait binding') + +abi = read_report('darwinboot-target-dyld-defaults-mach-init') +require(abi['original_dyld_mach_init_returned'] and abi['original_dyld_native_4k_abi_observed'] and + abi['original_dyld_page_abi'] == dict(_vm_page_size=4096, _vm_page_mask=4095, + _vm_page_shift=12, _vm_kernel_page_shift=12), 'current platform still qualifies genuine target dyld page ABI') +require(abi['kernel_observations']['_armvirt_platform_defaults_count'] == 1 and + not abi['bounded_timeout_interrupted'], 'actual defaults/ABI regression checkpoint') + +result = dict(candidate_sha256=plan['candidate_sha256'], checkpoints=5, defaults_nub_published=True, + unmatched_platform_refused=True, original_image4_aes_request_observed=True, + actual_wait_timeout=(1 << 64) - 1, original_ignition_handler_unchanged=True, + original_target_dyld_4k_abi_retained=True, + aes_provider_initialized=False, ignition_blob_published=False, + dyld_prepare_reached=False, functional_ipados_booted=False, hardware_tested=False) +(build / 'image4-ignition/service-checkpoints.json').write_text(json.dumps(result, indent=2) + '\n') +print('PASS: actual defaults enumeration, refusal control and original Image4 request/wait for IOAESAccelerator') +print('AES service and ignition publication remain missing; no successful startup is inferred from a timeout') diff --git a/scripts/check-iogpu-checkpoints.py b/scripts/check-iogpu-checkpoints.py new file mode 100644 index 0000000..8e9d19f --- /dev/null +++ b/scripts/check-iogpu-checkpoints.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 +"""Qualify actual original IOGPUFamily loading and native metaclass ABI evidence.""" +import hashlib +import json +from pathlib import Path +import re +import struct +import subprocess +import sys +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'kits/NativePrelink')) +from native_prelink import backed_offset, macho + + +def require(value, message): + if not value: + raise SystemExit(message) + + +source_hash = '257c6f553e0a30a577e8a53623fc742b46f337f55b912b5ff466c19b4c049828' +positive_hash = 'a66a3c60c82330cba7915a6dfd028cf6b2332e3919021f1fce9c0d0526498f99' +negative_hash = '043ec08b8ab0761412409fd2975e5a7d94ce8153ca1cd149bfe19f3053a72eb6' +directory = ROOT / 'build/native-prelink' +plans = {} +for stem, expected in (('native-owner-iogpu-candidate', positive_hash), + ('native-owner-iogpu-missing-candidate', negative_hash)): + path = directory / (stem + '.macho') + plan = json.loads(path.with_suffix('.json').read_text()) + require(plan['native_sha256'] == source_hash and plan['candidate_sha256'] == expected == + hashlib.sha256(path.read_bytes()).hexdigest() and plan['original_sha256'] == + 'f18b4df865405f2a287b84977d7aca745e056d656d9fc9df1662970ce2969de7' and + not plan['images_modified'] and not plan['source_inputs_modified'], 'Exact unchanged target and source/candidate identities') + plans[expected] = plan +require(len(plans[positive_hash]['components']) == 29 and len(plans[negative_hash]['components']) == 28 and + set(plans[positive_hash]['components']) - set(plans[negative_hash]['components']) == {'com.apple.iokit.IOGPUFamily'}, + 'Actual component present/absent closure') +image = (directory / 'native-owner-iogpu-candidate.macho').read_bytes() +segments = macho(image)['segments'] + + +def read(va, size): + at = backed_offset(segments, va, size, len(image)) + return image[at:at + size] + + +def report(label, expected=positive_hash): + d = json.loads((ROOT / ('build/darwinboot-owner-hvf-iogpu-' + label) / 'gdb-stop.json').read_text()) + r = {name: int(value, 16) for name, value in d['system_registers'].items()} + require(d['kernel_sha256'] == expected and d['qemu_accelerator'] == 'hvf' and not d['bounded_timeout_interrupted'] and + d['native_4k_mmu_observed'] and r['TCR_EL1'] >> 14 & 3 == 0 and r['TCR_EL1'] >> 30 & 3 == 2 and + r['SCTLR_EL1'] & 1 and r['TTBR0_EL1'] and r['TTBR1_EL1'] and + d['original_crypto_initialized_observed'] and d['original_crypto_table_owner_observed'] and + not d['sheng_hardware_tested'], 'Actual bounded native-4K XNU and original crypto') + return d + + +loaded, missing = report('load'), report('missing', negative_hash) +require(loaded['stop_symbol'] == missing['stop_symbol'] == '_native_iogpu_probe_checkpoint' and + loaded['native_iogpu_load_status'] == 0 and missing['native_iogpu_load_status'] == -603946998, + 'Real original component load and genuine component-absent refusal') +component = loaded['native_iogpu_component'] +require(component['actual_start_observed'] and component['original_entry_bytes_match'] and + component['start'] == '0xfffffff004bde714' and component['original_start'] == '0xfffffff005bde714' and + any(e['pc'] == component['start'] and e['symbol'] == 'com.apple.iokit.IOGPUFamily:kmod_start' for e in loaded['trace_events']), + 'Actual original IOGPU kmod start with unchanged entry instructions') + +symbols = subprocess.check_output(['xcrun', 'nm', '-U', directory / 'native-owner-iogpu-candidate.macho'], text=True) + + +def symbol(name): + match = re.search(r'^([0-9a-f]+)\s+\S\s+' + re.escape(name) + '$', symbols, re.M) + require(match, 'Qualified actual native symbol: ' + name) + return int(match[1], 16) + + +def owner(va): + return any(e['module'] == 'com.apple.iokit.IOGPUFamily' and e['segment'] == '__DATA' and + e['va'] <= va < e['va'] + e['size'] for e in plans[positive_hash]['extents']) + + +records = {} +for label, name, size, parent in ( + ('gpu', 'IOGPU', 0x2e0, 0xfffffff006aeb688), + ('device', 'IOGPUDevice', 0x100, 0xfffffff006aeb278), + ('userclient', 'IOGPUDeviceUserClient', 0x130, symbol('__ZN12IOUserClient10gMetaClassE'))): + d = report(label + '-class') + args = [int(value, 16) for value in d['function_entry_arguments']] + raw = bytes.fromhex(d['entry_regions']['0:0:80']['bytes']) + require(d['function_return_stack_matched'] and d['entry_kernel_bytes_match']['bytes'] == (name + '\0').encode().hex() and + args[2:4] == [parent, size] and struct.unpack_from('>5)&0x7ffff;imm-=0x80000 if imm&0x40000 else 0 + putq(pc+imm*4,value) + DICT=0x200000010;BOOL=0x200000020;TRUE=0x200000030;OBJ=0x200000040;EMPTY=0x200000050 + # Resolve original imported data symbols; retain all original instructions. + bind_literal(0x100046fb8,0x200000060) + bind_literal(0x10003b1a0,DICT);bind_literal(0x10002dd6c,DICT);bind_literal(0x10002dd9c,BOOL);bind_literal(0x100004150,TRUE) + encoded=struct.unpack_from('start]+[sec['va']+sec['size']]) + size=end-start;source=bytearray(obj[sec['fileoff']+start-sec['va']:sec['fileoff']+end-sec['va']]) + target=names[symbol];at=backed_offset(parsed['segments'],target,size,len(raw));linked=bytearray(raw[at:at+size]) + ro,rc=struct.unpack_from('len(set(masked))*4,'Actual linked own method instructions '+part) + code_checks.append(dict(method=part,bytes=size,relocated_words=sorted(set(masked)),masked_sha256=hashlib.sha256(source).hexdigest())) +observations={} +for case in ('query','headless','default-off'): + path=ROOT/f'build/darwinboot-owner-hvf-mobile-provider-{case}/gdb-stop.json';d=json.loads(path.read_text()) + require(d['kernel_sha256']==plan['candidate_sha256'] and not d['bounded_timeout_interrupted'] and d['qemu_accelerator']=='hvf','Actual bounded same-candidate boot') + r={k:int(v,16) for k,v in d['system_registers'].items()} + require(d['native_4k_mmu_observed'] and (r['TCR_EL1']>>14)&3==0 and (r['TCR_EL1']>>30)&3==2 + and r['SCTLR_EL1']&1 and r['TTBR0_EL1'] and r['TTBR1_EL1'],'Actual active native-4K CPU translations') + v=d['actual_boot_video']['pe_video'] + stage=field(d,'_native_mobile_provider_stage');meta=field(d,'_native_mobile_provider_meta_status') + if case=='query': + require(d['stop_symbol']=='_native_mobile_display_provider_checkpoint' and stage==5 and meta==0 + and field(d,'_native_mobile_provider_query_status')==0 and field(d,'_native_mobile_provider_parent_match')==1 + and field(d,'_native_mobile_provider_released')==1,'Real original client start/query/stop/release and typed parent relationship') + dimensions=field(d,'_native_mobile_provider_dimensions',16) + require(dimensions==field(d,'_native_mobile_provider_expected',16)==v['v_width']|(v['v_height']<<64) + and v['v_width']==800 and v['v_height']==600 and v['v_depth']==32 and v['v_rowBytes']==3200, + 'Actual original selector output matches actual GOP/PE video') + elif case=='headless': + require(d['stop_symbol']=='_native_mobile_display_provider_checkpoint' and stage==2 and meta==0 + and not field(d,'_native_mobile_provider_object',8) and not field(d,'_native_mobile_provider_client',8) + and field(d,'_native_mobile_provider_query_status')==0xffffffff and not any(v.values()), + 'Actual no-video backend creation refusal, without invented display geometry') + else: + require(not stage and meta==0xffffffff and d['original_launchd_main_executed'] + and d['followed_main_entry']['first_instruction_completed'] and d['original_amfi_table_owner_observed'] + and d['owner_root_binding_audit']['boot_root_accepted'] + and d['owner_root_binding_audit']['_native_owner_root_apple_autherr']==80 + and set(d['direct_user_commpage_shifts'].values())=={12} + and d['actual_signature_failures']==dict(_cs_validate_page_bad_hash=0,_cs_validate_page_no_hash=0), + 'Disabled provider path preserves genuine original launchd, AMFI/root and native user page ABI') + observations[case]=dict(checkpoint_sha256=sha(path),stage=stage,actual_tcr=r['TCR_EL1']) +(OUT/'native-checkpoint.json').write_text(json.dumps(dict(candidate_sha256=plan['candidate_sha256'], + kit_revision=revision, + source_sha256=build['source_sha256'],implemented_entries=[182,193,249],actual_original_client_selector=8, + actual_original_client_start_query_stop_release=True,actual_dimensions=[800,600],headless_refused=True, + linked_method_checks=code_checks,observations=observations,native_cpu_page_bytes=4096, + unclassified_extension_entries=91,display_published=False,presentation=False,gpu_execution=False, + active_gpu_smmu_qualified=False,metal_device=False,original_desktop=False),indent=2)+'\n') +print('PASS: real original framebuffer client queries actual 800x600 provider, then stops/releases; headless refuses') +print('Default-off original launchd/AMFI/root/user-4K preserved; display publication and presentation remain incomplete') diff --git a/scripts/check-mobile-graphics-checkpoints.py b/scripts/check-mobile-graphics-checkpoints.py new file mode 100644 index 0000000..5226123 --- /dev/null +++ b/scripts/check-mobile-graphics-checkpoints.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +"""Qualify original display-family initialization and bounded class ABI evidence.""" +import hashlib +import json +from pathlib import Path +import re +import struct +import subprocess +import sys + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'kits/NativePrelink')) +from native_prelink import backed_offset, macho + + +def require(value, message): + if not value: + raise SystemExit(message) + + +source_hash = '8b78575592b37bfc8a180fb4a029383287b3a9a032841dfc4efcb5cdeb656cdb' +positive_hash = 'aded28d3c61dafaf7b8a7c96b14427a798e9e95470ae6f287bcd038c07626798' +negative_hash = 'e3748f2c9e4f0d06d5ed925d278662c207cf14efb0a2d8a70c28f389848b81f1' +directory = ROOT / 'build/native-prelink' +images = {} +for stem, expected in (('native-owner-mobile-graphics-candidate', positive_hash), + ('native-owner-mobile-graphics-missing-candidate', negative_hash)): + path = directory / (stem + '.macho') + plan = json.loads(path.with_suffix('.json').read_text()) + require(plan['native_sha256'] == source_hash and plan['candidate_sha256'] == expected == + hashlib.sha256(path.read_bytes()).hexdigest(), 'Exact source/candidate identities') + images[expected] = plan +require(len(images[positive_hash]['components']) == 28 and len(images[negative_hash]['components']) == 18 and + 'com.apple.iokit.IOMobileGraphicsFamily' not in images[negative_hash]['components'], 'Actual component-present/absent candidates') +image = (directory / 'native-owner-mobile-graphics-candidate.macho').read_bytes() +segments = macho(image)['segments'] + + +def read(va, size): + at = backed_offset(segments, va, size, len(image)) + return image[at:at + size] + + +def report(label, expected=positive_hash): + d = json.loads((ROOT / ('build/darwinboot-owner-hvf-mobile-graphics-' + label) / 'gdb-stop.json').read_text()) + r = {name: int(value, 16) for name, value in d['system_registers'].items()} + require(d['kernel_sha256'] == expected and d['qemu_accelerator'] == 'hvf' and not d['bounded_timeout_interrupted'] and + d['native_4k_mmu_observed'] and r['TCR_EL1'] >> 14 & 3 == 0 and r['TCR_EL1'] >> 30 & 3 == 2 and + r['SCTLR_EL1'] & 1 and r['TTBR0_EL1'] and r['TTBR1_EL1'] and + d['original_crypto_initialized_observed'] and d['original_crypto_table_owner_observed'] and + not d['sheng_hardware_tested'], 'Bounded actual native-4K kernel and original crypto') + return d + + +loaded, missing = report('load'), report('missing', negative_hash) +require(loaded['stop_symbol'] == missing['stop_symbol'] == '_native_mobile_graphics_probe_checkpoint' and + loaded['native_mobile_graphics_load_status'] == 0 and missing['native_mobile_graphics_load_status'] == -603946998, + 'Real load returns zero; missing prelinked component returns kOSKextReturnDisabled (0xdc00800a)') +component = loaded['native_mobile_graphics_component'] +require(component['actual_start_observed'] and component['original_entry_bytes_match'] and + component['start'] == '0xfffffff0057a2654' and component['original_start'] == '0xfffffff0067a2654' and + any(e['pc'] == component['start'] and e['symbol'] == 'com.apple.iokit.IOMobileGraphicsFamily:kmod_start' for e in loaded['trace_events']), + 'Actual original kmod start and original instruction bytes') +print('PASS: actual original IOMobileGraphicsFamily starts and loads; component-absent control genuinely refuses') + +symbols = subprocess.check_output(['xcrun', 'nm', '-U', directory / 'native-owner-mobile-graphics-candidate.macho'], text=True) +def symbol(name): + return int(re.search(r'^([0-9a-f]+)\s+\S\s+' + re.escape(name) + '$', symbols, re.M)[1], 16) + +records = {} +for label, name, size, parent in ( + ('framebuffer', 'IOMobileFramebuffer', 0x90, symbol('__ZN9IOService10gMetaClassE')), + ('userclient', 'IOMobileFramebufferUserClient', 0x538, symbol('__ZN12IOUserClient10gMetaClassE')), + ('service', 'IOMobileFramebufferService', 0x90, 0xfffffff006b313d8), + ('legacy', 'IOMobileFramebufferLegacy', 0x18a0, 0xfffffff006b31400)): + d = report(label + '-class') + args = [int(value, 16) for value in d['function_entry_arguments']] + raw = bytes.fromhex(d['entry_regions']['0:0:80']['bytes']) + require(d['function_return_stack_matched'] and d['entry_kernel_bytes_match']['bytes'] == (name + '\0').encode().hex() and + args[2:4] == [parent, size] and struct.unpack_from('> 5) & 0x7ffff) << 2 | ((adrp >> 29) & 3) + if immediate & (1 << 20): + immediate -= 1 << 21 + vptr = (pc & ~4095) + (immediate << 12) + ((add >> 10) & 4095) + 16 + alloc, = struct.unpack('> 14 & 3 == 0 and r['TCR_EL1'] >> 30 & 3 == 2 and + r['SCTLR_EL1'] & 1 and r['TTBR0_EL1'] and r['TTBR1_EL1'] and + d['original_crypto_initialized_observed'] and d['original_crypto_table_owner_observed'] and + d['native_mobile_graphics_load_status'] == 0 and + d['native_mobile_graphics_component']['original_entry_bytes_match'], 'Bounded actual native-4K kernel and genuine original family loading') + client = d['native_mobile_graphics_client'] + if label == 'client-default-off': + require(client == dict(abstract_null_mask=0, client_object=0, client_vptr=0, client_meta=0, + client_size=0, client_init=-1, client_released=0, selector_results=[0xffffffff] * 3), + 'The independent client instrument remains default-off') + else: + require(client['abstract_null_mask'] == 7 and client['client_object'] >= 0xfffffff000000000 and + client['client_vptr'] == 0xfffffff005f34dd0 and client['client_meta'] == 0xfffffff006b313b0 and + client['client_size'] == 0x538 and client['client_init'] == client['client_released'] == 1 and + client['selector_results'] == [0xe0000001] * 3, + 'Actual original abstract factory refusals, client construction/init/release and three unsigned-range refusals') + if label != 'client': + require(d['original_launchd_main_executed'] and d['followed_main_entry']['first_instruction_completed'] and + int(d['pstate'], 16) & 31 == 0 and + d['direct_user_commpage_shifts'] == {'fffff4025:1': 12, 'fffff4037:1': 12} and + d['owner_root_binding_audit']['boot_root_accepted'] and d['owner_root_binding_audit']['_native_owner_root_apple_autherr'] == 80 and + d['owner_boot_manifest']['status'] == 0 and d['original_amfi_table_owner_observed'] and + d['actual_signature_failures'] == dict(_cs_validate_page_bad_hash=0, _cs_validate_page_no_hash=0) and + d['native_virtio_integrity']['_native_virtio_integrity_failures'] == 0, + 'Original launchd survives enabled/default-off client controls with intact trust, signatures and user 4K') + reports[label] = d + +factories = [('IOMobileFramebuffer', 0xfffffff00577b420, 0xfffffff006b313d8), + ('IOMobileFramebufferService', 0xfffffff00577b620, 0xfffffff006b31400), + ('IOMobileFramebufferLegacy', 0xfffffff0057866d4, 0xfffffff006b35940)] +for name, pc, meta in factories: + require(any(e['symbol'] == name + '::MetaClass::alloc' and int(e['pc'], 16) == pc and int(e['x0'], 16) == meta + for e in reports['client']['trace_events']), 'Actual original abstract factory must run: ' + name) +require(reports['client']['stop_symbol'] == '_native_mobile_graphics_client_checkpoint' and + any(e['symbol'] == 'IOMobileFramebufferUserClient::externalMethod' and e['x1'] == '0x5a' and + int(e['x0'], 16) == reports['client-launchd-main']['native_mobile_graphics_client']['client_object'] + for e in reports['client-launchd-main']['trace_events']), 'Actual original request entry on the allocated client') +print('PASS: all three original abstract factories actually run and return null; concrete original client allocates, initializes and releases') +print('PASS: selectors 90/0x80000000/0xffffffff really refuse; original launchd main, native 4K and trust survive enabled/default-off controls') diff --git a/scripts/check-mount-phase-checkpoints.py b/scripts/check-mount-phase-checkpoints.py new file mode 100644 index 0000000..102f77e --- /dev/null +++ b/scripts/check-mount-phase-checkpoints.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python3 +"""Qualify original launchd mount phase arguments and its genuine missing input.""" +import json +from pathlib import Path +import runpy + +ROOT = Path(__file__).resolve().parents[1] +base = runpy.run_path(str(ROOT / 'scripts/check-original-launchd-integrity-checkpoints.py')) +require = base['require'] +spawn, output = [json.loads((ROOT / ('build/darwinboot-owner-hvf-' + name) / 'gdb-stop.json').read_text()) + for name in ('mount-spawn', 'mount-output')] +for d in (spawn, output): + require(d['kernel_sha256'] == base['positive']['kernel_sha256'] and not d['bounded_timeout_interrupted'] and + d['owner_root_binding_audit']['boot_root_accepted'] and d['original_amfi_table_owner_observed'] and + d['actual_signature_failures'] == dict(_cs_validate_page_bad_hash=0, _cs_validate_page_no_hash=0), 'Same genuine native-4K original OS/security path') +require(spawn['stop_symbol'] == '_posix_spawn' and spawn['function_return_stack_matched'] and int(spawn['x0'], 16) == 0 and + spawn['spawn_parent_is_initproc'] and spawn['entry_string_events'][-1]['value'] == '/sbin/mount' and + bytes.fromhex(spawn['entry_indirect_memory']['1:0:4']['bytes']) == b'\x03\0\0\0', 'Real PID-1 parent successfully spawns mount PID 3') +vector = spawn['entry_user_vectors']['1:18:10'] +require(vector['terminated'] and [x['value'] for x in vector['values']] == ['mount', '-P', '1'] and + not any(x['truncated'] for x in vector['values']), 'Capture complete actual mount phase-1 argv') +require(output['critical_process_exit'] == dict(process='mount[3] exited', signal=0, status=66, cs_status='') and + output['syscall_start_event']['pc'] == '0xfffffff00bcb312c' and not output['syscall_log_exhausted'], 'Retain actual critical exit and bounded phase-selected observations') +writes = [bytes.fromhex(e['write_bytes']).decode(errors='replace') for e in output['syscall_events'] + if e['number'] in (4, 397) and 'write_bytes' in e and not e['write_bytes'].startswith('E')] +require(any('data volume missing, but not required in env: 0' in x for x in writes) and + any('failed to get boot-manifest-hash' in x for x in writes) and + any('failed to get boot manifest hash - No such file or directory' in x for x in writes), + 'Actual original stderr identifies missing manifest hash; do not guess that missing Data is the current cause') +print('PASS: original launchd PID 1 spawns mount -P 1 as PID 3; genuine missing boot-manifest-hash causes exit 66') diff --git a/scripts/check-native-4k-policy.py b/scripts/check-native-4k-policy.py new file mode 100755 index 0000000..0265727 --- /dev/null +++ b/scripts/check-native-4k-policy.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +"""Execute XNU's actual page-size policy at RAM boundaries, without a VM.""" +import json +from pathlib import Path +import shutil +import subprocess + +ROOT = Path(__file__).resolve().parents[1] +source = (ROOT / '.cache/xnu-work/osfmk/arm/arm_init.c').read_text() +start = source.index('\t\t/* Native non-Apple 4K builds expose the hardware page size at any RAM size. */') +end = source.index('#ifdef APPLETYPHOON', start) +policy = source[start:end] +output = ROOT / 'build/xnu/native-4k-policy' +output.mkdir(parents=True, exist_ok=True) +program = output / 'policy.c' +program.write_text('''#include +#include +#include +#define PAGE_MAX_SHIFT 14 +int PAGE_SHIFT_CONST; +unsigned int page_shift_user32; +static void policy(uint64_t memory) { + struct { uint64_t memSize; } value = { memory }, *args = &value; + (void)args; +''' + policy + ''' +} +int main(int argc, char **argv) { + if (argc != 2) return 1; + policy(strtoull(argv[1], 0, 0)); + printf("%d %u\\n", PAGE_SHIFT_CONST, page_shift_user32); + return 0; +} +''') +cases = [0x10000000, 0x40000000, 0x40001000, 0x80000000, 0x200000000] +report = {} +profiles = [('native-4k', ['-DARM_PGSHIFT=12'], False, False), + ('apple-16k-control', ['-DARM_PGSHIFT=14', '-D__ARM_16K_PG__=1', '-DAPPLE_ARM64_ARCH_FAMILY=1'], True, False), + ('apple-4k-control', ['-DARM_PGSHIFT=12', '-DAPPLE_ARM64_ARCH_FAMILY=1'], False, True)] +for name, flags, sixteen, apple_four in profiles: + executable = output / name + subprocess.run([shutil.which('clang'), '-Wall', '-Wextra', '-Werror', *flags, + str(program), '-o', str(executable)], check=True) + values = [] + for memory in cases: + actual = tuple(map(int, subprocess.check_output([executable, hex(memory)], text=True).split())) + expected = ((14, 14) if sixteen else ((14 if memory > 0x40000000 else 12), 14) + if apple_four else (12, 12)) + if actual != expected: + raise SystemExit(f'{name}, RAM={memory:#x}: got {actual}, expected {expected}') + values.append({'ram': hex(memory), 'kernel_shift': actual[0], 'user32_shift': actual[1]}) + report[name] = values + print(f'{name}: {len(cases)} RAM boundaries pass') +report['scope'] = 'Host execution of the actual extracted arm_init policy; not an iPadOS userspace test' +(output / 'report.json').write_text(json.dumps(report, indent=2) + '\n') diff --git a/scripts/check-native-aes-checkpoints.py b/scripts/check-native-aes-checkpoints.py new file mode 100644 index 0000000..d2285b3 --- /dev/null +++ b/scripts/check-native-aes-checkpoints.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python3 +"""Verify genuine CBC/refusal tests and original dyld prepare/cache observations.""" +import hashlib +import json +from pathlib import Path +import struct +import sys + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'kits/NativePrelink')) +import native_prelink as np +build = ROOT / 'build' + + +def require(value, message): + if not value: + raise SystemExit('FAIL: ' + message) + + +def plan(name): + path = build / 'native-prelink' / (name + '.macho') + data = path.read_bytes() + result = json.loads(path.with_suffix('.json').read_text()) + require(hashlib.sha256(data).hexdigest() == result['candidate_sha256'], 'candidate identity') + require(not result['images_modified'] and not result['source_inputs_modified'] and + result['original_sha256'] == 'f18b4df865405f2a287b84977d7aca745e056d656d9fc9df1662970ce2969de7', + 'unchanged original input provenance') + parsed = np.macho(data) + for extent in result['extents']: + if extent['segment'] == '__TEXT_EXEC': + at = np.backed_offset(parsed['segments'], extent['va'], extent['size'], len(data)) + require(hashlib.sha256(data[at:at + extent['size']]).hexdigest() == + result['original_executable_sha256'][extent['module']], 'unchanged actual original executable extent') + return result + + +positive = plan('native-aes-api-candidate') +negative = plan('native-aes-kat-negative-candidate') +require(positive['original_executable_sha256'] == negative['original_executable_sha256'], + 'all genuine original component code retained in both controls') +own = (ROOT / 'drivers/NativeAES/NativeAES.cpp').read_text() +control = (build / 'native-aes-kat-negative/NativeAES.cpp').read_text() +require(own.count('0x76,0x49,0xab,0xac') == 1 and + control == own.replace('0x76,0x49,0xab,0xac', '0x77,0x49,0xab,0xac'), + 'negative control changes only one own known-answer byte') +bindings = json.loads((build / 'native-aes/bindings.json').read_text()) +require(bindings['compiled_subclass_slots_checked'] == 181 and + not bindings['original_instructions_modified'], 'compiled ordinary iOS subclass dispatch') +require(hashlib.sha256((build / 'native-aes/NativeAES.o').read_bytes()).hexdigest() == + bindings['compiled_subclass_sha256'], 'qualified subclass object identity') + + +def report(name, candidate): + d = json.loads((build / name / 'gdb-stop.json').read_text()) + require(d['kernel_sha256'] == candidate['candidate_sha256'] and not d['bounded_timeout_interrupted'], + 'actual bounded checkpoint with exact candidate') + r = {k: int(v, 16) for k, v in d['system_registers'].items()} + require(r['TCR_EL1'] >> 14 & 3 == 0 and r['TCR_EL1'] >> 30 & 3 == 2 and + r['SCTLR_EL1'] & 1 and r['TTBR0_EL1'] and r['TTBR1_EL1'], 'actual native hardware 4K') + return d + + +started = report('darwinboot-native-aes-api-start-stack', positive) +refused = report('darwinboot-native-aes-kat-refusal-stack', negative) +for d, expected in ((started, 1), (refused, 0)): + require(d['function_return_stack_matched'] and d['sp'] == d['function_entry_stack'] and + d['pc'] == d['function_return_target'] and int(d['x0'], 16) == expected, + 'actual original-thread start return, not another driver at the shared caller') + require(d['original_crypto_initialized_observed'], 'genuine original crypto registration') +require(started['native_aes_globals'] == dict(_native_aes_kat_status=1, _native_aes_interface_status=1, + _native_aes_registered=1, _native_aes_completed=3, _native_aes_unsupported_key=1), + 'AES-128/192/256 answers, descriptor/raw/in-place operations, bounds and hardware-key refusal') +require(refused['native_aes_globals']['_native_aes_kat_status'] == 2 and + refused['native_aes_globals']['_native_aes_registered'] == 0, + 'wrong own known answer genuinely prevents service publication') + +fixture = json.loads((build / 'target-dyld-fixture/inputs.json').read_text()) +dyld = (build / 'target-dyld-fixture/dyld').read_bytes() +require(hashlib.sha256(dyld).hexdigest() == 'f8df3c58fae6e409db740df22c98aa699aee110b73523541a63f1bd5e23cf705', + 'unchanged supplied ordinary ARM64 dyld') +prepared = report('darwinboot-target-dyld-aes-api-prepare', positive) +cache = report('darwinboot-target-dyld-cache-loader', positive) +for d in (prepared, cache): + require(d['target_inputs']['root_sha256'] == fixture['root_sha256'] and + d['target_inputs']['trust_bundle_sha256'] == fixture['trust_bundle_sha256'], 'same original target root/trust input') + require(d['original_dyld_page_abi'] == dict(_vm_page_size=4096, _vm_page_mask=4095, + _vm_page_shift=12, _vm_kernel_page_shift=12), 'original dyld native page ABI') + g = d['kernel_observations'] + require(g['_native_aes_kat_status'] == g['_native_aes_interface_status'] == g['_native_aes_registered'] == 1, + 'genuine tested AES service available at original user checkpoint') + require(g['_cs_validate_page_bad_hash'] == g['_cs_validate_page_no_hash'] == 0, 'genuine code-page integrity') + checkpoint = d['followed_user_entry']['checkpoint'] + offset = int(checkpoint['pc'], 16) - int(d['original_dyld_slide'], 16) + require(checkpoint['observed'] and int(checkpoint['pstate'], 16) & 15 == 0 and + checkpoint['bytes'] == dyld[offset:offset + 16].hex(), 'byte-matched actual original EL0 checkpoint') + require(not d['target_launchd_main_executed'] and not d['functional_ipados_boot_demonstrated'] and + not d['sheng_hardware_tested'], 'component checkpoint scope') +require(int(prepared['followed_user_entry']['checkpoint']['pc'], 16) - int(prepared['original_dyld_slide'], 16) == + fixture['dyld_symbols']['__ZN5dyld4L7prepareERNS_4APIsEPKN5dyld313MachOAnalyzerE'] + 4, + 'actual original dyld4::prepare entry') +returned = cache['followed_user_entry']['function_return'] +require(cache['original_dyld_cache_loader_returned'] and returned['observed'] and + int(returned['pstate'], 16) & 15 == 0 and int(returned['x0'], 16) == 0, + 'actual original cache loader returned false, not a mapped-cache success') +words = struct.unpack('<8Q', bytes.fromhex(cache['followed_user_entry']['checkpoint_argument_memory']['1:40']['bytes'])) +require(words[0:2] == (0, 0), 'no actual cache mapping in the loader result') +offset = words[2] - int(cache['original_dyld_slide'], 16) +require(0 <= offset < len(dyld) and dyld[offset:offset + 256].split(b'\0', 1)[0] == + b'shared cache file open() failed', 'actual error pointer identifies the original input message') +require(any(e['number'] == 5 and e.get('path') == '/System/Library/Caches/com.apple.dyld/' + for e in cache['syscall_events']), 'actual attempted shared-cache directory open') +result = dict(candidate_sha256=positive['candidate_sha256'], negative_candidate_sha256=negative['candidate_sha256'], + checked_qemu_reports=4, actual_thread_returns_verified=True, genuine_crypto_known_answers_passed=True, + wrong_known_answer_prevents_registration=True, original_component_code_unchanged=True, + original_dyld_prepare_entered=True, original_cache_loader_returned=True, + original_cache_loader_success=False, original_cache_load_address=0, + original_error_message_pointer_target='shared cache file open() failed', + native_4k=True, target_launchd_main_executed=False, functional_ipados_booted=False, hardware_tested=False) +(build / 'native-aes/checkpoints.json').write_text(json.dumps(result, indent=2) + '\n') +print('PASS: genuine AES answers/interfaces, wrong-answer service refusal and original native-4K dyld prepare/cache loader') +print('Cache directory is missing in the two-file root; no cache mapping, launchd main, iPadOS UI or hardware boot is claimed') diff --git a/scripts/check-native-apfs-file-checkpoints.py b/scripts/check-native-apfs-file-checkpoints.py new file mode 100644 index 0000000..589a270 --- /dev/null +++ b/scripts/check-native-apfs-file-checkpoints.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 +"""Qualify genuine APFS file signatures and native-4K faults, including refusal.""" +import hashlib +import json +from pathlib import Path +import struct +import sys + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'kits/NativePrelink')) +import native_prelink as np +build = ROOT / 'build' + + +def require(value, message): + if not value: + raise SystemExit('FAIL: ' + message) + + +kernel = build / 'native-prelink/native-apfs-data-candidate.macho' +data = kernel.read_bytes() +plan = json.loads(kernel.with_suffix('.json').read_text()) +require(hashlib.sha256(data).hexdigest() == plan['candidate_sha256'] == + '8db7506164cb7772c6af3e8bd21b4a9ee3a951c5c611550deae923ed1558628f' and + plan['native_sha256'] == 'bad8c94bce1fc6d995671b60f68605ac550cce6074ac6562b27e47162e97427d', 'exact native source/candidate identities') +require(plan['original_sha256'] == 'f18b4df865405f2a287b84977d7aca745e056d656d9fc9df1662970ce2969de7' and + not plan['source_inputs_modified'] and not plan['images_modified'], 'unchanged supplied ordinary ARM64 input') +image = np.macho(data) +for e in plan['extents']: + if e['segment'] == '__TEXT_EXEC': + at = np.backed_offset(image['segments'], e['va'], e['size'], len(data)) + require(hashlib.sha256(data[at:at + e['size']]).hexdigest() == plan['original_executable_sha256'][e['module']], 'original executable extent retained') + + +def report(name, candidate=plan['candidate_sha256']): + d = json.loads((build / name / 'gdb-stop.json').read_text()) + require(d['kernel_sha256'] == candidate and not d['bounded_timeout_interrupted'], 'bounded exact-candidate observation') + r = {k: int(v, 16) for k, v in d['system_registers'].items()} + require(r['TCR_EL1'] >> 14 & 3 == 0 and r['TCR_EL1'] >> 30 & 3 == 2 and + r['SCTLR_EL1'] & 1 and r['TTBR0_EL1'] and r['TTBR1_EL1'], 'active native 4K hardware roots') + return d + + +positive = report('darwinboot-native-apfs-file-plain') +fault = report('darwinboot-native-apfs-file-cdhash-fault') +mode0 = report('darwinboot-native-apfs-file-ro') +directory = report('darwinboot-native-apfs-file-directory') +old_lookup = report('darwinboot-native-apfs-file', '213d08f953fe38d6b2aab67a2de78aafb2d15af76bbd2cc7582358597a161e5a') +require(old_lookup['native_user_image_sha256'] == directory['native_user_image_sha256'] and + old_lookup['native_apfs_data_mount']['mkdir_result'] == 2 and + directory['native_apfs_data_mount']['mkdir_result'] == 0, 'same own program qualifies the intermediate CREATE lookup correction') +require(mode0['native_apfs_data_mount']['mkdir_result'] == 0 and + mode0['native_apfs_data_mount']['mount_result'] == 1 and + mode0['native_apfs_data_mount']['mount_nzcv'] >> 29 & 1 and + mode0['native_apfs_data_mount']['stage'] == 1, 'original user-root mount mode is genuinely refused') +fixture = build / 'native-apfs-file-final-fixture' +metadata = json.loads((fixture / 'file-probe.json').read_text()) +own = (fixture / 'launchd.macho').read_bytes() +trust = (fixture / 'offline-file-trust.bundle').read_bytes() +require(metadata['apfs_mount_mode'] == 1 and metadata['apfs_data_mount'] and + metadata['actual_volume_device'] == '/dev/disk1s1' and not (fixture / 'root.img').exists(), 'actual data-mount mode with a one-file RAM instrument') +require(positive['native_user_image_sha256'] == fault['native_user_image_sha256'] == hashlib.sha256(own).hexdigest() and + positive['trust_cache_bundle_sha256'] == fault['trust_cache_bundle_sha256'] == hashlib.sha256(trust).hexdigest(), 'identical signed own process and explicit offline file trust') +config = (build / 'darwinboot-native-apfs-file-plain/esp/EFI/CrabApple/crabapple.toml').read_text() +require('rd=md0 ' in config and 'initrd = "Payloads/launchd.macho"' in config, 'explicit own one-file RAM root') +require(positive['native_user_4k_abi_observed'] and positive['native_user_el0_observed'] and + all(v == 12 for v in positive['commpage_user64_shifts'].values()), 'actual own EL0 syscall and commpage 4K ABI') +m = positive['native_apfs_data_mount'] +require(m['mkdir_result'] == m['mount_result'] == m['mkdir_nzcv'] == m['mount_nzcv'] == 0 and m['stage'] == 2, 'real devfs directory and read-only APFS data mount succeed') +f = positive['native_file_probe'] +require(f['open_result'] >= 0 and f['fcntl_result'] == 0 and f['mmap_result'] % 4096 == 0 and f['mmap_result'] > 0 and + f['open_nzcv'] == f['fcntl_nzcv'] == f['mmap_nzcv'] == 0 and + f['signature_end'] == metadata['code_limit'] == 32768 and f['stage'] == 4 and + f['native_pages_read'] == metadata['expected_native_pages'] == 8 and + f['page_word_xor'] == metadata['expected_page_word_xor'], 'actual APFS open, F_ADDFILESIGS_RETURN, RX mmap and eight 4K accesses') +for d in (positive, fault): + require(d['original_crypto_initialized_observed'] and d['original_amfi_table_owner_observed'], 'genuine original crypto and AMFI remain active') + g = d['native_virtio_globals'] + require(g['_native_virtio_read_requests'] == 138 and g['_native_virtio_read_failures'] == 0 and + g['_native_virtio_sectors'] * 512 == 3303014400, 'actual original data-volume I/O') + h = d['native_signature_hash_cache'] + require(h['_cs_native4k_full_blocks'] == 5 and h['_cs_native4k_ready_blobs'] == 2, 'real original-block authentication before native-4K faults') +require(positive['code_signing_hash_counters']['_cs_validate_page_bad_hash'] == 0 and + positive['code_signing_hash_counters']['_cs_validate_page_no_hash'] == 0, 'positive signature policy remains satisfied') +source = ROOT / '.cache/firmware/target-volumes/systemos-mount/System/Library/Caches/com.apple.dyld/dyld_shared_cache_arm64.02' +cache = source.read_bytes() +require(hashlib.sha256(cache).hexdigest() == metadata['original_file_sha256'] == + '3d923c404fe519184a99042b60a4cb3d37247dcb8b070242ed3e670d4d523d04', 'supplied original cache file remains unchanged') +sigoff, sigsize = struct.unpack_from('I', signature, 8)[0]): + _, at = struct.unpack_from('>II', signature, 12 + index * 8) + magic, size = struct.unpack_from('>II', signature, at) + if magic == 0xfade0c02: + directories.append(signature[at:at + size]) +require(len(directories) == 1 and directories[0][39] == 14, 'actual original 16K CodeDirectory remains intact') +cd = directories[0] +hashoff = struct.unpack_from('>I', cd, 16)[0] +require(hashlib.sha256(cd).digest()[:20].hex() == metadata['original_cdhash'] and + all(hashlib.sha256(cache[i:i + 16384]).digest() == cd[hashoff + i // 16384 * 32:hashoff + (i // 16384 + 1) * 32] + for i in range(0, 32768, 16384)), 'original signing blocks and actual CodeDirectory identity') +event = fault['own_mapped_page_mutation'] +require(event['file_page'] == 4096 and event['byte_in_page'] == 7 and + event['before'] == cache[4103] and event['after'] == cache[4103] ^ 1 and + event['page_sha256_before'] == hashlib.sha256(cache[4096:8192]).hexdigest() and + event['code_directory_hash'] == metadata['original_cdhash'] == '9d037a68ad6490e103e44e47086e279a2923b04e' and + event['compiled_cdhash_field_offset'] == 0x58 and not event['registers_or_validation_state_modified'], 'actual signed original-cache identity and one-byte VM input fault') +require(fault['code_signing_hash_counters']['_cs_validate_page_bad_hash'] == 1 and + fault['code_signing_hash_counters']['_cs_validate_page_no_hash'] == 0 and + fault['fatal_user_exit']['cs_status'] == 'CS_KILLED' and + fault['fatal_user_exit']['namespace'] == 3 and fault['fatal_user_exit']['code'] == 2, 'genuine CODESIGNING bad-page refusal') +mount_results = struct.unpack('<5Q', bytes.fromhex(fault['native_user_memory']['_apfs_mount_results'])) +file_results = struct.unpack('<10Q', bytes.fromhex(fault['native_user_memory']['_file_probe_results'])) +require(mount_results == (0, 0, 0, 0, 2) and file_results[9] == 3 and + file_results[8] == 32768, 'fault occurs after genuine data mount, signature registration and mmap') +result = dict(candidate_sha256=plan['candidate_sha256'], native_sha256=plan['native_sha256'], + checked_qemu_reports=5, original_executable_code_modified=False, target_cache_modified=False, + real_apfs_data_mount=True, native_4k=True, original_signing_block_bytes=16384, + actual_rx_fault_pages=8, dma_reads=138, dma_failures=0, + cache_page_fault_cdhash=event['code_directory_hash'], actual_bad_hash_refused=True, + original_user_root_mount_mode_refused=True, explicit_offline_ram_root=True, + production_root_authenticated=False, original_dyld_cache_loader_qualified=False, + functional_ipados_booted=False, hardware_tested=False) +(build / 'native-apfs-file-checkpoints.json').write_text(json.dumps(result, indent=2) + '\n') +print('PASS: genuine APFS data mount, original 16K-signed cache file and 8 native-4K RX faults; identity-qualified bad page is refused') +print('Explicit own RAM root/file trust is an offline instrument; production root, complete dyld cache, UI and hardware remain unqualified') diff --git a/scripts/check-native-arch-timer-checkpoints.py b/scripts/check-native-arch-timer-checkpoints.py new file mode 100644 index 0000000..aea19e5 --- /dev/null +++ b/scripts/check-native-arch-timer-checkpoints.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +"""Require real architectural timer boot without the BCM-specific early write.""" +import hashlib +import json +from pathlib import Path +import runpy + +ROOT = Path(__file__).resolve().parents[1] +base = runpy.run_path(str(ROOT / 'scripts/check-original-launchd-main-checkpoints.py')) +require = base['require'] +expected = base['candidate']('native-owner-arch-timer-candidate', + '8bd4387f7a1b83ceb76ed9d8d7ab4c32c497a48de00eb32e8d080d6c80582a90', + '655825844686147147aba8e76ca02c2871a1b95df170e332e92a49e2df604d47') +directory = ROOT / 'build/native-arch-timer' +r = json.loads((directory / 'qualification.json').read_text()) +sequence = bytes.fromhex('af0038d5ef1d4092000088d200bc70d30000019100080f8b011e8052010000b9df3f03d5') +for mode, stem, count in (('legacy', 'legacy-start', 1), ('architectural', 'architectural-start', 0)): + obj = (directory / (stem + '.o')).read_bytes() + args = json.loads((directory / (stem + '.o.json')).read_text().rstrip(',\n'))['arguments'] + require(hashlib.sha256(obj).hexdigest() == r[mode + '_object_sha256'] and obj.count(sequence) == count and + 'XNU_NATIVE_ARCH_TIMER=' + str(1 - count) in args, + 'Actual assembly command/object must retain legacy or remove the complete specific MMIO sequence') +image = (ROOT / 'build/native-prelink/native-owner-arch-timer-candidate.macho').read_bytes() +require(image.count(sequence) == 0 and r['linked_sequence_count'] == 0, 'Linked candidate contains no legacy routing sequence') +d = base['report']('owner-hvf-arch-timer-launchd-main') +require(d['kernel_sha256'] == expected and d['native_4k_mmu_observed'] and not d['bounded_timeout_interrupted'] and + d['stop_symbol'] == 'original-launchd-main' and d['original_launchd_main_executed'] and + d['followed_main_entry']['first_instruction_completed'] and int(d['pstate'], 16) & 31 == 0 and + d['direct_user_commpage_shifts'] == {'fffff4025:1': 12, 'fffff4037:1': 12}, + 'Actual original launchd must still execute at native-4K EL0') +a = d['armcpu_globals'] +require(a['_armcpu_registered'] == a['_armcpu_gic_ready'] == a['_gCPUsRunning'] == 1 and + a['_armcpu_timer_interrupts'] == 243286 and a['_armcpu_ipi_interrupts'] == 0, + 'Real architectural GIC/timer must interrupt; no observed IPI or SMP claim') +require('owner_cache_carrier' not in d and d['actual_root_composition']['_native_owner_cache_native_union'] == 1 and + d['owner_root_binding_audit']['boot_root_accepted'] and d['owner_boot_manifest']['status'] == 0 and + d['original_amfi_table_owner_observed'] and d['actual_signature_failures'] == + dict(_cs_validate_page_bad_hash=0, _cs_validate_page_no_hash=0) and + not d['functional_ipados_boot_demonstrated'] and not d['sheng_hardware_tested'], + 'Preserve owner authentication, original policy and honest offline scope') +print('PASS: legacy BCM pre-MMU write absent from actual architectural entry object and linked candidate') +print('PASS: 243286 actual timer interrupts, original launchd LC_MAIN at EL0, native hardware 4K and user shifts 12/12') diff --git a/scripts/check-native-data-mount-path.py b/scripts/check-native-data-mount-path.py new file mode 100644 index 0000000..aff6ac7 --- /dev/null +++ b/scripts/check-native-data-mount-path.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +"""Host-check the actual native Data mount decision path with API fault injection. +The API stand-ins do not execute APFS or qualify hardware integration. +""" +from pathlib import Path +import subprocess,json,hashlib +root=Path(__file__).resolve().parents[1];out=root/'build/cpu-desktop/data-mount-path';out.mkdir(parents=True,exist_ok=True) +s=(root/'.cache/xnu-work/bsd/kern/kern_exec.c').read_text();start=s.index('static int\nnative_owner_data_prepare(');end=s.index('\n#endif',s.index('\ndone:',start)) +function=s[start:end] +prelude=r''' +#include +#include +#include +#include +#include +#include +#define XNU_NATIVE_RAM_BLOCK 1 +#define XNU_OWNER_DATA_VOLUME_UUID "18AB5557-DC2D-469B-8758-855E60F5B0DF" +#define NODEV ((dev_t)-1) +#define NULLVP NULL +#define MNT_RDONLY 1 +#define strlcpy(d,s,n) ((void)snprintf(d,n,"%s",s)) +#include +typedef void *proc_t;typedef uintptr_t user64_addr_t; +struct statfs {char f_fstypename[16];};struct fake {struct statfs s;}; +typedef struct fake *vnode_t;typedef struct fake *mount_t; +struct mount_args {uintptr_t type,path;unsigned flags;uintptr_t data;}; +int native_owner_data_status,native_owner_data_stage,native_owner_data_readonly; +dev_t native_owner_data_device;uint64_t native_owner_data_seed_base,native_owner_data_seed_size; +uint8_t native_owner_data_seed_sha256[32];char native_owner_data_device_path[64]; +static int scenario,mounts;static struct fake object; +static int native_ram_block_volume_seed(const char *uuid,dev_t *dev,char *path,size_t n,uint64_t *base,uint64_t *bytes,uint8_t *hash){ + assert(!strcmp(uuid,XNU_OWNER_DATA_VOLUME_UUID)); + if(scenario==1)return 0; + *dev=42;snprintf(path,n,"/dev/disk2s1");*base=0;*bytes=scenario==2?4097:0x80000000;memset(hash,0xa5,32);return 1; +} +static int copyout(const void *p,uintptr_t to,size_t n){if(scenario==8)return EFAULT;memcpy((void*)to,p,n);return 0;} +static int mount(proc_t p,struct mount_args *a,int32_t *result){ + (void)p;(void)result;++mounts; + assert(!strcmp((char*)a->type,"apfs")&&!strcmp((char*)a->path,"/private/var")&&!a->flags); + uintptr_t spec;memcpy(&spec,(void*)a->data,8);assert(!strcmp((char*)spec,"/dev/disk2s1")); + return scenario==3?EACCES:0; +} +static void *vfs_context_current(void){return NULL;} +static int vnode_lookup(const char *p,int flags,vnode_t *v,void *ctx){(void)flags;(void)ctx;assert(!strcmp(p,"/private/var"));*v=&object;return scenario==4?ENOENT:0;} +static mount_t vnode_mount(vnode_t v){return v;} +static vnode_t vfs_devvp(mount_t m){return m;} +static dev_t vnode_specrdev(vnode_t v){(void)v;return scenario==5?43:42;} +static void vnode_put(vnode_t v){(void)v;} +static unsigned vfs_flags(mount_t m){(void)m;return scenario==6?MNT_RDONLY:0;} +static struct statfs *vfs_statfs(mount_t m){snprintf(m->s.f_fstypename,sizeof(m->s.f_fstypename),"%s",scenario==7?"hfs":"apfs");return &m->s;} +''' +main=r''' +int main(void){ + const int expected[]={0,ENOENT,EINVAL,EACCES,ENOENT,EINVAL,EINVAL,EINVAL,EFAULT}; + for(scenario=0;scenario<9;++scenario){ + uint8_t scratch[2048]={0};mounts=0;native_owner_data_stage=0; + assert(native_owner_data_prepare(NULL,(uintptr_t)scratch)==expected[scenario]); + assert(native_owner_data_status==expected[scenario]); + if(scenario==0){assert(native_owner_data_stage==4&&mounts==1&&native_owner_data_device==42&&!native_owner_data_readonly); + assert(native_owner_data_seed_base==0&&native_owner_data_seed_size==0x80000000);for(unsigned i=0;i<32;++i)assert(native_owner_data_seed_sha256[i]==0xa5);} + if(scenario==1||scenario==2||scenario==8)assert(mounts==0); + } + puts("PASS: actual native mount path without md0; 8 provider/range/mount/copyout/device/readonly/filesystem refusal paths"); +} +''' +(out/'test.c').write_text(prelude+function+main) +subprocess.run(['clang','-std=c11','-Wall','-Wextra','-Werror','-O1','-g','-fsanitize=address,undefined',str(out/'test.c'),'-o',str(out/'test')],check=True,timeout=60) +r=subprocess.run([str(out/'test')],check=True,text=True,capture_output=True,timeout=30) +(out/'report.json').write_text(json.dumps({'function_sha256':hashlib.sha256(function.encode()).hexdigest(),'passed':True,'cases':9,'apis_mocked':True,'actual_apfs':False,'hardware_observed':False},indent=2)+'\n');print(r.stdout,end='') diff --git a/scripts/check-native-fp-context.py b/scripts/check-native-fp-context.py new file mode 100644 index 0000000..7bd8de6 --- /dev/null +++ b/scripts/check-native-fp-context.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +"""Verify actual full SIMD preservation on the original-root native-4K instrument.""" +import hashlib +import json +from pathlib import Path +import struct +import subprocess + +ROOT=Path(__file__).resolve().parents[1];OUT=ROOT/'build/native-fp-context' +def require(value,reason): + if not value:raise SystemExit(reason) +def sha(path):return hashlib.sha256(Path(path).read_bytes()).hexdigest() +build=json.loads((OUT/'build-report.json').read_text()) +manifest=json.loads((OUT/'test-inputs.json').read_text()) +data=json.loads((OUT/'data-report.json').read_text()) +path=ROOT/'build/darwinboot-owner-hvf-fp-context-system-final/gdb-stop.json';d=json.loads(path.read_text()) +plan=json.loads((ROOT/'build/native-prelink/native-owner-legacy-display-info-candidate.json').read_text()) +require(sha(OUT/'air-native')==sha(OUT/'launchd.macho')==build['process_sha256']==data['process_sha256']==d['native_user_image_sha256'] and + sha(OUT/'test-inputs.json')==build['manifest_sha256']==data['manifest_sha256'] and + sha(ROOT/'scripts/qemu-entry/native-fp-context.inc')==manifest['source_sha256'], + 'Exact own source, signed process and staged instrument manifest') +require(sha(OUT/'compiler-data-32.cdr')==data['sha256']==d['scratch_data']['actual_seed_sha256'] and + sha(OUT/'offline-trust.bundle')==build['trust_bundle_sha256']==d['trust_cache_bundle_sha256'], + 'Exact real mutable Data input and trust bundle') +original=(ROOT/'build/original-static-system-trust/original-static-system.bundle').read_bytes() +trust=(OUT/'offline-trust.bundle').read_bytes() +require(struct.unpack_from('start and frequency==int(d['system_registers']['CNTFRQ_EL0'],16) and 0.025<=(end-start)/frequency<5, + 'Actual architectural counter confirms bounded blocking-time execution') +require(d['original_amfi_table_owner_observed'] and d['owner_root_binding_audit']['boot_root_accepted'] and + d['owner_root_binding_audit']['_native_owner_root_apple_autherr']==80 and + d['actual_signature_failures']==dict(_cs_validate_page_bad_hash=0,_cs_validate_page_no_hash=0), + 'Original AMFI, original Apple rejection, independent original-root acceptance and no code-sign hash failures') +(OUT/'checkpoint.json').write_text(json.dumps(dict(checkpoint_sha256=sha(path),candidate_sha256=plan['candidate_sha256'], + process_sha256=build['process_sha256'],simd_registers=32,bits_per_register=128,iterations=iterations, + blocking_thread_switches=32,elapsed_seconds=(end-start)/frequency,native_cpu_4k=True, + original_root=True,mockfs_boot=False,original_desktop=False,adreno_execution=False,hardware_tested=False),indent=2)+'\n') +print('PASS: all 32 full SIMD registers survive real syscalls and 32 blocking thread switches; native 4K/original root/AMFI intact') +print('CPU context instrument only; it does not qualify system UI or Adreno execution') diff --git a/scripts/check-native-integrity-checkpoints.py b/scripts/check-native-integrity-checkpoints.py new file mode 100644 index 0000000..ef446ce --- /dev/null +++ b/scripts/check-native-integrity-checkpoints.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python3 +"""Verify saved, bounded QEMU observations of native-4K signed-code validation.""" +import argparse +import hashlib +import json +from pathlib import Path +import struct + +ROOT = Path(__file__).resolve().parents[1] +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument('--build', type=Path, default=ROOT / 'build') +args = parser.parse_args() +build = args.build + + +def require(condition, message): + if not condition: + raise SystemExit('FAIL: ' + message) + + +candidate = json.loads((build / 'native-prelink/native-hash-cache-runtime.json').read_text()) +kernel_sha = candidate['candidate_sha256'] +fixture = (build / 'native-user-16k-signature/launchd.macho').read_bytes() +trust = json.loads((build / 'native-user-16k-signature/trustcache.json').read_text()) +require(hashlib.sha256(fixture).hexdigest() == trust['process_sha256'], 'own input identity') +observed = [] + + +def report(name, mmu=True): + d = json.loads((build / name / 'gdb-stop.json').read_text()) + require(d['kernel_sha256'] == kernel_sha, name + ': different kernel') + require(not d['bounded_timeout_interrupted'], name + ': deadline interrupted') + r = {k: int(v, 16) for k, v in d['system_registers'].items()} + require((r['TCR_EL1'] >> 14 & 3) == 0 and (r['TCR_EL1'] >> 30 & 3) == 2, + name + ': active hardware granules') + require(r['SCTLR_EL1'] & 1 and r['TTBR0_EL1'] and r['TTBR1_EL1'], name + ': active MMU') + if mmu: + require(d['native_4k_mmu_observed'] and not d['ipados_userspace_boot_demonstrated'], + name + ': observation scope') + require(d['original_amfi_table_owner_observed'] and d['original_crypto_initialized_observed'], + name + ': original security providers') + observed.append(name) + return d + + +def user(d): + require(d['native_user_el0_observed'] and int(d['pstate'], 16) & 15 == 0, 'actual EL0') + require(d['native_user_4k_abi_observed'], 'actual userspace 4K ABI') + require(d['native_user_syscalls'] == dict(pid=1, sysctl_result=0, page_size=4096, + old_length=4, getpid_carry=0, sysctl_carry=0), 'actual syscalls') + require(set(d['commpage_user64_shifts'].values()) == {12}, 'both commpage shifts') + + +def cache(d): + c = d['native_signature_hash_cache'] + require(c['_cs_native4k_hash_cache_version'] == 1, 'integrity cache version') + return tuple(c[n] for n in ('_cs_native4k_full_blocks', '_cs_native4k_page_checks', + '_cs_native4k_ready_blobs')) + + +positive = report('darwinboot-native-4k-16k-integrity-runtime') +user(positive) +require(positive['native_user_image_sha256'] == trust['process_sha256'], '16K-signed process identity') +require(positive['trust_cache_bundle_sha256'] == trust['bundle_sha256'], 'explicit test trust identity') +require(cache(positive) == (3, 3, 1), 'whole-block authentication before native page checks') +require(positive['code_signing_hash_counters']['_cs_validate_page_bad_hash'] == 0, 'positive hash result') + +inspection = report('darwinboot-native-cache-readonly', mmu=False) +require(int(inspection['x0'], 16) == 0, 'real authentication function returned success') +cd = bytes.fromhex(inspection['entry_indirect_memory']['1:80:200']['bytes']) +length = struct.unpack_from('>I', cd, 4)[0] +require(length <= len(cd) and cd[39] == 14, 'unchanged guest CodeDirectory granule') +require(hashlib.sha256(cd[:length]).hexdigest()[:40] == trust['cdhash'], 'unchanged guest CDHash') +limit = struct.unpack_from('>I', cd, 32)[0] +require(limit == 32976 and struct.unpack_from('>I', cd, 28)[0] == 3, 'original signed extent') +expected = b''.join(hashlib.sha256(fixture[o:min(o + 4096, limit)]).digest() + for o in range(0, limit, 4096)) +actual = inspection['entry_indirect_memory']['1:b8:1000'] +digests = bytes.fromhex(actual['bytes']) +require(digests[:len(expected)] == expected and not any(digests[len(expected):]), + 'derived page digests and zeroed allocation padding') +t = actual['translation'] +require(t['leaf_level'] == 3 and t['leaf_bytes'] == 4096 and t['ap'] == 2 and t['pxn'] and t['uxn'], + 'actual hardware read-only, non-executable 4K digest mapping') + +for quarter in range(4): + d = report(f'darwinboot-native-16k-bad-quarter-{quarter}') + mutated = bytearray(fixture) + mutated[0x380 if quarter == 0 else quarter * 4096] ^= 1 + require(d['native_user_image_sha256'] == hashlib.sha256(mutated).hexdigest(), 'tamper identity') + require(not d['native_user_el0_observed'] and cache(d) == (0, 0, 0), 'no unverified publication/EL0') + require(d['code_signing_hash_counters']['_cs_validate_page_bad_hash'] == 1, 'full-block tamper refusal') + require((d['exec_policy_refusal']['namespace'], d['exec_policy_refusal']['code']) == (9, 8), + 'actual EXEC/SECURITY_POLICY refusal') + +regression = report('darwinboot-native-hash-cache-4k-regression') +user(regression) +require(cache(regression) == (0, 0, 0), 'original 4K integrity path retained') + +backing = report('darwinboot-native-cache-post-auth-virtual-md0') +user(backing) +mutation = backing['own_input_mutation'] +require(mutation['input_sha256_before'] == trust['process_sha256'] and mutation['byte_offset'] == 0x380, + 'identified backing-input mutation') +require(not mutation['registers_or_validation_state_modified'] and mutation['before'] ^ mutation['after'] == 1, + 'backing mutation scope') +require(backing['code_signing_hash_counters']['_cs_validate_page_bad_hash'] == 0, + 'authentic already-cached vnode pages still validate') + +fault = report('darwinboot-native-cache-mapped-page-refusal') +mutation = fault['own_mapped_page_mutation'] +require(mutation['page_sha256_before'] == hashlib.sha256(fixture[:4096]).hexdigest(), 'actual fault-page identity') +require(mutation['file_page'] == 0 and mutation['byte_in_page'] == 0x380 and + mutation['before'] ^ mutation['after'] == 1 and not mutation['registers_or_validation_state_modified'], + 'actual mapped-page mutation scope') +require(cache(fault) == (3, 2, 1), 'post-authentication native page check') +require(fault['code_signing_hash_counters']['_cs_validate_page_bad_hash'] == 1 and + not fault['native_user_el0_observed'], 'actual fault-page rejection before EL0') +require(fault['stop_symbol'] == '_panic_with_options' and + (fault['fatal_user_exit']['namespace'], fault['fatal_user_exit']['code']) == (2, 12), + 'observed initproc failure; do not relabel it CODESIGNING/INVALID_PAGE') + +result = dict(kernel_sha256=kernel_sha, checkpoints=observed, checkpoint_count=len(observed), + source='XNU 10063.141.1 with explicit native integrity-cache opt-in', + own_16k_signed_process_el0=True, native_4k_hardware_observed=True, + original_codedirectory_unchanged=True, digest_mapping_readonly_nx=True, + all_four_pre_authentication_quarters_refused=True, mapped_page_tamper_refused=True, + target_ipados_userspace_executed=False, hardware_tested=False) +(build / 'native-integrity-checkpoints.json').write_text(json.dumps(result, indent=2) + '\n') +print(f'PASS: {len(observed)} saved QEMU checkpoints; original 16K signing blocks, native 4K hardware/EL0') +print('Own static fixture only; target iPadOS execution and sheng hardware remain unverified') diff --git a/scripts/check-native-legacy-display.py b/scripts/check-native-legacy-display.py new file mode 100644 index 0000000..d7671dc --- /dev/null +++ b/scripts/check-native-legacy-display.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 +"""Qualify original Legacy lifecycle and preserve its real backend refusal.""" +import hashlib +import json +from pathlib import Path +import sys + +ROOT = Path(__file__).resolve().parents[1] +OUT = ROOT / 'build/native-legacy-display' +sys.path.insert(0, str(ROOT / 'kits/NativePrelink')) +from native_prelink import macho, backed_offset + + +def require(value, reason): + if not value: + raise SystemExit(reason) + + +def sha(path): + return hashlib.sha256(Path(path).read_bytes()).hexdigest() + + +build = json.loads((OUT / 'build-report.json').read_text()) +for name, digest in build['source_sha256'].items(): + require(sha(ROOT / name) == digest, 'Exact current source: ' + name) +for name, key in [('OriginalLegacyFramebufferABI.h', 'generated_abi_sha256'), + ('NativeLegacyDisplay.o', 'compiled_object_sha256'), + ('NativeLegacyDisplay.late.o', 'late_object_sha256'), + ('libNativeLegacyDisplay.late.a', 'archive_sha256')]: + require(sha(OUT / name) == build[key], 'Exact actual build: ' + name) +require(build['virtual_entries'] == 358 and build['owned_entries'] == + [0, 1, 7, 84, 211, 213, 245, 246, 288, 306, 307, 337, 338, 345], + 'Only qualified own methods and guarded abstract leaves') + +candidate = ROOT / 'build/native-prelink/native-owner-legacy-display-gop-candidate.macho' +plan = json.loads(candidate.with_suffix('.json').read_text()) +raw = candidate.read_bytes() +image = macho(raw) +require(sha(candidate) == plan['candidate_sha256'], 'Exact observed candidate') +native = ROOT / 'build/xnu/ios-obj/DEVELOPMENT_ARM64_BCM2837/mach.development.bcm2837.unstripped.noctf' +require(sha(native) == plan['native_sha256'], 'Current native source kernel') +for family, digest in plan['original_executable_sha256'].items(): + extents = [e for e in plan['extents'] if e['module'] == family and e['segment'] == '__TEXT_EXEC'] + require(len(extents) == 1, 'Unambiguous original executable extent') + e = extents[0] + at = backed_offset(image['segments'], e['va'], e['size'], len(raw)) + require(hashlib.sha256(raw[at:at + e['size']]).hexdigest() == digest, + 'Complete original instructions unchanged: ' + family) + +observations = {} +for case in ['init', 'start', 'headless', 'default-off', 'start-error']: + path = ROOT / ('build/darwinboot-owner-hvf-legacy-display-' + case + '-gop-control/gdb-stop.json') + d = json.loads(path.read_text()) + require(d['kernel_sha256'] == plan['candidate_sha256'] and + d['qemu_accelerator'] == 'hvf' and not d['bounded_timeout_interrupted'], + 'Same actual bounded candidate: ' + case) + regs = {k: int(v, 16) for k, v in d['system_registers'].items()} + require(d['native_4k_mmu_observed'] and (regs['TCR_EL1'] >> 14) & 3 == 0 and + (regs['TCR_EL1'] >> 30) & 3 == 2 and regs['SCTLR_EL1'] & 1 and + regs['TTBR0_EL1'] and regs['TTBR1_EL1'], 'Actual native CPU 4K: ' + case) + state = d['native_legacy_display'] + require(not state['gop_map'] and not state['gop_width'], 'Backend gate remains disabled: ' + case) + require(not state['guard_slot'], 'No unclassified backend leaf executed: ' + case) + video = d['actual_boot_video']['pe_video'] + if case == 'headless': + require(state['stage'] == 2 and state['meta_status'] == 0 and + not state['object'] and not any(video.values()), 'No-video creation refusal') + elif case == 'default-off': + require(state['stage'] == 0 and state['meta_status'] == -1 and + d['original_launchd_main_executed'] and d['followed_main_entry']['first_instruction_completed'] and + d['original_amfi_table_owner_observed'] and d['owner_root_binding_audit']['boot_root_accepted'] and + d['owner_root_binding_audit']['_native_owner_root_apple_autherr'] == 80 and + set(d['direct_user_commpage_shifts'].values()) == {12} and + d['actual_signature_failures'] == dict(_cs_validate_page_bad_hash=0, _cs_validate_page_no_hash=0), + 'Default-off original launchd/AMFI/root/native user 4K') + else: + require(state['meta_status'] == 0 and state['init_result'] == 1 and + state['object_size'] == 0x18a0 and state['parent_match'] == 1 and state['object'], + 'Real original init/object/parent: ' + case) + if case == 'start-error': + require(int(d['pc'], 16) == 0xfffffff0057870b0 and int(d['x30'], 16) == 0xfffffff005787f1c and + d['argument_memory']['x1'] == 'start_hardware failed\n' and state['stage'] == 5, + 'Real original failure and exact original callsite') + else: + require(d['stop_symbol'] == '_native_legacy_display_checkpoint' and + state['stage'] == 8 and state['released'] == 1, 'Actual completed release: ' + case) + if case.startswith('start'): + require(state['probe_match'] == 1 and not state['start_result'] and + state['surface_status'] == 0 and state['surface_found'] == 1 and + state['video_base'] == video['v_baseAddr'] and + state['video_bytes'] == video['v_rowBytes'] * video['v_height'] == 1920000, + 'Actual GOP resources and real IOSurface dependency; original start refuses') + observations[case] = dict(checkpoint_sha256=sha(path), state=state, actual_tcr=regs['TCR_EL1']) + +for case in ['mapped', 'publication']: + path = ROOT / ('build/darwinboot-owner-hvf-legacy-display-start-gop-' + case + '/gdb-stop.json') + d = json.loads(path.read_text()) + require(d['kernel_sha256'] == plan['candidate_sha256'] and not d['bounded_timeout_interrupted'] and + d['qemu_accelerator'] == 'hvf' and d['native_4k_mmu_observed'], 'Actual gated GOP observation') + state = d['native_legacy_display'] + video = d['actual_boot_video']['pe_video'] + require(state['stage'] == 5 and state['init_result'] == state['probe_match'] == state['parent_match'] == 1 and + state['surface_status'] == 0 and state['surface_found'] == 1 and not state['guard_slot'] and + not state['released'] and state['object_size'] == 6304, 'Original start reaches actual backend') + require(state['gop_map'] and state['gop_virtual'] and + state['gop_physical'] == state['video_base'] == video['v_baseAddr'] and + state['gop_bytes'] == state['video_bytes'] == video['v_rowBytes'] * video['v_height'] == 1920000 and + state['gop_width'] == video['v_width'] == 800 and state['gop_height'] == video['v_height'] == 600, + 'Real retained framebuffer mapping and original geometry getter use actual GOP mode') + if case == 'mapped': + require(d['stop_symbol'] == '_native_legacy_gop_mapping_checkpoint', 'Actual mapped checkpoint') + else: + require(int(d['pc'], 16) == 0xfffffff0057880d4, 'Original service-publication call not yet executed') + observations['gop-' + case] = dict(checkpoint_sha256=sha(path), state=state) + +(OUT / 'native-checkpoint.json').write_text(json.dumps(dict( + candidate_sha256=plan['candidate_sha256'], native_sha256=plan['native_sha256'], + source_sha256=build['source_sha256'], observations=observations, + actual_original_init_and_release=True, actual_original_start=False, + original_start_failure='start_hardware failed', original_instructions_modified=False, + actual_gop_mapping_and_mode=True, original_publication_boundary_reached=True, + display_backend_complete=False, presentation=False, system_metal_device=False, + original_desktop=False, hardware_tested=False), indent=2) + '\n') +print('PASS: actual GOP mapping/original mode getter and original pre-publication boundary; gated-off backend refuses') +print('Headless refuses; default-off original launchd/AMFI/root/native CPU/user 4K preserved; desktop absent') diff --git a/scripts/check-native-legacy-logical-pm.py b/scripts/check-native-legacy-logical-pm.py new file mode 100644 index 0000000..fce5a3b --- /dev/null +++ b/scripts/check-native-legacy-logical-pm.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +"""Check actual IOKit logical scanout callbacks, controls and next refusal.""" +import hashlib +import json +from pathlib import Path +import struct +import sys +ROOT=Path(__file__).resolve().parents[1] +sys.path.insert(0,str(ROOT/'kits/NativePrelink')) +from native_prelink import macho,backed_offset +def require(v,s): + if not v:raise SystemExit(s) +def load(p):return json.loads(p.read_text()) +def sha(p):return hashlib.sha256(p.read_bytes()).hexdigest() +build=load(ROOT/'build/native-legacy-display/build-report.json') +for name,digest in build['source_sha256'].items():require(sha(ROOT/name)==digest,'Exact compiled source '+name) +require(sha(ROOT/'build/native-legacy-display/libNativeLegacyDisplay.late.a')==build['archive_sha256'] and + build['owned_entries']==[0,1,7,14,84,86,87,141,155,184,188,211,213,245,246,288,306,307,337,338,345], + 'Exact compiled archive and qualified lifecycle overrides') +kernel=ROOT/'build/native-prelink/native-owner-legacy-display-pm-candidate.macho';plan=load(kernel.with_suffix('.json')) +raw=kernel.read_bytes();image=macho(raw) +require(sha(kernel)==plan['candidate_sha256'] and sha(ROOT/'build/xnu/ios-obj/DEVELOPMENT_ARM64_BCM2837/mach.development.bcm2837.unstripped.noctf')==plan['native_sha256'], 'Exact current native candidate') +for module,digest in plan['original_executable_sha256'].items(): + e=next(e for e in plan['extents'] if e['module']==module and e['segment']=='__TEXT_EXEC') + o=backed_offset(image['segments'],e['va'],e['size'],len(raw)) + require(hashlib.sha256(raw[o:o+e['size']]).hexdigest()==digest,'Unchanged whole original executable '+module) +paths=[ROOT/('build/darwinboot-owner-hvf-offline-legacy-swap-pm'+suffix+'/gdb-stop.json') for suffix in ('','-submit','-validate')] +observations=[load(p) for p in paths] +for d in observations: + l=d['native_legacy_display'] + require(d['kernel_sha256']==plan['candidate_sha256'] and not d['bounded_timeout_interrupted'] and + d['native_4k_mmu_observed'] and d['original_amfi_table_owner_observed'] and + d['owner_root_binding_audit']['boot_root_accepted'] and + d['actual_signature_failures']==dict(_cs_validate_page_bad_hash=0,_cs_validate_page_no_hash=0), 'Native 4K and original AMFI/root') + require(l['pm_stage']==4 and l['pm_status']==0 and l['pm_callbacks']==1 and l['pm_calls']==4 and + l['pm_ordinal']==1 and l['stage']==10 and l['guard_slot']==0,'Actual IOKit registration/request/callback and original return') +d,submit,validation=observations +values=struct.unpack('<16Q',bytes.fromhex(d['entry_user_state_candidates'][0]['register_memory']['13:80']['bytes'])) +require(d['stop_symbol']=='_exception_triage' and values[:6]==(0x47505357,1,4096,4096,800,600) and + values[7:12]==(480000,0,1,0,0xe00002c2) and values[13]>=24000000 and values[14:]==(1,4), 'Real GPU pixels and deeper original BadArgument after one actual second') +tail=bytes.fromhex(submit['entry_regions']['0:1000:800']['bytes']) +require(int(submit['pc'],16)==0xfffffff00578f224 and struct.unpack_from('<2I',tail,5836-4096)==(1,0), 'Actual original controller scheduling state is 1; no panel power completion') +require(validation['function_return_stack_matched'] and int(validation['x0'],16)==0 and + int(validation['finished_function_entry'],16)==0xfffffff005795bb4 and int(validation['pc'],16)==0xfffffff005790ed0, + 'Original mandatory backend validator really returns false to its unchanged caller') +surface=bytes.fromhex(validation['entry_indirect_memory']['1:a40:100']['bytes']) +require(struct.unpack_from('<3Q',surface,88)==(800,600,3200) and struct.unpack_from('1 and v[12]>1 and v[4]!=v[12] and v[5:7]==v[13:15]==(4096,4096) and v[7] and v[15] and v[7]!=v[15] and v[8] and v[16] and v[8]!=v[16] and v[9:12]==(4096,0,4) and v[17:23]==(4096,1,4,0,0,1),'Both actual workers qualify their new original-shader path, real original IOSurfaces, foreign refusal, finish and actual waitpid exit zero') +b=d['native_gpu_broker'];require(b['stage']==3 and not b['status'] and b['clients']==b['closed']==2 and not b['active'] and not b['failures'] and b['refusals']==1 and b['requests']==137 and b['reply_bytes']==144974,'Exact real broker operations and one original foreign-resource refusal') +c=b['actual_cpu_contexts'];require(len(c)==2 and {r['pid'] for r in c}=={v[4],v[12]} and len({r['TTBR0_EL1'] for r in c})==len({r['task'] for r in c})==2,'Two distinct actual worker tasks and TTBR0 contexts') +for r in c:require(r['native_cpu_4k_accepted'] and (int(r['TCR_EL1'],16)>>14)&3==0 and (int(r['TCR_EL1'],16)>>30)&3==2 and int(r['SCTLR_EL1'],16)&1 and int(r['TTBR0_EL1'],16) and int(r['TTBR1_EL1'],16),'Actual active native CPU TG0/TG1 4K, MMU and TTBR state') +require(not any(d['native_gpu_shared_events'].values()) and d['native_original_provider']['stage']==0 and not d['native_original_provider']['allocated'],'No unrelated capability tests or original provider allocation/publication') +require(provenance['host_exit']==0 and provenance['api_validation_requested'] and provenance['report_sha256']==sha(folder/'host-enabled-report.json') and provenance['candidate_sha256']==plan['candidate_sha256'] and sha(ROOT/'build/host-metal/host-metal-originals')==provenance['process_sha256'],'Exact independent actual API-validation host provenance') +for name,digest in provenance['backend_sources'].items():require(sha(ROOT/name)==digest,'Exact host backend source '+name) +ops=host['operations'];require(host['transport_healthy'] and host['session_finished'] and host['opened_sessions']==host['closed_sessions']==2 and not host['cpu_pixel_renderer'] and not host['adreno_execution'] and not host['guest_system_device_discovery'] and not host['original_desktop'] and not host['failed_commands'] and not host['open_encoders'] and not host['aborted_encoders'] and not host['pending_submissions'] and len(ops)==b['requests'] and sum(32+o['result_bytes'] for o in ops)==b['reply_bytes'] and sum(bool(o['error']) for o in ops)==1,'Independent complete operation/result accounting and actual healthy cleanup') +require(all(o['request']==i+1 for i,o in enumerate(ops)),'Exact host global framing') +sessions={r['session'] for r in c} +for sid in sessions: + sequence=[o['client_request'] for o in ops if o['session']==sid];require(sequence==list(range(1,len(sequence)+1)),'Refused frontend descriptors do not consume actual RPC sequence numbers') +rows=host['vertex_pipelines'];require(len(rows)==2 and {r['session'] for r in rows}==sessions,'One real original vertex pipeline per isolated worker') +for r in rows:require(r['vertex']=='std_vert0_lpf' and r['fragment']=='attachment_clear_frag_lpf' and r['attributes']==[dict(index=i,format=f,offset=o,buffer=0) for i,f,o in ((0,31,0),(1,29,16),(3,31,24))] and r['layouts']==[dict(index=0,stride=40,step=1,rate=1)],'Actual host Metal descriptor matches original native SDK fields') +black=bytes(4);red=bytes((0,0,255,255));left=(black*32+red*32)*64;right=(red*32+black*32)*64 +left_hash=hashlib.sha256(left).hexdigest();right_hash=hashlib.sha256(right).hexdigest();reads=host['texture_readbacks'] +require(len(reads)==8 and sum(r['sha256']==left_hash for r in reads)==2 and sum(r['sha256']==right_hash for r in reads)==2,'Four complete geometry-dependent GPU readbacks, every one of 16384 new vertex pixels independently checked') +textures={r['texture'] for r in reads if r['sha256'] in (left_hash,right_hash)} +require(len(textures)==2 and not any(r['texture'] in textures for r in host['texture_uploads']),'New vertex output textures never receive uploaded CPU pixels') +for texture in textures:require([r['sha256'] for r in reads if r['texture']==texture]==[left_hash,right_hash],'Each actual worker updates real vertex buffer coverage on the same GPU target') +require(len(host['completed_commands'])==7 and all(r['status']==4 and 0= 29 and + ram['state']['_native_ram_block_failures'] == 0 and not ram['input']['initial_seed_signature_verified'] and + not ram['input']['production_storage_qualified'], 'Actual RAM storage bytes/read requests, without a persistent or signed Data claim') +entries = category['registry_string_events'] +require(any(e['key'] == 'UUID' and e['value'] == '18AB5557-DC2D-469B-8758-855E60F5B0DF' for e in entries) and + any(e['key'] == 'BSD Name' and e['value'] == 'disk1s1' for e in entries), 'Original APFS really publishes the observed scratch volume') +for d in (mount, timezone, abort): + data = d['scratch_data'] + require(data['actual_seed_sha256'] == digest and data['actual_seed_bytes'] == len(seed) and + data['actual_device_path'] == '/dev/disk1s1' and data['state'] == dict(_native_owner_data_status=0, + _native_owner_data_stage=4, _native_owner_data_readonly=0, _native_owner_data_device=16777218) and + d['native_ram_block']['state']['_native_ram_block_writes'] >= 30, 'Actual selected device mounts writable and completes real writes') +require(mount['bounded_timeout_interrupted'] and mount['stop_symbol'] == 'timeout-interrupt' and + mount['native_ram_block']['state']['_native_ram_block_reads'] == 517 and mount['native_ram_block']['state']['_native_ram_block_writes'] == 1509, + 'A 45-second continuation with later process crashes is a bounded observation, not functional UI success') +require(timezone['stop_symbol'] == '_symlink' and timezone['function_return_stack_matched'] and int(timezone['x0'], 16) == 0 and + timezone['entry_string_events'][-1]['matched'] and timezone['entry_string_events'][-1]['value'] == '/var/db/timezone/zoneinfo', + 'Previously failing original timezone syscall must now genuinely return zero') +reason = bytes.fromhex(abort['entry_indirect_memory']['1:20:100']['bytes']).split(b'\0')[0].decode() +require(abort['stop_symbol'] == '_abort_with_payload' and abort['function_return_stack_matched'] and int(abort['x0'], 16) == 0 and + reason.startswith("couldn't dlopen libobjc-trampolines.dylib:") and 'no such file, not in dyld cache' in reason, + 'Preserve the next actual original abort, do not replace it with success') +require(wrong['scratch_data']['state'] == dict(_native_owner_data_status=2, _native_owner_data_stage=1, + _native_owner_data_readonly=-1, _native_owner_data_device=-1) and wrong['native_ram_block']['state']['_native_ram_block_writes'] == 0 and + wrong['stop_symbol'] == '_panic' and wrong['argument_memory']['x0'].startswith('offline scratch Data mount failed'), 'Wrong expected UUID must refuse before a writable mount') +audit = json.loads((ROOT / 'build/native-ram-block/bindings.json').read_text()) +require(audit['virtual_methods_checked'] == 190 and audit['concrete_overrides'] == 24 and len(audit['bindings']) == 11 and + audit['unresolved_original_imports'] == 0 and audit['compiled_object_sha256'] == hashlib.sha256((ROOT / 'build/native-ram-block/NativeRamBlock.o').read_bytes()).hexdigest(), + 'Actual compiled RAM object retains original storage ABI/imports') +print('PASS: original APFS discovers actual RAM volume; native-4K writable Data performs real reads/writes') +print('PASS: original timezone syscall returns zero; wrong UUID refuses before writes') +print('Next observed abort: missing libobjc-trampolines path; functional UI, keybags, firmware enrollment and sheng hardware remain open') diff --git a/scripts/check-native-virtio-checkpoints.py b/scripts/check-native-virtio-checkpoints.py new file mode 100644 index 0000000..ea95166 --- /dev/null +++ b/scripts/check-native-virtio-checkpoints.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +"""Check real read-only descriptor DMA, separately from storage publication.""" +import hashlib +import json +from pathlib import Path +import re +import struct +import subprocess +import sys + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'kits/NativePrelink')) +import native_prelink as np + + +def require(value, message): + if not value: + raise SystemExit('FAIL: ' + message) + + +build = ROOT / 'build' +kernel = build / 'native-prelink/native-virtio-firstread-candidate.macho' +data = kernel.read_bytes() +plan = json.loads(kernel.with_suffix('.json').read_text()) +require(hashlib.sha256(data).hexdigest() == plan['candidate_sha256'] == + '213d08f953fe38d6b2aab67a2de78aafb2d15af76bbd2cc7582358597a161e5a', 'exact observed candidate') +require(plan['native_sha256'] == 'e0f9be317b31c53582802746fb792c45f7d14a554f00dac6deaec701b2636c1b' and + plan['original_sha256'] == 'f18b4df865405f2a287b84977d7aca745e056d656d9fc9df1662970ce2969de7' and + not plan['source_inputs_modified'] and not plan['images_modified'], 'native and original input provenance') +image = np.macho(data) +require(struct.unpack_from('> 14 & 3 == 0 and r['TCR_EL1'] >> 30 & 3 == 2 and + r['SCTLR_EL1'] & 1 and r['TTBR0_EL1'] and r['TTBR1_EL1'], 'active native 4K translation roots') + return d + + +read = report('darwinboot-virtio-firstread') +configured = report('darwinboot-virtio-platform-configure') +require(read['stop_symbol'] == '_native_virtio_read_checkpoint' and read['original_crypto_initialized_observed'], 'actual read checkpoint and original crypto') +require(configured['function_return_stack_matched'] and configured['sp'] == configured['function_entry_stack'] and + configured['pc'] == configured['function_return_target'] and int(configured['x0'], 16) == 1, 'actual platform configure returns true on same stack') +for d in (read, configured): + g = d['armvirt_globals'] + require(g['_armvirt_platform_configured'] == 1 and g['_armvirt_platform_child_count'] == 37 and + all(g['_armvirt_platform_' + n + '_count'] == 1 for n in ('cpu', 'bus', 'defaults')), 'actual platform resource enumeration') +g = read['native_virtio_globals'] +require(g['_native_virtio_ready'] == 1 and g['_native_virtio_read_requests'] == 1 and + g['_native_virtio_read_failures'] == 0, 'one genuine successful DMA request') +raw = ROOT / '.cache/firmware/target-volumes/044-76034-038.dmg' +with raw.open('rb') as stream: + sector = stream.read(512) +digest = hashlib.sha256(sector).hexdigest() +input_record = read['virtio_block_input'] +require(input_record['read_only'] and input_record['format'] == 'raw' and + input_record['name'] == raw.name and input_record['bytes'] == raw.stat().st_size == 3303014400 and + g['_native_virtio_sectors'] * 512 == raw.stat().st_size, 'real read-only input and device capacity') +require(sector[32:36] == b'NXSB' and digest == read['native_virtio_first_sector_sha256'] == + input_record['first_sector_sha256'] == '42b1178767bb27611cd1836cdd5779c48bfca8a3e719ec06dc4c8ef78d5e6387', 'actual DMA data matches original raw sector') +fdt = build / 'virtio-block-fixture/qemu-virt.dtb' +dts = subprocess.check_output(['dtc', '-I', 'dtb', '-O', 'dts', str(fdt)], text=True, stderr=subprocess.DEVNULL) +windows = [] +for node in re.findall(r'virtio_mmio@[0-9a-f]+\s*\{([^}]+)\}', dts): + match = re.search(r'reg = <([^>]+)>', node) + if match and 'dma-coherent;' in node and '"virtio,mmio"' in node: + cells = [int(v, 16) for v in match[1].split()] + require(len(cells) == 4, 'actual FDT two-cell address and size') + windows.append(((cells[0] << 32) | cells[1], (cells[2] << 32) | cells[3])) +require(len(windows) == 32 and (g['_native_virtio_mmio_physical'], 0x200) in windows, 'discovered active address belongs to actual coherent FDT resources') +bindings = json.loads((build / 'native-virtio-block/bindings.json').read_text()) +require(bindings['virtual_methods_checked'] == 190 and bindings['concrete_overrides'] == 25 and + bindings['unresolved_original_imports'] == 0 and len(bindings['bindings']) == 11 and + not bindings['original_code_modified'], 'qualified original storage dispatch and C2/D2 imports') +media = json.loads((build / 'darwinboot-virtio-media-properties/gdb-stop.json').read_text()) +require(media['kernel_sha256'] == plan['candidate_sha256'] and media['bounded_timeout_interrupted'] and + not media['registry_string_log_exhausted'], 'bounded media observation with complete property log') +properties = media['registry_string_events'] +volume_uuid = '8846FE78-2910-4297-AF97-D59BB6F8FC16' +uuid_event = next(e for e in properties if e['key'] == 'UUID' and e['value'] == volume_uuid) +volume = uuid_event['entry'] +volume_properties = {e['key']: e['value'] for e in properties if e['entry'] == volume} +require(volume_properties['Status'] == 'Online' and volume_properties['BSD Name'] == 'disk1s1' and + volume_properties['Content'] == '41504653-0000-11AA-AA11-00306543ECAC' and + volume_properties['Sealed'] == 'Yes', 'actual original APFS volume publication') +apfs = next(e for e in plan['extents'] if e['module'] == 'com.apple.filesystems.apfs' and e['segment'] == '__TEXT_EXEC') +require(apfs['va'] <= int(uuid_event['caller'], 16) < apfs['va'] + apfs['size'], 'volume UUID published by unchanged original APFS') +exports = np.exports(data, image) +read_method = '__ZN17NativeVirtioBlock16doAsyncReadWriteEP18IOMemoryDescriptoryyP19IOStorageAttributesP19IOStorageCompletion' +require(any(read_method in exports.get(int(e['pc'], 16), []) for e in media['trace_events']), 'actual public storage request executed') +symbol_output = subprocess.check_output(['xcrun', 'nm', '-U', str(kernel)], text=True) +symbols = {n: int(a, 16) for a, n in re.findall(r'^([0-9a-fA-F]+)\s+\S\s+(\S+)$', symbol_output, re.M)} +def observed(name): + size = 8 if name.endswith(('sectors', 'physical')) else 4 + return int.from_bytes(bytes.fromhex(media['memory'][f'{symbols[name]:x}:{size:x}']), 'little') +require(observed('_native_virtio_read_requests') == 62 and observed('_native_virtio_read_failures') == 0 and + observed('_native_virtio_ready') == 1, 'actual storage stack completed 62 reads without DMA failure') +result = dict(candidate_sha256=plan['candidate_sha256'], native_sha256=plan['native_sha256'], + checked_qemu_reports=3, read_only=True, first_sector_sha256=digest, + input_bytes=raw.stat().st_size, capacity_sectors=g['_native_virtio_sectors'], + discovered_mmio_physical=hex(g['_native_virtio_mmio_physical']), + fdt_sha256=hashlib.sha256(fdt.read_bytes()).hexdigest(), + actual_descriptor_dma_observed=True, native_4k=True, + original_executable_code_unchanged=True, platform_configure_return=1, + public_storage_read_observed=True, completed_dma_reads=62, + iomedia_attachment_qualified=True, volume_uuid=volume_uuid, observed_bsd_name='disk1s1', + apfs_mount_qualified=False, + functional_ipados_booted=False, hardware_tested=False) +(build / 'native-virtio-checkpoints.json').write_text(json.dumps(result, indent=2) + '\n') +print('PASS: real read-only VirtIO DMA matches original SystemOS sector; native 4K and actual platform return verified') +print('Original APFS publishes the matching Online SystemOS volume after 62 real reads; mount, iPadOS UI and hardware remain unqualified') diff --git a/scripts/check-offline-legacy-swap.py b/scripts/check-offline-legacy-swap.py new file mode 100644 index 0000000..aec04c5 --- /dev/null +++ b/scripts/check-offline-legacy-swap.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +"""Qualify GPU pixels and original swap return separately from actual scanout.""" +import hashlib +import json +from pathlib import Path +import struct +ROOT=Path(__file__).resolve().parents[1];OUT=ROOT/'build/offline-legacy-swap' +def sha(p):return hashlib.sha256(p.read_bytes()).hexdigest() +def require(v,s): + if not v:raise SystemExit(s) +def load(p):return json.loads(p.read_text()) +build=load(OUT/'build-report.json');manifest=load(OUT/'test-inputs.json');data=load(OUT/'data-report.json') +require(sha(OUT/'air-native')==build['process_sha256']==data['process_sha256'] and + sha(OUT/'test-inputs.json')==build['manifest_sha256']==data['manifest_sha256'], 'Exact own executable and manifest') +for name,digest in manifest['sources'].items():require(sha(ROOT/name)==digest,'Exact source '+name) +path=ROOT/'build/darwinboot-owner-hvf-offline-legacy-swap/gdb-stop.json';d=load(path) +require(sha(OUT/'compiler-data-32.cdr')==data['sha256']==d['scratch_data']['actual_seed_sha256'] and + sha(OUT/'offline-trust.bundle')==build['trust_bundle_sha256']==d['trust_cache_bundle_sha256'], 'Actual own Data/trust') +def modules(raw): + n=struct.unpack_from('0 and values[13]>=24000000 and + values[14:]==(1,4), 'Real full GPU-pixel check, original statuses and actual one-second queue observation interval') +host=load(OUT/'host-report.json');ops=host['operations'] +require(host['host_device']=='Apple M1' and not host['cpu_pixel_renderer'] and not host['adreno_execution'] and + not host['guest_system_device_discovery'] and not host['original_desktop'] and len(ops)==609 and + ops[0]['operation']==0x100 and ops[-1]['operation']==0x10f and + all(o['request']==i+1 and not o['error'] for i,o in enumerate(ops)), 'Actual complete ordered host GPU session') +require(len(host['completed_commands'])==len(host['texture_readbacks'])==1, 'One real GPU frame') +command=host['completed_commands'][0];readback=host['texture_readbacks'][0] +require(command['status']==4 and 01 and d['actual_process']['name']=='air-native' and + d['actual_signature_failures']['_cs_validate_page_bad_hash']==1 and not d['actual_signature_failures']['_cs_validate_page_no_hash'], + 'Actual worker code fault rejects a bad signed code page, without missing-signature fallback') + mount=directory/'inspect-mount';mount.mkdir(exist_ok=True) + attached=plistlib.loads(subprocess.check_output(['hdiutil','attach',str(directory/'compiler-data-32.cdr'),'-readonly','-nobrowse','-mountpoint',str(mount),'-plist'],timeout=30)) + try: + target=mount/'tmp/native-metal-addons/NativeOfflineMetal.bundle/NativeOfflineMetal' + pristine=(directory/'NativeOfflineMetal.bundle/NativeOfflineMetal').read_bytes();broken=target.read_bytes();offset=data['addon']['tamper_file_offset'] + require(len(pristine)==len(broken) and [i for i,(a,b) in enumerate(zip(pristine,broken)) if a!=b]==[offset] and + pristine[offset]^broken[offset]==1 and sha(target)==data['addon']['actual_plugin_sha256'] and + data['addon']['original_signature_retained'],'Actual staged negative file differs by exactly the recorded own instruction byte') + detail=subprocess.run(['codesign','--display','--verbose=4',target],capture_output=True,text=True,check=True,timeout=30).stderr + invalid=subprocess.run(['codesign','--verify','--strict',target],capture_output=True,timeout=30) + require(build['plugin_cdhash'] in detail and invalid.returncode!=0,'Actual unchanged own CodeDirectory has trusted hash but its changed code fails host verification too') + finally:subprocess.run(['hdiutil','detach',attached['system-entities'][0]['dev-entry']],check=True,timeout=30,capture_output=True) +(OUT/'addon-controls-checkpoint.json').write_text(json.dumps(dict(candidate_sha256=plan['candidate_sha256'], + positive_checkpoint_sha256=sha(OUT/'addon-checkpoint.json'),control_report_sha256=[sha(p) for p in observations], + mount_permission_refused=True,tampered_own_signed_code_refused=True,original_signature_checks_preserved=True, + readonly_own_addon_view=True,mutable_test_backing=True,production_addon_authorization=False, + system_metal=False,original_desktop=False,adreno_execution=False,active_native_adreno_4k=False),indent=2)+'\n') +print('PASS: identical own code loses bindfs access without its signed permission; one changed own plugin instruction faults in real code-signature validation before GPU access') diff --git a/scripts/check-offline-metal-sharing.py b/scripts/check-offline-metal-sharing.py new file mode 100644 index 0000000..6707668 --- /dev/null +++ b/scripts/check-offline-metal-sharing.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python3 +"""Qualify real original IOSurface Mach authorization, coherence and lifetime.""" +import argparse +import runpy +import subprocess +from pathlib import Path +p=argparse.ArgumentParser(description=__doc__);p.add_argument('--renderbox-draw',action='store_true');p.add_argument('--depth-stencil',action='store_true');p.add_argument('--private-texture',action='store_true');p.add_argument('--renderbox-device',action='store_true');p.add_argument('--plugin-bundle',action='store_true');p.add_argument('--plugin-addon',action='store_true');a=p.parse_args() +if a.renderbox_draw: + if a.depth_stencil or a.private_texture:p.error('Select one independent renderer/resource instrument') + a.renderbox_device=True +if a.depth_stencil: + if a.private_texture:p.error('Select one independent resource instrument') + a.renderbox_device=True +if a.private_texture:a.renderbox_device=True +if a.renderbox_device:a.plugin_addon=True +if a.plugin_addon:a.plugin_bundle=True +if a.renderbox_device:subprocess.run(['python3',str(Path(__file__).with_name('audit-target-renderbox-device.py'))],check=True,timeout=60,capture_output=True) +n=runpy.run_path(__file__.replace('check-offline-metal-sharing.py','check-offline-metal-processes.py')) +globals().update({k:n[k] for k in ('ROOT','OUT','require','load','sha','modules','original','plan','cpu_contexts','text_code','json','struct','hashlib','plistlib','subprocess','re')}) +kind='offline-metal-depth-stencil' if a.depth_stencil else 'offline-metal-private-texture' if a.private_texture else 'offline-renderbox-draw' if a.renderbox_draw else 'offline-renderbox-device' if a.renderbox_device else 'offline-metal-addon' if a.plugin_addon else 'offline-metal-plugin' if a.plugin_bundle else 'offline-metal-shared-surfaces' +directory=ROOT/'build'/kind;exe=directory/'air-native' +build=load(directory/'build-report.json');data=load(directory/'data-report.json');manifest=load(directory/'test-inputs.json') +require(sha(exe)==build['process_sha256']==data['process_sha256'] and + sha(directory/'test-inputs.json')==build['manifest_sha256']==data['manifest_sha256'] and + sha(directory/'compiler-data-32.cdr')==data['sha256'] and + sha(directory/'offline-trust.bundle')==build['trust_bundle_sha256'],'Exact signed sharing instrument, Data, trust and manifest') +for name,digest in manifest['sources'].items():require(sha(ROOT/name)==digest,'Exact compiled sharing source '+name) +require(struct.unpack_from('<4I',exe.read_bytes())==(0xfeedfacf,0x100000c,0,2),'Actual own ARM64 subtype-0 process') +subprocess.run(['codesign','--verify','--strict',exe],check=True,timeout=30,capture_output=True) +detail=subprocess.run(['codesign','--display','--verbose=4',exe],capture_output=True,text=True,check=True,timeout=30).stderr +require(re.search(r'^Page size=4096$',detail,re.M) and build['cdhash'] in detail,'Actual own 4K CodeDirectory') +entitlements=plistlib.loads(subprocess.check_output(['codesign','-d','--entitlements',':-',str(exe)],stderr=subprocess.DEVNULL)) +trust_modules=modules((directory/'offline-trust.bundle').read_bytes()) +expected={'com.apple.private.security.no-container':True, + 'com.apple.security.iokit-user-client-class':['NativeOfflineGPUClient','IOSurfaceRootUserClient']} +if a.plugin_addon:expected['com.apple.private.bindfs-allow']=True +elif a.plugin_bundle:expected['com.apple.private.security.no-sandbox']=True +require(entitlements==build['entitlements']==expected and trust_modules[1:]==original,'Exact own diagnostic permissions and unchanged original trust modules') +if a.plugin_bundle: + plugin=directory/'NativeOfflineMetal.bundle';binary=plugin/'NativeOfflineMetal' + require(sha(binary)==build['plugin_sha256'] and sha(plugin/'Info.plist')==build['plugin_info_sha256'] and + sha(plugin/'_CodeSignature/CodeResources')==build['plugin_resources_sha256'],'Exact actual own signed plugin and bundle resources') + require(struct.unpack_from('<4I',binary.read_bytes())==(0xfeedfacf,0x100000c,0,6),'Actual own ARM64 subtype-0 dylib') + info=plistlib.loads((plugin/'Info.plist').read_bytes()) + require(info['NSPrincipalClass']=='NativeOfflineMetalPlugin' and info['CFBundleIdentifier']=='org.blu-sharky.NativeOfflineMetal','Actual principal-class bundle metadata') + subprocess.run(['codesign','--verify','--strict',plugin],check=True,timeout=30,capture_output=True) + detail=subprocess.run(['codesign','--display','--verbose=4',plugin],capture_output=True,text=True,check=True,timeout=30).stderr + require(re.search(r'^Page size=4096$',detail,re.M) and build['plugin_cdhash'] in detail and len(trust_modules)==3 and + struct.unpack_from('1 and v[12]>1 and v[4]!=v[12] and + v[5:7]==v[13:15]==(4096,4096) and v[7] and v[15] and v[7]!=v[15] and + v[8]==v[16]!=0 and v[10:12]==(0,4) and v[18:23]==(1,4,0,0,1) and + len(set(v[26:29]))==3 and v[9]==v[28] and v[17] not in v[26:29], + 'Two actual children share the same original surface through real Mach descriptors, reject a non-surface right, observe changed seeds and exit successfully') +sessions=cpu_contexts(d,(v[4],v[12]));broker=d['native_gpu_broker'] +if a.plugin_addon: + addon=data['addon'];require(addon['volume_uuid']==data['volume_uuid'] and addon['plugin_sha256']==build['plugin_sha256'] and + addon['offline_only'] and addon['mutable_backing'] and addon['read_only_bindfs_view'] and not addon['production_root_authorization'] and + v[29]&33==33 and v[30] and v[31] and v[30]!=v[31], + 'Actual read-only union bindfs addon on own RAM Data and surviving original lower-volume extension; no immutable addon or production-root claim') +expected_requests=207 if a.renderbox_draw else 243 if a.depth_stencil else 115 if a.private_texture else 69 if a.renderbox_device else 67 +expected_reply=257426 if a.renderbox_draw else 241058 if a.depth_stencil else 118864 if a.private_texture else 84560 if a.renderbox_device else 84496 +require(broker['stage']==3 and broker['status']==0 and broker['clients']==broker['closed']==2 and + not broker['active'] and broker['requests']==expected_requests and broker['reply_bytes']==expected_reply and not broker['failures'] and broker['refusals']==(6 if a.depth_stencil else 4 if a.private_texture else 0), + 'Actual complete private broker lifetimes and real protocol results') +host_path=directory/'host-cpu-contexts-report.json';host=load(host_path);ops=host['operations'] +require(host['host_device']=='Apple M1' and host['isolated_sessions'] and host['opened_sessions']==host['closed_sessions']==2 and + not host['cpu_pixel_renderer'] and not host['guest_system_device_discovery'] and not host['adreno_execution'] and not host['original_desktop'] and + len(ops)==expected_requests and all(o['request']==i+1 for i,o in enumerate(ops)) and sum(bool(o['error']) for o in ops)==(6 if a.depth_stencil else 4 if a.private_texture else 0),'Actual real host GPU ledger and explicit offline scope') +if a.renderbox_device: + # The exact own instrument calls the unchanged original initializer and + # requires the real device/queue getters before either worker can proceed. + # Original RenderBox allocates one real initial buffer per worker. + buffers=[o for o in ops if o['operation']==0x111] + require(len(buffers)==(4 if a.renderbox_draw else 2) and {o['session'] for o in buffers}==set(sessions.values()) and + len({o['handle'] for o in buffers})==len(buffers) and all(o['handle'] and o['input_bytes']==16 for o in buffers), + 'Both original RenderBox constructors actually allocate a GPU buffer in their own independent namespace') + for o in [next(b for b in buffers if b['session']==sid) for sid in sessions.values()]: + first_texture=next(t for t in ops if t['operation']==0x104 and t['session']==o['session']) + require(o['request']close1 and ops[-1]['operation']==0x141,'Peer genuinely draws after creator session closes and actual parent waitpid reaps creator') +pixels=[0xff996633,0xff996633,0xff3366cc,0xff3366cc,0xffff00ff] +readbacks=[r for r in host['texture_readbacks'] if r['texture'] in (v[7],v[15])];require(len(readbacks)==5 and [r['texture'] for r in readbacks]==[v[7],v[15],v[15],v[7],v[15]],'Exact whole-surface import/coherence/lifetime readbacks') +for r,pixel in zip(readbacks,pixels):require(r['bytes']==16384 and r['sha256']==hashlib.sha256(struct.pack('> 14 & 3 == 0 and r['TCR_EL1'] >> 30 & 3 == 2 and + r['SCTLR_EL1'] & 1 and r['TTBR0_EL1'] and r['TTBR1_EL1'], 'active native 4K hardware roots') + return d + + +loaded = report('darwinboot-apfs-load-start-trace', positive) +missing = report('darwinboot-apfs-load-missing', negative) +registered = report('darwinboot-apfs-vfs-register', positive) +require(loaded['native_apfs_globals']['_native_apfs_load_status'] == 0 and + loaded['original_crypto_initialized_observed'], 'genuine kext load API returned success after original crypto') +start = positive['module_entrypoints']['com.apple.filesystems.apfs']['start_va'] +require(any(int(e['pc'], 16) == start and e.get('symbol') == 'com.apple.filesystems.apfs:kmod_start' + for e in loaded['trace_events']), 'actual original APFS kmod entry executed') +require(missing['native_apfs_globals']['_native_apfs_load_status'] != 0 and + missing['native_apfs_globals']['_native_apfs_registered'] == 0 and + missing['native_apfs_globals']['_native_apfs_vfsops'] == 0, 'missing original module is not reported as loaded/registered') +exports = np.exports(data, image) +require('_vfs_fsadd' in exports.get(int(registered['finished_function_entry'], 16), []), 'actual source VFS registration entry') +require(registered['function_return_stack_matched'] and registered['sp'] == registered['function_entry_stack'] and + registered['pc'] == registered['function_return_target'] and int(registered['x0'], 16) == 0, + 'actual same-stack VFS registration returned zero') +text = next(e for e in positive['extents'] if e['module'] == 'com.apple.filesystems.apfs' and e['segment'] == '__TEXT_EXEC') +require(text['va'] <= int(registered['pc'], 16) < text['va'] + text['size'], 'real VFS registration caller belongs to original APFS') + + +def layout(name): + obj = build / 'mockfs-fixture/mockfs_vfsops.o' + dwarf = subprocess.check_output(['xcrun', 'dwarfdump', '--name=' + name, '--show-children', obj], text=True) + offsets = {} + for member in dwarf.split('DW_TAG_member')[1:]: + field = re.search(r'DW_AT_name\s+\("([^"]+)"\)', member) + offset = re.search(r'DW_AT_data_member_location\s+\((0x[0-9a-f]+)\)', member) + if field and offset: + offsets[field[1]] = int(offset[1], 16) + return offsets + + +table_layout, ops_layout = layout('vfstable'), layout('vfsops') +require(table_layout['vfc_vfsops'] == 0 and table_layout['vfc_name'] == 8, 'compiled actual VFS table ABI') +ops = registered['entry_indirect_memory']['0:0:90'] +table = registered['entry_indirect_memory']['1:0:80'] +ops_address = int(ops['pointer'], 16) +table_bytes = bytes.fromhex(table['bytes']) +require(struct.unpack_from('> 14 & 3 == 0 and r['TCR_EL1'] >> 30 & 3 == 2 and + r['SCTLR_EL1'] & 1 and r['TTBR0_EL1'] and r['TTBR1_EL1'], 'active native 4K hardware roots') + return d + + +media = report('darwinboot-virtio-original-os-media', True) +mounted = report('darwinboot-virtio-original-os-bsd-mount') +refused = report('darwinboot-virtio-apfs-bsd-mount') +rootauth = report('darwinboot-original-os-dyld-prepare') +for d in (media, mounted): + i = d['virtio_block_input'] + require(i['name'] == '044-76040-038.dmg' and i['source_format'] == 'UDIF' and + i['source_sha256'] == '3f99b859c9728379110b66657c7b15285bcb1925ad8b5726542e2cdcf8b1028d' and + i['source_bytes'] == 5157406069 and i['bytes'] == 6077580800 and + i['read_only'] and i['host_bus_protocol'] == 'Disk Image' and + i['host_media_writable'] is False, 'exact supplied OS image through actual read-only virtual media') +require(not media['registry_string_log_exhausted'], 'complete bounded original OS property observation') +properties = media['registry_string_events'] +volume_uuid = '04769D26-2CA0-4825-B443-E2E5F88300F2' +entry = next(e['entry'] for e in properties if e['key'] == 'UUID' and e['value'] == volume_uuid) +v = {e['key']: e['value'] for e in properties if e['entry'] == entry} +require(v['Status'] == 'Online' and v['Sealed'] == 'No' and v['BSD Name'] == 'disk1s1' and + v['FullName'] == 'DawnUpdate21H461.J120J121J207J208OS', 'actual original OS volume publication') +config = (build / 'darwinboot-virtio-original-os-bsd-mount/esp/EFI/CrabApple/crabapple.toml').read_text() +require('rd=' + v['BSD Name'] + ' ' in config and 'initrd =' not in config, 'root chosen from actual published volume; no RAM root') +require(mounted['finished_function_entry'] == operation and mounted['function_return_stack_matched'] and + mounted['sp'] == mounted['function_entry_stack'] and mounted['pc'] == mounted['function_return_target'] and + int(mounted['x0'], 16) == 0, 'genuine original APFS mount returns zero on same stack') +g = mounted['native_virtio_globals'] +require(g['_native_virtio_ready'] == 1 and g['_native_virtio_read_requests'] == 167 and + g['_native_virtio_read_failures'] == 0 and g['_native_virtio_sectors'] * 512 == 6077580800, + 'actual OS root mount disk reads and capacity') +require(mounted['original_crypto_initialized_observed'], 'genuine original crypto remains initialized') +require(refused['finished_function_entry'] == operation and not refused['function_return_stack_matched'] and + 'Failed to find the root snapshot' in refused['argument_memory']['x0'] and + 'Rooting from the live fs of a sealed volume is not allowed on a RELEASE build' in refused['argument_memory']['x0'], + 'original sealed SystemCryptex root refusal retained') +apfs = next(e for e in plan['extents'] if e['module'] == 'com.apple.filesystems.apfs' and e['segment'] == '__TEXT_EXEC') +require(apfs['va'] <= int(refused['x30'], 16) < apfs['va'] + apfs['size'], 'actual refusal caller belongs to original APFS') +symbols = subprocess.check_output(['xcrun', 'nm', '-U', build / 'native-prelink/native-virtio-firstread-candidate.macho'], text=True) +panic = re.search(r'^([0-9a-f]+)\s+\S\s+_panic$', symbols, re.M) +require(panic and int(refused['pc'], 16) == int(panic[1], 16), 'genuine source panic receives original APFS refusal') +require(int(rootauth['pc'], 16) == int(panic[1], 16) and + rootauth['argument_memory']['x0'] == 'rootvp not authenticated after mounting @%s:%d' and + not rootauth.get('followed_user_entry') and not rootauth['syscall_events'] and + rootauth['kernel_observations']['_native_virtio_read_requests'] == 167 and + rootauth['kernel_observations']['_native_virtio_read_failures'] == 0, + 'genuine post-mount root authentication refuses the unsealed restore volume before userspace') +matched_kernel = build / 'native-prelink/native-apfs-data-candidate.macho' +matched_plan = json.loads(matched_kernel.with_suffix('.json').read_text()) +require(hashlib.sha256(matched_kernel.read_bytes()).hexdigest() == matched_plan['candidate_sha256'] == + '8db7506164cb7772c6af3e8bd21b4a9ee3a951c5c611550deae923ed1558628f' and + matched_plan['original_executable_sha256'] == plan['original_executable_sha256'], 'exact UUID-match candidate retains the same original modules') +matched = report('darwinboot-original-os-uuid-match-mount', candidate=matched_plan['candidate_sha256']) +require(matched['finished_function_entry'] == operation and matched['function_return_stack_matched'] and + matched['pc'] == matched['function_return_target'] and matched['sp'] == matched['function_entry_stack'] and + int(matched['x0'], 16) == 0, 'actual UUID-selected original mount returns zero on same stack') +selection = matched['root_selection'] +require(selection['volume_uuid'] == volume_uuid and not selection['bsd_name_override'] and + not selection['root_authentication_bypassed'], 'volume matching preserves authentication policy') +matched_dir = build / 'darwinboot-original-os-uuid-match-mount/esp/EFI/CrabApple' +config = (matched_dir / 'crabapple.toml').read_text() +require('rd=' not in config and 'boot-uuid=' not in config and 'initrd =' not in config, 'no BSD-name/resource/RAM override') +tree = (matched_dir / 'Payloads/kernel.macho.adt').read_bytes() + + +def dt_node(at): + count, children = struct.unpack_from('' + chosen['root-matching'].rstrip(b'\0') + b'') +require(matching == {'IOProviderClass': 'IOMedia', 'IOPropertyMatch': {'UUID': volume_uuid}}, 'actual DT selector matches only the original published volume UUID') +require(matched['native_virtio_globals']['_native_virtio_read_requests'] == 167 and + matched['native_virtio_globals']['_native_virtio_read_failures'] == 0, 'actual UUID-root mount I/O') +result = dict(candidate_sha256=plan['candidate_sha256'], checked_qemu_reports=5, + original_os_volume_uuid=volume_uuid, observed_bsd_name=v['BSD Name'], + original_apfs_mount_entry=operation, same_stack_mount_return=0, native_4k=True, + read_only_os_virtual_media=True, os_mount_completed_reads=167, + sealed_cryptex_root_refused=True, original_code_modified=False, + os_vfs_mount_completed=True, root_authentication_enforced=True, + unsealed_os_root_authentication_refused=True, + uuid_match_candidate_sha256=matched_plan['candidate_sha256'], + direct_uuid_root_match_qualified=True, uuid_match_mount_return=0, + original_userspace_executed_from_apfs_qualified=False, + functional_ipados_booted=False, hardware_tested=False) +(build / 'original-apfs-root-checkpoints.json').write_text(json.dumps(result, indent=2) + '\n') +print('PASS: original supplied OS APFS root mount returns 0 under native 4K after 167 successful DMA reads') +print('Sealed Cryptex root and unauthenticated OS root are genuinely refused; no APFS userspace/UI boot is claimed') +print('Standard DT matching also mounts the real UUID-selected volume without a BSD-name override') diff --git a/scripts/check-original-backboard-haptic-wait.py b/scripts/check-original-backboard-haptic-wait.py new file mode 100644 index 0000000..d076d2e --- /dev/null +++ b/scripts/check-original-backboard-haptic-wait.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +import sys,struct,json,argparse,hashlib +from pathlib import Path +ROOT=Path(__file__).resolve().parents[1] +a=argparse.ArgumentParser(description="Inspect original backboardd's haptic group wait; static analysis only") +a.add_argument('--backboardd',type=Path,required=True);a.add_argument('--report',type=Path,required=True);args=a.parse_args() +sys.path.insert(0,str(ROOT/'kits/NativePrelink'));from native_prelink import macho,backed_offset +raw=args.backboardd.read_bytes();sha=hashlib.sha256(raw).hexdigest() +assert sha=='6e362a46a2b46eb2741ecc6fcb50ffecb2fbc299641159045475361e62b34634' +m=macho(raw) +def rd(va,n):return raw[backed_offset(m['segments'],va,n,len(raw)):][:n] +def u32(va):return struct.unpack('>63:return 0 + v &= (1<<36)-1 + return v if v>=0x100000000 else 0x100000000+v +def cs(va):return rd(va,256).split(b'\0')[0].decode(errors='replace') +sections={};at=32 +for i in range(struct.unpack_from('>26==(0b100101 if link else 0b000101) + n=word&0x3ffffff;n-=1<<26 if n&(1<<25) else 0 + return pc+n*4 +def adr(pc): + word=u32(pc);assert word&0x9f000000==0x10000000 + n=((word>>5)&0x7ffff)<<2|((word>>29)&3);n-=1<<21 if n&(1<<20) else 0 + return pc+n +assert adr(0x10002fef4)==0x1000303c4 +assert u32(0x1000303dc)==0x92800001 # mov x1,-1, DISPATCH_TIME_FOREVER +assert branch(0x1000303e0)==0x10009765c +assert branch(0x1000303f0,False)==0x100099d00 +assert cs(ptr(0x100103de8))=='_queue_applyDefinitions:' +assert branch(0x100023284)==0x100096d68 # original main run-loop call +report=dict(passed=True,backboardd_sha256=sha,analysis='static-original-ARM64', + method=methods[0x10002fe4c],method_start=hex(0x10002fe4c),block=hex(0x1000303c4), + group_wait_return=hex(0x1000303e4),timeout='DISPATCH_TIME_FOREVER', + following_selector=cs(ptr(0x100103de8)),main_runloop_return=hex(0x100023288), + UI_blocking_dependency_proven=False,hardware_executed=False) +args.report.parent.mkdir(parents=True,exist_ok=True);args.report.write_text(json.dumps(report,indent=2)+'\n') +print('PASS: original applyDefinitions block waits forever for the haptic controller group; no claim that this causes the UI stall') diff --git a/scripts/check-original-bics-environment.py b/scripts/check-original-bics-environment.py new file mode 100644 index 0000000..dd881b1 --- /dev/null +++ b/scripts/check-original-bics-environment.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +"""Run the unchanged bics boot-environment branch offline with explicit API stand-ins.""" +import argparse,hashlib,json,struct,sys +from pathlib import Path +ROOT=Path(__file__).resolve().parents[1] +p=argparse.ArgumentParser(description=__doc__);p.add_argument('--daemon',type=Path,required=True);p.add_argument('--python-modules',type=Path,required=True);p.add_argument('--report',type=Path,required=True);a=p.parse_args() +sys.path[:0]=[str(a.python_modules),str(ROOT/'kits/NativePrelink')] +from native_prelink import macho,backed_offset +from unicorn import Uc,UC_ARCH_ARM64,UC_MODE_ARM,UC_HOOK_CODE +from unicorn.arm64_const import UC_ARM64_REG_SP,UC_ARM64_REG_X0,UC_ARM64_REG_X1,UC_ARM64_REG_X2,UC_ARM64_REG_X3,UC_ARM64_REG_X29,UC_ARM64_REG_LR,UC_ARM64_REG_PC,UC_ARM64_REG_CPACR_EL1 +raw=a.daemon.read_bytes();sha=hashlib.sha256(raw).hexdigest();assert sha=='2d1f67b75fb33d091396c70aee627bb6a2c8dcdb03ffa7401378b7a688088b53';m=macho(raw) +def at(va,size):return raw[backed_offset(m['segments'],va,size,len(raw)):][:size] +assert at(0x100002658,4)==struct.pack('> 14 & 3 == 0 and regs['TCR_EL1'] >> 30 & 3 == 2 and + regs['SCTLR_EL1'] & 1 and regs['TTBR0_EL1'] and regs['TTBR1_EL1'], 'Active native 4K hardware translation is required') + require(set(d['commpage_user64_shifts'].values()) == {12}, 'Both real commpages must retain shift 12') + require(d['original_amfi_table_owner_observed'] and d['original_crypto_initialized_observed'], + 'Use genuine active AMFI/corecrypto') + require(d['native_virtio_globals']['_native_virtio_read_requests'] == 1879 and + d['native_virtio_globals']['_native_virtio_read_failures'] == 0 and d['virtio_block_input']['read_only'], + 'Actual original APFS reads must complete without failure') + cache = d['native_signature_hash_cache'] + require(cache['_cs_native4k_full_blocks'] == 6562 and cache['_cs_native4k_ready_blobs'] == 2 and + cache['_cs_native4k_page_checks'] == 4, 'Authenticate all 6559 original signing blocks plus the own process') + require(not d['original_full_cache_loaded'] and not d['root_authentication_solved'], + 'This component test must not claim a full cache or production boot') +require(positive['original_cache_leaf_executed'] and positive['original_cache_leaf_result'] == 0 and + positive['native_user_4k_abi_observed'], 'The actual public original function must return under the real 4K ABI') +entry = positive['followed_user_entry'] +require(entry['entry_bytes_after_fault'] == leaf['function_bytes'][:32] and int(entry['pstate'], 16) & 15 == 0, + 'Read original instruction bytes after the genuine first fault at EL0') +require(entry['checkpoint']['observed'] and entry['checkpoint']['bytes'].startswith('c0035fd6') and + entry['function_return']['observed'] and int(entry['function_return']['pstate'], 16) & 15 == 0, + 'Observe the real original success RET and caller return') +call = [e for e in positive['syscall_events'] if e['number'] == 24] +require(len(call) == 1 and int(call[0]['user_pc'], 16) == int(entry['user_entry'], 16) + 8 and + call[0]['user_lr'] == entry['function_return']['target'], 'Original code must execute its own real getuid SVC') +require(positive['code_signing_hash_counters'] == dict(_cs_validate_page_bad_hash=0, _cs_validate_page_no_hash=0), + 'Positive code faults must validate without missing hashes') +fault = negative['own_mapped_page_mutation'] +require(fault['code_directory_hash'] == leaf['original_cdhash'] and + fault['page_sha256_before'] == leaf['original_page_sha256'] and + fault['file_page'] == leaf['native_page_offset'] and fault['byte_in_page'] == leaf['within_page_offset'] and + fault['before'] ^ fault['after'] == 1 and not fault['registers_or_validation_state_modified'], + 'Mutate only the identity-qualified actual original code-page input') +require(negative['code_signing_hash_counters'] == dict(_cs_validate_page_bad_hash=1, _cs_validate_page_no_hash=0) and + negative['fatal_user_exit']['cs_status'] == 'CS_KILLED' and negative['fatal_user_exit']['namespace'] == 3 and + negative['fatal_user_exit']['code'] == 2 and not negative['original_cache_leaf_executed'] and + not any(e['number'] == 24 for e in negative['syscall_events']), 'Genuine signature checking must refuse before original SVC') +print('PASS: original 16K-signed cache getuid executes its real SVC and returns at native 4K') +print('PASS: same-input, CDHash-qualified code-page mutation is genuinely refused before the original SVC') +print('Full shared cache, launchd main, production root trust and tablet hardware remain unqualified') diff --git a/scripts/check-original-cache-file-checkpoints.py b/scripts/check-original-cache-file-checkpoints.py new file mode 100644 index 0000000..f44a8e7 --- /dev/null +++ b/scripts/check-original-cache-file-checkpoints.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +"""Verify unchanged target cache data through real native-4K file APIs in QEMU.""" +import hashlib +import json +from pathlib import Path +import struct + +ROOT = Path(__file__).resolve().parents[1] +build = ROOT / 'build' +fixture = build / 'native-cache-file-fixture' +metadata = json.loads((fixture / 'file-probe.json').read_text()) +candidate = json.loads((build / 'native-prelink/native-readonly-file-candidate.json').read_text()) + + +def require(value, message): + if not value: + raise SystemExit('FAIL: ' + message) + + +image = (fixture / 'root.img').read_bytes() +magic, version, count, own_at, own_size, file_at, file_size = struct.unpack_from('<8sII4Q', image) +require((magic, version, count) == (b'N4KROOT\0', 1, 2), 'offline input format') +own = image[own_at:own_at + own_size] +target = image[file_at:file_at + file_size] +require(own == (fixture / 'launchd.macho').read_bytes(), 'unchanged own process') +require(hashlib.sha256(target).hexdigest() == metadata['original_file_sha256'] == + '3d923c404fe519184a99042b60a4cb3d37247dcb8b070242ed3e670d4d523d04', 'exact supplied target file') +require(hashlib.sha256(image).hexdigest() == metadata['root_sha256'], 'input container identity') +trust = json.loads((fixture / 'offline-file-trust.json').read_text()) +own_trust = json.loads((fixture / 'trustcache.json').read_text()) +require(trust['cdhashes'] == sorted((metadata['original_cdhash'], own_trust['cdhash'])) and + not trust['apple_trust_chain_verified'] and not trust['original_apple_trust_cache_modified'], + 'explicit offline trust scope') + + +def report(name, direct=False, own_sha=metadata['own_process_sha256']): + d = json.loads((build / name / 'gdb-stop.json').read_text()) + require(d['kernel_sha256'] == candidate['candidate_sha256'], 'same exact source/component candidate') + require(not d['bounded_timeout_interrupted'], name + ': no timeout inference') + r = {k: int(v, 16) for k, v in d['system_registers'].items()} + require(r['TCR_EL1'] >> 14 & 3 == 0 and r['TCR_EL1'] >> 30 & 3 == 2 and + r['SCTLR_EL1'] & 1 and r['TTBR0_EL1'] and r['TTBR1_EL1'], 'actual native 4K MMU') + if not direct: + require(d['original_amfi_table_owner_observed'] and d['original_crypto_initialized_observed'], + 'original security providers') + require(d['native_user_el0_observed'] and d['native_user_4k_abi_observed'] and + not d['ipados_userspace_boot_demonstrated'], 'own process ABI and scope') + require(d['native_user_image_sha256'] == own_sha, 'own process identity') + return d + + +refusal = report('darwinboot-original-cache-readonly-file') +require(refusal['offline_root_sha256'] == metadata['root_sha256'] and + refusal['trust_cache_bundle_sha256'] == own_trust['bundle_sha256'], 'own-only trust control') +f = refusal['native_file_probe'] +require(f['open_result'] == 0 and f['open_nzcv'] >> 29 & 1 == 0 and f['stage'] == 1 and + f['fcntl_result'] == 1 and f['fcntl_nzcv'] >> 29 & 1 == 1, 'real registration EPERM control') +require(refusal['native_signature_hash_cache']['_cs_native4k_ready_blobs'] == 0, 'no untrusted publication') + +positive = report('darwinboot-original-cache-owner-trust') +require(positive['offline_root_sha256'] == metadata['root_sha256'] and + positive['trust_cache_bundle_sha256'] == trust['bundle_sha256'], 'explicit file-test trust') +f = positive['native_file_probe'] +require(f['stage'] == 4 and f['open_result'] == 0 and f['fcntl_result'] == 0 and f['mmap_result'] != 0, + 'genuine open/signature registration/mmap completed') +require(not any(f[n] >> 29 & 1 for n in ('open_nzcv', 'fcntl_nzcv', 'mmap_nzcv')), 'no file syscall errors') +require(f['signature_end'] == metadata['code_limit'] and f['native_pages_read'] == 8 and + f['page_word_xor'] == metadata['expected_page_word_xor'], 'actual original page reads') +c = positive['native_signature_hash_cache'] +require((c['_cs_native4k_full_blocks'], c['_cs_native4k_page_checks'], c['_cs_native4k_ready_blobs']) == + (2, 8, 1), 'two original signing blocks and eight actual native page checks') +require(set(positive['code_signing_hash_counters'].values()) == {0}, 'no missing/bad page hashes') + +inspection = report('darwinboot-original-cache-guest-signature', direct=True) +require(int(inspection['x0'], 16) == 0 and inspection['skipped_target_hits'] == 1, 'actual second blob authentication') +captured = inspection['entry_indirect_memory'] +cd = bytes.fromhex(captured['1:80:200']['bytes']) +length = struct.unpack_from('>I', cd, 4)[0] +require(length <= len(cd) and cd[39] == 14 and hashlib.sha256(cd[:length]).hexdigest()[:40] == + metadata['original_cdhash'], 'unchanged actual guest target CodeDirectory/CDHash') +cache = captured['1:b8:1000'] +digests = bytes.fromhex(cache['bytes']) +expected = b''.join(hashlib.sha256(target[o:o + 4096]).digest() for o in range(0, metadata['code_limit'], 4096)) +require(digests[:len(expected)] == expected and not any(digests[len(expected):]), 'actual target derived hashes/padding') +t = cache['translation'] +require(t['leaf_level'] == 3 and t['leaf_bytes'] == 4096 and t['ap'] == 2 and t['pxn'] and t['uxn'], + 'actual target digest mapping read-only/NX/native4K') + +negative = report('darwinboot-original-cache-bad-native-page') +bad = bytearray(image) +bad[file_at + 4096] ^= 1 +require(hashlib.sha256(bad).hexdigest() == negative['offline_root_sha256'] and + negative['trust_cache_bundle_sha256'] == trust['bundle_sha256'], 'one-bit corruption, unchanged trust') +f = negative['native_file_probe'] +require(f['stage'] == 1 and f['fcntl_result'] == 85 and f['fcntl_nzcv'] >> 29 & 1 == 1 and + f['native_pages_read'] == 0, 'real EBADEXEC before mapping corrupted target file') +require(negative['code_signing_hash_counters']['_cs_validate_page_bad_hash'] == 1 and + negative['native_signature_hash_cache']['_cs_native4k_ready_blobs'] == 0, 'original target hash enforced') + +legacy = report('darwinboot-two-file-legacy-root-regression', + own_sha='8e6cb80348e018c780e3a8b71bed6015e80e932b23be378bfcdd67846277b9a0') +malformed = report('darwinboot-two-file-malformed-root', direct=True) +require(int(malformed['x0'], 16) == 22 and malformed.get('finished_function_entry'), + 'overlapping malformed root returns EINVAL through actual mount function cleanup') + +result = dict(candidate_sha256=candidate['candidate_sha256'], original_file_sha256=metadata['original_file_sha256'], + original_cdhash=metadata['original_cdhash'], original_file_bytes_modified=False, + original_hash_blocks_authenticated=2, native_4k_pages_checked=8, checkpoints=6, + explicit_offline_file_trust=True, apple_boot_trust_chain_verified=False, + target_component_code_executed=False, target_ipados_booted=False, sheng_hardware_tested=False) +(build / 'original-cache-file-checkpoints.json').write_text(json.dumps(result, indent=2) + '\n') +print('PASS: unchanged original cache file; real registration, native-4K mappings/faults and corruption refusal') +print('Explicit offline file trust; no original target code execution, iPadOS boot or hardware claim') diff --git a/scripts/check-original-cpu-render.py b/scripts/check-original-cpu-render.py new file mode 100644 index 0000000..f1ca9ca --- /dev/null +++ b/scripts/check-original-cpu-render.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +"""Verify the retained original render scheduling/gate observations, not a UI.""" +import hashlib +import json +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +base = ROOT / 'build/cpu-desktop' +summary = {} +for case in ('software-trace', 'software-gates'): + path = base / case / 'gdb-stop.json' + data = json.loads(path.read_text()) + assert data['kernel_sha256'] == 'fdcf9e7621deb9df77798de62615a358fe52450b54f5e4a468d2dd76f7738e18' + assert data['native_4k_mmu_observed'] and data['original_launchd_main_executed'] + assert data['owner_root_binding_audit']['boot_root_accepted'] + assert data['original_amfi_table_owner_observed'] and not any(data['actual_signature_failures'].values()) + assert set(data['direct_user_commpage_shifts'].values()) == {12} + assert not data['sheng_hardware_tested'] and not data['functional_ipados_boot_demonstrated'] + observed = data['followed_display_query'] + assert observed['activated_process']['name'] == 'backboardd' + plan = observed['plan'] + assert plan['original_files_modified'] is False + sites = {site['name']: site for site in plan['trace_sites']} + counts, results = {}, {} + for event in observed['trace_events']: + name = event['name']; site = sites[name] + assert event['process']['name'] == 'backboardd' and not int(event['pstate'],16) & 31 + assert int(event['pc'],16) == site['address'] + event['process']['shared_region_slide'] + assert event['entry_bytes'] == site['prefix'] + counts[name] = counts.get(name,0) + 1 + assert counts[name] <= site.get('max_hits',1) + results.setdefault(name, []).append(int(event['arguments'][0],16)) + assert data['native_legacy_display']['transfer_calls'] == 0 + assert data['native_legacy_display']['guard_slot'] == 0 + assert data['bounded_timeout_interrupted'] + if case == 'software-trace': + assert counts == {'render_for_time':1, 'display_timer':1} + assert set(observed['trace_pending']) == {'render_update','software_renderer'} + else: + for name in ('can_update_enabled_result','can_update_return','is_ready_return'): + assert counts[name] == 8 and 1 in results[name] + assert observed['trace_pending'] == ['render_update'] + summary[case] = dict(evidence_sha256=hashlib.sha256(path.read_bytes()).hexdigest(), events=counts, + pending=observed['trace_pending']) +summary.update(original_software_renderer_execution_observed=False, desktop_observed=False, hardware_observed=False) +(base/'verified-observations.json').write_text(json.dumps(summary,indent=2)+'\n') +print('PASS: genuine original EL0 scheduling and enabled/update/ready checks; no software-renderer/frame/desktop claim') diff --git a/scripts/check-original-display-exception-checkpoints.py b/scripts/check-original-display-exception-checkpoints.py new file mode 100644 index 0000000..5e5b207 --- /dev/null +++ b/scripts/check-original-display-exception-checkpoints.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +"""Require actual original FrontBoard EL0 exception and matching cache symbolication.""" +import json +from pathlib import Path +import runpy +import struct + +ROOT = Path(__file__).resolve().parents[1] +base = runpy.run_path(str(ROOT / 'scripts/check-native-arch-timer-checkpoints.py')) +require = base['require'] +d = json.loads((ROOT / 'build/darwinboot-owner-hvf-arch-timer-first-exception/gdb-stop.json').read_text()) +s = json.loads((ROOT / 'build/native-user-exception/arch-timer-qualified-symbolication.json').read_text()) +require(d['kernel_sha256'] == s['kernel_sha256'] == base['expected'] and + d['stop_symbol'] == '_exception_triage' and not d['bounded_timeout_interrupted'] and + int(d['observed_function_entry_arguments'][0], 16) == 6 and + int(d['observed_function_entry_arguments'][2], 16) == 2 and len(d['entry_user_state_candidates']) == 1, + 'Real two-code EXC_BREAKPOINT entry on the exact architectural timer candidate') +u = d['entry_user_state_candidates'][0] +require(struct.unpack('<2Q', bytes.fromhex(d['entry_regions']['1:0:10']['bytes'])) == (1, int(u['user_pc'], 16)) and + int(u['cpsr'], 16) & 31 == 0 and int(u['esr'], 16) >> 26 == 0x3c and + u['instruction_bytes'].startswith('000020d4200020d4') and u['user_pc'] == u['user_lr'], + 'Actual saved ABI, EL0, exception code, ESR and original BRK must agree') +require(s['unique_instruction_match'] == dict(file='dyld_shared_cache_arm64.09', offset=103477996, unslid_pc=6966604524) and + int(s['cache_slide'], 16) + 6966604524 == int(u['user_pc'], 16) and s['local_symbols_uuid_matches'] and + s['frames'][0]['nearest_local_symbol'] == '-[FBSDisplayMonitor _initWithBookendObserver:transformer:]' and + s['frames'][0]['offset'] == 1148 and + s['frames'][1]['nearest_local_symbol'] == '___34+[FBDisplayManager sharedInstance]_block_invoke' and + any(f['image'] == '/System/Library/PrivateFrameworks/SpringBoard.framework/SpringBoard' for f in s['frames']) and + s['frames'][-1]['image'] == '/usr/lib/dyld', 'Original uniquely matched code and UUID-bound symbols qualify the actual call chain') +require(s['static_original_assertion'] == dict(literal_va='0x1e7a9bf68', text='failed to find the main CADisplay', runtime_string_captured=False), + 'Distinguish actual original assertion-reference disassembly from a runtime log') +require(d['native_4k_mmu_observed'] and d['actual_signature_failures'] == dict(_cs_validate_page_bad_hash=0, _cs_validate_page_no_hash=0) and + d['owner_root_binding_audit']['boot_root_accepted'] and d['owner_boot_manifest']['status'] == 0 and + d['actual_root_composition']['_native_owner_cache_native_union'] == 1 and 'owner_cache_carrier' not in d and + not s['functional_ipados_ui_demonstrated'] and not s['original_files_modified'], 'No code tampering, carrier exception or functional UI claim') +print('PASS: actual EL0 FrontBoard display-monitor BRK, through SpringBoard framework, matched to unchanged cache instructions and symbols') +print('Open display assertion: failed to find the main CADisplay; no framebuffer service or functional UI success claimed') diff --git a/scripts/check-original-display-mode.py b/scripts/check-original-display-mode.py new file mode 100644 index 0000000..1e04904 --- /dev/null +++ b/scripts/check-original-display-mode.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +"""Verify the original SpringBoard initializer really returns the native GOP mode.""" +import json +from pathlib import Path +import runpy +import struct +ROOT=Path(__file__).resolve().parents[1] +base=runpy.run_path(str(ROOT/'scripts/check-system-framebuffer-client.py')) +require=base['require'];sha=base['sha'];plan=base['plan'] +require(base['args'].profile=='scale','Use the current qualified unit-scale candidate') +path=ROOT/'build/darwinboot-owner-hvf-legacy-display-start-scale-mode/gdb-stop.json';d=json.loads(path.read_text()) +require(d['kernel_sha256']==plan['candidate_sha256'] and not d['bounded_timeout_interrupted'] and + d['stop_symbol']=='original-display-mode-return' and d['original_launchd_main_executed'] and + d['followed_main_entry']['first_instruction_completed'],'Real original boot and selected mode-return observation') +f=d['followed_display_query'];p=f['plan'] +require(p['mode']=='user-return' and p['query']==0x19f3d0500 and p['return_checkpoint']==476 and + p['original_mode_initializer_sha256']=='ca4857e6617670f2369a0a6eb68be09e29135ce5dfda19ac1ea829229f0cc0ec' and + f['observed'] and f['entry_process']['name']==f['actual_process']['name']=='SpringBoard' and + f['entry_process']['thread']==f['actual_process']['thread']==f['activated_process']['thread'], + 'Exact unchanged original initializer and actual selected SpringBoard thread') +slide=int(f['cache_slide'],16) +require(int(f['query'],16)==p['query']+slide and int(f['epilogue_pc'],16)==p['query']+slide+476 and + f['entry_process']['shared_region_slide']==slide and + f['caller_restored'] and f['return_pc']==f['caller']==d['pc'] and + f['return_stack']==f['entry_stack']==d['sp'] and + f['entry_bytes']==f['query_bytes_after_fault']==p['query_prefix'] and int(d['pstate'],16)&31==0, + 'Actual cache activation, unmodified instructions and real RET restore the original caller/stack at EL0') +args=[int(v,16) for v in f['entry_arguments']] +raw=bytes.fromhex(f['returned_object_bytes']) +require(int(f['return_value'],16)==args[0]!=0 and len(raw)==80,'Real nonnull original mode object returned') +isa,width,height,preferred,override,refresh,gamut,hdr,rotation,virtual=struct.unpack_from('<4Q2d3QB',raw) +require(isa and (width,height,preferred)==tuple(args[2:5])==(800,600,1) and + override==0.0 and refresh==60.0 and (gamut,hdr,rotation,virtual)==(0,0,0,0), + 'Original returned object contains the actual GOP dimensions and explicit 1:1 logical scale') +state=d['native_legacy_display'];video=d['actual_boot_video']['pe_video'] +require(state['stage']==10 and state['start_result']==state['dt_attached']==state['registration_calls']==1 and + state['unit_scale_status']==0 and state['unit_scale_calls']==state['chosen_scale']==1 and + state['chosen_rotation']==0 and not state['guard_slot'] and + state['gop_physical']==video['v_baseAddr'] and state['gop_width']==width and state['gop_height']==height and + state['gop_bytes']==video['v_rowBytes']*height==1920000, + 'Unit-scale policy is bound to the actual retained GOP mode/resource, without a guessed panel mode') +require(d['native_4k_mmu_observed'] and set(d['direct_user_commpage_shifts'].values())=={12} and + d['original_amfi_table_owner_observed'] and d['owner_root_binding_audit']['boot_root_accepted'] and + d['owner_root_binding_audit']['_native_owner_root_apple_autherr']==80 and + d['actual_signature_failures']==dict(_cs_validate_page_bad_hash=0,_cs_validate_page_no_hash=0), + 'Active native CPU/user 4K, original AMFI/Apple rejection and independent owner-root acceptance remain intact') +(ROOT/'build/native-legacy-display/unit-scale-mode-checkpoint.json').write_text(json.dumps(dict( + candidate_sha256=plan['candidate_sha256'],native_sha256=plan['native_sha256'],checkpoint_sha256=sha(path), + actual_process=f['actual_process'],actual_original_initializer_return=True, + width=width,height=height,logical_scale=preferred,original_refresh_rate_field=refresh, + panel_timing_measured=False,original_instructions_modified=False,swap_contract_complete=False, + system_metal_device=False,original_desktop=False,adreno_execution=False,hardware_tested=False),indent=2)+'\n') +print('PASS: unchanged original SpringBoard mode initializer returns a real 800x600, scale-1 mode to its actual caller') +print('Original validation passes naturally; actual swaps, system Metal, desktop and Adreno remain open') diff --git a/scripts/check-original-gpu-attach-only.py b/scripts/check-original-gpu-attach-only.py new file mode 100644 index 0000000..163f537 --- /dev/null +++ b/scripts/check-original-gpu-attach-only.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +"""Qualify the attach-only original IOGPU factory boundary. + +Mode 7 attaches the original provider but deliberately skips registerService, +then supplies a real descriptor at slot 170. The factory still returns its +genuine kIOReturnInternalError, proving attachment alone is insufficient and +the registration path (required slot 84) is part of the client contract. +""" +import json +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +REPORT = ROOT / 'build/darwinboot-owner-hvf-original-descriptor-provision-attached-only-0x1/gdb-stop.json' +CHECKPOINT = ROOT / 'build/native-original-provider/attach-only-checkpoint.json' + +report = json.loads(REPORT.read_text()) +provider = report.get('native_original_provider') +if not provider: + raise SystemExit('Missing original provider observation') +if provider.get('callback') != 170: + raise SystemExit(f"Expected descriptor slot 170, got {provider.get('callback')!r}") +if provider.get('descriptor_provisioned') != 1: + raise SystemExit('The real descriptor was not provisioned') +if provider.get('attached') != 1 or provider.get('registered') != 0: + raise SystemExit('This checkpoint requires attached-only state') +if provider.get('factory_result') != -536870199: + raise SystemExit(f"Expected genuine kIOReturnInternalError, got {provider.get('factory_result')!r}") +if provider.get('client_instances_live') != 0: + raise SystemExit('No client instance may survive') +if not report.get('native_4k_mmu_observed'): + raise SystemExit('Native 4K must be active') +result = { + 'kernel_sha256': report['kernel_sha256'], + 'provider_attached': True, + 'provider_registered': False, + 'descriptor_provisioned': True, + 'descriptor_hook': 170, + 'client_factory_result': provider['factory_result'], + 'client_factory_result_name': 'kIOReturnInternalError', + 'client_instances_live': 0, + 'registration_required_before_factory': True, + 'device_or_mapping_supplied': False, + 'gpu_submission': False, + 'system_metal_device': False, + 'native_4k_observed': True, + 'qemu_only': True, + 'sheng_hardware_tested': False, +} +CHECKPOINT.parent.mkdir(parents=True, exist_ok=True) +CHECKPOINT.write_text(json.dumps(result, indent=2) + '\n') +print('PASS: attach-only provider reaches slot 170 with a real descriptor') +print('PASS: factory still returns genuine internal error without registration') +print('Open: genuine registration/slot-84 semantics, slot 189 device/resource/queue, GPU execution') diff --git a/scripts/check-original-gpu-attach.py b/scripts/check-original-gpu-attach.py new file mode 100644 index 0000000..27283ec --- /dev/null +++ b/scripts/check-original-gpu-attach.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 +"""Qualify the original IOGPU provider attach boundary. + +Mode 6 attaches and registers the original provider. Attachment succeeds, but +the original registerService immediately reaches its required slot-84 +registration-save hook, which this instrument never fakes; the probe stops +there. No client factory, descriptor, device, mapping, queue, GPU execution or +Metal device is reached. +""" +import json +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +REPORT = ROOT / 'build/darwinboot-owner-hvf-original-descriptor-provision-attached-0x1/gdb-stop.json' +CHECKPOINT = ROOT / 'build/native-original-provider/attach-checkpoint.json' + +report = json.loads(REPORT.read_text()) +provider = report.get('native_original_provider') +if not provider: + raise SystemExit('Missing original provider observation') +if provider.get('callback') != 84: + raise SystemExit(f"Expected required slot-84 registration hook, got {provider.get('callback')!r}") +if provider.get('attached') != 1: + raise SystemExit('The provider attachment did not succeed') +if provider.get('registered') != 0: + raise SystemExit('Registration must not have completed through the fake path') +if provider.get('descriptor_provisioned') != 0 or provider.get('descriptor_refused') != 0: + raise SystemExit('The client factory must not reach the descriptor hook in this boundary') +if provider.get('client_instances_live') != 0: + raise SystemExit('No client instance may survive') +if not report.get('native_4k_mmu_observed'): + raise SystemExit('Native 4K must be active') +result = { + 'kernel_sha256': report['kernel_sha256'], + 'provider_attached': True, + 'provider_registered': False, + 'registration_required_hook': 84, + 'client_factory_reached': False, + 'descriptor_provisioned': False, + 'client_instances_live': 0, + 'device_or_mapping_supplied': False, + 'gpu_submission': False, + 'system_metal_device': False, + 'native_4k_observed': True, + 'qemu_only': True, + 'sheng_hardware_tested': False, +} +CHECKPOINT.parent.mkdir(parents=True, exist_ok=True) +CHECKPOINT.write_text(json.dumps(result, indent=2) + '\n') +print('PASS: real original provider attachment succeeds') +print('PASS: genuine registration-save slot 84 is reached and not faked') +print('Open: slot-84 semantics, attach-only factory, device/resource/queue, GPU execution') diff --git a/scripts/check-original-gpu-client-abi.py b/scripts/check-original-gpu-client-abi.py new file mode 100644 index 0000000..b05c72c --- /dev/null +++ b/scripts/check-original-gpu-client-abi.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +"""Check genuine original userclient lifecycle/refusal without a publication claim.""" +import hashlib,json,struct,sys +from pathlib import Path +ROOT=Path(__file__).resolve().parents[1];sys.path.insert(0,str(ROOT/'kits/NativePrelink')) +from native_prelink import macho,backed_offset,exports + +def load(p):return json.loads(p.read_text()) +def sha(p):return hashlib.sha256(p.read_bytes()).hexdigest() +def require(ok,why): + if not ok:raise SystemExit(why) +base=ROOT/'build/native-prelink/native-owner-original-client-abi-candidate';plan=load(base.with_suffix('.json'));raw=base.with_suffix('.macho').read_bytes();segments=macho(raw)['segments'] +require(sha(base.with_suffix('.macho'))==plan['candidate_sha256'] and sha(ROOT/'build/xnu/ios-obj/DEVELOPMENT_ARM64_BCM2837/mach.development.bcm2837.unstripped.noctf')==plan['native_sha256'],'Exact current native kernel/candidate') +reference=load(ROOT/'build/native-prelink/native-owner-metal-shared-events-candidate.json');require(plan['components']==reference['components'] and plan['original_executable_sha256']==reference['original_executable_sha256'] and plan['early_security_initializers']==reference['early_security_initializers'],'Same 29 unchanged original components/security entry') +for module,digest in plan['original_executable_sha256'].items(): + e=next(e for e in plan['extents'] if e['module']==module and e['segment']=='__TEXT_EXEC');o=backed_offset(segments,e['va'],e['size'],len(raw));require(hashlib.sha256(raw[o:o+e['size']]).hexdigest()==digest,'Actual whole original executable identity '+module) +build=load(ROOT/'build/native-offline-accelerator/build-report.json');broker_build=load(ROOT/'build/native-offline-gpu/build-report.json');abi=load(ROOT/'build/gpu-analysis/original-iogpu-userclient-abi.json') +for manifest in (build['source_sha256'],broker_build['source_sha256']): + for rel,digest in manifest.items():require(sha(ROOT/rel)==digest,'Exact compiled own source '+rel) +require(sha(ROOT/'build/native-offline-accelerator/libNativeOfflineAccelerator.late.a')==build['archive_sha256'] and sha(ROOT/'build/native-offline-gpu/libNativeOfflineGPU.late.a')==broker_build['archive_sha256'] and sha(ROOT/'build/gpu-analysis/original-iogpu-userclient-abi.json')==build['original_client_abi_sha256'],'Actual compiled archives and original ABI inputs') +reports=[] +for suffix in ('','-missing-base','-default-off'): + fixture=ROOT/('build/darwinboot-owner-hvf-original-client-abi'+suffix);d=load(fixture/'gdb-stop.json');sink=load(fixture/'console-sink.json') + require(d['kernel_sha256']==plan['candidate_sha256'] and not d['bounded_timeout_interrupted'] and d['qemu_accelerator']=='hvf' and d['native_4k_mmu_observed'] and not d['sheng_hardware_tested'],'Actual bounded new native-4K kernel observation') + r={k:int(v,16) for k,v in d['system_registers'].items()};require((r['TCR_EL1']>>14)&3==0 and (r['TCR_EL1']>>30)&3==2 and r['SCTLR_EL1']&1 and r['TTBR0_EL1'] and r['TTBR1_EL1'],'Actual active CPU TCR/TTBR/SCTLR, independent of GPU SMMU') + require(sink['protocol_responses_sent']==0 and 0<=sink['bytes_observed']<=65536 and not sink['errors'] and not sink['gpu_execution'] and not sink['system_device_published'],'Bounded firmware console prefix receives zero protocol responses; no GPU execution/publication') + console=d['native_console_globals'];require(all(console['_native_console_'+name]==0 for name in ('tx_requests','rx_requests','failures','tx_bytes','rx_bytes')),'Actual kernel transport sends/receives no GPU protocol bytes after the independent firmware prefix') + require(d['original_crypto_initialized_observed'] and d['original_crypto_table_owner_observed'],'Actual unchanged original crypto initialization') + b=d['native_gpu_broker'];require(not b['clients'] and not b['requests'] and not b['actual_cpu_contexts'] and not any(d['native_gpu_shared_events'].values()),'No actual GPU userclient, command or shared capability opened') + if suffix!='-default-off':require(d['stop_symbol']=='_native_original_client_checkpoint' and b['stage']==3 and not b['status'],'Actual early private hook boundary, not an OS/desktop boot') + reports.append(d) +positive,missing,off=reports;p=positive['native_original_client'];m=missing['native_original_client'];z=off['native_original_client'] +names={name:address for address,group in exports(raw,macho(raw)).items() for name in group} +require(p['parent']==names['__ZN12IOUserClient10gMetaClassE'],'Actual live parent is this new kernel original native IOUserClient metaclass') +def table(address,count): + o=backed_offset(segments,address,count*8,len(raw));return struct.unpack_from('<'+'Q'*count,raw,o) +base_table=table(names['__ZTV12IOUserClient']+16,186);actual_table=table(p['vptr'],189) +require([i for i,(a,b) in enumerate(zip(base_table,actual_table)) if a!=b]==[0,1,7,14,86,87,168,169,172,185] and actual_table[14]==int(abi['free'],16) and actual_table[86]==int(abi['start'],16),'Observed real object uses untouched original operational methods and the actual new native ABI prefix') +require(p['stage']==3 and not p['status'] and p['allocated']==p['initialized']==p['parent_match']==p['released']==p['provider_initialized']==1 and not p['unrelated_match'] and not p['attached'] and not p['wrong_start'] and p['size']==304 and p['parent_size']==216 and p['object'] and p['meta']==int(abi['metaclass'],16) and p['vptr']==int(abi['vptr'],16),'Real original concrete class C2/initialization, genuine ancestry and wrong-provider start refusal, without any attachment') +require(p['instances_live']==p['instances_before']+1 and p['instances_after']==p['instances_before'],'Actual original instance count rises/falls through original free/D2') +require(m['stage']==1 and m['status']==0xe00002f0-(1<<32) and not m['meta'] and not m['allocated'] and not m['initialized'] and not m['released'],'Original class absent genuinely refuses with native kIOReturnNotFound before allocation') +require(z['stage']==0 and z['status']==-1 and all(not value for key,value in z.items() if key!='status') and off['stop_symbol']=='original-launchd-main' and off['original_launchd_main_executed'] and off['followed_main_entry']['first_instruction_completed'] and off['owner_root_binding_audit']['boot_root_accepted'] and off['original_amfi_table_owner_observed'] and off['actual_signature_failures']==dict(_cs_validate_page_bad_hash=0,_cs_validate_page_no_hash=0) and off['direct_user_commpage_shifts']=={'fffff4025:1':12,'fffff4037:1':12},'Default-off new kernel genuinely executes unchanged original launchd main/root/AMFI/native-4K ABI with no original-client allocation') +(ROOT/'build/native-offline-accelerator/original-client-checkpoint.json').write_text(json.dumps(dict(candidate_sha256=plan['candidate_sha256'],native_sha256=plan['native_sha256'],actual_original_userclient_instance=True,actual_original_init_and_free=True,wrong_provider_start_refused=True,instance_count_restored=True,positive_report_sha256=sha(ROOT/'build/darwinboot-owner-hvf-original-client-abi/gdb-stop.json'),missing_report_sha256=sha(ROOT/'build/darwinboot-owner-hvf-original-client-abi-missing-base/gdb-stop.json'),default_off_report_sha256=sha(ROOT/'build/darwinboot-owner-hvf-original-client-abi-default-off/gdb-stop.json'),no_original_client_published=True,normal_system_authorization=False,actual_gpu_execution=False,original_desktop=False,active_native_adreno_4k=False),indent=2)+'\n') +print('PASS: genuine original IOGPU userclient C2/init/free, real wrong-provider refusal and restored instance count; missing/default-off controls pass') +print('No operational client/provider, normal system authorization, GPU execution or original desktop is qualified') diff --git a/scripts/check-original-gpu-client-factory.py b/scripts/check-original-gpu-client-factory.py new file mode 100755 index 0000000..a22fb53 --- /dev/null +++ b/scripts/check-original-gpu-client-factory.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +"""Check real original client factory/task/subtype and first descriptor boundary.""" +import argparse,hashlib,json,struct,subprocess,sys +from pathlib import Path +ROOT=Path(__file__).resolve().parents[1];sys.path.insert(0,str(ROOT/'kits/NativePrelink')) +from native_prelink import macho,backed_offset +def require(value,why): + if not value:raise SystemExit(why) +def load(path):return json.loads(path.read_text()) +def sha(path):return hashlib.sha256(path.read_bytes()).hexdigest() +p=argparse.ArgumentParser(description=__doc__);p.add_argument('--candidate-base',type=Path,default=ROOT/'build/native-prelink/native-owner-original-descriptor-refusal-candidate');args=p.parse_args();base=args.candidate_base.resolve() +# This independently qualifies all unchanged operational provider addresses, +# real original lifetime, required configureDevice hook and default-off boot. +subprocess.run(['python3',ROOT/'scripts/check-original-gpu-provider.py','--candidate-base',base],check=True,timeout=30) +plan=load(base.with_suffix('.json'));raw=base.with_suffix('.macho').read_bytes();segments=macho(raw)['segments'] +abi=load(ROOT/'build/gpu-analysis/original-iogpu-userclient-abi.json');contract=load(ROOT/'build/gpu-analysis/original-iogpu-client-contract.json') +require(contract['original_kernel_sha256']==plan['original_sha256'] and contract['original_iogpu_text_sha256']==plan['original_executable_sha256']['com.apple.iokit.IOGPUFamily'],'Exact supplied target/client contract identity') +for field in ('new_user_client','custom_client_initialize','initialize_device','cleanup'): + extent=contract[field];at=backed_offset(segments,int(extent['address'],16),extent['bytes'],len(raw)) + require(hashlib.sha256(raw[at:at+extent['bytes']]).hexdigest()==extent['sha256'],'Actual unchanged original executable contract '+field) +rows=[] +for kind in (1,0xabcd0001,0): + folder=ROOT/'build'/('darwinboot-owner-hvf-original-client-factory-'+hex(kind));path=folder/'gdb-stop.json';d=load(path);sink=load(folder/'console-sink.json');provider=d['native_original_provider'] + require(d['kernel_sha256']==plan['candidate_sha256'] and not d['bounded_timeout_interrupted'] and d['qemu_accelerator']=='hvf' and d['native_4k_mmu_observed'] and not d['sheng_hardware_tested'] and d['stop_symbol']=='_native_original_provider_checkpoint','Actual bounded new-kernel factory observation') + registers={k:int(v,16) for k,v in d['system_registers'].items()} + require((registers['TCR_EL1']>>14)&3==0 and (registers['TCR_EL1']>>30)&3==2 and registers['SCTLR_EL1']&1 and registers['TTBR0_EL1'] and registers['TTBR1_EL1'],'Actual active native CPU4K hardware translation') + require(d['original_crypto_initialized_observed'] and d['original_crypto_table_owner_observed'] and sink['protocol_responses_sent']==0 and 0<=sink['bytes_observed']<=65536 and not sink['errors'] and not sink['gpu_execution'] and not sink['system_device_published'],'Actual original crypto and bounded response-free console sink') + console=d['native_console_globals'];require(all(console['_native_console_'+name]==0 for name in ('tx_requests','rx_requests','failures','tx_bytes','rx_bytes')),'No GPU host protocol traffic') + broker=d['native_gpu_broker'];require(broker['stage']==3 and not broker['status'] and not broker['clients'] and not broker['requests'] and not broker['actual_cpu_contexts'] and not any(d['native_gpu_shared_events'].values()),'No private broker client, GPU operation or shared capability opened') + require(provider['allocated']==provider['initialized']==provider['parent_match']==1 and provider['size']==736 and provider['object'] and provider['backend'] and provider['client_type']==kind and not provider['registered'] and not provider['start_returned'] and not provider['start_result'],'Actual unpublished own provider initialized; no successful provider start/publication') + if kind: + require(provider['stage']==4 and provider['status']==0xe00002c7-(1<<32) and provider['callback']==170 and not provider['released'] and not provider['factory_returned'] and not provider['factory_result'],'Real factory deliberately stops at unimplemented descriptor callback before returning a client') + require(provider['instances_live']==provider['instances_before']+1 and provider['client_instances_live']==provider['client_instances_before']+1,'Actual own provider and actual original client each have a live instance') + require(provider['client_instances_at_callback']==provider['client_instances_live'] and not provider['descriptor_refused'] and not provider['factory_out_client'],'Default capture never substitutes a descriptor refusal or returned client') + frame=provider['frame'];capture=provider['factory_client_capture'];client=int(capture['address'],16);memory=bytes.fromhex(capture['bytes']) + require(len(memory)==304 and int(capture['pointer'],16)==frame[1]==client+contract['shared_descriptor_out_offset'] and frame[0]==provider['object'] and frame[9]==int(contract['first_required_callback_caller_lr'],16),'Actual original client storage/output and original descriptor call site/LR') + require(struct.unpack_from('>16,'Real original client class, provider and scalar high16 subtype, no substituted vtable/object') + require(struct.unpack_from('>14)&3==0 and (frame[10]>>30)&3==2 and frame[11]&1 and frame[12] and frame[13] and frame[14]%16==0,'Actual naked callback native4K translation registers and aligned SP') + else: + require(provider['stage']==3 and provider['status']==provider['factory_result']==0xe00002c2-(1<<32) and provider['factory_returned']==provider['released']==1 and not provider['callback'] and not any(provider['frame']) and not provider['attached'] and provider['instances_after']==provider['instances_before'] and provider['client_instances_live']==provider['client_instances_before'],'Original invalid type refuses with real kIOReturnBadArgument, creates no client and fully releases provider') + rows.append(dict(open_type=kind,report_sha256=sha(path),required_slot=provider['callback'],client_started=False,factory_returned=bool(provider['factory_returned']),provider_released=bool(provider['released']))) +out=ROOT/'build/native-original-provider/client-factory-checkpoint.json' +out.write_text(json.dumps(dict(candidate_sha256=plan['candidate_sha256'],native_sha256=plan['native_sha256'],contract_sha256=sha(ROOT/'build/gpu-analysis/original-iogpu-client-contract.json'),observations=rows,actual_original_client_factory=True,actual_original_task_subtype_initialization=True,shared_descriptor_implemented=False,device_factory_implemented=False,original_client_started=False,provider_started=False,normal_device_discovery=False,original_desktop=False,adreno_execution=False,active_native_adreno_4k=False),indent=2)+'\n') +print('PASS: genuine original provider factory/custom task+subtype/client-start reach required descriptor slot170; high16 subtype and invalid-type cleanup controls') +print('The required descriptor is absent and the hook never returns; no successful client/provider startup, desktop or Adreno execution is claimed') diff --git a/scripts/check-original-gpu-descriptor-provision.py b/scripts/check-original-gpu-descriptor-provision.py new file mode 100644 index 0000000..64f40d8 --- /dev/null +++ b/scripts/check-original-gpu-descriptor-provision.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +"""Qualify the real descriptor-provision boundary of the original IOGPU provider. + +The opt-in instrument supplies one real owned IOBufferMemoryDescriptor at the +original slot-170 hook. The provider object is still deliberately unattached +and unregistered, so the original client factory stops with its genuine +kIOReturnInternalError. No device, mapping, queue, GPU submission, Metal +device or desktop is claimed. +""" +import json +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +REPORT = ROOT / 'build/darwinboot-owner-hvf-original-descriptor-provision-0x1/gdb-stop.json' +CHECKPOINT = ROOT / 'build/native-original-provider/descriptor-provision-checkpoint.json' + +report = json.loads(REPORT.read_text()) +provider = report.get('native_original_provider') +if not provider: + raise SystemExit('Missing original provider observation') +if provider.get('descriptor_provisioned') != 1: + raise SystemExit('No real descriptor was provisioned') +if provider.get('descriptor_refused') != 0: + raise SystemExit('Descriptor provision must not take the refusal path') +if provider.get('callback') != 170: + raise SystemExit(f"Expected slot-170 observation, got {provider.get('callback')!r}") +if provider.get('attached') != 0 or provider.get('registered') != 0: + raise SystemExit('This checkpoint requires the deliberately unattached provider') +if provider.get('factory_result') != -536870199: + raise SystemExit(f"Expected genuine kIOReturnInternalError, got {provider.get('factory_result')!r}") +if provider.get('client_instances_live') != 0: + raise SystemExit('No client instance may survive this boundary') +if not report.get('native_4k_mmu_observed'): + raise SystemExit('Native 4K must be active') +result = { + 'kernel_sha256': report['kernel_sha256'], + 'descriptor_provisioned': True, + 'descriptor_hook': 170, + 'provider_allocated': bool(provider.get('allocated')), + 'provider_initialized': bool(provider.get('initialized')), + 'provider_attached': False, + 'provider_registered': False, + 'client_factory_result': provider['factory_result'], + 'client_factory_result_name': 'kIOReturnInternalError', + 'client_instances_live': 0, + 'mapping_or_device_supplied': False, + 'gpu_submission': False, + 'system_metal_device': False, + 'native_4k_observed': True, + 'qemu_only': True, + 'sheng_hardware_tested': False, +} +CHECKPOINT.parent.mkdir(parents=True, exist_ok=True) +CHECKPOINT.write_text(json.dumps(result, indent=2) + '\n') +print('PASS: real descriptor reaches original slot 170') +print('PASS: genuine factory internal error from the unattached provider') +print('Open: provider attach/register, slot 189 device, mapping/queue, GPU execution') diff --git a/scripts/check-original-gpu-descriptor-refusal.py b/scripts/check-original-gpu-descriptor-refusal.py new file mode 100755 index 0000000..6cb0987 --- /dev/null +++ b/scripts/check-original-gpu-descriptor-refusal.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python3 +"""Qualify genuine original client cleanup after an explicit absent descriptor.""" +import argparse,hashlib,json,subprocess +from pathlib import Path +ROOT=Path(__file__).resolve().parents[1] +def load(path):return json.loads(path.read_text()) +def sha(path):return hashlib.sha256(path.read_bytes()).hexdigest() +def require(ok,why): + if not ok:raise SystemExit(why) +p=argparse.ArgumentParser(description=__doc__);p.add_argument('--candidate-base',type=Path,default=ROOT/'build/native-prelink/native-owner-original-descriptor-refusal-candidate');args=p.parse_args();base=args.candidate_base.resolve() +subprocess.run(['python3',ROOT/'scripts/check-original-gpu-client-factory.py','--candidate-base',base],check=True,timeout=30) +plan=load(base.with_suffix('.json'));contract=load(ROOT/'build/gpu-analysis/original-iogpu-client-contract.json');rows=[] +for kind in (1,0xabcd0001): + folder=ROOT/'build'/('darwinboot-owner-hvf-original-client-descriptor-refusal-'+hex(kind));report=folder/'gdb-stop.json';d=load(report);sink=load(folder/'console-sink.json');provider=d['native_original_provider'] + require(d['kernel_sha256']==plan['candidate_sha256'] and not d['bounded_timeout_interrupted'] and d['qemu_accelerator']=='hvf' and d['native_4k_mmu_observed'] and not d['sheng_hardware_tested'] and d['stop_symbol']=='_native_original_provider_checkpoint','Actual bounded absent-descriptor observation') + require(d['original_crypto_initialized_observed'] and d['original_crypto_table_owner_observed'] and sink['protocol_responses_sent']==0 and 0<=sink['bytes_observed']<=65536 and not sink['errors'] and not sink['gpu_execution'] and not sink['system_device_published'],'Real original crypto and response-free console control') + broker=d['native_gpu_broker'];console=d['native_console_globals'] + require(broker['stage']==3 and not broker['status'] and not broker['clients'] and not broker['requests'] and not broker['actual_cpu_contexts'] and not any(d['native_gpu_shared_events'].values()) and all(console['_native_console_'+name]==0 for name in ('tx_requests','rx_requests','failures','tx_bytes','rx_bytes')),'Zero actual GPU commands, broker clients or shared capabilities') + require(provider['stage']==3 and provider['status']==provider['factory_result']==0xe00002c9-(1<<32) and provider['factory_returned']==provider['descriptor_refused']==1 and not provider['factory_out_client'],'Original factory returned genuine failure with NULL output after actual absent descriptor') + require(provider['allocated']==provider['initialized']==provider['parent_match']==provider['released']==1 and provider['size']==736 and provider['client_type']==kind and provider['callback']==170 and not provider['attached'] and not provider['registered'] and not provider['start_returned'] and not provider['start_result'],'Actual real initialization/refusal and fully detached unpublished provider release') + require(provider['instances_live']==provider['instances_before']+1 and provider['instances_after']==provider['instances_before'] and provider['client_instances_at_callback']==provider['client_instances_before']+1 and provider['client_instances_live']==provider['client_instances_before'],'Both actual original client and own provider instance counts restored after genuine original error/cleanup') + frame=provider['frame'];require(frame[0]==provider['object'] and frame[1] and frame[9]==int(contract['first_required_callback_caller_lr'],16),'Actual absent-descriptor callback comes from original client call site') + registers={name:int(value,16) for name,value in d['system_registers'].items()} + for tcr,sctlr,ttbr0,ttbr1 in ((frame[10],frame[11],frame[12],frame[13]),(registers['TCR_EL1'],registers['SCTLR_EL1'],registers['TTBR0_EL1'],registers['TTBR1_EL1'])): + require((tcr>>14)&3==0 and (tcr>>30)&3==2 and sctlr&1 and ttbr0 and ttbr1,'Actual native CPU4K translation both at callback and after original cleanup') + require(frame[14]%16==0 and 'factory_client_capture' not in provider,'Real original SP alignment; observer never dereferences freed client storage') + rows.append(dict(open_type=kind,report_sha256=sha(report),original_result=provider['factory_result'],client_count_restored=True,provider_count_restored=True)) +(ROOT/'build/native-original-provider/descriptor-refusal-checkpoint.json').write_text(json.dumps(dict(candidate_sha256=plan['candidate_sha256'],native_sha256=plan['native_sha256'],observations=rows,actual_original_error_cleanup=True,absent_descriptor=True,backend_resources_implemented=False,successful_client_start=False,successful_provider_start=False,normal_device_discovery=False,original_desktop=False,adreno_execution=False,active_native_adreno_4k=False),indent=2)+'\n') +print('PASS: genuine NULL descriptor refusal executes original client failure/detach/release; both original-client/provider counts restored, output NULL, nativeCPU4K and no GPU traffic') +print('Real backend resources, successful startup, original desktop and Adreno/SMMU remain unimplemented/unqualified') diff --git a/scripts/check-original-gpu-provider.py b/scripts/check-original-gpu-provider.py new file mode 100644 index 0000000..18b05a4 --- /dev/null +++ b/scripts/check-original-gpu-provider.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +"""Check unpublished original provider lifetime and its real first required callback.""" +import argparse,hashlib,json,struct,sys +from pathlib import Path +ROOT=Path(__file__).resolve().parents[1];sys.path.insert(0,str(ROOT/'kits/NativePrelink')) +from native_prelink import macho,backed_offset,exports + +def load(p):return json.loads(p.read_text()) +def sha(p):return hashlib.sha256(p.read_bytes()).hexdigest() +def require(ok,why): + if not ok:raise SystemExit(why) +p=argparse.ArgumentParser(description=__doc__);p.add_argument('--candidate-base',type=Path,default=ROOT/'build/native-prelink/native-owner-original-descriptor-refusal-candidate');args=p.parse_args() +base=args.candidate_base.resolve();plan=load(base.with_suffix('.json'));raw=base.with_suffix('.macho').read_bytes();segments=macho(raw)['segments'] +require(sha(base.with_suffix('.macho'))==plan['candidate_sha256'] and sha(ROOT/'build/xnu/ios-obj/DEVELOPMENT_ARM64_BCM2837/mach.development.bcm2837.unstripped.noctf')==plan['native_sha256'],'Exact current native kernel/candidate') +reference=load(ROOT/'build/native-prelink/native-owner-metal-shared-events-candidate.json');require(plan['components']==reference['components'] and plan['original_executable_sha256']==reference['original_executable_sha256'] and plan['early_security_initializers']==reference['early_security_initializers'],'Same 29 unchanged original components/security entry') +for module,digest in plan['original_executable_sha256'].items(): + e=next(e for e in plan['extents'] if e['module']==module and e['segment']=='__TEXT_EXEC');o=backed_offset(segments,e['va'],e['size'],len(raw));require(hashlib.sha256(raw[o:o+e['size']]).hexdigest()==digest,'Actual whole original executable identity '+module) +build=load(ROOT/'build/native-original-provider/build-report.json');broker_build=load(ROOT/'build/native-offline-gpu/build-report.json');abi=load(ROOT/'build/gpu-analysis/original-iogpu-provider-abi.json') +for manifest in (build['source_sha256'],broker_build['source_sha256']): + for rel,digest in manifest.items():require(sha(ROOT/rel)==digest,'Exact compiled own source '+rel) +require(sha(ROOT/'build/native-original-provider/libNativeIOGPUProviderProbe.late.a')==build['archive_sha256'] and sha(ROOT/'build/native-offline-gpu/libNativeOfflineGPU.late.a')==broker_build['archive_sha256'] and sha(ROOT/'build/gpu-analysis/original-iogpu-provider-abi.json')==build['abi_sha256'],'Actual compiled archives and original ABI inputs') +reports=[] +for suffix in ('','-start','-missing-base','-default-off'): + fixture=ROOT/('build/darwinboot-owner-hvf-original-provider-start' if suffix=='-start' else 'build/darwinboot-owner-hvf-original-provider-abi'+suffix);d=load(fixture/'gdb-stop.json');sink=load(fixture/'console-sink.json') + require(d['kernel_sha256']==plan['candidate_sha256'] and not d['bounded_timeout_interrupted'] and d['qemu_accelerator']=='hvf' and d['native_4k_mmu_observed'] and not d['sheng_hardware_tested'],'Actual bounded new native-4K kernel observation') + r={k:int(v,16) for k,v in d['system_registers'].items()};require((r['TCR_EL1']>>14)&3==0 and (r['TCR_EL1']>>30)&3==2 and r['SCTLR_EL1']&1 and r['TTBR0_EL1'] and r['TTBR1_EL1'],'Actual active CPU TCR/TTBR/SCTLR, independent of GPU SMMU') + require(sink['protocol_responses_sent']==0 and 0<=sink['bytes_observed']<=65536 and not sink['errors'] and not sink['gpu_execution'] and not sink['system_device_published'],'Bounded firmware console prefix receives zero protocol responses; no GPU execution/publication') + console=d['native_console_globals'];require(all(console['_native_console_'+name]==0 for name in ('tx_requests','rx_requests','failures','tx_bytes','rx_bytes')),'Actual kernel transport sends/receives no GPU protocol bytes after the independent firmware prefix') + require(d['original_crypto_initialized_observed'] and d['original_crypto_table_owner_observed'],'Actual unchanged original crypto initialization') + b=d['native_gpu_broker'];require(not b['clients'] and not b['requests'] and not b['actual_cpu_contexts'] and not any(d['native_gpu_shared_events'].values()),'No actual GPU userclient, command or shared capability opened') + if suffix!='-default-off':require(d['stop_symbol']=='_native_original_provider_checkpoint' and b['stage']==3 and not b['status'],'Actual early private hook boundary, not an OS/desktop boot') + reports.append(d) +positive,started,missing,off=reports +p=positive['native_original_provider'];t=started['native_original_provider'];m=missing['native_original_provider'];z=off['native_original_provider'] +names={name:address for address,group in exports(raw,macho(raw)).items() for name in group} +def table(address,count): + o=backed_offset(segments,address,count*8,len(raw));return struct.unpack_from('<'+'Q'*count,raw,o) +original=table(abi['vptr'],215) +reference_raw=(ROOT/'build/native-prelink/native-owner-renderbox-presentation-candidate.macho').read_bytes() +reference_exports=exports(reference_raw,macho(reference_raw)) +expected=[] +for address in abi['inherited_entries']: + aliases=reference_exports.get(address,[]) + relocated={names[name] for name in aliases if name in names} + require(len(relocated)<=1,'Unambiguous native inherited export relocation') + expected.append(next(iter(relocated)) if relocated else address) +require(list(original)==expected,'Whole original table preserves original implementations with exact new native export relocation') +for obj in (p,t): + require(obj['parent']==abi['metaclass'] and obj['size']==736 and obj['allocated']==obj['initialized']==obj['parent_match']==1 and obj['object'] and obj['backend'],'Actual own object has genuine original parent and storage extent') + require(not obj['attached'] and not obj['registered'] and not obj['start_returned'] and not obj['start_result'],'No attachment, publication or successful original startup') + require(obj['instances_live']==obj['instances_before']+1,'Actual own instance count increases') + actual=table(obj['vptr'],215) + require([i for i,(a,b) in enumerate(zip(original,actual)) if a!=b]==build['own_entries'],'Only own metadata, nonreturning required hooks and publication guard differ') + for slot in (14,81,82,86,87,111,141):require(actual[slot]==original[slot],'Actual operational slot still targets original code '+str(slot)) +require(p['stage']==3 and not p['status'] and p['released']==1 and p['instances_after']==p['instances_before'] and not p['callback'] and not any(p['frame']),'Original C2/init/free/D2 restores actual instance count with no required hook') +require(t['stage']==4 and t['status']==0xe00002c7-(1<<32) and t['callback']==195 and not t['released'] and not t['instances_after'],'Real original start intentionally stops at first unimplemented required hook; no cleanup or successful return claimed') +f=t['frame'];require(f[0]==t['object'] and f[1]==t['backend'] and f[9]==abi['first_required_dispatch']['pc']+20,'Actual this/provider registers and caller LR come from the exact original start call site') +require((f[10]>>14)&3==0 and (f[10]>>30)&3==2 and f[11]&1 and f[12] and f[13] and f[14]%16==0,'Naked required hook captures actual native CPU 4K translation and aligned original SP') +require(m['stage']==1 and m['status']==0xe00002f0-(1<<32) and not m['parent'] and not m['allocated'] and not m['initialized'] and not m['released'],'Missing original class returns real kIOReturnNotFound before own registration/allocation') +require(z['stage']==0 and z['status']==-1 and all(not any(value) if isinstance(value,list) else not value for key,value in z.items() if key!='status') and off['stop_symbol']=='original-launchd-main' and off['original_launchd_main_executed'] and off['followed_main_entry']['first_instruction_completed'] and off['owner_root_binding_audit']['boot_root_accepted'] and off['original_amfi_table_owner_observed'] and off['actual_signature_failures']==dict(_cs_validate_page_bad_hash=0,_cs_validate_page_no_hash=0) and off['direct_user_commpage_shifts']=={'fffff4025:1':12,'fffff4037:1':12},'Default-off actually executes unchanged original launchd/root/AMFI/native4K userspace with zero provider allocation') +result=dict(candidate_sha256=plan['candidate_sha256'],native_sha256=plan['native_sha256'],actual_original_protected_constructor_destructor=True,actual_original_init_free=True,lifecycle_instance_count_restored=True,actual_original_start_entered=True,actual_first_required_slot=195,actual_this_provider_and_caller=True,required_callback_implemented=False,start_successful=False,start_object_released=False,normal_system_device_published=False,normal_system_authorization=False,actual_gpu_execution=False,original_desktop=False,active_native_adreno_4k=False,reports_sha256={name:sha(ROOT/'build'/name/'gdb-stop.json') for name in ('darwinboot-owner-hvf-original-provider-abi','darwinboot-owner-hvf-original-provider-start','darwinboot-owner-hvf-original-provider-abi-missing-base','darwinboot-owner-hvf-original-provider-abi-default-off')}) +(ROOT/'build/native-original-provider/runtime-checkpoint.json').write_text(json.dumps(result,indent=2)+'\n') +print('PASS: original C2/init/free/D2 lifetime; actual original start reaches required slot195 with real this/provider/LR; missing/default-off controls') +print('The required hook deliberately does not return. No successful startup, publication, GPU execution, active Adreno SMMU or desktop is qualified') diff --git a/scripts/check-original-launchd-integrity-checkpoints.py b/scripts/check-original-launchd-integrity-checkpoints.py new file mode 100644 index 0000000..a41cb42 --- /dev/null +++ b/scripts/check-original-launchd-integrity-checkpoints.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 +"""Require real refusal of one mutated original launchd code-page byte.""" +import hashlib +import json +from pathlib import Path +import runpy + +ROOT = Path(__file__).resolve().parents[1] +base = runpy.run_path(str(ROOT / 'scripts/check-original-launchd-main-checkpoints.py')) +require, positive = base['require'], base['main'] +d = json.loads((ROOT / 'build/darwinboot-owner-hvf-launchd-main-code-fault/gdb-stop.json').read_text()) +require(d['kernel_sha256'] == positive['kernel_sha256'] and + d['trust_cache_bundle_sha256'] == positive['trust_cache_bundle_sha256'] and + d['virtio_block_input'] == positive['virtio_block_input'] and + d['virtio_extra_block_input'] == positive['virtio_extra_block_input'] and + not d['bounded_timeout_interrupted'], 'Require the exact same kernel, original media and trust input') +fault = d['own_mapped_page_mutation'] +require(fault and fault['file_page'] == 81920 and fault['byte_in_page'] == 3976 and + fault['before'] == 255 and fault['after'] == 254 and + fault['page_sha256_before'] == '32f1004a16014e6f56fc8b18dcb4e7bed2bb38353be71725b881cfc09d462621' and + fault['code_directory_hash'] == '1d52c69747dffc4b1937d241bdbc9cee3b22df18' and + fault['compiled_cdhash_field_offset'] == 88 and not fault['registers_or_validation_state_modified'], + 'Apply exactly one SHA/CDHash-qualified actual vnode-page input fault; no validation state changes') +require(d['original_launchd_page_fault_input'] == dict( + file_sha256='8cec9a3c44a443b08e977ccc5d577850316ad428c575dc01f09b018dbd5b6a25', + code_directory_hash='1d52c69747dffc4b1937d241bdbc9cee3b22df18', offset=85896, + page_sha256=fault['page_sha256_before'], original_files_modified=False, + signatures_or_trust_modified=False, actual_fault_applied=True), 'Preserve the unchanged supplied signed launchd input') +require(d['actual_signature_failures'] == dict(_cs_validate_page_bad_hash=1, _cs_validate_page_no_hash=0) and + d['fatal_user_exit'] == dict(cs_status='CS_KILLED', prefix='initproc failed to start', namespace=3, code=2, description='none') and + not d['original_launchd_main_executed'] and d['mapped_page_validation_entries'] == 1656 and + d['mapped_page_validation_entries'] <= d['mapped_page_validation_entry_limit'] == 4096, + 'Genuine validation must kill the original process before main with a real bad hash') +require(d['owner_root_binding_audit']['boot_root_accepted'] and d['original_amfi_table_owner_observed'] and + d['original_crypto_initialized_observed'] and d['native_virtio_globals']['_native_virtio_read_failures'] == 0 and + d['native_virtio_integrity']['_native_virtio_integrity_failures'] == 0, 'Fault is after authenticated I/O, with real AMFI/crypto intact') +r = {k: int(v, 16) for k, v in d['system_registers'].items()} +require(r['TCR_EL1'] >> 14 & 3 == 0 and r['TCR_EL1'] >> 30 & 3 == 2 and r['SCTLR_EL1'] & 1 and + r['TTBR0_EL1'] and r['TTBR1_EL1'], 'Real refusal retains native hardware 4K') +print('PASS: one actual original launchd code byte is refused before LC_MAIN, bad hash 1/no hash 0 and CS_KILLED 3/2') diff --git a/scripts/check-original-launchd-main-checkpoints.py b/scripts/check-original-launchd-main-checkpoints.py new file mode 100644 index 0000000..0a009dd --- /dev/null +++ b/scripts/check-original-launchd-main-checkpoints.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +"""Qualify original Sandbox policy and original launchd LC_MAIN execution at native 4K.""" +import hashlib +import json +from pathlib import Path +import struct +import sys + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'kits/NativePrelink')) +import native_prelink as np + + +def require(value, message): + if not value: + raise SystemExit(message) + + +def report(name): + return json.loads((ROOT / ('build/darwinboot-' + name) / 'gdb-stop.json').read_text()) + + +original = (ROOT / '.cache/firmware/kernelcache.macho/kernelcache.release.ipad7.decompressed').read_bytes() +require(hashlib.sha256(original).hexdigest() == 'f18b4df865405f2a287b84977d7aca745e056d656d9fc9df1662970ce2969de7', 'Require supplied target') +trust = (ROOT / 'build/original-static-system-trust/original-static-system.bundle').read_bytes() +require(hashlib.sha256(trust).hexdigest() == '7416ca4f8377146626bb284e2133c76bee6cb4e3ee0af5d648b186235a7323a7' and + struct.unpack_from('> 14 & 3 == 0 and r['TCR_EL1'] >> 30 & 3 == 2 and r['SCTLR_EL1'] & 1 and + r['TTBR0_EL1'] and r['TTBR1_EL1'] and d['kernel_page_globals'] == + dict(_PAGE_SHIFT_CONST=12, _page_shift=12, _page_shift_user32=12, _page_size=4096), 'Actual hardware and kernel runtime remain native 4K') + require(d['original_amfi_table_owner_observed'] and d['original_crypto_initialized_observed'] and + d['original_crypto_table_owner_observed'] and d['owner_root_binding_audit']['boot_root_accepted'] and + d['owner_root_binding_audit']['_native_owner_root_apple_autherr'] == 80 and + d['actual_signature_failures'] == dict(_cs_validate_page_bad_hash=0, _cs_validate_page_no_hash=0), 'Original AMFI/crypto, owner root and code validation remain genuine') + require(d['native_virtio_globals']['_native_virtio_read_failures'] == 0 and + d['native_virtio_integrity']['_native_virtio_integrity_failures'] == 0, 'Keep actual media integrity') +require(policy['stop_symbol'] == '___mac_syscall' and policy['function_return_stack_matched'] and int(policy['x0'], 16) == 0 and + policy['entry_string_events'][-1]['value'] == 'Sandbox' and policy['entry_string_events'][-1]['matched'] and + bytes.fromhex(policy['entry_indirect_memory']['1:0:40']['bytes']).startswith(b'Sandbox\0'), 'Original registered Sandbox handler really accepts the selected request') +launchd = (ROOT / '.cache/firmware/target-volumes/os-mount/sbin/launchd').read_bytes() +require(hashlib.sha256(launchd).hexdigest() == main['original_launchd_main_input']['sha256'] == + '8cec9a3c44a443b08e977ccc5d577850316ad428c575dc01f09b018dbd5b6a25', 'Require unchanged supplied ARM64 launchd') +m = main['followed_main_entry'] +header = int(m['mach_header'], 16) +require(main['original_launchd_main_executed'] and main['stop_symbol'] == 'original-launchd-main' and + m['observed'] and m['first_instruction_completed'] and m['entry_offset'] == 85896 and + int(m['pc'], 16) == header + 85896 and int(m['checkpoint_pc'], 16) == int(main['pc'], 16) == header + 85900 and + int(m['pstate'], 16) & 15 == int(m['checkpoint_pstate'], 16) & 15 == int(main['pstate'], 16) & 15 == 0 and + m['entry_bytes_after_fault'] == launchd[85896:85912].hex() and m['mach_header_bytes'] == launchd[:32].hex(), + 'Actual original LC_MAIN and completed first instruction must match the original bytes at EL0') +require(struct.unpack('<2Q', bytes.fromhex(m['startup_stack_bytes'])) == (header, 1) and + m['argc_on_startup_stack'] == 1 and int(m['arguments'][0], 16) == 1 and + main['direct_user_commpage_shifts'] == {'fffff4025:1': 12, 'fffff4037:1': 12}, 'Follow actual dyld startup stack and real user commpage values, no guessed ASLR or ABI') +require(main['native_signature_hash_cache'] == dict(_cs_native4k_hash_cache_version=1, _cs_native4k_full_blocks=201083, + _cs_native4k_page_checks=1543, _cs_native4k_ready_blobs=53) and + main['owner_cache_carrier']['_native_owner_cache_carrier_accepts'] == 53 and + main['owner_cache_carrier']['_native_owner_cache_carrier_rejects'] == 0, 'Real original cache is authenticated and used through native-4K faults') +require(continuation['stop_symbol'] == '_panic_with_options' and continuation['panic_with_options']['format'].startswith('%s -- no exit reason available'), 'Preserve later real critical-process exit') +args = struct.unpack_from('<4Q', bytes.fromhex(continuation['stack_bytes'])) +raw_name = continuation['stack_pointer_memory']['0']['bytes'] +require(bytes.fromhex(raw_name).split(b'\0')[0] == b'mount[3] exited' and args[1] == 0 and args[2] == 66 and + bytes.fromhex(continuation['stack_pointer_memory']['3']['bytes']).split(b'\0')[0] == b'' and + continuation['native_signature_hash_cache']['_cs_native4k_page_checks'] == 2187, + 'Actual mount child exits 66 normally; do not mislabel this as an initproc code-sign kill') +print('PASS: unchanged original Sandbox policy is active; unchanged original launchd starts LC_MAIN at native-4K EL0') +print('PASS: actual first main instruction, original bytes, user shifts 12/12 and genuine later mount-child status 66') +print('Functional iPadOS UI, writable Data integration, firmware enrollment and sheng hardware remain open') diff --git a/scripts/check-original-mode-validation.py b/scripts/check-original-mode-validation.py new file mode 100644 index 0000000..b9acb93 --- /dev/null +++ b/scripts/check-original-mode-validation.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +"""Qualify the real SpringBoard mode-validation failure, without suppressing it.""" +import hashlib +import json +import mmap +from pathlib import Path +import struct + +ROOT=Path(__file__).resolve().parents[1] +def require(value,reason): + if not value:raise SystemExit(reason) +def sha(path):return hashlib.sha256(Path(path).read_bytes()).hexdigest() +# This is the immutable historical failure, independent of subsequent owned +# driver builds. Keep its exact candidate identity and original image input. +candidate=ROOT/'build/native-prelink/native-owner-legacy-display-info-candidate.macho' +plan=json.loads(candidate.with_suffix('.json').read_text()) +require(sha(candidate)==plan['candidate_sha256']=='195cbd11ae3f63d8589c2c931c09ed292bb1e599398bcd920b13a893ec48c23e' and + plan['original_sha256']=='f18b4df865405f2a287b84977d7aca745e056d656d9fc9df1662970ce2969de7', + 'Exact previously qualified original-family baseline candidate; subsequent builds do not replace it') +path=ROOT/'build/darwinboot-owner-hvf-legacy-display-start-info-exception-SpringBoard-registers/gdb-stop.json' +d=json.loads(path.read_text()) +symbol_path=ROOT/'build/native-legacy-display/info-SpringBoard-registers-symbolication.json' +s=json.loads(symbol_path.read_text()) +require(d['kernel_sha256']==s['kernel_sha256']==plan['candidate_sha256'] and + not d['bounded_timeout_interrupted'] and d['stop_symbol']=='_exception_triage' and + d['actual_process']['name']==d['entry_process_name']=='SpringBoard', + 'Actual selected SpringBoard exception on the exact original-family candidate') +events=d['entry_process_events'] +require(events and events[-1]['matched'] and events[-1]['process']==d['actual_process'] and + all(not e['matched'] and e['process']['name']!='SpringBoard' for e in events[:-1]) and + d['skipped_target_hits']==len(events)-1 and d['skipped_target_hits']>0, + 'Real bounded process selection skips unrelated exceptions without changing their execution') +require(len(d['entry_user_state_candidates'])==1,'Exactly one actual ARM64 saved EL0 state') +u=d['entry_user_state_candidates'][0] +regs=struct.unpack_from('<33Q',bytes.fromhex(u['saved_bytes']),8) +require(int(d['observed_function_entry_arguments'][0],16)==6 and + struct.unpack('<2Q',bytes.fromhex(d['entry_regions']['1:0:10']['bytes']))==(1,regs[32]) and + int(u['cpsr'],16)&31==0 and int(u['esr'],16)>>26==0x3c and + u['instruction_bytes'].startswith('000020d4'), + 'Actual EXC_BREAKPOINT code, EL0 state, syndrome and unchanged BRK agree') +require(s['local_symbols_uuid_matches'] and + int(s['frames'][0]['unslid_va'],16)==0x19f3d0744 and s['frames'][0]['offset']==580 and + s['frames'][0]['nearest_local_symbol']=='-[FBSDisplayMode _initWithWidth:height:preferredScale:scaleOverride:refreshRate:gamut:hdr:rotation:virtual:validityCheck:]' and + any(f['nearest_local_symbol']=='-[FBSDisplayConfiguration _initWithImmutableDisplay:originalDisplay:assertIfInvalid:]' for f in s['frames']), + 'UUID-bound original symbols identify the actual display mode validation call chain') +region=u['register_memory']['13:50'];raw=bytes.fromhex(region['bytes']) +require(int(region['address'],16)==regs[19] and len(raw)==80,'Read the actual preserved mode object through saved user x19') +isa,width,height,preferred,override,refresh,gamut,hdr,rotation,virtual=struct.unpack_from('<4Q2d3QB',raw) +require(isa and width==regs[25]==800 and height==regs[26]==600 and + preferred==regs[24]==0 and override==0.0 and refresh==60.0 and + (gamut,hdr,rotation,virtual)==(0,0,0,0), + 'Actual 800x600 mode has zero preferred/override scale and nonzero 60 Hz rate') +# The exact original initializer writes these fields and evaluates its own +# validity check. A missing capability is evidence to implement, not a reason +# to bypass this check or change original instructions. +cache=ROOT/'.cache/firmware/target-volumes/systemos-mount/System/Library/Caches/com.apple.dyld' +matched=False +for file in sorted(cache.glob('dyld_shared_cache_arm64*')): + if file.suffix=='.symbols':continue + with file.open('rb') as stream,mmap.mmap(stream.fileno(),0,access=mmap.ACCESS_READ) as b: + table,count=struct.unpack_from('<2I',b,16) + for index in range(count): + va,size,offset,_,protection=struct.unpack_from('<3Q2I',b,table+index*32) + if va<=0x19f3d0500 and 0x19f3d0500+584<=va+size: + at=offset+0x19f3d0500-va + require(protection&4 and hashlib.sha256(b[at:at+584]).hexdigest()== + 'ca4857e6617670f2369a0a6eb68be09e29135ce5dfda19ac1ea829229f0cc0ec', + 'Exact unchanged original mode initializer and validation instructions') + matched=True +require(matched,'Original executable mode initializer exists') +state=d['native_legacy_display'] +require(state['stage']==10 and state['start_result']==state['dt_attached']==1 and + state['client_calls']>=2 and not state['guard_slot'] and + d['native_4k_mmu_observed'] and d['original_amfi_table_owner_observed'] and + d['owner_root_binding_audit']['boot_root_accepted'] and + d['actual_signature_failures']==dict(_cs_validate_page_bad_hash=0,_cs_validate_page_no_hash=0), + 'Real display service/client state, native 4K, original AMFI and independent root acceptance remain intact') +out=ROOT/'build/native-legacy-display/original-mode-validation-checkpoint.json' +out.write_text(json.dumps(dict(candidate_sha256=plan['candidate_sha256'],checkpoint_sha256=sha(path), + symbolication_sha256=sha(symbol_path),actual_process=d['actual_process'], + mode=dict(width=width,height=height,preferred_scale=preferred,scale_override=override,refresh_rate=refresh), + historical_baseline=True,actual_original_validation_failure=True,original_instructions_modified=False, + original_desktop=False,system_metal_device=False,adreno_execution=False,hardware_tested=False),indent=2)+'\n') +print('PASS: actual original SpringBoard mode-validation failure qualified; 800x600/60 Hz, both scales zero') +print('Historical missing-scale baseline only; current successful mode return is qualified by check-original-display-mode.py') diff --git a/scripts/check-original-queue-completion.py b/scripts/check-original-queue-completion.py new file mode 100644 index 0000000..9fe453c --- /dev/null +++ b/scripts/check-original-queue-completion.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +"""Verify real original queue completion and native wait, with a pending control.""" +import hashlib +import json +from pathlib import Path +import struct +import sys +import zlib +ROOT=Path(__file__).resolve().parents[1];OUT=ROOT/'build/native-legacy-display' +sys.path.insert(0,str(ROOT/'kits/NativePrelink')) +from native_prelink import macho,backed_offset +def require(v,s): + if not v:raise SystemExit(s) +def load(p):return json.loads(p.read_text()) +def sha(p):return hashlib.sha256(p.read_bytes()).hexdigest() +build=load(OUT/'build-report.json') +for name,digest in build['source_sha256'].items():require(sha(ROOT/name)==digest,'Exact compiled source '+name) +require(sha(OUT/'libNativeLegacyDisplay.late.a')==build['archive_sha256'] and + build['guarded_backend_leaves']==[211,245,246,306,307,338,345] and + any('completeCurrentSwap' in k for k in build['bindings']) and build['owned_entries']==[0,1,7,14,84,86,87,141,155,184,188,211,213,245,246,288,290,296,306,307,337,338,345], + 'Actual archive and qualified original queue-kick override') +kernel=ROOT/'build/native-prelink/native-owner-legacy-display-completion-candidate.macho';plan=load(kernel.with_suffix('.json')) +raw=kernel.read_bytes();image=macho(raw) +require(sha(kernel)==plan['candidate_sha256'] and sha(ROOT/'build/xnu/ios-obj/DEVELOPMENT_ARM64_BCM2837/mach.development.bcm2837.unstripped.noctf')==plan['native_sha256'],'Exact current native candidate') +for module,digest in plan['original_executable_sha256'].items(): + e=next(e for e in plan['extents'] if e['module']==module and e['segment']=='__TEXT_EXEC') + o=backed_offset(image['segments'],e['va'],e['size'],len(raw)) + require(hashlib.sha256(raw[o:o+e['size']]).hexdigest()==digest,'Whole original executable unchanged '+module) +paths=[ROOT/('build/'+name+'/gdb-stop.json') for name in ( + 'darwinboot-owner-hvf-offline-legacy-swap-wait-completion-driver-pm', + 'darwinboot-owner-hvf-offline-legacy-swap-wait-retirement-disabled-completion-driver-pm')] +observations=[load(p) for p in paths] +for index,d in enumerate(observations): + l=d['native_legacy_display'] + require(d['kernel_sha256']==plan['candidate_sha256'] and not d['bounded_timeout_interrupted'] and + d['native_4k_mmu_observed'] and d['original_amfi_table_owner_observed'] and + d['owner_root_binding_audit']['boot_root_accepted'] and + d['actual_signature_failures']==dict(_cs_validate_page_bad_hash=0,_cs_validate_page_no_hash=0) and + l['stage']==10 and not l['guard_slot'] and l['object_size']==6328 and + l['swap_validation_calls']==1 and l['swap_validation_result']==1 and l['swap_validation_reason']==0, + 'Actual original provider/validation, native 4K and original AMFI/root') + require(d['stop_symbol']=='_exception_triage' and d['actual_process']['name']=='air-native' and d['actual_process']['pid']==1,'Actual own process boundary') + u=d['entry_user_state_candidates'][0];v=struct.unpack('<16Q',bytes.fromhex(u['register_memory']['13:80']['bytes'])) + require(int(u['cpsr'],16)&31==0 and u['instruction_bytes'].startswith('000020d4') and + v[:6]==(0x47505357,1,4096,4096,800,600) and v[7:12]==(480000,0,1,0,0) and + v[13]>=24000000 and v[14:]==((0 if index==0 else 0xe00002d5),4),'Actual completed GPU pixels, original API statuses and scheduling interval') +d,disabled=observations;l=d['native_legacy_display'];c=disabled['native_legacy_display'] +require(l['transfer_calls']==1 and l['transfer_status']==0 and l['transfer_bytes']==1920000 and + l['transfer_request']==l['swap_request'] and l['transfer_request'] and + c['transfer_calls']==1 and c['transfer_bytes']==1920000 and c['transfer_status']==0,'Both runs actually transfer the completed GPU frame') +require(l['retirement_calls']==1 and l['retired_generation']==l['presented_generation']==l['retired_frames']==1 and + l['retired_current']==l['retired_pending']==0 and c['retirement_calls']==0, + 'Original successful completion/generation advancement and actual empty queue; disabled control remains uncompleted') +gpu=ROOT/'build/offline-legacy-swap-wait';program=load(gpu/'build-report.json');data=load(gpu/'data-report.json');manifest=load(gpu/'test-inputs.json') +require(sha(gpu/'air-native')==program['process_sha256']==data['process_sha256'] and + sha(gpu/'test-inputs.json')==program['manifest_sha256']==data['manifest_sha256'] and + sha(gpu/'compiler-data-32.cdr')==data['sha256'],'Exact own signed/staged GPU process') +for name,digest in manifest['sources'].items():require(sha(ROOT/name)==digest,'Exact GPU source '+name) +for observation in observations: + require(observation['scratch_data']['actual_seed_sha256']==data['sha256'] and + observation['trust_cache_bundle_sha256']==program['trust_bundle_sha256']==sha(gpu/'offline-trust.bundle'),'Actual own Data/trust identity') +def modules(b): + n=struct.unpack_from('IIBBBBB',b,16)==(800,600,8,2,0,0,0),'Exact inspected actual QMP RGB PNG') +# Read the actual PNG; do not generate, modify or reconstruct a screenshot. +at=8;compressed=b'' +while atI',b,at)[0];kind=b[at+4:at+8];body=b[at+8:at+8+n] + if kind==b'IDAT':compressed+=body + at+=12+n +pixels=zlib.decompress(compressed);require(len(pixels)==600*2401,'Actual PNG row extent') +previous=bytearray(2400);rgb=bytearray() +for row in range(600): + filt=pixels[row*2401];current=bytearray(pixels[row*2401+1:(row+1)*2401]);require(filt<=4,'Standard PNG filter') + for i in range(2400): + left=current[i-3] if i>=3 else 0;above=previous[i];corner=previous[i-3] if i>=3 else 0 + if filt==4: + p=left+above-corner;dist=[abs(p-v) for v in (left,above,corner)];predict=(left,above,corner)[dist.index(min(dist))] + else:predict=(0,left,above,(left+above)//2)[filt] + current[i]=(current[i]+predict)&255 + rgb+=current;previous=current +require(rgb==bytes((51,102,153))*480000,'Every one of the 480,000 actual displayed pixels matches the GPU frame') +control=disabled['qemu_screen_capture'] +require(control['sha256']==sha(paths[1].parent/control['file'])==capture['sha256'],'Unretired control displays the same real GPU pixels but original wait refuses completion') +(OUT/'queue-completion-checkpoint.json').write_text(json.dumps(dict(candidate_sha256=sha(kernel),reports=[sha(p) for p in paths], + actual_gpu_pixels_visible_in_qemu=True,all_displayed_pixels_checked=480000,active_native_cpu_4k_leaves=469, + original_queue_retirement_implemented=True,original_wait_returned_success=True,pending_control_not_ready=True,display_fence_completed=False,physical_vblank=False, + system_metal=False,original_desktop=False,adreno_execution=False),indent=2)+'\n') +print('PASS: real GPU frame, original queue completion, original nonblocking/blocking wait and pending NotReady control') +print('Both actual QEMU screens match the GPU frame; 469 actual CPU 4K leaves; system Metal/desktop/Adreno remain open') diff --git a/scripts/check-original-queue-kick.py b/scripts/check-original-queue-kick.py new file mode 100644 index 0000000..ab8a79f --- /dev/null +++ b/scripts/check-original-queue-kick.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 +"""Check actual pending-request ownership at the original no-op queue kick.""" +import hashlib +import json +from pathlib import Path +import struct +import sys +ROOT=Path(__file__).resolve().parents[1] +sys.path.insert(0,str(ROOT/'kits/NativePrelink')) +from native_prelink import macho,backed_offset +def require(v,s): + if not v:raise SystemExit(s) +kernel=ROOT/'build/native-prelink/native-owner-legacy-display-validate-candidate.macho' +raw=kernel.read_bytes();image=macho(raw) +require(hashlib.sha256(raw).hexdigest()=='0c8e25d34b4e4987bd96794abecab882f796992691b008779e77a5111cb84497','Exact historical validation candidate') +offset=backed_offset(image['segments'],0xfffffff005794030,16,len(raw)) +require(raw[offset:offset+16]==bytes.fromhex('ff4300d1e00700f9ff430091c0035fd6'),'Exact unchanged original no-op queue-kick body') +path=ROOT/'build/darwinboot-owner-hvf-offline-legacy-swap-driver-pm-kick/gdb-stop.json' +d=json.loads(path.read_text());l=d['native_legacy_display'] +require(d['kernel_sha256']==hashlib.sha256(raw).hexdigest() and not d['bounded_timeout_interrupted'] and + int(d['pc'],16)==0xfffffff005794030 and int(d['x30'],16)==0xfffffff00579196c and + d['primary_start_event']['pc']=='0xfffffff00bf4e18c','Real original queue kick reached after actual own validator checkpoint') +tail=bytes.fromhex(d['entry_regions']['0:1000:800']['bytes']) +current,pending=struct.unpack_from('<2Q',tail,5472-4096) +require(current==0 and pending==l['swap_request'] and pending and l['swap_validation_result']==1 and + l['swap_validation_reason']==0 and l['swap_validation_calls']==1 and not l['guard_slot'], + 'Actual pending queue owns the identical validated original request, with no current request') +require(d['native_4k_mmu_observed'] and d['original_amfi_table_owner_observed'] and d['owner_root_binding_audit']['boot_root_accepted'],'Original AMFI/root and native 4K intact') +print('PASS: original queue owns the real validated request; its unchanged queue-kick implementation does no work') diff --git a/scripts/check-original-queue-transfer.py b/scripts/check-original-queue-transfer.py new file mode 100644 index 0000000..d4d8b77 --- /dev/null +++ b/scripts/check-original-queue-transfer.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +"""Verify actual GPU pixels on QEMU scanout through the original pending queue.""" +import hashlib +import json +from pathlib import Path +import struct +import sys +import zlib +ROOT=Path(__file__).resolve().parents[1];OUT=ROOT/'build/native-legacy-display' +sys.path.insert(0,str(ROOT/'kits/NativePrelink')) +from native_prelink import macho,backed_offset +def require(v,s): + if not v:raise SystemExit(s) +def load(p):return json.loads(p.read_text()) +def sha(p):return hashlib.sha256(p.read_bytes()).hexdigest() +build=load(OUT/'build-report.json') +for name,digest in build['source_sha256'].items():require(sha(ROOT/name)==digest,'Exact compiled source '+name) +require(sha(OUT/'libNativeLegacyDisplay.late.a')==build['archive_sha256'] and + build['owned_entries']==[0,1,7,14,84,86,87,141,155,184,188,211,213,245,246,288,290,296,306,307,337,338,345], + 'Actual archive and qualified original queue-kick override') +kernel=ROOT/'build/native-prelink/native-owner-legacy-display-transfer-ready-candidate.macho';plan=load(kernel.with_suffix('.json')) +raw=kernel.read_bytes();image=macho(raw) +require(sha(kernel)==plan['candidate_sha256'] and sha(ROOT/'build/xnu/ios-obj/DEVELOPMENT_ARM64_BCM2837/mach.development.bcm2837.unstripped.noctf')==plan['native_sha256'],'Exact current native candidate') +for module,digest in plan['original_executable_sha256'].items(): + e=next(e for e in plan['extents'] if e['module']==module and e['segment']=='__TEXT_EXEC') + o=backed_offset(image['segments'],e['va'],e['size'],len(raw)) + require(hashlib.sha256(raw[o:o+e['size']]).hexdigest()==digest,'Whole original executable unchanged '+module) +paths=[ROOT/('build/'+name+'/gdb-stop.json') for name in ( + 'darwinboot-owner-hvf-offline-legacy-swap-transfer-ready-driver-pm', + 'darwinboot-owner-hvf-offline-legacy-swap-transfer-disabled-transfer-ready-driver-pm')] +observations=[load(p) for p in paths] +for d in observations: + l=d['native_legacy_display'] + require(d['kernel_sha256']==plan['candidate_sha256'] and not d['bounded_timeout_interrupted'] and + d['native_4k_mmu_observed'] and d['original_amfi_table_owner_observed'] and + d['owner_root_binding_audit']['boot_root_accepted'] and + d['actual_signature_failures']==dict(_cs_validate_page_bad_hash=0,_cs_validate_page_no_hash=0) and + l['stage']==10 and not l['guard_slot'] and l['object_size']==6328 and + l['swap_validation_calls']==1 and l['swap_validation_result']==1 and l['swap_validation_reason']==0, + 'Actual original provider/validation, native 4K and original AMFI/root') + require(d['stop_symbol']=='_exception_triage' and d['actual_process']['name']=='air-native' and d['actual_process']['pid']==1,'Actual own process boundary') + u=d['entry_user_state_candidates'][0];v=struct.unpack('<16Q',bytes.fromhex(u['register_memory']['13:80']['bytes'])) + require(int(u['cpsr'],16)&31==0 and u['instruction_bytes'].startswith('000020d4') and + v[:6]==(0x47505357,1,4096,4096,800,600) and v[7:12]==(480000,0,1,0,0) and + v[13]>=24000000 and v[14:]==(1,4),'Actual completed GPU pixels, original API statuses and scheduling interval') +d,disabled=observations;l=d['native_legacy_display'];c=disabled['native_legacy_display'] +require(l['transfer_calls']==1 and l['transfer_status']==0 and l['transfer_bytes']==1920000 and + l['transfer_request']==l['swap_request'] and l['transfer_request'] and + c['transfer_calls']==c['transfer_bytes']==0 and c['transfer_status']==-1,'Actual identical queued request copied once; disabled control does not transfer') +gpu=ROOT/'build/offline-legacy-swap';program=load(gpu/'build-report.json');data=load(gpu/'data-report.json');manifest=load(gpu/'test-inputs.json') +require(sha(gpu/'air-native')==program['process_sha256']==data['process_sha256'] and + sha(gpu/'test-inputs.json')==program['manifest_sha256']==data['manifest_sha256'] and + sha(gpu/'compiler-data-32.cdr')==data['sha256'],'Exact own signed/staged GPU process') +for name,digest in manifest['sources'].items():require(sha(ROOT/name)==digest,'Exact GPU source '+name) +for observation in observations: + require(observation['scratch_data']['actual_seed_sha256']==data['sha256'] and + observation['trust_cache_bundle_sha256']==program['trust_bundle_sha256']==sha(gpu/'offline-trust.bundle'),'Actual own Data/trust identity') +def modules(b): + n=struct.unpack_from('IIBBBBB',b,16)==(800,600,8,2,0,0,0),'Exact inspected actual QMP RGB PNG') +# Read the actual PNG; do not generate, modify or reconstruct a screenshot. +at=8;compressed=b'' +while atI',b,at)[0];kind=b[at+4:at+8];body=b[at+8:at+8+n] + if kind==b'IDAT':compressed+=body + at+=12+n +pixels=zlib.decompress(compressed);require(len(pixels)==600*2401,'Actual PNG row extent') +previous=bytearray(2400);rgb=bytearray() +for row in range(600): + filt=pixels[row*2401];current=bytearray(pixels[row*2401+1:(row+1)*2401]);require(filt<=4,'Standard PNG filter') + for i in range(2400): + left=current[i-3] if i>=3 else 0;above=previous[i];corner=previous[i-3] if i>=3 else 0 + if filt==4: + p=left+above-corner;dist=[abs(p-v) for v in (left,above,corner)];predict=(left,above,corner)[dist.index(min(dist))] + else:predict=(0,left,above,(left+above)//2)[filt] + current[i]=(current[i]+predict)&255 + rgb+=current;previous=current +require(rgb==bytes((51,102,153))*480000,'Every one of the 480,000 actual displayed pixels matches the GPU frame') +control=disabled['qemu_screen_capture'] +require(control['sha256']==sha(paths[1].parent/control['file'])=='e90f3f3195332e37f9a95d7416dc79ca2fa63d08ed3e170f71704a5678000aaf','Disabled control retains actual previous loader frame') +(OUT/'queue-transfer-checkpoint.json').write_text(json.dumps(dict(candidate_sha256=sha(kernel),reports=[sha(p) for p in paths], + actual_gpu_pixels_visible_in_qemu=True,all_displayed_pixels_checked=480000,active_native_cpu_4k_leaves=469, + original_queue_retirement_implemented=False,display_fence_completed=False,physical_vblank=False, + system_metal=False,original_desktop=False,adreno_execution=False),indent=2)+'\n') +print('PASS: real GPU -> original IOSurface/pending queue -> actual QEMU scanout; all 480,000 pixels match') +print('469 actual native CPU 4K leaves; disabled control unchanged; queue retirement/system Metal/desktop remain open') diff --git a/scripts/check-original-render-server.py b/scripts/check-original-render-server.py new file mode 100644 index 0000000..9b3256a --- /dev/null +++ b/scripts/check-original-render-server.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +"""Verify the real render-server reply and independently captured QEMU scanout.""" +import hashlib +import json +from pathlib import Path +import plistlib +import struct +ROOT=Path(__file__).resolve().parents[1] +def require(value,reason): + if not value:raise SystemExit(reason) +def sha(path):return hashlib.sha256(Path(path).read_bytes()).hexdigest() +plan=json.loads((ROOT/'build/native-prelink/native-owner-legacy-display-scale-candidate.json').read_text()) +query_path=ROOT/'build/darwinboot-owner-hvf-legacy-display-start-scale-query/gdb-stop.json' +screen_path=ROOT/'build/darwinboot-owner-hvf-legacy-display-start-scale-exception-SpringBoard-screen/gdb-stop.json' +d=json.loads(query_path.read_text());screen=json.loads(screen_path.read_text()) +for observation in (d,screen): + require(observation['kernel_sha256']==plan['candidate_sha256'] and observation['native_4k_mmu_observed'] and + observation['original_amfi_table_owner_observed'] and observation['owner_root_binding_audit']['boot_root_accepted'] and + observation['actual_signature_failures']==dict(_cs_validate_page_bad_hash=0,_cs_validate_page_no_hash=0), + 'Same exact candidate, real native 4K, original AMFI/root and zero signature failures') +require(d['stop_symbol']=='original-display-query-return' and not d['bounded_timeout_interrupted'], + 'Actual unchanged original display query returns normally') +f=d['followed_display_query'] +require(f['observed'] and f['entry_process']['name']==f['actual_process']['name']=='SpringBoard' and + f['entry_process']['thread']==f['actual_process']['thread'] and f['caller_restored'] and + f['entry_bytes']==f['query_bytes_after_fault']==f['plan']['query_prefix'], + 'Real selected original query, unchanged instructions and restored caller') +port,reply=f['checkpoints'] +require(port['stage']=='server-port' and int(port['x0'],16)!=0 and reply['stage']=='display-reply' and int(reply['x0'],16)==0, + 'Real nonzero CARenderServer port and actual successful CASGetDisplays reply') +body=bytes.fromhex(reply['output_bytes']);require(len(body)==int.from_bytes(bytes.fromhex(reply['output_length']),'little'), + 'Actual reply extent matches captured bytes') +displays=plistlib.loads(body) +primary=[v for v in displays if v.get('kCADisplayDeviceName')=='primary'] +require(isinstance(displays,list) and len(primary)==1 and primary[0]['kCADisplayName']=='LCD' and + primary[0]['kCADisplayFrameCount']==0 and primary[0]['kCADisplayDirtyFrameCount']==0 and + int.from_bytes(bytes.fromhex(f['main_display_bytes']),'little') and int.from_bytes(bytes.fromhex(f['displays_bytes']),'little'), + 'The original server reports its LCD and original nonnull display objects, with initial counters zero') +capture=screen['qemu_screen_capture'];image=screen_path.parent/capture['file'];raw=image.read_bytes() +require(screen['bounded_timeout_interrupted'] and capture['width']==800 and capture['height']==600 and + raw[:8]==bytes.fromhex('89504e470d0a1a0a') and struct.unpack_from('>2I',raw,16)==(800,600) and + len(raw)==capture['bytes'] and sha(image)==capture['sha256'], + 'Actual QMP screendump produces the recorded bounded-run 800x600 PNG') +# This exact screenshot was visually inspected. It is still the loader GOP +# content, not evidence of an original UI frame. No image reconstruction or +# CPU renderer is used to create the QMP output. +require(capture['sha256']=='e90f3f3195332e37f9a95d7416dc79ca2fa63d08ed3e170f71704a5678000aaf', + 'Exact independently inspected loader-content screenshot; changed images require fresh inspection') +metal_path=ROOT/'build/darwinboot-owner-hvf-legacy-display-start-scale-metal/gdb-stop.json' +metal=json.loads(metal_path.read_text());m=metal['followed_display_query'];mp=m['plan'] +require(metal['kernel_sha256']==plan['candidate_sha256'] and not metal['bounded_timeout_interrupted'] and + metal['stop_symbol']=='original-default-metal-device-return' and m['observed'] and + m['entry_process']['name']==m['actual_process']['name']=='backboardd' and + m['entry_process']['thread']==m['actual_process']['thread'] and m['caller_restored'] and + m['return_pc']==m['caller']==metal['pc'] and m['return_stack']==m['entry_stack']==metal['sp'] and + m['entry_bytes']==m['query_bytes_after_fault']==mp['query_prefix'] and + mp['query']==0x187ac8668 and mp['return_checkpoint']==176 and + mp.get('original_function_sha256',mp.get('original_mode_initializer_sha256'))== + '8005e91f7637cf800246ae2130936f45cb57d6a0891a77c61f1f563e9b058d20' and + int(m['return_value'],16)==0 and m['returned_object_bytes'] is None, + 'Actual unchanged original MTLCreateSystemDefaultDevice returns nil to the real backboardd caller') +require(metal['native_4k_mmu_observed'] and metal['original_amfi_table_owner_observed'] and + metal['owner_root_binding_audit']['boot_root_accepted'] and + metal['actual_signature_failures']==dict(_cs_validate_page_bad_hash=0,_cs_validate_page_no_hash=0), + 'Original Metal observation preserves AMFI/root/native CPU 4K') +(ROOT/'build/native-legacy-display/render-server-checkpoint.json').write_text(json.dumps(dict( + candidate_sha256=plan['candidate_sha256'],query_checkpoint_sha256=sha(query_path),screen_checkpoint_sha256=sha(screen_path),metal_checkpoint_sha256=sha(metal_path), + screenshot_sha256=capture['sha256'],separate_bounded_vm_observations=True, + original_render_server_replied=True,actual_original_backboardd_metal_device=None,initial_primary_frame_count=0,actual_scanout_content='previous loader frame', + original_desktop=False,system_metal_device=False,adreno_execution=False,hardware_tested=False),indent=2)+'\n') +print('PASS: real original render server replies successfully with an LCD; actual QMP scanout capture qualified') +print('Actual backboardd receives nil from original Metal discovery; initial frames are zero and scanout remains loader content') diff --git a/scripts/check-original-sandbox-iokit.py b/scripts/check-original-sandbox-iokit.py new file mode 100644 index 0000000..0dffec2 --- /dev/null +++ b/scripts/check-original-sandbox-iokit.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +"""Record actual original Sandbox scoped IOKit authorization, including real denial.""" +import hashlib,json,re,struct,subprocess,sys +from pathlib import Path +ROOT=Path(__file__).resolve().parents[1];sys.path.insert(0,str(ROOT/'kits/NativePrelink')) +from native_prelink import macho,backed_offset +def require(value,why): + if not value:raise SystemExit(why) +def load(path):return json.loads(path.read_text()) +def sha(path):return hashlib.sha256(path.read_bytes()).hexdigest() +def modules(raw): + n=struct.unpack_from('1 and v[30]==0,'Actual entitled issuer open, both token issuances, real separate consumer and normal exit; no consumer opens succeeded') +# Child fields 6..21 are copied verbatim to parent fields 9..24. +require(v[9] and all(v[i]==0xe00002e2 for i in (10,12,14,17,19)) and all(v[i]==0 for i in (11,13,16,18)),'Actual unentitled consumer stays NotPermitted before, during both real consumed scopes and after each successful release') +require(v[23]-d['actual_process']['shared_region_slide']==0x1dd55bce8 and v[24]-d['actual_process']['shared_region_slide']==0x1dd55b9ac,'Exact unchanged original consume/release implementations, not local replacements') +require(d['entry_register_selection']==dict(index=0,expected='0x6',observed='0x6') and d['skipped_target_hits']==3,'Read-only observer selects actual EXC_BREAKPOINT after three earlier exception entries; target exception delivery is unmodified') +b=d['native_gpu_broker'];require(b['stage']==0 and b['clients']==b['requests']==b['closed']==b['active']==0,'No GPU broker publication, protocol traffic or fabricated rendering in this authorization observation') +require(all(d['kernel_page_globals'][k]==x for k,x in [('_PAGE_SHIFT_CONST',12),('_page_shift',12),('_page_shift_user32',12),('_page_size',4096)]) and int(d['system_registers']['TCR_EL1'],16)==0x22a51a251a and int(d['system_registers']['SCTLR_EL1'],16)&1,'Active native CPU registers and full native userspace 4K globals') +consumer=folder/'sandbox-consumer';require(sha(consumer)==build['consumer_sha256'],'Exact independently signed consumer input') +consumer_entitlements=subprocess.check_output(['codesign','-d','--entitlements',':-',consumer],stderr=subprocess.DEVNULL) +issuer_entitlements=subprocess.check_output(['codesign','-d','--entitlements',':-',folder/'air-native'],stderr=subprocess.DEVNULL) +import plistlib +require(plistlib.loads(consumer_entitlements)=={'com.apple.private.security.no-container':True} and plistlib.loads(issuer_entitlements)==build['entitlements'] and build['entitlements']=={'com.apple.private.security.no-container':True,'com.apple.security.iokit-user-client-class':['NativeOfflineGPUClient','IOSurfaceRootUserClient']},'Consumer has no IOKit allowlist; issuer retains only exact own diagnostic scopes') +subprocess.run(['codesign','--verify','--strict',consumer],check=True,capture_output=True,timeout=30) +code=[] +for file in (folder/'air-native',consumer): + binary=file.read_bytes();section=next(sec for seg in macho(binary)['segments'] for sec in seg['sections'] if seg['name']=='__TEXT' and sec['name']=='__text');code.append(binary[section['fileoff']:section['fileoff']+section['size']]) +require(code[0]==code[1],'Actual issuer and consumer machine instructions identical; only their signatures/scopes differ') +(folder/'checkpoint.json').write_text(json.dumps(dict(candidate_sha256=plan['candidate_sha256'],issuer_sha256=build['process_sha256'],consumer_sha256=build['consumer_sha256'],qemu_report_sha256=sha(fixture/'gdb-stop.json'),same_machine_instructions=True,entitled_issuer_open=True,real_tokens_issued=2,real_consume_release_pairs=2,consumer_opens_succeeded=0,consumer_status='NotPermitted',authorization_route_qualified=False,original_apps_modified=False,original_sandbox_unchanged=True,original_desktop=False,adreno_execution=False),indent=2)+'\n') +print('PASS observation: entitled issuer opens; two genuine Sandbox tokens consume/release, but consumer remains NotPermitted') +print('NEGATIVE authorization result: scoped tokens do not grant this target profile IOKit access; no system-client integration qualified') diff --git a/scripts/check-original-swap-gate.py b/scripts/check-original-swap-gate.py new file mode 100644 index 0000000..d796992 --- /dev/null +++ b/scripts/check-original-swap-gate.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +"""Verify the actual original valid-packet branch that bypasses submission.""" +import hashlib +import json +from pathlib import Path +import struct +import sys +ROOT=Path(__file__).resolve().parents[1] +sys.path.insert(0,str(ROOT/'kits/NativePrelink')) +from native_prelink import macho,backed_offset +def require(v,s): + if not v:raise SystemExit(s) +def sha(p):return hashlib.sha256(p.read_bytes()).hexdigest() +kernel=ROOT/'build/native-prelink/native-owner-legacy-display-scale-candidate.macho' +raw=kernel.read_bytes();image=macho(raw);plan=json.loads(kernel.with_suffix('.json').read_text()) +require(sha(kernel)==plan['candidate_sha256'],'Exact observed kernel') +def read(a,n): + o=backed_offset(image['segments'],a,n,len(raw));return raw[o:o+n] +# Exact original path: load self+5836, compare to zero, set bypass byte +# when zero, then skip the actual submit helper if packet-valid AND bypass. +words=struct.unpack('<21I',read(0xfffffff00579056c,84)) +require(words==(0xf94063e8,0xb956cd08,0x71000108,0x1a9f07e8,0x370000a8, + 0x14000001,0x52800028,0x3819f3a8,0x14000001,0x3859e3a8, + 0x360001c8,0x14000001,0x3859f3a8,0x36000088,0x14000001, + 0xb81ac3bf,0x14000007,0xf94063e0,0xf9404be8,0xf9400101, + 0x940000fc),'Exact original gate branch and submit call bytes') +path=ROOT/'build/darwinboot-owner-hvf-offline-legacy-swap-decision/gdb-stop.json' +d=json.loads(path.read_text()) +require(d['kernel_sha256']==plan['candidate_sha256'] and not d['bounded_timeout_interrupted'] and + int(d['pc'],16)==0xfffffff005790574 and int(d['x8'],16)==0, + 'Actual original load observes submission-control value zero') +frame=d['frame_chain'][0];local=bytes.fromhex(frame['local_bytes']);fp=int(frame['fp'],16);base=int(frame['local_address'],16) +require(local[fp-98-base]==1,'Actual original packet-validation result true') +packet=bytes.fromhex(d['frame_indirect_memory']['0:-28:508']['bytes']) +require(len(packet)==1288 and struct.unpack_from('<2I',packet,152)==(1,1) and + struct.unpack_from('<2I',packet,180)==(800,600) and + struct.unpack_from('<2I',packet,276)==(800,600),'Actual generation, surface ID and rectangles reach kernel') +require(d['native_4k_mmu_observed'] and d['original_amfi_table_owner_observed'] and + d['owner_root_binding_audit']['boot_root_accepted'] and not d['native_legacy_display']['guard_slot'], + 'Native 4K, original root/AMFI and unchanged backend remain intact') +(ROOT/'build/offline-legacy-swap/gate-checkpoint.json').write_text(json.dumps(dict( + report_sha256=sha(path),kernel_sha256=sha(kernel),original_packet_valid=True,actual_control_value=0, + branch_analysis='original valid-and-bypass path skips actual submit helper', + private_state_modified=False,original_desktop=False),indent=2)+'\n') +print('PASS: actual original packet is valid, but its real zero submission control selects the original bypass path') diff --git a/scripts/check-owner-boot-manifest-checkpoints.py b/scripts/check-owner-boot-manifest-checkpoints.py new file mode 100644 index 0000000..1191398 --- /dev/null +++ b/scripts/check-owner-boot-manifest-checkpoints.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +"""Qualify owner image-set metadata, original boot tasks and a genuine Data write refusal.""" +import hashlib +import json +from pathlib import Path +import runpy + +ROOT = Path(__file__).resolve().parents[1] +base = runpy.run_path(str(ROOT / 'scripts/check-original-launchd-main-checkpoints.py')) +require = base['require'] +expected = base['candidate']('native-owner-boot-manifest-candidate', + 'c8718350684b5041ccb639c84c3b55296b7d92e5a893186afa589e323bb6d15c', + '4b9ae2ac623da3cca0cc11733775508f841e8acfeefedfa3258b2c88ad64f93a') + +domain = b'VerifiedRoot.OwnerBootSet.v1\0' +parts = [] +for name, role, size, digest in ( + ('os', 1, 47481664, 'fd86b95b7033e352843f98d1204ade29459bf608a7e1ccb1a9b04ac93533292eb39de7c334ebe98b43d0db470fa16e77'), + ('systemos', 2, 25805344, '13148bb42c8ce4b56d835d89f91a4cea1ac0482111dc34a00cfddfee36037ce504923baae4bae2bae6da7d8229c53e54')): + data = (ROOT / ('build/verified-root-' + name) / 'manifest.bin').read_bytes() + require(len(data) == size and hashlib.sha384(data).hexdigest() == digest, 'Require exact original signed manifests, no padded/key bytes') + parts.append(bytes([role]) + hashlib.sha384(data).digest()) +digest = hashlib.sha384(domain + b''.join(parts)).hexdigest() +require(digest == '4787d7d9d6900c21e7fdd7b2192a932648aa024d158d1e0250c15a29d710ff7eca70dad2819ede499301dec31d10ba61', 'Canonical role/domain encoding') +positive, output, symlink, conflict = [base['report']('owner-hvf-' + name) for name in + ('boot-manifest', 'boot-manifest-output', 'tzinit-symlink', 'boot-manifest-conflict')] +for d in (positive, output, symlink, conflict): + require(d['kernel_sha256'] == expected and d['qemu_accelerator'] == 'hvf' and not d['bounded_timeout_interrupted'] and + d['trust_cache_bundle_sha256'] == hashlib.sha256(base['trust']).hexdigest(), 'Require exact bounded candidate and unchanged original trust') + r = {k: int(v, 16) for k, v in d['system_registers'].items()} + require(r['TCR_EL1'] >> 14 & 3 == 0 and r['TCR_EL1'] >> 30 & 3 == 2 and r['SCTLR_EL1'] & 1 and + r['TTBR0_EL1'] and r['TTBR1_EL1'] and d['kernel_page_globals'] == + dict(_PAGE_SHIFT_CONST=12, _page_shift=12, _page_shift_user32=12, _page_size=4096), 'Actual MMU/runtime must remain native 4K') + require(d['owner_root_binding_audit']['boot_root_accepted'] and d['owner_root_binding_audit']['_native_owner_root_apple_autherr'] == 80 and + d['original_amfi_table_owner_observed'] and d['original_crypto_initialized_observed'] and d['original_crypto_table_owner_observed'] and + d['actual_signature_failures'] == dict(_cs_validate_page_bad_hash=0, _cs_validate_page_no_hash=0) and + d['native_virtio_globals']['_native_virtio_read_failures'] == 0 and d['native_virtio_integrity']['_native_virtio_integrity_failures'] == 0, + 'Preserve original AMFI/crypto, actual owner root and real disk/signature failures') +for d in (positive, output, symlink): + require(d['owner_boot_manifest']['status'] == 0 and d['owner_boot_manifest']['sha384'] == digest and + not d['owner_boot_manifest']['firmware_personalization'] and not d['owner_boot_manifest'].get('conflicting_untrusted_dt_input', False), + 'Real kernel publication/readback must match exact authenticated host envelopes in the owner domain') + require(d['owner_cache_mount']['_native_owner_cache_status'] == 0 and + d['native_signature_hash_cache']['_cs_native4k_full_blocks'] == 201083 and + d['native_signature_hash_cache']['_cs_native4k_ready_blobs'] == 53 and + d['native_signature_hash_cache']['_cs_native4k_page_checks'] > 2000, 'Original complete cache must still pass genuine validation') +for d in (positive, output): + require(d['critical_process_exit'] == dict(process='tzinit[17] exited', signal=0, status=30, cs_status=''), 'Preserve the real later timezone failure') +writes = [bytes.fromhex(e['write_bytes']).decode(errors='replace') for e in output['syscall_events'] + if e['number'] in (4, 397) and 'write_bytes' in e and not e['write_bytes'].startswith('E')] +for text in ('(mount-phase-1)', '(data-protection)', '(mount-phase-2)', '(init-with-data-volume)', + '(fips)', 'all tests PASSED', 'result = true', '(keybag)', 'FATAL KEYBAG ERROR', '(usermanagerd)'): + require(any(text in x for x in writes), 'Require actual original boot output: ' + text) +require(any(digest.upper() in x and '/private/preboot/' in x for x in writes) and + not any('failed to get boot-manifest-hash' in x for x in writes) and + any(e['number'] == 1 and int(e['arguments'][0], 16) == 0 for e in output['syscall_events']), 'Original consumers use the owner hash and earlier boot children exit zero') +require(symlink['stop_symbol'] == '_symlink' and symlink['function_return_stack_matched'] and int(symlink['x0'], 16) == 30 and + symlink['entry_string_events'][-1]['matched'] and symlink['entry_string_events'][-1]['value'] == '/var/db/timezone/zoneinfo', 'Actual selected timezone symlink returns EROFS on the same stack') +for request, path in [('1:0:80', '/usr/share/zoneinfo.default'), ('1:8:80', '/var/db/timezone/zoneinfo')]: + require(bytes.fromhex(symlink['entry_indirect_memory'][request]['bytes']).split(b'\0')[0].decode() == path, 'Actual syscall target/link arguments') +require(conflict['owner_boot_manifest']['conflicting_untrusted_dt_input'] and conflict['owner_boot_manifest']['status'] == 3 and + conflict['owner_boot_manifest']['sha384'] == '00' * 48 and + conflict['owner_cache_mount']['_native_owner_cache_status'] == 80 and + conflict['native_signature_hash_cache']['_cs_native4k_page_checks'] == 0, 'An existing untrusted identity must cause real startup refusal before userspace') +print('PASS: authenticated owner image-set SHA-384 matches real kernel publication/readback; conflicting identity refuses startup') +print('PASS: original mount phases and userspace corecrypto FIPS POST complete; actual timezone symlink returns EROFS') +print('Writable Data/keybags, firmware enrollment, functional iPadOS UI and sheng hardware remain open') diff --git a/scripts/check-owner-cache-carrier-checkpoints.py b/scripts/check-owner-cache-carrier-checkpoints.py new file mode 100644 index 0000000..bc42404 --- /dev/null +++ b/scripts/check-owner-cache-carrier-checkpoints.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +"""Qualify independent owner cache carrier mapping and preserve original downstream refusals.""" +import hashlib +import json +from pathlib import Path +import struct +import sys + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'kits/NativePrelink')) +import native_prelink as np + + +def require(value, message): + if not value: + raise SystemExit(message) + + +def report(name): + return json.loads((ROOT / ('build/darwinboot-' + name) / 'gdb-stop.json').read_text()) + + +original = (ROOT / '.cache/firmware/kernelcache.macho/kernelcache.release.ipad7.decompressed').read_bytes() +require(hashlib.sha256(original).hexdigest() == 'f18b4df865405f2a287b84977d7aca745e056d656d9fc9df1662970ce2969de7', 'Require supplied target') +trust = (ROOT / 'build/original-static-system-trust/original-static-system.bundle').read_bytes() +require(hashlib.sha256(trust).hexdigest() == '7416ca4f8377146626bb284e2133c76bee6cb4e3ee0af5d648b186235a7323a7' and + struct.unpack_from('> 14 & 3 == 0 and regs['TCR_EL1'] >> 30 & 3 == 2 and regs['SCTLR_EL1'] & 1 and + regs['TTBR0_EL1'] and regs['TTBR1_EL1'] and d['commpage_user64_shifts'] == + dict(_commpage_rw_addr=12, _commpage_kernel_ro_addr=12), 'Actual native-4K mappings and commpage shift data') + require(d['actual_physical_memory']['_gPhysSize'] == 0x20000000 and d['actual_physical_memory']['_mem_actual'] == 0x20000000 and + d['original_amfi_table_owner_observed'] and d['original_crypto_initialized_observed'] and + d['original_crypto_table_owner_observed'] and d['owner_root_binding_audit']['boot_root_accepted'] and + d['owner_root_binding_audit']['_native_owner_root_apple_autherr'] == 80, 'Real owned memory and owner proof retain original AMFI/crypto') + require(d['native_virtio_globals']['_native_virtio_read_failures'] == 0 and + d['native_virtio_integrity']['_native_virtio_integrity_failures'] == 0 and + d['actual_signature_failures'] == dict(_cs_validate_page_bad_hash=0, _cs_validate_page_no_hash=0), 'No unauthenticated I/O or invalid code pages') +for d, status, checks, accepts, rejects in ((positive, 0, 254, 53, 0), (negative, 1, 212, 0, 1)): + require(d['stop_symbol'] == '_shared_region_map_and_slide_2_np' and d['function_return_stack_matched'] and + int(d['x0'], 16) == status and d['pc'] == d['function_return_target'], 'Actual same-stack shared-region result') + carrier = d['owner_cache_carrier'] + require(carrier['_native_owner_cache_carrier_accepts'] == accepts and carrier['_native_owner_cache_carrier_rejects'] == rejects and + carrier['_native_owner_cache_carrier_root_device'] == 0x01000006 and carrier['_native_owner_cache_carrier_file_device'] == 0x01000004, + 'Actual OS/SystemOS mounted devices must pass only the expected signed UUID') + require(d['native_signature_hash_cache'] == dict(_cs_native4k_hash_cache_version=1, _cs_native4k_full_blocks=201083, + _cs_native4k_page_checks=checks, _cs_native4k_ready_blobs=53), 'Retain complete original preflight and native-page checks') +policies = {positive['native_virtio_integrity'][name] for name in ( + '_native_virtio_verified_manifest_pointer', '_native_virtio_verified_key_pointer')} +observed = set() +for capture in positive['indirect_memory'].values(): + pointer = int(capture['pointer'], 16) + if pointer not in policies: continue + t = capture['translation'] + require(t['leaf_level'] == 3 and t['leaf_bytes'] == 4096 and t['ap'] == 2 and t['pxn'] and t['uxn'], 'Actual private policy/key copies retain native-4K RO/NX leaves') + observed.add(pointer) +require(observed == policies and len(policies) == 2, 'Observe both real private policy and key mappings') +require(not positive['cstring_events'] and any("not on process' root volume nor preboot volume" in e['value'] for e in negative['cstring_events']), + 'Wrong expected UUID must preserve the actual original carrier refusal') +require(continuation['stop_symbol'] == '_panic_with_options' and continuation['fatal_user_exit'] == + dict(cs_status='', prefix='initproc failed to start', namespace=7, code=1, + description='_container_init(): sandbox_get_container_expected() failed: %d') and + continuation['native_signature_hash_cache']['_cs_native4k_page_checks'] == 1392, + 'Preserve the actual subsequent original system-library initialization refusal') +require(missing_policy['stop_symbol'] == '___mac_syscall' and missing_policy['function_return_stack_matched'] and + int(missing_policy['x0'], 16) == 103 and missing_policy['entry_string_events'][-1]['matched'] and + missing_policy['entry_string_events'][-1]['value'] == 'Sandbox' and + bytes.fromhex(missing_policy['entry_indirect_memory']['1:0:40']['bytes']).startswith(b'Sandbox\0'), + 'Actual selected Sandbox request genuinely returns ENOPOLICY=103; no handler is fabricated') +print('PASS: actual owner-authorized 53-file shared-region mapping returns 0 at native 4K; wrong UUID still returns EPERM') +print('PASS: actual private policy/key RO/NX 4K PTEs; original system library runs until genuine missing Sandbox policy') +print('Launchd main, full iPadOS UI, firmware enrollment and bare-metal PMGR remain open') diff --git a/scripts/check-owner-cache-preflight-checkpoints.py b/scripts/check-owner-cache-preflight-checkpoints.py new file mode 100644 index 0000000..c8998bd --- /dev/null +++ b/scripts/check-owner-cache-preflight-checkpoints.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +"""Qualify real 512-MiB ownership, complete original cache preflight and genuine mount-policy refusal.""" +import hashlib +import json +from pathlib import Path +import struct +import sys + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'kits/NativePrelink')) +import native_prelink as np + + +def require(value, message): + if not value: + raise SystemExit(message) + + +def report(name): + return json.loads((ROOT / ('build/darwinboot-' + name) / 'gdb-stop.json').read_text()) + + +runs = [report(name) for name in ('owner-hvf-512-shared-region', 'owner-hvf-512-cache-permission')] +candidate = ROOT / 'build/native-prelink/native-owner-dt-panic-candidate' +plan = json.loads(candidate.with_suffix('.json').read_text()) +kernel = candidate.with_suffix('.macho').read_bytes() +require(hashlib.sha256(kernel).hexdigest() == plan['candidate_sha256'] == + 'f3611f1aa0a472122196134101237384de931d4d78d6f02d3c7013a63405898d' and plan['native_sha256'] == + '926d1e6bc223b12d482e5b48c0e3fd43173108168f54c2e1cb5cf3042f0f8eae' and len(plan['components']) == 15, + 'Require exact original-component mount candidate') +original = (ROOT / '.cache/firmware/kernelcache.macho/kernelcache.release.ipad7.decompressed').read_bytes() +require(hashlib.sha256(original).hexdigest() == 'f18b4df865405f2a287b84977d7aca745e056d656d9fc9df1662970ce2969de7', 'Require supplied target') +old, meta = np.macho(original), np.macho(kernel) +for e in plan['extents']: + if e['segment'] == '__TEXT_EXEC': + at = np.backed_offset(meta['segments'], e['va'], e['size'], len(kernel)) + prev = np.backed_offset(old['segments'], e['va'] - plan['slide'], e['size'], len(original)) + require(kernel[at:at + e['size']] == original[prev:prev + e['size']], 'Every original executable extent must remain unchanged') +trust = (ROOT / 'build/original-static-system-trust/original-static-system.bundle').read_bytes() +require(hashlib.sha256(trust).hexdigest() == '7416ca4f8377146626bb284e2133c76bee6cb4e3ee0af5d648b186235a7323a7' and + struct.unpack_from('> 14 & 3 == 0 and r['TCR_EL1'] >> 30 & 3 == 2 and r['SCTLR_EL1'] & 1 and + r['TTBR0_EL1'] and r['TTBR1_EL1'] and r['CNTFRQ_EL0'] == 24000000 and d['kernel_page_globals'] == + dict(_PAGE_SHIFT_CONST=12, _page_shift=12, _page_shift_user32=12, _page_size=4096), 'Actual native-4K hardware and ABI globals') + require(d['original_amfi_table_owner_observed'] and d['original_crypto_initialized_observed'] and + d['original_crypto_table_owner_observed'] and d['owner_root_binding_audit']['boot_root_accepted'] and + d['owner_root_binding_audit']['_native_owner_root_apple_autherr'] == 80, 'Real AMFI/crypto and owner root proof') + require(d['actual_physical_memory'] == dict(_gPhysBase=0x4807c000, _gPhysSize=0x20000000, + _mem_size=0x20000000, _mem_actual=0x20000000), 'All actual native memory sizes must match the real UEFI allocation') + require(d['actual_signature_failures'] == dict(_cs_validate_page_bad_hash=0, _cs_validate_page_no_hash=0) and + d['native_signature_hash_cache'] == dict(_cs_native4k_hash_cache_version=1, _cs_native4k_full_blocks=201083, + _cs_native4k_page_checks=212, _cs_native4k_ready_blobs=53), 'Actual complete original signing-block preflight and native-page checks') + require(d['native_virtio_globals']['_native_virtio_read_failures'] == 0 and + d['native_virtio_integrity']['_native_virtio_integrity_failures'] == 0 and + d['native_virtio_integrity']['_native_virtio_verified_devices'] == 2, 'Both media remain independently authenticated') + require(d['actual_panic_region'] == dict(_gPanicBase=0, _gPanicSize=0, _panic_info=0) and + d['dyld_startup_flags'] == dict(_dyld_flags=8, _native_owner_cache_dyld_flags=8, actual_commpage_flags=8), 'Retain actual authenticated startup mode and no phantom panic mapping') + state = d['owner_cache_mount'] + require(state['_native_owner_cache_stage'] == 6 and state['_native_owner_cache_status'] == 0 and + state['_native_owner_cache_mount_status'] == 0 and state['_native_owner_cache_readonly'] == 1 and + state['_native_owner_cache_authorized'] == state['_native_owner_cache_root_recheck'] == 1 and + state['_native_owner_cache_selected_device'] == state['_native_owner_cache_mounted_device'], 'Actual readonly SystemOS mount and separate root recheck') + require(d['stop_symbol'] == '_shared_region_map_and_slide_2_np' and d['finished_function_entry'] == '0xfffffff00bd5cb28' and + d['function_return_stack_matched'] and d['function_return_target'] == d['pc'] and int(d['x0'], 16) == 1 and + not d['ipados_userspace_boot_demonstrated'], 'Actual shared-region syscall returns EPERM on the same stack; no mapping claim') +permission = runs[1] +require(permission['cstring_start_event']['pc'] == '0xfffffff00bd5e26c' and not permission['cstring_log_exhausted'] and + any(e['value'].endswith("not on process' root volume nor preboot volume\n") and e['caller'] == '0xfffffff00bd5f4a0' + for e in permission['cstring_events']), 'Observe actual native carrier-volume policy refusal after setup begins') +for name in ('owner-hvf-512-shared-region', 'owner-hvf-512-cache-permission'): + folder = ROOT / ('build/darwinboot-' + name) + require(hashlib.sha256((folder / 'esp/EFI/CrabApple/Payloads/DarwinBoot.efi').read_bytes()).hexdigest() == + 'b5bf6baa3bad99ad2a10791359480af77053c828c0635ddbc65e6d30c1a3c8e6', 'Pin exact tested DarwinBoot loader') + require('owned=0x20000000' in (folder / 'gdb-uart.log').read_text(), 'Actual UEFI handoff must own 512 MiB') +print('PASS: real 512-MiB ownership, 53 original signature objects, 201083 whole signing blocks and 212 native-4K pages') +print('PASS: shared-region syscall genuinely returns EPERM; original cache carrier-volume policy is the observed cause') +print('Full shared-cache mapping, launchd main and bare-metal PMGR remain open') diff --git a/scripts/check-owner-cryptex-fallback-checkpoints.py b/scripts/check-owner-cryptex-fallback-checkpoints.py new file mode 100644 index 0000000..e697f91 --- /dev/null +++ b/scripts/check-owner-cryptex-fallback-checkpoints.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +"""Record the genuine failed standard dyld fallback-path experiment.""" +from pathlib import Path +import runpy + +ROOT = Path(__file__).resolve().parents[1] +base = runpy.run_path(str(ROOT / 'scripts/check-native-ram-data-checkpoints.py')) +require = base['require'] +expected = base['base']['candidate']('native-owner-cryptex-files-candidate', + 'e57f643394ee87c83ff292db2469a5897656aafd020b50be62b823025eef0457', + 'f261e44dac494e1060bfcbd233c97988159221f92462c242b77acab7d513f7ec') +d = base['base']['report']('owner-hvf-cryptex-files-first-abort') +require(d['kernel_sha256'] == expected and d['native_4k_mmu_observed'] and not d['bounded_timeout_interrupted'] and + d['scratch_data']['state']['_native_owner_data_status'] == 0 and d['scratch_data']['state']['_native_owner_data_readonly'] == 0 and + d['owner_root_binding_audit']['boot_root_accepted'] and d['owner_boot_manifest']['status'] == 0 and + d['original_amfi_table_owner_observed'] and d['actual_signature_failures'] == base['abort']['actual_signature_failures'], + 'Same genuine owner root, writable Data, original AMFI and native-4K conditions') +reason = bytes.fromhex(d['entry_indirect_memory']['1:20:100']['bytes']).split(b'\0')[0].decode() +require(d['stop_symbol'] == '_abort_with_payload' and d['function_return_stack_matched'] and reason == base['reason'], + 'Standard fallback env must not be reported as resolving the same actual original missing-library abort') +require(d['native_ram_block']['state']['_native_ram_block_writes'] > 0 and + d['native_ram_block']['state']['_native_ram_block_failures'] == 0, 'Retain real mutable Data I/O') +print('PASS: explicit standard fallback-path experiment preserves the same original missing-library abort; no path fix is claimed') diff --git a/scripts/check-owner-dyld-startup-checkpoints.py b/scripts/check-owner-dyld-startup-checkpoints.py new file mode 100644 index 0000000..1c29e6a --- /dev/null +++ b/scripts/check-owner-dyld-startup-checkpoints.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +"""Qualify owner dyld startup, original cache preflight and genuine PMGR refusal.""" +import hashlib +import json +from pathlib import Path +import struct +import sys + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'kits/NativePrelink')) +import native_prelink as np + + +def require(value, message): + if not value: + raise SystemExit(message) + + +def report(name): + return json.loads((ROOT / ('build/darwinboot-' + name) / 'gdb-stop.json').read_text()) + + +positive, missing = (report(name) for name in ( + 'owner-cache-shared-region', 'owner-startup-missing-refusal')) +candidate = ROOT / 'build/native-prelink/native-owner-dyld-startup-candidate' +plan = json.loads(candidate.with_suffix('.json').read_text()) +kernel = candidate.with_suffix('.macho').read_bytes() +require(hashlib.sha256(kernel).hexdigest() == plan['candidate_sha256'] == + 'cef9ebddda8afc0e407cf933e0bfc97fabb6e754a0a2e3fcf740ef02188a07b3' and plan['native_sha256'] == + '6c4a55cadc58e7c680cd8d0c9e67b5f166354ff4149a12078d3f79719870b064' and len(plan['components']) == 15, + 'Require exact original-component mount candidate') +original = (ROOT / '.cache/firmware/kernelcache.macho/kernelcache.release.ipad7.decompressed').read_bytes() +require(hashlib.sha256(original).hexdigest() == 'f18b4df865405f2a287b84977d7aca745e056d656d9fc9df1662970ce2969de7', 'Require supplied target') +old, meta = np.macho(original), np.macho(kernel) +for e in plan['extents']: + if e['segment'] == '__TEXT_EXEC': + at = np.backed_offset(meta['segments'], e['va'], e['size'], len(kernel)) + prev = np.backed_offset(old['segments'], e['va'] - plan['slide'], e['size'], len(original)) + require(kernel[at:at + e['size']] == original[prev:prev + e['size']], 'Every original executable extent must remain unchanged') +trust = (ROOT / 'build/original-static-system-trust/original-static-system.bundle').read_bytes() +require(hashlib.sha256(trust).hexdigest() == '7416ca4f8377146626bb284e2133c76bee6cb4e3ee0af5d648b186235a7323a7' and + struct.unpack_from('> 14 & 3 == 0 and regs['TCR_EL1'] >> 30 & 3 == 2 and regs['SCTLR_EL1'] & 1 and + regs['TTBR0_EL1'] and regs['TTBR1_EL1'] and d['original_amfi_table_owner_observed'] and + d['original_crypto_initialized_observed'] and d['original_crypto_table_owner_observed'], 'Require actual native 4K and original AMFI/crypto') + root = d['owner_root_binding_audit'] + require(root['boot_root_accepted'] and root['_native_owner_root_apple_autherr'] == 80 and + root['actual_uuid'] == '04769D26-2CA0-4825-B443-E2E5F88300F2', 'Retain separate actual owner root proof and Apple error') + require(d['native_virtio_globals']['_native_virtio_read_failures'] == 0 and + d['native_virtio_integrity']['_native_virtio_integrity_failures'] == 0, 'No unauthenticated I/O') + require(d['stop_symbol'] == '_panic' and not d['ipados_userspace_boot_demonstrated'], 'Keep genuine remaining refusal') +state = positive['owner_cache_mount'] +require(state['_native_owner_cache_stage'] == 6 and state['_native_owner_cache_status'] == 0 and + state['_native_owner_cache_mount_status'] == 0 and state['_native_owner_cache_readonly'] == 1 and + state['_native_owner_cache_authorized'] == 1 and state['_native_owner_cache_root_recheck'] == 1 and + state['_native_owner_cache_selected_device'] == state['_native_owner_cache_mounted_device'] == 0x01000004 and + state['actual_device_path'] == '/dev/disk2s1', 'Require actual readonly SystemOS mount and both owner proofs') +require(positive['dyld_startup_flags'] == dict(_dyld_flags=8, _native_owner_cache_dyld_flags=8, actual_commpage_flags=8), + 'Observe bit 3 in actual kernel and commpage state') +require(positive['native_signature_hash_cache'] == dict(_cs_native4k_hash_cache_version=1, + _cs_native4k_full_blocks=53998, _cs_native4k_page_checks=68, _cs_native4k_ready_blobs=17), 'Pin original whole-file authentication and genuine native-4K page validation') +require(positive['native_virtio_globals']['_native_virtio_read_requests'] == 16305 and + positive['native_virtio_integrity']['_native_virtio_verified_blocks'] == 220805 and + positive['native_virtio_integrity']['_native_virtio_verified_devices'] == 2, 'Pin actual verified DMA preflight') +require('AMFI: No PMGR?' in positive['argument_memory']['x0'] and positive['panic_caller_component'] == + dict(module='com.apple.driver.AppleMobileFileIntegrity', segment='__TEXT_EXEC', offset='0x1f420'), 'Preserve original AMFI hardware dependency') +# Actual native local symbols identify an asynchronous worker; do not label this as cache file 17 rejection. +import subprocess +symbols = {} +for line in subprocess.check_output(['xcrun', 'nm', '-n', candidate.with_suffix('.macho')], text=True).splitlines(): + fields = line.split() + if len(fields) == 3: + try: symbols[fields[2]] = int(fields[0], 16) + except ValueError: pass +returns = {int(frame['lr'], 16) for frame in positive['frame_chain']} +require(all(symbols[name] + offset in returns for name, offset in ( + ('_thread_call_invoke', 0x78), ('_thread_call_thread', 0x220), ('_Call_continuation', 0x44))), 'Require actual thread-call worker stack') +require(not any(e.get('symbol') == '_shared_region_map_and_slide_2_np' for e in positive['trace_events']), 'Do not claim full shared-region mapping') +state = missing['owner_cache_mount'] +require(missing['dyld_startup_flags'] == dict(_dyld_flags=0, _native_owner_cache_dyld_flags=0, actual_commpage_flags=0) and + missing['argument_memory']['x0'].startswith('offline owner SystemOS mount failed: %d') and + struct.unpack_from('> 5) & 0x3fff +if imm & 0x2000: imm -= 0x4000 +require(insn(0x32ce0) == 0x39416288 and word & 0x7f000000 == 0x37000000 and + ((word >> 19) & 31) == 3 and word & 31 == 8 and 0x32ce4 + imm * 4 == 0x32e28, 'Actual target tests skipIgnition bit 3') +word = insn(0x32d18) +imm = word & 0x3ffffff +if imm & 0x2000000: imm -= 0x4000000 +require(word >> 26 == 0x25 and 0x32d18 + imm * 4 == 0x34224, 'Original bit 3 skips actual ignite call') +print('PASS: owner-gated actual dyld/commpage bit 3; missing SystemOS refuses with all flags zero') +print('PASS: original cache preflight checks 53998 signed blocks, publishes 17 blobs and validates 68 native-4K pages') +print('Original asynchronous AMFI PMGR panic retained; full cache mapping, launchd main and hardware remain open') diff --git a/scripts/check-owner-policy-bundle-checkpoints.py b/scripts/check-owner-policy-bundle-checkpoints.py new file mode 100644 index 0000000..1fbb672 --- /dev/null +++ b/scripts/check-owner-policy-bundle-checkpoints.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +"""Qualify independent signed policies per device and real ambiguity refusal.""" +import hashlib +import json +from pathlib import Path +import struct +import sys + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'kits/NativePrelink')) +import native_prelink as np + + +def require(value, message): + if not value: + raise SystemExit(message) + + +original = (ROOT / '.cache/firmware/kernelcache.macho/kernelcache.release.ipad7.decompressed').read_bytes() +require(hashlib.sha256(original).hexdigest() == 'f18b4df865405f2a287b84977d7aca745e056d656d9fc9df1662970ce2969de7', 'Require supplied target') +old = np.macho(original) +os_policy = (ROOT / 'build/verified-root-os/manifest.bin').read_bytes() +system_policy = (ROOT / 'build/verified-root-systemos/manifest.bin').read_bytes() +key = (ROOT / 'build/verified-root-os/trusted-key.bin').read_bytes() +cases = ( + ('darwinboot-owner-two-verified-devices', 'native-owner-policy-bundle-candidate', '1623845aad68c0c2c31d29c1d7047be900be26071c364d5826d213ed536da170', 2), + ('darwinboot-owner-policy-ambiguity-refusal', 'native-owner-policy-ambiguous-candidate', 'ce61f3a8723834292485e95c5f8c0ca96697d282486f12b0c7def50f51cba700', 3), +) +reports = [] +for directory, name, identity, count in cases: + d = json.loads((ROOT / 'build' / directory / 'gdb-stop.json').read_text()) + reports.append(d) + plan = json.loads((ROOT / 'build/native-prelink' / (name + '.json')).read_text()) + data = (ROOT / 'build/native-prelink' / (name + '.macho')).read_bytes() + meta = np.macho(data) + exports = np.exports(data, meta) + symbols = {name: address for address, names in exports.items() for name in names} + require(d['kernel_sha256'] == plan['candidate_sha256'] == hashlib.sha256(data).hexdigest() == identity and + not d['bounded_timeout_interrupted'] and len(plan['components']) == 15, 'Require exact bounded original-component candidates') + + def read(address, length): + at = np.backed_offset(meta['segments'], address, length, len(data)) + return data[at:at + length] + + require(struct.unpack('> 14 & 3 == 0 and regs['TCR_EL1'] >> 30 & 3 == 2 and regs['SCTLR_EL1'] & 1 and + regs['TTBR0_EL1'] and regs['TTBR1_EL1'] and d['original_crypto_initialized_observed'] and + d['original_crypto_table_owner_observed'] and d['function_return_stack_matched'], 'Require actual native-4K return and real crypto provider') + +d, refusal = reports +state, io, integrity = d['owner_root_binding_audit'], d['native_virtio_globals'], d['native_virtio_integrity'] +require(d['stop_symbol'] == '_native_owner_root_accepted_checkpoint' and state['boot_root_accepted'] and + state['_native_owner_root_apple_autherr'] == 80 and state['_native_owner_root_mount_device'] == + state['_native_owner_root_selected_device'] == state['_native_verified_block_root_device'] == 0x01000006 and + state['actual_uuid'] == '04769D26-2CA0-4825-B443-E2E5F88300F2' and state['_native_verified_block_root_media_matches'] == 1 and + state['_native_verified_block_root_uuid_match'] == 1 and d['original_amfi_table_owner_observed'], 'Root binding must follow actual media despite changed device enumeration') +require(d['virtio_block_input']['read_only'] and d['virtio_block_input']['bytes'] == 6077580800 and + d['virtio_extra_block_input']['read_only'] and d['virtio_extra_block_input']['bytes'] == 3303014400 and + d['virtio_extra_block_input']['first_sector_sha256'] == + '42b1178767bb27611cd1836cdd5779c48bfca8a3e719ec06dc4c8ef78d5e6387', 'Require both actual unchanged read-only backends') +require(integrity['_native_virtio_verified_devices'] == 2 and integrity['_native_virtio_ambiguous_policies'] == 0 and + integrity['_native_virtio_verified_blocks'] == 391 and integrity['_native_virtio_integrity_failures'] == 0 and + io['_native_virtio_read_requests'] == 229 and io['_native_virtio_read_failures'] == 0, 'Each device must independently authenticate before readiness, with actual protected reads') +copies = list(d['indirect_memory'].values()) +require(len(copies) == 2 and any(bytes.fromhex(v['bytes']) == system_policy[:32] for v in copies) and + any(bytes.fromhex(v['bytes']) == key[:32] for v in copies), 'Observe actual second-device policy and independent key copies') +for value in copies: + t = value['translation'] + require(t['leaf_level'] == 3 and t['leaf_bytes'] == 4096 and t['ap'] == 2 and t['pxn'] and t['uxn'], 'Actual per-device private policy/key pages must be RO/NX native-4K leaves') +integrity, io = refusal['native_virtio_integrity'], refusal['native_virtio_globals'] +require(refusal['stop_symbol'] == '__ZN17NativeVirtioBlock21configureVerificationEv' and int(refusal['x0'], 16) == 0 and + integrity['_native_virtio_ambiguous_policies'] == 1 and integrity['_native_virtio_verified_devices'] == 0 and + integrity['_native_virtio_manifest_ready'] == 0 and integrity['_native_virtio_verified_blocks'] == 0 and + integrity['_native_virtio_verified_manifest_pointer'] == 0 and integrity['_native_virtio_verified_key_pointer'] == 0 and + io['_native_virtio_ready'] == 0 and io['_native_virtio_read_requests'] == 0 and not refusal['indirect_memory'], + 'A genuine duplicated policy must return false before allocation, DMA or media publication') +abi = json.loads((ROOT / 'build/native-virtio-block-verified-bundle/bindings.json').read_text()) +require(abi['virtual_methods_checked'] == 190 and abi['concrete_overrides'] == 25 and abi['unresolved_original_imports'] == 0, 'Concrete storage ABI must remain intact') +print('PASS: both original media independently authenticate and read; actual OS root remains bound to its signed UUID') +print('PASS: identical duplicated signed policy refuses before private allocation, DMA or readiness') +print('This qualifies two protected devices, not SystemCryptex mounting, full shared-cache loading or hardware boot') diff --git a/scripts/check-owner-root-checkpoints.py b/scripts/check-owner-root-checkpoints.py new file mode 100644 index 0000000..b262e77 --- /dev/null +++ b/scripts/check-owner-root-checkpoints.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +"""Qualify actual owner root enforcement and independent identity refusal.""" +import hashlib +import json +from pathlib import Path +import struct +import sys +import uuid + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'kits/NativePrelink')) +import native_prelink as np + + +def require(value, message): + if not value: + raise SystemExit(message) + + +original = (ROOT / '.cache/firmware/kernelcache.macho/kernelcache.release.ipad7.decompressed').read_bytes() +require(hashlib.sha256(original).hexdigest() == 'f18b4df865405f2a287b84977d7aca745e056d656d9fc9df1662970ce2969de7', + 'Require the supplied ordinary-ARM64 target') +old = np.macho(original) +policy = (ROOT / 'build/verified-root-os/manifest.bin').read_bytes() +wrong_policy = (ROOT / 'build/verified-root-os-xnu-wrong-uuid/wrong-uuid-manifest.bin').read_bytes() +anchor = (ROOT / 'build/verified-root-os/trusted-key.bin').read_bytes() +require(policy[544:] == wrong_policy[544:] and policy[:40] == wrong_policy[:40] and policy[56:160] == wrong_policy[56:160] and + policy[40:56] != wrong_policy[40:56], 'Negative policy changes only signed UUID/signature, preserving every original hash') +require(str(uuid.UUID(bytes=policy[40:56])).upper() == '04769D26-2CA0-4825-B443-E2E5F88300F2' and + struct.unpack_from('> 14 & 3 == 0 and regs['TCR_EL1'] >> 30 & 3 == 2 and regs['SCTLR_EL1'] & 1 and + regs['TTBR0_EL1'] and regs['TTBR1_EL1'], 'Require actual native hardware 4K, not source constants') + require(d['original_crypto_initialized_observed'] and d['original_crypto_table_owner_observed'] and + d['original_amfi_table_owner_observed'] and d['virtio_block_input']['read_only'], 'Original AMFI/crypto must remain active') + state, io, integrity = d['owner_root_binding_audit'], d['native_virtio_globals'], d['native_virtio_integrity'] + require(state['_native_owner_root_apple_autherr'] == 80 and state['_native_owner_root_readonly'] == 1 and + state['_native_owner_root_trust_enabled'] == enabled and state['_native_owner_root_decision'] == decision and + state['_native_owner_root_accepted'] == accepted and state['boot_root_accepted'] == bool(accepted), + 'Actual enforcement must retain Apple EAUTH and independently require real owner proof') + require(state['_native_owner_root_mount_device'] == state['_native_owner_root_selected_device'] == + state['_native_verified_block_root_device'] != 0 and state['_native_verified_block_root_media_matches'] == 1 and + state['_native_verified_block_root_provider_depth'] == 9 and state['_native_verified_block_root_uuid_match'] == decision and + state['_native_verified_block_wrong_uuid_refused'] == 1 and + state['actual_uuid'] == '04769D26-2CA0-4825-B443-E2E5F88300F2', 'Require actual mounted dev_t, unique IOMedia and typed verified provider') + require(io['_native_virtio_ready'] == 1 and io['_native_virtio_sectors'] == 11870275 and + io['_native_virtio_read_requests'] >= 167 and io['_native_virtio_read_failures'] == 0 and + integrity['_native_virtio_manifest_ready'] == 1 and integrity['_native_virtio_verified_blocks'] >= 329 and + integrity['_native_virtio_integrity_failures'] == 0, 'Real signed policy and every consumed DMA snapshot must qualify') + if name in ('accepted', 'wrong-uuid-refusal'): + copies = list(d['indirect_memory'].values()) + require(len(copies) == 2, 'Require actual private policy/key copies') + for copy in copies: + t = copy['translation'] + require(t['leaf_level'] == 3 and t['leaf_bytes'] == 4096 and t['ap'] == 2 and t['pxn'] and t['uxn'], + 'Actual policy/key copies must retain read-only NX native-4K PTEs') + if not accepted: + require(d['stop_symbol'] == '_panic' and d['argument_memory']['x0'].startswith('rootvp not authenticated after mounting'), + 'Default-off and genuine signed wrong-identity policies must reach actual enforcement refusal') + +d = reports['accepted'] +require(d['stop_symbol'] == '_native_owner_root_accepted_checkpoint' and d['function_return_stack_matched'] and + [int(x, 16) for x in d['function_entry_arguments'][:2]] == [80, 1], + 'Observe actual same-stack control flow after enforcement, never a pre-enforcement intent flag') +d = reports['original-continuation'] +require(d['stop_symbol'] == '_panic' and d['argument_memory']['x0'].startswith('Process 1 exec of %s failed, errno %d') and + struct.unpack_from('> 14 & 3 == 0 and regs['TCR_EL1'] >> 30 & 3 == 2 and regs['SCTLR_EL1'] & 1 and + regs['TTBR0_EL1'] and regs['TTBR1_EL1'], 'Require active native 4K hardware translation') +require(positive['original_amfi_table_owner_observed'] and positive['original_crypto_table_owner_observed'] and + positive['original_crypto_initialized_observed'] and positive['owner_root_binding_audit']['boot_root_accepted'] and + positive['owner_root_binding_audit']['_native_owner_root_apple_autherr'] == 80, 'Root must still use genuine owner enforcement and original AMFI/crypto') +io, integrity = positive['native_virtio_globals'], positive['native_virtio_integrity'] +require(io['_native_virtio_read_requests'] == 226 and io['_native_virtio_read_failures'] == 0 and + integrity['_native_virtio_verified_blocks'] == 526 and integrity['_native_virtio_integrity_failures'] == 0, + 'Original compressed-file execution must consume only actual authenticated I/O') +require(any(e['symbol'] == component + ':kmod_start' for e in positive['trace_events']) and + any(e['symbol'] == '_exec_mach_imgact' for e in positive['trace_events']), 'Observe real original module start and Mach-O activation') +fatal = positive['fatal_user_exit'] +require(positive['stop_symbol'] == '_panic_with_options' and fatal['namespace'] == 6 and fatal['code'] == 1 and + '/usr/lib/libSystem.B.dylib' in fatal['description'] and '/sbin/launchd' in fatal['description'] and + 'no dyld cache' in fatal['description'] and any(e['number'] == 5 and e.get('path') == + '/System/Library/Caches/com.apple.dyld/' for e in positive['syscall_events']), 'Preserve genuine later dyld missing-cache exit and actual default directory request') +print('PASS: real permissions succeed; absent decompressor returns ENOTSUP; original component supplies actual type-4 callback') +print('PASS: 15 original executable extents remain unchanged; authenticated native4K root reaches genuine dyld missing-cache exit') +print('No launchd main, complete shared-cache loading, production secure boot or hardware boot is claimed') diff --git a/scripts/check-owner-root-crypto-checkpoints.py b/scripts/check-owner-root-crypto-checkpoints.py new file mode 100644 index 0000000..91f5233 --- /dev/null +++ b/scripts/check-owner-root-crypto-checkpoints.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +"""Check real host complete-data verification and native original crypto.""" +import hashlib +import json +from pathlib import Path +import struct +import sys + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'kits/NativePrelink')) +import native_prelink as np + + +def require(value, message): + if not value: + raise SystemExit(message) + + +host = json.loads((ROOT / 'build/verified-root-systemos/verification.json').read_text()) +inputs = json.loads((ROOT / 'build/verified-root-xnu/inputs.json').read_text()) +d = json.loads((ROOT / 'build/darwinboot-owner-crypto-native/gdb-stop.json').read_text()) +plan = json.loads((ROOT / 'build/native-prelink/native-owner-crypto-candidate.json').read_text()) +require(host['real_c_signature_and_block_verification'] and host['blocks_verified'] == 806400 and + host['image_sha256'] == 'b4908cde41d23e6bc05e6af51a2c9eeb00cf0f863a6a18b1600c96a3fd70414f', + 'Host evidence must cover every original data block') +require(inputs['original_image_sha256'] == host['image_sha256'] and + inputs['trusted_key_sha256'] == host['trusted_key_sha256'], 'Native probe must use the actual host-qualified input/anchor') +require(d['kernel_sha256'] == plan['candidate_sha256'] == + '5da12edb422be5c8043fd524fcabdda0b3a878179cc65ca16efcde04aab5a54e' and + plan['native_sha256'] == '6cf58f6c0a60dbfa8b20419e8549513a8bb493d357641aed2f9af160a7cd62bf' and + len(plan['components']) == 14 and not d['bounded_timeout_interrupted'] and d['function_return_stack_matched'], + 'Require the exact same-stack bounded original-component native probe') +regs = {k: int(v, 16) for k, v in d['system_registers'].items()} +require(regs['TCR_EL1'] >> 14 & 3 == 0 and regs['TCR_EL1'] >> 30 & 3 == 2 and + regs['SCTLR_EL1'] & 1 and regs['TTBR0_EL1'] and regs['TTBR1_EL1'], 'Active hardware translation must be native 4K') +p = d['owner_root_crypto_probe'] +require(p['_native_verified_root_status'] == 1 and p['results'] == [0, 4, 3, 3, 4, 5, 2, 1] and + p['_native_verified_root_signature_calls'] == 4 and p['_native_verified_root_block_count'] == 806400 and + not p['boot_root_authenticated'], 'Real RSA/header/table/data/key/generation outcomes differ') +require(d['original_crypto_initialized_observed'] and d['original_crypto_table_owner_observed'] and + any(e['module'] == 'com.apple.driver.AppleMobileFileIntegrity' and e['segment'] == '__DATA_CONST' and + e['va'] <= p['_amfi'] < e['va'] + e['size'] for e in plan['extents']), + 'Original registered crypto and AMFI must remain active') +for name in ('0:0:40', '1:0:20'): + t = d['entry_indirect_memory'][name]['translation'] + require(t['leaf_level'] == 3 and t['leaf_bytes'] == 4096 and t['ap'] == 2 and t['pxn'] and t['uxn'], + 'Actual captured metadata/key PTEs must be read-only, NX and native 4K') +require(d['entry_indirect_memory']['0:0:40']['bytes'].startswith('56524f4f54303100') and + d['entry_indirect_memory']['1:0:20']['bytes'].startswith('56524f4f544b4559'), 'Observe the actual original input/anchor prefixes') +kernel = (ROOT / 'build/native-prelink/native-owner-crypto-candidate.macho').read_bytes() +original = (ROOT / '.cache/firmware/kernelcache.macho/kernelcache.release.ipad7.decompressed').read_bytes() +meta, old = np.macho(kernel), np.macho(original) +require(hashlib.sha256(kernel).hexdigest() == plan['candidate_sha256'] and hashlib.sha256(original).hexdigest() == plan['original_sha256'], + 'Native/original image identities differ') +for e in plan['extents']: + if e['segment'] != '__TEXT_EXEC': + continue + at = np.backed_offset(meta['segments'], e['va'], e['size'], len(kernel)) + previous = np.backed_offset(old['segments'], e['va'] - plan['slide'], e['size'], len(original)) + require(kernel[at:at + e['size']] == original[previous:previous + e['size']], 'Original executable bytes must not change') +print('PASS: complete original host data verification and actual original RSA/SHA crypto at native 4K') +print('PASS: native signature/table/data/key/generation refusals and read-only NX metadata/key PTEs') +print('Only four original data samples are checked in QEMU; runtime block-path/root acceptance and hardware remain open') diff --git a/scripts/check-owner-systemos-mount-checkpoints.py b/scripts/check-owner-systemos-mount-checkpoints.py new file mode 100644 index 0000000..6c915f2 --- /dev/null +++ b/scripts/check-owner-systemos-mount-checkpoints.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +"""Qualify actual owner-authenticated SystemOS mounting and genuine refusals.""" +import hashlib +import json +from pathlib import Path +import struct +import sys + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'kits/NativePrelink')) +import native_prelink as np + + +def require(value, message): + if not value: + raise SystemExit(message) + + +def report(name): + return json.loads((ROOT / ('build/darwinboot-' + name) / 'gdb-stop.json').read_text()) + + +mounted, missing, continuation, file_call = (report(name) for name in ( + 'owner-systemos-mounted', 'owner-systemos-missing-enforced', 'owner-original-cache-load', 'owner-cache-openat')) +candidate = ROOT / 'build/native-prelink/native-owner-systemos-mount-candidate' +plan = json.loads(candidate.with_suffix('.json').read_text()) +kernel = candidate.with_suffix('.macho').read_bytes() +require(hashlib.sha256(kernel).hexdigest() == plan['candidate_sha256'] == + '6631d27898e27565fb9ce8eb03cc40f2d74fb1aeb36c69da1e1a0f893bf0fe51' and plan['native_sha256'] == + 'dfdc5b20b8562885de687afe8bf4fda8e9da9b4b20d7111a12ffe985d9a84d71' and len(plan['components']) == 15, + 'Require exact original-component mount candidate') +original = (ROOT / '.cache/firmware/kernelcache.macho/kernelcache.release.ipad7.decompressed').read_bytes() +require(hashlib.sha256(original).hexdigest() == 'f18b4df865405f2a287b84977d7aca745e056d656d9fc9df1662970ce2969de7', 'Require supplied target') +old, meta = np.macho(original), np.macho(kernel) +for e in plan['extents']: + if e['segment'] == '__TEXT_EXEC': + at = np.backed_offset(meta['segments'], e['va'], e['size'], len(kernel)) + prev = np.backed_offset(old['segments'], e['va'] - plan['slide'], e['size'], len(original)) + require(kernel[at:at + e['size']] == original[prev:prev + e['size']], 'Every original executable extent must remain unchanged') +trust = (ROOT / 'build/original-static-system-trust/original-static-system.bundle').read_bytes() +require(hashlib.sha256(trust).hexdigest() == '7416ca4f8377146626bb284e2133c76bee6cb4e3ee0af5d648b186235a7323a7' and + struct.unpack_from('> 14 & 3 == 0 and regs['TCR_EL1'] >> 30 & 3 == 2 and regs['SCTLR_EL1'] & 1 and + regs['TTBR0_EL1'] and regs['TTBR1_EL1'] and d['original_amfi_table_owner_observed'] and + d['original_crypto_initialized_observed'] and d['original_crypto_table_owner_observed'], 'Require active native 4K and genuine original crypto/AMFI') + root = d['owner_root_binding_audit'] + require(root['boot_root_accepted'] and root['_native_owner_root_apple_autherr'] == 80 and + root['actual_uuid'] == '04769D26-2CA0-4825-B443-E2E5F88300F2', 'Original OS root proof must remain intact') + require(d['native_virtio_globals']['_native_virtio_read_failures'] == 0 and + d['native_virtio_integrity']['_native_virtio_integrity_failures'] == 0, 'No unauthenticated I/O may be exposed') +for d in (mounted, continuation, file_call): + state = d['owner_cache_mount'] + require(state['_native_owner_cache_stage'] == 6 and state['_native_owner_cache_status'] == 0 and + state['_native_owner_cache_mount_status'] == 0 and state['_native_owner_cache_readonly'] == 1 and + state['_native_owner_cache_authorized'] == 1 and state['_native_owner_cache_root_recheck'] == 1 and + state['_native_owner_cache_selected_device'] == state['_native_owner_cache_mounted_device'] == 0x01000004 and + state['actual_device_path'] == '/dev/disk2s1' and d['native_virtio_integrity']['_native_virtio_verified_devices'] == 2, + 'Real APFS user mount, actual dev_t/UUID binding and separate root recheck must pass') +require(mounted['stop_symbol'] == '_native_owner_cache_checkpoint' and mounted['function_return_stack_matched'] and + mounted['native_virtio_globals']['_native_virtio_read_requests'] == 304 and + mounted['native_virtio_integrity']['_native_virtio_verified_blocks'] == 478, 'Observe actual post-mount same-stack checkpoint') +for value in mounted['indirect_memory'].values(): + t = value['translation'] + require(t['leaf_level'] == 3 and t['leaf_bytes'] == 4096 and t['ap'] == 2 and t['pxn'] and t['uxn'], 'Private metadata and key copies retain native-4K RO/NX PTEs') +state = missing['owner_cache_mount'] +require(missing['stop_symbol'] == '_panic' and missing['argument_memory']['x0'].startswith('offline owner SystemOS mount failed: %d') and + struct.unpack_from('= 5 * 1024**3, 'Disk floor' + + +def run(cmd, **kw): + return subprocess.check_output([str(x) for x in cmd], text=True, timeout=90, **kw) + + +def function_body(source, marker): + start = source.index(marker) + brace = source.index('{', start) + depth = 1 + end = brace + 1 + while depth: + depth += (source[end] == '{') - (source[end] == '}') + end += 1 + return source[start:end] + + +original = (ROOT/'refer/xnu/osfmk/kern/debug.c').read_text() +header = (ROOT/'refer/xnu/osfmk/kern/debug.h').read_text() +assert re.search(r'#define DEBUGGER_INTERNAL_OPTION_THREAD_BACKTRACE\s+0x200ULL', header) +assert re.search(r'#define DEBUGGER_INTERNAL_OPTIONS_MASK\s+\(DEBUGGER_INTERNAL_OPTION_THREAD_BACKTRACE\)', header) +wrappers = [function_body(original, '\nvoid\n'+name+'(') + for name in ('panic', 'panic_with_options', 'panic_with_data')] +renderer = (ROOT/'scripts/xnu-panic-screen.c').read_text() +direct = function_body(renderer, 'void panic_screen (') +common = function_body(renderer, 'void panic_screen_common (') +direct_asm = direct[direct.index('__asm__'):direct.rindex(';')+1].replace('panic_screen_main', '_collect_arguments') +common_asm = common[common.index('__asm__'):common.rindex(';')+1].replace('panic_screen_main', '_collect_arguments') +fixture = r''' +#include +#include +#include +#include +#include +typedef unsigned char uuid_t[16]; +_Static_assert(sizeof(va_list) == sizeof(char *), "Darwin ARM64 stack argument pointer"); +#define DEBUGGER_INTERNAL_OPTION_THREAD_BACKTRACE 0x200ULL +#define DEBUGGER_INTERNAL_OPTIONS_MASK (DEBUGGER_INTERNAL_OPTION_THREAD_BACKTRACE) +static char collected[256]; +static unsigned records, data_calls; +__attribute__((used,noinline)) void collect_arguments(const char *fmt,const unsigned long *slots) { + va_list ap=(char *)slots; + vsnprintf(collected,sizeof(collected),fmt,ap);++records; +} +static void ext_paniclog_panic_with_data(uuid_t uuid,void *addr,uint32_t length) { + assert(uuid[0]==0x5a && addr && length==7);++data_calls; +} +''' +fixture += '\n__attribute__((naked,noinline)) void panic_trap_to_debugger(const char *fmt,va_list *ap,unsigned reason,void *ctx,uint64_t options,void *data,unsigned long caller) {'+common_asm+'}\n' +fixture += '\n__attribute__((naked,noinline)) void direct_entry(const char *fmt,...) {'+direct_asm+'}\n' +fixture += '\n'.join(wrappers).replace('\npanic(', '\nfixture_panic(').replace('\npanic_with_options(', '\nfixture_panic_with_options(').replace('\npanic_with_data(', '\nfixture_panic_with_data(') +fixture += r''' +int main(void) { + direct_entry("direct %s %u %llx", "argument", 37U, 0x123456789abcdef0ULL); + assert(!strcmp(collected,"direct argument 37 123456789abcdef0")); + fixture_panic("plain %s %u %llx", "argument", 41U, 0xfedcba9876543210ULL); + assert(!strcmp(collected,"plain argument 41 fedcba9876543210")); + fixture_panic_with_options(9,(void *)0x1234,0x240,"options %s %u %llx", "argument", 43U, 0x100000001ULL); + assert(!strcmp(collected,"options argument 43 100000001")); + uuid_t uuid={0x5a};char data[7]={0}; + fixture_panic_with_data(uuid,data,sizeof(data),"data %s %u %llx", "argument", 47U, 0x1122334455667788ULL); + assert(!strcmp(collected,"data argument 47 1122334455667788")); + assert(records==4 && data_calls==1); + puts("PASS: actual Darwin ARM64 argument adapters, direct entry and all three source panic wrappers"); +} +''' +(out/'abi.c').write_text(fixture) +run(['clang','-std=c11','-O2','-Wall','-Wextra','-Werror','-Wno-unused-parameter',out/'abi.c','-o',out/'abi']) +print(run([out/'abi']), end='') + +spec = importlib.util.spec_from_file_location('panic_probe', ROOT/'scripts/patch-xnu-entry-probe.py') +probe = importlib.util.module_from_spec(spec) +spec.loader.exec_module(probe) +before = args.kernel.read_bytes() +probe.load_segments(before) +names = probe.rebase(args.kernel, before) +expected_vinfo=names.get('_boot_progress_panic_vinfo',names.get('_vinfo')) +assert expected_vinfo is not None and probe.VINFO_VA==expected_vinfo, 'Wrong panic mapping prefix' +symbols = probe.kernel_symbols(args.kernel) +starts = sorted(symbols) + + +def calls(name): + low, high = probe.function_range(symbols, name) + result = [] + for va in range(low, high, 4): + word = struct.unpack_from('> 26 == 0b100101: + n = word & 0x3ffffff + n -= (1 << 26) if n & (1 << 25) else 0 + result.append(va+n*4) + return result + + +for name in ('_panic', '_panic_with_options', '_panic_with_data'): + assert names['_panic_trap_to_debugger'] in calls(name), name+' does not use the expected real common funnel' + +env = dict(os.environ) +env['PATH'] = '/opt/homebrew/opt/llvm/bin:/opt/homebrew/opt/lld/bin:' + env['PATH'] +with tempfile.TemporaryDirectory(dir=out) as temp: + temp = Path(temp) + copy = temp/'candidate.macho' + shutil.copyfile(args.kernel, copy) + log = run(['python3',ROOT/'scripts/patch-xnu-entry-probe.py',copy,'--panic-screen','--no-reset'],env=env,cwd=ROOT) + (out/'patch.log').write_text(log) + after = copy.read_bytes() + common_site = names['_panic_trap_to_debugger'] + panic_cave = probe.CAVES[2]+probe.CAVE_SIZE + code, load, entry = probe.assemble_c(ROOT/'scripts/xnu-panic-screen.c',panic_cave,temp,names) + alternate = probe.panic_common_entry(temp,code,load) + assert load+len(code) <= probe.PANIC_CAVE_END + for site, target in [(probe.PANIC_VA,entry),(common_site,alternate)]: + assert struct.unpack_from('> 14 & 3 == 0 and r['TCR_EL1'] >> 30 & 3 == 2 and + r['SCTLR_EL1'] & 1 and r['TTBR0_EL1'] and r['TTBR1_EL1'] and + d['kernel_page_globals'] == dict(_PAGE_SHIFT_CONST=12, _page_shift=12, _page_shift_user32=12, _page_size=4096), + 'Bounded real HVF observation with active native-4K hardware and runtime globals') + require(not d['sheng_hardware_tested'] and not d['functional_ipados_boot_demonstrated'], 'Offline scope only') + return d + + +probe = candidate('native-owner-pgz-fixed-probe-candidate', + '85a6eb541ca5771042dcbf3d167f20e8f0c7eddab00a7e6486b7cd851b0e83a7', + 'fd558d17129ae7c6ab53711f2e7d437985a34c43debfd4f51418124e6e35515b') +for mode, label, stage, reason in ( + (1, 'positive', 4, None), + (2, 'misaligned', 2, "isn't aligned to zone element size"), + (3, 'wrong-zone', 2, 'not in the expected zone'), + (4, 'freed', 3, 'probabilistic gzalloc use-after-free')): + d = report(label, probe) + a = d['actual_zone_pgz_probe'] + require(a['case'] == mode and a['stage'] == stage and a['metadata'] >> 12 == 10 and + a['metadata'] & 1023 == a['zone'] and a['backing'] != a['alias'] and + a['backing'] & 4095 == a['alias'] & 4095 and (a['alias'] & 4095) + 192 <= 4096 and + a['argument'] == a['alias'] + (mode == 2) and + (a['expected_zone'] != a['zone']) == (mode == 3), 'Actual controlled PGZ allocation and argument') + require(d['stop_symbol'] == ('_panic' if reason else '_native_zone_pgz_probe_checkpoint') and + (reason is None or reason in d['argument_memory']['x0']), 'Positive completion or exact genuine refusal') +print('PASS: real PGZ secondary-page allocation validates, writes through and reclaims; misaligned/wrong-zone/freed controls refuse') + +normal = candidate('native-owner-pgz-fixed-candidate', + 'ba60f353a90b61fb6d63425648aeb4475a2c237ed19800d60d2f583cea035bb3', + 'a4a5fd00dbfb3d41ffe7913cc31b14a2db58e27f5d89129d64018e534b9dc309') +symbols = subprocess.check_output(['xcrun', 'nm', '-U', ROOT / 'build/native-prelink/native-owner-pgz-fixed-candidate.macho'], text=True) +require('_native_zone_pgz_probe_' not in symbols, 'Allocator instrument absent from normal candidate') +main = report('launchd-main', normal) +require(main['stop_symbol'] == 'original-launchd-main' and main['original_launchd_main_executed'] and + main['followed_main_entry']['first_instruction_completed'] and int(main['pstate'], 16) & 31 == 0 and + main['direct_user_commpage_shifts'] == {'fffff4025:1': 12, 'fffff4037:1': 12}, + 'Unchanged original launchd executes at EL0 with actual native-4K userspace commpages') +require(main['owner_root_binding_audit']['boot_root_accepted'] and + main['owner_root_binding_audit']['_native_owner_root_apple_autherr'] == 80 and + main['owner_boot_manifest']['status'] == 0 and main['actual_root_composition']['_native_owner_cache_native_union'] == 1 and + main['original_amfi_table_owner_observed'] and main['original_crypto_initialized_observed'] and + main['actual_signature_failures'] == dict(_cs_validate_page_bad_hash=0, _cs_validate_page_no_hash=0) and + main['native_virtio_integrity']['_native_virtio_integrity_failures'] == 0, + 'Real independent owner roots, original AMFI/crypto and intact storage/code integrity') +print('PASS: probe-disabled original launchd executes with owner roots, original AMFI and native-4K hardware/userspace ABI') + +later = report('first-exception', normal) +s = json.loads((ROOT / 'build/native-user-exception/pgz-fixed-symbolication.json').read_text()) +require(later['stop_symbol'] == '_exception_triage' and s['kernel_sha256'] == normal and + int(later['observed_function_entry_arguments'][0], 16) == 6 and + int(later['observed_function_entry_arguments'][2], 16) == 2 and + len(later['entry_user_state_candidates']) == 1, 'Actual subsequent two-code user exception, not a kernel panic') +u = later['entry_user_state_candidates'][0] +require(struct.unpack('<2Q', bytes.fromhex(later['entry_regions']['1:0:10']['bytes'])) == (1, int(u['user_pc'], 16)) and + int(u['cpsr'], 16) & 31 == 0 and int(u['esr'], 16) >> 26 == 0x3c and + u['instruction_bytes'].startswith('000020d4200020d4') and s['local_symbols_uuid_matches'] and + s['unique_instruction_match'] == dict(file='dyld_shared_cache_arm64.09', offset=103477996, unslid_pc=6966604524) and + int(s['cache_slide'], 16) + 6966604524 == int(u['user_pc'], 16) and + s['frames'][0]['nearest_local_symbol'] == '-[FBSDisplayMonitor _initWithBookendObserver:transformer:]' and + s['static_original_assertion'] == dict(literal_va='0x1e7a9bf68', text='failed to find the main CADisplay', runtime_string_captured=False), + 'Cross-qualified unchanged FrontBoard EL0 BRK and original assertion reference') +require(later['actual_signature_failures'] == dict(_cs_validate_page_bad_hash=0, _cs_validate_page_no_hash=0) and + later['native_signature_hash_cache']['_cs_native4k_full_blocks'] == 201083 and + later['native_signature_hash_cache']['_cs_native4k_ready_blobs'] == 53 and + later['native_signature_hash_cache']['_cs_native4k_page_checks'] == 15861 and + later['owner_root_binding_audit']['boot_root_accepted'] and later['original_amfi_table_owner_observed'] and + later['native_virtio_integrity']['_native_virtio_integrity_failures'] == 0, + 'Actual continued original startup preserves code/storage integrity and authenticated root') +print('PASS: later original FrontBoard display assertion, 15861 native-4K page validations, zero bad/missing hashes; UI remains open') diff --git a/scripts/check-pgz-legacy-checkpoint.py b/scripts/check-pgz-legacy-checkpoint.py new file mode 100644 index 0000000..360eeb9 --- /dev/null +++ b/scripts/check-pgz-legacy-checkpoint.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +"""Require a real allocated PGZ secondary-page object falsely refused before the fix.""" +import hashlib +import json +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +def require(value, text): + if not value: raise SystemExit(text) +p = ROOT / 'build/native-prelink/native-owner-pgz-legacy-probe-candidate' +plan = json.loads(p.with_suffix('.json').read_text()) +require(plan['native_sha256'] == 'bd62a6409a7ba85849dc8d5bcfe844dcd8d92308d56ca1500ebebe0a9b38e066' and + hashlib.sha256(p.with_suffix('.macho').read_bytes()).hexdigest() == plan['candidate_sha256'] == + '855faea2c3dea3e363615283415c1af2b9eec7de4997e51f292e84c91cfa5f27', 'Exact pre-fix probe kernel') +d = json.loads((ROOT / 'build/darwinboot-owner-hvf-pgz-legacy-positive/gdb-stop.json').read_text()) +a = d['actual_zone_pgz_probe'] +require(d['kernel_sha256'] == plan['candidate_sha256'] and d['native_4k_mmu_observed'] and + not d['bounded_timeout_interrupted'] and d['stop_symbol'] == '_panic' and + a['case'] == 1 and a['stage'] == 2 and a['expected_zone'] == a['zone'] and + a['metadata'] >> 12 == 10 and a['metadata'] & 1023 == a['zone'] and + a['backing'] != a['alias'] == a['argument'] and a['backing'] & 4095 == a['alias'] & 4095 and + (a['alias'] & 4095) + 192 <= 4096 and d['argument_memory']['x0'].startswith( + 'metadata %p corresponding to addr %p being freed to zone') and + 'chunk len greater than max' in d['argument_memory']['x0'], + 'Actual live correct-zone PGZ object must reproduce the earlier false refusal; no timeout or invalid fixture') +print('PASS: pre-fix native-4K kernel genuinely rejects an allocated correct-zone PGZ object after its ordinary backing passed alignment') diff --git a/scripts/check-prelink-rebuild-equivalence.py b/scripts/check-prelink-rebuild-equivalence.py new file mode 100644 index 0000000..1519001 --- /dev/null +++ b/scripts/check-prelink-rebuild-equivalence.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +"""Compare rebuild bytes, excluding only UUID, N_OSO timestamps and derived KCID. + +This does not qualify a replacement boot image. Keep the exact image that was +executed; the check only explains nondeterminism in a source rebuild. +""" +import argparse +import hashlib +import json +from pathlib import Path +import re +import struct + + +def normalized(data): + b = bytearray(data) + if struct.unpack_from(' len(b): + raise ValueError('Malformed load command') + if command == 0x1b: + if size != 24: raise ValueError('Malformed UUID') + b[at+8:at+24] = bytes(16); counts['uuid'] += 1 + elif command == 2: + offset, count, _, _ = struct.unpack_from('<4I', b, at+8) + if offset + count*16 > len(b): raise ValueError('Malformed symbols') + for n in range(count): + item = offset+n*16 + if b[item+4] == 0x66: # N_OSO: object-path debug entry, value is timestamp + b[item+8:item+16] = bytes(8); counts['oso_timestamps'] += 1 + elif command == 0x19 and bytes(b[at+8:at+24]).rstrip(b'\0') == b'__PRELINK_INFO': + offset, length = struct.unpack_from(' len(b): raise ValueError('Malformed prelink info') + for m in re.finditer(rb'_PrelinkKCID([A-Za-z0-9+/]{22}==)',b[offset:offset+length]): + lo, hi = m.span(1); b[offset+lo:offset+hi] = b'0'*(hi-lo); counts['derived_kcid'] += 1 + at += size + if counts['uuid'] != 1 or counts['derived_kcid'] != 1 or not counts['oso_timestamps']: + raise ValueError('Unexpected identity/debug metadata') + return bytes(b), counts + + +def main(): + p=argparse.ArgumentParser(description=__doc__) + p.add_argument('--qualified',required=True,type=Path) + p.add_argument('--rebuilt',required=True,type=Path) + p.add_argument('--output',required=True,type=Path) + a=p.parse_args();a.output.unlink(missing_ok=True) + old=a.qualified.read_bytes();new=a.rebuilt.read_bytes() + x,xc=normalized(old);y,yc=normalized(new) + if x != y or xc != yc: raise ValueError('Rebuild differs beyond UUID/debug timestamps/derived KCID') + result={'passed':True,'qualified_sha256':hashlib.sha256(old).hexdigest(), + 'rebuilt_sha256':hashlib.sha256(new).hexdigest(),'normalized_sha256':hashlib.sha256(x).hexdigest(), + 'ignored_metadata_fields':xc,'replacement_image_qualified':False} + a.output.parent.mkdir(parents=True,exist_ok=True);a.output.write_text(json.dumps(result,indent=2)+'\n') + print('PASS: rebuild differs only in enumerated identity/debug metadata; retain executed image') + + +if __name__=='__main__': main() diff --git a/scripts/check-prelink-xml.py b/scripts/check-prelink-xml.py new file mode 100644 index 0000000..8e2ad55 --- /dev/null +++ b/scripts/check-prelink-xml.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +"""Check generated metadata with the pinned XNU XML tag/string lexer on host. + +This checks lexer compatibility, not the complete kernel object parser or boot. +The actual source functions are compiled unchanged into ignored build output. +""" +import argparse +from pathlib import Path +import subprocess +import sys + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'kits/NativePrelink')) +from native_prelink import macho + +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument('candidate', type=Path) +args = parser.parse_args() +data = args.candidate.read_bytes() +info = next(s for s in macho(data)['segments'] if s['name'] == '__PRELINK_INFO') +xml = data[info['fileoff']:info['fileoff'] + info['filesize']].split(b'\0')[0] + b'\0' +source = (ROOT / 'refer/xnu/libkern/libkern/c++/OSUnserializeXMLSharedImplementation.h').read_text() +start = source.index('#define TAG_MAX_LENGTH') +end = source.index('static long long\ngetNumber', start) +lexer = source[start:end] +output = ROOT / 'build/native-prelink/xml-lexer' +output.mkdir(parents=True, exist_ok=True) +(output / 'metadata.xml').write_bytes(xml) +program = r'''#include +#include +#include +#include +typedef struct { const char *parseBuffer; int parseBufferIndex; int lineNumber; } parser_state_t; +#define safe_free(pointer, size) free(pointer) +''' + lexer + r''' +int main(int argc, char **argv) { + if (argc != 2) return 2; + FILE *f = fopen(argv[1], "rb"); if (!f) return 2; + fseek(f, 0, SEEK_END); long size = ftell(f); rewind(f); + char *data = calloc((size_t)size + 1, 1); if (!data) return 2; + if (fread(data, 1, (size_t)size, f) != (size_t)size) return 2; + fclose(f); + parser_state_t parser = {data, 0, 1}; parser_state_t *state = &parser; unsigned count = 0; + while (currentChar()) { + if (currentChar() != '<') { (void)nextChar(); continue; } + int before = state->parseBufferIndex; + char tag[TAG_MAX_LENGTH], attributes[TAG_MAX_ATTRIBUTES][TAG_MAX_LENGTH], values[TAG_MAX_ATTRIBUTES][TAG_MAX_LENGTH]; + int attr_count; + int type = getTag(state, tag, &attr_count, attributes, values); + if (!type || ((!strcmp(tag, "true") || !strcmp(tag, "false")) && type != TAG_EMPTY)) { + fprintf(stderr, "Actual XNU tag lexer rejects byte %d\n", before); return 1; + } + if (type == TAG_START && (!strcmp(tag, "key") || !strcmp(tag, "string"))) { + int allocated; char *decoded = getString(state, &allocated); + if (!decoded) { fprintf(stderr, "Actual XNU string lexer rejects byte %d\n", before); return 1; } + free(decoded); + } + count++; + } + printf("PASS: actual pinned XNU tag/string lexer accepts %u tags; full object parser not claimed\n", count); + free(data); return 0; +} +''' +(output / 'lexer.c').write_text(program) +subprocess.run(['xcrun', 'clang', '-std=c11', '-Wall', '-Wextra', '-Werror', + str(output / 'lexer.c'), '-o', str(output / 'check')], check=True) +subprocess.run([str(output / 'check'), str(output / 'metadata.xml')], check=True) diff --git a/scripts/check-provisioned-data.py b/scripts/check-provisioned-data.py new file mode 100644 index 0000000..1abff37 --- /dev/null +++ b/scripts/check-provisioned-data.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +"""Qualify the usermanagerd session-setup checkpoint with the provisioned Data. + +The retained run is a bounded QEMU/HVF boot of the 29-component candidate with +the empty directory skeleton and the genuine -apfs_shared_datavolume argument. +It proves the original user-management stack creates its own personas and +foreground user; it does not prove a SpringBoard/backboardd launch, a display, +a desktop or any hardware result. +""" +import json +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +REPORT = ROOT / 'build/darwinboot-armdt-bootdata/gdb-stop.json' +CHECKPOINT = ROOT / 'build/armdt-platform/data-provision-checkpoint.json' + +report = json.loads(REPORT.read_text()) +if not report.get('native_4k_mmu_observed'): + raise SystemExit('Native 4K must be active') +data = (report.get('scratch_data') or {}).get('state', {}) +if data.get('_native_owner_data_status') != 0 or data.get('_native_owner_data_readonly') != 0: + raise SystemExit('The Data volume must be writable through the real backend') +writes = [] +for event in report.get('syscall_events', []): + if event.get('number') in (4, 397) and event.get('write_bytes'): + try: + writes.append(bytes.fromhex(event['write_bytes']).decode('utf-8', 'replace')) + except ValueError: + continue +text = '\n'.join(writes) +required = ( + 'MKB_INIT: No system keybag loaded.', + '****** DONE MKB_INIT ****', + 'Creating New Primary Session with uid 501', + 'Created default user 501', + 'Loading foreground user 501', + 'Created Personal Persona Successfully', + 'Created System Persona Successfully', +) +for item in required: + if item not in text: + raise SystemExit(f'Missing original usermanagerd output: {item}') +if 'FATAL KEYBAG ERROR' in text or 'REBOOTING INTO RECOVERY MODE' in text: + raise SystemExit('The run still took the fatal keybag/recovery path') +if report.get('critical_process_exit'): + raise SystemExit('A critical userspace process still exits') +last = max((event.get('elapsed_seconds', 0) for event in report.get('syscall_events', [])), default=0) +result = { + 'kernel_sha256': report['kernel_sha256'], + 'data_fixture_sha256': (report.get('scratch_data') or {}).get('input', {}).get('sha256'), + 'data_writable': True, + 'keybag_no_system_keybag_allowlist': True, + 'usermanagerd_created_sessions': True, + 'foreground_user': 501, + 'personas_created': ['System', 'Default', 'Personal'], + 'capture_last_seconds': last, + 'capture_exhausted': bool(report.get('syscall_log_exhausted')), + 'springboard_observed_in_capture': 'SpringBoard' in text, + 'backboardd_observed_in_capture': 'backboardd' in text, + 'native_4k_observed': True, + 'qemu_only': True, + 'sep_or_system_keybag_present': False, + 'functional_ipados_boot_demonstrated': False, +} +CHECKPOINT.parent.mkdir(parents=True, exist_ok=True) +CHECKPOINT.write_text(json.dumps(result, indent=2) + '\n') +print('PASS: provisioned Data reaches original usermanagerd session setup') +print('Open: SpringBoard/backboardd, display/GPU, UI; capture may end before later processes') diff --git a/scripts/check-qemu-ramfb-geometry.py b/scripts/check-qemu-ramfb-geometry.py new file mode 100644 index 0000000..2c5dfc1 --- /dev/null +++ b/scripts/check-qemu-ramfb-geometry.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 +"""Cross-check real QEMU scanout dimensions against actual native boot video.""" +import argparse +import hashlib +import json +from pathlib import Path +import struct + +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument('observation', type=Path) +parser.add_argument('--width', type=int, default=3048) +parser.add_argument('--height', type=int, default=2032) +args = parser.parse_args() +if not 1 <= args.width <= 4096 or not 1 <= args.height <= 4096: + parser.error('Dimensions must be bounded positive values') +report = json.loads(args.observation.read_text()) +video = report['actual_boot_video'] +boot, pe = video['boot_args'], video['pe_video'] +assert video['qemu_ramfb_attached'] +assert (boot['width'], boot['height'], boot['depth']) == (args.width, args.height, 32) +assert (pe['v_width'], pe['v_height'], pe['v_depth']) == (args.width, args.height, 32) +assert boot['physical_base'] == pe['v_baseAddr'] != 0 +assert boot['row_bytes'] == pe['v_rowBytes'] >= args.width * 4 +capture = report['qemu_screen_capture'] +png = args.observation.parent / capture['file'] +data = png.read_bytes() +assert data[:8] == b'\x89PNG\r\n\x1a\n' +assert struct.unpack_from('>2I', data, 16) == (args.width, args.height) +assert hashlib.sha256(data).hexdigest() == capture['sha256'] +tcr = int(report['system_registers']['TCR_EL1'], 16) +assert (tcr >> 14) & 3 == 0 and (tcr >> 30) & 3 == 2 +result = dict(kernel_sha256=report['kernel_sha256'], + evidence='actual QEMU ramfb scanout plus native boot/PE video observations', + width=args.width, height=args.height, row_bytes=boot['row_bytes'], + screen_sha256=capture['sha256'], native_cpu_granule_bytes=4096, + native_legacy_display=report.get('native_legacy_display'), + hardware_tested=False, physical_display_qualified=False) +args.observation.with_name('ramfb-geometry-checked.json').write_text(json.dumps(result, indent=2)+'\n') +print(f'PASS: actual QEMU scanout, boot_args and PE video all {args.width}x{args.height}; active CPU 4K') diff --git a/scripts/check-qemu-vmm-checkpoints.py b/scripts/check-qemu-vmm-checkpoints.py new file mode 100644 index 0000000..f67feb6 --- /dev/null +++ b/scripts/check-qemu-vmm-checkpoints.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +"""Check actual original AMFI sysctl discovery of the offline QEMU identity.""" +import hashlib +import json +from pathlib import Path +import struct +import sys + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'kits/NativePrelink')) +import native_prelink as np + + +def require(value, message): + if not value: + raise SystemExit(message) + + +def tree(blob): + def read(at): + count, children = struct.unpack_from('> 14 & 3 == 0 and r['TCR_EL1'] >> 30 & 3 == 2 and r['SCTLR_EL1'] & 1 and + r['TTBR0_EL1'] and r['TTBR1_EL1'], 'Observe actual native 4K geometry') +binary = (ROOT / 'build/native-prelink/native-owner-dyld-startup-candidate.macho').read_bytes() +require(hashlib.sha256(binary).hexdigest() == d['kernel_sha256'], 'Exact original-component candidate') +meta = np.macho(binary) +def word(pc): + return struct.unpack_from('1 and + d['actual_signature_failures']==dict(_cs_validate_page_bad_hash=0,_cs_validate_page_no_hash=0) and + d['scratch_data']['actual_seed_sha256']==data['sha256'] and d['trust_cache_bundle_sha256']==build['trust_bundle_sha256'], + 'Real original-runtime drawing worker, native CPU 4K, unchanged original AMFI and authenticated root') +u=d['entry_user_state_candidates'][0];v=struct.unpack('<32Q',bytes.fromhex(u['register_memory']['13:100']['bytes'])) +reason=bytes.fromhex(u['register_memory']['14:200']['bytes']).split(b'\0')[0].decode() +require(u['pointer']==d['actual_process']['user_state'] and u['instruction_bytes'].startswith('000020d4') and + int(u['cpsr'],16)&31==0 and v[:6]==(0x52424641,d['actual_process']['pid'],4096,4096,64,24) and + reason.startswith('Actual uncaught NSInvalidArgumentException: -[OMQueueObject getGPUPriority]: unrecognized selector sent to instance '), + 'Actual original drawing worker raises the precise missing queue-priority selector; own handler records refusal and stops') +rows=d['native_gpu_broker']['actual_cpu_contexts'];n['cpu_contexts'](d,tuple(r['pid'] for r in rows)) +require(d['native_gpu_broker']['requests']==8 and not d['native_gpu_broker']['failures'] and not d['native_gpu_broker']['refusals'], + 'Observed early boundary precedes the subsequent sharing/completion test') +(n['OUT']/'renderbox-drawing-refusal.json').write_text(json.dumps(dict(report_sha256=sha(report), + own_process_sha256=build['process_sha256'],candidate_sha256=d['kernel_sha256'],reason=reason, + original_renderbox_device_initialized=True,original_renderbox_frame_qualified=False, + original_desktop=False,adreno_execution=False,active_native_adreno_4k=False),indent=2)+'\n') +print('PASS: precise real RenderBox drawing refusal captured; no renderer frame or desktop is qualified') diff --git a/scripts/check-renderbox-function-constants.py b/scripts/check-renderbox-function-constants.py new file mode 100644 index 0000000..1244dc3 --- /dev/null +++ b/scripts/check-renderbox-function-constants.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +"""Qualify the real AIR fallback and function-constant boundary without claiming a frame.""" +import runpy +import sys +from pathlib import Path +script=Path(__file__).with_name('check-offline-metal-sharing.py');saved=sys.argv;sys.argv=[str(script),'--depth-stencil'] +try:n=runpy.run_path(str(script)) +finally:sys.argv=saved +ROOT=n['ROOT'];require=n['require'];load=n['load'];sha=n['sha'];struct=n['struct'];json=n['json'];subprocess=n['subprocess'] +folder=ROOT/'build/offline-renderbox-draw';build=load(folder/'build-report.json');data=load(folder/'data-report.json') +require(sha(folder/'air-native')==build['process_sha256']==data['process_sha256'] and sha(folder/'test-inputs.json')==build['manifest_sha256']==data['manifest_sha256'] and + sha(folder/'compiler-data-32.cdr')==data['sha256'] and sha(folder/'offline-trust.bundle')==build['trust_bundle_sha256'] and + sha(folder/'NativeOfflineMetal.bundle/NativeOfflineMetal')==build['plugin_sha256'],'Exact actual own original-renderer instrument, bundle, Data and trust') +for name,digest in load(folder/'test-inputs.json')['sources'].items():require(sha(ROOT/name)==digest,'Exact actual executed drawing source '+name) +for path in (folder/'air-native',folder/'NativeOfflineMetal.bundle'):subprocess.run(['codesign','--verify','--strict',path],check=True,timeout=30,capture_output=True) +p=ROOT/'build/darwinboot-owner-hvf-offline-renderbox-draw-cpu-contexts/gdb-stop.json';d=load(p) +require(d['kernel_sha256']==n['plan']['candidate_sha256'] and not d['bounded_timeout_interrupted'] and d['stop_symbol']=='_exception_triage' and + d['native_4k_mmu_observed'] and d['original_amfi_table_owner_observed'] and d['owner_root_binding_audit']['boot_root_accepted'] and + d['actual_signature_failures']==dict(_cs_validate_page_bad_hash=0,_cs_validate_page_no_hash=0) and d['actual_process']['pid']>1 and + d['scratch_data']['actual_seed_sha256']==data['sha256'] and d['trust_cache_bundle_sha256']==build['trust_bundle_sha256'], + 'Real original-runtime renderer observation, native CPU 4K, original AMFI/root and exact inputs') +u=d['entry_user_state_candidates'][0];memory=u['register_memory'];v=struct.unpack('<32Q',bytes.fromhex(memory['13:100']['bytes'])) +require(u['pointer']==d['actual_process']['user_state'] and not int(u['cpsr'],16)&31 and u['instruction_bytes'].startswith('000020d4') and + v[:6]==(0x52424641,d['actual_process']['pid'],4096,4096,73,24), + 'Actual own diagnostic stops exactly at the original function-constant request, not renderer completion') +raw=bytes.fromhex(memory['14:1000']['bytes']);require(b'\0' in raw,'Complete actual diagnostic C string precedes any unavailable tail page') +detail=raw.split(b'\0',1)[0].decode();require(detail.startswith('Actual function specialization plane__vertex values class MTLFunctionConstantValuesInternal size 16') and + 'index: 0; type: MTLDataTypeUInt4; value: (2, 0, 5502022, 4)' in detail and + 'newIndexedConstantArray{@16@0:8}' in detail and 'newNamedConstantArray{@16@0:8}' in detail, + 'Original runtime supplies an actual UInt4 constant and exposes original typed accessor ABIs') +broker=d['native_gpu_broker'];require(broker['requests']==42 and broker['reply_bytes']==4878 and broker['clients']==broker['active']==2 and + not broker['closed'] and not broker['failures'] and not broker['refusals'],'Real two-client original AIR-library fallback, still incomplete') +n['cpu_contexts'](d,tuple(r['pid'] for r in broker['actual_cpu_contexts'])) +host_path=folder/'host-cpu-contexts-report.json';host=load(host_path);ops=host['operations'];libraries=[o for o in ops if o['operation']==0x102] +require(len(ops)==42 and len(libraries)==2 and len({o['session'] for o in libraries})==2 and + all(o['handle'] and not o['error'] and o['result_bytes']==1591 for o in libraries) and + not host['transport_healthy'] and not host['session_finished'] and not host['completed_commands'] and not host['open_encoders'], + 'Both unchanged original RenderBox AIR libraries actually load, with no completed drawing or invented success') +require(len(host['aborted_encoders'])==2 and {o['session'] for o in host['aborted_encoders']}=={o['session'] for o in libraries} and + all(o['encoding_ended'] and o['actual_command_status']==1 and not o['gpu_submission_created_by_cleanup'] and o['reason']=='transport-lost' for o in host['aborted_encoders']), + 'Both real abandoned encoders end without committing the interrupted commands; full partial ledger retained') +require('Command encoder released without endEncoding' not in (folder/'host-cpu-contexts-stderr.log').read_text(),'No host Metal encoder-deallocation assertion') +(n['OUT']/'renderbox-function-constant-boundary.json').write_text(json.dumps(dict(candidate_sha256=d['kernel_sha256'],report_sha256=sha(p),host_report_sha256=sha(host_path), + actual_function='plane__vertex',actual_constant_type='MTLDataTypeUInt4',actual_constant_value=[2,0,5502022,4],actual_original_air_fallback=True, + original_renderbox_frame=False,original_desktop=False,adreno_execution=False,active_native_adreno_4k=False),indent=2)+'\n') +print('PASS: actual original AIR-library fallback and UInt4 specialization boundary; abandoned encoders cleanly end without invented GPU submission') diff --git a/scripts/check-renderbox-pipeline-boundary.py b/scripts/check-renderbox-pipeline-boundary.py new file mode 100644 index 0000000..58945fa --- /dev/null +++ b/scripts/check-renderbox-pipeline-boundary.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +"""Qualify the exact original pipeline-library crash, not a renderer frame.""" +import runpy +import sys +from pathlib import Path +script=Path(__file__).with_name('check-offline-metal-sharing.py');saved=sys.argv;sys.argv=[str(script),'--depth-stencil'] +try:n=runpy.run_path(str(script)) +finally:sys.argv=saved +ROOT=n['ROOT'];require=n['require'];load=n['load'];sha=n['sha'];struct=n['struct'];json=n['json'];subprocess=n['subprocess'] +folder=ROOT/'build/offline-renderbox-draw';build=load(folder/'build-report.json');data=load(folder/'data-report.json') +require(sha(folder/'air-native')==build['process_sha256']==data['process_sha256'] and sha(folder/'test-inputs.json')==build['manifest_sha256']==data['manifest_sha256'] and + sha(folder/'compiler-data-32.cdr')==data['sha256'] and sha(folder/'offline-trust.bundle')==build['trust_bundle_sha256'] and + sha(folder/'NativeOfflineMetal.bundle/NativeOfflineMetal')==build['plugin_sha256'],'Exact actual own original-renderer instrument, bundle, Data and trust') +for name,digest in load(folder/'test-inputs.json')['sources'].items():require(sha(ROOT/name)==digest,'Exact actual executed drawing source '+name) +for path in (folder/'air-native',folder/'NativeOfflineMetal.bundle'):subprocess.run(['codesign','--verify','--strict',path],check=True,timeout=30,capture_output=True) +p=ROOT/'build/darwinboot-owner-hvf-offline-renderbox-draw-cpu-contexts/gdb-stop.json';d=load(p) +require(d['kernel_sha256']==n['plan']['candidate_sha256'] and not d['bounded_timeout_interrupted'] and d['stop_symbol']=='_exception_triage' and + d['native_4k_mmu_observed'] and d['original_amfi_table_owner_observed'] and d['owner_root_binding_audit']['boot_root_accepted'] and + d['actual_signature_failures']==dict(_cs_validate_page_bad_hash=0,_cs_validate_page_no_hash=0) and d['actual_process']['pid']>1 and + d['scratch_data']['actual_seed_sha256']==data['sha256'] and d['trust_cache_bundle_sha256']==build['trust_bundle_sha256'], + 'Real original-runtime renderer fault, native CPU 4K, preserved original AMFI/root and exact actual inputs') +u=d['entry_user_state_candidates'][0];registers=struct.unpack_from('<33Q',bytes.fromhex(u['saved_bytes']),8);slide=d['actual_process']['shared_region_slide'];pc=int(u['user_pc'],16)-slide +require(u['pointer']==d['actual_process']['user_state'] and not int(u['cpsr'],16)&31 and int(d['observed_function_entry_arguments'][0],16)==1 and + pc==0x187acab64 and int(u['far'],16)==0x10 and registers[23]==0 and int(u['esr'],16)>>26==0x24 and + any(int(f['lr'],16)-slide==0x1e2618924 for f in u['frames']), + 'Actual original Metal pipeline-loader null internal object and exact original RenderBox caller; not own EL0 success breakpoint') +a=runpy.run_path(str(ROOT/'scripts/audit-target-metal-device.py'));require(bytes.fromhex(u['instruction_bytes'])==a['read'](pc,32),'All observed fault instructions equal unchanged original target cache bytes') +broker=d['native_gpu_broker'];require(broker['requests']==40 and broker['reply_bytes']==1632 and not broker['failures'] and not broker['refusals'], + 'Captured renderer boundary genuinely advances beyond supported attachment/state requests without transport refusal') +n['cpu_contexts'](d,tuple(r['pid'] for r in broker['actual_cpu_contexts'])) +log=folder/'host-cpu-contexts-stderr.log';require('Command encoder released without endEncoding' in log.read_text() and not (folder/'host-cpu-contexts-report.json').stat().st_size, + 'Actual interrupted-encoder cleanup assertion and missing partial host ledger are preserved as a failure, not successful rendering') +(n['OUT']/'renderbox-pipeline-boundary.json').write_text(json.dumps(dict(candidate_sha256=d['kernel_sha256'],report_sha256=sha(p), + actual_original_cache_fault_pc=hex(pc),null_internal_field_address='0x10',actual_original_caller='newPipelineLibraryWithFilePath:error:', + interrupted_encoder_cleanup_failed=True,original_renderbox_frame=False,original_desktop=False,adreno_execution=False,active_native_adreno_4k=False),indent=2)+'\n') +print('PASS: exact original RenderBox pipeline-library fault and interrupted-encoder cleanup failure qualified; no renderer frame claimed') diff --git a/scripts/check-renderbox-queue-priority.py b/scripts/check-renderbox-queue-priority.py new file mode 100644 index 0000000..0794ba6 --- /dev/null +++ b/scripts/check-renderbox-queue-priority.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +"""Qualify actual original RenderBox queue queries up to its next real refusal.""" +import argparse +import hashlib +import json +from pathlib import Path +import runpy +import struct +import subprocess +import sys +p=argparse.ArgumentParser(description=__doc__);p.add_argument('--enqueue',action='store_true');p.add_argument('--texture-refusal',action='store_true');p.add_argument('--private-texture-refusal',action='store_true');p.add_argument('--stencil-texture-refusal',action='store_true');a=p.parse_args() +if a.stencil_texture_refusal:a.private_texture_refusal=True +if a.private_texture_refusal:a.texture_refusal=True +if a.texture_refusal:a.enqueue=True +ROOT=Path(__file__).resolve().parents[1] +def load(p):return json.loads(p.read_text()) +def sha(p):return hashlib.sha256(p.read_bytes()).hexdigest() +def require(value,message): + if not value:raise SystemExit(message) +subprocess.run(['python3',ROOT/'scripts/audit-target-renderbox-device.py'],check=True,timeout=60,capture_output=True) +sys.path.insert(0,str(ROOT/'kits/NativePrelink'));from native_prelink import macho,backed_offset +kernel=ROOT/('build/native-prelink/native-owner-offline-gpu-texture-copy-candidate.macho' if a.stencil_texture_refusal else 'build/native-prelink/native-owner-offline-gpu-enqueue-candidate.macho' if a.enqueue else 'build/native-prelink/native-owner-offline-gpu-priority-candidate.macho');plan=load(kernel.with_suffix('.json'));raw=kernel.read_bytes();image=macho(raw) +require(sha(kernel)==plan['candidate_sha256'] and sha(ROOT/'build/xnu/ios-obj/DEVELOPMENT_ARM64_BCM2837/mach.development.bcm2837.unstripped.noctf')==plan['native_sha256'],'Exact current native kernel and original-component candidate') +for module,digest in plan['original_executable_sha256'].items(): + e=next(e for e in plan['extents'] if e['module']==module and e['segment']=='__TEXT_EXEC');at=backed_offset(image['segments'],e['va'],e['size'],len(raw)) + require(hashlib.sha256(raw[at:at+e['size']]).hexdigest()==digest,'Whole original executable remains unchanged '+module) +folder=ROOT/'build/offline-renderbox-draw';build=load(folder/'build-report.json');data=load(folder/'data-report.json') +require(sha(folder/'air-native')==build['process_sha256']==data['process_sha256'] and sha(folder/'test-inputs.json')==build['manifest_sha256']==data['manifest_sha256'] and + sha(folder/'compiler-data-32.cdr')==data['sha256'] and sha(folder/'offline-trust.bundle')==build['trust_bundle_sha256'] and + sha(folder/'NativeOfflineMetal.bundle/NativeOfflineMetal')==build['plugin_sha256'],'Exact own drawing process, bundle, Data, manifest and trust') +for name,digest in load(folder/'test-inputs.json')['sources'].items():require(sha(ROOT/name)==digest,'Exact executed source '+name) +for path in (folder/'air-native',folder/'NativeOfflineMetal.bundle'): + subprocess.run(['codesign','--verify','--strict',path],check=True,timeout=30,capture_output=True) +report=ROOT/'build/darwinboot-owner-hvf-offline-renderbox-draw-cpu-contexts/gdb-stop.json';d=load(report) +require(d['kernel_sha256']==plan['candidate_sha256'] and not d['bounded_timeout_interrupted'] and d['stop_symbol']=='_exception_triage' and + d['native_4k_mmu_observed'] and d['original_amfi_table_owner_observed'] and d['owner_root_binding_audit']['boot_root_accepted'] and + d['actual_signature_failures']==dict(_cs_validate_page_bad_hash=0,_cs_validate_page_no_hash=0) and + d['scratch_data']['actual_seed_sha256']==data['sha256'] and d['trust_cache_bundle_sha256']==build['trust_bundle_sha256'], + 'Actual original runtime, native CPU 4K, original AMFI and authenticated root') +u=d['entry_user_state_candidates'][0];v=struct.unpack('<32Q',bytes.fromhex(u['register_memory']['13:100']['bytes'])) +reason=bytes.fromhex(u['register_memory']['14:200']['bytes']).split(b'\0')[0].decode() +require(u['pointer']==d['actual_process']['user_state'] and int(u['cpsr'],16)&31==0 and u['instruction_bytes'].startswith('000020d4') and + v[:6]==(0x52424641,d['actual_process']['pid'],4096,4096,70 if a.texture_refusal else 64,24) and + (reason==('Actual own texture refusal type 2 format 253 extent 64x64 depth 1 levels 1 samples 1 array 1 usage 4 storage 2 cache 0 hazard 0 optimized 1 surface 0 format 0 planes 0 plane 0' if a.stencil_texture_refusal else 'Actual own texture refusal type 2 format 65 extent 64x64 depth 1 levels 1 samples 1 array 1 usage 4 storage 2 cache 0 hazard 0 optimized 1 surface 0 format 0 planes 0 plane 0' if a.private_texture_refusal else 'Actual own texture refusal type 2 format 70 extent 64x64 depth 1 levels 1 samples 1 array 1 usage 5 storage 0 cache 0 hazard 0 optimized 1 surface 1 format 1380401729 planes 0 plane 0') if a.texture_refusal else reason.startswith('Actual uncaught NSInvalidArgumentException: -[OMCommandObject '+('addCompletedHandler:' if a.enqueue else 'enqueue')+']: unrecognized selector sent to instance ')), + 'Actual original RenderBox has advanced past priority queries to the next genuine resource/submission boundary') +broker=d['native_gpu_broker'];rows=broker['actual_cpu_contexts'];require(len(rows)==2 and len({r['task'] for r in rows})==len({r['TTBR0_EL1'] for r in rows})==2,'Two actual independent worker CPU contexts') +for r in rows: + tcr=int(r['TCR_EL1'],16);require(tcr>>14&3==0 and tcr>>30&3==2 and int(r['SCTLR_EL1'],16)&1 and int(r['TTBR0_EL1'],16) and int(r['TTBR1_EL1'],16) and r['native_cpu_4k_accepted'],'Actual worker MMU/TTBR/native-4K state') +host=load(folder/'host-cpu-contexts-report.json');priorities=host['queue_priorities'];ops=host['operations'] +require(not host['transport_healthy'] and not host['session_finished'] and not host['completed_commands'] and + not host['cpu_pixel_renderer'] and not host['original_desktop'] and not host['adreno_execution'] and + len(ops)==broker['requests']==(32 if a.stencil_texture_refusal else 30 if a.private_texture_refusal else 16 if a.enqueue else 14) and broker['reply_bytes']==(1376 if a.stencil_texture_refusal else 1312 if a.private_texture_refusal else 864 if a.enqueue else 784) and not broker['failures'] and broker['refusals']==(2 if a.private_texture_refusal else 0) and + all(o['request']==i+1 for i,o in enumerate(ops)) and sum(bool(o['error']) for o in ops)==(2 if a.private_texture_refusal else 0) and all(o['operation']==0x104 and o['error']==-4 for o in ops if o['error']), + 'Exact incomplete real GPU ledger, with no invented frame, completion or successful close') +require(len(priorities)==4 and {p['session'] for p in priorities}=={r['session'] for r in rows} and + all(not p['writing'] and p['before']==p['after']==p['result']==(2 if p['background'] else 1) for p in priorities) and + len({p['queue'] for p in priorities})==2 and all({p['background'] for p in priorities if p['session']==r['session']}=={0,1} for r in rows), + 'Both original RenderBox workers query actual distinct foreground/background GPU priorities') +if a.private_texture_refusal: + textures=[o for o in ops if o['operation']==0x104 and not o['error']] + require(len(textures)==(4 if a.stencil_texture_refusal else 2) and {o['session'] for o in textures}=={r['session'] for r in rows} and + len(host['texture_uploads'])==10 and sum(r['bytes'] for r in host['texture_uploads'])==32768, + 'Both original RGBA IOSurface textures genuinely allocate and upload before the next private RG16Float request refuses') +if a.enqueue: + enqueued=host['enqueued_commands'];require(len(enqueued)==host['pending_enqueues']==2 and + {r['session'] for r in enqueued}=={r['session'] for r in rows} and all(r['status']==1 for r in enqueued) and + all(ops[r['operation_request']-1]['operation']==0x121 and not ops[r['operation_request']-1]['error'] for r in enqueued), + 'Both original RenderBox command buffers are really Enqueued, still pending and never reported Completed') +for name,digest in load(ROOT/'build/host-metal/qualification.json')['backend_sources'].items():require(sha(ROOT/name)==digest,'Exact actual host backend '+name) +(ROOT/('build/native-offline-gpu/renderbox-stencil-texture-refusal.json' if a.stencil_texture_refusal else 'build/native-offline-gpu/renderbox-private-texture-refusal.json' if a.private_texture_refusal else 'build/native-offline-gpu/renderbox-texture-refusal.json' if a.texture_refusal else 'build/native-offline-gpu/renderbox-enqueue.json' if a.enqueue else 'build/native-offline-gpu/renderbox-queue-priority.json')).write_text(json.dumps(dict(candidate_sha256=sha(kernel),native_sha256=plan['native_sha256'], + report_sha256=sha(report),host_report_sha256=sha(folder/'host-cpu-contexts-report.json'),actual_priority_queries=4,actual_enqueued_commands=2 if a.enqueue else 0,actual_native_cpu_4k_workers=2, + next_refusal=reason,original_renderbox_frame=False,original_desktop=False,adreno_execution=False,active_native_adreno_4k=False),indent=2)+'\n') +if a.stencil_texture_refusal:print('PASS: original RenderBox RGBA and private RG16Float textures allocate; next actual Stencil8 request refuses before CPU fallback') +elif a.private_texture_refusal:print('PASS: original RenderBox RGBA IOSurfaces allocate; next actual private RG16Float texture refuses before CPU fallback, without a GPU frame claim') +elif a.texture_refusal:print('PASS: actual original RenderBox RGBA8/RGBA IOSurface request refuses in the own frontend before any CPU fallback; no GPU frame claimed') +else:print('PASS: original RenderBox priority queries'+(' and actual GPU command enqueue' if a.enqueue else '')+' reach the native-4K broker; the next genuine refusal remains explicit') diff --git a/scripts/check-scratch-data-checkpoints.py b/scripts/check-scratch-data-checkpoints.py new file mode 100644 index 0000000..e0ae466 --- /dev/null +++ b/scripts/check-scratch-data-checkpoints.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +"""Qualify a genuine APFS/memdev refusal without misreporting writable Data success.""" +import hashlib +from pathlib import Path +import runpy + +ROOT = Path(__file__).resolve().parents[1] +base = runpy.run_path(str(ROOT / 'scripts/check-original-launchd-main-checkpoints.py')) +require = base['require'] +expected = base['candidate']('native-owner-scratch-data-candidate', + '3a4359eb443e706802f7672e4165f2f69cc79fcc498b73381353c72db7db776d', + 'a85a4fb651ca8cde566291e75989331d5d44d0d45bb71ebc1a7f98e397aa12e7') +seed = (ROOT / 'build/owner-data-fixture/owner-data-32.cdr').read_bytes() +digest = 'd7cd79af299b9dfc1b26ed886000c455f950470c26cffdf796478609bffb6aa9' +require(len(seed) == 33554432 and seed[32:36] == b'NXSB' and hashlib.sha256(seed).hexdigest() == digest, 'Exact host-formatted scratch APFS input remains unchanged') +positive, absent = [base['report']('owner-hvf-scratch-data' + suffix) for suffix in ('', '-absent')] +for d in (positive, absent): + require(d['kernel_sha256'] == expected and not d['bounded_timeout_interrupted'] and d['native_4k_mmu_observed'] and + d['owner_root_binding_audit']['boot_root_accepted'] and d['owner_root_binding_audit']['_native_owner_root_apple_autherr'] == 80 and + d['owner_cache_mount']['_native_owner_cache_status'] == 0 and d['owner_boot_manifest']['status'] == 0 and + d['original_amfi_table_owner_observed'] and d['original_crypto_table_owner_observed'], 'Keep actual native-4K authenticated original disk root/cache path') + require(d['stop_symbol'] == '_panic' and d['argument_memory']['x0'].startswith('offline scratch Data mount failed: %d') and + d['actual_signature_failures'] == dict(_cs_validate_page_bad_hash=0, _cs_validate_page_no_hash=0) and + d['native_virtio_globals']['_native_virtio_read_failures'] == 0 and d['native_virtio_integrity']['_native_virtio_integrity_failures'] == 0, + 'Real mount failure must stop startup, without disk/signature failure substitution') +data = positive['scratch_data'] +require(data['input']['sha256'] == data['actual_seed_sha256'] == digest and data['actual_seed_bytes'] == data['input']['bytes'] == len(seed) and + data['input']['guest_mutable'] and not data['input']['initial_seed_signature_verified'] and not data['input']['production_storage_qualified'], + 'Hash the actual loaded memdev bytes; never label an unsigned mutable seed as an authenticated OS image') +require(data['state'] == dict(_native_owner_data_status=2, _native_owner_data_stage=2, _native_owner_data_readonly=-1, _native_owner_data_device=-1), + 'Original APFS mount of md0 returns real ENOENT before a Data mount exists') +data = absent['scratch_data'] +require(data['input'] is None and data['actual_seed_bytes'] == 0 and data['actual_seed_sha256'] == '00' * 32 and + data['state'] == dict(_native_owner_data_status=22, _native_owner_data_stage=0, _native_owner_data_readonly=-1, _native_owner_data_device=-1), + 'Missing scratch media must refuse early, not continue on a read-only Data namespace') +print('PASS: actual 32 MiB memdev bytes match host seed; genuine APFS mount refuses ENOENT 2') +print('PASS: absent scratch media refuses EINVAL 22; writable Data remains unqualified') diff --git a/scripts/check-smmu4k-audit.py b/scripts/check-smmu4k-audit.py new file mode 100644 index 0000000..c84e507 --- /dev/null +++ b/scripts/check-smmu4k-audit.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +"""Bounded physical-graph audit qualification; host buffers are not hardware.""" +import hashlib,json,re,subprocess +from pathlib import Path +ROOT=Path(__file__).resolve().parents[1] +kit=ROOT/'kits/ArmSmmu4K';out=ROOT/'build/arm-smmu-4k-audit' +out.mkdir(parents=True,exist_ok=True);(out/'report.json').unlink(missing_ok=True) +sources=[kit/(name+'.c') for name in ['ArmSmmu4K','ArmSmmu4KContext','ArmSmmu4KProgram','ArmSmmu4KAudit','ArmSmmu4KTableMemory']] +test=out/'test-audit' +subprocess.run(['xcrun','clang','-std=c11','-O2','-Wall','-Wextra','-Werror','-fsanitize=address,undefined',*sources,kit/'test_audit.c','-o',test],check=True,timeout=60) +run=subprocess.run([test],capture_output=True,text=True,timeout=60) +print(run.stdout,end='') +if run.returncode:raise SystemExit(run.stderr) +counts=re.search(r'(\d+) physical-graph profiles, (\d+) descriptor mutations, (\d+) read failures',run.stdout) +if not counts:raise SystemExit('Missing actual audit coverage') +(out/'host.log').write_text(run.stdout) +def sha(p):return hashlib.sha256(p.read_bytes()).hexdigest() +inputs=sources+[s.with_suffix('.h') for s in sources]+[kit/'test_audit.c',Path(__file__)] +(out/'report.json').write_text(json.dumps(dict(passed=True,sources={str(p.relative_to(ROOT)):sha(p) for p in inputs}, + profiles=int(counts[1]),descriptor_mutations=int(counts[2]),physical_read_failures=int(counts[3]), + host_asan_ubsan=True,all_reachable_slots_audited=True,physical_graph_not_rebuilt=True, + relocated_owned_tables=True,cross_region_full_pool_checks=True,actual_volatile_adapter_on_host_buffers=True, + physical_labels_synthetic=True,actual_hardware_tables_observed=False,gpu_execution_qualified=False),indent=2)+'\n') +print('PASS: bounded physical graph, real volatile-load adapter on host buffers; active-context integration/native ABI checked by check-sheng-gpu-smmu.py') diff --git a/scripts/check-smmu4k-checkpoints.py b/scripts/check-smmu4k-checkpoints.py new file mode 100644 index 0000000..9148645 --- /dev/null +++ b/scripts/check-smmu4k-checkpoints.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +"""Check native inactive tables against actual XNU allocation and CPU mappings.""" +import hashlib +import json +from pathlib import Path +import struct +import sys +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'kits/NativePrelink')) +from native_prelink import macho + + +def require(value, message): + if not value: + raise SystemExit(message) + + +# Filled from the exact post-shareability build, never from target binaries in Git. +plan = json.loads((ROOT / 'build/native-prelink/native-owner-smmu4k-outer-candidate.json').read_text()) +source_hash = 'ad9041065a939cdc04235faa1f88bda693a04666cae66625cb5fa1c01ebc0fe7' +candidate_hash = '5a838595de36a70224e2a21b24f42db62b734f7b35376ff7a0b1ee699b51efb0' +image = (ROOT / 'build/native-prelink/native-owner-smmu4k-outer-candidate.macho').read_bytes() +require(plan['native_sha256'] == source_hash and plan['candidate_sha256'] == candidate_hash == + hashlib.sha256(image).hexdigest() and plan['original_sha256'] == + 'f18b4df865405f2a287b84977d7aca745e056d656d9fc9df1662970ce2969de7' and + len(plan['components']) == 29 and not plan['images_modified'] and not plan['source_inputs_modified'], + 'Exact source/candidate and unchanged target provenance') +build = json.loads((ROOT / 'build/smmu4k-probe/build-report.json').read_text()) +require(build['kit_revision'] == '32c74a03c8bbbdb82fed1cb6ee59a5824878f81f' and not build['gpu_execution'] and not build['submission_authorized'], + 'Exact inactive-table implementation; no GPU execution claim') +for name, digest in build['object_sha256'].items(): + raw = (ROOT / 'build/smmu4k-probe' / name).read_bytes() + parsed = macho(raw) + version = next(at for cmd, at, _ in parsed['commands'] if cmd == 0x32) + require(hashlib.sha256(raw).hexdigest() == digest and parsed['kind'] == 1 and + struct.unpack_from('> 14) & 3 == 0 and (r['TCR_EL1'] >> 30) & 3 == 2 and + r['SCTLR_EL1'] & 1 and r['TTBR0_EL1'] and r['TTBR1_EL1'] and + d['original_crypto_initialized_observed'] and d['original_crypto_table_owner_observed'], + 'Bounded real XNU execution and active native-4K CPU translation') + state = d['native_smmu4k'] + require(not state['smmu_activated'] and not state['gpu_execution_qualified'], 'Do not qualify inactive tables as live SMMU/GPU execution') + if label != 'default-off': + require(state['requested'] and state['status'] == 0 and state['table_pages'] == 5 and state['leaf_pages'] == 2 and + state['refusal_mask'] == 63 and state['output_preserved'] == 1, 'Real native table building and six transactional refusal controls') + else: + require(not state['requested'] and state['status'] == -1 and + all(state[k] == 0 for k in ('table_pages', 'leaf_pages', 'refusal_mask', 'output_preserved', 'allocations_released')) and + int(state['cpu_mair'], 16) == 0 and set(state['pool_physical'] + state['data_physical']) == {'0x0'}, + 'Default-off must perform no diagnostic allocation or table work') + if label != 'native': + require(int(state['pool_pointer'], 16) == int(state['data_pointer'], 16) == 0 and + state['allocations_released'] == (label == 'enabled-main') and + d['original_launchd_main_executed'] and d['followed_main_entry']['first_instruction_completed'] and + int(d['pstate'], 16) & 31 == 0 and + d['direct_user_commpage_shifts'] == {'fffff4025:1': 12, 'fffff4037:1': 12} and + d['owner_root_binding_audit']['boot_root_accepted'] and + d['owner_root_binding_audit']['_native_owner_root_apple_autherr'] == 80 and + d['original_amfi_table_owner_observed'] and d['actual_signature_failures'] == + dict(_cs_validate_page_bad_hash=0, _cs_validate_page_no_hash=0) and + d['native_virtio_integrity']['_native_virtio_integrity_failures'] == 0, + 'Native allocations returned before original EL0 launchd; security/root and user-4K remain intact') + reports[label] = d + +native = reports['native'] +state = native['native_smmu4k'] +require(native['stop_symbol'] == '_native_smmu4k_checkpoint' and not state['allocations_released'], 'Real checkpoint retains actual allocated input/output for inspection') +registers = {k: int(v, 16) for k, v in native['system_registers'].items()} +mask = 0x0000fffffffff000 +for label, count in (('pool', 6), ('data', 2)): + capture = state[label] + va = int(state[label + '_pointer'], 16) + physical = [int(v, 16) for v in state[label + '_physical']] + span = capture['cpu_mappings'] + require(va and not va & 4095 and physical == list(range(physical[0], physical[0] + count * 4096, 4096)) and + span['address'] == hex(va) and span['bytes'] == count * 4096 and len(span['pages']) == count, + 'Real allocated aligned/contiguous physical extent and complete CPU mapping coverage') + for i, page in enumerate(span['pages']): + pointer = va + i * 4096 + require(int(page['virtual_address'], 16) == pointer and int(page['physical_address'], 16) == physical[i] and + page['leaf_level'] == 3 and page['leaf_bytes'] == 4096 and page['ap'] == 0 and + page['pxn'] and page['uxn'] and page['accessed'] and page['shareability'] == 2 and + (int(state['cpu_mair'], 16) >> (8 * page['attr_index'])) & 255 == 255, + 'Every actual allocation page uses a real 4K RW/NX WB/outer-shareable CPU leaf') + table = registers['TTBR1_EL1'] & mask + bits = 64 - ((registers['TCR_EL1'] >> 16) & 63) + start = 4 - ((bits - 12 + 8) // 9) + walk = page['walk'] + require(len(walk) == 4 - start, 'Complete actual TTBR1 physical walk') + for level, step in zip(range(start, 4), walk): + descriptor = int(step['descriptor'], 16) + address = table + ((((pointer & ((1 << bits) - 1)) >> (39 - 9 * level)) & 511) * 8) + require(step['level'] == level and int(step['descriptor_pa'], 16) == address and descriptor & 3 == 3, + 'Recorded physical descriptors must follow the active TTBR and exact native 4K indices') + table = descriptor & mask + require(table == physical[i], 'Actual CPU leaf resolves to the independently captured kernel physical address') + require(len(capture['chunks']) == count, 'Complete allocated memory capture') + for i, chunk in enumerate(capture['chunks']): + require(int(chunk['pointer'], 16) == va and int(chunk['address'], 16) == va + i * 4096 and + len(bytes.fromhex(chunk['bytes'])) == 4096, 'Each actual allocation page is read without changing it') + +pool = b''.join(bytes.fromhex(c['bytes']) for c in state['pool']['chunks']) +data = b''.join(bytes.fromhex(c['bytes']) for c in state['data']['chunks']) +require(data == b'\x39' * 8192 and pool[5 * 4096:] == b'\xa5' * 4096, 'Actual resource bytes and unused table-pool page remain intact') +root = int(state['pool_physical'][0], 16) +expected = {0x1ff000: int(state['data_physical'][0], 16), 0x200000: int(state['data_physical'][1], 16)} +seen, leaves = set(), {} + + +def visit(index, level, prefix): + require(0 <= index < 5 and index not in seen, 'Owned noncyclic inactive table graph') + seen.add(index) + for slot in range(512): + descriptor, = struct.unpack_from(' +typedef uint32_t u32; +#define BIT(n) (UINT32_C(1)<<(n)) +#define GENMASK(h,l) ((UINT32_MAX>>(31-(h))) & (UINT32_MAX<<(l))) +#define FIELD_PREP(mask,val) (((uint32_t)(val)<<__builtin_ctz((uint32_t)(mask))) & (mask)) +struct io_pgtable_cfg {unsigned quirks;struct {struct {unsigned tg,sh,orgn,irgn,tsz,ips;} tcr;} arm_lpae_s1_cfg;}; +''' +harness=''' +uint32_t source_tcr(unsigned bits,unsigned upper,unsigned coherent) { + struct io_pgtable_cfg c={0};c.quirks=upper?IO_PGTABLE_QUIRK_ARM_TTBR1:0; + c.arm_lpae_s1_cfg.tcr.tg=upper?2:0;c.arm_lpae_s1_cfg.tcr.sh=coherent?3:2; + c.arm_lpae_s1_cfg.tcr.orgn=coherent?1:0;c.arm_lpae_s1_cfg.tcr.irgn=coherent?1:0; + c.arm_lpae_s1_cfg.tcr.tsz=64-bits;return arm_smmu_lpae_tcr(&c); +} +uint32_t source_tcr2(unsigned output_code) { + struct io_pgtable_cfg c={0};c.arm_lpae_s1_cfg.tcr.ips=output_code; + return arm_smmu_lpae_tcr2(&c); +} +''' +oracle=out/'source-oracle.c';oracle.write_text(prelude+'\n'.join([quirk[0],*macros,*functions])+harness) +sources=[kit/'ArmSmmu4K.c',kit/'ArmSmmu4KContext.c'] +test=out/'test-context' +subprocess.run(['xcrun','clang','-std=c11','-O2','-Wall','-Wextra','-Werror','-fsanitize=address,undefined',*sources,kit/'test_context.c',oracle,'-o',test],check=True,timeout=60) +result=subprocess.run([test],check=True,capture_output=True,text=True,timeout=30);print(result.stdout,end='') +match=re.search(r'(\d+) context images.*; (\d+) one-bit readback refusals; (\d+) TCR2.TBI/SCTLR.ASIDPNE readback tolerances',result.stdout) +if not match or tuple(map(int,match.groups()))!=(7344,2805408,36720):raise SystemExit('Actual bounded test counts differ') +objects=[];imports={} +for source in sources: + obj=out/(source.stem+'.o') + subprocess.run(['xcrun','clang','-target','arm64-apple-ios17.0','-std=c11','-O2','-Wall','-Wextra','-Werror','-Wno-incompatible-sysroot','-ffreestanding','-mgeneral-regs-only','-c',source,'-o',obj],check=True,timeout=30) + raw=obj.read_bytes();image=macho(raw);version=next(at for command,at,_ in image['commands'] if command==0x32) + if struct.unpack_from('<4I',raw)!=(0xfeedfacf,0x100000c,0,1) or image['kind']!=1 or struct.unpack_from(' +typedef uint64_t arm_lpae_iopte; +#define BIT(n) (1u<<(n)) +'''+block(header,'enum io_pgtable_fmt {')+';\n'+'\n'.join(selected.values())+''' +struct arm_lpae_io_pgtable {struct {enum io_pgtable_fmt fmt;struct {unsigned quirks;} cfg;} iop;}; +''' +oracle=out/'original-permissions.c';oracle.write_text(prelude+function+''' +uint64_t source_permission_pte(unsigned permissions,unsigned cache) { + struct arm_lpae_io_pgtable data={.iop={.fmt=ARM_64_LPAE_S1}}; + int prot=IOMMU_READ; + if(permissions&2)prot|=IOMMU_WRITE; + if(!(permissions&4))prot|=IOMMU_NOEXEC; + if(permissions&8)prot|=IOMMU_PRIV; + if(cache)prot|=IOMMU_CACHE; + return arm_lpae_prot_to_pte(&data,prot); +} +''') +gem=block((cache/names[3]).read_text(),'int msm_gem_prot(') +gmu=block((cache/names[4]).read_text(),'static int a6xx_gmu_memory_alloc(') +if 'MSM_BO_MAP_PRIV' not in gmu or 'MSM_BO_MAP_PRIV' not in gem or 'IOMMU_PRIV' not in gem or 'IOMMU_NOEXEC' in gem: + raise SystemExit('Pinned GMU/MSM mapping policy changed') +sources=[kit/'ArmSmmu4K.c',kit/'test_permissions.c'];test=out/'test-permissions' +subprocess.run(['xcrun','clang','-std=c11','-O2','-Wall','-Wextra','-Werror','-fsanitize=address,undefined',*sources,oracle,'-o',test],check=True,timeout=60) +r=subprocess.run([test],check=True,capture_output=True,text=True,timeout=30);print(r.stdout,end='') +if '544 lower/upper' not in r.stdout:raise SystemExit('Permission case count changed') +(out/'host.log').write_text(r.stdout) +(out/'report.json').write_text(json.dumps(dict(passed=True,linux_revision=pins['linux']['commit'],pinned_sources={n:sha(cache/n) for n in names}, + sources={str(p.relative_to(ROOT)):sha(p) for p in sources+[kit/'ArmSmmu4K.h',Path(__file__)]},oracle_sha256=sha(oracle), + profiles=544,leaves=1088,host_asan_ubsan=True,cpu_permissions_modified=False,active_hardware_mapping_qualified=False),indent=2)+'\n') diff --git a/scripts/check-smmu4k-program.py b/scripts/check-smmu4k-program.py new file mode 100644 index 0000000..3290705 --- /dev/null +++ b/scripts/check-smmu4k-program.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +"""Qualify actual context programming code offline; never open a device.""" +import hashlib,json,re,struct,subprocess,sys +from pathlib import Path +ROOT=Path(__file__).resolve().parents[1];sys.path.insert(0,str(ROOT/'kits/NativePrelink')) +from native_prelink import macho +kit=ROOT/'kits/ArmSmmu4K';out=ROOT/'build/arm-smmu-4k-program';out.mkdir(exist_ok=True) +pins=json.loads((ROOT/'scripts/data/adreno-inputs.json').read_text());cache=ROOT/'.cache/adreno-source' +names=['drivers/iommu/arm/arm-smmu/arm-smmu.h','drivers/iommu/arm/arm-smmu/arm-smmu.c'] +def sha(path):return hashlib.sha256(path.read_bytes()).hexdigest() +for name in names: + if sha(cache/name)!=pins['linux']['files'][name]['sha256']:raise SystemExit('Pinned source differs: '+name) +header=(cache/names[0]).read_text();implementation=(cache/names[1]).read_text() +definitions={} +for text in [header,implementation]: + for match in re.finditer(r'^#define\s+(\w+)(\([^\n]*?\))?\s+((?:[^\n]*\\\n)*[^\n]*)',text,re.M): + definitions[match[1]]=match[0] +registers=['ARM_SMMU_GR0_ID0','ARM_SMMU_GR0_ID1','ARM_SMMU_GR0_ID2','ARM_SMMU_CB_SCTLR', + 'ARM_SMMU_CB_TCR2','ARM_SMMU_CB_TTBR0','ARM_SMMU_CB_TTBR1','ARM_SMMU_CB_TCR', + 'ARM_SMMU_CB_S1_MAIR0','ARM_SMMU_CB_S1_MAIR1','ARM_SMMU_CB_FSR','ARM_SMMU_CB_S1_TLBIASID', + 'ARM_SMMU_CB_TLBSYNC','ARM_SMMU_CB_TLBSTATUS','ARM_SMMU_CB_FSR_FAULT','ARM_SMMU_GR1_CBA2R(0)','QCOM_DUMMY_VAL'] +selected={} +def include(name): + if name in selected:return + if name not in definitions:raise SystemExit('Missing actual source definition '+name) + selected[name]=definitions[name] + for dependency in set(re.findall(r'ARM_SMMU_[A-Z0-9_]+',definitions[name]))-{name}:include(dependency) +for expression in registers:include(expression.split('(')[0]) +oracle=out/'source-registers.c' +oracle.write_text('#include \n#define BIT(n) (UINT32_C(1)<<(n))\n#define GENMASK(h,l) ((UINT32_MAX>>(31-(h))) & (UINT32_MAX<<(l)))\n'+ + '\n'.join(selected.values())+'\nuint32_t source_register(unsigned i){const uint32_t r[]={'+','.join(registers)+'};return r[i];}\n') +# Check the pinned writer's required TCR-before-TTBR ordering, independently +# of our event fixture. This is source evidence, not timing or MMIO evidence. +writer=implementation[implementation.index('void arm_smmu_write_context_bank'):] +writer=writer[:writer.index('\n}',writer.index('{'))+2] +if not writer.index('ARM_SMMU_CB_TCR2') +#include +#include +typedef uint8_t u8;typedef uint16_t u16;typedef uint32_t u32; +#define BIT(n) (UINT32_C(1)<<(n)) +#define GENMASK(h,l) ((UINT32_MAX>>(31-(h))) & (UINT32_MAX<<(l))) +#define FIELD_PREP(mask,value) (((u32)(value)<<__builtin_ctz(mask))&(mask)) +'''+ '\n'.join(selected.values())+'\n' +for signature in ['enum arm_smmu_s2cr_type {','enum arm_smmu_s2cr_privcfg {','struct arm_smmu_smr {','struct arm_smmu_s2cr {']: + prelude+=block(h,signature)+';\n' +prelude+='''struct arm_smmu_device; +struct arm_smmu_impl {void (*write_s2cr)(struct arm_smmu_device *,int);}; +struct arm_smmu_device {struct arm_smmu_smr *smrs;struct arm_smmu_s2cr *s2crs;unsigned features;const struct arm_smmu_impl *impl;}; +static u32 sink_smr,sink_s2cr; +static void arm_smmu_gr0_write(struct arm_smmu_device *s,unsigned reg,u32 value) { + (void)s;if(reg==ARM_SMMU_GR0_SMR(0))sink_smr=value;else if(reg==ARM_SMMU_GR0_S2CR(0))sink_s2cr=value; +} +''' +oracle=out/'original-streams.c';oracle.write_text(prelude+block(c,'static void arm_smmu_write_smr(')+'\n'+block(c,'static void arm_smmu_write_s2cr(')+''' +void source_stream(unsigned exid,unsigned sid,unsigned mask,unsigned valid,unsigned bank,u32 *smr,u32 *s2cr) { + struct arm_smmu_smr m={.mask=mask,.id=sid,.valid=valid}; + struct arm_smmu_s2cr r={.type=S2CR_TYPE_TRANS,.privcfg=S2CR_PRIVCFG_DEFAULT,.cbndx=bank}; + struct arm_smmu_device s={&m,&r,exid?ARM_SMMU_FEAT_EXIDS:0,NULL}; + arm_smmu_write_s2cr(&s,0);arm_smmu_write_smr(&s,0);*smr=sink_smr;*s2cr=sink_s2cr; +} +uint32_t source_stream_register(unsigned i){const uint32_t r[]={'''+','.join(registers)+'''};return r[i];} +''') +sources=[kit/(n+'.c') for n in ['ArmSmmu4K','ArmSmmu4KContext','ArmSmmu4KProgram','ArmSmmu4KStreams']] +test=out/'test-streams';subprocess.run(['xcrun','clang','-std=c11','-O2','-Wall','-Wextra','-Werror','-fsanitize=address,undefined', + '-DSOURCE_ORACLE',*sources,kit/'test_streams.c',oracle,'-o',test],check=True,timeout=60) +r=subprocess.run([test],check=True,capture_output=True,text=True,timeout=30);print(r.stdout,end='');(out/'host.log').write_text(r.stdout) +flags=['-target','arm64-apple-ios17.0','-O2','-Wall','-Wextra','-Werror','-Wno-incompatible-sysroot','-ffreestanding','-mgeneral-regs-only'] +objdump=Path(subprocess.check_output(['brew','--prefix','llvm'],text=True).strip())/'bin/llvm-objdump' +objects=[];provided=set();imports=set() +for source in sources: + obj=out/(source.stem+'.o');subprocess.run(['xcrun','clang',*flags,'-std=c11','-c',source,'-o',obj],check=True,timeout=30) + raw=obj.read_bytes() + if struct.unpack_from('<4I',raw)!=(0xfeedfacf,0x100000c,0,1):raise SystemExit('Ordinary ARM64 required') + at=32;ios17=False + for _ in range(struct.unpack_from(' 1 for entry in by_name['SpringBoard']): + raise SystemExit('SpringBoard must have a real userspace PID') +data = (report.get('scratch_data') or {}).get('state', {}) +if data.get('_native_owner_data_status') != 0 or data.get('_native_owner_data_readonly') != 0: + raise SystemExit('The Data volume must be writable') +result = { + 'kernel_sha256': report['kernel_sha256'], + 'process_count': len(entries), + 'launchd_pid': by_name['launchd'][0]['pid'], + 'backboardd_pids': [entry['pid'] for entry in by_name['backboardd']], + 'springboard_pids': [entry['pid'] for entry in by_name['SpringBoard']], + 'usermanagerd_pids': [entry['pid'] for entry in by_name['usermanagerd']], + 'runningboardd_pids': [entry['pid'] for entry in by_name['runningboardd']], + 'names_present': list(required), + 'data_writable': True, + 'native_4k_observed': True, + 'qemu_only': True, + 'display_service_observed': False, + 'gpu_execution_observed': False, + 'visible_ui_observed': False, + 'sheng_hardware_tested': False, + 'ipados_userspace_boot_demonstrated': True, + 'functional_ipados_ui_demonstrated': False, +} +CHECKPOINT.parent.mkdir(parents=True, exist_ok=True) +CHECKPOINT.write_text(json.dumps(result, indent=2) + '\n') +print('PASS: actual allproc table contains launchd, backboardd and SpringBoard') +print('PASS: 114-process-class original userland is alive under native 4K') +print('Open: display service, GPU execution, visible UI, hardware') diff --git a/scripts/check-system-framebuffer-client.py b/scripts/check-system-framebuffer-client.py new file mode 100644 index 0000000..bb502c8 --- /dev/null +++ b/scripts/check-system-framebuffer-client.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 +"""Verify real GOP display discovery and the unchanged original client factory.""" +import argparse +import hashlib +import mmap +import struct +import json +from pathlib import Path +import subprocess +import sys + +ROOT=Path(__file__).resolve().parents[1];OUT=ROOT/'build/native-legacy-display' +parser=argparse.ArgumentParser(description=__doc__);parser.add_argument('--profile',choices=['pitch','info','scale'],default='scale');args=parser.parse_args() +sys.path.insert(0,str(ROOT/'kits/NativePrelink')) +from native_prelink import macho,backed_offset,unserialize_xml +def require(v,reason): + if not v:raise SystemExit(reason) +def sha(p):return hashlib.sha256(Path(p).read_bytes()).hexdigest() +build=json.loads((OUT/'build-report.json').read_text()) +for name,digest in build['source_sha256'].items():require(sha(ROOT/name)==digest,'Exact current compiled source '+name) +revision=subprocess.check_output(['git','-C',ROOT/'drivers/NativeMobileDisplay','rev-parse','HEAD'],text=True).strip() +for name in ['NativeLegacyDisplay.cpp','NativeGOPPresenter.cpp','NativeGOPWire.h']: + data=subprocess.check_output(['git','-C',ROOT/'drivers/NativeMobileDisplay','show',revision+':'+name]) + require(hashlib.sha256(data).hexdigest()==sha(ROOT/'drivers/NativeMobileDisplay'/name),'Exact committed driver source') +require(sha(OUT/'libNativeLegacyDisplay.late.a')==build['archive_sha256'] and + sha(OUT/'OriginalLegacyFramebufferABI.h')==build['generated_abi_sha256'],'Exact late archive and generated ABI') +require(build['virtual_entries']==358 and build['owned_entries']==[0,1,7,14,84,87,141,184,188,211,213,245,246,288,306,307,337,338,345], + 'Exact qualified own overrides and remaining guarded leaves') +audit=json.loads((ROOT/'build/display-analysis/mobile-graphics-legacy-abi.json').read_text()) +require(int(build['bindings']['__ZTV25IOMobileFramebufferLegacy'],16)==int(audit['legacy_vptr'],16)-16, + 'Apple-kext qualified super calls bind the real original vtable header') +candidate=ROOT/('build/native-prelink/native-owner-legacy-display-'+args.profile+'-candidate.macho');plan=json.loads(candidate.with_suffix('.json').read_text()) +raw=candidate.read_bytes();image=macho(raw) +require(sha(candidate)==plan['candidate_sha256'] and sha(ROOT/'build/xnu/ios-obj/DEVELOPMENT_ARM64_BCM2837/mach.development.bcm2837.unstripped.noctf')==plan['native_sha256'], + 'Exact current source kernel and original-family candidate') +for module,digest in plan['original_executable_sha256'].items(): + extent=next(e for e in plan['extents'] if e['module']==module and e['segment']=='__TEXT_EXEC') + at=backed_offset(image['segments'],extent['va'],extent['size'],len(raw)) + require(hashlib.sha256(raw[at:at+extent['size']]).hexdigest()==digest,'Whole original executable unchanged: '+module) + +# Qualify the actual uint32 output / IOReturn slot-188 contract and its +# missing-property result, rather than giving an opaque method a guessed ABI. +for address,size,digest in [ + (0xfffffff005798184,312,'6c161fa48818f254191d915cc99318e085a1555d7ddbd003b86ccd75034ba363'), + (0xfffffff005778168,56,'b925b4b5bb870637ace1f46bc14010bf5d473edbeb6eaecd9d898fedbc8ee465'), + (0xfffffff00577a554,88,'6d8ebdd3591475c418f3632d112cad88a12cc4a53fa6733190531aa0e56d46c8')]: + at=backed_offset(image['segments'],address,size,len(raw)) + require(hashlib.sha256(raw[at:at+size]).hexdigest()==digest,'Exact original dot-pitch implementation/callers') +pitch_path=ROOT/('build/darwinboot-owner-hvf-legacy-display-start-'+args.profile+'-dotpitch/gdb-stop.json') +pitch=json.loads(pitch_path.read_text()) +require(pitch['kernel_sha256']==plan['candidate_sha256'] and not pitch['bounded_timeout_interrupted'] and + int(pitch['pc'],16)==0xfffffff005778194 and int(pitch['x0'],16)==0xe00002f0, + 'Actual original caller receives genuine missing dot-pitch error') +frame=pitch['frame_chain'][1];data=bytes.fromhex(frame['local_bytes']) +offset=int(pitch['frame_chain'][0]['fp'],16)+16-int(frame['local_address'],16) +require(data[offset:offset+4]==bytes(4),'Actual uint32 output is initialized to zero on missing metadata') +require(pitch['native_legacy_display']['client_calls']>=2 and not pitch['native_legacy_display']['guard_slot'], + 'Real system clients reach the query without a backend guard') + +frame_info_observation=None +if args.profile in ('info','scale'): + require(build['guarded_backend_leaves']==[211,245,246,306,307,337,338,345], + 'Eight unclassified leaves remain guarded; optional frame tracing is explicitly unsupported') + for address,size,digest in [ + (0xfffffff005778688,152,'1dda59338b977c6c2ea0556d397e5ae26ace30d6c8c86fdceea18036ffb5b4c7'), + (0xfffffff00577a718,88,'9ffaf7219e50c6134bdc72fe51eff26de43ed27de4fe37fd0a05fff68771dc84'), + (0xfffffff005f35bb0,24,'75e1cd2260e703a28d2a5e7cf2ed1bbe0b1d4b49359860c12d55634dc85a805b')]: + at=backed_offset(image['segments'],address,size,len(raw)) + require(hashlib.sha256(raw[at:at+size]).hexdigest()==digest,'Exact original frame-info ABI and selector 74 dispatch') + # Original userspace enumerates fixed 64-byte key names with strndup; + # failure / zero count follows its real optional-capability path. + cache=ROOT/'.cache/firmware/target-volumes/systemos-mount/System/Library/Caches/com.apple.dyld' + wanted={0x1b63d53f8:(432,'6a6d646a0ffbbd9ff528d7ec73b7b451af4f33c765ceb02c06c6bffdad77d881'), + 0x1b63d65b0:(172,'e506fe8d7ec92144364bcdbcc5bf519569da6f30b319e24ccfc9a44803a228af')} + found=set() + for path in sorted(cache.glob('dyld_shared_cache_arm64*')): + if path.suffix=='.symbols':continue + with path.open('rb') as stream,mmap.mmap(stream.fileno(),0,access=mmap.ACCESS_READ) as mapping: + require(mapping[:16].rstrip(bytes(1))==b'dyld_v1 arm64','Original ARM64 shared-cache format') + table,count=struct.unpack_from('<2I',mapping,16) + for index in range(count): + va,size,offset,_,protection=struct.unpack_from('<3Q2I',mapping,table+index*32) + for address,(length,digest) in wanted.items(): + if va<=address and address+length<=va+size: + at=offset+address-va + require(protection&4 and hashlib.sha256(mapping[at:at+length]).hexdigest()==digest, + 'Exact unchanged original userspace trace-key enumeration') + found.add(address) + require(found==set(wanted),'Both original userspace frame-info functions are present') + path=ROOT/('build/darwinboot-owner-hvf-legacy-display-start-'+args.profile+'-frameinfo-return/gdb-stop.json') + d=json.loads(path.read_text()) + require(d['kernel_sha256']==plan['candidate_sha256'] and not d['bounded_timeout_interrupted'] and + int(d['pc'],16)==0xfffffff0057786e8 and int(d['x0'],16)==0xe00002c7, + 'Original frame-info caller receives genuine unsupported status') + data=bytes.fromhex(d['stack_bytes']);require(data[16:24]==bytes(8),'Both actual frame-info scalar outputs are zero') + require(bytes.fromhex(d['frame_indirect_memory']['0:-20:40']['bytes'])==bytes(64),'Actual fixed-size trace-key output is cleared') + require(not d['native_legacy_display']['guard_slot'] and d['native_legacy_display']['client_calls']>=2, + 'Optional frame-info query returns after actual system-client creation') + frame_info_observation=dict(checkpoint_sha256=sha(path),status=0xe00002c7,trace_keys_supported=False) + +observations={} +for case,directory in [('client','start-'+args.profile+'-client'),('init','init-'+args.profile+'-control'),('default-off','default-off-'+args.profile+'-control')]: + path=ROOT/('build/darwinboot-owner-hvf-legacy-display-'+directory+'/gdb-stop.json');d=json.loads(path.read_text()) + require(d['kernel_sha256']==plan['candidate_sha256'] and not d['bounded_timeout_interrupted'] and d['qemu_accelerator']=='hvf','Actual bounded same-candidate observation '+case) + regs={k:int(v,16) for k,v in d['system_registers'].items()} + require(d['native_4k_mmu_observed'] and (regs['TCR_EL1']>>14)&3==0 and (regs['TCR_EL1']>>30)&3==2 and + regs['SCTLR_EL1']&1 and regs['TTBR0_EL1'] and regs['TTBR1_EL1'],'Actual native 4K CPU translation') + state=d['native_legacy_display'];require(not state['guard_slot'],'No unclassified backend called before this boundary') + if args.profile=='scale': + require(state['unit_scale_calls']==(1 if case=='client' else 0) and + state['chosen_scale']==(1 if case=='client' else 0) and not state['chosen_rotation'] and + (state['unit_scale_status']==0 if case=='client' else state['unit_scale_status']!=0), + 'Actual unit-scale metadata is published only after the explicit mapped GOP system experiment') + if case=='init': + require(state['stage']==8 and state['released']==1 and state['init_result']==state['parent_match']==1 and + state['object_size']==6320 and not state['registration_calls'] and not state['dt_attached'] and not state['client_calls'], + 'Real initialized derived object releases through original free without publication') + elif case=='client': + require(d['stop_symbol']=='_native_legacy_client_checkpoint' and state['stage']==10 and + state['init_result']==state['start_result']==state['probe_match']==state['parent_match']==1 and + state['object_size']==6320 and state['registration_calls']==state['dt_attached']==1 and not state['released'], + 'Original start completes and actual GOP subsystem is registered/attached') + video=d['actual_boot_video']['pe_video'] + require(state['gop_map'] and state['gop_virtual'] and state['gop_physical']==video['v_baseAddr']==state['video_base'] and + state['gop_bytes']==state['video_bytes']==video['v_rowBytes']*video['v_height']==1920000 and + state['gop_width']==video['v_width']==800 and state['gop_height']==video['v_height']==600, + 'Actual retained GOP mapping and original mode') + props=unserialize_xml(d['native_legacy_properties'].encode()) + require(props['IOMFB Debug Info']['Global']['width']==800 and props['IOMFB Debug Info']['Global']['height']==600 and + 'NativeGOPFramebufferMapping' not in props,'Real original property serialization with no unencodable mapping object') + require(state['client_calls']==1 and state['client_type']==state['client_status']==0 and state['client_pid']>1 and + state['client_task'] and state['client_object'] and d['native_legacy_client_class']=='IOMobileFramebufferUserClient', + 'Original system process creates the genuine original userclient through original newUserClient') + else: + require(d['original_launchd_main_executed'] and d['followed_main_entry']['first_instruction_completed'] and + state['stage']==0 and state['meta_status']==-1 and not state['registration_calls'] and not state['dt_attached'] and not state['client_calls'], + 'Default-off original launchd without display experiment') + if case!='init': + require(d['original_amfi_table_owner_observed'] and d['owner_root_binding_audit']['boot_root_accepted'] and + d['owner_root_binding_audit']['_native_owner_root_apple_autherr']==80 and + d['actual_signature_failures']==dict(_cs_validate_page_bad_hash=0,_cs_validate_page_no_hash=0), + 'Original AMFI/Apple rejection and independent owner-root acceptance') + shifts=d['commpage_user64_shifts'] if case=='client' else d['direct_user_commpage_shifts'] + require(set(shifts.values())=={12},'Actual native user page ABI') + observations[case]=dict(checkpoint_sha256=sha(path),state=state,actual_tcr=regs['TCR_EL1']) +(OUT/'system-client-checkpoint.json').write_text(json.dumps(dict(candidate_sha256=plan['candidate_sha256'],native_sha256=plan['native_sha256'], + driver_revision=revision,observations=observations,frame_info=frame_info_observation,dot_pitch_checkpoint_sha256=sha(pitch_path),missing_dot_pitch_return=0xe00002f0,actual_original_start=True,actual_system_client_created=True, + original_instructions_modified=False,swap_contract_complete=False,system_metal_device=False,original_desktop=False, + adreno_execution=False,hardware_tested=False),indent=2)+'\n') +print('PASS: real GOP subsystem discovery, original framebuffer start and original system-process userclient creation') +print('Init/release and default-off controls, original AMFI/root/native 4K pass; actual swaps, system Metal and desktop remain open') diff --git a/scripts/check-target-display-discovery-checkpoints.py b/scripts/check-target-display-discovery-checkpoints.py new file mode 100644 index 0000000..01063b8 --- /dev/null +++ b/scripts/check-target-display-discovery-checkpoints.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +"""Qualify actual SpringBoard display replies and backboardd registry discovery.""" +import hashlib +import json +from pathlib import Path +import plistlib + +ROOT = Path(__file__).resolve().parents[1] +expected = 'a4a5fd00dbfb3d41ffe7913cc31b14a2db58e27f5d89129d64018e534b9dc309' +native = 'ba60f353a90b61fb6d63425648aeb4475a2c237ed19800d60d2f583cea035bb3' +image = ROOT / 'build/native-prelink/native-owner-pgz-fixed-candidate.macho' + + +def require(value, message): + if not value: + raise SystemExit(message) + + +require(hashlib.sha256(image.read_bytes()).hexdigest() == expected, 'Exact unchanged native candidate') +for actor, mode, function in (('springboard', 'query', 0x189549038), ('backboard', 'discovery', 0x1b63da630)): + d = json.loads((ROOT / f'build/darwinboot-owner-hvf-{actor}-display-{mode}/gdb-stop.json').read_text()) + q = d['followed_display_query'] + p = q['plan'] + r = {name: int(value, 16) for name, value in d['system_registers'].items()} + require(d['kernel_sha256'] == expected and p['native_sha256'] == native and d['qemu_accelerator'] == 'hvf' and + not d['bounded_timeout_interrupted'] and d['native_4k_mmu_observed'] and + r['TCR_EL1'] >> 14 & 3 == 0 and r['TCR_EL1'] >> 30 & 3 == 2 and r['SCTLR_EL1'] & 1 and + r['TTBR0_EL1'] and r['TTBR1_EL1'] and d['direct_user_commpage_shifts'] == {'fffff4025:1': 12, 'fffff4037:1': 12}, + 'Bounded native-4K hardware and actual user ABI') + require(d['original_launchd_main_executed'] and d['followed_main_entry']['first_instruction_completed'] and + q['observed'] and q['caller_restored'] and q['entry_bytes'] == q['query_bytes_after_fault'] == p['query_prefix'] and + int(q['entry_pstate'], 16) & 31 == 0 and q['entry_stack'] == q['return_stack'] and + q['entry_process'] == q['actual_process'] and + q['actual_process']['proc'] == q['activated_process']['proc'] and + q['actual_process']['task'] == q['activated_process']['task'] and + q['actual_process']['name'] == ('SpringBoard' if mode == 'query' else 'backboardd'), + 'Exact actual process/thread, authenticated original instructions and same-stack completion') + slide = int(q['cache_slide'], 16) + require(q['actual_process']['shared_region_slide'] == slide and slide & 4095 == 0 and + 0 <= slide <= p['max_slide'] and int(q['query'], 16) == function + slide and + p['cache_uuid'] == '590d839a82d230f1a17d0cc053420f73' and + p['local_symbols_uuid'] == 'a1a859a2479c31998a34f4afbd808cf5' and + p['main_import_offset'] == 0x6c0d0 and p['unslid_import'] == 0x1f4843c60 and + p['query'] == function and not p['original_files_modified'], + 'Original cache/local symbols, main binding and actual selected task shared-region slide') + if mode == 'query': + require(any(e.get('result') == 0 and e.get('return_process') == q['activated_process'] for e in q['activation_events']), + 'Real selected-process shared-region syscall returned success before observing its user code') + else: + require(not q['activation_events'] and q['actual_process']['shared_region_slide'] == int(q['launchd_cache_slide'], 16), + 'Actual discovery task region independently confirms the initial address; no invented syscall activation') + require(d['owner_root_binding_audit']['boot_root_accepted'] and + d['owner_root_binding_audit']['_native_owner_root_apple_autherr'] == 80 and + d['owner_boot_manifest']['status'] == 0 and d['original_amfi_table_owner_observed'] and + d['original_crypto_initialized_observed'] and + d['actual_signature_failures'] == dict(_cs_validate_page_bad_hash=0, _cs_validate_page_no_hash=0) and + d['native_virtio_integrity']['_native_virtio_integrity_failures'] == 0 and + not d['functional_ipados_boot_demonstrated'] and not d['sheng_hardware_tested'], + 'Original security checks, real independent owner roots and offline scope') + if mode == 'query': + port, reply = q['checkpoints'] + require(port['stage'] == 'server-port' and int(port['x0'], 16) != 0 and reply['stage'] == 'display-reply' and + int(reply['x0'], 16) == 0 and len(bytes.fromhex(reply['output_bytes'])) == + int.from_bytes(bytes.fromhex(reply['output_length']), 'little') == 42 and + plistlib.loads(bytes.fromhex(reply['output_bytes'])) == [] and + q['main_display_bytes'] == q['displays_bytes'] == '00' * 8 and q['return_pc'] == q['caller'], + 'Actual SpringBoard server request succeeds with a real empty plist array and no main display') + print('PASS: actual SpringBoard CASGetDisplays succeeds with []; original query returns with no main CADisplay') + else: + event, = q['checkpoints'] + require(event['stage'] == 'registry-arm-io' and int(event['x0'], 16) != 0 and + q['discovery_counts']['reference_count'] == q['discovery_counts']['enumerated_count'] == '00' * 4 and + q['discovery_counts']['display_count'] == 'E14' and + int(q['epilogue_pc'], 16) == int(q['query'], 16) + p['return_checkpoint'] and + 'return_pc' not in q, 'Actual backboardd finds arm-io but counts zero display pipes/instances; unreadable length is not zero') + print('PASS: actual backboardd original discovery reaches its RET with zero display pipes/instances; no RET execution claim') + +d = json.loads((ROOT / 'build/darwinboot-owner-hvf-display-exception-process/gdb-stop.json').read_text()) +s = json.loads((ROOT / 'build/native-user-exception/display-process-symbolication.json').read_text()) +require(d['kernel_sha256'] == s['kernel_sha256'] == expected and d['stop_symbol'] == '_exception_triage' and + not d['bounded_timeout_interrupted'] and d['actual_process']['name'] == 'SpringBoard' and + int(d['observed_function_entry_arguments'][0], 16) == 6 and + s['local_symbols_uuid_matches'] and s['frames'][0]['nearest_local_symbol'] == + '-[FBSDisplayMonitor _initWithBookendObserver:transformer:]' and + s['static_original_assertion'] == dict(literal_va='0x1e7a9bf68', text='failed to find the main CADisplay', runtime_string_captured=False), + 'Actual SpringBoard process, original FrontBoard exception and UUID-bound original assertion reference') +print('PASS: original display-monitor exception belongs to real SpringBoard; no functional UI or hardware qualification') diff --git a/scripts/check-target-dyld-checkpoints.py b/scripts/check-target-dyld-checkpoints.py new file mode 100644 index 0000000..9ae4173 --- /dev/null +++ b/scripts/check-target-dyld-checkpoints.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +"""Verify genuine original dyld execution and its native-4K page ABI in QEMU.""" +import hashlib +import json +from pathlib import Path +import struct + +ROOT = Path(__file__).resolve().parents[1] +build = ROOT / 'build' +fixture = build / 'target-dyld-fixture' +inputs = json.loads((fixture / 'inputs.json').read_text()) +candidate = json.loads((build / 'native-prelink/native-target-dyld-candidate.json').read_text()) + + +def require(value, message): + if not value: + raise SystemExit('FAIL: ' + message) + + +for item in inputs['files']: + data = (fixture / Path(item['path']).name).read_bytes() + require(hashlib.sha256(data).hexdigest() == item['sha256'], 'unchanged supplied ' + item['path']) + require(struct.unpack_from('<3I', data) == (0xfeedfacf, 0x100000c, 0), 'ordinary ARM64 target') +root = (fixture / 'root.img').read_bytes() +header = struct.unpack_from('<8sII4Q', root) +require(root[header[3]:header[3] + header[4]] == (fixture / 'launchd').read_bytes() and + root[header[5]:header[5] + header[6]] == (fixture / 'dyld').read_bytes(), 'byte-exact original files in RAM root') +require(hashlib.sha256(root).hexdigest() == inputs['root_sha256'], 'root identity') +trust = (fixture / 'original-static-trust.bundle').read_bytes() +require(trust[:8] == struct.pack('<2I', 1, 8) and hashlib.sha256(trust[8:]).hexdigest() == + inputs['original_static_module_sha256'] == 'da88460e6e513438d5307e5c526f76497a74754476a24da29f239f0c90120fc2', + 'unchanged original static module in loader offset-table container') +require(hashlib.sha256(trust).hexdigest() == inputs['trust_bundle_sha256'], 'trust input identity') +dyld = (fixture / 'dyld').read_bytes() +counts = [] +for name in ('darwinboot-target-dyld-cross-page', 'darwinboot-target-dyld-mach-init'): + d = json.loads((build / name / 'gdb-stop.json').read_text()) + require(d['kernel_sha256'] == candidate['candidate_sha256'] and not d['bounded_timeout_interrupted'], + 'same bounded native kernel/component candidate') + require(d['target_inputs']['root_sha256'] == inputs['root_sha256'] and + d['target_inputs']['trust_bundle_sha256'] == inputs['trust_bundle_sha256'], 'exact original boot inputs') + r = {k: int(v, 16) for k, v in d['system_registers'].items()} + require(r['TCR_EL1'] >> 14 & 3 == 0 and r['TCR_EL1'] >> 30 & 3 == 2 and + r['SCTLR_EL1'] & 1 and r['TTBR0_EL1'] and r['TTBR1_EL1'], 'actual native hardware 4K MMU') + g = d['kernel_observations'] + require((g['_PAGE_SHIFT_CONST'], g['_page_shift'], g['_page_size']) == (12, 12, 4096), 'kernel page ABI') + require(g['_cs_validate_page_bad_hash'] == g['_cs_validate_page_no_hash'] == 0, 'genuine code-page validation') + require(any(e['module'] == 'com.apple.driver.AppleMobileFileIntegrity' and e['segment'] == '__DATA_CONST' + and e['va'] <= g['_amfi'] < e['va'] + e['size'] for e in candidate['extents']), 'original AMFI interface owner') + require(len(d['indirect_memory']) == 2 and all(v['bytes'] == '0c' for v in d['indirect_memory'].values()), + 'actual legacy and read-only commpage user shifts') + e = d['followed_user_entry'] + require(e['observed'] and int(e['pstate'], 16) & 15 == 0, 'actual original dyld entry at EL0') + preferred = inputs['dyld_preferred_entry'] + require(e['entry_bytes'] == dyld[preferred:preferred + 16].hex(), 'unchanged original entry instructions') + slide = int(e['user_entry'], 16) - preferred + require(slide == int(d['original_dyld_slide'], 16), 'observed ASLR slide, not a guessed entry') + checkpoint = e['checkpoint'] + require(checkpoint['observed'] and int(checkpoint['pstate'], 16) & 15 == 0, 'original user checkpoint reached') + confirmed = [] + for step in e['steps']: + at = int(step['pc'], 16) - slide + if int(step['pstate'], 16) & 15 == 0 and 0 <= at < 0x80000 - 16 and step['bytes'] == dyld[at:at + 16].hex(): + confirmed.append(step['pc']) + require(len(set(confirmed)) >= 8 and len(confirmed) == d['original_dyld_el0_instruction_observations'], + 'distinct byte-matched original instructions during real execution') + counts.append(len(confirmed)) + require(not d['target_launchd_main_executed'] and not d['functional_ipados_boot_demonstrated'] and + not d['sheng_hardware_tested'], 'qualified checkpoint scope') + if name.endswith('mach-init'): + require(d['memory']['fffff4025:1'] == d['memory']['fffff4037:1'] == '0c', 'actual original user commpage ABI bytes') + returned = e['function_return'] + require(returned['observed'] and int(returned['pstate'], 16) & 15 == 0 and int(returned['x0'], 16) == 0, + 'genuine original mach_init returned successfully at EL0') + require(d['original_dyld_page_abi'] == dict(_vm_page_size=4096, _vm_page_mask=4095, + _vm_page_shift=12, _vm_kernel_page_shift=12) and d['original_dyld_native_4k_abi_observed'], + 'original target dyld initialized its own native-4K globals') + for symbol, expected in d['original_dyld_page_abi'].items(): + width = 8 if symbol in ('_vm_page_size', '_vm_page_mask') else 4 + request = f'{inputs["dyld_symbols"][symbol] - preferred}:{width:x}' + actual = d['followed_relative_memory'][request] + require(int(actual['address'], 16) == slide + inputs['dyld_symbols'][symbol] and + int.from_bytes(bytes.fromhex(actual['bytes']), 'little') == expected, 'actual target global location/value') + +negative = json.loads((build / 'darwinboot-target-dyld-tamper-refusal/gdb-stop.json').read_text()) +control = negative['negative_input_control'] +require(negative['kernel_sha256'] == candidate['candidate_sha256'] and not negative['bounded_timeout_interrupted'], + 'same bounded target code-page refusal candidate') +require(negative['native_4k_mmu_observed'] and + negative['target_inputs']['trust_bundle_sha256'] == inputs['trust_bundle_sha256'], + 'native hardware and unchanged original trust in refusal control') +require(control['file_offset'] == 0x372b8 and control['before'] ^ control['after'] == 1 and + not control['signatures_or_trust_modified'] and not control['supplied_original_file_modified'], + 'isolated one-bit target code-page fault') +damaged = bytearray(root) +damaged[header[5] + control['file_offset']] ^= 1 +require(hashlib.sha256(damaged).hexdigest() == control['input_copy_sha256'], 'exact negative input identity') +require(negative['kernel_observations']['_cs_validate_page_bad_hash'] == 1 and + negative['target_process_exit'] == dict(namespace=3, code=2) and + not negative['followed_user_entry']['function_return']['observed'], 'genuine CODESIGNING/INVALID_PAGE refusal') + +result = dict(candidate_sha256=candidate['candidate_sha256'], target=inputs['target'], checkpoints=3, + byte_matched_el0_instruction_observations=counts, original_dyld_executed=True, + original_mach_init_returned=True, original_dyld_page_size=4096, + original_files_and_static_module_unchanged=True, hardware_translation_native_4k=True, + corrupt_original_code_page_refused=True, + target_launchd_main_executed=False, functional_ipados_booted=False, + apple_boot_trust_chain_verified=False, sheng_hardware_tested=False) +(build / 'target-dyld-checkpoints.json').write_text(json.dumps(result, indent=2) + '\n') +print('PASS: unchanged original dyld executes at native-4K EL0; genuine mach_init returns with page_size=4096') +print('Original launchd main, shared-cache loading, functional iPadOS and hardware remain unqualified') diff --git a/scripts/check-turnip-darwin-host.py b/scripts/check-turnip-darwin-host.py new file mode 100644 index 0000000..9d3845f --- /dev/null +++ b/scripts/check-turnip-darwin-host.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +"""Drive the Turnip darwin backend through a real Vulkan instance against the host copy of the NativeAdreno kernel stack. + +The kernel half is the real GPU context bring-up, VM, ring, submit and user-client code over software SMMU/CP peers +(tests/hostsim/host_gpu_service.cpp); an IOKit shim answers the backend's IOKit calls. Nothing here runs a shader or touches +hardware: the software CP retires every submission at once. Needs the host Turnip build (build-turnip-darwin.py --target host).""" +import os +from pathlib import Path +import subprocess +import sys + +ROOT = Path(__file__).resolve().parents[1] +kit = ROOT / 'drivers/NativeAdreno' +sim = kit / 'tests/hostsim' +out = ROOT / 'build/turnip-darwin-host' +out.mkdir(parents=True, exist_ok=True) +smmu = ROOT / 'kits/ArmSmmu4K' +library = ROOT / 'build/mesa-tu/src/freedreno/vulkan/libvulkan_freedreno.dylib' +if not library.exists(): + sys.exit('Build the host Turnip first: python3 scripts/build-turnip-darwin.py --target host') + +includes = out / 'host-include' +for name in ['IOKit/IOTypes.h', 'IOKit/IOReturn.h', 'IOKit/IOBufferMemoryDescriptor.h', 'IOKit/IOLib.h', 'kern/task.h']: + path = includes / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text('#include "XnuMemoryFixture.h"\n') + +objects = [] +for stem in ['ArmSmmu4KContext', 'ArmSmmu4KProgram', 'ArmSmmu4KLive', 'ArmSmmu4KStreams', 'ArmSmmu4KMmio']: + obj = out / (stem + '.o') + subprocess.run(['xcrun', 'clang', '-std=c11', '-O1', '-fPIC', '-c', smmu / (stem + '.c'), '-o', obj], check=True) + objects.append(obj) +obj = out / 'AdrenoPM4.o' +subprocess.run(['xcrun', 'clang', '-std=c11', '-O1', '-fPIC', '-c', ROOT / 'kits/AdrenoPM4/AdrenoPM4.c', '-o', obj], check=True) +objects.append(obj) + +service = out / 'host_gpu_service.o' +subprocess.run(['xcrun', 'clang++', '-std=c++23', '-O1', '-fPIC', '-I', includes, '-I', sim.parent, '-I', kit, '-I', smmu, + '-I', ROOT / 'kits/AdrenoPM4', '-c', sim / 'host_gpu_service.cpp', '-o', service], check=True) +core = [] +for name in ['NativeAdrenoMemory', 'NativeAdrenoSmmuMemory', 'NativeAdrenoVm', 'NativeAdrenoRing', 'NativeAdrenoSubmit', + 'NativeAdrenoUserClient', 'NativeAdrenoTimer', 'NativeAdrenoSmmuContext', 'NativeAdrenoGpuContext']: + obj = out / (name + '.o') + subprocess.run(['xcrun', 'clang++', '-std=c++23', '-O1', '-fPIC', '-I', includes, '-I', sim.parent, '-I', kit, '-I', smmu, + '-I', ROOT / 'kits/AdrenoPM4', '-c', kit / (name + '.cpp'), '-o', obj], check=True) + core.append(obj) +shim_obj = out / 'host_iokit_shim.o' +subprocess.run(['xcrun', 'clang++', '-std=c++17', '-O1', '-fPIC', '-c', sim / 'host_iokit_shim.cpp', '-o', shim_obj], check=True) +shim = out / 'libhostiokit.dylib' +subprocess.run(['xcrun', 'clang++', '-dynamiclib', '-o', shim, service, shim_obj, *core, *objects, + '-framework', 'IOKit', '-framework', 'CoreFoundation'], check=True) +smoke = out / 'vulkan_smoke' +subprocess.run(['xcrun', 'clang', '-std=c11', '-O1', '-I', '/opt/homebrew/include', sim / 'vulkan_smoke.c', '-o', smoke, + '-Wl,-rpath,' + str(out)], check=True) + +plugin_smoke = out / 'plugin_smoke' +subprocess.run([sys.executable, ROOT / 'scripts/gen-turnip-smoke-shaders.py'], check=True) +subprocess.run(['xcrun', 'clang++', '-std=c++17', '-O1', '-I', ROOT / 'drivers/NativeMetal', '-I', ROOT / '.cache/mesa-work/include', + '-I', ROOT / 'build/turnip-smoke', + sim / 'plugin_smoke.cpp', '-o', plugin_smoke], check=True) +env = dict(os.environ, DYLD_INSERT_LIBRARIES=str(shim), MESA_LOG='') +env.pop('TU_DARWIN_SERVICE', None) # the default is the production transport: accelerator service, open type 0x4147, tunnel through one admitted selector +result = subprocess.run([str(smoke), str(library)], env=env, capture_output=True, text=True, timeout=300) +(out / 'smoke.log').write_text(result.stdout + result.stderr) +print(result.stdout, end='') +if result.stderr.strip(): + print(result.stderr[-3000:], end='') +if result.returncode or 'PASS' not in result.stdout: + sys.exit('Turnip darwin host smoke failed (exit %d); see %s' % (result.returncode, out / 'smoke.log')) +env_plugin = dict(env) +plugin = subprocess.run([str(plugin_smoke), str(library)], env=env_plugin, capture_output=True, text=True, timeout=300) +(out / 'plugin_smoke.log').write_text(plugin.stdout + plugin.stderr) +print(plugin.stdout, end='') +if plugin.returncode: + sys.exit('The plugin Turnip sequence did not behave as expected on the host (exit %d); see %s' % (plugin.returncode, out / 'plugin_smoke.log')) +# Watchdog: block the first BoAlloc for 17 s; the 15 s sample must name the last marker (6004), show the worker waiting (not running) and +# carry at least one return address inside the Turnip image ('v' frames). +stalled = subprocess.run([str(plugin_smoke), str(library)], env=dict(env_plugin, HOSTSIM_STALL='17'), capture_output=True, text=True, timeout=300) +(out / 'plugin_watchdog.log').write_text(stalled.stdout + stalled.stderr) +lines = [l for l in stalled.stdout.splitlines() if l.startswith('watchdog: W ')] +print('\n'.join(lines)) +if stalled.returncode or len(lines) != 1 or not lines[0].startswith('watchdog: W 6004 ') or ' v' not in lines[0] or 'kern' in lines[0]: + sys.exit('The plugin watchdog did not sample the stalled worker as expected; see %s' % (out / 'plugin_watchdog.log')) +print('PASS: the plugin watchdog sampled a worker blocked in vkCreateDevice (one line, marker 6004, frames inside Turnip)') +print('PASS: the plugin Turnip sequence (fill, compute pipeline, render pass) built, submitted and retired every command buffer over the host kernel stack (the software CP executes nothing, so no result is verified by design)') +print('PASS: Turnip darwin backend drove a Vulkan instance, device, memory mapping and a submission over the host kernel stack; no GPU') diff --git a/scripts/check-ufs-original-trust-evidence.py b/scripts/check-ufs-original-trust-evidence.py new file mode 100644 index 0000000..6c537f4 --- /dev/null +++ b/scripts/check-ufs-original-trust-evidence.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +"""Read-only closeout for the IMG_2974 correction; never issues device commands.""" +import hashlib +import json +from pathlib import Path +import shutil +import subprocess + +ROOT = Path(__file__).resolve().parents[1] +BASE = ROOT / 'build/ufs-root-diagnostics/exec-policy' +TRUST = '7416ca4f8377146626bb284e2133c76bee6cb4e3ee0af5d648b186235a7323a7' + + +def require(value, reason): + if not value: + raise ValueError(reason) + + +def sha(path): + with path.open('rb') as f: + return hashlib.file_digest(f, 'sha256').hexdigest() + + +def load(path): + return json.loads(path.read_text()) + + +def main(): + decision = BASE / 'decision.json' + decision.write_text(json.dumps(dict(offline_preparation_complete=False, device_action_authorized=False))+'\n') + release = load(BASE / 'release.json') + for name, digest in release['source_sha256'].items(): + require(sha(ROOT/name) == digest, 'Changed source: '+name) + for name, digest in release['evidence_sha256'].items(): + require(sha(ROOT/name) == digest, 'Changed evidence: '+name) + previous = load(ROOT/'build/ufs-root-diagnostics/va64/release.json') + ufs = ROOT/'drivers/NativeUfs' + require(subprocess.check_output(['git','-C',str(ufs),'rev-parse','HEAD'],text=True).strip() == + previous['native_ufs_commit'], 'Changed UFS revision') + for name, digest in previous['source_sha256'].items(): + require(sha(ufs/name) == digest, 'Changed UFS implementation: '+name) + require(sha(ROOT/release['candidate']) == previous['candidate_sha256'] == release['candidate_sha256'], + 'Kernel candidate changed') + image = ROOT/release['image'] + require(sha(image) == release['image_sha256'] and image.stat().st_size == release['image_bytes'], 'Wrong image') + package = load(BASE/'package-audit/report.json') + for field in ('image_sha256','candidate_sha256','embedded_kernel_sha256'): + require(package[field] == release[field], 'Wrong package '+field) + old_package = load(ROOT/'build/ufs-root-diagnostics/va64/package-audit/report.json') + for field in ('config_sha256','data_seed_sha256','boot_args','root_uuid','owned_memory'): + require(package[field] == old_package[field], 'Changed configuration: '+field) + require(package['original_trust_cache_sha256'] == TRUST and + package['patch_reproduction'] == 'byte-for-byte match', 'Missing exact original trust/renderer') + negative = load(BASE/'no-trust/gdb-stop.json') + positive = load(BASE/'original-trust/gdb-stop.json') + require(negative['exec_policy_refusal'] == dict(process='launchd',namespace=9,code=8,description='none') and + not negative['bounded_timeout_interrupted'], 'Missing genuine absent-trust refusal') + entry = positive['followed_user_entry'] + require(entry['observed'] and int(entry['pstate'],16)&31 == 0 and + entry['entry_bytes'] == 'e00300911fec7c921d0080d21e0080d2' and + positive['trust_cache_bundle_sha256'] == TRUST, 'Original dyld EL0 entry not observed') + for result in (negative, positive): + require(result['kernel_sha256'] == release['candidate_sha256'] and result['qemu_accelerator']=='tcg' and + not result['offline_qemu_vmm_present_declared'] and result['original_amfi_table_owner_observed'] and + result['owner_root_binding_audit']['boot_root_accepted'] and + result['native_virtio_integrity']['_native_virtio_integrity_failures']==0, + 'Root/AMFI/comparison evidence mismatch') + tcr=int(result['system_registers']['TCR_EL1'],16) + require((tcr>>14&3)==0 and (tcr>>30&3)==2, 'CPU not native 4K') + for name, count in [('renderer-exec',10),('renderer-ufs',28)]: + r=load(BASE/name/'report.json') + require(len(r['matched_lines'])==count and r['renderer_bytes']<=2608 and + not r['hardware_or_xnu_boot_qualified'], 'Renderer incomplete: '+name) + require('PASS:' in (BASE/'trust-refusal.log').read_text(), 'Missing changed-cache refusal') + require(all(r['refused'] and r['stale_report_removed'] for r in load(BASE/'package-refusals.json')), + 'Package refusal failed') + require(shutil.disk_usage(ROOT).free>=5*1024**3, 'Free space below floor') + extended=load(BASE/'original-trust-180/gdb-stop.json') + require(extended['kernel_sha256']==release['candidate_sha256'] and + extended['trust_cache_bundle_sha256']==TRUST, 'Wrong extended observation') + decision.write_text(json.dumps(dict(offline_preparation_complete=True, required_offline_items_remaining=[], + release_sha256=sha(BASE/'release.json'),image_sha256=release['image_sha256'], + device_action_authorized=False,boot_sent=False,hardware_success=False, + original_dyld_el0_observed_offline=True, + original_launchd_main_observed_offline=extended.get('original_launchd_main_executed',False), + extended_observation_timeout=extended['bounded_timeout_interrupted'], + ufs_smmu_mode='unchanged physical bypass',gpu_4k_qualified=False,production_ipados_ready=False),indent=2)+'\n') + print('PASS: original-trust batch complete; required offline items remaining: 0; no device operation') + + +if __name__=='__main__': + main() diff --git a/scripts/check-ufs-preboot-evidence.py b/scripts/check-ufs-preboot-evidence.py new file mode 100644 index 0000000..f59f0dc --- /dev/null +++ b/scripts/check-ufs-preboot-evidence.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python3 +"""Close the finite offline prerequisites for the held UFS diagnostic boot. + +Consume existing observations instead of rerunning unchanged experiments. +No USB, device commands, image creation, or userdata-image reads. Completion +is not permission to boot, hardware success, or production iPadOS readiness. +""" +import argparse +import hashlib +import json +from pathlib import Path +import shutil +import subprocess + +ROOT = Path(__file__).resolve().parents[1] + + +def digest(path): + with path.open('rb') as stream: + return hashlib.file_digest(stream, 'sha256').hexdigest() + + +def require(condition, reason): + if not condition: + raise ValueError(reason) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--evidence', type=Path, default=ROOT/'build/ufs-root-diagnostics') + parser.add_argument('--image', type=Path, required=True) + parser.add_argument('--candidate', type=Path, required=True) + parser.add_argument('--output', type=Path, required=True) + args = parser.parse_args() + # Invalidate an earlier decision before checking changed inputs. An error + # must never leave a stale "complete" file at the requested output path. + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(dict(offline_preparation_complete=False, + status='checking; incomplete unless replaced by a passing decision', + device_action_authorized=False, user_hold_active=True), indent=2)+'\n') + evidence = args.evidence.resolve() + ufs = ROOT/'drivers/NativeUfs' + inputs = {} + gates = [] + + def load(name): + path = evidence/name + inputs[name] = digest(path) + return json.loads(path.read_text()) + + def log_pass(name, marker='PASS:'): + path = evidence/name + inputs[name] = digest(path) + require(marker in path.read_text(), 'Missing successful evidence: '+name) + + def source_hashes(report, root, key='source_sha256'): + for name, expected in report[key].items(): + require(digest(root/name) == expected, 'Evidence source changed: '+name) + + release = load('publication/release.json') + require(args.image.name == release['image'] and args.image.stat().st_size == release['size'], 'Wrong held image') + require(digest(args.image) == release['image_sha256'], 'Held image changed') + require(digest(args.candidate) == release['candidate_sha256'], 'Held candidate changed') + # Later NativeUfs commits add tests only. Do not silently accept a driver + # change just because an old report has a PASS string. + revision = release['native_ufs_commit'] + original_files = subprocess.check_output(['git','-C',str(ufs),'ls-tree','--name-only',revision],text=True).splitlines() + original_files = {n for n in original_files if n.endswith(('.cpp','.h'))} + current_files = {p.name for p in ufs.iterdir() if p.suffix in ('.cpp','.h')} + require(original_files == current_files, 'Production source set changed') + for name in original_files: + original = subprocess.check_output(['git','-C',str(ufs),'show',revision+':'+name]) + require((ufs/name).read_bytes() == original, 'Production source differs from held release: '+name) + gates.append('frozen image, candidate and production source identity') + + for name in ('TransferPoll','RootCommands','SmmuTakeover'): + log_pass('integration/'+name+'-test.log') + old_command = load('integration/release/report.json') + for name in ('RootCommands.h','RootQueue.h','RootTransportLayout.h','TransferPoll.h','tests/RootCommandsBridge.cpp'): + require(digest(ufs/name) == old_command['source_sha256'][name], 'Command fault-test implementation changed') + # The SMMU planner/test and publication coordinator/test are byte-identical + # to their recorded tested revisions; diagnostic formatting is checked later. + for revision, names in [('d3a0ba8', ('SmmuTakeover.h','tests/SmmuTakeover.cpp','tests/TransferPoll.cpp','tests/RootCommands.cpp')), + (release['native_ufs_commit'], ('RootPublication.h','tests/RootPublication.cpp'))]: + for name in names: + require((ufs/name).read_bytes() == subprocess.check_output(['git','-C',str(ufs),'show',revision+':'+name]), + 'Host fault-test source changed: '+name) + gates.append('queue, completion, protocol and SMMU refusal tests') + + protocol = load('publication/qemu/report.json') + source_hashes(protocol, ufs) + source_hashes(protocol, ROOT/'kits/VerifiedRoot', 'verified_root_sha256') + require({r['dma'] for r in protocol['results']} == {'0x40010000','0x8cb5e9000'}, 'Missing modeled DMA addresses') + for result in protocol['results']: + require(all(result[k] == 'pass' for k in ('nop','read_4k','read_128k','bad_lba_no_copy')), 'UFS model failed') + require(all(v == 'pass' for v in result['root_path'].values()), 'Signed root path model failed') + for name in ('RootLayout','RootWindowRead','RootPublication'): + log_pass('publication/host/'+name+'.log') + gates.append('GPT, manifest, owner verification, window reads and publication failure handling') + + arm = load('allocation/arm64-sharing/report.json') + source_hashes(arm, ufs) + require({r['case'] for r in arm['results']} == {'low','high','high-inner-shareable','truncated-high','virtual-as-physical'}, + 'Missing ARM64 case') + require(all(r['passed'] for r in arm['results']), 'ARM64 command case failed') + legacy = load('arm64-command/legacy/report.json') + source_hashes(legacy, ufs) + require(legacy['passed'] and legacy['cases'] == 14, 'Historical descriptor comparison failed') + disassembly = load('arm64-command/held-kernel-audit.json') + require(disassembly['candidate_sha256'] == release['candidate_sha256'], 'Wrong disassembled candidate') + require(digest(evidence/'arm64-command/held-kernel-command.disasm') == disassembly['disassembly_sha256'], 'Disassembly changed') + gates.append('ARM64 cache/MMIO execution, DMA width and held-kernel instruction audit') + + xnu = load('allocation/tcg/gdb-stop.json') + require(xnu['kernel_sha256'] == release['candidate_sha256'] and not xnu['bounded_timeout_interrupted'], 'Wrong/incomplete XNU run') + allocation = xnu['ufs_allocation_exercise'] + require(all(allocation[k] for k in ('passed','registers_restored','debugger_invoked')), 'Allocation exercise incomplete') + require(allocation['source_sha256'] == digest(ROOT/'scripts/qemu-entry/ufs_allocation.py'), 'Allocation observer changed') + require(len(allocation['pages']) == 34 and len(allocation['calls']) == 39 and allocation['contiguous'] == 139264, + 'Incomplete actual allocation observation') + require(allocation['ufs_counters_before'] == allocation['ufs_counters_after'] and + not any(allocation['ufs_counters_after'].values()), 'Unexpected UFS activity') + base = int(allocation['physical'],16) + for index, page in enumerate(allocation['pages']): + require(int(page['pa'],16) == base+index*4096 and page['level'] == 3 and page['ap'] == 0 and + page['accessed'] and page['mair_byte'] == 255 and page['shareability'] == 2, 'Unexpected active DMA PTE') + gates.append('actual XNU allocation, every active 4K DMA PTE and bounds') + + pmgr = load('allocation/tcg/pmgr-checked.json') + require(pmgr['kernel_sha256'] == release['candidate_sha256'] and pmgr['amfi_lookup_qualified'] and + pmgr['mode'] == 'amfi' and pmgr['accelerator'] == 'tcg', 'AMFI checkpoint not qualified') + prelink = json.loads(args.candidate.with_suffix('.json').read_text()) + original = json.loads((args.candidate.parent/'native-owner-stage5-candidate.json').read_text()) + require(prelink['candidate_sha256'] == release['candidate_sha256'] and not prelink['source_inputs_modified'], 'Prelink identity mismatch') + require(len(prelink['original_executable_sha256']) == 29 and + prelink['original_executable_sha256'] == original['original_executable_sha256'], 'Original executable components changed') + log_pass('publication/crossbuild.log', '-Werror build and actual ARM64/iOS 17 object audit passed') + gates.append('strict driver build, original component hashes and original AMFI checkpoint') + + package = load('publication/final-package-audit/report.json') + for field in ('image_sha256','candidate_sha256','embedded_kernel_sha256'): + require(package[field] == release[field], 'Final-container identity mismatch: '+field) + require(package['patch_reproduction'] == 'byte-for-byte match' and package['owned_memory'] == '0x40000000', 'Wrong packaged kernel/window') + require(package['boot_args'] == ['-v','serial=3','debug=0x14e','kt_fixed_zones=120','sheng-pmgr=2','native-ufs-root=1','native-ufs-diagnostics=1'], + 'Unexpected probe/reset/security arguments') + log_pass('publication/audit-controls.log', 'PASS: 15 negative packaging controls') + gates.append('actual Android/FAT/kernel/firmware/ADT/root UUID/Data consistency and refusal controls') + + renderer = load('publication/renderer/report.json') + require(len(renderer['matched_lines']) == 28 and not renderer['hardware_or_xnu_boot_qualified'], 'Incomplete/mislabeled screen observation') + for prefix in ('QUEUE ','FAULT ','UIC ','IO ','Waiting for signed UUID ','TCR_EL1','max_mem'): + require(any(line.startswith(prefix) for line in renderer['matched_lines']), 'Missing screen evidence: '+prefix) + log_pass('publication/renderer.log', 'draws all 28 expected lines') + gates.append('single-screen diagnostic capacity and actual ARM64 renderer') + + require(shutil.disk_usage(ROOT).free >= 5*1024**3, 'Free space below user floor') + report = dict(scope='Offline prerequisites for the frozen UFS diagnostic boot only', + offline_preparation_complete=True, required_offline_items_remaining=[], + device_action_authorized=False, user_hold_active=True, hardware_success=False, + production_ipados_ready=False, boot_sent=False, gates=gates, + image=release['image'], image_sha256=release['image_sha256'], + candidate_sha256=release['candidate_sha256'], evidence_sha256=inputs, + production_revision=release['native_ufs_commit'], + source_revision=subprocess.check_output(['git','rev-parse','HEAD'],cwd=ROOT,text=True).strip(), + checker_sha256=digest(Path(__file__)), + reopen_only_for=['changed candidate/source/configuration', 'new hardware observation', + 'new concrete contradictory evidence', 'changed user objective']) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(report,indent=2)+'\n') + print(f'PASS: {len(gates)} prerequisite groups; required offline items remaining: 0; device hold remains active') + + +if __name__ == '__main__': + main() diff --git a/scripts/check-ufs-va64-evidence.py b/scripts/check-ufs-va64-evidence.py new file mode 100644 index 0000000..6c6e666 --- /dev/null +++ b/scripts/check-ufs-va64-evidence.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +"""Validate the frozen post-IMG_2973 UFS release without USB or repeated tests. + +This consumes a recorded release manifest; it cannot approve changed sources, +replace hardware evidence, authorize a boot, or qualify GPU translation. +""" +import hashlib +import json +from pathlib import Path +import shutil +import subprocess + +ROOT = Path(__file__).resolve().parents[1] +BASE = ROOT / 'build/ufs-root-diagnostics/va64' +UFS = ROOT / 'drivers/NativeUfs' + + +def require(condition, reason): + if not condition: + raise ValueError(reason) + + +def sha(path): + with path.open('rb') as stream: + return hashlib.file_digest(stream, 'sha256').hexdigest() + + +def load(path): + return json.loads(path.read_text()) + + +def main(): + output = BASE / 'decision.json' + output.write_text(json.dumps(dict(offline_preparation_complete=False, + device_action_authorized=False, status='checking')) + '\n') + release = load(BASE / 'release.json') + for name, digest in release['evidence_sha256'].items(): + require(sha(ROOT / name) == digest, 'Changed evidence: ' + name) + for name, digest in release['source_sha256'].items(): + require(sha(UFS / name) == digest, 'Changed driver source: ' + name) + require((UFS / name).read_bytes() == subprocess.check_output( + ['git', '-C', str(UFS), 'show', release['native_ufs_commit'] + ':' + name]), + 'Driver differs from recorded revision: ' + name) + require(subprocess.check_output(['git','-C',str(UFS),'status','--porcelain'],text=True) == '', + 'Driver checkout is dirty') + image = ROOT / release['image'] + candidate = ROOT / release['candidate'] + require(image.stat().st_size == release['image_bytes'] and sha(image) == release['image_sha256'], + 'Wrong frozen boot image') + require(sha(candidate) == release['candidate_sha256'], 'Wrong candidate') + old = load(ROOT / 'build/ufs-root-diagnostics/closeout/decision.json') + for name, digest in old['evidence_sha256'].items(): + require(sha(ROOT / 'build/ufs-root-diagnostics' / name) == digest, 'Historical evidence changed: ' + name) + require(old['offline_preparation_complete'] and not old['required_offline_items_remaining'], + 'Historical closeout incomplete') + retained = load(BASE / 'retained-evidence.json') + for name, digest in retained['unchanged_production_sha256'].items(): + require(sha(UFS / name) == digest, 'Retained evidence source changed: ' + name) + previous = subprocess.check_output(['git','-C',str(UFS),'show', + retained['previous_production_revision'] + ':NativeUfs.cpp'], text=True) + current = (UFS / 'NativeUfs.cpp').read_text() + def allocator(text): + return text.split(' gHost.dma = ',1)[1].split(' uint32_t report[3]',1)[0] + require(allocator(previous) == allocator(current), 'Actual XNU allocation evidence cannot be retained') + arm64 = load(BASE / 'arm64/report.json') + for name, digest in arm64['source_sha256'].items(): + require(sha(UFS / name) == digest, 'ARM64 observation source changed: ' + name) + require(len(arm64['results']) == 5 and all(r['passed'] for r in arm64['results']), 'ARM64 cases incomplete') + require('PASS: SMMU geometry/conflicts, VA64 high-DMA regression' in + (BASE / 'host/SmmuTakeover.log').read_text(), 'SMMU tests failed') + require('-Werror build and actual ARM64/iOS 17 object audit passed' in + (BASE / 'crossbuild.log').read_text(), 'Strict build failed') + prelink = load(candidate.with_suffix('.json')) + original = load(ROOT / 'build/native-prelink/native-owner-stage5-candidate.json') + require(len(prelink['original_executable_sha256']) == 29 and + prelink['original_executable_sha256'] == original['original_executable_sha256'], + 'Original executable components changed') + observed = load(BASE / 'amfi-tcg/gdb-stop.json') + require(observed['kernel_sha256'] == release['candidate_sha256'] and + not observed['bounded_timeout_interrupted'] and int(observed['pc'],16) == 0xfffffff004aaeb54, + 'Wrong actual XNU checkpoint') + pmgr = load(BASE / 'amfi-tcg/pmgr-checked.json') + require(pmgr['kernel_sha256'] == release['candidate_sha256'] and pmgr['amfi_lookup_qualified'] and + pmgr['accelerator'] == 'tcg', 'Wrong AMFI validation') + tcr = int(pmgr['tcr_el1'],16) + require((tcr >> 14 & 3) == 0 and (tcr >> 30 & 3) == 2, 'Active CPU translation is not 4K') + package = load(BASE / 'package-audit/report.json') + for field in ('image_sha256','candidate_sha256','embedded_kernel_sha256'): + require(package[field] == release[field], 'Package identity differs: ' + field) + require(package['patch_reproduction'] == 'byte-for-byte match', 'Kernel patch not reproduced') + previous_package = load(ROOT / 'build/ufs-root-diagnostics/publication/final-package-audit/report.json') + for field in ('data_seed_sha256','config_sha256','boot_args','root_uuid','owned_memory'): + require(package[field] == previous_package[field], 'Unexpected configuration change: ' + field) + # Packaging deliberately generates fresh entropy. The recorded comparison + # of both embedded ADTs must contain only this 64-byte property change. + changes = load(BASE / 'adt-diff.json') + require(len(changes) == 1 and changes[0]['node'] == '/device-tree/chosen' and + changes[0]['key'] == 'random-seed' and + len(bytes.fromhex(changes[0]['old'])) == len(bytes.fromhex(changes[0]['new'])) == 64, + 'Unexpected ADT difference') + require(shutil.disk_usage(ROOT).free >= 5*1024**3, 'Free space below 5 GiB') + result = dict(offline_preparation_complete=True, required_offline_items_remaining=[], + release_sha256=sha(BASE/'release.json'), checker_sha256=sha(Path(__file__)), + native_ufs_commit=release['native_ufs_commit'], image_sha256=release['image_sha256'], + scope='One continuous UFS diagnostic RAM boot, following IMG_2973', + device_action_authorized=False, boot_sent=False, hardware_success=False, + cpu_4k_observation='Actual XNU in TCG; prior device TCR retained separately', + ufs_smmu_mode='Physical bypass; VA64 context readback required; no page tables', + gpu_4k_qualified=False, production_ipados_ready=False, + retained_tests='Unchanged queue/commands/GPT/verification/publication/renderer and historical XNU allocator', + new_tests='SMMU register fixture, ARM64 command cases, strict build, actual XNU AMFI, exact final package') + output.write_text(json.dumps(result,indent=2)+'\n') + print('PASS: frozen VA64 release; required offline items remaining: 0; no hardware command sent') + + +if __name__ == '__main__': + main() diff --git a/scripts/check-verified-block-checkpoints.py b/scripts/check-verified-block-checkpoints.py new file mode 100644 index 0000000..1f89011 --- /dev/null +++ b/scripts/check-verified-block-checkpoints.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +"""Qualify authenticated real block I/O and refusal before byte exposure.""" +import hashlib +import json +from pathlib import Path +import sys + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'kits/NativePrelink')) +import native_prelink as np + + +def require(value, message): + if not value: + raise SystemExit(message) + + +positive = json.loads((ROOT / 'build/darwinboot-verified-block-cache-getuid/gdb-stop.json').read_text()) +negative = json.loads((ROOT / 'build/darwinboot-verified-block-data-refusal/gdb-stop.json').read_text()) +expected = [('native-verified-block-candidate', '36058ec38b3477e7eda6651dd1308432b13afda79783f720e8bf75059a9485b0'), + ('native-verified-block-fault-candidate', 'bc583a1b3e006606f16340d7af95ad966e2566cf33538a10098beaf2b8fbb59b')] +source = (ROOT / '.cache/firmware/kernelcache.macho/kernelcache.release.ipad7.decompressed').read_bytes() +old = np.macho(source) +require(hashlib.sha256(source).hexdigest() == 'f18b4df865405f2a287b84977d7aca745e056d656d9fc9df1662970ce2969de7', 'Require original ordinary-ARM64 target') +policy = (ROOT / 'build/verified-root-systemos/manifest.bin').read_bytes() +key = (ROOT / 'build/verified-root-systemos/trusted-key.bin').read_bytes() +for d, (name, identity) in zip((positive, negative), expected): + plan = json.loads((ROOT / 'build/native-prelink' / (name + '.json')).read_text()) + kernel = (ROOT / 'build/native-prelink' / (name + '.macho')).read_bytes() + meta = np.macho(kernel) + require(d['kernel_sha256'] == plan['candidate_sha256'] == identity == hashlib.sha256(kernel).hexdigest() and + not d['bounded_timeout_interrupted'], 'Require exact bounded real candidates') + exports = np.exports(kernel, meta) + for symbol, wanted in (('_verified_root_manifest', policy), ('_verified_root_key', key)): + address = next(a for a, names in exports.items() if symbol in names) + at = np.backed_offset(meta['segments'], address, len(wanted), len(kernel)) + require(kernel[at:at + len(wanted)] == wanted, 'Both candidates must use the identical actual signed policy and anchor') + for e in plan['extents']: + if e['segment'] != '__TEXT_EXEC': + continue + at = np.backed_offset(meta['segments'], e['va'], e['size'], len(kernel)) + previous = np.backed_offset(old['segments'], e['va'] - plan['slide'], e['size'], len(source)) + require(kernel[at:at + e['size']] == source[previous:previous + e['size']], 'Original executable code must remain unchanged') + regs = {k: int(v, 16) for k, v in d['system_registers'].items()} + require(regs['TCR_EL1'] >> 14 & 3 == 0 and regs['TCR_EL1'] >> 30 & 3 == 2 and + regs['SCTLR_EL1'] & 1 and regs['TTBR0_EL1'] and regs['TTBR1_EL1'], 'Require actual native hardware 4K') + require(d['original_crypto_initialized_observed'] and d['original_crypto_table_owner_observed'] and + d['virtio_block_input']['read_only'] and d['native_virtio_integrity']['_native_virtio_manifest_ready'] == 1, + 'Real crypto, signed policy and read-only backend must qualify before DMA data use') + captures = [v for v in d['indirect_memory'].values() if v['bytes'].startswith(('56524f4f54303100', '56524f4f544b4559'))] + require(len(captures) == 2, 'Capture actual private metadata/key copies') + for c in captures: + t = c['translation'] + require(t['leaf_level'] == 3 and t['leaf_bytes'] == 4096 and t['ap'] == 2 and t['pxn'] and t['uxn'], + 'Actual private policy/anchor copies must have read-only NX native-4K PTEs') +require(positive['original_cache_leaf_executed'] and positive['original_cache_leaf_result'] == 0 and + positive['native_user_4k_abi_observed'] and positive['original_amfi_table_owner_observed'] and + positive['native_apfs_data_mount']['mount_result'] == 0 and + positive['native_apfs_data_mount']['stage'] == 2, 'Original APFS and real original cache SVC must still work') +require(positive['native_virtio_globals']['_native_virtio_read_requests'] == 2035 and + positive['native_virtio_globals']['_native_virtio_read_failures'] == 0 and + positive['native_virtio_integrity']['_native_virtio_verified_blocks'] == 26520 and + positive['native_virtio_integrity']['_native_virtio_integrity_failures'] == 0, + 'Every actual consumed transport span must pass data authentication') +require(positive['code_signing_hash_counters'] == dict(_cs_validate_page_bad_hash=0, _cs_validate_page_no_hash=0) and + positive['native_signature_hash_cache']['_cs_native4k_full_blocks'] == 6562 and + positive['native_virtio_first_sector_sha256'] == '42b1178767bb27611cd1836cdd5779c48bfca8a3e719ec06dc4c8ef78d5e6387', + 'Original data/signatures must retain their real successful checks') +require(negative['function_return_stack_matched'] and int(negative['x0'], 16) == 0xe00002c1 and + negative['native_virtio_integrity']['_native_virtio_integrity_failures'] == 1 and + negative['native_virtio_integrity']['_native_virtio_verified_blocks'] == 0 and + negative['native_virtio_globals']['_native_virtio_ready'] == 0 and + negative['native_virtio_globals']['_native_virtio_read_requests'] == 1 and + negative['native_virtio_globals']['_native_virtio_read_failures'] == 0, + 'Authentic DMA with a changed private snapshot must return actual I/O refusal before publication') +for mode in ('', '-verified', '-verified-fault'): + binding = json.loads((ROOT / ('build/native-virtio-block' + mode) / 'bindings.json').read_text()) + require(binding['virtual_methods_checked'] == 190 and binding['concrete_overrides'] == 25 and + binding['unresolved_original_imports'] == 0, 'All actual concrete storage ABIs must remain qualified') +require('-DNATIVE_VERIFIED_BLOCK_INJECT_FAULT=1' in json.loads((ROOT / 'build/native-virtio-block-verified-fault/compile-command.json').read_text()) and + '-DNATIVE_VERIFIED_BLOCK_INJECT_FAULT=1' not in json.loads((ROOT / 'build/native-virtio-block-verified/compile-command.json').read_text()), + 'The negative source-only snapshot injection must be explicit and absent from the positive build') +print('PASS: owner-authenticated real DMA -> CPU snapshot -> original APFS -> original cache code at native 4K') +print('PASS: one changed DMA snapshot byte returns genuine kIOReturnNotPrivileged before media publication') +print('Checks count repeated authenticated reads, not unique/full runtime coverage; production root and hardware remain open') diff --git a/scripts/check-watchdogd-branches.py b/scripts/check-watchdogd-branches.py new file mode 100644 index 0000000..93323d2 --- /dev/null +++ b/scripts/check-watchdogd-branches.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +"""Execute original watchdogd connection branches offline, with explicit API stubs. +No VM boot, OS-byte patch, signature change, device access or hardware proof. +""" +import argparse, hashlib, json, struct, sys +from pathlib import Path +ROOT=Path(__file__).resolve().parents[1] +a=argparse.ArgumentParser(description=__doc__) +a.add_argument('--watchdogd',type=Path,required=True) +a.add_argument('--python-modules',type=Path,required=True) +a.add_argument('--report',type=Path,required=True) +args=a.parse_args();sys.path[:0]=[str(args.python_modules),str(ROOT/'kits/NativePrelink')] +from unicorn import Uc,UC_ARCH_ARM64,UC_MODE_ARM,UC_HOOK_CODE +from unicorn.arm64_const import UC_ARM64_REG_X0,UC_ARM64_REG_X1,UC_ARM64_REG_X3,UC_ARM64_REG_LR,UC_ARM64_REG_PC,UC_ARM64_REG_SP +from native_prelink import macho,backed_offset +raw=args.watchdogd.read_bytes();sha=hashlib.sha256(raw).hexdigest() +assert sha=='ded4c688e1c2add74acf3e7f9eea5fe2c2c569180ce81492221877944d913a01','Unqualified watchdogd revision' +m=macho(raw) +def word(pc):return struct.unpack_from('>5)&0x7ffff;imm-=0x80000 if imm&0x40000 else 0 + u.mem_write(pc+imm*4,struct.pack('>5)&0x7ffff;imm-=0x80000 if imm&0x40000 else 0 + u.mem_write(0x100008d80+imm*4,struct.pack('\n' + '#include \n' + f'.if ARM_PGSHIFT != {shift}\n.error "page shift"\n.endif\n' + f'.if ((TCR_EL1_BOOT >> 14) & 3) != {tg0}\n.error "TG0"\n.endif\n' + f'.if ((TCR_EL1_BOOT >> 30) & 3) != {tg1}\n.error "TG1"\n.endif\n' + '.data\n.quad TCR_EL1_BOOT\n.quad ARM_PGBYTES\n') + path = output / f'{profile}.S' + path.write_text(text) + command = [compiler, '--target=aarch64-none-elf', '-D__arm64__=1', + '-DASSEMBLER', '-D__ASSEMBLER__', f'-DARM64_BOARD_CONFIG_{profile}', '-ffreestanding', + '-Werror', '-c', '-I', str(source / 'osfmk'), + '-I', str(source / 'pexpert'), '-I', str(source / 'EXTERNAL_HEADERS'), + str(path), '-o', str(output / f'{profile}.o')] + subprocess.run(command, check=True) + binary = output / f'{profile}.bin' + subprocess.run([shutil.which('llvm-objcopy'), '--dump-section', f'.data={binary}', + str(output / f'{profile}.o')], check=True) + tcr, page = struct.unpack('[BCLSTU])(?P\d+)x(?P\d+)\s+' + r'(?:(?Pf5)\s+' + r'\s+'.join('(?P%s)' % (i, TOKEN) for i in range(5, 9)) + + r'(?: p(?P-?\d+)/e(?P\d+))?|' + + r'\s+'.join('(?P%s)' % (i, TOKEN) for i in range(6)) + r'\s+!(?P\d+))') +DETAIL = re.compile(r'(?P[BCLSTU]) FP(?P01|09|17|25)\s+' + r'\s+'.join('(?P%s)' % (i, TOKEN) for i in range(8))) + + +def address(token): + result = 0 + for digit in token: + result = result * 64 + DIGITS.index(digit) + if result >= 1 << 39: + raise ValueError('token exceeds native user address range: ' + token) + return hex(result) + + +def decode(text): + entries = {} + for m in PATTERN.finditer(text): + role, index, count = m['role'], int(m['index']), int(m['count']) + key = (role, index) + entry = entries.setdefault(key, dict(role=role, index=index, count=count)) + if entry['count'] != count: + raise ValueError('mixing different captures for one thread group') + if m['extension']: + entry['frames_5_8'] = [address(m['f%d' % i]) for i in range(5, 9)] + if m['peer'] is not None: + entry.update(peer=int(m['peer']), peer_error=int(m['peer_error'])) + else: + entry.update(pc=address(m['a0']), lr=address(m['a1']), + frames_1_4=[address(m['a%d' % i]) for i in range(2, 6)], error=int(m['error'])) + for m in DETAIL.finditer(text): + key = (m['role'], 1) + entry = entries.setdefault(key, dict(role=m['role'], index=1)) + first = int(m['first']) + entry['frames_%d_%d' % (first, first + 7)] = [address(m['f%d' % i]) for i in range(8)] + return list(entries.values()) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('input', nargs='?', type=Path, help='transcribed screen text; otherwise stdin') + args = parser.parse_args() + results = decode(args.input.read_text() if args.input else sys.stdin.read()) + if not results: + parser.error('no complete UI trace rows found') + print(json.dumps(results, indent=2)) diff --git a/scripts/derive-a740-hwinit.py b/scripts/derive-a740-hwinit.py new file mode 100644 index 0000000..11960aa --- /dev/null +++ b/scripts/derive-a740-hwinit.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +"""Derive the A740 hw_init register table (a6xx_gpu.c hw_init, a6xx_set_hwcg, a6xx_set_cp_protect) from pinned sources. + +Offsets come from the pinned XMLs with exact A7XX variant selection (a6xx.xml first, then adreno_common.xml); the hwcg and +protect values from the pinned a6xx_catalog.c (a740_hwcg, a730_protect). Writes drivers/NativeAdreno/NativeAdrenoA740HwInit.h; +--check compares instead.""" +import re, sys +from pathlib import Path +ROOT = Path(__file__).resolve().parents[1] +SRC = ROOT / '.cache/adreno-source/drivers/gpu/drm/msm' +XML = SRC / 'registers/adreno' +ORDER = ['A2XX', 'A3XX', 'A4XX', 'A5XX', 'A6XX', 'A7XX', 'A8XX'] +OUT = ROOT / 'drivers/NativeAdreno/NativeAdrenoA740HwInit.h' + +def matches(variants): + if variants is None: + return True + for part in variants.split(','): + part = part.strip() + if '-' in part: + left, right = part.split('-', 1) + low = ORDER.index(left) if left else 0 + high = ORDER.index(right) if right else len(ORDER) - 1 + if low <= ORDER.index('A7XX') <= high: + return True + elif part == 'A7XX': + return True + return False + +def resolve(name, files, kind='reg(?:32|64)'): + for f in files: + text = (XML / f).read_text() + hits = set() + for m in re.finditer(r'<(' + kind + r') offset="(0x[0-9a-fA-F]+)" name="' + name + r'"([^>]*)>', text): + v = re.search(r'variants="([^"]+)"', m.group(3)) + if matches(v.group(1) if v else None): + hits.add(int(m.group(2), 16)) + if len(hits) > 1: + raise SystemExit('ambiguous ' + name + ' in ' + f) + if hits: + return hits.pop() + raise SystemExit('missing ' + name) + +GPU = ['GBIF_HALT', 'RBBM_GBIF_HALT', 'RBBM_SECVID_TSB_CNTL', 'RBBM_SECVID_TSB_TRUSTED_BASE', + 'RBBM_SECVID_TSB_TRUSTED_SIZE', 'RBBM_GBIF_CLIENT_QOS_CNTL', 'UCHE_GBIF_GX_CONFIG', + 'RBBM_PERFCTR_GPU_BUSY_MASKED', 'UCHE_TRAP_BASE', 'UCHE_WRITE_THRU_BASE', 'UCHE_GMEM_RANGE_MIN', + 'UCHE_GMEM_RANGE_MAX', 'UCHE_CACHE_WAYS', 'CP_AHB_CNTL', 'RBBM_PERFCTR_CNTL', 'RBBM_INTERFACE_HANG_INT_CNTL', + 'UCHE_CLIENT_PF', 'CP_PROTECT_CNTL', 'UCHE_CMDQ_CONFIG', 'CP_APRIV_CNTL', 'CP_BV_APRIV_CNTL', + 'CP_LPAC_APRIV_CNTL', 'RB_CMP_DBG_ECO_CNTL', 'RBBM_INT_0_MASK', 'CP_SQE_INSTR_BASE', 'CP_RB_BASE', + 'CP_RB_CNTL', 'CP_RB_RPTR_ADDR', 'CP_BV_RB_RPTR_ADDR', 'CP_SQE_CNTL', 'CP_RB_RPTR', 'CP_RB_WPTR', + 'RBBM_STATUS', 'RBBM_INT_0_STATUS', 'CP_HW_FAULT', 'CP_INTERRUPT_STATUS', 'RBBM_CLOCK_CNTL', + 'RBBM_SECVID_TRUST_CNTL', 'RBBM_STATUS3', 'CP_SQE_STAT_ADDR', 'CP_SQE_STAT_DATA', + 'RB_NC_MODE_CNTL', 'TPL1_NC_MODE_CNTL', 'SP_NC_MODE_CNTL', 'CP_APERTURE_CNTL_HOST', 'GRAS_NC_MODE_CNTL', + 'UCHE_MODE_CNTL', 'RBBM_NC_MODE_CNTL', 'RBBM_INT_CLEAR_CMD', 'CP_ALWAYS_ON_COUNTER'] +GMU = ['GPU_GMU_AO_GMU_CGC_MODE_CNTL', 'GPU_GMU_AO_GMU_CGC_DELAY_CNTL', 'GPU_GMU_AO_GMU_CGC_HYST_CNTL'] + +def main(): + catalog = (SRC / 'adreno/a6xx_catalog.c').read_text() + defines = [(n, resolve(n, ['a6xx.xml', 'adreno_common.xml'])) for n in GPU] + defines += [(n, resolve(n, ['a6xx_gmu.xml'])) for n in GMU] + defines.append(('CP_PROTECT', resolve('CP_PROTECT', ['a6xx.xml'], 'array'))) + block = re.search(r'a740_hwcg\[\]\s*=\s*\{(.*?)\{\},\s*\};', catalog, re.S).group(1) + hwcg = [] + for name, value in re.findall(r'\{\s*REG_A[67]XX_([A-Z0-9_]+),\s*(0x[0-9a-fA-F]+)\s*\}', block): + hwcg.append((name, resolve(name, ['a6xx.xml', 'adreno_common.xml']), int(value, 16))) + if len(hwcg) != block.count('REG_'): + raise SystemExit('unparsed a740_hwcg entry') + pblock = re.search(r'a730_protect_regs\[\]\s*=\s*\{(.*?)\};', catalog, re.S).group(1) + pblock = re.sub(r'/\*.*?\*/', '', pblock, flags=re.S) + protect = [] + for entry in [e.strip() for e in pblock.split(',\n')]: + entry = entry.strip().rstrip(',') + if not entry: + continue + if entry == '0': + protect.append(0) + continue + m = re.fullmatch(r'A6XX_PROTECT_(NORDWR|RDONLY)\((0x[0-9a-fA-F]+),\s*(0x[0-9a-fA-F]+)\)', entry) + if not m: + raise SystemExit('unparsed protect entry ' + entry) + reg, length = int(m.group(2), 16), int(m.group(3), 16) + protect.append(((1 << 31) if m.group(1) == 'NORDWR' else 0) | ((length & 0x3fff) << 18) | (reg & 0x3ffff)) + count_max = int(re.search(r'DECLARE_ADRENO_PROTECT\(a730_protect,\s*(\d+)\)', catalog).group(1)) + # a6xx_set_cp_protect: entries 0..count-2 at their index (zeros skipped), the last one at count_max-1. + writes = [(i, v) for i, v in enumerate(protect[:-1]) if v] + [(count_max - 1, protect[-1])] + lines = ['/* SPDX-License-Identifier: MIT */', '#ifndef NATIVE_ADRENO_A740_HWINIT_H', '#define NATIVE_ADRENO_A740_HWINIT_H', + '#include ', + '/* Generated by scripts/derive-a740-hwinit.py from the pinned a6xx.xml/adreno_common.xml/a6xx_gmu.xml (A7XX', + ' * variants) and a6xx_catalog.c (a740_hwcg, a730_protect). Dword register offsets. Do not edit by hand. */'] + for name, offset in defines: + lines.append('#define A740_%s 0x%05x' % (name, offset)) + lines.append('static const uint32_t NativeAdrenoA740Hwcg[][2]={') + lines += [' {0x%05x,0x%08x}, /* %s */' % (o, v, n) for n, o, v in hwcg] + lines.append('};') + lines.append('#define NATIVE_ADRENO_A740_HWCG_COUNT %d' % len(hwcg)) + lines.append('/* {CP_PROTECT index, value} in a6xx_set_cp_protect order. */') + lines.append('static const uint32_t NativeAdrenoA740Protect[][2]={') + lines += [' {%2d,0x%08x},' % w for w in writes] + lines.append('};') + lines.append('#define NATIVE_ADRENO_A740_PROTECT_COUNT %d' % len(writes)) + lines.append('#endif') + text = '\n'.join(lines) + '\n' + if '--check' in sys.argv[1:]: + if OUT.read_text() != text: + raise SystemExit('NativeAdrenoA740HwInit.h is stale') + print('PASS: A740 hw_init table matches the pinned sources') + return + OUT.write_text(text) + print('PASS: %d registers, %d hwcg entries, %d protect writes' % (len(defines), len(hwcg), len(writes))) + +main() diff --git a/scripts/derive-a740-pwrup.py b/scripts/derive-a740-pwrup.py new file mode 100644 index 0000000..8144ffc --- /dev/null +++ b/scripts/derive-a740-pwrup.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +"""Derive the A740 preemption register list from pinned sources only.""" +import hashlib, json, re, sys +from pathlib import Path +ROOT = Path(__file__).resolve().parents[1] +XML = ROOT / '.cache/adreno-source/drivers/gpu/drm/msm/registers/adreno' +CATALOG = ROOT / '.cache/adreno-source/drivers/gpu/drm/msm/adreno/a6xx_catalog.c' +ORDER = ['A2XX', 'A3XX', 'A4XX', 'A5XX', 'A6XX', 'A7XX', 'A8XX'] +TARGET = 'A7XX' + +def matches(variants): + if variants is None: + return True + if '-' in variants: + left, right = variants.split('-', 1) + low = ORDER.index(left) if left else 0 + high = ORDER.index(right) if right else len(ORDER) - 1 + return low <= ORDER.index(TARGET) <= high + return variants == TARGET + +def main(): + check_only = '--check' in sys.argv[1:] + text = CATALOG.read_text() + block = re.search(r'a7xx_pwrup_reglist_regs\[\]\s*=\s*\{(.*?)\};', text, re.S) + if not block: + raise SystemExit('A740 preemption list absent from the pinned catalog') + catalog = [] + for entry in block.group(1).split(','): + entry = entry.strip() + if not entry: + continue + found = re.fullmatch(r'REG_(?:A[0-9]XX_)?([A-Z0-9_]+?)(?:\s*\+\s*(\d+))?', entry) + if not found: + raise SystemExit('Unsupported catalog entry: ' + entry) + catalog.append((found.group(1), int(found.group(2) or 0))) + xmls = {(XML / 'a6xx.xml').read_text(): 'a6xx.xml', + (XML / 'adreno_common.xml').read_text(): 'adreno_common.xml'} + resolved = [] + for name, plus in catalog: + hits = [] + for source, filename in xmls.items(): + for m in re.finditer(r']*)>', source): + variant = re.search(r'variants="([^"]+)"', m.group(3)) + if matches(variant.group(1) if variant else None): + hits.append((int(m.group(2), 16), m.group(1), filename)) + offsets = {h[0] for h in hits} + if len(offsets) != 1: + raise SystemExit('Ambiguous or missing ' + name + ': ' + repr(hits)) + resolved.append(offsets.pop() + plus) + if check_only: + current = (ROOT / 'drivers/NativeAdreno/NativeAdrenoPwrupRegs.h').read_text() + expected = [int(v, 16) for v in re.findall(r'0x([0-9a-fA-F]{8}),', current)] + if expected != resolved: + raise SystemExit('A740 power-up register table is stale: ' + repr((expected, resolved))) + print('PASS: pinned catalogue resolves to the committed %d A740 power-up registers' % len(resolved)) + return + header = '''/* SPDX-License-Identifier: MIT */ +#ifndef NATIVE_ADRENO_PWRUP_REGS_H +#define NATIVE_ADRENO_PWRUP_REGS_H +#include +/* Generated by scripts/derive-a740-pwrup.py from the pinned + * a6xx_catalog.c list and the pinned XMLs, selecting only entries whose + * variants include A7XX. Do not edit by hand. */ +static const uint32_t NativeAdrenoA740PwrupRegs[]={ +''' + ''.join(' 0x%08x,%s\n' % (v, '' if i % 4 != 3 else '') for i, v in enumerate(resolved)) + '''}; +#define NATIVE_ADRENO_A740_PWRUP_COUNT \\ + (sizeof(NativeAdrenoA740PwrupRegs)/sizeof(NativeAdrenoA740PwrupRegs[0])) +#endif +''' + (ROOT / 'drivers/NativeAdreno/NativeAdrenoPwrupRegs.h').write_text(header) + report = dict(catalog=[{'name': n, 'plus': p} for n, p in catalog], + resolved=resolved, count=len(resolved), + xml_sha256={f: hashlib.sha256((XML / f).read_bytes()).hexdigest() + for f in ('a6xx.xml', 'adreno_common.xml')}, + catalog_sha256=hashlib.sha256(CATALOG.read_bytes()).hexdigest()) + out = ROOT / 'build/adreno-pwrup'; out.mkdir(parents=True, exist_ok=True) + (out / 'report.json').write_text(json.dumps(report, indent=2) + '\n') + print('PASS: %d A740 preemption registers resolved with exact A7XX variant selection' % len(resolved)) + +if __name__ == '__main__': + main() diff --git a/scripts/extract-kdk-support.py b/scripts/extract-kdk-support.py new file mode 100755 index 0000000..ce3c9f9 --- /dev/null +++ b/scripts/extract-kdk-support.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +"""Stream a verified KDK PBZX payload, extracting only selected reference files.""" +import argparse +import contextlib +import sys +import lzma +from pathlib import Path +import struct +import subprocess +import time + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('payload', type=Path, help='PBZX file, or - for a streamed package payload') + parser.add_argument('--include', action='append', help='bsdtar include pattern; default: kernel support archives') + parser.add_argument('--output', type=Path, default=Path('.cache/kdk-root')) + args = parser.parse_args() + args.output.mkdir(parents=True, exist_ok=True) + started = time.monotonic() + patterns = args.include or ['*KernelSupport/*.a'] + command = ['bsdtar', '-x', '-f', '-', '--no-same-owner', '-C', str(args.output)] + for pattern in patterns: + command.extend(['--include', pattern]) + process = subprocess.Popen(command, stdin=subprocess.PIPE) + count = 0 + total = 0 + try: + with (contextlib.nullcontext(sys.stdin.buffer) if str(args.payload) == '-' + else args.payload.open('rb')) as source: + if source.read(4) != b'pbzx': + raise ValueError('Expected PBZX payload') + maximum = struct.unpack('>Q', source.read(8))[0] + if maximum != 16 * 1024**2: + raise ValueError('Unexpected PBZX chunk size') + while header := source.read(16): + if len(header) != 16: + raise ValueError('Truncated chunk header') + expected, compressed = struct.unpack('>QQ', header) + if expected > maximum or compressed > 2 * maximum: + raise ValueError('Invalid PBZX chunk extent') + block = source.read(compressed) + if len(block) != compressed: + raise ValueError('Truncated PBZX block') + if block.startswith(b'\xfd7zXZ\0'): + block = lzma.decompress(block, memlimit=256 * 1024**2) + if len(block) != expected: + raise ValueError('Decompressed PBZX chunk has wrong length') + process.stdin.write(block) + total += len(block) + count += 1 + process.stdin.close() + if process.wait() != 0: + raise RuntimeError('KDK support archive extraction failed') + finally: + if process.poll() is None: + process.terminate() + process.wait(timeout=5) + files = [path for path in args.output.rglob('*') if path.is_file()] + if not files: + raise SystemExit('No selected KDK files were extracted') + print(f'Selected output contains {len(files)} files from {count} chunks ({total} streamed bytes) ' + f'in {time.monotonic() - started:.1f}s') diff --git a/scripts/extract-target-volume.py b/scripts/extract-target-volume.py new file mode 100644 index 0000000..b831786 --- /dev/null +++ b/scripts/extract-target-volume.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +"""Extract one manifest-selected volume from the supplied iPad7,1 iPadOS 17 IPSW.""" +import argparse +import hashlib +import json +from pathlib import Path +import plistlib +import shutil +import zipfile + +ROOT = Path(__file__).resolve().parents[1] +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument('component', choices=['OS', 'RestoreRamDisk', 'Cryptex1,SystemOS', 'Cryptex1,AppOS', 'StaticTrustCache', 'Cryptex1,SystemTrustCache']) +parser.add_argument('--ipsw', type=Path, default=ROOT / 'refer/iPad_Pro_HFR_17.7.11_21H461_Restore.ipsw') +args = parser.parse_args() +out = ROOT / '.cache/firmware/target-volumes' +out.mkdir(parents=True, exist_ok=True) +with zipfile.ZipFile(args.ipsw) as archive: + manifest = plistlib.loads(archive.read('BuildManifest.plist')) + if (manifest['ProductVersion'] != '17.7.11' or manifest['ProductBuildVersion'] != '21H461' + or 'iPad7,1' not in manifest['SupportedProductTypes']): + raise SystemExit('Use only the supplied iPad7,1 iPadOS 17.7.11 / 21H461 input') + identity = next(item for item in manifest['BuildIdentities'] + if item['Info']['DeviceClass'] == 'j120ap' and item['Info']['RestoreBehavior'] == 'Erase' + and int(item['ApChipID'], 16) == 0x8011 and int(item['ApBoardID'], 16) == 12) + component = identity['Manifest'][args.component] + member = component['Info']['Path'] + entry = archive.getinfo(member) + destination = out / Path(member).name + if destination.exists(): + raise SystemExit(f'Input already exists; inspect its saved hash before replacing: {destination.name}') + if shutil.disk_usage(out).free - entry.file_size < 25 * 1024 ** 3: + raise SystemExit('Extraction would leave less than 5 GiB free') + temporary = destination.with_suffix('.partial') + digest = hashlib.sha256() + count = 0 + try: + with archive.open(entry) as source, temporary.open('xb') as target: + while chunk := source.read(8 * 1024 * 1024): + target.write(chunk) + digest.update(chunk) + count += len(chunk) + if count != entry.file_size: + raise SystemExit('Incomplete volume extraction') + temporary.rename(destination) + finally: + temporary.unlink(missing_ok=True) + report = {'product': 'iPad7,1', 'board': 'j120ap', 'version': manifest['ProductVersion'], + 'build': manifest['ProductBuildVersion'], 'identity': 'Erase', 'component': args.component, + 'member': member, 'size': count, 'sha256': digest.hexdigest(), + 'zip_crc_verified': True, 'mounted': False, 'userspace_boot_demonstrated': False} + destination.with_suffix('.json').write_text(json.dumps(report, indent=2) + '\n') + print(json.dumps(report, indent=2)) diff --git a/scripts/fetch-adreno-inputs.py b/scripts/fetch-adreno-inputs.py new file mode 100644 index 0000000..be9d5f9 --- /dev/null +++ b/scripts/fetch-adreno-inputs.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +"""Fetch only SHA-pinned Adreno source and firmware; never execute firmware.""" +import base64 +import concurrent.futures +import hashlib +import json +from pathlib import Path +import urllib.request + +ROOT = Path(__file__).resolve().parents[1] +pins = json.loads((ROOT / 'scripts/data/adreno-inputs.json').read_text()) + + +def fetch(item): + path, url, expected, blob, encoding = item + if path.exists(): + data = path.read_bytes() + else: + request = urllib.request.Request(url, headers={'User-Agent': 'sheng-offline-adreno-audit'}) + with urllib.request.urlopen(request, timeout=30) as response: + data = response.read() + if blob: + data = base64.b64decode(json.loads(data)['content']) + elif encoding == 'gitiles-base64': + data = base64.b64decode(data, validate=False) + if hashlib.sha256(data).hexdigest() != expected['sha256'] or (expected.get('bytes') and len(data) != expected['bytes']): + raise ValueError('Pinned input hash/size differs: ' + path.name) + if blob and hashlib.sha1(('blob ' + str(len(data)) + '\0').encode() + data).hexdigest() != blob: + raise ValueError('Pinned Git blob identity differs: ' + path.name) + if not path.exists(): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + return len(data) + + +if __name__ == '__main__': + jobs = [] + linux = pins['linux'] + for name, expected in linux['files'].items(): + url = linux['repository'].replace('https://github.com/', 'https://raw.githubusercontent.com/') + '/' + linux['commit'] + '/' + name + jobs.append((ROOT / '.cache/adreno-source' / name, url, expected, None, None)) + mesa = pins['mesa'] + for name, expected in mesa['files'].items(): + url = mesa['mirror'] + '/+/' + mesa['commit'] + '/' + name + '?format=TEXT' + jobs.append((ROOT / '.cache/adreno-source/mesa' / name, url, expected, None, 'gitiles-base64')) + firmware = pins['firmware'] + for name, expected in firmware['files'].items(): + blob = expected['git_blob_sha1'] + url = firmware['repository'].replace('https://github.com/', 'https://api.github.com/repos/') + '/git/blobs/' + blob + jobs.append((ROOT / '.cache/firmware/sheng-gpu' / name, url, expected, blob, None)) + with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor: + sizes = list(executor.map(fetch, jobs)) + print(f'PASS: {len(sizes)} exact source/firmware inputs, {sum(sizes)} bytes; no GPU execution/authenticity claim') diff --git a/scripts/gen-turnip-replay.py b/scripts/gen-turnip-replay.py new file mode 100644 index 0000000..664915c --- /dev/null +++ b/scripts/gen-turnip-replay.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +"""Record Turnip's first submission (the plugin's fill) on the host simulation and embed it for a prefix replay on the tablet. + +The host plugin smoke (scripts/check-turnip-darwin-host.py builds it) runs the same Mesa source as the tablet's Turnip against the host +copy of the kernel stack, with FD_RD_DUMP writing every command-stream buffer. The first submission's buffers (GPU address, size, contents) +and its single IB are written to build/turnip-smoke/TurnipReplay.h together with the dword offset after every TYPE7 packet, so the plugin +can submit growing prefixes of the IB and report the first one the CP does not finish. Generated, never committed.""" +import glob +import gzip +import os +from pathlib import Path +import shutil +import struct +import subprocess +import sys + +ROOT = Path(__file__).resolve().parents[1] +HOST = ROOT / 'build/turnip-darwin-host' +OUT = ROOT / 'build/turnip-replay' +HEADER = ROOT / 'build/turnip-smoke/TurnipReplay.h' +RD_GPUADDR, RD_BUFFER_CONTENTS, RD_CMDSTREAM_ADDR = 3, 12, 6 + +if not (HOST / 'plugin_smoke').exists(): + subprocess.run([sys.executable, ROOT / 'scripts/check-turnip-darwin-host.py'], check=True) +shutil.rmtree(OUT, ignore_errors=True) +OUT.mkdir(parents=True) +env = dict(os.environ, DYLD_INSERT_LIBRARIES=str(HOST / 'libhostiokit.dylib'), MESA_LOG='', TU_DEBUG='nocb', FD_RD_DUMP='enable', HOSTSIM_PAGE_SIZE='4096', + FD_RD_DUMP_PATH=str(OUT)) +env.pop('TU_DARWIN_SERVICE', None) +subprocess.run([str(HOST / 'plugin_smoke'), str(ROOT / 'build/mesa-tu/src/freedreno/vulkan/libvulkan_freedreno.dylib')], env=env, + check=True, capture_output=True, timeout=300) +first = sorted(glob.glob(str(OUT / '*submit00000.rd'))) +if not first: + raise SystemExit('no rd dump of the first submission') +data = gzip.decompress(Path(first[0]).read_bytes()) +buffers, streams, at = [], [], 0 +while at + 8 <= len(data): + kind, size = struct.unpack_from('> 28 == 7: + count = header & 0x3fff + opcode = (header >> 16) & 0x7f + k += 1 + count + if opcode in (COND_EXEC, COND_REG_EXEC) and count: + covered = max(covered, k + (words[k - 1] & 0xffffff)) + if k >= covered: + cuts.append(k) + ops.append(opcode) + elif header >> 28 == 4: + k += 1 + (header & 0x7f) + else: + raise SystemExit('unknown packet header %08x at dword %d' % (header, k)) +if cuts[-1] != len(words): + cuts.append(len(words)) + ops.append(0) + +lines = ['// Generated by scripts/gen-turnip-replay.py from a host rd dump of Turnip\'s first submission. Not committed.', '#pragma once', + '#include ', 'namespace turnipreplay {', + 'struct Buffer {uint64_t iova; uint64_t bytes; uint32_t words; const uint32_t *data;};'] +for index, (iova, length, contents) in enumerate(buffers): + used = len(contents.rstrip(b'\0')) + used = (used + 3) // 4 + values = struct.unpack_from('<%dI' % used, contents) if used else () + lines.append('static const uint32_t kData%d[] = {%s};' % (index, ','.join('0x%x' % v for v in values) or '0')) +lines.append('static const Buffer kBuffers[] = {%s};' % ','.join( + '{0x%xull, 0x%xull, %d, kData%d}' % (iova, (length + 4095) & ~4095, (len(contents.rstrip(b'\0')) + 3) // 4, i) + for i, (iova, length, contents) in enumerate(buffers))) +lines.append('static const uint64_t kIbIova = 0x%xull;' % ib) +lines.append('static const uint32_t kCuts[] = {%s};' % ','.join(map(str, cuts))) +lines.append('static const uint8_t kCutOps[] = {%s};' % ','.join('0x%x' % o for o in ops)) +lines.append('} // namespace turnipreplay') +HEADER.parent.mkdir(parents=True, exist_ok=True) +HEADER.write_text('\n'.join(lines) + '\n') +print('PASS: Turnip first submission: %d buffers, IB 0x%x (%d dwords), %d prefixes -> %s' % (len(buffers), ib, len(words), len(cuts), + HEADER.relative_to(ROOT))) diff --git a/scripts/gen-turnip-smoke-shaders.py b/scripts/gen-turnip-smoke-shaders.py new file mode 100644 index 0000000..ec80104 --- /dev/null +++ b/scripts/gen-turnip-smoke-shaders.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +"""Compile the Turnip smoke shaders (drivers/NativeMetal/shaders, GLSL) to SPIR-V with glslc, validate them with spirv-val and write +build/turnip-smoke/NativeTurnipShaders.h (word arrays). Build product only; the GLSL is the source.""" +from pathlib import Path +import struct +import subprocess +import tempfile + +ROOT = Path(__file__).resolve().parents[1] +OUT = ROOT / 'build/turnip-smoke' +OUT.mkdir(parents=True, exist_ok=True) +lines = ['// Generated by scripts/gen-turnip-smoke-shaders.py from drivers/NativeMetal/shaders; do not edit.', '#pragma once', '#include '] +with tempfile.TemporaryDirectory() as tmp: + for source in sorted((ROOT / 'drivers/NativeMetal/shaders').glob('smoke_*')): + spv = Path(tmp) / (source.name + '.spv') + subprocess.run(['glslc', '--target-env=vulkan1.1', '-O', str(source), '-o', str(spv)], check=True) + subprocess.run(['spirv-val', '--target-env', 'vulkan1.1', str(spv)], check=True) + data = spv.read_bytes() + words = struct.unpack('<%dI' % (len(data) // 4), data) + name = 'kSpv_' + source.name.replace('.', '_') + lines.append('static const uint32_t %s[] = {' % name) + for i in range(0, len(words), 8): + lines.append(' ' + ', '.join('0x%08x' % w for w in words[i:i + 8]) + ',') + lines.append('};') +(OUT / 'NativeTurnipShaders.h').write_text('\n'.join(lines) + '\n') +print('PASS: smoke shaders compiled and validated ->', OUT / 'NativeTurnipShaders.h') diff --git a/scripts/inspect-ipsw.py b/scripts/inspect-ipsw.py new file mode 100755 index 0000000..f2ad6d5 --- /dev/null +++ b/scripts/inspect-ipsw.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +"""Inspect a local restore image without extracting its root filesystem.""" +import argparse +import hashlib +import json +from pathlib import Path +import plistlib +import shutil +import zipfile + + +def inspect(path, extract=False, output=Path('.cache/firmware')): + with zipfile.ZipFile(path) as archive: + manifest = plistlib.loads(archive.read('BuildManifest.plist')) + if 'iPad7,1' not in manifest['SupportedProductTypes']: + raise ValueError('IPSW does not support iPad7,1') + if not manifest['ProductVersion'].startswith('17.'): + raise ValueError('IPSW is not iPadOS 17') + identity = next(b for b in manifest['BuildIdentities'] + if b['Info']['DeviceClass'] == 'j120ap' + and b['Info']['RestoreBehavior'] == 'Erase') + report = { + 'version': manifest['ProductVersion'], + 'build': manifest['ProductBuildVersion'], + 'products': manifest['SupportedProductTypes'], + 'board': identity['Info']['DeviceClass'], + 'chip_id': identity['ApChipID'], + 'board_id': identity['ApBoardID'], + 'components': {}, + } + names = {kind: identity['Manifest'][kind]['Info']['Path'] + for kind in ('KernelCache', 'DeviceTree', 'RestoreRamDisk', 'OS')} + needed = sum(archive.getinfo(names[k]).file_size + for k in ('KernelCache', 'DeviceTree')) + if extract and shutil.disk_usage('.').free - needed < 5 * 1024**3: + raise OSError('Extraction would leave less than 5 GiB free') + for kind, name in names.items(): + entry = {'member': name, 'size': archive.getinfo(name).file_size} + if kind in ('KernelCache', 'DeviceTree'): + data = archive.read(name) + entry['sha256'] = hashlib.sha256(data).hexdigest() + entry['magic_hex'] = data[:16].hex() + if extract: + output.mkdir(parents=True, exist_ok=True) + destination = output / Path(name).name + destination.write_bytes(data) + entry['extracted'] = destination.as_posix() + report['components'][kind] = entry + return report + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('ipsw', type=Path) + parser.add_argument('--extract', action='store_true') + parser.add_argument('--output', type=Path, default=Path('.cache/firmware')) + args = parser.parse_args() + print(json.dumps(inspect(args.ipsw, args.extract, args.output), indent=2)) diff --git a/scripts/inspect-prelink-callsite.py b/scripts/inspect-prelink-callsite.py new file mode 100644 index 0000000..6a5b5b8 --- /dev/null +++ b/scripts/inspect-prelink-callsite.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +"""Decode an observed prelinked callsite and resolve real kernel/GOT targets.""" +import argparse +from pathlib import Path +import shutil +import struct +import subprocess +import sys + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'kits/NativePrelink')) +from native_prelink import backed_offset, exports, macho + +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument('candidate', type=Path) +parser.add_argument('--at', required=True, type=lambda value: int(value, 0)) +parser.add_argument('--bytes', type=lambda value: int(value, 0), default=256) +args = parser.parse_args() +if args.at % 4 or args.bytes % 4 or not 0 < args.bytes <= 4096: + parser.error('Use aligned AArch64 instructions and at most 4096 bytes') +tool = shutil.which('llvm-mc') +if not tool: + raise SystemExit('llvm-mc is required on PATH') +data = args.candidate.read_bytes() +image = macho(data) +symbols = exports(data, image) + + +def read(va, size): + at = backed_offset(image['segments'], va, size, len(data)) + return data[at:at + size] + + +def signed(value, bits): + return value - (1 << bits) if value & (1 << (bits - 1)) else value + + +def adrp(word, pc): + return (pc & ~4095) + (signed(((word >> 5) & 0x7ffff) << 2 | ((word >> 29) & 3), 21) << 12) + + +def literal(va): + for segment in image['segments']: + if segment['va'] <= va < segment['va'] + segment['filesize']: + raw = read(va, min(180, segment['va'] + segment['filesize'] - va)).split(b'\0', 1)[0] + if raw and all(byte in (9, 10, 13) or 32 <= byte < 127 for byte in raw): + return repr(raw.decode()) + return '' + + +def branch_target(va): + if va in symbols: + return ','.join(symbols[va]) + try: + first, second, third = struct.unpack('<3I', read(va, 12)) + if (first & 0x9f000000 == 0x90000000 and second & 0xffc00000 == 0xf9400000 + and third & 0xfffffc1f == 0xd61f0000 and first & 31 == (second >> 5) & 31 + and second & 31 == (third >> 5) & 31): + slot = adrp(first, va) + ((second >> 10) & 0xfff) * 8 + target, = struct.unpack(' {target:#x} ' + ','.join(symbols.get(target, [])) + except ValueError: + pass + return '' + + +blob = read(args.at, args.bytes) +text = '\n'.join(' '.join(f'0x{x:02x}' for x in blob[i:i + 4]) for i in range(0, len(blob), 4)) +result = subprocess.run([tool, '--disassemble', '--triple=aarch64-apple-darwin'], + input=text, capture_output=True, text=True, check=True) +lines = [line.strip() for line in result.stdout.splitlines() if line.strip() and line.strip() != '.text'] +if len(lines) != len(blob) // 4: + raise SystemExit('Decoder omitted an instruction; refuse to mislabel addresses') +previous = None +for index, line in enumerate(lines): + va = args.at + index * 4 + word, = struct.unpack_from(' {target:#x} ' + branch_target(target) + if word & 0x7f000000 == 0x11000000 and previous is not None: + prev_word, prev_pc = previous + if prev_word & 0x9f000000 == 0x90000000 and prev_word & 31 == (word >> 5) & 31: + target = adrp(prev_word, prev_pc) + (((word >> 10) & 0xfff) << (12 if word & (1 << 22) else 0)) + note = f' -> {target:#x} ' + literal(target) + print(f'{va:#018x} {line}{note}') + previous = (word, va) diff --git a/scripts/inspect-target-prelink.py b/scripts/inspect-target-prelink.py new file mode 100755 index 0000000..bbca066 --- /dev/null +++ b/scripts/inspect-target-prelink.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python3 +"""Inventory the supplied MH_EXECUTE cache and extract one kext's segments. + +These remain prelinked bytes. This does not produce a relocated or loadable +standalone kext, and does not modify the supplied kernelcache. +""" +import argparse +import base64 +import hashlib +import json +from pathlib import Path +import struct +import xml.etree.ElementTree as ET + +ROOT = Path(__file__).resolve().parents[1] +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument('--bundle', default='com.apple.kec.corecrypto') +args = parser.parse_args() +source = ROOT / '.cache/firmware/kernelcache.macho/kernelcache.release.ipad7.decompressed' +data = source.read_bytes() +assert hashlib.sha256(data).hexdigest() == 'f18b4df865405f2a287b84977d7aca745e056d656d9fc9df1662970ce2969de7' + + +def commands(at): + magic, cpu, subtype, _, count, size = struct.unpack_from('<6I', data, at) + assert (magic, cpu, subtype) == (0xfeedfacf, 0x100000c, 0) + offset = at + 32 + for _ in range(count): + command, length = struct.unpack_from('= 8 and offset + length <= at + 32 + size <= len(data) + yield command, offset + offset += length + assert offset == at + 32 + size + + +def segments(at): + result = [] + for command, offset in commands(at): + if command == 0x19: + name, va, size, fileoff, filesize = struct.unpack_from('<16s4Q', data, offset + 8) + result.append({'name': name.rstrip(b'\0').decode(), 'va': va, + 'size': size, 'fileoff': fileoff, 'filesize': filesize}) + return result + + +parent = segments(0) + + +def read_virtual(address, length): + for segment in parent: + if segment['va'] <= address and address + length <= segment['va'] + segment['filesize']: + offset = segment['fileoff'] + address - segment['va'] + assert offset + length <= len(data) + return data[offset:offset + length] + raise ValueError(f'VA extent is not backed by cache bytes: {address:#x}+{length:#x}') + + +refs = {} + + +def unserialize(element): + if 'IDREF' in element.attrib: + return refs[element.attrib['IDREF']] + tag = element.tag + if tag == 'dict': + children = list(element) + assert len(children) % 2 == 0 + value = {unserialize(children[i]): unserialize(children[i + 1]) + for i in range(0, len(children), 2)} + elif tag == 'array': + value = [unserialize(child) for child in element] + elif tag == 'integer': + value = int(element.text or '0', 0) + elif tag in ('string', 'key'): + value = element.text or '' + elif tag == 'data': + value = base64.b64decode(element.text or '') + elif tag in ('true', 'false'): + value = tag == 'true' + else: + raise ValueError(f'Unsupported OSUnserializeXML tag: {tag}') + if 'ID' in element.attrib: + refs[element.attrib['ID']] = value + return value + + +info_segment = next(segment for segment in parent if segment['name'] == '__PRELINK_INFO') +xml = read_virtual(info_segment['va'], info_segment['filesize']).split(b'\0')[0] +prelink = unserialize(ET.fromstring(xml)) +entries = prelink['_PrelinkInfoDictionary'] +inventory = [{key: entry.get(key) for key in ['CFBundleIdentifier', 'CFBundleVersion', + '_PrelinkExecutableLoadAddr', '_PrelinkExecutableSize', 'OSBundleLibraries']} + for entry in entries] +output = ROOT / 'build/target-kexts' +output.mkdir(parents=True, exist_ok=True) +(output / 'inventory.json').write_text(json.dumps(inventory, indent=2) + '\n') +entry = next(entry for entry in entries if entry['CFBundleIdentifier'] == args.bundle) +address = entry['_PrelinkExecutableLoadAddr'] +header_segment = next(segment for segment in parent + if segment['va'] <= address < segment['va'] + segment['filesize']) +header = header_segment['fileoff'] + address - header_segment['va'] +destination = output / entry['CFBundleIdentifier'] +destination.mkdir(exist_ok=True) +extracted = [] +for segment in segments(header): + if not segment['filesize']: + continue + blob = read_virtual(segment['va'], segment['filesize']) + (destination / f'{segment["name"]}.bin').write_bytes(blob) + extracted.append({**segment, 'sha256': hashlib.sha256(blob).hexdigest()}) +# mach/kmod.h uses #pragma pack(4), including its 64-bit compatibility ABI. +kmod = read_virtual(entry['_PrelinkKmodInfo'], 196) +start, stop = struct.unpack_from(' ` fails on this host (fastboot 37 reads the file end to +copy an AVB footer and aborts with "Failed reading from " before sending anything); for +sparse input it skips that step. All-zero 4096-byte blocks become FILL chunks with value 0 (written +as zeros, never DONT_CARE), everything else RAW chunks, so the expanded content equals the input +byte for byte. The result is verified by expanding it and comparing SHA-256 with the input. +""" +import argparse +import hashlib +from pathlib import Path +import struct +import sys + +BLOCK = 4096 +MAGIC = 0xED26FF3A +RAW, FILL, DONT_CARE, CRC32 = 0xCAC1, 0xCAC2, 0xCAC3, 0xCAC4 +MAX_RAW_BLOCKS = 4096 # 16 MiB per RAW chunk +ZERO = bytes(BLOCK) + + +def convert(source, target): + size = source.stat().st_size + if size % BLOCK: + raise SystemExit('Input size must be a multiple of 4096') + total_blocks = size // BLOCK + chunks = 0 + with open(source, 'rb') as src, open(target, 'wb') as out: + out.write(b'\0' * 28) # header, rewritten at the end + pending = bytearray() # blocks of the current RAW run + pending_blocks = 0 + zero_run = 0 + + def flush_raw(): + nonlocal pending, pending_blocks, chunks + if pending_blocks: + out.write(struct.pack(' len(data) or end - start < 24 or struct.unpack_from('= data[at:at + 20]): + return False + return True + + +if not valid_like_the_loader(bundle): + raise SystemExit('The trust bundle would be refused by the boot loader') +out = ROOT / 'build/adreno-metal-plugin/owner-trust.bundle' +out.write_bytes(bundle) +(ROOT / 'build/adreno-metal-plugin/owner-trust.json').write_text(json.dumps(dict( + bundle_sha256=hashlib.sha256(bundle).hexdigest(), own_cdhashes=[h.hex() for h in hashes], original_modules_unchanged=True, + apple_trust_chain_verified=False, purpose='owner trust for the Adreno Metal plugin only'), indent=2) + '\n') +print('PASS: owner trust bundle (own module + both original modules)', hashlib.sha256(bundle).hexdigest()) diff --git a/scripts/prepare-air-nir-diagnostic.py b/scripts/prepare-air-nir-diagnostic.py new file mode 100644 index 0000000..9bc5981 --- /dev/null +++ b/scripts/prepare-air-nir-diagnostic.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python3 +"""Clone our mutable APFS Data fixture and stage only own code plus exact AIR input.""" +import argparse +import hashlib +import json +from pathlib import Path +import plistlib +import re +import shutil +import subprocess +import sys +ROOT=Path(__file__).resolve().parents[1] +p=argparse.ArgumentParser(description=__doc__);p.add_argument('--broker-processes',action='store_true');p.add_argument('--broker-policy',choices=['full','without-container','without-client','parent-client','unrelated-client'],default='full');p.add_argument('--legacy-swap',action='store_true');p.add_argument('--legacy-swap-wait',action='store_true');p.add_argument('--framebuffer-geometry',action='store_true');p.add_argument('--framebuffer-power',action='store_true');p.add_argument('--framebuffer-protocol',action='store_true');p.add_argument('--empty-end',action='store_true');p.add_argument('--display-environment',action='store_true');p.add_argument('--fp-context',action='store_true');p.add_argument('--compose',action='store_true');p.add_argument('--resources',action='store_true');p.add_argument('--present',action='store_true');p.add_argument('--bgra',action='store_true');p.add_argument('--iosurface',action='store_true');p.add_argument('--offline-metal-api',action='store_true');p.add_argument('--offline-gpu-transport',action='store_true');p.add_argument('--metal-runtime',action='store_true');p.add_argument('--library-input',action='store_true');p.add_argument('--fragment-inputs',action='store_true');p.add_argument('--library-tests',action='store_true');p.add_argument('--vpc-tests',action='store_true');p.add_argument('--case',choices=['positive','bad-input'],default='positive');p.add_argument('--async-commands',action='store_true');p.add_argument('--fences',action='store_true');p.add_argument('--sandbox-extension',action='store_true');p.add_argument('--metal-layer',action='store_true');p.add_argument('--renderbox-present',type=int,choices=[1,2]);p.add_argument('--depth-stencil',action='store_true');p.add_argument('--private-texture',action='store_true');p.add_argument('--renderbox-draw',action='store_true');p.add_argument('--renderbox-device',action='store_true');p.add_argument('--plugin-control',choices=['full','without-mount-permission','tampered-code'],default='full');p.add_argument('--plugin-addon',action='store_true');p.add_argument('--plugin-bundle',action='store_true');p.add_argument('--shared-surfaces',action='store_true',help='Pass actual IOSurface Mach rights between two GPU processes') +p.add_argument('--shared-events',action='store_true',help='Real native shared-event Mach capabilities between two processes') +p.add_argument('--shared-notifications',action='store_true',help='Original listener queue delivery and bounded context cancellation') +p.add_argument('--shared-coding',action='store_true',help='Original NSXPC shared-handle secure coding, separate from cross-process authority') +p.add_argument('--shared-cross-process',action='store_true',help='Actual original NSXPC endpoint and handle coding between two native GPU workers') +p.add_argument('--vertex-inputs',action='store_true',help='Actual original QuartzCore vertex layouts and native GPU buffers') +args=p.parse_args() +if args.vertex_inputs and any((args.shared_events, args.shared_notifications, args.shared_coding, args.shared_cross_process, args.async_commands, args.fences, args.metal_layer, args.renderbox_present, args.depth_stencil, args.private_texture, args.renderbox_draw, args.renderbox_device, args.plugin_addon, args.plugin_bundle, args.shared_surfaces, args.sandbox_extension, args.legacy_swap, args.legacy_swap_wait, args.compose, args.resources, args.present)):p.error("Select the independent vertex input instrument") +if args.vertex_inputs:args.broker_processes=True +if args.shared_notifications and (args.shared_coding or args.shared_cross_process):p.error('Select notifications or secure coding as independent instruments') +if args.shared_cross_process:args.shared_coding=True +if args.shared_coding:args.shared_events=True +if args.shared_notifications:args.shared_events=True +if args.shared_events:args.broker_processes=True +if args.async_commands:args.fences=True +if args.fences:args.metal_layer=True +if args.sandbox_extension:args.broker_processes=True +if args.metal_layer: + if args.renderbox_present or args.depth_stencil or args.private_texture or args.renderbox_draw:p.error('Select one independent layer/renderer instrument') + args.renderbox_device=True +if args.renderbox_present: + if args.depth_stencil or args.private_texture or args.renderbox_draw:p.error('Select one independent renderer/presentation instrument') + args.renderbox_device=True +if args.depth_stencil: + if args.private_texture or args.renderbox_draw:p.error('Select one independent resource or renderer instrument') + args.renderbox_device=True +if args.private_texture: + if args.renderbox_draw:p.error('Select one independent resource or original renderer instrument') + args.renderbox_device=True +if args.renderbox_draw:args.renderbox_device=True +if args.renderbox_device: + if args.plugin_control!='full':p.error('RenderBox device observation is a separate instrument') + args.plugin_addon=True +if args.plugin_control!='full':args.plugin_addon=True +if args.plugin_addon:args.plugin_bundle=True +if args.plugin_bundle:args.shared_surfaces=True +if args.shared_surfaces:args.broker_processes=True +if args.shared_surfaces and args.broker_policy!='full':p.error('Sharing instrument requires both existing scoped diagnostic permissions') +if args.broker_policy!='full' and not args.broker_processes:p.error('Permission controls require the GPU process instrument') +if args.broker_processes: + if not args.offline_metal_api:p.error('GPU process instrument requires --offline-metal-api') + if args.legacy_swap or args.legacy_swap_wait or args.compose or args.resources or args.present:p.error('Select one GPU runtime instrument') + args.bgra=True +if args.legacy_swap_wait:args.legacy_swap=True +if args.empty_end and not args.framebuffer_protocol:p.error('--empty-end requires the framebuffer protocol instrument') +if args.legacy_swap and (args.compose or args.resources or args.present):p.error('Select one presentation instrument') +if args.legacy_swap: + if not args.offline_metal_api:p.error('--legacy-swap requires --offline-metal-api') + args.bgra=True +if args.compose and args.resources:p.error('Select composition or resource instrument') +if args.compose:args.present=True +if args.resources:args.present=True +if args.present:args.bgra=True +if args.bgra:args.iosurface=True +if args.iosurface and not args.offline_metal_api:raise SystemExit('IOSurface requires the Metal API instrument') +if args.fragment_inputs and not args.library_input:raise SystemExit('Fragment inputs require direct libraries') +if sum((args.framebuffer_geometry,args.framebuffer_power,args.framebuffer_protocol,args.display_environment,args.fp_context,args.library_input,args.library_tests,args.metal_runtime,args.vpc_tests,args.offline_gpu_transport,args.offline_metal_api))>1:raise SystemExit('Select one independent instrument') +broker_kind='offline-metal-vertex-inputs' if args.vertex_inputs else 'offline-metal-shared-cross-process' if args.shared_cross_process else 'offline-metal-shared-coding' if args.shared_coding else 'offline-metal-shared-notifications' if args.shared_notifications else 'offline-metal-shared-events' if args.shared_events else 'offline-metal-async' if args.async_commands else 'offline-metal-fences' if args.fences else 'offline-sandbox-extension' if args.sandbox_extension else 'offline-metal-layer' if args.metal_layer else ('offline-renderbox-present-'+str(args.renderbox_present)) if args.renderbox_present else 'offline-metal-depth-stencil' if args.depth_stencil else 'offline-metal-private-texture' if args.private_texture else 'offline-renderbox-draw' if args.renderbox_draw else 'offline-renderbox-device' if args.renderbox_device else ('offline-metal-addon'+('' if args.plugin_control=='full' else '-'+args.plugin_control)) if args.plugin_addon else 'offline-metal-plugin' if args.plugin_bundle else 'offline-metal-shared-surfaces' if args.shared_surfaces else 'offline-metal-processes' +OUT=ROOT/(('build/'+broker_kind+('' if args.broker_policy=='full' else '-'+args.broker_policy)) if args.broker_processes else 'build/framebuffer-geometry-diagnostic' if args.framebuffer_geometry else 'build/framebuffer-power-diagnostic' if args.framebuffer_power else 'build/offline-legacy-swap-wait' if args.legacy_swap_wait else 'build/offline-legacy-swap' if args.legacy_swap else 'build/framebuffer-empty-protocol-diagnostic' if args.empty_end else 'build/framebuffer-protocol-diagnostic' if args.framebuffer_protocol else 'build/display-environment-diagnostic' if args.display_environment else 'build/native-fp-context' if args.fp_context else 'build/offline-gop-composition' if args.compose else 'build/offline-gop-resources' if args.resources else 'build/offline-gop-presentation' if args.present else 'build/offline-metal-bgra' if args.bgra else 'build/offline-metal-iosurface' if args.iosurface else 'build/offline-metal-api' if args.offline_metal_api else 'build/offline-gpu-transport' if args.offline_gpu_transport else 'build/adreno-vpc-native-tests' if args.vpc_tests else 'build/metal-runtime-diagnostic' if args.metal_runtime else 'build/air-library-native-tests' if args.library_tests else 'build/air-fragment-inputs-diagnostic' if args.fragment_inputs else 'build/air-library-diagnostic' if args.library_input else 'build/air-nir-diagnostic') +suffix='' if args.case=='positive' else '-bad-input' +def sha(path):return hashlib.sha256(Path(path).read_bytes()).hexdigest() +source=ROOT/'build/owner-data-fixture/owner-data-32.cdr';seed=json.loads(source.with_name('seed.json').read_text()) +if sha(source)!=seed['sha256']:raise SystemExit('Own Data seed identity mismatch') +process=OUT/'air-native';report=json.loads((OUT/'build-report.json').read_text()) +if sha(process)!=report['process_sha256']:raise SystemExit('Own signed compiler changed') +manifest=(OUT/'test-inputs.json') if (args.bgra or args.fp_context or args.display_environment or args.framebuffer_protocol or args.framebuffer_power or args.framebuffer_geometry) else ROOT/('build/offline-metal-iosurface/test-inputs.json' if args.iosurface else 'build/offline-metal-api/test-inputs.json' if args.offline_metal_api else 'build/offline-gpu-transport/test-inputs.json' if args.offline_gpu_transport else 'build/adreno-vpc-native-tests/test-inputs.json' if args.vpc_tests else 'build/gpu-analysis/metal-device-abi.json' if args.metal_runtime else 'build/air-library-native-tests/test-inputs.json' if args.library_tests else 'scripts/data/native-air-fragment-inputs.json' if args.fragment_inputs else 'scripts/data/native-air-diagnostic-inputs.json') +if sha(manifest)!=report['manifest_sha256']:raise SystemExit('Input manifest differs from signed instrument') +cases=[] if (args.framebuffer_geometry or args.framebuffer_power or args.framebuffer_protocol or args.display_environment or args.fp_context or args.metal_runtime or args.library_tests or args.vpc_tests or args.offline_gpu_transport or args.offline_metal_api) else json.loads(manifest.read_text())['cases'] +inputs=[] +for case in ([] if args.library_input else cases): + bitcode=ROOT/f"build/gpu-analysis/air/{case['library']}/module-{case['index']}.bc" + if sha(bitcode)!=case['input_sha256']:raise SystemExit('Original AIR differs') + inputs.append((case,bitcode)) +image=OUT/('compiler-data'+suffix+'-32.cdr') +if image.exists():raise SystemExit('Refuse to overwrite a diagnostic Data image') +shutil.copyfile(source,image);mount=OUT/'data-mount';mount.mkdir(exist_ok=True) +attached=plistlib.loads(subprocess.check_output(['hdiutil','attach',str(image),'-readwrite','-nobrowse','-owners','off','-mountpoint',str(mount),'-plist'],timeout=30)) +device=attached['system-entities'][0]['dev-entry'] +addon_report={} +try: + info=plistlib.loads(subprocess.check_output(['diskutil','info','-plist',str(mount)],timeout=30)) + if info['VolumeUUID']!=seed['volume_uuid']:raise SystemExit('Cloned Data UUID differs') + if shutil.disk_usage(mount).free65536:failures.append('Bounded read-only console prefix exceeded');break + except OSError as error: + if not stop.is_set():failures.append(str(error)) + finally: + if connection:connection.close() + thread=threading.Thread(target=sink,daemon=True);thread.start() + try:subprocess.run([*cmd,'--virtio-console-socket',str(path)],cwd=ROOT,check=True,timeout=180) + finally:stop.set();listener.close();thread.join(timeout=3) + fixture.joinpath('console-sink.json').write_text(json.dumps(dict(bytes_observed=len(capture),sha256=hashlib.sha256(capture).hexdigest(),errors=failures, + protocol_responses_sent=0,gpu_execution=False,system_device_published=False),indent=2)+'\n') + if failures:raise SystemExit('; '.join(failures)) +print('Actual original-base lifecycle observation captured; no device attached/published and no GPU protocol replies supplied') diff --git a/scripts/probe-air-nir-native.py b/scripts/probe-air-nir-native.py new file mode 100644 index 0000000..9db950c --- /dev/null +++ b/scripts/probe-air-nir-native.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +"""Run one bounded actual iOS17 compiler/control boot; this does not emulate a GPU.""" +import argparse +from pathlib import Path +import subprocess +ROOT=Path(__file__).resolve().parents[1] +p=argparse.ArgumentParser(description=__doc__) +p.add_argument('--case',required=True,choices=['exit','bad-input','no-trust','default-off']) +group=p.add_mutually_exclusive_group() +group.add_argument('--library-input',action='store_true',help='Read whole original OS libraries through the native compiler') +group.add_argument('--vpc-tests',action='store_true',help='Run native encoder plus original source-reference VPC tests') +group.add_argument('--library-tests',action='store_true',help='Run actual native container refusal/guard tests') +group.add_argument('--metal-runtime',action='store_true',help='Observe genuine original Metal ABI and missing provider') +p.add_argument('--fragment-inputs',action='store_true',help='Include the five new center-perspective original fragments') +a=p.parse_args() +if a.fragment_inputs and not a.library_input:p.error('Fragment inputs require direct original libraries') +if a.case=='bad-input' and (a.library_input or a.library_tests or a.metal_runtime or a.vpc_tests):p.error('Bad extracted AIR input belongs to the original compiler control') +kind='adreno-vpc-native-tests' if a.vpc_tests else 'air-fragment-inputs-diagnostic' if a.fragment_inputs else 'air-library-diagnostic' if a.library_input else 'air-library-native-tests' if a.library_tests else 'metal-runtime-diagnostic' if a.metal_runtime else 'air-nir-diagnostic' +name=('adreno-vpc-controls' if a.case=='exit' else 'adreno-vpc-controls-'+a.case) if a.vpc_tests else 'air-fragment-inputs-'+a.case if a.fragment_inputs else 'air-library-'+a.case if a.library_input else ('air-library-controls' if a.case=='exit' else 'air-library-controls-'+a.case) if a.library_tests else ('metal-runtime-complete' if a.case=='exit' else 'metal-runtime-'+a.case) if a.metal_runtime else 'air17-'+a.case +output='build/darwinboot-owner-hvf-'+name +trust='build/original-static-system-trust/original-static-system.bundle' if a.case=='no-trust' else 'build/'+kind+'/offline-trust.bundle' +data='build/'+kind+'/compiler-data'+('-bad-input' if a.case=='bad-input' else '')+'-32.cdr' +cmd=['python3','scripts/probe-darwinboot-xnu.py', + '--kernel','build/native-prelink/native-owner-air-outer-candidate.macho', + '--prelink-report','build/native-prelink/native-owner-air-outer-candidate.json', + '--output',output,'--virtio-udif-image','.cache/firmware/target-volumes/044-76040-038.dmg', + '--virtio-extra-block-image','.cache/firmware/target-volumes/044-76034-038.dmg', + '--virtio-mmio-fdt','build/virtio-block-fixture/qemu-virt.dtb', + '--root-match-volume-uuid','04769D26-2CA0-4825-B443-E2E5F88300F2','--trust-cache',trust, + '--armvirt','--armcpu','--qemu-vmm-present','--qemu-accel','hvf', + '--owned-memory-size','0x20000000','--nvram-proxy','--crypto-hash-method','sha2-384', + '--amfi-platform-only','--load-apfs','--inspect-owner-root','--inspect-dyld-flags','--check-commpage', + '--skip-initializer-traces','--timeout','60','--data-ramdisk',data,'--qemu-ramfb','--inspect-boot-video'] +cmd+=['--follow-target-main'] if a.case=='default-off' else ['--native-air-init','--break-symbol','_exit','--read-entry-region','1:0:8'] +if a.metal_runtime:cmd+=['--load-iogpu'] +r=subprocess.run(cmd,cwd=ROOT,timeout=180) +if a.case=='no-trust': + import json,struct + d=json.loads((ROOT/output/'gdb-stop.json').read_text()) + args=struct.unpack_from('<5Q',bytes.fromhex(d['stack_bytes'])) + if r.returncode!=1 or d['stop_symbol']!='_panic' or args[2:4]!=(9,8) or not d['argument_memory']['x0'].startswith('unexpected SIGKILL of %s %s with reason'):raise SystemExit('Expected genuine EXEC security-policy refusal') + print('PASS: absent own test trust produces genuine EXEC security-policy refusal') +else: + if r.returncode:raise SystemExit(r.returncode) + if a.case in ('exit','bad-input'): + import json,struct + d=json.loads((ROOT/output/'gdb-stop.json').read_text()) + expected=62 if a.case=='bad-input' else 93 if a.vpc_tests else 90 if a.library_tests else 74 if a.metal_runtime else 73 + if d['stop_symbol']!='_exit' or struct.unpack_from('300 and not args.live_hid_dir:parser.error('Deadlines above 300 seconds require explicit live HID control') +if args.inspect_ui_threads: + args.inspect_current_process = True + args.inspect_process_list = True +if args.boot_progress_clock_fixture: + if args.break_symbol != '_boot_progress_stop_checkpoint' or args.qemu_accel != 'tcg': + parser.error('Clock fixture requires TCG and the diagnostic stop checkpoint') + args.trace_symbol.append('_boot_progress_poll') +data_seed = None +if args.data_ramdisk: + if args.native_user or not args.prelink_report or not args.root_match_volume_uuid: + parser.error('Scratch Data requires the exact original OS root/prelink path') + import sys + sys.path.insert(0, str(ROOT / 'drivers/DarwinBoot/Test')) + from packed_ramdisk import inspect as inspect_ramdisk + try: + data_seed, data_header = inspect_ramdisk(args.data_ramdisk) + except ValueError as error: + parser.error(str(error)) + if data_header[32:36] != b'NXSB': + parser.error('Scratch Data must expand to an actual raw APFS container') + data_seed.update(guest_mutable=True, production_storage_qualified=False, + initial_seed_signature_verified=False) + +main_fault = args.mutate_target_main_page +if main_fault and (not args.follow_target_main or args.mutate_own_vnode_page is not None or args.mutate_cache_vnode_page is not None or args.mutate_own_ramdisk_after_auth is not None): + parser.error('Original main-page fault requires its exact main observer and no other input fault') +main_input = None +display_plan = None +process_plan = None +if args.follow_original_user_plan: + if not args.follow_target_main or any((args.follow_display_mode,args.follow_metal_device,args.follow_display_query,args.follow_display_discovery)): + parser.error('Original user plan requires original launchd main and no other display observer') + display_plan=args.follow_original_user_plan + selected_plan=json.loads(display_plan.read_text()) + if selected_plan.get('mode') not in ('user-return','user-trace') or selected_plan.get('original_files_modified') is not False: + parser.error('Require a read-only original user-return or user-trace plan') +if args.entry_process_name and not args.inspect_current_process:parser.error('Process-name selection requires --inspect-current-process') +if args.inspect_current_process: + if not args.prelink_report: + parser.error('Process inspection requires the exact original-component candidate') + subprocess.run(['python3', ROOT / 'scripts/audit-target-display-query.py'], check=True) + process_plan = ROOT / 'build/display-analysis/query-plan.json' +if args.follow_display_mode or args.follow_metal_device: + if not args.follow_target_main or args.follow_display_query or args.follow_display_discovery or (args.follow_display_mode and args.follow_metal_device):parser.error('Mode return requires original main and no other display observer') + subprocess.run(['python3',ROOT/'scripts/prepare-original-display-mode-observer.py',*(['--metal-device'] if args.follow_metal_device else [])],check=True) + display_plan=ROOT/('build/display-analysis/metal-device-return-plan.json' if args.follow_metal_device else 'build/display-analysis/mode-plan.json') +if args.follow_display_query or args.follow_display_discovery: + if not args.follow_target_main: + parser.error('Display query observation requires actual original launchd main') + if args.follow_display_query and args.follow_display_discovery: + parser.error('Select one original display observation') + subprocess.run(['python3', ROOT / 'scripts/audit-target-display-query.py', *(['--discovery'] if args.follow_display_discovery else [])], check=True) + display_plan = ROOT / ('build/display-analysis/discovery-plan.json' if args.follow_display_discovery else 'build/display-analysis/query-plan.json') +if args.native_air_init and args.follow_target_main: + parser.error('Own compiler init and original launchd main are distinct observations') +if args.follow_target_main: + if args.native_user or args.finish_function or args.break_symbol != '_panic' or args.break_address is not None or not args.prelink_report: + parser.error('Original main observation requires its exact original-component candidate and genuine OS root, without another checkpoint') + main_bytes = (args.target_main_executable or ROOT / '.cache/firmware/target-volumes/os-mount/sbin/launchd').read_bytes() + if hashlib.sha256(main_bytes).hexdigest() != '8cec9a3c44a443b08e977ccc5d577850316ad428c575dc01f09b018dbd5b6a25': + raise SystemExit('Expected the unchanged supplied original launchd') + sys.path.insert(0, str(ROOT / 'kits/NativePrelink')) + from native_prelink import macho + main_image = macho(main_bytes) + entries = [struct.unpack_from(' 1: + dt_command += ['--smp', str(args.smp)] +if args.negative_platform_match: + if not args.armvirt: + parser.error('--negative-platform-match requires --armvirt') + dt_command.extend(['--platform-compatible', 'unmatched-test-platform']) +if args.armdt_platform and args.negative_platform_match: + parser.error('Select either the sheng-shaped root or the unmatched control') +if args.native_user: + dt_command.append('--ramdisk') +if args.data_ramdisk: + dt_command.append('--data-ramdisk') +if args.nvram_proxy: + dt_command.append('--nvram-proxy') +if args.conflicting_boot_manifest: + if not args.prelink_report or args.native_user: + parser.error('Manifest conflict control requires the original OS prelink path') + dt_command.append('--conflicting-boot-manifest') +if args.crypto_hash_method: + dt_command.extend(['--crypto-hash-method', args.crypto_hash_method]) +if args.osenvironment: + dt_command.extend(['--osenvironment', args.osenvironment]) +if args.amfi_platform_only: + dt_command.append('--amfi-platform-only') +if args.trust_cache: + dt_command.append('--trust-cache') +if args.virtio_mmio_fdt: + dt_command += ['--virtio-mmio-fdt', args.virtio_mmio_fdt] +if args.root_match_volume_uuid: + dt_command += ['--root-match-volume-uuid', str(args.root_match_volume_uuid)] +subprocess.run(dt_command, check=True) +import re as _re +if not _re.fullmatch(r'[A-Za-z0-9_=. +\-]*', args.kernel_extra_args): + raise SystemExit('Kernel extra arguments contain unsupported characters') +kernel_args = ('-v serial=3 debug=0x14e kt_fixed_zones=120 ' + args.kernel_extra_args).strip() +(esp / 'EFI/CrabApple/crabapple.toml').write_text(f'''version = 1 +[boot] +timeout = 1 +default = "source-4k" +[kits] +autoload = false +[[driver]] +path = "Payloads/DarwinBoot.efi" +[[entry]] +id = "source-4k" +title = "Source native 4K XNU" +type = "xnu" +path = "Payloads/kernel.macho" +args = "{kernel_args}" +''') +user_addresses = None +if args.qemu_ramfb_high_resolution: + if not args.qemu_ramfb: + parser.error('High-resolution fixture requires the actual QEMU ramfb') + driver = ROOT / 'build/DarwinBoot/DEBUG_CLANGPDB/AARCH64/QemuRamfbGeometry.efi' + shutil.copyfile(driver, payloads / driver.name) + config = esp / 'EFI/CrabApple/crabapple.toml' + config.write_text(config.read_text().replace('[[entry]]', + '[[driver]]\npath = "Payloads/QemuRamfbGeometry.efi"\n[[entry]]', 1)) +if args.native_user: + user_directory = args.native_user_image.parent if args.native_user_image else ROOT / 'build/native-user-fixture' + shutil.copyfile(args.native_user_image or user_directory / 'launchd.macho', payloads / 'launchd.macho') + user_addresses = json.loads((user_directory / 'addresses.json').read_text()) + root_name = 'launchd.macho' + if args.native_user_ramdisk: + root_name = 'root.img' + shutil.copyfile(args.native_user_ramdisk, payloads / root_name) + config = esp / 'EFI/CrabApple/crabapple.toml' + config.write_text(config.read_text().replace('args = "-v', f'initrd = "Payloads/{root_name}"\nargs = "rd=md0 -v')) + if args.break_symbol == '_panic': + args.break_symbol = '_native_user_checkpoint' +if args.data_ramdisk: + shutil.copyfile(args.data_ramdisk, payloads / 'scratch-data.img') + config = esp / 'EFI/CrabApple/crabapple.toml' + config.write_text(config.read_text().replace('args = "-v', 'initrd = "Payloads/scratch-data.img"\nargs = "-v')) +if args.verified_root_probe: + config = esp / 'EFI/CrabApple/crabapple.toml' + config.write_text(config.read_text().replace('args = "', 'args = "offline-owner-root-check=1 ', 1)) +if args.load_apfs: + config = esp / 'EFI/CrabApple/crabapple.toml' + config.write_text(config.read_text().replace('args = "', 'args = "offline-apfs-load=1 ', 1)) +if args.load_iogpu: + config = esp / 'EFI/CrabApple/crabapple.toml' + config.write_text(config.read_text().replace('args = "', 'args = "offline-iogpu-load=1 ', 1)) +if args.load_mobile_graphics: + config = esp / 'EFI/CrabApple/crabapple.toml' + config.write_text(config.read_text().replace('args = "', 'args = "offline-mobile-graphics-load=1 ', 1)) +if args.native_air_init: + if not args.data_ramdisk or not args.trust_cache or not args.inspect_owner_root: + parser.error('Offline AIR init requires explicit Data, trust bundle and real owner-root checks') + config = esp / 'EFI/CrabApple/crabapple.toml' + config.write_text(config.read_text().replace('args = "', 'args = "offline-air-init=1 ', 1)) +if args.offline_gpu_broker: + if not args.native_air_init:parser.error('Private GPU broker requires the explicit owned diagnostic') + config.write_text(config.read_text().replace('args = "','args = "offline-gpu-broker=1 ',1)) +if args.accelerator_abi_probe: + if not args.offline_gpu_broker:parser.error('Accelerator ABI observation requires the explicit private broker startup boundary') + config.write_text(config.read_text().replace('args = "','args = "offline-accelerator-abi=1 ',1)) +if args.original_client_abi_probe: + if not args.offline_gpu_broker:parser.error('Original client ABI observation requires the explicit private broker lifecycle hook') + config.write_text(config.read_text().replace('args = "','args = "offline-original-client-abi=1 ',1)) +if args.original_provider_probe: + if not args.offline_gpu_broker:parser.error('Provider ABI observation requires the explicit private backend hook') + if not 0<=args.original_provider_client_type<=0xffffffff:parser.error('Original client type must fit uint32') + mode={'lifecycle':1,'start':2,'client-factory':3,'client-refusal':4,'descriptor-provision':5,'descriptor-provision-attached':6,'descriptor-provision-attached-only':7}[args.original_provider_probe] + config.write_text(config.read_text().replace('args = "','args = "offline-original-provider-abi='+str(mode)+' offline-original-client-type='+str(args.original_provider_client_type)+' ',1)) +if args.constant_uploads_probe: + config = esp / 'EFI/CrabApple/crabapple.toml' + config.write_text(config.read_text().replace('args = "', 'args = "offline-constants=1 ', 1)) +if args.smmu4k_probe: + config = esp / 'EFI/CrabApple/crabapple.toml' + config.write_text(config.read_text().replace('args = "', 'args = "offline-smmu4k=1 ', 1)) +if args.adreno_pm4_probe: + config = esp / 'EFI/CrabApple/crabapple.toml' + config.write_text(config.read_text().replace('args = "', 'args = "offline-adreno-pm4=1 ', 1)) +if args.adreno_firmware_probe: + config = esp / 'EFI/CrabApple/crabapple.toml' + config.write_text(config.read_text().replace('args = "', 'args = "offline-adreno-firmware=1 ', 1)) +if args.mobile_graphics_client_probe: + config = esp / 'EFI/CrabApple/crabapple.toml' + config.write_text(config.read_text().replace('args = "', 'args = "offline-mobile-graphics-client=1 ', 1)) +if args.legacy_display_phase: + if not args.mobile_graphics_client_probe:parser.error('Legacy lifecycle requires the explicit original-family client gate') + config.write_text(config.read_text().replace('args = "', f'args = "offline-legacy-phase={args.legacy_display_phase} ', 1)) +if args.live_hid_dir and not args.hid_home_input:parser.error('Live HID requires the explicit offline input source') +if args.hid_home_input and not args.load_hid_input: + parser.error('Offline Home report requires original HID loading') +if args.load_hid_input: + if not (args.load_mobile_graphics and args.prelink_report and args.legacy_display_phase==4): + parser.error('HID observation requires the original system/display late-init experiment') + config=esp/'EFI/CrabApple/crabapple.toml' + config.write_text(config.read_text().replace('args = "','args = "offline-hid-input=1 ',1)) + if args.hid_home_input:config.write_text(config.read_text().replace('args = "','args = "offline-hid-home=1 ',1)) +if args.smp > 1: + if not args.armcpu: + parser.error('--smp requires --armcpu') + config=esp/'EFI/CrabApple/crabapple.toml' + config.write_text(config.read_text().replace('args = "','args = "sheng-smp=1 ',1)) +if args.original_cpu_present: + if not (args.legacy_display_transfer and args.load_mobile_graphics and args.prelink_report) or args.native_air_init or args.native_user: + parser.error('Original CPU presentation requires original system graphics and validated copy, without an owned renderer') + config=esp/'EFI/CrabApple/crabapple.toml' + config.write_text(config.read_text().replace('args = "','args = "offline-legacy-cpu-present=1 ',1)) +if args.legacy_display_retire: + if not (args.legacy_display_transfer and args.native_air_init):parser.error('Original retirement requires actual validated GOP transfer and the explicit owned diagnostic') + config=esp/'EFI/CrabApple/crabapple.toml' + config.write_text(config.read_text().replace('args = "','args = "offline-legacy-retire=1 ',1)) +if args.legacy_display_transfer: + if not args.legacy_display_validate:parser.error('Bounded GOP transfer requires actual original surface validation') + config=esp/'EFI/CrabApple/crabapple.toml' + config.write_text(config.read_text().replace('args = "','args = "offline-legacy-transfer=1 ',1)) +if args.legacy_display_validate: + if not args.legacy_display_logical_pm:parser.error('Original full-frame validation requires logical scanout scheduling') + config=esp/'EFI/CrabApple/crabapple.toml' + config.write_text(config.read_text().replace('args = "','args = "offline-legacy-validate=1 ',1)) +if args.legacy_display_logical_pm: + if not (args.legacy_display_gop and args.legacy_display_register and args.legacy_display_phase==4):parser.error('Logical scanout PM requires the explicit GOP phase-4 experiment') + config=esp/'EFI/CrabApple/crabapple.toml' + config.write_text(config.read_text().replace('args = "','args = "offline-legacy-pm=1 ',1)) +if args.legacy_display_unit_scale: + config=esp/'EFI/CrabApple/crabapple.toml' + config.write_text(config.read_text().replace('args = \"','args = \"offline-legacy-unit-scale=1 ',1)) +if args.legacy_display_gop: + if args.legacy_display_phase not in (2,4):parser.error('Legacy GOP backend requires the explicit start phase') + config.write_text(config.read_text().replace('args = "', 'args = "offline-legacy-gop=1 ', 1)) +if args.legacy_display_unit_scale and not (args.legacy_display_gop and args.legacy_display_register and args.legacy_display_phase==4):parser.error('Unit-scale policy requires the actual GOP-backed phase-4 system experiment') +if args.legacy_display_register: + if args.legacy_display_phase!=4 or not args.legacy_display_gop:parser.error('Original client experiment requires explicit phase 4 and genuine GOP backend') + config.write_text(config.read_text().replace('args = "','args = "offline-legacy-register=1 ',1)) +if args.legacy_display_register_only: + if not args.qemu_ramfb:parser.error('Register-only display needs the actual GOP ramfb') + config.write_text(config.read_text().replace('args = "','args = "offline-mobile-graphics-load=1 offline-legacy-gop=1 offline-legacy-phase=4 offline-legacy-register=1 ',1)) +if args.root_volume_uuid: + config = esp / 'EFI/CrabApple/crabapple.toml' + config.write_text(config.read_text().replace('args = "', f'args = "rd=uuid boot-uuid={str(args.root_volume_uuid).upper()} ', 1)) +if args.root_bsd_name: + config = esp / 'EFI/CrabApple/crabapple.toml' + config.write_text(config.read_text().replace('args = "', f'args = "rd={args.root_bsd_name} ', 1)) +if args.zone_pgz_probe_case: + config = esp / 'EFI/CrabApple/crabapple.toml' + config.write_text(config.read_text().replace('args = "', f'args = "offline-zone-pgz-case={args.zone_pgz_probe_case} ', 1)) +if args.native_data_user_image: + user_addresses=json.loads((args.native_data_user_image.parent/'addresses.json').read_text()) + args.native_user=True # Observation only; ESP already retains the original OS and actual Data. +symbols = subprocess.check_output(['xcrun', 'nm', '-U', args.kernel], text=True) +panic = re.search(r'^([0-9a-fA-F]+)\s+\S\s+' + re.escape(args.break_symbol) + '$', symbols, re.M) +if args.break_address is not None: + panic = {1: f'{args.break_address:x}'} +if args.follow_cache_leaf: + leaf_record = json.loads((args.native_user_image.parent / 'cache-leaf.json').read_text()) + if hashlib.sha256(args.native_user_image.read_bytes()).hexdigest() != leaf_record['own_process_sha256']: + raise SystemExit('Cache-code fixture identity mismatch') + args.break_symbol = '_original_cache_leaf_return' + panic = {1: f"{user_addresses['_cache_leaf_dispatch']:x}"} +elif args.native_user and args.break_symbol == '_native_user_checkpoint': + panic = {1: f"{user_addresses['_checkpoint']:x}"} +if not panic: + raise SystemExit(f'Cannot resolve {args.break_symbol} in this kernel') +page_globals = {} +compression_requests = {} +if args.inspect_process_list: + match = re.search(r'^([0-9a-fA-F]+)\s+\S\s+_allproc$', symbols, re.M) + if not match: + raise SystemExit('Cannot resolve the actual kernel allproc list') + process_list_args = ['--allproc-address', '0x' + match[1], + '--proc-list-offset', '0', '--proc-pid-offset', '96', + '--proc-comm-offset', '1468', '--proc-stat-offset', '100'] +else: + process_list_args = [] +gdb_command = ['python3', ROOT / 'scripts/qemu-entry/gdb-stop.py', f'0x{panic[1]}', + '--fixture', output, '--esp', esp, '--kernel', args.kernel, + '--system-registers', '--timeout', str(args.timeout), '--qemu-accel', args.qemu_accel, + '--qemu-memory-mib', str(args.qemu_memory_mib)] +gdb_command += process_list_args +if args.capture_scratch_data: + if not args.data_ramdisk: + parser.error('Scratch Data capture requires an actual Data RAM disk input') + pointers = [] + for name in ('_native_owner_data_seed_base', '_native_owner_data_seed_size'): + match = re.search(r'^([0-9a-fA-F]+)\s+\S\s+' + name + '$', symbols, re.M) + if not match: + raise SystemExit('Missing exact native Data capture symbol: ' + name) + pointers.append(match[1]) + gdb_command += ['--capture-scratch-data', ':'.join(pointers + [f'{data_seed["bytes"]:x}'])] +if args.inspect_ui_threads: + gdb_command += ['--proc-list-max', '512'] +if args.boot_progress_clock_fixture: + addresses=[] + for name in ('_boot_progress_poll','_boot_progress_epoch','_boot_progress_frequency'): + match=re.search(r'^([0-9a-fA-F]+)\s+\S\s+'+name+'$',symbols,re.M) + if not match:raise SystemExit('Missing diagnostic-only clock symbol '+name) + addresses.append(match[1]) + gdb_command += ['--boot-progress-clock-fixture',':'.join(addresses+[str(args.boot_progress_clock_fixture)])] +if args.exercise_ufs_allocation: + gdb_command += ['--exercise-ufs-allocation'] +if args.inspect_mac_policies: + objects=ROOT/'build/xnu/ios-obj/DEVELOPMENT_ARM64_BCM2837/security/DEVELOPMENT' + layout={};dwarf={} + for structure,obj,names in ( + ('mac_policy_list','mac_base.o',('numloaded','max','maxindex','staticmax','entries')), + ('mac_policy_list_element','mac_base.o',('mpc',)), + ('mac_policy_conf','mac_vfs.o',('mpc_name','mpc_fullname','mpc_ops')), + ('mac_policy_ops','mac_vfs.o',('mpo_proc_check_fork','mpo_vnode_check_exec','mpo_vnode_check_signature','mpo_proc_check_launch_constraints','mpo_file_check_mmap','mpo_policy_syscall'))): + text=subprocess.check_output(['xcrun','dwarfdump','--name='+structure,'--show-children',objects/obj],text=True) + dwarf[obj]=hashlib.sha256((objects/obj).read_bytes()).hexdigest();layout[structure]={} + for name in names: + match=re.search(r'DW_AT_name\s+\("'+name+r'"\)[\s\S]*?DW_AT_data_member_location\s+\((0x[0-9a-f]+)\)',text) + if not match:raise SystemExit('Cannot qualify actual MAC policy member '+name) + layout[structure][name]=int(match[1],16) + match=re.search(r'^([0-9a-fA-F]+)\s+\S\s+_mac_policy_list$',symbols,re.M) + if not match:raise SystemExit('Cannot locate actual MAC policy list') + plan=output/'mac-policy-layout.json';plan.write_text(json.dumps(dict(address=int(match[1],16),layout=layout,object_sha256=dwarf),indent=2)+'\n') + gdb_command+=['--mac-policy-layout',plan] +if args.entry_min_pid and (not args.entry_process_name or not 1<=args.entry_min_pid<1<<31):parser.error('Minimum PID requires a named process and a positive bounded PID') +if args.entry_process_name:gdb_command+=['--entry-process-name',args.entry_process_name,'--entry-min-pid',str(args.entry_min_pid)] +if args.capture_qemu_screen:gdb_command+=['--capture-screen'] +if args.qemu_ramfb: + gdb_command += ['--qemu-ramfb'] +if args.virtio_console_socket: + gdb_command += ['--virtio-console-socket', args.virtio_console_socket] +if args.follow_target_main: + gdb_command += ['--follow-entrypoint-arg', '1', '--follow-main-entry-offset', str(main_input['entry_offset']), + '--read-memory', 'fffff4025:1', '--read-memory', 'fffff4037:1'] +if display_plan: + if json.loads(display_plan.read_text())['native_sha256'] != prelink_report['native_sha256']: + raise SystemExit('Display observer layout/symbols must belong to this exact source kernel') + gdb_command += ['--main-cache-query-plan', display_plan] +if process_plan: + if json.loads(process_plan.read_text())['native_sha256'] != prelink_report['native_sha256']: + raise SystemExit('Process observer layout must belong to this exact source kernel') + gdb_command += ['--process-layout-plan', process_plan] +if args.inspect_ui_threads: + gdb_command.append('--inspect-ui-threads') +iogpu_component = None +if args.load_iogpu and 'com.apple.iokit.IOGPUFamily' in prelink_report['components']: + sys.path.insert(0, str(ROOT / 'kits/NativePrelink')) + from native_prelink import macho, backed_offset + target = prelink_report['module_entrypoints']['com.apple.iokit.IOGPUFamily']['start_va'] + candidate = args.kernel.read_bytes() + at = backed_offset(macho(candidate)['segments'], target, 16, len(candidate)) + iogpu_component = dict(start=hex(target), original_start=hex(target - prelink_report['slide']), + entry_prefix=candidate[at:at + 16].hex()) + gdb_command += ['--trace-address', hex(target), '--read-memory', f'{target:x}:10'] +mobile_component = None +if args.load_mobile_graphics and 'com.apple.iokit.IOMobileGraphicsFamily' in prelink_report['components']: + sys.path.insert(0, str(ROOT / 'kits/NativePrelink')) + from native_prelink import macho, backed_offset + target = prelink_report['module_entrypoints']['com.apple.iokit.IOMobileGraphicsFamily']['start_va'] + candidate = args.kernel.read_bytes() + at = backed_offset(macho(candidate)['segments'], target, 16, len(candidate)) + mobile_component = dict(start=hex(target), original_start=hex(target - prelink_report['slide']), + entry_prefix=candidate[at:at + 16].hex(), entry_trace_requested=not (args.trace_mobile_graphics_factories or args.follow_display_mode or args.follow_metal_device)) + if not (args.trace_mobile_graphics_factories or args.follow_display_mode or args.follow_metal_device): + gdb_command += ['--trace-address', hex(target)] + gdb_command += ['--read-memory', f'{target:x}:10'] +if args.inspect_compression: + match = re.search(r'^([0-9a-fA-F]+)\s+\S\s+_decompressors$', symbols, re.M) + if not match or not prelink_report: + raise SystemExit('Compression inspection requires actual native registration storage and its exact original-component candidate') + for kind in (3, 4): + request = f'{int(match[1], 16) + kind * 8:x}:0:30' + compression_requests[str(kind)] = request + gdb_command += ['--read-indirect', request] +if args.armvirt: + gdb_command.extend(['--gic-version', '3']) +if args.smp > 1: + gdb_command.extend(['--smp', str(args.smp)]) +if block_image: + gdb_command += ['--virtio-udif-image' if args.virtio_udif_image else '--virtio-block-image', block_image] + if args.missing_virtio_device: + gdb_command += ['--missing-virtio-device'] +if args.virtio_extra_block_image: + gdb_command += ['--virtio-extra-block-image', args.virtio_extra_block_image] +if args.entry_after: + match = re.search(r'^([0-9a-fA-F]+)\s+\S\s+' + re.escape(args.entry_after) + '$', symbols, re.M) + if not match: + raise SystemExit('Cannot resolve actual primary observation activation entry') + gdb_command += ['--primary-start-address', '0x' + match[1]] +if args.log_kernel_strings_after: + for name, option in ((args.log_kernel_strings_after, '--cstring-start-address'), ('_printf', '--cstring-log-address')): + match = re.search(r'^([0-9a-fA-F]+)\s+\S\s+' + re.escape(name) + '$', symbols, re.M) + if not match: + raise SystemExit(f'Cannot resolve actual delayed log function {name}') + gdb_command += [option, '0x' + match[1]] +if args.log_registry_strings: + match = re.search(r'^([0-9a-fA-F]+)\s+\S\s+__ZN15IORegistryEntry11setPropertyEPKcS1_$', symbols, re.M) + if not match: + raise SystemExit('Cannot resolve actual registry string-property setter') + gdb_command += ['--registry-string-log-address', '0x' + match[1]] +if args.follow_cache_leaf: + gdb_command += ['--also-stop', hex(user_addresses['_checkpoint'])] + gdb_command += ['--follow-entrypoint-arg', '0', '--follow-entrypoint-checkpoint-offset', + str(leaf_record['cache_leaf_input']['success_return_offset']), '--finish-followed-checkpoint'] +for number in args.log_syscall_number: + gdb_command += ['--syscall-number', str(number)] +if args.log_syscalls_after: + match = re.search(r'^([0-9a-fA-F]+)\s+\S\s+' + re.escape(args.log_syscalls_after) + '$', symbols, re.M) + if not match: + raise SystemExit('Cannot resolve actual syscall-log activation entry') + gdb_command += ['--syscall-start-address', '0x' + match[1]] +if args.log_syscalls or args.log_syscalls_after: + for name in ('_unix_syscall', '_mach_syscall'): + match = re.search(r'^([0-9a-fA-F]+)\s+\S\s+' + name + '$', symbols, re.M) + if not match: + raise SystemExit('Cannot resolve the actual syscall handler') + gdb_command += ['--syscall-log-address', '0x' + match[1]] +if args.inspect_entry_translations: + gdb_command += ['--inspect-entry-translations'] +for request in args.read_frame_indirect: + gdb_command += ['--read-frame-indirect', request] +if args.read_frame_locals: + gdb_command.append('--read-frame-locals') +for request in args.read_saved_user_register:gdb_command+=['--read-saved-user-register',request] +if args.inspect_entry_user_state: + gdb_command.append('--inspect-entry-user-state') +if args.finish_function: + gdb_command.append('--finish-function') + if args.verified_root_probe: + gdb_command += ['--read-entry-indirect', '0:0:40', '--read-entry-indirect', '1:0:20'] +for option, value in (('--entry-kernel-bytes-field', args.entry_kernel_bytes_field), + ('--entry-register-equals', args.entry_register_equals)): + if value: + gdb_command += [option, value] +gdb_command += ['--entry-max-skips', str(args.entry_max_skips)] +for request in args.read_entry_region: + gdb_command += ['--read-entry-region', request] +if args.entry_user_string_field: + gdb_command += ['--entry-user-string-field', args.entry_user_string_field] +for request in args.read_entry_indirect: + gdb_command += ['--read-entry-indirect', request] +for request in args.read_entry_user_vector: + gdb_command += ['--read-entry-user-vector', request] +for request in args.read_entry_user_indirect: + gdb_command += ['--read-entry-user-indirect', request] + if args.load_apfs and args.break_symbol == '_vfs_fsadd': + gdb_command += ['--read-entry-indirect', '0:0:90', '--read-entry-indirect', '1:0:80'] +fallback_symbols = {} +for fallback_name in ('_panic', '_panic_with_options'): + if args.break_symbol == fallback_name: + continue + fallback = re.search(r'^([0-9a-fA-F]+)\s+\S\s+' + fallback_name + '$', symbols, re.M) + if fallback: + gdb_command.extend(['--also-stop', f'0x{fallback[1]}']) + fallback_symbols[int(fallback[1], 16)] = fallback_name +if args.kext_logs: + match = re.search(r'^([0-9a-fA-F]+)\s+\S\s+_OSKextLog$', symbols, re.M) + if not match: + raise SystemExit('Cannot resolve OSKextLog') + gdb_command.extend(['--kext-log-address', f'0x{match[1]}']) +trace_symbols = {} +if iogpu_component: + trace_symbols[int(iogpu_component['start'], 16)] = 'com.apple.iokit.IOGPUFamily:kmod_start' +if mobile_component: + if not (args.follow_display_mode or args.follow_metal_device):trace_symbols[int(mobile_component['start'],16)]='com.apple.iokit.IOMobileGraphicsFamily:kmod_start' + if args.mobile_graphics_client_probe and not (args.follow_display_mode or args.follow_metal_device): + entries = ((0xfffffff00677b420, 'IOMobileFramebuffer::MetaClass::alloc'), + (0xfffffff00677b620, 'IOMobileFramebufferService::MetaClass::alloc'), + (0xfffffff0067866d4, 'IOMobileFramebufferLegacy::MetaClass::alloc')) if args.trace_mobile_graphics_factories else ( + (0xfffffff006778d40, 'IOMobileFramebufferUserClient::externalMethod'),) + for original, name in entries: + address = original + prelink_report['slide'] + trace_symbols[address] = name + gdb_command += ['--trace-address', hex(address)] +for name in args.trace_symbol: + match = re.search(r'^([0-9a-fA-F]+)\s+\S\s+' + re.escape(name) + '$', symbols, re.M) + if not match: + raise SystemExit(f'Cannot resolve trace symbol {name}') + trace_symbols[int(match[1], 16)] = name + gdb_command.extend(['--trace-address', f'0x{match[1]}']) +if prelink_report and not args.skip_initializer_traces: + for identifier in ('com.apple.kec.corecrypto', 'com.apple.driver.AppleMobileFileIntegrity', + 'com.apple.AppleFSCompression.AppleFSCompressionTypeZlib'): + if identifier not in prelink_report['module_entrypoints']: + continue + address = prelink_report['module_entrypoints'][identifier]['start_va'] + trace_symbols[address] = identifier + ':kmod_start' + gdb_command.extend(['--trace-address', hex(address)]) + if args.load_apfs and 'com.apple.filesystems.apfs' in prelink_report['module_entrypoints']: + address = prelink_report['module_entrypoints']['com.apple.filesystems.apfs']['start_va'] + trace_symbols[address] = 'com.apple.filesystems.apfs:kmod_start' + gdb_command.extend(['--trace-address', hex(address)]) + for identifier, address in prelink_report.get('early_security_initializers', {}).items(): + trace_symbols[address] = identifier + ':early_security_init' + gdb_command.extend(['--trace-address', hex(address)]) + for name in ('_register_crypto_functions', '_register_and_init_prng', '_amfi_interface_register', '_mac_policy_register'): + match = re.search(r'^([0-9a-fA-F]+)\s+\S\s+' + name + '$', symbols, re.M) + if match: + trace_symbols[int(match[1], 16)] = name + gdb_command.extend(['--trace-address', f'0x{match[1]}']) +inspections = [('_PAGE_SHIFT_CONST', 4), ('_page_shift', 4), ('_page_shift_user32', 4), ('_page_size', 8)] +if args.qemu_accel == 'hvf': + inspections += [('_initproc', 8), ('_gPanicBase', 8), ('_gPanicSize', 4), ('_panic_info', 8), + ('_gPhysBase', 8), ('_gPhysSize', 8), ('_mem_size', 8), ('_mem_actual', 8), + ('_cs_validate_page_bad_hash', 8), ('_cs_validate_page_no_hash', 8)] +owner_cache_globals = ('_native_owner_cache_stage', '_native_owner_cache_status', '_native_owner_cache_mount_status', + '_native_owner_cache_readonly', '_native_owner_cache_authorized', '_native_owner_cache_root_recheck', + '_native_owner_cache_selected_device', '_native_owner_cache_mounted_device') +has_owner_cache = re.search(r'\s_native_owner_cache_stage$', symbols, re.M) is not None +if has_owner_cache: + inspections += [(name, 4) for name in owner_cache_globals] + [('_native_owner_cache_device_path', 64)] +carrier_globals = (('_native_owner_cache_carrier_accepts', 8), ('_native_owner_cache_carrier_rejects', 8), + ('_native_owner_cache_carrier_root_device', 4), ('_native_owner_cache_carrier_file_device', 4)) +has_owner_carrier = re.search(r'\s_native_owner_cache_carrier_accepts$', symbols, re.M) is not None +if has_owner_carrier: + inspections += list(carrier_globals) +has_owner_manifest = re.search(r'\s_native_owner_boot_manifest_status$', symbols, re.M) is not None +if args.conflicting_boot_manifest and not has_owner_manifest: + parser.error('Conflict refusal requires the enabled owner metadata publisher') +if has_owner_manifest: + inspections += [('_native_owner_boot_manifest_status', 4), ('_native_owner_boot_manifest_hash', 48)] +has_owner_data = re.search(r'\s_native_owner_data_status$', symbols, re.M) is not None +if has_owner_data: + inspections += [(name, 4) for name in ('_native_owner_data_status', '_native_owner_data_stage', + '_native_owner_data_readonly', '_native_owner_data_device')] + inspections += [('_native_owner_data_seed_base', 8), ('_native_owner_data_seed_size', 8), ('_native_owner_data_seed_sha256', 32)] +has_owner_data_path = re.search(r'\s_native_owner_data_device_path$', symbols, re.M) is not None +if has_owner_data_path: + inspections += [('_native_owner_data_device_path', 64)] +composition_globals = (('_native_owner_cache_actual_mount_flags', 8), ('_native_owner_cache_original_root', 8), + ('_native_owner_cache_current_root', 8), ('_native_owner_cache_native_union', 4), ('_rootvnode', 8)) +has_composition = re.search(r'\s_native_owner_cache_native_union$', symbols, re.M) is not None +if has_composition: + inspections += list(composition_globals) +ram_block_globals = (('_native_ram_block_ready', 4), ('_native_ram_block_bytes', 8), + ('_native_ram_block_reads', 8), ('_native_ram_block_writes', 8), ('_native_ram_block_failures', 8)) +has_ram_block = re.search(r'\s_native_ram_block_ready$', symbols, re.M) is not None +console_globals = tuple(('_native_console_' + name, size) for name, size in ( + ('ready', 4), ('tx_requests', 4), ('rx_requests', 4), ('failures', 4), + ('tx_bytes', 8), ('rx_bytes', 8), ('mmio', 8))) +has_console = re.search(r'\s_native_console_ready$', symbols, re.M) is not None +if has_console: + inspections += list(console_globals) +gpu_broker_globals=tuple(('_native_gpu_broker_'+name,size) for name,size in ( + ('stage',4),('status',4),('clients',4),('active',4),('closed',4),('requests',4),('failures',4),('refusals',4),('reply_bytes',8))) +has_gpu_broker=re.search(r'\s_native_gpu_broker_stage$',symbols,re.M) is not None +if has_gpu_broker:inspections+=list(gpu_broker_globals) +has_gpu_cpu_contexts=re.search(r'\s_native_gpu_broker_cpu_context_count$',symbols,re.M) is not None +if has_gpu_cpu_contexts:inspections+=[('_native_gpu_broker_cpu_context_count',4),('_native_gpu_broker_cpu_contexts',512)] +gpu_shared_globals=tuple('_native_gpu_shared_'+name for name in ('exports','imports','drops','refusals','tokens_live','ports')) +has_gpu_shared_contexts=all(re.search(r'\s'+name+'$',symbols,re.M) for name in gpu_shared_globals) +if has_gpu_shared_contexts:inspections.extend((name,4) for name in gpu_shared_globals) +accelerator_globals=tuple(('_native_accelerator_'+name,size) for name,size in ( + ('stage',4),('status',4),('allocated',4),('initialized',4),('parent_match',4),('unrelated_match',4),('attached',4),('released',4),('size',4), + ('instances_before',4),('instances_live',4),('instances_after',4),('object',8),('vptr',8),('parent',8),('superclass',8))) +has_accelerator=re.search(r'\s_native_accelerator_stage$',symbols,re.M) is not None +if args.accelerator_abi_probe and not has_accelerator:parser.error('Explicit accelerator ABI component is absent from this kernel') +if has_accelerator:inspections+=list(accelerator_globals) +original_client_globals=tuple(('_native_original_client_'+name,size) for name,size in ( + ('stage',4),('status',4),('allocated',4),('initialized',4),('parent_match',4),('unrelated_match',4),('attached',4),('released',4),('size',4),('parent_size',4), + ('instances_before',4),('instances_live',4),('instances_after',4),('wrong_start',4),('provider_initialized',4),('object',8),('vptr',8),('meta',8),('parent',8))) +has_original_client=re.search(r'\s_native_original_client_stage$',symbols,re.M) is not None +if args.original_client_abi_probe and not has_original_client:parser.error('Explicit original-client ABI instrument is absent from this kernel') +if has_original_client:inspections+=list(original_client_globals) +original_provider_globals=tuple(('_native_original_provider_'+name,size) for name,size in ( + ('stage',4),('status',4),('allocated',4),('initialized',4),('parent_match',4),('attached',4),('registered',4),('released',4),('size',4), + ('instances_before',4),('instances_live',4),('instances_after',4),('callback',4),('start_returned',4),('start_result',4),('object',8),('vptr',8),('parent',8),('backend',8))) +# Optional new factory counters keep older exact kernels observable. +original_provider_globals+=tuple(('_native_original_provider_'+name,size) for name,size in ( + ('factory_returned',4),('factory_result',4),('client_type',4),('client_instances_before',4),('client_instances_live',4),('client_instances_at_callback',4),('descriptor_refused',4),('descriptor_provisioned',4),('factory_out_client',8)) + if re.search(r'\s_native_original_provider_'+name+'$',symbols,re.M)) +has_original_provider=re.search(r'\s_native_original_provider_stage$',symbols,re.M) is not None +if args.original_provider_probe and not has_original_provider:parser.error('Original provider instrument absent from kernel') +if has_original_provider:inspections+=list(original_provider_globals)+[('_native_original_provider_frame',128)] +provider_client_capture=None +if args.original_provider_probe=='client-factory' and args.original_provider_client_type&0xffff==1: + frame=re.search(r'^([0-9a-fA-F]+)\s+\S\s+_native_original_provider_frame$',symbols,re.M) + if not frame:parser.error('Actual original provider frame absent') + # Read-only: captured x1 is the original client's +280 output field. The + # independently audited original call site determines the containing object. + provider_client_capture=f'{int(frame[1],16)+8:x}:-118:130' + gdb_command+=['--read-indirect',provider_client_capture] + +gop_globals=tuple(('_native_gop_'+name,size) for name,size in ( + ('ready',4),('failures',4),('transfers',8),('bytes',8),('physical',8),('virtual',8))) +has_gop=re.search(r'\s_native_gop_ready$',symbols,re.M) is not None +if has_gop:inspections+=list(gop_globals) +legacy_globals = tuple(('_native_legacy_'+name,size) for name,size in ( + ('stage',4),('meta_status',4),('init_result',4),('start_result',4),('probe_match',4),('parent_match',4), + ('object_size',4),('released',4),('object',8),('nub',8),('workloop',8),('guard_slot',4),('guard_object',8),('guard_caller',8), + ('surface_status',4),('surface_found',4),('video_base',8),('video_bytes',8))) +has_legacy = re.search(r'\s_native_legacy_stage$',symbols,re.M) is not None +if re.search(r'\s_native_legacy_gop_map$',symbols,re.M): + legacy_globals+=tuple(('_native_legacy_gop_'+name,size) for name,size in ( + ('map',8),('virtual',8),('physical',8),('bytes',8),('width',4),('height',4))) +if re.search(r'\s_native_legacy_transfer_calls$',symbols,re.M): + legacy_globals+=tuple(('_native_legacy_'+name,size) for name,size in (('transfer_calls',4),('transfer_status',4),('transfer_bytes',8),('transfer_request',8))) +if re.search(r'\s_native_legacy_retirement_calls$',symbols,re.M): + legacy_globals+=tuple(('_native_legacy_'+name,size) for name,size in (('retirement_calls',4),('retired_generation',4),('presented_generation',4),('retired_frames',4),('retired_current',8),('retired_pending',8))) +if re.search(r'\s_native_legacy_swap_validation_calls$',symbols,re.M): + legacy_globals+=tuple(('_native_legacy_'+name,size) for name,size in (('swap_validation_calls',4),('swap_validation_result',4),('swap_validation_reason',4),('swap_request',8),('swap_surface',8))) +if re.search(r'\s_native_legacy_pm_status$',symbols,re.M): + legacy_globals+=tuple(('_native_legacy_'+name,4) for name in ('pm_stage','pm_status','pm_calls','pm_callbacks','pm_ordinal')) +if re.search(r'\s_native_legacy_unit_scale_status$',symbols,re.M): + legacy_globals+=tuple(('_native_legacy_'+name,4) for name in ('unit_scale_status','unit_scale_calls','chosen_scale','chosen_rotation')) +has_legacy_client=re.search(r'\s_native_legacy_client_calls$',symbols,re.M) is not None +if has_legacy_client: + legacy_globals+=tuple(('_native_legacy_'+name,size) for name,size in ( + ('registration_calls',4),('properties_length',4),('client_calls',4),('client_type',4),('client_pid',4),('client_status',4),('client_task',8),('client_object',8))) + if re.search(r'\s_native_legacy_dt_attached$',symbols,re.M):legacy_globals+=(('_native_legacy_dt_attached',4),) + inspections+=[('_native_legacy_properties',4096),('_native_legacy_client_class',64)] +if has_legacy:inspections += list(legacy_globals) +ufs_globals = tuple(('_native_ufs_'+name,size) for name,size in ( + ('probe_calls',4),('matched',4),('probe_requested',4),('mapped',4),('status',4), + ('cap',4),('version',4),('hcs',4),('hce',4),('utrl_ready',4),('uic_status',4),('uic_result',4))) +has_ufs = re.search(r'\s_native_ufs_matched$',symbols,re.M) is not None +if has_ufs: inspections += list(ufs_globals) +if args.data_ramdisk and not (has_owner_data or has_ram_block): + parser.error('Scratch Data requires its explicitly enabled kernel mount or storage instrument') +if has_ram_block: + inspections += list(ram_block_globals) + [('_native_ram_block_seed_sha256', 32)] +if args.inspect_dyld_flags: + inspections += [('_dyld_flags', 8), ('_commpage_rw_addr', 8)] + if re.search(r'\s_native_owner_cache_dyld_flags$', symbols, re.M): + inspections += [('_native_owner_cache_dyld_flags', 8)] +if args.owner_root_audit or args.owner_root_accepted or args.inspect_owner_root: + inspections += [('_native_owner_root_apple_autherr', 4), ('_native_owner_root_decision', 4), ('_native_owner_root_readonly', 4), ('_native_owner_root_mount_device', 4), ('_native_owner_root_selected_device', 4), + ('_native_verified_block_root_device', 4), ('_native_verified_block_root_media_matches', 4), + ('_native_verified_block_root_provider_depth', 4), ('_native_verified_block_root_uuid_match', 4), + ('_native_verified_block_wrong_uuid_refused', 4), ('_native_verified_block_root_uuid', 37), + ('_native_owner_root_trust_enabled', 4), ('_native_owner_root_accepted', 4), ('_amfi', 8)] +if args.verified_root_probe: + inspections += [('_native_verified_root_status', 4), ('_native_verified_root_results', 32), + ('_native_verified_root_signature_calls', 4), ('_native_verified_root_block_count', 8), + ('_native_verified_root_manifest_pointer', 8), ('_amfi', 8)] +has_provider = re.search(r'\s_source_corecrypto14_status$', symbols, re.M) is not None +extra_integrity_globals = [name for name in ('_native_virtio_verified_devices', '_native_virtio_ambiguous_policies') + if re.search(r'\s' + name + '$', symbols, re.M)] +inspections += [(name, 4) for name in extra_integrity_globals] +if args.require_crypto14 and not has_provider: + raise SystemExit('This kernel has no source corecrypto 14 provider') +if has_provider: + inspections += [('_source_corecrypto14_status', 4), ('_g_crypto_funcs', 8), ('_prng_ready', 4)] +if prelink_report: + inspections += [('_g_crypto_funcs', 8), ('_crypto_init', 1), ('_prng_ready', 4)] + if prelink_report.get('early_security_initializers'): + inspections += [('_mac_late', 4)] +has_pthread = re.search(r'\s_source_pthread_status$', symbols, re.M) is not None +if has_pthread: + inspections += [('_source_pthread_status', 4), ('_pthread_functions', 8), ('_pthread519_pthread_kern', 8)] +aes_globals = ('_native_aes_kat_status', '_native_aes_interface_status', '_native_aes_registered', + '_native_aes_completed', '_native_aes_unsupported_key') +has_native_aes = re.search(r'\s_native_aes_kat_status$', symbols, re.M) is not None +if has_native_aes: + aes_globals = tuple(name for name in aes_globals if re.search(r'\s' + name + '$', symbols, re.M)) + inspections += [(name, 4) for name in aes_globals] +if args.load_apfs: + inspections += [('_native_apfs_load_status', 4), ('_native_apfs_registered', 4), ('_native_apfs_vfsops', 8)] +has_hid_probe = re.search(r'\s_native_hid_stage$', symbols, re.M) is not None +if args.load_hid_input and not has_hid_probe: + parser.error('Build the original HID initialization observation first') +if has_hid_probe: + inspections += [('_native_hid_stage',4),('_native_hid_resource_found',4),('_native_hid_load_status',4),('_native_hid_matching_load_status',4),('_native_hid_class_sizes',20)] + inspections += [('_native_hid_input_meta_status',4),('_native_hid_input_object',8),('_native_hid_input_events',32),('_native_hid_input_results',32)] +has_iogpu_probe = re.search(r'\s_native_iogpu_load_status$', symbols, re.M) is not None +if args.load_iogpu and not has_iogpu_probe: + parser.error('Build the explicit original IOGPUFamily load diagnostic first') +if has_iogpu_probe: + inspections += [('_native_iogpu_load_status', 4)] +has_mobile_graphics_probe = re.search(r'\s_native_mobile_graphics_load_status$', symbols, re.M) is not None +constant_fields = ('status', 'refusal_mask', 'output_preserved', 'written_bytes', 'padded_bytes', 'allocations_released') +constant_pointers = ('a_pointer', 'b_pointer', 'output_pointer') +has_constant_probe = re.search(r'\s_native_constants_status$', symbols, re.M) is not None +if (args.constant_uploads_probe or args.inspect_constant_uploads) and not has_constant_probe: + parser.error('Build the explicit native constant upload diagnostic archive first') +if has_constant_probe: + inspections += [('_native_constants_' + name, 4) for name in constant_fields] + inspections += [('_native_constants_' + name, 8) for name in constant_pointers] + inspections += [('_native_constants_physical', 48)] +smmu4k_fields = ('status', 'table_pages', 'leaf_pages', 'refusal_mask', 'output_preserved', 'allocations_released') +has_smmu4k_probe = re.search(r'\s_native_smmu4k_status$', symbols, re.M) is not None +if (args.smmu4k_probe or args.inspect_smmu4k) and not has_smmu4k_probe: + parser.error('Build the explicit native SMMU4K diagnostic archive first') +if has_smmu4k_probe: + inspections += [('_native_smmu4k_' + name, 4) for name in smmu4k_fields] + inspections += [('_native_smmu4k_' + name, 8) for name in ('pool_pointer', 'data_pointer', 'cpu_mair')] + inspections += [('_native_smmu4k_pool_physical', 48), ('_native_smmu4k_data_physical', 16)] +pm4_fields = ('status', 'packets', 'refusal_mask', 'output_preserved', 'bit_refusals', 'input_preserved') +has_pm4_probe = re.search(r'\s_native_adreno_pm4_status$', symbols, re.M) is not None +if args.adreno_pm4_probe and not has_pm4_probe: + parser.error('Build the explicit native Adreno PM4 diagnostic archive first') +if has_pm4_probe: + inspections += [('_native_adreno_pm4_' + name, 4) for name in pm4_fields] +has_adreno_probe = re.search(r'\s_native_adreno_firmware_status$', symbols, re.M) is not None +if (args.adreno_firmware_probe or args.inspect_adreno_firmware) and not has_adreno_probe: + parser.error('Build the explicit native Adreno firmware diagnostic archive first') +adreno_fields = ('status', 'records', 'payload_bytes', 'refusal_mask', 'output_preserved') +if has_adreno_probe: + inspections += [('_native_adreno_firmware_' + name, 4) for name in adreno_fields] + inspections += [('_native_adreno_firmware_sha256', 32), ('_native_adreno_firmware_pointer', 8), + ('_native_adreno_firmware_image_size', 8)] +if has_mobile_graphics_probe: + inspections += [('_native_mobile_graphics_load_status', 4)] +has_mobile_client_probe = re.search(r'\s_native_mobile_graphics_client_results$', symbols, re.M) is not None +if re.search(r'\s_native_mobile_provider_stage$', symbols, re.M): + inspections += [(name, size) for name, size in ( + ('_native_mobile_provider_stage', 4), ('_native_mobile_provider_meta_status', 4), + ('_native_mobile_provider_query_status', 4), ('_native_mobile_provider_object', 8), + ('_native_mobile_provider_client', 8), ('_native_mobile_provider_dimensions', 16), + ('_native_mobile_provider_expected', 16), ('_native_mobile_provider_parent_match', 4), + ('_native_mobile_provider_released', 4))] +mobile_client_fields = ('abstract_null_mask', 'client_object', 'client_vptr', 'client_meta', + 'client_size', 'client_init', 'client_released') +if has_mobile_client_probe: + inspections += [('_native_mobile_graphics_' + name, 8 if name in ('client_object', 'client_vptr', 'client_meta') else 4) + for name in mobile_client_fields] + inspections += [('_native_mobile_graphics_client_results', 12)] +if block_image and re.search(r'\s_native_virtio_manifest_ready$', symbols, re.M): + inspections += [('_native_virtio_manifest_ready', 4), ('_native_virtio_verified_blocks', 8), + ('_native_virtio_integrity_failures', 4), ('_native_virtio_verified_manifest_pointer', 8), + ('_native_virtio_verified_key_pointer', 8)] +if block_image: + inspections += [('_native_virtio_ready', 4), ('_native_virtio_read_requests', 4), ('_native_virtio_read_failures', 4), + ('_native_virtio_sectors', 8), ('_native_virtio_mmio_physical', 8), ('_native_virtio_first_sector_sha256', 32)] +if args.native_user: + inspections += [('_amfi', 8), ('_cs_validate_page_bad_hash', 8), ('_cs_validate_page_no_hash', 8)] +native_hash_cache = re.search(r'\s_cs_native4k_hash_cache_version$', symbols, re.M) is not None +if args.mutate_own_vnode_page is not None or cache_fault or main_fault: + if not native_hash_cache: + raise SystemExit('Mapped-page fault injection requires the native integrity-cache kernel') + entry = re.search(r'^([0-9a-fA-F]+)\s+\S\s+_cs_validate_hash$', symbols, re.M) + if not entry: + raise SystemExit('Cannot resolve actual page-integrity validation') + image_bytes = main_bytes if main_fault else (args.cache_page_source if cache_fault else payloads / 'launchd.macho').read_bytes() + expected_cache_sha = (leaf_record['cache_leaf_input']['file_sha256'] if args.follow_cache_leaf else + '3d923c404fe519184a99042b60a4cb3d37247dcb8b070242ed3e670d4d523d04') + if cache_fault and hashlib.sha256(image_bytes).hexdigest() != expected_cache_sha: + raise SystemExit('Expected the exact unchanged supplied cache input identified by this probe') + offset = main_input['entry_offset'] if main_fault else args.mutate_cache_vnode_page if cache_fault else args.mutate_own_vnode_page + page = offset & ~4095 + if offset < 0 or page + 4096 > len(image_bytes): + raise SystemExit('Select a byte in a complete own fixture page') + address = int(entry[1], 16) + trace_symbols[address] = '_cs_validate_hash' + gdb_command.extend(['--trace-address', hex(address), '--mutate-mapped-vnode-page', + f'{address:x}:{page:x}:{offset & 4095:x}:{hashlib.sha256(image_bytes[page:page + 4096]).hexdigest()}']) + if cache_fault or main_fault: + dwarf = subprocess.check_output(['xcrun', 'dwarfdump', '--name=cs_blob', '--show-children', + ROOT / 'build/xnu/ios-obj/DEVELOPMENT_ARM64_BCM2837/bsd/DEVELOPMENT/ubc_subr.o'], text=True) + field = re.search(r'DW_AT_name\s+\("csb_cdhash"\)[\s\S]*?DW_AT_data_member_location\s+\((0x[0-9a-f]+)\)', dwarf) + if main_fault: + original_record = json.loads((ROOT / 'build/target-dyld-fixture/inputs.json').read_text()) + if original_record['files'][0]['sha256'] != hashlib.sha256(image_bytes).hexdigest(): + raise SystemExit('Require the exact unchanged supplied launchd signing identity') + probe_metadata = dict(original_file_sha256=original_record['files'][0]['sha256'], original_cdhash=original_record['original_cdhashes'][0]) + gdb_command += ['--mapped-vnode-max-hits', '4096'] + else: + probe_metadata = (dict(original_file_sha256=leaf_record['cache_leaf_input']['file_sha256'], + original_cdhash=leaf_record['cache_leaf_input']['original_cdhash']) if args.follow_cache_leaf else + json.loads((args.native_user_image.parent / 'file-probe.json').read_text())) + if not field or probe_metadata['original_file_sha256'] != hashlib.sha256(image_bytes).hexdigest(): + raise SystemExit('Require actual compiled cs_blob layout and exact original cache probe identity') + gdb_command += ['--mapped-vnode-cdhash', f'{int(field[1], 16):x}:' + probe_metadata['original_cdhash']] +if args.mutate_own_ramdisk_after_auth is not None: + if not native_hash_cache: + raise SystemExit('Post-authentication mutation requires the native integrity-cache kernel') + entry = re.search(r'^([0-9a-fA-F]+)\s+\S\s+_cs_native4k_prepare$', symbols, re.M) + md0 = re.search(r'^([0-9a-fA-F]+)\s+\S\s+_mdev$', symbols, re.M) + if not entry or not md0: + raise SystemExit('Cannot resolve actual authentication/md0 state symbols') + image_bytes = (payloads / 'launchd.macho').read_bytes() + address = int(entry[1], 16) + trace_symbols[address] = '_cs_native4k_prepare' + gdb_command.extend(['--trace-address', hex(address), '--mutate-md0-after-return', + f'{address:x}:{md0[1]}:{len(image_bytes):x}:{hashlib.sha256(image_bytes).hexdigest()}:{args.mutate_own_ramdisk_after_auth:x}']) +if native_hash_cache: + inspections += [('_cs_native4k_hash_cache_version', 4), ('_cs_native4k_full_blocks', 8), + ('_cs_native4k_page_checks', 8), ('_cs_native4k_ready_blobs', 8)] +if args.armvirt and re.search(r'\s_armvirt_platform_configured$', symbols, re.M): + inspections += [('_armvirt_platform_configured', 4), ('_armvirt_platform_child_count', 4), + ('_armvirt_platform_cpu_count', 4), ('_armvirt_platform_bus_count', 4)] + if re.search(r'\s_armvirt_platform_defaults_count$', symbols, re.M): + inspections += [('_armvirt_platform_defaults_count', 4)] +if re.search(r'\s_armdt_platform_configured$', symbols, re.M): + inspections += [('_armdt_platform_configured', 4), ('_armdt_platform_child_count', 4), + ('_armdt_platform_cpu_count', 4), ('_armdt_platform_bus_count', 4), + ('_armdt_platform_defaults_count', 4), ('_armdt_platform_nvram_proxy', 4)] + if re.search(r'\s_armdt_platform_device_count$', symbols, re.M): + inspections += [('_armdt_platform_device_count', 4)] +if args.armcpu: + inspections += [('_armcpu_registered', 4), ('_armcpu_gic_ready', 4), ('_armcpu_timer_interrupts', 8), + ('_armcpu_ipi_interrupts', 8), ('_gCPUsRunning', 1)] + if args.smp > 1: + inspections += [('_armcpu_smp_running', 4), ('_armcpu_smp_step', 32), ('_armcpu_smp_psci_rc', 32), ('_armcpu_smp_conduit', 4)] +inspections = list(dict.fromkeys(inspections)) +pgz_probe_globals = [('case', 4), ('stage', 4), ('zone', 4), ('expected_zone', 4), ('metadata', 4), + ('backing', 8), ('alias', 8), ('argument', 8)] +if args.zone_pgz_probe_case: + inspections += [('_native_zone_pgz_probe_' + name, width) for name, width in pgz_probe_globals] +pgz_fix_counter = re.search(r'\s_native_zone_pgz_alignment_decodes$', symbols, re.M) is not None +if pgz_fix_counter: + inspections += [('_native_zone_pgz_alignment_decodes', 8)] +video_layout = None +if args.inspect_boot_video: + obj = ROOT / 'build/xnu/ios-obj/DEVELOPMENT_ARM64_BCM2837/pexpert/DEVELOPMENT/pe_init.o' + state_dwarf = subprocess.check_output(['xcrun', 'dwarfdump', '--name=PE_state', '--show-children', obj], text=True) + video_dwarf = subprocess.check_output(['xcrun', 'dwarfdump', '--name=PE_Video', '--show-children', obj], text=True) + def member_offset(text, name): + match = re.search(r'DW_TAG_member[\s\S]*?DW_AT_name\s+\("' + re.escape(name) + r'"\)[\s\S]*?DW_AT_data_member_location\s+\((0x[0-9a-f]+)\)', text) + if not match: + raise SystemExit('Cannot derive actual PE video layout from compiler DWARF') + return int(match[1], 16) + size = re.search(r'DW_TAG_structure_type[\s\S]*?DW_AT_byte_size\s+\((0x[0-9a-f]+)\)', state_dwarf) + if not size: + raise SystemExit('Cannot derive actual PE state size') + video_layout = dict(state_size=int(size[1], 16), video=member_offset(state_dwarf, 'video'), + boot_args=member_offset(state_dwarf, 'bootArgs'), + fields={n: member_offset(video_dwarf, n) for n in + ('v_baseAddr', 'v_rowBytes', 'v_width', 'v_height', 'v_depth', 'v_display')}, + object_sha256=hashlib.sha256(obj.read_bytes()).hexdigest()) + inspections += [('_PE_state', video_layout['state_size'])] +if args.inspect_sheng_pmgr: + inspections += [('_sheng_pmgr_observation', 16 * 8)] +has_boot_progress = re.search(r'\s_boot_progress_observation$', symbols, re.M) is not None +if has_boot_progress: + inspections += [('_boot_progress_observation', 24 * 8), ('_boot_progress_frozen', 4)] + if re.search(r'\s_boot_progress_display_owned$', symbols, re.M): + inspections += [('_boot_progress_display_owned', 4)] +for name, width in inspections: + match = re.search(r'^([0-9a-fA-F]+)\s+\S\s+' + re.escape(name) + '$', symbols, re.M) + if not match: + raise SystemExit(f'Cannot resolve {name}') + request = f'{match[1]}:{width:x}' + page_globals[name] = request + gdb_command.extend(['--read-memory', request]) +if video_layout: + video_args_request = f"{int(page_globals['_PE_state'].split(':')[0], 16) + video_layout['boot_args']:x}:28:30" + gdb_command += ['--read-indirect', video_args_request] +gop_captures=[];gop_spans=[] +if args.inspect_gop_scanout_bytes: + size=args.inspect_gop_scanout_bytes + if not has_gop or not video_layout or not 4096<=size<=4*1024*1024:parser.error('Scanout observation requires actual presenter/video and a bounded 4 KiB..4 MiB extent') + pointer=page_globals['_native_legacy_gop_virtual' if args.legacy_display_transfer else '_native_gop_virtual'].split(':')[0] + for offset in sorted({0,(size//2)&~4095,size-4096}): + request=f'{pointer}:{offset:x}:1000';gop_captures.append(request);gdb_command+=['--read-indirect',request] + for offset in range(0,size,1024*1024): + request=f'{pointer}:{min(size-offset,1024*1024):x}:{offset:x}';gop_spans.append(request);gdb_command+=['--inspect-indirect-kernel-range',request] +constant_captures = {} +if args.inspect_constant_uploads: + gdb_command += ['--allow-unmapped-kernel-pages'] + for label in ('a', 'b', 'output'): + address = page_globals['_native_constants_' + label + '_pointer'].split(':')[0] + span = address + ':2000' + capture = address + ':0:1000' + gdb_command += ['--read-indirect', capture, '--inspect-indirect-kernel-range', span] + constant_captures[label] = dict(span=span, capture=capture) +smmu4k_captures = {} +if args.inspect_smmu4k: + for label, pages in (('pool', 6), ('data', 2)): + address = page_globals['_native_smmu4k_' + label + '_pointer'].split(':')[0] + span = address + ':' + f'{pages * 4096:x}' + captures = [address + f':{i * 4096:x}:1000' for i in range(pages)] + for request in captures: + gdb_command += ['--read-indirect', request] + gdb_command += ['--inspect-indirect-kernel-range', span] + smmu4k_captures[label] = dict(span=span, chunks=captures) +if args.inspect_adreno_firmware: + adreno_input_request = page_globals['_native_adreno_firmware_pointer'].split(':')[0] + ':0:20' + gdb_command += ['--read-indirect', adreno_input_request] + match = re.search(r'^([0-9a-fA-F]+)\s+\S\s+_native_adreno_firmware_image$', symbols, re.M) + if not match: + raise SystemExit('Cannot resolve the actual constant GMU input') + adreno_input_bytes = json.loads((ROOT / 'scripts/data/adreno-inputs.json').read_text())['firmware']['files']['qcom/gmu_gen70200.bin']['bytes'] + adreno_input_range = f'{int(match[1], 16):x}:{adreno_input_bytes:x}' + gdb_command += ['--inspect-kernel-range', adreno_input_range] +if block_image and '_native_virtio_verified_manifest_pointer' in page_globals and not args.skip_policy_pointer_reads: + for name in ('_native_virtio_verified_manifest_pointer', '_native_virtio_verified_key_pointer'): + gdb_command += ['--read-indirect', page_globals[name].split(':')[0] + ':0:20'] +if args.inspect_dyld_flags: + flags_request = page_globals['_commpage_rw_addr'].split(':')[0] + ':160:8' + gdb_command += ['--read-indirect', flags_request] +user_requests = {} +if args.native_user: + user_inspections = [('_results', 32), ('_user_page_size', 8), ('_old_length', 8)] + if '_fp_context_results' in user_addresses: + user_inspections += [('_fp_context_results',80),('_fp_context_actual',512),('_fp_context_expected',512)] + if '_file_probe_results' in user_addresses: + user_inspections += [('_file_probe_results', 80), ('_file_signature', 24)] + if '_cache_leaf_results' in user_addresses: + user_inspections += [('_cache_leaf_results', 72), ('_cache_leaf_target', 8)] + if '_apfs_launchd_results' in user_addresses: + user_inspections += [('_apfs_launchd_results', 104)] + if '_apfs_mount_results' in user_addresses: + user_inspections += [('_apfs_mount_results', 40)] + for name, length in user_inspections: + request = f'{user_addresses[name]:x}:{length:x}' + user_requests[name] = request + gdb_command.extend(['--read-memory', request]) +commpage_requests = {} +if args.check_commpage: + for name in ('_commpage_rw_addr', '_commpage_kernel_ro_addr'): + match = re.search(r'^([0-9a-fA-F]+)\s+\S\s+' + name + '$', symbols, re.M) + if not match: + raise SystemExit(f'Cannot resolve {name}') + request = f'{match[1]}:25:1' + commpage_requests[name] = request + gdb_command.extend(['--read-indirect', request]) +if args.live_hid_dir:gdb_command.extend(['--live-hid-dir',str(args.live_hid_dir.resolve())]) +subprocess.run(gdb_command, check=True) +report = json.loads((output / 'gdb-stop.json').read_text()) +if has_hid_probe: + report['native_hid'] = {name: int.from_bytes(bytes.fromhex(report['memory'][page_globals['_native_hid_'+name]]),'little',signed=name.endswith('load_status')) + for name in ('stage','resource_found','load_status','matching_load_status')} + report['native_hid']['class_sizes'] = list(struct.unpack('<5I',bytes.fromhex(report['memory'][page_globals['_native_hid_class_sizes']]))) + for name in ('input_meta_status','input_object'): + report['native_hid'][name]=int.from_bytes(bytes.fromhex(report['memory'][page_globals['_native_hid_'+name]]),'little',signed=name=='input_meta_status') + for name,fmt in (('input_events','<8I'),('input_results','<8i')): + report['native_hid'][name]=list(struct.unpack(fmt,bytes.fromhex(report['memory'][page_globals['_native_hid_'+name]]))) +if has_boot_progress: + values = struct.unpack('<24Q', bytes.fromhex(report['memory'][page_globals['_boot_progress_observation']])) + report['boot_progress'] = dict(zip(('enabled','bound','seconds','activity_at','irqs','monitor_ticks', + 'dropped','reason','reads','verified','failures','last_pc','last_psr','last_sp','tcr','ttbr0','ttbr1', + 'sctlr','stage_records','log_lines','width','height','stride','frame'), values)) + report['boot_progress']['frozen'] = int.from_bytes(bytes.fromhex(report['memory'][page_globals['_boot_progress_frozen']]), 'little') + if '_boot_progress_display_owned' in page_globals: + report['boot_progress']['display_owned'] = int.from_bytes(bytes.fromhex(report['memory'][page_globals['_boot_progress_display_owned']]), 'little') +if args.zone_pgz_probe_case: + report['actual_zone_pgz_probe'] = {name: int.from_bytes(bytes.fromhex(report['memory'][page_globals['_native_zone_pgz_probe_' + name]]), 'little') + for name, _ in pgz_probe_globals} +if pgz_fix_counter: + report['actual_zone_pgz_alignment_decodes'] = int.from_bytes(bytes.fromhex(report['memory'][page_globals['_native_zone_pgz_alignment_decodes']]), 'little') +if video_layout: + pe = bytes.fromhex(report['memory'][page_globals['_PE_state']]) + boot = bytes.fromhex(report['indirect_memory'][video_args_request]['bytes']) + report['actual_boot_video'] = dict(qemu_ramfb_attached=args.qemu_ramfb, compiled_layout=video_layout, + boot_args=dict(zip(('physical_base', 'display', 'row_bytes', 'width', 'height', 'depth'), struct.unpack('<6Q', boot))), + pe_video={name: struct.unpack_from('> 14 & 3) != 0 or (tcr >> 30 & 3) != 2 or not registers['SCTLR_EL1'] & 1: + raise SystemExit('FAIL: active native 4K translation was not observed') +if not registers['TTBR0_EL1'] or not registers['TTBR1_EL1']: + raise SystemExit('FAIL: active translation bases are missing') +uart = (output / 'gdb-uart.log').read_text(errors='replace') +if 'DARWINBOOT: inputs validated; exiting boot services' not in uart or 'ASSERT' in uart: + raise SystemExit('FAIL: the UEFI loader path was not established') +for event in report.get('trace_events', []): + event['symbol'] = trace_symbols[int(event['pc'], 16)] +if args.qemu_accel == 'hvf': + report['actual_panic_region'] = {name: int.from_bytes(bytes.fromhex(report['memory'][page_globals[name]]), 'little') + for name in ('_gPanicBase', '_gPanicSize', '_panic_info')} +if args.qemu_accel == 'hvf': + report['actual_initproc_pointer'] = int.from_bytes(bytes.fromhex(report['memory'][page_globals['_initproc']]), 'little') + if args.break_symbol == '_posix_spawn' and 'function_entry_arguments' in report: + report['spawn_parent_is_initproc'] = int(report['function_entry_arguments'][0], 16) == report['actual_initproc_pointer'] + report['actual_physical_memory'] = {name: int.from_bytes(bytes.fromhex(report['memory'][page_globals[name]]), 'little') + for name in ('_gPhysBase', '_gPhysSize', '_mem_size', '_mem_actual')} + report['actual_signature_failures'] = {name: int.from_bytes(bytes.fromhex(report['memory'][page_globals[name]]), 'little') + for name in ('_cs_validate_page_bad_hash', '_cs_validate_page_no_hash')} +report['native_4k_mmu_observed'] = True +report['ipados_userspace_boot_demonstrated'] = False +report['functional_ipados_boot_demonstrated'] = False +report['sheng_hardware_tested'] = False +if args.inspect_compression: + report['compression_registrations'] = {kind: report['indirect_memory'][request] + for kind, request in compression_requests.items()} +if args.inspect_sheng_pmgr: + values = struct.unpack('<16Q', bytes.fromhex(report['memory'][page_globals['_sheng_pmgr_observation']])) + report['sheng_pmgr_observation'] = dict(zip(('stage', 'status', 'parent_size', 'own_size', + 'provider', 'nub', 'bound_provider', 'read_before', 'read_after', 'own_before', + 'own_after', 'nub_before', 'nub_after', 'attached', 'registered', 'mode'), values)) +report['kernel_page_globals'] = {name: int.from_bytes(bytes.fromhex(report['memory'][request]), 'little') + for name, request in page_globals.items() if name in dict(inspections[:4])} +for name, value in report['kernel_page_globals'].items(): + if value != (4096 if name == '_page_size' else 12): + raise SystemExit(f'FAIL: {name} reports {value}, inconsistent with native 4K policy') +text_symbols = sorted((int(address, 16), name) for address, name in + re.findall(r'^([0-9a-fA-F]+)\s+\S\s+(_\S+)$', symbols, re.M)) +fault_pc = int(report['x1'], 16) +preceding = [(address, name) for address, name in text_symbols if address <= fault_pc] +if args.break_symbol == '_panic': + report['panic_x1_nearest_text_symbol'] = preceding[-1][1] if preceding else None +if report.get('argument_memory', {}).get('x0', '').startswith('%s at pc'): + arguments = struct.unpack_from('<4Q', bytes.fromhex(report['stack_bytes'])) + report['kernel_exception'] = {'pc': hex(arguments[1]), 'lr': hex(arguments[2])} + preceding = [(address, name) for address, name in text_symbols if address <= arguments[1]] + if preceding: + report['kernel_exception']['nearest_symbol'] = preceding[-1][1] + report['kernel_exception']['symbol_offset'] = hex(arguments[1] - preceding[-1][0]) + saved = report.get('stack_pointer_memory', {}).get('3', {}).get('bytes', '') + if len(saved) >= 592 and not saved.startswith('E'): + state = bytes.fromhex(saved) + if struct.unpack_from('8:raise SystemExit('CPU context observer capacity exceeded') + raw=bytes.fromhex(report['memory'][page_globals['_native_gpu_broker_cpu_contexts']]);contexts=[] + for index in range(count): + v=struct.unpack_from('<8Q',raw,index*64) + contexts.append(dict(pid=v[0],session=v[1],TCR_EL1=hex(v[2]),TTBR0_EL1=hex(v[3]),TTBR1_EL1=hex(v[4]),SCTLR_EL1=hex(v[5]),task=hex(v[6]),native_cpu_4k_accepted=bool(v[7]))) + report['native_gpu_broker']['actual_cpu_contexts']=contexts +if has_gpu_shared_contexts: + report['native_gpu_shared_events']={name.removeprefix('_native_gpu_shared_'):int.from_bytes(bytes.fromhex(report['memory'][page_globals[name]]),'little') for name in gpu_shared_globals} +if has_gop: + report['native_gop_presenter'] = {name.removeprefix('_native_gop_'):int.from_bytes(bytes.fromhex(report['memory'][page_globals[name]]),'little') + for name,_ in gop_globals} +if args.inspect_gop_scanout_bytes: + expected=args.inspect_gop_scanout_bytes;video=report['actual_boot_video']['pe_video'] + if video['v_rowBytes']*video['v_height']!=expected:raise SystemExit('Actual GOP extent differs from requested inspection') + report['gop_scanout_observation']=dict(provider='original-legacy-gop' if args.legacy_display_transfer else 'own-gop-presenter',bytes=expected,captures=[report['indirect_memory'][r] for r in gop_captures], + translations=[report['kernel_range_translations'][r] for r in gop_spans]) +if has_legacy: + report['native_legacy_display'] = {name.removeprefix('_native_legacy_'):int.from_bytes(bytes.fromhex(report['memory'][page_globals[name]]),'little',signed=name.endswith('_status')) + for name,_ in legacy_globals} +if has_legacy_client: + report['native_legacy_properties']=bytes.fromhex(report['memory'][page_globals['_native_legacy_properties']]).split(b'\0',1)[0].decode('utf8','replace') + report['native_legacy_client_class']=bytes.fromhex(report['memory'][page_globals['_native_legacy_client_class']]).split(b'\0',1)[0].decode('utf8','replace') +if has_ufs: + report['native_ufs']={name.removeprefix('_native_ufs_'):int.from_bytes(bytes.fromhex(report['memory'][page_globals[name]]),'little') + for name,_ in ufs_globals} +if '_native_virtio_manifest_ready' in page_globals: + report['native_virtio_integrity'] = {name: int.from_bytes(bytes.fromhex(report['memory'][page_globals[name]]), 'little') + for name in ('_native_virtio_manifest_ready', '_native_virtio_verified_blocks', '_native_virtio_integrity_failures', + '_native_virtio_verified_manifest_pointer', '_native_virtio_verified_key_pointer')} + report['native_virtio_integrity'].update({name: int.from_bytes(bytes.fromhex(report['memory'][page_globals[name]]), 'little') + for name in extra_integrity_globals}) +if native_hash_cache: + report['native_signature_hash_cache'] = { + name: int.from_bytes(bytes.fromhex(report['memory'][page_globals[name]]), 'little') + for name in ('_cs_native4k_hash_cache_version', '_cs_native4k_full_blocks', + '_cs_native4k_page_checks', '_cs_native4k_ready_blobs')} +if any(name.startswith('_armvirt_platform_') for name, _ in inspections): + report['armvirt_globals'] = {name: int.from_bytes(bytes.fromhex(report['memory'][page_globals[name]]), 'little') + for name, _ in inspections if name.startswith('_armvirt_platform_')} +if any(name.startswith('_armdt_platform_') for name, _ in inspections): + report['armdt_globals'] = {name: int.from_bytes(bytes.fromhex(report['memory'][page_globals[name]]), 'little') + for name, _ in inspections if name.startswith('_armdt_platform_')} +if args.armcpu: + report['armcpu_globals'] = {name: int.from_bytes(bytes.fromhex(report['memory'][page_globals[name]]), 'little') + for name in ('_armcpu_registered', '_armcpu_gic_ready', '_armcpu_timer_interrupts', + '_armcpu_ipi_interrupts', '_gCPUsRunning')} + if args.smp > 1: + report['armcpu_smp'] = {'running': int.from_bytes(bytes.fromhex(report['memory'][page_globals['_armcpu_smp_running']]), 'little'), + 'conduit': int.from_bytes(bytes.fromhex(report['memory'][page_globals['_armcpu_smp_conduit']]), 'little'), + 'step': list(struct.unpack('<8I', bytes.fromhex(report['memory'][page_globals['_armcpu_smp_step']]))), + 'psci_rc': list(struct.unpack('<8i', bytes.fromhex(report['memory'][page_globals['_armcpu_smp_psci_rc']])))} +if args.native_user: + report['amfi_interface_pointer'] = int.from_bytes(bytes.fromhex(report['memory'][page_globals['_amfi']]), 'little') + report['code_signing_hash_counters'] = { + name: int.from_bytes(bytes.fromhex(report['memory'][page_globals[name]]), 'little') + for name in ('_cs_validate_page_bad_hash', '_cs_validate_page_no_hash')} + if prelink_report: + pointer = report['amfi_interface_pointer'] + report['original_amfi_table_owner_observed'] = any( + e['module'] == 'com.apple.driver.AppleMobileFileIntegrity' + and e['segment'] == '__DATA_CONST' and e['va'] <= pointer < e['va'] + e['size'] + for e in prelink_report['extents']) + report['native_user_memory'] = {name: report['memory'][request] for name, request in user_requests.items()} + report['native_user_el0_observed'] = (actual_target and (args.break_symbol == '_native_user_checkpoint' or args.follow_cache_leaf) + and int(report['pstate'], 16) & 15 == 0) + if report['native_user_el0_observed']: + pid, sysctl_result, pid_flags, sysctl_flags = struct.unpack('<4Q', bytes.fromhex(report['native_user_memory']['_results'])) + page_size = int.from_bytes(bytes.fromhex(report['native_user_memory']['_user_page_size']), 'little') + old_length = int.from_bytes(bytes.fromhex(report['native_user_memory']['_old_length']), 'little') + report['native_user_syscalls'] = {'pid': pid, 'sysctl_result': sysctl_result, + 'page_size': page_size, 'old_length': old_length, + 'getpid_carry': (pid_flags >> 29) & 1, + 'sysctl_carry': (sysctl_flags >> 29) & 1} + report['native_user_4k_abi_observed'] = (pid == 1 and sysctl_result == 0 and page_size == 4096 + and old_length == 4 and not ((pid_flags | sysctl_flags) & (1 << 29))) + if '_file_probe_results' in user_requests: + values = struct.unpack('<10Q', bytes.fromhex(report['native_user_memory']['_file_probe_results'])) + report['native_file_probe'] = dict(zip(('open_result', 'open_nzcv', 'fcntl_result', 'fcntl_nzcv', + 'mmap_result', 'mmap_nzcv', 'native_pages_read', 'page_word_xor', 'signature_end', 'stage'), values)) + if '_cache_leaf_results' in user_requests: + values = struct.unpack('<9Q', bytes.fromhex(report['native_user_memory']['_cache_leaf_results'])) + report['native_cache_leaf_pre_call'] = dict(zip(('open_result', 'open_nzcv', 'fcntl_result', 'fcntl_nzcv', + 'mmap_result', 'mmap_nzcv', 'stage', 'stored_return', 'stored_return_nzcv'), values)) + if '_apfs_launchd_results' in user_requests: + values = struct.unpack('<13Q', bytes.fromhex(report['native_user_memory']['_apfs_launchd_results'])) + report['apfs_launchd_component'] = dict(zip(('mkdir', 'mkdir_nzcv', 'os_mount', 'os_mount_nzcv', + 'cache_mount', 'cache_mount_nzcv', 'chroot', 'chroot_nzcv', 'chdir', 'chdir_nzcv', + 'exec_return', 'exec_nzcv', 'stage'), values)) + if '_apfs_mount_results' in user_requests: + values = struct.unpack('<5Q', bytes.fromhex(report['native_user_memory']['_apfs_mount_results'])) + report['native_apfs_data_mount'] = dict(zip(('mkdir_result', 'mkdir_nzcv', 'mount_result', 'mount_nzcv', 'stage'), values)) +if args.check_commpage: + report['commpage_user64_shifts'] = {name: int(report['indirect_memory'][request]['bytes'], 16) + for name, request in commpage_requests.items()} + if any(shift != 12 for shift in report['commpage_user64_shifts'].values()): + raise SystemExit(f"Incorrect native user ABI: {report['commpage_user64_shifts']}") +(output / 'gdb-stop.json').write_text(json.dumps(report, indent=2) + '\n') +print('PASS: real UEFI -> CrabApple -> DarwinBoot -> active native 4K XNU translation') +print(json.dumps(report['kernel_page_globals'], indent=2)) +if has_provider: + print(json.dumps(report['crypto14_globals'], indent=2)) +if 'armvirt_globals' in report: + print(json.dumps(report['armvirt_globals'], indent=2)) +if 'armdt_globals' in report: + print(json.dumps(report['armdt_globals'], indent=2)) +if args.armcpu: + print(json.dumps(report['armcpu_globals'], indent=2)) +if prelink_report: + print(json.dumps(report['prelinked_crypto_globals'], indent=2)) +if has_pthread: + print(json.dumps(report['pthread_globals'], indent=2)) +if args.follow_display_query or args.follow_display_discovery: + print(f'Observed {report["stop_symbol"]}; functional iPadOS and sheng remain unverified') +elif report.get('original_launchd_main_executed'): + print('Observed original launchd main executing at native-4K EL0; functional iPadOS and sheng remain unverified') +else: + print(f'Observed {report["stop_symbol"]}; functional iPadOS remains unverified') +if report.get('native_file_probe'): + print('Own file syscall probe:', json.dumps(report['native_file_probe'])) +if not actual_target: + raise SystemExit(f'Unexpected {report["stop_symbol"]} before {args.break_symbol}; inspect captured state') + +if args.native_user and args.break_symbol == '_native_user_checkpoint' and not report.get('native_user_4k_abi_observed'): + raise SystemExit('Native EL0/SVC/page-size ABI proof failed; inspect captured state') + +if prelink_report and not report['original_crypto_initialized_observed']: + raise SystemExit('The original prelinked crypto provider has not initialized') + +if args.follow_cache_leaf and not report.get('original_cache_leaf_executed'): + raise SystemExit('Original 16K-signed cache-code execution is not established') + +if args.verified_root_probe and report['owner_root_crypto_probe']['_native_verified_root_status'] != 1: + raise SystemExit('Real native owner signature and data-hash verification has not passed') +if args.break_symbol == '_native_owner_cache_checkpoint' and report['owner_cache_mount']['_native_owner_cache_status'] != 0: + raise SystemExit('The real original SystemOS mount or owner media binding refused; inspect the actual result') diff --git a/scripts/probe-keybag-bypass.py b/scripts/probe-keybag-bypass.py new file mode 100644 index 0000000..5ebdd92 --- /dev/null +++ b/scripts/probe-keybag-bypass.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python3 +"""Reproduce and qualify the real shared-Data keybag path offline. + +This runs the exact 29-component candidate with a writable APFS Data fixture +and the genuine Apple boot argument `-apfs_shared_datavolume`. It verifies +that keybagd takes its no-system-keybag allow-list path instead of the fatal +recovery path, and records the next real userspace input failure. QEMU/HVF +only: no sheng hardware, SEP, keybag, UI or desktop is claimed. +""" +import argparse +import hashlib +import json +from pathlib import Path +import plistlib +import shutil +import subprocess +import sys + +ROOT = Path(__file__).resolve().parents[1] +BASE = ROOT / 'build/native-prelink/native-owner-armdt-nubs-candidate' +DATA_SEED = ROOT / 'build/offline-renderbox-present-2/compiler-data-32.cdr' +DATA = ROOT / 'build/offline-renderbox-present-2/compiler-data-32-keybags.cdr' +OUTPUT = ROOT / 'build/darwinboot-armdt-keybag-bypass' +CHECKPOINT = ROOT / 'build/armdt-platform/keybag-bypass.json' + +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument('--check-existing', action='store_true') +parser.add_argument('--output', type=Path, default=OUTPUT) +args = parser.parse_args() + +if not BASE.with_suffix('.macho').exists() or not BASE.with_suffix('.json').exists(): + raise SystemExit('Missing the native owner ArmDT candidate') +plan = json.loads(BASE.with_suffix('.json').read_text()) +if hashlib.sha256(BASE.with_suffix('.macho').read_bytes()).hexdigest() != plan['candidate_sha256']: + raise SystemExit('Candidate report does not identify this exact image') + + +def ensure_data_fixture(): + if DATA.exists(): + return + shutil.copyfile(DATA_SEED, DATA) + mount = ROOT / 'build/offline-renderbox-present-2/data-keybags-mount' + mount.mkdir(parents=True, exist_ok=True) + info = plistlib.loads(subprocess.check_output( + ['hdiutil', 'attach', '-nobrowse', '-owners', 'off', '-mountpoint', str(mount), + str(DATA), '-plist'])) + device = info['system-entities'][0]['dev-entry'] + try: + (mount / 'keybags/backup').mkdir(parents=True, exist_ok=True) + finally: + subprocess.run(['hdiutil', 'detach', device], check=True, capture_output=True) + mount.rmdir() + + +def run_probe(): + if args.output.exists(): + raise SystemExit(f'Preserve the existing observation: {args.output}') + ensure_data_fixture() + command = [ + sys.executable, str(ROOT / 'scripts/probe-darwinboot-xnu.py'), + '--kernel', str(BASE.with_suffix('.macho')), + '--prelink-report', str(BASE.with_suffix('.json')), + '--output', str(args.output), + '--virtio-udif-image', '.cache/firmware/target-volumes/044-76040-038.dmg', + '--virtio-extra-block-image', '.cache/firmware/target-volumes/044-76034-038.dmg', + '--virtio-mmio-fdt', 'build/virtio-block-fixture/qemu-virt.dtb', + '--root-match-volume-uuid', '04769D26-2CA0-4825-B443-E2E5F88300F2', + '--trust-cache', 'build/original-static-system-trust/original-static-system.bundle', + '--armvirt', '--armdt-platform', '--armcpu', '--qemu-vmm-present', + '--qemu-accel', 'hvf', '--owned-memory-size', '0x20000000', '--nvram-proxy', + '--crypto-hash-method', 'sha2-384', '--amfi-platform-only', '--load-apfs', + '--inspect-owner-root', '--inspect-dyld-flags', '--check-commpage', + '--skip-initializer-traces', '--timeout', '60', + '--data-ramdisk', str(DATA), + '--kernel-extra-args=-apfs_shared_datavolume', + '--log-syscalls-after', '_posix_spawn', + '--log-syscall-number', '4', '--log-syscall-number', '397', + '--log-syscall-number', '1', '--log-syscall-number', '59', + '--log-syscall-number', '10', '--log-syscall-number', '57', + '--log-syscall-number', '128', '--log-syscall-number', '136', + ] + subprocess.run(command, cwd=ROOT, check=True, timeout=240) + + +if not args.check_existing: + run_probe() + +report_path = args.output / 'gdb-stop.json' +if not report_path.exists(): + raise SystemExit(f'Missing observation: {report_path}') +report = json.loads(report_path.read_text()) +if report['kernel_sha256'] != plan['candidate_sha256']: + raise SystemExit('Observation is from a different candidate') +if not report.get('native_4k_mmu_observed'): + raise SystemExit('Native 4K must be active') +data_state = (report.get('scratch_data') or {}).get('state', {}) +if data_state.get('_native_owner_data_status') != 0 or data_state.get('_native_owner_data_readonly') != 0: + raise SystemExit('The Data volume must mount writable through the real backend') +writes = [] +for event in report.get('syscall_events', []): + if event.get('number') in (4, 397) and event.get('write_bytes'): + try: + writes.append(bytes.fromhex(event['write_bytes']).decode('utf-8', 'replace')) + except ValueError: + continue +text = '\n'.join(writes) +required = ( + 'bootarg = 1', + 'MKB_INIT: No system keybag loaded.', + '****** DONE MKB_INIT ****', + 'usermanagerd', +) +for item in required: + if item not in text: + raise SystemExit(f'Missing real keybag/usermanagerd output: {item}') +if 'FATAL KEYBAG ERROR' in text or 'REBOOTING INTO RECOVERY MODE' in text: + raise SystemExit('The run still took the fatal keybag/recovery path') +if report.get('critical_process_exit'): + raise SystemExit('A critical userspace process still exits') +result = { + 'kernel_sha256': report['kernel_sha256'], + 'boot_argument': '-apfs_shared_datavolume', + 'keybag_bootarg': 1, + 'no_system_keybag_allowlist': True, + 'recovery_path': False, + 'data_writable': True, + 'usermanagerd_reached': True, + 'next_input': 'persona/session state on the empty Data fixture', + 'native_4k_observed': True, + 'qemu_only': True, + 'sep_or_system_keybag_present': False, + 'springboard_or_backboard_observed': False, + 'ipados_userspace_boot_demonstrated': False, +} +CHECKPOINT.parent.mkdir(parents=True, exist_ok=True) +CHECKPOINT.write_text(json.dumps(result, indent=2) + '\n') +print('PASS: shared-Data bootarg selects the real no-system-keybag allow-list path') +print('PASS: writable Data and usermanagerd reached; no recovery and no fabricated key') +print('Open: persona/session provisioning, SpringBoard/backboardd, display/GPU, hardware') diff --git a/scripts/probe-metal-vulkan-host.py b/scripts/probe-metal-vulkan-host.py new file mode 100644 index 0000000..8c75a0b --- /dev/null +++ b/scripts/probe-metal-vulkan-host.py @@ -0,0 +1,236 @@ +#!/usr/bin/env python3 +"""Bounded target AIR parsing/translation; profiles are hypothetical, never GPU evidence.""" +import argparse +from collections import Counter +import hashlib +import json +import os +from pathlib import Path +import shutil +import signal +import struct +import subprocess +import time + +ROOT = Path(__file__).resolve().parents[1] +PINS = json.loads((ROOT / 'scripts/data/gpu-metal-vulkan-inputs.json').read_text()) +SOURCE_PIN = PINS['translator']['commit'] +INVENTORY_PIN = PINS['target_air_inventory_sha256'] +LIBRARIES = { + 'QuartzCore': '606f913adf520420f9b4bb666efb782a68c830f082f1e559e17b20d0d13adbc9', + 'RenderBox': 'd76883f626b1527c36fb0940cf27dcade503a55cf2fa05c8f7a247a40ccebec4', +} + + +def sha(path): + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def output_stores(words): + """Count direct/access-chain stores to declared SPIR-V Output variables.""" + variables, bases, stores = set(), {}, [] + offset = 5 + while offset < len(words): + count, opcode = words[offset] >> 16, words[offset] & 0xffff + if not count or offset + count > len(words): + raise ValueError('Invalid SPIR-V instruction extent') + instruction = words[offset:offset + count] + if opcode == 59 and count >= 4 and instruction[3] == 3: + variables.add(instruction[2]) + elif opcode in (65, 66, 67) and count >= 4: + bases[instruction[2]] = instruction[3] + elif opcode == 62 and count >= 3: + stores.append(instruction[1]) + offset += count + observed = 0 + for pointer in stores: + visited = set() + while pointer in bases and pointer not in visited: + visited.add(pointer) + pointer = bases[pointer] + observed += pointer in variables + return dict(declared_output_variables=len(variables), output_store_instructions=observed) + + +def worker(command, directory): + """A separate process group per attempt; watch RSS and bound output and time.""" + directory.mkdir() + start = time.monotonic() + peak = 0 + failure = None + with (directory / 'stdout.json').open('wb') as out, (directory / 'stderr.log').open('wb') as err: + process = subprocess.Popen([str(x) for x in command], stdout=out, stderr=err, start_new_session=True) + try: + while process.poll() is None: + sample = subprocess.run(['ps', '-o', 'rss=', '-p', str(process.pid)], + capture_output=True, text=True, timeout=1) + if sample.returncode == 0 and sample.stdout.strip(): + peak = max(peak, int(sample.stdout.strip()) * 1024) + if peak > 500 * 1024**2: + failure = 'resident-memory budget exceeded' + elif time.monotonic() - start > 20: + failure = '20-second attempt deadline exceeded' + elif sum((directory / name).stat().st_size for name in ('stdout.json', 'stderr.log')) > 64 * 1024**2: + failure = '64-MiB output budget exceeded' + if failure: + os.killpg(process.pid, signal.SIGKILL) + break + time.sleep(0.025) + finally: + if process.poll() is None: + os.killpg(process.pid, signal.SIGKILL) + process.wait(timeout=2) + elapsed = time.monotonic() - start + if elapsed > 20: + failure = failure or '20-second attempt deadline exceeded' + detail = (directory / 'stderr.log').read_text(errors='replace')[-4000:] + result = dict(returncode=process.returncode, seconds=elapsed, sampled_peak_rss_bytes=peak, + memory_measurement='25-ms RSS samples; not a peak-allocation proof', + budget_failure=failure, diagnostic=detail) + value = None + if process.returncode == 0 and not failure: + value = json.loads((directory / 'stdout.json').read_text()) + return result, value + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--air-input-dir', required=True, type=Path) + parser.add_argument('--compiler-build', type=Path, default=ROOT / 'build/gpu-001/compiler') + parser.add_argument('--output', type=Path, default=ROOT / 'build/gpu-001/translation') + parser.add_argument('--profile', type=Path, default=ROOT / '.cache/gpu-reference/metal2vulkan/crates/metal2vulkan-validation/profiles/vk12-bare.json') + parser.add_argument('--compile-entrypoints', action='store_true') + parser.add_argument('--vertex-descriptor', type=Path, + help='Explicit diagnostic layout, not an observed system pipeline') + parser.add_argument('--color-attachments', type=Path, + help='Explicit attachment-index/pixel-format map for paired pipelines') + parser.add_argument('--require-fragment-output', action='store_true', + help='Reject validation-only empty fragments in selected pipeline checks') + parser.add_argument('--pipeline', action='append', default=[], metavar='LIB:INDEX:LIB:INDEX') + args = parser.parse_args() + output = args.output.resolve() + if output.exists(): + raise SystemExit('Use a fresh output directory; never reuse an earlier success report') + if shutil.disk_usage(ROOT).free < 6 * 1024**3: + raise SystemExit('Preserve 5 GiB free plus the bounded output allowance') + build = json.loads((args.compiler_build / 'build-report.json').read_text()) + binary = Path(build['binary']) + if build['translator_revision'] != SOURCE_PIN or sha(binary) != build['binary_sha256'] or \ + sha(ROOT / 'drivers/NativeMetal/tools/MetalVulkanProbe.rs') != build['source_sha256']: + raise SystemExit('Compiler/source identity differs from its build report') + validator = shutil.which('spirv-val') + if not validator: + raise SystemExit('spirv-val is required; translation alone cannot pass validation') + inventory_path = args.air_input_dir / 'target-inputs.json' + if sha(inventory_path) != INVENTORY_PIN: + raise SystemExit('Target AIR audit differs from the pinned original inventory') + audit = json.loads(inventory_path.read_text()) + recorded = {Path(x['file']).parts[-2].removesuffix('.framework'): x['sha256'] for x in audit['inputs']} + if recorded != LIBRARIES or len(audit['modules']) != 240: + raise SystemExit('Only the existing 240-module iPad7,1 target inventory is accepted') + modules = {(m['library'], m['index']): m for m in audit['modules']} + if len(modules) != 240: + raise SystemExit('Duplicate module identity') + + def source(module): + p = args.air_input_dir / module['library'] / f'module-{module["index"]}.bc' + if sha(p) != module['bitcode_sha256']: + raise ValueError('Original AIR differs from inventory') + return p + + # Verify every source before creating any result. + for module in modules.values(): + source(module) + output.mkdir(parents=True) + cases = output / 'cases' + cases.mkdir() + parsed = [] + compiled = [] + unexpected = [] + for key, module in modules.items(): + tag = f'{key[0]}-{key[1]}' + result, value = worker([binary, '--list', source(module)], cases / ('parse-' + tag)) + result.update(library=key[0], index=key[1], input_sha256=module['bitcode_sha256']) + if value is not None: + result['functions'] = value['functions'] + if not any(f['name'] == module['name'] for f in value['functions']): + unexpected.append('Original function absent from parsed module: ' + tag) + elif result['returncode'] != 2 or result['budget_failure']: + unexpected.append('Parser failure: ' + tag) + parsed.append(result) + + jobs = [] + if args.compile_entrypoints: + jobs.extend(([m], f'{k[0]}-{k[1]}') for k, m in modules.items() if m['stage'] in ('kernel', 'vertex')) + for item in args.pipeline: + a, i, b, j = item.split(':') + jobs.append(([modules[(a, int(i))], modules[(b, int(j))]], item.replace(':', '-'))) + for members, tag in jobs: + attempt_start = time.monotonic() + command = [binary] + if args.vertex_descriptor: + command.extend(['--vertex-descriptor', args.vertex_descriptor]) + if args.color_attachments: + command.extend(['--color-attachments', args.color_attachments]) + command.append(args.profile) + for member in members: + command.extend([source(member), member['name']]) + case = cases / ('compile-' + tag) + result, value = worker(command, case) + result['inputs'] = [{k: m[k] for k in ('library', 'index', 'bitcode_sha256')} for m in members] + if value is not None: + result['artifact'] = value + for index, stage in enumerate(value['stages']): + words = stage.pop('spirv') + if len(words) < 5 or words[0] != 0x07230203: + raise ValueError('Invalid SPIR-V header from translator') + stage.update(output_stores(words)) + if args.require_fragment_output and stage['stage'] == 'VK_SHADER_STAGE_FRAGMENT_BIT' and not stage['output_store_instructions']: + unexpected.append('Fragment has no output-store instructions: ' + tag) + data = struct.pack('<' + 'I' * len(words), *words) + path = case / f'stage-{index}.spv' + path.write_bytes(data) + stage.update(bytes=len(data), sha256=sha(path), validation=[]) + for env in value['validation_envs']: + remaining = 20 - (time.monotonic() - attempt_start) + if remaining <= 0: + result['budget_failure'] = '20-second translation/validation deadline exceeded' + unexpected.append('Attempt deadline exceeded: ' + tag) + break + try: + validation = subprocess.run([validator, '--target-env', env, + *value['validation_relaxations'], str(path)], + capture_output=True, text=True, timeout=remaining) + except subprocess.TimeoutExpired: + result['budget_failure'] = '20-second translation/validation deadline exceeded' + unexpected.append('Validation deadline exceeded: ' + tag) + break + stage['validation'].append(dict(environment=env, returncode=validation.returncode, + diagnostic=validation.stderr[-4000:])) + if validation.returncode: + unexpected.append('SPIR-V validation failed: ' + tag) + (case / 'stdout.json').unlink() + result['semantic_equivalence_qualified'] = False + elif result['returncode'] != 2 or result['budget_failure']: + unexpected.append('Compiler failure: ' + tag) + compiled.append(result) + result['translation_and_validation_seconds'] = time.monotonic() - attempt_start + for module in modules.values(): + source(module) + report = dict(compiler=build, input_inventory_sha256=sha(inventory_path), profile_sha256=sha(args.profile), + vertex_descriptor_sha256=sha(args.vertex_descriptor) if args.vertex_descriptor else None, + color_attachments_sha256=sha(args.color_attachments) if args.color_attachments else None, + profile_scope='synthetic Vulkan minimum; not observed Adreno capabilities', + validator_version=subprocess.check_output([validator, '--version'], text=True).strip(), + validator_sha256=sha(Path(validator).resolve()), parse_results=parsed, compile_results=compiled, + parse_counts=dict(Counter(str(x['returncode']) for x in parsed)), + compile_counts=dict(Counter(str(x['returncode']) for x in compiled)), + unexpected_failures=unexpected, original_inputs_unchanged=True, + gpu_execution=False, active_gpu_smmu_4k_qualified=False, deployment_ready=False) + (output / 'report.json').write_text(json.dumps(report, indent=2) + '\n') + print(json.dumps({k: report[k] for k in ('parse_counts', 'compile_counts', 'unexpected_failures', 'deployment_ready')})) + return bool(unexpected) + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/scripts/probe-mobile-display-provider.py b/scripts/probe-mobile-display-provider.py new file mode 100644 index 0000000..ca2f600 --- /dev/null +++ b/scripts/probe-mobile-display-provider.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 +"""Run one bounded original-client/provider ABI boot; no display is published.""" +import argparse +from pathlib import Path +import subprocess +ROOT=Path(__file__).resolve().parents[1] +p=argparse.ArgumentParser(description=__doc__) +p.add_argument('--case',choices=['query','headless','default-off'],required=True) +a=p.parse_args() +base='build/native-prelink/native-owner-mobile-provider-query-candidate' +cmd=['python3','scripts/probe-darwinboot-xnu.py','--kernel',base+'.macho', + '--prelink-report',base+'.json','--output','build/darwinboot-owner-hvf-mobile-provider-'+a.case, + '--virtio-udif-image','.cache/firmware/target-volumes/044-76040-038.dmg', + '--virtio-extra-block-image','.cache/firmware/target-volumes/044-76034-038.dmg', + '--virtio-mmio-fdt','build/virtio-block-fixture/qemu-virt.dtb', + '--root-match-volume-uuid','04769D26-2CA0-4825-B443-E2E5F88300F2', + '--trust-cache','build/original-static-system-trust/original-static-system.bundle', + '--armvirt','--armcpu','--qemu-vmm-present','--qemu-accel','hvf', + '--owned-memory-size','0x20000000','--nvram-proxy','--crypto-hash-method','sha2-384', + '--amfi-platform-only','--load-apfs','--skip-initializer-traces','--timeout','60', + '--data-ramdisk','build/owner-data-fixture/owner-data-32.cdr','--load-mobile-graphics'] +if a.case!='headless':cmd+=['--qemu-ramfb'] +if a.case=='default-off':cmd+=['--follow-target-main','--inspect-owner-root','--check-commpage'] +else:cmd+=['--mobile-graphics-client-probe','--skip-policy-pointer-reads', + '--break-symbol','_native_mobile_display_provider_checkpoint'] +cmd+=['--inspect-boot-video'] +subprocess.run(cmd,cwd=ROOT,check=True,timeout=180) diff --git a/scripts/probe-native-legacy-display.py b/scripts/probe-native-legacy-display.py new file mode 100644 index 0000000..37ab3da --- /dev/null +++ b/scripts/probe-native-legacy-display.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +"""Run one bounded original Legacy init/start/control using real GOP resources.""" +import argparse +from pathlib import Path +import subprocess +ROOT=Path(__file__).resolve().parents[1] +p=argparse.ArgumentParser(description=__doc__);p.add_argument('--case',required=True,choices=['init','start','headless','default-off']);p.add_argument('--original-error',action='store_true',help='Observe the actual original start failure report before cleanup') +p.add_argument('--gop',choices=['mapped','publication'],help='Use actual GOP mapping backend; stop before any service publication') +p.add_argument('--system',choices=['started','client','mode','query','metal','protocol','empty-protocol','power','geometry','dotpitch','frameinfo','frameinfo-return','exception','panic'],help='Explicit genuine GOP-backed original system-client experiment') +p.add_argument('--profile',choices=['late','gop','system','system-dt','pitch','info','scale','pm','validate','transfer-ready'],default='gop',help='Exact retained candidate profile') +p.add_argument('--process',help='Observe actual exceptions from this named process only') +p.add_argument('--read-user-register',action='append',default=[],help='Read bounded saved user pointer memory at the actual exception') +p.add_argument('--capture-screen',action='store_true',help='Capture the actual QEMU console without resuming the guest') +a=p.parse_args() +if a.read_user_register and a.system!='exception':p.error('--read-user-register requires --system exception') +if a.process and a.system!='exception':p.error('--process requires --system exception') +if a.system: + if a.gop or a.original_error or a.case!='start':p.error('System-client observation requires start without other GOP/error observations') + if a.profile not in ('system','system-dt','pitch','info','scale','pm','validate','transfer-ready'):a.profile='system-dt' +if a.gop and a.profile!='gop':p.error('GOP observations require the GOP candidate profile') +base='build/native-prelink/native-owner-legacy-display-'+a.profile+'-candidate' +cmd=['python3','scripts/probe-darwinboot-xnu.py','--kernel',base+'.macho','--prelink-report',base+'.json', + '--output','build/darwinboot-owner-hvf-legacy-display-'+a.case+('-error' if a.original_error else '')+('-'+a.profile+'-'+(a.system or 'control') if a.profile.startswith('system') or a.profile in ('pitch','info','scale','pm','validate','transfer-ready') else '-gop-'+(a.gop or 'control') if a.profile=='gop' else ''), + '--virtio-udif-image','.cache/firmware/target-volumes/044-76040-038.dmg', + '--virtio-extra-block-image','.cache/firmware/target-volumes/044-76034-038.dmg', + '--virtio-mmio-fdt','build/virtio-block-fixture/qemu-virt.dtb', + '--root-match-volume-uuid','04769D26-2CA0-4825-B443-E2E5F88300F2', + '--trust-cache','build/original-static-system-trust/original-static-system.bundle', + '--armvirt','--armcpu','--qemu-vmm-present','--qemu-accel','hvf','--owned-memory-size','0x20000000', + '--nvram-proxy','--crypto-hash-method','sha2-384','--amfi-platform-only','--load-apfs','--load-mobile-graphics', + '--skip-initializer-traces','--timeout','60','--data-ramdisk','build/owner-data-fixture/owner-data-32.cdr','--inspect-boot-video'] +if a.process: + cmd[cmd.index('--output')+1]+='-'+a.process + cmd+=['--entry-process-name',a.process,'--entry-max-skips','128'] +if a.read_user_register: + cmd[cmd.index('--output')+1]+='-registers' + for request in a.read_user_register:cmd+=['--read-saved-user-register',request] +if a.capture_screen: + cmd[cmd.index('--output')+1]+='-screen' + cmd+=['--capture-qemu-screen'] +if a.case!='headless':cmd+=['--qemu-ramfb'] +if a.case=='default-off':cmd+=['--follow-target-main','--inspect-owner-root','--check-commpage'] +else: + cmd+=['--mobile-graphics-client-probe','--legacy-display-phase','4' if a.system else '2' if a.case=='start' else '1'] + if a.system: + cmd+=['--legacy-display-gop','--legacy-display-register'] + if a.profile in ('scale','pm','validate','transfer-ready'):cmd+=['--legacy-display-unit-scale'] + if a.profile in ('pm','validate','transfer-ready'):cmd+=['--legacy-display-logical-pm'] + if a.profile in ('validate','transfer-ready'):cmd+=['--legacy-display-validate'] + if a.profile=='transfer-ready':cmd+=['--legacy-display-transfer'] + if a.system in ('protocol','empty-protocol','power','geometry'): + instrument='framebuffer-geometry-diagnostic' if a.system=='geometry' else 'framebuffer-power-diagnostic' if a.system=='power' else 'framebuffer-empty-protocol-diagnostic' if a.system=='empty-protocol' else 'framebuffer-protocol-diagnostic' + cmd[cmd.index('--data-ramdisk')+1]='build/'+instrument+'/compiler-data-32.cdr' + cmd[cmd.index('--trust-cache')+1]='build/'+instrument+'/offline-trust.bundle' + cmd+=['--native-air-init','--break-symbol','_exception_triage','--inspect-current-process','--inspect-entry-user-state','--read-saved-user-register','13:80','--inspect-owner-root','--check-commpage'] + elif a.system=='metal':cmd+=['--follow-target-main','--follow-metal-device','--inspect-owner-root','--check-commpage'] + elif a.system=='query':cmd+=['--follow-target-main','--follow-display-query','--inspect-owner-root','--check-commpage'] + elif a.system=='mode':cmd+=['--follow-target-main','--follow-display-mode','--inspect-owner-root','--check-commpage'] + elif a.system=='started':cmd+=['--break-symbol','_native_legacy_display_checkpoint','--skip-policy-pointer-reads'] + elif a.system=='client':cmd+=['--break-symbol','_native_legacy_client_checkpoint','--inspect-current-process','--inspect-owner-root','--inspect-dyld-flags','--check-commpage'] + elif a.system=='exception':cmd+=['--break-symbol','_exception_triage','--inspect-current-process','--inspect-entry-user-state','--read-entry-region','1:0:10','--read-frame-locals','--inspect-owner-root','--skip-policy-pointer-reads'] + elif a.system=='frameinfo-return':cmd+=['--break-address','0xfffffff0057786e8','--read-frame-locals','--read-frame-indirect','0:-20:40','--inspect-owner-root','--skip-policy-pointer-reads'] + elif a.system=='frameinfo':cmd+=['--break-address','0xfffffff0057786e4','--read-frame-locals','--read-entry-region','2:0:4','--read-entry-region','3:0:40','--inspect-owner-root','--skip-policy-pointer-reads'] + elif a.system=='dotpitch':cmd+=['--break-address','0xfffffff005778194','--read-frame-locals','--inspect-owner-root','--skip-policy-pointer-reads'] + else: + # The general APFS probe intentionally replaces its default panic + # target with its load checkpoint unless observing the owner-root + # path. Keep that explicit observation for the real panic target. + cmd+=['--break-symbol','_panic','--inspect-owner-root','--skip-policy-pointer-reads'] + elif a.gop: + if a.case!='start' or a.original_error:p.error('GOP observation requires start without --original-error') + cmd+=['--legacy-display-gop','--skip-policy-pointer-reads'] + cmd+=(['--break-symbol','_native_legacy_gop_mapping_checkpoint'] if a.gop=='mapped' else ['--break-address','0xfffffff0057880d4']) + elif a.original_error: + if a.case!='start':p.error('Original start failure observation requires --case start') + cmd+=['--break-address','0xfffffff0057870b0','--read-frame-locals','--skip-policy-pointer-reads'] + else:cmd+=['--break-symbol','_native_legacy_display_checkpoint','--skip-policy-pointer-reads'] +subprocess.run(cmd,cwd=ROOT,check=True,timeout=180) diff --git a/scripts/probe-offline-gpu-transport.py b/scripts/probe-offline-gpu-transport.py new file mode 100644 index 0000000..616cf42 --- /dev/null +++ b/scripts/probe-offline-gpu-transport.py @@ -0,0 +1,169 @@ +#!/usr/bin/env python3 +"""Bounded genuine native iOS17 -> VirtIO -> host Metal GPU round-trip.""" +import argparse +import json +from pathlib import Path +import subprocess +import tempfile +import time +ROOT=Path(__file__).resolve().parents[1];OUT=ROOT/'build/offline-gpu-transport' +p=argparse.ArgumentParser(description=__doc__) +p.add_argument('--broker-processes',action='store_true',help='Two actual original-userland GPU processes through the private kernel broker') +p.add_argument('--broker-spawn-observe',choices=['posix_spawn','fork1','exec_activate_image','exec_mach_imgact','mac_vnode_check_exec'],help='Read a genuine original spawn boundary and return without claiming GPU completion') +p.add_argument('--broker-map-observe',action='store_true',help='Read actual executable mmap MAC return and registered hooks for the own plugin') +p.add_argument('--broker-map-policy',choices=['mac','AMFI','Sandbox'],default='mac',help='Read a genuine registered mapping hook, without changing policy') +p.add_argument('--legacy-swap-wait',action='store_true',help='Read actual original SwapWaitWithTimeout status after first GPU frame') +p.add_argument('--legacy-queue-retirement',action='store_true',help='Use original queue completion only after actual first-frame scanout transfer') +p.add_argument('--legacy-swap',action='store_true',help='Submit real GPU IOSurface through original framebuffer transaction') +p.add_argument('--legacy-queue-transfer',action='store_true',help='Transfer one completed actual GPU surface from the real pending queue without retiring it') +p.add_argument('--legacy-gop-validation',action='store_true',help='Qualify the actual one-full-frame IOSurface validator') +p.add_argument('--legacy-logical-pm',action='store_true',help='Use genuine IOKit logical scanout scheduling, without panel power claims') +p.add_argument('--legacy-swap-submit',action='store_true',help='Observe unchanged original kernel swap_submit entry and real marshaled packet') +p.add_argument('--legacy-swap-decision',action='store_true',help='Observe original validated-swap submission gate, without changing its state') +p.add_argument('--legacy-swap-validate',action='store_true',help='Observe original mandatory backend validation return and actual IOSurface') +p.add_argument('--legacy-swap-kick',action='store_true',help='Observe original queue-kick entry only after actual resource validation') +p.add_argument('--iosurface',action='store_true',help='Exercise real original IOSurface-backed Metal textures') +p.add_argument('--bgra',action='store_true',help='Exercise actual BGRA IOSurface uploads, GPU drawing and readback') +p.add_argument('--present',action='store_true',help='Transfer actual display-size GPU output to genuine GOP scanout') +p.add_argument('--resources',action='store_true',help='Use real GPU buffers, blit, original buffer-driven vertex and indexed draws') +p.add_argument('--compose',action='store_true',help='Use actual GPU texture reads, clipping and blend state for layer composition') +p.add_argument('--metal-api',action='store_true',help='Run the partial native Metal object/command frontend') +p.add_argument('--case',choices=['positive','missing-device','default-off','validation-disabled','transfer-disabled','retirement-disabled','without-container','without-client'],default='positive') +p.add_argument('--depth-stencil',action='store_true');p.add_argument('--private-texture',action='store_true');p.add_argument('--renderbox-draw',action='store_true');p.add_argument('--renderbox-device',action='store_true');p.add_argument('--plugin-control',choices=['full','without-mount-permission','tampered-code'],default='full');p.add_argument('--plugin-addon',action='store_true');p.add_argument('--plugin-bundle',action='store_true');p.add_argument('--shared-surfaces',action='store_true',help='Pass actual IOSurface Mach rights between two GPU processes') +a=p.parse_args() +if a.depth_stencil: + if a.private_texture or a.renderbox_draw:p.error('Select one independent resource or renderer instrument') + a.renderbox_device=True +if a.private_texture: + if a.renderbox_draw:p.error('Select one independent resource or original renderer instrument') + a.renderbox_device=True +if a.renderbox_draw:a.renderbox_device=True +if a.renderbox_device: + if a.plugin_control!='full':p.error('RenderBox device observation is a separate instrument') + a.plugin_addon=True +if a.plugin_control!='full':a.plugin_addon=True +if a.plugin_addon:a.plugin_bundle=True +if a.plugin_bundle:a.shared_surfaces=True +if a.broker_map_policy!='mac':a.broker_map_observe=True +if a.shared_surfaces:a.broker_processes=True +if a.shared_surfaces and a.case!='positive':p.error('Use --broker-processes for the independently qualified broker controls') +if a.broker_map_observe and (not a.plugin_bundle or a.broker_spawn_observe):p.error('Select only the explicit own plugin mmap observer') +if a.broker_processes: + if a.legacy_swap or a.legacy_swap_wait or a.compose or a.resources or a.present or a.case not in ('positive','missing-device','default-off','without-container','without-client'):p.error('Select one GPU process instrument or its missing-device/default-off control') + a.metal_api=True;a.bgra=True +if a.case in ('without-container','without-client') and not a.broker_processes:p.error('Permission controls require the private GPU instrument') +if a.broker_spawn_observe and (not a.broker_processes or a.case!='positive'):p.error('Spawn observation requires the enabled private GPU instrument') +if a.legacy_swap_wait:a.legacy_swap=True +if a.legacy_queue_retirement:a.legacy_queue_transfer=True +if a.legacy_swap_wait and not a.legacy_queue_retirement:p.error('The bounded wait instrument requires the original completion candidate') +if a.legacy_queue_retirement and a.case not in ('positive','retirement-disabled'):p.error('Select completion or its same-candidate disabled control') +if a.case=='retirement-disabled' and not a.legacy_queue_retirement:p.error('Retirement-disabled requires the real completion candidate') +if a.legacy_queue_transfer:a.legacy_gop_validation=True +if a.case=='transfer-disabled' and not a.legacy_queue_transfer:p.error('Transfer-disabled requires the actual queue transfer candidate') +if a.legacy_gop_validation:a.legacy_logical_pm=True +if a.case=='validation-disabled' and (not a.legacy_gop_validation or a.legacy_swap_validate):p.error('Validation-disabled control requires the own validator candidate and its normal return observation') +if a.legacy_logical_pm and not a.legacy_swap:p.error('Logical scanout scheduling requires the original swap instrument') +if a.legacy_swap_submit and not a.legacy_swap:p.error('--legacy-swap-submit requires --legacy-swap') +if a.legacy_swap_decision and (not a.legacy_swap or a.legacy_swap_submit):p.error('Select one original swap observation') +if a.legacy_swap_validate and (not a.legacy_swap or a.legacy_swap_submit or a.legacy_swap_decision):p.error('Select one original swap observation') +if a.legacy_swap_kick and (not a.legacy_gop_validation or a.legacy_swap_validate or a.legacy_swap_submit or a.legacy_swap_decision or a.case!='positive'):p.error('Queue kick requires the enabled validator and its positive GPU input') +if a.legacy_swap and (a.compose or a.resources or a.present):p.error('Select one presentation instrument') +if a.legacy_swap:a.bgra=True +if a.compose and a.resources:p.error('Select composition or resource instrument') +if a.compose:a.present=True +if a.resources:a.present=True +if a.present:a.bgra=True +if a.bgra:a.iosurface=True +if a.iosurface and not a.metal_api:p.error('IOSurface requires --metal-api') +broker_kind='offline-metal-depth-stencil' if a.depth_stencil else 'offline-metal-private-texture' if a.private_texture else 'offline-renderbox-draw' if a.renderbox_draw else 'offline-renderbox-device' if a.renderbox_device else ('offline-metal-addon'+('' if a.plugin_control=='full' else '-'+a.plugin_control)) if a.plugin_addon else 'offline-metal-plugin' if a.plugin_bundle else 'offline-metal-shared-surfaces' if a.shared_surfaces else 'offline-metal-processes' +kind=(broker_kind+('-'+a.case if a.case in ('without-container','without-client') else '')) if a.broker_processes else 'offline-legacy-swap-wait' if a.legacy_swap_wait else 'offline-legacy-swap' if a.legacy_swap else 'offline-gop-composition' if a.compose else 'offline-gop-resources' if a.resources else 'offline-gop-presentation' if a.present else 'offline-metal-bgra' if a.bgra else 'offline-metal-iosurface' if a.iosurface else 'offline-metal-api' if a.metal_api else 'offline-gpu-transport' +OUT=ROOT/'build'/kind +base='build/native-prelink/native-owner-offline-gpu-depth-stencil-candidate' if a.broker_processes else 'build/native-prelink/native-owner-legacy-display-completion-candidate' if a.legacy_queue_retirement else 'build/native-prelink/native-owner-legacy-display-transfer-ready-candidate' if a.legacy_queue_transfer else 'build/native-prelink/native-owner-legacy-display-validate-candidate' if a.legacy_gop_validation else 'build/native-prelink/native-owner-legacy-display-pm-candidate' if a.legacy_logical_pm else 'build/native-prelink/native-owner-legacy-display-scale-candidate' if a.legacy_swap else 'build/native-prelink/native-owner-gop-present-candidate' if a.present else 'build/native-prelink/native-owner-console-bounds-candidate' +oslib=ROOT/'.cache/firmware/target-volumes/os-mount/System/Library' +paths=[oslib/'Frameworks/QuartzCore.framework/default.metallib',oslib/'PrivateFrameworks/RenderBox.framework/default.metallib'] +cmd=['python3','scripts/probe-darwinboot-xnu.py','--kernel',base+'.macho','--prelink-report',base+'.json', + '--output','build/darwinboot-owner-hvf-'+kind+('-cpu-contexts' if a.broker_processes else '')+('' if a.case in ('positive','without-container','without-client') else '-'+a.case), + '--virtio-udif-image','.cache/firmware/target-volumes/044-76040-038.dmg', + '--virtio-extra-block-image','.cache/firmware/target-volumes/044-76034-038.dmg', + '--virtio-mmio-fdt','build/virtio-block-fixture/qemu-virt.dtb', + '--root-match-volume-uuid','04769D26-2CA0-4825-B443-E2E5F88300F2', + '--trust-cache','build/'+kind+'/offline-trust.bundle', + '--armvirt','--armcpu','--qemu-vmm-present','--qemu-accel','hvf', + '--owned-memory-size','0x20000000','--nvram-proxy','--crypto-hash-method','sha2-384', + '--amfi-platform-only','--load-apfs','--inspect-owner-root','--inspect-dyld-flags','--check-commpage', + '--skip-initializer-traces','--timeout','60','--data-ramdisk','build/'+kind+'/compiler-data-32.cdr', + '--qemu-ramfb','--inspect-boot-video'] +if a.legacy_queue_retirement and a.case!='retirement-disabled':cmd+=['--legacy-display-retire'] +if a.legacy_queue_transfer: + cmd[cmd.index('--output')+1]+='-completion' if a.legacy_queue_retirement else '-transfer-ready' + if a.case not in ('default-off','validation-disabled','transfer-disabled'):cmd+=['--legacy-display-transfer','--inspect-gop-scanout-bytes','1920000'] +if a.legacy_gop_validation: + cmd[cmd.index('--output')+1]+='-driver' + if a.case not in ('default-off','validation-disabled'):cmd+=['--legacy-display-validate'] +if a.legacy_logical_pm: + cmd[cmd.index('--output')+1]+='-pm' + if a.case!='default-off':cmd+=['--legacy-display-logical-pm'] +if a.iosurface:cmd+=['--load-iogpu'] +if a.broker_processes and a.case!='default-off':cmd+=['--offline-gpu-broker'] +if a.broker_processes and a.case in ('without-container','without-client'):cmd+=['--inspect-mac-policies'] +if a.legacy_swap and a.case!='default-off':cmd+=['--load-mobile-graphics','--mobile-graphics-client-probe','--legacy-display-phase','4','--legacy-display-gop','--legacy-display-register','--legacy-display-unit-scale'] +if a.present and a.case!='default-off':cmd+=['--load-mobile-graphics','--mobile-graphics-client-probe','--legacy-display-phase','3'] +if a.present and a.case=='positive':cmd+=['--inspect-gop-scanout-bytes','1920000'] +if a.case=='default-off':cmd+=['--follow-target-main'] +elif a.broker_processes: + cmd+=['--native-air-init'] + if a.broker_map_observe: + cmd[cmd.index('--output')+1]+='-mmap-workers'+('' if a.broker_map_policy=='mac' else '-'+a.broker_map_policy.lower()) + if a.broker_map_policy=='mac':cmd+=['--break-symbol','_mac_file_check_mmap'] + else: + observation=json.loads((ROOT/'build/darwinboot-owner-hvf-offline-metal-plugin-cpu-contexts-mmap-workers/gdb-stop.json').read_text()) + if observation['kernel_sha256']!=json.loads((ROOT/(base+'.json')).read_text())['candidate_sha256']:raise SystemExit('Registered policy observation belongs to another kernel') + hook=next(r for r in observation['registered_mac_policies']['policies'] if r['name']==a.broker_map_policy)['hooks']['mpo_file_check_mmap'] + if not int(hook['address'],16) or len(hook['instructions'])!=128:raise SystemExit('Missing genuine registered mapping hook') + cmd+=['--break-address',hook['address']] + cmd+=['--finish-function','--inspect-current-process','--entry-process-name','air-native','--entry-min-pid','2','--entry-register-equals','2:5' if a.broker_map_policy=='mac' else '3:5','--entry-max-skips','1024','--inspect-mac-policies','--read-entry-region','1:0:80','--read-frame-locals'] + elif a.broker_spawn_observe: + cmd[cmd.index('--output')+1]+='-'+a.broker_spawn_observe+('-policies' if a.broker_spawn_observe=='mac_vnode_check_exec' else '') + cmd+=['--break-symbol','_'+a.broker_spawn_observe,'--finish-function','--inspect-current-process','--entry-process-name','air-native','--entry-max-skips','128','--read-frame-locals'] + if a.broker_spawn_observe=='mac_vnode_check_exec':cmd+=['--inspect-mac-policies'] + if a.broker_spawn_observe=='posix_spawn':cmd+=['--read-entry-region','1:0:40'] + elif a.case=='missing-device':cmd+=['--break-symbol','_native_gpu_broker_checkpoint'] + elif a.plugin_control=='tampered-code':cmd+=['--break-symbol','_cs_invalid_page','--inspect-current-process','--inspect-entry-user-state'] + else:cmd+=['--break-symbol','_exception_triage','--inspect-current-process','--inspect-entry-user-state','--read-saved-user-register','13:100'] +elif a.legacy_swap_kick: + cmd[cmd.index('--output')+1]+='-kick' + if a.legacy_queue_transfer:cmd+=['--native-air-init','--break-symbol','_native_legacy_transfer_observed','--read-entry-region','1:0:a80','--capture-qemu-screen'] + else:cmd+=['--native-air-init','--break-address','0xfffffff005794030','--entry-after','_native_legacy_swap_validation_checkpoint','--read-entry-region','0:1000:800','--read-frame-locals','--capture-qemu-screen'] +elif a.legacy_swap_validate: + cmd[cmd.index('--output')+1]+='-validate' + if a.legacy_gop_validation:cmd+=['--native-air-init','--break-symbol','_native_legacy_swap_validation_checkpoint','--read-entry-region','2:0:a80','--read-entry-region','3:0:100','--read-frame-locals','--capture-qemu-screen'] + else:cmd+=['--native-air-init','--break-address','0xfffffff005795bb4','--finish-function','--read-entry-region','1:0:a80','--read-entry-indirect','1:a40:100','--read-frame-locals','--capture-qemu-screen'] +elif a.legacy_swap_decision: + cmd[cmd.index('--output')+1]+='-decision' + cmd+=['--native-air-init','--break-address','0xfffffff005790574','--read-frame-locals','--read-frame-indirect','0:-28:508','--capture-qemu-screen'] +elif a.legacy_swap_submit: + cmd[cmd.index('--output')+1]+='-submit' + cmd+=['--native-air-init','--break-address','0xfffffff00578f224','--read-entry-region','0:1000:800','--read-entry-region','1:0:508','--read-entry-region','2:0:100','--read-frame-locals','--capture-qemu-screen'] +elif a.legacy_swap:cmd+=['--native-air-init','--break-symbol','_exception_triage','--inspect-current-process','--inspect-entry-user-state','--read-saved-user-register','13:80','--capture-qemu-screen'] +else:cmd+=['--native-air-init','--break-symbol','_exit','--read-entry-region','1:0:8'] +if a.plugin_bundle and not a.broker_map_observe and a.plugin_control!='tampered-code':cmd+=['--read-saved-user-register','14:1000' if a.renderbox_draw else '14:200'] +if a.case in ('missing-device','default-off'): + subprocess.run(cmd,cwd=ROOT,timeout=180,check=True) + print('Absent console/default-off control observation recorded');raise SystemExit(0) +with tempfile.TemporaryDirectory(prefix='native-gpu-',dir='/tmp') as directory: + sock=Path(directory)/'gpu.sock' + host_suffix=('-cpu-contexts' if a.broker_processes else '')+('-'+a.broker_spawn_observe if a.broker_spawn_observe else '')+(('-mmap-workers'+('' if a.broker_map_policy=='mac' else '-'+a.broker_map_policy.lower())) if a.broker_map_observe else '')+('-disabled' if a.case=='validation-disabled' else '-transfer-disabled' if a.case=='transfer-disabled' else '')+('-retire-disabled' if a.case=='retirement-disabled' else '')+('-completion' if a.legacy_queue_retirement else '-transfer-ready' if a.legacy_queue_transfer else '')+('-driver' if a.legacy_gop_validation else '')+('-pm' if a.legacy_logical_pm else '')+('-kick' if a.legacy_swap_kick else '-validate' if a.legacy_swap_validate else '-decision' if a.legacy_swap_decision else '-submit' if a.legacy_swap_submit else '') + with (OUT/('host'+host_suffix+'-report.json')).open('w') as report,(OUT/('host'+host_suffix+'-stderr.log')).open('w') as errors: + host=subprocess.Popen([str(ROOT/'build/host-metal/host-metal-originals'),*map(str,paths),'--metal-session-socket' if a.broker_processes else '--metal-socket' if a.metal_api else '--socket',str(sock)],stdout=report,stderr=errors) + try: + deadline=time.monotonic()+15 + while not sock.exists(): + if host.poll() is not None or time.monotonic()>=deadline:raise SystemExit('Host GPU socket did not become ready') + time.sleep(.02) + cmd+=['--virtio-console-socket',str(sock)] + result=subprocess.run(cmd,cwd=ROOT,timeout=180) + if result.returncode:raise SystemExit(result.returncode) + if not a.broker_spawn_observe and not a.broker_map_observe and a.case!='without-container' and a.plugin_control=='full' and not a.renderbox_device and host.wait(timeout=5):raise SystemExit('Real host GPU backend failed') + finally: + if host.poll() is None:host.terminate();host.wait(timeout=5) +print('Offline transport observation recorded; inspect actual native exit, DMA and host GPU results') diff --git a/scripts/probe-original-gpu-client-abi.py b/scripts/probe-original-gpu-client-abi.py new file mode 100644 index 0000000..3506041 --- /dev/null +++ b/scripts/probe-original-gpu-client-abi.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 +"""Bounded real original IOGPU userclient lifecycle; no device/client publication.""" +import argparse,hashlib,json,shutil,socket,subprocess,tempfile,threading +from pathlib import Path +ROOT=Path(__file__).resolve().parents[1] +p=argparse.ArgumentParser(description=__doc__);p.add_argument('--missing-base',action='store_true');p.add_argument('--default-off',action='store_true');a=p.parse_args() +if a.missing_base and a.default_off:p.error('Select one independent control') +if subprocess.run(['pgrep','-f','qemu-system-aarch64'],capture_output=True).returncode==0:raise SystemExit('Another QEMU is running') +if shutil.disk_usage(ROOT).free<5*1024**3+256*1024**2:raise SystemExit('Preserve 5 GiB free') +base=ROOT/'build/native-prelink/native-owner-original-client-abi-candidate';fixture=ROOT/'build'/('darwinboot-owner-hvf-original-client-abi'+('-missing-base' if a.missing_base else '-default-off' if a.default_off else '')) +if (fixture/'gdb-stop.json').exists():raise SystemExit('Preserve the existing observation before another run') +build=json.loads((ROOT/'build/native-offline-accelerator/build-report.json').read_text()) +for rel,digest in build['source_sha256'].items(): + if hashlib.sha256((ROOT/rel).read_bytes()).hexdigest()!=digest:raise SystemExit('Exact compiled source required: '+rel) +cmd=['python3','scripts/probe-darwinboot-xnu.py','--kernel',str(base.with_suffix('.macho')),'--prelink-report',str(base.with_suffix('.json')),'--output',str(fixture), + '--virtio-udif-image','.cache/firmware/target-volumes/044-76040-038.dmg','--virtio-extra-block-image','.cache/firmware/target-volumes/044-76034-038.dmg', + '--virtio-mmio-fdt','build/virtio-block-fixture/qemu-virt.dtb','--root-match-volume-uuid','04769D26-2CA0-4825-B443-E2E5F88300F2', + '--trust-cache','build/offline-renderbox-present-2/offline-trust.bundle','--armvirt','--armcpu','--qemu-vmm-present','--qemu-accel','hvf','--owned-memory-size','0x20000000', + '--nvram-proxy','--crypto-hash-method','sha2-384','--amfi-platform-only','--load-apfs','--inspect-owner-root','--inspect-dyld-flags','--check-commpage', + '--skip-initializer-traces','--skip-policy-pointer-reads','--timeout','60','--data-ramdisk','build/offline-renderbox-present-2/compiler-data-32.cdr','--qemu-ramfb','--inspect-boot-video', + '--offline-gpu-broker','--native-air-init','--original-client-abi-probe','--break-symbol','_native_original_client_checkpoint'] +if not a.missing_base:cmd+=['--load-iogpu'] +if a.default_off: + cmd.remove('--original-client-abi-probe');cmd.remove('--offline-gpu-broker');cmd.remove('--native-air-init');cmd[cmd.index('--break-symbol')+1]='_panic';cmd+=['--follow-target-main'] +with tempfile.TemporaryDirectory(prefix='accel-abi-',dir='/tmp') as directory: + path=Path(directory)/'console.sock';listener=socket.socket(socket.AF_UNIX,socket.SOCK_STREAM);listener.bind(str(path));listener.listen(1);listener.settimeout(1) + stop=threading.Event();capture=bytearray();failures=[] + def sink(): + connection=None + try: + while not stop.is_set(): + try:connection,_=listener.accept();break + except socket.timeout:continue + if connection: + connection.settimeout(1) + while not stop.is_set(): + try:chunk=connection.recv(4096) + except socket.timeout:continue + if not chunk:break + capture.extend(chunk) + if len(capture)>65536:failures.append('Bounded read-only console prefix exceeded');break + except OSError as error: + if not stop.is_set():failures.append(str(error)) + finally: + if connection:connection.close() + thread=threading.Thread(target=sink,daemon=True);thread.start() + try:subprocess.run([*cmd,'--virtio-console-socket',str(path)],cwd=ROOT,check=True,timeout=180) + finally:stop.set();listener.close();thread.join(timeout=3) + fixture.joinpath('console-sink.json').write_text(json.dumps(dict(bytes_observed=len(capture),sha256=hashlib.sha256(capture).hexdigest(),errors=failures, + protocol_responses_sent=0,gpu_execution=False,system_device_published=False),indent=2)+'\n') + if failures:raise SystemExit('; '.join(failures)) +print('Actual original userclient lifecycle/control observed; no device/client attached or published and no GPU protocol replies supplied') diff --git a/scripts/probe-original-gpu-provider.py b/scripts/probe-original-gpu-provider.py new file mode 100644 index 0000000..532f6ee --- /dev/null +++ b/scripts/probe-original-gpu-provider.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +"""Bounded unpublished original provider lifecycle or first required-hook observation.""" +import argparse,hashlib,json,shutil,socket,subprocess,tempfile,threading +from pathlib import Path +ROOT=Path(__file__).resolve().parents[1] +p=argparse.ArgumentParser(description=__doc__);p.add_argument('--missing-base',action='store_true');p.add_argument('--default-off',action='store_true');p.add_argument('--start',action='store_true');p.add_argument('--client-factory',action='store_true');p.add_argument('--descriptor-refusal',action='store_true');p.add_argument('--descriptor-provision',action='store_true',help='Supply one real owned IOMemoryDescriptor at the original slot-170 hook; no device/mapping/GPU success');p.add_argument('--descriptor-provision-attached',action='store_true',help='Attach/register the original provider before supplying the descriptor; still no device/GPU success');p.add_argument('--descriptor-provision-attached-only',action='store_true',help='Attach without registerService, then supply the descriptor; still no device/GPU success');p.add_argument('--armdt-platform',action='store_true',help='Use the sheng-shaped DT root with actual QEMU GIC resources; still no sheng MMIO');p.add_argument('--client-type',type=lambda value:int(value,0),default=1);p.add_argument('--candidate-base',type=Path,default=ROOT/'build/native-prelink/native-owner-original-descriptor-refusal-candidate');a=p.parse_args() +if sum((a.missing_base,a.default_off,a.start,a.client_factory,a.descriptor_refusal,a.descriptor_provision,a.descriptor_provision_attached,a.descriptor_provision_attached_only))>1:p.error('Select one independent lifecycle, start, client factory, refusal or descriptor-provision observation') +if not 0<=a.client_type<=0xffffffff or a.client_type!=1 and not (a.client_factory or a.descriptor_refusal or a.descriptor_provision or a.descriptor_provision_attached or a.descriptor_provision_attached_only):p.error('Actual uint32 open type requires the client-factory instrument') +if subprocess.run(['pgrep','-f','qemu-system-aarch64'],capture_output=True).returncode==0:raise SystemExit('Another QEMU is running') +if shutil.disk_usage(ROOT).free<5*1024**3+256*1024**2:raise SystemExit('Preserve 5 GiB free') +base=a.candidate_base.resolve();fixture=ROOT/'build'/(('darwinboot-owner-hvf-original-descriptor-provision-attached-only-'+hex(a.client_type) if a.descriptor_provision_attached_only else 'darwinboot-owner-hvf-original-descriptor-provision-attached-'+hex(a.client_type) if a.descriptor_provision_attached else 'darwinboot-owner-hvf-original-descriptor-provision-'+hex(a.client_type) if a.descriptor_provision else 'darwinboot-owner-hvf-original-client-descriptor-refusal-'+hex(a.client_type) if a.descriptor_refusal else 'darwinboot-owner-hvf-original-client-factory-'+hex(a.client_type) if a.client_factory else 'darwinboot-owner-hvf-original-provider-start' if a.start else 'darwinboot-owner-hvf-original-provider-abi')+('-missing-base' if a.missing_base else '-default-off' if a.default_off else '')) +if (fixture/'gdb-stop.json').exists():raise SystemExit('Preserve the existing observation before another run') +build=json.loads((ROOT/'build/native-original-provider/build-report.json').read_text()) +for rel,digest in build['source_sha256'].items(): + if hashlib.sha256((ROOT/rel).read_bytes()).hexdigest()!=digest:raise SystemExit('Exact compiled source required: '+rel) +cmd=['python3','scripts/probe-darwinboot-xnu.py','--kernel',str(base.with_suffix('.macho')),'--prelink-report',str(base.with_suffix('.json')),'--output',str(fixture), + '--virtio-udif-image','.cache/firmware/target-volumes/044-76040-038.dmg','--virtio-extra-block-image','.cache/firmware/target-volumes/044-76034-038.dmg', + '--virtio-mmio-fdt','build/virtio-block-fixture/qemu-virt.dtb','--root-match-volume-uuid','04769D26-2CA0-4825-B443-E2E5F88300F2', + '--trust-cache','build/offline-renderbox-present-2/offline-trust.bundle','--armvirt','--armcpu','--qemu-vmm-present','--qemu-accel','hvf','--owned-memory-size','0x20000000', + '--nvram-proxy','--crypto-hash-method','sha2-384','--amfi-platform-only','--load-apfs','--inspect-owner-root','--inspect-dyld-flags','--check-commpage', + '--skip-initializer-traces','--skip-policy-pointer-reads','--timeout','60','--data-ramdisk','build/offline-renderbox-present-2/compiler-data-32.cdr','--qemu-ramfb','--inspect-boot-video', + '--offline-gpu-broker','--native-air-init','--original-provider-probe','descriptor-provision-attached-only' if a.descriptor_provision_attached_only else 'descriptor-provision-attached' if a.descriptor_provision_attached else 'descriptor-provision' if a.descriptor_provision else 'client-refusal' if a.descriptor_refusal else 'client-factory' if a.client_factory else 'start' if a.start else 'lifecycle','--break-symbol','_native_original_provider_checkpoint'] +if a.client_factory or a.descriptor_refusal:cmd+=['--original-provider-client-type',str(a.client_type)] +if a.armdt_platform: + if '--armvirt' not in cmd:raise SystemExit('The sheng-shaped platform needs the offline QEMU GIC fixture') + cmd.insert(cmd.index('--armvirt')+1,'--armdt-platform') +if not a.missing_base:cmd+=['--load-iogpu'] +if a.default_off: + cmd.remove('--original-provider-probe');cmd.remove('lifecycle');cmd.remove('--offline-gpu-broker');cmd.remove('--native-air-init');cmd[cmd.index('--break-symbol')+1]='_panic';cmd+=['--follow-target-main'] +with tempfile.TemporaryDirectory(prefix='accel-abi-',dir='/tmp') as directory: + path=Path(directory)/'console.sock';listener=socket.socket(socket.AF_UNIX,socket.SOCK_STREAM);listener.bind(str(path));listener.listen(1);listener.settimeout(1) + stop=threading.Event();capture=bytearray();failures=[] + def sink(): + connection=None + try: + while not stop.is_set(): + try:connection,_=listener.accept();break + except socket.timeout:continue + if connection: + connection.settimeout(1) + while not stop.is_set(): + try:chunk=connection.recv(4096) + except socket.timeout:continue + if not chunk:break + capture.extend(chunk) + if len(capture)>65536:failures.append('Bounded read-only console prefix exceeded');break + except OSError as error: + if not stop.is_set():failures.append(str(error)) + finally: + if connection:connection.close() + thread=threading.Thread(target=sink,daemon=True);thread.start() + try:subprocess.run([*cmd,'--virtio-console-socket',str(path)],cwd=ROOT,check=True,timeout=180) + finally:stop.set();listener.close();thread.join(timeout=3) + fixture.joinpath('console-sink.json').write_text(json.dumps(dict(bytes_observed=len(capture),sha256=hashlib.sha256(capture).hexdigest(),errors=failures, + protocol_responses_sent=0,gpu_execution=False,system_device_published=False),indent=2)+'\n') + if failures:raise SystemExit('; '.join(failures)) +print('Actual unpublished provider lifecycle/required-hook boundary observed; no device/client published or successful GPU startup claimed') diff --git a/scripts/probe-original-sandbox-iokit.py b/scripts/probe-original-sandbox-iokit.py new file mode 100644 index 0000000..b4a25ef --- /dev/null +++ b/scripts/probe-original-sandbox-iokit.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python3 +"""Observe actual original Sandbox scoped IOKit access without changing original apps.""" +import argparse,hashlib,json,shutil,subprocess +from pathlib import Path +ROOT=Path(__file__).resolve().parents[1] +p=argparse.ArgumentParser(description=__doc__);a=p.parse_args() +def sha(path):return hashlib.sha256(path.read_bytes()).hexdigest() +if subprocess.run(['pgrep','-f','qemu-system-aarch64'],capture_output=True).returncode==0:raise SystemExit('Another QEMU is running') +if shutil.disk_usage(ROOT).free<5*1024**3+256*1024**2:raise SystemExit('Preserve 5 GiB free before allocating this fixture') +kind='offline-sandbox-extension';out=ROOT/'build'/kind +fixture=ROOT/'build'/('darwinboot-owner-hvf-'+kind) +if (fixture/'gdb-stop.json').exists():raise SystemExit('Preserve the existing observation before another run') +manifest=json.loads((out/'test-inputs.json').read_text()) +for name,digest in manifest['sources'].items(): + if sha(ROOT/name)!=digest:raise SystemExit('Exact actual instrument source required: '+name) +base=ROOT/'build/native-prelink/native-owner-accelerator-abi-candidate';plan=json.loads(base.with_suffix('.json').read_text()) +if sha(base.with_suffix('.macho'))!=plan['candidate_sha256']:raise SystemExit('Exact immutable candidate required') +cmd=['python3',str(ROOT/'scripts/probe-darwinboot-xnu.py'),'--kernel',str(base.with_suffix('.macho')),'--prelink-report',str(base.with_suffix('.json')),'--output',str(fixture), + '--virtio-udif-image','.cache/firmware/target-volumes/044-76040-038.dmg','--virtio-extra-block-image','.cache/firmware/target-volumes/044-76034-038.dmg', + '--virtio-mmio-fdt','build/virtio-block-fixture/qemu-virt.dtb','--root-match-volume-uuid','04769D26-2CA0-4825-B443-E2E5F88300F2', + '--trust-cache',str(out/'offline-trust.bundle'),'--armvirt','--armcpu','--qemu-vmm-present','--qemu-accel','hvf','--owned-memory-size','0x20000000', + '--nvram-proxy','--crypto-hash-method','sha2-384','--amfi-platform-only','--load-apfs','--load-iogpu','--inspect-owner-root','--inspect-dyld-flags','--check-commpage', + '--skip-initializer-traces','--timeout','60','--data-ramdisk',str(out/'compiler-data-32.cdr'),'--qemu-ramfb','--inspect-boot-video', + '--load-mobile-graphics','--mobile-graphics-client-probe','--legacy-display-phase','4','--legacy-display-gop','--legacy-display-register','--legacy-display-unit-scale', + '--legacy-display-logical-pm','--legacy-display-validate','--legacy-display-transfer','--inspect-gop-scanout-bytes','1920000', + '--native-air-init','--break-symbol','_exception_triage','--entry-process-name','air-native','--entry-register-equals','0:6','--inspect-current-process','--inspect-entry-user-state','--read-saved-user-register','13:100', + '--read-saved-user-register','14:1000','--capture-qemu-screen'] +cmd+=['--legacy-display-retire'] +subprocess.run(cmd,cwd=ROOT,check=True,timeout=180) +print('Actual original Sandbox scoped authorization observation captured; inspect real issue/consume/open/release outcomes') diff --git a/scripts/probe-stock-entry.py b/scripts/probe-stock-entry.py new file mode 100755 index 0000000..1e27c00 --- /dev/null +++ b/scripts/probe-stock-entry.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python3 +"""Observe the first exception from an unchanged Mach-O on QEMU virt. + +This tests the instruction entry boundary, not an XNU platform or OS boot. +The fixture supplies skeletal boot arguments and an optional test device tree. +""" +import argparse +import hashlib +import json +from pathlib import Path +import re +import shutil +import struct +import subprocess +import time + +ROOT = Path(__file__).resolve().parent.parent +PHYS_BASE = 0x50000000 +BOOT_ARGS_PA = 0x5ff00000 + + +def prepare(kernel, output, phys_base=PHYS_BASE, boot_args_pa=BOOT_ARGS_PA, device_tree=None, + command_line='-v serial=3 debug=0x14e'): + data = kernel.read_bytes() + if data[:4] != b'\xcf\xfa\xed\xfe': + raise ValueError('Expected decompressed little-endian 64-bit Mach-O') + cpu, subtype, kind, count, commands = struct.unpack_from(' len(data) - 32: + raise ValueError('Expected AArch64 MH_EXECUTE with complete load commands') + offset = 32 + segments = [] + entry = None + for _ in range(count): + if offset + 8 > 32 + commands: + raise ValueError('Truncated Mach-O command') + command, size = struct.unpack_from(' 32 + commands: + raise ValueError('Invalid Mach-O command size') + if command == 0x19: + if size < 72: + raise ValueError('Truncated LC_SEGMENT_64') + name, address, length, fileoff, filesize = struct.unpack_from('<16sQQQQ', data, offset + 8) + if filesize > length or fileoff + filesize > len(data): + raise ValueError('Invalid segment file extent') + if length: + segments.append((name.rstrip(b'\0').decode(), address, length, fileoff, filesize)) + elif command == 5: + if size != 288 or struct.unpack_from(' right[1]: + raise ValueError('Overlapping virtual segments') + base = segments[0][1] + end = max(s[1] + s[2] for s in segments) + if not 0x40000000 <= phys_base < boot_args_pa or phys_base % 4096: + raise ValueError('Physical base must be page-aligned in the fixture RAM') + if boot_args_pa % 4096 or boot_args_pa + 0x4000 > 0x60000000: + raise ValueError('Boot arguments must be page-aligned in fixture RAM') + if end - base > boot_args_pa - phys_base or not base <= entry < end: + raise ValueError('Kernel does not fit the fixed test memory layout') + image = bytearray(end - base) + for name, address, length, fileoff, filesize in segments: + image[address - base:address - base + filesize] = data[fileoff:fileoff + filesize] + (output / 'kernel-flat.bin').write_bytes(image) + boot = bytearray(0x4000) + struct.pack_into('= 1024: + raise ValueError('Command line exceeds boot_args capacity') + boot[0x6c:0x6c + len(encoded_command_line)] = encoded_command_line + struct.pack_into(' len(boot) - 0x1000: + raise ValueError('Test device tree exceeds reserved boot buffer') + boot[0x1000:0x1000 + len(tree)] = tree + struct.pack_into(' 1024 * 1024): + break + time.sleep(0.05) + finally: + if process.poll() is None: + process.terminate() + try: + process.wait(timeout=2) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + text = log.read_text(errors='replace') if log.exists() else '' + exceptions = trace.read_text(errors='replace') if trace.exists() else '' + report.update(cpu=args.cpu, seconds=round(time.monotonic() - started, 3), + outcome='exception' if ('exception captured' in text or 'Taking exception' in exceptions) + else 'no-exception-captured', + os_boot_demonstrated=False) + report['qemu_exception_trace'] = exceptions[:4096] + fault = re.search(r'with ELR 0x([0-9a-f]+)', exceptions) + if fault: + pc = int(fault[1], 16) + in_virtual_segment = any(int(s['va'], 16) <= pc < int(s['va'], 16) + s['size'] + for s in report['segments']) + in_physical_image = args.phys_base <= pc < args.phys_base + report['mapped_bytes'] + virtual = pc if in_virtual_segment else ( + pc - args.phys_base + int(report['virtual_base'], 16) if in_physical_image else None) + report['fault_pc'] = hex(pc) + report['fault_address_space'] = ('kernel-virtual' if in_virtual_segment else + 'physical' if in_physical_image else 'unmapped') + if virtual is not None: + report['fault_va'] = hex(virtual) + binary = args.kernel.read_bytes() + offset = 32 + for _ in range(struct.unpack_from('> 19) & 3}_{(opcode >> 16) & 7}' + f'_C{(opcode >> 12) & 15}_C{(opcode >> 8) & 15}_{(opcode >> 5) & 7}') + offset += size + for register in ('CurrentEL', 'MMFR0', 'CNTFRQ', 'ESR_EL1', 'ELR_EL1', 'FAR_EL1'): + match = re.search(register + r'=0x([0-9a-f]+)', text) + if match: + report[register] = '0x' + match[1] + (output / f'{args.cpu}.json').write_text(json.dumps(report, indent=2) + '\n') + print(text, end='') + print(json.dumps({k: v for k, v in report.items() if k != 'segments'}, indent=2)) + if 'entering unchanged kernel' not in text: + raise SystemExit('Probe did not reach the entry boundary; inspect stderr log') diff --git a/scripts/probe-target-dyld.py b/scripts/probe-target-dyld.py new file mode 100644 index 0000000..e85d67a --- /dev/null +++ b/scripts/probe-target-dyld.py @@ -0,0 +1,252 @@ +#!/usr/bin/env python3 +"""Observe genuine supplied dyld instructions at EL0 on native-4K XNU, offline.""" +import argparse +import hashlib +import json +from pathlib import Path +import re +import shutil +import subprocess + +ROOT = Path(__file__).resolve().parents[1] +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument('--kernel', type=Path, default=ROOT / 'build/native-prelink/native-target-dyld-candidate.macho') +parser.add_argument('--output', type=Path, default=ROOT / 'build/darwinboot-target-dyld-entry') +parser.add_argument('--mach-init', action='store_true', help='Observe genuine original mach_init returning and qualify its page ABI') +parser.add_argument('--prepare', action='store_true', help='Continue genuine original startup toward its dependent-library preparation') +parser.add_argument('--load-cache', action='store_true', help='Observe the original loadDyldCache function returning to its real caller') +parser.add_argument('--log-syscalls', action='store_true', help='Record bounded actual saved syscall states to diagnose startup waits') +parser.add_argument('--log-service-matching', action='store_true', help='Observe genuine IOService class-match requests and their callers') +parser.add_argument('--os-image', type=Path, help='Original supplied OS UDIF, already attached read-only; replace the minimal RAM root') +parser.add_argument('--root-bsd-name', help='Actual previously observed OS volume BSD name') +parser.add_argument('--virtio-mmio-fdt', type=Path, help='Actual QEMU FDT for the corresponding offline block transport') +parser.add_argument('--apfs-component-launcher', type=Path, help='Own RAM launcher for two read-only data mounts/chroot/exec; not a production root') +parser.add_argument('--systemos-image', type=Path, help='Second existing raw SystemCryptex image for the component launcher') +parser.add_argument('--timeout', type=int, choices=range(1, 61), default=25, help='Bounded guest deadline, at most 60 seconds') +parser.add_argument('--corrupt-dyld-byte', type=lambda v: int(v, 0), + help='Negative control only: flip one code byte in the isolated RAM root, leaving original signatures/trust unchanged') +args = parser.parse_args() +if sum((args.mach_init, args.prepare, args.load_cache)) > 1: + parser.error('Choose one original dyld checkpoint') +if args.corrupt_dyld_byte is not None and not args.mach_init: + parser.error('The corruption refusal control requires --mach-init') +if args.apfs_component_launcher: + if not all((args.os_image, args.systemos_image, args.virtio_mmio_fdt)) or args.root_bsd_name or args.corrupt_dyld_byte is not None: + parser.error('The component launcher requires both original disk inputs and the FDT, without a boot-root override') +elif args.systemos_image: + parser.error('--systemos-image requires the explicit own component launcher') +elif any((args.os_image, args.root_bsd_name, args.virtio_mmio_fdt)): + if not all((args.os_image, args.root_bsd_name, args.virtio_mmio_fdt)) or args.corrupt_dyld_byte is not None: + parser.error('A real OS root requires its image, actual BSD name and actual FDT, without RAM-root corruption') + if not re.fullmatch(r'disk[0-9]+(?:s[0-9]+)*', args.root_bsd_name): + parser.error('Expected an actual observed disk BSD name') +output = args.output.resolve() +fixture = ROOT / 'build/target-dyld-fixture' +inputs = json.loads((fixture / 'inputs.json').read_text()) +prelink = json.loads(args.kernel.with_suffix('.json').read_text()) +if hashlib.sha256(args.kernel.read_bytes()).hexdigest() != prelink['candidate_sha256']: + raise SystemExit('Candidate identity mismatch') +subprocess.run(['python3', ROOT / 'scripts/audit-xnu-kernel.py', args.kernel], check=True) +esp = output / 'esp' +payloads = esp / 'EFI/CrabApple/Payloads' +payloads.mkdir(parents=True, exist_ok=True) +(esp / 'EFI/BOOT').mkdir(exist_ok=True) +for source, target in [ + (ROOT / 'crabapple/Build/Out/AARCH64-DEBUG/CrabApple.efi', esp / 'EFI/BOOT/BOOTAA64.EFI'), + (ROOT / 'build/DarwinBoot/DEBUG_CLANGPDB/AARCH64/DarwinBoot.efi', payloads / 'DarwinBoot.efi'), + (args.kernel, payloads / 'kernel.macho'), + (fixture / 'original-static-trust.bundle', payloads / 'kernel.macho.trustcache')]: + shutil.copyfile(source, target) +if args.apfs_component_launcher: + launcher = args.apfs_component_launcher.parent + launcher_metadata = json.loads((launcher / 'apfs-launchd.json').read_text()) + if (hashlib.sha256(args.apfs_component_launcher.read_bytes()).hexdigest() != launcher_metadata['own_process_sha256'] or + hashlib.sha256((launcher / 'apfs-launchd-trust.bundle').read_bytes()).hexdigest() != launcher_metadata['trust_bundle_sha256']): + raise SystemExit('Component launcher and explicit trust bundle identities must match their build record') + if launcher_metadata['stop_before_exec']: + raise SystemExit('The original-dyld probe needs a launcher that actually execs') + shutil.copyfile(args.apfs_component_launcher, payloads / 'root.img') + shutil.copyfile(launcher / 'apfs-launchd-trust.bundle', payloads / 'kernel.macho.trustcache') +elif not args.os_image: + shutil.copyfile(fixture / 'root.img', payloads / 'root.img') +if not args.apfs_component_launcher and ((not args.os_image and hashlib.sha256((payloads / 'root.img').read_bytes()).hexdigest() != inputs['root_sha256']) or \ + hashlib.sha256((payloads / 'kernel.macho.trustcache').read_bytes()).hexdigest() != inputs['trust_bundle_sha256']): + raise SystemExit('Exact original target inputs were not preserved') +negative_input = None +if args.corrupt_dyld_byte is not None: + import struct + image = bytearray((payloads / 'root.img').read_bytes()) + _, _, _, _, _, dyld_offset, dyld_size = struct.unpack_from('<8sII4Q', image) + at = args.corrupt_dyld_byte + if not 0 <= at < min(dyld_size, 0x80000): + raise SystemExit('Select a byte in original dyld code, not its signature') + before = image[dyld_offset + at] + image[dyld_offset + at] ^= 1 + (payloads / 'root.img').write_bytes(image) + negative_input = dict(file='usr/lib/dyld', file_offset=at, before=before, after=before ^ 1, + input_copy_sha256=hashlib.sha256(image).hexdigest(), signatures_or_trust_modified=False, + supplied_original_file_modified=False, input_fault_instrument_only=True) +dt_command = ['python3', ROOT / 'scripts/qemu-entry/make-dt.py', '--output', payloads / 'kernel.macho.adt', + '--dram-size', '0x40000000', '--armvirt', '--armcpu', '--nvram-proxy', + '--crypto-hash-method', 'sha2-384', '--amfi-platform-only', '--trust-cache'] +if args.os_image: + dt_command += ['--virtio-mmio-fdt', args.virtio_mmio_fdt] +if not args.os_image or args.apfs_component_launcher: + dt_command += ['--ramdisk'] +subprocess.run(dt_command, check=True) +(esp / 'EFI/CrabApple/crabapple.toml').write_text('''version = 1 +[boot] +timeout = 1 +default = "target-dyld" +[kits] +autoload = false +[[driver]] +path = "Payloads/DarwinBoot.efi" +[[entry]] +id = "target-dyld" +title = "Original target dyld offline native 4K probe" +type = "xnu" +path = "Payloads/kernel.macho" +initrd = "Payloads/root.img" +args = "rd=md0 -v serial=3 debug=0x14e kt_fixed_zones=120" +''') +if args.os_image: + if 'com.apple.filesystems.apfs' not in prelink['components']: + raise SystemExit('The genuine original APFS component is required') + config = esp / 'EFI/CrabApple/crabapple.toml' + if args.apfs_component_launcher: + config.write_text(config.read_text().replace('rd=md0 ', 'rd=md0 offline-apfs-load=1 ')) + else: + config.write_text(config.read_text().replace('initrd = "Payloads/root.img"\n', '').replace( + 'rd=md0 ', f'rd={args.root_bsd_name} offline-apfs-load=1 ')) +symbols = subprocess.check_output(['xcrun', 'nm', '-U', args.kernel], text=True) +addresses = {name: int(at, 16) for at, name in re.findall(r'^([0-9a-fA-F]+)\s+\S\s+(_\S+)$', symbols, re.M)} +checkpoint = inputs['dyld_symbols']['_mach_init'] if args.mach_init else inputs['dyld_startup_checkpoint'] +if args.prepare: + checkpoint = inputs['dyld_symbols']['__ZN5dyld4L7prepareERNS_4APIsEPKN5dyld313MachOAnalyzerE'] + 4 +if args.load_cache: + dyld_path = fixture / 'dyld' + expected = next(f['sha256'] for f in inputs['files'] if f['path'] == 'usr/lib/dyld') + if hashlib.sha256(dyld_path.read_bytes()).hexdigest() != expected: + raise SystemExit('Unexpected original dyld symbol input') + dyld_names = subprocess.check_output(['xcrun', 'nm', '-U', dyld_path], text=True) + cache_name = '__ZN5dyld313loadDyldCacheERKNS_18SharedCacheOptionsEPNS_19SharedCacheLoadInfoE' + cache_match = re.search(r'^([0-9a-fA-F]+)\s+\S\s+' + re.escape(cache_name) + '$', dyld_names, re.M) + if not cache_match: + raise SystemExit('The actual original cache loader symbol is unavailable') + checkpoint = int(cache_match[1], 16) + 4 +command = ['python3', ROOT / 'scripts/qemu-entry/gdb-stop.py', hex(addresses['_thread_setentrypoint']), + '--fixture', output, '--esp', esp, '--kernel', args.kernel, '--gic-version', '3', '--system-registers', + '--timeout', str(args.timeout), '--follow-entrypoint-arg', '1', '--step-followed-entry', '64', + '--follow-entrypoint-checkpoint-offset=' + str(checkpoint - inputs['dyld_preferred_entry'])] +if args.os_image: + command += ['--virtio-udif-image', args.os_image] +if args.apfs_component_launcher: + command += ['--virtio-extra-block-image', args.systemos_image, '--skip-target-hits', '1'] +dyld_requests = {} +if args.mach_init or args.prepare or args.load_cache: + if args.mach_init or args.load_cache: + command += ['--finish-followed-checkpoint'] + if args.load_cache: + command += ['--read-followed-checkpoint-arg', '1:40'] + for name, width in dict(_vm_page_size=8, _vm_page_mask=8, _vm_page_shift=4, _vm_kernel_page_shift=4).items(): + request = f'{inputs["dyld_symbols"][name] - inputs["dyld_preferred_entry"]}:{width:x}' + dyld_requests[name] = request + command += ['--read-followed-relative', request] +for name in ('_panic', '_panic_with_options'): + command += ['--also-stop', hex(addresses[name])] +if args.log_syscalls: + for name in ('_unix_syscall', '_mach_syscall'): + command += ['--syscall-log-address', hex(addresses[name])] +if args.log_service_matching: + command += ['--cstring-log-address', hex(addresses['__ZN9IOService15serviceMatchingEPKcP12OSDictionary'])] +inspections = dict(_PAGE_SHIFT_CONST=4, _page_shift=4, _page_size=8, + _cs_validate_page_bad_hash=8, _cs_validate_page_no_hash=8, _amfi=8, + _armcpu_timer_interrupts=8, _armcpu_ipi_interrupts=8, _armcpu_gic_ready=4) +if '_armvirt_platform_defaults_count' in addresses: + inspections['_armvirt_platform_defaults_count'] = 4 +if args.os_image: + inspections.update(_native_virtio_ready=4, _native_virtio_read_requests=4, _native_virtio_read_failures=4, + _native_virtio_sectors=8, _native_virtio_mmio_physical=8) +for name in ('_native_aes_kat_status', '_native_aes_interface_status', '_native_aes_registered', + '_native_aes_completed', '_native_aes_unsupported_key'): + if name in addresses: + inspections[name] = 4 +requests = {name: f'{addresses[name]:x}:{width:x}' for name, width in inspections.items()} +for request in requests.values(): + command += ['--read-memory', request] +for name in ('_commpage_rw_addr', '_commpage_kernel_ro_addr'): + command += ['--read-indirect', f'{addresses[name]:x}:25:1'] +command += ['--read-memory', 'fffff4025:1', '--read-memory', 'fffff4037:1'] +subprocess.run(command, check=True) +report_path = output / 'gdb-stop.json' +d = json.loads(report_path.read_text()) +r = {k: int(v, 16) for k, v in d['system_registers'].items()} +native = (r['TCR_EL1'] >> 14 & 3 == 0 and r['TCR_EL1'] >> 30 & 3 == 2 and + r['SCTLR_EL1'] & 1 and r['TTBR0_EL1'] and r['TTBR1_EL1']) +d['native_4k_mmu_observed'] = bool(native) +d['target_inputs'] = inputs +if args.os_image: + d['target_inputs'] = {k: v for k, v in inputs.items() if k not in ('root_sha256', 'root_bytes')} + if args.apfs_component_launcher: + d['boot_root_input'] = 'own RAM fixture; original APFS volumes are read-only data mounts' + d['offline_component_launcher'] = launcher_metadata + d['root_authentication_solved'] = False + d['actual_trust_bundle_sha256'] = hashlib.sha256((payloads / 'kernel.macho.trustcache').read_bytes()).hexdigest() + d['actual_ram_root_sha256'] = hashlib.sha256((payloads / 'root.img').read_bytes()).hexdigest() + else: + d['boot_root_input'] = 'actual supplied OS APFS volume; no RAM root' + d['selected_bsd_name'] = args.root_bsd_name +if negative_input: + d['negative_input_control'] = negative_input +d['kernel_observations'] = {name: int.from_bytes(bytes.fromhex(d['memory'][request]), 'little') + for name, request in requests.items()} +entry = d.get('followed_user_entry', {}) +dyld = (fixture / 'dyld').read_bytes() +slide = int(entry['user_entry'], 16) - inputs['dyld_preferred_entry'] if entry else None +confirmed = [] +if entry.get('observed') and int(entry['pstate'], 16) & 15 == 0 and \ + entry['entry_bytes'] == dyld[inputs['dyld_preferred_entry']:inputs['dyld_preferred_entry'] + 16].hex(): + for step in entry['steps']: + offset = int(step['pc'], 16) - slide + if int(step['pstate'], 16) & 15 == 0 and 0 <= offset < 0x80000 - 16 and \ + step['bytes'] == dyld[offset:offset + 16].hex(): + confirmed.append(step['pc']) +d['original_dyld_el0_instruction_observations'] = len(confirmed) +d['original_dyld_slide'] = hex(slide) if slide is not None else None +if args.mach_init or args.prepare or args.load_cache: + d['original_dyld_page_abi'] = {} + for name, request in dyld_requests.items(): + raw = d.get('followed_relative_memory', {}).get(request, {}).get('bytes', 'E14') + d['original_dyld_page_abi'][name] = int.from_bytes(bytes.fromhex(raw), 'little') if not raw.startswith('E') else None + if args.mach_init: + d['original_dyld_mach_init_returned'] = entry.get('function_return', {}).get('observed', False) + if args.prepare: + d['original_dyld_prepare_entered'] = entry.get('checkpoint', {}).get('observed', False) + if args.load_cache: + d['original_dyld_cache_loader_returned'] = entry.get('function_return', {}).get('observed', False) + d['original_dyld_cache_loader_symbol'] = cache_name + d['original_dyld_native_4k_abi_observed'] = ((entry.get('function_return', {}).get('observed', False) if args.mach_init or args.load_cache else + entry.get('checkpoint', {}).get('observed', False)) and + d['original_dyld_page_abi'] == dict(_vm_page_size=4096, _vm_page_mask=4095, _vm_page_shift=12, _vm_kernel_page_shift=12)) +if (d.get('argument_memory', {}).get('x3') or '').startswith('%s %s -- exit reason namespace'): + import struct + raw = d['stack_bytes'] + if not raw.startswith('E'): + exit_args = struct.unpack_from('<5Q', bytes.fromhex(raw)) + d['target_process_exit'] = dict(namespace=exit_args[2] & 0xffffffff, code=exit_args[3]) +d['target_launchd_main_executed'] = False +d['functional_ipados_boot_demonstrated'] = False +d['sheng_hardware_tested'] = False +report_path.write_text(json.dumps(d, indent=2) + '\n') +if negative_input: + if native and not d['bounded_timeout_interrupted'] and not entry.get('function_return', {}).get('observed') and \ + d['kernel_observations']['_cs_validate_page_bad_hash'] >= 1: + print('PASS: corrupted original dyld input copy was refused by genuine code-page validation') + raise SystemExit(0) + raise SystemExit('Original dyld corruption refusal is not yet established; inspect the bounded report') +if not native or d['bounded_timeout_interrupted'] or len(confirmed) < 8 or \ + ((args.mach_init or args.prepare or args.load_cache) and not d['original_dyld_native_4k_abi_observed']): + raise SystemExit('Target dyld execution is not yet established; inspect the bounded report') +print(f'PASS: {len(confirmed)} byte-matched original dyld instruction observations at native-4K EL0') +print('Original static trust module is unchanged; authenticated boot chain and functional iPadOS remain unqualified') diff --git a/scripts/qemu-entry/bootshim-fixture.S b/scripts/qemu-entry/bootshim-fixture.S new file mode 100644 index 0000000..c9a28b6 --- /dev/null +++ b/scripts/qemu-entry/bootshim-fixture.S @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: BSD-2-Clause-Patent +.section .text, "ax" +.global _start +_start: +#ifndef PAYLOAD + msr daifset, #15 + ldr x0, =0x40010000 + ldr x19, =(SHIM_BASE + SHIM_SIZE) + ldr x20, =FD_BASE + ldr x1, =SHIM_BASE + br x1 +#else + ldr x1, =0x40010000 + cmp x0, x1 + b.ne failed + adr x1, _start + cmp x1, x20 + b.ne failed + mov x1, x19 + mov x2, x20 + ldr x3, =FD_SIZE +1: + ldp x4, x5, [x1], #16 + ldp x6, x7, [x2], #16 + eor x4, x4, x6 + eor x5, x5, x7 + orr x4, x4, x5 + cbnz x4, failed + subs x3, x3, #16 + b.ne 1b + adr x0, passed + b puts +failed: + adr x0, failure +puts: + mov x1, #0x09000000 +2: + ldrb w2, [x0], #1 + cbz w2, done +3: + ldr w3, [x1, #0x18] + tbnz w3, #5, 3b + str w2, [x1] + b 2b +done: + wfe + b done +passed: + .asciz "BOOTSHIM: full copy, destination entry and x0 PASS\n" +failure: + .asciz "BOOTSHIM: FAIL\n" +#endif diff --git a/scripts/qemu-entry/entry.S b/scripts/qemu-entry/entry.S new file mode 100644 index 0000000..fa1de6c --- /dev/null +++ b/scripts/qemu-entry/entry.S @@ -0,0 +1,94 @@ +// SPDX-License-Identifier: BSD-2-Clause-Patent +.section .text.boot, "ax" +.global _start +_start: + msr daifset, #15 + adr x0, vectors + msr vbar_el1, x0 + isb + adr x0, banner + bl puts + mrs x0, CurrentEL + bl hex + adr x0, granules + bl puts + mrs x0, id_aa64mmfr0_el1 + bl hex + adr x0, frequency + bl puts + mrs x0, cntfrq_el0 + bl hex + adr x0, entering + bl puts + ldr x0, =BOOT_ARGS_PA + ldr x1, =KERNEL_ENTRY_PA + br x1 + +exception: + adr x0, trapped + bl puts + mrs x0, esr_el1 + bl hex + adr x0, faultpc + bl puts + mrs x0, elr_el1 + bl hex + adr x0, faultaddress + bl puts + mrs x0, far_el1 + bl hex + adr x0, complete + bl puts +1: + wfe + b 1b + +puts: + mov x1, #0x09000000 +1: + ldrb w2, [x0], #1 + cbz w2, 3f +2: + ldr w3, [x1, #0x18] + tbnz w3, #5, 2b + str w2, [x1] + b 1b +3: + ret + +hex: + mov x1, #0x09000000 + mov x2, #60 + adr x5, digits +1: + lsr x3, x0, x2 + and x3, x3, #15 + ldrb w4, [x5, x3] +2: + ldr w6, [x1, #0x18] + tbnz w6, #5, 2b + str w4, [x1] + subs x2, x2, #4 + b.ge 1b + mov w4, #13 + str w4, [x1] + mov w4, #10 + str w4, [x1] + ret + +.balign 2048 +vectors: +.rept 16 + b exception + .balign 128 +.endr + +banner: .asciz "ENTRY-PROBE: CurrentEL=0x" +granules: .asciz "ENTRY-PROBE: MMFR0=0x" +frequency: .asciz "ENTRY-PROBE: CNTFRQ=0x" +entering: .asciz "ENTRY-PROBE: entering unchanged kernel\r\n" +trapped: .asciz "ENTRY-PROBE: ESR_EL1=0x" +faultpc: .asciz "ENTRY-PROBE: ELR_EL1=0x" +faultaddress: .asciz "ENTRY-PROBE: FAR_EL1=0x" +complete: .asciz "ENTRY-PROBE: exception captured\r\n" +digits: .ascii "0123456789abcdef" diff --git a/scripts/qemu-entry/gdb-stop.py b/scripts/qemu-entry/gdb-stop.py new file mode 100755 index 0000000..9038ae5 --- /dev/null +++ b/scripts/qemu-entry/gdb-stop.py @@ -0,0 +1,1729 @@ +#!/usr/bin/env python3 +"""Stop the prepared QEMU entry fixture at an address and record registers.""" +import argparse +import hashlib +import json +import platform +from pathlib import Path +import shutil +import socket +import stat +import struct +import subprocess +import time +import tempfile +import xml.etree.ElementTree as ET + +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument('address', type=lambda value: int(value, 0)) +parser.add_argument('--qemu-accel', choices=['tcg', 'hvf'], default='tcg', help='Offline QEMU accelerator; HVF is restricted to macOS ARM64 ESP tests') +parser.add_argument('--qemu-memory-mib', type=int, choices=[512, 1024, 2048, 4096, 8192], + help='Actual VM RAM; defaults remain 1024 MiB for ESP and 512 MiB for raw fixtures') +parser.add_argument('--fixture', type=Path, default=Path('build/xnu-entry')) +parser.add_argument('--read-memory', action='append', default=[], metavar='HEX_ADDRESS:HEX_LENGTH') +parser.add_argument('--read-indirect', action='append', default=[], metavar='HEX_POINTER_ADDRESS:HEX_OFFSET:HEX_LENGTH') +parser.add_argument('--mac-policy-layout',type=Path,help='Compiler-derived bounded read-only registered policy observer') +parser.add_argument('--system-registers', action='store_true') +parser.add_argument('--live-hid-dir',type=Path,help='QEMU-only own HID mailbox at actual timer callbacks; bounded by timeout') +parser.add_argument('--exercise-ufs-allocation', action='store_true', + help='Offline only: invoke actual XNU UFS-sized allocation APIs after the checkpoint; never maps UFS hardware') +parser.add_argument('--esp', type=Path, help='Boot a read-only ESP through real QEMU UEFI instead of the raw fixture') +parser.add_argument('--kernel', type=Path, help='Kernel whose SHA-256 identifies an ESP test') +block_group = parser.add_mutually_exclusive_group() +block_group.add_argument('--virtio-block-image', type=Path, help='Attach an existing sector-aligned raw file as a strictly read-only modern MMIO disk') +block_group.add_argument('--virtio-udif-image', type=Path, help='Read an already attached read-only UDIF image through its verified host virtual media') +parser.add_argument('--virtio-extra-block-image', type=Path, help='Second existing raw file, strictly read-only, for offline two-volume component tests') +parser.add_argument('--missing-virtio-device', action='store_true', help='Refusal control: leave described MMIO windows empty while retaining the read-only input identity') +parser.add_argument('--capture-screen',action='store_true',help='Read the actual stopped QEMU console through QMP screendump into a PNG; no guest execution or drawing') +parser.add_argument('--qemu-ramfb', action='store_true', help='Attach actual offline QEMU ramfb for UEFI GOP handoff qualification') +parser.add_argument('--virtio-console-socket', type=Path, help='Connect an existing private Unix socket to real modern VirtIO console port zero') +parser.add_argument('--firmware', type=Path, help='QEMU AArch64 UEFI code FD for an ESP test') +parser.add_argument('--timeout', type=float, default=3, help='Maximum seconds waiting for the guest breakpoint') +parser.add_argument('--primary-start-address', type=lambda value: int(value, 0), help='Arm the primary breakpoint only after this actual one-shot kernel entry') +parser.add_argument('--trace-address', action='append', default=[], type=lambda value: int(value, 0)) +parser.add_argument('--syscall-number', type=int, action='append', default=[], help='Capture only these genuine syscall numbers, preserving the bounded event budget') +parser.add_argument('--syscall-start-address', type=lambda value: int(value, 0), help='Arm syscall saved-state observation only after this actual kernel entry') +parser.add_argument('--kext-log-address', type=lambda value: int(value, 0), help='Capture up to 256 real OSKextLog calls within the same guest deadline') +parser.add_argument('--syscall-log-address', type=lambda value: int(value, 0), action='append', default=[], + help='Observe genuine Darwin 23 ARM64 syscall handlers; at most 512 saved states in the same deadline') +parser.add_argument('--cstring-log-address', type=lambda value: int(value, 0), action='append', default=[], + help='Observe at most 128 genuine calls whose first argument is a kernel C string') +parser.add_argument('--cstring-start-address', type=lambda value: int(value, 0), help='Arm C-string observation only after this real entrypoint; conserves hardware breakpoints and the bounded log budget') +parser.add_argument('--registry-string-log-address', type=lambda value: int(value, 0), + help='Observe at most 128 real IORegistryEntry::setProperty(char const*, char const*) calls') +parser.add_argument('--also-stop', action='append', default=[], type=lambda value: int(value, 0)) +parser.add_argument('--finish-function', action='store_true', help='After the entry breakpoint, stop at its caller return address') +parser.add_argument('--skip-target-hits', type=int, default=0, + help='Skip at most 16 earlier calls to the primary breakpoint within the same deadline') +parser.add_argument('--follow-entrypoint-arg', type=int, choices=range(4), + help='Observe the actual user PC passed in x0-x3, then follow it without changing state') +parser.add_argument('--follow-main-entry-offset', type=lambda value: int(value, 0), help='Observe the main executable LC_MAIN offset relative to its actual Mach header on the original dyld startup stack') +parser.add_argument('--main-cache-query-plan', type=Path, help='Follow an audited original cache display query after real main entry; read-only hardware breakpoints') +parser.add_argument('--process-layout-plan', type=Path, help='Inspect current process identity with compiler-qualified kernel layout') +parser.add_argument('--allproc-address', type=lambda value: int(value, 0), help='Read the actual kernel allproc list once at the final stop') +parser.add_argument('--proc-list-offset', type=int, default=0, help='Byte offset of p_list.le_next in struct proc') +parser.add_argument('--proc-pid-offset', type=int, default=96, help='Byte offset of p_pid') +parser.add_argument('--proc-comm-offset', type=int, default=1468, help='Byte offset of p_comm') +parser.add_argument('--proc-stat-offset', type=int, default=100, help='Byte offset of p_stat') +parser.add_argument('--proc-list-max', type=int, default=128, help='Bounded number of actual process entries') +parser.add_argument('--inspect-ui-threads', action='store_true', help='Read original UI task thread lists and kernel-owned saved user registers, without changing address spaces') +parser.add_argument('--capture-scratch-data', metavar='BASE_SYMBOL:SIZE_SYMBOL:EXPECTED_BYTES', + help='Read the exact native mutable Data RAM range into an offline snapshot; never flush or modify guest state') + +parser.add_argument('--step-followed-entry', type=int, default=0, + help='Execute at most 64 real instructions after reaching that user entry') +parser.add_argument('--follow-entrypoint-checkpoint-offset', type=lambda value: int(value, 0), + help='After entry, continue normally to this source-derived relative user checkpoint') +parser.add_argument('--finish-followed-checkpoint', action='store_true', + help='Observe that real user function returning to its caller, without state changes') +parser.add_argument('--read-followed-checkpoint-arg', action='append', default=[], metavar='ARG:HEX_LENGTH', + help='Read at most 4096 bytes at an original user checkpoint argument after its real return') +parser.add_argument('--read-followed-relative', action='append', default=[], metavar='SIGNED_OFFSET:HEX_LENGTH', + help='Read bounded user data relative to the actual observed user entry') +parser.add_argument('--entry-kernel-bytes-field', help='Select a real call by ARG:HEX_OFFSET:HEX_BYTES in bounded kernel memory') +parser.add_argument('--entry-process-name', help='Select a real primary call by compiler-qualified current process name; bounded read-only observation') +parser.add_argument('--entry-min-pid',type=int,default=0,help='Minimum actual PID for an explicitly named process predicate') +parser.add_argument('--entry-register-equals', help='Select a real call by ARG:HEX_VALUE; observation only') +parser.add_argument('--entry-max-skips', type=int, default=16, help='Bounded real entry selection attempts, at most 65536') +parser.add_argument('--read-entry-region', action='append', default=[], help='Read a bounded direct kernel region ARG:HEX_OFFSET:HEX_LENGTH at entry or after real return') +parser.add_argument('--read-saved-user-register',action='append',default=[],help='At an ABI-qualified saved EL0 state, read bounded user memory through HEX_REGISTER:HEX_LENGTH') +parser.add_argument('--inspect-entry-user-state', action='store_true', help='Read ABI-qualified user saved states from bounded saved registers and compiler-derived current-thread linkage') +parser.add_argument('--read-frame-locals', action='store_true', help='Read at most 192 bytes below each of at most 16 actual kernel frame pointers') +parser.add_argument('--read-frame-indirect', action='append', default=[], help='Read kernel memory through a real saved frame slot FRAME:SIGNED_HEX_OFFSET:HEX_LENGTH') +parser.add_argument('--entry-user-string-field', metavar='ARG:HEX_OFFSET:VALUE', help='Select an actual function call by a bounded user C-string pointer field; skip at most 16 other calls') +parser.add_argument('--read-entry-indirect', action='append', default=[], metavar='ARG:HEX_OFFSET:HEX_LENGTH', + help='After function return, read a pointer field from an original x0-x3 argument') +parser.add_argument('--read-entry-user-vector', action='append', default=[], metavar='ARG:HEX_OFFSET:HEX_COUNT', help='Read a bounded user argv vector reached through an actual kernel argument field after return') +parser.add_argument('--read-entry-user-indirect', action='append', default=[], metavar='ARG:HEX_OFFSET:HEX_LENGTH', + help='After function return, read an explicitly user-space pointer field from an original argument') +parser.add_argument('--inspect-entry-translations', action='store_true', help='Walk actual 4K TTBR1 tables for captured entry pointers using QEMU physical reads') +parser.add_argument('--allow-unmapped-kernel-pages', action='store_true', help='Diagnostic only: record invalid descriptors in requested kernel spans instead of requiring every page mapped') +parser.add_argument('--inspect-indirect-kernel-range', action='append', default=[], metavar='HEX_POINTER_ADDRESS:HEX_BYTES[:HEX_OFFSET]', + help='Walk a bounded kernel extent addressed by an actual captured global pointer') +parser.add_argument('--inspect-kernel-range', action='append', default=[], metavar='HEX_VA:HEX_BYTES', + help='Walk every actual hardware page covering an explicit kernel range, at most four 1-MiB ranges') +parser.add_argument('--mutate-md0-after-return', metavar='ENTRY_PC:MD0_VA:IMAGE_BYTES:SHA256:OFFSET', + help='Offline input-fault test: XOR one SHA-identified own md0 byte after a successful function return') +parser.add_argument('--mutate-mapped-vnode-page', metavar='ENTRY_PC:FILE_PAGE:BYTE:SHA256', + help='Offline fault test: XOR one SHA-identified actual vnode page at cs_validate_hash entry') +parser.add_argument('--mapped-vnode-max-hits', type=int, default=16, help='Maximum real validation entries before a selected page fault, bounded to 8192') +parser.add_argument('--mapped-vnode-cdhash', metavar='FIELD_OFFSET:CDHASH', + help='Require this actual cs_blob CodeDirectory identity before page fault injection') +parser.add_argument('--gic-version', choices=['2', '3'], default='2') +parser.add_argument('--smp', type=int, default=1, help='QEMU CPU count (offline secondary-CPU start check)') +parser.add_argument('--boot-progress-clock-fixture', help='Own diagnostic-only clock epoch: ENTRY:EPOCH:FREQUENCY:SECONDS') +args = parser.parse_args() +progress_clock=None +if args.boot_progress_clock_fixture: + parts=args.boot_progress_clock_fixture.split(':') + if len(parts)!=4:parser.error('Invalid diagnostic clock fixture') + progress_clock=tuple(int(x,16) for x in parts[:3])+(int(parts[3]),) + if (not args.esp or not args.kernel or args.qemu_accel!='tcg' or + progress_clock[0] not in args.trace_address or progress_clock[3] not in (31,121) or + any(not 0xfffffff000000000<=v<0xfffffffffffffff0 for v in progress_clock[:3]) or + progress_clock[0]%4 or progress_clock[1]%8 or progress_clock[2]%8): + parser.error('Clock fixture requires identified XNU TCG and its exact diagnostic symbols') +if args.exercise_ufs_allocation and (not args.esp or not args.kernel or not args.system_registers or args.qemu_accel != 'tcg'): + parser.error('UFS allocation exercise requires an identified full XNU ESP boot, TCG and system registers') +cache_query_plan = None +if args.main_cache_query_plan: + if args.follow_main_entry_offset is None: + parser.error('Cache query observation requires actual main entry') + cache_query_plan = json.loads(args.main_cache_query_plan.read_text()) +process_plan = json.loads(args.process_layout_plan.read_text()) if args.process_layout_plan else cache_query_plan +if args.inspect_ui_threads and (not process_plan or not process_plan.get('thread_snapshot_layout') or args.allproc_address is None): + parser.error('UI thread snapshots require compiler-qualified layouts and the process list') +if process_plan and not args.system_registers: + parser.error('Actual process inspection requires real system-register observation') +if args.qemu_accel == 'hvf' and (platform.system() != 'Darwin' or platform.machine() != 'arm64' or not args.esp): + parser.error('HVF requires a macOS ARM64 ESP test') +if args.primary_start_address is not None and (not args.esp or args.primary_start_address in + [args.address, *args.also_stop, *args.trace_address, *args.syscall_log_address, *args.cstring_log_address, + args.syscall_start_address, args.cstring_start_address, args.registry_string_log_address]): + parser.error('Primary activation requires a distinct actual ESP kernel entry') +if args.syscall_start_address is not None and (not args.syscall_log_address or args.syscall_start_address in [args.address, *args.also_stop, *args.syscall_log_address, *args.trace_address] or args.syscall_start_address == args.cstring_start_address): + parser.error('Delayed syscall logging requires a distinct activation entrypoint and logger') +if args.cstring_start_address is not None and (not args.cstring_log_address or args.cstring_start_address in [args.address, *args.also_stop, *args.cstring_log_address, *args.trace_address]): + parser.error('Delayed C-string logging requires a distinct activation entrypoint and logger') +if not 0 <= args.skip_target_hits <= 16: + parser.error('--skip-target-hits must be between 0 and 16') +if not 0 <= args.step_followed_entry <= 64 or (args.step_followed_entry and args.follow_entrypoint_arg is None): + parser.error('--step-followed-entry requires an entrypoint argument and 0..64 instructions') +if args.follow_entrypoint_arg is not None and (args.finish_function or not args.esp): + parser.error('Following an actual entrypoint requires an ESP and conflicts with --finish-function') +if args.follow_main_entry_offset is not None and (args.follow_entrypoint_arg is None or args.follow_entrypoint_checkpoint_offset is not None or + args.step_followed_entry or not 0 < args.follow_main_entry_offset < 16 * 1024 * 1024 or args.follow_main_entry_offset & 3): + parser.error('Main observation requires a bounded aligned LC_MAIN offset and a followed entry, without another checkpoint or stepping') +if args.follow_entrypoint_checkpoint_offset is not None and args.follow_entrypoint_arg is None: + parser.error('A relative user checkpoint requires --follow-entrypoint-arg') +if args.finish_followed_checkpoint and args.follow_entrypoint_checkpoint_offset is None: + parser.error('--finish-followed-checkpoint requires a relative checkpoint') +if args.read_followed_checkpoint_arg and not args.finish_followed_checkpoint: + parser.error('--read-followed-checkpoint-arg requires --finish-followed-checkpoint') +if args.read_followed_relative and args.follow_entrypoint_arg is None: + parser.error('--read-followed-relative requires an actual followed entry') +if (args.read_entry_indirect or args.read_entry_user_indirect) and not args.finish_function: + parser.error('--read-entry-indirect requires --finish-function') +if args.read_entry_user_vector and not args.finish_function: + parser.error('User argv observation requires a real finished function call') +if args.read_entry_user_indirect and args.inspect_entry_translations: + parser.error('User pointer capture is separate from the existing TTBR1-only translation walker') +if set(args.read_entry_indirect) & set(args.read_entry_user_indirect): + parser.error('An entry pointer request must select exactly one address space') +if args.inspect_entry_translations and not args.system_registers: + parser.error('--inspect-entry-translations requires --system-registers') +kernel_ranges = [] +for request in args.inspect_kernel_range: + va, size = (int(part, 16) for part in request.split(':')) + if not args.system_registers or len(args.inspect_kernel_range) > 4 or not 0 < size <= 1024 * 1024 or not 0xfffffff000000000 <= va < va + size < 1 << 64: + parser.error('Kernel range translation inspection requires system registers and at most four bounded kernel spans') + kernel_ranges.append((request, va, size)) +indirect_kernel_ranges = [] +for request in args.inspect_indirect_kernel_range: + parts=request.split(':') + if len(parts) not in (2,3):parser.error('Indirect kernel range requires pointer, bytes and optional offset') + address,size=(int(part,16) for part in parts[:2]);offset=int(parts[2],16) if len(parts)==3 else 0 + if (not args.system_registers or len(args.inspect_kernel_range) + len(args.inspect_indirect_kernel_range) > 4 or + not 0 < size <= 1024 * 1024 or not 0<=offset<=1024*1024*1024 or not 0xfffffff000000000 <= address < address + 8 < 1 << 64): + parser.error('Indirect kernel ranges require system registers and at most four total bounded kernel spans') + indirect_kernel_ranges.append((request, address, size,offset)) +if not 1 <= args.mapped_vnode_max_hits <= 8192: + parser.error('Mapped-page validation entry limit must be 1..8192') +entry_kernel_bytes = None +entry_register_equal = None +if args.entry_kernel_bytes_field: + parts = args.entry_kernel_bytes_field.split(':') + if len(parts) != 3 or not args.finish_function or not args.esp: + parser.error('Kernel field selection requires a real finished ESP function call') + entry_kernel_bytes = (int(parts[0], 16), int(parts[1], 16), bytes.fromhex(parts[2])) + if not 0 <= entry_kernel_bytes[0] < 8 or not 0 <= entry_kernel_bytes[1] <= 4096 or not 1 <= len(entry_kernel_bytes[2]) <= 128: + parser.error('Read a bounded field from kernel x0-x7') +if args.entry_register_equals: + parts = args.entry_register_equals.split(':') + if len(parts) != 2 or not args.esp or not (args.finish_function or (args.inspect_entry_user_state and args.entry_process_name and process_plan)): + parser.error('Register selection requires a finished ESP function call or a named real user-state boundary') + entry_register_equal = (int(parts[0], 16), int(parts[1], 16)) + if not 0 <= entry_register_equal[0] < 8 or not 0 <= entry_register_equal[1] < 1 << 64: + parser.error('Select an actual 64-bit entry argument') +if args.entry_process_name and (not process_plan or not args.esp or not 1 <= len(args.entry_process_name.encode('ascii')) <= 16): + parser.error('Process selection requires exact compiled process layout, ESP and a 1..16-byte ASCII process name') +if args.entry_min_pid and (not args.entry_process_name or not 1<=args.entry_min_pid<1<<31):parser.error('Minimum PID requires a named process and a positive bounded PID') +if not 1 <= args.entry_max_skips <= 65536 or (args.entry_max_skips > 16 and not (entry_kernel_bytes or args.entry_process_name)): + parser.error('Large bounded entry selection requires an explicit kernel-byte identity') +saved_user_requests=[] +for request in args.read_saved_user_register: + index,length=(int(v,16) for v in request.split(':')) + if not args.inspect_entry_user_state or len(args.read_saved_user_register)>4 or not 0<=index<=30 or not 0 deadline: + raise + time.sleep(0.01) + + with connection: + from gdb_packets import Packets + packet = Packets(connection).request + + def feature(name): + offset = 0 + text = '' + while True: + chunk = packet(f'qXfer:features:read:{name}:{offset:x},1000') + if not chunk or chunk[0] not in 'ml': + raise RuntimeError(f'QEMU did not provide {name}') + text += chunk[1:] + offset += len(chunk) - 1 + if chunk[0] == 'l': + # QEMU's target.xml uses xi:include without declaring xi. + if 'xi:include' in text and 'xmlns:xi=' not in text: + text = text.replace(' args.mapped_vnode_max_hits: + raise RuntimeError('Selected vnode page was not found within the bounded validation entries') + matched_page = False + if current[1] and current[2] == mapped_mutation[1]: + raw = packet(f'm{current[3]:x},1000') + if len(raw) != 8192 or raw.startswith('E'): + raise RuntimeError('Cannot inspect the actual validation input page') + matched_page = hashlib.sha256(bytes.fromhex(raw)).hexdigest() == mapped_mutation[3] + if matched_page and mapped_cdhash: + observed_cdhash = packet(f'm{current[0] + mapped_cdhash[0]:x},14') + if observed_cdhash.startswith('E'): + raise RuntimeError('Cannot inspect actual cs_blob CodeDirectory identity') + matched_page = observed_cdhash == mapped_cdhash[1] + if matched_page: + original = bytes.fromhex(raw)[mapped_mutation[2]] + target = current[3] + mapped_mutation[2] + if packet(f'M{target:x},1:{original ^ 1:02x}') != 'OK' or packet(f'm{target:x},1') != f'{original ^ 1:02x}': + raise RuntimeError('Own mapped-page byte mutation failed') + mapped_mutation_event = {'entry': hex(current[32]), 'pager': hex(current[1]), + 'file_page': current[2], 'data_kva': hex(current[3]), 'byte_in_page': mapped_mutation[2], + 'page_sha256_before': mapped_mutation[3], 'before': original, 'after': original ^ 1, + 'registers_or_validation_state_modified': False} + if mapped_cdhash: + mapped_mutation_event['cs_blob'] = hex(current[0]) + mapped_mutation_event['code_directory_hash'] = observed_cdhash + mapped_mutation_event['compiled_cdhash_field_offset'] = mapped_cdhash[0] + else: + if packet(f'z1,{current[32]:x},4') != 'OK': + raise RuntimeError('Cannot step over a non-target page validation') + packet('s') + if packet(f'Z1,{current[32]:x},4') != 'OK': + raise RuntimeError('Cannot restore mapped-page validation tracepoint') + return True + trace_events.append({'pc': hex(current[32]), 'x0': hex(current[0]), + 'x1': hex(current[1]), 'lr': hex(current[30]), + 'elapsed_seconds': round(time.monotonic() - started, 3)}) + if packet(f'z1,{current[32]:x},4') != 'OK': + raise RuntimeError('Cannot retire the applied mapped-page fault breakpoint') + trace_remaining.discard(current[32]) + return True + + def continue_observing(): + global cstring_start_event, syscall_start_event, primary_start_event + while True: + connection.settimeout(max(0.001, deadline - time.monotonic())) + timed_out = False + try: + stopped = packet('c') + except TimeoutError: + stopped = packet(None) + timed_out = True + current = struct.unpack_from('<33Q', bytes.fromhex(packet('g'))) + if not timed_out and live_hid and current[32]==live_hid.address: + connection.settimeout(3) + if live_hid.checkpoint(packet,time.monotonic()-started):return stopped,current,False + if packet(f'z1,{current[32]:x},4')!='OK':raise RuntimeError('Cannot step own HID timer checkpoint') + packet('s') + if struct.unpack_from('<33Q',bytes.fromhex(packet('g')))[32]==current[32]: + raise RuntimeError('Own HID checkpoint instruction did not advance') + if packet(f'Z1,{current[32]:x},4')!='OK':raise RuntimeError('Cannot restore own HID timer checkpoint') + continue + if not timed_out and args.primary_start_address is not None and current[32] == args.primary_start_address: + primary_start_event = dict(pc=hex(current[32]), lr=hex(current[30]), + elapsed_seconds=round(time.monotonic() - started, 3)) + if packet(f'z1,{current[32]:x},4') != 'OK' or packet(f'Z1,{args.address:x},4') != 'OK': + raise RuntimeError('Cannot activate the actual primary entry observation') + continue + if not timed_out and observe_mapped_fault(current): + continue + if not timed_out and args.syscall_start_address is not None and current[32] == args.syscall_start_address: + syscall_start_event = {'pc': hex(current[32]), 'lr': hex(current[30]), + 'elapsed_seconds': round(time.monotonic() - started, 3)} + if packet(f'z1,{current[32]:x},4') != 'OK': + raise RuntimeError('Cannot retire syscall activation breakpoint') + for address in set(args.syscall_log_address): + if packet(f'Z1,{address:x},4') != 'OK': + raise RuntimeError('Cannot arm delayed syscall observation') + continue + if not timed_out and args.cstring_start_address is not None and current[32] == args.cstring_start_address: + cstring_start_event = {'pc': hex(current[32]), 'lr': hex(current[30]), + 'elapsed_seconds': round(time.monotonic() - started, 3)} + if packet(f'z1,{current[32]:x},4') != 'OK': + raise RuntimeError('Cannot retire kernel-string activation breakpoint') + for address in set(args.cstring_log_address): + if packet(f'Z1,{address:x},4') != 'OK': + raise RuntimeError('Cannot arm delayed kernel-string observation') + continue + if not timed_out and args.registry_string_log_address and current[32] == args.registry_string_log_address: + event = {'entry': hex(current[0]), 'caller': hex(current[30]), + 'elapsed_seconds': round(time.monotonic() - started, 3)} + for index, name in ((1, 'key'), (2, 'value')): + if current[index] < 0xfffffff000000000: + raise RuntimeError('Expected actual kernel registry string arguments') + raw = packet(f'm{current[index]:x},{min(256, 4096 - (current[index] & 4095)):x}') + if raw.startswith('E'): + raise RuntimeError('Cannot read actual registry property argument') + event[name] = bytes.fromhex(raw).split(b'\0', 1)[0].decode(errors='replace') + registry_string_events.append(event) + if packet(f'z1,{current[32]:x},4') != 'OK': + raise RuntimeError('Cannot remove registry string observation breakpoint') + packet('s') + if len(registry_string_events) < 128 and packet(f'Z1,{current[32]:x},4') != 'OK': + raise RuntimeError('Cannot restore registry string observation breakpoint') + continue + if not timed_out and current[32] in args.cstring_log_address: + if current[0] < 0xfffffff000000000: + raise RuntimeError('Expected a kernel C-string call argument') + raw = packet(f'm{current[0]:x},100') + if raw.startswith('E'): + raise RuntimeError('Cannot read the observed kernel string') + cstring_events.append({'handler': hex(current[32]), 'caller': hex(current[30]), + 'argument': hex(current[0]), 'stack_bytes': packet(f'm{current[31]:x},80'), 'value': bytes.fromhex(raw).split(b'\0', 1)[0].decode(errors='replace'), + 'elapsed_seconds': round(time.monotonic() - started, 3)}) + if packet(f'z1,{current[32]:x},4') != 'OK': + raise RuntimeError('Cannot remove kernel string observation breakpoint') + packet('s') + if len(cstring_events) < 128 and packet(f'Z1,{current[32]:x},4') != 'OK': + raise RuntimeError('Cannot restore kernel string observation breakpoint') + if len(cstring_events) == 128: + for address in set(args.cstring_log_address) - {current[32]}: + if packet(f'z1,{address:x},4') != 'OK': + raise RuntimeError('Cannot retire exhausted string observation breakpoint') + continue + if timed_out or current[32] not in args.syscall_log_address: + return stopped, current, timed_out + raw = packet(f'm{current[0]:x},128') + if len(raw) != 592 or raw.startswith('E'): + raise RuntimeError('Cannot read actual ARM64 syscall saved state') + state = bytes.fromhex(raw) + if struct.unpack_from('<2I', state) != (21, 72): + raise RuntimeError('Unexpected saved-state ABI for pinned Darwin 23 ARM64') + regs = struct.unpack_from('<33Q', state, 8) + event = {'number': struct.unpack_from(' 0: + event['write_bytes'] = packet(f'm{regs[1]:x},{min(256, regs[2]):x}') + event['write_truncated'] = regs[2] > 256 + if event['number'] in (5, 10, 21, 57, 128, 136, 242, 345, 398) and 0 < regs[0] < 0xfffffff000000000: + count = min(256, 4096 - (regs[0] & 4095)) + raw = packet(f'm{regs[0]:x},{count:x}') + event['path_bytes'] = raw + if not raw.startswith('E'): + event['path'] = bytes.fromhex(raw).split(b'\0', 1)[0].decode(errors='replace') + if event['number'] in (57, 128) and 0 < regs[1] < 0xfffffff000000000: + count = min(256, 4096 - (regs[1] & 4095)) + raw = packet(f'm{regs[1]:x},{count:x}') + event['second_path_bytes'] = raw + if not raw.startswith('E'): + event['second_path'] = bytes.fromhex(raw).split(b'\0', 1)[0].decode(errors='replace') + if not args.syscall_number or event['number'] in args.syscall_number: + syscall_events.append(event) + if packet(f'z1,{current[32]:x},4') != 'OK': + raise RuntimeError('Cannot remove syscall observation breakpoint') + packet('s') + if len(syscall_events) < 512 and packet(f'Z1,{current[32]:x},4') != 'OK': + raise RuntimeError('Cannot restore syscall observation breakpoint') + if len(syscall_events) == 512: + for address in set(args.syscall_log_address) - {current[32]}: + if packet(f'z1,{address:x},4') != 'OK': + raise RuntimeError('Cannot retire exhausted syscall observation breakpoint') + + while True: + stop, values, interrupted = continue_observing() + connection.settimeout(3) + entry_string_match = True + if not interrupted and values[32] == args.address and entry_string is not None: + index, offset, expected = entry_string + raw = packet(f'm{values[index] + offset:x},8') + if len(raw) != 16 or raw.startswith('E'): + raise RuntimeError('Cannot read actual entry string pointer field') + pointer = int.from_bytes(bytes.fromhex(raw), 'little') + if not 0 < pointer < 0xfffffff000000000 - len(expected) - 1: + raise RuntimeError('Expected a bounded non-null user C-string pointer') + raw = packet(f'm{pointer:x},{len(expected) + 1:x}') + if len(raw) != 2 * (len(expected) + 1) or raw.startswith('E'): + raise RuntimeError('Cannot read actual user entry string') + entry_string_match = bytes.fromhex(raw) == expected + b'\0' + entry_string_events.append({'pc': hex(values[32]), 'pointer': hex(pointer), + 'value': bytes.fromhex(raw).split(b'\0')[0].decode(errors='replace'), + 'matched': entry_string_match, 'elapsed_seconds': round(time.monotonic() - started, 3)}) + process_match = True + if not interrupted and values[32] == args.address and args.entry_process_name: + identity=process_identity();process_match=identity['name']==args.entry_process_name and identity['pid']>=args.entry_min_pid + entry_process_events.append(dict(pc=hex(values[32]),process=identity,matched=process_match, + elapsed_seconds=round(time.monotonic()-started,3))) + kernel_match = register_match = True + if not interrupted and values[32] == args.address and entry_kernel_bytes: + index, offset, expected = entry_kernel_bytes + pointer = values[index] + offset + kernel_match = 0xfffffff000000000 <= pointer <= (1 << 64) - len(expected) + raw = packet(f'm{pointer:x},{len(expected):x}') if kernel_match else '' + kernel_match = raw == expected.hex() + if not interrupted and values[32] == args.address and entry_register_equal: + index, expected_value = entry_register_equal + register_match = values[index] == expected_value + if not interrupted and values[32] == args.address and entry_kernel_bytes and kernel_match and register_match: + entry_kernel_bytes_match = dict(pc=hex(values[32]), address=hex(pointer), bytes=raw, + elapsed_seconds=round(time.monotonic() - started, 3)) + if not interrupted and values[32] == args.address and (skipped_target_hits < args.skip_target_hits or not entry_string_match or not kernel_match or not register_match or not process_match): + skipped_target_hits += 1 + if skipped_target_hits > args.entry_max_skips: + raise RuntimeError('Entry predicate did not match within the bounded skipped calls') + if packet(f'z1,{args.address:x},4') != 'OK': + raise RuntimeError('Cannot step over an earlier primary breakpoint call') + packet('s') + if packet(f'Z1,{args.address:x},4') != 'OK': + raise RuntimeError('Cannot restore primary breakpoint') + continue + if not interrupted and mutation_return is not None and values[32] == mutation_return: + if values[0] != 0 or packet(f'z1,{mutation_return:x},4') != 'OK': + raise RuntimeError('Own-input mutation requires successful authentication return') + raw = packet(f'm{mutation[1]:x},10') + if len(raw) != 32 or raw.startswith('E'): + raise RuntimeError('Cannot inspect actual md0 state') + pages, count, flags = struct.unpack(' count * 4096: + raise RuntimeError(f'Expected initialized md0 containing the own input: pages={pages:#x}, size={count:#x}, flags={flags:#x}') + base = pages * 4096 + old_mode = packet('qqemu.PhyMemMode') + input_mode = '1' if flags & 4 else '0' + if old_mode not in ('0', '1') or packet('Qqemu.PhyMemMode:' + input_mode) != 'OK': + raise RuntimeError('Own-input memory inspection unavailable') + try: + digest = hashlib.sha256() + original = None + for offset in range(0, mutation[2], 4096): + size = min(4096, mutation[2] - offset) + raw = packet(f'm{base + offset:x},{size:x}') + if len(raw) != size * 2 or raw.startswith('E'): + raise RuntimeError('Cannot read the complete own md0 image') + chunk = bytes.fromhex(raw) + digest.update(chunk) + if offset <= mutation[4] < offset + size: + original = chunk[mutation[4] - offset] + if digest.hexdigest() != mutation[3] or original is None: + raise RuntimeError('Actual md0 bytes do not match the explicit own-input SHA-256') + target = base + mutation[4] + if packet(f'M{target:x},1:{original ^ 1:02x}') != 'OK' or packet(f'm{target:x},1') != f'{original ^ 1:02x}': + raise RuntimeError('Own-input byte mutation failed') + mutation_event = {'trigger': hex(mutation[0]), 'return_pc': hex(mutation_return), + 'md0_va': hex(mutation[1]), 'input_address': hex(base), 'input_bytes': mutation[2], + 'address_space': 'guest_physical' if input_mode == '1' else 'guest_virtual', + 'input_sha256_before': digest.hexdigest(), 'byte_offset': mutation[4], + 'before': original, 'after': original ^ 1, 'md0_flags': flags, + 'registers_or_validation_state_modified': False} + finally: + if packet('Qqemu.PhyMemMode:' + old_mode) != 'OK': + raise RuntimeError('Cannot restore debugger virtual-memory mode') + mutation_return = None + continue + if not interrupted and args.kext_log_address and values[32] == args.kext_log_address: + kext_log_hits += 1 + if values[1] & 7 in (1, 2): + raw_format = packet(f'm{values[2]:x},200') + raw_stack = packet(f'm{values[31]:x},40') + event = {'pc': hex(values[32]), 'level': values[1] & 7, + 'format_bytes': raw_format, 'stack_bytes': raw_stack} + if not raw_format.startswith('E'): + event['format'] = bytes.fromhex(raw_format).split(b'\0', 1)[0].decode(errors='replace') + event['argument_strings'] = {} + if not raw_stack.startswith('E'): + for index, pointer in enumerate(struct.unpack_from('<4Q', bytes.fromhex(raw_stack))): + if pointer >= 0xfffffff000000000: + raw = packet(f'm{pointer:x},100') + if not raw.startswith('E'): + event['argument_strings'][str(index)] = bytes.fromhex(raw).split(b'\0', 1)[0].decode(errors='replace') + kext_log_events.append(event) + if packet(f'z1,{values[32]:x},4') != 'OK': + raise RuntimeError('Cannot remove repeated logpoint') + if kext_log_hits < 256: + packet('s') + if packet(f'Z1,{values[32]:x},4') != 'OK': + raise RuntimeError('Cannot restore repeated logpoint') + continue + if interrupted or values[32] not in trace_remaining: + break + trace_events.append({'pc': hex(values[32]), 'x0': hex(values[0]), + 'x1': hex(values[1]), 'lr': hex(values[30]), + 'elapsed_seconds': round(time.monotonic() - started, 3)}) + if progress_clock and values[32]==progress_clock[0]: + epoch_raw=packet(f'm{progress_clock[1]:x},8') + freq_raw=packet(f'm{progress_clock[2]:x},8') + if len(epoch_raw)!=16 or len(freq_raw)!=16 or epoch_raw.startswith('E') or freq_raw.startswith('E'): + raise RuntimeError('Cannot read initialized diagnostic clock') + before=int.from_bytes(bytes.fromhex(epoch_raw),'little') + frequency=int.from_bytes(bytes.fromhex(freq_raw),'little') + if not 1000<=frequency<=10**10:raise RuntimeError('Invalid actual counter frequency') + after=(before-progress_clock[3]*frequency)&((1<<64)-1) + raw=after.to_bytes(8,'little').hex() + if packet(f'M{progress_clock[1]:x},8:{raw}')!='OK' or packet(f'm{progress_clock[1]:x},8')!=raw: + raise RuntimeError('Cannot qualify diagnostic-only epoch write/readback') + progress_clock_event=dict(advance_seconds=progress_clock[3],frequency=frequency, + epoch_before=before,epoch_after=after,entry=hex(values[32]), + changed_only_own_diagnostic_epoch=True,hardware_or_os_clock_modified=False) + if mutation is not None and values[32] == mutation[0]: + mutation_return = values[30] + if mutation_return in {args.address, *args.also_stop, *trace_remaining}: + raise RuntimeError('Mutation return must be distinct from existing breakpoints') + if packet(f'Z1,{mutation_return:x},4') != 'OK': + raise RuntimeError('Cannot observe authentication return') + if packet(f'z1,{values[32]:x},4') != 'OK': + raise RuntimeError('Cannot remove one-shot tracepoint') + trace_remaining.remove(values[32]) + connection.settimeout(3) + followed_entry = None + followed_main = None + followed_display = None + if args.follow_entrypoint_arg is not None and values[32] == args.address: + target = values[args.follow_entrypoint_arg] + if not 0 < target < 0xfffffff000000000 or target & 3: + raise RuntimeError('Expected an actual aligned user entrypoint') + followed_entry = {'kernel_entry': hex(values[32]), 'argument': args.follow_entrypoint_arg, + 'user_entry': hex(target), 'observed': False, 'steps': []} + if packet(f'z1,{args.address:x},4') != 'OK' or packet(f'Z1,{target:x},4') != 'OK': + raise RuntimeError('Cannot observe actual dynamic user entrypoint') + stop, values, interrupted = continue_observing() + connection.settimeout(3) + if not interrupted and values[32] == target: + followed_entry['observed'] = True + followed_entry['pstate'] = hex(struct.unpack_from(' activation_limit: + followed_display['activation_limit_reached'] = True + for address in activations: + if packet(f'z1,{address:x},4') != 'OK': + raise RuntimeError('Cannot retire bounded activation observation') + break + if identity['name'] != plan['expected_process']: + address = values[32] + if packet(f'z1,{address:x},4') != 'OK': + raise RuntimeError('Cannot retire an unrelated activation entry') + packet('s') + if packet(f'Z1,{address:x},4') != 'OK': + raise RuntimeError('Cannot restore process activation observation') + continue + activation_return, activation_stack = values[30], values[31] + for address in activations: + if packet(f'z1,{address:x},4') != 'OK': + raise RuntimeError('Cannot retire actual shared-region activation entries') + if packet(f'Z1,{activation_return:x},4') != 'OK': + raise RuntimeError('Cannot observe actual shared-region syscall return') + stop, values, interrupted = continue_observing() + connection.settimeout(3) + if interrupted or values[32] != activation_return or values[31] != activation_stack or process_identity()['thread'] != identity['thread']: + raise RuntimeError('Shared-region startup must return on the selected actual thread/stack') + if packet(f'z1,{activation_return:x},4') != 'OK': + raise RuntimeError('Cannot retire shared-region syscall return observation') + event['result'] = values[0] + identity = process_identity() + event['return_process'] = identity + if values[0] == 0 and identity['shared_region_slide'] is not None: + slide = identity['shared_region_slide'] + if not 0 <= slide <= plan['max_slide'] or slide & 4095: + raise RuntimeError('Selected process must have an actual native-page-aligned cache slide') + query = plan['query'] + slide + followed_display.update(cache_slide=hex(slide), query=hex(query), activated_process=identity) + break + for address in activations: + if packet(f'Z1,{address:x},4') != 'OK': + raise RuntimeError('Cannot observe the next real shared-region operation') + activated = plan['mode'] == 'discovery' or 'activated_process' in followed_display + if activated and plan['mode'] == 'user-trace': + sites = plan['trace_sites'] + if not 1 <= len(sites) <= 4: + raise RuntimeError('Original trace requires one to four bounded sites') + pending = {} + hits = {} + for site in sites: + address = site['address'] + slide + prefix = site['prefix'] + if address in pending or address & 3 or not 0 < address < 0xfffffff000000000 - 16 or len(bytes.fromhex(prefix)) != 16: + raise RuntimeError('Invalid or duplicate original trace site') + pending[address] = site + hits[address] = 0 + if not 1 <= site.get('max_hits', 1) <= 8: + raise RuntimeError('Original trace hit budget must be one to eight') + if packet(f'Z1,{address:x},4') != 'OK': + raise RuntimeError('Cannot install original trace site') + events, skipped = [], [] + followed_display['trace_events'] = events + followed_display['trace_skipped_processes'] = skipped + while pending: + stop, values, interrupted = continue_observing() + connection.settimeout(3) + if interrupted: + break + address = values[32] + if address not in pending: + raise RuntimeError('Unexpected stop during original user trace') + identity = process_identity() + site = pending[address] + prefix = packet(f'm{address:x},10') + pstate = struct.unpack_from(' 8: + raise RuntimeError('Original trace memory reads exceed bound') + event['memory'] = [] + for request in reads: + reg, offset, size = request['register'], request['offset'], request['size'] + if not 0 <= reg <= 30 or not 0 <= offset <= 65536 or not 1 <= size <= 256: + raise RuntimeError('Invalid bounded original trace memory read') + target = values[reg] + offset + read = dict(request, address=hex(target)) + if request.get('indirect'): + raw = packet(f'm{target:x},8') if 0 < target < 0xfffffff000000000 - 8 else '' + read['pointer_bytes'] = raw + target = int.from_bytes(bytes.fromhex(raw), 'little') if len(raw) == 16 and not raw.startswith('E') else 0 + read['target'] = hex(target) + read['bytes'] = packet(f'm{target:x},{size:x}') if 0 < target < 0xfffffff000000000 - size else None + event['memory'].append(read) + events.append(event) + hits[address] += 1 + if hits[address] == site.get('max_hits', 1): + del pending[address] + else: + packet('s') + if packet(f'Z1,{address:x},4') != 'OK': + raise RuntimeError('Cannot restore bounded original trace site') + else: + skipped.append(identity) + if len(skipped) > 64: + raise RuntimeError('Original trace exceeded process selection bound') + packet('s') + if packet(f'Z1,{address:x},4') != 'OK': + raise RuntimeError('Cannot restore process-selected trace site') + followed_display['trace_pending'] = [site['name'] for site in pending.values()] + followed_display['observed'] = not pending + for address in pending: + if packet(f'z1,{address:x},4') != 'OK': + raise RuntimeError('Cannot retire pending original trace site') + activated = False + if activated and packet(f'Z1,{query:x},4') != 'OK': + raise RuntimeError('Cannot observe the original display query') + if activated: + stop, values, interrupted = continue_observing() + connection.settimeout(3) + skipped_processes = [] + while activated and not interrupted and values[32] == query: + identity = process_identity() + if identity['name'] == plan['expected_process']: + followed_display['entry_process'] = identity + if plan['mode'] == 'discovery': + if identity['shared_region_slide'] != slide: + raise RuntimeError('Actual discovery task cache slide differs from the bound observation address') + followed_display['activated_process'] = identity + break + skipped_processes.append(identity) + if len(skipped_processes) > 64 or packet(f'z1,{query:x},4') != 'OK': + raise RuntimeError('Display observation exceeded bounded process selection') + packet('s') + if packet(f'Z1,{query:x},4') != 'OK': + raise RuntimeError('Cannot restore the process-selected display observation') + stop, values, interrupted = continue_observing() + connection.settimeout(3) + followed_display['skipped_processes'] = skipped_processes + if activated and not interrupted and values[32] == query: + prefix = packet(f'm{query:x},10') + pstate = struct.unpack_from('64 or packet(f'z1,{final_target:x},4')!='OK':raise RuntimeError('Bounded user-return frame selection failed') + packet('s') + if packet(f'Z1,{final_target:x},4')!='OK':raise RuntimeError('Cannot restore original user-return breakpoint') + stop,values,interrupted=continue_observing();connection.settimeout(3) + if not interrupted and values[32]==final_target and values[31]==entry_stack: + if packet(f'm{final_target:x},4')!='c0035fd6' or values[30]!=caller or packet(f'z1,{final_target:x},4')!='OK': + raise RuntimeError('Require unchanged original RET and restored caller/stack') + returned=values[0] + object_bytes=packet(f'm{returned:x},{plan["object_bytes"]:x}') if 0 64 or packet(f'z1,{final_target:x},4') != 'OK': + raise RuntimeError('Display return exceeded the bounded unrelated frames') + packet('s') + if packet(f'Z1,{final_target:x},4') != 'OK': + raise RuntimeError('Cannot restore original query return observation') + stop, values, interrupted = continue_observing() + connection.settimeout(3) + if not interrupted and values[32] == final_target and values[31] == entry_stack: + if packet(f'm{final_target:x},4') != 'c0035fd6' or packet(f'z1,{final_target:x},4') != 'OK': + raise RuntimeError('Require the unchanged actual original RET instruction') + followed_display.update(epilogue_pc=hex(values[32]), epilogue_lr=hex(values[30]), + caller_restored=values[30] == caller) + if plan['mode'] == 'query': + stop = packet('s') + values = struct.unpack_from('<33Q', bytes.fromhex(packet('g'))) + if values[32] != caller or values[31] != entry_stack: + raise RuntimeError('Original user RET did not restore its actual caller/stack') + followed_display['return_pc'] = hex(values[32]) + followed_display.update(observed=True, completion_pc=hex(values[32]), + return_stack=hex(values[31]), other_stacks_skipped=unrelated, + query_bytes_after_fault=packet(f'm{query:x},10')) + if plan['mode'] == 'query': + followed_display.update(main_display_bytes=packet(f'm{plan["main_display"] + slide:x},8'), + displays_bytes=packet(f'm{plan["displays"] + slide:x},8')) + if plan['mode'] == 'discovery': + followed_display['discovery_counts'] = {name: packet(f'm{plan[name] + slide:x},4') + for name in ('reference_count', 'enumerated_count', 'display_count')} + if args.follow_entrypoint_checkpoint_offset is not None: + checkpoint = target + args.follow_entrypoint_checkpoint_offset + if not 0 < checkpoint < 0xfffffff000000000 or checkpoint & 3: + raise RuntimeError('Expected an aligned relative user checkpoint') + if packet(f'Z1,{checkpoint:x},4') != 'OK': + raise RuntimeError('Cannot observe relative user checkpoint') + stop, values, interrupted = continue_observing() + connection.settimeout(3) + checkpoint_ready = not interrupted and values[32] == checkpoint + followed_entry['checkpoint'] = {'target': hex(checkpoint), 'observed': checkpoint_ready, + 'pc': hex(values[32]), + 'pstate': hex(struct.unpack_from(' 64: + raise RuntimeError('Too many unrelated thread returns at the shared caller') + if packet(f'z1,{return_target:x},4') != 'OK': + raise RuntimeError('Cannot step over another thread at the shared caller') + packet('s') + if packet(f'Z1,{return_target:x},4') != 'OK': + raise RuntimeError('Cannot restore the original thread return observation') + stop, values, interrupted = continue_observing() + connection.settimeout(3) + report = {f'x{i}': hex(value) for i, value in enumerate(values[:31])} + if process_plan: + report['process_layout_audit']=dict(native_sha256=process_plan.get('native_sha256'),layout=process_plan['process_layout']) + report.update(sp=hex(values[31]), pc=hex(values[32]), stop=stop, + pstate=hex(struct.unpack_from(' 16 * 1024 * 1024 or not 0 < length <= 4096 or not 0 < address < 0xfffffff000000000: + raise ValueError('Read bounded data from the observed user image') + report['followed_relative_memory'][request] = {'address': hex(address), + 'bytes': packet(f'm{address:x},{length:x}')} + if mapped_mutation is not None: + report['own_mapped_page_mutation'] = mapped_mutation_event + report['mapped_page_validation_entries'] = mapped_mutation_hits + report['mapped_page_validation_entry_limit'] = args.mapped_vnode_max_hits + if progress_clock: + report['boot_progress_clock_fixture']=progress_clock_event + if finished_entry is not None: + report['finished_function_entry'] = hex(finished_entry) + report['function_return_target'] = hex(return_target) + report['function_entry_arguments'] = [hex(value) for value in entry_arguments] + report['function_entry_stack'] = hex(return_stack) + report['function_return_stack_matched'] = values[31] == return_stack and values[32] == return_target and not interrupted + report['function_return_other_stacks_skipped'] = other_stack_returns + report['entry_regions'] = {} + region_arguments = entry_arguments + if region_arguments is None and values[32] == args.address and not interrupted: + region_arguments = values[:8] + report['observed_function_entry_arguments'] = [hex(v) for v in region_arguments] + if region_arguments is not None: + for request in args.read_entry_region: + index, offset, length = (int(part, 16) for part in request.split(':')) + if not 0 <= index < 8 or not 0 <= offset <= 4096 or not 0 < length <= 4096: + raise ValueError('Read bounded direct entry regions from kernel x0-x7') + pointer = region_arguments[index] + offset + if not 0xfffffff000000000 <= pointer <= (1 << 64) - length: + raise ValueError('Entry region must be actual bounded kernel memory') + raw = packet(f'm{pointer:x},{length:x}') + if len(raw) != 2 * length or raw.startswith('E'): + raise RuntimeError('Cannot read actual entry argument region') + report['entry_regions'][request] = dict(address=hex(pointer), bytes=raw) + report['entry_indirect_memory'] = {} + if entry_arguments is not None: + for request in args.read_entry_indirect + args.read_entry_user_indirect: + index, offset, length = (int(part, 16) for part in request.split(':')) + if not 0 <= index < 4 or not 0 <= offset <= 4096 or not 0 < length <= 4096: + raise ValueError('Read bounded pointer fields from entry arguments x0-x3') + raw = packet(f'm{entry_arguments[index] + offset:x},8') + if len(raw) != 16 or raw.startswith('E'): + raise RuntimeError('Cannot read original function argument pointer') + pointer = int.from_bytes(bytes.fromhex(raw), 'little') + user_pointer = request in args.read_entry_user_indirect + if user_pointer and not 0 < pointer < 0xfffffff000000000 - length: + raise RuntimeError('Expected a non-null bounded user virtual pointer') + if not user_pointer and pointer < 0xfffffff000000000: + raise RuntimeError('Expected a kernel virtual pointer') + report['entry_indirect_memory'][request] = {'pointer': hex(pointer), + 'bytes': packet(f'm{pointer:x},{length:x}')} + if user_pointer: + report['entry_indirect_memory'][request]['address_space'] = 'user' + report['entry_user_vectors'] = {} + if finished_entry is not None: + for request in args.read_entry_user_vector: + index, offset, count = (int(part, 16) for part in request.split(':')) + if not 0 <= index < 4 or not 0 <= offset <= 4096 or not 1 <= count <= 16 or entry_arguments[index] < 0xfffffff000000000: + raise ValueError('Read a bounded user vector field from actual kernel x0-x3') + raw = packet(f'm{entry_arguments[index] + offset:x},8') + if len(raw) != 16 or raw.startswith('E'): + raise RuntimeError('Cannot read actual user argv pointer field') + base = int.from_bytes(bytes.fromhex(raw), 'little') + if not 0 < base < 0xfffffff000000000 - count * 8: + raise RuntimeError('Expected a bounded user argv vector') + capture = {'pointer': hex(base), 'values': [], 'terminated': False} + for slot in range(count): + raw = packet(f'm{base + slot * 8:x},8') + if len(raw) != 16 or raw.startswith('E'): + raise RuntimeError('Cannot read actual user argv element') + pointer = int.from_bytes(bytes.fromhex(raw), 'little') + if not pointer: + capture['terminated'] = True + break + if pointer >= 0xfffffff000000000: + raise RuntimeError('Expected a user argv string') + raw = packet(f'm{pointer:x},{min(256, 4096 - (pointer & 4095)):x}') + if raw.startswith('E'): + raise RuntimeError('Cannot read actual user argv string') + value = bytes.fromhex(raw) + capture['values'].append({'pointer': hex(pointer), 'value': value.split(b'\0')[0].decode(errors='replace'), + 'truncated': b'\0' not in value}) + report['entry_user_vectors'][request] = capture + if args.inspect_entry_user_state and values[32] == args.address and not interrupted: + report['entry_user_state_candidates'] = [] + state_pointers=[(f'x{index}',values[index]) for index in range(19,29)] + if process_plan: + current=process_identity() + if current.get('user_state'):state_pointers.append(('current-thread.upcb',int(current['user_state'],16))) + seen_state_pointers=set() + for source,pointer in state_pointers: + if pointer in seen_state_pointers:continue + seen_state_pointers.add(pointer) + if not 0xfffffff000000000 <= pointer < (1 << 64) - 304 or pointer % 8: + continue + raw = packet(f'm{pointer:x},130') + if raw.startswith('E') or len(raw) != 608: + continue + saved = bytes.fromhex(raw) + if struct.unpack_from('<2I', saved) != (21, 72): + continue + regs = struct.unpack_from('<33Q', saved, 8) + cpsr = struct.unpack_from(' 0x10000 or frame % 16: + break + frame_raw = packet(f'm{frame:x},10') + if frame_raw.startswith('E') or len(frame_raw) != 32: + break + parent, caller = struct.unpack('<2Q', bytes.fromhex(frame_raw)) + candidate['frames'].append(dict(fp=hex(frame), lr=hex(caller))) + if parent <= frame: + break + frame = parent + report['entry_user_state_candidates'].append(candidate) + report['stack_bytes'] = packet(f'm{values[31]:x},100') + report['frame_chain'] = [] + frame = values[29] + for _ in range(16): + if frame < values[31] or frame - values[31] > 0x10000 or frame % 16: + break + raw = packet(f'm{frame:x},10') + if len(raw) != 32 or raw.startswith('E'): + break + parent, caller = struct.unpack('<2Q', bytes.fromhex(raw)) + report['frame_chain'].append({'fp': hex(frame), 'lr': hex(caller)}) + if args.read_frame_locals and frame >= 0xfffffff000000000: + start = max(values[31], frame - 192) + report['frame_chain'][-1]['local_address'] = hex(start) + report['frame_chain'][-1]['local_bytes'] = packet(f'm{start:x},{frame + 16 - start:x}') + if parent <= frame: + break + frame = parent + report['frame_indirect_memory'] = {} + for request in args.read_frame_indirect: + index, offset, length = (int(part, 16) for part in request.split(':')) + if not 0 <= index < 16 or not -192 <= offset <= 8 or offset % 8 or not 0 < length <= 4096: + raise ValueError('Read a bounded aligned kernel frame slot and at most 4096 pointed bytes') + if index >= len(report['frame_chain']): + continue + frame = int(report['frame_chain'][index]['fp'], 16) + if frame < 0xfffffff000000000: + continue + raw = packet(f'm{frame + offset:x},8') + if raw.startswith('E') or len(raw) != 16: + raise RuntimeError('Cannot read actual kernel frame pointer slot') + pointer = int.from_bytes(bytes.fromhex(raw), 'little') + if not 0xfffffff000000000 <= pointer <= (1 << 64) - length: + raise RuntimeError('Actual frame slot is not a bounded kernel pointer') + report['frame_indirect_memory'][request] = dict(slot_address=hex(frame + offset), + pointer=hex(pointer), bytes=packet(f'm{pointer:x},{length:x}')) + report['stack_pointer_memory'] = {} + stack_raw = bytes.fromhex(report['stack_bytes']) if not report['stack_bytes'].startswith('E') else b'' + stack_pointers = struct.unpack_from('<5Q', stack_raw) if len(stack_raw) >= 40 else () + for index, pointer in enumerate(stack_pointers): + if pointer >= 0xfffffff000000000: + raw = packet(f'm{pointer:x},130') + report['stack_pointer_memory'][str(index)] = {'address': hex(pointer), 'bytes': raw} + saved = report['stack_pointer_memory'].get('3', {}).get('bytes', '') + if len(saved) == 608 and not saved.startswith('E'): + raw_state = bytes.fromhex(saved) + if struct.unpack_from('<2I', raw_state) == (21, 72): + report['saved_exception_registers'] = {} + report['saved_exception_argument_memory'] = {} + for index in range(4): + value, = struct.unpack_from('= 0xfffffff000000000: + raw = packet(f'm{value:x},100') + if not raw.startswith('E'): + report['saved_exception_argument_memory'][f'x{index}'] = bytes.fromhex(raw).split(b'\0', 1)[0].decode(errors='replace') + report['memory'] = {} + for request in args.read_memory: + address, length = (int(part, 16) for part in request.split(':')) + if not 0 < length <= 4096: + raise ValueError('Inspect at most 4096 bytes per request') + report['memory'][request] = packet(f'm{address:x},{length:x}') + report['indirect_memory'] = {} + if args.mac_policy_layout: + plan=json.loads(args.mac_policy_layout.read_text());layout=plan['layout'];base=plan['address'] + def mac_read(address,size): + if address<0xfffffff000000000 or not 0= 1 << 64: + raise RuntimeError('Invalid indirect memory pointer: ' + request) + report['indirect_memory'][request] = {'pointer': hex(base), + 'address': hex(base + offset), 'bytes': packet(f'm{base + offset:x},{length:x}')} + if args.system_registers: + target = feature('target.xml') + report['system_registers'] = {} + wanted = {'TCR_EL1', 'TTBR0_EL1', 'TTBR1_EL1', 'SCTLR_EL1', 'CNTFRQ_EL0', + 'ESR_EL1', 'FAR_EL1', 'ELR_EL1', 'SPSR_EL1', 'ID_AA64MMFR1_EL1', 'ID_AA64MMFR0_EL1', 'MIDR_EL1'} + if process_plan: + wanted.add('TPIDR_EL1') + for include in target.findall('{http://www.w3.org/2001/XInclude}include'): + document = feature(include.attrib['href']) + for register in document.findall('reg'): + if register.attrib['name'] in wanted: + raw = packet(f'p{int(register.attrib["regnum"]):x}') + value = int.from_bytes(bytes.fromhex(raw), 'little') + report['system_registers'][register.attrib['name']] = hex(value) + if followed_display and followed_display['observed']: + followed_display['actual_process'] = process_identity() + if args.process_layout_plan: + try: + report['actual_process'] = process_identity() + except RuntimeError as error: + if not args.inspect_ui_threads or args.entry_process_name: + raise + # An arbitrary final snapshot may interrupt a kernel + # thread with no BSD process. Preserve that limitation; + # the selected UI tasks are independently owner-checked. + report['actual_process_error'] = str(error) + if args.inspect_entry_translations or kernel_ranges or indirect_kernel_ranges: + regs = {name: int(value, 16) for name, value in report['system_registers'].items()} + tcr = regs['TCR_EL1'] + bits = 64 - ((tcr >> 16) & 63) + if (tcr >> 30) & 3 != 2 or not 25 <= bits <= 48: + raise RuntimeError('Translation inspection requires actual 4K TTBR1 configuration') + + def translate_kernel_pointer(va): + input_va = va & ((1 << bits) - 1) + table = regs['TTBR1_EL1'] & 0x0000fffffffff000 + levels = (bits - 12 + 8) // 9 + walk = [] + for level in range(4 - levels, 4): + shift = 12 + 9 * (3 - level) + address = table + ((input_va >> shift) & 511) * 8 + raw = packet(f'm{address:x},8') + if len(raw) != 16 or raw.startswith('E'): + raise RuntimeError('Cannot read actual page-table descriptor') + descriptor = int.from_bytes(bytes.fromhex(raw), 'little') + walk.append({'level': level, 'descriptor_pa': hex(address), 'descriptor': hex(descriptor)}) + if not descriptor & 1: + if args.allow_unmapped_kernel_pages: + return {'walk': walk, 'mapped': False} + raise RuntimeError('Captured pointer has no valid hardware translation') + if level < 3 and descriptor & 3 == 3: + table = descriptor & 0x0000fffffffff000 + continue + if (level == 3 and descriptor & 3 != 3) or (level < 3 and descriptor & 3 != 1): + raise RuntimeError('Invalid leaf descriptor type') + return {'walk': walk, 'leaf_level': level, + 'leaf_bytes': 1 << shift, 'ap': (descriptor >> 6) & 3, + 'attr_index': (descriptor >> 2) & 7, 'shareability': (descriptor >> 8) & 3, + 'accessed': bool(descriptor & (1 << 10)), + 'pxn': bool(descriptor & (1 << 53)), 'uxn': bool(descriptor & (1 << 54)), + 'physical_address': hex((descriptor & 0x0000fffffffff000 & ~((1 << shift) - 1)) | (va & ((1 << shift) - 1)))} + raise RuntimeError('No hardware translation leaf') + + resolved_ranges = [(name, va, size, None) for name, va, size in kernel_ranges] + for name, pointer_address, size,offset in indirect_kernel_ranges: + raw = packet(f'm{pointer_address:x},8') + if len(raw) != 16 or raw.startswith('E'): + raise RuntimeError('Cannot read actual kernel range pointer') + va = int.from_bytes(bytes.fromhex(raw), 'little')+offset + if not 0xfffffff000000000 <= va < va + size < 1 << 64: + raise RuntimeError('Invalid actual indirect kernel range') + resolved_ranges.append((name, va, size, pointer_address)) + + old_mode = packet('qqemu.PhyMemMode') + if old_mode not in ('0', '1') or packet('Qqemu.PhyMemMode:1') != 'OK': + raise RuntimeError('QEMU physical debugger reads are unavailable') + try: + if args.inspect_entry_translations: + for capture in [*report['entry_indirect_memory'].values(), *report['indirect_memory'].values()]: + capture['translation'] = translate_kernel_pointer(int(capture['pointer'], 16)) + if resolved_ranges: + report['kernel_range_translations'] = {} + for name, va, size, pointer_address in resolved_ranges: + pages = [dict(virtual_address=hex(page), **translate_kernel_pointer(page)) + for page in range(va & ~4095, (va + size + 4095) & ~4095, 4096)] + report['kernel_range_translations'][name] = dict(address=hex(va), bytes=size, pages=pages) + if pointer_address is not None: + report['kernel_range_translations'][name]['pointer_address'] = hex(pointer_address) + finally: + if packet('Qqemu.PhyMemMode:' + old_mode) != 'OK': + raise RuntimeError('Cannot restore debugger virtual-memory mode') + report['argument_memory'] = {} + for index, value in enumerate(values[:4]): + if value >= 0xfffffff000000000: + raw = packet(f'm{value:x},100') + if not raw.startswith('E'): + report['argument_memory'][f'x{index}'] = bytes.fromhex(raw).split(b'\0')[0].decode(errors='replace') + if args.capture_screen: + try: + with socket.socket(socket.AF_UNIX,socket.SOCK_STREAM) as qmp: + qmp.settimeout(5);qmp.connect(str(screen_socket.resolve())) + with qmp.makefile('rwb',buffering=0) as stream: + if 'QMP' not in json.loads(stream.readline()):raise RuntimeError('Invalid actual QMP greeting') + def screen_request(command_name,arguments,identifier): + qmp.sendall((json.dumps(dict(execute=command_name,arguments=arguments,id=identifier))+'\n').encode()) + while True: + response=json.loads(stream.readline()) + if response.get('id')!=identifier:continue + if 'error' in response:raise RuntimeError(str(response['error'])) + if 'return' not in response:raise RuntimeError('Missing actual QMP response') + return response['return'] + screen_request('qmp_capabilities',{},1) + screen=fixture/'screen.png' + screen_request('screendump',dict(filename=str(screen.resolve()),format='png'),2) + data=screen.read_bytes() + if data[:8]!=bytes.fromhex('89504e470d0a1a0a'):raise RuntimeError('QEMU did not produce an actual PNG') + width,height=struct.unpack_from('>2I',data,16) + report['qemu_screen_capture']=dict(file='screen.png',sha256=hashlib.sha256(data).hexdigest(),bytes=len(data),width=width,height=height) + except Exception as error: + report['qemu_screen_capture']=dict(error=str(error)) + if args.allproc_address is not None: + def mem_bytes(address, length): + if not 0xfffffff000000000 <= address < (1 << 64) - length: + return None + raw = packet(f'm{address:x},{length:x}') + if len(raw) != length * 2 or raw.startswith('E'): + return None + return bytes.fromhex(raw) + entries = [] + node = None + head = mem_bytes(args.allproc_address, 8) + if head: + node = int.from_bytes(head, 'little') + seen = set() + while node and node not in seen and len(entries) < args.proc_list_max: + seen.add(node) + pid_raw = mem_bytes(node + args.proc_pid_offset, 4) + comm_raw = mem_bytes(node + args.proc_comm_offset, 17) + stat_raw = mem_bytes(node + args.proc_stat_offset, 1) + next_raw = mem_bytes(node + args.proc_list_offset, 8) + if pid_raw is None or next_raw is None: + break + entries.append(dict( + address=hex(node), + pid=int.from_bytes(pid_raw, 'little', signed=True), + name=(comm_raw.split(b'\0', 1)[0].decode('utf-8', 'replace') if comm_raw else ''), + state=(stat_raw[0] if stat_raw else None))) + node = int.from_bytes(next_raw, 'little') + report['process_list'] = entries + if args.inspect_ui_threads: + from thread_snapshot import capture + report['ui_thread_snapshots'] = capture(packet, entries, process_plan, report['system_registers']) + if args.capture_scratch_data: + base_symbol, size_symbol, expected = data_capture + def data_word(address): + if not 0xfffffff000000000 <= address < (1 << 64) - 8: + raise ValueError('Invalid native Data capture pointer') + raw = packet(f'm{address:x},8') + if raw.startswith('E') or len(raw) != 16: + raise ValueError('Cannot read native Data capture pointer') + return int.from_bytes(bytes.fromhex(raw), 'little') + base, size = data_word(base_symbol), data_word(size_symbol) + if size != expected or base & 4095 or not 0xfffffff000000000 <= base < (1 << 64) - size: + raise ValueError('Actual native Data range differs from the bounded input') + if shutil.disk_usage(fixture).free < 5 * 1024**3 + size: + raise ValueError('Preserve 5 GiB free before exporting Data') + destination = fixture / 'scratch-data-snapshot.apfs' + digest = hashlib.sha256() + with destination.open('xb') as output: + for offset in range(0, size, 16384): + count = min(16384, size - offset) + raw = packet(f'm{base + offset:x},{count:x}') + if raw.startswith('E') or len(raw) != count * 2: + raise ValueError('Incomplete scratch Data snapshot') + block = bytes.fromhex(raw) + if offset == 0 and block[32:36] != b'NXSB': + raise ValueError('Actual Data container header differs') + output.write(block) + digest.update(block) + report['scratch_data_snapshot'] = dict(file=destination.name, bytes=size, + address=hex(base), sha256=digest.hexdigest(), read_only=True, + guest_flushed=False, consistency='paused live block image; filesystem may be unclean') + if args.exercise_ufs_allocation: + if interrupted or values[32] != args.address: + raise RuntimeError('Allocation exercise requires the requested checkpoint') + from ufs_allocation import exercise + report['ufs_allocation_exercise'] = exercise(packet, feature, continue_observing, + args.kernel, report['system_registers'], fixture, args.esp) + (fixture / 'gdb-stop.json').write_text(json.dumps(report, indent=2) + '\n') + print(json.dumps({key: report[key] for key in ['x0', 'x1', 'x29', 'x30', 'sp', 'pc', 'stop']}, indent=2)) + print(json.dumps(report['argument_memory'], ensure_ascii=False)) + if args.system_registers: + print(json.dumps(report['system_registers'], indent=2)) + except Exception as error: + (fixture / 'gdb-failure.json').write_text(json.dumps({ + 'kernel_sha256': state['sha256'], 'qemu_accelerator': args.qemu_accel, + 'qemu_memory_mib': qemu_memory_mib, + 'qemu_cpu_model': 'cortex-a710' if args.qemu_accel == 'tcg' else 'host', + 'error': str(error), 'qemu_exit_code': process.poll(), + 'trace_events': locals().get('trace_events', []), + # A late diagnostic read can fail at an early boot panic. Preserve + # already captured registers/arguments without certifying a stop. + 'partial_report': locals().get('report', {}), + 'checkpoint_observed': False}, indent=2) + '\n') + raise + finally: + process.terminate() + try: + process.wait(timeout=2) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + if args.capture_screen:screen_directory.cleanup() diff --git a/scripts/qemu-entry/gdb_packets.py b/scripts/qemu-entry/gdb_packets.py new file mode 100644 index 0000000..a1a8589 --- /dev/null +++ b/scripts/qemu-entry/gdb_packets.py @@ -0,0 +1,43 @@ +"""Buffered packet reception for the bounded QEMU GDB observer.""" + + +class Packets: + def __init__(self, connection): + self.connection = connection + self.pending = bytearray() + + def request(self, value): + if value is None: + self.connection.settimeout(3) + self.connection.sendall(b'\x03') + else: + payload = value.encode() + self.connection.sendall(b'$' + payload + b'#' + f'{sum(payload) % 256:02x}'.encode()) + return self.receive() + + def receive(self): + # Retain partial packets across socket timeouts. The caller may interrupt + # a running target afterwards; socket.makefile would become unusable. + while True: + start = self.pending.find(b'$') + if start < 0: + self.pending.clear() # acknowledgements, before a reply + else: + del self.pending[:start] + end = self.pending.find(b'#', 1) + if end > 1024 * 1024: + raise RuntimeError('GDB response exceeds observer bound') + if end >= 0 and len(self.pending) >= end + 3: + data = self.pending[1:end] + checksum = self.pending[end + 1:end + 3] + del self.pending[:end + 3] + if int(checksum, 16) != sum(data) % 256: + raise RuntimeError('GDB checksum mismatch') + self.connection.sendall(b'+') + return data.decode() + if len(self.pending) > 1024 * 1024: + raise RuntimeError('GDB response exceeds observer bound') + chunk = self.connection.recv(65536) + if not chunk: + raise RuntimeError('GDB connection closed') + self.pending.extend(chunk) diff --git a/scripts/qemu-entry/link.ld b/scripts/qemu-entry/link.ld new file mode 100644 index 0000000..ea65984 --- /dev/null +++ b/scripts/qemu-entry/link.ld @@ -0,0 +1,7 @@ +/* SPDX-License-Identifier: BSD-2-Clause-Patent */ +ENTRY(_start) +SECTIONS { + . = 0x40080000; + .text : { *(.text.boot) *(.text*) *(.rodata*) } + /DISCARD/ : { *(.comment) *(.note*) } +} diff --git a/scripts/qemu-entry/live-hid.py b/scripts/qemu-entry/live-hid.py new file mode 100644 index 0000000..bb66486 --- /dev/null +++ b/scripts/qemu-entry/live-hid.py @@ -0,0 +1,71 @@ +"""Own-source HID command mailbox at a genuine timer checkpoint; QEMU only.""" +import hashlib,json,subprocess +from pathlib import Path + +FIELDS={'native_hid_stage':4,'native_hid_load_status':4,'native_hid_matching_load_status':4, + 'native_hid_input_meta_status':4,'native_hid_input_object':8,'native_hid_input_command':4, + 'native_hid_input_events':32,'native_hid_input_results':32,'native_hid_input_registry':2048, + 'native_legacy_transfer_calls':4,'native_legacy_retirement_calls':4} + +class LiveHid: + def __init__(self,kernel,directory): + self.directory=Path(directory);self.directory.mkdir(parents=True,exist_ok=True) + self.last_id=0;self.events=[] + output=subprocess.check_output(['xcrun','nm','-U',str(kernel)],text=True,timeout=15) + names={p[2][1:]:int(p[0],16) for line in output.splitlines() if len(p:=line.split())==3} + self.address=names['native_hid_live_checkpoint'];self.fields={n:(names[n],size) for n,size in FIELDS.items()} + for n,size in {'native_keystore_stage':4,'native_keystore_load_status':4, + 'native_keystore_service':8,'native_keystore_class_size':4, + 'osvariant_status':8, 'native_keyevents_stage':4, 'native_keyevents_status':4, + 'native_keyevents_service':8, 'native_keyevents_refused_opens':4}.items(): + if n in names:self.fields[n]=(names[n],size) + for n,size in {'native_legacy_stage':4, 'native_legacy_transfer_status':4, + 'native_legacy_transfer_bytes':8, 'native_legacy_swap_validation_calls':4, + 'native_legacy_swap_validation_result':4, 'native_legacy_swap_validation_reason':4}.items(): + if n in names:self.fields[n]=(names[n],size) + # Synthetic touch (QEMU only): the same published finger set the hardware touch driver writes (see TouchProbe.cpp). + self.touch={n:names[n] for n in ('native_touch_seq','native_touch_count','native_touch_xs','native_touch_ys','native_touch_ids') if n in names} + self.identity={'kernel_sha256':hashlib.sha256(Path(kernel).read_bytes()).hexdigest(),'checkpoint':hex(self.address)} + self.publish({'state':'waiting-for-real-timer','identity':self.identity}) + def publish(self,status): + temporary=self.directory/'status.tmp';temporary.write_text(json.dumps(status,indent=2)+'\n');temporary.replace(self.directory/'status.json') + def checkpoint(self,packet,elapsed): + data={} + for name,(address,size) in self.fields.items(): + raw=packet(f'm{address:x},{size:x}') + if raw.startswith('E') or len(raw)!=size*2:raise RuntimeError('Cannot read owned HID observation '+name) + b=bytes.fromhex(raw) + if name.endswith('registry'):value=b.split(b'\0',1)[0].decode(errors='replace') + elif size==32:value=[int.from_bytes(b[i:i+4],'little',signed=name.endswith('results')) for i in range(0,32,4)] + else:value=int.from_bytes(b,'little',signed=name.endswith('status')) + data[name]=value + request_path=self.directory/'command.json';stop=False + if request_path.exists(): + if request_path.stat().st_size>1024:raise RuntimeError('Oversized live HID command') + request=json.loads(request_path.read_text()) + touch=request.get('action') in ('touch','lift') + if (set(request)!={'id','action'} and not (touch and set(request)<={'id','action','x','y'})) or type(request['id']) is not int or not 1<=request['id']<=10000 or request['action'] not in ('home','stop','touch','lift'): + raise RuntimeError('Invalid live HID command; only bounded Home, stop, touch or lift accepted') + if request['id']>self.last_id and touch: + if len(self.touch)!=5:raise RuntimeError('Kernel has no touch publication symbols') + def put(address,value): + if packet(f'M{address:x},4:'+int(value&0xffffffff).to_bytes(4,'little').hex())!='OK':raise RuntimeError('Touch publication write failed') + seq=int.from_bytes(bytes.fromhex(packet(f"m{self.touch['native_touch_seq']:x},4")),'little') + if request['action']=='touch': + x,y=request['x'],request['y'] + if type(x) is not int or type(y) is not int or not 0<=x<30480 or not 0<=y<20320:raise RuntimeError('Touch coordinate outside the 30480x20320 axes') + put(self.touch['native_touch_ids'],1);put(self.touch['native_touch_xs'],x);put(self.touch['native_touch_ys'],y);put(self.touch['native_touch_count'],1) + else:put(self.touch['native_touch_count'],0) + put(self.touch['native_touch_seq'],(seq+2)&0xfffffffe) + self.last_id=request['id'];self.events.append({**request,'elapsed_seconds':round(elapsed,3)}) + elif request['id']>self.last_id: + if request['action']=='stop':stop=True;self.last_id=request['id'] + elif data['native_hid_input_command']==0: + address=self.fields['native_hid_input_command'][0] + if packet(f'M{address:x},4:01000000')!='OK' or packet(f'm{address:x},4')!='01000000': + raise RuntimeError('Owned input mailbox write/readback failed') + self.last_id=request['id'];data['native_hid_input_command']=1 + if self.last_id==request['id']:self.events.append({**request,'elapsed_seconds':round(elapsed,3)}) + self.publish({'state':'stopping' if stop else 'running','identity':self.identity,'elapsed_seconds':round(elapsed,3), + 'accepted_command_id':self.last_id,'commands':self.events,'observed':data}) + return stop diff --git a/scripts/qemu-entry/make-dt.py b/scripts/qemu-entry/make-dt.py new file mode 100755 index 0000000..ee39344 --- /dev/null +++ b/scripts/qemu-entry/make-dt.py @@ -0,0 +1,187 @@ +#!/usr/bin/env python3 +"""Generate a synthetic Apple-format DT for the single-CPU QEMU entry fixture. + +This is neither sheng board data nor an iPad device identity. The arm-io +range describes QEMU PL011 solely to pass early platform discovery. +""" +from pathlib import Path +import argparse +import os +import struct +import zlib +import uuid +import platform +import subprocess + + +def node(name, properties=(), children=()): + properties = [('name', name.encode() + b'\0'), *properties] + result = struct.pack(' 255: + checksum = (checksum & 255) + (checksum >> 8) + raw[1] = checksum + return raw + + image = bytearray(8192) + image[:16] = header(b'nvram', 32) + image[32:48] = header(b'common', 16 + 2048) + free_at = 32 + 16 + 2048 + image[free_at:free_at + 16] = header(b'free', len(image) - free_at) + struct.pack_into(' 1 and not args.armcpu): + parser.error('--smp is 1..8 and needs --armcpu') +if args.root_match_volume_uuid and (not args.virtio_mmio_fdt or args.ramdisk): + parser.error('Direct volume matching requires a real offline disk and conflicts with a RAM root') +if args.data_ramdisk and (not args.root_match_volume_uuid or args.ramdisk): + parser.error('Scratch Data requires actual disk root matching and no RAM root') +root_matching = [] +if args.root_match_volume_uuid: + xml = ('IOProviderClassIOMedia' + 'IOPropertyMatchUUID' + + str(args.root_match_volume_uuid).upper() + '\0') + root_matching = [('root-matching', xml.encode())] +virtio_nodes = [] +if args.virtio_mmio_fdt: + if not args.armvirt: + parser.error('--virtio-mmio-fdt requires --armvirt') + blob = args.virtio_mmio_fdt.read_bytes() + magic, total, structure, strings, _, version, _, _, string_bytes, structure_bytes = struct.unpack_from('>10I', blob) + if magic != 0xd00dfeed or total > len(blob) or version != 17 or strings + string_bytes > total or structure + structure_bytes > total: + raise SystemExit('Unexpected actual QEMU FDT') + stack, entries = [], [] + cursor = structure + while cursor < structure + structure_bytes: + tag = struct.unpack_from('>I', blob, cursor)[0] + cursor += 4 + if tag == 1: + end = blob.index(b'\0', cursor) + stack.append(dict(name=blob[cursor:end].decode(), properties={})) + cursor = (end + 4) & ~3 + elif tag == 2: + entry = stack.pop() + properties = entry['properties'] + if b'virtio,mmio' in properties.get('compatible', b'').split(b'\0'): + parent = stack[-1]['properties'] + if (parent.get('#address-cells'), parent.get('#size-cells')) != (struct.pack('>I', 2), struct.pack('>I', 2)): + raise SystemExit('Unexpected QEMU VirtIO parent address encoding') + values = struct.unpack('>4I', properties['reg']) + address, size = (values[0] << 32) | values[1], (values[2] << 32) | values[3] + if not 0x08000000 <= address < address + size <= 0x0c000000 or size < 0x108 or 'dma-coherent' not in properties: + raise SystemExit('VirtIO resource is outside the established QEMU bus or not coherent') + entries.append((address, size)) + elif tag == 3: + length, name_offset = struct.unpack_from('>II', blob, cursor) + cursor += 8 + if name_offset >= string_bytes or cursor + length > structure + structure_bytes: + raise SystemExit('Malformed QEMU FDT property') + name = blob[strings + name_offset:blob.index(b'\0', strings + name_offset)].decode() + stack[-1]['properties'][name] = blob[cursor:cursor + length] + cursor = (cursor + length + 3) & ~3 + elif tag == 9: + break + elif tag != 4: + raise SystemExit('Unknown QEMU FDT tag') + if not entries or len(entries) != len(set(entries)): + raise SystemExit('Missing or repeated actual VirtIO resource windows') + for index, (address, size) in enumerate(sorted(entries)): + virtio_nodes.append(node('virtio-mmio-' + str(index), [('compatible', b'virtio,mmio\0'), + ('reg', struct.pack(' 1 else []), node('defaults', + [('vmm-present', struct.pack('> 14) & 3 or (tcr >> 30) & 3 != 2: + raise ValueError('Stack snapshot requires active native 4K CPU translation') + old_mode = packet('qqemu.PhyMemMode') + if old_mode not in ('0', '1') or packet('Qqemu.PhyMemMode:1') != 'OK': + raise ValueError('QEMU physical debugger reads unavailable') + try: + def physical(address, size): + if not 0 < address < (1 << 48) - size or not 0 < size <= 16: + raise ValueError('Invalid bounded physical read') + raw = packet(f'm{address:x},{size:x}') + if raw.startswith('E') or len(raw) != size * 2: + raise ValueError('Unbacked physical memory') + return bytes.fromhex(raw) + + for row in results: + if row.get('page_shift') != 12 or row.get('root_level') not in (0, 1, 2): + row['stack_error'] = 'Unqualified native 4K task geometry' + continue + + def user_record(va): + va_bits = 64 - (tcr & 63) + root_bits = va_bits - (12 + 9 * (3 - row['root_level'])) + if not 1 <= root_bits <= 9: + raise ValueError('Unsupported task root-table extent') + if not 0 < va < 1 << va_bits or va & 15: + raise ValueError('Invalid user frame address') + table = row['ttep'] + # A native 4K hierarchy can have a smaller initial table: + # T0SZ=26 uses 256 L1 entries, aligned to 2048 bytes. + if table & ((8 << root_bits) - 1): + raise ValueError('Unaligned task page table') + for level in range(row['root_level'], 4): + shift = 12 + 9 * (3 - level) + descriptor = int.from_bytes(physical(table + ((va >> shift) & 511) * 8, 8), 'little') + if descriptor & 1 == 0: + raise ValueError('User frame is not resident') + base = descriptor & 0x0000fffffffff000 + if level < 3 and descriptor & 3 == 3: + table = base + continue + if (level == 3 and descriptor & 3 != 3) or (level < 3 and descriptor & 3 != 1): + raise ValueError('Invalid user frame page descriptor') + if not descriptor & (1 << 6): + raise ValueError('User frame page lacks EL0 access') + return physical((base & ~((1 << shift) - 1)) | (va & ((1 << shift) - 1)), 16) + raise ValueError('Missing user frame leaf') + + for item in row['threads']: + if 'user_fp' not in item: + continue + fp, sp = int(item['user_fp'], 16), int(item['user_sp'], 16) + item['user_frames'] = [] + item['user_stack_complete'] = False + try: + for _ in range(32): + if not fp: + break + if not sp <= fp < sp + 65536: + raise ValueError('Frame exceeds bounded stack window') + next_fp, lr = struct.unpack('<2Q', user_record(fp)) + item['user_frames'].append(dict(fp=hex(fp), lr=hex(lr))) + if next_fp and next_fp <= fp: + raise ValueError('Non-increasing user frame chain') + fp = next_fp + item['user_stack_complete'] = fp == 0 + except ValueError as error: + item['stack_error'] = str(error) + finally: + if packet('Qqemu.PhyMemMode:' + old_mode) != 'OK': + raise RuntimeError('Cannot restore debugger virtual-memory mode') + return dict(native_sha256=plan['native_sha256'], read_only=True, + process_limit=16, threads_per_process_limit=64, frames_per_thread_limit=32, + stack_window_bytes=65536, stack_source='actual-native-4k-task-tables', + selected_processes=len(selected), processes=results) diff --git a/scripts/qemu-entry/ufs_allocation.py b/scripts/qemu-entry/ufs_allocation.py new file mode 100644 index 0000000..ce62b86 --- /dev/null +++ b/scripts/qemu-entry/ufs_allocation.py @@ -0,0 +1,186 @@ +"""Debugger-invoked actual XNU allocation test in an owned, bounded QEMU guest. + +This changes guest registers/stack and allocates guest RAM. It is not a +natural driver start, a UFS model, or hardware evidence. No kernel code is +patched; the held candidate is the input. All addresses come from its symbols +or active translation tables. The enclosing runner always terminates QEMU. +""" +import hashlib +import json +import re +import struct +import subprocess +import shlex +import tomllib + + +def walk4k(read_u64, tcr, ttbr1, mair, va): + bits = 64 - ((tcr >> 16) & 63) + if (tcr >> 30) & 3 != 2 or not 25 <= bits <= 48: + raise RuntimeError('Require active native 4 KiB kernel translation') + if va >> bits != (1 << (64-bits))-1: + raise RuntimeError('Require a canonical kernel virtual address') + table = ttbr1 & 0x0000fffffffff000 + input_va = va & ((1 << bits) - 1) + chain = [] + for level in range(4 - (bits - 12 + 8) // 9, 4): + shift = 12 + 9 * (3 - level) + address = table + ((input_va >> shift) & 511) * 8 + descriptor = read_u64(address) + chain.append(dict(level=level, descriptor_pa=hex(address), descriptor=hex(descriptor))) + if not descriptor & 1: + raise RuntimeError('Allocated page is unmapped') + if level < 3 and descriptor & 3 == 3: + table = descriptor & 0x0000fffffffff000 + continue + if level == 0 or (descriptor & 3) != (3 if level == 3 else 1): + raise RuntimeError('Invalid hardware leaf') + pa = (descriptor & 0x0000fffffffff000 & ~((1 << shift)-1)) | (va & ((1 << shift)-1)) + attr = (descriptor >> 2) & 7 + return dict(va=hex(va), pa=hex(pa), level=level, attr_index=attr, + mair_byte=(mair >> (8*attr)) & 255, shareability=(descriptor >> 8) & 3, + ap=(descriptor >> 6) & 3, accessed=bool(descriptor & 1024), walk=chain) + raise RuntimeError('No valid leaf') + + +def exercise(packet, feature, resume, kernel, system_registers, output, esp): + config = tomllib.loads((esp/'EFI/CrabApple/crabapple.toml').read_text()) + entries = [e for e in config['entry'] if e['id'] == config['boot']['default']] + if len(entries) != 1: + raise RuntimeError('Require an unambiguous prepared boot entry') + boot_args = shlex.split(entries[0].get('args','')) + if any(a.split('=',1)[0] in ('native-ufs-root','native-ufs-probe') for a in boot_args): + raise RuntimeError('Allocation-only exercise must not enable UFS startup') + symbols = subprocess.check_output(['xcrun', 'nm', '--demangle', str(kernel)], text=True) + + def symbol(name): + found = re.findall(r'^([0-9a-fA-F]+)\s+\S\s+' + re.escape(name) + '$', symbols, re.M) + if len(found) != 1: + raise RuntimeError('Missing/ambiguous actual kernel symbol: ' + name) + return int(found[0], 16) + + functions = {name: symbol(name) for name in ( + 'IOBufferMemoryDescriptor::inTaskWithOptions(task*, unsigned int, unsigned long, unsigned long)', + 'IOBufferMemoryDescriptor::getBytesNoCopy()', + 'IOGeneralMemoryDescriptor::getPhysicalSegment(unsigned long long, unsigned long long*, unsigned int)')} + allocate, get_bytes, get_segment = functions.values() + + def read(address, count): + data = packet(f'm{address:x},{count:x}') + if data.startswith('E') or len(data) != count * 2: + raise RuntimeError(f'Guest memory read failed: {address:x}/{count:x}') + return bytes.fromhex(data) + + def write(address, data): + if packet(f'M{address:x},{len(data):x}:' + data.hex()) != 'OK': + raise RuntimeError('Guest scratch write failed') + + def u64(address): + return int.from_bytes(read(address, 8), 'little') + + original = packet('g') + registers = struct.unpack_from('<33Q', bytes.fromhex(original)) + saved_sp, return_pc = registers[31:33] + scratch = saved_sp - 0x80 + saved_stack = read(scratch, 0x80) + call_sp = saved_sp - 0x100 + events = [] + report = dict(debugger_invoked=True, natural_driver_start=False, kernel_code_patched=False, + allocation_bytes=139264, alignment=4096, options='0x813', + functions={k: hex(v) for k, v in functions.items()}, calls=events, + source_sha256=hashlib.sha256(open(__file__, 'rb').read()).hexdigest()) + + def call(address, *args): + for index, value in list(enumerate(args)) + [(30, return_pc), (31, call_sp), (32, address)]: + if packet(f'P{index:x}=' + int(value).to_bytes(8, 'little').hex()) != 'OK': + raise RuntimeError('Cannot set guest call registers') + stop, values, interrupted = resume() + event = dict(function=hex(address), arguments=[hex(v) for v in args], + result=hex(values[0]), pc=hex(values[32]), sp=hex(values[31]), + interrupted=interrupted) + events.append(event) + if interrupted or values[32] != return_pc or values[31] != call_sp: + raise RuntimeError('Guest allocation API did not return to its bounded caller') + return values[0] + + regs = {name: int(value, 16) for name, value in system_registers.items()} + tcr = regs['TCR_EL1'] + bits = 64 - ((tcr >> 16) & 63) + if (tcr >> 30) & 3 != 2 or not 25 <= bits <= 48 or not regs['SCTLR_EL1'] & 1: + raise RuntimeError('Require active native 4 KiB kernel translation') + mair = None + for include in feature('target.xml').findall('{http://www.w3.org/2001/XInclude}include'): + for reg in feature(include.attrib['href']).findall('reg'): + if reg.attrib['name'] == 'MAIR_EL1': + mair = int.from_bytes(bytes.fromhex(packet(f'p{int(reg.attrib["regnum"]):x}')), 'little') + if mair is None: + raise RuntimeError('Cannot inspect active memory attributes') + report['system_registers'] = dict(system_registers, MAIR_EL1=hex(mair)) + counters = {name: symbol(name) for name in ('_native_ufs_probe_requested', '_native_ufs_mapped')} + before = {name: int.from_bytes(read(address, 4), 'little') for name, address in counters.items()} + if any(before.values()): + raise RuntimeError('Allocation exercise requires UFS probes/MMIO disabled') + + try: + task = u64(symbol('_kernel_task')) + descriptor = call(allocate, task, 0x813, 139264, 4096) + if not descriptor: + raise RuntimeError('Actual XNU allocation failed') + va = call(get_bytes, descriptor) + write(scratch, bytes(8)) + pa = call(get_segment, descriptor, 0, scratch, 0x800) + contiguous = u64(scratch) + if not va or not pa or (va | pa) & 4095 or contiguous < 139264: + raise RuntimeError('Actual XNU DMA span is invalid') + report.update(descriptor=hex(descriptor), virtual=hex(va), physical=hex(pa), contiguous=contiguous) + # Every page is queried through the actual API, independently of its + # initial aggregate contiguous length, then compared with active PTEs. + segments = [] + for offset in range(0, 139264, 4096): + actual = call(get_segment, descriptor, offset, scratch, 0x800) + length = u64(scratch) + if actual != pa+offset or length < 139264-offset: + raise RuntimeError('Actual segment walk disagrees with contiguous allocation') + segments.append(dict(offset=offset, pa=hex(actual), length=length)) + # Exact final byte and out-of-range refusal, with nonzero output sentinel. + if call(get_segment, descriptor, 139263, scratch, 0x800) != pa+139263 or u64(scratch) != 1: + raise RuntimeError('Last-byte extent mismatch') + write(scratch, bytes([0xa5])*8) + if call(get_segment, descriptor, 139264, scratch, 0x800) or u64(scratch): + raise RuntimeError('Out-of-range segment accepted') + old_mode = packet('qqemu.PhyMemMode') + if old_mode != '0' or packet('Qqemu.PhyMemMode:1') != 'OK': + raise RuntimeError('Physical page-table inspection unavailable') + try: + pages = [walk4k(u64, tcr, regs['TTBR1_EL1'], mair, va+offset) for offset in range(0,139264,4096)] + finally: + if packet('Qqemu.PhyMemMode:' + old_mode) != 'OK': + raise RuntimeError('Cannot restore virtual memory access') + report.update(segments=segments, pages=pages) + for offset, page in zip(range(0,139264,4096), pages): + if int(page['pa'],16) != pa+offset or not page['accessed'] or page['ap'] != 0: + raise RuntimeError('Active PTE disagrees with actual allocator/segment API') + # XNU pmap.c's default mapping uses SH_OUTER_MEMORY (2), which + # includes inner sharing; do not assume the standalone guest's 3. + if page['mair_byte'] != 0xff or page['shareability'] != 2: + raise RuntimeError('Expected XNU normal WBWA outer-shareable DMA mapping') + report.update(passed=True, + retained_until_guest_exit=True, + scope='Actual XNU allocation and active CPU mapping in TCG; no UFS DMA, SM8550 SMMU or hardware timing') + except Exception as error: + report.update(passed=False, error=str(error)) + raise + finally: + write(scratch, saved_stack) + if packet('G' + original) != 'OK' or packet('g') != original: + raise RuntimeError('Cannot restore original general/vector register state') + report['registers_restored'] = True + after = {name: int.from_bytes(read(address,4),'little') for name,address in counters.items()} + report['ufs_counters_before'] = before + report['ufs_counters_after'] = after + if after != before: + report.update(passed=False, error='Unexpected UFS access during allocation exercise') + (output/'ufs-allocation.json').write_text(json.dumps(report,indent=2)+'\n') + if after != before: + raise RuntimeError('Unexpected UFS access during allocation exercise') + return report diff --git a/scripts/qemu-entry/verified-root-probe.c b/scripts/qemu-entry/verified-root-probe.c new file mode 100644 index 0000000..baa6fd4 --- /dev/null +++ b/scripts/qemu-entry/verified-root-probe.c @@ -0,0 +1,88 @@ +/* SPDX-License-Identifier: BSD-2-Clause-Patent */ +/* Development-only crypto instrument. It never changes root/AMFI policy. */ +#include "verified_root.h" +#include +#include +#include +#include +#include + +void native_verified_root_probe(void); +void native_verified_root_checkpoint(const volatile void *, const volatile void *); +extern const struct vr_crypto vr_xnu_crypto; +extern const uint8_t verified_root_manifest[], verified_root_key[], verified_root_samples[]; +extern const uint64_t verified_root_manifest_bytes, verified_root_sample_indices[4], verified_root_sample_lengths[4]; +volatile uint32_t native_verified_root_status = 0; +volatile uint32_t native_verified_root_results[8] = {0}; +volatile uint32_t native_verified_root_signature_calls = 0; +volatile uintptr_t native_verified_root_manifest_pointer = 0; +volatile uintptr_t native_verified_root_key_pointer = 0; +volatile uint64_t native_verified_root_block_count = 0; + +static int hash(const void *p, size_t n, uint8_t out[32]) { + return vr_xnu_crypto.sha256(p, n, out); +} +static int verify(const uint8_t key[VR_KEY_BYTES], const uint8_t digest[32], const uint8_t signature[VR_SIGNATURE_BYTES]) { + native_verified_root_signature_calls++; + return vr_xnu_crypto.rsa3072_sha256(key, digest, signature); +} +static const struct vr_crypto provider = {hash, verify}; + +__attribute__((noinline)) void +native_verified_root_checkpoint(const volatile void *manifest, const volatile void *key) +{ + __asm__ volatile("" : : "r"(manifest), "r"(key) : "memory"); +} + +void +native_verified_root_probe(void) +{ + struct vr_view view, refused; + vm_offset_t copy = 0; + vm_size_t copy_bytes = round_page((vm_size_t)verified_root_manifest_bytes); + uint8_t wrong_key[VR_KEY_BYTES]; + uint8_t *block = NULL; + int all_blocks = 0; + native_verified_root_manifest_pointer = (uintptr_t)verified_root_manifest; + native_verified_root_key_pointer = (uintptr_t)verified_root_key; + native_verified_root_results[0] = vr_open(verified_root_manifest, verified_root_manifest_bytes, + verified_root_key, 1, &provider, &view); + if (native_verified_root_results[0] != VR_OK) goto done; + native_verified_root_block_count = view.block_count; + for (unsigned i = 0; i < 4; ++i) { + if (vr_verify_block(&view, verified_root_sample_indices[i], verified_root_samples + i * VR_BLOCK_BYTES, + (size_t)verified_root_sample_lengths[i], &provider) == VR_OK) all_blocks++; + } + native_verified_root_results[1] = (uint32_t)all_blocks; + if (kmem_alloc(kernel_map, ©, copy_bytes, + KMA_KOBJECT | KMA_DATA, VM_KERN_MEMORY_SECURITY) != KERN_SUCCESS) goto done; + memcpy((void *)copy, verified_root_manifest, verified_root_manifest_bytes); + uint8_t *changed = (uint8_t *)copy; + changed[VR_HEADER_BYTES] ^= 1; + native_verified_root_results[2] = vr_open(changed, verified_root_manifest_bytes, verified_root_key, 1, &provider, &refused); + changed[VR_HEADER_BYTES] ^= 1; + changed[40] ^= 1; + native_verified_root_results[3] = vr_open(changed, verified_root_manifest_bytes, verified_root_key, 1, &provider, &refused); + changed[40] ^= 1; + changed[VR_HEADER_BYTES + VR_SIGNATURE_BYTES] ^= 1; + native_verified_root_results[4] = vr_open(changed, verified_root_manifest_bytes, verified_root_key, 1, &provider, &refused); + block = kalloc_data(VR_BLOCK_BYTES, Z_WAITOK); + memcpy(block, verified_root_samples, VR_BLOCK_BYTES); + block[7] ^= 1; + native_verified_root_results[5] = vr_verify_block(&view, 0, block, VR_BLOCK_BYTES, &provider); + memcpy(wrong_key, verified_root_key, VR_KEY_BYTES); + wrong_key[100] ^= 1; + native_verified_root_results[6] = vr_open(verified_root_manifest, verified_root_manifest_bytes, + wrong_key, 1, &provider, &refused); + native_verified_root_results[7] = vr_open(verified_root_manifest, verified_root_manifest_bytes, + verified_root_key, 2, &provider, &refused); + if (native_verified_root_results[1] == 4 && native_verified_root_results[2] == VR_SIGNATURE && + native_verified_root_results[3] == VR_SIGNATURE && native_verified_root_results[4] == VR_HASH_TABLE && + native_verified_root_results[5] == VR_BLOCK && native_verified_root_results[6] == VR_KEY && + native_verified_root_results[7] == VR_FORMAT && !refused.hashes && native_verified_root_signature_calls == 4) + native_verified_root_status = 1; +done: + if (block) kfree_data(block, VR_BLOCK_BYTES); + if (copy) kmem_free(kernel_map, copy, copy_bytes); + native_verified_root_checkpoint(&native_verified_root_manifest_pointer, &native_verified_root_key_pointer); +} diff --git a/scripts/requirements-adreno-compiler.txt b/scripts/requirements-adreno-compiler.txt new file mode 100644 index 0000000..fe418a8 --- /dev/null +++ b/scripts/requirements-adreno-compiler.txt @@ -0,0 +1,6 @@ +meson==1.9.1 +Mako==1.3.10 +MarkupSafe==3.0.4 +packaging==25.0 +PyYAML==6.0.3 +ply==3.11 diff --git a/scripts/save-turnip-darwin-patch.py b/scripts/save-turnip-darwin-patch.py new file mode 100644 index 0000000..4937c6b --- /dev/null +++ b/scripts/save-turnip-darwin-patch.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +"""Regenerate patches/mesa/turnip-darwin-kmd.patch from the working Mesa tree. + +The patch is the difference between "pinned Mesa + the other recorded patches" and the working tree for the files the +Darwin backend touches, so it can be applied last. Edit the Mesa tree, run this, then run build-adreno-compiler-host.py to +prove the tree equals the recorded patches.""" +import json +import os +from pathlib import Path +import shutil +import subprocess +import tempfile + +ROOT = Path(__file__).resolve().parents[1] +MESA = ROOT / '.cache/mesa-work' +NEW = ['src/freedreno/vulkan/tu_knl_darwin.cc'] +MINE = ['meson.options', 'src/freedreno/vulkan/meson.build', 'src/freedreno/vulkan/tu_knl.cc', + 'src/freedreno/vulkan/tu_knl.h', 'src/freedreno/vulkan/tu_device.cc', + 'src/vulkan/wsi/wsi_common_metal_layer.m', 'src/util/cache_ops_aarch64.c'] + NEW +OUT = ROOT / 'patches/mesa/turnip-darwin-kmd.patch' +others = [p for p in json.loads((ROOT / 'scripts/data/adreno-compiler-inputs.json').read_text())['patches'] + if Path(p) != OUT.relative_to(ROOT)] + + +def git(tmp, *args, check=True): + return subprocess.run(['git', '-c', 'user.email=a@b', '-c', 'user.name=n', *args], cwd=tmp, check=check, + capture_output=True, text=True).stdout + + +tmp = Path(tempfile.mkdtemp(dir=ROOT / '.cache')) +try: + git(tmp, 'init', '-q') + for f in MINE: + if f in NEW: + continue + (tmp / f).parent.mkdir(parents=True, exist_ok=True) + (tmp / f).write_bytes(subprocess.run(['git', 'show', 'HEAD:' + f], cwd=MESA, check=True, capture_output=True).stdout) + git(tmp, 'add', '-A') + git(tmp, 'commit', '-q', '-m', 'base') + for patch in others: + for f in MINE: + git(tmp, 'apply', '--include=' + f, str(ROOT / patch), check=False) + git(tmp, 'add', '-A') + git(tmp, 'commit', '-q', '-m', 'others', '--allow-empty') + for f in MINE: + (tmp / f).parent.mkdir(parents=True, exist_ok=True) + shutil.copy(MESA / f, tmp / f) + for f in NEW: + git(tmp, 'add', '-N', f) + OUT.write_text(git(tmp, 'diff', '--binary')) +finally: + shutil.rmtree(tmp) +print('wrote', OUT.relative_to(ROOT), len(OUT.read_text().splitlines()), 'lines') diff --git a/scripts/save-xnu-patch.py b/scripts/save-xnu-patch.py new file mode 100644 index 0000000..7c2e118 --- /dev/null +++ b/scripts/save-xnu-patch.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 +"""Regenerate one incremental XNU patch from the working tree: save-xnu-patch.py NAME FILE... + +The result is the difference between "pinned XNU + every patch that sorts before NAME" and the working tree, for the named +files only, written to patches/xnu/NAME. zzz-boot-progress.patch must be reversed in the working tree first (the build +script reverses and re-applies it around the others).""" +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile + +ROOT = Path(__file__).resolve().parents[1] +XNU = ROOT / '.cache/xnu-work' +name, files = sys.argv[1], sys.argv[2:] +earlier = [p for p in sorted((ROOT / 'patches/xnu').glob('*.patch')) if p.name < name] + + +def git(tmp, *args, check=True): + return subprocess.run(['git', '-c', 'user.email=a@b', '-c', 'user.name=n', *args], cwd=tmp, check=check, + capture_output=True, text=True).stdout + + +tmp = Path(tempfile.mkdtemp(dir=ROOT / '.cache')) +try: + git(tmp, 'init', '-q') + for f in files: + (tmp / f).parent.mkdir(parents=True, exist_ok=True) + (tmp / f).write_bytes(subprocess.run(['git', 'show', 'HEAD:' + f], cwd=XNU, check=True, capture_output=True).stdout) + git(tmp, 'add', '-A') + git(tmp, 'commit', '-q', '-m', 'base') + for patch in earlier: + for f in files: + git(tmp, 'apply', '--include=' + f, str(patch), check=False) + git(tmp, 'add', '-A') + git(tmp, 'commit', '-q', '-m', 'earlier', '--allow-empty') + for f in files: + shutil.copy(XNU / f, tmp / f) + diff = git(tmp, 'diff', '--binary') +finally: + shutil.rmtree(tmp) +(ROOT / 'patches/xnu' / name).write_text(diff) +print('wrote patches/xnu/%s (%d lines)' % (name, len(diff.splitlines()))) diff --git a/scripts/seal-source-crypto-kernel.py b/scripts/seal-source-crypto-kernel.py new file mode 100755 index 0000000..61ab3ee --- /dev/null +++ b/scripts/seal-source-crypto-kernel.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +"""Fill the source provider's normal POST HMAC field after final kernel link. + +Implements the source fipspost_get_hmac/hmacfiletool build contract: HMAC-SHA256 +with a one-byte zero key over the first __TEXT[ _EXEC ]/__text section. +No selftest or integrity check is disabled. Only locally compiled input is used. +""" +import argparse +import hashlib +import hmac +import json +from pathlib import Path +import struct + +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument('kernel', type=Path) +args = parser.parse_args() +data = bytearray(args.kernel.read_bytes()) +if struct.unpack_from('<4I', data) != (0xfeedfacf, 0x100000c, 0, 2): + raise SystemExit('Expected a source-built ordinary ARM64 MH_EXECUTE kernel') +if b'_source_corecrypto14_bootstrap\0' not in data: + raise SystemExit('Input does not contain the explicit source-provider bootstrap') +offset = 32 +code = None +integrity = None +segments = [] +for _ in range(struct.unpack_from('> 26 != 0x3c or + bytes.fromhex(u['instruction_bytes'])[:4] != bytes.fromhex('000020d4')): + raise SystemExit('Require cross-qualified actual EL0 BRK exception/code/PC/syndrome') + +source = ROOT / 'refer/dyld' +if subprocess.check_output(['git', '-C', source, 'rev-parse', 'HEAD'], text=True).strip() != 'a571176e8e00c47e95b95e3156820ebec0cbd5e6': + raise SystemExit('Unexpected pinned cache-format reference') +work = ROOT / 'build/native-user-exception' +work.mkdir(parents=True, exist_ok=True) +cpp = work / 'offsets.cpp' +cpp.write_text('''#include +#include +#include "dyld_cache_format.h" +int main(){printf("%zu %zu %zu %zu %zu %zu %zu %zu %zu %zu\\n", +offsetof(dyld_cache_header,imagesTextOffset),offsetof(dyld_cache_header,localSymbolsOffset), +offsetof(dyld_cache_header,sharedRegionStart),sizeof(dyld_cache_image_text_info), +sizeof(dyld_cache_local_symbols_info),sizeof(dyld_cache_local_symbols_entry_64), +sizeof(dyld_cache_local_symbols_entry),offsetof(dyld_cache_header,symbolFileUUID), +offsetof(dyld_cache_header,mappingWithSlideOffset),sizeof(dyld_cache_mapping_and_slide_info));} +''') +subprocess.run(['xcrun', 'clang++', '-std=c++17', '-I', source / 'cache-builder', cpp, '-o', work / 'offsets'], check=True) +textoff, localoff, regionoff, textsize, infosize, entrysize, _, uuidfield, slidefield, slidesize = map( + int, subprocess.check_output([work / 'offsets'], text=True).split()) +if (textsize, infosize, entrysize) != (32, 24, 16): + raise SystemExit('Unexpected original cache ABI') +cache = ROOT / '.cache/firmware/target-volumes/systemos-mount/System/Library/Caches/com.apple.dyld' +needle = bytes.fromhex(u['instruction_bytes']) +hits = [] +catalog = [] +for path in sorted(cache.glob('dyld_shared_cache_arm64*')): + if path.suffix == '.symbols': + continue + with path.open('rb') as stream, mmap.mmap(stream.fileno(), 0, access=mmap.ACCESS_READ) as b: + if b[:16].rstrip(b'\0') != b'dyld_v1 arm64': + raise SystemExit('Unexpected original cache architecture') + mo, mc = struct.unpack_from('<2I', b, 16) + mappings = [struct.unpack_from('<3Q2I', b, mo + i * 32) for i in range(mc)] + so, sc = struct.unpack_from('<2I', b, slidefield) + for i in range(sc): + catalog.append((path, struct.unpack_from('<7Q2I', b, so + i * slidesize))) + pos = b.find(needle) + while pos >= 0: + for va, size, off, _, prot in mappings: + if off <= pos and pos + len(needle) <= off + size and prot & 4: + hits.append(dict(file=path.name, offset=pos, unslid_pc=va + pos - off)) + pos = b.find(needle, pos + 1) +if len(hits) != 1: + raise SystemExit('Actual instruction bytes must uniquely identify an executable cache mapping') +hit = hits[0] +slide = int(u['user_pc'], 16) - hit['unslid_pc'] +b = (cache / 'dyld_shared_cache_arm64').read_bytes() +region, maximum = struct.unpack_from('<2Q', b, regionoff)[0], struct.unpack_from(' nc: + raise SystemExit('Symbol entry exceeds the actual local table') + for i in range(start, start + count): + string, kind, section, _, value = struct.unpack_from(' best[0]): + best = value, string + if best: + value, string = best + if string >= ss: + raise SystemExit('Invalid original symbol string offset') + pos = lo + so + string + row.update(nearest_local_symbol=symbols[pos:symbols.find(b'\0', pos)].decode(), offset=va - value) + rows.append(row) +result = dict(kernel_sha256=d['kernel_sha256'], actual_exception='EXC_BREAKPOINT', actual_codes=list(code), + unique_instruction_match=hit, cache_slide=hex(slide), frames=rows, + local_symbols_uuid_matches=True, functional_ipados_ui_demonstrated=False, + original_files_modified=False) + +def read_original(va, length): + for path, mapping in catalog: + start, size, off = mapping[:3] + if start <= va and va + length <= start + size: + with path.open('rb') as stream: + stream.seek(off + va - start) + return stream.read(length), path, mapping + raise SystemExit('Requested static address is not in an original cache mapping') + +# Qualify the literal referenced by this observed original failure block; +# this is static disassembly evidence, not a captured runtime log message. +if rows[0].get('nearest_local_symbol') == '-[FBSDisplayMonitor _initWithBookendObserver:transformer:]': + address = hit['unslid_pc'] - 0x48 + words = struct.unpack('<2I', read_original(address, 8)[0]) + adrp, add = words + if adrp & 0x9f00001f != 0x90000002 or add & 0xffc003ff != 0x91000042: + raise SystemExit('Original assertion reference instructions differ') + immediate = ((adrp >> 29) & 3) | (((adrp >> 5) & 0x7ffff) << 2) + if immediate & (1 << 20): + immediate -= 1 << 21 + literal = (address & ~4095) + (immediate << 12) + ((add >> 10) & 4095) + raw, path, mapping = read_original(literal, 32) + pointer, length = struct.unpack_from('<2Q', raw, 16) + with path.open('rb') as stream: + stream.seek(mapping[3]) + info = stream.read(40) + if struct.unpack_from(' DarwinBoot -> ExitBootServices -> synthetic Mach-O.""" +import argparse +from pathlib import Path +import shutil +import subprocess + +ROOT = Path(__file__).resolve().parents[1] +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument('--memory-size', type=lambda value: int(value, 0), choices=[0x10000000, 0x20000000]) +parser.add_argument('--packed-ramdisk', action='store_true', help='Exercise real packed seed expansion before firmware exit') +parser.add_argument('--gop', action='store_true', help='Exercise the actual QEMU ramfb UEFI GOP handoff') +parser.add_argument('--gop-high-resolution', action='store_true', help='Exercise real QEMU ramfb 3048x2032 allocation and mode programming') +parser.add_argument('--ramfb-geometry-refusal', action='store_true', help='Load the geometry fixture without a GOP and require a clean refusal') +args = parser.parse_args() +if args.ramfb_geometry_refusal and (args.gop or args.gop_high_resolution): + parser.error('The refusal control requires no GOP device') +if args.gop_high_resolution: + args.gop = True +OUTPUT = ROOT / ('build/darwinboot-test' if not args.memory_size else f'build/darwinboot-test-{args.memory_size >> 20}m') +if args.gop: + OUTPUT = OUTPUT.with_name(OUTPUT.name + '-gop') +if args.packed_ramdisk: + OUTPUT = OUTPUT.with_name(OUTPUT.name + '-packed') +if args.gop_high_resolution: + OUTPUT = OUTPUT.with_name(OUTPUT.name + '-3048x2032') +if args.ramfb_geometry_refusal: + OUTPUT = OUTPUT.with_name(OUTPUT.name + '-geometry-refusal') +ESP = OUTPUT / 'esp' +PAYLOADS = ESP / 'EFI/CrabApple/Payloads' + +OUTPUT.mkdir(parents=True, exist_ok=True) +PAYLOADS.mkdir(parents=True, exist_ok=True) +subprocess.run(['python3', ROOT / 'drivers/DarwinBoot/Test/make-fixtures.py', PAYLOADS, + *(['--memory-size', str(args.memory_size)] if args.memory_size else []), + *(['--gop'] if args.gop else []), + *(['--gop-width', '3048', '--gop-height', '2032'] if args.gop_high_resolution else [])], check=True) +if args.packed_ramdisk: + import sys + sys.path.insert(0, str(ROOT / 'drivers/DarwinBoot/Test')) + from packed_ramdisk import pack + raw = PAYLOADS / 'ramdisk.bin' + raw.write_bytes(raw.read_bytes() + bytes(-raw.stat().st_size % 4096)) + packed = PAYLOADS / 'ramdisk.packed' + packed.unlink(missing_ok=True) + pack(raw, packed) + packed.replace(raw) +(ESP / 'EFI/BOOT').mkdir(exist_ok=True) +shutil.copyfile(ROOT / 'crabapple/Build/Out/AARCH64-DEBUG/CrabApple.efi', ESP / 'EFI/BOOT/BOOTAA64.EFI') +for name in ('DarwinBoot', 'DarwinBootValidation'): + shutil.copyfile(ROOT / f'build/DarwinBoot/DEBUG_CLANGPDB/AARCH64/{name}.efi', PAYLOADS / f'{name}.efi') +(ESP / 'EFI/CrabApple/crabapple.toml').write_text('''version = 1 +[boot] +timeout = 1 +default = "synthetic" +[kits] +autoload = false +[[driver]] +path = "Payloads/DarwinBoot.efi" +[[driver]] +path = "Payloads/DarwinBootValidation.efi" +[[entry]] +id = "synthetic" +title = "Synthetic Darwin ABI entry" +type = "xnu" +path = "Payloads/kernel.macho" +args = "darwinboot.fixture=1" +initrd = "Payloads/ramdisk.bin" +''') +if args.gop_high_resolution or args.ramfb_geometry_refusal: + driver = ROOT / 'build/DarwinBoot/DEBUG_CLANGPDB/AARCH64/QemuRamfbGeometry.efi' + shutil.copyfile(driver, PAYLOADS / driver.name) + config = ESP / 'EFI/CrabApple/crabapple.toml' + config.write_text(config.read_text().replace('[[entry]]', + '[[driver]]\npath = "Payloads/QemuRamfbGeometry.efi"\n[[entry]]', 1)) +subprocess.run(['python3', ROOT / 'crabapple/Scripts/qemu-run.py', ESP, + '--log', OUTPUT / 'qemu.log', '--cpus', '1', *([] if args.gop else ['--no-gop']), '--timeout', '25', + '--finish-on-expect', + '--expect', 'DARWINBOOT: loader v2 installed EL1', + '--expect', 'DARWINBOOT-VALIDATION: 33 rejected requests; firmware retained PASS', + '--expect', 'DARWINBOOT-VIDEO-VALIDATION: 8 malformed/unsupported GOP field sets refused PASS', + '--expect', 'DARWINBOOT: inputs validated; exiting boot services', + *(['--expect', 'QEMU-RAMFB-GEOMETRY: actual SetMode 3048x2032'] if args.gop_high_resolution else []), + *(['--expect', 'QEMU-RAMFB-GEOMETRY: REFUSED no GOP'] if args.ramfb_geometry_refusal else []), + '--expect', ('DARWINBOOT-FIXTURE: actual GOP boot video after ExitBootServices PASS' if args.gop else + 'DARWINBOOT-FIXTURE: boot_args, RAMDisk, TrustCache, zero-fill and MMU-off entry PASS'), + '--reject', r'DARWINBOOT[^\n]*FAIL', '--reject', 'ASSERT'], check=True)