okooooooo/UniXNU
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(gpu): import the native Adreno and Metal stack

Browse Source 
Bring the current NativeAdreno, NativeMetal, AdrenoFirmware, AdrenoPM4,
AirNir and ArmSmmu4K sources, the Mesa pipeline-cache and builtin-shader
patches, and the Turnip host cache tools.
This commit is contained in:
rpone committed 2026-10-20 18:08:40 +08:00
1 parent cb6dcea743
commit 6c56ecec19
64 files changed
+6586 -286
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

drivers/NativeAdreno/NativeAdrenoGpuContext.cpp
+15 -1
View File
@@ -3,6 +3,8 @@
#include "ArmSmmu4KMmio.h"
#include <IOKit/IOLib.h>
extern "C" volatile uint32_t native_adreno_pool_mib;
extern "C" volatile uint32_t native_adreno_pool_stats[8],native_adreno_pool_status;
static const uint64_t kIovas[GpuBufferCount]={GpuSqeIova,GpuRingIova,GpuMemptrsIova,GpuPwrupIova,GpuScratchIova};
static const size_t kBytes[GpuBufferCount]={GpuSqeBytes,GpuRingBytes,GpuPageBytes,GpuPageBytes,GpuPageBytes};
IOReturn NativeAdrenoGpuContext::bringUp(const ArmSmmu4KIo *io,unsigned bank,unsigned asid,const unsigned slots[2],
@@ -25,6 +27,15 @@ IOReturn NativeAdrenoGpuContext::bringUp(const ArmSmmu4KIo *io,unsigned bank,uns
if(report->output_bits>40)report->output_bits=40;
if(!report->output_bits)status=kIOReturnUnsupported;
}
/* The BO pool is taken first, while the VM still has long contiguous runs (the board sets the size from sheng-gpu-pool; 0 = scattered BOs only).
* A pool that cannot be had at one size is tried at the next smaller; none is not an error. */
if(!status && native_adreno_pool_mib) {
for(uint32_t mib=native_adreno_pool_mib;mib>=16;mib>>=1) {
const IOReturn pooled=NativeAdrenoMemory::reservePool(report->output_bits,(size_t)mib<<20);
native_adreno_pool_stats[0]=pooled==kIOReturnSuccess?mib:0;native_adreno_pool_status=(uint32_t)pooled;
if(pooled==kIOReturnSuccess)break;
}
}
if(!status)status=tables_.allocate(TablePages*0x1000,report->output_bits,true,TablePages*0x1000);
for(unsigned i=0;!status && i<GpuBufferCount;++i)
status=buffers_[i].allocate(kBytes[i],report->output_bits,true,kBytes[i]);
@@ -32,6 +43,7 @@ IOReturn NativeAdrenoGpuContext::bringUp(const ArmSmmu4KIo *io,unsigned bank,uns
step(Prepare);
status=smmu_.prepare(&tables_,0,0,GpuApertureBytes,NativeAdrenoSmmuContext::MaxLeaves,
bank,asid,1000,report->output_bits);
if(!status)smmu_.setAliasScratch(alias_scratch_,sizeof alias_scratch_/sizeof *alias_scratch_);
}
if(!status) {
step(Program);
@@ -53,7 +65,9 @@ IOReturn NativeAdrenoGpuContext::bringUp(const ArmSmmu4KIo *io,unsigned bank,uns
if(!status) {
step(VmAttach);
NativeAdrenoVmRegion lower={&tables_,smmu_.arenaHandle(),smmu_.stateHandle()};
status=vm_.attach(io,smmu_.configHandle(),smmu_.streamsHandle(),&lower,nullptr,16384,262144);
/* 65536 spans: 256 MiB of BOs even when every page is its own span. The Metal runtime pools its memory in 64 MiB blocks,
* which XNU rarely finds physically contiguous; one scattered block is 16384 spans (int-070 ran out at 16384). */
status=vm_.attach(io,smmu_.configHandle(),smmu_.streamsHandle(),&lower,nullptr,131072,262144);
}
if(!status) {
step(Map);
drivers/NativeAdreno/NativeAdrenoGpuContext.h
+2
View File
@@ -24,6 +24,8 @@ class NativeAdrenoGpuContext final {
uint64_t handles_[GpuBufferCount]{};
NativeAdrenoSmmuContext smmu_;
NativeAdrenoVm vm_;
/* The alias checks' sort scratch: one entry per range of the smaller compared side, at most half the aperture's leaves. */
uint32_t alias_scratch_[NativeAdrenoSmmuContext::MaxLeaves/2];
bool ready_=false,used_=false;
public:
enum Step : uint32_t {Allocate=1,Prepare,Program,Attach,VmAttach,Map,Ready};
drivers/NativeAdreno/NativeAdrenoRing.cpp
+11
View File
@@ -64,6 +64,9 @@ static uint32_t ilog2(size_t value)
{
uint32_t result=0;while(value>1) {value>>=1;++result;}return result;
}
/* The last submissions as the CP will read them: [slot] fence, start index, end index, word count, then up to 112 ring words from the start. The board log
* prints them, so a fence the GPU never wrote can be compared with the packets that should have written it. */
extern "C" {volatile uint32_t native_adreno_ring_log[4][4+112];volatile uint32_t native_adreno_ring_log_count;}
IOReturn NativeAdrenoRing::poison()
{
poisoned_=true;return kIOReturnIOError;
@@ -229,6 +232,14 @@ IOReturn NativeAdrenoRing::submitWords(const uint32_t *words,size_t word_count,u
if(status)return status;
/* Whole-ring publication is deliberate: a submission may wrap, and one
* clean of the owned pool is cheaper than a split range proof. */
{
const uint32_t slot=native_adreno_ring_log_count&3;
const uint32_t total=(end-next_)&mask;
volatile uint32_t *record=native_adreno_ring_log[slot];
record[0]=seqno_+1;record[1]=next_;record[2]=end;record[3]=total;
for(uint32_t i=0;i<112;++i)record[4+i]=i<total?ring_[(next_+i)&mask]:0;
native_adreno_ring_log_count=native_adreno_ring_log_count+1;
}
if(cache_.publish(cache_.cookie,ring_,ring_bytes_))return poison();
if(io_.barrier(io_.cookie))return poison();
if(io_.write32(io_.cookie,RingWptr,end))return poison();
drivers/NativeAdreno/NativeAdrenoRing.h
+1
View File
@@ -93,6 +93,7 @@ public:
uint32_t timeout_us,uint32_t *observed);
bool poisoned() const {return poisoned_;}
uint32_t outstanding() const {return seqno_-completed_;}
uint32_t lastSubmitted() const {return seqno_;} /* the newest fence number handed out */
uint32_t completedFence() const {return completed_;}
uint32_t submissions() const {return submissions_;}
uint32_t readPointer() const {return rptr_shadow_;}
drivers/NativeAdreno/NativeAdrenoService.cpp
+10
View File
@@ -46,6 +46,8 @@ extern "C" const uint64_t native_adreno_firmware_image_size __attribute__((weak)
extern "C" const uint8_t native_adreno_sqe_firmware_image[] __attribute__((weak));
extern "C" const uint64_t native_adreno_sqe_firmware_image_size __attribute__((weak));
#include "NativeAdrenoUserClient.h"
/* Physical address of the fence page (the CP writes its timestamps there): the board log reads it from DRAM and through the CPU cache. */
extern "C" {volatile uint64_t native_adreno_fence_phys=0;}
class NativeAdrenoGpuClient;
static IOUserClient *createGpuClient(task_t,void *,UInt32,OSDictionary *,NativeAdrenoUserClient *);
@@ -181,8 +183,12 @@ uint64_t NativeAdrenoGpuService::ring_now(void *)
NativeAdrenoKernelTime::now_us(nullptr,&microseconds);
return microseconds;
}
extern "C" uint64_t (*native_adreno_vm_now_us)(void);
extern "C" volatile uint32_t native_adreno_pool_mib;
static uint64_t vm_clock_us(void) {uint64_t microseconds=0;NativeAdrenoKernelTime::now_us(nullptr,&microseconds);return microseconds;}
IOService *NativeAdrenoGpuService::probe(IOService *provider,SInt32 *score)
{
native_adreno_vm_now_us=vm_clock_us;
if(!advertisesAdreno(provider))return nullptr;
if(score)*score=1000;
return this;
@@ -289,6 +295,9 @@ bool NativeAdrenoGpuService::start(IOService *provider)
* bank, attaches the stream and maps the GMU payloads. Do not run it before the read-only power/clock observation
* (sheng-gpu-power, docs/gpu-hardware-plan.md) shows the GPU SMMU powered and writable: the apps SMMU on this board is
* hypervisor-managed (docs/native-ufs.md) and the adreno SMMU has not been shown to differ. */
{
uint32_t mib=384;PE_parse_boot_argn("sheng-gpu-pool",&mib,sizeof(mib));native_adreno_pool_mib=mib;
}
uint32_t bringup=0;
if(PE_parse_boot_argn("sheng-gpu-bringup",&bringup,sizeof(bringup)) && bringup) {
NativeAdrenoSmmuService *node=OSDynamicCast(NativeAdrenoSmmuService,smmu);
@@ -414,6 +423,7 @@ bool NativeAdrenoGpuService::start(IOService *provider)
ring_status=sqe_mem_.allocate(size_t(sqe_bytes),40,true,size_t(sqe_bytes));
if(!ring_status)ring_status=vm_.map(&ring_mem_,1,ring_iova,3,0,2,&ring_map_);
if(!ring_status)ring_status=vm_.map(&fence_mem_,1,fence_iova,3,0,2,&fence_map_);
if(!ring_status && fence_mem_.spanCount())native_adreno_fence_phys=fence_mem_.spans()[0].physical;
if(!ring_status)ring_status=vm_.map(&sqe_mem_,1,sqe_iova,3,0,2,&sqe_map_);
if(ring_status) {
IOLog("NativeAdreno: ring/SQE setup stopped at 0x%x\n",ring_status);
drivers/NativeAdreno/NativeAdrenoSmmuContext.h
+3
View File
@@ -52,6 +52,9 @@ public:
const ArmSmmu4KLiveArena *arena() const {return &arena_;}
/* Handles for a NativeAdrenoVm that will map buffers through this bank. */
ArmSmmu4KLiveArena *arenaHandle() {return &arena_;}
/* Scratch the kit's physical-alias checks sort in (ArmSmmu4KLiveArena::alias_scratch), owned by the caller for this context's
* lifetime; without it those checks compare pairwise. Only after prepare. */
void setAliasScratch(uint32_t *entries,size_t count) {arena_.alias_scratch=entries;arena_.alias_scratch_entries=entries?count:0;}
ArmSmmu4KLiveState *stateHandle() {return &state_;}
ArmSmmu4KContextConfig *configHandle() {return &config_;}
ArmSmmu4KStreamState *streamsHandle() {return &streams_;}
drivers/NativeAdreno/NativeAdrenoSmmuMemory.cpp
+1 -1
View File
@@ -29,7 +29,7 @@ IOReturn NativeAdrenoPrepareSmmuMemory(NativeAdrenoMemory *memory,unsigned input
memory->spans()[0].offset || memory->spans()[0].bytes!=memory->bytes() ||
!memory->bytes() || (memory->bytes()&4095))return kIOReturnUnsupported;
*arena={{input_bits,output_bits,memory->spans()[0].physical,budget},iova,bytes,upper,
static_cast<volatile uint64_t *>(memory->mapping()),memory->bytes()/4096};
static_cast<volatile uint64_t *>(memory->mapping()),memory->bytes()/4096,nullptr,0};
ArmSmmu4KStatus status=ArmSmmu4KLivePrepare(arena,state);
if(status!=ArmSmmu4KSuccess)return status==ArmSmmu4KCapacity?kIOReturnNoResources:kIOReturnBadArgument;
ArmSmmu4KLiveResult audit;
drivers/NativeAdreno/NativeAdrenoSubmit.cpp
+75 -7
View File
@@ -7,8 +7,20 @@
* and IB_SIZE in dwords (bits 0..19), exactly as the pinned adreno_pm4.xml
* domain states and Linux msm_gem_submit.c converts the userspace byte size. */
enum : uint32_t {SubmitIndirectBuffer=0x3f,SubmitIbWords=3,SubmitMaxJobs=8,
SubmitMaxBuffers=16,SubmitMaxIbBytes=0xfffffu*4u};
SubmitMaxBuffers=16,SubmitMaxIbBytes=0xfffffu*4u,
SubmitAuditBytes=256u*1024u}; /* IB bytes the write-combine audit compares per submission */
extern "C" {volatile uint32_t native_adreno_coherence[8];}
/* The submission (1-based count) at which the write-combine audit found a dirty IB line or failed; 0 = never. */
extern "C" {volatile uint32_t native_adreno_wc_distrusted;}
/* The write-combine audit before each submission while trusted: [0] submissions audited, [1] IB lines compared, [2] dirty lines,
* [3] stale clean lines, [4] the last audit failure (any failure also ends the trust), [5] pages found back at the default attribute
* and write-combined again. */
extern "C" {volatile uint32_t native_adreno_wc_audit[6];}
/* Fence waits: [0] calls, [1] timeouts, [2] the fence of the last timed-out wait, [3] the fence value the GPU had written then, [4] the newest fence handed out,
* [5] the newest completed, [6] waits for a fence no job holds (observeFence path), [7] the last fence refused as never submitted. */
extern "C" {volatile uint32_t native_adreno_fence_wait[8];}
/* The IBs of the last submissions: [slot] fence, iova low/high, bytes, then the first 28 dwords of the first IB (read as the kernel maps it). */
extern "C" {volatile uint32_t native_adreno_ib_log[4][4+28];volatile uint32_t native_adreno_ib_log_count;}
IOReturn NativeAdrenoSubmit::poison()
{
poisoned_=true;return kIOReturnIOError;
@@ -38,6 +50,7 @@ IOReturn NativeAdrenoSubmit::release(Job &job)
if(retired)status=retired;else ++stats_.references_retired;
}
if(status)return poison();
if(vm_->acquireImported())return poison();
job={};return kIOReturnSuccess;
}
IOReturn NativeAdrenoSubmit::detach()
@@ -100,6 +113,40 @@ IOReturn NativeAdrenoSubmit::submit(const NativeAdrenoSubmitBuffer *buffers,size
/* int-070: the first IB word as the kernel's cacheable alias and DRAM see it before any maintenance, then DRAM after the CPU
* caches over every mapped BO are cleaned and invalidated. The board shows the last submission's record on a stall. */
uint32_t cached=~0u,before=~0u,ignored=0,after=~0u;
/* Write-combined stores this CPU made before the call (the client's command words) complete before anything below. */
__asm__ volatile("dsb sy":::"memory");
/* int-098/099: with write-combined user BOs and no maintenance the CP stopped at the end of the first IB, as in int-065..069 when an
* IB sat in a CPU cache. While write-combined BOs are trusted, every line of this submission's IBs is compared as a CPU cache and
* DRAM hold it (bounded). A dirty line means a client view still caches its BOs: maintain every BO from now on, this submission
* included (the audit already wrote the IB lines back; the rest of the BOs may hold more). */
if(vm_->writeCombinedTrusted()) {
/* int-100: the dirty IB page was back at XNU's default (cacheable) attribute. Write-combine every such page again first; the
* pmap writes its lines back on the way, so the audit below should then find none. */
uint32_t reset=0;
IOReturn audited=vm_->reassertWriteCombined(&reset);
native_adreno_wc_audit[5]=native_adreno_wc_audit[5]+reset;
uint32_t lines=0,dirty=0,stale=0;size_t budget=SubmitAuditBytes;
/* The re-assertion above is the correction and runs every time; the audit only detects, and found no dirty line in int-141's
* 133798, so after the first 16 submissions it compares every eighth one's IBs (it cost about 2 ms each). */
++audit_tick_;
const bool audit_now=audit_tick_<=16 || (audit_tick_&7)==1;
if(!audit_now)budget=0;
for(size_t i=0;i<count && budget && !audited;++i) { /* a failed re-assertion skips the audit and ends the trust */
const size_t bytes=buffers[i].bytes<budget?(size_t)buffers[i].bytes:budget;
audited=vm_->auditRange(buffers[i].iova,bytes,&lines,&dirty,&stale);budget-=bytes;
}
if(audit_now && audited!=kIOReturnUnsupported) { /* no board hook (host): nothing to compare */
native_adreno_wc_audit[0]=native_adreno_wc_audit[0]+1;
native_adreno_wc_audit[1]=native_adreno_wc_audit[1]+lines;
native_adreno_wc_audit[2]=native_adreno_wc_audit[2]+dirty;
native_adreno_wc_audit[3]=native_adreno_wc_audit[3]+stale;
if(audited)native_adreno_wc_audit[4]=audited;
}
if(dirty || (audited && audited!=kIOReturnUnsupported)) {
vm_->distrustWriteCombined();
native_adreno_wc_distrusted=native_adreno_coherence[0]+1;
}
}
(void)vm_->sampleWord(buffers[0].iova,&cached,&before);
IOReturn status=vm_->publishMapped();
if(!status) {
@@ -118,6 +165,14 @@ IOReturn NativeAdrenoSubmit::submit(const NativeAdrenoSubmitBuffer *buffers,size
for(size_t i=0;i<handle_count;++i) {vm_->retireJobAfterCompletion(handles[i]);++stats_.references_retired;}
return status;
}
{
const uint32_t slot=native_adreno_ib_log_count&3;uint32_t head[28];
(void)vm_->peekWords(buffers[0].iova,head,28);
native_adreno_ib_log[slot][0]=ring_->lastSubmitted()+1;native_adreno_ib_log[slot][1]=(uint32_t)buffers[0].iova;
native_adreno_ib_log[slot][2]=(uint32_t)(buffers[0].iova>>32);native_adreno_ib_log[slot][3]=(uint32_t)buffers[0].bytes;
for(unsigned i=0;i<28;++i)native_adreno_ib_log[slot][4+i]=head[i];
native_adreno_ib_log_count=native_adreno_ib_log_count+1;
}
status=ring_->submitFramed(words,word_count,&value);
if(status) {
/* A poisoned ring may already have consumed the WPTR write, so the
@@ -154,17 +209,30 @@ IOReturn NativeAdrenoSubmit::wait(uint32_t fence,const NativeAdrenoRingClock *cl
{
if(!attached_ || poisoned_)return kIOReturnNotReady;
Job *job=find(fence);
uint32_t observed=0;
native_adreno_fence_wait[0]=native_adreno_fence_wait[0]+1;
if(!job) {
const NativeAdrenoRingFence passed=ring_->observeFence(fence,clock,timeout_us,nullptr);
native_adreno_fence_wait[6]=native_adreno_fence_wait[6]+1;
const NativeAdrenoRingFence passed=ring_->observeFence(fence,clock,timeout_us,&observed);
if(passed==NativeAdrenoRingFencePoisoned)return poison();
if(passed==NativeAdrenoRingFenceArgument)return kIOReturnBadArgument;
if(passed==NativeAdrenoRingFenceTimeout) {++stats_.timeouts;return kIOReturnTimeout;}
if(passed==NativeAdrenoRingFenceArgument) {native_adreno_fence_wait[7]=fence;return kIOReturnBadArgument;}
if(passed==NativeAdrenoRingFenceTimeout) {
++stats_.timeouts;
native_adreno_fence_wait[1]=native_adreno_fence_wait[1]+1;native_adreno_fence_wait[2]=fence;native_adreno_fence_wait[3]=observed;
native_adreno_fence_wait[4]=ring_->lastSubmitted();native_adreno_fence_wait[5]=ring_->completedFence();
return kIOReturnTimeout;
}
return kIOReturnSuccess;
}
const NativeAdrenoRingFence result=ring_->wait(fence,clock,timeout_us,nullptr);
const NativeAdrenoRingFence result=ring_->wait(fence,clock,timeout_us,&observed);
if(result==NativeAdrenoRingFencePoisoned)return poison();
if(result==NativeAdrenoRingFenceArgument)return kIOReturnBadArgument;
if(result==NativeAdrenoRingFenceTimeout) {++stats_.timeouts;return kIOReturnTimeout;}
if(result==NativeAdrenoRingFenceArgument) {native_adreno_fence_wait[7]=fence;return kIOReturnBadArgument;}
if(result==NativeAdrenoRingFenceTimeout) {
++stats_.timeouts;
native_adreno_fence_wait[1]=native_adreno_fence_wait[1]+1;native_adreno_fence_wait[2]=fence;native_adreno_fence_wait[3]=observed;
native_adreno_fence_wait[4]=ring_->lastSubmitted();native_adreno_fence_wait[5]=ring_->completedFence();
return kIOReturnTimeout;
}
IOReturn status=release(*job);
if(status)return status;
++stats_.completed;
drivers/NativeAdreno/NativeAdrenoSubmit.h
+4
View File
@@ -13,6 +13,9 @@ struct NativeAdrenoSubmitBuffer {
* through DRAM before and through DRAM after the CPU cache maintenance of every mapped BO (~0 when unreadable); [4] samples whose
* three values differ and [5..7] the first such sample. */
extern "C" volatile uint32_t native_adreno_coherence[8];
/* The write-combine audit (int-100), see NativeAdrenoSubmit.cpp, and the submission (1-based) that ended the trust; 0 = never. */
extern "C" volatile uint32_t native_adreno_wc_audit[6];
extern "C" volatile uint32_t native_adreno_wc_distrusted;
struct NativeAdrenoSubmitStats {
uint32_t submissions,completed,timeouts,poisoned_jobs;
size_t references_held,references_retired,peak_jobs;
@@ -38,6 +41,7 @@ class NativeAdrenoSubmit final {
NativeAdrenoRing *ring_=nullptr;
NativeAdrenoSubmitStats stats_{};
bool attached_=false,poisoned_=false;
uint32_t audit_tick_=0; /* submissions since attach; every eighth one audits its IB lines */
Job *find(uint32_t fence);
Job *free();
IOReturn release(Job &);
drivers/NativeAdreno/NativeAdrenoVm.cpp
+215 -41
View File
@@ -1,11 +1,24 @@
/* SPDX-License-Identifier: MIT */
#include "NativeAdrenoVm.h"
#include "NativeAdrenoVmAlias.h"
extern "C" volatile uint32_t native_adreno_trace_stage;
extern "C" {int (*native_adreno_dram_read32)(uint64_t,uint32_t *);}
extern "C" {int (*native_adreno_dram_read32)(uint64_t,uint32_t *);int (*native_adreno_cached_read32)(uint64_t,uint32_t *);
int (*native_adreno_line_audit)(uint64_t,size_t,uint32_t *,uint32_t *);}
#include <string.h>
#include <IOKit/IOLib.h>
/* 1 once the board proved the GPU's cacheable inner-shareable accesses coherent with the CPU's caches in both directions (the A740's
* ADRENO_QUIRK_HAS_CACHED_COHERENT behind the dma-coherent GPU SMMU, which Linux maps with IOMMU_CACHE): a mapping with write-back
* attributes then needs no CPU cache maintenance. 0 keeps every mapping maintained as before. Counters: maintenance passes skipped for
* coherent mappings at map, unmap, submission and completion. */
extern "C" {volatile uint32_t native_adreno_io_coherent;volatile uint32_t native_adreno_coherent_skips[4];}
static bool coherentMapping(unsigned writeback) {return writeback && native_adreno_io_coherent;}
static bool overlap(uint64_t a,uint64_t ab,uint64_t b,uint64_t bb)
{return a<b+bb && b<a+ab;}
/* Mappings of one cache/shareability/privilege class may share pages; only pages shared across classes refuse. */
static unsigned aliasClass(unsigned permissions,unsigned writeback,unsigned shareability)
{return writeback|(shareability<<1)|((permissions&ArmSmmu4KPrivileged)?8u:0u);}
static void reverse(ArmSmmu4KRange *first,ArmSmmu4KRange *last)
{while(first<last && first<--last) {const ArmSmmu4KRange t=*first;*first++=*last;*last=t;}}
static bool validBinding(const NativeAdrenoVmRegion &r,unsigned upper)
{
return r.tables && r.arena && r.state && !r.tables->quarantined() &&
@@ -21,6 +34,24 @@ ArmSmmu4KLiveRegion NativeAdrenoVm::region(unsigned upper) const
return {regions_[upper].arena,regions_[upper].state,
ranges_+(upper?lower_count_:0),upper?upper_count_:lower_count_};
}
/* Board hook and per-phase time of the two table transactions, for the log: [0] map, [1] unmap; each row is count, the phases in order, the slowest call
* and the bytes mapped (int-134: BoMap averaged 135 ms and the desktop crawled). */
extern "C" {uint64_t (*native_adreno_vm_now_us)(void)=nullptr;volatile uint64_t native_adreno_vm_phase_us[2][8];}
namespace {
struct PhaseClock {
uint64_t last=native_adreno_vm_now_us?native_adreno_vm_now_us():0,began=last;
void lap(unsigned row,unsigned slot) {
if(!native_adreno_vm_now_us)return;
const uint64_t now=native_adreno_vm_now_us();native_adreno_vm_phase_us[row][slot]+=now-last;last=now;
}
void done(unsigned row,uint64_t bytes) {
if(!native_adreno_vm_now_us)return;
const uint64_t total=native_adreno_vm_now_us()-began;
native_adreno_vm_phase_us[row][0]+=1;native_adreno_vm_phase_us[row][7]+=bytes;
if(total>native_adreno_vm_phase_us[row][6])native_adreno_vm_phase_us[row][6]=total;
}
};
}
ArmSmmu4KStatus NativeAdrenoVm::observe() const
{
ArmSmmu4KStreamResult routes;
@@ -30,6 +61,16 @@ ArmSmmu4KStatus NativeAdrenoVm::observe() const
if(status)return status;
return ArmSmmu4KObserveStreams(&io_,streams_,context_,&routes);
}
ArmSmmu4KStatus NativeAdrenoVm::observeStreamsOnly() const
{
ArmSmmu4KStreamResult routes;
return ArmSmmu4KObserveStreams(&io_,streams_,context_,&routes);
}
ArmSmmu4KStatus NativeAdrenoVm::observeCheap()
{
if(++since_audit_>=64) {since_audit_=0;return observe();}
return observeStreamsOnly();
}
IOReturn NativeAdrenoVm::fail(ArmSmmu4KStatus status)
{
poisoned_=true;result_.operation=status;result_.complete=false;
@@ -44,7 +85,7 @@ IOReturn NativeAdrenoVm::attach(const ArmSmmu4KIo *io,ArmSmmu4KContextConfig *co
size_t capacity,unsigned budget)
{
if(attached_ || ranges_ || poisoned_)return kIOReturnBusy;
if(!io || !context || !streams || (!lower && !upper) || !capacity || capacity>65536 ||
if(!io || !context || !streams || (!lower && !upper) || !capacity || capacity>262144 ||
!budget || budget>1048576 || (lower && !validBinding(*lower,0)) || (upper && !validBinding(*upper,1)))
return kIOReturnBadArgument;
ranges_=IONewData(ArmSmmu4KRange,capacity);if(!ranges_)return kIOReturnNoMemory;
@@ -114,50 +155,100 @@ IOReturn NativeAdrenoVm::preflight(const NativeAdrenoMemory *memory,unsigned upp
return kIOReturnBadArgument;
for(const auto &r:regions_)if(r.arena && overlap(span.physical,span.bytes,
r.arena->config.table_physical,r.arena->capacity_pages*UINT64_C(4096)))return kIOReturnBadArgument;
for(size_t i=0;i<spanCount();++i)if(overlap(span.physical,span.bytes,ranges_[i].physical,ranges_[i].bytes) &&
(writeback!=ranges_[i].writeback || shareability!=ranges_[i].shareability ||
((permissions^ranges_[i].permissions)&ArmSmmu4KPrivileged)))return kIOReturnBadArgument;
offset+=span.bytes;
}
return offset==memory->bytes()?kIOReturnSuccess:kIOReturnBadArgument;
if(offset!=memory->bytes())return kIOReturnBadArgument;
/* Only another cache/shareability/privilege class can conflict. Reuse the
* arena's caller-owned temporary alias scratch before the table transaction
* uses it. Index the smaller side and query the other in O(log n); neither
* the resource spans nor the actual page tables are modified here. */
const unsigned mine=aliasClass(permissions,writeback,shareability);
size_t other=0;
for(size_t i=0;i<spanCount();++i)
if(aliasClass(ranges_[i].permissions,ranges_[i].writeback,ranges_[i].shareability)!=mine)++other;
if(!other)return kIOReturnSuccess;
const bool indexNew=memory->spanCount()<=other;
const size_t smaller=indexNew?memory->spanCount():other;
uint32_t *order=a->alias_scratch;
if(order && smaller<=a->alias_scratch_entries) {
if(indexNew) {
for(size_t i=0;i<smaller;++i)order[i]=(uint32_t)i;
NativeAdrenoPhysicalIndex index(memory->spans(),order,smaller);
for(size_t i=0;i<spanCount();++i) {
const auto &r=ranges_[i];
if(aliasClass(r.permissions,r.writeback,r.shareability)!=mine && index.overlaps(r.physical,r.bytes))return kIOReturnBadArgument;
}
} else {
size_t n=0;
for(size_t i=0;i<spanCount();++i)
if(aliasClass(ranges_[i].permissions,ranges_[i].writeback,ranges_[i].shareability)!=mine)order[n++]=(uint32_t)i;
NativeAdrenoPhysicalIndex index(ranges_,order,n);
for(size_t j=0;j<memory->spanCount();++j)
if(index.overlaps(memory->spans()[j].physical,memory->spans()[j].bytes))return kIOReturnBadArgument;
}
return kIOReturnSuccess;
}
/* Small/unconfigured callers retain the conservative allocation-free path. */
for(size_t i=0;i<spanCount();++i) {
const ArmSmmu4KRange &r=ranges_[i];
if(aliasClass(r.permissions,r.writeback,r.shareability)==mine)continue;
for(size_t j=0;j<memory->spanCount();++j)
if(overlap(memory->spans()[j].physical,memory->spans()[j].bytes,r.physical,r.bytes))return kIOReturnBadArgument;
}
return kIOReturnSuccess;
}
NativeAdrenoVm::Mapping *NativeAdrenoVm::find(uint64_t handle)
{if(handle && mappings_)for(size_t i=0;i<NativeAdrenoVmMaxObjects;++i)if(mappings_[i].handle==handle && mappings_[i].memory)return &mappings_[i];return nullptr;}
void NativeAdrenoVm::insert(const ArmSmmu4KRange &range,unsigned upper)
/* The n ranges just past the table's end (one mapping's spans, consecutive in IOVA and all in one gap) take their sorted place. */
void NativeAdrenoVm::insert(size_t n,unsigned upper)
{
size_t at=0,count=spanCount();while(at<count && ranges_[at].iova<range.iova)++at;
for(size_t i=count;i>at;--i)ranges_[i]=ranges_[i-1];ranges_[at]=range;
if(upper)++upper_count_;else ++lower_count_;
const size_t count=spanCount();
size_t at=0;while(at<count && ranges_[at].iova<ranges_[count].iova)++at;
reverse(ranges_+at,ranges_+count);reverse(ranges_+count,ranges_+count+n);reverse(ranges_+at,ranges_+count+n);
if(upper)upper_count_+=n;else lower_count_+=n;
}
void NativeAdrenoVm::erase(size_t at,unsigned upper)
void NativeAdrenoVm::erase(size_t at,size_t n,unsigned upper)
{
size_t count=spanCount();for(size_t i=at+1;i<count;++i)ranges_[i-1]=ranges_[i];
if(upper)--upper_count_;else --lower_count_;
const size_t count=spanCount();
memmove(ranges_+at,ranges_+at+n,(count-at-n)*sizeof(*ranges_));
if(upper)upper_count_-=n;else lower_count_-=n;
}
IOReturn NativeAdrenoVm::map(NativeAdrenoMemory *memory,unsigned upper,uint64_t iova,
unsigned permissions,unsigned writeback,unsigned shareability,uint64_t *handle)
{
if(!handle)return kIOReturnBadArgument;*handle=0;result_={};
if(!attached_ || poisoned_)return kIOReturnNotReady;
PhaseClock clock;
IOReturn checked=preflight(memory,upper,iova,permissions,writeback,shareability);if(checked)return checked;
if(next_handle_==UINT64_MAX)return kIOReturnNoResources;
ArmSmmu4KStatus status=observe();if(status)return fail(status);
clock.lap(0,5); /* Previously omitted preflight; map slot 5 was unused. */
ArmSmmu4KStatus status=observeCheap();if(status)return fail(status);
clock.lap(0,1);
checked=memory->holdForDevice();if(checked)return checked;
Mapping *record=nullptr;for(size_t i=0;i<NativeAdrenoVmMaxObjects;++i)if(!mappings_[i].memory){record=&mappings_[i];break;}
if(!record) {memory->quarantine();return fail(ArmSmmu4KStateMismatch);}
*record={memory,0,iova,memory->spanCount(),0,permissions,writeback,shareability,0};++objects_;
checked=memory->publishForDevice(0,memory->bytes(),permissions&ArmSmmu4KWrite);
if(checked)return fail(ArmSmmu4KIoError);
for(size_t i=0;i<record->spans;++i) {
const auto &span=memory->spans()[i];
ArmSmmu4KRange range={iova+span.offset,span.physical,span.bytes,permissions,writeback,shareability};
ArmSmmu4KLiveRegion lower=region(0),high=region(1);ArmSmmu4KLiveResult update;
status=ArmSmmu4KLiveUpdateContext(&io_,streams_->bank,context_,lower.arena?&lower:nullptr,
high.arena?&high:nullptr,upper,&visibility_[upper],&range,1,budget_,&update);
if(status)return fail(status);
insert(range,upper);++record->mapped_spans;++result_.completed_spans;
if(coherentMapping(writeback))native_adreno_coherent_skips[0]=native_adreno_coherent_skips[0]+1;
else {
checked=memory->publishForDevice(0,memory->bytes(),permissions&ArmSmmu4KWrite,distrust_write_combined_);
if(checked)return fail(ArmSmmu4KIoError);
}
status=observe();if(status)return fail(status);
clock.lap(0,2);
/* Every span in one table transaction, staged in the table's free tail (preflight checked the room). */
const size_t spans=record->spans;
ArmSmmu4KRange *changes=ranges_+spanCount();
for(size_t i=0;i<spans;++i) {
const auto &span=memory->spans()[i];
changes[i]={iova+span.offset,span.physical,span.bytes,permissions,writeback,shareability};
}
ArmSmmu4KLiveRegion lower=region(0),high=region(1);ArmSmmu4KLiveResult update;
status=ArmSmmu4KLiveUpdateContextBatch(&io_,streams_->bank,context_,lower.arena?&lower:nullptr,
high.arena?&high:nullptr,upper,&visibility_[upper],changes,spans,1,budget_,&update);
if(status)return fail(status);
clock.lap(0,3);
insert(spans,upper);record->mapped_spans=spans;result_.completed_spans=spans;
status=observeStreamsOnly();if(status)return fail(status);
clock.lap(0,4);clock.done(0,memory->bytes());
record->handle=next_handle_++;*handle=record->handle;result_.complete=true;return kIOReturnSuccess;
}
IOReturn NativeAdrenoVm::unmap(uint64_t handle)
@@ -166,31 +257,43 @@ IOReturn NativeAdrenoVm::unmap(uint64_t handle)
result_={};if(!attached_ || poisoned_)return kIOReturnNotReady;
Mapping *record=find(handle);if(!record)return kIOReturnBadArgument;
if(record->jobs)return kIOReturnBusy;
ArmSmmu4KStatus status=observe();
PhaseClock clock;
ArmSmmu4KStatus status=observeCheap();
native_adreno_trace_stage=11;
if(status)return fail(status);
clock.lap(1,1);
unsigned upper=regions_[1].arena && record->iova>=regions_[1].arena->iova;
if(record->permissions&ArmSmmu4KWrite) {
if(coherentMapping(record->writeback))native_adreno_coherent_skips[1]=native_adreno_coherent_skips[1]+1;
else if(record->permissions&ArmSmmu4KWrite) {
if(record->memory->acquireDeviceWrites(0,record->memory->bytes())!=kIOReturnSuccess)
return fail(ArmSmmu4KIoError);
}
for(size_t i=0;i<record->spans;++i) {
clock.lap(1,2);
/* The mapping's spans are consecutive in the table; unmap them in one transaction. */
const size_t spans=record->spans;
size_t at=0;while(at<spanCount() && ranges_[at].iova!=record->iova)++at;
if(spans>spanCount()-at)return fail(ArmSmmu4KStateMismatch);
for(size_t i=0;i<spans;++i) {
const auto &span=record->memory->spans()[i];
ArmSmmu4KRange change={record->iova+span.offset,span.physical,span.bytes,record->permissions,record->writeback,record->shareability};
size_t at=0;while(at<spanCount() && ranges_[at].iova!=change.iova)++at;
if(at==spanCount())return fail(ArmSmmu4KStateMismatch);
ArmSmmu4KLiveRegion lower=region(0),high=region(1);ArmSmmu4KLiveResult update;
native_adreno_trace_stage=12;
status=ArmSmmu4KLiveUpdateContext(&io_,streams_->bank,context_,lower.arena?&lower:nullptr,
high.arena?&high:nullptr,upper,&visibility_[upper],&change,0,budget_,&update);
native_adreno_trace_stage=13;
if(status)return fail(status);
erase(at,upper);--record->mapped_spans;++result_.completed_spans;
const ArmSmmu4KRange &r=ranges_[at+i];
if(r.iova!=record->iova+span.offset || r.physical!=span.physical || r.bytes!=span.bytes ||
r.permissions!=record->permissions || r.writeback!=record->writeback || r.shareability!=record->shareability)
return fail(ArmSmmu4KStateMismatch);
}
status=observe();
ArmSmmu4KLiveRegion lower=region(0),high=region(1);ArmSmmu4KLiveResult update;
native_adreno_trace_stage=12;
status=ArmSmmu4KLiveUpdateContextBatch(&io_,streams_->bank,context_,lower.arena?&lower:nullptr,
high.arena?&high:nullptr,upper,&visibility_[upper],ranges_+at,spans,0,budget_,&update);
native_adreno_trace_stage=13;
if(status)return fail(status);
clock.lap(1,3);
erase(at,spans,upper);record->mapped_spans=0;result_.completed_spans=spans;
status=observeStreamsOnly();
native_adreno_trace_stage=14;
if(status)return fail(status);
clock.lap(1,4);
if(record->memory->retireAfterQuiescence()!=kIOReturnSuccess)return fail(ArmSmmu4KStateMismatch);
clock.lap(1,5);clock.done(1,record->memory->bytes());
*record={};--objects_;result_.complete=true;return kIOReturnSuccess;
}
IOReturn NativeAdrenoVm::describe(uint64_t handle,uint64_t *iova,size_t *bytes,
@@ -218,8 +321,20 @@ IOReturn NativeAdrenoVm::publishMapped()
if(!attached_ || poisoned_)return kIOReturnNotReady;
for(size_t i=0;i<NativeAdrenoVmMaxObjects;++i) {
const Mapping &m=mappings_[i];
if(!m.memory || m.memory->imported())continue;
if(m.memory->publishForDevice(0,m.memory->bytes(),true))return fail(ArmSmmu4KIoError);
if(!m.memory)continue;
if(coherentMapping(m.writeback)) {native_adreno_coherent_skips[2]=native_adreno_coherent_skips[2]+1;continue;}
if(m.memory->publishForDevice(0,m.memory->bytes(),true,distrust_write_combined_))return fail(ArmSmmu4KIoError);
}
return kIOReturnSuccess;
}
IOReturn NativeAdrenoVm::acquireImported()
{
if(!attached_ || poisoned_)return kIOReturnNotReady;
for(size_t i=0;i<NativeAdrenoVmMaxObjects;++i) {
const Mapping &m=mappings_[i];
if(!m.memory || !m.memory->imported() || !(m.permissions&ArmSmmu4KWrite))continue;
if(coherentMapping(m.writeback)) {native_adreno_coherent_skips[3]=native_adreno_coherent_skips[3]+1;continue;}
if(m.memory->acquireDeviceWrites(0,m.memory->bytes()))return fail(ArmSmmu4KIoError);
}
return kIOReturnSuccess;
}
@@ -238,13 +353,72 @@ IOReturn NativeAdrenoVm::sampleWord(uint64_t iova,uint32_t *cached,uint32_t *dra
const auto &span=m.memory->spans()[j];
if(offset<span.offset || offset-span.offset>=span.bytes)continue;
uint32_t value=0;
/* DRAM first: the cached reader writes its line back, so read after it DRAM would always agree (int-099). */
if(native_adreno_dram_read32 && !native_adreno_dram_read32(span.physical+(offset-span.offset),&value))*dram=value;
/* A write-combined BO's own mapping is non-cacheable: read the cacheable alias instead, so a dirty line is still seen. */
if(native_adreno_cached_read32 && !native_adreno_cached_read32(span.physical+(offset-span.offset),&value))*cached=value;
break;
}
return kIOReturnSuccess;
}
return kIOReturnBadArgument;
}
IOReturn NativeAdrenoVm::reassertWriteCombined(uint32_t *reset) const
{
if(!reset)return kIOReturnBadArgument;
if(!attached_ || poisoned_)return kIOReturnNotReady;
for(size_t i=0;i<NativeAdrenoVmMaxObjects;++i) {
const Mapping &m=mappings_[i];
if(!m.memory)continue;
const IOReturn status=m.memory->reassertWriteCombined(reset);
if(status)return status;
}
return kIOReturnSuccess;
}
IOReturn NativeAdrenoVm::auditRange(uint64_t iova,size_t bytes,uint32_t *lines,uint32_t *dirty,uint32_t *stale) const
{
if(!lines || !dirty || !stale)return kIOReturnBadArgument;
if(!attached_ || poisoned_)return kIOReturnNotReady;
if(!native_adreno_line_audit)return kIOReturnUnsupported;
if(!bytes || iova>UINT64_MAX-bytes)return kIOReturnBadArgument;
for(size_t i=0;i<NativeAdrenoVmMaxObjects;++i) {
const Mapping &m=mappings_[i];
if(!m.memory || iova<m.iova || iova-m.iova>=m.memory->bytes())continue;
if(!m.memory->writeCombined())return kIOReturnSuccess; /* maintained before every submission anyway */
/* Whole lines; the range is clipped to its BO. */
size_t at=(size_t)(iova-m.iova)&~(size_t)63;
const size_t end=(size_t)(iova-m.iova)+bytes>m.memory->bytes()?m.memory->bytes():(size_t)(iova-m.iova)+bytes;
while(at<end) {
const NativeAdrenoPhysicalSpan *span=nullptr;
for(size_t j=0;j<m.memory->spanCount();++j) {
const auto &s=m.memory->spans()[j];
if(at>=s.offset && at-s.offset<s.bytes) {span=&s;break;}
}
if(!span)return kIOReturnBadArgument;
const uint64_t physical=span->physical+(at-span->offset);
size_t chunk=4096-(size_t)(physical&4095);
if(chunk>span->offset+span->bytes-at)chunk=span->offset+span->bytes-at;
if(chunk>end-at)chunk=(end-at+63)&~(size_t)63;
if(native_adreno_line_audit(physical,chunk,dirty,stale))return kIOReturnIOError;
*lines+=(uint32_t)(chunk/64);at+=chunk;
}
return kIOReturnSuccess;
}
return kIOReturnBadArgument;
}
IOReturn NativeAdrenoVm::peekWords(uint64_t iova,uint32_t *out,size_t count) const
{
if(!out || !attached_ || poisoned_)return kIOReturnBadArgument;
for(size_t k=0;k<count;++k)out[k]=0;
for(size_t i=0;i<NativeAdrenoVmMaxObjects;++i) {
const Mapping &m=mappings_[i];
if(!m.memory || !m.memory->mapping() || iova<m.iova || iova-m.iova>=m.memory->bytes())continue;
const size_t offset=(size_t)(iova-m.iova);
for(size_t k=0;k<count && offset+4*k+4<=m.memory->bytes();++k)out[k]=static_cast<const volatile uint32_t *>(m.memory->mapping())[offset/4+k];
return kIOReturnSuccess;
}
return kIOReturnBadArgument;
}
IOReturn NativeAdrenoVm::holdJob(uint64_t handle)
{
if(!attached_ || poisoned_)return kIOReturnNotReady;
drivers/NativeAdreno/NativeAdrenoVm.h
+35 -6
View File
@@ -29,6 +29,12 @@ struct NativeAdrenoVmResult {
enum : size_t {NativeAdrenoVmMaxObjects=1024};
/* Board hook for sampleWord(): reads one aligned word of physical memory through a fresh uncached mapping; 0 on success. */
extern "C" int (*native_adreno_dram_read32)(uint64_t physical,uint32_t *value);
/* The board's reader through the kernel's cacheable physical aperture (null on the host: the BO's own mapping is read). */
extern "C" int (*native_adreno_cached_read32)(uint64_t physical,uint32_t *value);
/* Board hook for auditRange(): [physical, physical+bytes), whole 64-byte lines inside one page, as the cacheable physical aperture and
* DRAM (a fresh uncached mapping) read it. A line that differs is cleaned and invalidated and DRAM read again: when DRAM then holds the
* cached copy the line was dirty (*dirty), when DRAM is unchanged it was a stale clean copy (*stale). 0 on success. */
extern "C" int (*native_adreno_line_audit)(uint64_t physical,size_t bytes,uint32_t *dirty,uint32_t *stale);
class NativeAdrenoVm final {
struct Mapping {
NativeAdrenoMemory *memory=nullptr;
@@ -46,14 +52,19 @@ class NativeAdrenoVm final {
size_t capacity_=0,lower_count_=0,upper_count_=0,objects_=0;
unsigned budget_=0;
uint64_t next_handle_=1;
bool attached_=false,poisoned_=false;
bool attached_=false,poisoned_=false,distrust_write_combined_=false;
NativeAdrenoVmResult result_{};
ArmSmmu4KLiveRegion region(unsigned) const;
ArmSmmu4KStatus observe() const;
/* The route check alone: the table walk is the update's own (it verifies every region before the first store and after the last), so
* map/unmap pay a full extra audit only every 64th operation (int-139: 4 full walks of the 2 GiB aperture made every map 70 ms). */
ArmSmmu4KStatus observeStreamsOnly() const;
ArmSmmu4KStatus observeCheap();
unsigned since_audit_=0;
IOReturn fail(ArmSmmu4KStatus);
Mapping *find(uint64_t);
void insert(const ArmSmmu4KRange &,unsigned);
void erase(size_t,unsigned);
void insert(size_t,unsigned);
void erase(size_t,size_t,unsigned);
IOReturn preflight(const NativeAdrenoMemory *,unsigned,uint64_t,unsigned,unsigned,unsigned) const;
public:
NativeAdrenoVm()=default;
@@ -75,15 +86,33 @@ public:
* synchronizes by ASID; this is the stronger form the submit path uses before each submission (int-070: on sheng the CP read
* zeros from an IB whose table entry and DRAM were both correct, after the address had been mapped to other pages before). */
IOReturn invalidateTlb();
/* Cleans and invalidates the CPU caches over every mapped kernel-owned BO (whole objects; imported BOs are their owner's
* business), so no CPU cache line can hold data the GPU should see or hide what it wrote. Lines are physically tagged, so
* this also reaches lines a user mapping of the same pages left behind. int-070 runs it before every submission. */
/* Cleans and invalidates the CPU caches over every mapped BO (whole objects; imported ones through their physical pages), so
* no CPU cache line can hold data the GPU should see or hide what it wrote. Lines are physically tagged, so this also reaches
* lines a user mapping of the same pages left behind. int-070 runs it before every submission. */
IOReturn publishMapped();
/* The submit path saw a CPU-cached word that DRAM did not hold yet in a client's BO: from now on write-combined BOs are maintained
* like any other (sticky; the safe behaviour of int-070..097). */
void distrustWriteCombined() {distrust_write_combined_=true;}
bool writeCombinedTrusted() const {return !distrust_write_combined_;}
/* After a submission completed: cleans and invalidates the CPU caches over every GPU-writable imported BO, so the owner's mapping
* (an IOSurface) reads what the GPU wrote, not a line prefetched while the GPU ran. Jobs name only their IBs, so every such BO
* is covered. int-071: an imported render target read back 672 of 4096 texels stale. */
IOReturn acquireImported();
/* Diagnostic, never a proof of coherence: the 32-bit word at a mapped GPU address as the kernel's cacheable alias reads it
* and, when the board installed native_adreno_dram_read32, as DRAM reads it through an uncached mapping (else ~0). */
IOReturn sampleWord(uint64_t iova,uint32_t *cached,uint32_t *dram) const;
/* Every line of [iova, iova+bytes) that lies in a write-combined BO, through native_adreno_line_audit: *dirty counts lines a CPU
* cache held newer than DRAM (a client view that caches after all; each such line is written back by the audit), *stale clean
* copies, *lines the lines compared. kIOReturnUnsupported without a board hook. int-099 sampled one word and wrote its line back
* before reading DRAM, so it could never see a dirty line. */
/* Before a submission while write-combined BOs are trusted: every page of every mapped write-combined BO that XNU set back to the
* default attribute is write-combined again (and its lines written back); *reset counts the pages. */
IOReturn reassertWriteCombined(uint32_t *reset) const;
IOReturn auditRange(uint64_t iova,size_t bytes,uint32_t *lines,uint32_t *dirty,uint32_t *stale) const;
/* Read-only description of a live mapping; never pins or extends it. */
IOReturn describe(uint64_t handle,uint64_t *iova,size_t *bytes,unsigned *permissions) const;
/* Up to `count` dwords of a mapped kernel-owned BO at a GPU address (through the kernel's own mapping), for the board log. */
IOReturn peekWords(uint64_t iova,uint32_t *out,size_t count) const;
/* Only a scheduler's actual successful GPU completion may call this.
* A timeout, interrupt bit alone or CPU-written timestamp is insufficient. */
IOReturn retireJobAfterCompletion(uint64_t handle);
drivers/NativeAdreno/NativeAdrenoVmAlias.h
+48
View File
@@ -0,0 +1,48 @@
/* SPDX-License-Identifier: MIT */
#ifndef NATIVE_ADRENO_VM_ALIAS_H
#define NATIVE_ADRENO_VM_ALIAS_H
#include <stddef.h>
#include <stdint.h>
/* Temporary CPU validation index, never a GPU mapping or translation table.
* Range has physical/bytes fields. The caller has validated nonempty ranges
* with nonoverflowing ends and supplies count valid indices in owned scratch.
* Original ranges are immutable. One serialized owner uses scratch at a time. */
template<class Range> class NativeAdrenoPhysicalIndex final {
const Range *ranges_;
uint32_t *order_;
size_t count_;
uint64_t end(uint32_t i) const {return ranges_[i].physical+ranges_[i].bytes;}
bool less(uint32_t a,uint32_t b) const {
return ranges_[a].physical<ranges_[b].physical ||
(ranges_[a].physical==ranges_[b].physical && ranges_[a].bytes<ranges_[b].bytes);
}
void sift(size_t root,size_t count) {
while(root<count/2) {
size_t child=root*2+1;
if(child+1<count && less(order_[child],order_[child+1]))++child;
if(!less(order_[root],order_[child]))break;
uint32_t t=order_[root];order_[root]=order_[child];order_[child]=t;root=child;
}
}
public:
NativeAdrenoPhysicalIndex(const Range *ranges,uint32_t *order,size_t count):ranges_(ranges),order_(order),count_(count) {
for(size_t i=count/2;i-->0;)sift(i,count);
for(size_t n=count;n>1;) {--n;uint32_t t=order_[0];order_[0]=order_[n];order_[n]=t;sift(0,n);}
/* Discard an interval contained by an earlier interval. The remaining
* starts AND ends are monotonic, even for nested/duplicate/overlapping
* inputs. Each query then needs a binary search, with no backward scan. */
size_t kept=0;
for(size_t i=0;i<count;++i) {
uint32_t index=order_[i];
if(!kept || end(index)>end(order_[kept-1]))order_[kept++]=index;
}
count_=kept;
}
bool overlaps(uint64_t physical,uint64_t bytes) const {
size_t lo=0,hi=count_;
while(lo<hi) {size_t mid=lo+(hi-lo)/2;if(end(order_[mid])<=physical)lo=mid+1;else hi=mid;}
return lo<count_ && ranges_[order_[lo]].physical<physical+bytes;
}
};
#endif
drivers/NativeAdreno/tests/XnuMemoryFixture.h
+16 -7
View File
@@ -15,19 +15,23 @@ using addr64_t=uint64_t;
constexpr IOReturn kIOReturnSuccess=0,kIOReturnBadArgument=1,kIOReturnBusy=2,
kIOReturnNoResources=3,kIOReturnNoMemory=4,kIOReturnUnsupported=5,
kIOReturnNotReady=6,kIOReturnIOError=7,kIOReturnTimeout=8;
constexpr unsigned kIODirectionInOut=3,kIOMemoryMapperNone=0x800,kIOMemoryPhysicallyContiguous=0x10;
constexpr unsigned kIODirectionOut=2,kIODirectionInOut=3,kIOMemoryMapperNone=0x800,kIOMemoryPhysicallyContiguous=0x10;
inline void *kernel_task=reinterpret_cast<void *>(1);
using task_t=void *;
enum SegmentMode {Contiguous,Scattered,ZeroPhysical,UnalignedPhysical,ShortSegment,HighPhysical,Paired};
/* ScatterUnlessContiguous: what the tablet does, one physical page per span unless the caller asked for contiguous memory. */
enum SegmentMode {Contiguous,Scattered,ZeroPhysical,UnalignedPhysical,ShortSegment,HighPhysical,Paired,ScatterUnlessContiguous};
class IOBufferMemoryDescriptor;
struct MemoryFixture {
unsigned created=0,prepared=0,completed=0,released=0,cache_calls=0,segment_calls=0;
unsigned allocation_fail_count=0;
bool allocation_fail=false,prepare_fail=false,complete_fail=false,metadata_fail=false;
bool write_protected_imports=false; /* user ranges that can only be pinned for reading: prepare() of a writable one reports KERN_PROTECTION_FAILURE */
bool null_mapping=false,unaligned_mapping=false,wrong_length=false;
unsigned cache_fail_call=0;
uint64_t next_physical=0x30000000;
size_t contiguous_limit=0; /* nonzero: physically contiguous requests above this many bytes fail (a fragmented tablet) */
unsigned imported=0,imports_live=0;
unsigned physical_calls=0;size_t physical_bytes=0; /* NativeAdrenoMaintainPhysicalPages doubles */
IOReturn cpu_status=kIOReturnSuccess;
SegmentMode mode=Contiguous;
std::vector<IOBufferMemoryDescriptor *> live;
@@ -51,12 +55,14 @@ public:
size_t bytes=0;
uint64_t mask=0;
uint64_t physical_base=0;
bool contiguous=false;
static IOBufferMemoryDescriptor *inTaskWithPhysicalMask(void *task,unsigned options,size_t bytes,uint64_t mask) {
if(task!=kernel_task || !(options&kIOMemoryMapperNone))std::abort();
if(fixture.allocation_fail)return nullptr;
if(fixture.allocation_fail_count) {--fixture.allocation_fail_count;return nullptr;}
if(fixture.contiguous_limit && (options&kIOMemoryPhysicallyContiguous) && bytes>fixture.contiguous_limit)return nullptr;
auto *d=new IOBufferMemoryDescriptor;
d->bytes=bytes;d->mask=mask;d->physical_base=fixture.next_physical;
d->bytes=bytes;d->mask=mask;d->physical_base=fixture.next_physical;d->contiguous=options&kIOMemoryPhysicallyContiguous;
fixture.next_physical+=2*bytes+0x10000;
if(posix_memalign(&d->raw,4096,bytes+4096))std::abort();
std::memset(d->raw,0xa5,bytes+4096);++fixture.created;fixture.live.push_back(d);return d;
@@ -73,7 +79,10 @@ public:
if((offset&4095) || offset>=bytes || options!=kIOMemoryMapperNone)std::abort();
++fixture.segment_calls;*length=bytes-offset;
uint64_t base=physical_base,physical=base+offset;
if(fixture.mode==Scattered)physical=base+2*offset;
if(fixture.mode==Scattered || (fixture.mode==ScatterUnlessContiguous && !contiguous)) {
physical=base+2*offset;
if(fixture.mode==ScatterUnlessContiguous)*length=4096;
}
if(fixture.mode==Paired)physical=base+offset+((offset/8192)*4096);
if(offset==4096) {
if(fixture.mode==ZeroPhysical)physical=0;
@@ -87,14 +96,14 @@ public:
/* A user range pinned with withAddressRange/prepare: pages at a fixed physical stride (every other page), so imports are scattered. */
class IOMemoryDescriptor {
public:
uint64_t address=0,bytes=0,physical_base=0;
uint64_t address=0,bytes=0,physical_base=0;unsigned options=0;
static IOMemoryDescriptor *withAddressRange(uint64_t address,uint64_t bytes,unsigned options,task_t task) {
if(!task || task==kernel_task || !(options&kIOMemoryMapperNone))std::abort();
if(fixture.allocation_fail)return nullptr;
auto *d=new IOMemoryDescriptor;d->address=address;d->bytes=bytes;d->physical_base=fixture.next_physical;
auto *d=new IOMemoryDescriptor;d->options=options;d->address=address;d->bytes=bytes;d->physical_base=fixture.next_physical;
fixture.next_physical+=2*bytes+0x10000;++fixture.imported;++fixture.imports_live;return d;
}
IOReturn prepare() {++fixture.prepared;return fixture.prepare_fail?kIOReturnIOError:kIOReturnSuccess;}
IOReturn prepare() {++fixture.prepared;if(fixture.write_protected_imports && (options&1))return 2;return fixture.prepare_fail?kIOReturnIOError:kIOReturnSuccess;}
IOReturn complete() {++fixture.completed;return fixture.complete_fail?kIOReturnIOError:kIOReturnSuccess;}
void release() {--fixture.imports_live;delete this;}
uint64_t getLength() {return bytes;}
drivers/NativeAdreno/tests/hostsim/builtin_cache.cpp
+73
View File
@@ -0,0 +1,73 @@
/* SPDX-License-Identifier: MIT */
/* Actual Turnip builtin export/embedding and lifetime checks. The mandatory
* software peer executes no shader. No hardware timing claim follows. */
#include <vulkan/vulkan.h>
#include "util/blake3/blake3.h"
#include <algorithm>
#include <chrono>
#include <cstdio>
#include <cstring>
#include <dlfcn.h>
#include <fstream>
#include <iterator>
#include <vector>
#include <unistd.h>
#include <TargetConditionals.h>
#define REQUIRE(x) do {if(!(x)){std::fprintf(stderr,"FAIL builtin line %d: %s\n",__LINE__,#x);return 1;}} while(0)
#define LOAD(name) auto name=reinterpret_cast<PFN_##name>(get(instance,#name));REQUIRE(name)
int main(int argc,char **argv) {
if(argc!=4)return 2;
bool produce=!std::strcmp(argv[3],"produce"),consume=!std::strcmp(argv[3],"consume"),refuse=!std::strcmp(argv[3],"refuse");
REQUIRE(produce||consume||refuse);
#if TARGET_OS_IPHONE
REQUIRE(getpagesize()==4096 && sysconf(_SC_PAGESIZE)==4096 && !getenv("HOSTSIM_PAGE_SIZE"));
#endif
auto live=reinterpret_cast<unsigned(*)()>(dlsym(RTLD_DEFAULT,"host_gpu_live_objects"));
auto submitted=reinterpret_cast<unsigned(*)()>(dlsym(RTLD_DEFAULT,"host_gpu_submissions"));REQUIRE(live&&submitted);
void *library=dlopen(argv[1],RTLD_NOW|RTLD_LOCAL);if(!library)std::fprintf(stderr,"%s\n",dlerror());REQUIRE(library);
auto get=reinterpret_cast<PFN_vkGetInstanceProcAddr>(dlsym(library,"vk_icdGetInstanceProcAddr"));REQUIRE(get);
auto exportPack=reinterpret_cast<VkResult(*)(VkDevice,size_t *,void *)>(dlsym(library,"tu_export_builtin_shader_pack"));
auto validate=reinterpret_cast<VkResult(*)(VkDevice,size_t,const void *)>(dlsym(library,"tu_validate_builtin_shader_pack"));REQUIRE(exportPack&&validate);
VkInstance instance=VK_NULL_HANDLE;LOAD(vkCreateInstance);
VkApplicationInfo app={};app.sType=VK_STRUCTURE_TYPE_APPLICATION_INFO;app.apiVersion=VK_API_VERSION_1_2;
VkInstanceCreateInfo ii={};ii.sType=VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO;ii.pApplicationInfo=&app;
REQUIRE(vkCreateInstance(&ii,nullptr,&instance)==VK_SUCCESS);
LOAD(vkDestroyInstance);LOAD(vkEnumeratePhysicalDevices);LOAD(vkGetPhysicalDeviceQueueFamilyProperties);
LOAD(vkCreateDevice);LOAD(vkDestroyDevice);LOAD(vkDeviceWaitIdle);
uint32_t n=1;VkPhysicalDevice physical=VK_NULL_HANDLE;REQUIRE(vkEnumeratePhysicalDevices(instance,&n,&physical)==VK_SUCCESS&&n==1);
VkQueueFamilyProperties families[8]={};n=8;vkGetPhysicalDeviceQueueFamilyProperties(physical,&n,families);
REQUIRE(n>0&&n<=8);int family=-1;for(unsigned i=0;i<n;++i)if(families[i].queueFlags&VK_QUEUE_GRAPHICS_BIT){family=(int)i;break;}REQUIRE(family>=0);
float priority=1;VkDeviceQueueCreateInfo qi={};qi.sType=VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO;qi.queueFamilyIndex=family;qi.queueCount=1;qi.pQueuePriorities=&priority;
VkDeviceCreateInfo di={};di.sType=VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO;di.queueCreateInfoCount=1;di.pQueueCreateInfos=&qi;
VkDevice device=VK_NULL_HANDLE;auto begin=std::chrono::steady_clock::now();const VkResult created=vkCreateDevice(physical,&di,nullptr,&device);
const double elapsed=std::chrono::duration<double,std::micro>(std::chrono::steady_clock::now()-begin).count();
size_t bytes=0;
if(refuse)REQUIRE(created==VK_ERROR_INITIALIZATION_FAILED&&device==VK_NULL_HANDLE);
else {
REQUIRE(created==VK_SUCCESS&&device);REQUIRE(exportPack(device,&bytes,nullptr)==VK_SUCCESS&&bytes>128&&bytes<4*1024*1024);
std::vector<unsigned char> record(bytes);REQUIRE(exportPack(device,&bytes,record.data())==VK_SUCCESS&&validate(device,bytes,record.data())==VK_SUCCESS);
unsigned char guard[16];std::memset(guard,0xa5,sizeof(guard));size_t shortSize=sizeof(guard);
REQUIRE(exportPack(device,&shortSize,guard)==VK_INCOMPLETE&&shortSize==bytes&&std::all_of(std::begin(guard),std::end(guard),[](auto x){return x==0xa5;}));
auto bad=record;bad[bytes-1]^=1;
REQUIRE(validate(device,bytes,bad.data())==VK_ERROR_INITIALIZATION_FAILED);
REQUIRE(validate(device,bytes-1,record.data())==VK_ERROR_INITIALIZATION_FAILED);
REQUIRE(validate(device,0,nullptr)==VK_ERROR_INITIALIZATION_FAILED);
REQUIRE(validate(device,4*1024*1024+1,reinterpret_cast<void *>(1))==VK_ERROR_INITIALIZATION_FAILED);
REQUIRE(record.size()>256&&!std::memcmp(record.data(),"TUBUILT1",8));
auto reseal=[](std::vector<unsigned char> &value){blake3_hasher h;blake3_hasher_init(&h);blake3_hasher_update(&h,value.data(),value.size()-32);blake3_hasher_finalize(&h,value.data()+value.size()-32,32);};
auto changed=[&](size_t at,uint32_t value){auto mutated=record;REQUIRE(at+4<=mutated.size()-32);std::memcpy(mutated.data()+at,&value,4);reseal(mutated);REQUIRE(validate(device,mutated.size(),mutated.data())==VK_ERROR_INITIALIZATION_FAILED);return 0;};
REQUIRE(changed(16,0)==0); // exact source/profile identity, valid checksum
REQUIRE(changed(80,0)==0); // actual GPU identity
REQUIRE(changed(12,16)==0); // incomplete variant set
REQUIRE(changed(144,1)==0); // duplicated/reordered entry
REQUIRE(changed(148,0x10000000)==0); // entry allocation bound
uint32_t keyBytes=0;std::memcpy(&keyBytes,record.data()+116,4);REQUIRE(keyBytes<1024);
REQUIRE(changed(152+keyBytes+8,0xffffffff)==0); // unchecked IR3 allocation in the old generic reader
bad=record;bad.insert(bad.end()-32,0x7f);reseal(bad);REQUIRE(validate(device,bad.size(),bad.data())==VK_ERROR_INITIALIZATION_FAILED);
if(produce){std::ofstream output(argv[2],std::ios::binary);REQUIRE(output);output.write(reinterpret_cast<char *>(record.data()),record.size());REQUIRE(output.good());}
else {std::ifstream input(argv[2],std::ios::binary);REQUIRE(input);std::vector<unsigned char> previous((std::istreambuf_iterator<char>(input)),{});REQUIRE(record==previous);}
REQUIRE(vkDeviceWaitIdle(device)==VK_SUCCESS);vkDestroyDevice(device,nullptr);
}
vkDestroyInstance(instance,nullptr);REQUIRE(live()==0&&submitted()==0);
std::printf("PASS: builtin cache %s pid=%d variants=17 bytes=%zu device_us=%.1f negatives=%u live=0 submissions=0; software peer, no shader execution\n",argv[3],getpid(),bytes,elapsed,refuse?0:11);
}
drivers/NativeAdreno/tests/hostsim/deferred_bo.cpp
+160
View File
@@ -0,0 +1,160 @@
/* SPDX-License-Identifier: MIT */
/* Actual Vulkan/Turnip/KMD control flow over the software kernel peer. It runs no
* shader: pending fences, retained native objects/views/IOVAs and error handling
* are checked independently of rendering or hardware timing. */
#include <vulkan/vulkan.h>
#include "NativeAdrenoABI.h"
#include <atomic>
#include <chrono>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <dlfcn.h>
#include <thread>
#include <vector>
#define REQUIRE(x) do {if (!(x)) {std::fprintf(stderr,"FAIL line %d: %s\n",__LINE__,#x);return 1;}} while(0)
#define OK(x) REQUIRE((x)==VK_SUCCESS)
#define LOAD(name) auto name=(PFN_##name)get(instance,#name);REQUIRE(name)
struct Peer {
void (*hold)(int);
std::atomic<bool> release{false},forced{false};
std::thread worker;
explicit Peer(void (*fn)(int)):hold(fn){hold(1);}
void watchdog() {
worker=std::thread([this]{
auto end=std::chrono::steady_clock::now()+std::chrono::milliseconds(500);
while(!release.load() && std::chrono::steady_clock::now()<end)
std::this_thread::sleep_for(std::chrono::milliseconds(1));
if(!release.load())forced=true;
hold(0);
});
}
void finish(){release=true;hold(0);if(worker.joinable())worker.join();}
~Peer(){finish();}
};
int main(int argc,char **argv) {
if(argc!=3)return 2;
const bool pressure=!std::strcmp(argv[2],"pressure"),busy=!std::strcmp(argv[2],"busy");
const bool fenceError=!std::strcmp(argv[2],"fence-error"),freeError=!std::strcmp(argv[2],"free-error");
const bool leaseError=!std::strcmp(argv[2],"lease-error"),viewError=!std::strcmp(argv[2],"view-error");
const bool cleanupError=!std::strcmp(argv[2],"map-cleanup-error");
const bool fatal=fenceError||freeError||leaseError||viewError||cleanupError;
void *library=dlopen(argv[1],RTLD_NOW|RTLD_LOCAL);REQUIRE(library);
auto get=(PFN_vkGetInstanceProcAddr)dlsym(library,"vk_icdGetInstanceProcAddr");REQUIRE(get);
auto live=(unsigned(*)())dlsym(RTLD_DEFAULT,"host_gpu_live_objects");
auto submissions=(unsigned(*)())dlsym(RTLD_DEFAULT,"host_gpu_submissions");
auto successes=(unsigned(*)(uint32_t))dlsym(RTLD_DEFAULT,"host_gpu_successes");
auto lastIova=(uint64_t(*)())dlsym(RTLD_DEFAULT,"host_gpu_last_map_iova");
auto hold=(void(*)(int))dlsym(RTLD_DEFAULT,"host_gpu_hold_completion");
auto inject=(void(*)(uint32_t,int,uint32_t))dlsym(RTLD_DEFAULT,"host_gpu_inject_failure");
REQUIRE(live&&submissions&&successes&&lastIova&&hold&&inject);
VkInstance instance=VK_NULL_HANDLE;
LOAD(vkCreateInstance);
VkApplicationInfo app={.sType=VK_STRUCTURE_TYPE_APPLICATION_INFO,.pApplicationName="deferred BO control",.apiVersion=VK_API_VERSION_1_1};
VkInstanceCreateInfo ii={.sType=VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO,.pApplicationInfo=&app};
OK(vkCreateInstance(&ii,nullptr,&instance));
LOAD(vkEnumeratePhysicalDevices);LOAD(vkGetPhysicalDeviceMemoryProperties);LOAD(vkGetPhysicalDeviceQueueFamilyProperties);
LOAD(vkCreateDevice);LOAD(vkAllocateMemory);LOAD(vkFreeMemory);LOAD(vkMapMemory);
LOAD(vkCreateBuffer);LOAD(vkDestroyBuffer);LOAD(vkGetBufferMemoryRequirements);LOAD(vkBindBufferMemory);
LOAD(vkGetDeviceQueue);LOAD(vkCreateCommandPool);LOAD(vkAllocateCommandBuffers);LOAD(vkBeginCommandBuffer);
LOAD(vkCmdFillBuffer);LOAD(vkEndCommandBuffer);LOAD(vkQueueSubmit);LOAD(vkDeviceWaitIdle);
LOAD(vkDestroyCommandPool);LOAD(vkDestroyDevice);LOAD(vkDestroyInstance);
uint32_t count=1;VkPhysicalDevice physical;OK(vkEnumeratePhysicalDevices(instance,&count,&physical));REQUIRE(count==1);
VkPhysicalDeviceMemoryProperties memory={};vkGetPhysicalDeviceMemoryProperties(physical,&memory);
int type=-1;
for(uint32_t i=0;i<memory.memoryTypeCount;i++)if((memory.memoryTypes[i].propertyFlags&(VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT|VK_MEMORY_PROPERTY_HOST_COHERENT_BIT))==(VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT|VK_MEMORY_PROPERTY_HOST_COHERENT_BIT)){type=(int)i;break;}
REQUIRE(type>=0);
VkQueueFamilyProperties families[8];count=8;vkGetPhysicalDeviceQueueFamilyProperties(physical,&count,families);
int family=-1;for(uint32_t i=0;i<count;i++)if(families[i].queueFlags&VK_QUEUE_GRAPHICS_BIT){family=(int)i;break;}
REQUIRE(family>=0);
float priority=1;
VkDeviceQueueCreateInfo qi={.sType=VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO,.queueFamilyIndex=(uint32_t)family,.queueCount=1,.pQueuePriorities=&priority};
VkDeviceCreateInfo di={.sType=VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO,.queueCreateInfoCount=1,.pQueueCreateInfos=&qi};
VkDevice device;OK(vkCreateDevice(physical,&di,nullptr,&device));
VkQueue queue;vkGetDeviceQueue(device,(uint32_t)family,0,&queue);
VkBufferCreateInfo bi={.sType=VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO,.size=4096,.usage=VK_BUFFER_USAGE_TRANSFER_DST_BIT,.sharingMode=VK_SHARING_MODE_EXCLUSIVE};
VkBuffer buffer;OK(vkCreateBuffer(device,&bi,nullptr,&buffer));
VkMemoryRequirements req;vkGetBufferMemoryRequirements(device,buffer,&req);REQUIRE(req.memoryTypeBits&(1u<<type));
VkMemoryAllocateInfo ai={.sType=VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO,.allocationSize=req.size,.memoryTypeIndex=(uint32_t)type};
VkDeviceMemory used;OK(vkAllocateMemory(device,&ai,nullptr,&used));OK(vkBindBufferMemory(device,buffer,used,0));
ai.allocationSize=16384;
std::vector<VkDeviceMemory> unused(pressure?257:1);
uint64_t oldIova=0;
for(auto &allocation:unused){OK(vkAllocateMemory(device,&ai,nullptr,&allocation));void *address=nullptr;OK(vkMapMemory(device,allocation,0,VK_WHOLE_SIZE,0,&address));if(!oldIova)oldIova=lastIova();}
VkCommandPoolCreateInfo pi={.sType=VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO,.queueFamilyIndex=(uint32_t)family};
VkCommandPool pool;OK(vkCreateCommandPool(device,&pi,nullptr,&pool));
VkCommandBufferAllocateInfo ci={.sType=VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO,.commandPool=pool,.level=VK_COMMAND_BUFFER_LEVEL_PRIMARY,.commandBufferCount=1};
VkCommandBuffer commands;OK(vkAllocateCommandBuffers(device,&ci,&commands));
VkCommandBufferBeginInfo begin={.sType=VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO};OK(vkBeginCommandBuffer(commands,&begin));
vkCmdFillBuffer(commands,buffer,0,4096,0x5a5a5a5a);OK(vkEndCommandBuffer(commands));
Peer peer(hold);
unsigned beforeSubmit=submissions();
VkSubmitInfo si={.sType=VK_STRUCTURE_TYPE_SUBMIT_INFO,.commandBufferCount=1,.pCommandBuffers=&commands};
OK(vkQueueSubmit(queue,1,&si,VK_NULL_HANDLE));REQUIRE(submissions()>beforeSubmit);
const unsigned beforeLive=live(),beforeFree=successes(NativeAdrenoMethodBoFree),beforeUnmap=successes(NativeAdrenoMethodBoCpuUnmap);
if(fenceError)inject(NativeAdrenoMethodWaitFence,7,1); // fixture kIOReturnIOError, before mutation
if(!pressure)peer.watchdog();
for(unsigned i=0;i<(pressure?256u:1u);i++){
vkFreeMemory(device,unused[i],nullptr);unused[i]=VK_NULL_HANDLE;
REQUIRE(!peer.forced.load());
REQUIRE(successes(NativeAdrenoMethodBoFree)==beforeFree);
REQUIRE(successes(NativeAdrenoMethodBoCpuUnmap)==beforeUnmap);
}
REQUIRE(live()==beforeLive);
VkDeviceMemory probe=VK_NULL_HANDLE,after=VK_NULL_HANDLE;
if(cleanupError){
inject(NativeAdrenoMethodBoMap,7,1);inject(NativeAdrenoMethodBoFree,2,1);
VkDeviceMemory refused=VK_NULL_HANDLE;
REQUIRE(vkAllocateMemory(device,&ai,nullptr,&refused)==VK_ERROR_DEVICE_LOST);
REQUIRE(live()==beforeLive+1);
REQUIRE(successes(NativeAdrenoMethodBoFree)==beforeFree);
REQUIRE(successes(NativeAdrenoMethodBoCpuUnmap)==beforeUnmap);
}
if(!fatal && !pressure){OK(vkAllocateMemory(device,&ai,nullptr,&probe));REQUIRE(lastIova()!=oldIova);REQUIRE(successes(NativeAdrenoMethodBoFree)==beforeFree);}
if(pressure){
peer.watchdog();
vkFreeMemory(device,unused.back(),nullptr);unused.back()=VK_NULL_HANDLE;
REQUIRE(peer.forced.load());
REQUIRE(successes(NativeAdrenoMethodBoFree)==beforeFree+257);
REQUIRE(successes(NativeAdrenoMethodBoCpuUnmap)==beforeUnmap+257);
}
peer.finish();
if(!pressure&&!busy&&!fatal){
OK(vkDeviceWaitIdle(device));
REQUIRE(successes(NativeAdrenoMethodBoFree)==beforeFree+1);
REQUIRE(successes(NativeAdrenoMethodBoCpuUnmap)==beforeUnmap+1);
}
if(busy)inject(NativeAdrenoMethodBoFree,2,1); // temporary busy must retain the IOVA
if(freeError)inject(NativeAdrenoMethodBoFree,7,1);
if(leaseError)inject(NativeAdrenoMethodBoCpuUnmap,7,1);
if(viewError)setenv("HOSTSIM_FAIL_USER_UNMAP","1",1);
VkResult result=vkAllocateMemory(device,&ai,nullptr,&after);
if(fatal){
REQUIRE(result==VK_ERROR_DEVICE_LOST);
REQUIRE(successes(NativeAdrenoMethodBoFree)==beforeFree);
REQUIRE(successes(NativeAdrenoMethodBoCpuUnmap)==beforeUnmap+(freeError?1u:0u));
} else {
REQUIRE(result==VK_SUCCESS);
if(busy){
REQUIRE(lastIova()!=oldIova);REQUIRE(successes(NativeAdrenoMethodBoFree)==beforeFree);
VkDeviceMemory retry;OK(vkAllocateMemory(device,&ai,nullptr,&retry));
REQUIRE(successes(NativeAdrenoMethodBoFree)==beforeFree+1);
REQUIRE(successes(NativeAdrenoMethodBoCpuUnmap)==beforeUnmap+1);
vkFreeMemory(device,retry,nullptr);
} else REQUIRE(successes(NativeAdrenoMethodBoFree)>=beforeFree+(pressure?257u:1u));
}
unsetenv("HOSTSIM_FAIL_USER_UNMAP");inject(~0u,0,0);
if(!fatal)OK(vkDeviceWaitIdle(device));
if(probe)vkFreeMemory(device,probe,nullptr);
if(after)vkFreeMemory(device,after,nullptr);
vkDestroyCommandPool(device,pool,nullptr);vkDestroyBuffer(device,buffer,nullptr);vkFreeMemory(device,used,nullptr);
vkDestroyDevice(device,nullptr);vkDestroyInstance(instance,nullptr);
// A failed fence/view/lease release retains native ownership in this narrow peer;
// it deliberately does not manufacture a successful IOKit close or recovery.
if(!fatal)REQUIRE(live()==0);
std::printf("PASS: deferred BO %s; native objects at exit=%u; software control only, no shader execution\n",argv[2],live());
return 0;
}
drivers/NativeAdreno/tests/hostsim/host_gpu_service.cpp
+64 -6
View File
@@ -9,7 +9,10 @@
#include "NativeAdrenoTimer.h"
#include <chrono>
#include <cstdio>
#include <cstdlib>
#include <thread>
#include <atomic>
#include <mutex>
IOReturn NativeAdrenoQueryCpu4K(NativeAdrenoCpuState *state) {*state={};return kIOReturnSuccess;}
IOReturn NativeAdrenoMaintainOwnedPages(void *,size_t,NativeAdrenoCacheOperation) {return kIOReturnSuccess;}
@@ -31,15 +34,27 @@ int sbarrier(void *) {return 0;}
std::vector<uint32_t> regs(0x10000);
volatile uint32_t *memptrs=nullptr;
unsigned submissions=0;
std::atomic<bool> holdCompletion{false};
std::recursive_mutex serviceLock;
unsigned calls[32]={},successes[32]={};
uint32_t faultRemaining[32]={};
IOReturn faultStatus[32]={};
uint64_t lastMapIova=0;
void completeIfReleased() {
if(!holdCompletion.load(std::memory_order_acquire)) {
regs[0x0806]=regs[0x0807];
if(memptrs)memptrs[1]=submissions;
}
}
int cwrite(void *,uint32_t reg,uint32_t v) {
regs[reg]=v;
if(reg==0x0807) {regs[0x0806]=v;++submissions;if(memptrs)memptrs[1]=submissions;}
if(reg==0x0807) {++submissions;completeIfReleased();}
return 0;
}
int cread(void *,uint32_t reg,uint32_t *v) {*v=regs[reg];return 0;}
int cread(void *,uint32_t reg,uint32_t *v) {completeIfReleased();*v=regs[reg];return 0;}
int cbarrier(void *) {return 0;}
int cpublish(void *,const volatile uint32_t *,size_t) {return 0;}
int cacquire(void *,volatile uint32_t *,size_t) {return 0;}
int cacquire(void *,volatile uint32_t *,size_t) {completeIfReleased();return 0;}
uint64_t cnow(void *) {
return uint64_t(std::chrono::duration_cast<std::chrono::microseconds>(
std::chrono::steady_clock::now().time_since_epoch()).count());
@@ -66,6 +81,11 @@ bool built=false,opened=false;
bool build() {
if(built)return true;
/* HOSTSIM_SCATTER=1: non-contiguous BOs come back one physical page per span, as on the tablet (int-070's bridge ran out of
* device memory there while the contiguous host model passed). */
if(const char *scatter=std::getenv("HOSTSIM_SCATTER");scatter && *scatter=='1')fixture.mode=ScatterUnlessContiguous;
/* HOSTSIM_CONTIGUOUS_MAX=<bytes>: larger physically contiguous allocations fail, so they fall back to scattered pages. */
if(const char *limit=std::getenv("HOSTSIM_CONTIGUOUS_MAX"))fixture.contiguous_limit=std::strtoull(limit,nullptr,0);
std::memset(mmio,0,sizeof(mmio));
mmio[0]=0x002f0406;mmio[0x20/4]=0x4c017e18;mmio[0x24/4]=0x40000016;mmio[0x28/4]=0x1433;
for(unsigned i=0;i<24;++i)mmio[(0xc00+4*i)/4]=0x20000;
@@ -93,22 +113,28 @@ IOReturn translate(IOReturn status) {return status;}
extern "C" {
/* Return codes are the fixture's IOReturn values; the shim translates them. */
int host_gpu_open() {
std::lock_guard<std::recursive_mutex> guard(serviceLock);
if(!build() || opened)return kIOReturnBusy;
const IOReturn st=client.open(NativeAdrenoAbiVersion,context->vm(),&execbuf,&ring,31,40);
if(st) {std::fprintf(stderr,"host gpu: client open failed %d\n",st);return kIOReturnNotReady;}
std::setvbuf(stderr,nullptr,_IONBF,0);
client.attachTimer(&timer);
client.attachDeviceInfo(&info);
opened=true;
return kIOReturnSuccess;
}
int host_gpu_close() {
std::lock_guard<std::recursive_mutex> guard(serviceLock);
if(!opened)return kIOReturnNotReady;
const IOReturn status=client.releaseAll(&clock_,2000000);
if(!status)opened=false;
return translate(status);
}
int host_gpu_call(uint32_t method,const void *in,size_t in_size,void *out,size_t out_size) {
std::lock_guard<std::recursive_mutex> guard(serviceLock);
if(!opened)return kIOReturnNotReady;
if(method<32)++calls[method];
if(method<32 && faultRemaining[method]){--faultRemaining[method];return faultStatus[method];}
/* As the IOKit wrapper does (NativeAdrenoPublish.cpp): an import is the calling task's pages, here a stand-in task. */
if(method==NativeAdrenoMethodBoImport) {
if(!in || in_size!=sizeof(NativeAdrenoAbiBoImport) || !out || out_size!=sizeof(NativeAdrenoAbiBoAllocReply))return kIOReturnBadArgument;
@@ -118,13 +144,45 @@ int host_gpu_call(uint32_t method,const void *in,size_t in_size,void *out,size_t
if(!status)std::memcpy(out,&reply,sizeof(reply));
return translate(status);
}
return translate(client.dispatch(method,in,in_size,out,out_size,&clock_));
const IOReturn status=client.dispatch(method,in,in_size,out,out_size,&clock_);
if(!status && method<32)++successes[method];
if(!status && method==NativeAdrenoMethodBoMap && in)
lastMapIova=static_cast<const NativeAdrenoAbiBoMap *>(in)->iova;
/* HOSTSIM_TRACE=1: every failing call, and every allocation size, on stderr. */
static const bool trace=std::getenv("HOSTSIM_TRACE") && *std::getenv("HOSTSIM_TRACE")=='1';
if(trace && method==NativeAdrenoMethodBoAlloc && in)
std::fprintf(stderr,"hostgpu alloc %llu -> %d spans-live %zu\n",(unsigned long long)static_cast<const NativeAdrenoAbiBoAlloc *>(in)->bytes,
status,context->vm()->spanCount());
else if(trace && status)std::fprintf(stderr,"hostgpu method %u -> %d\n",method,status);
return translate(status);
}
int host_gpu_map(uint32_t token,void **address,uint64_t *bytes) {
std::lock_guard<std::recursive_mutex> guard(serviceLock);
NativeAdrenoMemory *memory=opened?client.memoryForToken(token):nullptr;
if(!memory)return kIOReturnBadArgument;
*address=memory->mapping();*bytes=memory->bytes();return kIOReturnSuccess;
}
unsigned host_gpu_submissions() {return submissions;}
unsigned host_gpu_live_objects() {return unsigned(client.liveObjects());}
unsigned host_gpu_submissions() {std::lock_guard<std::recursive_mutex> guard(serviceLock);return submissions;}
unsigned host_gpu_live_objects() {std::lock_guard<std::recursive_mutex> guard(serviceLock);return unsigned(client.liveObjects());}
unsigned host_gpu_calls(uint32_t method) {std::lock_guard<std::recursive_mutex> guard(serviceLock);return method<32?calls[method]:0;}
unsigned host_gpu_successes(uint32_t method) {std::lock_guard<std::recursive_mutex> guard(serviceLock);return method<32?successes[method]:0;}
uint64_t host_gpu_last_map_iova() {std::lock_guard<std::recursive_mutex> guard(serviceLock);return lastMapIova;}
// Only the atomic peer-release flag changes off-thread. The next actual ring read/acquire
// publishes the software peer's completion under the service lock, like the existing CP model.
void host_gpu_hold_completion(int hold) {holdCompletion.store(hold!=0,std::memory_order_release);}
void host_gpu_inject_failure(uint32_t method,int status,uint32_t count) {
std::lock_guard<std::recursive_mutex> guard(serviceLock);
if(method<32){faultStatus[method]=status;faultRemaining[method]=count;}
else std::memset(faultRemaining,0,sizeof faultRemaining);
}
}
/* Physical-page maintenance (imported BOs): record it; no host page is touched. */
/* Host double: the page attribute change of user BOs has no host equivalent (no host page is touched). */
IOReturn NativeAdrenoSetPagesCacheMode(const NativeAdrenoPhysicalSpan *spans,size_t count,bool){return spans && count?kIOReturnSuccess:kIOReturnBadArgument;}
IOReturn NativeAdrenoReassertWriteCombined(const NativeAdrenoPhysicalSpan *spans,size_t count,uint32_t *reset)
{return spans && count && reset?kIOReturnSuccess:kIOReturnBadArgument;}
IOReturn NativeAdrenoMaintainPhysicalPages(uint64_t physical,size_t bytes,NativeAdrenoCacheOperation op)
{
if(!physical || !bytes || ((physical|bytes)&4095) || op>NativeAdrenoInvalidate)std::abort();
++fixture.physical_calls;fixture.physical_bytes+=bytes;return kIOReturnSuccess;
}
drivers/NativeAdreno/tests/hostsim/host_iokit_shim.cpp
+3 -1
View File
@@ -85,7 +85,9 @@ static kern_return_t shim_IOConnectMapMemory64(io_connect_t,uint32_t type,task_p
if(kr==KERN_SUCCESS) {*address=reinterpret_cast<mach_vm_address_t>(mapping);*size=bytes;}
return kr;
}
static kern_return_t shim_IOConnectUnmapMemory64(io_connect_t,uint32_t,task_port_t,mach_vm_address_t) {return KERN_SUCCESS;}
static kern_return_t shim_IOConnectUnmapMemory64(io_connect_t,uint32_t,task_port_t,mach_vm_address_t) {
return getenv("HOSTSIM_FAIL_USER_UNMAP") ? kIOReturnIOError : KERN_SUCCESS;
}
/* HOSTSIM_PAGE_SIZE=4096: Turnip sizes and places BOs with the page size (os_get_page_size, getpagesize), 16 KiB on this Mac and 4 KiB on the
* tablet, so a recording meant to match the tablet's GPU address layout reports the tablet's page size. */
static long pageSize(){static long p=-1;if(p<0)p=getenv("HOSTSIM_PAGE_SIZE")?atol(getenv("HOSTSIM_PAGE_SIZE")):0;return p;}
drivers/NativeAdreno/tests/hostsim/pipeline_cache.cpp
+129
View File
@@ -0,0 +1,129 @@
/* SPDX-License-Identifier: MIT */
/* Real Turnip shader compilation/cache import over the software kernel peer.
* No command is submitted and no shader is executed. */
#include <vulkan/vulkan.h>
#include "NativeTurnipShaders.h"
#include <chrono>
#include <cstdio>
#include <cstring>
#include <dlfcn.h>
#include <fstream>
#include <iterator>
#include <vector>
#define REQUIRE(x) do {if (!(x)) {std::fprintf(stderr,"FAIL line %d: %s\n",__LINE__,#x);return 1;}} while (0)
#define OK(x) REQUIRE((x)==VK_SUCCESS)
#define LOAD(name) auto name=(PFN_##name)get(instance,#name);REQUIRE(name)
int main(int argc,char **argv) {
if(argc!=4)return 2;
const bool produce=!std::strcmp(argv[3],"produce");
const bool mismatch=!std::strcmp(argv[3],"uuid-miss");
const bool shaderMismatch=!std::strcmp(argv[3],"shader-miss");
REQUIRE(produce||mismatch||shaderMismatch||!std::strcmp(argv[3],"consume"));
void *library=dlopen(argv[1],RTLD_NOW|RTLD_LOCAL);
if(!library)std::fprintf(stderr,"dlopen: %s\n",dlerror());
REQUIRE(library);
auto get=(PFN_vkGetInstanceProcAddr)dlsym(library,"vk_icdGetInstanceProcAddr");REQUIRE(get);
auto submissions=(unsigned(*)())dlsym(RTLD_DEFAULT,"host_gpu_submissions");
auto live=(unsigned(*)())dlsym(RTLD_DEFAULT,"host_gpu_live_objects");REQUIRE(submissions&&live);
VkInstance instance=VK_NULL_HANDLE;
LOAD(vkCreateInstance);
VkApplicationInfo app={.sType=VK_STRUCTURE_TYPE_APPLICATION_INFO,.pApplicationName="native pipeline cache control",.apiVersion=VK_API_VERSION_1_2};
VkInstanceCreateInfo ii={.sType=VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO,.pApplicationInfo=&app};
OK(vkCreateInstance(&ii,nullptr,&instance));
LOAD(vkEnumeratePhysicalDevices);LOAD(vkGetPhysicalDeviceProperties);LOAD(vkGetPhysicalDeviceFeatures2);
LOAD(vkGetPhysicalDeviceQueueFamilyProperties);LOAD(vkEnumerateDeviceExtensionProperties);
LOAD(vkCreateDevice);LOAD(vkDestroyDevice);LOAD(vkDestroyInstance);LOAD(vkDeviceWaitIdle);
LOAD(vkCreateShaderModule);LOAD(vkDestroyShaderModule);
LOAD(vkCreateDescriptorSetLayout);LOAD(vkDestroyDescriptorSetLayout);
LOAD(vkCreatePipelineLayout);LOAD(vkDestroyPipelineLayout);
LOAD(vkCreatePipelineCache);LOAD(vkGetPipelineCacheData);LOAD(vkDestroyPipelineCache);
LOAD(vkCreateComputePipelines);LOAD(vkDestroyPipeline);
uint32_t count=1;VkPhysicalDevice physical;OK(vkEnumeratePhysicalDevices(instance,&count,&physical));REQUIRE(count==1);
VkPhysicalDeviceProperties properties={};vkGetPhysicalDeviceProperties(physical,&properties);
count=0;OK(vkEnumerateDeviceExtensionProperties(physical,nullptr,&count,nullptr));
std::vector<VkExtensionProperties> listed(count);OK(vkEnumerateDeviceExtensionProperties(physical,nullptr,&count,listed.data()));
const char *extensions[]={VK_EXT_PIPELINE_CREATION_CACHE_CONTROL_EXTENSION_NAME,VK_EXT_PIPELINE_CREATION_FEEDBACK_EXTENSION_NAME};
for(auto name:extensions){bool found=false;for(auto &e:listed)found|=!std::strcmp(e.extensionName,name);REQUIRE(found);}
VkPhysicalDevicePipelineCreationCacheControlFeatures control={.sType=VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PIPELINE_CREATION_CACHE_CONTROL_FEATURES};
VkPhysicalDeviceFeatures2 features={.sType=VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2,.pNext=&control};
vkGetPhysicalDeviceFeatures2(physical,&features);REQUIRE(control.pipelineCreationCacheControl);
VkQueueFamilyProperties families[8];count=8;vkGetPhysicalDeviceQueueFamilyProperties(physical,&count,families);
int family=-1;for(uint32_t i=0;i<count;i++)if(families[i].queueFlags&VK_QUEUE_COMPUTE_BIT){family=(int)i;break;}REQUIRE(family>=0);
float priority=1;
VkDeviceQueueCreateInfo qi={.sType=VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO,.queueFamilyIndex=(uint32_t)family,.queueCount=1,.pQueuePriorities=&priority};
VkDeviceCreateInfo di={.sType=VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO,.pNext=&control,.queueCreateInfoCount=1,.pQueueCreateInfos=&qi,
.enabledExtensionCount=2,.ppEnabledExtensionNames=extensions};
VkDevice device;OK(vkCreateDevice(physical,&di,nullptr,&device));
const unsigned submitted=submissions();
VkDescriptorSetLayoutBinding binding={.binding=0,.descriptorType=VK_DESCRIPTOR_TYPE_STORAGE_BUFFER,.descriptorCount=1,.stageFlags=VK_SHADER_STAGE_COMPUTE_BIT};
VkDescriptorSetLayoutCreateInfo si={.sType=VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,.bindingCount=1,.pBindings=&binding};
VkDescriptorSetLayout set;OK(vkCreateDescriptorSetLayout(device,&si,nullptr,&set));
VkPushConstantRange push={.stageFlags=VK_SHADER_STAGE_COMPUTE_BIT,.offset=0,.size=8};
VkPipelineLayoutCreateInfo li={.sType=VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO,.setLayoutCount=1,.pSetLayouts=&set,.pushConstantRangeCount=1,.pPushConstantRanges=&push};
VkPipelineLayout layout;OK(vkCreatePipelineLayout(device,&li,nullptr,&layout));
std::vector<uint32_t> spirv(std::begin(kSpv_smoke_fill_comp),std::end(kSpv_smoke_fill_comp));
if(shaderMismatch){
unsigned changed=0;
for(size_t i=5;i<spirv.size();){
unsigned words=spirv[i]>>16;REQUIRE(words&&i+words<=spirv.size());
// Own fixture: change OpExecutionMode LocalSize from 64x1x1 to 32x1x1.
if(spirv[i]==0x00060010&&spirv[i+2]==17){REQUIRE(spirv[i+3]==64);spirv[i+3]=32;++changed;}
i+=words;
}
REQUIRE(changed==1);
}
VkShaderModuleCreateInfo mi={.sType=VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO,.codeSize=spirv.size()*sizeof(uint32_t),.pCode=spirv.data()};
VkShaderModule module;OK(vkCreateShaderModule(device,&mi,nullptr,&module));
std::vector<char> data;
if(!produce){
std::ifstream input(argv[2],std::ios::binary);REQUIRE(input);
data.assign(std::istreambuf_iterator<char>(input),{});REQUIRE(data.size()>32);
uint32_t words[4];std::memcpy(words,data.data(),sizeof words);
REQUIRE(words[0]==32&&words[1]==VK_PIPELINE_CACHE_HEADER_VERSION_ONE);
REQUIRE(words[2]==properties.vendorID&&words[3]==properties.deviceID);
REQUIRE(!std::memcmp(data.data()+16,properties.pipelineCacheUUID,VK_UUID_SIZE));
if(mismatch)data[16]^=1;
}
VkPipelineCacheCreateInfo ci={.sType=VK_STRUCTURE_TYPE_PIPELINE_CACHE_CREATE_INFO,.initialDataSize=data.size(),.pInitialData=data.data()};
VkPipelineCache cache;OK(vkCreatePipelineCache(device,&ci,nullptr,&cache));
VkPipelineCreationFeedback whole={},stage={};
VkPipelineCreationFeedbackCreateInfo feedback={.sType=VK_STRUCTURE_TYPE_PIPELINE_CREATION_FEEDBACK_CREATE_INFO,
.pPipelineCreationFeedback=&whole,.pipelineStageCreationFeedbackCount=1,.pPipelineStageCreationFeedbacks=&stage};
VkComputePipelineCreateInfo pi={.sType=VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO,.pNext=&feedback,
.flags=VK_PIPELINE_CREATE_FAIL_ON_PIPELINE_COMPILE_REQUIRED_BIT,
.stage={.sType=VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,.stage=VK_SHADER_STAGE_COMPUTE_BIT,.module=module,.pName="main"},
.layout=layout};
VkPipeline pipeline=VK_NULL_HANDLE;
auto began=std::chrono::steady_clock::now();
VkResult strict=vkCreateComputePipelines(device,cache,1,&pi,nullptr,&pipeline);
double strictUs=std::chrono::duration<double,std::micro>(std::chrono::steady_clock::now()-began).count();
double compileUs=0;
if(produce||mismatch||shaderMismatch){
REQUIRE(strict==VK_PIPELINE_COMPILE_REQUIRED&&pipeline==VK_NULL_HANDLE);
if(produce){
pi.flags=0;whole={};stage={};began=std::chrono::steady_clock::now();
OK(vkCreateComputePipelines(device,cache,1,&pi,nullptr,&pipeline));
compileUs=std::chrono::duration<double,std::micro>(std::chrono::steady_clock::now()-began).count();
REQUIRE(whole.flags&VK_PIPELINE_CREATION_FEEDBACK_VALID_BIT);
REQUIRE(!(whole.flags&VK_PIPELINE_CREATION_FEEDBACK_APPLICATION_PIPELINE_CACHE_HIT_BIT));
size_t bytes=0;OK(vkGetPipelineCacheData(device,cache,&bytes,nullptr));REQUIRE(bytes>32);
data.resize(bytes);OK(vkGetPipelineCacheData(device,cache,&bytes,data.data()));
std::ofstream output(argv[2],std::ios::binary);REQUIRE(output);output.write(data.data(),bytes);REQUIRE(output.good());
}
}else{
REQUIRE(strict==VK_SUCCESS&&pipeline!=VK_NULL_HANDLE);
REQUIRE((whole.flags&(VK_PIPELINE_CREATION_FEEDBACK_VALID_BIT|VK_PIPELINE_CREATION_FEEDBACK_APPLICATION_PIPELINE_CACHE_HIT_BIT))==
(VK_PIPELINE_CREATION_FEEDBACK_VALID_BIT|VK_PIPELINE_CREATION_FEEDBACK_APPLICATION_PIPELINE_CACHE_HIT_BIT));
}
if(pipeline)vkDestroyPipeline(device,pipeline,nullptr);
vkDestroyPipelineCache(device,cache,nullptr);vkDestroyShaderModule(device,module,nullptr);
vkDestroyPipelineLayout(device,layout,nullptr);vkDestroyDescriptorSetLayout(device,set,nullptr);
OK(vkDeviceWaitIdle(device));REQUIRE(submissions()==submitted);
vkDestroyDevice(device,nullptr);vkDestroyInstance(instance,nullptr);REQUIRE(live()==0);
char uuid[33]={};for(unsigned i=0;i<16;i++)std::snprintf(uuid+2*i,3,"%02x",properties.pipelineCacheUUID[i]);
std::printf("PASS: Turnip pipeline cache %s; uuid=%s bytes=%zu strict=%d feedback=0x%x strict_us=%.1f compile_us=%.1f; no shader execution\n",
argv[3],uuid,data.size(),strict,whole.flags,strictUs,compileUs);
return 0;
}
drivers/NativeAdreno/tests/hostsim/pipeline_cache_ios.cpp
+130
View File
@@ -0,0 +1,130 @@
/* SPDX-License-Identifier: MIT */
/* QEMU-only compiler/cache instrument. The unchanged iOS Turnip library talks
* to the authored software peer, never to a physical GPU. Separate children
* preserve cold process boundaries; the worker normally links the own peer,
* and original dyld/AMFI decide loading and interposition. */
#define main pipeline_cache_case_main
#include "pipeline_cache.cpp"
#undef main
#include <spawn.h>
#include <sys/wait.h>
#include <unistd.h>
#include <cerrno>
#include <cstdlib>
#include <fcntl.h>
#include <sys/mount.h>
#include <sys/stat.h>
#include <CoreFoundation/CoreFoundation.h>
#include <IOKit/IOKitLib.h>
extern char **environ;
static uint64_t result[48];
alignas(4096) static unsigned char snapshot[65536];
alignas(4096) static unsigned char workerOutput[4096];
static const char *workerLog="/private/var/tmp/turnip-compiler-worker.log";
static const char *driverPath="/System/Library/Extensions/OfflineTurnipCompiler/libvulkan_freedreno.dylib";
static void stop(unsigned failure) {
result[1]=failure;
if(FILE *file=fopen(workerLog,"rb")) {
result[11]=fread(workerOutput,1,sizeof workerOutput-1,file);
fclose(file);
}
__asm__ volatile("mov x13, %0\nmov x14, %1\nmov x15, %2\nbrk #0"::"r"(result),"r"(snapshot),"r"(workerOutput):"x13","x14","x15","memory");
_exit(99);
}
static void mountCompilerView() {
char uuid[37]={};FILE *file=fopen("/private/var/tmp/compiler-data-uuid","rb");
if(!file)stop(60);
size_t count=fread(uuid,1,37,file);fclose(file);
if(count!=36 || uuid[36])stop(61);
CFStringRef identity=CFStringCreateWithCString(nullptr,uuid,kCFStringEncodingASCII);
CFMutableDictionaryRef match=IOServiceMatching("IOMedia");
if(!identity || !match)stop(62);
const void *keys[]={CFSTR("UUID")},*values[]={identity};
CFDictionaryRef properties=CFDictionaryCreate(nullptr,keys,values,1,&kCFTypeDictionaryKeyCallBacks,&kCFTypeDictionaryValueCallBacks);
if(!properties)stop(63);
CFDictionarySetValue(match,CFSTR("IOPropertyMatch"),properties);CFRelease(properties);
io_iterator_t iterator=0;
if(IOServiceGetMatchingServices(0,match,&iterator) || !iterator)stop(64);
char device[64]={};unsigned found=0;
while(io_service_t media=IOIteratorNext(iterator)) {
++found;
CFTypeRef observed=IORegistryEntryCreateCFProperty(media,CFSTR("UUID"),nullptr,0);
CFTypeRef name=IORegistryEntryCreateCFProperty(media,CFSTR("BSD Name"),nullptr,0);
if(!observed || !CFEqual(observed,identity) || !name || CFGetTypeID(name)!=CFStringGetTypeID())stop(65);
char bsd[56]={};
if(!CFStringGetCString((CFStringRef)name,bsd,sizeof bsd,kCFStringEncodingASCII) || strncmp(bsd,"disk",4))stop(66);
for(const char *p=bsd;*p;++p)if(!((*p>='a'&&*p<='z')||(*p>='0'&&*p<='9')))stop(67);
bool backed=false;io_registry_entry_t node=media;
for(unsigned depth=0;node && depth<64;++depth) {
if(IOObjectConformsTo(node,"NativeRamBlock")){backed=true;break;}
io_registry_entry_t above=0;auto error=IORegistryEntryGetParentEntry(node,"IOService",&above);
if(node!=media)IOObjectRelease(node);
node=error?0:above;
}
if(node && node!=media)IOObjectRelease(node);
if(!backed)stop(68);
snprintf(device,sizeof device,"/dev/%s",bsd);
CFRelease(observed);CFRelease(name);IOObjectRelease(media);
}
IOObjectRelease(iterator);CFRelease(identity);
if(found!=1 || !device[0])stop(69);
const char *point="/System/Library/Extensions",*directory="/private/var/tmp/native-compiler-addons";
struct stat block={},source={},before={},after={},original={},code={};struct statfs fs={};
if(stat(device,&block) || !S_ISBLK(block.st_mode) || stat(point,&before) || stat(directory,&source) || source.st_dev!=block.st_rdev)stop(70);
if(mount("bindfs",point,MNT_RDONLY|MNT_UNION,(void *)directory))stop(71);
if(statfs(point,&fs) || (fs.f_flags&(MNT_RDONLY|MNT_UNION))!=(MNT_RDONLY|MNT_UNION) ||
strcmp(fs.f_mntfromname,directory) || strcmp(fs.f_fstypename,"bindfs") || stat(point,&after) ||
after.st_dev==before.st_dev || stat(driverPath,&code) || code.st_dev!=after.st_dev ||
stat("/System/Library/Extensions/AGXMetalA10.bundle/Info.plist",&original) || original.st_dev!=before.st_dev)stop(72);
result[38]=fs.f_flags;result[39]=(uint32_t)before.st_dev;result[40]=(uint32_t)after.st_dev;
result[41]=(uint32_t)original.st_dev;result[42]=(uint32_t)code.st_dev;result[43]=found;result[44]=1;
}
int main(int argc,char **argv) {
if(argc==4) {
// Never emulate a page-size answer in the target OS process.
if(getpagesize()!=4096 || sysconf(_SC_PAGESIZE)!=4096 || getenv("HOSTSIM_PAGE_SIZE"))return 92;
if(!strcmp(argv[3],"startup-control"))return 0;
return pipeline_cache_case_main(argc,argv);
}
result[0]=0x54435051494f5331ull;
result[2]=(uint64_t)getpid();result[3]=(uint64_t)getpagesize();result[4]=(uint64_t)sysconf(_SC_PAGESIZE);
if(argc!=1 || getpid()!=1 || result[3]!=4096 || result[4]!=4096)stop(1);
mountCompilerView();
if(setenv("MESA_SHADER_CACHE_DISABLE","true",1) || unsetenv("DYLD_INSERT_LIBRARIES") || unsetenv("HOSTSIM_PAGE_SIZE"))stop(2);
if(unlink("/private/var/tmp/turnip-compiler-cache.bin") && errno!=ENOENT)stop(3);
if(unlink(workerLog) && errno!=ENOENT)stop(5);
posix_spawn_file_actions_t actions;
if(posix_spawn_file_actions_init(&actions) ||
posix_spawn_file_actions_addopen(&actions,STDOUT_FILENO,workerLog,O_WRONLY|O_CREAT|O_APPEND,0600) ||
posix_spawn_file_actions_adddup2(&actions,STDOUT_FILENO,STDERR_FILENO))stop(4);
const char *modes[]={"produce","consume","uuid-miss","shader-miss"};
for(unsigned i=0;i<4;++i) {
char *arguments[]={const_cast<char *>("compiler-worker"),const_cast<char *>(driverPath),
const_cast<char *>("/private/var/tmp/turnip-compiler-cache.bin"),const_cast<char *>(modes[i]),nullptr};
pid_t child=0;
int error=posix_spawn(&child,"/private/var/tmp/compiler-worker",&actions,nullptr,arguments,environ);
result[12+3*i]=(uint64_t)child;result[13+3*i]=(uint64_t)error;
if(error || child<=1)stop(10+i);
int status=0;
while(waitpid(child,&status,0)<0)if(errno!=EINTR)stop(20+i);
result[14+3*i]=(uint64_t)status;
if(!WIFEXITED(status) || WEXITSTATUS(status)!=0)stop(30+i);
result[5]=i+1;
}
posix_spawn_file_actions_destroy(&actions);
FILE *file=fopen("/private/var/tmp/turnip-compiler-cache.bin","rb");
if(!file)stop(40);
const size_t bytes=fread(snapshot,1,sizeof(snapshot),file);
const bool ok=!ferror(file) && bytes>32 && bytes<sizeof(snapshot) && feof(file);
fclose(file);
if(!ok)stop(41);
result[6]=bytes;
uint32_t header[4];memcpy(header,snapshot,sizeof header);
result[7]=header[0];result[8]=header[1];result[9]=header[2];result[10]=header[3];
memcpy(result+24,snapshot+16,VK_UUID_SIZE);
stop(0);
}
drivers/NativeAdreno/tests/test_submit.cpp
+69 -2
View File
@@ -79,7 +79,7 @@ static void smmuReset(void)
for(unsigned i=0;i<8;++i)mmio[(0xc00+4*i)/4]=0x20000;
mmio[0x800/4]=1u<<31|9;mmio[0xc00/4]=3;
}
static int exposed(Path &p)
static int exposed(Path &p,bool write_combined=false)
{
smmuReset();cp=CpPeer{};clock_us=0;
std::memset(ringmem,0xef,sizeof(ringmem));std::memset(fencemem,0,sizeof(fencemem));
@@ -100,6 +100,7 @@ static int exposed(Path &p)
NativeAdrenoVmRegion lower={&p.tables,&p.arena,&p.state};
if(p.vm.attach(&smmuio,&context,&streams,&lower,nullptr,32,1000))return kIOReturnBadArgument;
if(p.object.allocate(8192,40,false,8192))return kIOReturnNoMemory;
if(write_combined && p.object.makeWriteCombined())return kIOReturnNoMemory;
if(p.vm.map(&p.object,0,0x800000,3,0,2,&p.handle))return kIOReturnBadArgument;
if(p.ring.attach(&ringio,&ringcache,ringmem,sizeof(ringmem),0x100000,fencemem,sizeof(fencemem),0x108000))
return kIOReturnBadArgument;
@@ -147,6 +148,62 @@ int main()
CHECK(p.submit.wait(2,&ringclock,1000)==kIOReturnSuccess);
CHECK(p.submit.detach()==kIOReturnSuccess);
}
/* int-100: a write-combined BO is not maintained while the IB audit finds no dirty line; a dirty line or a failed audit ends the
* trust, and that submission already maintains every BO through its physical pages. */
{
Path p;CHECK(exposed(p,true)==kIOReturnSuccess);
NativeAdrenoSubmitBuffer buffer{p.handle,0x800000,4096};
static std::vector<uint64_t> audited;static uint32_t dirty_lines,fail;
audited.clear();dirty_lines=0;fail=0;
native_adreno_line_audit=[](uint64_t physical,size_t bytes,uint32_t *dirty,uint32_t *stale) {
audited.push_back(physical);audited.push_back(bytes);
if(fail)return 1;
*dirty+=dirty_lines;*stale+=1;return 0;};
const uint32_t submissions=native_adreno_wc_audit[0],lines=native_adreno_wc_audit[1];
const unsigned physical=fixture.physical_calls,owned=fixture.cache_calls;
uint32_t fence=0;
CHECK(p.submit.submit(&buffer,1,0,&fence)==kIOReturnSuccess && fence==1);
CHECK(audited.size()==2 && audited[0]==p.object.spans()[0].physical && audited[1]==4096);
CHECK(native_adreno_wc_audit[0]==submissions+1 && native_adreno_wc_audit[1]==lines+64 && native_adreno_wc_distrusted==0);
CHECK(fixture.physical_calls==physical && fixture.cache_calls==owned && p.vm.writeCombinedTrusted()); /* no maintenance */
fencemem[1]=1;CHECK(p.submit.wait(1,&ringclock,1000)==kIOReturnSuccess);
dirty_lines=2;
CHECK(p.submit.submit(&buffer,1,0,&fence)==kIOReturnSuccess && fence==2);
CHECK(!p.vm.writeCombinedTrusted() && native_adreno_wc_distrusted==native_adreno_coherence[0]);
CHECK(fixture.physical_calls>physical); /* this submission maintained the write-combined BO */
const size_t calls=audited.size();
fencemem[1]=2;CHECK(p.submit.wait(2,&ringclock,1000)==kIOReturnSuccess);
CHECK(p.submit.submit(&buffer,1,0,&fence)==kIOReturnSuccess && audited.size()==calls); /* no audit once distrusted */
fencemem[1]=3;CHECK(p.submit.wait(3,&ringclock,1000)==kIOReturnSuccess);
CHECK(p.submit.detach()==kIOReturnSuccess);
}
/* The audit compares the first 16 submissions after attach, then every eighth (17, 25, ...); the re-assertion runs at each. */
{
Path p;CHECK(exposed(p,true)==kIOReturnSuccess);
NativeAdrenoSubmitBuffer buffer{p.handle,0x800000,4096};
static unsigned calls;calls=0;
native_adreno_line_audit=[](uint64_t,size_t,uint32_t *,uint32_t *stale) {++calls;*stale+=0;return 0;};
const uint32_t submissions=native_adreno_wc_audit[0];
for(uint32_t n=1;n<=24;++n) {
uint32_t fence=0;
CHECK(p.submit.submit(&buffer,1,0,&fence)==kIOReturnSuccess && fence==n);
fencemem[1]=n;CHECK(p.submit.wait(n,&ringclock,1000)==kIOReturnSuccess);
}
CHECK(calls==17 && native_adreno_wc_audit[0]==submissions+17 && p.vm.writeCombinedTrusted());
native_adreno_line_audit=nullptr;
CHECK(p.submit.detach()==kIOReturnSuccess);
}
{
Path p;CHECK(exposed(p,true)==kIOReturnSuccess);
NativeAdrenoSubmitBuffer buffer{p.handle,0x800000,4096};
native_adreno_line_audit=[](uint64_t,size_t,uint32_t *,uint32_t *) {return 1;};
uint32_t fence=0;
CHECK(p.submit.submit(&buffer,1,0,&fence)==kIOReturnSuccess && fence==1);
CHECK(!p.vm.writeCombinedTrusted() && native_adreno_wc_audit[4]==kIOReturnIOError);
native_adreno_line_audit=nullptr;native_adreno_wc_distrusted=0;
fencemem[1]=1;CHECK(p.submit.wait(1,&ringclock,1000)==kIOReturnSuccess);
CHECK(p.submit.detach()==kIOReturnSuccess);
}
/* Refusals precede references, ring bytes and WPTR updates. */
{
Path p;CHECK(exposed(p)==kIOReturnSuccess);
@@ -197,5 +254,15 @@ int main()
CHECK(submit.detach()==kIOReturnNotReady);
CHECK(submit.submit(nullptr,1,0,nullptr)==kIOReturnBadArgument);
}
std::printf("PASS: 1 execbuf submission with exact IB packet and fence, 8 pre-reference refusals, 1 duplicate-buffer reference case and 1 poisoned-commit retention case over software SMMU/CP peers\n");
std::printf("PASS: 1 execbuf submission with exact IB packet and fence, 8 pre-reference refusals, 1 duplicate-buffer reference case and 1 poisoned-commit retention case over software SMMU/CP peers; 2 write-combine audit cases (trusted, dirty and failed)\n");
}
/* Physical-page maintenance (imported BOs): record it; no host page is touched. */
/* Host double: the page attribute change of user BOs has no host equivalent (no host page is touched). */
IOReturn NativeAdrenoSetPagesCacheMode(const NativeAdrenoPhysicalSpan *spans,size_t count,bool){return spans && count?kIOReturnSuccess:kIOReturnBadArgument;}
IOReturn NativeAdrenoReassertWriteCombined(const NativeAdrenoPhysicalSpan *spans,size_t count,uint32_t *reset)
{return spans && count && reset?kIOReturnSuccess:kIOReturnBadArgument;}
IOReturn NativeAdrenoMaintainPhysicalPages(uint64_t physical,size_t bytes,NativeAdrenoCacheOperation op)
{
if(!physical || !bytes || ((physical|bytes)&4095) || op>NativeAdrenoInvalidate)std::abort();
++fixture.physical_calls;fixture.physical_bytes+=bytes;return kIOReturnSuccess;
}
drivers/NativeAdreno/tests/test_vm.cpp
+94 -5
View File
@@ -3,6 +3,7 @@
* Proves ownership/lifetime/refusal logic only; no hardware is touched. */
#include "XnuMemoryFixture.h"
#include "NativeAdrenoVm.h"
#include <chrono>
#include <cstdio>
#define CHECK(x) do {if(!(x)){std::fprintf(stderr,"line %d: %s\n",__LINE__,#x);std::exit(1);}}while(0)
#define BYTES 0x40000
@@ -24,6 +25,8 @@ static ArmSmmu4KStream routes[2]={{2,0},{4,1}};
static ArmSmmu4KStreamResult route_result;
static ArmSmmu4KContextConfig context;
static ArmSmmu4KContextImage image;
extern "C" {extern uint64_t (*native_adreno_vm_now_us)(void);extern volatile uint64_t native_adreno_vm_phase_us[2][8];}
static uint64_t nowUs() {return (uint64_t)std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::steady_clock::now().time_since_epoch()).count();}
static int event(void){return ++f.calls==f.fail_call;}
static int read32(void *cookie,size_t at,uint32_t *v)
{CHECK(cookie==&f && at%4==0 && at<=BYTES-4);if(event())return 1;*v=memory[at/4];return 0;}
@@ -43,6 +46,7 @@ struct Fixture {
ArmSmmu4KLiveState state{};
ArmSmmu4KLiveVisibility visibility;
NativeAdrenoVm vm;
std::vector<uint32_t> aliasScratch;
};
static void mmioReset(void)
{
@@ -54,11 +58,12 @@ static void mmioReset(void)
memory[0x800/4]=1u<<31|9;memory[0xc00/4]=3;
}
/* Real prepared arena, enabled actual context image and exact stream routes. */
static IOReturn exposed(Fixture &s)
static IOReturn exposed(Fixture &s,size_t capacity=32,size_t pages=16)
{
mmioReset();
IOReturn status=s.tables.allocate(32*4096,40,true,32*4096);if(status)return status;
IOReturn smmu=NativeAdrenoPrepareSmmuMemory(&s.tables,32,40,0x800000,65536,0,16,
const size_t tablePages=std::max(size_t(32),pages/512+8);
IOReturn status=s.tables.allocate(tablePages*4096,40,true,tablePages*4096);if(status)return status;
IOReturn smmu=NativeAdrenoPrepareSmmuMemory(&s.tables,32,40,0x800000,pages*4096,0,pages,
&s.arena,&s.state,&s.visibility);
if(smmu)return smmu;
context=(ArmSmmu4KContextConfig){.lower=&s.arena.config,.lower_tables=&s.state.tables,
@@ -74,7 +79,7 @@ static IOReturn exposed(Fixture &s)
if(ArmSmmu4KAttachStreams(&io,&streams,routes,2,2,&context,&route_result)!=ArmSmmu4KSuccess)
return kIOReturnBadArgument;
NativeAdrenoVmRegion lower={&s.tables,&s.arena,&s.state};
return s.vm.attach(&io,&context,&streams,&lower,nullptr,32,1000);
return s.vm.attach(&io,&context,&streams,&lower,nullptr,capacity,1000);
}
static IOReturn object(Fixture &s,size_t pages)
{return s.object.allocate(pages*4096,40,false,pages*4096);}
@@ -133,6 +138,80 @@ int main()
CHECK(s.tables.retireAfterQuiescence()==kIOReturnSuccess && s.tables.release()==kIOReturnSuccess);
CHECK(fixture.live.empty() && fixture.metadata.empty());
}
/* Scattered BOs, one span per page, mapped in one transaction each into the middle, the end and the start of the table
* (each map and unmap audits the whole table against the sorted spans), and a span-capacity refusal before ownership. */
{
Fixture s;
CHECK(exposed(s,8)==kIOReturnSuccess); /* aperture 0x800000..0x810000, 8 spans */
fixture.mode=Scattered;
NativeAdrenoMemory first,middle,last,huge;
CHECK(!first.allocate(4096,40,false,4096) && !middle.allocate(4*4096,40,false,4*4096) && !last.allocate(4096,40,false,4096));
CHECK(middle.spanCount()==4);
uint64_t a=0,b=0,c=0,iova=0;size_t bytes=0;unsigned permissions=0;
CHECK(!s.vm.map(&first,0,0x800000,3,0,2,&a) && !s.vm.map(&last,0,0x808000,3,0,2,&c));
CHECK(!s.vm.map(&middle,0,0x802000,3,0,2,&b) && s.vm.spanCount()==6 && s.vm.lastResult().completed_spans==4);
CHECK(!s.vm.describe(b,&iova,&bytes,&permissions) && iova==0x802000 && bytes==4*4096);
CHECK(!s.vm.unmap(b) && s.vm.spanCount()==2 && s.vm.lastResult().completed_spans==4);
CHECK(!s.vm.map(&middle,0,0x80a000,3,0,2,&b) && s.vm.spanCount()==6);
CHECK(!s.vm.unmap(a) && s.vm.spanCount()==5);
CHECK(!s.vm.map(&first,0,0x800000,3,0,2,&a) && s.vm.spanCount()==6);
CHECK(!huge.allocate(3*4096,40,false,3*4096) && huge.spanCount()==3);
uint64_t refused=0;const unsigned calls=f.calls;
CHECK(s.vm.map(&huge,0,0x802000,3,0,2,&refused)==kIOReturnNoResources && !refused);
CHECK(!huge.deviceReferences() && f.calls==calls && s.vm.spanCount()==6 && !s.vm.poisoned());
CHECK(!s.vm.unmap(a) && !s.vm.unmap(b) && !s.vm.unmap(c) && s.vm.spanCount()==0);
fixture.mode=Contiguous;
CHECK(!first.release() && !middle.release() && !last.release() && !huge.release());
CHECK(s.vm.detach()==kIOReturnSuccess);
CHECK(s.tables.retireAfterQuiescence()==kIOReturnSuccess && s.tables.release()==kIOReturnSuccess);
CHECK(fixture.live.empty() && fixture.metadata.empty());
}
/* Physical overlap remains allowed only within the same attribute class.
* Check scratch and fallback, both smaller-side choices, and refusals before
* MMIO/ownership changes. These are physical-page identities in the peer. */
for(bool scratch:{false,true}) {
Fixture s;CHECK(exposed(s)==kIOReturnSuccess);
s.aliasScratch.resize(32);
if(scratch){s.arena.alias_scratch=s.aliasScratch.data();s.arena.alias_scratch_entries=s.aliasScratch.size();}
NativeAdrenoMemory first,second,many;
CHECK(!first.allocate(4096,40,false,4096));
fixture.next_physical=first.spans()[0].physical;
CHECK(!second.allocate(4096,40,false,4096));
fixture.mode=Scattered;fixture.next_physical=first.spans()[0].physical;
CHECK(!many.allocate(8192,40,false,8192)&&many.spanCount()==2);fixture.mode=Contiguous;
uint64_t a=0,b=0;CHECK(!s.vm.map(&first,0,0x800000,3,0,2,&a));
CHECK(!s.vm.map(&second,0,0x802000,1,0,2,&b));CHECK(!s.vm.unmap(b));
const unsigned calls=f.calls;
for(unsigned change=0;change<3;++change) {
CHECK(s.vm.map(&second,0,0x802000,change==2?9:3,change==0?1:0,change==1?3:2,&b)==kIOReturnBadArgument);
CHECK(!b&&!second.deviceReferences()&&f.calls==calls&&!s.vm.poisoned());
}
CHECK(s.vm.map(&many,0,0x802000,3,1,3,&b)==kIOReturnBadArgument);
CHECK(!b&&!many.deviceReferences()&&f.calls==calls&&!s.vm.poisoned());
CHECK(!s.vm.unmap(a)&&!first.release()&&!second.release()&&!many.release());
CHECK(!s.vm.detach()&&!s.tables.retireAfterQuiescence()&&!s.tables.release());
CHECK(fixture.live.empty()&&fixture.metadata.empty());
}
/* A large different-class scattered map: the old preflight did n*n physical
* comparisons despite the table updater already having alias scratch. */
{
constexpr size_t n=8192;
Fixture s;CHECK(exposed(s,2*n,2*n)==kIOReturnSuccess);
s.aliasScratch.resize(2*n);s.arena.alias_scratch=s.aliasScratch.data();s.arena.alias_scratch_entries=s.aliasScratch.size();
fixture.mode=Scattered;NativeAdrenoMemory first,second;
CHECK(!first.allocate(n*4096,40,false,n*4096)&&first.spanCount()==n);
CHECK(!second.allocate(n*4096,40,false,n*4096)&&second.spanCount()==n);
uint64_t a=0,b=0;CHECK(!s.vm.map(&first,0,0x800000,3,0,2,&a));
native_adreno_vm_now_us=nowUs;uint64_t before=native_adreno_vm_phase_us[0][5];
CHECK(!s.vm.map(&second,0,0x800000+n*4096,3,1,3,&b));
std::printf("VM alias preflight diagnostic: %zu x %zu different-class spans, %llu host us (not hardware timing)\n",n,n,
(unsigned long long)(native_adreno_vm_phase_us[0][5]-before));
native_adreno_vm_now_us=nullptr;fixture.mode=Contiguous;
CHECK(s.vm.spanCount()==2*n&&!s.vm.unmap(b)&&!s.vm.unmap(a)&&!first.release()&&!second.release());
CHECK(!s.vm.detach()&&!s.tables.retireAfterQuiescence()&&!s.tables.release());
CHECK(fixture.live.empty()&&fixture.metadata.empty());
}
std::printf("PASS: VM alias index/fallback preserve attribute-class refusals before MMIO/ownership; 8192-span different-class map/unmap\n");
/* Post-hold CPU publication failure: poison, retain everything, refuse. */
{
Fixture s;
@@ -150,5 +229,15 @@ int main()
CHECK(s.object.release()==kIOReturnBusy && s.tables.release()==kIOReturnBusy);
CHECK(!fixture.live.empty());
}
std::printf("PASS: 1 native map/hold/unmap transaction, 13 pre-ownership refusals and 1 post-hold poison/retention case over a software SMMU register peer\n");
std::printf("PASS: 1 native map/hold/unmap transaction, 13 pre-ownership refusals, 1 scattered-span ordering and capacity case and 1 post-hold poison/retention case over a software SMMU register peer\n");
}
/* Physical-page maintenance (imported BOs): record it; no host page is touched. */
/* Host double: the page attribute change of user BOs has no host equivalent (no host page is touched). */
IOReturn NativeAdrenoSetPagesCacheMode(const NativeAdrenoPhysicalSpan *spans,size_t count,bool){return spans && count?kIOReturnSuccess:kIOReturnBadArgument;}
IOReturn NativeAdrenoReassertWriteCombined(const NativeAdrenoPhysicalSpan *spans,size_t count,uint32_t *reset)
{return spans && count && reset?kIOReturnSuccess:kIOReturnBadArgument;}
IOReturn NativeAdrenoMaintainPhysicalPages(uint64_t physical,size_t bytes,NativeAdrenoCacheOperation op)
{
if(!physical || !bytes || ((physical|bytes)&4095) || op>NativeAdrenoInvalidate)std::abort();
++fixture.physical_calls;fixture.physical_bytes+=bytes;return kIOReturnSuccess;
}
drivers/NativeAdreno/tests/test_vm_alias.cpp
+46
View File
@@ -0,0 +1,46 @@
/* SPDX-License-Identifier: MIT */
#include "NativeAdrenoVmAlias.h"
#include <algorithm>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <vector>
#define CHECK(x) do {if(!(x)){std::fprintf(stderr,"line %d: %s\n",__LINE__,#x);std::exit(1);}}while(0)
struct Range {uint64_t physical,bytes;};
static uint64_t randomState=0x28372934;
static uint64_t next() {randomState^=randomState<<13;randomState^=randomState>>7;randomState^=randomState<<17;return randomState;}
static bool oracle(const std::vector<Range> &ranges,uint64_t start,uint64_t bytes) {
for(const auto &r:ranges)if(start<r.physical+r.bytes && r.physical<start+bytes)return true;
return false;
}
int main() {
unsigned cases=0,queries=0;
for(unsigned profile=0;profile<2048;++profile) {
std::vector<Range> ranges;
const unsigned count=profile%65;
for(unsigned i=0;i<count;++i)ranges.push_back({(next()%128)*4096,(1+next()%64)*4096});
// Explicit nesting plus a gap catches a binary search using starts alone.
if(profile==1)ranges={{0x1000,0x10000},{0x2000,0x1000},{0x3000,0x1000},{0x20000,0x1000}};
if(profile==2)ranges={{0x1000,0x1000},{0x1000,0x1000},{0x2000,0x1000},{0x4000,0x1000}};
const auto original=ranges;
// Filtered index subsets exercise caller-provided nonconsecutive indices.
std::vector<uint32_t> scratch(ranges.size()+2,0xdeadbeef);
std::vector<Range> selected;
size_t n=0;
for(size_t i=0;i<ranges.size();++i)if(profile%3 || i%2){scratch[1+n++]=(uint32_t)i;selected.push_back(ranges[i]);}
NativeAdrenoPhysicalIndex index(ranges.data(),scratch.data()+1,n);
for(unsigned q=0;q<256;++q) {
uint64_t start=q*4096,bytes=(1+(q%7))*4096;
CHECK(index.overlaps(start,bytes)==oracle(selected,start,bytes));++queries;
}
CHECK(scratch.front()==0xdeadbeef && scratch.back()==0xdeadbeef);
CHECK(ranges.size()==original.size() && (!ranges.size() || !std::memcmp(ranges.data(),original.data(),ranges.size()*sizeof(Range))));
++cases;
}
// Scale well beyond the bounded oracle profiles: distinct spans with gaps.
std::vector<Range> ranges(65536);std::vector<uint32_t> order(ranges.size());
for(size_t i=0;i<ranges.size();++i){ranges[i]={0x100000+(ranges.size()-1-i)*8192,4096};order[i]=(uint32_t)i;}
NativeAdrenoPhysicalIndex index(ranges.data(),order.data(),order.size());
for(size_t i=0;i<ranges.size();++i){CHECK(index.overlaps(0x100000+i*8192,4096));CHECK(!index.overlaps(0x101000+i*8192,4096));}
std::printf("PASS: physical alias index %u oracle profiles / %u queries; 65536 reversed spans and exact gap boundaries\n",cases,queries);
}
drivers/NativeMetal/NativeBridgeMetalFrontend.h
+13
View File
@@ -11,5 +11,18 @@ extern "C" {
id native_bridge_metal_open(const char *loader, const char *cache, uint64_t registryID, void (*missing)(const char *, const char *));
// Bare AIR bitcode as a library (the bridge's fixtures; Metal itself accepts only metallibs).
id native_bridge_metal_library_with_air(const void *bytes, size_t length);
// One line for the plugin's status report: command buffers committed/failed locally, pipelines made, refusals, the last refusal's text.
size_t native_bridge_metal_status(char *out, size_t size);
// The pixel formats of the textures asked for (first eight distinct) and how many texture requests the runtime refused.
size_t native_bridge_metal_formats(char *out, size_t size);
size_t native_bridge_metal_telemetry(char *out, size_t size);
// Structured cumulative artifact/driver-cache counters for offline qualification.
struct NmbShaderCacheStats;
bool native_bridge_metal_shader_cache_stats(struct NmbShaderCacheStats *out);
bool native_bridge_metal_work_stats(uint64_t out[10]);
// The same fixed CPU, memory and pipeline workload on the tablet and the host, so their times compare directly (int-145): compute, first-touch
// page faults, memset bandwidth, CommonCrypto SHA-256, then the first `pairs` vertex/fragment pairs of the given QuartzCore metallib made into
// BGRA8 render states and drawn once each (pipeline_timing_host's draw=1 order). The results become the telemetry's Y rows; also returned.
size_t native_bridge_metal_benchmark(id device, const char *metallib, unsigned pairs, char *out, size_t size);
}
#endif
drivers/NativeMetal/NativeBridgeMetalFrontend.mm
+803 -91
View File
File diff suppressed because it is too large. Load diff
drivers/NativeMetal/bridge/NativeMetalObjects.h
+68
View File
@@ -13,6 +13,7 @@ extern "C" {
#endif
typedef struct NmbDevice NmbDevice;
typedef struct NmbBuffer NmbBuffer;
typedef struct NmbHeap NmbHeap;
typedef struct NmbTexture NmbTexture;
typedef struct NmbLibrary NmbLibrary;
typedef struct NmbFunction NmbFunction;
@@ -36,6 +37,28 @@ NmbDevice *nmb_device_open(const char *loader, const char *cache);
void nmb_device_release(NmbDevice *);
size_t nmb_device_name(const NmbDevice *, char *out, size_t size);
uint64_t nmb_device_max_buffer_length(const NmbDevice *);
// Pipeline makings (count, total us, longest us, waited us, failed), sink messages (info, warning+), shader cache hits/misses/writes.
bool nmb_device_work_stats(const NmbDevice *, uint64_t out[10]);
/* Artifact lookups and driver pipeline records are separate cache layers. */
struct NmbShaderCacheStats {
uint64_t hits, misses, rejected, read_failures, writes, write_failures, evictions;
uint64_t driver_loads, driver_hits, driver_misses, driver_saves, driver_truncated, driver_mismatched, driver_unchanged;
};
bool nmb_device_shader_cache_stats(const NmbDevice *, struct NmbShaderCacheStats *);
size_t nmb_last_warning(char *out, size_t size);
bool nmb_device_max_threads_per_threadgroup(const NmbDevice *, uint64_t out[3]);
/* Device queries the runtime answers by Metal selector: Apple's private counts and flags where the device has an answer of its own,
* and the public queries. A selector's kind says its result and argument. */
enum {NMB_ANSWER_COUNT = 0, NMB_ANSWER_FLAG = 1, NMB_ANSWER_COUNT_ARG = 2, NMB_ANSWER_FLAG_ARG = 3};
const char *nmb_device_answer_selector(size_t index, uint32_t *kind /* NMB_ANSWER_* */); /* NULL past the last */
bool nmb_device_answer(const NmbDevice *, const char *selector, uint64_t arg, uint64_t *out); /* false: no answer, nmb_last_error says why */
size_t nmb_device_vendor_name(const NmbDevice *, char *out, size_t size);
uint64_t nmb_device_linear_bytes_per_row(const NmbDevice *, uint64_t format, uint64_t width); /* 0: refused */
typedef struct NmbBinaryArchive NmbBinaryArchive;
NmbBinaryArchive *nmb_binary_archive_new(NmbDevice *, const char *path /* NULL: empty */);
void nmb_binary_archive_release(NmbBinaryArchive *);
bool nmb_binary_archive_serialize(NmbBinaryArchive *, const char *path);
NmbBuffer *nmb_buffer_new(NmbDevice *, uint64_t length, uint64_t options, const void *bytes /* NULL: zeroed */);
/* Wraps page-aligned memory the caller keeps mapped until the buffer is released (an IOSurface's base, host pages). */
@@ -48,9 +71,25 @@ uint64_t nmb_buffer_gpu_address(const NmbBuffer *);
struct NmbTextureDescriptor {
uint64_t type, format, width, height, depth, levels, samples, array_length, usage, options;
};
/* Heaps: type 0 automatic / 1 placement; offset UINT64_MAX asks an automatic heap to place the resource. */
NmbHeap *nmb_heap_new(NmbDevice *, uint64_t type, uint64_t size, uint64_t options);
void nmb_heap_release(NmbHeap *);
uint64_t nmb_heap_size(const NmbHeap *);
uint64_t nmb_heap_used_size(const NmbHeap *);
uint64_t nmb_heap_allocated_size(const NmbHeap *);
uint64_t nmb_heap_max_available_size(const NmbHeap *, uint64_t alignment);
uint64_t nmb_heap_set_purgeable_state(const NmbHeap *, uint64_t state);
NmbBuffer *nmb_heap_new_buffer(const NmbHeap *, uint64_t length, uint64_t options, uint64_t offset);
NmbTexture *nmb_heap_new_texture(const NmbHeap *, const struct NmbTextureDescriptor *, uint64_t offset);
NmbTexture *nmb_texture_new(NmbDevice *, const struct NmbTextureDescriptor *);
/* A linear texture over a buffer's bytes; iosurface marks the buffer as standing for a surface without planes (Metal's IOSurface rules). */
NmbTexture *nmb_texture_new_on_buffer(NmbBuffer *, const struct NmbTextureDescriptor *, uint64_t offset, uint64_t bytes_per_row, bool iosurface);
NmbTexture *nmb_texture_new_linear(NmbBuffer *, const struct NmbTextureDescriptor *, uint64_t offset, uint64_t bytes_per_row, uint64_t bytes_per_image);
/* whole: a view of every level and slice as format; else levels and slices are {start, count} and kind the view's texture type. */
NmbTexture *nmb_texture_new_view(NmbTexture *, uint64_t format, uint64_t kind, const uint64_t levels[2], const uint64_t slices[2], bool whole);
/* setPurgeableState: with Metal's raw state; the answer's raw state, 0 where it fails. */
uint64_t nmb_texture_set_purgeable_state(const NmbTexture *, uint64_t state);
uint64_t nmb_buffer_set_purgeable_state(const NmbBuffer *, uint64_t state);
void nmb_texture_release(NmbTexture *);
void nmb_texture_describe(const NmbTexture *, struct NmbTextureDescriptor *out);
bool nmb_texture_replace(NmbTexture *, const uint64_t region[6], uint64_t level, uint64_t slice, const void *bytes, uint64_t bytes_per_row,
@@ -91,6 +130,21 @@ struct NmbRenderPipelineDescriptor {
struct NmbVertexLayout layouts[31];
};
NmbRenderPipeline *nmb_render_pipeline_new(NmbDevice *, const struct NmbRenderPipelineDescriptor *);
/* Descriptor binaryArchives and MTLPipelineOption bits. NULL options means defaults.
* The archive array and its handles must remain valid for the duration of this call. */
struct NmbPipelineOptions {uint64_t flags; NmbBinaryArchive *const *archives; size_t archive_count;};
struct NmbComputePipelineDescriptor {
NmbFunction *function;
uint64_t max_threads, multiple_of_execution_width, indirect, max_call_stack_depth, adding_binary_functions;
uint64_t mutability[31];
};
NmbComputePipeline *nmb_compute_pipeline_new_with_options(NmbDevice *, NmbFunction *, const struct NmbPipelineOptions *);
NmbComputePipeline *nmb_compute_pipeline_new_with_descriptor(NmbDevice *, const struct NmbComputePipelineDescriptor *, const struct NmbPipelineOptions *);
NmbRenderPipeline *nmb_render_pipeline_new_with_options(NmbDevice *, const struct NmbRenderPipelineDescriptor *, const struct NmbPipelineOptions *);
/* The archive makes the state as the matching new call does and keeps what it used (addRender/ComputePipelineFunctionsWithDescriptor:). */
bool nmb_binary_archive_add_render(NmbBinaryArchive *, const struct NmbRenderPipelineDescriptor *);
bool nmb_binary_archive_add_compute(NmbBinaryArchive *, NmbFunction *);
bool nmb_binary_archive_add_compute_descriptor(NmbBinaryArchive *, const struct NmbComputePipelineDescriptor *);
void nmb_render_pipeline_release(NmbRenderPipeline *);
struct NmbSamplerDescriptor {
@@ -109,9 +163,19 @@ NmbQueue *nmb_queue_new(NmbDevice *);
void nmb_queue_release(NmbQueue *);
NmbCommandBuffer *nmb_command_buffer_new(NmbQueue *);
void nmb_command_buffer_release(NmbCommandBuffer *);
/* 1 storage dependency, 0 unrelated/already complete, -1 unknown (caller must wait).
* Includes texture roots, buffer-backed textures, heaps and overlapping imported host ranges.
* This frontend does not rebind/remap live backing ranges. */
int nmb_command_buffer_uses_buffer(const NmbCommandBuffer *, const NmbBuffer *);
int nmb_command_buffer_uses_texture(const NmbCommandBuffer *, const NmbTexture *);
/* The handler runs once, on a runtime thread, after the command buffer completed or failed. */
bool nmb_command_buffer_on_completed(NmbCommandBuffer *, void (*handler)(void *context), void *context);
/* Runs on actual scheduling, before completion. Not called if submission fails before scheduling.
* The caller keeps context alive until completion (or a refused commit). */
bool nmb_command_buffer_on_scheduled(NmbCommandBuffer *, void (*handler)(void *context), void *context);
bool nmb_command_buffer_commit(NmbCommandBuffer *);
/* Wait for scheduling and scheduled handlers only; false on a refused wait. */
bool nmb_command_buffer_wait_scheduled(NmbCommandBuffer *);
/* 0 completed in time, 1 timed out (timeout_ms 0 waits without a bound). */
int nmb_command_buffer_wait(NmbCommandBuffer *, uint64_t timeout_ms);
/* Metal's MTLCommandBufferStatus (0 not enqueued .. 4 completed, 5 error). */
@@ -146,6 +210,7 @@ struct NmbPassAttachment {
struct NmbRenderPassDescriptor {
struct NmbPassAttachment colors[8], depth, stencil;
uint64_t width, height, array_length, samples;
NmbBuffer *visibility; /* the pass's visibility result buffer, or null */
};
NmbEncoder *nmb_render_encoder_new(NmbCommandBuffer *, const struct NmbRenderPassDescriptor *);
bool nmb_encoder_end(NmbEncoder *);
@@ -169,6 +234,9 @@ bool nmb_render_set_scissor(NmbEncoder *, const uint64_t rect[4]);
bool nmb_render_set_raster(NmbEncoder *, uint64_t cull_mode, uint64_t winding, uint64_t fill_mode);
bool nmb_render_set_stencil_reference(NmbEncoder *, uint32_t front, uint32_t back);
bool nmb_render_set_blend_color(NmbEncoder *, const float color[4]);
bool nmb_render_set_depth_bias(NmbEncoder *, float bias, float slope_scale, float clamp);
bool nmb_render_set_line_width(NmbEncoder *, float width);
bool nmb_render_set_visibility_mode(NmbEncoder *, uint64_t mode, uint64_t offset);
bool nmb_render_draw(NmbEncoder *, uint64_t primitive, uint32_t start, uint32_t count, uint32_t instances, uint32_t base_instance);
bool nmb_render_draw_indexed(NmbEncoder *, uint64_t primitive, uint32_t count, uint64_t index_type, NmbBuffer *indices, uint64_t offset,
uint32_t instances, int32_t base_vertex, uint32_t base_instance);
drivers/NativeMetal/bridge/NativeMetalObjects.rs
+661 -14
View File
@@ -13,14 +13,14 @@ use metal2vulkan::metal::{
TextureUsage, TriangleFillMode, VertexFormat, VertexStepFunction, Winding,
};
use metal2vulkan::pipeline::{
ColorAttachmentDescriptor, RenderPipelineDescriptor, StageSource, VertexAttributeDescriptor, VertexBufferLayoutDescriptor,
ColorAttachmentDescriptor, ComputePipelineDescriptor, RenderPipelineDescriptor, StageSource, VertexAttributeDescriptor, VertexBufferLayoutDescriptor,
VertexDescriptor,
};
use metal2vulkan_runtime::{
BlitCommandEncoder, BlitOption, Buffer, ClearColor, CommandBuffer, CommandQueue, ComputeCommandEncoder,
ComputePipelineState, DepthStencilState, Device, DeviceOptions, Event, EventRef, Fence, Function, HostMemory, IoSurfacePlane, Library, LoadAction,
MetalLevel, Origin, PipelineOptions, Region, RenderCommandEncoder, RenderPassDescriptor, RenderPipelineState, ResourceOptions,
SamplerState, ScissorRect, SharedEvent, SharedEventListener, Size, StoreAction, Texture, TextureDescriptor, Viewport,
BinaryArchive, BinaryArchiveDescriptor, BlitCommandEncoder, FamilyRow, FeatureRow, GpuFamily, RowState, BlitOption, Buffer, ClearColor, CommandBuffer, CommandQueue, ComputeCommandEncoder,
ComputePipelineState, DebugMessage, DepthStencilState, Device, DeviceOptions, MessageSeverity, MessageSink, Event, EventRef, Fence, Function, Heap, HeapDescriptor, HeapType, HostMemory, IoSurfacePlane, Library, LoadAction,
MetalLevel, Origin, PipelineOption, PipelineOptions, PurgeableState, PrivateCount, PrivateFlag, PrivateProperties, MAX_COLOR_ATTACHMENTS, Region, RenderCommandEncoder, RenderPassDescriptor, RenderPipelineState, ResourceOptions,
SamplerState, ScissorRect, SharedEvent, SharedEventListener, Size, StoreAction, Texture, TextureDescriptor, Viewport, VisibilityResultMode,
};
use std::cell::RefCell;
use std::ffi::{c_char, c_void, CStr};
@@ -140,7 +140,14 @@ pub unsafe extern "C" fn nmb_function_constant_air(length: *mut usize) -> *const
// ---- device ----
pub struct NmbDevice(Device);
pub struct NmbDevice(Device, std::sync::OnceLock<PrivateProperties>, std::sync::Mutex<()>);
impl NmbDevice {
/// The runtime judges the private properties by the family rows once.
fn private(&self) -> &PrivateProperties {
self.1.get_or_init(|| self.0.private_properties())
}
}
/// # Safety
/// `loader` and `cache` are C strings.
@@ -156,10 +163,114 @@ pub unsafe extern "C" fn nmb_device_open(loader: *const c_char, cache: *const c_
options.loader = Some(PathBuf::from(loader.to_string_lossy().into_owned()));
options.metal_level = Some(MetalLevel::NEWEST);
options.shader_cache = metal2vulkan_runtime::ShaderCacheOptions::in_directory(cache.to_string_lossy().into_owned());
Ok(boxed(NmbDevice(Device::system_default_with(&options).map_err(e("device"))?)))
options.messages = Some(MessageSink::new(observe_message));
Ok(boxed(NmbDevice(Device::system_default_with(&options).map_err(e("device"))?, std::sync::OnceLock::new(), std::sync::Mutex::new(()))))
})
}
// What the runtime says through the device's sink, kept as numbers for the plugin's telemetry: each render VkPipeline making the
// creation log (METAL2VULKAN_LOG_PIPELINES) reports ("... made in <ms> ms[ after waiting <ms> ms for its turn]"), and the other
// messages by severity with the last warning's text.
static PIPELINE_MAKINGS: [std::sync::atomic::AtomicU64; 5] = [const { std::sync::atomic::AtomicU64::new(0) }; 5]; // n, us, max us, waited us, failed
static OTHER_MESSAGES: [std::sync::atomic::AtomicU64; 2] = [const { std::sync::atomic::AtomicU64::new(0) }; 2]; // info, warning or worse
static LAST_WARNING: std::sync::Mutex<String> = std::sync::Mutex::new(String::new());
fn millis_after(text: &str, marker: &str) -> Option<f64> {
let at = text.find(marker)? + marker.len();
let rest = &text[at..];
let end = rest.find(' ').unwrap_or(rest.len());
rest[..end].parse().ok()
}
fn observe_message(message: &DebugMessage) {
use std::sync::atomic::Ordering::Relaxed;
if message.id_name == "metal2vulkan-pipeline-creation" {
let made = millis_after(&message.text, " made in ");
let us = made.or_else(|| millis_after(&message.text, " failed in ")).map_or(0, |ms| (ms * 1000.0) as u64);
PIPELINE_MAKINGS[0].fetch_add(1, Relaxed);
PIPELINE_MAKINGS[1].fetch_add(us, Relaxed);
PIPELINE_MAKINGS[2].fetch_max(us, Relaxed);
PIPELINE_MAKINGS[3].fetch_add(millis_after(&message.text, " after waiting ").map_or(0, |ms| (ms * 1000.0) as u64), Relaxed);
if made.is_none() {
PIPELINE_MAKINGS[4].fetch_add(1, Relaxed);
}
return;
}
if message.severity >= MessageSeverity::Warning {
OTHER_MESSAGES[1].fetch_add(1, Relaxed);
if let Ok(mut last) = LAST_WARNING.lock() {
*last = message.text.chars().take(160).collect();
}
} else {
OTHER_MESSAGES[0].fetch_add(1, Relaxed);
}
}
/// The sink's counts into the caller's ten words: pipeline makings (count, total us, longest us, waited us, failed), messages (info,
/// warning or worse), and the device's shader cache hits, misses and writes.
#[unsafe(no_mangle)]
pub extern "C" fn nmb_device_work_stats(device: *const NmbDevice, out: *mut u64) -> bool {
use std::sync::atomic::Ordering::Relaxed;
guard(false, || {
if out.is_null() {
return Err("null stats".into());
}
// SAFETY: the header's ten words.
let out = unsafe { std::slice::from_raw_parts_mut(out, 10) };
for (i, v) in PIPELINE_MAKINGS.iter().enumerate() {
out[i] = v.load(Relaxed);
}
out[5] = OTHER_MESSAGES[0].load(Relaxed);
out[6] = OTHER_MESSAGES[1].load(Relaxed);
let cache = get(device)?.0.shader_cache_statistics();
out[7] = cache.hits;
out[8] = cache.misses;
out[9] = cache.writes;
Ok(true)
})
}
/// Separate artifact and driver-record counters, mirroring the C ABI.
#[repr(C)]
pub struct NmbShaderCacheStats {
pub hits: u64,
pub misses: u64,
pub rejected: u64,
pub read_failures: u64,
pub writes: u64,
pub write_failures: u64,
pub evictions: u64,
pub driver_loads: u64,
pub driver_hits: u64,
pub driver_misses: u64,
pub driver_saves: u64,
pub driver_truncated: u64,
pub driver_mismatched: u64,
pub driver_unchanged: u64,
}
#[unsafe(no_mangle)]
pub extern "C" fn nmb_device_shader_cache_stats(device: *const NmbDevice, out: *mut NmbShaderCacheStats) -> bool {
guard(false, || {
let out = get_mut(out)?;
let stats = get(device)?.0.shader_cache_statistics();
*out = NmbShaderCacheStats {
hits: stats.hits, misses: stats.misses, rejected: stats.rejected,
read_failures: stats.read_failures, writes: stats.writes, write_failures: stats.write_failures, evictions: stats.evictions,
driver_loads: stats.driver_loads, driver_hits: stats.driver_pipeline_hits, driver_misses: stats.driver_pipeline_misses,
driver_saves: stats.driver_saves, driver_truncated: stats.driver_data_truncated,
driver_mismatched: stats.driver_data_mismatched, driver_unchanged: stats.driver_data_unchanged,
};
Ok(true)
})
}
/// The last warning the device's sink received, as a C string; returns its length.
#[unsafe(no_mangle)]
pub extern "C" fn nmb_last_warning(out: *mut c_char, size: usize) -> usize {
guard(0, || Ok(copy_out(&LAST_WARNING.lock().map(|s| s.clone()).unwrap_or_default(), out, size)))
}
#[unsafe(no_mangle)]
pub extern "C" fn nmb_device_release(device: *mut NmbDevice) {
release(device)
@@ -175,6 +286,188 @@ pub extern "C" fn nmb_device_max_buffer_length(device: *const NmbDevice) -> u64
guard(0, || Ok(get(device)?.0.properties().max_buffer_length))
}
/// `maxThreadsPerThreadgroup` into the caller's three words.
#[unsafe(no_mangle)]
pub extern "C" fn nmb_device_max_threads_per_threadgroup(device: *const NmbDevice, out: *mut u64) -> bool {
guard(false, || {
if out.is_null() {
return Err("null size".into());
}
let s = get(device)?.0.properties().max_threads_per_threadgroup;
// SAFETY: the header's three words.
unsafe { std::slice::from_raw_parts_mut(out, 3) }.copy_from_slice(&[s.width, s.height, s.depth]);
Ok(true)
})
}
/// The device selectors `nmb_device_answer` takes beyond Apple's private counts and flags: the selector and its kind
/// (the header's NMB_ANSWER_* values).
const PUBLIC_ANSWERS: &[(&str, u32)] = &[
("maxThreadgroupMemoryLength", 0),
("argumentBuffersSupport", 0),
("readWriteTextureSupport", 0),
("maxColorAttachments", 0),
("minConstantBufferAlignmentBytes", 0),
("deviceLinearReadOnlyTextureAlignmentBytes", 0),
("areRasterOrderGroupsSupported", 1),
("supportsRasterOrderGroups", 1),
("isDepth24Stencil8PixelFormatSupported", 1),
("areProgrammableSamplePositionsSupported", 1),
("supports32BitFloatFiltering", 1),
("supportsMemorylessRenderTargets", 1),
("supportsSamplerReductionMode", 1),
("supportsYCBCRFormats12", 1),
("supportsYCBCRPackedFormats12", 1),
("supportsSIMDGroup", 1),
("minimumLinearTextureAlignmentForPixelFormat:", 2),
("supportsTextureSampleCount:", 3),
("supportsVertexAmplificationCount:", 3),
("supportsRasterizationRateMapWithLayerCount:", 3),
("supportsFamily:", 3),
("supportsPrimitiveType:", 3),
("supportsTextureWriteRoundingMode:", 3),
];
/// Every selector `nmb_device_answer` takes, with its kind, as C strings.
fn answer_selectors() -> &'static [(std::ffi::CString, u32)] {
static ALL: std::sync::OnceLock<Vec<(std::ffi::CString, u32)>> = std::sync::OnceLock::new();
ALL.get_or_init(|| {
let counts = PrivateCount::ALL.into_iter().map(|c| (c.selector(), 0));
let flags = PrivateFlag::ALL.into_iter().map(|f| (f.selector(), 1));
counts
.chain(flags)
.chain(PUBLIC_ANSWERS.iter().copied())
.map(|(s, kind)| (std::ffi::CString::new(s).expect("selector"), kind))
.collect()
})
}
/// The selector at `index` of the ones `nmb_device_answer` takes and its kind; null past the last.
///
/// # Safety
/// `kind` is writable.
#[unsafe(no_mangle)]
pub unsafe extern "C" fn nmb_device_answer_selector(index: usize, kind: *mut u32) -> *const c_char {
let Some((selector, k)) = answer_selectors().get(index) else { return std::ptr::null() };
// SAFETY: the caller's contract.
if let Some(kind) = unsafe { kind.as_mut() } {
*kind = *k;
}
selector.as_ptr()
}
fn device_answer(device: &NmbDevice, selector: &str, arg: u64) -> Result<u64, String> {
let d = &device.0;
let p = d.properties();
let none = || format!("the device has no answer for {selector}");
if let Some(c) = PrivateCount::ALL.into_iter().find(|c| c.selector() == selector) {
return device.private().count(c).value().ok_or_else(none);
}
if let Some(f) = PrivateFlag::ALL.into_iter().find(|f| f.selector() == selector) {
return device.private().flag(f).value().map(u64::from).ok_or_else(none);
}
let gap = |r: Result<bool, metal2vulkan_runtime::RuntimeGap>| r.map(u64::from).map_err(|g| format!("{selector}: {g:?}"));
let format_supported = |f: PixelFormat| d.pixel_format_capabilities(f) != metal2vulkan_runtime::FormatCapabilities::NONE;
Ok(match selector {
"maxThreadgroupMemoryLength" => p.max_threadgroup_memory_length,
"argumentBuffersSupport" => p.argument_buffers_support.as_raw() as u64,
"readWriteTextureSupport" => p.read_write_texture_support.as_raw() as u64,
// The runtime's pass takes MAX_COLOR_ATTACHMENTS; the device's Vulkan limits bound what a pipeline links.
"maxColorAttachments" => {
let l = &p.description.limits;
(MAX_COLOR_ATTACHMENTS as u64).min(l.max_color_attachments.min(l.max_fragment_output_attachments) as u64)
}
// The runtime binds a buffer at any offset within it (render.rs, setVertexBuffer); 4 is a scalar's alignment, which the shader's
// own loads need.
"minConstantBufferAlignmentBytes" => 4,
"deviceLinearReadOnlyTextureAlignmentBytes" => {
device.private().count(PrivateCount::DeviceLinearTextureAlignmentBytes).value().ok_or_else(none)?
}
"areRasterOrderGroupsSupported" | "supportsRasterOrderGroups" => gap(p.are_raster_order_groups_supported())?,
"isDepth24Stencil8PixelFormatSupported" => gap(p.is_depth24_stencil8_pixel_format_supported())?,
"areProgrammableSamplePositionsSupported" => gap(p.are_programmable_sample_positions_supported())?,
"supports32BitFloatFiltering" => p.supports_32bit_float_filtering as u64,
// A Memoryless texture is a transient render attachment (texture.rs), which every device makes.
"supportsMemorylessRenderTargets" => 1,
"supportsSamplerReductionMode" => gap(p.supports_sampler_reduction_mode())?,
// Apple's SIMD-group functions: the family's SIMD-scoped barrier and permute rows are exact on the device.
"supportsSIMDGroup" => {
let rows = d.family_rows(GpuFamily::Apple9);
let exact = |r: FeatureRow| rows.iter().any(|a| a.row == FamilyRow::Feature(r) && a.state == RowState::Exact);
(exact(FeatureRow::SimdBarrier) && exact(FeatureRow::SimdPermute)) as u64
}
"supportsYCBCRFormats12" => format_supported(PixelFormat::YCBCR124202P) as u64,
"supportsYCBCRPackedFormats12" => format_supported(PixelFormat::YCBCR124202PPACKED) as u64,
"minimumLinearTextureAlignmentForPixelFormat:" => p
.minimum_linear_texture_alignment_for_pixel_format(raw::<PixelFormat>(arg, "pixel format")?)
.map_err(e("minimum linear texture alignment"))?,
"supportsTextureSampleCount:" => gap(p.supports_texture_sample_count(arg))?,
"supportsVertexAmplificationCount:" => gap(p.supports_vertex_amplification_count(arg))?,
"supportsRasterizationRateMapWithLayerCount:" => gap(p.supports_rasterization_rate_map_with_layer_count(arg))?,
"supportsFamily:" => d.supports_family(arg as i64) as u64,
"supportsPrimitiveType:" => p.supports_primitive_type(arg) as u64,
"supportsTextureWriteRoundingMode:" => p.supports_texture_write_rounding_mode(arg) as u64,
_ => return Err(format!("{selector} is not a device query of the bridge")),
})
}
/// The runtime's answer on the device to `selector` (one `nmb_device_answer_selector` lists), with `arg` for a selector that takes one.
/// False, with the reason, where the device has no answer of its own.
///
/// # Safety
/// `selector` is a C string; `out` is writable.
#[unsafe(no_mangle)]
pub unsafe extern "C" fn nmb_device_answer(device: *const NmbDevice, selector: *const c_char, arg: u64, out: *mut u64) -> bool {
guard(false, || {
if selector.is_null() || out.is_null() {
return Err("null selector or answer".into());
}
// SAFETY: the caller's contract.
let selector = unsafe { CStr::from_ptr(selector) }.to_str().map_err(e("selector"))?;
let answer = device_answer(get(device)?, selector, arg)?;
// SAFETY: as above.
unsafe { *out = answer };
Ok(true)
})
}
/// `requiredLinearTextureBytesPerRowForDescriptor:` (Apple SPI): a row of `width` texels of `format` rounded up to the format's
/// linear alignment, from the runtime's format description; 0, with the reason, for a format no buffer can back.
#[unsafe(no_mangle)]
pub extern "C" fn nmb_device_linear_bytes_per_row(device: *const NmbDevice, format: u64, width: u64) -> u64 {
guard(0, || {
let d = &get(device)?.0;
let alignment = d
.properties()
.minimum_linear_texture_alignment_for_pixel_format(raw::<PixelFormat>(format, "pixel format")?)
.map_err(e("linear texture"))?;
let info = d.pixel_format_info(format);
// Flags bit 0x400: a format of blocks, which the alignment query has refused already.
if info.flags & 0x400 != 0 || info.words[0] == 0 {
return Err(format!("pixel format {format} has no texel size"));
}
Ok((width * info.words[0]).div_ceil(alignment) * alignment)
})
}
/// `vendorName`: the Vulkan device's PCI vendor, named where the ID is a known GPU vendor's, else the ID itself.
#[unsafe(no_mangle)]
pub extern "C" fn nmb_device_vendor_name(device: *const NmbDevice, out: *mut c_char, size: usize) -> usize {
guard(0, || {
let id = get(device)?.0.properties().vendor_id;
let name = match id {
0x5143 => "Qualcomm".to_string(),
0x106b => "Apple".to_string(),
0x10de => "NVIDIA".to_string(),
0x1002 => "AMD".to_string(),
0x8086 => "Intel".to_string(),
0x13b5 => "ARM".to_string(),
_ => format!("0x{id:04x}"),
};
Ok(copy_out(&name, out, size))
})
}
// ---- buffers ----
pub struct NmbBuffer(Buffer);
@@ -203,11 +496,23 @@ pub unsafe extern "C" fn nmb_buffer_new(device: *mut NmbDevice, length: u64, opt
pub unsafe extern "C" fn nmb_buffer_new_no_copy(device: *mut NmbDevice, pointer: *mut c_void, length: u64, options_word: u64)
-> *mut NmbBuffer {
guard(std::ptr::null_mut(), || {
let device = &get(device)?.0;
let device = get(device)?;
// SAFETY: the caller's contract.
let memory = unsafe { HostMemory::new(NonNull::new(pointer as *mut u8).ok_or("null memory")?) };
// SAFETY: as above.
let buffer = unsafe { device.new_buffer_with_bytes_no_copy(memory, length, options(options_word)?, None) };
let modes = options(options_word)?;
// The runtime's live-import lookup and subsequent join are separate:
// simultaneous first constructors can allocate distinct imports of the
// same pages. Cover every native ABI no-copy call, including mixtures
// of raw buffers and IOSurface textures. Each call still gets its own
// buffer object. This ABI passes neither a dma-buf source nor a runtime
// deallocator; no caller callback runs under this gate. Drop it before
// returning/formatting a failure; GPU work and resource release never
// take it. A device's unrelated APIs and other devices remain separate.
let buffer = {
let _gate = device.2.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
// SAFETY: as above.
unsafe { device.0.new_buffer_with_bytes_no_copy(memory, length, modes, None) }
};
Ok(boxed(NmbBuffer(buffer.map_err(e("no-copy buffer"))?)))
})
}
@@ -232,6 +537,79 @@ pub extern "C" fn nmb_buffer_gpu_address(buffer: *const NmbBuffer) -> u64 {
guard(0, || Ok(get(buffer)?.0.gpu_address().0))
}
// ---- heaps ----
pub struct NmbHeap(Heap);
/// `newHeapWithDescriptor:`: type (0 automatic, 1 placement), size, resource options. Sparse heaps are not made here.
#[unsafe(no_mangle)]
pub extern "C" fn nmb_heap_new(device: *mut NmbDevice, heap_type: u64, size: u64, options_word: u64) -> *mut NmbHeap {
guard(std::ptr::null_mut(), || {
let device = &get(device)?.0;
let heap_type = HeapType::from_raw(heap_type).map_err(e("heap type"))?;
if heap_type == HeapType::Sparse {
return Err("sparse heaps are not implemented".into());
}
let descriptor = HeapDescriptor::new(heap_type, size, options(options_word)?);
Ok(boxed(NmbHeap(device.new_heap(&descriptor).map_err(e("heap"))?)))
})
}
#[unsafe(no_mangle)]
pub extern "C" fn nmb_heap_release(heap: *mut NmbHeap) {
release(heap)
}
#[unsafe(no_mangle)]
pub extern "C" fn nmb_heap_size(heap: *const NmbHeap) -> u64 {
guard(0, || Ok(get(heap)?.0.size()))
}
#[unsafe(no_mangle)]
pub extern "C" fn nmb_heap_used_size(heap: *const NmbHeap) -> u64 {
guard(0, || Ok(get(heap)?.0.used_size()))
}
#[unsafe(no_mangle)]
pub extern "C" fn nmb_heap_allocated_size(heap: *const NmbHeap) -> u64 {
guard(0, || Ok(get(heap)?.0.current_allocated_size()))
}
#[unsafe(no_mangle)]
pub extern "C" fn nmb_heap_max_available_size(heap: *const NmbHeap, alignment: u64) -> u64 {
guard(0, || Ok(get(heap)?.0.max_available_size(alignment).map_err(e("heap available size"))?))
}
#[unsafe(no_mangle)]
pub extern "C" fn nmb_heap_set_purgeable_state(heap: *const NmbHeap, state: u64) -> u64 {
guard(0, || {
let state = PurgeableState::from_raw(state).map_err(e("purgeable state"))?;
Ok(get(heap)?.0.set_purgeable_state(state).map_err(e("heap purgeable state"))?.raw())
})
}
/// A buffer in the heap: automatic heaps place it, a placement heap takes `offset` (`u64::MAX` asks for automatic placement).
#[unsafe(no_mangle)]
pub extern "C" fn nmb_heap_new_buffer(heap: *const NmbHeap, length: u64, options_word: u64, offset: u64) -> *mut NmbBuffer {
guard(std::ptr::null_mut(), || {
let heap = &get(heap)?.0;
let options = options(options_word)?;
let buffer = if offset == u64::MAX { heap.new_buffer(length, options) } else { heap.new_buffer_at_offset(length, options, offset) };
Ok(boxed(NmbBuffer(buffer.map_err(e("heap buffer"))?)))
})
}
/// A texture in the heap, placed as `nmb_heap_new_buffer` places a buffer.
#[unsafe(no_mangle)]
pub extern "C" fn nmb_heap_new_texture(heap: *const NmbHeap, descriptor: *const NmbTextureDescriptor, offset: u64) -> *mut NmbTexture {
guard(std::ptr::null_mut(), || {
let heap = &get(heap)?.0;
let descriptor = texture_descriptor(descriptor)?;
let texture = if offset == u64::MAX { heap.new_texture(&descriptor) } else { heap.new_texture_at_offset(&descriptor, offset) };
Ok(boxed(NmbTexture(texture.map_err(e("heap texture"))?)))
})
}
// ---- textures ----
#[repr(C)]
@@ -289,6 +667,57 @@ pub extern "C" fn nmb_texture_new_on_buffer(buffer: *mut NmbBuffer, descriptor:
})
}
/// `newLinearTextureWithDescriptor:offset:bytesPerRow:bytesPerImage:` (Apple SPI).
#[unsafe(no_mangle)]
pub extern "C" fn nmb_texture_new_linear(buffer: *mut NmbBuffer, descriptor: *const NmbTextureDescriptor, offset: u64, bytes_per_row: u64,
bytes_per_image: u64) -> *mut NmbTexture {
guard(std::ptr::null_mut(), || {
let descriptor = texture_descriptor(descriptor)?;
let texture = get(buffer)?.0.new_linear_texture(&descriptor, offset, bytes_per_row, bytes_per_image);
Ok(boxed(NmbTexture(texture.map_err(e("linear texture"))?)))
})
}
/// A view of `texture`: of the whole texture as `format` where `whole`, else of `levels` and `slices` (start, count) as `format` and `kind`.
#[unsafe(no_mangle)]
pub extern "C" fn nmb_texture_new_view(texture: *mut NmbTexture, format: u64, kind: u64, levels: *const u64, slices: *const u64, whole: bool)
-> *mut NmbTexture {
guard(std::ptr::null_mut(), || {
let t = &get(texture)?.0;
let format = raw::<PixelFormat>(format, "pixel format")?;
let view = if whole {
t.new_texture_view(format)
} else {
if levels.is_null() || slices.is_null() {
return Err("null levels or slices".into());
}
// SAFETY: the header's two words each.
let (l, s) = unsafe { (std::slice::from_raw_parts(levels, 2), std::slice::from_raw_parts(slices, 2)) };
let ranges = (l[0]..l[0].checked_add(l[1]).ok_or("levels overflow")?, s[0]..s[0].checked_add(s[1]).ok_or("slices overflow")?);
t.new_texture_view_with_levels_slices(format, raw::<TextureType>(kind, "texture type")?, ranges.0, ranges.1)
};
Ok(boxed(NmbTexture(view.map_err(e("texture view"))?)))
})
}
/// `setPurgeableState:` on a texture: the runtime's answer as Metal's raw state, 0 where it fails.
#[unsafe(no_mangle)]
pub extern "C" fn nmb_texture_set_purgeable_state(texture: *const NmbTexture, state: u64) -> u64 {
guard(0, || {
let state = PurgeableState::from_raw(state).map_err(e("purgeable state"))?;
Ok(get(texture)?.0.set_purgeable_state(state).map_err(e("purgeable state"))?.raw())
})
}
/// `setPurgeableState:` on a buffer, as on a texture.
#[unsafe(no_mangle)]
pub extern "C" fn nmb_buffer_set_purgeable_state(buffer: *const NmbBuffer, state: u64) -> u64 {
guard(0, || {
let state = PurgeableState::from_raw(state).map_err(e("purgeable state"))?;
Ok(get(buffer)?.0.set_purgeable_state(state).map_err(e("purgeable state"))?.raw())
})
}
#[unsafe(no_mangle)]
pub extern "C" fn nmb_texture_release(texture: *mut NmbTexture) {
release(texture)
@@ -486,12 +915,61 @@ pub struct NmbDepthStencil(DepthStencilState);
#[unsafe(no_mangle)]
pub extern "C" fn nmb_compute_pipeline_new(device: *mut NmbDevice, function: *mut NmbFunction) -> *mut NmbComputePipeline {
// SAFETY: null options names no array, the existing handles follow this ABI.
unsafe { nmb_compute_pipeline_new_with_options(device, function, std::ptr::null()) }
}
/// # Safety
/// `options` is null or a valid record with a counted array of live archive handles.
#[unsafe(no_mangle)]
pub unsafe extern "C" fn nmb_compute_pipeline_new_with_options(device: *mut NmbDevice, function: *mut NmbFunction,
options: *const NmbPipelineOptions) -> *mut NmbComputePipeline {
guard(std::ptr::null_mut(), || {
let state = get(device)?.0.new_compute_pipeline_state_with_function(&get(function)?.0, &PipelineOptions::default());
let resolved = unsafe { pipeline_options(options)? };
let state = get(device)?.0.new_compute_pipeline_state_with_function(&get(function)?.0, &resolved);
Ok(boxed(NmbComputePipeline(state.map_err(e("compute pipeline"))?)))
})
}
#[repr(C)]
pub struct NmbComputePipelineDescriptor {
pub function: *mut NmbFunction,
pub max_threads: u64,
pub multiple_of_execution_width: u64,
pub indirect: u64,
pub max_call_stack_depth: u64,
pub adding_binary_functions: u64,
pub mutability: [u64; 31],
}
fn compute_descriptor(descriptor: *const NmbComputePipelineDescriptor) -> Result<ComputePipelineDescriptor, String> {
let d = get(descriptor)?;
let mut out = ComputePipelineDescriptor::new(StageSource::Function(get(d.function)?.0.function_descriptor()));
out.max_total_threads_per_threadgroup = if d.max_threads == 0 { None } else { Some(u32::try_from(d.max_threads).map_err(e("compute max threads"))?) };
out.threadgroup_size_is_multiple_of_execution_width = d.multiple_of_execution_width != 0;
out.support_indirect_command_buffers = d.indirect != 0;
out.max_call_stack_depth = u32::try_from(d.max_call_stack_depth).map_err(e("compute stack depth"))?;
out.support_adding_binary_functions = d.adding_binary_functions != 0;
for (index, &mutability) in d.mutability.iter().enumerate() {
if mutability != 0 {
out.buffers.insert(index as u32, raw::<metal2vulkan::metal::Mutability>(mutability, "buffer mutability")?);
}
}
Ok(out)
}
/// # Safety
/// `options` is null or a valid record with a counted array of live archive handles.
#[unsafe(no_mangle)]
pub unsafe extern "C" fn nmb_compute_pipeline_new_with_descriptor(device: *mut NmbDevice, descriptor: *const NmbComputePipelineDescriptor,
options: *const NmbPipelineOptions) -> *mut NmbComputePipeline {
guard(std::ptr::null_mut(), || {
let resolved = unsafe { pipeline_options(options)? };
let state = get(device)?.0.new_compute_pipeline_state(&compute_descriptor(descriptor)?, &resolved).map_err(e("compute pipeline"))?;
Ok(boxed(NmbComputePipeline(state)))
})
}
#[unsafe(no_mangle)]
pub extern "C" fn nmb_compute_pipeline_release(pipeline: *mut NmbComputePipeline) {
release(pipeline)
@@ -555,8 +1033,50 @@ fn optional_format(value: u64) -> Result<Option<PixelFormat>, String> {
#[unsafe(no_mangle)]
pub extern "C" fn nmb_render_pipeline_new(device: *mut NmbDevice, descriptor: *const NmbRenderPipelineDescriptor) -> *mut NmbRenderPipeline {
// SAFETY: null options names no array; the device/descriptor handles follow the existing ABI.
unsafe { nmb_render_pipeline_new_with_options(device, descriptor, std::ptr::null()) }
}
#[repr(C)]
pub struct NmbPipelineOptions {
pub flags: u64,
pub archives: *const *mut NmbBinaryArchive,
pub archive_count: usize,
}
/// # Safety
/// The options record and its counted archive array are valid for this call.
unsafe fn pipeline_options(options: *const NmbPipelineOptions) -> Result<PipelineOptions, String> {
let mut resolved = PipelineOptions::default();
if let Some(options) = unsafe { options.as_ref() } {
resolved.options = PipelineOption::from_raw(options.flags);
if options.archive_count > 0 {
if options.archives.is_null() || options.archive_count > isize::MAX as usize / std::mem::size_of::<*mut NmbBinaryArchive>() {
return Err("invalid binary archive array".into());
}
let archives = unsafe { std::slice::from_raw_parts(options.archives, options.archive_count) };
for &archive in archives {
resolved.binary_archives.push(get(archive)?.0.clone());
}
}
}
Ok(resolved)
}
/// # Safety
/// `options` is null or a valid record; its array holds `archive_count` live archive handles.
#[unsafe(no_mangle)]
pub unsafe extern "C" fn nmb_render_pipeline_new_with_options(device: *mut NmbDevice, descriptor: *const NmbRenderPipelineDescriptor,
options: *const NmbPipelineOptions) -> *mut NmbRenderPipeline {
guard(std::ptr::null_mut(), || {
let device = &get(device)?.0;
let resolved = unsafe { pipeline_options(options)? };
let state = device.new_render_pipeline_state(&render_descriptor(descriptor)?, &resolved).map_err(e("render pipeline"))?;
Ok(boxed(NmbRenderPipeline(state)))
})
}
fn render_descriptor(descriptor: *const NmbRenderPipelineDescriptor) -> Result<RenderPipelineDescriptor, String> {
let d = get(descriptor)?;
let mut out = RenderPipelineDescriptor::new(StageSource::Function(get(d.vertex)?.0.function_descriptor()));
if !d.fragment.is_null() {
@@ -599,9 +1119,7 @@ pub extern "C" fn nmb_render_pipeline_new(device: *mut NmbDevice, descriptor: *c
}
out.vertex_descriptor = Some(inputs);
}
let state = device.new_render_pipeline_state(&out, &PipelineOptions::default()).map_err(e("render pipeline"))?;
Ok(boxed(NmbRenderPipeline(state)))
})
Ok(out)
}
#[unsafe(no_mangle)]
@@ -609,6 +1127,67 @@ pub extern "C" fn nmb_render_pipeline_release(pipeline: *mut NmbRenderPipeline)
release(pipeline)
}
// ---- binary archives ----
pub struct NmbBinaryArchive(BinaryArchive);
/// # Safety
/// `path` is null (an empty archive) or a C string naming the archive's file.
#[unsafe(no_mangle)]
pub unsafe extern "C" fn nmb_binary_archive_new(device: *mut NmbDevice, path: *const c_char) -> *mut NmbBinaryArchive {
guard(std::ptr::null_mut(), || {
// SAFETY: the caller's contract.
let url = (!path.is_null()).then(|| PathBuf::from(unsafe { CStr::from_ptr(path) }.to_string_lossy().into_owned()));
let archive = get(device)?.0.new_binary_archive(&BinaryArchiveDescriptor::new(url)).map_err(e("binary archive"))?;
Ok(boxed(NmbBinaryArchive(archive)))
})
}
#[unsafe(no_mangle)]
pub extern "C" fn nmb_binary_archive_release(archive: *mut NmbBinaryArchive) {
release(archive)
}
#[unsafe(no_mangle)]
pub extern "C" fn nmb_binary_archive_add_render(archive: *mut NmbBinaryArchive, descriptor: *const NmbRenderPipelineDescriptor) -> bool {
guard(false, || {
get(archive)?.0.add_render_pipeline_functions(&render_descriptor(descriptor)?).map_err(e("archive render pipeline"))?;
Ok(true)
})
}
#[unsafe(no_mangle)]
pub extern "C" fn nmb_binary_archive_add_compute(archive: *mut NmbBinaryArchive, function: *mut NmbFunction) -> bool {
guard(false, || {
let descriptor = ComputePipelineDescriptor::new(StageSource::Function(get(function)?.0.function_descriptor()));
get(archive)?.0.add_compute_pipeline_functions(&descriptor).map_err(e("archive compute pipeline"))?;
Ok(true)
})
}
#[unsafe(no_mangle)]
pub extern "C" fn nmb_binary_archive_add_compute_descriptor(archive: *mut NmbBinaryArchive, descriptor: *const NmbComputePipelineDescriptor) -> bool {
guard(false, || {
get(archive)?.0.add_compute_pipeline_functions(&compute_descriptor(descriptor)?).map_err(e("archive compute pipeline"))?;
Ok(true)
})
}
/// # Safety
/// `path` is a C string.
#[unsafe(no_mangle)]
pub unsafe extern "C" fn nmb_binary_archive_serialize(archive: *mut NmbBinaryArchive, path: *const c_char) -> bool {
guard(false, || {
if path.is_null() {
return Err("null path".into());
}
// SAFETY: the caller's contract.
let path = PathBuf::from(unsafe { CStr::from_ptr(path) }.to_string_lossy().into_owned());
get(archive)?.0.serialize_to_url(&path).map_err(e("archive serialize"))?;
Ok(true)
})
}
#[repr(C)]
pub struct NmbSamplerDescriptor {
pub min_filter: u64,
@@ -725,6 +1304,22 @@ pub extern "C" fn nmb_command_buffer_release(command_buffer: *mut NmbCommandBuff
release(command_buffer)
}
#[unsafe(no_mangle)]
pub extern "C" fn nmb_command_buffer_uses_buffer(command_buffer: *const NmbCommandBuffer, buffer: *const NmbBuffer) -> i32 {
guard(-1, || {
let used = get(command_buffer)?.0.used_resources();
Ok(i32::from(used.references_resource(get(buffer)?.0.as_resource())))
})
}
#[unsafe(no_mangle)]
pub extern "C" fn nmb_command_buffer_uses_texture(command_buffer: *const NmbCommandBuffer, texture: *const NmbTexture) -> i32 {
guard(-1, || {
let used = get(command_buffer)?.0.used_resources();
Ok(i32::from(used.references_resource(get(texture)?.0.as_resource())))
})
}
struct Context(*mut c_void);
// SAFETY: the context is the caller's, handed back on the runtime's thread as the header states.
unsafe impl Send for Context {}
@@ -743,6 +1338,28 @@ pub extern "C" fn nmb_command_buffer_on_completed(command_buffer: *mut NmbComman
})
}
#[unsafe(no_mangle)]
pub extern "C" fn nmb_command_buffer_on_scheduled(command_buffer: *mut NmbCommandBuffer, handler: Option<extern "C" fn(*mut c_void)>,
context: *mut c_void) -> bool {
guard(false, || {
let handler = handler.ok_or("null handler")?;
let context = Context(context);
get(command_buffer)?.0.add_scheduled_handler(Box::new(move |_| {
let context = context;
handler(context.0)
})).map_err(e("scheduled handler"))?;
Ok(true)
})
}
#[unsafe(no_mangle)]
pub extern "C" fn nmb_command_buffer_wait_scheduled(command_buffer: *mut NmbCommandBuffer) -> bool {
guard(false, || {
get(command_buffer)?.0.wait_until_scheduled().map_err(e("wait scheduled"))?;
Ok(true)
})
}
#[unsafe(no_mangle)]
pub extern "C" fn nmb_command_buffer_commit(command_buffer: *mut NmbCommandBuffer) -> bool {
guard(false, || {
@@ -958,6 +1575,7 @@ pub struct NmbRenderPassDescriptor {
pub height: u64,
pub array_length: u64,
pub samples: u64,
pub visibility: *const NmbBuffer,
}
fn texture_of(a: &NmbPassAttachment) -> Result<Option<Texture>, String> {
@@ -999,6 +1617,9 @@ pub extern "C" fn nmb_render_encoder_new(command_buffer: *mut NmbCommandBuffer,
pass.render_target_height = d.height;
pass.render_target_array_length = d.array_length;
pass.default_raster_sample_count = d.samples;
if !d.visibility.is_null() {
pass.visibility_result_buffer = Some(get(d.visibility)?.0.clone());
}
let encoder = get(command_buffer)?.0.render_command_encoder(&pass).map_err(e("render encoder"))?;
Ok(boxed(NmbEncoder::Render(encoder)))
})
@@ -1229,6 +1850,32 @@ pub extern "C" fn nmb_render_set_stencil_reference(encoder: *mut NmbEncoder, fro
})
}
#[unsafe(no_mangle)]
pub extern "C" fn nmb_render_set_depth_bias(encoder: *mut NmbEncoder, bias: f32, slope_scale: f32, clamp: f32) -> bool {
guard(false, || {
render(encoder)?.set_depth_bias(bias, slope_scale, clamp).map_err(e("depth bias"))?;
Ok(true)
})
}
/// SPI `setLineWidth:`: the runtime accepts Metal's default width of 1 and refuses any other (Apple's wide-line rule is unmeasured).
#[unsafe(no_mangle)]
pub extern "C" fn nmb_render_set_line_width(encoder: *mut NmbEncoder, width: f32) -> bool {
guard(false, || {
render(encoder)?.set_line_width(width).map_err(e("line width"))?;
Ok(true)
})
}
#[unsafe(no_mangle)]
pub extern "C" fn nmb_render_set_visibility_mode(encoder: *mut NmbEncoder, mode: u64, offset: u64) -> bool {
guard(false, || {
let mode = VisibilityResultMode::from_raw(mode).map_err(e("visibility result mode"))?;
render(encoder)?.set_visibility_result_mode(mode, offset).map_err(e("visibility result mode"))?;
Ok(true)
})
}
#[unsafe(no_mangle)]
pub extern "C" fn nmb_render_set_blend_color(encoder: *mut NmbEncoder, color: *const f32) -> bool {
guard(false, || {
drivers/NativeMetal/bridge/archive_host.mm
+58
View File
@@ -0,0 +1,58 @@
// SPDX-License-Identifier: MIT
// Two processes, distinct initially empty caches: archive selection cannot be satisfied by
// the producer's live pipeline memo. A strict empty/changed-descriptor miss must refuse.
#include "FrontendChecks.h"
#include "NativeMetalObjects.h"
#include "../NativeBridgeMetalFrontend.h"
int main(int argc, char **argv) {
if (argc != 5) return 2;
@autoreleasepool {
const bool produce = !strcmp(argv[4], "produce");
id<MTLDevice> device = native_bridge_metal_open(argv[1], argv[2], 0x6e6d62, nullptr);
if (!device) return 2;
size_t bytes = 0;
const void *air = nmb_fixture_air(&bytes);
id<MTLLibrary> library = native_bridge_metal_library_with_air(air, bytes);
if (!library) return 2;
auto descriptor = [MTLRenderPipelineDescriptor new];
descriptor.vertexFunction = [library newFunctionWithName:@"colored_vertex"];
descriptor.fragmentFunction = [library newFunctionWithName:@"colored_fragment"];
descriptor.colorAttachments[0].pixelFormat = MTLPixelFormatRGBA8Unorm;
auto archiveDescriptor = [MTLBinaryArchiveDescriptor new];
NSURL *url = [NSURL fileURLWithPath:@(argv[3])];
if (!produce) archiveDescriptor.url = url;
NSError *error = nil;
id<MTLBinaryArchive> archive = [device newBinaryArchiveWithDescriptor:archiveDescriptor error:&error];
if (!archive) {std::printf("FAIL open archive: %s\n", error.localizedDescription.UTF8String);return 1;}
descriptor.binaryArchives = @[archive];
if (produce) {
id state = [device newRenderPipelineStateWithDescriptor:descriptor options:MTLPipelineOptionFailOnBinaryArchiveMiss reflection:nil error:&error];
if (state || !error) {std::puts("FAIL empty archive miss was silently compiled");return 1;}
if (![archive addRenderPipelineFunctionsWithDescriptor:descriptor error:&error] || ![archive serializeToURL:url error:&error]) {
std::printf("FAIL archive add/save: %s\n", error.localizedDescription.UTF8String);return 1;
}
}
error = nil;
id<MTLRenderPipelineState> state = [device newRenderPipelineStateWithDescriptor:descriptor options:MTLPipelineOptionFailOnBinaryArchiveMiss reflection:nil error:&error];
if (!state || error) {std::printf("FAIL strict archive hit: %s\n", error.localizedDescription.UTF8String);return 1;}
// A format not added to the archive must still miss, even after a valid pipeline is live.
descriptor.colorAttachments[0].pixelFormat = MTLPixelFormatBGRA8Unorm;
if ([device newRenderPipelineStateWithDescriptor:descriptor options:MTLPipelineOptionFailOnBinaryArchiveMiss reflection:nil error:&error] || !error) {
std::puts("FAIL changed-descriptor archive miss");return 1;
}
auto td = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatRGBA8Unorm width:16 height:16 mipmapped:NO];
td.usage = MTLTextureUsageRenderTarget;
id<MTLTexture> target = [device newTextureWithDescriptor:td];
id<MTLCommandBuffer> cb = [[device newCommandQueue] commandBuffer];
if (!target || !cb) return 2;
frontendchecks::Report report = {};
frontendchecks::gReport = &report;
frontendchecks::drawQuad(cb, state, target, nil, frontendchecks::kQuarter, 6);
if (!frontendchecks::finish(cb, "archived draw")) return 1;
uint8_t pixels[16 * 16 * 4] = {};
[target getBytes:pixels bytesPerRow:64 fromRegion:MTLRegionMake2D(0, 0, 16, 16) mipmapLevel:0];
if (frontendchecks::wrongTexels(pixels, 64, 16)) {std::puts("FAIL archived render pixels");return 1;}
std::printf("PASS: %s render archive, strict hit/miss and GPU pixel check (host only)\n", produce ? "produced" : "reloaded");
}
}
drivers/NativeMetal/bridge/carenderer_host.mm
+165
View File
@@ -0,0 +1,165 @@
// SPDX-License-Identifier: MIT
// Host instrument: the Mac's own QuartzCore renders a layer tree through CARenderer into a texture of the native frontend's device (bridge
// over a host Vulkan driver, MoltenVK on the Mac). CoreAnimation's Metal renderer then drives the frontend as the target's render server
// will, and the pixels it produced are checked. With NMB_DEVICE_BASE_NSOBJECT=1 every device selector the renderer sends that the frontend
// lacks is reported as missing. The Mac's QuartzCore is not the target's: this names gaps and checks results, it qualifies nothing on iPadOS.
#import <Foundation/Foundation.h>
#import <Metal/Metal.h>
#import <QuartzCore/QuartzCore.h>
#import <CoreGraphics/CoreGraphics.h>
#import <objc/runtime.h>
#include <cstdio>
#include <vector>
#include <set>
#include <string>
#include "../NativeBridgeMetalFrontend.h"
#include <cstring>
#include <ctime>
#import <CoreImage/CoreImage.h>
extern "C" size_t native_bridge_metal_telemetry(char *, size_t);
static std::set<std::string> missing;
static void onMissing(const char *cls, const char *sel) {
if (missing.insert(std::string(cls) + " " + sel).second) std::fprintf(stderr, "missing selector: %s %s\n", cls, sel);
}
// NMB_DISCOVER=1 (instrument only): an unimplemented selector is logged and answers zero instead of raising, so one run lists every gap on
// the renderer's path. Results after a zero answer mean nothing; the frontend itself never answers this way.
static const char *protocolTypes(SEL selector) {
Protocol *protocols[] = {@protocol(MTLDevice), @protocol(MTLTexture), @protocol(MTLBuffer), @protocol(MTLResource), @protocol(MTLCommandQueue),
@protocol(MTLCommandBuffer), @protocol(MTLRenderCommandEncoder), @protocol(MTLBlitCommandEncoder),
@protocol(MTLComputeCommandEncoder), @protocol(MTLCommandEncoder), @protocol(MTLLibrary), @protocol(MTLFunction),
@protocol(MTLRenderPipelineState), @protocol(MTLComputePipelineState), @protocol(MTLSamplerState),
@protocol(MTLDepthStencilState), @protocol(MTLHeap), @protocol(MTLFence), @protocol(MTLEvent), @protocol(MTLSharedEvent)};
for (Protocol *p : protocols)
for (BOOL required : {YES, NO}) {
objc_method_description m = protocol_getMethodDescription(p, selector, required, YES);
if (m.types) return m.types;
}
return nullptr;
}
static NSMethodSignature *discoverSignature(id self, SEL _cmd, SEL selector) {
const char *types = protocolTypes(selector);
if (!types) {onMissing(class_getName(object_getClass(self)), (std::string(sel_getName(selector)) + " (no protocol signature)").c_str()); types = "@@:";}
return [NSMethodSignature signatureWithObjCTypes:types];
}
static void discoverForward(id self, SEL _cmd, NSInvocation *invocation) {
onMissing(class_getName(object_getClass(self)), sel_getName(invocation.selector));
const NSUInteger length = invocation.methodSignature.methodReturnLength;
if (length) {std::vector<uint8_t> zero(length); [invocation setReturnValue:zero.data()];}
}
static void discover(Class cls) {
class_addMethod(cls, @selector(methodSignatureForSelector:), (IMP)discoverSignature, "@@::");
class_addMethod(cls, @selector(forwardInvocation:), (IMP)discoverForward, "v@:@");
}
static CGColorRef rgba(double r, double g, double b, double a) {
CGColorSpaceRef space = CGColorSpaceCreateWithName(kCGColorSpaceSRGB);
const CGFloat c[4] = {r, g, b, a};
CGColorRef color = CGColorCreate(space, c);
CGColorSpaceRelease(space);
return color;
}
// argv: loader, shader cache. Prints PASS or the first wrong pixel.
int main(int argc, char **argv) {
@autoreleasepool {
if (argc < 3) return 2;
// NMB_SYSTEM_DEVICE=1: the Mac's own Metal device, the instrument's baseline.
id<MTLDevice> device = getenv("NMB_SYSTEM_DEVICE") ? MTLCreateSystemDefaultDevice() : native_bridge_metal_open(argv[1], argv[2], 0x6e6d62, onMissing);
if (!device) {std::printf("FAIL device\n"); return 1;}
if (getenv("NMB_DISCOVER")) {discover(object_getClass(device)); discover(objc_getClass("NBObject"));}
const NSUInteger side = 64;
MTLTextureDescriptor *d = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatBGRA8Unorm width:side height:side mipmapped:NO];
d.usage = MTLTextureUsageRenderTarget | MTLTextureUsageShaderRead;
d.storageMode = MTLStorageModeShared;
id<MTLTexture> target = [device newTextureWithDescriptor:d];
id<MTLCommandQueue> queue = [device newCommandQueue];
if (!target || !queue) {std::printf("FAIL target or queue\n"); return 1;}
// Blue background; a red square in the top-left quarter; a half-transparent green square in the bottom-right quarter.
CALayer *root = [CALayer layer];
root.frame = CGRectMake(0, 0, side, side);
root.backgroundColor = rgba(0, 0, 1, 1);
CALayer *red = [CALayer layer];
red.frame = CGRectMake(0, 0, side / 2, side / 2);
red.backgroundColor = rgba(1, 0, 0, 1);
CALayer *green = [CALayer layer];
green.frame = CGRectMake(side / 2, side / 2, side / 2, side / 2);
green.backgroundColor = rgba(0, 1, 0, 1);
green.opacity = 0.5;
[root addSublayer:red];
[root addSublayer:green];
CARenderer *renderer = [CARenderer rendererWithMTLTexture:target options:@{kCARendererMetalCommandQueue: queue}];
if (!renderer) {std::printf("FAIL CARenderer refused the frontend's texture\n"); return 1;}
renderer.layer = root;
renderer.bounds = CGRectMake(0, 0, side, side);
// The layers' implicit transaction would commit at the run loop's end; the renderer sees only committed trees.
// NMB_BLUR_TIMING=1: also render a blurred, shadowed layer for several frames and print the frontend's creation telemetry (pipeline
// counts and times, VkPipeline makings), so the cost of the pipelines CoreAnimation's own renderer asks for can be measured.
if (getenv("NMB_BLUR_TIMING")) {
CALayer *blurred = [CALayer layer];
blurred.frame = CGRectMake(4, 4, side - 8, side - 8);
blurred.backgroundColor = rgba(1, 1, 0, 0.7);
blurred.cornerRadius = 9;
blurred.shadowOpacity = 0.8;
blurred.shadowRadius = 5;
CIFilter *blur = [CIFilter filterWithName:@"CIGaussianBlur"];
[blur setValue:@6 forKey:kCIInputRadiusKey];
blurred.filters = @[blur];
[root addSublayer:blurred];
[CATransaction flush];
for (int frame = 0; frame < 3; ++frame) {
const uint64_t t0 = clock_gettime_nsec_np(CLOCK_UPTIME_RAW);
[renderer beginFrameAtTime:CACurrentMediaTime() timeStamp:NULL];
[renderer addUpdateRect:renderer.bounds];
[renderer render];
[renderer endFrame];
id<MTLCommandBuffer> fence = [queue commandBuffer];
[fence commit];
[fence waitUntilCompleted];
std::printf("BLUR frame %d %.1f ms\n", frame, (clock_gettime_nsec_np(CLOCK_UPTIME_RAW) - t0) / 1e6);
}
static char table[16384];
native_bridge_metal_telemetry(table, sizeof table);
for (char *line = std::strtok(table, "\n"); line; line = std::strtok(nullptr, "\n"))
if (line[0] == 'T' || line[0] == 'V' || line[0] == 'S' || line[0] == 'W') std::printf("TELEMETRY %s\n", line);
[blurred removeFromSuperlayer];
[CATransaction flush];
}
[renderer beginFrameAtTime:CACurrentMediaTime() timeStamp:NULL];
[renderer addUpdateRect:renderer.bounds];
[renderer render];
[renderer endFrame];
// CARenderer commits its own command buffers; one more on the same queue completes after them.
id<MTLCommandBuffer> last = [queue commandBuffer];
[last commit];
[last waitUntilCompleted];
std::vector<uint8_t> pixels(side * side * 4);
[target getBytes:pixels.data() bytesPerRow:side * 4 fromRegion:MTLRegionMake2D(0, 0, side, side) mipmapLevel:0];
// BGRA. Row 0 of the texture is the layer tree's y 0 (the Mac's Metal device renders it so, NMB_SYSTEM_DEVICE=1): red top-left,
// green bottom-right.
auto at = [&](NSUInteger x, NSUInteger y) {return &pixels[(y * side + x) * 4];};
struct Probe {NSUInteger x, y; int b, g, r;} probes[] = {
{8, 8, 0, 0, 255}, // red quarter
{side - 8, side - 8, 128, 128, 0}, // green at 0.5 over blue: (0, 0.5, 0.5)
{side - 8, 8, 255, 0, 0}, // background
{8, side - 8, 255, 0, 0}, // background
};
bool ok = true;
for (auto &p : probes) {
const uint8_t *px = at(p.x, p.y);
if (std::abs(px[0] - p.b) > 3 || std::abs(px[1] - p.g) > 3 || std::abs(px[2] - p.r) > 3 || px[3] != 255) {
std::printf("FAIL pixel (%lu,%lu) = b%u g%u r%u a%u, want b%d g%d r%d\n", (unsigned long)p.x, (unsigned long)p.y, px[0], px[1], px[2], px[3],
p.b, p.g, p.r);
ok = false;
}
}
std::printf("missing selectors: %zu\n", missing.size());
if (!ok) return 1;
std::printf("PASS: the Mac's CARenderer rendered a layer tree on %s\n", device.name.UTF8String);
return 0;
}
}
drivers/NativeMetal/bridge/compute_archive_host.mm
+90
View File
@@ -0,0 +1,90 @@
// SPDX-License-Identifier: MIT
// Separate processes/cache directories distinguish archive consumption from live memo hits.
#import <Foundation/Foundation.h>
#import <Metal/Metal.h>
#include "NativeMetalObjects.h"
#include "../NativeBridgeMetalFrontend.h"
#include <cstdio>
#include <cstring>
#define REQUIRE(condition) do {if (!(condition)) {std::printf("FAIL line %d: %s; %s\n", __LINE__, #condition, error.localizedDescription.UTF8String ?: "");return 1;}} while (0)
int main(int argc, char **argv) {
if (argc != 5) return 2;
@autoreleasepool {
NSError *error = nil;
const bool produce = !strcmp(argv[4], "produce");
id<MTLDevice> device = native_bridge_metal_open(argv[1], argv[2], 0x6e6d62, nullptr);
REQUIRE(device);
size_t bytes = 0;
const void *air = nmb_function_constant_air(&bytes);
id<MTLLibrary> library = native_bridge_metal_library_with_air(air, bytes);
REQUIRE(library);
auto constants = [MTLFunctionConstantValues new];
int mode = 2;
[constants setConstantValue:&mode type:MTLDataTypeInt atIndex:0];
id<MTLFunction> function = [library newFunctionWithName:@"fcfold" constantValues:constants error:&error];
REQUIRE(function && !error);
REQUIRE([device respondsToSelector:@selector(newComputePipelineStateWithDescriptor:options:reflection:error:)]);
auto descriptor = [MTLComputePipelineDescriptor new];
descriptor.computeFunction = function;
descriptor.maxTotalThreadsPerThreadgroup = 32;
descriptor.buffers[1].mutability = MTLMutabilityMutable;
descriptor.label = @"archived compute";
auto archiveDescriptor = [MTLBinaryArchiveDescriptor new];
NSURL *url = [NSURL fileURLWithPath:@(argv[3])];
if (!produce) archiveDescriptor.url = url;
id<MTLBinaryArchive> archive = [device newBinaryArchiveWithDescriptor:archiveDescriptor error:&error];
REQUIRE(archive && !error);
descriptor.binaryArchives = @[archive];
if (produce) {
id<MTLComputePipelineState> miss = [device newComputePipelineStateWithDescriptor:descriptor options:MTLPipelineOptionFailOnBinaryArchiveMiss reflection:nil error:&error];
if (miss || !error) {
std::printf("FAIL strict empty archive: state=%s error=%s\n", miss ? "created" : "nil", error.localizedDescription.UTF8String ?: "nil");
return 1;
}
REQUIRE([archive addComputePipelineFunctionsWithDescriptor:descriptor error:&error] && !error);
REQUIRE([archive serializeToURL:url error:&error] && !error);
}
id<MTLComputePipelineState> state = [device newComputePipelineStateWithDescriptor:descriptor options:MTLPipelineOptionFailOnBinaryArchiveMiss reflection:nil error:&error];
REQUIRE(state && !error && state.maxTotalThreadsPerThreadgroup == 32);
REQUIRE([state.label isEqualToString:descriptor.label]);
// Both descriptor fields and function constants must retain their archive identity.
descriptor.maxTotalThreadsPerThreadgroup = 16;
REQUIRE(![device newComputePipelineStateWithDescriptor:descriptor options:MTLPipelineOptionFailOnBinaryArchiveMiss reflection:nil error:&error] && error);
descriptor.maxTotalThreadsPerThreadgroup = 32;
descriptor.buffers[1].mutability = MTLMutabilityDefault;
REQUIRE(![device newComputePipelineStateWithDescriptor:descriptor options:MTLPipelineOptionFailOnBinaryArchiveMiss reflection:nil error:&error] && error);
descriptor.buffers[1].mutability = MTLMutabilityMutable;
mode = 3;
[constants setConstantValue:&mode type:MTLDataTypeInt atIndex:0];
descriptor.computeFunction = [library newFunctionWithName:@"fcfold" constantValues:constants error:&error];
REQUIRE(descriptor.computeFunction && !error);
REQUIRE(![device newComputePipelineStateWithDescriptor:descriptor options:MTLPipelineOptionFailOnBinaryArchiveMiss reflection:nil error:&error] && error);
descriptor.computeFunction = function;
descriptor.stageInputDescriptor = [MTLStageInputOutputDescriptor stageInputOutputDescriptor];
descriptor.stageInputDescriptor.attributes[0].format = MTLAttributeFormatFloat;
// Unsupported descriptor state must be refused by both population and creation.
REQUIRE(![archive addComputePipelineFunctionsWithDescriptor:descriptor error:&error] && error);
REQUIRE(![device newComputePipelineStateWithDescriptor:descriptor options:MTLPipelineOptionNone reflection:nil error:&error] && error);
descriptor.stageInputDescriptor = nil;
const float inputValue = 2;
id<MTLBuffer> input = [device newBufferWithBytes:&inputValue length:4 options:MTLResourceStorageModeShared];
id<MTLBuffer> output = [device newBufferWithLength:128 options:MTLResourceStorageModeShared];
id<MTLCommandBuffer> command = [[device newCommandQueue] commandBuffer];
id<MTLComputeCommandEncoder> encoder = [command computeCommandEncoder];
REQUIRE(input && output && encoder);
[encoder setComputePipelineState:state];
[encoder setBuffer:input offset:0 atIndex:0];
[encoder setBuffer:output offset:0 atIndex:1];
[encoder dispatchThreads:MTLSizeMake(1, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];
[encoder endEncoding];
[command commit];
[command waitUntilCompleted];
REQUIRE(command.status == MTLCommandBufferStatusCompleted);
REQUIRE(((const float *)output.contents)[24] == 4000);
// Check the no-list behavior only after the cold archive consumer and GPU verdict.
REQUIRE([device newComputePipelineStateWithFunction:function options:MTLPipelineOptionFailOnBinaryArchiveMiss reflection:nil error:&error] && !error);
std::printf("PASS: %s compute archive, strict descriptor misses and host GPU output\n", produce ? "produced" : "reloaded");
}
}
drivers/NativeMetal/bridge/driver_cache_host.cpp
+60
View File
@@ -0,0 +1,60 @@
// SPDX-License-Identifier: MIT
// The default compute-function path used by the target; no explicit binary archive.
// Runs over Turnip and the software kernel peer: no shader output is claimed.
#include "NativeMetalObjects.h"
#include <cstdio>
#include <cstring>
#include <dlfcn.h>
#define REQUIRE(x) do {if (!(x)) {std::fprintf(stderr,"FAIL line %d: %s; %s\n",__LINE__,#x,nmb_last_error());return 1;}} while (0)
int main(int argc,char **argv) {
if(argc!=4)return 2;
const bool consume=!std::strcmp(argv[3],"consume");
REQUIRE(consume||!std::strcmp(argv[3],"produce"));
auto submissions=(unsigned(*)())dlsym(RTLD_DEFAULT,"host_gpu_submissions");REQUIRE(submissions);
NmbDevice *device=nmb_device_open(argv[1],argv[2]);REQUIRE(device);
NmbShaderCacheStats before={},after={};REQUIRE(nmb_device_shader_cache_stats(device,&before));
size_t size=0;const void *air=nmb_fixture_air(&size);
NmbLibrary *library=nmb_library_new(device,air,size,true);REQUIRE(library);
NmbFunction *function=nmb_function_new(library,"scale_and_offset");REQUIRE(function);
NmbComputePipeline *pipeline=nmb_compute_pipeline_new(device,function);REQUIRE(pipeline);
float input[64]={};
NmbBuffer *source=nmb_buffer_new(device,sizeof input,0,input);
NmbBuffer *destination=nmb_buffer_new(device,sizeof input,0,nullptr);REQUIRE(source&&destination);
NmbQueue *queue=nmb_queue_new(device);REQUIRE(queue);
NmbCommandBuffer *command=nmb_command_buffer_new(queue);REQUIRE(command);
NmbEncoder *encoder=nmb_compute_encoder_new(command);REQUIRE(encoder);
REQUIRE(nmb_compute_set_pipeline(encoder,pipeline));
REQUIRE(nmb_encoder_set_buffer(encoder,0,source,0,0));
REQUIRE(nmb_encoder_set_buffer(encoder,0,destination,0,1));
const float scale=2;REQUIRE(nmb_encoder_set_bytes(encoder,0,&scale,sizeof scale,2));
const uint64_t threads[]={64,1,1};
REQUIRE(nmb_compute_dispatch(encoder,threads,threads,true));
REQUIRE(nmb_encoder_end(encoder));
const unsigned submitted=submissions();
REQUIRE(nmb_command_buffer_commit(command));
REQUIRE(nmb_command_buffer_wait(command,10000)==0);
REQUIRE(nmb_command_buffer_status(command)==4&&submissions()>submitted);
REQUIRE(nmb_device_shader_cache_stats(device,&after));
std::printf("cache %s: artifacts hit=%llu miss=%llu write=%llu; driver load=%llu hit=%llu miss=%llu save=%llu same=%llu; rejected=%llu readfail=%llu writefail=%llu truncated=%llu mismatched=%llu\n",argv[3],
(unsigned long long)(after.hits-before.hits),(unsigned long long)(after.misses-before.misses),(unsigned long long)(after.writes-before.writes),
(unsigned long long)(after.driver_loads-before.driver_loads),(unsigned long long)(after.driver_hits-before.driver_hits),
(unsigned long long)(after.driver_misses-before.driver_misses),(unsigned long long)(after.driver_saves-before.driver_saves),
(unsigned long long)(after.driver_unchanged-before.driver_unchanged),(unsigned long long)after.rejected,(unsigned long long)after.read_failures,
(unsigned long long)after.write_failures,(unsigned long long)after.driver_truncated,(unsigned long long)after.driver_mismatched);
if(consume){
REQUIRE(after.hits>before.hits&&after.misses==before.misses);
REQUIRE(after.driver_loads>before.driver_loads&&after.driver_hits>before.driver_hits);
REQUIRE(after.driver_misses==before.driver_misses);
}else{
REQUIRE(after.misses>before.misses&&after.writes>before.writes);
REQUIRE(after.driver_misses>before.driver_misses&&after.driver_saves>before.driver_saves);
}
REQUIRE(!after.read_failures&&!after.write_failures&&!after.rejected&&!after.driver_truncated&&!after.driver_mismatched);
nmb_command_buffer_release(command);nmb_queue_release(queue);
nmb_buffer_release(destination);nmb_buffer_release(source);nmb_compute_pipeline_release(pipeline);
nmb_function_release(function);nmb_library_release(library);nmb_device_release(device);
std::printf("PASS: default compute %s uses separate artifact/Turnip caches; software transport, results unchecked\n",argv[3]);
return 0;
}
drivers/NativeMetal/bridge/frontend_host.mm
+1 -1
View File
@@ -23,7 +23,7 @@ int main(int argc, char **argv) {
if (!library || !fcLibrary) {std::printf("FAIL fixture libraries\n"); return 1;}
frontendchecks::Report r = {};
frontendchecks::run(device, library, fcLibrary, argc > 3 && argv[3][0] ? @(argv[3]) : nil, &r);
if (r.qcFunctions) std::printf("QuartzCore: %u functions, kernels %u/%u, pairs %u/%u\n", r.qcFunctions, r.qcKernelsOk, r.qcKernels, r.qcPairsOk, r.qcPairs);
if (r.qcFunctions) std::printf("QuartzCore bare-descriptor probe: %u functions, kernels %u/%u, pairs %u/%u (not complete shader coverage)\n", r.qcFunctions, r.qcKernelsOk, r.qcKernels, r.qcPairsOk, r.qcPairs);
for (auto &m : missing) std::printf("missing selector: %s\n", m.c_str());
if (r.failed) {std::printf("FAILED %u checks, first: %s\n", r.failed, r.first); return 1;}
std::printf("PASS: native Metal frontend over the bridge on %s: compute, render, IOSurface render, surface sampling, texture compute, "
drivers/NativeMetal/bridge/graphics_cache_loader.cpp
+115
View File
@@ -0,0 +1,115 @@
/* SPDX-License-Identifier: MIT */
/* Offline qualification only: forward to the actual Vulkan compiler, enabling
* its reported cache-control feature and optionally forbidding compilation.
* A negative control corrupts only a private copy of a cache UUID. Never deploy. */
#include <vulkan/vulkan.h>
#include <atomic>
#include <cstdlib>
#include <cstring>
#include <dlfcn.h>
#include <mutex>
#include <vector>
static PFN_vkGetInstanceProcAddr realInstance;
static std::atomic<PFN_vkGetDeviceProcAddr> realDevice;
static std::once_flag initialized;
static std::atomic<VkInstance> activeInstance;
static std::atomic<bool> strict, badUuid;
// calls, success, compile-required, valid feedback, whole-cache hit, UUID mutations, devices,
// calls with an application cache, their valid feedback and their cache hits
static std::atomic<uint64_t> counts[10];
static void initialize() {
std::call_once(initialized,[]{
const char *path=std::getenv("NMB_TEST_REAL_VULKAN_LOADER");
void *library=path?dlopen(path,RTLD_NOW|RTLD_LOCAL):nullptr;
if(!library)std::abort();
realInstance=reinterpret_cast<PFN_vkGetInstanceProcAddr>(dlsym(library,"vkGetInstanceProcAddr"));
if(!realInstance)realInstance=reinterpret_cast<PFN_vkGetInstanceProcAddr>(dlsym(library,"vk_icdGetInstanceProcAddr"));
if(!realInstance)std::abort();
});
}
static VKAPI_ATTR VkResult VKAPI_CALL createDevice(VkPhysicalDevice physical,const VkDeviceCreateInfo *info,
const VkAllocationCallbacks *allocator,VkDevice *device) {
initialize();
// The qualified runtime requests Vulkan 1.2. Refuse an unexpected chain
// rather than duplicate a promoted feature or alter caller-owned structures.
for(auto p=static_cast<const VkBaseInStructure *>(info->pNext);p;p=p->pNext)
if(p->sType==VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_3_FEATURES ||
p->sType==VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PIPELINE_CREATION_CACHE_CONTROL_FEATURES)return VK_ERROR_FEATURE_NOT_PRESENT;
auto query=reinterpret_cast<PFN_vkGetPhysicalDeviceFeatures2>(realInstance(activeInstance.load(),"vkGetPhysicalDeviceFeatures2"));
if(!query)return VK_ERROR_FEATURE_NOT_PRESENT;
VkPhysicalDevicePipelineCreationCacheControlFeatures supported={};supported.sType=VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PIPELINE_CREATION_CACHE_CONTROL_FEATURES;
VkPhysicalDeviceFeatures2 features={};features.sType=VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2;features.pNext=&supported;
query(physical,&features);if(!supported.pipelineCreationCacheControl)return VK_ERROR_FEATURE_NOT_PRESENT;
std::vector<const char *> extensions;
if(info->enabledExtensionCount)extensions.assign(info->ppEnabledExtensionNames,info->ppEnabledExtensionNames+info->enabledExtensionCount);
bool listed=false;for(auto name:extensions)listed|=!std::strcmp(name,VK_EXT_PIPELINE_CREATION_CACHE_CONTROL_EXTENSION_NAME);
if(!listed)extensions.push_back(VK_EXT_PIPELINE_CREATION_CACHE_CONTROL_EXTENSION_NAME);
supported.pNext=const_cast<void *>(info->pNext);
VkDeviceCreateInfo copy=*info;copy.pNext=&supported;copy.enabledExtensionCount=(uint32_t)extensions.size();copy.ppEnabledExtensionNames=extensions.data();
auto real=reinterpret_cast<PFN_vkCreateDevice>(realInstance(activeInstance.load(),"vkCreateDevice"));
if(!real)return VK_ERROR_INITIALIZATION_FAILED;
const auto result=real(physical,&copy,allocator,device);if(result==VK_SUCCESS)counts[6]++;
return result;
}
static VKAPI_ATTR VkResult VKAPI_CALL createCache(VkDevice device,const VkPipelineCacheCreateInfo *info,
const VkAllocationCallbacks *allocator,VkPipelineCache *cache) {
VkPipelineCacheCreateInfo copy=*info;std::vector<unsigned char> bytes;
if(badUuid.load() && info->initialDataSize) {
if(info->initialDataSize<32)return VK_ERROR_INITIALIZATION_FAILED;
const auto *source=static_cast<const unsigned char *>(info->pInitialData);
bytes.assign(source,source+info->initialDataSize);bytes[16]^=1;copy.pInitialData=bytes.data();counts[5]++;
}
auto real=reinterpret_cast<PFN_vkCreatePipelineCache>(realDevice.load()(device,"vkCreatePipelineCache"));
return real(device,&copy,allocator,cache);
}
static VKAPI_ATTR VkResult VKAPI_CALL createGraphics(VkDevice device,VkPipelineCache cache,uint32_t count,
const VkGraphicsPipelineCreateInfo *infos,const VkAllocationCallbacks *allocator,VkPipeline *pipelines) {
if(!count || count>64)return VK_ERROR_INITIALIZATION_FAILED;
std::vector<VkGraphicsPipelineCreateInfo> copies(infos,infos+count);
std::vector<VkPipelineCreationFeedback> feedback(count);
std::vector<VkPipelineCreationFeedbackCreateInfo> additions(count);
std::vector<const VkPipelineCreationFeedback *> observed(count);
for(uint32_t i=0;i<count;++i) {
if(strict.load())copies[i].flags|=VK_PIPELINE_CREATE_FAIL_ON_PIPELINE_COMPILE_REQUIRED_BIT;
for(auto p=static_cast<const VkBaseInStructure *>(copies[i].pNext);p;p=p->pNext)
if(p->sType==VK_STRUCTURE_TYPE_PIPELINE_CREATION_FEEDBACK_CREATE_INFO)
observed[i]=reinterpret_cast<const VkPipelineCreationFeedbackCreateInfo *>(p)->pPipelineCreationFeedback;
if(!observed[i]) {
auto &a=additions[i];a.sType=VK_STRUCTURE_TYPE_PIPELINE_CREATION_FEEDBACK_CREATE_INFO;
a.pNext=copies[i].pNext;a.pPipelineCreationFeedback=&feedback[i];copies[i].pNext=&a;observed[i]=&feedback[i];
}
}
auto real=reinterpret_cast<PFN_vkCreateGraphicsPipelines>(realDevice.load()(device,"vkCreateGraphicsPipelines"));
counts[0]+=count;if(cache)counts[7]+=count;const auto result=real(device,cache,count,copies.data(),allocator,pipelines);
if(result==VK_PIPELINE_COMPILE_REQUIRED)counts[2]++;
for(uint32_t i=0;i<count;++i)if(pipelines[i]) {
counts[1]++;
if(observed[i] && (observed[i]->flags&VK_PIPELINE_CREATION_FEEDBACK_VALID_BIT)) {
counts[3]++;if(cache)counts[8]++;
if(observed[i]->flags&VK_PIPELINE_CREATION_FEEDBACK_APPLICATION_PIPELINE_CACHE_HIT_BIT){counts[4]++;if(cache)counts[9]++;}
}
}
return result;
}
extern "C" VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL vkGetDeviceProcAddr(VkDevice device,const char *name) {
initialize();
if(!std::strcmp(name,"vkCreateGraphicsPipelines"))return reinterpret_cast<PFN_vkVoidFunction>(createGraphics);
if(!std::strcmp(name,"vkCreatePipelineCache"))return reinterpret_cast<PFN_vkVoidFunction>(createCache);
return realDevice.load()(device,name);
}
extern "C" VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL vkGetInstanceProcAddr(VkInstance instance,const char *name) {
initialize();
if(instance){
activeInstance=instance;
realDevice=reinterpret_cast<PFN_vkGetDeviceProcAddr>(realInstance(instance,"vkGetDeviceProcAddr"));
if(!realDevice.load())std::abort();
}
if(!std::strcmp(name,"vkGetDeviceProcAddr"))return reinterpret_cast<PFN_vkVoidFunction>(vkGetDeviceProcAddr);
if(!std::strcmp(name,"vkCreateDevice"))return reinterpret_cast<PFN_vkVoidFunction>(createDevice);
if(!std::strcmp(name,"vkCreateGraphicsPipelines"))return reinterpret_cast<PFN_vkVoidFunction>(createGraphics);
if(!std::strcmp(name,"vkCreatePipelineCache"))return reinterpret_cast<PFN_vkVoidFunction>(createCache);
return realInstance(instance,name);
}
extern "C" void nmb_test_graphics_cache_control(bool forbidCompilation,bool corruptUuid) {strict=forbidCompilation;badUuid=corruptUuid;}
extern "C" void nmb_test_graphics_cache_counts(uint64_t out[10]) {for(unsigned i=0;i<10;++i)out[i]=counts[i].load();}
drivers/NativeMetal/bridge/import_count_loader.cpp
+60
View File
@@ -0,0 +1,60 @@
/* SPDX-License-Identifier: MIT */
/* Host-test Vulkan forwarding loader. Every request reaches the real driver;
* count successful external-host-memory imports without changing any request,
* code, driver identity, memory contents, fence or GPU result. Never deployed. */
#include <vulkan/vulkan.h>
#include <atomic>
#include <chrono>
#include <cstdlib>
#include <cstring>
#include <dlfcn.h>
#include <mutex>
#include <thread>
static PFN_vkGetInstanceProcAddr realInstance;
static PFN_vkGetDeviceProcAddr realDevice;
static std::once_flag initialized;
static std::atomic<uint64_t> imported;
static std::atomic<unsigned> importDelayMs;
static void initialize() {
std::call_once(initialized,[]{
const char *path=std::getenv("NMB_TEST_REAL_VULKAN_LOADER");
void *library=path?dlopen(path,RTLD_NOW|RTLD_LOCAL):nullptr;
if(!library)std::abort();
realInstance=reinterpret_cast<PFN_vkGetInstanceProcAddr>(dlsym(library,"vkGetInstanceProcAddr"));
realDevice=reinterpret_cast<PFN_vkGetDeviceProcAddr>(dlsym(library,"vkGetDeviceProcAddr"));
if(!realInstance || !realDevice)std::abort();
});
}
static VKAPI_ATTR VkResult VKAPI_CALL allocate(VkDevice device,const VkMemoryAllocateInfo *info,
const VkAllocationCallbacks *allocator,VkDeviceMemory *memory) {
initialize();
bool hostImport=false;
for(auto node=static_cast<const VkBaseInStructure *>(info->pNext);node;node=node->pNext)
if(node->sType==VK_STRUCTURE_TYPE_IMPORT_MEMORY_HOST_POINTER_INFO_EXT){hostImport=true;break;}
// A bounded, one-shot delay makes a concurrent first-lookup miss
// reproducible. The actual allocation and its result are unchanged.
if(hostImport) {
const unsigned delay=importDelayMs.exchange(0);
if(delay)std::this_thread::sleep_for(std::chrono::milliseconds(delay));
}
auto real=reinterpret_cast<PFN_vkAllocateMemory>(realDevice(device,"vkAllocateMemory"));
const VkResult result=real(device,info,allocator,memory);
if(result==VK_SUCCESS && hostImport)imported++;
return result;
}
extern "C" VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL vkGetDeviceProcAddr(VkDevice device,const char *name) {
initialize();
if(!std::strcmp(name,"vkAllocateMemory"))return reinterpret_cast<PFN_vkVoidFunction>(allocate);
return realDevice(device,name);
}
extern "C" VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL vkGetInstanceProcAddr(VkInstance instance,const char *name) {
initialize();
if(!std::strcmp(name,"vkGetDeviceProcAddr"))return reinterpret_cast<PFN_vkVoidFunction>(vkGetDeviceProcAddr);
if(!std::strcmp(name,"vkAllocateMemory"))return reinterpret_cast<PFN_vkVoidFunction>(allocate);
return realInstance(instance,name);
}
extern "C" uint64_t nmb_test_vk_host_import_count() {return imported.load();}
extern "C" void nmb_test_vk_host_import_delay(unsigned milliseconds) {
importDelayMs.store(milliseconds<=100?milliseconds:100);
}
drivers/NativeMetal/bridge/no_copy_import_host.mm
+144
View File
@@ -0,0 +1,144 @@
// SPDX-License-Identifier: MIT
// Actual public Metal no-copy calls through the native bridge. Every buffer
// remains a distinct owner; count real Vulkan imports and check real GPU bytes.
#import <Foundation/Foundation.h>
#import <Metal/Metal.h>
#import <IOSurface/IOSurfaceRef.h>
#include <array>
#include <atomic>
#include <condition_variable>
#include <cstdio>
#include <cstring>
#include <dlfcn.h>
#include <memory>
#include <mutex>
#include <sys/mman.h>
#include <thread>
#include <unistd.h>
#include <vector>
#include "../NativeBridgeMetalFrontend.h"
constexpr unsigned workers=16;
struct State {std::array<std::atomic<unsigned>,workers> released{};std::atomic<unsigned> errors{0};};
static bool creation(id<MTLDevice> device,bool mixed,uint64_t (*imports)(),void (*delay)(unsigned)) {
auto state=std::make_shared<State>();bool gpu=true;
void *mapping=nullptr;size_t bytes=0;IOSurfaceRef surface=nullptr;
uint64_t allocated=0;
@autoreleasepool {
if(mixed) {
surface=IOSurfaceCreate((__bridge CFDictionaryRef)@{(id)kIOSurfaceWidth:@64,(id)kIOSurfaceHeight:@16,
(id)kIOSurfaceBytesPerElement:@4,(id)kIOSurfaceBytesPerRow:@256,(id)kIOSurfacePixelFormat:@((uint32_t)0x52474241)});
if(!surface)return false;
mapping=IOSurfaceGetBaseAddress(surface);bytes=IOSurfaceGetAllocSize(surface);
} else {
bytes=(size_t)getpagesize();mapping=mmap(nullptr,bytes,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANON,-1,0);
if(mapping==MAP_FAILED)return false;
}
if(!mapping || bytes<4096)return false;
memset(mapping,0xcc,bytes);
auto buffers=[NSMutableArray array],textures=[NSMutableArray array];
std::mutex startLock;std::condition_variable ready;unsigned waiting=0;bool start=false;
std::vector<std::thread> threads;
const uint64_t before=imports();delay(25);
for(unsigned i=0;i<workers;++i)threads.emplace_back([&,i]{
{std::unique_lock<std::mutex> lock(startLock);++waiting;ready.notify_all();ready.wait(lock,[&]{return start;});}
@autoreleasepool {
if(mixed && (i&1)) {
auto d=[MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatRGBA8Unorm width:64 height:16 mipmapped:NO];
d.storageMode=MTLStorageModeShared;d.usage=MTLTextureUsageShaderRead|MTLTextureUsageShaderWrite;
id<MTLTexture> texture=[device newTextureWithDescriptor:d iosurface:surface plane:0];
if(!texture){state->errors++;return;}
@synchronized(textures){[textures addObject:texture];}
} else {
auto owner=state;const unsigned ordinal=i;
id<MTLBuffer> buffer=[device newBufferWithBytesNoCopy:mapping length:bytes options:MTLResourceStorageModeShared
deallocator:^(void *,NSUInteger){owner->released[ordinal]++;}];
if(!buffer){state->errors++;return;}
@synchronized(buffers){[buffers addObject:buffer];}
}
}
});
{std::unique_lock<std::mutex> lock(startLock);ready.wait(lock,[&]{return waiting==workers;});start=true;ready.notify_all();}
for(auto &thread:threads)thread.join();
allocated=imports()-before;
const unsigned expected=mixed?workers/2:workers;
if(state->errors || buffers.count!=expected || textures.count!=(mixed?workers/2:0))return false;
@autoreleasepool {
// The API still returns distinct buffer objects and independent
// callbacks; only the driver's import is shared.
if([NSSet setWithArray:buffers].count!=expected)return false;
}
id<MTLCommandQueue> queue=[device newCommandQueue];
id<MTLCommandBuffer> write=[queue commandBuffer];id<MTLBlitCommandEncoder> fill=[write blitCommandEncoder];
[fill fillBuffer:buffers[0] range:NSMakeRange(0,4096) value:0x5a];[fill endEncoding];[write commit];[write waitUntilCompleted];
// Use an explicit completed producer, so correctness here does not
// assume implicit synchronization of arbitrary no-copy Metal aliases.
id<MTLBuffer> output=[device newBufferWithLength:workers*4096 options:MTLResourceStorageModeShared];
id<MTLCommandBuffer> read=[queue commandBuffer];id<MTLBlitCommandEncoder> copy=[read blitCommandEncoder];
if(!queue || !write || !fill || !output || !read || !copy)return false;
NSUInteger at=0;
for(id<MTLBuffer> buffer in buffers){[copy copyFromBuffer:buffer sourceOffset:0 toBuffer:output destinationOffset:at size:4096];at+=4096;}
for(id<MTLTexture> texture in textures){
[copy copyFromTexture:texture sourceSlice:0 sourceLevel:0 sourceOrigin:MTLOriginMake(0,0,0) sourceSize:MTLSizeMake(64,16,1)
toBuffer:output destinationOffset:at destinationBytesPerRow:256 destinationBytesPerImage:4096];at+=4096;
}
[copy endEncoding];[read commit];[read waitUntilCompleted];
gpu=write.status==MTLCommandBufferStatusCompleted && read.status==MTLCommandBufferStatusCompleted && at==workers*4096;
for(unsigned i=0;i<workers*4096;++i)if(((const unsigned char *)output.contents)[i]!=0x5a){gpu=false;break;}
}
bool owners=false;
for(unsigned attempt=0;attempt<300;++attempt) {
owners=true;
for(unsigned i=0;i<workers;++i)if(state->released[i].load()!=(mixed&&(i&1)?0u:1u))owners=false;
if(owners)break;usleep(1000);
}
if(surface)CFRelease(surface);else munmap(mapping,bytes);
std::printf("mode=%s buffers=%u vk_imports=%llu gpu=%s independent_owners=%s\n",mixed?"mixed":"raw",mixed?workers/2:workers,
(unsigned long long)allocated,gpu?"pass":"FAIL",owners?"pass":"FAIL");
return allocated==1 && gpu && owners;
}
static bool reentry(id<MTLDevice> device) {
const size_t page=(size_t)getpagesize();void *mapping=mmap(nullptr,page*2,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANON,-1,0);
if(mapping==MAP_FAILED)return false;
auto finished=dispatch_semaphore_create(0);auto state=std::make_shared<State>();
@autoreleasepool {
id<MTLBuffer> outer=[device newBufferWithBytesNoCopy:mapping length:page options:MTLResourceStorageModeShared
deallocator:^(void *,NSUInteger){
state->released[0]++;
dispatch_async(dispatch_get_global_queue(QOS_CLASS_DEFAULT,0),^{
@autoreleasepool {
id<MTLBuffer> nested=[device newBufferWithBytesNoCopy:(char *)mapping+page length:page options:MTLResourceStorageModeShared
deallocator:^(void *,NSUInteger){state->released[1]++;}];
if(!nested)state->errors++;
}
dispatch_semaphore_signal(finished);
});
if(dispatch_semaphore_wait(finished,dispatch_time(DISPATCH_TIME_NOW,3*NSEC_PER_SEC)))state->errors++;
}];
if(!outer)return false;
}
const bool ok=!state->errors && state->released[0]==1 && state->released[1]==1;
// On a timeout keep the test-owned mapping alive until process exit;
// never turn a lifetime failure into a submitted dangling pointer.
if(ok)munmap(mapping,page*2);
std::printf("deallocator cross-thread reentry=%s\n",ok?"pass":"FAIL");return ok;
}
int main(int argc,char **argv) {
if(argc!=4)return 2;
@autoreleasepool {
id<MTLDevice> device=native_bridge_metal_open(argv[1],argv[2],0x6e6d62,nullptr);if(!device)return 2;
void *loader=dlopen(argv[1],RTLD_NOW|RTLD_LOCAL);if(!loader)return 2;
auto imports=(uint64_t (*)())dlsym(loader,"nmb_test_vk_host_import_count");
auto delay=(void (*)(unsigned))dlsym(loader,"nmb_test_vk_host_import_delay");
if(!imports || !delay)return 2;
bool ok=false;
if(!strcmp(argv[3],"raw"))ok=creation(device,false,imports,delay);
else if(!strcmp(argv[3],"mixed"))ok=creation(device,true,imports,delay);
else if(!strcmp(argv[3],"reentry"))ok=reentry(device);
if(!ok)return 1;
std::puts("PASS: native no-copy construction coalesces real imports while retaining independent objects, GPU bytes and callbacks");
}
}
drivers/NativeMetal/bridge/pipeline_timing_host.mm
+130
View File
@@ -0,0 +1,130 @@
// SPDX-License-Identifier: MIT
// Host instrument: how long the target's own QuartzCore shaders take through the native frontend (metal2vulkan, then the Vulkan driver given as
// loader: the Adreno Turnip over the host simulation, or MoltenVK). Every <name>_vert_<p> / <name>_frag_<p> pair becomes a BGRA8 render pipeline
// and every kernel a compute pipeline, each timed; with METAL2VULKAN_LOG_PIPELINES=1 the runtime also prints the VkPipeline creation time of each,
// which separates the driver's compile from the translation. The host CPU is not the tablet's: compare the split, not the absolute numbers.
// argv: loader, shader cache dir, QuartzCore default.metallib, [limit], [draw]. With draw=1 each render state made is drawn once (three
// vertices into a 64x64 BGRA8 target, a zeroed buffer at every vertex and fragment index 0..7), which makes its VkPipeline: the runtime
// makes it at the draw, so without a draw the creation time is the translation alone.
#import <Foundation/Foundation.h>
#import <Metal/Metal.h>
#include <algorithm>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <ctime>
#include <vector>
#include "NativeBridgeMetalFrontend.h"
extern "C" size_t native_bridge_metal_telemetry(char *, size_t);
static double nowMs() {return clock_gettime_nsec_np(CLOCK_UPTIME_RAW) / 1e6;}
static void missing(const char *c, const char *s) {std::fprintf(stderr, "missing %s %s\n", c, s);}
int main(int argc, char **argv) {
if (argc < 4) {std::fprintf(stderr, "usage: loader cache metallib [limit]\n"); return 2;}
@autoreleasepool {
const unsigned limit = argc > 4 ? (unsigned)std::atoi(argv[4]) : 0;
const bool draw = argc > 5 && std::atoi(argv[5]) == 1;
double t0 = nowMs();
id<MTLDevice> device = native_bridge_metal_open(argv[1], argv[2], 0, missing);
if (!device) {std::printf("open failed\n"); return 1;}
std::printf("open %.1f ms\n", nowMs() - t0);
// draw=2: the plugin's on-device benchmark (native_bridge_metal_benchmark) with the same pair count, for the host side of the comparison.
if (argc > 5 && std::atoi(argv[5]) == 2) {
static char rows[512];
native_bridge_metal_benchmark(device, argv[3], limit ? limit : 8, rows, sizeof rows);
std::printf("%s", rows);
return 0;
}
NSError *error = nil;
t0 = nowMs();
id<MTLLibrary> qc = [device newLibraryWithURL:[NSURL fileURLWithPath:@(argv[3])] error:&error];
if (!qc) {std::printf("library failed: %s\n", error.localizedDescription.UTF8String); return 1;}
NSArray<NSString *> *names = qc.functionNames;
std::printf("library %.1f ms, %lu functions\n", nowMs() - t0, (unsigned long)names.count);
std::vector<double> function, pipeline, drawn;
id<MTLCommandQueue> queue = [device newCommandQueue];
MTLTextureDescriptor *td = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatBGRA8Unorm width:64 height:64 mipmapped:NO];
td.usage = MTLTextureUsageRenderTarget;
id<MTLTexture> target = [device newTextureWithDescriptor:td];
id<MTLBuffer> zeros = [device newBufferWithLength:65536 options:MTLResourceStorageModeShared];
unsigned made = 0, refused = 0, n = 0;
for (NSString *name in names) {
if (limit && n >= limit) break;
double f0 = nowMs();
id<MTLFunction> f = [qc newFunctionWithName:name];
double f1 = nowMs();
function.push_back(f1 - f0);
if (f.functionType == MTLFunctionTypeKernel) {
++n;
double p0 = nowMs();
id state = [device newComputePipelineStateWithFunction:f error:&error];
double p1 = nowMs();
pipeline.push_back(p1 - p0);
std::printf("K %-60s fn=%7.1f pipe=%8.1f %s\n", name.UTF8String, f1 - f0, p1 - p0, state ? "ok" : error.localizedDescription.UTF8String);
state ? ++made : ++refused;
} else if (f.functionType == MTLFunctionTypeVertex) {
NSString *fragmentName = [name stringByReplacingOccurrencesOfString:@"_vert_" withString:@"_frag_"];
if ([fragmentName isEqualToString:name] || ![names containsObject:fragmentName]) continue;
++n;
double g0 = nowMs();
id<MTLFunction> fragment = [qc newFunctionWithName:fragmentName];
double g1 = nowMs();
MTLRenderPipelineDescriptor *d = [MTLRenderPipelineDescriptor new];
d.vertexFunction = f;
d.fragmentFunction = fragment;
d.colorAttachments[0].pixelFormat = MTLPixelFormatBGRA8Unorm;
double p0 = nowMs();
id state = [device newRenderPipelineStateWithDescriptor:d error:&error];
double p1 = nowMs();
pipeline.push_back(p1 - p0);
double took = 0;
if (state && draw) {
double d0 = nowMs();
MTLRenderPassDescriptor *pass = [MTLRenderPassDescriptor renderPassDescriptor];
pass.colorAttachments[0].texture = target;
pass.colorAttachments[0].loadAction = MTLLoadActionClear;
pass.colorAttachments[0].storeAction = MTLStoreActionStore;
id<MTLCommandBuffer> commands = [queue commandBuffer];
id<MTLRenderCommandEncoder> encoder = [commands renderCommandEncoderWithDescriptor:pass];
[encoder setRenderPipelineState:state];
for (NSUInteger i = 0; i < 8; ++i) {
[encoder setVertexBuffer:zeros offset:0 atIndex:i];
[encoder setFragmentBuffer:zeros offset:0 atIndex:i];
}
[encoder drawPrimitives:MTLPrimitiveTypeTriangle vertexStart:0 vertexCount:3];
[encoder endEncoding];
[commands commit];
[commands waitUntilCompleted];
took = nowMs() - d0;
drawn.push_back(took);
}
std::printf("R %-60s fn=%7.1f+%7.1f pipe=%8.1f draw=%8.1f %s\n", name.UTF8String, f1 - f0, g1 - g0, p1 - p0, took,
state ? "ok" : error.localizedDescription.UTF8String);
state ? ++made : ++refused;
}
std::fflush(stdout);
}
std::sort(pipeline.begin(), pipeline.end());
double total = 0;
for (double v : pipeline) total += v;
double ftotal = 0;
for (double v : function) ftotal += v;
if (!pipeline.empty())
std::printf("SUMMARY pipelines=%zu made=%u refused=%u total=%.0f ms median=%.1f ms p90=%.1f ms max=%.1f ms | functions=%zu total=%.0f ms\n",
pipeline.size(), made, refused, total, pipeline[pipeline.size() / 2], pipeline[pipeline.size() * 9 / 10], pipeline.back(),
function.size(), ftotal);
if (!drawn.empty()) {
std::sort(drawn.begin(), drawn.end());
double dt = 0;
for (double v : drawn) dt += v;
static char table[8192];
native_bridge_metal_telemetry(table, sizeof table);
for (char *line = std::strtok(table, "\n"); line; line = std::strtok(nullptr, "\n"))
if (line[0] == 'V' || line[0] == 'W' || line[0] == 'S') std::printf("TELEMETRY %s\n", line);
std::printf("DRAWS n=%zu total=%.0f ms median=%.1f ms max=%.1f ms\n", drawn.size(), dt, drawn[drawn.size() / 2], drawn.back());
}
}
return 0;
}
drivers/NativeMetal/bridge/priming_cache_host.cpp
+40
View File
@@ -0,0 +1,40 @@
/* SPDX-License-Identifier: MIT */
/* Device open/close only. Actual Turnip compiler; mandatory software peer.
* No authored command, shader execution, timing or hardware claim. */
#include "NativeMetalObjects.h"
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <dlfcn.h>
#include <string>
#include <unistd.h>
#include <TargetConditionals.h>
#define REQUIRE(x) do {if(!(x)){std::fprintf(stderr,"FAIL priming line %d: %s; %s\n",__LINE__,#x,nmb_last_error());return 1;}} while(0)
int main(int argc,char **argv) {
if(argc!=4)return 2;
const bool produce=!std::strcmp(argv[3],"produce"),consume=!std::strcmp(argv[3],"consume"),uuid=!std::strcmp(argv[3],"uuid-miss"),cold=!std::strcmp(argv[3],"cold-miss");
REQUIRE(produce||consume||uuid||cold);
#if TARGET_OS_IPHONE
REQUIRE(getpagesize()==4096 && sysconf(_SC_PAGESIZE)==4096 && !getenv("HOSTSIM_PAGE_SIZE"));
#endif
REQUIRE(!setenv("NMB_TEST_REAL_VULKAN_LOADER",argv[1],1) && !setenv("MESA_SHADER_CACHE_DISABLE","true",1) && !setenv("NMB_TEST_FORCE_PRIME_STRICT",produce?"0":"1",1));
const char *path=getenv("NMB_TEST_PRIMING_CACHE_LOADER");REQUIRE(path);
void *loader=dlopen(path,RTLD_NOW|RTLD_LOCAL);REQUIRE(loader);
auto control=reinterpret_cast<void(*)(bool,bool)>(dlsym(loader,"nmb_test_graphics_cache_control"));
auto graphics=reinterpret_cast<void(*)(uint64_t *)>(dlsym(loader,"nmb_test_graphics_cache_counts"));
auto compute=reinterpret_cast<void(*)(uint64_t *)>(dlsym(loader,"nmb_test_prime_compute_counts"));
auto live=reinterpret_cast<unsigned(*)()>(dlsym(RTLD_DEFAULT,"host_gpu_live_objects"));
auto submitted=reinterpret_cast<unsigned(*)()>(dlsym(RTLD_DEFAULT,"host_gpu_submissions"));
REQUIRE(control&&graphics&&compute&&live&&submitted);control(false,uuid);
NmbDevice *device=nmb_device_open(path,argv[2]);uint64_t c[5]={},g[10]={};compute(c);graphics(g);
if(cold||uuid){REQUIRE(!device && c[2]>0);if(uuid)REQUIRE(g[5]>0);}
else {
REQUIRE(device && c[0] && c[1]==c[0] && !c[2] && c[3]==c[0] && g[0] && g[1]==g[0] && !g[2] && g[3]==g[0]);
if(consume)REQUIRE(c[4]==c[0] && g[4]==g[0]);
nmb_device_release(device);
}
REQUIRE(live()==0);
std::printf("PASS: device priming %s pid=%d compute=%llu/%llu/%llu/%llu/%llu graphics=%llu/%llu/%llu/%llu/%llu uuid_mutations=%llu submissions=%u live=0; software peer, no shader execution\n",argv[3],getpid(),
(unsigned long long)c[0],(unsigned long long)c[1],(unsigned long long)c[2],(unsigned long long)c[3],(unsigned long long)c[4],
(unsigned long long)g[0],(unsigned long long)g[1],(unsigned long long)g[2],(unsigned long long)g[3],(unsigned long long)g[4],(unsigned long long)g[5],submitted());
}
drivers/NativeMetal/bridge/priming_cache_ios.cpp
+62
View File
@@ -0,0 +1,62 @@
/* SPDX-License-Identifier: MIT */
/* Owned offline QEMU harness for the real iOS device-priming cache. Shared
* helpers retain exact RAM-volume identity, readonly view and bounded capture.
* The optional first-consumer marker is part of the hashed initial test image. */
#define main unused_ui_cold_suite_main
#include "quartzcore_cache_ios.cpp"
#undef main
int main(int argc,char **argv) {
(void)argv;prefix="/private/var/tmp/priming";workerLog="/private/var/tmp/priming-worker.log";
bool first=false;
if(FILE *file=fopen("/private/var/tmp/priming-first-consumer","rb")) {
char marker[32]={};const auto n=fread(marker,1,sizeof marker,file);fclose(file);
if(n!=strlen("priming-first-v1\n") || memcmp(marker,"priming-first-v1\n",n))stop(96);
first=true;
} else if(errno!=ENOENT)stop(97);
result[0]=first?0x5052494d494f5332ull:0x5052494d494f5331ull;
result[2]=getpid();result[3]=getpagesize();result[4]=sysconf(_SC_PAGESIZE);
if(argc!=1 || getpid()!=1 || result[3]!=4096 || result[4]!=4096)stop(1);
mountCompilerView();
if(setenv("MESA_SHADER_CACHE_DISABLE","true",1) || unsetenv("DYLD_INSERT_LIBRARIES") || unsetenv("HOSTSIM_PAGE_SIZE") ||
setenv("NMB_TEST_PRIMING_CACHE_LOADER","/System/Library/Extensions/OfflineTurnipCompiler/libOfflinePrimingCacheControl.dylib",1))stop(2);
const std::string cache=std::string(prefix)+".cache",empty=std::string(prefix)+".empty",copy=std::string(prefix)+".archive";
std::array<unsigned char,128> initial={};
if(first){summarize();memcpy(initial.data(),snapshot,initial.size());}
else {struct stat old={};if(!lstat(copy.c_str(),&old) || errno!=ENOENT)stop(3);}
if(unlink(workerLog) && errno!=ENOENT)stop(5);
posix_spawn_file_actions_t actions;
if(posix_spawn_file_actions_init(&actions) || posix_spawn_file_actions_addopen(&actions,STDOUT_FILENO,workerLog,O_WRONLY|O_CREAT|O_APPEND,0600) ||
posix_spawn_file_actions_adddup2(&actions,STDOUT_FILENO,STDERR_FILENO))stop(4);
const char *coldModes[]={"cold-miss","produce","consume","uuid-miss"},*firstModes[]={"consume","uuid-miss","cold-miss"};
const unsigned cases=first?3:4;
for(unsigned i=0;i<cases;++i) {
const char *mode=first?firstModes[i]:coldModes[i];const auto &directory=first && i==2?empty:cache;
char *arguments[]={const_cast<char *>("compiler-worker"),const_cast<char *>(driverPath),const_cast<char *>(directory.c_str()),const_cast<char *>(mode),nullptr};
pid_t child=0;int error=posix_spawn(&child,"/private/var/tmp/compiler-worker",&actions,nullptr,arguments,environ);
result[12+3*i]=child;result[13+3*i]=error;if(error || child<=1)stop(10+i);
int status=0;while(waitpid(child,&status,0)<0)if(errno!=EINTR)stop(20+i);
result[14+3*i]=status;if(!WIFEXITED(status) || WEXITSTATUS(status)!=0)stop(30+i);result[5]=i+1;
}
posix_spawn_file_actions_destroy(&actions);
// A diagnostic duplicate of the sole actual driver record, not a Metal
// binary archive. The full directory is independently hashed by summarize.
if(!first) {
std::error_code ec;std::filesystem::path record;unsigned files=0;
std::filesystem::recursive_directory_iterator iterator(cache,ec),end;if(ec)stop(98);
for(;iterator!=end;iterator.increment(ec)) {
if(ec)stop(99);const auto status=iterator->symlink_status(ec);
if(ec || std::filesystem::is_symlink(status))stop(100);
if(std::filesystem::is_directory(status))continue;
if(!std::filesystem::is_regular_file(status))stop(101);
record=iterator->path();++files;
}
if(ec || files!=1 || !std::filesystem::copy_file(record,copy,std::filesystem::copy_options::none,ec) || ec)stop(102);
}
summarize();
if(first) {
if(memcmp(initial.data(),snapshot,initial.size()))stop(103);
memcpy(snapshot+256,initial.data(),initial.size());result[6]=384;result[45]=2;result[46]=1;
}
stop(0);
}
drivers/NativeMetal/bridge/priming_cache_loader.cpp
+74
View File
@@ -0,0 +1,74 @@
/* SPDX-License-Identifier: MIT */
/* Offline only. Preserve the existing graphics/UUID counters and forward real
* calls, additionally forbid/observe compute and graphics during device open. */
#define vkGetInstanceProcAddr baselineGetInstanceProcAddr
#define vkGetDeviceProcAddr baselineGetDeviceProcAddr
#include "graphics_cache_loader.cpp"
#undef vkGetInstanceProcAddr
#undef vkGetDeviceProcAddr
static std::atomic<uint64_t> computeCounts[5]; // calls, successes, required, valid, app hits
static bool forceCompilationForbidden() {
const char *value=std::getenv("NMB_TEST_FORCE_PRIME_STRICT");return value && !std::strcmp(value,"1");
}
static bool hasFlags2(const void *next) {
for(auto p=static_cast<const VkBaseInStructure *>(next);p;p=p->pNext)
if(p->sType==VK_STRUCTURE_TYPE_PIPELINE_CREATE_FLAGS_2_CREATE_INFO_KHR)return true;
return false;
}
static VKAPI_ATTR VkResult VKAPI_CALL primeGraphics(VkDevice device,VkPipelineCache cache,uint32_t count,
const VkGraphicsPipelineCreateInfo *infos,const VkAllocationCallbacks *allocator,VkPipeline *pipelines) {
if(!count || count>64)return VK_ERROR_INITIALIZATION_FAILED;
std::vector<VkGraphicsPipelineCreateInfo> copies(infos,infos+count);
if(forceCompilationForbidden())for(auto &info:copies) {
// Explicit refusal rather than silently missing flags promoted to pNext.
if(hasFlags2(info.pNext))return VK_ERROR_FEATURE_NOT_PRESENT;
info.flags|=VK_PIPELINE_CREATE_FAIL_ON_PIPELINE_COMPILE_REQUIRED_BIT;
}
return createGraphics(device,cache,count,copies.data(),allocator,pipelines);
}
static VKAPI_ATTR VkResult VKAPI_CALL primeCompute(VkDevice device,VkPipelineCache cache,uint32_t count,
const VkComputePipelineCreateInfo *infos,const VkAllocationCallbacks *allocator,VkPipeline *pipelines) {
if(!count || count>64)return VK_ERROR_INITIALIZATION_FAILED;
std::vector<VkComputePipelineCreateInfo> copies(infos,infos+count);
std::vector<VkPipelineCreationFeedback> feedback(count);
std::vector<VkPipelineCreationFeedbackCreateInfo> additions(count);
std::vector<const VkPipelineCreationFeedback *> observed(count);
for(uint32_t i=0;i<count;++i) {
if(forceCompilationForbidden()) {
if(hasFlags2(copies[i].pNext))return VK_ERROR_FEATURE_NOT_PRESENT;
copies[i].flags|=VK_PIPELINE_CREATE_FAIL_ON_PIPELINE_COMPILE_REQUIRED_BIT;
}
for(auto p=static_cast<const VkBaseInStructure *>(copies[i].pNext);p;p=p->pNext)
if(p->sType==VK_STRUCTURE_TYPE_PIPELINE_CREATION_FEEDBACK_CREATE_INFO)
observed[i]=reinterpret_cast<const VkPipelineCreationFeedbackCreateInfo *>(p)->pPipelineCreationFeedback;
if(!observed[i]) {
auto &a=additions[i];a.sType=VK_STRUCTURE_TYPE_PIPELINE_CREATION_FEEDBACK_CREATE_INFO;
a.pNext=copies[i].pNext;a.pPipelineCreationFeedback=&feedback[i];copies[i].pNext=&a;observed[i]=&feedback[i];
}
}
auto real=reinterpret_cast<PFN_vkCreateComputePipelines>(realDevice.load()(device,"vkCreateComputePipelines"));
computeCounts[0]+=count;const auto result=real(device,cache,count,copies.data(),allocator,pipelines);
if(result==VK_PIPELINE_COMPILE_REQUIRED)computeCounts[2]++;
for(uint32_t i=0;i<count;++i)if(pipelines[i]) {
computeCounts[1]++;
if(observed[i] && (observed[i]->flags&VK_PIPELINE_CREATION_FEEDBACK_VALID_BIT)) {
computeCounts[3]++;
if(observed[i]->flags&VK_PIPELINE_CREATION_FEEDBACK_APPLICATION_PIPELINE_CACHE_HIT_BIT)computeCounts[4]++;
}
}
return result;
}
extern "C" VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL vkGetDeviceProcAddr(VkDevice device,const char *name) {
if(!std::strcmp(name,"vkCreateGraphicsPipelines"))return reinterpret_cast<PFN_vkVoidFunction>(primeGraphics);
if(!std::strcmp(name,"vkCreateComputePipelines"))return reinterpret_cast<PFN_vkVoidFunction>(primeCompute);
return baselineGetDeviceProcAddr(device,name);
}
extern "C" VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL vkGetInstanceProcAddr(VkInstance instance,const char *name) {
const auto result=baselineGetInstanceProcAddr(instance,name);
if(!std::strcmp(name,"vkGetDeviceProcAddr"))return reinterpret_cast<PFN_vkVoidFunction>(vkGetDeviceProcAddr);
if(!std::strcmp(name,"vkCreateGraphicsPipelines"))return reinterpret_cast<PFN_vkVoidFunction>(primeGraphics);
if(!std::strcmp(name,"vkCreateComputePipelines"))return reinterpret_cast<PFN_vkVoidFunction>(primeCompute);
return result;
}
extern "C" void nmb_test_prime_compute_counts(uint64_t out[5]) {for(unsigned i=0;i<5;++i)out[i]=computeCounts[i].load();}
drivers/NativeMetal/bridge/quartzcore_blur_host.mm
+202
View File
@@ -0,0 +1,202 @@
// SPDX-License-Identifier: MIT
// Execute unchanged target blur functions on a real host GPU. The reference
// is an authored seven-tap binomial convolution, not a deployment renderer.
#import <Foundation/Foundation.h>
#import <Metal/Metal.h>
#include <algorithm>
#include <cmath>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <vector>
#include "../NativeBridgeMetalFrontend.h"
#include "NativeMetalObjects.h"
constexpr unsigned width=32,height=24;
struct Vertex {float position[4],uv[2],unusedColor[4];};
static const Vertex vertices[]={
{{-1,-1,0,1},{0,1},{}},{{1,-1,0,1},{1,1},{}},{{-1,1,0,1},{0,0},{}},
{{-1,1,0,1},{0,0},{}},{{1,-1,0,1},{1,1},{}},{{1,1,0,1},{1,0},{}}};
struct alignas(16) UniformF {float offset[8][2],weight[8],divide;unsigned char padding[12];};
struct alignas(8) UniformH {float offset[8][2];_Float16 weight[8],divide;unsigned char padding[6];};
static_assert(sizeof(UniformF)==112 && sizeof(UniformH)==88);
static MTLRenderPipelineDescriptor *blurDescriptor(id<MTLLibrary> library,bool half) {
NSString *suffix=half?@"lph":@"lpf";auto d=[MTLRenderPipelineDescriptor new];
d.vertexFunction=[library newFunctionWithName:[@"downsample_blur_vert_" stringByAppendingString:suffix]];
d.fragmentFunction=[library newFunctionWithName:[@"narrow_blur_7_frag_" stringByAppendingString:suffix]];
d.colorAttachments[0].pixelFormat=MTLPixelFormatRGBA16Float;
auto vd=[MTLVertexDescriptor vertexDescriptor];
const unsigned indices[]={0,1,3},offsets[]={0,16,24};
const MTLVertexFormat formats[]={MTLVertexFormatFloat4,MTLVertexFormatFloat2,MTLVertexFormatFloat4};
for(unsigned i=0;i<3;++i){vd.attributes[indices[i]].format=formats[i];vd.attributes[indices[i]].offset=offsets[i];vd.attributes[indices[i]].bufferIndex=0;}
vd.layouts[0].stride=sizeof(Vertex);d.vertexDescriptor=vd;return d;
}
static MTLRenderPipelineDescriptor *namedDescriptor(id<MTLLibrary> library,NSString *vertex,NSString *fragment) {
auto d=[MTLRenderPipelineDescriptor new];d.vertexFunction=[library newFunctionWithName:vertex];d.fragmentFunction=[library newFunctionWithName:fragment];
d.colorAttachments[0].pixelFormat=MTLPixelFormatBGRA8Unorm;
auto vd=[MTLVertexDescriptor vertexDescriptor];
const MTLVertexFormat formats[]={MTLVertexFormatFloat4,MTLVertexFormatFloat2,MTLVertexFormatFloat2,MTLVertexFormatFloat4,MTLVertexFormatFloat};
const unsigned offsets[]={0,16,24,32,48};
for(unsigned i=0;i<5;++i){vd.attributes[i].format=formats[i];vd.attributes[i].bufferIndex=0;vd.attributes[i].offset=offsets[i];}
vd.layouts[0].stride=64;d.vertexDescriptor=vd;
if([vertex hasPrefix:@"path_blit_"]){d.colorAttachments[1].pixelFormat=MTLPixelFormatBGRA8Unorm;d.colorAttachments[2].pixelFormat=MTLPixelFormatR32Float;}
return d;
}
static bool cacheState(const char *phase) {
NmbShaderCacheStats s={};uint64_t work[10]={};
if(!native_bridge_metal_shader_cache_stats(&s) || !native_bridge_metal_work_stats(work))return false;
std::printf("cache %s artifact_hit=%llu miss=%llu write=%llu driver_load=%llu hit=%llu miss=%llu save=%llu same=%llu vk_makings=%llu\n",phase,
(unsigned long long)s.hits,(unsigned long long)s.misses,(unsigned long long)s.writes,
(unsigned long long)s.driver_loads,(unsigned long long)s.driver_hits,(unsigned long long)s.driver_misses,
(unsigned long long)s.driver_saves,(unsigned long long)s.driver_unchanged,(unsigned long long)work[0]);
return !s.rejected && !s.read_failures && !s.write_failures && !s.driver_truncated && !s.driver_mismatched && !work[4];
}
static bool inventory(id<MTLDevice> device,id<MTLLibrary> library) {
unsigned vertex=0,fragment=0,kernels=0,ordinary=0,tile=0,pairs=0,bare=0,attributes=0,attachments=0;
for(NSString *name in library.functionNames) {
id<MTLFunction> f=[library newFunctionWithName:name];if(!f)return false;
NSError *error=nil;
if(f.functionType==MTLFunctionTypeKernel) {
++kernels;
if([device newComputePipelineStateWithFunction:f error:&error])++ordinary;
else if([error.localizedDescription containsString:@"NeedsInput(TileAttachments)"])++tile;
else {std::printf("FAIL unclassified compute %s: %s\n",name.UTF8String,error.localizedDescription.UTF8String);return false;}
} else if(f.functionType==MTLFunctionTypeFragment)++fragment;
else if(f.functionType==MTLFunctionTypeVertex) {
++vertex;NSString *other=[name stringByReplacingOccurrencesOfString:@"_vert_" withString:@"_frag_"];
if([other isEqualToString:name] || ![library.functionNames containsObject:other])continue;
++pairs;auto d=[MTLRenderPipelineDescriptor new];d.vertexFunction=f;d.fragmentFunction=[library newFunctionWithName:other];
d.colorAttachments[0].pixelFormat=MTLPixelFormatBGRA8Unorm;
if([device newRenderPipelineStateWithDescriptor:d error:&error])++bare;
else if([error.localizedDescription containsString:@"InvalidDescriptor(VertexAttribute"])++attributes;
else if([error.localizedDescription containsString:@"InvalidDescriptor(ColorAttachment"])++attachments;
else {std::printf("FAIL unclassified render %s: %s\n",name.UTF8String,error.localizedDescription.UTF8String);return false;}
} else return false;
}
std::printf("inventory functions=167 vertex=%u fragment=%u kernels=%u ordinary=%u tile_requires=%u pairs=%u bare_ok=%u missing_vertex=%u missing_attachment=%u\n",
vertex,fragment,kernels,ordinary,tile,pairs,bare,attributes,attachments);
return vertex==41 && fragment==111 && kernels==15 && ordinary==4 && tile==11 && pairs==22 && bare==14 && attributes==6 && attachments==2;
}
static std::vector<_Float16> convolution(const std::vector<_Float16> &input,bool vertical) {
// Independent integer stencil: coefficients sum to 64, clamp at the edge.
constexpr unsigned weight[]={1,6,15,20,15,6,1};std::vector<_Float16> result(input.size());
for(unsigned y=0;y<height;++y)for(unsigned x=0;x<width;++x)for(unsigned c=0;c<4;++c) {
float sum=0;
for(int tap=-3;tap<=3;++tap) {
const int sx=std::clamp((int)x+(vertical?0:tap),0,(int)width-1);
const int sy=std::clamp((int)y+(vertical?tap:0),0,(int)height-1);
sum+=(float)input[(sy*width+sx)*4+c]*weight[tap+3];
}
result[(y*width+x)*4+c]=(_Float16)(sum/64);
}
return result;
}
static bool encode(id<MTLCommandBuffer> command,id<MTLRenderPipelineState> state,id<MTLTexture> source,
id<MTLTexture> destination,id<MTLSamplerState> sampler,bool half,bool vertical) {
auto pass=[MTLRenderPassDescriptor renderPassDescriptor];pass.colorAttachments[0].texture=destination;
pass.colorAttachments[0].loadAction=MTLLoadActionDontCare;pass.colorAttachments[0].storeAction=MTLStoreActionStore;
id<MTLRenderCommandEncoder> encoder=[command renderCommandEncoderWithDescriptor:pass];if(!encoder)return false;
[encoder setRenderPipelineState:state];[encoder setVertexBytes:vertices length:sizeof vertices atIndex:0];
const float identity[]={1,0,0,0,0,1,0,0,0,0,1,0,0,0,0,1},texmat[]={1,1,0,0};
[encoder setVertexBytes:identity length:sizeof identity atIndex:2];[encoder setVertexBytes:texmat length:sizeof texmat atIndex:3];
[encoder setFragmentTexture:source atIndex:3];[encoder setFragmentSamplerState:sampler atIndex:0];
// Two bilinear pairs exactly represent the seven authored stencil taps.
UniformF f={};UniformH h={};
f.offset[0][vertical?1:0]=0.6f/(vertical?height:width);
f.offset[1][vertical?1:0]=(15.0f/7)/(vertical?height:width);
f.weight[0]=25.0f/64;f.weight[1]=7.0f/64;
if(half) {
memcpy(h.offset,f.offset,sizeof h.offset);h.weight[0]=(_Float16)f.weight[0];h.weight[1]=(_Float16)f.weight[1];
const _Float16 scale=1;[encoder setFragmentBytes:&h length:sizeof h atIndex:1];[encoder setFragmentBytes:&scale length:sizeof scale atIndex:6];
} else {
const float scale=1;[encoder setFragmentBytes:&f length:sizeof f atIndex:1];[encoder setFragmentBytes:&scale length:sizeof scale atIndex:6];
}
[encoder drawPrimitives:MTLPrimitiveTypeTriangle vertexStart:0 vertexCount:6];[encoder endEncoding];return true;
}
static bool runBlur(id<MTLDevice> device,id<MTLRenderPipelineState> state,bool half) {
std::vector<_Float16> data(width*height*4);
for(unsigned y=0;y<height;++y)for(unsigned x=0;x<width;++x) {
auto p=data.data()+(y*width+x)*4;p[0]=(_Float16)((x%7)/7.0f);p[1]=(_Float16)((y%5)/5.0f);
p[2]=(_Float16)(x==width/2 && y==height/2);p[3]=1;
}
const auto expected=convolution(convolution(data,false),true);
auto td=[MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatRGBA16Float width:width height:height mipmapped:NO];
td.storageMode=MTLStorageModeShared;td.usage=MTLTextureUsageShaderRead|MTLTextureUsageRenderTarget;
id<MTLTexture> input=[device newTextureWithDescriptor:td],middle=[device newTextureWithDescriptor:td],output=[device newTextureWithDescriptor:td];
auto sd=[MTLSamplerDescriptor new];sd.minFilter=MTLSamplerMinMagFilterLinear;sd.magFilter=MTLSamplerMinMagFilterLinear;
sd.sAddressMode=MTLSamplerAddressModeClampToEdge;sd.tAddressMode=MTLSamplerAddressModeClampToEdge;
id<MTLSamplerState> sampler=[device newSamplerStateWithDescriptor:sd];id<MTLCommandQueue> queue=[device newCommandQueue];
if(!input || !middle || !output || !sampler || !queue)return false;
[input replaceRegion:MTLRegionMake2D(0,0,width,height) mipmapLevel:0 withBytes:data.data() bytesPerRow:width*8];
float worst=0;
for(unsigned round=0;round<6;++round){@autoreleasepool {
NmbShaderCacheStats before={},after={};uint64_t workBefore[10]={},workAfter[10]={};
if(!native_bridge_metal_shader_cache_stats(&before) || !native_bridge_metal_work_stats(workBefore))return false;
id<MTLCommandBuffer> command=[queue commandBuffer];
if(!encode(command,state,input,middle,sampler,half,false) || !encode(command,state,middle,output,sampler,half,true))return false;
[command commit];[command waitUntilCompleted];
if(command.status!=MTLCommandBufferStatusCompleted){std::printf("FAIL blur command: %s\n",command.error.localizedDescription.UTF8String);return false;}
std::vector<_Float16> got(data.size());[output getBytes:got.data() bytesPerRow:width*8 fromRegion:MTLRegionMake2D(0,0,width,height) mipmapLevel:0];
for(size_t i=0;i<got.size();++i){const float error=std::fabs((float)got[i]-(float)expected[i]);
if(!std::isfinite((float)got[i]) || error>0.003f){std::printf("FAIL blur %s round=%u element=%zu got=%g expected=%g\n",half?"half":"float",round,i,(double)got[i],(double)expected[i]);return false;}
worst=std::max(worst,error);
}
if(!native_bridge_metal_shader_cache_stats(&after) || !native_bridge_metal_work_stats(workAfter))return false;
std::printf("round %s %u vk_makings=%llu artifact_hit=%llu miss=%llu driver_load=%llu hit=%llu miss=%llu\n",half?"half":"float",round,
(unsigned long long)(workAfter[0]-workBefore[0]),(unsigned long long)(after.hits-before.hits),(unsigned long long)(after.misses-before.misses),
(unsigned long long)(after.driver_loads-before.driver_loads),(unsigned long long)(after.driver_hits-before.driver_hits),(unsigned long long)(after.driver_misses-before.driver_misses));
if(round && (workAfter[0]!=workBefore[0] || after.misses!=before.misses || after.driver_misses!=before.driver_misses)) {
std::puts("FAIL repeated identical blur performs new pipeline/translation work");return false;
}
}}
std::printf("blur %s rounds=6 passes=12 texels=%u max_error=%g GPU=pass\n",half?"half":"float",width*height,worst);return true;
}
int main(int argc,char **argv) {
if(argc!=6 || (strcmp(argv[5],"produce") && strcmp(argv[5],"consume") && strcmp(argv[5],"inventory")))return 2;
if(setenv("METAL2VULKAN_LOG_PIPELINES","1",1))return 2;
const bool produce=!strcmp(argv[5],"produce");
@autoreleasepool {
id<MTLDevice> device=native_bridge_metal_open(argv[1],argv[2],0x6e6d62,nullptr);if(!device)return 2;
NSError *error=nil;id<MTLLibrary> library=[device newLibraryWithFile:@(argv[3]) error:&error];if(!library)return 2;
if(library.functionNames.count!=167)return 1;
if(!strcmp(argv[5],"inventory"))return inventory(device,library)?0:1;
auto ad=[MTLBinaryArchiveDescriptor new];NSURL *url=[NSURL fileURLWithPath:@(argv[4])];if(!produce)ad.url=url;
id<MTLBinaryArchive> archive=[device newBinaryArchiveWithDescriptor:ad error:&error];if(!archive)return 1;
if(!cacheState("opened"))return 1;
if(produce) {
auto empty=blurDescriptor(library,false);empty.binaryArchives=@[archive];error=nil;
if([device newRenderPipelineStateWithDescriptor:empty options:MTLPipelineOptionFailOnBinaryArchiveMiss reflection:nil error:&error] || !error)return 1;
}
unsigned pairs=0;
for(NSString *name in library.functionNames) {
id<MTLFunction> function=[library newFunctionWithName:name];if(function.functionType!=MTLFunctionTypeVertex)continue;
NSString *fragment=[name stringByReplacingOccurrencesOfString:@"_vert_" withString:@"_frag_"];
if([fragment isEqualToString:name] || ![library.functionNames containsObject:fragment])continue;
auto d=namedDescriptor(library,name,fragment);d.binaryArchives=@[archive];
if(produce && ![archive addRenderPipelineFunctionsWithDescriptor:d error:&error]){
std::printf("FAIL archive add %s: %s\n",name.UTF8String,error.localizedDescription.UTF8String);return 1;}
error=nil;
if(![device newRenderPipelineStateWithDescriptor:d options:MTLPipelineOptionFailOnBinaryArchiveMiss reflection:nil error:&error] || error){
std::printf("FAIL strict pair %s: %s\n",name.UTF8String,error.localizedDescription.UTF8String);return 1;}
++pairs;
}
if(pairs!=22)return 1;
std::printf("valid descriptor pairs=%u strict archive=pass\n",pairs);
for(bool half:{false,true}) {
auto d=blurDescriptor(library,half);d.binaryArchives=@[archive];
if(produce && ![archive addRenderPipelineFunctionsWithDescriptor:d error:&error])return 1;
error=nil;id<MTLRenderPipelineState> state=[device newRenderPipelineStateWithDescriptor:d options:MTLPipelineOptionFailOnBinaryArchiveMiss reflection:nil error:&error];
if(!state){std::printf("FAIL blur pipeline: %s\n",error.localizedDescription.UTF8String);return 1;}
const auto format=d.colorAttachments[0].pixelFormat;d.colorAttachments[0].pixelFormat=MTLPixelFormatBGRA8Unorm;error=nil;
if([device newRenderPipelineStateWithDescriptor:d options:MTLPipelineOptionFailOnBinaryArchiveMiss reflection:nil error:&error] || !error)return 1;
d.colorAttachments[0].pixelFormat=format;
if(!runBlur(device,state,half))return 1;
}
if(produce && ![archive serializeToURL:url error:&error])return 1;
if(!cacheState("finished"))return 1;
uint64_t work[10]={};if(!native_bridge_metal_work_stats(work) || !work[0])return 1; // logging must be active, never a zero-counter shortcut
std::puts("PASS: unchanged target blur functions execute repeated separable Gaussian reference cases on the host GPU");
}
}
drivers/NativeMetal/bridge/quartzcore_cache.mm
+112
View File
@@ -0,0 +1,112 @@
// SPDX-License-Identifier: MIT
// Actual compiler/cache instrument. No draw is encoded. The runtime may initialize
// neutral resources over the mandatory software peer; it executes no shader.
#import <Metal/Metal.h>
#include <TargetConditionals.h>
#include <CommonCrypto/CommonDigest.h>
#include "NativeMetalObjects.h"
#include <algorithm>
#include <array>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <dlfcn.h>
#include <fstream>
#include <iterator>
#include <string>
#include <vector>
#include <unistd.h>
#define REQUIRE(x) do {if(!(x)){std::fprintf(stderr,"FAIL UI cache line %d: %s; %s\n",__LINE__,#x,nmb_last_error());return 1;}} while(0)
static NmbRenderPipelineDescriptor descriptor(NmbLibrary *library,const std::string &vertex,const std::string &fragment,bool blur) {
NmbRenderPipelineDescriptor d={};d.vertex=nmb_function_new(library,vertex.c_str());d.fragment=nmb_function_new(library,fragment.c_str());
d.colors[0].format=blur?MTLPixelFormatRGBA16Float:MTLPixelFormatBGRA8Unorm;
if(vertex.rfind("path_blit_",0)==0){d.colors[1].format=MTLPixelFormatBGRA8Unorm;d.colors[2].format=MTLPixelFormatR32Float;}
for(auto &c:d.colors)if(c.format){c.write_mask=MTLColorWriteMaskAll;c.source_rgb=MTLBlendFactorOne;c.source_alpha=MTLBlendFactorOne;}
d.samples=1;d.rasterization=1;d.vertex_inputs=1;
if(blur) {
d.attributes[0]={MTLVertexFormatFloat4,0,0};d.attributes[1]={MTLVertexFormatFloat2,16,0};d.attributes[3]={MTLVertexFormatFloat4,24,0};
d.layouts[0]={40,MTLVertexStepFunctionPerVertex,1};
} else {
const uint64_t formats[]={MTLVertexFormatFloat4,MTLVertexFormatFloat2,MTLVertexFormatFloat2,MTLVertexFormatFloat4,MTLVertexFormatFloat};
const uint64_t offsets[]={0,16,24,32,48};for(unsigned i=0;i<5;++i)d.attributes[i]={formats[i],offsets[i],0};
d.layouts[0]={64,MTLVertexStepFunctionPerVertex,1};
}
return d;
}
int main(int argc,char **argv) {
if(argc!=4)return 2;
const bool produce=!std::strcmp(argv[3],"produce"),consume=!std::strcmp(argv[3],"consume"),uuid=!std::strcmp(argv[3],"uuid-miss"),empty=!std::strcmp(argv[3],"empty-driver");
REQUIRE(produce||consume||uuid||empty);
#if TARGET_OS_IPHONE
REQUIRE(getpagesize()==4096 && sysconf(_SC_PAGESIZE)==4096 && !getenv("HOSTSIM_PAGE_SIZE"));
#endif
REQUIRE(!setenv("MESA_SHADER_CACHE_DISABLE","true",1) && !setenv("METAL2VULKAN_LOG_PIPELINES","1",1) && !setenv("NMB_TEST_REAL_VULKAN_LOADER",argv[1],1));
std::string path=argv[1];path=path.substr(0,path.find_last_of('/')+1)+"libOfflineGraphicsCacheControl.dylib";
if(const char *override=getenv("NMB_TEST_GRAPHICS_CACHE_LOADER"))path=override;
void *loader=dlopen(path.c_str(),RTLD_NOW|RTLD_LOCAL);REQUIRE(loader);
auto control=reinterpret_cast<void(*)(bool,bool)>(dlsym(loader,"nmb_test_graphics_cache_control"));
auto counters=reinterpret_cast<void(*)(uint64_t *)>(dlsym(loader,"nmb_test_graphics_cache_counts"));
auto submitted=reinterpret_cast<unsigned(*)()>(dlsym(RTLD_DEFAULT,"host_gpu_submissions"));REQUIRE(control&&counters&&submitted);
const unsigned submissions=submitted();control(false,false);
const std::string prefix=argv[2],archivePath=prefix+".archive",cachePath=prefix+(empty?".empty":".cache");
NmbDevice *device=nmb_device_open(path.c_str(),cachePath.c_str());REQUIRE(device);
uint64_t deviceCounts[10]={};counters(deviceCounts);control(!produce,uuid);
const char *libraryPath="/System/Library/Frameworks/QuartzCore.framework/default.metallib";
if(const char *override=getenv("NMB_TEST_QUARTZCORE_LIBRARY"))libraryPath=override;
std::ifstream input(libraryPath,std::ios::binary);REQUIRE(input);
std::vector<char> bytes(9670832);input.read(bytes.data(),bytes.size());REQUIRE(input.good() && input.peek()==EOF);
unsigned char digest[CC_SHA256_DIGEST_LENGTH];CC_SHA256(bytes.data(),(CC_LONG)bytes.size(),digest);
char digestHex[65]={};for(unsigned i=0;i<32;++i)std::snprintf(digestHex+2*i,3,"%02x",digest[i]);
REQUIRE(!std::strcmp(digestHex,"606f913adf520420f9b4bb666efb782a68c830f082f1e559e17b20d0d13adbc9"));
NmbLibrary *library=nmb_library_new(device,bytes.data(),bytes.size(),false);REQUIRE(library&&nmb_library_function_count(library)==167);
std::vector<std::string> names;for(unsigned i=0;i<167;++i){char name[256]={};REQUIRE(nmb_library_function_name(library,i,name,sizeof name)<sizeof name);names.emplace_back(name);}
std::vector<NmbRenderPipelineDescriptor> recipes;
for(const auto &name:names) {
const auto marker=name.find("_vert_");if(marker==std::string::npos)continue;
auto fragment=name;fragment.replace(marker,6,"_frag_");if(std::find(names.begin(),names.end(),fragment)==names.end())continue;
recipes.push_back(descriptor(library,name,fragment,false));
}
REQUIRE(recipes.size()==22);
for(const auto *suffix:{"lpf","lph"})recipes.push_back(descriptor(library,std::string("downsample_blur_vert_")+suffix,std::string("narrow_blur_7_frag_")+suffix,true));
for(const auto &d:recipes)REQUIRE(d.vertex&&d.fragment&&nmb_function_type(d.vertex)==1&&nmb_function_type(d.fragment)==2);
NmbBinaryArchive *archive=nmb_binary_archive_new(device,produce?nullptr:archivePath.c_str());REQUIRE(archive);
NmbPipelineOptions options={MTLPipelineOptionFailOnBinaryArchiveMiss,&archive,1};
uint64_t coldRequired=0;bool coldObject=false;
if(produce) {
// Exercise the real driver's compile-forbidden cold refusal, separately
// from strict binary-archive misses. Clear only the test policy afterwards.
control(true,false);auto cold=nmb_render_pipeline_new(device,&recipes.back());
coldObject=cold!=nullptr;if(cold)nmb_render_pipeline_release(cold);
uint64_t counts[10]={};counters(counts);coldRequired=counts[2];REQUIRE(coldRequired>0);control(false,false);
}
uint64_t before[10]={};counters(before);unsigned made=0;
for(const auto &d:recipes) {
if(produce)REQUIRE(nmb_binary_archive_add_render(archive,&d));
NmbRenderPipeline *state=nmb_render_pipeline_new_with_options(device,&d,&options);
if(uuid||empty){if(state)nmb_render_pipeline_release(state);break;}
REQUIRE(state);nmb_render_pipeline_release(state);++made;
}
uint64_t counts[10]={};counters(counts);
std::fprintf(stderr,"graphics results %s: calls=%llu success=%llu required=%llu valid=%llu hits=%llu cached=%llu/%llu/%llu\n",argv[3],
(unsigned long long)(counts[0]-before[0]),(unsigned long long)(counts[1]-before[1]),(unsigned long long)(counts[2]-before[2]),
(unsigned long long)(counts[3]-before[3]),(unsigned long long)(counts[4]-before[4]),(unsigned long long)(counts[7]-before[7]),
(unsigned long long)(counts[8]-before[8]),(unsigned long long)(counts[9]-before[9]));
if(uuid||empty) {REQUIRE(counts[2]>before[2]);if(uuid)REQUIRE(counts[5]>before[5]);}
else REQUIRE(made==24 && counts[1]>before[1]);
if(consume)REQUIRE(counts[2]==before[2] && counts[7]>before[7] && counts[8]-before[8]==counts[7]-before[7] && counts[9]-before[9]==counts[7]-before[7]);
if(produce)REQUIRE(nmb_binary_archive_serialize(archive,archivePath.c_str()));
NmbShaderCacheStats stats={};REQUIRE(nmb_device_shader_cache_stats(device,&stats));
REQUIRE(!stats.rejected&&!stats.read_failures&&!stats.write_failures&&!stats.driver_truncated&&!stats.driver_mismatched);
if(consume)REQUIRE(!stats.misses&&!stats.writes&&!stats.driver_misses&&stats.driver_loads);
const unsigned initializationSubmissions=submitted()-submissions;
std::printf("PASS: QuartzCore compiler %s pid=%d recipes=%u calls=%llu success=%llu required=%llu valid=%llu hits=%llu uuid_mutations=%llu devices=%llu artifacts=%llu/%llu/%llu driver=%llu/%llu/%llu/%llu cached_calls=%llu/%llu/%llu device_graphics=%llu cold_required=%llu cold_object=%u init_submissions=%u; no shader execution\n",argv[3],getpid(),made,
(unsigned long long)(counts[0]-before[0]),(unsigned long long)(counts[1]-before[1]),(unsigned long long)(counts[2]-before[2]),
(unsigned long long)(counts[3]-before[3]),(unsigned long long)(counts[4]-before[4]),(unsigned long long)(counts[5]-before[5]),(unsigned long long)counts[6],
(unsigned long long)stats.hits,(unsigned long long)stats.misses,(unsigned long long)stats.writes,
(unsigned long long)stats.driver_loads,(unsigned long long)stats.driver_hits,(unsigned long long)stats.driver_misses,(unsigned long long)stats.driver_saves,(unsigned long long)(counts[7]-before[7]),
(unsigned long long)(counts[8]-before[8]),(unsigned long long)(counts[9]-before[9]),(unsigned long long)deviceCounts[0],(unsigned long long)coldRequired,(unsigned)coldObject,initializationSubmissions);
nmb_binary_archive_release(archive);for(const auto &d:recipes){nmb_function_release(d.vertex);nmb_function_release(d.fragment);}
nmb_library_release(library);nmb_device_release(device);
return 0;
}
drivers/NativeMetal/bridge/quartzcore_cache_ios.cpp
+144
View File
@@ -0,0 +1,144 @@
/* SPDX-License-Identifier: MIT */
/* Owned QEMU-only PID 1 harness. Run the actual iOS UI compiler in four cold
* children, flush own mutable Data, then expose a bounded manifest checkpoint.
* Original dyld/AMFI, signatures, read-only OS and exact RAM-volume selection
* remain in force. The linked software peer executes no shader. */
#include <CoreFoundation/CoreFoundation.h>
#include <IOKit/IOKitLib.h>
#include <CommonCrypto/CommonDigest.h>
#include <spawn.h>
#include <sys/wait.h>
#include <sys/mount.h>
#include <sys/stat.h>
#include <unistd.h>
#include <fcntl.h>
#include <cerrno>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <algorithm>
#include <array>
#include <filesystem>
#include <string>
#include <vector>
extern char **environ;
static uint64_t result[48];
alignas(4096) static unsigned char snapshot[4096];
alignas(4096) static unsigned char workerOutput[4096];
static const char *workerLog="/private/var/tmp/quartzcore-compiler-worker.log";
static const char *driverPath="/System/Library/Extensions/OfflineTurnipCompiler/libvulkan_freedreno.dylib";
static const char *prefix="/private/var/tmp/quartzcore-ui";
[[noreturn]] static void stop(unsigned failure) {
result[1]=failure;
if(FILE *file=fopen(workerLog,"rb")) {
result[11]=fread(workerOutput,1,sizeof workerOutput-1,file);
if(fgetc(file)!=EOF || ferror(file))result[11]=sizeof workerOutput;
fclose(file);
}
__asm__ volatile("mov x13, %0\nmov x14, %1\nmov x15, %2\nbrk #0"::"r"(result),"r"(snapshot),"r"(workerOutput):"x13","x14","x15","memory");
_exit(99);
}
static void mountCompilerView() {
char uuid[37]={};FILE *file=fopen("/private/var/tmp/compiler-data-uuid","rb");
if(!file)stop(60);
const size_t count=fread(uuid,1,37,file);fclose(file);
if(count!=36 || uuid[36])stop(61);
CFStringRef identity=CFStringCreateWithCString(nullptr,uuid,kCFStringEncodingASCII);
CFMutableDictionaryRef match=IOServiceMatching("IOMedia");if(!identity || !match)stop(62);
const void *keys[]={CFSTR("UUID")},*values[]={identity};
CFDictionaryRef properties=CFDictionaryCreate(nullptr,keys,values,1,&kCFTypeDictionaryKeyCallBacks,&kCFTypeDictionaryValueCallBacks);
if(!properties)stop(63);
CFDictionarySetValue(match,CFSTR("IOPropertyMatch"),properties);CFRelease(properties);
io_iterator_t iterator=0;if(IOServiceGetMatchingServices(0,match,&iterator) || !iterator)stop(64);
char device[64]={};unsigned found=0;
while(io_service_t media=IOIteratorNext(iterator)) {
++found;
CFTypeRef observed=IORegistryEntryCreateCFProperty(media,CFSTR("UUID"),nullptr,0);
CFTypeRef name=IORegistryEntryCreateCFProperty(media,CFSTR("BSD Name"),nullptr,0);
if(!observed || !CFEqual(observed,identity) || !name || CFGetTypeID(name)!=CFStringGetTypeID())stop(65);
char bsd[56]={};if(!CFStringGetCString((CFStringRef)name,bsd,sizeof bsd,kCFStringEncodingASCII) || strncmp(bsd,"disk",4))stop(66);
for(const char *p=bsd;*p;++p)if(!((*p>='a'&&*p<='z')||(*p>='0'&&*p<='9')))stop(67);
bool backed=false;io_registry_entry_t node=media;
for(unsigned depth=0;node && depth<64;++depth) {
if(IOObjectConformsTo(node,"NativeRamBlock")){backed=true;break;}
io_registry_entry_t above=0;auto error=IORegistryEntryGetParentEntry(node,"IOService",&above);
if(node!=media)IOObjectRelease(node);node=error?0:above;
}
if(node && node!=media)IOObjectRelease(node);if(!backed)stop(68);
snprintf(device,sizeof device,"/dev/%s",bsd);CFRelease(observed);CFRelease(name);IOObjectRelease(media);
}
IOObjectRelease(iterator);CFRelease(identity);if(found!=1 || !device[0])stop(69);
const char *point="/System/Library/Extensions",*directory="/private/var/tmp/native-compiler-addons";
struct stat block={},source={},before={},after={},original={},code={};struct statfs fs={};
if(stat(device,&block) || !S_ISBLK(block.st_mode) || stat(point,&before) || stat(directory,&source) || source.st_dev!=block.st_rdev)stop(70);
if(mount("bindfs",point,MNT_RDONLY|MNT_UNION,(void *)directory))stop(71);
if(statfs(point,&fs) || (fs.f_flags&(MNT_RDONLY|MNT_UNION))!=(MNT_RDONLY|MNT_UNION) ||
strcmp(fs.f_mntfromname,directory) || strcmp(fs.f_fstypename,"bindfs") || stat(point,&after) ||
after.st_dev==before.st_dev || stat(driverPath,&code) || code.st_dev!=after.st_dev ||
stat("/System/Library/Extensions/AGXMetalA10.bundle/Info.plist",&original) || original.st_dev!=before.st_dev)stop(72);
result[38]=fs.f_flags;result[39]=(uint32_t)before.st_dev;result[40]=(uint32_t)after.st_dev;
result[41]=(uint32_t)original.st_dev;result[42]=(uint32_t)code.st_dev;result[43]=found;result[44]=1;
}
static std::array<unsigned char,32> digestFile(const std::filesystem::path &path,uint64_t *length) {
struct stat st={};if(lstat(path.c_str(),&st) || !S_ISREG(st.st_mode) || st.st_size<0 || st.st_size>16*1024*1024)stop(80);
FILE *file=fopen(path.c_str(),"rb");if(!file)stop(81);
CC_SHA256_CTX hash;CC_SHA256_Init(&hash);unsigned char buffer[16384];uint64_t total=0;
for(;;){const size_t n=fread(buffer,1,sizeof buffer,file);if(n){CC_SHA256_Update(&hash,buffer,(CC_LONG)n);total+=n;}if(n<sizeof buffer)break;}
const bool good=!ferror(file) && feof(file) && total==(uint64_t)st.st_size;fclose(file);if(!good)stop(82);
std::array<unsigned char,32> digest={};CC_SHA256_Final(digest.data(),&hash);*length=total;return digest;
}
static void summarize() {
const std::filesystem::path directory=std::string(prefix)+".cache",archive=std::string(prefix)+".archive";
std::error_code ec;std::vector<std::string> names;
std::filesystem::recursive_directory_iterator iterator(directory,std::filesystem::directory_options::none,ec),end;
if(ec)stop(83);
for(;iterator!=end;iterator.increment(ec)) {
if(ec)stop(84);
const auto status=iterator->symlink_status(ec);if(ec || std::filesystem::is_symlink(status))stop(85);
if(std::filesystem::is_directory(status))continue;
if(!std::filesystem::is_regular_file(status) || names.size()>=2048)stop(86);
auto name=iterator->path().lexically_relative(directory).generic_string();
if(name.empty() || name.size()>192 || name.find("..")!=std::string::npos)stop(87);
for(unsigned char c:name)if(c<33 || c>126)stop(88);
names.push_back(name);
}
if(ec || names.empty())stop(89);std::sort(names.begin(),names.end());
CC_SHA256_CTX manifest;CC_SHA256_Init(&manifest);uint64_t total=0;
for(const auto &name:names) {
uint64_t size=0;const auto digest=digestFile(directory/name,&size);total+=size;if(total>64*1024*1024)stop(90);
const uint32_t length=(uint32_t)name.size();CC_SHA256_Update(&manifest,&length,sizeof length);
CC_SHA256_Update(&manifest,name.data(),length);CC_SHA256_Update(&manifest,&size,sizeof size);CC_SHA256_Update(&manifest,digest.data(),digest.size());
}
uint64_t archiveBytes=0,sourceBytes=0;const auto archiveDigest=digestFile(archive,&archiveBytes);
const auto sourceDigest=digestFile("/System/Library/Frameworks/QuartzCore.framework/default.metallib",&sourceBytes);
auto *words=reinterpret_cast<uint64_t *>(snapshot);words[0]=0x515455495041434bull;words[1]=archiveBytes;
memcpy(snapshot+16,archiveDigest.data(),32);words[6]=names.size();words[7]=total;
CC_SHA256_Final(snapshot+64,&manifest);memcpy(snapshot+96,sourceDigest.data(),32);
result[6]=128;result[7]=sourceBytes;result[8]=names.size();result[9]=total;result[10]=archiveBytes;
int fd=open(archive.c_str(),O_RDONLY);if(fd<0 || fsync(fd))stop(91);close(fd);
sync();result[45]=1;result[47]=1;
}
int main(int argc,char **argv) {
(void)argv;result[0]=0x515443504b494f31ull;result[2]=(uint64_t)getpid();result[3]=getpagesize();result[4]=sysconf(_SC_PAGESIZE);
if(argc!=1 || getpid()!=1 || result[3]!=4096 || result[4]!=4096)stop(1);
mountCompilerView();
if(setenv("MESA_SHADER_CACHE_DISABLE","true",1) || unsetenv("DYLD_INSERT_LIBRARIES") || unsetenv("HOSTSIM_PAGE_SIZE") ||
unsetenv("NMB_TEST_GRAPHICS_CACHE_LOADER") || unsetenv("NMB_TEST_QUARTZCORE_LIBRARY"))stop(2);
// A fresh fixture must never reuse the earlier run's products.
struct stat old={};if(!lstat((std::string(prefix)+".archive").c_str(),&old) || errno!=ENOENT)stop(3);
if(unlink(workerLog) && errno!=ENOENT)stop(5);
posix_spawn_file_actions_t actions;
if(posix_spawn_file_actions_init(&actions) || posix_spawn_file_actions_addopen(&actions,STDOUT_FILENO,workerLog,O_WRONLY|O_CREAT|O_APPEND,0600) ||
posix_spawn_file_actions_adddup2(&actions,STDOUT_FILENO,STDERR_FILENO))stop(4);
const char *modes[]={"produce","consume","uuid-miss","empty-driver"};
for(unsigned i=0;i<4;++i) {
char *arguments[]={const_cast<char *>("compiler-worker"),const_cast<char *>(driverPath),const_cast<char *>(prefix),const_cast<char *>(modes[i]),nullptr};
pid_t child=0;int error=posix_spawn(&child,"/private/var/tmp/compiler-worker",&actions,nullptr,arguments,environ);
result[12+3*i]=(uint64_t)child;result[13+3*i]=(uint64_t)error;if(error || child<=1)stop(10+i);
int status=0;while(waitpid(child,&status,0)<0)if(errno!=EINTR)stop(20+i);
result[14+3*i]=(uint64_t)status;if(!WIFEXITED(status) || WEXITSTATUS(status)!=0)stop(30+i);result[5]=i+1;
}
posix_spawn_file_actions_destroy(&actions);summarize();stop(0);
}
drivers/NativeMetal/bridge/quartzcore_cache_provisioned_ios.cpp
+32
View File
@@ -0,0 +1,32 @@
/* SPDX-License-Identifier: MIT */
/* QEMU-only first-consumer harness. Reuse the exact volume/manifest/checkpoint
* helpers, but never call the cold producer suite. Initial Data already holds
* the actual iOS-generated archive and caches at the consumer's normal paths. */
#define main unused_cold_suite_main
#include "quartzcore_cache_ios.cpp"
#undef main
int main(int argc,char **argv) {
(void)argv;result[0]=0x515443504b494f32ull;result[2]=(uint64_t)getpid();result[3]=getpagesize();result[4]=sysconf(_SC_PAGESIZE);
if(argc!=1 || getpid()!=1 || result[3]!=4096 || result[4]!=4096)stop(1);
mountCompilerView();
if(setenv("MESA_SHADER_CACHE_DISABLE","true",1) || unsetenv("DYLD_INSERT_LIBRARIES") || unsetenv("HOSTSIM_PAGE_SIZE") ||
unsetenv("NMB_TEST_GRAPHICS_CACHE_LOADER") || unsetenv("NMB_TEST_QUARTZCORE_LIBRARY"))stop(2);
// Hash the actual initial-image bytes before any compiler worker is spawned.
summarize();std::array<unsigned char,128> initial={};memcpy(initial.data(),snapshot,initial.size());
if(unlink(workerLog) && errno!=ENOENT)stop(5);
posix_spawn_file_actions_t actions;
if(posix_spawn_file_actions_init(&actions) || posix_spawn_file_actions_addopen(&actions,STDOUT_FILENO,workerLog,O_WRONLY|O_CREAT|O_APPEND,0600) ||
posix_spawn_file_actions_adddup2(&actions,STDOUT_FILENO,STDERR_FILENO))stop(4);
const char *modes[]={"consume","uuid-miss","empty-driver"};
for(unsigned i=0;i<3;++i) {
char *arguments[]={const_cast<char *>("compiler-worker"),const_cast<char *>(driverPath),const_cast<char *>(prefix),const_cast<char *>(modes[i]),nullptr};
pid_t child=0;int error=posix_spawn(&child,"/private/var/tmp/compiler-worker",&actions,nullptr,arguments,environ);
result[12+3*i]=(uint64_t)child;result[13+3*i]=(uint64_t)error;if(error || child<=1)stop(10+i);
int status=0;while(waitpid(child,&status,0)<0)if(errno!=EINTR)stop(20+i);
result[14+3*i]=(uint64_t)status;if(!WIFEXITED(status) || WEXITSTATUS(status)!=0)stop(30+i);result[5]=i+1;
}
posix_spawn_file_actions_destroy(&actions);summarize();
if(memcmp(initial.data(),snapshot,initial.size()))stop(95);
memcpy(snapshot+256,initial.data(),initial.size());result[6]=384;result[45]=2;result[46]=1;stop(0);
}
drivers/NativeMetal/bridge/resource_lifetime_host.mm
+127
View File
@@ -0,0 +1,127 @@
// SPDX-License-Identifier: MIT
// Actual host GPU work waits on an unsignaled event while every caller-owned
// no-copy buffer/view reference is dropped. Its deallocator must not run yet.
#import <Foundation/Foundation.h>
#import <Metal/Metal.h>
#include <atomic>
#include <cstdio>
#include <cstring>
#include <memory>
#include <sys/mman.h>
#include <unistd.h>
#include "../NativeBridgeMetalFrontend.h"
#include "NativeMetalObjects.h"
struct State {std::atomic<unsigned> released{0};};
int main(int argc, char **argv) {
if (argc != 3) return 2;
@autoreleasepool {
id<MTLDevice> device = native_bridge_metal_open(argv[1], argv[2], 0x6e6d62, nullptr);
if (!device) return 2;
unsigned failures = 0;
size_t airBytes = 0;
const void *air = nmb_fixture_air(&airBytes);
id<MTLLibrary> library = native_bridge_metal_library_with_air(air, airBytes);
id<MTLComputePipelineState> pipeline = [device newComputePipelineStateWithFunction:[library newFunctionWithName:@"scale_and_offset"] error:nil];
if (!pipeline) return 2;
for (unsigned variant = 0; variant < 7; ++variant) {
auto state = std::make_shared<State>();
const size_t page = (size_t)getpagesize();
void *pages = mmap(nullptr, page, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0);
if (pages == MAP_FAILED) return 2;
memset(pages, variant == 6 ? 0xcc : 0x5a, page);
if (variant == 5) for (unsigned i = 0; i < 256; ++i) ((float *)pages)[i] = (float)i;
id<MTLBuffer> output = [device newBufferWithLength:4096 options:MTLResourceStorageModeShared];
id<MTLSharedEvent> event = [device newSharedEvent];
id<MTLCommandBuffer> commands = [[device newCommandQueue] commandBuffer];
__weak id weakBuffer;
if (!output || !event || !commands) return 2;
@autoreleasepool {
// Deliberately keep the mapping owned by the test, so the old
// early-deallocator failure cannot make the pending GPU read
// unmapped memory. No pixel or GPU completion is synthesized.
id<MTLBuffer> buffer = [device newBufferWithBytesNoCopy:pages length:page options:MTLResourceStorageModeShared
deallocator:^(void *, NSUInteger){state->released.fetch_add(1);}];
if (!buffer) return 2;
weakBuffer = buffer;
[commands encodeWaitForEvent:event value:1];
if (variant == 5) {
id<MTLComputeCommandEncoder> compute = [commands computeCommandEncoder];
[compute setComputePipelineState:pipeline];
[compute setBuffer:buffer offset:0 atIndex:0];
[compute setBuffer:output offset:0 atIndex:1];
const float scale = 2;
[compute setBytes:&scale length:sizeof scale atIndex:2];
[compute dispatchThreads:MTLSizeMake(256,1,1) threadsPerThreadgroup:MTLSizeMake(64,1,1)];
[compute endEncoding];
} else if (variant == 1 || variant == 6) {
auto d = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatRGBA8Unorm width:16 height:16 mipmapped:NO];
d.storageMode = MTLStorageModeShared;
d.usage = MTLTextureUsageShaderRead | MTLTextureUsagePixelFormatView | MTLTextureUsageRenderTarget;
id<MTLTexture> root = [buffer newTextureWithDescriptor:d offset:0 bytesPerRow:256];
id<MTLTexture> view = [[root newTextureViewWithPixelFormat:MTLPixelFormatRGBA8Unorm]
newTextureViewWithPixelFormat:MTLPixelFormatRGBA8Unorm];
if (!view) return 2;
if (variant == 6) {
auto pass = [MTLRenderPassDescriptor renderPassDescriptor];
pass.colorAttachments[0].texture = view;
pass.colorAttachments[0].loadAction = MTLLoadActionClear;
pass.colorAttachments[0].storeAction = MTLStoreActionStore;
pass.colorAttachments[0].clearColor = MTLClearColorMake(90.0/255,90.0/255,90.0/255,90.0/255);
[[commands renderCommandEncoderWithDescriptor:pass] endEncoding];
}
if (variant == 1) {
id<MTLBlitCommandEncoder> blit = [commands blitCommandEncoder];
[blit copyFromTexture:view sourceSlice:0 sourceLevel:0 sourceOrigin:MTLOriginMake(0,0,0) sourceSize:MTLSizeMake(16,16,1)
toBuffer:output destinationOffset:0 destinationBytesPerRow:64 destinationBytesPerImage:1024];
[blit endEncoding];
}
} else {
id<MTLBlitCommandEncoder> blit = [commands blitCommandEncoder];
if (variant == 2) [blit fillBuffer:buffer range:NSMakeRange(0, 1024) value:0x37];
[blit copyFromBuffer:buffer sourceOffset:0 toBuffer:output destinationOffset:0 size:1024];
id<MTLBuffer> missing = nil;
if (variant == 3) [blit fillBuffer:missing range:NSMakeRange(0, 16) value:0]; // local refusal, never submit
[blit endEncoding];
}
}
const unsigned before = state->released.load();
if (before || !weakBuffer) ++failures;
unsigned pending = before;
if (variant != 4) {
[commands commit];
// Completion must remain pending until the actual event is
// signaled, except a locally refused command which submitted
// no GPU work. Check with bounded dispatch waits.
auto finished = dispatch_semaphore_create(0);
dispatch_async(dispatch_get_global_queue(QOS_CLASS_DEFAULT, 0), ^{
@autoreleasepool {[commands waitUntilCompleted];}
dispatch_semaphore_signal(finished);
});
const bool early = dispatch_semaphore_wait(finished, dispatch_time(DISPATCH_TIME_NOW, 100 * NSEC_PER_MSEC)) == 0;
pending = state->released.load();
if (variant != 3 && (early || pending || !weakBuffer)) ++failures;
event.signaledValue = 1;
if (!early && dispatch_semaphore_wait(finished, dispatch_time(DISPATCH_TIME_NOW, 3 * NSEC_PER_SEC))) return 1;
if (commands.status != (variant == 3 ? MTLCommandBufferStatusError : MTLCommandBufferStatusCompleted)) ++failures;
if (variant != 3) for (unsigned i = 0; i < (variant == 5 ? 256 : 1024); ++i) {
const bool bad = variant == 5 ? ((const float *)output.contents)[i] != 3.0f*i
: variant == 6 ? ((const unsigned char *)pages)[(i / 64) * 256 + i % 64] != 0x5a
: ((const unsigned char *)output.contents)[i] != (variant == 2 ? 0x37 : 0x5a);
if (bad) {++failures;break;}
}
if (state->released.load() != 1 || weakBuffer) ++failures; // even while commands remains retained
}
commands = nil;
// The real completion callback can release its own reference just
// after waitUntilCompleted returns. Bound that scheduling margin.
for (unsigned i = 0; i < 300 && state->released.load() == 0; ++i) usleep(1000);
if (state->released.load() != 1 || weakBuffer) ++failures;
std::printf("case %u before=%u pending=%u released=%u\n", variant, before, pending, state->released.load());
munmap(pages, page);
}
if (failures) {std::printf("FAIL: resource owner lifetime (%u checks)\n", failures);return 1;}
std::puts("PASS: no-copy owners/views survive encoding and gated GPU work, release once after completion/refusal/discard; actual GPU bytes match");
}
}
drivers/NativeMetal/bridge/resource_wait_host.mm
+127
View File
@@ -0,0 +1,127 @@
// SPDX-License-Identifier: MIT
// Keep real work pending behind an unsignaled shared event. Waiting for an unrelated
// resource must return; waiting for its storage, views or IOSurface alias must not.
#import <Foundation/Foundation.h>
#import <Metal/Metal.h>
#import <IOSurface/IOSurfaceRef.h>
#include <cstdio>
#include <cstring>
#include <memory>
#include <sys/mman.h>
#include <unistd.h>
#include "../NativeBridgeMetalFrontend.h"
@protocol ResourceWait
- (void)waitUntilComplete;
@end
static dispatch_time_t seconds(unsigned n) {return dispatch_time(DISPATCH_TIME_NOW, n * NSEC_PER_SEC);}
int main(int argc, char **argv) {
if (argc != 3) return 2;
@autoreleasepool {
id<MTLDevice> device = native_bridge_metal_open(argv[1], argv[2], 0x6e6d62, nullptr);
if (!device) return 2;
for (unsigned variant = 0; variant < 9; ++variant) {
@autoreleasepool {
id<MTLBuffer> output = [device newBufferWithLength:4096 options:MTLResourceStorageModeShared];
id<MTLBuffer> input = [device newBufferWithLength:4096 options:MTLResourceStorageModeShared];
id query = output;
id<MTLTexture> target = nil;
IOSurfaceRef surface = nullptr;
void *pages = nullptr;
auto td = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatRGBA8Unorm width:16 height:16 mipmapped:NO];
td.storageMode = MTLStorageModeShared;
td.usage = MTLTextureUsageShaderRead | MTLTextureUsageShaderWrite | MTLTextureUsagePixelFormatView;
if (variant == 0) query = input; // never used by the pending fill
if (variant == 2) {
id<MTLTexture> texture = [output newTextureWithDescriptor:td offset:0 bytesPerRow:256];
query = [texture newTextureViewWithPixelFormat:MTLPixelFormatRGBA8Unorm];
}
if (variant == 3 || variant == 7) {
if (variant == 7) td.storageMode = MTLStorageModePrivate;
target = [device newTextureWithDescriptor:td];
query = [[target newTextureViewWithPixelFormat:MTLPixelFormatRGBA8Unorm] newTextureViewWithPixelFormat:MTLPixelFormatRGBA8Unorm];
}
if (variant == 4) {
NSDictionary *p = @{(id)kIOSurfaceWidth:@16, (id)kIOSurfaceHeight:@16,
(id)kIOSurfaceBytesPerElement:@4, (id)kIOSurfacePixelFormat:@((uint32_t)0x52474241)};
surface = IOSurfaceCreate((__bridge CFDictionaryRef)p);
target = [device newTextureWithDescriptor:td iosurface:surface plane:0];
query = [device newTextureWithDescriptor:td iosurface:surface plane:0];
}
if (variant == 5) {
auto hd = [MTLHeapDescriptor new];hd.size = 65536;hd.storageMode = MTLStorageModeShared;
id<MTLHeap> heap = [device newHeapWithDescriptor:hd];
output = [heap newBufferWithLength:4096 options:MTLResourceStorageModeShared];
query = [heap newBufferWithLength:4096 options:MTLResourceStorageModeShared];
}
if (variant == 6) {
const size_t page = (size_t)getpagesize();
pages = mmap(nullptr, page * 2, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0);
if (pages == MAP_FAILED) return 2;
auto keep = std::shared_ptr<void>(pages, [page](void *p){munmap(p, page * 2);});
auto release = ^(void *, NSUInteger){(void)keep;};
output = [device newBufferWithBytesNoCopy:pages length:page * 2 options:MTLResourceStorageModeShared deallocator:release];
query = [device newBufferWithBytesNoCopy:(char *)pages + page length:page options:MTLResourceStorageModeShared deallocator:release];
}
if (variant == 8) {
output = [device newBufferWithLength:4096 options:MTLResourceStorageModePrivate];
query = output;
}
if (!output || !input || !query || ((variant == 3 || variant == 4 || variant == 7) && !target)) {
std::printf("FAIL resource allocation %u\n", variant);return 2;
}
memset(input.contents, 0x5a, 4096);
id<MTLSharedEvent> event = [device newSharedEvent];
id<MTLCommandBuffer> commands = [[device newCommandQueue] commandBuffer];
[commands encodeWaitForEvent:event value:1];
id<MTLBlitCommandEncoder> blit = [commands blitCommandEncoder];
if (target) [blit copyFromBuffer:input sourceOffset:0 sourceBytesPerRow:64 sourceBytesPerImage:1024 sourceSize:MTLSizeMake(16,16,1)
toTexture:target destinationSlice:0 destinationLevel:0 destinationOrigin:MTLOriginMake(0,0,0)];
else [blit fillBuffer:output range:NSMakeRange(0, output.length) value:0x5a];
[blit endEncoding];[commands commit];
auto entered = dispatch_semaphore_create(0), returned = dispatch_semaphore_create(0);
dispatch_async(dispatch_get_global_queue(QOS_CLASS_DEFAULT, 0), ^{
dispatch_semaphore_signal(entered);
[(id<ResourceWait>)query waitUntilComplete];
dispatch_semaphore_signal(returned);
});
if (dispatch_semaphore_wait(entered, seconds(3))) return 1;
const bool early = dispatch_semaphore_wait(returned, dispatch_time(DISPATCH_TIME_NOW, 250 * NSEC_PER_MSEC)) == 0;
event.signaledValue = 1;
if (!early && dispatch_semaphore_wait(returned, seconds(3))) {std::puts("FAIL resource wait stuck");return 1;}
[commands waitUntilCompleted];
if (commands.status != MTLCommandBufferStatusCompleted || early != (variant == 0)) {
std::printf("FAIL variant=%u early=%d status=%lu\n", variant, early, (unsigned long)commands.status);return 1;
}
if (target) {
unsigned char pixels[1024] = {};
if (variant == 7) {
memset(input.contents, 0, 4096);
id<MTLCommandBuffer> readback = [[device newCommandQueue] commandBuffer];
id<MTLBlitCommandEncoder> copy = [readback blitCommandEncoder];
[copy copyFromTexture:target sourceSlice:0 sourceLevel:0 sourceOrigin:MTLOriginMake(0,0,0) sourceSize:MTLSizeMake(16,16,1)
toBuffer:input destinationOffset:0 destinationBytesPerRow:64 destinationBytesPerImage:1024];
[copy endEncoding];[readback commit];[readback waitUntilCompleted];
if (readback.status != MTLCommandBufferStatusCompleted) return 1;
memcpy(pixels, input.contents, sizeof pixels);
} else [target getBytes:pixels bytesPerRow:64 fromRegion:MTLRegionMake2D(0,0,16,16) mipmapLevel:0];
for (unsigned char pixel : pixels) if (pixel != 0x5a) return 1;
} else if (variant == 8) {
memset(input.contents, 0, 4096);
id<MTLCommandBuffer> readback = [[device newCommandQueue] commandBuffer];
id<MTLBlitCommandEncoder> copy = [readback blitCommandEncoder];
[copy copyFromBuffer:output sourceOffset:0 toBuffer:input destinationOffset:0 size:4096];
[copy endEncoding];[readback commit];[readback waitUntilCompleted];
if (readback.status != MTLCommandBufferStatusCompleted) return 1;
for (NSUInteger i = 0; i < 4096; ++i) if (((const unsigned char *)input.contents)[i] != 0x5a) return 1;
} else for (NSUInteger i = 0; i < output.length; ++i) if (((const unsigned char *)output.contents)[i] != 0x5a) return 1;
// The shared deallocator owns no-copy pages until both imports release them.
query = nil;output = nil;target = nil;
if (surface) CFRelease(surface);
}
}
std::puts("PASS: unrelated resource wait returns while work is gated; storage, views, heap and IOSurface/no-copy aliases wait for real completion");
}
}
drivers/NativeMetal/bridge/scheduled_host.mm
+70
View File
@@ -0,0 +1,70 @@
// SPDX-License-Identifier: MIT
// A blocked completed handler must not hold waitUntilScheduled. No hardware timing claim:
// the semaphores create a deterministic distinction between scheduling and completion.
#import <Foundation/Foundation.h>
#import <Metal/Metal.h>
#include <atomic>
#include <cstdio>
#include <initializer_list>
#include "../NativeBridgeMetalFrontend.h"
static dispatch_time_t deadline() {return dispatch_time(DISPATCH_TIME_NOW, 3 * NSEC_PER_SEC);}
__attribute__((noinline)) static id nothing() {return nil;}
@protocol CompletionQueue
- (void)setCompletionQueue:(dispatch_queue_t)queue;
@end
int main(int argc, char **argv) {
if (argc < 3) return 2;
@autoreleasepool {
id<MTLDevice> device = native_bridge_metal_open(argv[1], argv[2], 0x6e6d62, nullptr);
if (!device) return 2;
id<MTLCommandQueue> queue = [device newCommandQueue];
id<MTLBuffer> buffer = [device newBufferWithLength:256 options:MTLResourceStorageModeShared];
if (!queue || !buffer) return 2;
for (unsigned variant : {0u, 1u, 2u, 3u}) {
const bool refused = variant == 1 || variant == 3;
[(id<CompletionQueue>)queue setCompletionQueue:variant == 2 ? dispatch_queue_create("scheduled-test", DISPATCH_QUEUE_SERIAL) : nil];
id<MTLCommandBuffer> cb = [queue commandBuffer];
auto entered = dispatch_semaphore_create(0), release = dispatch_semaphore_create(0);
auto returned = dispatch_semaphore_create(0);
std::atomic<unsigned> scheduledCount{0}, completedCount{0};
auto scheduled = &scheduledCount, completed = &completedCount;
[cb addScheduledHandler:^(id<MTLCommandBuffer> command) {
// This also catches callbacks deferred to Completed by the frontend.
if (command.status == MTLCommandBufferStatusScheduled) ++*scheduled;
}];
[cb addCompletedHandler:^(id<MTLCommandBuffer>) {
++*completed;
dispatch_semaphore_signal(entered);
dispatch_semaphore_wait(release, deadline());
}];
id<MTLBlitCommandEncoder> blit = [cb blitCommandEncoder];
if (variant == 1) {
// A frontend encoding failure has no scheduling event to announce.
[blit fillBuffer:(id<MTLBuffer>)nothing() range:NSMakeRange(0, 16) value:0];
} else {
[blit fillBuffer:buffer range:NSMakeRange(0, 256) value:0x5a];
}
if (variant != 3) [blit endEncoding]; // variant 3: runtime refuses commit with an encoder open
[cb commit];
if (dispatch_semaphore_wait(entered, deadline())) {std::puts("FAIL completion never arrived");return 1;}
dispatch_async(dispatch_get_global_queue(QOS_CLASS_DEFAULT, 0), ^{
[cb waitUntilScheduled];
dispatch_semaphore_signal(returned);
});
const bool independent = dispatch_semaphore_wait(returned, dispatch_time(DISPATCH_TIME_NOW, 300 * NSEC_PER_MSEC)) == 0;
dispatch_semaphore_signal(release);
if (!independent && dispatch_semaphore_wait(returned, deadline())) {std::puts("FAIL scheduled wait stuck");return 1;}
[cb waitUntilCompleted];
if (variant == 3) [blit endEncoding];
if (!independent || completedCount != 1 || scheduledCount != (refused ? 0u : 1u) ||
cb.status != (refused ? MTLCommandBufferStatusError : MTLCommandBufferStatusCompleted)) {
std::printf("FAIL variant=%u independent=%d scheduled=%u completed=%u status=%lu\n",
variant, independent, scheduledCount.load(), completedCount.load(), (unsigned long)cb.status);
return 1;
}
}
std::puts("PASS: scheduling precedes completion on runtime/serial queues; scheduled waits ignore blocked completed handlers; frontend/runtime refusals announce no scheduling");
}
}
drivers/NativeMetal/bridge/surface_flush_host.mm
+102
View File
@@ -0,0 +1,102 @@
// SPDX-License-Identifier: MIT
// Count actual frontend cache-maintenance calls while forwarding every call
// to libSystem. GPU writes and alias reads must still reach the real pages.
#import <Foundation/Foundation.h>
#import <Metal/Metal.h>
#import <IOSurface/IOSurfaceRef.h>
#include <atomic>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <dlfcn.h>
#include <vector>
#include "../NativeBridgeMetalFrontend.h"
static std::atomic<uint64_t> calls, bytes;
extern "C" void sys_dcache_flush(void *address, size_t length) {
using Flush = void (*)(void *, size_t);
static Flush real = reinterpret_cast<Flush>(dlsym(RTLD_NEXT, "sys_dcache_flush"));
if (!real || real == sys_dcache_flush) std::abort();
calls++;bytes += length;
real(address, length);
}
static bool run(id<MTLDevice> device, unsigned aliases, unsigned allocations) {
@autoreleasepool {
constexpr unsigned width = 64, height = 16, row = width * 4, frameBytes = row * height;
std::vector<IOSurfaceRef> surfaces;
auto textures = [NSMutableArray array];
auto descriptor = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatRGBA8Unorm
width:width height:height mipmapped:NO];
descriptor.storageMode = MTLStorageModeShared;
descriptor.usage = MTLTextureUsageShaderRead | MTLTextureUsageShaderWrite | MTLTextureUsagePixelFormatView;
for (unsigned i = 0; i < allocations; ++i) {
NSDictionary *properties = @{(id)kIOSurfaceWidth:@(width), (id)kIOSurfaceHeight:@(height),
(id)kIOSurfaceBytesPerElement:@4, (id)kIOSurfaceBytesPerRow:@(row),
(id)kIOSurfacePixelFormat:@((uint32_t)0x52474241)};
IOSurfaceRef surface = IOSurfaceCreate((__bridge CFDictionaryRef)properties);
if (!surface || IOSurfaceLock(surface, 0, nullptr)) return false;
memset(IOSurfaceGetBaseAddress(surface), i ? 0xa7 : 0xcc, IOSurfaceGetAllocSize(surface));
if (IOSurfaceUnlock(surface, 0, nullptr)) return false;
surfaces.push_back(surface);
}
for (unsigned i = 0; i < aliases; ++i) {
// Include independent imports as well as nested views of an import.
id<MTLTexture> texture = [device newTextureWithDescriptor:descriptor iosurface:surfaces[i % allocations] plane:0];
if (i & 1) texture = [[texture newTextureViewWithPixelFormat:MTLPixelFormatRGBA8Unorm]
newTextureViewWithPixelFormat:MTLPixelFormatRGBA8Unorm];
if (!texture) return false;
[textures addObject:texture];
}
std::vector<unsigned char> upload(frameBytes, 0x5a);
id<MTLBuffer> input = [device newBufferWithBytes:upload.data() length:frameBytes options:MTLResourceStorageModeShared];
id<MTLBuffer> output = [device newBufferWithLength:aliases * frameBytes options:MTLResourceStorageModeShared];
id<MTLCommandBuffer> command = [[device newCommandQueue] commandBuffer];
id<MTLBlitCommandEncoder> blit = [command blitCommandEncoder];
if (!input || !output || !blit) return false;
[blit copyFromBuffer:input sourceOffset:0 sourceBytesPerRow:row sourceBytesPerImage:frameBytes
sourceSize:MTLSizeMake(width,height,1) toTexture:textures[0] destinationSlice:0 destinationLevel:0
destinationOrigin:MTLOriginMake(0,0,0)];
for (unsigned i = 0; i < aliases; ++i)
[blit copyFromTexture:textures[i] sourceSlice:0 sourceLevel:0 sourceOrigin:MTLOriginMake(0,0,0)
sourceSize:MTLSizeMake(width,height,1) toBuffer:output destinationOffset:i * frameBytes
destinationBytesPerRow:row destinationBytesPerImage:frameBytes];
[blit endEncoding];
const uint64_t beforeCalls = calls.load(), beforeBytes = bytes.load();
[command commit];[command waitUntilCompleted];
uint64_t expectedBytes = 0;
for (IOSurfaceRef surface : surfaces) expectedBytes += 2 * IOSurfaceGetAllocSize(surface);
bool ok = command.status == MTLCommandBufferStatusCompleted;
for (unsigned i = 0; i < aliases; ++i) for (unsigned j = 0; j < frameBytes; ++j)
if (((const unsigned char *)output.contents)[i * frameBytes + j] != (i % allocations ? 0xa7 : 0x5a)) ok = false;
// The GPU also changed the first surface; check its actual shared bytes.
for (unsigned y = 0; y < height; ++y) for (unsigned x = 0; x < row; ++x)
if (((const unsigned char *)IOSurfaceGetBaseAddress(surfaces[0]))[y * IOSurfaceGetBytesPerRow(surfaces[0]) + x] != 0x5a) ok = false;
const uint64_t actualCalls = calls.load() - beforeCalls, actualBytes = bytes.load() - beforeBytes;
std::printf("aliases=%u allocations=%u flushes=%llu bytes=%llu expected=%u/%llu pixels=%s\n", aliases, allocations,
(unsigned long long)actualCalls, (unsigned long long)actualBytes, 2 * allocations,
(unsigned long long)expectedBytes, ok ? "pass" : "FAIL");
ok &= actualCalls == 2 * allocations && actualBytes == expectedBytes;
for (IOSurfaceRef surface : surfaces) CFRelease(surface);
return ok;
}
}
int main(int argc, char **argv) {
if (argc != 3) return 2;
@autoreleasepool {
id<MTLDevice> device = native_bridge_metal_open(argv[1], argv[2], 0x6e6d62, nullptr);
if (!device) return 2;
bool ok = run(device, 1, 1);
ok &= run(device, 128, 1);
ok &= run(device, 128, 2);
char telemetry[6144] = {};
native_bridge_metal_telemetry(telemetry, sizeof telemetry);
const char *line = strstr(telemetry, "J surface cache ");
unsigned long long count = 0, maintained = 0, duplicates = 0;
if (!line || sscanf(line, "J surface cache n=%llu bytes=%llu dup=%llu", &count, &maintained, &duplicates) != 3 ||
count != calls.load() || maintained != bytes.load() || duplicates != 506) ok = false;
if (!ok) {std::puts("FAIL: duplicate IOSurface cache maintenance");return 1;}
std::puts("PASS: each live surface range flushed once per phase, independent imports/views deduplicated, distinct surfaces and GPU bytes preserved");
}
}
drivers/NativeMetal/bridge/surface_import_host.mm
+173
View File
@@ -0,0 +1,173 @@
// SPDX-License-Identifier: MIT
// Count real frontend no-copy imports/releases, forwarding every call to the
// unmodified bridge. Use actual GPU pixels and bounded pending-work lifetimes.
#import <Foundation/Foundation.h>
#import <Metal/Metal.h>
#import <IOSurface/IOSurfaceRef.h>
#include <atomic>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <dlfcn.h>
#include <mutex>
#include <set>
#include <vector>
#include <unistd.h>
#include "../NativeBridgeMetalFrontend.h"
#include "NativeMetalObjects.h"
static std::atomic<unsigned> imports, releases;
static std::mutex handlesLock;
static std::set<NmbBuffer *> handles;
static uint64_t (*vkImports)();
extern "C" NmbBuffer *tracked_surface_import(NmbDevice *d, void *p, uint64_t n, uint64_t options) {
NmbBuffer *buffer = nmb_buffer_new_no_copy(d,p,n,options);
if (buffer) {std::lock_guard<std::mutex> lock(handlesLock);handles.insert(buffer);imports++;}
return buffer;
}
extern "C" void tracked_buffer_release(NmbBuffer *buffer) {
{std::lock_guard<std::mutex> lock(handlesLock);if(handles.erase(buffer))releases++;}
nmb_buffer_release(buffer);
}
static bool drained(unsigned expected) {
for(unsigned i=0;i<300 && releases.load()!=expected;++i)usleep(1000);
return releases.load()==expected;
}
static IOSurfaceRef makeSurface(unsigned byte) {
IOSurfaceRef surface=IOSurfaceCreate((__bridge CFDictionaryRef)@{
(id)kIOSurfaceWidth:@64,(id)kIOSurfaceHeight:@16,(id)kIOSurfaceBytesPerElement:@4,
(id)kIOSurfaceBytesPerRow:@256,(id)kIOSurfacePixelFormat:@((uint32_t)0x52474241)});
if(!surface || IOSurfaceLock(surface,0,nullptr))return nullptr;
memset(IOSurfaceGetBaseAddress(surface),byte,IOSurfaceGetAllocSize(surface));
if(IOSurfaceUnlock(surface,0,nullptr)){CFRelease(surface);return nullptr;}
return surface;
}
static MTLTextureDescriptor *descriptor() {
auto d=[MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatRGBA8Unorm width:64 height:16 mipmapped:NO];
d.storageMode=MTLStorageModeShared;
d.usage=MTLTextureUsageShaderRead|MTLTextureUsageShaderWrite|MTLTextureUsagePixelFormatView;
return d;
}
static bool aliases(id<MTLDevice> device,unsigned count,unsigned allocations,bool parallel) {
const unsigned before=imports.load(),released=releases.load();bool ok=true;
const uint64_t vkBefore=vkImports?vkImports():0;
@autoreleasepool {
std::vector<IOSurfaceRef> surfaces;
for(unsigned i=0;i<allocations;++i){auto s=makeSurface(i?0xa7:0xcc);if(!s)return false;surfaces.push_back(s);}
auto textures=[NSMutableArray array];auto d=descriptor();
if(parallel) {
std::atomic<unsigned> failures{0};auto *failed=&failures;
const IOSurfaceRef surface=surfaces[0];
dispatch_apply(count,dispatch_get_global_queue(QOS_CLASS_DEFAULT,0),^(size_t i){
(void)i;
@autoreleasepool {
id<MTLTexture> texture=[device newTextureWithDescriptor:d iosurface:surface plane:0];
if(!texture){(*failed)++;return;}
@synchronized(textures){[textures addObject:texture];}
}
});
ok=failures.load()==0;
} else for(unsigned i=0;i<count;++i) {
id<MTLTexture> texture=[device newTextureWithDescriptor:d iosurface:surfaces[i%allocations] plane:0];
if(i&1)texture=[[texture newTextureViewWithPixelFormat:MTLPixelFormatRGBA8Unorm] newTextureViewWithPixelFormat:MTLPixelFormatRGBA8Unorm];
if(!texture)return false;[textures addObject:texture];
}
if(textures.count!=count)return false;
std::vector<unsigned char> data(4096,0x5a);
id<MTLBuffer> input=[device newBufferWithBytes:data.data() length:data.size() options:MTLResourceStorageModeShared];
id<MTLBuffer> output=[device newBufferWithLength:count*4096 options:MTLResourceStorageModeShared];
id<MTLCommandBuffer> command=[[device newCommandQueue] commandBuffer];id<MTLBlitCommandEncoder> copy=[command blitCommandEncoder];
if(!input || !output || !copy)return false;
[copy copyFromBuffer:input sourceOffset:0 sourceBytesPerRow:256 sourceBytesPerImage:4096 sourceSize:MTLSizeMake(64,16,1)
toTexture:textures[0] destinationSlice:0 destinationLevel:0 destinationOrigin:MTLOriginMake(0,0,0)];
for(unsigned i=0;i<count;++i)
[copy copyFromTexture:textures[i] sourceSlice:0 sourceLevel:0 sourceOrigin:MTLOriginMake(0,0,0) sourceSize:MTLSizeMake(64,16,1)
toBuffer:output destinationOffset:i*4096 destinationBytesPerRow:256 destinationBytesPerImage:4096];
[copy endEncoding];[command commit];[command waitUntilCompleted];
ok &= command.status==MTLCommandBufferStatusCompleted;
bool mismatch=false;
for(unsigned i=0;i<count;++i)for(unsigned j=0;j<4096;++j)
if(((const unsigned char *)output.contents)[i*4096+j]!=(i%allocations?0xa7:0x5a)) {
if(!mismatch)std::printf("first mismatch alias=%u byte=%u actual=%02x expected=%02x\n",i,j,
((const unsigned char *)output.contents)[i*4096+j],i%allocations?0xa7:0x5a);
mismatch=true;ok=false;
}
for(IOSurfaceRef surface:surfaces)CFRelease(surface);
}
const unsigned made=imports.load()-before;
const bool retired=drained(released+made);
// More than the metadata bound may evict live weak associations. The
// runtime must still share each live import and retire every real owner.
const bool reuse=allocations>256?(made>=allocations && made<=count):made==allocations;
const uint64_t actualVk=vkImports?vkImports()-vkBefore:0;
std::printf("aliases=%u surfaces=%u parallel=%d imports=%u releases=%u vk_imports=%llu gpu=%s retired=%s reuse=%s\n",
count,allocations,parallel,made,releases.load()-released,(unsigned long long)actualVk,ok?"pass":"FAIL",retired?"pass":"FAIL",reuse?"pass":"FAIL");
return ok && retired && reuse && (!vkImports || actualVk==allocations);
}
static bool expires(id<MTLDevice> device) {
const unsigned before=imports.load(),released=releases.load();IOSurfaceRef surface=makeSurface(0x35);if(!surface)return false;
bool ok=true;
for(unsigned i=0;i<2;++i) {
@autoreleasepool {id<MTLTexture> texture=[device newTextureWithDescriptor:descriptor() iosurface:surface plane:0];if(!texture)ok=false;}
ok &= drained(released+i+1) && imports.load()==before+i+1;
}
CFRelease(surface);
std::printf("expired owners imports=%u releases=%u result=%s\n",imports.load()-before,releases.load()-released,ok?"pass":"FAIL");
return ok;
}
static bool pending(id<MTLDevice> device) {
const unsigned before=imports.load(),released=releases.load();bool ok=true;
@autoreleasepool {
IOSurfaceRef surface=makeSurface(0x79);if(!surface)return false;
id<MTLSharedEvent> event=[device newSharedEvent];id<MTLCommandBuffer> command=[[device newCommandQueue] commandBuffer];
id<MTLBuffer> output=[device newBufferWithLength:4096 options:MTLResourceStorageModeShared];
if(!event || !command || !output)return false;
@autoreleasepool {
id<MTLTexture> texture=[device newTextureWithDescriptor:descriptor() iosurface:surface plane:0];if(!texture)return false;
[command encodeWaitForEvent:event value:1];id<MTLBlitCommandEncoder> copy=[command blitCommandEncoder];
[copy copyFromTexture:texture sourceSlice:0 sourceLevel:0 sourceOrigin:MTLOriginMake(0,0,0) sourceSize:MTLSizeMake(64,16,1)
toBuffer:output destinationOffset:0 destinationBytesPerRow:256 destinationBytesPerImage:4096];
[copy endEncoding];[command commit];
}
auto finished=dispatch_semaphore_create(0);
dispatch_async(dispatch_get_global_queue(QOS_CLASS_DEFAULT,0),^{@autoreleasepool{[command waitUntilCompleted];}dispatch_semaphore_signal(finished);});
if(!dispatch_semaphore_wait(finished,dispatch_time(DISPATCH_TIME_NOW,100*NSEC_PER_MSEC)))ok=false;
@autoreleasepool {
id<MTLTexture> alias=[device newTextureWithDescriptor:descriptor() iosurface:surface plane:0];
if(!alias || imports.load()!=before+1 || releases.load()!=released)ok=false;
// Drop the caller's last surface reference before the GPU reads it.
CFRelease(surface);surface=nullptr;
}
event.signaledValue=1;
if(dispatch_semaphore_wait(finished,dispatch_time(DISPATCH_TIME_NOW,3*NSEC_PER_SEC)))return false;
if(command.status!=MTLCommandBufferStatusCompleted)ok=false;
for(unsigned i=0;i<4096;++i)if(((const unsigned char *)output.contents)[i]!=0x79)ok=false;
// Completed commands must release the import even if still retained.
ok &= drained(released+1);
}
std::printf("pending owners imports=%u releases=%u result=%s\n",imports.load()-before,releases.load()-released,ok?"pass":"FAIL");
return ok;
}
int main(int argc,char **argv) {
if(argc!=3 && !(argc==4 && !strcmp(argv[3],"parallel")))return 2;
@autoreleasepool {
id<MTLDevice> device=native_bridge_metal_open(argv[1],argv[2],0x6e6d62,nullptr);if(!device)return 2;
void *loader=dlopen(argv[1],RTLD_NOW|RTLD_LOCAL);if(!loader)return 2;
vkImports=reinterpret_cast<uint64_t (*)()>(dlsym(loader,"nmb_test_vk_host_import_count"));
if(getenv("NMB_TEST_REAL_VULKAN_LOADER") && !vkImports)return 2;
if(argc==4)return aliases(device,64,1,true)?0:1;
bool ok=aliases(device,1,1,false);
ok &= aliases(device,128,1,false);
ok &= aliases(device,128,2,false);
ok &= aliases(device,64,1,true);
ok &= expires(device);
ok &= pending(device);
ok &= aliases(device,514,257,false);
if(!ok || imports.load()!=releases.load()){std::puts("FAIL: live IOSurface import reuse");return 1;}
std::puts("PASS: live IOSurface aliases reuse imports with real GPU bytes; distinct, parallel, expired and pending owners preserve lifetimes");
}
}
drivers/NativeMetal/bridge/telemetry_host.mm
+56
View File
@@ -0,0 +1,56 @@
// SPDX-License-Identifier: MIT
// Interleave encoding on two queues, then reverse commit order. Global deltas assign
// the first buffer's render work to the second buffer's blits; per-buffer counts must not.
#include "FrontendChecks.h"
#include "NativeMetalObjects.h"
#include "../NativeBridgeMetalFrontend.h"
#include <map>
#include <string>
int main(int argc, char **argv) {
if (argc != 3) return 2;
@autoreleasepool {
id<MTLDevice> device = native_bridge_metal_open(argv[1], argv[2], 0x6e6d62, nullptr);
if (!device) return 2;
size_t length = 0;
const void *air = nmb_fixture_air(&length);
id<MTLLibrary> library = native_bridge_metal_library_with_air(air, length);
frontendchecks::Report report = {};
frontendchecks::gReport = &report;
id<MTLRenderPipelineState> pipeline = frontendchecks::pipeline(device, library, @"colored_fragment", MTLPixelFormatRGBA8Unorm);
auto td = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatRGBA8Unorm width:16 height:16 mipmapped:NO];
td.usage = MTLTextureUsageRenderTarget;
id<MTLTexture> texture = [device newTextureWithDescriptor:td];
id<MTLBuffer> buffer = [device newBufferWithLength:256 options:MTLResourceStorageModeShared];
id<MTLCommandBuffer> render = [[device newCommandQueue] commandBuffer];
id<MTLCommandBuffer> copy = [[device newCommandQueue] commandBuffer];
if (!library || !pipeline || !texture || !buffer || !render || !copy) return 2;
frontendchecks::drawQuad(render, pipeline, texture, nil, frontendchecks::kQuarter, 6);
for (unsigned i = 0; i < 3; ++i) {
id<MTLBlitCommandEncoder> blit = [copy blitCommandEncoder];
[blit fillBuffer:buffer range:NSMakeRange(0, 256) value:0x5a];
[blit endEncoding];
}
frontendchecks::drawQuad(render, pipeline, texture, nil, frontendchecks::kQuarter, 6);
[copy commit];[render commit];
if (!frontendchecks::completed(copy, "blit") || !frontendchecks::completed(render, "render")) return 1;
char text[4608] = {};
native_bridge_metal_telemetry(text, sizeof text);
std::map<unsigned, std::pair<unsigned, unsigned>> counts;
for (char *line = strtok(text, "\n"); line; line = strtok(nullptr, "\n")) {
unsigned index, sec, ms, passes, draws, kpx, dispatches, blits;
if (sscanf(line, "K%u t=%u.%us rp%u dr%u kpx%u cs%u bl%u", &index, &sec, &ms, &passes, &draws, &kpx, &dispatches, &blits) != 8) continue;
std::printf("%s\n", line);
counts[index] = {passes, blits};
if (draws != (index == 2 ? 2u : 0u)) {std::puts("FAIL cross-buffer draw attribution");return 1;}
}
if (counts.size() != 2 || counts[1] != std::make_pair(0u, 3u) || counts[2] != std::make_pair(2u, 0u)) {
std::puts("FAIL interleaved render/blit attribution");return 1;
}
uint8_t pixels[16 * 16 * 4] = {};
[texture getBytes:pixels bytesPerRow:64 fromRegion:MTLRegionMake2D(0, 0, 16, 16) mipmapLevel:0];
if (frontendchecks::wrongTexels(pixels, 64, 16)) return 1;
for (unsigned i = 0; i < 256; ++i) if (((const uint8_t *)buffer.contents)[i] != 0x5a) return 1;
std::puts("PASS: opposite encode/commit orders retain per-buffer render/blit counts and correct host GPU pixels");
}
}
kits/ArmSmmu4K/ArmSmmu4KLive.c
+223 -57
View File
@@ -42,7 +42,7 @@ static uint64_t descriptor(const ArmSmmu4KRange *r,uint64_t iova)
if(r->writeback)attrs|=1u<<2;
return (r->physical+(iova-r->iova))|attrs;
}
static uint64_t expected(const ArmSmmu4KRange *ranges,size_t count,uint64_t iova)
static const ArmSmmu4KRange *find(const ArmSmmu4KRange *ranges,size_t count,uint64_t iova)
{
size_t low=0,high=count;
while(low<high) {
@@ -50,12 +50,34 @@ static uint64_t expected(const ArmSmmu4KRange *ranges,size_t count,uint64_t iova
const ArmSmmu4KRange *r=&ranges[mid];
if(iova<r->iova)high=mid;
else if(iova-r->iova>=r->bytes)low=mid+1;
else return descriptor(r,iova);
else return r;
}
return 0;
}
static int contains(const ArmSmmu4KRange *r,uint64_t iova)
{return r && iova>=r->iova && iova-r->iova<r->bytes;}
/* The walk visits addresses in ascending order, so one cursor per range list replaces a binary search for every leaf: lower_bound once per
* leaf page, then forward. A range is behind the address when the address lies past its end (written as a difference: native addresses
* may end at 2^64). */
static size_t first_not_behind(const ArmSmmu4KRange *ranges,size_t count,uint64_t va)
{
size_t low=0,high=count;
while(low<high) {
size_t mid=low+(high-low)/2;
if(va>=ranges[mid].iova && va-ranges[mid].iova>=ranges[mid].bytes)low=mid+1;
else high=mid;
}
return low;
}
static const ArmSmmu4KRange *scan(const ArmSmmu4KRange *ranges,size_t count,size_t *cursor,uint64_t va)
{
while(*cursor<count && va>=ranges[*cursor].iova && va-ranges[*cursor].iova>=ranges[*cursor].bytes)++*cursor;
if(*cursor<count && va>=ranges[*cursor].iova && va-ranges[*cursor].iova<ranges[*cursor].bytes)return &ranges[*cursor];
return 0;
}
static uint64_t expected(const ArmSmmu4KRange *ranges,size_t count,uint64_t iova)
{
const ArmSmmu4KRange *r=find(ranges,count,iova);
return r?descriptor(r,iova):0;
}
static ArmSmmu4KStatus validate_range(const ArmSmmu4KLiveArena *a,const ArmSmmu4KRange *r)
{
uint64_t physical_limit=UINT64_C(1)<<a->config.output_bits;
@@ -75,8 +97,97 @@ static int conflicting_alias(const ArmSmmu4KRange *a,const ArmSmmu4KRange *b)
((a->permissions^b->permissions)&ArmSmmu4KPrivileged)) &&
a->physical<b->physical+b->bytes && b->physical<a->physical+a->bytes;
}
/* Only ranges of different cache/shareability/privilege classes can alias in conflict, and of two conflicting ranges at most one
* is in the class most ranges of both sets have. So for every other class k, the members of class k in one set are compared with
* the members of other classes in the other set, both ways. Each such comparison sorts the smaller side by physical address in
* the arena's scratch and looks every range of the larger side up in it: a scattered buffer is one range per physical page, and
* with coherent imports (write-back) beside the GPU's own non-cacheable buffers a pairwise loop over both was some 2.4 billion
* comparisons per map (int-144: 368 ms per map on average, 8 s at worst). Without scratch, or a side larger than it, the
* comparison is the pairwise loop. */
static unsigned alias_class(const ArmSmmu4KRange *r)
{return (r->writeback&1u)|((r->shareability&3u)<<1)|((r->permissions&ArmSmmu4KPrivileged)?8u:0u);}
typedef struct {
const ArmSmmu4KRange *ranges;size_t count;
unsigned cls;int same; /* members: class == cls (same) or class != cls */
} AliasSide;
static int alias_member(const AliasSide *s,size_t i)
{return (alias_class(&s->ranges[i])==s->cls)==(s->same!=0);}
static void sift(const ArmSmmu4KRange *r,uint32_t *h,size_t root,size_t n)
{
for(;;) {
size_t child=2*root+1;
if(child>=n)return;
if(child+1<n && r[h[child+1]].physical>r[h[child]].physical)++child;
if(r[h[root]].physical>=r[h[child]].physical)return;
uint32_t t=h[root];h[root]=h[child];h[child]=t;root=child;
}
}
static int alias_pass(const ArmSmmu4KLiveArena *arena,const AliasSide *p,const AliasSide *q)
{
size_t np=0,nq=0;
for(size_t i=0;i<p->count;++i)np+=(size_t)alias_member(p,i);
for(size_t j=0;j<q->count;++j)nq+=(size_t)alias_member(q,j);
if(!np || !nq)return 0;
const AliasSide *sorted=np<=nq?p:q,*probe=np<=nq?q:p;
const size_t n=np<=nq?np:nq;
uint32_t *h=arena?arena->alias_scratch:0;
if(!h || n>arena->alias_scratch_entries || sorted->count>UINT32_MAX) {
for(size_t i=0;i<p->count;++i)if(alias_member(p,i))
for(size_t j=0;j<q->count;++j)if(alias_member(q,j) && conflicting_alias(&p->ranges[i],&q->ranges[j]))return 1;
return 0;
}
const ArmSmmu4KRange *r=sorted->ranges;
size_t k=0;
for(size_t i=0;i<sorted->count;++i)if(alias_member(sorted,i)) {
h[k++]=(uint32_t)i;
}
for(size_t i=n/2;i-->0;)sift(r,h,i,n);
for(size_t end=n;end>1;) {--end;uint32_t t=h[0];h[0]=h[end];h[end]=t;sift(r,h,0,end);}
/* Drop intervals contained by an earlier interval. Both starts and ends of
* the retained indices are now monotonic, even with nested/duplicate runs.
* A distant long run cannot force gap queries to scan short runs backward.
* Every compared pair belongs to different classes by AliasSide's filter. */
size_t kept=0;
for(size_t i=0;i<n;++i) {
uint32_t index=h[i];
if(!kept || r[index].physical+r[index].bytes>r[h[kept-1]].physical+r[h[kept-1]].bytes)h[kept++]=index;
}
for(size_t j=0;j<probe->count;++j) {
if(!alias_member(probe,j))continue;
const ArmSmmu4KRange *y=&probe->ranges[j];
size_t lo=0,hi=kept; /* first interval ending after y's start */
while(lo<hi) {size_t mid=lo+(hi-lo)/2;if(r[h[mid]].physical+r[h[mid]].bytes<=y->physical)lo=mid+1;else hi=mid;}
if(lo<kept && r[h[lo]].physical<y->physical+y->bytes)return 1;
}
return 0;
}
static int any_conflicting_alias(const ArmSmmu4KLiveArena *arena,const ArmSmmu4KRange *a,size_t na,
const ArmSmmu4KRange *b,size_t nb)
{
if(!na || !nb)return 0;
size_t members[16]={0};
for(size_t i=0;i<na;++i)++members[alias_class(&a[i])];
for(size_t j=0;j<nb;++j)++members[alias_class(&b[j])];
unsigned reference=0;
for(unsigned k=1;k<16;++k)if(members[k]>members[reference])reference=k;
const int self=a==b && na==nb;
for(unsigned k=0;k<16;++k) {
if(k==reference || !members[k])continue;
const AliasSide ak={a,na,k,1},bo={b,nb,k,0};
if(alias_pass(arena,&ak,&bo))return 1;
if(self)continue;
const AliasSide bk={b,nb,k,1},ao={a,na,k,0};
if(alias_pass(arena,&bk,&ao))return 1;
}
return 0;
}
static ArmSmmu4KStatus validate_ranges_checked(const ArmSmmu4KLiveArena *a,
const ArmSmmu4KRange *ranges,size_t count,size_t *leaves,int aliases);
static ArmSmmu4KStatus validate_ranges(const ArmSmmu4KLiveArena *a,
const ArmSmmu4KRange *ranges,size_t count,size_t *leaves)
{return validate_ranges_checked(a,ranges,count,leaves,1);}
static ArmSmmu4KStatus validate_ranges_checked(const ArmSmmu4KLiveArena *a,
const ArmSmmu4KRange *ranges,size_t count,size_t *leaves,int aliases)
{
*leaves=0;
if((count && !ranges) || count>a->config.leaf_budget ||
@@ -85,25 +196,29 @@ static ArmSmmu4KStatus validate_ranges(const ArmSmmu4KLiveArena *a,
ArmSmmu4KStatus status=validate_range(a,&ranges[i]);
if(status!=ArmSmmu4KSuccess)return status;
if(i && ranges[i].iova<=ranges[i-1].iova+(ranges[i-1].bytes-1))return ArmSmmu4KOverlap;
for(size_t j=0;j<i;++j)if(conflicting_alias(&ranges[i],&ranges[j]))return ArmSmmu4KOverlap;
size_t pages=(size_t)(ranges[i].bytes/PAGE);
if(pages>a->config.leaf_budget-*leaves)return ArmSmmu4KBudget;
*leaves+=pages;
}
return ArmSmmu4KSuccess;
return aliases && any_conflicting_alias(a,ranges,count,ranges,count)?ArmSmmu4KOverlap:ArmSmmu4KSuccess;
}
/* Traverse the actual fixed topology, never generate/copy a reference table.
* A fixed owned aperture reserves empty intermediate tables; every other slot
* must be zero. Resource expectations are metadata, not translated page tables. */
static ArmSmmu4KStatus walk(const ArmSmmu4KLiveArena *a,const ArmSmmu4KResult *shape,
unsigned level,size_t page,uint64_t prefix,size_t *next,unsigned build,
const ArmSmmu4KRange *ranges,size_t count,const ArmSmmu4KRange *change,
const ArmSmmu4KRange *ranges,size_t count,const ArmSmmu4KRange *changes,size_t changed,
unsigned map,ArmSmmu4KLiveResult *r)
{
uint64_t limit=UINT64_C(1)<<a->config.input_bits;
uint64_t low=a->iova&(limit-1),high=low+a->bytes-1;
unsigned shift=39-9*level;
unsigned width=level==shape->start_level?a->config.input_bits-shift:9;
size_t range_cursor=0,change_cursor=0;
if(level==3) {
uint64_t first=a->upper_region ? prefix|~(limit-1) : prefix;
range_cursor=first_not_behind(ranges,count,first);change_cursor=first_not_behind(changes,changed,first);
}
for(unsigned slot=0;slot<512;++slot) {
uint64_t begin=prefix+((uint64_t)slot<<shift),value=0;
int reserved=slot<(1u<<width) && begin<=high && low<begin+(UINT64_C(1)<<shift);
@@ -114,24 +229,45 @@ static ArmSmmu4KStatus walk(const ArmSmmu4KLiveArena *a,const ArmSmmu4KResult *s
value=(a->config.table_physical+child*PAGE)|3u;
} else if(level==3 && reserved && !build) {
uint64_t va=a->upper_region ? begin|~(limit-1) : begin;
value=contains(change,va)?(map?descriptor(change,va):0):expected(ranges,count,va);
const ArmSmmu4KRange *c=scan(changes,changed,&change_cursor,va);
if(c)value=map?descriptor(c,va):0;
else {
const ArmSmmu4KRange *r0=scan(ranges,count,&range_cursor,va);
value=r0?descriptor(r0,va):0;
}
}
size_t index=page*512+slot;
if(build)a->mapping[index]=value;
else {++r->descriptor_reads;if(a->mapping[index]!=value)return ArmSmmu4KStateMismatch;}
if(level<3 && reserved) {
ArmSmmu4KStatus status=walk(a,shape,level+1,child,begin,next,build,ranges,count,change,map,r);
ArmSmmu4KStatus status=walk(a,shape,level+1,child,begin,next,build,ranges,count,changes,changed,map,r);
if(status!=ArmSmmu4KSuccess)return status;
}
}
return ArmSmmu4KSuccess;
}
static ArmSmmu4KStatus leaf_index(const ArmSmmu4KLiveArena *a,
const ArmSmmu4KResult *shape,uint64_t va,size_t *index,ArmSmmu4KLiveResult *r);
/* The changed leaves alone, through the table path each one's address takes: before the store (after=0) each holds what the
* operation replaces (nothing for a map, its range's descriptor for an unmap), after it what the operation wrote. */
static ArmSmmu4KStatus verify_changed(const ArmSmmu4KLiveArena *a,const ArmSmmu4KResult *shape,
const ArmSmmu4KRange *changes,size_t changed,unsigned map,unsigned after,ArmSmmu4KLiveResult *r)
{
for(size_t i=0;i<changed;++i)for(uint64_t offset=0;offset<changes[i].bytes;offset+=PAGE) {
const uint64_t va=changes[i].iova+offset;size_t index;
ArmSmmu4KStatus status=leaf_index(a,shape,va,&index,r);if(status!=ArmSmmu4KSuccess)return status;
const uint64_t want=(map==after)?descriptor(&changes[i],va):0;
++r->descriptor_reads;
if(a->mapping[index]!=want)return ArmSmmu4KStateMismatch;
}
return ArmSmmu4KSuccess;
}
static ArmSmmu4KStatus verify(const ArmSmmu4KLiveArena *a,const ArmSmmu4KResult *shape,
const ArmSmmu4KRange *ranges,size_t count,const ArmSmmu4KRange *change,
const ArmSmmu4KRange *ranges,size_t count,const ArmSmmu4KRange *changes,size_t changed,
unsigned map,ArmSmmu4KLiveResult *r)
{
size_t next=1;
ArmSmmu4KStatus status=walk(a,shape,shape->start_level,0,0,&next,0,ranges,count,change,map,r);
ArmSmmu4KStatus status=walk(a,shape,shape->start_level,0,0,&next,0,ranges,count,changes,changed,map,r);
return status==ArmSmmu4KSuccess && next!=shape->table_pages?ArmSmmu4KStateMismatch:status;
}
ArmSmmu4KStatus ArmSmmu4KLivePrepare(const ArmSmmu4KLiveArena *a,ArmSmmu4KLiveState *state)
@@ -141,7 +277,7 @@ ArmSmmu4KStatus ArmSmmu4KLivePrepare(const ArmSmmu4KLiveArena *a,ArmSmmu4KLiveSt
ArmSmmu4KStatus status=layout(a,&shape);
if(status!=ArmSmmu4KSuccess)return status;
size_t next=1;
status=walk(a,&shape,shape.start_level,0,0,&next,1,0,0,0,0,0);
status=walk(a,&shape,shape.start_level,0,0,&next,1,0,0,0,0,0,0);
if(status!=ArmSmmu4KSuccess || next!=shape.table_pages) {state->poisoned=1;return ArmSmmu4KStateMismatch;}
state->tables=shape;state->generation=0;state->prepared=1;
return ArmSmmu4KSuccess;
@@ -166,7 +302,7 @@ ArmSmmu4KStatus ArmSmmu4KLiveAudit(const ArmSmmu4KLiveArena *a,const ArmSmmu4KLi
size_t leaves=0;
status=validate_ranges(a,ranges,count,&leaves);if(status!=ArmSmmu4KSuccess)return status;
if(leaves!=state->tables.leaf_pages)return ArmSmmu4KArgument;
status=verify(a,&shape,ranges,count,0,0,r);
status=verify(a,&shape,ranges,count,0,0,0,r);
r->complete=status==ArmSmmu4KSuccess;return status;
}
static ArmSmmu4KStatus leaf_index(const ArmSmmu4KLiveArena *a,
@@ -189,7 +325,7 @@ static int overlaps(uint64_t a,uint64_t abytes,uint64_t b,uint64_t bbytes)
static int touches_pool(const ArmSmmu4KRange *r,const ArmSmmu4KLiveArena *a)
{return overlaps(r->physical,r->bytes,a->config.table_physical,a->capacity_pages*PAGE);}
static ArmSmmu4KStatus region_metadata(const ArmSmmu4KLiveRegion *region,
const ArmSmmu4KConfig *config,const ArmSmmu4KResult *root,unsigned upper)
const ArmSmmu4KConfig *config,const ArmSmmu4KResult *root,unsigned upper,int aliases)
{
if(!region)return (!config && !root)?ArmSmmu4KSuccess:ArmSmmu4KArgument;
if(!region->arena || !region->state || region->arena->upper_region!=upper ||
@@ -201,27 +337,26 @@ static ArmSmmu4KStatus region_metadata(const ArmSmmu4KLiveRegion *region,
if(r->root_physical!=shape.root_physical || r->mair!=shape.mair || r->table_pages!=shape.table_pages ||
r->start_level!=shape.start_level || r->upper_region!=shape.upper_region)return ArmSmmu4KArgument;
size_t leaves=0;
status=validate_ranges(region->arena,region->ranges,region->count,&leaves);if(status)return status;
status=validate_ranges_checked(region->arena,region->ranges,region->count,&leaves,aliases);if(status)return status;
return leaves==r->leaf_pages?ArmSmmu4KSuccess:ArmSmmu4KArgument;
}
/* aliases=0 (an update that is not its region's whole-verification one) skips the existing ranges' alias checks against each other
* and across the regions: each range was checked against all of them when it was mapped, and the 64th update re-proves them. */
static ArmSmmu4KStatus context_metadata(const ArmSmmu4KContextConfig *context,
const ArmSmmu4KLiveRegion *lower,const ArmSmmu4KLiveRegion *upper)
const ArmSmmu4KLiveRegion *lower,const ArmSmmu4KLiveRegion *upper,int aliases)
{
if(!context || (!lower && !upper))return ArmSmmu4KArgument;
ArmSmmu4KStatus status=region_metadata(lower,context->lower,context->lower_tables,0);if(status)return status;
status=region_metadata(upper,context->upper,context->upper_tables,1);if(status)return status;
ArmSmmu4KStatus status=region_metadata(lower,context->lower,context->lower_tables,0,aliases);if(status)return status;
status=region_metadata(upper,context->upper,context->upper_tables,1,aliases);if(status)return status;
if(lower && upper) {
const ArmSmmu4KLiveArena *a=lower->arena,*b=upper->arena;
if(lower->state==upper->state || a->config.output_bits!=b->config.output_bits ||
overlaps(a->config.table_physical,a->capacity_pages*PAGE,b->config.table_physical,b->capacity_pages*PAGE) ||
overlaps((uintptr_t)a->mapping,a->capacity_pages*PAGE,(uintptr_t)b->mapping,b->capacity_pages*PAGE))
return ArmSmmu4KOverlap;
for(size_t i=0;i<lower->count;++i) {
if(touches_pool(&lower->ranges[i],b))return ArmSmmu4KOverlap;
for(size_t j=0;j<upper->count;++j)
if(conflicting_alias(&lower->ranges[i],&upper->ranges[j]))return ArmSmmu4KOverlap;
}
for(size_t i=0;i<lower->count;++i)if(touches_pool(&lower->ranges[i],b))return ArmSmmu4KOverlap;
for(size_t i=0;i<upper->count;++i)if(touches_pool(&upper->ranges[i],a))return ArmSmmu4KOverlap;
if(aliases && any_conflicting_alias(lower->arena,lower->ranges,lower->count,upper->ranges,upper->count))return ArmSmmu4KOverlap;
}
return ArmSmmu4KSuccess;
}
@@ -230,23 +365,24 @@ ArmSmmu4KStatus ArmSmmu4KLiveAuditContext(const ArmSmmu4KContextConfig *context,
{
if(!r)return ArmSmmu4KArgument;
*r=(ArmSmmu4KLiveResult){0};r->must_retain_resources=1;
ArmSmmu4KStatus status=context_metadata(context,lower,upper);if(status)return status;
ArmSmmu4KStatus status=context_metadata(context,lower,upper,1);if(status)return status;
const ArmSmmu4KLiveRegion *regions[]={lower,upper};
for(unsigned i=0;i<2;++i)if(regions[i]) {
const ArmSmmu4KLiveRegion *v=regions[i];
status=verify(v->arena,&v->state->tables,v->ranges,v->count,0,0,r);if(status)return status;
status=verify(v->arena,&v->state->tables,v->ranges,v->count,0,0,0,r);if(status)return status;
}
r->complete=1;return ArmSmmu4KSuccess;
}
static ArmSmmu4KStatus update(const ArmSmmu4KLiveArena *a,ArmSmmu4KLiveState *state,
const ArmSmmu4KIo *io,unsigned bank,const ArmSmmu4KContextConfig *context,
const ArmSmmu4KLiveVisibility *visibility,const ArmSmmu4KRange *before,size_t count,
const ArmSmmu4KLiveRegion *peer,const ArmSmmu4KRange *change,unsigned map,unsigned budget,ArmSmmu4KLiveResult *r)
const ArmSmmu4KLiveRegion *peer,const ArmSmmu4KRange *changes,size_t change_count,unsigned map,unsigned budget,
ArmSmmu4KLiveResult *r)
{
if(!r)return ArmSmmu4KArgument;
*r=(ArmSmmu4KLiveResult){0};r->must_retain_resources=1;
if(!state || !state->prepared || state->poisoned || !context ||
!visibility || !visibility->publish_table_page || map>1 ||
!visibility || !visibility->publish_table_page || map>1 || !change_count ||
!budget || budget>1048576 || state->generation==UINT64_MAX)return ArmSmmu4KArgument;
ArmSmmu4KResult shape;
ArmSmmu4KStatus status=layout(a,&shape);
@@ -258,31 +394,51 @@ static ArmSmmu4KStatus update(const ArmSmmu4KLiveArena *a,ArmSmmu4KLiveState *st
state->tables.mair!=shape.mair || state->tables.table_pages!=shape.table_pages ||
state->tables.start_level!=shape.start_level || state->tables.upper_region!=shape.upper_region)
return ArmSmmu4KArgument;
/* Every 64th update of a region (its first included) re-proves the whole region: the existing ranges against each other and every
* table descriptor before and after. The others prove what they change: each changed leaf holds its old descriptor before the store
* and its new one after it, the existing ranges are still sorted, disjoint and in budget, and the changes conflict with none of
* them. The ranges were each checked as changes when they were added, and the tables were last proved whole at most 63 updates ago.
* A whole-region walk read every descriptor of a 2 GiB aperture four times per update (int-143: 10 ms per buffer map). */
const int whole=(state->generation&63)==0;
r->whole_verification=whole?1u:0u;
size_t leaves=0;
status=validate_ranges(a,before,count,&leaves);if(status!=ArmSmmu4KSuccess)return status;
status=validate_ranges_checked(a,before,count,&leaves,whole);if(status!=ArmSmmu4KSuccess)return status;
if(leaves!=state->tables.leaf_pages)return ArmSmmu4KArgument;
status=validate_range(a,change);if(status!=ArmSmmu4KSuccess)return status;
/* The changes: sorted, disjoint, each valid; together within the leaf budget. */
size_t changed=0;
if(!changes)return ArmSmmu4KBadRange;
if(change_count>a->config.leaf_budget)return ArmSmmu4KArgument;
for(size_t i=0;i<change_count;++i) {
status=validate_range(a,&changes[i]);if(status!=ArmSmmu4KSuccess)return status;
if(i && changes[i].iova<=changes[i-1].iova+(changes[i-1].bytes-1))return ArmSmmu4KOverlap;
size_t pages=(size_t)(changes[i].bytes/PAGE);
if(pages>a->config.leaf_budget-changed)return ArmSmmu4KBudget;
changed+=pages;
}
if(peer) {
ArmSmmu4KLiveRegion target={a,state,before,count};
status=context_metadata(context,a->upper_region?peer:&target,a->upper_region?&target:peer);
status=context_metadata(context,a->upper_region?peer:&target,a->upper_region?&target:peer,whole);
if(status)return status;
if(touches_pool(change,peer->arena))return ArmSmmu4KOverlap;
if(map)for(size_t i=0;i<peer->count;++i)
if(conflicting_alias(change,&peer->ranges[i]))return ArmSmmu4KOverlap;
for(size_t i=0;i<change_count;++i)if(touches_pool(&changes[i],peer->arena))return ArmSmmu4KOverlap;
if(map && any_conflicting_alias(a,changes,change_count,peer->ranges,peer->count))return ArmSmmu4KOverlap;
}
size_t changed=(size_t)(change->bytes/PAGE);
if(map && changed>a->config.leaf_budget-leaves)return ArmSmmu4KBudget;
if(map)for(size_t i=0;i<count;++i)if(conflicting_alias(change,&before[i]))return ArmSmmu4KOverlap;
for(uint64_t offset=0;offset<change->bytes;offset+=PAGE) {
uint64_t va=change->iova+offset,old=expected(before,count,va);
if(map?old!=0:old!=descriptor(change,va))return ArmSmmu4KStateMismatch;
if(map && (any_conflicting_alias(a,changes,change_count,changes,change_count) ||
any_conflicting_alias(a,changes,change_count,before,count)))return ArmSmmu4KOverlap;
for(size_t i=0;i<change_count;++i)for(uint64_t offset=0;offset<changes[i].bytes;offset+=PAGE) {
uint64_t va=changes[i].iova+offset,old=expected(before,count,va);
if(map?old!=0:old!=descriptor(&changes[i],va))return ArmSmmu4KStateMismatch;
}
status=ArmSmmu4KObserveOwnedContext(io,bank,context,&r->synchronization);
if(status!=ArmSmmu4KSuccess)goto poisoned;
status=verify(a,&shape,before,count,0,0,r);if(status!=ArmSmmu4KSuccess)goto poisoned;
if(peer) {
status=verify(peer->arena,&peer->state->tables,peer->ranges,peer->count,0,0,r);
if(status)goto poisoned;
if(whole) {
status=verify(a,&shape,before,count,0,0,0,r);if(status!=ArmSmmu4KSuccess)goto poisoned;
if(peer) {
status=verify(peer->arena,&peer->state->tables,peer->ranges,peer->count,0,0,0,r);
if(status)goto poisoned;
}
} else {
status=verify_changed(a,&shape,changes,change_count,map,0,r);if(status!=ArmSmmu4KSuccess)goto poisoned;
}
ArmSmmu4KProgramResult initial=r->synchronization;
status=ArmSmmu4KObserveOwnedContext(io,bank,context,&r->synchronization);
@@ -292,15 +448,16 @@ static ArmSmmu4KStatus update(const ArmSmmu4KLiveArena *a,ArmSmmu4KLiveState *st
{status=ArmSmmu4KStateMismatch;goto poisoned;}
/* All validation precedes the first store. Caller keeps every master
* quiesced and retains both old/new object references through this call. */
for(uint64_t offset=0;offset<change->bytes;offset+=PAGE) {
uint64_t va=change->iova+offset;size_t index;
for(size_t i=0;i<change_count;++i)for(uint64_t offset=0;offset<changes[i].bytes;offset+=PAGE) {
uint64_t va=changes[i].iova+offset;size_t index;
status=leaf_index(a,&shape,va,&index,r);if(status!=ArmSmmu4KSuccess)goto poisoned;
a->mapping[index]=map?descriptor(change,va):0;++r->descriptor_writes;
a->mapping[index]=map?descriptor(&changes[i],va):0;++r->descriptor_writes;
}
/* Sorted changes reach leaf pages in order, so each changed page is published once. */
size_t last_page=SIZE_MAX;
for(uint64_t offset=0;offset<change->bytes;offset+=PAGE) {
for(size_t i=0;i<change_count;++i)for(uint64_t offset=0;offset<changes[i].bytes;offset+=PAGE) {
size_t index;
status=leaf_index(a,&shape,change->iova+offset,&index,r);if(status!=ArmSmmu4KSuccess)goto poisoned;
status=leaf_index(a,&shape,changes[i].iova+offset,&index,r);if(status!=ArmSmmu4KSuccess)goto poisoned;
size_t page=index/512;
if(page!=last_page) {
if(visibility->publish_table_page(visibility->cookie,a->mapping+page*512,4096)) {status=ArmSmmu4KIoError;goto poisoned;}
@@ -309,10 +466,14 @@ static ArmSmmu4KStatus update(const ArmSmmu4KLiveArena *a,ArmSmmu4KLiveState *st
}
status=ArmSmmu4KSynchronizeOwnedContext(io,bank,context,budget,&r->synchronization);
if(status!=ArmSmmu4KSuccess)goto poisoned;
status=verify(a,&shape,before,count,change,map,r);if(status!=ArmSmmu4KSuccess)goto poisoned;
if(peer) {
status=verify(peer->arena,&peer->state->tables,peer->ranges,peer->count,0,0,r);
if(status)goto poisoned;
if(whole) {
status=verify(a,&shape,before,count,changes,change_count,map,r);if(status!=ArmSmmu4KSuccess)goto poisoned;
if(peer) {
status=verify(peer->arena,&peer->state->tables,peer->ranges,peer->count,0,0,0,r);
if(status)goto poisoned;
}
} else {
status=verify_changed(a,&shape,changes,change_count,map,1,r);if(status!=ArmSmmu4KSuccess)goto poisoned;
}
ArmSmmu4KProgramResult after;
status=ArmSmmu4KObserveOwnedContext(io,bank,context,&after);if(status!=ArmSmmu4KSuccess)goto poisoned;
@@ -329,18 +490,23 @@ ArmSmmu4KStatus ArmSmmu4KLiveUpdate(const ArmSmmu4KLiveArena *a,ArmSmmu4KLiveSta
const ArmSmmu4KIo *io,unsigned bank,const ArmSmmu4KContextConfig *context,
const ArmSmmu4KLiveVisibility *visibility,const ArmSmmu4KRange *before,size_t count,
const ArmSmmu4KRange *change,unsigned map,unsigned budget,ArmSmmu4KLiveResult *r)
{return update(a,state,io,bank,context,visibility,before,count,0,change,map,budget,r);}
ArmSmmu4KStatus ArmSmmu4KLiveUpdateContext(const ArmSmmu4KIo *io,unsigned bank,
{return update(a,state,io,bank,context,visibility,before,count,0,change,1,map,budget,r);}
ArmSmmu4KStatus ArmSmmu4KLiveUpdateContextBatch(const ArmSmmu4KIo *io,unsigned bank,
const ArmSmmu4KContextConfig *context,const ArmSmmu4KLiveRegion *lower,
const ArmSmmu4KLiveRegion *upper,unsigned change_upper,const ArmSmmu4KLiveVisibility *visibility,
const ArmSmmu4KRange *change,unsigned map,unsigned budget,ArmSmmu4KLiveResult *r)
const ArmSmmu4KRange *changes,size_t change_count,unsigned map,unsigned budget,ArmSmmu4KLiveResult *r)
{
if(!r)return ArmSmmu4KArgument;
*r=(ArmSmmu4KLiveResult){0};r->must_retain_resources=1;
if(change_upper>1)return ArmSmmu4KArgument;
ArmSmmu4KStatus status=context_metadata(context,lower,upper);if(status)return status;
const ArmSmmu4KLiveRegion *target=change_upper?upper:lower,*peer=change_upper?lower:upper;
if(!target)return ArmSmmu4KArgument;
if(!target || !target->state)return ArmSmmu4KArgument;
ArmSmmu4KStatus status=context_metadata(context,lower,upper,(target->state->generation&63)==0);if(status)return status;
return update(target->arena,target->state,io,bank,context,visibility,target->ranges,target->count,
peer,change,map,budget,r);
peer,changes,change_count,map,budget,r);
}
ArmSmmu4KStatus ArmSmmu4KLiveUpdateContext(const ArmSmmu4KIo *io,unsigned bank,
const ArmSmmu4KContextConfig *context,const ArmSmmu4KLiveRegion *lower,
const ArmSmmu4KLiveRegion *upper,unsigned change_upper,const ArmSmmu4KLiveVisibility *visibility,
const ArmSmmu4KRange *change,unsigned map,unsigned budget,ArmSmmu4KLiveResult *r)
{return ArmSmmu4KLiveUpdateContextBatch(io,bank,context,lower,upper,change_upper,visibility,change,1,map,budget,r);}
kits/ArmSmmu4K/ArmSmmu4KLive.h
+17
View File
@@ -11,6 +11,10 @@ typedef struct {
unsigned upper_region;
volatile uint64_t *mapping; /* Real writable CPU mapping of owned pool. */
size_t capacity_pages;
/* Optional caller-owned scratch for the physical-alias checks (one entry per range of the smaller compared side), used by one
* call at a time; absent or too small, those checks compare pairwise. Never read as state. */
uint32_t *alias_scratch;
size_t alias_scratch_entries;
} ArmSmmu4KLiveArena;
typedef struct {
ArmSmmu4KResult tables;
@@ -33,6 +37,7 @@ typedef struct {
typedef struct {
size_t descriptor_reads,descriptor_writes,published_pages;
unsigned complete,translation_removed,must_retain_resources;
unsigned whole_verification; /* this update re-proved the whole region (every 64th of a region) */
ArmSmmu4KProgramResult synchronization;
} ArmSmmu4KLiveResult;
/* Prepare an INACTIVE empty native4K arena. Preallocate only real intermediate
@@ -89,6 +94,18 @@ ArmSmmu4KStatus ArmSmmu4KLiveUpdateContext(const ArmSmmu4KIo *,unsigned bank,
const ArmSmmu4KLiveRegion *upper,unsigned change_upper,
const ArmSmmu4KLiveVisibility *,const ArmSmmu4KRange *change,unsigned map,
unsigned poll_budget,ArmSmmu4KLiveResult *);
/* UpdateContext for several ranges of one region in one transaction: changes are
* sorted by native IOVA and disjoint; map wants each entirely invalid, unmap
* each exactly mapped. Every check of UpdateContext covers all of them before
* the first store; their leaf pages are published once, the ASID synchronized
* once and both graphs rechecked once. A buffer of scattered physical pages is
* one change per page, so its mapping costs one transaction, not one per page.
* The changes may lie inside the target region's ranges (an unmap). */
ArmSmmu4KStatus ArmSmmu4KLiveUpdateContextBatch(const ArmSmmu4KIo *,unsigned bank,
const ArmSmmu4KContextConfig *,const ArmSmmu4KLiveRegion *lower,
const ArmSmmu4KLiveRegion *upper,unsigned change_upper,
const ArmSmmu4KLiveVisibility *,const ArmSmmu4KRange *changes,size_t change_count,
unsigned map,unsigned poll_budget,ArmSmmu4KLiveResult *);
#ifdef __cplusplus
}
#endif
kits/ArmSmmu4K/test_alias.c
+113
View File
@@ -0,0 +1,113 @@
/* SPDX-License-Identifier: MIT */
/* The sorted physical-alias check against the pairwise definition (conflicting_alias over every pair) on random sets: scattered
* single pages and long runs, up to eight attribute classes, self and cross comparisons, with scratch, without it and with too
* little of it; then one int-144-sized case (24k coherent import pages beside 82k non-cacheable pages) for its cost. */
#ifndef ARM_SMMU4K_ALIAS_SOURCE
#define ARM_SMMU4K_ALIAS_SOURCE "ArmSmmu4KLive.c"
#endif
#include ARM_SMMU4K_ALIAS_SOURCE
#include <stdio.h>
#include <stdlib.h>
#include <time.h>
#ifdef ARM_SMMU4K_TEST_COVERAGE
/* Compiler basic-block coverage counts, test-only. Production needs no hook or
* counter. Volatile keeps the caller's reads intact before coverage insertion. */
static volatile uint64_t alias_steps;
__attribute__((no_sanitize("coverage"))) void __sanitizer_cov_trace_pc_guard(uint32_t *guard)
{(void)guard;++alias_steps;}
__attribute__((no_sanitize("coverage"))) void __sanitizer_cov_trace_pc_guard_init(uint32_t *start,uint32_t *end)
{if(start==end || *start)return;for(uint32_t *p=start;p<end;++p)*p=1;}
#endif
#define CHECK(x) do{if(!(x)){fprintf(stderr,"%s:%d: %s\n",__FILE__,__LINE__,#x);exit(1);}}while(0)
static uint64_t seed=0x9e3779b97f4a7c15ull;
static uint64_t rnd(void){seed^=seed<<13;seed^=seed>>7;seed^=seed<<17;return seed;}
static int pairwise(const ArmSmmu4KRange *a,size_t na,const ArmSmmu4KRange *b,size_t nb)
{
for(size_t i=0;i<na;++i)for(size_t j=0;j<nb;++j) {
const int different=a[i].writeback!=b[j].writeback || a[i].shareability!=b[j].shareability ||
((a[i].permissions^b[j].permissions)&ArmSmmu4KPrivileged);
if(different && a[i].physical<b[j].physical+b[j].bytes && b[j].physical<a[i].physical+a[i].bytes)return 1;
}
return 0;
}
static void fill(ArmSmmu4KRange *r,size_t n,unsigned classes,uint64_t span)
{
for(size_t i=0;i<n;++i) {
const unsigned c=(unsigned)(rnd()%classes);
const uint64_t pages=(rnd()%8)?1:1+rnd()%64;
r[i]=(ArmSmmu4KRange){0x100000+i*0x100000,0x80000000+(rnd()%span)*4096,pages*4096,
(c&2)?3u|ArmSmmu4KPrivileged:3u,c&1u,(c&4u)?3u:2u};
}
}
static uint32_t scratch[1u<<18];
int main(void)
{
static ArmSmmu4KRange a[4096],b[4096];
ArmSmmu4KLiveArena with={0},without={0},small={0};
with.alias_scratch=scratch;with.alias_scratch_entries=sizeof scratch/sizeof *scratch;
small.alias_scratch=scratch;small.alias_scratch_entries=3;
unsigned cases=0,conflicts=0;
for(unsigned round=0;round<3000;++round) {
const size_t na=1+rnd()%(round<2000?40:600),nb=1+rnd()%(round<2000?40:600);
const unsigned classes=1+(unsigned)(rnd()%8);
const uint64_t span=1+rnd()%(round&1?200000:4000);
fill(a,na,classes,span);fill(b,nb,classes,span);
const int cross=pairwise(a,na,b,nb),self=pairwise(a,na,a,na);
CHECK(any_conflicting_alias(&with,a,na,b,nb)==cross && any_conflicting_alias(&with,b,nb,a,na)==cross);
CHECK(any_conflicting_alias(&without,a,na,b,nb)==cross && any_conflicting_alias(&small,a,na,b,nb)==cross);
CHECK(any_conflicting_alias(&with,a,na,a,na)==self && any_conflicting_alias(&small,a,na,a,na)==self);
cases+=5;conflicts+=(unsigned)(cross+self);
}
/* int-144's shape: 23855 coherent import pages and 82000 non-cacheable pages, disjoint, then one aliased page. */
enum {Imports=23855,Owned=82000};
ArmSmmu4KRange *all=calloc(Imports+Owned,sizeof *all);CHECK(all);
for(size_t i=0;i<Imports+Owned;++i) {
const int coherent=i<Imports;
all[i]=(ArmSmmu4KRange){0x100000+i*4096,(coherent?UINT64_C(0x800000000):UINT64_C(0x900000000))+(rnd()%0x100000)*4096*2+
(coherent?0:4096),4096,3,coherent?1u:0u,coherent?3u:2u};
}
const clock_t began=clock();
CHECK(!any_conflicting_alias(&with,all,Imports+Owned,all,Imports+Owned));
CHECK(!any_conflicting_alias(&with,all,Imports,all+Imports,Owned));
all[Imports+Owned-1].physical=all[17].physical;
CHECK(any_conflicting_alias(&with,all,Imports+Owned,all,Imports+Owned));
CHECK(any_conflicting_alias(&with,all+Imports+Owned-1,1,all,Imports));
const double ms=1000.0*(double)(clock()-began)/CLOCKS_PER_SEC;
CHECK(ms<2000);
free(all);
printf("PASS: %u sorted alias checks equal the pairwise definition (%u conflicting) with, without and with too little scratch; "
"a 23855+82000-page mixed-class set checked 4 times in %.0f ms\n",cases,conflicts,ms);
/* One long interval lies beyond every probe. Its length must not make each
* gap probe scan all preceding small intervals. Equal interval lengths in
* the earlier large case do not expose that degeneration. */
enum {Mixed=8192};
ArmSmmu4KRange *left=calloc(Mixed+1,sizeof *left),*right=calloc(Mixed+2,sizeof *right);CHECK(left && right);
for(size_t i=0;i<Mixed+2;++i) {
if(i<Mixed)left[i]=(ArmSmmu4KRange){0,0x40000000+i*8192,4096,3,0,2};
right[i]=(ArmSmmu4KRange){0,0x40001000+i*8192,4096,3,1,3};
}
left[Mixed]=(ArmSmmu4KRange){0,UINT64_C(0x200000000),UINT64_C(0x10000000),3,0,2};
#ifdef ARM_SMMU4K_TEST_COVERAGE
alias_steps=0;
#endif
const clock_t mixed_began=clock();
CHECK(!any_conflicting_alias(&with,left,Mixed+1,right,Mixed+2));
#ifdef ARM_SMMU4K_TEST_COVERAGE
const uint64_t work=alias_steps;
printf("mixed alias diagnostic: %u intervals, %llu coverage steps, %.1f host ms (not hardware timing)\n",2*Mixed+3,
(unsigned long long)work,1000.0*(double)(clock()-mixed_began)/CLOCKS_PER_SEC);fflush(stdout);
CHECK(work>0 && work<UINT64_C(512)*(2*Mixed+3));
#else
(void)mixed_began;
#endif
CHECK(!any_conflicting_alias(&with,right,Mixed+2,left,Mixed+1));
right[Mixed+1].physical=left[Mixed].physical+4096;
CHECK(any_conflicting_alias(&with,left,Mixed+1,right,Mixed+2));
free(left);free(right);
#ifdef ARM_SMMU4K_TEST_COVERAGE
puts("PASS: mixed-size alias gap/long-range controls and bounded compiler-coverage work");
#else
puts("PASS: mixed-size alias gap/long-range controls; work counter not enabled");
#endif
return 0;
}
kits/ArmSmmu4K/test_live.c
+92 -3
View File
@@ -22,6 +22,7 @@ typedef struct {
} Fixture;
static Fixture f;
static ArmSmmu4KLiveArena arena;
static uint32_t alias_scratch[64]; /* the main arena's alias checks sort; the peer's compare pairwise */
static ArmSmmu4KLiveState state;
static ArmSmmu4KContextConfig context;
static ArmSmmu4KIo io;
@@ -97,7 +98,7 @@ static void setup(unsigned bits,unsigned upper,unsigned output,unsigned spacing,
memset(&state,0,sizeof(state));memset(&f,0,sizeof(f));memset(tables,0xa5,sizeof(tables));
pair_mode=0;
arena=(ArmSmmu4KLiveArena){{bits,output,0x10000000,16},
(upper?UINT64_MAX-(UINT64_C(1)<<bits)+1:0)+low,4*4096,upper,tables,POOL_PAGES};
(upper?UINT64_MAX-(UINT64_C(1)<<bits)+1:0)+low,4*4096,upper,tables,POOL_PAGES,alias_scratch,64};
CHECK(ArmSmmu4KLivePrepare(&arena,&state)==ArmSmmu4KSuccess);
CHECK(state.prepared && !state.poisoned && !state.tables.leaf_pages && state.tables.table_pages<=POOL_PAGES);
ArmSmmu4KLiveResult audit;
@@ -140,7 +141,8 @@ static void complete(unsigned removed,size_t pages)
CHECK(result.complete && result.translation_removed==removed && !result.must_retain_resources);
CHECK(!state.poisoned && state.tables.leaf_pages==pages && result.synchronization.tlb_synchronized);
CHECK(f.tlbi_calls==1 && f.sync_calls==1 && f.synced && f.barriers>=4);
CHECK(result.descriptor_reads>=state.tables.table_pages*1024);
/* A whole-region update reads every descriptor; the others read each changed leaf's path before and after the store. */
CHECK(result.whole_verification?result.descriptor_reads>=state.tables.table_pages*1024:result.descriptor_reads>=2);
}
static unsigned profiles(void)
{
@@ -263,7 +265,7 @@ static void setup_pair(unsigned bits,unsigned output,unsigned spacing)
memset(&peer_state,0,sizeof(peer_state));
unsigned peer_bits=80-bits;
peer_arena=(ArmSmmu4KLiveArena){{peer_bits,output,0x11000000,16},
UINT64_MAX-(UINT64_C(1)<<peer_bits)+1+0x1ff000,4*4096,1,peer_tables,POOL_PAGES};
UINT64_MAX-(UINT64_C(1)<<peer_bits)+1+0x1ff000,4*4096,1,peer_tables,POOL_PAGES,0,0}; /* pairwise alias checks */
CHECK(!ArmSmmu4KLivePrepare(&peer_arena,&peer_state));
context.upper=&peer_arena.config;context.upper_tables=&peer_state.tables;
/* Explicitly reset the HOST REGISTER fixture, never a live hardware bank. */
@@ -352,6 +354,91 @@ static unsigned pair_failures(void)
CHECK(!state.poisoned && !peer_state.poisoned);
return controls;
}
static ArmSmmu4KStatus batch(const ArmSmmu4KRange *before,size_t count,const ArmSmmu4KRange *changes,size_t n,unsigned map)
{
ArmSmmu4KLiveRegion lo={&arena,&state,before,count};
return ArmSmmu4KLiveUpdateContextBatch(&io,0,&context,&lo,0,0,&visibility,changes,n,map,4,&result);
}
/* Four scattered physical pages (descending addresses, never adjacent) over the arena's 2 MiB boundary. */
static void scattered(ArmSmmu4KRange pages[4])
{for(unsigned i=0;i<4;++i)pages[i]=(ArmSmmu4KRange){arena.iova+4096*i,0x30000000+0x20000*(3-i),4096,3,0,2};}
static unsigned batches(void)
{
unsigned controls=0;
setup(48,0,48,0,0x1ff000);ArmSmmu4KRange pages[4];scattered(pages);
memcpy(snapshot,tables,sizeof(tables));
ArmSmmu4KRange swapped[2]={pages[1],pages[0]};
CHECK(batch(0,0,swapped,2,1)==ArmSmmu4KOverlap);++controls;
ArmSmmu4KRange alias[2]={pages[0],pages[1]};alias[1].physical=alias[0].physical;alias[1].writeback=1;alias[1].shareability=3;
CHECK(batch(0,0,alias,2,1)==ArmSmmu4KOverlap);++controls;
alias[1].writeback=0;alias[1].shareability=2;alias[1].permissions|=ArmSmmu4KPrivileged;
CHECK(batch(0,0,alias,2,1)==ArmSmmu4KOverlap);++controls;
CHECK(batch(0,0,pages,0,1)==ArmSmmu4KArgument);++controls;
CHECK(batch(0,0,0,1,1)==ArmSmmu4KBadRange);++controls;
CHECK(!f.calls && !state.poisoned && !memcmp(snapshot,tables,sizeof(tables)));
/* One transaction: four leaves written, two leaf pages published once each, one TLB synchronization. */
CHECK(batch(0,0,pages,4,1)==ArmSmmu4KSuccess);complete(0,4);
CHECK(result.descriptor_writes==4 && result.published_pages==2 && state.generation==1);
for(unsigned i=0;i<4;++i)CHECK(pte(pages[i].iova)==(pages[i].physical|source_permission_pte(3,0)|3));
++controls;
/* Any already mapped page refuses the whole batch before MMIO or a store. */
reset_counters();memcpy(snapshot,tables,sizeof(tables));
CHECK(batch(pages,4,pages+2,2,1)==ArmSmmu4KStateMismatch && !f.calls && !memcmp(snapshot,tables,sizeof(tables)));++controls;
/* Unmap a sub-batch given as part of the region's own ranges. */
reset_counters();CHECK(batch(pages,4,pages+1,2,0)==ArmSmmu4KSuccess);complete(1,2);
CHECK(pte(pages[0].iova) && !pte(pages[1].iova) && !pte(pages[2].iova) && pte(pages[3].iova));++controls;
ArmSmmu4KRange rest[2]={pages[0],pages[3]},wrong[2]={pages[0],pages[3]};wrong[1].physical+=4096;
reset_counters();CHECK(batch(rest,2,wrong,2,0)==ArmSmmu4KStateMismatch && !f.calls);++controls;
reset_counters();CHECK(batch(rest,2,rest,2,0)==ArmSmmu4KSuccess);complete(1,0);++controls;
/* Every register access of a batch failing, before or after its write, poisons and retains. */
setup(48,0,48,0,0x1ff000);scattered(pages);
CHECK(batch(0,0,pages,4,1)==ArmSmmu4KSuccess);size_t calls=f.calls;
for(unsigned unmap=0;unmap<2;++unmap)for(unsigned after=0;after<2;++after)for(size_t failed=1;failed<=calls;++failed) {
setup(48,0,48,0,0x1ff000);scattered(pages);
if(unmap) {CHECK(batch(0,0,pages,4,1)==ArmSmmu4KSuccess);reset_counters();}
f.fail_call=failed;f.after_write=after;
CHECK(batch(unmap?pages:0,unmap?4:0,pages,4,!unmap)==ArmSmmu4KIoError);
CHECK(state.poisoned && result.must_retain_resources && !result.complete);++controls;
}
return controls;
}
/* The leaf slot of va in the fixture's pool, by the oracle's own walk. */
static uint64_t *leaf_of(uint64_t va)
{
size_t at=0;unsigned level=state.tables.start_level;
for(;;) {
unsigned shift=12+9*(3-level),width=level==state.tables.start_level?arena.config.input_bits-shift:9;
uint64_t *entry=&tables[at+((va>>shift)&((1u<<width)-1))];
if(level==3)return entry;
CHECK((*entry&3)==3);
at=(size_t)(((*entry&UINT64_C(0x0000fffffffff000))-arena.config.table_physical)/8);++level;
}
}
/* Between whole-region updates (every 64th of a region) an update proves the leaves it changes: a changed leaf holding something
* else refuses before any store; a corrupted descriptor elsewhere is found by the next whole-region update. */
static unsigned incremental(void)
{
unsigned count=0;
setup(48,1,48,0,0x1ff000);
ArmSmmu4KRange r=resource(1,3,1),r2=r;r2.iova+=4096;r2.physical+=0x10000;
CHECK(update(0,0,&r,1)==ArmSmmu4KSuccess && result.whole_verification);
*leaf_of(r2.iova)=(r2.physical|0x703);reset_counters();
CHECK(update(&r,1,&r2,1)==ArmSmmu4KStateMismatch && !result.whole_verification);
CHECK(state.poisoned && !result.descriptor_writes && !f.writes && !result.complete);++count;
setup(48,1,48,0,0x1ff000);
CHECK(update(0,0,&r,1)==ArmSmmu4KSuccess);
const ArmSmmu4KRange both[2]={r,r2};
while(state.generation<63) {
reset_counters();CHECK(update(&r,1,&r2,1)==ArmSmmu4KSuccess && !result.whole_verification);complete(0,2);
CHECK(pte(r2.iova)==(r2.physical|source_permission_pte(3,1)|3));
reset_counters();CHECK(update(both,2,&r2,0)==ArmSmmu4KSuccess && !result.whole_verification);complete(1,1);
CHECK(!pte(r2.iova));
}
reset_counters();CHECK(update(&r,1,&r2,1)==ArmSmmu4KSuccess && !result.whole_verification && state.generation==64);
*leaf_of(r.iova+8*4096)=UINT64_C(0x40000703);reset_counters();
CHECK(update(both,2,&r2,0)==ArmSmmu4KStateMismatch && result.whole_verification && state.poisoned && !f.writes);++count;
return count;
}
int main(void)
{
unsigned p=profiles(),fcount=failures(),m=mutations();refusals_and_partial();
@@ -359,4 +446,6 @@ int main(void)
puts("Host fixtures only: no physical table provenance, cache-coherence, hardware TLB, GPU execution or fence qualification");
unsigned pairs=pair_profiles(),pair_faults=pair_failures();
printf("PASS: %u dual-region native4K profiles and %u dual-region failure/alias/isolation controls, both graphs audited and retained on uncertainty\n",pairs,pair_faults);
printf("PASS: %u batched scattered-page map/unmap transactions and refusals: one publication per leaf page, one TLB synchronization each\n",batches());
printf("PASS: %u incremental-verification controls: a changed leaf is proved before and after its store, the whole region every 64th update\n",incremental());
}
patches/mesa/turnip-builtin-shader-pack.patch
+456
View File
@@ -0,0 +1,456 @@
diff --git a/meson.options b/meson.options
index 680d08f..8b92a2b 100644
--- a/meson.options
+++ b/meson.options
@@ -914,3 +914,6 @@ option('offline-air-nir-dir', type : 'string', value : '',
description : 'Optional external AIR frontend source directory for offline tests')
option('offline-air-nir-llvm-config', type : 'string', value : '',
description : 'Explicit LLVM config executable for the offline AIR frontend')
+
+option('tu-builtin-source-id', type : 'string', value : '', description : 'Exact builtin compiler source and profile identity')
+option('tu-builtin-pack-header', type : 'string', value : '', description : 'Explicit generated builtin pack header; empty retains compiler path')
diff --git a/src/freedreno/vulkan/meson.build b/src/freedreno/vulkan/meson.build
index 762ecc8..b544a9f 100644
--- a/src/freedreno/vulkan/meson.build
+++ b/src/freedreno/vulkan/meson.build
@@ -139,7 +139,11 @@ endif
if freedreno_kmds.contains('darwin')
tu_flags += '-DTU_HAS_DARWIN'
- libtu_files += files('tu_knl_darwin.cc')
+ libtu_files += files('tu_knl_darwin.cc', 'tu_builtin_cache.cc')
+ tu_flags += '-DTU_BUILTIN_SOURCE_ID="' + get_option('tu-builtin-source-id') + '"'
+ if get_option('tu-builtin-pack-header') != ''
+ tu_flags += '-DTU_BUILTIN_PACK_HEADER="' + get_option('tu-builtin-pack-header') + '"'
+ endif
libtu_includes += include_directories(get_option('tu-darwin-abi-dir'), is_system : false)
tu_deps += dependency('appleframeworks', modules : ['IOKit', 'CoreGraphics', 'Foundation', 'QuartzCore'])
# The wsi layer's device entrypoint table references weak symbols that nothing defines; ld64 needs them allowed.
diff --git a/src/freedreno/vulkan/tu_builtin_cache.cc b/src/freedreno/vulkan/tu_builtin_cache.cc
new file mode 100644
index 0000000..1251f35
--- /dev/null
+++ b/src/freedreno/vulkan/tu_builtin_cache.cc
@@ -0,0 +1,259 @@
+/* SPDX-License-Identifier: MIT */
+/* Driver-owned, build-time embedded builtin variants. This is not a general
+ * untrusted shader import API. Pack generation must use this exact compiler
+ * source/profile and GPU, and the final driver must carry the normal owner
+ * signature. No disk I/O, cache thread, fake completion or shader execution. */
+#include "tu_builtin_cache.h"
+#include "tu_device.h"
+#include "tu_shader.h"
+#include "tu_cs.h"
+#include "ir3/ir3_compiler.h"
+#include "ir3/ir3_shader.h"
+#include "util/blob.h"
+#include "util/mesa-blake3.h"
+
+#ifdef TU_BUILTIN_PACK_HEADER
+#include TU_BUILTIN_PACK_HEADER
+#endif
+
+static constexpr unsigned variant_count = GLOBAL_SH_COUNT + 2;
+static constexpr size_t max_pack_size = 4 * 1024 * 1024;
+static constexpr size_t max_variant_size = 256 * 1024;
+
+static void
+identity(struct tu_device *dev, struct blob *out)
+{
+ blob_write_bytes(out, "TUBUILT1", 8);
+ blob_write_uint32(out, 1);
+ blob_write_uint32(out, variant_count);
+ blob_write_bytes(out, TU_BUILTIN_SOURCE_ID, strlen(TU_BUILTIN_SOURCE_ID));
+ blob_write_uint32(out, dev->physical_device->dev_id.gpu_id);
+ blob_write_uint64(out, dev->physical_device->dev_id.chip_id);
+ blob_write_uint64(out, ir3_shader_debug_hash_key());
+ blob_write_uint32(out, sizeof(struct ir3_shader_variant));
+ blob_write_uint32(out, VARIANT_CACHE_SIZE);
+ blob_write_uint32(out, sizeof(struct ir3_const_state));
+ blob_write_uint32(out, sizeof(struct ir3_shader_key));
+ const struct ir3_compiler_options *o = &dev->compiler->options;
+ uint32_t flags = o->push_ubo_with_preamble | (o->disable_cache << 1) |
+ (o->storage_16bit << 2) | (o->storage_8bit << 3) |
+ (o->lower_base_vertex << 4) | (o->shared_push_consts << 5) |
+ (o->dual_color_blend_by_location << 6);
+ blob_write_uint32(out, flags);
+ blob_write_uint32(out, o->bindless_fb_read_descriptor);
+ blob_write_uint32(out, o->bindless_fb_read_slot);
+ blob_write_uint64(out, o->uche_trap_base);
+}
+
+static bool
+available(const struct blob_reader *in, size_t bytes)
+{
+ return !in->overrun && bytes <= (size_t)(in->end - in->current);
+}
+
+/* Read only after bounding the record and every allocation. Unlike the
+ * general IR3 disk-cache reader, failure is checked before dereferencing an
+ * allocation, and no allocation size is taken from an unchecked short blob. */
+static struct ir3_shader_variant *
+read_variant(struct blob_reader *in, struct ir3_compiler *compiler, void *owner)
+{
+ if (!available(in, sizeof(struct ir3_shader_key) + 8))
+ return NULL;
+ struct ir3_shader_key key;
+ blob_copy_bytes(in, &key, sizeof(key));
+ const struct ir3_shader_key zero_key = {};
+ if (memcmp(&key, &zero_key, sizeof(key)))
+ return NULL;
+ const uint32_t stage = blob_read_uint32(in);
+ const uint32_t mergedregs = blob_read_uint32(in);
+ if ((stage != MESA_SHADER_VERTEX && stage != MESA_SHADER_FRAGMENT) || mergedregs > 1)
+ return NULL;
+ struct ir3_shader_variant *v = rzalloc(owner, struct ir3_shader_variant);
+ if (!v)
+ return NULL;
+ v->compiler = compiler;
+ v->key = key;
+ v->type = (mesa_shader_stage) stage;
+ v->mergedregs = mergedregs;
+ v->const_state = rzalloc(v, struct ir3_const_state);
+ if (!v->const_state)
+ return NULL;
+ unsigned count = stage == MESA_SHADER_VERTEX && ir3_has_binning_vs(&key) ? 2 : 1;
+ for (unsigned i = 0; i < count; ++i) {
+ struct ir3_shader_variant *part = v;
+ if (i) {
+ part = rzalloc(v, struct ir3_shader_variant);
+ if (!part)
+ return NULL;
+ v->binning = part;
+ part->compiler = compiler;
+ part->key = key;
+ part->type = MESA_SHADER_VERTEX;
+ part->mergedregs = mergedregs;
+ part->binning_pass = true;
+ part->nonbinning = v;
+ part->const_state = v->const_state;
+ }
+ if (!available(in, VARIANT_CACHE_SIZE))
+ return NULL;
+ blob_copy_bytes(in, VARIANT_CACHE_PTR(part), VARIANT_CACHE_SIZE);
+ const size_t size = part->info.size;
+ if (!size || size > max_variant_size || size % 4 ||
+ part->info.sizedwords * sizeof(uint32_t) > size ||
+ part->info.constant_data_offset > size || !available(in, size))
+ return NULL;
+ part->bin = (uint32_t *) ralloc_size(part, size);
+ if (!part->bin)
+ return NULL;
+ blob_copy_bytes(in, part->bin, size);
+ if (!i) {
+ if (!available(in, sizeof(*part->const_state)))
+ return NULL;
+ blob_copy_bytes(in, part->const_state, sizeof(*part->const_state));
+ }
+ if (!compiler->info->props.load_shader_consts_via_preamble) {
+ if (!available(in, 4))
+ return NULL;
+ unsigned n = blob_read_uint32(in);
+ if (n > max_variant_size / 4 || !available(in, (size_t)n * 4))
+ return NULL;
+ part->imm_state.size = part->imm_state.count = n;
+ if (n) {
+ part->imm_state.values = (uint32_t *) ralloc_size(part, (size_t)n * 4);
+ if (!part->imm_state.values)
+ return NULL;
+ blob_copy_bytes(in, part->imm_state.values, (size_t)n * 4);
+ }
+ }
+ }
+ return v;
+}
+
+static bool
+decode(struct tu_device *dev, const void *data, size_t size, void *owner,
+ struct ir3_shader_variant **variants)
+{
+ if (strlen(TU_BUILTIN_SOURCE_ID) != 64 || !data || size > max_pack_size || size < 32)
+ return false;
+ uint8_t digest[BLAKE3_OUT_LEN];
+ _mesa_blake3_compute(data, size - sizeof(digest), digest);
+ if (memcmp(digest, (const uint8_t *)data + size - sizeof(digest), sizeof(digest)))
+ return false;
+ struct blob header;
+ blob_init(&header);
+ identity(dev, &header);
+ const bool matches = !header.out_of_memory && header.size <= size - sizeof(digest) &&
+ !memcmp(header.data, data, header.size);
+ const size_t prefix = header.size;
+ blob_finish(&header);
+ if (!matches)
+ return false;
+ struct blob_reader in;
+ blob_reader_init(&in, (const uint8_t *)data + prefix, size - prefix - sizeof(digest));
+ for (unsigned i = 0; i < variant_count; ++i) {
+ if (!available(&in, 8) || blob_read_uint32(&in) != i)
+ return false;
+ const size_t length = blob_read_uint32(&in);
+ if (!length || length > max_variant_size || !available(&in, length))
+ return false;
+ struct blob_reader entry;
+ blob_reader_init(&entry, in.current, length);
+ variants[i] = read_variant(&entry, dev->compiler, owner);
+ if (!variants[i] || entry.overrun || entry.current != entry.end ||
+ variants[i]->type != (i < 3 ? MESA_SHADER_VERTEX : MESA_SHADER_FRAGMENT))
+ return false;
+ in.current += length;
+ }
+ uint64_t words = 0;
+ for (unsigned i = 0; i < GLOBAL_SH_COUNT; ++i)
+ words += align(variants[i]->info.sizedwords, 32);
+ return words <= ARRAY_SIZE(((struct tu6_global *)0)->shaders) &&
+ !in.overrun && in.current == in.end;
+}
+
+VkResult
+tu_builtin_pack_load(struct tu_device *dev)
+{
+#ifdef TU_BUILTIN_PACK_HEADER
+ void *owner = ralloc_context(NULL);
+ if (!owner)
+ return VK_ERROR_OUT_OF_HOST_MEMORY;
+ if (!decode(dev, tu_builtin_pack_bytes, sizeof(tu_builtin_pack_bytes), owner, dev->builtin_variants)) {
+ ralloc_free(owner);
+ memset(dev->builtin_variants, 0, sizeof(dev->builtin_variants));
+ /* A requested pack is mandatory: no silent runtime compiler fallback. */
+ return VK_ERROR_INITIALIZATION_FAILED;
+ }
+ dev->builtin_cache_owner = owner;
+#endif
+ return VK_SUCCESS;
+}
+
+/* Explicit offline tool interfaces. They never load a pack into a running
+ * device, replace a shader, submit commands, or change Vulkan feature reports. */
+extern "C" __attribute__((visibility("default"))) VkResult
+tu_export_builtin_shader_pack(VkDevice handle, size_t *size, void *data)
+{
+ VK_FROM_HANDLE(tu_device, dev, handle);
+ if (!dev || !size || strlen(TU_BUILTIN_SOURCE_ID) != 64)
+ return VK_ERROR_INITIALIZATION_FAILED;
+ struct blob out;
+ blob_init(&out);
+ identity(dev, &out);
+ for (unsigned i = 0; i < variant_count; ++i) {
+ const struct ir3_shader_variant *v = i < GLOBAL_SH_COUNT ? dev->global_shader_variants[i] :
+ (i == GLOBAL_SH_COUNT ? dev->empty_fs->variant : dev->empty_fs_fdm->variant);
+ if (!v) {
+ blob_finish(&out);
+ return VK_ERROR_INITIALIZATION_FAILED;
+ }
+ struct blob entry;
+ blob_init(&entry);
+ ir3_store_variant(&entry, v);
+ blob_write_uint32(&out, i);
+ blob_write_uint32(&out, entry.size);
+ blob_write_bytes(&out, entry.data, entry.size);
+ bool failed = entry.out_of_memory;
+ blob_finish(&entry);
+ if (failed) {
+ blob_finish(&out);
+ return VK_ERROR_OUT_OF_HOST_MEMORY;
+ }
+ }
+ if (out.out_of_memory || out.size > max_pack_size - BLAKE3_OUT_LEN) {
+ blob_finish(&out);
+ return VK_ERROR_OUT_OF_HOST_MEMORY;
+ }
+ uint8_t digest[BLAKE3_OUT_LEN];
+ _mesa_blake3_compute(out.data, out.size, digest);
+ blob_write_bytes(&out, digest, sizeof(digest));
+ if (out.out_of_memory || out.size > max_pack_size) {
+ blob_finish(&out);
+ return VK_ERROR_OUT_OF_HOST_MEMORY;
+ }
+ VkResult result = VK_SUCCESS;
+ if (data) {
+ if (*size < out.size)
+ result = VK_INCOMPLETE;
+ else
+ memcpy(data, out.data, out.size);
+ }
+ *size = out.size;
+ blob_finish(&out);
+ return result;
+}
+
+extern "C" __attribute__((visibility("default"))) VkResult
+tu_validate_builtin_shader_pack(VkDevice handle, size_t size, const void *data)
+{
+ VK_FROM_HANDLE(tu_device, dev, handle);
+ if (!dev)
+ return VK_ERROR_INITIALIZATION_FAILED;
+ void *owner = ralloc_context(NULL);
+ if (!owner)
+ return VK_ERROR_OUT_OF_HOST_MEMORY;
+ struct ir3_shader_variant *variants[variant_count] = {};
+ bool valid = decode(dev, data, size, owner, variants);
+ ralloc_free(owner);
+ return valid ? VK_SUCCESS : VK_ERROR_INITIALIZATION_FAILED;
+}
diff --git a/src/freedreno/vulkan/tu_builtin_cache.h b/src/freedreno/vulkan/tu_builtin_cache.h
new file mode 100644
index 0000000..74e97e6
--- /dev/null
+++ b/src/freedreno/vulkan/tu_builtin_cache.h
@@ -0,0 +1,8 @@
+/* SPDX-License-Identifier: MIT */
+#ifndef TU_BUILTIN_CACHE_H
+#define TU_BUILTIN_CACHE_H
+#include "tu_common.h"
+VkResult tu_builtin_pack_load(struct tu_device *dev);
+extern "C" VkResult tu_export_builtin_shader_pack(VkDevice device, size_t *size, void *data);
+extern "C" VkResult tu_validate_builtin_shader_pack(VkDevice device, size_t size, const void *data);
+#endif
diff --git a/src/freedreno/vulkan/tu_clear_blit.cc b/src/freedreno/vulkan/tu_clear_blit.cc
index 47a2fc6..1d33085 100644
--- a/src/freedreno/vulkan/tu_clear_blit.cc
+++ b/src/freedreno/vulkan/tu_clear_blit.cc
@@ -981,33 +981,39 @@ static void
compile_shader(struct tu_device *dev, struct nir_shader *nir,
unsigned consts, unsigned *offset, enum global_shader idx)
{
- nir->options = ir3_get_compiler_options(dev->compiler);
+ /* A provisioned variant bypasses NIR/compiler work while retaining the
+ * original global-BO upload, addresses and command emission below. */
+ struct ir3_shader_variant *so = dev->builtin_variants[idx];
+ struct ir3_shader *sh = NULL;
+ if (so) {
+ ralloc_free(nir);
+ } else {
+ nir->options = ir3_get_compiler_options(dev->compiler);
- nir_assign_io_var_locations(nir, nir_var_shader_in);
- nir_assign_io_var_locations(nir, nir_var_shader_out);
+ nir_assign_io_var_locations(nir, nir_var_shader_in);
+ nir_assign_io_var_locations(nir, nir_var_shader_out);
- struct ir3_const_allocations const_allocs = {};
- if (consts > 0)
- ir3_const_alloc(&const_allocs, IR3_CONST_ALLOC_UBO_RANGES, align(consts, 8), 1);
+ struct ir3_const_allocations const_allocs = {};
+ if (consts > 0)
+ ir3_const_alloc(&const_allocs, IR3_CONST_ALLOC_UBO_RANGES, align(consts, 8), 1);
- const struct ir3_shader_options options = {
- .api_wavesize = IR3_SINGLE_OR_DOUBLE,
- .real_wavesize = IR3_SINGLE_OR_DOUBLE,
- .const_allocs = const_allocs,
- .fragdata_dynamic_remap =
- idx >= GLOBAL_SH_VS_CLEAR && idx <= GLOBAL_SH_FS_CLEAR_MAX,
- };
+ const struct ir3_shader_options options = {
+ .api_wavesize = IR3_SINGLE_OR_DOUBLE,
+ .real_wavesize = IR3_SINGLE_OR_DOUBLE,
+ .const_allocs = const_allocs,
+ .fragdata_dynamic_remap =
+ idx >= GLOBAL_SH_VS_CLEAR && idx <= GLOBAL_SH_FS_CLEAR_MAX,
+ };
- ir3_finalize_nir(dev->compiler, &options.nir_options, nir);
+ ir3_finalize_nir(dev->compiler, &options.nir_options, nir);
- ir3_nir_lower_io(nir);
+ ir3_nir_lower_io(nir);
- struct ir3_shader *sh =
- ir3_shader_from_nir(dev->compiler, nir, &options);
+ sh = ir3_shader_from_nir(dev->compiler, nir, &options);
- struct ir3_shader_key key = {};
- struct ir3_shader_variant *so =
- ir3_shader_get_variant(sh, &key, false, false, NULL, NULL);
+ struct ir3_shader_key key = {};
+ so = ir3_shader_get_variant(sh, &key, false, false, NULL, NULL);
+ }
struct tu6_global *global = dev->global_bo_map;
@@ -1045,6 +1051,8 @@ tu_destroy_clear_blit_shaders(struct tu_device *dev)
if (dev->global_shaders[i])
ir3_shader_destroy(dev->global_shaders[i]);
}
+ ralloc_free(dev->builtin_cache_owner);
+ dev->builtin_cache_owner = NULL;
}
enum r3d_type {
diff --git a/src/freedreno/vulkan/tu_device.cc b/src/freedreno/vulkan/tu_device.cc
index 3b512f8..527a4d4 100644
--- a/src/freedreno/vulkan/tu_device.cc
+++ b/src/freedreno/vulkan/tu_device.cc
@@ -34,6 +34,7 @@
#include "fdl/freedreno_layout.h"
#include "tu_acceleration_structure.h"
#include "tu_clear_blit.h"
+#include "tu_builtin_cache.h"
#include "tu_cmd_buffer.h"
#include "tu_cs.h"
#include "tu_descriptor_set.h"
@@ -3049,6 +3050,11 @@ tu_CreateDevice(VkPhysicalDevice physicalDevice,
global = (struct tu6_global *)device->global_bo->map;
device->global_bo_map = global;
+#ifdef TU_HAS_DARWIN
+ result = tu_builtin_pack_load(device);
+ if (result != VK_SUCCESS)
+ goto fail_null_accel_struct;
+#endif
tu_init_clear_blit_shaders(device);
if (device->vk.enabled_features.accelerationStructure &&
diff --git a/src/freedreno/vulkan/tu_device.h b/src/freedreno/vulkan/tu_device.h
index ce1d494..356ab86 100644
--- a/src/freedreno/vulkan/tu_device.h
+++ b/src/freedreno/vulkan/tu_device.h
@@ -136,6 +136,8 @@ struct tu_physical_device
uint64_t va_size;
bool has_cached_coherent_memory;
+ /* The kernel pins host pages as BOs (VK_EXT_external_memory_host). */
+ bool has_host_import;
bool has_cached_non_coherent_memory;
/* Index for device local, host-coherent, host-cached memory.
* Only set to positive index when overwriting device-local, uncached memory
@@ -310,6 +312,8 @@ struct tu_device
int fd;
struct ir3_compiler *compiler;
+ void *builtin_cache_owner;
+ struct ir3_shader_variant *builtin_variants[GLOBAL_SH_COUNT + 2];
/* Backup in-memory cache to be used if the app doesn't provide one */
struct vk_pipeline_cache *mem_cache;
diff --git a/src/freedreno/vulkan/tu_shader.cc b/src/freedreno/vulkan/tu_shader.cc
index 8c228e9..d99d55c 100644
--- a/src/freedreno/vulkan/tu_shader.cc
+++ b/src/freedreno/vulkan/tu_shader.cc
@@ -3858,10 +3858,21 @@ tu_empty_fs_create(struct tu_device *dev, struct tu_shader **shader,
for (unsigned i = 0; i < MAX_SETS; i++)
(*shader)->dynamic_descriptor_sizes[i] = -1;
- struct ir3_shader *ir3_shader =
- ir3_shader_from_nir(dev->compiler, fs_b.shader, &options);
- (*shader)->variant = ir3_shader_create_variant(ir3_shader, &key, false);
- ir3_shader_destroy(ir3_shader);
+ struct ir3_shader_variant *cached =
+ dev->builtin_variants[GLOBAL_SH_COUNT + fragment_density_map];
+ if (cached) {
+ /* The shader object owns its variant independently of the global
+ * builtin context: normal shader destruction remains unchanged. */
+ ralloc_steal(NULL, cached);
+ (*shader)->variant = cached;
+ dev->builtin_variants[GLOBAL_SH_COUNT + fragment_density_map] = NULL;
+ ralloc_free(fs_b.shader);
+ } else {
+ struct ir3_shader *ir3_shader =
+ ir3_shader_from_nir(dev->compiler, fs_b.shader, &options);
+ (*shader)->variant = ir3_shader_create_variant(ir3_shader, &key, false);
+ ir3_shader_destroy(ir3_shader);
+ }
return tu_upload_shader(dev, *shader);
}
patches/mesa/turnip-darwin-kmd.patch
+187 -33
View File
@@ -170,6 +170,17 @@ index 82efb07..3b512f8 100644
VkResult
tu_allocate_lazy_memory(struct tu_device *dev,
struct tu_device_memory *mem)
diff --git a/src/freedreno/vulkan/tu_device.h b/src/freedreno/vulkan/tu_device.h
--- a/src/freedreno/vulkan/tu_device.h
+++ b/src/freedreno/vulkan/tu_device.h
@@ -300,6 +300,7 @@
{
struct vk_device vk;
struct tu_instance *instance;
+ void *darwin_retire; /* Darwin KMD's bounded deferred BO retirement state. */
struct tu_queue *queues[TU_MAX_QUEUE_FAMILIES];
int queue_count[TU_MAX_QUEUE_FAMILIES];
diff --git a/src/freedreno/vulkan/tu_knl.cc b/src/freedreno/vulkan/tu_knl.cc
index ab27798..9c545c6 100644
--- a/src/freedreno/vulkan/tu_knl.cc
@@ -257,10 +268,10 @@ index 4c2bf6a..a8ccd76 100644
VkResult tu_knl_drm_msm_load(struct tu_instance *instance,
diff --git a/src/freedreno/vulkan/tu_knl_darwin.cc b/src/freedreno/vulkan/tu_knl_darwin.cc
new file mode 100644
index 0000000..8601aaa
index 0000000..b03c6ee
--- /dev/null
+++ b/src/freedreno/vulkan/tu_knl_darwin.cc
@@ -0,0 +1,967 @@
@@ -0,0 +1,1110 @@
+/*
+ * Native Darwin kernel backend for Turnip: talks to the NativeAdreno IOKit user client (NativeAdrenoABI.h, version 1).
+ * Modelled on tu_knl_kgsl.cc (no libdrm) and tu_knl_drm_msm.cc (the client chooses GPU addresses).
@@ -315,6 +326,23 @@ index 0000000..8601aaa
+ NativeAdrenoAbiParams params;
+} dw = { .lock = PTHREAD_MUTEX_INITIALIZER };
+
+/* Preserve the native handle, CPU view and IOVA until the captured ring fence
+ * actually completed. An unrelated later submission never extends this lease.
+ * The queue is bounded; pressure may wait, but ordinary destruction does not. */
+enum { DARWIN_RETIRED_BOS = 256 };
+struct darwin_retired_bo {
+ uint64_t handle, iova, size;
+ void *mapping;
+ uint32_t token, fence;
+};
+struct darwin_retire_state {
+ pthread_mutex_t lock;
+ struct darwin_retired_bo entries[DARWIN_RETIRED_BOS];
+ unsigned head, count;
+ VkResult failure;
+ uint64_t enqueued, reaped, pressure_waits;
+};
+
+static kern_return_t
+dw_call(uint32_t method, const void *in, size_t in_size, void *out,
+ size_t out_size)
@@ -418,7 +446,7 @@ index 0000000..8601aaa
+
+/* Waits for a fence in short kernel waits so the connection lock is never held for long. */
+static VkResult
+dw_wait_fence_locked_free(struct tu_device *dev, uint32_t fence, uint64_t abs_timeout_ns)
+dw_wait_fence_raw(struct tu_device *dev, uint32_t fence, uint64_t abs_timeout_ns)
+{
+ for (;;) {
+ uint64_t now = os_time_get_nano();
@@ -438,6 +466,17 @@ index 0000000..8601aaa
+ }
+}
+
+static VkResult darwin_collect_retired(struct tu_device *dev);
+
+static VkResult
+dw_wait_fence_locked_free(struct tu_device *dev, uint32_t fence, uint64_t abs_timeout_ns)
+{
+ VkResult result = dw_wait_fence_raw(dev, fence, abs_timeout_ns);
+ /* Fence observation is also a retirement opportunity, so a quiet application
+ * need not allocate another BO to release completed deferred storage. */
+ return result == VK_SUCCESS ? darwin_collect_retired(dev) : result;
+}
+
+static VkResult
+darwin_queue_wait_fence(struct tu_queue *queue, uint32_t fence, uint64_t timeout_ns)
+{
@@ -494,6 +533,87 @@ index 0000000..8601aaa
+ mtx_unlock(&dev->vma_mutex);
+}
+
+/* Called under retire->lock. All fence polls are nonblocking. Failed/uncertain
+ * releases retain the record and the IOVA; a busy BO is retried by a later sweep. */
+static VkResult
+darwin_reap_locked(struct tu_device *dev, struct darwin_retire_state *retire)
+{
+ if (retire->failure != VK_SUCCESS)
+ return retire->failure;
+ /* Cache only within this sweep. Persisting a bare 32-bit fence across sweeps
+ * could mistake a later wrapped value for an old observed completion. */
+ uint32_t completed_fence = 0;
+ bool have_completed_fence = false;
+ while (retire->count) {
+ struct darwin_retired_bo *bo = &retire->entries[retire->head];
+ if (bo->fence && (!have_completed_fence || bo->fence != completed_fence)) {
+ VkResult result = dw_wait_fence_raw(dev, bo->fence, 0);
+ if (result == VK_TIMEOUT)
+ return VK_SUCCESS;
+ if (result != VK_SUCCESS)
+ return retire->failure = result;
+ have_completed_fence = true;
+ completed_fence = bo->fence;
+ }
+ if (bo->token) {
+ if (bo->mapping) {
+ kern_return_t kr = IOConnectUnmapMemory64(dw.conn, bo->token, mach_task_self(),
+ (mach_vm_address_t)(uintptr_t)bo->mapping);
+ if (kr != KERN_SUCCESS)
+ return retire->failure = vk_device_set_lost(&dev->vk, "deferred CPU unmap failed: 0x%x", kr);
+ bo->mapping = NULL;
+ }
+ NativeAdrenoAbiBoCpuUnmap undo = {.handle = bo->handle, .token = bo->token};
+ kern_return_t kr = dw_call(NativeAdrenoMethodBoCpuUnmap, &undo, sizeof(undo), NULL, 0);
+ if (kr != KERN_SUCCESS)
+ return retire->failure = vk_device_set_lost(&dev->vk, "deferred CPU lease release failed: 0x%x", kr);
+ bo->token = 0;
+ }
+ NativeAdrenoAbiBoFree release = {.handle = bo->handle};
+ kern_return_t kr = dw_call(NativeAdrenoMethodBoFree, &release, sizeof(release), NULL, 0);
+ if (kr == kIOReturnBusy)
+ return VK_SUCCESS;
+ if (kr != KERN_SUCCESS)
+ return retire->failure = vk_device_set_lost(&dev->vk, "deferred BoFree failed: 0x%x", kr);
+ darwin_free_iova(dev, bo->iova, bo->size);
+ *bo = {};
+ retire->head = (retire->head + 1) % DARWIN_RETIRED_BOS;
+ --retire->count;
+ ++retire->reaped;
+ }
+ return VK_SUCCESS;
+}
+
+static VkResult
+darwin_collect_retired(struct tu_device *dev)
+{
+ struct darwin_retire_state *retire = (struct darwin_retire_state *)dev->darwin_retire;
+ pthread_mutex_lock(&retire->lock);
+ VkResult result = darwin_reap_locked(dev, retire);
+ pthread_mutex_unlock(&retire->lock);
+ return result;
+}
+
+static VkResult
+darwin_map_failure(struct tu_device *dev, uint64_t handle, uint64_t iova,
+ uint64_t size, kern_return_t map_error)
+{
+ NativeAdrenoAbiBoFree release = {.handle = handle};
+ kern_return_t cleanup = dw_call(NativeAdrenoMethodBoFree, &release, sizeof(release), NULL, 0);
+ if (cleanup == KERN_SUCCESS) {
+ darwin_free_iova(dev, iova, size);
+ return vk_errorf(dev, VK_ERROR_OUT_OF_DEVICE_MEMORY, "BoMap failed: 0x%x", map_error);
+ }
+ /* A failed map may have modified tables. Unknown cleanup never authorizes
+ * reusing its IOVA, even though no tu_bo could be published to the caller. */
+ struct darwin_retire_state *retire = (struct darwin_retire_state *)dev->darwin_retire;
+ pthread_mutex_lock(&retire->lock);
+ retire->failure = vk_device_set_lost(&dev->vk, "BoMap cleanup failed: 0x%x; retaining native allocation and IOVA", cleanup);
+ VkResult result = retire->failure;
+ pthread_mutex_unlock(&retire->lock);
+ return result;
+}
+
+/* ---- buffer objects ---- */
+
+static VkResult
@@ -507,6 +627,10 @@ index 0000000..8601aaa
+ if (lazy_vma)
+ return vk_error(dev, VK_ERROR_FEATURE_NOT_PRESENT);
+
+ VkResult collected = darwin_collect_retired(dev);
+ if (collected != VK_SUCCESS)
+ return collected;
+
+ uint64_t iova;
+ VkResult result = darwin_allocate_iova(dev, size, client_iova, flags, &iova);
+ if (result != VK_SUCCESS)
@@ -530,10 +654,7 @@ index 0000000..8601aaa
+ NativeAdrenoAbiBoMapReply mapped = {};
+ kr = dw_call(NativeAdrenoMethodBoMap, &map, sizeof(map), &mapped, sizeof(mapped));
+ if (kr != KERN_SUCCESS) {
+ NativeAdrenoAbiBoFree release = {.handle = reply.handle};
+ dw_call(NativeAdrenoMethodBoFree, &release, sizeof(release), NULL, 0);
+ darwin_free_iova(dev, iova, size);
+ return vk_errorf(dev, VK_ERROR_OUT_OF_DEVICE_MEMORY, "BoMap failed: 0x%x", kr);
+ return darwin_map_failure(dev, reply.handle, iova, size, kr);
+ }
+
+ uint32_t gem_handle = (uint32_t)reply.handle;
@@ -572,6 +693,9 @@ index 0000000..8601aaa
+{
+ if (!(dw.params.flags & NativeAdrenoParamsHostImport))
+ return vk_error(dev, VK_ERROR_INVALID_EXTERNAL_HANDLE);
+ VkResult collected = darwin_collect_retired(dev);
+ if (collected != VK_SUCCESS)
+ return collected;
+ uint64_t iova;
+ VkResult result = darwin_allocate_iova(dev, size, 0, TU_BO_ALLOC_NO_FLAGS, &iova);
+ if (result != VK_SUCCESS)
@@ -588,10 +712,7 @@ index 0000000..8601aaa
+ NativeAdrenoAbiBoMapReply mapped = {};
+ kr = dw_call(NativeAdrenoMethodBoMap, &map, sizeof(map), &mapped, sizeof(mapped));
+ if (kr != KERN_SUCCESS) {
+ NativeAdrenoAbiBoFree release = {.handle = reply.handle};
+ dw_call(NativeAdrenoMethodBoFree, &release, sizeof(release), NULL, 0);
+ darwin_free_iova(dev, iova, size);
+ return vk_errorf(dev, VK_ERROR_OUT_OF_DEVICE_MEMORY, "BoMap of an import failed: 0x%x", kr);
+ return darwin_map_failure(dev, reply.handle, iova, size, kr);
+ }
+
+ uint32_t gem_handle = (uint32_t)reply.handle;
@@ -690,14 +811,15 @@ index 0000000..8601aaa
+
+/* Waits until every submission made so far has retired, so a freed buffer cannot still be in flight. The kernel also
+ * retires completed jobs itself; this only covers the window before the GPU passes the last fence. */
+static void
+static VkResult
+darwin_wait_idle(struct tu_device *dev)
+{
+ pthread_mutex_lock(&dw.lock);
+ uint32_t last = dw.last_fence;
+ pthread_mutex_unlock(&dw.lock);
+ if (last)
+ dw_wait_fence_locked_free(dev, last, os_time_get_nano() + 5000000000ull);
+ return dw_wait_fence_locked_free(dev, last, os_time_get_nano() + 5000000000ull);
+ return VK_SUCCESS;
+}
+
+static void
@@ -715,30 +837,38 @@ index 0000000..8601aaa
+ tu_debug_bos_del(dev, bo);
+ tu_dump_bo_del(dev, bo);
+
+ /* An import has no token: its mapping is the owner's. */
+ if (bo->map && bo->darwin_token) {
+ if (bo->map && bo->darwin_token)
+ TU_RMV(bo_unmap, dev, bo);
+ IOConnectUnmapMemory64(dw.conn, bo->darwin_token, mach_task_self(),
+ (mach_vm_address_t)(uintptr_t)bo->map);
+ NativeAdrenoAbiBoCpuUnmap undo = {.handle = bo->darwin_handle, .token = bo->darwin_token};
+ dw_call(NativeAdrenoMethodBoCpuUnmap, &undo, sizeof(undo), NULL, 0);
+ }
+
+ TU_RMV(bo_destroy, dev, bo);
+
+ darwin_wait_idle(dev);
+
+ NativeAdrenoAbiBoFree release = {.handle = bo->darwin_handle};
+ kern_return_t kr = dw_call(NativeAdrenoMethodBoFree, &release, sizeof(release), NULL, 0);
+ if (kr == kIOReturnBusy) {
+ /* A job still references it; give the CP a moment and retry once. */
+ darwin_wait_idle(dev);
+ kr = dw_call(NativeAdrenoMethodBoFree, &release, sizeof(release), NULL, 0);
+ struct darwin_retire_state *retire = (struct darwin_retire_state *)dev->darwin_retire;
+ pthread_mutex_lock(&retire->lock);
+ darwin_reap_locked(dev, retire);
+ if (retire->count == DARWIN_RETIRED_BOS && retire->failure == VK_SUCCESS) {
+ ++retire->pressure_waits;
+ pthread_mutex_unlock(&retire->lock);
+ VkResult waited = darwin_wait_idle(dev);
+ pthread_mutex_lock(&retire->lock);
+ if (waited != VK_SUCCESS)
+ retire->failure = vk_device_set_lost(&dev->vk, "deferred BO pressure wait failed");
+ else
+ darwin_reap_locked(dev, retire);
+ }
+ if (kr != KERN_SUCCESS)
+ mesa_loge("BoFree failed: 0x%x (the address range is leaked)", kr);
+ else
+ darwin_free_iova(dev, bo->iova, bo->size);
+ if (retire->count < DARWIN_RETIRED_BOS) {
+ pthread_mutex_lock(&dw.lock);
+ uint32_t fence = dw.last_fence;
+ pthread_mutex_unlock(&dw.lock);
+ unsigned slot = (retire->head + retire->count) % DARWIN_RETIRED_BOS;
+ retire->entries[slot] = {bo->darwin_handle, bo->iova, bo->size, bo->map, bo->darwin_token, fence};
+ ++retire->count;
+ ++retire->enqueued;
+ darwin_reap_locked(dev, retire);
+ } else {
+ /* No reusable range is reported on error. The kernel still owns the handle
+ * and view; connection teardown must drain them, even if this record cannot fit. */
+ retire->failure = vk_device_set_lost(&dev->vk, "deferred BO queue remains full; retaining native allocation");
+ }
+ pthread_mutex_unlock(&retire->lock);
+
+ /* The BO structs live in a sparse array: zero the entry to mark it free. */
+ memset(bo, 0, sizeof(*bo));
@@ -1057,6 +1187,18 @@ index 0000000..8601aaa
+ if (result != VK_SUCCESS)
+ return result;
+ }
+ struct darwin_retire_state *retire = (struct darwin_retire_state *)
+ vk_zalloc(&dev->vk.alloc, sizeof(*retire), 8, VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
+ if (!retire) {
+ dw_close_connection();
+ return vk_error(dev, VK_ERROR_OUT_OF_HOST_MEMORY);
+ }
+ if (pthread_mutex_init(&retire->lock, NULL)) {
+ vk_free(&dev->vk.alloc, retire);
+ dw_close_connection();
+ return vk_error(dev, VK_ERROR_INITIALIZATION_FAILED);
+ }
+ dev->darwin_retire = retire;
+ return VK_SUCCESS;
+}
+
@@ -1065,6 +1207,18 @@ index 0000000..8601aaa
+{
+ /* Closing the connection makes the kernel drain the GPU and free every object this client created. */
+ dw_close_connection();
+ struct darwin_retire_state *retire = (struct darwin_retire_state *)dev->darwin_retire;
+ if (retire) {
+ /* Some CreateDevice failure paths already destroyed the VMA heap/mutex.
+ * Never return ranges here: they are either gone or about to be destroyed. */
+ if (debug_get_bool_option("TU_DARWIN_RETIRE_TRACE", false))
+ mesa_logi("darwin BO retirement: enqueued=%llu reaped=%llu pending-at-close=%u pressure=%llu",
+ (unsigned long long)retire->enqueued, (unsigned long long)retire->reaped,
+ retire->count, (unsigned long long)retire->pressure_waits);
+ pthread_mutex_destroy(&retire->lock);
+ vk_free(&dev->vk.alloc, retire);
+ dev->darwin_retire = NULL;
+ }
+}
+
+static int
patches/mesa/turnip-empty-scissor.patch
+22
View File
@@ -0,0 +1,22 @@
diff --git a/src/freedreno/vulkan/tu_pipeline.cc b/src/freedreno/vulkan/tu_pipeline.cc
index dba43f9..8b010ed 100644
--- a/src/freedreno/vulkan/tu_pipeline.cc
+++ b/src/freedreno/vulkan/tu_pipeline.cc
@@ -2854,13 +2858,16 @@ template <chip CHIP>
static unsigned
tu6_scissor_size(struct tu_device *dev, const struct vk_viewport_state *vp)
{
- return 1 + vp->scissor_count * 2;
+ /* A pipeline without viewport state (rasterizer discard) has no scissors: a type-4 packet of zero registers is invalid. */
+ return vp->scissor_count ? 1 + vp->scissor_count * 2 : 0;
}
template <chip CHIP>
void
tu6_emit_scissor(struct tu_cs *cs, const struct vk_viewport_state *vp)
{
+ if (!vp->scissor_count)
+ return;
tu_cs_emit_pkt4(cs, GRAS_SC_SCREEN_SCISSOR_TL(CHIP, 0).reg, vp->scissor_count * 2);
for (uint32_t i = 0; i < vp->scissor_count; i++) {
patches/mesa/turnip-graphics-cache-feedback.patch
+25
View File
@@ -0,0 +1,25 @@
diff --git a/src/freedreno/vulkan/tu_pipeline.cc b/src/freedreno/vulkan/tu_pipeline.cc
--- a/src/freedreno/vulkan/tu_pipeline.cc
+++ b/src/freedreno/vulkan/tu_pipeline.cc
@@ -1958,7 +1958,10 @@
if (!executable_info) {
cache_hit = true;
- bool application_cache_hit = false;
+ /* Every required shader (and retained NIR, when requested) must come
+ * from the application's cache. Starting false makes this AND reduction
+ * report a miss even when compile-forbidden creation succeeds. */
+ bool application_cache_hit = true;
unsigned char shader_blake3[BLAKE3_KEY_LEN + 1];
memcpy(shader_blake3, pipeline_blake3, sizeof(pipeline_blake3));
@@ -1997,7 +2000,8 @@
cache_hit &= !!nir_shaders;
}
- if (application_cache_hit && builder->cache != builder->device->mem_cache) {
+ if (cache_hit && application_cache_hit &&
+ builder->cache != builder->device->mem_cache) {
pipeline_feedback.flags |=
VK_PIPELINE_CREATION_FEEDBACK_APPLICATION_PIPELINE_CACHE_HIT_BIT;
}
scripts/build-turnip-darwin.py
+43 -9
View File
@@ -5,17 +5,20 @@
cross-builds the ARM64/iOS 17 library. Neither runs a GPU: the backend talks to the NativeAdreno IOKit service, which
exists only on the tablet. Output stays under build/ and is never committed."""
import argparse
import hashlib
import json
import os
from pathlib import Path
import shutil
import subprocess
import sys
from mesa_patch_identity import validate_recorded_patches
ROOT = Path(__file__).resolve().parents[1]
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--target', choices=('host', 'ios'), default='host')
parser.add_argument('--configure-only', action='store_true')
parser.add_argument('--builtin-pack', type=Path, help='Embed an exact-source/profile/GPU-bound own builtin pack; no runtime compiler fallback')
args = parser.parse_args()
SOURCE = ROOT / '.cache/mesa-work'
OUT = ROOT / ('build/mesa-tu' if args.target == 'host' else 'build/mesa-tu-ios')
@@ -29,9 +32,7 @@ def read(cmd):
pins = json.loads((ROOT / 'scripts/data/adreno-compiler-inputs.json').read_text())
if read(['git', '-C', SOURCE, 'rev-parse', 'HEAD']) != pins['mesa']['commit']:
sys.exit('Pinned Mesa source differs')
for patch in pins['patches']:
if subprocess.run(['git', '-C', SOURCE, 'apply', '--reverse', '--check', ROOT / patch], capture_output=True).returncode:
sys.exit('Recorded patch is not applied to the Mesa tree: ' + patch)
validate_recorded_patches(ROOT, SOURCE, pins)
if not shutil.which('glslangValidator'):
sys.exit('glslangValidator is required by the Vulkan runtime build (brew install glslang)')
@@ -57,16 +58,49 @@ if args.target == 'ios':
''.join('%s = %r\n' % (k, flags) for k in ('c_args', 'cpp_args', 'objc_args', 'c_link_args', 'cpp_link_args',
'objc_link_args')))
setup += ['--cross-file', cross]
# Release with NDEBUG for the tablet: Mesa's default debugoptimized keeps assertions and NIR validation after
# every pass, which the tablet's pipeline makings paid for (int-143: 58 pipeline states took 68 s). The host
# build keeps them for the simulation checks.
options += ['-Dlibunwind=disabled', '-Dlmsensors=disabled', '-Dvalgrind=disabled', '-Dexpat=disabled',
'-Dzstd=disabled']
'-Dzstd=disabled', '-Dbuildtype=release', '-Db_ndebug=true']
env.update(CC=find('clang'), CXX=find('clang++'), CFLAGS='', CXXFLAGS='', CPPFLAGS='', LDFLAGS='')
else:
env.update(CC='/usr/bin/clang', CXX='/usr/bin/clang++')
# Bind raw variant layout and code to every actual source edit, untracked
# source addition, compiler, target and build option. The generated embedded
# header stays outside SOURCE and is deliberately excluded from this key;
# otherwise the pack would make its own producer identity circular.
source_diff = subprocess.check_output(['git', '-C', SOURCE, 'diff', 'HEAD', '--binary'])
untracked = subprocess.check_output(['git', '-C', SOURCE, 'ls-files', '--others', '--exclude-standard', '-z']).split(b'\0')
source_extras = {name.decode(): hashlib.sha256((SOURCE / name.decode()).read_bytes()).hexdigest()
for name in untracked if name and (SOURCE / name.decode()).is_file()}
profile = dict(mesa_revision=pins['mesa']['commit'], target=args.target,
source_diff_sha256=hashlib.sha256(source_diff).hexdigest(), extra_sources=source_extras,
patch_guard_sha256=hashlib.sha256((ROOT / 'scripts/mesa_patch_identity.py').read_bytes()).hexdigest(),
compiler=read([env['CXX'], '--version']), options=options,
build_script_sha256=hashlib.sha256(Path(__file__).read_bytes()).hexdigest())
source_id = hashlib.sha256(json.dumps(profile, sort_keys=True, separators=(',', ':')).encode()).hexdigest()
OUT.mkdir(parents=True, exist_ok=True)
pack_header = ''
pack_sha = None
if args.builtin_pack:
pack = args.builtin_pack.resolve().read_bytes()
if not 112 <= len(pack) <= 4 * 1024**2 or pack[:8] != b'TUBUILT1' or pack[16:80] != source_id.encode():
sys.exit('Builtin pack is not from this exact compiler source/profile; regenerate rather than relabel it')
header = OUT / 'tu_builtin_pack.generated.h'
header.write_text('/* Generated own compiler output: never commit. */\nstatic const unsigned char tu_builtin_pack_bytes[] = {\n' +
''.join(','.join('0x%02x' % b for b in pack[i:i+16]) + ',\n' for i in range(0,len(pack),16)) + '};\n')
pack_header = str(header)
pack_sha = hashlib.sha256(pack).hexdigest()
(OUT / 'builtin-build-profile.json').write_text(json.dumps(dict(source_id=source_id, profile=profile,
embedded_pack_sha256=pack_sha, builtin_pack_required=bool(args.builtin_pack)), indent=2) + '\n')
options += ['-Dtu-builtin-source-id=' + source_id, '-Dtu-builtin-pack-header=' + pack_header]
log = LOGS / ('mesa-tu-%s-configure.log' % args.target)
if not (OUT / 'build.ninja').exists():
with log.open('w') as stream:
if subprocess.run([str(c) for c in setup + options], env=env, stdout=stream, stderr=subprocess.STDOUT).returncode:
sys.exit('Configure failed; see ' + str(log) + '\n' + '\n'.join(log.read_text().splitlines()[-12:]))
with log.open('w') as stream:
# An existing build directory takes the current options (meson configure), so an option added here reaches it.
command = setup + options if not (OUT / 'build.ninja').exists() else [ROOT / '.cache/mesa-venv/bin/meson', 'configure', OUT] + options
if subprocess.run([str(c) for c in command], env=env, stdout=stream, stderr=subprocess.STDOUT).returncode:
sys.exit('Configure failed; see ' + str(log) + '\n' + '\n'.join(log.read_text().splitlines()[-12:]))
if args.configure_only:
print('PASS: Turnip darwin-kmd configuration (%s)' % args.target)
sys.exit(0)
@@ -75,7 +109,7 @@ with build_log.open('w') as stream:
result = subprocess.run(['ninja', '-C', str(OUT), '-j6'], env=env, stdout=stream, stderr=subprocess.STDOUT)
if result.returncode:
sys.exit('Build failed; see ' + str(build_log) + '\n' + '\n'.join(
l for l in build_log.read_text().splitlines() if 'error' in l.lower())[:3000])
l for l in build_log.read_text().splitlines() if ': error:' in l or 'fatal error:' in l or l.startswith('FAILED:'))[:3000])
library = OUT / 'src/freedreno/vulkan/libvulkan_freedreno.dylib'
symbols = subprocess.run(['nm', '-gU', str(library)], capture_output=True, text=True).stdout
if ' T _vk_icdGetInstanceProcAddr' not in symbols:
scripts/check-priming-cache-host.py
+178
View File
@@ -0,0 +1,178 @@
#!/usr/bin/env python3
"""Qualify real device-priming cache hits while all compilation is forbidden.
Actual Turnip/IR3 plus the mandatory software transport; no shader execution,
physical GPU, or deadline claim. Preserve exact source/driver/cache identities.
"""
import argparse
import hashlib
import json
import os
from pathlib import Path
import re
import shutil
import subprocess
ROOT = Path(__file__).resolve().parents[1]
KIT = ROOT / 'drivers/NativeMetal'
OUT = ROOT / 'build/priming-cache'
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--output', type=Path, required=True)
parser.add_argument('--baseline', type=Path, help='Optional exact preserved pre-fix executable for a before/after refusal control')
parser.add_argument('--cross-build-ios', action='store_true', help='Link/sign the same instrument for iOS; never run or stage hardware')
args = parser.parse_args()
work = args.output.resolve()
def require(value, reason):
if not value:
raise SystemExit(reason)
def sha(path):
return hashlib.sha256(path.read_bytes()).hexdigest()
require(not work.exists(), 'Preserve the existing controls; choose a fresh output directory')
require(shutil.disk_usage(ROOT).free > 5 * 1024**3 + 256 * 1024**2, 'Preserve 5 GiB and bounded allowance')
OUT.mkdir(exist_ok=True)
work.mkdir(parents=True)
loader, program = OUT / 'libOfflinePrimingCacheControl.dylib', OUT / 'priming_cache_host'
bridge = ROOT / 'build/native-metal-bridge/host/libnative_metal_bridge.a'
driver = ROOT / 'build/mesa-tu/src/freedreno/vulkan/libvulkan_freedreno.dylib'
peer = ROOT / 'build/turnip-darwin-host/libhostiokit.dylib'
flags = ['xcrun', 'clang++', '-O1', '-std=c++17', '-Wall', '-Wextra', '-Werror']
subprocess.run([*flags, '-dynamiclib', '-I', ROOT / '.cache/mesa-work/include', KIT / 'bridge/priming_cache_loader.cpp', '-o', loader], check=True, timeout=60)
subprocess.run([*flags, KIT / 'bridge/priming_cache_host.cpp', bridge, '-Wl,-dead_strip', '-Wl,-S',
'-framework', 'Foundation', '-framework', 'CoreFoundation', '-framework', 'IOKit', '-o', program], check=True, timeout=90)
ios_link = None
if args.cross_build_ios:
import plistlib
import struct
target = OUT / 'ios'
target.mkdir(exist_ok=True)
sdk = subprocess.check_output(['xcrun', '--sdk', 'iphoneos', '--show-sdk-path'], text=True, timeout=10).strip()
ios_bridge = ROOT / 'build/native-metal-bridge/ios/libnative_metal_bridge.a'
peer_dir = ROOT / 'build/turnip-ios-compiler-peer'
ios_peer = peer_dir / 'libOfflineTurnipCompilerPeer.dylib'
peer_build = json.loads((peer_dir / 'build-report.json').read_text())
require(sha(ios_peer) == peer_build['outputs'][ios_peer.name]['sha256'], 'Exact own signed software peer')
for name, digest in peer_build['sources'].items():
require(sha(ROOT / name) == digest, 'Exact software peer source: ' + name)
policy = peer_dir / 'compiler-entitlements.plist'
require(plistlib.loads(policy.read_bytes()) == {'com.apple.private.security.no-container': True,
'com.apple.private.bindfs-allow': True}, 'Exact own offline permissions; no no-sandbox')
ios_flags = [*flags, '-target', 'arm64-apple-ios17.0', '-isysroot', sdk, '-fstack-protector-strong']
ios_loader, worker = target / loader.name, target / 'compiler-worker'
subprocess.run([*ios_flags, '-dynamiclib', '-install_name', '@rpath/' + loader.name, '-I', ROOT / '.cache/mesa-work/include',
KIT / 'bridge/priming_cache_loader.cpp', '-o', ios_loader], check=True, timeout=60)
subprocess.run([*ios_flags, KIT / 'bridge/priming_cache_host.cpp', ios_bridge, '-Wl,-dead_strip', '-Wl,-S',
'-Wl,-needed_library,' + str(ios_peer), '-Wl,-rpath,/System/Library/Extensions/OfflineTurnipCompiler',
'-framework', 'Foundation', '-framework', 'CoreFoundation', '-framework', 'IOKit', '-o', worker], check=True, timeout=120)
outputs = {}
for path in [ios_loader, worker]:
command = ['codesign', '--force', '--sign', '-', '--pagesize', '4096', '--identifier', 'org.blu-sharky.offline-priming-cache.' + path.stem]
if path == worker:
command += ['--entitlements', str(policy)]
subprocess.run([*command, str(path)], check=True, timeout=30)
subprocess.run(['codesign', '--verify', '--strict', str(path)], check=True, capture_output=True, timeout=30)
detail = subprocess.run(['codesign', '--display', '--verbose=4', str(path)], capture_output=True, text=True, check=True, timeout=30).stderr
cdhash = re.search(r'^CDHash=([a-f0-9]{40})$', detail, re.M)
raw = path.read_bytes()
require(struct.unpack_from('<3I', raw) == (0xfeedfacf, 0x100000c, 0) and cdhash and re.search(r'^Page size=4096$', detail, re.M),
'Actual own ordinary ARM64 4K CodeDirectory')
cursor, builds, ids = 32, [], []
for _ in range(struct.unpack_from('<I', raw, 16)[0]):
command, size = struct.unpack_from('<2I', raw, cursor)
require(size >= 8 and cursor + size <= len(raw), 'Bounded actual Mach-O')
if command == 0x32:
builds.append(struct.unpack_from('<2I', raw, cursor + 8))
if command == 0x1b:
ids.append(raw[cursor + 8:cursor + 24].hex())
cursor += size
require(builds == [(2, 0x110000)] and len(ids) == 1, 'Actual iOS17 build/UUID')
outputs[path.name] = dict(sha256=sha(path), bytes=len(raw), cdhash=cdhash[1], macho_uuid=ids[0], platform=2, cpu_subtype=0, signature_page_bytes=4096)
ios_link = dict(evidence='cross-link/signature verification only, not iOS execution', outputs=outputs,
bridge_sha256=sha(ios_bridge), peer_sha256=sha(ios_peer), peer_report_sha256=sha(peer_dir / 'build-report.json'))
env = {k: v for k, v in os.environ.items() if not k.startswith(('IR3_', 'FD_MESA_', 'NIR_', 'TU_', 'HOSTSIM_', 'MESA_', 'NMB_TEST_'))}
env.update(DYLD_INSERT_LIBRARIES=str(peer), NMB_TEST_PRIMING_CACHE_LOADER=str(loader))
cache = work / 'control.cache'
pattern = re.compile(r'^PASS: device priming (\S+) pid=(\d+) compute=(\d+)/(\d+)/(\d+)/(\d+)/(\d+) graphics=(\d+)/(\d+)/(\d+)/(\d+)/(\d+) uuid_mutations=(\d+) submissions=(\d+) live=0; software peer, no shader execution$', re.M)
fields = ['pid', 'compute_calls', 'compute_success', 'compute_required', 'compute_valid', 'compute_hits',
'graphics_calls', 'graphics_success', 'graphics_required', 'graphics_valid', 'graphics_hits', 'uuid_mutations', 'submissions']
cases = {}
def case(label, mode, directory):
run = subprocess.run([str(program), str(driver), str(directory), mode], env=env, capture_output=True, text=True, timeout=60)
log = work / (label + '.log')
log.write_text(run.stdout + run.stderr)
row = pattern.search(run.stdout)
require(run.returncode == 0 and row and row[1] == mode, 'Actual priming control failed: ' + label)
data = dict(zip(fields, map(int, row.groups()[1:])))
require(data['submissions'] == 1 and data['compute_calls'] == 1, 'Full priming still submits its barriers; observed non-generated compute shape')
if mode in ('cold-miss', 'uuid-miss'):
require(data['compute_success'] == 0 and data['compute_required'] == 1 and data['graphics_calls'] == 0,
'Actual cold/invalid priming fails instead of silently compiling')
if mode == 'uuid-miss':
require(data['uuid_mutations'] > 0, 'Actual driver cache UUID refusal control')
else:
require(data['compute_success'] == data['compute_valid'] == data['graphics_calls'] == data['graphics_success'] == data['graphics_valid'] == 1,
'Both actual helper creations and their feedback are valid')
if mode == 'consume':
require(data['compute_hits'] == data['graphics_hits'] == 1 and data['compute_required'] == data['graphics_required'] == 0,
'Both helpers truly hit with compilation forbidden')
data['log_sha256'] = sha(log)
cases[label] = data
print(row[0], flush=True)
case('cold-miss', 'cold-miss', cache)
case('produce', 'produce', cache)
records = [p for p in cache.rglob('*') if p.is_file()]
require(len(records) == 1 and records[0].stat().st_size < 16 * 1024**2, 'Exactly one bounded native priming record')
record = records[0]
original = record.read_bytes()
require(original.startswith(b'M2VPPLC\0') and hashlib.sha256(original[:-32]).digest() == original[-32:], 'Actual driver record and checksum')
case('consume', 'consume', cache)
case('uuid-miss', 'uuid-miss', cache)
require(record.read_bytes() == original, 'Warm and UUID controls preserve producer record bytes')
for label in ['truncated-miss', 'checksum-miss']:
directory = work / (label + '.cache')
shutil.copytree(cache, directory)
bad = directory / record.relative_to(cache)
if label == 'truncated-miss':
bad.write_bytes(original[:20])
else:
data = bytearray(original);data[-33] ^= 1;bad.write_bytes(data)
case(label, 'cold-miss', directory)
require(len({v['pid'] for v in cases.values()}) == len(cases), 'Independent cold processes for all controls')
baseline_evidence = None
if args.baseline:
baseline = args.baseline.resolve()
baseline_report = ROOT / 'build/ui-shader-coverage/turnip-graphics/committed-qualified/checkpoint.json'
old = json.loads(baseline_report.read_text())
require(baseline.is_file() and sha(baseline) == old['process']['sha256'], 'Exact previously qualified executable with uncached priming')
old_env = dict(env, NMB_TEST_GRAPHICS_CACHE_LOADER=str(loader), NMB_TEST_FORCE_PRIME_STRICT='1')
run = subprocess.run([str(baseline), str(driver), str(work / 'control'), 'consume'], env=old_env, capture_output=True, text=True, timeout=60)
log = work / 'baseline.log'
log.write_text(run.stdout + run.stderr)
require(run.returncode != 0 and 'device: Vulkan(VkResultCode(1000297000))' in log.read_text(),
'Old executable must still fail during device open, using this exact populated cache directory')
require(record.read_bytes() == original, 'The old refusal also preserves all producer record bytes')
baseline_evidence = dict(process_sha256=sha(baseline), log_sha256=sha(log), exit_code=run.returncode)
patches = KIT / 'bridge/patches'
report = dict(evidence='real host Turnip compiler, mandatory software peer; no shader execution', checker_sha256=sha(Path(__file__)),
sources_sha256={str(p.relative_to(ROOT)): sha(p) for p in [KIT / 'bridge/priming_cache_host.cpp', KIT / 'bridge/priming_cache_loader.cpp', KIT / 'bridge/graphics_cache_loader.cpp']},
runtime_base_revision=subprocess.check_output(['git', '-C', str(ROOT / '.cache/gpu-reference/metal2vulkan'), 'rev-parse', 'HEAD'], text=True).strip(),
runtime_patches_sha256={p.name: sha(p) for p in sorted(patches.glob('*.patch'))}, host_bridge_sha256=sha(bridge),
driver_sha256=sha(driver), peer_sha256=sha(peer), ios_cross_link=ios_link, process_sha256=sha(program), loader_sha256=sha(loader),
cache_record_path=str(record.relative_to(cache)), cache_record_sha256=sha(record), cache_record_bytes=len(original),
driver_cache_uuid=original[20:36].hex(), cases=cases, baseline=baseline_evidence,
exit_guard_priming_preserved=True, generated_command_priming_qualified=False, ios_execution=False, shader_execution=False,
physical_device_tested=False, active_native_adreno_4k=False, complete_ui_variant_coverage=False, hardware_ready=False, stable_120_fps_qualified=False)
(work / 'checkpoint.json').write_text(json.dumps(report, indent=2) + '\n')
(OUT / 'checkpoint.json').write_text(json.dumps(report, indent=2) + '\n')
print('PASS: both priming helpers hit with compilation forbidden; invalid data refuses and leaves no live software objects')
if baseline_evidence:
print('PASS: preserved pre-fix executable still fails in device open with the same populated cache')
scripts/check-turnip-pipeline-cache-host.py
+82
View File
@@ -0,0 +1,82 @@
#!/usr/bin/env python3
"""Qualify Turnip cache import without executing a shader or contacting hardware.
Requires the built host Turnip and the software peer from check-turnip-darwin-host.py.
Each cache case is a fresh process with Mesa's disk shader cache disabled. The
consumer must succeed under FAIL_ON_PIPELINE_COMPILE_REQUIRED and report an
application cache hit. This does not qualify cross-build cache portability.
"""
import hashlib
import json
import os
from pathlib import Path
import re
import struct
import subprocess
import tempfile
ROOT = Path(__file__).resolve().parents[1]
OUT = ROOT / 'build/turnip-darwin-host'
LIBRARY = ROOT / 'build/mesa-tu/src/freedreno/vulkan/libvulkan_freedreno.dylib'
IOS = ROOT / 'build/mesa-tu-ios/src/freedreno/vulkan/libvulkan_freedreno.dylib'
SHIM = OUT / 'libhostiokit.dylib'
SHADERS = ROOT / 'build/turnip-smoke/NativeTurnipShaders.h'
for required in [LIBRARY, SHIM, SHADERS]:
if not required.is_file():
raise SystemExit('Build the host Turnip/peer/own shader fixtures first: ' + str(required))
def binary_identity(path):
data = path.read_bytes()
if struct.unpack_from('<I', data)[0] != 0xfeedfacf:
raise SystemExit('Expected a thin little-endian Mach-O64: ' + str(path))
count, size = struct.unpack_from('<II', data, 16)
offset, end = 32, 32 + size
if end > len(data):
raise SystemExit('Truncated Mach-O load commands')
cpu, subtype = struct.unpack_from('<II', data, 4)
if cpu != 0x100000c or subtype != 0:
raise SystemExit('Expected ordinary ARM64 subtype 0: ' + str(path))
result = dict(sha256=hashlib.sha256(data).hexdigest(), cpu_type=cpu, cpu_subtype=subtype)
for _ in range(count):
command, length = struct.unpack_from('<II', data, offset)
if length < 8 or offset + length > end:
raise SystemExit('Invalid Mach-O load command')
if command == 0x1b and length == 24: # LC_UUID
result['macho_uuid'] = data[offset + 8:offset + 24].hex()
if command == 0x32 and length >= 24: # LC_BUILD_VERSION
result['platform'] = struct.unpack_from('<I', data, offset + 8)[0]
offset += length
if offset != end or 'macho_uuid' not in result:
raise SystemExit('Missing/invalid Mach-O UUID')
return result
executable = OUT / 'pipeline_cache'
subprocess.run(['xcrun', 'clang++', '-std=c++23', '-O1', '-Wall', '-Wextra', '-Werror',
'-Wno-unused-parameter', '-Wno-unused-const-variable', '-I', '/opt/homebrew/include',
'-I', SHADERS.parent, ROOT / 'drivers/NativeAdreno/tests/hostsim/pipeline_cache.cpp',
'-o', executable], check=True)
env = {k: v for k, v in os.environ.items() if not k.startswith(('IR3_', 'FD_MESA_', 'NIR_', 'TU_', 'HOSTSIM_', 'MESA_'))}
env.update(DYLD_INSERT_LIBRARIES=str(SHIM), MESA_SHADER_CACHE_DISABLE='true', MESA_LOG='')
report = dict(evidence='host Turnip compilation/cache import over software kernel transport; no shader execution',
host=binary_identity(LIBRARY),
own_shader_header_sha256=hashlib.sha256(SHADERS.read_bytes()).hexdigest(),
cross_platform_portability_qualified=False, cases={})
if IOS.is_file():
report['ios_static'] = binary_identity(IOS)
report['host_ios_build_uuid_differ'] = report['host']['macho_uuid'] != report['ios_static']['macho_uuid']
with tempfile.TemporaryDirectory(prefix='pipeline-cache-', dir=OUT) as temporary:
for mode in ['produce', 'consume', 'uuid-miss', 'shader-miss']:
run = subprocess.run([str(executable), str(LIBRARY), str(Path(temporary) / 'compute.bin'), mode],
env=env, text=True, capture_output=True, timeout=120)
log = OUT / ('pipeline-cache-' + mode + '.log')
log.write_text(run.stdout + run.stderr)
match = re.search(r'^PASS: Turnip pipeline cache (\S+); uuid=([0-9a-f]{32}) bytes=(\d+) strict=(\d+) feedback=0x([0-9a-f]+) strict_us=([\d.]+) compile_us=([\d.]+); no shader execution$', run.stdout, re.M)
if run.returncode or not match or match[1] != mode:
raise SystemExit('Turnip pipeline cache %s failed; see %s' % (mode, log))
report['cases'][mode] = dict(pipeline_cache_uuid=match[2], bytes=int(match[3]), strict_result=int(match[4]),
feedback_flags=int(match[5], 16), strict_us=float(match[6]), compile_us=float(match[7]))
print(match[0], flush=True)
(OUT / 'pipeline-cache-report.json').write_text(json.dumps(report, indent=2) + '\n')
print('PASS: separate-process Turnip cache hit and UUID/shader miss controls; host only, cross-platform portability unqualified')
scripts/data/adreno-compiler-inputs.json
+4 -1
View File
@@ -49,7 +49,10 @@
"patches/mesa/ir3-darwin-build-id.patch",
"patches/mesa/ir3-portable-asm-roundtrip.patch",
"patches/mesa/offline-adreno-compiler-probe.patch",
"patches/mesa/turnip-darwin-kmd.patch"
"patches/mesa/turnip-darwin-kmd.patch",
"patches/mesa/turnip-graphics-cache-feedback.patch",
"patches/mesa/turnip-empty-scissor.patch",
"patches/mesa/turnip-builtin-shader-pack.patch"
],
"integer_fma_reference": {
"repository": "https://github.com/ucb-bar/berkeley-softfloat-3",
scripts/mesa_patch_identity.py
+38
View File
@@ -0,0 +1,38 @@
# SPDX-License-Identifier: MIT
"""Verify overlapping recorded Mesa patches without mutating the source tree."""
from pathlib import Path
import re
import subprocess
import tempfile
def validate_recorded_patches(root, source, pins):
paths = set()
for patch in pins['patches']:
text = (root / patch).read_text()
paths.update(re.findall(r'^diff --git a/(\S+) b/\1$', text, re.M))
changed = set(subprocess.check_output(['git', '-C', source, 'diff', 'HEAD', '--name-only'], text=True).splitlines())
changed.update(subprocess.check_output(['git', '-C', source, 'ls-files', '--others', '--exclude-standard'], text=True).splitlines())
if changed - paths:
raise SystemExit('Unrecorded Mesa source changes: ' + ', '.join(sorted(changed - paths)))
with tempfile.TemporaryDirectory(prefix='mesa-patch-check-', dir=root / '.cache') as name:
stripped = Path(name)
subprocess.run(['git', 'init', '-q', str(stripped)], check=True, capture_output=True, timeout=20)
for path in paths:
actual = source / path
if actual.is_file():
if actual.is_symlink():
raise SystemExit('Unexpected source symlink: ' + path)
destination = stripped / path
destination.parent.mkdir(parents=True, exist_ok=True)
destination.write_bytes(actual.read_bytes())
for patch in reversed(pins['patches']):
result = subprocess.run(['git', 'apply', '--reverse', str(root / patch)], cwd=stripped,
capture_output=True, text=True, timeout=20)
if result.returncode:
raise SystemExit('Recorded Mesa patch sequence differs: ' + patch + '\n' + result.stderr)
for path in paths:
original = subprocess.run(['git', '-C', source, 'show', 'HEAD:' + path], capture_output=True, timeout=20)
actual = stripped / path
if (original.returncode == 0 and (not actual.is_file() or actual.read_bytes() != original.stdout)) or (original.returncode != 0 and actual.exists()):
raise SystemExit('Recorded Mesa patches do not reconstruct pinned source: ' + path)
scripts/probe-turnip-internal-compiler-host.py
+120
View File
@@ -0,0 +1,120 @@
#!/usr/bin/env python3
"""Count real IR3 backend entries hidden below public pipeline cache feedback.
Unchanged actual host Turnip plus the mandatory software transport. LLDB stops
at entries and immediately resumes; no shader executes and timing is unqualified.
The canonical driver, original inputs and prior qualified packs stay unchanged.
"""
import argparse
from collections import Counter
import hashlib
import json
import os
from pathlib import Path
import re
import shlex
import shutil
import subprocess
ROOT = Path(__file__).resolve().parents[1]
def sha(path):
return hashlib.sha256(path.read_bytes()).hexdigest()
def require(condition, reason):
if not condition:
raise SystemExit(reason)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--output', type=Path, required=True)
parser.add_argument('--qualification', type=Path, required=True,
help='Exact passing check-priming-cache-host output directory')
args = parser.parse_args()
output, qualification = args.output.resolve(), args.qualification.resolve()
require(not output.exists(), 'Preserve existing evidence; choose a fresh directory')
require(shutil.disk_usage(ROOT).free > 5 * 1024**3 + 32 * 1024**2, 'Preserve 5 GiB and bounded logs')
previous_path = qualification / 'checkpoint.json'
previous = json.loads(previous_path.read_text())
paths = dict(program='build/priming-cache/priming_cache_host',
driver='build/mesa-tu/src/freedreno/vulkan/libvulkan_freedreno.dylib',
peer='build/turnip-darwin-host/libhostiokit.dylib',
loader='build/priming-cache/libOfflinePrimingCacheControl.dylib')
# This is an additional observation of an exact previously qualified pair.
# Never silently use a newly rebuilt compiler, wrapper or executable.
for name, digest in previous['sources_sha256'].items():
require(sha(ROOT / name) == digest, 'Exact qualified source: ' + name)
require(sha(ROOT / paths['driver']) == previous['driver_sha256'], 'Exact previously qualified host compiler')
for name, digest in previous['runtime_patches_sha256'].items():
require(sha(ROOT / 'drivers/NativeMetal/bridge/patches' / name) == digest, 'Exact qualified runtime patch: ' + name)
for name, field in [('program', 'process_sha256'), ('peer', 'peer_sha256'), ('loader', 'loader_sha256')]:
require(sha(ROOT / paths[name]) == previous[field], 'Exact qualified binary: ' + name)
require(sha(ROOT / 'build/native-metal-bridge/host/libnative_metal_bridge.a') == previous['host_bridge_sha256'], 'Exact linked runtime input')
output.mkdir(parents=True)
identities = {name: sha(ROOT / path) for name, path in paths.items()}
cases = {}
cache = output / 'cache'
produced = None
for mode in ('cold-miss', 'produce', 'consume'):
work = output / mode
work.mkdir()
config = dict(root=str(ROOT), work=str(work), cache=str(cache), mode=mode, **paths)
config_path = work / 'config.json'
config_path.write_text(json.dumps(config, indent=2) + '\n')
commands = work / 'commands.lldb'
commands.write_text('command script import ' + shlex.quote(str(ROOT / 'scripts/turnip_compiler_lldb.py')) +
'\nscript turnip_compiler_lldb.run(lldb.debugger)\n')
env = dict(os.environ, NMB_OFFLINE_COMPILER_CONFIG=str(config_path))
run = subprocess.run(['xcrun', 'lldb', '--batch', '--source', str(commands)],
capture_output=True, text=True, env=env, timeout=60)
(work / 'debugger.log').write_text(run.stdout + run.stderr)
require(run.returncode == 0, 'Bounded own debugger failed: ' + mode)
trace_path = work / 'trace.json'
trace = json.loads(trace_path.read_text())
require(trace['launch_success'] and trace['exited'] and trace['exit_status'] == 0 and
trace['breakpoint_locations'] == 1 and not trace['trace_overflow'], 'Actual complete compiler trace: ' + mode)
text = (work / 'worker.stdout').read_text()
pattern = (r'^PASS: device priming ' + mode + r' pid=\d+ compute=(\d+)/(\d+)/(\d+)/(\d+)/(\d+) '
r'graphics=(\d+)/(\d+)/(\d+)/(\d+)/(\d+) uuid_mutations=0 submissions=1 live=0; software peer, no shader execution$')
row = re.search(pattern, text, re.M)
require(row, 'Complete actual worker feedback: ' + mode)
c, g = tuple(map(int, row.groups()[:5])), tuple(map(int, row.groups()[5:]))
require((c, g) == (((1, 0, 1, 0, 0), (0, 0, 0, 0, 0)) if mode == 'cold-miss' else
((1, 1, 0, 1, int(mode == 'consume')), (1, 1, 0, 1, int(mode == 'consume')))),
'Actual empty refusal/producer/strict-consumer feedback: ' + mode)
counts = Counter(entry['scope'] for entry in trace['rows'])
require(counts['clear-blit'] == 18 and counts['empty'] == 2, 'Exact observed internal startup compiler calls')
require(all(entry['stack'][0] == 'ir3_compile_shader_nir' and
any('tu_CreateDevice(' in s for s in entry['stack']) for entry in trace['rows'] if entry['scope'] != 'other'),
'Actual IR3 entries occur inside original device initialization')
require(counts['other'] == 0 if mode != 'produce' else counts['other'] > 0,
'Strict public-pipeline refusal/hits distinguish actual helper backend compilation')
records = {str(p.relative_to(cache)): dict(bytes=p.stat().st_size, sha256=sha(p))
for p in cache.rglob('*') if p.is_file()}
if mode == 'cold-miss':
require(not records, 'Cold refusal never writes a successful helper cache')
elif mode == 'produce':
require(len(records) == 1, 'Exactly one bounded own priming record')
produced = records
else:
require(records == produced, 'Strict consumer leaves every producer-cache byte unchanged')
cases[mode] = dict(compiler_entries=len(trace['rows']), scopes=dict(counts), compute=c, graphics=g,
trace_sha256=sha(trace_path), worker_stdout_sha256=sha(work / 'worker.stdout'))
print(mode, dict(counts), flush=True)
require(identities == {name: sha(ROOT / path) for name, path in paths.items()}, 'Canonical compiler/fixture binaries unchanged')
report = dict(evidence='actual host Turnip IR3 backend entries under LLDB; software peer executes no shader',
qualification_sha256=sha(previous_path), identities=identities, cases=cases, cache_manifest=produced,
checker_sha256=sha(Path(__file__)), callbacks_sha256=sha(ROOT / 'scripts/turnip_compiler_lldb.py'),
mesa_base_revision=subprocess.check_output(['git', '-C', ROOT / '.cache/mesa-work', 'rev-parse', 'HEAD'], text=True).strip(),
patches_sha256={p: sha(ROOT / p) for p in json.loads((ROOT / 'scripts/data/adreno-compiler-inputs.json').read_text())['patches']},
timing_qualified=False, shader_execution=False, physical_device_tested=False,
active_native_adreno_4k=False, hardware_ready=False, stable_120_fps_qualified=False)
(output / 'checkpoint.json').write_text(json.dumps(report, indent=2) + '\n')
print('PASS: real internal compilation persists despite strict public-pipeline hits; no timing/hardware qualification')
if __name__ == '__main__':
main()
scripts/save-turnip-builtin-patch.py
+38
View File
@@ -0,0 +1,38 @@
#!/usr/bin/env python3
"""Record only the builtin-cache change over the other pinned Mesa patches."""
import json
from pathlib import Path
import shutil
import subprocess
import tempfile
ROOT = Path(__file__).resolve().parents[1]
MESA = ROOT / '.cache/mesa-work'
OUT = ROOT / 'patches/mesa/turnip-builtin-shader-pack.patch'
FILES = ['meson.options', 'src/freedreno/vulkan/meson.build', 'src/freedreno/vulkan/tu_device.cc',
'src/freedreno/vulkan/tu_device.h', 'src/freedreno/vulkan/tu_shader.cc',
'src/freedreno/vulkan/tu_clear_blit.cc']
NEW = ['src/freedreno/vulkan/tu_builtin_cache.cc', 'src/freedreno/vulkan/tu_builtin_cache.h']
def git(directory, *args):
return subprocess.check_output(['git', '-c', 'user.email=a@b', '-c', 'user.name=n', *args], cwd=directory)
with tempfile.TemporaryDirectory(dir=ROOT / '.cache', prefix='builtin-patch-') as name:
tmp = Path(name)
git(tmp, 'init', '-q')
for path in FILES:
(tmp / path).parent.mkdir(parents=True, exist_ok=True)
(tmp / path).write_bytes(git(MESA, 'show', 'HEAD:' + path))
for patch in json.loads((ROOT / 'scripts/data/adreno-compiler-inputs.json').read_text())['patches']:
if patch != str(OUT.relative_to(ROOT)):
git(tmp, 'apply', *['--include=' + path for path in FILES], str(ROOT / patch))
git(tmp, 'add', '-A')
git(tmp, 'commit', '-q', '-m', 'recorded-base')
for path in FILES + NEW:
(tmp / path).parent.mkdir(parents=True, exist_ok=True)
shutil.copyfile(MESA / path, tmp / path)
git(tmp, 'add', '-N', *NEW)
OUT.write_bytes(git(tmp, 'diff', '--binary'))
print('Recorded builtin-cache source patch')
scripts/turnip_compiler_lldb.py
+55
View File
@@ -0,0 +1,55 @@
# SPDX-License-Identifier: MIT
"""Bounded debugger callbacks for the own host-only compiler probe.
No patched instruction, changed argument, forced return, hardware connection,
or shader execution. Breakpoints perturb timing; only invocation counts qualify.
"""
import json
import os
from pathlib import Path
import lldb
ROWS = []
LIMIT = 256
def entered(frame, location, internal_dict):
if len(ROWS) == LIMIT:
return True
thread = frame.GetThread()
stack = [thread.GetFrameAtIndex(i).GetFunctionName() or '?'
for i in range(min(thread.GetNumFrames(), 32))]
scope = ('clear-blit' if any('tu_init_clear_blit_shaders' in s for s in stack)
else 'empty' if any('tu_init_empty_shaders' in s for s in stack)
else 'other')
ROWS.append(dict(scope=scope, stack=stack))
return False
def run(debugger):
config = json.loads(Path(os.environ['NMB_OFFLINE_COMPILER_CONFIG']).read_text())
root, work = Path(config['root']), Path(config['work'])
debugger.SetAsync(False)
target = debugger.CreateTarget(str(root / config['program']))
breakpoint = target.BreakpointCreateByName('ir3_compile_shader_nir', 'libvulkan_freedreno.dylib')
breakpoint.SetScriptCallbackFunction(__name__ + '.entered')
launch = lldb.SBLaunchInfo([str(root / config['driver']), config['cache'], config['mode']])
environment = {k: v for k, v in os.environ.items() if not k.startswith(
('IR3_', 'FD_MESA_', 'NIR_', 'TU_', 'HOSTSIM_', 'MESA_', 'NMB_TEST_'))}
environment.update(DYLD_INSERT_LIBRARIES=str(root / config['peer']),
NMB_TEST_PRIMING_CACHE_LOADER=str(root / config['loader']))
launch.SetEnvironmentEntries([k + '=' + v for k, v in environment.items()], True)
launch.AddOpenFileAction(1, str(work / 'worker.stdout'), False, True)
launch.AddOpenFileAction(2, str(work / 'worker.stderr'), False, True)
error = lldb.SBError()
process = target.Launch(launch, error)
exited = process.GetState() == lldb.eStateExited
result = dict(launch_success=error.Success(), exited=exited,
exit_status=process.GetExitStatus() if exited else None,
breakpoint_locations=breakpoint.GetNumLocations(),
rows=ROWS, trace_limit=LIMIT, trace_overflow=len(ROWS) == LIMIT,
timing_qualified=False, physical_device_tested=False, shader_execution=False)
if not exited:
process.Kill()
(work / 'trace.json').write_text(json.dumps(result, indent=2) + '\n')
print('Recorded %d actual backend entries; timing is unqualified' % len(ROWS))