From 66abbc3d301ef5bbe81a293e6c524ffd7ebd2e35 Mon Sep 17 00:00:00 2001 From: seedds Date: Sun, 25 Oct 2026 11:31:14 +0800 Subject: [PATCH] Cache Metal 4 pipeline bases with per-game archives Adapt PR #122: compile reusable bases with unspecialized colour state, specialize per draw state, and persist bases between launches. Keep iOS opt-in and the MTLDevice fallback. Preserve rasterizing depth-only pipelines through the legacy path. Share in-progress base builds, including errors, while unrelated bases compile in parallel. Defer timed flushes during compaction and finish flush/cleanup together; synchronize the pending count with serializer capture. Checked on Apple M4 Pro: t_pso_archive 48 checks with Metal validation, t_pso_pending 48, t_integer_blend 63, t_geometry 95, and t_pso_skey pass. The Breath of the Wild Mac host builds and nvn_mtl.m passes the iOS arm64 syntax check. Real-game memory usage and iPhone runtime behavior were not measured. Co-authored-by: v0lumehi --- dm/gfx/README.md | 3 + dm/gfx/nvn/nvn_mtl.m | 263 ++++++++++++++++++++++++++++++++--- dm/gfx/tests/.gitignore | 2 + dm/gfx/tests/Makefile | 5 +- dm/gfx/tests/t_pso_archive.m | 220 +++++++++++++++++++++++++++++ docs/development.md | 13 ++ 6 files changed, 486 insertions(+), 20 deletions(-) create mode 100644 dm/gfx/tests/t_pso_archive.m diff --git a/dm/gfx/README.md b/dm/gfx/README.md index 99b1ce7..cbf50d3 100644 --- a/dm/gfx/README.md +++ b/dm/gfx/README.md @@ -20,6 +20,7 @@ For the build steps of the whole project see [README.md](../../README.md) and [d - Memory pools are created as zero-copy `MTLBuffer`s, with the GPU address equal to the CPU address, so only unified-memory devices are supported. - The visible effects of "GPU completion" (sync objects becoming ready, events writing their value, counters) happen only after the Metal command buffer has actually completed. - Pipeline states and depth-stencil states are cached by state hash; at startup, shaders and pipelines are pre-built in the background according to the warm-up manifests (the bundled seed `pso_manifest.tsv`, then this device's `pso_user.tsv`; see below). A pipeline that fails to build is cached as failed and reported once with its shader keys. + - Render pipelines are made through `MTL4Compiler` from a base whose colour state is left unspecialized, one base per pipeline less its colour state, specialized per colour state; the bases are kept in the per-game pipeline archive (`pso_archive/` beside `pso_user.tsv`) and taken from it at the next start (`pso_build`; see docs/development.md, "Pipeline bases and the pipeline archive"). - The game's packaged textures are in Tegra block-linear layout and are deswizzled before upload (see `tex/README.md`). - Presentation: the game's swapchain texture is drawn to a `CAMetalLayer`, and presentation timing is scheduled by the pacer in `nvn/nvn_pace.h`. @@ -150,6 +151,8 @@ Resource locations: | `DM_PSO_MANIFEST=path` or `off` | Use a different warm-up manifest, or turn warm-up off | | `DM_PSO_RECORD=path` | Appends the shaders and pipelines used in this run to a file, for re-recording the warm-up manifest | | `DM_PSO_USER=path` | Where to keep the per-device list of shaders and pipelines built on the spot (default: `pso_user.tsv` in the parent of `DM_SAVE`) | +| `DM_PSO_ARCHIVE=folder` or `off` | Where to keep the per-game pipeline archive of pipeline bases (default: `pso_archive/` beside `pso_user.tsv`), or no archive | +| `DM_PSO_MTL4=0` / `=1` | `0`: build render pipelines through the `MTLDevice` as before, with no unspecialized bases and no archive (the Mac's default is on). `1`: turn them on on iOS, where they are off by default | Logging and capture: diff --git a/dm/gfx/nvn/nvn_mtl.m b/dm/gfx/nvn/nvn_mtl.m index cf59a56..b2777c7 100644 --- a/dm/gfx/nvn/nvn_mtl.m +++ b/dm/gfx/nvn/nvn_mtl.m @@ -3045,6 +3045,27 @@ void mtl_set_depth_zero(int on) { int n = atomic_load(&g_vs_loaded); if (n) MLOG("[RED] depth mode changed after %d vertex shaders were loaded: they keep the old convention", n); } +// The Metal 4 function descriptor of each function pipelines are built from (pso_build): MTL4Compiler takes a library, an entry name and the constant values a function was +// specialized with, not an MTLFunction, so every function made from a library is noted here as it is made. A function with no note (the diagnostic fragment overrides +// DM_NVN_HILITE and DM_NVN_MAGSCAN) keeps the MTLDevice path +static os_unfair_lock g_fn4_lock = OS_UNFAIR_LOCK_INIT; +static NSMapTable *g_fn4; // id → MTL4FunctionDescriptor +static id fn4_note(id f, id lib, NSString *name, MTLFunctionConstantValues *cv) { + if (!f || !lib || !name) return f; + MTL4LibraryFunctionDescriptor *ld = [MTL4LibraryFunctionDescriptor new]; ld.library = lib; ld.name = name; + MTL4FunctionDescriptor *d = ld; + if (cv) { MTL4SpecializedFunctionDescriptor *sd = [MTL4SpecializedFunctionDescriptor new]; sd.functionDescriptor = ld; sd.constantValues = cv; d = sd; } + os_unfair_lock_lock(&g_fn4_lock); + if (!g_fn4) g_fn4 = [[NSMapTable alloc] initWithKeyOptions:NSPointerFunctionsStrongMemory | NSPointerFunctionsObjectPointerPersonality valueOptions:NSPointerFunctionsStrongMemory capacity:0]; // by identity: a function made again from the same library and name (other constants: bf_fn) is another function + [g_fn4 setObject:d forKey:f]; + os_unfair_lock_unlock(&g_fn4_lock); + return f; +} +static MTL4FunctionDescriptor *fn4_of(id f) { + if (!f) return nil; + os_unfair_lock_lock(&g_fn4_lock); MTL4FunctionDescriptor *d = [g_fn4 objectForKey:f]; os_unfair_lock_unlock(&g_fn4_lock); + return d; +} // layer: a vertex shader's layer entry points (entry_layer / entry_layer_z, which also write the layer) instead of main0 / main0_z, for a program whose geometry shader is a slice writer (prog_fold_gs) static NvShader *shader_load(NSString *pc, int stage, int layer) { NSError *e = nil; @@ -3076,15 +3097,15 @@ static NvShader *shader_load(NSString *pc, int stage, int layer) { if (layer) entry = atomic_load(&g_depth_zero) ? j[@"entry_layer_z"] : j[@"entry_layer"]; } if (j[@"entry_geometry_capture"]) { - NSError *ge = nil; - o->geo_fn = [lib newFunctionWithName:j[@"entry_geometry_capture"] constantValues:[MTLFunctionConstantValues new] error:&ge]; + NSError *ge = nil; MTLFunctionConstantValues *gcv = [MTLFunctionConstantValues new]; + o->geo_fn = fn4_note([lib newFunctionWithName:j[@"entry_geometry_capture"] constantValues:gcv error:&ge], lib, j[@"entry_geometry_capture"], gcv); if (!o->geo_fn) MLOG("[RED] vertex shader %s: geometry capture entry failed: %s", pc.UTF8String, ge.localizedDescription.UTF8String); } NSDictionary *geo = j[@"geometry"]; if (j[@"geometry_reader"]) o->geo_output = [j[@"geometry_output"] unsignedIntValue]; if (geo) { o->geo_vertices = [geo[@"vertices"] unsignedIntValue]; o->geo_invocations = [geo[@"invocations"] unsignedIntValue]; o->geo_output = [geo[@"output"] unsignedIntValue]; o->geo_stride = [geo[@"stride"] unsignedIntValue]; o->geo_index_stride = [geo[@"index_stride"] unsignedIntValue]; } - o->fn = [lib newFunctionWithName:entry]; o->stage = stage; strlcpy(o->key, pc.UTF8String, sizeof o->key); o->ventry = entry; + o->fn = fn4_note([lib newFunctionWithName:entry], lib, entry, nil); o->stage = stage; strlcpy(o->key, pc.UTF8String, sizeof o->key); o->ventry = entry; // writes the layer (render_target_array_index, named in the library's metadata): its passes are layered and its pipelines get an input class (pso_desc). A library with layer entry points // names it for those alone: its main0 / main0_z write no layer if (stage == 0) o->layer_out = layer || (t_lib_layer && !j[@"entry_layer"]); @@ -3092,18 +3113,18 @@ static NvShader *shader_load(NSString *pc, int stage, int layer) { // texel buffer (spvLinearTextureAlignmentOverride, used only through is_function_constant_defined; one of Pokémon Legends: Z-A's imgui pixel shaders, 2bd5037003980716): an empty set of // values keeps its default. Without it the first pipeline with that shader failed Metal's validation (`fragmentFunction main0 cannot be used to build a pipeline state`) if (j[@"entry_tf"]) { // transform feedback capture entry point (dmsh_build.py vs_tf) and the bytes per vertex of each feedback buffer - o->tf_fn = [lib newFunctionWithName:j[@"entry_tf"]]; NSArray *ts = j[@"tf_strides"]; + o->tf_fn = fn4_note([lib newFunctionWithName:j[@"entry_tf"]], lib, j[@"entry_tf"], nil); NSArray *ts = j[@"tf_strides"]; for (NSUInteger b = 0; b < 4 && b < ts.count; b++) o->tf_stride[b] = (uint16_t)[ts[b] unsignedIntValue]; if (!o->tf_fn) MLOG("[RED] shader %s: no transform feedback entry point %s in its library", pc.UTF8String, [j[@"entry_tf"] UTF8String]); } - if (stage == 1 && j[@"entry_u"]) { o->fn_u = [lib newFunctionWithName:j[@"entry_u"]]; if (!o->fn_u) MLOG("[RED] shader %s: no integer-output entry point %s in its library", pc.UTF8String, [j[@"entry_u"] UTF8String]); } - if (stage == 1 && j[@"entry_m"]) { o->fn_m = [lib newFunctionWithName:j[@"entry_m"]]; o->mask_m = (uint8_t)[j[@"uint_mask_m"] unsignedIntValue]; + if (stage == 1 && j[@"entry_u"]) { o->fn_u = fn4_note([lib newFunctionWithName:j[@"entry_u"]], lib, j[@"entry_u"], nil); if (!o->fn_u) MLOG("[RED] shader %s: no integer-output entry point %s in its library", pc.UTF8String, [j[@"entry_u"] UTF8String]); } + if (stage == 1 && j[@"entry_m"]) { o->fn_m = fn4_note([lib newFunctionWithName:j[@"entry_m"]], lib, j[@"entry_m"], nil); o->mask_m = (uint8_t)[j[@"uint_mask_m"] unsignedIntValue]; if (!o->fn_m) MLOG("[RED] shader %s: no mixed integer-output entry point %s in its library", pc.UTF8String, [j[@"entry_m"] UTF8String]); } if (o->fn.functionConstantsDictionary.count) { - NSError *fe = nil; id f = [lib newFunctionWithName:entry constantValues:[MTLFunctionConstantValues new] error:&fe]; + NSError *fe = nil; MTLFunctionConstantValues *cv = [MTLFunctionConstantValues new]; id f = fn4_note([lib newFunctionWithName:entry constantValues:cv error:&fe], lib, entry, cv); if (f) o->fn = f; else MLOG("[RED] shader %s: specializing its function constants failed: %s", pc.UTF8String, fe.localizedDescription.UTF8String); - if (o->fn_u) { f = [lib newFunctionWithName:j[@"entry_u"] constantValues:[MTLFunctionConstantValues new] error:&fe]; o->fn_u = f; } - if (o->fn_m) { f = [lib newFunctionWithName:j[@"entry_m"] constantValues:[MTLFunctionConstantValues new] error:&fe]; o->fn_m = f; } + if (o->fn_u) { f = fn4_note([lib newFunctionWithName:j[@"entry_u"] constantValues:cv error:&fe], lib, j[@"entry_u"], cv); o->fn_u = f; } + if (o->fn_m) { f = fn4_note([lib newFunctionWithName:j[@"entry_m"] constantValues:cv error:&fe], lib, j[@"entry_m"], cv); o->fn_m = f; } } if (o->rf.nb) { o->ae = [o->fn newArgumentEncoderWithBufferIndex:o->rf.abi]; @@ -3958,8 +3979,8 @@ static id vin_fn(NvShader *vs) { // the shadow-input entry point, if (!vs->vfn_state) { vs->vfn_state = 2; size_t kl = strlen(vs->key); NSString *path = [NSString stringWithFormat:@"%@/%.*s_bv.metallib", g_msl_dir, (int)(kl > 2 ? kl - 2 : kl), vs->key]; - NSError *e = nil; id lib = lib_from_file(path, &e); id f = lib && vs->ventry ? [lib newFunctionWithName:vs->ventry] : nil; - if (f && f.functionConstantsDictionary.count) f = [lib newFunctionWithName:vs->ventry constantValues:[MTLFunctionConstantValues new] error:&e]; // as shader_load + NSError *e = nil; id lib = lib_from_file(path, &e); id f = lib && vs->ventry ? fn4_note([lib newFunctionWithName:vs->ventry], lib, vs->ventry, nil) : nil; + if (f && f.functionConstantsDictionary.count) { MTLFunctionConstantValues *cv = [MTLFunctionConstantValues new]; f = fn4_note([lib newFunctionWithName:vs->ventry constantValues:cv error:&e], lib, vs->ventry, cv); } // as shader_load const char *why = !lib ? "no library" : !f ? "no entry point" : NULL; if (f && vs->rf.nb && [f newArgumentEncoderWithBufferIndex:vs->rf.abi].encodedLength != vs->arg_len) { why = "its argument buffer differs from the base library's"; f = nil; } if (f) { vs->vfn = f; vs->vfn_state = 1; g_vin_loaded++; } @@ -4289,6 +4310,205 @@ static MTLRenderPipelineDescriptor *pso_desc(const pso_key *k, NvShader *vs, NvS d.vertexDescriptor = vd; return d; } +// ---- Metal 4 pipelines: a base per pipeline with its colour state unspecialized, a per-game archive of the bases ---- +// pso_desc stays the description of a pipeline. pso_build turns it into an MTL4RenderPipelineDescriptor whose colour attachments (format, blending, factors, operations, write mask) +// are unspecialized, builds that base once (MTL4Compiler, or takes it from the game's pipeline archive) and makes each pipeline from it by specialization: about 4 µs on an M2 Max, +// where building a pipeline whose colour state alone differs took 3-5 ms. Breath of the Wild's 4,829 recorded pipelines came from 1,818 such bases. Metal 4 pipelines name no +// depth / stencil format, so a base serves every depth / stencil attachment. +// The archive keeps the bases beside pso_user.tsv, in pso_archive/: files written by an MTL4PipelineDataSetSerializer that captures binaries only (with the descriptor capture as +// well it writes no archive and reports no error). Each serialization writes what was captured since the one before, so the folder holds one file per flush; all of them are opened +// at start, and a base found in one is taken from it directly (MTL4Archive), not through the compiler's lookupArchives, which would capture it again into the next file. A base +// taken from an archive costs about 0.06 ms, against 10-80 ms to build one. +// DM_PSO_MTL4=0: the MTLDevice path as before (no bases, no archive). DM_PSO_ARCHIVE=off: no archive; DM_PSO_ARCHIVE=: the archive in that folder. +// On by default on the Mac only; on iOS DM_PSO_MTL4=1 turns it on. The archive's footprint (with the pipeline prebuild, about 850 MB more for Breath of the Wild) is not measured +// against an iPhone's limit yet, and an iPhone cannot be given an archive made offline (metal-tt rejects the Metal 4 serializer's pipeline script) +static int pso4_on(void) { + static int v = -1; + if (v < 0) { + const char *e = getenv("DM_PSO_MTL4"); +#if TARGET_OS_IPHONE + v = e && !strcmp(e, "1"); + if (!v) MLOG("render pipelines through the MTLDevice (iOS: DM_PSO_MTL4=1 turns on the Metal 4 bases and the pipeline archive)"); +#else + v = !(e && !strcmp(e, "0")); + if (!v) MLOG("DM_PSO_MTL4=0: render pipelines are built through the MTLDevice (no bases, no pipeline archive)"); +#endif + } + return v; +} +static id g_comp4; static id g_ser4; +static NSArray> *g_arch4; static NSString *g_arch_dir; static int g_arch_seq; +static dispatch_queue_t g_arch_q; static _Atomic int g_arch_new, g_arch_sched; +// Compiles with the serializer attached hold this shared, a flush holds it alone: a compile that finishes during serializeAsArchiveAndFlushToURL crashed in Metal +// (-[_MTLBinaryArchive newArchiverIdWithBinaryKey:...]). Specializations and archive lookups do not take it (they capture nothing) +static pthread_rwlock_t g_arch_rw = PTHREAD_RWLOCK_INITIALIZER; +// Compaction: every start writes a file or more (each flush captures only what is new; Breath of the Wild 5-7 per start), and each file a base is not in costs a lookup about +// 0.06 ms. With more than PSO_ARCH_COMPACT files, the bases the prewarm takes from them are taken through the compiler with those files as its lookup archives, which captures them +// again, so the flush after the prewarm writes them into one file; the old files are then removed (open archives stay usable for the rest of the run). Bases the prewarm did not +// take (pso_user.tsv never named them) go with them and are built again when drawn +enum { PSO_ARCH_COMPACT = 8 }; +static MTL4CompilerTaskOptions *g_compact_opts; static NSArray *g_compact_files; static _Atomic int g_compacted; +static _Atomic uint64_t g_pb_built, g_pb_archived, g_pb_reused, g_pb_spec, g_pb_legacy, g_pb_base_ns; // bases compiled / taken from the archive / reused, specializations, MTLDevice builds +typedef struct { const void *vf, *ff; uint32_t samples, a2c, rast, topo; uint32_t va[31][3], vl[31][3]; } pbase_key; // what a base depends on: pso_desc's descriptor less the colour attachments +static nvn_hmap g_pbase = {.klen = sizeof(pbase_key)}; +// Only callers for the same base wait on this group; unrelated bases compile in parallel. +@interface PbaseBuild : NSObject { @public dispatch_group_t done; id ps; NSError *error; } @end +@implementation PbaseBuild @end +static nvn_hmap g_pbase_pending = {.klen = sizeof(pbase_key)}; +static os_unfair_lock g_pbase_lock = OS_UNFAIR_LOCK_INIT; +static NSString *pso_user_path(void); +static void pso4_init(void) { + static dispatch_once_t once; + dispatch_once(&once, ^{ + if (!pso4_on() || ![(id)g_dev respondsToSelector:@selector(newCompilerWithDescriptor:error:)]) return; // no Metal 4 compiler (t_pso_pending's fake device): the MTLDevice path + const char *e = getenv("DM_PSO_ARCHIVE"); NSString *user = pso_user_path(); + g_arch_dir = e && !strcmp(e, "off") ? nil : e && *e ? @(e) : user ? [user.stringByDeletingLastPathComponent stringByAppendingPathComponent:@"pso_archive"] : nil; + id ser = nil; + if (g_arch_dir) { + g_arch_q = dispatch_queue_create("dm.nvn.pso-archive", dispatch_queue_attr_make_with_qos_class(DISPATCH_QUEUE_SERIAL, QOS_CLASS_UTILITY, 0)); + MTL4PipelineDataSetSerializerDescriptor *sd = [MTL4PipelineDataSetSerializerDescriptor new]; sd.configuration = MTL4PipelineDataSetSerializerConfigurationCaptureBinaries; + ser = [g_dev newPipelineDataSetSerializerWithDescriptor:sd]; + NSMutableArray> *arch = [NSMutableArray array]; uint64_t t0 = bp_now(); unsigned long long bytes = 0; + NSMutableArray *runf = [NSMutableArray array]; + NSArray *names = [[[NSFileManager.defaultManager contentsOfDirectoryAtPath:g_arch_dir error:nil] filteredArrayUsingPredicate:[NSPredicate predicateWithFormat:@"self ENDSWITH '.mtl4a'"]] + sortedArrayUsingSelector:@selector(compare:)]; + for (NSString *n in names.reverseObjectEnumerator) { // newest first + NSString *p = [g_arch_dir stringByAppendingPathComponent:n]; NSError *ae = nil; + id a = [g_dev newArchiveWithURL:[NSURL fileURLWithPath:p] error:&ae]; + if (a) { [arch addObject:a]; [runf addObject:p]; bytes += [[NSFileManager.defaultManager attributesOfItemAtPath:p error:nil] fileSize]; } + else MLOG("[RED] pipeline archive file %s not usable (%s): its bases are built again", p.UTF8String, ae.localizedDescription.UTF8String); + g_arch_seq = MAX(g_arch_seq, n.intValue + 1); + } + g_arch4 = arch; + if (runf.count > PSO_ARCH_COMPACT) { g_compact_opts = [MTL4CompilerTaskOptions new]; g_compact_opts.lookupArchives = arch; g_compact_files = runf; } + MLOG("pipeline archive %s: %lu files, %.1f MB, opened in %.1f ms", g_arch_dir.UTF8String, (unsigned long)arch.count, bytes / 1048576.0, (bp_now() - t0) / 1e6); + } + MTL4CompilerDescriptor *cd = [MTL4CompilerDescriptor new]; cd.pipelineDataSetSerializer = ser; + NSError *ce = nil; g_comp4 = [g_dev newCompilerWithDescriptor:cd error:&ce]; + if (!g_comp4) MLOG("[RED] no Metal 4 compiler (%s): render pipelines are built through the MTLDevice", ce.localizedDescription.UTF8String); + else g_ser4 = ser; + }); +} +// Writes what the serializer captured since the last flush as the next file of the archive (on g_arch_q); g_arch_flush_ok: the last one wrote its file +static int g_arch_flush_ok; +static void arch_flush(void) { + // Keep all recaptured bases in one serialization, even when a prewarm lasts beyond the timer. + if (g_compact_opts && atomic_load(&g_warm_running)) return; + g_arch_flush_ok = 0; + pthread_rwlock_wrlock(&g_arch_rw); + int n = atomic_exchange(&g_arch_new, 0); + if (!n || !g_ser4 || !g_arch_dir) { pthread_rwlock_unlock(&g_arch_rw); return; } + [NSFileManager.defaultManager createDirectoryAtPath:g_arch_dir withIntermediateDirectories:YES attributes:nil error:nil]; + NSString *path = [g_arch_dir stringByAppendingPathComponent:[NSString stringWithFormat:@"%05d.mtl4a", g_arch_seq++]], *tmp = [path stringByAppendingString:@".tmp"]; + uint64_t t0 = bp_now(); NSError *e = nil; + BOOL ok = [g_ser4 serializeAsArchiveAndFlushToURL:[NSURL fileURLWithPath:tmp] error:&e]; pthread_rwlock_unlock(&g_arch_rw); + ok = ok && !rename(tmp.fileSystemRepresentation, path.fileSystemRepresentation); + if (!ok) { MLOG("[RED] pipeline archive: writing %s failed%s%s", path.UTF8String, e ? ": " : "", e ? e.localizedDescription.UTF8String : ""); unlink(tmp.fileSystemRepresentation); return; } + g_arch_flush_ok = 1; + MLOG("pipeline archive: %d new pipeline bases written to %s (%.1f MB, %.0f ms)", n, path.lastPathComponent.UTF8String, + [[NSFileManager.defaultManager attributesOfItemAtPath:path error:nil] fileSize] / 1048576.0, (bp_now() - t0) / 1e6); +} +static void arch_compact_finish(void); +static void arch_note_new(void) { // a base was compiled: flush 10 s later (one flush for a burst of them) + atomic_fetch_add(&g_arch_new, 1); + if (!g_arch_q || atomic_exchange(&g_arch_sched, 1)) return; + dispatch_after(dispatch_time(DISPATCH_TIME_NOW, 10 * (int64_t)NSEC_PER_SEC), g_arch_q, ^{ atomic_store(&g_arch_sched, 0); arch_flush(); if (!atomic_load(&g_warm_running)) arch_compact_finish(); }); +} +static void arch_compact_finish(void) { // after the flush that follows the prewarm: the old files go once their bases are in the new one + if (!g_compact_files || !g_compacted) return; + if (!g_arch_flush_ok) { MLOG("[RED] pipeline archive: compaction left the old files (the flush failed)"); return; } + for (NSString *f in g_compact_files) unlink(f.fileSystemRepresentation); + MLOG("pipeline archive compacted: %lu files gave %d bases to the file just written and were removed", (unsigned long)g_compact_files.count, (int)g_compacted); + g_compact_files = nil; +} +static void arch_flush_now(void) { if (g_arch_q) dispatch_async(g_arch_q, ^{ arch_flush(); arch_compact_finish(); }); } +static void pso_col4(MTL4RenderPipelineColorAttachmentDescriptor *a, MTLRenderPipelineColorAttachmentDescriptor *c) { + a.pixelFormat = c.pixelFormat; a.writeMask = c.writeMask; a.blendingState = c.blendingEnabled ? MTL4BlendStateEnabled : MTL4BlendStateDisabled; + a.sourceRGBBlendFactor = c.sourceRGBBlendFactor; a.destinationRGBBlendFactor = c.destinationRGBBlendFactor; a.rgbBlendOperation = c.rgbBlendOperation; + a.sourceAlphaBlendFactor = c.sourceAlphaBlendFactor; a.destinationAlphaBlendFactor = c.destinationAlphaBlendFactor; a.alphaBlendOperation = c.alphaBlendOperation; +} +// pso_desc's descriptor as a Metal 4 one: unspec leaves every colour attachment unspecialized (the base), else copies them. nil when a function has no Metal 4 descriptor (fn4_of) +static MTL4RenderPipelineDescriptor *pso_desc4(MTLRenderPipelineDescriptor *d, int unspec) { + // Metal 4 requires a fragment function when rasterization is enabled. Depth-only draws still need rasterization. + if (d.rasterizationEnabled && !d.fragmentFunction) return nil; + MTL4FunctionDescriptor *vf = fn4_of(d.vertexFunction), *ff = d.fragmentFunction ? fn4_of(d.fragmentFunction) : nil; + if (!vf || (d.fragmentFunction && !ff)) return nil; + MTL4RenderPipelineDescriptor *m = [MTL4RenderPipelineDescriptor new]; + m.vertexFunctionDescriptor = vf; m.fragmentFunctionDescriptor = ff; m.vertexDescriptor = d.vertexDescriptor; + m.rasterSampleCount = d.rasterSampleCount; m.alphaToCoverageState = d.alphaToCoverageEnabled ? MTL4AlphaToCoverageStateEnabled : MTL4AlphaToCoverageStateDisabled; + m.rasterizationEnabled = d.rasterizationEnabled; m.inputPrimitiveTopology = d.inputPrimitiveTopology; + for (int i = 0; i < 8; i++) { + MTL4RenderPipelineColorAttachmentDescriptor *a = m.colorAttachments[i]; + if (!unspec) { pso_col4(a, d.colorAttachments[i]); continue; } + a.pixelFormat = MTLPixelFormatUnspecialized; a.writeMask = MTLColorWriteMaskUnspecialized; a.blendingState = MTL4BlendStateUnspecialized; + a.sourceRGBBlendFactor = a.destinationRGBBlendFactor = a.sourceAlphaBlendFactor = a.destinationAlphaBlendFactor = MTLBlendFactorUnspecialized; + a.rgbBlendOperation = a.alphaBlendOperation = MTLBlendOperationUnspecialized; + } + return m; +} +static void pbase_key_of(MTLRenderPipelineDescriptor *d, pbase_key *k) { + memset(k, 0, sizeof *k); + k->vf = (__bridge const void *)d.vertexFunction; k->ff = (__bridge const void *)d.fragmentFunction; + k->samples = (uint32_t)d.rasterSampleCount; k->a2c = d.alphaToCoverageEnabled; k->rast = d.rasterizationEnabled; k->topo = (uint32_t)d.inputPrimitiveTopology; + MTLVertexDescriptor *vd = d.vertexDescriptor; + for (NSUInteger i = 0; vd && i < 31; i++) { + MTLVertexAttributeDescriptor *a = vd.attributes[i]; MTLVertexBufferLayoutDescriptor *l = vd.layouts[i]; + k->va[i][0] = (uint32_t)a.format; k->va[i][1] = (uint32_t)a.offset; k->va[i][2] = (uint32_t)a.bufferIndex; + k->vl[i][0] = (uint32_t)l.stride; k->vl[i][1] = (uint32_t)l.stepFunction; k->vl[i][2] = (uint32_t)l.stepRate; + } +} +// The pipeline pso_desc describes: its base (built, from the archive or already made), then specialized with its colour state. Called on several threads +static id pso_build(MTLRenderPipelineDescriptor *d, NSError **e) { + pso4_init(); + int rast = d.rasterizationEnabled; + MTL4RenderPipelineDescriptor *b = g_comp4 ? pso_desc4(d, rast) : nil; // with rasterization off there is no colour state to leave open: the base is the pipeline + if (!b) { g_pb_legacy++; return [g_dev newRenderPipelineStateWithDescriptor:d error:e]; } + pbase_key bk; pbase_key_of(d, &bk); + os_unfair_lock_lock(&g_pbase_lock); + id base = hm_get_id(&g_pbase, &bk); + PbaseBuild *job = nil; int owner = 0; + if (!base) { + job = hm_get_id(&g_pbase_pending, &bk); + if (!job) { job = [PbaseBuild new]; job->done = dispatch_group_create(); dispatch_group_enter(job->done); hm_put_id(&g_pbase_pending, &bk, job); owner = 1; } + } + os_unfair_lock_unlock(&g_pbase_lock); + if (job && !owner) { dispatch_group_wait(job->done, DISPATCH_TIME_FOREVER); base = job->ps; if (!base) { if (e) *e = job->error; return nil; } } + if (base) g_pb_reused++; + else { + uint64_t t0 = bp_now(); + for (id a in g_arch4) if ((base = [a newRenderPipelineStateWithDescriptor:b error:nil])) break; + if (base) g_pb_archived++; + pthread_rwlock_rdlock(&g_arch_rw); + if (base && g_compact_opts && atomic_load(&g_warm_running)) { // compaction (see g_compact_opts): taken again through the compiler, which captures it + id cb = [g_comp4 newRenderPipelineStateWithDescriptor:b compilerTaskOptions:g_compact_opts error:nil]; + if (cb) { base = cb; atomic_fetch_add(&g_arch_new, 1); g_compacted++; } + } + if (!base) { + NSError *be = nil; base = [g_comp4 newRenderPipelineStateWithDescriptor:b compilerTaskOptions:nil error:&be]; job->error = be; + if (base) { g_pb_built++; if (g_ser4) arch_note_new(); } + } + pthread_rwlock_unlock(&g_arch_rw); + g_pb_base_ns += bp_now() - t0; + os_unfair_lock_lock(&g_pbase_lock); + job->ps = base; + if (base) hm_put_id(&g_pbase, &bk, base); + hm_rel(nvn_hm_del(&g_pbase_pending, &bk)); dispatch_group_leave(job->done); + os_unfair_lock_unlock(&g_pbase_lock); + if (!base) { if (e) *e = job->error; return nil; } + } + if (!rast) return base; + MTL4RenderPipelineDescriptor *s = (MTL4RenderPipelineDescriptor *)[base newRenderPipelineDescriptorForSpecialization]; + for (int i = 0; i < 8; i++) pso_col4(s.colorAttachments[i], d.colorAttachments[i]); + id ps = [g_comp4 newRenderPipelineStateBySpecializationWithDescriptor:s pipeline:base error:e]; + if (ps) g_pb_spec++; + return ps; +} +// What no pipeline depends on leaves the key: the blend state of targets whose blending is off (pso_desc reads blend[l] only where blend_en has bit l). Breath of the Wild recorded +// 4,829 keys for 1,982 different pipelines, 2,845 of them told apart by such blend state alone. The depth / stencil formats stay: a Metal 4 base names none (pbase_key leaves them +// out, so keys that differ only there cost one specialization each), but a pipeline built through the MTLDevice must name the pass's +static void pso_key_norm(pso_key *k) { + for (int i = 0; i < 8; i++) if (!(k->blend_en >> i & 1)) memset(&k->blend[i], 0, sizeof k->blend[i]); +} // Pipeline key: current state S + pass formats (normal pass; a merged pass additionally carries lmap / specialized function, filled in by pso_get) static void pso_key_fill(pso_key *k, NvShader *vs, NvShader *fs, const uint32_t cfmt[8], uint32_t dfmt, uint32_t sfmt, uint32_t samples, int lmap) { memset(k, 0, sizeof *k); @@ -4298,6 +4518,7 @@ static void pso_key_fill(pso_key *k, NvShader *vs, NvShader *fs, const uint32_t for (int i = 0; i < 8; i++) if (k->cfmt[i] || lmap) k->blend[i] = S.blend[i]; const nvn_refl *rf = &vs->rf; for (int v = 0; v < rf->nv; v++) { int a = rf->v[v].nvn; if (a < 16 && a < S.nattr) { k->attr[a] = S.attr[a]; int st = S.attr[a].stream; if (st < 16) k->stream[st] = S.stream[st]; } } + pso_key_norm(k); } // ---- Parallel precompile: pipelines met for the first time used to be compiled synchronously one by one on the submit thread; 9 misses in a row in one frame = 284 ms (heavy scene on a real device). // After satisfying the first pipeline miss in a submission, scan ahead once along the remaining commands (replaying state only, no drawing) and kick a bounded number of normal-pass pipelines to the background; @@ -4325,7 +4546,7 @@ static int pso_pend_kick(const pso_key *k, NvShader *vs, NvShader *fs) { // No if (at >= 0 || slot < 0) return 0; MTLRenderPipelineDescriptor *d = pso_desc(k, vs, fs, nil, 0); dispatch_async(dispatch_get_global_queue(QOS_CLASS_USER_INITIATED, 0), ^{ - NSError *e = nil; id ps = [g_dev newRenderPipelineStateWithDescriptor:d error:&e]; + NSError *e = nil; id ps = pso_build(d, &e); pthread_mutex_lock(&g_pp_mu); g_pp[slot].ps = ps; g_pp[slot].done = 1; pthread_cond_broadcast(&g_pp_cv); pthread_mutex_unlock(&g_pp_mu); }); return 1; @@ -4368,13 +4589,13 @@ static __unsafe_unretained id pso_get_(NvProg *pr, MTLRe if (ps) g_pp_taken++; else { MTLRenderPipelineDescriptor *d = pso_desc(&k, vs, fs, ms_fn ? ms_fn : g_pso_fsfn, g_pso_lmap != NULL); - ps = [g_dev newRenderPipelineStateWithDescriptor:d error:&e]; + ps = pso_build(d, &e); // A vertex shader that writes the layer (render_target_array_index) needs the pipeline's input primitive class, which the reflection does not record and the pipeline key does not hold. // Pokémon Legends: Z-A has two (b29f08619f379a83, 57246f9a04af8529, layered full-screen passes); Metal names the reason, the shader is marked and every later pipeline with it gets the // triangle class (layered draws are triangle lists; a line or point draw with such a shader would fail validation) if (!ps && !vs->layer_out && [e.localizedDescription containsString:@"render_target_array_index"]) { vs->layer_out = 1; MLOG("vertex shader %s writes the layer: its pipelines get the triangle input primitive class", vs->key); - e = nil; d = pso_desc(&k, vs, fs, ms_fn ? ms_fn : g_pso_fsfn, g_pso_lmap != NULL); ps = [g_dev newRenderPipelineStateWithDescriptor:d error:&e]; + e = nil; d = pso_desc(&k, vs, fs, ms_fn ? ms_fn : g_pso_fsfn, g_pso_lmap != NULL); ps = pso_build(d, &e); } } g_bp_pso_n++; g_bp_pso_ns += bp_now() - bt; @@ -4444,10 +4665,10 @@ static void warm_worker(NSArray *paths) { if (sk.fz_desc.kind == 1 && fs) { int fx[32]; for (int i = 0; i < 32; i++) fx[i] = sk.fz_desc.fx[i]; fsfn = bf_fn(fs, fx, sk.fz_desc.fbmask, sk.k.lmap); } if (!fsfn) { g_warm_skip_fz++; continue; } } - pso_key k = sk.k; k.vs = (__bridge void *)vs; k.fs = (__bridge void *)fs; k.fsfn = (__bridge void *)fsfn; - NSError *e = nil; id ps = [g_dev newRenderPipelineStateWithDescriptor:pso_desc(&k, vs, fs, fsfn, sk.merged) error:&e]; + pso_key k = sk.k; k.vs = (__bridge void *)vs; k.fs = (__bridge void *)fs; k.fsfn = (__bridge void *)fsfn; pso_key_norm(&k); // rows written before the key was normalized + NSError *e = nil; id ps = pso_build(pso_desc(&k, vs, fs, fsfn, sk.merged), &e); if (!ps && !vs->layer_out && [e.localizedDescription containsString:@"render_target_array_index"]) { // a vertex shader that writes the layer (see pso_get_) - vs->layer_out = 1; e = nil; ps = [g_dev newRenderPipelineStateWithDescriptor:pso_desc(&k, vs, fs, fsfn, sk.merged) error:&e]; } + vs->layer_out = 1; e = nil; ps = pso_build(pso_desc(&k, vs, fs, fsfn, sk.merged), &e); } if (!ps) { MLOG("[RED] background render pipeline creation failed: %s", e.localizedDescription.UTF8String); continue; } WarmItem *item = [WarmItem new]; item->kind = 1; item->vs_key = vkey; item->fs_key = fkey; item->vs = vs; item->fs = fs; item->fsfn = fsfn; item->pso = k; item->object = ps; warm_enqueue(item); g_warm_rps++; @@ -4464,8 +4685,11 @@ static void warm_worker(NSArray *paths) { } MLOG("pipeline prewarm done: shaders %llu / render pipelines %llu (specialized fragment skipped %llu) / compute pipelines %llu, background time %.1f ms; shader keys not in the shader map (another game version) %llu", g_warm_shaders, g_warm_rps, g_warm_skip_fz, g_warm_cps, (bp_now() - g_warm_t0) / 1e6, g_warm_skip_stale); + if (g_comp4) MLOG("pipeline bases so far: %llu compiled, %llu from the archive, %llu reused (%.1f ms for the bases); %llu specializations, %llu pipelines without a base (MTLDevice)", + (unsigned long long)g_pb_built, (unsigned long long)g_pb_archived, (unsigned long long)g_pb_reused, g_pb_base_ns / 1e6, (unsigned long long)g_pb_spec, (unsigned long long)g_pb_legacy); } atomic_store(&g_warm_running, 0); + arch_flush_now(); // flush and finish compaction together, after the prewarm stops recapturing bases } // Wait for the background warm-up to finish (at most timeout_ms): returns how many milliseconds were actually waited, 0 if it is not running. On first install the system shader cache is fully cold; if warm-up is slower than the player entering a run, // pipelines not yet warmed are compiled on the spot on the submit thread = in-run stutter; the loading screen on entering a run blocks the player anyway, so let it load a little longer (called where hle_nvn.c presents). @@ -4492,6 +4716,7 @@ static void warm_start(void) { paths = all; } if (!g_warm_items) g_warm_items = [NSMutableArray array]; + pso4_init(); // the pipeline archive opens before the prewarm builds from it g_warm_t0 = bp_now(); dispatch_queue_attr_t attr = dispatch_queue_attr_make_with_qos_class(DISPATCH_QUEUE_SERIAL, QOS_CLASS_USER_INITIATED, 0); // Not optional background work: it decides whether the pipelines are ready before the player enters a run dispatch_queue_t q = dispatch_queue_create("dm.nvn.pso-warm", attr); @@ -6030,7 +6255,7 @@ static id bf_fn(NvShader *fs, const int *fx, uint8_t fbmask, const for (int i = 0; i < fs->rf.nfc; i++) { int v = fx[i]; [cv setConstantValue:&v type:MTLDataTypeInt atIndex:(NSUInteger)i]; } for (int q = 0; q < 8; q++) { bool b = fbmask >> q & 1; [cv setConstantValue:&b type:MTLDataTypeBool atIndex:(NSUInteger)(32 + q)]; } for (int j = 0; j < fs->rf.nfo; j++) for (int q = 0; q < 8; q++) { bool b = lmap[q] == fs->rf.fo_color[j]; [cv setConstantValue:&b type:MTLDataTypeBool atIndex:(NSUInteger)(64 + 8 * j + q)]; } - NSError *e = nil; f = [lib newFunctionWithName:@"main0" constantValues:cv error:&e]; + NSError *e = nil; f = fn4_note([lib newFunctionWithName:@"main0" constantValues:cv error:&e], lib, @"main0", cv); if (!f) { MLOG("[RED] readable-tile version specialization failed %s (%s): %s", fs->key, k.UTF8String, e.localizedDescription.UTF8String); os_unfair_lock_unlock(&g_fz_fn_lock); return nil; } g_bf_fn_cache[cacheKey] = f; os_unfair_lock_unlock(&g_fz_fn_lock); return f; } @@ -6714,7 +6939,7 @@ static __unsafe_unretained id tf_pso(NvShader *vs) { __unsafe_unretained id cached = hm_get_id(&g_pso, &k); if (cached) return cached; if (hm_get_id(&g_pso_fail, &k)) return nil; - NSError *e = nil; id ps = [g_dev newRenderPipelineStateWithDescriptor:pso_desc(&k, vs, nil, nil, 0) error:&e]; + NSError *e = nil; id ps = pso_build(pso_desc(&k, vs, nil, nil, 0), &e); if (!ps) { MLOG("[RED] failed to create the transform feedback pipeline (vertex %s): %s", vs->key, e.localizedDescription.UTF8String); hm_put_id(&g_pso_fail, &k, NSNull.null); return nil; } hm_put_id(&g_pso, &k, ps); return ps; diff --git a/dm/gfx/tests/.gitignore b/dm/gfx/tests/.gitignore index 26a1b89..f181051 100644 --- a/dm/gfx/tests/.gitignore +++ b/dm/gfx/tests/.gitignore @@ -74,3 +74,5 @@ t_upgrade_alias t_upgrade_alias.dSYM/ t_stencil_view t_stencil_view.dSYM/ +t_pso_archive +t_pso_archive.dSYM/ diff --git a/dm/gfx/tests/Makefile b/dm/gfx/tests/Makefile index 205a0f6..735471f 100644 --- a/dm/gfx/tests/Makefile +++ b/dm/gfx/tests/Makefile @@ -79,6 +79,8 @@ t_rt_depth_ro: t_rt_depth_ro.m $(MTL_T_DEP) clang $(CFLAGS) -fobjc-arc -o $@ t_rt_depth_ro.m $(MTL_T_SRC) $(FW) t_pso_pending: t_pso_pending.m $(MTL_T_DEP) clang $(CFLAGS) -fobjc-arc -o $@ t_pso_pending.m $(MTL_T_SRC) $(FW) +t_pso_archive: t_pso_archive.m $(MTL_T_DEP) + clang $(CFLAGS) -fobjc-arc -o $@ t_pso_archive.m $(MTL_T_SRC) $(FW) t_pso_skey: t_pso_skey.m ../nvn/nvn_pso_skey.h ../nvn/nvn_obj.h clang $(CFLAGS) -fobjc-arc -o $@ t_pso_skey.m -framework Foundation t_view_swizzle: t_view_swizzle.m $(MTL_T_DEP) @@ -106,7 +108,7 @@ t_stencil_view: t_stencil_view.m $(MTL_T_DEP) # The Vulkan layer (#included) over the whole NVN layer, whose device and present path it uses t_vk_desc: t_vk_desc.m ../vk/hle_vk.m $(MTL_T_DEP) clang $(CFLAGS) -I../../third_party/vulkan-headers/include -fobjc-arc -o $@ t_vk_desc.m $(MTL_T_SRC) ../nvn/nvn_mtl.m $(FW) -run: t_nvn t_dirty_slots t_dirty_race t_builtin t_vin t_tbuf2d t_deswz_gpu t_pace t_fstep t_refl t_hmap t_cover t_fuse_check t_present_crop t_reorder_chain t_defer_chain t_zskip t_xv t_ub t_vbuf t_geometry t_mip_extent t_rt_range_shape t_rt_range_ro t_rt_range t_rt_depth_ro t_pso_skey t_pso_pending t_pfv t_view_levels t_view_swizzle t_vk_desc t_integer_blend t_primitive_restart t_clear_ds t_upgrade_alias t_stencil_view +run: t_nvn t_dirty_slots t_dirty_race t_builtin t_vin t_tbuf2d t_deswz_gpu t_pace t_fstep t_refl t_hmap t_cover t_fuse_check t_present_crop t_reorder_chain t_defer_chain t_zskip t_xv t_ub t_vbuf t_geometry t_mip_extent t_rt_range_shape t_rt_range_ro t_rt_range t_rt_depth_ro t_pso_skey t_pso_archive t_pso_pending t_pfv t_view_levels t_view_swizzle t_vk_desc t_integer_blend t_primitive_restart t_clear_ds t_upgrade_alias t_stencil_view ./t_nvn ./t_dirty_slots ./t_dirty_race @@ -141,6 +143,7 @@ run: t_nvn t_dirty_slots t_dirty_race t_builtin t_vin t_tbuf2d t_deswz_gpu t_pac DM_NVN_UNWRITTEN_DC=0 MTL_DEBUG_LAYER=1 ./t_rt_range --sample-attachment --both-flags ./t_pso_skey ./t_pso_pending + MTL_DEBUG_LAYER=1 ./t_pso_archive MTL_DEBUG_LAYER=1 ./t_pfv MTL_DEBUG_LAYER=1 ./t_view_levels MTL_DEBUG_LAYER=1 ./t_view_swizzle diff --git a/dm/gfx/tests/t_pso_archive.m b/dm/gfx/tests/t_pso_archive.m new file mode 100644 index 0000000..fa8e409 --- /dev/null +++ b/dm/gfx/tests/t_pso_archive.m @@ -0,0 +1,220 @@ +// Metal 4 pipelines (pso_build): one base per pipeline with its colour state unspecialized, made into pipelines by specialization, kept in the pipeline archive. +// Checks: pipelines that differ only in colour state share one base; a specialized pipeline draws the same pixels as one built through the MTLDevice; a flushed archive +// file serves the base again without compiling; a function with no Metal 4 descriptor keeps the MTLDevice path; pso_key_norm drops what no pipeline depends on; compaction +// merges the files the prewarm took bases from into one despite a mid-prewarm flush; depth-only draws write depth; concurrent colour variants compile one base +#include "../nvn/nvn_mtl.m" +uint64_t g_t0_ns; +int dm_guest_as_contains(uintptr_t address) { (void)address; return 0; } +void *front_metal_layer(void) { return NULL; } +module_t *module_of(const void *p) { (void)p; return NULL; } +module_t g_mod[MAX_MOD]; int g_nmod; +void svc_thread_dump(void) {} +uint64_t now_ns(void) { return bp_now(); } +static int checks, failed; +#define EXPECT(name, want, got) do { uint64_t w = (want), g = (got); checks++; if (w != g) { fprintf(stderr, "%s: expected %llu, got %llu\n", name, (unsigned long long)w, (unsigned long long)g); failed++; } } while (0) + +static id g_lib, g_lib2; // g_lib2: the same source again, for functions with no Metal 4 descriptor (a library hands out one function object per name) +static id g_q; +// Hold the first base build while other callers request colour variants and an unrelated base. +@interface ArchiveGateCompiler : NSObject { +@public id compiler; dispatch_semaphore_t entered, release; _Atomic int calls; int fail; +} +@end +@implementation ArchiveGateCompiler +- (id)forwardingTargetForSelector:(SEL)selector { return compiler; } +- (id)newRenderPipelineStateWithDescriptor:(MTL4RenderPipelineDescriptor *)d compilerTaskOptions:(MTL4CompilerTaskOptions *)opts error:(NSError **)error { + int call = atomic_fetch_add(&calls, 1); + if (!call) { dispatch_semaphore_signal(entered); dispatch_semaphore_wait(release, DISPATCH_TIME_FOREVER); } + if (fail) { if (error) *error = [NSError errorWithDomain:@"test" code:1 userInfo:nil]; return nil; } + return [compiler newRenderPipelineStateWithDescriptor:d compilerTaskOptions:opts error:error]; +} +@end +// a full-screen triangle; colour 0 = (0.5, 0.25, 1, 0.5), colour 1 = (0.2, 0.4, 0.6, 0.8); attribute 0 (a float4) scales nothing, so a vertex descriptor is part of the pipeline +static MTLRenderPipelineDescriptor *desc(MTLPixelFormat f0, int blend, MTLPixelFormat f1, MTLColorWriteMask m1, int registered) { + MTLRenderPipelineDescriptor *d = [MTLRenderPipelineDescriptor new]; + id v = [g_lib2 newFunctionWithName:@"vs"], f = [g_lib2 newFunctionWithName:@"fs"]; + static id rv, rf; // the registered ones are made once, as shader_load makes a shader's functions once + if (registered) { if (!rv) { rv = fn4_note([g_lib newFunctionWithName:@"vs"], g_lib, @"vs", nil); rf = fn4_note([g_lib newFunctionWithName:@"fs"], g_lib, @"fs", nil); } v = rv; f = rf; } + d.vertexFunction = v; d.fragmentFunction = f; + MTLVertexDescriptor *vd = [MTLVertexDescriptor vertexDescriptor]; + vd.attributes[0].format = MTLVertexFormatFloat4; vd.attributes[0].bufferIndex = 16; vd.layouts[16].stride = 16; vd.layouts[16].stepFunction = MTLVertexStepFunctionConstant; vd.layouts[16].stepRate = 0; + d.vertexDescriptor = vd; + d.colorAttachments[0].pixelFormat = f0; + if (blend) { d.colorAttachments[0].blendingEnabled = YES; d.colorAttachments[0].sourceRGBBlendFactor = d.colorAttachments[0].sourceAlphaBlendFactor = MTLBlendFactorSourceAlpha; + d.colorAttachments[0].destinationRGBBlendFactor = d.colorAttachments[0].destinationAlphaBlendFactor = MTLBlendFactorOneMinusSourceAlpha; } + d.colorAttachments[1].pixelFormat = f1; d.colorAttachments[1].writeMask = m1; + d.depthAttachmentPixelFormat = MTLPixelFormatDepth32Float; // a Metal 4 pipeline names no depth format; drawn into a pass with this depth attachment + return d; +} +// draws with ps into fresh targets cleared to (1, 0, 0, 1) and returns texel (1, 1) of both as floats +static void draw_with(id ps, MTLPixelFormat f0, MTLPixelFormat f1, float out[2][4]) { + MTLRenderPassDescriptor *rp = [MTLRenderPassDescriptor new]; id t[2]; + MTLPixelFormat fm[2] = { f0, f1 }; + for (int i = 0; i < 2; i++) { + MTLTextureDescriptor *td = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:fm[i] width:4 height:4 mipmapped:NO]; td.usage = MTLTextureUsageRenderTarget; td.storageMode = MTLStorageModeShared; + t[i] = [g_dev newTextureWithDescriptor:td]; + rp.colorAttachments[i].texture = t[i]; rp.colorAttachments[i].loadAction = MTLLoadActionClear; rp.colorAttachments[i].clearColor = MTLClearColorMake(1, 0, 0, 1); rp.colorAttachments[i].storeAction = MTLStoreActionStore; + } + MTLTextureDescriptor *dd = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatDepth32Float width:4 height:4 mipmapped:NO]; dd.usage = MTLTextureUsageRenderTarget; dd.storageMode = MTLStorageModePrivate; + rp.depthAttachment.texture = [g_dev newTextureWithDescriptor:dd]; rp.depthAttachment.loadAction = MTLLoadActionClear; rp.depthAttachment.storeAction = MTLStoreActionDontCare; + static id one; if (!one) { float v[4] = { 1, 1, 1, 1 }; one = [g_dev newBufferWithBytes:v length:16 options:MTLResourceStorageModeShared]; } + id cb = [g_q commandBuffer]; id e = [cb renderCommandEncoderWithDescriptor:rp]; + [e setRenderPipelineState:ps]; [e setVertexBuffer:one offset:0 atIndex:16]; [e drawPrimitives:MTLPrimitiveTypeTriangle vertexStart:0 vertexCount:3]; [e endEncoding]; + [cb commit]; [cb waitUntilCompleted]; + for (int i = 0; i < 2; i++) { + if (fm[i] == MTLPixelFormatRGBA8Unorm) { uint8_t p[4]; [t[i] getBytes:p bytesPerRow:16 fromRegion:MTLRegionMake2D(1, 1, 1, 1) mipmapLevel:0]; for (int c = 0; c < 4; c++) out[i][c] = p[c] / 255.0f; } + else { __fp16 p[4]; [t[i] getBytes:p bytesPerRow:32 fromRegion:MTLRegionMake2D(1, 1, 1, 1) mipmapLevel:0]; for (int c = 0; c < 4; c++) out[i][c] = p[c]; } + } +} +static void same_pixels(const char *name, MTLPixelFormat f0, int blend, MTLPixelFormat f1, MTLColorWriteMask m1) { + NSError *e = nil; + id a = pso_build(desc(f0, blend, f1, m1, 1), &e), b = [g_dev newRenderPipelineStateWithDescriptor:desc(f0, blend, f1, m1, 0) error:&e]; + checks++; if (!a || !b) { fprintf(stderr, "%s: no pipeline (%s)\n", name, e.localizedDescription.UTF8String); failed++; return; } + float pa[2][4], pb[2][4]; draw_with(a, f0, f1, pa); draw_with(b, f0, f1, pb); + checks++; if (memcmp(pa, pb, sizeof pa)) { fprintf(stderr, "%s: the specialized pipeline drew (%g %g %g %g | %g %g %g %g), the MTLDevice one (%g %g %g %g | %g %g %g %g)\n", name, + pa[0][0], pa[0][1], pa[0][2], pa[0][3], pa[1][0], pa[1][1], pa[1][2], pa[1][3], pb[0][0], pb[0][1], pb[0][2], pb[0][3], pb[1][0], pb[1][1], pb[1][2], pb[1][3]); failed++; } +} +static void depth_only(void) { + MTLRenderPipelineDescriptor *d = desc(MTLPixelFormatInvalid, 0, MTLPixelFormatInvalid, MTLColorWriteMaskNone, 1); + d.fragmentFunction = nil; NSError *err = nil; uint64_t legacy = g_pb_legacy; + id ps = pso_build(d, &err); + EXPECT("depth only: pipeline", 1, ps != nil); + EXPECT("depth only: legacy path", legacy + 1, g_pb_legacy); + if (!ps) return; + MTLTextureDescriptor *td = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatDepth32Float width:4 height:4 mipmapped:NO]; + td.usage = MTLTextureUsageRenderTarget; td.storageMode = MTLStorageModePrivate; + id t = [g_dev newTextureWithDescriptor:td]; + MTLRenderPassDescriptor *rp = [MTLRenderPassDescriptor new]; rp.depthAttachment.texture = t; + rp.depthAttachment.loadAction = MTLLoadActionClear; rp.depthAttachment.clearDepth = 1; rp.depthAttachment.storeAction = MTLStoreActionStore; + MTLDepthStencilDescriptor *ds = [MTLDepthStencilDescriptor new]; ds.depthWriteEnabled = YES; ds.depthCompareFunction = MTLCompareFunctionAlways; + float v[4] = {1, 1, 1, 1}; id vb = [g_dev newBufferWithBytes:v length:sizeof v options:MTLResourceStorageModeShared]; + id out = [g_dev newBufferWithLength:1024 options:MTLResourceStorageModeShared]; + id cb = [g_q commandBuffer]; id enc = [cb renderCommandEncoderWithDescriptor:rp]; + [enc setRenderPipelineState:ps]; [enc setDepthStencilState:[g_dev newDepthStencilStateWithDescriptor:ds]]; + [enc setVertexBuffer:vb offset:0 atIndex:16]; [enc drawPrimitives:MTLPrimitiveTypeTriangle vertexStart:0 vertexCount:3]; [enc endEncoding]; + id blit = [cb blitCommandEncoder]; + [blit copyFromTexture:t sourceSlice:0 sourceLevel:0 sourceOrigin:MTLOriginMake(0, 0, 0) sourceSize:MTLSizeMake(4, 4, 1) + toBuffer:out destinationOffset:0 destinationBytesPerRow:256 destinationBytesPerImage:1024]; [blit endEncoding]; + [cb commit]; [cb waitUntilCompleted]; + EXPECT("depth only: command completed", MTLCommandBufferStatusCompleted, cb.status); + EXPECT("depth only: rasterization wrote zero depth", 1, ((float *)out.contents)[65] == 0.0f); +} +static void concurrent_bases(int fail) { + nvn_hm_clear(&g_pbase, hm_rel); + ArchiveGateCompiler *gate = [ArchiveGateCompiler new]; gate->compiler = g_comp4; gate->fail = fail; + gate->entered = dispatch_semaphore_create(0); gate->release = dispatch_semaphore_create(0); g_comp4 = (id)gate; + NSMutableArray *ds = [NSMutableArray array]; + for (int i = 0; i < 8; i++) { + MTLRenderPipelineDescriptor *d = desc(MTLPixelFormatRGBA8Unorm, i & 1, MTLPixelFormatRGBA8Unorm, i, 1); + d.vertexDescriptor.layouts[16].stride = 512; [ds addObject:d]; + } + dispatch_group_t group = dispatch_group_create(); __block int good = 0; NSObject *lock = [NSObject new]; + for (MTLRenderPipelineDescriptor *d in ds) dispatch_group_async(group, dispatch_get_global_queue(QOS_CLASS_USER_INITIATED, 0), ^{ + @autoreleasepool { NSError *err = nil; id ps = pso_build(d, &err); + @synchronized(lock) { good += fail ? !ps && [err.domain isEqualToString:@"test"] : ps != nil; } + } + }); + EXPECT("concurrent: first compiler entered", 0, dispatch_semaphore_wait(gate->entered, dispatch_time(DISPATCH_TIME_NOW, 5 * NSEC_PER_SEC))); + // An unrelated key must complete while the first remains blocked. + MTLRenderPipelineDescriptor *other = [ds[0] copy]; other.vertexDescriptor = [other.vertexDescriptor copy]; other.vertexDescriptor.layouts[16].stride = 1024; + dispatch_group_t separate = dispatch_group_create(); + dispatch_group_async(separate, dispatch_get_global_queue(QOS_CLASS_USER_INITIATED, 0), ^{ @autoreleasepool { pso_build(other, NULL); } }); + EXPECT("concurrent: unrelated base progresses", 0, dispatch_group_wait(separate, dispatch_time(DISPATCH_TIME_NOW, 5 * NSEC_PER_SEC))); + dispatch_semaphore_signal(gate->release); + dispatch_group_wait(group, DISPATCH_TIME_FOREVER); dispatch_group_wait(separate, DISPATCH_TIME_FOREVER); + EXPECT("concurrent: all callers receive result", 8, good); + if (!fail) EXPECT("concurrent: one compile per base", 2, gate->calls); + EXPECT("concurrent: no unfinished builds", 0, g_pbase_pending.n); + g_comp4 = gate->compiler; + if (fail) { NSError *err = nil; EXPECT("concurrent: failed base can retry", 1, pso_build(ds[0], &err) != nil); } +} +int main(void) { @autoreleasepool { + char dir[] = "/tmp/t_pso_archive.XXXXXX"; if (!mkdtemp(dir)) abort(); + setenv("DM_PSO_ARCHIVE", dir, 1); + mtl_init(); g_q = [g_dev newCommandQueue]; NSError *err = nil; + NSString *src = @"#include \nusing namespace metal;\nstruct V { float4 p [[position]]; };\nstruct I { float4 a [[attribute(0)]]; };\n" + "vertex V vs(I in [[stage_in]], uint id [[vertex_id]]) { V o; float2 uv = float2((id << 1) & 2, id & 2); o.p = float4(uv * 2 - 1, 0, in.a.w); return o; }\n" + "struct O { float4 c0 [[color(0)]]; float4 c1 [[color(1)]]; };\nfragment O fs() { return { float4(0.5, 0.25, 1.0, 0.5), float4(0.2, 0.4, 0.6, 0.8) }; }\n"; + g_lib = [g_dev newLibraryWithSource:src options:nil error:&err]; + if (!g_lib) { fprintf(stderr, "%s\n", err.localizedDescription.UTF8String); return 1; } + g_lib2 = [g_dev newLibraryWithSource:[src stringByAppendingString:@"// another library: the same source would come back as the same library\n"] options:nil error:&err]; + // pipelines differing only in colour state share one base + same_pixels("RGBA8 blended, RGBA16F all channels", MTLPixelFormatRGBA8Unorm, 1, MTLPixelFormatRGBA16Float, MTLColorWriteMaskAll); + EXPECT("first pipeline: its base compiled", 1, g_pb_built); + same_pixels("RGBA16F, RGBA8 red and green", MTLPixelFormatRGBA16Float, 0, MTLPixelFormatRGBA8Unorm, MTLColorWriteMaskRed | MTLColorWriteMaskGreen); + same_pixels("RGBA8, RGBA8 no channel", MTLPixelFormatRGBA8Unorm, 0, MTLPixelFormatRGBA8Unorm, MTLColorWriteMaskNone); + EXPECT("other colour states: the same base", 1, g_pb_built); + EXPECT("other colour states: base reused", 2, g_pb_reused); + EXPECT("one specialization per pipeline", 3, g_pb_spec); + // the archive: flushed to a file, a later start takes the base from it without compiling + EXPECT("one base waits for the archive", 1, g_arch_new); + arch_flush(); + NSString *file = [@(dir) stringByAppendingPathComponent:@"00000.mtl4a"]; + EXPECT("archive file written", 1, [NSFileManager.defaultManager fileExistsAtPath:file]); + EXPECT("nothing left to write", 0, g_arch_new); + nvn_hm_clear(&g_pbase, hm_rel); // as a new start: no base in memory, the archive opened + g_arch4 = @[[g_dev newArchiveWithURL:[NSURL fileURLWithPath:file] error:&err]]; + uint64_t built = g_pb_built; + same_pixels("from the archive: RGBA8 blended, RGBA16F", MTLPixelFormatRGBA8Unorm, 1, MTLPixelFormatRGBA16Float, MTLColorWriteMaskAll); + EXPECT("base from the archive", 1, g_pb_archived); + EXPECT("no base compiled", built, g_pb_built); + // compaction: with more than PSO_ARCH_COMPACT files, the bases the prewarm takes from them are captured again; after the flush that follows, one file holds them and the old ones are gone + { + char cdir[] = "/tmp/t_pso_compact.XXXXXX"; if (!mkdtemp(cdir)) abort(); + NSString *saved = g_arch_dir; g_arch_dir = @(cdir); nvn_hm_clear(&g_pbase, hm_rel); g_arch4 = @[]; + NSMutableArray *descs = [NSMutableArray array]; + for (int i = 0; i <= PSO_ARCH_COMPACT; i++) { // one base per file: the stride of the layout tells them apart + MTLRenderPipelineDescriptor *cd = desc(MTLPixelFormatRGBA8Unorm, 0, MTLPixelFormatRGBA8Unorm, MTLColorWriteMaskAll, 1); + cd.vertexDescriptor.layouts[16].stride = 16 * (i + 2); [descs addObject:cd]; + if (!pso_build(cd, &err)) { fprintf(stderr, "compaction: no pipeline %d\n", i); failed++; } + arch_flush(); + } + NSArray *names = [[NSFileManager.defaultManager contentsOfDirectoryAtPath:@(cdir) error:nil] sortedArrayUsingSelector:@selector(compare:)]; + EXPECT("compaction: one file per flush", PSO_ARCH_COMPACT + 1, names.count); + NSMutableArray> *open = [NSMutableArray array]; NSMutableArray *paths = [NSMutableArray array]; + for (NSString *n in names.reverseObjectEnumerator) { NSString *f = [@(cdir) stringByAppendingPathComponent:n]; [paths addObject:f]; [open addObject:[g_dev newArchiveWithURL:[NSURL fileURLWithPath:f] error:&err]]; } + nvn_hm_clear(&g_pbase, hm_rel); g_arch4 = open; // as a new start with that many files + g_compact_opts = [MTL4CompilerTaskOptions new]; g_compact_opts.lookupArchives = open; g_compact_files = paths; g_compacted = 0; + uint64_t arch0 = g_pb_archived, built0 = g_pb_built; + atomic_store(&g_warm_running, 1); + for (MTLRenderPipelineDescriptor *cd in descs) pso_build(cd, &err); + dispatch_sync(g_arch_q, ^{ arch_flush(); }); // the timer fires while prewarm still has other work + EXPECT("compaction: timed flush keeps recaptured bases pending", descs.count, g_arch_new); + EXPECT("compaction: timed flush adds no partial file", PSO_ARCH_COMPACT + 1, [NSFileManager.defaultManager contentsOfDirectoryAtPath:@(cdir) error:nil].count); + atomic_store(&g_warm_running, 0); + EXPECT("compaction: every base from the archive", arch0 + descs.count, g_pb_archived); + EXPECT("compaction: none compiled", built0, g_pb_built); + EXPECT("compaction: every base captured again", descs.count, g_compacted); + arch_flush_now(); dispatch_sync(g_arch_q, ^{}); + names = [NSFileManager.defaultManager contentsOfDirectoryAtPath:@(cdir) error:nil]; + EXPECT("compaction: one file left", 1, names.count); + nvn_hm_clear(&g_pbase, hm_rel); g_compact_opts = nil; g_compacted = 0; + g_arch4 = @[[g_dev newArchiveWithURL:[NSURL fileURLWithPath:[@(cdir) stringByAppendingPathComponent:names.firstObject]] error:&err]]; + arch0 = g_pb_archived; built0 = g_pb_built; + for (MTLRenderPipelineDescriptor *cd in descs) pso_build(cd, &err); + EXPECT("compaction: the new file holds every base", arch0 + descs.count, g_pb_archived); + EXPECT("compaction: none compiled from the new file", built0, g_pb_built); + [NSFileManager.defaultManager removeItemAtPath:@(cdir) error:nil]; g_arch_dir = saved; g_arch4 = @[]; + } + // a function with no Metal 4 descriptor: the MTLDevice path + uint64_t legacy = g_pb_legacy; + id l = pso_build(desc(MTLPixelFormatRGBA8Unorm, 0, MTLPixelFormatRGBA8Unorm, MTLColorWriteMaskAll, 0), &err); + EXPECT("unregistered function: a pipeline", 1, l != nil); + EXPECT("unregistered function: MTLDevice build", legacy + 1, g_pb_legacy); + depth_only(); + concurrent_bases(0); + concurrent_bases(1); + // key normalization + pso_key k; memset(&k, 0, sizeof k); k.blend_en = 0x5; k.dfmt = MTLPixelFormatDepth32Float; k.sfmt = MTLPixelFormatStencil8; + for (int i = 0; i < 8; i++) { k.blend[i].src_rgb = 5; k.blend[i].dst_rgb = 6; } + pso_key_norm(&k); + EXPECT("blend of an enabled target kept", 5, k.blend[0].src_rgb); + EXPECT("blend of an enabled target kept (2)", 6, k.blend[2].dst_rgb); + EXPECT("blend of a disabled target dropped", 0, k.blend[1].src_rgb); + EXPECT("blend of a disabled target dropped (7)", 0, k.blend[7].dst_rgb); + EXPECT("depth format kept (an MTLDevice pipeline names it)", MTLPixelFormatDepth32Float, k.dfmt); + EXPECT("stencil format kept", MTLPixelFormatStencil8, k.sfmt); + [NSFileManager.defaultManager removeItemAtPath:@(dir) error:nil]; + printf("t_pso_archive: %d checks, %d failed\n", checks, failed); + return failed != 0; +} } diff --git a/docs/development.md b/docs/development.md index 67c14e3..64acdae 100644 --- a/docs/development.md +++ b/docs/development.md @@ -203,6 +203,19 @@ The lines name shaders by content key and hold no compiled code and nothing pers Record on the game version you ship. At launch the log names each list it read and its line count (`pipeline prewarm list …: N lines`), or says that `pso_user.tsv` does not exist yet. During rendering, a pipeline miss compiles and caches the demanded pipeline before scanning subsequent draws for speculative work. The scan admits at most two new jobs, with at most two speculative compilations outstanding; completed predictions remain available by exact key. This limit does not apply to the startup warmer. `t_pso_pending` checks scheduling and ownership with a gated compiler; it does not establish that the two-job budget is optimal on every device or remove the cost of compiling an unseen pipeline. +### Pipeline bases and the pipeline archive + +Concurrent requests for the same base share its in-progress build and result; unrelated bases can compile in parallel. Rasterizing pipelines without a fragment function use the `MTLDevice` path so depth-only draws keep their depth writes: Metal 4 requires a fragment function when rasterization is enabled. Transform-feedback pipelines, whose rasterization is disabled, can still use Metal 4. + +Render pipelines are built through Metal 4's `MTL4Compiler` (`pso_build` in `dm/gfx/nvn/nvn_mtl.m`). `pso_desc` still describes each pipeline; from it the host builds a *base* whose colour attachments (format, blending, write mask) are left unspecialized, once per base, and makes every pipeline from its base by specialization, which takes a few microseconds instead of the 3–5 ms of a pipeline build. Metal 4 pipelines name no depth or stencil format, so one base serves every depth/stencil attachment. Breath of the Wild's 4,829 recorded pipeline keys came from 1,818 bases; 2,845 of the keys differed only in the blend factors of targets whose blending was off, which the key no longer holds (`pso_key_norm`). + +The bases are kept in a per-game pipeline archive in `pso_archive/` beside `pso_user.tsv`. Each file holds what one flush captured (`NNNNN.mtl4a`). A flush runs after the prewarm and 10 s after a burst of new bases. At start the host opens every file, and a base found in one is taken from it directly, in about 0.06 ms where building it takes 10–80 ms. Only bases no file holds are compiled, so each file holds new bases only. The serializer must capture binaries only: with descriptor capture as well, it writes no archive and reports no error. A flush and a compile must not overlap (a compile that finished during a flush crashed in Metal), so compiles hold a read lock and a flush the write lock (`g_arch_rw`). + +Every start adds a file or more: Breath of the Wild wrote 5–7 per start, and each file a base is not in costs a lookup about 0.06 ms. With more than 8 files (`PSO_ARCH_COMPACT`) the archive is compacted: the bases the prewarm takes from the old files are taken again through the compiler, with those files as its lookup archives, which captures them; the flush after the prewarm writes them into one file, and the old files are removed (`arch_compact_finish`). Bases the prewarm did not take (no `pso_user.tsv` line names them) go with the old files and are built again when drawn. The log says `pipeline archive compacted: N files gave M bases …`. + +The log shows `pipeline archive …: N files, … MB, opened in … ms` at start, `pipeline archive: N new pipeline bases written to …` after each flush, and after the prewarm a line counting the bases compiled, taken from the archive and reused. `DM_PSO_ARCHIVE=off` turns the archive off; `DM_PSO_ARCHIVE=` puts it elsewhere. `DM_PSO_MTL4=0` builds every pipeline through the `MTLDevice` as before, with no bases and no archive. The Metal 4 path is on by default on the Mac only. On iOS it stays off unless `DM_PSO_MTL4=1`: the archive's memory is not measured against an iPhone's limit yet, and an archive cannot be made offline for an iPhone, because `metal-tt` rejects the pipeline script the Metal 4 serializer writes. iOS is a later step. Pipelines whose functions have no Metal 4 descriptor (the diagnostic `DM_NVN_HILITE` and `DM_NVN_MAGSCAN` overrides) always take the `MTLDevice` path. + +During compaction, timed flushes defer while prewarm runs. Once it finishes, one archive-queue task flushes the recaptured bases and removes the old files. The pending count and serializer capture are protected by the same compile/flush lock, so a flush consumes the count for the bases it writes. ## Power