feat(turnip): release cached BOs on OS memory pressure

The retire queue and the BO cache grow with demand instead of holding
256 and 1024 entries. The cache gives everything back when libdispatch
reports memory pressure (DISPATCH_SOURCE_TYPE_MEMORYPRESSURE, warning
or critical) and when an allocation or GPU range cannot be had; the 2 s
idle eviction, the 100 ms kernel poll and the seven-eighths VA rule are
gone. Fence waits for idle and for ring space have no fixed bound: the
kernel's hangcheck ends a hang. TU_DARWIN_BO_CACHE=0 turns the cache off
for the host checks of the release paths.

patches/mesa/turnip-darwin-kmd.patch

@@ -274,7 +274,7 @@ new file mode 100644
 index 0000000..b03c6ee
 --- /dev/null
 +++ b/src/freedreno/vulkan/tu_knl_darwin.cc
-@@ -0,0 +1,1554 @@
+@@ -0,0 +1,1566 @@
 +/*
 + * Native Darwin kernel backend for Turnip: talks to the NativeAdreno IOKit user client (NativeAdrenoABI.h, version 1).
 + * Modelled on tu_knl_kgsl.cc (no libdrm) and tu_knl_drm_msm.cc (the client chooses GPU addresses).
@@ -290,11 +290,13 @@ index 0000000..b03c6ee
 +#include <unistd.h>
 +
 +#include <IOKit/IOKitLib.h>
++#include <dispatch/dispatch.h>
 +#include <mach/mach.h>
 +
 +#include "NativeAdrenoABI.h"
 +
 +#include "util/os_time.h"
++#include "util/u_atomic.h"
 +#include "util/rb_tree.h"
 +#include "util/timespec.h"
 +#include "util/u_debug.h"
@@ -334,8 +336,7 @@ index 0000000..b03c6ee
 +
 +/* Preserve the native handle, CPU view and IOVA until the captured ring fence
 + * actually completed. An unrelated later submission never extends this lease.
-+ * The queue is bounded; pressure may wait, but ordinary destruction does not. */
-+enum { DARWIN_RETIRED_BOS = 256 };
++ * The queue grows with what is waiting; destruction never waits. */
 +struct darwin_retired_bo {
 +   uint64_t handle, iova, size;
 +   void *mapping;
@@ -347,13 +348,12 @@ index 0000000..b03c6ee
 + * allocation of its class: one BoAlloc + BoMap (the SMMU table update, about a millisecond) and one BoFree + unmap saved per reuse.
 + * Classes are the size rounded up to a quarter of its power of two (4 KiB steps up to 16 KiB). Replayable BOs (client addresses) and
 + * host imports never enter it. */
-+enum { DARWIN_CACHE_ENTRIES = 1024 };
-+/* No byte limit: as Metal's own resource cache does, the cache gives memory back when the system asks and when it goes unused. Every
-+ * Metal client draws from the one kernel pool, so the kernel reports pressure on it (NativeAdrenoParamsPoolPressure in Query's flags)
-+ * and each client then empties its cache; an entry nobody took for DARWIN_CACHE_IDLE_NS is released anyway. */
-+enum : uint64_t { DARWIN_CACHE_IDLE_NS = 2000000000ull, DARWIN_PRESSURE_POLL_NS = 100000000ull };
++/* No byte or entry limit: as Metal's own resource cache does, the cache gives everything back when the system reports memory pressure
++ * (DISPATCH_SOURCE_TYPE_MEMORYPRESSURE, warning or critical, the signal Apple's frameworks purge on) and when an allocation or a GPU range
++ * cannot be had (then the allocation is tried again). */
++static volatile uint32_t darwin_memory_pressure;   /* bumped by the memory-pressure source; each device flushes once per change */
 +struct darwin_cached_bo {
-+   uint64_t handle, iova, size, stamp, cached_ns;
++   uint64_t handle, iova, size, stamp;
 +   void *mapping;
 +   uint32_t token;
 +   bool read_only;
@@ -379,13 +379,14 @@ index 0000000..b03c6ee
 +
 +struct darwin_retire_state {
 +   pthread_mutex_t lock;
-+   struct darwin_retired_bo entries[DARWIN_RETIRED_BOS];
-+   unsigned head, count;
++   struct darwin_retired_bo *entries;   /* a ring of `capacity` records, grown when full */
++   unsigned capacity, head, count;
 +   VkResult failure;
 +   uint64_t enqueued, reaped, pressure_waits;
-+   struct darwin_cached_bo cache[DARWIN_CACHE_ENTRIES];
-+   unsigned cache_count;
-+   uint64_t cache_bytes, cache_stamp, pressure_polled_ns;
++   struct darwin_cached_bo *cache;
++   unsigned cache_count, cache_capacity;
++   uint64_t cache_bytes, cache_stamp;
++   uint32_t pressure_seen;
 +   struct darwin_bo_cache_stats cache_stats;
 +   struct darwin_va va;
 +};
@@ -834,15 +835,15 @@ index 0000000..b03c6ee
 +static bool
 +darwin_cache_put(struct tu_device *dev, struct darwin_retire_state *retire, const struct darwin_retired_bo *bo)
 +{
-+   while (retire->cache_count == DARWIN_CACHE_ENTRIES) {
-+      unsigned oldest = 0;
-+      for (unsigned i = 1; i < retire->cache_count; ++i)
-+         if (retire->cache[i].stamp < retire->cache[oldest].stamp)
-+            oldest = i;
-+      if (darwin_cache_evict(dev, retire, oldest) != VK_SUCCESS)
-+         return false;
++   if (retire->cache_count == retire->cache_capacity) {
++      const unsigned capacity = retire->cache_capacity ? retire->cache_capacity * 2 : 64;
++      struct darwin_cached_bo *cache = (struct darwin_cached_bo *)realloc(retire->cache, capacity * sizeof(*cache));
++      if (!cache)
++         return false;   /* the BO is released instead */
++      retire->cache = cache;
++      retire->cache_capacity = capacity;
 +   }
-+   retire->cache[retire->cache_count++] = {bo->handle, bo->iova, bo->size, ++retire->cache_stamp, os_time_get_nano(), bo->mapping, bo->token,
++   retire->cache[retire->cache_count++] = {bo->handle, bo->iova, bo->size, ++retire->cache_stamp, bo->mapping, bo->token,
 +                                           bo->read_only};
 +   retire->cache_bytes += bo->size;
 +   if (retire->cache_bytes > retire->cache_stats.bytes_peak)
@@ -888,28 +889,31 @@ index 0000000..b03c6ee
 +   return any;
 +}
 +
-+/* Called under retire->lock, at most every DARWIN_PRESSURE_POLL_NS: entries idle for DARWIN_CACHE_IDLE_NS go back, and all of them when
-+ * the kernel reports pressure on the shared pool. */
++/* Called under retire->lock: once per memory-pressure notification the whole cache goes back. */
 +static void
 +darwin_cache_trim(struct tu_device *dev, struct darwin_retire_state *retire)
 +{
-+   const uint64_t now = os_time_get_nano();
-+   if (now - retire->pressure_polled_ns < DARWIN_PRESSURE_POLL_NS)
++   const uint32_t pressure = p_atomic_read(&darwin_memory_pressure);
++   if (pressure == retire->pressure_seen)
 +      return;
-+   retire->pressure_polled_ns = now;
-+   /* This process's own GPU addresses are the other shared resource: cached BOs keep theirs, so past seven eighths of the range the
-+    * cache goes back too (the render server's mappings climbed through its whole range while its cache kept them, i-2740ce770). */
-+   const uint64_t va_size = dev->physical_device->va_size;
-+   const bool va_pressure = va_size && retire->va.used > va_size - va_size / 8;
-+   NativeAdrenoAbiParams params = {};
-+   if (va_pressure || (dw_call(NativeAdrenoMethodQuery, NULL, 0, &params, sizeof(params)) == KERN_SUCCESS &&
-+                       (params.flags & NativeAdrenoParamsPoolPressure))) {
-+      darwin_cache_flush(dev, retire);
-+      return;
-+   }
-+   for (unsigned i = retire->cache_count; i-- > 0;)
-+      if (now - retire->cache[i].cached_ns >= DARWIN_CACHE_IDLE_NS)
-+         darwin_cache_evict(dev, retire, i);
++   retire->pressure_seen = pressure;
++   darwin_cache_flush(dev, retire);
++}
++
++/* The process's memory-pressure source, made once: libdispatch delivers the kernel's memorystatus levels to it. */
++static void
++darwin_watch_memory_pressure(void)
++{
++   static pthread_once_t once = PTHREAD_ONCE_INIT;
++   pthread_once(&once, [] {
++      dispatch_source_t source = dispatch_source_create(DISPATCH_SOURCE_TYPE_MEMORYPRESSURE, 0,
++                                                        DISPATCH_MEMORYPRESSURE_WARN | DISPATCH_MEMORYPRESSURE_CRITICAL,
++                                                        dispatch_get_global_queue(QOS_CLASS_UTILITY, 0));
++      if (!source)
++         return;
++      dispatch_source_set_event_handler_f(source, [](void *) { p_atomic_inc(&darwin_memory_pressure); });
++      dispatch_activate(source);
++   });
 +}
 +
 +/* Called under retire->lock. All fence polls are nonblocking. Failed/uncertain
@@ -928,7 +932,7 @@ index 0000000..b03c6ee
 +   const unsigned total = retire->count;
 +   unsigned kept = 0;
 +   for (unsigned i = 0; i < total; ++i) {
-+      struct darwin_retired_bo *bo = &retire->entries[(retire->head + i) % DARWIN_RETIRED_BOS];
++      struct darwin_retired_bo *bo = &retire->entries[(retire->head + i) % retire->capacity];
 +      bool ready = true;
 +      if (bo->fence) {
 +         if (have_pending_fence && bo->fence == pending_fence) {
@@ -957,12 +961,12 @@ index 0000000..b03c6ee
 +      if (ready) {
 +         ++retire->reaped;
 +      } else {
-+         retire->entries[(retire->head + kept) % DARWIN_RETIRED_BOS] = *bo;
++         retire->entries[(retire->head + kept) % retire->capacity] = *bo;
 +         ++kept;
 +      }
 +   }
 +   for (unsigned i = kept; i < total; ++i)
-+      retire->entries[(retire->head + i) % DARWIN_RETIRED_BOS] = {};
++      retire->entries[(retire->head + i) % retire->capacity] = {};
 +   retire->count = kept;
 +   return VK_SUCCESS;
 +}
@@ -1017,7 +1021,9 @@ index 0000000..b03c6ee
 +      return collected;
 +
 +   struct darwin_retire_state *retire = (struct darwin_retire_state *)dev->darwin_retire;
-+   const bool cacheable = !(flags & TU_BO_ALLOC_REPLAYABLE);
++   /* TU_DARWIN_BO_CACHE=0 turns the cache off (the host checks of the release paths need every free to reach the kernel). */
++   static const bool cache_enabled = debug_get_bool_option("TU_DARWIN_BO_CACHE", true);
++   const bool cacheable = cache_enabled && !(flags & TU_BO_ALLOC_REPLAYABLE);
 +   const bool read_only = !!(flags & TU_BO_ALLOC_GPU_READ_ONLY);
 +   uint64_t alloc_size = cacheable ? darwin_size_class(size) : size;
 +
@@ -1258,7 +1264,7 @@ index 0000000..b03c6ee
 +   uint32_t last = dw.last_fence;
 +   pthread_mutex_unlock(&dw.lock);
 +   if (last)
-+      return dw_wait_fence_locked_free(dev, last, os_time_get_nano() + 5000000000ull);
++      return dw_wait_fence_locked_free(dev, last, UINT64_MAX);   /* the kernel's hangcheck ends a hang */
 +   return VK_SUCCESS;
 +}
 +
@@ -1284,30 +1290,33 @@ index 0000000..b03c6ee
 +   struct darwin_retire_state *retire = (struct darwin_retire_state *)dev->darwin_retire;
 +   pthread_mutex_lock(&retire->lock);
 +   darwin_reap_locked(dev, retire);
-+   if (retire->count == DARWIN_RETIRED_BOS && retire->failure == VK_SUCCESS) {
-+      ++retire->pressure_waits;
-+      pthread_mutex_unlock(&retire->lock);
-+      VkResult waited = darwin_wait_idle(dev);
-+      pthread_mutex_lock(&retire->lock);
-+      if (waited != VK_SUCCESS)
-+         retire->failure = vk_device_set_lost(&dev->vk, "deferred BO pressure wait failed");
-+      else
-+         darwin_reap_locked(dev, retire);
++   if (retire->count == retire->capacity && retire->failure == VK_SUCCESS) {
++      /* The ring grows, its records in order from the head. */
++      const unsigned capacity = retire->capacity ? retire->capacity * 2 : 64;
++      struct darwin_retired_bo *entries = (struct darwin_retired_bo *)calloc(capacity, sizeof(*entries));
++      if (entries) {
++         for (unsigned i = 0; i < retire->count; ++i)
++            entries[i] = retire->entries[(retire->head + i) % retire->capacity];
++         free(retire->entries);
++         retire->entries = entries;
++         retire->capacity = capacity;
++         retire->head = 0;
++      }
 +   }
-+   if (retire->count < DARWIN_RETIRED_BOS) {
++   if (retire->count < retire->capacity) {
 +      pthread_mutex_lock(&dw.lock);
 +      uint32_t fence = dw.last_fence;
 +      pthread_mutex_unlock(&dw.lock);
-+      unsigned slot = (retire->head + retire->count) % DARWIN_RETIRED_BOS;
++      unsigned slot = (retire->head + retire->count) % retire->capacity;
 +      retire->entries[slot] = {bo->darwin_handle, bo->iova, bo->darwin_alloc_size ? bo->darwin_alloc_size : bo->size, bo->map,
 +                               bo->darwin_token, fence, bo->darwin_alloc_size != 0, bo->darwin_read_only};
 +      ++retire->count;
 +      ++retire->enqueued;
 +      darwin_reap_locked(dev, retire);
-+   } else {
++   } else if (retire->failure == VK_SUCCESS) {
 +      /* No reusable range is reported on error. The kernel still owns the handle
 +       * and view; connection teardown must drain them, even if this record cannot fit. */
-+      retire->failure = vk_device_set_lost(&dev->vk, "deferred BO queue remains full; retaining native allocation");
++      retire->failure = vk_device_set_lost(&dev->vk, "deferred BO queue cannot grow; retaining native allocation");
 +   }
 +   pthread_mutex_unlock(&retire->lock);
 +
@@ -1532,8 +1541,8 @@ index 0000000..b03c6ee
 +   dw.last_fence = fence;
 +}
 +
-+/* One kernel submission of up to NativeAdrenoAbiMaxBuffers indirect buffers; waits for the oldest job when all eight kernel job slots
-+ * are taken. */
++/* One kernel submission of up to NativeAdrenoAbiMaxBuffers indirect buffers; while the kernel's ring has no room (kIOReturnBusy, as msm
++ * waits for ring space) it waits for this client's oldest job, or yields when none is in flight. */
 +static VkResult
 +darwin_submit_chunk(struct tu_queue *queue, const NativeAdrenoAbiSubmitBuffer *entries,
 +                    unsigned count, uint32_t *fence)
@@ -1545,7 +1554,7 @@ index 0000000..b03c6ee
 +   for (unsigned i = 0; i < count; i++)
 +      req.buffers[i] = entries[i];
 +
-+   for (unsigned attempt = 0; attempt < 64; attempt++) {
++   for (;;) {
 +      NativeAdrenoAbiSubmitReply reply = {};
 +      kern_return_t kr = dw_call(NativeAdrenoMethodSubmit, &req, sizeof(req), &reply, sizeof(reply));
 +      if (kr == KERN_SUCCESS) {
@@ -1559,11 +1568,10 @@ index 0000000..b03c6ee
 +      uint32_t oldest = dw.inflight_count ? dw.inflight[0] : 0;
 +      pthread_mutex_unlock(&dw.lock);
 +      if (!oldest)
-+         usleep(100);
-+      else if (dw_wait_fence_locked_free(dev, oldest, os_time_get_nano() + 5000000000ull) != VK_SUCCESS)
-+         return vk_device_set_lost(&dev->vk, "submit: GPU did not retire a job");
++         sched_yield();
++      else if (dw_wait_fence_locked_free(dev, oldest, UINT64_MAX) != VK_SUCCESS)
++         return vk_device_set_lost(&dev->vk, "submit: waiting for ring space failed");
 +   }
-+   return vk_device_set_lost(&dev->vk, "submit: no free job slot");
 +}
 +
 +static VkResult
@@ -1641,6 +1649,8 @@ index 0000000..b03c6ee
 +   }
 +   dev->darwin_retire = retire;
 +   darwin_stats_state = retire;
++   darwin_watch_memory_pressure();
++   retire->pressure_seen = p_atomic_read(&darwin_memory_pressure);
 +   return VK_SUCCESS;
 +}
 +
@@ -1663,6 +1673,8 @@ index 0000000..b03c6ee
 +      rb_tree_foreach_safe(struct darwin_va_hole, hole, &retire->va.address, by_address)
 +         free(hole);
 +      pthread_mutex_destroy(&retire->lock);
++      free(retire->entries);
++      free(retire->cache);
 +      vk_free(&dev->vk.alloc, retire);
 +      dev->darwin_retire = NULL;
 +   }

scripts/check-turnip-darwin-host.py

@@ -77,13 +77,15 @@ subprocess.run(['xcrun', 'clang++', '-std=c++23', '-O1', '-Wall', '-Wextra', '-W
 deferred_env = dict(env, TU_DARWIN_RETIRE_TRACE='1')
 deferred_env.pop('HOSTSIM_FAIL_USER_UNMAP', None)
 for mode in ['normal', 'busy', 'fence-error', 'free-error', 'lease-error', 'view-error', 'map-cleanup-error', 'pressure']:
-    test = subprocess.run([str(deferred), str(library), mode], env=deferred_env, capture_output=True, text=True, timeout=60)
+    # The release paths need every free to reach the kernel: only the normal mode runs with the BO cache on.
+    mode_env = dict(deferred_env, TU_DARWIN_BO_CACHE='1' if mode == 'normal' else '0')
+    test = subprocess.run([str(deferred), str(library), mode], env=mode_env, capture_output=True, text=True, timeout=60)
     (out / ('deferred-' + mode + '.log')).write_text(test.stdout + test.stderr)
     verdict = [line for line in test.stdout.splitlines() if line.startswith('PASS:')]
     if test.returncode or not verdict:
         sys.exit('Deferred BO %s failed (exit %d); see %s' % (mode, test.returncode, out / ('deferred-' + mode + '.log')))
-    if mode == 'pressure' and 'pressure=1' not in test.stderr:
-        sys.exit('Deferred BO pressure control did not reach its bounded queue limit')
+    if mode == 'pressure' and 'pressure=0' not in test.stderr:
+        sys.exit('Deferred BO pressure control waited for the GPU instead of growing its queue')
     print(verdict[0])
 # Driver cache controls compile Adreno code but submit/execute no shader.
 subprocess.run([sys.executable, ROOT / 'scripts/check-turnip-pipeline-cache-host.py'], check=True)