Rescue stalled display links with bounded retries (#198)

Switch active-app presents to drawable acquisition after 500 ms of display-link
silence, then retry with exponential backoff capped at 60 seconds. Preserve the
requested pacing mode, synchronize rescue state, and prevent focus timestamp
underflow from bypassing the foreground grace period.

Add CPU regression coverage in CI, a real Metal layer ownership check, and the
mechanism reference with upstream provenance and validation limits.

Validated: 43 CPU rescue checks; 47 checks with real Metal drawable acquisition;
34 present/crop and 10 pacing checks; Wonder Mac host build; all 133 iOS source
compile checks; documentation tests. Windowed gameplay recovery and iPhone
validation remain outstanding.

Co-authored-by: Aleksandr Svidchenkov <a.svidchenkov@gmail.com>

.github/workflows/tests.yml

@@ -48,7 +48,7 @@ jobs:
         run: |
           mkdir -p logs
           cd dm/gfx/tests
-          tests="t_dirty_slots t_dirty_race t_pace t_fstep t_hmap t_cover t_pso_skey t_refl t_rt_range_ro"
+          tests="t_dirty_slots t_dirty_race t_pace t_fstep t_hmap t_cover t_pso_skey t_refl t_rt_range_ro t_present_rescue"
           make GAME="$GAME" -j3 $tests 2>&1 | tee ../../../logs/gfx-build.log
           failed=0
           for t in $tests; do

dm/gfx/nvn/nvn_mtl.m

@@ -8830,6 +8830,13 @@ static double g_link_vperiod, g_link_vhalf;   // virtual ticks (link_range): sec
 #define STALL_NS 100000000ull
 #define STALL_RED_NS 2000000000ull
 static uint64_t g_stall_t0, g_stall_drop, g_stall_n; static int g_stall_red, g_stall_logged;
+// When the display link stays silent for LINK_RESCUE_NS while the app is active, presents stop waiting for it: mode 0 (the present thread takes drawables itself), so the
+// picture keeps moving instead of standing still on dropped frames (d2r-mobile afad65c, via MacNativeGames cf1e629: an iPhone's link stopped while active and the picture
+// froze for 80 s; macOS also pauses the link of a window it takes for covered). The link is tried again after g_rescue_back_ns (link_sync), which doubles on each new
+// stall up to 60 s. The requested g_dm_present_link stays unchanged; all rescue state changes take g_pq_mu, and link_sync handles the link object on the enqueue thread.
+#define LINK_RESCUE_NS 500000000ull
+static _Atomic int g_rescue_mode;   // Also read by link_on without g_pq_mu, to stop pacing on silent ticks immediately
+static uint64_t g_rescue_back_at, g_rescue_back_ns = 3000000000ull, g_rescue_n;
 // The display link watchdog only recognizes the "active" state: when the App loses active status (pulling down Control Center / notifications, incoming call, system popup) and in the background the system does not call back anyway, and the picture is covered too -- the frame-drop fallback stays as before,
 // but it is not counted as stuck and no failure is reported; the stall is measured from the later of "the most recent callback / the most recent activation" (A16 logs: 8–17 seconds inactive and 17 seconds in the background were both reported as "stuck in foreground")
 __attribute__((weak)) int dm_focus_now(void) { return 1; }   // The host's hle_oe.c provides the strong definition; when graphics unit tests do not link the host it is treated as always active
@@ -8849,12 +8856,12 @@ __attribute__((weak)) uint64_t dm_focus_in_since(void) { return 0; }
         next += vp; tol += 2 * g_link_vhalf;   // a tick lies up to half a refresh off its slot, so two of them up to one refresh nearer each other than the slots
     }
     pthread_mutex_lock(&g_link_mu);
-    g_link_tick_ns = now_ns();
+    __atomic_store_n(&g_link_tick_ns, now_ns(), __ATOMIC_RELAXED);
     g_link_tick_n++;
     pthread_cond_broadcast(&g_link_cv);
     pthread_mutex_unlock(&g_link_mu);
     pthread_mutex_lock(&g_pq_mu);
-    if (!g_dm_present_link || g_pq_read == g_pq_in) { pthread_mutex_unlock(&g_pq_mu); return; }
+    if (!g_dm_present_link || g_rescue_mode || g_pq_read == g_pq_in) { pthread_mutex_unlock(&g_pq_mu); return; }
     // The previous frame's present interval was >= 2 (period > 25 ms): keep it on screen for its whole period before taking the next one, as the Switch flips only every N vertical syncs. This also holds while the link runs at 60 Hz (Mario 1.2.1 sets 2 for single frames, see nvn_game_fps in hle_nvn.c);
     //   before, every tick took the newest frame, so that frame, carrying the extra time step the game advanced for it, was shown for only 16.7 ms and the game looked like it jumped ahead
     if (g_link_hold_until > 0 && update.targetPresentationTimestamp < g_link_hold_until - tol) { g_link_held++; pthread_mutex_unlock(&g_pq_mu); return; }
@@ -8885,7 +8892,7 @@ __attribute__((weak)) uint64_t dm_focus_in_since(void) { return 0; }
 
 static CAMetalDisplayLink *g_link; static CFRunLoopRef g_link_rl; static int g_link_started;
 static int g_link_owns_layer, g_classic_busy;   // Under g_pq_mu: nextDrawable throws while the layer has the display link object attached, so the present thread that takes its own drawable and the display link object are mutually exclusive
-static int link_on(void) { return g_dm_present_link && g_link != nil; }
+static int link_on(void) { return g_dm_present_link && !atomic_load(&g_rescue_mode) && g_link != nil; }
 // Screens whose refresh rate is not a multiple of the link's (144 or 165 Hz against 60 or 30): asked for 60 Hz, the system fires the link on every third refresh of a 165 Hz screen,
 // 55 Hz, and a game paced by the link runs at 55 fps (27.5 at 30). There the link runs at the screen's rate and only the refreshes nearest a steady clock of the wanted rate count as
 // ticks (g_link_vperiod, NvMetalDisplayLinkDelegate): 60 a second on average, each frame shown on the refreshes nearest its 1/60 s slot. DM_LINK_VTICK=0 keeps the system's choice.
@@ -8922,8 +8929,16 @@ static void *link_main(void *arg) {   // Display link thread: started when switc
     return NULL;
 }
 static void link_sync(void) {   // Called every frame by the game's present thread: aligns the display link object's state to g_dm_present_link and the power-saving mode (can change at run time)
-    static int applied; int want = g_dm_present_link;
-    if (want < 0 || want > 2) { g_dm_present_link = applied == 3 ? 1 : applied; return; }
+    static int applied;
+    pthread_mutex_lock(&g_pq_mu);
+    int want = g_dm_present_link;
+    if (want < 0 || want > 2) { g_dm_present_link = applied == 3 ? 1 : applied; pthread_mutex_unlock(&g_pq_mu); return; }
+    if (g_rescue_mode && (want != g_rescue_mode || now_ns() >= g_rescue_back_at)) {
+        g_rescue_mode = 0; g_stall_t0 = 0; __atomic_store_n(&g_link_tick_ns, now_ns(), __ATOMIC_RELAXED);
+        MLOG("display link: rescue ended, trying pacing mode %d (frame %llu)", want, (unsigned long long)atomic_load(&g_cur_frame));
+    }
+    if (g_rescue_mode) want = 0;
+    pthread_mutex_unlock(&g_pq_mu);
     if (want && mtl_fps_target() == 30) want = 3;   // Power-saving mode: display link at 30 Hz
     if (want) front_screen_match(want == 3 ? 30 : want == 2 ? 120 : 60);
     // The screen's rate changed (switched by front_screen_match, or another screen): the range again, virtual ticks or none. Taken only once the link object exists, so a change
@@ -9155,9 +9170,15 @@ void mtl_present_wait(uint64_t tk) {
         if (!g_stall_t0) { g_stall_t0 = last ? last : now; g_stall_drop = 0; g_stall_red = 0; g_stall_logged = 0; g_stall_n++; }
         g_stall_drop += n;
         int active = dm_focus_now() == 1; uint64_t since = dm_focus_in_since(), from = g_stall_t0 > since ? g_stall_t0 : since;
-        if (active && !g_stall_logged && now - from >= STALL_NS) { g_stall_logged = 1;
+        uint64_t elapsed = now >= from ? now - from : 0;   // Focus can return after the earlier now sample, including while waiting for an in-flight present
+        if (active && !g_stall_logged && elapsed >= STALL_NS) { g_stall_logged = 1;
             MLOG("display link silent for %.0f ms (active): dropping %llu queued frames not yet displayed, the game continues (stall #%llu; back to normal when the link returns)", (now - from) / 1e6, (unsigned long long)n, (unsigned long long)g_stall_n); }
-        if (active && !g_stall_red && now - from >= STALL_RED_NS) { g_stall_red = 1; MLOG("[RED] display link silent for %.1f s while active; presenting relies entirely on fallback frame drops, the picture is frozen", (now - from) / 1e9); }
+        if (active && !g_rescue_mode && g_dm_present_link && elapsed >= LINK_RESCUE_NS) {
+            g_rescue_mode = g_dm_present_link; g_rescue_n++; g_rescue_back_at = now + g_rescue_back_ns;
+            MLOG("display link silent for %.0f ms while active: the present thread takes drawables itself (rescue #%llu), the link is tried again in %.0f s", (now - from) / 1e6, (unsigned long long)g_rescue_n, g_rescue_back_ns / 1e9);
+            g_rescue_back_ns = g_rescue_back_ns < 30000000000ull ? g_rescue_back_ns * 2 : 60000000000ull;
+        }
+        if (active && !g_stall_red && elapsed >= STALL_RED_NS) { g_stall_red = 1; MLOG("[RED] display link silent for %.1f s while active; presenting relies entirely on fallback frame drops, the picture is frozen", elapsed / 1e9); }
     }
     pthread_mutex_unlock(&g_pq_mu);
 }

dm/gfx/tests/.gitignore

@@ -20,6 +20,8 @@ t_fuse_check
 t_fuse_check.dSYM/
 t_present_crop
 t_present_crop.dSYM/
+t_present_rescue
+t_present_rescue.dSYM/
 t_reorder_chain
 t_reorder_chain.dSYM/
 t_mip_extent

dm/gfx/tests/Makefile

@@ -65,6 +65,12 @@ check_t_readback: t_readback
 run: check_t_readback
 t_present_crop: t_present_crop.m ../nvn/hle_nvn.c ../../host/crash.c ../../host/hle_bigstack.c t_host_stubs.c $(GFX) $(GFX_H)
 	clang $(CFLAGS) -fobjc-arc -o $@ t_present_crop.m ../../host/crash.c ../../host/hle_bigstack.c t_host_stubs.c $(filter-out ../nvn/nvn_mtl.m,$(GFX)) $(FW)
+t_present_rescue: t_present_rescue.m t_present_rescue_focus.c $(MTL_T_DEP)
+	clang $(CFLAGS) -fobjc-arc -o $@ t_present_rescue.m t_present_rescue_focus.c $(MTL_T_SRC) $(FW)
+.PHONY: check_t_present_rescue
+check_t_present_rescue: t_present_rescue
+	MTL_DEBUG_LAYER=1 ./t_present_rescue --metal
+run: check_t_present_rescue
 t_present_snap: t_present_snap.m $(MTL_T_DEP)
 	clang $(CFLAGS) -fobjc-arc -o $@ t_present_snap.m ../../host/crash.c ../../host/hle_bigstack.c t_host_stubs.c $(filter-out ../nvn/nvn_mtl.m,$(GFX)) $(FW)
 .PHONY: check_t_present_snap

dm/gfx/tests/t_present_rescue.m

@@ -0,0 +1,112 @@
+// Exercise the production watchdog with a silent display link and a controlled clock; no GPU or game files. --metal also checks real layer ownership.
+#import <QuartzCore/CAMetalDisplayLink.h>
+#import <QuartzCore/CAMetalLayer.h>
+@interface SilentDisplayLink : NSObject
+@property CAFrameRateRange preferredFrameRateRange;
+@property float preferredFrameLatency;
+@property (weak) id<CAMetalDisplayLinkDelegate> delegate;
+@property BOOL paused;
+@property BOOL invalidated;
+- (instancetype)initWithMetalLayer:(CAMetalLayer *)layer;
+- (void)addToRunLoop:(NSRunLoop *)loop forMode:(NSRunLoopMode)mode;
+- (void)invalidate;
+@end
+@implementation SilentDisplayLink
+- (instancetype)initWithMetalLayer:(CAMetalLayer *)layer { (void)layer; return [super init]; }
+- (void)addToRunLoop:(NSRunLoop *)loop forMode:(NSRunLoopMode)mode { (void)mode; [loop addPort:[NSMachPort port] forMode:NSDefaultRunLoopMode]; }
+- (void)invalidate { self.invalidated = YES; }
+@end
+#define CAMetalDisplayLink SilentDisplayLink
+#include "../nvn/nvn_mtl.m"
+#undef CAMetalDisplayLink
+static uint64_t test_now;
+int test_focus = 1;
+uint64_t test_focus_since;
+void *front_metal_layer(void) { return NULL; }
+module_t *module_of(const void *p) { (void)p; return NULL; }
+module_t g_mod[MAX_MOD]; int g_nmod;
+void svc_thread_dump(void) {}
+uint64_t now_ns(void) { return test_now; }
+static int checks, failed;
+#define CHECK(name, ok) do { int pass = !!(ok); checks++; failed += !pass; printf("%s: %s\n", name, pass ? "PASS" : "FAIL"); } while (0)
+static void reset(void) {
+    test_now = 10000000000ull; test_focus = 1; test_focus_since = 0;
+    g_dm_present_link = 1; g_rescue_mode = 0; g_rescue_back_ns = 3000000000ull; g_rescue_n = 0;
+    g_link_tick_ns = test_now - 1000000000ull; g_stall_t0 = g_link_tick_ns;
+    g_stall_logged = g_stall_red = 0; g_stall_drop = 0;
+    g_pq_in = g_pq_read = g_pq_done = 0; g_link_owns_layer = 1;
+}
+static void wait_frame(void) {
+    uint64_t ticket = ++g_pq_in;
+    mtl_present_wait(ticket);
+    CHECK("stalled frame releases its acquire ticket", g_pq_done == ticket);
+}
+static void thresholds(void) {
+    reset(); g_link = [SilentDisplayLink new];
+    CHECK("a link paces before rescue", mtl_link_active());
+    g_stall_t0 = g_link_tick_ns = test_now - 499999999ull; wait_frame();
+    CHECK("499 ms does not trigger rescue", !g_rescue_mode);
+    test_now++; wait_frame();
+    CHECK("500 ms triggers rescue", g_rescue_mode == 1);
+    CHECK("rescue preserves the requested mode", g_dm_present_link == 1);
+    CHECK("rescue stops display-link pacing", !mtl_link_active());
+    g_link = nil;
+    uint64_t deadline = g_rescue_back_at;
+    test_now = deadline - 1; link_sync(); CHECK("no retry before 3 s", g_rescue_mode == 1);
+    test_now++; link_sync(); CHECK("retry after 3 s", !g_rescue_mode && g_dm_present_link == 1);
+    CHECK("retry starts a fresh stall clock", !g_stall_t0 && g_link_tick_ns == test_now);
+    reset(); test_focus = 2; wait_frame(); CHECK("inactive time does not trigger rescue", !g_rescue_mode);
+    test_focus = 3; wait_frame(); CHECK("background time does not trigger rescue", !g_rescue_mode);
+    test_focus = 1; test_focus_since = test_now; wait_frame(); CHECK("foreground starts a fresh grace period", !g_rescue_mode);
+    test_now += 500000000ull; wait_frame(); CHECK("500 ms active after foreground triggers rescue", g_rescue_mode == 1);
+    reset(); test_focus_since = test_now + 1; wait_frame();
+    CHECK("a newer focus timestamp cannot underflow the grace period", !g_rescue_mode);
+}
+static void backoff(void) {
+    reset(); const uint64_t seconds[] = {3, 6, 12, 24, 48, 60, 60};
+    for (unsigned i = 0; i < sizeof seconds / sizeof seconds[0]; i++) {
+        g_stall_t0 = g_link_tick_ns = test_now - 500000000ull; wait_frame();
+        CHECK("retry delay doubles and stops at 60 s", g_rescue_back_at - test_now == seconds[i] * 1000000000ull);
+        test_now = g_rescue_back_at; link_sync();
+    }
+}
+static void settings(void) {
+    for (int mode = 0; mode <= 2; mode += 2) {
+        reset(); wait_frame(); g_dm_present_link = mode;
+        uint64_t deadline = g_rescue_back_at;
+        link_sync(); CHECK("a new runtime mode ends rescue immediately", !g_rescue_mode);
+        test_now = deadline; link_sync();
+        CHECK("retry honors a new runtime pacing mode", g_dm_present_link == mode && !g_rescue_mode);
+    }
+}
+static void lifecycle(void) {
+    reset(); SilentDisplayLink *link = [SilentDisplayLink new];
+    g_link = link; g_link_started = 1; g_link_rl = CFRunLoopGetCurrent();
+    link_sync(); wait_frame(); link_sync();
+    CHECK("layer stays with the link until its run loop invalidates it", g_link_owns_layer);
+    CFRunLoopRunInMode(kCFRunLoopDefaultMode, 0.01, false);
+    CHECK("rescue invalidates the link before releasing the layer", link.invalidated && !g_link && !g_link_owns_layer);
+    g_link_started = 0;
+}
+static void metal_lifecycle(void) {
+    id<MTLDevice> device = MTLCreateSystemDefaultDevice();
+    CHECK("Metal device available", device != nil); if (!device) return;
+    CAMetalLayer *layer = [CAMetalLayer layer]; layer.device = device; layer.drawableSize = CGSizeMake(8, 8);
+    // Use the real class despite the compile-time double above. A paused link on an offscreen layer needs no visible window.
+    reset(); g_link = [[NSClassFromString(@"CAMetalDisplayLink") alloc] initWithMetalLayer:layer];
+    g_link.paused = YES; [g_link addToRunLoop:[NSRunLoop currentRunLoop] forMode:NSDefaultRunLoopMode];
+    g_link_started = 1; g_link_rl = CFRunLoopGetCurrent();
+    link_sync(); wait_frame(); link_sync();
+    CFRunLoopRunInMode(kCFRunLoopDefaultMode, 0.01, false);
+    CHECK("real link releases the layer during rescue", !g_link && !g_link_owns_layer);
+    id<CAMetalDrawable> drawable = [layer nextDrawable];
+    CHECK("present thread can take a real drawable after rescue", drawable != nil);
+    [drawable present]; g_link_started = 0;
+}
+int main(int argc, char **argv) { @autoreleasepool {
+    alarm(10); setbuf(stdout, NULL);
+    if (argc > 2 || (argc == 2 && strcmp(argv[1], "--metal"))) return 2;
+    thresholds(); backoff(); settings(); lifecycle();
+    if (argc == 2) metal_lifecycle();
+    printf("present rescue: %d checks, %d failed\n", checks, failed); return failed != 0;
+} }

dm/gfx/tests/t_present_rescue_focus.c

@@ -0,0 +1,6 @@
+// Strong focus hooks override the graphics layer's weak, always-active test defaults.
+#include <stdint.h>
+extern int test_focus;
+extern uint64_t test_focus_since;
+int dm_focus_now(void) { return test_focus; }
+uint64_t dm_focus_in_since(void) { return test_focus_since; }

docs/agents/symptoms.md

@@ -209,6 +209,7 @@ One line per symptom in the [subsystem references](../reference/): what you see
 - slow motion when the GPU is the bottleneck: a game's interval controller that times the GPU never steps up (GitHub issue #9). → [GPU timestamps per submission segment](../reference/frame-rate.md#gpu-timestamps-per-submission-segment)
 - the game presents at interval 1, yet every frame takes exactly 33.3 ms of "frame work"; after only the limiter was raised, everything ran twice as fast on the iPhone. → [A render-thread limiter with a constant and a fixed time step](../reference/frame-rate.md#a-render-thread-limiter-with-a-constant-and-a-fixed-time-step)
 - 55 fps with every frame 18.2 ms, the main thread asleep most of the time; every 11th frame stays on screen two vsyncs. → [A frame-end wait polled in 2 ms sleeps](../reference/frame-rate.md#a-frame-end-wait-polled-in-2-ms-sleeps)
+- the app is active and the game keeps running, but the picture freezes while display-link callbacks stop; the log repeatedly drops queued frames. → [Present thread rescues a silent display link](../reference/frame-rate.md#present-thread-rescues-a-silent-display-link)
 - "sometimes the speed increases when jumping" (Mario 1.2.1 on the iPhone): in `display` mode interval-2 frames were shown for 16.7 ms and the game visibly jumped ahead. "When pressing B, Link's move pace is speeding up" (Breath of the Wild on the iPhone); `old frames dropped by merging`. → [Frames of a dynamic present interval held on screen and at the acquire](../reference/frame-rate.md#frames-of-a-dynamic-present-interval-held-on-screen-and-at-the-acquire)
 - frames miss the screen while the GPU stays at its lowest clock (444 MHz on an M2 Max); more work (a higher render scale) only makes the frames slower. → [Present snapshots: the game's swapchain texture free at once](../reference/frame-rate.md#present-snapshots-the-games-swapchain-texture-free-at-once)
 - a steady game whose frame-rate counter wanders between 50 and 60 on the iPhone only; the `display link up to tick` lines show about as many empty ticks as frames dropped by merging: frames arrive two in one tick and none in the next. → [Vulkan acquire and vsync event follow the display link](../reference/frame-rate.md#vulkan-acquire-and-vsync-event-follow-the-display-link)

docs/reference/frame-rate.md

@@ -39,6 +39,15 @@ Frame pacing: the host's frame step, engine frame limiters, present intervals an
 
 ## Present interval and display rate
 
+### Present thread rescues a silent display link
+
+- **Symptom:** the app is active and the game keeps running, but the picture freezes while display-link callbacks stop; the log repeatedly drops queued frames.
+- **Cause:** dropping frames releases the game's swapchain textures but never displays them. A display link may stop while the app is active, including an occluded Mac window.
+- **Fix:** after 500 ms of silence while active, `mtl_present_wait` requests rescue. `link_sync` invalidates the display link on its run loop before releasing the layer to the present thread, which takes its own drawables in mode 0. Retry delays are 3, 6, 12, 24, 48 and then at most 60 seconds. Rescue state changes take `g_pq_mu`; the requested `g_dm_present_link` stays unchanged, and a new runtime mode ends rescue immediately. A retry resets stall timing. Inactive and background time do not count towards the 500 ms threshold, including a focus timestamp newer than the watchdog's clock sample.
+- **Found on:** d2r-mobile commit `afad65c` (an iPhone picture froze for 80 seconds), carried through MacNativeGames `cf1e629` and [PR #198](https://github.com/seedds/nxnative/pull/198). That upstream title has no profile in this repository; no local game reproduction is claimed.
+- **Check:** `make -C dm/gfx/tests GAME=010015100B514000 t_present_rescue` and `dm/gfx/tests/t_present_rescue` exercise the production wait and retry paths with a controlled clock and a silent link double, without a GPU. They cover foreground grace, backoff saturation, runtime mode changes, and layer release only after run-loop invalidation. The test also runs in CPU graphics CI. `make -C dm/gfx/tests GAME=010015100B514000 check_t_present_rescue` adds a paused real Metal display link on an offscreen Mac layer: rescue invalidates it, then `nextDrawable` succeeds (47 checks, zero failures with Metal validation). `DM_TEST_LINK_STALL=<start seconds>,<duration seconds>` pauses a real window's display link for manual verification.
+- **Not checked:** gameplay during stalled-link recovery or the iPhone; the PR's gameplay runs were headless.
+
 ### Frames of a dynamic present interval held on screen and at the acquire
 
 - **Symptom:** "sometimes the speed increases when jumping" (Mario 1.2.1 on the iPhone): in `display` mode interval-2 frames were shown for 16.7 ms and the game visibly jumped ahead. "When pressing B, Link's move pace is speeding up" (Breath of the Wild on the iPhone); `old frames dropped by merging`.