Rescue stalled display links with bounded retries (#198)

Switch active-app presents to drawable acquisition after 500 ms of display-link
silence, then retry with exponential backoff capped at 60 seconds. Preserve the
requested pacing mode, synchronize rescue state, and prevent focus timestamp
underflow from bypassing the foreground grace period.

Add CPU regression coverage in CI, a real Metal layer ownership check, and the
mechanism reference with upstream provenance and validation limits.

Validated: 43 CPU rescue checks; 47 checks with real Metal drawable acquisition;
34 present/crop and 10 pacing checks; Wonder Mac host build; all 133 iOS source
compile checks; documentation tests. Windowed gameplay recovery and iPhone
validation remain outstanding.

Co-authored-by: Aleksandr Svidchenkov <a.svidchenkov@gmail.com>

dm/gfx/nvn/nvn_mtl.m

@@ -8830,6 +8830,13 @@ static double g_link_vperiod, g_link_vhalf;   // virtual ticks (link_range): sec
 #define STALL_NS 100000000ull
 #define STALL_RED_NS 2000000000ull
 static uint64_t g_stall_t0, g_stall_drop, g_stall_n; static int g_stall_red, g_stall_logged;
+// When the display link stays silent for LINK_RESCUE_NS while the app is active, presents stop waiting for it: mode 0 (the present thread takes drawables itself), so the
+// picture keeps moving instead of standing still on dropped frames (d2r-mobile afad65c, via MacNativeGames cf1e629: an iPhone's link stopped while active and the picture
+// froze for 80 s; macOS also pauses the link of a window it takes for covered). The link is tried again after g_rescue_back_ns (link_sync), which doubles on each new
+// stall up to 60 s. The requested g_dm_present_link stays unchanged; all rescue state changes take g_pq_mu, and link_sync handles the link object on the enqueue thread.
+#define LINK_RESCUE_NS 500000000ull
+static _Atomic int g_rescue_mode;   // Also read by link_on without g_pq_mu, to stop pacing on silent ticks immediately
+static uint64_t g_rescue_back_at, g_rescue_back_ns = 3000000000ull, g_rescue_n;
 // The display link watchdog only recognizes the "active" state: when the App loses active status (pulling down Control Center / notifications, incoming call, system popup) and in the background the system does not call back anyway, and the picture is covered too -- the frame-drop fallback stays as before,
 // but it is not counted as stuck and no failure is reported; the stall is measured from the later of "the most recent callback / the most recent activation" (A16 logs: 8–17 seconds inactive and 17 seconds in the background were both reported as "stuck in foreground")
 __attribute__((weak)) int dm_focus_now(void) { return 1; }   // The host's hle_oe.c provides the strong definition; when graphics unit tests do not link the host it is treated as always active
@@ -8849,12 +8856,12 @@ __attribute__((weak)) uint64_t dm_focus_in_since(void) { return 0; }
         next += vp; tol += 2 * g_link_vhalf;   // a tick lies up to half a refresh off its slot, so two of them up to one refresh nearer each other than the slots
     }
     pthread_mutex_lock(&g_link_mu);
-    g_link_tick_ns = now_ns();
+    __atomic_store_n(&g_link_tick_ns, now_ns(), __ATOMIC_RELAXED);
     g_link_tick_n++;
     pthread_cond_broadcast(&g_link_cv);
     pthread_mutex_unlock(&g_link_mu);
     pthread_mutex_lock(&g_pq_mu);
-    if (!g_dm_present_link || g_pq_read == g_pq_in) { pthread_mutex_unlock(&g_pq_mu); return; }
+    if (!g_dm_present_link || g_rescue_mode || g_pq_read == g_pq_in) { pthread_mutex_unlock(&g_pq_mu); return; }
     // The previous frame's present interval was >= 2 (period > 25 ms): keep it on screen for its whole period before taking the next one, as the Switch flips only every N vertical syncs. This also holds while the link runs at 60 Hz (Mario 1.2.1 sets 2 for single frames, see nvn_game_fps in hle_nvn.c);
     //   before, every tick took the newest frame, so that frame, carrying the extra time step the game advanced for it, was shown for only 16.7 ms and the game looked like it jumped ahead
     if (g_link_hold_until > 0 && update.targetPresentationTimestamp < g_link_hold_until - tol) { g_link_held++; pthread_mutex_unlock(&g_pq_mu); return; }
@@ -8885,7 +8892,7 @@ __attribute__((weak)) uint64_t dm_focus_in_since(void) { return 0; }
 
 static CAMetalDisplayLink *g_link; static CFRunLoopRef g_link_rl; static int g_link_started;
 static int g_link_owns_layer, g_classic_busy;   // Under g_pq_mu: nextDrawable throws while the layer has the display link object attached, so the present thread that takes its own drawable and the display link object are mutually exclusive
-static int link_on(void) { return g_dm_present_link && g_link != nil; }
+static int link_on(void) { return g_dm_present_link && !atomic_load(&g_rescue_mode) && g_link != nil; }
 // Screens whose refresh rate is not a multiple of the link's (144 or 165 Hz against 60 or 30): asked for 60 Hz, the system fires the link on every third refresh of a 165 Hz screen,
 // 55 Hz, and a game paced by the link runs at 55 fps (27.5 at 30). There the link runs at the screen's rate and only the refreshes nearest a steady clock of the wanted rate count as
 // ticks (g_link_vperiod, NvMetalDisplayLinkDelegate): 60 a second on average, each frame shown on the refreshes nearest its 1/60 s slot. DM_LINK_VTICK=0 keeps the system's choice.
@@ -8922,8 +8929,16 @@ static void *link_main(void *arg) {   // Display link thread: started when switc
     return NULL;
 }
 static void link_sync(void) {   // Called every frame by the game's present thread: aligns the display link object's state to g_dm_present_link and the power-saving mode (can change at run time)
-    static int applied; int want = g_dm_present_link;
-    if (want < 0 || want > 2) { g_dm_present_link = applied == 3 ? 1 : applied; return; }
+    static int applied;
+    pthread_mutex_lock(&g_pq_mu);
+    int want = g_dm_present_link;
+    if (want < 0 || want > 2) { g_dm_present_link = applied == 3 ? 1 : applied; pthread_mutex_unlock(&g_pq_mu); return; }
+    if (g_rescue_mode && (want != g_rescue_mode || now_ns() >= g_rescue_back_at)) {
+        g_rescue_mode = 0; g_stall_t0 = 0; __atomic_store_n(&g_link_tick_ns, now_ns(), __ATOMIC_RELAXED);
+        MLOG("display link: rescue ended, trying pacing mode %d (frame %llu)", want, (unsigned long long)atomic_load(&g_cur_frame));
+    }
+    if (g_rescue_mode) want = 0;
+    pthread_mutex_unlock(&g_pq_mu);
     if (want && mtl_fps_target() == 30) want = 3;   // Power-saving mode: display link at 30 Hz
     if (want) front_screen_match(want == 3 ? 30 : want == 2 ? 120 : 60);
     // The screen's rate changed (switched by front_screen_match, or another screen): the range again, virtual ticks or none. Taken only once the link object exists, so a change
@@ -9155,9 +9170,15 @@ void mtl_present_wait(uint64_t tk) {
         if (!g_stall_t0) { g_stall_t0 = last ? last : now; g_stall_drop = 0; g_stall_red = 0; g_stall_logged = 0; g_stall_n++; }
         g_stall_drop += n;
         int active = dm_focus_now() == 1; uint64_t since = dm_focus_in_since(), from = g_stall_t0 > since ? g_stall_t0 : since;
-        if (active && !g_stall_logged && now - from >= STALL_NS) { g_stall_logged = 1;
+        uint64_t elapsed = now >= from ? now - from : 0;   // Focus can return after the earlier now sample, including while waiting for an in-flight present
+        if (active && !g_stall_logged && elapsed >= STALL_NS) { g_stall_logged = 1;
             MLOG("display link silent for %.0f ms (active): dropping %llu queued frames not yet displayed, the game continues (stall #%llu; back to normal when the link returns)", (now - from) / 1e6, (unsigned long long)n, (unsigned long long)g_stall_n); }
-        if (active && !g_stall_red && now - from >= STALL_RED_NS) { g_stall_red = 1; MLOG("[RED] display link silent for %.1f s while active; presenting relies entirely on fallback frame drops, the picture is frozen", (now - from) / 1e9); }
+        if (active && !g_rescue_mode && g_dm_present_link && elapsed >= LINK_RESCUE_NS) {
+            g_rescue_mode = g_dm_present_link; g_rescue_n++; g_rescue_back_at = now + g_rescue_back_ns;
+            MLOG("display link silent for %.0f ms while active: the present thread takes drawables itself (rescue #%llu), the link is tried again in %.0f s", (now - from) / 1e6, (unsigned long long)g_rescue_n, g_rescue_back_ns / 1e9);
+            g_rescue_back_ns = g_rescue_back_ns < 30000000000ull ? g_rescue_back_ns * 2 : 60000000000ull;
+        }
+        if (active && !g_stall_red && elapsed >= STALL_RED_NS) { g_stall_red = 1; MLOG("[RED] display link silent for %.1f s while active; presenting relies entirely on fallback frame drops, the picture is frozen", elapsed / 1e9); }
     }
     pthread_mutex_unlock(&g_pq_mu);
 }

dm/gfx/tests/.gitignore

@@ -20,6 +20,8 @@ t_fuse_check
 t_fuse_check.dSYM/
 t_present_crop
 t_present_crop.dSYM/
+t_present_rescue
+t_present_rescue.dSYM/
 t_reorder_chain
 t_reorder_chain.dSYM/
 t_mip_extent

dm/gfx/tests/Makefile

@@ -65,6 +65,12 @@ check_t_readback: t_readback
 run: check_t_readback
 t_present_crop: t_present_crop.m ../nvn/hle_nvn.c ../../host/crash.c ../../host/hle_bigstack.c t_host_stubs.c $(GFX) $(GFX_H)
 	clang $(CFLAGS) -fobjc-arc -o $@ t_present_crop.m ../../host/crash.c ../../host/hle_bigstack.c t_host_stubs.c $(filter-out ../nvn/nvn_mtl.m,$(GFX)) $(FW)
+t_present_rescue: t_present_rescue.m t_present_rescue_focus.c $(MTL_T_DEP)
+	clang $(CFLAGS) -fobjc-arc -o $@ t_present_rescue.m t_present_rescue_focus.c $(MTL_T_SRC) $(FW)
+.PHONY: check_t_present_rescue
+check_t_present_rescue: t_present_rescue
+	MTL_DEBUG_LAYER=1 ./t_present_rescue --metal
+run: check_t_present_rescue
 t_present_snap: t_present_snap.m $(MTL_T_DEP)
 	clang $(CFLAGS) -fobjc-arc -o $@ t_present_snap.m ../../host/crash.c ../../host/hle_bigstack.c t_host_stubs.c $(filter-out ../nvn/nvn_mtl.m,$(GFX)) $(FW)
 .PHONY: check_t_present_snap

dm/gfx/tests/t_present_rescue.m

@@ -0,0 +1,112 @@
+// Exercise the production watchdog with a silent display link and a controlled clock; no GPU or game files. --metal also checks real layer ownership.
+#import <QuartzCore/CAMetalDisplayLink.h>
+#import <QuartzCore/CAMetalLayer.h>
+@interface SilentDisplayLink : NSObject
+@property CAFrameRateRange preferredFrameRateRange;
+@property float preferredFrameLatency;
+@property (weak) id<CAMetalDisplayLinkDelegate> delegate;
+@property BOOL paused;
+@property BOOL invalidated;
+- (instancetype)initWithMetalLayer:(CAMetalLayer *)layer;
+- (void)addToRunLoop:(NSRunLoop *)loop forMode:(NSRunLoopMode)mode;
+- (void)invalidate;
+@end
+@implementation SilentDisplayLink
+- (instancetype)initWithMetalLayer:(CAMetalLayer *)layer { (void)layer; return [super init]; }
+- (void)addToRunLoop:(NSRunLoop *)loop forMode:(NSRunLoopMode)mode { (void)mode; [loop addPort:[NSMachPort port] forMode:NSDefaultRunLoopMode]; }
+- (void)invalidate { self.invalidated = YES; }
+@end
+#define CAMetalDisplayLink SilentDisplayLink
+#include "../nvn/nvn_mtl.m"
+#undef CAMetalDisplayLink
+static uint64_t test_now;
+int test_focus = 1;
+uint64_t test_focus_since;
+void *front_metal_layer(void) { return NULL; }
+module_t *module_of(const void *p) { (void)p; return NULL; }
+module_t g_mod[MAX_MOD]; int g_nmod;
+void svc_thread_dump(void) {}
+uint64_t now_ns(void) { return test_now; }
+static int checks, failed;
+#define CHECK(name, ok) do { int pass = !!(ok); checks++; failed += !pass; printf("%s: %s\n", name, pass ? "PASS" : "FAIL"); } while (0)
+static void reset(void) {
+    test_now = 10000000000ull; test_focus = 1; test_focus_since = 0;
+    g_dm_present_link = 1; g_rescue_mode = 0; g_rescue_back_ns = 3000000000ull; g_rescue_n = 0;
+    g_link_tick_ns = test_now - 1000000000ull; g_stall_t0 = g_link_tick_ns;
+    g_stall_logged = g_stall_red = 0; g_stall_drop = 0;
+    g_pq_in = g_pq_read = g_pq_done = 0; g_link_owns_layer = 1;
+}
+static void wait_frame(void) {
+    uint64_t ticket = ++g_pq_in;
+    mtl_present_wait(ticket);
+    CHECK("stalled frame releases its acquire ticket", g_pq_done == ticket);
+}
+static void thresholds(void) {
+    reset(); g_link = [SilentDisplayLink new];
+    CHECK("a link paces before rescue", mtl_link_active());
+    g_stall_t0 = g_link_tick_ns = test_now - 499999999ull; wait_frame();
+    CHECK("499 ms does not trigger rescue", !g_rescue_mode);
+    test_now++; wait_frame();
+    CHECK("500 ms triggers rescue", g_rescue_mode == 1);
+    CHECK("rescue preserves the requested mode", g_dm_present_link == 1);
+    CHECK("rescue stops display-link pacing", !mtl_link_active());
+    g_link = nil;
+    uint64_t deadline = g_rescue_back_at;
+    test_now = deadline - 1; link_sync(); CHECK("no retry before 3 s", g_rescue_mode == 1);
+    test_now++; link_sync(); CHECK("retry after 3 s", !g_rescue_mode && g_dm_present_link == 1);
+    CHECK("retry starts a fresh stall clock", !g_stall_t0 && g_link_tick_ns == test_now);
+    reset(); test_focus = 2; wait_frame(); CHECK("inactive time does not trigger rescue", !g_rescue_mode);
+    test_focus = 3; wait_frame(); CHECK("background time does not trigger rescue", !g_rescue_mode);
+    test_focus = 1; test_focus_since = test_now; wait_frame(); CHECK("foreground starts a fresh grace period", !g_rescue_mode);
+    test_now += 500000000ull; wait_frame(); CHECK("500 ms active after foreground triggers rescue", g_rescue_mode == 1);
+    reset(); test_focus_since = test_now + 1; wait_frame();
+    CHECK("a newer focus timestamp cannot underflow the grace period", !g_rescue_mode);
+}
+static void backoff(void) {
+    reset(); const uint64_t seconds[] = {3, 6, 12, 24, 48, 60, 60};
+    for (unsigned i = 0; i < sizeof seconds / sizeof seconds[0]; i++) {
+        g_stall_t0 = g_link_tick_ns = test_now - 500000000ull; wait_frame();
+        CHECK("retry delay doubles and stops at 60 s", g_rescue_back_at - test_now == seconds[i] * 1000000000ull);
+        test_now = g_rescue_back_at; link_sync();
+    }
+}
+static void settings(void) {
+    for (int mode = 0; mode <= 2; mode += 2) {
+        reset(); wait_frame(); g_dm_present_link = mode;
+        uint64_t deadline = g_rescue_back_at;
+        link_sync(); CHECK("a new runtime mode ends rescue immediately", !g_rescue_mode);
+        test_now = deadline; link_sync();
+        CHECK("retry honors a new runtime pacing mode", g_dm_present_link == mode && !g_rescue_mode);
+    }
+}
+static void lifecycle(void) {
+    reset(); SilentDisplayLink *link = [SilentDisplayLink new];
+    g_link = link; g_link_started = 1; g_link_rl = CFRunLoopGetCurrent();
+    link_sync(); wait_frame(); link_sync();
+    CHECK("layer stays with the link until its run loop invalidates it", g_link_owns_layer);
+    CFRunLoopRunInMode(kCFRunLoopDefaultMode, 0.01, false);
+    CHECK("rescue invalidates the link before releasing the layer", link.invalidated && !g_link && !g_link_owns_layer);
+    g_link_started = 0;
+}
+static void metal_lifecycle(void) {
+    id<MTLDevice> device = MTLCreateSystemDefaultDevice();
+    CHECK("Metal device available", device != nil); if (!device) return;
+    CAMetalLayer *layer = [CAMetalLayer layer]; layer.device = device; layer.drawableSize = CGSizeMake(8, 8);
+    // Use the real class despite the compile-time double above. A paused link on an offscreen layer needs no visible window.
+    reset(); g_link = [[NSClassFromString(@"CAMetalDisplayLink") alloc] initWithMetalLayer:layer];
+    g_link.paused = YES; [g_link addToRunLoop:[NSRunLoop currentRunLoop] forMode:NSDefaultRunLoopMode];
+    g_link_started = 1; g_link_rl = CFRunLoopGetCurrent();
+    link_sync(); wait_frame(); link_sync();
+    CFRunLoopRunInMode(kCFRunLoopDefaultMode, 0.01, false);
+    CHECK("real link releases the layer during rescue", !g_link && !g_link_owns_layer);
+    id<CAMetalDrawable> drawable = [layer nextDrawable];
+    CHECK("present thread can take a real drawable after rescue", drawable != nil);
+    [drawable present]; g_link_started = 0;
+}
+int main(int argc, char **argv) { @autoreleasepool {
+    alarm(10); setbuf(stdout, NULL);
+    if (argc > 2 || (argc == 2 && strcmp(argv[1], "--metal"))) return 2;
+    thresholds(); backoff(); settings(); lifecycle();
+    if (argc == 2) metal_lifecycle();
+    printf("present rescue: %d checks, %d failed\n", checks, failed); return failed != 0;
+} }

dm/gfx/tests/t_present_rescue_focus.c

@@ -0,0 +1,6 @@
+// Strong focus hooks override the graphics layer's weak, always-active test defaults.
+#include <stdint.h>
+extern int test_focus;
+extern uint64_t test_focus_since;
+int dm_focus_now(void) { return test_focus; }
+uint64_t dm_focus_in_since(void) { return test_focus_since; }