From 1fffff784dacb82cd1f62bd9e91178e50b5ca253 Mon Sep 17 00:00:00 2001 From: Acelogic <7362641+Acelogic@users.noreply.github.com> Date: Sat, 24 Oct 2026 21:54:41 -0400 Subject: [PATCH] tools: replay early swap completion in the pacing world With -DADRENO_PRESENT_EARLY_DONE=1 the presentation replay mirrors AdrenoDisplayEarlyDone.h step for step (the completion at the copy in l_present, arrivals in l_swap_begin, the tick state in l_tick) and runs early_done_scenarios (early-done.json): the 2026-10-24 busy desktop (org2's 117 windows: 49.8 copies/s with a SLOT WindowServer of 5-9 ms frames, 20% of them 14-30 ms), the same with b12v23's coherent maintenance, a FREE WindowServer, a quiet cursor desktop and next2c's sporadic heavy frames, each with the mode off, always and adaptive, under the deployed switches (relatch, urgent polling, EP1/EP2, no EP3 assist). Every earlier output (beam, damage, latency, pacing, pipeline, presentation, presentsync, relatch, vsync .json) is byte for byte the define-off build's, and the define-off build's equal the previous tree's. Busy SLOT: 49.8 -> 56.3 copies/s (coherent 51.8 -> 57.9), input latency mean +7 ms; quiet: unchanged in adaptive mode, +14 ms always; sporadic: 56.5 -> 58.1 adaptive, 59.4 always. --- tools/replay-adreno-early-done.c | 99 ++++++++++++++++++++++++++++++ tools/replay-adreno-latency.c | 36 +++++++++++ tools/replay-adreno-pacing.c | 22 +++++++ tools/replay-adreno-presentation.c | 3 + 4 files changed, 160 insertions(+) create mode 100644 tools/replay-adreno-early-done.c diff --git a/tools/replay-adreno-early-done.c b/tools/replay-adreno-early-done.c new file mode 100644 index 0000000..06318c0 --- /dev/null +++ b/tools/replay-adreno-early-done.c @@ -0,0 +1,99 @@ +/* SPDX-License-Identifier: MIT */ +/* Early completion (gpu/adreno-kext/kext/src/AdrenoDisplayEarlyDone.h): included at the end of + * tools/replay-adreno-pacing.c under -DADRENO_PRESENT_EARLY_DONE=1, after relatch_scenarios. Without the define this + * file adds nothing; with it every other output is byte for byte the define-off build's (the knobs below are 0 there). + * + * The world is the present-pacing one (the A16 beam and latch, the sync gate, the relatch, the pipelined submit lock, + * app IBs) with the deployed kext's switches: relatch and urgent polling on, early release EP1/EP2 on, EP3's assist off + * (the sparse kext compiles it out). AdrenoDisplay's early completion is mirrored step for step in + * replay-adreno-latency.c (l_present, l_swap_begin's arrival, l_tick); mode 1 completes every layer-0 swap at its copy, + * mode 2 only while a late frame engaged it (hold 2 s). + * + * Calibration (the 2026-10-24 desktop runs): + * - busy: installed-fresh-desktop-20261024-org2's 117 busy windows (gfx-probe load): 49.9 copies/s median, 18% of the + * ticks without a new swap, q 6.2 ms, one 1.35 ms IB per frame, TO_DEVICE + FROM_DEVICE 3.4 ms per IB (98 MiB). Only + * input-latency's SLOT WindowServer (two swaps outstanding, the next frame when one completes) reproduces that rate, + * with a render time of 5-9 ms and 20% of the frames 14-30 ms (FREE gives 57-59 copies/s for the same frames). + * - busy_coherent: the same frames with b12v23's maintenance (--gpu-coherent: 1.05 ms and 40 MiB per IB); + * - quiet: the cursor moving over an idle desktop, a layer-0 swap every refresh and no IB (b12v20/b12v21: 59.8/s); + * - sporadic: next2c's idle windows (57.9/s): the quiet frames plus 5% of frames with an IB whose kext time is 13 ms + * (TO_DEVICE 7 ms over 208 MiB and 6 ms of preparation) and 3.4 ms of GPU. + * The model has no guest-wide stalls, so its intervals over 34 ms stay under the hardware's (org2: 5.4%). */ +static void early_done_scenarios(const char *dir) +{ + unsigned c0 = checks, f0 = failures, cc0 = cost_checks, cf0 = cost_failures; + char path[4096]; snprintf(path, sizeof path, "%s/early-done.json", dir); + RJ = fopen(path, "w"); assert(RJ); rfirst = 0; + fprintf(RJ, "{\"scenarios\":{"); + const uint64_t M = MS; + struct ecase { struct rcase c; unsigned spike_pct; uint64_t spike_min, spike_max; }; + const struct ecase cases[] = { + {{"busy", WS_SLOT, 5 * M, 9 * M, 100, {900 * US, 1400 * US, 3 * M}, {1400 * US, 3 * M, 8 * M}, {75, 23, 2}, + 2200 * US, 3400 * US, 500 * US, 1 * M, 1200 * M, {1 * M, 3 * M, 10 * M}, {3 * M, 10 * M, 30 * M}, {80, 15, 5}, + 1 * M, 300 * US, 2, 1 * M, 6 * M, 3}, 20, 14 * M, 30 * M}, + {{"busy_coherent", WS_SLOT, 5 * M, 9 * M, 100, {900 * US, 1400 * US, 3 * M}, {1400 * US, 3 * M, 8 * M}, {75, 23, 2}, + 400 * US, 900 * US, 300 * US, 700 * US, 1200 * M, {1 * M, 3 * M, 10 * M}, {3 * M, 10 * M, 30 * M}, {80, 15, 5}, + 1 * M, 300 * US, 2, 1 * M, 6 * M, 3}, 20, 14 * M, 30 * M}, + {{"busy", WS_FREE, 5 * M, 9 * M, 100, {900 * US, 1400 * US, 3 * M}, {1400 * US, 3 * M, 8 * M}, {75, 23, 2}, + 2200 * US, 3400 * US, 500 * US, 1 * M, 1200 * M, {1 * M, 3 * M, 10 * M}, {3 * M, 10 * M, 30 * M}, {80, 15, 5}, + 1 * M, 300 * US, 2, 1 * M, 6 * M, 3}, 20, 14 * M, 30 * M}, + {{"quiet", WS_SLOT, 2 * M, 3600 * US, 0, {0, 0, 0}, {0, 0, 0}, {100, 0, 0}, + 0, 0, 0, 0, 0, {0, 0, 0}, {0, 0, 0}, {100, 0, 0}, 0, 0, 0, 0, 0, 0}, 0, 0, 0}, + {{"sporadic", WS_SLOT, 2 * M, 3600 * US, 5, {3 * M, 3 * M, 3 * M}, {4 * M, 4 * M, 4 * M}, {100, 0, 0}, + 12 * M, 14 * M, 1 * M, 2 * M, 0, {0, 0, 0}, {0, 0, 0}, {100, 0, 0}, 0, 0, 0, 0, 0, 0}, 0, 0, 0}, + }; + const unsigned ncases = sizeof cases / sizeof cases[0]; + for (unsigned i = 0; i < ncases; i++) { + const struct ecase *e = &cases[i]; + struct rres x[3]; + for (int mode = 0; mode < 3; mode++) { + char tag[32]; snprintf(tag, sizeof tag, "_done%d", mode); + rcase_noassist = 1; rcase_copydone = mode; rcase_ed_hold = 2000 * M; + rcase_spike_pct = e->spike_pct; rcase_spike_min = e->spike_min; rcase_spike_max = e->spike_max; + x[mode] = r_case(&e->c, RP_ALL, 1500, 1000, 0, tag); + } + rcase_noassist = 0; rcase_copydone = 0; rcase_ed_hold = 0; rcase_spike_pct = 0; rcase_spike_min = rcase_spike_max = 0; + const struct rres *t = &x[0], *a = &x[1], *d = &x[2]; + /* mode 0 measures only: nothing completed at a copy; the others never complete without a copy */ + CHECK(t->ed.early == 0 && t->ed.engages == 0); + CHECK(a->ed.early <= a->copies && d->ed.early <= d->copies); + CHECK(a->ed.engages == 0); /* mode 1 has nothing to engage */ + /* never fewer copies than today (1% noise) */ + COST(a->cps_x10 * 100 >= t->cps_x10 * 99 && d->cps_x10 * 100 >= t->cps_x10 * 99); + if (e->c.ws == WS_SLOT && t->ed.late * 50 > t->copies) { + /* WindowServer late on more than 2% of the frames: both modes recover most of the missed refreshes */ + COST(a->cps_x10 >= t->cps_x10 + 30 && d->cps_x10 >= t->cps_x10 + 25); + COST(d->ed.engages > 0 && d->ed.early > 0); + } + if (!t->ed.late) { + /* nothing late: the adaptive mode never engages and the run is today's frame for frame */ + COST(d->ed.engages == 0 && d->ed.early == 0 && d->cps_x10 == t->cps_x10 && d->r.in_mean_us == t->r.in_mean_us); + } + char line[512]; + int n = adreno_perf_line_done(line, sizeof line, i + 1, &d->ed, 0); + CHECK(n > 0 && n <= ADRENO_PERF_LINE_MAX); + printf("early-done %s_%s: copies/s today %llu.%llu, always %llu.%llu, adaptive %llu.%llu; >34 ms %llu.%llu%% / " + "%llu.%llu%% / %llu.%llu%%; late frames today %llu of %llu copies; input mean %llu / %llu / %llu us, " + "p95 %llu / " + "%llu / %llu us; q %llu / %llu / %llu us; adaptive engaged %llu times, %llu of %llu copies completed early\n", + e->c.name, ws_name[e->c.ws], (unsigned long long)t->cps_x10 / 10, (unsigned long long)t->cps_x10 % 10, + (unsigned long long)a->cps_x10 / 10, (unsigned long long)a->cps_x10 % 10, + (unsigned long long)d->cps_x10 / 10, (unsigned long long)d->cps_x10 % 10, + (unsigned long long)t->over34_pm / 10, (unsigned long long)t->over34_pm % 10, + (unsigned long long)a->over34_pm / 10, (unsigned long long)a->over34_pm % 10, + (unsigned long long)d->over34_pm / 10, (unsigned long long)d->over34_pm % 10, + (unsigned long long)t->ed.late, (unsigned long long)t->copies, + (unsigned long long)t->r.in_mean_us, (unsigned long long)a->r.in_mean_us, (unsigned long long)d->r.in_mean_us, + (unsigned long long)t->r.in_p95_us, (unsigned long long)a->r.in_p95_us, (unsigned long long)d->r.in_p95_us, + (unsigned long long)t->r.q_us, (unsigned long long)a->r.q_us, (unsigned long long)d->r.q_us, + (unsigned long long)d->ed.engages, (unsigned long long)d->ed.early, (unsigned long long)d->copies); + n = adreno_perf_line_done(line, sizeof line, i + 1, &t->ed, 0); + printf("early-done %s_%s today's arrivals: %s\n", e->c.name, ws_name[e->c.ws], line); + } + fprintf(RJ, "},\"checks\":%u,\"failures\":%u,\"cost_checks\":%u,\"cost_failures\":%u}\n", checks - c0, failures - f0, + cost_checks - cc0, cost_failures - cf0); + fclose(RJ); + printf("{\"early_done_checks\":%u,\"early_done_failures\":%u,\"early_done_cost_checks\":%u," + "\"early_done_cost_failures\":%u}\n", + checks - c0, failures - f0, cost_checks - cc0, cost_failures - cf0); +} diff --git a/tools/replay-adreno-latency.c b/tools/replay-adreno-latency.c index e40322f..bac6361 100644 --- a/tools/replay-adreno-latency.c +++ b/tools/replay-adreno-latency.c @@ -57,6 +57,16 @@ /* agents/present-pacing FAILURE-MODES.md: the b12v17 pipeline with its submit lock, and the relatch (relatch_scenarios) */ #include "../gpu/adreno-kext/kext/src/AdrenoDisplayRelatch.h" #endif +#ifndef ADRENO_PRESENT_EARLY_DONE +#define ADRENO_PRESENT_EARLY_DONE 0 +#endif +#if ADRENO_PRESENT_EARLY_DONE +#if !ADRENO_PRESENT_RELATCH +#error "the early-completion scenarios run in the present-pacing world (-DADRENO_PRESENT_RELATCH=1)" +#endif +/* AdrenoDisplayEarlyDone.h: a layer-0 swap completed right after its copy, always or while WindowServer is behind (early_done_scenarios) */ +#include "../gpu/adreno-kext/kext/src/AdrenoDisplayEarlyDone.h" +#endif enum { WS_SLOT = 0, WS_EVENT = 1, WS_TIMER = 2, WS_FREE = 3 }; enum { LP_BEAM = 0, LP_LATCH = 1, LP_HOLD = 2, LP_HOLD_NOGATE = 3 }; @@ -144,6 +154,10 @@ struct lworld { uint64_t c_ovl, c_tmr, c_pre, c_ast, c_app, c_apps, c_polls, c_late_unmark; uint64_t ret_at[LMAX]; /* the submission's return: G1 busy until then, S returns then */ #endif +#if ADRENO_PRESENT_EARLY_DONE + /* AdrenoDisplay's fEarly: mode = complete_at_copy (1 always, as the pipeline what-if; 2 adaptive), its hold */ + struct adreno_early_done ed; uint64_t ed_hold; +#endif }; /* ---- WindowServer ---- */ @@ -334,13 +348,25 @@ static void l_present(struct lworld *l, uint32_t id, uint8_t layers) /* presen l->presented = 1; if (layers & 1) l_copy(l, id); #if ADRENO_PIPELINE +#if ADRENO_PRESENT_EARLY_DONE + /* earlyDoneNowGated by mode (1 is also gpu-pipeline's what-if, whose hold rows keep it: the kext refuses the hold + * with early completion, the early-completion scenarios never run the hold) */ + if ((layers & 1) && l->complete_at_copy && adreno_early_active(&l->ed, l->w.now)) { + l->early_done = id; adreno_early_completed(&l->ed); ws_completed(l, id); + } +#else if ((layers & 1) && l->complete_at_copy) { l->early_done = id; ws_completed(l, id); } /* what-if */ #endif +#endif } static void l_pop(struct lworld *l) { memmove(l->q, l->q + 1, (l->qn - 1) * sizeof l->q[0]); l->qn--; } static void l_swap_begin(struct lworld *l, int in_tick) /* swap_begin_gated */ { struct bworld *w = &l->w; +#if ADRENO_PRESENT_EARLY_DONE + if (!in_tick && l->qn && !l->pending_since && (l->q[0].layers & 1)) + adreno_early_arrival(&l->ed, w->now, l->qn == 1); /* swap_begin_gated, F1's read */ +#endif if (!in_tick && l->qn && !l->pending_since) l->pending_since = w->now; /* F1 */ #if ADRENO_LATCH if (!in_tick && l->pending_since && l->h.released_at) adreno_hold_arrival(&l->h, tb(w, l->pending_since)); @@ -462,12 +488,19 @@ static void l_tick(struct lworld *l) /* vsyncGated */ w->in_tick = 0; if (!held) { if (l->rel_armed && l->cur && l->presented) l->rel_armed = 0; +#endif +#if ADRENO_PRESENT_EARLY_DONE + const int ed_had = l->cur && l->presented && (l->cur_layers & 1); /* completed here, or held since its copy */ + const uint64_t ed_copies = l->copies; #endif if (l->cur && l->presented) l_complete(l); w->in_tick = 1; if (l_gate(l)) { l->gated++; l_arm(l); } else l_swap_begin(l, 1); #if ADRENO_PRESENT_RELATCH if (l->p2) p2_relatch_arm(l); /* RL4: the GO's frame unused and its head deferred */ +#endif +#if ADRENO_PRESENT_EARLY_DONE + adreno_early_tick(&l->ed, w->now, ed_had, l->copies != ed_copies || (l->qn && (l->q[0].layers & 1))); #endif w->b.go_row0 = 0; w->in_tick = 0; #if ADRENO_LATCH @@ -545,6 +578,9 @@ static void l_start(struct lworld *l) adreno_beam_latch(&w->b, l->policy == LP_BEAM ? 0 : 3 * ms); adreno_hold_init(&l->h, l->policy == LP_HOLD || l->policy == LP_HOLD_NOGATE, ms * 3 / 2, ms * ADRENO_HOLD_FLOOR_US / 1000); l->gate_ns = l->policy == LP_LATCH || l->policy == LP_HOLD ? 4 * MS : 0; +#endif +#if ADRENO_PRESENT_EARLY_DONE + adreno_early_init(&l->ed, (uint32_t)l->complete_at_copy, 16666667, l->ed_hold ? l->ed_hold : 2000 * MS, MS); #endif l->started_at = w->now; if (l->ws_kind == WS_EVENT) l->events = 1; diff --git a/tools/replay-adreno-pacing.c b/tools/replay-adreno-pacing.c index 177bf47..62edd15 100644 --- a/tools/replay-adreno-pacing.c +++ b/tools/replay-adreno-pacing.c @@ -277,8 +277,19 @@ struct rres { uint64_t copies, iv[5], over34_pm, cps_x10, defer, swaps, max_wait_us, gpu_frames, marked, apps; uint64_t ovl, tmr, pre, ast, app_fin, polls, rl_armed, rl_go, rl_late, rl_gone, rl_gated, rl_ovt, rl_polls, rl_lead_min_us; uint64_t hist[ADRENO_RELATCH_BINS]; +#if ADRENO_PRESENT_EARLY_DONE + struct adreno_early_done ed; /* the run's early completions and arrivals (not windowed) */ +#endif }; static FILE *RJ; static int rfirst; +#if ADRENO_PRESENT_EARLY_DONE +/* replay-adreno-early-done.c's knobs, all 0 for every row above (their outputs stay byte for byte as without the define): + * the swap completed right after its copy (lworld complete_at_copy, AdrenoDisplay's adreno_present_early_done=1), + * WindowServer's render spikes, and EP3's assist off (the deployed sparse kext compiles it out). */ +static int rcase_copydone, rcase_noassist; +static unsigned rcase_spike_pct; +static uint64_t rcase_spike_min, rcase_spike_max, rcase_ed_hold; +#endif static struct rres r_case(const struct rcase *c, int rp, unsigned swaps, uint64_t guard_us, unsigned seed, const char *tag) { static struct lworld l; @@ -297,6 +308,11 @@ static struct rres r_case(const struct rcase *c, int rp, unsigned swaps, uint64_ l.early = rp == RP_EARLY || rp == RP_EARLY_NOASSIST || rp == RP_ALL || rp == RP_ALL_PIPE2; l.assist = l.early && rp != RP_EARLY_NOASSIST ? 250 * US : 0; l.tmr_period = 500 * US; l.urgent_period = relatch && rp != RP_RELATCH_NOURGENT ? 100 * US : 0; +#if ADRENO_PRESENT_EARLY_DONE + l.complete_at_copy = rcase_copydone; l.ed_hold = rcase_ed_hold; + l.spike_pct = rcase_spike_pct; l.spike_min = rcase_spike_min; l.spike_max = rcase_spike_max; + if (rcase_noassist) l.assist = 0; +#endif l.multi_pct = c->multi_pct; l.w.stall_pct = c->stall_pct; l.w.stall_min = c->stall_min; l.w.stall_max = c->stall_max; l.tmr_at = l.enter_at = l.rl_at = LINF; @@ -331,6 +347,9 @@ static struct rres r_case(const struct rcase *c, int rp, unsigned swaps, uint64_ x.rl_armed = l.rl.armed; x.rl_go = l.rl.go; x.rl_late = l.rl.late; x.rl_gone = l.rl.gone; x.rl_gated = l.rl.gated; x.rl_ovt = l.rl.ovt; x.rl_polls = l.rl.polls; x.rl_lead_min_us = tb_ns(&l.w, l.w.tb_off + l.rl.lead_min) / 1000; memcpy(x.hist, l.rl.hist, sizeof x.hist); +#if ADRENO_PRESENT_EARLY_DONE + x.ed = l.ed; +#endif char name[128]; snprintf(name, sizeof name, "%s_%s_%s%s", c->name, ws_name[c->ws], rp_name[rp], tag ? tag : ""); const struct lres *r = &x.r; fprintf(RJ, "%s\"%s\":{\"frames\":%llu,\"torn\":%llu,\"ovt\":%llu,\"beam_forced\":%llu,\"fps_x10\":%llu,\"copies_x10\":%llu," @@ -493,4 +512,7 @@ static void relatch_scenarios(const char *dir) printf("{\"relatch_checks\":%u,\"relatch_failures\":%u,\"relatch_cost_checks\":%u,\"relatch_cost_failures\":%u}\n", checks - c0, failures - f0, cost_checks - cc0, cost_failures - cf0); } +#if ADRENO_PRESENT_EARLY_DONE +#include "replay-adreno-early-done.c" +#endif #endif diff --git a/tools/replay-adreno-presentation.c b/tools/replay-adreno-presentation.c index fdb62ec..efe2767 100644 --- a/tools/replay-adreno-presentation.c +++ b/tools/replay-adreno-presentation.c @@ -1153,6 +1153,9 @@ int main(int argc,char **argv) { #endif #if ADRENO_PRESENT_RELATCH relatch_scenarios(argv[1]); +#if ADRENO_PRESENT_EARLY_DONE + early_done_scenarios(argv[1]); +#endif #endif #endif #endif