Acelogic
e128ca1abe
installed-fresh-desktop-20261024-next1 stopped after 1,104 s with exit 7: a RETYPE to ROZONE was refused because sprr_views_prepare found the kernel views at SPRR_VIEW_TABLES (128). Its context.bin shows that 114 of the 118 level-3 copies still held selector2 leaves (122 ROZONE protects, 3 releases): each RO-zone frame lands in its own 2 MiB table, so the live set itself reached the pool. In the 86 preserved runs of 6 minutes or more, releases stayed at 2 to 8. SNAP_VIEW_RECLAIM (opt-in, default 0): - SPRR_VIEW_TABLES becomes 256. struct sprr_views grows by 8,192 bytes and 65 of them precede pauth_bank, so PAUTH_BANK_CONTEXT moves under the define (still asserted). The publication rows, the row-empty checks (sprr_publish_row_empty) and USER_PUBLISH_MAX follow the capacity. Context +1.0 MiB, root_ext +1.5 MiB of the arena. - sprr_views_reclaim retires every copied table no CPU needs: a level-3 table without a valid selector2/4/9 leaf, an upper table none of whose entries leads to a needed copy, never the root. It reads every source before changing anything, compacts the metadata (kept tables in order, retired sets parked past count at SPRR_VIEW_RETIRED), then calls quiesce. kernel_views_quiesce drops the reuse snapshot and the publication rows, and shadow_tlb_sync syncs all six views (a retired table's parent entry names its canonical source again, an identical translation), publishes them and runs TLBI ALLE1IS. Only after that may add() reuse the six pages at that position; after a failed quiesce they stay unusable. The trigger is lazy: kernel_views_prepare_change, used by the ROZONE and TXM-default retypes, reclaims only when a prepare is refused with fewer than three free slots, then prepares the whole change again, because the reclaim also retires paths an earlier leaf of the change prepared (not protected until the commit). prepare_path refuses a path that does not fit before adding any of it. Nothing runs before the pool is full: the refresh, PPERM and sync paths are unchanged. Off, the monitor is byte-identical: next1's flags give 458,552 bytes, sha256 86bb02ea9e03...; so does the build without SNAP_EDP_BACKLIGHT and the define-free one. On, the code grows by about 1.3 KiB, which needs the code reservation of the next commit. kernel_sync_view's frame grows from 2,336 to 4,528 bytes (64 KiB EL2 stacks; worst direct-call chain 13.2 KiB, unchanged). The mirrored multipage_close in tools/test-multipage-fast.c and the source check in tools/test-kernel-ops-perf-source.py follow the row helper. Hardware: not run.
99 lines
5.2 KiB
C
99 lines
5.2 KiB
C
/* SPDX-License-Identifier: MIT */
|
|
#pragma once
|
|
#include "sprr-view.h"
|
|
#include "sprr-reuse.h"
|
|
#include "sprr-publish.h"
|
|
|
|
/* Kernel PPERM variants (docs/KERNEL-PPERM-VARIANTS-20261016.md). Per CPU, a
|
|
* second private copy column that is never linked into the views. A flip of
|
|
* SPRR_KERNEL_VIEW_PPERM exchanges the CPU's whole column, so each set stays
|
|
* self-linked (entry_decode links a copied child through the same column), and
|
|
* the caller installs the new root. The caller owns the service lock,
|
|
* publication, the TTBR1 write and the invalidation. */
|
|
struct sprr_view_variants {
|
|
uint64_t copy[SPRR_VIEW_TABLES][SPRR_VIEW_CPUS];
|
|
uint64_t source[SPRR_VIEW_TABLES][SPRR_VIEW_CPUS];
|
|
uint64_t count[SPRR_VIEW_CPUS],pperm[SPRR_VIEW_CPUS],saves[SPRR_VIEW_CPUS];
|
|
uint64_t swaps,exact,synced,reserved,verified,mismatches,fallbacks,errors;
|
|
};
|
|
static inline int sprr_variants_shape(const struct sprr_views *v,unsigned cpu)
|
|
{return cpu<SPRR_VIEW_CPUS && v->count && v->count<=SPRR_VIEW_TABLES;}
|
|
static inline int sprr_variants_aliases(const struct sprr_view_variants *a,
|
|
const struct sprr_views *v,uint64_t pa)
|
|
{
|
|
for(unsigned n=0;n<v->count;n++) {
|
|
if(v->tables[n].source==pa)return 1;
|
|
for(unsigned cpu=0;cpu<SPRR_VIEW_CPUS;cpu++)if(v->tables[n].copy[cpu]==pa)return 1;
|
|
}
|
|
for(unsigned n=0;n<SPRR_VIEW_TABLES;n++)for(unsigned cpu=0;cpu<SPRR_VIEW_CPUS;cpu++)
|
|
if(a->copy[n][cpu]==pa)return 1;
|
|
return 0;
|
|
}
|
|
/* Allocate this CPU's missing variant pages. alloc must return fresh, zeroed
|
|
* pages whose zeroing the table walker already sees. Pages are kept on
|
|
* failure; a refused page is never recorded. */
|
|
static inline int sprr_variants_reserve(struct sprr_view_variants *a,const struct sprr_views *v,
|
|
const struct shadow *s,unsigned cpu)
|
|
{
|
|
if(!sprr_variants_shape(v,cpu) || !s->alloc) {a->errors++;return 0;}
|
|
for(unsigned n=0;n<v->count;n++) {
|
|
if(a->copy[n][cpu])continue;
|
|
uint64_t pa=shadow_alloc_page(s);
|
|
if(!pa || (pa&4095) || sprr_variants_aliases(a,v,pa)) {a->errors++;return 0;}
|
|
a->copy[n][cpu]=pa;a->reserved++;
|
|
}
|
|
return 1;
|
|
}
|
|
/* 1 iff this CPU's variant column equals the translation of the current
|
|
* canonical words under pperm. Legal only when sprr_reuse_current() held at
|
|
* the start of the transaction: then an unchanged r->saves since the swap-out
|
|
* means no canonical kernel word changed (every change re-saves before the
|
|
* next PPERM transaction), and a kernel copy decodes only
|
|
* SPRR_KERNEL_VIEW_PPERM. A column never swapped out has saves 0. */
|
|
static inline int sprr_variants_exact(const struct sprr_view_variants *a,const struct sprr_views *v,
|
|
const struct sprr_reuse_state *r,unsigned cpu,uint64_t pperm)
|
|
{
|
|
if(!sprr_variants_shape(v,cpu) || a->count[cpu]!=v->count || !a->saves[cpu] ||
|
|
a->saves[cpu]!=r->saves || ((a->pperm[cpu]^pperm)&SPRR_KERNEL_VIEW_PPERM) ||
|
|
(pperm&~SPRR_PPERM_MUTABLE)!=SPRR_PPERM_INITIAL)return 0;
|
|
for(unsigned n=0;n<v->count;n++)
|
|
if(!a->copy[n][cpu] || a->source[n][cpu]!=v->tables[n].source)return 0;
|
|
return 1;
|
|
}
|
|
/* Exchange this CPU's column with its reserved variant column. The outgoing
|
|
* column becomes the variant for `before` at snapshot generation `saves`:
|
|
* legal only when it equals that translation (the snapshot was current at the
|
|
* start of this transaction) and is fully published (an empty pending row,
|
|
* which also keeps every pending bit naming the page it was set for). */
|
|
static inline int sprr_variants_swap(struct sprr_view_variants *a,struct sprr_views *v,
|
|
const struct sprr_publish_state *p,unsigned cpu,uint64_t before,uint64_t saves)
|
|
{
|
|
if(!sprr_variants_shape(v,cpu) || !saves || p->root!=v->canonical_root ||
|
|
p->count!=v->count || !sprr_publish_row_empty(p,cpu) ||
|
|
(before&~SPRR_PPERM_MUTABLE)!=SPRR_PPERM_INITIAL) {a->errors++;return 0;}
|
|
for(unsigned n=0;n<v->count;n++)
|
|
if(!a->copy[n][cpu] || (a->copy[n][cpu]&4095)) {a->errors++;return 0;}
|
|
for(unsigned n=0;n<v->count;n++) {
|
|
uint64_t out=v->tables[n].copy[cpu];
|
|
v->tables[n].copy[cpu]=a->copy[n][cpu];a->copy[n][cpu]=out;
|
|
a->source[n][cpu]=v->tables[n].source;
|
|
}
|
|
a->count[cpu]=v->count;a->pperm[cpu]=before;a->saves[cpu]=saves;a->swaps++;
|
|
return 1;
|
|
}
|
|
/* sprr_reuse_repin after a swap: the snapshot names this CPU's new column.
|
|
* Legal only when sprr_reuse_current() held at the start of the transaction,
|
|
* no canonical word or other column changed, and the new column now equals
|
|
* its translation under pperm (an exact swap, or a swap then a full sync). */
|
|
static inline int sprr_variants_repin(struct sprr_reuse_state *r,const struct sprr_views *v,
|
|
unsigned cpu,uint64_t pperm)
|
|
{
|
|
if(!sprr_variants_shape(v,cpu) || r->count!=v->count || r->root!=v->canonical_root ||
|
|
(pperm&~SPRR_PPERM_MUTABLE)!=SPRR_PPERM_INITIAL)return 0;
|
|
for(unsigned n=0;n<v->count;n++)
|
|
if(r->tables[n].source!=v->tables[n].source || r->tables[n].level!=v->tables[n].level)
|
|
return 0;
|
|
for(unsigned n=0;n<v->count;n++)r->tables[n].copy[cpu]=v->tables[n].copy[cpu];
|
|
return sprr_reuse_repin(r,v,cpu,pperm);
|
|
}
|