test(host): pin R5/R6 semantics — usage bucket mapping and model-switch signal
Verified against the harness source before locking in: - agent-loop appends request/header (reason 'change' when the envelope differs) BEFORE request/context per request, so a model switch is always a header change — firing the model event only there equals the durable record, and route/capacity metadata can never double-fire it. - dsh-llm TokenUsage buckets are disjoint: billed prompt-side = inputTokens + cacheReadTokens + cacheWriteTokens, outputTokens already contains reasoningTokens — the fold's prompt/output mapping was already correct; comments now cite the official semantics. No behavior change; new assertions cover the switch event (exactly one, from model-a to model-b), provider-only changes (route update, not a switch), and the usage mapping (1000 = 800+150+50; output 30).
This commit is contained in:
1 parent
3e74f354e1
commit
9e6675ce1d
2 files changed
+40
-1
No files matched your search
@@ -341,8 +341,15 @@ export function applyTimeline(state: TimelineState, event: TimelineEvent, bounds
|
||||
// every schema); per-tool prices above are display-only rankings.
|
||||
s.toolsTokens = estimateToolsTotal(tools)
|
||||
s.systemTokens = estimateSystem(header.system)
|
||||
// Current route/model: the durable request envelope is the source of
|
||||
// truth (request/context is only route/capacity metadata, appended
|
||||
// AFTER request/header per request — see agent-loop buildRequestHeader).
|
||||
if (header.config && typeof header.config.model === 'string') s.model = header.config.model
|
||||
if (header.config && typeof header.config.provider === 'string') s.provider = header.config.provider
|
||||
// A model switch has no dedicated durable event: it is a request
|
||||
// header that differs from the previous one, logged with reason
|
||||
// 'change' ('initial' opens a session, 'resume' reopens it). Firing
|
||||
// only here keeps the model event list equal to the durable record.
|
||||
if (data?.reason === 'change' && s.model && s.lastModel && s.model !== s.lastModel) {
|
||||
s.events.push({ seq: event.seq, time: event.time, kind: 'model', from: s.lastModel, to: s.model })
|
||||
}
|
||||
@@ -425,6 +432,11 @@ export function applyTimeline(state: TimelineState, event: TimelineEvent, bounds
|
||||
total,
|
||||
}
|
||||
if (usage && typeof usage.inputTokens === 'number') {
|
||||
// Official TokenUsage semantics (dsh-llm): the buckets are disjoint —
|
||||
// inputTokens is uncached input only, cache read/write are separate,
|
||||
// and billed prompt-side = input + cacheRead + cacheWrite. outputTokens
|
||||
// already includes reasoningTokens. No separate prompt/output field
|
||||
// exists in the durable vocabulary.
|
||||
record.prompt = usage.inputTokens + (usage.cacheReadTokens || 0) + (usage.cacheWriteTokens || 0)
|
||||
if (typeof usage.outputTokens === 'number') record.output = usage.outputTokens
|
||||
}
|
||||
|
||||
+28
-1
@@ -236,4 +236,31 @@ assert.ok(bounded.nodes.length <= 2, 'maxNodes bounds the served surface slice')
|
||||
assert.ok(bounded.requests.length <= 8, 'maxKeptTurns bounds the retained history (400-turn fixture trimmed by whole turns)')
|
||||
assert.equal(typeof cfgDef.schema.safeParse(bounded).success, 'boolean', 'custom-bounds view still passes the unit schema')
|
||||
|
||||
console.log('✔ host half functional test passed (projection unit, fold semantics, attribution, retention, shadow-price seqs, reference stability, determinism, config bounds)')
|
||||
// -- model-switch semantics (R6): the only durable signal is a request
|
||||
// header differing from the previous one (reason 'change'); route/capacity
|
||||
// metadata (request/context) must NOT fire a switch event by itself, and a
|
||||
// provider-only change with the same model is not a switch either --
|
||||
const switchLog = drive([
|
||||
{ seq: 1, type: 'request/header', time: 1000, data: { reason: 'initial', header: { system: 's', config: { model: 'model-a', provider: 'deepseek' } } } },
|
||||
{ seq: 2, type: 'request/header', time: 2000, data: { reason: 'change', header: { system: 's', config: { model: 'model-b', provider: 'deepseek' } } } },
|
||||
{ seq: 3, type: 'request/context', time: 2000, data: { model: 'model-b', provider: 'deepseek', contextWindow: 64000 } },
|
||||
{ seq: 4, type: 'request/header', time: 3000, data: { reason: 'change', header: { system: 's', config: { model: 'model-b', provider: 'openai' } } } },
|
||||
{ seq: 5, type: 'request/header', time: 4000, data: { reason: 'change', header: { system: 's', config: { model: 'model-b', provider: 'openai' } } } },
|
||||
])
|
||||
const switchEvents = switchLog.events.filter(e => e.kind === 'model')
|
||||
assert.equal(switchEvents.length, 1, 'exactly one model-switch event')
|
||||
assert.equal(switchEvents[0].from, 'model-a', 'switch records the previous model')
|
||||
assert.equal(switchEvents[0].to, 'model-b', 'switch records the new model')
|
||||
assert.equal(switchLog.model, 'model-b', 'current model follows the header')
|
||||
assert.equal(switchLog.provider, 'openai', 'provider-only change updates the route, not a switch')
|
||||
assert.equal(switchLog.contextWindow, 64000, 'request/context supplies route capacity')
|
||||
|
||||
// Usage mapping (R5): prompt-side = input + cacheRead + cacheWrite (billed
|
||||
// input), output = outputTokens; reasoningTokens stay inside outputTokens.
|
||||
const usageLog = drive([
|
||||
{ seq: 1, type: 'assistant/message', time: 1000, data: { turn: 1, step: 1, usage: { inputTokens: 800, cacheReadTokens: 150, cacheWriteTokens: 50, outputTokens: 30, reasoningTokens: 10 }, message: { content: [] } } },
|
||||
])
|
||||
assert.equal(usageLog.requests[0].prompt, 1000, 'prompt side sums the disjoint input buckets')
|
||||
assert.equal(usageLog.requests[0].output, 30, 'output is outputTokens (reasoning already inside)')
|
||||
|
||||
console.log('✔ host half functional test passed (projection unit, fold semantics, attribution, retention, shadow-price seqs, reference stability, determinism, config bounds, model switch, usage mapping)')
|
||||
Reference in new issue
Block a user