From ffdd36aa1d4f0a04034d832200fd003bc9d2bb63 Mon Sep 17 00:00:00 2001 From: Kaede Akino Date: Tue, 25 Aug 2026 22:45:58 +0800 Subject: [PATCH] Move session context off the cached prefix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every cache these providers keep is a prefix cache, and the working directory of a terminal is the most volatile value the app has. Putting one inside the other meant a single `cd` between turns changed byte zero of the request and missed the entire replayed conversation — on a BYOK key, in an app whose whole purpose is moving around a filesystem. Nothing sets `cache_control` yet, so Anthropic was unaffected, but OpenAI and Google cache prefixes automatically: this was already costing money, and it was a trap laid for the day explicit caching lands. The context now rides the tail of the message list, immediately before the turn it describes. History the model has already seen is immutable, so `[system + history]` keeps hitting and the divergence is the newest turn, which was never cacheable anyway. The system prompt goes back to being byte-identical across every run, conversation, and user. It is sent but not kept. `snapshot.messages` is the durable transcript, and a block added there would be persisted, replayed, and — since restore projects user rows into the visible list — eventually shown in the user's own chat bubble. Leaving it out costs the next run one uncached message at the tail and nothing before it. Two consecutive user messages are already routine here: `.tool` rows become `user` rows on all three adapters, so any parallel tool call produces them. Co-Authored-By: Claude Opus 5 (1M context) --- Termind/Application/Agent/AgentRuntime.swift | 43 +++++++++++++++-- TermindTests/AgentRuntimeTests.swift | 51 ++++++++++++++------ 2 files changed, 75 insertions(+), 19 deletions(-) diff --git a/Termind/Application/Agent/AgentRuntime.swift b/Termind/Application/Agent/AgentRuntime.swift index 1b2642db..633ffb9c 100644 --- a/Termind/Application/Agent/AgentRuntime.swift +++ b/Termind/Application/Agent/AgentRuntime.swift @@ -249,6 +249,26 @@ actor AgentRuntime { Task { try? await repository.record(event, runID: runID) } } + /// Puts the session context into what is *sent* without putting it into + /// what is *kept*. + /// + /// `snapshot.messages` is the durable transcript, so a context block added + /// there would be persisted, replayed, and — since restore projects user + /// rows into the visible list — eventually shown in the user's own chat + /// bubble. Dropping it costs the next run one uncached message at the tail + /// and nothing before it, because history the model already saw is + /// unchanged either way. + private static func injecting( + _ context: AgentTranscriptMessage?, + at index: Int, + into messages: [AgentTranscriptMessage] + ) -> [AgentTranscriptMessage] { + guard let context, index <= messages.count else { return messages } + var result = messages + result.insert(context, at: index) + return result + } + private func run(_ request: AgentRunRequest) async { var snapshot = AgentRunSnapshot( conversationID: request.conversationID, @@ -258,9 +278,20 @@ actor AgentRuntime { ) activeRunID = snapshot.id let runID = snapshot.id - let effectiveSystemPrompt = [request.systemPrompt, request.sessionContext.modelContext] - .compactMap { $0 } - .joined(separator: "\n\n") + // Session context rides the tail of the message list, not the system + // prompt. Every cache these providers keep is a *prefix* cache, and the + // working directory of a terminal is the most volatile thing we know: + // one `cd` between turns used to change byte zero of the request and + // miss the entire replayed conversation. Here the divergence is the + // newest turn, which was never cacheable anyway, and the system prompt + // stays byte-identical across every run, conversation, and user. + let sessionContext = request.sessionContext.modelContext.map { + AgentTranscriptMessage(role: .user, text: $0) + } + // Prior history ends here, and everything a run appends — tool results, + // steering, assistant turns — goes after the prompt, so this index + // stays put for the whole run. + let sessionContextIndex = snapshot.messages.count snapshot.messages.append(AgentTranscriptMessage(role: .user, text: request.prompt)) emit(.runState(.running)) await persist(snapshot) @@ -274,8 +305,10 @@ actor AgentRuntime { steering.removeAll() } let turn = try await assistantTurn( - systemPrompt: effectiveSystemPrompt, - messages: snapshot.messages + systemPrompt: request.systemPrompt, + messages: Self.injecting( + sessionContext, at: sessionContextIndex, into: snapshot.messages + ) ) snapshot.messages.append(turn.message) emit(.messageFinished(turn.message)) diff --git a/TermindTests/AgentRuntimeTests.swift b/TermindTests/AgentRuntimeTests.swift index ceeca218..f3d3bee7 100644 --- a/TermindTests/AgentRuntimeTests.swift +++ b/TermindTests/AgentRuntimeTests.swift @@ -87,9 +87,14 @@ private struct RecordingScriptedAgentModel: AgentModelStreaming { @Suite struct AgentRuntimeTests { + /// The system prompt is the cached prefix of every request, and a terminal's + /// working directory is the most volatile value the app has. Putting one + /// inside the other meant a single `cd` between turns changed byte zero and + /// missed the whole replayed conversation. @Test - func sessionContextIsStableAcrossToolTurnsAndExcludedFromTranscript() async { + func sessionContextRidesTheMessageTailAndLeavesTheSystemPromptAlone() async { let requests = RequestRecorder() + let repository = InMemoryAgentRunRepository() let model = RecordingScriptedAgentModel( turns: [ [ @@ -115,7 +120,8 @@ struct AgentRuntimeTests { } let runtime = AgentRuntime( model: model, tools: [tool], - approval: AgentApprovalBroker(reviewer: nil, manualApproval: { _ in .allow }) + approval: AgentApprovalBroker(reviewer: nil, manualApproval: { _ in .allow }), + repository: repository ) let context = SessionContext( hostName: "edge-01", @@ -126,27 +132,44 @@ struct AgentRuntimeTests { architecture: "aarch64" ) ) + var runID: UUID? - for await _ in await runtime.start(AgentRunRequest( + for await event in await runtime.start(AgentRunRequest( conversationID: UUID(), prompt: "inspect", permissionMode: .askForApproval, sessionContext: context - )) {} + )) { + runID = runID ?? event.associatedRunID + } let captured = await requests.values #expect(captured.count == 2) - #expect(captured.allSatisfy { - $0.systemPrompt.contains("host: edge-01") - && $0.systemPrompt.contains("cwd: /srv/app") - && $0.systemPrompt.contains("distribution_id: alpine") - && $0.systemPrompt.contains("kernel_release: 6.6.44-0-lts") - && $0.systemPrompt.contains("architecture: aarch64") + // Byte-identical to the default across every run, which is the whole + // point: nothing session-specific may reach the cached prefix. + #expect(captured.allSatisfy { $0.systemPrompt == AgentSystemPrompt.default }) + + // One block, carrying everything, sitting between the replayed history + // and the turn it describes. + #expect(captured.allSatisfy { request in + request.messages.filter { $0.text.contains("cwd: /srv/app") }.count == 1 }) #expect(captured.allSatisfy { request in - request.messages.allSatisfy { - !$0.text.contains("distribution_id:") && !$0.text.contains("cwd:") - } + guard let index = request.messages.firstIndex(where: { + $0.text.contains("Session context") + }) else { return false } + let block = request.messages[index].text + return block.contains("host: edge-01") + && block.contains("distribution_id: alpine") + && block.contains("kernel_release: 6.6.44-0-lts") + && block.contains("architecture: aarch64") + && request.messages[index + 1].text == "inspect" }) - #expect(captured.first?.messages.first?.text == "inspect") + + // Present in what is sent, absent from what is kept — otherwise it would + // be replayed, and restore would project it into the user's own bubble. + let id = try! #require(runID) + let saved = try! #require(await repository.snapshot(id)).messages + #expect(saved.contains { $0.text == "inspect" }) + #expect(!saved.contains { $0.text.contains("Session context") }) } @Test