diff --git a/Termind/App/AppModel.swift b/Termind/App/AppModel.swift index 6e766397..3d629420 100644 --- a/Termind/App/AppModel.swift +++ b/Termind/App/AppModel.swift @@ -507,7 +507,12 @@ final class AppModel { let summary = try await agentCompaction.summarize( span.summarize, instructions: instructions, model: model ) - conversation.applyCompaction(summary: summary, cut: span.cut) + guard let record = conversation.prepareCompaction( + summary: summary, cut: span.cut + ) else { + throw AgentCompactionError.nothingToCompact + } + try await persistAndApplyCompaction(record, to: conversation, cut: span.cut) conversation.endCompaction() return true } catch { @@ -516,6 +521,17 @@ final class AppModel { } } + /// The atomicity boundary for compaction: persistence happens first, and + /// the observable model changes only after that succeeds. + func persistAndApplyCompaction( + _ record: AgentCompactionRecord, to conversation: ConversationModel, cut: Int + ) async throws { + try await agentRunRepository.persistCompaction( + record, conversationID: conversation.id + ) + conversation.applyCompaction(record, cut: cut) + } + /// Compacts before a run when the conversation is close to the window. /// /// Threshold only. Recovering a turn that has *already* overflowed means @@ -807,6 +823,8 @@ final class AppModel { else { return } let stored = StoredAgentConversation( id: fork.id, messages: fork.transcript, + modelTranscript: fork.transcript, + compactions: fork.compactionRecords, messageRunIDs: fork.messageRunIDs, runs: Array(fork.runSummaries.values), toolCards: fork.toolCards.map { diff --git a/Termind/Application/Agent/AgentRunPersisting.swift b/Termind/Application/Agent/AgentRunPersisting.swift index 19d33894..f0945c8b 100644 --- a/Termind/Application/Agent/AgentRunPersisting.swift +++ b/Termind/Application/Agent/AgentRunPersisting.swift @@ -20,6 +20,9 @@ nonisolated enum AgentConversationPersistenceError: LocalizedError, Sendable { nonisolated protocol AgentRunPersisting: Sendable { func save(_ snapshot: AgentRunSnapshot) async throws func record(_ event: AgentEvent, runID: UUID) async throws + func persistCompaction( + _ record: AgentCompactionRecord, conversationID: UUID + ) async throws func loadConversations() async throws -> [StoredAgentConversation] func deleteConversation(_ id: UUID) async throws /// Records the generated summary. Separate from `save` because it arrives @@ -46,6 +49,10 @@ nonisolated protocol AgentRunPersisting: Sendable { nonisolated struct StoredAgentConversation: Sendable { var id: UUID var messages: [AgentTranscriptMessage] + /// The independently restored model-facing projection. `nil` preserves the + /// old memberwise-initializer behavior for callers that have no compaction. + var modelTranscript: [AgentTranscriptMessage]? = nil + var compactions: [AgentCompactionRecord] = [] /// UI-only ownership restored from message rows. Keeping it outside the /// transcript prevents run identifiers leaking into provider requests. var messageRunIDs: [UUID: UUID] = [:] @@ -57,6 +64,80 @@ nonisolated struct StoredAgentConversation: Sendable { var metadata = StoredAgentConversationMetadata() } +nonisolated enum AgentCompactionProjection { + struct Restored: Sendable { + var messages: [AgentTranscriptMessage] + var modelTranscript: [AgentTranscriptMessage] + var records: [AgentCompactionRecord] + } + + /// Separates legacy inline markers from raw display history, merges them + /// with dedicated records, and applies the newest valid boundary to the + /// model-facing transcript. + static func restore( + messages: [AgentTranscriptMessage], records stored: [AgentCompactionRecord] + ) -> Restored { + let raw = messages.filter { !$0.isCompaction } + let inline = messages.filter(\.isCompaction) + var legacy: [AgentCompactionRecord] = [] + // Dedicated records are always newer than inline markers from the old + // format. Negative ordinals preserve that relationship without + // trusting wall-clock time across launches. + var nextSequence = -inline.count + + for message in inline { + // Legacy markers may have been assigned to the next run, so their + // array position is not a trustworthy boundary. Their timestamp was + // deliberately copied from the last summarized message. + let boundaryIndex = raw.lastIndex { $0.createdAt <= message.createdAt } + let summarized = boundaryIndex.map { raw[...$0] } ?? raw[...] + let summarizedTokens = summarized.reduce(0) { + $0 + AgentContextEstimator.tokens( + inCharacters: AgentContextEstimator.characterUnits(in: $1.text) + ) + } + legacy.append(AgentCompactionRecord( + summaryMessageID: message.id, + compactedThroughMessageID: boundaryIndex.map { raw[$0].id }, + sequence: nextSequence, + summary: message.text, + summarizedTokens: summarizedTokens, + keptTokens: AgentContextEstimator.tokens( + inCharacters: AgentContextEstimator.characterUnits(in: message.text) + ), + displayAnchor: message.createdAt, + compactedAt: message.createdAt + )) + nextSequence += 1 + } + + let records = (stored + legacy).sorted { + $0.sequence == $1.sequence + ? $0.compactedAt < $1.compactedAt : $0.sequence < $1.sequence + } + return Restored( + messages: raw, + modelTranscript: modelTranscript(messages: raw, records: records), + records: records + ) + } + + static func modelTranscript( + messages: [AgentTranscriptMessage], records: [AgentCompactionRecord] + ) -> [AgentTranscriptMessage] { + for record in records.reversed() { + guard let boundary = record.compactedThroughMessageID else { + return [record.summaryMessage] + messages + } + guard let index = messages.firstIndex(where: { $0.id == boundary }) else { + continue + } + return [record.summaryMessage] + messages[messages.index(after: index)...] + } + return messages + } +} + nonisolated struct StoredAgentConversationMetadata: Hashable, Sendable { var customTitle = "" var isPinned = false @@ -88,16 +169,38 @@ actor InMemoryAgentRunRepository: AgentRunPersisting { private var storedToolCards: [UUID: [StoredAgentToolCard]] = [:] private var deletedConversationIDs = Set() private var metadata: [UUID: StoredAgentConversationMetadata] = [:] + private var compactions: [UUID: [AgentCompactionRecord]] = [:] func save(_ snapshot: AgentRunSnapshot) { guard !deletedConversationIDs.contains(snapshot.conversationID) else { return } - snapshots[snapshot.id] = snapshot + var stored = snapshot + stored.messages.removeAll(where: \.isCompaction) + snapshots[stored.id] = stored } func record(_ event: AgentEvent, runID: UUID) {} + func persistCompaction( + _ record: AgentCompactionRecord, conversationID: UUID + ) throws { + guard !deletedConversationIDs.contains(conversationID), + snapshots.values.contains(where: { $0.conversationID == conversationID }) + else { throw AgentConversationPersistenceError.conversationNotFound } + if let boundary = record.compactedThroughMessageID { + guard snapshots.values.contains(where: { snapshot in + snapshot.conversationID == conversationID + && snapshot.messages.contains(where: { $0.id == boundary }) + }) else { throw AgentConversationPersistenceError.messageNotFound } + } + guard compactions[conversationID]?.contains(where: { $0.id == record.id }) != true + else { return } + compactions[conversationID, default: []].append(record) + } func loadConversations() -> [StoredAgentConversation] { Dictionary(grouping: snapshots.values, by: \.conversationID).map { id, runs in let ordered = runs.sorted { $0.startedAt < $1.startedAt } var seen = Set() - let messages = ordered.flatMap(\.messages).filter { seen.insert($0.id).inserted } + let storedMessages = ordered.flatMap(\.messages).filter { seen.insert($0.id).inserted } + let projection = AgentCompactionProjection.restore( + messages: storedMessages, records: compactions[id] ?? [] + ) var messageRunIDs: [UUID: UUID] = [:] for run in ordered { for message in run.messages where messageRunIDs[message.id] == nil { @@ -105,7 +208,10 @@ actor InMemoryAgentRunRepository: AgentRunPersisting { } } return StoredAgentConversation( - id: id, messages: messages, messageRunIDs: messageRunIDs, + id: id, messages: projection.messages, + modelTranscript: projection.modelTranscript, + compactions: projection.records, + messageRunIDs: messageRunIDs, runs: ordered.map { AgentRunSummary( id: $0.id, state: $0.state, startedAt: $0.startedAt, @@ -158,9 +264,14 @@ actor InMemoryAgentRunRepository: AgentRunPersisting { }) else { throw AgentConversationPersistenceError.messageNotFound } let targetStart = target.startedAt let removedRunIDs = Set(ordered.filter { $0.startedAt >= targetStart }.map(\.id)) + let removedMessageIDs = Set(ordered.filter { removedRunIDs.contains($0.id) } + .flatMap(\.messages).map(\.id)) snapshots = snapshots.filter { $0.value.conversationID != id || $0.value.startedAt < targetStart } + compactions[id]?.removeAll { + $0.compactedThroughMessageID.map(removedMessageIDs.contains) == true + } storedToolCards[id]?.removeAll { removedRunIDs.contains($0.invocation.runID) } let replacement = AgentRunSnapshot( conversationID: id, state: .completed, @@ -186,12 +297,14 @@ actor InMemoryAgentRunRepository: AgentRunPersisting { id: run.id, conversationID: conversation.id, state: run.state, permissionMode: .askForApproval, messages: conversation.messages.filter { - (conversation.messageRunIDs[$0.id] ?? fallbackRunID) == run.id + !$0.isCompaction + && (conversation.messageRunIDs[$0.id] ?? fallbackRunID) == run.id }, startedAt: run.startedAt, finishedAt: run.finishedAt ) } storedToolCards[conversation.id] = conversation.toolCards + compactions[conversation.id] = conversation.compactions titles[conversation.id] = conversation.title metadata[conversation.id] = conversation.metadata } @@ -199,6 +312,7 @@ actor InMemoryAgentRunRepository: AgentRunPersisting { deletedConversationIDs.insert(id) titles.removeValue(forKey: id) metadata.removeValue(forKey: id) + compactions.removeValue(forKey: id) storedToolCards.removeValue(forKey: id) snapshots = snapshots.filter { $0.value.conversationID != id } } diff --git a/Termind/Application/FeatureModels/AssistantLibraryModel.swift b/Termind/Application/FeatureModels/AssistantLibraryModel.swift index 717d6fe7..bea76035 100644 --- a/Termind/Application/FeatureModels/AssistantLibraryModel.swift +++ b/Termind/Application/FeatureModels/AssistantLibraryModel.swift @@ -71,7 +71,8 @@ final class AssistantConversationStore { let conversation = ConversationModel( id: value.id, messages: value.messages.compactMap { message in - switch message.role { + guard !message.isCompaction else { return nil } + return switch message.role { case .user: ChatMessage( id: message.id, role: .user, @@ -94,10 +95,11 @@ final class AssistantConversationStore { }, // Tool rows are dropped from the visible list because they are // drawn as cards, but the model still needs them next turn. - transcript: value.messages, + transcript: value.modelTranscript ?? value.messages, runState: value.lastRunState, messageRunIDs: value.messageRunIDs, runs: value.runs, + compactionRecords: value.compactions, createdAt: value.metadata.createdAt, updatedAt: value.metadata.updatedAt, sortIndex: value.metadata.sortIndex diff --git a/Termind/Domain/Agent/AgentTypes.swift b/Termind/Domain/Agent/AgentTypes.swift index 76f27c44..62040f57 100644 --- a/Termind/Domain/Agent/AgentTypes.swift +++ b/Termind/Domain/Agent/AgentTypes.swift @@ -605,6 +605,31 @@ nonisolated struct AgentTranscriptMessage: Identifiable, Hashable, Sendable, Cod var createdAt: Date = .now } +/// A durable replacement for the model-facing prefix of one conversation. +/// +/// The original messages stay in run storage for the reader. This record says +/// which raw message the summary replaces through, so persistence can rebuild a +/// compact model transcript without deleting the display history or assigning +/// the synthetic summary to a run that never produced it. +nonisolated struct AgentCompactionRecord: Identifiable, Hashable, Sendable, Codable { + var id: UUID = UUID() + var summaryMessageID: UUID = UUID() + var compactedThroughMessageID: UUID? + var sequence: Int + var summary: String + var summarizedTokens: Int + var keptTokens: Int + var displayAnchor: Date + var compactedAt: Date = .now + + var summaryMessage: AgentTranscriptMessage { + AgentTranscriptMessage( + id: summaryMessageID, role: .user, text: summary, + isCompaction: true, createdAt: displayAnchor + ) + } +} + nonisolated struct AgentRunSnapshot: Identifiable, Hashable, Sendable, Codable { var id: UUID = UUID() var conversationID: UUID diff --git a/Termind/Features/AI/ConversationModel.swift b/Termind/Features/AI/ConversationModel.swift index 4f575bc8..04b93969 100644 --- a/Termind/Features/AI/ConversationModel.swift +++ b/Termind/Features/AI/ConversationModel.swift @@ -490,6 +490,23 @@ nonisolated struct AgentCompactionMark: Identifiable, Hashable, Sendable { var summarizedTokens: Int var keptTokens: Int var createdAt: Date = .now + + init( + id: UUID = UUID(), summarizedTokens: Int, keptTokens: Int, + createdAt: Date = .now + ) { + self.id = id + self.summarizedTokens = summarizedTokens + self.keptTokens = keptTokens + self.createdAt = createdAt + } + + init(_ record: AgentCompactionRecord) { + self.init( + id: record.id, summarizedTokens: record.summarizedTokens, + keptTokens: record.keptTokens, createdAt: record.displayAnchor + ) + } } /// A presented plan and what became of it. @@ -667,10 +684,11 @@ final class ConversationModel: Identifiable { /// Plans presented in this conversation, oldest first. private(set) var plans: [AgentPlanCard] = [] - /// Where history stopped being sent, oldest first. Display only — the - /// authoritative record of a compaction is the `isCompaction` turn in - /// `transcript`, which is what survives a relaunch. + /// Where history stopped being sent, oldest first. Display only — durable + /// `compactionRecords` are authoritative across relaunches; their synthetic + /// transcript turns exist only in the model-facing projection. private(set) var compactions: [AgentCompactionMark] = [] + private(set) var compactionRecords: [AgentCompactionRecord] = [] /// Ephemeral UI state. Successful compactions become boundary marks; a /// failed attempt stays readable for this process without entering the @@ -803,6 +821,7 @@ final class ConversationModel: Identifiable { runState: AgentRunState = .idle, messageRunIDs: [UUID: UUID] = [:], runs: [AgentRunSummary] = [], + compactionRecords: [AgentCompactionRecord] = [], createdAt: Date = .now, updatedAt: Date = .now, sortIndex: Double? = nil, contextEstimatePublishInterval: Duration = .milliseconds(250) @@ -821,6 +840,8 @@ final class ConversationModel: Identifiable { self.messages = messages self.transcript = transcript self.messageRunIDs = messageRunIDs + self.compactionRecords = compactionRecords + compactions = compactionRecords.map(AgentCompactionMark.init) runSummaries = Dictionary(uniqueKeysWithValues: runs.map { ($0.id, $0) }) self.runState = runState runtimeState = runState @@ -1028,9 +1049,20 @@ final class ConversationModel: Identifiable { run.id = newID return run } + let clonedCompactions = compactionRecords.compactMap { record -> AgentCompactionRecord? in + guard let summaryMessageID = messageIDMap[record.summaryMessageID] else { return nil } + var copy = record + copy.id = UUID() + copy.summaryMessageID = summaryMessageID + // The fork stores only its compact model history, so its summary is + // the base context rather than a boundary into raw source rows. + copy.compactedThroughMessageID = nil + return copy + } let fork = ConversationModel( messages: visibleMessages, transcript: cloned, runState: .completed, messageRunIDs: clonedMessageRunIDs, runs: clonedRuns, + compactionRecords: clonedCompactions, createdAt: .now, updatedAt: .now ) let resolvedTitle = forkTitle ?? Self.numberedForkTitle(base: derivedTitle, index: 1) @@ -1333,15 +1365,11 @@ final class ConversationModel: Identifiable { } } - /// Replaces everything before `cut` with one synthetic turn carrying the - /// summary. - /// - /// `transcript` — what the model is sent — is rewritten. `messages` — what - /// the reader sees — is not: a marker is added instead, so the conversation - /// stays scrollable and nothing the user wrote disappears. That is the same - /// rule `AgentContextPipeline` states for every other transform. - func applyCompaction(summary: String, cut: Int) { - guard cut > 0, cut <= transcript.count else { return } + /// Builds the durable record without changing the conversation. AppModel + /// persists this value before applying it, so a failed save cannot leave + /// memory and disk describing different contexts. + func prepareCompaction(summary: String, cut: Int) -> AgentCompactionRecord? { + guard cut > 0, cut <= transcript.count else { return nil } let summarized = transcript[.. 0 && cut <= transcript.count) + let replacement = record.summaryMessage let removedIDs = transcript[.. AgentCompactionRecord? { + guard let record = prepareCompaction(summary: summary, cut: cut) else { return nil } + applyCompaction(record, cut: cut) + return record + } + /// Records what the user decided about a plan. /// /// Implementing turns plan mode off, because that is what the decision diff --git a/Termind/Infrastructure/Database/AgentPersistence.swift b/Termind/Infrastructure/Database/AgentPersistence.swift index 205409fa..926813e0 100644 --- a/Termind/Infrastructure/Database/AgentPersistence.swift +++ b/Termind/Infrastructure/Database/AgentPersistence.swift @@ -113,6 +113,46 @@ final class PersistentAgentMessage { } } +/// A compaction is conversation state, not output produced by an agent run. +/// Keeping it in its own table preserves the complete run/message graph for the +/// transcript reader while giving restore an exact model-context boundary. +@Model +final class PersistentAgentCompaction { + @Attribute(.unique) var id: UUID = UUID() + var conversationID: UUID = UUID() + var summaryMessageID: UUID = UUID() + var compactedThroughMessageID: UUID? + var sequence = 0 + var summary = "" + var summarizedTokens = 0 + var keptTokens = 0 + var displayAnchor: Date = Date() + var compactedAt: Date = Date() + + init(_ record: AgentCompactionRecord, conversationID: UUID) { + id = record.id + self.conversationID = conversationID + summaryMessageID = record.summaryMessageID + compactedThroughMessageID = record.compactedThroughMessageID + sequence = record.sequence + summary = AgentPersistenceRedactor.visibleText(record.summary) + summarizedTokens = record.summarizedTokens + keptTokens = record.keptTokens + displayAnchor = record.displayAnchor + compactedAt = record.compactedAt + } + + var value: AgentCompactionRecord { + AgentCompactionRecord( + id: id, summaryMessageID: summaryMessageID, + compactedThroughMessageID: compactedThroughMessageID, + sequence: sequence, summary: summary, + summarizedTokens: summarizedTokens, keptTokens: keptTokens, + displayAnchor: displayAnchor, compactedAt: compactedAt + ) + } +} + /// V3 storage for original context snapshots. A separate model keeps the V2 /// message schema frozen and makes the migration an additive lightweight one. @Model @@ -258,13 +298,14 @@ final class SwiftDataAgentRunRepository: AgentRunPersisting, Sendable { // turns are replayed into every later run, so filtering by `runID` made // each save re-insert the whole history — colliding with the unique // constraint and rewriting rows that were already correct. - let ids = Set(snapshot.messages.map(\.id)) + let durableMessages = snapshot.messages.filter { !$0.isCompaction } + let ids = Set(durableMessages.map(\.id)) let stored = try context.fetch(FetchDescriptor( predicate: #Predicate { ids.contains($0.id) } )) let storedIDs = Set(stored.map(\.id)) var inserted = false - for (sequence, message) in snapshot.messages.enumerated() + for (sequence, message) in durableMessages.enumerated() where !storedIDs.contains(message.id) { context.insert(PersistentAgentMessage( @@ -277,6 +318,43 @@ final class SwiftDataAgentRunRepository: AgentRunPersisting, Sendable { try context.save() } + @MainActor + func persistCompaction( + _ record: AgentCompactionRecord, conversationID: UUID + ) async throws { + guard !deletedConversationIDs.contains(conversationID) else { + throw AgentConversationPersistenceError.conversationNotFound + } + let conversations = try context.fetch(FetchDescriptor( + predicate: #Predicate { $0.id == conversationID } + )) + guard conversations.first != nil else { + throw AgentConversationPersistenceError.conversationNotFound + } + if let boundary = record.compactedThroughMessageID { + let runIDs = Set(try context.fetch(FetchDescriptor()) + .filter { $0.conversationID == conversationID }.map(\.id)) + let rows = try context.fetch(FetchDescriptor( + predicate: #Predicate { $0.id == boundary } + )) + guard rows.contains(where: { runIDs.contains($0.runID) }) else { + throw AgentConversationPersistenceError.messageNotFound + } + } + let recordID = record.id + let existing = try context.fetch(FetchDescriptor( + predicate: #Predicate { $0.id == recordID } + )) + guard existing.isEmpty else { return } + context.insert(PersistentAgentCompaction(record, conversationID: conversationID)) + do { + try context.save() + } catch { + context.rollback() + throw error + } + } + @MainActor func record(_ event: AgentEvent, runID: UUID) async throws { guard !deletedRunIDs.contains(runID) else { return } @@ -321,6 +399,7 @@ final class SwiftDataAgentRunRepository: AgentRunPersisting, Sendable { ) let runs = try context.fetch(FetchDescriptor()) let messages = try context.fetch(FetchDescriptor()) + let compactionRows = try context.fetch(FetchDescriptor()) let attachmentRows = try context.fetch( FetchDescriptor() ) @@ -392,13 +471,20 @@ final class SwiftDataAgentRunRepository: AgentRunPersisting, Sendable { let cards = invocations.filter { runIDs.contains($0.runID) } .sorted { $0.createdAt < $1.createdAt } .compactMap { storedToolCard($0, results: results, reviews: reviews) } - return StoredAgentConversation( - id: conversation.id, - messages: Self.compacted(Self.recovering( + let projection = AgentCompactionProjection.restore( + messages: Self.recovering( transcript, results: results.filter { runIDs.contains($0.runID) }, invocations: invocations.filter { runIDs.contains($0.runID) } - )), + ), + records: compactionRows.filter { $0.conversationID == conversation.id } + .map(\.value) + ) + return StoredAgentConversation( + id: conversation.id, + messages: projection.messages, + modelTranscript: projection.modelTranscript, + compactions: projection.records, messageRunIDs: messageRunIDs, runs: matchingRuns.compactMap { run in guard let state = AgentRunState(rawValue: run.state) else { return nil } @@ -519,6 +605,7 @@ final class SwiftDataAgentRunRepository: AgentRunPersisting, Sendable { let removedRuns = Array(runs[targetIndex...]) let removedRunIDs = Set(removedRuns.map(\.id)) + let removedMessageIDs = Set(allMessages.filter { removedRunIDs.contains($0.runID) }.map(\.id)) deletedRunIDs.formUnion(removedRunIDs) let earlierRunIDs = Set(runs[..()) + where row.conversationID == id + && row.compactedThroughMessageID.map(removedMessageIDs.contains) == true + { + context.delete(row) + } let replacementRun = AgentRunSnapshot( conversationID: id, state: .completed, @@ -599,6 +692,7 @@ final class SwiftDataAgentRunRepository: AgentRunPersisting, Sendable { let validRunIDs = Set(summaries.map(\.id)) let fallbackRunID = summaries[0].id for (sequence, message) in value.messages.enumerated() { + guard !message.isCompaction else { continue } context.insert(PersistentAgentMessage( message, runID: value.messageRunIDs[message.id].flatMap { @@ -608,6 +702,9 @@ final class SwiftDataAgentRunRepository: AgentRunPersisting, Sendable { )) insertContextAttachments(for: message) } + for record in value.compactions { + context.insert(PersistentAgentCompaction(record, conversationID: value.id)) + } for card in value.toolCards { var invocation = card.invocation if !validRunIDs.contains(invocation.runID) { @@ -644,6 +741,8 @@ final class SwiftDataAgentRunRepository: AgentRunPersisting, Sendable { for row in conversations where row.id == id { context.delete(row) } let orderRows = try context.fetch(FetchDescriptor()) for row in orderRows where row.conversationID == id { context.delete(row) } + for row in try context.fetch(FetchDescriptor()) + where row.conversationID == id { context.delete(row) } try context.save() } @@ -684,23 +783,6 @@ final class SwiftDataAgentRunRepository: AgentRunPersisting, Sendable { /// gap, and does for anything still missing after this. Doing it here first /// is what gives the model the output the tool actually produced instead of /// being told the outcome is unknown when it is sitting in the database. - /// Re-applies a compaction that was recorded before the process ended. - /// - /// History is rebuilt from every stored message of every run, so a - /// conversation that was compacted would come back whole and be sent whole — - /// the compaction silently undone at exactly the point the window was - /// already tight. The synthetic summary turn carries `isCompaction`, and - /// everything before the newest one is dropped. - /// - /// The **last** marker, not the first: a conversation compacted twice has - /// two, and the older summary was itself folded into the newer one. - static func compacted( - _ messages: [AgentTranscriptMessage] - ) -> [AgentTranscriptMessage] { - guard let cut = messages.lastIndex(where: \.isCompaction) else { return messages } - return Array(messages[cut...]) - } - static func recovering( _ transcript: [AgentTranscriptMessage], results: [PersistentAgentToolResult], diff --git a/Termind/Infrastructure/Database/HostPersistence.swift b/Termind/Infrastructure/Database/HostPersistence.swift index 2b6496df..2548f7b4 100644 --- a/Termind/Infrastructure/Database/HostPersistence.swift +++ b/Termind/Infrastructure/Database/HostPersistence.swift @@ -550,12 +550,20 @@ enum TermindSchemaV8: VersionedSchema { } } +enum TermindSchemaV9: VersionedSchema { + static let versionIdentifier = Schema.Version(9, 0, 0) + + static var models: [any PersistentModel.Type] { + TermindSchemaV8.models + [PersistentAgentCompaction.self] + } +} + enum TermindMigrationPlan: SchemaMigrationPlan { static var schemas: [any VersionedSchema.Type] { [ TermindSchemaV1.self, TermindSchemaV2.self, TermindSchemaV3.self, TermindSchemaV4.self, TermindSchemaV5.self, TermindSchemaV6.self, - TermindSchemaV7.self, TermindSchemaV8.self, + TermindSchemaV7.self, TermindSchemaV8.self, TermindSchemaV9.self, ] } @@ -591,13 +599,16 @@ enum TermindMigrationPlan: SchemaMigrationPlan { MigrationStage.lightweight( fromVersion: TermindSchemaV7.self, toVersion: TermindSchemaV8.self ), + MigrationStage.lightweight( + fromVersion: TermindSchemaV8.self, toVersion: TermindSchemaV9.self + ), ] } } @MainActor final class HostPersistence { - static let schema = Schema(versionedSchema: TermindSchemaV8.self) + static let schema = Schema(versionedSchema: TermindSchemaV9.self) let context: ModelContext diff --git a/TermindTests/AgentCompactionTests.swift b/TermindTests/AgentCompactionTests.swift index 988206c0..6d110804 100644 --- a/TermindTests/AgentCompactionTests.swift +++ b/TermindTests/AgentCompactionTests.swift @@ -289,6 +289,44 @@ struct AgentCompactionTests { #expect(conversation.compactions.count == 1) } + @Test + func preparingACompactionDoesNotMutateTheConversation() throws { + let conversation = ConversationModel() + for index in 0..<4 { + conversation.appendUserTurn("turn \(index) " + filler(tokens: 300)) + } + let span = try #require(conversation.compactionSpan(keepRecentTokens: 300)) + let transcript = conversation.transcript + let estimate = conversation.contextEstimate + + let record = try #require(conversation.prepareCompaction( + summary: "summary", cut: span.cut + )) + + #expect(conversation.transcript == transcript) + #expect(conversation.contextEstimate == estimate) + #expect(conversation.compactions.isEmpty) + #expect(record.compactedThroughMessageID == transcript[span.cut - 1].id) + } + + @Test + func aRepeatedCompactionInheritsTheRawBoundaryWhenItOnlyFoldsTheOldSummary() throws { + let conversation = ConversationModel() + conversation.appendUserTurn("old " + filler(tokens: 600)) + conversation.appendUserTurn("recent " + filler(tokens: 20)) + let firstSpan = try #require(conversation.compactionSpan(keepRecentTokens: 20)) + let first = try #require(conversation.applyCompaction( + summary: "first summary", cut: firstSpan.cut + )) + + let second = try #require(conversation.prepareCompaction( + summary: "second summary", cut: 1 + )) + + #expect(second.compactedThroughMessageID == first.compactedThroughMessageID) + #expect(second.sequence == first.sequence + 1) + } + @Test func compactionActivityIsEphemeralAndNeverBecomesAMessage() throws { let conversation = ConversationModel() @@ -358,27 +396,58 @@ struct AgentCompactionTests { /// it whole — undoing the compaction at the point it was most needed. @Test func restoreDoesNotResurrectCompactedHistory() { - let restored = SwiftDataAgentRunRepository.compacted([ - user("very old"), - assistant("older reply"), - AgentTranscriptMessage(role: .user, text: "## Goal\nfirst", isCompaction: true), - user("recent"), - AgentTranscriptMessage(role: .user, text: "## Goal\nsecond", isCompaction: true), - user("newest"), - ]) + let at: (TimeInterval) -> Date = { Date(timeIntervalSinceReferenceDate: $0) } + let restored = AgentCompactionProjection.restore(messages: [ + AgentTranscriptMessage(role: .user, text: "very old", createdAt: at(1)), + AgentTranscriptMessage(role: .assistant, text: "older reply", createdAt: at(2)), + AgentTranscriptMessage( + role: .user, text: "## Goal\nfirst", isCompaction: true, createdAt: at(2) + ), + AgentTranscriptMessage(role: .user, text: "recent", createdAt: at(3)), + AgentTranscriptMessage( + role: .user, text: "## Goal\nsecond", isCompaction: true, createdAt: at(3) + ), + AgentTranscriptMessage(role: .user, text: "newest", createdAt: at(4)), + ], records: []) // The last marker, not the first: a conversation compacted twice has // two, and the older summary was folded into the newer one. - #expect(restored.count == 2) - #expect(restored.first?.text.contains("second") == true) - #expect(restored.last?.text == "newest") + #expect(restored.modelTranscript.count == 2) + #expect(restored.modelTranscript.first?.text.contains("second") == true) + #expect(restored.modelTranscript.last?.text == "newest") + #expect(restored.messages.count == 4) } @Test func restoreLeavesAnUncompactedConversationAlone() { let messages = [user("a"), assistant("b")] - #expect(SwiftDataAgentRunRepository.compacted(messages).count == 2) + let restored = AgentCompactionProjection.restore(messages: messages, records: []) + #expect(restored.modelTranscript == messages) + } + + @Test + func legacyInlineMarkerIsHiddenFromDisplayAndKeptForTheModel() { + let old = AgentTranscriptMessage( + role: .user, text: "old", createdAt: Date(timeIntervalSinceReferenceDate: 1) + ) + let marker = AgentTranscriptMessage( + role: .user, text: "legacy summary", isCompaction: true, + createdAt: Date(timeIntervalSinceReferenceDate: 1) + ) + let recent = AgentTranscriptMessage( + role: .assistant, text: "recent", + createdAt: Date(timeIntervalSinceReferenceDate: 2) + ) + + let restored = AgentCompactionProjection.restore( + messages: [old, marker, recent], records: [] + ) + + #expect(restored.messages.map(\.id) == [old.id, recent.id]) + #expect(restored.modelTranscript.map(\.id) == [marker.id, recent.id]) + #expect(restored.records.count == 1) + #expect(restored.records.first?.summaryMessageID == marker.id) } // MARK: - The summarizer's prompt diff --git a/TermindTests/AgentPersistenceTests.swift b/TermindTests/AgentPersistenceTests.swift index 961d486f..d8aea50c 100644 --- a/TermindTests/AgentPersistenceTests.swift +++ b/TermindTests/AgentPersistenceTests.swift @@ -173,6 +173,48 @@ struct AgentPersistenceTests { ).isEmpty) } + @Test + func schemaV8MigratesToV9WithEmptyCompactionBacking() throws { + let directory = FileManager.default.temporaryDirectory + .appendingPathComponent("TermindAgentV9Migration-\(UUID().uuidString)") + try FileManager.default.createDirectory( + at: directory, withIntermediateDirectories: true + ) + defer { try? FileManager.default.removeItem(at: directory) } + let storeURL = directory.appendingPathComponent("Termind.store") + let legacySchema = Schema(versionedSchema: TermindSchemaV8.self) + let conversationID = UUID() + + var legacyContainer: ModelContainer? = try ModelContainer( + for: legacySchema, + configurations: [ + localModelConfiguration("Migration", schema: legacySchema, url: storeURL) + ] + ) + legacyContainer?.mainContext.insert(PersistentAgentConversation( + id: conversationID, title: "Existing" + )) + try legacyContainer?.mainContext.save() + legacyContainer = nil + + let migrated = try ModelContainer( + for: HostPersistence.schema, + migrationPlan: TermindMigrationPlan.self, + configurations: [ + localModelConfiguration( + "Migration", schema: HostPersistence.schema, url: storeURL + ) + ] + ) + + #expect(try migrated.mainContext.fetch( + FetchDescriptor() + ).map(\.id) == [conversationID]) + #expect(try migrated.mainContext.fetch( + FetchDescriptor() + ).isEmpty) + } + @Test func displayReasoningRoundTripsRedactedWithoutProviderState() async throws { let configuration = localModelConfiguration( @@ -786,6 +828,10 @@ struct AgentPersistenceTests { let forkID = UUID() let originalMessageID = UUID() let forkMessageID = UUID() + let forkCompaction = AgentCompactionRecord( + compactedThroughMessageID: nil, sequence: 0, summary: "Fork base", + summarizedTokens: 10, keptTokens: 2, displayAnchor: .now + ) try await repository.save(AgentRunSnapshot( conversationID: originalID, state: .completed, permissionMode: .askForApproval, @@ -797,16 +843,204 @@ struct AgentPersistenceTests { id: forkID, messages: [AgentTranscriptMessage( id: forkMessageID, role: .user, text: "Original", authoredText: "Original" - )], lastRunState: .completed, + )], compactions: [forkCompaction], lastRunState: .completed, metadata: StoredAgentConversationMetadata(customTitle: "Original (1)") )) let restored = try await repository.loadConversations() #expect(Set(restored.map(\.id)) == Set([originalID, forkID])) #expect(restored.first { $0.id == forkID }?.messages.first?.id == forkMessageID) + #expect(restored.first { $0.id == forkID }?.modelTranscript?.first?.id + == forkCompaction.summaryMessageID) #expect(restored.first { $0.id == forkID }?.metadata.customTitle == "Original (1)") try await repository.deleteConversation(originalID) #expect(try await repository.loadConversations().map(\.id) == [forkID]) + try await repository.deleteConversation(forkID) + #expect(try container.mainContext.fetch( + FetchDescriptor() + ).isEmpty) + } + + @Test + func compactionPersistsImmediatelyWithoutReplacingDisplayHistory() async throws { + let configuration = localModelConfiguration( + "AgentCompactionPersistenceTests", schema: HostPersistence.schema, + isStoredInMemoryOnly: true + ) + let container = try ModelContainer( + for: HostPersistence.schema, configurations: [configuration] + ) + let conversationID = UUID() + let old = AgentTranscriptMessage( + role: .user, text: "Old question " + String(repeating: "x", count: 4_000) + ) + let answer = AgentTranscriptMessage( + role: .assistant, text: "Old answer " + String(repeating: "y", count: 4_000) + ) + let recent = AgentTranscriptMessage(role: .user, text: "Recent question") + let repository = SwiftDataAgentRunRepository(context: container.mainContext) + try await repository.save(AgentRunSnapshot( + conversationID: conversationID, state: .completed, + permissionMode: .askForApproval, messages: [old, answer, recent] + )) + let record = AgentCompactionRecord( + compactedThroughMessageID: answer.id, sequence: 0, + summary: "## Goal\nKeep the recent work", + summarizedTokens: 20, keptTokens: 8, + displayAnchor: answer.createdAt + ) + + try await repository.persistCompaction(record, conversationID: conversationID) + + let reopened = SwiftDataAgentRunRepository(context: container.mainContext) + let restored = try #require(try await reopened.loadConversations().first) + #expect(restored.messages.map(\.id) == [old.id, answer.id, recent.id]) + #expect(restored.modelTranscript?.map(\.id) == [record.summaryMessageID, recent.id]) + #expect(restored.modelTranscript?.first?.isCompaction == true) + #expect(restored.compactions == [record]) + let store = AssistantConversationStore() + store.restore([restored]) + let live = try #require(store.conversation(id: conversationID)) + #expect(live.messages.map(\.id) == [old.id, answer.id, recent.id]) + #expect(live.transcript.map(\.id) == [record.summaryMessageID, recent.id]) + #expect(live.compactions.map(\.id) == [record.id]) + let fullContext = ConversationModel(transcript: restored.messages).contextEstimate.total + #expect(live.contextEstimate.total < fullContext) + #expect(try container.mainContext.fetch( + FetchDescriptor() + ).count == 1) + } + + @Test + func aRunAfterCompactionDoesNotPersistTheSyntheticSummaryAsAMessage() async throws { + let configuration = localModelConfiguration( + "AgentCompactionNextRunTests", schema: HostPersistence.schema, + isStoredInMemoryOnly: true + ) + let container = try ModelContainer( + for: HostPersistence.schema, configurations: [configuration] + ) + let conversationID = UUID() + let old = AgentTranscriptMessage(role: .user, text: "Old") + let recent = AgentTranscriptMessage(role: .assistant, text: "Recent") + let repository = SwiftDataAgentRunRepository(context: container.mainContext) + try await repository.save(AgentRunSnapshot( + conversationID: conversationID, state: .completed, + permissionMode: .askForApproval, messages: [old, recent] + )) + let record = AgentCompactionRecord( + compactedThroughMessageID: old.id, sequence: 0, summary: "Summary", + summarizedTokens: 10, keptTokens: 2, displayAnchor: old.createdAt + ) + try await repository.persistCompaction(record, conversationID: conversationID) + let followUp = AgentTranscriptMessage(role: .user, text: "Continue") + try await repository.save(AgentRunSnapshot( + conversationID: conversationID, state: .completed, + permissionMode: .askForApproval, + messages: [record.summaryMessage, recent, followUp] + )) + + let messageRows = try container.mainContext.fetch( + FetchDescriptor() + ) + let compactionMessageCount = messageRows.filter { $0.isCompaction }.count + #expect(compactionMessageCount == 0) + #expect(!messageRows.contains { $0.id == record.summaryMessageID }) + let restored = try #require(try await repository.loadConversations().first) + #expect(restored.messages.map(\.id) == [old.id, recent.id, followUp.id]) + #expect(restored.modelTranscript?.map(\.id) == [ + record.summaryMessageID, recent.id, followUp.id, + ]) + } + + @Test + func newestOfTwoCompactionsWinsAfterRestore() async throws { + let configuration = localModelConfiguration( + "AgentRepeatedCompactionTests", schema: HostPersistence.schema, + isStoredInMemoryOnly: true + ) + let container = try ModelContainer( + for: HostPersistence.schema, configurations: [configuration] + ) + let conversationID = UUID() + let messages = [ + AgentTranscriptMessage(role: .user, text: "One"), + AgentTranscriptMessage(role: .assistant, text: "Two"), + AgentTranscriptMessage(role: .user, text: "Three"), + AgentTranscriptMessage(role: .assistant, text: "Four"), + ] + let repository = SwiftDataAgentRunRepository(context: container.mainContext) + try await repository.save(AgentRunSnapshot( + conversationID: conversationID, state: .completed, + permissionMode: .askForApproval, messages: messages + )) + let first = AgentCompactionRecord( + compactedThroughMessageID: messages[1].id, sequence: 0, summary: "First", + summarizedTokens: 10, keptTokens: 2, displayAnchor: messages[1].createdAt, + compactedAt: Date(timeIntervalSinceReferenceDate: 1) + ) + let second = AgentCompactionRecord( + compactedThroughMessageID: messages[2].id, sequence: 1, summary: "Second", + summarizedTokens: 12, keptTokens: 2, displayAnchor: messages[2].createdAt, + compactedAt: Date(timeIntervalSinceReferenceDate: 2) + ) + try await repository.persistCompaction(first, conversationID: conversationID) + try await repository.persistCompaction(second, conversationID: conversationID) + + let restored = try #require(try await repository.loadConversations().first) + #expect(restored.messages.map(\.id) == messages.map(\.id)) + #expect(restored.modelTranscript?.map(\.id) == [second.summaryMessageID, messages[3].id]) + #expect(restored.compactions == [first, second]) + } + + @Test + func truncationDeletesOnlyCompactionsWhoseBoundaryWasRemoved() async throws { + let configuration = localModelConfiguration( + "AgentCompactionTruncationTests", schema: HostPersistence.schema, + isStoredInMemoryOnly: true + ) + let container = try ModelContainer( + for: HostPersistence.schema, configurations: [configuration] + ) + let conversationID = UUID() + let firstUser = AgentTranscriptMessage(role: .user, text: "One") + let firstAnswer = AgentTranscriptMessage(role: .assistant, text: "Answer one") + let secondUser = AgentTranscriptMessage(role: .user, text: "Two") + var firstRun = AgentRunSnapshot( + conversationID: conversationID, state: .completed, + permissionMode: .askForApproval, messages: [firstUser, firstAnswer], + startedAt: Date(timeIntervalSinceReferenceDate: 1) + ) + firstRun.finishedAt = Date(timeIntervalSinceReferenceDate: 2) + var secondRun = AgentRunSnapshot( + conversationID: conversationID, state: .completed, + permissionMode: .askForApproval, + messages: [firstUser, firstAnswer, secondUser], + startedAt: Date(timeIntervalSinceReferenceDate: 3) + ) + secondRun.finishedAt = Date(timeIntervalSinceReferenceDate: 4) + let repository = SwiftDataAgentRunRepository(context: container.mainContext) + try await repository.save(firstRun) + try await repository.save(secondRun) + let kept = AgentCompactionRecord( + compactedThroughMessageID: firstAnswer.id, sequence: 0, summary: "Kept", + summarizedTokens: 8, keptTokens: 2, displayAnchor: firstAnswer.createdAt + ) + let removed = AgentCompactionRecord( + compactedThroughMessageID: secondUser.id, sequence: 1, summary: "Removed", + summarizedTokens: 10, keptTokens: 2, displayAnchor: secondUser.createdAt + ) + try await repository.persistCompaction(kept, conversationID: conversationID) + try await repository.persistCompaction(removed, conversationID: conversationID) + + try await repository.truncateConversation( + conversationID, fromUserMessageID: secondUser.id, + replacingWith: AgentTranscriptMessage(role: .user, text: "Edited") + ) + + let restored = try #require(try await repository.loadConversations().first) + #expect(restored.compactions == [kept]) + #expect(restored.modelTranscript?.first?.id == kept.summaryMessageID) } } diff --git a/TermindTests/AgentRuntimeTests.swift b/TermindTests/AgentRuntimeTests.swift index defb1b24..9d94f30f 100644 --- a/TermindTests/AgentRuntimeTests.swift +++ b/TermindTests/AgentRuntimeTests.swift @@ -921,6 +921,9 @@ private actor RecordingRunRepository: AgentRunPersisting { var recorded: [AgentEvent] = [] func save(_ snapshot: AgentRunSnapshot) {} func record(_ event: AgentEvent, runID: UUID) { recorded.append(event) } + func persistCompaction( + _ record: AgentCompactionRecord, conversationID: UUID + ) {} func loadConversations() -> [StoredAgentConversation] { [] } func deleteConversation(_ id: UUID) {} func setTitle(_ title: String, conversationID: UUID) {} diff --git a/TermindTests/WindowStateTests.swift b/TermindTests/WindowStateTests.swift index 15d55b73..d5571568 100644 --- a/TermindTests/WindowStateTests.swift +++ b/TermindTests/WindowStateTests.swift @@ -277,6 +277,36 @@ struct WindowStateTests { #expect(window.assistantOperationError == "Storage unavailable") } + @Test + func failedCompactionSaveDoesNotChangeTheInMemoryContext() async throws { + let repository = FailingConversationOperationRepository() + let hosts = HostLibraryModel() + let app = AppModel( + hosts: hosts, + dependencies: AppDependencies( + hostRepository: InMemoryHostRepository(), outboxRunner: nil, + agentRunRepository: repository + ) + ) + let conversation = ConversationModel() + conversation.appendUserTurn("old " + String(repeating: "x", count: 4_000)) + conversation.appendUserTurn("recent") + let span = try #require(conversation.compactionSpan(keepRecentTokens: 2)) + let record = try #require(conversation.prepareCompaction( + summary: "summary", cut: span.cut + )) + let transcript = conversation.transcript + let estimate = conversation.contextEstimate + + await #expect(throws: (any Error).self) { + try await app.persistAndApplyCompaction(record, to: conversation, cut: span.cut) + } + + #expect(conversation.transcript == transcript) + #expect(conversation.contextEstimate == estimate) + #expect(conversation.compactions.isEmpty) + } + /// Each window used to build its own `ConversationModel` for the same /// conversation id, so a reply streaming in one window was invisible in the /// other and both could start a run against the same stored history. @@ -1165,6 +1195,11 @@ private actor FailingConversationOperationRepository: AgentRunPersisting { func save(_ snapshot: AgentRunSnapshot) {} func record(_ event: AgentEvent, runID: UUID) {} + func persistCompaction( + _ record: AgentCompactionRecord, conversationID: UUID + ) throws { + throw StorageFailure() + } func loadConversations() -> [StoredAgentConversation] { [] } func deleteConversation(_ id: UUID) {} func setTitle(_ title: String, conversationID: UUID) {}