From f3b4226555bbc401b94b95642af26f2c8ec7e4a2 Mon Sep 17 00:00:00 2001 From: tianma-if Date: Wed, 7 Oct 2026 19:36:19 +0800 Subject: [PATCH] feat(speech): transcribe note media attachments without external downloads --- apps/api/src/ai-routes.test.mjs | 31 ++ apps/api/src/ai-routes.ts | 325 ++++++++++++++++++ apps/api/src/ai-service.ts | 195 +++++++++++ apps/api/src/resource-transcription.test.mjs | 75 ++++ apps/api/src/resource-transcription.ts | 79 +++++ apps/web/src/components/EditorPane.tsx | 72 ++++ apps/web/src/components/SettingsPane.tsx | 2 + .../dialogs/AttachmentTranscriptDialog.tsx | 52 +++ .../components/editor/EditorPaneChrome.tsx | 15 +- .../settings/SpeechProviderCard.tsx | 325 ++++++++++++++++++ .../settings/SpeechTranscriptionCard.tsx | 274 +++++++++++++++ apps/web/src/lib/plugins/plugin-host.ts | 11 +- docs/plugin-development.md | 4 + docs/plugin-development.zh-CN.md | 4 + docs/video-clip-spec.md | 10 +- docs/video-clip-spec.zh-CN.md | 10 +- .../0065_restore_speech_transcription.sql | 42 +++ packages/client/src/index.ts | 63 ++++ packages/plugin-api/dist/index.d.ts | 5 + packages/plugin-api/src/index.ts | 1 + packages/shared/src/i18n/en-US.ts | 48 +++ packages/shared/src/i18n/ja.ts | 48 +++ packages/shared/src/i18n/pl.ts | 48 +++ packages/shared/src/i18n/zh-CN.ts | 48 +++ packages/shared/src/schemas.ts | 36 ++ packages/shared/src/types.ts | 29 ++ ...transcription-retirement-migration.test.ts | 13 +- 27 files changed, 1856 insertions(+), 9 deletions(-) create mode 100644 apps/api/src/resource-transcription.test.mjs create mode 100644 apps/api/src/resource-transcription.ts create mode 100644 apps/web/src/components/dialogs/AttachmentTranscriptDialog.tsx create mode 100644 apps/web/src/components/settings/SpeechProviderCard.tsx create mode 100644 apps/web/src/components/settings/SpeechTranscriptionCard.tsx create mode 100644 migrations/0065_restore_speech_transcription.sql diff --git a/apps/api/src/ai-routes.test.mjs b/apps/api/src/ai-routes.test.mjs index af72b279..14099540 100644 --- a/apps/api/src/ai-routes.test.mjs +++ b/apps/api/src/ai-routes.test.mjs @@ -107,6 +107,37 @@ const validSettings = { }; describe("AI route contracts", () => { + test("speech settings stay encrypted and transcription requires a note-owned resource", async () => { + const app = createApp(); + const { environment: databaseEnvironment } = createDatabaseEnvironment(); + const saved = await app.request( + "/api/v1/ai/transcription-providers", + { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ ...validSettings, initialModelId: "whisper-1" }), + }, + databaseEnvironment, + ); + expect(saved.status).toBe(201); + expect(JSON.stringify(await saved.json())).not.toContain("secret"); + + const transcript = await app.request( + "/api/v1/memos/missing/resources/missing/transcribe", + { method: "POST" }, + databaseEnvironment, + ); + expect(transcript.status).toBe(404); + + const scopedApp = createApp({ currentAuth: { ...auth, kind: "agent", scopes: ["read:resources"] } }); + const denied = await scopedApp.request( + "/api/v1/memos/missing/resources/missing/transcribe", + { method: "POST" }, + databaseEnvironment, + ); + expect(denied.status).toBe(403); + }); + test("accepts the shared semantic action catalog with required parameters", () => { for (const action of AI_ACTIONS) { const parsed = AiGenerateSchema.safeParse({ diff --git a/apps/api/src/ai-routes.ts b/apps/api/src/ai-routes.ts index 55c90eba..3ea07537 100644 --- a/apps/api/src/ai-routes.ts +++ b/apps/api/src/ai-routes.ts @@ -1,6 +1,10 @@ import { AiDefaultModelUpdateSchema, AiGenerateSchema, + AiTranscriptionDefaultModelUpdateSchema, + AiTranscriptionModelCreateSchema, + AiTranscriptionProviderCreateSchema, + AiTranscriptionProviderUpdateSchema, InfographicAgentRequestSchema, AiModelConfigCreateSchema, AiProviderConfigCreateSchema, @@ -40,6 +44,11 @@ import { normalizeAiGenerationText, normalizeAiBaseUrl, prepareAiGeneration, + aiTranscriptionDefaultStatement, + getAiTranscriptionModel, + getAiTranscriptionProvider, + getAiTranscriptionSettings, + getDefaultAiTranscriptionModelId, resolvePrimaryAiCredentialEncryptionKey, streamAiGeneration, testAiModel, @@ -59,6 +68,7 @@ import { } from "./video-outline"; import { encryptSecret } from "./secret-encryption"; import { listTagSummaries } from "./tag-service"; +import { transcribeNoteResource } from "./resource-transcription"; type AiRouteDependencies = { isDemoMode: (environment: Bindings) => boolean; @@ -109,6 +119,28 @@ const readSettings = (context: AppContext, dependencies: AiRouteDependencies) => context.env, ); +const readTranscriptionSettings = (context: AppContext, dependencies: AiRouteDependencies) => + getAiTranscriptionSettings( + context.env.storage.db, + getWorkspaceId(context), + encryptionConfigured(context), + dependencies.isDemoMode(context.env), + context.env, + ); + +const invalidTranscriptionSettings = ( + result: { success: boolean }, + context: Context, +) => { + if (result.success) return; + return apiError( + context, + "ai_transcription_request_invalid", + "The speech transcription settings are invalid.", + 400, + ); +}; + const denyMutation = (context: AppContext, dependencies: AiRouteDependencies) => { const denied = requireUser(context); if (denied) return denied; @@ -214,6 +246,299 @@ export const registerAiRoutes = (app: Hono, dependencies: AiRouteDepende return context.json(await readSettings(context, dependencies)); }); + app.get("/api/v1/ai/transcription-settings", async (context) => { + const denied = requireUser(context); + if (denied) return denied; + return context.json(await readTranscriptionSettings(context, dependencies)); + }); + + app.post("/api/v1/memos/:memoId/resources/:resourceId/transcribe", async (context) => { + const denied = requireScopes(context, "read:resources", "ai:generate"); + if (denied) return denied; + if (dependencies.isDemoMode(context.env)) { + return forbidden(context, "Speech transcription is unavailable in demo mode."); + } + try { + return context.json(await transcribeNoteResource( + context.env, + getWorkspaceId(context), + context.req.param("memoId"), + context.req.param("resourceId"), + fetch, + context.req.raw.signal, + )); + } catch (error) { + return withAiError(context, error, "ai_transcription_failed"); + } + }); + + app.post( + "/api/v1/ai/transcription-providers", + zValidator("json", AiTranscriptionProviderCreateSchema, invalidTranscriptionSettings), + async (context) => { + const denied = denyMutation(context, dependencies); + if (denied) return denied; + try { + const input = context.req.valid("json"); + const workspaceId = getWorkspaceId(context); + const providerId = createId("atp"); + const modelConfigId = input.initialModelId ? createId("atm") : null; + const now = isoNow(); + const statements = [ + context.env.storage.db.prepare( + `INSERT INTO ai_transcription_providers ( + id, workspace_id, provider, display_name, base_url, api_key_encrypted, + is_enabled, created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, + ).bind( + providerId, + workspaceId, + input.provider, + input.displayName, + normalizeAiBaseUrl(input.baseUrl), + await encryptSecret(input.apiKey, requireEncryptionKey(context)), + input.isEnabled ? 1 : 0, + now, + now, + ), + ]; + if (modelConfigId && input.initialModelId) { + statements.push(context.env.storage.db.prepare( + `INSERT INTO ai_transcription_models ( + id, provider_id, model_id, display_name, created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?)`, + ).bind( + modelConfigId, + providerId, + input.initialModelId, + input.initialModelId.slice(0, 80), + now, + now, + )); + if (input.isEnabled && !(await getDefaultAiTranscriptionModelId(context.env.storage.db, workspaceId))) { + statements.push(aiTranscriptionDefaultStatement( + context.env.storage.db, + workspaceId, + modelConfigId, + now, + )); + } + } + statements.push(auditStatement( + context.env.storage.db, + "user", + context.get("auth").actorId, + "workspace.ai_transcription_provider.create", + "ai_transcription_provider", + providerId, + { isEnabled: input.isEnabled, provider: input.provider, modelId: input.initialModelId ?? null }, + )); + await context.env.storage.db.batch(statements); + return context.json(await readTranscriptionSettings(context, dependencies), 201); + } catch (error) { + return withAiError(context, error, "ai_transcription_provider_create_failed"); + } + }, + ); + + app.put( + "/api/v1/ai/transcription-providers/:providerId", + zValidator("json", AiTranscriptionProviderUpdateSchema, invalidTranscriptionSettings), + async (context) => { + const denied = denyMutation(context, dependencies); + if (denied) return denied; + try { + const input = context.req.valid("json"); + const providerId = context.req.param("providerId"); + const workspaceId = getWorkspaceId(context); + const existing = await getAiTranscriptionProvider(context.env.storage.db, workspaceId, providerId); + if (!existing) return notFound(context, "Speech service not found."); + const now = isoNow(); + const apiKeyEncrypted = input.apiKey + ? await encryptSecret(input.apiKey, requireEncryptionKey(context)) + : existing.api_key_encrypted; + await context.env.storage.db.batch([ + context.env.storage.db.prepare( + `UPDATE ai_transcription_providers SET + provider = ?, display_name = ?, base_url = ?, api_key_encrypted = ?, + is_enabled = ?, updated_at = ? + WHERE id = ? AND workspace_id = ?`, + ).bind( + input.provider, + input.displayName, + normalizeAiBaseUrl(input.baseUrl), + apiKeyEncrypted, + input.isEnabled ? 1 : 0, + now, + providerId, + workspaceId, + ), + auditStatement( + context.env.storage.db, + "user", + context.get("auth").actorId, + "workspace.ai_transcription_provider.update", + "ai_transcription_provider", + providerId, + { isEnabled: input.isEnabled, provider: input.provider, apiKeyUpdated: Boolean(input.apiKey) }, + ), + ]); + return context.json(await readTranscriptionSettings(context, dependencies)); + } catch (error) { + return withAiError(context, error, "ai_transcription_provider_update_failed"); + } + }, + ); + + app.delete("/api/v1/ai/transcription-providers/:providerId", async (context) => { + const denied = denyMutation(context, dependencies); + if (denied) return denied; + const providerId = context.req.param("providerId"); + const workspaceId = getWorkspaceId(context); + const existing = await getAiTranscriptionProvider(context.env.storage.db, workspaceId, providerId); + if (!existing) return notFound(context, "Speech service not found."); + await context.env.storage.db.batch([ + context.env.storage.db.prepare( + `DELETE FROM ai_transcription_providers WHERE id = ? AND workspace_id = ?`, + ).bind(providerId, workspaceId), + auditStatement( + context.env.storage.db, + "user", + context.get("auth").actorId, + "workspace.ai_transcription_provider.delete", + "ai_transcription_provider", + providerId, + {}, + ), + ]); + return context.json(await readTranscriptionSettings(context, dependencies)); + }); + + app.post( + "/api/v1/ai/transcription-providers/:providerId/models", + zValidator("json", AiTranscriptionModelCreateSchema, invalidTranscriptionSettings), + async (context) => { + const denied = denyMutation(context, dependencies); + if (denied) return denied; + const providerId = context.req.param("providerId"); + const workspaceId = getWorkspaceId(context); + const provider = await getAiTranscriptionProvider(context.env.storage.db, workspaceId, providerId); + if (!provider) return notFound(context, "Speech service not found."); + const input = context.req.valid("json"); + const modelConfigId = createId("atm"); + const now = isoNow(); + const statements = [ + context.env.storage.db.prepare( + `INSERT INTO ai_transcription_models ( + id, provider_id, model_id, display_name, created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?)`, + ).bind( + modelConfigId, + providerId, + input.modelId, + input.displayName ?? input.modelId.slice(0, 80), + now, + now, + ), + ]; + if (provider.is_enabled && !(await getDefaultAiTranscriptionModelId(context.env.storage.db, workspaceId))) { + statements.push(aiTranscriptionDefaultStatement( + context.env.storage.db, + workspaceId, + modelConfigId, + now, + )); + } + statements.push(auditStatement( + context.env.storage.db, + "user", + context.get("auth").actorId, + "workspace.ai_transcription_model.create", + "ai_transcription_model", + modelConfigId, + { providerId, modelId: input.modelId }, + )); + try { + await context.env.storage.db.batch(statements); + return context.json(await readTranscriptionSettings(context, dependencies), 201); + } catch (error) { + return withAiError(context, error, "ai_transcription_model_create_failed"); + } + }, + ); + + app.delete( + "/api/v1/ai/transcription-providers/:providerId/models/:modelConfigId", + async (context) => { + const denied = denyMutation(context, dependencies); + if (denied) return denied; + const workspaceId = getWorkspaceId(context); + const providerId = context.req.param("providerId"); + const modelConfigId = context.req.param("modelConfigId"); + const model = await getAiTranscriptionModel(context.env.storage.db, workspaceId, modelConfigId); + if (!model || model.provider_id !== providerId) { + return notFound(context, "Speech model not found."); + } + await context.env.storage.db.batch([ + context.env.storage.db.prepare( + `DELETE FROM ai_transcription_models WHERE id = ? AND provider_id = ?`, + ).bind(modelConfigId, providerId), + auditStatement( + context.env.storage.db, + "user", + context.get("auth").actorId, + "workspace.ai_transcription_model.delete", + "ai_transcription_model", + modelConfigId, + { providerId, modelId: model.model_id }, + ), + ]); + return context.json(await readTranscriptionSettings(context, dependencies)); + }, + ); + + app.put( + "/api/v1/ai/transcription-default-model", + zValidator("json", AiTranscriptionDefaultModelUpdateSchema, invalidTranscriptionSettings), + async (context) => { + const denied = denyMutation(context, dependencies); + if (denied) return denied; + const input = context.req.valid("json"); + const workspaceId = getWorkspaceId(context); + if (input.modelConfigId) { + const model = await context.env.storage.db.prepare( + `SELECT models.id + FROM ai_transcription_models AS models + JOIN ai_transcription_providers AS providers ON providers.id = models.provider_id + WHERE models.id = ? AND providers.workspace_id = ? AND providers.is_enabled = 1 + LIMIT 1`, + ).bind(input.modelConfigId, workspaceId).first<{ id: string }>(); + if (!model) { + return apiError( + context, + "ai_transcription_default_unavailable", + "The selected speech model belongs to a disabled or missing service.", + 400, + ); + } + } + const now = isoNow(); + await context.env.storage.db.batch([ + aiTranscriptionDefaultStatement(context.env.storage.db, workspaceId, input.modelConfigId, now), + auditStatement( + context.env.storage.db, + "user", + context.get("auth").actorId, + "workspace.ai_transcription_default.update", + "workspace", + workspaceId, + { modelConfigId: input.modelConfigId }, + ), + ]); + return context.json(await readTranscriptionSettings(context, dependencies)); + }, + ); + app.post( "/api/v1/ai/providers", zValidator("json", AiProviderConfigCreateSchema), diff --git a/apps/api/src/ai-service.ts b/apps/api/src/ai-service.ts index 7b11b2ab..620b3717 100644 --- a/apps/api/src/ai-service.ts +++ b/apps/api/src/ai-service.ts @@ -8,6 +8,10 @@ import type { AiProvider, AiProviderConfig, AiSettings, + AiTranscriptionModel, + AiTranscriptionProvider, + AiTranscriptionStandard, + AiTranscriptionSettings, AiTargetLanguage, AiTone, } from "@edgeever/shared"; @@ -224,6 +228,197 @@ export const getAiSettings = async ( export const normalizeAiBaseUrl = (value: string) => value.trim().replace(/\/+$/, ""); +export type AiTranscriptionProviderRow = { + id: string; + workspace_id: string; + provider: AiTranscriptionStandard; + display_name: string; + base_url: string; + api_key_encrypted: string; + is_enabled: number; + created_at: string; + updated_at: string; +}; + +export type AiTranscriptionModelRow = { + id: string; + provider_id: string; + model_id: string; + display_name: string; + created_at: string; + updated_at: string; +}; + +const selectTranscriptionProviderSql = `SELECT + id, workspace_id, provider, display_name, base_url, api_key_encrypted, is_enabled, created_at, updated_at + FROM ai_transcription_providers`; + +const selectTranscriptionModelSql = `SELECT + id, provider_id, model_id, display_name, created_at, updated_at + FROM ai_transcription_models`; + +export const getAiTranscriptionProvider = ( + db: DatabaseAdapter, + workspaceId: string, + providerId: string, +) => db.prepare( + `${selectTranscriptionProviderSql} WHERE id = ? AND workspace_id = ? LIMIT 1`, +).bind(providerId, workspaceId).first(); + +export const getAiTranscriptionModel = ( + db: DatabaseAdapter, + workspaceId: string, + modelConfigId: string, +) => db.prepare( + `${selectTranscriptionModelSql} + WHERE id = ? AND provider_id IN ( + SELECT id FROM ai_transcription_providers WHERE workspace_id = ? + ) + LIMIT 1`, +).bind(modelConfigId, workspaceId).first(); + +export const getDefaultAiTranscriptionModelId = async (db: DatabaseAdapter, workspaceId: string) => { + const row = await db.prepare( + `SELECT default_model_id + FROM ai_transcription_workspace_settings + WHERE workspace_id = ? + LIMIT 1`, + ).bind(workspaceId).first<{ default_model_id: string | null }>(); + return row?.default_model_id ?? null; +}; + +export const aiTranscriptionDefaultStatement = ( + db: DatabaseAdapter, + workspaceId: string, + modelConfigId: string | null, + now: string, +) => db.prepare( + `INSERT INTO ai_transcription_workspace_settings ( + workspace_id, default_model_id, created_at, updated_at + ) VALUES (?, ?, ?, ?) + ON CONFLICT(workspace_id) DO UPDATE SET + default_model_id = excluded.default_model_id, + updated_at = excluded.updated_at`, +).bind(workspaceId, modelConfigId, now, now); + +const mapAiTranscriptionModel = (row: AiTranscriptionModelRow): AiTranscriptionModel => ({ + id: row.id, + providerId: row.provider_id, + modelId: row.model_id, + displayName: row.display_name, +}); + +const mapAiTranscriptionProvider = ( + row: AiTranscriptionProviderRow, + models: AiTranscriptionModelRow[], + credentialsUnavailable = false, +): AiTranscriptionProvider => ({ + id: row.id, + provider: row.provider, + displayName: row.display_name, + baseUrl: row.base_url, + isEnabled: Boolean(row.is_enabled), + hasApiKey: Boolean(row.api_key_encrypted), + models: models.filter((model) => model.provider_id === row.id).map(mapAiTranscriptionModel), + ...(credentialsUnavailable ? { credentialsUnavailable: true } : {}), +}); + +const withAiTranscriptionCredentialAvailability = async ( + row: AiTranscriptionProviderRow, + models: AiTranscriptionModelRow[], + encryptionConfigured: boolean, + environment?: AiCredentialEnvironment, +): Promise => { + const mapped = mapAiTranscriptionProvider(row, models); + if (!encryptionConfigured || !environment || !row.api_key_encrypted) return mapped; + try { + await decryptAiCredential(row.api_key_encrypted, environment); + return mapped; + } catch { + return mapAiTranscriptionProvider(row, models, true); + } +}; + +export const getAiTranscriptionSettings = async ( + db: DatabaseAdapter, + workspaceId: string, + encryptionConfigured: boolean, + readOnly: boolean, + environment?: AiCredentialEnvironment, +): Promise => { + const [providersResult, modelsResult, defaultModelId] = await Promise.all([ + db.prepare( + `${selectTranscriptionProviderSql} WHERE workspace_id = ? ORDER BY created_at ASC, id ASC`, + ).bind(workspaceId).all(), + db.prepare( + `${selectTranscriptionModelSql} + WHERE provider_id IN ( + SELECT id FROM ai_transcription_providers WHERE workspace_id = ? + ) + ORDER BY created_at ASC, id ASC`, + ).bind(workspaceId).all(), + getDefaultAiTranscriptionModelId(db, workspaceId), + ]); + const providers = await Promise.all(providersResult.results.map((provider) => + withAiTranscriptionCredentialAvailability(provider, modelsResult.results, encryptionConfigured, environment))); + const defaultProvider = providers.find((provider) => + provider.models.some((model) => model.id === defaultModelId)); + const enabled = Boolean( + encryptionConfigured + && defaultProvider?.isEnabled + && defaultProvider.hasApiKey + && !defaultProvider.credentialsUnavailable + && defaultProvider.models.some((model) => model.id === defaultModelId && model.modelId), + ); + return { + providers, + defaultModelId, + enabled, + encryptionConfigured, + readOnly, + }; +}; + +export const prepareAiTranscriptionCredentials = async ( + db: DatabaseAdapter, + workspaceId: string, + environment: AiCredentialEnvironment, +) => { + const row = await db.prepare( + `SELECT + models.model_id, + providers.provider, + providers.base_url, + providers.api_key_encrypted + FROM ai_transcription_workspace_settings AS settings + JOIN ai_transcription_models AS models ON models.id = settings.default_model_id + JOIN ai_transcription_providers AS providers ON providers.id = models.provider_id + WHERE settings.workspace_id = ? + AND providers.workspace_id = ? + AND providers.is_enabled = 1 + LIMIT 1`, + ).bind(workspaceId, workspaceId).first<{ + model_id: string; + provider: AiTranscriptionStandard; + base_url: string; + api_key_encrypted: string; + }>(); + if (!row?.base_url || !row.model_id || !row.api_key_encrypted || !row.provider) { + return { enabled: false as const }; + } + try { + return { + enabled: true as const, + provider: row.provider, + baseUrl: normalizeAiBaseUrl(row.base_url), + modelId: row.model_id, + apiKey: await decryptAiCredential(row.api_key_encrypted, environment), + }; + } catch { + return { enabled: false as const }; + } +}; + const loadAiRuntime = () => import("./ai-runtime"); export const createAiModel = async (config: { diff --git a/apps/api/src/resource-transcription.test.mjs b/apps/api/src/resource-transcription.test.mjs new file mode 100644 index 00000000..cd640e82 --- /dev/null +++ b/apps/api/src/resource-transcription.test.mjs @@ -0,0 +1,75 @@ +import { describe, expect, test } from "bun:test"; +import { resolvePrimaryAiCredentialEncryptionKey } from "./ai-service.ts"; +import { encryptSecret } from "./secret-encryption.ts"; +import { isTranscribableAttachment, transcribeNoteResource } from "./resource-transcription.ts"; + +const makeEnvironment = async (resource) => { + const env = { + EDGE_EVER_AUTH_PASSWORD: "test-password-long-enough", + storage: { + resources: { + get: async () => ({ body: new Blob(["audio bytes"]).stream() }), + }, + db: { + prepare: (sql) => ({ + bind: (...args) => ({ + first: async () => { + if (sql.includes("FROM resources r")) return args[0] === resource.id && args[1] === "ws_one" ? resource : null; + if (sql.includes("FROM ai_transcription_workspace_settings AS settings")) return { + model_id: "whisper-1", + provider: "openai-compatible", + base_url: "https://speech.example/v1", + api_key_encrypted: await encryptSecret("secret-token", resolvePrimaryAiCredentialEncryptionKey(env)), + }; + if (sql.includes("FROM object_storage_configs")) return { id: "builtin", provider: "builtin" }; + return null; + }, + }), + }), + }, + }, + }; + return env; +}; + +const resource = { + id: "res_one", + memo_id: "memo_one", + kind: "attachment", + filename: "meeting.mp4", + mime_type: "video/mp4", + byte_size: 11, + storage_config_id: "builtin", + object_key: "workspace/meeting.mp4", +}; + +describe("note attachment transcription boundary", () => { + test("accepts only supported audio and video attachments", () => { + expect(isTranscribableAttachment(resource)).toBe(true); + expect(isTranscribableAttachment({ ...resource, filename: "meeting.exe" })).toBe(false); + expect(isTranscribableAttachment({ ...resource, mime_type: "application/octet-stream" })).toBe(false); + expect(isTranscribableAttachment({ ...resource, kind: "image" })).toBe(false); + }); + + test("rejects an attachment from another note before reading it", async () => { + const env = await makeEnvironment(resource); + await expect(transcribeNoteResource(env, "ws_one", "memo_other", "res_one", () => { + throw new Error("Provider must not be called"); + })).rejects.toMatchObject({ code: "resource_not_found", status: 404 }); + }); + + test("sends only the note attachment to the configured model and returns text", async () => { + const env = await makeEnvironment(resource); + const calls = []; + const result = await transcribeNoteResource(env, "ws_one", "memo_one", "res_one", async (url, init) => { + calls.push({ url, init }); + return Response.json({ text: " Meeting notes. " }); + }); + expect(result).toEqual({ text: "Meeting notes.", resourceId: "res_one", filename: "meeting.mp4" }); + expect(calls).toHaveLength(1); + expect(calls[0].url).toBe("https://speech.example/v1/audio/transcriptions"); + expect(calls[0].init.headers.Authorization).toBe("Bearer secret-token"); + expect(calls[0].init.body.get("model")).toBe("whisper-1"); + expect((await calls[0].init.body.get("file").text())).toBe("audio bytes"); + }); +}); diff --git a/apps/api/src/resource-transcription.ts b/apps/api/src/resource-transcription.ts new file mode 100644 index 00000000..4e0ee377 --- /dev/null +++ b/apps/api/src/resource-transcription.ts @@ -0,0 +1,79 @@ +import { AppError } from "./app-error"; +import { prepareAiTranscriptionCredentials } from "./ai-service"; +import { resolveObjectStorage } from "./object-storage"; +import { getResourceRow } from "./resource-service"; +import type { Bindings } from "./api-context"; + +export const MAX_TRANSCRIPTION_BYTES = 24 * 1024 * 1024; + +const SUPPORTED_EXTENSIONS = new Set(["flac", "mp3", "mp4", "mpeg", "mpga", "m4a", "ogg", "wav", "webm"]); + +export const isTranscribableAttachment = (resource: { + kind: string; + mime_type: string | null; + filename: string | null; +}) => { + if (resource.kind !== "attachment") return false; + const extension = /\.([a-z0-9]+)$/i.exec(resource.filename ?? "")?.[1]?.toLowerCase(); + if (!extension || !SUPPORTED_EXTENSIONS.has(extension)) return false; + const mime = (resource.mime_type ?? "").toLowerCase(); + return mime.startsWith("audio/") || mime === "video/mp4" || mime === "video/webm"; +}; + +export const transcribeNoteResource = async ( + env: Bindings, + workspaceId: string, + memoId: string, + resourceId: string, + fetchImpl: typeof fetch = fetch, + abortSignal?: AbortSignal, +) => { + const resource = await getResourceRow(env.storage.db, workspaceId, resourceId); + if (!resource || resource.memo_id !== memoId) { + throw new AppError("resource_not_found", "This attachment does not belong to the note.", 404); + } + if (!isTranscribableAttachment(resource)) { + throw new AppError("unsupported_media_type", "Choose an audio or MP4/WebM video attachment.", 415); + } + if (resource.byte_size < 1 || resource.byte_size > MAX_TRANSCRIPTION_BYTES) { + throw new AppError("audio_too_large", "The attachment must be 24 MiB or smaller.", 413); + } + const credentials = await prepareAiTranscriptionCredentials(env.storage.db, workspaceId, env); + if (!credentials.enabled) { + throw new AppError("ai_transcription_not_configured", "Choose a default speech model first.", 409); + } + + const source = await resolveObjectStorage(env, resource.storage_config_id); + const object = await source.store.get(resource.object_key); + if (!object) throw new AppError("resource_not_found", "Attachment bytes are unavailable.", 404); + const bytes = await new Response(object.body).arrayBuffer(); + if (bytes.byteLength < 1 || bytes.byteLength > MAX_TRANSCRIPTION_BYTES) { + throw new AppError("audio_too_large", "The attachment must be 24 MiB or smaller.", 413); + } + + const body = new FormData(); + body.set("model", credentials.modelId); + body.set("file", new File([bytes], resource.filename || "recording.mp3", { + type: resource.mime_type || "application/octet-stream", + })); + let response: Response; + try { + response = await fetchImpl(`${credentials.baseUrl}/audio/transcriptions`, { + method: "POST", + headers: { Authorization: `Bearer ${credentials.apiKey}` }, + body, + signal: abortSignal + ? AbortSignal.any([abortSignal, AbortSignal.timeout(180_000)]) + : AbortSignal.timeout(180_000), + }); + } catch { + throw new AppError("ai_transcription_failed", "The speech service could not be reached.", 502); + } + if (!response.ok) { + throw new AppError("ai_transcription_failed", `The speech service returned HTTP ${response.status}.`, 502); + } + const result = await response.json().catch(() => null) as { text?: unknown } | null; + const text = typeof result?.text === "string" ? result.text.trim() : ""; + if (!text) throw new AppError("empty_transcript", "The speech service returned no transcript.", 502); + return { text, resourceId, filename: resource.filename ?? "" }; +}; diff --git a/apps/web/src/components/EditorPane.tsx b/apps/web/src/components/EditorPane.tsx index c6f836f5..06ac7ccf 100644 --- a/apps/web/src/components/EditorPane.tsx +++ b/apps/web/src/components/EditorPane.tsx @@ -103,6 +103,7 @@ import { sanitizeAndScopeCss } from "@/lib/css-sandbox"; import { RevisionHistoryDialog } from "./dialogs/RevisionHistoryDialog"; import { ExternalLinkDialog } from "./dialogs/ExternalLinkDialog"; import { MathFormulaDialog } from "./dialogs/MathFormulaDialog"; +import { AttachmentTranscriptDialog } from "./dialogs/AttachmentTranscriptDialog"; import { EditorBlockDragHandle } from "./editor/EditorBlockDragHandle"; import { applyMathFormula, @@ -497,6 +498,14 @@ const RichEditorPane = ({ canRemove: false, }); const [mathFormulaOpen, setMathFormulaOpen] = useState(false); + const [attachmentTranscript, setAttachmentTranscript] = useState<{ + memoId: string; + filename: string; + text: string; + loading: boolean; + error: string | null; + } | null>(null); + const attachmentTranscriptAbortRef = useRef(null); const [mathFormulaDraft, setMathFormulaDraft] = useState(null); const { menuTarget: resourceMenuTarget, @@ -1820,6 +1829,11 @@ const RichEditorPane = ({ resetResourceActions(); }, [memo?.id, isMarkdownMode, resetResourceActions]); + useEffect(() => () => { + attachmentTranscriptAbortRef.current?.abort(); + attachmentTranscriptAbortRef.current = null; + }, [memo?.id]); + useEffect(() => () => { if (resourceMenuHideTimerRef.current !== null) { window.clearTimeout(resourceMenuHideTimerRef.current); @@ -3115,6 +3129,30 @@ const RichEditorPane = ({ downloadResourceDirectly(target); }, [clearResourceActionError, downloadResourceDirectly, hideResourceMenu]); + const handleResourceTranscribe = useCallback(async (target: ResourceMenuTarget) => { + const currentMemoId = memoRef.current?.id; + if (!currentMemoId || !target.resourceId) return; + attachmentTranscriptAbortRef.current?.abort(); + const controller = new AbortController(); + attachmentTranscriptAbortRef.current = controller; + hideResourceMenu(); + setAttachmentTranscript({ memoId: currentMemoId, filename: target.filename, text: "", loading: true, error: null }); + try { + const result = await api.transcribeNoteResource(currentMemoId, target.resourceId, controller.signal); + if (attachmentTranscriptAbortRef.current !== controller) return; + setAttachmentTranscript((current) => current?.memoId === currentMemoId && current.filename === target.filename + ? { ...current, text: result.text, loading: false } + : current); + } catch (error) { + if (attachmentTranscriptAbortRef.current !== controller || controller.signal.aborted) return; + setAttachmentTranscript((current) => current?.memoId === currentMemoId && current.filename === target.filename + ? { ...current, loading: false, error: error instanceof Error ? error.message : t("speechTranscription.recognizeFailed") } + : current); + } finally { + if (attachmentTranscriptAbortRef.current === controller) attachmentTranscriptAbortRef.current = null; + } + }, [hideResourceMenu, t]); + const handleResourceSaveAs = useCallback(async (target: ResourceMenuTarget) => { hideResourceMenu(); clearResourceActionError(); @@ -3602,6 +3640,7 @@ const RichEditorPane = ({ const resourceMenuLabels = { download: t("editor.resourceActions.download"), saveAs: t("editor.resourceActions.saveAs"), + transcribe: t("editor.resourceActions.transcribe"), rename: t("editor.resourceActions.rename"), delete: t("editor.resourceActions.delete"), unavailable: t("editor.resourceActions.unavailable"), @@ -4350,9 +4389,17 @@ const RichEditorPane = ({ resourceMenuTarget.resourceId && !resourceMenuTarget.resourceId.startsWith("local_resource_") )) )} + canTranscribe={Boolean( + memo?.id && + resourceMenuTarget.kind === "attachment" && + resourceMenuTarget.resourceId && + !resourceMenuTarget.resourceId.startsWith("local_resource_") && + /\.(flac|mp3|mp4|mpeg|mpga|m4a|ogg|wav|webm)$/i.test(resourceMenuTarget.filename) + )} labels={resourceMenuLabels} onDownload={() => void handleResourceDownload(resourceMenuTarget)} onSaveAs={() => void handleResourceSaveAs(resourceMenuTarget)} + onTranscribe={() => void handleResourceTranscribe(resourceMenuTarget)} onRename={() => openResourceDialog("rename", resourceMenuTarget)} onDelete={() => openResourceDialog("delete", resourceMenuTarget)} onMouseEnter={cancelResourceMenuHide} @@ -4360,6 +4407,31 @@ const RichEditorPane = ({ /> )} + { + if (!open) { + attachmentTranscriptAbortRef.current?.abort(); + attachmentTranscriptAbortRef.current = null; + setAttachmentTranscript(null); + } + }} + onInsert={() => { + if (!editor || !attachmentTranscript?.text || attachmentTranscript.memoId !== memo?.id) return; + const paragraphs = attachmentTranscript.text.split(/\n+/).map((line) => line.trim()).filter(Boolean); + editor.chain().focus().insertContentAt(editor.state.doc.content.size, [ + { type: "heading", attrs: { level: 2 }, content: [{ type: "text", text: t("speechTranscription.resultTitle") }] }, + ...paragraphs.map((line) => ({ type: "paragraph", content: [{ type: "text", text: line }] })), + ]).run(); + setAttachmentTranscript(null); + }} + /> + + ); case "mcp": diff --git a/apps/web/src/components/dialogs/AttachmentTranscriptDialog.tsx b/apps/web/src/components/dialogs/AttachmentTranscriptDialog.tsx new file mode 100644 index 00000000..238c6906 --- /dev/null +++ b/apps/web/src/components/dialogs/AttachmentTranscriptDialog.tsx @@ -0,0 +1,52 @@ +import { useState } from "react"; +import { useTranslation } from "react-i18next"; +import { Button } from "@/components/ui/button"; +import { Dialog, DialogContent, DialogHeader, DialogTitle } from "@/components/ui/dialog"; + +export const AttachmentTranscriptDialog = ({ + open, + filename, + text, + loading, + error, + canInsert, + onOpenChange, + onInsert, +}: { + open: boolean; + filename: string; + text: string; + loading: boolean; + error: string | null; + canInsert: boolean; + onOpenChange: (open: boolean) => void; + onInsert: () => void; +}) => { + const { t } = useTranslation(); + const [copied, setCopied] = useState(false); + return ( + + + + {t("speechTranscription.resultTitle")} + +

{filename}

+ {loading ?

{t("speechTranscription.recognizing")}

: null} + {error ?

{error}

: null} + {text ? ( + <> +

{t("speechTranscription.attachmentNotice")}

+