feat(plugin-api): transcribe plugin-owned media on client

apps/web/src/lib/plugins/plugin-host.ts

@@ -1091,6 +1091,15 @@ export class EdgeEverPluginHost {
           lifetime.signal.throwIfAborted();
           return result;
         },
+        transcribeMedia: async (media, options) => {
+          assertPermission(manifest, 'ai:generate');
+          lifetime.signal.throwIfAborted();
+          const signal = AbortSignal.any([lifetime.signal, ...(options?.signal ? [options.signal] : [])]);
+          const { transcribeMediaBlob } = await import("@/lib/transcribe-note-resource");
+          const result = await transcribeMediaBlob(media, signal);
+          signal.throwIfAborted();
+          return result;
+        },
       },
       notes: {
         query: async (input = {}) => {

apps/web/src/lib/transcribe-note-resource.test.mjs

@@ -2,7 +2,7 @@ import { describe, expect, test } from "bun:test";
 import { readFile } from "node:fs/promises";
 import { fileURLToPath } from "node:url";
 import { ALL_FORMATS, BufferSource, Input } from "mediabunny";
-import { extractAudioParts, testSpeechService, transcribePreparedAudioParts } from "./transcribe-note-resource.ts";
+import { extractAudioParts, testSpeechService, transcribeMediaBlob, transcribePreparedAudioParts } from "./transcribe-note-resource.ts";
 import { SpeechProviderReachabilityError } from "./speech-transcription-error.ts";
 
 const fixturePath = fileURLToPath(new URL("./fixtures/transcription-video.mp4", import.meta.url));
@@ -52,6 +52,53 @@ describe("client-side note media preparation", () => {
     input.dispose();
   });
 
+  test.each([
+    ["transcription-audio.mp3", "audio/mpeg"],
+    ["transcription-video.mp4", "video/mp4"],
+  ])("transcribes a plugin-supplied %s locally without uploading it", async (filename, mediaType) => {
+    const bytes = await readFile(fileURLToPath(new URL(`./fixtures/${filename}`, import.meta.url)));
+    const source = new File([bytes], filename, { type: mediaType });
+    const requests = [];
+    const controller = new AbortController();
+    const result = await transcribeMediaBlob(source, controller.signal, {
+      getSettings: async () => ({
+        enabled: true,
+        encryptionConfigured: true,
+        readOnly: false,
+        defaultModelId: "model-1",
+        providers: [{
+          id: "provider-1",
+          provider: "openai-compatible",
+          displayName: "Speech",
+          baseUrl: "https://speech.example/v1",
+          isEnabled: true,
+          hasApiKey: true,
+          models: [{ id: "model-1", providerId: "provider-1", modelId: "whisper-1", displayName: "Whisper" }],
+        }],
+      }),
+      getCredential: async (providerId, signal) => {
+        requests.push({ kind: "credential", providerId, signal });
+        return { apiKey: "test-token" };
+      },
+      providerFetch: async (url, init) => {
+        requests.push({ kind: "provider", url: String(url), init });
+        return Response.json({ text: "Recognized speech" });
+      },
+    });
+    expect(result).toEqual({ text: "Recognized speech" });
+    expect(requests.map(({ kind }) => kind)).toEqual(["credential", "provider"]);
+    expect(requests[0]).toMatchObject({ providerId: "provider-1", signal: controller.signal });
+    expect(requests[1].url).toBe("https://speech.example/v1/audio/transcriptions");
+    expect(requests[1].init.signal.aborted).toBe(false);
+    expect(requests[1].init.headers.Authorization).toBe("Bearer test-token");
+    expect(requests[1].init.body.get("model")).toBe("whisper-1");
+    const submittedAudio = requests[1].init.body.get("file");
+    expect(submittedAudio).toBeInstanceOf(File);
+    expect(submittedAudio.type).toStartWith("audio/");
+    controller.abort();
+    expect(requests[1].init.signal.aborted).toBe(true);
+  });
+
   test.each([
     "speech-service-check.en-US.mp3",
     "speech-service-check.zh-CN.mp3",

apps/web/src/lib/transcribe-note-resource.ts

@@ -170,6 +170,41 @@ export async function transcribeNoteResource(
   return { text, resourceId, filename: target.filename };
 }
 
+export async function transcribeMediaBlob(
+  media: Blob,
+  signal?: AbortSignal,
+  transport: {
+    getSettings?: typeof api.getAiTranscriptionSettings;
+    getCredential?: typeof api.getAiTranscriptionDirectCredential;
+    providerFetch?: typeof fetch;
+  } = {},
+): Promise<{ text: string }> {
+  if (!(media instanceof Blob)) throw new TypeError("A media Blob or File is required.");
+  if (!Number.isSafeInteger(media.size) || media.size < 1 || media.size > MAX_SOURCE_BYTES) {
+    throw new Error("The media is unavailable or exceeds the 1 GiB limit.");
+  }
+  signal?.throwIfAborted();
+  const settings = await (transport.getSettings ?? api.getAiTranscriptionSettings)();
+  signal?.throwIfAborted();
+  const provider = settings.providers.find((item) => item.isEnabled
+    && item.models.some((model) => model.id === settings.defaultModelId));
+  const model = provider?.models.find((item) => item.id === settings.defaultModelId);
+  if (!settings.enabled || !provider || !model || !provider.hasApiKey) {
+    throw new Error("No speech recognition model is enabled.");
+  }
+  const { apiKey } = await (transport.getCredential ?? api.getAiTranscriptionDirectCredential)(provider.id, signal);
+  signal?.throwIfAborted();
+  const readRange = async (start: number, end: number) =>
+    new Uint8Array(await media.slice(start, end).arrayBuffer());
+  const text = await transcribePreparedAudioParts(
+    { baseUrl: provider.baseUrl.trim().replace(/\/+$/, ""), modelId: model.modelId, apiKey },
+    extractAudioParts(media.size, readRange, signal),
+    transport.providerFetch ?? directProviderFetch,
+    signal,
+  );
+  return { text };
+}
+
 export async function transcribePreparedAudioParts(
   target: { baseUrl: string; modelId: string; apiKey: string },
   parts: AsyncIterable<File>,

docs/plugin-development.md

@@ -600,12 +600,17 @@ const result = await context.ai.generate({
 });
 const transcript = await context.ai.transcribeResource(noteId, resourceId);
 // { text, resourceId, filename }. The attachment must already belong to the note; online video URLs are not accepted.
+const ownMedia = new File([audioBytes], "recording.mp3", { type: "audio/mpeg" });
+const { text } = await context.ai.transcribeMedia(ownMedia, { signal: controller.signal });
+// A plugin can also pass a video File or Blob; the host extracts its audio locally.
 ```
 
 `system` is limited to 8,000 characters and `prompt` to 90,000. `maxOutputTokens` must be a positive integer and defaults to 3,000 when omitted; the host does not impose a maximum output-token value. Generation is limited to 120 seconds. The model or provider may impose its own limit or reject a request based on available credits. The backend requires an interactive user session, disables AI in public demo mode, and redacts provider errors. AI calls have a four-request per-workspace guard in each backend instance; this is not a distributed quota. Model charges follow the configured provider. Plugin deactivation aborts outstanding calls.
 
 `transcribeResource` accepts only audio or video attachments already uploaded to a note in the current workspace. The client reads the attachment in ranges through the existing resource API, extracts its audio track, and splits it into audio segments of at most 24 MB, then sends them directly to the user's configured speech model service. The instance validates attachment ownership and gives the signed-in client the model configuration; it does not extract or convert audio or relay provider requests. Browser calls require provider CORS support; the desktop main process calls the provider locally. The plugin receives text and decides how to display or save it. This API does not resolve URLs, download third-party media, or expose speech model credentials to plugins.
 
+`transcribeMedia` accepts a plugin-supplied audio or video `Blob`/`File` without uploading it to a note. The client obtains the configured default speech model and credential through the existing authenticated settings APIs, extracts and segments the media locally with the same 1 GiB source, 24 MB segment, and 64-segment limits, and calls the speech provider directly. The API returns only `{ text }`; it does not return the credential. Enabled plugins run as trusted client JavaScript, so this is not a credential isolation boundary. The method does not fetch a URL; a plugin is responsible for obtaining authorized media bytes. Provider charges and browser CORS requirements still apply. Aborting the supplied signal or disabling the plugin stops the call.
+
 The default `network.fetch(url, init)` transport is a trusted browser request. It accepts arbitrary HTTP/HTTPS destinations, methods, bodies, request headers such as `Authorization`, and the requested browser credential mode. It remains subject to the runtime browser's CORS and cookie policy. `networkHosts` is legacy descriptive metadata and is not a security boundary. To read a cross-origin public feed or API without credentials, explicitly select `transport: "public"`; listing `network` and `network:public` remains useful disclosure but is optional.
 
 ```json

docs/plugin-development.zh-CN.md

@@ -600,12 +600,17 @@ const result = await context.ai.generate({
 });
 const transcript = await context.ai.transcribeResource(noteId, resourceId);
 // { text, resourceId, filename }。附件必须已属于该笔记；不接受在线视频 URL。
+const ownMedia = new File([audioBytes], "recording.mp3", { type: "audio/mpeg" });
+const { text } = await context.ai.transcribeMedia(ownMedia, { signal: controller.signal });
+// 也可传入视频 File 或 Blob；宿主会在客户端提取音轨。
 ```
 
 `system` 最多 8,000 字符，`prompt` 最多 90,000 字符。`maxOutputTokens` 必须是正整数，省略时默认 3,000；宿主不设置输出 token 的最大值。生成最长 120 秒。模型或供应商可能有自己的限制，也可能因可用额度不足拒绝请求。后端要求交互式用户会话，公开演示模式禁用 AI，供应商错误脱敏。每个后端实例对每工作区的 AI 调用设置四路并发保护，不是分布式配额。模型费用沿用已配置供应商的计费；停用插件会中止其调用。
 
 `transcribeResource` 仅识别当前工作区笔记中已上传的音视频附件；客户端经原有资源接口分段读取附件、提取音轨并切成不超过 24 MB 的音频片段，直接发送到用户配置的语音模型服务。实例校验附件归属并把模型配置交给已登录客户端，不提取或转换音轨，也不转发模型请求。浏览器直连要求模型服务支持跨域请求；桌面端从本机主进程直连。插件拿到文本后自行决定如何展示或写入笔记。它不解析 URL、不下载第三方媒体，也不向插件返回语音模型密钥。
 
+`transcribeMedia` 接受插件提供的音频或视频 `Blob`／`File`，无需先上传为笔记附件。客户端经现有的已认证设置接口获取默认语音模型及凭据，在本机提取和切分媒体，沿用 1 GiB 源文件、24 MB 单段和 64 段的限制，直接调用语音服务。API 只返回 `{ text }`，不把凭据作为返回值。已启用插件以受信任的客户端 JavaScript 运行，因此这不是凭据隔离边界。此方法不抓取 URL；插件须自行取得有权处理的媒体字节。模型费用与浏览器跨域要求仍适用。传入的取消信号或停用插件都会中止调用。
+
 默认的 `network.fetch(url, init)` 是受信任的浏览器请求，可以访问任意 HTTP／HTTPS 地址，使用任意方法、正文、`Authorization` 等请求头以及调用方指定的浏览器凭据模式；它仍受所在运行时的 CORS 与 Cookie 策略约束。`networkHosts` 仅为兼容旧版保留，不是安全边界。需要无凭据读取跨域公开订阅或 API 时，显式选择 `transport: "public"` 即可；列出 `network` 和 `network:public` 仍有助于披露用途，但不是必需条件：
 
 ```json

docs/video-clip-spec.md

@@ -6,7 +6,7 @@ Supported pages are ordinary YouTube videos and Shorts, and ordinary Bilibili vi
 
 ## Speech transcription for note attachments
 
-Users can explicitly transcribe an audio or MP4/WebM video attachment already uploaded to a note. A workspace configures an OpenAI-compatible speech provider and default model. The API accepts only a note ID and an attachment ID belonging to that note; it does not accept a media URL. The instance validates attachment ownership and gives the signed-in client the current speech model URL, name, and credential; the credential is encrypted at rest on the instance. The web/desktop client reads the attachment in ranges through the existing resource API, extracts its audio track locally, and splits it by size or duration. The client sends each audio segment of at most 24 MB directly to the configured `/audio/transcriptions` service. The instance serves the original attachment bytes but does not extract or convert audio or relay provider requests. Browser calls require the provider to support cross-origin requests and put the credential in the current browser session; the desktop main process calls the provider locally. The result is shown for copying or insertion into the current note. Plugins can use the same client-side capability through `context.ai.transcribeResource(noteId, resourceId)` but do not receive provider credentials. Attachments remain subject to the normal upload limit; unreadable formats or audio tracks return an error.
+Users can explicitly transcribe an audio or MP4/WebM video attachment already uploaded to a note. A workspace configures an OpenAI-compatible speech provider and default model. The note-resource API accepts only a note ID and an attachment ID belonging to that note; it does not accept a media URL. The instance validates attachment ownership and gives the signed-in client the current speech model URL, name, and credential; the credential is encrypted at rest on the instance. The web/desktop client reads the attachment in ranges through the existing resource API, extracts its audio track locally, and splits it by size or duration. The client sends each audio segment of at most 24 MB directly to the configured `/audio/transcriptions` service. The instance serves the original attachment bytes but does not extract or convert audio or relay provider requests. Browser calls require the provider to support cross-origin requests and put the credential in the current browser session; the desktop main process calls the provider locally. The result is shown for copying or insertion into the current note. Plugins can use the same client-side capability through `context.ai.transcribeResource(noteId, resourceId)`, or pass their own audio/video `Blob` to `context.ai.transcribeMedia(blob)`; neither method returns provider credentials or accepts a media URL. Attachments remain subject to the normal upload limit; unreadable formats or audio tracks return an error.
 
 When adding a service, users can test the unsaved URL, model ID, and token. When editing, they can select an existing model and test with either the stored token or a replacement. The current client sends a short bundled speech sample directly to the model service and shows its returned transcript. Testing does not save the form and may incur a small provider charge. Browser CORS limitations are the same as for normal transcription.
 

docs/video-clip-spec.zh-CN.md

@@ -6,7 +6,7 @@
 
 ## 笔记附件语音识别
 
-用户可以在已上传到笔记的音频或 MP4/WebM 视频附件上主动选择「转录」。工作区配置 OpenAI 兼容语音模型和默认模型；API 仅接受笔记 ID 与归属该笔记的附件 ID，不接受媒体 URL。实例校验附件归属，并把当前语音模型地址、名称及凭据交给已登录客户端；凭据在实例上加密保存。Web/桌面客户端经原有资源接口分段读取附件，在本机提取音轨并按大小或时长切段，由客户端把每段不超过 24 MB 的音频直接发送到用户配置的 `/audio/transcriptions` 服务。实例只提供原有附件字节，不提取或转换音轨，也不转发模型请求。Web 浏览器中的直连要求模型服务支持跨域请求，并会使模型凭据进入当前浏览器会话；桌面端由本机主进程直接请求模型服务。识别文本先展示给用户，可复制或插入当前笔记。插件可通过 `context.ai.transcribeResource(noteId, resourceId)` 调用同一客户端能力，但不会拿到模型凭据。附件仍受普通上传上限约束；无法读取的媒体格式或音轨会明确报错。
+用户可以在已上传到笔记的音频或 MP4/WebM 视频附件上主动选择「转录」。工作区配置 OpenAI 兼容语音模型和默认模型；笔记附件 API 仅接受笔记 ID 与归属该笔记的附件 ID，不接受媒体 URL。实例校验附件归属，并把当前语音模型地址、名称及凭据交给已登录客户端；凭据在实例上加密保存。Web/桌面客户端经原有资源接口分段读取附件，在本机提取音轨并按大小或时长切段，由客户端把每段不超过 24 MB 的音频直接发送到用户配置的 `/audio/transcriptions` 服务。实例只提供原有附件字节，不提取或转换音轨，也不转发模型请求。Web 浏览器中的直连要求模型服务支持跨域请求，并会使模型凭据进入当前浏览器会话；桌面端由本机主进程直接请求模型服务。识别文本先展示给用户，可复制或插入当前笔记。插件可通过 `context.ai.transcribeResource(noteId, resourceId)` 调用同一客户端能力，也可把自己的音视频 `Blob` 交给 `context.ai.transcribeMedia(blob)`；两种方法都不在返回值中提供模型凭据，也不接受媒体 URL。附件仍受普通上传上限约束；无法读取的媒体格式或音轨会明确报错。
 
 添加服务时可用尚未保存的地址、模型 ID 和 Token 测试连接；编辑服务时可选择已添加的模型，并沿用已保存的 Token 或输入新 Token 测试。测试从当前客户端直连模型服务，发送一段内置短语音并显示返回的文本；测试不保存表单，可能产生少量模型费用。浏览器跨域限制与真实转录相同。
 

packages/plugin-api/README.md

@@ -44,6 +44,8 @@ await context.ui.openNote(noteId, { search: "New task" }); // requires ui:naviga
 
 Bulk-indexing plugins can page through `context.notes.queryContent()`. The API also exposes workspace templates, full live-editor reads and range edits, template mutation events, and programmatic opening of a plugin's own registered panels.
 
+Speech plugins can transcribe a note attachment with `context.ai.transcribeResource(noteId, resourceId)`, or pass an audio/video `Blob` or `File` to `context.ai.transcribeMedia(media)`. EdgeEver extracts audio on the client and calls the user's configured speech provider directly. Neither method accepts a media URL or returns the provider credential.
+
 Dashboard and workflow panels can ask EdgeEver to render system chrome (header actions, search, tabs, selects, empty states) through `mount` context `shell.set()`, while the plugin keeps owning the body DOM.
 
 Canvas-style plugins can read and conflict-safely replace resource bytes, open full-screen guarded panels with JSON state, and register constrained block embed renderers:

packages/plugin-api/dist/index.d.ts

@@ -486,6 +486,11 @@ export interface PluginContext {
             resourceId: string;
             filename: string;
         }>;
+        transcribeMedia(media: Blob, options?: {
+            signal?: AbortSignal;
+        }): Promise<{
+            text: string;
+        }>;
     };
     notes: {
         query(input?: PluginNoteQuery): Promise<PluginNoteQueryResult>;

packages/plugin-api/src/index.ts

@@ -498,6 +498,7 @@ export interface PluginContext {
     status(): Promise<{ configured: boolean; modelName?: string }>;
     generate(input: { system: string; prompt: string; maxOutputTokens?: number; signal?: AbortSignal }): Promise<{ text: string }>;
     transcribeResource(noteId: string, resourceId: string): Promise<{ text: string; resourceId: string; filename: string }>;
+    transcribeMedia(media: Blob, options?: { signal?: AbortSignal }): Promise<{ text: string }>;
   };
   notes: {
     query(input?: PluginNoteQuery): Promise<PluginNoteQueryResult>;