From 730a1adf93d185423a13fb51c54c43bca2416830 Mon Sep 17 00:00:00 2001 From: Ismail Pelaseyed Date: Tue, 24 Mar 2026 14:18:27 +0100 Subject: [PATCH] feat: add Telegram voice/audio transcription via whisper.cpp (#210) * feat: add Telegram voice/audio transcription via whisper.cpp Telegram voice notes and audio attachments are now transcribed locally using whisper.cpp before being sent to the agent. Includes auto-download of Whisper models, ffmpeg fallback for non-WAV formats, configurable settings under telegram.audioInput, and unit tests for the new STT layer. * fix: harden whisper audio setup Avoid Linux model download failures, remove unused STT dependencies, and keep local Cursor state changes isolated to the intended files. * refactor: share Telegram audio handler Reduce duplicate Telegram voice and audio handling so transcription flow changes stay in sync. --- .cursor/hooks/state/continual-learning.json | 8 + .gitignore | 1 + README.md | 65 +++++ src/audio/stt/engine.ts | 28 ++ src/audio/stt/whisper-cpp.test.ts | 86 +++++++ src/audio/stt/whisper-cpp.ts | 270 ++++++++++++++++++++ src/telegram/audio-input.test.ts | 41 +++ src/telegram/audio-input.ts | 116 +++++++++ src/telegram/bridge.ts | 144 ++++++++--- src/utils/settings.ts | 46 ++++ src/utils/telegram-audio-settings.test.ts | 43 ++++ 11 files changed, 810 insertions(+), 38 deletions(-) create mode 100644 .cursor/hooks/state/continual-learning.json create mode 100644 src/audio/stt/engine.ts create mode 100644 src/audio/stt/whisper-cpp.test.ts create mode 100644 src/audio/stt/whisper-cpp.ts create mode 100644 src/telegram/audio-input.test.ts create mode 100644 src/telegram/audio-input.ts create mode 100644 src/utils/telegram-audio-settings.test.ts diff --git a/.cursor/hooks/state/continual-learning.json b/.cursor/hooks/state/continual-learning.json new file mode 100644 index 0000000..4831afd --- /dev/null +++ b/.cursor/hooks/state/continual-learning.json @@ -0,0 +1,8 @@ +{ + "version": 1, + "lastRunAtMs": 0, + "turnsSinceLastRun": 4, + "lastTranscriptMtimeMs": null, + "lastProcessedGenerationId": "518874b0-1435-472e-9c42-c54214a9ae4e", + "trialStartedAtMs": null +} diff --git a/.gitignore b/.gitignore index c6fc0d1..5027a3d 100644 --- a/.gitignore +++ b/.gitignore @@ -103,5 +103,6 @@ temp/ # Cursor config (tracked — do not ignore) # .cursor/ +.cursor/hooks/state/ .agents skills-lock.json diff --git a/README.md b/README.md index 481b330..87932eb 100644 --- a/README.md +++ b/README.md @@ -183,6 +183,71 @@ Optional: **`GROK_BASE_URL`** (default `https://api.x.ai/v1`), **`GROK_MODEL`**, 3. Start **`grok`**, open **`/remote-control`** → **Telegram** if needed, then in Telegram DM your bot: **`/pair`**, enter the **6-character code** in the terminal when asked. 4. First user must be approved once; after that, it’s remembered. **Keep the CLI process running** while you use the bot (long polling lives in that process). +### Voice & audio messages + +Send a voice note or audio attachment in Telegram and Grok will transcribe it locally with **[whisper.cpp](https://github.com/ggml-org/whisper.cpp)** before passing the text to the agent. No cloud STT service is involved — everything runs on your machine. + +#### Prerequisites + +| Dependency | Why | Install (macOS) | +|---|---|---| +| **whisper-cli** | Runs the actual speech-to-text inference | `brew install whisper-cpp` | +| **ffmpeg** | Converts Telegram voice notes (OGG/Opus) to WAV for whisper.cpp | `brew install ffmpeg` | + +After installing, verify both are available: + +```bash +whisper-cli -h +ffmpeg -version +``` + +#### Download a Whisper model + +Grok CLI auto-downloads the configured model on first use, but you can pre-download it: + +```bash +mkdir -p ~/.grok/models/stt/whisper.cpp +curl -L https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-tiny.en.bin \ + -o ~/.grok/models/stt/whisper.cpp/ggml-tiny.en.bin +``` + +Available models (trade size for accuracy): `tiny.en` (75 MB), `base.en` (142 MB), `small.en` (466 MB). + +#### Configure in `~/.grok/user-settings.json` + +```json +{ + "telegram": { + "botToken": "YOUR_BOT_TOKEN", + "audioInput": { + "enabled": true, + "binaryPath": "/opt/homebrew/bin/whisper-cli", + "model": "tiny.en", + "modelPath": "~/.grok/models/stt/whisper.cpp/ggml-tiny.en.bin", + "autoDownloadModel": true, + "language": "en" + } + } +} +``` + +| Setting | Default | Description | +|---|---|---| +| `enabled` | `true` | Set to `false` to ignore voice/audio messages entirely. | +| `binaryPath` | `whisper-cli` | Absolute path or command name for the whisper.cpp CLI binary. | +| `model` | `tiny.en` | Model alias used for auto-download resolution. | +| `modelPath` | _(auto-resolved)_ | Explicit path to a `.bin` model file. Overrides `model` + auto-download. | +| `autoDownloadModel` | `true` | Download the model into `~/.grok/models/stt/whisper.cpp` on first use. | +| `language` | `en` | Whisper language code passed to the CLI. | + +#### Test it + +1. Restart the CLI after changing settings: `bun run build && node dist/index.js` +2. Send a **3–5 second voice note** with a clear spoken sentence (e.g. "list the files in the current directory"). +3. The bot should reply as if you had typed that sentence. + +Very short or silent voice notes may produce an empty transcript — the agent will tell you the audio was blank. + Treat the bot token like a password. --- diff --git a/src/audio/stt/engine.ts b/src/audio/stt/engine.ts new file mode 100644 index 0000000..0b803f0 --- /dev/null +++ b/src/audio/stt/engine.ts @@ -0,0 +1,28 @@ +import type { TelegramSettings } from "../../utils/settings"; +import { resolveTelegramAudioInputSettings } from "../../utils/settings"; +import { WhisperCppSttEngine, type WhisperCppSttEngineConfig, type WhisperCppTranscriptionResult } from "./whisper-cpp"; + +export interface AudioTranscriptionInput { + audioPath: string; +} + +export type AudioTranscriptionResult = WhisperCppTranscriptionResult; + +export interface AudioTranscriptionEngine { + transcribe(input: AudioTranscriptionInput): Promise; +} + +export function createTelegramAudioInputEngine( + telegramSettings: TelegramSettings | undefined, +): AudioTranscriptionEngine { + const resolved = resolveTelegramAudioInputSettings(telegramSettings); + const config: WhisperCppSttEngineConfig = { + binaryPath: resolved.binaryPath, + model: resolved.model, + modelPath: resolved.modelPath, + autoDownloadModel: resolved.autoDownloadModel, + language: resolved.language, + }; + + return new WhisperCppSttEngine(config); +} diff --git a/src/audio/stt/whisper-cpp.test.ts b/src/audio/stt/whisper-cpp.test.ts new file mode 100644 index 0000000..cb62e94 --- /dev/null +++ b/src/audio/stt/whisper-cpp.test.ts @@ -0,0 +1,86 @@ +import fs from "fs"; +import os from "os"; +import path from "path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { + getWhisperCppModelCacheDir, + getWhisperCppModelCachePath, + getWhisperCppModelDownloadUrl, + getWhisperCppModelFileName, + normalizeWhisperTranscript, + resolveWhisperCppModelPath, +} from "./whisper-cpp"; + +describe("whisper.cpp helpers", () => { + let tempDir: string; + const originalHome = process.env.HOME; + const originalFetch = global.fetch; + + beforeEach(() => { + tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "grok-whisper-test-")); + }); + + afterEach(() => { + fs.rmSync(tempDir, { recursive: true, force: true }); + process.env.HOME = originalHome; + global.fetch = originalFetch; + vi.restoreAllMocks(); + }); + + it("builds stable cache paths for whisper.cpp models", () => { + expect(getWhisperCppModelCacheDir(tempDir)).toBe(path.join(tempDir, ".grok", "models", "stt", "whisper.cpp")); + expect(getWhisperCppModelFileName("base.en")).toBe("ggml-base.en.bin"); + expect(getWhisperCppModelCachePath("base.en", tempDir)).toBe( + path.join(tempDir, ".grok", "models", "stt", "whisper.cpp", "ggml-base.en.bin"), + ); + expect(getWhisperCppModelDownloadUrl("base.en")).toBe( + "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base.en.bin", + ); + }); + + it("normalizes multi-line whisper transcripts without blank lines", () => { + expect(normalizeWhisperTranscript(" hello world \n\n second line \n")).toBe("hello world\nsecond line"); + }); + + it("uses an explicit model path when provided", async () => { + const modelPath = path.join(tempDir, "custom-model.bin"); + fs.writeFileSync(modelPath, "model"); + + await expect( + resolveWhisperCppModelPath({ + modelPath, + autoDownloadModel: false, + }), + ).resolves.toBe(modelPath); + }); + + it("fails clearly when auto download is disabled and the model is missing", async () => { + await expect( + resolveWhisperCppModelPath({ + model: "tiny.en", + autoDownloadModel: false, + }), + ).rejects.toThrow("Whisper.cpp model ggml-tiny.en.bin is missing."); + }); + + it("downloads a missing model into the cache directory and cleans up temp files", async () => { + process.env.HOME = tempDir; + const fetchMock = vi.fn().mockResolvedValue( + new Response("model-bytes", { + status: 200, + headers: { "Content-Type": "application/octet-stream" }, + }), + ); + global.fetch = fetchMock as unknown as typeof fetch; + + const modelPath = await resolveWhisperCppModelPath({ + model: "tiny.en", + }); + + expect(modelPath).toBe(path.join(tempDir, ".grok", "models", "stt", "whisper.cpp", "ggml-tiny.en.bin")); + expect(fs.readFileSync(modelPath, "utf8")).toBe("model-bytes"); + expect(fs.readdirSync(path.dirname(modelPath)).filter((entry) => entry.startsWith(".grok-whisper-model-"))).toEqual( + [], + ); + }); +}); diff --git a/src/audio/stt/whisper-cpp.ts b/src/audio/stt/whisper-cpp.ts new file mode 100644 index 0000000..5d029ad --- /dev/null +++ b/src/audio/stt/whisper-cpp.ts @@ -0,0 +1,270 @@ +import { spawn } from "child_process"; +import { createWriteStream, existsSync } from "fs"; +import { access, mkdir, mkdtemp, readFile, rename, rm } from "fs/promises"; +import os from "os"; +import path from "path"; +import { Readable } from "stream"; +import { pipeline } from "stream/promises"; + +export const DEFAULT_WHISPER_CPP_BINARY = "whisper-cli"; +export const DEFAULT_WHISPER_CPP_MODEL = "tiny.en"; +const DEFAULT_WHISPER_LANGUAGE = "en"; +const MODEL_ID_PATTERN = /^[a-z0-9][a-z0-9._-]*$/i; + +export interface WhisperCppSttEngineConfig { + binaryPath?: string; + model?: string; + modelPath?: string; + autoDownloadModel?: boolean; + language?: string; +} + +export interface WhisperCppTranscriptionResult { + text: string; + engine: "whisper.cpp"; + model: string; + modelPath: string; +} + +export class WhisperCppSttEngine { + constructor(private readonly config: WhisperCppSttEngineConfig) {} + + async transcribe(input: { audioPath: string }): Promise { + const binaryPath = this.config.binaryPath?.trim() || DEFAULT_WHISPER_CPP_BINARY; + const model = normalizeWhisperCppModel(this.config.model); + const modelPath = await resolveWhisperCppModelPath({ + model, + modelPath: this.config.modelPath, + autoDownloadModel: this.config.autoDownloadModel !== false, + }); + const language = this.config.language?.trim() || DEFAULT_WHISPER_LANGUAGE; + const workDir = await mkdtemp(path.join(os.tmpdir(), "grok-whisper-")); + + try { + const text = await transcribeWithOptionalFfmpegFallback({ + binaryPath, + modelPath, + language, + audioPath: input.audioPath, + workDir, + }); + + return { + text, + engine: "whisper.cpp", + model, + modelPath, + }; + } finally { + await rm(workDir, { recursive: true, force: true }); + } + } +} + +export function getWhisperCppModelCacheDir(homeDir = os.homedir()): string { + return path.join(homeDir, ".grok", "models", "stt", "whisper.cpp"); +} + +export function getWhisperCppModelFileName(model = DEFAULT_WHISPER_CPP_MODEL): string { + return `ggml-${normalizeWhisperCppModel(model)}.bin`; +} + +export function getWhisperCppModelCachePath(model = DEFAULT_WHISPER_CPP_MODEL, homeDir = os.homedir()): string { + return path.join(getWhisperCppModelCacheDir(homeDir), getWhisperCppModelFileName(model)); +} + +export function getWhisperCppModelDownloadUrl(model = DEFAULT_WHISPER_CPP_MODEL): string { + return `https://huggingface.co/ggerganov/whisper.cpp/resolve/main/${getWhisperCppModelFileName(model)}`; +} + +export function normalizeWhisperTranscript(text: string): string { + return text + .split(/\r?\n/) + .map((line) => line.trim()) + .filter(Boolean) + .join("\n") + .trim(); +} + +export async function resolveWhisperCppModelPath(opts: { + model?: string; + modelPath?: string; + autoDownloadModel?: boolean; +}): Promise { + const model = normalizeWhisperCppModel(opts.model); + const overridePath = opts.modelPath?.trim(); + if (overridePath) { + await ensureFileReadable(overridePath, `Configured whisper.cpp model not found: ${overridePath}`); + return overridePath; + } + + const cachePath = getWhisperCppModelCachePath(model); + if (existsSync(cachePath)) { + return cachePath; + } + + if (opts.autoDownloadModel === false) { + throw new Error( + `Whisper.cpp model ${getWhisperCppModelFileName(model)} is missing. ` + + `Set telegram.audioInput.modelPath or enable autoDownloadModel.`, + ); + } + + await downloadWhisperCppModel(model, cachePath); + return cachePath; +} + +async function transcribeWithOptionalFfmpegFallback(opts: { + binaryPath: string; + modelPath: string; + language: string; + audioPath: string; + workDir: string; +}): Promise { + const directOutputBase = path.join(opts.workDir, "transcript"); + + try { + return await runWhisperCpp({ + binaryPath: opts.binaryPath, + modelPath: opts.modelPath, + language: opts.language, + audioPath: opts.audioPath, + outputBase: directOutputBase, + }); + } catch (directError: unknown) { + const convertedPath = await maybeConvertAudioWithFfmpeg(opts.audioPath, opts.workDir); + if (!convertedPath) { + throw directError; + } + + try { + return await runWhisperCpp({ + binaryPath: opts.binaryPath, + modelPath: opts.modelPath, + language: opts.language, + audioPath: convertedPath, + outputBase: path.join(opts.workDir, "transcript-converted"), + }); + } catch (convertedError: unknown) { + const directMessage = directError instanceof Error ? directError.message : String(directError); + const convertedMessage = convertedError instanceof Error ? convertedError.message : String(convertedError); + throw new Error(`${directMessage}\nffmpeg fallback failed: ${convertedMessage}`); + } + } +} + +async function runWhisperCpp(opts: { + binaryPath: string; + modelPath: string; + language: string; + audioPath: string; + outputBase: string; +}): Promise { + const args = [ + "--model", + opts.modelPath, + "--file", + opts.audioPath, + "--language", + opts.language, + "--output-txt", + "--output-file", + opts.outputBase, + ]; + + await runCommand(opts.binaryPath, args); + const transcriptPath = `${opts.outputBase}.txt`; + const transcript = normalizeWhisperTranscript(await readFile(transcriptPath, "utf8")); + + if (!transcript) { + throw new Error("whisper.cpp returned an empty transcript."); + } + + return transcript; +} + +async function maybeConvertAudioWithFfmpeg(audioPath: string, workDir: string): Promise { + if (path.extname(audioPath).toLowerCase() === ".wav") { + return null; + } + + const convertedPath = path.join(workDir, "input.wav"); + try { + await runCommand("ffmpeg", ["-y", "-i", audioPath, "-ar", "16000", "-ac", "1", "-c:a", "pcm_s16le", convertedPath]); + return convertedPath; + } catch { + return null; + } +} + +async function downloadWhisperCppModel(model: string, destinationPath: string): Promise { + await mkdir(path.dirname(destinationPath), { recursive: true, mode: 0o700 }); + const response = await fetch(getWhisperCppModelDownloadUrl(model)); + if (!response.ok || !response.body) { + throw new Error(`Failed to download whisper.cpp model ${model}: ${response.status} ${response.statusText}`); + } + + // Keep the temp file on the target filesystem so the final rename stays atomic. + const tempDir = await mkdtemp(path.join(path.dirname(destinationPath), ".grok-whisper-model-")); + const tempPath = path.join(tempDir, getWhisperCppModelFileName(model)); + + try { + const stream = createWriteStream(tempPath, { mode: 0o600 }); + await pipeline(Readable.fromWeb(response.body as globalThis.ReadableStream), stream); + await rename(tempPath, destinationPath); + } finally { + await rm(tempDir, { recursive: true, force: true }); + } +} + +async function ensureFileReadable(filePath: string, message: string): Promise { + try { + await access(filePath); + } catch { + throw new Error(message); + } +} + +function normalizeWhisperCppModel(model = DEFAULT_WHISPER_CPP_MODEL): string { + const normalized = model.trim() || DEFAULT_WHISPER_CPP_MODEL; + if (!MODEL_ID_PATTERN.test(normalized)) { + throw new Error(`Invalid whisper.cpp model name: ${model}`); + } + return normalized; +} + +function runCommand(command: string, args: string[]): Promise { + return new Promise((resolve, reject) => { + const child = spawn(command, args, { + stdio: ["ignore", "pipe", "pipe"], + }); + + let stdout = ""; + let stderr = ""; + + child.stdout?.on("data", (chunk: Buffer | string) => { + stdout += chunk.toString(); + }); + child.stderr?.on("data", (chunk: Buffer | string) => { + stderr += chunk.toString(); + }); + + child.on("error", (error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") { + reject(new Error(`Command not found: ${command}`)); + return; + } + reject(error); + }); + + child.on("close", (code) => { + if (code === 0) { + resolve(); + return; + } + + const details = stderr.trim() || stdout.trim() || `exit code ${code ?? "unknown"}`; + reject(new Error(`${command} failed: ${details}`)); + }); + }); +} diff --git a/src/telegram/audio-input.test.ts b/src/telegram/audio-input.test.ts new file mode 100644 index 0000000..f6c80a3 --- /dev/null +++ b/src/telegram/audio-input.test.ts @@ -0,0 +1,41 @@ +import { describe, expect, it } from "vitest"; +import { formatTelegramAudioTranscript, getTelegramAudioSource } from "./audio-input"; + +describe("telegram audio input helpers", () => { + it("extracts a voice source from a telegram message payload", () => { + expect( + getTelegramAudioSource({ + voice: { + file_id: "voice-file", + mime_type: "audio/ogg", + }, + }), + ).toEqual({ + kind: "voice", + fileId: "voice-file", + mimeType: "audio/ogg", + }); + }); + + it("extracts an audio attachment source from a telegram message payload", () => { + expect( + getTelegramAudioSource({ + audio: { + file_id: "audio-file", + file_name: "song.mp3", + mime_type: "audio/mpeg", + }, + }), + ).toEqual({ + kind: "audio", + fileId: "audio-file", + fileName: "song.mp3", + mimeType: "audio/mpeg", + }); + }); + + it("formats user-visible transcript prefixes for voice and audio messages", () => { + expect(formatTelegramAudioTranscript("voice", "hello there")).toBe("[Voice transcript] hello there"); + expect(formatTelegramAudioTranscript("audio", "demo")).toBe("[Audio transcript] demo"); + }); +}); diff --git a/src/telegram/audio-input.ts b/src/telegram/audio-input.ts new file mode 100644 index 0000000..6ee41ef --- /dev/null +++ b/src/telegram/audio-input.ts @@ -0,0 +1,116 @@ +import { mkdtemp, rm, writeFile } from "fs/promises"; +import os from "os"; +import path from "path"; +import { createTelegramAudioInputEngine } from "../audio/stt/engine"; +import type { TelegramSettings } from "../utils/settings"; +import { resolveTelegramAudioInputSettings } from "../utils/settings"; + +export interface TelegramAudioSource { + kind: "voice" | "audio"; + fileId: string; + fileName?: string; + mimeType?: string; +} + +export interface TelegramFileApi { + getFile(fileId: string): Promise<{ file_path?: string }>; +} + +export interface TelegramAudioTranscription { + promptText: string; + userContent: string; +} + +export function getTelegramAudioSource(message: { + voice?: { file_id: string; mime_type?: string }; + audio?: { file_id: string; file_name?: string; mime_type?: string }; +}): TelegramAudioSource | null { + if (message.voice) { + return { + kind: "voice", + fileId: message.voice.file_id, + mimeType: message.voice.mime_type, + }; + } + + if (message.audio) { + return { + kind: "audio", + fileId: message.audio.file_id, + fileName: message.audio.file_name, + mimeType: message.audio.mime_type, + }; + } + + return null; +} + +export function formatTelegramAudioTranscript(kind: TelegramAudioSource["kind"], transcript: string): string { + const label = kind === "voice" ? "Voice" : "Audio"; + return `[${label} transcript] ${transcript}`; +} + +export async function transcribeTelegramAudioMessage(opts: { + api: TelegramFileApi; + token: string; + source: TelegramAudioSource; + telegramSettings: TelegramSettings | undefined; +}): Promise { + const audioSettings = resolveTelegramAudioInputSettings(opts.telegramSettings); + if (!audioSettings.enabled) { + throw new Error("Telegram audio input is disabled in settings."); + } + + const tempDir = await mkdtemp(path.join(os.tmpdir(), "grok-telegram-audio-")); + try { + const file = await opts.api.getFile(opts.source.fileId); + if (!file.file_path) { + throw new Error("Telegram did not return a downloadable file path for this audio message."); + } + + const response = await fetch(`https://api.telegram.org/file/bot${opts.token}/${file.file_path}`); + if (!response.ok) { + throw new Error(`Telegram audio download failed: ${response.status} ${response.statusText}`); + } + + const extension = inferTelegramAudioExtension(opts.source, file.file_path); + const audioPath = path.join(tempDir, `input${extension}`); + const bytes = Buffer.from(await response.arrayBuffer()); + await writeFile(audioPath, bytes, { mode: 0o600 }); + + const engine = createTelegramAudioInputEngine(opts.telegramSettings); + const result = await engine.transcribe({ audioPath }); + + return { + promptText: result.text, + userContent: formatTelegramAudioTranscript(opts.source.kind, result.text), + }; + } finally { + await rm(tempDir, { recursive: true, force: true }); + } +} + +function inferTelegramAudioExtension(source: TelegramAudioSource, filePath?: string): string { + const fromPath = filePath ? path.extname(filePath) : ""; + if (fromPath) return fromPath; + + const fromName = source.fileName ? path.extname(source.fileName) : ""; + if (fromName) return fromName; + + switch ((source.mimeType || "").toLowerCase()) { + case "audio/ogg": + return ".ogg"; + case "audio/opus": + return ".opus"; + case "audio/mpeg": + return ".mp3"; + case "audio/mp4": + case "audio/x-m4a": + return ".m4a"; + case "audio/wav": + case "audio/x-wav": + return ".wav"; + default: + return source.kind === "voice" ? ".ogg" : ".bin"; + } +} diff --git a/src/telegram/bridge.ts b/src/telegram/bridge.ts index d995573..87879b9 100644 --- a/src/telegram/bridge.ts +++ b/src/telegram/bridge.ts @@ -2,6 +2,7 @@ import { Bot } from "grammy"; import type { Agent } from "../agent/agent"; import type { ToolCall, ToolResult } from "../types/index"; import { loadUserSettings, resolveTelegramStreamSettings } from "../utils/settings"; +import { getTelegramAudioSource, transcribeTelegramAudioMessage } from "./audio-input"; import { splitTelegramMessage, TELEGRAM_MAX_MESSAGE } from "./limits"; import { registerPairingCode } from "./pairing"; import { runTelegramPartialReply } from "./preview-stream"; @@ -32,34 +33,56 @@ export function createTelegramBridge(opts: TelegramBridgeOptions): TelegramBridg const bot = new Bot(opts.token); let running = false; - bot.command("start", async (ctx) => { - await ctx.reply("Send /pair to link this chat to Grok CLI, then approve the code in the terminal."); - }); + const buildTurnKey = (ctx: { chat: { id: number }; message: { message_id: number } }) => + `telegram:${ctx.chat.id}:${ctx.message.message_id}`; - bot.command("pair", async (ctx) => { + const ensureApprovedUser = async (ctx: { from?: { id?: number }; reply: (text: string) => Promise }) => { const userId = ctx.from?.id; - if (userId === undefined) return; - const code = registerPairingCode(userId); - await ctx.reply(`Your pairing code: ${code}\nEnter this code in Grok CLI (/remote-control → Telegram) to approve.`); - }); - - bot.on("message:text", async (ctx) => { - const text = ctx.message.text; - if (text.startsWith("/")) return; - - const userId = ctx.from?.id; - if (userId === undefined) return; + if (userId === undefined) return null; const approved = opts.getApprovedUserIds(); if (!approved.includes(userId)) { await ctx.reply("Not paired yet. Send /pair to get a code, then approve in Grok CLI."); - return; + return null; } + return userId; + }; + + const replyTurnError = async ( + ctx: { reply: (text: string) => Promise; chat: { id: number }; message: { message_id: number } }, + userId: number, + message: string, + ) => { + const clipped = message.slice(0, TELEGRAM_MAX_MESSAGE); + opts.onAssistantMessage?.({ + turnKey: buildTurnKey(ctx), + userId, + content: `Error: ${clipped}`, + done: true, + }); + try { + await ctx.reply(`Error: ${clipped}`); + } catch { + /* user blocked bot or chat forbids messages */ + } + }; + + const runAgentTurn = async ( + ctx: { + chat: { id: number }; + from?: { id?: number }; + message: { message_id: number; message_thread_id?: number }; + reply: (text: string) => Promise; + }, + userId: number, + userContent: string, + promptText: string, + ) => { await opts.coordinator.run(async () => { try { - const turnKey = `telegram:${ctx.chat.id}:${ctx.message.message_id}`; - opts.onUserMessage?.({ turnKey, userId, content: text }); + const turnKey = buildTurnKey(ctx); + opts.onUserMessage?.({ turnKey, userId, content: userContent }); const agent = opts.getTelegramAgent(userId); const stream = resolveTelegramStreamSettings(loadUserSettings().telegram); @@ -72,7 +95,7 @@ export function createTelegramBridge(opts: TelegramBridgeOptions): TelegramBridg ); let acc = ""; try { - for await (const chunk of agent.processMessage(text)) { + for await (const chunk of agent.processMessage(promptText)) { switch (chunk.type) { case "content": if (chunk.content) { @@ -99,13 +122,7 @@ export function createTelegramBridge(opts: TelegramBridgeOptions): TelegramBridg } } catch (err: unknown) { const msg = err instanceof Error ? err.message : String(err); - opts.onAssistantMessage?.({ - turnKey, - userId, - content: `Error: ${msg.slice(0, TELEGRAM_MAX_MESSAGE)}`, - done: true, - }); - await ctx.reply(`Error: ${msg.slice(0, TELEGRAM_MAX_MESSAGE)}`); + await replyTurnError(ctx, userId, msg); return; } finally { stopTyping(); @@ -123,7 +140,7 @@ export function createTelegramBridge(opts: TelegramBridgeOptions): TelegramBridg chatId: ctx.chat.id, messageThreadId: ctx.message.message_thread_id, typingIndicator: stream.typingIndicator, - stream: agent.processMessage(text), + stream: agent.processMessage(promptText), onAssistantMessage: (event) => { opts.onAssistantMessage?.({ turnKey, @@ -146,19 +163,70 @@ export function createTelegramBridge(opts: TelegramBridgeOptions): TelegramBridg }); } catch (err: unknown) { const msg = err instanceof Error ? err.message : String(err); - opts.onAssistantMessage?.({ - turnKey: `telegram:${ctx.chat.id}:${ctx.message.message_id}`, - userId, - content: `Error: ${msg.slice(0, TELEGRAM_MAX_MESSAGE)}`, - done: true, - }); - try { - await ctx.reply(`Error: ${msg.slice(0, TELEGRAM_MAX_MESSAGE)}`); - } catch { - /* user blocked bot or chat forbids messages */ - } + await replyTurnError(ctx, userId, msg); } }); + }; + + const handleAudioMessage = async (ctx: { + chat: { id: number }; + from?: { id?: number }; + message: { + message_id: number; + message_thread_id?: number; + voice?: { file_id: string; mime_type?: string }; + audio?: { file_id: string; file_name?: string; mime_type?: string }; + }; + reply: (text: string) => Promise; + }) => { + const userId = await ensureApprovedUser(ctx); + if (userId === null) return; + + const source = getTelegramAudioSource(ctx.message); + if (!source) return; + + try { + await bot.api.sendChatAction(ctx.chat.id, "typing"); + const transcription = await transcribeTelegramAudioMessage({ + api: bot.api, + token: opts.token, + source, + telegramSettings: loadUserSettings().telegram, + }); + await runAgentTurn(ctx, userId, transcription.userContent, transcription.promptText); + } catch (err: unknown) { + const msg = err instanceof Error ? err.message : String(err); + await replyTurnError(ctx, userId, `Audio transcription failed: ${msg}`); + } + }; + + bot.command("start", async (ctx) => { + await ctx.reply("Send /pair to link this chat to Grok CLI, then approve the code in the terminal."); + }); + + bot.command("pair", async (ctx) => { + const userId = ctx.from?.id; + if (userId === undefined) return; + const code = registerPairingCode(userId); + await ctx.reply(`Your pairing code: ${code}\nEnter this code in Grok CLI (/remote-control → Telegram) to approve.`); + }); + + bot.on("message:text", async (ctx) => { + const text = ctx.message.text; + if (text.startsWith("/")) return; + + const userId = await ensureApprovedUser(ctx); + if (userId === null) return; + + await runAgentTurn(ctx, userId, text, text); + }); + + bot.on("message:voice", async (ctx) => { + await handleAudioMessage(ctx); + }); + + bot.on("message:audio", async (ctx) => { + await handleAudioMessage(ctx); }); bot.catch((err) => { diff --git a/src/utils/settings.ts b/src/utils/settings.ts index 6de5d4b..07b0ca5 100644 --- a/src/utils/settings.ts +++ b/src/utils/settings.ts @@ -5,6 +5,27 @@ import { DEFAULT_MODEL, getEffectiveReasoningEffort, getModelIds, normalizeModel import type { ReasoningEffort } from "../types/index"; export type TelegramStreamingMode = "off" | "partial"; +export type TelegramAudioInputEngine = "whisper.cpp"; + +export const DEFAULT_TELEGRAM_AUDIO_INPUT_BINARY = "whisper-cli"; +export const DEFAULT_TELEGRAM_AUDIO_INPUT_MODEL = "tiny.en"; + +export interface TelegramAudioInputSettings { + /** Enable Telegram voice/audio transcription before sending text to the agent. Default: true. */ + enabled?: boolean; + /** Reserved for future providers. The current built-in engine is whisper.cpp. */ + engine?: TelegramAudioInputEngine; + /** Path or command name for the whisper.cpp CLI. Default: whisper-cli on PATH. */ + binaryPath?: string; + /** Whisper.cpp model alias used for cache/download resolution. Default: tiny.en. */ + model?: string; + /** Optional override for an exact ggml model path on disk. */ + modelPath?: string; + /** Automatically download missing models into ~/.grok/models/stt. Default: true. */ + autoDownloadModel?: boolean; + /** Whisper language code passed to the CLI. Default: en. */ + language?: string; +} export interface TelegramSettings { botToken?: string; @@ -16,6 +37,7 @@ export interface TelegramSettings { typingIndicator?: boolean; /** Reserved: Bot API `sendMessageDraft` for private DMs (not implemented yet). */ nativeDrafts?: boolean; + audioInput?: TelegramAudioInputSettings; } export type McpRemoteTransport = "http" | "sse"; @@ -144,6 +166,10 @@ export function saveUserSettings(partial: Partial): void { telegram: { ...current.telegram, ...partial.telegram, + audioInput: { + ...current.telegram?.audioInput, + ...partial.telegram?.audioInput, + }, sessionsByUserId: { ...current.telegram?.sessionsByUserId, ...partial.telegram?.sessionsByUserId, @@ -231,6 +257,26 @@ export function resolveTelegramStreamSettings(t: TelegramSettings | undefined): }; } +export function resolveTelegramAudioInputSettings(t: TelegramSettings | undefined): { + enabled: boolean; + engine: TelegramAudioInputEngine; + binaryPath: string; + model: string; + modelPath: string | undefined; + autoDownloadModel: boolean; + language: string; +} { + return { + enabled: t?.audioInput?.enabled !== false, + engine: "whisper.cpp", + binaryPath: t?.audioInput?.binaryPath?.trim() || DEFAULT_TELEGRAM_AUDIO_INPUT_BINARY, + model: t?.audioInput?.model?.trim() || DEFAULT_TELEGRAM_AUDIO_INPUT_MODEL, + modelPath: t?.audioInput?.modelPath?.trim() || undefined, + autoDownloadModel: t?.audioInput?.autoDownloadModel !== false, + language: t?.audioInput?.language?.trim() || "en", + }; +} + export function loadMcpServers(): McpServerConfig[] { return loadUserSettings().mcp?.servers ?? []; } diff --git a/src/utils/telegram-audio-settings.test.ts b/src/utils/telegram-audio-settings.test.ts new file mode 100644 index 0000000..4851953 --- /dev/null +++ b/src/utils/telegram-audio-settings.test.ts @@ -0,0 +1,43 @@ +import { describe, expect, it } from "vitest"; +import { + DEFAULT_TELEGRAM_AUDIO_INPUT_BINARY, + DEFAULT_TELEGRAM_AUDIO_INPUT_MODEL, + resolveTelegramAudioInputSettings, +} from "./settings"; + +describe("resolveTelegramAudioInputSettings", () => { + it("returns whisper.cpp defaults when audio input is unset", () => { + expect(resolveTelegramAudioInputSettings(undefined)).toEqual({ + enabled: true, + engine: "whisper.cpp", + binaryPath: DEFAULT_TELEGRAM_AUDIO_INPUT_BINARY, + model: DEFAULT_TELEGRAM_AUDIO_INPUT_MODEL, + modelPath: undefined, + autoDownloadModel: true, + language: "en", + }); + }); + + it("preserves explicit telegram audio overrides", () => { + expect( + resolveTelegramAudioInputSettings({ + audioInput: { + enabled: false, + binaryPath: "/opt/whisper-cli", + model: "base.en", + modelPath: "/tmp/ggml-base.en.bin", + autoDownloadModel: false, + language: "fr", + }, + }), + ).toEqual({ + enabled: false, + engine: "whisper.cpp", + binaryPath: "/opt/whisper-cli", + model: "base.en", + modelPath: "/tmp/ggml-base.en.bin", + autoDownloadModel: false, + language: "fr", + }); + }); +});