feat: add Telegram voice/audio transcription via whisper.cpp (#210)

* feat: add Telegram voice/audio transcription via whisper.cpp

Telegram voice notes and audio attachments are now transcribed locally
using whisper.cpp before being sent to the agent. Includes auto-download
of Whisper models, ffmpeg fallback for non-WAV formats, configurable
settings under telegram.audioInput, and unit tests for the new STT layer.

* fix: harden whisper audio setup

Avoid Linux model download failures, remove unused STT dependencies, and keep local Cursor state changes isolated to the intended files.

* refactor: share Telegram audio handler

Reduce duplicate Telegram voice and audio handling so transcription flow changes stay in sync.

.cursor/hooks/state/continual-learning.json

@@ -0,0 +1,8 @@
+{
+  "version": 1,
+  "lastRunAtMs": 0,
+  "turnsSinceLastRun": 4,
+  "lastTranscriptMtimeMs": null,
+  "lastProcessedGenerationId": "518874b0-1435-472e-9c42-c54214a9ae4e",
+  "trialStartedAtMs": null
+}

.gitignore

@@ -103,5 +103,6 @@ temp/
 
 # Cursor config (tracked — do not ignore)
 # .cursor/
+.cursor/hooks/state/
 .agents
 skills-lock.json

README.md

@@ -183,6 +183,71 @@ Optional: **`GROK_BASE_URL`** (default `https://api.x.ai/v1`), **`GROK_MODEL`**,
 3. Start **`grok`**, open **`/remote-control`** → **Telegram** if needed, then in Telegram DM your bot: **`/pair`**, enter the **6-character code** in the terminal when asked.
 4. First user must be approved once; after that, it’s remembered. **Keep the CLI process running** while you use the bot (long polling lives in that process).
 
+### Voice & audio messages
+
+Send a voice note or audio attachment in Telegram and Grok will transcribe it locally with **[whisper.cpp](https://github.com/ggml-org/whisper.cpp)** before passing the text to the agent. No cloud STT service is involved — everything runs on your machine.
+
+#### Prerequisites
+
+| Dependency | Why | Install (macOS) |
+|---|---|---|
+| **whisper-cli** | Runs the actual speech-to-text inference | `brew install whisper-cpp` |
+| **ffmpeg** | Converts Telegram voice notes (OGG/Opus) to WAV for whisper.cpp | `brew install ffmpeg` |
+
+After installing, verify both are available:
+
+```bash
+whisper-cli -h
+ffmpeg -version
+```
+
+#### Download a Whisper model
+
+Grok CLI auto-downloads the configured model on first use, but you can pre-download it:
+
+```bash
+mkdir -p ~/.grok/models/stt/whisper.cpp
+curl -L https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-tiny.en.bin \
+  -o ~/.grok/models/stt/whisper.cpp/ggml-tiny.en.bin
+```
+
+Available models (trade size for accuracy): `tiny.en` (75 MB), `base.en` (142 MB), `small.en` (466 MB).
+
+#### Configure in `~/.grok/user-settings.json`
+
+```json
+{
+  "telegram": {
+    "botToken": "YOUR_BOT_TOKEN",
+    "audioInput": {
+      "enabled": true,
+      "binaryPath": "/opt/homebrew/bin/whisper-cli",
+      "model": "tiny.en",
+      "modelPath": "~/.grok/models/stt/whisper.cpp/ggml-tiny.en.bin",
+      "autoDownloadModel": true,
+      "language": "en"
+    }
+  }
+}
+```
+
+| Setting | Default | Description |
+|---|---|---|
+| `enabled` | `true` | Set to `false` to ignore voice/audio messages entirely. |
+| `binaryPath` | `whisper-cli` | Absolute path or command name for the whisper.cpp CLI binary. |
+| `model` | `tiny.en` | Model alias used for auto-download resolution. |
+| `modelPath` | _(auto-resolved)_ | Explicit path to a `.bin` model file. Overrides `model` + auto-download. |
+| `autoDownloadModel` | `true` | Download the model into `~/.grok/models/stt/whisper.cpp` on first use. |
+| `language` | `en` | Whisper language code passed to the CLI. |
+
+#### Test it
+
+1. Restart the CLI after changing settings: `bun run build && node dist/index.js`
+2. Send a **3–5 second voice note** with a clear spoken sentence (e.g. "list the files in the current directory").
+3. The bot should reply as if you had typed that sentence.
+
+Very short or silent voice notes may produce an empty transcript — the agent will tell you the audio was blank.
+
 Treat the bot token like a password.
 
 ---

src/audio/stt/engine.ts

@@ -0,0 +1,28 @@
+import type { TelegramSettings } from "../../utils/settings";
+import { resolveTelegramAudioInputSettings } from "../../utils/settings";
+import { WhisperCppSttEngine, type WhisperCppSttEngineConfig, type WhisperCppTranscriptionResult } from "./whisper-cpp";
+
+export interface AudioTranscriptionInput {
+  audioPath: string;
+}
+
+export type AudioTranscriptionResult = WhisperCppTranscriptionResult;
+
+export interface AudioTranscriptionEngine {
+  transcribe(input: AudioTranscriptionInput): Promise<AudioTranscriptionResult>;
+}
+
+export function createTelegramAudioInputEngine(
+  telegramSettings: TelegramSettings | undefined,
+): AudioTranscriptionEngine {
+  const resolved = resolveTelegramAudioInputSettings(telegramSettings);
+  const config: WhisperCppSttEngineConfig = {
+    binaryPath: resolved.binaryPath,
+    model: resolved.model,
+    modelPath: resolved.modelPath,
+    autoDownloadModel: resolved.autoDownloadModel,
+    language: resolved.language,
+  };
+
+  return new WhisperCppSttEngine(config);
+}

src/audio/stt/whisper-cpp.test.ts

@@ -0,0 +1,86 @@
+import fs from "fs";
+import os from "os";
+import path from "path";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import {
+  getWhisperCppModelCacheDir,
+  getWhisperCppModelCachePath,
+  getWhisperCppModelDownloadUrl,
+  getWhisperCppModelFileName,
+  normalizeWhisperTranscript,
+  resolveWhisperCppModelPath,
+} from "./whisper-cpp";
+
+describe("whisper.cpp helpers", () => {
+  let tempDir: string;
+  const originalHome = process.env.HOME;
+  const originalFetch = global.fetch;
+
+  beforeEach(() => {
+    tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "grok-whisper-test-"));
+  });
+
+  afterEach(() => {
+    fs.rmSync(tempDir, { recursive: true, force: true });
+    process.env.HOME = originalHome;
+    global.fetch = originalFetch;
+    vi.restoreAllMocks();
+  });
+
+  it("builds stable cache paths for whisper.cpp models", () => {
+    expect(getWhisperCppModelCacheDir(tempDir)).toBe(path.join(tempDir, ".grok", "models", "stt", "whisper.cpp"));
+    expect(getWhisperCppModelFileName("base.en")).toBe("ggml-base.en.bin");
+    expect(getWhisperCppModelCachePath("base.en", tempDir)).toBe(
+      path.join(tempDir, ".grok", "models", "stt", "whisper.cpp", "ggml-base.en.bin"),
+    );
+    expect(getWhisperCppModelDownloadUrl("base.en")).toBe(
+      "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base.en.bin",
+    );
+  });
+
+  it("normalizes multi-line whisper transcripts without blank lines", () => {
+    expect(normalizeWhisperTranscript("  hello world \n\n second line  \n")).toBe("hello world\nsecond line");
+  });
+
+  it("uses an explicit model path when provided", async () => {
+    const modelPath = path.join(tempDir, "custom-model.bin");
+    fs.writeFileSync(modelPath, "model");
+
+    await expect(
+      resolveWhisperCppModelPath({
+        modelPath,
+        autoDownloadModel: false,
+      }),
+    ).resolves.toBe(modelPath);
+  });
+
+  it("fails clearly when auto download is disabled and the model is missing", async () => {
+    await expect(
+      resolveWhisperCppModelPath({
+        model: "tiny.en",
+        autoDownloadModel: false,
+      }),
+    ).rejects.toThrow("Whisper.cpp model ggml-tiny.en.bin is missing.");
+  });
+
+  it("downloads a missing model into the cache directory and cleans up temp files", async () => {
+    process.env.HOME = tempDir;
+    const fetchMock = vi.fn().mockResolvedValue(
+      new Response("model-bytes", {
+        status: 200,
+        headers: { "Content-Type": "application/octet-stream" },
+      }),
+    );
+    global.fetch = fetchMock as unknown as typeof fetch;
+
+    const modelPath = await resolveWhisperCppModelPath({
+      model: "tiny.en",
+    });
+
+    expect(modelPath).toBe(path.join(tempDir, ".grok", "models", "stt", "whisper.cpp", "ggml-tiny.en.bin"));
+    expect(fs.readFileSync(modelPath, "utf8")).toBe("model-bytes");
+    expect(fs.readdirSync(path.dirname(modelPath)).filter((entry) => entry.startsWith(".grok-whisper-model-"))).toEqual(
+      [],
+    );
+  });
+});

src/audio/stt/whisper-cpp.ts

@@ -0,0 +1,270 @@
+import { spawn } from "child_process";
+import { createWriteStream, existsSync } from "fs";
+import { access, mkdir, mkdtemp, readFile, rename, rm } from "fs/promises";
+import os from "os";
+import path from "path";
+import { Readable } from "stream";
+import { pipeline } from "stream/promises";
+
+export const DEFAULT_WHISPER_CPP_BINARY = "whisper-cli";
+export const DEFAULT_WHISPER_CPP_MODEL = "tiny.en";
+const DEFAULT_WHISPER_LANGUAGE = "en";
+const MODEL_ID_PATTERN = /^[a-z0-9][a-z0-9._-]*$/i;
+
+export interface WhisperCppSttEngineConfig {
+  binaryPath?: string;
+  model?: string;
+  modelPath?: string;
+  autoDownloadModel?: boolean;
+  language?: string;
+}
+
+export interface WhisperCppTranscriptionResult {
+  text: string;
+  engine: "whisper.cpp";
+  model: string;
+  modelPath: string;
+}
+
+export class WhisperCppSttEngine {
+  constructor(private readonly config: WhisperCppSttEngineConfig) {}
+
+  async transcribe(input: { audioPath: string }): Promise<WhisperCppTranscriptionResult> {
+    const binaryPath = this.config.binaryPath?.trim() || DEFAULT_WHISPER_CPP_BINARY;
+    const model = normalizeWhisperCppModel(this.config.model);
+    const modelPath = await resolveWhisperCppModelPath({
+      model,
+      modelPath: this.config.modelPath,
+      autoDownloadModel: this.config.autoDownloadModel !== false,
+    });
+    const language = this.config.language?.trim() || DEFAULT_WHISPER_LANGUAGE;
+    const workDir = await mkdtemp(path.join(os.tmpdir(), "grok-whisper-"));
+
+    try {
+      const text = await transcribeWithOptionalFfmpegFallback({
+        binaryPath,
+        modelPath,
+        language,
+        audioPath: input.audioPath,
+        workDir,
+      });
+
+      return {
+        text,
+        engine: "whisper.cpp",
+        model,
+        modelPath,
+      };
+    } finally {
+      await rm(workDir, { recursive: true, force: true });
+    }
+  }
+}
+
+export function getWhisperCppModelCacheDir(homeDir = os.homedir()): string {
+  return path.join(homeDir, ".grok", "models", "stt", "whisper.cpp");
+}
+
+export function getWhisperCppModelFileName(model = DEFAULT_WHISPER_CPP_MODEL): string {
+  return `ggml-${normalizeWhisperCppModel(model)}.bin`;
+}
+
+export function getWhisperCppModelCachePath(model = DEFAULT_WHISPER_CPP_MODEL, homeDir = os.homedir()): string {
+  return path.join(getWhisperCppModelCacheDir(homeDir), getWhisperCppModelFileName(model));
+}
+
+export function getWhisperCppModelDownloadUrl(model = DEFAULT_WHISPER_CPP_MODEL): string {
+  return `https://huggingface.co/ggerganov/whisper.cpp/resolve/main/${getWhisperCppModelFileName(model)}`;
+}
+
+export function normalizeWhisperTranscript(text: string): string {
+  return text
+    .split(/\r?\n/)
+    .map((line) => line.trim())
+    .filter(Boolean)
+    .join("\n")
+    .trim();
+}
+
+export async function resolveWhisperCppModelPath(opts: {
+  model?: string;
+  modelPath?: string;
+  autoDownloadModel?: boolean;
+}): Promise<string> {
+  const model = normalizeWhisperCppModel(opts.model);
+  const overridePath = opts.modelPath?.trim();
+  if (overridePath) {
+    await ensureFileReadable(overridePath, `Configured whisper.cpp model not found: ${overridePath}`);
+    return overridePath;
+  }
+
+  const cachePath = getWhisperCppModelCachePath(model);
+  if (existsSync(cachePath)) {
+    return cachePath;
+  }
+
+  if (opts.autoDownloadModel === false) {
+    throw new Error(
+      `Whisper.cpp model ${getWhisperCppModelFileName(model)} is missing. ` +
+        `Set telegram.audioInput.modelPath or enable autoDownloadModel.`,
+    );
+  }
+
+  await downloadWhisperCppModel(model, cachePath);
+  return cachePath;
+}
+
+async function transcribeWithOptionalFfmpegFallback(opts: {
+  binaryPath: string;
+  modelPath: string;
+  language: string;
+  audioPath: string;
+  workDir: string;
+}): Promise<string> {
+  const directOutputBase = path.join(opts.workDir, "transcript");
+
+  try {
+    return await runWhisperCpp({
+      binaryPath: opts.binaryPath,
+      modelPath: opts.modelPath,
+      language: opts.language,
+      audioPath: opts.audioPath,
+      outputBase: directOutputBase,
+    });
+  } catch (directError: unknown) {
+    const convertedPath = await maybeConvertAudioWithFfmpeg(opts.audioPath, opts.workDir);
+    if (!convertedPath) {
+      throw directError;
+    }
+
+    try {
+      return await runWhisperCpp({
+        binaryPath: opts.binaryPath,
+        modelPath: opts.modelPath,
+        language: opts.language,
+        audioPath: convertedPath,
+        outputBase: path.join(opts.workDir, "transcript-converted"),
+      });
+    } catch (convertedError: unknown) {
+      const directMessage = directError instanceof Error ? directError.message : String(directError);
+      const convertedMessage = convertedError instanceof Error ? convertedError.message : String(convertedError);
+      throw new Error(`${directMessage}\nffmpeg fallback failed: ${convertedMessage}`);
+    }
+  }
+}
+
+async function runWhisperCpp(opts: {
+  binaryPath: string;
+  modelPath: string;
+  language: string;
+  audioPath: string;
+  outputBase: string;
+}): Promise<string> {
+  const args = [
+    "--model",
+    opts.modelPath,
+    "--file",
+    opts.audioPath,
+    "--language",
+    opts.language,
+    "--output-txt",
+    "--output-file",
+    opts.outputBase,
+  ];
+
+  await runCommand(opts.binaryPath, args);
+  const transcriptPath = `${opts.outputBase}.txt`;
+  const transcript = normalizeWhisperTranscript(await readFile(transcriptPath, "utf8"));
+
+  if (!transcript) {
+    throw new Error("whisper.cpp returned an empty transcript.");
+  }
+
+  return transcript;
+}
+
+async function maybeConvertAudioWithFfmpeg(audioPath: string, workDir: string): Promise<string | null> {
+  if (path.extname(audioPath).toLowerCase() === ".wav") {
+    return null;
+  }
+
+  const convertedPath = path.join(workDir, "input.wav");
+  try {
+    await runCommand("ffmpeg", ["-y", "-i", audioPath, "-ar", "16000", "-ac", "1", "-c:a", "pcm_s16le", convertedPath]);
+    return convertedPath;
+  } catch {
+    return null;
+  }
+}
+
+async function downloadWhisperCppModel(model: string, destinationPath: string): Promise<void> {
+  await mkdir(path.dirname(destinationPath), { recursive: true, mode: 0o700 });
+  const response = await fetch(getWhisperCppModelDownloadUrl(model));
+  if (!response.ok || !response.body) {
+    throw new Error(`Failed to download whisper.cpp model ${model}: ${response.status} ${response.statusText}`);
+  }
+
+  // Keep the temp file on the target filesystem so the final rename stays atomic.
+  const tempDir = await mkdtemp(path.join(path.dirname(destinationPath), ".grok-whisper-model-"));
+  const tempPath = path.join(tempDir, getWhisperCppModelFileName(model));
+
+  try {
+    const stream = createWriteStream(tempPath, { mode: 0o600 });
+    await pipeline(Readable.fromWeb(response.body as globalThis.ReadableStream<Uint8Array>), stream);
+    await rename(tempPath, destinationPath);
+  } finally {
+    await rm(tempDir, { recursive: true, force: true });
+  }
+}
+
+async function ensureFileReadable(filePath: string, message: string): Promise<void> {
+  try {
+    await access(filePath);
+  } catch {
+    throw new Error(message);
+  }
+}
+
+function normalizeWhisperCppModel(model = DEFAULT_WHISPER_CPP_MODEL): string {
+  const normalized = model.trim() || DEFAULT_WHISPER_CPP_MODEL;
+  if (!MODEL_ID_PATTERN.test(normalized)) {
+    throw new Error(`Invalid whisper.cpp model name: ${model}`);
+  }
+  return normalized;
+}
+
+function runCommand(command: string, args: string[]): Promise<void> {
+  return new Promise((resolve, reject) => {
+    const child = spawn(command, args, {
+      stdio: ["ignore", "pipe", "pipe"],
+    });
+
+    let stdout = "";
+    let stderr = "";
+
+    child.stdout?.on("data", (chunk: Buffer | string) => {
+      stdout += chunk.toString();
+    });
+    child.stderr?.on("data", (chunk: Buffer | string) => {
+      stderr += chunk.toString();
+    });
+
+    child.on("error", (error: NodeJS.ErrnoException) => {
+      if (error.code === "ENOENT") {
+        reject(new Error(`Command not found: ${command}`));
+        return;
+      }
+      reject(error);
+    });
+
+    child.on("close", (code) => {
+      if (code === 0) {
+        resolve();
+        return;
+      }
+
+      const details = stderr.trim() || stdout.trim() || `exit code ${code ?? "unknown"}`;
+      reject(new Error(`${command} failed: ${details}`));
+    });
+  });
+}

src/telegram/audio-input.test.ts

@@ -0,0 +1,41 @@
+import { describe, expect, it } from "vitest";
+import { formatTelegramAudioTranscript, getTelegramAudioSource } from "./audio-input";
+
+describe("telegram audio input helpers", () => {
+  it("extracts a voice source from a telegram message payload", () => {
+    expect(
+      getTelegramAudioSource({
+        voice: {
+          file_id: "voice-file",
+          mime_type: "audio/ogg",
+        },
+      }),
+    ).toEqual({
+      kind: "voice",
+      fileId: "voice-file",
+      mimeType: "audio/ogg",
+    });
+  });
+
+  it("extracts an audio attachment source from a telegram message payload", () => {
+    expect(
+      getTelegramAudioSource({
+        audio: {
+          file_id: "audio-file",
+          file_name: "song.mp3",
+          mime_type: "audio/mpeg",
+        },
+      }),
+    ).toEqual({
+      kind: "audio",
+      fileId: "audio-file",
+      fileName: "song.mp3",
+      mimeType: "audio/mpeg",
+    });
+  });
+
+  it("formats user-visible transcript prefixes for voice and audio messages", () => {
+    expect(formatTelegramAudioTranscript("voice", "hello there")).toBe("[Voice transcript] hello there");
+    expect(formatTelegramAudioTranscript("audio", "demo")).toBe("[Audio transcript] demo");
+  });
+});

src/telegram/audio-input.ts

@@ -0,0 +1,116 @@
+import { mkdtemp, rm, writeFile } from "fs/promises";
+import os from "os";
+import path from "path";
+import { createTelegramAudioInputEngine } from "../audio/stt/engine";
+import type { TelegramSettings } from "../utils/settings";
+import { resolveTelegramAudioInputSettings } from "../utils/settings";
+
+export interface TelegramAudioSource {
+  kind: "voice" | "audio";
+  fileId: string;
+  fileName?: string;
+  mimeType?: string;
+}
+
+export interface TelegramFileApi {
+  getFile(fileId: string): Promise<{ file_path?: string }>;
+}
+
+export interface TelegramAudioTranscription {
+  promptText: string;
+  userContent: string;
+}
+
+export function getTelegramAudioSource(message: {
+  voice?: { file_id: string; mime_type?: string };
+  audio?: { file_id: string; file_name?: string; mime_type?: string };
+}): TelegramAudioSource | null {
+  if (message.voice) {
+    return {
+      kind: "voice",
+      fileId: message.voice.file_id,
+      mimeType: message.voice.mime_type,
+    };
+  }
+
+  if (message.audio) {
+    return {
+      kind: "audio",
+      fileId: message.audio.file_id,
+      fileName: message.audio.file_name,
+      mimeType: message.audio.mime_type,
+    };
+  }
+
+  return null;
+}
+
+export function formatTelegramAudioTranscript(kind: TelegramAudioSource["kind"], transcript: string): string {
+  const label = kind === "voice" ? "Voice" : "Audio";
+  return `[${label} transcript] ${transcript}`;
+}
+
+export async function transcribeTelegramAudioMessage(opts: {
+  api: TelegramFileApi;
+  token: string;
+  source: TelegramAudioSource;
+  telegramSettings: TelegramSettings | undefined;
+}): Promise<TelegramAudioTranscription> {
+  const audioSettings = resolveTelegramAudioInputSettings(opts.telegramSettings);
+  if (!audioSettings.enabled) {
+    throw new Error("Telegram audio input is disabled in settings.");
+  }
+
+  const tempDir = await mkdtemp(path.join(os.tmpdir(), "grok-telegram-audio-"));
+  try {
+    const file = await opts.api.getFile(opts.source.fileId);
+    if (!file.file_path) {
+      throw new Error("Telegram did not return a downloadable file path for this audio message.");
+    }
+
+    const response = await fetch(`https://api.telegram.org/file/bot${opts.token}/${file.file_path}`);
+    if (!response.ok) {
+      throw new Error(`Telegram audio download failed: ${response.status} ${response.statusText}`);
+    }
+
+    const extension = inferTelegramAudioExtension(opts.source, file.file_path);
+    const audioPath = path.join(tempDir, `input${extension}`);
+    const bytes = Buffer.from(await response.arrayBuffer());
+    await writeFile(audioPath, bytes, { mode: 0o600 });
+
+    const engine = createTelegramAudioInputEngine(opts.telegramSettings);
+    const result = await engine.transcribe({ audioPath });
+
+    return {
+      promptText: result.text,
+      userContent: formatTelegramAudioTranscript(opts.source.kind, result.text),
+    };
+  } finally {
+    await rm(tempDir, { recursive: true, force: true });
+  }
+}
+
+function inferTelegramAudioExtension(source: TelegramAudioSource, filePath?: string): string {
+  const fromPath = filePath ? path.extname(filePath) : "";
+  if (fromPath) return fromPath;
+
+  const fromName = source.fileName ? path.extname(source.fileName) : "";
+  if (fromName) return fromName;
+
+  switch ((source.mimeType || "").toLowerCase()) {
+    case "audio/ogg":
+      return ".ogg";
+    case "audio/opus":
+      return ".opus";
+    case "audio/mpeg":
+      return ".mp3";
+    case "audio/mp4":
+    case "audio/x-m4a":
+      return ".m4a";
+    case "audio/wav":
+    case "audio/x-wav":
+      return ".wav";
+    default:
+      return source.kind === "voice" ? ".ogg" : ".bin";
+  }
+}

src/telegram/bridge.ts

@@ -2,6 +2,7 @@ import { Bot } from "grammy";
 import type { Agent } from "../agent/agent";
 import type { ToolCall, ToolResult } from "../types/index";
 import { loadUserSettings, resolveTelegramStreamSettings } from "../utils/settings";
+import { getTelegramAudioSource, transcribeTelegramAudioMessage } from "./audio-input";
 import { splitTelegramMessage, TELEGRAM_MAX_MESSAGE } from "./limits";
 import { registerPairingCode } from "./pairing";
 import { runTelegramPartialReply } from "./preview-stream";
@@ -32,34 +33,56 @@ export function createTelegramBridge(opts: TelegramBridgeOptions): TelegramBridg
   const bot = new Bot(opts.token);
   let running = false;
 
-  bot.command("start", async (ctx) => {
-    await ctx.reply("Send /pair to link this chat to Grok CLI, then approve the code in the terminal.");
-  });
+  const buildTurnKey = (ctx: { chat: { id: number }; message: { message_id: number } }) =>
+    `telegram:${ctx.chat.id}:${ctx.message.message_id}`;
 
-  bot.command("pair", async (ctx) => {
+  const ensureApprovedUser = async (ctx: { from?: { id?: number }; reply: (text: string) => Promise<unknown> }) => {
     const userId = ctx.from?.id;
-    if (userId === undefined) return;
-    const code = registerPairingCode(userId);
-    await ctx.reply(`Your pairing code: ${code}\nEnter this code in Grok CLI (/remote-control → Telegram) to approve.`);
-  });
-
-  bot.on("message:text", async (ctx) => {
-    const text = ctx.message.text;
-    if (text.startsWith("/")) return;
-
-    const userId = ctx.from?.id;
-    if (userId === undefined) return;
+    if (userId === undefined) return null;
 
     const approved = opts.getApprovedUserIds();
     if (!approved.includes(userId)) {
       await ctx.reply("Not paired yet. Send /pair to get a code, then approve in Grok CLI.");
-      return;
+      return null;
     }
 
+    return userId;
+  };
+
+  const replyTurnError = async (
+    ctx: { reply: (text: string) => Promise<unknown>; chat: { id: number }; message: { message_id: number } },
+    userId: number,
+    message: string,
+  ) => {
+    const clipped = message.slice(0, TELEGRAM_MAX_MESSAGE);
+    opts.onAssistantMessage?.({
+      turnKey: buildTurnKey(ctx),
+      userId,
+      content: `Error: ${clipped}`,
+      done: true,
+    });
+    try {
+      await ctx.reply(`Error: ${clipped}`);
+    } catch {
+      /* user blocked bot or chat forbids messages */
+    }
+  };
+
+  const runAgentTurn = async (
+    ctx: {
+      chat: { id: number };
+      from?: { id?: number };
+      message: { message_id: number; message_thread_id?: number };
+      reply: (text: string) => Promise<unknown>;
+    },
+    userId: number,
+    userContent: string,
+    promptText: string,
+  ) => {
     await opts.coordinator.run(async () => {
       try {
-        const turnKey = `telegram:${ctx.chat.id}:${ctx.message.message_id}`;
-        opts.onUserMessage?.({ turnKey, userId, content: text });
+        const turnKey = buildTurnKey(ctx);
+        opts.onUserMessage?.({ turnKey, userId, content: userContent });
         const agent = opts.getTelegramAgent(userId);
         const stream = resolveTelegramStreamSettings(loadUserSettings().telegram);
 
@@ -72,7 +95,7 @@ export function createTelegramBridge(opts: TelegramBridgeOptions): TelegramBridg
           );
           let acc = "";
           try {
-            for await (const chunk of agent.processMessage(text)) {
+            for await (const chunk of agent.processMessage(promptText)) {
               switch (chunk.type) {
                 case "content":
                   if (chunk.content) {
@@ -99,13 +122,7 @@ export function createTelegramBridge(opts: TelegramBridgeOptions): TelegramBridg
             }
           } catch (err: unknown) {
             const msg = err instanceof Error ? err.message : String(err);
-            opts.onAssistantMessage?.({
-              turnKey,
-              userId,
-              content: `Error: ${msg.slice(0, TELEGRAM_MAX_MESSAGE)}`,
-              done: true,
-            });
-            await ctx.reply(`Error: ${msg.slice(0, TELEGRAM_MAX_MESSAGE)}`);
+            await replyTurnError(ctx, userId, msg);
             return;
           } finally {
             stopTyping();
@@ -123,7 +140,7 @@ export function createTelegramBridge(opts: TelegramBridgeOptions): TelegramBridg
           chatId: ctx.chat.id,
           messageThreadId: ctx.message.message_thread_id,
           typingIndicator: stream.typingIndicator,
-          stream: agent.processMessage(text),
+          stream: agent.processMessage(promptText),
           onAssistantMessage: (event) => {
             opts.onAssistantMessage?.({
               turnKey,
@@ -146,19 +163,70 @@ export function createTelegramBridge(opts: TelegramBridgeOptions): TelegramBridg
         });
       } catch (err: unknown) {
         const msg = err instanceof Error ? err.message : String(err);
-        opts.onAssistantMessage?.({
-          turnKey: `telegram:${ctx.chat.id}:${ctx.message.message_id}`,
-          userId,
-          content: `Error: ${msg.slice(0, TELEGRAM_MAX_MESSAGE)}`,
-          done: true,
-        });
-        try {
-          await ctx.reply(`Error: ${msg.slice(0, TELEGRAM_MAX_MESSAGE)}`);
-        } catch {
-          /* user blocked bot or chat forbids messages */
-        }
+        await replyTurnError(ctx, userId, msg);
       }
     });
+  };
+
+  const handleAudioMessage = async (ctx: {
+    chat: { id: number };
+    from?: { id?: number };
+    message: {
+      message_id: number;
+      message_thread_id?: number;
+      voice?: { file_id: string; mime_type?: string };
+      audio?: { file_id: string; file_name?: string; mime_type?: string };
+    };
+    reply: (text: string) => Promise<unknown>;
+  }) => {
+    const userId = await ensureApprovedUser(ctx);
+    if (userId === null) return;
+
+    const source = getTelegramAudioSource(ctx.message);
+    if (!source) return;
+
+    try {
+      await bot.api.sendChatAction(ctx.chat.id, "typing");
+      const transcription = await transcribeTelegramAudioMessage({
+        api: bot.api,
+        token: opts.token,
+        source,
+        telegramSettings: loadUserSettings().telegram,
+      });
+      await runAgentTurn(ctx, userId, transcription.userContent, transcription.promptText);
+    } catch (err: unknown) {
+      const msg = err instanceof Error ? err.message : String(err);
+      await replyTurnError(ctx, userId, `Audio transcription failed: ${msg}`);
+    }
+  };
+
+  bot.command("start", async (ctx) => {
+    await ctx.reply("Send /pair to link this chat to Grok CLI, then approve the code in the terminal.");
+  });
+
+  bot.command("pair", async (ctx) => {
+    const userId = ctx.from?.id;
+    if (userId === undefined) return;
+    const code = registerPairingCode(userId);
+    await ctx.reply(`Your pairing code: ${code}\nEnter this code in Grok CLI (/remote-control → Telegram) to approve.`);
+  });
+
+  bot.on("message:text", async (ctx) => {
+    const text = ctx.message.text;
+    if (text.startsWith("/")) return;
+
+    const userId = await ensureApprovedUser(ctx);
+    if (userId === null) return;
+
+    await runAgentTurn(ctx, userId, text, text);
+  });
+
+  bot.on("message:voice", async (ctx) => {
+    await handleAudioMessage(ctx);
+  });
+
+  bot.on("message:audio", async (ctx) => {
+    await handleAudioMessage(ctx);
   });
 
   bot.catch((err) => {

src/utils/settings.ts

@@ -5,6 +5,27 @@ import { DEFAULT_MODEL, getEffectiveReasoningEffort, getModelIds, normalizeModel
 import type { ReasoningEffort } from "../types/index";
 
 export type TelegramStreamingMode = "off" | "partial";
+export type TelegramAudioInputEngine = "whisper.cpp";
+
+export const DEFAULT_TELEGRAM_AUDIO_INPUT_BINARY = "whisper-cli";
+export const DEFAULT_TELEGRAM_AUDIO_INPUT_MODEL = "tiny.en";
+
+export interface TelegramAudioInputSettings {
+  /** Enable Telegram voice/audio transcription before sending text to the agent. Default: true. */
+  enabled?: boolean;
+  /** Reserved for future providers. The current built-in engine is whisper.cpp. */
+  engine?: TelegramAudioInputEngine;
+  /** Path or command name for the whisper.cpp CLI. Default: whisper-cli on PATH. */
+  binaryPath?: string;
+  /** Whisper.cpp model alias used for cache/download resolution. Default: tiny.en. */
+  model?: string;
+  /** Optional override for an exact ggml model path on disk. */
+  modelPath?: string;
+  /** Automatically download missing models into ~/.grok/models/stt. Default: true. */
+  autoDownloadModel?: boolean;
+  /** Whisper language code passed to the CLI. Default: en. */
+  language?: string;
+}
 
 export interface TelegramSettings {
   botToken?: string;
@@ -16,6 +37,7 @@ export interface TelegramSettings {
   typingIndicator?: boolean;
   /** Reserved: Bot API `sendMessageDraft` for private DMs (not implemented yet). */
   nativeDrafts?: boolean;
+  audioInput?: TelegramAudioInputSettings;
 }
 
 export type McpRemoteTransport = "http" | "sse";
@@ -144,6 +166,10 @@ export function saveUserSettings(partial: Partial<UserSettings>): void {
           telegram: {
             ...current.telegram,
             ...partial.telegram,
+            audioInput: {
+              ...current.telegram?.audioInput,
+              ...partial.telegram?.audioInput,
+            },
             sessionsByUserId: {
               ...current.telegram?.sessionsByUserId,
               ...partial.telegram?.sessionsByUserId,
@@ -231,6 +257,26 @@ export function resolveTelegramStreamSettings(t: TelegramSettings | undefined):
   };
 }
 
+export function resolveTelegramAudioInputSettings(t: TelegramSettings | undefined): {
+  enabled: boolean;
+  engine: TelegramAudioInputEngine;
+  binaryPath: string;
+  model: string;
+  modelPath: string | undefined;
+  autoDownloadModel: boolean;
+  language: string;
+} {
+  return {
+    enabled: t?.audioInput?.enabled !== false,
+    engine: "whisper.cpp",
+    binaryPath: t?.audioInput?.binaryPath?.trim() || DEFAULT_TELEGRAM_AUDIO_INPUT_BINARY,
+    model: t?.audioInput?.model?.trim() || DEFAULT_TELEGRAM_AUDIO_INPUT_MODEL,
+    modelPath: t?.audioInput?.modelPath?.trim() || undefined,
+    autoDownloadModel: t?.audioInput?.autoDownloadModel !== false,
+    language: t?.audioInput?.language?.trim() || "en",
+  };
+}
+
 export function loadMcpServers(): McpServerConfig[] {
   return loadUserSettings().mcp?.servers ?? [];
 }

src/utils/telegram-audio-settings.test.ts

@@ -0,0 +1,43 @@
+import { describe, expect, it } from "vitest";
+import {
+  DEFAULT_TELEGRAM_AUDIO_INPUT_BINARY,
+  DEFAULT_TELEGRAM_AUDIO_INPUT_MODEL,
+  resolveTelegramAudioInputSettings,
+} from "./settings";
+
+describe("resolveTelegramAudioInputSettings", () => {
+  it("returns whisper.cpp defaults when audio input is unset", () => {
+    expect(resolveTelegramAudioInputSettings(undefined)).toEqual({
+      enabled: true,
+      engine: "whisper.cpp",
+      binaryPath: DEFAULT_TELEGRAM_AUDIO_INPUT_BINARY,
+      model: DEFAULT_TELEGRAM_AUDIO_INPUT_MODEL,
+      modelPath: undefined,
+      autoDownloadModel: true,
+      language: "en",
+    });
+  });
+
+  it("preserves explicit telegram audio overrides", () => {
+    expect(
+      resolveTelegramAudioInputSettings({
+        audioInput: {
+          enabled: false,
+          binaryPath: "/opt/whisper-cli",
+          model: "base.en",
+          modelPath: "/tmp/ggml-base.en.bin",
+          autoDownloadModel: false,
+          language: "fr",
+        },
+      }),
+    ).toEqual({
+      enabled: false,
+      engine: "whisper.cpp",
+      binaryPath: "/opt/whisper-cli",
+      model: "base.en",
+      modelPath: "/tmp/ggml-base.en.bin",
+      autoDownloadModel: false,
+      language: "fr",
+    });
+  });
+});