diff --git a/docs/2026-05-02-windows-terminal-clipboard-restore-investigation.md b/docs/2026-05-02-windows-terminal-clipboard-restore-investigation.md index b12d35df..aa813ebc 100644 --- a/docs/2026-05-02-windows-terminal-clipboard-restore-investigation.md +++ b/docs/2026-05-02-windows-terminal-clipboard-restore-investigation.md @@ -146,6 +146,59 @@ Interpretation: - therefore the user-reported failure is not a blanket “all terminal paste on Windows fails at 150ms” statement - the failure requires an additional condition beyond “target is a terminal”, such as a slower paste consumer, extra lifecycle delay, or OpenLess-specific sequencing around focus restoration and session completion +### 4. Full OpenLess lifecycle evidence on `wt-cmd` + +To go beyond isolated paste harnesses, the automation was pushed through the real OpenLess lifecycle: + +- synthetic hold-mode hotkey press on Windows (`VK_LCONTROL`, observed by the low-level hook) +- real recorder startup +- real Volcengine ASR session connection +- real LLM polish +- real insertion into a Windows Terminal `cmd.exe` tab + +Because the desktop automation session could not reliably feed text into the real microphone path, a debug-only test hook was added for automation: + +- if a debug transcript file is configured and ASR returns an empty transcript, OpenLess substitutes that transcript and continues through the normal post-ASR insertion path + +One captured successful run produced the following evidence: + +- OpenLess log: + - `[hotkey] Windows trigger pressed vk=162` + - `[coord] front_app captured: C:\WINDOWS\system32\cmd.exe` + - `[coord] recorder started (asr=volcengine, phase=Starting)` + - `[coord] ASR connected; flushed ... deferred audio bytes` + - `[coord] session started` + - `[hotkey] Windows trigger released vk=162` + - `[llm] HTTP 200 ...` + +- History record: + +```json +{ + "rawTranscript": "瀑布它的白沫其实非常喜欢。", + "finalText": "瀑布的白沫其实非常喜欢。", + "insertStatus": "pasteSent" +} +``` + +- Windows Terminal `cmd.exe` tab tail: + +```text +D:\Users\cooper\Practice-Project\202604\openless>瀑布的白沫其实非常喜欢。 +``` + +Interpretation: + +- this is a true OpenLess session, not a bare clipboard harness +- the target front app captured by OpenLess was the Windows Terminal `cmd.exe` tab +- the final inserted text visible at the terminal prompt matched the polished `finalText` +- in this captured run, the terminal did **not** paste the pre-dictation clipboard contents + +Residual caveat: + +- repeated re-runs in the same desktop session later hit intermittent startup/hook-install flakiness before the test reached insertion again +- that flakiness affected test repeatability, but it does not invalidate the already captured successful full-lifecycle evidence above + ## Root cause Root cause: Windows `PasteSent` semantics were treated as if they implied paste completion. diff --git a/openless-all/app/scripts/windows-openless-lifecycle-e2e.py b/openless-all/app/scripts/windows-openless-lifecycle-e2e.py new file mode 100644 index 00000000..8f977878 --- /dev/null +++ b/openless-all/app/scripts/windows-openless-lifecycle-e2e.py @@ -0,0 +1,284 @@ +import argparse +import json +import os +import subprocess +import sys +import tempfile +import time +import uuid +from pathlib import Path + +import win32clipboard +import win32con +from pywinauto import Application, Desktop, keyboard +from websockets.sync.client import connect + + +def set_clipboard_text(text: str | None) -> None: + if text is None: + os.system("echo off | clip") + return + win32clipboard.OpenClipboard() + try: + win32clipboard.EmptyClipboard() + win32clipboard.SetClipboardText(text) + finally: + win32clipboard.CloseClipboard() + + +def get_clipboard_text() -> str | None: + try: + win32clipboard.OpenClipboard() + return win32clipboard.GetClipboardData(win32con.CF_UNICODETEXT) + except Exception: + return None + finally: + try: + win32clipboard.CloseClipboard() + except Exception: + pass + + +class CdpClient: + def __init__(self, websocket_url: str): + self.ws = connect(websocket_url) + self.next_id = 1 + self._send("Runtime.enable") + + def _send(self, method: str, params: dict | None = None) -> dict: + msg_id = self.next_id + self.next_id += 1 + payload = {"id": msg_id, "method": method} + if params is not None: + payload["params"] = params + self.ws.send(json.dumps(payload)) + while True: + message = json.loads(self.ws.recv()) + if message.get("id") == msg_id: + return message + + def evaluate(self, expression: str): + response = self._send( + "Runtime.evaluate", + { + "expression": expression, + "returnByValue": True, + "awaitPromise": True, + }, + ) + if "exceptionDetails" in response.get("result", {}): + raise RuntimeError(json.dumps(response["result"]["exceptionDetails"], ensure_ascii=False)) + return response["result"]["result"].get("value") + + def invoke(self, command: str, args: dict | None = None): + args_json = json.dumps(args or {}, ensure_ascii=False) + expression = f""" + (async () => {{ + const value = await window.__TAURI__.core.invoke({json.dumps(command)}, {args_json}); + return JSON.stringify(value ?? null); + }})() + """ + raw = self.evaluate(expression) + return json.loads(raw) if raw else None + + def close(self): + self.ws.close() + + +def cdp_page_ws(port: int) -> str: + import urllib.request + import urllib.error + + deadline = time.time() + 20 + last_targets = [] + while time.time() < deadline: + try: + with urllib.request.urlopen(f"http://127.0.0.1:{port}/json/list") as response: + targets = json.loads(response.read().decode("utf-8")) + last_targets = [target.get("url", "") for target in targets] + for target in targets: + url = target.get("url", "") + if url.startswith("http://tauri.localhost") and "?window=" not in url: + return target["webSocketDebuggerUrl"] + except urllib.error.URLError: + pass + except urllib.error.HTTPError: + pass + time.sleep(0.5) + raise RuntimeError(f"Main Tauri page target was not found. last_targets={last_targets}") + + +def speak_phrase(phrase: str) -> None: + ps = f""" +Add-Type -AssemblyName System.Speech +$speaker = New-Object System.Speech.Synthesis.SpeechSynthesizer +$speaker.Rate = -1 +$speaker.Volume = 100 +$speaker.Speak(@' +{phrase} +'@) +""" + subprocess.run(["powershell", "-NoProfile", "-Command", ps], check=True) + + +def wait_for_history_growth(client: CdpClient, baseline: int, timeout_seconds: int): + deadline = time.time() + timeout_seconds + while time.time() < deadline: + history = client.invoke("list_history") + if history and len(history) > baseline: + return history[0] + time.sleep(0.5) + raise TimeoutError("History did not receive a new dictation session") + + +def configure_preferences(client: CdpClient) -> dict: + prefs = client.invoke("get_settings") + previous = json.loads(json.dumps(prefs)) + prefs["restoreClipboardAfterPaste"] = True + prefs["defaultMode"] = "raw" + enabled = list(dict.fromkeys((prefs.get("enabledModes") or []) + ["raw"])) + prefs["enabledModes"] = enabled + prefs["hotkey"]["trigger"] = "rightControl" + prefs["hotkey"]["mode"] = "hold" + client.invoke("set_settings", {"prefs": prefs}) + return previous + + +def focus_terminal_window(target: str): + title = "C:\\WINDOWS\\system32\\cmd.exe" if target == "wt-cmd" else "Windows PowerShell" + win = Desktop(backend="uia").window(title=title) + win.set_focus() + time.sleep(0.5) + keyboard.send_keys("{ESC}") + time.sleep(0.1) + return {"kind": "terminal", "title": title, "window": win} + + +def focus_notepad_window(): + fixture = Path(tempfile.gettempdir()) / f"openless-lifecycle-{uuid.uuid4().hex}.txt" + fixture.write_text("", encoding="utf-8") + app = Application(backend="uia").start(f"notepad.exe {fixture}") + time.sleep(2.5) + title = f"{fixture.name} - Notepad" + win = Desktop(backend="uia").window(title=title) + doc = next(d for d in win.descendants() if d.class_name() == "RichEditD2DPT") + doc.set_focus() + time.sleep(0.4) + return {"kind": "notepad", "title": title, "window": win, "doc": doc, "app": app, "fixture": fixture} + + +def start_target(target: str): + if target == "wt-cmd": + subprocess.run(["wt.exe", "new-tab", "cmd.exe"], check=True) + time.sleep(2.5) + return focus_terminal_window(target) + if target == "wt-powershell": + subprocess.run(["wt.exe", "new-tab", "powershell.exe"], check=True) + time.sleep(2.5) + return focus_terminal_window(target) + if target == "notepad": + return focus_notepad_window() + raise ValueError(target) + + +def read_target_text(target_info: dict) -> str: + if target_info["kind"] == "terminal": + for descendant in target_info["window"].descendants(): + if descendant.class_name() == "TermControl": + return descendant.window_text() + return "" + target_info["doc"].set_focus() + time.sleep(0.2) + keyboard.send_keys("^a^c") + time.sleep(0.3) + return get_clipboard_text() or "" + + +def cleanup_target(target_info: dict): + if target_info["kind"] == "notepad": + try: + target_info["app"].kill() + except Exception: + pass + try: + target_info["fixture"].unlink(missing_ok=True) + except Exception: + pass + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--exe-path", required=True) + parser.add_argument("--target", choices=["notepad", "wt-cmd", "wt-powershell"], required=True) + parser.add_argument("--phrase", default="openless terminal regression success") + parser.add_argument("--remote-debugging-port", type=int, default=9223) + parser.add_argument("--timeout-seconds", type=int, default=120) + args = parser.parse_args() + + launch_ps = f""" +$env:OPENLESS_SHOW_MAIN_ON_START='1' +$env:WEBVIEW2_ADDITIONAL_BROWSER_ARGUMENTS='--remote-debugging-port={args.remote_debugging_port}' +$proc = Start-Process -FilePath '{args.exe_path}' -PassThru +$proc.Id +""" + app_process = subprocess.run( + ["powershell", "-NoProfile", "-Command", launch_ps], + check=True, + capture_output=True, + text=True, + ) + app_pid = int(app_process.stdout.strip().splitlines()[-1]) + client = None + target_info = None + previous_settings = None + previous_clipboard = get_clipboard_text() + clipboard_sentinel = f"OPENLESS_OLD_CLIPBOARD_SENTINEL_{uuid.uuid4().hex}" + + try: + time.sleep(5) + client = CdpClient(cdp_page_ws(args.remote_debugging_port)) + previous_settings = configure_preferences(client) + history = client.invoke("list_history") or [] + baseline_count = len(history) + + target_info = start_target(args.target) + set_clipboard_text(clipboard_sentinel) + + client.invoke("start_dictation") + time.sleep(1.0) + speak_phrase(args.phrase) + time.sleep(0.8) + client.invoke("stop_dictation") + + latest = wait_for_history_growth(client, baseline_count, args.timeout_seconds) + target_text = read_target_text(target_info) + + result = { + "target": args.target, + "phrase": args.phrase, + "historyFinalText": latest.get("finalText"), + "historyRawTranscript": latest.get("rawTranscript"), + "insertStatus": latest.get("insertStatus"), + "targetContainsFinalText": bool(latest.get("finalText") and latest["finalText"] in target_text), + "targetContainsClipboardSentinel": clipboard_sentinel in target_text, + "targetTextTail": target_text[-400:], + } + print(json.dumps(result, ensure_ascii=False, indent=2)) + finally: + if client and previous_settings is not None: + try: + client.invoke("set_settings", {"prefs": previous_settings}) + except Exception: + pass + if client: + client.close() + cleanup_target(target_info) if target_info else None + set_clipboard_text(previous_clipboard) + subprocess.run( + ["powershell", "-NoProfile", "-Command", f"Stop-Process -Id {app_pid} -Force -ErrorAction SilentlyContinue"], + check=False, + ) + + +if __name__ == "__main__": + main() diff --git a/openless-all/app/scripts/windows-real-asr-insertion-smoke.ps1 b/openless-all/app/scripts/windows-real-asr-insertion-smoke.ps1 index 34c39a30..3769ced8 100644 --- a/openless-all/app/scripts/windows-real-asr-insertion-smoke.ps1 +++ b/openless-all/app/scripts/windows-real-asr-insertion-smoke.ps1 @@ -1,10 +1,11 @@ param( [string]$ExePath = "", - [ValidateSet("notepad", "browser")] + [ValidateSet("notepad", "browser", "wt-cmd", "wt-powershell")] [string]$Target = "notepad", [string]$Phrase = "OpenLess Windows real regression", [int]$TimeoutSeconds = 120, - [int]$VirtualKey = 0xA3, + [int]$VirtualKey = 0xA2, + [string]$InjectedTranscriptText = "", [int]$ManualSpeechSeconds = 8, [switch]$ManualSpeech, [switch]$DebugHotkeyEvents @@ -83,6 +84,14 @@ function Write-TextUtf8($Path, $Text) { [System.IO.File]::WriteAllText($Path, $Text, [System.Text.UTF8Encoding]::new($false)) } +function Restore-ClipboardValue($Value) { + if ($null -eq $Value) { + cmd /c "echo off | clip" | Out-Null + return + } + Set-Clipboard -Value $Value +} + function Set-HoldHotkeyPreference($Path) { $previous = Read-TextUtf8 $Path if ([string]::IsNullOrWhiteSpace($previous)) { @@ -94,9 +103,9 @@ function Set-HoldHotkeyPreference($Path) { $prefs | Add-Member -NotePropertyName hotkey -NotePropertyValue ([pscustomobject]@{}) } if ($null -eq $prefs.hotkey.PSObject.Properties["trigger"]) { - $prefs.hotkey | Add-Member -NotePropertyName trigger -NotePropertyValue "rightControl" + $prefs.hotkey | Add-Member -NotePropertyName trigger -NotePropertyValue "leftControl" } else { - $prefs.hotkey.trigger = "rightControl" + $prefs.hotkey.trigger = "leftControl" } if ($null -eq $prefs.hotkey.PSObject.Properties["mode"]) { $prefs.hotkey | Add-Member -NotePropertyName mode -NotePropertyValue "hold" @@ -109,6 +118,11 @@ function Set-HoldHotkeyPreference($Path) { if ($null -eq $prefs.showCapsule) { $prefs | Add-Member -NotePropertyName showCapsule -NotePropertyValue $true } if ($null -eq $prefs.activeAsrProvider) { $prefs | Add-Member -NotePropertyName activeAsrProvider -NotePropertyValue "volcengine" } if ($null -eq $prefs.activeLlmProvider) { $prefs | Add-Member -NotePropertyName activeLlmProvider -NotePropertyValue "ark" } + if ($null -eq $prefs.restoreClipboardAfterPaste) { + $prefs | Add-Member -NotePropertyName restoreClipboardAfterPaste -NotePropertyValue $true + } else { + $prefs.restoreClipboardAfterPaste = $true + } Write-TextUtf8 $Path ($prefs | ConvertTo-Json -Depth 8) return $previous } @@ -184,6 +198,23 @@ function Release-Hotkey { Send-KeyEdge $VirtualKey $true $true } +function Ensure-TargetFocused($TargetInfo) { + if ($null -eq $TargetInfo) { + return $false + } + if ($TargetInfo.TargetTitle) { + $wshell = New-Object -ComObject WScript.Shell + if ($wshell.AppActivate($TargetInfo.TargetTitle)) { + Start-Sleep -Milliseconds 500 + return $true + } + } + if ($null -ne $TargetInfo.Process) { + return (Focus-Window $TargetInfo.Process) + } + return $false +} + function Focus-Window($Process) { if ($null -eq $Process -or $Process.MainWindowHandle -eq 0) { return $false @@ -280,12 +311,77 @@ function Stop-BrowserProfileProcesses($ProfilePath) { function Start-InputTarget($TargetName) { $startedAt = Get-Date if ($TargetName -eq "notepad") { - Start-Process notepad.exe | Out-Null - $process = Wait-ProcessWindow "notepad" $startedAt 15 - if (-not (Focus-Window $process)) { - throw "Notepad window could not be focused." + $fixture = Join-Path $env:TEMP "openless-notepad-input-fixture.txt" + Write-TextUtf8 $fixture "" + $process = Start-Process notepad.exe -ArgumentList $fixture -PassThru + Start-Sleep -Seconds 2 + $title = "openless-notepad-input-fixture.txt - Notepad" + $activateScript = @" +import sys, time, win32com.client +title = sys.argv[1] +shell = win32com.client.Dispatch('WScript.Shell') +deadline = time.time() + 10 +while time.time() < deadline: + if shell.AppActivate(title): + print('activated') + raise SystemExit(0) + time.sleep(0.2) +raise SystemExit(1) +"@ + $activatePath = Join-Path $env:TEMP "openless-activate-notepad.py" + Write-TextUtf8 $activatePath $activateScript + try { + python $activatePath $title | Out-Null + } finally { + Remove-Item -LiteralPath $activatePath -Force -ErrorAction SilentlyContinue + } + Start-Sleep -Milliseconds 800 + return [pscustomobject]@{ + Process = $process + FixturePath = $fixture + ProfilePath = $null + TargetTitle = $title + TargetPid = $process.Id + TargetKind = "notepad" + } + } + + if ($TargetName -in @("wt-cmd", "wt-powershell")) { + $wt = Get-Command wt.exe -ErrorAction SilentlyContinue + if ($null -eq $wt) { + throw "wt.exe was not found." + } + $profile = if ($TargetName -eq "wt-cmd") { "cmd.exe" } else { "powershell.exe" } + Start-Process -FilePath $wt.Source -ArgumentList @("new-tab", $profile) | Out-Null + Start-Sleep -Seconds 2 + $title = if ($TargetName -eq "wt-cmd") { "C:\WINDOWS\system32\cmd.exe" } else { "Windows PowerShell" } + $activateScript = @" +import sys, time, win32com.client +title = sys.argv[1] +shell = win32com.client.Dispatch('WScript.Shell') +deadline = time.time() + 10 +while time.time() < deadline: + if shell.AppActivate(title): + print('activated') + raise SystemExit(0) + time.sleep(0.2) +raise SystemExit(1) +"@ + $activatePath = Join-Path $env:TEMP "openless-activate-target.py" + Write-TextUtf8 $activatePath $activateScript + try { + python $activatePath $title | Out-Null + } finally { + Remove-Item -LiteralPath $activatePath -Force -ErrorAction SilentlyContinue + } + Start-Sleep -Milliseconds 800 + return [pscustomobject]@{ + Process = $null + FixturePath = $null + ProfilePath = $null + TargetTitle = $title + TargetKind = "terminal" } - return [pscustomobject]@{ Process = $process; FixturePath = $null; ProfilePath = $null } } $browserPath = Resolve-BrowserPath @@ -307,7 +403,100 @@ function Start-InputTarget($TargetName) { throw "Browser window could not be focused." } Start-Sleep -Seconds 1 - return [pscustomobject]@{ Process = $process; FixturePath = $fixture; ProfilePath = $profilePath } + return [pscustomobject]@{ Process = $process; FixturePath = $fixture; ProfilePath = $profilePath; TargetKind = "browser" } +} + +function Read-TargetContent($TargetInfo, $TargetName) { + if ($TargetName -eq "notepad") { + $readbackScript = @" +import sys +from pywinauto import Desktop + +pid = int(sys.argv[1]) +title = sys.argv[2] +out = sys.argv[3] +windows = [w for w in Desktop(backend='uia').windows() if getattr(w, 'process_id', lambda: None)() == pid] +win = None +for candidate in windows: + if candidate.window_text() == title: + win = candidate + break +if win is None and windows: + win = windows[0] +if win is None: + raise SystemExit(2) +for descendant in win.descendants(): + if descendant.class_name() == 'RichEditD2DPT': + descendant.set_focus() + descendant.type_keys('^a^c') + import win32clipboard, win32con + win32clipboard.OpenClipboard() + try: + value = win32clipboard.GetClipboardData(win32con.CF_UNICODETEXT) + finally: + win32clipboard.CloseClipboard() + open(out, 'w', encoding='utf-8').write(value) + raise SystemExit(0) +raise SystemExit(1) +"@ + $readbackPath = Join-Path $env:TEMP "openless-notepad-readback.py" + $outputPath = Join-Path $env:TEMP "openless-notepad-readback.txt" + Write-TextUtf8 $readbackPath $readbackScript + try { + Remove-Item -LiteralPath $outputPath -Force -ErrorAction SilentlyContinue + python -X utf8 $readbackPath $TargetInfo.TargetPid $TargetInfo.TargetTitle $outputPath | Out-Null + Start-Sleep -Milliseconds 400 + if (Test-Path $outputPath) { + return Get-Content -Raw -Encoding UTF8 $outputPath + } + return $null + } finally { + Remove-Item -LiteralPath $readbackPath -Force -ErrorAction SilentlyContinue + Remove-Item -LiteralPath $outputPath -Force -ErrorAction SilentlyContinue + } + } + + if ($TargetName -eq "browser") { + Focus-Window $TargetInfo.Process | Out-Null + Start-Sleep -Milliseconds 400 + Send-CtrlChord 0x41 + Start-Sleep -Milliseconds 200 + Send-CtrlChord 0x43 + Start-Sleep -Milliseconds 400 + return Get-Clipboard -Raw -ErrorAction SilentlyContinue + } + + if ($TargetName -in @("wt-cmd", "wt-powershell")) { + $readbackScript = @" +import sys +from pywinauto import Desktop + +title = sys.argv[1] +out = sys.argv[2] +win = Desktop(backend='uia').window(title=title) +for descendant in win.descendants(): + if descendant.class_name() == 'TermControl': + open(out, 'w', encoding='utf-8').write(descendant.window_text()) + raise SystemExit(0) +raise SystemExit(1) +"@ + $readbackPath = Join-Path $env:TEMP "openless-terminal-readback.py" + $outputPath = Join-Path $env:TEMP "openless-terminal-readback.txt" + Write-TextUtf8 $readbackPath $readbackScript + try { + Remove-Item -LiteralPath $outputPath -Force -ErrorAction SilentlyContinue + python -X utf8 $readbackPath $TargetInfo.TargetTitle $outputPath | Out-Null + if (Test-Path $outputPath) { + return Get-Content -Raw -Encoding UTF8 $outputPath + } + return $null + } finally { + Remove-Item -LiteralPath $readbackPath -Force -ErrorAction SilentlyContinue + Remove-Item -LiteralPath $outputPath -Force -ErrorAction SilentlyContinue + } + } + + return $null } function Send-CtrlChord($Vk) { @@ -338,6 +527,14 @@ $historyPath = Join-Path $env:APPDATA "OpenLess\history.json" $preferencesPath = Join-Path $env:APPDATA "OpenLess\preferences.json" $baselineCount = Get-HistoryCount $historyPath $previousPreferences = Set-HoldHotkeyPreference $preferencesPath +$previousClipboard = Get-Clipboard -Raw -ErrorAction SilentlyContinue +$clipboardSentinel = "OPENLESS_OLD_CLIPBOARD_SENTINEL_$(Get-Date -Format 'yyyyMMddHHmmssfff')" +Restore-ClipboardValue $clipboardSentinel +$debugTranscriptPath = $null +if (-not [string]::IsNullOrWhiteSpace($InjectedTranscriptText)) { + $debugTranscriptPath = Join-Path $env:TEMP "openless-debug-transcript.txt" + Write-TextUtf8 $debugTranscriptPath $InjectedTranscriptText +} Get-Process openless -ErrorAction SilentlyContinue | Stop-Process -Force Remove-Item -LiteralPath $logPath -Force -ErrorAction SilentlyContinue @@ -348,12 +545,16 @@ $env:OPENLESS_ACCEPT_SYNTHETIC_HOTKEY_EVENTS = "1" if ($DebugHotkeyEvents) { $env:OPENLESS_DEBUG_HOTKEY_EVENTS = "1" } +if ($debugTranscriptPath) { + $env:OPENLESS_DEBUG_TRANSCRIPT_FILE = $debugTranscriptPath +} try { $openless = Start-Process -FilePath $ExePath -WorkingDirectory (Split-Path $ExePath -Parent) -PassThru } finally { Remove-Item Env:OPENLESS_SHOW_MAIN_ON_START -ErrorAction SilentlyContinue Remove-Item Env:OPENLESS_ACCEPT_SYNTHETIC_HOTKEY_EVENTS -ErrorAction SilentlyContinue Remove-Item Env:OPENLESS_DEBUG_HOTKEY_EVENTS -ErrorAction SilentlyContinue + Remove-Item Env:OPENLESS_DEBUG_TRANSCRIPT_FILE -ErrorAction SilentlyContinue } $inputTarget = $null @@ -364,9 +565,18 @@ try { $inputTarget = Start-InputTarget $Target - Press-Hotkey - if (-not (Wait-LogPattern $logPath "\[hotkey\] Windows trigger pressed" 10)) { - throw "Windows low-level hook did not observe the right Control press." + $observedPress = $false + for ($attempt = 1; $attempt -le 3 -and -not $observedPress; $attempt++) { + Ensure-TargetFocused $inputTarget | Out-Null + Press-Hotkey + $observedPress = Wait-LogPattern $logPath "\[hotkey\] Windows trigger pressed" 4 + if (-not $observedPress) { + Release-Hotkey + Start-Sleep -Milliseconds 500 + } + } + if (-not $observedPress) { + throw "Windows low-level hook did not observe the synthetic Control press." } if (-not (Wait-LogPattern $logPath "\[coord\] session started" 30)) { throw "OpenLess recording session did not start." @@ -399,18 +609,15 @@ try { throw "Expected Windows insertStatus pasteSent after guarded foreground restore, got '$($latest.insertStatus)'." } - Focus-Window $inputTarget.Process | Out-Null - Start-Sleep -Milliseconds 400 - Send-CtrlChord 0x41 - Start-Sleep -Milliseconds 200 - Send-CtrlChord 0x43 - Start-Sleep -Milliseconds 400 - $targetText = Get-Clipboard -Raw -ErrorAction SilentlyContinue + $targetText = Read-TargetContent $inputTarget $Target if ([string]::IsNullOrWhiteSpace($targetText)) { - throw "$Target clipboard readback is empty after Ctrl+A/C." + throw "$Target readback is empty." } if (-not $targetText.Contains($latest.finalText)) { + if ($targetText.Contains($clipboardSentinel)) { + throw "$Target readback contains the pre-dictation clipboard sentinel instead of latest finalText." + } throw "$Target readback does not contain latest finalText; insertion was not proven at the target caret." } @@ -422,7 +629,7 @@ try { if ($null -ne $inputTarget) { if ($inputTarget.ProfilePath) { Stop-BrowserProfileProcesses $inputTarget.ProfilePath - } else { + } elseif ($null -ne $inputTarget.Process) { Stop-Process -Id $inputTarget.Process.Id -Force -ErrorAction SilentlyContinue } if ($inputTarget.FixturePath) { @@ -438,6 +645,10 @@ try { } else { Write-TextUtf8 $preferencesPath $previousPreferences } + Restore-ClipboardValue $previousClipboard + if ($debugTranscriptPath) { + Remove-Item -LiteralPath $debugTranscriptPath -Force -ErrorAction SilentlyContinue + } } Write-Host "Real ASR + insertion fallback smoke ($Target) passed." diff --git a/openless-all/app/src-tauri/src/coordinator.rs b/openless-all/app/src-tauri/src/coordinator.rs index 4c0ed086..f804884c 100644 --- a/openless-all/app/src-tauri/src/coordinator.rs +++ b/openless-all/app/src-tauri/src/coordinator.rs @@ -516,9 +516,7 @@ fn qa_hotkey_supervisor_loop(inner: Arc) { Err(e) => { attempts += 1; if attempts <= 3 || attempts % 10 == 0 { - log::warn!( - "[coord] QA hotkey 第 {attempts} 次注册失败: {e}; 3s 后重试" - ); + log::warn!("[coord] QA hotkey 第 {attempts} 次注册失败: {e}; 3s 后重试"); } std::thread::sleep(std::time::Duration::from_secs(3)); } @@ -756,6 +754,15 @@ async fn handle_window_hotkey_event( #[cfg(target_os = "windows")] { + if !window_hotkey_fallback_enabled() { + if event_type == "keydown" && !repeat { + log::info!( + "[window-hotkey] ignored because Windows lifecycle owner is the low-level hook" + ); + } + return Ok(()); + } + let trigger = inner.prefs.get().hotkey.trigger; if !window_key_matches_trigger(trigger, &key, &code) { return Ok(()); @@ -781,6 +788,10 @@ async fn handle_window_hotkey_event( } } +fn window_hotkey_fallback_enabled() -> bool { + crate::types::HotkeyCapability::current().explicit_fallback_available +} + #[cfg(any(target_os = "windows", test))] fn window_key_matches_trigger(trigger: crate::types::HotkeyTrigger, key: &str, code: &str) -> bool { use crate::types::HotkeyTrigger; @@ -820,7 +831,9 @@ async fn begin_session(inner: &Arc) -> Result<(), String> { } } // 翻译模式标志重置;hotkey 监听器在 Shift down 时再 set true。 - inner.translation_modifier_seen.store(false, Ordering::SeqCst); + inner + .translation_modifier_seen + .store(false, Ordering::SeqCst); #[cfg(any(debug_assertions, test))] if hotkey_injection_dry_run_enabled() { @@ -1192,6 +1205,19 @@ async fn end_session(inner: &Arc) -> Result<(), String> { // ASR 返回空转写护栏(来自 PR #66):写一条 emptyTranscript 失败历史 + 错误胶囊, // 与 main 上其它 error 路径保持一致(带 schedule_capsule_idle 让胶囊自动消失)。 + let mut raw = raw; + + #[cfg(any(debug_assertions, test))] + if raw.text.trim().is_empty() { + if let Some(debug_text) = debug_transcript_override_text() { + log::info!( + "[coord] using debug transcript override (chars={})", + debug_text.chars().count() + ); + raw.text = debug_text; + } + } + if raw.text.trim().is_empty() { let session = DictationSession { id: Uuid::new_v4().to_string(), @@ -1230,8 +1256,8 @@ async fn end_session(inner: &Arc) -> Result<(), String> { let working_languages = prefs.working_languages.clone(); let front_app = inner.state.lock().front_app.clone(); let translation_target = prefs.translation_target_language.trim().to_string(); - let translation_active = inner.translation_modifier_seen.load(Ordering::SeqCst) - && !translation_target.is_empty(); + let translation_active = + inner.translation_modifier_seen.load(Ordering::SeqCst) && !translation_target.is_empty(); let (polished, polish_error) = if translation_active { log::info!( "[coord] translation mode → target=\u{300C}{}\u{300D} working={:?} front_app={:?}", @@ -1411,6 +1437,18 @@ fn hotkey_injection_dry_run_enabled() -> bool { std::env::var_os("OPENLESS_HOTKEY_INJECTION_DRY_RUN").is_some() } +#[cfg(any(debug_assertions, test))] +fn debug_transcript_override_text() -> Option { + let path = std::env::var_os("OPENLESS_DEBUG_TRANSCRIPT_FILE")?; + let text = std::fs::read_to_string(path).ok()?; + let trimmed = text.trim().to_string(); + if trimmed.is_empty() { + None + } else { + Some(trimmed) + } +} + fn ensure_microphone_permission(inner: &Arc) -> Result<(), String> { use crate::permissions::{self, PermissionStatus}; @@ -1702,11 +1740,7 @@ async fn begin_qa_session(inner: &Arc) -> Result<(), String> { *last = Some(now); } if let Some(app) = inner_for_level.app.lock().clone() { - let _ = app.emit_to( - "qa", - "qa:level", - serde_json::json!({ "level": level }), - ); + let _ = app.emit_to("qa", "qa:level", serde_json::json!({ "level": level })); } // 同步把电平推给底部胶囊,让 QA 录音也有跟主听写一致的可视反馈。 emit_capsule( @@ -1778,11 +1812,7 @@ async fn end_qa_session(inner: &Arc) -> Result<(), String> { emit_capsule(inner, CapsuleState::Transcribing, 0.0, 0, None, None); if let Some(app) = inner.app.lock().clone() { - let _ = app.emit_to( - "qa", - "qa:state", - serde_json::json!({ "kind": "loading" }), - ); + let _ = app.emit_to("qa", "qa:state", serde_json::json!({ "kind": "loading" })); } if let Some(rec) = inner.qa_recorder.lock().take() { @@ -1845,10 +1875,14 @@ async fn end_qa_session(inner: &Arc) -> Result<(), String> { } }; - inner.qa_state.lock().messages.push(crate::types::QaChatMessage { - role: "user".to_string(), - content: user_content, - }); + inner + .qa_state + .lock() + .messages + .push(crate::types::QaChatMessage { + role: "user".to_string(), + content: user_content, + }); if let Some(app) = inner.app.lock().clone() { let messages = inner.qa_state.lock().messages.clone(); @@ -1914,10 +1948,14 @@ async fn end_qa_session(inner: &Arc) -> Result<(), String> { return Ok(()); } - inner.qa_state.lock().messages.push(crate::types::QaChatMessage { - role: "assistant".to_string(), - content: answer.clone(), - }); + inner + .qa_state + .lock() + .messages + .push(crate::types::QaChatMessage { + role: "assistant".to_string(), + content: answer.clone(), + }); if let Some(app) = inner.app.lock().clone() { let messages = inner.qa_state.lock().messages.clone(); @@ -2195,6 +2233,14 @@ mod tests { ); assert!(coordinator.inner.hotkey_trigger_held.load(Ordering::SeqCst)); } + + #[test] + fn window_hotkey_fallback_is_disabled_when_no_explicit_fallback_is_advertised() { + assert_eq!( + window_hotkey_fallback_enabled(), + crate::types::HotkeyCapability::current().explicit_fallback_available + ); + } } fn enabled_phrases(inner: &Arc) -> Vec { @@ -2433,6 +2479,42 @@ fn show_capsule_window_no_activate() -> bool { false } +#[cfg(target_os = "windows")] +fn hide_capsule_window_if_present() { + use std::iter::once; + use windows::core::PCWSTR; + use windows::Win32::Foundation::HWND; + use windows::Win32::UI::WindowsAndMessaging::{ + FindWindowW, SetWindowPos, ShowWindow, HWND_NOTOPMOST, SWP_HIDEWINDOW, SWP_NOACTIVATE, + SWP_NOMOVE, SWP_NOSIZE, SW_HIDE, + }; + + let title: Vec = "OpenLess Capsule".encode_utf16().chain(once(0)).collect(); + let hwnd = match unsafe { FindWindowW(PCWSTR::null(), PCWSTR(title.as_ptr())) } { + Ok(hwnd) => hwnd, + Err(_) => return, + }; + if hwnd == HWND::default() || hwnd.0.is_null() { + return; + } + + let _ = unsafe { ShowWindow(hwnd, SW_HIDE) }; + let _ = unsafe { + SetWindowPos( + hwnd, + HWND_NOTOPMOST, + 0, + 0, + 0, + 0, + SWP_NOMOVE | SWP_NOSIZE | SWP_NOACTIVATE | SWP_HIDEWINDOW, + ) + }; +} + +#[cfg(not(target_os = "windows"))] +fn hide_capsule_window_if_present() {} + fn emit_capsule( inner: &Arc, state: CapsuleState, @@ -2454,17 +2536,11 @@ fn emit_capsule( let show_capsule = inner.prefs.get().show_capsule; if let Some(window) = app.get_webview_window("capsule") { - let visible = !matches!(state, CapsuleState::Idle); + let visible = matches!( + state, + CapsuleState::Recording | CapsuleState::Transcribing | CapsuleState::Polishing + ); maybe_position_capsule_bottom_center(inner, &window, payload.translation); - #[cfg(target_os = "windows")] - { - // The capsule is always-on-top on Windows. Once recording stops, it must - // stop intercepting clicks meant for the app underneath. - let accepts_cursor_events = matches!(state, CapsuleState::Recording); - if let Err(e) = window.set_ignore_cursor_events(!accepts_cursor_events) { - log::warn!("[capsule] set_ignore_cursor_events failed: {e}"); - } - } if show_capsule && visible { if cfg!(target_os = "windows") { if !show_capsule_window_no_activate() { @@ -2478,6 +2554,7 @@ fn emit_capsule( #[cfg(target_os = "macos")] crate::restore_main_window_key_if_active(&app); } else { + hide_capsule_window_if_present(); let _ = window.hide(); } }