test(windows): automate openless lifecycle evidence

This commit is contained in:
Cooper-X-Oak committed 2026-05-02 04:47:37 +08:00
1 parent 23207beb8e
commit 734c430339
4 files changed
+681 -56

No files matched your search

@@ -146,6 +146,59 @@ Interpretation:
- therefore the user-reported failure is not a blanket “all terminal paste on Windows fails at 150ms” statement
- the failure requires an additional condition beyond “target is a terminal”, such as a slower paste consumer, extra lifecycle delay, or OpenLess-specific sequencing around focus restoration and session completion
### 4. Full OpenLess lifecycle evidence on `wt-cmd`
To go beyond isolated paste harnesses, the automation was pushed through the real OpenLess lifecycle:
- synthetic hold-mode hotkey press on Windows (`VK_LCONTROL`, observed by the low-level hook)
- real recorder startup
- real Volcengine ASR session connection
- real LLM polish
- real insertion into a Windows Terminal `cmd.exe` tab
Because the desktop automation session could not reliably feed text into the real microphone path, a debug-only test hook was added for automation:
- if a debug transcript file is configured and ASR returns an empty transcript, OpenLess substitutes that transcript and continues through the normal post-ASR insertion path
One captured successful run produced the following evidence:
- OpenLess log:
- `[hotkey] Windows trigger pressed vk=162`
- `[coord] front_app captured: C:\WINDOWS\system32\cmd.exe`
- `[coord] recorder started (asr=volcengine, phase=Starting)`
- `[coord] ASR connected; flushed ... deferred audio bytes`
- `[coord] session started`
- `[hotkey] Windows trigger released vk=162`
- `[llm] HTTP 200 ...`
- History record:
```json
{
"rawTranscript": "瀑布它的白沫其实非常喜欢。",
"finalText": "瀑布的白沫其实非常喜欢。",
"insertStatus": "pasteSent"
}
```
- Windows Terminal `cmd.exe` tab tail:
```text
D:\Users\cooper\Practice-Project\202604\openless>瀑布的白沫其实非常喜欢。
```
Interpretation:
- this is a true OpenLess session, not a bare clipboard harness
- the target front app captured by OpenLess was the Windows Terminal `cmd.exe` tab
- the final inserted text visible at the terminal prompt matched the polished `finalText`
- in this captured run, the terminal did **not** paste the pre-dictation clipboard contents
Residual caveat:
- repeated re-runs in the same desktop session later hit intermittent startup/hook-install flakiness before the test reached insertion again
- that flakiness affected test repeatability, but it does not invalidate the already captured successful full-lifecycle evidence above
## Root cause
Root cause: Windows `PasteSent` semantics were treated as if they implied paste completion.
@@ -0,0 +1,284 @@
import argparse
import json
import os
import subprocess
import sys
import tempfile
import time
import uuid
from pathlib import Path
import win32clipboard
import win32con
from pywinauto import Application, Desktop, keyboard
from websockets.sync.client import connect
def set_clipboard_text(text: str | None) -> None:
if text is None:
os.system("echo off | clip")
return
win32clipboard.OpenClipboard()
try:
win32clipboard.EmptyClipboard()
win32clipboard.SetClipboardText(text)
finally:
win32clipboard.CloseClipboard()
def get_clipboard_text() -> str | None:
try:
win32clipboard.OpenClipboard()
return win32clipboard.GetClipboardData(win32con.CF_UNICODETEXT)
except Exception:
return None
finally:
try:
win32clipboard.CloseClipboard()
except Exception:
pass
class CdpClient:
def __init__(self, websocket_url: str):
self.ws = connect(websocket_url)
self.next_id = 1
self._send("Runtime.enable")
def _send(self, method: str, params: dict | None = None) -> dict:
msg_id = self.next_id
self.next_id += 1
payload = {"id": msg_id, "method": method}
if params is not None:
payload["params"] = params
self.ws.send(json.dumps(payload))
while True:
message = json.loads(self.ws.recv())
if message.get("id") == msg_id:
return message
def evaluate(self, expression: str):
response = self._send(
"Runtime.evaluate",
{
"expression": expression,
"returnByValue": True,
"awaitPromise": True,
},
)
if "exceptionDetails" in response.get("result", {}):
raise RuntimeError(json.dumps(response["result"]["exceptionDetails"], ensure_ascii=False))
return response["result"]["result"].get("value")
def invoke(self, command: str, args: dict | None = None):
args_json = json.dumps(args or {}, ensure_ascii=False)
expression = f"""
(async () => {{
const value = await window.__TAURI__.core.invoke({json.dumps(command)}, {args_json});
return JSON.stringify(value ?? null);
}})()
"""
raw = self.evaluate(expression)
return json.loads(raw) if raw else None
def close(self):
self.ws.close()
def cdp_page_ws(port: int) -> str:
import urllib.request
import urllib.error
deadline = time.time() + 20
last_targets = []
while time.time() < deadline:
try:
with urllib.request.urlopen(f"http://127.0.0.1:{port}/json/list") as response:
targets = json.loads(response.read().decode("utf-8"))
last_targets = [target.get("url", "") for target in targets]
for target in targets:
url = target.get("url", "")
if url.startswith("http://tauri.localhost") and "?window=" not in url:
return target["webSocketDebuggerUrl"]
except urllib.error.URLError:
pass
except urllib.error.HTTPError:
pass
time.sleep(0.5)
raise RuntimeError(f"Main Tauri page target was not found. last_targets={last_targets}")
def speak_phrase(phrase: str) -> None:
ps = f"""
Add-Type -AssemblyName System.Speech
$speaker = New-Object System.Speech.Synthesis.SpeechSynthesizer
$speaker.Rate = -1
$speaker.Volume = 100
$speaker.Speak(@'
{phrase}
'@)
"""
subprocess.run(["powershell", "-NoProfile", "-Command", ps], check=True)
def wait_for_history_growth(client: CdpClient, baseline: int, timeout_seconds: int):
deadline = time.time() + timeout_seconds
while time.time() < deadline:
history = client.invoke("list_history")
if history and len(history) > baseline:
return history[0]
time.sleep(0.5)
raise TimeoutError("History did not receive a new dictation session")
def configure_preferences(client: CdpClient) -> dict:
prefs = client.invoke("get_settings")
previous = json.loads(json.dumps(prefs))
prefs["restoreClipboardAfterPaste"] = True
prefs["defaultMode"] = "raw"
enabled = list(dict.fromkeys((prefs.get("enabledModes") or []) + ["raw"]))
prefs["enabledModes"] = enabled
prefs["hotkey"]["trigger"] = "rightControl"
prefs["hotkey"]["mode"] = "hold"
client.invoke("set_settings", {"prefs": prefs})
return previous
def focus_terminal_window(target: str):
title = "C:\\WINDOWS\\system32\\cmd.exe" if target == "wt-cmd" else "Windows PowerShell"
win = Desktop(backend="uia").window(title=title)
win.set_focus()
time.sleep(0.5)
keyboard.send_keys("{ESC}")
time.sleep(0.1)
return {"kind": "terminal", "title": title, "window": win}
def focus_notepad_window():
fixture = Path(tempfile.gettempdir()) / f"openless-lifecycle-{uuid.uuid4().hex}.txt"
fixture.write_text("", encoding="utf-8")
app = Application(backend="uia").start(f"notepad.exe {fixture}")
time.sleep(2.5)
title = f"{fixture.name} - Notepad"
win = Desktop(backend="uia").window(title=title)
doc = next(d for d in win.descendants() if d.class_name() == "RichEditD2DPT")
doc.set_focus()
time.sleep(0.4)
return {"kind": "notepad", "title": title, "window": win, "doc": doc, "app": app, "fixture": fixture}
def start_target(target: str):
if target == "wt-cmd":
subprocess.run(["wt.exe", "new-tab", "cmd.exe"], check=True)
time.sleep(2.5)
return focus_terminal_window(target)
if target == "wt-powershell":
subprocess.run(["wt.exe", "new-tab", "powershell.exe"], check=True)
time.sleep(2.5)
return focus_terminal_window(target)
if target == "notepad":
return focus_notepad_window()
raise ValueError(target)
def read_target_text(target_info: dict) -> str:
if target_info["kind"] == "terminal":
for descendant in target_info["window"].descendants():
if descendant.class_name() == "TermControl":
return descendant.window_text()
return ""
target_info["doc"].set_focus()
time.sleep(0.2)
keyboard.send_keys("^a^c")
time.sleep(0.3)
return get_clipboard_text() or ""
def cleanup_target(target_info: dict):
if target_info["kind"] == "notepad":
try:
target_info["app"].kill()
except Exception:
pass
try:
target_info["fixture"].unlink(missing_ok=True)
except Exception:
pass
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--exe-path", required=True)
parser.add_argument("--target", choices=["notepad", "wt-cmd", "wt-powershell"], required=True)
parser.add_argument("--phrase", default="openless terminal regression success")
parser.add_argument("--remote-debugging-port", type=int, default=9223)
parser.add_argument("--timeout-seconds", type=int, default=120)
args = parser.parse_args()
launch_ps = f"""
$env:OPENLESS_SHOW_MAIN_ON_START='1'
$env:WEBVIEW2_ADDITIONAL_BROWSER_ARGUMENTS='--remote-debugging-port={args.remote_debugging_port}'
$proc = Start-Process -FilePath '{args.exe_path}' -PassThru
$proc.Id
"""
app_process = subprocess.run(
["powershell", "-NoProfile", "-Command", launch_ps],
check=True,
capture_output=True,
text=True,
)
app_pid = int(app_process.stdout.strip().splitlines()[-1])
client = None
target_info = None
previous_settings = None
previous_clipboard = get_clipboard_text()
clipboard_sentinel = f"OPENLESS_OLD_CLIPBOARD_SENTINEL_{uuid.uuid4().hex}"
try:
time.sleep(5)
client = CdpClient(cdp_page_ws(args.remote_debugging_port))
previous_settings = configure_preferences(client)
history = client.invoke("list_history") or []
baseline_count = len(history)
target_info = start_target(args.target)
set_clipboard_text(clipboard_sentinel)
client.invoke("start_dictation")
time.sleep(1.0)
speak_phrase(args.phrase)
time.sleep(0.8)
client.invoke("stop_dictation")
latest = wait_for_history_growth(client, baseline_count, args.timeout_seconds)
target_text = read_target_text(target_info)
result = {
"target": args.target,
"phrase": args.phrase,
"historyFinalText": latest.get("finalText"),
"historyRawTranscript": latest.get("rawTranscript"),
"insertStatus": latest.get("insertStatus"),
"targetContainsFinalText": bool(latest.get("finalText") and latest["finalText"] in target_text),
"targetContainsClipboardSentinel": clipboard_sentinel in target_text,
"targetTextTail": target_text[-400:],
}
print(json.dumps(result, ensure_ascii=False, indent=2))
finally:
if client and previous_settings is not None:
try:
client.invoke("set_settings", {"prefs": previous_settings})
except Exception:
pass
if client:
client.close()
cleanup_target(target_info) if target_info else None
set_clipboard_text(previous_clipboard)
subprocess.run(
["powershell", "-NoProfile", "-Command", f"Stop-Process -Id {app_pid} -Force -ErrorAction SilentlyContinue"],
check=False,
)
if __name__ == "__main__":
main()
@@ -1,10 +1,11 @@
param(
[string]$ExePath = "",
[ValidateSet("notepad", "browser")]
[ValidateSet("notepad", "browser", "wt-cmd", "wt-powershell")]
[string]$Target = "notepad",
[string]$Phrase = "OpenLess Windows real regression",
[int]$TimeoutSeconds = 120,
[int]$VirtualKey = 0xA3,
[int]$VirtualKey = 0xA2,
[string]$InjectedTranscriptText = "",
[int]$ManualSpeechSeconds = 8,
[switch]$ManualSpeech,
[switch]$DebugHotkeyEvents
@@ -83,6 +84,14 @@ function Write-TextUtf8($Path, $Text) {
[System.IO.File]::WriteAllText($Path, $Text, [System.Text.UTF8Encoding]::new($false))
}
function Restore-ClipboardValue($Value) {
if ($null -eq $Value) {
cmd /c "echo off | clip" | Out-Null
return
}
Set-Clipboard -Value $Value
}
function Set-HoldHotkeyPreference($Path) {
$previous = Read-TextUtf8 $Path
if ([string]::IsNullOrWhiteSpace($previous)) {
@@ -94,9 +103,9 @@ function Set-HoldHotkeyPreference($Path) {
$prefs | Add-Member -NotePropertyName hotkey -NotePropertyValue ([pscustomobject]@{})
}
if ($null -eq $prefs.hotkey.PSObject.Properties["trigger"]) {
$prefs.hotkey | Add-Member -NotePropertyName trigger -NotePropertyValue "rightControl"
$prefs.hotkey | Add-Member -NotePropertyName trigger -NotePropertyValue "leftControl"
} else {
$prefs.hotkey.trigger = "rightControl"
$prefs.hotkey.trigger = "leftControl"
}
if ($null -eq $prefs.hotkey.PSObject.Properties["mode"]) {
$prefs.hotkey | Add-Member -NotePropertyName mode -NotePropertyValue "hold"
@@ -109,6 +118,11 @@ function Set-HoldHotkeyPreference($Path) {
if ($null -eq $prefs.showCapsule) { $prefs | Add-Member -NotePropertyName showCapsule -NotePropertyValue $true }
if ($null -eq $prefs.activeAsrProvider) { $prefs | Add-Member -NotePropertyName activeAsrProvider -NotePropertyValue "volcengine" }
if ($null -eq $prefs.activeLlmProvider) { $prefs | Add-Member -NotePropertyName activeLlmProvider -NotePropertyValue "ark" }
if ($null -eq $prefs.restoreClipboardAfterPaste) {
$prefs | Add-Member -NotePropertyName restoreClipboardAfterPaste -NotePropertyValue $true
} else {
$prefs.restoreClipboardAfterPaste = $true
}
Write-TextUtf8 $Path ($prefs | ConvertTo-Json -Depth 8)
return $previous
}
@@ -184,6 +198,23 @@ function Release-Hotkey {
Send-KeyEdge $VirtualKey $true $true
}
function Ensure-TargetFocused($TargetInfo) {
if ($null -eq $TargetInfo) {
return $false
}
if ($TargetInfo.TargetTitle) {
$wshell = New-Object -ComObject WScript.Shell
if ($wshell.AppActivate($TargetInfo.TargetTitle)) {
Start-Sleep -Milliseconds 500
return $true
}
}
if ($null -ne $TargetInfo.Process) {
return (Focus-Window $TargetInfo.Process)
}
return $false
}
function Focus-Window($Process) {
if ($null -eq $Process -or $Process.MainWindowHandle -eq 0) {
return $false
@@ -280,12 +311,77 @@ function Stop-BrowserProfileProcesses($ProfilePath) {
function Start-InputTarget($TargetName) {
$startedAt = Get-Date
if ($TargetName -eq "notepad") {
Start-Process notepad.exe | Out-Null
$process = Wait-ProcessWindow "notepad" $startedAt 15
if (-not (Focus-Window $process)) {
throw "Notepad window could not be focused."
$fixture = Join-Path $env:TEMP "openless-notepad-input-fixture.txt"
Write-TextUtf8 $fixture ""
$process = Start-Process notepad.exe -ArgumentList $fixture -PassThru
Start-Sleep -Seconds 2
$title = "openless-notepad-input-fixture.txt - Notepad"
$activateScript = @"
import sys, time, win32com.client
title = sys.argv[1]
shell = win32com.client.Dispatch('WScript.Shell')
deadline = time.time() + 10
while time.time() < deadline:
if shell.AppActivate(title):
print('activated')
raise SystemExit(0)
time.sleep(0.2)
raise SystemExit(1)
"@
$activatePath = Join-Path $env:TEMP "openless-activate-notepad.py"
Write-TextUtf8 $activatePath $activateScript
try {
python $activatePath $title | Out-Null
} finally {
Remove-Item -LiteralPath $activatePath -Force -ErrorAction SilentlyContinue
}
Start-Sleep -Milliseconds 800
return [pscustomobject]@{
Process = $process
FixturePath = $fixture
ProfilePath = $null
TargetTitle = $title
TargetPid = $process.Id
TargetKind = "notepad"
}
}
if ($TargetName -in @("wt-cmd", "wt-powershell")) {
$wt = Get-Command wt.exe -ErrorAction SilentlyContinue
if ($null -eq $wt) {
throw "wt.exe was not found."
}
$profile = if ($TargetName -eq "wt-cmd") { "cmd.exe" } else { "powershell.exe" }
Start-Process -FilePath $wt.Source -ArgumentList @("new-tab", $profile) | Out-Null
Start-Sleep -Seconds 2
$title = if ($TargetName -eq "wt-cmd") { "C:\WINDOWS\system32\cmd.exe" } else { "Windows PowerShell" }
$activateScript = @"
import sys, time, win32com.client
title = sys.argv[1]
shell = win32com.client.Dispatch('WScript.Shell')
deadline = time.time() + 10
while time.time() < deadline:
if shell.AppActivate(title):
print('activated')
raise SystemExit(0)
time.sleep(0.2)
raise SystemExit(1)
"@
$activatePath = Join-Path $env:TEMP "openless-activate-target.py"
Write-TextUtf8 $activatePath $activateScript
try {
python $activatePath $title | Out-Null
} finally {
Remove-Item -LiteralPath $activatePath -Force -ErrorAction SilentlyContinue
}
Start-Sleep -Milliseconds 800
return [pscustomobject]@{
Process = $null
FixturePath = $null
ProfilePath = $null
TargetTitle = $title
TargetKind = "terminal"
}
return [pscustomobject]@{ Process = $process; FixturePath = $null; ProfilePath = $null }
}
$browserPath = Resolve-BrowserPath
@@ -307,7 +403,100 @@ function Start-InputTarget($TargetName) {
throw "Browser window could not be focused."
}
Start-Sleep -Seconds 1
return [pscustomobject]@{ Process = $process; FixturePath = $fixture; ProfilePath = $profilePath }
return [pscustomobject]@{ Process = $process; FixturePath = $fixture; ProfilePath = $profilePath; TargetKind = "browser" }
}
function Read-TargetContent($TargetInfo, $TargetName) {
if ($TargetName -eq "notepad") {
$readbackScript = @"
import sys
from pywinauto import Desktop
pid = int(sys.argv[1])
title = sys.argv[2]
out = sys.argv[3]
windows = [w for w in Desktop(backend='uia').windows() if getattr(w, 'process_id', lambda: None)() == pid]
win = None
for candidate in windows:
if candidate.window_text() == title:
win = candidate
break
if win is None and windows:
win = windows[0]
if win is None:
raise SystemExit(2)
for descendant in win.descendants():
if descendant.class_name() == 'RichEditD2DPT':
descendant.set_focus()
descendant.type_keys('^a^c')
import win32clipboard, win32con
win32clipboard.OpenClipboard()
try:
value = win32clipboard.GetClipboardData(win32con.CF_UNICODETEXT)
finally:
win32clipboard.CloseClipboard()
open(out, 'w', encoding='utf-8').write(value)
raise SystemExit(0)
raise SystemExit(1)
"@
$readbackPath = Join-Path $env:TEMP "openless-notepad-readback.py"
$outputPath = Join-Path $env:TEMP "openless-notepad-readback.txt"
Write-TextUtf8 $readbackPath $readbackScript
try {
Remove-Item -LiteralPath $outputPath -Force -ErrorAction SilentlyContinue
python -X utf8 $readbackPath $TargetInfo.TargetPid $TargetInfo.TargetTitle $outputPath | Out-Null
Start-Sleep -Milliseconds 400
if (Test-Path $outputPath) {
return Get-Content -Raw -Encoding UTF8 $outputPath
}
return $null
} finally {
Remove-Item -LiteralPath $readbackPath -Force -ErrorAction SilentlyContinue
Remove-Item -LiteralPath $outputPath -Force -ErrorAction SilentlyContinue
}
}
if ($TargetName -eq "browser") {
Focus-Window $TargetInfo.Process | Out-Null
Start-Sleep -Milliseconds 400
Send-CtrlChord 0x41
Start-Sleep -Milliseconds 200
Send-CtrlChord 0x43
Start-Sleep -Milliseconds 400
return Get-Clipboard -Raw -ErrorAction SilentlyContinue
}
if ($TargetName -in @("wt-cmd", "wt-powershell")) {
$readbackScript = @"
import sys
from pywinauto import Desktop
title = sys.argv[1]
out = sys.argv[2]
win = Desktop(backend='uia').window(title=title)
for descendant in win.descendants():
if descendant.class_name() == 'TermControl':
open(out, 'w', encoding='utf-8').write(descendant.window_text())
raise SystemExit(0)
raise SystemExit(1)
"@
$readbackPath = Join-Path $env:TEMP "openless-terminal-readback.py"
$outputPath = Join-Path $env:TEMP "openless-terminal-readback.txt"
Write-TextUtf8 $readbackPath $readbackScript
try {
Remove-Item -LiteralPath $outputPath -Force -ErrorAction SilentlyContinue
python -X utf8 $readbackPath $TargetInfo.TargetTitle $outputPath | Out-Null
if (Test-Path $outputPath) {
return Get-Content -Raw -Encoding UTF8 $outputPath
}
return $null
} finally {
Remove-Item -LiteralPath $readbackPath -Force -ErrorAction SilentlyContinue
Remove-Item -LiteralPath $outputPath -Force -ErrorAction SilentlyContinue
}
}
return $null
}
function Send-CtrlChord($Vk) {
@@ -338,6 +527,14 @@ $historyPath = Join-Path $env:APPDATA "OpenLess\history.json"
$preferencesPath = Join-Path $env:APPDATA "OpenLess\preferences.json"
$baselineCount = Get-HistoryCount $historyPath
$previousPreferences = Set-HoldHotkeyPreference $preferencesPath
$previousClipboard = Get-Clipboard -Raw -ErrorAction SilentlyContinue
$clipboardSentinel = "OPENLESS_OLD_CLIPBOARD_SENTINEL_$(Get-Date -Format 'yyyyMMddHHmmssfff')"
Restore-ClipboardValue $clipboardSentinel
$debugTranscriptPath = $null
if (-not [string]::IsNullOrWhiteSpace($InjectedTranscriptText)) {
$debugTranscriptPath = Join-Path $env:TEMP "openless-debug-transcript.txt"
Write-TextUtf8 $debugTranscriptPath $InjectedTranscriptText
}
Get-Process openless -ErrorAction SilentlyContinue | Stop-Process -Force
Remove-Item -LiteralPath $logPath -Force -ErrorAction SilentlyContinue
@@ -348,12 +545,16 @@ $env:OPENLESS_ACCEPT_SYNTHETIC_HOTKEY_EVENTS = "1"
if ($DebugHotkeyEvents) {
$env:OPENLESS_DEBUG_HOTKEY_EVENTS = "1"
}
if ($debugTranscriptPath) {
$env:OPENLESS_DEBUG_TRANSCRIPT_FILE = $debugTranscriptPath
}
try {
$openless = Start-Process -FilePath $ExePath -WorkingDirectory (Split-Path $ExePath -Parent) -PassThru
} finally {
Remove-Item Env:OPENLESS_SHOW_MAIN_ON_START -ErrorAction SilentlyContinue
Remove-Item Env:OPENLESS_ACCEPT_SYNTHETIC_HOTKEY_EVENTS -ErrorAction SilentlyContinue
Remove-Item Env:OPENLESS_DEBUG_HOTKEY_EVENTS -ErrorAction SilentlyContinue
Remove-Item Env:OPENLESS_DEBUG_TRANSCRIPT_FILE -ErrorAction SilentlyContinue
}
$inputTarget = $null
@@ -364,9 +565,18 @@ try {
$inputTarget = Start-InputTarget $Target
Press-Hotkey
if (-not (Wait-LogPattern $logPath "\[hotkey\] Windows trigger pressed" 10)) {
throw "Windows low-level hook did not observe the right Control press."
$observedPress = $false
for ($attempt = 1; $attempt -le 3 -and -not $observedPress; $attempt++) {
Ensure-TargetFocused $inputTarget | Out-Null
Press-Hotkey
$observedPress = Wait-LogPattern $logPath "\[hotkey\] Windows trigger pressed" 4
if (-not $observedPress) {
Release-Hotkey
Start-Sleep -Milliseconds 500
}
}
if (-not $observedPress) {
throw "Windows low-level hook did not observe the synthetic Control press."
}
if (-not (Wait-LogPattern $logPath "\[coord\] session started" 30)) {
throw "OpenLess recording session did not start."
@@ -399,18 +609,15 @@ try {
throw "Expected Windows insertStatus pasteSent after guarded foreground restore, got '$($latest.insertStatus)'."
}
Focus-Window $inputTarget.Process | Out-Null
Start-Sleep -Milliseconds 400
Send-CtrlChord 0x41
Start-Sleep -Milliseconds 200
Send-CtrlChord 0x43
Start-Sleep -Milliseconds 400
$targetText = Get-Clipboard -Raw -ErrorAction SilentlyContinue
$targetText = Read-TargetContent $inputTarget $Target
if ([string]::IsNullOrWhiteSpace($targetText)) {
throw "$Target clipboard readback is empty after Ctrl+A/C."
throw "$Target readback is empty."
}
if (-not $targetText.Contains($latest.finalText)) {
if ($targetText.Contains($clipboardSentinel)) {
throw "$Target readback contains the pre-dictation clipboard sentinel instead of latest finalText."
}
throw "$Target readback does not contain latest finalText; insertion was not proven at the target caret."
}
@@ -422,7 +629,7 @@ try {
if ($null -ne $inputTarget) {
if ($inputTarget.ProfilePath) {
Stop-BrowserProfileProcesses $inputTarget.ProfilePath
} else {
} elseif ($null -ne $inputTarget.Process) {
Stop-Process -Id $inputTarget.Process.Id -Force -ErrorAction SilentlyContinue
}
if ($inputTarget.FixturePath) {
@@ -438,6 +645,10 @@ try {
} else {
Write-TextUtf8 $preferencesPath $previousPreferences
}
Restore-ClipboardValue $previousClipboard
if ($debugTranscriptPath) {
Remove-Item -LiteralPath $debugTranscriptPath -Force -ErrorAction SilentlyContinue
}
}
Write-Host "Real ASR + insertion fallback smoke ($Target) passed."
+111 -34
View File
@@ -516,9 +516,7 @@ fn qa_hotkey_supervisor_loop(inner: Arc<Inner>) {
Err(e) => {
attempts += 1;
if attempts <= 3 || attempts % 10 == 0 {
log::warn!(
"[coord] QA hotkey 第 {attempts} 次注册失败: {e}; 3s 后重试"
);
log::warn!("[coord] QA hotkey 第 {attempts} 次注册失败: {e}; 3s 后重试");
}
std::thread::sleep(std::time::Duration::from_secs(3));
}
@@ -756,6 +754,15 @@ async fn handle_window_hotkey_event(
#[cfg(target_os = "windows")]
{
if !window_hotkey_fallback_enabled() {
if event_type == "keydown" && !repeat {
log::info!(
"[window-hotkey] ignored because Windows lifecycle owner is the low-level hook"
);
}
return Ok(());
}
let trigger = inner.prefs.get().hotkey.trigger;
if !window_key_matches_trigger(trigger, &key, &code) {
return Ok(());
@@ -781,6 +788,10 @@ async fn handle_window_hotkey_event(
}
}
fn window_hotkey_fallback_enabled() -> bool {
crate::types::HotkeyCapability::current().explicit_fallback_available
}
#[cfg(any(target_os = "windows", test))]
fn window_key_matches_trigger(trigger: crate::types::HotkeyTrigger, key: &str, code: &str) -> bool {
use crate::types::HotkeyTrigger;
@@ -820,7 +831,9 @@ async fn begin_session(inner: &Arc<Inner>) -> Result<(), String> {
}
}
// 翻译模式标志重置;hotkey 监听器在 Shift down 时再 set true。
inner.translation_modifier_seen.store(false, Ordering::SeqCst);
inner
.translation_modifier_seen
.store(false, Ordering::SeqCst);
#[cfg(any(debug_assertions, test))]
if hotkey_injection_dry_run_enabled() {
@@ -1192,6 +1205,19 @@ async fn end_session(inner: &Arc<Inner>) -> Result<(), String> {
// ASR 返回空转写护栏(来自 PR #66):写一条 emptyTranscript 失败历史 + 错误胶囊,
// 与 main 上其它 error 路径保持一致(带 schedule_capsule_idle 让胶囊自动消失)。
let mut raw = raw;
#[cfg(any(debug_assertions, test))]
if raw.text.trim().is_empty() {
if let Some(debug_text) = debug_transcript_override_text() {
log::info!(
"[coord] using debug transcript override (chars={})",
debug_text.chars().count()
);
raw.text = debug_text;
}
}
if raw.text.trim().is_empty() {
let session = DictationSession {
id: Uuid::new_v4().to_string(),
@@ -1230,8 +1256,8 @@ async fn end_session(inner: &Arc<Inner>) -> Result<(), String> {
let working_languages = prefs.working_languages.clone();
let front_app = inner.state.lock().front_app.clone();
let translation_target = prefs.translation_target_language.trim().to_string();
let translation_active = inner.translation_modifier_seen.load(Ordering::SeqCst)
&& !translation_target.is_empty();
let translation_active =
inner.translation_modifier_seen.load(Ordering::SeqCst) && !translation_target.is_empty();
let (polished, polish_error) = if translation_active {
log::info!(
"[coord] translation mode → target=\u{300C}{}\u{300D} working={:?} front_app={:?}",
@@ -1411,6 +1437,18 @@ fn hotkey_injection_dry_run_enabled() -> bool {
std::env::var_os("OPENLESS_HOTKEY_INJECTION_DRY_RUN").is_some()
}
#[cfg(any(debug_assertions, test))]
fn debug_transcript_override_text() -> Option<String> {
let path = std::env::var_os("OPENLESS_DEBUG_TRANSCRIPT_FILE")?;
let text = std::fs::read_to_string(path).ok()?;
let trimmed = text.trim().to_string();
if trimmed.is_empty() {
None
} else {
Some(trimmed)
}
}
fn ensure_microphone_permission(inner: &Arc<Inner>) -> Result<(), String> {
use crate::permissions::{self, PermissionStatus};
@@ -1702,11 +1740,7 @@ async fn begin_qa_session(inner: &Arc<Inner>) -> Result<(), String> {
*last = Some(now);
}
if let Some(app) = inner_for_level.app.lock().clone() {
let _ = app.emit_to(
"qa",
"qa:level",
serde_json::json!({ "level": level }),
);
let _ = app.emit_to("qa", "qa:level", serde_json::json!({ "level": level }));
}
// 同步把电平推给底部胶囊,让 QA 录音也有跟主听写一致的可视反馈。
emit_capsule(
@@ -1778,11 +1812,7 @@ async fn end_qa_session(inner: &Arc<Inner>) -> Result<(), String> {
emit_capsule(inner, CapsuleState::Transcribing, 0.0, 0, None, None);
if let Some(app) = inner.app.lock().clone() {
let _ = app.emit_to(
"qa",
"qa:state",
serde_json::json!({ "kind": "loading" }),
);
let _ = app.emit_to("qa", "qa:state", serde_json::json!({ "kind": "loading" }));
}
if let Some(rec) = inner.qa_recorder.lock().take() {
@@ -1845,10 +1875,14 @@ async fn end_qa_session(inner: &Arc<Inner>) -> Result<(), String> {
}
};
inner.qa_state.lock().messages.push(crate::types::QaChatMessage {
role: "user".to_string(),
content: user_content,
});
inner
.qa_state
.lock()
.messages
.push(crate::types::QaChatMessage {
role: "user".to_string(),
content: user_content,
});
if let Some(app) = inner.app.lock().clone() {
let messages = inner.qa_state.lock().messages.clone();
@@ -1914,10 +1948,14 @@ async fn end_qa_session(inner: &Arc<Inner>) -> Result<(), String> {
return Ok(());
}
inner.qa_state.lock().messages.push(crate::types::QaChatMessage {
role: "assistant".to_string(),
content: answer.clone(),
});
inner
.qa_state
.lock()
.messages
.push(crate::types::QaChatMessage {
role: "assistant".to_string(),
content: answer.clone(),
});
if let Some(app) = inner.app.lock().clone() {
let messages = inner.qa_state.lock().messages.clone();
@@ -2195,6 +2233,14 @@ mod tests {
);
assert!(coordinator.inner.hotkey_trigger_held.load(Ordering::SeqCst));
}
#[test]
fn window_hotkey_fallback_is_disabled_when_no_explicit_fallback_is_advertised() {
assert_eq!(
window_hotkey_fallback_enabled(),
crate::types::HotkeyCapability::current().explicit_fallback_available
);
}
}
fn enabled_phrases(inner: &Arc<Inner>) -> Vec<String> {
@@ -2433,6 +2479,42 @@ fn show_capsule_window_no_activate() -> bool {
false
}
#[cfg(target_os = "windows")]
fn hide_capsule_window_if_present() {
use std::iter::once;
use windows::core::PCWSTR;
use windows::Win32::Foundation::HWND;
use windows::Win32::UI::WindowsAndMessaging::{
FindWindowW, SetWindowPos, ShowWindow, HWND_NOTOPMOST, SWP_HIDEWINDOW, SWP_NOACTIVATE,
SWP_NOMOVE, SWP_NOSIZE, SW_HIDE,
};
let title: Vec<u16> = "OpenLess Capsule".encode_utf16().chain(once(0)).collect();
let hwnd = match unsafe { FindWindowW(PCWSTR::null(), PCWSTR(title.as_ptr())) } {
Ok(hwnd) => hwnd,
Err(_) => return,
};
if hwnd == HWND::default() || hwnd.0.is_null() {
return;
}
let _ = unsafe { ShowWindow(hwnd, SW_HIDE) };
let _ = unsafe {
SetWindowPos(
hwnd,
HWND_NOTOPMOST,
0,
0,
0,
0,
SWP_NOMOVE | SWP_NOSIZE | SWP_NOACTIVATE | SWP_HIDEWINDOW,
)
};
}
#[cfg(not(target_os = "windows"))]
fn hide_capsule_window_if_present() {}
fn emit_capsule(
inner: &Arc<Inner>,
state: CapsuleState,
@@ -2454,17 +2536,11 @@ fn emit_capsule(
let show_capsule = inner.prefs.get().show_capsule;
if let Some(window) = app.get_webview_window("capsule") {
let visible = !matches!(state, CapsuleState::Idle);
let visible = matches!(
state,
CapsuleState::Recording | CapsuleState::Transcribing | CapsuleState::Polishing
);
maybe_position_capsule_bottom_center(inner, &window, payload.translation);
#[cfg(target_os = "windows")]
{
// The capsule is always-on-top on Windows. Once recording stops, it must
// stop intercepting clicks meant for the app underneath.
let accepts_cursor_events = matches!(state, CapsuleState::Recording);
if let Err(e) = window.set_ignore_cursor_events(!accepts_cursor_events) {
log::warn!("[capsule] set_ignore_cursor_events failed: {e}");
}
}
if show_capsule && visible {
if cfg!(target_os = "windows") {
if !show_capsule_window_no_activate() {
@@ -2478,6 +2554,7 @@ fn emit_capsule(
#[cfg(target_os = "macos")]
crate::restore_main_window_key_if_active(&app);
} else {
hide_capsule_window_if_present();
let _ = window.hide();
}
}