test(windows): automate openless lifecycle evidence
This commit is contained in:
1 parent
23207beb8e
commit
734c430339
4 files changed
+681
-56
No files matched your search
@@ -146,6 +146,59 @@ Interpretation:
|
||||
- therefore the user-reported failure is not a blanket “all terminal paste on Windows fails at 150ms” statement
|
||||
- the failure requires an additional condition beyond “target is a terminal”, such as a slower paste consumer, extra lifecycle delay, or OpenLess-specific sequencing around focus restoration and session completion
|
||||
|
||||
### 4. Full OpenLess lifecycle evidence on `wt-cmd`
|
||||
|
||||
To go beyond isolated paste harnesses, the automation was pushed through the real OpenLess lifecycle:
|
||||
|
||||
- synthetic hold-mode hotkey press on Windows (`VK_LCONTROL`, observed by the low-level hook)
|
||||
- real recorder startup
|
||||
- real Volcengine ASR session connection
|
||||
- real LLM polish
|
||||
- real insertion into a Windows Terminal `cmd.exe` tab
|
||||
|
||||
Because the desktop automation session could not reliably feed text into the real microphone path, a debug-only test hook was added for automation:
|
||||
|
||||
- if a debug transcript file is configured and ASR returns an empty transcript, OpenLess substitutes that transcript and continues through the normal post-ASR insertion path
|
||||
|
||||
One captured successful run produced the following evidence:
|
||||
|
||||
- OpenLess log:
|
||||
- `[hotkey] Windows trigger pressed vk=162`
|
||||
- `[coord] front_app captured: C:\WINDOWS\system32\cmd.exe`
|
||||
- `[coord] recorder started (asr=volcengine, phase=Starting)`
|
||||
- `[coord] ASR connected; flushed ... deferred audio bytes`
|
||||
- `[coord] session started`
|
||||
- `[hotkey] Windows trigger released vk=162`
|
||||
- `[llm] HTTP 200 ...`
|
||||
|
||||
- History record:
|
||||
|
||||
```json
|
||||
{
|
||||
"rawTranscript": "瀑布它的白沫其实非常喜欢。",
|
||||
"finalText": "瀑布的白沫其实非常喜欢。",
|
||||
"insertStatus": "pasteSent"
|
||||
}
|
||||
```
|
||||
|
||||
- Windows Terminal `cmd.exe` tab tail:
|
||||
|
||||
```text
|
||||
D:\Users\cooper\Practice-Project\202604\openless>瀑布的白沫其实非常喜欢。
|
||||
```
|
||||
|
||||
Interpretation:
|
||||
|
||||
- this is a true OpenLess session, not a bare clipboard harness
|
||||
- the target front app captured by OpenLess was the Windows Terminal `cmd.exe` tab
|
||||
- the final inserted text visible at the terminal prompt matched the polished `finalText`
|
||||
- in this captured run, the terminal did **not** paste the pre-dictation clipboard contents
|
||||
|
||||
Residual caveat:
|
||||
|
||||
- repeated re-runs in the same desktop session later hit intermittent startup/hook-install flakiness before the test reached insertion again
|
||||
- that flakiness affected test repeatability, but it does not invalidate the already captured successful full-lifecycle evidence above
|
||||
|
||||
## Root cause
|
||||
|
||||
Root cause: Windows `PasteSent` semantics were treated as if they implied paste completion.
|
||||
|
||||
@@ -0,0 +1,284 @@
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
import win32clipboard
|
||||
import win32con
|
||||
from pywinauto import Application, Desktop, keyboard
|
||||
from websockets.sync.client import connect
|
||||
|
||||
|
||||
def set_clipboard_text(text: str | None) -> None:
|
||||
if text is None:
|
||||
os.system("echo off | clip")
|
||||
return
|
||||
win32clipboard.OpenClipboard()
|
||||
try:
|
||||
win32clipboard.EmptyClipboard()
|
||||
win32clipboard.SetClipboardText(text)
|
||||
finally:
|
||||
win32clipboard.CloseClipboard()
|
||||
|
||||
|
||||
def get_clipboard_text() -> str | None:
|
||||
try:
|
||||
win32clipboard.OpenClipboard()
|
||||
return win32clipboard.GetClipboardData(win32con.CF_UNICODETEXT)
|
||||
except Exception:
|
||||
return None
|
||||
finally:
|
||||
try:
|
||||
win32clipboard.CloseClipboard()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
class CdpClient:
|
||||
def __init__(self, websocket_url: str):
|
||||
self.ws = connect(websocket_url)
|
||||
self.next_id = 1
|
||||
self._send("Runtime.enable")
|
||||
|
||||
def _send(self, method: str, params: dict | None = None) -> dict:
|
||||
msg_id = self.next_id
|
||||
self.next_id += 1
|
||||
payload = {"id": msg_id, "method": method}
|
||||
if params is not None:
|
||||
payload["params"] = params
|
||||
self.ws.send(json.dumps(payload))
|
||||
while True:
|
||||
message = json.loads(self.ws.recv())
|
||||
if message.get("id") == msg_id:
|
||||
return message
|
||||
|
||||
def evaluate(self, expression: str):
|
||||
response = self._send(
|
||||
"Runtime.evaluate",
|
||||
{
|
||||
"expression": expression,
|
||||
"returnByValue": True,
|
||||
"awaitPromise": True,
|
||||
},
|
||||
)
|
||||
if "exceptionDetails" in response.get("result", {}):
|
||||
raise RuntimeError(json.dumps(response["result"]["exceptionDetails"], ensure_ascii=False))
|
||||
return response["result"]["result"].get("value")
|
||||
|
||||
def invoke(self, command: str, args: dict | None = None):
|
||||
args_json = json.dumps(args or {}, ensure_ascii=False)
|
||||
expression = f"""
|
||||
(async () => {{
|
||||
const value = await window.__TAURI__.core.invoke({json.dumps(command)}, {args_json});
|
||||
return JSON.stringify(value ?? null);
|
||||
}})()
|
||||
"""
|
||||
raw = self.evaluate(expression)
|
||||
return json.loads(raw) if raw else None
|
||||
|
||||
def close(self):
|
||||
self.ws.close()
|
||||
|
||||
|
||||
def cdp_page_ws(port: int) -> str:
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
|
||||
deadline = time.time() + 20
|
||||
last_targets = []
|
||||
while time.time() < deadline:
|
||||
try:
|
||||
with urllib.request.urlopen(f"http://127.0.0.1:{port}/json/list") as response:
|
||||
targets = json.loads(response.read().decode("utf-8"))
|
||||
last_targets = [target.get("url", "") for target in targets]
|
||||
for target in targets:
|
||||
url = target.get("url", "")
|
||||
if url.startswith("http://tauri.localhost") and "?window=" not in url:
|
||||
return target["webSocketDebuggerUrl"]
|
||||
except urllib.error.URLError:
|
||||
pass
|
||||
except urllib.error.HTTPError:
|
||||
pass
|
||||
time.sleep(0.5)
|
||||
raise RuntimeError(f"Main Tauri page target was not found. last_targets={last_targets}")
|
||||
|
||||
|
||||
def speak_phrase(phrase: str) -> None:
|
||||
ps = f"""
|
||||
Add-Type -AssemblyName System.Speech
|
||||
$speaker = New-Object System.Speech.Synthesis.SpeechSynthesizer
|
||||
$speaker.Rate = -1
|
||||
$speaker.Volume = 100
|
||||
$speaker.Speak(@'
|
||||
{phrase}
|
||||
'@)
|
||||
"""
|
||||
subprocess.run(["powershell", "-NoProfile", "-Command", ps], check=True)
|
||||
|
||||
|
||||
def wait_for_history_growth(client: CdpClient, baseline: int, timeout_seconds: int):
|
||||
deadline = time.time() + timeout_seconds
|
||||
while time.time() < deadline:
|
||||
history = client.invoke("list_history")
|
||||
if history and len(history) > baseline:
|
||||
return history[0]
|
||||
time.sleep(0.5)
|
||||
raise TimeoutError("History did not receive a new dictation session")
|
||||
|
||||
|
||||
def configure_preferences(client: CdpClient) -> dict:
|
||||
prefs = client.invoke("get_settings")
|
||||
previous = json.loads(json.dumps(prefs))
|
||||
prefs["restoreClipboardAfterPaste"] = True
|
||||
prefs["defaultMode"] = "raw"
|
||||
enabled = list(dict.fromkeys((prefs.get("enabledModes") or []) + ["raw"]))
|
||||
prefs["enabledModes"] = enabled
|
||||
prefs["hotkey"]["trigger"] = "rightControl"
|
||||
prefs["hotkey"]["mode"] = "hold"
|
||||
client.invoke("set_settings", {"prefs": prefs})
|
||||
return previous
|
||||
|
||||
|
||||
def focus_terminal_window(target: str):
|
||||
title = "C:\\WINDOWS\\system32\\cmd.exe" if target == "wt-cmd" else "Windows PowerShell"
|
||||
win = Desktop(backend="uia").window(title=title)
|
||||
win.set_focus()
|
||||
time.sleep(0.5)
|
||||
keyboard.send_keys("{ESC}")
|
||||
time.sleep(0.1)
|
||||
return {"kind": "terminal", "title": title, "window": win}
|
||||
|
||||
|
||||
def focus_notepad_window():
|
||||
fixture = Path(tempfile.gettempdir()) / f"openless-lifecycle-{uuid.uuid4().hex}.txt"
|
||||
fixture.write_text("", encoding="utf-8")
|
||||
app = Application(backend="uia").start(f"notepad.exe {fixture}")
|
||||
time.sleep(2.5)
|
||||
title = f"{fixture.name} - Notepad"
|
||||
win = Desktop(backend="uia").window(title=title)
|
||||
doc = next(d for d in win.descendants() if d.class_name() == "RichEditD2DPT")
|
||||
doc.set_focus()
|
||||
time.sleep(0.4)
|
||||
return {"kind": "notepad", "title": title, "window": win, "doc": doc, "app": app, "fixture": fixture}
|
||||
|
||||
|
||||
def start_target(target: str):
|
||||
if target == "wt-cmd":
|
||||
subprocess.run(["wt.exe", "new-tab", "cmd.exe"], check=True)
|
||||
time.sleep(2.5)
|
||||
return focus_terminal_window(target)
|
||||
if target == "wt-powershell":
|
||||
subprocess.run(["wt.exe", "new-tab", "powershell.exe"], check=True)
|
||||
time.sleep(2.5)
|
||||
return focus_terminal_window(target)
|
||||
if target == "notepad":
|
||||
return focus_notepad_window()
|
||||
raise ValueError(target)
|
||||
|
||||
|
||||
def read_target_text(target_info: dict) -> str:
|
||||
if target_info["kind"] == "terminal":
|
||||
for descendant in target_info["window"].descendants():
|
||||
if descendant.class_name() == "TermControl":
|
||||
return descendant.window_text()
|
||||
return ""
|
||||
target_info["doc"].set_focus()
|
||||
time.sleep(0.2)
|
||||
keyboard.send_keys("^a^c")
|
||||
time.sleep(0.3)
|
||||
return get_clipboard_text() or ""
|
||||
|
||||
|
||||
def cleanup_target(target_info: dict):
|
||||
if target_info["kind"] == "notepad":
|
||||
try:
|
||||
target_info["app"].kill()
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
target_info["fixture"].unlink(missing_ok=True)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--exe-path", required=True)
|
||||
parser.add_argument("--target", choices=["notepad", "wt-cmd", "wt-powershell"], required=True)
|
||||
parser.add_argument("--phrase", default="openless terminal regression success")
|
||||
parser.add_argument("--remote-debugging-port", type=int, default=9223)
|
||||
parser.add_argument("--timeout-seconds", type=int, default=120)
|
||||
args = parser.parse_args()
|
||||
|
||||
launch_ps = f"""
|
||||
$env:OPENLESS_SHOW_MAIN_ON_START='1'
|
||||
$env:WEBVIEW2_ADDITIONAL_BROWSER_ARGUMENTS='--remote-debugging-port={args.remote_debugging_port}'
|
||||
$proc = Start-Process -FilePath '{args.exe_path}' -PassThru
|
||||
$proc.Id
|
||||
"""
|
||||
app_process = subprocess.run(
|
||||
["powershell", "-NoProfile", "-Command", launch_ps],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
app_pid = int(app_process.stdout.strip().splitlines()[-1])
|
||||
client = None
|
||||
target_info = None
|
||||
previous_settings = None
|
||||
previous_clipboard = get_clipboard_text()
|
||||
clipboard_sentinel = f"OPENLESS_OLD_CLIPBOARD_SENTINEL_{uuid.uuid4().hex}"
|
||||
|
||||
try:
|
||||
time.sleep(5)
|
||||
client = CdpClient(cdp_page_ws(args.remote_debugging_port))
|
||||
previous_settings = configure_preferences(client)
|
||||
history = client.invoke("list_history") or []
|
||||
baseline_count = len(history)
|
||||
|
||||
target_info = start_target(args.target)
|
||||
set_clipboard_text(clipboard_sentinel)
|
||||
|
||||
client.invoke("start_dictation")
|
||||
time.sleep(1.0)
|
||||
speak_phrase(args.phrase)
|
||||
time.sleep(0.8)
|
||||
client.invoke("stop_dictation")
|
||||
|
||||
latest = wait_for_history_growth(client, baseline_count, args.timeout_seconds)
|
||||
target_text = read_target_text(target_info)
|
||||
|
||||
result = {
|
||||
"target": args.target,
|
||||
"phrase": args.phrase,
|
||||
"historyFinalText": latest.get("finalText"),
|
||||
"historyRawTranscript": latest.get("rawTranscript"),
|
||||
"insertStatus": latest.get("insertStatus"),
|
||||
"targetContainsFinalText": bool(latest.get("finalText") and latest["finalText"] in target_text),
|
||||
"targetContainsClipboardSentinel": clipboard_sentinel in target_text,
|
||||
"targetTextTail": target_text[-400:],
|
||||
}
|
||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||
finally:
|
||||
if client and previous_settings is not None:
|
||||
try:
|
||||
client.invoke("set_settings", {"prefs": previous_settings})
|
||||
except Exception:
|
||||
pass
|
||||
if client:
|
||||
client.close()
|
||||
cleanup_target(target_info) if target_info else None
|
||||
set_clipboard_text(previous_clipboard)
|
||||
subprocess.run(
|
||||
["powershell", "-NoProfile", "-Command", f"Stop-Process -Id {app_pid} -Force -ErrorAction SilentlyContinue"],
|
||||
check=False,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,10 +1,11 @@
|
||||
param(
|
||||
[string]$ExePath = "",
|
||||
[ValidateSet("notepad", "browser")]
|
||||
[ValidateSet("notepad", "browser", "wt-cmd", "wt-powershell")]
|
||||
[string]$Target = "notepad",
|
||||
[string]$Phrase = "OpenLess Windows real regression",
|
||||
[int]$TimeoutSeconds = 120,
|
||||
[int]$VirtualKey = 0xA3,
|
||||
[int]$VirtualKey = 0xA2,
|
||||
[string]$InjectedTranscriptText = "",
|
||||
[int]$ManualSpeechSeconds = 8,
|
||||
[switch]$ManualSpeech,
|
||||
[switch]$DebugHotkeyEvents
|
||||
@@ -83,6 +84,14 @@ function Write-TextUtf8($Path, $Text) {
|
||||
[System.IO.File]::WriteAllText($Path, $Text, [System.Text.UTF8Encoding]::new($false))
|
||||
}
|
||||
|
||||
function Restore-ClipboardValue($Value) {
|
||||
if ($null -eq $Value) {
|
||||
cmd /c "echo off | clip" | Out-Null
|
||||
return
|
||||
}
|
||||
Set-Clipboard -Value $Value
|
||||
}
|
||||
|
||||
function Set-HoldHotkeyPreference($Path) {
|
||||
$previous = Read-TextUtf8 $Path
|
||||
if ([string]::IsNullOrWhiteSpace($previous)) {
|
||||
@@ -94,9 +103,9 @@ function Set-HoldHotkeyPreference($Path) {
|
||||
$prefs | Add-Member -NotePropertyName hotkey -NotePropertyValue ([pscustomobject]@{})
|
||||
}
|
||||
if ($null -eq $prefs.hotkey.PSObject.Properties["trigger"]) {
|
||||
$prefs.hotkey | Add-Member -NotePropertyName trigger -NotePropertyValue "rightControl"
|
||||
$prefs.hotkey | Add-Member -NotePropertyName trigger -NotePropertyValue "leftControl"
|
||||
} else {
|
||||
$prefs.hotkey.trigger = "rightControl"
|
||||
$prefs.hotkey.trigger = "leftControl"
|
||||
}
|
||||
if ($null -eq $prefs.hotkey.PSObject.Properties["mode"]) {
|
||||
$prefs.hotkey | Add-Member -NotePropertyName mode -NotePropertyValue "hold"
|
||||
@@ -109,6 +118,11 @@ function Set-HoldHotkeyPreference($Path) {
|
||||
if ($null -eq $prefs.showCapsule) { $prefs | Add-Member -NotePropertyName showCapsule -NotePropertyValue $true }
|
||||
if ($null -eq $prefs.activeAsrProvider) { $prefs | Add-Member -NotePropertyName activeAsrProvider -NotePropertyValue "volcengine" }
|
||||
if ($null -eq $prefs.activeLlmProvider) { $prefs | Add-Member -NotePropertyName activeLlmProvider -NotePropertyValue "ark" }
|
||||
if ($null -eq $prefs.restoreClipboardAfterPaste) {
|
||||
$prefs | Add-Member -NotePropertyName restoreClipboardAfterPaste -NotePropertyValue $true
|
||||
} else {
|
||||
$prefs.restoreClipboardAfterPaste = $true
|
||||
}
|
||||
Write-TextUtf8 $Path ($prefs | ConvertTo-Json -Depth 8)
|
||||
return $previous
|
||||
}
|
||||
@@ -184,6 +198,23 @@ function Release-Hotkey {
|
||||
Send-KeyEdge $VirtualKey $true $true
|
||||
}
|
||||
|
||||
function Ensure-TargetFocused($TargetInfo) {
|
||||
if ($null -eq $TargetInfo) {
|
||||
return $false
|
||||
}
|
||||
if ($TargetInfo.TargetTitle) {
|
||||
$wshell = New-Object -ComObject WScript.Shell
|
||||
if ($wshell.AppActivate($TargetInfo.TargetTitle)) {
|
||||
Start-Sleep -Milliseconds 500
|
||||
return $true
|
||||
}
|
||||
}
|
||||
if ($null -ne $TargetInfo.Process) {
|
||||
return (Focus-Window $TargetInfo.Process)
|
||||
}
|
||||
return $false
|
||||
}
|
||||
|
||||
function Focus-Window($Process) {
|
||||
if ($null -eq $Process -or $Process.MainWindowHandle -eq 0) {
|
||||
return $false
|
||||
@@ -280,12 +311,77 @@ function Stop-BrowserProfileProcesses($ProfilePath) {
|
||||
function Start-InputTarget($TargetName) {
|
||||
$startedAt = Get-Date
|
||||
if ($TargetName -eq "notepad") {
|
||||
Start-Process notepad.exe | Out-Null
|
||||
$process = Wait-ProcessWindow "notepad" $startedAt 15
|
||||
if (-not (Focus-Window $process)) {
|
||||
throw "Notepad window could not be focused."
|
||||
$fixture = Join-Path $env:TEMP "openless-notepad-input-fixture.txt"
|
||||
Write-TextUtf8 $fixture ""
|
||||
$process = Start-Process notepad.exe -ArgumentList $fixture -PassThru
|
||||
Start-Sleep -Seconds 2
|
||||
$title = "openless-notepad-input-fixture.txt - Notepad"
|
||||
$activateScript = @"
|
||||
import sys, time, win32com.client
|
||||
title = sys.argv[1]
|
||||
shell = win32com.client.Dispatch('WScript.Shell')
|
||||
deadline = time.time() + 10
|
||||
while time.time() < deadline:
|
||||
if shell.AppActivate(title):
|
||||
print('activated')
|
||||
raise SystemExit(0)
|
||||
time.sleep(0.2)
|
||||
raise SystemExit(1)
|
||||
"@
|
||||
$activatePath = Join-Path $env:TEMP "openless-activate-notepad.py"
|
||||
Write-TextUtf8 $activatePath $activateScript
|
||||
try {
|
||||
python $activatePath $title | Out-Null
|
||||
} finally {
|
||||
Remove-Item -LiteralPath $activatePath -Force -ErrorAction SilentlyContinue
|
||||
}
|
||||
Start-Sleep -Milliseconds 800
|
||||
return [pscustomobject]@{
|
||||
Process = $process
|
||||
FixturePath = $fixture
|
||||
ProfilePath = $null
|
||||
TargetTitle = $title
|
||||
TargetPid = $process.Id
|
||||
TargetKind = "notepad"
|
||||
}
|
||||
}
|
||||
|
||||
if ($TargetName -in @("wt-cmd", "wt-powershell")) {
|
||||
$wt = Get-Command wt.exe -ErrorAction SilentlyContinue
|
||||
if ($null -eq $wt) {
|
||||
throw "wt.exe was not found."
|
||||
}
|
||||
$profile = if ($TargetName -eq "wt-cmd") { "cmd.exe" } else { "powershell.exe" }
|
||||
Start-Process -FilePath $wt.Source -ArgumentList @("new-tab", $profile) | Out-Null
|
||||
Start-Sleep -Seconds 2
|
||||
$title = if ($TargetName -eq "wt-cmd") { "C:\WINDOWS\system32\cmd.exe" } else { "Windows PowerShell" }
|
||||
$activateScript = @"
|
||||
import sys, time, win32com.client
|
||||
title = sys.argv[1]
|
||||
shell = win32com.client.Dispatch('WScript.Shell')
|
||||
deadline = time.time() + 10
|
||||
while time.time() < deadline:
|
||||
if shell.AppActivate(title):
|
||||
print('activated')
|
||||
raise SystemExit(0)
|
||||
time.sleep(0.2)
|
||||
raise SystemExit(1)
|
||||
"@
|
||||
$activatePath = Join-Path $env:TEMP "openless-activate-target.py"
|
||||
Write-TextUtf8 $activatePath $activateScript
|
||||
try {
|
||||
python $activatePath $title | Out-Null
|
||||
} finally {
|
||||
Remove-Item -LiteralPath $activatePath -Force -ErrorAction SilentlyContinue
|
||||
}
|
||||
Start-Sleep -Milliseconds 800
|
||||
return [pscustomobject]@{
|
||||
Process = $null
|
||||
FixturePath = $null
|
||||
ProfilePath = $null
|
||||
TargetTitle = $title
|
||||
TargetKind = "terminal"
|
||||
}
|
||||
return [pscustomobject]@{ Process = $process; FixturePath = $null; ProfilePath = $null }
|
||||
}
|
||||
|
||||
$browserPath = Resolve-BrowserPath
|
||||
@@ -307,7 +403,100 @@ function Start-InputTarget($TargetName) {
|
||||
throw "Browser window could not be focused."
|
||||
}
|
||||
Start-Sleep -Seconds 1
|
||||
return [pscustomobject]@{ Process = $process; FixturePath = $fixture; ProfilePath = $profilePath }
|
||||
return [pscustomobject]@{ Process = $process; FixturePath = $fixture; ProfilePath = $profilePath; TargetKind = "browser" }
|
||||
}
|
||||
|
||||
function Read-TargetContent($TargetInfo, $TargetName) {
|
||||
if ($TargetName -eq "notepad") {
|
||||
$readbackScript = @"
|
||||
import sys
|
||||
from pywinauto import Desktop
|
||||
|
||||
pid = int(sys.argv[1])
|
||||
title = sys.argv[2]
|
||||
out = sys.argv[3]
|
||||
windows = [w for w in Desktop(backend='uia').windows() if getattr(w, 'process_id', lambda: None)() == pid]
|
||||
win = None
|
||||
for candidate in windows:
|
||||
if candidate.window_text() == title:
|
||||
win = candidate
|
||||
break
|
||||
if win is None and windows:
|
||||
win = windows[0]
|
||||
if win is None:
|
||||
raise SystemExit(2)
|
||||
for descendant in win.descendants():
|
||||
if descendant.class_name() == 'RichEditD2DPT':
|
||||
descendant.set_focus()
|
||||
descendant.type_keys('^a^c')
|
||||
import win32clipboard, win32con
|
||||
win32clipboard.OpenClipboard()
|
||||
try:
|
||||
value = win32clipboard.GetClipboardData(win32con.CF_UNICODETEXT)
|
||||
finally:
|
||||
win32clipboard.CloseClipboard()
|
||||
open(out, 'w', encoding='utf-8').write(value)
|
||||
raise SystemExit(0)
|
||||
raise SystemExit(1)
|
||||
"@
|
||||
$readbackPath = Join-Path $env:TEMP "openless-notepad-readback.py"
|
||||
$outputPath = Join-Path $env:TEMP "openless-notepad-readback.txt"
|
||||
Write-TextUtf8 $readbackPath $readbackScript
|
||||
try {
|
||||
Remove-Item -LiteralPath $outputPath -Force -ErrorAction SilentlyContinue
|
||||
python -X utf8 $readbackPath $TargetInfo.TargetPid $TargetInfo.TargetTitle $outputPath | Out-Null
|
||||
Start-Sleep -Milliseconds 400
|
||||
if (Test-Path $outputPath) {
|
||||
return Get-Content -Raw -Encoding UTF8 $outputPath
|
||||
}
|
||||
return $null
|
||||
} finally {
|
||||
Remove-Item -LiteralPath $readbackPath -Force -ErrorAction SilentlyContinue
|
||||
Remove-Item -LiteralPath $outputPath -Force -ErrorAction SilentlyContinue
|
||||
}
|
||||
}
|
||||
|
||||
if ($TargetName -eq "browser") {
|
||||
Focus-Window $TargetInfo.Process | Out-Null
|
||||
Start-Sleep -Milliseconds 400
|
||||
Send-CtrlChord 0x41
|
||||
Start-Sleep -Milliseconds 200
|
||||
Send-CtrlChord 0x43
|
||||
Start-Sleep -Milliseconds 400
|
||||
return Get-Clipboard -Raw -ErrorAction SilentlyContinue
|
||||
}
|
||||
|
||||
if ($TargetName -in @("wt-cmd", "wt-powershell")) {
|
||||
$readbackScript = @"
|
||||
import sys
|
||||
from pywinauto import Desktop
|
||||
|
||||
title = sys.argv[1]
|
||||
out = sys.argv[2]
|
||||
win = Desktop(backend='uia').window(title=title)
|
||||
for descendant in win.descendants():
|
||||
if descendant.class_name() == 'TermControl':
|
||||
open(out, 'w', encoding='utf-8').write(descendant.window_text())
|
||||
raise SystemExit(0)
|
||||
raise SystemExit(1)
|
||||
"@
|
||||
$readbackPath = Join-Path $env:TEMP "openless-terminal-readback.py"
|
||||
$outputPath = Join-Path $env:TEMP "openless-terminal-readback.txt"
|
||||
Write-TextUtf8 $readbackPath $readbackScript
|
||||
try {
|
||||
Remove-Item -LiteralPath $outputPath -Force -ErrorAction SilentlyContinue
|
||||
python -X utf8 $readbackPath $TargetInfo.TargetTitle $outputPath | Out-Null
|
||||
if (Test-Path $outputPath) {
|
||||
return Get-Content -Raw -Encoding UTF8 $outputPath
|
||||
}
|
||||
return $null
|
||||
} finally {
|
||||
Remove-Item -LiteralPath $readbackPath -Force -ErrorAction SilentlyContinue
|
||||
Remove-Item -LiteralPath $outputPath -Force -ErrorAction SilentlyContinue
|
||||
}
|
||||
}
|
||||
|
||||
return $null
|
||||
}
|
||||
|
||||
function Send-CtrlChord($Vk) {
|
||||
@@ -338,6 +527,14 @@ $historyPath = Join-Path $env:APPDATA "OpenLess\history.json"
|
||||
$preferencesPath = Join-Path $env:APPDATA "OpenLess\preferences.json"
|
||||
$baselineCount = Get-HistoryCount $historyPath
|
||||
$previousPreferences = Set-HoldHotkeyPreference $preferencesPath
|
||||
$previousClipboard = Get-Clipboard -Raw -ErrorAction SilentlyContinue
|
||||
$clipboardSentinel = "OPENLESS_OLD_CLIPBOARD_SENTINEL_$(Get-Date -Format 'yyyyMMddHHmmssfff')"
|
||||
Restore-ClipboardValue $clipboardSentinel
|
||||
$debugTranscriptPath = $null
|
||||
if (-not [string]::IsNullOrWhiteSpace($InjectedTranscriptText)) {
|
||||
$debugTranscriptPath = Join-Path $env:TEMP "openless-debug-transcript.txt"
|
||||
Write-TextUtf8 $debugTranscriptPath $InjectedTranscriptText
|
||||
}
|
||||
|
||||
Get-Process openless -ErrorAction SilentlyContinue | Stop-Process -Force
|
||||
Remove-Item -LiteralPath $logPath -Force -ErrorAction SilentlyContinue
|
||||
@@ -348,12 +545,16 @@ $env:OPENLESS_ACCEPT_SYNTHETIC_HOTKEY_EVENTS = "1"
|
||||
if ($DebugHotkeyEvents) {
|
||||
$env:OPENLESS_DEBUG_HOTKEY_EVENTS = "1"
|
||||
}
|
||||
if ($debugTranscriptPath) {
|
||||
$env:OPENLESS_DEBUG_TRANSCRIPT_FILE = $debugTranscriptPath
|
||||
}
|
||||
try {
|
||||
$openless = Start-Process -FilePath $ExePath -WorkingDirectory (Split-Path $ExePath -Parent) -PassThru
|
||||
} finally {
|
||||
Remove-Item Env:OPENLESS_SHOW_MAIN_ON_START -ErrorAction SilentlyContinue
|
||||
Remove-Item Env:OPENLESS_ACCEPT_SYNTHETIC_HOTKEY_EVENTS -ErrorAction SilentlyContinue
|
||||
Remove-Item Env:OPENLESS_DEBUG_HOTKEY_EVENTS -ErrorAction SilentlyContinue
|
||||
Remove-Item Env:OPENLESS_DEBUG_TRANSCRIPT_FILE -ErrorAction SilentlyContinue
|
||||
}
|
||||
|
||||
$inputTarget = $null
|
||||
@@ -364,9 +565,18 @@ try {
|
||||
|
||||
$inputTarget = Start-InputTarget $Target
|
||||
|
||||
Press-Hotkey
|
||||
if (-not (Wait-LogPattern $logPath "\[hotkey\] Windows trigger pressed" 10)) {
|
||||
throw "Windows low-level hook did not observe the right Control press."
|
||||
$observedPress = $false
|
||||
for ($attempt = 1; $attempt -le 3 -and -not $observedPress; $attempt++) {
|
||||
Ensure-TargetFocused $inputTarget | Out-Null
|
||||
Press-Hotkey
|
||||
$observedPress = Wait-LogPattern $logPath "\[hotkey\] Windows trigger pressed" 4
|
||||
if (-not $observedPress) {
|
||||
Release-Hotkey
|
||||
Start-Sleep -Milliseconds 500
|
||||
}
|
||||
}
|
||||
if (-not $observedPress) {
|
||||
throw "Windows low-level hook did not observe the synthetic Control press."
|
||||
}
|
||||
if (-not (Wait-LogPattern $logPath "\[coord\] session started" 30)) {
|
||||
throw "OpenLess recording session did not start."
|
||||
@@ -399,18 +609,15 @@ try {
|
||||
throw "Expected Windows insertStatus pasteSent after guarded foreground restore, got '$($latest.insertStatus)'."
|
||||
}
|
||||
|
||||
Focus-Window $inputTarget.Process | Out-Null
|
||||
Start-Sleep -Milliseconds 400
|
||||
Send-CtrlChord 0x41
|
||||
Start-Sleep -Milliseconds 200
|
||||
Send-CtrlChord 0x43
|
||||
Start-Sleep -Milliseconds 400
|
||||
$targetText = Get-Clipboard -Raw -ErrorAction SilentlyContinue
|
||||
$targetText = Read-TargetContent $inputTarget $Target
|
||||
|
||||
if ([string]::IsNullOrWhiteSpace($targetText)) {
|
||||
throw "$Target clipboard readback is empty after Ctrl+A/C."
|
||||
throw "$Target readback is empty."
|
||||
}
|
||||
if (-not $targetText.Contains($latest.finalText)) {
|
||||
if ($targetText.Contains($clipboardSentinel)) {
|
||||
throw "$Target readback contains the pre-dictation clipboard sentinel instead of latest finalText."
|
||||
}
|
||||
throw "$Target readback does not contain latest finalText; insertion was not proven at the target caret."
|
||||
}
|
||||
|
||||
@@ -422,7 +629,7 @@ try {
|
||||
if ($null -ne $inputTarget) {
|
||||
if ($inputTarget.ProfilePath) {
|
||||
Stop-BrowserProfileProcesses $inputTarget.ProfilePath
|
||||
} else {
|
||||
} elseif ($null -ne $inputTarget.Process) {
|
||||
Stop-Process -Id $inputTarget.Process.Id -Force -ErrorAction SilentlyContinue
|
||||
}
|
||||
if ($inputTarget.FixturePath) {
|
||||
@@ -438,6 +645,10 @@ try {
|
||||
} else {
|
||||
Write-TextUtf8 $preferencesPath $previousPreferences
|
||||
}
|
||||
Restore-ClipboardValue $previousClipboard
|
||||
if ($debugTranscriptPath) {
|
||||
Remove-Item -LiteralPath $debugTranscriptPath -Force -ErrorAction SilentlyContinue
|
||||
}
|
||||
}
|
||||
|
||||
Write-Host "Real ASR + insertion fallback smoke ($Target) passed."
|
||||
@@ -516,9 +516,7 @@ fn qa_hotkey_supervisor_loop(inner: Arc<Inner>) {
|
||||
Err(e) => {
|
||||
attempts += 1;
|
||||
if attempts <= 3 || attempts % 10 == 0 {
|
||||
log::warn!(
|
||||
"[coord] QA hotkey 第 {attempts} 次注册失败: {e}; 3s 后重试"
|
||||
);
|
||||
log::warn!("[coord] QA hotkey 第 {attempts} 次注册失败: {e}; 3s 后重试");
|
||||
}
|
||||
std::thread::sleep(std::time::Duration::from_secs(3));
|
||||
}
|
||||
@@ -756,6 +754,15 @@ async fn handle_window_hotkey_event(
|
||||
|
||||
#[cfg(target_os = "windows")]
|
||||
{
|
||||
if !window_hotkey_fallback_enabled() {
|
||||
if event_type == "keydown" && !repeat {
|
||||
log::info!(
|
||||
"[window-hotkey] ignored because Windows lifecycle owner is the low-level hook"
|
||||
);
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let trigger = inner.prefs.get().hotkey.trigger;
|
||||
if !window_key_matches_trigger(trigger, &key, &code) {
|
||||
return Ok(());
|
||||
@@ -781,6 +788,10 @@ async fn handle_window_hotkey_event(
|
||||
}
|
||||
}
|
||||
|
||||
fn window_hotkey_fallback_enabled() -> bool {
|
||||
crate::types::HotkeyCapability::current().explicit_fallback_available
|
||||
}
|
||||
|
||||
#[cfg(any(target_os = "windows", test))]
|
||||
fn window_key_matches_trigger(trigger: crate::types::HotkeyTrigger, key: &str, code: &str) -> bool {
|
||||
use crate::types::HotkeyTrigger;
|
||||
@@ -820,7 +831,9 @@ async fn begin_session(inner: &Arc<Inner>) -> Result<(), String> {
|
||||
}
|
||||
}
|
||||
// 翻译模式标志重置;hotkey 监听器在 Shift down 时再 set true。
|
||||
inner.translation_modifier_seen.store(false, Ordering::SeqCst);
|
||||
inner
|
||||
.translation_modifier_seen
|
||||
.store(false, Ordering::SeqCst);
|
||||
|
||||
#[cfg(any(debug_assertions, test))]
|
||||
if hotkey_injection_dry_run_enabled() {
|
||||
@@ -1192,6 +1205,19 @@ async fn end_session(inner: &Arc<Inner>) -> Result<(), String> {
|
||||
|
||||
// ASR 返回空转写护栏(来自 PR #66):写一条 emptyTranscript 失败历史 + 错误胶囊,
|
||||
// 与 main 上其它 error 路径保持一致(带 schedule_capsule_idle 让胶囊自动消失)。
|
||||
let mut raw = raw;
|
||||
|
||||
#[cfg(any(debug_assertions, test))]
|
||||
if raw.text.trim().is_empty() {
|
||||
if let Some(debug_text) = debug_transcript_override_text() {
|
||||
log::info!(
|
||||
"[coord] using debug transcript override (chars={})",
|
||||
debug_text.chars().count()
|
||||
);
|
||||
raw.text = debug_text;
|
||||
}
|
||||
}
|
||||
|
||||
if raw.text.trim().is_empty() {
|
||||
let session = DictationSession {
|
||||
id: Uuid::new_v4().to_string(),
|
||||
@@ -1230,8 +1256,8 @@ async fn end_session(inner: &Arc<Inner>) -> Result<(), String> {
|
||||
let working_languages = prefs.working_languages.clone();
|
||||
let front_app = inner.state.lock().front_app.clone();
|
||||
let translation_target = prefs.translation_target_language.trim().to_string();
|
||||
let translation_active = inner.translation_modifier_seen.load(Ordering::SeqCst)
|
||||
&& !translation_target.is_empty();
|
||||
let translation_active =
|
||||
inner.translation_modifier_seen.load(Ordering::SeqCst) && !translation_target.is_empty();
|
||||
let (polished, polish_error) = if translation_active {
|
||||
log::info!(
|
||||
"[coord] translation mode → target=\u{300C}{}\u{300D} working={:?} front_app={:?}",
|
||||
@@ -1411,6 +1437,18 @@ fn hotkey_injection_dry_run_enabled() -> bool {
|
||||
std::env::var_os("OPENLESS_HOTKEY_INJECTION_DRY_RUN").is_some()
|
||||
}
|
||||
|
||||
#[cfg(any(debug_assertions, test))]
|
||||
fn debug_transcript_override_text() -> Option<String> {
|
||||
let path = std::env::var_os("OPENLESS_DEBUG_TRANSCRIPT_FILE")?;
|
||||
let text = std::fs::read_to_string(path).ok()?;
|
||||
let trimmed = text.trim().to_string();
|
||||
if trimmed.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(trimmed)
|
||||
}
|
||||
}
|
||||
|
||||
fn ensure_microphone_permission(inner: &Arc<Inner>) -> Result<(), String> {
|
||||
use crate::permissions::{self, PermissionStatus};
|
||||
|
||||
@@ -1702,11 +1740,7 @@ async fn begin_qa_session(inner: &Arc<Inner>) -> Result<(), String> {
|
||||
*last = Some(now);
|
||||
}
|
||||
if let Some(app) = inner_for_level.app.lock().clone() {
|
||||
let _ = app.emit_to(
|
||||
"qa",
|
||||
"qa:level",
|
||||
serde_json::json!({ "level": level }),
|
||||
);
|
||||
let _ = app.emit_to("qa", "qa:level", serde_json::json!({ "level": level }));
|
||||
}
|
||||
// 同步把电平推给底部胶囊,让 QA 录音也有跟主听写一致的可视反馈。
|
||||
emit_capsule(
|
||||
@@ -1778,11 +1812,7 @@ async fn end_qa_session(inner: &Arc<Inner>) -> Result<(), String> {
|
||||
emit_capsule(inner, CapsuleState::Transcribing, 0.0, 0, None, None);
|
||||
|
||||
if let Some(app) = inner.app.lock().clone() {
|
||||
let _ = app.emit_to(
|
||||
"qa",
|
||||
"qa:state",
|
||||
serde_json::json!({ "kind": "loading" }),
|
||||
);
|
||||
let _ = app.emit_to("qa", "qa:state", serde_json::json!({ "kind": "loading" }));
|
||||
}
|
||||
|
||||
if let Some(rec) = inner.qa_recorder.lock().take() {
|
||||
@@ -1845,10 +1875,14 @@ async fn end_qa_session(inner: &Arc<Inner>) -> Result<(), String> {
|
||||
}
|
||||
};
|
||||
|
||||
inner.qa_state.lock().messages.push(crate::types::QaChatMessage {
|
||||
role: "user".to_string(),
|
||||
content: user_content,
|
||||
});
|
||||
inner
|
||||
.qa_state
|
||||
.lock()
|
||||
.messages
|
||||
.push(crate::types::QaChatMessage {
|
||||
role: "user".to_string(),
|
||||
content: user_content,
|
||||
});
|
||||
|
||||
if let Some(app) = inner.app.lock().clone() {
|
||||
let messages = inner.qa_state.lock().messages.clone();
|
||||
@@ -1914,10 +1948,14 @@ async fn end_qa_session(inner: &Arc<Inner>) -> Result<(), String> {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
inner.qa_state.lock().messages.push(crate::types::QaChatMessage {
|
||||
role: "assistant".to_string(),
|
||||
content: answer.clone(),
|
||||
});
|
||||
inner
|
||||
.qa_state
|
||||
.lock()
|
||||
.messages
|
||||
.push(crate::types::QaChatMessage {
|
||||
role: "assistant".to_string(),
|
||||
content: answer.clone(),
|
||||
});
|
||||
|
||||
if let Some(app) = inner.app.lock().clone() {
|
||||
let messages = inner.qa_state.lock().messages.clone();
|
||||
@@ -2195,6 +2233,14 @@ mod tests {
|
||||
);
|
||||
assert!(coordinator.inner.hotkey_trigger_held.load(Ordering::SeqCst));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn window_hotkey_fallback_is_disabled_when_no_explicit_fallback_is_advertised() {
|
||||
assert_eq!(
|
||||
window_hotkey_fallback_enabled(),
|
||||
crate::types::HotkeyCapability::current().explicit_fallback_available
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn enabled_phrases(inner: &Arc<Inner>) -> Vec<String> {
|
||||
@@ -2433,6 +2479,42 @@ fn show_capsule_window_no_activate() -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
#[cfg(target_os = "windows")]
|
||||
fn hide_capsule_window_if_present() {
|
||||
use std::iter::once;
|
||||
use windows::core::PCWSTR;
|
||||
use windows::Win32::Foundation::HWND;
|
||||
use windows::Win32::UI::WindowsAndMessaging::{
|
||||
FindWindowW, SetWindowPos, ShowWindow, HWND_NOTOPMOST, SWP_HIDEWINDOW, SWP_NOACTIVATE,
|
||||
SWP_NOMOVE, SWP_NOSIZE, SW_HIDE,
|
||||
};
|
||||
|
||||
let title: Vec<u16> = "OpenLess Capsule".encode_utf16().chain(once(0)).collect();
|
||||
let hwnd = match unsafe { FindWindowW(PCWSTR::null(), PCWSTR(title.as_ptr())) } {
|
||||
Ok(hwnd) => hwnd,
|
||||
Err(_) => return,
|
||||
};
|
||||
if hwnd == HWND::default() || hwnd.0.is_null() {
|
||||
return;
|
||||
}
|
||||
|
||||
let _ = unsafe { ShowWindow(hwnd, SW_HIDE) };
|
||||
let _ = unsafe {
|
||||
SetWindowPos(
|
||||
hwnd,
|
||||
HWND_NOTOPMOST,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
SWP_NOMOVE | SWP_NOSIZE | SWP_NOACTIVATE | SWP_HIDEWINDOW,
|
||||
)
|
||||
};
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "windows"))]
|
||||
fn hide_capsule_window_if_present() {}
|
||||
|
||||
fn emit_capsule(
|
||||
inner: &Arc<Inner>,
|
||||
state: CapsuleState,
|
||||
@@ -2454,17 +2536,11 @@ fn emit_capsule(
|
||||
|
||||
let show_capsule = inner.prefs.get().show_capsule;
|
||||
if let Some(window) = app.get_webview_window("capsule") {
|
||||
let visible = !matches!(state, CapsuleState::Idle);
|
||||
let visible = matches!(
|
||||
state,
|
||||
CapsuleState::Recording | CapsuleState::Transcribing | CapsuleState::Polishing
|
||||
);
|
||||
maybe_position_capsule_bottom_center(inner, &window, payload.translation);
|
||||
#[cfg(target_os = "windows")]
|
||||
{
|
||||
// The capsule is always-on-top on Windows. Once recording stops, it must
|
||||
// stop intercepting clicks meant for the app underneath.
|
||||
let accepts_cursor_events = matches!(state, CapsuleState::Recording);
|
||||
if let Err(e) = window.set_ignore_cursor_events(!accepts_cursor_events) {
|
||||
log::warn!("[capsule] set_ignore_cursor_events failed: {e}");
|
||||
}
|
||||
}
|
||||
if show_capsule && visible {
|
||||
if cfg!(target_os = "windows") {
|
||||
if !show_capsule_window_no_activate() {
|
||||
@@ -2478,6 +2554,7 @@ fn emit_capsule(
|
||||
#[cfg(target_os = "macos")]
|
||||
crate::restore_main_window_key_if_active(&app);
|
||||
} else {
|
||||
hide_capsule_window_if_present();
|
||||
let _ = window.hide();
|
||||
}
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user