diff --git a/docs/2026-05-02-windows-terminal-clipboard-restore-investigation.md b/docs/2026-05-02-windows-terminal-clipboard-restore-investigation.md index aa813ebc..b785ec46 100644 --- a/docs/2026-05-02-windows-terminal-clipboard-restore-investigation.md +++ b/docs/2026-05-02-windows-terminal-clipboard-restore-investigation.md @@ -199,6 +199,71 @@ Residual caveat: - repeated re-runs in the same desktop session later hit intermittent startup/hook-install flakiness before the test reached insertion again - that flakiness affected test repeatability, but it does not invalidate the already captured successful full-lifecycle evidence above +## 5. Repeatable full-lifecycle regression after automation hardening + +After hardening the automation path, the full OpenLess lifecycle was run through a stable route: + +- launch OpenLess with WebView2 remote debugging enabled +- drive lifecycle by invoking Tauri commands from the main webview (`start_dictation` / `stop_dictation`) +- keep real focus-target capture and real insertion behavior +- use a debug-only transcript override only when ASR would otherwise be empty in this desktop environment +- read back target content directly from UIA controls instead of recycling clipboard-based readback + +Targets exercised: + +- `Windows Terminal` `cmd.exe` tab +- `Windows Terminal` `PowerShell` tab +- `Notepad` + +Representative results: + +```json +{ + "target": "wt-cmd", + "historyFinalText": "openless terminal regression success", + "insertStatus": "pasteSent", + "targetContainsFinalText": true, + "targetContainsClipboardSentinel": false +} +{ + "target": "wt-powershell", + "historyFinalText": "openless terminal regression success", + "insertStatus": "pasteSent", + "targetContainsFinalText": true, + "targetContainsClipboardSentinel": false +} +{ + "target": "notepad", + "historyFinalText": "openless terminal regression success", + "insertStatus": "pasteSent", + "targetContainsFinalText": true, + "targetContainsClipboardSentinel": false +} +``` + +Repeatability observed in the current session: + +- `wt-cmd`: multiple successful runs with final text visible at the terminal prompt +- `wt-powershell`: successful run with final text visible at the terminal prompt +- `notepad`: two consecutive successful runs after switching readback from clipboard-based copy to direct UIA text capture + +Updated interpretation: + +- the originally suspected “terminal paste always restores the old clipboard before paste lands” is **not** reproducible as a general rule in the current full-lifecycle automation +- once the automation path is stabilized, all three tested targets receive the intended final text while `insertStatus` remains `pasteSent` +- the clipboard timing race is still real in isolation for slow consumers, but the complete OpenLess lifecycle on this machine does not reproduce the stale-clipboard failure for: + - `wt-cmd` + - `wt-powershell` + - `notepad` + +Most likely current conclusion: + +- the user-reported bug depends on an additional condition not captured in the hardened automation path +- plausible candidates remain: + - a different terminal host/session state + - a different target application than the tested Windows Terminal tabs + - another timing-sensitive environment factor outside the core insertion code + ## Root cause Root cause: Windows `PasteSent` semantics were treated as if they implied paste completion. diff --git a/openless-all/app/scripts/windows-openless-lifecycle-e2e.py b/openless-all/app/scripts/windows-openless-lifecycle-e2e.py index 8f977878..a4565a95 100644 --- a/openless-all/app/scripts/windows-openless-lifecycle-e2e.py +++ b/openless-all/app/scripts/windows-openless-lifecycle-e2e.py @@ -86,23 +86,28 @@ class CdpClient: def cdp_page_ws(port: int) -> str: - import urllib.request - import urllib.error - deadline = time.time() + 20 last_targets = [] while time.time() < deadline: try: - with urllib.request.urlopen(f"http://127.0.0.1:{port}/json/list") as response: - targets = json.loads(response.read().decode("utf-8")) + response = subprocess.run( + [ + "powershell", + "-NoProfile", + "-Command", + f"(Invoke-WebRequest -UseBasicParsing http://127.0.0.1:{port}/json/list).Content", + ], + capture_output=True, + text=True, + check=True, + ) + targets = json.loads(response.stdout) last_targets = [target.get("url", "") for target in targets] for target in targets: url = target.get("url", "") if url.startswith("http://tauri.localhost") and "?window=" not in url: return target["webSocketDebuggerUrl"] - except urllib.error.URLError: - pass - except urllib.error.HTTPError: + except Exception: pass time.sleep(0.5) raise RuntimeError(f"Main Tauri page target was not found. last_targets={last_targets}") @@ -146,7 +151,16 @@ def configure_preferences(client: CdpClient) -> dict: def focus_terminal_window(target: str): title = "C:\\WINDOWS\\system32\\cmd.exe" if target == "wt-cmd" else "Windows PowerShell" - win = Desktop(backend="uia").window(title=title) + win = None + for candidate in Desktop(backend="uia").windows(): + try: + if candidate.class_name() == "CASCADIA_HOSTING_WINDOW_CLASS" and candidate.window_text() == title: + win = candidate + break + except Exception: + continue + if win is None: + raise RuntimeError(f"terminal window not found for title={title}") win.set_focus() time.sleep(0.5) keyboard.send_keys("{ESC}") @@ -187,11 +201,7 @@ def read_target_text(target_info: dict) -> str: if descendant.class_name() == "TermControl": return descendant.window_text() return "" - target_info["doc"].set_focus() - time.sleep(0.2) - keyboard.send_keys("^a^c") - time.sleep(0.3) - return get_clipboard_text() or "" + return target_info["doc"].window_text() def cleanup_target(target_info: dict): @@ -211,13 +221,20 @@ def main(): parser.add_argument("--exe-path", required=True) parser.add_argument("--target", choices=["notepad", "wt-cmd", "wt-powershell"], required=True) parser.add_argument("--phrase", default="openless terminal regression success") + parser.add_argument("--injected-transcript-text", default="") parser.add_argument("--remote-debugging-port", type=int, default=9223) parser.add_argument("--timeout-seconds", type=int, default=120) args = parser.parse_args() + debug_transcript_path = "" + if args.injected_transcript_text.strip(): + debug_transcript_path = str(Path(tempfile.gettempdir()) / "openless-debug-transcript-e2e.txt") + Path(debug_transcript_path).write_text(args.injected_transcript_text, encoding="utf-8") + launch_ps = f""" $env:OPENLESS_SHOW_MAIN_ON_START='1' $env:WEBVIEW2_ADDITIONAL_BROWSER_ARGUMENTS='--remote-debugging-port={args.remote_debugging_port}' +$env:OPENLESS_DEBUG_TRANSCRIPT_FILE='{debug_transcript_path}' $proc = Start-Process -FilePath '{args.exe_path}' -PassThru $proc.Id """ @@ -246,8 +263,11 @@ $proc.Id client.invoke("start_dictation") time.sleep(1.0) - speak_phrase(args.phrase) - time.sleep(0.8) + if args.injected_transcript_text.strip(): + time.sleep(1.0) + else: + speak_phrase(args.phrase) + time.sleep(0.8) client.invoke("stop_dictation") latest = wait_for_history_growth(client, baseline_count, args.timeout_seconds) @@ -278,6 +298,11 @@ $proc.Id ["powershell", "-NoProfile", "-Command", f"Stop-Process -Id {app_pid} -Force -ErrorAction SilentlyContinue"], check=False, ) + if debug_transcript_path: + try: + Path(debug_transcript_path).unlink(missing_ok=True) + except Exception: + pass if __name__ == "__main__": diff --git a/openless-all/app/scripts/windows-real-asr-insertion-smoke.ps1 b/openless-all/app/scripts/windows-real-asr-insertion-smoke.ps1 index 3769ced8..b1809151 100644 --- a/openless-all/app/scripts/windows-real-asr-insertion-smoke.ps1 +++ b/openless-all/app/scripts/windows-real-asr-insertion-smoke.ps1 @@ -374,12 +374,31 @@ raise SystemExit(1) } finally { Remove-Item -LiteralPath $activatePath -Force -ErrorAction SilentlyContinue } + $handleLookup = @" +import sys +from pywinauto import Desktop + +title = sys.argv[1] +for window in Desktop(backend='uia').windows(): + if window.class_name() == 'CASCADIA_HOSTING_WINDOW_CLASS' and window.window_text() == title: + print(window.handle) + raise SystemExit(0) +raise SystemExit(1) +"@ + $handlePath = Join-Path $env:TEMP "openless-terminal-handle.py" + Write-TextUtf8 $handlePath $handleLookup + try { + $targetHandle = [int](python -X utf8 $handlePath $title) + } finally { + Remove-Item -LiteralPath $handlePath -Force -ErrorAction SilentlyContinue + } Start-Sleep -Milliseconds 800 return [pscustomobject]@{ Process = $null FixturePath = $null ProfilePath = $null TargetTitle = $title + TargetHandle = $targetHandle TargetKind = "terminal" } } @@ -427,14 +446,7 @@ if win is None: raise SystemExit(2) for descendant in win.descendants(): if descendant.class_name() == 'RichEditD2DPT': - descendant.set_focus() - descendant.type_keys('^a^c') - import win32clipboard, win32con - win32clipboard.OpenClipboard() - try: - value = win32clipboard.GetClipboardData(win32con.CF_UNICODETEXT) - finally: - win32clipboard.CloseClipboard() + value = descendant.window_text() open(out, 'w', encoding='utf-8').write(value) raise SystemExit(0) raise SystemExit(1) @@ -471,9 +483,9 @@ raise SystemExit(1) import sys from pywinauto import Desktop -title = sys.argv[1] +handle = int(sys.argv[1]) out = sys.argv[2] -win = Desktop(backend='uia').window(title=title) +win = Desktop(backend='uia').window(handle=handle) for descendant in win.descendants(): if descendant.class_name() == 'TermControl': open(out, 'w', encoding='utf-8').write(descendant.window_text()) @@ -485,7 +497,7 @@ raise SystemExit(1) Write-TextUtf8 $readbackPath $readbackScript try { Remove-Item -LiteralPath $outputPath -Force -ErrorAction SilentlyContinue - python -X utf8 $readbackPath $TargetInfo.TargetTitle $outputPath | Out-Null + python -X utf8 $readbackPath $TargetInfo.TargetHandle $outputPath | Out-Null if (Test-Path $outputPath) { return Get-Content -Raw -Encoding UTF8 $outputPath }