fix(chatgpt): read replies from AX tree instead of clipboard shortcut (#106)

package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@jackwener/opencli",
-  "version": "1.0.1",
+  "version": "1.0.2",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@jackwener/opencli",
-      "version": "1.0.1",
+      "version": "1.0.2",
       "hasInstallScript": true,
       "license": "Apache-2.0",
       "dependencies": {
@@ -1557,7 +1557,6 @@
       "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==",
       "dev": true,
       "license": "MIT",
-      "peer": true,
       "engines": {
         "node": ">=12"
       },
@@ -1753,7 +1752,6 @@
       "integrity": "sha512-5C1sg4USs1lfG0GFb2RLXsdpXqBSEhAaA/0kPL01wxzpMqLILNxIxIOKiILz+cdg/pLnOUxFYOR5yhHU666wbw==",
       "dev": true,
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "esbuild": "~0.27.0",
         "get-tsconfig": "^4.7.5"
@@ -1795,7 +1793,6 @@
       "integrity": "sha512-fPGaRNj9Zytaf8LEiBhY7Z6ijnFKdzU/+mL8EFBaKr7Vw1/FWcTBAMW0wLPJAGMPX38ZPVCVgLceWiEqeoqL2Q==",
       "dev": true,
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "@oxc-project/runtime": "0.115.0",
         "lightningcss": "^1.32.0",

src/clis/chatgpt/ask.ts

@@ -1,6 +1,7 @@
 import { execSync, spawnSync } from 'node:child_process';
 import { cli, Strategy } from '../../registry.js';
 import type { IPage } from '../../types.js';
+import { getVisibleChatMessages } from './ax.js';
 
 export const askCommand = cli({
   site: 'chatgpt',
@@ -21,6 +22,7 @@ export const askCommand = cli({
     // Backup clipboard
     let clipBackup = '';
     try { clipBackup = execSync('pbpaste', { encoding: 'utf-8' }); } catch {}
+    const messagesBefore = getVisibleChatMessages();
 
     // Send the message
     spawnSync('pbcopy', { input: text });
@@ -35,33 +37,30 @@ export const askCommand = cli({
                 "-e 'end tell'";
     execSync(cmd);
 
-    // Clear clipboard marker
-    spawnSync('pbcopy', { input: '__OPENCLI_WAITING__' });
+    // Restore clipboard after the prompt is sent.
+    if (clipBackup) spawnSync('pbcopy', { input: clipBackup });
 
-    // Wait for response, then read it
-    const pollInterval = 3;
+    // Wait for response, then read the latest visible assistant message from the AX tree.
+    const pollInterval = 1;
     const maxPolls = Math.ceil(timeout / pollInterval);
     let response = '';
 
     for (let i = 0; i < maxPolls; i++) {
-      // Wait
       execSync(`sleep ${pollInterval}`);
-
-      // Try Cmd+Shift+C to copy the latest response
       execSync("osascript -e 'tell application \"ChatGPT\" to activate'");
-      execSync("osascript -e 'tell application \"System Events\" to keystroke \"c\" using {command down, shift down}'");
-      execSync("osascript -e 'delay 0.3'");
+      execSync("osascript -e 'delay 0.2'");
 
-      const copied = execSync('pbpaste', { encoding: 'utf-8' }).trim();
-      if (copied && copied !== '__OPENCLI_WAITING__' && copied !== text) {
-        response = copied;
+      const messagesNow = getVisibleChatMessages();
+      if (messagesNow.length <= messagesBefore.length) continue;
+
+      const newMessages = messagesNow.slice(messagesBefore.length);
+      const candidate = [...newMessages].reverse().find((message) => message !== text);
+      if (candidate) {
+        response = candidate;
         break;
       }
     }
 
-    // Restore clipboard
-    if (clipBackup) spawnSync('pbcopy', { input: clipBackup });
-
     if (!response) {
       return [
         { Role: 'User', Text: text },

src/clis/chatgpt/ax.ts

@@ -0,0 +1,81 @@
+import { execFileSync } from 'node:child_process';
+
+const AX_READ_SCRIPT = `
+import Cocoa
+import ApplicationServices
+
+func attr(_ el: AXUIElement, _ name: String) -> AnyObject? {
+    var value: CFTypeRef?
+    guard AXUIElementCopyAttributeValue(el, name as CFString, &value) == .success else { return nil }
+    return value as AnyObject?
+}
+
+func s(_ el: AXUIElement, _ name: String) -> String? {
+    if let v = attr(el, name) as? String, !v.isEmpty { return v }
+    return nil
+}
+
+func children(_ el: AXUIElement) -> [AXUIElement] {
+    (attr(el, kAXChildrenAttribute as String) as? [AnyObject] ?? []).map { $0 as! AXUIElement }
+}
+
+func collectLists(_ el: AXUIElement, into out: inout [AXUIElement]) {
+    let role = s(el, kAXRoleAttribute as String) ?? ""
+    if role == kAXListRole as String { out.append(el) }
+    for c in children(el) { collectLists(c, into: &out) }
+}
+
+func collectTexts(_ el: AXUIElement, into out: inout [String]) {
+    let role = s(el, kAXRoleAttribute as String) ?? ""
+    if role == kAXStaticTextRole as String {
+        if let text = s(el, kAXDescriptionAttribute as String), !text.isEmpty {
+            out.append(text)
+        }
+    }
+    for c in children(el) { collectTexts(c, into: &out) }
+}
+
+guard let app = NSRunningApplication.runningApplications(withBundleIdentifier: "com.openai.chat").first else {
+    fputs("ChatGPT not running\\n", stderr)
+    exit(1)
+}
+
+let axApp = AXUIElementCreateApplication(app.processIdentifier)
+guard let win = attr(axApp, kAXFocusedWindowAttribute as String) as! AXUIElement? else {
+    fputs("No focused ChatGPT window\\n", stderr)
+    exit(1)
+}
+
+var lists: [AXUIElement] = []
+collectLists(win, into: &lists)
+
+var best: [String] = []
+for list in lists {
+    var texts: [String] = []
+    collectTexts(list, into: &texts)
+    if texts.count > best.count {
+        best = texts
+    }
+}
+
+let data = try! JSONSerialization.data(withJSONObject: best, options: [])
+print(String(data: data, encoding: .utf8)!)
+`;
+
+export function getVisibleChatMessages(): string[] {
+  const output = execFileSync('swift', ['-'], {
+    input: AX_READ_SCRIPT,
+    encoding: 'utf-8',
+    maxBuffer: 10 * 1024 * 1024,
+  }).trim();
+
+  if (!output) return [];
+
+  const parsed = JSON.parse(output);
+  if (!Array.isArray(parsed)) return [];
+
+  return parsed
+    .filter((item): item is string => typeof item === 'string')
+    .map((item) => item.replace(/[\uFFFC\u200B-\u200D\uFEFF]/g, '').trim())
+    .filter((item) => item.length > 0);
+}

src/clis/chatgpt/read.ts

@@ -1,6 +1,7 @@
 import { execSync } from 'node:child_process';
 import { cli, Strategy } from '../../registry.js';
 import type { IPage } from '../../types.js';
+import { getVisibleChatMessages } from './ax.js';
 
 export const readCommand = cli({
   site: 'chatgpt',
@@ -14,17 +15,14 @@ export const readCommand = cli({
   func: async (page: IPage | null) => {
     try {
       execSync("osascript -e 'tell application \"ChatGPT\" to activate'");
-      execSync("osascript -e 'delay 0.5'");
-      execSync("osascript -e 'tell application \"System Events\" to keystroke \"c\" using {command down, shift down}'");
       execSync("osascript -e 'delay 0.3'");
+      const messages = getVisibleChatMessages();
 
-      const result = execSync('pbpaste', { encoding: 'utf-8' }).trim();
-      
-      if (!result) {
-        return [{ Role: 'System', Text: 'No text was copied. Is there a response in the chat?' }];
+      if (!messages.length) {
+        return [{ Role: 'System', Text: 'No visible chat messages were found in the current ChatGPT window.' }];
       }
 
-      return [{ Role: 'Assistant', Text: result }];
+      return [{ Role: 'Assistant', Text: messages[messages.length - 1] }];
     } catch (err: any) {
       throw new Error("Failed to read from ChatGPT: " + err.message);
     }