From ffdd64b8d7d2e7ffc12edb4e83c35cc45a90a844 Mon Sep 17 00:00:00 2001
From: hy <huangyuan@onecat.cn>
Date: Wed, 7 Jan 2026 18:53:26 +0800
Subject: [PATCH] =?UTF-8?q?feat(=E5=AD=98=E5=82=A8):=20=E4=BC=98=E5=8C=96?=
 =?UTF-8?q?=E5=85=A8=E6=96=87=E6=90=9C=E7=B4=A2=E9=85=8D=E7=BD=AE=E5=B9=B6?=
 =?UTF-8?q?=E7=AE=80=E5=8C=96=E6=96=87=E6=9C=AC=E5=A4=84=E7=90=86?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

将全文搜索配置改为使用ngram分词器，提高搜索准确性
移除冗余的文本分词处理，直接使用原始文本进行搜索
---
 src/main/services/storage/UnifiedStore.ts | 20 +++++++++++++++-----
 1 file changed, 15 insertions(+), 5 deletions(-)

diff --git a/src/main/services/storage/UnifiedStore.ts b/src/main/services/storage/UnifiedStore.ts
index 7991dd3f..c5f763f2 100644
--- a/src/main/services/storage/UnifiedStore.ts
+++ b/src/main/services/storage/UnifiedStore.ts
@@ -137,7 +137,13 @@ export class UnifiedStore {
         // },
         ftsIndexConfig: {
           column: 'tokenizedText',
-          options: { config: lancedb.Index.fts() }
+          options: {
+            config: lancedb.Index.fts({
+              baseTokenizer: 'ngram',
+              ngramMinLength: 2,
+              ngramMaxLength: 3
+            })
+          }
         }
       }
       // Add more table configurations here as needed
@@ -200,7 +206,7 @@ export class UnifiedStore {
     const data = vectors.map((vector, i) => ({
       vector: Array.from(vector),
       text: chunks[i].text,
-      tokenizedText: this.tokenize(chunks[i].text, true),
+      tokenizedText: chunks[i].text,
       id: chunks[i].id,
       filename: chunks[i].filename,
       createdAt: Date.now()
@@ -274,9 +280,13 @@ export class UnifiedStore {
     if (!tableNames.includes(this.TABLE_CHUNK)) return []
 
     const table = await this.db!.openTable(this.TABLE_CHUNK)
-    const q = this.tokenize(query, false)
-    console.log('tokenize query:', q)
-    const results = await table.search(q).limit(limit).toArray()
+    const results = await table
+      .query()
+      .fullTextSearch(query, {
+        columns: ['tokenizedText']
+      })
+      .limit(limit)
+      .toArray()
 
     return results.map((item) => ({
       ...item,