253 lines
7.7 KiB
Python
253 lines
7.7 KiB
Python
import os
|
|
import sys
|
|
|
|
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
|
|
import json
|
|
from typing import List, Dict, Tuple
|
|
import numpy as np
|
|
from sklearn.feature_extraction.text import TfidfVectorizer
|
|
from sklearn.metrics.pairwise import cosine_similarity
|
|
import glob
|
|
import sqlite3
|
|
import re
|
|
from datetime import datetime
|
|
import random
|
|
from src.llm_models.utils_model import LLMRequest
|
|
from src.config.config import global_config
|
|
|
|
|
|
def clean_group_name(name: str) -> str:
|
|
"""清理群组名称，只保留中文和英文字符"""
|
|
cleaned = re.sub(r"[^\u4e00-\u9fa5a-zA-Z]", "", name)
|
|
if not cleaned:
|
|
cleaned = datetime.now().strftime("%Y%m%d")
|
|
return cleaned
|
|
|
|
|
|
def get_group_name(stream_id: str) -> str:
|
|
"""从数据库中获取群组名称"""
|
|
conn = sqlite3.connect("data/maibot.db")
|
|
cursor = conn.cursor()
|
|
|
|
cursor.execute(
|
|
"""
|
|
SELECT group_name, user_nickname, platform
|
|
FROM chat_streams
|
|
WHERE stream_id = ?
|
|
""",
|
|
(stream_id,),
|
|
)
|
|
|
|
result = cursor.fetchone()
|
|
conn.close()
|
|
|
|
if result:
|
|
group_name, user_nickname, platform = result
|
|
if group_name:
|
|
return clean_group_name(group_name)
|
|
if user_nickname:
|
|
return clean_group_name(user_nickname)
|
|
if platform:
|
|
return clean_group_name(f"{platform}{stream_id[:8]}")
|
|
return stream_id
|
|
|
|
|
|
def load_expressions(chat_id: str) -> List[Dict]:
|
|
"""加载指定群聊的表达方式"""
|
|
style_file = os.path.join("data", "expression", "learnt_style", str(chat_id), "expressions.json")
|
|
|
|
style_exprs = []
|
|
|
|
if os.path.exists(style_file):
|
|
with open(style_file, "r", encoding="utf-8") as f:
|
|
style_exprs = json.load(f)
|
|
|
|
# 如果表达方式超过10个，随机选择10个
|
|
if len(style_exprs) > 50:
|
|
style_exprs = random.sample(style_exprs, 50)
|
|
print(f"\n从 {len(style_exprs)} 个表达方式中随机选择了 10 个进行匹配")
|
|
|
|
return style_exprs
|
|
|
|
|
|
def find_similar_expressions_tfidf(
|
|
input_text: str, expressions: List[Dict], mode: str = "both", top_k: int = 10
|
|
) -> List[Tuple[str, str, float]]:
|
|
"""使用TF-IDF方法找出与输入文本最相似的top_k个表达方式"""
|
|
if not expressions:
|
|
return []
|
|
|
|
# 准备文本数据
|
|
if mode == "style":
|
|
texts = [expr["style"] for expr in expressions]
|
|
elif mode == "situation":
|
|
texts = [expr["situation"] for expr in expressions]
|
|
else: # both
|
|
texts = [f"{expr['situation']} {expr['style']}" for expr in expressions]
|
|
|
|
texts.append(input_text) # 添加输入文本
|
|
|
|
# 使用TF-IDF向量化
|
|
vectorizer = TfidfVectorizer()
|
|
tfidf_matrix = vectorizer.fit_transform(texts)
|
|
|
|
# 计算余弦相似度
|
|
similarity_matrix = cosine_similarity(tfidf_matrix)
|
|
|
|
# 获取输入文本的相似度分数（最后一行）
|
|
scores = similarity_matrix[-1][:-1] # 排除与自身的相似度
|
|
|
|
# 获取top_k的索引
|
|
top_indices = np.argsort(scores)[::-1][:top_k]
|
|
|
|
# 获取相似表达
|
|
similar_exprs = []
|
|
for idx in top_indices:
|
|
if scores[idx] > 0: # 只保留有相似度的
|
|
similar_exprs.append((expressions[idx]["style"], expressions[idx]["situation"], scores[idx]))
|
|
|
|
return similar_exprs
|
|
|
|
|
|
async def find_similar_expressions_embedding(
|
|
input_text: str, expressions: List[Dict], mode: str = "both", top_k: int = 5
|
|
) -> List[Tuple[str, str, float]]:
|
|
"""使用嵌入模型找出与输入文本最相似的top_k个表达方式"""
|
|
if not expressions:
|
|
return []
|
|
|
|
# 准备文本数据
|
|
if mode == "style":
|
|
texts = [expr["style"] for expr in expressions]
|
|
elif mode == "situation":
|
|
texts = [expr["situation"] for expr in expressions]
|
|
else: # both
|
|
texts = [f"{expr['situation']} {expr['style']}" for expr in expressions]
|
|
|
|
# 获取嵌入向量
|
|
llm_request = LLMRequest(global_config.model.embedding)
|
|
text_embeddings = []
|
|
for text in texts:
|
|
embedding = await llm_request.get_embedding(text)
|
|
if embedding:
|
|
text_embeddings.append(embedding)
|
|
|
|
input_embedding = await llm_request.get_embedding(input_text)
|
|
if not input_embedding or not text_embeddings:
|
|
return []
|
|
|
|
# 计算余弦相似度
|
|
text_embeddings = np.array(text_embeddings)
|
|
similarities = np.dot(text_embeddings, input_embedding) / (
|
|
np.linalg.norm(text_embeddings, axis=1) * np.linalg.norm(input_embedding)
|
|
)
|
|
|
|
# 获取top_k的索引
|
|
top_indices = np.argsort(similarities)[::-1][:top_k]
|
|
|
|
# 获取相似表达
|
|
similar_exprs = []
|
|
for idx in top_indices:
|
|
if similarities[idx] > 0: # 只保留有相似度的
|
|
similar_exprs.append((expressions[idx]["style"], expressions[idx]["situation"], similarities[idx]))
|
|
|
|
return similar_exprs
|
|
|
|
|
|
async def main():
|
|
# 获取所有群聊ID
|
|
style_dirs = glob.glob(os.path.join("data", "expression", "learnt_style", "*"))
|
|
chat_ids = [os.path.basename(d) for d in style_dirs]
|
|
|
|
if not chat_ids:
|
|
print("没有找到任何群聊的表达方式数据")
|
|
return
|
|
|
|
print("可用的群聊:")
|
|
for i, chat_id in enumerate(chat_ids, 1):
|
|
group_name = get_group_name(chat_id)
|
|
print(f"{i}. {group_name}")
|
|
|
|
while True:
|
|
try:
|
|
choice = int(input("\n请选择要分析的群聊编号 (输入0退出): "))
|
|
if choice == 0:
|
|
break
|
|
if 1 <= choice <= len(chat_ids):
|
|
chat_id = chat_ids[choice - 1]
|
|
break
|
|
print("无效的选择，请重试")
|
|
except ValueError:
|
|
print("请输入有效的数字")
|
|
|
|
if choice == 0:
|
|
return
|
|
|
|
# 加载表达方式
|
|
style_exprs = load_expressions(chat_id)
|
|
|
|
group_name = get_group_name(chat_id)
|
|
print(f"\n已选择群聊：{group_name}")
|
|
|
|
# 选择匹配模式
|
|
print("\n请选择匹配模式：")
|
|
print("1. 匹配表达方式")
|
|
print("2. 匹配情景")
|
|
print("3. 两者都考虑")
|
|
|
|
while True:
|
|
try:
|
|
mode_choice = int(input("\n请选择匹配模式 (1-3): "))
|
|
if 1 <= mode_choice <= 3:
|
|
break
|
|
print("无效的选择，请重试")
|
|
except ValueError:
|
|
print("请输入有效的数字")
|
|
|
|
mode_map = {1: "style", 2: "situation", 3: "both"}
|
|
mode = mode_map[mode_choice]
|
|
|
|
# 选择匹配方法
|
|
print("\n请选择匹配方法：")
|
|
print("1. TF-IDF方法")
|
|
print("2. 嵌入模型方法")
|
|
|
|
while True:
|
|
try:
|
|
method_choice = int(input("\n请选择匹配方法 (1-2): "))
|
|
if 1 <= method_choice <= 2:
|
|
break
|
|
print("无效的选择，请重试")
|
|
except ValueError:
|
|
print("请输入有效的数字")
|
|
|
|
while True:
|
|
input_text = input("\n请输入要匹配的文本（输入q退出）: ")
|
|
if input_text.lower() == "q":
|
|
break
|
|
|
|
if not input_text.strip():
|
|
continue
|
|
|
|
if method_choice == 1:
|
|
similar_exprs = find_similar_expressions_tfidf(input_text, style_exprs, mode)
|
|
else:
|
|
similar_exprs = await find_similar_expressions_embedding(input_text, style_exprs, mode)
|
|
|
|
if similar_exprs:
|
|
print("\n找到以下相似表达：")
|
|
for style, situation, score in similar_exprs:
|
|
print(f"\n\033[33m表达方式：{style}\033[0m")
|
|
print(f"\033[32m对应情景：{situation}\033[0m")
|
|
print(f"相似度：{score:.3f}")
|
|
print("-" * 20)
|
|
else:
|
|
print("\n没有找到相似的表达方式")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
import asyncio
|
|
|
|
asyncio.run(main())
|