大幅优化 Bangumi Data 检索性能与速度,避免中低端机器开启为负优化 (#301)
* bangumi-data-util.js:大幅优化数据查询性能 * tmdb-util.js:异步查询本地 Bangumi Data * animeko.js:异步查询本地 Bangumi Data * bahamut.js:异步查询本地 Bangumi Data * bilibili.js:异步查询本地 Bangumi Data * dandan.js:异步查询本地 Bangumi Data * bangumi-data-util.js:兼容旧版数据缓存 * tmdb-util.js:调整日志输出 * renren.js:捕获空弹幕数据,直接返回空数组 * renren.js:还原修改,因为并不有效 Removed checks for 'Document not found' error in response handling. * merge-util.js:再次修复Re0最新季因共识差导致animeko映射错误 * bangumi-data-util.js:再次优化,引入倒排索引避免未来数据越来越多检索变慢 * dandan.js:如果结果命中用户需要的季,则跳过非目标季相关请求 减轻容易报错的详情接口压力 * merge-util.js:只有 1 个资源时不进行合并计算 * bahamut.js:如果结果命中用户需要的季,则跳过非目标季相关请求 * Add extractSeasonNumberFromAnimeTitle to imports * Add extractSeasonNumberFromAnimeTitle to imports * Add extractSeasonNumberFromAnimeTitle to imports * bilibili.js:更新 Bangumi Data 数据对齐匹配逻辑,修复中配版对齐错误 * bahamut.js:同步更新 Bangumi Data 数据对齐匹配逻辑 * bahamut.js:补全 Bangumi Data 数据对齐标识,修复对齐后TMDB标题替换错误启动 * tmdb-util.js:调整智能标题替换执行逻辑,无有效资源时不再进行无效计算
This commit is contained in:
1 parent
a51fb8d408
commit
8e3d208c2f
7 files changed
+400
-128
No files matched your search
@@ -37,9 +37,9 @@ export default class AnimekoSource extends BaseSource {
|
||||
*/
|
||||
async search(keyword) {
|
||||
if (globals.useBangumiData) {
|
||||
const localMatches = searchBangumiData(keyword, ['bangumi']);
|
||||
const localMatches = await searchBangumiData(keyword, ['bangumi']);
|
||||
if (localMatches.length > 0) {
|
||||
log("info", `[Animeko] Bangumi-Data 命中 ${localMatches.length} 条数据`);
|
||||
log("info", `[Animeko] Bangumi-Data 本地命中 ${localMatches.length} 条数据`);
|
||||
return this.transformResults(localMatches.map(m => {
|
||||
const displayTitle = m.titles.find(t => t && t.includes(keyword)) || m.titles[1] || m.title;
|
||||
const finalTitle = displayTitle + (m.titleSuffix || '');
|
||||
|
||||
@@ -6,7 +6,7 @@ import { generateValidStartDate } from "../utils/time-util.js";
|
||||
import { addAnime, removeEarliestAnime } from "../utils/cache-util.js";
|
||||
import { simplized, traditionalized } from "../utils/zh-util.js";
|
||||
import { getTmdbJaOriginalTitle, smartTitleReplace } from "../utils/tmdb-util.js";
|
||||
import { strictTitleMatch, normalizeSpaces } from "../utils/common-util.js";
|
||||
import { strictTitleMatch, normalizeSpaces, getExplicitSeasonNumber, extractSeasonNumberFromAnimeTitle } from "../utils/common-util.js";
|
||||
import { SegmentListResponse } from '../models/dandan-model.js';
|
||||
import { searchBangumiData } from '../utils/bangumi-data-util.js';
|
||||
|
||||
@@ -21,7 +21,7 @@ export default class BahamutSource extends BaseSource {
|
||||
let localMatches = [];
|
||||
// 提前获取本地匹配结果
|
||||
if (globals.useBangumiData) {
|
||||
localMatches = searchBangumiData(keyword, ['gamer', 'gamer_hk']);
|
||||
localMatches = await searchBangumiData(keyword, ['gamer', 'gamer_hk']);
|
||||
log("info", `[Bahamut] Bangumi-Data 本地命中 ${localMatches.length} 条数据`);
|
||||
}
|
||||
|
||||
@@ -161,11 +161,10 @@ export default class BahamutSource extends BaseSource {
|
||||
for (const item of finalResults) {
|
||||
if (!item || !item.acg_sn) continue;
|
||||
|
||||
// 依据 acg_sn 匹配本地数据
|
||||
const matchedLocal = localMatches.find(m =>
|
||||
String(item.acg_sn) === String(m.siteId) ||
|
||||
(item.title && m.title === item.title)
|
||||
);
|
||||
// 对齐逻辑:优先精准匹配 acg_sn,其次降级匹配原名
|
||||
const matchedLocal =
|
||||
localMatches.find(m => String(item.acg_sn) === String(m.siteId)) ||
|
||||
localMatches.find(m => item.title && m.title === item.title);
|
||||
|
||||
if (matchedLocal) {
|
||||
const originalBahamutTitle = item.title;
|
||||
@@ -175,6 +174,7 @@ export default class BahamutSource extends BaseSource {
|
||||
// 注入本地别名和优选标题,同时挂载精准类型
|
||||
item.title = finalTitle;
|
||||
item._displayTitle = finalTitle;
|
||||
item.isLocalPriority = true;
|
||||
item.aliases = [...matchedLocal.titles];
|
||||
|
||||
// 将原始网络标题加入别名池,防止后续匹配时丢失源站的精确特征
|
||||
@@ -337,8 +337,28 @@ export default class BahamutSource extends BaseSource {
|
||||
const cnAlias = filtered.length > 0 ? filtered[0]._tmdbCnAlias : null;
|
||||
smartTitleReplace(filtered, cnAlias);
|
||||
|
||||
// 提取搜索词中的明确季度信息
|
||||
const querySeason = getExplicitSeasonNumber(queryTitle);
|
||||
|
||||
// 初始列表预过滤机制:若用户指定了季度,优先检查初始结果中是否已包含匹配项
|
||||
let matchedAnimes = filtered;
|
||||
|
||||
if (querySeason !== null) {
|
||||
const seasonFiltered = filtered.filter(anime => {
|
||||
const titleToCheck = anime._displayTitle || anime.title;
|
||||
const s = extractSeasonNumberFromAnimeTitle(titleToCheck).season;
|
||||
return s === querySeason || (querySeason === 1 && s === null);
|
||||
});
|
||||
|
||||
// 如果已命中目标,减少详情请求量
|
||||
if (seasonFiltered.length > 0) {
|
||||
matchedAnimes = seasonFiltered;
|
||||
log("info", `[Bahamut] 结果已命中目标季(第${querySeason}季),跳过非目标季相关请求`);
|
||||
}
|
||||
}
|
||||
|
||||
// 使用 map 和 async 时需要返回 Promise 数组,并等待所有 Promise 完成
|
||||
const processBahamutAnimes = await Promise.all(filtered.map(async (anime) => {
|
||||
const processBahamutAnimes = await Promise.all(matchedAnimes.map(async (anime) => {
|
||||
try {
|
||||
const epData = await this.getEpisodes(anime.video_sn);
|
||||
const detail = epData.video;
|
||||
|
||||
@@ -288,7 +288,7 @@ export default class BilibiliSource extends BaseSource {
|
||||
let localMatches = [];
|
||||
if (globals.useBangumiData) {
|
||||
// 获取本地匹配条目
|
||||
localMatches = searchBangumiData(keyword, [
|
||||
localMatches = await searchBangumiData(keyword, [
|
||||
'bilibili', 'bilibili_hk_mo_tw', 'bilibili_hk_mo', 'bilibili_tw'
|
||||
]);
|
||||
log("info", `[Bilibili] Bangumi-Data 本地命中 ${localMatches.length} 条数据`);
|
||||
@@ -321,15 +321,14 @@ export default class BilibiliSource extends BaseSource {
|
||||
const seenIds = new Set();
|
||||
const consumedLocalMdIds = new Set();
|
||||
|
||||
// 处理网络结果:对齐本地数据进行信息增强
|
||||
// 对齐 Bangumi Data 进行信息强化
|
||||
for (const item of networkResults) {
|
||||
if (!item || (!item.mediaId && !item.season_id)) continue;
|
||||
|
||||
// 对齐逻辑:优先匹配 mdId,其次匹配原名
|
||||
const matchedLocal = localMatches.find(m =>
|
||||
(item.mdId && item.mdId === `md${m.siteId}`) ||
|
||||
(item.org_title && m.title === item.org_title)
|
||||
);
|
||||
// 对齐逻辑:优先精准匹配 mdId,其次降级匹配原名
|
||||
const matchedLocal =
|
||||
localMatches.find(m => item.mdId && item.mdId === `md${m.siteId}`) ||
|
||||
localMatches.find(m => item.org_title && m.title === item.org_title);
|
||||
|
||||
if (matchedLocal) {
|
||||
const displayTitle = matchedLocal.titles.find(t => t && t.includes(keyword)) || matchedLocal.titles[1] || matchedLocal.title;
|
||||
|
||||
@@ -12,7 +12,7 @@ import MangoSource from "./mango.js";
|
||||
import BilibiliSource from "./bilibili.js";
|
||||
import YoukuSource from "./youku.js";
|
||||
import BahamutSource from "./bahamut.js";
|
||||
import { titleMatches, getExplicitSeasonNumber } from "../utils/common-util.js";
|
||||
import { titleMatches, getExplicitSeasonNumber, extractSeasonNumberFromAnimeTitle } from "../utils/common-util.js";
|
||||
import { searchBangumiData } from '../utils/bangumi-data-util.js';
|
||||
|
||||
const tencentSource = new TencentSource();
|
||||
@@ -37,7 +37,7 @@ export default class DandanSource extends BaseSource {
|
||||
*/
|
||||
async search(keyword, isFallback = false) {
|
||||
if (globals.useBangumiData && !isFallback) {
|
||||
const localMatches = searchBangumiData(keyword, ['anidb']);
|
||||
const localMatches = await searchBangumiData(keyword, ['anidb']);
|
||||
if (localMatches.length > 0) {
|
||||
log("info", `[Dandan] Bangumi-Data 本地命中 ${localMatches.length} 条数据`);
|
||||
return localMatches.map(m => {
|
||||
@@ -314,8 +314,30 @@ export default class DandanSource extends BaseSource {
|
||||
const existingIds = new Set();
|
||||
const queue = [];
|
||||
|
||||
// 初始化任务队列与去重池:将所有初始搜索结果载入队列,标记为非相关作品
|
||||
for (const anime of sourceAnimes) {
|
||||
// 提取搜索词中的明确季度信息
|
||||
const querySeason = getExplicitSeasonNumber(queryTitle);
|
||||
|
||||
// 初始列表预过滤机制:若用户指定了季度,优先检查初始结果中是否已包含匹配项
|
||||
let matchedAnimes = sourceAnimes;
|
||||
let isTargetFoundInInitial = false;
|
||||
|
||||
if (querySeason !== null) {
|
||||
const filtered = sourceAnimes.filter(anime => {
|
||||
const titleToCheck = anime._displayTitle || anime.animeTitle;
|
||||
const s = extractSeasonNumberFromAnimeTitle(titleToCheck).season;
|
||||
return s === querySeason || (querySeason === 1 && s === null);
|
||||
});
|
||||
|
||||
// 如果已命中目标,减少详情请求量
|
||||
if (filtered.length > 0) {
|
||||
matchedAnimes = filtered;
|
||||
isTargetFoundInInitial = true;
|
||||
log("info", `[Dandan] 结果已命中目标季(第${querySeason}季),跳过非目标季相关请求`);
|
||||
}
|
||||
}
|
||||
|
||||
// 初始化任务队列与去重池:将筛选后的条目载入队列,标记为非相关作品
|
||||
for (const anime of matchedAnimes) {
|
||||
existingIds.add(anime.animeId);
|
||||
queue.push({ ...anime, isRelated: false });
|
||||
}
|
||||
@@ -334,11 +356,14 @@ export default class DandanSource extends BaseSource {
|
||||
// 计算当前作品标题与用户原始搜索词的相似度
|
||||
const similarity = this.calculateSimilarity(queryTitle, anime.animeTitle);
|
||||
|
||||
// 关联挖掘控制逻辑:仅当用户未指定明确季度,或者初始扫描未命中目标季度时,才执行相关作品的深度展开
|
||||
const canExpandRelateds = !isTargetFoundInInitial;
|
||||
|
||||
// 相似度高于10%时,对每个关联作品单独判断是否符合展开条件:
|
||||
// 关联作品标题含季度信息(避免范围发散),或初始搜索结果不少于25个(API25个结果上限,用相关作品突破)
|
||||
if (similarity >= 0.1 && details.relateds && Array.isArray(details.relateds)) {
|
||||
if (similarity >= 0.1 && details.relateds && Array.isArray(details.relateds) && canExpandRelateds) {
|
||||
for (const rel of details.relateds) {
|
||||
const hasSeason = getExplicitSeasonNumber(rel.animeTitle) !== null;
|
||||
const hasSeason = extractSeasonNumberFromAnimeTitle(rel.animeTitle).season !== null;
|
||||
if (!existingIds.has(rel.animeId) && (hasSeason || initialCount >= 25)) {
|
||||
existingIds.add(rel.animeId);
|
||||
queue.push({
|
||||
|
||||
@@ -4,7 +4,7 @@ import { pipeline } from 'stream/promises';
|
||||
import fetch from 'node-fetch';
|
||||
import { globals } from '../configs/globals.js';
|
||||
import { log } from './log-util.js';
|
||||
import { titleMatches } from './common-util.js';
|
||||
import { titleMatches, normalizeSpaces, getExplicitSeasonNumber, extractSeasonNumberFromAnimeTitle } from './common-util.js';
|
||||
import { simplized, traditionalized } from './zh-util.js';
|
||||
|
||||
// =====================
|
||||
@@ -17,11 +17,49 @@ let isDownloading = false;
|
||||
let downloadLockTime = 0;
|
||||
let memoryFootprintMB = '0.00';
|
||||
let hasLoggedCacheWarning = false;
|
||||
let charInvertedIndex = new Map();
|
||||
|
||||
// 定义缓存目录和文件名
|
||||
// 定义缓存目录/文件名
|
||||
const CACHE_DIR = path.join(process.cwd(), '.cache');
|
||||
const CACHE_FILENAME = 'bangumi-data-cache.json';
|
||||
const DOWNLOAD_TIMEOUT_MS = 20000;
|
||||
const queryCache = new Map();
|
||||
|
||||
// 预编译全局正则表达式
|
||||
const VALID_DUB_REGEX = /(?:普通[话話]|[国國][语語]|中文配音|中配|中文|[粤粵][语語]配音|[粤粵]配|[粤粵][语語]|[台臺]配|[台臺][语語]|港配|港[语語]|字幕|助[听聽]|日[语語]|日配|原版|原[声聲])(?:版)?/;
|
||||
const SUFFIX_CLEAN_REGEX = /(?:\s+|^)(?:第?\s*(?:\d+|[一二三四五六七八九十]+)\s*[季期部]|season\s*\d+|s\d+|part\s*\d+|act\s*\d+|phase\s*\d+|the\s+final\s+season|(?:movie|film|ova|oad|sp|剧场版|劇場版|续[篇集]|外传)(?![a-z]))|[::~~]|\s+.*?篇|(?<=\s|^)\d+$/gi;
|
||||
const WHITESPACE_REGEX = /\s+/g;
|
||||
const COMMA_SPLIT_REGEX = /[,,]/;
|
||||
const COLON_SPLIT_REGEX = /[::]/;
|
||||
|
||||
/**
|
||||
* 构建内存级倒排特征索引字典
|
||||
* 提取每个条目的指纹字符,映射至该字符关联的条目数组序号中
|
||||
* @param {Array<Object>} items - 已精简的条目数组
|
||||
*/
|
||||
function buildInvertedIndex(items) {
|
||||
const newCharIndex = new Map();
|
||||
const len = items.length;
|
||||
|
||||
for (let i = 0; i < len; i++) {
|
||||
const flatText = items[i]._flatText;
|
||||
if (!flatText) continue;
|
||||
|
||||
const uniqueChars = new Set(flatText);
|
||||
for (const char of uniqueChars) {
|
||||
if (char.trim() === '') continue;
|
||||
|
||||
let idArray = newCharIndex.get(char);
|
||||
if (!idArray) {
|
||||
idArray = [];
|
||||
newCharIndex.set(char, idArray);
|
||||
}
|
||||
idArray.push(i);
|
||||
}
|
||||
}
|
||||
|
||||
charInvertedIndex = newCharIndex;
|
||||
}
|
||||
|
||||
/**
|
||||
* 初始化 Bangumi Data 数据源
|
||||
@@ -83,6 +121,16 @@ export async function initBangumiData(deployPlatform, isDataDependentRequest = f
|
||||
|
||||
memoryCache = JSON.parse(fs.readFileSync(cachePath, 'utf-8'));
|
||||
|
||||
// 兼容旧版缓存:静默热升级,应用预处理规则生成特征指纹并更新磁盘文件
|
||||
if (memoryCache?.items?.length > 0 && memoryCache.items[0]._flatText === undefined) {
|
||||
memoryCache = pruneBangumiData(memoryCache);
|
||||
fs.writeFileSync(cachePath, JSON.stringify(memoryCache), 'utf-8');
|
||||
buildInvertedIndex(memoryCache.items);
|
||||
} else if (memoryCache?.items?.length > 0) {
|
||||
// 内存读取流程:倒排索引结构非持久化存储,需依据内存数据触发重建
|
||||
buildInvertedIndex(memoryCache.items);
|
||||
}
|
||||
|
||||
const memAfter = process.memoryUsage().heapUsed;
|
||||
const loadTimeMs = Date.now() - startTime;
|
||||
memoryFootprintMB = Math.max(0, (memAfter - memBefore) / 1024 / 1024).toFixed(2);
|
||||
@@ -146,14 +194,16 @@ const ALLOWED_SITES = new Set([
|
||||
|
||||
/**
|
||||
* 精简 Bangumi Data 数据结构
|
||||
* 提取搜索逻辑必须的字段,并过滤掉不包含目标站点标识的动漫条目,控制常驻内存占用
|
||||
* * @param {Object} rawData - 原始完整版 Bangumi Data JSON 对象
|
||||
* 提取检索与渲染必需的字段,过滤无关站点,并在此时提取倒排特征文本
|
||||
* 预处理后的数据会随主流程落盘,消除运行时的冷启动开销
|
||||
* @param {Object} rawData - 原始完整版 Bangumi Data JSON 对象
|
||||
* @returns {Object} 包含精简后 items 数组的对象
|
||||
*/
|
||||
function pruneBangumiData(rawData) {
|
||||
if (!rawData || !rawData.items) return { items: [] };
|
||||
|
||||
const prunedItems = [];
|
||||
|
||||
for (const item of rawData.items) {
|
||||
// 筛选当前条目下属于允许列表的站点信息
|
||||
const validSites = [];
|
||||
@@ -170,7 +220,7 @@ function pruneBangumiData(rawData) {
|
||||
// 丢弃不包含任何目标站点的条目,避免占用内存
|
||||
if (validSites.length === 0) continue;
|
||||
|
||||
// 构建精简版条目,仅保留核心检索字段
|
||||
// 构建精简版条目,保留核心数据
|
||||
const prunedItem = {
|
||||
title: item.title,
|
||||
type: item.type,
|
||||
@@ -179,8 +229,22 @@ function pruneBangumiData(rawData) {
|
||||
if (item.begin) prunedItem.begin = item.begin;
|
||||
if (item.titleTranslate) prunedItem.titleTranslate = item.titleTranslate;
|
||||
|
||||
// 构建聚合特征指纹:合并所有标题,强制转换为简体,并进行统一字符集规范化及小写转换
|
||||
let str = item.title;
|
||||
if (item.titleTranslate) {
|
||||
for (const lang in item.titleTranslate) {
|
||||
const transArr = item.titleTranslate[lang];
|
||||
for (let j = 0; j < transArr.length; j++) {
|
||||
str += transArr[j];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const normalizedStr = simplized(str);
|
||||
prunedItem._flatText = normalizeSpaces(normalizedStr).toLowerCase().replace(SUFFIX_CLEAN_REGEX, '');
|
||||
prunedItems.push(prunedItem);
|
||||
}
|
||||
|
||||
return { items: prunedItems };
|
||||
}
|
||||
|
||||
@@ -229,6 +293,8 @@ async function downloadAndCache(cachePath) {
|
||||
const resultData = await response.json();
|
||||
const originalCount = resultData.items ? resultData.items.length : 0;
|
||||
|
||||
if (controller.signal.aborted) throw new Error('Aborted before parsing');
|
||||
|
||||
// 立即执行数据裁剪以释放内存
|
||||
const localParseStartTime = Date.now();
|
||||
const prunedData = pruneBangumiData(resultData);
|
||||
@@ -236,7 +302,7 @@ async function downloadAndCache(cachePath) {
|
||||
|
||||
let tempFilePath = null;
|
||||
|
||||
if (cachePath) {
|
||||
if (cachePath && !controller.signal.aborted) {
|
||||
// 物理磁盘模式:为每个并发流生成独立的临时文件,写入精简后的数据
|
||||
tempFilePath = `${cachePath}.tmp${index}`;
|
||||
fs.writeFileSync(tempFilePath, JSON.stringify(prunedData), 'utf-8');
|
||||
@@ -277,6 +343,8 @@ async function downloadAndCache(cachePath) {
|
||||
// 数据处理流
|
||||
memoryCache = winner.resultData;
|
||||
parseTimeMs = winner.pruneCost;
|
||||
buildInvertedIndex(memoryCache.items);
|
||||
queryCache.clear();
|
||||
|
||||
const memAfter = process.memoryUsage().heapUsed;
|
||||
const totalTime = ((Date.now() - startTime) / 1000).toFixed(2);
|
||||
@@ -295,109 +363,240 @@ async function downloadAndCache(cachePath) {
|
||||
|
||||
/**
|
||||
* 在 Bangumi Data 中搜索匹配的动漫条目
|
||||
* 采用倒排索引结合特征词清洗逻辑进行初筛,随后执行精准匹配
|
||||
* 支持多语言标题检索、特定站点过滤以及配音/区域版本解析
|
||||
* @param {string} keyword - 搜索关键词
|
||||
* @param {Array<string>} siteKeys - 需要匹配的源站点标识数组
|
||||
* @returns {Array<Object>} 匹配的动漫条目数组,包含解析后的展示标题、ID及类型等元数据
|
||||
* @returns {Array<Object>} 匹配的动漫条目数组
|
||||
*/
|
||||
export function searchBangumiData(keyword, siteKeys) {
|
||||
export async function searchBangumiData(keyword, siteKeys) {
|
||||
if (!memoryCache || !memoryCache.items) return [];
|
||||
|
||||
const validDubRegex = /(?:普通[话話]|[国國][语語]|中文配音|中配|中文|[粤粵][语語]配音|[粤粵]配|[粤粵][语語]|[台臺]配|[台臺][语語]|港配|港[语語]|字幕|助[听聽]|日[语語]|日配|原版|原[声聲])(?:版)?/;
|
||||
const results = [];
|
||||
let searchPromise = queryCache.get(keyword);
|
||||
|
||||
// 简繁转换搜索关键词
|
||||
let searchTerms = [keyword];
|
||||
try {
|
||||
searchTerms = [...new Set([keyword, simplized(keyword), traditionalized(keyword)])];
|
||||
} catch (e) {
|
||||
}
|
||||
if (!searchPromise) {
|
||||
searchPromise = (async () => {
|
||||
const matched = [];
|
||||
|
||||
for (const item of memoryCache.items) {
|
||||
// 构建完整的搜索标题池
|
||||
const titles = [item.title];
|
||||
if (item.titleTranslate) {
|
||||
for (const lang in item.titleTranslate) {
|
||||
titles.push(...item.titleTranslate[lang]);
|
||||
// 保留完整的繁简变体数组用于最终的精准校验阶段
|
||||
let searchTerms = [keyword];
|
||||
try {
|
||||
searchTerms = [...new Set([keyword, simplized(keyword), traditionalized(keyword)])];
|
||||
} catch (e) {}
|
||||
|
||||
// 提取核心检索词:基于底层指纹的简体单向收束特性,仅提取简体核心词用于极速初筛
|
||||
const unifiedKeyword = simplized(keyword);
|
||||
const coreKws = [unifiedKeyword].map(kw => {
|
||||
let core = kw.replace(SUFFIX_CLEAN_REGEX, '');
|
||||
core = normalizeSpaces(core).toLowerCase();
|
||||
// 当规范化后字符过短时,降级使用仅规范化的原始词作为初筛条件
|
||||
return core.length >= 2 ? core : normalizeSpaces(kw).toLowerCase();
|
||||
}).filter(k => k.length > 0);
|
||||
|
||||
let candidateIndices = null;
|
||||
|
||||
// 倒排索引初筛逻辑
|
||||
if (typeof charInvertedIndex !== 'undefined' && charInvertedIndex.size > 0 && coreKws.length > 0) {
|
||||
const globalCandidates = new Set();
|
||||
|
||||
for (const kw of coreKws) {
|
||||
// 剔除空格后,获取当前搜索词的所有去重单字
|
||||
const chars = Array.from(new Set(kw.replace(WHITESPACE_REGEX, '')));
|
||||
if (chars.length === 0) continue;
|
||||
|
||||
// 按字在库中出现的频率从小到大排序,优先处理包含条目最少的字以减少运算量
|
||||
chars.sort((a, b) => {
|
||||
const lenA = charInvertedIndex.get(a)?.length || 0;
|
||||
const lenB = charInvertedIndex.get(b)?.length || 0;
|
||||
return lenA - lenB;
|
||||
});
|
||||
|
||||
// 如果出现频率最低的字不存在,表明该词无对应条目
|
||||
const rarestCharArr = charInvertedIndex.get(chars[0]);
|
||||
if (!rarestCharArr || rarestCharArr.length === 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// 初始化当前词的候选池
|
||||
let localCandidates = new Set(rarestCharArr);
|
||||
|
||||
// 依次与后续字的集合求交集
|
||||
for (let i = 1; i < chars.length; i++) {
|
||||
const nextCharArr = charInvertedIndex.get(chars[i]);
|
||||
if (!nextCharArr) {
|
||||
localCandidates.clear(); break;
|
||||
}
|
||||
|
||||
const nextCharSet = new Set(nextCharArr);
|
||||
for (const idx of localCandidates) {
|
||||
if (!nextCharSet.has(idx)) {
|
||||
localCandidates.delete(idx);
|
||||
}
|
||||
}
|
||||
if (localCandidates.size === 0) break;
|
||||
}
|
||||
|
||||
// 合并到全局候选池
|
||||
for (const idx of localCandidates) {
|
||||
globalCandidates.add(idx);
|
||||
}
|
||||
}
|
||||
|
||||
candidateIndices = Array.from(globalCandidates);
|
||||
}
|
||||
else {
|
||||
// 索引未就绪时,降级使用全量扫描
|
||||
candidateIndices = memoryCache.items.map((_, i) => i);
|
||||
}
|
||||
}
|
||||
|
||||
const isMatch = titles.some(t => t && searchTerms.some(kw => titleMatches(t, kw)));
|
||||
const items = memoryCache.items;
|
||||
const termsLen = searchTerms.length;
|
||||
|
||||
if (isMatch && item.sites) {
|
||||
// 捕获所有符合条件的站点记录,支持同源多区域版本(如大陆版和港澳台版共存)
|
||||
const matchedSites = item.sites.filter(s => siteKeys.includes(s.site));
|
||||
// 核心匹配逻辑:校验候选项的主标题及所有多语言翻译版本
|
||||
for (let i = 0; i < candidateIndices.length; i++) {
|
||||
const item = items[candidateIndices[i]];
|
||||
|
||||
for (const matchedSite of matchedSites) {
|
||||
// 标准化媒体类型映射
|
||||
let typeStr = "TV动画";
|
||||
let typeId = "tvseries";
|
||||
if (item.type === 'movie') { typeStr = "剧场版"; typeId = "movie"; }
|
||||
else if (item.type === 'tv') { typeStr = "TV动画"; typeId = "tvseries"; }
|
||||
else if (item.type === 'ova') { typeStr = "OVA"; typeId = "ova"; }
|
||||
else if (item.type === 'web') { typeStr = "WEB动画"; typeId = "web"; }
|
||||
let isMatch = false;
|
||||
for (let k = 0; k < termsLen; k++) {
|
||||
const kw = searchTerms[k];
|
||||
|
||||
// 提取区域版本基础后缀
|
||||
let baseSuffix = "";
|
||||
if (['bilibili_hk_mo_tw', 'bilibili_hk_mo', 'bilibili_tw'].includes(matchedSite.site)) {
|
||||
baseSuffix = "(港澳台)";
|
||||
if (titleMatches(item.title, kw)) {
|
||||
isMatch = true; break;
|
||||
}
|
||||
|
||||
if (item.titleTranslate) {
|
||||
for (const lang in item.titleTranslate) {
|
||||
const transArr = item.titleTranslate[lang];
|
||||
for (let j = 0; j < transArr.length; j++) {
|
||||
if (transArr[j] && titleMatches(transArr[j], kw)) {
|
||||
isMatch = true; break;
|
||||
}
|
||||
}
|
||||
if (isMatch) break;
|
||||
}
|
||||
}
|
||||
if (isMatch) break;
|
||||
}
|
||||
|
||||
// 解析衍生配音版本与当前主条目的附加描述
|
||||
let additionalDubs = [];
|
||||
let mainItemDubs = [];
|
||||
|
||||
if (matchedSite.comment) {
|
||||
const dubs = matchedSite.comment.split(/[,,]/);
|
||||
for (const dub of dubs) {
|
||||
const parts = dub.split(/[::]/);
|
||||
if (parts.length >= 2) {
|
||||
const dubName = parts[0].trim();
|
||||
const dubId = parts[1].trim();
|
||||
if (dubId && validDubRegex.test(dubName)) {
|
||||
additionalDubs.push({
|
||||
title: item.title,
|
||||
titles: [...titles],
|
||||
begin: item.begin,
|
||||
siteId: dubId,
|
||||
matchedSiteKey: matchedSite.site,
|
||||
type: item.type,
|
||||
typeStr: typeStr,
|
||||
typeId: typeId,
|
||||
titleSuffix: ` ${dubName}${baseSuffix}`
|
||||
});
|
||||
}
|
||||
} else if (parts.length === 1 && parts[0].trim()) {
|
||||
const dubName = parts[0].trim();
|
||||
if (validDubRegex.test(dubName)) {
|
||||
mainItemDubs.push(dubName);
|
||||
// 装载匹配结果及其全量标题上下文
|
||||
if (isMatch) {
|
||||
const titles = [item.title];
|
||||
if (item.titleTranslate) {
|
||||
for (const lang in item.titleTranslate) {
|
||||
const transArr = item.titleTranslate[lang];
|
||||
for (let j = 0; j < transArr.length; j++) {
|
||||
titles.push(transArr[j]);
|
||||
}
|
||||
}
|
||||
}
|
||||
matched.push({ item, titles });
|
||||
}
|
||||
|
||||
// 组装当前主条目的最终后缀标签
|
||||
let currentItemSuffix = "";
|
||||
if (mainItemDubs.length > 0) {
|
||||
currentItemSuffix += ` ${mainItemDubs.join(' ')}`;
|
||||
}
|
||||
currentItemSuffix += baseSuffix;
|
||||
|
||||
results.push({
|
||||
title: item.title,
|
||||
titles: [...titles],
|
||||
begin: item.begin,
|
||||
siteId: matchedSite.id,
|
||||
matchedSiteKey: matchedSite.site,
|
||||
type: item.type,
|
||||
typeStr: typeStr,
|
||||
typeId: typeId,
|
||||
titleSuffix: currentItemSuffix
|
||||
});
|
||||
|
||||
// 推送提取出的衍生条目
|
||||
results.push(...additionalDubs);
|
||||
}
|
||||
return matched;
|
||||
})();
|
||||
|
||||
// 维护缓存容量池,基于 LRU 策略淘汰旧任务
|
||||
if (queryCache.size > 100) {
|
||||
const firstKey = queryCache.keys().next().value;
|
||||
queryCache.delete(firstKey);
|
||||
}
|
||||
queryCache.set(keyword, searchPromise);
|
||||
}
|
||||
|
||||
const matchedItems = await searchPromise;
|
||||
let finalItems = matchedItems;
|
||||
|
||||
// 基于搜索词提取明确的季度信息并执行结果精准过滤
|
||||
const querySeason = getExplicitSeasonNumber(keyword);
|
||||
if (querySeason !== null) {
|
||||
const tempFiltered = matchedItems.filter(({ item }) => {
|
||||
let itemSeason = extractSeasonNumberFromAnimeTitle(item.title).season;
|
||||
if (itemSeason === null && item.titleTranslate) {
|
||||
for (const lang in item.titleTranslate) {
|
||||
const transArr = item.titleTranslate[lang];
|
||||
for (let j = 0; j < transArr.length; j++) {
|
||||
const s = extractSeasonNumberFromAnimeTitle(transArr[j]).season;
|
||||
if (s !== null) { itemSeason = s; break; }
|
||||
}
|
||||
if (itemSeason !== null) break;
|
||||
}
|
||||
}
|
||||
|
||||
// 搜索指定续作(>1)时,标题必须明确包含该季度标识
|
||||
if (querySeason > 1) {
|
||||
return (itemSeason || 1) === querySeason;
|
||||
}
|
||||
// 搜索第1季时,拦截明确标明为其他季度(如第2季、第3季)的结果
|
||||
else if (querySeason === 1) {
|
||||
return itemSeason === null || itemSeason === 1;
|
||||
}
|
||||
return true;
|
||||
});
|
||||
|
||||
// 仅当过滤后结果不为空时应用,防止由于解析缺失导致结果被误杀清空
|
||||
if (tempFiltered.length > 0) {
|
||||
finalItems = tempFiltered;
|
||||
}
|
||||
}
|
||||
|
||||
const validDubRegex = VALID_DUB_REGEX;
|
||||
const results = [];
|
||||
|
||||
// 特定站点分发与区域版本/配音版本构建
|
||||
for (const { item, titles } of finalItems) {
|
||||
if (!item.sites) continue;
|
||||
|
||||
const matchedSites = item.sites.filter(s => siteKeys.includes(s.site));
|
||||
for (const matchedSite of matchedSites) {
|
||||
let typeStr = "TV动画"; let typeId = "tvseries";
|
||||
if (item.type === 'movie') { typeStr = "剧场版"; typeId = "movie"; }
|
||||
else if (item.type === 'tv') { typeStr = "TV动画"; typeId = "tvseries"; }
|
||||
else if (item.type === 'ova') { typeStr = "OVA"; typeId = "ova"; }
|
||||
else if (item.type === 'web') { typeStr = "WEB动画"; typeId = "web"; }
|
||||
|
||||
let baseSuffix = "";
|
||||
if (['bilibili_hk_mo_tw', 'bilibili_hk_mo', 'bilibili_tw'].includes(matchedSite.site)) {
|
||||
baseSuffix = "(港澳台)";
|
||||
}
|
||||
|
||||
let additionalDubs = [];
|
||||
let mainItemDubs = [];
|
||||
|
||||
if (matchedSite.comment) {
|
||||
const dubs = matchedSite.comment.split(COMMA_SPLIT_REGEX);
|
||||
for (const dub of dubs) {
|
||||
const parts = dub.split(COLON_SPLIT_REGEX);
|
||||
if (parts.length >= 2) {
|
||||
const dubName = parts[0].trim(); const dubId = parts[1].trim();
|
||||
if (dubId && validDubRegex.test(dubName)) {
|
||||
additionalDubs.push({
|
||||
title: item.title, titles: [...titles], begin: item.begin,
|
||||
siteId: dubId, matchedSiteKey: matchedSite.site,
|
||||
type: item.type, typeStr: typeStr, typeId: typeId, titleSuffix: ` ${dubName}${baseSuffix}`
|
||||
});
|
||||
}
|
||||
} else if (parts.length === 1 && parts[0].trim()) {
|
||||
const dubName = parts[0].trim();
|
||||
if (validDubRegex.test(dubName)) {
|
||||
mainItemDubs.push(dubName);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 组装当前主条目的最终后缀标签
|
||||
let currentItemSuffix = "";
|
||||
if (mainItemDubs.length > 0) { currentItemSuffix += ` ${mainItemDubs.join(' ')}`; }
|
||||
currentItemSuffix += baseSuffix;
|
||||
|
||||
results.push({
|
||||
title: item.title, titles: [...titles], begin: item.begin,
|
||||
siteId: matchedSite.id, matchedSiteKey: matchedSite.site,
|
||||
type: item.type, typeStr: typeStr, typeId: typeId, titleSuffix: currentItemSuffix
|
||||
});
|
||||
|
||||
// 推送提取出的衍生条目
|
||||
results.push(...additionalDubs);
|
||||
}
|
||||
}
|
||||
return results;
|
||||
@@ -412,6 +611,8 @@ export function clearBangumiDataCache() {
|
||||
const itemCount = memoryCache.items ? memoryCache.items.length : 0;
|
||||
memoryCache = null; // 切断引用,等待 GC 回收
|
||||
memoryCacheTime = 0; // 重置寿命时钟
|
||||
queryCache.clear(); // 释放查询缓存
|
||||
charInvertedIndex.clear(); // 释放倒排索引内存
|
||||
const totalMemMB = (process.memoryUsage().rss / 1024 / 1024).toFixed(2);
|
||||
log("info", `[Bangumi-Data] 内存缓存已主动释放 (原条目数: ${itemCount},释放: ${memoryFootprintMB} MB,当前项目总占用: ${totalMemMB} MB)`);
|
||||
memoryFootprintMB = '0.00'; // 重置探针
|
||||
|
||||
@@ -508,6 +508,7 @@ function calculateSimilarity(str1, str2) {
|
||||
if (!str1 || !str2) return 0;
|
||||
const s1 = cleanTitleForSimilarity(str1);
|
||||
const s2 = cleanTitleForSimilarity(str2);
|
||||
if (s1 === '' && s2 === '') return 0.0;
|
||||
if (s1 === s2) return 1.0;
|
||||
const len1 = s1.length, len2 = s2.length;
|
||||
const maxLen = Math.max(len1, len2), minLen = Math.min(len1, len2);
|
||||
@@ -1411,6 +1412,7 @@ function findBestAlignmentOffset(primaryLinks, secondaryLinks, seriesLangA = 'Un
|
||||
let totalTextScore = 0, rawTextScoreSum = 0, matchCount = 0;
|
||||
let numericDiffs = new Map();
|
||||
let hasSeasonShiftMatch = false;
|
||||
let lastPNumLocal = null;
|
||||
|
||||
for (let i = 0; i < secondaryLinks.length; i++) {
|
||||
const pIndex = i + offset;
|
||||
@@ -1453,10 +1455,23 @@ function findBestAlignmentOffset(primaryLinks, secondaryLinks, seriesLangA = 'Un
|
||||
rawTextScoreSum += sim;
|
||||
if (infoA.num !== null && infoB.num !== null && infoA.num === infoB.num) pairScore += MergeWeights.EP_ALIGN.NUMERIC_MATCH;
|
||||
|
||||
let weightMultiplier = 1.0;
|
||||
if (infoA.num !== null && !infoA.isSpecial) {
|
||||
if (lastPNumLocal !== null && (infoA.num - lastPNumLocal > 1)) {
|
||||
weightMultiplier = 0.1;
|
||||
} else {
|
||||
lastPNumLocal = infoA.num;
|
||||
}
|
||||
}
|
||||
|
||||
pairScore *= weightMultiplier;
|
||||
sim *= weightMultiplier;
|
||||
|
||||
rawTextScoreSum += sim;
|
||||
totalTextScore += pairScore;
|
||||
if (infoA.num !== null && infoB.num !== null) {
|
||||
const diffKey = (infoB.num - infoA.num).toFixed(4);
|
||||
numericDiffs.set(diffKey, (numericDiffs.get(diffKey) || 0) + 1);
|
||||
numericDiffs.set(diffKey, (numericDiffs.get(diffKey) || 0) + weightMultiplier);
|
||||
}
|
||||
matchCount++;
|
||||
}
|
||||
@@ -2417,6 +2432,9 @@ function detectCollectionCandidates(curAnimes) {
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
export async function applyMergeLogic(curAnimes, detailStore = null) {
|
||||
if (!curAnimes || curAnimes.length < 2) {
|
||||
return;
|
||||
}
|
||||
const groups = globals.mergeSourcePairs;
|
||||
if (!groups || groups.length === 0) return;
|
||||
|
||||
@@ -2585,7 +2603,9 @@ export async function applyMergeLogic(curAnimes, detailStore = null) {
|
||||
items.forEach(item => remainingCandidates.push(item));
|
||||
});
|
||||
|
||||
if (remainingCandidates.length > 0) {
|
||||
const uniqueRemainingSources = new Set(remainingCandidates.map(a => a.source));
|
||||
|
||||
if (remainingCandidates.length > 0 && uniqueRemainingSources.size >= 2) {
|
||||
log("info", `[Merge] [Phase 2] 启动标准回退匹配: 剩余 ${remainingCandidates.length} 个资源。`);
|
||||
sortCandidates(remainingCandidates, "Phase 2");
|
||||
|
||||
|
||||
@@ -214,13 +214,13 @@ export async function getTmdbJaOriginalTitle(title, signal = null, sourceLabel =
|
||||
|
||||
// 优先尝试使用本地 Bangumi Data 获取原名与翻译,零延迟且无需 API Key
|
||||
if (globals.useBangumiData) {
|
||||
const localMatches = searchBangumiData(cleanTitle, ['tmdb', 'bangumi', 'anidb']);
|
||||
const localMatches = await searchBangumiData(cleanTitle, ['tmdb', 'bangumi', 'anidb']);
|
||||
if (localMatches && localMatches.length > 0) {
|
||||
const m = localMatches[0]; // 取第一个最佳匹配
|
||||
const displayTitle = m.titles.find(t => t && t.includes(cleanTitle)) || m.titles[1] || m.title;
|
||||
const jaOriginalTitle = m.title; // Bangumi Data 的主标题就是原名
|
||||
|
||||
log("info", `[TMDB] 命中本地 Bangumi Data,提取原名成功: 原名=${jaOriginalTitle}, 别名=${displayTitle}`);
|
||||
log("info", `[TMDB] Bangumi-Data 本地命中,提取原名成功: 原名=${jaOriginalTitle}, 别名=${displayTitle}`);
|
||||
return { title: jaOriginalTitle, cnAlias: displayTitle };
|
||||
}
|
||||
}
|
||||
@@ -684,7 +684,19 @@ export function cleanSearchQuery(title) {
|
||||
export function smartTitleReplace(animes, cnAlias) {
|
||||
if (!animes || animes.length === 0 || !cnAlias) return;
|
||||
|
||||
log("info", `[TMDB] 启动智能替换,目标别名: "${cnAlias}",待处理条目: ${animes.length}`);
|
||||
let validCount = 0;
|
||||
// 遍历列表执行属性兜底赋值,并统计实际需要执行标题替换的有效条目数
|
||||
for (const anime of animes) {
|
||||
anime._displayTitle = anime._displayTitle || anime.title || "";
|
||||
if (!(anime.isLocalPriority || anime._displayTitle.includes(cnAlias))) {
|
||||
validCount++;
|
||||
}
|
||||
}
|
||||
|
||||
// 若有效替换条目数为0,说明均已处理或无需处理,直接静默退出
|
||||
if (validCount === 0) return;
|
||||
|
||||
log("info", `[TMDB] 启动智能替换,目标别名: "${cnAlias}",待处理条目: ${validCount}`);
|
||||
|
||||
// 计算所有标题主体部分的 LCP (最长公共前缀)
|
||||
const baseTitles = animes.map(a => {
|
||||
@@ -705,36 +717,31 @@ export function smartTitleReplace(animes, cnAlias) {
|
||||
log("info", `[TMDB] 计算出最长公共前缀 (LCP): "${lcp}"`);
|
||||
}
|
||||
|
||||
// 执行具体的智能替换策略
|
||||
for (const anime of animes) {
|
||||
const originalTitle = anime.title || "";
|
||||
|
||||
// 防止本地 Bangumi Data 标题被替换与防止重复标题
|
||||
if (anime.isLocalPriority || originalTitle.includes(cnAlias)) {
|
||||
anime._displayTitle = anime._displayTitle || originalTitle;
|
||||
continue;
|
||||
}
|
||||
// 过滤已被本地数据处理或已含目标别名的条目
|
||||
if (anime.isLocalPriority || originalTitle.includes(cnAlias)) continue;
|
||||
|
||||
// 策略 A: LCP 模式
|
||||
if (lcp && lcp.length > 1 && originalTitle.startsWith(lcp)) {
|
||||
const suffix = originalTitle.substring(lcp.length).trim();
|
||||
anime._displayTitle = suffix ? `${cnAlias}${suffix.match(/^[~~::]/) ? '' : ' '}${suffix}` : cnAlias;
|
||||
log("info", `[TMDB] [LCP模式] "${originalTitle}" -> "${anime._displayTitle}"`);
|
||||
}
|
||||
// 策略 B: 分隔符模式
|
||||
else {
|
||||
} else {
|
||||
const match = originalTitle.match(SEPARATOR_REGEX);
|
||||
if (match) {
|
||||
const prefix = originalTitle.substring(0, match.index).trim();
|
||||
const suffix = originalTitle.substring(match.index);
|
||||
|
||||
// 防止替换必要前缀
|
||||
// 策略 B1: 前缀保护模式(防止截断季数等特征前缀)
|
||||
if (prefix && SUFFIX_PATTERN.test(prefix)) {
|
||||
const subMatch = suffix.trim().match(SEPARATOR_REGEX);
|
||||
const subSuffix = subMatch ? suffix.trim().substring(subMatch.index) : '';
|
||||
anime._displayTitle = `${prefix} ${cnAlias}${subSuffix}`;
|
||||
log("info", `[TMDB] [前缀保护模式] "${originalTitle}" -> "${anime._displayTitle}"`);
|
||||
} else {
|
||||
// 常规分隔符替换
|
||||
// 策略 B2: 常规分隔符模式
|
||||
anime._displayTitle = cnAlias + suffix;
|
||||
log("info", `[TMDB] [分隔符模式] "${originalTitle}" -> "${anime._displayTitle}"`);
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user