From b4342404893640aaae6155cebb312c09c38100b3 Mon Sep 17 00:00:00 2001 From: fanlusky <1730976608@qq.com> Date: Mon, 12 Oct 2026 03:44:25 +0800 Subject: [PATCH] =?UTF-8?q?feat(engine):=20=E6=A8=A1=E7=B3=8A=E8=AF=BB?= =?UTF-8?q?=E9=9F=B3=E6=8C=89=E8=AF=8D=E9=A2=91=E4=B8=8E=E5=8E=9F=E8=AF=BB?= =?UTF-8?q?=E9=9F=B3=E6=B7=B7=E6=8E=92=EF=BC=8C=E6=95=B4=E5=8F=A5=E8=AF=8D?= =?UTF-8?q?=E6=A0=BC=E6=94=B6=E6=A8=A1=E7=B3=8A=E8=AF=BB=E9=9F=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 开模糊音时模糊读音的候选不再整组垫在末尾:同组内按「权重 × 0.2」与敲出读音的词库行 归并,min 出「民 明 名 命 敏…」而不是先翻完 min 的生僻字。词格的跨度查询同时收模糊变体, 每个模糊音节扣 1.0(log10),lin'yi'bian 解出「另一边」;整句 canonical 记真实读音, 造词不写错音。切分与纠错的裁决打分不带模糊读音,避免字面读法借模糊分压过纠错。 双拼同步。eval_quanpin_sentence 新增 fuzzy=<规则位> 参数。 --- engine/direct_helpcode/direct_lattice.cpp | 10 ++-- engine/quanpin/fuzzy_pinyin.h | 53 ++++++++++++++++++++++ engine/quanpin/quanpin_dictionary.cpp | 48 +++++++++++++++----- engine/quanpin/quanpin_dictionary.h | 8 +++- engine/quanpin/quanpin_query.cpp | 46 +++++++++++++++++-- engine/quanpin/quanpin_query.h | 6 ++- engine/quanpin/word_lattice.cpp | 14 +++--- engine/quanpin/word_lattice.h | 2 + engine/shuangpin/engine.cpp | 16 +++---- engine/shuangpin/shuangpin_dictionary.cpp | 12 ++++- engine/shuangpin/shuangpin_dictionary.h | 3 ++ engine/tests/src/eval_quanpin_sentence.cpp | 19 ++++++-- engine/tests/src/test_fuzzy_pinyin.cpp | 37 +++++++++++++++ 13 files changed, 231 insertions(+), 43 deletions(-) diff --git a/engine/direct_helpcode/direct_lattice.cpp b/engine/direct_helpcode/direct_lattice.cpp index 32d77e17..2f827a25 100644 --- a/engine/direct_helpcode/direct_lattice.cpp +++ b/engine/direct_helpcode/direct_lattice.cpp @@ -238,9 +238,10 @@ class WordGraphBuilder } std::vector words; - std::int64_t span_total = 0; + // 按读音分开求和,模糊读音的行不压低敲出读音的占比,见 quanpin::build_graph。 + std::unordered_map reading_totals; for (const auto *row : accepted) - span_total += row->weight > 0 ? row->weight : 0; + reading_totals[row->key] += row->weight > 0 ? row->weight : 0; const std::string span_key = quanpin::join_segments(span); for (const auto *row : accepted) { @@ -250,13 +251,14 @@ class WordGraphBuilder if (model_) { word.index = model_->index(word.word); - word.base_score = - dictionary_tiebreak(row->weight, options_) + reading_prior(row->weight, span_total, options_); + word.base_score = dictionary_tiebreak(row->weight, options_) + + reading_prior(row->weight, reading_totals[row->key], options_); } else { word.base_score = heuristic_log_prob(row->weight, sequence.size(), options_); } + word.base_score -= row->penalty; words.push_back(std::move(word)); } return words; diff --git a/engine/quanpin/fuzzy_pinyin.h b/engine/quanpin/fuzzy_pinyin.h index 8935ce90..4af20d9d 100644 --- a/engine/quanpin/fuzzy_pinyin.h +++ b/engine/quanpin/fuzzy_pinyin.h @@ -1,5 +1,6 @@ #pragma once #include "../core/fuzzy_pinyin_options.h" +#include "../core/word_item.h" #include "quanpin_utils.h" #include #include @@ -7,6 +8,19 @@ namespace quanpin { +// 模糊读音的折扣。开模糊音等于说「这个音我可能没分清」,模糊读音应当与敲出的读音按词频混排, +// 而不是整组垫在后面:min 下的「明」(286 万)该排在「敏」(20 万)前,更该排在 min 那几十个 +// 权重为 0 的生僻字前。折扣保证敲出读音的常用字(「民」90 万)仍压得住模糊读音。 +constexpr double kFuzzyWeightRatio = 0.2; +// 词格里路径每走一个模糊音节扣的分(log10)。拿 eval_quanpin_sentence 的合法输入集加模糊音 +// 例句标定:只开 in/ing 时 0.7~1.0 都不改变合法输入的首选,lin'yi'bian 照样解出「另一边」; +// 规则全开时 wo'xiang 要 1.5 才不被 ian/iang 读成「我先」,但那时 an/ang 的「晚宴上买」又会 +// 压过「蜿蜒山脉的另一边」。取 1.0(模糊读音的句概率要高出 10 倍才翻盘),按常见的只开几条 +// 规则调。 +constexpr double kFuzzyLatticePenaltyLog = 1.0; +// 词格每个跨度最多再查几条模糊变体。跨度最长 7 个音节,规则全开时变体按乘积增长。 +constexpr std::size_t kFuzzyLatticeVariantLimit = 16; + inline std::vector fuzzy_syllables(const std::string &syllable, metasequoia::FuzzyPinyinOptions options) { using R = metasequoia::FuzzyPinyinRule; @@ -92,4 +106,43 @@ inline std::vector fuzzy_segmentations(const Segments &segments, metas paths.erase(std::remove(paths.begin(), paths.end(), segments), paths.end()); return paths; } + +// 把模糊候选归并进精确候选。group 是候选覆盖的输入长度,大的在前。精确列表的次序原样保留: +// 整句、保护位、调频与纠错裁决的结果都留在原处(不能按 group 重排它——纠错读法的字母比敲的少, +// 重排会把 jiuzheeyang 裁决出的「就这样」压到字面读法下面)。模糊行按 group、权重排好后 +// 逐个插到第一个「比它短」或「同长而权重低于它折后权重」的精确词库行前面。 +template +std::vector merge_fuzzy_candidates(std::vector exact, std::vector fuzzy, Group group) +{ + std::unordered_set seen; + for (const auto &item : exact) + seen.insert(item.word); + fuzzy.erase(std::remove_if(fuzzy.begin(), fuzzy.end(), + [&](const WordItem &item) { return !seen.insert(item.word).second; }), + fuzzy.end()); + std::stable_sort(fuzzy.begin(), fuzzy.end(), [&](const WordItem &a, const WordItem &b) { + const auto ga = group(a), gb = group(b); + return ga != gb ? ga > gb : a.weight > b.weight; + }); + const auto ranked_by_weight = [](const WordItem &item) { + return !item.fuzzy && + (item.source == CandidateSource::Database || item.source == CandidateSource::UserDatabase); + }; + std::vector merged; + merged.reserve(exact.size() + fuzzy.size()); + std::size_t i = 0, j = 0; + while (i < exact.size() || j < fuzzy.size()) + { + bool take_fuzzy = i == exact.size(); + if (!take_fuzzy && j < fuzzy.size() && ranked_by_weight(exact[i])) + { + const auto ge = group(exact[i]), gf = group(fuzzy[j]); + take_fuzzy = ge != gf ? gf > ge + : static_cast(exact[i].weight) < + static_cast(fuzzy[j].weight) * kFuzzyWeightRatio; + } + merged.push_back(std::move(take_fuzzy ? fuzzy[j++] : exact[i++])); + } + return merged; +} } // namespace quanpin diff --git a/engine/quanpin/quanpin_dictionary.cpp b/engine/quanpin/quanpin_dictionary.cpp index 290aeee5..df6082da 100644 --- a/engine/quanpin/quanpin_dictionary.cpp +++ b/engine/quanpin/quanpin_dictionary.cpp @@ -321,6 +321,13 @@ unsigned literal_mixed_match(const WordItem &item, const std::string &raw_input) : 0; } +// 读音就是 key 这条切分的行(整键块)。整句行按敲的读音(pinyin)认:词格整句的 +// canonical_pinyin 是它实际走的读音,开着模糊音时可能与切分不同(min 走成 ming)。 +bool reads_full_key(const WordItem &item, const std::string &key) +{ + return item.canonical_pinyin == key || (item.sentence_association && item.pinyin == key); +} + void keep_literal_mixed_words_first(std::vector &candidates, const std::string &raw_input) { // Stable groups preserve dictionary/user order without manufacturing frequency weights. @@ -673,7 +680,7 @@ std::vector QuanpinDictionary::query_exact(const std::string &raw_inpu std::vector winner_prefix; for (auto &item : query_series(raw_input, winner_key, winner_segments)) { - (item.canonical_pinyin == winner_key ? winner_full : winner_prefix).push_back(std::move(item)); + (reads_full_key(item, winner_key) ? winner_full : winner_prefix).push_back(std::move(item)); } // 贵档胜者没有整词、也没有整句行时(整句来源只开了重排、模型没载入), // 整键块是空的;不能让主切的行顶回首位,由胜者的最长前缀词领衔。 @@ -1246,7 +1253,7 @@ std::vector QuanpinDictionary::arbitrate_literal_segmentations(const s const std::string primary_key = quanpin::join_segments(segments); const std::string best_key = quanpin::join_segments(*best_cut); const auto is_sentence_of = [](const std::string &key) { - return [&key](const WordItem &item) { return item.canonical_pinyin == key; }; + return [&key](const WordItem &item) { return reads_full_key(item, key); }; }; const bool alternative_leads = *best_score > *primary_score && !user_prefers_reading(primary_key); const double gap = alternative_leads ? *best_score - *primary_score : *primary_score - *best_score; @@ -1296,7 +1303,7 @@ std::vector QuanpinDictionary::arbitrate_literal_segmentations(const s std::vector winner_prefix; for (auto &item : query_series(raw_input, best_key, *best_cut)) { - (item.canonical_pinyin == best_key ? winner_full : winner_prefix).push_back(std::move(item)); + (reads_full_key(item, best_key) ? winner_full : winner_prefix).push_back(std::move(item)); } if (winner_full.empty()) { @@ -1319,7 +1326,7 @@ std::vector QuanpinDictionary::arbitrate_literal_segmentations(const s std::vector rest; for (auto &item : result) { - if (item.canonical_pinyin == primary_key) + if (reads_full_key(item, primary_key)) { primary_sentences.push_back(std::move(item)); continue; @@ -1469,7 +1476,9 @@ std::optional QuanpinDictionary::lattice_best_path(const q }; lattice_options.collocation_weight = sentence_association_.collocation_weight; } - const auto lookup = lattice_lookup(lattice_options.span_limit); + // 不带模糊音:这个分用来裁决切分与纠错,模糊读音只给敲定的切分多出整句,不参与投票。 + // 带上它,字面读法会借模糊读音的分压过纠错(jiuzheeyang 的「就这恶言」压过「就这样」)。 + const auto lookup = lattice_lookup(lattice_options.span_limit, false); auto cut_paths = quanpin::decode_word_lattice(cut, lookup, lattice_options); std::optional best; if (!cut_paths.empty()) @@ -1804,15 +1813,20 @@ void QuanpinDictionary::reset_cache() lattice_span_cache_.clear(); } -quanpin::WordLatticeLookup QuanpinDictionary::lattice_lookup(int limit) +quanpin::WordLatticeLookup QuanpinDictionary::lattice_lookup(int limit, bool with_fuzzy) { - return [this, lookup = quanpin::make_lattice_db_lookup(db_, statement_cache_, limit), - limit](const quanpin::Segments &span) { + const auto fuzzy = with_fuzzy ? fuzzy_pinyin_ : metasequoia::FuzzyPinyinOptions{}; + return [this, lookup = quanpin::make_lattice_db_lookup(db_, statement_cache_, limit, fuzzy), limit, + fuzzy](const quanpin::Segments &span) { // 超过上限整体清掉,免得一次长时间输入把内存撑大(同双拼的 direct_span_cache_)。 constexpr std::size_t kLatticeSpanCacheLimit = 8192; std::string key = quanpin::join_segments(span); key.push_back('#'); key += std::to_string(limit); + if (fuzzy.rules) + { + key += "#f"; + } auto found = lattice_span_cache_.find(key); if (found == lattice_span_cache_.end()) { @@ -1826,6 +1840,18 @@ quanpin::WordLatticeLookup QuanpinDictionary::lattice_lookup(int limit) }; } +void QuanpinDictionary::set_fuzzy_pinyin(metasequoia::FuzzyPinyinOptions options) +{ + if (fuzzy_pinyin_.rules == options.rules) + { + return; + } + fuzzy_pinyin_ = options; + // 词格跨度、最佳路径分和按它们排出的候选列表都按旧规则算出。规则只在设置页改, + // 清一次比给每层缓存的键都带上规则简单,也不会让关掉模糊音后读到带模糊行的旧列表。 + reset_cache(); +} + void QuanpinDictionary::reset_sentence_cache() { // 词格整句只在 query_series 里合入,结果只经 series_cache_ 缓存;cache_ 是逐前缀的 @@ -2178,13 +2204,14 @@ std::vector QuanpinDictionary::fuzzy_candidates(const std::string &seg std::vector QuanpinDictionary::query(const std::string &raw_input, const std::string &segmentation, unsigned autocorrect, metasequoia::FuzzyPinyinOptions fuzzy) { + // 词格要按同一套规则收模糊读音,先于 query_exact 下发。 + set_fuzzy_pinyin(fuzzy); auto result = query_exact(raw_input, segmentation, autocorrect); if (fuzzy.rules && !raw_input.empty()) { // Keep ordinary cache slots free of preference-specific candidates. const auto typed = segmentation.empty() ? quanpin::join_segments(resolve_segments(raw_input, segmentation)) : segmentation; - append_unique_words(result, fuzzy_candidates(typed, fuzzy)); // Specificity is how many typed letters a candidate accounts for, not the raw // length of its stored key: alternative cuts ("xi'e" for "xie", "you'di'an" // for "you'dian") and fuzzy variants carry apostrophes, so comparing pinyin @@ -2197,8 +2224,7 @@ std::vector QuanpinDictionary::query(const std::string &raw_input, con } return letters; }; - std::stable_sort(result.begin(), result.end(), - [&](const WordItem &a, const WordItem &b) { return matched_letters(a) > matched_letters(b); }); + result = quanpin::merge_fuzzy_candidates(std::move(result), fuzzy_candidates(typed, fuzzy), matched_letters); } // Labeling runs after every mutation (including the fuzzy merge) so the // returned list and the published candidate list always agree. diff --git a/engine/quanpin/quanpin_dictionary.h b/engine/quanpin/quanpin_dictionary.h index 919c56c1..dc3d47c0 100644 --- a/engine/quanpin/quanpin_dictionary.h +++ b/engine/quanpin/quanpin_dictionary.h @@ -155,8 +155,10 @@ class QuanpinDictionary std::optional lattice_best_path(const quanpin::Segments &cut); // 词格的跨度查询,套一层按(跨度, limit)的缓存:每多敲一个字母整句都要重解,同一输入的 // 几种字面切分、主切与纠错读法之间也大量共用跨度,不缓存就一遍遍打同样的 SQL。结果与 - // 直查逐行相同;随 reset_cache 清空(词库被写或换了开关时)。 - quanpin::WordLatticeLookup lattice_lookup(int limit); + // 直查逐行相同;随 reset_cache 清空(词库被写或换了开关时)。with_fuzzy 为假时不收模糊读音, + // 见 lattice_best_path。 + quanpin::WordLatticeLookup lattice_lookup(int limit, bool with_fuzzy = true); + void set_fuzzy_pinyin(metasequoia::FuzzyPinyinOptions options); bool user_prefers_reading(const std::string &key); static void append_unique_words(std::vector &result, const std::vector &extra); void mark_autocorrect_candidates(std::vector &candidates, const std::string &raw_input); @@ -215,6 +217,8 @@ class QuanpinDictionary neural::LazySentenceModel neural_keyboard_model_; // 整句候选来源与去重补位选项,默认全关;由 set_sentence_association 随请求更新。 SentenceAssociationOptions sentence_association_; + // 词格跨度查询带的模糊音规则,由 query 随请求更新,变化时清缓存(整句与裁决都按它算)。 + metasequoia::FuzzyPinyinOptions fuzzy_pinyin_; // 神经重排的上下文(光标前已上屏的文本)。宿主没给就是空串。 std::string rescoring_context_; std::unordered_map statement_cache_; diff --git a/engine/quanpin/quanpin_query.cpp b/engine/quanpin/quanpin_query.cpp index 530d07b4..178c14ba 100644 --- a/engine/quanpin/quanpin_query.cpp +++ b/engine/quanpin/quanpin_query.cpp @@ -2,10 +2,12 @@ #include "../contracts/dictionary/format.h" #include "quanpin_query.h" +#include "fuzzy_pinyin.h" #include "quanpin_utils.h" #include "../shuangpin/shuangpin_utils.h" #include #include +#include #include #include #include @@ -1280,9 +1282,9 @@ const std::string &canonical_syllable(const std::string &syllable) } WordLatticeLookup make_lattice_db_lookup(sqlite3 *db, std::unordered_map &statement_cache, - int span_limit) + int span_limit, metasequoia::FuzzyPinyinOptions fuzzy) { - return [db, &statement_cache, span_limit](const Segments &span) { + return [db, &statement_cache, span_limit, fuzzy](const Segments &span) { Segments normalized; normalized.reserve(span.size()); for (const auto &syllable : span) @@ -1293,11 +1295,47 @@ WordLatticeLookup make_lattice_db_lookup(sqlite3 *db, std::unordered_map keys{normalized}; + for (auto &variant : fuzzy_segmentations(normalized, fuzzy, kFuzzyLatticeVariantLimit)) + { + for (auto &syllable : variant) + syllable = canonical_syllable(syllable); + keys.push_back(std::move(variant)); + } + // 模糊变体与敲出的读音同批查(同表一条 SQL)。LIMIT 按表计,放宽到每个键一份, + // 免得高频的模糊读音把敲出读音的行挤出去;折算之后再截回 span_limit。 + const int limit = span_limit * static_cast(keys.size()); + const auto rows = query_exact_segmentations_keyed_flat(keys, db, statement_cache, limit); std::vector lexemes; lexemes.reserve(rows.size()); for (const auto &row : rows) - lexemes.push_back({row.key, row.value, row.weight}); + { + LatticeLexeme lexeme{row.key, row.value, row.weight}; + if (keys.size() > 1) + { + // 每个与敲出读音不同的音节扣一份分:一个词里错两个音,比只错一个更不可能。 + const auto syllables = split_segments(row.key); + for (size_t i = 0; i < syllables.size() && i < normalized.size(); ++i) + lexeme.penalty += syllables[i] == normalized[i] ? 0.0 : kFuzzyLatticePenaltyLog; + } + lexemes.push_back(std::move(lexeme)); + } + if (keys.size() > 1) + { + // 按折后权重排序再截断:与 build_graph 一样,参与解码的只是排在前面的 span_limit 行。 + std::stable_sort(lexemes.begin(), lexemes.end(), [](const LatticeLexeme &a, const LatticeLexeme &b) { + return static_cast(a.weight) * std::pow(10.0, -a.penalty) > + static_cast(b.weight) * std::pow(10.0, -b.penalty); + }); + // 同一个字两种读音都收(多音字),只留排在前面的那行。 + std::unordered_set seen; + lexemes.erase( + std::remove_if(lexemes.begin(), lexemes.end(), + [&](const LatticeLexeme &lexeme) { return !seen.insert(lexeme.value).second; }), + lexemes.end()); + if (lexemes.size() > static_cast(std::max(0, span_limit))) + lexemes.resize(static_cast(std::max(0, span_limit))); + } return lexemes; }; } diff --git a/engine/quanpin/quanpin_query.h b/engine/quanpin/quanpin_query.h index 3bca0f8a..b2431386 100644 --- a/engine/quanpin/quanpin_query.h +++ b/engine/quanpin/quanpin_query.h @@ -1,5 +1,6 @@ #pragma once +#include "../core/fuzzy_pinyin_options.h" #include "word_lattice.h" #include @@ -80,7 +81,10 @@ std::vector query_exact_segmentations_keyed_flat( // 词格的跨度查询:只返回键与跨度完全相等的行,没有前缀区间与简拼降级——整句 // 联想出来的每个词,读音都必须精确对上用户敲的那段拼音。因此也不需要 QuerySource: // 区分全拼/双拼的只有简拼那几级。 +// 开着模糊音时同时收跨度的模糊变体:行的 key 是词库里的真实读音,每个与敲出读音不同的音节 +// 在 penalty 上记一份 kFuzzyLatticePenaltyLog。整句的 canonical_pinyin 因此落在真实读音上, +// 选中后造词不会把错音写进词库。 WordLatticeLookup make_lattice_db_lookup(sqlite3 *db, std::unordered_map &statement_cache, - int span_limit); + int span_limit, metasequoia::FuzzyPinyinOptions fuzzy = {}); } // namespace quanpin diff --git a/engine/quanpin/word_lattice.cpp b/engine/quanpin/word_lattice.cpp index 9c1fea04..017f0f07 100644 --- a/engine/quanpin/word_lattice.cpp +++ b/engine/quanpin/word_lattice.cpp @@ -194,11 +194,12 @@ std::vector> build_graph(const Segments &syllables, con } const auto &rows = pins.empty() ? cached->second : constrained_rows; const size_t take = (std::min)(rows.size(), static_cast(std::max(0, options.span_limit))); - // 先验的分母:跨度内实际参与解码的那些行。行数被 span_limit 截断, - // 截掉的都是权重最低的尾巴,对和的影响可以忽略。 - std::int64_t span_total = 0; + // 先验的分母:跨度内实际参与解码、读音相同的那些行。行数被 span_limit 截断, + // 截掉的都是权重最低的尾巴,对和的影响可以忽略。按读音分开求和:模糊音把别的 + // 读音的行混进同一跨度,算进一个分母会把敲出读音的行的占比压低。 + std::unordered_map reading_totals; for (size_t i = 0; i < take; ++i) - span_total += rows[i].weight > 0 ? rows[i].weight : 0; + reading_totals[rows[i].key] += rows[i].weight > 0 ? rows[i].weight : 0; for (size_t i = 0; i < take; ++i) { const auto &row = rows[i]; @@ -212,13 +213,14 @@ std::vector> build_graph(const Segments &syllables, con if (model) { edge.index = model->index(edge.word); - edge.base_score = - dictionary_tiebreak(edge.weight, options) + reading_prior(edge.weight, span_total, options); + edge.base_score = dictionary_tiebreak(edge.weight, options) + + reading_prior(edge.weight, reading_totals[row.key], options); } else { edge.base_score = heuristic_log_prob(edge.weight, end - start, options); } + edge.base_score -= row.penalty; graph[start].push_back(std::move(edge)); } } diff --git a/engine/quanpin/word_lattice.h b/engine/quanpin/word_lattice.h index 4517684d..82221e22 100644 --- a/engine/quanpin/word_lattice.h +++ b/engine/quanpin/word_lattice.h @@ -60,6 +60,8 @@ struct LatticeLexeme std::string key; std::string value; std::int64_t weight = 0; + // 从边分里扣掉的分(与路径分同单位)。模糊读音的行(key 与跨度不同)靠它让位于敲出的读音。 + double penalty = 0.0; }; // 句中辅助码:第 syllable 个音节(0 起,按 decode 收到的 syllables 计)只能解成 accept diff --git a/engine/shuangpin/engine.cpp b/engine/shuangpin/engine.cpp index 48da2e20..c0e4c9f8 100644 --- a/engine/shuangpin/engine.cpp +++ b/engine/shuangpin/engine.cpp @@ -1,10 +1,10 @@ #include "engine.h" +#include "../quanpin/fuzzy_pinyin.h" #include "shuangpin_query.h" #include "shuangpin_utils.h" #include -#include #include namespace @@ -126,6 +126,8 @@ std::vector ShuangpinEngine::query(const QueryRequest &request) : SyllableHelpcodes{}); // 直接辅助码开着时词格的跨度查询走与解析器共用的缓存,见 ShuangpinDictionary::lattice_lookup。 dictionary_.set_direct_span_cache_enabled(request.enable_direct_helpcode); + // 整句同样要收模糊读音,见 QuanpinDictionary::set_fuzzy_pinyin。 + dictionary_.set_fuzzy_pinyin(request.fuzzy_pinyin); auto candidates = query_unfiltered(request); dictionary_.filter_by_syllable_helpcodes(candidates); return candidates; @@ -139,6 +141,7 @@ bool ShuangpinEngine::resolve_direct_helpcode(QueryRequest &request) } // 解码要用的搭配模型跟着整句选项走,先下发,和 query 的顺序一致。 dictionary_.set_sentence_association(request.sentence_association); + dictionary_.set_fuzzy_pinyin(request.fuzzy_pinyin); return dictionary_.resolve_direct_helpcode(request); } @@ -268,13 +271,6 @@ std::vector ShuangpinEngine::append_fuzzy(std::vector exact, fuzzy = HelpcodeUtils::filter_candidates_with_double_helpcodes(fuzzy, helpcodes, helpcodes_.get()); else if (helpcodes.size() == 1) fuzzy = HelpcodeUtils::reorder_candidates_with_single_helpcode(fuzzy, helpcodes, helpcodes_.get()); - std::unordered_set seen; - for (const auto &item : exact) - seen.insert(item.word); - for (auto &item : fuzzy) - if (seen.insert(item.word).second) - exact.push_back(std::move(item)); - std::stable_sort(exact.begin(), exact.end(), - [](const auto &a, const auto &b) { return a.pinyin.size() > b.pinyin.size(); }); - return exact; + return quanpin::merge_fuzzy_candidates(std::move(exact), std::move(fuzzy), + [](const WordItem &item) { return item.pinyin.size(); }); } diff --git a/engine/shuangpin/shuangpin_dictionary.cpp b/engine/shuangpin/shuangpin_dictionary.cpp index ad16b1ad..bf7ff0af 100644 --- a/engine/shuangpin/shuangpin_dictionary.cpp +++ b/engine/shuangpin/shuangpin_dictionary.cpp @@ -1186,9 +1186,19 @@ void ShuangpinDictionary::reset_cache() } } +void ShuangpinDictionary::set_fuzzy_pinyin(metasequoia::FuzzyPinyinOptions options) +{ + if (fuzzy_pinyin_.rules == options.rules) + { + return; + } + fuzzy_pinyin_ = options; + reset_cache(); +} + quanpin::WordLatticeLookup ShuangpinDictionary::lattice_lookup(int limit) { - auto lookup = quanpin::make_lattice_db_lookup(quanpin_db_, quanpin_statement_cache_, limit); + auto lookup = quanpin::make_lattice_db_lookup(quanpin_db_, quanpin_statement_cache_, limit, fuzzy_pinyin_); if (!direct_span_cache_enabled_) { return lookup; diff --git a/engine/shuangpin/shuangpin_dictionary.h b/engine/shuangpin/shuangpin_dictionary.h index 311f3593..709f4970 100644 --- a/engine/shuangpin/shuangpin_dictionary.h +++ b/engine/shuangpin/shuangpin_dictionary.h @@ -95,6 +95,8 @@ class ShuangpinDictionary { direct_span_cache_enabled_ = enabled; } + // 词格跨度查询带的模糊音规则,见 QuanpinDictionary::set_fuzzy_pinyin。变化时清缓存。 + void set_fuzzy_pinyin(metasequoia::FuzzyPinyinOptions options); explicit ShuangpinDictionary(const ShuangpinProfile &profile = GetXiaoheShuangpinProfile(), metasequoia::RuntimePaths paths = metasequoia::RuntimePaths::legacy()); @@ -139,6 +141,7 @@ class ShuangpinDictionary // (reset_cache)缓存随之清空。 quanpin::WordLatticeLookup lattice_lookup(int limit); bool direct_span_cache_enabled_ = false; + metasequoia::FuzzyPinyinOptions fuzzy_pinyin_; std::unordered_map> direct_span_cache_; void reset_cache_if_database_changed(); // 这个字能不能落在 helpcode 约束的音节上。 diff --git a/engine/tests/src/eval_quanpin_sentence.cpp b/engine/tests/src/eval_quanpin_sentence.cpp index a6028792..c09a03e6 100644 --- a/engine/tests/src/eval_quanpin_sentence.cpp +++ b/engine/tests/src/eval_quanpin_sentence.cpp @@ -3,7 +3,7 @@ // // 不是测试:不断言,也不进 ctest。改切分或整句打分前后各跑一遍对比,例如 // set METASEQUOIA_IME_DATA_DIR=<数据目录的副本> -// eval_quanpin_sentence engine\tests\data\quanpin_sentence_cases.tsv [repeats] [lattice] +// eval_quanpin_sentence engine\tests\data\quanpin_sentence_cases.tsv [repeats] [lattice] [fuzzy=] // 只敲键和重置,从不选词,不写用户日志;仍请指向副本,别用正在用的数据目录。 #include "../../core/data_path.h" #include "../../core/input_session.h" @@ -71,7 +71,7 @@ int main(int argc, char **argv) { if (argc < 2) { - std::fprintf(stderr, "usage: eval_quanpin_sentence [repeats] [lattice]\n"); + std::fprintf(stderr, "usage: eval_quanpin_sentence [repeats] [lattice] [fuzzy=]\n"); return 2; } const auto cases = load_cases(argv[1]); @@ -79,7 +79,17 @@ int main(int argc, char **argv) // 与出厂默认一致:词格整句 + 神经速度档 + 万象搭配加分(模型在数据目录里时)。 // 第 3 个参数写 lattice 时只开词格,用来把神经重排的耗时单独拆出来。 - const bool lattice_only = argc > 3 && std::string(argv[3]) == "lattice"; + // 任意位置写 fuzzy=<规则位>(如 fuzzy=0x7ff)时带模糊音跑,看合法输入的首选有没有被模糊读音抢走。 + bool lattice_only = false; + metasequoia::FuzzyPinyinOptions fuzzy; + for (int i = 3; i < argc; ++i) + { + const std::string arg = argv[i]; + if (arg == "lattice") + lattice_only = true; + else if (arg.rfind("fuzzy=", 0) == 0) + fuzzy.rules = static_cast(std::strtoul(arg.c_str() + 6, nullptr, 0)); + } SentenceAssociationOptions options; options.word_lattice = true; options.neural_keyboard = !lattice_only; @@ -95,7 +105,8 @@ int main(int argc, char **argv) { metasequoia::InputSession session(SchemeType::Quanpin, setting.autocorrect_types); session.set_sentence_association(options); - std::printf("== %s ==\n", setting.name); + session.set_fuzzy_pinyin_options(fuzzy); + std::printf("== %s%s ==\n", setting.name, fuzzy.rules ? ", fuzzy on" : ""); std::map> by_category; std::vector per_key; for (const Case &item : cases) diff --git a/engine/tests/src/test_fuzzy_pinyin.cpp b/engine/tests/src/test_fuzzy_pinyin.cpp index ad145b14..b4761647 100644 --- a/engine/tests/src/test_fuzzy_pinyin.cpp +++ b/engine/tests/src/test_fuzzy_pinyin.cpp @@ -119,6 +119,20 @@ int main() insert_weighted("jian", "检", 500000); insert_weighted("ji'an", "吉安", 766925); insert_weighted("ji'an", "积案", 9420); + // 混排:权重取自出货词库。min 的生僻字权重为 0,旧实现把 ming 整组垫在它们后面。 + insert_weighted("min", "民", 902697); + insert_weighted("min", "敏", 196615); + insert_weighted("min", "缗", 0); + insert_weighted("min", "黾", 0); + insert_weighted("ming", "明", 2864492); + insert_weighted("ming", "名", 1804559); + insert_weighted("ming", "瞑", 6405); + // 整句:lin'yi'bian 没有整词,词格要走模糊读音的 ling'yi(另一)才拼得出「另一边」。 + insert_weighted("lin", "林", 1435296); + insert_weighted("ling", "另", 608814); + insert_weighted("yi", "一", 5000000); + insert_weighted("bian", "边", 800000); + insert_weighted("ling'yi", "另一", 50000); sqlite3_close(db); std::filesystem::create_directories(directory / "helpcodes"); std::ofstream(directory / "helpcodes" / "helpcode.txt") << "中=ab\n宗=cd\n国=ef\n"; @@ -155,6 +169,29 @@ int main() const auto youdian_list = dictionary.query("youdian", "you'dian", 0u, fuzzy_on); require(youdian_list.at(0).word == "邮电" && position(youdian_list, "尤迪安") > 0, "longer alternative key outranked the exact reading"); + // 模糊读音按折后权重与敲出的读音混排:常用的「明」「名」进到「敏」前面,生僻字垫底, + // 敲出读音的首字「民」不让位。 + const FuzzyPinyinOptions in_ing{static_cast(FuzzyPinyinRule::IN_ING)}; + const auto min_list = dictionary.query("min", "min", 0u, in_ing); + require(min_list.at(0).word == "民", "exact reading lost the lead"); + require(position(min_list, "明") < position(min_list, "敏") && + position(min_list, "名") < position(min_list, "敏"), + "frequent fuzzy reading was not interleaved"); + require(position(min_list, "明") < position(min_list, "缗") && + position(min_list, "敏") < position(min_list, "瞑"), + "fuzzy merge ignored weights"); + require(min_list.at(position(min_list, "明")).fuzzy && !min_list.at(0).fuzzy, "fuzzy flag lost"); + // 整句:词格收模糊读音的跨度,canonical 记真实读音,pinyin 仍是敲的。 + SentenceAssociationOptions lattice_only; + lattice_only.word_lattice = true; + dictionary.set_sentence_association(lattice_only); + const auto sentence_list = dictionary.query("linyibian", "lin'yi'bian", 0u, in_ing); + const auto &sentence = sentence_list.at(position(sentence_list, "另一边")); + require(sentence.pinyin == "lin'yi'bian" && sentence.canonical_pinyin == "ling'yi'bian", + "fuzzy sentence identity lost"); + require(!contains(dictionary.query("linyibian", "lin'yi'bian", 0u), "另一边"), + "fuzzy lattice leaked into exact cache"); + dictionary.set_sentence_association({}); require(quanpin::fuzzy_syllables("zh", {0x7ff}) == std::vector{"zh"}, "incomplete initial changed"); require(quanpin::fuzzy_syllables("bian", {1u << 6}) == std::vector{"bian"},