From a60e2948dbba29f32bec58ee4bf44d68ce953a0b Mon Sep 17 00:00:00 2001 From: "Lu Fan(SonnyCalcr)" <1730976608@qq.com> Date: Tue, 20 Oct 2026 08:47:27 +0800 Subject: [PATCH] =?UTF-8?q?fix(engine):=20=E6=95=B4=E5=8F=A5=E8=81=94?= =?UTF-8?q?=E6=83=B3=E6=8C=89=20P(=E8=AF=BB=E9=9F=B3|=E5=AD=97)=20?= =?UTF-8?q?=E5=8E=8B=E4=BD=8F=E5=A4=9A=E9=9F=B3=E5=AD=97=E7=9A=84=E7=94=9F?= =?UTF-8?q?=E5=83=BB=E8=AF=BB=E9=9F=B3=20(#1123)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit tai'zhuan'qian'le 出「大赚钱了」：大 也读 tai，词库里这一行权重 4719，占 tai 的 0.47%，已有的 reading_prior（看 P(字|拼音)，1% 免罚线）只扣 0.33，翻不过语言模型 对「大赚」约 1.6 的偏好。换个角度看，大 读 tai 只占它全部权重的 0.14%——这才是 「这个字几乎不这么读」，语言模型只认汉字，这一维要由读音补上。 词格跨度查询对单字行加一项 P(读音|字) 惩罚：占比低于 10% 的部分按 log10 扣分， 上限 2.5。长 chang、重 chong、行 hang 这类常见第二读音不扣；大 tai/dai、会 kuai、 都 du、给 ji 扣 1.5~2。只罚单字，词本身就是读音的证据（会计、成都、大夫）。多音字 各读音的权重和在第一次整句联想时扫一遍单字表载入，全拼与双拼（含直接辅助码）共用。 自建 111 句多音字评测从 79 提到 82，无回归；5% 档翻不过「大赚钱了」，20% 与 10% 结果相同，取保守的一边。test_engine_smoke 加固定权重的断言，句子评测集补两条。 --- engine/quanpin/quanpin_dictionary.cpp | 8 +- engine/quanpin/quanpin_dictionary.h | 3 + engine/quanpin/quanpin_query.cpp | 94 +++++++++++++++++++- engine/quanpin/quanpin_query.h | 13 ++- engine/shuangpin/shuangpin_dictionary.cpp | 7 +- engine/shuangpin/shuangpin_dictionary.h | 2 + engine/tests/data/quanpin_sentence_cases.tsv | 2 + engine/tests/src/test_engine_smoke.cpp | 50 +++++++++++ 8 files changed, 172 insertions(+), 7 deletions(-) diff --git a/engine/quanpin/quanpin_dictionary.cpp b/engine/quanpin/quanpin_dictionary.cpp index 99184e28..5a4bdb26 100644 --- a/engine/quanpin/quanpin_dictionary.cpp +++ b/engine/quanpin/quanpin_dictionary.cpp @@ -1890,8 +1890,12 @@ void QuanpinDictionary::reset_cache() quanpin::WordLatticeLookup QuanpinDictionary::lattice_lookup(int limit, bool with_fuzzy) { const auto fuzzy = with_fuzzy ? fuzzy_pinyin_ : metasequoia::FuzzyPinyinOptions{}; - return [this, lookup = quanpin::make_lattice_db_lookup(db_, statement_cache_, limit, fuzzy), limit, - fuzzy](const quanpin::Segments &span) { + if (!char_reading_totals_) + { + char_reading_totals_ = quanpin::load_char_reading_totals(db_); + } + return [this, lookup = quanpin::make_lattice_db_lookup(db_, statement_cache_, limit, fuzzy, char_reading_totals_), + limit, fuzzy](const quanpin::Segments &span) { // 超过上限整体清掉，免得一次长时间输入把内存撑大（同双拼的 direct_span_cache_）。 constexpr std::size_t kLatticeSpanCacheLimit = 8192; std::string key = quanpin::join_segments(span); diff --git a/engine/quanpin/quanpin_dictionary.h b/engine/quanpin/quanpin_dictionary.h index 3e6c5140..e5de70d8 100644 --- a/engine/quanpin/quanpin_dictionary.h +++ b/engine/quanpin/quanpin_dictionary.h @@ -204,6 +204,9 @@ class QuanpinDictionary CircularBuffer> lattice_best_path_cache_; // 见 lattice_lookup。 std::unordered_map> lattice_span_cache_; + // 多音单字各读音的权重和，词格第一次查跨度时载入，此后不随 reset_cache 重载。 + // 见 quanpin::load_char_reading_totals。 + std::shared_ptr char_reading_totals_; sqlite3 *db_ = nullptr; sqlite3_int64 data_version_ = -1; metasequoia::RuntimePaths paths_; diff --git a/engine/quanpin/quanpin_query.cpp b/engine/quanpin/quanpin_query.cpp index 1fdf1c26..0af2778a 100644 --- a/engine/quanpin/quanpin_query.cpp +++ b/engine/quanpin/quanpin_query.cpp @@ -1286,10 +1286,96 @@ std::vector query_exact_segmentations_keyed_flat( return result; } -WordLatticeLookup make_lattice_db_lookup(sqlite3 *db, std::unordered_map &statement_cache, - int span_limit, metasequoia::FuzzyPinyinOptions fuzzy) +std::shared_ptr load_char_reading_totals(sqlite3 *db) { - return [db, &statement_cache, span_limit, fuzzy](const Segments &span) { + auto totals = std::make_shared(); + if (db == nullptr) + return totals; + + std::vector tables; + sqlite3_stmt *statement = nullptr; + if (sqlite3_prepare_v2( + db, "SELECT name FROM sqlite_master WHERE type = 'table' AND name LIKE 'tbl\\_1\\_%' ESCAPE '\\'", -1, + &statement, nullptr) != SQLITE_OK) + return totals; + while (sqlite3_step(statement) == SQLITE_ROW) + { + const auto *name = reinterpret_cast(sqlite3_column_text(statement, 0)); + if (name != nullptr) + tables.emplace_back(name); + } + sqlite3_finalize(statement); + + // 先按字累计权重和读音数，再只留多音字：单读音的字占比恒为 1，存着只占内存。 + std::unordered_map> sums; + for (const auto &table : tables) + { + const std::string sql = "SELECT value, weight FROM " + table; + if (sqlite3_prepare_v2(db, sql.c_str(), -1, &statement, nullptr) != SQLITE_OK) + continue; + while (sqlite3_step(statement) == SQLITE_ROW) + { + const auto *value = reinterpret_cast(sqlite3_column_text(statement, 0)); + if (value == nullptr) + continue; + auto &[sum, readings] = sums[value]; + sum += (std::max)(sqlite3_column_int64(statement, 1), sqlite3_int64{0}); + ++readings; + } + sqlite3_finalize(statement); + } + for (const auto &[value, sum] : sums) + { + if (sum.second > 1) + totals->emplace(value, sum.first); + } + return totals; +} + +namespace +{ + +// P(读音|字) 的免罚线与扣分上限（log10）。 +// +// 已有的 reading_prior 看的是 P(字|拼音)：这一行占该拼音全部行的权重比。它分得出权重为 0 的 +// 「卷」读 gun，却分不出「大」读 tai——大 的 tai 行有 4719，占 tai 的 0.47%，只扣 0.33，翻不过 +// 语言模型对「大赚」的偏好（差 1.6 左右），tai'zhuan'qian'le 于是出「大赚钱了」。同一行换个 +// 角度看，大 的 da 行有 327 万，读 tai 的只占 0.14%：这才是「这个字几乎不这么读」。语言模型 +// 只认汉字，这一维只能由读音补上（噪声信道里的 P(拼音|字)）。 +// +// 免罚线定在 10%：长 chang 42%、重 chong 20%、行 hang 15% 这类常见的第二读音一分不扣， +// 觉 jiao、还 huan（8%~9%）只扣零点零几，着 zhao/zhuo、地 di（3%~4%）扣 0.4~0.5。真正的 +// 生僻读音——大 tai/dai、会 kuai、都 du、给 ji——都在 0.1%~0.3%，扣 1.5~2。了 liao（0.47%） +// 扣 1.3，「受不了」「去不了」仍靠语言模型的上下文赢下 料/聊。5% 那一档还差一点翻不过 +// 「大赚钱了」，20% 在评测句上与 10% 结果相同，取保守的一边。上限只管权重为 0 的读音，那些 +// reading_prior 已经扣过一大截。 +// +// 只罚单字：多音字的少见读音落在词里（会计、成都、大夫）时整词就是读音的证据。没有同音 +// 竞争的字（得 只有它一个读 dei）各条路径扣得一样多，排序不受影响。 +constexpr double kRareCharReadingShare = 0.1; +constexpr double kRareCharReadingFloor = 2.5; + +double rare_char_reading_penalty(const LatticeLexeme &lexeme, const CharReadingTotals *totals) +{ + if (totals == nullptr) + return 0.0; + const auto found = totals->find(lexeme.value); + if (found == totals->end() || found->second <= 0) + return 0.0; + const double weight = lexeme.weight > 0 ? static_cast(lexeme.weight) : 0.0; + const double share = (weight + 1.0) / (static_cast(found->second) + 1.0); + if (share >= kRareCharReadingShare) + return 0.0; + return (std::min)(-std::log10(share / kRareCharReadingShare), kRareCharReadingFloor); +} + +} // namespace + +WordLatticeLookup make_lattice_db_lookup(sqlite3 *db, std::unordered_map &statement_cache, + int span_limit, metasequoia::FuzzyPinyinOptions fuzzy, + std::shared_ptr char_totals) +{ + return [db, &statement_cache, span_limit, fuzzy, char_totals = std::move(char_totals)](const Segments &span) { // ü 的两种写法归一到词库存的那一种。intact_pinyin_list() 两种都认，可词库只存 // 一种：切出来的 jv、lue 这类音节一行都查不到，精确键查询查空，整个词格就断在 // 这里——wo'men'lue'de 解不出任何路径，长句联想直接消失。这不是模糊音降级，lue @@ -1320,6 +1406,8 @@ WordLatticeLookup make_lattice_db_lookup(sqlite3 *db, std::unordered_map 1) { // 每个与敲出读音不同的音节扣一份分：一个词里错两个音，比只错一个更不可能。 diff --git a/engine/quanpin/quanpin_query.h b/engine/quanpin/quanpin_query.h index 28e0c0b6..105832c5 100644 --- a/engine/quanpin/quanpin_query.h +++ b/engine/quanpin/quanpin_query.h @@ -5,6 +5,7 @@ #include +#include #include #include #include @@ -85,13 +86,23 @@ std::vector query_exact_segmentations_keyed_flat( const std::vector &segmentations, sqlite3 *db, std::unordered_map &statement_cache, int limit = 128); +// 多音单字在各读音下的权重和（单字 → Σweight），只收词库里挂在两个以上读音下的字。 +// 词格用它算 P(读音|字)，见 make_lattice_db_lookup。 +using CharReadingTotals = std::unordered_map; +// 扫一遍全部单音节表（五万行量级）。词库被学习写入时不必重载：调频改动的权重相对 +// 出厂权重的量级可以忽略，这里只要分出「大」读 da 还是读 tai 这种数量级上的差别。 +std::shared_ptr load_char_reading_totals(sqlite3 *db); + // 词格的跨度查询：只返回键与跨度完全相等的行，没有前缀区间与简拼降级——整句 // 联想出来的每个词，读音都必须精确对上用户敲的那段拼音。因此也不需要 QuerySource： // 区分全拼/双拼的只有简拼那几级。 // 开着模糊音时同时收跨度的模糊变体：行的 key 是词库里的真实读音，每个与敲出读音不同的音节 // 在 penalty 上记一份 kFuzzyLatticePenaltyLog。整句的 canonical_pinyin 因此落在真实读音上， // 选中后造词不会把错音写进词库。 +// 给了 char_totals 时，单字行按这个字少见的读音在 penalty 上扣分（见 quanpin_query.cpp 的 +// rare_char_reading_penalty）。 WordLatticeLookup make_lattice_db_lookup(sqlite3 *db, std::unordered_map &statement_cache, - int span_limit, metasequoia::FuzzyPinyinOptions fuzzy = {}); + int span_limit, metasequoia::FuzzyPinyinOptions fuzzy = {}, + std::shared_ptr char_totals = nullptr); } // namespace quanpin diff --git a/engine/shuangpin/shuangpin_dictionary.cpp b/engine/shuangpin/shuangpin_dictionary.cpp index bee643e8..5ea5cf69 100644 --- a/engine/shuangpin/shuangpin_dictionary.cpp +++ b/engine/shuangpin/shuangpin_dictionary.cpp @@ -1205,7 +1205,12 @@ void ShuangpinDictionary::set_fuzzy_pinyin(metasequoia::FuzzyPinyinOptions optio quanpin::WordLatticeLookup ShuangpinDictionary::lattice_lookup(int limit) { - auto lookup = quanpin::make_lattice_db_lookup(quanpin_db_, quanpin_statement_cache_, limit, fuzzy_pinyin_); + if (!char_reading_totals_) + { + char_reading_totals_ = quanpin::load_char_reading_totals(quanpin_db_); + } + auto lookup = quanpin::make_lattice_db_lookup(quanpin_db_, quanpin_statement_cache_, limit, fuzzy_pinyin_, + char_reading_totals_); if (!direct_span_cache_enabled_) { return lookup; diff --git a/engine/shuangpin/shuangpin_dictionary.h b/engine/shuangpin/shuangpin_dictionary.h index 9d9dc1b3..c35941c7 100644 --- a/engine/shuangpin/shuangpin_dictionary.h +++ b/engine/shuangpin/shuangpin_dictionary.h @@ -145,6 +145,8 @@ class ShuangpinDictionary bool direct_span_cache_enabled_ = false; metasequoia::FuzzyPinyinOptions fuzzy_pinyin_; std::unordered_map> direct_span_cache_; + // 见 QuanpinDictionary::char_reading_totals_。 + std::shared_ptr char_reading_totals_; void reset_cache_if_database_changed(); // 这个字能不能落在 helpcode 约束的音节上。 bool accepts_syllable_char(const SyllableHelpcode &helpcode, const std::string &hanzi) const; diff --git a/engine/tests/data/quanpin_sentence_cases.tsv b/engine/tests/data/quanpin_sentence_cases.tsv index 5839bd51..1bf0f43e 100644 --- a/engine/tests/data/quanpin_sentence_cases.tsv +++ b/engine/tests/data/quanpin_sentence_cases.tsv @@ -30,6 +30,8 @@ xianzaijiuqu 现在就去 plain ganxienidebangzhu 感谢你的帮助 plain anzhaoguiding 按照规定 plain tiananmenguangchang 天安门广场 plain +taizhuanqianle 太赚钱了 plain 大 也读 tai，但只占它全部权重的 0.14%；语言模型偏爱「大赚」，靠 P(读音|字) 压回去 +womendouquguochengdu 我们都去过成都 plain 都 读 du 是生僻读音，「去过程都」靠同一条读音惩罚压回去 zaijinuta 再激怒他 plain 贪心切分在 jin 上卡死的合法输入；再/在 两可，只盯切分不回退 fangan 方案 word danganguan 档案馆 word diff --git a/engine/tests/src/test_engine_smoke.cpp b/engine/tests/src/test_engine_smoke.cpp index a5c2e531..8f630f6c 100644 --- a/engine/tests/src/test_engine_smoke.cpp +++ b/engine/tests/src/test_engine_smoke.cpp @@ -3,6 +3,7 @@ #include "../../english/english_dictionary.h" #include "../../japanese/romaji_converter.h" #include "../../neural/neural_decoder.h" +#include "../../quanpin/quanpin_query.h" #include "../../quanpin/word_lattice.h" #include "../../user_dictionary/user_dictionary_journal.h" #include "test_directory_cleanup.h" @@ -88,6 +89,55 @@ int run_test() throw std::runtime_error("Kana conversion changed during the platform port."); } + { + // 多音字的生僻读音：大 读 tai 只占 大 全部权重的 0.14%，单字行要在 penalty 上扣分， + // 否则 tai'zhuan'qian'le 会被语言模型带成「大赚钱了」。太/台 只有这一个读音，不扣； + // 长 读 chang 占四成，是常见读音，也不扣。权重取自出厂词库。 + sqlite3 *db = nullptr; + if (sqlite3_open(":memory:", &db) != SQLITE_OK) + throw std::runtime_error("Failed to open the in-memory dictionary."); + const char *fixture = "CREATE TABLE tbl_1_d(key TEXT,jp TEXT,value TEXT,weight INTEGER);" + "CREATE TABLE tbl_1_t(key TEXT,jp TEXT,value TEXT,weight INTEGER);" + "CREATE TABLE tbl_1_c(key TEXT,jp TEXT,value TEXT,weight INTEGER);" + "CREATE TABLE tbl_1_z(key TEXT,jp TEXT,value TEXT,weight INTEGER);" + "INSERT INTO tbl_1_d VALUES('da','d','大',3273479),('dai','d','大',4993);" + "INSERT INTO tbl_1_t VALUES('tai','t','太',768757),('tai','t','台',51622)," + "('tai','t','大',4719);" + "INSERT INTO tbl_1_c VALUES('chang','c','长',340269);" + "INSERT INTO tbl_1_z VALUES('zhang','z','长',462258);"; + if (sqlite3_exec(db, fixture, nullptr, nullptr, nullptr) != SQLITE_OK) + throw std::runtime_error("Failed to build the polyphone fixture."); + + const auto totals = quanpin::load_char_reading_totals(db); + if (totals->size() != 2 || totals->count("大") != 1 || totals->count("长") != 1) + throw std::runtime_error("Reading totals must keep exactly the polyphonic characters."); + + std::unordered_map statements; + const auto penalty_of = [&](const std::string &syllable, const std::string &word) { + const auto lookup = quanpin::make_lattice_db_lookup(db, statements, 32, {}, totals); + for (const auto &row : lookup({syllable})) + { + if (row.value == word) + return row.penalty; + } + throw std::runtime_error("Polyphone fixture row missing: " + word); + }; + const double rare = penalty_of("tai", "大"); + if (rare < 1.5 || penalty_of("tai", "太") != 0.0 || penalty_of("tai", "台") != 0.0 || + penalty_of("chang", "长") != 0.0 || penalty_of("da", "大") != 0.0) + throw std::runtime_error("Rare polyphone readings must be penalized, common ones must not: " + + std::to_string(rare)); + // 不给 totals 时保持旧行为，一分不扣。 + for (const auto &row : quanpin::make_lattice_db_lookup(db, statements, 32)({"tai"})) + { + if (row.penalty != 0.0) + throw std::runtime_error("Lookup without reading totals must not penalize."); + } + for (auto &[sql, statement] : statements) + sqlite3_finalize(statement); + sqlite3_close(db); + } + { // Neural-only mode still asks the lattice for twelve internal alternatives, but it must not // expose the lattice's own pick while the asynchronous neural result is pending.