From e32eeadec970fd35655352002dc0e592bae7e33f Mon Sep 17 00:00:00 2001 From: fanlusky <1730976608@qq.com> Date: Sat, 19 Sep 2026 22:14:57 +0800 Subject: [PATCH] =?UTF-8?q?fix(engine):=20=E8=B0=83=E9=A2=91=E6=AF=94?= =?UTF-8?q?=E8=BE=83=E9=9B=86=E8=B7=A8=E9=94=AE=E5=90=88=E5=B9=B6=EF=BC=8C?= =?UTF-8?q?=E6=8C=89=E6=9D=83=E9=87=8D=E8=80=8C=E9=9D=9E=E6=98=BE=E7=A4=BA?= =?UTF-8?q?=E9=A1=BA=E5=BA=8F=E5=8F=96=E5=9F=BA=E5=87=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 打 jian 时列表里混着另一种切分 ji'an 的词。吉安 在出货词库里权重是 1，压在同组的 积案(9420)/几案(5999) 底下，而 adjust_candidate_ranking 按音节数圈比较集，于是 吉安 只跟 ji'an 这一组比，可学权重封顶一万出头，永远追不上 见 的 3460998——用户 选多少次，这个候选都纹丝不动。 音节数这个尺度是 #400 加的，为的是挡住另一个 bug：显示列表里备选切分词会被保护位 顶到最前面，中点数学却假定列表按权重降序，于是拿 西鄂(w=6) 当基准把 写 写成 506。 根子在「显示顺序 != 权重顺序」，按音节数滤掉跨表行只是绕开了症状，代价是把每个候选 关进自己的组里。 改成在比较集上直接按权重降序排一次，不变式在源头恢复：任何行都不可能坐在高于自身 权重的位置上充当基准，西鄂 那条路被堵死；同时比较集不再按音节数切开，吉安 的基准 取到整个列表最重的 见，置顶一次即越过，promote 模式连选五次一路走到首位。写入仍只 落在 entry_key 的行上，#36 的护栏不变。 test_user_dictionary_journal 新增两条回归（跨键置顶、连续 promote 收敛到首位）， #400 的两条断言原样保留并仍然通过。 --- .../user_dictionary_journal.cpp | 49 ++++--- .../src/test_user_dictionary_journal.cpp | 127 +++++++++++++++++- 2 files changed, 152 insertions(+), 24 deletions(-) diff --git a/engine/user_dictionary/user_dictionary_journal.cpp b/engine/user_dictionary/user_dictionary_journal.cpp index 2732e92b..a687aa07 100644 --- a/engine/user_dictionary/user_dictionary_journal.cpp +++ b/engine/user_dictionary/user_dictionary_journal.cpp @@ -917,33 +917,42 @@ bool adjust_candidate_ranking(const std::string &main_db_path, const std::string } } - // Single-letter and jianpin contexts show several entry keys in one list - // (context "y" mixes 一/yi with 有/you, nine-key digit contexts mix whole - // single-char tables). Rank within the selected word's own scale: ranking - // against only the rows sharing entry_key made the selection permanently - // rank 0 and no weight was ever written, but ranking against the whole - // visible list lets different weight scales mix (single-char corpus counts - // vs phrase weights). A promoted alternative-segmentation word at the top - // of the list then became the weight basis and the midpoint math wrote - // garbage (西鄂 w=6 as top of "xie" turned 写 into 506). The scale is the - // syllable count: tables of equal length stay comparable. Writes still go - // to entry_key rows only, so a rebalance can never land on another key's - // row the way it did in #36. - const auto ranking_scale = [kind](const std::string &key) -> std::size_t { - return kind == DictionaryKind::Wubi ? 0 : pinyin_segments(key).size(); - }; - const std::size_t entry_scale = ranking_scale(entry_key); + // One list mixes several entry keys: single-letter and jianpin contexts show + // 一/yi next to 有/you, nine-key digits mix whole single-char tables, and a + // re-segmentation shows 吉安 (ji'an) inside the jian list. All of them have to + // share one scale, because the user's 调频 is "move this above what I can see". + // + // The midpoint math below assumes this vector is sorted by weight, descending: + // it brackets the target position between its neighbours' weights and splits + // the gap. The displayed order does NOT satisfy that -- an alternative + // segmentation takes a protected slot near the top and pinned candidates are + // hoisted to fixed positions, both regardless of weight. Ranking on display + // order is what wrote 写 as 506 (= 西鄂's weight 6 + 500) in #400. Sorting by + // weight fixes that at its source, so no weight basis can ever come from a row + // sitting above its own weight. + // + // #400 instead restricted the set to keys with the same syllable count. That + // also kept 西鄂 out, but it walled every candidate into its own group: 吉安 + // (ji'an, w=1) could only ever be compared against 积案/几案/急案/即按, so its + // learnable weight was capped around 10k and it could never pass 见 (jian, + // w=3460998) no matter how many times the user picked it. Weight order removes + // the wall without bringing the bug back. Writes still go to entry_key rows + // only, so a rebalance cannot land on another key's row the way it did in #36. std::vector database_candidates; - std::vector owns_entry_key; for (const auto &item : ordered_candidates) { if (item.source != CandidateSource::Database && item.source != CandidateSource::UserDatabase) continue; + database_candidates.push_back(item); + } + std::stable_sort(database_candidates.begin(), database_candidates.end(), + [](const WordItem &lhs, const WordItem &rhs) { return lhs.weight > rhs.weight; }); + std::vector owns_entry_key; + owns_entry_key.reserve(database_candidates.size()); + for (const auto &item : database_candidates) + { const std::string item_key = kind == DictionaryKind::Wubi ? item.pinyin : candidate_dictionary_key(item, context_key); - if (entry_scale != 0 && ranking_scale(item_key) != entry_scale) - continue; - database_candidates.push_back(item); owns_entry_key.push_back(item_key == entry_key); } const auto selected = std::find_if(database_candidates.begin(), database_candidates.end(), diff --git a/server/tests/src/test_user_dictionary_journal.cpp b/server/tests/src/test_user_dictionary_journal.cpp index 9a8dbf67..cf0d0a50 100644 --- a/server/tests/src/test_user_dictionary_journal.cpp +++ b/server/tests/src/test_user_dictionary_journal.cpp @@ -3,6 +3,8 @@ #include "engine/user_dictionary/user_dictionary_journal.h" #include +#include +#include #include #include #include @@ -404,11 +406,11 @@ TEST_CASE(UserDictionaryPromoteDoesNotCrushPrefixSinglesFromSeriesQuery) std::filesystem::remove_all(directory); } -// 回归：调频的权重基准只能取与选中词同尺度（同音节数）的候选。备选切分词（xi'e 的 -// 西鄂 w=6）在旧显示排序下会坐在 xie 列表最前面，拿它当基准会把写（原 605147）写成 +// 回归：调频的权重基准必须取自按权重排好序的比较集，不能按显示顺序取。备选切分词 +// （xi'e 的西鄂 w=6）会坐在 xie 显示列表最前面，拿它当基准会把写（原 605147）写成 // 6+500=506，权重反而低于些/血/谢——这就是用户机器上「置顶写之后写还是不在前排」的根因。 // 同音节的多个表（九宫格数字上下混多个单字表）仍互相比权，不受影响。 -TEST_CASE(UserDictionaryPromoteBasesWeightOnTheSelectedCandidatesOwnScale) +TEST_CASE(UserDictionaryPromoteBasesWeightOnTheHeaviestCandidateNotTheTopmost) { const auto directory = std::filesystem::temp_directory_path() / ("msime-cross-table-ranking-" + std::to_string(GetCurrentProcessId())); @@ -458,6 +460,121 @@ TEST_CASE(UserDictionaryPromoteBasesWeightOnTheSelectedCandidatesOwnScale) std::filesystem::remove_all(directory); } +// 回归：调频必须能把另一种切分的词调到整个列表的首位。打 jian 时列表里混着 ji'an 的 +// 吉安（出货词库权重是 1，压在同组的积案 9420 底下），旧实现按音节数圈比较集，吉安 +// 只跟积案/几案比，可学权重封顶一万出头，怎么调都追不上 见 的 3460998——用户看到的就是 +// 「调多少次都没用」。比较集不再按音节数切开之后，基准取到 见，一次置顶就越过它。 +TEST_CASE(UserDictionaryPromoteLiftsAnAlternativeSegmentationAcrossKeys) +{ + const auto directory = + std::filesystem::temp_directory_path() / ("msime-cross-key-ranking-" + std::to_string(GetCurrentProcessId())); + std::filesystem::create_directories(directory); + const auto user_path = directory / "msime_user.db"; + const auto main_path = directory / "msime.db"; + { + TestDatabase db(main_path); + db.exec("CREATE TABLE tbl_1_j(key TEXT,jp TEXT,value TEXT,weight INTEGER);" + "CREATE TABLE tbl_2_j(key TEXT,jp TEXT,value TEXT,weight INTEGER);" + "INSERT INTO tbl_1_j VALUES(" + "'jian','j','见',3460998)," + "('jian','j','间',3067939)," + "('jian','j','剑',1151704);" + "INSERT INTO tbl_2_j VALUES(" + "'ji''an','ja','积案',9420)," + "('ji''an','ja','几案',5999)," + "('ji''an','ja','吉安',1);"); + } + std::vector candidates = { + {"jian", "见", 3460998, CandidateSource::Database, "jian"}, + {"jian", "间", 3067939, CandidateSource::Database, "jian"}, + {"jian", "剑", 1151704, CandidateSource::Database, "jian"}, + {"jian", "积案", 9420, CandidateSource::Database, "ji'an"}, + {"jian", "几案", 5999, CandidateSource::Database, "ji'an"}, + {"jian", "吉安", 1, CandidateSource::Database, "ji'an"}, + }; + + bool ranking_changed = false; + REQUIRE(user_dictionary::adjust_candidate_ranking(test::Utf8(main_path), test::Utf8(user_path), "jian", candidates, + "ji'an", "吉安", "pin", 1, 1, true, &ranking_changed)); + REQUIRE(ranking_changed); + { + TestDatabase db(main_path); + // 基准是整个列表最重的 见，不再是 ji'an 这一组的头部 积案。 + REQUIRE(db.scalar_int64("SELECT weight FROM tbl_2_j WHERE value='吉安'") > 3460998); + // 写入仍只落在 entry_key 的行上：jian 那张表一行都没动（#36 的护栏）。 + REQUIRE_EQ(db.scalar_int64("SELECT weight FROM tbl_1_j WHERE value='见'"), 3460998); + REQUIRE_EQ(db.scalar_int64("SELECT weight FROM tbl_1_j WHERE value='间'"), 3067939); + REQUIRE_EQ(db.scalar_int64("SELECT weight FROM tbl_2_j WHERE value='积案'"), 9420); + } + std::filesystem::remove_all(directory); +} + +// 回归：不置顶、按 promote 模式连着选几次，也要能把吉安一路顶到首位。每一轮都照着 +// 库里的新权重重排候选列表，模拟用户重新打一遍 jian 看到的顺序。 +TEST_CASE(UserDictionaryRepeatedPromotionWalksAnAlternativeSegmentationToTheTop) +{ + const auto directory = + std::filesystem::temp_directory_path() / ("msime-cross-key-walk-" + std::to_string(GetCurrentProcessId())); + std::filesystem::create_directories(directory); + const auto user_path = directory / "msime_user.db"; + const auto main_path = directory / "msime.db"; + { + TestDatabase db(main_path); + db.exec("CREATE TABLE tbl_1_j(key TEXT,jp TEXT,value TEXT,weight INTEGER);" + "CREATE TABLE tbl_2_j(key TEXT,jp TEXT,value TEXT,weight INTEGER);" + "INSERT INTO tbl_1_j VALUES(" + "'jian','j','见',3460998)," + "('jian','j','间',3067939)," + "('jian','j','剑',1151704)," + "('jian','j','件',935235)," + "('jian','j','建',598616);" + "INSERT INTO tbl_2_j VALUES(" + "'ji''an','ja','积案',9420)," + "('ji''an','ja','几案',5999)," + "('ji''an','ja','吉安',1);"); + } + const std::vector> rows = { + {"jian", "见"}, {"jian", "间"}, {"jian", "剑"}, {"jian", "件"}, + {"jian", "建"}, {"ji'an", "积案"}, {"ji'an", "几案"}, {"ji'an", "吉安"}, + }; + const auto current_list = [&]() { + TestDatabase db(main_path); + std::vector list; + for (const auto &row : rows) + { + const std::string table = row.first == "jian" ? "tbl_1_j" : "tbl_2_j"; + const auto sql = "SELECT weight FROM " + table + " WHERE value='" + row.second + "'"; + list.push_back({"jian", row.second, db.scalar_int64(sql.c_str()), CandidateSource::Database, row.first}); + } + std::stable_sort(list.begin(), list.end(), + [](const WordItem &lhs, const WordItem &rhs) { return lhs.weight > rhs.weight; }); + return list; + }; + const auto rank_of = [](const std::vector &list, const std::string &word) { + for (std::size_t i = 0; i < list.size(); ++i) + if (list[i].word == word) + return i; + return list.size(); + }; + + // promote 模式一次最多前进到第 5 位，所以从第 8 位走到首位需要几轮。上限给 8 轮， + // 断言的是「会收敛」，不是精确的轮数。 + std::size_t rounds = 0; + while (rank_of(current_list(), "吉安") != 0 && rounds < 8) + { + const auto list = current_list(); + REQUIRE(user_dictionary::adjust_candidate_ranking(test::Utf8(main_path), test::Utf8(user_path), "jian", list, + "ji'an", "吉安", "promote", 1, 1, false)); + ++rounds; + } + REQUIRE_EQ(rank_of(current_list(), "吉安"), static_cast(0)); + { + TestDatabase db(main_path); + REQUIRE_EQ(db.scalar_int64("SELECT weight FROM tbl_1_j WHERE value='见'"), 3460998); + } + std::filesystem::remove_all(directory); +} + TEST_CASE(UserDictionaryEqualWeightPromoteDoesNotWriteNegatives) { const auto directory = std::filesystem::temp_directory_path() / @@ -571,7 +688,9 @@ TEST_CASE(UserDictionaryLocalRebalanceKeepsPositiveSameKeyWeights) std::filesystem::remove_all(directory); } -TEST_CASE(UserDictionaryRankingIgnoresShorterKeysWithoutCanonicalPinyin) +// 候选没带 canonical_pinyin 时，键要从 context_key 按字数截出来（先 -> xian， +// 现网 -> xian'wang）。较短的那个键现在也参与比权，但写入必须只落在 entry_key 的行上。 +TEST_CASE(UserDictionaryRankingKeepsWritesOnTheEntryKeyWhenKeysAreReconstructed) { const auto directory = std::filesystem::temp_directory_path() / ("msime-reconstructed-key-ranking-" + std::to_string(GetCurrentProcessId()));