From fc90cb1f2c765976ae2c3e48e5b9ea47766685a6 Mon Sep 17 00:00:00 2001 From: Jia Sui Date: Mon, 12 Oct 2026 17:23:30 +0800 Subject: [PATCH] =?UTF-8?q?fix(engine):=20=E7=BA=AF=E7=AE=80=E6=8B=BC?= =?UTF-8?q?=E8=AF=BB=E6=B3=95=E5=90=A6=E5=86=B3=E6=9D=83=EF=BC=8C=E7=BA=A0?= =?UTF-8?q?=E9=94=99=E4=B8=8D=E5=86=8D=E5=8A=AB=E6=8C=81=20nbn=20=E7=AD=89?= =?UTF-8?q?=E7=AE=80=E6=8B=BC=E5=80=99=E9=80=89=20(#881)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(engine): 纯简拼读法否决权，纠错不再劫持 nbn 等简拼候选 纯简拼形输入（切分全是单字母声母）若词库 jp 列有该输入读法，纠错不得接管： nbn 是「能不能」、nqn 是「女青年」、bqng 是「并且能够」，纠错却把它们改写成 字母更近的错串候选（难/帮），顶掉真实命中的简拼词。 输入形状分不干净——sahng 也全是合法声母却必须保留乱序纠错；唯一分得开的是 词库有没有这个简拼读法，故判定落在同时持有 DB 与纠错掩码的查询层。 - 新增 quanpin::has_pure_jianpin_reading（jp 列 LIMIT 1 查询） - QuanpinDictionary::effective_autocorrect_types 统一 query_exact 与 insert_word_to_series_cache 的有效掩码，避免 series 缓存键分歧 - 非纯简拼（shng=[sh,n,g]、shbng、sahng/shangg/sjhang）纠错不变 * fix(engine): 纯简拼整词改为排在纠错前面，不再整个关掉纠错 原方案在词库有纯简拼读法时把纠错掩码置 0，漏了元音的真手误（bng 想打 bang） 也随之纠不了，而且只有词库层知道纠错被关，预编辑与选词消耗仍按原掩码判断。 改为不动掩码：纠错照常进行，纠错分支里字面结果中的纯简拼整词 （literal_jianpin_pinyin_match：两个及以上单字母声母盖住整串，逐个是正读首字母， a/e/o 不算）和混合简拼一样放进领衔组。纠错读法排在后面，仍可选、仍带纠错标记。 判定直接看已查回的字面结果，不新增 SQL；掩码不变，缓存键也不变， 因此删掉 has_pure_jianpin_reading 与 effective_autocorrect_types。 测试补上预编辑（n'b'n）、选前缀单字只消耗自己的声母、bng 仍保留「帮」。 --------- Co-authored-by: fanlusky <1730976608@qq.com> --- engine/quanpin/quanpin_dictionary.cpp | 11 ++- engine/quanpin/quanpin_utils.cpp | 25 ++++++- engine/quanpin/quanpin_utils.h | 3 + engine/tests/src/test_input_session.cpp | 90 ++++++++++++++++++++++++ engine/tests/src/test_pinyin.cpp | 29 ++++++++ server/tests/src/test_quanpin_scheme.cpp | 31 ++++++++ 6 files changed, 186 insertions(+), 3 deletions(-) diff --git a/engine/quanpin/quanpin_dictionary.cpp b/engine/quanpin/quanpin_dictionary.cpp index ae162e96..ca240be8 100644 --- a/engine/quanpin/quanpin_dictionary.cpp +++ b/engine/quanpin/quanpin_dictionary.cpp @@ -321,6 +321,12 @@ unsigned literal_mixed_match(const WordItem &item, const std::string &raw_input) : 0; } +bool literal_jianpin_match(const WordItem &item, const std::string &raw_input) +{ + return (item.source == CandidateSource::Database || item.source == CandidateSource::UserDatabase) && + quanpin::literal_jianpin_pinyin_match(raw_input, item.pinyin, item.canonical_pinyin); +} + // 读音就是 key 这条切分的行（整键块）。整句行按敲的读音（pinyin）认：词格整句的 // canonical_pinyin 是它实际走的读音，开着模糊音时可能与切分不同（min 走成 ming）。 bool reads_full_key(const WordItem &item, const std::string &key) @@ -515,9 +521,12 @@ std::vector QuanpinDictionary::query_exact(const std::string &raw_inpu const std::string fallback_segmentation = segmentation.empty() ? quanpin::join_segments(segments) : segmentation; const auto literal = query_series(raw_input, fallback_segmentation, segments); + // 纯简拼字面整词与混合简拼同样领衔：全辅音串照常进纠错（形状分不出 nbn 与漏了元音 + // 的 bng），但词库里真有这个简拼读法时（nbn「能不能」、bqng「并且能够」），纠错读法 + // 只排在它后面，仍可选、仍带纠错标记。 for (const auto &item : literal) { - if (literal_mixed_match(item, raw_input)) + if (literal_mixed_match(item, raw_input) || literal_jianpin_match(item, raw_input)) literal_mixed.push_back(item); } append_unique_words(result, literal); diff --git a/engine/quanpin/quanpin_utils.cpp b/engine/quanpin/quanpin_utils.cpp index ad5d5b3f..a8d64912 100644 --- a/engine/quanpin/quanpin_utils.cpp +++ b/engine/quanpin/quanpin_utils.cpp @@ -50,6 +50,25 @@ unsigned literal_mixed_pinyin_match(const std::string &raw, const std::string &c return full && initial ? (tail.empty() ? 1u : 2u) : 0u; } +bool literal_jianpin_pinyin_match(const std::string &raw, const std::string &code, const std::string &canonical) +{ + if (raw.find('\'') != std::string::npos || canonical.empty()) + return false; + const auto typed = split_segments(code); + const auto reading = split_segments(canonical); + if (typed.size() < 2 || typed.size() != reading.size()) + return false; + std::string letters; + for (size_t i = 0; i < typed.size(); ++i) + { + if (typed[i].size() != 1 || !prefix_pinyin_set().count(typed[i]) || intact_pinyin_set().count(typed[i]) || + !intact_pinyin_set().count(reading[i]) || reading[i][0] != typed[i][0]) + return false; + letters += typed[i]; + } + return letters == raw; +} + namespace { struct SparsePinyinFallbackEntry @@ -214,8 +233,10 @@ bool looks_like_jianpin_spelling(const std::string &pinyin) // trailing letter ("zheg" = zhe + g) to count as jianpin intent. The cut // backtracks like the query-side segmentation ("zaijinut" = zai'ji'nu + t). // All-consonant strings match zero syllables and return false on purpose: - // the engine has no multi-letter jianpin, so correction is the only useful - // reading of e.g. "bqng" -> bang. + // shape alone cannot tell a pure jianpin ("nbn" = 能不能) from a dropped + // vowel ("bng" -> bang), so both still enter correction. The dictionary + // layer then puts a literal pure-jianpin whole word ahead of the corrected + // readings (literal_jianpin_pinyin_match), keeping the correction behind it. if (is_complete_pinyin_input(pinyin)) { // Fully reduced to legal syllables: a legitimate spelling, not a typo. diff --git a/engine/quanpin/quanpin_utils.h b/engine/quanpin/quanpin_utils.h index 1759a97e..0f8a518f 100644 --- a/engine/quanpin/quanpin_utils.h +++ b/engine/quanpin/quanpin_utils.h @@ -29,6 +29,9 @@ bool has_only_complete_pinyin_segments(const Segments &segments); // 0: no literal mixed match; 1: whole input; 2: one unfinished 1-2-letter syllable remains. // Both full syllables and single-letter initials must match the candidate's canonical key. unsigned literal_mixed_pinyin_match(const std::string &raw, const std::string &code, const std::string &canonical); +// 纯简拼字面整词：两个及以上单字母声母（a/e/o 是完整音节，不算）盖住整串输入，且逐个是 +// 候选正读音节的首字母（nbn = n'b'n -> neng'bu'neng）。前缀单字和带完整音节的混合简拼不算。 +bool literal_jianpin_pinyin_match(const std::string &raw, const std::string &code, const std::string &canonical); SyllableGraph build_syllable_graph(const std::string &pinyin); std::vector enumerate_complete_segmentations(const SyllableGraph &graph, size_t path_limit = 32); std::vector cut_one_piece_greedy(const std::string &pinyin, bool intact_only); diff --git a/engine/tests/src/test_input_session.cpp b/engine/tests/src/test_input_session.cpp index 7826f74a..bb33821b 100644 --- a/engine/tests/src/test_input_session.cpp +++ b/engine/tests/src/test_input_session.cpp @@ -1525,6 +1525,95 @@ void run_autocorrect_generated_space_tests(const std::filesystem::path &data_dir } } +// 纯简拼形输入（每个字母都是一个声母）如果词库里真有它的简拼整词，由字面整词领衔： +// nbn/nqn/bqng 分别命中词库 jp 列的 能不能/女青年/并且能够，纠错读法（难/帮）只排在 +// 后面，仍可选、仍带纠错标记。预编辑按字面简拼切分，选前缀单字只消耗它自己的字母。 +// 漏了元音的 bng 即使有简拼整词「不能够」，「帮」也不能丢。shng 含双字母切分 sh， +// 不是纯简拼，漏字纠错照旧。 +void run_autocorrect_pure_jianpin_guard_tests(const std::filesystem::path &data_directory) +{ + const std::filesystem::path directory = data_directory / "autocorrect-pure-jianpin"; + std::filesystem::create_directories(directory); + { + Database database(directory / "msime.db"); + database.execute("CREATE TABLE tbl_1_s(key TEXT,jp TEXT,value TEXT,weight INTEGER);" + "INSERT INTO tbl_1_s VALUES('shang','s','上',900);" + "CREATE TABLE tbl_1_n(key TEXT,jp TEXT,value TEXT,weight INTEGER);" + "INSERT INTO tbl_1_n VALUES('nan','n','难',900);" + "INSERT INTO tbl_1_n VALUES('neng','n','能',800);" + "CREATE TABLE tbl_1_b(key TEXT,jp TEXT,value TEXT,weight INTEGER);" + "INSERT INTO tbl_1_b VALUES('bang','b','帮',900);" + "CREATE TABLE tbl_3_b(key TEXT,jp TEXT,value TEXT,weight INTEGER);" + "INSERT INTO tbl_3_b VALUES('bu''neng''gou','bng','不能够',2000);" + "CREATE TABLE tbl_3_n(key TEXT,jp TEXT,value TEXT,weight INTEGER);" + "INSERT INTO tbl_3_n VALUES('neng''bu''neng','nbn','能不能',5000);" + "INSERT INTO tbl_3_n VALUES('nv''qing''nian','nqn','女青年',3000);" + "CREATE TABLE tbl_4_b(key TEXT,jp TEXT,value TEXT,weight INTEGER);" + "INSERT INTO tbl_4_b VALUES('bing''qie''neng''gou','bqng','并且能够',4000);"); + } + + metasequoia::RuntimePaths paths; + paths.resources = directory; + paths.user_data = directory; + paths.cache = directory; + paths.dictionaries = directory; + const unsigned all = quanpin::kAutocorrectTransposition | quanpin::kAutocorrectNeighbor | + quanpin::kAutocorrectDeletion | quanpin::kAutocorrectInsertion; + + struct Case + { + const char *input; + const char *word; + const char *display; + const char *corrected; + }; + for (const Case &test : {Case{"nbn", "能不能", "n'b'n", "难"}, Case{"nqn", "女青年", "n'q'n", "难"}, + Case{"bqng", "并且能够", "b'q'n'g", "帮"}, Case{"bng", "不能够", "b'n'g", "帮"}}) + { + metasequoia::InputSession session(SchemeType::Quanpin, all, true, true, true, paths); + type(session, test.input); + require(!session.candidates().empty() && session.candidates().front().word == test.word && + session.candidates().front().corrected_from.empty(), + (std::string("A pure-jianpin whole word must lead, uncorrected: ") + test.input).c_str()); + const auto corrected = find_candidate_index(session, test.corrected); + require(corrected < session.candidates().size() && session.candidates()[corrected].corrected_from == test.input, + (std::string("The corrected reading must stay behind the jianpin word: ") + test.input).c_str()); + require(session.get_pinyin_segmentation_with_cases() == test.display, + (std::string("The preedit must follow the literal jianpin reading: ") + test.input + " -> " + + session.get_pinyin_segmentation_with_cases()) + .c_str()); + } + + // 选纯简拼的前缀单字只消耗它自己的声母，后面的 bn 留着继续输入。 + { + metasequoia::InputSession session(SchemeType::Quanpin, all, true, true, true, paths); + type(session, "nbn"); + const auto committed = session.select_candidate(candidate_index(session, "能")); + require(committed.commit == "能" && session.has_composition() && session.preedit() == "bn", + "Selecting a jianpin prefix character must consume only its own initial."); + } + + // 含双字母切分的输入不是纯简拼，漏字纠错照旧命中「上」并记下敲入的字母。 + { + metasequoia::InputSession session(SchemeType::Quanpin, all, true, true, true, paths); + type(session, "shng"); + const auto found = find_candidate_index(session, "上"); + require(found < session.candidates().size() && session.candidates()[found].corrected_from == "shng", + "A non-pure-jianpin typo (shng) must still be corrected with the typed letters recorded."); + } + + // 纠错关时逐位不变：纯简拼输入仍由字面简拼读法领衔，且无纠错标记。 + { + metasequoia::InputSession session(SchemeType::Quanpin, 0u, true, true, true, paths); + type(session, "bqng"); + require(!session.candidates().empty() && session.candidates().front().word == "并且能够" && + session.candidates().front().corrected_from.empty(), + "With autocorrection off a pure-jianpin reading must still lead, uncorrected."); + } + + std::filesystem::remove_all(directory); +} + void run_caret_prefix_session_tests(const std::filesystem::path &data_directory) { const std::filesystem::path directory = data_directory / "caret-prefix"; @@ -2521,6 +2610,7 @@ int run_test() run_autocorrect_costlier_context_tests(data_directory); run_autocorrect_generated_space_tests(data_directory); + run_autocorrect_pure_jianpin_guard_tests(data_directory); #endif #ifndef METASEQUOIA_SKIP_FREQUENCY_TESTS diff --git a/engine/tests/src/test_pinyin.cpp b/engine/tests/src/test_pinyin.cpp index ba409028..df78c059 100644 --- a/engine/tests/src/test_pinyin.cpp +++ b/engine/tests/src/test_pinyin.cpp @@ -919,12 +919,21 @@ std::filesystem::path create_autocorrect_probe_database() throw std::runtime_error("Failed to create the autocorrect probe database."); } const char *sql = "CREATE TABLE tbl_1_s(key TEXT,jp TEXT,value TEXT,weight INTEGER);" + "CREATE TABLE tbl_1_n(key TEXT,jp TEXT,value TEXT,weight INTEGER);" + "CREATE TABLE tbl_1_b(key TEXT,jp TEXT,value TEXT,weight INTEGER);" "CREATE TABLE tbl_2_s(key TEXT,jp TEXT,value TEXT,weight INTEGER);" "CREATE TABLE tbl_2_j(key TEXT,jp TEXT,value TEXT,weight INTEGER);" + "CREATE TABLE tbl_3_n(key TEXT,jp TEXT,value TEXT,weight INTEGER);" "CREATE TABLE tbl_4_s(key TEXT,jp TEXT,value TEXT,weight INTEGER);" + "CREATE TABLE tbl_4_b(key TEXT,jp TEXT,value TEXT,weight INTEGER);" "INSERT INTO tbl_1_s VALUES('shang','s','上',100);" + "INSERT INTO tbl_1_n VALUES('nan','n','难',100);" + "INSERT INTO tbl_1_b VALUES('bang','b','帮',100);" "INSERT INTO tbl_2_s VALUES('shang''zhi','sz','上至',100);" "INSERT INTO tbl_2_j VALUES('jian''du','jd','监督',100);" + "INSERT INTO tbl_3_n VALUES('neng''bu''neng','nbn','能不能',5000);" + "INSERT INTO tbl_3_n VALUES('nv''qing''nian','nqn','女青年',3000);" + "INSERT INTO tbl_4_b VALUES('bing''qie''neng''gou','bqng','并且能够',4000);" "INSERT INTO tbl_4_s VALUES('sa''huang''na''ge','shng','撒谎那个',1000);"; const int result = sqlite3_exec(db, sql, nullptr, nullptr, nullptr); sqlite3_close(db); @@ -1123,6 +1132,26 @@ void test_quanpin_autocorrect_switches_and_guard() { QuanpinDictionary dictionary(db_path.string()); + // 纯简拼字面整词领衔：纯简拼形输入（每个字母都是一个声母）且词库里有这个简拼整词时， + // 它排在纠错读法前面且无 corrected_from；纠错读法仍留在后面、带纠错标记。 + // 放在既有断言之前：本函数后段有一条与本任务无关的既有失败会抛出，其后的断言不再执行。 + const unsigned all_four = + transposition_only | neighbor_only | quanpin::kAutocorrectDeletion | quanpin::kAutocorrectInsertion; + const auto pure_nbn = dictionary.query("nbn", "", all_four); + expect(!pure_nbn.empty() && pure_nbn.front().word == "能不能" && pure_nbn.front().corrected_from.empty(), + "A pure-jianpin input with a dictionary reading must lead uncorrected (nbn -> 能不能)."); + expect(std::any_of(pure_nbn.begin(), pure_nbn.end(), + [](const WordItem &item) { return item.word == "难" && item.corrected_from == "nbn"; }), + "The corrected reading must stay behind the jianpin word (nbn -> 难)."); + const auto pure_bqng = dictionary.query("bqng", "", all_four); + expect(!pure_bqng.empty() && pure_bqng.front().word == "并且能够" && pure_bqng.front().corrected_from.empty(), + "A pure-jianpin input with a dictionary reading must lead uncorrected (bqng -> 并且能够)."); + // 非纯简拼（shng = sh + n + g，含双字母切分）不受保护：漏字纠错照旧命中「上」。 + const auto non_pure_shng = dictionary.query("shng", "sh'n'g", all_four); + expect(!non_pure_shng.empty() && non_pure_shng.front().word == "上" && + non_pure_shng.front().corrected_from == "shng", + "A non-pure-jianpin typo must still be corrected (shng -> 上)."); + const auto corrected = dictionary.query("sahng", "sa'h'n'g", both); expect(!corrected.empty() && corrected.front().word == "上", "Both switches on must put the corrected candidate first."); diff --git a/server/tests/src/test_quanpin_scheme.cpp b/server/tests/src/test_quanpin_scheme.cpp index f6cb3a35..6d31c9e8 100644 --- a/server/tests/src/test_quanpin_scheme.cpp +++ b/server/tests/src/test_quanpin_scheme.cpp @@ -978,16 +978,24 @@ std::filesystem::path CreateAutocorrectDatabase() throw std::runtime_error("Failed to create temporary autocorrect database."); } const char *sql = "CREATE TABLE tbl_1_s(key TEXT,jp TEXT,value TEXT,weight INTEGER);" + "CREATE TABLE tbl_1_n(key TEXT,jp TEXT,value TEXT,weight INTEGER);" + "CREATE TABLE tbl_1_b(key TEXT,jp TEXT,value TEXT,weight INTEGER);" "CREATE TABLE tbl_2_s(key TEXT,jp TEXT,value TEXT,weight INTEGER);" "CREATE TABLE tbl_2_g(key TEXT,jp TEXT,value TEXT,weight INTEGER);" "CREATE TABLE tbl_2_j(key TEXT,jp TEXT,value TEXT,weight INTEGER);" "CREATE TABLE tbl_2_q(key TEXT,jp TEXT,value TEXT,weight INTEGER);" + "CREATE TABLE tbl_3_n(key TEXT,jp TEXT,value TEXT,weight INTEGER);" "CREATE TABLE tbl_4_s(key TEXT,jp TEXT,value TEXT,weight INTEGER);" + "CREATE TABLE tbl_4_b(key TEXT,jp TEXT,value TEXT,weight INTEGER);" "INSERT INTO tbl_1_s VALUES('shang','s','上',100);" + "INSERT INTO tbl_1_n VALUES('nan','n','难',100);" + "INSERT INTO tbl_1_b VALUES('bang','b','帮',100);" "INSERT INTO tbl_2_s VALUES('shang''zhi','sz','上至',100);" "INSERT INTO tbl_2_g VALUES('guan''li','gl','管理',100);" "INSERT INTO tbl_2_j VALUES('jian''du','jd','监督',100);" "INSERT INTO tbl_2_q VALUES('quan''li','ql','权利',100);" + "INSERT INTO tbl_3_n VALUES('neng''bu''neng','nbn','能不能',5000);" + "INSERT INTO tbl_4_b VALUES('bing''qie''neng''gou','bqng','并且能够',4000);" "INSERT INTO tbl_4_s VALUES('sa''huang''na''ge','shng','撒谎那个',1000);"; const int result = sqlite3_exec(db, sql, nullptr, nullptr, nullptr); sqlite3_close(db); @@ -1063,6 +1071,29 @@ TEST_CASE(QuanpinDictionaryAutocorrectDeletionTypo) std::none_of(legacy.begin(), legacy.end(), [](const WordItem &item) { return item.corrected_from == "shng"; })); } +TEST_CASE(QuanpinDictionaryPureJianpinWordLeadsAutocorrect) +{ + const auto db_path = CreateAutocorrectDatabase(); + QuanpinDictionary dictionary(test::Utf8(db_path)); + const unsigned all = quanpin::kAutocorrectTransposition | quanpin::kAutocorrectNeighbor | + quanpin::kAutocorrectDeletion | quanpin::kAutocorrectInsertion; + + // 纯简拼字面整词领衔：纯简拼形输入（每个字母都是一个声母）且词库里有这个简拼整词时， + // 它排在纠错读法前面且无 corrected_from；纠错读法仍留在后面、带纠错标记。 + const auto bqng = dictionary.query("bqng", "", all); + REQUIRE(!bqng.empty()); + REQUIRE_EQ(bqng.front().word, std::string("并且能够")); + REQUIRE(bqng.front().corrected_from.empty()); + REQUIRE(std::any_of(bqng.begin(), bqng.end(), + [](const WordItem &item) { return item.word == "帮" && item.corrected_from == "bqng"; })); + + // 非纯简拼（shng = sh + n + g，含双字母切分）不受保护：漏字纠错照旧命中「上」。 + const auto shng = dictionary.query("shng", "sh'n'g", all); + REQUIRE(!shng.empty()); + REQUIRE_EQ(shng.front().word, std::string("上")); + REQUIRE_EQ(shng.front().corrected_from, std::string("shng")); +} + TEST_CASE(QuanpinDictionaryAutocorrectInsertionTypo) { const auto db_path = CreateAutocorrectDatabase();