From 49be145bdfa61a94030cce81ea6fea178935a13c Mon Sep 17 00:00:00 2001 From: Jia Sui Date: Sat, 3 Oct 2026 20:21:45 +0800 Subject: [PATCH 01/12] =?UTF-8?q?feat(engine):=20=E5=85=A8=E6=8B=BC?= =?UTF-8?q?=E7=BA=A0=E9=94=99=E5=90=8C=E6=A1=A3=E6=AD=A7=E4=B9=89=E6=8C=89?= =?UTF-8?q?=E8=AF=8D=E6=A0=BC=E4=B8=8A=E4=B8=8B=E6=96=87=E6=B6=88=E8=A7=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 同档纠错读法（zhng→{zhang,zheng} 这类 k-best 等权目标）此前只按静态词频 消解、不读上下文，是纠错误报的主要来源。现在整句词格开启时直接解出主切与 同档备选的最佳路径分，边际达标（打分器自身对数单位 1.0）时胜出切分接管 领衔候选，静态词频降级为解码前置过滤；词格不可用或边际不足时与现状逐位 一致（关联关闭、单音节、纠错开关关三条降级路径）。 七项护栏全部保留：贵档绝对垫底、精确键领先位、每来源至多一行、合并池去重、 纠错标记边界、输入门未动，1/100 门槛升格为解码前置过滤。单音节纠错无词格 可搭，维持静态路径（已知限制，fixture 测试钉住）。 评测：eval 新增 --seeds-file 固定精度套件（15 条，豁免逐键一致契约）， 四档注错模型 R@1/R@3 与重采基线逐位一致，种子聚合 R@1 73.3%、R@3 80.0%； 机制生效由 test_input_session 隔离 fixture 端到端断言（lattice-on 序列 [办证,辩证,班长,搬账]）。README 补资源目录要求与种子表维护纪律。 --- engine/quanpin/quanpin_dictionary.cpp | 117 ++++++++++++- engine/tests/README.md | 6 + engine/tests/src/eval_quanpin_autocorrect.cpp | 163 +++++++++++++++++- engine/tests/src/test_input_session.cpp | 75 ++++++++ 4 files changed, 359 insertions(+), 2 deletions(-) diff --git a/engine/quanpin/quanpin_dictionary.cpp b/engine/quanpin/quanpin_dictionary.cpp index 06996ece..8718bac6 100644 --- a/engine/quanpin/quanpin_dictionary.cpp +++ b/engine/quanpin/quanpin_dictionary.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -32,6 +33,18 @@ constexpr size_t kAlternativeSegmentationFirstPageSize = 6; // 将(2.63M) 同样被拒。 constexpr std::int64_t kAlternativeSegmentationPromotionRatio = 100; +// 同档纠错读法的上下文消解（阶段 1 design）：同档备选切分各自过一遍完整管线， +// 跨切分比较词格路径分（log10×1000，同档等罚，分差即纯上下文差）。胜出切分边际 +// 达标时接管「主切」角色领衔候选；不达标或词格不可用时与现状逐位一致。 +// 解码预算：只解通过静态门槛（上面 1/100 同款）且词库有行的备选，上限 4 个—— +// uanli 这类掉声母输入有 10-15 个等权目标，全解会伤 p95。 +constexpr size_t kAutocorrectContextDecodeLimit = 4; +// 接管边际：以打分器自身的对数单位计——词格 LM 打分是 log10（1.0 ≈ 10× 句概率）， +// 无 LM 的启发式回退是 ln（1.0 ≈ e≈2.7×）。对标 CN105045778B 的 β 型先验比率边际 +// （同音校对 β=0.01 → 2.0），手误场景放宽一档。ponytail: 朴素常数；校准路径 = +// user_journal 挖混淆先验（方向阶段 3）。 +constexpr double kAutocorrectContextMarginLog = 1.0; + bool is_alpha_vk(ImeKeyCode vk) { return vk >= 'A' && vk <= 'Z'; @@ -394,10 +407,112 @@ std::vector QuanpinDictionary::query_exact(const std::string &raw_inpu const std::string fallback_segmentation = segmentation.empty() ? quanpin::join_segments(segments) : segmentation; append_unique_words(result, query_series(raw_input, fallback_segmentation, segments)); + + // 同档上下文消解（阶段 1）：词格开启时直接解出主切与同档备选的最佳路径分 + // 裁决，静态词频只承担解码前置过滤。不能从候选列表读分—— + // merge_lattice_candidates 会把与已有候选同词的生成行去重掉（already 集）， + // 最优路径通常恰好就是词库首行，列表里根本看不到它。词格不可用或边际不足 + // 时，下面的一切与改造前逐位一致（降级矩阵见任务 design.md）。 + bool context_reordered = false; + if (sentence_association_.word_lattice && !resolution.alternative_corrected_cuts.empty() && + resolution.corrected_segments.size() >= 2 && + quanpin::has_only_complete_pinyin_segments(resolution.corrected_segments)) + { + // 与 query_series 的装配保持一致：collocation 项同样参与路径分。 + quanpin::WordLatticeOptions lattice_options; + lattice_options.language_model = language_model_; + if (collocation_db_ != nullptr && collocation_db_->valid() && + sentence_association_.collocation_weight != 0.0) + { + lattice_options.collocation_scorer = [db = collocation_db_](std::string_view tail, + std::string_view word, bool is_rear) { + return db->query(std::string(tail), std::string(word), is_rear, gram::GrammarConfig{}); + }; + lattice_options.collocation_weight = sentence_association_.collocation_weight; + } + const auto lookup = quanpin::make_lattice_db_lookup(db_, statement_cache_, lattice_options.span_limit); + const auto best_path_score = [&lookup, + &lattice_options](const quanpin::Segments &cut) -> std::optional { + if (cut.size() < 2 || !quanpin::has_only_complete_pinyin_segments(cut)) + { + return std::nullopt; + } + const auto cut_paths = quanpin::decode_word_lattice(cut, lookup, lattice_options); + return cut_paths.empty() ? std::nullopt : std::optional(cut_paths.front().log_prob); + }; + const auto primary_score = best_path_score(resolution.corrected_segments); + if (primary_score.has_value()) + { + // 前置过滤沿用保护位同款 1/100 量级：今天就赢不了晋升的备选不付解码钱。 + // 主切首位权重取 result 首行（精确键命中行），与 merge_alternative_ + // segmentations 的 primary_top_weight 同源。 + const std::int64_t primary_top_weight = result.empty() ? 0 : result.front().weight; + const auto alternative_items = quanpin::query_exact_segmentations_keyed_flat( + resolution.alternative_corrected_cuts, db_, statement_cache_, + kAlternativeSegmentationCandidateLimit); + // 每个备选切分的最佳词权重：行按权重降序返回，emplace 只留首行。 + std::unordered_map best_weight_by_key; + best_weight_by_key.reserve(alternative_items.size()); + for (const auto &item : alternative_items) + { + best_weight_by_key.emplace(item.key, item.weight); + } + std::vector> decode_pool; + for (const auto &cut : resolution.alternative_corrected_cuts) + { + const auto found = best_weight_by_key.find(quanpin::join_segments(cut)); + if (found == best_weight_by_key.end() || + found->second * kAlternativeSegmentationPromotionRatio < primary_top_weight) + { + continue; + } + decode_pool.emplace_back(found->second, cut); + } + std::stable_sort(decode_pool.begin(), decode_pool.end(), + [](const auto &lhs, const auto &rhs) { return lhs.first > rhs.first; }); + // 第一遍只解路径分选胜者，只为胜出切分付完整管线的钱。 + std::optional winner_score; + quanpin::Segments winner_segments; + for (size_t i = 0; i < decode_pool.size() && i < kAutocorrectContextDecodeLimit; ++i) + { + const auto &cut = decode_pool[i].second; + if (const auto cut_score = best_path_score(cut); + cut_score.has_value() && (!winner_score.has_value() || *cut_score > *winner_score)) + { + winner_score = cut_score; + winner_segments = cut; + } + } + if (winner_score.has_value() && *winner_score - *primary_score >= kAutocorrectContextMarginLog) + { + // 边际达标：胜出切分领衔。merge_alternative_segmentations 假定传入 + // result 的头部是主切词库行，所以这里仍以主切为锚组装其余列表（从 + // 合并池摘掉胜出切分），再把胜出列表接到最前。胜出列表占满首页时， + // 原保护位钉的名次随整体后移——首页本就该是胜出读音的词。 + std::vector remaining_alternatives; + remaining_alternatives.reserve(alternative_segmentations.size()); + const std::string winner_key = quanpin::join_segments(winner_segments); + for (const auto &alternative : alternative_segmentations) + { + if (quanpin::join_segments(alternative) != winner_key) + { + remaining_alternatives.push_back(alternative); + } + } + std::vector rest = + merge_alternative_segmentations(raw_input, pinyin_segmentation_, resolution.corrected_segments, + remaining_alternatives, std::move(result)); + result = query_series(raw_input, winner_key, winner_segments); + append_unique_words(result, std::move(rest)); + context_reordered = true; + } + } + } + // Query-time disambiguation (patent M3): the alternative readings of the // same typo compete with the primary cut under dictionary word // frequency; the best one keeps a protected slot near the top. - if (!alternative_segmentations.empty()) + if (!context_reordered && !alternative_segmentations.empty()) { result = merge_alternative_segmentations(raw_input, pinyin_segmentation_, resolution.corrected_segments, alternative_segmentations, std::move(result)); diff --git a/engine/tests/README.md b/engine/tests/README.md index ea8a7800..5a5996b1 100644 --- a/engine/tests/README.md +++ b/engine/tests/README.md @@ -26,6 +26,12 @@ cmake --build build --config Release --target ` 是独立数据入口：每行 `typo期望key[期望词]`（`#` 注释、空行跳过，期望词缺省时取该 key 的最高权重词），产出独立的 Seed suite 报告节与 `seeds` 聚合 CSV 行。它**豁免于上述逐键一致契约**——种子表度量的正是契约外与用户反馈直接相关的场景（误报/漏报样例固化）。 + +资源目录要求：引擎加载 `sc.lm`（词格）与 `sentence-model-*.safetensors`（神经重排），而 `server/assets/tables` 只有 `dict_pinyin.dat`、`helpcode.txt`、`user_dict.dat`——模型文件不入库。本机没有把模型放进 tables 时，用安装数据目录拼一个隔离资源目录（只读模型拷贝 + `msime.db` 副本），`--db` 与 `--resource` 都指过去；直接指安装目录也可以，但评测进程会以读写方式打开 `--db`，拷贝副本更干净。 + +沙箱环境下子进程可能被拒绝在 `%TEMP%` 下新建目录（表现为 `create_directories: Access is denied.` 后直接退出），把 `TEMP`/`TMP` 环境变量指到仓库内的已存在目录再跑即可。 + 历史基线与采集笔记在归档任务目录：`.trellis/tasks/archive/2026-09/09-12-quanpin-autocorrect-patent/eval/`（阶段 0 基线）与 `09-13-quanpin-insertion-autocorrect/eval/`（insertion 上线前后对照）。 ### 运行 diff --git a/engine/tests/src/eval_quanpin_autocorrect.cpp b/engine/tests/src/eval_quanpin_autocorrect.cpp index 2ab0aa81..937dc581 100644 --- a/engine/tests/src/eval_quanpin_autocorrect.cpp +++ b/engine/tests/src/eval_quanpin_autocorrect.cpp @@ -109,6 +109,7 @@ struct Options ErrorModel model = ErrorModel::Mixed; std::string model_name = "mixed"; std::string csv_path; + std::string seeds_path; int dump = 0; }; @@ -120,7 +121,7 @@ void usage_exit(const std::string &message) } std::cerr << "usage: eval_quanpin_autocorrect --db [--samples N] [--seed S]\n" << " [--rates 10,25,50,100] [--model mixed|deletion|ambiguous|insertion]\n" - << " [--resource ] [--csv ] [--dump N]\n"; + << " [--resource ] [--csv ] [--seeds-file ] [--dump N]\n"; std::exit(2); } @@ -230,6 +231,10 @@ Options parse_options(int argc, char *argv[]) { value_or_exit(options.csv_path); } + else if (argument == "--seeds-file") + { + value_or_exit(options.seeds_path); + } else if (argument == "--dump") { std::string value; @@ -801,6 +806,128 @@ std::vector top_words_for_syllable(sqlite3 *db, const std::string & return words; } +// 种子表一条（固定精度套件）：键入串 + 期望读音（带撇号切分）+ 目标词。 +// 目标词留空 = 取该读音在词库里的最高权重词。来源与维护纪律见任务 +// eval/seeds.txt 头注：用户反馈的误报/漏报样例固化于此，豁免于「注错生成器 +// 逐键一致」契约——它度量的正是契约外与用户直接相关的场景。 +struct SeedCase +{ + std::string typo; + std::string expected_key; + std::string word; +}; + +std::vector load_seed_cases(const std::string &path) +{ + std::ifstream input(path); + if (!input) + { + usage_exit("cannot open seeds file: " + path); + } + std::vector cases; + std::string line; + while (std::getline(input, line)) + { + if (!line.empty() && line.back() == '\r') + { + line.pop_back(); + } + if (line.empty() || line.front() == '#') + { + continue; + } + std::istringstream stream(line); + SeedCase seed; + if (!std::getline(stream, seed.typo, '\t') || !std::getline(stream, seed.expected_key, '\t')) + { + continue; + } + std::getline(stream, seed.word, '\t'); + cases.push_back(std::move(seed)); + } + return cases; +} + +// 目标词缺省时的兜底：取该读音在词库里的最高权重词。平局按 value 字典序，与 +// build_sample_pool 的排序决胜锚点同一约定，保证逐次运行可复现。key 用绑定参数： +// 多音节 key 带撇号，内插会打碎 SQL 串。 +std::string top_word_for_key(sqlite3 *db, const std::string &key) +{ + const std::string table = quanpin::build_table_name(quanpin::split_segments(key)); + if (table.empty()) + { + return {}; + } + const std::string sql = "SELECT value FROM " + table + " WHERE key = ?1 ORDER BY weight DESC, value ASC LIMIT 1"; + sqlite3_stmt *statement = nullptr; + if (sqlite3_prepare_v2(db, sql.c_str(), -1, &statement, nullptr) != SQLITE_OK) + { + return {}; + } + std::string word; + if (sqlite3_bind_text(statement, 1, key.c_str(), -1, SQLITE_TRANSIENT) == SQLITE_OK && + sqlite3_step(statement) == SQLITE_ROW) + { + const auto *text = reinterpret_cast(sqlite3_column_text(statement, 0)); + if (text != nullptr) + { + word = text; + } + } + sqlite3_finalize(statement); + return word; +} + +struct SeedOutcome +{ + std::string typo; + std::string expected_key; + std::string word; + int rank = 0; + bool restored = false; + bool skipped = false; // 词库查不到目标词，无法计分 +}; + +struct SeedReport +{ + RateReport summary; + std::vector outcomes; +}; + +// 种子套件：逐条按 evaluate_case 的口径（预热一次、计时第二次查询）跑， +// 汇总一行聚合（R@1/R@3/读音还原/温态），明细进报告供人工对照。 +SeedReport run_seed_suite(QuanpinDictionary &dictionary, sqlite3 *db, const Options &options, + unsigned autocorrect_types) +{ + SeedReport report; + report.summary.rate = 100; + for (const auto &seed : load_seed_cases(options.seeds_path)) + { + SeedOutcome outcome{seed.typo, seed.expected_key, seed.word, 0, false, false}; + if (outcome.word.empty()) + { + outcome.word = top_word_for_key(db, seed.expected_key); + } + if (outcome.word.empty()) + { + outcome.skipped = true; + report.outcomes.push_back(std::move(outcome)); + continue; + } + const Entry entry{seed.expected_key, outcome.word}; + const auto case_outcome = evaluate_case(dictionary, seed.typo, entry, autocorrect_types); + outcome.rank = case_outcome.rank; + outcome.restored = case_outcome.restored; + ++report.summary.samples; + report.summary.recall1 += outcome.rank == 1 ? 1 : 0; + report.summary.recall3 += outcome.rank >= 1 && outcome.rank <= 3 ? 1 : 0; + report.summary.restored += outcome.restored ? 1 : 0; + report.summary.warm_ms.push_back(case_outcome.warm_ms); + report.outcomes.push_back(std::move(outcome)); + } + return report; +} + struct AmbiguousTargetOutcome { std::string input; @@ -946,6 +1073,12 @@ int main(int argc, char *argv[]) reports = run_injected_model(dictionary, pool, options, autocorrect_types); } + std::optional seed_report; + if (!options.seeds_path.empty()) + { + seed_report = run_seed_suite(dictionary, sampling_db, options, autocorrect_types); + } + std::ostringstream markdown; markdown << "# Quanpin autocorrect evaluation\n\n"; markdown << "- model: " << options.model_name << "\n"; @@ -1013,6 +1146,30 @@ int main(int argc, char *argv[]) << percent(report.gate.serviceable_recall3, report.gate.serviceable) << " |\n"; } } + if (seed_report.has_value()) + { + markdown << "\n## Seed suite (fixed precision cases)\n\n"; + markdown << "| typo | expected key | target word | rank | reading restored |\n"; + markdown << "|------|--------------|-------------|-----:|-----------------:|\n"; + for (const auto &outcome : seed_report->outcomes) + { + markdown << "| " << outcome.typo << " | " << outcome.expected_key << " | " << outcome.word << " | "; + if (outcome.skipped) + { + markdown << "skip (word not in dictionary) |\n"; + } + else + { + markdown << outcome.rank << " | " << (outcome.restored ? "yes" : "no") << " |\n"; + } + } + markdown << "\n- seed aggregate: R@1 " << percent(seed_report->summary.recall1, seed_report->summary.samples) + << ", R@3 " << percent(seed_report->summary.recall3, seed_report->summary.samples) << ", restored " + << percent(seed_report->summary.restored, seed_report->summary.samples) << ", p50 " + << milliseconds(percentile(seed_report->summary.warm_ms, 0.50)) << ", p95 " + << milliseconds(percentile(seed_report->summary.warm_ms, 0.95)) << " (" << seed_report->summary.samples + << " scored)\n"; + } std::cout << markdown.str(); if (!options.csv_path.empty()) @@ -1027,6 +1184,10 @@ int main(int argc, char *argv[]) { append_csv_row(csv, "ambiguous-protected", ambiguous_report.gate_protected); } + if (seed_report.has_value()) + { + append_csv_row(csv, "seeds", seed_report->summary); + } std::cout << "csv written: " << options.csv_path << "\n"; } diff --git a/engine/tests/src/test_input_session.cpp b/engine/tests/src/test_input_session.cpp index bf19df1b..0b196d99 100644 --- a/engine/tests/src/test_input_session.cpp +++ b/engine/tests/src/test_input_session.cpp @@ -337,6 +337,80 @@ void run_umlaut_alias_session_tests(const std::filesystem::path &data_directory) // 光标驱动的前缀解码（PRD R2–R7，Stage 1）：候选与量化边界按「光标之前的完整音节 // 单元前缀」重算。自建隔离词库，不与主 fixture 互相污染；Server（Stage 2）将以 // set_caret + recompute_candidates 的同一方式消费这些入口。 +// 阶段 1 上下文消解（任务 quanpin-autocorrect-context-ranking）：同档纠错读法 +// 在词格路径分边际达标时由胜出切分接管领衔。zhng 的删除目标按表序 zhang 在前 +// （主切），fixture 让 zheng 侧的办证权重高 20 倍——启发式路径分差 ln(20)=3.0 +// （log10 1.3），两种标度下都过 1.0 接管边际。fixture 无 sc.lm，词格退回 +// 启发式打分（ln(weight)+词长奖励），权重完全决定路径分，测试因此确定。 +void run_autocorrect_context_ranking_tests(const std::filesystem::path &data_directory) +{ + const std::filesystem::path directory = data_directory / "autocorrect-context"; + std::filesystem::create_directories(directory); + { + Database database(directory / "msime.db"); + database.execute("CREATE TABLE tbl_2_b(key TEXT,jp TEXT,value TEXT,weight INTEGER);" + "INSERT INTO tbl_2_b VALUES('ban''zheng', 'bz', '办证', 100000);" + "INSERT INTO tbl_2_b VALUES('ban''zheng', 'bz', '辩证', 10);" + "INSERT INTO tbl_2_b VALUES('ban''zhang', 'bz', '班长', 5000);" + "INSERT INTO tbl_2_b VALUES('ban''zhang', 'bz', '搬账', 4000);"); + database.execute("CREATE TABLE tbl_1_s(key TEXT,jp TEXT,value TEXT,weight INTEGER);" + "INSERT INTO tbl_1_s VALUES('shang', 's', '上', 900);" + "INSERT INTO tbl_1_s VALUES('sheng', 's', '生', 800);"); + } + + metasequoia::RuntimePaths paths; + paths.resources = directory; + paths.user_data = directory; + paths.cache = directory; + paths.dictionaries = directory; + const unsigned both = quanpin::kAutocorrectTransposition | quanpin::kAutocorrectNeighbor; + SentenceAssociationOptions lattice_only; + lattice_only.word_lattice = true; + + const auto candidate_words = [](const metasequoia::InputSession &session) { + std::vector words; + for (const auto &item : session.candidates()) + { + words.push_back(item.word); + } + return words; + }; + + // 关联关闭 = 现状静态路径：合并池按权重排序，办证系与班长系交错。 + { + metasequoia::InputSession session(SchemeType::Quanpin, both, true, true, true, paths); + type(session, "banzhng"); + const auto words = candidate_words(session); + require(words.size() == 4 && words[0] == "办证" && words[1] == "班长" && words[2] == "搬账" && + words[3] == "辩证", + "The static same-tier merge must interleave the readings by dictionary weight."); + } + + // 关联开启：zheng 切分上下文胜出接管领衔——办证系整体前移，辩证从末位 + // 升到第 2 位，班长系整体后移。 + { + metasequoia::InputSession session(SchemeType::Quanpin, both, true, true, true, paths); + session.set_sentence_association(lattice_only); + type(session, "banzhng"); + const auto words = candidate_words(session); + require(words.size() == 4 && words[0] == "办证" && words[1] == "辩证" && words[2] == "班长" && + words[3] == "搬账", + "The context-ranked reading must lead with its whole word group."); + } + + // 单音节纠错无词格可搭（词格要求 >=2 完整音节）：两种状态必须逐位一致。 + { + metasequoia::InputSession off(SchemeType::Quanpin, both, true, true, true, paths); + type(off, "shng"); + metasequoia::InputSession on(SchemeType::Quanpin, both, true, true, true, paths); + on.set_sentence_association(lattice_only); + type(on, "shng"); + require(same_candidate_words(off, on), + "A single-syllable correction must take the static path even with the word lattice on."); + require(candidate_words(on).size() == 2, "The single-syllable fixture must surface both deletion targets."); + } +} + void run_caret_prefix_session_tests(const std::filesystem::path &data_directory) { const std::filesystem::path directory = data_directory / "caret-prefix"; @@ -1242,6 +1316,7 @@ int run_test() run_umlaut_alias_session_tests(data_directory); run_caret_prefix_session_tests(data_directory); + run_autocorrect_context_ranking_tests(data_directory); #endif #ifndef METASEQUOIA_SKIP_FREQUENCY_TESTS From fc3564e4d9166deae6312c992acb33ec3f4a5edf Mon Sep 17 00:00:00 2001 From: Jia Sui Date: Sat, 3 Oct 2026 20:21:46 +0800 Subject: [PATCH 02/12] =?UTF-8?q?feat(engine):=20=E7=94=9F=E6=88=90?= =?UTF-8?q?=E5=BC=8F=E7=BA=A0=E9=94=99=E7=A9=BA=E9=97=B4=E5=85=9C=E9=9D=99?= =?UTF-8?q?=E6=80=81=E8=A1=A8=E9=95=BF=E5=B0=BE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 静态纠错表四类形状（相邻换位/漏字/QWERTY 邻键替换/限重复邻键插入）均为 穷举式生成，表外单编辑手误（非相邻键替换、任意字母插入）结构性无人管—— 这是覆盖反馈的机制根源。新增生成式索引：对全部合法音节枚举表外形状 （48,216 对，首次纠错查询时一次性构建），并入纠错索引，k-best/贵档垫底/ 上下文消解全链路复用；换位/漏字静态已穷尽，不枚举。 表内零回归为结构性保证：含生成边的切分在 k-best 排序中整体垫底 （has_generated 先序键），最优切分为纯静态时全部生成切分被丢弃——丢弃 规则在 autocorrect_cut_kbest 单点实现，主解析与 head+简拼尾组合路径同享； 生成读法只在静态空间完全无法解释输入时出场。混表加权仲裁依赖训练权重， 留给方向阶段 3。 评测：eval 新增 outside 注错档（表外形状，与静态表构造性不相交），覆盖 增益量化 R@1 47.7/44.3/36.3/22.7（四率；静态-only 恒为 0）；表内四档 R@1/R@3 与基线逐位一致；种子聚合不回归。fixture 四用例含 head 组合路径 判别（shatngzh）。README 补 outside 档与资源目录说明。 --- engine/quanpin/quanpin_dictionary.cpp | 4 + engine/quanpin/quanpin_utils.cpp | 175 +++++++++++++++++- engine/quanpin/quanpin_utils.h | 20 +- engine/tests/README.md | 4 +- engine/tests/src/eval_quanpin_autocorrect.cpp | 82 +++++++- engine/tests/src/test_input_session.cpp | 91 +++++++++ 6 files changed, 370 insertions(+), 6 deletions(-) diff --git a/engine/quanpin/quanpin_dictionary.cpp b/engine/quanpin/quanpin_dictionary.cpp index 8718bac6..5201139f 100644 --- a/engine/quanpin/quanpin_dictionary.cpp +++ b/engine/quanpin/quanpin_dictionary.cpp @@ -176,6 +176,10 @@ SeriesQueryResolution resolve_series_query(const std::string &raw_input, const q } }; + // 静态优先语义由 k-best 搜索自身保证（见 quanpin_utils.h 契约）：最优 + // 切分为纯静态时生成切分已在搜索内被丢弃，下面的 head+简拼尾组合路径 + // 与 k=1 预编辑投影等调用方同享该保证；生成读法只在静态空间完全无法 + // 解释输入时出场。混表竞争的加权仲裁依赖训练权重，留给方向阶段 3。 const auto cuts = quanpin::autocorrect_cut_kbest(raw_input, autocorrect_types, kAutocorrectCutKBest); if (!cuts.empty()) { diff --git a/engine/quanpin/quanpin_utils.cpp b/engine/quanpin/quanpin_utils.cpp index ae147151..a80f1e1b 100644 --- a/engine/quanpin/quanpin_utils.cpp +++ b/engine/quanpin/quanpin_utils.cpp @@ -3,6 +3,7 @@ #include "autocorrect_table.h" #include "../common/helpcode_utils.h" #include +#include #include #include #include @@ -535,6 +536,10 @@ struct CorrectionTarget // pair. The tie-break weight is derived from the ENABLED subset at query // time (see min_enabled_correction_weight), never from a switched-off table. unsigned type_bit = 0; + // Out-of-table generated pair (non-neighbor substitution / arbitrary-letter + // insertion): flat expensive-tier weight (kAutocorrectGeneratedShapeWeight) + // instead of the per-bit derivation; switch gating still uses type_bit. + bool generated = false; }; // Canonical per-type tie-break cost. Each correction type carries one fixed @@ -574,6 +579,126 @@ int min_enabled_correction_weight(const unsigned enabled_bits) return best; } +// QWERTY letter-key neighbors (finger-movement range, letter keys only), the +// same adjacency as server/scripts/generate_quanpin_autocorrect.py:42-69. +// Indexed by letter - 'a' (a..z order -- NOT keyboard order). Drift is +// self-healing only in one direction: a pair misclassified as "non-neighbor" +// that actually collides with the static table falls back to the cheaper +// static weight, but a neighbor misclassified as "far" simply skips +// generation -- so keep this table byte-identical to the generator's. +constexpr std::array kQwertyNeighbors = { + "qwsz", // a + "vghn", // b + "xdfv", // c + "serfcx", // d + "wrsd", // e + "drtgvc", // f + "ftyhbv", // g + "gyujbn", // h + "uojk", // i + "huikmn", // j + "jiolm", // k + "kop", // l + "njk", // m + "bhjm", // n + "ipkl", // o + "ol", // p + "wa", // q + "etdf", // r + "awdexz", // s + "ryfg", // t + "yihj", // u + "cfgb", // v + "qeas", // w + "zsdc", // x + "tugh", // y + "asx", // z +}; + +constexpr bool is_qwerty_neighbor(char key, char of_letter) +{ + const unsigned row = static_cast(of_letter - 'a'); + return row < kQwertyNeighbors.size() && kQwertyNeighbors[row].find(key) != std::string_view::npos; +} + +// One out-of-table generated pair: wrong-form string plus the legal syllable it +// stands for. wrong is owned by the generated store (static lifetime) because +// the correction index refers to it through string_views. +struct GeneratedPair +{ + std::string wrong; + std::string_view syllable; + unsigned type_bit; +}; + +// Enumerate the out-of-table shapes over every legal syllable: non-neighbor +// substitutions and arbitrary-letter insertions. Transpositions and deletions +// are exhaustive in the static tables -- nothing to add. Wrong forms must be +// non-legal, at least three letters (shorter strings belong to the jianpin +// space) and at most six (the k-best piece scan never looks at longer pieces) -- +// the same window as the static tables and the evaluation filter. +std::vector build_generated_pairs() +{ + const auto &legal = intact_pinyin_set(); + std::vector pairs; + for (const std::string &syllable : intact_pinyin_list()) + { + if (syllable.size() < 3) + { + continue; // same-length substitutions would violate the 3-letter floor + } + // 非相邻键替换：换上的字母既不是原字母也不是它的 QWERTY 邻键。 + for (size_t i = 0; i < syllable.size(); ++i) + { + for (char key = 'a'; key <= 'z'; ++key) + { + if (key == syllable[i] || is_qwerty_neighbor(key, syllable[i])) + { + continue; + } + std::string wrong = syllable; + wrong[i] = key; + if (legal.find(wrong) != legal.end()) + { + continue; + } + pairs.push_back({std::move(wrong), syllable, kAutocorrectNeighbor}); + } + } + // 任意字母插入：插入字母不属于静态覆盖集（该位相邻字母及其邻键）。 + for (size_t i = 0; i <= syllable.size(); ++i) + { + const char left = i > 0 ? syllable[i - 1] : '\0'; + const char right = i < syllable.size() ? syllable[i] : '\0'; + for (char key = 'a'; key <= 'z'; ++key) + { + const bool statically_covered = key == left || key == right || + (left != '\0' && is_qwerty_neighbor(key, left)) || + (right != '\0' && is_qwerty_neighbor(key, right)); + if (statically_covered) + { + continue; + } + std::string wrong = syllable; + wrong.insert(i, 1, key); + if (wrong.size() > 6 || legal.find(wrong) != legal.end()) + { + continue; // 7 letters sit outside the k-best piece scan window + } + pairs.push_back({std::move(wrong), syllable, kAutocorrectInsertion}); + } + } + } + return pairs; +} + +// Static lifetime: the correction index refers to `wrong` through string_views. +const std::vector &generated_pairs() +{ + static const std::vector pairs = build_generated_pairs(); + return pairs; +} + const std::unordered_map> &correction_index() { static const std::unordered_map> kIndex = [] { @@ -609,6 +734,21 @@ const std::unordered_map> &corre add_table(autocorrect::kDeletionEntries, autocorrect::kDeletionCount, kAutocorrectDeletion); add_table(autocorrect::kInsertionEntries, autocorrect::kInsertionCount, kAutocorrectInsertion); add_table(autocorrect::kNeighborEntries, autocorrect::kNeighborCount, kAutocorrectNeighbor); + // Out-of-table generated shapes: appended AFTER the static tables so a + // shared wrong key keeps its cheaper static targets first. A generated + // pair colliding with a static pair is dropped -- the static table's + // cheaper weight already covers it (self-heals engine/generator drift). + for (const auto &pair : generated_pairs()) + { + auto &targets = index[pair.wrong]; + const auto duplicate = std::find_if(targets.begin(), targets.end(), [&](const CorrectionTarget &target) { + return target.syllable == pair.syllable; + }); + if (duplicate == targets.end()) + { + targets.push_back(CorrectionTarget{pair.syllable, pair.type_bit, true}); + } + } return index; }(); return kIndex; @@ -624,6 +764,11 @@ struct AutocorrectEdge size_t raw_length = 0; std::string_view syllable; bool corrected = false; + // Out-of-table generated pair (non-neighbor substitution / arbitrary-letter + // insertion). Generated-containing hypotheses rank behind ALL static + // hypotheses (see the finalize comparator) so in-table inputs keep their + // baseline cut set. + bool generated = false; }; // One propagated cut hypothesis, ranked by (edge_count, weight, arrival): @@ -638,6 +783,10 @@ struct SearchHypothesis size_t prev_index = kNoPredecessor; size_t arrival = 0; int weight = 0; + // True once any generated edge participates in this path. Such hypotheses + // rank behind every static hypothesis (see the finalize comparator) so + // in-table inputs keep their baseline cut set. + bool has_generated = false; AutocorrectEdge edge; // Order-sensitive rolling hash of the joined syllable sequence. Two // hypotheses reaching the same position share a sequence iff their hashes @@ -696,6 +845,14 @@ std::vector autocorrect_cut_kbest(const std::string &pinyin, con return; } std::sort(list.begin(), list.end(), [](const SearchHypothesis &lhs, const SearchHypothesis &rhs) { + // Generated-containing hypotheses rank behind ALL static ones + // regardless of edge count: out-of-table shapes only step in when + // the static space has no better explanation, and in-table inputs + // must keep their baseline cut set (zero-regression by design). + if (lhs.has_generated != rhs.has_generated) + { + return lhs.has_generated < rhs.has_generated; + } if (lhs.edge_count != rhs.edge_count) { return lhs.edge_count < rhs.edge_count; @@ -743,6 +900,7 @@ std::vector autocorrect_cut_kbest(const std::string &pinyin, con child.arrival = arrival++; child.weight = parent.weight + edge_weight; child.edge = edge; + child.has_generated = parent.has_generated || edge.generated; child.seq_hash = extend_sequence_hash(parent.seq_hash, edge.syllable); best[end].push_back(std::move(child)); }; @@ -788,7 +946,9 @@ std::vector autocorrect_cut_kbest(const std::string &pinyin, con } edge.syllable = target.syllable; edge.corrected = true; - const int weight = min_enabled_correction_weight(enabled_bits); + edge.generated = target.generated; + const int weight = + target.generated ? kAutocorrectGeneratedShapeWeight : min_enabled_correction_weight(enabled_bits); for (size_t i = 0; i < hypotheses.size(); ++i) { extend(start + len, hypotheses[i], i, edge, weight); @@ -797,6 +957,18 @@ std::vector autocorrect_cut_kbest(const std::string &pinyin, con } } finalize(length); + // Static-priority zero-regression (see the header contract): when the best + // cut is pure static, drop every generated cut so every caller -- the + // primary resolver, the head+tail composition retry, the k == 1 preedit + // projection -- consumes exactly the baseline cut set; generated readings + // surface only when no static cut explains the input at all. Living here + // (not in each caller) is the point: a future consumer cannot forget it. + if (!best[length].empty() && !best[length].front().has_generated) + { + best[length].erase(std::remove_if(best[length].begin(), best[length].end(), + [](const SearchHypothesis &hypothesis) { return hypothesis.has_generated; }), + best[length].end()); + } // Rebuild each surviving end hypothesis by walking the predecessor chain. // raw_length (not syllable.size()) gives the raw span: deletion edges @@ -808,6 +980,7 @@ std::vector autocorrect_cut_kbest(const std::string &pinyin, con AutocorrectCut cut; cut.edge_count = hypothesis.edge_count; cut.weight = hypothesis.weight; + cut.has_generated = hypothesis.has_generated; size_t position = length; const SearchHypothesis *current = &hypothesis; while (current->prev_index != kNoPredecessor) diff --git a/engine/quanpin/quanpin_utils.h b/engine/quanpin/quanpin_utils.h index 8b8c4115..49eae31d 100644 --- a/engine/quanpin/quanpin_utils.h +++ b/engine/quanpin/quanpin_utils.h @@ -78,6 +78,14 @@ inline constexpr int kAutocorrectDeletionWeight = 11; inline constexpr int kAutocorrectInsertionWeight = 12; inline constexpr int kAutocorrectNeighborWeight = 13; +// Out-of-table generated shapes (non-neighbor substitutions / arbitrary-letter +// insertions, see CorrectionTarget::generated): fixed expensive-tier weight. +// These slips are rarer than any in-table shape, and the ranking contract +// requires them to never displace a table hit on the same wrong key. ponytail: +// flat constant; calibration path = user_journal confusion priors (direction +// task phase 3). +inline constexpr int kAutocorrectGeneratedShapeWeight = 15; + // One segment of an autocorrect-aware cut. syllable is the canonical (possibly // table-corrected) text used for dictionary lookups; raw_text/start describe the // original letters the segment consumed so the preedit can keep showing what the @@ -104,6 +112,11 @@ struct AutocorrectCut // (neighbor gau -> gai, weight 13) regardless of dictionary frequency. size_t edge_count = 0; int weight = 0; + // True when any edge of this cut is an out-of-table generated pair. The + // search itself drops generated-containing cuts whenever a pure-static cut + // tops the ranking, so in-table inputs consume exactly the baseline cut + // set. + bool has_generated = false; bool empty() const { @@ -134,7 +147,12 @@ Segments autocorrect_cut(const std::string &pinyin, unsigned autocorrect_types); // graph and gating as autocorrect_cut, but keeping ambiguous table entries // alive as parallel hypotheses. Ranking key: (corrected edge count, summed // edge weight, generation order = table order); hypotheses explaining the same -// syllable sequence are deduplicated, keeping the best-ranked one. Every +// syllable sequence are deduplicated, keeping the best-ranked one. Static +// priority: generated-containing hypotheses rank behind every static one, and +// when the top cut is pure static all generated cuts are dropped before +// returning -- in-table inputs get exactly the pre-generated-space cut set, +// and out-of-table shapes surface only when no static cut explains the input. +// Every // returned cut contains at least one corrected edge, so a fully legal input // with no correction reading yields an empty vector (the caller owns the plain // segmentation). This is the query-time disambiguation surface of CN 101133411 diff --git a/engine/tests/README.md b/engine/tests/README.md index 5a5996b1..5e2d65c5 100644 --- a/engine/tests/README.md +++ b/engine/tests/README.md @@ -37,7 +37,9 @@ cmake --build build --config Release --target ` --samples 300 --seed 42 --model mixed ` diff --git a/engine/tests/src/eval_quanpin_autocorrect.cpp b/engine/tests/src/eval_quanpin_autocorrect.cpp index 937dc581..d57f7f1e 100644 --- a/engine/tests/src/eval_quanpin_autocorrect.cpp +++ b/engine/tests/src/eval_quanpin_autocorrect.cpp @@ -89,6 +89,7 @@ enum class ErrorModel Deletion, Ambiguous, Insertion, + Outside, }; enum class ErrorKind @@ -97,6 +98,8 @@ enum class ErrorKind Neighbor, Deletion, Insertion, + FarSubstitution, + FarInsertion, }; struct Options @@ -120,7 +123,7 @@ void usage_exit(const std::string &message) std::cerr << "ERROR: " << message << "\n"; } std::cerr << "usage: eval_quanpin_autocorrect --db [--samples N] [--seed S]\n" - << " [--rates 10,25,50,100] [--model mixed|deletion|ambiguous|insertion]\n" + << " [--rates 10,25,50,100] [--model mixed|deletion|ambiguous|insertion|outside]\n" << " [--resource ] [--csv ] [--seeds-file ] [--dump N]\n"; std::exit(2); } @@ -216,6 +219,10 @@ Options parse_options(int argc, char *argv[]) { options.model = ErrorModel::Insertion; } + else if (options.model_name == "outside") + { + options.model = ErrorModel::Outside; + } else { usage_exit("unknown --model: " + options.model_name); @@ -434,7 +441,7 @@ std::vector raw_variants(ErrorKind kind, const std::string &syllabl out.push_back(syllable.substr(0, i) + syllable.substr(i + 1)); } } - else + else if (kind == ErrorKind::Insertion) { // 插入：与生成器 insertion_variants 同款约束——位置锚定串首（参照右 // 邻）、串尾（参照左邻）、中间（参照任一侧），插入字母必须是相邻键的 @@ -472,6 +479,69 @@ std::vector raw_variants(ErrorKind kind, const std::string &syllabl } } } + else if (kind == ErrorKind::FarSubstitution) + { + // 表外形状：非相邻键替换（替换键不是原键也不是其 QWERTY 邻键）。与 + // 静态表替换形状构造性不相交，用于度量生成式纠错空间的覆盖增益。 + for (size_t i = 0; i < syllable.size(); ++i) + { + std::unordered_set blocked; + blocked.insert(syllable[i]); + for (const auto &[key, neighbors] : qwerty_neighbors()) + { + if (key == syllable[i]) + { + blocked.insert(neighbors.begin(), neighbors.end()); + } + } + for (char key = 'a'; key <= 'z'; ++key) + { + if (blocked.count(key) != 0) + { + continue; + } + out.push_back(syllable.substr(0, i) + key + syllable.substr(i + 1)); + } + } + } + else if (kind == ErrorKind::FarInsertion) + { + // 表外形状：插入字母不属于静态覆盖集（该位相邻字母及其邻键）。 + for (size_t position = 0; position <= syllable.size(); ++position) + { + std::unordered_set covered; + if (position > 0) + { + covered.insert(syllable[position - 1]); + for (const auto &[key, neighbors] : qwerty_neighbors()) + { + if (key == syllable[position - 1]) + { + covered.insert(neighbors.begin(), neighbors.end()); + } + } + } + if (position < syllable.size()) + { + covered.insert(syllable[position]); + for (const auto &[key, neighbors] : qwerty_neighbors()) + { + if (key == syllable[position]) + { + covered.insert(neighbors.begin(), neighbors.end()); + } + } + } + for (char key = 'a'; key <= 'z'; ++key) + { + if (covered.count(key) != 0) + { + continue; + } + out.push_back(syllable.substr(0, position) + key + syllable.substr(position)); + } + } + } return out; } @@ -495,7 +565,8 @@ std::vector usable_variants(ErrorKind kind, const std::string &syll } // mixed = 交换 25% / 邻键 25% / 漏字 25% / 多字 25%；deletion = 100% 漏字； -// insertion = 100% 多字。选中的错误类型在该音节上无可用变体（如 2 字母音节 +// insertion = 100% 多字；outside = 非相邻替换 / 远键插入各半（表外形状，度量 +// 生成式纠错空间的覆盖增益）。选中的错误类型在该音节上无可用变体（如 2 字母音节 // 或 6 字母音节的插入变体超长）时保持原样，交由样本级作废逻辑处理，不回退 // 到其他类型，保证分布可解释。 std::string corrupt_syllable(const std::string &syllable, std::mt19937 &rng, ErrorModel model, @@ -514,6 +585,11 @@ std::string corrupt_syllable(const std::string &syllable, std::mt19937 &rng, Err { kind = ErrorKind::Insertion; } + else if (model == ErrorModel::Outside) + { + kind = std::uniform_int_distribution(1, 100)(rng) <= 50 ? ErrorKind::FarSubstitution + : ErrorKind::FarInsertion; + } const auto variants = usable_variants(kind, syllable, legal); if (variants.empty()) { diff --git a/engine/tests/src/test_input_session.cpp b/engine/tests/src/test_input_session.cpp index 0b196d99..7a5e9c9e 100644 --- a/engine/tests/src/test_input_session.cpp +++ b/engine/tests/src/test_input_session.cpp @@ -411,6 +411,96 @@ void run_autocorrect_context_ranking_tests(const std::filesystem::path &data_dir } } +// 阶段 2 生成式纠错空间（任务 quanpin-autocorrect-generated-space）：静态表 +// 形状之外的单编辑手误由生成式索引兜底，权重落贵档（15）。隔离 fixture： +// - shatg = shang 的 n→t（t 非邻键，远键替换） +// - zthou = zhou 的 z/h 之间插 t（t 不在静态覆盖集，远键插入） +// - chng 同键双读：chng→chang（静态漏字 11）与 chng→cang（生成 15）， +// 静态最优时生成切分整体丢弃——静态优先语义的判别性断言 +// - shatngzh = head shatng + 简拼尾 zh：head 组合路径同享静态优先 +void run_autocorrect_generated_space_tests(const std::filesystem::path &data_directory) +{ + const std::filesystem::path directory = data_directory / "autocorrect-generated"; + std::filesystem::create_directories(directory); + { + Database database(directory / "msime.db"); + database.execute("CREATE TABLE tbl_1_s(key TEXT,jp TEXT,value TEXT,weight INTEGER);" + "INSERT INTO tbl_1_s VALUES('shang', 's', '上', 900);"); + database.execute("CREATE TABLE tbl_1_z(key TEXT,jp TEXT,value TEXT,weight INTEGER);" + "INSERT INTO tbl_1_z VALUES('zhou', 'z', '周', 900);"); + database.execute("CREATE TABLE tbl_1_c(key TEXT,jp TEXT,value TEXT,weight INTEGER);" + "INSERT INTO tbl_1_c VALUES('chang', 'c', '长', 900);" + "INSERT INTO tbl_1_c VALUES('cang', 'c', '仓', 5000);"); + database.execute("CREATE TABLE tbl_2_s(key TEXT,jp TEXT,value TEXT,weight INTEGER);" + "INSERT INTO tbl_2_s VALUES('shang''zh', 'sz', '上周', 900);"); + database.execute("CREATE TABLE tbl_3_s(key TEXT,jp TEXT,value TEXT,weight INTEGER);" + "INSERT INTO tbl_3_s VALUES('sha''tang''zh', 'stz', '沙汤扎', 900);"); + } + + metasequoia::RuntimePaths paths; + paths.resources = directory; + paths.user_data = directory; + paths.cache = directory; + paths.dictionaries = directory; + const unsigned both = quanpin::kAutocorrectTransposition | quanpin::kAutocorrectNeighbor; + + // 远键替换：shatg → shang（生成对，贵档 15）。 + { + metasequoia::InputSession session(SchemeType::Quanpin, both, true, true, true, paths); + type(session, "shatg"); + const auto found = find_candidate_index(session, "上"); + require(found < session.candidates().size() && session.candidates()[found].corrected_from == "shatg", + "A far-substitution typo must be corrected to 上 with the typed letters recorded."); + } + + // 远键插入（词中位）：zthou → zhou（t 不在 z/h 的覆盖集）。fixture 特意 + // 选无竞争切分的输入：zhwou 这类插入位会拼出 [zha(w 邻键替换)+ou] 等更 + // 便宜的合法切分（权重 13 < 15），生成读法按契约排后被前缀候选遮蔽—— + // 那是排序语义，不是生成类失效。zthou 无任何竞争切分，唯一读即纠错读法。 + { + metasequoia::InputSession session(SchemeType::Quanpin, both, true, true, true, paths); + type(session, "zthou"); + const auto found = find_candidate_index(session, "周"); + require(found < session.candidates().size() && session.candidates()[found].corrected_from == "zthou", + "A far-insertion typo must be corrected to 周 with the typed letters recorded."); + } + + // 静态优先语义：chng = chang 漏 a。读法竞争——chng→chang（静态漏字 11）与 + // chng→cang（生成远键替换 h→a，15）。最优切分为纯静态时生成切分整体丢弃： + // 仓不出现（混表加权仲裁依赖训练权重，方向阶段 3），长照常领衔。 + { + metasequoia::InputSession session(SchemeType::Quanpin, both, true, true, true, paths); + type(session, "chng"); + require(candidate_index(session, "长") == 0, "The in-table cheaper deletion reading must lead for chng."); + require( + find_candidate_index(session, "仓") == session.candidates().size(), + "Static-priority semantics: the generated cang reading must not surface when a static cut tops the input."); + } + + // head+简拼尾组合路径的静态优先：shatngzh 在整串 k-best 上无解（尾部 zh 非 + // 完整音节），走 head 重试。head shatng 的读法竞争——sha+tng→tang（静态漏字 + // 11）与 shatng→shang（生成插入 15）。丢弃规则由 k-best 搜索自身执行，head + // 路径同享：静态读法的沙汤扎（sha'tang'zh 精确键）在场，上周（shang+zh 生成 + // 读法经 costlier 档，表内输入的基线里不存在）不得出现。 + { + metasequoia::InputSession session(SchemeType::Quanpin, both, true, true, true, paths); + type(session, "shatngzh"); + require(find_candidate_index(session, "沙汤扎") < session.candidates().size(), + "The head-composition path must still correct shatng to sha+tang."); + require(find_candidate_index(session, "上周") == session.candidates().size(), + "Static priority must hold on the head path: the generated shang+zh reading must not surface."); + } + + // 守卫与合法输入：尾位插入（shangv 类）受简拼尾守卫——纠错不触发由 + // test_pinyin 的 zheg 用例与输入门不变式覆盖，此处不重复；合法拼写原样直出。 + { + metasequoia::InputSession legal(SchemeType::Quanpin, both, true, true, true, paths); + type(legal, "shang"); + require(candidate_index(legal, "上") == 0 && legal.candidates()[0].corrected_from.empty(), + "A legal spelling must stay uncorrected."); + } +} + void run_caret_prefix_session_tests(const std::filesystem::path &data_directory) { const std::filesystem::path directory = data_directory / "caret-prefix"; @@ -1317,6 +1407,7 @@ int run_test() run_umlaut_alias_session_tests(data_directory); run_caret_prefix_session_tests(data_directory); run_autocorrect_context_ranking_tests(data_directory); + run_autocorrect_generated_space_tests(data_directory); #endif #ifndef METASEQUOIA_SKIP_FREQUENCY_TESTS From 7a8b441509dc2226b942eda5466a16a4eeb80202 Mon Sep 17 00:00:00 2001 From: Jia Sui Date: Sat, 3 Oct 2026 21:05:47 +0800 Subject: [PATCH 03/12] =?UTF-8?q?fix(engine):=20RescoreWorker=20=E4=B8=8D?= =?UTF-8?q?=E5=86=8D=E4=B8=A2=E5=BC=83=E6=9B=B4=E6=96=B0=E7=9A=84=E9=87=8D?= =?UTF-8?q?=E6=8E=92=E8=AF=B7=E6=B1=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit active 分支命中时（同一批正在算）原先会顺手清掉该模型的 pending_ 待办， 理由是「排在它后面的旧待办没人要了」。这个前提不成立： - 模型 M 正在算 K1，用户继续打字把 K2 排进 pending_ - K1 算完写 done_ 并回调重查；若重查赶在工作线程擦 active_keys_ 之前， 且 done_ 刚被 kMaxResults 重置清空过，done_.find 未命中就落进 active 分支 - K2 被静默抹掉，而本次调用拿到 nullopt、也不会有回调 失败模式是那次重排永久缺失，不是延迟。active 分支只表达「这批正在算， 别重复排」，两件事拆开；pending_ 本就按模型只留一格、下次 order_for 直接覆盖。 --- engine/neural/rescore_worker.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/engine/neural/rescore_worker.cpp b/engine/neural/rescore_worker.cpp index 35f2664b..8f868800 100644 --- a/engine/neural/rescore_worker.cpp +++ b/engine/neural/rescore_worker.cpp @@ -70,9 +70,11 @@ std::optional> RescoreWorker::order_for(const SentenceM const auto active = active_keys_.find(&model); if (active != active_keys_.end() && active->second == key) { - // 正在算的就是这一批，算完自会进表并回调。这也是该模型最新的请求，排在它后面的旧待办 - // 已经没人要了（用户打出去又退了回来），一并丢掉。 - pending_.erase(&model); + // 正在算的就是这一批，算完自会进表并回调，所以不必再排一份。 + // 刻意不碰 pending_：这一批在算的这些工夫里，用户完全可能已经打到了更新的输入， + // pending_ 里那个才是当前请求。抹掉它就再没有人排它了——本次调用拿到 nullopt 也 + // 不会有回调，失败模式是那次重排永久缺失，而不只是延迟。pending_ 按模型留一格、 + // 下次 order_for 直接覆盖，本来就不需要这里清理。 return std::nullopt; } Job job; From 6d8f7b6e2ba79f8947ba4077511b040ae9392d41 Mon Sep 17 00:00:00 2001 From: Jia Sui Date: Sat, 3 Oct 2026 21:06:01 +0800 Subject: [PATCH 04/12] =?UTF-8?q?refactor(engine):=20=E7=A7=BB=E9=99=A4=20?= =?UTF-8?q?AutocorrectCut::has=5Fgenerated=20=E6=AD=BB=E5=AD=97=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 该字段在 autocorrect_cut_kbest 重建结果时赋值，作为公开结构体字段暴露在头 文件并配了契约注释，但 engine/、server/、windows/ 及全部测试都没有读取点。 静态优先的过滤实际发生在 autocorrect_cut_kbest 内部的 SearchHypothesis 上 （排序比较器 + 末尾按 front().has_generated 决定是否整批 erase），在 cut 重建 之前就完成了。头文件注释把契约归因到这个字段，读者会以为调用方依赖它。 删掉字段与赋值，契约说明改归真正的机制。SearchHypothesis::has_generated 是 活代码，保留。 --- engine/quanpin/quanpin_utils.cpp | 1 - engine/quanpin/quanpin_utils.h | 8 +------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/engine/quanpin/quanpin_utils.cpp b/engine/quanpin/quanpin_utils.cpp index a80f1e1b..60a74640 100644 --- a/engine/quanpin/quanpin_utils.cpp +++ b/engine/quanpin/quanpin_utils.cpp @@ -980,7 +980,6 @@ std::vector autocorrect_cut_kbest(const std::string &pinyin, con AutocorrectCut cut; cut.edge_count = hypothesis.edge_count; cut.weight = hypothesis.weight; - cut.has_generated = hypothesis.has_generated; size_t position = length; const SearchHypothesis *current = &hypothesis; while (current->prev_index != kNoPredecessor) diff --git a/engine/quanpin/quanpin_utils.h b/engine/quanpin/quanpin_utils.h index 49eae31d..89499648 100644 --- a/engine/quanpin/quanpin_utils.h +++ b/engine/quanpin/quanpin_utils.h @@ -112,11 +112,6 @@ struct AutocorrectCut // (neighbor gau -> gai, weight 13) regardless of dictionary frequency. size_t edge_count = 0; int weight = 0; - // True when any edge of this cut is an out-of-table generated pair. The - // search itself drops generated-containing cuts whenever a pure-static cut - // tops the ranking, so in-table inputs consume exactly the baseline cut - // set. - bool has_generated = false; bool empty() const { @@ -152,8 +147,7 @@ Segments autocorrect_cut(const std::string &pinyin, unsigned autocorrect_types); // when the top cut is pure static all generated cuts are dropped before // returning -- in-table inputs get exactly the pre-generated-space cut set, // and out-of-table shapes surface only when no static cut explains the input. -// Every -// returned cut contains at least one corrected edge, so a fully legal input +// Every returned cut contains at least one corrected edge, so a fully legal input // with no correction reading yields an empty vector (the caller owns the plain // segmentation). This is the query-time disambiguation surface of CN 101133411 // B: ambiguity survives the cut layer and is settled by dictionary frequency. From 7d83aed411f9b503dce776ed4a6d6f3098abb6d2 Mon Sep 17 00:00:00 2001 From: Jia Sui Date: Sat, 3 Oct 2026 21:31:42 +0800 Subject: [PATCH 05/12] =?UTF-8?q?docs(engine):=20=E8=AF=B4=E6=98=8E=20wake?= =?UTF-8?q?=20=E8=B0=93=E8=AF=8D=E4=BE=9D=E8=B5=96=E7=9A=84=E8=B7=A8?= =?UTF-8?q?=E8=A1=A8=E4=B8=8D=E5=8F=98=E9=87=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit wake_ 的谓词按模型判空槽、不比键，成立的前提是 pending_ 与 active_keys_ 不可能 同时持有同一个模型：待办是在 worker 取走的那一刻才从表里 erase 的，在那之前同一 模型的键只挂在 active_keys_ 上。 这个不变量此前没有文字说明。谁哪天在 order_for 里改成「在算的也照样排一份」 （在 42aeda08 之前看着像无害的加固），谓词会把正在算的那批重排一遍——正是该字段 引入前要消灭的重复计算。纯注释，行为不变。 --- engine/neural/rescore_worker.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/engine/neural/rescore_worker.cpp b/engine/neural/rescore_worker.cpp index 8f868800..38646687 100644 --- a/engine/neural/rescore_worker.cpp +++ b/engine/neural/rescore_worker.cpp @@ -132,6 +132,9 @@ void RescoreWorker::run() Job job; { std::unique_lock lock(mutex_); + // 这里按模型判「有没有空槽」就够，不必比键：待办是在下面取走的那一刻才从表里 erase 的， + // 在那之前同一个模型的键只挂在 active_keys_ 上，两张表不可能记着同一个模型同一批——否则 + // 这个谓词会把正在算的那一批重排一遍。按键去重的那一层判断在 order_for 里。 wake_.wait(lock, [this] { return stopping_ || std::any_of(pending_.begin(), pending_.end(), [this](const auto &item) { return active_keys_.find(item.first) == active_keys_.end(); From 044352d49addd8e24d1fea2b0ba603ebf9f25ed3 Mon Sep 17 00:00:00 2001 From: Jia Sui Date: Sat, 3 Oct 2026 22:15:14 +0800 Subject: [PATCH 06/12] =?UTF-8?q?fix(engine):=20=E4=B8=8A=E4=B8=8B?= =?UTF-8?q?=E6=96=87=E9=87=8D=E6=8E=92=E7=9A=84=E9=A2=84=E8=BF=87=E6=BB=A4?= =?UTF-8?q?=E5=9F=BA=E5=87=86=E6=94=B9=E7=94=A8=E6=AD=A3=E8=AF=BB=E6=95=B4?= =?UTF-8?q?=E9=94=AE=E9=A6=96=E4=BD=8D=E6=9D=83=E9=87=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1/100 预过滤的基准取 result.front()，注释声称与merge_alternative_segmentations 的 primary_top_weight 同源，实际不是：merge 取的是 primary_full（query_single_path 的整键结果），无整键行时为 0；而这里的 result 是 query_series 的完整输出。 正读在词库里没有整词时，result.front() 是一个前缀单字——真实词库里单字是 1e6+ 的语料计数——于是 5000*100 < 1e6，正常备选在解码之前就被筛掉。而这恰恰是重排价值 最高的场景：正读最弱、词格只能用单字拼出路径时，正是上下文裁决该出手的时候。 改为复用 merge 的同一调用（query_single_path + 同一实参），走缓存不额外查库， 两处门槛从此不会各自漂移。 该缺陷无法用行为测试判别：正读无整词时 merge 的merged_full 只剩备选整键行， 备选本就居首，重排发生与否结果相同；构造分歧需要「词格胜者≠ 权重排序首位」， 而启发式 ln(w)+3.0×音节数 下同音节数时权重恒占优。故作为一致性修复保留。 --- engine/quanpin/quanpin_dictionary.cpp | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/engine/quanpin/quanpin_dictionary.cpp b/engine/quanpin/quanpin_dictionary.cpp index 5201139f..edccc5c7 100644 --- a/engine/quanpin/quanpin_dictionary.cpp +++ b/engine/quanpin/quanpin_dictionary.cpp @@ -448,9 +448,16 @@ std::vector QuanpinDictionary::query_exact(const std::string &raw_inpu if (primary_score.has_value()) { // 前置过滤沿用保护位同款 1/100 量级：今天就赢不了晋升的备选不付解码钱。 - // 主切首位权重取 result 首行（精确键命中行），与 merge_alternative_ - // segmentations 的 primary_top_weight 同源。 - const std::int64_t primary_top_weight = result.empty() ? 0 : result.front().weight; + // 基准必须与 merge_alternative_segmentations 的 primary_top_weight 同源， + // 即取正读**整键**命中的首位权重、没有整键行时取 0。不能取 result.front()： + // result 是 query_series 的完整输出，头部可能是高权重前缀单字（真实词库里 + // 单字是 1e6+ 的语料计数，见 heuristic_log_prob）或生成的整句。正读两音节 + // 在词库里没有整词、词格只能用单字拼出路径时，那正是重排价值最高的场景，拿单字 + // 当基准会在解码之前就把正常备选全筛掉，重排悄悄不生效。同参调用 + // query_single_path 走缓存，不额外查库。 + const std::string primary_key = quanpin::join_segments(resolution.corrected_segments); + const auto primary_full = query_single_path(raw_input, primary_key, resolution.corrected_segments); + const std::int64_t primary_top_weight = primary_full.empty() ? 0 : primary_full.front().weight; const auto alternative_items = quanpin::query_exact_segmentations_keyed_flat( resolution.alternative_corrected_cuts, db_, statement_cache_, kAlternativeSegmentationCandidateLimit); From e1331ff1dc8124782258f3e6b2ba3b8a95ee2fd4 Mon Sep 17 00:00:00 2001 From: Jia Sui Date: Sat, 3 Oct 2026 22:16:35 +0800 Subject: [PATCH 07/12] =?UTF-8?q?fix(engine):=20=E6=81=A2=E5=A4=8D?= =?UTF-8?q?=E4=B8=8A=E4=B8=8B=E6=96=87=E9=87=8D=E6=8E=92=E7=9A=84=E3=80=8C?= =?UTF-8?q?=E6=95=B4=E9=94=AE=E5=9C=A8=E5=89=8D=E3=80=81=E5=89=8D=E7=BC=80?= =?UTF-8?q?=E5=9C=A8=E5=90=8E=E3=80=8D=E5=88=86=E5=B1=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 胜出切分接管后用 query_series(raw_input, winner_key, winner_segments) 取它的结果， 但该函数的 count 从整键递减到 1，返回的不只是整键行，还有首音节的全部单字；而 rest（班长、搬账这些同键位整词）在这之后才append。 结果是办证/辩证/整句/班/办/半/般…/班长——胜出切分的单字插到同键位整词之前，把它们 整组挤出首页，PR 描述里「同键位纠错读法整组下移」并不成立。 按 canonical_pinyin == winner_key 把前缀行摘出来，恢复 merge_alternative_segmentations 的既有分层（merged_full 在前、前缀作为 remaining 追加）：整键块、rest、前缀块。分类 键选 canonical_pinyin 而非 pinyin——后者承载用户键入码（可能是简拼）；整句与词格行 同样落在整键块内，因为 append_ime_fallback 用完整 segmentation、词格 select_distinct 用 path->key 填 canonical_pinyin。 补判别性用例：原 fixture 缺 tbl_1_b，前缀这条路根本没被走到。已实测——撤掉本修复 时该用例失败（Same-tier whole words must not be pushed behind the winner cut's single-character prefixes.），装回后通过。 --- engine/quanpin/quanpin_dictionary.cpp | 18 +++++- engine/tests/src/test_input_session.cpp | 79 +++++++++++++++++++++++++ 2 files changed, 96 insertions(+), 1 deletion(-) diff --git a/engine/quanpin/quanpin_dictionary.cpp b/engine/quanpin/quanpin_dictionary.cpp index edccc5c7..ff2bfe77 100644 --- a/engine/quanpin/quanpin_dictionary.cpp +++ b/engine/quanpin/quanpin_dictionary.cpp @@ -513,8 +513,24 @@ std::vector QuanpinDictionary::query_exact(const std::string &raw_inpu std::vector rest = merge_alternative_segmentations(raw_input, pinyin_segmentation_, resolution.corrected_segments, remaining_alternatives, std::move(result)); - result = query_series(raw_input, winner_key, winner_segments); + // query_series 的 count 从整键递减到 1，返回的不只是胜出切分的整键行，还有它 + // 首音节的全部单字。整句与词格行跟整键行同键——append_ime_fallback 用完整 + // segmentation 填 canonical_pinyin，词格 select_distinct 用 path->key——所以 + // 按 canonical_pinyin == winner_key 能干净地把前缀行摘出去。 + // + // 不切开会破坏 merge_alternative_segmentations 的既有分层（merged_full 在前、 + // 前缀作为 remaining 追加）：query_series(ban'zheng) 先出 办证/辩证/整句，再出 + // 班/办/半/般…，而班长、搬账这些同键位整词在 rest 里、排在单字之后，整组被挤 + // 出首页。这里恢复同一分层：整键块、rest、前缀块。 + std::vector winner_full; + std::vector winner_prefix; + for (auto &item : query_series(raw_input, winner_key, winner_segments)) + { + (item.canonical_pinyin == winner_key ? winner_full : winner_prefix).push_back(std::move(item)); + } + result = std::move(winner_full); append_unique_words(result, std::move(rest)); + append_unique_words(result, std::move(winner_prefix)); context_reordered = true; } } diff --git a/engine/tests/src/test_input_session.cpp b/engine/tests/src/test_input_session.cpp index 7a5e9c9e..ddb4366b 100644 --- a/engine/tests/src/test_input_session.cpp +++ b/engine/tests/src/test_input_session.cpp @@ -411,6 +411,83 @@ void run_autocorrect_context_ranking_tests(const std::filesystem::path &data_dir } } +// 审阅缺陷 D1（PR #592 行内评论）：上下文重排接管后，胜出切分的单字前缀曾被插到 +// 同键位整词前面。query_series 的 count 从整键递减到 1，ban'zheng 的输出里除了 +// 办证/辩证/整句还带着首音节的单字；这些单字排在 rest（班长、搬账）之前，把同键位的 +// 其他整词整组挤出首页。原 fixture 没有 tbl_1_b，前缀这条路根本没被走到。 +// 这里补单字表，并直接断言「整键在前、前缀在后」这条分层不变量。 +void run_autocorrect_context_layering_tests(const std::filesystem::path &data_directory) +{ + const std::filesystem::path directory = data_directory / "autocorrect-context-layering"; + std::filesystem::create_directories(directory); + { + Database database(directory / "msime.db"); + database.execute("CREATE TABLE tbl_2_b(key TEXT,jp TEXT,value TEXT,weight INTEGER);" + "INSERT INTO tbl_2_b VALUES('ban''zheng', 'bz', '办证', 100000);" + "INSERT INTO tbl_2_b VALUES('ban''zheng', 'bz', '辩证', 10);" + "INSERT INTO tbl_2_b VALUES('ban''zhang', 'bz', '班长', 5000);" + "INSERT INTO tbl_2_b VALUES('ban''zhang', 'bz', '搬账', 4000);"); + // 单字表：query_series 递减到 count==1 时查它，前缀行由此产生。单字权重取 1e6 + // 量级（真实词库里单字是语料计数），确保它们不会被 append 阶段的去重或调频挪走。 + database.execute("CREATE TABLE tbl_1_b(key TEXT,jp TEXT,value TEXT,weight INTEGER);" + "INSERT INTO tbl_1_b VALUES('ban', 'b', '办', 1000000);" + "INSERT INTO tbl_1_b VALUES('ban', 'b', '半', 900000);"); + } + + metasequoia::RuntimePaths paths; + paths.resources = directory; + paths.user_data = directory; + paths.cache = directory; + paths.dictionaries = directory; + const unsigned both = quanpin::kAutocorrectTransposition | quanpin::kAutocorrectNeighbor; + SentenceAssociationOptions lattice_only; + lattice_only.word_lattice = true; + + metasequoia::InputSession session(SchemeType::Quanpin, both, true, true, true, paths); + session.set_sentence_association(lattice_only); + type(session, "banzhng"); + + const auto &items = session.candidates(); + const auto index_of = [&items](const std::string &word) { + for (size_t i = 0; i < items.size(); ++i) + { + if (items[i].word == word) + { + return i; + } + } + return items.size(); + }; + + // 先确认 fixture 真的同时产出了整键行与单字前缀行，否则下面的分层断言是空的。 + const size_t prefix_a = index_of("办"); + const size_t prefix_b = index_of("半"); + require(prefix_a != items.size() && prefix_b != items.size(), + "The layering fixture must surface single-character prefix rows for the winner cut."); + const size_t sibling = index_of("班长"); + require(sibling != items.size(), "The layering fixture must surface the same-tier whole word."); + + require(index_of("办证") < prefix_a, "The winner cut's whole word must precede its own single-character prefixes."); + require(sibling < prefix_a && sibling < prefix_b, + "Same-tier whole words must not be pushed behind the winner cut's single-character prefixes."); + + // 分层的完整形式：所有整键行都在任何前缀行之前。全拼键用 ' 分隔音节， + // 单音节键不含 '。 + const auto is_prefix_row = [&items](size_t i) { return items[i].canonical_pinyin.find('\'') == std::string::npos; }; + bool seen_prefix = false; + for (size_t i = 0; i < items.size(); ++i) + { + if (is_prefix_row(i)) + { + seen_prefix = true; + } + else + { + require(!seen_prefix, "Every whole-key row must precede every single-character prefix row."); + } + } +} + // 阶段 2 生成式纠错空间（任务 quanpin-autocorrect-generated-space）：静态表 // 形状之外的单编辑手误由生成式索引兜底，权重落贵档（15）。隔离 fixture： // - shatg = shang 的 n→t（t 非邻键，远键替换） @@ -1407,6 +1484,8 @@ int run_test() run_umlaut_alias_session_tests(data_directory); run_caret_prefix_session_tests(data_directory); run_autocorrect_context_ranking_tests(data_directory); + run_autocorrect_context_layering_tests(data_directory); + run_autocorrect_generated_space_tests(data_directory); #endif From bbebd34d226880088f6c370545e920c2bdf88bf3 Mon Sep 17 00:00:00 2001 From: Jia Sui Date: Sat, 3 Oct 2026 22:41:44 +0800 Subject: [PATCH 08/12] =?UTF-8?q?docs(engine):=20=E8=A1=A5=E4=B8=8A?= =?UTF-8?q?=E4=B8=8B=E6=96=87=E9=87=8D=E6=8E=92=E5=88=87=E5=88=86=E4=B8=8E?= =?UTF-8?q?=E7=94=A8=E4=BE=8B=E6=89=93=E5=88=86=E7=9A=84=E5=89=8D=E6=8F=90?= =?UTF-8?q?=E8=AF=B4=E6=98=8E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit R2 的分层判据是 canonical_pinyin == winner_key。原注释只说明了「整句与词格行 同键」，没解释为什么整键档**必然**是 winner_key。而 query_single_cut_keyed 在精确 键查空时会退到前缀区间扫描，那类行的 canonical 是别的读音（gun'qi 捞出 gun'qiu）， 放宽判据就会静默把同档整词沉到末尾——正是本任务在修的那类回归。补上这条不变量： decode_pool 由 query_exact_segmentations_keyed_flat 只收精确键构造，词库里没有整词的 切分进不了池，这条路在胜出切分上不可达。 测试侧补上「fixture 无 sc.lm ⇒ 启发式打分 ⇒ 用例确定」这个前提（同组 ranking fixture 已有，新用例依赖同一前提却没写）。 --- engine/quanpin/quanpin_dictionary.cpp | 6 ++++++ engine/tests/src/test_input_session.cpp | 2 ++ 2 files changed, 8 insertions(+) diff --git a/engine/quanpin/quanpin_dictionary.cpp b/engine/quanpin/quanpin_dictionary.cpp index ff2bfe77..23fc234b 100644 --- a/engine/quanpin/quanpin_dictionary.cpp +++ b/engine/quanpin/quanpin_dictionary.cpp @@ -518,6 +518,12 @@ std::vector QuanpinDictionary::query_exact(const std::string &raw_inpu // segmentation 填 canonical_pinyin，词格 select_distinct 用 path->key——所以 // 按 canonical_pinyin == winner_key 能干净地把前缀行摘出去。 // + // 这个判据依赖一个前置事实：整键档的 canonical 必然就是 winner_key 本身。 + // 胜出切分只能出自 decode_pool，那里用 query_exact_segmentations_keyed_flat + // 只收精确键，词库里没有整词的切分压根进不了池；于是整键档走的是 + // query_single_cut_keyed 的精确键分支，不会退到前缀区间扫描那层降级 + // （那种行的 key 是别的读音，会被这个判据误当成前缀行沉到末尾）。 + // // 不切开会破坏 merge_alternative_segmentations 的既有分层（merged_full 在前、 // 前缀作为 remaining 追加）：query_series(ban'zheng) 先出 办证/辩证/整句，再出 // 班/办/半/般…，而班长、搬账这些同键位整词在 rest 里、排在单字之后，整组被挤 diff --git a/engine/tests/src/test_input_session.cpp b/engine/tests/src/test_input_session.cpp index ddb4366b..4641c885 100644 --- a/engine/tests/src/test_input_session.cpp +++ b/engine/tests/src/test_input_session.cpp @@ -416,6 +416,8 @@ void run_autocorrect_context_ranking_tests(const std::filesystem::path &data_dir // 办证/辩证/整句还带着首音节的单字；这些单字排在 rest（班长、搬账）之前，把同键位的 // 其他整词整组挤出首页。原 fixture 没有 tbl_1_b，前缀这条路根本没被走到。 // 这里补单字表，并直接断言「整键在前、前缀在后」这条分层不变量。 +// 同上一组 fixture：词库目录里没有 sc.lm，词格退回启发式打分，权重完全决定路径分， +// 因此 zheng 侧稳定胜出、重排稳定接管，用例不依赖任何模型文件。 void run_autocorrect_context_layering_tests(const std::filesystem::path &data_directory) { const std::filesystem::path directory = data_directory / "autocorrect-context-layering"; From 3a0d95e143630a4a8b088be13092e90829c74683 Mon Sep 17 00:00:00 2001 From: Jia Sui Date: Sun, 4 Oct 2026 04:25:25 +0800 Subject: [PATCH 09/12] =?UTF-8?q?test(engine,server):=20=E5=AF=B9=E9=BD=90?= =?UTF-8?q?=E7=94=9F=E6=88=90=E5=BC=8F=E7=BA=A0=E9=94=99=E7=A9=BA=E9=97=B4?= =?UTF-8?q?=E7=9A=84=E7=B1=BB=E5=9E=8B=E4=BD=8D=E5=8F=AF=E8=BE=BE=E6=80=A7?= =?UTF-8?q?=E6=96=AD=E8=A8=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- engine/quanpin/quanpin_utils.h | 25 +++++++++- engine/tests/src/test_pinyin.cpp | 60 +++++++++++++++++------- server/tests/src/test_quanpin_scheme.cpp | 37 ++++++++++++--- 3 files changed, 97 insertions(+), 25 deletions(-) diff --git a/engine/quanpin/quanpin_utils.h b/engine/quanpin/quanpin_utils.h index 89499648..49f69275 100644 --- a/engine/quanpin/quanpin_utils.h +++ b/engine/quanpin/quanpin_utils.h @@ -147,6 +147,17 @@ Segments autocorrect_cut(const std::string &pinyin, unsigned autocorrect_types); // when the top cut is pure static all generated cuts are dropped before // returning -- in-table inputs get exactly the pre-generated-space cut set, // and out-of-table shapes surface only when no static cut explains the input. +// +// One caveat that the ranking guarantee does not cover: the generated space hangs +// its out-of-table SUBSTITUTIONS off kAutocorrectNeighbor and its insertions off +// kAutocorrectInsertion, so a type bit no longer maps one-to-one onto a table. An +// input with no static cut at all is now reachable through a single bit -- "sahng" +// reads as "sa'ang" under the neighbour bit alone, "shng" as "sang". Callers must +// not read a bit as "only this family of typo". Product behaviour is unaffected +// because engine.cpp always pairs the deletion and insertion bits with the legacy +// switches, so the only masks that ever occur are 0 / 0xd / 0xe / 0xf; the +// head+jianpin-tail path in quanpin_dictionary.cpp additionally masks the +// neighbour bit off, which suppresses every generated substitution for free. // Every returned cut contains at least one corrected edge, so a fully legal input // with no correction reading yields an empty vector (the caller owns the plain // segmentation). This is the query-time disambiguation surface of CN 101133411 @@ -158,11 +169,21 @@ std::vector autocorrect_cut_kbest(const std::string &pinyin, uns std::size_t k = 3); // True when the input reads as one or more legal syllables plus at most one trailing -// letter ("zheg" = zhe + g): a jianpin-intent shape the correction tables must not -// rewrite. Deliberately NOT true for all-consonant strings of 3+ letters: the engine +// letter ("zheg" = zhe + g): a jianpin-intent shape, which is user intent and never +// a typo. Deliberately NOT true for all-consonant strings of 3+ letters: the engine // has no multi-letter jianpin, so correction is the only useful reading of e.g. // "bqng" -> bang. Inputs with manual delimiters return false; the correction path // excludes them on its own. +// +// Who enforces it: the CALLERS, not the search above. autocorrect_cut, +// autocorrect_cut_kbest and autocorrect_cut_detail never consult this predicate, so +// calling them directly on a jianpin shape still cuts one -- "zheg" reads as "zhei" +// through the generated space, "zher" as "zhe" through the insertion table (both +// pinned by engine/tests/src/test_pinyin.cpp). Both production callers apply the +// guard before reaching the search: quanpin_dictionary.cpp (query layer) and +// input_session_composition.cpp (preedit layer). A new caller has to do the same; +// none of these signatures carries the constraint, so omitting it fails silently +// rather than at compile time. bool looks_like_syllable_with_jianpin_tail(const std::string &pinyin); } // namespace quanpin diff --git a/engine/tests/src/test_pinyin.cpp b/engine/tests/src/test_pinyin.cpp index 9aa5aea2..4580c72b 100644 --- a/engine/tests/src/test_pinyin.cpp +++ b/engine/tests/src/test_pinyin.cpp @@ -947,12 +947,17 @@ void test_quanpin_autocorrect_switches_and_guard() const unsigned both = transposition_only | neighbor_only; // Cut-level switch matrix: 'sahng' is a transposition fix, 'shabg' a neighbor - // fix, and each type bit must enable exactly its own family (AC1-AC4). + // fix (AC1-AC4). Since the generated correction space was added, the neighbour + // bit also carries out-of-table substitutions, so a bit no longer maps onto + // exactly one table and the 'must NOT correct' rows had to give up their empty + // expectation for the input's widened reading. The masks involved are + // unreachable in the product: engine.cpp always pairs the deletion and + // insertion bits with the legacy switches, so only 0xd/0xe/0xf ever occur. expect(quanpin::autocorrect_cut("sahng", none).empty(), "Both switches off must disable the correction cut."); expect(quanpin::join_segments(quanpin::autocorrect_cut("sahng", transposition_only)) == "shang", "Transposition-only must correct 'sahng'."); - expect(quanpin::autocorrect_cut("sahng", neighbor_only).empty(), - "Neighbor-only must not correct the transposition case 'sahng'."); + expect(quanpin::join_segments(quanpin::autocorrect_cut("sahng", neighbor_only)) == "sa'ang", + "Neighbor-only reads 'sahng' as sa + ang through a generated substitution."); expect(quanpin::join_segments(quanpin::autocorrect_cut("sahng", both)) == "shang", "Both switches on must correct 'sahng'."); @@ -990,19 +995,26 @@ void test_quanpin_autocorrect_switches_and_guard() "'gau' must correct to 'gua' via transposition once the guard lets it through."); expect(quanpin::join_segments(quanpin::autocorrect_cut("bqng", neighbor_only)) == "bang", "'bqng' -> bang must remain a valid neighbor correction."); - expect(quanpin::autocorrect_cut("zheg", both).empty(), - "The cleaned table must offer no correction path for 'zheg' (2-letter keys are gone)."); + // "zheg" is jianpin intent (zhe + g). The static tables still offer no path + // for it -- 2-letter keys are gone -- but the generated space substitutes the + // trailing letter ("zheg" -> "zhei", weight 15), so the cut is no longer empty. + // Production is unchanged: with the deletion bit on (0xd/0xe/0xf) the static + // reading "zheng" costs 11 and wins, and this search consults no guard by + // design -- the callers screen jianpin intent first. + expect(quanpin::join_segments(quanpin::autocorrect_cut("zheg", both)) == "zhei", + "The cleaned static table has no path for 'zheg', but a generated one exists."); - // Deletion bit (phase 2): "shng" (dropped "a") is a deletion fix; only - // that bit may correct it, and the legacy switches keep their own families. + // Deletion bit (phase 2): "shng" (dropped "a") is a deletion fix, and that + // bit still corrects it directly. The legacy switches keep their own + // families apart from where the generated space widened the neighbour bit. const unsigned deletion_only = quanpin::kAutocorrectDeletion; const unsigned all = transposition_only | neighbor_only | deletion_only; expect(quanpin::join_segments(quanpin::autocorrect_cut("shng", deletion_only)) == "shang", "Deletion-only must correct 'shng'."); expect(quanpin::autocorrect_cut("shng", transposition_only).empty(), "Transposition-only must not correct the deletion case 'shng'."); - expect(quanpin::autocorrect_cut("shng", neighbor_only).empty(), - "Neighbor-only must not correct the deletion case 'shng'."); + expect(quanpin::join_segments(quanpin::autocorrect_cut("shng", neighbor_only)) == "sang", + "Neighbor-only reads 'shng' as sang through a generated substitution."); expect(quanpin::autocorrect_cut("sahng", deletion_only).empty() == false && quanpin::join_segments(quanpin::autocorrect_cut("sahng", deletion_only)) == "sa'hang", "Bits gate tables, not intents: deletion-only still explains 'sahng' via sa + hng -> hang."); @@ -1010,8 +1022,8 @@ void test_quanpin_autocorrect_switches_and_guard() "Deletion-only must not correct the neighbor case 'shabg'."); expect(quanpin::join_segments(quanpin::autocorrect_cut("shngzhk", all)) == "shang'zhi", "A deletion edge followed by a neighbor edge must cut 'shngzhk'."); - expect(quanpin::autocorrect_cut("shngzhk", both).empty(), - "Without the deletion bit 'shngzhk' must stay unexplained."); + expect(quanpin::join_segments(quanpin::autocorrect_cut("shngzhk", both)) == "sang'zhi", + "Without the deletion bit 'shngzhk' still cuts: generated sang + static zhk -> zhi."); // Insertion bit (fourth type): "shangg" (doubled g) and "sjhang" (j is a // QWERTY neighbor of h) are insertion fixes; only that bit may correct them. @@ -1124,9 +1136,16 @@ void test_quanpin_autocorrect_switches_and_guard() expect(!transposed.empty() && transposed.front().word == "上", "Transposition-only must correct 'sahng' at the dictionary layer (AC2)."); + // AC2 gating is verified through the correction marker rather than through + // 上's presence: a sequence-prefix query hits the 'shang' key through 'sha' + // whether or not a correction happened (see the AC3 case below for the same + // effect). Now that the generated space routes neighbor-only 'sahng' through + // the correction path (as sa + ang), 上 comes back with an empty + // corrected_from -- the transposition reading is still not produced. const auto neighbor_denied = dictionary.query("sahng", "sa'h'n'g", neighbor_only); - expect(std::none_of(neighbor_denied.begin(), neighbor_denied.end(), is_shang), - "Neighbor-only must not correct the transposition case 'sahng' (AC2)."); + expect(std::none_of(neighbor_denied.begin(), neighbor_denied.end(), + [](const WordItem &item) { return item.corrected_from == "sahng"; }), + "Neighbor-only must not produce a corrected candidate for the transposition case 'sahng' (AC2)."); const auto neighbor_corrected = dictionary.query("shabg", "sha'b'g", neighbor_only); expect(!neighbor_corrected.empty() && neighbor_corrected.front().word == "上", @@ -1302,10 +1321,17 @@ void test_quanpin_autocorrect_display() sahnghao_cut.segments[1].start == 5 && !sahnghao_cut.segments[1].corrected, "The untouched tail of 'sahnghao' must keep its raw span."); - expect(quanpin::autocorrect_cut_detail("zheg", both).empty(), - "The cleaned table must leave the jianpin shape 'zheg' unexplained by the BFS."); - expect(quanpin::autocorrect_cut_detail("keneng", both).empty(), - "A fully legal spelling must produce no correction cut."); + const auto zheg_cut = quanpin::autocorrect_cut_detail("zheg", both); + expect(zheg_cut.segments.size() == 1 && zheg_cut.segments[0].syllable == "zhei", + "The detail cut reads the jianpin shape 'zheg' as 'zhei' via a generated substitution."); + // "keneng" is ken + eng, both legal syllables. It used to yield no cut, but + // not because a guard stopped it -- this search consults no guard, as the + // "zher" case above documents -- rather because the static tables happened to + // lack the "eng" -> "ang" pair. The generated space supplies it, so the input + // now cuts. Jianpin intent stays protected by the callers, which apply + // looks_like_syllable_with_jianpin_tail before reaching this search. + expect(!quanpin::autocorrect_cut_detail("keneng", both).empty(), + "A fully legal spelling now cuts too, via a generated 'eng' -> 'ang' substitution."); expect(quanpin::autocorrect_cut_detail("sahng", none).empty(), "Both switches off must disable the range-carrying cut too."); expect(quanpin::autocorrect_cut_detail("xi'an", both).empty(), diff --git a/server/tests/src/test_quanpin_scheme.cpp b/server/tests/src/test_quanpin_scheme.cpp index 1b6499cf..2743b73a 100644 --- a/server/tests/src/test_quanpin_scheme.cpp +++ b/server/tests/src/test_quanpin_scheme.cpp @@ -676,10 +676,19 @@ TEST_CASE(QuanpinAutocorrectCutGatesEachTypeIndependently) const unsigned neighbor_only = quanpin::kAutocorrectNeighbor; const unsigned both = transposition_only | neighbor_only; + // The generated correction space attaches its out-of-table substitutions to + // kAutocorrectNeighbor, so that bit no longer maps one-to-one onto the static + // neighbour table: swapping "sang"'s leading vowel for "h" is not a QWERTY + // neighbour pair (kQwertyNeighbors['a'] == "qwsz"), and the generated space + // explains "sahng" as "sa'ang" under the neighbour bit alone. The cases below + // used to pin that one-to-one relation and are kept, with the widened reading + // spelled out. The mask is unreachable in the product: engine.cpp assembles the + // deletion and insertion bits alongside the legacy switches, so the only + // non-zero masks are 0xd/0xe/0xf -- the widening never reaches a user. // "sahng" is a transposition fix; only that bit may correct it. REQUIRE(quanpin::autocorrect_cut("sahng", none).empty()); REQUIRE_EQ(quanpin::join_segments(quanpin::autocorrect_cut("sahng", transposition_only)), std::string("shang")); - REQUIRE(quanpin::autocorrect_cut("sahng", neighbor_only).empty()); + REQUIRE_EQ(quanpin::join_segments(quanpin::autocorrect_cut("sahng", neighbor_only)), std::string("sa'ang")); // "shabg" is a neighbor fix; only that bit may correct it. REQUIRE(quanpin::autocorrect_cut("shabg", none).empty()); REQUIRE(quanpin::autocorrect_cut("shabg", transposition_only).empty()); @@ -749,7 +758,17 @@ TEST_CASE(QuanpinAutocorrectCutRejectsInputsOutOfScope) // Legal input: the caller must only invoke autocorrect when the correction // cut already failed, so a fully legal spelling yields no correction. REQUIRE(quanpin::autocorrect_cut("shang", both).empty()); - REQUIRE(quanpin::autocorrect_cut("keneng", both).empty()); + // "keneng" is a jianpin intent shape: ken + eng with zero corrected letters, + // and looks_like_syllable_with_jianpin_tail agrees (pinned by + // QuanpinJianpinShapeGuardBlocksCorrection below). That guard is the callers' + // responsibility (quanpin_dictionary.cpp, input_session_composition.cpp) and + // autocorrect_cut deliberately does not consult it -- test_pinyin.cpp's "zher" + // case documents that same contract, where the correction cut exists by design. + // What changed here is coverage, not the guard: the generated space supplies + // "eng" -> "ang" as an out-of-table substitution, so the search now explains the + // input where the static tables happened not to. Unreachable in production for + // the same mask reason as QuanpinAutocorrectCutGatesEachTypeIndependently. + REQUIRE_EQ(quanpin::join_segments(quanpin::autocorrect_cut("keneng", both)), std::string("ken'ang")); // Manual delimiter: user-intended boundary, never rewritten. REQUIRE(quanpin::autocorrect_cut("xi'an", both).empty()); // No correction path: unresolvable garbage. @@ -851,8 +870,11 @@ TEST_CASE(QuanpinAutocorrectCutMixedDeletionAndNeighbor) // "shngzhk" mixes a deletion edge (shng -> shang) with a neighbor edge // (zhk -> zhi): two corrected edges, weights 11 + 13. REQUIRE_EQ(quanpin::join_segments(quanpin::autocorrect_cut("shngzhk", all)), std::string("shang'zhi")); - // Without the deletion bit the same input stays unexplained. - REQUIRE(quanpin::autocorrect_cut("shngzhk", both).empty()); + // Without the deletion bit the generated space still explains it: "shng" + // becomes "sang" through an out-of-table substitution on the leading vowel, + // then the static neighbour edge zhk -> zhi carries the rest (15 + 13 = 28). + // The deletion-bit reading above still wins because it costs 11 + 13 = 24. + REQUIRE_EQ(quanpin::join_segments(quanpin::autocorrect_cut("shngzhk", both)), std::string("sang'zhi")); // A two-letter jianpin leftover ("zh") stays out of correction scope even // with deletion on: the tables hold no 2-letter keys. @@ -867,10 +889,13 @@ TEST_CASE(QuanpinAutocorrectCutGatesDeletionIndependently) const unsigned deletion_only = quanpin::kAutocorrectDeletion; const unsigned all = transposition_only | neighbor_only | deletion_only; - // "shng" is a deletion fix; only that bit may correct it. + // "shng" is a deletion fix; only that bit may correct it directly. REQUIRE(quanpin::autocorrect_cut("shng", none).empty()); REQUIRE(quanpin::autocorrect_cut("shng", transposition_only).empty()); - REQUIRE(quanpin::autocorrect_cut("shng", neighbor_only).empty()); + // ...except that the generated space also reads "shng" as "sang" (out-of-table + // substitution on the leading vowel, weight 15) under the neighbour bit. + // The direct deletion reading costs 11 and still wins wherever both bits are on. + REQUIRE_EQ(quanpin::join_segments(quanpin::autocorrect_cut("shng", neighbor_only)), std::string("sang")); REQUIRE_EQ(quanpin::join_segments(quanpin::autocorrect_cut("shng", deletion_only)), std::string("shang")); REQUIRE_EQ(quanpin::join_segments(quanpin::autocorrect_cut("shng", all)), std::string("shang")); From 7e1ae10da2849bb9a8e5646616790c546c272932 Mon Sep 17 00:00:00 2001 From: Jia Sui Date: Sun, 4 Oct 2026 05:21:45 +0800 Subject: [PATCH 10/12] =?UTF-8?q?docs(engine):=20=E6=9B=B4=E6=AD=A3?= =?UTF-8?q?=E7=94=9F=E6=88=90=E5=BC=8F=E7=A9=BA=E9=97=B4=E5=8F=AF=E8=BE=BE?= =?UTF-8?q?=E6=80=A7=E8=AE=BA=E6=96=AD=EF=BC=8C=E5=88=A0=E9=99=A4=E5=AF=B9?= =?UTF-8?q?=E7=94=A8=E6=88=B7=E4=B8=8D=E5=8F=AF=E8=A7=81=E7=9A=84=E9=94=99?= =?UTF-8?q?=E8=AF=AF=E7=BB=93=E8=AE=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- engine/quanpin/quanpin_utils.h | 15 +++++++++++---- engine/tests/src/test_pinyin.cpp | 24 ++++++++++++++++-------- server/tests/src/test_quanpin_scheme.cpp | 8 +++++--- 3 files changed, 32 insertions(+), 15 deletions(-) diff --git a/engine/quanpin/quanpin_utils.h b/engine/quanpin/quanpin_utils.h index 49f69275..54461291 100644 --- a/engine/quanpin/quanpin_utils.h +++ b/engine/quanpin/quanpin_utils.h @@ -153,11 +153,18 @@ Segments autocorrect_cut(const std::string &pinyin, unsigned autocorrect_types); // kAutocorrectInsertion, so a type bit no longer maps one-to-one onto a table. An // input with no static cut at all is now reachable through a single bit -- "sahng" // reads as "sa'ang" under the neighbour bit alone, "shng" as "sang". Callers must -// not read a bit as "only this family of typo". Product behaviour is unaffected +// not read a bit as "only this family of typo". +// +// Scope of that: the BARE-bit readings above are unreachable in the product, // because engine.cpp always pairs the deletion and insertion bits with the legacy -// switches, so the only masks that ever occur are 0 / 0xd / 0xe / 0xf; the -// head+jianpin-tail path in quanpin_dictionary.cpp additionally masks the -// neighbour bit off, which suppresses every generated substitution for free. +// switches, so the only masks that ever occur are 0 / 0xd / 0xe / 0xf. The +// generated space itself is NOT unreachable: 0xe and 0xf still carry the +// neighbour bit and all three carry the insertion bit, so it does add +// user-visible recall (far substitutions under 0xe/0xf, far insertions under +// 0xd). That recall is the intended gain, pinned end-to-end by +// engine/tests/src/test_input_session.cpp ("shatg" -> shang, "zthou" -> zhou). +// The head+jianpin-tail fallback in quanpin_dictionary.cpp additionally masks the +// neighbour bit off, which suppresses generated substitutions on that path only. // Every returned cut contains at least one corrected edge, so a fully legal input // with no correction reading yields an empty vector (the caller owns the plain // segmentation). This is the query-time disambiguation surface of CN 101133411 diff --git a/engine/tests/src/test_pinyin.cpp b/engine/tests/src/test_pinyin.cpp index 4580c72b..b9e27b4e 100644 --- a/engine/tests/src/test_pinyin.cpp +++ b/engine/tests/src/test_pinyin.cpp @@ -950,9 +950,14 @@ void test_quanpin_autocorrect_switches_and_guard() // fix (AC1-AC4). Since the generated correction space was added, the neighbour // bit also carries out-of-table substitutions, so a bit no longer maps onto // exactly one table and the 'must NOT correct' rows had to give up their empty - // expectation for the input's widened reading. The masks involved are - // unreachable in the product: engine.cpp always pairs the deletion and - // insertion bits with the legacy switches, so only 0xd/0xe/0xf ever occur. + // expectation for the input's widened reading. The bare bits used here + // (neighbour alone, T|N) are unreachable in the product: engine.cpp always + // pairs the deletion and insertion bits with the legacy switches, so only + // 0xd/0xe/0xf ever occur. That does NOT make the generated space unreachable + // -- 0xe/0xf still carry the neighbour bit and all three carry the insertion + // bit -- it only means these particular one-bit rows cannot be triggered by a + // user. The space itself is reachable and is covered by the session-level + // cases below plus test_input_session.cpp. expect(quanpin::autocorrect_cut("sahng", none).empty(), "Both switches off must disable the correction cut."); expect(quanpin::join_segments(quanpin::autocorrect_cut("sahng", transposition_only)) == "shang", "Transposition-only must correct 'sahng'."); @@ -1137,11 +1142,14 @@ void test_quanpin_autocorrect_switches_and_guard() "Transposition-only must correct 'sahng' at the dictionary layer (AC2)."); // AC2 gating is verified through the correction marker rather than through - // 上's presence: a sequence-prefix query hits the 'shang' key through 'sha' - // whether or not a correction happened (see the AC3 case below for the same - // effect). Now that the generated space routes neighbor-only 'sahng' through - // the correction path (as sa + ang), 上 comes back with an empty - // corrected_from -- the transposition reading is still not produced. + // 上's presence. Under neighbor_only the generated space now explains + // 'sahng' as sa + ang, so the correction path runs; 上 still comes back, but + // via the correction-mode segmentation 'shang' (cut_pinyin_by_mode lists it + // as an alternative), which hits the 'shang' key exactly and therefore + // carries no corrected_from. The transposition reading is not produced, so + // nothing is marked. Note this is a different route than the AC3 case below, + // where 'shabg' only reaches the key by a 'sha' PREFIX scan -- same symptom + // (an unmarked 上), different mechanism. const auto neighbor_denied = dictionary.query("sahng", "sa'h'n'g", neighbor_only); expect(std::none_of(neighbor_denied.begin(), neighbor_denied.end(), [](const WordItem &item) { return item.corrected_from == "sahng"; }), diff --git a/server/tests/src/test_quanpin_scheme.cpp b/server/tests/src/test_quanpin_scheme.cpp index 2743b73a..59a8bb18 100644 --- a/server/tests/src/test_quanpin_scheme.cpp +++ b/server/tests/src/test_quanpin_scheme.cpp @@ -682,9 +682,11 @@ TEST_CASE(QuanpinAutocorrectCutGatesEachTypeIndependently) // neighbour pair (kQwertyNeighbors['a'] == "qwsz"), and the generated space // explains "sahng" as "sa'ang" under the neighbour bit alone. The cases below // used to pin that one-to-one relation and are kept, with the widened reading - // spelled out. The mask is unreachable in the product: engine.cpp assembles the - // deletion and insertion bits alongside the legacy switches, so the only - // non-zero masks are 0xd/0xe/0xf -- the widening never reaches a user. + // spelled out. The bare bit used here is unreachable in the product: engine.cpp + // assembles the deletion and insertion bits alongside the legacy switches, so + // the only non-zero masks are 0xd/0xe/0xf. That makes THIS row untriggerable by + // a user; it does not make the generated space unreachable, because 0xe/0xf + // still carry the neighbour bit and all three carry the insertion bit. // "sahng" is a transposition fix; only that bit may correct it. REQUIRE(quanpin::autocorrect_cut("sahng", none).empty()); REQUIRE_EQ(quanpin::join_segments(quanpin::autocorrect_cut("sahng", transposition_only)), std::string("shang")); From 0952333f94ef1d54ba0eb00097c0fae480d0fd66 Mon Sep 17 00:00:00 2001 From: Jia Sui Date: Sun, 4 Oct 2026 08:16:18 +0800 Subject: [PATCH 11/12] =?UTF-8?q?test(server):=20=E6=9B=B4=E6=AD=A3?= =?UTF-8?q?=E4=B8=89=E5=A4=84=E4=B8=8E=E7=94=9F=E6=88=90=E5=BC=8F=E7=A9=BA?= =?UTF-8?q?=E9=97=B4=E4=B8=8D=E7=AC=A6=E7=9A=84=E7=94=A8=E4=BE=8B=E5=90=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/tests/src/test_quanpin_scheme.cpp | 35 ++++++++++++++---------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/server/tests/src/test_quanpin_scheme.cpp b/server/tests/src/test_quanpin_scheme.cpp index 59a8bb18..a545f22d 100644 --- a/server/tests/src/test_quanpin_scheme.cpp +++ b/server/tests/src/test_quanpin_scheme.cpp @@ -669,7 +669,7 @@ TEST_CASE(QuanpinAutocorrectInsertionTableEntriesAreLegal) } } -TEST_CASE(QuanpinAutocorrectCutGatesEachTypeIndependently) +TEST_CASE(QuanpinAutocorrectCutTypeBitSwitchMatrix) { const unsigned none = 0; const unsigned transposition_only = quanpin::kAutocorrectTransposition; @@ -722,6 +722,24 @@ TEST_CASE(QuanpinJianpinShapeGuardBlocksCorrection) REQUIRE(!quanpin::looks_like_syllable_with_jianpin_tail("wj")); } +// The counterpart of QuanpinJianpinShapeGuardBlocksCorrection: the shape predicate +// says "jianpin intent", and the correction search still cuts it anyway. Both are +// correct -- the guard is applied by the callers, not here. +TEST_CASE(QuanpinAutocorrectCutExplainsJianpinShapeByGeneratedSubstitution) +{ + const unsigned both = quanpin::kAutocorrectTransposition | quanpin::kAutocorrectNeighbor; + // "keneng" is ken + eng with zero corrected letters. That guard is the CALLERS' + // responsibility (quanpin_dictionary.cpp, input_session_composition.cpp) and + // autocorrect_cut deliberately does not consult it -- test_pinyin.cpp's "zher" + // case documents that same contract, where the correction cut exists by design. + // What the generated space changed is coverage, not the guard: it supplies + // "eng" -> "ang" as an out-of-table substitution, so the search now explains an + // input the static tables happened not to. develop returned empty here only + // because no static deletion pair happened to exist, not because a guard fired. + // The mask is unreachable in production: see QuanpinAutocorrectCutTypeBitSwitchMatrix. + REQUIRE_EQ(quanpin::join_segments(quanpin::autocorrect_cut("keneng", both)), std::string("ken'ang")); +} + TEST_CASE(QuanpinAutocorrectCutSingleSyllableTranspositions) { const unsigned both = quanpin::kAutocorrectTransposition | quanpin::kAutocorrectNeighbor; @@ -752,7 +770,7 @@ TEST_CASE(QuanpinAutocorrectCutPrefersFewestCorrections) REQUIRE_EQ(quanpin::join_segments(quanpin::autocorrect_cut("sahnguai", both)), std::string("shan'guai")); } -TEST_CASE(QuanpinAutocorrectCutRejectsInputsOutOfScope) +TEST_CASE(QuanpinAutocorrectCutRejectsInputsWithNoCorrectionPath) { const unsigned both = quanpin::kAutocorrectTransposition | quanpin::kAutocorrectNeighbor; // The mask short-circuits before any table lookup. @@ -760,17 +778,6 @@ TEST_CASE(QuanpinAutocorrectCutRejectsInputsOutOfScope) // Legal input: the caller must only invoke autocorrect when the correction // cut already failed, so a fully legal spelling yields no correction. REQUIRE(quanpin::autocorrect_cut("shang", both).empty()); - // "keneng" is a jianpin intent shape: ken + eng with zero corrected letters, - // and looks_like_syllable_with_jianpin_tail agrees (pinned by - // QuanpinJianpinShapeGuardBlocksCorrection below). That guard is the callers' - // responsibility (quanpin_dictionary.cpp, input_session_composition.cpp) and - // autocorrect_cut deliberately does not consult it -- test_pinyin.cpp's "zher" - // case documents that same contract, where the correction cut exists by design. - // What changed here is coverage, not the guard: the generated space supplies - // "eng" -> "ang" as an out-of-table substitution, so the search now explains the - // input where the static tables happened not to. Unreachable in production for - // the same mask reason as QuanpinAutocorrectCutGatesEachTypeIndependently. - REQUIRE_EQ(quanpin::join_segments(quanpin::autocorrect_cut("keneng", both)), std::string("ken'ang")); // Manual delimiter: user-intended boundary, never rewritten. REQUIRE(quanpin::autocorrect_cut("xi'an", both).empty()); // No correction path: unresolvable garbage. @@ -883,7 +890,7 @@ TEST_CASE(QuanpinAutocorrectCutMixedDeletionAndNeighbor) REQUIRE(quanpin::autocorrect_cut("sahngzh", all).empty()); } -TEST_CASE(QuanpinAutocorrectCutGatesDeletionIndependently) +TEST_CASE(QuanpinAutocorrectCutDeletionBitSwitchMatrix) { const unsigned none = 0; const unsigned transposition_only = quanpin::kAutocorrectTransposition; From a219e499f6dd4686515c7303b068d70d858f4a27 Mon Sep 17 00:00:00 2001 From: fanlusky <1730976608@qq.com> Date: Sun, 4 Oct 2026 14:35:50 +0800 Subject: [PATCH 12/12] =?UTF-8?q?fix(engine):=20=E5=85=A8=E6=8B=BC?= =?UTF-8?q?=E7=BA=A0=E9=94=99=E4=B8=8A=E4=B8=8B=E6=96=87=E6=8E=A5=E7=AE=A1?= =?UTF-8?q?=E8=AE=A9=E4=BD=8D=E4=BA=8E=E7=94=A8=E6=88=B7=E5=9C=A8=E4=B8=BB?= =?UTF-8?q?=E5=88=87=E8=AF=BB=E9=9F=B3=E4=B8=8A=E7=9A=84=E9=80=89=E6=8B=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 加载 sc.lm 时词格路径分里的词库权重只剩 dictionary_tiebreak（≤0.07）和 reading_prior，调频改的权重翻不过 1.0 的接管边际：banzhng 的用户每次都选 班长，办证系照样每次领衔，调频在这组词上被架空。 接管前加一道判据：主切键在用户日志里有 upsert，且主切整键首位权重不低于 胜出切分首位（即用户已把主切读音选到前面），就不接管、走按权重排序的静态 合并。出货词频本身仍交给上下文裁决；日志只在边际达标后查一次。 测试用手写 ARPA 作 sc.lm，让 LM 稳定偏向办证，覆盖无用户记录照常接管、 用户选过班长不再接管、用户记录未越过胜者照常接管三种情形。 --- engine/quanpin/quanpin_dictionary.cpp | 18 +++- engine/tests/src/test_input_session.cpp | 83 +++++++++++++++++++ .../user_dictionary_journal.cpp | 11 +++ .../user_dictionary/user_dictionary_journal.h | 3 + 4 files changed, 114 insertions(+), 1 deletion(-) diff --git a/engine/quanpin/quanpin_dictionary.cpp b/engine/quanpin/quanpin_dictionary.cpp index 23fc234b..d251f597 100644 --- a/engine/quanpin/quanpin_dictionary.cpp +++ b/engine/quanpin/quanpin_dictionary.cpp @@ -483,6 +483,7 @@ std::vector QuanpinDictionary::query_exact(const std::string &raw_inpu [](const auto &lhs, const auto &rhs) { return lhs.first > rhs.first; }); // 第一遍只解路径分选胜者，只为胜出切分付完整管线的钱。 std::optional winner_score; + std::int64_t winner_top_weight = 0; quanpin::Segments winner_segments; for (size_t i = 0; i < decode_pool.size() && i < kAutocorrectContextDecodeLimit; ++i) { @@ -491,10 +492,25 @@ std::vector QuanpinDictionary::query_exact(const std::string &raw_inpu cut_score.has_value() && (!winner_score.has_value() || *cut_score > *winner_score)) { winner_score = cut_score; + winner_top_weight = decode_pool[i].first; winner_segments = cut; } } - if (winner_score.has_value() && *winner_score - *primary_score >= kAutocorrectContextMarginLog) + // 用户在主切读音上的选择优先于上下文。调频只改词库权重，而加载 LM 时 + // 路径分里的权重只剩 dictionary_tiebreak（≤0.07）和 reading_prior，翻不过 + // 1.0 的边际：不拦的话，用户每次都选班长，办证系照样每次领衔。调频把选中词 + // 抬到它越过的词之上，所以「主切整键首位权重不低于胜出切分首位」正是用户已经 + // 把主切读音选到前面的样子；再要求主切键有用户 upsert，把这条限制在用户 + // 表达过偏好的键上，出货词频本身仍交给上下文裁决。不接管时走下面的静态 + // 合并，它按权重排序，调频照常生效。只在边际达标后才查日志，不给每次按键加查询。 + const auto user_prefers_primary = [&] { + return primary_top_weight >= winner_top_weight && + user_dictionary::has_user_upsert_for_key( + metasequoia::path_to_utf8(paths_.user(metasequoia::assets::user_journal)), + user_dictionary::DictionaryKind::Pinyin, primary_key); + }; + if (winner_score.has_value() && *winner_score - *primary_score >= kAutocorrectContextMarginLog && + !user_prefers_primary()) { // 边际达标：胜出切分领衔。merge_alternative_segmentations 假定传入 // result 的头部是主切词库行，所以这里仍以主切为锚组装其余列表（从 diff --git a/engine/tests/src/test_input_session.cpp b/engine/tests/src/test_input_session.cpp index 4641c885..b2d6d25b 100644 --- a/engine/tests/src/test_input_session.cpp +++ b/engine/tests/src/test_input_session.cpp @@ -490,6 +490,88 @@ void run_autocorrect_context_layering_tests(const std::filesystem::path &data_di } } +// 用户在主切读音上的选择优先于上下文接管。加载 LM 时路径分只剩 dictionary_tiebreak +// 一点点词库权重，调频翻不过接管边际，所以前两组 fixture 的启发式打分测不出来：这里 +// 放一份手写 ARPA 当 sc.lm（kenlm 按文本格式载入），让 LM 稳定偏向办证（log10 差 2.5）， +// 同时把班长的词库权重设成调频后的样子——越过办证。 +void run_autocorrect_context_user_choice_tests(const std::filesystem::path &data_directory) +{ + const auto make_fixture = [&data_directory](const char *name, int banzhang_weight) { + const std::filesystem::path directory = data_directory / name; + std::filesystem::create_directories(directory); + { + Database database(directory / "msime.db"); + database.execute("CREATE TABLE tbl_2_b(key TEXT,jp TEXT,value TEXT,weight INTEGER);" + "INSERT INTO tbl_2_b VALUES('ban''zheng', 'bz', '办证', 100000);" + "INSERT INTO tbl_2_b VALUES('ban''zheng', 'bz', '辩证', 10);" + "INSERT INTO tbl_2_b VALUES('ban''zhang', 'bz', '搬账', 4000);"); + const std::string banzhang = + "INSERT INTO tbl_2_b VALUES('ban''zhang', 'bz', '班长', " + std::to_string(banzhang_weight) + ");"; + database.execute(banzhang.c_str()); + } + // 二进制写：文本模式在 Windows 上会把换行写成 \r\n，kenlm 的 ARPA 解析不认。 + std::ofstream arpa(directory / metasequoia::assets::language_model, std::ios::binary); + arpa << "\\data\\\n" + "ngram 1=7\n" + "ngram 2=1\n" + "\n" + "\\1-grams:\n" + "-1.0\t\t0\n" + "-99\t\t0\n" + "-1.0\t\t0\n" + "-1.0\t办证\t0\n" + "-3.5\t班长\t0\n" + "-3.5\t搬账\t0\n" + "-4.0\t辩证\t0\n" + "\n" + "\\2-grams:\n" + "-0.5\t 办证\n" + "\n" + "\\end\\\n"; + require(static_cast(arpa), "Failed to write the user-choice language model fixture."); + metasequoia::RuntimePaths paths; + paths.resources = directory; + paths.user_data = directory; + paths.cache = directory; + paths.dictionaries = directory; + return paths; + }; + const unsigned both = quanpin::kAutocorrectTransposition | quanpin::kAutocorrectNeighbor; + SentenceAssociationOptions lattice_only; + lattice_only.word_lattice = true; + const auto first_word = [both, &lattice_only](const metasequoia::RuntimePaths &paths) { + metasequoia::InputSession session(SchemeType::Quanpin, both, true, true, true, paths); + session.set_sentence_association(lattice_only); + type(session, "banzhng"); + require(!session.candidates().empty(), "The user-choice fixture produced no candidates."); + return session.candidates().front().word; + }; + + { + const auto paths = make_fixture("autocorrect-context-user-choice", 150000); + // 没有用户记录：班长权重再高也只是出货词频，仍交给上下文裁决。这一步同时确认 + // ARPA 真的载入了——启发式打分下班长权重更高，不会是办证领衔。 + require(first_word(paths) == "办证", + "Without a user record the language model must still let the context reading lead."); + + require(user_dictionary::record_upsert(metasequoia::path_to_utf8(paths.user(metasequoia::assets::user_journal)), + user_dictionary::DictionaryKind::Pinyin, "ban'zhang", "班长", 150000), + "Failed to write the user-choice journal record."); + require(first_word(paths) == "班长", + "A word the user ranked above the context winner must not be displaced by the context takeover."); + } + + { + // 用户动过主切的键，但主切首位仍低于胜出切分首位：用户并没有把这组读音选到前面，照常接管。 + const auto paths = make_fixture("autocorrect-context-user-touched", 5000); + require(user_dictionary::record_upsert(metasequoia::path_to_utf8(paths.user(metasequoia::assets::user_journal)), + user_dictionary::DictionaryKind::Pinyin, "ban'zhang", "搬账", 4000), + "Failed to write the user-touched journal record."); + require(first_word(paths) == "办证", + "A user record that does not outrank the context winner must not block the takeover."); + } +} + // 阶段 2 生成式纠错空间（任务 quanpin-autocorrect-generated-space）：静态表 // 形状之外的单编辑手误由生成式索引兜底，权重落贵档（15）。隔离 fixture： // - shatg = shang 的 n→t（t 非邻键，远键替换） @@ -1487,6 +1569,7 @@ int run_test() run_caret_prefix_session_tests(data_directory); run_autocorrect_context_ranking_tests(data_directory); run_autocorrect_context_layering_tests(data_directory); + run_autocorrect_context_user_choice_tests(data_directory); run_autocorrect_generated_space_tests(data_directory); #endif diff --git a/engine/user_dictionary/user_dictionary_journal.cpp b/engine/user_dictionary/user_dictionary_journal.cpp index afd00f1c..f8218b02 100644 --- a/engine/user_dictionary/user_dictionary_journal.cpp +++ b/engine/user_dictionary/user_dictionary_journal.cpp @@ -581,6 +581,17 @@ bool is_user_inserted(const std::string &user_db_path, DictionaryKind kind, cons sqlite3_column_int(stmt.get(), 0) != 0; } +bool has_user_upsert_for_key(const std::string &user_db_path, DictionaryKind kind, const std::string &key) +{ + UserDatabase db(user_db_path); + if (!db) + return false; + auto stmt = prepare(db.get(), "SELECT 1 FROM user_dictionary_operations " + "WHERE dictionary=?1 AND key=?2 AND operation='upsert' LIMIT 1"); + return stmt && bind_text(stmt.get(), 1, kind_name(kind)) && bind_text(stmt.get(), 2, key) && + sqlite3_step(stmt.get()) == SQLITE_ROW; +} + bool record_pinyin_upsert_from_database(const std::string &main_db_path, const std::string &key, const std::string &value, const std::string &user_db_path) { diff --git a/engine/user_dictionary/user_dictionary_journal.h b/engine/user_dictionary/user_dictionary_journal.h index 1f641e85..24178289 100644 --- a/engine/user_dictionary/user_dictionary_journal.h +++ b/engine/user_dictionary/user_dictionary_journal.h @@ -28,6 +28,9 @@ bool record_delete(const std::string &user_db_path, DictionaryKind kind, const s const std::string &value); bool is_user_inserted(const std::string &user_db_path, DictionaryKind kind, const std::string &key, const std::string &value); +// 该键下是否有用户留下的权重（调频或造词的 upsert，不看具体词）。删除记录不算： +// 它表达的是「不要这个词」，不是对这组读音的偏好。 +bool has_user_upsert_for_key(const std::string &user_db_path, DictionaryKind kind, const std::string &key); bool ensure_user_database(const std::string &user_db_path); // Switch an existing dictionary database (msime.db) to WAL so the key thread's // queries never wait for a learning write's commit. The user journal does this