diff --git a/engine/quanpin/quanpin_dictionary.cpp b/engine/quanpin/quanpin_dictionary.cpp index 8718bac6..5201139f 100644 --- a/engine/quanpin/quanpin_dictionary.cpp +++ b/engine/quanpin/quanpin_dictionary.cpp @@ -176,6 +176,10 @@ SeriesQueryResolution resolve_series_query(const std::string &raw_input, const q } }; + // 静态优先语义由 k-best 搜索自身保证（见 quanpin_utils.h 契约）：最优 + // 切分为纯静态时生成切分已在搜索内被丢弃，下面的 head+简拼尾组合路径 + // 与 k=1 预编辑投影等调用方同享该保证；生成读法只在静态空间完全无法 + // 解释输入时出场。混表竞争的加权仲裁依赖训练权重，留给方向阶段 3。 const auto cuts = quanpin::autocorrect_cut_kbest(raw_input, autocorrect_types, kAutocorrectCutKBest); if (!cuts.empty()) { diff --git a/engine/quanpin/quanpin_utils.cpp b/engine/quanpin/quanpin_utils.cpp index ae147151..a80f1e1b 100644 --- a/engine/quanpin/quanpin_utils.cpp +++ b/engine/quanpin/quanpin_utils.cpp @@ -3,6 +3,7 @@ #include "autocorrect_table.h" #include "../common/helpcode_utils.h" #include +#include #include #include #include @@ -535,6 +536,10 @@ struct CorrectionTarget // pair. The tie-break weight is derived from the ENABLED subset at query // time (see min_enabled_correction_weight), never from a switched-off table. unsigned type_bit = 0; + // Out-of-table generated pair (non-neighbor substitution / arbitrary-letter + // insertion): flat expensive-tier weight (kAutocorrectGeneratedShapeWeight) + // instead of the per-bit derivation; switch gating still uses type_bit. + bool generated = false; }; // Canonical per-type tie-break cost. Each correction type carries one fixed @@ -574,6 +579,126 @@ int min_enabled_correction_weight(const unsigned enabled_bits) return best; } +// QWERTY letter-key neighbors (finger-movement range, letter keys only), the +// same adjacency as server/scripts/generate_quanpin_autocorrect.py:42-69. +// Indexed by letter - 'a' (a..z order -- NOT keyboard order). Drift is +// self-healing only in one direction: a pair misclassified as "non-neighbor" +// that actually collides with the static table falls back to the cheaper +// static weight, but a neighbor misclassified as "far" simply skips +// generation -- so keep this table byte-identical to the generator's. +constexpr std::array kQwertyNeighbors = { + "qwsz", // a + "vghn", // b + "xdfv", // c + "serfcx", // d + "wrsd", // e + "drtgvc", // f + "ftyhbv", // g + "gyujbn", // h + "uojk", // i + "huikmn", // j + "jiolm", // k + "kop", // l + "njk", // m + "bhjm", // n + "ipkl", // o + "ol", // p + "wa", // q + "etdf", // r + "awdexz", // s + "ryfg", // t + "yihj", // u + "cfgb", // v + "qeas", // w + "zsdc", // x + "tugh", // y + "asx", // z +}; + +constexpr bool is_qwerty_neighbor(char key, char of_letter) +{ + const unsigned row = static_cast(of_letter - 'a'); + return row < kQwertyNeighbors.size() && kQwertyNeighbors[row].find(key) != std::string_view::npos; +} + +// One out-of-table generated pair: wrong-form string plus the legal syllable it +// stands for. wrong is owned by the generated store (static lifetime) because +// the correction index refers to it through string_views. +struct GeneratedPair +{ + std::string wrong; + std::string_view syllable; + unsigned type_bit; +}; + +// Enumerate the out-of-table shapes over every legal syllable: non-neighbor +// substitutions and arbitrary-letter insertions. Transpositions and deletions +// are exhaustive in the static tables -- nothing to add. Wrong forms must be +// non-legal, at least three letters (shorter strings belong to the jianpin +// space) and at most six (the k-best piece scan never looks at longer pieces) -- +// the same window as the static tables and the evaluation filter. +std::vector build_generated_pairs() +{ + const auto &legal = intact_pinyin_set(); + std::vector pairs; + for (const std::string &syllable : intact_pinyin_list()) + { + if (syllable.size() < 3) + { + continue; // same-length substitutions would violate the 3-letter floor + } + // 非相邻键替换：换上的字母既不是原字母也不是它的 QWERTY 邻键。 + for (size_t i = 0; i < syllable.size(); ++i) + { + for (char key = 'a'; key <= 'z'; ++key) + { + if (key == syllable[i] || is_qwerty_neighbor(key, syllable[i])) + { + continue; + } + std::string wrong = syllable; + wrong[i] = key; + if (legal.find(wrong) != legal.end()) + { + continue; + } + pairs.push_back({std::move(wrong), syllable, kAutocorrectNeighbor}); + } + } + // 任意字母插入：插入字母不属于静态覆盖集（该位相邻字母及其邻键）。 + for (size_t i = 0; i <= syllable.size(); ++i) + { + const char left = i > 0 ? syllable[i - 1] : '\0'; + const char right = i < syllable.size() ? syllable[i] : '\0'; + for (char key = 'a'; key <= 'z'; ++key) + { + const bool statically_covered = key == left || key == right || + (left != '\0' && is_qwerty_neighbor(key, left)) || + (right != '\0' && is_qwerty_neighbor(key, right)); + if (statically_covered) + { + continue; + } + std::string wrong = syllable; + wrong.insert(i, 1, key); + if (wrong.size() > 6 || legal.find(wrong) != legal.end()) + { + continue; // 7 letters sit outside the k-best piece scan window + } + pairs.push_back({std::move(wrong), syllable, kAutocorrectInsertion}); + } + } + } + return pairs; +} + +// Static lifetime: the correction index refers to `wrong` through string_views. +const std::vector &generated_pairs() +{ + static const std::vector pairs = build_generated_pairs(); + return pairs; +} + const std::unordered_map> &correction_index() { static const std::unordered_map> kIndex = [] { @@ -609,6 +734,21 @@ const std::unordered_map> &corre add_table(autocorrect::kDeletionEntries, autocorrect::kDeletionCount, kAutocorrectDeletion); add_table(autocorrect::kInsertionEntries, autocorrect::kInsertionCount, kAutocorrectInsertion); add_table(autocorrect::kNeighborEntries, autocorrect::kNeighborCount, kAutocorrectNeighbor); + // Out-of-table generated shapes: appended AFTER the static tables so a + // shared wrong key keeps its cheaper static targets first. A generated + // pair colliding with a static pair is dropped -- the static table's + // cheaper weight already covers it (self-heals engine/generator drift). + for (const auto &pair : generated_pairs()) + { + auto &targets = index[pair.wrong]; + const auto duplicate = std::find_if(targets.begin(), targets.end(), [&](const CorrectionTarget &target) { + return target.syllable == pair.syllable; + }); + if (duplicate == targets.end()) + { + targets.push_back(CorrectionTarget{pair.syllable, pair.type_bit, true}); + } + } return index; }(); return kIndex; @@ -624,6 +764,11 @@ struct AutocorrectEdge size_t raw_length = 0; std::string_view syllable; bool corrected = false; + // Out-of-table generated pair (non-neighbor substitution / arbitrary-letter + // insertion). Generated-containing hypotheses rank behind ALL static + // hypotheses (see the finalize comparator) so in-table inputs keep their + // baseline cut set. + bool generated = false; }; // One propagated cut hypothesis, ranked by (edge_count, weight, arrival): @@ -638,6 +783,10 @@ struct SearchHypothesis size_t prev_index = kNoPredecessor; size_t arrival = 0; int weight = 0; + // True once any generated edge participates in this path. Such hypotheses + // rank behind every static hypothesis (see the finalize comparator) so + // in-table inputs keep their baseline cut set. + bool has_generated = false; AutocorrectEdge edge; // Order-sensitive rolling hash of the joined syllable sequence. Two // hypotheses reaching the same position share a sequence iff their hashes @@ -696,6 +845,14 @@ std::vector autocorrect_cut_kbest(const std::string &pinyin, con return; } std::sort(list.begin(), list.end(), [](const SearchHypothesis &lhs, const SearchHypothesis &rhs) { + // Generated-containing hypotheses rank behind ALL static ones + // regardless of edge count: out-of-table shapes only step in when + // the static space has no better explanation, and in-table inputs + // must keep their baseline cut set (zero-regression by design). + if (lhs.has_generated != rhs.has_generated) + { + return lhs.has_generated < rhs.has_generated; + } if (lhs.edge_count != rhs.edge_count) { return lhs.edge_count < rhs.edge_count; @@ -743,6 +900,7 @@ std::vector autocorrect_cut_kbest(const std::string &pinyin, con child.arrival = arrival++; child.weight = parent.weight + edge_weight; child.edge = edge; + child.has_generated = parent.has_generated || edge.generated; child.seq_hash = extend_sequence_hash(parent.seq_hash, edge.syllable); best[end].push_back(std::move(child)); }; @@ -788,7 +946,9 @@ std::vector autocorrect_cut_kbest(const std::string &pinyin, con } edge.syllable = target.syllable; edge.corrected = true; - const int weight = min_enabled_correction_weight(enabled_bits); + edge.generated = target.generated; + const int weight = + target.generated ? kAutocorrectGeneratedShapeWeight : min_enabled_correction_weight(enabled_bits); for (size_t i = 0; i < hypotheses.size(); ++i) { extend(start + len, hypotheses[i], i, edge, weight); @@ -797,6 +957,18 @@ std::vector autocorrect_cut_kbest(const std::string &pinyin, con } } finalize(length); + // Static-priority zero-regression (see the header contract): when the best + // cut is pure static, drop every generated cut so every caller -- the + // primary resolver, the head+tail composition retry, the k == 1 preedit + // projection -- consumes exactly the baseline cut set; generated readings + // surface only when no static cut explains the input at all. Living here + // (not in each caller) is the point: a future consumer cannot forget it. + if (!best[length].empty() && !best[length].front().has_generated) + { + best[length].erase(std::remove_if(best[length].begin(), best[length].end(), + [](const SearchHypothesis &hypothesis) { return hypothesis.has_generated; }), + best[length].end()); + } // Rebuild each surviving end hypothesis by walking the predecessor chain. // raw_length (not syllable.size()) gives the raw span: deletion edges @@ -808,6 +980,7 @@ std::vector autocorrect_cut_kbest(const std::string &pinyin, con AutocorrectCut cut; cut.edge_count = hypothesis.edge_count; cut.weight = hypothesis.weight; + cut.has_generated = hypothesis.has_generated; size_t position = length; const SearchHypothesis *current = &hypothesis; while (current->prev_index != kNoPredecessor) diff --git a/engine/quanpin/quanpin_utils.h b/engine/quanpin/quanpin_utils.h index 8b8c4115..49eae31d 100644 --- a/engine/quanpin/quanpin_utils.h +++ b/engine/quanpin/quanpin_utils.h @@ -78,6 +78,14 @@ inline constexpr int kAutocorrectDeletionWeight = 11; inline constexpr int kAutocorrectInsertionWeight = 12; inline constexpr int kAutocorrectNeighborWeight = 13; +// Out-of-table generated shapes (non-neighbor substitutions / arbitrary-letter +// insertions, see CorrectionTarget::generated): fixed expensive-tier weight. +// These slips are rarer than any in-table shape, and the ranking contract +// requires them to never displace a table hit on the same wrong key. ponytail: +// flat constant; calibration path = user_journal confusion priors (direction +// task phase 3). +inline constexpr int kAutocorrectGeneratedShapeWeight = 15; + // One segment of an autocorrect-aware cut. syllable is the canonical (possibly // table-corrected) text used for dictionary lookups; raw_text/start describe the // original letters the segment consumed so the preedit can keep showing what the @@ -104,6 +112,11 @@ struct AutocorrectCut // (neighbor gau -> gai, weight 13) regardless of dictionary frequency. size_t edge_count = 0; int weight = 0; + // True when any edge of this cut is an out-of-table generated pair. The + // search itself drops generated-containing cuts whenever a pure-static cut + // tops the ranking, so in-table inputs consume exactly the baseline cut + // set. + bool has_generated = false; bool empty() const { @@ -134,7 +147,12 @@ Segments autocorrect_cut(const std::string &pinyin, unsigned autocorrect_types); // graph and gating as autocorrect_cut, but keeping ambiguous table entries // alive as parallel hypotheses. Ranking key: (corrected edge count, summed // edge weight, generation order = table order); hypotheses explaining the same -// syllable sequence are deduplicated, keeping the best-ranked one. Every +// syllable sequence are deduplicated, keeping the best-ranked one. Static +// priority: generated-containing hypotheses rank behind every static one, and +// when the top cut is pure static all generated cuts are dropped before +// returning -- in-table inputs get exactly the pre-generated-space cut set, +// and out-of-table shapes surface only when no static cut explains the input. +// Every // returned cut contains at least one corrected edge, so a fully legal input // with no correction reading yields an empty vector (the caller owns the plain // segmentation). This is the query-time disambiguation surface of CN 101133411 diff --git a/engine/tests/README.md b/engine/tests/README.md index 5a5996b1..5e2d65c5 100644 --- a/engine/tests/README.md +++ b/engine/tests/README.md @@ -37,7 +37,9 @@ cmake --build build --config Release --target ` --samples 300 --seed 42 --model mixed ` diff --git a/engine/tests/src/eval_quanpin_autocorrect.cpp b/engine/tests/src/eval_quanpin_autocorrect.cpp index 937dc581..d57f7f1e 100644 --- a/engine/tests/src/eval_quanpin_autocorrect.cpp +++ b/engine/tests/src/eval_quanpin_autocorrect.cpp @@ -89,6 +89,7 @@ enum class ErrorModel Deletion, Ambiguous, Insertion, + Outside, }; enum class ErrorKind @@ -97,6 +98,8 @@ enum class ErrorKind Neighbor, Deletion, Insertion, + FarSubstitution, + FarInsertion, }; struct Options @@ -120,7 +123,7 @@ void usage_exit(const std::string &message) std::cerr << "ERROR: " << message << "\n"; } std::cerr << "usage: eval_quanpin_autocorrect --db [--samples N] [--seed S]\n" - << " [--rates 10,25,50,100] [--model mixed|deletion|ambiguous|insertion]\n" + << " [--rates 10,25,50,100] [--model mixed|deletion|ambiguous|insertion|outside]\n" << " [--resource ] [--csv ] [--seeds-file ] [--dump N]\n"; std::exit(2); } @@ -216,6 +219,10 @@ Options parse_options(int argc, char *argv[]) { options.model = ErrorModel::Insertion; } + else if (options.model_name == "outside") + { + options.model = ErrorModel::Outside; + } else { usage_exit("unknown --model: " + options.model_name); @@ -434,7 +441,7 @@ std::vector raw_variants(ErrorKind kind, const std::string &syllabl out.push_back(syllable.substr(0, i) + syllable.substr(i + 1)); } } - else + else if (kind == ErrorKind::Insertion) { // 插入：与生成器 insertion_variants 同款约束——位置锚定串首（参照右 // 邻）、串尾（参照左邻）、中间（参照任一侧），插入字母必须是相邻键的 @@ -472,6 +479,69 @@ std::vector raw_variants(ErrorKind kind, const std::string &syllabl } } } + else if (kind == ErrorKind::FarSubstitution) + { + // 表外形状：非相邻键替换（替换键不是原键也不是其 QWERTY 邻键）。与 + // 静态表替换形状构造性不相交，用于度量生成式纠错空间的覆盖增益。 + for (size_t i = 0; i < syllable.size(); ++i) + { + std::unordered_set blocked; + blocked.insert(syllable[i]); + for (const auto &[key, neighbors] : qwerty_neighbors()) + { + if (key == syllable[i]) + { + blocked.insert(neighbors.begin(), neighbors.end()); + } + } + for (char key = 'a'; key <= 'z'; ++key) + { + if (blocked.count(key) != 0) + { + continue; + } + out.push_back(syllable.substr(0, i) + key + syllable.substr(i + 1)); + } + } + } + else if (kind == ErrorKind::FarInsertion) + { + // 表外形状：插入字母不属于静态覆盖集（该位相邻字母及其邻键）。 + for (size_t position = 0; position <= syllable.size(); ++position) + { + std::unordered_set covered; + if (position > 0) + { + covered.insert(syllable[position - 1]); + for (const auto &[key, neighbors] : qwerty_neighbors()) + { + if (key == syllable[position - 1]) + { + covered.insert(neighbors.begin(), neighbors.end()); + } + } + } + if (position < syllable.size()) + { + covered.insert(syllable[position]); + for (const auto &[key, neighbors] : qwerty_neighbors()) + { + if (key == syllable[position]) + { + covered.insert(neighbors.begin(), neighbors.end()); + } + } + } + for (char key = 'a'; key <= 'z'; ++key) + { + if (covered.count(key) != 0) + { + continue; + } + out.push_back(syllable.substr(0, position) + key + syllable.substr(position)); + } + } + } return out; } @@ -495,7 +565,8 @@ std::vector usable_variants(ErrorKind kind, const std::string &syll } // mixed = 交换 25% / 邻键 25% / 漏字 25% / 多字 25%；deletion = 100% 漏字； -// insertion = 100% 多字。选中的错误类型在该音节上无可用变体（如 2 字母音节 +// insertion = 100% 多字；outside = 非相邻替换 / 远键插入各半（表外形状，度量 +// 生成式纠错空间的覆盖增益）。选中的错误类型在该音节上无可用变体（如 2 字母音节 // 或 6 字母音节的插入变体超长）时保持原样，交由样本级作废逻辑处理，不回退 // 到其他类型，保证分布可解释。 std::string corrupt_syllable(const std::string &syllable, std::mt19937 &rng, ErrorModel model, @@ -514,6 +585,11 @@ std::string corrupt_syllable(const std::string &syllable, std::mt19937 &rng, Err { kind = ErrorKind::Insertion; } + else if (model == ErrorModel::Outside) + { + kind = std::uniform_int_distribution(1, 100)(rng) <= 50 ? ErrorKind::FarSubstitution + : ErrorKind::FarInsertion; + } const auto variants = usable_variants(kind, syllable, legal); if (variants.empty()) { diff --git a/engine/tests/src/test_input_session.cpp b/engine/tests/src/test_input_session.cpp index 0b196d99..7a5e9c9e 100644 --- a/engine/tests/src/test_input_session.cpp +++ b/engine/tests/src/test_input_session.cpp @@ -411,6 +411,96 @@ void run_autocorrect_context_ranking_tests(const std::filesystem::path &data_dir } } +// 阶段 2 生成式纠错空间（任务 quanpin-autocorrect-generated-space）：静态表 +// 形状之外的单编辑手误由生成式索引兜底，权重落贵档（15）。隔离 fixture： +// - shatg = shang 的 n→t（t 非邻键，远键替换） +// - zthou = zhou 的 z/h 之间插 t（t 不在静态覆盖集，远键插入） +// - chng 同键双读：chng→chang（静态漏字 11）与 chng→cang（生成 15）， +// 静态最优时生成切分整体丢弃——静态优先语义的判别性断言 +// - shatngzh = head shatng + 简拼尾 zh：head 组合路径同享静态优先 +void run_autocorrect_generated_space_tests(const std::filesystem::path &data_directory) +{ + const std::filesystem::path directory = data_directory / "autocorrect-generated"; + std::filesystem::create_directories(directory); + { + Database database(directory / "msime.db"); + database.execute("CREATE TABLE tbl_1_s(key TEXT,jp TEXT,value TEXT,weight INTEGER);" + "INSERT INTO tbl_1_s VALUES('shang', 's', '上', 900);"); + database.execute("CREATE TABLE tbl_1_z(key TEXT,jp TEXT,value TEXT,weight INTEGER);" + "INSERT INTO tbl_1_z VALUES('zhou', 'z', '周', 900);"); + database.execute("CREATE TABLE tbl_1_c(key TEXT,jp TEXT,value TEXT,weight INTEGER);" + "INSERT INTO tbl_1_c VALUES('chang', 'c', '长', 900);" + "INSERT INTO tbl_1_c VALUES('cang', 'c', '仓', 5000);"); + database.execute("CREATE TABLE tbl_2_s(key TEXT,jp TEXT,value TEXT,weight INTEGER);" + "INSERT INTO tbl_2_s VALUES('shang''zh', 'sz', '上周', 900);"); + database.execute("CREATE TABLE tbl_3_s(key TEXT,jp TEXT,value TEXT,weight INTEGER);" + "INSERT INTO tbl_3_s VALUES('sha''tang''zh', 'stz', '沙汤扎', 900);"); + } + + metasequoia::RuntimePaths paths; + paths.resources = directory; + paths.user_data = directory; + paths.cache = directory; + paths.dictionaries = directory; + const unsigned both = quanpin::kAutocorrectTransposition | quanpin::kAutocorrectNeighbor; + + // 远键替换：shatg → shang（生成对，贵档 15）。 + { + metasequoia::InputSession session(SchemeType::Quanpin, both, true, true, true, paths); + type(session, "shatg"); + const auto found = find_candidate_index(session, "上"); + require(found < session.candidates().size() && session.candidates()[found].corrected_from == "shatg", + "A far-substitution typo must be corrected to 上 with the typed letters recorded."); + } + + // 远键插入（词中位）：zthou → zhou（t 不在 z/h 的覆盖集）。fixture 特意 + // 选无竞争切分的输入：zhwou 这类插入位会拼出 [zha(w 邻键替换)+ou] 等更 + // 便宜的合法切分（权重 13 < 15），生成读法按契约排后被前缀候选遮蔽—— + // 那是排序语义，不是生成类失效。zthou 无任何竞争切分，唯一读即纠错读法。 + { + metasequoia::InputSession session(SchemeType::Quanpin, both, true, true, true, paths); + type(session, "zthou"); + const auto found = find_candidate_index(session, "周"); + require(found < session.candidates().size() && session.candidates()[found].corrected_from == "zthou", + "A far-insertion typo must be corrected to 周 with the typed letters recorded."); + } + + // 静态优先语义：chng = chang 漏 a。读法竞争——chng→chang（静态漏字 11）与 + // chng→cang（生成远键替换 h→a，15）。最优切分为纯静态时生成切分整体丢弃： + // 仓不出现（混表加权仲裁依赖训练权重，方向阶段 3），长照常领衔。 + { + metasequoia::InputSession session(SchemeType::Quanpin, both, true, true, true, paths); + type(session, "chng"); + require(candidate_index(session, "长") == 0, "The in-table cheaper deletion reading must lead for chng."); + require( + find_candidate_index(session, "仓") == session.candidates().size(), + "Static-priority semantics: the generated cang reading must not surface when a static cut tops the input."); + } + + // head+简拼尾组合路径的静态优先：shatngzh 在整串 k-best 上无解（尾部 zh 非 + // 完整音节），走 head 重试。head shatng 的读法竞争——sha+tng→tang（静态漏字 + // 11）与 shatng→shang（生成插入 15）。丢弃规则由 k-best 搜索自身执行，head + // 路径同享：静态读法的沙汤扎（sha'tang'zh 精确键）在场，上周（shang+zh 生成 + // 读法经 costlier 档，表内输入的基线里不存在）不得出现。 + { + metasequoia::InputSession session(SchemeType::Quanpin, both, true, true, true, paths); + type(session, "shatngzh"); + require(find_candidate_index(session, "沙汤扎") < session.candidates().size(), + "The head-composition path must still correct shatng to sha+tang."); + require(find_candidate_index(session, "上周") == session.candidates().size(), + "Static priority must hold on the head path: the generated shang+zh reading must not surface."); + } + + // 守卫与合法输入：尾位插入（shangv 类）受简拼尾守卫——纠错不触发由 + // test_pinyin 的 zheg 用例与输入门不变式覆盖，此处不重复；合法拼写原样直出。 + { + metasequoia::InputSession legal(SchemeType::Quanpin, both, true, true, true, paths); + type(legal, "shang"); + require(candidate_index(legal, "上") == 0 && legal.candidates()[0].corrected_from.empty(), + "A legal spelling must stay uncorrected."); + } +} + void run_caret_prefix_session_tests(const std::filesystem::path &data_directory) { const std::filesystem::path directory = data_directory / "caret-prefix"; @@ -1317,6 +1407,7 @@ int run_test() run_umlaut_alias_session_tests(data_directory); run_caret_prefix_session_tests(data_directory); run_autocorrect_context_ranking_tests(data_directory); + run_autocorrect_generated_space_tests(data_directory); #endif #ifndef METASEQUOIA_SKIP_FREQUENCY_TESTS