diff --git a/engine/AGENTS.md b/engine/AGENTS.md index e95f0378..2af6e111 100644 --- a/engine/AGENTS.md +++ b/engine/AGENTS.md @@ -57,7 +57,8 @@ ctest --test-dir build --output-on-failure --timeout 20 - 平局时偏好更短的首音节(`quanpin/quanpin_utils.cpp` 的 `cut_one_piece_min_segments`)。**这个判据是对的,不要翻成前向最长匹配**——在 48068 条带词典切分的多音节词上量过,翻向会从 99.605% 掉到 99.393%,因为拼音的 `n`/`ng` 歧义让前向最长匹配在中文上不成立 - 歧义切分交给词典裁决,走 `quanpin_dictionary.cpp` 的备选切分路径,受 `kMaxSyllablesForMultipleSegmentations` 限制 -- `query()` 在每次按键上都会跑。往里加无条件的枚举或查询前先量耗时,`tests/src/test_pinyin.cpp` 里那个 timings 用例只打印不断言,拦不住回归 +- 上一条是单词级(有整词就按词频比)。没有任何切分能命中整词的整句(`yibananzhao` 的一般按照 / 一把难找),由 `arbitrate_literal_segmentations` 对等音节数的字面切分各解一遍词格、按路径分裁决,只在整句来源开着时介入;预编辑跟着首选的切分走。上面的「平局取短首音节」在这里只是同分时的判据 +- `query()` 在每次按键上都会跑。往里加无条件的枚举或查询前先量耗时,`tests/src/test_pinyin.cpp` 里那个 timings 用例只打印不断言,拦不住回归。改切分或整句打分时,用 `eval_quanpin_sentence`(句子级评测集在 `tests/data/quanpin_sentence_cases.tsv`)对着真实词库的副本跑前后对比,它同时报每键耗时 ## 数据 diff --git a/engine/CMakeLists.txt b/engine/CMakeLists.txt index cb50b52c..914f350a 100644 --- a/engine/CMakeLists.txt +++ b/engine/CMakeLists.txt @@ -307,6 +307,10 @@ if(BUILD_TESTING) add_executable(bench_input_session tests/src/bench_input_session.cpp) target_link_libraries(bench_input_session PRIVATE MetasequoiaIme::Engine) + # Sentence-level segmentation evaluation (tests/data/quanpin_sentence_cases.tsv); run by hand, not by ctest. + add_executable(eval_quanpin_sentence tests/src/eval_quanpin_sentence.cpp) + target_link_libraries(eval_quanpin_sentence PRIVATE MetasequoiaIme::Engine) + # Run this benchmark by hand with the released dictionaries and neural models. add_executable(bench_neural_rescore tests/src/bench_neural_rescore.cpp) target_link_libraries(bench_neural_rescore PRIVATE MetasequoiaIme::Engine) diff --git a/engine/core/input_session_composition.cpp b/engine/core/input_session_composition.cpp index 21cf5fb7..f5f52c18 100644 --- a/engine/core/input_session_composition.cpp +++ b/engine/core/input_session_composition.cpp @@ -412,6 +412,46 @@ std::string BuildQuanpinAutocorrectDisplay(const QueryRequest &request) return letters_rewritten ? cased : base; } +// 首选是同一串字母的另一种字面切分时(yibananzhao 由 yi'ban'an'zhao 的「一般按照」领衔, +// fangan 由 fang'an 的「方案」领衔),预编辑跟着首选分,而不是停在方案层的最少音节切分上, +// 否则用户看着 yi'ba'nan'zhao 选「一般按照」。只认读音字母与敲的字母逐个相同、盖住整串、 +// 不带纠错标记的首选(辅助码、手动分隔符都不满足),而且音节数与方案层切分相同:那正是 +// 只差一处 n/ng 边界的等长歧义;音节数不同的读法(nue 的字面整词「怒鹅」nu'e)不在此列, +// 预编辑照旧。否则返回空串。 +std::string BuildQuanpinDisplayFromLiteralHead(const QueryRequest &request, const WordItem &head) +{ + const std::string &cased = request.raw_input_with_cases.empty() ? request.raw_input : request.raw_input_with_cases; + if (!head.corrected_from.empty() || request.raw_input.find('\'') != std::string::npos || + cased.size() != request.raw_input.size()) + { + return {}; + } + const auto syllables = quanpin::split_segments(head.pinyin); + const std::string &scheme_segmentation = + request.normalized_segmentation.empty() ? request.segmentation : request.normalized_segmentation; + if (syllables.size() < 2 || syllables.size() != quanpin::split_segments(scheme_segmentation).size()) + { + return {}; + } + std::vector boundaries; + size_t end = 0; + for (const auto &syllable : syllables) + { + if (request.raw_input.compare(end, syllable.size(), syllable) != 0) + { + return {}; + } + end += syllable.size(); + boundaries.push_back(end); + } + if (end != request.raw_input.size()) + { + return {}; + } + boundaries.pop_back(); + return RebuildQuanpinDisplayAtBoundaries(cased, boundaries); +} + // 合法输入上的纠错领衔时(ziazheliya 出「在这里呀」、jiuzheeyang 出「就这样」),预编辑 // 按领衔读法分隔成 zia'zhe'li'ya、jiu'zhee'yang,而不是字面切分 zi'a'zhe'li'ya——后者 // 和首选对不上。边界取纠错切分记录的原始区间(多字、漏字纠错和原始字母不等长); @@ -620,7 +660,11 @@ std::string InputSession::build_pinyin_segmentation_with_cases(bool shuangpin_ra } if (!candidates().empty()) { - std::string display = BuildQuanpinDisplayFromLeadingCorrection(request(), candidates().front()); + std::string display = BuildQuanpinDisplayFromLiteralHead(request(), candidates().front()); + if (display.empty()) + { + display = BuildQuanpinDisplayFromLeadingCorrection(request(), candidates().front()); + } if (!display.empty()) { return display; diff --git a/engine/quanpin/quanpin_dictionary.cpp b/engine/quanpin/quanpin_dictionary.cpp index 1f813b0e..290aeee5 100644 --- a/engine/quanpin/quanpin_dictionary.cpp +++ b/engine/quanpin/quanpin_dictionary.cpp @@ -137,6 +137,10 @@ constexpr double kLegalInputLengthChangePenaltyLog = 3.0; // 原读法领衔时,纠错读法离它在这个分差以内才出现在第 2 位;更远的不出现。没有它, // 每个碰巧能纠成别的音节的输入都会在第 2 位挂一个语言模型都不认的读法。 constexpr double kLegalInputRunnerUpGapLog = 3.0; +// 合法输入的等音节数字面切分(ban'an / ba'nan):最多枚举几条、最多多解几条词格。 +// 每多解一条,按键路径上就多一次词格解码;绝大多数输入只有一两处歧义边界。 +constexpr size_t kLiteralSegmentationEnumerateLimit = 8; +constexpr size_t kLiteralSegmentationDecodeLimit = 3; quanpin::Segments cut_syllables(const quanpin::AutocorrectCut &cut) { @@ -342,8 +346,9 @@ std::string escape_sql_text(std::string text) QuanpinDictionary::QuanpinDictionary(std::string db_path, metasequoia::RuntimePaths paths) : cache_(128), series_cache_(128), segmentation_cache_(128), prefix_miss_cache_(128), resolution_cache_(128), - paths_(std::move(paths)), decoder_(paths_.resource(metasequoia::assets::pinyin_model), - paths_.user(metasequoia::assets::pinyin_user_dictionary)), + lattice_best_path_cache_(64), paths_(std::move(paths)), + decoder_(paths_.resource(metasequoia::assets::pinyin_model), + paths_.user(metasequoia::assets::pinyin_user_dictionary)), language_model_(&ngram::shared_language_model(paths_.resource(metasequoia::assets::language_model))), neural_keyboard_model_(metasequoia::path_to_utf8(paths_.resource(metasequoia::assets::neural_model_keyboard))), db_path_(db_path.empty() ? metasequoia::path_to_utf8(paths_.dictionary(metasequoia::assets::main_dictionary)) @@ -715,10 +720,13 @@ std::vector QuanpinDictionary::query_exact(const std::string &raw_inpu result = merge_alternative_segmentations(raw_input, pinyin_segmentation_, segments, alternative_segmentations, std::move(result)); } + // 字面切分先争出领衔,纠错读法再与领衔的那条比:纠错胜出时第 2 位是得分最高的字面读法。 + quanpin::Segments leading_segments; + result = arbitrate_literal_segmentations(raw_input, segments, std::move(result), leading_segments); if (!resolution.legal_corrected_cuts.empty()) { - result = - arbitrate_legal_corrections(raw_input, segments, resolution.legal_corrected_cuts, std::move(result)); + result = arbitrate_legal_corrections(raw_input, leading_segments, resolution.legal_corrected_cuts, + std::move(result)); } keep_literal_whole_word_first(result, raw_input); } @@ -921,9 +929,9 @@ std::vector QuanpinDictionary::query_series(const std::string &raw_inp } if (sentence_association_.word_lattice || !neural_rerankers.empty()) { - quanpin::merge_lattice_candidates( - result, segments, quanpin::make_lattice_db_lookup(db_, statement_cache_, lattice_options.span_limit), - segmentation.empty() ? raw_input : segmentation, lattice_options, neural_rerankers); + quanpin::merge_lattice_candidates(result, segments, lattice_lookup(lattice_options.span_limit), + segmentation.empty() ? raw_input : segmentation, lattice_options, + neural_rerankers); } } @@ -1165,6 +1173,177 @@ std::vector QuanpinDictionary::merge_alternative_segmentations( return merged; } +// 合法输入常有不止一种等音节数的切分:yibananzhao 既是 yi'ba'nan'zhao 也是 +// yi'ban'an'zhao,最少音节规则平局时取较短的首音节,于是固定落在前者、出「一把难找」。 +// 切分是在查词之前定的,没有上下文,这里把它交给整句打分: +// - 任一条切分在词库里有整词(fang'an 的方案 / fan'gan 的反感、西安)时不介入,那是 +// merge_alternative_segmentations 按词频比的单词级歧义,已经调好,不该被整句分翻动; +// - 否则各切分解一遍词格,路径分最高的领衔,同分仍是原切分(最少音节规则降为平局判据); +// - 输的一方分差不超过 kLegalInputRunnerUpGapLog 时,它的整句占第 2 位;更远就不出, +// 免得每个带 n/ng 歧义的输入都在第 2 位挂一句不通的话。 +// 只在整句来源开着时介入:整句来源全关时没有整句可比,结果与改造前逐位一致。用户在原 +// 切分键上调过频或造过词时,原切分不让出领衔(同 arbitrate_legal_corrections)。 +std::vector QuanpinDictionary::arbitrate_literal_segmentations(const std::string &raw_input, + const quanpin::Segments &segments, + std::vector result, + quanpin::Segments &leading_segments) +{ + leading_segments = segments; + if (!sentence_association_.any_sentence_source() || result.empty() || segments.size() < 2 || + raw_input.find('\'') != std::string::npos || !quanpin::has_only_complete_pinyin_segments(segments)) + { + return result; + } + + // 主切分必须就是字面切分之一(别名改写过字母的读法不算),备选是其余等音节数的切分。 + bool primary_is_literal = false; + std::vector alternatives; + for (auto &cut : quanpin::enumerate_min_segmentations(raw_input, kLiteralSegmentationEnumerateLimit)) + { + cut = normalize_umlaut_aliases(std::move(cut)); + if (cut == segments) + { + primary_is_literal = true; + } + else if (alternatives.size() < kLiteralSegmentationDecodeLimit) + { + alternatives.push_back(std::move(cut)); + } + } + if (!primary_is_literal || alternatives.empty()) + { + return result; + } + + std::vector all_cuts = alternatives; + all_cuts.push_back(segments); + if (!quanpin::query_exact_segmentations_keyed_flat(all_cuts, db_, statement_cache_, 1).empty()) + { + return result; + } + + const auto primary_score = lattice_best_path_score(segments); + if (!primary_score.has_value()) + { + return result; + } + std::optional best_score; + const quanpin::Segments *best_cut = nullptr; + for (const auto &cut : alternatives) + { + const auto score = lattice_best_path_score(cut); + if (score.has_value() && (!best_score.has_value() || *score > *best_score)) + { + best_score = score; + best_cut = &cut; + } + } + if (best_cut == nullptr) + { + return result; + } + + const std::string primary_key = quanpin::join_segments(segments); + const std::string best_key = quanpin::join_segments(*best_cut); + const auto is_sentence_of = [](const std::string &key) { + return [&key](const WordItem &item) { return item.canonical_pinyin == key; }; + }; + const bool alternative_leads = *best_score > *primary_score && !user_prefers_reading(primary_key); + const double gap = alternative_leads ? *best_score - *primary_score : *primary_score - *best_score; + if (!alternative_leads) + { + if (gap > kLegalInputRunnerUpGapLog) + { + return result; + } + // 第 2 位只要一句。Trigram 整句开着时直接用打分时解出的那条最优路径,与 merge_lattice_candidates + // 造的行同形;不必为这一行再跑一遍带神经重排的 query_series。只开神经等来源时 Trigram 行不显示, + // 那就照常查一遍,取它的整键块首行(词库里没有这条切分的整词,上面已经排除,整键块就是整句行)。 + std::optional runner; + if (sentence_association_.word_lattice) + { + if (const auto path = lattice_best_path(*best_cut)) + { + runner = WordItem(best_key, path->sentence, static_cast(path->log_prob * 1000.0), + CandidateSource::Generated, path->key); + runner->sentence_association = true; + } + } + else + { + const auto alternative = query_series(raw_input, best_key, *best_cut); + const auto found = std::find_if(alternative.begin(), alternative.end(), is_sentence_of(best_key)); + if (found != alternative.end()) + { + runner = *found; + } + } + if (!runner.has_value()) + { + return result; + } + std::vector merged; + merged.push_back(result.front()); + append_unique_words(merged, {*runner}); + append_unique_words(merged, result); + return merged; + } + + leading_segments = *best_cut; + // 同 query_exact 里纠错的上下文消解:胜出切分的整键块(整句行)领衔,其余照旧, + // 胜出切分的前缀词垫底;整键块为空时(整句只开了重排、模型没载入)由前缀词领衔。 + std::vector winner_full; + std::vector winner_prefix; + for (auto &item : query_series(raw_input, best_key, *best_cut)) + { + (item.canonical_pinyin == best_key ? winner_full : winner_prefix).push_back(std::move(item)); + } + if (winner_full.empty()) + { + winner_full.swap(winner_prefix); + } + if (winner_full.empty()) + { + leading_segments = segments; + return result; + } + // 原切分的整句:分差以内占第 2 位,更远就整组不出(前缀词照留)。 + // 原切分的前缀区间扫描会捞出胜出切分的词,读音却记在原切分上(yi'ba 扫出「一般」,pinyin 是 + // yi'ba):同一个词两边都有时换成胜出切分那一行,选它才按 yi'ban 消耗字母。位置不变。 + std::unordered_map winner_prefix_by_word; + for (const auto &item : winner_prefix) + { + winner_prefix_by_word.emplace(item.word, &item); + } + std::vector primary_sentences; + std::vector rest; + for (auto &item : result) + { + if (item.canonical_pinyin == primary_key) + { + primary_sentences.push_back(std::move(item)); + continue; + } + const auto same_word = winner_prefix_by_word.find(item.word); + rest.push_back(same_word == winner_prefix_by_word.end() ? std::move(item) : *same_word->second); + } + std::vector merged; + merged.push_back(winner_full.front()); + if (gap <= kLegalInputRunnerUpGapLog && !primary_sentences.empty()) + { + append_unique_words(merged, {primary_sentences.front()}); + append_unique_words(merged, winner_full); + append_unique_words(merged, primary_sentences); + } + else + { + append_unique_words(merged, winner_full); + } + append_unique_words(merged, rest); + append_unique_words(merged, winner_prefix); + return merged; +} + // 合法输入上的手误(ziazheliya、jioa、jiuzheeyang、nia)按噪声信道整句打分: // 总分 = 词格最佳路径分 - 每处纠正的手误代价(换位 2.0,多字/漏字 3.0) // 原读法代价为 0。最高分领衔,另一方的最佳读法占第 2 位:纠错领衔时原读法永远在 @@ -1263,11 +1442,22 @@ std::vector QuanpinDictionary::arbitrate_legal_corrections( } std::optional QuanpinDictionary::lattice_best_path_score(const quanpin::Segments &cut) +{ + const auto path = lattice_best_path(cut); + return path.has_value() ? std::optional(path->log_prob) : std::nullopt; +} + +std::optional QuanpinDictionary::lattice_best_path(const quanpin::Segments &cut) { if (cut.empty() || !quanpin::has_only_complete_pinyin_segments(cut)) { return std::nullopt; } + const std::string cache_key = quanpin::join_segments(cut); + if (const auto *cached = lattice_best_path_cache_.find(cache_key)) + { + return *cached; + } // 与 query_series 的装配保持一致:collocation 项同样参与路径分。 quanpin::WordLatticeOptions lattice_options; lattice_options.language_model = language_model_; @@ -1279,9 +1469,15 @@ std::optional QuanpinDictionary::lattice_best_path_score(const quanpin:: }; lattice_options.collocation_weight = sentence_association_.collocation_weight; } - const auto lookup = quanpin::make_lattice_db_lookup(db_, statement_cache_, lattice_options.span_limit); - const auto cut_paths = quanpin::decode_word_lattice(cut, lookup, lattice_options); - return cut_paths.empty() ? std::nullopt : std::optional(cut_paths.front().log_prob); + const auto lookup = lattice_lookup(lattice_options.span_limit); + auto cut_paths = quanpin::decode_word_lattice(cut, lookup, lattice_options); + std::optional best; + if (!cut_paths.empty()) + { + best = std::move(cut_paths.front()); + } + lattice_best_path_cache_.insert(cache_key, best); + return best; } // 用户在这个读音键上有过调频或造词记录。只在边际已达标、要决定是否接管时才查, @@ -1604,6 +1800,30 @@ void QuanpinDictionary::reset_cache() series_cache_.clear(); segmentation_cache_.clear(); prefix_miss_cache_.clear(); + lattice_best_path_cache_.clear(); + lattice_span_cache_.clear(); +} + +quanpin::WordLatticeLookup QuanpinDictionary::lattice_lookup(int limit) +{ + return [this, lookup = quanpin::make_lattice_db_lookup(db_, statement_cache_, limit), + limit](const quanpin::Segments &span) { + // 超过上限整体清掉,免得一次长时间输入把内存撑大(同双拼的 direct_span_cache_)。 + constexpr std::size_t kLatticeSpanCacheLimit = 8192; + std::string key = quanpin::join_segments(span); + key.push_back('#'); + key += std::to_string(limit); + auto found = lattice_span_cache_.find(key); + if (found == lattice_span_cache_.end()) + { + if (lattice_span_cache_.size() >= kLatticeSpanCacheLimit) + { + lattice_span_cache_.clear(); + } + found = lattice_span_cache_.emplace(std::move(key), lookup(span)).first; + } + return found->second; + }; } void QuanpinDictionary::reset_sentence_cache() diff --git a/engine/quanpin/quanpin_dictionary.h b/engine/quanpin/quanpin_dictionary.h index f2d8d224..919c56c1 100644 --- a/engine/quanpin/quanpin_dictionary.h +++ b/engine/quanpin/quanpin_dictionary.h @@ -138,6 +138,12 @@ class QuanpinDictionary const std::string &raw_input, const std::string &primary_segmentation, const quanpin::Segments &primary_segments, const std::vector &alternative_segmentations, std::vector result); + // 合法输入的等音节数字面切分(yi'ba'nan'zhao / yi'ban'an'zhao)按整句打分争领衔。 + // leading_segments 返回领衔的字面切分,交给 arbitrate_legal_corrections 当原读法。 + std::vector arbitrate_literal_segmentations(const std::string &raw_input, + const quanpin::Segments &segments, + std::vector result, + quanpin::Segments &leading_segments); std::vector arbitrate_legal_corrections(const std::string &raw_input, const quanpin::Segments &plain_segments, const std::vector &corrected_cuts, @@ -145,6 +151,12 @@ class QuanpinDictionary // 词格对一条完整切分的最佳路径分(加载了 sc.lm 时是 log10,否则是启发式的 ln)。 // 与整句候选开关无关;单音节也能打分。不是全合法音节或解不出路径时为空。 std::optional lattice_best_path_score(const quanpin::Segments &cut); + // 同上,返回整条最佳路径。按切分缓存:一次按键里同一条切分会被字面切分与纠错两轮裁决各打一次分。 + std::optional lattice_best_path(const quanpin::Segments &cut); + // 词格的跨度查询,套一层按(跨度, limit)的缓存:每多敲一个字母整句都要重解,同一输入的 + // 几种字面切分、主切与纠错读法之间也大量共用跨度,不缓存就一遍遍打同样的 SQL。结果与 + // 直查逐行相同;随 reset_cache 清空(词库被写或换了开关时)。 + quanpin::WordLatticeLookup lattice_lookup(int limit); bool user_prefers_reading(const std::string &key); static void append_unique_words(std::vector &result, const std::vector &extra); void mark_autocorrect_candidates(std::vector &candidates, const std::string &raw_input); @@ -179,6 +191,10 @@ class QuanpinDictionary // cleared with the dictionary caches: its value depends only on the input // and the static correction tables, never on dictionary rows. CircularBuffer resolution_cache_; + // 见 lattice_best_path。随 reset_cache 清空:分数取决于词库行和整句选项(搭配模型)。 + CircularBuffer> lattice_best_path_cache_; + // 见 lattice_lookup。 + std::unordered_map> lattice_span_cache_; sqlite3 *db_ = nullptr; sqlite3_int64 data_version_ = -1; metasequoia::RuntimePaths paths_; diff --git a/engine/quanpin/quanpin_utils.cpp b/engine/quanpin/quanpin_utils.cpp index 6e8aa37a..ad5d5b3f 100644 --- a/engine/quanpin/quanpin_utils.cpp +++ b/engine/quanpin/quanpin_utils.cpp @@ -414,6 +414,68 @@ std::vector cut_one_piece_min_segments(const std::string &pinyin, b return solve(solve, 0); } +std::vector enumerate_min_segmentations(const std::string &pinyin, const size_t path_limit) +{ + std::vector result; + if (pinyin.empty() || path_limit == 0) + { + return result; + } + const auto &pinyin_set = intact_pinyin_set(); + constexpr size_t kMaxSyllableLength = 6; + constexpr size_t kUnreachable = static_cast(-1); + const size_t length = pinyin.size(); + // min_count[i]:从 i 切到末尾最少几个音节。只沿「这一刀之后仍是最少音节」的边往下走, + // 枚举出来的就只有等音节数的切分,组合数只随歧义边界的个数增长。 + std::vector min_count(length + 1, kUnreachable); + min_count[length] = 0; + for (size_t start = length; start-- > 0;) + { + for (size_t len = 1; len <= kMaxSyllableLength && start + len <= length; ++len) + { + if (min_count[start + len] != kUnreachable && + pinyin_set.find(pinyin.substr(start, len)) != pinyin_set.end()) + { + min_count[start] = (std::min)(min_count[start], min_count[start + len] + 1); + } + } + } + if (min_count[0] == kUnreachable) + { + return result; + } + + Segments current; + const auto walk = [&](auto &&self, const size_t start) -> void { + if (result.size() >= path_limit) + { + return; + } + if (start == length) + { + result.push_back(current); + return; + } + for (size_t len = 1; len <= kMaxSyllableLength && start + len <= length; ++len) + { + if (min_count[start + len] == kUnreachable || min_count[start + len] + 1 != min_count[start]) + { + continue; + } + std::string piece = pinyin.substr(start, len); + if (pinyin_set.find(piece) == pinyin_set.end()) + { + continue; + } + current.push_back(std::move(piece)); + self(self, start + len); + current.pop_back(); + } + }; + walk(walk, 0); + return result; +} + bool is_complete_pinyin_input(const std::string &pinyin) { if (pinyin.empty()) diff --git a/engine/quanpin/quanpin_utils.h b/engine/quanpin/quanpin_utils.h index 3b8b09b3..1759a97e 100644 --- a/engine/quanpin/quanpin_utils.h +++ b/engine/quanpin/quanpin_utils.h @@ -33,6 +33,9 @@ SyllableGraph build_syllable_graph(const std::string &pinyin); std::vector enumerate_complete_segmentations(const SyllableGraph &graph, size_t path_limit = 32); std::vector cut_one_piece_greedy(const std::string &pinyin, bool intact_only); std::vector cut_one_piece_min_segments(const std::string &pinyin, bool intact_only); +// 音节数与 cut_one_piece_min_segments 相同的全部合法切分(最多 path_limit 条),即只在 +// ban'an / ba'nan 这类等长歧义边界上分叉。cut_one_piece_min_segments 的结果总在其中。 +std::vector enumerate_min_segmentations(const std::string &pinyin, size_t path_limit); bool is_complete_pinyin_input(const std::string &pinyin); // 把切好的全拼串换成 Google 两个解码器(本地 googlepinyinime-rev 与 inputtools // 云输入)认的 ü 写法:nve→nue、lve→lue、jv→ju……只在送进这两个解码器前用, diff --git a/engine/tests/data/quanpin_sentence_cases.tsv b/engine/tests/data/quanpin_sentence_cases.tsv new file mode 100644 index 00000000..5839bd51 --- /dev/null +++ b/engine/tests/data/quanpin_sentence_cases.tsv @@ -0,0 +1,38 @@ +# 全拼整句级切分评测集。eval_quanpin_sentence 逐条敲入「输入」,看首选是不是「期望首选」。 +# 列:输入期望首选类别备注 +# 类别: +# ambiguous 同一串字母有两种等音节数的合法切分,正确的是最少音节规则平局时不选的那一种 +# control 同上有歧义,但最少音节规则本来就选对了;改动后不能变差 +# plain 没有切分歧义的普通整句和单词,用来盯回归 +# word 备选切分在词库里有整词,仍按词频比较(不应受整句打分影响) +# typo 合法输入上的手误(需要打开纠错) +yibananzhao 一般按照 ambiguous yi'ban'an'zhao / yi'ba'nan'zhao +woxianganzhaojihuazuo 我想按照计划做 ambiguous xiang'an / xian'gan +qinganzhaoshunxupaidui 请按照顺序排队 ambiguous qing'an / qin'gan +tamenyijinganzhaoyaoqiuzuole 他们已经按照要求做了 ambiguous jing'an / jin'gan +zhegefanganhenhao 这个方案很好 ambiguous fang'an / fan'gan +tadefanganbeitongguole 他的方案被通过了 ambiguous fang'an / fan'gan +fanganyijingdingle 方案已经定了 ambiguous fang'an / fan'gan +zhegexiangmudefangan 这个项目的方案 ambiguous fang'an / fan'gan +danganziliao 档案资料 ambiguous dang'an / dan'gan +anzhaoxinfanganzhixing 按照新方案执行 ambiguous fang'an / fan'gan +tazhengzaiganhuo 他正在干活 control zheng'zai'gan +jinanshidaxue 济南市大学 control ji'nan / jin'an +xinanjiaotongdaxue 西南交通大学 control xi'nan / xin'an +dananbusi 大难不死 control da'nan / dan'an +womendouhenganxieni 我们都很感谢你 control hen'gan / heng'an +tazhenganjing 他真干净 control zhen'gan / zheng'an +nihao 你好 plain +womenyiqiqu 我们一起去 plain +jintiantianqibucuo 今天天气不错 plain +zhegewentiyinggaizenmejiejue 这个问题应该怎么解决 plain +xianzaijiuqu 现在就去 plain +ganxienidebangzhu 感谢你的帮助 plain +anzhaoguiding 按照规定 plain +tiananmenguangchang 天安门广场 plain +zaijinuta 再激怒他 plain 贪心切分在 jin 上卡死的合法输入;再/在 两可,只盯切分不回退 +fangan 方案 word +danganguan 档案馆 word +changanjie 长安街 word +ziazheliya 在这里呀 typo +jiuzheeyang 就这样 typo diff --git a/engine/tests/src/eval_quanpin_sentence.cpp b/engine/tests/src/eval_quanpin_sentence.cpp new file mode 100644 index 00000000..a6028792 --- /dev/null +++ b/engine/tests/src/eval_quanpin_sentence.cpp @@ -0,0 +1,139 @@ +// 全拼整句级切分评测:逐条敲入 tests/data/quanpin_sentence_cases.tsv 的输入,看首选、第 2 位和 +// 预编辑,并统计每键耗时。 +// +// 不是测试:不断言,也不进 ctest。改切分或整句打分前后各跑一遍对比,例如 +// set METASEQUOIA_IME_DATA_DIR=<数据目录的副本> +// eval_quanpin_sentence engine\tests\data\quanpin_sentence_cases.tsv [repeats] [lattice] +// 只敲键和重置,从不选词,不写用户日志;仍请指向副本,别用正在用的数据目录。 +#include "../../core/data_path.h" +#include "../../core/input_session.h" +#include "../../core/sentence_association_options.h" +#include "../../quanpin/quanpin_utils.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace +{ +struct Case +{ + std::string input; + std::string expected; + std::string category; +}; + +std::vector load_cases(const char *path) +{ + std::vector cases; + std::ifstream file(path, std::ios::binary); + std::string line; + while (std::getline(file, line)) + { + if (!line.empty() && line.back() == '\r') + line.pop_back(); + if (line.empty() || line.front() == '#') + continue; + std::istringstream fields(line); + Case item; + std::getline(fields, item.input, '\t'); + std::getline(fields, item.expected, '\t'); + std::getline(fields, item.category, '\t'); + if (!item.input.empty() && !item.expected.empty()) + cases.push_back(std::move(item)); + } + return cases; +} + +double percentile(std::vector values, double p) +{ + if (values.empty()) + return 0.0; + std::sort(values.begin(), values.end()); + const size_t index = (std::min)(values.size() - 1, static_cast(p * (values.size() - 1) + 0.5)); + return values[index]; +} + +struct Setting +{ + const char *name; + unsigned autocorrect_types; +}; +} // namespace + +int main(int argc, char **argv) +{ + if (argc < 2) + { + std::fprintf(stderr, "usage: eval_quanpin_sentence [repeats] [lattice]\n"); + return 2; + } + const auto cases = load_cases(argv[1]); + const int repeats = argc > 2 ? (std::max)(1, std::atoi(argv[2])) : 3; + + // 与出厂默认一致:词格整句 + 神经速度档 + 万象搭配加分(模型在数据目录里时)。 + // 第 3 个参数写 lattice 时只开词格,用来把神经重排的耗时单独拆出来。 + const bool lattice_only = argc > 3 && std::string(argv[3]) == "lattice"; + SentenceAssociationOptions options; + options.word_lattice = true; + options.neural_keyboard = !lattice_only; + const auto collocation = metasequoia::data_directory() / "models" / "zh-moqi" / "zh-moqi.gram"; + if (std::filesystem::exists(collocation)) + options.collocation_model = metasequoia::path_to_utf8(collocation); + + const unsigned all_types = quanpin::kAutocorrectTransposition | quanpin::kAutocorrectNeighbor | + quanpin::kAutocorrectDeletion | quanpin::kAutocorrectInsertion; + const Setting settings[] = {{"autocorrect off", 0}, {"autocorrect on", all_types}}; + + for (const Setting &setting : settings) + { + metasequoia::InputSession session(SchemeType::Quanpin, setting.autocorrect_types); + session.set_sentence_association(options); + std::printf("== %s ==\n", setting.name); + std::map> by_category; + std::vector per_key; + for (const Case &item : cases) + { + std::string head; + std::string second; + std::string display; + // 第一轮预热,只记后面几轮的耗时;结果每轮都一样,取最后一轮。 + for (int round = -1; round < repeats; ++round) + { + session.reset_state(); + session.reset_cache(); + for (const char character : item.input) + { + const auto start = std::chrono::steady_clock::now(); + session.handle_character(character); + const double ms = + std::chrono::duration(std::chrono::steady_clock::now() - start).count(); + if (round >= 0) + per_key.push_back(ms); + } + const auto &candidates = session.candidates(); + head = candidates.empty() ? std::string{} : candidates[0].word; + second = candidates.size() < 2 ? std::string{} : candidates[1].word; + display = session.get_pinyin_segmentation_with_cases(); + } + const bool pass = head == item.expected; + auto &counts = by_category[item.category]; + counts.first += pass ? 1 : 0; + counts.second += 1; + std::printf("%s %-30s %-10s 1:%s 2:%s [%s]\n", pass ? "PASS" : "FAIL", item.input.c_str(), + item.category.c_str(), head.c_str(), second.c_str(), display.c_str()); + } + for (const auto &[category, counts] : by_category) + std::printf(" %-10s %d/%d\n", category.c_str(), counts.first, counts.second); + std::printf(" ms per key: median %.2f p90 %.2f max %.2f\n\n", percentile(per_key, 0.5), + percentile(per_key, 0.9), percentile(per_key, 1.0)); + session.reset_state(); + } + return 0; +} diff --git a/engine/tests/src/test_input_session.cpp b/engine/tests/src/test_input_session.cpp index 7b870576..0f3e0ef0 100644 --- a/engine/tests/src/test_input_session.cpp +++ b/engine/tests/src/test_input_session.cpp @@ -1246,6 +1246,91 @@ void run_autocorrect_costlier_context_tests(const std::filesystem::path &data_di std::filesystem::remove_all(directory); } +// 合法输入的等音节数字面切分按整句打分争领衔(arbitrate_literal_segmentations)。 +// yibananzhao 的最少音节规则平局时取较短首音节,落在 yi'ba'nan'zhao;fixture 只给 +// yi'ban'an'zhao 一侧词组(一般、按照),另一侧只有单字。词库目录没有 sc.lm,词格退回 +// 启发式打分(单字 ln(w/1e6)、词组 ln(w)+3×音节数),分差完全由权重决定,用例因此确定。 +void run_literal_segmentation_arbitration_tests(const std::filesystem::path &data_directory) +{ + require(quanpin::enumerate_min_segmentations("yibananzhao", 8).size() == 2, + "Both four-syllable readings of yibananzhao must be enumerated."); + require(quanpin::enumerate_min_segmentations("nihao", 8).size() == 1, + "An unambiguous input must have exactly one minimal segmentation."); + + const std::filesystem::path directory = data_directory / "literal-segmentation"; + std::filesystem::create_directories(directory); + { + Database database(directory / "msime.db"); + database.execute("CREATE TABLE tbl_2_y(key TEXT,jp TEXT,value TEXT,weight INTEGER);" + "INSERT INTO tbl_2_y VALUES('yi''ban', 'yb', '一般', 1000);"); + database.execute("CREATE TABLE tbl_2_a(key TEXT,jp TEXT,value TEXT,weight INTEGER);" + "INSERT INTO tbl_2_a VALUES('an''zhao', 'az', '按照', 1000);"); + database.execute("CREATE TABLE tbl_1_y(key TEXT,jp TEXT,value TEXT,weight INTEGER);" + "INSERT INTO tbl_1_y VALUES('yi', 'y', '一', 1000);"); + database.execute("CREATE TABLE tbl_1_b(key TEXT,jp TEXT,value TEXT,weight INTEGER);" + "INSERT INTO tbl_1_b VALUES('ba', 'b', '把', 1000);"); + database.execute("CREATE TABLE tbl_1_n(key TEXT,jp TEXT,value TEXT,weight INTEGER);" + "INSERT INTO tbl_1_n VALUES('nan', 'n', '难', 1000);"); + database.execute("CREATE TABLE tbl_1_z(key TEXT,jp TEXT,value TEXT,weight INTEGER);" + "INSERT INTO tbl_1_z VALUES('zhao', 'z', '找', 1000);"); + // fanganhao:fan'gan'hao(主切)与 fang'an'hao 只差 ln(1000/600)=0.5,主切领衔, + // 另一读法的整句在分差以内,占第 2 位。 + database.execute("CREATE TABLE tbl_2_f(key TEXT,jp TEXT,value TEXT,weight INTEGER);" + "INSERT INTO tbl_2_f VALUES('fan''gan', 'fg', '反感', 1000);" + "INSERT INTO tbl_2_f VALUES('fang''an', 'fa', '方案', 600);"); + database.execute("CREATE TABLE tbl_1_h(key TEXT,jp TEXT,value TEXT,weight INTEGER);" + "INSERT INTO tbl_1_h VALUES('hao', 'h', '好', 1000);"); + } + + metasequoia::RuntimePaths paths; + paths.resources = directory; + paths.user_data = directory; + paths.cache = directory; + paths.dictionaries = directory; + SentenceAssociationOptions lattice_only; + lattice_only.word_lattice = true; + const auto has_word = [](const metasequoia::InputSession &target, const std::string &word) { + const auto &items = target.candidates(); + return std::any_of(items.begin(), items.end(), [&](const WordItem &item) { return item.word == word; }); + }; + + // 另一种切分的整句分高出一截:它领衔,预编辑跟着它分;原切分的整句分差太远,不占第 2 位。 + { + metasequoia::InputSession session(SchemeType::Quanpin, 0, true, true, true, paths); + session.set_sentence_association(lattice_only); + type(session, "yibananzhao"); + require(!session.candidates().empty() && session.candidates().front().word == "一般按照", + "The better-scoring literal segmentation must lead."); + require(session.get_pinyin_segmentation_with_cases() == "yi'ban'an'zhao", + "The preedit must follow the leading segmentation."); + require(!has_word(session, "一把难找"), "A far-behind literal sentence must not be listed."); + (void)session.select_candidate(candidate_index(session, "一般")); + require(session.has_composition() && session.preedit() == "anzhao" && + session.candidates().front().word == "按照", + "Selecting a prefix of the leading segmentation must leave the rest of it."); + } + // 整句来源全关:没有整句可比,与改造前一致,仍按最少音节切分。 + { + metasequoia::InputSession session(SchemeType::Quanpin, 0, true, true, true, paths); + type(session, "yibananzhao"); + require(session.get_pinyin_segmentation_with_cases() == "yi'ba'nan'zhao" && !has_word(session, "一般按照"), + "Without any sentence source the segmentation must not be re-arbitrated."); + } + // 主切领衔、另一读法分差以内:它的整句紧跟在第 2 位。 + { + metasequoia::InputSession session(SchemeType::Quanpin, 0, true, true, true, paths); + session.set_sentence_association(lattice_only); + type(session, "fanganhao"); + const auto &candidates = session.candidates(); + require(candidates.size() >= 2 && candidates[0].word == "反感好" && candidates[1].word == "方案好", + "A close runner-up segmentation must take the second slot."); + require(session.get_pinyin_segmentation_with_cases() == "fan'gan'hao", + "The preedit must stay on the leading segmentation."); + } + + std::filesystem::remove_all(directory); +} + // 多字、漏字纠错的读音和原始字母不等长,选词时要按纠错切分记录的原始区间消耗字母, // 不能按读音长度:buuhui 选「不会」曾剩下一个 i,选「张」会把 zhngg 一起吞掉。 void run_autocorrect_selection_consumption_tests(const std::filesystem::path &data_directory) @@ -2432,6 +2517,7 @@ int run_test() run_autocorrect_context_user_choice_tests(data_directory); run_legal_input_correction_tests(data_directory); run_autocorrect_selection_consumption_tests(data_directory); + run_literal_segmentation_arbitration_tests(data_directory); run_autocorrect_costlier_context_tests(data_directory); run_autocorrect_generated_space_tests(data_directory);