Files
msime-windows/engine/quanpin/engine.cpp
T
Jia Sui 3c2f3ae374 feat(engine): 全拼纠错补齐漏字/多字,k-best 切分按词频消解歧义
此前纠错只有交换(sahng -> shang)与 QWERTY 邻键(shabg -> shang)两类,
生成器在离线阶段就把「一个错串对应多个音节」的变体整个丢掉,运行时也只
取代价最低的一条切分。三类输入因此没有出路:漏字的(shng)、多敲一键的
(shangg、sjhang)、漏声母连基础切分都切不出来的(uanli 想打 quan'li)。

本次按专利 CN 101133411 B 的思路重排整条链路:错串可能对应的读音全部
保留到运行时,先由 k-best 切分带出平行读音,再让词典词频决定谁排前面。

- 纠错类型从两类扩到四类:新增漏字(shng -> shang)与多字(shangg 双打、
  sjhang 邻键 -> shang)。两个新位跟随既有 autocorrect_transposition /
  autocorrect_neighbor 开关联动(任一打开即生效),定义保持独立,留待
  将来的独立配置键。查询与 preedit 重建共用同一映射,两侧纠正空间一致。
- 生成器从两表扩到四表并保留歧义变体(ahan -> {shan, zhan});同一条
  (错串, 正确) 对被两条规则产出时在两张表各留一份,保证每个类型开关
  单独有效。正确音节收敛进 kCorrectSyllables 单表、行内改存 16-bit
  下标,一万七千多行不再各自重复音节字符串,--check 拦截过期表头。
- 切分层新增 autocorrect_cut_kbest:按位置自左向右做 top-k 假设传播,
  排序键是(纠错边数,边权之和,生成序)——最少纠错优先的契约不变,
  权重只在同边数内打破平局,引擎绝不用一条额外纠错换更低权重。边权取
  专利 [0052] 允许的估常量(交换 10 / 漏字 11 / 多字 12 / 邻键 13),
  标定路径是挖掘用户日志。漏字/多字边不等长,切分段改按 raw 长度推进
  位置,preedit 分隔跟着原始字母的区间走。
- 词典层固定 k=9,为离线评测定值:漏声母键 uan 有 10-15 个等权漏字
  目标,k=3 按表序截断够不到 quan'li 的正确读音;k=9 提升 R@1/R@3
  且 p95 不变。主切分仍是查询键,其余读音交给
  merge_alternative_segmentations 与主读法在词频下竞争;备选读音在
  缓存命中路径同样发布,corrected_from 标记对全部纠错读音生效;纠错
  读音排在音形别名读音之前。结尾单插入(shangg = shang + g)天然是
  简拼尾形状,仍被简拼形守卫拦下——多字纠错的可达面在中间/开头插入
  与多音节输入。
- 纠错门放宽:!segments.empty() 从门里去掉。BFS 工作在原始字母上,
  基础切分为空的丢声母输入恰是最需要纠错的一批,jian -> ian 现在能
  走到纠错候选并按词频合并到首位。简拼形守卫保留:合法音节加至多一个
  尾字母(zheg)是用户意图,任何纠错表都不得改写。
- preedit 补两道防线:zheg 在显式三位掩码下仍走简拼守卫,不被漏字表
  重排为 zheng;只有 BFS 切分的读音与 scheme 别名层完全一致时才按
  BFS 位置重画分隔符,否则 sahng 会被切成 sa'hng 而候选实际来自
  别名读音 shang。

新增独立评测工具 eval_quanpin_autocorrect:从真实 msime.db 采样高频
词条,按交换/邻键/漏字/多字模型注错,测 R@1/R@3、读音还原率与温态
p50/p95,固定种子可复现。它是评测工具不是测试,永远不进 CI,作为
纠错阈值与门逻辑改动的零回归标尺;注错规则与生成器逐键一致,改任一
侧须同一提交同步另一侧。

表格测试改为按 (错串, 正确) 对断言唯一并加覆盖门(合计 >= 4700 条),
新增歧义保留、漏字/多字变体、k-best 排序与权重定序、preedit 守卫等
用例;engine/tests 的 README 与 AGENTS.md 同步写清 imetest 与评测
工具两个产物的边界。
2026-09-13 11:04:34 +08:00

148 lines
5.8 KiB
C++

#include "engine.h"
#include "../common/helpcode_utils.h"
#include "quanpin_query.h"
#include "quanpin_utils.h"
namespace
{ // The request carries one bool per correction type; the dictionary layer gates
// the whole feature with a single mask, so bridge the two here. Design D2: any
// correction switch on also enables the deletion bit -- the user's intent to
// have typos fixed covers dropped letters -- while both off stays 0. The same
// linkage carries the insertion bit (design D4 of the insertion task).
unsigned autocorrect_types_from_request(const QueryRequest &request)
{
const unsigned legacy =
(request.enable_quanpin_autocorrect_transposition ? quanpin::kAutocorrectTransposition : 0u) |
(request.enable_quanpin_autocorrect_neighbor ? quanpin::kAutocorrectNeighbor : 0u);
return legacy == 0 ? 0u : legacy | quanpin::kAutocorrectDeletion | quanpin::kAutocorrectInsertion;
}
} // namespace
QuanpinEngine::QuanpinEngine(metasequoia::RuntimePaths paths)
: dictionary_({}, paths),
helpcodes_(HelpcodeUtils::load_helpcode_keymap(paths.resources, HelpcodeUtils::selected_helpcode_schema()))
{
}
QuanpinEngine::~QuanpinEngine() = default;
std::optional<WordItem> QuanpinEngine::find_candidate(const std::string &key, const std::string &value)
{
return dictionary_.find_candidate(key, value);
}
std::vector<WordItem> QuanpinEngine::query(const QueryRequest &request)
{
if (!request.valid)
{
return {};
}
const size_t helpcode_length =
request.enable_quanpin_helpcode
? quanpin::detect_active_helpcode_length(request.raw_input, request.raw_input_with_cases)
: 0;
if (helpcode_length == 2)
{
const std::string base_raw_input =
quanpin::strip_active_helpcodes(request.raw_input, request.raw_input_with_cases);
const auto cuts = quanpin::cut_pinyin_by_mode(base_raw_input, "correction");
const std::string base_segmentation = quanpin::join_segments(cuts.front());
const std::string help_codes = request.raw_input.substr(request.raw_input.size() - 2, 2);
const auto base_candidates = dictionary_.query(base_raw_input, base_segmentation,
autocorrect_types_from_request(request), request.fuzzy_pinyin);
return HelpcodeUtils::filter_candidates_with_double_helpcodes(base_candidates, help_codes, helpcodes_.get());
}
if (helpcode_length == 1)
{
const std::string base_raw_input =
quanpin::strip_active_helpcodes(request.raw_input, request.raw_input_with_cases);
const auto cuts = quanpin::cut_pinyin_by_mode(base_raw_input, "correction");
const std::string base_segmentation = quanpin::join_segments(cuts.front());
const std::string help_code = request.raw_input.substr(request.raw_input.size() - 1, 1);
const auto base_candidates = dictionary_.query(base_raw_input, base_segmentation,
autocorrect_types_from_request(request), request.fuzzy_pinyin);
return HelpcodeUtils::reorder_candidates_with_single_helpcode(base_candidates, help_code, helpcodes_.get());
}
return dictionary_.query(request.raw_input, request.segmentation, autocorrect_types_from_request(request),
request.fuzzy_pinyin);
}
bool QuanpinEngine::expand_initial_candidates(const QueryRequest &request, std::vector<WordItem> &candidates)
{
if (request.scheme != SchemeType::Quanpin ||
(request.enable_quanpin_helpcode &&
quanpin::detect_active_helpcode_length(request.raw_input, request.raw_input_with_cases) > 0))
{
return false;
}
const auto segments = quanpin::split_segments(request.segmentation);
if (segments.empty() || segments.front().size() != 1)
{
return false;
}
return dictionary_.expand_initial_candidates(segments.front(), candidates);
}
int QuanpinEngine::handleVkCode(ImeKeyCode vk, ImeModifierMask modifiers_down, ImeCharacter wch)
{
return dictionary_.handleVkCode(vk, modifiers_down, wch);
}
int QuanpinEngine::create_word(std::string pinyin, std::string word)
{
return dictionary_.create_word(std::move(pinyin), std::move(word));
}
int QuanpinEngine::create_word_from_canonical_pinyin(std::string pinyin, std::string word)
{
return dictionary_.create_word_from_canonical_pinyin(std::move(pinyin), std::move(word));
}
int QuanpinEngine::update_weight_by_word(std::string word)
{
return dictionary_.update_weight_by_word(std::move(word));
}
int QuanpinEngine::update_weight_by_pinyin_and_word(std::string pinyin, std::string word)
{
return dictionary_.update_weight_by_pinyin_and_word(std::move(pinyin), std::move(word));
}
int QuanpinEngine::delete_by_pinyin_and_word(std::string pinyin, std::string word)
{
return dictionary_.delete_by_pinyin_and_word(std::move(pinyin), std::move(word));
}
int QuanpinEngine::insert_word_to_series_cache(const std::string &pinyin, const std::string &word,
CandidateSource source)
{
return dictionary_.insert_word_to_series_cache(pinyin, word, source);
}
int QuanpinEngine::insert_word_to_series_cache(const QueryRequest &request, const std::string &word,
CandidateSource source)
{
return dictionary_.insert_word_to_series_cache(request.raw_input, request.segmentation,
autocorrect_types_from_request(request), word, source);
}
std::string QuanpinEngine::search_sentence_from_ime_engine(const std::string &user_pinyin)
{
return dictionary_.search_sentence_from_ime_engine(user_pinyin);
}
void QuanpinEngine::reset_state()
{
dictionary_.reset_state();
}
void QuanpinEngine::reset_cache()
{
dictionary_.reset_cache();
}