Files
msime-windows/engine/quanpin/engine.cpp
T
fanlusky e365a8d1b5 perf(engine): 大模型改为只读映射并按需载入,缩短每键查询路径
- sc.lm 改用 kenlm 的 LAZY 载入:Windows 上默认的 POPULATE_OR_READ 会退成 READ,
  把 33 MB 整份拷进私有堆且永不释放;现在是 MapViewOfFile 只读映射,可被系统回收
- dict_japanese.dat 在 Windows 上同样改为只读映射,与 POSIX 分支一致
- 神经整句模型改为开关首次生效时才载入(LazySentenceModel),默认关闭的桌面模型
  不再占用约 24 MB
- 新增 reset_sentence_cache():重排结果与上文只影响词格整句,只清 series 一层,
  逐前缀查库与切分缓存保持热;set_rescoring_context 也改用它
- CircularBuffer 新增 find():命中时返回指针而不复制整份候选列表,并改为真正的 LRU;
  全拼、双拼每键的命中路径改用它
- Google 解码器去掉每次调用前的 im_reset_search,沿用上游的公共前缀增量搜索
- 测试:出货 sc.lm 能映射载入并正常打分;增量搜索与从头搜结果逐步一致
2026-09-30 19:58:18 +08:00

158 lines
6.2 KiB
C++

#include "engine.h"
#include "../common/helpcode_utils.h"
#include "quanpin_query.h"
#include "quanpin_utils.h"
namespace
{ // The request carries one bool per correction type; the dictionary layer gates
// the whole feature with a single mask, so bridge the two here. Design D2: any
// correction switch on also enables the deletion bit -- the user's intent to
// have typos fixed covers dropped letters -- while both off stays 0. The same
// linkage carries the insertion bit (design D4 of the insertion task).
unsigned autocorrect_types_from_request(const QueryRequest &request)
{
const unsigned legacy =
(request.enable_quanpin_autocorrect_transposition ? quanpin::kAutocorrectTransposition : 0u) |
(request.enable_quanpin_autocorrect_neighbor ? quanpin::kAutocorrectNeighbor : 0u);
return legacy == 0 ? 0u : legacy | quanpin::kAutocorrectDeletion | quanpin::kAutocorrectInsertion;
}
} // namespace
QuanpinEngine::QuanpinEngine(metasequoia::RuntimePaths paths)
: dictionary_({}, paths),
helpcodes_(HelpcodeUtils::load_helpcode_keymap(paths.resources, HelpcodeUtils::selected_helpcode_schema()))
{
}
QuanpinEngine::~QuanpinEngine() = default;
std::optional<WordItem> QuanpinEngine::find_candidate(const std::string &key, const std::string &value)
{
return dictionary_.find_candidate(key, value);
}
std::vector<WordItem> QuanpinEngine::query(const QueryRequest &request)
{
if (!request.valid)
{
return {};
}
// 整句联想开关随请求下发到词典层,四条整句来源据此各自门控(见 query_series)。
dictionary_.set_sentence_association(request.sentence_association);
// 前文同理随请求下发;换了前文,按旧前文重排出来的缓存顺序作废。
dictionary_.set_rescoring_context(request.rescoring_context);
const size_t helpcode_length =
request.enable_quanpin_helpcode
? quanpin::detect_active_helpcode_length(request.raw_input, request.raw_input_with_cases)
: 0;
if (helpcode_length == 2)
{
const std::string base_raw_input =
quanpin::strip_active_helpcodes(request.raw_input, request.raw_input_with_cases);
const auto cuts = quanpin::cut_pinyin_by_mode(base_raw_input, "correction");
const std::string base_segmentation = quanpin::join_segments(cuts.front());
const std::string help_codes = request.raw_input.substr(request.raw_input.size() - 2, 2);
const auto base_candidates = dictionary_.query(base_raw_input, base_segmentation,
autocorrect_types_from_request(request), request.fuzzy_pinyin);
return HelpcodeUtils::filter_candidates_with_double_helpcodes(base_candidates, help_codes, helpcodes_.get());
}
if (helpcode_length == 1)
{
const std::string base_raw_input =
quanpin::strip_active_helpcodes(request.raw_input, request.raw_input_with_cases);
const auto cuts = quanpin::cut_pinyin_by_mode(base_raw_input, "correction");
const std::string base_segmentation = quanpin::join_segments(cuts.front());
const std::string help_code = request.raw_input.substr(request.raw_input.size() - 1, 1);
const auto base_candidates = dictionary_.query(base_raw_input, base_segmentation,
autocorrect_types_from_request(request), request.fuzzy_pinyin);
return HelpcodeUtils::reorder_candidates_with_single_helpcode(base_candidates, help_code, helpcodes_.get());
}
return dictionary_.query(request.raw_input, request.segmentation, autocorrect_types_from_request(request),
request.fuzzy_pinyin);
}
bool QuanpinEngine::expand_initial_candidates(const QueryRequest &request, std::vector<WordItem> &candidates)
{
if (request.scheme != SchemeType::Quanpin ||
(request.enable_quanpin_helpcode &&
quanpin::detect_active_helpcode_length(request.raw_input, request.raw_input_with_cases) > 0))
{
return false;
}
const auto segments = quanpin::split_segments(request.segmentation);
if (segments.empty() || segments.front().size() != 1)
{
return false;
}
return dictionary_.expand_initial_candidates(segments.front(), candidates);
}
int QuanpinEngine::handleVkCode(ImeKeyCode vk, ImeModifierMask modifiers_down, ImeCharacter wch)
{
return dictionary_.handleVkCode(vk, modifiers_down, wch);
}
int QuanpinEngine::create_word(std::string pinyin, std::string word)
{
return dictionary_.create_word(std::move(pinyin), std::move(word));
}
int QuanpinEngine::create_word_from_canonical_pinyin(std::string pinyin, std::string word)
{
return dictionary_.create_word_from_canonical_pinyin(std::move(pinyin), std::move(word));
}
int QuanpinEngine::update_weight_by_word(std::string word)
{
return dictionary_.update_weight_by_word(std::move(word));
}
int QuanpinEngine::update_weight_by_pinyin_and_word(std::string pinyin, std::string word)
{
return dictionary_.update_weight_by_pinyin_and_word(std::move(pinyin), std::move(word));
}
int QuanpinEngine::delete_by_pinyin_and_word(std::string pinyin, std::string word)
{
return dictionary_.delete_by_pinyin_and_word(std::move(pinyin), std::move(word));
}
int QuanpinEngine::insert_word_to_series_cache(const std::string &pinyin, const std::string &word,
CandidateSource source)
{
return dictionary_.insert_word_to_series_cache(pinyin, word, source);
}
int QuanpinEngine::insert_word_to_series_cache(const QueryRequest &request, const std::string &word,
CandidateSource source)
{
return dictionary_.insert_word_to_series_cache(request.raw_input, request.segmentation,
autocorrect_types_from_request(request), word, source);
}
std::string QuanpinEngine::search_sentence_from_ime_engine(const std::string &user_pinyin)
{
return dictionary_.search_sentence_from_ime_engine(user_pinyin);
}
void QuanpinEngine::reset_state()
{
dictionary_.reset_state();
}
void QuanpinEngine::reset_cache()
{
dictionary_.reset_cache();
}
void QuanpinEngine::reset_sentence_cache()
{
dictionary_.reset_sentence_cache();
}