#include "direct_resolver.h" #include "../quanpin/quanpin_query.h" #include "../shuangpin/shuangpin_query.h" #include #include namespace direct_helpcode { namespace { // 缓存条目上限:超过就整体清掉,免得一次长时间输入把内存撑大。 constexpr std::size_t kSpanCacheLimit = 8192; constexpr std::size_t kResolutionCacheLimit = 256; constexpr std::size_t kCollocationMemoLimit = 65536; std::string lowercase(std::string text) { for (char &ch : text) ch = static_cast(std::tolower(static_cast(ch))); return text; } bool is_single_utf8_char(const std::string &text) { if (text.empty()) return false; const auto lead = static_cast(text[0]); const std::size_t len = lead < 0x80 ? 1 : (lead >> 5) == 0x6 ? 2 : (lead >> 4) == 0xE ? 3 : 4; return text.size() == len; } bool accepts_char(const HelpcodeUtils::Keymap *keymap, const std::string &hanzi, char first, char second) { if (keymap == nullptr) return false; const auto found = keymap->find(hanzi); if (found == keymap->end()) return false; SyllableHelpcode helpcode; helpcode.first = first; helpcode.second = second; return syllable_helpcode_matches(helpcode, found->second); } // 两个相邻音节之间要不要放一个手动分隔符:前一个音节后面挂着辅码、任一个是单键声母、或原串里 // 两者之间本来就隔着别的字符时都要放。其余情况是连续的两键音节,贪心切分自己就切得出来,不放 // 分隔符正是为了让它们和普通双拼走完全相同的路径。 bool needs_delimiter(const SyllableSpelling &previous, const SyllableSpelling ¤t) { return previous.end != previous.begin + previous.code.size() || previous.kind == SpellingKind::Initial || current.kind == SpellingKind::Initial || current.begin != previous.end; } } // namespace void apply_spellings(QueryRequest &request, const std::string &typed, const std::vector &spellings, const ShuangpinProfile &profile) { std::string clean; std::vector source_index; std::vector> decorations; SyllableHelpcodes helpcodes; for (std::size_t i = 0; i < spellings.size(); ++i) { const auto &spelling = spellings[i]; if (i > 0 && needs_delimiter(spellings[i - 1], spelling)) { const auto &previous = spellings[i - 1]; clean.push_back('\''); // 和句中辅助码一致:分隔符对应前一个音节辅码段的起点。 source_index.push_back(previous.begin + previous.code.size()); } for (std::size_t j = 0; j < spelling.code.size(); ++j) { clean.push_back(spelling.code[j]); source_index.push_back(spelling.begin + j); } const std::size_t code_end = spelling.begin + spelling.code.size(); if (spelling.end > code_end) decorations.emplace_back(i, typed.substr(code_end, spelling.end - code_end)); if (spelling.has_aux()) { SyllableHelpcode helpcode; helpcode.syllable = i; helpcode.first = spelling.first; helpcode.second = spelling.second; helpcodes.push_back(helpcode); } } source_index.push_back(typed.size()); request.raw_input_with_syllable_helpcodes = typed; request.raw_input = clean; request.raw_input_with_cases = clean; request.syllable_helpcodes = std::move(helpcodes); request.enable_mid_sentence_helpcode = true; request.enable_shuangpin_helpcode = false; request.direct_helpcode = true; request.direct_helpcode_source_index = std::move(source_index); request.direct_helpcode_decorations = std::move(decorations); request.valid = !clean.empty(); if (!request.valid) return; const std::string raw_segmentation = shuangpin::segment_input(clean, profile); request.raw_segmentation = raw_segmentation; request.normalized_segmentation = shuangpin::to_quanpin_segmentation(raw_segmentation, profile); request.segmentation = request.normalized_segmentation; request.normalized_input = shuangpin::normalize_input(clean, profile); } Resolver::Resolver(const ShuangpinProfile &profile) : profile_(profile) { } void Resolver::reset_cache() { span_cache_.clear(); word_memo_.clear(); collocation_memo_.clear(); resolution_cache_.clear(); last_sentence_ = nullptr; } void Resolver::build_aux_index(const ResolveContext &context) { aux_index_built_ = true; if (!context.single_char_rows || context.keymap == nullptr) return; // 一次扫完词库的单字行,对应万象词库里为每个字派生的 拼音;辅码 拼写。比按音节逐个查(每个音节一条 // 4096 行的查询)便宜得多,也不会把第一次碰到某个音节的那一键拖慢。 for (const auto &[quanpin, hanzi] : context.single_char_rows()) { const auto code = context.keymap->find(hanzi); if (code == context.keymap->end() || code->second.empty()) continue; auto &aux = aux_index_[quanpin]; const char lead = code->second[0]; if (lead >= 'a' && lead <= 'z') aux.first[static_cast(lead - 'a')] = true; if (code->second.size() > 1) aux.pairs.insert(code->second.substr(0, 2)); } } const std::vector &Resolver::span_rows(const quanpin::Segments &span, bool constrained, const ResolveContext &context) { std::string key = quanpin::join_segments(span); if (constrained) key.push_back('#'); auto found = span_cache_.find(key); if (found != span_cache_.end()) return found->second; if (span_cache_.size() >= kSpanCacheLimit) span_cache_.clear(); auto rows = context.lookup ? context.lookup(span, constrained) : std::vector{}; return span_cache_.emplace(std::move(key), std::move(rows)).first->second; } bool Resolver::has_aux(const std::string &quanpin, char first, char second, const ResolveContext &context) { if (context.keymap == nullptr || first < 'a' || first > 'z') return false; if (!aux_index_built_) build_aux_index(context); auto found = aux_index_.find(quanpin); if (found == aux_index_.end()) { // 全量扫描里没有这个读音(没有全量扫描,或词库键的写法不同,如 ü):退回按音节查一次,结果同样 // 记下,之后不再查。 SyllableAux aux; for (const auto &row : span_rows({quanpin}, true, context)) { if (!is_single_utf8_char(row.value)) continue; const auto code = context.keymap->find(row.value); if (code == context.keymap->end() || code->second.empty()) continue; const char lead = code->second[0]; if (lead >= 'a' && lead <= 'z') aux.first[static_cast(lead - 'a')] = true; if (code->second.size() > 1) aux.pairs.insert(code->second.substr(0, 2)); } found = aux_index_.emplace(quanpin, std::move(aux)).first; } if (second == 0) return found->second.first[static_cast(first - 'a')]; return found->second.pairs.count(std::string{first, second}) > 0; } bool Resolver::resolve(QueryRequest &request, const ResolveContext &context) { const std::string typed = request.raw_input_with_cases.empty() ? request.raw_input : request.raw_input_with_cases; if (typed.empty()) return false; last_sentence_ = nullptr; if (context.keymap != indexed_keymap_) { reset_cache(); aux_index_.clear(); aux_index_built_ = false; indexed_keymap_ = context.keymap; } auto cached = resolution_cache_.find(typed); if (cached == resolution_cache_.end()) { const auto graph = build_spelling_graph(typed, profile_, [&](const std::string &quanpin, char first, char second) { return has_aux(quanpin, first, second, context); }); Resolution resolution; // 只有一条完整切分时没有可比的切法,跳过整句解码;整句留给词典层自己的词格去解。 if (!graph.empty() && !single_path(graph, resolution.spellings)) { const SpanLookup lookup = [&](const quanpin::Segments &span, bool constrained) { return span_rows(span, constrained, context); }; const CharAccept accept = [&](const std::string &hanzi, char first, char second) { return accepts_char(context.keymap, hanzi, first, second); }; if (word_memo_.size() >= kSpanCacheLimit) word_memo_.clear(); // 搭配分只取决于(尾窗、词、是否句尾),是纯函数:记下来跨假设、跨按键复用,结果与不缓存完全一致。 // 解析器一次要比较所有切法,边数远多于单一切分的词格,不缓存时 .gram 查询是解码的大头。 DecodeOptions options = context.options; if (options.collocation_scorer && options.collocation_weight > 0.0) { options.collocation_scorer = [this, &context](std::string_view tail, std::string_view word, bool is_rear) { // 复用同一个键缓冲:命中时不分配,只有未命中插入时才拷一份。 std::string &key = collocation_key_; key.clear(); key.append(tail).push_back('\x1f'); key.append(word).push_back(is_rear ? '1' : '0'); auto found = collocation_memo_.find(key); if (found != collocation_memo_.end()) return found->second; if (collocation_memo_.size() >= kCollocationMemoLimit) collocation_memo_.clear(); const double score = context.options.collocation_scorer(tail, word, is_rear); collocation_memo_.emplace(key, score); return score; }; } if (auto path = decode_best_path(graph, lookup, accept, options, &word_memo_)) { resolution.spellings = path->syllables; if (path->complete && !path->sentence.empty()) resolution.sentence = std::move(path); } } if (resolution_cache_.size() >= kResolutionCacheLimit) resolution_cache_.clear(); cached = resolution_cache_.emplace(typed, std::move(resolution)).first; } const auto &spellings = cached->second.spellings; if (spellings.empty()) return false; if (cached->second.sentence) last_sentence_ = &*cached->second.sentence; // 没有辅码、也没有吃掉分隔符以外的字符时,选中的切分就是普通双拼的切分:请求原样放行, // 不带辅码的输入和关着开关时走完全一样的路。 const bool plain = std::none_of(spellings.begin(), spellings.end(), [](const SyllableSpelling &spelling) { return spelling.end != spelling.begin + spelling.code.size(); }); if (plain) { std::string joined; for (std::size_t i = 0; i < spellings.size(); ++i) { if (i > 0) joined.push_back('\''); joined += spellings[i].code; } if (joined == shuangpin::segment_input(lowercase(typed), profile_)) return false; } apply_spellings(request, typed, spellings, profile_); return true; } } // namespace direct_helpcode