fix(engine): 日语预测紧跟整读音转换,不再被前缀片段压后

完整读音时的候选顺序改为:最佳整句路径与整读音精确词条在前,前缀预测随后,
其余整句结果(次优路径、假名兜底、只覆盖前缀的词条)殿后。预测只取读音严格
长于输入的词条,不再重复扫描精确词条;预测权重不高于其上方的转换,避免按权
重重排时把顺序翻回去。测试补齐头文件、清理目录不再抛异常,并新增前缀片段回归。
This commit is contained in:
fanlusky committed 2026-10-08 16:48:39 +08:00
1 parent 4234f6deb2
commit 9a08042586
4 files changed
+61 -9

No files matched your search

@@ -343,6 +343,24 @@ std::vector<JapaneseLemma> JapaneseSentenceDecoder::PrefixLemmas(const std::stri
return BestLemmas(matches, limit);
}
std::vector<JapaneseLemma> JapaneseSentenceDecoder::LongerPrefixLemmas(const std::string &reading_prefix,
size_t limit) const
{
if (!ready_ || reading_prefix.empty() || limit == 0)
return {};
std::vector<std::uint32_t> matches;
for (size_t index = LowerBoundReading(reading_prefix); index < token_count_; ++index)
{
const auto token_reading = Reading(TokenAt(index));
if (token_reading.size() < reading_prefix.size() ||
token_reading.compare(0, reading_prefix.size(), reading_prefix) != 0)
break;
if (token_reading.size() > reading_prefix.size())
matches.push_back(static_cast<std::uint32_t>(index));
}
return BestLemmas(matches, limit);
}
std::vector<JapaneseLemma> JapaneseSentenceDecoder::PrefixLemmasContinuing(const std::string &reading_prefix,
const std::vector<std::string> &next_kana,
size_t limit) const
@@ -38,6 +38,8 @@ class JapaneseSentenceDecoder
std::vector<SentenceCandidate> Decode(const std::string &reading, size_t limit = 8) const;
std::vector<JapaneseLemma> ExactLemmas(const std::string &reading, size_t limit = 32) const;
std::vector<JapaneseLemma> PrefixLemmas(const std::string &reading_prefix, size_t limit = 32) const;
// Like PrefixLemmas, but only lemmas whose reading is strictly longer than the prefix.
std::vector<JapaneseLemma> LongerPrefixLemmas(const std::string &reading_prefix, size_t limit = 32) const;
std::vector<JapaneseLemma> PrefixLemmasContinuing(const std::string &reading_prefix,
const std::vector<std::string> &next_kana,
size_t limit = 32) const;
@@ -121,19 +121,38 @@ std::vector<WordItem> JapaneseCandidateProvider::query(const QueryRequest &reque
}
}
japanese::JapaneseMatrixSearch search(*sentence_decoder_);
for (const auto &sentence : search.SearchConverted(conversion, 12))
{
AppendUnique(candidates, seen, request.raw_input_with_cases, sentence.text, 900000 - sentence.cost,
CandidateSource::Database);
}
const auto sentences = search.SearchConverted(conversion, 12);
size_t next_sentence = 0;
// Once the reading is complete, convert what was typed before predicting
// longer words. Keep the prediction-first path above for pending romaji.
// longer words: the best whole-reading path and exact lemmas, then
// predictions, then the remaining sentence tail (other paths, kana
// fallbacks and lemmas covering only a prefix of the reading). Keep the
// prediction-first path above for pending romaji.
if (conversion.pending.empty() && conversion.hiragana.size() >= 6)
{
for (const auto &lemma : sentence_decoder_->PrefixLemmas(conversion.hiragana, 16))
AppendUnique(candidates, seen, request.raw_input_with_cases, lemma.surface, 980000 - lemma.word_cost,
std::int64_t weight_floor = 980000;
const auto append_exact = [&](const std::string &text, std::int64_t weight) {
const size_t before = candidates.size();
AppendUnique(candidates, seen, request.raw_input_with_cases, text, weight, CandidateSource::Database);
if (candidates.size() != before)
weight_floor = (std::min)(weight_floor, weight);
};
if (!sentences.empty())
append_exact(sentences[next_sentence++].text, 900000 - sentences.front().cost);
for (const auto &lemma : sentence_decoder_->ExactLemmas(conversion.hiragana, 16))
append_exact(lemma.surface, 900000 - lemma.word_cost);
// Predictions must not outweigh the conversions above them, or a
// weight-based re-sort would undo this order.
for (const auto &lemma : sentence_decoder_->LongerPrefixLemmas(conversion.hiragana, 16))
AppendUnique(candidates, seen, request.raw_input_with_cases, lemma.surface,
(std::min)(weight_floor, 980000 - static_cast<std::int64_t>(lemma.word_cost)),
CandidateSource::Database);
}
for (; next_sentence < sentences.size(); ++next_sentence)
{
AppendUnique(candidates, seen, request.raw_input_with_cases, sentences[next_sentence].text,
900000 - sentences[next_sentence].cost, CandidateSource::Database);
}
}
if (ensure_query_statement())
+14 -1
View File
@@ -5,9 +5,14 @@
#include <sqlite3.h>
#include <algorithm>
#include <chrono>
#include <cstdint>
#include <cstdio>
#include <filesystem>
#include <fstream>
#include <stdexcept>
#include <string>
#include <system_error>
#include <vector>
namespace
{
@@ -112,6 +117,13 @@ void run_test(const std::filesystem::path &root)
const auto sentence = query(provider, "watashiha");
require(position(sentence, "私は") < position(sentence, "私は走る"),
"Prediction displaced a complete reading spanning multiple lemmas.");
// Lemmas covering only a prefix of the reading (川, 蚊) are not conversions of
// what was typed, so they must not push the whole-word prediction down.
const auto kawai = query(provider, "kawai");
require(position(kawai, "可愛い") < position(kawai, "川"), "Prefix fragment 川 outranked a prediction.");
require(position(kawai, "可愛い") < position(kawai, "蚊"), "Prefix fragment 蚊 outranked a prediction.");
require(hashi[position(hashi, "走る")].weight <= hashi[position(hashi, "箸")].weight,
"A prediction outweighs the exact conversion ranked above it.");
const auto pending = query(provider, "kaw");
require(!pending.empty() && pending.front().word == "可愛い",
@@ -147,6 +159,7 @@ int main()
std::fprintf(stderr, "%s\n", error.what());
result = 1;
}
std::filesystem::remove_all(root);
std::error_code ignored;
std::filesystem::remove_all(root, ignored);
return result;
}