diff --git a/engine/japanese/japanese_sentence_decoder.cpp b/engine/japanese/japanese_sentence_decoder.cpp index 1f9a367a..6e762b5a 100644 --- a/engine/japanese/japanese_sentence_decoder.cpp +++ b/engine/japanese/japanese_sentence_decoder.cpp @@ -343,6 +343,24 @@ std::vector JapaneseSentenceDecoder::PrefixLemmas(const std::stri return BestLemmas(matches, limit); } +std::vector JapaneseSentenceDecoder::LongerPrefixLemmas(const std::string &reading_prefix, + size_t limit) const +{ + if (!ready_ || reading_prefix.empty() || limit == 0) + return {}; + std::vector matches; + for (size_t index = LowerBoundReading(reading_prefix); index < token_count_; ++index) + { + const auto token_reading = Reading(TokenAt(index)); + if (token_reading.size() < reading_prefix.size() || + token_reading.compare(0, reading_prefix.size(), reading_prefix) != 0) + break; + if (token_reading.size() > reading_prefix.size()) + matches.push_back(static_cast(index)); + } + return BestLemmas(matches, limit); +} + std::vector JapaneseSentenceDecoder::PrefixLemmasContinuing(const std::string &reading_prefix, const std::vector &next_kana, size_t limit) const diff --git a/engine/japanese/japanese_sentence_decoder.h b/engine/japanese/japanese_sentence_decoder.h index f732abb9..4ea564f6 100644 --- a/engine/japanese/japanese_sentence_decoder.h +++ b/engine/japanese/japanese_sentence_decoder.h @@ -38,6 +38,8 @@ class JapaneseSentenceDecoder std::vector Decode(const std::string &reading, size_t limit = 8) const; std::vector ExactLemmas(const std::string &reading, size_t limit = 32) const; std::vector PrefixLemmas(const std::string &reading_prefix, size_t limit = 32) const; + // Like PrefixLemmas, but only lemmas whose reading is strictly longer than the prefix. + std::vector LongerPrefixLemmas(const std::string &reading_prefix, size_t limit = 32) const; std::vector PrefixLemmasContinuing(const std::string &reading_prefix, const std::vector &next_kana, size_t limit = 32) const; diff --git a/engine/providers/japanese_candidate_provider.cpp b/engine/providers/japanese_candidate_provider.cpp index b8000b72..32f57c99 100644 --- a/engine/providers/japanese_candidate_provider.cpp +++ b/engine/providers/japanese_candidate_provider.cpp @@ -121,19 +121,38 @@ std::vector JapaneseCandidateProvider::query(const QueryRequest &reque } } japanese::JapaneseMatrixSearch search(*sentence_decoder_); - for (const auto &sentence : search.SearchConverted(conversion, 12)) - { - AppendUnique(candidates, seen, request.raw_input_with_cases, sentence.text, 900000 - sentence.cost, - CandidateSource::Database); - } + const auto sentences = search.SearchConverted(conversion, 12); + size_t next_sentence = 0; // Once the reading is complete, convert what was typed before predicting - // longer words. Keep the prediction-first path above for pending romaji. + // longer words: the best whole-reading path and exact lemmas, then + // predictions, then the remaining sentence tail (other paths, kana + // fallbacks and lemmas covering only a prefix of the reading). Keep the + // prediction-first path above for pending romaji. if (conversion.pending.empty() && conversion.hiragana.size() >= 6) { - for (const auto &lemma : sentence_decoder_->PrefixLemmas(conversion.hiragana, 16)) - AppendUnique(candidates, seen, request.raw_input_with_cases, lemma.surface, 980000 - lemma.word_cost, + std::int64_t weight_floor = 980000; + const auto append_exact = [&](const std::string &text, std::int64_t weight) { + const size_t before = candidates.size(); + AppendUnique(candidates, seen, request.raw_input_with_cases, text, weight, CandidateSource::Database); + if (candidates.size() != before) + weight_floor = (std::min)(weight_floor, weight); + }; + if (!sentences.empty()) + append_exact(sentences[next_sentence++].text, 900000 - sentences.front().cost); + for (const auto &lemma : sentence_decoder_->ExactLemmas(conversion.hiragana, 16)) + append_exact(lemma.surface, 900000 - lemma.word_cost); + // Predictions must not outweigh the conversions above them, or a + // weight-based re-sort would undo this order. + for (const auto &lemma : sentence_decoder_->LongerPrefixLemmas(conversion.hiragana, 16)) + AppendUnique(candidates, seen, request.raw_input_with_cases, lemma.surface, + (std::min)(weight_floor, 980000 - static_cast(lemma.word_cost)), CandidateSource::Database); } + for (; next_sentence < sentences.size(); ++next_sentence) + { + AppendUnique(candidates, seen, request.raw_input_with_cases, sentences[next_sentence].text, + 900000 - sentences[next_sentence].cost, CandidateSource::Database); + } } if (ensure_query_statement()) diff --git a/engine/tests/src/test_japanese_candidates.cpp b/engine/tests/src/test_japanese_candidates.cpp index ca597eca..8e1cc725 100644 --- a/engine/tests/src/test_japanese_candidates.cpp +++ b/engine/tests/src/test_japanese_candidates.cpp @@ -5,9 +5,14 @@ #include #include #include +#include #include +#include #include #include +#include +#include +#include namespace { @@ -112,6 +117,13 @@ void run_test(const std::filesystem::path &root) const auto sentence = query(provider, "watashiha"); require(position(sentence, "私は") < position(sentence, "私は走る"), "Prediction displaced a complete reading spanning multiple lemmas."); + // Lemmas covering only a prefix of the reading (川, 蚊) are not conversions of + // what was typed, so they must not push the whole-word prediction down. + const auto kawai = query(provider, "kawai"); + require(position(kawai, "可愛い") < position(kawai, "川"), "Prefix fragment 川 outranked a prediction."); + require(position(kawai, "可愛い") < position(kawai, "蚊"), "Prefix fragment 蚊 outranked a prediction."); + require(hashi[position(hashi, "走る")].weight <= hashi[position(hashi, "箸")].weight, + "A prediction outweighs the exact conversion ranked above it."); const auto pending = query(provider, "kaw"); require(!pending.empty() && pending.front().word == "可愛い", @@ -147,6 +159,7 @@ int main() std::fprintf(stderr, "%s\n", error.what()); result = 1; } - std::filesystem::remove_all(root); + std::error_code ignored; + std::filesystem::remove_all(root, ignored); return result; }