fix(engine): 日语预测紧跟整读音转换,不再被前缀片段压后
完整读音时的候选顺序改为:最佳整句路径与整读音精确词条在前,前缀预测随后, 其余整句结果(次优路径、假名兜底、只覆盖前缀的词条)殿后。预测只取读音严格 长于输入的词条,不再重复扫描精确词条;预测权重不高于其上方的转换,避免按权 重重排时把顺序翻回去。测试补齐头文件、清理目录不再抛异常,并新增前缀片段回归。
This commit is contained in:
1 parent
4234f6deb2
commit
9a08042586
4 files changed
+61
-9
No files matched your search
@@ -343,6 +343,24 @@ std::vector<JapaneseLemma> JapaneseSentenceDecoder::PrefixLemmas(const std::stri
|
||||
return BestLemmas(matches, limit);
|
||||
}
|
||||
|
||||
std::vector<JapaneseLemma> JapaneseSentenceDecoder::LongerPrefixLemmas(const std::string &reading_prefix,
|
||||
size_t limit) const
|
||||
{
|
||||
if (!ready_ || reading_prefix.empty() || limit == 0)
|
||||
return {};
|
||||
std::vector<std::uint32_t> matches;
|
||||
for (size_t index = LowerBoundReading(reading_prefix); index < token_count_; ++index)
|
||||
{
|
||||
const auto token_reading = Reading(TokenAt(index));
|
||||
if (token_reading.size() < reading_prefix.size() ||
|
||||
token_reading.compare(0, reading_prefix.size(), reading_prefix) != 0)
|
||||
break;
|
||||
if (token_reading.size() > reading_prefix.size())
|
||||
matches.push_back(static_cast<std::uint32_t>(index));
|
||||
}
|
||||
return BestLemmas(matches, limit);
|
||||
}
|
||||
|
||||
std::vector<JapaneseLemma> JapaneseSentenceDecoder::PrefixLemmasContinuing(const std::string &reading_prefix,
|
||||
const std::vector<std::string> &next_kana,
|
||||
size_t limit) const
|
||||
|
||||
@@ -38,6 +38,8 @@ class JapaneseSentenceDecoder
|
||||
std::vector<SentenceCandidate> Decode(const std::string &reading, size_t limit = 8) const;
|
||||
std::vector<JapaneseLemma> ExactLemmas(const std::string &reading, size_t limit = 32) const;
|
||||
std::vector<JapaneseLemma> PrefixLemmas(const std::string &reading_prefix, size_t limit = 32) const;
|
||||
// Like PrefixLemmas, but only lemmas whose reading is strictly longer than the prefix.
|
||||
std::vector<JapaneseLemma> LongerPrefixLemmas(const std::string &reading_prefix, size_t limit = 32) const;
|
||||
std::vector<JapaneseLemma> PrefixLemmasContinuing(const std::string &reading_prefix,
|
||||
const std::vector<std::string> &next_kana,
|
||||
size_t limit = 32) const;
|
||||
|
||||
@@ -121,19 +121,38 @@ std::vector<WordItem> JapaneseCandidateProvider::query(const QueryRequest &reque
|
||||
}
|
||||
}
|
||||
japanese::JapaneseMatrixSearch search(*sentence_decoder_);
|
||||
for (const auto &sentence : search.SearchConverted(conversion, 12))
|
||||
{
|
||||
AppendUnique(candidates, seen, request.raw_input_with_cases, sentence.text, 900000 - sentence.cost,
|
||||
CandidateSource::Database);
|
||||
}
|
||||
const auto sentences = search.SearchConverted(conversion, 12);
|
||||
size_t next_sentence = 0;
|
||||
// Once the reading is complete, convert what was typed before predicting
|
||||
// longer words. Keep the prediction-first path above for pending romaji.
|
||||
// longer words: the best whole-reading path and exact lemmas, then
|
||||
// predictions, then the remaining sentence tail (other paths, kana
|
||||
// fallbacks and lemmas covering only a prefix of the reading). Keep the
|
||||
// prediction-first path above for pending romaji.
|
||||
if (conversion.pending.empty() && conversion.hiragana.size() >= 6)
|
||||
{
|
||||
for (const auto &lemma : sentence_decoder_->PrefixLemmas(conversion.hiragana, 16))
|
||||
AppendUnique(candidates, seen, request.raw_input_with_cases, lemma.surface, 980000 - lemma.word_cost,
|
||||
std::int64_t weight_floor = 980000;
|
||||
const auto append_exact = [&](const std::string &text, std::int64_t weight) {
|
||||
const size_t before = candidates.size();
|
||||
AppendUnique(candidates, seen, request.raw_input_with_cases, text, weight, CandidateSource::Database);
|
||||
if (candidates.size() != before)
|
||||
weight_floor = (std::min)(weight_floor, weight);
|
||||
};
|
||||
if (!sentences.empty())
|
||||
append_exact(sentences[next_sentence++].text, 900000 - sentences.front().cost);
|
||||
for (const auto &lemma : sentence_decoder_->ExactLemmas(conversion.hiragana, 16))
|
||||
append_exact(lemma.surface, 900000 - lemma.word_cost);
|
||||
// Predictions must not outweigh the conversions above them, or a
|
||||
// weight-based re-sort would undo this order.
|
||||
for (const auto &lemma : sentence_decoder_->LongerPrefixLemmas(conversion.hiragana, 16))
|
||||
AppendUnique(candidates, seen, request.raw_input_with_cases, lemma.surface,
|
||||
(std::min)(weight_floor, 980000 - static_cast<std::int64_t>(lemma.word_cost)),
|
||||
CandidateSource::Database);
|
||||
}
|
||||
for (; next_sentence < sentences.size(); ++next_sentence)
|
||||
{
|
||||
AppendUnique(candidates, seen, request.raw_input_with_cases, sentences[next_sentence].text,
|
||||
900000 - sentences[next_sentence].cost, CandidateSource::Database);
|
||||
}
|
||||
}
|
||||
|
||||
if (ensure_query_statement())
|
||||
|
||||
@@ -5,9 +5,14 @@
|
||||
#include <sqlite3.h>
|
||||
#include <algorithm>
|
||||
#include <chrono>
|
||||
#include <cstdint>
|
||||
#include <cstdio>
|
||||
#include <filesystem>
|
||||
#include <fstream>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
#include <system_error>
|
||||
#include <vector>
|
||||
|
||||
namespace
|
||||
{
|
||||
@@ -112,6 +117,13 @@ void run_test(const std::filesystem::path &root)
|
||||
const auto sentence = query(provider, "watashiha");
|
||||
require(position(sentence, "私は") < position(sentence, "私は走る"),
|
||||
"Prediction displaced a complete reading spanning multiple lemmas.");
|
||||
// Lemmas covering only a prefix of the reading (川, 蚊) are not conversions of
|
||||
// what was typed, so they must not push the whole-word prediction down.
|
||||
const auto kawai = query(provider, "kawai");
|
||||
require(position(kawai, "可愛い") < position(kawai, "川"), "Prefix fragment 川 outranked a prediction.");
|
||||
require(position(kawai, "可愛い") < position(kawai, "蚊"), "Prefix fragment 蚊 outranked a prediction.");
|
||||
require(hashi[position(hashi, "走る")].weight <= hashi[position(hashi, "箸")].weight,
|
||||
"A prediction outweighs the exact conversion ranked above it.");
|
||||
|
||||
const auto pending = query(provider, "kaw");
|
||||
require(!pending.empty() && pending.front().word == "可愛い",
|
||||
@@ -147,6 +159,7 @@ int main()
|
||||
std::fprintf(stderr, "%s\n", error.what());
|
||||
result = 1;
|
||||
}
|
||||
std::filesystem::remove_all(root);
|
||||
std::error_code ignored;
|
||||
std::filesystem::remove_all(root, ignored);
|
||||
return result;
|
||||
}
|
||||
Reference in new issue
Block a user