Implement the whole-sentence algorithm and fuzzy query for Japanese.

server/CMakeLists.txt

@@ -178,6 +178,7 @@ set(SRC_FILES_IME_ENGINE
     "./MetasequoiaImeEngine/providers/japanese_candidate_provider.cpp"
     "./MetasequoiaImeEngine/japanese/romaji_converter.cpp"
     "./MetasequoiaImeEngine/japanese/japanese_sentence_decoder.cpp"
+    "./MetasequoiaImeEngine/japanese/japanese_matrix_search.cpp"
     "./MetasequoiaImeEngine/quanpin/engine.cpp"
     "./MetasequoiaImeEngine/quanpin/quanpin_dictionary.cpp"
     "./MetasequoiaImeEngine/quanpin/quanpin_query.cpp"

server/src/session/engine_input_session.cpp

@@ -515,7 +515,7 @@ IInputSession::CloudQueryState EngineInputSession::get_cloud_query_state() const
     {
         state.cache_key = request().raw_input;
         state.committed_pinyin = request().raw_input;
-        state.should_query = japanese::ConvertRomaji(request().raw_input).complete;
+        state.should_query = !request().raw_input.empty();
         state.query_text = state.should_query ? request().raw_input : std::string{};
         return state;
     }

server/tests/CMakeLists.txt

@@ -52,6 +52,7 @@ set(TEST_SOURCE_FILES
     "${CMAKE_SOURCE_DIR}/MetasequoiaImeEngine/providers/japanese_candidate_provider.cpp"
     "${CMAKE_SOURCE_DIR}/MetasequoiaImeEngine/japanese/romaji_converter.cpp"
     "${CMAKE_SOURCE_DIR}/MetasequoiaImeEngine/japanese/japanese_sentence_decoder.cpp"
+    "${CMAKE_SOURCE_DIR}/MetasequoiaImeEngine/japanese/japanese_matrix_search.cpp"
     "${CMAKE_SOURCE_DIR}/MetasequoiaImeEngine/quanpin/engine.cpp"
     "${CMAKE_SOURCE_DIR}/MetasequoiaImeEngine/quanpin/quanpin_dictionary.cpp"
     "${CMAKE_SOURCE_DIR}/MetasequoiaImeEngine/quanpin/quanpin_utils.cpp"

server/tests/src/test_japanese_romaji.cpp

@@ -1,8 +1,10 @@
 #include "tests/includes/test_framework.h"
 #include "MetasequoiaImeEngine/japanese/romaji_converter.h"
 #include "MetasequoiaImeEngine/japanese/japanese_sentence_decoder.h"
+#include "MetasequoiaImeEngine/japanese/japanese_matrix_search.h"
 #include "MetasequoiaImeEngine/providers/japanese_candidate_provider.h"
 #include "MetasequoiaImeEngine/schemes/japanese_romaji_scheme.h"
+#include <algorithm>
 #include <cctype>
 #include <filesystem>
 #include <sqlite3.h>
@@ -19,6 +21,17 @@ void InputRomaji(JapaneseRomajiScheme &scheme, const std::string &keys)
     }
 }
 
+bool ContainsWord(const std::vector<WordItem> &items, const std::string &word)
+{
+    return std::any_of(items.begin(), items.end(), [&](const WordItem &item) { return item.word == word; });
+}
+
+size_t WordIndex(const std::vector<WordItem> &items, const std::string &word)
+{
+    const auto found = std::find_if(items.begin(), items.end(), [&](const WordItem &item) { return item.word == word; });
+    return found == items.end() ? items.size() : static_cast<size_t>(found - items.begin());
+}
+
 std::filesystem::path CreateJapaneseDatabase()
 {
     const auto path = std::filesystem::temp_directory_path() / "msime-japanese-provider-test.db";
@@ -29,7 +42,10 @@ std::filesystem::path CreateJapaneseDatabase()
     const char *sql =
         "CREATE TABLE japanese_lexicon(code TEXT,value TEXT,weight INTEGER,PRIMARY KEY(code,value));"
         "INSERT INTO japanese_lexicon VALUES('qa','亜',20);"
-        "INSERT INTO japanese_lexicon VALUES('qa','会',10);";
+        "INSERT INTO japanese_lexicon VALUES('qa','会',10);"
+        "INSERT INTO japanese_lexicon VALUES('qwatashi','私',30);"
+        "INSERT INTO japanese_lexicon VALUES('kawaii','かわいい',40);"
+        "INSERT INTO japanese_lexicon VALUES('qkawaii','可愛い',50);";
     const int result = sqlite3_exec(db, sql, nullptr, nullptr, nullptr);
     sqlite3_close(db);
     if (result != SQLITE_OK)
@@ -45,7 +61,8 @@ TEST_CASE(JapaneseRomajiConvertsCommonImeSpellings)
     REQUIRE_EQ(japanese::ConvertRomaji("gakkou").hiragana, std::string("がっこう"));
     REQUIRE_EQ(japanese::ConvertRomaji("shin'you").hiragana, std::string("しんよう"));
     REQUIRE_EQ(japanese::HiraganaToKatakana("にほんご"), std::string("ニホンゴ"));
-    REQUIRE(!japanese::ConvertRomaji("k").complete);
+    REQUIRE_EQ(japanese::HiraganaToRomaji("かわいい"), std::string("kawaii"));
+    REQUIRE_EQ(japanese::HiraganaToRomaji("にほんご"), std::string("nihongo"));
 }
 
 TEST_CASE(JapaneseRomajiSchemePreservesTypedCodeAndShowsKanaSegmentation)
@@ -70,9 +87,8 @@ TEST_CASE(JapaneseProviderCombinesGeneratedKanaAndSqliteCandidates)
         kana.raw_input_with_cases = "nihongo";
         kana.valid = true;
         const auto kana_candidates = provider.query(kana);
-        REQUIRE(kana_candidates.size() >= static_cast<size_t>(2));
-        REQUIRE_EQ(kana_candidates[0].word, std::string("にほんご"));
-        REQUIRE_EQ(kana_candidates[1].word, std::string("ニホンゴ"));
+        REQUIRE(ContainsWord(kana_candidates, "にほんご"));
+        REQUIRE(ContainsWord(kana_candidates, "ニホンゴ"));
 
         QueryRequest direct;
         direct.scheme = SchemeType::JapaneseRomaji;
@@ -83,10 +99,94 @@ TEST_CASE(JapaneseProviderCombinesGeneratedKanaAndSqliteCandidates)
         REQUIRE_EQ(direct_candidates.size(), static_cast<size_t>(2));
         REQUIRE_EQ(direct_candidates[0].word, std::string("亜"));
         REQUIRE_EQ(direct_candidates[1].word, std::string("会"));
+
+        QueryRequest prefix;
+        prefix.scheme = SchemeType::JapaneseRomaji;
+        prefix.raw_input = "qwat";
+        prefix.raw_input_with_cases = "qwat";
+        prefix.valid = true;
+        const auto prefix_candidates = provider.query(prefix);
+        bool found_watashi = false;
+        for (const auto &item : prefix_candidates)
+            found_watashi = found_watashi || item.word == "私";
+        REQUIRE(found_watashi);
+
+        QueryRequest fuzzy;
+        fuzzy.scheme = SchemeType::JapaneseRomaji;
+        fuzzy.raw_input = "kaw";
+        fuzzy.raw_input_with_cases = "kaw";
+        fuzzy.valid = true;
+        const auto fuzzy_candidates = provider.query(fuzzy);
+        REQUIRE(ContainsWord(fuzzy_candidates, "か"));
+        REQUIRE(ContainsWord(fuzzy_candidates, "かわいい") || ContainsWord(fuzzy_candidates, "可愛い"));
+        const size_t phrase_index = (std::min)(WordIndex(fuzzy_candidates, "かわいい"), WordIndex(fuzzy_candidates, "可愛い"));
+        REQUIRE(phrase_index < WordIndex(fuzzy_candidates, "か"));
     }
     std::filesystem::remove(path);
 }
 
+TEST_CASE(JapaneseRomajiPrefixMapsToKanaLikeHalfSpellingIds)
+{
+    const auto kana = japanese::KanaForRomajiPrefix("k");
+    bool has_ka = false;
+    bool has_kyo = false;
+    for (const auto &item : kana)
+    {
+        has_ka = has_ka || item == "か";
+        has_kyo = has_kyo || item == "きょ";
+    }
+    REQUIRE(has_ka);
+    REQUIRE(has_kyo);
+    REQUIRE(japanese::KanaForRomajiPrefix("").empty());
+}
+
+TEST_CASE(JapaneseProviderShowsPhrasesBeforeConvertedKana)
+{
+    const auto path = CreateJapaneseDatabase();
+    {
+        JapaneseCandidateProvider provider(path.string());
+        QueryRequest request;
+        request.scheme = SchemeType::JapaneseRomaji;
+        request.raw_input = "nihong";
+        request.raw_input_with_cases = "nihong";
+        request.valid = true;
+        const auto candidates = provider.query(request);
+        REQUIRE(ContainsWord(candidates, "にほん"));
+        REQUIRE(ContainsWord(candidates, "ニホン"));
+    }
+    std::filesystem::remove(path);
+}
+
+TEST_CASE(JapaneseMatrixSearchDecodesWholeSentenceWhenModelAvailable)
+{
+    const auto workspace = std::filesystem::path(__FILE__).parent_path().parent_path().parent_path().parent_path();
+    const auto model = workspace / "MetasequoiaImeDict" / "out" / "dict_japanese.dat";
+    if (!std::filesystem::is_regular_file(model)) return;
+
+    japanese::JapaneseSentenceDecoder decoder(model.string());
+    REQUIRE(decoder.ready());
+    japanese::JapaneseMatrixSearch search(decoder);
+    const auto complete = search.Search("nihongo", 12);
+    REQUIRE(!complete.empty());
+    bool found = false;
+    for (const auto &candidate : complete)
+        found = found || candidate.text == "日本語";
+    REQUIRE(found);
+
+    const auto prefix = search.Search("nihong", 12);
+    bool found_prefix = false;
+    for (const auto &candidate : prefix)
+        found_prefix = found_prefix || candidate.text == "日本語";
+    REQUIRE(found_prefix);
+
+    const auto fuzzy = decoder.PrefixLemmas("かわ", 32);
+    bool found_kawaii = false;
+    for (const auto &lemma : fuzzy)
+        found_kawaii = found_kawaii || lemma.surface == "可愛い" || lemma.reading == "かわいい" ||
+                       lemma.surface == "かわいい";
+    REQUIRE(found_kawaii);
+}
+
 TEST_CASE(JapaneseSentenceDecoderReadsGeneratedMozcModelWhenAvailable)
 {
     const auto workspace = std::filesystem::path(__FILE__).parent_path().parent_path().parent_path().parent_path();