diff --git a/server/CMakeLists.txt b/server/CMakeLists.txt index 57d407bd..b4e9f646 100644 --- a/server/CMakeLists.txt +++ b/server/CMakeLists.txt @@ -178,6 +178,7 @@ set(SRC_FILES_IME_ENGINE "./MetasequoiaImeEngine/providers/japanese_candidate_provider.cpp" "./MetasequoiaImeEngine/japanese/romaji_converter.cpp" "./MetasequoiaImeEngine/japanese/japanese_sentence_decoder.cpp" + "./MetasequoiaImeEngine/japanese/japanese_matrix_search.cpp" "./MetasequoiaImeEngine/quanpin/engine.cpp" "./MetasequoiaImeEngine/quanpin/quanpin_dictionary.cpp" "./MetasequoiaImeEngine/quanpin/quanpin_query.cpp" diff --git a/server/MetasequoiaImeEngine b/server/MetasequoiaImeEngine index aa7b52bb..d6f1491b 160000 --- a/server/MetasequoiaImeEngine +++ b/server/MetasequoiaImeEngine @@ -1 +1 @@ -Subproject commit aa7b52bb5d2f03f67ffc692acfbe2e4e77b87023 +Subproject commit d6f1491ba731dacfbdc73440ed7a05b6ee09711a diff --git a/server/src/session/engine_input_session.cpp b/server/src/session/engine_input_session.cpp index a251ea01..b85ab5b3 100644 --- a/server/src/session/engine_input_session.cpp +++ b/server/src/session/engine_input_session.cpp @@ -515,7 +515,7 @@ IInputSession::CloudQueryState EngineInputSession::get_cloud_query_state() const { state.cache_key = request().raw_input; state.committed_pinyin = request().raw_input; - state.should_query = japanese::ConvertRomaji(request().raw_input).complete; + state.should_query = !request().raw_input.empty(); state.query_text = state.should_query ? request().raw_input : std::string{}; return state; } diff --git a/server/tests/CMakeLists.txt b/server/tests/CMakeLists.txt index b775dfde..aa174d96 100644 --- a/server/tests/CMakeLists.txt +++ b/server/tests/CMakeLists.txt @@ -52,6 +52,7 @@ set(TEST_SOURCE_FILES "${CMAKE_SOURCE_DIR}/MetasequoiaImeEngine/providers/japanese_candidate_provider.cpp" "${CMAKE_SOURCE_DIR}/MetasequoiaImeEngine/japanese/romaji_converter.cpp" "${CMAKE_SOURCE_DIR}/MetasequoiaImeEngine/japanese/japanese_sentence_decoder.cpp" + "${CMAKE_SOURCE_DIR}/MetasequoiaImeEngine/japanese/japanese_matrix_search.cpp" "${CMAKE_SOURCE_DIR}/MetasequoiaImeEngine/quanpin/engine.cpp" "${CMAKE_SOURCE_DIR}/MetasequoiaImeEngine/quanpin/quanpin_dictionary.cpp" "${CMAKE_SOURCE_DIR}/MetasequoiaImeEngine/quanpin/quanpin_utils.cpp" diff --git a/server/tests/src/test_japanese_romaji.cpp b/server/tests/src/test_japanese_romaji.cpp index 2fc33803..fef7d502 100644 --- a/server/tests/src/test_japanese_romaji.cpp +++ b/server/tests/src/test_japanese_romaji.cpp @@ -1,8 +1,10 @@ #include "tests/includes/test_framework.h" #include "MetasequoiaImeEngine/japanese/romaji_converter.h" #include "MetasequoiaImeEngine/japanese/japanese_sentence_decoder.h" +#include "MetasequoiaImeEngine/japanese/japanese_matrix_search.h" #include "MetasequoiaImeEngine/providers/japanese_candidate_provider.h" #include "MetasequoiaImeEngine/schemes/japanese_romaji_scheme.h" +#include #include #include #include @@ -19,6 +21,17 @@ void InputRomaji(JapaneseRomajiScheme &scheme, const std::string &keys) } } +bool ContainsWord(const std::vector &items, const std::string &word) +{ + return std::any_of(items.begin(), items.end(), [&](const WordItem &item) { return item.word == word; }); +} + +size_t WordIndex(const std::vector &items, const std::string &word) +{ + const auto found = std::find_if(items.begin(), items.end(), [&](const WordItem &item) { return item.word == word; }); + return found == items.end() ? items.size() : static_cast(found - items.begin()); +} + std::filesystem::path CreateJapaneseDatabase() { const auto path = std::filesystem::temp_directory_path() / "msime-japanese-provider-test.db"; @@ -29,7 +42,10 @@ std::filesystem::path CreateJapaneseDatabase() const char *sql = "CREATE TABLE japanese_lexicon(code TEXT,value TEXT,weight INTEGER,PRIMARY KEY(code,value));" "INSERT INTO japanese_lexicon VALUES('qa','亜',20);" - "INSERT INTO japanese_lexicon VALUES('qa','会',10);"; + "INSERT INTO japanese_lexicon VALUES('qa','会',10);" + "INSERT INTO japanese_lexicon VALUES('qwatashi','私',30);" + "INSERT INTO japanese_lexicon VALUES('kawaii','かわいい',40);" + "INSERT INTO japanese_lexicon VALUES('qkawaii','可愛い',50);"; const int result = sqlite3_exec(db, sql, nullptr, nullptr, nullptr); sqlite3_close(db); if (result != SQLITE_OK) @@ -45,7 +61,8 @@ TEST_CASE(JapaneseRomajiConvertsCommonImeSpellings) REQUIRE_EQ(japanese::ConvertRomaji("gakkou").hiragana, std::string("がっこう")); REQUIRE_EQ(japanese::ConvertRomaji("shin'you").hiragana, std::string("しんよう")); REQUIRE_EQ(japanese::HiraganaToKatakana("にほんご"), std::string("ニホンゴ")); - REQUIRE(!japanese::ConvertRomaji("k").complete); + REQUIRE_EQ(japanese::HiraganaToRomaji("かわいい"), std::string("kawaii")); + REQUIRE_EQ(japanese::HiraganaToRomaji("にほんご"), std::string("nihongo")); } TEST_CASE(JapaneseRomajiSchemePreservesTypedCodeAndShowsKanaSegmentation) @@ -70,9 +87,8 @@ TEST_CASE(JapaneseProviderCombinesGeneratedKanaAndSqliteCandidates) kana.raw_input_with_cases = "nihongo"; kana.valid = true; const auto kana_candidates = provider.query(kana); - REQUIRE(kana_candidates.size() >= static_cast(2)); - REQUIRE_EQ(kana_candidates[0].word, std::string("にほんご")); - REQUIRE_EQ(kana_candidates[1].word, std::string("ニホンゴ")); + REQUIRE(ContainsWord(kana_candidates, "にほんご")); + REQUIRE(ContainsWord(kana_candidates, "ニホンゴ")); QueryRequest direct; direct.scheme = SchemeType::JapaneseRomaji; @@ -83,10 +99,94 @@ TEST_CASE(JapaneseProviderCombinesGeneratedKanaAndSqliteCandidates) REQUIRE_EQ(direct_candidates.size(), static_cast(2)); REQUIRE_EQ(direct_candidates[0].word, std::string("亜")); REQUIRE_EQ(direct_candidates[1].word, std::string("会")); + + QueryRequest prefix; + prefix.scheme = SchemeType::JapaneseRomaji; + prefix.raw_input = "qwat"; + prefix.raw_input_with_cases = "qwat"; + prefix.valid = true; + const auto prefix_candidates = provider.query(prefix); + bool found_watashi = false; + for (const auto &item : prefix_candidates) + found_watashi = found_watashi || item.word == "私"; + REQUIRE(found_watashi); + + QueryRequest fuzzy; + fuzzy.scheme = SchemeType::JapaneseRomaji; + fuzzy.raw_input = "kaw"; + fuzzy.raw_input_with_cases = "kaw"; + fuzzy.valid = true; + const auto fuzzy_candidates = provider.query(fuzzy); + REQUIRE(ContainsWord(fuzzy_candidates, "か")); + REQUIRE(ContainsWord(fuzzy_candidates, "かわいい") || ContainsWord(fuzzy_candidates, "可愛い")); + const size_t phrase_index = (std::min)(WordIndex(fuzzy_candidates, "かわいい"), WordIndex(fuzzy_candidates, "可愛い")); + REQUIRE(phrase_index < WordIndex(fuzzy_candidates, "か")); } std::filesystem::remove(path); } +TEST_CASE(JapaneseRomajiPrefixMapsToKanaLikeHalfSpellingIds) +{ + const auto kana = japanese::KanaForRomajiPrefix("k"); + bool has_ka = false; + bool has_kyo = false; + for (const auto &item : kana) + { + has_ka = has_ka || item == "か"; + has_kyo = has_kyo || item == "きょ"; + } + REQUIRE(has_ka); + REQUIRE(has_kyo); + REQUIRE(japanese::KanaForRomajiPrefix("").empty()); +} + +TEST_CASE(JapaneseProviderShowsPhrasesBeforeConvertedKana) +{ + const auto path = CreateJapaneseDatabase(); + { + JapaneseCandidateProvider provider(path.string()); + QueryRequest request; + request.scheme = SchemeType::JapaneseRomaji; + request.raw_input = "nihong"; + request.raw_input_with_cases = "nihong"; + request.valid = true; + const auto candidates = provider.query(request); + REQUIRE(ContainsWord(candidates, "にほん")); + REQUIRE(ContainsWord(candidates, "ニホン")); + } + std::filesystem::remove(path); +} + +TEST_CASE(JapaneseMatrixSearchDecodesWholeSentenceWhenModelAvailable) +{ + const auto workspace = std::filesystem::path(__FILE__).parent_path().parent_path().parent_path().parent_path(); + const auto model = workspace / "MetasequoiaImeDict" / "out" / "dict_japanese.dat"; + if (!std::filesystem::is_regular_file(model)) return; + + japanese::JapaneseSentenceDecoder decoder(model.string()); + REQUIRE(decoder.ready()); + japanese::JapaneseMatrixSearch search(decoder); + const auto complete = search.Search("nihongo", 12); + REQUIRE(!complete.empty()); + bool found = false; + for (const auto &candidate : complete) + found = found || candidate.text == "日本語"; + REQUIRE(found); + + const auto prefix = search.Search("nihong", 12); + bool found_prefix = false; + for (const auto &candidate : prefix) + found_prefix = found_prefix || candidate.text == "日本語"; + REQUIRE(found_prefix); + + const auto fuzzy = decoder.PrefixLemmas("かわ", 32); + bool found_kawaii = false; + for (const auto &lemma : fuzzy) + found_kawaii = found_kawaii || lemma.surface == "可愛い" || lemma.reading == "かわいい" || + lemma.surface == "かわいい"; + REQUIRE(found_kawaii); +} + TEST_CASE(JapaneseSentenceDecoderReadsGeneratedMozcModelWhenAvailable) { const auto workspace = std::filesystem::path(__FILE__).parent_path().parent_path().parent_path().parent_path();