diff --git a/server/MetasequoiaImeEngine b/server/MetasequoiaImeEngine index c98e4bbc..22ced21d 160000 --- a/server/MetasequoiaImeEngine +++ b/server/MetasequoiaImeEngine @@ -1 +1 @@ -Subproject commit c98e4bbc01ddabd31a3ea2f77022557b0543ec6f +Subproject commit 22ced21d308cf22f7dc88b810f1b0c3c1087f602 diff --git a/server/src/ipc/event_listener.cpp b/server/src/ipc/event_listener.cpp index 55e1c542..d3d03078 100644 --- a/server/src/ipc/event_listener.cpp +++ b/server/src/ipc/event_listener.cpp @@ -782,6 +782,7 @@ struct Task uint64_t english_generation = 0; std::string session_pinyin; std::string session_word; + bool session_pinyin_is_canonical = false; int candidate_one_based_index = 0; int fixed_position = 0; }; @@ -799,6 +800,7 @@ std::string CurrentRankingContextKey() std::string CandidateDatabaseKey(const WordItem &item, const std::string &context_key) { + if (!item.canonical_pinyin.empty()) return item.canonical_pinyin; if (g_inputSession->get_pinyin_sequence().size() == 1) return item.pinyin; auto segments = quanpin::split_segments(context_key); const size_t han_count = HelpcodeUtils::count_han_chars(item.word); @@ -818,7 +820,8 @@ void ProcessSelectionKey(UINT keycode, uint64_t client_id, uint64_t activation_e void ApplyCloudCandidate(const std::string &candidate, const std::string &pinyin, uint64_t generation); void ApplyAiCandidate(const std::string &candidate, const std::string &identity, uint64_t generation); void ApplyEnglishCandidates(std::vector candidates, const std::string &input, uint64_t generation); -void EnqueueStoreUserPhraseTask(const std::string &pinyin, const std::string &word); +void EnqueueStoreUserPhraseTask(const std::string &pinyin, const std::string &word, + bool pinyin_is_canonical = false); void EnqueuePinCandidateTask(const std::string &pinyin, const std::string &word); bool ResolveCandidateItem(int one_based_index, WordItem &item); bool SendCurrentDataToClient(uint64_t client_id, uint64_t activation_epoch, uint64_t request_id); @@ -944,7 +947,15 @@ void WorkerThread() } case TaskType::StoreUserPhrase: { - g_inputSession->store_user_phrase(task.session_pinyin, task.session_word); + if (task.session_pinyin_is_canonical) + { + g_inputSession->store_user_phrase_from_canonical_pinyin(task.session_pinyin, + task.session_word); + } + else + { + g_inputSession->store_user_phrase(task.session_pinyin, task.session_word); + } g_inputSession->reset_cache(); break; } @@ -1255,7 +1266,8 @@ void EnqueueEnglishCandidates(std::vector candidates, const std::strin pipe_queueCv.notify_one(); } -void EnqueueStoreUserPhraseTask(const std::string &pinyin, const std::string &word) +void EnqueueStoreUserPhraseTask(const std::string &pinyin, const std::string &word, + bool pinyin_is_canonical) { { std::lock_guard lock(queueMutex); @@ -1263,6 +1275,7 @@ void EnqueueStoreUserPhraseTask(const std::string &pinyin, const std::string &wo task.type = TaskType::StoreUserPhrase; task.session_pinyin = pinyin; task.session_word = word; + task.session_pinyin_is_canonical = pinyin_is_canonical; taskQueue.push(std::move(task)); } pipe_queueCv.notify_one(); @@ -2619,11 +2632,11 @@ void ProcessSelectionKey(UINT keycode, uint64_t client_id, uint64_t activation_e ? g_inputSession->get_cloud_query_state().committed_pinyin : std::string{}; isNeedUpdateWeight = false; } - auto selection_transition = g_inputSession->advance_composition_after_selection(curWordPinyin, curWord); + auto selection_transition = g_inputSession->advance_composition_after_selection( + curWordPinyin, curWord, curWordItem.canonical_pinyin); const bool isNeedCreateWord = selection_transition.continues_composition; if (isNeedCreateWord) - { /* 将上屏的汉字字符串所对应的拼音比实际的拼音要短的话，同时，preedit - 拼音的纯拼音版本(去除辅助码)的每一个分词都是完整的拼音 */ + { /* 候选只消耗了输入的一部分，继续使用剩余输入造词。完整拼音和简拼均可进入。 */ /* 打开造词开关 */ GlobalIme::composition.creating_word.active = true; Global::MsgTypeToTsf = Global::DataFromServerMsgType::NeedToCreateWord; @@ -2657,9 +2670,13 @@ void ProcessSelectionKey(UINT keycode, uint64_t client_id, uint64_t activation_e Global::candidate_ui.selected_text = string_to_wstring(CandidateTextForOutput(GlobalIme::composition.creating_word.word)); - // 这里异步处理，不然有可能会阻塞住 TSF 端读取 pipe 导致超时 - EnqueueStoreUserPhraseTask(GlobalIme::composition.creating_word.pinyin, - GlobalIme::composition.creating_word.word); + if (creating_word_progress.can_store) + { + // 这里异步处理，不然有可能会阻塞住 TSF 端读取 pipe 导致超时 + EnqueueStoreUserPhraseTask(GlobalIme::composition.creating_word.pinyin, + GlobalIme::composition.creating_word.word, + /*pinyin_is_canonical=*/true); + } /* 清理 */ GlobalIme::composition.clear_creating_word(); diff --git a/server/src/session/engine_input_session.cpp b/server/src/session/engine_input_session.cpp index eb6dbd45..5036934b 100644 --- a/server/src/session/engine_input_session.cpp +++ b/server/src/session/engine_input_session.cpp @@ -45,6 +45,58 @@ size_t raw_length_for_normalized_prefix(const std::string &raw_input, size_t nor return raw_length; } +void remove_consumed_leading_separators(std::string &raw_input, std::string &raw_input_with_cases) +{ + size_t count = 0; + while (count < raw_input.size() && raw_input[count] == '\'') + { + ++count; + } + raw_input.erase(0, count); + + count = 0; + while (count < raw_input_with_cases.size() && raw_input_with_cases[count] == '\'') + { + ++count; + } + raw_input_with_cases.erase(0, count); +} + +std::string normalize_canonical_pinyin_for_word(const std::string &pinyin, const std::string &word) +{ + if (pinyin.empty()) + { + return {}; + } + + const auto segments = quanpin::split_segments(pinyin); + if (segments.empty() || segments.size() != HelpcodeUtils::count_han_chars(word)) + { + return {}; + } + for (const auto &segment : segments) + { + if (segment.empty() || !quanpin::is_complete_pinyin_input(segment)) + { + return {}; + } + } + return quanpin::join_segments(segments); +} + +std::string append_canonical_pinyin(const std::string &prefix, const std::string &suffix) +{ + if (prefix.empty()) + { + return suffix; + } + if (suffix.empty()) + { + return {}; + } + return prefix + "'" + suffix; +} + struct ShuangpinCompositionBase { std::string raw_input; @@ -67,8 +119,27 @@ ShuangpinCompositionBase ResolveShuangpinCompositionBase(const QueryRequest &req return base; } + const auto has_complete_unseparated_base = [&](size_t helpcode_length) { + if (base.effective_raw_input.size() <= helpcode_length) + { + return false; + } + const size_t pure_length = base.effective_raw_input.size() - helpcode_length; + const size_t raw_prefix_length = + raw_length_for_normalized_prefix(base.raw_input, pure_length); + // An apostrophe immediately before the suffix makes that suffix a + // user-defined pinyin segment, not an auxiliary code. + if (raw_prefix_length < base.raw_input.size() && + base.raw_input[raw_prefix_length] == '\'') + { + return false; + } + return shuangpin::is_complete_input( + base.raw_input.substr(0, raw_prefix_length), profile); + }; + if (ShuangpinUtil::IsFullHelpMode(base.effective_raw_input_with_cases, profile) && - base.effective_raw_input.size() >= 2) + has_complete_unseparated_base(2)) { base.helpcode_length = 2; return base; @@ -76,8 +147,7 @@ ShuangpinCompositionBase ResolveShuangpinCompositionBase(const QueryRequest &req if (base.effective_raw_input.size() % 2 == 1 && base.effective_raw_input.size() > 1) { - const std::string pure_raw_input = base.effective_raw_input.substr(0, base.effective_raw_input.size() - 1); - if (shuangpin::is_complete_input(pure_raw_input, profile)) + if (has_complete_unseparated_base(1)) { base.helpcode_length = 1; } @@ -280,6 +350,14 @@ int EngineInputSession::store_user_phrase(std::string pinyin, std::string word) return session_.create_word(std::move(pinyin), std::move(word)); } +int EngineInputSession::store_user_phrase_from_canonical_pinyin(std::string pinyin, std::string word) +{ + // Both quanpin and shuangpin ultimately share the canonical quanpin + // dictionary. Do not feed a complete quanpin key back through the active + // shuangpin profile a second time. + return quanpin_engine_.create_word_from_canonical_pinyin(std::move(pinyin), std::move(word)); +} + int EngineInputSession::pin_candidate(std::string pinyin, std::string word) { return session_.update_weight_by_pinyin_and_word(std::move(pinyin), std::move(word)); @@ -303,9 +381,12 @@ int EngineInputSession::cache_dynamic_candidate(const std::string &pinyin, const } IInputSession::SelectionTransition EngineInputSession::advance_composition_after_selection( - const std::string &selected_pinyin, const std::string &selected_word) + const std::string &selected_pinyin, + const std::string &selected_word, + const std::string &selected_canonical_pinyin) { SelectionTransition transition; + transition.selected_canonical_pinyin = selected_canonical_pinyin; if (is_wubi()) { transition.full_pure_pinyin = request().normalized_input; @@ -338,9 +419,11 @@ IInputSession::SelectionTransition EngineInputSession::advance_composition_after const size_t rest_end = raw_length_for_normalized_prefix(base.raw_input_with_cases, total_input_length - base.helpcode_length); const std::string rest_pinyin_sequence = base.raw_input.substr(rest_start, rest_end - rest_start); - const std::string rest_pinyin_sequence_with_cases = + std::string normalized_rest = rest_pinyin_sequence; + std::string cased_rest = base.raw_input_with_cases.substr(rest_start, rest_end - rest_start); - session_.replace_shuangpin_raw_input(rest_pinyin_sequence, rest_pinyin_sequence_with_cases); + remove_consumed_leading_separators(normalized_rest, cased_rest); + session_.replace_shuangpin_raw_input(normalized_rest, cased_rest); } } else @@ -351,7 +434,7 @@ IInputSession::SelectionTransition EngineInputSession::advance_composition_after } transition.continues_composition = - consumed_length < transition.full_pure_pinyin.size() && is_all_complete_pure_pinyin(); + consumed_length < transition.full_pure_pinyin.size(); if (transition.continues_composition) { @@ -361,7 +444,10 @@ IInputSession::SelectionTransition EngineInputSession::advance_composition_after base.raw_input.substr(consumed_raw_length, base.raw_input.size() - consumed_raw_length); const std::string rest_pinyin_sequence_with_cases = base.raw_input_with_cases.substr( consumed_raw_length, base.raw_input_with_cases.size() - consumed_raw_length); - session_.replace_shuangpin_raw_input(rest_pinyin_sequence, rest_pinyin_sequence_with_cases); + std::string normalized_rest = rest_pinyin_sequence; + std::string cased_rest = rest_pinyin_sequence_with_cases; + remove_consumed_leading_separators(normalized_rest, cased_rest); + session_.replace_shuangpin_raw_input(normalized_rest, cased_rest); } } @@ -385,13 +471,14 @@ IInputSession::SelectionTransition EngineInputSession::advance_composition_after transition.continues_composition = !selected_pure_pinyin.empty() && selected_pure_pinyin.size() < transition.full_pure_pinyin.size() && - is_all_complete_pure_pinyin(); + consumed_raw_length < raw_input_without_helpcodes.size(); if (transition.continues_composition) { - const std::string rest_raw_input = raw_input_without_helpcodes.substr(consumed_raw_length); - const std::string rest_raw_input_with_cases = + std::string rest_raw_input = raw_input_without_helpcodes.substr(consumed_raw_length); + std::string rest_raw_input_with_cases = raw_input_with_cases_without_helpcodes.substr(consumed_raw_length); + remove_consumed_leading_separators(rest_raw_input, rest_raw_input_with_cases); session_.replace_quanpin_raw_input(rest_raw_input, rest_raw_input_with_cases); transition.current_segmentation = get_pinyin_segmentation(); transition.current_segmentation_with_cases = get_pinyin_segmentation_with_cases(); @@ -469,19 +556,23 @@ IInputSession::CreatingWordProgress EngineInputSession::update_creating_word_pro progress.word = current_word + selected_word; progress.preedit = progress.word; progress.completed = true; - return progress; - } - progress.pinyin = current_pinyin.empty() ? selection_transition.full_pure_pinyin : current_pinyin; - progress.word = current_word + selected_word; - progress.preedit = progress.word + selection_transition.current_segmentation_with_cases; - if (is_shuangpin()) - { - progress.completed = HelpcodeUtils::count_han_chars(progress.word) * 2 == progress.pinyin.size(); + progress.can_store = false; return progress; } - const auto cuts = quanpin::cut_pinyin_by_mode(progress.pinyin, "correction"); - progress.completed = !cuts.empty() && cuts.front().size() == HelpcodeUtils::count_han_chars(progress.word); + const std::string selected_canonical = + normalize_canonical_pinyin_for_word(selection_transition.selected_canonical_pinyin, selected_word); + const bool prior_parts_are_storeable = current_word.empty() || !current_pinyin.empty(); + if (prior_parts_are_storeable && !selected_canonical.empty()) + { + progress.pinyin = append_canonical_pinyin(current_pinyin, selected_canonical); + } + progress.word = current_word + selected_word; + progress.preedit = progress.word + selection_transition.current_segmentation_with_cases; + progress.completed = !selection_transition.continues_composition; + progress.can_store = + progress.completed && + !normalize_canonical_pinyin_for_word(progress.pinyin, progress.word).empty(); return progress; } diff --git a/server/src/session/engine_input_session.h b/server/src/session/engine_input_session.h index c72a4b47..d4743e03 100644 --- a/server/src/session/engine_input_session.h +++ b/server/src/session/engine_input_session.h @@ -35,12 +35,15 @@ class EngineInputSession : public IInputSession void set_pinyin_sequence_with_cases(const std::string &pinyin_sequence) override; int store_user_phrase(std::string pinyin, std::string word) override; + int store_user_phrase_from_canonical_pinyin(std::string pinyin, std::string word) override; int pin_candidate(std::string pinyin, std::string word) override; int remove_candidate(std::string pinyin, std::string word) override; int cache_dynamic_candidate(const std::string &pinyin, const std::string &word, CandidateSource source) override; - SelectionTransition advance_composition_after_selection(const std::string &selected_pinyin, - const std::string &selected_word) override; + SelectionTransition advance_composition_after_selection( + const std::string &selected_pinyin, + const std::string &selected_word, + const std::string &selected_canonical_pinyin) override; CloudQueryState get_cloud_query_state() const override; CreatingWordProgress update_creating_word_progress(const std::string ¤t_pinyin, const std::string ¤t_word, diff --git a/server/src/session/input_session.h b/server/src/session/input_session.h index 7cc1d1c7..4305df7d 100644 --- a/server/src/session/input_session.h +++ b/server/src/session/input_session.h @@ -18,6 +18,7 @@ class IInputSession std::string full_pure_pinyin; std::string current_segmentation; std::string current_segmentation_with_cases; + std::string selected_canonical_pinyin; }; struct CloudQueryState @@ -34,6 +35,7 @@ class IInputSession std::string word; std::string preedit; bool completed = false; + bool can_store = false; }; virtual ~IInputSession() = default; @@ -65,12 +67,15 @@ class IInputSession virtual void set_pinyin_sequence_with_cases(const std::string &pinyin_sequence) = 0; virtual int store_user_phrase(std::string pinyin, std::string word) = 0; + virtual int store_user_phrase_from_canonical_pinyin(std::string pinyin, std::string word) = 0; virtual int pin_candidate(std::string pinyin, std::string word) = 0; virtual int remove_candidate(std::string pinyin, std::string word) = 0; virtual int cache_dynamic_candidate(const std::string &pinyin, const std::string &word, CandidateSource source) = 0; - virtual SelectionTransition advance_composition_after_selection(const std::string &selected_pinyin, - const std::string &selected_word) = 0; + virtual SelectionTransition advance_composition_after_selection( + const std::string &selected_pinyin, + const std::string &selected_word, + const std::string &selected_canonical_pinyin) = 0; virtual CloudQueryState get_cloud_query_state() const = 0; virtual CreatingWordProgress update_creating_word_progress(const std::string ¤t_pinyin, const std::string ¤t_word, diff --git a/server/src/session/shuangpin_input_session.cpp b/server/src/session/shuangpin_input_session.cpp index 1b5d2f48..9b113f10 100644 --- a/server/src/session/shuangpin_input_session.cpp +++ b/server/src/session/shuangpin_input_session.cpp @@ -1,7 +1,10 @@ #include "shuangpin_input_session.h" #include "config/ime_config.h" #include "MetasequoiaImeEngine/common/helpcode_utils.h" +#include "MetasequoiaImeEngine/quanpin/quanpin_query.h" +#include "MetasequoiaImeEngine/quanpin/quanpin_utils.h" #include "MetasequoiaImeEngine/shuangpin/shuangpin_utils.h" +#include #include namespace @@ -135,6 +138,11 @@ int ShuangpinInputSession::store_user_phrase(std::string pinyin, std::string wor return dictionary_->create_word(std::move(pinyin), std::move(word)); } +int ShuangpinInputSession::store_user_phrase_from_canonical_pinyin(std::string pinyin, std::string word) +{ + return dictionary_->create_word_from_quanpin(std::move(pinyin), std::move(word)); +} + int ShuangpinInputSession::pin_candidate(std::string pinyin, std::string word) { return dictionary_->update_weight_by_pinyin_and_word(std::move(pinyin), std::move(word)); @@ -157,14 +165,16 @@ int ShuangpinInputSession::cache_dynamic_candidate(const std::string &pinyin, co IInputSession::SelectionTransition ShuangpinInputSession::advance_composition_after_selection(const std::string &selected_pinyin, - const std::string &selected_word) + const std::string &selected_word, + const std::string &selected_canonical_pinyin) { (void)selected_word; SelectionTransition transition; + transition.selected_canonical_pinyin = selected_canonical_pinyin; transition.full_pure_pinyin = dictionary_->get_pure_pinyin_sequence(); transition.continues_composition = - selected_pinyin.size() < transition.full_pure_pinyin.size() && dictionary_->is_all_complete_pure_pinyin(); + !selected_pinyin.empty() && selected_pinyin.size() < transition.full_pure_pinyin.size(); if (transition.continues_composition) { @@ -216,9 +226,26 @@ ShuangpinInputSession::update_creating_word_progress(const std::string ¤t_ const SelectionTransition &selection_transition) const { CreatingWordProgress progress; - progress.pinyin = current_pinyin.empty() ? selection_transition.full_pure_pinyin : current_pinyin; + const auto canonical_segments = + quanpin::split_segments(selection_transition.selected_canonical_pinyin); + const bool selected_is_canonical = + !canonical_segments.empty() && + canonical_segments.size() == HelpcodeUtils::count_han_chars(selected_word) && + std::all_of(canonical_segments.begin(), canonical_segments.end(), [](const std::string &segment) { + return !segment.empty() && quanpin::is_complete_pinyin_input(segment); + }); + const bool prior_parts_are_storeable = current_word.empty() || !current_pinyin.empty(); + if (selected_is_canonical && prior_parts_are_storeable) + { + const std::string selected = quanpin::join_segments(canonical_segments); + progress.pinyin = current_pinyin.empty() ? selected : current_pinyin + "'" + selected; + } progress.word = current_word + selected_word; progress.preedit = progress.word + selection_transition.current_segmentation_with_cases; - progress.completed = HelpcodeUtils::count_han_chars(progress.word) * 2 == progress.pinyin.size(); + progress.completed = !selection_transition.continues_composition; + const auto all_segments = quanpin::split_segments(progress.pinyin); + progress.can_store = + progress.completed && !progress.pinyin.empty() && + all_segments.size() == HelpcodeUtils::count_han_chars(progress.word); return progress; } diff --git a/server/src/session/shuangpin_input_session.h b/server/src/session/shuangpin_input_session.h index c3d91c47..8c9a0dff 100644 --- a/server/src/session/shuangpin_input_session.h +++ b/server/src/session/shuangpin_input_session.h @@ -33,12 +33,15 @@ class ShuangpinInputSession : public IInputSession void set_pinyin_sequence_with_cases(const std::string &pinyin_sequence) override; int store_user_phrase(std::string pinyin, std::string word) override; + int store_user_phrase_from_canonical_pinyin(std::string pinyin, std::string word) override; int pin_candidate(std::string pinyin, std::string word) override; int remove_candidate(std::string pinyin, std::string word) override; int cache_dynamic_candidate(const std::string &pinyin, const std::string &word, CandidateSource source) override; - SelectionTransition advance_composition_after_selection(const std::string &selected_pinyin, - const std::string &selected_word) override; + SelectionTransition advance_composition_after_selection( + const std::string &selected_pinyin, + const std::string &selected_word, + const std::string &selected_canonical_pinyin) override; CloudQueryState get_cloud_query_state() const override; CreatingWordProgress update_creating_word_progress(const std::string ¤t_pinyin, const std::string ¤t_word, diff --git a/server/tests/src/test_engine_shuangpin_session.cpp b/server/tests/src/test_engine_shuangpin_session.cpp index 8a13b4b4..bc0de011 100644 --- a/server/tests/src/test_engine_shuangpin_session.cpp +++ b/server/tests/src/test_engine_shuangpin_session.cpp @@ -1,5 +1,7 @@ #include "tests/includes/test_framework.h" #include "src/session/engine_input_session.h" +#include "MetasequoiaImeEngine/common/helpcode_utils.h" +#include "MetasequoiaImeEngine/quanpin/quanpin_query.h" #include "MetasequoiaImeEngine/shuangpin/shuangpin_dictionary.h" #include @@ -14,6 +16,21 @@ void InputLetters(EngineInputSession &session, const std::string &keys) session.handle_key(static_cast(upper), 0, static_cast(ch)); } } + +void InputSequence(EngineInputSession &session, const std::string &keys) +{ + for (const char ch : keys) + { + if (ch == '\'') + { + session.handle_key(VK_OEM_7, 0, L'\''); + continue; + } + const bool is_upper = ch >= 'A' && ch <= 'Z'; + const char upper = is_upper ? ch : static_cast(ch - ('a' - 'A')); + session.handle_key(static_cast(upper), 0, static_cast(ch)); + } +} } TEST_CASE(EngineShuangpinSessionContinuesCompositionWithoutHelpcode) @@ -21,7 +38,7 @@ TEST_CASE(EngineShuangpinSessionContinuesCompositionWithoutHelpcode) EngineInputSession session(SchemeType::Shuangpin); InputLetters(session, "xitele"); - const auto transition = session.advance_composition_after_selection("xi", "西"); + const auto transition = session.advance_composition_after_selection("xi", "西", "xi"); REQUIRE(transition.continues_composition); REQUIRE_EQ(session.get_pinyin_sequence(), std::string("tele")); } @@ -31,7 +48,7 @@ TEST_CASE(EngineShuangpinSessionContinuesCompositionWithSingleHelpcode) EngineInputSession session(SchemeType::Shuangpin); InputLetters(session, "xitelea"); - const auto transition = session.advance_composition_after_selection("xi", "西"); + const auto transition = session.advance_composition_after_selection("xi", "西", "xi"); REQUIRE(transition.continues_composition); REQUIRE_EQ(session.get_pinyin_sequence(), std::string("tele")); } @@ -41,7 +58,7 @@ TEST_CASE(EngineShuangpinSessionContinuesCompositionWithDoubleHelpcode) EngineInputSession session(SchemeType::Shuangpin); InputLetters(session, "xiteleaA"); - const auto transition = session.advance_composition_after_selection("xi", "西"); + const auto transition = session.advance_composition_after_selection("xi", "西", "xi"); REQUIRE(transition.continues_composition); REQUIRE_EQ(session.get_pinyin_sequence(), std::string("tele")); } @@ -126,7 +143,7 @@ TEST_CASE(EngineQuanpinSessionContinuesCompositionForCreatingWord) EngineInputSession session(SchemeType::Quanpin); InputLetters(session, "xitele"); - const auto transition = session.advance_composition_after_selection("xi", "西"); + const auto transition = session.advance_composition_after_selection("xi", "西", "xi"); REQUIRE(transition.continues_composition); REQUIRE_EQ(session.get_pinyin_sequence(), std::string("tele")); REQUIRE_EQ(session.get_pinyin_segmentation_with_cases(), std::string("te'le")); @@ -137,34 +154,193 @@ TEST_CASE(EngineQuanpinSessionCompletesCreatingWordProgress) EngineInputSession session(SchemeType::Quanpin); InputLetters(session, "xitele"); - const auto first_transition = session.advance_composition_after_selection("xi", "西"); + const auto first_transition = session.advance_composition_after_selection("xi", "西", "xi"); const auto first_progress = session.update_creating_word_progress("", "", "西", first_transition); REQUIRE(!first_progress.completed); - REQUIRE_EQ(first_progress.pinyin, std::string("xitele")); + REQUIRE_EQ(first_progress.pinyin, std::string("xi")); REQUIRE_EQ(first_progress.word, std::string("西")); REQUIRE_EQ(first_progress.preedit, std::string("西te'le")); - const auto second_transition = session.advance_composition_after_selection("te'le", "特乐"); + const auto second_transition = + session.advance_composition_after_selection("te'le", "特乐", "te'le"); REQUIRE(!second_transition.continues_composition); const auto second_progress = session.update_creating_word_progress(first_progress.pinyin, first_progress.word, "特乐", second_transition); REQUIRE(second_progress.completed); - REQUIRE_EQ(second_progress.pinyin, std::string("xitele")); + REQUIRE(second_progress.can_store); + REQUIRE_EQ(second_progress.pinyin, std::string("xi'te'le")); REQUIRE_EQ(second_progress.word, std::string("西特乐")); } +TEST_CASE(EngineQuanpinAbbreviationsContinueAndCreateWithCanonicalPinyin) +{ + EngineInputSession session(SchemeType::Quanpin); + InputLetters(session, "zgrm"); + + const auto first = + session.advance_composition_after_selection("z'g", "中国", "zhong'guo"); + REQUIRE(first.continues_composition); + REQUIRE_EQ(session.get_pinyin_sequence(), std::string("rm")); + REQUIRE_EQ(session.get_pinyin_segmentation_with_cases(), std::string("r'm")); + + const auto first_progress = + session.update_creating_word_progress("", "", "中国", first); + REQUIRE(!first_progress.completed); + REQUIRE_EQ(first_progress.pinyin, std::string("zhong'guo")); + REQUIRE_EQ(first_progress.word, std::string("中国")); + + const auto second = + session.advance_composition_after_selection("r'm", "人民", "ren'min"); + REQUIRE(!second.continues_composition); + const auto completed = session.update_creating_word_progress( + first_progress.pinyin, first_progress.word, "人民", second); + REQUIRE(completed.completed); + REQUIRE(completed.can_store); + REQUIRE_EQ(completed.pinyin, std::string("zhong'guo'ren'min")); + REQUIRE_EQ(completed.word, std::string("中国人民")); +} + +TEST_CASE(EngineQuanpinAbbreviationCandidatesRetainDatabaseCanonicalPinyin) +{ + EngineInputSession session(SchemeType::Quanpin); + InputLetters(session, "zgrm"); + + const auto &candidates = session.get_candidates(); + const auto candidate = std::find_if(candidates.begin(), candidates.end(), [](const IInputSession::WordItem &item) { + return item.source == CandidateSource::Database && item.pinyin != item.canonical_pinyin; + }); + REQUIRE(candidate != candidates.end()); + REQUIRE(!candidate->canonical_pinyin.empty()); + REQUIRE_EQ(quanpin::split_segments(candidate->canonical_pinyin).size(), + HelpcodeUtils::count_han_chars(candidate->word)); +} + +TEST_CASE(EngineQuanpinSelectionPreservesManualSeparatorsInRemainingInput) +{ + EngineInputSession session(SchemeType::Quanpin); + InputSequence(session, "zgrm'gh"); + + const auto first = + session.advance_composition_after_selection("z'g", "中国", "zhong'guo"); + REQUIRE(first.continues_composition); + REQUIRE_EQ(session.get_pinyin_sequence(), std::string("rm'gh")); + REQUIRE_EQ(session.get_pinyin_sequence_with_cases(), std::string("rm'gh")); + + const auto second = + session.advance_composition_after_selection("r'm", "人民", "ren'min"); + REQUIRE(second.continues_composition); + REQUIRE_EQ(session.get_pinyin_sequence(), std::string("gh")); +} + +TEST_CASE(EngineQuanpinSelectionConsumesOnlyTheLeadingManualBoundary) +{ + EngineInputSession session(SchemeType::Quanpin); + InputSequence(session, "zg'rm'gh"); + + const auto transition = + session.advance_composition_after_selection("z'g", "中国", "zhong'guo"); + REQUIRE(transition.continues_composition); + REQUIRE_EQ(session.get_pinyin_sequence(), std::string("rm'gh")); +} + +TEST_CASE(EngineShuangpinIncompleteManualSegmentsContinueCreatingWord) +{ + EngineInputSession session(SchemeType::Shuangpin); + InputSequence(session, "v'x'r'm"); + + const auto first = + session.advance_composition_after_selection("v", "中", "zhong"); + REQUIRE(first.continues_composition); + REQUIRE_EQ(session.get_pinyin_sequence(), std::string("x'r'm")); + + const auto first_progress = + session.update_creating_word_progress("", "", "中", first); + REQUIRE(!first_progress.completed); + REQUIRE_EQ(first_progress.pinyin, std::string("zhong")); + REQUIRE_EQ(first_progress.word, std::string("中")); + + const auto second = + session.advance_composition_after_selection("x", "西", "xi"); + REQUIRE(second.continues_composition); + REQUIRE_EQ(session.get_pinyin_sequence(), std::string("r'm")); + const auto second_progress = session.update_creating_word_progress( + first_progress.pinyin, first_progress.word, "西", second); + + const auto third = + session.advance_composition_after_selection("r", "人", "ren"); + REQUIRE(third.continues_composition); + REQUIRE_EQ(session.get_pinyin_sequence(), std::string("m")); + const auto third_progress = session.update_creating_word_progress( + second_progress.pinyin, second_progress.word, "人", third); + + const auto fourth = + session.advance_composition_after_selection("m", "民", "min"); + REQUIRE(!fourth.continues_composition); + const auto completed = session.update_creating_word_progress( + third_progress.pinyin, third_progress.word, "民", fourth); + REQUIRE(completed.completed); + REQUIRE(completed.can_store); + REQUIRE_EQ(completed.pinyin, std::string("zhong'xi'ren'min")); + REQUIRE_EQ(completed.word, std::string("中西人民")); +} + +TEST_CASE(EngineQuanpinIncompleteUppercaseSuffixIsNotConsumedAsHelpcode) +{ + EngineInputSession session(SchemeType::Quanpin); + InputSequence(session, "zgR"); + + const auto transition = + session.advance_composition_after_selection("z", "中", "zhong"); + REQUIRE(transition.continues_composition); + REQUIRE_EQ(session.get_pinyin_sequence(), std::string("gr")); + REQUIRE_EQ(session.get_pinyin_sequence_with_cases(), std::string("gR")); +} + +TEST_CASE(EngineQuanpinCompleteHelpcodeIsDiscardedButManualRemainderIsPreserved) +{ + EngineInputSession session(SchemeType::Quanpin); + InputSequence(session, "ni'shuo'neNV"); + + const auto transition = + session.advance_composition_after_selection("ni'shuo", "你说", "ni'shuo"); + REQUIRE(transition.continues_composition); + REQUIRE_EQ(session.get_pinyin_sequence(), std::string("ne")); + REQUIRE_EQ(session.get_pinyin_sequence_with_cases(), std::string("ne")); +} + +TEST_CASE(EngineCreatingWordDoesNotStoreWhenAnySelectedPartLacksCanonicalPinyin) +{ + EngineInputSession session(SchemeType::Quanpin); + InputLetters(session, "zgrm"); + + const auto first = session.advance_composition_after_selection("z'g", "中国", ""); + REQUIRE(first.continues_composition); + const auto first_progress = + session.update_creating_word_progress("", "", "中国", first); + REQUIRE(first_progress.pinyin.empty()); + + const auto second = + session.advance_composition_after_selection("r'm", "人民", "ren'min"); + const auto completed = session.update_creating_word_progress( + first_progress.pinyin, first_progress.word, "人民", second); + REQUIRE(completed.completed); + REQUIRE(!completed.can_store); + REQUIRE(completed.pinyin.empty()); +} + TEST_CASE(EngineQuanpinSessionContinuesCompositionAfterMultiSyllableSelection) { EngineInputSession session(SchemeType::Quanpin); InputLetters(session, "zhengxianghuafen"); - const auto transition = session.advance_composition_after_selection("zheng'xiang", "正向"); + const auto transition = + session.advance_composition_after_selection("zheng'xiang", "正向", "zheng'xiang"); REQUIRE(transition.continues_composition); REQUIRE_EQ(session.get_pinyin_sequence(), std::string("huafen")); REQUIRE_EQ(session.get_pinyin_segmentation_with_cases(), std::string("hua'fen")); const auto progress = session.update_creating_word_progress("", "", "正向", transition); - REQUIRE_EQ(progress.pinyin, std::string("zhengxianghuafen")); + REQUIRE_EQ(progress.pinyin, std::string("zheng'xiang")); REQUIRE_EQ(progress.word, std::string("正向")); REQUIRE_EQ(progress.preedit, std::string("正向hua'fen")); } @@ -174,14 +350,15 @@ TEST_CASE(EngineQuanpinSessionContinuesCompositionWithoutRetainingHelpcodes) EngineInputSession session(SchemeType::Quanpin); InputLetters(session, "nishuoneNV"); - const auto transition = session.advance_composition_after_selection("ni'shuo", "你说"); + const auto transition = + session.advance_composition_after_selection("ni'shuo", "你说", "ni'shuo"); REQUIRE(transition.continues_composition); REQUIRE_EQ(session.get_pinyin_sequence(), std::string("ne")); REQUIRE_EQ(session.get_pinyin_sequence_with_cases(), std::string("ne")); REQUIRE_EQ(session.get_pinyin_segmentation_with_cases(), std::string("ne")); const auto progress = session.update_creating_word_progress("", "", "你说", transition); - REQUIRE_EQ(progress.pinyin, std::string("nishuone")); + REQUIRE_EQ(progress.pinyin, std::string("ni'shuo")); REQUIRE_EQ(progress.word, std::string("你说")); REQUIRE_EQ(progress.preedit, std::string("你说ne")); } @@ -291,7 +468,7 @@ TEST_CASE(EngineShuangpinInitialVQueriesZhCandidatesAndExpands) const auto &initial_candidates = session.get_candidates(); REQUIRE_EQ(initial_candidates.size(), static_cast(24)); REQUIRE(std::all_of(initial_candidates.begin(), initial_candidates.end(), [](const IInputSession::WordItem &item) { - return item.pinyin.rfind("zh", 0) == 0; + return item.pinyin == "v" && item.canonical_pinyin.rfind("zh", 0) == 0; })); const auto initial_count = initial_candidates.size(); @@ -299,7 +476,7 @@ TEST_CASE(EngineShuangpinInitialVQueriesZhCandidatesAndExpands) const auto &expanded_candidates = session.get_candidates(); REQUIRE(expanded_candidates.size() > initial_count); REQUIRE(std::all_of(expanded_candidates.begin(), expanded_candidates.end(), [](const IInputSession::WordItem &item) { - return item.pinyin.rfind("zh", 0) == 0; + return item.pinyin == "v" && item.canonical_pinyin.rfind("zh", 0) == 0; })); }