diff --git a/engine/core/input_session.h b/engine/core/input_session.h index db700dec..52f0b7b7 100644 --- a/engine/core/input_session.h +++ b/engine/core/input_session.h @@ -180,6 +180,10 @@ class InputSession std::vector segment_raw_boundaries() const; std::string get_quanpin() const; bool is_all_complete_pure_pinyin() const; + // 整串输入能不能读成拼音：全拼按「完整音节或音节前缀」（含简拼声母，bus = bu + s）切完整串， + // 双拼按完整音节码、末尾允许单独一个声母键。不要求音节完整，也不看词库有没有词。混输里用它 + // 决定英文前缀补全词的默认位置：能读成拼音的输入，用户多半在打中文，补全词默认退到首页末位。 + bool reads_as_pinyin() const; // The current composition is a complete four-letter wubi code the wubi table answered with // exactly one row. Hosts decide whether to auto-commit on this; the engine only reports the fact. // The pinyin candidates mixed input appends are deliberately not counted: a code the table did diff --git a/engine/core/input_session_composition.cpp b/engine/core/input_session_composition.cpp index e27a9539..1cec3a7a 100644 --- a/engine/core/input_session_composition.cpp +++ b/engine/core/input_session_composition.cpp @@ -462,6 +462,26 @@ bool InputSession::is_all_complete_pure_pinyin() const return !segmentation.empty() && quanpin::is_complete_pinyin_input(segmentation); } +bool InputSession::reads_as_pinyin() const +{ + if (has_active_helpcode()) + return true; + std::string raw = remove_delimiters(request().raw_input); + std::transform(raw.begin(), raw.end(), raw.begin(), + [](unsigned char ch) { return static_cast(std::tolower(ch)); }); + if (raw.empty()) + return false; + if (is_shuangpin()) + { + // 双拼两键一个音节，打到一半时末尾会剩一个声母键。 + return raw.size() == 1 || shuangpin::is_complete_input(raw, shuangpin_profile_) || + shuangpin::is_complete_input(raw.substr(0, raw.size() - 1), shuangpin_profile_); + } + if (current_scheme_type() == SchemeType::Quanpin) + return !quanpin::cut_one_piece_min_segments(raw, false).empty(); + return false; +} + bool InputSession::wubi_unique_four_code() const { // Both modes spell words rather than codes; a wubi code is not composed inside them. No host diff --git a/engine/tests/src/test_english_input_session.cpp b/engine/tests/src/test_english_input_session.cpp index ed0bac8c..d073d045 100644 --- a/engine/tests/src/test_english_input_session.cpp +++ b/engine/tests/src/test_english_input_session.cpp @@ -239,6 +239,80 @@ int run_test() "AND key='nimbus' AND value='Nimbus' AND operation='upsert'") == 1200, "English ranking was not journaled for upgrade replay."); } + + // 混输列表里的中文候选不能当英文调频的基准：中文词库的权重是另一个量纲，拿它 + 1000 会让 + // 选过一次的英文永久压过「在」这种高频字。 + const std::vector mixed_ranked{WordItem("ni", "你", 19195987, CandidateSource::Database), + WordItem("nimbus", "Nimbus", 1200, CandidateSource::EnglishDictionary), + WordItem("ninja", "Ninja", 200, CandidateSource::EnglishDictionary)}; + require(user_dictionary::adjust_english_candidate_ranking(english_db, user_db, "english:nin", mixed_ranked, + "ninja", "Ninja", "promote", 1, 1, false), + "English ranking in a mixed list failed."); + { + Database database(english_mode_directory / "english.db"); + require(database.query_integer("SELECT weight FROM english_words WHERE word='ninja' AND " + "display='Ninja'") == 2200, + "English ranking used a Chinese candidate's weight as its baseline."); + } + } + { + const std::string user_db = user_dictionary::default_user_db_path(); + require(!user_dictionary::english_slot(user_db, "zai").has_value(), "An unlearned English slot was stored."); + + // 补全词默认在首页末位（下标 6）：每选中一次按 promote 的名次规则前移，能一路升到首位。 + const int expected_promotions[] = {4, 3, 2, 1, 0}; + int english_index = 6; + for (const int expected : expected_promotions) + { + require(user_dictionary::learn_english_slot_selection(user_db, "zai", english_index, "promote", 1, 1) && + user_dictionary::english_slot(user_db, "zai") == expected, + "Selecting the slotted English candidate did not promote it by rank."); + english_index = expected; + } + // 已在首位：没有可学的。 + require(user_dictionary::learn_english_slot_selection(user_db, "zai", 0, "promote", 1, 1) && + user_dictionary::english_slot(user_db, "zai") == 0, + "Selecting a first English candidate changed its slot."); + + // 英文在首位时越过它选了第二个候选（「在」）：英文后退一位，不会赶不走。 + require(user_dictionary::learn_english_slot_bypass(user_db, "zai", 0, 1, "promote", 1, 1, 6) && + user_dictionary::english_slot(user_db, "zai") == 1, + "Bypassing a first English candidate did not demote it."); + // 选的候选离得太远、调频目标没越过英文，不算越过。 + require(user_dictionary::learn_english_slot_bypass(user_db, "zai", 1, 7, "promote", 1, 1, 6) && + user_dictionary::english_slot(user_db, "zai") == 1, + "A pick that does not pass the English candidate demoted it."); + + // trigger_count：累计够次数才动。 + require(user_dictionary::learn_english_slot_bypass(user_db, "bus", 1, 2, "promote", 1, 2, 6) && + !user_dictionary::english_slot(user_db, "bus").has_value(), + "An English slot moved before reaching the trigger count."); + require(user_dictionary::learn_english_slot_bypass(user_db, "bus", 1, 2, "promote", 1, 2, 6) && + user_dictionary::english_slot(user_db, "bus") == 2, + "An English slot did not move at the trigger count."); + + // 已在首页末位不再后退；调频关闭时不学。 + require(user_dictionary::learn_english_slot_bypass(user_db, "bus", 6, 7, "promote", 1, 1, 6) && + user_dictionary::english_slot(user_db, "bus") == 2, + "An English slot moved past the last slot of the first page."); + require(user_dictionary::set_english_slot(user_db, "github", 3) && + user_dictionary::learn_english_slot_selection(user_db, "github", 3, "disabled", 1, 1) && + user_dictionary::english_slot(user_db, "github") == 3, + "English slot learning ran with frequency adjustment disabled."); + } + { + const auto reads_as_pinyin = [](SchemeType scheme, const std::string &input) { + metasequoia::InputSession session(scheme); + type(session, input); + return session.reads_as_pinyin(); + }; + require(reads_as_pinyin(SchemeType::Quanpin, "zai"), "A complete syllable did not read as pinyin."); + require(reads_as_pinyin(SchemeType::Quanpin, "bus"), "A syllable with a jianpin tail did not read as pinyin."); + require(!reads_as_pinyin(SchemeType::Quanpin, "github"), "An English-only spelling read as pinyin."); + require(!reads_as_pinyin(SchemeType::Quanpin, "upstream"), "An English-only spelling read as pinyin."); + require(reads_as_pinyin(SchemeType::Shuangpin, "zd"), "A complete shuangpin code did not read as pinyin."); + require(reads_as_pinyin(SchemeType::Shuangpin, "zdb"), + "A shuangpin code with a pending initial did not read as pinyin."); } dedicated.set_dedicated_english_mode(false); require(!dedicated.dedicated_english_mode() && !dedicated.has_composition(), diff --git a/engine/user_dictionary/user_dictionary_journal.cpp b/engine/user_dictionary/user_dictionary_journal.cpp index 467fca66..c4ab6b1d 100644 --- a/engine/user_dictionary/user_dictionary_journal.cpp +++ b/engine/user_dictionary/user_dictionary_journal.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -131,6 +132,8 @@ bool ensure_schema(sqlite3 *db) "position INTEGER NOT NULL CHECK(position BETWEEN 1 AND 5)," "PRIMARY KEY(context_key,entry_key,value),UNIQUE(context_key,position));" "CREATE TABLE IF NOT EXISTS quick_phrase_slots(" + "code TEXT PRIMARY KEY,slot INTEGER NOT NULL CHECK(slot>=0));" + "CREATE TABLE IF NOT EXISTS english_slots(" "code TEXT PRIMARY KEY,slot INTEGER NOT NULL CHECK(slot>=0));"; if (sqlite3_exec(db, sql, nullptr, nullptr, nullptr) != SQLITE_OK) return false; @@ -626,15 +629,21 @@ bool adjust_english_candidate_ranking(const std::string &english_db_path, const } } - const auto selected = std::find_if(ordered_candidates.begin(), ordered_candidates.end(), [&](const WordItem &item) { - return item.source == CandidateSource::EnglishDictionary && item.pinyin == entry_key && item.word == value; + // 只和英文候选比。混输列表里还有中文，它们的权重是另一个词库的量纲：拿「全表最高 + 1000」 + // 当新权重，选一次英文就会压过「在」这种一千多万的高频字，而且权重按词全局生效，之后凡是 + // 前缀能查出它的输入都受影响。英文排在中文的第几位由按输入记的英文槽位决定（english_slot）。 + std::vector english_candidates; + std::copy_if(ordered_candidates.begin(), ordered_candidates.end(), std::back_inserter(english_candidates), + [](const WordItem &item) { return item.source == CandidateSource::EnglishDictionary; }); + const auto selected = std::find_if(english_candidates.begin(), english_candidates.end(), [&](const WordItem &item) { + return item.pinyin == entry_key && item.word == value; }); - if (selected == ordered_candidates.end()) + if (selected == english_candidates.end()) { rollback_user(); return false; } - const size_t rank = static_cast(selected - ordered_candidates.begin()); + const size_t rank = static_cast(selected - english_candidates.begin()); if (rank == 0) { const bool committed = execute_sql(user_db.get(), "COMMIT"); @@ -652,10 +661,10 @@ bool adjust_english_candidate_ranking(const std::string &english_db_path, const else if (!force_top && mode == "promote") target = rank > 4 ? 4 : rank - 1; const std::int64_t maximum_weight = - std::accumulate(ordered_candidates.begin(), ordered_candidates.end(), std::int64_t{0}, + std::accumulate(english_candidates.begin(), english_candidates.end(), std::int64_t{0}, [](std::int64_t maximum, const WordItem &item) { return (std::max)(maximum, item.weight); }); const std::int64_t new_weight = - target == 0 ? (std::max)(maximum_weight, selected->weight) + 1000 : ordered_candidates[target - 1].weight + 1; + target == 0 ? (std::max)(maximum_weight, selected->weight) + 1000 : english_candidates[target - 1].weight + 1; auto english_db = open_database(english_db_path, SQLITE_OPEN_READWRITE); auto update = english_db @@ -835,11 +844,12 @@ bool write_quick_phrase_slot(sqlite3 *db, const std::string &schema, const std:: } // 计一次数；到 trigger_count 时置 reached 并清零，计数不会越过状态导出允许的上限。 -bool count_quick_phrase_selection(sqlite3 *db, const std::string &schema, const std::string &code, - const char *direction, int trigger_count, bool &reached) +// context_prefix 区分快捷短语组和英文槽位，两者的计数互不干扰。 +bool count_slot_selection(sqlite3 *db, const std::string &schema, const std::string &context_prefix, + const std::string &code, const char *direction, int trigger_count, bool &reached) { reached = false; - const std::string context = kQuickPhraseSelectionContext + code; + const std::string context = context_prefix + code; { auto counter = prepare(db, "INSERT INTO " + schema + "candidate_selection_state(context_key,entry_key,value,selection_count)" @@ -901,7 +911,8 @@ bool learn_quick_phrase_selection(const std::string &main_db_path, const std::st if (slot == 0 && first_in_group) return commit(); bool reached = false; - if (!count_quick_phrase_selection(database.get(), schema, code, "promote", trigger_count, reached)) + if (!count_slot_selection(database.get(), schema, kQuickPhraseSelectionContext, code, "promote", trigger_count, + reached)) return rollback(); if (!reached) return commit(); @@ -963,13 +974,97 @@ bool learn_quick_phrase_bypass(const std::string &user_db_path, const std::strin if (target > static_cast(slot)) return commit(); bool reached = false; - if (!count_quick_phrase_selection(db.get(), "", code, "bypass", trigger_count, reached)) + if (!count_slot_selection(db.get(), "", kQuickPhraseSelectionContext, code, "bypass", trigger_count, reached)) return rollback(); if (reached && !write_quick_phrase_slot(db.get(), "", code, (std::min)(slot + 1, max_slot))) return rollback(); return commit(); } +namespace +{ +const std::string kEnglishSlotSelectionContext = "english_slot:"; + +bool write_english_slot(sqlite3 *db, const std::string &code, int slot) +{ + auto stmt = prepare(db, "INSERT INTO english_slots(code,slot) VALUES(?1,?2)" + " ON CONFLICT(code) DO UPDATE SET slot=excluded.slot"); + return stmt && bind_text(stmt.get(), 1, code) && + sqlite3_bind_int(stmt.get(), 2, (std::max)(0, slot)) == SQLITE_OK && sqlite3_step(stmt.get()) == SQLITE_DONE; +} + +// 选择计数与写槽位放在一个事务里；write_slot 只在计数到 trigger_count 时调用。 +template +bool count_and_write_english_slot(const std::string &user_db_path, const std::string &code, const char *direction, + int trigger_count, WriteSlot write_slot) +{ + UserDatabase db(user_db_path); + if (!db || !execute_sql(db.get(), "BEGIN IMMEDIATE")) + return false; + const auto rollback = [&]() { + (void)execute_sql(db.get(), "ROLLBACK"); + return false; + }; + bool reached = false; + if (!count_slot_selection(db.get(), "", kEnglishSlotSelectionContext, code, direction, trigger_count, reached)) + return rollback(); + if (reached && !write_slot(db.get())) + return rollback(); + return execute_sql(db.get(), "COMMIT") || rollback(); +} +} // namespace + +std::optional english_slot(const std::string &user_db_path, const std::string &code) +{ + if (code.empty()) + return std::nullopt; + UserDatabase db(user_db_path); + auto stmt = db ? prepare(db.get(), "SELECT slot FROM english_slots WHERE code=?1") : Stmt{}; + if (!stmt || !bind_text(stmt.get(), 1, code) || sqlite3_step(stmt.get()) != SQLITE_ROW) + return std::nullopt; + return (std::max)(0, sqlite3_column_int(stmt.get(), 0)); +} + +bool set_english_slot(const std::string &user_db_path, const std::string &code, int slot) +{ + if (code.empty()) + return false; + UserDatabase db(user_db_path); + return db && write_english_slot(db.get(), code, slot); +} + +bool learn_english_slot_selection(const std::string &user_db_path, const std::string &code, int english_index, + const std::string &mode, int linear_step, int trigger_count) +{ + if (code.empty() || english_index < 0) + return false; + // 已经在首位就是默认上屏的那一个，没有可学的。 + if (mode == "disabled" || english_index == 0) + return true; + // 和中文调频同一套名次规则：按它此刻在列表里的名次算目标位置。 + const int target = + static_cast(ranking_target(static_cast(english_index), mode, (std::max)(1, linear_step), false)); + return count_and_write_english_slot(user_db_path, code, "promote", trigger_count, + [&](sqlite3 *db) { return write_english_slot(db, code, target); }); +} + +bool learn_english_slot_bypass(const std::string &user_db_path, const std::string &code, int english_index, + int selected_index, const std::string &mode, int linear_step, int trigger_count, + int max_slot) +{ + if (code.empty() || english_index < 0 || selected_index <= english_index) + return false; + if (mode == "disabled" || english_index >= max_slot) + return true; + // 和快捷短语组同一个判据：选中的候选按调频模式的目标位置落在英文的位置或更前，才算越过了它。 + const size_t target = ranking_target(static_cast(selected_index), mode, (std::max)(1, linear_step), false); + if (target > static_cast(english_index)) + return true; + return count_and_write_english_slot(user_db_path, code, "bypass", trigger_count, [&](sqlite3 *db) { + return write_english_slot(db, code, (std::min)(english_index + 1, max_slot)); + }); +} + bool learn_entered_english_word(const std::string &english_db_path, const std::string &user_db_path, const std::string &display, std::int64_t weight) { diff --git a/engine/user_dictionary/user_dictionary_journal.h b/engine/user_dictionary/user_dictionary_journal.h index b7f217ab..e86f20eb 100644 --- a/engine/user_dictionary/user_dictionary_journal.h +++ b/engine/user_dictionary/user_dictionary_journal.h @@ -55,6 +55,22 @@ bool learn_quick_phrase_selection(const std::string &main_db_path, const std::st bool learn_quick_phrase_bypass(const std::string &user_db_path, const std::string &code, int ordinary_rank, const std::string &mode, int linear_step, int trigger_count, int max_slot); +// 中英混输里英文候选的调频状态，思路同快捷短语：英文词库和中文词库的权重不在同一个量纲上， +// 不能比大小，所以按输入（小写）记一个槽位：首个英文候选在整个候选列表里的下标，0 是首位。 +// 没有记录时由调用方决定默认位置（精确匹配紧跟第一个中文候选，补全词在首页末位）。 +std::optional english_slot(const std::string &user_db_path, const std::string &code); +// 用户显式置顶英文候选：直接写槽位，不走计数。 +bool set_english_slot(const std::string &user_db_path, const std::string &code, int slot); +// 用户选中了占着槽位的英文候选，english_index 是它此刻在列表里的下标。累计到 trigger_count +// 次后，按调频模式从这个名次算出目标位置写成槽位，和中文调频一样可以一路升到首位。 +bool learn_english_slot_selection(const std::string &user_db_path, const std::string &code, int english_index, + const std::string &mode, int linear_step, int trigger_count); +// 用户越过英文候选选了排在它后面的候选。english_index / selected_index 是两者在列表里的下标， +// 选中的候选按调频模式的目标位置越过英文时，累计到 trigger_count 次后英文后退一位，最多到 max_slot。 +bool learn_english_slot_bypass(const std::string &user_db_path, const std::string &code, int english_index, + int selected_index, const std::string &mode, int linear_step, int trigger_count, + int max_slot); + struct ReplayResult { int applied = 0;