diff --git a/server/MetasequoiaImeEngine b/server/MetasequoiaImeEngine index aada04c4..2422fd13 160000 --- a/server/MetasequoiaImeEngine +++ b/server/MetasequoiaImeEngine @@ -1 +1 @@ -Subproject commit aada04c4c98487dd5117a7461c0ded8500435cc5 +Subproject commit 2422fd13838aad63dd74efbaf35706e53693d091 diff --git a/server/assets/config/config.toml b/server/assets/config/config.toml index 60060f9a..076a56e6 100644 --- a/server/assets/config/config.toml +++ b/server/assets/config/config.toml @@ -189,6 +189,15 @@ quanpin_helpcode_schema = "ziranma" # 是否在候选窗口中显示全拼辅助码 show_qp_helpcode_in_candidate_window = true +# +# Quanpin +# 全拼 +# +[quanpin] +# 全拼自动纠错：遇到按键窜乱（相邻字母交换，如 sahng）或邻键误按（如 shabg）时， +# 自动按最接近的正确拼音查询候选。仅当输入串不是合法拼音组合时触发，不影响正常输入。 +autocorrect = true + # # Keybindings # 快捷键 diff --git a/server/scripts/generate_quanpin_autocorrect.py b/server/scripts/generate_quanpin_autocorrect.py new file mode 100644 index 00000000..32a261f6 --- /dev/null +++ b/server/scripts/generate_quanpin_autocorrect.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 +"""Generate MetasequoiaImeEngine/quanpin/autocorrect_table.h from quanpin_utils.cpp. + +The intact pinyin list is parsed directly from the engine source, so the +generated table can never drift from the runtime syllable list. + +Usage: + python scripts/generate_quanpin_autocorrect.py # regenerate the header + python scripts/generate_quanpin_autocorrect.py --check # verify the header is up to date (CI) +""" + +from __future__ import annotations + +import pathlib +import re +import sys + +ENGINE_DIR = pathlib.Path(__file__).resolve().parent.parent / "MetasequoiaImeEngine" +UTILS_PATH = ENGINE_DIR / "quanpin" / "quanpin_utils.cpp" +OUT_PATH = ENGINE_DIR / "quanpin" / "autocorrect_table.h" + +# QWERTY letter-key neighbors (finger-movement range, letter keys only). +QWERTY_NEIGHBORS = { + "q": ["w", "a"], + "w": ["q", "e", "s", "a"], + "e": ["w", "r", "s", "d"], + "r": ["e", "t", "d", "f"], + "t": ["r", "y", "f", "g"], + "y": ["t", "u", "g", "h"], + "u": ["y", "i", "h", "j"], + "i": ["u", "o", "j", "k"], + "o": ["i", "p", "k", "l"], + "p": ["o", "l"], + "a": ["q", "w", "s", "z"], + "s": ["a", "w", "d", "e", "x", "z"], + "d": ["s", "e", "r", "f", "c", "x"], + "f": ["d", "r", "t", "g", "v", "c"], + "g": ["f", "t", "y", "h", "b", "v"], + "h": ["g", "y", "u", "j", "n", "b"], + "j": ["h", "u", "i", "k", "m", "n"], + "k": ["j", "i", "o", "l", "m"], + "l": ["k", "o", "p"], + "z": ["a", "s", "x"], + "x": ["z", "s", "d", "c"], + "c": ["x", "d", "f", "v"], + "v": ["c", "f", "g", "b"], + "b": ["v", "g", "h", "n"], + "n": ["b", "h", "j", "m"], + "m": ["n", "j", "k"], +} + +HEADER_TEMPLATE = """\ +// Generated by scripts/generate_quanpin_autocorrect.py -- DO NOT EDIT. +// Quanpin typing autocorrection table: wrong spelling -> correct syllable. +// Rules: single adjacent-letter transposition, or single neighbor-key +// substitution on a QWERTY layout, applied to one legal syllable. +// Every key is a non-legal pinyin string and maps to exactly one syllable. +#pragma once + +#include + +namespace quanpin::autocorrect +{{ +struct Entry +{{ + const char *wrong; + const char *correct; +}}; + +inline constexpr Entry kEntries[] = {{ +{entries} +}}; + +inline constexpr std::size_t kEntryCount = sizeof(kEntries) / sizeof(kEntries[0]); + +}} // namespace quanpin::autocorrect +""" + + +def extract_intact_pinyin_list() -> list[str]: + text = UTILS_PATH.read_text(encoding="utf-8") + match = re.search(r"intact_pinyin_list\(\)[\s\S]*?kList = \{([\s\S]*?)\};", text) + if not match: + raise SystemExit("ERROR: cannot find intact_pinyin_list in " + str(UTILS_PATH)) + return re.findall(r'"([a-z]+)"', match.group(1)) + + +def build_entries(syllables: list[str]) -> dict[str, str]: + valid = set(syllables) + mapping: dict[str, str] = {} + + def add_variant(variant: str, correct: str) -> None: + if ( + variant == correct + or variant in valid + or len(variant) < 2 + or len(variant) > 6 + or "'" in variant + ): + return + existing = mapping.get(variant) + if existing is None: + mapping[variant] = correct + elif existing != correct: + # Ambiguous: the same wrong string would map to two syllables. + # Drop it entirely so the runtime never guesses. + del mapping[variant] + + for syllable in syllables: + if len(syllable) < 2: + continue + # Adjacent transpositions: sahng -> shang. + for i in range(len(syllable) - 1): + variant = syllable[:i] + syllable[i + 1] + syllable[i] + syllable[i + 2 :] + add_variant(variant, syllable) + # Neighbor-key substitutions: shabg -> shang (n -> b). + for i, ch in enumerate(syllable): + for neighbor in QWERTY_NEIGHBORS.get(ch, ()): + add_variant(syllable[:i] + neighbor + syllable[i + 1 :], syllable) + + return mapping + + +def render_header(entries: dict[str, str]) -> str: + lines = [] + for wrong, correct in sorted(entries.items()): + lines.append(f' {{"{wrong}", "{correct}"}},') + return HEADER_TEMPLATE.format(entries="\n".join(lines)) + + +def main() -> int: + syllables = extract_intact_pinyin_list() + if not syllables: + raise SystemExit("ERROR: extracted empty pinyin list") + entries = build_entries(syllables) + rendered = render_header(entries) + + if "--check" in sys.argv[1:]: + if not OUT_PATH.exists(): + raise SystemExit(f"ERROR: {OUT_PATH} is missing") + if OUT_PATH.read_text(encoding="utf-8") != rendered: + raise SystemExit(f"ERROR: {OUT_PATH} is stale; rerun generate_quanpin_autocorrect.py") + print(f"OK: {OUT_PATH} is up to date ({len(entries)} entries)") + return 0 + + OUT_PATH.write_text(rendered, encoding="utf-8") + print(f"Wrote {OUT_PATH} with {len(entries)} entries " + f"(from {len(syllables)} legal syllables)") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) \ No newline at end of file diff --git a/server/src/config/ime_config.cpp b/server/src/config/ime_config.cpp index 199d7f6b..4999e624 100644 --- a/server/src/config/ime_config.cpp +++ b/server/src/config/ime_config.cpp @@ -70,6 +70,7 @@ std::string g_shuangpin_helpcode_schema = "lantian"; std::string g_quanpin_helpcode_schema = "lantian"; bool g_show_shuangpin_helpcode_in_candidate_window = true; bool g_show_quanpin_helpcode_in_candidate_window = true; +bool g_quanpin_autocorrect_enabled = true; bool g_floating_toolbar_enabled = true; FloatingToolbarItemsConfig g_floating_toolbar_items; double g_floating_toolbar_scale = 1.0; @@ -665,6 +666,7 @@ bool LoadImeConfig() tbl["helpcode"]["show_sp_helpcode_in_candidate_window"].value_or(true); g_show_quanpin_helpcode_in_candidate_window = tbl["helpcode"]["show_qp_helpcode_in_candidate_window"].value_or(true); + g_quanpin_autocorrect_enabled = tbl["quanpin"]["autocorrect"].value_or(true); g_floating_toolbar_enabled = tbl["general"]["floating_toolbar"].value_or(true); // Read the old candidate-only key as a migration fallback. New writes // use the unified key. @@ -1826,6 +1828,21 @@ bool SetConfiguredQuanpinHelpcodeEnabled(bool enabled) return true; } +bool GetConfiguredQuanpinAutocorrectEnabled() +{ + return g_quanpin_autocorrect_enabled; +} + +bool SetConfiguredQuanpinAutocorrectEnabled(bool enabled) +{ + if (!WriteConfiguredValue("quanpin", "autocorrect", enabled ? "true" : "false")) + { + return false; + } + g_quanpin_autocorrect_enabled = enabled; + return true; +} + const std::string &GetConfiguredQuanpinHelpcodeSchema() { return g_quanpin_helpcode_schema; diff --git a/server/src/config/ime_config.h b/server/src/config/ime_config.h index dfcff364..258dbc04 100644 --- a/server/src/config/ime_config.h +++ b/server/src/config/ime_config.h @@ -191,6 +191,8 @@ const std::string &GetConfiguredShuangpinHelpcodeSchema(); bool SetConfiguredShuangpinHelpcodeSchema(const std::string &schema); bool GetConfiguredQuanpinHelpcodeEnabled(); bool SetConfiguredQuanpinHelpcodeEnabled(bool enabled); +bool GetConfiguredQuanpinAutocorrectEnabled(); +bool SetConfiguredQuanpinAutocorrectEnabled(bool enabled); const std::string &GetConfiguredQuanpinHelpcodeSchema(); bool SetConfiguredQuanpinHelpcodeSchema(const std::string &schema); bool GetConfiguredShowShuangpinHelpcodeInCandidateWindow(); diff --git a/server/src/session/engine_input_session.cpp b/server/src/session/engine_input_session.cpp index 67d5a603..49407c49 100644 --- a/server/src/session/engine_input_session.cpp +++ b/server/src/session/engine_input_session.cpp @@ -185,6 +185,7 @@ EngineInputSession::EngineInputSession(SchemeType scheme_type, const ShuangpinPr SelectConfiguredHelpcodeSchema(scheme_type); session_.set_shuangpin_helpcode_enabled(GetConfiguredShuangpinHelpcodeEnabled()); session_.set_quanpin_helpcode_enabled(GetConfiguredQuanpinHelpcodeEnabled()); + session_.set_quanpin_autocorrect_enabled(GetConfiguredQuanpinAutocorrectEnabled()); } void EngineInputSession::handle_key(UINT vk, UINT modifiers_down, WCHAR wch) @@ -192,6 +193,7 @@ void EngineInputSession::handle_key(UINT vk, UINT modifiers_down, WCHAR wch) SelectConfiguredHelpcodeSchema(current_scheme_type()); session_.set_shuangpin_helpcode_enabled(GetConfiguredShuangpinHelpcodeEnabled()); session_.set_quanpin_helpcode_enabled(GetConfiguredQuanpinHelpcodeEnabled()); + session_.set_quanpin_autocorrect_enabled(GetConfiguredQuanpinAutocorrectEnabled()); session_.handle_key(vk, modifiers_down, wch); } @@ -200,6 +202,7 @@ void EngineInputSession::recompute_candidates() SelectConfiguredHelpcodeSchema(current_scheme_type()); session_.set_shuangpin_helpcode_enabled(GetConfiguredShuangpinHelpcodeEnabled()); session_.set_quanpin_helpcode_enabled(GetConfiguredQuanpinHelpcodeEnabled()); + session_.set_quanpin_autocorrect_enabled(GetConfiguredQuanpinAutocorrectEnabled()); if (has_pending_pinyin_sequence_ || has_pending_pinyin_sequence_with_cases_) { apply_pending_sequence(); diff --git a/server/tests/src/test_quanpin_scheme.cpp b/server/tests/src/test_quanpin_scheme.cpp index 7327634c..dec5f985 100644 --- a/server/tests/src/test_quanpin_scheme.cpp +++ b/server/tests/src/test_quanpin_scheme.cpp @@ -1,5 +1,6 @@ #include "tests/includes/test_framework.h" #include "MetasequoiaImeEngine/common/helpcode_utils.h" +#include "MetasequoiaImeEngine/quanpin/autocorrect_table.h" #include "MetasequoiaImeEngine/quanpin/quanpin_dictionary.h" #include "MetasequoiaImeEngine/quanpin/quanpin_query.h" #include "MetasequoiaImeEngine/quanpin/quanpin_utils.h" @@ -8,6 +9,7 @@ #include #include #include +#include #include #include @@ -414,6 +416,167 @@ TEST_CASE(QuanpinSparsePinyinFallbackSegmentsPreserveSuffixSegments) REQUIRE_EQ(quanpin::join_segments(dia_fallbacks[1]), std::string("di")); } +TEST_CASE(QuanpinAutocorrectTableHasNoCollisionsWithLegalPinyin) +{ + const auto &legal = quanpin::intact_pinyin_set(); + std::unordered_set wrong_keys; + for (const auto &entry : quanpin::autocorrect::kEntries) + { + const std::string wrong(entry.wrong); + REQUIRE(!wrong.empty()); + REQUIRE(!legal.count(wrong)); // a key must never shadow a legal syllable + REQUIRE(legal.count(entry.correct)); // the correction must be a legal syllable + REQUIRE(wrong_keys.insert(wrong).second); // keys must be unique + } + REQUIRE(wrong_keys.size() > 1000); +} + +TEST_CASE(QuanpinAutocorrectCutSingleSyllableTranspositions) +{ + REQUIRE_EQ(quanpin::join_segments(quanpin::autocorrect_cut("sahng")), std::string("shang")); + REQUIRE_EQ(quanpin::join_segments(quanpin::autocorrect_cut("hsang")), std::string("shang")); + REQUIRE_EQ(quanpin::join_segments(quanpin::autocorrect_cut("shagn")), std::string("shang")); +} + +TEST_CASE(QuanpinAutocorrectCutNeighborKeySubstitutions) +{ + REQUIRE_EQ(quanpin::join_segments(quanpin::autocorrect_cut("shabg")), std::string("shang")); // n -> b + REQUIRE_EQ(quanpin::join_segments(quanpin::autocorrect_cut("sgang")), std::string("shang")); // h -> g +} + +TEST_CASE(QuanpinAutocorrectCutMixedMultisyllableInput) +{ + REQUIRE_EQ(quanpin::join_segments(quanpin::autocorrect_cut("sahngzhi")), std::string("shang'zhi")); + REQUIRE_EQ(quanpin::join_segments(quanpin::autocorrect_cut("sahngzhk")), std::string("shang'zhi")); // i -> k +} + +TEST_CASE(QuanpinAutocorrectCutPrefersFewestCorrections) +{ + // sahnguai has two plausible readings; the one with fewer corrected edges + // (shan + guai, one mistake) wins over shang + hai (two mistakes). + REQUIRE_EQ(quanpin::join_segments(quanpin::autocorrect_cut("sahnguai")), std::string("shan'guai")); +} + +TEST_CASE(QuanpinAutocorrectCutRejectsInputsOutOfScope) +{ + // Legal input: the caller must only invoke autocorrect when the correction + // cut already failed, so a fully legal spelling yields no correction. + REQUIRE(quanpin::autocorrect_cut("shang").empty()); + REQUIRE(quanpin::autocorrect_cut("keneng").empty()); + // Manual delimiter: user-intended boundary, never rewritten. + REQUIRE(quanpin::autocorrect_cut("xi'an").empty()); + // No correction path: unresolvable garbage. + REQUIRE(quanpin::autocorrect_cut("xxxxx").empty()); + REQUIRE(quanpin::autocorrect_cut("qzzvv").empty()); + // Beyond the per-input correction edge budget. + REQUIRE(quanpin::autocorrect_cut("sahngsahngsahngsahng").empty()); +} + +TEST_CASE(QuanpinAutocorrectCutRespectsEdgeBudget) +{ + REQUIRE_EQ(quanpin::join_segments(quanpin::autocorrect_cut("sahngsahngsahng")), + std::string("shang'shang'shang")); +} + +namespace +{ +std::filesystem::path CreateAutocorrectDatabase() +{ + const auto path = std::filesystem::temp_directory_path() / "msime-quanpin-autocorrect-test.db"; + std::filesystem::remove(path); + sqlite3 *db = nullptr; + if (sqlite3_open(path.string().c_str(), &db) != SQLITE_OK) + { + throw std::runtime_error("Failed to create temporary autocorrect database."); + } + const char *sql = + "CREATE TABLE tbl_1_s(key TEXT,jp TEXT,value TEXT,weight INTEGER);" + "CREATE TABLE tbl_2_s(key TEXT,jp TEXT,value TEXT,weight INTEGER);" + "CREATE TABLE tbl_4_s(key TEXT,jp TEXT,value TEXT,weight INTEGER);" + "INSERT INTO tbl_1_s VALUES('shang','s','上',100);" + "INSERT INTO tbl_2_s VALUES('shang''zhi','sz','上至',100);" + "INSERT INTO tbl_4_s VALUES('sa''huang''na''ge','shng','撒谎那个',1000);"; + const int result = sqlite3_exec(db, sql, nullptr, nullptr, nullptr); + sqlite3_close(db); + if (result != SQLITE_OK) + { + std::filesystem::remove(path); + throw std::runtime_error("Failed to initialize temporary autocorrect database."); + } + return path; +} +} // namespace + +TEST_CASE(QuanpinDictionaryAutocorrectPutsCorrectedCandidateFirst) +{ + const auto db_path = CreateAutocorrectDatabase(); + QuanpinDictionary dictionary(db_path.string()); + + const auto candidates = dictionary.query("sahng", "sa'h'n'g", true); + REQUIRE(!candidates.empty()); + // The corrected candidate leads, and its pinyin is the corrected key so + // that selection/weight updates land on the right dictionary entry. + REQUIRE_EQ(candidates.front().word, std::string("上")); + REQUIRE_EQ(candidates.front().pinyin, std::string("shang")); + REQUIRE_EQ(candidates.front().canonical_pinyin, std::string("shang")); + // The original greedy candidates stay behind as a fallback tail. + REQUIRE(std::any_of(candidates.begin(), candidates.end(), + [](const WordItem &item) { return item.word == "撒谎那个"; })); + REQUIRE(candidates.front().word != "撒谎那个"); +} + +TEST_CASE(QuanpinDictionaryAutocorrectNeighborKeySubstitution) +{ + const auto db_path = CreateAutocorrectDatabase(); + QuanpinDictionary dictionary(db_path.string()); + + const auto candidates = dictionary.query("shabg", "sha'b'g", true); + REQUIRE(!candidates.empty()); + REQUIRE_EQ(candidates.front().word, std::string("上")); +} + +TEST_CASE(QuanpinDictionaryAutocorrectMultisyllableInput) +{ + const auto db_path = CreateAutocorrectDatabase(); + QuanpinDictionary dictionary(db_path.string()); + + const auto candidates = dictionary.query("sahngzhi", "sa'h'n'g'zhi", true); + REQUIRE(!candidates.empty()); + REQUIRE_EQ(candidates.front().word, std::string("上至")); +} + +TEST_CASE(QuanpinDictionaryAutocorrectDisabledKeepsLegacyBehavior) +{ + const auto db_path = CreateAutocorrectDatabase(); + QuanpinDictionary dictionary(db_path.string()); + + const auto candidates = dictionary.query("sahng", "sa'h'n'g", false); + REQUIRE(!candidates.empty()); + REQUIRE(std::none_of(candidates.begin(), candidates.end(), + [](const WordItem &item) { return item.word == "上"; })); + REQUIRE(std::any_of(candidates.begin(), candidates.end(), + [](const WordItem &item) { return item.word == "撒谎那个"; })); +} + +TEST_CASE(QuanpinDictionaryAutocorrectLeavesLegalInputsUntouched) +{ + const auto db_path = CreateAutocorrectDatabase(); + QuanpinDictionary dictionary(db_path.string()); + + // Corrected results live in their own cache slot; a plain spelling never + // sees the fallback tail produced for the corrected spelling. + (void)dictionary.query("sahng", "sa'h'n'g", true); + const auto plain = dictionary.query("shang", "shang", true); + REQUIRE(!plain.empty()); + REQUIRE_EQ(plain.front().word, std::string("上")); + REQUIRE(std::none_of(plain.begin(), plain.end(), + [](const WordItem &item) { return item.word == "撒谎那个"; })); + + const auto manual = dictionary.query("xi'an", "xi'an", true); + const auto manual_off = dictionary.query("xi'an", "xi'an", false); + REQUIRE_EQ(manual.size(), manual_off.size()); +} + namespace { struct HelpcodeSample