feat(quanpin): wire up typing autocorrect config and tests
- config.toml [quanpin] autocorrect = true (default on) - ime_config: GetConfiguredQuanpinAutocorrectEnabled / SetConfiguredQuanpinAutocorrectEnabled with template-preserving write - engine_input_session injects the flag into the engine session - tests: autocorrect table integrity, autocorrect_cut units, dictionary-level corrected-candidate-first behavior, legacy behavior when disabled, and cache isolation for plain spellings - bump MetasequoiaImeEngine to 2422fd1
This commit is contained in:
1 parent
9e4296d1c5
commit
bfd42d314e
7 files changed
+348
-1
No files matched your search
Submodule server/MetasequoiaImeEngine updated: aada04c4c9...2422fd1383.
@@ -189,6 +189,15 @@ quanpin_helpcode_schema = "ziranma"
|
||||
# 是否在候选窗口中显示全拼辅助码
|
||||
show_qp_helpcode_in_candidate_window = true
|
||||
|
||||
#
|
||||
# Quanpin
|
||||
# 全拼
|
||||
#
|
||||
[quanpin]
|
||||
# 全拼自动纠错:遇到按键窜乱(相邻字母交换,如 sahng)或邻键误按(如 shabg)时,
|
||||
# 自动按最接近的正确拼音查询候选。仅当输入串不是合法拼音组合时触发,不影响正常输入。
|
||||
autocorrect = true
|
||||
|
||||
#
|
||||
# Keybindings
|
||||
# 快捷键
|
||||
|
||||
@@ -0,0 +1,153 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate MetasequoiaImeEngine/quanpin/autocorrect_table.h from quanpin_utils.cpp.
|
||||
|
||||
The intact pinyin list is parsed directly from the engine source, so the
|
||||
generated table can never drift from the runtime syllable list.
|
||||
|
||||
Usage:
|
||||
python scripts/generate_quanpin_autocorrect.py # regenerate the header
|
||||
python scripts/generate_quanpin_autocorrect.py --check # verify the header is up to date (CI)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pathlib
|
||||
import re
|
||||
import sys
|
||||
|
||||
ENGINE_DIR = pathlib.Path(__file__).resolve().parent.parent / "MetasequoiaImeEngine"
|
||||
UTILS_PATH = ENGINE_DIR / "quanpin" / "quanpin_utils.cpp"
|
||||
OUT_PATH = ENGINE_DIR / "quanpin" / "autocorrect_table.h"
|
||||
|
||||
# QWERTY letter-key neighbors (finger-movement range, letter keys only).
|
||||
QWERTY_NEIGHBORS = {
|
||||
"q": ["w", "a"],
|
||||
"w": ["q", "e", "s", "a"],
|
||||
"e": ["w", "r", "s", "d"],
|
||||
"r": ["e", "t", "d", "f"],
|
||||
"t": ["r", "y", "f", "g"],
|
||||
"y": ["t", "u", "g", "h"],
|
||||
"u": ["y", "i", "h", "j"],
|
||||
"i": ["u", "o", "j", "k"],
|
||||
"o": ["i", "p", "k", "l"],
|
||||
"p": ["o", "l"],
|
||||
"a": ["q", "w", "s", "z"],
|
||||
"s": ["a", "w", "d", "e", "x", "z"],
|
||||
"d": ["s", "e", "r", "f", "c", "x"],
|
||||
"f": ["d", "r", "t", "g", "v", "c"],
|
||||
"g": ["f", "t", "y", "h", "b", "v"],
|
||||
"h": ["g", "y", "u", "j", "n", "b"],
|
||||
"j": ["h", "u", "i", "k", "m", "n"],
|
||||
"k": ["j", "i", "o", "l", "m"],
|
||||
"l": ["k", "o", "p"],
|
||||
"z": ["a", "s", "x"],
|
||||
"x": ["z", "s", "d", "c"],
|
||||
"c": ["x", "d", "f", "v"],
|
||||
"v": ["c", "f", "g", "b"],
|
||||
"b": ["v", "g", "h", "n"],
|
||||
"n": ["b", "h", "j", "m"],
|
||||
"m": ["n", "j", "k"],
|
||||
}
|
||||
|
||||
HEADER_TEMPLATE = """\
|
||||
// Generated by scripts/generate_quanpin_autocorrect.py -- DO NOT EDIT.
|
||||
// Quanpin typing autocorrection table: wrong spelling -> correct syllable.
|
||||
// Rules: single adjacent-letter transposition, or single neighbor-key
|
||||
// substitution on a QWERTY layout, applied to one legal syllable.
|
||||
// Every key is a non-legal pinyin string and maps to exactly one syllable.
|
||||
#pragma once
|
||||
|
||||
#include <cstddef>
|
||||
|
||||
namespace quanpin::autocorrect
|
||||
{{
|
||||
struct Entry
|
||||
{{
|
||||
const char *wrong;
|
||||
const char *correct;
|
||||
}};
|
||||
|
||||
inline constexpr Entry kEntries[] = {{
|
||||
{entries}
|
||||
}};
|
||||
|
||||
inline constexpr std::size_t kEntryCount = sizeof(kEntries) / sizeof(kEntries[0]);
|
||||
|
||||
}} // namespace quanpin::autocorrect
|
||||
"""
|
||||
|
||||
|
||||
def extract_intact_pinyin_list() -> list[str]:
|
||||
text = UTILS_PATH.read_text(encoding="utf-8")
|
||||
match = re.search(r"intact_pinyin_list\(\)[\s\S]*?kList = \{([\s\S]*?)\};", text)
|
||||
if not match:
|
||||
raise SystemExit("ERROR: cannot find intact_pinyin_list in " + str(UTILS_PATH))
|
||||
return re.findall(r'"([a-z]+)"', match.group(1))
|
||||
|
||||
|
||||
def build_entries(syllables: list[str]) -> dict[str, str]:
|
||||
valid = set(syllables)
|
||||
mapping: dict[str, str] = {}
|
||||
|
||||
def add_variant(variant: str, correct: str) -> None:
|
||||
if (
|
||||
variant == correct
|
||||
or variant in valid
|
||||
or len(variant) < 2
|
||||
or len(variant) > 6
|
||||
or "'" in variant
|
||||
):
|
||||
return
|
||||
existing = mapping.get(variant)
|
||||
if existing is None:
|
||||
mapping[variant] = correct
|
||||
elif existing != correct:
|
||||
# Ambiguous: the same wrong string would map to two syllables.
|
||||
# Drop it entirely so the runtime never guesses.
|
||||
del mapping[variant]
|
||||
|
||||
for syllable in syllables:
|
||||
if len(syllable) < 2:
|
||||
continue
|
||||
# Adjacent transpositions: sahng -> shang.
|
||||
for i in range(len(syllable) - 1):
|
||||
variant = syllable[:i] + syllable[i + 1] + syllable[i] + syllable[i + 2 :]
|
||||
add_variant(variant, syllable)
|
||||
# Neighbor-key substitutions: shabg -> shang (n -> b).
|
||||
for i, ch in enumerate(syllable):
|
||||
for neighbor in QWERTY_NEIGHBORS.get(ch, ()):
|
||||
add_variant(syllable[:i] + neighbor + syllable[i + 1 :], syllable)
|
||||
|
||||
return mapping
|
||||
|
||||
|
||||
def render_header(entries: dict[str, str]) -> str:
|
||||
lines = []
|
||||
for wrong, correct in sorted(entries.items()):
|
||||
lines.append(f' {{"{wrong}", "{correct}"}},')
|
||||
return HEADER_TEMPLATE.format(entries="\n".join(lines))
|
||||
|
||||
|
||||
def main() -> int:
|
||||
syllables = extract_intact_pinyin_list()
|
||||
if not syllables:
|
||||
raise SystemExit("ERROR: extracted empty pinyin list")
|
||||
entries = build_entries(syllables)
|
||||
rendered = render_header(entries)
|
||||
|
||||
if "--check" in sys.argv[1:]:
|
||||
if not OUT_PATH.exists():
|
||||
raise SystemExit(f"ERROR: {OUT_PATH} is missing")
|
||||
if OUT_PATH.read_text(encoding="utf-8") != rendered:
|
||||
raise SystemExit(f"ERROR: {OUT_PATH} is stale; rerun generate_quanpin_autocorrect.py")
|
||||
print(f"OK: {OUT_PATH} is up to date ({len(entries)} entries)")
|
||||
return 0
|
||||
|
||||
OUT_PATH.write_text(rendered, encoding="utf-8")
|
||||
print(f"Wrote {OUT_PATH} with {len(entries)} entries "
|
||||
f"(from {len(syllables)} legal syllables)")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -70,6 +70,7 @@ std::string g_shuangpin_helpcode_schema = "lantian";
|
||||
std::string g_quanpin_helpcode_schema = "lantian";
|
||||
bool g_show_shuangpin_helpcode_in_candidate_window = true;
|
||||
bool g_show_quanpin_helpcode_in_candidate_window = true;
|
||||
bool g_quanpin_autocorrect_enabled = true;
|
||||
bool g_floating_toolbar_enabled = true;
|
||||
FloatingToolbarItemsConfig g_floating_toolbar_items;
|
||||
double g_floating_toolbar_scale = 1.0;
|
||||
@@ -665,6 +666,7 @@ bool LoadImeConfig()
|
||||
tbl["helpcode"]["show_sp_helpcode_in_candidate_window"].value_or(true);
|
||||
g_show_quanpin_helpcode_in_candidate_window =
|
||||
tbl["helpcode"]["show_qp_helpcode_in_candidate_window"].value_or(true);
|
||||
g_quanpin_autocorrect_enabled = tbl["quanpin"]["autocorrect"].value_or(true);
|
||||
g_floating_toolbar_enabled = tbl["general"]["floating_toolbar"].value_or(true);
|
||||
// Read the old candidate-only key as a migration fallback. New writes
|
||||
// use the unified key.
|
||||
@@ -1826,6 +1828,21 @@ bool SetConfiguredQuanpinHelpcodeEnabled(bool enabled)
|
||||
return true;
|
||||
}
|
||||
|
||||
bool GetConfiguredQuanpinAutocorrectEnabled()
|
||||
{
|
||||
return g_quanpin_autocorrect_enabled;
|
||||
}
|
||||
|
||||
bool SetConfiguredQuanpinAutocorrectEnabled(bool enabled)
|
||||
{
|
||||
if (!WriteConfiguredValue("quanpin", "autocorrect", enabled ? "true" : "false"))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
g_quanpin_autocorrect_enabled = enabled;
|
||||
return true;
|
||||
}
|
||||
|
||||
const std::string &GetConfiguredQuanpinHelpcodeSchema()
|
||||
{
|
||||
return g_quanpin_helpcode_schema;
|
||||
|
||||
@@ -191,6 +191,8 @@ const std::string &GetConfiguredShuangpinHelpcodeSchema();
|
||||
bool SetConfiguredShuangpinHelpcodeSchema(const std::string &schema);
|
||||
bool GetConfiguredQuanpinHelpcodeEnabled();
|
||||
bool SetConfiguredQuanpinHelpcodeEnabled(bool enabled);
|
||||
bool GetConfiguredQuanpinAutocorrectEnabled();
|
||||
bool SetConfiguredQuanpinAutocorrectEnabled(bool enabled);
|
||||
const std::string &GetConfiguredQuanpinHelpcodeSchema();
|
||||
bool SetConfiguredQuanpinHelpcodeSchema(const std::string &schema);
|
||||
bool GetConfiguredShowShuangpinHelpcodeInCandidateWindow();
|
||||
|
||||
@@ -185,6 +185,7 @@ EngineInputSession::EngineInputSession(SchemeType scheme_type, const ShuangpinPr
|
||||
SelectConfiguredHelpcodeSchema(scheme_type);
|
||||
session_.set_shuangpin_helpcode_enabled(GetConfiguredShuangpinHelpcodeEnabled());
|
||||
session_.set_quanpin_helpcode_enabled(GetConfiguredQuanpinHelpcodeEnabled());
|
||||
session_.set_quanpin_autocorrect_enabled(GetConfiguredQuanpinAutocorrectEnabled());
|
||||
}
|
||||
|
||||
void EngineInputSession::handle_key(UINT vk, UINT modifiers_down, WCHAR wch)
|
||||
@@ -192,6 +193,7 @@ void EngineInputSession::handle_key(UINT vk, UINT modifiers_down, WCHAR wch)
|
||||
SelectConfiguredHelpcodeSchema(current_scheme_type());
|
||||
session_.set_shuangpin_helpcode_enabled(GetConfiguredShuangpinHelpcodeEnabled());
|
||||
session_.set_quanpin_helpcode_enabled(GetConfiguredQuanpinHelpcodeEnabled());
|
||||
session_.set_quanpin_autocorrect_enabled(GetConfiguredQuanpinAutocorrectEnabled());
|
||||
session_.handle_key(vk, modifiers_down, wch);
|
||||
}
|
||||
|
||||
@@ -200,6 +202,7 @@ void EngineInputSession::recompute_candidates()
|
||||
SelectConfiguredHelpcodeSchema(current_scheme_type());
|
||||
session_.set_shuangpin_helpcode_enabled(GetConfiguredShuangpinHelpcodeEnabled());
|
||||
session_.set_quanpin_helpcode_enabled(GetConfiguredQuanpinHelpcodeEnabled());
|
||||
session_.set_quanpin_autocorrect_enabled(GetConfiguredQuanpinAutocorrectEnabled());
|
||||
if (has_pending_pinyin_sequence_ || has_pending_pinyin_sequence_with_cases_)
|
||||
{
|
||||
apply_pending_sequence();
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
#include "tests/includes/test_framework.h"
|
||||
#include "MetasequoiaImeEngine/common/helpcode_utils.h"
|
||||
#include "MetasequoiaImeEngine/quanpin/autocorrect_table.h"
|
||||
#include "MetasequoiaImeEngine/quanpin/quanpin_dictionary.h"
|
||||
#include "MetasequoiaImeEngine/quanpin/quanpin_query.h"
|
||||
#include "MetasequoiaImeEngine/quanpin/quanpin_utils.h"
|
||||
@@ -8,6 +9,7 @@
|
||||
#include <filesystem>
|
||||
#include <sqlite3.h>
|
||||
#include <stdexcept>
|
||||
#include <unordered_set>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
@@ -414,6 +416,167 @@ TEST_CASE(QuanpinSparsePinyinFallbackSegmentsPreserveSuffixSegments)
|
||||
REQUIRE_EQ(quanpin::join_segments(dia_fallbacks[1]), std::string("di"));
|
||||
}
|
||||
|
||||
TEST_CASE(QuanpinAutocorrectTableHasNoCollisionsWithLegalPinyin)
|
||||
{
|
||||
const auto &legal = quanpin::intact_pinyin_set();
|
||||
std::unordered_set<std::string> wrong_keys;
|
||||
for (const auto &entry : quanpin::autocorrect::kEntries)
|
||||
{
|
||||
const std::string wrong(entry.wrong);
|
||||
REQUIRE(!wrong.empty());
|
||||
REQUIRE(!legal.count(wrong)); // a key must never shadow a legal syllable
|
||||
REQUIRE(legal.count(entry.correct)); // the correction must be a legal syllable
|
||||
REQUIRE(wrong_keys.insert(wrong).second); // keys must be unique
|
||||
}
|
||||
REQUIRE(wrong_keys.size() > 1000);
|
||||
}
|
||||
|
||||
TEST_CASE(QuanpinAutocorrectCutSingleSyllableTranspositions)
|
||||
{
|
||||
REQUIRE_EQ(quanpin::join_segments(quanpin::autocorrect_cut("sahng")), std::string("shang"));
|
||||
REQUIRE_EQ(quanpin::join_segments(quanpin::autocorrect_cut("hsang")), std::string("shang"));
|
||||
REQUIRE_EQ(quanpin::join_segments(quanpin::autocorrect_cut("shagn")), std::string("shang"));
|
||||
}
|
||||
|
||||
TEST_CASE(QuanpinAutocorrectCutNeighborKeySubstitutions)
|
||||
{
|
||||
REQUIRE_EQ(quanpin::join_segments(quanpin::autocorrect_cut("shabg")), std::string("shang")); // n -> b
|
||||
REQUIRE_EQ(quanpin::join_segments(quanpin::autocorrect_cut("sgang")), std::string("shang")); // h -> g
|
||||
}
|
||||
|
||||
TEST_CASE(QuanpinAutocorrectCutMixedMultisyllableInput)
|
||||
{
|
||||
REQUIRE_EQ(quanpin::join_segments(quanpin::autocorrect_cut("sahngzhi")), std::string("shang'zhi"));
|
||||
REQUIRE_EQ(quanpin::join_segments(quanpin::autocorrect_cut("sahngzhk")), std::string("shang'zhi")); // i -> k
|
||||
}
|
||||
|
||||
TEST_CASE(QuanpinAutocorrectCutPrefersFewestCorrections)
|
||||
{
|
||||
// sahnguai has two plausible readings; the one with fewer corrected edges
|
||||
// (shan + guai, one mistake) wins over shang + hai (two mistakes).
|
||||
REQUIRE_EQ(quanpin::join_segments(quanpin::autocorrect_cut("sahnguai")), std::string("shan'guai"));
|
||||
}
|
||||
|
||||
TEST_CASE(QuanpinAutocorrectCutRejectsInputsOutOfScope)
|
||||
{
|
||||
// Legal input: the caller must only invoke autocorrect when the correction
|
||||
// cut already failed, so a fully legal spelling yields no correction.
|
||||
REQUIRE(quanpin::autocorrect_cut("shang").empty());
|
||||
REQUIRE(quanpin::autocorrect_cut("keneng").empty());
|
||||
// Manual delimiter: user-intended boundary, never rewritten.
|
||||
REQUIRE(quanpin::autocorrect_cut("xi'an").empty());
|
||||
// No correction path: unresolvable garbage.
|
||||
REQUIRE(quanpin::autocorrect_cut("xxxxx").empty());
|
||||
REQUIRE(quanpin::autocorrect_cut("qzzvv").empty());
|
||||
// Beyond the per-input correction edge budget.
|
||||
REQUIRE(quanpin::autocorrect_cut("sahngsahngsahngsahng").empty());
|
||||
}
|
||||
|
||||
TEST_CASE(QuanpinAutocorrectCutRespectsEdgeBudget)
|
||||
{
|
||||
REQUIRE_EQ(quanpin::join_segments(quanpin::autocorrect_cut("sahngsahngsahng")),
|
||||
std::string("shang'shang'shang"));
|
||||
}
|
||||
|
||||
namespace
|
||||
{
|
||||
std::filesystem::path CreateAutocorrectDatabase()
|
||||
{
|
||||
const auto path = std::filesystem::temp_directory_path() / "msime-quanpin-autocorrect-test.db";
|
||||
std::filesystem::remove(path);
|
||||
sqlite3 *db = nullptr;
|
||||
if (sqlite3_open(path.string().c_str(), &db) != SQLITE_OK)
|
||||
{
|
||||
throw std::runtime_error("Failed to create temporary autocorrect database.");
|
||||
}
|
||||
const char *sql =
|
||||
"CREATE TABLE tbl_1_s(key TEXT,jp TEXT,value TEXT,weight INTEGER);"
|
||||
"CREATE TABLE tbl_2_s(key TEXT,jp TEXT,value TEXT,weight INTEGER);"
|
||||
"CREATE TABLE tbl_4_s(key TEXT,jp TEXT,value TEXT,weight INTEGER);"
|
||||
"INSERT INTO tbl_1_s VALUES('shang','s','上',100);"
|
||||
"INSERT INTO tbl_2_s VALUES('shang''zhi','sz','上至',100);"
|
||||
"INSERT INTO tbl_4_s VALUES('sa''huang''na''ge','shng','撒谎那个',1000);";
|
||||
const int result = sqlite3_exec(db, sql, nullptr, nullptr, nullptr);
|
||||
sqlite3_close(db);
|
||||
if (result != SQLITE_OK)
|
||||
{
|
||||
std::filesystem::remove(path);
|
||||
throw std::runtime_error("Failed to initialize temporary autocorrect database.");
|
||||
}
|
||||
return path;
|
||||
}
|
||||
} // namespace
|
||||
|
||||
TEST_CASE(QuanpinDictionaryAutocorrectPutsCorrectedCandidateFirst)
|
||||
{
|
||||
const auto db_path = CreateAutocorrectDatabase();
|
||||
QuanpinDictionary dictionary(db_path.string());
|
||||
|
||||
const auto candidates = dictionary.query("sahng", "sa'h'n'g", true);
|
||||
REQUIRE(!candidates.empty());
|
||||
// The corrected candidate leads, and its pinyin is the corrected key so
|
||||
// that selection/weight updates land on the right dictionary entry.
|
||||
REQUIRE_EQ(candidates.front().word, std::string("上"));
|
||||
REQUIRE_EQ(candidates.front().pinyin, std::string("shang"));
|
||||
REQUIRE_EQ(candidates.front().canonical_pinyin, std::string("shang"));
|
||||
// The original greedy candidates stay behind as a fallback tail.
|
||||
REQUIRE(std::any_of(candidates.begin(), candidates.end(),
|
||||
[](const WordItem &item) { return item.word == "撒谎那个"; }));
|
||||
REQUIRE(candidates.front().word != "撒谎那个");
|
||||
}
|
||||
|
||||
TEST_CASE(QuanpinDictionaryAutocorrectNeighborKeySubstitution)
|
||||
{
|
||||
const auto db_path = CreateAutocorrectDatabase();
|
||||
QuanpinDictionary dictionary(db_path.string());
|
||||
|
||||
const auto candidates = dictionary.query("shabg", "sha'b'g", true);
|
||||
REQUIRE(!candidates.empty());
|
||||
REQUIRE_EQ(candidates.front().word, std::string("上"));
|
||||
}
|
||||
|
||||
TEST_CASE(QuanpinDictionaryAutocorrectMultisyllableInput)
|
||||
{
|
||||
const auto db_path = CreateAutocorrectDatabase();
|
||||
QuanpinDictionary dictionary(db_path.string());
|
||||
|
||||
const auto candidates = dictionary.query("sahngzhi", "sa'h'n'g'zhi", true);
|
||||
REQUIRE(!candidates.empty());
|
||||
REQUIRE_EQ(candidates.front().word, std::string("上至"));
|
||||
}
|
||||
|
||||
TEST_CASE(QuanpinDictionaryAutocorrectDisabledKeepsLegacyBehavior)
|
||||
{
|
||||
const auto db_path = CreateAutocorrectDatabase();
|
||||
QuanpinDictionary dictionary(db_path.string());
|
||||
|
||||
const auto candidates = dictionary.query("sahng", "sa'h'n'g", false);
|
||||
REQUIRE(!candidates.empty());
|
||||
REQUIRE(std::none_of(candidates.begin(), candidates.end(),
|
||||
[](const WordItem &item) { return item.word == "上"; }));
|
||||
REQUIRE(std::any_of(candidates.begin(), candidates.end(),
|
||||
[](const WordItem &item) { return item.word == "撒谎那个"; }));
|
||||
}
|
||||
|
||||
TEST_CASE(QuanpinDictionaryAutocorrectLeavesLegalInputsUntouched)
|
||||
{
|
||||
const auto db_path = CreateAutocorrectDatabase();
|
||||
QuanpinDictionary dictionary(db_path.string());
|
||||
|
||||
// Corrected results live in their own cache slot; a plain spelling never
|
||||
// sees the fallback tail produced for the corrected spelling.
|
||||
(void)dictionary.query("sahng", "sa'h'n'g", true);
|
||||
const auto plain = dictionary.query("shang", "shang", true);
|
||||
REQUIRE(!plain.empty());
|
||||
REQUIRE_EQ(plain.front().word, std::string("上"));
|
||||
REQUIRE(std::none_of(plain.begin(), plain.end(),
|
||||
[](const WordItem &item) { return item.word == "撒谎那个"; }));
|
||||
|
||||
const auto manual = dictionary.query("xi'an", "xi'an", true);
|
||||
const auto manual_off = dictionary.query("xi'an", "xi'an", false);
|
||||
REQUIRE_EQ(manual.size(), manual_off.size());
|
||||
}
|
||||
|
||||
namespace
|
||||
{
|
||||
struct HelpcodeSample
|
||||
|
||||
Reference in new issue
Block a user