diff --git a/server/MetasequoiaImeEngine b/server/MetasequoiaImeEngine index ddcb3227..804fcf19 160000 --- a/server/MetasequoiaImeEngine +++ b/server/MetasequoiaImeEngine @@ -1 +1 @@ -Subproject commit ddcb3227742dfeb0cc24800e17f219dadfb0a575 +Subproject commit 804fcf19232583a7eab4c36971291249554c8f46 diff --git a/server/src/settings/dictionary_manager.cpp b/server/src/settings/dictionary_manager.cpp index 5969f2d4..085cacc6 100644 --- a/server/src/settings/dictionary_manager.cpp +++ b/server/src/settings/dictionary_manager.cpp @@ -375,35 +375,7 @@ bool InsertChineseWord(sqlite3 *db, const quanpin::Segments &segments, const std bool ParseImportLine(const std::string &line, std::string &word, std::string &code, int &weight, std::string &message) { - std::istringstream iss(line); - std::string weight_str; - std::string extra; - if (!(iss >> word >> code >> weight_str) || (iss >> extra)) - { - message = "格式错误,应为:词语 全拼 权重(全拼可用 ' 分音节)"; - return false; - } - if (word.empty() || code.empty()) - { - message = "词语和拼音不能为空"; - return false; - } - if (weight_str.empty() || !std::all_of(weight_str.begin(), weight_str.end(), - [](unsigned char ch) { return std::isdigit(ch); })) - { - message = "权重必须是非负整数"; - return false; - } - try - { - weight = (std::max)(0, std::stoi(weight_str)); - } - catch (...) - { - message = "权重数值无效"; - return false; - } - return true; + return Validation::ParseCodedImportLine(line, word, code, weight, message); } json::object ImportChinese(const json::object &request) @@ -438,10 +410,7 @@ json::object ImportChinese(const json::object &request) { ++line_no; if (!line.empty() && line.back() == '\r') line.pop_back(); - const auto begin = line.find_first_not_of(" \t"); - if (begin == std::string::npos) continue; - const auto end = line.find_last_not_of(" \t"); - line = line.substr(begin, end - begin + 1); + if (line.find_first_not_of(" \t") == std::string::npos) continue; std::string word, code; int weight = 0; @@ -889,9 +858,118 @@ json::object HandleEnglish(const json::object &request) return Result(ok, ok ? std::string("英文词条") + label + "成功" : std::string(label) + "失败:" + sqlite3_errmsg(db.get())); } +json::object ImportWubi(const json::object &request) +{ + std::string content = StringValue(request, "content"); + if (content.size() >= 3 && static_cast(content[0]) == 0xEF && + static_cast(content[1]) == 0xBB && static_cast(content[2]) == 0xBF) + content.erase(0, 3); + if (content.find_first_not_of(" \t\r\n") == std::string::npos) + return Result(false, "文件内容为空"); + + std::string error; + Db db = OpenDatabase("msime.db", error); + if (!db) return Result(false, "打开五笔词库失败:" + error); + Stmt insert = Prepare(db.get(), + "INSERT INTO wubi86(key,value,weight) SELECT ?1,?2,?3 " + "WHERE NOT EXISTS (SELECT 1 FROM wubi86 WHERE key=?1 AND value=?2)", error); + if (!insert) return Result(false, "准备导入失败:" + error); + + const auto valid_code = [](const std::string &value) { + return !value.empty() && + std::all_of(value.begin(), value.end(), [](unsigned char ch) { return ch >= 'a' && ch <= 'z'; }); + }; + int inserted = 0; + int skipped = 0; + int failed = 0; + std::vector error_details; + const auto append_error = [&](int line_no, const std::string &detail) { + ++failed; + if (error_details.size() < 5) + error_details.push_back("第 " + std::to_string(line_no) + " 行:" + detail); + }; + + sqlite3_exec(db.get(), "BEGIN IMMEDIATE", nullptr, nullptr, nullptr); + std::istringstream stream(content); + std::string line; + int line_no = 0; + while (std::getline(stream, line)) + { + ++line_no; + if (!line.empty() && line.back() == '\r') line.pop_back(); + if (line.find_first_not_of(" \t") == std::string::npos) continue; + + std::string word; + std::string code; + std::string message; + int weight = 0; + if (!Validation::ParseCodedImportLine(line, word, code, weight, message)) + { + append_error(line_no, message); + continue; + } + std::transform(code.begin(), code.end(), code.begin(), + [](unsigned char ch) { return static_cast(std::tolower(ch)); }); + if (!valid_code(code)) + { + append_error(line_no, "五笔编码只能包含英文字母"); + continue; + } + + sqlite3_reset(insert.get()); + sqlite3_clear_bindings(insert.get()); + const bool ok = BindText(insert.get(), 1, code) && BindText(insert.get(), 2, word) && + sqlite3_bind_int(insert.get(), 3, weight) == SQLITE_OK && + sqlite3_step(insert.get()) == SQLITE_DONE; + if (!ok) + { + append_error(line_no, "写入失败:" + std::string(sqlite3_errmsg(db.get()))); + continue; + } + if (sqlite3_changes(db.get()) == 0) + { + ++skipped; + continue; + } + (void)user_dictionary::record_user_insert(user_dictionary::default_user_db_path(), + user_dictionary::DictionaryKind::Wubi, + code, word, weight); + ++inserted; + } + sqlite3_exec(db.get(), "COMMIT", nullptr, nullptr, nullptr); + + if (inserted == 0 && skipped == 0 && failed == 0) + return Result(false, "文件中没有可导入的词条"); + std::string message; + if (inserted > 0) message += "成功导入 " + std::to_string(inserted) + " 条"; + if (skipped > 0) + { + if (!message.empty()) message += ","; + message += "跳过 " + std::to_string(skipped) + " 条(已存在)"; + } + if (failed > 0) + { + if (!message.empty()) message += ","; + message += "失败 " + std::to_string(failed) + " 条"; + if (!error_details.empty()) + { + message += "。"; + for (size_t i = 0; i < error_details.size(); ++i) + { + if (i) message += ";"; + message += error_details[i]; + } + if (static_cast(error_details.size()) < failed) message += "等"; + } + } + if (inserted > 0) NotifyImeServerClearDictCache(); + return Result(inserted > 0 || (failed == 0 && skipped > 0), message); +} + json::object HandleWubi(const json::object &request) { const std::string action = StringValue(request, "action"); + if (action == "import") return ImportWubi(request); std::string code = StringValue(request, "code"); const std::string word = StringValue(request, "word"); const int weight = (std::max)(0, IntValue(request, "weight", 10)); diff --git a/server/src/settings/dictionary_validation.cpp b/server/src/settings/dictionary_validation.cpp index 95e69b1b..13b15a84 100644 --- a/server/src/settings/dictionary_validation.cpp +++ b/server/src/settings/dictionary_validation.cpp @@ -6,6 +6,7 @@ #include #include +#include namespace SettingsDictionary::Validation { @@ -53,6 +54,59 @@ bool NormalizeFullPinyin(const std::string &input, quanpin::Segments &segments, return without_delimiters == source_without_delimiters; } +bool ParseCodedImportLine(const std::string &line, std::string &word, std::string &code, int &weight, + std::string &message) +{ + const auto trim = [](std::string value) { + const auto begin = value.find_first_not_of(' '); + if (begin == std::string::npos) + return std::string{}; + const auto end = value.find_last_not_of(' '); + return value.substr(begin, end - begin + 1); + }; + + std::vector fields; + std::string::size_type start = 0; + for (;;) + { + const auto separator = line.find('\t', start); + fields.push_back(trim(line.substr(start, separator == std::string::npos ? separator : separator - start))); + if (separator == std::string::npos) + break; + start = separator + 1; + } + if (fields.size() != 3) + { + message = "格式错误,应为:词语编码权重"; + return false; + } + + word = fields[0]; + code = fields[1]; + const std::string &weight_text = fields[2]; + if (word.empty() || code.empty()) + { + message = "词语和编码不能为空"; + return false; + } + if (weight_text.empty() || + !std::all_of(weight_text.begin(), weight_text.end(), [](unsigned char ch) { return std::isdigit(ch); })) + { + message = "权重必须是非负整数"; + return false; + } + try + { + weight = std::stoi(weight_text); + } + catch (...) + { + message = "权重数值无效"; + return false; + } + return true; +} + bool QuickPhraseFitsNamedPipe(const std::string &phrase) { try diff --git a/server/src/settings/dictionary_validation.h b/server/src/settings/dictionary_validation.h index 607d4cbc..fbbd30ec 100644 --- a/server/src/settings/dictionary_validation.h +++ b/server/src/settings/dictionary_validation.h @@ -7,5 +7,7 @@ namespace SettingsDictionary::Validation { bool NormalizeFullPinyin(const std::string &input, quanpin::Segments &segments, std::string &normalized); +bool ParseCodedImportLine(const std::string &line, std::string &word, std::string &code, int &weight, + std::string &message); bool QuickPhraseFitsNamedPipe(const std::string &phrase); } // namespace SettingsDictionary::Validation diff --git a/server/tests/src/test_dictionary_validation.cpp b/server/tests/src/test_dictionary_validation.cpp index a524e940..a41738be 100644 --- a/server/tests/src/test_dictionary_validation.cpp +++ b/server/tests/src/test_dictionary_validation.cpp @@ -26,3 +26,22 @@ TEST_CASE(QuickPhraseValidationMatchesNamedPipeWcharCapacity) REQUIRE(SettingsDictionary::Validation::QuickPhraseFitsNamedPipe(std::string(197, 'a') + emoji)); REQUIRE(!SettingsDictionary::Validation::QuickPhraseFitsNamedPipe(std::string(198, 'a') + emoji)); } + +TEST_CASE(CodedDictionaryImportRequiresTabsAndPreservesSpacesInWords) +{ + std::string word; + std::string code; + std::string message; + int weight = -1; + + REQUIRE(SettingsDictionary::Validation::ParseCodedImportLine("包含 空格的词\tbao'han'kong'ge'de'ci\t123", word, + code, weight, message)); + REQUIRE_EQ(word, std::string("包含 空格的词")); + REQUIRE_EQ(code, std::string("bao'han'kong'ge'de'ci")); + REQUIRE_EQ(weight, 123); + + REQUIRE(!SettingsDictionary::Validation::ParseCodedImportLine("普通词 putongci 10", word, code, weight, message)); + REQUIRE(!SettingsDictionary::Validation::ParseCodedImportLine("普通词\tputongci", word, code, weight, message)); + REQUIRE(!SettingsDictionary::Validation::ParseCodedImportLine("普通词\tputongci\t10\textra", word, code, weight, + message)); +} diff --git a/server/tests/src/test_quanpin_scheme.cpp b/server/tests/src/test_quanpin_scheme.cpp index 8319489f..4f6b6ada 100644 --- a/server/tests/src/test_quanpin_scheme.cpp +++ b/server/tests/src/test_quanpin_scheme.cpp @@ -5,6 +5,9 @@ #include "MetasequoiaImeEngine/quanpin/quanpin_utils.h" #include "MetasequoiaImeEngine/schemes/quanpin_scheme.h" #include +#include +#include +#include #include #include @@ -14,6 +17,32 @@ void InputKey(QuanpinScheme &scheme, UINT vk, WCHAR wch, UINT modifiers_down = 0 { scheme.handle_key(vk, modifiers_down, wch); } + +std::filesystem::path CreatePinyinCacheDatabase() +{ + const auto path = std::filesystem::temp_directory_path() / "msime-pinyin-cache-refresh-test.db"; + std::filesystem::remove(path); + sqlite3 *db = nullptr; + if (sqlite3_open(path.string().c_str(), &db) != SQLITE_OK) + { + throw std::runtime_error("Failed to create temporary pinyin database."); + } + const char *sql = + "PRAGMA journal_mode=WAL;" + "CREATE TABLE tbl_1_n(key TEXT,jp TEXT,value TEXT,weight INTEGER);" + "CREATE TABLE tbl_1_a(key TEXT,jp TEXT,value TEXT,weight INTEGER);" + "CREATE TABLE tbl_2_a(key TEXT,jp TEXT,value TEXT,weight INTEGER);" + "CREATE TABLE tbl_3_a(key TEXT,jp TEXT,value TEXT,weight INTEGER);" + "INSERT INTO tbl_3_a VALUES('ao''shi''ke','ask','奥湿克',1);"; + const int result = sqlite3_exec(db, sql, nullptr, nullptr, nullptr); + sqlite3_close(db); + if (result != SQLITE_OK) + { + std::filesystem::remove(path); + throw std::runtime_error("Failed to initialize temporary pinyin database."); + } + return path; +} } TEST_CASE(QuanpinSchemeSpaceDoesNotResetComposition) @@ -114,6 +143,29 @@ TEST_CASE(QuanpinCandidateCacheKeepsManualSegmentationBoundariesDistinct) [&](const WordItem &item) { return item.word == automatic_only_candidate; })); } +TEST_CASE(QuanpinCandidateCacheDetectsExternalDictionaryWrites) +{ + const auto db_path = CreatePinyinCacheDatabase(); + { + QuanpinDictionary dictionary(db_path.string()); + const auto before = dictionary.query("aoshike", "ao'shi'ke"); + REQUIRE(std::none_of(before.begin(), before.end(), + [](const WordItem &item) { return item.word == "澳鳾科"; })); + + sqlite3 *writer = nullptr; + REQUIRE_EQ(sqlite3_open(db_path.string().c_str(), &writer), SQLITE_OK); + REQUIRE_EQ(sqlite3_exec(writer, + "INSERT INTO tbl_3_a VALUES('ao''shi''ke','ask','澳鳾科',1)", + nullptr, nullptr, nullptr), SQLITE_OK); + sqlite3_close(writer); + + const auto after = dictionary.query("aoshike", "ao'shi'ke"); + REQUIRE(std::any_of(after.begin(), after.end(), + [](const WordItem &item) { return item.word == "澳鳾科"; })); + } + std::filesystem::remove(db_path); +} + TEST_CASE(QuanpinSchemePreservesUppercaseRawInputForHelpcodes) { QuanpinScheme scheme;