Add preliminary support for Chinese-English mixed input.

This commit is contained in:
fanlusky committed 2026-07-14 19:00:09 +08:00
1 parent d4f2bcc91e
commit df6002c03e
15 files changed
+386 -6

No files matched your search

+4
View File
@@ -50,6 +50,8 @@ set(SRC_FILES
"./src/config/ime_config.cpp"
"./src/config/ime_config.h"
"./src/defines/defines.h"
"./src/english/english_ime.cpp"
"./src/english/english_ime.h"
"./src/global/globals.h"
"./src/ipc/event_listener.cpp"
"./src/ipc/event_listener.h"
@@ -89,6 +91,8 @@ set(SRC_FILES_IME_ENGINE
"./MetasequoiaImeEngine/common/helpcode_utils.cpp"
"./MetasequoiaImeEngine/common/string_utils.cpp"
"./MetasequoiaImeEngine/core/ime_session.cpp"
"./MetasequoiaImeEngine/english/english_dictionary.cpp"
"./MetasequoiaImeEngine/english/english_dictionary.h"
"./MetasequoiaImeEngine/providers/provider_registry.cpp"
"./MetasequoiaImeEngine/providers/pinyin_candidate_provider.cpp"
"./MetasequoiaImeEngine/providers/wubi_candidate_provider.cpp"
Submodule server/MetasequoiaImeEngine updated: d395e390aa...50b4507c2e.
+6
View File
@@ -67,6 +67,12 @@ If you want to build and run in **one step**, run the following command,
.\scripts\llaunch.ps1
```
## English prefix candidates
Place `english.db` next to `msime.db` in `%LOCALAPPDATA%\MetasequoiaImeTsf`. Set
`general.cn_en_mixed_input = true` in `config.toml` to enable asynchronous English prefix candidates for Quanpin and
Shuangpin.
## Developer Shortcuts
The server process includes a few built-in global shortcuts that are useful during development:
+2 -2
View File
@@ -20,8 +20,8 @@ paging_comma_period = true
paging_tab = true
paging_page_up_down = true
candidate_arrow_navigation = true
# 中英混输
cn_en_mixed_input = false
# 中文输入状态下显示英文前缀候选
cn_en_mixed_input = true
# 极简模式。极简模式下,候选窗口使用 d2d 绘制,界面只作展示候选项,无其他功能。
clean_mode = false
@@ -43,6 +43,8 @@ set(SRC_FILES
"./src/config/ime_config.cpp"
"./src/config/ime_config.h"
"./src/defines/defines.h"
"./src/english/english_ime.cpp"
"./src/english/english_ime.h"
"./src/global/globals.h"
"./src/ipc/event_listener.cpp"
"./src/ipc/event_listener.h"
@@ -79,6 +81,8 @@ set(SRC_FILES
set(SRC_FILES_IME_ENGINE
"./MetasequoiaImeEngine/common/string_utils.cpp"
"./MetasequoiaImeEngine/core/ime_session.cpp"
"./MetasequoiaImeEngine/english/english_dictionary.cpp"
"./MetasequoiaImeEngine/english/english_dictionary.h"
"./MetasequoiaImeEngine/providers/provider_registry.cpp"
"./MetasequoiaImeEngine/providers/pinyin_candidate_provider.cpp"
"./MetasequoiaImeEngine/quanpin/engine.cpp"
+7
View File
@@ -20,6 +20,7 @@ bool g_quanpin_helpcode_enabled = true;
bool g_show_shuangpin_helpcode_in_candidate_window = true;
bool g_show_quanpin_helpcode_in_candidate_window = true;
bool g_floating_toolbar_enabled = true;
bool g_english_candidates_enabled = false;
bool g_paging_minus_equal_enabled = true;
bool g_paging_comma_period_enabled = false;
bool g_paging_tab_enabled = true;
@@ -237,6 +238,7 @@ bool LoadImeConfig()
g_show_quanpin_helpcode_in_candidate_window =
tbl["helpcode"]["show_qp_helpcode_in_candidate_window"].value_or(true);
g_floating_toolbar_enabled = tbl["general"]["floating_toolbar"].value_or(true);
g_english_candidates_enabled = tbl["general"]["cn_en_mixed_input"].value_or(false);
const auto legacy_paging_mode = tbl["general"]["paging_mode"].value<std::string>();
g_paging_minus_equal_enabled =
tbl["general"]["paging_minus_equal"].value_or(!legacy_paging_mode || *legacy_paging_mode == "-/=");
@@ -517,6 +519,11 @@ bool SetConfiguredFloatingToolbarEnabled(bool enabled)
return true;
}
bool GetConfiguredEnglishCandidatesEnabled()
{
return g_english_candidates_enabled;
}
bool GetConfiguredPagingMinusEqualEnabled()
{
return g_paging_minus_equal_enabled;
+1
View File
@@ -27,6 +27,7 @@ bool GetConfiguredShowQuanpinHelpcodeInCandidateWindow();
bool SetConfiguredShowQuanpinHelpcodeInCandidateWindow(bool enabled);
bool GetConfiguredFloatingToolbarEnabled();
bool SetConfiguredFloatingToolbarEnabled(bool enabled);
bool GetConfiguredEnglishCandidatesEnabled();
bool GetConfiguredPagingMinusEqualEnabled();
bool SetConfiguredPagingMinusEqualEnabled(bool enabled);
bool GetConfiguredPagingCommaPeriodEnabled();
+135
View File
@@ -0,0 +1,135 @@
#include "english_ime.h"
#include "MetasequoiaImeEngine/english/english_dictionary.h"
#include <algorithm>
#include <atomic>
#include <condition_variable>
#include <memory>
#include <mutex>
#include <thread>
#include <utility>
namespace
{
constexpr size_t kMinimumPrefixLength = 2;
constexpr size_t kCandidateLimit = 5;
std::mutex g_mutex;
std::condition_variable g_cv;
std::thread g_worker;
std::atomic<bool> g_running{false};
std::atomic<uint64_t> g_generation{0};
std::string g_latest_input;
std::string g_db_path;
EnglishIme::ApplyCallback g_apply_callback;
std::string NormalizeInput(const std::string &input)
{
std::string normalized;
normalized.reserve(input.size());
for (const unsigned char ch : input)
{
if (ch >= 'a' && ch <= 'z')
{
normalized.push_back(static_cast<char>(ch));
}
else if (ch >= 'A' && ch <= 'Z')
{
normalized.push_back(static_cast<char>(ch + ('a' - 'A')));
}
else
{
return {};
}
}
return normalized;
}
void WorkerLoop()
{
EnglishDictionary dictionary(g_db_path);
uint64_t observed_generation = g_generation.load();
while (g_running)
{
std::unique_lock lock(g_mutex);
g_cv.wait(lock, [&] { return !g_running || g_generation.load() != observed_generation; });
if (!g_running)
{
break;
}
observed_generation = g_generation.load();
const std::string input = g_latest_input;
lock.unlock();
const std::string prefix = NormalizeInput(input);
if (prefix.size() < kMinimumPrefixLength)
{
continue;
}
auto candidates = dictionary.query_prefix(prefix, kCandidateLimit);
if (!g_running || g_generation.load() != observed_generation)
{
continue;
}
if (g_apply_callback)
{
g_apply_callback(std::move(candidates), input, observed_generation);
}
}
}
} // namespace
namespace EnglishIme
{
void Start(const std::string &db_path, ApplyCallback apply_callback)
{
if (g_running)
{
return;
}
g_db_path = db_path;
g_apply_callback = std::move(apply_callback);
g_running = true;
g_worker = std::thread(WorkerLoop);
}
void Stop()
{
if (!g_running)
{
return;
}
g_running = false;
g_cv.notify_all();
if (g_worker.joinable())
{
g_worker.join();
}
g_apply_callback = {};
}
void OnInputChanged(const std::string &input)
{
{
std::lock_guard lock(g_mutex);
g_latest_input = input;
g_generation.fetch_add(1);
}
g_cv.notify_one();
}
void Clear()
{
OnInputChanged("");
}
bool IsCurrent(const std::string &input, uint64_t generation)
{
std::lock_guard lock(g_mutex);
return g_running && g_generation.load() == generation && g_latest_input == input;
}
} // namespace EnglishIme
+19
View File
@@ -0,0 +1,19 @@
#pragma once
#include "MetasequoiaImeEngine/core/word_item.h"
#include <cstdint>
#include <functional>
#include <string>
#include <vector>
namespace EnglishIme
{
using ApplyCallback =
std::function<void(std::vector<WordItem> candidates, const std::string &input, uint64_t generation)>;
void Start(const std::string &db_path, ApplyCallback apply_callback);
void Stop();
void OnInputChanged(const std::string &input);
void Clear();
bool IsCurrent(const std::string &input, uint64_t generation);
} // namespace EnglishIme
+94 -3
View File
@@ -24,6 +24,7 @@
#include "ipc/event_listener.h"
#include "utils/ime_utils.h"
#include "cloud/cloud_ime.h"
#include "english/english_ime.h"
#include "config/ime_config.h"
#ifdef FANY_IPC_DEBUG
@@ -208,7 +209,7 @@ std::string BuildCurrentCandidatePage()
const auto &word = item.word;
std::string display_word = word;
if (show_helpcodes)
if (show_helpcodes && item.source != CandidateSource::EnglishDictionary)
{
display_word += HelpcodeUtils::compute_helpcodes(word, uppercase_all_helpcodes);
}
@@ -321,6 +322,7 @@ enum class TaskType
PuncSwitch,
DoubleSingleByteSwitch,
ApplyCloudCandidate,
ApplyEnglishCandidates,
StoreUserPhrase,
PinCandidate,
ClientDeactivated,
@@ -334,6 +336,9 @@ struct Task
std::string cloud_candidate;
std::string cloud_pinyin;
uint64_t cloud_generation = 0;
std::vector<WordItem> english_candidates;
std::string english_input;
uint64_t english_generation = 0;
std::string session_pinyin;
std::string session_word;
};
@@ -346,6 +351,7 @@ void HandleImeKey();
void ClearState();
void ProcessSelectionKey(UINT keycode);
void ApplyCloudCandidate(const std::string &candidate, const std::string &pinyin, uint64_t generation);
void ApplyEnglishCandidates(std::vector<WordItem> candidates, const std::string &input, uint64_t generation);
void EnqueueStoreUserPhraseTask(const std::string &pinyin, const std::string &word);
void EnqueuePinCandidateTask(const std::string &pinyin, const std::string &word);
void MainPipeClientThread(HANDLE clientPipe);
@@ -425,6 +431,11 @@ void WorkerThread()
break;
}
case TaskType::ApplyEnglishCandidates: {
ApplyEnglishCandidates(std::move(task.english_candidates), task.english_input, task.english_generation);
break;
}
case TaskType::StoreUserPhrase: {
g_inputSession->store_user_phrase(task.session_pinyin, task.session_word);
g_inputSession->reset_cache();
@@ -473,6 +484,20 @@ void EnqueueCloudCandidate(const std::string &candidate, const std::string &piny
pipe_queueCv.notify_one();
}
void EnqueueEnglishCandidates(std::vector<WordItem> candidates, const std::string &input, uint64_t generation)
{
{
std::lock_guard lock(queueMutex);
Task task;
task.type = TaskType::ApplyEnglishCandidates;
task.english_candidates = std::move(candidates);
task.english_input = input;
task.english_generation = generation;
taskQueue.push(std::move(task));
}
pipe_queueCv.notify_one();
}
void EnqueueStoreUserPhraseTask(const std::string &pinyin, const std::string &word)
{
{
@@ -828,6 +853,7 @@ void PrepareCandidateList()
auto &ui = Global::candidate_ui;
std::string pinyin = wstring_to_string(Global::PinyinString);
auto items = g_inputSession->get_candidates();
const std::string current_input = g_inputSession->get_pinyin_sequence_with_cases();
if (items.empty())
{
@@ -836,6 +862,17 @@ void PrepareCandidateList()
ui.set_items(std::move(items));
RefreshCandidatePageUi(false);
const SchemeType scheme = g_inputSession->current_scheme_type();
if (GetConfiguredEnglishCandidatesEnabled() && scheme != SchemeType::Wubi &&
!GlobalIme::composition.creating_word.active)
{
EnglishIme::OnInputChanged(current_input);
}
else
{
EnglishIme::Clear();
}
}
void ApplyCloudCandidate(const std::string &candidate, const std::string &pinyin, uint64_t generation)
@@ -858,8 +895,6 @@ void ApplyCloudCandidate(const std::string &candidate, const std::string &pinyin
if (Global::candidate_ui.items.empty())
return;
// 找一下看云候选词在当前候选列表里有没有,如果没有就插到第二位,如果有就不处理。就不需要判断字词的数量大于 2
// 的时候才插入云候选项了。
auto dup_it = std::find_if(Global::candidate_ui.items.begin(), Global::candidate_ui.items.end(),
[&](const DictionaryUlPb::WordItem &item) { return item.word == candidate; });
if (dup_it != Global::candidate_ui.items.end())
@@ -882,6 +917,53 @@ void ApplyCloudCandidate(const std::string &candidate, const std::string &pinyin
RefreshCandidatePageUi(true);
}
void ApplyEnglishCandidates(std::vector<WordItem> candidates, const std::string &input, uint64_t generation)
{
if (!GetConfiguredEnglishCandidatesEnabled() || !EnglishIme::IsCurrent(input, generation) ||
g_inputSession == nullptr || g_inputSession->current_scheme_type() == SchemeType::Wubi ||
g_inputSession->get_pinyin_sequence_with_cases() != input || GlobalIme::composition.creating_word.active)
{
return;
}
auto &items = Global::candidate_ui.items;
items.erase(std::remove_if(items.begin(), items.end(), [](const WordItem &item) {
return item.source == CandidateSource::EnglishDictionary;
}),
items.end());
std::vector<WordItem> unique_candidates;
for (auto &candidate : candidates)
{
const bool duplicate = std::any_of(items.begin(), items.end(),
[&](const WordItem &item) { return item.word == candidate.word; });
if (!duplicate)
{
unique_candidates.push_back(std::move(candidate));
}
}
if (!unique_candidates.empty())
{
size_t insert_index = items.empty() || items.front().source == CandidateSource::Fallback ? 0 : 1;
while (insert_index < items.size() && items[insert_index].source == CandidateSource::CloudSuggestion)
{
++insert_index;
}
items.insert(items.begin() + static_cast<std::ptrdiff_t>(insert_index), std::move(unique_candidates.front()));
for (size_t index = 1; index < unique_candidates.size(); ++index)
{
items.push_back(std::move(unique_candidates[index]));
}
}
Global::candidate_ui.item_total_count = static_cast<int>(items.size());
Global::candidate_ui.page_index = 0;
Global::candidate_ui.select_first_on_page();
Global::candidate_ui.clear_page();
RefreshCandidatePageUi(true);
}
/**
* @brief
*
@@ -1080,6 +1162,7 @@ void HandleImeKey()
void ClearState()
{
CloudIme::Clear();
EnglishIme::Clear();
/* Clear dict engine state */
g_inputSession->reset_state();
/* 造词的状态也要清理 */
@@ -1113,6 +1196,14 @@ void ProcessSelectionKey(UINT keycode)
Global::candidate_ui.items[index + Global::candidate_ui.page_index * Global::candidate_ui.page_size];
std::string curWord = curWordItem.word;
std::string curWordPinyin = curWordItem.pinyin;
if (curWordItem.source == CandidateSource::EnglishDictionary)
{
CloudIme::Clear();
EnglishIme::Clear();
g_inputSession->reset_state();
GlobalIme::composition.clear();
return;
}
std::string cloudCommittedPinyin;
if (curWordItem.source == CandidateSource::CloudSuggestion)
{
+1
View File
@@ -27,4 +27,5 @@ void ClearState();
void ProcessSelectionKey(UINT keycode);
void SendCurrentDataToActiveTsf();
void EnqueueCloudCandidate(const std::string &candidate, const std::string &pinyin, uint64_t generation);
void EnqueueEnglishCandidates(std::vector<WordItem> candidates, const std::string &input, uint64_t generation);
} // namespace FanyNamedPipe
+9
View File
@@ -3,12 +3,15 @@
#include "window/ime_windows.h"
#include <commctrl.h>
#include <thread>
#include <utility>
#include "ipc/ipc.h"
#include "config/ime_config.h"
#include <windows.h>
#include <fmt/xchar.h>
#include <spdlog/spdlog.h>
#include "cloud/cloud_ime.h"
#include "english/english_ime.h"
#include "utils/common_utils.h"
#include "session/session_factory.h"
namespace
@@ -27,6 +30,7 @@ const char *SchemeTypeToString(SchemeType scheme_type)
return "unknown";
}
}
} // namespace
int CALLBACK WinMain(_In_ HINSTANCE hInstance, _In_ HINSTANCE hPrevInstance, _In_ LPSTR lpCmdLine, _In_ int nCmdShow)
@@ -94,9 +98,14 @@ int CALLBACK WinMain(_In_ HINSTANCE hInstance, _In_ HINSTANCE hPrevInstance, _In
CloudIme::Start([](const std::string &candidate, const std::string &pinyin, uint64_t generation) {
FanyNamedPipe::EnqueueCloudCandidate(candidate, pinyin, generation);
});
EnglishIme::Start(CommonUtils::get_ime_data_path() + "\\english.db",
[](std::vector<WordItem> candidates, const std::string &input, uint64_t generation) {
FanyNamedPipe::EnqueueEnglishCandidates(std::move(candidates), input, generation);
});
int ret = CreateCandidateWindow(hInstance);
EnglishIme::Stop();
CloudIme::Stop();
pipe_running = false;
+8
View File
@@ -629,6 +629,10 @@ LRESULT CALLBACK WndProcCandWindow(HWND hwnd, UINT message, WPARAM wParam, LPARA
/* 先取出拼音和汉字 */
DictionaryUlPb::WordItem curWordItem =
Global::candidate_ui.items[zero_based + Global::candidate_ui.page_index * Global::candidate_ui.page_size];
if (curWordItem.source == CandidateSource::EnglishDictionary)
{
break;
}
std::string curWord = curWordItem.word;
std::string curWordPinyin = curWordItem.pinyin;
@@ -661,6 +665,10 @@ LRESULT CALLBACK WndProcCandWindow(HWND hwnd, UINT message, WPARAM wParam, LPARA
/* 先取出拼音和汉字 */
DictionaryUlPb::WordItem curWordItem =
Global::candidate_ui.items[zero_based + Global::candidate_ui.page_index * Global::candidate_ui.page_size];
if (curWordItem.source == CandidateSource::EnglishDictionary)
{
break;
}
std::string curWord = curWordItem.word;
std::string curWordPinyin = curWordItem.pinyin;
// 单字不删除,静默无任何操作来处理
+2
View File
@@ -7,6 +7,8 @@ set(TEST_SOURCE_FILES
"${CMAKE_CURRENT_SOURCE_DIR}/src/test_wubi.cpp"
"${CMAKE_CURRENT_SOURCE_DIR}/src/test_shuangpin_query.cpp"
"${CMAKE_CURRENT_SOURCE_DIR}/src/test_engine_shuangpin_session.cpp"
"${CMAKE_CURRENT_SOURCE_DIR}/src/test_english_dictionary.cpp"
"${CMAKE_SOURCE_DIR}/MetasequoiaImeEngine/english/english_dictionary.cpp"
"${CMAKE_SOURCE_DIR}/src/config/ime_config.cpp"
"${CMAKE_SOURCE_DIR}/src/session/engine_input_session.cpp"
"${CMAKE_SOURCE_DIR}/src/utils/common_utils.cpp"
@@ -0,0 +1,93 @@
#include "tests/includes/test_framework.h"
#include "MetasequoiaImeEngine/english/english_dictionary.h"
#include <chrono>
#include <filesystem>
#include <sqlite3.h>
#include <stdexcept>
#include <string>
namespace
{
class TemporaryEnglishDatabase
{
public:
TemporaryEnglishDatabase()
{
const auto suffix = std::to_string(std::chrono::steady_clock::now().time_since_epoch().count());
path_ = std::filesystem::temp_directory_path() / ("metasequoia_english_test_" + suffix + ".db");
sqlite3 *db = nullptr;
if (sqlite3_open(path_.string().c_str(), &db) != SQLITE_OK)
{
const std::string error = db == nullptr ? "unknown error" : sqlite3_errmsg(db);
if (db != nullptr)
{
sqlite3_close(db);
}
throw std::runtime_error("Unable to create English test database: " + error);
}
constexpr const char *sql =
"CREATE TABLE english_words ("
"word TEXT PRIMARY KEY COLLATE BINARY, display TEXT NOT NULL"
") WITHOUT ROWID;"
"INSERT INTO english_words VALUES ('hel', 'hel');"
"INSERT INTO english_words VALUES ('held', 'held');"
"INSERT INTO english_words VALUES ('hello', 'hello');"
"INSERT INTO english_words VALUES ('help', 'help');"
"INSERT INTO english_words VALUES ('helpful', 'helpful');"
"INSERT INTO english_words VALUES ('hero', 'hero');";
char *error_message = nullptr;
if (sqlite3_exec(db, sql, nullptr, nullptr, &error_message) != SQLITE_OK)
{
const std::string error = error_message == nullptr ? "unknown error" : error_message;
sqlite3_free(error_message);
sqlite3_close(db);
throw std::runtime_error("Unable to populate English test database: " + error);
}
sqlite3_close(db);
}
~TemporaryEnglishDatabase()
{
std::error_code error;
std::filesystem::remove(path_, error);
}
const std::filesystem::path &path() const
{
return path_;
}
private:
std::filesystem::path path_;
};
} // namespace
TEST_CASE(EnglishDictionaryOrdersByExactMatchThenLengthAndWord)
{
TemporaryEnglishDatabase database;
EnglishDictionary dictionary(database.path().string());
const auto candidates = dictionary.query_prefix("hel", 4);
REQUIRE_EQ(candidates.size(), static_cast<size_t>(4));
REQUIRE_EQ(candidates[0].word, std::string("hel"));
REQUIRE_EQ(candidates[1].word, std::string("held"));
REQUIRE_EQ(candidates[2].word, std::string("help"));
REQUIRE_EQ(candidates[3].word, std::string("hello"));
REQUIRE_EQ(candidates[0].pinyin, std::string("hel"));
REQUIRE(candidates[0].source == CandidateSource::EnglishDictionary);
}
TEST_CASE(EnglishDictionaryRestrictsResultsToTheRequestedPrefix)
{
TemporaryEnglishDatabase database;
EnglishDictionary dictionary(database.path().string());
const auto candidates = dictionary.query_prefix("help", 10);
REQUIRE_EQ(candidates.size(), static_cast<size_t>(2));
REQUIRE_EQ(candidates[0].word, std::string("help"));
REQUIRE_EQ(candidates[1].word, std::string("helpful"));
REQUIRE(dictionary.query_prefix("HEL", 5).empty());
}