feat(server): 万象模型的配置链路、模型包解析与按需下载
- [association] 新增五键:sentence_collocation_additive/rerank(设置页开关,默认
关)、sentence_collocation_model(模型包 id,下载器写回)、两个权重(配置文件
高级项,不进设置页);出厂模板与工作配置同步更新,通过五件套映射下发
- 模型包确定性布局 <DataDir>/models/<id>/<id>.gram,解析只是一次 stat;开关开
着但文件缺席即视为关闭,词格静默降级
- collocation_model.{h,cpp}:WinHTTP 按需下载(重定向放行、进度回报)+ BCrypt
SHA256 校验 + 同卷原子落位 + CC-BY-4.0 NOTICE。SHA256/字节数在常量里钉死:
上游 LTS release 会被原地重传,换字节必须显式更新并复跑离线评测
- CandidateSource::Collocation 落库/学习判定与神经整句同类,候选徽标〔万象〕
- 下载器编译进 Server 与独立设置进程两个宿主,状态以磁盘为准
This commit is contained in:
1 parent
cc738f19ca
commit
69092b9d9a
15 files changed
+532
-10
No files matched your search
@@ -27,3 +27,10 @@ log.txt
|
||||
# 由 scripts/fetch-neural-model.ps1 按 lock.json 从 release 重新取到并校验 sha256。
|
||||
/neural-model/*.safetensors
|
||||
__pycache__/
|
||||
|
||||
# 外部模型包:models/ 下是按需下载的大文件(如万象 .gram 400MB),来源与摘要
|
||||
# 记在调研文档与锁文件里,不进仓库。
|
||||
/models/
|
||||
|
||||
# 本地测试数据暂存(照 scripts/ci/test-server.ps1 的布局手工准备的)
|
||||
/stage/
|
||||
@@ -81,8 +81,18 @@ sentence_neural_desktop = false
|
||||
sentence_neural_keyboard = true
|
||||
# 某个模型的首选与已有候选重复时,改为显示该模型排名最高的下一条不同结果
|
||||
sentence_show_next_on_duplicate = false
|
||||
# 在候选窗的整句候选后显示来源标签(〔Trigram〕〔Unigram〕〔神经D〕〔神经K〕)
|
||||
# 在候选窗的整句候选后显示来源标签(〔Trigram〕〔Unigram〕〔神经D〕〔神经K〕〔万象〕)
|
||||
sentence_source_badge = false
|
||||
# 万象语法模型(octagram .gram,约 390MB,设置页下载):字级搭配强度表,与词表零耦合。
|
||||
# 两个机制共用一份模型包(models/<id>/<id>.gram),默认全关;模型未下载时开关不生效。
|
||||
# additive 在词格解码里逐边叠加搭配分,rerank 在 n-best 上按整句搭配分重排(〔万象〕行)。
|
||||
sentence_collocation_additive = false
|
||||
sentence_collocation_rerank = false
|
||||
# 模型包 id,由设置页下载器写回;也可手动放置后填 id(布局 models/<id>/<id>.gram)
|
||||
sentence_collocation_model = ""
|
||||
# 搭配项的线性权重(log10 域)。取值依据 docs/research 的离线评测扫描;不进设置页
|
||||
sentence_collocation_weight = 0.1
|
||||
sentence_collocation_rerank_weight = 0.05
|
||||
|
||||
#
|
||||
# 腾讯云机器翻译(竖排候选项本地词库未命中时的补全)
|
||||
|
||||
@@ -210,6 +210,8 @@ set(SRC_FILES
|
||||
"./src/settings/settings_launcher.h"
|
||||
"./src/settings/api_credential_test.cpp"
|
||||
"./src/settings/api_credential_test.h"
|
||||
"./src/settings/collocation_model.cpp"
|
||||
"./src/settings/collocation_model.h"
|
||||
"./src/skin/candidate_skin_catalog.cpp"
|
||||
"./src/skin/candidate_skin_catalog.h"
|
||||
"./src/session/engine_input_session.cpp"
|
||||
@@ -432,6 +434,8 @@ add_executable(${SETTINGS_EXECUTABLE_NAME} WIN32
|
||||
"./src/settings/settings_app.cpp"
|
||||
"./src/settings/api_credential_test.cpp"
|
||||
"./src/settings/api_credential_test.h"
|
||||
"./src/settings/collocation_model.cpp"
|
||||
"./src/settings/collocation_model.h"
|
||||
"./src/settings/settings_splash.cpp"
|
||||
"./src/settings/settings_splash.h"
|
||||
"./src/settings/settings_launcher.cpp"
|
||||
|
||||
@@ -81,8 +81,18 @@ sentence_neural_desktop = false
|
||||
sentence_neural_keyboard = true
|
||||
# 某个模型的首选与已有候选重复时,改为显示该模型排名最高的下一条不同结果
|
||||
sentence_show_next_on_duplicate = false
|
||||
# 在候选窗的整句候选后显示来源标签(〔Trigram〕〔Unigram〕〔神经D〕〔神经K〕)
|
||||
# 在候选窗的整句候选后显示来源标签(〔Trigram〕〔Unigram〕〔神经D〕〔神经K〕〔万象〕)
|
||||
sentence_source_badge = false
|
||||
# 万象语法模型(octagram .gram,约 390MB,设置页下载):字级搭配强度表,与词表零耦合。
|
||||
# 两个机制共用一份模型包(models/<id>/<id>.gram),默认全关;模型未下载时开关不生效。
|
||||
# additive 在词格解码里逐边叠加搭配分,rerank 在 n-best 上按整句搭配分重排(〔万象〕行)。
|
||||
sentence_collocation_additive = false
|
||||
sentence_collocation_rerank = false
|
||||
# 模型包 id,由设置页下载器写回;也可手动放置后填 id(布局 models/<id>/<id>.gram)
|
||||
sentence_collocation_model = ""
|
||||
# 搭配项的线性权重(log10 域)。取值依据 docs/research 的离线评测扫描;不进设置页
|
||||
sentence_collocation_weight = 0.1
|
||||
sentence_collocation_rerank_weight = 0.05
|
||||
|
||||
#
|
||||
# 腾讯云机器翻译(候选项本地词库未命中时的补全)
|
||||
|
||||
@@ -120,6 +120,13 @@ bool g_assoc_sentence_google = true;
|
||||
bool g_assoc_sentence_neural_desktop = false;
|
||||
bool g_assoc_sentence_neural_keyboard = true;
|
||||
bool g_assoc_sentence_show_next_on_duplicate = false;
|
||||
// 万象语法模型(octagram .gram):两个机制默认全关,模型 id 默认空(未下载/未选)。
|
||||
// 权重不进设置页,供配置文件高级调参;取值依据 docs/research 的离线评测扫描。
|
||||
bool g_assoc_sentence_collocation_additive = false;
|
||||
bool g_assoc_sentence_collocation_rerank = false;
|
||||
std::string g_assoc_sentence_collocation_model;
|
||||
double g_assoc_sentence_collocation_weight = 0.1;
|
||||
double g_assoc_sentence_collocation_rerank_weight = 0.05;
|
||||
// 候选窗整句候选后的来源标签(〔Trigram〕〔神经K〕等),默认不显示。
|
||||
bool g_assoc_sentence_source_badge = false;
|
||||
bool g_emoji_mixed_input_enabled = false;
|
||||
@@ -491,6 +498,12 @@ bool LoadImeConfig()
|
||||
g_assoc_sentence_neural_desktop = tbl["association"]["sentence_neural_desktop"].value_or(false);
|
||||
g_assoc_sentence_neural_keyboard = tbl["association"]["sentence_neural_keyboard"].value_or(true);
|
||||
g_assoc_sentence_show_next_on_duplicate = tbl["association"]["sentence_show_next_on_duplicate"].value_or(false);
|
||||
g_assoc_sentence_collocation_additive = tbl["association"]["sentence_collocation_additive"].value_or(false);
|
||||
g_assoc_sentence_collocation_rerank = tbl["association"]["sentence_collocation_rerank"].value_or(false);
|
||||
g_assoc_sentence_collocation_model = tbl["association"]["sentence_collocation_model"].value_or(std::string());
|
||||
g_assoc_sentence_collocation_weight = tbl["association"]["sentence_collocation_weight"].value_or(0.1);
|
||||
g_assoc_sentence_collocation_rerank_weight =
|
||||
tbl["association"]["sentence_collocation_rerank_weight"].value_or(0.05);
|
||||
g_assoc_sentence_source_badge = tbl["association"]["sentence_source_badge"].value_or(false);
|
||||
g_emoji_mixed_input_enabled = tbl["general"]["emoji_mixed_input"].value_or(false);
|
||||
g_kaomoji_mixed_input_enabled = tbl["general"]["kaomoji_mixed_input"].value_or(false);
|
||||
|
||||
@@ -361,6 +361,15 @@ bool GetConfiguredAssocSentenceShowNextOnDuplicate();
|
||||
bool SetConfiguredAssocSentenceShowNextOnDuplicate(bool enabled);
|
||||
bool GetConfiguredAssocSentenceSourceBadge();
|
||||
bool SetConfiguredAssocSentenceSourceBadge(bool enabled);
|
||||
// 万象语法模型([association] 段):两个机制开关进设置页,模型 id 由下载器写回,
|
||||
// 权重是配置文件里的高级调参项,不进设置页。
|
||||
bool GetConfiguredAssocSentenceCollocationAdditive();
|
||||
bool SetConfiguredAssocSentenceCollocationAdditive(bool enabled);
|
||||
bool GetConfiguredAssocSentenceCollocationRerank();
|
||||
bool SetConfiguredAssocSentenceCollocationRerank(bool enabled);
|
||||
bool GetConfiguredAssocSentenceCollocationRerankWeight();
|
||||
std::string GetConfiguredAssocSentenceCollocationModel();
|
||||
double GetConfiguredAssocSentenceCollocationWeight();
|
||||
bool GetConfiguredUnicodeModeEnabled();
|
||||
bool SetConfiguredUnicodeModeEnabled(bool enabled);
|
||||
// Shift+K 快捷短语模式。
|
||||
|
||||
@@ -208,6 +208,49 @@ bool SetConfiguredAssocSentenceNeuralKeyboard(bool enabled)
|
||||
return true;
|
||||
}
|
||||
|
||||
bool GetConfiguredAssocSentenceCollocationAdditive()
|
||||
{
|
||||
return g_assoc_sentence_collocation_additive;
|
||||
}
|
||||
bool SetConfiguredAssocSentenceCollocationAdditive(bool enabled)
|
||||
{
|
||||
if (!WriteConfiguredValue("association", "sentence_collocation_additive", enabled ? "true" : "false"))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
g_assoc_sentence_collocation_additive = enabled;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool GetConfiguredAssocSentenceCollocationRerank()
|
||||
{
|
||||
return g_assoc_sentence_collocation_rerank;
|
||||
}
|
||||
bool SetConfiguredAssocSentenceCollocationRerank(bool enabled)
|
||||
{
|
||||
if (!WriteConfiguredValue("association", "sentence_collocation_rerank", enabled ? "true" : "false"))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
g_assoc_sentence_collocation_rerank = enabled;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool GetConfiguredAssocSentenceCollocationRerankWeight()
|
||||
{
|
||||
return g_assoc_sentence_collocation_rerank_weight;
|
||||
}
|
||||
|
||||
std::string GetConfiguredAssocSentenceCollocationModel()
|
||||
{
|
||||
return g_assoc_sentence_collocation_model;
|
||||
}
|
||||
|
||||
double GetConfiguredAssocSentenceCollocationWeight()
|
||||
{
|
||||
return g_assoc_sentence_collocation_weight;
|
||||
}
|
||||
|
||||
bool GetConfiguredAssocSentenceShowNextOnDuplicate()
|
||||
{
|
||||
return g_assoc_sentence_show_next_on_duplicate;
|
||||
|
||||
@@ -164,6 +164,11 @@ extern bool g_assoc_sentence_neural_desktop;
|
||||
extern bool g_assoc_sentence_neural_keyboard;
|
||||
extern bool g_assoc_sentence_show_next_on_duplicate;
|
||||
extern bool g_assoc_sentence_source_badge;
|
||||
extern bool g_assoc_sentence_collocation_additive;
|
||||
extern bool g_assoc_sentence_collocation_rerank;
|
||||
extern std::string g_assoc_sentence_collocation_model;
|
||||
extern double g_assoc_sentence_collocation_weight;
|
||||
extern double g_assoc_sentence_collocation_rerank_weight;
|
||||
extern bool g_emoji_mixed_input_enabled;
|
||||
extern bool g_kaomoji_mixed_input_enabled;
|
||||
extern bool g_unicode_mode_enabled;
|
||||
|
||||
@@ -31,12 +31,14 @@ inline size_t CountCanonicalSyllables(const std::string &canonical_pinyin) noexc
|
||||
return 1 + static_cast<size_t>(std::count(canonical_pinyin.begin(), canonical_pinyin.end(), '\''));
|
||||
}
|
||||
|
||||
// 猜出来的整句来源:词格 Generated、Google 解码器 Fallback,以及神经整句
|
||||
// NeuralDesktop / NeuralKeyboard。它们都不是词库里已有的行,落库/学习判定同类处理。
|
||||
// 猜出来的整句来源:词格 Generated、Google 解码器 Fallback,神经整句
|
||||
// NeuralDesktop / NeuralKeyboard,以及万象语法模型重排的 Collocation。
|
||||
// 它们都不是词库里已有的行,落库/学习判定同类处理。
|
||||
constexpr bool IsGuessedSentenceSource(CandidateSource source) noexcept
|
||||
{
|
||||
return source == CandidateSource::Generated || source == CandidateSource::Fallback ||
|
||||
source == CandidateSource::NeuralDesktop || source == CandidateSource::NeuralKeyboard;
|
||||
source == CandidateSource::NeuralDesktop || source == CandidateSource::NeuralKeyboard ||
|
||||
source == CandidateSource::Collocation;
|
||||
}
|
||||
|
||||
// 整句候选独立上屏——不接在造词前缀后面、自己就是整条输入——时是否该落库。该落:整句是
|
||||
@@ -64,10 +66,11 @@ inline bool ShouldStoreEarlyReturnPhrase(CandidateSource source, bool creating_w
|
||||
const std::string &prefix_canonical_pinyin,
|
||||
const std::string &candidate_canonical_pinyin) noexcept
|
||||
{
|
||||
// 与词格 Generated 同一支:神经整句同样带 canonical quanpin,可以结束一段造词或独立
|
||||
// 上屏。Google Fallback 沿用旧语义仍不走这条造词落库路径。
|
||||
// 与词格 Generated 同一支:神经整句与万象重排句同样带 canonical quanpin,可以结束
|
||||
// 一段造词或独立上屏。Google Fallback 沿用旧语义仍不走这条造词落库路径。
|
||||
const bool early_return_source = source == CandidateSource::Generated || source == CandidateSource::NeuralDesktop ||
|
||||
source == CandidateSource::NeuralKeyboard;
|
||||
source == CandidateSource::NeuralKeyboard ||
|
||||
source == CandidateSource::Collocation;
|
||||
if (!early_return_source || candidate_canonical_pinyin.empty())
|
||||
{
|
||||
return false;
|
||||
|
||||
@@ -302,6 +302,8 @@ std::string BuildCurrentCandidatePage()
|
||||
view.badge = " 〔神经D〕";
|
||||
else if (item.source == CandidateSource::NeuralKeyboard)
|
||||
view.badge = " 〔神经K〕";
|
||||
else if (item.source == CandidateSource::Collocation)
|
||||
view.badge = " 〔万象〕";
|
||||
}
|
||||
view.fixed_position = item.fixed_position > 0;
|
||||
ApplyFixedPositionBadge(view, show_fixed_badge, fixed_badge_style);
|
||||
|
||||
@@ -4,6 +4,30 @@
|
||||
#include "engine/core/sentence_association_options.h"
|
||||
#include "engine/quanpin/quanpin_utils.h"
|
||||
|
||||
#include <filesystem>
|
||||
|
||||
namespace
|
||||
{
|
||||
// 模型包的确定性布局:<resources>/models/<id>/<id>.gram。下载器与手动放置都
|
||||
// 遵守它,解析因此只是一次 stat,不需要目录扫描,也不需要 model.toml。
|
||||
std::string ResolveCollocationModelPath(const std::string &model_id)
|
||||
{
|
||||
if (model_id.empty())
|
||||
{
|
||||
return {};
|
||||
}
|
||||
// 会话持有的是 legacy() 布局:资源根就是数据根,模型包随安装数据走。
|
||||
const std::filesystem::path file =
|
||||
metasequoia::RuntimePaths::legacy().resources / "models" / model_id / (model_id + ".gram");
|
||||
std::error_code error;
|
||||
if (!std::filesystem::is_regular_file(file, error) || error)
|
||||
{
|
||||
return {};
|
||||
}
|
||||
return file.u8string();
|
||||
}
|
||||
} // namespace
|
||||
|
||||
EngineInputSession::EngineInputSession(SchemeType scheme, const ShuangpinProfile &profile)
|
||||
: paths_(metasequoia::RuntimePaths::legacy()), session_(scheme, profile, paths_)
|
||||
{
|
||||
@@ -47,6 +71,18 @@ void EngineInputSession::ApplyConfiguration()
|
||||
association.neural_desktop = GetConfiguredAssocSentenceNeuralDesktop();
|
||||
association.neural_keyboard = GetConfiguredAssocSentenceNeuralKeyboard();
|
||||
association.show_next_on_duplicate = GetConfiguredAssocSentenceShowNextOnDuplicate();
|
||||
// 万象语法模型:两个机制共用一份模型包。模型 id 按 <resources>/models/<id>/<id>.gram
|
||||
// 解析成路径(下载器与手动放置都遵守这个命名);开关开着但文件不在就等于关,词格
|
||||
// 静默降级。解析是一次 stat,按键路径上可忽略。
|
||||
association.collocation_model = ResolveCollocationModelPath(GetConfiguredAssocSentenceCollocationModel());
|
||||
association.collocation_weight = GetConfiguredAssocSentenceCollocationWeight();
|
||||
association.collocation_rerank = GetConfiguredAssocSentenceCollocationRerank();
|
||||
association.collocation_rerank_weight = GetConfiguredAssocSentenceCollocationRerankWeight();
|
||||
if (association.collocation_model.empty())
|
||||
{
|
||||
// 模型缺席时连开关一起置空,避免词典层拿空路径做无谓解析。
|
||||
association.collocation_rerank = false;
|
||||
}
|
||||
session_.set_sentence_association(association);
|
||||
session_.set_shuangpin_preedit_uses_raw(GetConfiguredShuangpinPreeditMode() == "shuangpin");
|
||||
// 五笔拼音混输与 z 键角色是两个独立设置:混输是「同时给五笔和拼音候选」,z 键角色只管
|
||||
|
||||
@@ -0,0 +1,292 @@
|
||||
#include "collocation_model.h"
|
||||
|
||||
#include "engine/core/data_path.h"
|
||||
|
||||
#include <Windows.h>
|
||||
#include <bcrypt.h>
|
||||
#include <winhttp.h>
|
||||
|
||||
#include <atomic>
|
||||
#include <filesystem>
|
||||
#include <fstream>
|
||||
#include <mutex>
|
||||
#include <sstream>
|
||||
#include <string>
|
||||
#include <thread>
|
||||
#include <vector>
|
||||
|
||||
#pragma comment(lib, "winhttp.lib")
|
||||
#pragma comment(lib, "bcrypt.lib")
|
||||
|
||||
namespace collocation
|
||||
{
|
||||
namespace
|
||||
{
|
||||
|
||||
// 模型包的钉死输入。bytes 与 sha256 来自
|
||||
// https://github.com/amzxyz/RIME-LMDG/releases/tag/LTS 的资产清单,
|
||||
// 与 docs/research/2026-10-01-wanxiang-octagram-model.md 的评测基线同一份字节。
|
||||
constexpr const wchar_t *kModelHost = L"github.com";
|
||||
constexpr const wchar_t *kModelPath = L"/amzxyz/RIME-LMDG/releases/download/LTS/wanxiang-lts-zh-hans.gram";
|
||||
constexpr const char *kModelSha256 = "ae43724a9aa02b4c493c603025549d2cba6fa6a67ae3ae040e55efdfeeaa1183";
|
||||
constexpr long long kModelBytes = 409719852LL;
|
||||
constexpr const char *kModelId = "wanxiang-lts-zh-hans";
|
||||
|
||||
std::filesystem::path ModelDirectory()
|
||||
{
|
||||
return metasequoia::data_directory() / "models" / kModelId;
|
||||
}
|
||||
|
||||
std::filesystem::path ModelFile()
|
||||
{
|
||||
return ModelDirectory() / (std::string(kModelId) + ".gram");
|
||||
}
|
||||
|
||||
std::filesystem::path PartFile()
|
||||
{
|
||||
// 同一卷上的临时名,下载完成后原子 rename 到位。
|
||||
return metasequoia::data_directory() / "models" / (std::string(kModelId) + ".gram.part");
|
||||
}
|
||||
|
||||
std::string Sha256Hex(const std::filesystem::path &file)
|
||||
{
|
||||
BCRYPT_ALG_HANDLE algorithm = nullptr;
|
||||
BCRYPT_HASH_HANDLE hash = nullptr;
|
||||
std::string result;
|
||||
DWORD digest_length = 0, object_length = 0, callback = 0;
|
||||
if (BCryptOpenAlgorithmProvider(&algorithm, BCRYPT_SHA256_ALGORITHM, nullptr, 0) != 0 ||
|
||||
BCryptGetProperty(algorithm, BCRYPT_HASH_LENGTH, reinterpret_cast<PUCHAR>(&digest_length),
|
||||
sizeof(digest_length), &callback, 0) != 0 ||
|
||||
BCryptGetProperty(algorithm, BCRYPT_OBJECT_LENGTH, reinterpret_cast<PUCHAR>(&object_length),
|
||||
sizeof(object_length), &callback, 0) != 0)
|
||||
{
|
||||
if (algorithm)
|
||||
BCryptCloseAlgorithmProvider(algorithm, 0);
|
||||
return {};
|
||||
}
|
||||
std::vector<unsigned char> hash_object(object_length);
|
||||
if (BCryptCreateHash(algorithm, &hash, hash_object.data(), object_length, nullptr, 0, 0) == 0)
|
||||
{
|
||||
std::ifstream in(file, std::ios::binary);
|
||||
std::vector<char> buffer(1 << 20);
|
||||
bool ok = static_cast<bool>(in);
|
||||
while (ok && in)
|
||||
{
|
||||
in.read(buffer.data(), static_cast<std::streamsize>(buffer.size()));
|
||||
const auto read = in.gcount();
|
||||
if (read > 0)
|
||||
ok = BCryptHashData(hash, reinterpret_cast<PUCHAR>(buffer.data()), static_cast<ULONG>(read), 0) == 0;
|
||||
}
|
||||
if (ok)
|
||||
{
|
||||
std::vector<unsigned char> digest(digest_length);
|
||||
ok = BCryptFinishHash(hash, digest.data(), digest_length, 0) == 0;
|
||||
static const char *kHex = "0123456789abcdef";
|
||||
for (unsigned char byte : digest)
|
||||
{
|
||||
result.push_back(kHex[byte >> 4]);
|
||||
result.push_back(kHex[byte & 0x0F]);
|
||||
}
|
||||
}
|
||||
BCryptDestroyHash(hash);
|
||||
}
|
||||
BCryptCloseAlgorithmProvider(algorithm, 0);
|
||||
return result;
|
||||
}
|
||||
|
||||
enum class DownloadState
|
||||
{
|
||||
Idle,
|
||||
Downloading,
|
||||
Error,
|
||||
};
|
||||
|
||||
std::mutex g_mutex;
|
||||
DownloadState g_state = DownloadState::Idle;
|
||||
int g_progress = 0;
|
||||
std::string g_error;
|
||||
|
||||
void SetState(DownloadState state, int progress, std::string error)
|
||||
{
|
||||
std::lock_guard lock(g_mutex);
|
||||
g_state = state;
|
||||
g_progress = progress;
|
||||
g_error = std::move(error);
|
||||
}
|
||||
|
||||
// 后台线程:下载 -> 校验 -> 原子落位。全程只碰 PartFile,失败时不留半个模型
|
||||
// 在正式位置上。GitHub 的 release 资产经 302 跳转到 CDN,显式放开自动重定向。
|
||||
void DownloadThread()
|
||||
{
|
||||
SetState(DownloadState::Downloading, 0, {});
|
||||
|
||||
std::error_code fs_error;
|
||||
const std::filesystem::path directory = ModelDirectory();
|
||||
std::filesystem::create_directories(directory, fs_error);
|
||||
if (fs_error)
|
||||
{
|
||||
SetState(DownloadState::Error, 0, "无法创建模型目录");
|
||||
return;
|
||||
}
|
||||
|
||||
HINTERNET session = WinHttpOpen(L"MetasequoiaImeServer/1.0", WINHTTP_ACCESS_TYPE_DEFAULT_PROXY,
|
||||
WINHTTP_NO_PROXY_NAME, WINHTTP_NO_PROXY_BYPASS, 0);
|
||||
if (session == nullptr)
|
||||
{
|
||||
SetState(DownloadState::Error, 0, "无法初始化网络");
|
||||
return;
|
||||
}
|
||||
// 400MB 不是快操作:连接阶段给足超时,收发阶段不设上限靠进度与失败兜底。
|
||||
WinHttpSetTimeouts(session, 30000, 30000, 30000, 0);
|
||||
HINTERNET connection = WinHttpConnect(session, kModelHost, INTERNET_DEFAULT_HTTPS_PORT, 0);
|
||||
HINTERNET request = nullptr;
|
||||
if (connection != nullptr)
|
||||
{
|
||||
request = WinHttpOpenRequest(connection, L"GET", kModelPath, nullptr, WINHTTP_NO_REFERER,
|
||||
WINHTTP_DEFAULT_ACCEPT_TYPES, WINHTTP_FLAG_SECURE);
|
||||
if (request != nullptr)
|
||||
{
|
||||
DWORD policy = WINHTTP_OPTION_REDIRECT_POLICY_ALWAYS;
|
||||
WinHttpSetOption(request, WINHTTP_OPTION_REDIRECT_POLICY, &policy, sizeof(policy));
|
||||
}
|
||||
}
|
||||
bool ok = connection != nullptr && request != nullptr &&
|
||||
WinHttpSendRequest(request, WINHTTP_NO_ADDITIONAL_HEADERS, 0, WINHTTP_NO_REQUEST_DATA, 0, 0, 0) &&
|
||||
WinHttpReceiveResponse(request, nullptr);
|
||||
DWORD status = 0;
|
||||
DWORD status_size = sizeof(status);
|
||||
if (ok)
|
||||
{
|
||||
WinHttpQueryHeaders(request, WINHTTP_QUERY_STATUS_CODE | WINHTTP_QUERY_FLAG_NUMBER,
|
||||
WINHTTP_HEADER_NAME_BY_INDEX, &status, &status_size, WINHTTP_NO_HEADER_INDEX);
|
||||
ok = status == HTTP_STATUS_OK;
|
||||
}
|
||||
|
||||
std::ofstream out;
|
||||
if (ok)
|
||||
{
|
||||
out.open(PartFile(), std::ios::binary | std::ios::trunc);
|
||||
ok = static_cast<bool>(out);
|
||||
}
|
||||
|
||||
long long received = 0;
|
||||
while (ok)
|
||||
{
|
||||
DWORD available = 0;
|
||||
if (!WinHttpQueryDataAvailable(request, &available))
|
||||
{
|
||||
ok = false;
|
||||
break;
|
||||
}
|
||||
if (available == 0)
|
||||
break;
|
||||
std::vector<char> buffer(available);
|
||||
DWORD read = 0;
|
||||
if (!WinHttpReadData(request, buffer.data(), available, &read) || read == 0)
|
||||
{
|
||||
ok = false;
|
||||
break;
|
||||
}
|
||||
out.write(buffer.data(), static_cast<std::streamsize>(read));
|
||||
if (!out)
|
||||
{
|
||||
ok = false;
|
||||
break;
|
||||
}
|
||||
received += read;
|
||||
const long long total = kModelBytes;
|
||||
if (total > 0)
|
||||
{
|
||||
const int progress = static_cast<int>((std::min)(100LL, received * 100LL / total));
|
||||
SetState(DownloadState::Downloading, progress, {});
|
||||
}
|
||||
}
|
||||
if (request != nullptr)
|
||||
WinHttpCloseHandle(request);
|
||||
if (connection != nullptr)
|
||||
WinHttpCloseHandle(connection);
|
||||
WinHttpCloseHandle(session);
|
||||
out.close();
|
||||
|
||||
if (!ok)
|
||||
{
|
||||
SetState(DownloadState::Error, 0, status != 0 && status != HTTP_STATUS_OK ? "下载响应异常" : "下载中断");
|
||||
std::error_code cleanup;
|
||||
std::filesystem::remove(PartFile(), cleanup);
|
||||
return;
|
||||
}
|
||||
|
||||
// 字节级校验:LTS tag 会被上游原地重传,这一步是防漂移的唯一闸门。
|
||||
if (std::filesystem::file_size(PartFile()) != static_cast<unsigned long long>(kModelBytes))
|
||||
{
|
||||
SetState(DownloadState::Error, 0, "下载字节数与锁定值不符");
|
||||
std::error_code cleanup;
|
||||
std::filesystem::remove(PartFile(), cleanup);
|
||||
return;
|
||||
}
|
||||
const std::string digest = Sha256Hex(PartFile());
|
||||
if (digest != kModelSha256)
|
||||
{
|
||||
SetState(DownloadState::Error, 0, "SHA256 与锁定值不符,模型可能已被上游更换");
|
||||
std::error_code cleanup;
|
||||
std::filesystem::remove(PartFile(), cleanup);
|
||||
return;
|
||||
}
|
||||
|
||||
std::filesystem::rename(PartFile(), ModelFile(), fs_error);
|
||||
if (fs_error)
|
||||
{
|
||||
SetState(DownloadState::Error, 0, "模型落位失败");
|
||||
return;
|
||||
}
|
||||
|
||||
// CC-BY-4.0 的署名义务随模型走:NOTICE 与模型同目录,注明来源与锁定摘要。
|
||||
std::ofstream notice(ModelDirectory() / "NOTICE.md", std::ios::binary | std::ios::trunc);
|
||||
if (notice)
|
||||
{
|
||||
notice << "# 万象语法模型(" << kModelId << ")\n\n"
|
||||
<< "- 来源:" << "https://github.com/amzxyz/RIME-LMDG/releases/tag/LTS\n"
|
||||
<< "- 许可:CC-BY-4.0(© amzxyz / RIME-LMDG 项目)\n"
|
||||
<< "- 文件:" << kModelId << ".gram(" << kModelBytes << " 字节)\n"
|
||||
<< "- SHA256:" << kModelSha256 << "\n";
|
||||
}
|
||||
SetState(DownloadState::Idle, 100, {});
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
ModelStatus GetModelStatus()
|
||||
{
|
||||
std::error_code fs_error;
|
||||
if (std::filesystem::exists(ModelFile(), fs_error) && !fs_error &&
|
||||
static_cast<long long>(std::filesystem::file_size(ModelFile(), fs_error)) == kModelBytes && !fs_error)
|
||||
{
|
||||
return {"ready", 100, {}};
|
||||
}
|
||||
std::lock_guard lock(g_mutex);
|
||||
switch (g_state)
|
||||
{
|
||||
case DownloadState::Downloading:
|
||||
return {"downloading", g_progress, {}};
|
||||
case DownloadState::Error:
|
||||
return {"error", 0, g_error};
|
||||
default:
|
||||
return {"absent", 0, {}};
|
||||
}
|
||||
}
|
||||
|
||||
bool StartDownload()
|
||||
{
|
||||
{
|
||||
std::lock_guard lock(g_mutex);
|
||||
if (g_state == DownloadState::Downloading)
|
||||
return true;
|
||||
g_state = DownloadState::Downloading;
|
||||
g_progress = 0;
|
||||
g_error.clear();
|
||||
}
|
||||
std::thread(DownloadThread).detach();
|
||||
return true;
|
||||
}
|
||||
|
||||
} // namespace collocation
|
||||
@@ -0,0 +1,33 @@
|
||||
#pragma once
|
||||
|
||||
// 万象语法模型(octagram .gram)的按需下载与状态查询。
|
||||
//
|
||||
// 390MB 的模型不进安装包:设置页提供下载按钮,落到
|
||||
// <DataDir>/models/wanxiang-lts-zh-hans/wanxiang-lts-zh-hans.gram,
|
||||
// SHA256 与字节数在本文件钉死。上游 LTS release 会被原地重传(tag 不变、字节变),
|
||||
// 所以这里的摘要就是防漂移的锁;上游更新模型时必须显式改这里的常量并复跑
|
||||
// docs/research 的离线评测,而不是让旧字节静默失效。
|
||||
//
|
||||
// 状态以磁盘为准(存在且字节数吻合 = ready),下载态是各进程内存里的
|
||||
// runtime 状态:独立设置进程与 Server 各自能看到自己的下载进度。
|
||||
|
||||
#include <string>
|
||||
|
||||
namespace collocation
|
||||
{
|
||||
|
||||
struct ModelStatus
|
||||
{
|
||||
std::string state; // "absent" | "downloading" | "ready" | "error"
|
||||
int progress = 0; // 下载中 0-100;其余状态无意义
|
||||
std::string error; // state == "error" 时的人类可读原因
|
||||
};
|
||||
|
||||
// 磁盘现状 + 本进程下载态。轻量,可随配置快照频繁调用。
|
||||
ModelStatus GetModelStatus();
|
||||
|
||||
// 幂等启动后台下载:已在下载或已就绪时返回 true 不重复启动;磁盘空间不足、
|
||||
// 无法创建目录等同步失败返回 false 并把原因记入状态。完成/失败经状态查询可见。
|
||||
bool StartDownload();
|
||||
|
||||
} // namespace collocation
|
||||
@@ -5,6 +5,7 @@
|
||||
#include "resource/resource.h"
|
||||
#include "settings/settings_launcher.h"
|
||||
#include "settings/api_credential_test.h"
|
||||
#include "settings/collocation_model.h"
|
||||
#include "settings/settings_splash.h"
|
||||
#include "settings/dictionary_manager.h"
|
||||
#include "settings/serial_task_queue.h"
|
||||
@@ -506,7 +507,13 @@ std::wstring BuildConfigMessage(bool refresh_skin_catalog)
|
||||
{"sentence_neural_desktop", GetConfiguredAssocSentenceNeuralDesktop()},
|
||||
{"sentence_neural_keyboard", GetConfiguredAssocSentenceNeuralKeyboard()},
|
||||
{"sentence_show_next_on_duplicate", GetConfiguredAssocSentenceShowNextOnDuplicate()},
|
||||
{"sentence_source_badge", GetConfiguredAssocSentenceSourceBadge()}}},
|
||||
{"sentence_source_badge", GetConfiguredAssocSentenceSourceBadge()},
|
||||
{"sentence_collocation_additive", GetConfiguredAssocSentenceCollocationAdditive()},
|
||||
{"sentence_collocation_rerank", GetConfiguredAssocSentenceCollocationRerank()},
|
||||
{"sentence_collocation_model_status", [] {
|
||||
const auto status = collocation::GetModelStatus();
|
||||
return nlohmann::json{{"state", status.state}, {"progress", status.progress}, {"error", status.error}};
|
||||
}()}}},
|
||||
{"keybindings",
|
||||
{{"switch_language_shift", GetConfiguredSwitchLanguageShiftEnabled()},
|
||||
{"switch_language_ctrl", GetConfiguredSwitchLanguageCtrlEnabled()},
|
||||
@@ -880,6 +887,10 @@ bool ApplyConfigUpdate(const json::object &data)
|
||||
return SetConfiguredAssocSentenceShowNextOnDuplicate(json::value_to<bool>(data.at("value")));
|
||||
if (path == "association.sentence_source_badge")
|
||||
return SetConfiguredAssocSentenceSourceBadge(json::value_to<bool>(data.at("value")));
|
||||
if (path == "association.sentence_collocation_additive")
|
||||
return SetConfiguredAssocSentenceCollocationAdditive(json::value_to<bool>(data.at("value")));
|
||||
if (path == "association.sentence_collocation_rerank")
|
||||
return SetConfiguredAssocSentenceCollocationRerank(json::value_to<bool>(data.at("value")));
|
||||
if (path == "utility.unicode_mode")
|
||||
return SetConfiguredUnicodeModeEnabled(json::value_to<bool>(data.at("value")));
|
||||
if (path == "utility.quick_phrase")
|
||||
@@ -1327,6 +1338,16 @@ void HandleWebMessage(HWND hwnd, ICoreWebView2WebMessageReceivedEventArgs *args)
|
||||
if (!ec)
|
||||
ShellExecuteW(hwnd, L"open", directory.c_str(), nullptr, nullptr, SW_SHOWNORMAL);
|
||||
}
|
||||
else if (type == "collocationModelDownload")
|
||||
{
|
||||
// 后台线程下载,进度随下一次配置快照回给页面;页面在下载态轮询 configRequest。
|
||||
collocation::StartDownload();
|
||||
PostConfig(false);
|
||||
}
|
||||
else if (type == "collocationModelStatusRequest")
|
||||
{
|
||||
PostConfig(false);
|
||||
}
|
||||
else if (type == "configUpdate")
|
||||
{
|
||||
const auto data = value.at("data").as_object();
|
||||
|
||||
@@ -10,6 +10,7 @@
|
||||
#include "ipc/ipc.h"
|
||||
#include "ipc/event_listener.h"
|
||||
#include "skin/candidate_skin_catalog.h"
|
||||
#include "settings/collocation_model.h"
|
||||
#include "utils/common_utils.h"
|
||||
#include <dwmapi.h>
|
||||
#include <nlohmann/json.hpp>
|
||||
@@ -375,6 +376,16 @@ HRESULT OnControllerCreatedSettingsWnd( //
|
||||
if (!ec)
|
||||
ShellExecuteW(hwnd, L"open", directory.c_str(), nullptr, nullptr, SW_SHOWNORMAL);
|
||||
}
|
||||
else if (type == "collocationModelDownload")
|
||||
{
|
||||
// 后台线程下载,进度随下一次配置快照回给页面;页面在下载态轮询 configRequest。
|
||||
collocation::StartDownload();
|
||||
PostSettingsConfig();
|
||||
}
|
||||
else if (type == "collocationModelStatusRequest")
|
||||
{
|
||||
PostSettingsConfig();
|
||||
}
|
||||
else if (type == "configUpdate")
|
||||
{
|
||||
try
|
||||
@@ -1052,6 +1063,22 @@ HRESULT OnControllerCreatedSettingsWnd( //
|
||||
PostSettingsConfig();
|
||||
}
|
||||
}
|
||||
else if (path == "association.sentence_collocation_additive")
|
||||
{
|
||||
const bool value = json::value_to<bool>(data.at("value"));
|
||||
if (SetConfiguredAssocSentenceCollocationAdditive(value))
|
||||
{
|
||||
PostSettingsConfig();
|
||||
}
|
||||
}
|
||||
else if (path == "association.sentence_collocation_rerank")
|
||||
{
|
||||
const bool value = json::value_to<bool>(data.at("value"));
|
||||
if (SetConfiguredAssocSentenceCollocationRerank(value))
|
||||
{
|
||||
PostSettingsConfig();
|
||||
}
|
||||
}
|
||||
else if (path == "utility.unicode_mode")
|
||||
{
|
||||
const bool value = json::value_to<bool>(data.at("value"));
|
||||
@@ -1474,7 +1501,14 @@ void PostSettingsConfig()
|
||||
{"sentence_neural_desktop", GetConfiguredAssocSentenceNeuralDesktop()},
|
||||
{"sentence_neural_keyboard", GetConfiguredAssocSentenceNeuralKeyboard()},
|
||||
{"sentence_show_next_on_duplicate", GetConfiguredAssocSentenceShowNextOnDuplicate()},
|
||||
{"sentence_source_badge", GetConfiguredAssocSentenceSourceBadge()}}},
|
||||
{"sentence_source_badge", GetConfiguredAssocSentenceSourceBadge()},
|
||||
{"sentence_collocation_additive", GetConfiguredAssocSentenceCollocationAdditive()},
|
||||
{"sentence_collocation_rerank", GetConfiguredAssocSentenceCollocationRerank()},
|
||||
{"sentence_collocation_model_status",
|
||||
[] {
|
||||
const auto status = collocation::GetModelStatus();
|
||||
return nlohmann::json{{"state", status.state}, {"progress", status.progress}, {"error", status.error}};
|
||||
}()}}},
|
||||
{"keybindings",
|
||||
{{"switch_language_shift", GetConfiguredSwitchLanguageShiftEnabled()},
|
||||
{"switch_language_ctrl", GetConfiguredSwitchLanguageCtrlEnabled()},
|
||||
|
||||
Reference in new issue
Block a user