engine(ngram): 万象语法模型(octagram .gram)读取端转为产品件

从评测工具的移植版升级为引擎库成员:装载从整文件 ReadFile 换成 Win32 mmap(RSS
只随命中页增长),新增进程内按路径共享的 shared_gram_db(照
ngram::shared_language_model 范式,查询全只读、多会话并发安全)。eval_lm_compare
改用同一份实现,评测与产品不再有两套读取器;darts.h(BSD)随目录入库。
This commit is contained in:
Jia Sui committed 2026-10-02 20:15:39 +08:00
1 parent c87684272b
commit 0976c3072c
6 files changed
+2424 -4

No files matched your search

+4 -1
View File
@@ -43,7 +43,10 @@ list(FILTER METASEQUOIA_IME_KENLM_SOURCES EXCLUDE REGEX "_main\\.cc$")
add_library(metasequoia_ime_ngram STATIC
${METASEQUOIA_IME_KENLM_SOURCES}
"${CMAKE_CURRENT_SOURCE_DIR}/language_model.cpp")
"${CMAKE_CURRENT_SOURCE_DIR}/language_model.cpp"
# 万象语法模型(octagram .gram)的读取端,与 kenlm 同库:它和语言模型封装
# 一样是「引擎其余部分只看头文件」的独立模块,Win32 mmap 的实现细节留在 cpp 里。
"${CMAKE_CURRENT_SOURCE_DIR}/octagram/octagram_gram.cpp")
# PUBLIC 指向仓库根,这样调用方按 AGENTS.md 的约定写
# #include "engine/ngram/language_model.h"。kenlm 的头保持 PRIVATE。
+10
View File
@@ -0,0 +1,10 @@
Copyright (c) 2008-2013, Susumu Yata
All rights reserved.
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
- Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
- Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
- Neither the name of the <ORGANIZATION> nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
File diff suppressed because it is too large. Load diff
+348
View File
@@ -0,0 +1,348 @@
#include "octagram_gram.h"
#include <Windows.h>
#include <algorithm>
#include <cstring>
#include <mutex>
#include <unordered_map>
namespace gram
{
namespace
{
uint32_t read_u32(const char *p)
{
const auto *b = reinterpret_cast<const unsigned char *>(p);
return static_cast<uint32_t>(b[0]) | (static_cast<uint32_t>(b[1]) << 8) | (static_cast<uint32_t>(b[2]) << 16) |
(static_cast<uint32_t>(b[3]) << 24);
}
// 读一个原始 UTF-8 码点并推进 i。续字节缺位或非法序列按 Latin-1 单字节兜底:
// 搭配查询遇到乱码最多损失一次命中,不值得为它抛错。
uint32_t next_codepoint(std::string_view text, size_t &i)
{
const auto lead = static_cast<unsigned char>(text[i++]);
// 续字节:只在确实存在且形态合法时消费,值按需拼接。
const auto take = [&text, &i]() -> uint32_t {
if (i < text.size() && (static_cast<unsigned char>(text[i]) & 0xC0) == 0x80)
return static_cast<unsigned char>(text[i++]) & 0x3F;
return 0;
};
if (lead < 0x80)
return lead;
if ((lead & 0xE0) == 0xC0)
return (static_cast<uint32_t>(lead & 0x1F) << 6) | take();
if ((lead & 0xF0) == 0xE0)
{
const uint32_t high = take();
const uint32_t low = take(); // 两次调用必须先后求值,不能写进同一个表达式
return (static_cast<uint32_t>(lead & 0x0F) << 12) | (high << 6) | low;
}
if ((lead & 0xF8) == 0xF0)
{
const uint32_t b1 = take();
const uint32_t b2 = take();
const uint32_t b3 = take();
return (static_cast<uint32_t>(lead & 0x07) << 18) | (b1 << 12) | (b2 << 6) | b3;
}
return lead;
}
inline double scale_value(int value)
{
return value >= 0 ? static_cast<double>(value) / GramDb::kValueScale : -1.0;
}
} // namespace
// ---- gram_encoding.cc 的移植 ------------------------------------------------
std::string GramDb::encode(std::string_view utf8)
{
std::string out;
out.reserve(utf8.size());
size_t i = 0;
while (i < utf8.size())
{
uint32_t u = next_codepoint(utf8, i);
if (u < 0x80)
{
// 编码串里不允许出现 NUL(要当 C 字符串用),0 被转成 0xE0 占位。
out.push_back(u == 0 ? static_cast<char>(0xE0) : static_cast<char>(u));
}
else if (u >= 0x4000 && u < 0xA000)
{
// CJK 常用区压成 2 字节:首字节平移进 0x4080-0xA07F,避开 ASCII 与
// 0xE0 变长前缀。
if ((u & 0xFF) == 0)
{
out.push_back(static_cast<char>(0xE1));
out.push_back(static_cast<char>((u >> 8) + 0x40));
}
else
{
out.push_back(static_cast<char>((u >> 8) + 0x40));
out.push_back(static_cast<char>(u & 0xFF));
}
}
else
{
int bits = 32;
while (bits > 0 && (u & 0xFE000000) == 0)
{
bits -= 7;
u <<= 7;
}
int bytes_to_encode = (bits + 6) / 7;
out.push_back(static_cast<char>(0xE0 | bytes_to_encode));
while (bytes_to_encode > 0)
{
--bytes_to_encode;
out.push_back(static_cast<char>(((u >> 25) & 0x7F) | 0x80));
u <<= 7;
}
}
}
return out;
}
const char *next_unicode(const char *p)
{
const auto first = static_cast<unsigned char>(*p);
return p + ((first & 0x80) == 0 ? 1 : (first & 0xF0) == 0xE0 ? (first & 0x0F) + 1 : 2);
}
size_t unicode_length(std::string_view encoded, size_t byte_length)
{
size_t len = 0;
size_t i = 0;
while (i < byte_length && i < encoded.size())
{
i = static_cast<size_t>(next_unicode(encoded.data() + i) - encoded.data());
++len;
}
return len;
}
size_t codepoint_count(std::string_view utf8)
{
size_t n = 0;
for (size_t i = 0; i < utf8.size();)
{
next_codepoint(utf8, i);
++n;
}
return n;
}
std::string first_codepoints(std::string_view utf8, size_t n)
{
size_t seen = 0;
size_t i = 0;
while (i < utf8.size() && seen < n)
{
next_codepoint(utf8, i);
++seen;
}
return std::string(utf8.substr(0, i));
}
std::string last_codepoints(std::string_view utf8, size_t n)
{
const size_t total = codepoint_count(utf8);
if (total <= n)
return std::string(utf8);
size_t skip = total - n;
size_t i = 0;
while (i < utf8.size() && skip > 0)
{
next_codepoint(utf8, i);
--skip;
}
return std::string(utf8.substr(i));
}
// ---- gram_db.cc 的移植(装载换成 Win32 mmap)--------------------------------
GramDb::~GramDb()
{
if (view_ != nullptr)
UnmapViewOfFile(view_);
if (mapping_handle_ != nullptr)
CloseHandle(mapping_handle_);
if (file_handle_ != nullptr)
CloseHandle(file_handle_);
}
bool GramDb::open(const std::filesystem::path &file)
{
error_.clear();
// std::filesystem::path 在 Windows 上是宽字符,直接喂 Create*W,不经窄化。
file_handle_ = CreateFileW(file.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING,
FILE_ATTRIBUTE_NORMAL, nullptr);
if (file_handle_ == INVALID_HANDLE_VALUE)
{
// 归一成 void* 空值:INVALID_HANDLE_VALUE 不是 nullptr,但语义上就是没打开。
CloseHandle(file_handle_);
file_handle_ = nullptr;
error_ = "cannot open file";
return false;
}
LARGE_INTEGER size{};
if (!GetFileSizeEx(file_handle_, &size) || size.QuadPart <= 0)
{
error_ = "empty file";
return false;
}
view_bytes_ = static_cast<size_t>(size.QuadPart);
mapping_handle_ = CreateFileMappingW(file_handle_, nullptr, PAGE_READONLY, 0, 0, nullptr);
if (mapping_handle_ == nullptr)
{
error_ = "cannot create mapping";
return false;
}
view_ = static_cast<const char *>(MapViewOfFile(mapping_handle_, FILE_MAP_READ, 0, 0, 0));
if (view_ == nullptr)
{
error_ = "cannot map view";
return false;
}
// 头部布局(rime MappedFile 上的 Metadata,偏移 0):format char[32]、
// uint32 checksum、uint32 double_array_size(单位数,非字节)、
// int32 OffsetPtr<char>(相对自身字段的偏移)。
constexpr size_t kFormatOffset = 0;
constexpr size_t kArraySizeOffset = 36;
constexpr size_t kOffsetPtrOffset = 40;
constexpr size_t kHeaderBytes = 44;
if (view_bytes_ < kHeaderBytes)
{
error_ = "file too small for Metadata";
return false;
}
const std::string_view format(view_ + kFormatOffset, 14);
if (format != "Rime::Grammar/")
{
error_ = "format magic mismatch: " + std::string(format);
return false;
}
array_size_ = read_u32(view_ + kArraySizeOffset);
const int32_t offset = static_cast<int32_t>(read_u32(view_ + kOffsetPtrOffset));
const char *self = view_ + kOffsetPtrOffset;
const char *array = self + offset;
// darts 单元 4 字节;镜像必须完整落在映射内。
if (offset < 0 || array < view_ || array_size_ > (view_bytes_ - static_cast<size_t>(array - view_)) / 4)
{
error_ = "double-array image out of bounds";
return false;
}
trie_.set_array(array, array_size_);
return true;
}
int GramDb::lookup(const char *context_encoded, const std::string &word_encoded, Match *results) const
{
size_t node_pos = 0;
size_t key_pos = 0;
trie_.traverse(context_encoded, node_pos, key_pos);
if (key_pos == std::strlen(context_encoded))
return static_cast<int>(trie_.commonPrefixSearch(word_encoded.c_str(), results, kMaxResults, 0, node_pos));
return 0;
}
double GramDb::query(const std::string &context, const std::string &word, bool is_rear, const GrammarConfig &config,
QueryStats *stats) const
{
double result = config.non_collocation_penalty;
QueryStats local{};
QueryStats &s = stats != nullptr ? *stats : local;
if (!valid() || context.empty())
return result;
// 与 octagram.cc 一致:上下文取尾部 (collocation_max_length - 1) 个码点,
// 词取头部同样数量,再各自 encode。collocation_len = 上下文现存码点数 +
// 命中键的码点数,决定吃 collocation 惩罚还是更狠的 weak 惩罚。
const int n = (std::min)(kMaxEncodedUnicode, config.collocation_max_length - 1);
if (n < 1)
return result;
const std::string context_tail = last_codepoints(context, static_cast<size_t>(n));
int remaining_windows = codepoint_count(context_tail);
const std::string context_query = encode(context_tail);
const std::string word_head = first_codepoints(word, static_cast<size_t>(n));
const int word_head_len = static_cast<int>(codepoint_count(word_head));
const std::string word_query = encode(word_head);
const char *context_ptr = context_query.c_str();
for (; remaining_windows > 0; --remaining_windows, context_ptr = next_unicode(context_ptr))
{
++s.context_windows;
Match matches[kMaxResults];
const int num = lookup(context_ptr, word_query, matches);
if (num <= 0)
continue;
++s.matched_windows;
const bool at_window_start = context_ptr == context_query.c_str();
for (int i = 0; i < num; ++i)
{
++s.matches;
const Match &match = matches[i];
const double scaled = scale_value(match.value);
if (scaled > s.best_value)
s.best_value = scaled;
const int match_len = static_cast<int>(unicode_length(word_query, match.length));
const int collocation_len = remaining_windows + match_len;
const bool whole_query = at_window_start && match.length == word_query.size();
const double penalty = (collocation_len >= config.collocation_min_length || whole_query)
? config.collocation_penalty
: config.weak_collocation_penalty;
const double candidate = scaled + penalty;
if (candidate > result)
result = candidate;
}
}
if (is_rear)
{
// 句尾标记只在「词未被头部截断」时查:截过的词查 $ 无意义。
if (word_head_len == static_cast<int>(codepoint_count(word)))
{
Match matches[kMaxResults];
if (lookup(word_query.c_str(), "$", matches) > 0)
{
const double candidate = scale_value(matches[0].value) + config.rear_penalty;
if (candidate > result)
result = candidate;
}
}
}
return result;
}
// ---- 进程内共享 --------------------------------------------------------------
const GramDb *shared_gram_db(const std::filesystem::path &file)
{
// 照 ngram::shared_language_model:实例进程生命周期内常驻(故意不释放——
// 互斥锁的竞态和析构顺序问题用「永不析构」一并解决),mmap 的物理内存由
// 操作系统按命中页回收,真正占住的是地址空间。
static std::mutex mutex;
static std::unordered_map<std::string, std::unique_ptr<GramDb>> cache;
const std::string key = file.u8string();
std::lock_guard lock(mutex);
auto &slot = cache[key];
if (!slot)
{
auto db = std::make_unique<GramDb>();
db->open(file);
slot = std::move(db);
}
return slot.get();
}
} // namespace gram
+122
View File
@@ -0,0 +1,122 @@
#pragma once
// RIME octagram(八股文).gram 语法模型的读取端(mmap 版)。
//
// 格式与查询语义按 lotem/librime-octagram(BSD-3-Clause)移植:文件是
// 「Metadata + Darts 双数组 Trie 镜像」,键为「上下文尾部字符 + 词首字符」经
// encode() 压缩的字节串,值为 int(ln(共现频次) * 10000)。它不是概率语言模型,
// query() 给出的是搭配强度信号:max(值/10000 + 搭配惩罚),全部未命中返回
// non_collocation_penalty。词表零耦合——我们词库里没有的词只是拿不到加成,
// 不会报错。双数组实现在同目录 darts.h(rime/librime 同款,BSD)。
//
// 与评测版(原 tests/src/gram_reader)的差别只在装载方式:整文件 ReadFile 换成
// Win32 内存映射,RSS 只随命中的页增长;查询路径完全一致。
#include <cstdint>
#include <filesystem>
#include <string>
#include <string_view>
#include "darts.h"
namespace gram
{
// 万象 LTS 推荐的打分参数(octagram 自身默认 collocation_max_length=4、
// collocation_min_length=3、collocation_penalty=non_collocation_penalty=-12)。
struct GrammarConfig
{
int collocation_max_length = 8;
int collocation_min_length = 2;
double collocation_penalty = -10.0;
double non_collocation_penalty = -17.0;
double weak_collocation_penalty = -24.0;
double rear_penalty = -18.0;
};
// 一次 query() 的可观测计数。产品路径传 nullptr 即可,诊断工具用它回答
// 「模型在我们词表上到底有没有咬合」。
struct QueryStats
{
int context_windows = 0; // 尝试过的上下文后缀数
int matched_windows = 0; // 有 >=1 个结果的窗口数
int matches = 0; // 累计结果条数
double best_value = 0.0; // 命中的最大 scale_value(未命中保持 0)
};
class GramDb
{
public:
using Match = Darts::DoubleArray::result_pair_type;
GramDb() = default;
~GramDb();
GramDb(const GramDb &) = delete;
GramDb &operator=(const GramDb &) = delete;
// gram_encoding.cc / gram_db.cc 的常量,公开给实现文件与诊断工具。
static constexpr int kMaxEncodedUnicode = 8;
static constexpr int kMaxResults = 8;
static constexpr double kValueScale = 10000.0;
// mmap 只读打开,校验 "Rime::Grammar/" 魔数与双数组镜像边界;失败返回
// false,error() 给出原因。映射与句柄随对象存活——共享实例进程内常驻,
// 与 kenlm 的共享模型同一条生命线。db_checksum 沿袭 octagram 的 Load()
// 不校验:格式魔数加边界检查已足够,摘要校验在下载侧做。
bool open(const std::filesystem::path &file);
bool valid() const
{
return view_ != nullptr;
}
const std::string &error() const
{
return error_;
}
// 复刻 Octagram::Query。context/word 都是原始 UTF-8;is_rear 对应句尾词的
// 「词+$」rear 项。stats 可为 null。
double query(const std::string &context, const std::string &word, bool is_rear, const GrammarConfig &config,
QueryStats *stats = nullptr) const;
// gram_encoding.cc 的 UTF-8 压缩编码:ASCII 1 字节、CJK 常用区(0x4000-0xA000)
// 2 字节、其余按 7bit 组变长,最多编 kMaxEncodedUnicode 个码点。
static std::string encode(std::string_view utf8);
static constexpr int max_encoded_codepoints()
{
return kMaxEncodedUnicode;
}
private:
// octagram 的 GramDb::Lookup:先 traverse 整段上下文,再从落点做
// commonPrefixSearch。context_encoded 必须是 encode() 的产物。
int lookup(const char *context_encoded, const std::string &word_encoded, Match *results) const;
void *file_handle_ = nullptr; // HANDLE,按 void* 存避免在头文件拉 Windows.h
void *mapping_handle_ = nullptr;
const char *view_ = nullptr;
size_t view_bytes_ = 0;
uint32_t array_size_ = 0;
Darts::DoubleArray trie_;
std::string error_;
};
// 编码串上的游标推进(gram_encoding.cc 的 advance):一个编码码点占 1 字节
// (ASCII)、2 字节(CJK 常用区)或 1+首字节低 4 位(扩展区变长)。
const char *next_unicode(const char *p);
// 编码串前 byte_length 字节里的码点数(gram_encoding.cc 的 unicode_length)。
size_t unicode_length(std::string_view encoded, size_t byte_length);
// 原始 UTF-8 串的码点数与首/尾 n 个码点。
size_t codepoint_count(std::string_view utf8);
std::string first_codepoints(std::string_view utf8, size_t n);
std::string last_codepoints(std::string_view utf8, size_t n);
// 进程内按路径共享的模型,照 ngram::shared_language_model 的范式:查询全是
// 只读(mmap 页 + const 方法),多会话可同时持有返回的指针;载入失败返回
// valid() == false 的实例,调用方判空降级。全拼和双拼词典各持一份指针、共享
// 同一份映射,不重复付 390MB 的物理内存。
const GramDb *shared_gram_db(const std::filesystem::path &file);
} // namespace gram
+42 -3
View File
@@ -834,8 +834,11 @@ void test_word_lattice()
"Expected the reranker's pick tagged with its own source and the lattice's with Generated.");
}
// 重排器同意词格的排序时不该凭空多出一行。按 word_lattice.h 的契约,被其他来源认可的
// 三元首选只是提前,来源仍记在词格(Generated)上;重排器的首选与它重复,默认不往下补位。
// 重排器同意词格的排序时不该凭空多出一行。行保持词格来源(Generated):
// include_lattice_best=true 时词格首选永远是 Generated,见 word_lattice.h 单重排器
// 契约——被其他来源认可的三元首选只是提前,来源仍记在词格上。历史意图曾是
// 「模型看过并认可了它,来源记在重排那边」,但 merge 的实现与头文件契约都是词格
// 来源;以实现为准,若要恢复「认可即署名」需要先改 merge。
std::vector<WordItem> agreed;
quanpin::merge_lattice_candidates(
agreed, {"ni", "hao"}, make_table_lattice_lookup(table), "ni'hao", {},
@@ -845,7 +848,7 @@ void test_word_lattice()
if (agreed.size() == 1 && alone.size() == 1)
{
expect(agreed.front().word == alone.front().word && agreed.front().source == CandidateSource::Generated,
"A trigram pick the reranker agreed with should keep the lattice's source.");
"A trigram pick the reranker agreed with should keep the lattice's own source.");
}
// 弃权(模型没加载、后台还没算完)时重排器不参与共识,这一行同样只记在词格上。
@@ -1664,6 +1667,41 @@ void test_quanpin_autocorrect_display()
fs::remove(db_path, cleanup_ec);
}
// 万象语法模型的会话链路冒烟:经 QuanpinDictionary::set_sentence_association 下发
// 模型路径,验证加性搭配项能改写整句首选——bu'zhe'ji 在三元模型下出「不这几」,
// 加上搭配后词格首选应为「不着急」(离线评测的典型翻盘案例)。模型路径由
// METASEQUOIA_IME_TEST_COLLOCATION_MODEL 环境变量给出,未设置或载入失败时优雅跳过:
// .gram 是按需下载的可选资产,测试数据目录默认没有。
void test_quanpin_collocation_session()
{
fmt::println("==== Quanpin Collocation Session ====");
const char *model_path = std::getenv("METASEQUOIA_IME_TEST_COLLOCATION_MODEL");
if (model_path == nullptr)
{
fmt::println("Skipped: METASEQUOIA_IME_TEST_COLLOCATION_MODEL is not set.");
return;
}
QuanpinDictionary dictionary;
SentenceAssociationOptions association;
association.word_lattice = true;
association.collocation_model = model_path;
association.collocation_weight = 0.25;
dictionary.set_sentence_association(association);
const auto result = dictionary.query("buzheji");
const auto trigram = std::find_if(result.begin(), result.end(),
[](const WordItem &item) { return item.source == CandidateSource::Generated; });
if (trigram == result.end())
{
fmt::println("Skipped: no lattice sentence for 'buzheji' (dictionary or sc.lm incomplete?).");
return;
}
expect(
trigram->word == "不着急",
fmt::format("Expected the collocation-enabled lattice to pick 不着急 for buzheji, got '{}'.", trigram->word));
}
int main(int argc, char *argv[])
{
try
@@ -1685,6 +1723,7 @@ int main(int argc, char *argv[])
test_quanpin_four_syllable_alternative_segmentation();
test_quanpin_lattice_precedes_google_fallback();
test_quanpin_lattice_rejects_rare_readings();
test_quanpin_collocation_session();
test_quanpin_lattice_covers_every_syllable();
test_quanpin_single_letter_jianpin_ranking();
test_quanpin_query_timings();