Files
fanlusky 80b1fc42a1 feat(engine): 词格整句改用 kenlm 三元模型打分
词格原先的整句打分是一份没校准过的启发式 unigram,质量很差。这里内嵌 libime
在用的 kenlm 查询子集,改由它的三元模型 sc.lm 来打路径分。

- engine/ngram 只编 kenlm 的 query / trie-build 路径。不定义 WITH_THREADS,
  parallel_read.cc 因此退化成串行实现,整个子集零 Boost 依赖。
  KENLM_MAX_ORDER=3 必须与建模阶数一致,改它等于读不懂已发布的模型文件。
- ngram::LanguageModel 是唯一 include kenlm 头文件的地方,对外只给出一个纯
  std 的头,_HAS_AUTO_PTR_ETC 和 kenlm 的 include 目录都保持 PRIVATE。
- shared_language_model 按资源路径共享一份 mmap。kenlm 的查询是只读的且把
  上下文放在调用方的 State 里,所以并发会话不需要额外加锁。
- 模型缺失时 valid() 为假,词格退回原来的启发式打分:整句候选只是变差,不会
  停止工作。测试覆盖了缺模型降级与按路径共享。
- 出货的 ARPA 不带 <s>/</s>,所以解码从 null_state() 起算而非 begin_state()。

kenlm 是上游副本,不重排版、不进 clang-format,其 LGPL-2.1+ 与 BSD 例外清单
已补进 THIRD_PARTY_NOTICES.txt。googlepinyinime 那条 Fallback 整句路径不变。
2026-09-19 10:08:39 +08:00

193 lines
5.4 KiB
C++

// Copyright 2004 The RE2 Authors. All Rights Reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in string_piece.hh.
#include "string_piece.hh"
#include <algorithm>
#include <climits>
#ifndef HAVE_ICU
typedef StringPiece::size_type size_type;
void StringPiece::CopyToString(std::string* target) const {
target->assign(ptr_, length_);
}
size_type StringPiece::find(const StringPiece& s, size_type pos) const {
// Not sure why length_ < 0 was here since it's std::size_t.
if (/*length_ < 0 || */pos > static_cast<size_type>(length_))
return npos;
const char* result = std::search(ptr_ + pos, ptr_ + length_,
s.ptr_, s.ptr_ + s.length_);
const size_type xpos = result - ptr_;
return xpos + s.length_ <= length_ ? xpos : npos;
}
size_type StringPiece::find(char c, size_type pos) const {
if (length_ <= 0 || pos >= static_cast<size_type>(length_)) {
return npos;
}
const char* result = std::find(ptr_ + pos, ptr_ + length_, c);
return result != ptr_ + length_ ? result - ptr_ : npos;
}
size_type StringPiece::rfind(const StringPiece& s, size_type pos) const {
if (length_ < s.length_) return npos;
const size_t ulen = length_;
if (s.length_ == 0) return std::min(ulen, pos);
const char* last = ptr_ + std::min(ulen - s.length_, pos) + s.length_;
const char* result = std::find_end(ptr_, last, s.ptr_, s.ptr_ + s.length_);
return result != last ? result - ptr_ : npos;
}
size_type StringPiece::rfind(char c, size_type pos) const {
if (length_ <= 0) return npos;
for (int i = std::min(pos, static_cast<size_type>(length_ - 1));
i >= 0; --i) {
if (ptr_[i] == c) {
return i;
}
}
return npos;
}
// For each character in characters_wanted, sets the index corresponding
// to the ASCII code of that character to 1 in table. This is used by
// the find_.*_of methods below to tell whether or not a character is in
// the lookup table in constant time.
// The argument `table' must be an array that is large enough to hold all
// the possible values of an unsigned char. Thus it should be be declared
// as follows:
// bool table[UCHAR_MAX + 1]
static inline void BuildLookupTable(const StringPiece& characters_wanted,
bool* table) {
const size_type length = characters_wanted.length();
const char* const data = characters_wanted.data();
for (size_type i = 0; i < length; ++i) {
table[static_cast<unsigned char>(data[i])] = true;
}
}
size_type StringPiece::find_first_of(const StringPiece& s,
size_type pos) const {
if (length_ == 0 || s.length_ == 0)
return npos;
// Avoid the cost of BuildLookupTable() for a single-character search.
if (s.length_ == 1)
return find_first_of(s.ptr_[0], pos);
bool lookup[UCHAR_MAX + 1] = { false };
BuildLookupTable(s, lookup);
for (size_type i = pos; i < length_; ++i) {
if (lookup[static_cast<unsigned char>(ptr_[i])]) {
return i;
}
}
return npos;
}
size_type StringPiece::find_first_not_of(const StringPiece& s,
size_type pos) const {
if (length_ == 0)
return npos;
if (s.length_ == 0)
return 0;
// Avoid the cost of BuildLookupTable() for a single-character search.
if (s.length_ == 1)
return find_first_not_of(s.ptr_[0], pos);
bool lookup[UCHAR_MAX + 1] = { false };
BuildLookupTable(s, lookup);
for (size_type i = pos; i < length_; ++i) {
if (!lookup[static_cast<unsigned char>(ptr_[i])]) {
return i;
}
}
return npos;
}
size_type StringPiece::find_first_not_of(char c, size_type pos) const {
if (length_ == 0)
return npos;
for (; pos < length_; ++pos) {
if (ptr_[pos] != c) {
return pos;
}
}
return npos;
}
size_type StringPiece::find_last_of(const StringPiece& s, size_type pos) const {
if (length_ == 0 || s.length_ == 0)
return npos;
// Avoid the cost of BuildLookupTable() for a single-character search.
if (s.length_ == 1)
return find_last_of(s.ptr_[0], pos);
bool lookup[UCHAR_MAX + 1] = { false };
BuildLookupTable(s, lookup);
for (size_type i = std::min(pos, length_ - 1); ; --i) {
if (lookup[static_cast<unsigned char>(ptr_[i])])
return i;
if (i == 0)
break;
}
return npos;
}
size_type StringPiece::find_last_not_of(const StringPiece& s,
size_type pos) const {
if (length_ == 0)
return npos;
size_type i = std::min(pos, length_ - 1);
if (s.length_ == 0)
return i;
// Avoid the cost of BuildLookupTable() for a single-character search.
if (s.length_ == 1)
return find_last_not_of(s.ptr_[0], pos);
bool lookup[UCHAR_MAX + 1] = { false };
BuildLookupTable(s, lookup);
for (; ; --i) {
if (!lookup[static_cast<unsigned char>(ptr_[i])])
return i;
if (i == 0)
break;
}
return npos;
}
size_type StringPiece::find_last_not_of(char c, size_type pos) const {
if (length_ == 0)
return npos;
for (size_type i = std::min(pos, length_ - 1); ; --i) {
if (ptr_[i] != c)
return i;
if (i == 0)
break;
}
return npos;
}
StringPiece StringPiece::substr(size_type pos, size_type n) const {
if (pos > length_) pos = length_;
if (n > length_ - pos) n = length_ - pos;
return StringPiece(ptr_ + pos, n);
}
const size_type StringPiece::npos = size_type(-1);
#endif // !HAVE_ICU