fanlusky
80b1fc42a1
词格原先的整句打分是一份没校准过的启发式 unigram，质量很差。这里内嵌 libime 在用的 kenlm 查询子集，改由它的三元模型 sc.lm 来打路径分。 - engine/ngram 只编 kenlm 的 query / trie-build 路径。不定义 WITH_THREADS， parallel_read.cc 因此退化成串行实现，整个子集零 Boost 依赖。 KENLM_MAX_ORDER=3 必须与建模阶数一致，改它等于读不懂已发布的模型文件。 - ngram::LanguageModel 是唯一 include kenlm 头文件的地方，对外只给出一个纯 std 的头，_HAS_AUTO_PTR_ETC 和 kenlm 的 include 目录都保持 PRIVATE。 - shared_language_model 按资源路径共享一份 mmap。kenlm 的查询是只读的且把 上下文放在调用方的 State 里，所以并发会话不需要额外加锁。 - 模型缺失时 valid() 为假，词格退回原来的启发式打分：整句候选只是变差，不会 停止工作。测试覆盖了缺模型降级与按路径共享。 - 出货的 ARPA 不带 <s>/</s>，所以解码从 null_state() 起算而非 begin_state()。 kenlm 是上游副本，不重排版、不进 clang-format，其 LGPL-2.1+ 与 BSD 例外清单 已补进 THIRD_PARTY_NOTICES.txt。googlepinyinime 那条 Fallback 整句路径不变。
124 lines
2.9 KiB
C++
124 lines
2.9 KiB
C++
#ifndef UTIL_POOL_H
|
|
#define UTIL_POOL_H
|
|
|
|
#include <algorithm>
|
|
#include <cassert>
|
|
#include <cstring>
|
|
#include <vector>
|
|
|
|
#include <stdint.h>
|
|
|
|
namespace util {
|
|
|
|
/* Very simple pool. It can only allocate memory. And all of the memory it
|
|
* allocates must be freed at the same time.
|
|
*/
|
|
class Pool {
|
|
public:
|
|
Pool();
|
|
|
|
~Pool();
|
|
|
|
void *Allocate(std::size_t size) {
|
|
void *ret = current_;
|
|
current_ += size;
|
|
if (current_ > current_end_) {
|
|
ret = More(size);
|
|
}
|
|
#ifdef DEBUG
|
|
base_check_ = ret;
|
|
#endif
|
|
return ret;
|
|
}
|
|
|
|
/** Extend (or contract) the most recent allocation.
|
|
* @param base The base pointer of the allocation. This must must have been
|
|
* returned by the MOST RECENT call to Allocate or Continue.
|
|
* @param additional Change in the size.
|
|
*
|
|
* In most cases, more memory from the same page is used, in which case
|
|
* base is unchanged and the function returns false.
|
|
* If the page runs out, a new page is created and the memory (from base)
|
|
* is copied. The function returns true.
|
|
*
|
|
* @return Whether the base had to be changed due to allocating a page.
|
|
*/
|
|
bool Continue(void *&base, std::ptrdiff_t additional) {
|
|
#ifdef DEBUG
|
|
assert(base == base_check_);
|
|
#endif
|
|
current_ += additional;
|
|
if (current_ > current_end_) {
|
|
std::size_t new_total = current_ - static_cast<uint8_t*>(base);
|
|
void *new_base = More(new_total);
|
|
std::memcpy(new_base, base, new_total - additional);
|
|
base = new_base;
|
|
#ifdef DEBUG
|
|
base_check_ = base;
|
|
#endif
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
void FreeAll();
|
|
|
|
private:
|
|
void *More(std::size_t size);
|
|
|
|
std::vector<void *> free_list_;
|
|
|
|
uint8_t *current_, *current_end_;
|
|
|
|
#ifdef DEBUG
|
|
// For debugging, check that Continue came from the most recent call.
|
|
void *base_check_;
|
|
#endif // DEBUG
|
|
|
|
// no copying
|
|
Pool(const Pool &);
|
|
Pool &operator=(const Pool &);
|
|
};
|
|
|
|
/**
|
|
* Pool designed to allow limited freeing.
|
|
* Keeps a linked list of free elements in the free spaces.
|
|
* Will not reduce in size until FreeAll is called.
|
|
*/
|
|
class FreePool {
|
|
public:
|
|
explicit FreePool(std::size_t element_size)
|
|
: free_list_(NULL),
|
|
element_size_(element_size),
|
|
padded_size_(std::max(element_size_, sizeof(void*))) {}
|
|
|
|
void *Allocate() {
|
|
if (free_list_) {
|
|
void *ret = free_list_;
|
|
free_list_ = *reinterpret_cast<void**>(free_list_);
|
|
return ret;
|
|
} else {
|
|
return backing_.Allocate(padded_size_);
|
|
}
|
|
}
|
|
|
|
void Free(void *ptr) {
|
|
*reinterpret_cast<void**>(ptr) = free_list_;
|
|
free_list_ = ptr;
|
|
}
|
|
|
|
std::size_t ElementSize() const { return element_size_; }
|
|
|
|
private:
|
|
void *free_list_;
|
|
|
|
Pool backing_;
|
|
|
|
const std::size_t element_size_;
|
|
const std::size_t padded_size_;
|
|
};
|
|
|
|
} // namespace util
|
|
|
|
#endif // UTIL_POOL_H
|