fanlusky
80b1fc42a1
词格原先的整句打分是一份没校准过的启发式 unigram，质量很差。这里内嵌 libime 在用的 kenlm 查询子集，改由它的三元模型 sc.lm 来打路径分。 - engine/ngram 只编 kenlm 的 query / trie-build 路径。不定义 WITH_THREADS， parallel_read.cc 因此退化成串行实现，整个子集零 Boost 依赖。 KENLM_MAX_ORDER=3 必须与建模阶数一致，改它等于读不懂已发布的模型文件。 - ngram::LanguageModel 是唯一 include kenlm 头文件的地方，对外只给出一个纯 std 的头，_HAS_AUTO_PTR_ETC 和 kenlm 的 include 目录都保持 PRIVATE。 - shared_language_model 按资源路径共享一份 mmap。kenlm 的查询是只读的且把 上下文放在调用方的 State 里，所以并发会话不需要额外加锁。 - 模型缺失时 valid() 为假，词格退回原来的启发式打分：整句候选只是变差，不会 停止工作。测试覆盖了缺模型降级与按路径共享。 - 出货的 ARPA 不带 <s>/</s>，所以解码从 null_state() 起算而非 begin_state()。 kenlm 是上游副本，不重排版、不进 clang-format，其 LGPL-2.1+ 与 BSD 例外清单 已补进 THIRD_PARTY_NOTICES.txt。googlepinyinime 那条 Fallback 整句路径不变。
98 lines
2.4 KiB
C++
98 lines
2.4 KiB
C++
/* Like std::ofstream but without being incredibly slow. Backed by a raw fd.
|
|
* Supports most of the built-in types except for long double.
|
|
*/
|
|
#ifndef UTIL_FILE_STREAM_H
|
|
#define UTIL_FILE_STREAM_H
|
|
|
|
#include "fake_ostream.hh"
|
|
#include "file.hh"
|
|
#include "scoped.hh"
|
|
|
|
#include <cassert>
|
|
#include <cstring>
|
|
|
|
#include <stdint.h>
|
|
|
|
namespace util {
|
|
|
|
class FileStream : public FakeOStream<FileStream> {
|
|
public:
|
|
explicit FileStream(int out = -1, std::size_t buffer_size = 8192)
|
|
: buf_(util::MallocOrThrow(std::max<std::size_t>(buffer_size, kToStringMaxBytes))),
|
|
current_(static_cast<char*>(buf_.get())),
|
|
end_(current_ + std::max<std::size_t>(buffer_size, kToStringMaxBytes)),
|
|
fd_(out) {}
|
|
|
|
#if __cplusplus >= 201103L
|
|
FileStream(FileStream &&from) noexcept : buf_(from.buf_.release()), current_(from.current_), end_(from.end_), fd_(from.fd_) {
|
|
from.end_ = reinterpret_cast<char*>(from.buf_.get());
|
|
from.current_ = from.end_;
|
|
}
|
|
#endif
|
|
|
|
~FileStream() {
|
|
flush();
|
|
}
|
|
|
|
void SetFD(int to) {
|
|
flush();
|
|
fd_ = to;
|
|
}
|
|
|
|
FileStream &flush() {
|
|
if (current_ != buf_.get()) {
|
|
util::WriteOrThrow(fd_, buf_.get(), current_ - (char*)buf_.get());
|
|
current_ = static_cast<char*>(buf_.get());
|
|
}
|
|
return *this;
|
|
}
|
|
|
|
// For writes of arbitrary size.
|
|
FileStream &write(const void *data, std::size_t length) {
|
|
if (UTIL_LIKELY(current_ + length <= end_)) {
|
|
std::memcpy(current_, data, length);
|
|
current_ += length;
|
|
return *this;
|
|
}
|
|
flush();
|
|
if (current_ + length <= end_) {
|
|
std::memcpy(current_, data, length);
|
|
current_ += length;
|
|
} else {
|
|
util::WriteOrThrow(fd_, data, length);
|
|
}
|
|
return *this;
|
|
}
|
|
|
|
FileStream &seekp(uint64_t to) {
|
|
flush();
|
|
util::SeekOrThrow(fd_, to);
|
|
return *this;
|
|
}
|
|
|
|
protected:
|
|
friend class FakeOStream<FileStream>;
|
|
// For writes directly to buffer guaranteed to have amount < buffer size.
|
|
char *Ensure(std::size_t amount) {
|
|
if (UTIL_UNLIKELY(current_ + amount > end_)) {
|
|
flush();
|
|
assert(current_ + amount <= end_);
|
|
}
|
|
return current_;
|
|
}
|
|
|
|
void AdvanceTo(char *to) {
|
|
current_ = to;
|
|
assert(current_ <= end_);
|
|
}
|
|
|
|
private:
|
|
util::scoped_malloc buf_;
|
|
char *current_, *end_;
|
|
int fd_;
|
|
};
|
|
|
|
} // namespace
|
|
|
|
#endif
|