fanlusky
80b1fc42a1
词格原先的整句打分是一份没校准过的启发式 unigram，质量很差。这里内嵌 libime 在用的 kenlm 查询子集，改由它的三元模型 sc.lm 来打路径分。 - engine/ngram 只编 kenlm 的 query / trie-build 路径。不定义 WITH_THREADS， parallel_read.cc 因此退化成串行实现，整个子集零 Boost 依赖。 KENLM_MAX_ORDER=3 必须与建模阶数一致，改它等于读不懂已发布的模型文件。 - ngram::LanguageModel 是唯一 include kenlm 头文件的地方，对外只给出一个纯 std 的头，_HAS_AUTO_PTR_ETC 和 kenlm 的 include 目录都保持 PRIVATE。 - shared_language_model 按资源路径共享一份 mmap。kenlm 的查询是只读的且把 上下文放在调用方的 State 里，所以并发会话不需要额外加锁。 - 模型缺失时 valid() 为假，词格退回原来的启发式打分：整句候选只是变差，不会 停止工作。测试覆盖了缺模型降级与按路径共享。 - 出货的 ARPA 不带 <s>/</s>，所以解码从 null_state() 起算而非 begin_state()。 kenlm 是上游副本，不重排版、不进 clang-format，其 LGPL-2.1+ 与 BSD 例外清单 已补进 THIRD_PARTY_NOTICES.txt。googlepinyinime 那条 Fallback 整句路径不变。
107 lines
3.4 KiB
C++
107 lines
3.4 KiB
C++
#ifndef LM_BINARY_FORMAT_H
|
|
#define LM_BINARY_FORMAT_H
|
|
|
|
#include "config.hh"
|
|
#include "model_type.hh"
|
|
#include "read_arpa.hh"
|
|
|
|
#include "../util/file_piece.hh"
|
|
#include "../util/mmap.hh"
|
|
#include "../util/scoped.hh"
|
|
|
|
#include <cstddef>
|
|
#include <vector>
|
|
|
|
#include <stdint.h>
|
|
|
|
namespace lm {
|
|
namespace ngram {
|
|
|
|
extern const char *kModelNames[6];
|
|
|
|
/*Inspect a file to determine if it is a binary lm. If not, return false.
|
|
* If so, return true and set recognized to the type. This is the only API in
|
|
* this header designed for use by decoder authors.
|
|
*/
|
|
bool RecognizeBinary(const char *file, ModelType &recognized);
|
|
|
|
struct FixedWidthParameters {
|
|
unsigned char order;
|
|
float probing_multiplier;
|
|
// What type of model is this?
|
|
ModelType model_type;
|
|
// Does the end of the file have the actual strings in the vocabulary?
|
|
bool has_vocabulary;
|
|
unsigned int search_version;
|
|
};
|
|
|
|
// This is a macro instead of an inline function so constants can be assigned using it.
|
|
#define ALIGN8(a) ((std::ptrdiff_t(((a)-1)/8)+1)*8)
|
|
|
|
// Parameters stored in the header of a binary file.
|
|
struct Parameters {
|
|
FixedWidthParameters fixed;
|
|
std::vector<uint64_t> counts;
|
|
};
|
|
|
|
class BinaryFormat {
|
|
public:
|
|
explicit BinaryFormat(const Config &config);
|
|
|
|
// Reading a binary file:
|
|
// Takes ownership of fd
|
|
void InitializeBinary(int fd, ModelType model_type, unsigned int search_version, Parameters ¶ms);
|
|
// Used to read parts of the file to update the config object before figuring out full size.
|
|
void ReadForConfig(void *to, std::size_t amount, uint64_t offset_excluding_header) const;
|
|
// Actually load the binary file and return a pointer to the beginning of the search area.
|
|
void *LoadBinary(std::size_t size);
|
|
|
|
uint64_t VocabStringReadingOffset() const {
|
|
assert(vocab_string_offset_ != kInvalidOffset);
|
|
return vocab_string_offset_;
|
|
}
|
|
|
|
// Writing a binary file or initializing in RAM from ARPA:
|
|
// Size for vocabulary.
|
|
void *SetupJustVocab(std::size_t memory_size, uint8_t order);
|
|
// Warning: can change the vocaulary base pointer.
|
|
void *GrowForSearch(std::size_t memory_size, std::size_t vocab_pad, void *&vocab_base);
|
|
// Warning: can change vocabulary and search base addresses.
|
|
void WriteVocabWords(const std::string &buffer, void *&vocab_base, void *&search_base);
|
|
// Write the header at the beginning of the file.
|
|
void FinishFile(const Config &config, ModelType model_type, unsigned int search_version, const std::vector<uint64_t> &counts);
|
|
|
|
private:
|
|
void MapFile(void *&vocab_base, void *&search_base);
|
|
|
|
// Copied from configuration.
|
|
const Config::WriteMethod write_method_;
|
|
const char *write_mmap_;
|
|
util::LoadMethod load_method_;
|
|
|
|
// File behind memory, if any.
|
|
util::scoped_fd file_;
|
|
|
|
// If there is a file involved, a single mapping.
|
|
util::scoped_memory mapping_;
|
|
|
|
// If the data is only in memory, separately allocate each because the trie
|
|
// knows vocab's size before it knows search's size (because SRILM might
|
|
// have pruned).
|
|
util::scoped_memory memory_vocab_, memory_search_;
|
|
|
|
// Memory ranges. Note that these may not be contiguous and may not all
|
|
// exist.
|
|
std::size_t header_size_, vocab_size_, vocab_pad_;
|
|
// aka end of search.
|
|
uint64_t vocab_string_offset_;
|
|
|
|
static const uint64_t kInvalidOffset = (uint64_t)-1;
|
|
};
|
|
|
|
bool IsBinaryFormat(int fd);
|
|
|
|
} // namespace ngram
|
|
} // namespace lm
|
|
#endif // LM_BINARY_FORMAT_H
|