fanlusky
80b1fc42a1
词格原先的整句打分是一份没校准过的启发式 unigram，质量很差。这里内嵌 libime 在用的 kenlm 查询子集，改由它的三元模型 sc.lm 来打路径分。 - engine/ngram 只编 kenlm 的 query / trie-build 路径。不定义 WITH_THREADS， parallel_read.cc 因此退化成串行实现，整个子集零 Boost 依赖。 KENLM_MAX_ORDER=3 必须与建模阶数一致，改它等于读不懂已发布的模型文件。 - ngram::LanguageModel 是唯一 include kenlm 头文件的地方，对外只给出一个纯 std 的头，_HAS_AUTO_PTR_ETC 和 kenlm 的 include 目录都保持 PRIVATE。 - shared_language_model 按资源路径共享一份 mmap。kenlm 的查询是只读的且把 上下文放在调用方的 State 里，所以并发会话不需要额外加锁。 - 模型缺失时 valid() 为假，词格退回原来的启发式打分：整句候选只是变差，不会 停止工作。测试覆盖了缺模型降级与按路径共享。 - 出货的 ARPA 不带 <s>/</s>，所以解码从 null_state() 起算而非 begin_state()。 kenlm 是上游副本，不重排版、不进 clang-format，其 LGPL-2.1+ 与 BSD 例外清单 已补进 THIRD_PARTY_NOTICES.txt。googlepinyinime 那条 Fallback 整句路径不变。
101 lines
3.4 KiB
C++
101 lines
3.4 KiB
C++
#ifndef UTIL_PROXY_ITERATOR_H
|
|
#define UTIL_PROXY_ITERATOR_H
|
|
|
|
#include <cstddef>
|
|
#include <iterator>
|
|
|
|
/* This is a RandomAccessIterator that uses a proxy to access the underlying
|
|
* data. Useful for packing data at bit offsets but still using STL
|
|
* algorithms.
|
|
*
|
|
* Normally I would use boost::iterator_facade but some people are too lazy to
|
|
* install boost and still want to use my language model. It's amazing how
|
|
* many operators an iterator has.
|
|
*
|
|
* The Proxy needs to provide:
|
|
* class InnerIterator;
|
|
* InnerIterator &Inner();
|
|
* const InnerIterator &Inner() const;
|
|
*
|
|
* InnerIterator has to implement:
|
|
* operator==(InnerIterator)
|
|
* operator<(InnerIterator)
|
|
* operator+=(std::ptrdiff_t)
|
|
* operator-(InnerIterator)
|
|
* and of course whatever Proxy needs to dereference it.
|
|
*
|
|
* It's also a good idea to specialize std::swap for Proxy.
|
|
*/
|
|
|
|
namespace util {
|
|
template <class Proxy> class ProxyIterator {
|
|
private:
|
|
// Self.
|
|
typedef ProxyIterator<Proxy> S;
|
|
typedef typename Proxy::InnerIterator InnerIterator;
|
|
|
|
public:
|
|
typedef std::random_access_iterator_tag iterator_category;
|
|
typedef typename Proxy::value_type value_type;
|
|
typedef std::ptrdiff_t difference_type;
|
|
typedef Proxy reference;
|
|
typedef ProxyIterator<Proxy> * pointer;
|
|
|
|
ProxyIterator() {}
|
|
|
|
// For cast from non const to const.
|
|
template <class AlternateProxy> ProxyIterator(const ProxyIterator<AlternateProxy> &in) : p_(*in) {}
|
|
explicit ProxyIterator(const Proxy &p) : p_(p) {}
|
|
|
|
/* // p_'s swap does value swapping, but here we want iterator swapping
|
|
friend inline void swap(ProxyIterator<Proxy> &first, ProxyIterator<Proxy> &second) {
|
|
swap(first.I(), second.I());
|
|
}*/
|
|
|
|
// p_'s operator= does value copying, but here we want iterator copying.
|
|
S &operator=(const S &other) {
|
|
I() = other.I();
|
|
return *this;
|
|
}
|
|
|
|
bool operator==(const S &other) const { return I() == other.I(); }
|
|
bool operator!=(const S &other) const { return !(*this == other); }
|
|
bool operator<(const S &other) const { return I() < other.I(); }
|
|
bool operator>(const S &other) const { return other < *this; }
|
|
bool operator<=(const S &other) const { return !(*this > other); }
|
|
bool operator>=(const S &other) const { return !(*this < other); }
|
|
|
|
S &operator++() { return *this += 1; }
|
|
S operator++(int) { S ret(*this); ++*this; return ret; }
|
|
S &operator+=(std::ptrdiff_t amount) { I() += amount; return *this; }
|
|
S operator+(std::ptrdiff_t amount) const { S ret(*this); ret += amount; return ret; }
|
|
|
|
S &operator--() { return *this -= 1; }
|
|
S operator--(int) { S ret(*this); --*this; return ret; }
|
|
S &operator-=(std::ptrdiff_t amount) { I() += (-amount); return *this; }
|
|
S operator-(std::ptrdiff_t amount) const { S ret(*this); ret -= amount; return ret; }
|
|
|
|
std::ptrdiff_t operator-(const S &other) const { return I() - other.I(); }
|
|
|
|
Proxy operator*() const { return p_; }
|
|
Proxy *operator->() { return &p_; }
|
|
const Proxy *operator->() const { return &p_; }
|
|
Proxy operator[](std::ptrdiff_t amount) const { return *(*this + amount); }
|
|
|
|
const InnerIterator &Inner() { return p_.Inner(); }
|
|
|
|
private:
|
|
InnerIterator &I() { return p_.Inner(); }
|
|
const InnerIterator &I() const { return p_.Inner(); }
|
|
|
|
Proxy p_;
|
|
};
|
|
|
|
template <class Proxy> ProxyIterator<Proxy> operator+(std::ptrdiff_t amount, const ProxyIterator<Proxy> &it) {
|
|
return it + amount;
|
|
}
|
|
|
|
} // namespace util
|
|
|
|
#endif // UTIL_PROXY_ITERATOR_H
|