词格原先的整句打分是一份没校准过的启发式 unigram,质量很差。这里内嵌 libime 在用的 kenlm 查询子集,改由它的三元模型 sc.lm 来打路径分。 - engine/ngram 只编 kenlm 的 query / trie-build 路径。不定义 WITH_THREADS, parallel_read.cc 因此退化成串行实现,整个子集零 Boost 依赖。 KENLM_MAX_ORDER=3 必须与建模阶数一致,改它等于读不懂已发布的模型文件。 - ngram::LanguageModel 是唯一 include kenlm 头文件的地方,对外只给出一个纯 std 的头,_HAS_AUTO_PTR_ETC 和 kenlm 的 include 目录都保持 PRIVATE。 - shared_language_model 按资源路径共享一份 mmap。kenlm 的查询是只读的且把 上下文放在调用方的 State 里,所以并发会话不需要额外加锁。 - 模型缺失时 valid() 为假,词格退回原来的启发式打分:整句候选只是变差,不会 停止工作。测试覆盖了缺模型降级与按路径共享。 - 出货的 ARPA 不带 <s>/</s>,所以解码从 null_state() 起算而非 begin_state()。 kenlm 是上游副本,不重排版、不进 clang-format,其 LGPL-2.1+ 与 BSD 例外清单 已补进 THIRD_PARTY_NOTICES.txt。googlepinyinime 那条 Fallback 整句路径不变。
147 lines
4.8 KiB
C++
147 lines
4.8 KiB
C++
#ifndef UTIL_JOINT_SORT_H
|
|
#define UTIL_JOINT_SORT_H
|
|
|
|
/* A terrifying amount of C++ to coax std::sort into soring one range while
|
|
* also permuting another range the same way.
|
|
*/
|
|
|
|
#include "proxy_iterator.hh"
|
|
|
|
#include <algorithm>
|
|
#include <functional>
|
|
|
|
namespace util {
|
|
|
|
namespace detail {
|
|
|
|
template <class KeyIter, class ValueIter> class JointProxy;
|
|
|
|
template <class KeyIter, class ValueIter> class JointIter {
|
|
public:
|
|
JointIter() {}
|
|
|
|
JointIter(const KeyIter &key_iter, const ValueIter &value_iter) : key_(key_iter), value_(value_iter) {}
|
|
|
|
bool operator==(const JointIter<KeyIter, ValueIter> &other) const { return key_ == other.key_; }
|
|
|
|
bool operator<(const JointIter<KeyIter, ValueIter> &other) const { return (key_ < other.key_); }
|
|
|
|
std::ptrdiff_t operator-(const JointIter<KeyIter, ValueIter> &other) const { return key_ - other.key_; }
|
|
|
|
JointIter<KeyIter, ValueIter> &operator+=(std::ptrdiff_t amount) {
|
|
key_ += amount;
|
|
value_ += amount;
|
|
return *this;
|
|
}
|
|
|
|
friend void swap(JointIter &first, JointIter &second) {
|
|
using std::swap;
|
|
swap(first.key_, second.key_);
|
|
swap(first.value_, second.value_);
|
|
}
|
|
|
|
void DeepSwap(JointIter &other) {
|
|
using std::swap;
|
|
swap(*key_, *other.key_);
|
|
swap(*value_, *other.value_);
|
|
}
|
|
|
|
private:
|
|
friend class JointProxy<KeyIter, ValueIter>;
|
|
KeyIter key_;
|
|
ValueIter value_;
|
|
};
|
|
|
|
template <class KeyIter, class ValueIter> class JointProxy {
|
|
private:
|
|
typedef JointIter<KeyIter, ValueIter> InnerIterator;
|
|
|
|
public:
|
|
typedef struct {
|
|
typename std::iterator_traits<KeyIter>::value_type key;
|
|
typename std::iterator_traits<ValueIter>::value_type value;
|
|
const typename std::iterator_traits<KeyIter>::value_type &GetKey() const { return key; }
|
|
} value_type;
|
|
|
|
JointProxy(const KeyIter &key_iter, const ValueIter &value_iter) : inner_(key_iter, value_iter) {}
|
|
JointProxy(const JointProxy<KeyIter, ValueIter> &other) : inner_(other.inner_) {}
|
|
|
|
operator value_type() const {
|
|
value_type ret;
|
|
ret.key = *inner_.key_;
|
|
ret.value = *inner_.value_;
|
|
return ret;
|
|
}
|
|
|
|
JointProxy &operator=(const JointProxy &other) {
|
|
*inner_.key_ = *other.inner_.key_;
|
|
*inner_.value_ = *other.inner_.value_;
|
|
return *this;
|
|
}
|
|
|
|
JointProxy &operator=(const value_type &other) {
|
|
*inner_.key_ = other.key;
|
|
*inner_.value_ = other.value;
|
|
return *this;
|
|
}
|
|
|
|
typename std::iterator_traits<KeyIter>::reference GetKey() const {
|
|
return *(inner_.key_);
|
|
}
|
|
|
|
friend void swap(JointProxy<KeyIter, ValueIter> first, JointProxy<KeyIter, ValueIter> second) {
|
|
first.Inner().DeepSwap(second.Inner());
|
|
}
|
|
|
|
private:
|
|
friend class ProxyIterator<JointProxy<KeyIter, ValueIter> >;
|
|
|
|
InnerIterator &Inner() { return inner_; }
|
|
const InnerIterator &Inner() const { return inner_; }
|
|
InnerIterator inner_;
|
|
};
|
|
|
|
template <class Proxy, class Less> class LessWrapper : public std::binary_function<const typename Proxy::value_type &, const typename Proxy::value_type &, bool> {
|
|
public:
|
|
explicit LessWrapper(const Less &less) : less_(less) {}
|
|
|
|
bool operator()(const Proxy &left, const Proxy &right) const {
|
|
return less_(left.GetKey(), right.GetKey());
|
|
}
|
|
bool operator()(const Proxy &left, const typename Proxy::value_type &right) const {
|
|
return less_(left.GetKey(), right.GetKey());
|
|
}
|
|
bool operator()(const typename Proxy::value_type &left, const Proxy &right) const {
|
|
return less_(left.GetKey(), right.GetKey());
|
|
}
|
|
bool operator()(const typename Proxy::value_type &left, const typename Proxy::value_type &right) const {
|
|
return less_(left.GetKey(), right.GetKey());
|
|
}
|
|
|
|
private:
|
|
const Less less_;
|
|
};
|
|
|
|
} // namespace detail
|
|
|
|
template <class KeyIter, class ValueIter> class PairedIterator : public ProxyIterator<detail::JointProxy<KeyIter, ValueIter> > {
|
|
public:
|
|
PairedIterator(const KeyIter &key, const ValueIter &value) :
|
|
ProxyIterator<detail::JointProxy<KeyIter, ValueIter> >(detail::JointProxy<KeyIter, ValueIter>(key, value)) {}
|
|
};
|
|
|
|
template <class KeyIter, class ValueIter, class Less> void JointSort(const KeyIter &key_begin, const KeyIter &key_end, const ValueIter &value_begin, const Less &less) {
|
|
ProxyIterator<detail::JointProxy<KeyIter, ValueIter> > full_begin(detail::JointProxy<KeyIter, ValueIter>(key_begin, value_begin));
|
|
detail::LessWrapper<detail::JointProxy<KeyIter, ValueIter>, Less> less_wrap(less);
|
|
std::sort(full_begin, full_begin + (key_end - key_begin), less_wrap);
|
|
}
|
|
|
|
|
|
template <class KeyIter, class ValueIter> void JointSort(const KeyIter &key_begin, const KeyIter &key_end, const ValueIter &value_begin) {
|
|
JointSort(key_begin, key_end, value_begin, std::less<typename std::iterator_traits<KeyIter>::value_type>());
|
|
}
|
|
|
|
} // namespace util
|
|
|
|
#endif // UTIL_JOINT_SORT_H
|