fanlusky
80b1fc42a1
词格原先的整句打分是一份没校准过的启发式 unigram，质量很差。这里内嵌 libime 在用的 kenlm 查询子集，改由它的三元模型 sc.lm 来打路径分。 - engine/ngram 只编 kenlm 的 query / trie-build 路径。不定义 WITH_THREADS， parallel_read.cc 因此退化成串行实现，整个子集零 Boost 依赖。 KENLM_MAX_ORDER=3 必须与建模阶数一致，改它等于读不懂已发布的模型文件。 - ngram::LanguageModel 是唯一 include kenlm 头文件的地方，对外只给出一个纯 std 的头，_HAS_AUTO_PTR_ETC 和 kenlm 的 include 目录都保持 PRIVATE。 - shared_language_model 按资源路径共享一份 mmap。kenlm 的查询是只读的且把 上下文放在调用方的 State 里，所以并发会话不需要额外加锁。 - 模型缺失时 valid() 为假，词格退回原来的启发式打分：整句候选只是变差，不会 停止工作。测试覆盖了缺模型降级与按路径共享。 - 出货的 ARPA 不带 <s>/</s>，所以解码从 null_state() 起算而非 begin_state()。 kenlm 是上游副本，不重排版、不进 clang-format，其 LGPL-2.1+ 与 BSD 例外清单 已补进 THIRD_PARTY_NOTICES.txt。googlepinyinime 那条 Fallback 整句路径不变。
81 lines
3.3 KiB
C++
81 lines
3.3 KiB
C++
#ifndef UTIL_MULTI_INTERSECTION_H
|
|
#define UTIL_MULTI_INTERSECTION_H
|
|
|
|
#include <boost/optional.hpp>
|
|
#include <boost/range/iterator_range.hpp>
|
|
|
|
#include <algorithm>
|
|
#include <functional>
|
|
#include <vector>
|
|
|
|
namespace util {
|
|
|
|
namespace detail {
|
|
template <class Range> struct RangeLessBySize : public std::binary_function<const Range &, const Range &, bool> {
|
|
bool operator()(const Range &left, const Range &right) const {
|
|
return left.size() < right.size();
|
|
}
|
|
};
|
|
|
|
/* Takes sets specified by their iterators and a boost::optional containing
|
|
* the lowest intersection if any. Each set must be sorted in increasing
|
|
* order. sets is changed to truncate the beginning of each sequence to the
|
|
* location of the match or an empty set. Precondition: sets is not empty
|
|
* since the intersection over null is the universe and this function does not
|
|
* know the universe.
|
|
*/
|
|
template <class Iterator, class Less> boost::optional<typename std::iterator_traits<Iterator>::value_type> FirstIntersectionSorted(std::vector<boost::iterator_range<Iterator> > &sets, const Less &less = std::less<typename std::iterator_traits<Iterator>::value_type>()) {
|
|
typedef std::vector<boost::iterator_range<Iterator> > Sets;
|
|
typedef typename std::iterator_traits<Iterator>::value_type Value;
|
|
|
|
assert(!sets.empty());
|
|
|
|
if (sets.front().empty()) return boost::optional<Value>();
|
|
// Possibly suboptimal to copy for general Value; makes unsigned int go slightly faster.
|
|
Value highest(sets.front().front());
|
|
for (typename Sets::iterator i(sets.begin()); i != sets.end(); ) {
|
|
i->advance_begin(std::lower_bound(i->begin(), i->end(), highest, less) - i->begin());
|
|
if (i->empty()) return boost::optional<Value>();
|
|
if (less(highest, i->front())) {
|
|
highest = i->front();
|
|
// start over
|
|
i = sets.begin();
|
|
} else {
|
|
++i;
|
|
}
|
|
}
|
|
return boost::optional<Value>(highest);
|
|
}
|
|
|
|
} // namespace detail
|
|
|
|
template <class Iterator, class Less> boost::optional<typename std::iterator_traits<Iterator>::value_type> FirstIntersection(std::vector<boost::iterator_range<Iterator> > &sets, const Less less) {
|
|
assert(!sets.empty());
|
|
|
|
std::sort(sets.begin(), sets.end(), detail::RangeLessBySize<boost::iterator_range<Iterator> >());
|
|
return detail::FirstIntersectionSorted(sets, less);
|
|
}
|
|
|
|
template <class Iterator> boost::optional<typename std::iterator_traits<Iterator>::value_type> FirstIntersection(std::vector<boost::iterator_range<Iterator> > &sets) {
|
|
return FirstIntersection(sets, std::less<typename std::iterator_traits<Iterator>::value_type>());
|
|
}
|
|
|
|
template <class Iterator, class Output, class Less> void AllIntersection(std::vector<boost::iterator_range<Iterator> > &sets, Output &out, const Less less) {
|
|
typedef typename std::iterator_traits<Iterator>::value_type Value;
|
|
assert(!sets.empty());
|
|
|
|
std::sort(sets.begin(), sets.end(), detail::RangeLessBySize<boost::iterator_range<Iterator> >());
|
|
boost::optional<Value> ret;
|
|
for (boost::optional<Value> ret; (ret = detail::FirstIntersectionSorted(sets, less)); sets.front().advance_begin(1)) {
|
|
out(*ret);
|
|
}
|
|
}
|
|
|
|
template <class Iterator, class Output> void AllIntersection(std::vector<boost::iterator_range<Iterator> > &sets, Output &out) {
|
|
AllIntersection(sets, out, std::less<typename std::iterator_traits<Iterator>::value_type>());
|
|
}
|
|
|
|
} // namespace util
|
|
|
|
#endif // UTIL_MULTI_INTERSECTION_H
|