词格原先的整句打分是一份没校准过的启发式 unigram,质量很差。这里内嵌 libime 在用的 kenlm 查询子集,改由它的三元模型 sc.lm 来打路径分。 - engine/ngram 只编 kenlm 的 query / trie-build 路径。不定义 WITH_THREADS, parallel_read.cc 因此退化成串行实现,整个子集零 Boost 依赖。 KENLM_MAX_ORDER=3 必须与建模阶数一致,改它等于读不懂已发布的模型文件。 - ngram::LanguageModel 是唯一 include kenlm 头文件的地方,对外只给出一个纯 std 的头,_HAS_AUTO_PTR_ETC 和 kenlm 的 include 目录都保持 PRIVATE。 - shared_language_model 按资源路径共享一份 mmap。kenlm 的查询是只读的且把 上下文放在调用方的 State 里,所以并发会话不需要额外加锁。 - 模型缺失时 valid() 为假,词格退回原来的启发式打分:整句候选只是变差,不会 停止工作。测试覆盖了缺模型降级与按路径共享。 - 出货的 ARPA 不带 <s>/</s>,所以解码从 null_state() 起算而非 begin_state()。 kenlm 是上游副本,不重排版、不进 clang-format,其 LGPL-2.1+ 与 BSD 例外清单 已补进 THIRD_PARTY_NOTICES.txt。googlepinyinime 那条 Fallback 整句路径不变。
157 lines
3.4 KiB
C++
157 lines
3.4 KiB
C++
#ifndef UTIL_PCQUEUE_H
|
|
#define UTIL_PCQUEUE_H
|
|
|
|
#include "exception.hh"
|
|
|
|
#include <boost/interprocess/sync/interprocess_semaphore.hpp>
|
|
#include <boost/scoped_array.hpp>
|
|
#include <boost/thread/mutex.hpp>
|
|
#include <boost/utility.hpp>
|
|
|
|
#include <cerrno>
|
|
|
|
#ifdef __APPLE__
|
|
#include <mach/semaphore.h>
|
|
#include <mach/task.h>
|
|
#include <mach/mach_traps.h>
|
|
#include <mach/mach.h>
|
|
#endif // __APPLE__
|
|
|
|
namespace util {
|
|
|
|
/* OS X Maverick and Boost interprocess were doing "Function not implemented."
|
|
* So this is my own wrapper around the mach kernel APIs.
|
|
*/
|
|
#ifdef __APPLE__
|
|
|
|
#define MACH_CALL(call) UTIL_THROW_IF(KERN_SUCCESS != (call), Exception, "Mach call failure")
|
|
|
|
class Semaphore {
|
|
public:
|
|
explicit Semaphore(int value) : task_(mach_task_self()) {
|
|
MACH_CALL(semaphore_create(task_, &back_, SYNC_POLICY_FIFO, value));
|
|
}
|
|
|
|
~Semaphore() {
|
|
MACH_CALL(semaphore_destroy(task_, back_));
|
|
}
|
|
|
|
void wait() {
|
|
MACH_CALL(semaphore_wait(back_));
|
|
}
|
|
|
|
void post() {
|
|
MACH_CALL(semaphore_signal(back_));
|
|
}
|
|
|
|
private:
|
|
semaphore_t back_;
|
|
task_t task_;
|
|
};
|
|
|
|
inline void WaitSemaphore(Semaphore &semaphore) {
|
|
semaphore.wait();
|
|
}
|
|
|
|
#else
|
|
typedef boost::interprocess::interprocess_semaphore Semaphore;
|
|
|
|
inline void WaitSemaphore (Semaphore &on) {
|
|
while (1) {
|
|
try {
|
|
on.wait();
|
|
break;
|
|
}
|
|
catch (boost::interprocess::interprocess_exception &e) {
|
|
if (e.get_native_error() != EINTR) {
|
|
throw;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
#endif // __APPLE__
|
|
|
|
/**
|
|
* Producer consumer queue safe for multiple producers and multiple consumers.
|
|
* T must be default constructable and have operator=.
|
|
* The value is copied twice for Consume(T &out) or three times for Consume(),
|
|
* so larger objects should be passed via pointer.
|
|
* Strong exception guarantee if operator= throws. Undefined if semaphores throw.
|
|
*/
|
|
template <class T> class PCQueue : boost::noncopyable {
|
|
public:
|
|
explicit PCQueue(size_t size)
|
|
: empty_(size), used_(0),
|
|
storage_(new T[size]),
|
|
end_(storage_.get() + size),
|
|
produce_at_(storage_.get()),
|
|
consume_at_(storage_.get()) {}
|
|
|
|
// Add a value to the queue.
|
|
void Produce(const T &val) {
|
|
WaitSemaphore(empty_);
|
|
{
|
|
boost::unique_lock<boost::mutex> produce_lock(produce_at_mutex_);
|
|
try {
|
|
*produce_at_ = val;
|
|
}
|
|
catch (...) {
|
|
empty_.post();
|
|
throw;
|
|
}
|
|
if (++produce_at_ == end_) produce_at_ = storage_.get();
|
|
}
|
|
used_.post();
|
|
}
|
|
|
|
// Consume a value, assigning it to out.
|
|
T& Consume(T &out) {
|
|
WaitSemaphore(used_);
|
|
{
|
|
boost::unique_lock<boost::mutex> consume_lock(consume_at_mutex_);
|
|
try {
|
|
out = *consume_at_;
|
|
}
|
|
catch (...) {
|
|
used_.post();
|
|
throw;
|
|
}
|
|
if (++consume_at_ == end_) consume_at_ = storage_.get();
|
|
}
|
|
empty_.post();
|
|
return out;
|
|
}
|
|
|
|
// Convenience version of Consume that copies the value to return.
|
|
// The other version is faster.
|
|
T Consume() {
|
|
T ret;
|
|
Consume(ret);
|
|
return ret;
|
|
}
|
|
|
|
private:
|
|
// Number of empty spaces in storage_.
|
|
Semaphore empty_;
|
|
// Number of occupied spaces in storage_.
|
|
Semaphore used_;
|
|
|
|
boost::scoped_array<T> storage_;
|
|
|
|
T *const end_;
|
|
|
|
// Index for next write in storage_.
|
|
T *produce_at_;
|
|
boost::mutex produce_at_mutex_;
|
|
|
|
// Index for next read from storage_.
|
|
T *consume_at_;
|
|
boost::mutex consume_at_mutex_;
|
|
|
|
};
|
|
|
|
} // namespace util
|
|
|
|
#endif // UTIL_PCQUEUE_H
|