fanlusky
80b1fc42a1
词格原先的整句打分是一份没校准过的启发式 unigram，质量很差。这里内嵌 libime 在用的 kenlm 查询子集，改由它的三元模型 sc.lm 来打路径分。 - engine/ngram 只编 kenlm 的 query / trie-build 路径。不定义 WITH_THREADS， parallel_read.cc 因此退化成串行实现，整个子集零 Boost 依赖。 KENLM_MAX_ORDER=3 必须与建模阶数一致，改它等于读不懂已发布的模型文件。 - ngram::LanguageModel 是唯一 include kenlm 头文件的地方，对外只给出一个纯 std 的头，_HAS_AUTO_PTR_ETC 和 kenlm 的 include 目录都保持 PRIVATE。 - shared_language_model 按资源路径共享一份 mmap。kenlm 的查询是只读的且把 上下文放在调用方的 State 里，所以并发会话不需要额外加锁。 - 模型缺失时 valid() 为假，词格退回原来的启发式打分：整句候选只是变差，不会 停止工作。测试覆盖了缺模型降级与按路径共享。 - 出货的 ARPA 不带 <s>/</s>，所以解码从 null_state() 起算而非 begin_state()。 kenlm 是上游副本，不重排版、不进 clang-format，其 LGPL-2.1+ 与 BSD 例外清单 已补进 THIRD_PARTY_NOTICES.txt。googlepinyinime 那条 Fallback 整句路径不变。
160 lines
4.3 KiB
C++
160 lines
4.3 KiB
C++
#ifndef UTIL_EXCEPTION_H
|
|
#define UTIL_EXCEPTION_H
|
|
|
|
#include "string_stream.hh"
|
|
|
|
#include <exception>
|
|
#include <limits>
|
|
#include <string>
|
|
#include <stdint.h>
|
|
|
|
namespace util {
|
|
|
|
template <class Except, class Data> typename Except::template ExceptionTag<Except&>::Identity operator<<(Except &e, const Data &data);
|
|
|
|
class Exception : public std::exception {
|
|
public:
|
|
Exception() throw();
|
|
virtual ~Exception() throw();
|
|
|
|
const char *what() const throw() { return what_.str().c_str(); }
|
|
|
|
// For use by the UTIL_THROW macros.
|
|
void SetLocation(
|
|
const char *file,
|
|
unsigned int line,
|
|
const char *func,
|
|
const char *child_name,
|
|
const char *condition);
|
|
|
|
private:
|
|
template <class Except, class Data> friend typename Except::template ExceptionTag<Except&>::Identity operator<<(Except &e, const Data &data);
|
|
|
|
// This helps restrict operator<< defined below.
|
|
template <class T> struct ExceptionTag {
|
|
typedef T Identity;
|
|
};
|
|
|
|
StringStream what_;
|
|
};
|
|
|
|
/* This implements the normal operator<< for Exception and all its children.
|
|
* SFINAE means it only applies to Exception. Think of this as an ersatz
|
|
* boost::enable_if.
|
|
*/
|
|
template <class Except, class Data> typename Except::template ExceptionTag<Except&>::Identity operator<<(Except &e, const Data &data) {
|
|
e.what_ << data;
|
|
return e;
|
|
}
|
|
|
|
#ifdef __GNUC__
|
|
#define UTIL_FUNC_NAME __PRETTY_FUNCTION__
|
|
#else
|
|
#ifdef _WIN32
|
|
#define UTIL_FUNC_NAME __FUNCTION__
|
|
#else
|
|
#define UTIL_FUNC_NAME NULL
|
|
#endif
|
|
#endif
|
|
|
|
/* Create an instance of Exception, add the message Modify, and throw it.
|
|
* Modify is appended to the what() message and can contain << for ostream
|
|
* operations.
|
|
*
|
|
* do .. while kludge to swallow trailing ; character
|
|
* http://gcc.gnu.org/onlinedocs/cpp/Swallowing-the-Semicolon.html .
|
|
* Arg can be a constructor argument to the exception.
|
|
*/
|
|
#define UTIL_THROW_BACKEND(Condition, Exception, Arg, Modify) do { \
|
|
Exception UTIL_e Arg; \
|
|
UTIL_e.SetLocation(__FILE__, __LINE__, UTIL_FUNC_NAME, #Exception, Condition); \
|
|
UTIL_e << Modify; \
|
|
throw UTIL_e; \
|
|
} while (0)
|
|
|
|
#define UTIL_THROW_ARG(Exception, Arg, Modify) \
|
|
UTIL_THROW_BACKEND(NULL, Exception, Arg, Modify)
|
|
|
|
#define UTIL_THROW(Exception, Modify) \
|
|
UTIL_THROW_BACKEND(NULL, Exception, , Modify);
|
|
|
|
#define UTIL_THROW2(Modify) \
|
|
UTIL_THROW_BACKEND(NULL, util::Exception, , Modify);
|
|
|
|
#if __GNUC__ >= 3
|
|
#define UTIL_UNLIKELY(x) __builtin_expect (!!(x), 0)
|
|
#else
|
|
#define UTIL_UNLIKELY(x) (x)
|
|
#endif
|
|
|
|
#if __GNUC__ >= 3
|
|
#define UTIL_LIKELY(x) __builtin_expect (!!(x), 1)
|
|
#else
|
|
#define UTIL_LIKELY(x) (x)
|
|
#endif
|
|
|
|
#define UTIL_THROW_IF_ARG(Condition, Exception, Arg, Modify) do { \
|
|
if (UTIL_UNLIKELY(Condition)) { \
|
|
UTIL_THROW_BACKEND(#Condition, Exception, Arg, Modify); \
|
|
} \
|
|
} while (0)
|
|
|
|
#define UTIL_THROW_IF(Condition, Exception, Modify) \
|
|
UTIL_THROW_IF_ARG(Condition, Exception, , Modify)
|
|
|
|
#define UTIL_THROW_IF2(Condition, Modify) \
|
|
UTIL_THROW_IF_ARG(Condition, util::Exception, , Modify)
|
|
|
|
// Exception that records errno and adds it to the message.
|
|
class ErrnoException : public Exception {
|
|
public:
|
|
ErrnoException() throw();
|
|
|
|
virtual ~ErrnoException() throw();
|
|
|
|
int Error() const throw() { return errno_; }
|
|
|
|
private:
|
|
int errno_;
|
|
};
|
|
|
|
// file wasn't there, or couldn't be open for some reason
|
|
class FileOpenException : public Exception {
|
|
public:
|
|
FileOpenException() throw() {}
|
|
~FileOpenException() throw() {}
|
|
};
|
|
|
|
// Utilities for overflow checking.
|
|
class OverflowException : public Exception {
|
|
public:
|
|
OverflowException() throw();
|
|
~OverflowException() throw();
|
|
};
|
|
|
|
template <unsigned len> inline std::size_t CheckOverflowInternal(uint64_t value) {
|
|
UTIL_THROW_IF(value > static_cast<uint64_t>(std::numeric_limits<std::size_t>::max()), OverflowException, "Integer overflow detected. This model is too big for 32-bit code.");
|
|
return static_cast<std::size_t>(value);
|
|
}
|
|
|
|
template <> inline std::size_t CheckOverflowInternal<8>(uint64_t value) {
|
|
return value;
|
|
}
|
|
|
|
inline std::size_t CheckOverflow(uint64_t value) {
|
|
return CheckOverflowInternal<sizeof(std::size_t)>(value);
|
|
}
|
|
|
|
#if defined(_WIN32) || defined(_WIN64)
|
|
/* Thrown for Windows specific operations. */
|
|
class WindowsException : public Exception {
|
|
public:
|
|
WindowsException() throw();
|
|
~WindowsException() throw();
|
|
};
|
|
#endif
|
|
|
|
} // namespace util
|
|
|
|
#endif // UTIL_EXCEPTION_H
|