From 0976c3072cb54b4dff8a42d9d837f2b85ce78e38 Mon Sep 17 00:00:00 2001 From: Jia Sui Date: Fri, 2 Oct 2026 20:15:39 +0800 Subject: [PATCH] =?UTF-8?q?engine(ngram):=20=E4=B8=87=E8=B1=A1=E8=AF=AD?= =?UTF-8?q?=E6=B3=95=E6=A8=A1=E5=9E=8B=EF=BC=88octagram=20.gram=EF=BC=89?= =?UTF-8?q?=E8=AF=BB=E5=8F=96=E7=AB=AF=E8=BD=AC=E4=B8=BA=E4=BA=A7=E5=93=81?= =?UTF-8?q?=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 从评测工具的移植版升级为引擎库成员：装载从整文件 ReadFile 换成 Win32 mmap（RSS 只随命中页增长），新增进程内按路径共享的 shared_gram_db（照 ngram::shared_language_model 范式，查询全只读、多会话并发安全）。eval_lm_compare 改用同一份实现，评测与产品不再有两套读取器；darts.h（BSD）随目录入库。 --- engine/ngram/CMakeLists.txt | 5 +- engine/ngram/octagram/COPYING.darts-clone | 10 + engine/ngram/octagram/darts.h | 1898 +++++++++++++++++++++ engine/ngram/octagram/octagram_gram.cpp | 348 ++++ engine/ngram/octagram/octagram_gram.h | 122 ++ engine/tests/src/test_pinyin.cpp | 45 +- 6 files changed, 2424 insertions(+), 4 deletions(-) create mode 100644 engine/ngram/octagram/COPYING.darts-clone create mode 100644 engine/ngram/octagram/darts.h create mode 100644 engine/ngram/octagram/octagram_gram.cpp create mode 100644 engine/ngram/octagram/octagram_gram.h diff --git a/engine/ngram/CMakeLists.txt b/engine/ngram/CMakeLists.txt index 4a60235c..e8fb8892 100644 --- a/engine/ngram/CMakeLists.txt +++ b/engine/ngram/CMakeLists.txt @@ -43,7 +43,10 @@ list(FILTER METASEQUOIA_IME_KENLM_SOURCES EXCLUDE REGEX "_main\\.cc$") add_library(metasequoia_ime_ngram STATIC ${METASEQUOIA_IME_KENLM_SOURCES} - "${CMAKE_CURRENT_SOURCE_DIR}/language_model.cpp") + "${CMAKE_CURRENT_SOURCE_DIR}/language_model.cpp" + # 万象语法模型（octagram .gram）的读取端，与 kenlm 同库：它和语言模型封装 + # 一样是「引擎其余部分只看头文件」的独立模块，Win32 mmap 的实现细节留在 cpp 里。 + "${CMAKE_CURRENT_SOURCE_DIR}/octagram/octagram_gram.cpp") # PUBLIC 指向仓库根，这样调用方按 AGENTS.md 的约定写 # #include "engine/ngram/language_model.h"。kenlm 的头保持 PRIVATE。 diff --git a/engine/ngram/octagram/COPYING.darts-clone b/engine/ngram/octagram/COPYING.darts-clone new file mode 100644 index 00000000..cd30e91f --- /dev/null +++ b/engine/ngram/octagram/COPYING.darts-clone @@ -0,0 +1,10 @@ +Copyright (c) 2008-2013, Susumu Yata +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +- Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. +- Neither the name of the nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/engine/ngram/octagram/darts.h b/engine/ngram/octagram/darts.h new file mode 100644 index 00000000..632b3cbd --- /dev/null +++ b/engine/ngram/octagram/darts.h @@ -0,0 +1,1898 @@ +#ifndef DARTS_H_ +#define DARTS_H_ + +#include +#include +#include + +#define DARTS_VERSION "0.32" + +// DARTS_THROW() throws a whose message starts with the +// file name and the line number. For example, DARTS_THROW("error message") at +// line 123 of "darts.h" throws a which has a pointer to +// "darts.h:123: exception: error message". The message is available by using +// what() as well as that of . +#define DARTS_INT_TO_STR(value) #value +#define DARTS_LINE_TO_STR(line) DARTS_INT_TO_STR(line) +#define DARTS_LINE_STR DARTS_LINE_TO_STR(__LINE__) +#define DARTS_THROW(msg) throw Darts::Details::Exception( \ + __FILE__ ":" DARTS_LINE_STR ": exception: " msg) + +namespace Darts { + +// The following namespace hides the internal types and classes. +namespace Details { + +// This header assumes that and are 32-bit integer types. +// +// Darts-clone keeps values associated with keys. The type of the values is +// . Note that the values must be positive integers because the +// most significant bit (MSB) of each value is used to represent whether the +// corresponding unit is a leaf or not. Also, the keys are represented by +// sequences of s. is the unsigned type of . +typedef char char_type; +typedef unsigned char uchar_type; +typedef int value_type; + +// The main structure of Darts-clone is an array of s, and the +// unit type is actually a wrapper of . +typedef unsigned int id_type; + +// is the type of callback functions for reporting the +// progress of building a dictionary. See also build() of . +// The 1st argument receives the progress value and the 2nd argument receives +// the maximum progress value. A usage example is to show the progress +// percentage, 100.0 * (the 1st argument) / (the 2nd argument). +typedef int (*progress_func_type)(std::size_t, std::size_t); + +// is the type of double-array units and it is a wrapper of +// in practice. +class DoubleArrayUnit { + public: + DoubleArrayUnit() : unit_() {} + + // has_leaf() returns whether a leaf unit is immediately derived from the + // unit (true) or not (false). + bool has_leaf() const { + return ((unit_ >> 8) & 1) == 1; + } + // value() returns the value stored in the unit, and thus value() is + // available when and only when the unit is a leaf unit. + value_type value() const { + return static_cast(unit_ & ((1U << 31) - 1)); + } + + // label() returns the label associated with the unit. Note that a leaf unit + // always returns an invalid label. For this feature, leaf unit's label() + // returns an that has the MSB of 1. + id_type label() const { + return unit_ & ((1U << 31) | 0xFF); + } + // offset() returns the offset from the unit to its derived units. + id_type offset() const { + return (unit_ >> 10) << ((unit_ & (1U << 9)) >> 6); + } + + private: + id_type unit_; + + // Copyable. +}; + +// Darts-clone throws an for memory allocation failure, invalid +// arguments or a too large offset. The last case means that there are too many +// keys in the given set of keys. Note that the `msg' of must be a +// constant or static string because an keeps only a pointer to +// that string. +class Exception : public std::exception { + public: + explicit Exception(const char *msg = NULL) throw() : msg_(msg) {} + Exception(const Exception &rhs) throw() : msg_(rhs.msg_) {} + virtual ~Exception() throw() {} + + // overrides what() of . + virtual const char *what() const throw() { + return (msg_ != NULL) ? msg_ : ""; + } + + private: + const char *msg_; + + // Disallows operator=. + Exception &operator=(const Exception &); +}; + +} // namespace Details + +// is the interface of Darts-clone. Note that other +// classes should not be accessed from outside. +// +// has 4 template arguments but only the 3rd one is used as +// the type of values. Note that the given is used only from outside, and +// the internal value type is not changed from . +// In build(), given values are casted from to +// by using static_cast. On the other hand, values are casted from +// to in searching dictionaries. +template +class DoubleArrayImpl { + public: + // Even if this is changed, the internal value type is still + // . Other types, such as 64-bit integer types + // and floating-point number types, should not be used. + typedef T value_type; + // A key is reprenseted by a sequence of s. For example, + // exactMatchSearch() takes a . + typedef Details::char_type key_type; + // In searching dictionaries, the values associated with the matched keys are + // stored into or returned as s. + typedef value_type result_type; + + // enables applications to get the lengths of the matched + // keys in addition to the values. + struct result_pair_type { + value_type value; + std::size_t length; + }; + + // The constructor initializes member variables with 0 and NULLs. + DoubleArrayImpl() : size_(0), array_(NULL), buf_(NULL) {} + // The destructor frees memory allocated for units and then initializes + // member variables with 0 and NULLs. + virtual ~DoubleArrayImpl() { + clear(); + } + + // has 2 kinds of set_result()s. The 1st set_result() is to + // set a value to a . The 2nd set_result() is to set a value and + // a length to a . By using set_result()s, search methods + // can return the 2 kinds of results in the same way. + // Why the set_result()s are non-static? It is for compatibility. + // + // The 1st set_result() takes a length as the 3rd argument but it is not + // used. If a compiler does a good job, codes for getting the length may be + // removed. + void set_result(value_type *result, value_type value, std::size_t) const { + *result = value; + } + // The 2nd set_result() uses both `value' and `length'. + void set_result(result_pair_type *result, + value_type value, std::size_t length) const { + result->value = value; + result->length = length; + } + + // set_array() calls clear() in order to free memory allocated to the old + // array and then sets a new array. This function is useful to set a memory- + // mapped array. Note that the array set by set_array() is not freed in + // clear() and the destructor of . + // set_array() can also set the size of the new array but the size is not + // used in search methods. So it works well even if the 2nd argument is 0 or + // omitted. Remember that size() and total_size() returns 0 in such a case. + void set_array(const void *ptr, std::size_t size = 0) { + clear(); + array_ = static_cast(ptr); + size_ = size; + } + // array() returns a pointer to the array of units. + const void *array() const { + return array_; + } + + // clear() frees memory allocated to units and then initializes member + // variables with 0 and NULLs. Note that clear() does not free memory if the + // array of units was set by set_array(). In such a case, `array_' is not + // NULL and `buf_' is NULL. + void clear() { + size_ = 0; + array_ = NULL; + if (buf_ != NULL) { + delete[] buf_; + buf_ = NULL; + } + } + + // unit_size() returns the size of each unit. The size must be 4 bytes. + std::size_t unit_size() const { + return sizeof(unit_type); + } + // size() returns the number of units. It can be 0 if set_array() is used. + std::size_t size() const { + return size_; + } + // total_size() returns the number of bytes allocated to the array of units. + // It can be 0 if set_array() is used. + std::size_t total_size() const { + return unit_size() * size(); + } + // nonzero_size() exists for compatibility. It always returns the number of + // units because it takes long time to count the number of non-zero units. + std::size_t nonzero_size() const { + return size(); + } + + // build() constructs a dictionary from given key-value pairs. If `lengths' + // is NULL, `keys' is handled as an array of zero-terminated strings. If + // `values' is NULL, the index in `keys' is associated with each key, i.e. + // the ith key has (i - 1) as its value. + // Note that the key-value pairs must be arranged in key order and the values + // must not be negative. Also, if there are duplicate keys, only the first + // pair will be stored in the resultant dictionary. + // `progress_func' is a pointer to a callback function. If it is not NULL, + // it will be called in build() so that the caller can check the progress of + // dictionary construction. For details, please see the definition of + // . + // The return value of build() is 0, and it indicates the success of the + // operation. Otherwise, build() throws a , which is a + // derived class of . + // build() uses another construction algorithm if `values' is not NULL. In + // this case, Darts-clone uses a Directed Acyclic Word Graph (DAWG) instead + // of a trie because a DAWG is likely to be more compact than a trie. + int build(std::size_t num_keys, const key_type * const *keys, + const std::size_t *lengths = NULL, const value_type *values = NULL, + Details::progress_func_type progress_func = NULL); + + // open() reads an array of units from the specified file. And if it goes + // well, the old array will be freed and replaced with the new array read + // from the file. `offset' specifies the number of bytes to be skipped before + // reading an array. `size' specifies the number of bytes to be read from the + // file. If the `size' is 0, the whole file will be read. + // open() returns 0 iff the operation succeeds. Otherwise, it returns a + // non-zero value or throws a . The exception is thrown + // when and only when a memory allocation fails. + int open(const char *file_name, const char *mode = "rb", + std::size_t offset = 0, std::size_t size = 0); + // save() writes the array of units into the specified file. `offset' + // specifies the number of bytes to be skipped before writing the array. + // open() returns 0 iff the operation succeeds. Otherwise, it returns a + // non-zero value. + int save(const char *file_name, const char *mode = "wb", + std::size_t offset = 0) const; + + // The 1st exactMatchSearch() tests whether the given key exists or not, and + // if it exists, its value and length are set to `result'. Otherwise, the + // value and the length of `result' are set to -1 and 0 respectively. + // Note that if `length' is 0, `key' is handled as a zero-terminated string. + // `node_pos' specifies the start position of matching. This argument enables + // the combination of exactMatchSearch() and traverse(). For example, if you + // want to test "xyzA", "xyzBC", and "xyzDE", you can use traverse() to get + // the node position corresponding to "xyz" and then you can use + // exactMatchSearch() to test "A", "BC", and "DE" from that position. + // Note that the length of `result' indicates the length from the `node_pos'. + // In the above example, the lengths are { 1, 2, 2 }, not { 4, 5, 5 }. + template + void exactMatchSearch(const key_type *key, U &result, + std::size_t length = 0, std::size_t node_pos = 0) const { + result = exactMatchSearch(key, length, node_pos); + } + // The 2nd exactMatchSearch() returns a result instead of updating the 2nd + // argument. So, the following exactMatchSearch() has only 3 arguments. + template + inline U exactMatchSearch(const key_type *key, std::size_t length = 0, + std::size_t node_pos = 0) const; + + // commonPrefixSearch() searches for keys which match a prefix of the given + // string. If `length' is 0, `key' is handled as a zero-terminated string. + // The values and the lengths of at most `max_num_results' matched keys are + // stored in `results'. commonPrefixSearch() returns the number of matched + // keys. Note that the return value can be larger than `max_num_results' if + // there are more than `max_num_results' matches. If you want to get all the + // results, allocate more spaces and call commonPrefixSearch() again. + // `node_pos' works as well as in exactMatchSearch(). + template + inline std::size_t commonPrefixSearch(const key_type *key, U *results, + std::size_t max_num_results, std::size_t length = 0, + std::size_t node_pos = 0) const; + + // In Darts-clone, a dictionary is a deterministic finite-state automaton + // (DFA) and traverse() tests transitions on the DFA. The initial state is + // `node_pos' and traverse() chooses transitions labeled key[key_pos], + // key[key_pos + 1], ... in order. If there is not a transition labeled + // key[key_pos + i], traverse() terminates the transitions at that state and + // returns -2. Otherwise, traverse() ends without a termination and returns + // -1 or a nonnegative value, -1 indicates that the final state was not an + // accept state. When a nonnegative value is returned, it is the value + // associated with the final accept state. That is, traverse() returns the + // value associated with the given key if it exists. Note that traverse() + // updates `node_pos' and `key_pos' after each transition. + inline value_type traverse(const key_type *key, std::size_t &node_pos, + std::size_t &key_pos, std::size_t length = 0) const; + + private: + typedef Details::uchar_type uchar_type; + typedef Details::id_type id_type; + typedef Details::DoubleArrayUnit unit_type; + + std::size_t size_; + const unit_type *array_; + unit_type *buf_; + + // Disallows copy and assignment. + DoubleArrayImpl(const DoubleArrayImpl &); + DoubleArrayImpl &operator=(const DoubleArrayImpl &); +}; + +// is the typical instance of . It uses +// as the type of values and it is suitable for most cases. +typedef DoubleArrayImpl DoubleArray; + +// The interface section ends here. For using Darts-clone, there is no need +// to read the remaining section, which gives the implementation of +// Darts-clone. + +// +// Member functions of DoubleArrayImpl (except build()). +// + +template +int DoubleArrayImpl::open(const char *file_name, + const char *mode, std::size_t offset, std::size_t size) { +#ifdef _MSC_VER + std::FILE *file; + if (::fopen_s(&file, file_name, mode) != 0) { + return -1; + } +#else + std::FILE *file = std::fopen(file_name, mode); + if (file == NULL) { + return -1; + } +#endif + + if (size == 0) { + if (std::fseek(file, 0, SEEK_END) != 0) { + std::fclose(file); + return -1; + } + size = std::ftell(file) - offset; + } + + if (std::fseek(file, offset, SEEK_SET) != 0) { + std::fclose(file); + return -1; + } + + size /= unit_size(); + unit_type *buf; + try { + buf = new unit_type[size]; + } catch (const std::bad_alloc &) { + std::fclose(file); + DARTS_THROW("failed to open double-array: std::bad_alloc"); + } + + if (std::fread(buf, unit_size(), size, file) != size) { + std::fclose(file); + delete[] buf; + return -1; + } + std::fclose(file); + + clear(); + + size_ = size; + array_ = buf; + buf_ = buf; + return 0; +} + +template +int DoubleArrayImpl::save(const char *file_name, + const char *mode, std::size_t) const { + if (size() == 0) { + return -1; + } + +#ifdef _MSC_VER + std::FILE *file; + if (::fopen_s(&file, file_name, mode) != 0) { + return -1; + } +#else + std::FILE *file = std::fopen(file_name, mode); + if (file == NULL) { + return -1; + } +#endif + + if (std::fwrite(array_, unit_size(), size(), file) != size()) { + std::fclose(file); + return -1; + } + std::fclose(file); + return 0; +} + +template +template +inline U DoubleArrayImpl::exactMatchSearch(const key_type *key, + std::size_t length, std::size_t node_pos) const { + U result; + set_result(&result, static_cast(-1), 0); + + unit_type unit = array_[node_pos]; + if (length != 0) { + for (std::size_t i = 0; i < length; ++i) { + node_pos ^= unit.offset() ^ static_cast(key[i]); + unit = array_[node_pos]; + if (unit.label() != static_cast(key[i])) { + return result; + } + } + } else { + for ( ; key[length] != '\0'; ++length) { + node_pos ^= unit.offset() ^ static_cast(key[length]); + unit = array_[node_pos]; + if (unit.label() != static_cast(key[length])) { + return result; + } + } + } + + if (!unit.has_leaf()) { + return result; + } + unit = array_[node_pos ^ unit.offset()]; + set_result(&result, static_cast(unit.value()), length); + return result; +} + +template +template +inline std::size_t DoubleArrayImpl::commonPrefixSearch( + const key_type *key, U *results, std::size_t max_num_results, + std::size_t length, std::size_t node_pos) const { + std::size_t num_results = 0; + + unit_type unit = array_[node_pos]; + node_pos ^= unit.offset(); + if (length != 0) { + for (std::size_t i = 0; i < length; ++i) { + node_pos ^= static_cast(key[i]); + unit = array_[node_pos]; + if (unit.label() != static_cast(key[i])) { + return num_results; + } + + node_pos ^= unit.offset(); + if (unit.has_leaf()) { + if (num_results < max_num_results) { + set_result(&results[num_results], static_cast( + array_[node_pos].value()), i + 1); + } + ++num_results; + } + } + } else { + for ( ; key[length] != '\0'; ++length) { + node_pos ^= static_cast(key[length]); + unit = array_[node_pos]; + if (unit.label() != static_cast(key[length])) { + return num_results; + } + + node_pos ^= unit.offset(); + if (unit.has_leaf()) { + if (num_results < max_num_results) { + set_result(&results[num_results], static_cast( + array_[node_pos].value()), length + 1); + } + ++num_results; + } + } + } + + return num_results; +} + +template +inline typename DoubleArrayImpl::value_type +DoubleArrayImpl::traverse(const key_type *key, + std::size_t &node_pos, std::size_t &key_pos, std::size_t length) const { + id_type id = static_cast(node_pos); + unit_type unit = array_[id]; + + if (length != 0) { + for ( ; key_pos < length; ++key_pos) { + id ^= unit.offset() ^ static_cast(key[key_pos]); + unit = array_[id]; + if (unit.label() != static_cast(key[key_pos])) { + return static_cast(-2); + } + node_pos = id; + } + } else { + for ( ; key[key_pos] != '\0'; ++key_pos) { + id ^= unit.offset() ^ static_cast(key[key_pos]); + unit = array_[id]; + if (unit.label() != static_cast(key[key_pos])) { + return static_cast(-2); + } + node_pos = id; + } + } + + if (!unit.has_leaf()) { + return static_cast(-1); + } + unit = array_[id ^ unit.offset()]; + return static_cast(unit.value()); +} + +namespace Details { + +// +// Memory management of array. +// + +template +class AutoArray { + public: + explicit AutoArray(T *array = NULL) : array_(array) {} + ~AutoArray() { + clear(); + } + + const T &operator[](std::size_t id) const { + return array_[id]; + } + T &operator[](std::size_t id) { + return array_[id]; + } + + bool empty() const { + return array_ == NULL; + } + + void clear() { + if (array_ != NULL) { + delete[] array_; + array_ = NULL; + } + } + void swap(AutoArray *array) { + T *temp = array_; + array_ = array->array_; + array->array_ = temp; + } + void reset(T *array = NULL) { + AutoArray(array).swap(this); + } + + private: + T *array_; + + // Disallows copy and assignment. + AutoArray(const AutoArray &); + AutoArray &operator=(const AutoArray &); +}; + +// +// Memory management of resizable array. +// + +template +class AutoPool { + public: + AutoPool() : buf_(), size_(0), capacity_(0) {} + ~AutoPool() { clear(); } + + const T &operator[](std::size_t id) const { + return *(reinterpret_cast(&buf_[0]) + id); + } + T &operator[](std::size_t id) { + return *(reinterpret_cast(&buf_[0]) + id); + } + + bool empty() const { + return size_ == 0; + } + std::size_t size() const { + return size_; + } + + void clear() { + resize(0); + buf_.clear(); + size_ = 0; + capacity_ = 0; + } + + void push_back(const T &value) { + append(value); + } + void pop_back() { + (*this)[--size_].~T(); + } + + void append() { + if (size_ == capacity_) + resize_buf(size_ + 1); + new(&(*this)[size_++]) T; + } + void append(const T &value) { + if (size_ == capacity_) + resize_buf(size_ + 1); + new(&(*this)[size_++]) T(value); + } + + void resize(std::size_t size) { + while (size_ > size) { + (*this)[--size_].~T(); + } + if (size > capacity_) { + resize_buf(size); + } + while (size_ < size) { + new(&(*this)[size_++]) T; + } + } + void resize(std::size_t size, const T &value) { + while (size_ > size) { + (*this)[--size_].~T(); + } + if (size > capacity_) { + resize_buf(size); + } + while (size_ < size) { + new(&(*this)[size_++]) T(value); + } + } + + void reserve(std::size_t size) { + if (size > capacity_) { + resize_buf(size); + } + } + + private: + AutoArray buf_; + std::size_t size_; + std::size_t capacity_; + + // Disallows copy and assignment. + AutoPool(const AutoPool &); + AutoPool &operator=(const AutoPool &); + + void resize_buf(std::size_t size); +}; + +template +void AutoPool::resize_buf(std::size_t size) { + std::size_t capacity; + if (size >= capacity_ * 2) { + capacity = size; + } else { + capacity = 1; + while (capacity < size) { + capacity <<= 1; + } + } + + AutoArray buf; + try { + buf.reset(new char[sizeof(T) * capacity]); + } catch (const std::bad_alloc &) { + DARTS_THROW("failed to resize pool: std::bad_alloc"); + } + + if (size_ > 0) { + T *src = reinterpret_cast(&buf_[0]); + T *dest = reinterpret_cast(&buf[0]); + for (std::size_t i = 0; i < size_; ++i) { + new(&dest[i]) T(src[i]); + src[i].~T(); + } + } + + buf_.swap(&buf); + capacity_ = capacity; +} + +// +// Memory management of stack. +// + +template +class AutoStack { + public: + AutoStack() : pool_() {} + ~AutoStack() { + clear(); + } + + const T &top() const { + return pool_[size() - 1]; + } + T &top() { + return pool_[size() - 1]; + } + + bool empty() const { + return pool_.empty(); + } + std::size_t size() const { + return pool_.size(); + } + + void push(const T &value) { + pool_.push_back(value); + } + void pop() { + pool_.pop_back(); + } + + void clear() { + pool_.clear(); + } + + private: + AutoPool pool_; + + // Disallows copy and assignment. + AutoStack(const AutoStack &); + AutoStack &operator=(const AutoStack &); +}; + +// +// Succinct bit vector. +// + +class BitVector { + public: + BitVector() : units_(), ranks_(), num_ones_(0), size_(0) {} + ~BitVector() { + clear(); + } + + bool operator[](std::size_t id) const { + return (units_[id / UNIT_SIZE] >> (id % UNIT_SIZE) & 1) == 1; + } + + id_type rank(std::size_t id) const { + std::size_t unit_id = id / UNIT_SIZE; + return ranks_[unit_id] + pop_count(units_[unit_id] + & (~0U >> (UNIT_SIZE - (id % UNIT_SIZE) - 1))); + } + + void set(std::size_t id, bool bit) { + if (bit) { + units_[id / UNIT_SIZE] |= 1U << (id % UNIT_SIZE); + } else { + units_[id / UNIT_SIZE] &= ~(1U << (id % UNIT_SIZE)); + } + } + + bool empty() const { + return units_.empty(); + } + std::size_t num_ones() const { + return num_ones_; + } + std::size_t size() const { + return size_; + } + + void append() { + if ((size_ % UNIT_SIZE) == 0) { + units_.append(0); + } + ++size_; + } + void build(); + + void clear() { + units_.clear(); + ranks_.clear(); + } + + private: + enum { UNIT_SIZE = sizeof(id_type) * 8 }; + + AutoPool units_; + AutoArray ranks_; + std::size_t num_ones_; + std::size_t size_; + + // Disallows copy and assignment. + BitVector(const BitVector &); + BitVector &operator=(const BitVector &); + + static id_type pop_count(id_type unit) { + unit = ((unit & 0xAAAAAAAA) >> 1) + (unit & 0x55555555); + unit = ((unit & 0xCCCCCCCC) >> 2) + (unit & 0x33333333); + unit = ((unit >> 4) + unit) & 0x0F0F0F0F; + unit += unit >> 8; + unit += unit >> 16; + return unit & 0xFF; + } +}; + +inline void BitVector::build() { + try { + ranks_.reset(new id_type[units_.size()]); + } catch (const std::bad_alloc &) { + DARTS_THROW("failed to build rank index: std::bad_alloc"); + } + + num_ones_ = 0; + for (std::size_t i = 0; i < units_.size(); ++i) { + ranks_[i] = num_ones_; + num_ones_ += pop_count(units_[i]); + } +} + +// +// Keyset. +// + +template +class Keyset { + public: + Keyset(std::size_t num_keys, const char_type * const *keys, + const std::size_t *lengths, const T *values) : + num_keys_(num_keys), keys_(keys), lengths_(lengths), values_(values) {} + + std::size_t num_keys() const { + return num_keys_; + } + const char_type *keys(std::size_t id) const { + return keys_[id]; + } + uchar_type keys(std::size_t key_id, std::size_t char_id) const { + if (has_lengths() && char_id >= lengths_[key_id]) + return '\0'; + return keys_[key_id][char_id]; + } + + bool has_lengths() const { + return lengths_ != NULL; + } + std::size_t lengths(std::size_t id) const { + if (has_lengths()) { + return lengths_[id]; + } + std::size_t length = 0; + while (keys_[id][length] != '\0') { + ++length; + } + return length; + } + + bool has_values() const { + return values_ != NULL; + } + const value_type values(std::size_t id) const { + if (has_values()) { + return static_cast(values_[id]); + } + return static_cast(id); + } + + private: + std::size_t num_keys_; + const char_type * const * keys_; + const std::size_t *lengths_; + const T *values_; + + // Disallows copy and assignment. + Keyset(const Keyset &); + Keyset &operator=(const Keyset &); +}; + +// +// Node of Directed Acyclic Word Graph (DAWG). +// + +class DawgNode { + public: + DawgNode() : child_(0), sibling_(0), label_('\0'), + is_state_(false), has_sibling_(false) {} + + void set_child(id_type child) { + child_ = child; + } + void set_sibling(id_type sibling) { + sibling_ = sibling; + } + void set_value(value_type value) { + child_ = value; + } + void set_label(uchar_type label) { + label_ = label; + } + void set_is_state(bool is_state) { + is_state_ = is_state; + } + void set_has_sibling(bool has_sibling) { + has_sibling_ = has_sibling; + } + + id_type child() const { + return child_; + } + id_type sibling() const { + return sibling_; + } + value_type value() const { + return static_cast(child_); + } + uchar_type label() const { + return label_; + } + bool is_state() const { + return is_state_; + } + bool has_sibling() const { + return has_sibling_; + } + + id_type unit() const { + if (label_ == '\0') { + return (child_ << 1) | (has_sibling_ ? 1 : 0); + } + return (child_ << 2) | (is_state_ ? 2 : 0) | (has_sibling_ ? 1 : 0); + } + + private: + id_type child_; + id_type sibling_; + uchar_type label_; + bool is_state_; + bool has_sibling_; + + // Copyable. +}; + +// +// Fixed unit of Directed Acyclic Word Graph (DAWG). +// + +class DawgUnit { + public: + explicit DawgUnit(id_type unit = 0) : unit_(unit) {} + DawgUnit(const DawgUnit &unit) : unit_(unit.unit_) {} + + DawgUnit &operator=(id_type unit) { + unit_ = unit; + return *this; + } + + id_type unit() const { + return unit_; + } + + id_type child() const { + return unit_ >> 2; + } + bool has_sibling() const { + return (unit_ & 1) == 1; + } + value_type value() const { + return static_cast(unit_ >> 1); + } + bool is_state() const { + return (unit_ & 2) == 2; + } + + private: + id_type unit_; + + // Copyable. +}; + +// +// Directed Acyclic Word Graph (DAWG) builder. +// + +class DawgBuilder { + public: + DawgBuilder() : nodes_(), units_(), labels_(), is_intersections_(), + table_(), node_stack_(), recycle_bin_(), num_states_(0) {} + ~DawgBuilder() { + clear(); + } + + id_type root() const { + return 0; + } + + id_type child(id_type id) const { + return units_[id].child(); + } + id_type sibling(id_type id) const { + return units_[id].has_sibling() ? (id + 1) : 0; + } + int value(id_type id) const { + return units_[id].value(); + } + + bool is_leaf(id_type id) const { + return label(id) == '\0'; + } + uchar_type label(id_type id) const { + return labels_[id]; + } + + bool is_intersection(id_type id) const { + return is_intersections_[id]; + } + id_type intersection_id(id_type id) const { + return is_intersections_.rank(id) - 1; + } + + std::size_t num_intersections() const { + return is_intersections_.num_ones(); + } + + std::size_t size() const { + return units_.size(); + } + + void init(); + void finish(); + + void insert(const char *key, std::size_t length, value_type value); + + void clear(); + + private: + enum { INITIAL_TABLE_SIZE = 1 << 10 }; + + AutoPool nodes_; + AutoPool units_; + AutoPool labels_; + BitVector is_intersections_; + AutoPool table_; + AutoStack node_stack_; + AutoStack recycle_bin_; + std::size_t num_states_; + + // Disallows copy and assignment. + DawgBuilder(const DawgBuilder &); + DawgBuilder &operator=(const DawgBuilder &); + + void flush(id_type id); + + void expand_table(); + + id_type find_unit(id_type id, id_type *hash_id) const; + id_type find_node(id_type node_id, id_type *hash_id) const; + + bool are_equal(id_type node_id, id_type unit_id) const; + + id_type hash_unit(id_type id) const; + id_type hash_node(id_type id) const; + + id_type append_node(); + id_type append_unit(); + + void free_node(id_type id) { + recycle_bin_.push(id); + } + + static id_type hash(id_type key) { + key = ~key + (key << 15); // key = (key << 15) - key - 1; + key = key ^ (key >> 12); + key = key + (key << 2); + key = key ^ (key >> 4); + key = key * 2057; // key = (key + (key << 3)) + (key << 11); + key = key ^ (key >> 16); + return key; + } +}; + +inline void DawgBuilder::init() { + table_.resize(INITIAL_TABLE_SIZE, 0); + + append_node(); + append_unit(); + + num_states_ = 1; + + nodes_[0].set_label(0xFF); + node_stack_.push(0); +} + +inline void DawgBuilder::finish() { + flush(0); + + units_[0] = nodes_[0].unit(); + labels_[0] = nodes_[0].label(); + + nodes_.clear(); + table_.clear(); + node_stack_.clear(); + recycle_bin_.clear(); + + is_intersections_.build(); +} + +inline void DawgBuilder::insert(const char *key, std::size_t length, + value_type value) { + if (value < 0) { + DARTS_THROW("failed to insert key: negative value"); + } else if (length == 0) { + DARTS_THROW("failed to insert key: zero-length key"); + } + + id_type id = 0; + std::size_t key_pos = 0; + + for ( ; key_pos <= length; ++key_pos) { + id_type child_id = nodes_[id].child(); + if (child_id == 0) { + break; + } + + uchar_type key_label = static_cast(key[key_pos]); + if (key_pos < length && key_label == '\0') { + DARTS_THROW("failed to insert key: invalid null character"); + } + + uchar_type unit_label = nodes_[child_id].label(); + if (key_label < unit_label) { + DARTS_THROW("failed to insert key: wrong key order"); + } else if (key_label > unit_label) { + nodes_[child_id].set_has_sibling(true); + flush(child_id); + break; + } + id = child_id; + } + + if (key_pos > length) { + return; + } + + for ( ; key_pos <= length; ++key_pos) { + uchar_type key_label = static_cast( + (key_pos < length) ? key[key_pos] : '\0'); + id_type child_id = append_node(); + + if (nodes_[id].child() == 0) { + nodes_[child_id].set_is_state(true); + } + nodes_[child_id].set_sibling(nodes_[id].child()); + nodes_[child_id].set_label(key_label); + nodes_[id].set_child(child_id); + node_stack_.push(child_id); + + id = child_id; + } + nodes_[id].set_value(value); +} + +inline void DawgBuilder::clear() { + nodes_.clear(); + units_.clear(); + labels_.clear(); + is_intersections_.clear(); + table_.clear(); + node_stack_.clear(); + recycle_bin_.clear(); + num_states_ = 0; +} + +inline void DawgBuilder::flush(id_type id) { + while (node_stack_.top() != id) { + id_type node_id = node_stack_.top(); + node_stack_.pop(); + + if (num_states_ >= table_.size() - (table_.size() >> 2)) { + expand_table(); + } + + id_type num_siblings = 0; + for (id_type i = node_id; i != 0; i = nodes_[i].sibling()) { + ++num_siblings; + } + + id_type hash_id; + id_type match_id = find_node(node_id, &hash_id); + if (match_id != 0) { + is_intersections_.set(match_id, true); + } else { + id_type unit_id = 0; + for (id_type i = 0; i < num_siblings; ++i) { + unit_id = append_unit(); + } + for (id_type i = node_id; i != 0; i = nodes_[i].sibling()) { + units_[unit_id] = nodes_[i].unit(); + labels_[unit_id] = nodes_[i].label(); + --unit_id; + } + match_id = unit_id + 1; + table_[hash_id] = match_id; + ++num_states_; + } + + for (id_type i = node_id, next; i != 0; i = next) { + next = nodes_[i].sibling(); + free_node(i); + } + + nodes_[node_stack_.top()].set_child(match_id); + } + node_stack_.pop(); +} + +inline void DawgBuilder::expand_table() { + std::size_t table_size = table_.size() << 1; + table_.clear(); + table_.resize(table_size, 0); + + for (std::size_t i = 1; i < units_.size(); ++i) { + id_type id = static_cast(i); + if (labels_[id] == '\0' || units_[id].is_state()) { + id_type hash_id; + find_unit(id, &hash_id); + table_[hash_id] = id; + } + } +} + +inline id_type DawgBuilder::find_unit(id_type id, id_type *hash_id) const { + *hash_id = hash_unit(id) % table_.size(); + for ( ; ; *hash_id = (*hash_id + 1) % table_.size()) { + id_type unit_id = table_[*hash_id]; + if (unit_id == 0) { + break; + } + + // There must not be the same unit. + } + return 0; +} + +inline id_type DawgBuilder::find_node(id_type node_id, + id_type *hash_id) const { + *hash_id = hash_node(node_id) % table_.size(); + for ( ; ; *hash_id = (*hash_id + 1) % table_.size()) { + id_type unit_id = table_[*hash_id]; + if (unit_id == 0) { + break; + } + + if (are_equal(node_id, unit_id)) { + return unit_id; + } + } + return 0; +} + +inline bool DawgBuilder::are_equal(id_type node_id, id_type unit_id) const { + for (id_type i = nodes_[node_id].sibling(); i != 0; + i = nodes_[i].sibling()) { + if (units_[unit_id].has_sibling() == false) { + return false; + } + ++unit_id; + } + if (units_[unit_id].has_sibling() == true) { + return false; + } + + for (id_type i = node_id; i != 0; i = nodes_[i].sibling(), --unit_id) { + if (nodes_[i].unit() != units_[unit_id].unit() || + nodes_[i].label() != labels_[unit_id]) { + return false; + } + } + return true; +} + +inline id_type DawgBuilder::hash_unit(id_type id) const { + id_type hash_value = 0; + for ( ; id != 0; ++id) { + id_type unit = units_[id].unit(); + uchar_type label = labels_[id]; + hash_value ^= hash((label << 24) ^ unit); + + if (units_[id].has_sibling() == false) { + break; + } + } + return hash_value; +} + +inline id_type DawgBuilder::hash_node(id_type id) const { + id_type hash_value = 0; + for ( ; id != 0; id = nodes_[id].sibling()) { + id_type unit = nodes_[id].unit(); + uchar_type label = nodes_[id].label(); + hash_value ^= hash((label << 24) ^ unit); + } + return hash_value; +} + +inline id_type DawgBuilder::append_unit() { + is_intersections_.append(); + units_.append(); + labels_.append(); + + return static_cast(is_intersections_.size() - 1); +} + +inline id_type DawgBuilder::append_node() { + id_type id; + if (recycle_bin_.empty()) { + id = static_cast(nodes_.size()); + nodes_.append(); + } else { + id = recycle_bin_.top(); + nodes_[id] = DawgNode(); + recycle_bin_.pop(); + } + return id; +} + +// +// Unit of double-array builder. +// + +class DoubleArrayBuilderUnit { + public: + DoubleArrayBuilderUnit() : unit_(0) {} + + void set_has_leaf(bool has_leaf) { + if (has_leaf) { + unit_ |= 1U << 8; + } else { + unit_ &= ~(1U << 8); + } + } + void set_value(value_type value) { + unit_ = value | (1U << 31); + } + void set_label(uchar_type label) { + unit_ = (unit_ & ~0xFFU) | label; + } + void set_offset(id_type offset) { + if (offset >= 1U << 29) { + DARTS_THROW("failed to modify unit: too large offset"); + } + unit_ &= (1U << 31) | (1U << 8) | 0xFF; + if (offset < 1U << 21) { + unit_ |= (offset << 10); + } else { + unit_ |= (offset << 2) | (1U << 9); + } + } + + private: + id_type unit_; + + // Copyable. +}; + +// +// Extra unit of double-array builder. +// + +class DoubleArrayBuilderExtraUnit { + public: + DoubleArrayBuilderExtraUnit() : prev_(0), next_(0), + is_fixed_(false), is_used_(false) {} + + void set_prev(id_type prev) { + prev_ = prev; + } + void set_next(id_type next) { + next_ = next; + } + void set_is_fixed(bool is_fixed) { + is_fixed_ = is_fixed; + } + void set_is_used(bool is_used) { + is_used_ = is_used; + } + + id_type prev() const { + return prev_; + } + id_type next() const { + return next_; + } + bool is_fixed() const { + return is_fixed_; + } + bool is_used() const { + return is_used_; + } + + private: + id_type prev_; + id_type next_; + bool is_fixed_; + bool is_used_; + + // Copyable. +}; + +// +// DAWG -> double-array converter. +// + +class DoubleArrayBuilder { + public: + explicit DoubleArrayBuilder(progress_func_type progress_func) + : progress_func_(progress_func), units_(), extras_(), labels_(), + table_(), extras_head_(0) {} + ~DoubleArrayBuilder() { + clear(); + } + + template + void build(const Keyset &keyset); + void copy(std::size_t *size_ptr, DoubleArrayUnit **buf_ptr) const; + + void clear(); + + private: + static const std::size_t BLOCK_SIZE = 256; + static const std::size_t NUM_EXTRA_BLOCKS = 16; + static const std::size_t NUM_EXTRAS = BLOCK_SIZE * NUM_EXTRA_BLOCKS; + + enum { UPPER_MASK = 0xFF << 21 }; + enum { LOWER_MASK = 0xFF }; + + typedef DoubleArrayBuilderUnit unit_type; + typedef DoubleArrayBuilderExtraUnit extra_type; + + progress_func_type progress_func_; + AutoPool units_; + AutoArray extras_; + AutoPool labels_; + AutoArray table_; + id_type extras_head_; + + // Disallows copy and assignment. + DoubleArrayBuilder(const DoubleArrayBuilder &); + DoubleArrayBuilder &operator=(const DoubleArrayBuilder &); + + std::size_t num_blocks() const { + return units_.size() / BLOCK_SIZE; + } + + const extra_type &extras(id_type id) const { + return extras_[id % NUM_EXTRAS]; + } + extra_type &extras(id_type id) { + return extras_[id % NUM_EXTRAS]; + } + + template + void build_dawg(const Keyset &keyset, DawgBuilder *dawg_builder); + void build_from_dawg(const DawgBuilder &dawg); + void build_from_dawg(const DawgBuilder &dawg, + id_type dawg_id, id_type dic_id); + id_type arrange_from_dawg(const DawgBuilder &dawg, + id_type dawg_id, id_type dic_id); + + template + void build_from_keyset(const Keyset &keyset); + template + void build_from_keyset(const Keyset &keyset, std::size_t begin, + std::size_t end, std::size_t depth, id_type dic_id); + template + id_type arrange_from_keyset(const Keyset &keyset, std::size_t begin, + std::size_t end, std::size_t depth, id_type dic_id); + + id_type find_valid_offset(id_type id) const; + bool is_valid_offset(id_type id, id_type offset) const; + + void reserve_id(id_type id); + void expand_units(); + + void fix_all_blocks(); + void fix_block(id_type block_id); +}; + +template +void DoubleArrayBuilder::build(const Keyset &keyset) { + if (keyset.has_values()) { + Details::DawgBuilder dawg_builder; + build_dawg(keyset, &dawg_builder); + build_from_dawg(dawg_builder); + dawg_builder.clear(); + } else { + build_from_keyset(keyset); + } +} + +inline void DoubleArrayBuilder::copy(std::size_t *size_ptr, + DoubleArrayUnit **buf_ptr) const { + if (size_ptr != NULL) { + *size_ptr = units_.size(); + } + if (buf_ptr != NULL) { + *buf_ptr = new DoubleArrayUnit[units_.size()]; + unit_type *units = reinterpret_cast(*buf_ptr); + for (std::size_t i = 0; i < units_.size(); ++i) { + units[i] = units_[i]; + } + } +} + +inline void DoubleArrayBuilder::clear() { + units_.clear(); + extras_.clear(); + labels_.clear(); + table_.clear(); + extras_head_ = 0; +} + +template +void DoubleArrayBuilder::build_dawg(const Keyset &keyset, + DawgBuilder *dawg_builder) { + dawg_builder->init(); + for (std::size_t i = 0; i < keyset.num_keys(); ++i) { + dawg_builder->insert(keyset.keys(i), keyset.lengths(i), keyset.values(i)); + if (progress_func_ != NULL) { + progress_func_(i + 1, keyset.num_keys() + 1); + } + } + dawg_builder->finish(); +} + +inline void DoubleArrayBuilder::build_from_dawg(const DawgBuilder &dawg) { + std::size_t num_units = 1; + while (num_units < dawg.size()) { + num_units <<= 1; + } + units_.reserve(num_units); + + table_.reset(new id_type[dawg.num_intersections()]); + for (std::size_t i = 0; i < dawg.num_intersections(); ++i) { + table_[i] = 0; + } + + extras_.reset(new extra_type[NUM_EXTRAS]); + + reserve_id(0); + extras(0).set_is_used(true); + units_[0].set_offset(1); + units_[0].set_label('\0'); + + if (dawg.child(dawg.root()) != 0) { + build_from_dawg(dawg, dawg.root(), 0); + } + + fix_all_blocks(); + + extras_.clear(); + labels_.clear(); + table_.clear(); +} + +inline void DoubleArrayBuilder::build_from_dawg(const DawgBuilder &dawg, + id_type dawg_id, id_type dic_id) { + id_type dawg_child_id = dawg.child(dawg_id); + if (dawg.is_intersection(dawg_child_id)) { + id_type intersection_id = dawg.intersection_id(dawg_child_id); + id_type offset = table_[intersection_id]; + if (offset != 0) { + offset ^= dic_id; + if (!(offset & UPPER_MASK) || !(offset & LOWER_MASK)) { + if (dawg.is_leaf(dawg_child_id)) { + units_[dic_id].set_has_leaf(true); + } + units_[dic_id].set_offset(offset); + return; + } + } + } + + id_type offset = arrange_from_dawg(dawg, dawg_id, dic_id); + if (dawg.is_intersection(dawg_child_id)) { + table_[dawg.intersection_id(dawg_child_id)] = offset; + } + + do { + uchar_type child_label = dawg.label(dawg_child_id); + id_type dic_child_id = offset ^ child_label; + if (child_label != '\0') { + build_from_dawg(dawg, dawg_child_id, dic_child_id); + } + dawg_child_id = dawg.sibling(dawg_child_id); + } while (dawg_child_id != 0); +} + +inline id_type DoubleArrayBuilder::arrange_from_dawg(const DawgBuilder &dawg, + id_type dawg_id, id_type dic_id) { + labels_.resize(0); + + id_type dawg_child_id = dawg.child(dawg_id); + while (dawg_child_id != 0) { + labels_.append(dawg.label(dawg_child_id)); + dawg_child_id = dawg.sibling(dawg_child_id); + } + + id_type offset = find_valid_offset(dic_id); + units_[dic_id].set_offset(dic_id ^ offset); + + dawg_child_id = dawg.child(dawg_id); + for (std::size_t i = 0; i < labels_.size(); ++i) { + id_type dic_child_id = offset ^ labels_[i]; + reserve_id(dic_child_id); + + if (dawg.is_leaf(dawg_child_id)) { + units_[dic_id].set_has_leaf(true); + units_[dic_child_id].set_value(dawg.value(dawg_child_id)); + } else { + units_[dic_child_id].set_label(labels_[i]); + } + + dawg_child_id = dawg.sibling(dawg_child_id); + } + extras(offset).set_is_used(true); + + return offset; +} + +template +void DoubleArrayBuilder::build_from_keyset(const Keyset &keyset) { + std::size_t num_units = 1; + while (num_units < keyset.num_keys()) { + num_units <<= 1; + } + units_.reserve(num_units); + + extras_.reset(new extra_type[NUM_EXTRAS]); + + reserve_id(0); + extras(0).set_is_used(true); + units_[0].set_offset(1); + units_[0].set_label('\0'); + + if (keyset.num_keys() > 0) { + build_from_keyset(keyset, 0, keyset.num_keys(), 0, 0); + } + + fix_all_blocks(); + + extras_.clear(); + labels_.clear(); +} + +template +void DoubleArrayBuilder::build_from_keyset(const Keyset &keyset, + std::size_t begin, std::size_t end, std::size_t depth, id_type dic_id) { + id_type offset = arrange_from_keyset(keyset, begin, end, depth, dic_id); + + while (begin < end) { + if (keyset.keys(begin, depth) != '\0') { + break; + } + ++begin; + } + if (begin == end) { + return; + } + + std::size_t last_begin = begin; + uchar_type last_label = keyset.keys(begin, depth); + while (++begin < end) { + uchar_type label = keyset.keys(begin, depth); + if (label != last_label) { + build_from_keyset(keyset, last_begin, begin, + depth + 1, offset ^ last_label); + last_begin = begin; + last_label = keyset.keys(begin, depth); + } + } + build_from_keyset(keyset, last_begin, end, depth + 1, offset ^ last_label); +} + +template +id_type DoubleArrayBuilder::arrange_from_keyset(const Keyset &keyset, + std::size_t begin, std::size_t end, std::size_t depth, id_type dic_id) { + labels_.resize(0); + + value_type value = -1; + for (std::size_t i = begin; i < end; ++i) { + uchar_type label = keyset.keys(i, depth); + if (label == '\0') { + if (keyset.has_lengths() && depth < keyset.lengths(i)) { + DARTS_THROW("failed to build double-array: " + "invalid null character"); + } else if (keyset.values(i) < 0) { + DARTS_THROW("failed to build double-array: negative value"); + } + + if (value == -1) { + value = keyset.values(i); + } + if (progress_func_ != NULL) { + progress_func_(i + 1, keyset.num_keys() + 1); + } + } + + if (labels_.empty()) { + labels_.append(label); + } else if (label != labels_[labels_.size() - 1]) { + if (label < labels_[labels_.size() - 1]) { + DARTS_THROW("failed to build double-array: wrong key order"); + } + labels_.append(label); + } + } + + id_type offset = find_valid_offset(dic_id); + units_[dic_id].set_offset(dic_id ^ offset); + + for (std::size_t i = 0; i < labels_.size(); ++i) { + id_type dic_child_id = offset ^ labels_[i]; + reserve_id(dic_child_id); + if (labels_[i] == '\0') { + units_[dic_id].set_has_leaf(true); + units_[dic_child_id].set_value(value); + } else { + units_[dic_child_id].set_label(labels_[i]); + } + } + extras(offset).set_is_used(true); + + return offset; +} + +inline id_type DoubleArrayBuilder::find_valid_offset(id_type id) const { + if (extras_head_ >= units_.size()) { + return units_.size() | (id & LOWER_MASK); + } + + id_type unfixed_id = extras_head_; + do { + id_type offset = unfixed_id ^ labels_[0]; + if (is_valid_offset(id, offset)) { + return offset; + } + unfixed_id = extras(unfixed_id).next(); + } while (unfixed_id != extras_head_); + + return units_.size() | (id & LOWER_MASK); +} + +inline bool DoubleArrayBuilder::is_valid_offset(id_type id, + id_type offset) const { + if (extras(offset).is_used()) { + return false; + } + + id_type rel_offset = id ^ offset; + if ((rel_offset & LOWER_MASK) && (rel_offset & UPPER_MASK)) { + return false; + } + + for (std::size_t i = 1; i < labels_.size(); ++i) { + if (extras(offset ^ labels_[i]).is_fixed()) { + return false; + } + } + + return true; +} + +inline void DoubleArrayBuilder::reserve_id(id_type id) { + if (id >= units_.size()) { + expand_units(); + } + + if (id == extras_head_) { + extras_head_ = extras(id).next(); + if (extras_head_ == id) { + extras_head_ = units_.size(); + } + } + extras(extras(id).prev()).set_next(extras(id).next()); + extras(extras(id).next()).set_prev(extras(id).prev()); + extras(id).set_is_fixed(true); +} + +inline void DoubleArrayBuilder::expand_units() { + id_type src_num_units = units_.size(); + id_type src_num_blocks = num_blocks(); + + id_type dest_num_units = src_num_units + BLOCK_SIZE; + id_type dest_num_blocks = src_num_blocks + 1; + + if (dest_num_blocks > NUM_EXTRA_BLOCKS) { + fix_block(src_num_blocks - NUM_EXTRA_BLOCKS); + } + + units_.resize(dest_num_units); + + if (dest_num_blocks > NUM_EXTRA_BLOCKS) { + for (std::size_t id = src_num_units; id < dest_num_units; ++id) { + extras(id).set_is_used(false); + extras(id).set_is_fixed(false); + } + } + + for (id_type i = src_num_units + 1; i < dest_num_units; ++i) { + extras(i - 1).set_next(i); + extras(i).set_prev(i - 1); + } + + extras(src_num_units).set_prev(dest_num_units - 1); + extras(dest_num_units - 1).set_next(src_num_units); + + extras(src_num_units).set_prev(extras(extras_head_).prev()); + extras(dest_num_units - 1).set_next(extras_head_); + + extras(extras(extras_head_).prev()).set_next(src_num_units); + extras(extras_head_).set_prev(dest_num_units - 1); +} + +inline void DoubleArrayBuilder::fix_all_blocks() { + id_type begin = 0; + if (num_blocks() > NUM_EXTRA_BLOCKS) { + begin = num_blocks() - NUM_EXTRA_BLOCKS; + } + id_type end = num_blocks(); + + for (id_type block_id = begin; block_id != end; ++block_id) { + fix_block(block_id); + } +} + +inline void DoubleArrayBuilder::fix_block(id_type block_id) { + id_type begin = block_id * BLOCK_SIZE; + id_type end = begin + BLOCK_SIZE; + + id_type unused_offset = 0; + for (id_type offset = begin; offset != end; ++offset) { + if (!extras(offset).is_used()) { + unused_offset = offset; + break; + } + } + + for (id_type id = begin; id != end; ++id) { + if (!extras(id).is_fixed()) { + reserve_id(id); + units_[id].set_label(static_cast(id ^ unused_offset)); + } + } +} + +} // namespace Details + +// +// Member function build() of DoubleArrayImpl. +// + +template +int DoubleArrayImpl::build(std::size_t num_keys, + const key_type * const *keys, const std::size_t *lengths, + const value_type *values, Details::progress_func_type progress_func) { + Details::Keyset keyset(num_keys, keys, lengths, values); + + Details::DoubleArrayBuilder builder(progress_func); + builder.build(keyset); + + std::size_t size = 0; + unit_type *buf = NULL; + builder.copy(&size, &buf); + + clear(); + + size_ = size; + array_ = buf; + buf_ = buf; + + if (progress_func != NULL) { + progress_func(num_keys + 1, num_keys + 1); + } + + return 0; +} + +} // namespace Darts + +#undef DARTS_INT_TO_STR +#undef DARTS_LINE_TO_STR +#undef DARTS_LINE_STR +#undef DARTS_THROW + +#endif // DARTS_H_ diff --git a/engine/ngram/octagram/octagram_gram.cpp b/engine/ngram/octagram/octagram_gram.cpp new file mode 100644 index 00000000..9c2cd432 --- /dev/null +++ b/engine/ngram/octagram/octagram_gram.cpp @@ -0,0 +1,348 @@ +#include "octagram_gram.h" + +#include + +#include +#include +#include +#include + +namespace gram +{ +namespace +{ + +uint32_t read_u32(const char *p) +{ + const auto *b = reinterpret_cast(p); + return static_cast(b[0]) | (static_cast(b[1]) << 8) | (static_cast(b[2]) << 16) | + (static_cast(b[3]) << 24); +} + +// 读一个原始 UTF-8 码点并推进 i。续字节缺位或非法序列按 Latin-1 单字节兜底： +// 搭配查询遇到乱码最多损失一次命中，不值得为它抛错。 +uint32_t next_codepoint(std::string_view text, size_t &i) +{ + const auto lead = static_cast(text[i++]); + // 续字节：只在确实存在且形态合法时消费，值按需拼接。 + const auto take = [&text, &i]() -> uint32_t { + if (i < text.size() && (static_cast(text[i]) & 0xC0) == 0x80) + return static_cast(text[i++]) & 0x3F; + return 0; + }; + if (lead < 0x80) + return lead; + if ((lead & 0xE0) == 0xC0) + return (static_cast(lead & 0x1F) << 6) | take(); + if ((lead & 0xF0) == 0xE0) + { + const uint32_t high = take(); + const uint32_t low = take(); // 两次调用必须先后求值，不能写进同一个表达式 + return (static_cast(lead & 0x0F) << 12) | (high << 6) | low; + } + if ((lead & 0xF8) == 0xF0) + { + const uint32_t b1 = take(); + const uint32_t b2 = take(); + const uint32_t b3 = take(); + return (static_cast(lead & 0x07) << 18) | (b1 << 12) | (b2 << 6) | b3; + } + return lead; +} + +inline double scale_value(int value) +{ + return value >= 0 ? static_cast(value) / GramDb::kValueScale : -1.0; +} + +} // namespace + +// ---- gram_encoding.cc 的移植 ------------------------------------------------ + +std::string GramDb::encode(std::string_view utf8) +{ + std::string out; + out.reserve(utf8.size()); + size_t i = 0; + while (i < utf8.size()) + { + uint32_t u = next_codepoint(utf8, i); + if (u < 0x80) + { + // 编码串里不允许出现 NUL（要当 C 字符串用），0 被转成 0xE0 占位。 + out.push_back(u == 0 ? static_cast(0xE0) : static_cast(u)); + } + else if (u >= 0x4000 && u < 0xA000) + { + // CJK 常用区压成 2 字节：首字节平移进 0x4080-0xA07F，避开 ASCII 与 + // 0xE0 变长前缀。 + if ((u & 0xFF) == 0) + { + out.push_back(static_cast(0xE1)); + out.push_back(static_cast((u >> 8) + 0x40)); + } + else + { + out.push_back(static_cast((u >> 8) + 0x40)); + out.push_back(static_cast(u & 0xFF)); + } + } + else + { + int bits = 32; + while (bits > 0 && (u & 0xFE000000) == 0) + { + bits -= 7; + u <<= 7; + } + int bytes_to_encode = (bits + 6) / 7; + out.push_back(static_cast(0xE0 | bytes_to_encode)); + while (bytes_to_encode > 0) + { + --bytes_to_encode; + out.push_back(static_cast(((u >> 25) & 0x7F) | 0x80)); + u <<= 7; + } + } + } + return out; +} + +const char *next_unicode(const char *p) +{ + const auto first = static_cast(*p); + return p + ((first & 0x80) == 0 ? 1 : (first & 0xF0) == 0xE0 ? (first & 0x0F) + 1 : 2); +} + +size_t unicode_length(std::string_view encoded, size_t byte_length) +{ + size_t len = 0; + size_t i = 0; + while (i < byte_length && i < encoded.size()) + { + i = static_cast(next_unicode(encoded.data() + i) - encoded.data()); + ++len; + } + return len; +} + +size_t codepoint_count(std::string_view utf8) +{ + size_t n = 0; + for (size_t i = 0; i < utf8.size();) + { + next_codepoint(utf8, i); + ++n; + } + return n; +} + +std::string first_codepoints(std::string_view utf8, size_t n) +{ + size_t seen = 0; + size_t i = 0; + while (i < utf8.size() && seen < n) + { + next_codepoint(utf8, i); + ++seen; + } + return std::string(utf8.substr(0, i)); +} + +std::string last_codepoints(std::string_view utf8, size_t n) +{ + const size_t total = codepoint_count(utf8); + if (total <= n) + return std::string(utf8); + size_t skip = total - n; + size_t i = 0; + while (i < utf8.size() && skip > 0) + { + next_codepoint(utf8, i); + --skip; + } + return std::string(utf8.substr(i)); +} + +// ---- gram_db.cc 的移植（装载换成 Win32 mmap）-------------------------------- + +GramDb::~GramDb() +{ + if (view_ != nullptr) + UnmapViewOfFile(view_); + if (mapping_handle_ != nullptr) + CloseHandle(mapping_handle_); + if (file_handle_ != nullptr) + CloseHandle(file_handle_); +} + +bool GramDb::open(const std::filesystem::path &file) +{ + error_.clear(); + + // std::filesystem::path 在 Windows 上是宽字符，直接喂 Create*W，不经窄化。 + file_handle_ = CreateFileW(file.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL, nullptr); + if (file_handle_ == INVALID_HANDLE_VALUE) + { + // 归一成 void* 空值：INVALID_HANDLE_VALUE 不是 nullptr，但语义上就是没打开。 + CloseHandle(file_handle_); + file_handle_ = nullptr; + error_ = "cannot open file"; + return false; + } + + LARGE_INTEGER size{}; + if (!GetFileSizeEx(file_handle_, &size) || size.QuadPart <= 0) + { + error_ = "empty file"; + return false; + } + view_bytes_ = static_cast(size.QuadPart); + + mapping_handle_ = CreateFileMappingW(file_handle_, nullptr, PAGE_READONLY, 0, 0, nullptr); + if (mapping_handle_ == nullptr) + { + error_ = "cannot create mapping"; + return false; + } + view_ = static_cast(MapViewOfFile(mapping_handle_, FILE_MAP_READ, 0, 0, 0)); + if (view_ == nullptr) + { + error_ = "cannot map view"; + return false; + } + + // 头部布局（rime MappedFile 上的 Metadata，偏移 0）：format char[32]、 + // uint32 checksum、uint32 double_array_size（单位数，非字节）、 + // int32 OffsetPtr（相对自身字段的偏移）。 + constexpr size_t kFormatOffset = 0; + constexpr size_t kArraySizeOffset = 36; + constexpr size_t kOffsetPtrOffset = 40; + constexpr size_t kHeaderBytes = 44; + if (view_bytes_ < kHeaderBytes) + { + error_ = "file too small for Metadata"; + return false; + } + const std::string_view format(view_ + kFormatOffset, 14); + if (format != "Rime::Grammar/") + { + error_ = "format magic mismatch: " + std::string(format); + return false; + } + array_size_ = read_u32(view_ + kArraySizeOffset); + const int32_t offset = static_cast(read_u32(view_ + kOffsetPtrOffset)); + const char *self = view_ + kOffsetPtrOffset; + const char *array = self + offset; + // darts 单元 4 字节；镜像必须完整落在映射内。 + if (offset < 0 || array < view_ || array_size_ > (view_bytes_ - static_cast(array - view_)) / 4) + { + error_ = "double-array image out of bounds"; + return false; + } + trie_.set_array(array, array_size_); + return true; +} + +int GramDb::lookup(const char *context_encoded, const std::string &word_encoded, Match *results) const +{ + size_t node_pos = 0; + size_t key_pos = 0; + trie_.traverse(context_encoded, node_pos, key_pos); + if (key_pos == std::strlen(context_encoded)) + return static_cast(trie_.commonPrefixSearch(word_encoded.c_str(), results, kMaxResults, 0, node_pos)); + return 0; +} + +double GramDb::query(const std::string &context, const std::string &word, bool is_rear, const GrammarConfig &config, + QueryStats *stats) const +{ + double result = config.non_collocation_penalty; + QueryStats local{}; + QueryStats &s = stats != nullptr ? *stats : local; + if (!valid() || context.empty()) + return result; + + // 与 octagram.cc 一致：上下文取尾部 (collocation_max_length - 1) 个码点， + // 词取头部同样数量，再各自 encode。collocation_len = 上下文现存码点数 + + // 命中键的码点数，决定吃 collocation 惩罚还是更狠的 weak 惩罚。 + const int n = (std::min)(kMaxEncodedUnicode, config.collocation_max_length - 1); + if (n < 1) + return result; + + const std::string context_tail = last_codepoints(context, static_cast(n)); + int remaining_windows = codepoint_count(context_tail); + const std::string context_query = encode(context_tail); + const std::string word_head = first_codepoints(word, static_cast(n)); + const int word_head_len = static_cast(codepoint_count(word_head)); + const std::string word_query = encode(word_head); + + const char *context_ptr = context_query.c_str(); + for (; remaining_windows > 0; --remaining_windows, context_ptr = next_unicode(context_ptr)) + { + ++s.context_windows; + Match matches[kMaxResults]; + const int num = lookup(context_ptr, word_query, matches); + if (num <= 0) + continue; + ++s.matched_windows; + const bool at_window_start = context_ptr == context_query.c_str(); + for (int i = 0; i < num; ++i) + { + ++s.matches; + const Match &match = matches[i]; + const double scaled = scale_value(match.value); + if (scaled > s.best_value) + s.best_value = scaled; + const int match_len = static_cast(unicode_length(word_query, match.length)); + const int collocation_len = remaining_windows + match_len; + const bool whole_query = at_window_start && match.length == word_query.size(); + const double penalty = (collocation_len >= config.collocation_min_length || whole_query) + ? config.collocation_penalty + : config.weak_collocation_penalty; + const double candidate = scaled + penalty; + if (candidate > result) + result = candidate; + } + } + + if (is_rear) + { + // 句尾标记只在「词未被头部截断」时查：截过的词查 $ 无意义。 + if (word_head_len == static_cast(codepoint_count(word))) + { + Match matches[kMaxResults]; + if (lookup(word_query.c_str(), "$", matches) > 0) + { + const double candidate = scale_value(matches[0].value) + config.rear_penalty; + if (candidate > result) + result = candidate; + } + } + } + return result; +} + +// ---- 进程内共享 -------------------------------------------------------------- + +const GramDb *shared_gram_db(const std::filesystem::path &file) +{ + // 照 ngram::shared_language_model：实例进程生命周期内常驻（故意不释放—— + // 互斥锁的竞态和析构顺序问题用「永不析构」一并解决），mmap 的物理内存由 + // 操作系统按命中页回收，真正占住的是地址空间。 + static std::mutex mutex; + static std::unordered_map> cache; + const std::string key = file.u8string(); + std::lock_guard lock(mutex); + auto &slot = cache[key]; + if (!slot) + { + auto db = std::make_unique(); + db->open(file); + slot = std::move(db); + } + return slot.get(); +} + +} // namespace gram diff --git a/engine/ngram/octagram/octagram_gram.h b/engine/ngram/octagram/octagram_gram.h new file mode 100644 index 00000000..8cefd24f --- /dev/null +++ b/engine/ngram/octagram/octagram_gram.h @@ -0,0 +1,122 @@ +#pragma once + +// RIME octagram（八股文）.gram 语法模型的读取端（mmap 版）。 +// +// 格式与查询语义按 lotem/librime-octagram（BSD-3-Clause）移植：文件是 +// 「Metadata + Darts 双数组 Trie 镜像」，键为「上下文尾部字符 + 词首字符」经 +// encode() 压缩的字节串，值为 int(ln(共现频次) * 10000)。它不是概率语言模型， +// query() 给出的是搭配强度信号：max(值/10000 + 搭配惩罚)，全部未命中返回 +// non_collocation_penalty。词表零耦合——我们词库里没有的词只是拿不到加成， +// 不会报错。双数组实现在同目录 darts.h（rime/librime 同款，BSD）。 +// +// 与评测版（原 tests/src/gram_reader）的差别只在装载方式：整文件 ReadFile 换成 +// Win32 内存映射，RSS 只随命中的页增长；查询路径完全一致。 + +#include +#include +#include +#include + +#include "darts.h" + +namespace gram +{ + +// 万象 LTS 推荐的打分参数（octagram 自身默认 collocation_max_length=4、 +// collocation_min_length=3、collocation_penalty=non_collocation_penalty=-12）。 +struct GrammarConfig +{ + int collocation_max_length = 8; + int collocation_min_length = 2; + double collocation_penalty = -10.0; + double non_collocation_penalty = -17.0; + double weak_collocation_penalty = -24.0; + double rear_penalty = -18.0; +}; + +// 一次 query() 的可观测计数。产品路径传 nullptr 即可，诊断工具用它回答 +// 「模型在我们词表上到底有没有咬合」。 +struct QueryStats +{ + int context_windows = 0; // 尝试过的上下文后缀数 + int matched_windows = 0; // 有 >=1 个结果的窗口数 + int matches = 0; // 累计结果条数 + double best_value = 0.0; // 命中的最大 scale_value（未命中保持 0） +}; + +class GramDb +{ + public: + using Match = Darts::DoubleArray::result_pair_type; + + GramDb() = default; + ~GramDb(); + GramDb(const GramDb &) = delete; + GramDb &operator=(const GramDb &) = delete; + + // gram_encoding.cc / gram_db.cc 的常量，公开给实现文件与诊断工具。 + static constexpr int kMaxEncodedUnicode = 8; + static constexpr int kMaxResults = 8; + static constexpr double kValueScale = 10000.0; + + // mmap 只读打开，校验 "Rime::Grammar/" 魔数与双数组镜像边界；失败返回 + // false，error() 给出原因。映射与句柄随对象存活——共享实例进程内常驻， + // 与 kenlm 的共享模型同一条生命线。db_checksum 沿袭 octagram 的 Load() + // 不校验：格式魔数加边界检查已足够，摘要校验在下载侧做。 + bool open(const std::filesystem::path &file); + + bool valid() const + { + return view_ != nullptr; + } + const std::string &error() const + { + return error_; + } + + // 复刻 Octagram::Query。context/word 都是原始 UTF-8；is_rear 对应句尾词的 + // 「词+$」rear 项。stats 可为 null。 + double query(const std::string &context, const std::string &word, bool is_rear, const GrammarConfig &config, + QueryStats *stats = nullptr) const; + + // gram_encoding.cc 的 UTF-8 压缩编码：ASCII 1 字节、CJK 常用区（0x4000-0xA000） + // 2 字节、其余按 7bit 组变长，最多编 kMaxEncodedUnicode 个码点。 + static std::string encode(std::string_view utf8); + static constexpr int max_encoded_codepoints() + { + return kMaxEncodedUnicode; + } + + private: + // octagram 的 GramDb::Lookup：先 traverse 整段上下文，再从落点做 + // commonPrefixSearch。context_encoded 必须是 encode() 的产物。 + int lookup(const char *context_encoded, const std::string &word_encoded, Match *results) const; + + void *file_handle_ = nullptr; // HANDLE，按 void* 存避免在头文件拉 Windows.h + void *mapping_handle_ = nullptr; + const char *view_ = nullptr; + size_t view_bytes_ = 0; + uint32_t array_size_ = 0; + Darts::DoubleArray trie_; + std::string error_; +}; + +// 编码串上的游标推进（gram_encoding.cc 的 advance）：一个编码码点占 1 字节 +// （ASCII）、2 字节（CJK 常用区）或 1+首字节低 4 位（扩展区变长）。 +const char *next_unicode(const char *p); + +// 编码串前 byte_length 字节里的码点数（gram_encoding.cc 的 unicode_length）。 +size_t unicode_length(std::string_view encoded, size_t byte_length); + +// 原始 UTF-8 串的码点数与首/尾 n 个码点。 +size_t codepoint_count(std::string_view utf8); +std::string first_codepoints(std::string_view utf8, size_t n); +std::string last_codepoints(std::string_view utf8, size_t n); + +// 进程内按路径共享的模型，照 ngram::shared_language_model 的范式：查询全是 +// 只读（mmap 页 + const 方法），多会话可同时持有返回的指针；载入失败返回 +// valid() == false 的实例，调用方判空降级。全拼和双拼词典各持一份指针、共享 +// 同一份映射，不重复付 390MB 的物理内存。 +const GramDb *shared_gram_db(const std::filesystem::path &file); + +} // namespace gram diff --git a/engine/tests/src/test_pinyin.cpp b/engine/tests/src/test_pinyin.cpp index ebd05943..9aa5aea2 100644 --- a/engine/tests/src/test_pinyin.cpp +++ b/engine/tests/src/test_pinyin.cpp @@ -834,8 +834,11 @@ void test_word_lattice() "Expected the reranker's pick tagged with its own source and the lattice's with Generated."); } - // 重排器同意词格的排序时不该凭空多出一行。按 word_lattice.h 的契约，被其他来源认可的 - // 三元首选只是提前，来源仍记在词格（Generated）上；重排器的首选与它重复，默认不往下补位。 + // 重排器同意词格的排序时不该凭空多出一行。行保持词格来源（Generated）： + // include_lattice_best=true 时词格首选永远是 Generated，见 word_lattice.h 单重排器 + // 契约——被其他来源认可的三元首选只是提前，来源仍记在词格上。历史意图曾是 + // 「模型看过并认可了它，来源记在重排那边」，但 merge 的实现与头文件契约都是词格 + // 来源；以实现为准，若要恢复「认可即署名」需要先改 merge。 std::vector agreed; quanpin::merge_lattice_candidates( agreed, {"ni", "hao"}, make_table_lattice_lookup(table), "ni'hao", {}, @@ -845,7 +848,7 @@ void test_word_lattice() if (agreed.size() == 1 && alone.size() == 1) { expect(agreed.front().word == alone.front().word && agreed.front().source == CandidateSource::Generated, - "A trigram pick the reranker agreed with should keep the lattice's source."); + "A trigram pick the reranker agreed with should keep the lattice's own source."); } // 弃权（模型没加载、后台还没算完）时重排器不参与共识，这一行同样只记在词格上。 @@ -1664,6 +1667,41 @@ void test_quanpin_autocorrect_display() fs::remove(db_path, cleanup_ec); } +// 万象语法模型的会话链路冒烟：经 QuanpinDictionary::set_sentence_association 下发 +// 模型路径，验证加性搭配项能改写整句首选——bu'zhe'ji 在三元模型下出「不这几」， +// 加上搭配后词格首选应为「不着急」（离线评测的典型翻盘案例）。模型路径由 +// METASEQUOIA_IME_TEST_COLLOCATION_MODEL 环境变量给出，未设置或载入失败时优雅跳过： +// .gram 是按需下载的可选资产，测试数据目录默认没有。 +void test_quanpin_collocation_session() +{ + fmt::println("==== Quanpin Collocation Session ===="); + const char *model_path = std::getenv("METASEQUOIA_IME_TEST_COLLOCATION_MODEL"); + if (model_path == nullptr) + { + fmt::println("Skipped: METASEQUOIA_IME_TEST_COLLOCATION_MODEL is not set."); + return; + } + + QuanpinDictionary dictionary; + SentenceAssociationOptions association; + association.word_lattice = true; + association.collocation_model = model_path; + association.collocation_weight = 0.25; + dictionary.set_sentence_association(association); + + const auto result = dictionary.query("buzheji"); + const auto trigram = std::find_if(result.begin(), result.end(), + [](const WordItem &item) { return item.source == CandidateSource::Generated; }); + if (trigram == result.end()) + { + fmt::println("Skipped: no lattice sentence for 'buzheji' (dictionary or sc.lm incomplete?)."); + return; + } + expect( + trigram->word == "不着急", + fmt::format("Expected the collocation-enabled lattice to pick 不着急 for buzheji, got '{}'.", trigram->word)); +} + int main(int argc, char *argv[]) { try @@ -1685,6 +1723,7 @@ int main(int argc, char *argv[]) test_quanpin_four_syllable_alternative_segmentation(); test_quanpin_lattice_precedes_google_fallback(); test_quanpin_lattice_rejects_rare_readings(); + test_quanpin_collocation_session(); test_quanpin_lattice_covers_every_syllable(); test_quanpin_single_letter_jianpin_ranking(); test_quanpin_query_timings();