fanlusky
e365a8d1b5
- sc.lm 改用 kenlm 的 LAZY 载入：Windows 上默认的 POPULATE_OR_READ 会退成 READ， 把 33 MB 整份拷进私有堆且永不释放；现在是 MapViewOfFile 只读映射，可被系统回收 - dict_japanese.dat 在 Windows 上同样改为只读映射，与 POSIX 分支一致 - 神经整句模型改为开关首次生效时才载入（LazySentenceModel），默认关闭的桌面模型 不再占用约 24 MB - 新增 reset_sentence_cache()：重排结果与上文只影响词格整句，只清 series 一层， 逐前缀查库与切分缓存保持热；set_rescoring_context 也改用它 - CircularBuffer 新增 find()：命中时返回指针而不复制整份候选列表，并改为真正的 LRU； 全拼、双拼每键的命中路径改用它 - Google 解码器去掉每次调用前的 im_reset_search，沿用上游的公共前缀增量搜索 - 测试：出货 sc.lm 能映射载入并正常打分；增量搜索与从头搜结果逐步一致
94 lines
2.9 KiB
C++
94 lines
2.9 KiB
C++
#include "pinyin_decoder.h"
|
||
#include "data_path.h"
|
||
#include "../googlepinyinime-rev/src/include/pinyinime.h"
|
||
#include "../googlepinyinime-rev/src/include/lpicache.h"
|
||
#include <mutex>
|
||
#include <utf8.h>
|
||
|
||
namespace metasequoia
|
||
{
|
||
namespace
|
||
{
|
||
struct DecoderService
|
||
{
|
||
std::mutex mutex;
|
||
std::filesystem::path model;
|
||
std::filesystem::path user_dictionary;
|
||
bool ready = false;
|
||
std::size_t clients = 0;
|
||
~DecoderService()
|
||
{
|
||
ime_pinyin::im_close_decoder();
|
||
}
|
||
};
|
||
|
||
DecoderService &service()
|
||
{
|
||
static DecoderService instance;
|
||
return instance;
|
||
}
|
||
} // namespace
|
||
|
||
PinyinDecoder::PinyinDecoder(std::filesystem::path model, std::filesystem::path user_dictionary)
|
||
: model_(std::move(model)), user_dictionary_(std::move(user_dictionary))
|
||
{
|
||
auto &decoder = service();
|
||
std::lock_guard lock(decoder.mutex);
|
||
++decoder.clients;
|
||
}
|
||
|
||
PinyinDecoder::~PinyinDecoder()
|
||
{
|
||
auto &decoder = service();
|
||
std::lock_guard lock(decoder.mutex);
|
||
if (--decoder.clients == 0)
|
||
{
|
||
ime_pinyin::im_close_decoder();
|
||
decoder.ready = false;
|
||
decoder.model.clear();
|
||
decoder.user_dictionary.clear();
|
||
}
|
||
}
|
||
|
||
std::string PinyinDecoder::sentence(const std::string &pinyin) const
|
||
{
|
||
if (pinyin.empty() || pinyin.size() > 128 || model_.empty() || user_dictionary_.empty())
|
||
return {};
|
||
auto &decoder = service();
|
||
std::lock_guard lock(decoder.mutex);
|
||
if (decoder.model != model_ || decoder.user_dictionary != user_dictionary_)
|
||
{
|
||
ime_pinyin::im_close_decoder();
|
||
decoder.ready = false;
|
||
// The upstream lemma cache is also global and contains model-specific IDs.
|
||
auto &cache = ime_pinyin::LpiCache::get_instance();
|
||
for (ime_pinyin::uint16 id = 0; id < ime_pinyin::kFullSplIdStart; ++id)
|
||
cache.put_cache(id, nullptr, 0);
|
||
decoder.model = model_;
|
||
decoder.user_dictionary = user_dictionary_;
|
||
decoder.ready =
|
||
ime_pinyin::im_open_decoder(path_to_utf8(model_).c_str(), path_to_utf8(user_dictionary_).c_str());
|
||
if (!decoder.ready)
|
||
return {};
|
||
ime_pinyin::im_set_max_lens(128, 64);
|
||
}
|
||
if (!decoder.ready)
|
||
return {};
|
||
// 不先 im_reset_search：upstream 的 search 自己会与上一次的输入比公共前缀，从分歧处
|
||
// 回退再往后解，连续打字时每键只多解一个字母。前缀为零时它走的就是整段重置。
|
||
const auto count = ime_pinyin::im_search(pinyin.data(), pinyin.size());
|
||
for (std::size_t i = 0; i < count; ++i)
|
||
{
|
||
ime_pinyin::char16 buffer[256] = {};
|
||
if (!ime_pinyin::im_get_candidate(i, buffer, 255))
|
||
continue;
|
||
std::size_t length = 0;
|
||
while (length < 255 && buffer[length] != 0)
|
||
++length;
|
||
if (length)
|
||
return utf8::utf16to8(std::u16string(reinterpret_cast<const char16_t *>(buffer), length));
|
||
}
|
||
return {};
|
||
}
|
||
} // namespace metasequoia
|