From 1acc3d67458feb0a3a5cda326a7248908ed3b1a4 Mon Sep 17 00:00:00 2001 From: luojiyin Date: Sat, 3 Oct 2026 16:20:47 +0800 Subject: [PATCH 1/3] perf(engine): skip unused neural scoring positions Skip vocabulary projection for positions without targets. Remove the final unused Transformer position from each candidate row. Add coverage for empty candidates in mixed batches. Refs #593. --- engine/neural/sentence_model.cpp | 47 ++++++++++++++++------- engine/tests/src/test_neural_sentence.cpp | 17 ++++++++ 2 files changed, 50 insertions(+), 14 deletions(-) diff --git a/engine/neural/sentence_model.cpp b/engine/neural/sentence_model.cpp index 921289c5..50747706 100644 --- a/engine/neural/sentence_model.cpp +++ b/engine/neural/sentence_model.cpp @@ -849,7 +849,22 @@ std::vector SentenceModel::target_log_probs(const std::vector &hid { const std::size_t n_embd = config_.n_embd; const std::size_t vocab = config_.vocab; - std::vector logits(rows * vocab, 0.0f); + std::vector active_rows; + active_rows.reserve(rows); + for (std::size_t row = 0; row < rows; ++row) + { + if (targets[row] != kNoTarget) + { + active_rows.push_back(row); + } + } + std::vector out(rows, 0.0f); + if (active_rows.empty()) + { + return out; + } + + std::vector logits(active_rows.size() * vocab, 0.0f); // Output-major over the vocabulary, and widening each row's int8 once, for the same reasons as // linear(): the tied embedding table is the single largest matrix in the model. std::vector scratch(token_.quantized ? n_embd : 0, 0.0f); @@ -871,20 +886,16 @@ std::vector SentenceModel::target_log_probs(const std::vector &hid { weights = &token_.floats[id * n_embd]; } - for (std::size_t row = 0; row < rows; ++row) + for (std::size_t active = 0; active < active_rows.size(); ++active) { - logits[row * vocab + id] = dot(&hidden[row * n_embd], weights, n_embd) * scale; + logits[active * vocab + id] = dot(&hidden[active_rows[active] * n_embd], weights, n_embd) * scale; } } - std::vector out(rows, 0.0f); - for (std::size_t row = 0; row < rows; ++row) + for (std::size_t active = 0; active < active_rows.size(); ++active) { - if (targets[row] == kNoTarget) - { - continue; - } - const float *source = &logits[row * vocab]; + const std::size_t row = active_rows[active]; + const float *source = &logits[active * vocab]; float highest = -std::numeric_limits::infinity(); for (std::size_t id = 0; id < vocab; ++id) { @@ -953,12 +964,17 @@ std::vector SentenceModel::score_sentences(const std::string &context, longest = std::max(longest, ids.size()); tails.push_back(std::move(ids)); } - const std::size_t width = longest + 1; // the shared leading token, then the longest candidate + if (longest == 0) + { + return std::vector(texts.size(), 0.0); + } + // The last candidate token is a target only. Its hidden state predicts no scored character. + const std::size_t width = longest; const std::uint32_t last = full.back(); std::vector head(full.begin(), full.end() - 1); - // Head plus the widest row must fit the position table; the oldest context goes first. - const std::size_t room = limit > width ? limit - width : 0; + // Keep the old context limit so removing the unused final token does not change scores. + const std::size_t room = limit > longest + 1 ? limit - (longest + 1) : 0; if (head.size() > room) { head.erase(head.begin(), head.begin() + static_cast(head.size() - room)); @@ -973,9 +989,12 @@ std::vector SentenceModel::score_sentences(const std::string &context, flat[row * width] = last; for (std::size_t i = 0; i < tails[row].size(); ++i) { - flat[row * width + 1 + i] = tails[row][i]; // Position i predicts token i+1, so the leading token's slot carries the first character. targets[row * width + i] = tails[row][i]; + if (i + 1 < tails[row].size()) + { + flat[row * width + 1 + i] = tails[row][i]; + } } } diff --git a/engine/tests/src/test_neural_sentence.cpp b/engine/tests/src/test_neural_sentence.cpp index 0a2e5be9..c487b280 100644 --- a/engine/tests/src/test_neural_sentence.cpp +++ b/engine/tests/src/test_neural_sentence.cpp @@ -152,6 +152,23 @@ int run_test() } } + // Empty candidates have no targets. They must not read a padding logit. + { + const std::vector texts = {"", "上海", "", "伤害"}; + const std::vector scores = model->score_sentences("我今天想去", texts); + expect(scores.size() == texts.size(), "empty candidates should keep their batch positions"); + if (scores.size() == texts.size()) + { + for (std::size_t i = 0; i < texts.size(); ++i) + { + expect_near(scores[i], reference_score(*model, "我今天想去", texts[i]), 1e-3, + "empty candidates should not affect other scores"); + } + } + expect(model->score_sentences("", {""}) == std::vector{0.0}, + "a batch without targets should score zero"); + } + // ---- 批量与单条一致：padding 行不能污染同批的其它行 ---- { const std::string context = "我今天想去"; From 1b5e7a71e4c431f82c908e61d33ebb69236a1021 Mon Sep 17 00:00:00 2001 From: luojiyin Date: Sat, 3 Oct 2026 16:32:42 +0800 Subject: [PATCH 2/3] refactor(engine): name compact logit slots Use slot for compact logits and row for Transformer positions. --- engine/neural/sentence_model.cpp | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/engine/neural/sentence_model.cpp b/engine/neural/sentence_model.cpp index 50747706..bb467838 100644 --- a/engine/neural/sentence_model.cpp +++ b/engine/neural/sentence_model.cpp @@ -886,16 +886,17 @@ std::vector SentenceModel::target_log_probs(const std::vector &hid { weights = &token_.floats[id * n_embd]; } - for (std::size_t active = 0; active < active_rows.size(); ++active) + for (std::size_t slot = 0; slot < active_rows.size(); ++slot) { - logits[active * vocab + id] = dot(&hidden[active_rows[active] * n_embd], weights, n_embd) * scale; + const std::size_t row = active_rows[slot]; + logits[slot * vocab + id] = dot(&hidden[row * n_embd], weights, n_embd) * scale; } } - for (std::size_t active = 0; active < active_rows.size(); ++active) + for (std::size_t slot = 0; slot < active_rows.size(); ++slot) { - const std::size_t row = active_rows[active]; - const float *source = &logits[active * vocab]; + const std::size_t row = active_rows[slot]; + const float *source = &logits[slot * vocab]; float highest = -std::numeric_limits::infinity(); for (std::size_t id = 0; id < vocab; ++id) { From a18b3373c4504dd41568a1069d858e89f32113f9 Mon Sep 17 00:00:00 2001 From: luojiyin Date: Sat, 3 Oct 2026 16:40:07 +0800 Subject: [PATCH 3/3] ci(server): rerun checks after OpenCC file lock