Merge pull request #362 from EVEngine/codex/tensor-onnx-gpu-tts
feat(tensor): add quantized ONNX inference with reusable Vulkan sessions
This commit is contained in:
47 files changed
+5595
-98
No files matched your search
@@ -159,6 +159,26 @@ fp8/fp4 的 block scale 取 `maxAbs / 格式最大幅值`（e4m3=240、e2m1=6）
|
||||
量化正确性由 `tensor.quant.*` 与 `tensor.llm.quantizedDtypes` 测试覆盖（含
|
||||
GPU 编译路径），资产缺失时自动跳过。
|
||||
|
||||
## C++ 原生 ONNX 导入与量化 GPU 推理
|
||||
|
||||
`tensor/OnnxModel.h` 直接导入 ONNX，保留 int8/uint8 权重、仿射 scale/zero-point
|
||||
及精确 int32/int64 数据。`run()` 执行 CPU 参考路径；`runGpu()` 接收
|
||||
`createOnnxGpuCompute()` 返回的引擎 Gpgpu 适配器，在 Vulkan 设备线程上同步执行。
|
||||
GPU 负责量化矩阵乘、卷积、LSTM 投影、浮点神经网络计算；形状、索引、控制流、
|
||||
LSTM 门控仍由 CPU 处理；动态量化在 GPU 上计算，只回读少量校验标量。
|
||||
GPU 错误明确返回，不会自动改用 CPU 重试。
|
||||
|
||||
已用原版 Kokoro v1.1 INT8 模型跑通中文语音整图，包括 Loop/If/Sequence。
|
||||
接口使用 owning Result，输出不依赖模型生命周期，随机激励可指定种子。
|
||||
当前 ONNX GPU 执行已使用驻留缓冲区和批量 Sequence 提交；连续 GPU 节点之间不回读，
|
||||
仅在 CPU 数据依赖边界下载。应在推理循环外创建并保留 `createOnnxGpuCompute()`
|
||||
返回的 GPU 会话，跨调用复用管线、权重和缓冲区池；Graphics 销毁前自动清理，
|
||||
失效会话明确拒绝执行。实测同进程第二次合成约 2.60 秒，仍需优化 LSTM CPU 门控等开销。文字前处理、Squirrel
|
||||
绑定及 dialogue 播放接入不在此示例范围内。
|
||||
|
||||
完整构建命令、模型资源、数值契约、限制和可重复语音测试见
|
||||
[原生 ONNX 示例](../../../examples/tensor/onnx/README.md)。
|
||||
|
||||
## 常见问题
|
||||
|
||||
- 对 symbolic Tensor 调用 `get()`。
|
||||
@@ -231,6 +251,9 @@ GPU 编译路径），资产缺失时自动跳过。
|
||||
- 带 `update(dt)` 的系统应在 `eve_update` 调用；绘制方法应在 `eve_render` 调用。
|
||||
- 参数约束、默认值和返回类型以对应模块头文件及 `addFunc` 绑定为准；本文 API 快查与当前源码同步生成。
|
||||
- 张量支持 rank 1–6，dtype 为 float32 / int32；二元运算支持广播。
|
||||
- 原生 ONNX GPU 会话使用有界 shader 编译队列，`createOnnxGpuCompute(compilerWorkers)`
|
||||
可选 1–8 个 CPU 编译线程（默认 4）；同源码去重，编译器上下文和管线缓存均复用。
|
||||
Vulkan 管线创建与提交仍在设备线程，`runGpu` 保持同步返回；并发数应按目标构建实测选择。
|
||||
- GPU 路径：`compile()` 需要已初始化的 Vulkan Graphics（先创建窗口）。Windows
|
||||
上 GLSL→SPIR-V 由链接进引擎的 shaderc 静态库完成，不需要安装 glslc。
|
||||
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
cmake_minimum_required(VERSION 3.21)
|
||||
project(EveTensorOnnx LANGUAGES CXX)
|
||||
get_filename_component(EVE_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/../../.." ABSOLUTE)
|
||||
set(ZEROERR_SOURCE "${EVE_ROOT}/external/zeroerr" CACHE PATH "Initialized zeroerr source directory")
|
||||
set(CMAKE_CXX_STANDARD 20)
|
||||
set(CMAKE_CXX_STANDARD_REQUIRED ON)
|
||||
add_library(eve_tensor_onnx STATIC
|
||||
"${EVE_ROOT}/src/modules/tensor/AffineQuant.cpp"
|
||||
"${EVE_ROOT}/src/modules/tensor/OnnxReader.cpp"
|
||||
"${EVE_ROOT}/src/modules/tensor/OnnxModel.cpp"
|
||||
"${EVE_ROOT}/src/modules/tensor/OnnxRuntime.cpp"
|
||||
"${EVE_ROOT}/src/modules/tensor/OnnxExecute.cpp"
|
||||
"${EVE_ROOT}/src/modules/tensor/OnnxQuant.cpp"
|
||||
"${EVE_ROOT}/src/modules/tensor/OnnxGpuKernels.cpp"
|
||||
"${EVE_ROOT}/src/modules/tensor/OnnxShapeOps.cpp"
|
||||
"${EVE_ROOT}/src/modules/tensor/OnnxIndexOps.cpp"
|
||||
"${EVE_ROOT}/src/modules/tensor/OnnxNumeric.cpp"
|
||||
"${EVE_ROOT}/src/modules/tensor/OnnxNeural.cpp"
|
||||
"${EVE_ROOT}/src/modules/tensor/OnnxMisc.cpp"
|
||||
"${EVE_ROOT}/src/modules/tensor/OnnxLstm.cpp"
|
||||
"${EVE_ROOT}/src/modules/tensor/Tensor.cpp"
|
||||
"${EVE_ROOT}/src/modules/tensor/CpuKernels.cpp"
|
||||
"${EVE_ROOT}/src/engine/common/Exception.cpp"
|
||||
"${EVE_ROOT}/src/engine/common/RenderTrace.cpp")
|
||||
target_include_directories(eve_tensor_onnx PUBLIC "${EVE_ROOT}/src/modules" "${EVE_ROOT}/src/engine" "${ZEROERR_SOURCE}/include")
|
||||
if(MSVC)
|
||||
target_compile_options(eve_tensor_onnx PUBLIC /utf-8 /EHsc)
|
||||
endif()
|
||||
add_executable(onnx_probe main.cpp)
|
||||
target_link_libraries(onnx_probe PRIVATE eve_tensor_onnx)
|
||||
option(EVE_ONNX_TESTS "Build focused tensor ONNX contract tests" ON)
|
||||
if(EVE_ONNX_TESTS)
|
||||
set(BUILD_TEST OFF CACHE BOOL "" FORCE)
|
||||
set(BUILD_EXAMPLES OFF CACHE BOOL "" FORCE)
|
||||
add_subdirectory("${ZEROERR_SOURCE}" zeroerr)
|
||||
target_link_libraries(eve_tensor_onnx PUBLIC zeroerr)
|
||||
add_executable(onnx_contracts "${EVE_ROOT}/test/tensor_onnx.cpp")
|
||||
target_link_libraries(onnx_contracts PRIVATE eve_tensor_onnx zeroerr)
|
||||
enable_testing()
|
||||
foreach(case roundingAndRange exactAccumulators convolutionValidation)
|
||||
add_test(NAME "tensor.onnx.affine.${case}" COMMAND onnx_contracts "--testcase=tensor.onnx.affine.${case}")
|
||||
endforeach()
|
||||
add_test(NAME tensor.onnx.import.transactionAndDiagnostics COMMAND onnx_contracts "--testcase=tensor.onnx.import.transactionAndDiagnostics")
|
||||
add_test(NAME tensor.onnx.storage.aliasAndCpuMutation COMMAND onnx_contracts "--testcase=tensor.onnx.storage.aliasAndCpuMutation")
|
||||
add_test(NAME tensor.onnx.gpu.failureAndScope COMMAND onnx_contracts "--testcase=tensor.onnx.gpu.failureAndScope")
|
||||
add_test(NAME tensor.onnx.import.owningOutputs COMMAND onnx_contracts "--testcase=tensor.onnx.import.owningOutputs")
|
||||
endif()
|
||||
|
||||
if(NOT EVE_ONNX_TESTS)
|
||||
target_compile_definitions(eve_tensor_onnx PRIVATE ZEROERR_NO_ASSERT)
|
||||
target_compile_definitions(onnx_probe PRIVATE ZEROERR_NO_ASSERT)
|
||||
endif()
|
||||
|
||||
option(EVE_ONNX_VULKAN_PROBE "Enable standalone hardware parity provider (Vulkan SDK + shaderc)" OFF)
|
||||
if(EVE_ONNX_VULKAN_PROBE)
|
||||
find_package(Vulkan REQUIRED)
|
||||
find_library(ONNX_SHADERC NAMES shaderc_shared HINTS "$ENV{VULKAN_SDK}/Lib" REQUIRED)
|
||||
target_sources(onnx_probe PRIVATE VulkanProbeCompute.cpp)
|
||||
target_compile_definitions(onnx_probe PRIVATE EVE_ONNX_VULKAN_PROBE)
|
||||
target_link_libraries(onnx_probe PRIVATE Vulkan::Vulkan "${ONNX_SHADERC}")
|
||||
if(WIN32)
|
||||
add_custom_command(TARGET onnx_probe POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different "$ENV{VULKAN_SDK}/Bin/shaderc_shared.dll" "$<TARGET_FILE_DIR:onnx_probe>")
|
||||
endif()
|
||||
endif()
|
||||
@@ -0,0 +1,223 @@
|
||||
# Native quantized ONNX inference
|
||||
|
||||
Direct ONNX execution in EVEngine's tensor module, without ONNX Runtime, protobuf
|
||||
or Python at runtime. Original INT8 weights remain packed. The existing block
|
||||
quantization format is separate from ONNX affine quantization.
|
||||
|
||||
## Engine GPU path
|
||||
|
||||
After configuring the engine with tensor enabled:
|
||||
|
||||
```powershell
|
||||
cmake/with-msvc.cmd cmake --build build/win32-debug --target onnx_engine_probe --parallel 4
|
||||
$env:EVE_ONNX_GPU='1'
|
||||
build/win32-debug/test/onnx_engine_probe.exe model.int8.onnx audio audio.f32 0,73,127,172,88,108,169,4,0 style.f32
|
||||
```
|
||||
|
||||
The probe initializes headless Graphics and calls `createOnnxGpuCompute()`. It also
|
||||
checks device-unavailable behavior before initialization. `style.f32` contains 256
|
||||
little-endian floats; providing it also supplies speed=1. The IDs encode “你好。”.
|
||||
|
||||
The current minimal host profile also needs font and devtools to link its closure:
|
||||
`-DEVENGINE_PROFILE=minimal -DEVENGINE_MODULE_tensor=ON -DEVENGINE_MODULE_font=ON
|
||||
-DEVENGINE_MODULE_devtools=ON`. No manifest or default profile is changed here.
|
||||
|
||||
C++ callers check Results from `OnnxModel::load`, `createOnnxGpuCompute` and
|
||||
`model->runGpu(feeds, *compute, requestedNames)`. The result owns `outputs` and
|
||||
reports successful `dispatches` plus transfer, submission and allocation/reuse counters. `run(feeds, requestedNames)` selects CPU for
|
||||
reference execution. Empty requestedNames selects all graph outputs.
|
||||
|
||||
GPU kernels cover integer MatMul/Conv and LSTM projections, FP32 MatMul/ConvTranspose,
|
||||
broadcast arithmetic, activations, Softmax, sum/mean reductions, InstanceNormalization,
|
||||
Resize and CumSum. Dynamic quantization reduces range on GPU and reads only three
|
||||
validation scalars; packed activation generation stays on GPU. Shape/index/sequence/control
|
||||
operations, padding, other quantization variants, RNG and LSTM gates stay on CPU. GPU errors never trigger an automatic CPU retry.
|
||||
The GLSL adapter explicitly rejects WebGPU.
|
||||
|
||||
The adapter reuses the existing `gpgpu::Sequence` recording and `submitAsync`/`wait`
|
||||
mechanism used by tensor GPU execution. Consecutive GPU nodes share immutable device
|
||||
buffers. Identity/reshape aliases do not transfer data. CPU reads materialize host bytes;
|
||||
CPU writes detach shared storage. Packed initializers and recurrent weights are cached
|
||||
by owning storage identity across runs for immutable model initializers. Dead device allocations return to a size-bounded
|
||||
reuse pool. A batch flush occurs at a CPU-read boundary or after a bounded recording
|
||||
backlog, with no download at a backlog flush. The public runGpu call remains synchronous.
|
||||
|
||||
The static `GpuProgram` compiler is not an ONNX control-flow executor; ONNX uses the same
|
||||
Gpgpu buffers/sequences directly while retaining dynamic Loop/If scheduling and affine
|
||||
integer semantics. Device allocations are bounded to 512 MiB, with a 128 MiB host-input
|
||||
cache and size-compatible buffer reuse. All referenced buffers survive queued work.
|
||||
|
||||
Retain the provider returned by `createOnnxGpuCompute()` outside the dialogue/inference
|
||||
loop and pass it to every `runGpu` call. It owns a reusable compiled GPU session:
|
||||
shader pipelines (up to 1024 variants), model initializers (up to 128 MiB) and device
|
||||
buffer pools (up to 512 MiB) persist across calls. Shader cache keys include the complete
|
||||
generated source, including shapes, integer interpretation and LSTM direction offsets.
|
||||
A different shape compiles only missing variants. Dynamic activations remain per-call.
|
||||
Failed calls discard pending uploads before allowing cache reuse; compiled shaders stay.
|
||||
|
||||
The session subscribes to `Graphics::onResourcesRetiring`: resources are freed before
|
||||
Vulkan device teardown even if the session outlives Graphics. A retired session rejects
|
||||
GPU work; construct a new one for the new device. Destroying the session first cancels
|
||||
the subscription. All of this is device-thread-only. Feeds are borrowed during the call;
|
||||
outputs outlive the model and provider. Use the device's owning thread without
|
||||
concurrent provider calls. Independent CPU runs are thread-safe. No global RNG,
|
||||
background tasks, script callbacks or locks around compute calls are introduced.
|
||||
|
||||
## Standalone build and tests
|
||||
|
||||
```powershell
|
||||
cmake/with-msvc.cmd cmake -S examples/tensor/onnx -B build/tensor-onnx -G Ninja -DCMAKE_BUILD_TYPE=Release
|
||||
cmake/with-msvc.cmd cmake --build build/tensor-onnx --parallel 4
|
||||
ctest --test-dir build/tensor-onnx --output-on-failure
|
||||
build/tensor-onnx/onnx_probe.exe model.int8.onnx
|
||||
```
|
||||
|
||||
Initialize `external/zeroerr`, or set `ZEROERR_SOURCE` to an initialized checkout.
|
||||
`EVE_ONNX_TESTS=OFF` removes test linkage. The CPU path needs no Vulkan, SDL or VM.
|
||||
Tests and their library deliberately use the same assertion policy.
|
||||
|
||||
Optional `-DEVE_ONNX_VULKAN_PROBE=ON` enables a standalone hardware parity provider
|
||||
using Vulkan SDK and shaderc. It executes the same tensor-generated kernels on a
|
||||
physical GPU. Production uses the engine's Gpgpu adapter instead.
|
||||
|
||||
## Reproduce Chinese speech
|
||||
|
||||
Download `model.int8.onnx`, `voices.bin` and `tokens.txt` from the
|
||||
[Kokoro INT8 v1.1 package](https://huggingface.co/csukuangfj/kokoro-int8-multi-lang-v1_1/tree/main).
|
||||
Tested model: 114299010 bytes, SHA256
|
||||
`bda15858163726a492d02a9a727bc263551b86ac77f90812c4b30ff41d380e26`.
|
||||
|
||||
```powershell
|
||||
python -m pip install numpy onnx onnxruntime
|
||||
python scripts/run_kokoro_onnx.py --model model.int8.onnx --voices voices.bin --tokens tokens.txt --probe build/win32-debug/test/onnx_engine_probe.exe --work build/kokoro-gpu --gpu
|
||||
```
|
||||
|
||||
The utility prepares explicit phonemes and zf_001 voice data, invokes native inference,
|
||||
and writes `audio.wav`, `native.log` and `result.json`. It does not transform weights.
|
||||
Style selection follows [sherpa-onnx](https://github.com/k2-fsa/sherpa-onnx/blob/master/sherpa-onnx/csrc/offline-tts-kokoro-model.cc):
|
||||
index by phoneme length excluding boundary zeros. This is not yet a text normalizer,
|
||||
phonemizer or dialogue playback integration.
|
||||
|
||||
## Import and numerical contract
|
||||
|
||||
- IR 3–10, default opset 13–17, Microsoft opset 1; FP32, int8/uint8, int32/int64,
|
||||
bool and internal sequences. External/sparse tensor data and functions are rejected.
|
||||
- Bounded parsing checks truncation, sizes, duplicates and lexical captures. Nested
|
||||
Loop/If bodies are parsed; scheduling includes implicit captured values, including
|
||||
Kokoro captures appearing later in the serialized main graph.
|
||||
- Unknown operators are reported and required unknown nodes fail before dispatch.
|
||||
This bounded executor is not a complete ONNX schema validator.
|
||||
- Affine APIs reject nonfinite values. The ONNX DynamicQuantizeLinear bridge matches
|
||||
ORT CPU for NaN: exclude it from range estimation and emit byte code zero. Kokoro's
|
||||
exported STFT phase graph produces 0/0 at zero-amplitude bins. Infinity remains an
|
||||
error in quantization. Rounding is ties-to-even.
|
||||
- GPU integer accumulation is exact int32, with a conservative term-count overflow
|
||||
bound. CPU checks actual int64 sums before narrowing. Per-axis dequantization and
|
||||
forward/reverse/bidirectional quantized LSTM are supported.
|
||||
- `OnnxRunOptions.seed` controls per-node RNG streams. Floating CPU/GPU results use
|
||||
tolerances. `requireFinite` is a strict diagnostic that rejects intentional
|
||||
intermediate infinity/NaN; keep it false for Kokoro.
|
||||
- Specialized boundaries: integer MatMul uses rank>=2 A/rank-2 B and scalar zeros;
|
||||
Conv is 1D/2D NOTSET padding; ConvTranspose is 1D; nearest Resize uses
|
||||
asymmetric/floor coordinates; linear Resize interpolates the final axis; Loop
|
||||
supports carried tensors/sequences without scan outputs. Other variants fail.
|
||||
- Limits: 512 MiB model, 100000 recursive nodes, depth 16, rank 6, 128M elements per
|
||||
tensor, 10000 loop iterations, 1M execution steps. Live owned intermediates and
|
||||
copied outputs are bounded to 512 MiB; model/feed data and temporary workspaces
|
||||
are additional. GPU work is bounded to 65535 groups of 64.
|
||||
|
||||
## Validation and performance
|
||||
|
||||
```powershell
|
||||
python scripts/test_tensor_onnx.py --probe build/tensor-onnx/onnx_probe.exe --work build/onnx-cpu-fixtures --kokoro model.int8.onnx
|
||||
python scripts/test_tensor_onnx.py --probe build/tensor-onnx/onnx_probe.exe --work build/onnx-gpu-fixtures --kokoro model.int8.onnx --gpu
|
||||
```
|
||||
|
||||
Reference packages are test-only. Coverage includes exact integer results, floating
|
||||
kernels, shape operations, lexical If captures, Loop/Sequence, neural resampling,
|
||||
and all six original packed Kokoro LSTMs. Contract tests include malformed inputs,
|
||||
owning outputs, unsupported preflight, injected GPU failure and scope cleanup.
|
||||
|
||||
The original model generates 26400 finite 24 kHz samples (1.1 s of audio).
|
||||
The GPU implementation remains synchronous at `runGpu`; CPU boundaries include LSTM
|
||||
gates, data-dependent shape/index work and quantization validation scalars.
|
||||
|
||||
The hardware residency contract verifies that a 16-node Add chain with aliases uses
|
||||
only two input uploads, one download and one submission, on the first run. The second run performs zero uploads and zero shader compilations.
|
||||
Separate hardware tests cover staging-pool growth/reordering and descriptor retirement
|
||||
when unsubmitted sequences are canceled. Run them with:
|
||||
|
||||
```powershell
|
||||
cmake/with-msvc.cmd cmake --build build/win32-debug --target onnx_gpu_contracts --parallel 4
|
||||
$env:EVENGINE_VULKAN_VALIDATION='1'
|
||||
$cases = 'tensor.onnx.gpu.residentChain', 'gpgpu.sequence.stagingSizeReuse', 'gpgpu.sequence.cancelRecordingRetiresDescriptors', 'tensor.onnx.gpu.sessionShapesAndLifetime', 'tensor.onnx.gpu.parallelCompilation'
|
||||
foreach ($case in $cases) {
|
||||
build/win32-debug/test/onnx_gpu_contracts.exe "--testcase=$case"
|
||||
if ($LASTEXITCODE -ne 0) { throw "Failed: $case" }
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
The session now compiles different GLSL sources on a bounded CPU queue and defers
|
||||
pipeline creation/dispatch recording until a host-read or segment-budget boundary.
|
||||
Dispatches retain their buffers and preserve graph order. Identical pending sources
|
||||
share one future; completed pipelines persist across calls. All Vulkan access stays
|
||||
on the device thread. Error cleanup drains admitted CPU jobs and discards unsubmitted
|
||||
commands; compiler workers never capture a device or call the render tracer.
|
||||
|
||||
`createOnnxGpuCompute(compilerWorkers)` accepts 1..8 workers (default 4), with at most
|
||||
48 waiting compilation jobs. The probe exposes this as `EVE_ONNX_COMPILER_WORKERS`.
|
||||
This is an upper bound, not a promise of a speedup: dynamic CPU boundaries limit how
|
||||
many independent sources are available. Shaderc retains a compiler/options context
|
||||
per thread until that thread exits. Session workers are joined at destruction.
|
||||
Thread-safety follows the [shaderc C API contract](https://github.com/google/shaderc/blob/main/libshaderc/include/shaderc/shaderc.h).
|
||||
|
||||
Windows CMake now defines `EVE_HAS_SHADERC` on EVGpgpu itself. Previously its
|
||||
INTERFACE-only definition linked the library but left the implementation launching
|
||||
`glslc.exe`. A shaderc-enabled build now reports compilation failures directly and
|
||||
never requires that executable. The build without the shaderc library uses an external compiler path that retains unique
|
||||
temporary names through output consumption and explicitly restricts child handle
|
||||
inheritance so concurrent jobs cannot lock each other's temporary files.
|
||||
|
||||
Measured on the same native Windows Debug build and original model/input, with a
|
||||
fresh process/session (application shader cache empty; driver caches were not cleared):
|
||||
|
||||
- Previous synchronous compilation: first `runGpu` 35.72-36.99 s; repeat 2.60-2.64 s.
|
||||
- Four external compiler workers: first 19.89 s; repeat 2.69 s.
|
||||
- Four in-process workers with retained contexts: first 8.23-9.88 s; repeat 2.74-3.26 s.
|
||||
- One in-process worker with retained context: first 6.81-9.82 s; repeat 2.69-3.22 s.
|
||||
|
||||
Each new session compiled 370 variants; every repeat compiled zero (2608 cache hits).
|
||||
Audio was byte-identical across the old implementation, external/in-process compilers,
|
||||
and one/four-worker runs. Thus compiler wiring/context reuse supplies the main benefit;
|
||||
on this Debug workload, four workers did not consistently beat one. The final paired
|
||||
run measured 9.82 s cold / 3.22 s warm with one worker, and 9.88 s / 3.26 s with four.
|
||||
Timing varied between runs, so the ranges above are retained rather than selecting only the best result. Release performance is not
|
||||
established. Even the warm call exceeds the 1.1 s audio duration, so this is not real-time.
|
||||
|
||||
Set `EVE_ONNX_PROFILE=1` for wall-time buckets. `compile_task_ms` sums durations of
|
||||
worker tasks (overlapping durations, not elapsed time); `compile_wait_ms` is device-thread
|
||||
blocking on futures; `pipeline_ms` is Vulkan pipeline creation on that thread.
|
||||
`shader_ms` comprises the latter two. `compiler_peak_workers` reports observed concurrency.
|
||||
Allocation, upload/dispatch recording, submit/wait and host readback are also reported;
|
||||
these are not GPU timestamp-query kernel timings. An earlier four-worker cold measurement reported
|
||||
7.75 s aggregate compile tasks, 2.07 s compile waits and 2.86 s pipeline creation.
|
||||
The one-worker control measured 3.07 s aggregate tasks, 1.22 s waits and 2.65 s pipelines.
|
||||
|
||||
Reproduce both configurations with the same model/session in each process:
|
||||
|
||||
```powershell
|
||||
$env:EVE_ONNX_PROFILE='1'
|
||||
$env:EVE_ONNX_COMPILER_WORKERS='4' # use 1 for the single-worker control
|
||||
python scripts/run_kokoro_onnx.py --model model.int8.onnx --voices voices.bin --tokens tokens.txt --probe build/win32-debug/test/onnx_engine_probe.exe --work build/kokoro-repeat --gpu --repeat 2
|
||||
```
|
||||
|
||||
`result.json` includes per-iteration latency, compilation/cache-hit counts, peak workers
|
||||
and exact repeat verification. Hardware contracts cover serial/parallel dependent chains
|
||||
across submission boundaries, changed shapes/constants, cold cancellation/retry,
|
||||
compilation failure/recovery, optional-device absence and both device/session destruction orders.
|
||||
|
||||
Architecture rules applied: checked Results, ownership and device-thread contracts,
|
||||
transactional publication, optional-device present/absent tests, failure injection,
|
||||
deterministic RNG and foreign-format bounds. No ECS, Link, module boundary or engine
|
||||
persistence changes are introduced.
|
||||
@@ -0,0 +1,249 @@
|
||||
#include "VulkanProbeCompute.h"
|
||||
#include <shaderc/shaderc.h>
|
||||
#include <vulkan/vulkan.h>
|
||||
#include <algorithm>
|
||||
#include <cstring>
|
||||
#include <iostream>
|
||||
#include <stdexcept>
|
||||
|
||||
namespace {
|
||||
void check(VkResult r) {
|
||||
if (r != VK_SUCCESS) throw std::runtime_error("Vulkan failure: " + std::to_string(r));
|
||||
}
|
||||
class VulkanCompute final : public eve::tensor::OnnxCompute {
|
||||
size_t dispatches = 0;
|
||||
VkInstance instance = VK_NULL_HANDLE;
|
||||
VkPhysicalDevice physical = VK_NULL_HANDLE;
|
||||
VkDevice device = VK_NULL_HANDLE;
|
||||
VkQueue queue = VK_NULL_HANDLE;
|
||||
VkCommandPool commands = VK_NULL_HANDLE;
|
||||
VkPhysicalDeviceMemoryProperties memory{};
|
||||
struct Resources {
|
||||
VkDevice device;
|
||||
VkCommandPool pool;
|
||||
std::vector<VkBuffer> buffers;
|
||||
std::vector<VkDeviceMemory> memories;
|
||||
VkShaderModule shader = VK_NULL_HANDLE;
|
||||
VkDescriptorSetLayout setLayout = VK_NULL_HANDLE;
|
||||
VkPipelineLayout layout = VK_NULL_HANDLE;
|
||||
VkPipeline pipeline = VK_NULL_HANDLE;
|
||||
VkDescriptorPool descriptors = VK_NULL_HANDLE;
|
||||
VkCommandBuffer command = VK_NULL_HANDLE;
|
||||
VkFence fence = VK_NULL_HANDLE;
|
||||
~Resources() {
|
||||
if (fence) vkDestroyFence(device, fence, nullptr);
|
||||
if (command) vkFreeCommandBuffers(device, pool, 1, &command);
|
||||
if (descriptors) vkDestroyDescriptorPool(device, descriptors, nullptr);
|
||||
if (pipeline) vkDestroyPipeline(device, pipeline, nullptr);
|
||||
if (layout) vkDestroyPipelineLayout(device, layout, nullptr);
|
||||
if (setLayout) vkDestroyDescriptorSetLayout(device, setLayout, nullptr);
|
||||
if (shader) vkDestroyShaderModule(device, shader, nullptr);
|
||||
for (auto b : buffers) vkDestroyBuffer(device, b, nullptr);
|
||||
for (auto m : memories) vkFreeMemory(device, m, nullptr);
|
||||
}
|
||||
};
|
||||
|
||||
public:
|
||||
~VulkanCompute() override {
|
||||
if (commands) vkDestroyCommandPool(device, commands, nullptr);
|
||||
if (device) vkDestroyDevice(device, nullptr);
|
||||
if (instance) vkDestroyInstance(instance, nullptr);
|
||||
}
|
||||
void initialize() {
|
||||
VkApplicationInfo app{VK_STRUCTURE_TYPE_APPLICATION_INFO};
|
||||
app.pApplicationName = "EVEngine ONNX GPU parity";
|
||||
app.apiVersion = VK_API_VERSION_1_0;
|
||||
VkInstanceCreateInfo ci{VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO};
|
||||
ci.pApplicationInfo = &app;
|
||||
check(vkCreateInstance(&ci, nullptr, &instance));
|
||||
uint32_t count = 0;
|
||||
check(vkEnumeratePhysicalDevices(instance, &count, nullptr));
|
||||
std::vector<VkPhysicalDevice> devices(count);
|
||||
check(vkEnumeratePhysicalDevices(instance, &count, devices.data()));
|
||||
uint32_t family = UINT32_MAX;
|
||||
for (auto d : devices) {
|
||||
VkPhysicalDeviceProperties p;
|
||||
vkGetPhysicalDeviceProperties(d, &p);
|
||||
if (p.deviceType != VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU &&
|
||||
p.deviceType != VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU)
|
||||
continue;
|
||||
uint32_t n = 0;
|
||||
vkGetPhysicalDeviceQueueFamilyProperties(d, &n, nullptr);
|
||||
std::vector<VkQueueFamilyProperties> qs(n);
|
||||
vkGetPhysicalDeviceQueueFamilyProperties(d, &n, qs.data());
|
||||
for (uint32_t i = 0; i < n; ++i)
|
||||
if (qs[i].queueFlags & VK_QUEUE_COMPUTE_BIT) {
|
||||
physical = d;
|
||||
family = i;
|
||||
break;
|
||||
}
|
||||
if (physical) {
|
||||
std::cout << "ONNX_GPU_DEVICE " << p.deviceName << '\n';
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!physical) throw std::runtime_error("No hardware Vulkan compute device");
|
||||
vkGetPhysicalDeviceMemoryProperties(physical, &memory);
|
||||
float priority = 1;
|
||||
VkDeviceQueueCreateInfo q{VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO};
|
||||
q.queueFamilyIndex = family;
|
||||
q.queueCount = 1;
|
||||
q.pQueuePriorities = &priority;
|
||||
VkDeviceCreateInfo dc{VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO};
|
||||
dc.queueCreateInfoCount = 1;
|
||||
dc.pQueueCreateInfos = &q;
|
||||
check(vkCreateDevice(physical, &dc, nullptr, &device));
|
||||
vkGetDeviceQueue(device, family, 0, &queue);
|
||||
VkCommandPoolCreateInfo pc{VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO};
|
||||
pc.queueFamilyIndex = family;
|
||||
check(vkCreateCommandPool(device, &pc, nullptr, &commands));
|
||||
}
|
||||
eve::Result<std::vector<uint8_t>> dispatch(const eve::tensor::OnnxKernel& k) override {
|
||||
try {
|
||||
Resources r{device, commands};
|
||||
if (k.inputs.size() >= 8 || !k.outputBytes || !k.workItems || uint64_t(k.workItems) + 63 > 65535u * 64u)
|
||||
throw std::runtime_error("Invalid probe kernel extent");
|
||||
const uint32_t n = static_cast<uint32_t>(k.inputs.size() + 1);
|
||||
std::vector<VkDescriptorSetLayoutBinding> bindings(n);
|
||||
std::vector<VkDescriptorBufferInfo> infos(n);
|
||||
for (uint32_t i = 0; i < n; ++i) {
|
||||
const size_t bytes = i + 1 == n ? k.outputBytes : k.inputs[i].size();
|
||||
VkBufferCreateInfo bc{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO};
|
||||
bc.size = std::max(size_t(4), (bytes + 3) & ~size_t(3));
|
||||
bc.usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT;
|
||||
VkBuffer b;
|
||||
check(vkCreateBuffer(device, &bc, nullptr, &b));
|
||||
r.buffers.push_back(b);
|
||||
VkMemoryRequirements req;
|
||||
vkGetBufferMemoryRequirements(device, b, &req);
|
||||
uint32_t type = UINT32_MAX;
|
||||
for (uint32_t j = 0; j < memory.memoryTypeCount; ++j)
|
||||
if ((req.memoryTypeBits & (1u << j)) &&
|
||||
(memory.memoryTypes[j].propertyFlags &
|
||||
(VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT)) ==
|
||||
(VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT)) {
|
||||
type = j;
|
||||
break;
|
||||
}
|
||||
if (type == UINT32_MAX) throw std::runtime_error("No coherent host-visible Vulkan memory");
|
||||
VkMemoryAllocateInfo ma{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO};
|
||||
ma.allocationSize = req.size;
|
||||
ma.memoryTypeIndex = type;
|
||||
VkDeviceMemory m;
|
||||
check(vkAllocateMemory(device, &ma, nullptr, &m));
|
||||
r.memories.push_back(m);
|
||||
check(vkBindBufferMemory(device, b, m, 0));
|
||||
void* mapped;
|
||||
check(vkMapMemory(device, m, 0, bc.size, 0, &mapped));
|
||||
std::memset(mapped, 0, bc.size);
|
||||
if (i + 1 < n && bytes) std::memcpy(mapped, k.inputs[i].data(), bytes);
|
||||
vkUnmapMemory(device, m);
|
||||
bindings[i] = {i, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1, VK_SHADER_STAGE_COMPUTE_BIT, nullptr};
|
||||
infos[i] = {b, 0, bc.size};
|
||||
}
|
||||
auto compiler = shaderc_compiler_initialize();
|
||||
if (!compiler) throw std::runtime_error("shaderc initialization failed");
|
||||
auto compiled = shaderc_compile_into_spv(compiler, k.source.data(), k.source.size(), shaderc_compute_shader,
|
||||
"onnx.comp", "main", nullptr);
|
||||
shaderc_compiler_release(compiler);
|
||||
if (!compiled) throw std::runtime_error("shaderc compilation failed");
|
||||
if (shaderc_result_get_compilation_status(compiled) != shaderc_compilation_status_success) {
|
||||
std::string message = shaderc_result_get_error_message(compiled);
|
||||
shaderc_result_release(compiled);
|
||||
throw std::runtime_error(message);
|
||||
}
|
||||
std::vector<uint32_t> spv(shaderc_result_get_length(compiled) / 4);
|
||||
std::memcpy(spv.data(), shaderc_result_get_bytes(compiled), spv.size() * 4);
|
||||
shaderc_result_release(compiled);
|
||||
VkShaderModuleCreateInfo sm{VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO};
|
||||
sm.codeSize = spv.size() * 4;
|
||||
sm.pCode = spv.data();
|
||||
check(vkCreateShaderModule(device, &sm, nullptr, &r.shader));
|
||||
VkDescriptorSetLayoutCreateInfo sl{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO};
|
||||
sl.bindingCount = n;
|
||||
sl.pBindings = bindings.data();
|
||||
check(vkCreateDescriptorSetLayout(device, &sl, nullptr, &r.setLayout));
|
||||
VkPipelineLayoutCreateInfo pl{VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO};
|
||||
pl.setLayoutCount = 1;
|
||||
pl.pSetLayouts = &r.setLayout;
|
||||
check(vkCreatePipelineLayout(device, &pl, nullptr, &r.layout));
|
||||
VkComputePipelineCreateInfo cp{VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO};
|
||||
cp.layout = r.layout;
|
||||
cp.stage = {VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO};
|
||||
cp.stage.stage = VK_SHADER_STAGE_COMPUTE_BIT;
|
||||
cp.stage.module = r.shader;
|
||||
cp.stage.pName = "main";
|
||||
check(vkCreateComputePipelines(device, VK_NULL_HANDLE, 1, &cp, nullptr, &r.pipeline));
|
||||
VkDescriptorPoolSize ps{VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, n};
|
||||
VkDescriptorPoolCreateInfo dp{VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO};
|
||||
dp.maxSets = 1;
|
||||
dp.poolSizeCount = 1;
|
||||
dp.pPoolSizes = &ps;
|
||||
check(vkCreateDescriptorPool(device, &dp, nullptr, &r.descriptors));
|
||||
VkDescriptorSetAllocateInfo da{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO};
|
||||
da.descriptorPool = r.descriptors;
|
||||
da.descriptorSetCount = 1;
|
||||
da.pSetLayouts = &r.setLayout;
|
||||
VkDescriptorSet set;
|
||||
check(vkAllocateDescriptorSets(device, &da, &set));
|
||||
std::vector<VkWriteDescriptorSet> writes(n);
|
||||
for (uint32_t i = 0; i < n; ++i) {
|
||||
writes[i] = {VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET};
|
||||
writes[i].dstSet = set;
|
||||
writes[i].dstBinding = i;
|
||||
writes[i].descriptorCount = 1;
|
||||
writes[i].descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER;
|
||||
writes[i].pBufferInfo = &infos[i];
|
||||
}
|
||||
vkUpdateDescriptorSets(device, n, writes.data(), 0, nullptr);
|
||||
VkCommandBufferAllocateInfo ca{VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO};
|
||||
ca.commandPool = commands;
|
||||
ca.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY;
|
||||
ca.commandBufferCount = 1;
|
||||
check(vkAllocateCommandBuffers(device, &ca, &r.command));
|
||||
VkCommandBufferBeginInfo begin{VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO};
|
||||
check(vkBeginCommandBuffer(r.command, &begin));
|
||||
VkMemoryBarrier barrier{VK_STRUCTURE_TYPE_MEMORY_BARRIER};
|
||||
barrier.srcAccessMask = VK_ACCESS_HOST_WRITE_BIT;
|
||||
barrier.dstAccessMask = VK_ACCESS_SHADER_READ_BIT;
|
||||
vkCmdPipelineBarrier(r.command, VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0, 1,
|
||||
&barrier, 0, nullptr, 0, nullptr);
|
||||
vkCmdBindPipeline(r.command, VK_PIPELINE_BIND_POINT_COMPUTE, r.pipeline);
|
||||
vkCmdBindDescriptorSets(r.command, VK_PIPELINE_BIND_POINT_COMPUTE, r.layout, 0, 1, &set, 0, nullptr);
|
||||
vkCmdDispatch(r.command, (k.workItems + 63) / 64, 1, 1);
|
||||
barrier.srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT;
|
||||
barrier.dstAccessMask = VK_ACCESS_HOST_READ_BIT;
|
||||
vkCmdPipelineBarrier(r.command, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_HOST_BIT, 0, 1,
|
||||
&barrier, 0, nullptr, 0, nullptr);
|
||||
check(vkEndCommandBuffer(r.command));
|
||||
VkFenceCreateInfo fc{VK_STRUCTURE_TYPE_FENCE_CREATE_INFO};
|
||||
check(vkCreateFence(device, &fc, nullptr, &r.fence));
|
||||
VkSubmitInfo submit{VK_STRUCTURE_TYPE_SUBMIT_INFO};
|
||||
submit.commandBufferCount = 1;
|
||||
submit.pCommandBuffers = &r.command;
|
||||
check(vkQueueSubmit(queue, 1, &submit, r.fence));
|
||||
check(vkWaitForFences(device, 1, &r.fence, VK_TRUE, UINT64_MAX));
|
||||
std::vector<uint8_t> result(k.outputBytes);
|
||||
void* mapped;
|
||||
check(vkMapMemory(device, r.memories.back(), 0, k.outputBytes, 0, &mapped));
|
||||
std::memcpy(result.data(), mapped, k.outputBytes);
|
||||
vkUnmapMemory(device, r.memories.back());
|
||||
if (++dispatches % 100 == 0) std::cout << "GPU_DISPATCHES " << dispatches << std::endl;
|
||||
return eve::Result<std::vector<uint8_t>>::success(std::move(result));
|
||||
} catch (const std::exception& e) {
|
||||
return eve::Result<std::vector<uint8_t>>::failure(
|
||||
eve::Diagnostic::error(eve::DiagnosticCode::Failed, e.what()));
|
||||
}
|
||||
}
|
||||
};
|
||||
} // namespace
|
||||
eve::Result<std::unique_ptr<eve::tensor::OnnxCompute>> createVulkanProbeCompute() {
|
||||
try {
|
||||
auto p = std::make_unique<VulkanCompute>();
|
||||
p->initialize();
|
||||
return eve::Result<std::unique_ptr<eve::tensor::OnnxCompute>>::success(std::move(p));
|
||||
} catch (const std::exception& e) {
|
||||
return eve::Result<std::unique_ptr<eve::tensor::OnnxCompute>>::failure(
|
||||
eve::Diagnostic::error(eve::DiagnosticCode::Failed, e.what()));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
#pragma once
|
||||
#include "tensor/OnnxCompute.h"
|
||||
/** @brief Standalone hardware test provider; production uses createOnnxGpuCompute(). */
|
||||
[[nodiscard]] eve::Result<std::unique_ptr<eve::tensor::OnnxCompute>> createVulkanProbeCompute();
|
||||
@@ -0,0 +1,156 @@
|
||||
#include <chrono>
|
||||
#include <cstdlib>
|
||||
#include "tensor/OnnxModel.h"
|
||||
#ifdef EVE_ONNX_ENGINE_PROVIDER
|
||||
#include "common/CrashHandler.h"
|
||||
#include "graphics/Graphics.h"
|
||||
#include "tensor/OnnxCompute.h"
|
||||
#elif defined(EVE_ONNX_VULKAN_PROBE)
|
||||
#include "VulkanProbeCompute.h"
|
||||
#endif
|
||||
|
||||
#include <cstring>
|
||||
#include <fstream>
|
||||
#include <iostream>
|
||||
#include <sstream>
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
#ifdef EVE_ONNX_ENGINE_PROVIDER
|
||||
eve::installCrashHandler();
|
||||
#endif
|
||||
std::cout.setf(std::ios::unitbuf);
|
||||
if (argc < 2) {
|
||||
std::cerr << "Usage: onnx_probe MODEL [OUTPUT_NAME RAW_OUTPUT_FILE [COMMA_SEPARATED_TOKEN_IDS]]\n";
|
||||
return 2;
|
||||
}
|
||||
std::ifstream file(argv[1], std::ios::binary | std::ios::ate);
|
||||
if (!file || file.tellg() <= 0 || file.tellg() > 512 * 1024 * 1024) return 2;
|
||||
std::vector<uint8_t> bytes(static_cast<size_t>(file.tellg()));
|
||||
file.seekg(0);
|
||||
if (!file.read(reinterpret_cast<char*>(bytes.data()), bytes.size())) return 2;
|
||||
const auto importStart = std::chrono::steady_clock::now();
|
||||
auto imported = eve::tensor::OnnxModel::load(bytes);
|
||||
std::cout << "ONNX_IMPORT_TIME ms="
|
||||
<< std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - importStart).count()
|
||||
<< '\n';
|
||||
if (!imported.ok()) {
|
||||
std::cerr << imported.error()->message() << '\n';
|
||||
return 1;
|
||||
}
|
||||
auto model = std::move(imported.value());
|
||||
const auto info = model->info();
|
||||
std::cout << "ONNX_IMPORT nodes=" << info.nodeCount << " packed_bytes=" << info.initializerBytes
|
||||
<< " unsupported_nodes=" << info.unsupportedNodes.size() << '\n';
|
||||
if (argc < 4) {
|
||||
for (const auto& node : info.unsupportedNodes) std::cout << node << '\n';
|
||||
return 0;
|
||||
}
|
||||
std::vector<eve::tensor::OnnxNamedTensor> feeds;
|
||||
if (argc > 4) {
|
||||
std::vector<int64_t> ids;
|
||||
std::istringstream text(argv[4]);
|
||||
std::string item;
|
||||
try {
|
||||
while (std::getline(text, item, ',')) ids.push_back(std::stoll(item));
|
||||
} catch (const std::exception&) {
|
||||
return 2;
|
||||
}
|
||||
eve::tensor::OnnxTensor tokens{eve::tensor::OnnxElement::Int64,
|
||||
{1, static_cast<int64_t>(ids.size())},
|
||||
std::vector<uint8_t>(ids.size() * 8)};
|
||||
if (!ids.empty()) std::memcpy(tokens.bytes.data(), ids.data(), tokens.bytes.size());
|
||||
feeds.push_back({"tokens", std::move(tokens)});
|
||||
}
|
||||
if (argc > 5) {
|
||||
eve::tensor::OnnxTensor style{eve::tensor::OnnxElement::Float32, {1, 256}, std::vector<uint8_t>(1024)};
|
||||
std::ifstream voice(argv[5], std::ios::binary | std::ios::ate);
|
||||
if (!voice || voice.tellg() != 1024) {
|
||||
std::cerr << "Style file must contain 256 float32 values\n";
|
||||
return 2;
|
||||
}
|
||||
voice.seekg(0);
|
||||
if (!voice.read(reinterpret_cast<char*>(style.bytes.data()), 1024)) return 2;
|
||||
feeds.push_back({"style", std::move(style)});
|
||||
eve::tensor::OnnxTensor speed{eve::tensor::OnnxElement::Float32, {1}, std::vector<uint8_t>(4)};
|
||||
const float rate = 1;
|
||||
std::memcpy(speed.bytes.data(), &rate, 4);
|
||||
feeds.push_back({"speed", std::move(speed)});
|
||||
}
|
||||
const std::vector<std::string> outputs{argv[2]};
|
||||
size_t dispatches = 0;
|
||||
const bool gpu = std::getenv("EVE_ONNX_GPU") && std::string(std::getenv("EVE_ONNX_GPU")) == "1";
|
||||
eve::tensor::OnnxRunOptions options;
|
||||
options.requireFinite = std::getenv("EVE_ONNX_CHECK_FINITE") != nullptr;
|
||||
auto execute = [&]() -> eve::Result<std::vector<eve::tensor::OnnxNamedTensor>> {
|
||||
if (!gpu) return model->run(feeds, outputs, options);
|
||||
#if defined(EVE_ONNX_VULKAN_PROBE) || defined(EVE_ONNX_ENGINE_PROVIDER)
|
||||
#ifdef EVE_ONNX_ENGINE_PROVIDER
|
||||
auto unavailable = eve::tensor::createOnnxGpuCompute();
|
||||
if (unavailable.ok())
|
||||
return eve::Result<std::vector<eve::tensor::OnnxNamedTensor>>::failure(eve::Diagnostic::error(
|
||||
eve::DiagnosticCode::Failed, "Unexpected device before headless initialization"));
|
||||
std::cout << "ONNX_GPU_ABSENT_PASS\n";
|
||||
auto* graphics = eve::graphics::Graphics::create();
|
||||
graphics->initHeadless(64, 64);
|
||||
uint32_t compilerWorkers = 4;
|
||||
if (const auto* value = std::getenv("EVE_ONNX_COMPILER_WORKERS"))
|
||||
compilerWorkers = static_cast<uint32_t>(std::stoul(value));
|
||||
auto provider = eve::tensor::createOnnxGpuCompute(compilerWorkers);
|
||||
#else
|
||||
auto provider = createVulkanProbeCompute();
|
||||
#endif
|
||||
if (!provider.ok()) return eve::Result<std::vector<eve::tensor::OnnxNamedTensor>>::failure(provider.status());
|
||||
auto runOnce = [&](int iteration) {
|
||||
const auto runStart = std::chrono::steady_clock::now();
|
||||
auto result = model->runGpu(feeds, *provider.value(), outputs, options);
|
||||
std::cout << "ONNX_ITERATION index=" << iteration << " ms="
|
||||
<< std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - runStart).count();
|
||||
if (result.ok())
|
||||
std::cout << " compilations=" << result.value().transfers.shaderCompilations
|
||||
<< " compiler_peak_workers=" << result.value().transfers.compilerPeakWorkers
|
||||
<< " cache_hits=" << result.value().transfers.shaderCacheHits
|
||||
<< " upload_bytes=" << result.value().transfers.uploadedBytes;
|
||||
std::cout << '\n';
|
||||
return result;
|
||||
};
|
||||
auto result = runOnce(0);
|
||||
if (const auto* repeat = std::getenv("EVE_ONNX_REPEAT"); repeat && std::string(repeat) == "2") {
|
||||
if (!result.ok()) return eve::Result<std::vector<eve::tensor::OnnxNamedTensor>>::failure(result.status());
|
||||
auto previous = result.value().outputs;
|
||||
result = runOnce(1);
|
||||
if (result.ok()) {
|
||||
if (previous.size() != result.value().outputs.size())
|
||||
return eve::Result<std::vector<eve::tensor::OnnxNamedTensor>>::failure(
|
||||
eve::Diagnostic::error(eve::DiagnosticCode::Failed, "Repeat output count mismatch"));
|
||||
for (size_t i = 0; i < previous.size(); ++i)
|
||||
if (previous[i].tensor.bytes != result.value().outputs[i].tensor.bytes)
|
||||
return eve::Result<std::vector<eve::tensor::OnnxNamedTensor>>::failure(
|
||||
eve::Diagnostic::error(eve::DiagnosticCode::Failed, "Repeat output bytes differ"));
|
||||
std::cout << "ONNX_REPEAT_IDENTICAL_PASS\n";
|
||||
}
|
||||
}
|
||||
if (!result.ok()) return eve::Result<std::vector<eve::tensor::OnnxNamedTensor>>::failure(result.status());
|
||||
dispatches = result.value().dispatches;
|
||||
const auto& t = result.value().transfers;
|
||||
std::cout << "ONNX_TRANSFERS uploads=" << t.uploads << " downloads=" << t.downloads
|
||||
<< " submissions=" << t.submissions << " upload_bytes=" << t.uploadedBytes
|
||||
<< " download_bytes=" << t.downloadedBytes << " allocations=" << t.bufferAllocations
|
||||
<< " reuses=" << t.bufferReuses << " compilations=" << t.shaderCompilations
|
||||
<< " cache_hits=" << t.shaderCacheHits << '\n';
|
||||
return eve::Result<std::vector<eve::tensor::OnnxNamedTensor>>::success(std::move(result.value().outputs));
|
||||
#else
|
||||
return eve::Result<std::vector<eve::tensor::OnnxNamedTensor>>::failure(eve::Diagnostic::error(
|
||||
eve::DiagnosticCode::Unsupported, "Build EVE_ONNX_VULKAN_PROBE to enable GPU parity"));
|
||||
#endif
|
||||
};
|
||||
auto generated = execute();
|
||||
if (!generated.ok()) {
|
||||
std::cerr << generated.error()->path() << ": " << generated.error()->message() << '\n';
|
||||
return 1;
|
||||
}
|
||||
const auto& output = generated.value().front().tensor;
|
||||
std::ofstream target(argv[3], std::ios::binary);
|
||||
if (!target.write(reinterpret_cast<const char*>(output.bytes.data()), output.bytes.size())) return 2;
|
||||
std::cout << "ONNX_RUN_PASS backend=" << (gpu ? "vulkan_quantized" : "engine_cpu") << " dispatches=" << dispatches
|
||||
<< " bytes=" << output.bytes.size() << '\n';
|
||||
}
|
||||
@@ -348,7 +348,7 @@ POINTER_API = re.compile(
|
||||
r"^\s*(?:(?:\[\[nodiscard(?:\([^]]*\))?\]\]\s*)?"
|
||||
r"(?:static|virtual|inline|constexpr|explicit|const)\s+)*"
|
||||
r"(?:[A-Za-z_]\w*(?:::[A-Za-z_]\w*)*(?:\s*<[^;{}()]+>)?\s*)?\*\s*"
|
||||
r"(?:[A-Za-z_]\w*)?\s*(?:\(|;|=)"
|
||||
r"[A-Za-z_]\w*\s*(?:\(|;|=)"
|
||||
)
|
||||
# A bool query is a legitimate API shape when its name follows the project's
|
||||
# query vocabulary. Keep this list deliberately lexical: this checker does
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Exercise native Kokoro INT8 inference with explicit phonemes and a selected voice.
|
||||
|
||||
Python/ONNX/NumPy prepare test inputs and WAV output only. Inference runs in the
|
||||
native probe, without ONNX Runtime. Default phonemes say Chinese 'ni hao'.
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import subprocess
|
||||
import time
|
||||
import wave
|
||||
|
||||
import numpy as np
|
||||
import onnx
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--model', type=Path, required=True)
|
||||
parser.add_argument('--voices', type=Path, required=True)
|
||||
parser.add_argument('--tokens', type=Path, required=True)
|
||||
parser.add_argument('--probe', type=Path, required=True)
|
||||
parser.add_argument('--work', type=Path, required=True)
|
||||
parser.add_argument('--gpu', action='store_true')
|
||||
parser.add_argument('--repeat', type=int, choices=[1, 2], default=1)
|
||||
parser.add_argument('--speaker', type=int, default=3)
|
||||
parser.add_argument('--phonemes', default='ㄋ ㄧ 2 ㄏ ㄠ 3 .')
|
||||
args = parser.parse_args()
|
||||
args.work.mkdir(parents=True, exist_ok=True)
|
||||
model = onnx.load(args.model, load_external_data=False)
|
||||
metadata = {entry.key: entry.value for entry in model.metadata_props}
|
||||
dims = [int(d) for d in metadata['style_dim'].split(',')]
|
||||
speakers = int(metadata['n_speakers'])
|
||||
if dims[1:] != [1, 256] or not 0 <= args.speaker < speakers:
|
||||
raise ValueError('Unsupported voice shape or speaker index')
|
||||
vocab = {line.rsplit(' ', 1)[0]: int(line.rsplit(' ', 1)[1])
|
||||
for line in args.tokens.read_text(encoding='utf-8').splitlines()}
|
||||
phones = args.phonemes.split()
|
||||
if not 0 < len(phones) < dims[0]:
|
||||
raise ValueError('Phoneme count exceeds the voice table')
|
||||
ids = [0] + [vocab[p] for p in phones] + [0]
|
||||
voices = np.memmap(args.voices, mode='r', dtype='<f4')
|
||||
if voices.size != speakers * dims[0] * dims[2]:
|
||||
raise ValueError('Voice table size does not match model metadata')
|
||||
# sherpa-onnx indexes style by phoneme length, excluding the two boundary zeros.
|
||||
style = voices.reshape(speakers, dims[0], dims[2])[args.speaker, len(phones)]
|
||||
style_path = args.work / 'style.f32'
|
||||
style.tofile(style_path)
|
||||
raw = args.work / 'audio.f32'
|
||||
env = os.environ.copy()
|
||||
env['EVE_ONNX_GPU'] = '1' if args.gpu else '0'
|
||||
env.pop('EVE_ONNX_CHECK_FINITE', None)
|
||||
env['EVE_ONNX_REPEAT'] = str(args.repeat)
|
||||
started = time.perf_counter()
|
||||
run = subprocess.run([str(args.probe.resolve()), str(args.model.resolve()), 'audio',
|
||||
str(raw.resolve()), ','.join(map(str, ids)), str(style_path.resolve())],
|
||||
env=env, capture_output=True, text=True, encoding='utf-8', errors='replace')
|
||||
seconds = time.perf_counter() - started
|
||||
(args.work / 'native.log').write_text(run.stdout + run.stderr, encoding='utf-8')
|
||||
if run.returncode or 'ONNX_RUN_PASS' not in run.stdout:
|
||||
raise RuntimeError('Native inference failed; see ' + str(args.work / 'native.log'))
|
||||
dispatches = int(re.search(r'dispatches=(\d+)', run.stdout).group(1))
|
||||
if args.gpu and dispatches == 0:
|
||||
raise RuntimeError('GPU run completed without any GPU dispatch')
|
||||
audio = np.fromfile(raw, '<f4')
|
||||
if not audio.size or not np.isfinite(audio).all() or np.max(np.abs(audio)) == 0:
|
||||
raise RuntimeError('Empty, nonfinite or silent audio output')
|
||||
sample_rate = int(metadata['sample_rate'])
|
||||
with wave.open(str(args.work / 'audio.wav'), 'wb') as wav:
|
||||
wav.setnchannels(1)
|
||||
wav.setsampwidth(2)
|
||||
wav.setframerate(sample_rate)
|
||||
wav.writeframes((np.clip(audio, -1, 1) * 32767).astype('<i2').tobytes())
|
||||
report = dict(schema='eve.tensor.kokoro-probe/1', model_sha256=hashlib.sha256(args.model.read_bytes()).hexdigest(),
|
||||
model_bytes=args.model.stat().st_size, speaker=args.speaker, phonemes=phones,
|
||||
token_ids=ids, backend='gpu' if args.gpu else 'cpu', gpu_dispatches=dispatches,
|
||||
samples=int(audio.size), sample_rate=sample_rate, peak=float(np.max(np.abs(audio))),
|
||||
inference_wall_seconds=seconds, audio_seconds=audio.size / sample_rate)
|
||||
report['iterations'] = [{key: (float(value) if key == 'ms' else int(value))
|
||||
for key, value in re.findall(r'(\w+)=([\d.]+)', line)}
|
||||
for line in re.findall(r'ONNX_ITERATION ([^\r\n]+)', run.stdout)]
|
||||
if args.repeat == 2 and ('ONNX_REPEAT_IDENTICAL_PASS' not in run.stdout or len(report['iterations']) != 2):
|
||||
raise RuntimeError('Missing repeated-inference equivalence proof')
|
||||
transfers = re.search(r'ONNX_TRANSFERS ([^\r\n]+)', run.stdout)
|
||||
if transfers:
|
||||
report['transfers'] = {key: int(value) for key, value in re.findall(r'(\w+)=(\d+)', transfers.group(1))}
|
||||
(args.work / 'result.json').write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding='utf-8')
|
||||
print('KOKORO_ONNX_PASS', args.work / 'audio.wav', f'{dispatches} GPU dispatches, {seconds:.3f}s wall time')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,151 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Compare the native tensor ONNX probe against ONNX ReferenceEvaluator/ORT.
|
||||
|
||||
Python and ONNX Runtime are test-only dependencies; the engine never links them.
|
||||
Usage: python scripts/test_tensor_onnx.py --probe PATH --work PATH [--kokoro PATH]
|
||||
"""
|
||||
import argparse
|
||||
import os
|
||||
import re
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
|
||||
import numpy as np
|
||||
import onnx
|
||||
from onnx import helper as h, numpy_helper as nh
|
||||
from onnx.reference import ReferenceEvaluator
|
||||
import onnxruntime as ort
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--probe', type=Path, required=True)
|
||||
parser.add_argument('--work', type=Path, required=True)
|
||||
parser.add_argument('--kokoro', type=Path)
|
||||
parser.add_argument('--gpu', action='store_true')
|
||||
args = parser.parse_args()
|
||||
if args.gpu:
|
||||
os.environ['EVE_ONNX_GPU'] = '1'
|
||||
else:
|
||||
os.environ.pop('EVE_ONNX_GPU', None)
|
||||
args.work.mkdir(parents=True, exist_ok=True)
|
||||
probe = str(args.probe.resolve())
|
||||
results = []
|
||||
|
||||
def check(name, nodes, values, outputs, reference='ort', tolerance=1e-6):
|
||||
initializers = [nh.from_array(np.asarray(v), k) for k, v in values.items()]
|
||||
graph = h.make_graph(nodes, name, [], [h.make_empty_tensor_value_info(o) for o in outputs], initializers)
|
||||
model = h.make_model(graph, ir_version=7, opset_imports=[h.make_opsetid('', 14), h.make_opsetid('com.microsoft', 1)])
|
||||
path = args.work / (name + '.onnx')
|
||||
onnx.save(model, path)
|
||||
if reference == 'ort':
|
||||
options = ort.SessionOptions()
|
||||
options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_DISABLE_ALL
|
||||
expected = ort.InferenceSession(model.SerializeToString(), options, providers=['CPUExecutionProvider']).run(outputs, {})
|
||||
else:
|
||||
expected = ReferenceEvaluator(model).run(outputs, {})
|
||||
for output, ref in zip(outputs, expected):
|
||||
raw = args.work / (name + '-' + output.replace('/', '_') + '.raw')
|
||||
result = subprocess.run([probe, str(path), output, str(raw)], capture_output=True, text=True)
|
||||
if result.returncode:
|
||||
raise RuntimeError(name + ': ' + result.stdout + result.stderr)
|
||||
actual = np.fromfile(raw, dtype=ref.dtype).reshape(ref.shape)
|
||||
error = float(np.max(np.abs(actual.astype(np.float64) - ref.astype(np.float64)))) if ref.size else 0
|
||||
if np.issubdtype(ref.dtype, np.integer):
|
||||
np.testing.assert_array_equal(actual, ref)
|
||||
else:
|
||||
np.testing.assert_allclose(actual, ref, atol=tolerance, rtol=tolerance)
|
||||
dispatches=int(re.search(r'dispatches=(\d+)',result.stdout).group(1))
|
||||
if args.gpu and any(n.op_type in ['MatMulInteger','ConvInteger','DynamicQuantizeLSTM','MatMul','Add','Softmax','ReduceMean'] for n in nodes):
|
||||
assert dispatches>0, (name,result.stdout)
|
||||
results.append(dict(gpu_dispatches=dispatches, backend='gpu' if args.gpu else 'cpu', case=name, output=output, max_abs_error=error, tolerance=tolerance, reference=reference))
|
||||
print(name, output, 'PASS', error)
|
||||
|
||||
for i, x in enumerate([[-1., 0, 1], [0., 0], [2., 3.], [-3., -2.], [-0.01, 0.03], [np.nan,1.,-1.], [np.nan,np.nan]]):
|
||||
check('dynamic' + str(i), [h.make_node('DynamicQuantizeLinear', ['x'], ['q','s','z'])],
|
||||
{'x': np.array(x, np.float32)}, ['q','s','z'], reference='ort')
|
||||
for sign in [False, True]:
|
||||
dtype = np.int8 if sign else np.uint8
|
||||
check('ties' + str(sign), [h.make_node('QuantizeLinear', ['x','s','z'], ['q'])],
|
||||
{'x':np.array([-1000,-1.25,-0.75,-0.25,0.25,0.75,1.25,1000],np.float32),
|
||||
's':np.array(0.5,np.float32),'z':np.array(0 if sign else 128,dtype)}, ['q'])
|
||||
check('per_axis', [h.make_node('DequantizeLinear',['q','s','z'],['y'],axis=1)],
|
||||
{'q':np.arange(24,dtype=np.uint8).reshape(2,3,4),'s':np.array([.1,.2,.3],np.float32),'z':np.array([2,5,8],np.uint8)}, ['y'])
|
||||
check('integer_exact', [h.make_node('MatMulInteger',['a','b'],['y'])],
|
||||
{'a':np.full((1,1025),255,np.uint8),'b':np.full((1025,1),127,np.int8)}, ['y'])
|
||||
check('signed_zero', [h.make_node('MatMulInteger',['a','b','az','bz'],['y'])],
|
||||
{'a':np.array([[-128,127,0]],np.int8),'b':np.array([[3,4],[5,6],[7,8]],np.uint8),'az':np.array(-3,np.int8),'bz':np.array(4,np.uint8)}, ['y'])
|
||||
check('conv_group', [h.make_node('ConvInteger',['x','w','xz','wz'],['y'],group=2,pads=[1,0,0,1],dilations=[1,2])],
|
||||
{'x':np.arange(2*4*4,dtype=np.uint8).reshape(1,2,4,4),'w':np.array(range(8),np.uint8).reshape(2,1,2,2),'xz':np.array(7,np.uint8),'wz':np.array(2,np.uint8)}, ['y'])
|
||||
check('conv1d', [h.make_node('ConvInteger',['x','w'],['y'],pads=[1,2],strides=[2])],
|
||||
{'x':np.arange(9,dtype=np.uint8).reshape(1,1,9),'w':np.array([1,2,3],np.uint8).reshape(1,1,3)}, ['y'])
|
||||
check('integer64_shape', [h.make_node('Gather',['x','i'],['y'])],
|
||||
{'x':np.array([2**60+1,-2**60-3],np.int64),'i':np.array([1,0],np.int64)}, ['y'])
|
||||
check('reshape_transpose', [h.make_node('Reshape',['x','s'],['r']), h.make_node('Transpose',['r'],['y'],perm=[1,0])],
|
||||
{'x':np.arange(12,dtype=np.float32),'s':np.array([3,-1],np.int64)}, ['y'])
|
||||
check('slice_reverse',[h.make_node('Slice',['x','s','e','a','st'],['y'])],
|
||||
{'x':np.arange(24,dtype=np.int64).reshape(2,3,4),'s':np.array([-1,-1],np.int64),'e':np.array([-100,-100],np.int64),'a':np.array([0,2],np.int64),'st':np.array([-1,-2],np.int64)},['y'])
|
||||
check('broadcast_empty',[h.make_node('Expand',['x','s'],['y'])],{'x':np.ones((1,3),np.float32),'s':np.array([0,3],np.int64)},['y'])
|
||||
check('concat',[h.make_node('Concat',['x','b'],['y'],axis=-1)],{'x':np.arange(6,dtype=np.int64).reshape(2,3),'b':np.array([[20],[30]],np.int64)},['y'])
|
||||
check('where',[h.make_node('Where',['c','a','b'],['y'])],{'c':np.array([[True],[False]]),'a':np.array([2**60+1,2**60+2],np.int64),'b':np.array(0,np.int64)},['y'])
|
||||
check('range_reverse',[h.make_node('Range',['s','e','d'],['y'])],{'s':np.array(2**60+9,np.int64),'e':np.array(2**60,np.int64),'d':np.array(-2,np.int64)},['y'],reference='reference')
|
||||
check('topk_ties',[h.make_node('TopK',['x','k'],['y','indices'],axis=0)],{'x':np.array([[2,4],[2,3],[1,5]],np.float32),'k':np.array([2],np.int64)},['y','indices'])
|
||||
check('scatter_nd',[h.make_node('ScatterND',['x','i','u'],['y'])],{'x':np.zeros((3,2),np.float32),'i':np.array([[-1],[0]],np.int64),'u':np.array([[4,5],[6,7]],np.float32)},['y'])
|
||||
check('reduce_product',[h.make_node('ReduceProd',['x'],['y'],axes=[0],keepdims=0)],{'x':np.array([[2,3],[4,5]],np.int64)},['y'])
|
||||
check('broadcast_float',[h.make_node('Add',['a','b'],['y'])],{'a':np.arange(24,dtype=np.float32).reshape(2,3,4),'b':np.array([[.1],[.2],[.3]],np.float32)},['y'])
|
||||
for name,a,b in [('batched',np.arange(24,dtype=np.float32).reshape(2,3,4)/20,np.arange(20,dtype=np.float32).reshape(4,5)/30),('vector',np.array([1.,2.,3.],np.float32),np.arange(6,dtype=np.float32).reshape(3,2)),('dot',np.array([1.,2.],np.float32),np.array([3.,4.],np.float32))]:
|
||||
check('matmul_'+name,[h.make_node('MatMul',['a','b'],['y'])],{'a':a,'b':b},['y'])
|
||||
check('pow_negative',[h.make_node('Pow',['x','p'],['y'])],{'x':np.array([-3.,-2.,-.1,0,2],np.float32),'p':np.array(2,np.float32)},['y'])
|
||||
check('reduce_mean',[h.make_node('ReduceMean',['x'],['y'],axes=[0,2],keepdims=0)],{'x':np.arange(24,dtype=np.float32).reshape(2,3,4)/7},['y'])
|
||||
check('softmax_axis',[h.make_node('Softmax',['x'],['y'],axis=1)],{'x':np.arange(24,dtype=np.float32).reshape(2,3,4)/7},['y'])
|
||||
for op in ['Reciprocal','Floor','Round','Atan','LeakyRelu','Sqrt','Exp','Log','Sin','Cos','Tanh','Sigmoid']:
|
||||
check('unary_'+op,[h.make_node(op,['x'],['y'])],{'x':np.array([.25,.5,1.,1.5,2.5,3.5],np.float32)},['y'],tolerance=2e-6)
|
||||
check('atan_infinity',[h.make_node('Atan',['x'],['y'])],{'x':np.array([-np.inf,np.inf],np.float32)},['y'],tolerance=2e-6)
|
||||
check('instance_norm',[h.make_node('InstanceNormalization',['x','s','b'],['y'],epsilon=1e-5)],{'x':np.arange(24,dtype=np.float32).reshape(2,3,4)/11,'s':np.array([1,2,3],np.float32),'b':np.array([.1,.2,.3],np.float32)},['y'],tolerance=3e-5)
|
||||
check('transpose_group',[h.make_node('ConvTranspose',['x','w','b'],['y'],group=2,strides=[2],pads=[1,1],output_padding=[1])],{'x':np.arange(12,dtype=np.float32).reshape(1,4,3)/13,'w':np.arange(24,dtype=np.float32).reshape(4,2,3)/7,'b':np.arange(4,dtype=np.float32)},['y'],tolerance=3e-6)
|
||||
for mode in ['nearest','linear']:
|
||||
check('resize_'+mode,[h.make_node('Resize',['x','','s'],['y'],mode=mode,coordinate_transformation_mode='asymmetric' if mode=='nearest' else 'half_pixel',nearest_mode='floor')],{'x':np.arange(6,dtype=np.float32).reshape(1,2,3),'s':np.array([1,1,2.5],np.float32)},['y'],tolerance=3e-6)
|
||||
for mode in ['reflect','edge','constant']:
|
||||
check('pad_'+mode,[h.make_node('Pad',['x','p'],['y'],mode=mode)],{'x':np.arange(6,dtype=np.float32).reshape(2,3),'p':np.array([0,2,0,1],np.int64)},['y'])
|
||||
for exclusive in [0,1]:
|
||||
check('cumsum_'+str(exclusive),[h.make_node('CumSum',['x','a'],['y'],exclusive=exclusive,reverse=1)],{'x':np.arange(24,dtype=np.float32).reshape(2,3,4),'a':np.array(1,np.int64)},['y'])
|
||||
check('clip',[h.make_node('Clip',['x','lo','hi'],['y'])],{'x':np.arange(6,dtype=np.float32),'lo':np.array(1.,np.float32),'hi':np.array(4.,np.float32)},['y'])
|
||||
branch=h.make_graph([h.make_node('Identity',['captured'],['branch_result'])],'branch',[],[h.make_tensor_value_info('branch_result',onnx.TensorProto.FLOAT,[2])])
|
||||
check('if_lexical_capture',[h.make_node('If',['condition'],['y'],then_branch=branch,else_branch=branch),h.make_node('Identity',['x'],['captured'])],{'condition':np.array(True),'x':np.array([2,3],np.float32)},['y'])
|
||||
body=h.make_graph([h.make_node('Identity',['condition_in'],['condition_out']),h.make_node('SequenceInsert',['state_in','iteration'],['state_out'])],'body',
|
||||
[h.make_tensor_value_info('iteration',onnx.TensorProto.INT64,[]),h.make_tensor_value_info('condition_in',onnx.TensorProto.BOOL,[]),h.make_tensor_sequence_value_info('state_in',onnx.TensorProto.INT64,[])],
|
||||
[h.make_tensor_value_info('condition_out',onnx.TensorProto.BOOL,[]),h.make_tensor_sequence_value_info('state_out',onnx.TensorProto.INT64,[])])
|
||||
check('loop_sequence',[h.make_node('SequenceEmpty',[],['empty'],dtype=onnx.TensorProto.INT64),h.make_node('Loop',['trip','condition','empty'],['sequence'],body=body),h.make_node('ConcatFromSequence',['sequence'],['y'],axis=0,new_axis=1)],{'trip':np.array(3,np.int64),'condition':np.array(True)},['y'])
|
||||
check('split_sequence',[h.make_node('SplitToSequence',['x','split'],['seq'],axis=1),h.make_node('SequenceAt',['seq','index'],['y'])],{'x':np.arange(12,dtype=np.int64).reshape(2,6),'split':np.array([2,1,3],np.int64),'index':np.array(-1,np.int64)},['y'])
|
||||
rng=np.random.default_rng(3)
|
||||
for direction in ['forward','reverse','bidirectional']:
|
||||
d=2 if direction=='bidirectional' else 1
|
||||
values={'x':rng.normal(size=(4,2,3)).astype(np.float32),
|
||||
'w':rng.integers(-60,60,(d,3,8),dtype=np.int8),'r':rng.integers(-60,60,(d,2,8),dtype=np.int8),
|
||||
'b':rng.normal(0,.1,(d,16)).astype(np.float32),
|
||||
'ws':np.full((d,8),.01,np.float32),'wz':np.zeros((d,8),np.int8),
|
||||
'rs':np.full((d,8),.01,np.float32),'rz':np.zeros((d,8),np.int8)}
|
||||
check('lstm_'+direction,[h.make_node('DynamicQuantizeLSTM',['x','w','r','b','','','','','ws','wz','rs','rz'],['y','yh','yc'],domain='com.microsoft',hidden_size=2,direction=direction)],values,['y','yh','yc'],reference='ort',tolerance=.004)
|
||||
values['lengths']=np.array([4,2],np.int32)
|
||||
values['h']=rng.normal(0,.1,(d,2,2)).astype(np.float32)
|
||||
values['c']=rng.normal(0,.1,(d,2,2)).astype(np.float32)
|
||||
check('lstm_state_'+direction,[h.make_node('DynamicQuantizeLSTM',['x','w','r','b','lengths','h','c','','ws','wz','rs','rz'],['y','yh','yc'],domain='com.microsoft',hidden_size=2,direction=direction)],values,['y','yh','yc'],reference='ort',tolerance=.004)
|
||||
if args.kokoro:
|
||||
model=onnx.load(args.kokoro)
|
||||
initializers={x.name:x for x in model.graph.initializer}
|
||||
for index, lstm in enumerate(n for n in model.graph.node if n.op_type=='DynamicQuantizeLSTM'):
|
||||
# Keep the original packed weights, scales, and node attributes.
|
||||
constants={k:nh.to_array(initializers[k]) for k in lstm.input if k in initializers}
|
||||
weight=constants[lstm.input[1]]
|
||||
constants[lstm.input[0]]=rng.normal(0,.1,(3,1,weight.shape[1])).astype(np.float32)
|
||||
for i in [4,5,6,7]:
|
||||
if i<len(lstm.input) and lstm.input[i] and lstm.input[i] not in constants:
|
||||
lstm.input[i]=''
|
||||
check('kokoro_lstm_'+str(index),[lstm],constants,list(lstm.output),reference='ort',tolerance=1e-5)
|
||||
(args.work/'results.json').write_text(json.dumps(results,indent=2),encoding='utf-8')
|
||||
print('TENSOR_ONNX_PARITY_PASS',len(results),'outputs')
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
main()
|
||||
@@ -54,6 +54,19 @@ class ArchitectureContractTests(unittest.TestCase):
|
||||
self.assertIn("last-error-channel", codes)
|
||||
self.assertIn("undocumented-raw-pointer-api", codes)
|
||||
|
||||
def test_multiply_assignment_is_not_a_pointer_api(self):
|
||||
for text in ("n *= static_cast<size_t>(d);", "n *= count(shape);"):
|
||||
with self.subTest(text=text):
|
||||
lines = [contracts.SourceLine("fixture.h", 1, text)]
|
||||
self.assertEqual([], contracts.lint_api_shapes(lines))
|
||||
|
||||
def test_named_pointer_apis_still_require_lifetime_contracts(self):
|
||||
for text in ("Widget* get();", "Widget* value = makeWidget();"):
|
||||
with self.subTest(text=text):
|
||||
lines = [contracts.SourceLine("fixture.h", 1, text)]
|
||||
codes = {finding.code for finding in contracts.lint_api_shapes(lines)}
|
||||
self.assertIn("undocumented-raw-pointer-api", codes)
|
||||
|
||||
def test_new_link_and_system_need_catalogue_coverage(self):
|
||||
metadata = {
|
||||
"entries": [
|
||||
|
||||
@@ -488,7 +488,9 @@ if(_has_gpgpu)
|
||||
# INTERFACE: EVGpgpu is an OBJECT library; its private link deps would not
|
||||
# reach the final executable, so propagate the SDK archive to consumers.
|
||||
target_link_libraries(EVGpgpu INTERFACE ${EVE_SHADERC_LIB})
|
||||
target_compile_definitions(EVGpgpu INTERFACE EVE_HAS_SHADERC=1)
|
||||
# The implementation itself selects shaderc; INTERFACE would only define
|
||||
# the macro for consumers and silently leave VulkanUtil spawning glslc.
|
||||
target_compile_definitions(EVGpgpu PRIVATE EVE_HAS_SHADERC=1)
|
||||
message(STATUS "Gpgpu: in-process shaderc compiler: ${EVE_SHADERC_LIB}")
|
||||
else()
|
||||
message(STATUS "Gpgpu: shaderc lib not found; will spawn glslc at runtime")
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
#pragma once
|
||||
#include <cstdint>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include "common/Result.h"
|
||||
|
||||
namespace eve::gpgpu {
|
||||
class ComputeShader;
|
||||
/** @brief Compile GLSL compute source to owning SPIR-V without accessing Graphics or a GPU.
|
||||
* @param source Borrowed until return; independent calls may execute concurrently on worker threads.
|
||||
* @return Owning words, compilation diagnostic, or Unsupported on WebGPU. No callbacks; each thread retains its own
|
||||
* compiler context until thread exit.
|
||||
*/
|
||||
[[nodiscard]] Result<std::vector<uint32_t>> compileComputeSpirv(const std::string& source);
|
||||
/** @brief Create an owning compute pipeline from valid SPIR-V produced by compileComputeSpirv.
|
||||
* @param words Borrowed only until return; must contain a valid compute entry point named main.
|
||||
* @return Owning shader or a diagnostic; Unsupported on WebGPU.
|
||||
* @note Device thread only, no concurrent calls/reentrancy or Graphics retirement during the call.
|
||||
* The caller must destroy the shader before the active Graphics device retires.
|
||||
*/
|
||||
[[nodiscard]] Result<std::unique_ptr<ComputeShader>> createComputeShader(const std::vector<uint32_t>& words);
|
||||
} // namespace eve::gpgpu
|
||||
@@ -1,4 +1,5 @@
|
||||
#include "gpgpu/Gpgpu.h"
|
||||
#include "gpgpu/ComputeProgram.h"
|
||||
#include "gpgpu/ComputeShader.h"
|
||||
#include "gpgpu/EcsScriptPack.h"
|
||||
#include "gpgpu/GpuBuffer.h"
|
||||
@@ -204,6 +205,36 @@ bool Gpgpu::isAvailable() const {
|
||||
#endif
|
||||
}
|
||||
|
||||
Result<std::vector<uint32_t>> compileComputeSpirv(const std::string &source) {
|
||||
#ifdef EVENGINE_WEBGPU
|
||||
return Result<std::vector<uint32_t>>::failure(
|
||||
Diagnostic::error(DiagnosticCode::Unsupported, "GLSL compilation requires Vulkan"));
|
||||
#else
|
||||
try {
|
||||
return Result<std::vector<uint32_t>>::success(compileComputeGlsl(source));
|
||||
} catch (const std::exception &e) {
|
||||
return Result<std::vector<uint32_t>>::failure(Diagnostic::error(DiagnosticCode::Failed, e.what()));
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
Result<std::unique_ptr<ComputeShader>> createComputeShader(const std::vector<uint32_t> &words) {
|
||||
#ifdef EVENGINE_WEBGPU
|
||||
return Result<std::unique_ptr<ComputeShader>>::failure(
|
||||
Diagnostic::error(DiagnosticCode::Unsupported, "SPIR-V pipelines require Vulkan"));
|
||||
#else
|
||||
try {
|
||||
if (words.size() < 5 || words.front() != 0x07230203)
|
||||
return Result<std::unique_ptr<ComputeShader>>::failure(
|
||||
Diagnostic::error(DiagnosticCode::InvalidArgument, "Invalid SPIR-V header"));
|
||||
return Result<std::unique_ptr<ComputeShader>>::success(
|
||||
std::unique_ptr<ComputeShader>(vulkanNewShaderFromSpirv(words)));
|
||||
} catch (const std::exception &e) {
|
||||
return Result<std::unique_ptr<ComputeShader>>::failure(Diagnostic::error(DiagnosticCode::Failed, e.what()));
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
ComputeShader *Gpgpu::newShader(const std::string &source) {
|
||||
#ifdef EVENGINE_WEBGPU
|
||||
if (currentGraphicsBackend() != "webgpu")
|
||||
@@ -213,7 +244,11 @@ ComputeShader *Gpgpu::newShader(const std::string &source) {
|
||||
#else
|
||||
if (currentGraphicsBackend() != "vulkan")
|
||||
throw Exception("Gpgpu.newShader: requires vulkan Graphics backend");
|
||||
return vulkanNewShaderFromSpirv(compileComputeGlsl(source));
|
||||
auto compiled = compileComputeSpirv(source);
|
||||
if (!compiled.ok()) throw Exception("%s", compiled.error()->message().c_str());
|
||||
auto shader = createComputeShader(compiled.value());
|
||||
if (!shader.ok()) throw Exception("%s", shader.error()->message().c_str());
|
||||
return shader.value().release();
|
||||
#endif
|
||||
}
|
||||
|
||||
|
||||
@@ -26,7 +26,11 @@ public:
|
||||
/** @brief True when the active Graphics backend can run compute (device initialized). */
|
||||
bool isAvailable() const;
|
||||
|
||||
/** @brief Compile compute source for the active backend (Vulkan: GLSL; WebGPU: WGSL). */
|
||||
/** @brief Compatibility-only raw-owning shader factory (Vulkan: GLSL; WebGPU: WGSL).
|
||||
* Vulkan delegates to the
|
||||
* checked compileComputeSpirv/createComputeShader APIs.
|
||||
* Device thread only; caller must delete the shader
|
||||
* before Graphics retires. */
|
||||
ComputeShader *newShader(const std::string &source);
|
||||
|
||||
/** @brief Load precompiled compute bytecode from Filesystem path (Vulkan: SPIR-V). */
|
||||
|
||||
@@ -14,7 +14,7 @@ ComputeShader *vulkanNewShaderFromSpirv(const std::vector<uint32_t> &spv) {
|
||||
auto *vkg = requireVulkanGraphics();
|
||||
auto &device = vkg->getDevice();
|
||||
|
||||
auto *shader = new VulkanComputeShader();
|
||||
auto shader = std::make_unique<VulkanComputeShader>();
|
||||
shader->device_ = &device;
|
||||
shader->module_ = vkb::PipelineBuilder::createShaderModule(device.instance, spv);
|
||||
|
||||
@@ -56,11 +56,10 @@ ComputeShader *vulkanNewShaderFromSpirv(const std::vector<uint32_t> &spv) {
|
||||
cpInfo.layout = shader->pipelineLayout_;
|
||||
auto result = device->createComputePipeline(vk::PipelineCache{}, cpInfo, device.allocation_callbacks);
|
||||
if (result.result != vk::Result::eSuccess) {
|
||||
delete shader;
|
||||
throw Exception("Gpgpu.newShader: createComputePipeline failed");
|
||||
}
|
||||
shader->pipeline_ = result.value;
|
||||
return shader;
|
||||
return shader.release();
|
||||
}
|
||||
|
||||
GpuBuffer *vulkanNewBuffer(int byteSize, const std::string &usage) {
|
||||
|
||||
@@ -75,6 +75,16 @@ void VulkanSequence::destroy() {
|
||||
(*dev)->freeCommandBuffers(pool, submittedCommandBuffer);
|
||||
submittedCommandBuffer = vk::CommandBuffer{};
|
||||
}
|
||||
// Abandoning an unsubmitted recording must retire its descriptor leases too.
|
||||
// The caller keeps shaders alive until the Sequence has been destroyed.
|
||||
for (ComputeShader *shader : usedShaders) {
|
||||
auto *vs = dynamic_cast<VulkanComputeShader *>(shader);
|
||||
if (vs) {
|
||||
vs->endSequence();
|
||||
vs->releasePendingDescriptors(*dev);
|
||||
}
|
||||
}
|
||||
usedShaders.clear();
|
||||
if (fenceReady) {
|
||||
(*dev)->destroyFence(fence, dev->allocation_callbacks);
|
||||
fence = nullptr;
|
||||
@@ -117,15 +127,16 @@ void vulkanSequenceRecordUpload(VulkanSequence *seq, GpuBuffer *dst,
|
||||
vkb::GenericBuffer *staging = nullptr;
|
||||
for (size_t i = seq->stagingUsed; i < seq->stagingPool.size(); ++i) {
|
||||
if (seq->stagingPool[i].capacity >= vk::DeviceSize(nbytes)) {
|
||||
staging = &seq->stagingPool[i];
|
||||
std::swap(seq->stagingPool[seq->stagingUsed], seq->stagingPool[i]);
|
||||
staging = &seq->stagingPool[seq->stagingUsed];
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!staging) {
|
||||
seq->stagingPool.emplace_back(device, buf::eTransferSrc, vk::DeviceSize(nbytes),
|
||||
pfb::eHostVisible | pfb::eHostCoherent);
|
||||
staging = &seq->stagingPool.back();
|
||||
std::swap(seq->stagingPool[seq->stagingUsed], seq->stagingPool.back());
|
||||
staging = &seq->stagingPool[seq->stagingUsed];
|
||||
}
|
||||
++seq->stagingUsed;
|
||||
staging->updateLocal(vkb::FrameSlot::gpuIdle(), src, vk::DeviceSize(nbytes));
|
||||
|
||||
@@ -10,6 +10,7 @@
|
||||
#include <cstdlib>
|
||||
#include <cstring>
|
||||
#include <memory>
|
||||
#include <stdexcept>
|
||||
#include <vector>
|
||||
|
||||
#if defined(_WIN32)
|
||||
@@ -24,12 +25,26 @@
|
||||
namespace eve::gpgpu {
|
||||
namespace {
|
||||
|
||||
// Compiler failures must not call Exception's global render-tracer callback:
|
||||
// this path also executes on CPU workers without a Graphics lifetime.
|
||||
std::runtime_error compileError(const char *message) {
|
||||
return std::runtime_error(message);
|
||||
}
|
||||
|
||||
template <class... Args>
|
||||
std::runtime_error compileError(const char *format, Args... args) {
|
||||
const int count = std::snprintf(nullptr, 0, format, args...);
|
||||
if (count < 0) return std::runtime_error("Compute compilation failed");
|
||||
std::string text(static_cast<size_t>(count) + 1, '\0');
|
||||
std::snprintf(text.data(), text.size(), format, args...);
|
||||
text.pop_back();
|
||||
return std::runtime_error(text);
|
||||
}
|
||||
|
||||
std::vector<uint32_t> loadSpirvBytes(const void *data, size_t size) {
|
||||
if (!data || size < 4 || (size % 4) != 0)
|
||||
throw Exception("Gpgpu SPIR-V: invalid size %zu", size);
|
||||
if (!data || size < 4 || (size % 4) != 0) throw compileError("Gpgpu SPIR-V: invalid size %zu", size);
|
||||
const auto *words = static_cast<const uint32_t *>(data);
|
||||
if (words[0] != 0x07230203)
|
||||
throw Exception("Gpgpu SPIR-V: bad magic (expected 0x07230203)");
|
||||
if (words[0] != 0x07230203) throw compileError("Gpgpu SPIR-V: bad magic (expected 0x07230203)");
|
||||
return std::vector<uint32_t>(words, words + size / 4);
|
||||
}
|
||||
|
||||
@@ -85,40 +100,54 @@ namespace {
|
||||
#if defined(EVE_HAS_SHADERC)
|
||||
/** In-process GLSL -> SPIR-V via the Vulkan SDK's static shaderc library. */
|
||||
std::vector<uint32_t> compileComputeGlslInProcess(const std::string &glsl) {
|
||||
shaderc_compiler_t compiler = shaderc_compiler_initialize();
|
||||
if (!compiler) throw Exception("Gpgpu.newShader: shaderc_compiler_initialize failed");
|
||||
shaderc_compile_options_t options = shaderc_compile_options_initialize();
|
||||
if (!options) {
|
||||
shaderc_compiler_release(compiler);
|
||||
throw Exception("Gpgpu.newShader: shaderc_compile_options_initialize failed");
|
||||
}
|
||||
shaderc_compile_options_set_target_env(options, shaderc_target_env_vulkan,
|
||||
shaderc_env_version_vulkan_1_0);
|
||||
shaderc_compile_options_set_target_spirv(options, shaderc_spirv_version_1_0);
|
||||
struct CompilerContext {
|
||||
shaderc_compiler_t compiler = shaderc_compiler_initialize();
|
||||
shaderc_compile_options_t options = nullptr;
|
||||
CompilerContext() {
|
||||
if (!compiler) throw compileError("Gpgpu.newShader: shaderc_compiler_initialize failed");
|
||||
options = shaderc_compile_options_initialize();
|
||||
if (!options) {
|
||||
shaderc_compiler_release(compiler);
|
||||
throw compileError("Gpgpu.newShader: shaderc_compile_options_initialize failed");
|
||||
}
|
||||
shaderc_compile_options_set_target_env(options, shaderc_target_env_vulkan, shaderc_env_version_vulkan_1_0);
|
||||
shaderc_compile_options_set_target_spirv(options, shaderc_spirv_version_1_0);
|
||||
}
|
||||
~CompilerContext() {
|
||||
shaderc_compile_options_release(options);
|
||||
shaderc_compiler_release(compiler);
|
||||
}
|
||||
};
|
||||
// Keep glslang's compiler/built-in state alive across jobs and graph segments.
|
||||
// Each CPU thread owns its context; no device references survive here.
|
||||
thread_local CompilerContext context;
|
||||
auto compiler = context.compiler;
|
||||
auto options = context.options;
|
||||
shaderc_compilation_result_t result =
|
||||
shaderc_compile_into_spv(compiler, glsl.data(), glsl.size(),
|
||||
shaderc_glsl_compute_shader, "eve_compute.comp", "main",
|
||||
options);
|
||||
shaderc_compile_options_release(options);
|
||||
shaderc_compiler_release(compiler);
|
||||
if (!result) throw compileError("Gpgpu.newShader: shaderc returned no compilation result");
|
||||
struct CompilationResult {
|
||||
shaderc_compilation_result_t value;
|
||||
~CompilationResult() { shaderc_result_release(value); }
|
||||
} ownedResult{result};
|
||||
if (shaderc_result_get_compilation_status(result) != shaderc_compilation_status_success) {
|
||||
const std::string err = shaderc_result_get_error_message(result);
|
||||
shaderc_result_release(result);
|
||||
throw Exception("Gpgpu.newShader: shaderc failed:\n%s", err.c_str());
|
||||
throw compileError("Gpgpu.newShader: shaderc failed:\n%s", err.c_str());
|
||||
}
|
||||
const size_t len = shaderc_result_get_length(result);
|
||||
const char *bytes = shaderc_result_get_bytes(result);
|
||||
if (len == 0 || len % 4 != 0) {
|
||||
shaderc_result_release(result);
|
||||
throw Exception("Gpgpu.newShader: shaderc returned invalid SPIR-V");
|
||||
throw compileError("Gpgpu.newShader: shaderc returned invalid SPIR-V");
|
||||
}
|
||||
std::vector<uint32_t> spv(len / 4);
|
||||
std::memcpy(spv.data(), bytes, len);
|
||||
shaderc_result_release(result);
|
||||
return spv;
|
||||
}
|
||||
#endif // EVE_HAS_SHADERC
|
||||
|
||||
#if !defined(EVE_HAS_SHADERC)
|
||||
/** Locate a usable glslc.exe: VULKAN_SDK, common install roots, then PATH. */
|
||||
std::string findGlslc() {
|
||||
if (const char *sdk = std::getenv("VULKAN_SDK"); sdk && *sdk) {
|
||||
@@ -157,37 +186,45 @@ std::string findGlslc() {
|
||||
return {};
|
||||
}
|
||||
|
||||
#endif // !EVE_HAS_SHADERC
|
||||
} // namespace
|
||||
#endif
|
||||
|
||||
std::vector<uint32_t> compileComputeGlsl(const std::string &glsl) {
|
||||
if (glsl.empty()) throw Exception("Gpgpu.newShader: empty GLSL");
|
||||
if (glsl.empty()) throw compileError("Gpgpu.newShader: empty GLSL");
|
||||
#if defined(_WIN32)
|
||||
#if defined(EVE_HAS_SHADERC)
|
||||
try {
|
||||
return compileComputeGlslInProcess(glsl);
|
||||
} catch (...) {
|
||||
// Fall through to spawning glslc (SDK lib present but compile failed).
|
||||
}
|
||||
#endif
|
||||
// Compile errors remain errors; a packaged shaderc build never needs an SDK executable.
|
||||
return compileComputeGlslInProcess(glsl);
|
||||
#else
|
||||
const std::string glslc = findGlslc();
|
||||
if (glslc.empty())
|
||||
throw Exception("Gpgpu.newShader: glslc not found on Windows "
|
||||
"(install the Vulkan SDK or set VULKAN_SDK)");
|
||||
throw compileError(
|
||||
"Gpgpu.newShader: glslc not found on Windows "
|
||||
"(install the Vulkan SDK or set VULKAN_SDK)");
|
||||
|
||||
char tmpDir[MAX_PATH];
|
||||
if (GetTempPathA(MAX_PATH, tmpDir) == 0)
|
||||
throw Exception("Gpgpu.newShader: GetTempPath failed");
|
||||
if (GetTempPathA(MAX_PATH, tmpDir) == 0) throw compileError("Gpgpu.newShader: GetTempPath failed");
|
||||
char inPath[MAX_PATH];
|
||||
if (GetTempFileNameA(tmpDir, "eve", 0, inPath) == 0)
|
||||
throw Exception("Gpgpu.newShader: GetTempFileName failed");
|
||||
if (GetTempFileNameA(tmpDir, "eve", 0, inPath) == 0) throw compileError("Gpgpu.newShader: GetTempFileName failed");
|
||||
std::string outPath = std::string(inPath) + ".spv";
|
||||
const std::string errPath = outPath + ".err";
|
||||
struct TemporaryFiles {
|
||||
const char *input;
|
||||
const std::string &output;
|
||||
const std::string &error;
|
||||
~TemporaryFiles() {
|
||||
DeleteFileA(error.c_str());
|
||||
DeleteFileA(output.c_str());
|
||||
// Reserve the unique basename until both sidecars are consumed.
|
||||
DeleteFileA(input);
|
||||
}
|
||||
} temporary{inPath, outPath, errPath};
|
||||
|
||||
{
|
||||
FILE *f = nullptr;
|
||||
if (fopen_s(&f, inPath, "wb") != 0 || !f) {
|
||||
DeleteFileA(inPath);
|
||||
throw Exception("Gpgpu.newShader: failed to write temp GLSL");
|
||||
throw compileError("Gpgpu.newShader: failed to write temp GLSL");
|
||||
}
|
||||
fwrite(glsl.data(), 1, glsl.size(), f);
|
||||
fclose(f);
|
||||
@@ -196,64 +233,79 @@ std::vector<uint32_t> compileComputeGlsl(const std::string &glsl) {
|
||||
// Spawn glslc directly with CreateProcess: _popen routes through cmd.exe
|
||||
// /c whose quote-stripping rules mangle commands starting with a quoted
|
||||
// program path. Capture stderr/stdout into a sidecar file.
|
||||
const std::string errPath = outPath + ".err";
|
||||
std::string cmd = "\"" + glslc + "\" -fshader-stage=comp \"" + inPath + "\" -o \"" +
|
||||
outPath + "\"";
|
||||
STARTUPINFOA si{};
|
||||
si.cb = sizeof(si);
|
||||
STARTUPINFOEXA si{};
|
||||
si.StartupInfo.cb = sizeof(si);
|
||||
PROCESS_INFORMATION pi{};
|
||||
SECURITY_ATTRIBUTES sa{};
|
||||
sa.nLength = sizeof(sa);
|
||||
sa.bInheritHandle = TRUE;
|
||||
HANDLE errFile = CreateFileA(errPath.c_str(), GENERIC_WRITE, FILE_SHARE_READ, &sa,
|
||||
CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, nullptr);
|
||||
if (errFile != INVALID_HANDLE_VALUE) {
|
||||
si.dwFlags = STARTF_USESTDHANDLES;
|
||||
si.hStdInput = GetStdHandle(STD_INPUT_HANDLE);
|
||||
si.hStdOutput = errFile;
|
||||
si.hStdError = errFile;
|
||||
HANDLE errFile = CreateFileA(errPath.c_str(), GENERIC_WRITE, FILE_SHARE_READ | FILE_SHARE_DELETE, &sa,
|
||||
CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, nullptr);
|
||||
HANDLE input = CreateFileA("NUL", GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, &sa, OPEN_EXISTING,
|
||||
FILE_ATTRIBUTE_NORMAL, nullptr);
|
||||
if (errFile == INVALID_HANDLE_VALUE || input == INVALID_HANDLE_VALUE) {
|
||||
if (errFile != INVALID_HANDLE_VALUE) CloseHandle(errFile);
|
||||
if (input != INVALID_HANDLE_VALUE) CloseHandle(input);
|
||||
throw compileError("Gpgpu.newShader: failed to create compiler standard handles");
|
||||
}
|
||||
char *mutableCmd = _strdup(cmd.c_str());
|
||||
struct StandardHandles {
|
||||
HANDLE output, input;
|
||||
~StandardHandles() {
|
||||
CloseHandle(output);
|
||||
CloseHandle(input);
|
||||
}
|
||||
} handles{errFile, input};
|
||||
// Without an explicit list, concurrent CreateProcess calls inherit other
|
||||
// jobs' temporary-file handles and can prevent glslc from opening its input.
|
||||
SIZE_T attributeBytes = 0;
|
||||
InitializeProcThreadAttributeList(nullptr, 1, 0, &attributeBytes);
|
||||
std::vector<uint8_t> attributes(attributeBytes);
|
||||
si.lpAttributeList = reinterpret_cast<LPPROC_THREAD_ATTRIBUTE_LIST>(attributes.data());
|
||||
if (!InitializeProcThreadAttributeList(si.lpAttributeList, 1, 0, &attributeBytes))
|
||||
throw compileError("Gpgpu.newShader: failed to initialize process attributes");
|
||||
struct AttributeList {
|
||||
LPPROC_THREAD_ATTRIBUTE_LIST value;
|
||||
~AttributeList() { DeleteProcThreadAttributeList(value); }
|
||||
} attributeList{si.lpAttributeList};
|
||||
HANDLE inherited[] = {errFile, input};
|
||||
if (!UpdateProcThreadAttribute(si.lpAttributeList, 0, PROC_THREAD_ATTRIBUTE_HANDLE_LIST, inherited,
|
||||
sizeof(inherited), nullptr, nullptr))
|
||||
throw compileError("Gpgpu.newShader: failed to restrict compiler handle inheritance");
|
||||
si.StartupInfo.dwFlags = STARTF_USESTDHANDLES;
|
||||
si.StartupInfo.hStdInput = input;
|
||||
si.StartupInfo.hStdOutput = errFile;
|
||||
si.StartupInfo.hStdError = errFile;
|
||||
const BOOL spawned =
|
||||
CreateProcessA(glslc.c_str(), mutableCmd, nullptr, nullptr, TRUE, CREATE_NO_WINDOW,
|
||||
nullptr, nullptr, &si, &pi);
|
||||
free(mutableCmd);
|
||||
if (errFile != INVALID_HANDLE_VALUE) CloseHandle(errFile);
|
||||
CreateProcessA(glslc.c_str(), cmd.data(), nullptr, nullptr, TRUE,
|
||||
CREATE_NO_WINDOW | EXTENDED_STARTUPINFO_PRESENT, nullptr, nullptr, &si.StartupInfo, &pi);
|
||||
if (!spawned) {
|
||||
const DWORD errCode = GetLastError();
|
||||
DeleteFileA(inPath);
|
||||
DeleteFileA(outPath.c_str());
|
||||
DeleteFileA(errPath.c_str());
|
||||
throw Exception("Gpgpu.newShader: failed to launch glslc (error %lu)", errCode);
|
||||
throw compileError("Gpgpu.newShader: failed to launch glslc (error %lu)", errCode);
|
||||
}
|
||||
WaitForSingleObject(pi.hProcess, INFINITE);
|
||||
DWORD exitCode = 0;
|
||||
GetExitCodeProcess(pi.hProcess, &exitCode);
|
||||
CloseHandle(pi.hProcess);
|
||||
CloseHandle(pi.hThread);
|
||||
DeleteFileA(inPath);
|
||||
if (exitCode != 0) {
|
||||
std::string err;
|
||||
FILE *ef = nullptr;
|
||||
if (fopen_s(&ef, errPath.c_str(), "rb") == 0 && ef) {
|
||||
char buf[512];
|
||||
size_t n;
|
||||
while ((n = fread(buf, 1, sizeof(buf) - 1, ef)) > 0) {
|
||||
buf[n] = '\0';
|
||||
err += buf;
|
||||
}
|
||||
fclose(ef);
|
||||
HANDLE diagnostics =
|
||||
CreateFileA(errPath.c_str(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, nullptr,
|
||||
OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr);
|
||||
if (diagnostics != INVALID_HANDLE_VALUE) {
|
||||
char buffer[512];
|
||||
DWORD count = 0;
|
||||
while (ReadFile(diagnostics, buffer, sizeof(buffer), &count, nullptr) && count) err.append(buffer, count);
|
||||
CloseHandle(diagnostics);
|
||||
}
|
||||
DeleteFileA(outPath.c_str());
|
||||
DeleteFileA(errPath.c_str());
|
||||
throw Exception("Gpgpu.newShader: glslc failed (exit %lu):\n%s", exitCode, err.c_str());
|
||||
throw compileError("Gpgpu.newShader: glslc failed (exit %lu):\n%s", exitCode, err.c_str());
|
||||
}
|
||||
DeleteFileA(errPath.c_str());
|
||||
|
||||
FILE *f = nullptr;
|
||||
if (fopen_s(&f, outPath.c_str(), "rb") != 0 || !f) {
|
||||
DeleteFileA(outPath.c_str());
|
||||
throw Exception("Gpgpu.newShader: failed to open compiled SPIR-V");
|
||||
throw compileError("Gpgpu.newShader: failed to open compiled SPIR-V");
|
||||
}
|
||||
fseek(f, 0, SEEK_END);
|
||||
long sz = ftell(f);
|
||||
@@ -261,23 +313,22 @@ std::vector<uint32_t> compileComputeGlsl(const std::string &glsl) {
|
||||
std::vector<uint8_t> bytes(static_cast<size_t>(sz > 0 ? sz : 0));
|
||||
if (sz > 0 && fread(bytes.data(), 1, static_cast<size_t>(sz), f) != static_cast<size_t>(sz)) {
|
||||
fclose(f);
|
||||
DeleteFileA(outPath.c_str());
|
||||
throw Exception("Gpgpu.newShader: failed to read compiled SPIR-V");
|
||||
throw compileError("Gpgpu.newShader: failed to read compiled SPIR-V");
|
||||
}
|
||||
fclose(f);
|
||||
DeleteFileA(outPath.c_str());
|
||||
return loadSpirvBytes(bytes.data(), bytes.size());
|
||||
#endif // EVE_HAS_SHADERC
|
||||
#else
|
||||
char inPath[] = "/tmp/eve_comp_XXXXXX";
|
||||
int fd = mkstemp(inPath);
|
||||
if (fd < 0) throw Exception("Gpgpu.newShader: mkstemp failed");
|
||||
if (fd < 0) throw compileError("Gpgpu.newShader: mkstemp failed");
|
||||
std::string outPath = std::string(inPath) + ".spv";
|
||||
{
|
||||
ssize_t n = write(fd, glsl.data(), glsl.size());
|
||||
close(fd);
|
||||
if (n < 0 || size_t(n) != glsl.size()) {
|
||||
unlink(inPath);
|
||||
throw Exception("Gpgpu.newShader: failed to write temp GLSL");
|
||||
throw compileError("Gpgpu.newShader: failed to write temp GLSL");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -292,17 +343,17 @@ std::vector<uint32_t> compileComputeGlsl(const std::string &glsl) {
|
||||
unlink(inPath);
|
||||
if (status != 0) {
|
||||
unlink(outPath.c_str());
|
||||
throw Exception("Gpgpu.newShader: glslc failed:\n%s", err.c_str());
|
||||
throw compileError("Gpgpu.newShader: glslc failed:\n%s", err.c_str());
|
||||
}
|
||||
} else {
|
||||
unlink(inPath);
|
||||
throw Exception("Gpgpu.newShader: glslc not available (popen failed)");
|
||||
throw compileError("Gpgpu.newShader: glslc not available (popen failed)");
|
||||
}
|
||||
|
||||
FILE *f = fopen(outPath.c_str(), "rb");
|
||||
if (!f) {
|
||||
unlink(outPath.c_str());
|
||||
throw Exception("Gpgpu.newShader: failed to open compiled SPIR-V");
|
||||
throw compileError("Gpgpu.newShader: failed to open compiled SPIR-V");
|
||||
}
|
||||
fseek(f, 0, SEEK_END);
|
||||
long sz = ftell(f);
|
||||
@@ -311,7 +362,7 @@ std::vector<uint32_t> compileComputeGlsl(const std::string &glsl) {
|
||||
if (sz > 0 && fread(bytes.data(), 1, static_cast<size_t>(sz), f) != static_cast<size_t>(sz)) {
|
||||
fclose(f);
|
||||
unlink(outPath.c_str());
|
||||
throw Exception("Gpgpu.newShader: failed to read compiled SPIR-V");
|
||||
throw compileError("Gpgpu.newShader: failed to read compiled SPIR-V");
|
||||
}
|
||||
fclose(f);
|
||||
unlink(outPath.c_str());
|
||||
|
||||
@@ -31,6 +31,10 @@
|
||||
|
||||
struct aiMesh;
|
||||
|
||||
namespace eve {
|
||||
class Subscription;
|
||||
}
|
||||
|
||||
namespace eve::graphics {
|
||||
struct PbrSurface;
|
||||
|
||||
@@ -188,6 +192,13 @@ public:
|
||||
* callbacks.
|
||||
*/
|
||||
[[nodiscard]] std::weak_ptr<const void> resourceLifetime() const;
|
||||
/** @brief Register cleanup before this provider destroys its GPU device/resources.
|
||||
* @return Owning subscription, or Conflict if retirement has started.
|
||||
* @lifetime Retain the subscription until cleanup is no longer needed. Either destruction order is safe.
|
||||
* @thread Render thread only. Callback must not throw, recreate or destroy Graphics, or start GPU work.
|
||||
* @note Callbacks run without locks; disposing subscriptions during notification is supported. */
|
||||
[[nodiscard]] Result<eve::Subscription> onResourcesRetiring(std::function<void()> callback) const;
|
||||
|
||||
|
||||
/**
|
||||
* @brief Whether gbuffer-based post-process shaders (AO, GI) can be created on this
|
||||
|
||||
@@ -1,13 +1,16 @@
|
||||
#include <cstdio>
|
||||
#include <unordered_map>
|
||||
#include "common/Subscription.h"
|
||||
#include "graphics/Graphics.h"
|
||||
|
||||
namespace eve::graphics {
|
||||
namespace {
|
||||
// Render-thread-only observation registry. Erasing on destruction prevents address reuse
|
||||
// from making an old observer valid for a new Graphics instance.
|
||||
struct Lifetime {
|
||||
eve::Observer<> retiring;
|
||||
bool retired = false;
|
||||
};
|
||||
bool registryAlive = false;
|
||||
struct Registry {
|
||||
std::unordered_map<const Graphics*, std::shared_ptr<const void>> tokens;
|
||||
std::unordered_map<const Graphics*, std::shared_ptr<Lifetime>> tokens;
|
||||
Registry() { registryAlive = true; }
|
||||
~Registry() { registryAlive = false; }
|
||||
};
|
||||
@@ -15,13 +18,31 @@ auto& lifetimes() {
|
||||
static Registry registry;
|
||||
return registry.tokens;
|
||||
}
|
||||
auto lifetime(const Graphics* graphics) {
|
||||
auto& value = lifetimes()[graphics];
|
||||
if (!value) value = std::make_shared<Lifetime>();
|
||||
return value;
|
||||
}
|
||||
} // namespace
|
||||
std::weak_ptr<const void> Graphics::resourceLifetime() const {
|
||||
auto& token = lifetimes()[this];
|
||||
if (!token) token = std::make_shared<const int>(0);
|
||||
return token;
|
||||
auto value = lifetime(this);
|
||||
return value->retired ? std::weak_ptr<const void>{} : value;
|
||||
}
|
||||
Result<eve::Subscription> Graphics::onResourcesRetiring(std::function<void()> callback) const {
|
||||
auto value = lifetime(this);
|
||||
if (value->retired)
|
||||
return Result<eve::Subscription>::failure(
|
||||
Diagnostic::error(DiagnosticCode::Conflict, "Graphics resources are retiring"));
|
||||
return Result<eve::Subscription>::success(value->retiring.subscribe(std::move(callback)));
|
||||
}
|
||||
void Graphics::retireResourceLifetime() const {
|
||||
if (registryAlive) lifetimes().erase(this);
|
||||
if (!registryAlive) return;
|
||||
auto it = lifetimes().find(this);
|
||||
if (it == lifetimes().end() || it->second->retired) return;
|
||||
auto value = it->second;
|
||||
value->retired = true;
|
||||
const auto failures = value->retiring.notifyChecked([]() noexcept {});
|
||||
if (failures) std::fprintf(stderr, "Graphics resource retirement: %zu callback failures\n", failures);
|
||||
lifetimes().erase(this);
|
||||
}
|
||||
} // namespace eve::graphics
|
||||
@@ -292,6 +292,7 @@ std::string Graphics::getBackendName() const { return "vulkan"; }
|
||||
Graphics::Graphics() = default;
|
||||
|
||||
Graphics::~Graphics() {
|
||||
retireResourceLifetime();
|
||||
detachGraphicsArtifactProvider(this);
|
||||
if (!initialized) {
|
||||
// A process-level boot warmup may exist even when callers decide not
|
||||
|
||||
@@ -99,7 +99,10 @@ Graphics::Graphics() {
|
||||
}
|
||||
}
|
||||
|
||||
Graphics::~Graphics() { detachGraphicsArtifactProvider(this); }
|
||||
Graphics::~Graphics() {
|
||||
retireResourceLifetime();
|
||||
detachGraphicsArtifactProvider(this);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Init
|
||||
|
||||
@@ -0,0 +1,179 @@
|
||||
#include "tensor/AffineQuant.h"
|
||||
#include "tensor/AffineQuantInternal.h"
|
||||
#include "tensor/OnnxGpuKernels.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <cmath>
|
||||
#include <limits>
|
||||
|
||||
namespace eve::tensor::affine {
|
||||
namespace {
|
||||
template <class T>
|
||||
Result<T> fail(const char* message) {
|
||||
return Result<T>::failure(Diagnostic::error(DiagnosticCode::InvalidArgument, message));
|
||||
}
|
||||
int value(ByteView v, size_t i) {
|
||||
const int x = v.bytes[i];
|
||||
return v.signedValues && x >= 128 ? x - 256 : x;
|
||||
}
|
||||
bool validZero(int z, bool sign) { return sign ? z >= -128 && z <= 127 : z >= 0 && z <= 255; }
|
||||
double evenRound(double x) {
|
||||
const double lo = std::floor(x), fraction = x - lo;
|
||||
return lo + (fraction > 0.5 || (fraction == 0.5 && std::fmod(lo, 2.0) != 0));
|
||||
}
|
||||
bool fits(int64_t n) { return n >= INT32_MIN && n <= INT32_MAX; }
|
||||
bool product(size_t a, size_t b, size_t limit) { return b == 0 || a <= limit / b; }
|
||||
} // namespace
|
||||
|
||||
Result<std::vector<uint8_t>> quantize(std::span<const float> input, float scale, int zero, bool sign) {
|
||||
if (!(scale > 0) || !std::isfinite(scale) || !validZero(zero, sign))
|
||||
return fail<std::vector<uint8_t>>("Invalid affine scale or zero point");
|
||||
std::vector<uint8_t> out;
|
||||
out.reserve(input.size());
|
||||
for (float x : input) {
|
||||
if (!std::isfinite(x)) return fail<std::vector<uint8_t>>("Nonfinite quantization input");
|
||||
const double rounded = evenRound(static_cast<double>(x) / scale) + zero;
|
||||
const int q = static_cast<int>(std::clamp(rounded, sign ? -128.0 : 0.0, sign ? 127.0 : 255.0));
|
||||
out.push_back(static_cast<uint8_t>(q));
|
||||
}
|
||||
return Result<std::vector<uint8_t>>::success(std::move(out));
|
||||
}
|
||||
|
||||
Result<QuantizedActivation> dynamicQuantize(std::span<const float> input) {
|
||||
float lo = 0, hi = 0;
|
||||
for (float x : input) {
|
||||
if (!std::isfinite(x)) return fail<QuantizedActivation>("Nonfinite dynamic quantization input");
|
||||
lo = std::min(lo, x);
|
||||
hi = std::max(hi, x);
|
||||
}
|
||||
QuantizedActivation result;
|
||||
result.scale = hi == lo ? 1.f : static_cast<float>((static_cast<double>(hi) - lo) / 255.0);
|
||||
if (!(result.scale > 0) || !std::isfinite(result.scale))
|
||||
return fail<QuantizedActivation>("Dynamic quantization scale is not representable");
|
||||
result.zeroPoint = static_cast<uint8_t>(std::clamp(evenRound(-static_cast<double>(lo) / result.scale), 0.0, 255.0));
|
||||
auto bytes = quantize(input, result.scale, result.zeroPoint, false);
|
||||
if (!bytes.ok()) return Result<QuantizedActivation>::failure(bytes.status());
|
||||
result.values = std::move(bytes.value());
|
||||
return Result<QuantizedActivation>::success(std::move(result));
|
||||
}
|
||||
|
||||
Result<std::vector<float>> dequantize(ByteView input, std::span<const float> scales, std::span<const int32_t> zeros,
|
||||
size_t inner) {
|
||||
if (scales.empty() || scales.size() != zeros.size() || inner == 0 || !product(scales.size(), inner, SIZE_MAX) ||
|
||||
input.bytes.size() % (scales.size() * inner))
|
||||
return fail<std::vector<float>>("Invalid affine channel layout");
|
||||
for (size_t i = 0; i < scales.size(); ++i)
|
||||
if (!(scales[i] > 0) || !std::isfinite(scales[i]) || !validZero(zeros[i], input.signedValues))
|
||||
return fail<std::vector<float>>("Invalid affine scale or zero point");
|
||||
std::vector<float> out(input.bytes.size());
|
||||
for (size_t i = 0; i < out.size(); ++i) {
|
||||
const size_t channel = (i / inner) % scales.size();
|
||||
out[i] = static_cast<float>(value(input, i) - zeros[channel]) * scales[channel];
|
||||
}
|
||||
return Result<std::vector<float>>::success(std::move(out));
|
||||
}
|
||||
|
||||
Result<std::vector<int32_t>> matmul(ByteView a, ByteView b, size_t m, size_t k, size_t n, int aZero, int bZero,
|
||||
OnnxCompute* compute) {
|
||||
if (!validZero(aZero, a.signedValues) || !validZero(bZero, b.signedValues) || !product(m, k, SIZE_MAX) ||
|
||||
!product(k, n, SIZE_MAX) || !product(m, n, INT32_MAX) || a.bytes.size() != m * k || b.bytes.size() != k * n)
|
||||
return fail<std::vector<int32_t>>("Invalid integer matmul dimensions or zero points");
|
||||
if (compute && m && n && k) {
|
||||
try {
|
||||
const int32_t z = bZero;
|
||||
return Result<std::vector<int32_t>>::success(
|
||||
onnx_detail::gpuMatmul(*compute, a, b, m, k, n, aZero, {&z, 1}));
|
||||
} catch (const std::exception& e) {
|
||||
return Result<std::vector<int32_t>>::failure(Diagnostic::error(DiagnosticCode::Failed, e.what()));
|
||||
}
|
||||
}
|
||||
std::vector<int32_t> out(m * n);
|
||||
for (size_t r = 0; r < m; ++r)
|
||||
for (size_t c = 0; c < n; ++c) {
|
||||
int64_t sum = 0;
|
||||
for (size_t j = 0; j < k; ++j)
|
||||
sum += static_cast<int64_t>(value(a, r * k + j) - aZero) * (value(b, j * n + c) - bZero);
|
||||
if (!fits(sum)) return fail<std::vector<int32_t>>("Integer matmul accumulator overflow");
|
||||
out[r * n + c] = static_cast<int32_t>(sum);
|
||||
}
|
||||
return Result<std::vector<int32_t>>::success(std::move(out));
|
||||
}
|
||||
|
||||
Result<detail::ConvExtent> detail::validateConv(size_t xSize, size_t wSize, bool xSigned, bool wSigned,
|
||||
const ConvShape& s, int xZero, std::span<const int32_t> wZeros) {
|
||||
for (int d : {s.batch, s.channels, s.height, s.width, s.outputs, s.kernelH, s.kernelW, s.strideH, s.strideW,
|
||||
s.dilationH, s.dilationW, s.groups})
|
||||
if (d <= 0 || d > 65536) return fail<detail::ConvExtent>("Invalid integer convolution dimensions");
|
||||
if (s.channels % s.groups || s.outputs % s.groups || s.padTop < 0 || s.padBottom < 0 || s.padLeft < 0 ||
|
||||
s.padRight < 0 || !validZero(xZero, xSigned) ||
|
||||
(wZeros.size() != 1 && wZeros.size() != static_cast<size_t>(s.outputs)))
|
||||
return fail<detail::ConvExtent>("Invalid convolution groups, padding or zero points");
|
||||
for (int z : wZeros)
|
||||
if (!validZero(z, wSigned)) return fail<detail::ConvExtent>("Invalid convolution weight zero point");
|
||||
const int64_t h = static_cast<int64_t>(s.height) + s.padTop + s.padBottom -
|
||||
static_cast<int64_t>(s.dilationH) * (s.kernelH - 1) - 1;
|
||||
const int64_t width = static_cast<int64_t>(s.width) + s.padLeft + s.padRight -
|
||||
static_cast<int64_t>(s.dilationW) * (s.kernelW - 1) - 1;
|
||||
if (h < 0 || width < 0) return fail<detail::ConvExtent>("Convolution kernel exceeds padded input");
|
||||
const int64_t oh = h / s.strideH + 1, ow = width / s.strideW + 1;
|
||||
int64_t outCount = 1;
|
||||
for (int64_t d : {static_cast<int64_t>(s.batch), static_cast<int64_t>(s.outputs), oh, ow}) {
|
||||
if (d > INT32_MAX / outCount) return fail<detail::ConvExtent>("Convolution output too large");
|
||||
outCount *= d;
|
||||
}
|
||||
const int cg = s.channels / s.groups;
|
||||
size_t inputCount = 1, weightCount = 1;
|
||||
for (int d : {s.batch, s.channels, s.height, s.width}) {
|
||||
if (!product(inputCount, d, INT32_MAX)) return fail<detail::ConvExtent>("Convolution input too large");
|
||||
inputCount *= d;
|
||||
}
|
||||
for (int d : {s.outputs, cg, s.kernelH, s.kernelW}) {
|
||||
if (!product(weightCount, d, INT32_MAX)) return fail<detail::ConvExtent>("Convolution weights too large");
|
||||
weightCount *= d;
|
||||
}
|
||||
if (outCount > INT32_MAX || xSize != inputCount || wSize != weightCount)
|
||||
return fail<detail::ConvExtent>("Convolution buffer size mismatch or output too large");
|
||||
return Result<detail::ConvExtent>::success({oh, ow, outCount});
|
||||
}
|
||||
Result<std::vector<int32_t>> conv(ByteView x, ByteView w, const ConvShape& s, int xZero,
|
||||
std::span<const int32_t> wZeros, OnnxCompute* compute) {
|
||||
auto extent =
|
||||
detail::validateConv(x.bytes.size(), w.bytes.size(), x.signedValues, w.signedValues, s, xZero, wZeros);
|
||||
if (!extent.ok()) return Result<std::vector<int32_t>>::failure(extent.status());
|
||||
const auto [oh, ow, outCount] = extent.value();
|
||||
const int cg = s.channels / s.groups;
|
||||
if (compute) {
|
||||
try {
|
||||
return Result<std::vector<int32_t>>::success(onnx_detail::gpuConv(*compute, x, w, s, xZero, wZeros));
|
||||
} catch (const std::exception& e) {
|
||||
return Result<std::vector<int32_t>>::failure(Diagnostic::error(DiagnosticCode::Failed, e.what()));
|
||||
}
|
||||
}
|
||||
std::vector<int32_t> out(static_cast<size_t>(outCount));
|
||||
for (int b = 0; b < s.batch; ++b)
|
||||
for (int o = 0; o < s.outputs; ++o)
|
||||
for (int64_t y = 0; y < oh; ++y)
|
||||
for (int64_t z = 0; z < ow; ++z) {
|
||||
int64_t sum = 0;
|
||||
for (int c = 0; c < cg; ++c)
|
||||
for (int ky = 0; ky < s.kernelH; ++ky)
|
||||
for (int kx = 0; kx < s.kernelW; ++kx) {
|
||||
const int64_t iy = y * s.strideH - s.padTop + static_cast<int64_t>(ky) * s.dilationH;
|
||||
const int64_t ix = z * s.strideW - s.padLeft + static_cast<int64_t>(kx) * s.dilationW;
|
||||
if (iy < 0 || ix < 0 || iy >= s.height || ix >= s.width) continue;
|
||||
const size_t xi =
|
||||
((static_cast<size_t>(b) * s.channels + (o / (s.outputs / s.groups)) * cg + c) *
|
||||
s.height +
|
||||
iy) *
|
||||
s.width +
|
||||
ix;
|
||||
const size_t wi = ((static_cast<size_t>(o) * cg + c) * s.kernelH + ky) * s.kernelW + kx;
|
||||
sum += static_cast<int64_t>(value(x, xi) - xZero) *
|
||||
(value(w, wi) - wZeros[wZeros.size() == 1 ? 0 : o]);
|
||||
}
|
||||
if (!fits(sum)) return fail<std::vector<int32_t>>("Integer convolution accumulator overflow");
|
||||
out[((static_cast<size_t>(b) * s.outputs + o) * oh + y) * ow + z] = static_cast<int32_t>(sum);
|
||||
}
|
||||
return Result<std::vector<int32_t>>::success(std::move(out));
|
||||
}
|
||||
} // namespace eve::tensor::affine
|
||||
@@ -0,0 +1,77 @@
|
||||
#pragma once
|
||||
|
||||
#include "common/Result.h"
|
||||
|
||||
#include <cstdint>
|
||||
#include <span>
|
||||
#include <vector>
|
||||
|
||||
namespace eve::tensor {
|
||||
class OnnxCompute;
|
||||
}
|
||||
namespace eve::tensor::affine {
|
||||
|
||||
/** @brief ONNX unsigned activation quantization; owns bytes and scalar parameters. */
|
||||
struct QuantizedActivation {
|
||||
std::vector<uint8_t> values;
|
||||
float scale = 1;
|
||||
uint8_t zeroPoint = 0;
|
||||
};
|
||||
|
||||
/** @brief Borrowed packed 8-bit values, valid for the duration of a synchronous call. */
|
||||
struct ByteView {
|
||||
std::span<const uint8_t> bytes;
|
||||
bool signedValues = false;
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief Quantize finite FP32 activations with ONNX DynamicQuantizeLinear semantics.
|
||||
* @param input Borrowed flat input; not retained.
|
||||
* @return Owning uint8 values, scale, zero point; InvalidArgument for nonfinite input.
|
||||
* @note CPU, thread-safe, no callbacks; nearest-even rounding independent of fenv.
|
||||
*/
|
||||
[[nodiscard]] Result<QuantizedActivation> dynamicQuantize(std::span<const float> input);
|
||||
|
||||
/**
|
||||
* @brief Affine quantization to int8/uint8 bytes, saturating and rounding ties to even.
|
||||
* @return Owning packed bytes; rejects invalid scale, zero point or nonfinite input.
|
||||
* @note CPU, thread-safe; spans are borrowed only during the call.
|
||||
*/
|
||||
[[nodiscard]] Result<std::vector<uint8_t>> quantize(std::span<const float> input, float scale, int zeroPoint,
|
||||
bool signedValues);
|
||||
|
||||
/**
|
||||
* @brief Affine dequantization using scalar or per-axis scale/zero point.
|
||||
* @param inner Number of contiguous values following the channel axis.
|
||||
* @return Owning FP32 values; InvalidArgument on inconsistent channel layout.
|
||||
* @note CPU, thread-safe; all views are borrowed for the call, no callbacks.
|
||||
*/
|
||||
[[nodiscard]] Result<std::vector<float>> dequantize(ByteView input, std::span<const float> scales,
|
||||
std::span<const int32_t> zeros, size_t inner = 1);
|
||||
|
||||
/**
|
||||
* @brief Integer row-major [M,K] x [K,N], subtracting scalar zero points.
|
||||
* @return Exact owning int32 accumulators; rejects dimension and int32 overflow.
|
||||
* @param compute Optional borrowed GPU provider; null selects CPU. No retry on GPU failure.
|
||||
* @note Synchronous. CPU is thread-safe; GPU uses the provider thread. Weights remain packed.
|
||||
*/
|
||||
[[nodiscard]] Result<std::vector<int32_t>> matmul(ByteView a, ByteView b, size_t m, size_t k, size_t n, int aZero = 0,
|
||||
int bZero = 0, OnnxCompute* compute = nullptr);
|
||||
|
||||
/** @brief Explicit 1D/2D NCHW convolution geometry; missing 1D height is one. */
|
||||
struct ConvShape {
|
||||
int batch = 1, channels = 1, height = 1, width = 1;
|
||||
int outputs = 1, kernelH = 1, kernelW = 1;
|
||||
int strideH = 1, strideW = 1, dilationH = 1, dilationW = 1;
|
||||
int padTop = 0, padLeft = 0, padBottom = 0, padRight = 0, groups = 1;
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief Integer Conv, supporting groups, asymmetric padding and dilation.
|
||||
* @return Owning NCHW int32 values; rejects bad geometry or accumulator overflow.
|
||||
* @param compute Optional borrowed GPU provider; null selects CPU. No retry on GPU failure.
|
||||
* @note Synchronous. CPU is thread-safe; GPU uses the provider thread. Padding represents real zero.
|
||||
*/
|
||||
[[nodiscard]] Result<std::vector<int32_t>> conv(ByteView x, ByteView w, const ConvShape& shape, int xZero,
|
||||
std::span<const int32_t> wZeros, OnnxCompute* compute = nullptr);
|
||||
} // namespace eve::tensor::affine
|
||||
@@ -0,0 +1,9 @@
|
||||
#pragma once
|
||||
#include "tensor/AffineQuant.h"
|
||||
namespace eve::tensor::affine::detail {
|
||||
struct ConvExtent {
|
||||
int64_t height, width, count;
|
||||
};
|
||||
Result<ConvExtent> validateConv(size_t xSize, size_t wSize, bool xSigned, bool wSigned, const ConvShape&, int,
|
||||
std::span<const int32_t>);
|
||||
} // namespace eve::tensor::affine::detail
|
||||
@@ -0,0 +1,74 @@
|
||||
#pragma once
|
||||
#include <span>
|
||||
#include <stdexcept>
|
||||
#include "tensor/OnnxModel.h"
|
||||
#include "tensor/OnnxStorage.h"
|
||||
namespace eve::tensor::onnx_detail {
|
||||
// Internal copy-on-write host view. Const reads materialize only at CPU boundaries.
|
||||
// GPU aliases share immutable storage and never force a readback.
|
||||
class ByteStorage {
|
||||
mutable OnnxBuffer buffer_;
|
||||
std::shared_ptr<std::vector<uint8_t>> writable_;
|
||||
const std::vector<uint8_t>& get() const {
|
||||
if (!buffer_.host) {
|
||||
if (!buffer_.device) throw std::runtime_error("Missing ONNX storage");
|
||||
auto r = buffer_.device->readback();
|
||||
if (!r.ok()) throw std::runtime_error(r.error()->message());
|
||||
if (r.value().size() != buffer_.size) throw std::runtime_error("GPU byte count mismatch");
|
||||
buffer_.host = std::make_shared<const std::vector<uint8_t>>(std::move(r.value()));
|
||||
}
|
||||
return *buffer_.host;
|
||||
}
|
||||
std::vector<uint8_t>& write() {
|
||||
// CPU mutation creates a new authoritative storage identity.
|
||||
if (writable_ && buffer_.host.use_count() == 2) return *writable_;
|
||||
auto v = std::make_shared<std::vector<uint8_t>>(get());
|
||||
buffer_ = {v, {}, v->size()};
|
||||
writable_ = v;
|
||||
return *v;
|
||||
}
|
||||
|
||||
public:
|
||||
ByteStorage() : ByteStorage(std::vector<uint8_t>{}) {}
|
||||
ByteStorage(std::vector<uint8_t> v) : buffer_{std::make_shared<const std::vector<uint8_t>>(std::move(v)), {}, 0} {
|
||||
buffer_.size = buffer_.host->size();
|
||||
}
|
||||
ByteStorage(OnnxBuffer b) : buffer_(std::move(b)) {}
|
||||
void retainAcrossRuns() { buffer_.persistent = true; }
|
||||
const OnnxBuffer& buffer() const { return buffer_; }
|
||||
size_t size() const { return buffer_.size; }
|
||||
bool empty() const { return !size(); }
|
||||
/** @brief Borrow CPU data until this storage is mutated or destroyed; device-thread only for lazy data.
|
||||
* @lifetime Borrowed until mutation or destruction; use the provider thread for GPU readback.
|
||||
* @throws std::runtime_error On readback failure. Internal executor converts this to Result. */
|
||||
const uint8_t* data() const { return get().data(); }
|
||||
/** @brief Borrow mutable CPU data until mutation/destruction; detaches graph aliases before writing.
|
||||
* @lifetime Borrowed until mutation or destruction; use the provider thread for GPU readback.
|
||||
* @throws std::runtime_error On readback failure. Internal executor converts this to Result. */
|
||||
uint8_t* data() { return write().data(); }
|
||||
uint8_t operator[](size_t i) const { return get()[i]; }
|
||||
uint8_t& operator[](size_t i) { return write()[i]; }
|
||||
auto begin() { return write().begin(); }
|
||||
auto end() { return write().end(); }
|
||||
auto begin() const { return get().begin(); }
|
||||
auto end() const { return get().end(); }
|
||||
void resize(size_t n) {
|
||||
auto& v = write();
|
||||
v.resize(n);
|
||||
buffer_.size = n;
|
||||
}
|
||||
/** @brief Copy borrowed iterators into owning CPU storage.
|
||||
* @lifetime Input iterators are borrowed only for this call; all prior data pointers are invalidated. */
|
||||
template <class I>
|
||||
void assign(I a, I b) {
|
||||
*this = ByteStorage(std::vector<uint8_t>(a, b));
|
||||
}
|
||||
operator std::span<const uint8_t>() const { return get(); }
|
||||
operator std::vector<uint8_t>() const { return get(); }
|
||||
};
|
||||
struct RuntimeTensor {
|
||||
OnnxElement element = OnnxElement::Float32;
|
||||
std::vector<int64_t> shape;
|
||||
ByteStorage bytes;
|
||||
};
|
||||
} // namespace eve::tensor::onnx_detail
|
||||
@@ -0,0 +1,99 @@
|
||||
#pragma once
|
||||
#include <algorithm>
|
||||
#include <atomic>
|
||||
#include <chrono>
|
||||
#include <condition_variable>
|
||||
#include <deque>
|
||||
#include <future>
|
||||
#include <mutex>
|
||||
#include <thread>
|
||||
#include "gpgpu/ComputeProgram.h"
|
||||
|
||||
namespace eve::tensor::onnx_detail {
|
||||
// Session-owned CPU workers. Jobs own their source and never capture Graphics,
|
||||
// Run, buffers or callbacks. Joining the workers needs no device access.
|
||||
struct CompiledProgram {
|
||||
std::vector<uint32_t> words;
|
||||
double milliseconds = 0;
|
||||
};
|
||||
class CompilerQueue {
|
||||
using Job = std::packaged_task<CompiledProgram()>;
|
||||
std::mutex mutex;
|
||||
std::condition_variable ready;
|
||||
std::deque<Job> jobs;
|
||||
bool stopping = false;
|
||||
size_t inFlight = 0;
|
||||
std::vector<std::thread> workers;
|
||||
std::atomic<size_t> running{0}, peak{0};
|
||||
void stop() noexcept {
|
||||
{
|
||||
std::lock_guard lock(mutex);
|
||||
stopping = true;
|
||||
}
|
||||
ready.notify_all();
|
||||
for (auto& worker : workers)
|
||||
if (worker.joinable()) worker.join();
|
||||
}
|
||||
|
||||
public:
|
||||
explicit CompilerQueue(uint32_t count) {
|
||||
try {
|
||||
for (uint32_t i = 0; i < count; ++i)
|
||||
workers.emplace_back([this] {
|
||||
for (;;) {
|
||||
Job job;
|
||||
{
|
||||
std::unique_lock lock(mutex);
|
||||
ready.wait(lock, [this] { return stopping || !jobs.empty(); });
|
||||
if (jobs.empty()) return;
|
||||
job = std::move(jobs.front());
|
||||
jobs.pop_front();
|
||||
++inFlight;
|
||||
}
|
||||
const auto n = ++running;
|
||||
auto p = peak.load();
|
||||
while (p < n && !peak.compare_exchange_weak(p, n)) {
|
||||
}
|
||||
job(); // packaged_task transports failures to the owning device thread.
|
||||
--running;
|
||||
{
|
||||
std::lock_guard lock(mutex);
|
||||
--inFlight;
|
||||
}
|
||||
ready.notify_all();
|
||||
}
|
||||
});
|
||||
} catch (...) {
|
||||
stop();
|
||||
throw;
|
||||
}
|
||||
}
|
||||
~CompilerQueue() { stop(); }
|
||||
// Called on the device thread. Run bounds each segment to 48 operations;
|
||||
// this independent queue bound also protects accidental future callers.
|
||||
std::future<CompiledProgram> enqueue(std::string source) {
|
||||
Job job([source = std::move(source)] {
|
||||
const auto start = std::chrono::steady_clock::now();
|
||||
auto result = gpgpu::compileComputeSpirv(source);
|
||||
if (!result.ok()) throw std::runtime_error(result.error()->message() + "\nCompute source:\n" + source);
|
||||
return CompiledProgram{
|
||||
std::move(result.value()),
|
||||
std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - start).count()};
|
||||
});
|
||||
auto future = job.get_future();
|
||||
{
|
||||
std::lock_guard lock(mutex);
|
||||
if (stopping || jobs.size() >= 48) throw std::runtime_error("ONNX compiler queue unavailable");
|
||||
jobs.push_back(std::move(job));
|
||||
}
|
||||
ready.notify_one();
|
||||
return future;
|
||||
}
|
||||
size_t peakWorkers() const noexcept { return peak.load(); }
|
||||
void waitIdle() noexcept {
|
||||
std::unique_lock lock(mutex);
|
||||
ready.wait(lock, [this] { return jobs.empty() && inFlight == 0; });
|
||||
}
|
||||
void resetPeak() noexcept { peak = 0; }
|
||||
};
|
||||
} // namespace eve::tensor::onnx_detail
|
||||
@@ -0,0 +1,85 @@
|
||||
#pragma once
|
||||
#include <cstdint>
|
||||
#include <memory>
|
||||
#include <span>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include "common/Result.h"
|
||||
#include "tensor/OnnxStorage.h"
|
||||
|
||||
namespace eve::tensor {
|
||||
/** @brief Synchronous GPU kernel request; all inputs are borrowed only until dispatch returns. */
|
||||
struct OnnxKernel {
|
||||
std::string source;
|
||||
std::vector<std::span<const uint8_t>> inputs;
|
||||
size_t outputBytes = 0;
|
||||
uint32_t workItems = 0;
|
||||
};
|
||||
/**
|
||||
* @brief GPU execution boundary for native ONNX; retains no model and retains compiled resources for the lifetime of
|
||||
* the provider.
|
||||
* @note Use on the device's owning thread, without concurrent calls or reentrancy.
|
||||
* dispatch completes GPU work before returning host bytes; enqueue may defer completion. Errors never
|
||||
* trigger CPU execution. Integer kernels require exact int32 results.
|
||||
*/
|
||||
class OnnxCompute {
|
||||
public:
|
||||
/** @brief Release device resources after all synchronous calls have returned. */
|
||||
virtual ~OnnxCompute() = default;
|
||||
/** @brief Compatibility-only synchronous host-byte entry point; engine dispatch delegates to enqueue.
|
||||
* @note GLSL 450, local_size_x=64; inputs occupy bindings 0..N-1, output N. */
|
||||
[[nodiscard]] virtual Result<std::vector<uint8_t>> dispatch(const OnnxKernel& kernel) = 0;
|
||||
|
||||
/** @brief Queue a kernel using immutable host/device buffers; no readback is required.
|
||||
* @note Default provider executes synchronously through dispatch. Engine provider records a
|
||||
* resident graph segment inside runGpu, completing it at a host-read or resource-budget boundary.
|
||||
* Inputs are retained until completion; returned device storage expires when runGpu ends. */
|
||||
[[nodiscard]] virtual Result<OnnxBuffer> enqueue(const std::string& source, std::span<const OnnxBuffer> inputs,
|
||||
size_t outputBytes, uint32_t workItems) {
|
||||
OnnxKernel k{source, {}, outputBytes, workItems};
|
||||
std::vector<std::vector<uint8_t>> copies;
|
||||
copies.reserve(inputs.size());
|
||||
for (const auto& input : inputs) {
|
||||
if (input.host)
|
||||
k.inputs.push_back(*input.host);
|
||||
else {
|
||||
if (!input.device)
|
||||
return Result<OnnxBuffer>::failure(
|
||||
Diagnostic::error(DiagnosticCode::InvalidArgument, "Missing GPU input"));
|
||||
auto r = input.device->readback();
|
||||
if (!r.ok()) return Result<OnnxBuffer>::failure(r.status());
|
||||
copies.push_back(std::move(r.value()));
|
||||
k.inputs.push_back(copies.back());
|
||||
}
|
||||
}
|
||||
auto r = dispatch(k);
|
||||
if (!r.ok()) return Result<OnnxBuffer>::failure(r.status());
|
||||
auto bytes = std::make_shared<const std::vector<uint8_t>>(std::move(r.value()));
|
||||
return Result<OnnxBuffer>::success({bytes, {}, bytes->size()});
|
||||
}
|
||||
/** @brief Observable transfer counters for the current or most recent run. */
|
||||
virtual OnnxTransferStats transferStats() const { return {}; }
|
||||
|
||||
protected:
|
||||
friend class OnnxModel;
|
||||
/** @brief Start a synchronous run cache scope; invoked only by OnnxModel on the device thread. */
|
||||
virtual void beginRun() noexcept {}
|
||||
/** @brief Retire temporary activations/recordings; compiled resources may survive for later calls. */
|
||||
virtual void endRun() noexcept {}
|
||||
};
|
||||
/**
|
||||
* @brief Create a reusable GPU session for the active engine Gpgpu Vulkan device, or an error.
|
||||
* @note Retain and reuse this owning provider across runGpu calls. Pipelines, immutable initializers
|
||||
* and buffer pools persist until provider destruction or Graphics resource retirement. Cache bounds:
|
||||
* 1024 shader variants, 128 MiB initializer payload and 512 MiB device buffers.
|
||||
* CPU shader compilation uses a
|
||||
* bounded queue; pipeline creation and submission remain on the device thread.
|
||||
* @param compilerWorkers Number of CPU
|
||||
* compiler workers (1..8); default 4. Source is owned by each job.
|
||||
* @lifetime Graphics retirement clears device
|
||||
* resources before device destruction; either destruction order is supported. A retired session rejects further GPU
|
||||
* work; create a new session for a new device.
|
||||
* @thread Graphics thread only, no concurrent calls, reentrancy or device destruction during runGpu.
|
||||
*/
|
||||
[[nodiscard]] Result<std::unique_ptr<OnnxCompute>> createOnnxGpuCompute(uint32_t compilerWorkers = 4);
|
||||
} // namespace eve::tensor
|
||||
@@ -0,0 +1,377 @@
|
||||
#include "tensor/CpuKernels.h"
|
||||
#include "tensor/OnnxInternal.h"
|
||||
#include "tensor/Tensor.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <cmath>
|
||||
#include <numeric>
|
||||
#include <set>
|
||||
|
||||
namespace eve::tensor::onnx_detail {
|
||||
bool isSupported(const Node& n) {
|
||||
if (n.domain == "com.microsoft" && n.op == "DynamicQuantizeLSTM") {
|
||||
if (n.inputs.size() != 12 || n.outputs.empty() || n.outputs.size() > 3) return false;
|
||||
for (const auto& [key, a] : n.attrs) {
|
||||
if (key == "direction") {
|
||||
if (a.type != 3 || (a.text != "forward" && a.text != "reverse" && a.text != "bidirectional"))
|
||||
return false;
|
||||
} else if (key == "hidden_size" || key == "input_forget") {
|
||||
if (a.type != 2) return false;
|
||||
} else if (key == "clip") {
|
||||
if (a.type != 1) return false;
|
||||
} else
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
if (!n.domain.empty()) return false;
|
||||
if (n.op == "If" || n.op == "Loop") {
|
||||
const std::set<std::string> keys =
|
||||
n.op == "If" ? std::set<std::string>{"then_branch", "else_branch"} : std::set<std::string>{"body"};
|
||||
if (n.attrs.size() != keys.size() || n.outputs.empty() ||
|
||||
(n.op == "If" ? n.inputs.size() != 1 : n.inputs.size() < 2))
|
||||
return false;
|
||||
for (const auto& key : keys) {
|
||||
auto it = n.attrs.find(key);
|
||||
if (it == n.attrs.end() || it->second.type != 5 || !it->second.graph) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
static const std::map<std::string, std::set<std::string>> allowed{
|
||||
{"DynamicQuantizeLinear", {}},
|
||||
{"QuantizeLinear", {"axis"}},
|
||||
{"DequantizeLinear", {"axis"}},
|
||||
{"MatMulInteger", {}},
|
||||
{"ConvInteger", {"auto_pad", "dilations", "group", "kernel_shape", "pads", "strides"}},
|
||||
{"Constant", {"value"}},
|
||||
{"Clip", {}},
|
||||
{"Pad", {"mode"}},
|
||||
{"CumSum", {"exclusive", "reverse"}},
|
||||
{"SequenceEmpty", {"dtype"}},
|
||||
{"SequenceAt", {}},
|
||||
{"SequenceInsert", {}},
|
||||
{"SplitToSequence", {"axis", "keepdims"}},
|
||||
{"ConcatFromSequence", {"axis", "new_axis"}},
|
||||
{"RandomUniformLike", {"dtype", "seed", "low", "high"}},
|
||||
{"RandomNormalLike", {"dtype", "seed", "mean", "scale"}},
|
||||
{"InstanceNormalization", {"epsilon"}},
|
||||
{"ConvTranspose", {"auto_pad", "dilations", "group", "kernel_shape", "output_padding", "pads", "strides"}},
|
||||
{"Resize", {"coordinate_transformation_mode", "mode", "nearest_mode", "cubic_coeff_a"}},
|
||||
{"Range", {}},
|
||||
{"TopK", {"axis", "largest", "sorted"}},
|
||||
{"ScatterND", {"reduction"}},
|
||||
{"ScatterElements", {"axis", "reduction"}},
|
||||
{"ReduceProd", {"axes", "keepdims"}},
|
||||
{"ReduceMax", {"axes", "keepdims"}},
|
||||
{"Slice", {}},
|
||||
{"Concat", {"axis"}},
|
||||
{"Expand", {}},
|
||||
{"ConstantOfShape", {"value"}},
|
||||
{"Where", {}},
|
||||
{"Equal", {}},
|
||||
{"Less", {}},
|
||||
{"Greater", {}},
|
||||
{"And", {}},
|
||||
{"Not", {}},
|
||||
{"Reciprocal", {}},
|
||||
{"LeakyRelu", {"alpha"}},
|
||||
{"Floor", {}},
|
||||
{"Round", {}},
|
||||
{"Atan", {}},
|
||||
{"Identity", {}},
|
||||
{"Cast", {"to"}},
|
||||
{"Gather", {"axis"}},
|
||||
{"Shape", {"start", "end"}},
|
||||
{"Reshape", {"allowzero"}},
|
||||
{"Transpose", {"perm"}},
|
||||
{"Unsqueeze", {}},
|
||||
{"Squeeze", {}},
|
||||
{"MatMul", {}},
|
||||
{"Add", {}},
|
||||
{"Sub", {}},
|
||||
{"Mul", {}},
|
||||
{"Div", {}},
|
||||
{"Sqrt", {}},
|
||||
{"Exp", {}},
|
||||
{"Log", {}},
|
||||
{"Sin", {}},
|
||||
{"Cos", {}},
|
||||
{"Relu", {}},
|
||||
{"Sigmoid", {}},
|
||||
{"Tanh", {}},
|
||||
{"Neg", {}},
|
||||
{"Abs", {}},
|
||||
{"Pow", {}},
|
||||
{"Softmax", {"axis"}},
|
||||
{"ReduceMean", {"axes", "keepdims"}},
|
||||
{"ReduceSum", {"keepdims", "noop_with_empty_axes"}}};
|
||||
auto op = allowed.find(n.op);
|
||||
if (op == allowed.end()) return false;
|
||||
size_t minimum = 1, maximum = 1, outputs = 1;
|
||||
if (n.op == "Clip") maximum = 3;
|
||||
if (n.op == "Pad") {
|
||||
minimum = 2;
|
||||
maximum = 3;
|
||||
}
|
||||
if (n.op == "CumSum") minimum = maximum = 2;
|
||||
if (n.op == "SequenceEmpty") minimum = maximum = 0;
|
||||
if (n.op == "SequenceAt") minimum = maximum = 2;
|
||||
if (n.op == "SequenceInsert") {
|
||||
minimum = 2;
|
||||
maximum = 3;
|
||||
}
|
||||
if (n.op == "SplitToSequence") maximum = 2;
|
||||
if (n.op == "InstanceNormalization") minimum = maximum = 3;
|
||||
if (n.op == "ConvTranspose") {
|
||||
minimum = 2;
|
||||
maximum = 3;
|
||||
}
|
||||
if (n.op == "Resize") {
|
||||
minimum = 3;
|
||||
maximum = 4;
|
||||
}
|
||||
if (n.op == "Range" || n.op == "ScatterElements" || n.op == "ScatterND") minimum = maximum = 3;
|
||||
if (n.op == "TopK") {
|
||||
minimum = maximum = 2;
|
||||
outputs = 2;
|
||||
}
|
||||
if (n.op == "Slice") {
|
||||
minimum = 3;
|
||||
maximum = 5;
|
||||
}
|
||||
if (n.op == "Concat") maximum = 100000;
|
||||
if (n.op == "Where") minimum = maximum = 3;
|
||||
if (n.op == "Expand" || n.op == "Equal" || n.op == "Less" || n.op == "Greater" || n.op == "And")
|
||||
minimum = maximum = 2;
|
||||
if (n.op == "Constant") minimum = maximum = 0;
|
||||
if (n.op == "DynamicQuantizeLinear") outputs = 3;
|
||||
if (n.op == "QuantizeLinear" || n.op == "DequantizeLinear") {
|
||||
minimum = 2;
|
||||
maximum = 3;
|
||||
}
|
||||
if (n.op == "MatMulInteger" || n.op == "ConvInteger") {
|
||||
minimum = 2;
|
||||
maximum = 4;
|
||||
}
|
||||
if (n.op == "Gather" || n.op == "Reshape" || n.op == "Unsqueeze" || n.op == "MatMul" || n.op == "Add" ||
|
||||
n.op == "Sub" || n.op == "Mul" || n.op == "Div" || n.op == "Pow")
|
||||
minimum = maximum = 2;
|
||||
if (n.op == "Squeeze" || n.op == "ReduceSum") maximum = 2;
|
||||
if (n.inputs.size() < minimum || n.inputs.size() > maximum || n.outputs.size() != outputs) return false;
|
||||
for (const auto& [key, a] : n.attrs) {
|
||||
if (!op->second.contains(key)) return false;
|
||||
const int expected = (key == "alpha" || key == "epsilon" || key == "cubic_coeff_a" || key == "seed" ||
|
||||
key == "low" || key == "high" || key == "mean" || key == "scale")
|
||||
? 1
|
||||
: key == "value" ? 4
|
||||
: (key == "auto_pad" || key == "reduction" || key == "coordinate_transformation_mode" ||
|
||||
key == "mode" || key == "nearest_mode")
|
||||
? 3
|
||||
: (key == "perm" || key == "axes" || key == "dilations" || key == "kernel_shape" ||
|
||||
key == "pads" || key == "strides" || key == "output_padding")
|
||||
? 7
|
||||
: 2;
|
||||
if (a.type != expected) return false;
|
||||
if (key == "reduction" && a.text != "none") return false;
|
||||
if (key == "auto_pad" && a.text != "NOTSET") return false;
|
||||
if (key == "allowzero" && a.integer != 0) return false;
|
||||
}
|
||||
if (n.op == "Pad" && n.attrs.contains("mode") && n.attrs.at("mode").text != "constant" &&
|
||||
n.attrs.at("mode").text != "edge" && n.attrs.at("mode").text != "reflect")
|
||||
return false;
|
||||
if (n.op == "Resize") {
|
||||
const auto mode = n.attrs.contains("mode") ? n.attrs.at("mode").text : "nearest";
|
||||
const auto coordinate = n.attrs.contains("coordinate_transformation_mode")
|
||||
? n.attrs.at("coordinate_transformation_mode").text
|
||||
: "half_pixel";
|
||||
const auto nearest = n.attrs.contains("nearest_mode") ? n.attrs.at("nearest_mode").text : "round_prefer_floor";
|
||||
if (mode == "nearest") {
|
||||
if (coordinate != "asymmetric" || nearest != "floor") return false;
|
||||
} else if (mode != "linear" || (coordinate != "half_pixel" && coordinate != "asymmetric"))
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
namespace {
|
||||
std::vector<int> dims(const RuntimeTensor& x) {
|
||||
std::vector<int> out;
|
||||
for (auto d : x.shape) {
|
||||
if (d <= 0) throw Failure("Empty float kernel input", DiagnosticCode::Unsupported);
|
||||
out.push_back(static_cast<int>(d));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
std::vector<RuntimeTensor> single(RuntimeTensor x) { return {std::move(x)}; }
|
||||
} // namespace
|
||||
std::vector<RuntimeTensor> execute(const Node& n, const std::vector<const RuntimeTensor*>& in, OnnxCompute* compute) {
|
||||
if (n.domain == "com.microsoft" && n.op == "DynamicQuantizeLSTM") return executeQuantLstm(n, in, compute);
|
||||
if (n.op == "Constant") {
|
||||
auto a = n.attrs.find("value");
|
||||
if (a == n.attrs.end() || !a->second.tensor) throw Failure("Constant requires tensor value");
|
||||
return single(*a->second.tensor);
|
||||
}
|
||||
if (n.op.find("Quantize") != std::string::npos || n.op == "DequantizeLinear" || n.op == "MatMulInteger" ||
|
||||
n.op == "ConvInteger")
|
||||
return executeQuant(n, in, compute);
|
||||
const auto& x = required(in, 0);
|
||||
if (n.op == "Identity") return single(x);
|
||||
if (n.op == "Shape") {
|
||||
const int64_t rank = x.shape.size();
|
||||
auto norm = [rank](int64_t i) { return std::clamp(i < 0 ? i + rank : i, int64_t(0), rank); };
|
||||
const auto begin = norm(attr(n, "start", 0)), end = norm(attr(n, "end", rank));
|
||||
std::vector<int64_t> shape;
|
||||
if (begin < end) shape.assign(x.shape.begin() + begin, x.shape.begin() + end);
|
||||
return single(make(OnnxElement::Int64, {static_cast<int64_t>(shape.size())}, shape));
|
||||
}
|
||||
if (n.op == "Cast") {
|
||||
const auto target = static_cast<OnnxElement>(attr(n, "to", 0));
|
||||
if (compute && target == OnnxElement::Float32 && x.element == OnnxElement::Int32)
|
||||
return single(dispatchFloat(*compute, {&x}, x.shape, "y[i]=float(x0[i]);"));
|
||||
const auto size = elementSize(target);
|
||||
RuntimeTensor out{target, x.shape, std::vector<uint8_t>(count(x.shape) * size)};
|
||||
for (size_t i = 0; i < count(x.shape); ++i) {
|
||||
if (target == OnnxElement::Float32) {
|
||||
const float v =
|
||||
x.element == OnnxElement::Float32 ? read<float>(x, i) : static_cast<float>(integer(x, i));
|
||||
std::memcpy(out.bytes.data() + i * 4, &v, 4);
|
||||
} else {
|
||||
int64_t v;
|
||||
if (x.element == OnnxElement::Float32) {
|
||||
const double f = read<float>(x, i);
|
||||
if (!std::isfinite(f) || f < -9223372036854775808.0 || f >= 9223372036854775808.0)
|
||||
throw Failure("Cast overflow");
|
||||
v = target == OnnxElement::Bool ? (f != 0) : static_cast<int64_t>(f);
|
||||
} else
|
||||
v = integer(x, i);
|
||||
if (target == OnnxElement::Bool) v = v != 0;
|
||||
std::memcpy(out.bytes.data() + i * size, &v, size);
|
||||
}
|
||||
}
|
||||
return single(std::move(out));
|
||||
}
|
||||
if (n.op == "Gather") {
|
||||
const auto& idx = required(in, 1);
|
||||
const int a = axis(attr(n, "axis", 0), x.shape.size());
|
||||
if (idx.element != OnnxElement::Int32 && idx.element != OnnxElement::Int64)
|
||||
throw Failure("Gather indices must be int32/int64");
|
||||
size_t inner = 1, outer = 1;
|
||||
for (size_t i = a + 1; i < x.shape.size(); ++i) inner *= x.shape[i];
|
||||
for (int i = 0; i < a; ++i) outer *= x.shape[i];
|
||||
std::vector<int64_t> shape(x.shape.begin(), x.shape.begin() + a);
|
||||
shape.insert(shape.end(), idx.shape.begin(), idx.shape.end());
|
||||
shape.insert(shape.end(), x.shape.begin() + a + 1, x.shape.end());
|
||||
const size_t stride = inner * elementSize(x.element), ni = count(idx.shape);
|
||||
RuntimeTensor out{x.element, shape, std::vector<uint8_t>(count(shape) * elementSize(x.element))};
|
||||
for (size_t o = 0; o < outer; ++o)
|
||||
for (size_t i = 0; i < ni; ++i) {
|
||||
auto j = integer(idx, i);
|
||||
if (j < 0) j += x.shape[a];
|
||||
if (j < 0 || j >= x.shape[a]) throw Failure("Gather index out of range");
|
||||
if (stride)
|
||||
std::memcpy(out.bytes.data() + (o * ni + i) * stride,
|
||||
x.bytes.data() + (o * x.shape[a] + j) * stride, stride);
|
||||
}
|
||||
return single(std::move(out));
|
||||
}
|
||||
if (n.op == "Reshape" || n.op == "Unsqueeze" || n.op == "Squeeze") {
|
||||
auto shape = x.shape;
|
||||
if (n.op == "Reshape") {
|
||||
shape = ints(required(in, 1));
|
||||
int infer = -1;
|
||||
size_t known = 1;
|
||||
for (size_t i = 0; i < shape.size(); ++i) {
|
||||
if (shape[i] == 0) {
|
||||
if (i >= x.shape.size()) throw Failure("Reshape zero axis out of range");
|
||||
shape[i] = x.shape[i];
|
||||
}
|
||||
if (shape[i] == -1) {
|
||||
if (infer != -1) throw Failure("Multiple inferred dimensions");
|
||||
infer = static_cast<int>(i);
|
||||
} else {
|
||||
if (shape[i] < 0 || shape[i] > INT32_MAX ||
|
||||
(shape[i] && known > 128u * 1024u * 1024u / static_cast<size_t>(shape[i])))
|
||||
throw Failure("Reshape overflow");
|
||||
known *= static_cast<size_t>(shape[i]);
|
||||
}
|
||||
}
|
||||
if (infer >= 0) {
|
||||
if (!known || count(x.shape) % known) throw Failure("Invalid inferred shape");
|
||||
shape[infer] = count(x.shape) / known;
|
||||
}
|
||||
} else {
|
||||
std::vector<int64_t> axes;
|
||||
if (in.size() > 1 && in[1])
|
||||
axes = ints(*in[1]);
|
||||
else if (n.op == "Unsqueeze")
|
||||
throw Failure("Unsqueeze needs axes");
|
||||
else
|
||||
for (size_t i = 0; i < shape.size(); ++i)
|
||||
if (shape[i] == 1) axes.push_back(i);
|
||||
const size_t rank = shape.size() + (n.op == "Unsqueeze" ? axes.size() : 0);
|
||||
std::set<int> positions;
|
||||
for (auto a : axes)
|
||||
if (!positions.insert(axis(a, rank)).second) throw Failure("Duplicate squeeze axis");
|
||||
if (n.op == "Unsqueeze") {
|
||||
for (int a : positions) shape.insert(shape.begin() + a, 1);
|
||||
} else
|
||||
for (auto it = positions.rbegin(); it != positions.rend(); ++it) {
|
||||
if (shape[*it] != 1) throw Failure("Squeeze dimension not one");
|
||||
shape.erase(shape.begin() + *it);
|
||||
}
|
||||
}
|
||||
if (count(shape) != count(x.shape)) throw Failure("Reshape element count mismatch");
|
||||
RuntimeTensor out = x;
|
||||
out.shape = std::move(shape);
|
||||
return single(std::move(out));
|
||||
}
|
||||
if (n.op == "Transpose") {
|
||||
std::vector<int64_t> order = attrs(n, "perm", {});
|
||||
if (order.empty())
|
||||
for (size_t i = x.shape.size(); i > 0; --i) order.push_back(i - 1);
|
||||
if (order.size() != x.shape.size()) throw Failure("Invalid transpose rank");
|
||||
std::set<int64_t> used;
|
||||
std::vector<int64_t> shape;
|
||||
for (auto a : order) {
|
||||
if (a < 0 || a >= static_cast<int64_t>(x.shape.size()) || !used.insert(a).second)
|
||||
throw Failure("Invalid transpose permutation");
|
||||
shape.push_back(x.shape[a]);
|
||||
}
|
||||
RuntimeTensor out{x.element, shape, std::vector<uint8_t>(x.bytes.size())};
|
||||
const auto size = elementSize(x.element);
|
||||
std::vector<size_t> strides(x.shape.size(), 1);
|
||||
for (size_t i = x.shape.size(); i > 1; --i) strides[i - 2] = strides[i - 1] * x.shape[i - 1];
|
||||
for (size_t i = 0; i < count(shape); ++i) {
|
||||
size_t rest = i, source = 0;
|
||||
for (size_t j = shape.size(); j > 0; --j) {
|
||||
source += (rest % shape[j - 1]) * strides[order[j - 1]];
|
||||
rest /= shape[j - 1];
|
||||
}
|
||||
std::memcpy(out.bytes.data() + i * size, x.bytes.data() + source * size, size);
|
||||
}
|
||||
return single(std::move(out));
|
||||
}
|
||||
if (auto indexResult = executeIndex(n, in)) return std::move(*indexResult);
|
||||
if (auto shapeResult = executeShape(n, in)) return single(std::move(*shapeResult));
|
||||
if (auto miscResult = executeMisc(n, in, compute)) return single(std::move(*miscResult));
|
||||
if (auto neuralResult = executeNeural(n, in, compute)) return single(std::move(*neuralResult));
|
||||
if (auto numericResult = executeNumeric(n, in, compute)) return single(std::move(*numericResult));
|
||||
const auto a = floats(x);
|
||||
if (n.op == "Softmax") {
|
||||
auto xd = dims(x);
|
||||
std::vector<float> out(a.size());
|
||||
kernels::softmax(a.data(), xd.data(), static_cast<int>(xd.size()), axis(attr(n, "axis", -1), xd.size()), false,
|
||||
out.data());
|
||||
return single(make(OnnxElement::Float32, x.shape, out));
|
||||
}
|
||||
static const std::map<std::string, OpType> unary{
|
||||
{"Sqrt", OpType::Sqrt}, {"Exp", OpType::Exp}, {"Log", OpType::Log}, {"Sin", OpType::Sin},
|
||||
{"Cos", OpType::Cos}, {"Relu", OpType::Relu}, {"Sigmoid", OpType::Sigmoid}, {"Tanh", OpType::Tanh},
|
||||
{"Neg", OpType::Neg}, {"Abs", OpType::Abs}};
|
||||
auto op = unary.find(n.op);
|
||||
if (op == unary.end()) throw Failure("Unsupported operator", DiagnosticCode::Unsupported);
|
||||
std::vector<float> out(a.size());
|
||||
kernels::unaryOp(op->second, a.data(), static_cast<int>(a.size()), out.data(), 0, 0);
|
||||
return single(make(OnnxElement::Float32, x.shape, out));
|
||||
}
|
||||
} // namespace eve::tensor::onnx_detail
|
||||
@@ -0,0 +1,378 @@
|
||||
#include <algorithm>
|
||||
#include <chrono>
|
||||
#include <cstdlib>
|
||||
#include <iostream>
|
||||
#include <map>
|
||||
#include <unordered_map>
|
||||
#include "common/Subscription.h"
|
||||
#include "gpgpu/ComputeShader.h"
|
||||
#include "gpgpu/Gpgpu.h"
|
||||
#include "gpgpu/GpuBuffer.h"
|
||||
#include "gpgpu/Sequence.h"
|
||||
#include "graphics/Graphics.h"
|
||||
#include "tensor/OnnxCompiler.h"
|
||||
#include "tensor/OnnxCompute.h"
|
||||
namespace eve::tensor {
|
||||
namespace {
|
||||
using Clock = std::chrono::steady_clock;
|
||||
struct Timed {
|
||||
double& milliseconds;
|
||||
Clock::time_point start = Clock::now();
|
||||
~Timed() { milliseconds += std::chrono::duration<double, std::milli>(Clock::now() - start).count(); }
|
||||
};
|
||||
struct Run;
|
||||
struct DeviceBytes final : OnnxDeviceStorage {
|
||||
std::weak_ptr<Run> run;
|
||||
std::unique_ptr<gpgpu::GpuBuffer> buffer;
|
||||
size_t size = 0, capacity = 0;
|
||||
uint64_t epoch = 0;
|
||||
~DeviceBytes() override;
|
||||
Result<std::vector<uint8_t>> readback() const override;
|
||||
};
|
||||
struct ShaderEntry {
|
||||
std::future<onnx_detail::CompiledProgram> compilation;
|
||||
std::unique_ptr<gpgpu::ComputeShader> shader;
|
||||
};
|
||||
struct PendingDispatch {
|
||||
ShaderEntry* program;
|
||||
std::vector<std::shared_ptr<DeviceBytes>> bindings;
|
||||
uint32_t groups;
|
||||
};
|
||||
struct Run : std::enable_shared_from_this<Run> {
|
||||
gpgpu::Gpgpu& gp;
|
||||
std::unique_ptr<gpgpu::Sequence> sequence;
|
||||
std::unordered_map<std::string, ShaderEntry> shaders;
|
||||
onnx_detail::CompilerQueue compiler;
|
||||
std::vector<PendingDispatch> dispatches;
|
||||
double compileTaskMs = 0, compileWaitMs = 0, pipelineMs = 0;
|
||||
std::map<std::shared_ptr<const std::vector<uint8_t>>, std::shared_ptr<DeviceBytes>,
|
||||
std::owner_less<std::shared_ptr<const std::vector<uint8_t>>>>
|
||||
uploads, persistentUploads;
|
||||
std::vector<std::shared_ptr<const std::vector<uint8_t>>> uncommittedUploads;
|
||||
bool active = true;
|
||||
uint64_t epoch = 1;
|
||||
std::vector<std::shared_ptr<DeviceBytes>> pending;
|
||||
std::vector<std::weak_ptr<DeviceBytes>> resources;
|
||||
OnnxTransferStats stats;
|
||||
double shaderMs = 0, allocationMs = 0, uploadRecordMs = 0, dispatchRecordMs = 0, waitMs = 0, readbackMs = 0;
|
||||
size_t compiledShaders = 0;
|
||||
Clock::time_point started = Clock::now();
|
||||
size_t queued = 0, cachedBytes = 0, allocatedBytes = 0;
|
||||
std::multimap<size_t, std::unique_ptr<gpgpu::GpuBuffer>> freeBuffers;
|
||||
explicit Run(gpgpu::Gpgpu& g, uint32_t workers) : gp(g), sequence(g.newSequence()), compiler(workers) {
|
||||
sequence->begin();
|
||||
}
|
||||
~Run() {
|
||||
// Destroy an unsubmitted command sequence before its referenced resources.
|
||||
sequence.reset();
|
||||
for (auto& [key, entry] : shaders)
|
||||
if (entry.shader) entry.shader->clearBindings();
|
||||
for (auto& weak : resources)
|
||||
if (auto p = weak.lock()) p->buffer.reset();
|
||||
}
|
||||
void beginCall() noexcept {
|
||||
stats = {};
|
||||
compiler.resetPeak();
|
||||
compileTaskMs = compileWaitMs = pipelineMs = 0;
|
||||
shaderMs = allocationMs = uploadRecordMs = dispatchRecordMs = waitMs = readbackMs = 0;
|
||||
compiledShaders = 0;
|
||||
started = Clock::now();
|
||||
active = true;
|
||||
++epoch;
|
||||
}
|
||||
void endCall() noexcept {
|
||||
compiler.waitIdle();
|
||||
// An abandoned graph must not record or submit its deferred commands.
|
||||
// Unmaterialized jobs contain only CPU data; discard their future (including any
|
||||
// compilation exception) because the caller already received the graph failure.
|
||||
dispatches.clear();
|
||||
std::erase_if(shaders, [](const auto& item) { return !item.second.shader; });
|
||||
if (const auto* profile = std::getenv("EVE_ONNX_PROFILE"); profile && std::string(profile) == "1")
|
||||
std::cerr << "ONNX_PROFILE shader_ms=" << shaderMs << " allocation_ms=" << allocationMs
|
||||
<< " upload_record_ms=" << uploadRecordMs << " dispatch_record_ms=" << dispatchRecordMs
|
||||
<< " submit_wait_ms=" << waitMs << " readback_host_ms=" << readbackMs
|
||||
<< " compile_task_ms=" << compileTaskMs << " compile_wait_ms=" << compileWaitMs
|
||||
<< " pipeline_ms=" << pipelineMs << " compiler_peak_workers=" << compiler.peakWorkers()
|
||||
<< " compiled_shaders=" << compiledShaders << " resident_scope_ms="
|
||||
<< std::chrono::duration<double, std::milli>(Clock::now() - started).count() << '\n';
|
||||
sequence.reset();
|
||||
for (auto& [key, entry] : shaders)
|
||||
if (entry.shader) entry.shader->clearBindings();
|
||||
// Abandoned recorded uploads have not reached the device and cannot enter a persistent cache.
|
||||
for (const auto& key : uncommittedUploads) {
|
||||
if (auto it = persistentUploads.find(key); it != persistentUploads.end()) {
|
||||
cachedBytes -= key->size();
|
||||
persistentUploads.erase(it);
|
||||
}
|
||||
}
|
||||
uncommittedUploads.clear();
|
||||
pending.clear();
|
||||
uploads.clear();
|
||||
queued = 0;
|
||||
active = false;
|
||||
std::erase_if(resources, [](const auto& p) { return p.expired(); });
|
||||
}
|
||||
std::shared_ptr<DeviceBytes> allocate(size_t bytes) {
|
||||
Timed timing{allocationMs};
|
||||
if (!bytes || bytes > INT32_MAX - 3) throw std::runtime_error("Invalid resident buffer extent");
|
||||
auto p = std::make_shared<DeviceBytes>();
|
||||
p->run = shared_from_this();
|
||||
p->size = bytes;
|
||||
p->epoch = epoch;
|
||||
const auto capacity = std::max(size_t(4), (bytes + 3) & ~size_t(3));
|
||||
auto reusable = freeBuffers.lower_bound(capacity);
|
||||
if (reusable != freeBuffers.end() && reusable->first <= std::max(size_t(4096), capacity * 2)) {
|
||||
p->capacity = reusable->first;
|
||||
p->buffer = std::move(reusable->second);
|
||||
freeBuffers.erase(reusable);
|
||||
++stats.bufferReuses;
|
||||
} else {
|
||||
if (allocatedBytes + capacity > 512u * 1024u * 1024u) {
|
||||
for (const auto& [n, b] : freeBuffers) allocatedBytes -= n;
|
||||
freeBuffers.clear();
|
||||
}
|
||||
if (allocatedBytes + capacity > 512u * 1024u * 1024u)
|
||||
throw std::runtime_error("ONNX GPU resident memory limit exceeded");
|
||||
p->buffer.reset(gp.newBuffer(static_cast<int>(capacity)));
|
||||
p->capacity = capacity;
|
||||
allocatedBytes += capacity;
|
||||
++stats.bufferAllocations;
|
||||
}
|
||||
resources.push_back(p);
|
||||
return p;
|
||||
}
|
||||
void recordDispatches() {
|
||||
for (auto& command : dispatches) {
|
||||
auto& entry = *command.program;
|
||||
if (!entry.shader) {
|
||||
Timed shaderTiming{shaderMs};
|
||||
onnx_detail::CompiledProgram program;
|
||||
{
|
||||
Timed waitTiming{compileWaitMs};
|
||||
program = entry.compilation.get();
|
||||
}
|
||||
compileTaskMs += program.milliseconds;
|
||||
{
|
||||
Timed pipelineTiming{pipelineMs};
|
||||
auto result = gpgpu::createComputeShader(program.words);
|
||||
if (!result.ok()) throw std::runtime_error(result.error()->message());
|
||||
entry.shader = std::move(result.value());
|
||||
}
|
||||
}
|
||||
Timed timing{dispatchRecordMs};
|
||||
auto& shader = *entry.shader;
|
||||
for (size_t i = 0; i < command.bindings.size(); ++i)
|
||||
shader.bindBuffer(static_cast<int>(i), command.bindings[i]->buffer.get());
|
||||
sequence->recordDispatch(&shader, static_cast<int>(command.groups));
|
||||
shader.clearBindings();
|
||||
}
|
||||
stats.compilerPeakWorkers = compiler.peakWorkers();
|
||||
dispatches.clear();
|
||||
}
|
||||
void flush() {
|
||||
if (!queued) return;
|
||||
recordDispatches();
|
||||
Timed timing{waitMs};
|
||||
// Submit asynchronously through the existing sequence contract; CPU visibility is
|
||||
// requested only at a graph boundary. Retain every referenced allocation until wait.
|
||||
if (sequence->submitAsync() != gpgpu::SequenceStatus::Submitted ||
|
||||
sequence->wait() != gpgpu::SequenceStatus::Complete)
|
||||
throw std::runtime_error("ONNX GPU submission failed");
|
||||
uncommittedUploads.clear();
|
||||
++stats.submissions;
|
||||
pending.clear();
|
||||
queued = 0;
|
||||
sequence->begin();
|
||||
}
|
||||
std::shared_ptr<DeviceBytes> input(const OnnxBuffer& value) {
|
||||
if (value.device) {
|
||||
auto d = std::dynamic_pointer_cast<DeviceBytes>(value.device);
|
||||
if (!d || d->run.lock().get() != this || !d->buffer || d->epoch != epoch || !active)
|
||||
throw std::runtime_error("Foreign or expired GPU storage");
|
||||
return d;
|
||||
}
|
||||
if (!value.host || value.host->size() != value.size) throw std::runtime_error("Invalid host input storage");
|
||||
auto& cache = value.persistent ? persistentUploads : uploads;
|
||||
if (auto it = cache.find(value.host); it != cache.end()) {
|
||||
it->second->epoch = epoch;
|
||||
return it->second;
|
||||
}
|
||||
if (value.persistent && cachedBytes + value.size > 128u * 1024u * 1024u) {
|
||||
persistentUploads.clear();
|
||||
cachedBytes = 0;
|
||||
}
|
||||
auto d = allocate(std::max(size_t(1), value.size));
|
||||
Timed timing{uploadRecordMs};
|
||||
std::vector<uint8_t> padded(std::max(size_t(4), (value.size + 3) & ~size_t(3)), 0);
|
||||
std::copy(value.host->begin(), value.host->end(), padded.begin());
|
||||
sequence->recordUpload(d->buffer.get(), padded.data(), padded.size());
|
||||
++stats.uploads;
|
||||
stats.uploadedBytes += value.size;
|
||||
++queued;
|
||||
cache.emplace(value.host, d);
|
||||
if (value.persistent) {
|
||||
cachedBytes += value.size;
|
||||
uncommittedUploads.push_back(value.host);
|
||||
}
|
||||
pending.push_back(d);
|
||||
return d;
|
||||
}
|
||||
};
|
||||
DeviceBytes::~DeviceBytes() {
|
||||
if (buffer)
|
||||
if (auto owner = run.lock()) owner->freeBuffers.emplace(capacity, std::move(buffer));
|
||||
}
|
||||
Result<std::vector<uint8_t>> DeviceBytes::readback() const {
|
||||
try {
|
||||
auto owner = run.lock();
|
||||
if (!owner || !buffer || !owner->active || epoch != owner->epoch)
|
||||
throw std::runtime_error("ONNX GPU storage expired");
|
||||
// Dispatches must precede the download in the command stream. Compilation
|
||||
// waits are accounted separately from host readback and GPU submission.
|
||||
owner->recordDispatches();
|
||||
const auto start = Clock::now();
|
||||
const auto priorWait = owner->waitMs;
|
||||
std::unique_ptr<gpgpu::GpuBuffer> staging(owner->gp.newBuffer(static_cast<int>(size), "staging"));
|
||||
owner->sequence->recordDownload(buffer.get(), staging.get(), size);
|
||||
++owner->queued;
|
||||
owner->flush();
|
||||
std::vector<uint8_t> out(size);
|
||||
staging->downloadBytes(out.data(), size);
|
||||
owner->readbackMs +=
|
||||
std::chrono::duration<double, std::milli>(Clock::now() - start).count() - (owner->waitMs - priorWait);
|
||||
++owner->stats.downloads;
|
||||
owner->stats.downloadedBytes += size;
|
||||
return Result<std::vector<uint8_t>>::success(std::move(out));
|
||||
} catch (const std::exception& e) {
|
||||
return Result<std::vector<uint8_t>>::failure(Diagnostic::error(DiagnosticCode::Failed, e.what()));
|
||||
}
|
||||
}
|
||||
struct SessionState {
|
||||
std::shared_ptr<Run> run;
|
||||
bool retired = false;
|
||||
uint32_t compilerWorkers = 4;
|
||||
};
|
||||
class EngineCompute final : public OnnxCompute {
|
||||
std::shared_ptr<SessionState> state;
|
||||
eve::Subscription retirement;
|
||||
OnnxTransferStats last;
|
||||
bool active = false;
|
||||
|
||||
protected:
|
||||
void beginRun() noexcept override {
|
||||
if (state->run) state->run->beginCall();
|
||||
last = {};
|
||||
active = true;
|
||||
}
|
||||
void endRun() noexcept override {
|
||||
if (state->run) {
|
||||
last = state->run->stats;
|
||||
state->run->endCall();
|
||||
}
|
||||
active = false;
|
||||
}
|
||||
|
||||
public:
|
||||
EngineCompute(std::shared_ptr<SessionState> s, eve::Subscription token)
|
||||
: state(std::move(s)), retirement(std::move(token)) {}
|
||||
OnnxTransferStats transferStats() const override { return active && state->run ? state->run->stats : last; }
|
||||
Result<OnnxBuffer> enqueue(const std::string& source, std::span<const OnnxBuffer> inputs, size_t bytes,
|
||||
uint32_t work) override {
|
||||
try {
|
||||
auto& run = state->run;
|
||||
if (state->retired) throw std::runtime_error("ONNX GPU session device has been retired");
|
||||
if (!active) throw std::runtime_error("Resident enqueue requires an active runGpu scope");
|
||||
if (inputs.size() >= 8 || !bytes || !work || (uint64_t(work) + 63) / 64 > 65535)
|
||||
throw std::runtime_error("Invalid GPU kernel extent");
|
||||
if (!run) {
|
||||
auto* gp = gpgpu::Gpgpu::create();
|
||||
if (!gp || !gp->isAvailable()) throw std::runtime_error("ONNX GPU device unavailable");
|
||||
run = std::make_shared<Run>(*gp, state->compilerWorkers);
|
||||
}
|
||||
if (!run->sequence) {
|
||||
run->sequence.reset(run->gp.newSequence());
|
||||
run->sequence->begin();
|
||||
}
|
||||
// Bound pending resources and the existing shader descriptor pool (64 sets).
|
||||
if (run->queued >= 48) run->flush();
|
||||
auto it = run->shaders.find(source);
|
||||
if (it == run->shaders.end()) {
|
||||
if (run->shaders.size() >= 1024) {
|
||||
run->flush();
|
||||
run->shaders.clear();
|
||||
}
|
||||
++run->stats.shaderCompilations;
|
||||
++run->compiledShaders;
|
||||
it = run->shaders.emplace(source, ShaderEntry{run->compiler.enqueue(source), {}}).first;
|
||||
} else
|
||||
++run->stats.shaderCacheHits;
|
||||
PendingDispatch command{&it->second, {}, (work + 63) / 64};
|
||||
for (const auto& input : inputs) {
|
||||
auto d = run->input(input);
|
||||
command.bindings.push_back(d);
|
||||
run->pending.push_back(d);
|
||||
}
|
||||
auto out = run->allocate(bytes);
|
||||
command.bindings.push_back(out);
|
||||
run->dispatches.push_back(std::move(command));
|
||||
run->pending.push_back(out);
|
||||
++run->queued;
|
||||
return Result<OnnxBuffer>::success({{}, out, bytes});
|
||||
} catch (const std::exception& e) {
|
||||
return Result<OnnxBuffer>::failure(
|
||||
Diagnostic::error(DiagnosticCode::Failed, e.what(), {}, {}, "tensor.onnx.gpu"));
|
||||
}
|
||||
}
|
||||
Result<std::vector<uint8_t>> dispatch(const OnnxKernel& k) override {
|
||||
const bool temporary = !active;
|
||||
if (temporary) beginRun();
|
||||
struct Scope {
|
||||
EngineCompute& owner;
|
||||
bool temporary;
|
||||
~Scope() {
|
||||
if (temporary) owner.endRun();
|
||||
}
|
||||
} scope{*this, temporary};
|
||||
std::vector<OnnxBuffer> inputs;
|
||||
for (auto s : k.inputs) {
|
||||
auto p = std::make_shared<const std::vector<uint8_t>>(s.begin(), s.end());
|
||||
inputs.push_back({p, {}, p->size()});
|
||||
}
|
||||
auto r = enqueue(k.source, inputs, k.outputBytes, k.workItems);
|
||||
if (!r.ok()) return Result<std::vector<uint8_t>>::failure(r.status());
|
||||
return r.value().device->readback();
|
||||
}
|
||||
};
|
||||
} // namespace
|
||||
Result<std::unique_ptr<OnnxCompute>> createOnnxGpuCompute(uint32_t compilerWorkers) {
|
||||
if (compilerWorkers < 1 || compilerWorkers > 8)
|
||||
return Result<std::unique_ptr<OnnxCompute>>::failure(
|
||||
Diagnostic::error(DiagnosticCode::InvalidArgument, "Compiler worker count must be 1..8"));
|
||||
#ifdef EVENGINE_WEBGPU
|
||||
return Result<std::unique_ptr<OnnxCompute>>::failure(
|
||||
Diagnostic::error(DiagnosticCode::Unsupported, "ONNX GLSL kernels require the Vulkan backend"));
|
||||
#else
|
||||
try {
|
||||
auto* gp = gpgpu::Gpgpu::create();
|
||||
if (gp && gp->isAvailable()) {
|
||||
auto state = std::make_shared<SessionState>();
|
||||
state->compilerWorkers = compilerWorkers;
|
||||
std::weak_ptr<SessionState> weak = state;
|
||||
auto token = graphics::Graphics::create()->onResourcesRetiring([weak]() noexcept {
|
||||
if (auto s = weak.lock()) {
|
||||
s->retired = true;
|
||||
s->run.reset();
|
||||
}
|
||||
});
|
||||
if (!token.ok()) return Result<std::unique_ptr<OnnxCompute>>::failure(token.status());
|
||||
return Result<std::unique_ptr<OnnxCompute>>::success(
|
||||
std::make_unique<EngineCompute>(state, std::move(token.value())));
|
||||
}
|
||||
} catch (const std::exception& e) {
|
||||
return Result<std::unique_ptr<OnnxCompute>>::failure(Diagnostic::error(DiagnosticCode::Failed, e.what()));
|
||||
}
|
||||
return Result<std::unique_ptr<OnnxCompute>>::failure(Diagnostic::error(
|
||||
DiagnosticCode::Unsupported, "Initialize the engine Vulkan graphics device before ONNX GPU execution"));
|
||||
#endif
|
||||
}
|
||||
} // namespace eve::tensor
|
||||
@@ -0,0 +1,102 @@
|
||||
#include "tensor/OnnxGpuKernels.h"
|
||||
#include <sstream>
|
||||
#include "tensor/OnnxInternal.h"
|
||||
namespace eve::tensor::onnx_detail {
|
||||
namespace {
|
||||
std::string prefix(bool as, bool bs) {
|
||||
std::ostringstream s;
|
||||
s << "#version 450\nlayout(local_size_x=64) in;\n"
|
||||
"layout(std430,binding=0) readonly buffer A {uint a[];};\n"
|
||||
"layout(std430,binding=1) readonly buffer B {uint b[];};\n"
|
||||
"layout(std430,binding=2) readonly buffer Z {int zp[];};\n"
|
||||
"layout(std430,binding=3) writeonly buffer Y {int y[];};\n"
|
||||
"int av(uint i){int v=int((a[i/4]>>((i%4)*8))&255u);return "
|
||||
<< (as ? "v>=128?v-256:v" : "v")
|
||||
<< ";}\n"
|
||||
"int bv(uint i){int v=int((b[i/4]>>((i%4)*8))&255u);return "
|
||||
<< (bs ? "v>=128?v-256:v" : "v") << ";}\n";
|
||||
return s.str();
|
||||
}
|
||||
// Compatibility host-byte calls use the same resident kernel generators. The
|
||||
// adapter below requests completion through the provider's synchronous contract.
|
||||
class Synchronous final : public OnnxCompute {
|
||||
OnnxCompute& target;
|
||||
|
||||
public:
|
||||
explicit Synchronous(OnnxCompute& t) : target(t) {}
|
||||
Result<std::vector<uint8_t>> dispatch(const OnnxKernel& k) override { return target.dispatch(k); }
|
||||
};
|
||||
OnnxBuffer hostBuffer(std::span<const uint8_t> bytes) {
|
||||
auto p = std::make_shared<const std::vector<uint8_t>>(bytes.begin(), bytes.end());
|
||||
return {p, {}, p->size()};
|
||||
}
|
||||
std::vector<int32_t> integerOutput(const OnnxBuffer& result) {
|
||||
if (!result.host || result.host->size() % 4) throw Failure("Invalid synchronous integer output");
|
||||
std::vector<int32_t> out(result.host->size() / 4);
|
||||
std::memcpy(out.data(), result.host->data(), result.host->size());
|
||||
return out;
|
||||
}
|
||||
} // namespace
|
||||
std::vector<int32_t> gpuMatmul(OnnxCompute& d, affine::ByteView a, affine::ByteView b, size_t m, size_t k, size_t n,
|
||||
int az, std::span<const int32_t> bz) {
|
||||
Synchronous sync(d);
|
||||
return integerOutput(gpuMatmulResident(sync, hostBuffer(a.bytes), hostBuffer(b.bytes), a.signedValues,
|
||||
b.signedValues, m, k, n, az, bz));
|
||||
}
|
||||
std::vector<int32_t> gpuConv(OnnxCompute& d, affine::ByteView x, affine::ByteView w, const affine::ConvShape& c, int xz,
|
||||
std::span<const int32_t> wz) {
|
||||
Synchronous sync(d);
|
||||
return integerOutput(
|
||||
gpuConvResident(sync, hostBuffer(x.bytes), hostBuffer(w.bytes), x.signedValues, w.signedValues, c, xz, wz));
|
||||
}
|
||||
OnnxBuffer enqueueInteger(OnnxCompute& d, const std::string& source, OnnxBuffer a, OnnxBuffer b,
|
||||
std::span<const int32_t> z, size_t size, size_t terms) {
|
||||
if (terms > INT32_MAX / 65025 || !size || size > 64u * 65535u)
|
||||
throw Failure("GPU integer kernel exceeds exact accumulator or dispatch limit", DiagnosticCode::Unsupported);
|
||||
std::vector<uint8_t> bytes(z.size_bytes());
|
||||
std::memcpy(bytes.data(), z.data(), bytes.size());
|
||||
auto host = std::make_shared<const std::vector<uint8_t>>(std::move(bytes));
|
||||
const std::vector<OnnxBuffer> inputs{a, b, {host, {}, host->size()}};
|
||||
auto r = d.enqueue(source, inputs, size * 4, static_cast<uint32_t>(size));
|
||||
if (!r.ok()) throw Failure(r.error()->message(), r.error()->code());
|
||||
return std::move(r.value());
|
||||
}
|
||||
OnnxBuffer gpuMatmulResident(OnnxCompute& d, OnnxBuffer a, OnnxBuffer b, bool as, bool bs, size_t m, size_t k, size_t n,
|
||||
int az, std::span<const int32_t> bz, size_t bOffset) {
|
||||
if (bz.size() != 1 && bz.size() != n) throw Failure("GPU matmul zero-point extent mismatch");
|
||||
std::ostringstream s;
|
||||
s << prefix(as, bs) << "void main(){uint i=gl_GlobalInvocationID.x;if(i>=" << m * n << "u)return;"
|
||||
<< "uint r=i/" << n << "u,c=i%" << n << "u;int v=0;for(uint j=0;j<" << k << "u;++j)"
|
||||
<< "v+=(av(r*" << k << "u+j)-(" << az << "))*(bv(" << bOffset << "u+j*" << n << "u+c)-zp["
|
||||
<< (bz.size() == 1 ? "0" : "c") << "]);y[i]=v;}";
|
||||
return enqueueInteger(d, s.str(), a, b, bz, m * n, k);
|
||||
}
|
||||
|
||||
OnnxBuffer gpuConvResident(OnnxCompute& d, OnnxBuffer x, OnnxBuffer w, bool xs, bool ws, const affine::ConvShape& c,
|
||||
int xz, std::span<const int32_t> wz) {
|
||||
const int64_t oh =
|
||||
(int64_t(c.height) + c.padTop + c.padBottom - int64_t(c.dilationH) * (c.kernelH - 1) - 1) / c.strideH + 1;
|
||||
const int64_t ow =
|
||||
(int64_t(c.width) + c.padLeft + c.padRight - int64_t(c.dilationW) * (c.kernelW - 1) - 1) / c.strideW + 1;
|
||||
if ((oh - 1) * c.strideH > INT32_MAX || (ow - 1) * c.strideW > INT32_MAX ||
|
||||
(oh - 1) * c.strideH - c.padTop + int64_t(c.kernelH - 1) * c.dilationH > INT32_MAX ||
|
||||
(ow - 1) * c.strideW - c.padLeft + int64_t(c.kernelW - 1) * c.dilationW > INT32_MAX)
|
||||
throw Failure("GPU convolution coordinate overflow", DiagnosticCode::Unsupported);
|
||||
const size_t size = size_t(c.batch) * c.outputs * oh * ow;
|
||||
const int cg = c.channels / c.groups;
|
||||
std::ostringstream s;
|
||||
s << prefix(xs, ws) << "void main(){uint i=gl_GlobalInvocationID.x;if(i>=" << size << "u)return;"
|
||||
<< "int xx=int(i%" << ow << "u),yy=int(i/" << ow << "u%" << oh << "u),o=int(i/" << ow * oh << "u%" << c.outputs
|
||||
<< "u),batch=int(i/" << ow * oh * c.outputs << "u);int v=0;"
|
||||
<< "for(int ch=0;ch<" << cg << ";++ch)for(int ky=0;ky<" << c.kernelH << ";++ky)for(int kx=0;kx<" << c.kernelW
|
||||
<< ";++kx){"
|
||||
<< "int iy=yy*" << c.strideH << "-" << c.padTop << "+ky*" << c.dilationH << ",ix=xx*" << c.strideW << "-"
|
||||
<< c.padLeft << "+kx*" << c.dilationW << ";"
|
||||
<< "if(iy<0||ix<0||iy>=" << c.height << "||ix>=" << c.width << ")continue;"
|
||||
<< "uint ai=uint(((batch*" << c.channels << "+o/" << c.outputs / c.groups << "*" << cg << "+ch)*" << c.height
|
||||
<< "+iy)*" << c.width << "+ix);"
|
||||
<< "uint bi=uint(((o*" << cg << "+ch)*" << c.kernelH << "+ky)*" << c.kernelW << "+kx);"
|
||||
<< "v+=(av(ai)-(" << xz << "))*(bv(bi)-zp[" << (wz.size() == 1 ? "0" : "o") << "]);}y[i]=v;}";
|
||||
return enqueueInteger(d, s.str(), x, w, wz, size, size_t(cg) * c.kernelH * c.kernelW);
|
||||
}
|
||||
} // namespace eve::tensor::onnx_detail
|
||||
@@ -0,0 +1,14 @@
|
||||
#pragma once
|
||||
#include "tensor/AffineQuant.h"
|
||||
#include "tensor/OnnxCompute.h"
|
||||
namespace eve::tensor::onnx_detail {
|
||||
// Internal kernels; callers validate geometry and byte extents before dispatch.
|
||||
std::vector<int32_t> gpuMatmul(OnnxCompute& device, affine::ByteView a, affine::ByteView b, size_t m, size_t k,
|
||||
size_t n, int az, std::span<const int32_t> bz);
|
||||
std::vector<int32_t> gpuConv(OnnxCompute& device, affine::ByteView x, affine::ByteView w, const affine::ConvShape& s,
|
||||
int xz, std::span<const int32_t> wz);
|
||||
OnnxBuffer gpuMatmulResident(OnnxCompute&, OnnxBuffer, OnnxBuffer, bool, bool, size_t, size_t, size_t, int,
|
||||
std::span<const int32_t>, size_t bOffset = 0);
|
||||
OnnxBuffer gpuConvResident(OnnxCompute&, OnnxBuffer, OnnxBuffer, bool, bool, const affine::ConvShape&, int,
|
||||
std::span<const int32_t>);
|
||||
} // namespace eve::tensor::onnx_detail
|
||||
@@ -0,0 +1,180 @@
|
||||
#include <algorithm>
|
||||
#include <cmath>
|
||||
#include <limits>
|
||||
#include <numeric>
|
||||
#include <set>
|
||||
#include "tensor/OnnxInternal.h"
|
||||
|
||||
namespace eve::tensor::onnx_detail {
|
||||
std::optional<std::vector<RuntimeTensor>> executeIndex(const Node& n, const std::vector<const RuntimeTensor*>& in) {
|
||||
const auto& x = required(in, 0);
|
||||
auto one = [](RuntimeTensor t) { return std::vector<RuntimeTensor>{std::move(t)}; };
|
||||
if (n.op == "Range") {
|
||||
const auto& end = required(in, 1);
|
||||
const auto& delta = required(in, 2);
|
||||
if (count(x.shape) != 1 || count(end.shape) != 1 || count(delta.shape) != 1 || x.element != end.element ||
|
||||
x.element != delta.element)
|
||||
throw Failure("Range requires matching scalars");
|
||||
if (x.element == OnnxElement::Float32) {
|
||||
const double start = read<float>(x, 0), stop = read<float>(end, 0), step = read<float>(delta, 0);
|
||||
if (!std::isfinite(start) || !std::isfinite(stop) || !std::isfinite(step) || step == 0)
|
||||
throw Failure("Invalid Range");
|
||||
const double length = std::max(0., std::ceil((stop - start) / step));
|
||||
if (length > 128u * 1024u * 1024u) throw Failure("Range too large");
|
||||
std::vector<float> v(static_cast<size_t>(length));
|
||||
for (size_t i = 0; i < v.size(); ++i) v[i] = static_cast<float>(start + i * step);
|
||||
return one(make(x.element, {static_cast<int64_t>(v.size())}, v));
|
||||
}
|
||||
if (x.element != OnnxElement::Int64 && x.element != OnnxElement::Int32)
|
||||
throw Failure("Range integer dtype unsupported");
|
||||
const int64_t start = integer(x), stop = integer(end), step = integer(delta);
|
||||
if (!step || step == INT64_MIN) throw Failure("Invalid Range step");
|
||||
uint64_t length = 0;
|
||||
if (step > 0 && stop > start) length = 1 + (uint64_t(stop) - uint64_t(start) - 1) / uint64_t(step);
|
||||
if (step < 0 && stop < start) length = 1 + (uint64_t(start) - uint64_t(stop) - 1) / uint64_t(-step);
|
||||
if (length > 128u * 1024u * 1024u) throw Failure("Range too large");
|
||||
RuntimeTensor out{
|
||||
x.element, {static_cast<int64_t>(length)}, std::vector<uint8_t>(length * elementSize(x.element))};
|
||||
int64_t v = start;
|
||||
for (size_t i = 0; i < length; ++i) {
|
||||
std::memcpy(out.bytes.data() + i * elementSize(x.element), &v, elementSize(x.element));
|
||||
if (i + 1 < length) v += step;
|
||||
}
|
||||
return one(std::move(out));
|
||||
}
|
||||
if (n.op == "TopK") {
|
||||
const auto& kt = required(in, 1);
|
||||
if (count(kt.shape) != 1) throw Failure("TopK requires one K");
|
||||
int a = axis(attr(n, "axis", -1), x.shape.size());
|
||||
const int64_t k = integer(kt), size = x.shape[a];
|
||||
if (k < 0 || k > size) throw Failure("Invalid TopK K");
|
||||
size_t outer = 1, inner = 1;
|
||||
for (int j = 0; j < a; ++j) outer *= x.shape[j];
|
||||
for (size_t j = a + 1; j < x.shape.size(); ++j) inner *= x.shape[j];
|
||||
auto shape = x.shape;
|
||||
shape[a] = k;
|
||||
RuntimeTensor values{x.element, shape, std::vector<uint8_t>(count(shape) * elementSize(x.element))};
|
||||
std::vector<int64_t> indices(count(shape));
|
||||
for (size_t o = 0; o < outer; ++o)
|
||||
for (size_t i = 0; i < inner; ++i) {
|
||||
std::vector<int64_t> order(size);
|
||||
std::iota(order.begin(), order.end(), 0);
|
||||
std::stable_sort(order.begin(), order.end(), [&](int64_t aa, int64_t bb) {
|
||||
auto compare = [&](auto u, auto v) { return attr(n, "largest", 1) ? u > v : u < v; };
|
||||
const size_t ai = (o * size + aa) * inner + i, bi = (o * size + bb) * inner + i;
|
||||
return x.element == OnnxElement::Float32 ? compare(read<float>(x, ai), read<float>(x, bi))
|
||||
: compare(integer(x, ai), integer(x, bi));
|
||||
});
|
||||
for (int64_t j = 0; j < k; ++j) {
|
||||
size_t dst = (o * k + j) * inner + i, src = (o * size + order[j]) * inner + i;
|
||||
indices[dst] = order[j];
|
||||
std::memcpy(values.bytes.data() + dst * elementSize(x.element),
|
||||
x.bytes.data() + src * elementSize(x.element), elementSize(x.element));
|
||||
}
|
||||
}
|
||||
return std::vector<RuntimeTensor>{std::move(values), make(OnnxElement::Int64, shape, indices)};
|
||||
}
|
||||
if (n.op == "ScatterElements" || n.op == "ScatterND") {
|
||||
const auto& idx = required(in, 1);
|
||||
const auto& updates = required(in, 2);
|
||||
if (updates.element != x.element || (idx.element != OnnxElement::Int64 && idx.element != OnnxElement::Int32))
|
||||
throw Failure("Scatter dtype mismatch");
|
||||
auto out = x;
|
||||
const auto bytes = elementSize(x.element);
|
||||
if (n.op == "ScatterElements") {
|
||||
if (idx.shape != updates.shape || idx.shape.size() != x.shape.size())
|
||||
throw Failure("ScatterElements shape mismatch");
|
||||
int a = axis(attr(n, "axis", 0), x.shape.size());
|
||||
for (size_t j = 0; j < x.shape.size(); ++j)
|
||||
if (j != a && idx.shape[j] > x.shape[j]) throw Failure("Scatter index shape exceeds data");
|
||||
for (size_t i = 0; i < count(idx.shape); ++i) {
|
||||
size_t rest = i, source = 0, stride = 1;
|
||||
for (size_t j = x.shape.size(); j > 0; --j) {
|
||||
int64_t c = rest % idx.shape[j - 1];
|
||||
rest /= idx.shape[j - 1];
|
||||
if (j - 1 == a) {
|
||||
c = integer(idx, i);
|
||||
if (c < 0) c += x.shape[j - 1];
|
||||
}
|
||||
if (c < 0 || c >= x.shape[j - 1]) throw Failure("Scatter index out of range");
|
||||
source += c * stride;
|
||||
stride *= x.shape[j - 1];
|
||||
}
|
||||
std::memcpy(out.bytes.data() + source * bytes, updates.bytes.data() + i * bytes, bytes);
|
||||
}
|
||||
} else {
|
||||
if (idx.shape.empty() || idx.shape.back() < 1 || static_cast<size_t>(idx.shape.back()) > x.shape.size())
|
||||
throw Failure("ScatterND index rank mismatch");
|
||||
const size_t depth = idx.shape.back();
|
||||
std::vector<int64_t> expected(idx.shape.begin(), idx.shape.end() - 1);
|
||||
expected.insert(expected.end(), x.shape.begin() + depth, x.shape.end());
|
||||
if (expected != updates.shape) throw Failure("ScatterND updates shape mismatch");
|
||||
size_t chunk = 1;
|
||||
for (size_t j = depth; j < x.shape.size(); ++j) chunk *= x.shape[j];
|
||||
for (size_t i = 0; i < count(idx.shape) / depth; ++i) {
|
||||
size_t target = 0;
|
||||
for (size_t j = 0; j < depth; ++j) {
|
||||
auto c = integer(idx, i * depth + j);
|
||||
if (c < 0) c += x.shape[j];
|
||||
if (c < 0 || c >= x.shape[j]) throw Failure("ScatterND index out of range");
|
||||
target = target * x.shape[j] + c;
|
||||
}
|
||||
if (chunk)
|
||||
std::memcpy(out.bytes.data() + target * chunk * bytes, updates.bytes.data() + i * chunk * bytes,
|
||||
chunk * bytes);
|
||||
}
|
||||
}
|
||||
return one(std::move(out));
|
||||
}
|
||||
if (n.op == "ReduceProd" || n.op == "ReduceMax") {
|
||||
auto axes = attrs(n, "axes", {});
|
||||
if (axes.empty())
|
||||
for (size_t i = 0; i < x.shape.size(); ++i) axes.push_back(i);
|
||||
std::set<int> reduced;
|
||||
for (auto a : axes)
|
||||
if (!reduced.insert(axis(a, x.shape.size())).second) throw Failure("Duplicate reduction axis");
|
||||
auto mapShape = x.shape;
|
||||
std::vector<int64_t> shape;
|
||||
for (size_t j = 0; j < x.shape.size(); ++j) {
|
||||
if (reduced.contains(static_cast<int>(j))) {
|
||||
mapShape[j] = 1;
|
||||
if (attr(n, "keepdims", 1)) shape.push_back(1);
|
||||
} else
|
||||
shape.push_back(x.shape[j]);
|
||||
}
|
||||
if (x.element == OnnxElement::Float32) {
|
||||
std::vector<float> out(count(shape), n.op == "ReduceProd" ? 1.f : -std::numeric_limits<float>::infinity());
|
||||
for (size_t i = 0; i < count(x.shape); ++i) {
|
||||
size_t t = broadcastIndex(i, mapShape, x.shape);
|
||||
const float v = read<float>(x, i);
|
||||
out[t] = n.op == "ReduceProd" ? out[t] * v : std::max(out[t], v);
|
||||
}
|
||||
return one(make(x.element, shape, out));
|
||||
}
|
||||
if (x.element != OnnxElement::Int64 && x.element != OnnxElement::Int32)
|
||||
throw Failure("Integer reduction dtype unsupported");
|
||||
std::vector<int64_t> out(count(shape), n.op == "ReduceProd" ? 1 : INT64_MIN);
|
||||
for (size_t i = 0; i < count(x.shape); ++i) {
|
||||
auto& a = out[broadcastIndex(i, mapShape, x.shape)];
|
||||
const auto b = integer(x, i);
|
||||
if (n.op == "ReduceMax")
|
||||
a = std::max(a, b);
|
||||
else {
|
||||
if (a && b &&
|
||||
(a > 0 ? (b > 0 ? a > INT64_MAX / b : b < INT64_MIN / a)
|
||||
: (b > 0 ? a < INT64_MIN / b : a < INT64_MAX / b)))
|
||||
throw Failure("ReduceProd overflow");
|
||||
a *= b;
|
||||
}
|
||||
}
|
||||
if (x.element == OnnxElement::Int64) return one(make(x.element, shape, out));
|
||||
std::vector<int32_t> narrow;
|
||||
for (auto v : out) {
|
||||
if (v < INT32_MIN || v > INT32_MAX) throw Failure("Int32 reduction overflow");
|
||||
narrow.push_back(static_cast<int32_t>(v));
|
||||
}
|
||||
return one(make(x.element, shape, narrow));
|
||||
}
|
||||
return std::nullopt;
|
||||
}
|
||||
} // namespace eve::tensor::onnx_detail
|
||||
@@ -0,0 +1,140 @@
|
||||
#pragma once
|
||||
#include <cstring>
|
||||
#include <map>
|
||||
#include <optional>
|
||||
#include <stdexcept>
|
||||
#include <unordered_map>
|
||||
#include "tensor/OnnxByteStorage.h"
|
||||
#include "tensor/OnnxModel.h"
|
||||
|
||||
namespace eve::tensor::onnx_detail {
|
||||
struct Failure : std::runtime_error {
|
||||
DiagnosticCode code;
|
||||
std::string path;
|
||||
explicit Failure(std::string message, DiagnosticCode c = DiagnosticCode::InvalidArgument, std::string p = {})
|
||||
: std::runtime_error(std::move(message)), code(c), path(std::move(p)) {}
|
||||
};
|
||||
struct ModelData;
|
||||
struct Attribute {
|
||||
int type = 0;
|
||||
int64_t integer = 0;
|
||||
float real = 0;
|
||||
std::string text;
|
||||
std::vector<int64_t> integers;
|
||||
std::vector<float> reals;
|
||||
std::vector<std::string> strings;
|
||||
std::optional<RuntimeTensor> tensor;
|
||||
std::shared_ptr<ModelData> graph;
|
||||
};
|
||||
struct Node {
|
||||
std::string name, op, domain;
|
||||
std::vector<std::string> inputs, outputs;
|
||||
std::map<std::string, Attribute> attrs;
|
||||
};
|
||||
struct Input {
|
||||
int type = 0;
|
||||
std::vector<int64_t> shape;
|
||||
};
|
||||
struct ModelData {
|
||||
OnnxModelInfo info;
|
||||
std::vector<Node> nodes;
|
||||
std::unordered_map<std::string, RuntimeTensor> constants;
|
||||
std::map<std::string, Input> inputs;
|
||||
std::map<std::string, int64_t> opsets;
|
||||
};
|
||||
inline size_t elementSize(OnnxElement e) {
|
||||
switch (e) {
|
||||
case OnnxElement::Float32:
|
||||
case OnnxElement::Int32: return 4;
|
||||
case OnnxElement::Int64: return 8;
|
||||
case OnnxElement::UInt8:
|
||||
case OnnxElement::Int8:
|
||||
case OnnxElement::Bool: return 1;
|
||||
}
|
||||
throw Failure("Unsupported ONNX element type", DiagnosticCode::Unsupported);
|
||||
}
|
||||
inline size_t count(const std::vector<int64_t>& shape) {
|
||||
if (shape.size() > 6) throw Failure("ONNX rank exceeds six", DiagnosticCode::Unsupported);
|
||||
size_t n = 1;
|
||||
for (int64_t d : shape) {
|
||||
if (d < 0 || d > INT32_MAX || (d && n > (128u * 1024u * 1024u) / static_cast<size_t>(d)))
|
||||
throw Failure("Invalid or excessive tensor shape");
|
||||
n *= static_cast<size_t>(d);
|
||||
}
|
||||
return n;
|
||||
}
|
||||
inline void validate(const RuntimeTensor& v) {
|
||||
if (count(v.shape) * elementSize(v.element) != v.bytes.size()) throw Failure("Tensor byte count mismatch");
|
||||
}
|
||||
template <class T>
|
||||
T read(const RuntimeTensor& v, size_t i) {
|
||||
T x;
|
||||
std::memcpy(&x, v.bytes.data() + i * sizeof(T), sizeof(T));
|
||||
return x;
|
||||
}
|
||||
template <class T>
|
||||
RuntimeTensor make(OnnxElement e, std::vector<int64_t> shape, const std::vector<T>& values) {
|
||||
RuntimeTensor v{e, std::move(shape), std::vector<uint8_t>(values.size() * sizeof(T))};
|
||||
if (!values.empty()) std::memcpy(v.bytes.data(), values.data(), v.bytes.size());
|
||||
validate(v);
|
||||
return v;
|
||||
}
|
||||
inline int64_t integer(const RuntimeTensor& v, size_t i = 0) {
|
||||
if (i >= count(v.shape)) throw Failure("Integer index out of bounds");
|
||||
switch (v.element) {
|
||||
case OnnxElement::Int64: return read<int64_t>(v, i);
|
||||
case OnnxElement::Int32: return read<int32_t>(v, i);
|
||||
case OnnxElement::Int8: return v.bytes[i] >= 128 ? int(v.bytes[i]) - 256 : v.bytes[i];
|
||||
case OnnxElement::UInt8:
|
||||
case OnnxElement::Bool: return v.bytes[i];
|
||||
default: throw Failure("Expected integer tensor");
|
||||
}
|
||||
}
|
||||
inline std::vector<float> floats(const RuntimeTensor& v) {
|
||||
if (v.element != OnnxElement::Float32) throw Failure("Expected FP32 tensor");
|
||||
std::vector<float> out(count(v.shape));
|
||||
if (!out.empty()) std::memcpy(out.data(), v.bytes.data(), v.bytes.size());
|
||||
return out;
|
||||
}
|
||||
inline std::vector<int64_t> ints(const RuntimeTensor& v) {
|
||||
std::vector<int64_t> out(count(v.shape));
|
||||
for (size_t i = 0; i < out.size(); ++i) out[i] = integer(v, i);
|
||||
return out;
|
||||
}
|
||||
inline int64_t attr(const Node& n, const char* key, int64_t fallback) {
|
||||
auto it = n.attrs.find(key);
|
||||
return it == n.attrs.end() ? fallback : it->second.integer;
|
||||
}
|
||||
inline std::vector<int64_t> attrs(const Node& n, const char* key, std::vector<int64_t> fallback) {
|
||||
auto it = n.attrs.find(key);
|
||||
return it == n.attrs.end() ? fallback : it->second.integers;
|
||||
}
|
||||
inline const RuntimeTensor& required(const std::vector<const RuntimeTensor*>& in, size_t i) {
|
||||
if (i >= in.size() || !in[i]) throw Failure("Missing required input");
|
||||
return *in[i];
|
||||
}
|
||||
inline int axis(int64_t a, size_t rank) {
|
||||
if (a < 0) a += rank;
|
||||
if (a < 0 || a >= static_cast<int64_t>(rank)) throw Failure("Invalid axis");
|
||||
return static_cast<int>(a);
|
||||
}
|
||||
std::vector<int64_t> broadcast(const std::vector<int64_t>& a, const std::vector<int64_t>& b);
|
||||
size_t broadcastIndex(size_t i, const std::vector<int64_t>& shape, const std::vector<int64_t>& output);
|
||||
std::optional<std::vector<RuntimeTensor>> executeIndex(const Node&, const std::vector<const RuntimeTensor*>&);
|
||||
std::optional<RuntimeTensor> executeShape(const Node&, const std::vector<const RuntimeTensor*>&);
|
||||
RuntimeTensor dispatchFloat(OnnxCompute&, const std::vector<const RuntimeTensor*>&, const std::vector<int64_t>&,
|
||||
const std::string&, size_t work = 0);
|
||||
std::optional<RuntimeTensor> executeMisc(const Node&, const std::vector<const RuntimeTensor*>&, OnnxCompute*);
|
||||
std::optional<RuntimeTensor> executeNeural(const Node&, const std::vector<const RuntimeTensor*>&, OnnxCompute*);
|
||||
std::optional<RuntimeTensor> executeNumeric(const Node&, const std::vector<const RuntimeTensor*>&, OnnxCompute*);
|
||||
std::vector<OnnxNamedTensor> evaluate(const ModelData&, std::span<const OnnxNamedTensor>,
|
||||
const std::vector<std::string>&, OnnxCompute*, OnnxRunOptions options);
|
||||
ModelData parse(std::span<const uint8_t> bytes);
|
||||
bool isSupported(const Node& node);
|
||||
std::vector<RuntimeTensor> execute(const Node& node, const std::vector<const RuntimeTensor*>& inputs,
|
||||
OnnxCompute* compute = nullptr);
|
||||
std::vector<RuntimeTensor> executeQuant(const Node& node, const std::vector<const RuntimeTensor*>& inputs,
|
||||
OnnxCompute* compute = nullptr);
|
||||
std::vector<RuntimeTensor> executeQuantLstm(const Node& node, const std::vector<const RuntimeTensor*>& inputs,
|
||||
OnnxCompute* compute = nullptr);
|
||||
} // namespace eve::tensor::onnx_detail
|
||||
@@ -0,0 +1,142 @@
|
||||
#include "tensor/AffineQuant.h"
|
||||
#include "tensor/OnnxGpuKernels.h"
|
||||
#include "tensor/OnnxInternal.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <cmath>
|
||||
#include <limits>
|
||||
|
||||
namespace eve::tensor::onnx_detail {
|
||||
namespace {
|
||||
template <class T>
|
||||
T checked(Result<T> r) {
|
||||
if (!r.ok()) throw Failure(r.error()->message(), r.error()->code());
|
||||
return std::move(r.value());
|
||||
}
|
||||
void shape(const RuntimeTensor& t, const std::vector<int64_t>& expected) {
|
||||
if (t.shape != expected) throw Failure("LSTM input shape mismatch");
|
||||
}
|
||||
std::vector<float> optionalFloats(const std::vector<const RuntimeTensor*>& in, size_t i,
|
||||
const std::vector<int64_t>& dims) {
|
||||
if (i >= in.size() || !in[i]) return std::vector<float>(count(dims), 0);
|
||||
shape(*in[i], dims);
|
||||
return floats(*in[i]);
|
||||
}
|
||||
struct Weights {
|
||||
const RuntimeTensor& tensor;
|
||||
std::vector<float> scales;
|
||||
std::vector<int64_t> zeros;
|
||||
size_t directions, gates, rows;
|
||||
Weights(const RuntimeTensor& t, const RuntimeTensor& s, const RuntimeTensor& z, size_t d, size_t g, size_t r)
|
||||
: tensor(t), scales(floats(s)), zeros(ints(z)), directions(d), gates(g), rows(r) {
|
||||
if (t.element != OnnxElement::Int8 && t.element != OnnxElement::UInt8)
|
||||
throw Failure("LSTM weights must be 8-bit");
|
||||
if (t.element != z.element || s.shape != z.shape ||
|
||||
(s.shape != std::vector<int64_t>{static_cast<int64_t>(d)} &&
|
||||
s.shape != std::vector<int64_t>{static_cast<int64_t>(d), static_cast<int64_t>(g)}))
|
||||
throw Failure("LSTM quantization parameter shape mismatch");
|
||||
shape(t, {static_cast<int64_t>(d), static_cast<int64_t>(r), static_cast<int64_t>(g)});
|
||||
for (float v : scales)
|
||||
if (!(v > 0) || !std::isfinite(v)) throw Failure("Invalid LSTM weight scale");
|
||||
}
|
||||
std::vector<float> multiply(const affine::QuantizedActivation& a, size_t batch, size_t direction,
|
||||
OnnxCompute* compute) const {
|
||||
std::vector<float> out(batch * gates);
|
||||
const bool sign = tensor.element == OnnxElement::Int8;
|
||||
if (compute) {
|
||||
std::vector<int32_t> z(scales.size() == directions ? 1 : gates);
|
||||
for (size_t i = 0; i < z.size(); ++i) z[i] = static_cast<int32_t>(zeros[direction * z.size() + i]);
|
||||
ByteStorage activation(a.values);
|
||||
ByteStorage accumulator(gpuMatmulResident(*compute, activation.buffer(), tensor.bytes.buffer(), false, sign,
|
||||
batch, rows, gates, a.zeroPoint, z, direction * rows * gates));
|
||||
std::vector<int32_t> sums(batch * gates);
|
||||
std::memcpy(sums.data(), static_cast<const ByteStorage&>(accumulator).data(),
|
||||
sums.size() * sizeof(int32_t));
|
||||
for (size_t b = 0; b < batch; ++b)
|
||||
for (size_t g = 0; g < gates; ++g)
|
||||
out[b * gates + g] = static_cast<float>(sums[b * gates + g]) * a.scale *
|
||||
scales[direction * z.size() + (z.size() == 1 ? 0 : g)];
|
||||
return out;
|
||||
}
|
||||
// Per-output-channel affine zero points do not require unpacking a weight matrix.
|
||||
for (size_t b = 0; b < batch; ++b)
|
||||
for (size_t g = 0; g < gates; ++g) {
|
||||
const size_t p = scales.size() == directions ? direction : direction * gates + g;
|
||||
int64_t sum = 0;
|
||||
for (size_t k = 0; k < rows; ++k) {
|
||||
const int byte = tensor.bytes[(direction * rows + k) * gates + g];
|
||||
const int w = sign && byte >= 128 ? byte - 256 : byte;
|
||||
sum += static_cast<int64_t>(int(a.values[b * rows + k]) - a.zeroPoint) * (w - zeros[p]);
|
||||
}
|
||||
if (sum < INT32_MIN || sum > INT32_MAX) throw Failure("LSTM integer accumulator overflow");
|
||||
out[b * gates + g] = static_cast<float>(sum) * a.scale * scales[p];
|
||||
}
|
||||
return out;
|
||||
}
|
||||
};
|
||||
} // namespace
|
||||
std::vector<RuntimeTensor> executeQuantLstm(const Node& n, const std::vector<const RuntimeTensor*>& in,
|
||||
OnnxCompute* compute) {
|
||||
const auto& xt = required(in, 0);
|
||||
if (xt.shape.size() != 3) throw Failure("LSTM X must be [time,batch,input]");
|
||||
const auto x = floats(xt);
|
||||
const int64_t time = xt.shape[0], batch = xt.shape[1], input = xt.shape[2], hidden = attr(n, "hidden_size", 0);
|
||||
const auto direction = n.attrs.contains("direction") ? n.attrs.at("direction").text : "forward";
|
||||
const int64_t dirs = direction == "bidirectional" ? 2 : 1;
|
||||
if (time <= 0 || batch <= 0 || input <= 0 || hidden <= 0 || hidden > 16384)
|
||||
throw Failure("Invalid LSTM dimensions");
|
||||
const size_t gates = 4 * hidden;
|
||||
const size_t projectionSize = count({time, batch, static_cast<int64_t>(gates)});
|
||||
if (projectionSize > 128u * 1024u * 1024u) throw Failure("LSTM projection exceeds memory limit");
|
||||
Weights w(required(in, 1), required(in, 8), required(in, 9), dirs, gates, input);
|
||||
Weights r(required(in, 2), required(in, 10), required(in, 11), dirs, gates, hidden);
|
||||
auto biases = optionalFloats(in, 3, {dirs, 8 * hidden});
|
||||
auto h = optionalFloats(in, 5, {dirs, batch, hidden}), c = optionalFloats(in, 6, {dirs, batch, hidden});
|
||||
auto p = optionalFloats(in, 7, {dirs, 3 * hidden});
|
||||
std::vector<int64_t> lengths(batch, time);
|
||||
if (in.size() > 4 && in[4]) {
|
||||
if (in[4]->element != OnnxElement::Int32) throw Failure("LSTM sequence lengths require int32");
|
||||
shape(*in[4], {batch});
|
||||
lengths = ints(*in[4]);
|
||||
}
|
||||
for (auto len : lengths)
|
||||
if (len < 0 || len > time) throw Failure("Invalid LSTM sequence length");
|
||||
const int64_t coupled = attr(n, "input_forget", 0);
|
||||
if (coupled != 0 && coupled != 1) throw Failure("Invalid coupled gate flag");
|
||||
const float clip = n.attrs.contains("clip") ? n.attrs.at("clip").real : std::numeric_limits<float>::max();
|
||||
if (!(clip > 0) || !std::isfinite(clip)) throw Failure("Invalid LSTM clipping threshold");
|
||||
auto sigmoid = [clip](float z) { return 1.f / (1.f + std::exp(-std::clamp(z, -clip, clip))); };
|
||||
auto tanh = [clip](float z) { return std::tanh(std::clamp(z, -clip, clip)); };
|
||||
std::vector<float> y(count({time, dirs, batch, hidden}), 0);
|
||||
const auto qx = checked(affine::dynamicQuantize(x));
|
||||
for (int64_t d = 0; d < dirs; ++d) {
|
||||
const auto projected = w.multiply(qx, time * batch, d, compute);
|
||||
const bool reverse = direction == "reverse" || d == 1;
|
||||
for (int64_t step = 0; step < time; ++step) {
|
||||
const auto qh = checked(affine::dynamicQuantize(std::span(h).subspan(d * batch * hidden, batch * hidden)));
|
||||
const auto recurrent = r.multiply(qh, batch, d, compute);
|
||||
for (int64_t b = 0; b < batch; ++b) {
|
||||
if (step >= lengths[b]) continue;
|
||||
const int64_t t = reverse ? lengths[b] - 1 - step : step;
|
||||
const size_t state = (d * batch + b) * hidden;
|
||||
for (int64_t j = 0; j < hidden; ++j) {
|
||||
auto gate = [&](int g) {
|
||||
const size_t channel = g * hidden + j;
|
||||
return projected[(t * batch + b) * gates + channel] + recurrent[b * gates + channel] +
|
||||
biases[d * 8 * hidden + channel] + biases[d * 8 * hidden + gates + channel];
|
||||
};
|
||||
const float i = sigmoid(gate(0) + p[d * 3 * hidden + j] * c[state + j]);
|
||||
const float f =
|
||||
coupled ? 1 - i : sigmoid(gate(2) + p[d * 3 * hidden + 2 * hidden + j] * c[state + j]);
|
||||
c[state + j] = f * c[state + j] + i * tanh(gate(3));
|
||||
const float o = sigmoid(gate(1) + p[d * 3 * hidden + hidden + j] * c[state + j]);
|
||||
h[state + j] = o * tanh(c[state + j]);
|
||||
y[((t * dirs + d) * batch + b) * hidden + j] = h[state + j];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return {make(OnnxElement::Float32, {time, dirs, batch, hidden}, y),
|
||||
make(OnnxElement::Float32, {dirs, batch, hidden}, h), make(OnnxElement::Float32, {dirs, batch, hidden}, c)};
|
||||
}
|
||||
} // namespace eve::tensor::onnx_detail
|
||||
@@ -0,0 +1,107 @@
|
||||
#include <algorithm>
|
||||
#include <limits>
|
||||
#include <sstream>
|
||||
#include "tensor/OnnxInternal.h"
|
||||
|
||||
namespace eve::tensor::onnx_detail {
|
||||
std::optional<RuntimeTensor> executeMisc(const Node& n, const std::vector<const RuntimeTensor*>& in,
|
||||
OnnxCompute* compute) {
|
||||
const auto& x = required(in, 0);
|
||||
if (n.op == "Clip") {
|
||||
if (x.element != OnnxElement::Float32)
|
||||
throw Failure("Clip currently requires FP32", DiagnosticCode::Unsupported);
|
||||
auto low = make(x.element, {}, std::vector<float>{-std::numeric_limits<float>::infinity()}),
|
||||
high = make(x.element, {}, std::vector<float>{std::numeric_limits<float>::infinity()});
|
||||
const auto& a = in.size() > 1 && in[1] ? *in[1] : low;
|
||||
const auto& b = in.size() > 2 && in[2] ? *in[2] : high;
|
||||
if (a.element != x.element || b.element != x.element || count(a.shape) != 1 || count(b.shape) != 1)
|
||||
throw Failure("Clip bounds must be matching scalars");
|
||||
if (compute) return dispatchFloat(*compute, {&x, &a, &b}, x.shape, "y[i]=min(max(x0[i],x1[0]),x2[0]);");
|
||||
auto v = floats(x);
|
||||
for (auto& f : v) f = std::min(std::max(f, read<float>(a, 0)), read<float>(b, 0));
|
||||
return make(x.element, x.shape, v);
|
||||
}
|
||||
if (n.op == "CumSum") {
|
||||
const auto& at = required(in, 1);
|
||||
if (count(at.shape) != 1) throw Failure("CumSum axis must be scalar");
|
||||
const int a = axis(integer(at), x.shape.size());
|
||||
if (x.element != OnnxElement::Float32)
|
||||
throw Failure("CumSum currently requires FP32", DiagnosticCode::Unsupported);
|
||||
const bool reverse = attr(n, "reverse", 0) != 0, exclusive = attr(n, "exclusive", 0) != 0;
|
||||
size_t outer = 1, inner = 1;
|
||||
for (int j = 0; j < a; ++j) outer *= x.shape[j];
|
||||
for (size_t j = a + 1; j < x.shape.size(); ++j) inner *= x.shape[j];
|
||||
const auto width = x.shape[a];
|
||||
if (compute && outer && inner && width) {
|
||||
std::ostringstream s;
|
||||
s << "uint base=i/" << inner << "u*" << inner * width << "u+i%" << inner
|
||||
<< "u;precise float sum=0.0;for(uint j=0;j<" << width << "u;++j){uint index=base+"
|
||||
<< (reverse ? "(" + std::to_string(width - 1) + "u-j)" : "j") << "*" << inner << "u;"
|
||||
<< (exclusive ? "y[index]=sum;sum+=x0[index];" : "sum+=x0[index];y[index]=sum;") << "}";
|
||||
return dispatchFloat(*compute, {&x}, x.shape, s.str(), outer * inner);
|
||||
}
|
||||
auto out = floats(x);
|
||||
for (size_t o = 0; o < outer; ++o)
|
||||
for (size_t i = 0; i < inner; ++i) {
|
||||
float sum = 0;
|
||||
for (int64_t j = 0; j < width; ++j) {
|
||||
size_t p = (o * width + (reverse ? width - 1 - j : j)) * inner + i;
|
||||
const float v = out[p];
|
||||
if (exclusive) {
|
||||
out[p] = sum;
|
||||
sum += v;
|
||||
} else {
|
||||
sum += v;
|
||||
out[p] = sum;
|
||||
}
|
||||
}
|
||||
}
|
||||
return make(x.element, x.shape, out);
|
||||
}
|
||||
if (n.op == "Pad") {
|
||||
const auto pads = ints(required(in, 1));
|
||||
if (pads.size() != 2 * x.shape.size()) throw Failure("Pad rank mismatch");
|
||||
auto shape = x.shape;
|
||||
const auto mode = n.attrs.contains("mode") ? n.attrs.at("mode").text : "constant";
|
||||
for (size_t j = 0; j < shape.size(); ++j) {
|
||||
if (pads[j] < -INT32_MAX || pads[j] > INT32_MAX || pads[j + shape.size()] < -INT32_MAX ||
|
||||
pads[j + shape.size()] > INT32_MAX)
|
||||
throw Failure("Pad extent out of range");
|
||||
shape[j] += pads[j] + pads[j + shape.size()];
|
||||
if (mode != "constant" && x.shape[j] <= 0)
|
||||
throw Failure("Edge/reflect padding requires nonempty dimensions");
|
||||
if (mode == "reflect" && (pads[j] >= x.shape[j] || pads[j + shape.size()] >= x.shape[j]))
|
||||
throw Failure("Reflect padding exceeds input dimension");
|
||||
}
|
||||
const auto size = elementSize(x.element);
|
||||
std::vector<uint8_t> fill(size, 0);
|
||||
if (in.size() > 2 && in[2]) {
|
||||
if (in[2]->element != x.element || count(in[2]->shape) != 1)
|
||||
throw Failure("Pad value dtype/shape mismatch");
|
||||
fill = in[2]->bytes;
|
||||
}
|
||||
RuntimeTensor out{x.element, shape, std::vector<uint8_t>(count(shape) * size)};
|
||||
for (size_t i = 0; i < count(shape); ++i) {
|
||||
size_t rest = i, source = 0, stride = 1;
|
||||
bool outside = false;
|
||||
for (size_t j = shape.size(); j > 0; --j) {
|
||||
int64_t p = static_cast<int64_t>(rest % shape[j - 1]) - pads[j - 1];
|
||||
rest /= shape[j - 1];
|
||||
if (p < 0 || p >= x.shape[j - 1]) {
|
||||
if (mode == "constant")
|
||||
outside = true;
|
||||
else if (mode == "edge")
|
||||
p = std::clamp(p, int64_t(0), x.shape[j - 1] - 1);
|
||||
else
|
||||
p = p < 0 ? -p : 2 * x.shape[j - 1] - 2 - p;
|
||||
}
|
||||
if (!outside) source += p * stride;
|
||||
stride *= x.shape[j - 1];
|
||||
}
|
||||
std::memcpy(out.bytes.data() + i * size, outside ? fill.data() : x.bytes.data() + source * size, size);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
return std::nullopt;
|
||||
}
|
||||
} // namespace eve::tensor::onnx_detail
|
||||
@@ -0,0 +1,94 @@
|
||||
#include "tensor/OnnxCompute.h"
|
||||
#include "tensor/OnnxInternal.h"
|
||||
|
||||
#include <set>
|
||||
|
||||
namespace eve::tensor {
|
||||
struct OnnxModel::Impl {
|
||||
onnx_detail::ModelData model;
|
||||
};
|
||||
OnnxModel::OnnxModel(std::unique_ptr<Impl> impl) : impl_(std::move(impl)) {}
|
||||
OnnxModel::~OnnxModel() = default;
|
||||
OnnxModelInfo OnnxModel::info() const { return impl_->model.info; }
|
||||
Result<std::unique_ptr<OnnxModel>> OnnxModel::load(std::span<const uint8_t> bytes) {
|
||||
try {
|
||||
auto impl = std::make_unique<Impl>();
|
||||
impl->model = onnx_detail::parse(bytes);
|
||||
return Result<std::unique_ptr<OnnxModel>>::success(std::unique_ptr<OnnxModel>(new OnnxModel(std::move(impl))));
|
||||
} catch (const onnx_detail::Failure& e) {
|
||||
return Result<std::unique_ptr<OnnxModel>>::failure(
|
||||
Diagnostic::error(e.code, e.what(), {}, {}, "tensor.onnx.import"));
|
||||
} catch (const std::exception& e) {
|
||||
return Result<std::unique_ptr<OnnxModel>>::failure(
|
||||
Diagnostic::error(DiagnosticCode::Failed, e.what(), {}, {}, "tensor.onnx.import"));
|
||||
}
|
||||
}
|
||||
Result<std::vector<OnnxNamedTensor>> OnnxModel::run(std::span<const OnnxNamedTensor> feeds,
|
||||
std::span<const std::string> requested,
|
||||
OnnxRunOptions options) const {
|
||||
return runInternal(feeds, requested, nullptr, options);
|
||||
}
|
||||
Result<OnnxGpuResult> OnnxModel::runGpu(std::span<const OnnxNamedTensor> feeds, OnnxCompute& compute,
|
||||
std::span<const std::string> requested, OnnxRunOptions options) const {
|
||||
class Counter final : public OnnxCompute {
|
||||
public:
|
||||
OnnxCompute& target;
|
||||
size_t count = 0;
|
||||
explicit Counter(OnnxCompute& t) : target(t) {}
|
||||
Result<std::vector<uint8_t>> dispatch(const OnnxKernel& k) override {
|
||||
auto r = target.dispatch(k);
|
||||
if (r.ok()) ++count;
|
||||
return r;
|
||||
}
|
||||
Result<OnnxBuffer> enqueue(const std::string& source, std::span<const OnnxBuffer> inputs, size_t bytes,
|
||||
uint32_t work) override {
|
||||
auto r = target.enqueue(source, inputs, bytes, work);
|
||||
if (r.ok()) ++count;
|
||||
return r;
|
||||
}
|
||||
} counter(compute);
|
||||
compute.beginRun();
|
||||
struct RunScope {
|
||||
OnnxCompute& device;
|
||||
~RunScope() { device.endRun(); }
|
||||
} scope{compute};
|
||||
auto r = runInternal(feeds, requested, &counter, options);
|
||||
if (!r.ok()) return Result<OnnxGpuResult>::failure(r.status());
|
||||
return Result<OnnxGpuResult>::success({std::move(r.value()), counter.count, compute.transferStats()});
|
||||
}
|
||||
Result<std::vector<OnnxNamedTensor>> OnnxModel::runInternal(std::span<const OnnxNamedTensor> feeds,
|
||||
std::span<const std::string> requested,
|
||||
OnnxCompute* compute, OnnxRunOptions options) const {
|
||||
using namespace onnx_detail;
|
||||
std::string nodeName;
|
||||
try {
|
||||
const auto& model = impl_->model;
|
||||
const std::vector<std::string> targets =
|
||||
requested.empty() ? model.info.outputs : std::vector<std::string>(requested.begin(), requested.end());
|
||||
std::set<std::string> seenFeeds;
|
||||
for (const auto& f : feeds) {
|
||||
nodeName = f.name;
|
||||
if (count(f.tensor.shape) * elementSize(f.tensor.element) != f.tensor.bytes.size())
|
||||
throw Failure("Tensor byte count mismatch");
|
||||
auto input = model.inputs.find(f.name);
|
||||
if (input == model.inputs.end() || !seenFeeds.insert(f.name).second)
|
||||
throw Failure("Unknown or duplicate feed");
|
||||
if (static_cast<int>(f.tensor.element) != input->second.type ||
|
||||
f.tensor.shape.size() != input->second.shape.size())
|
||||
throw Failure("Feed type/rank mismatch");
|
||||
for (size_t i = 0; i < f.tensor.shape.size(); ++i)
|
||||
if (input->second.shape[i] >= 0 && input->second.shape[i] != f.tensor.shape[i])
|
||||
throw Failure("Feed dimension mismatch");
|
||||
}
|
||||
nodeName.clear();
|
||||
auto result = evaluate(model, feeds, targets, compute, options);
|
||||
return Result<std::vector<OnnxNamedTensor>>::success(std::move(result));
|
||||
} catch (const Failure& e) {
|
||||
return Result<std::vector<OnnxNamedTensor>>::failure(
|
||||
Diagnostic::error(e.code, e.what(), e.path.empty() ? nodeName : e.path, {}, "tensor.onnx.run"));
|
||||
} catch (const std::exception& e) {
|
||||
return Result<std::vector<OnnxNamedTensor>>::failure(
|
||||
Diagnostic::error(DiagnosticCode::Failed, e.what(), nodeName, {}, "tensor.onnx.run"));
|
||||
}
|
||||
}
|
||||
} // namespace eve::tensor
|
||||
@@ -0,0 +1,106 @@
|
||||
#pragma once
|
||||
|
||||
#include "common/Result.h"
|
||||
#include "tensor/OnnxStorage.h"
|
||||
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <memory>
|
||||
#include <span>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
namespace eve::tensor {
|
||||
class OnnxCompute;
|
||||
/** @brief ONNX wire element types; distinct from block-quantized Tensor storage. */
|
||||
enum class OnnxElement : int { Float32 = 1, UInt8 = 2, Int8 = 3, Int32 = 6, Int64 = 7, Bool = 9 };
|
||||
|
||||
/**
|
||||
* @brief Owning ONNX boundary tensor: row-major little-endian bytes and exact integer shape.
|
||||
* @note No borrowed storage. run() validates caller-provided values before execution.
|
||||
* Scalar shape is empty; zero-sized dimensions are supported. No implicit FP32 integer conversion.
|
||||
*/
|
||||
struct OnnxTensor {
|
||||
OnnxElement element = OnnxElement::Float32;
|
||||
std::vector<int64_t> shape;
|
||||
std::vector<uint8_t> bytes;
|
||||
};
|
||||
/** @brief Owning named feed/output value. */
|
||||
struct OnnxNamedTensor {
|
||||
std::string name;
|
||||
OnnxTensor tensor;
|
||||
};
|
||||
/** @brief Per-call deterministic RNG and optional strict finite-output diagnostic.
|
||||
* @note requireFinite rejects even intentional intermediate infinities/NaNs, so keep it false for Kokoro. */
|
||||
struct OnnxRunOptions {
|
||||
uint64_t seed = 0;
|
||||
bool requireFinite = false;
|
||||
};
|
||||
/** @brief Owning GPU execution output and completed dispatch count. */
|
||||
struct OnnxGpuResult {
|
||||
std::vector<OnnxNamedTensor> outputs;
|
||||
size_t dispatches = 0;
|
||||
OnnxTransferStats transfers;
|
||||
};
|
||||
/** @brief Owning admission report; unsupported nodes remain inspectable but cannot execute. */
|
||||
struct OnnxModelInfo {
|
||||
int64_t irVersion = 0;
|
||||
size_t nodeCount = 0, initializerBytes = 0;
|
||||
std::vector<std::string> inputs, outputs;
|
||||
std::vector<std::string> unsupportedNodes;
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief Native ONNX import and CPU/GPU execution using tensor kernels, without ONNX Runtime.
|
||||
* @note Owns immutable model data. Concurrent CPU run calls are independent; no script callbacks,
|
||||
* global RNG or background threads. runGpu uses a caller-provided device on its owning thread.
|
||||
* FP32 results use tolerance comparison. Seeded random excitation is local to a run.
|
||||
* Loading is transactional. Unknown protobuf metadata is ignored, unknown operators
|
||||
* are reported; unsupported semantic features fail explicitly, never become identity.
|
||||
*/
|
||||
class OnnxModel {
|
||||
public:
|
||||
/** @brief Destroy model-owned graph and packed initializers; outputs remain valid. */
|
||||
~OnnxModel();
|
||||
/**
|
||||
* @brief Import bounded in-memory ONNX ModelProto; copies all retained data.
|
||||
* @param bytes Borrowed input, not retained. Maximum 512 MiB, 100000 recursive nodes, graph depth 16, rank 6.
|
||||
* @return Owning model or ParseError/Unsupported/UnknownVersion; no partial publication.
|
||||
* @note Supports ONNX IR 3..10 and default-domain opset 13..17. External data is rejected.
|
||||
*/
|
||||
[[nodiscard]] static Result<std::unique_ptr<OnnxModel>> load(std::span<const uint8_t> bytes);
|
||||
/** @brief Return owning graph diagnostics; importing does not imply full operator support. */
|
||||
[[nodiscard]] OnnxModelInfo info() const;
|
||||
/**
|
||||
* @brief Execute selected named values, or graph outputs when requested is empty.
|
||||
* @param feeds Borrowed inputs, valid only during this call; never mutated or retained.
|
||||
* @param requested Borrowed output names; intermediate outputs may be selected for parity tests.
|
||||
* @return Owning outputs or structured node diagnostic; failed calls publish no outputs.
|
||||
* @note CPU only. Unsupported dependencies fail before executing the selected subgraph.
|
||||
*/
|
||||
[[nodiscard]] Result<std::vector<OnnxNamedTensor>> run(std::span<const OnnxNamedTensor> feeds,
|
||||
std::span<const std::string> requested = {},
|
||||
OnnxRunOptions options = {}) const;
|
||||
|
||||
/**
|
||||
* @brief Execute with GPU matrix, convolution, normalization, activation and resampling kernels.
|
||||
* @note Shape/index/control operations, padding and LSTM gates execute on CPU.
|
||||
* Dynamic quantization reads three validation scalars; activations remain on GPU.
|
||||
* Integer GEMM/Conv, LSTM projections, FP32 GEMM/ConvTranspose, elementwise math,
|
||||
* Softmax, sum/mean reductions, normalization, Resize and CumSum execute on GPU.
|
||||
* The borrowed compute provider must outlive this synchronous call; use its device thread.
|
||||
* No automatic CPU retry occurs after a GPU error. Outputs own their storage.
|
||||
*/
|
||||
[[nodiscard]] Result<OnnxGpuResult> runGpu(std::span<const OnnxNamedTensor> feeds, OnnxCompute& compute,
|
||||
std::span<const std::string> requested = {},
|
||||
OnnxRunOptions options = {}) const;
|
||||
|
||||
private:
|
||||
[[nodiscard]] Result<std::vector<OnnxNamedTensor>> runInternal(std::span<const OnnxNamedTensor> feeds,
|
||||
std::span<const std::string> requested,
|
||||
OnnxCompute* compute, OnnxRunOptions options) const;
|
||||
struct Impl;
|
||||
explicit OnnxModel(std::unique_ptr<Impl> impl);
|
||||
std::unique_ptr<Impl> impl_;
|
||||
};
|
||||
} // namespace eve::tensor
|
||||
@@ -0,0 +1,190 @@
|
||||
#include <algorithm>
|
||||
#include <cmath>
|
||||
#include <sstream>
|
||||
#include "tensor/OnnxInternal.h"
|
||||
|
||||
namespace eve::tensor::onnx_detail {
|
||||
std::optional<RuntimeTensor> executeNeural(const Node& n, const std::vector<const RuntimeTensor*>& in,
|
||||
OnnxCompute* compute) {
|
||||
const auto& x = required(in, 0);
|
||||
if (x.element != OnnxElement::Float32) return std::nullopt;
|
||||
if (n.op == "InstanceNormalization") {
|
||||
const auto& scale = required(in, 1);
|
||||
const auto& bias = required(in, 2);
|
||||
if (x.shape.size() < 3 || scale.shape != std::vector<int64_t>{x.shape[1]} || bias.shape != scale.shape ||
|
||||
scale.element != x.element || bias.element != x.element)
|
||||
throw Failure("InstanceNormalization shape/type mismatch");
|
||||
const float epsilon = n.attrs.contains("epsilon") ? n.attrs.at("epsilon").real : 1e-5f;
|
||||
if (!(epsilon > 0) || !std::isfinite(epsilon)) throw Failure("Invalid normalization epsilon");
|
||||
size_t spatial = 1;
|
||||
for (size_t j = 2; j < x.shape.size(); ++j) spatial *= x.shape[j];
|
||||
if (!spatial) throw Failure("Empty InstanceNormalization");
|
||||
if (compute) {
|
||||
std::ostringstream s;
|
||||
s.precision(9);
|
||||
s << std::scientific;
|
||||
s << "uint base=i*" << spatial << "u;precise float mean=0.0;for(uint j=0;j<" << spatial
|
||||
<< "u;++j)mean+=x0[base+j];mean/=" << spatial << ".0;precise float var=0.0;for(uint j=0;j<" << spatial
|
||||
<< "u;++j){float d=x0[base+j]-mean;var+=d*d;}float inv=inversesqrt(var/" << spatial << ".0+" << epsilon
|
||||
<< ");for(uint j=0;j<" << spatial << "u;++j)y[base+j]=(x0[base+j]-mean)*inv*x1[i%" << x.shape[1]
|
||||
<< "u]+x2[i%" << x.shape[1] << "u];";
|
||||
return dispatchFloat(*compute, {&x, &scale, &bias}, x.shape, s.str(), count(x.shape) / spatial);
|
||||
}
|
||||
auto out = floats(x);
|
||||
for (size_t i = 0; i < out.size() / spatial; ++i) {
|
||||
float mean = 0, var = 0;
|
||||
for (size_t j = 0; j < spatial; ++j) mean += out[i * spatial + j];
|
||||
mean /= spatial;
|
||||
for (size_t j = 0; j < spatial; ++j) {
|
||||
float d = out[i * spatial + j] - mean;
|
||||
var += d * d;
|
||||
}
|
||||
const float multiplier = read<float>(scale, i % x.shape[1]) / std::sqrt(var / spatial + epsilon),
|
||||
offset = read<float>(bias, i % x.shape[1]);
|
||||
for (size_t j = 0; j < spatial; ++j)
|
||||
out[i * spatial + j] = (out[i * spatial + j] - mean) * multiplier + offset;
|
||||
}
|
||||
return make(x.element, x.shape, out);
|
||||
}
|
||||
if (n.op == "ConvTranspose") {
|
||||
const auto& w = required(in, 1);
|
||||
if (x.shape.size() != 3 || w.shape.size() != 3 || w.element != x.element || w.shape[0] != x.shape[1])
|
||||
throw Failure("ConvTranspose requires NCL / CIO weights", DiagnosticCode::Unsupported);
|
||||
const auto strides = attrs(n, "strides", {1}), pads = attrs(n, "pads", {0, 0}),
|
||||
dilations = attrs(n, "dilations", {1}), extra = attrs(n, "output_padding", {0});
|
||||
if (strides.size() != 1 || pads.size() != 2 || dilations.size() != 1 || extra.size() != 1)
|
||||
throw Failure("ConvTranspose attribute rank mismatch");
|
||||
const int64_t stride = strides[0], dilation = dilations[0], groups = attr(n, "group", 1), channels = x.shape[1],
|
||||
length = x.shape[2], kernel = w.shape[2], cout = w.shape[1];
|
||||
if (stride <= 0 || stride > 65536 || dilation <= 0 || dilation > 65536 || groups <= 0 || channels % groups ||
|
||||
pads[0] < 0 || pads[1] < 0 || extra[0] < 0 || extra[0] >= std::max(stride, dilation) || length <= 0 ||
|
||||
kernel <= 0 || cout <= 0)
|
||||
throw Failure("Invalid ConvTranspose geometry");
|
||||
if (n.attrs.contains("kernel_shape") && attrs(n, "kernel_shape", {}) != std::vector<int64_t>{kernel})
|
||||
throw Failure("ConvTranspose kernel_shape mismatch");
|
||||
const int64_t width = stride * (length - 1) + extra[0] + dilation * (kernel - 1) + 1 - pads[0] - pads[1],
|
||||
outputs = cout * groups;
|
||||
if (width <= 0 || width > INT32_MAX) throw Failure("Invalid ConvTranspose output extent");
|
||||
std::vector<int64_t> shape{x.shape[0], outputs, width};
|
||||
count(shape);
|
||||
RuntimeTensor zeroBias = make(OnnxElement::Float32, {outputs}, std::vector<float>(outputs, 0));
|
||||
const auto& bias = in.size() > 2 && in[2] ? *in[2] : zeroBias;
|
||||
if (bias.element != x.element || bias.shape != zeroBias.shape) throw Failure("ConvTranspose bias mismatch");
|
||||
if (compute) {
|
||||
std::ostringstream s;
|
||||
s << "uint pos=i%" << width << "u,o=i/" << width << "u%" << outputs << "u,batch=i/" << width * outputs
|
||||
<< "u;precise float v=x2[o];for(uint c=0;c<" << channels / groups << "u;++c)for(uint j=0;j<" << kernel
|
||||
<< "u;++j){int z=int(pos)+" << pads[0] << "-int(j)*" << dilation << ";if(z<0||z%" << stride << "!=0||z/"
|
||||
<< stride << ">=" << length << ")continue;uint ch=o/" << cout << "u*" << channels / groups
|
||||
<< "u+c;v+=x0[(batch*" << channels << "u+ch)*" << length << "u+uint(z/" << stride << ")]*x1[(ch*" << cout
|
||||
<< "u+o%" << cout << "u)*" << kernel << "u+j];}y[i]=v;";
|
||||
return dispatchFloat(*compute, {&x, &w, &bias}, shape, s.str());
|
||||
}
|
||||
std::vector<float> out(count(shape));
|
||||
for (int64_t batch = 0; batch < x.shape[0]; ++batch)
|
||||
for (int64_t o = 0; o < outputs; ++o)
|
||||
for (int64_t pos = 0; pos < width; ++pos) {
|
||||
float v = read<float>(bias, o);
|
||||
for (int64_t c = 0; c < channels / groups; ++c)
|
||||
for (int64_t j = 0; j < kernel; ++j) {
|
||||
int64_t z = pos + pads[0] - j * dilation;
|
||||
if (z < 0 || z % stride || z / stride >= length) continue;
|
||||
int64_t ch = o / cout * (channels / groups) + c;
|
||||
v += read<float>(x, (batch * channels + ch) * length + z / stride) *
|
||||
read<float>(w, (ch * cout + o % cout) * kernel + j);
|
||||
}
|
||||
out[(batch * outputs + o) * width + pos] = v;
|
||||
}
|
||||
return make(x.element, shape, out);
|
||||
}
|
||||
if (n.op == "Resize") {
|
||||
if (in.size() > 1 && in[1] && !in[1]->bytes.empty())
|
||||
throw Failure("Resize ROI unsupported", DiagnosticCode::Unsupported);
|
||||
std::vector<double> scales;
|
||||
auto shape = x.shape;
|
||||
if (in.size() > 2 && in[2] && !in[2]->bytes.empty()) {
|
||||
auto s = floats(*in[2]);
|
||||
scales.assign(s.begin(), s.end());
|
||||
if (scales.size() != shape.size()) throw Failure("Resize scales rank mismatch");
|
||||
for (size_t j = 0; j < shape.size(); ++j) {
|
||||
if (!(scales[j] > 0) || !std::isfinite(scales[j]) || std::floor(shape[j] * scales[j]) > INT32_MAX)
|
||||
throw Failure("Invalid Resize scale");
|
||||
shape[j] = static_cast<int64_t>(std::floor(shape[j] * scales[j]));
|
||||
}
|
||||
} else {
|
||||
shape = ints(required(in, 3));
|
||||
if (shape.size() != x.shape.size()) throw Failure("Resize sizes rank mismatch");
|
||||
for (size_t j = 0; j < shape.size(); ++j) {
|
||||
if (x.shape[j] <= 0) throw Failure("Empty Resize input");
|
||||
scales.push_back(double(shape[j]) / x.shape[j]);
|
||||
}
|
||||
}
|
||||
count(shape);
|
||||
const auto mode = n.attrs.contains("mode") ? n.attrs.at("mode").text : "nearest",
|
||||
coordinate = n.attrs.contains("coordinate_transformation_mode")
|
||||
? n.attrs.at("coordinate_transformation_mode").text
|
||||
: "half_pixel";
|
||||
for (auto d : x.shape)
|
||||
if (d <= 0) throw Failure("Empty Resize input");
|
||||
if (mode == "nearest") {
|
||||
if (compute) {
|
||||
std::ostringstream s;
|
||||
s.precision(17);
|
||||
s << "uint rest=i,source=0u;";
|
||||
size_t stride = 1;
|
||||
for (size_t j = shape.size(); j > 0; --j) {
|
||||
s << "source+=uint(min(floor(float(rest%" << shape[j - 1] << "u)/" << std::scientific
|
||||
<< scales[j - 1] << ")," << double(x.shape[j - 1] - 1) << "))*" << stride
|
||||
<< "u;rest/=" << shape[j - 1] << "u;";
|
||||
stride *= x.shape[j - 1];
|
||||
}
|
||||
s << "y[i]=x0[source];";
|
||||
return dispatchFloat(*compute, {&x}, shape, s.str());
|
||||
}
|
||||
std::vector<float> out(count(shape));
|
||||
for (size_t i = 0; i < out.size(); ++i) {
|
||||
size_t rest = i, source = 0, stride = 1;
|
||||
for (size_t j = shape.size(); j > 0; --j) {
|
||||
source += std::min(int64_t(std::floor((rest % shape[j - 1]) / scales[j - 1])), x.shape[j - 1] - 1) *
|
||||
stride;
|
||||
rest /= shape[j - 1];
|
||||
stride *= x.shape[j - 1];
|
||||
}
|
||||
out[i] = read<float>(x, source);
|
||||
}
|
||||
return make(x.element, shape, out);
|
||||
}
|
||||
// The speech model interpolates its final axis only. Reject other geometries explicitly.
|
||||
for (size_t j = 0; j + 1 < shape.size(); ++j)
|
||||
if (shape[j] != x.shape[j])
|
||||
throw Failure("Linear Resize supports final axis only", DiagnosticCode::Unsupported);
|
||||
if (shape.empty()) throw Failure("Resize scalar unsupported");
|
||||
const auto width = shape.back(), old = x.shape.back();
|
||||
if (!width) return RuntimeTensor{x.element, shape, {}};
|
||||
auto position = [&](int64_t p) {
|
||||
return coordinate == "half_pixel" ? (p + .5) / scales.back() - .5 : p / scales.back();
|
||||
};
|
||||
if (compute) {
|
||||
std::ostringstream s;
|
||||
s.precision(17);
|
||||
s << std::scientific;
|
||||
s << "uint base=i/" << width << "u*" << old
|
||||
<< "u;float p=" << (coordinate == "half_pixel" ? "(float(i%" : "float(i%") << width
|
||||
<< (coordinate == "half_pixel" ? "u)+0.5)/" : "u)/") << scales.back()
|
||||
<< (coordinate == "half_pixel" ? "-0.5" : "") << ";float t=floor(p),f=p-t;uint a=uint(clamp(t,0.0,"
|
||||
<< double(old - 1) << ")),b=uint(clamp(t+1.0,0.0," << double(old - 1)
|
||||
<< "));y[i]=x0[base+a]*(1.0-f)+x0[base+b]*f;";
|
||||
return dispatchFloat(*compute, {&x}, shape, s.str());
|
||||
}
|
||||
std::vector<float> out(count(shape));
|
||||
for (size_t i = 0; i < out.size(); ++i) {
|
||||
double p = position(i % width), lo = std::floor(p), f = p - lo;
|
||||
size_t base = i / width * old;
|
||||
out[i] = static_cast<float>(read<float>(x, base + std::clamp(int64_t(lo), int64_t(0), old - 1)) * (1 - f) +
|
||||
read<float>(x, base + std::clamp(int64_t(lo + 1), int64_t(0), old - 1)) * f);
|
||||
}
|
||||
return make(x.element, shape, out);
|
||||
}
|
||||
return std::nullopt;
|
||||
}
|
||||
} // namespace eve::tensor::onnx_detail
|
||||
@@ -0,0 +1,257 @@
|
||||
#include <algorithm>
|
||||
#include <cmath>
|
||||
#include <limits>
|
||||
#include <set>
|
||||
#include <sstream>
|
||||
#include "tensor/OnnxCompute.h"
|
||||
#include "tensor/OnnxInternal.h"
|
||||
|
||||
namespace eve::tensor::onnx_detail {
|
||||
namespace {
|
||||
std::string literal(float v) {
|
||||
std::ostringstream s;
|
||||
s.precision(9);
|
||||
s << std::scientific << v;
|
||||
return s.str();
|
||||
}
|
||||
std::string indexCode(const std::vector<int64_t>& input, const std::vector<int64_t>& output, const char* index) {
|
||||
std::string result = "0u";
|
||||
size_t src = 1, dst = 1;
|
||||
for (size_t j = 0; j < output.size(); ++j) {
|
||||
if (j < input.size()) {
|
||||
const auto dim = input[input.size() - 1 - j];
|
||||
if (dim != 1)
|
||||
result += "+(" + std::string(index) + "/" + std::to_string(dst) + "u%" + std::to_string(dim) + "u)*" +
|
||||
std::to_string(src) + "u";
|
||||
src *= dim;
|
||||
}
|
||||
dst *= output[output.size() - 1 - j];
|
||||
}
|
||||
return result;
|
||||
}
|
||||
} // namespace
|
||||
RuntimeTensor dispatchFloat(OnnxCompute& compute, const std::vector<const RuntimeTensor*>& inputs,
|
||||
const std::vector<int64_t>& shape, const std::string& body, size_t work) {
|
||||
RuntimeTensor out{OnnxElement::Float32, shape, std::vector<uint8_t>(count(shape) * 4)};
|
||||
if (out.bytes.empty()) return out;
|
||||
if (!work) work = count(shape);
|
||||
if (work > 65535u * 64u) throw Failure("GPU float dispatch limit exceeded", DiagnosticCode::Unsupported);
|
||||
OnnxKernel k;
|
||||
std::vector<OnnxBuffer> buffers;
|
||||
k.outputBytes = out.bytes.size();
|
||||
k.workItems = static_cast<uint32_t>(work);
|
||||
k.source = "#version 450\nlayout(local_size_x=64)in;\n";
|
||||
for (size_t i = 0; i < inputs.size(); ++i) {
|
||||
if (inputs[i]->element != OnnxElement::Float32 && inputs[i]->element != OnnxElement::Int32)
|
||||
throw Failure("GPU float input type mismatch");
|
||||
buffers.push_back(inputs[i]->bytes.buffer());
|
||||
k.source += "layout(std430,binding=" + std::to_string(i) + ")readonly buffer B" + std::to_string(i) +
|
||||
(inputs[i]->element == OnnxElement::Int32 ? "{int x" : "{float x") + std::to_string(i) + "[];};\n";
|
||||
}
|
||||
k.source +=
|
||||
"layout(std430,binding=" + std::to_string(inputs.size()) +
|
||||
")writeonly buffer Y{float y[];};\nvoid main(){uint i=gl_GlobalInvocationID.x;if(i>=" + std::to_string(work) +
|
||||
"u)return;" + body + "}";
|
||||
auto r = compute.enqueue(k.source, buffers, k.outputBytes, k.workItems);
|
||||
if (!r.ok()) throw Failure(r.error()->message(), r.error()->code());
|
||||
if (r.value().size != out.bytes.size()) throw Failure("GPU output size mismatch");
|
||||
out.bytes = std::move(r.value());
|
||||
return out;
|
||||
}
|
||||
std::optional<RuntimeTensor> executeNumeric(const Node& n, const std::vector<const RuntimeTensor*>& in,
|
||||
OnnxCompute* compute) {
|
||||
const auto& x = required(in, 0);
|
||||
if (x.element != OnnxElement::Float32) return std::nullopt;
|
||||
if (n.op == "MatMul") {
|
||||
const auto& b = required(in, 1);
|
||||
if (b.element != x.element || x.shape.empty() || b.shape.empty()) throw Failure("MatMul type/rank mismatch");
|
||||
auto as = x.shape, bs = b.shape;
|
||||
const bool av = as.size() == 1, bv = bs.size() == 1;
|
||||
if (av) as.insert(as.begin(), 1);
|
||||
if (bv) bs.push_back(1);
|
||||
const auto m = as[as.size() - 2], k = as.back(), cols = bs.back();
|
||||
if (k != bs[bs.size() - 2]) throw Failure("MatMul inner dimension mismatch");
|
||||
std::vector<int64_t> ab(as.begin(), as.end() - 2), bb(bs.begin(), bs.end() - 2);
|
||||
auto batches = broadcast(ab, bb);
|
||||
auto shape = batches;
|
||||
shape.push_back(m);
|
||||
shape.push_back(cols);
|
||||
count(shape);
|
||||
if (av) shape.erase(shape.end() - 2);
|
||||
if (bv) shape.pop_back();
|
||||
if (compute && m && cols) {
|
||||
std::string body = "uint batch=i/" + std::to_string(m * cols) + "u,r=i/" + std::to_string(cols) + "u%" +
|
||||
std::to_string(m) + "u,c=i%" + std::to_string(cols) +
|
||||
"u;precise float v=0.0;for(uint j=0;j<" + std::to_string(k) + "u;++j)v+=x0[(" +
|
||||
indexCode(ab, batches, "batch") + ")*" + std::to_string(m * k) + "u+r*" +
|
||||
std::to_string(k) + "u+j]*x1[(" + indexCode(bb, batches, "batch") + ")*" +
|
||||
std::to_string(k * cols) + "u+j*" + std::to_string(cols) + "u+c];y[i]=v;";
|
||||
return dispatchFloat(*compute, {&x, &b}, shape, body);
|
||||
}
|
||||
std::vector<float> out(count(shape), 0);
|
||||
for (size_t batch = 0; batch < count(batches); ++batch)
|
||||
for (int64_t r = 0; r < m; ++r)
|
||||
for (int64_t c = 0; c < cols; ++c) {
|
||||
float sum = 0;
|
||||
for (int64_t j = 0; j < k; ++j)
|
||||
sum += read<float>(x, broadcastIndex(batch, ab, batches) * m * k + r * k + j) *
|
||||
read<float>(b, broadcastIndex(batch, bb, batches) * k * cols + j * cols + c);
|
||||
out[(batch * m + r) * cols + c] = sum;
|
||||
}
|
||||
return make(OnnxElement::Float32, shape, out);
|
||||
}
|
||||
static const std::set<std::string> binary{"Add", "Sub", "Mul", "Div", "Pow"};
|
||||
if (binary.contains(n.op)) {
|
||||
const auto& b = required(in, 1);
|
||||
if (b.element != x.element) throw Failure("Float arithmetic dtype mismatch");
|
||||
auto shape = broadcast(x.shape, b.shape);
|
||||
if (compute) {
|
||||
const std::string a = "x0[" + indexCode(x.shape, shape, "i") + "]",
|
||||
v = "x1[" + indexCode(b.shape, shape, "i") + "]";
|
||||
const std::string e = n.op == "Pow" ? "pow(" + a + "," + v + ")"
|
||||
: a +
|
||||
(n.op == "Add" ? "+"
|
||||
: n.op == "Sub" ? "-"
|
||||
: n.op == "Mul" ? "*"
|
||||
: "/") +
|
||||
v;
|
||||
if (n.op == "Pow")
|
||||
return dispatchFloat(*compute, {&x, &b}, shape,
|
||||
"float a=" + a + ",b=" + v +
|
||||
";float value;if(b==0.0)value=1.0;else "
|
||||
"if(a==0.0)value=b>0.0?0.0:uintBitsToFloat(0x7f800000u);else "
|
||||
"if(a<0.0)value=b==floor(b)?pow(-a,b)*(mod(abs(b),2.0)==0.0?1.0:-1.0):"
|
||||
"uintBitsToFloat(0x7fc00000u);else value=pow(a,b);y[i]=value;");
|
||||
return dispatchFloat(*compute, {&x, &b}, shape, "y[i]=" + e + ";");
|
||||
}
|
||||
std::vector<float> out(count(shape));
|
||||
for (size_t i = 0; i < out.size(); ++i) {
|
||||
const float a = read<float>(x, broadcastIndex(i, x.shape, shape)),
|
||||
v = read<float>(b, broadcastIndex(i, b.shape, shape));
|
||||
out[i] = n.op == "Add" ? a + v
|
||||
: n.op == "Sub" ? a - v
|
||||
: n.op == "Mul" ? a * v
|
||||
: n.op == "Div" ? a / v
|
||||
: std::pow(a, v);
|
||||
}
|
||||
return make(OnnxElement::Float32, shape, out);
|
||||
}
|
||||
if (n.op == "ReduceMean" || n.op == "ReduceSum") {
|
||||
auto axes = n.op == "ReduceMean" ? attrs(n, "axes", {})
|
||||
: in.size() > 1 && in[1] ? ints(*in[1])
|
||||
: std::vector<int64_t>{};
|
||||
if (axes.empty() && n.op == "ReduceSum" && attr(n, "noop_with_empty_axes", 0)) return x;
|
||||
if (axes.empty())
|
||||
for (size_t i = 0; i < x.shape.size(); ++i) axes.push_back(i);
|
||||
std::set<int> set;
|
||||
for (auto a : axes)
|
||||
if (!set.insert(axis(a, x.shape.size())).second) throw Failure("Duplicate reduction axis");
|
||||
std::vector<int64_t> outerShape, reducedShape, shape;
|
||||
std::vector<size_t> outerStrides, reducedStrides;
|
||||
size_t stride = count(x.shape);
|
||||
for (size_t j = 0; j < x.shape.size(); ++j) {
|
||||
if (!x.shape[j]) throw Failure("Empty reduction unsupported", DiagnosticCode::Unsupported);
|
||||
stride /= x.shape[j];
|
||||
if (set.contains(static_cast<int>(j))) {
|
||||
reducedShape.push_back(x.shape[j]);
|
||||
reducedStrides.push_back(stride);
|
||||
if (attr(n, "keepdims", 1)) shape.push_back(1);
|
||||
} else {
|
||||
outerShape.push_back(x.shape[j]);
|
||||
outerStrides.push_back(stride);
|
||||
shape.push_back(x.shape[j]);
|
||||
}
|
||||
}
|
||||
auto offset = [](size_t i, const std::vector<int64_t>& dims, const std::vector<size_t>& strides) {
|
||||
size_t p = 0;
|
||||
for (size_t j = dims.size(); j > 0; --j) {
|
||||
p += (i % dims[j - 1]) * strides[j - 1];
|
||||
i /= dims[j - 1];
|
||||
}
|
||||
return p;
|
||||
};
|
||||
auto code = [](const char* var, const std::vector<int64_t>& dims, const std::vector<size_t>& strides) {
|
||||
std::string s = "0u";
|
||||
size_t divisor = 1;
|
||||
for (size_t j = dims.size(); j > 0; --j) {
|
||||
s += "+(" + std::string(var) + "/" + std::to_string(divisor) + "u%" + std::to_string(dims[j - 1]) +
|
||||
"u)*" + std::to_string(strides[j - 1]) + "u";
|
||||
divisor *= dims[j - 1];
|
||||
}
|
||||
return s;
|
||||
};
|
||||
const size_t terms = count(reducedShape);
|
||||
if (compute)
|
||||
return dispatchFloat(*compute, {&x}, shape,
|
||||
"uint base=" + code("i", outerShape, outerStrides) +
|
||||
";precise float v=0.0;for(uint j=0;j<" + std::to_string(terms) +
|
||||
"u;++j)v+=x0[base+(" + code("j", reducedShape, reducedStrides) + ")];y[i]=v" +
|
||||
(n.op == "ReduceMean" ? "/" + std::to_string(terms) + ".0" : "") + ";");
|
||||
std::vector<float> out(count(shape));
|
||||
for (size_t i = 0; i < out.size(); ++i) {
|
||||
float v = 0;
|
||||
for (size_t j = 0; j < terms; ++j)
|
||||
v += read<float>(x, offset(i, outerShape, outerStrides) + offset(j, reducedShape, reducedStrides));
|
||||
out[i] = n.op == "ReduceMean" ? v / terms : v;
|
||||
}
|
||||
return make(OnnxElement::Float32, shape, out);
|
||||
}
|
||||
if (n.op == "Softmax" && compute) {
|
||||
int a = axis(attr(n, "axis", -1), x.shape.size());
|
||||
size_t inner = 1, outer = 1;
|
||||
for (int j = 0; j < a; ++j) outer *= x.shape[j];
|
||||
for (size_t j = a + 1; j < x.shape.size(); ++j) inner *= x.shape[j];
|
||||
const auto channels = x.shape[a];
|
||||
if (!channels || !inner) return RuntimeTensor{x.element, x.shape, {}};
|
||||
auto ch = std::to_string(channels), ins = std::to_string(inner);
|
||||
return dispatchFloat(*compute, {&x}, x.shape,
|
||||
"uint base=i/" + ins + "u*" + ins + "u*" + ch + "u+i%" + ins +
|
||||
"u;float m=-3.402823466e38;for(uint j=0;j<" + ch + "u;++j)m=max(m,x0[base+j*" + ins +
|
||||
"u]);float sum=0.0;for(uint j=0;j<" + ch + "u;++j)sum+=exp(x0[base+j*" + ins +
|
||||
"u]-m);for(uint j=0;j<" + ch + "u;++j)y[base+j*" + ins + "u]=exp(x0[base+j*" + ins +
|
||||
"u]-m)/sum;",
|
||||
outer * inner);
|
||||
}
|
||||
static const std::map<std::string, std::string> unary{{"Sqrt", "sqrt(v)"},
|
||||
{"Exp", "exp(v)"},
|
||||
{"Log", "log(v)"},
|
||||
{"Sin", "sin(v)"},
|
||||
{"Cos", "cos(v)"},
|
||||
{"Relu", "max(v,0.0)"},
|
||||
{"Sigmoid", "1.0/(1.0+exp(-v))"},
|
||||
{"Tanh", "isnan(v)?v:tanh(clamp(v,-10.0,10.0))"},
|
||||
{"Neg", "-v"},
|
||||
{"Abs", "abs(v)"},
|
||||
{"Reciprocal", "1.0/v"},
|
||||
{"Floor", "floor(v)"},
|
||||
{"Round", "roundEven(v)"},
|
||||
{"Atan", "isinf(v)?sign(v)*1.5707963267948966:atan(v)"}};
|
||||
if (n.op == "LeakyRelu" || unary.contains(n.op)) {
|
||||
const float alpha = n.attrs.contains("alpha") ? n.attrs.at("alpha").real : 0.01f;
|
||||
if (!std::isfinite(alpha)) throw Failure("Invalid LeakyRelu alpha");
|
||||
if (compute)
|
||||
return dispatchFloat(
|
||||
*compute, {&x}, x.shape,
|
||||
"float v=x0[i];y[i]=" + (n.op == "LeakyRelu" ? "v>=0.0?v:v*" + literal(alpha) : unary.at(n.op)) + ";");
|
||||
if (n.op != "LeakyRelu" && n.op != "Reciprocal" && n.op != "Floor" && n.op != "Round" && n.op != "Atan")
|
||||
return std::nullopt;
|
||||
auto out = floats(x);
|
||||
for (auto& v : out) {
|
||||
if (n.op == "LeakyRelu")
|
||||
v = v >= 0 ? v : alpha * v;
|
||||
else if (n.op == "Reciprocal")
|
||||
v = 1 / v;
|
||||
else if (n.op == "Floor")
|
||||
v = std::floor(v);
|
||||
else if (n.op == "Atan")
|
||||
v = std::atan(v);
|
||||
else {
|
||||
float lo = std::floor(v), f = v - lo;
|
||||
v = lo + (f > 0.5f || (f == 0.5f && std::fmod(lo, 2.f) != 0));
|
||||
}
|
||||
}
|
||||
return make(OnnxElement::Float32, x.shape, out);
|
||||
}
|
||||
return std::nullopt;
|
||||
}
|
||||
} // namespace eve::tensor::onnx_detail
|
||||
@@ -0,0 +1,196 @@
|
||||
#include "tensor/AffineQuant.h"
|
||||
#include "tensor/AffineQuantInternal.h"
|
||||
#include "tensor/OnnxGpuKernels.h"
|
||||
#include "tensor/OnnxInternal.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <cmath>
|
||||
|
||||
namespace eve::tensor::onnx_detail {
|
||||
namespace {
|
||||
affine::ByteView bytes(const RuntimeTensor& t) {
|
||||
if (t.element != OnnxElement::Int8 && t.element != OnnxElement::UInt8)
|
||||
throw Failure("Expected packed int8/uint8 tensor");
|
||||
return {t.bytes, t.element == OnnxElement::Int8};
|
||||
}
|
||||
template <class T>
|
||||
T checked(Result<T> result) {
|
||||
if (!result.ok()) throw Failure(result.error()->message(), result.error()->code());
|
||||
return std::move(result.value());
|
||||
}
|
||||
int zero(const std::vector<const RuntimeTensor*>& in, size_t i, const RuntimeTensor& owner) {
|
||||
if (i >= in.size() || !in[i]) return 0;
|
||||
if (in[i]->element != owner.element || count(in[i]->shape) != 1)
|
||||
throw Failure("Expected matching scalar zero point");
|
||||
return static_cast<int>(integer(*in[i]));
|
||||
}
|
||||
} // namespace
|
||||
std::vector<RuntimeTensor> executeQuant(const Node& n, const std::vector<const RuntimeTensor*>& in,
|
||||
OnnxCompute* compute) {
|
||||
const auto& x = required(in, 0);
|
||||
if (n.op == "DynamicQuantizeLinear") {
|
||||
if (compute && count(x.shape)) {
|
||||
if (x.element != OnnxElement::Float32) throw Failure("Expected FP32 quantization input");
|
||||
const size_t total = count(x.shape);
|
||||
auto stats = dispatchFloat(
|
||||
*compute, {&x}, {3},
|
||||
"float lo=0.0,hi=0.0,invalid=0.0;for(uint j=0;j<" + std::to_string(total) +
|
||||
"u;++j){float v=x0[j];if(isinf(v))invalid=1.0;if(!isnan(v)){lo=min(lo,v);hi=max(hi,v);}}float "
|
||||
"s=(hi==lo)?1.0:(hi-lo)/255.0;y[0]=s;y[1]=clamp(roundEven(-lo/s),0.0,255.0);y[2]=invalid;",
|
||||
1);
|
||||
auto values = floats(stats);
|
||||
if (values[2] != 0 || !std::isfinite(values[0]) || !(values[0] > 0))
|
||||
throw Failure("Nonfinite quantization input/range");
|
||||
auto scale = make(OnnxElement::Float32, {}, std::vector<float>{values[0]});
|
||||
auto zeroPoint = make(OnnxElement::UInt8, {}, std::vector<uint8_t>{static_cast<uint8_t>(values[1])});
|
||||
auto params = make(OnnxElement::Float32, {2}, std::vector<float>{values[0], values[1]});
|
||||
std::string source =
|
||||
"#version 450\nlayout(local_size_x=64)in;layout(std430,binding=0)readonly buffer X{float "
|
||||
"x[];};layout(std430,binding=1)readonly buffer P{float p[];};layout(std430,binding=2)writeonly buffer "
|
||||
"Y{uint y[];};void main(){uint word=gl_GlobalInvocationID.x;if(word>=" +
|
||||
std::to_string((total + 3) / 4) +
|
||||
"u)return;uint packed=0u;for(uint c=0u;c<4u;++c){uint j=word*4u+c;if(j<" + std::to_string(total) +
|
||||
"u){float v=x[j];uint "
|
||||
"q=isnan(v)?0u:uint(clamp(roundEven(v/p[0])+p[1],0.0,255.0));packed|=q<<(c*8u);}}y[word]=packed;}";
|
||||
const std::vector<OnnxBuffer> inputs{x.bytes.buffer(), params.bytes.buffer()};
|
||||
auto q = checked(compute->enqueue(source, inputs, total, static_cast<uint32_t>((total + 3) / 4)));
|
||||
return {{OnnxElement::UInt8, x.shape, std::move(q)}, std::move(scale), std::move(zeroPoint)};
|
||||
}
|
||||
auto input = floats(x);
|
||||
// ORT CPU DynamicQuantizeLinear ignores NaN for range estimation and stores code 0.
|
||||
// Exported Kokoro phase graphs produce 0/0 at zero-amplitude STFT bins.
|
||||
std::vector<size_t> nanPositions;
|
||||
for (size_t i = 0; i < input.size(); ++i)
|
||||
if (std::isnan(input[i])) {
|
||||
nanPositions.push_back(i);
|
||||
input[i] = 0;
|
||||
}
|
||||
auto q = checked(affine::dynamicQuantize(input));
|
||||
for (auto i : nanPositions) q.values[i] = 0;
|
||||
return {{OnnxElement::UInt8, x.shape, std::move(q.values)},
|
||||
make(OnnxElement::Float32, {}, std::vector<float>{q.scale}),
|
||||
make(OnnxElement::UInt8, {}, std::vector<uint8_t>{q.zeroPoint})};
|
||||
}
|
||||
if (n.op == "QuantizeLinear" || n.op == "DequantizeLinear") {
|
||||
const auto scales = floats(required(in, 1));
|
||||
if (required(in, 1).shape.size() > 1) throw Failure("Quantization scale must be scalar or vector");
|
||||
if (scales.empty()) throw Failure("Empty quantization scales");
|
||||
const RuntimeTensor* zp = in.size() > 2 ? in[2] : nullptr;
|
||||
const auto type = n.op == "DequantizeLinear" ? x.element : (zp ? zp->element : OnnxElement::UInt8);
|
||||
if (type != OnnxElement::UInt8 && type != OnnxElement::Int8)
|
||||
throw Failure("Unsupported quantization dtype", DiagnosticCode::Unsupported);
|
||||
if (zp && (zp->element != type || zp->shape != required(in, 1).shape))
|
||||
throw Failure("Scale/zero point mismatch");
|
||||
std::vector<int32_t> zeros(scales.size(), 0);
|
||||
if (zp)
|
||||
for (size_t i = 0; i < zeros.size(); ++i) zeros[i] = static_cast<int32_t>(integer(*zp, i));
|
||||
size_t inner = 1;
|
||||
if (scales.size() > 1) {
|
||||
const int a = axis(attr(n, "axis", 1), x.shape.size());
|
||||
if (static_cast<size_t>(x.shape[a]) != scales.size()) throw Failure("Scale channel count mismatch");
|
||||
for (size_t i = a + 1; i < x.shape.size(); ++i) inner *= x.shape[i];
|
||||
}
|
||||
if (n.op == "DequantizeLinear")
|
||||
return {make(OnnxElement::Float32, x.shape, checked(affine::dequantize(bytes(x), scales, zeros, inner)))};
|
||||
const auto values = floats(x);
|
||||
std::vector<uint8_t> out(values.size());
|
||||
for (size_t i = 0; i < values.size(); i += inner) {
|
||||
const size_t channel = (i / inner) % scales.size();
|
||||
auto q = checked(affine::quantize(std::span(values).subspan(i, inner), scales[channel], zeros[channel],
|
||||
type == OnnxElement::Int8));
|
||||
std::copy(q.begin(), q.end(), out.begin() + i);
|
||||
}
|
||||
return {{type, x.shape, std::move(out)}};
|
||||
}
|
||||
if (n.op == "MatMulInteger") {
|
||||
const auto& b = required(in, 1);
|
||||
if (x.shape.size() < 2 || b.shape.size() != 2)
|
||||
throw Failure("MatMulInteger requires rank >=2 A and rank 2 B", DiagnosticCode::Unsupported);
|
||||
const size_t k = x.shape.back(), columns = b.shape[1];
|
||||
if (k != static_cast<size_t>(b.shape[0]) || k == 0) throw Failure("Invalid MatMulInteger inner dimension");
|
||||
const size_t rows = count(x.shape) / k;
|
||||
auto shape = x.shape;
|
||||
shape.back() = columns;
|
||||
if (compute) {
|
||||
if ((x.element != OnnxElement::Int8 && x.element != OnnxElement::UInt8) ||
|
||||
(b.element != OnnxElement::Int8 && b.element != OnnxElement::UInt8))
|
||||
throw Failure("Expected packed integer matrices");
|
||||
std::vector<int32_t> bz{zero(in, 3, b)};
|
||||
return {{OnnxElement::Int32, shape,
|
||||
gpuMatmulResident(*compute, x.bytes.buffer(), b.bytes.buffer(), x.element == OnnxElement::Int8,
|
||||
b.element == OnnxElement::Int8, rows, k, columns, zero(in, 2, x), bz)}};
|
||||
}
|
||||
return {make(
|
||||
OnnxElement::Int32, shape,
|
||||
checked(affine::matmul(bytes(x), bytes(b), rows, k, columns, zero(in, 2, x), zero(in, 3, b), compute)))};
|
||||
}
|
||||
if (n.op == "ConvInteger") {
|
||||
const auto& w = required(in, 1);
|
||||
if ((x.shape.size() != 3 && x.shape.size() != 4) || w.shape.size() != x.shape.size())
|
||||
throw Failure("ConvInteger supports 1D/2D NCHW", DiagnosticCode::Unsupported);
|
||||
const size_t spatial = x.shape.size() - 2;
|
||||
if (n.attrs.contains("kernel_shape") &&
|
||||
n.attrs.at("kernel_shape").integers != std::vector<int64_t>(w.shape.begin() + 2, w.shape.end()))
|
||||
throw Failure("Convolution kernel_shape mismatch");
|
||||
auto strides = attrs(n, "strides", std::vector<int64_t>(spatial, 1));
|
||||
auto pads = attrs(n, "pads", std::vector<int64_t>(2 * spatial, 0));
|
||||
auto dilations = attrs(n, "dilations", std::vector<int64_t>(spatial, 1));
|
||||
if (strides.size() != spatial || pads.size() != 2 * spatial || dilations.size() != spatial)
|
||||
throw Failure("Invalid convolution attributes");
|
||||
auto narrow = [](int64_t v) {
|
||||
if (v < 0 || v > 65536) throw Failure("Convolution attribute out of range");
|
||||
return static_cast<int>(v);
|
||||
};
|
||||
affine::ConvShape s;
|
||||
s.batch = narrow(x.shape[0]);
|
||||
s.channels = narrow(x.shape[1]);
|
||||
s.outputs = narrow(w.shape[0]);
|
||||
s.width = narrow(x.shape.back());
|
||||
s.kernelW = narrow(w.shape.back());
|
||||
s.strideW = narrow(strides.back());
|
||||
s.dilationW = narrow(dilations.back());
|
||||
s.padLeft = narrow(pads[spatial - 1]);
|
||||
s.padRight = narrow(pads.back());
|
||||
s.groups = narrow(attr(n, "group", 1));
|
||||
if (spatial == 2) {
|
||||
s.height = narrow(x.shape[2]);
|
||||
s.kernelH = narrow(w.shape[2]);
|
||||
s.strideH = narrow(strides[0]);
|
||||
s.dilationH = narrow(dilations[0]);
|
||||
s.padTop = narrow(pads[0]);
|
||||
s.padBottom = narrow(pads[2]);
|
||||
}
|
||||
if (!s.groups || s.channels % s.groups || w.shape[1] != s.channels / s.groups)
|
||||
throw Failure("Convolution channel mismatch");
|
||||
std::vector<int32_t> wz{0};
|
||||
if (in.size() > 3 && in[3]) {
|
||||
if (in[3]->element != w.element) throw Failure("Weight zero-point type mismatch");
|
||||
wz.clear();
|
||||
for (auto z : ints(*in[3])) wz.push_back(static_cast<int32_t>(z));
|
||||
}
|
||||
if (compute) {
|
||||
if ((x.element != OnnxElement::Int8 && x.element != OnnxElement::UInt8) ||
|
||||
(w.element != OnnxElement::Int8 && w.element != OnnxElement::UInt8))
|
||||
throw Failure("Expected packed convolution inputs");
|
||||
const auto extent =
|
||||
checked(affine::detail::validateConv(x.bytes.size(), w.bytes.size(), x.element == OnnxElement::Int8,
|
||||
w.element == OnnxElement::Int8, s, zero(in, 2, x), wz));
|
||||
std::vector<int64_t> outputShape{s.batch, s.outputs};
|
||||
if (spatial == 2) outputShape.push_back(extent.height);
|
||||
outputShape.push_back(extent.width);
|
||||
return {{OnnxElement::Int32, outputShape,
|
||||
gpuConvResident(*compute, x.bytes.buffer(), w.bytes.buffer(), x.element == OnnxElement::Int8,
|
||||
w.element == OnnxElement::Int8, s, zero(in, 2, x), wz)}};
|
||||
}
|
||||
auto values = checked(affine::conv(bytes(x), bytes(w), s, zero(in, 2, x), wz, compute));
|
||||
std::vector<int64_t> shape{s.batch, s.outputs};
|
||||
if (spatial == 2)
|
||||
shape.push_back(
|
||||
(s.height + s.padTop + s.padBottom - int64_t(s.dilationH) * (s.kernelH - 1) - 1) / s.strideH + 1);
|
||||
shape.push_back((s.width + s.padLeft + s.padRight - int64_t(s.dilationW) * (s.kernelW - 1) - 1) / s.strideW +
|
||||
1);
|
||||
return {make(OnnxElement::Int32, shape, values)};
|
||||
}
|
||||
throw Failure("Unsupported quantized operation", DiagnosticCode::Unsupported);
|
||||
}
|
||||
} // namespace eve::tensor::onnx_detail
|
||||
@@ -0,0 +1,419 @@
|
||||
#include "tensor/OnnxInternal.h"
|
||||
|
||||
#include <bit>
|
||||
#include <functional>
|
||||
#include <set>
|
||||
|
||||
namespace eve::tensor::onnx_detail {
|
||||
namespace {
|
||||
struct Field {
|
||||
int id = 0, wire = 0;
|
||||
uint64_t number = 0;
|
||||
std::span<const uint8_t> bytes;
|
||||
std::string text() const { return {reinterpret_cast<const char*>(bytes.data()), bytes.size()}; }
|
||||
};
|
||||
class Reader {
|
||||
public:
|
||||
explicit Reader(std::span<const uint8_t> data) : data_(data) {}
|
||||
bool empty() const { return offset_ == data_.size(); }
|
||||
uint64_t varint() {
|
||||
uint64_t n = 0;
|
||||
for (int shift = 0; shift < 70; shift += 7) {
|
||||
if (offset_ == data_.size()) throw Failure("Truncated protobuf varint", DiagnosticCode::ParseError);
|
||||
const uint8_t b = data_[offset_++];
|
||||
if (shift == 63 && b > 1) throw Failure("Overflowing protobuf varint", DiagnosticCode::ParseError);
|
||||
n |= static_cast<uint64_t>(b & 127) << shift;
|
||||
if (!(b & 128)) return n;
|
||||
}
|
||||
throw Failure("Invalid protobuf varint", DiagnosticCode::ParseError);
|
||||
}
|
||||
std::span<const uint8_t> take(uint64_t n) {
|
||||
if (n > data_.size() - offset_) throw Failure("Truncated protobuf field", DiagnosticCode::ParseError);
|
||||
auto result = data_.subspan(offset_, static_cast<size_t>(n));
|
||||
offset_ += static_cast<size_t>(n);
|
||||
return result;
|
||||
}
|
||||
Field next() {
|
||||
const uint64_t tag = varint();
|
||||
if ((tag >> 3) == 0 || (tag >> 3) > 0x1fffffff)
|
||||
throw Failure("Invalid protobuf tag", DiagnosticCode::ParseError);
|
||||
Field f;
|
||||
f.id = static_cast<int>(tag >> 3);
|
||||
f.wire = tag & 7;
|
||||
switch (f.wire) {
|
||||
case 0: f.number = varint(); break;
|
||||
case 1: f.bytes = take(8); break;
|
||||
case 2: f.bytes = take(varint()); break;
|
||||
case 5: f.bytes = take(4); break;
|
||||
default: throw Failure("Unsupported protobuf wire type", DiagnosticCode::ParseError);
|
||||
}
|
||||
return f;
|
||||
}
|
||||
|
||||
private:
|
||||
std::span<const uint8_t> data_;
|
||||
size_t offset_ = 0;
|
||||
};
|
||||
void wire(const Field& f, int expected) {
|
||||
if (f.wire != expected) throw Failure("Unexpected protobuf field type", DiagnosticCode::ParseError);
|
||||
}
|
||||
void integers(const Field& f, std::vector<int64_t>& out) {
|
||||
if (f.wire == 0)
|
||||
out.push_back(std::bit_cast<int64_t>(f.number));
|
||||
else {
|
||||
wire(f, 2);
|
||||
Reader r(f.bytes);
|
||||
while (!r.empty()) out.push_back(std::bit_cast<int64_t>(r.varint()));
|
||||
}
|
||||
}
|
||||
void reals(const Field& f, std::vector<float>& out) {
|
||||
if (f.wire != 2 && f.wire != 5) throw Failure("Invalid float wire type", DiagnosticCode::ParseError);
|
||||
if (f.bytes.size() % 4) throw Failure("Truncated float payload", DiagnosticCode::ParseError);
|
||||
for (size_t i = 0; i < f.bytes.size(); i += 4) {
|
||||
float x;
|
||||
std::memcpy(&x, f.bytes.data() + i, 4);
|
||||
out.push_back(x);
|
||||
}
|
||||
}
|
||||
std::pair<std::string, RuntimeTensor> tensor(std::span<const uint8_t> data) {
|
||||
Reader r(data);
|
||||
RuntimeTensor v;
|
||||
std::string name;
|
||||
std::vector<int64_t> iv;
|
||||
std::vector<float> fv;
|
||||
bool raw = false, type = false;
|
||||
while (!r.empty()) {
|
||||
auto f = r.next();
|
||||
switch (f.id) {
|
||||
case 1: integers(f, v.shape); break;
|
||||
case 2:
|
||||
wire(f, 0);
|
||||
if (f.number > 9) throw Failure("Unsupported tensor element type", DiagnosticCode::Unsupported);
|
||||
v.element = static_cast<OnnxElement>(f.number);
|
||||
type = true;
|
||||
break;
|
||||
case 4: reals(f, fv); break;
|
||||
case 5:
|
||||
case 7: integers(f, iv); break;
|
||||
case 8:
|
||||
wire(f, 2);
|
||||
name = f.text();
|
||||
break;
|
||||
case 9:
|
||||
wire(f, 2);
|
||||
if (raw) throw Failure("Duplicate raw tensor data", DiagnosticCode::ParseError);
|
||||
v.bytes.assign(f.bytes.begin(), f.bytes.end());
|
||||
raw = true;
|
||||
break;
|
||||
case 14:
|
||||
wire(f, 0);
|
||||
if (f.number != 0) throw Failure("External tensor data unsupported", DiagnosticCode::Unsupported);
|
||||
break;
|
||||
case 3:
|
||||
case 6:
|
||||
case 10:
|
||||
case 11:
|
||||
case 13: throw Failure("External, segmented or unsupported tensor data", DiagnosticCode::Unsupported);
|
||||
default: break;
|
||||
}
|
||||
}
|
||||
if (!type) throw Failure("Tensor has no dtype", DiagnosticCode::ParseError);
|
||||
const size_t n = count(v.shape), size = elementSize(v.element);
|
||||
if (raw && (!iv.empty() || !fv.empty())) throw Failure("Ambiguous tensor payload", DiagnosticCode::ParseError);
|
||||
if (!raw) {
|
||||
if (v.element == OnnxElement::Float32) {
|
||||
if (fv.size() != n || !iv.empty())
|
||||
throw Failure("Invalid float tensor payload", DiagnosticCode::ParseError);
|
||||
v.bytes.resize(n * size);
|
||||
if (n) std::memcpy(v.bytes.data(), fv.data(), v.bytes.size());
|
||||
} else {
|
||||
if (iv.size() != n || !fv.empty())
|
||||
throw Failure("Invalid integer tensor payload", DiagnosticCode::ParseError);
|
||||
v.bytes.resize(n * size);
|
||||
for (size_t i = 0; i < n; ++i) {
|
||||
const auto x = iv[i];
|
||||
if ((v.element == OnnxElement::UInt8 && (x < 0 || x > 255)) ||
|
||||
(v.element == OnnxElement::Int8 && (x < -128 || x > 127)) ||
|
||||
(v.element == OnnxElement::Bool && x != 0 && x != 1) ||
|
||||
(v.element == OnnxElement::Int32 && (x < INT32_MIN || x > INT32_MAX)))
|
||||
throw Failure("Integer initializer out of range", DiagnosticCode::ParseError);
|
||||
std::memcpy(v.bytes.data() + i * size, &x, size);
|
||||
}
|
||||
}
|
||||
}
|
||||
validate(v);
|
||||
return {std::move(name), std::move(v)};
|
||||
}
|
||||
ModelData parseGraph(std::span<const uint8_t> bytes, size_t depth);
|
||||
std::pair<std::string, Attribute> attribute(std::span<const uint8_t> bytes, size_t depth) {
|
||||
Reader r(bytes);
|
||||
Attribute a;
|
||||
std::string name;
|
||||
while (!r.empty()) {
|
||||
auto f = r.next();
|
||||
switch (f.id) {
|
||||
case 1:
|
||||
wire(f, 2);
|
||||
name = f.text();
|
||||
break;
|
||||
case 20:
|
||||
wire(f, 0);
|
||||
a.type = static_cast<int>(f.number);
|
||||
break;
|
||||
case 2: {
|
||||
wire(f, 5);
|
||||
std::memcpy(&a.real, f.bytes.data(), 4);
|
||||
break;
|
||||
}
|
||||
case 3:
|
||||
wire(f, 0);
|
||||
a.integer = std::bit_cast<int64_t>(f.number);
|
||||
break;
|
||||
case 4:
|
||||
wire(f, 2);
|
||||
a.text = f.text();
|
||||
break;
|
||||
case 5:
|
||||
wire(f, 2);
|
||||
a.tensor = tensor(f.bytes).second;
|
||||
break;
|
||||
case 7: reals(f, a.reals); break;
|
||||
case 8: integers(f, a.integers); break;
|
||||
case 9:
|
||||
wire(f, 2);
|
||||
a.strings.push_back(f.text());
|
||||
break;
|
||||
case 6:
|
||||
wire(f, 2);
|
||||
a.graph = std::make_shared<ModelData>(parseGraph(f.bytes, depth + 1));
|
||||
break;
|
||||
case 11: throw Failure("Multiple graph attributes unsupported", DiagnosticCode::Unsupported);
|
||||
case 21: throw Failure("Function attribute references unsupported", DiagnosticCode::Unsupported);
|
||||
default: break;
|
||||
}
|
||||
}
|
||||
if (name.empty() || !a.type) throw Failure("Incomplete attribute", DiagnosticCode::ParseError);
|
||||
return {std::move(name), std::move(a)};
|
||||
}
|
||||
Node node(std::span<const uint8_t> bytes, size_t depth) {
|
||||
Reader r(bytes);
|
||||
Node n;
|
||||
while (!r.empty()) {
|
||||
auto f = r.next();
|
||||
if (f.id >= 1 && f.id <= 5) wire(f, 2);
|
||||
switch (f.id) {
|
||||
case 1: n.inputs.push_back(f.text()); break;
|
||||
case 2: n.outputs.push_back(f.text()); break;
|
||||
case 3: n.name = f.text(); break;
|
||||
case 4: n.op = f.text(); break;
|
||||
case 5:
|
||||
if (!n.attrs.insert(attribute(f.bytes, depth)).second)
|
||||
throw Failure("Duplicate attribute", DiagnosticCode::ParseError);
|
||||
break;
|
||||
case 7:
|
||||
wire(f, 2);
|
||||
n.domain = f.text();
|
||||
break;
|
||||
case 8: throw Failure("Function overloads unsupported", DiagnosticCode::Unsupported);
|
||||
default: break;
|
||||
}
|
||||
}
|
||||
if (n.op.empty() || n.outputs.empty()) throw Failure("Incomplete node", DiagnosticCode::ParseError);
|
||||
return n;
|
||||
}
|
||||
std::pair<std::string, Input> input(std::span<const uint8_t> bytes) {
|
||||
Reader r(bytes);
|
||||
std::string name;
|
||||
Input value;
|
||||
while (!r.empty()) {
|
||||
auto f = r.next();
|
||||
if (f.id == 1) {
|
||||
wire(f, 2);
|
||||
name = f.text();
|
||||
}
|
||||
if (f.id == 2) {
|
||||
wire(f, 2);
|
||||
Reader type(f.bytes);
|
||||
while (!type.empty()) {
|
||||
auto t = type.next();
|
||||
if (t.id == 4) {
|
||||
value.type = 0;
|
||||
continue;
|
||||
}
|
||||
if (t.id != 1) throw Failure("Non-tensor graph input", DiagnosticCode::Unsupported);
|
||||
wire(t, 2);
|
||||
Reader tt(t.bytes);
|
||||
while (!tt.empty()) {
|
||||
auto p = tt.next();
|
||||
if (p.id == 1) {
|
||||
wire(p, 0);
|
||||
value.type = static_cast<int>(p.number);
|
||||
}
|
||||
if (p.id == 2) {
|
||||
wire(p, 2);
|
||||
Reader shape(p.bytes);
|
||||
while (!shape.empty()) {
|
||||
auto d = shape.next();
|
||||
if (d.id != 1) continue;
|
||||
wire(d, 2);
|
||||
Reader dim(d.bytes);
|
||||
int64_t v = -1;
|
||||
while (!dim.empty()) {
|
||||
auto x = dim.next();
|
||||
if (x.id == 1) {
|
||||
wire(x, 0);
|
||||
v = std::bit_cast<int64_t>(x.number);
|
||||
}
|
||||
}
|
||||
value.shape.push_back(v);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (name.empty()) throw Failure("Empty value name", DiagnosticCode::ParseError);
|
||||
if (value.shape.size() > 6) throw Failure("Input rank exceeds six", DiagnosticCode::Unsupported);
|
||||
for (auto d : value.shape)
|
||||
if (d < -1 || d > INT32_MAX) throw Failure("Invalid interface dimension", DiagnosticCode::ParseError);
|
||||
return {name, value};
|
||||
}
|
||||
ModelData parseGraph(std::span<const uint8_t> bytes, size_t depth) {
|
||||
ModelData model;
|
||||
if (depth > 16) throw Failure("ONNX graph nesting limit exceeded", DiagnosticCode::ParseError);
|
||||
Reader g(bytes);
|
||||
while (!g.empty()) {
|
||||
auto f = g.next();
|
||||
if (f.id == 1) {
|
||||
wire(f, 2);
|
||||
if (model.nodes.size() >= 100000) throw Failure("Node limit exceeded", DiagnosticCode::ParseError);
|
||||
model.nodes.push_back(node(f.bytes, depth));
|
||||
}
|
||||
if (f.id == 5) {
|
||||
wire(f, 2);
|
||||
auto t = tensor(f.bytes);
|
||||
t.second.bytes.retainAcrossRuns();
|
||||
if (t.first.empty() || !model.constants.emplace(std::move(t)).second)
|
||||
throw Failure("Duplicate/empty initializer", DiagnosticCode::ParseError);
|
||||
}
|
||||
if (f.id == 11) {
|
||||
wire(f, 2);
|
||||
auto v = input(f.bytes);
|
||||
model.info.inputs.push_back(v.first);
|
||||
if (!model.inputs.emplace(std::move(v)).second)
|
||||
throw Failure("Duplicate graph input", DiagnosticCode::ParseError);
|
||||
}
|
||||
if (f.id == 12) {
|
||||
wire(f, 2);
|
||||
model.info.outputs.push_back(input(f.bytes).first);
|
||||
}
|
||||
if (f.id == 15) throw Failure("Sparse initializers unsupported", DiagnosticCode::Unsupported);
|
||||
}
|
||||
return model;
|
||||
}
|
||||
} // namespace
|
||||
|
||||
ModelData parse(std::span<const uint8_t> bytes) {
|
||||
if constexpr (std::endian::native != std::endian::little)
|
||||
throw Failure("ONNX requires little-endian host", DiagnosticCode::Unsupported);
|
||||
if (bytes.empty() || bytes.size() > 512u * 1024u * 1024u)
|
||||
throw Failure("Model size exceeds import limit", DiagnosticCode::ParseError);
|
||||
ModelData model;
|
||||
Reader r(bytes);
|
||||
std::span<const uint8_t> graph;
|
||||
while (!r.empty()) {
|
||||
auto f = r.next();
|
||||
if (f.id == 1) {
|
||||
wire(f, 0);
|
||||
model.info.irVersion = static_cast<int64_t>(f.number);
|
||||
}
|
||||
if (f.id == 7) {
|
||||
wire(f, 2);
|
||||
if (!graph.empty()) throw Failure("Duplicate graph", DiagnosticCode::ParseError);
|
||||
graph = f.bytes;
|
||||
}
|
||||
if (f.id == 8) {
|
||||
wire(f, 2);
|
||||
Reader ops(f.bytes);
|
||||
std::string domain;
|
||||
int64_t version = 0;
|
||||
while (!ops.empty()) {
|
||||
auto x = ops.next();
|
||||
if (x.id == 1) {
|
||||
wire(x, 2);
|
||||
domain = x.text();
|
||||
}
|
||||
if (x.id == 2) {
|
||||
wire(x, 0);
|
||||
version = x.number;
|
||||
}
|
||||
}
|
||||
if (!model.opsets.emplace(domain, version).second)
|
||||
throw Failure("Duplicate opset", DiagnosticCode::ParseError);
|
||||
}
|
||||
if (f.id == 20 || f.id == 25 || f.id == 26)
|
||||
throw Failure("Training/functions/device configuration unsupported", DiagnosticCode::Unsupported);
|
||||
}
|
||||
if (model.info.irVersion < 3 || model.info.irVersion > 10 || !model.opsets.contains("") ||
|
||||
model.opsets.at("") < 13 || model.opsets.at("") > 17)
|
||||
throw Failure("Unsupported ONNX IR/opset version", DiagnosticCode::UnknownVersion);
|
||||
if (model.opsets.contains("com.microsoft") && model.opsets.at("com.microsoft") != 1)
|
||||
throw Failure("Unsupported Microsoft opset", DiagnosticCode::UnknownVersion);
|
||||
if (graph.empty()) throw Failure("Missing ONNX graph", DiagnosticCode::ParseError);
|
||||
auto parsed = parseGraph(graph, 0);
|
||||
model.nodes = std::move(parsed.nodes);
|
||||
model.constants = std::move(parsed.constants);
|
||||
model.inputs = std::move(parsed.inputs);
|
||||
model.info.inputs = std::move(parsed.info.inputs);
|
||||
model.info.outputs = std::move(parsed.info.outputs);
|
||||
std::set<std::string> names;
|
||||
for (const auto& [name, v] : model.constants) {
|
||||
names.insert(name);
|
||||
model.info.initializerBytes += v.bytes.size();
|
||||
}
|
||||
for (const auto& [name, v] : model.inputs) names.insert(name);
|
||||
for (const auto& n : model.nodes) {
|
||||
if (!model.opsets.contains(n.domain)) throw Failure("Node domain has no opset", DiagnosticCode::ParseError);
|
||||
for (const auto& name : n.inputs)
|
||||
if (!name.empty() && !names.contains(name))
|
||||
throw Failure("Input not in topological order: " + name, DiagnosticCode::ParseError);
|
||||
for (const auto& name : n.outputs)
|
||||
if (!name.empty() && !names.insert(name).second)
|
||||
throw Failure("Duplicate graph value: " + name, DiagnosticCode::ParseError);
|
||||
if (!isSupported(n)) model.info.unsupportedNodes.push_back(n.name + " [" + n.domain + "::" + n.op + "]");
|
||||
}
|
||||
for (const auto& out : model.info.outputs)
|
||||
if (!names.contains(out)) throw Failure("Undefined graph output", DiagnosticCode::ParseError);
|
||||
size_t recursiveNodes = 0;
|
||||
std::function<void(const ModelData&, const std::set<std::string>&, bool)> validateNested;
|
||||
validateNested = [&](const ModelData& g, const std::set<std::string>& outer, bool root) {
|
||||
recursiveNodes += g.nodes.size();
|
||||
if (recursiveNodes > 100000) throw Failure("Recursive node limit exceeded", DiagnosticCode::ParseError);
|
||||
auto available = outer;
|
||||
std::set<std::string> locals;
|
||||
for (const auto& [name, t] : g.constants) locals.insert(name);
|
||||
for (const auto& [name, t] : g.inputs) locals.insert(name);
|
||||
for (const auto& n : g.nodes)
|
||||
for (const auto& out : n.outputs)
|
||||
if (!out.empty() && !locals.insert(out).second)
|
||||
throw Failure("Duplicate nested graph value: " + out, DiagnosticCode::ParseError);
|
||||
available.insert(locals.begin(), locals.end());
|
||||
for (const auto& n : g.nodes) {
|
||||
if (!model.opsets.contains(n.domain))
|
||||
throw Failure("Nested node domain has no opset", DiagnosticCode::ParseError);
|
||||
for (const auto& input : n.inputs)
|
||||
if (!input.empty() && !available.contains(input))
|
||||
throw Failure("Undefined lexical capture: " + input, DiagnosticCode::ParseError);
|
||||
if (!root && !isSupported(n))
|
||||
model.info.unsupportedNodes.push_back(n.name + " [" + n.domain + "::" + n.op + "]");
|
||||
for (const auto& [key, a] : n.attrs)
|
||||
if (a.graph) validateNested(*a.graph, available, false);
|
||||
}
|
||||
for (const auto& out : g.info.outputs)
|
||||
if (!available.contains(out)) throw Failure("Undefined nested output: " + out, DiagnosticCode::ParseError);
|
||||
};
|
||||
validateNested(model, {}, true);
|
||||
model.info.nodeCount = model.nodes.size();
|
||||
return model;
|
||||
}
|
||||
} // namespace eve::tensor::onnx_detail
|
||||
@@ -0,0 +1,360 @@
|
||||
#include <algorithm>
|
||||
#include <cmath>
|
||||
#include <functional>
|
||||
#include <queue>
|
||||
#include <random>
|
||||
#include <set>
|
||||
#include "tensor/OnnxInternal.h"
|
||||
|
||||
namespace eve::tensor::onnx_detail {
|
||||
namespace {
|
||||
struct Value {
|
||||
const RuntimeTensor* borrowed = nullptr;
|
||||
std::shared_ptr<const RuntimeTensor> owned;
|
||||
std::shared_ptr<std::vector<Value>> sequence;
|
||||
OnnxElement sequenceElement = OnnxElement::Float32;
|
||||
const RuntimeTensor& tensor() const {
|
||||
if (owned) return *owned;
|
||||
if (borrowed) return *borrowed;
|
||||
throw Failure("Expected tensor, received sequence");
|
||||
}
|
||||
};
|
||||
using Values = std::unordered_map<std::string, Value>;
|
||||
struct Context {
|
||||
OnnxCompute* compute;
|
||||
uint64_t seed;
|
||||
bool finite;
|
||||
size_t steps = 0, liveBytes = 0;
|
||||
std::unordered_map<const Node*, std::mt19937> randomStreams;
|
||||
};
|
||||
Value hold(RuntimeTensor t, Context& c) {
|
||||
validate(t);
|
||||
if (c.finite && t.element == OnnxElement::Float32)
|
||||
for (size_t i = 0; i < count(t.shape); ++i)
|
||||
if (!std::isfinite(read<float>(t, i))) throw Failure("Nonfinite float output");
|
||||
const auto size = t.bytes.size();
|
||||
if (size > 512u * 1024u * 1024u - c.liveBytes) throw Failure("ONNX live tensor memory limit exceeded");
|
||||
// Context encloses every runtime Value; escaping public outputs are owning copies.
|
||||
auto* raw = new RuntimeTensor(std::move(t));
|
||||
c.liveBytes += size;
|
||||
return {nullptr,
|
||||
std::shared_ptr<const RuntimeTensor>(raw,
|
||||
[&c, size](const RuntimeTensor* p) {
|
||||
delete p;
|
||||
c.liveBytes -= size;
|
||||
}),
|
||||
{}};
|
||||
}
|
||||
std::set<std::string> captures(const ModelData& graph);
|
||||
std::set<std::string> dependencies(const Node& n) {
|
||||
std::set<std::string> result;
|
||||
for (const auto& i : n.inputs)
|
||||
if (!i.empty()) result.insert(i);
|
||||
for (const auto& [key, a] : n.attrs)
|
||||
if (a.graph) {
|
||||
auto names = captures(*a.graph);
|
||||
result.insert(names.begin(), names.end());
|
||||
}
|
||||
return result;
|
||||
}
|
||||
std::set<std::string> captures(const ModelData& graph) {
|
||||
std::set<std::string> locals, result;
|
||||
for (const auto& [name, t] : graph.constants) locals.insert(name);
|
||||
for (const auto& [name, t] : graph.inputs) locals.insert(name);
|
||||
for (const auto& n : graph.nodes)
|
||||
for (const auto& out : n.outputs)
|
||||
if (!out.empty()) locals.insert(out);
|
||||
for (const auto& n : graph.nodes) {
|
||||
auto deps = dependencies(n);
|
||||
for (const auto& d : deps)
|
||||
if (!locals.contains(d)) result.insert(d);
|
||||
}
|
||||
for (const auto& out : graph.info.outputs)
|
||||
if (!locals.contains(out)) result.insert(out);
|
||||
return result;
|
||||
}
|
||||
void admit(const Node& n) {
|
||||
if (!isSupported(n))
|
||||
throw Failure(n.name + ": Unsupported ONNX node " + n.domain + "::" + n.op, DiagnosticCode::Unsupported,
|
||||
n.name);
|
||||
for (const auto& [key, a] : n.attrs)
|
||||
if (a.graph)
|
||||
for (const auto& child : a.graph->nodes) admit(child);
|
||||
}
|
||||
std::vector<size_t> plan(const ModelData& g, const std::vector<std::string>& requested) {
|
||||
std::unordered_map<std::string, size_t> producers;
|
||||
for (size_t i = 0; i < g.nodes.size(); ++i)
|
||||
for (const auto& name : g.nodes[i].outputs)
|
||||
if (!name.empty() && !producers.emplace(name, i).second) throw Failure("Duplicate graph output");
|
||||
std::vector<bool> selected(g.nodes.size(), false);
|
||||
std::vector<std::string> pending = requested;
|
||||
while (!pending.empty()) {
|
||||
auto name = std::move(pending.back());
|
||||
pending.pop_back();
|
||||
auto p = producers.find(name);
|
||||
if (p == producers.end() || selected[p->second]) continue;
|
||||
selected[p->second] = true;
|
||||
admit(g.nodes[p->second]);
|
||||
auto deps = dependencies(g.nodes[p->second]);
|
||||
pending.insert(pending.end(), deps.begin(), deps.end());
|
||||
}
|
||||
std::vector<size_t> indegree(g.nodes.size());
|
||||
std::vector<std::vector<size_t>> users(g.nodes.size());
|
||||
size_t total = 0;
|
||||
std::priority_queue<size_t, std::vector<size_t>, std::greater<size_t>> ready;
|
||||
for (size_t i = 0; i < g.nodes.size(); ++i)
|
||||
if (selected[i]) {
|
||||
++total;
|
||||
std::set<size_t> parents;
|
||||
for (const auto& name : dependencies(g.nodes[i])) {
|
||||
auto p = producers.find(name);
|
||||
if (p != producers.end()) parents.insert(p->second);
|
||||
}
|
||||
indegree[i] = parents.size();
|
||||
for (auto p : parents) users[p].push_back(i);
|
||||
if (parents.empty()) ready.push(i);
|
||||
}
|
||||
std::vector<size_t> order;
|
||||
while (!ready.empty()) {
|
||||
size_t i = ready.top();
|
||||
ready.pop();
|
||||
order.push_back(i);
|
||||
for (auto user : users[i])
|
||||
if (--indegree[user] == 0) ready.push(user);
|
||||
}
|
||||
if (order.size() != total) throw Failure("ONNX graph has cyclic lexical dependencies", DiagnosticCode::ParseError);
|
||||
return order;
|
||||
}
|
||||
std::vector<Value> run(const ModelData&, Values, const std::vector<std::string>&, Context&, size_t);
|
||||
std::vector<Value> control(const Node& n, const std::vector<Value>& in, const Values& values, Context& c,
|
||||
size_t depth) {
|
||||
auto scalar = [&](size_t i) {
|
||||
const auto& t = in.at(i).tensor();
|
||||
if (count(t.shape) != 1) throw Failure("Control input must contain one element");
|
||||
return integer(t);
|
||||
};
|
||||
auto seq = [&](size_t i) -> const std::vector<Value>& {
|
||||
if (!in.at(i).sequence) throw Failure("Expected sequence input");
|
||||
return *in[i].sequence;
|
||||
};
|
||||
auto sequence = [](std::vector<Value> v, OnnxElement type) {
|
||||
return Value{nullptr, {}, std::make_shared<std::vector<Value>>(std::move(v)), type};
|
||||
};
|
||||
if (n.op == "SequenceEmpty") {
|
||||
auto type = static_cast<OnnxElement>(attr(n, "dtype", 1));
|
||||
elementSize(type);
|
||||
return {sequence({}, type)};
|
||||
}
|
||||
if (n.op == "SequenceAt") {
|
||||
const auto& s = seq(0);
|
||||
int64_t i = scalar(1);
|
||||
if (i < 0) i += static_cast<int64_t>(s.size());
|
||||
if (i < 0 || static_cast<size_t>(i) >= s.size()) throw Failure("SequenceAt out of bounds");
|
||||
return {s[i]};
|
||||
}
|
||||
if (n.op == "SequenceInsert") {
|
||||
auto s = seq(0);
|
||||
if (s.size() >= 100000) throw Failure("Sequence length limit exceeded");
|
||||
int64_t i = in.size() > 2 ? scalar(2) : static_cast<int64_t>(s.size());
|
||||
if (i < 0) i += static_cast<int64_t>(s.size());
|
||||
if (i < 0 || static_cast<size_t>(i) > s.size()) throw Failure("SequenceInsert out of bounds");
|
||||
const auto& t = in.at(1).tensor();
|
||||
if (in[0].sequenceElement != t.element) throw Failure("Sequence element dtype mismatch");
|
||||
s.insert(s.begin() + i, in[1]);
|
||||
return {sequence(std::move(s), in[0].sequenceElement)};
|
||||
}
|
||||
if (n.op == "SplitToSequence") {
|
||||
const auto& x = in.at(0).tensor();
|
||||
int a = axis(attr(n, "axis", 0), x.shape.size());
|
||||
std::vector<int64_t> lengths;
|
||||
if (in.size() > 1 && (in[1].borrowed || in[1].owned)) {
|
||||
const auto& split = in[1].tensor();
|
||||
lengths = ints(split);
|
||||
if (split.shape.empty()) {
|
||||
if (lengths[0] <= 0) throw Failure("Invalid split size");
|
||||
int64_t width = lengths[0];
|
||||
lengths.clear();
|
||||
for (int64_t i = 0; i < x.shape[a]; i += width) {
|
||||
if (lengths.size() >= 100000) throw Failure("Sequence length limit exceeded");
|
||||
lengths.push_back(std::min(width, x.shape[a] - i));
|
||||
}
|
||||
}
|
||||
} else
|
||||
lengths.assign(x.shape[a], 1);
|
||||
if (lengths.size() > 100000) throw Failure("Sequence length limit exceeded");
|
||||
int64_t total = 0;
|
||||
for (auto size : lengths) {
|
||||
if (size < 0 || size > x.shape[a] - total) throw Failure("Invalid split lengths");
|
||||
total += size;
|
||||
}
|
||||
if (total != x.shape[a]) throw Failure("Split lengths do not cover axis");
|
||||
size_t outer = 1, inner = elementSize(x.element);
|
||||
for (int j = 0; j < a; ++j) outer *= x.shape[j];
|
||||
for (size_t j = a + 1; j < x.shape.size(); ++j) inner *= x.shape[j];
|
||||
std::vector<Value> s;
|
||||
size_t offset = 0;
|
||||
for (auto length : lengths) {
|
||||
auto shape = x.shape;
|
||||
shape[a] = length;
|
||||
if (!attr(n, "keepdims", 1)) {
|
||||
if (in.size() > 1 && (in[1].borrowed || in[1].owned))
|
||||
throw Failure("keepdims=0 with explicit split unsupported", DiagnosticCode::Unsupported);
|
||||
shape.erase(shape.begin() + a);
|
||||
}
|
||||
RuntimeTensor out{x.element, shape, std::vector<uint8_t>(count(shape) * elementSize(x.element))};
|
||||
for (size_t o = 0; o < outer; ++o)
|
||||
if (length)
|
||||
std::memcpy(out.bytes.data() + o * length * inner,
|
||||
x.bytes.data() + (o * x.shape[a] + offset) * inner, length * inner);
|
||||
s.push_back(hold(std::move(out), c));
|
||||
offset += length;
|
||||
}
|
||||
return {sequence(std::move(s), x.element)};
|
||||
}
|
||||
if (n.op == "ConcatFromSequence") {
|
||||
const auto& s = seq(0);
|
||||
if (s.empty()) throw Failure("Cannot concatenate empty sequence");
|
||||
std::vector<RuntimeTensor> tensors;
|
||||
std::vector<const RuntimeTensor*> inputs;
|
||||
for (const auto& v : s) {
|
||||
tensors.push_back(v.tensor());
|
||||
if (attr(n, "new_axis", 0)) {
|
||||
const int a = axis(attr(n, "axis", 0), tensors.back().shape.size() + 1);
|
||||
tensors.back().shape.insert(tensors.back().shape.begin() + a, 1);
|
||||
}
|
||||
}
|
||||
for (const auto& t : tensors) inputs.push_back(&t);
|
||||
Node concat = n;
|
||||
concat.op = "Concat";
|
||||
auto out = executeShape(concat, inputs);
|
||||
if (!out) throw Failure("ConcatFromSequence dispatch failed");
|
||||
return {hold(std::move(*out), c)};
|
||||
}
|
||||
if (n.op == "If") {
|
||||
if (in.at(0).tensor().element != OnnxElement::Bool) throw Failure("If condition must be boolean");
|
||||
auto key = scalar(0) ? "then_branch" : "else_branch";
|
||||
const auto& g = n.attrs.at(key).graph;
|
||||
if (!g) throw Failure("Missing If branch");
|
||||
return run(*g, values, g->info.outputs, c, depth + 1);
|
||||
}
|
||||
if (n.op == "Loop") {
|
||||
const auto& body = n.attrs.at("body").graph;
|
||||
if (!body) throw Failure("Missing Loop body");
|
||||
if (in.size() < 2 || body->info.inputs.size() != in.size() || body->info.outputs.size() != in.size() - 1 ||
|
||||
n.outputs.size() != in.size() - 2)
|
||||
throw Failure("Loop scan outputs unsupported", DiagnosticCode::Unsupported);
|
||||
const int64_t trips = (in[0].borrowed || in[0].owned) ? scalar(0) : 10000;
|
||||
if (trips < 0 || trips > 10000) throw Failure("Loop trip budget exceeded");
|
||||
bool condition = (in[1].borrowed || in[1].owned) ? scalar(1) != 0 : true;
|
||||
std::vector<Value> state(in.begin() + 2, in.end());
|
||||
for (int64_t i = 0; i < trips && condition; ++i) {
|
||||
auto scope = values;
|
||||
scope[body->info.inputs[0]] = hold(make(OnnxElement::Int64, {}, std::vector<int64_t>{i}), c);
|
||||
scope[body->info.inputs[1]] =
|
||||
hold(make(OnnxElement::Bool, {}, std::vector<uint8_t>{uint8_t(condition)}), c);
|
||||
for (size_t j = 0; j < state.size(); ++j) scope[body->info.inputs[j + 2]] = state[j];
|
||||
auto result = run(*body, std::move(scope), body->info.outputs, c, depth + 1);
|
||||
const auto& cond = result[0].tensor();
|
||||
if (cond.element != OnnxElement::Bool || count(cond.shape) != 1)
|
||||
throw Failure("Loop returned invalid condition");
|
||||
condition = integer(cond) != 0;
|
||||
state.assign(result.begin() + 1, result.end());
|
||||
}
|
||||
return state;
|
||||
}
|
||||
if (n.op == "RandomUniformLike" || n.op == "RandomNormalLike") {
|
||||
const auto& x = in.at(0).tensor();
|
||||
if (attr(n, "dtype", static_cast<int>(x.element)) != 1)
|
||||
throw Failure("Random output requires FP32", DiagnosticCode::Unsupported);
|
||||
uint64_t seed = c.seed;
|
||||
for (unsigned char ch : n.name) seed = (seed ^ ch) * 1099511628211ull;
|
||||
if (n.attrs.contains("seed")) {
|
||||
uint32_t bits;
|
||||
std::memcpy(&bits, &n.attrs.at("seed").real, 4);
|
||||
seed ^= bits;
|
||||
}
|
||||
auto [stream, inserted] = c.randomStreams.try_emplace(&n, static_cast<uint32_t>(seed ^ (seed >> 32)));
|
||||
auto& rng = stream->second;
|
||||
auto uniform = [&]() { return (double(rng()) + .5) / 4294967296.; };
|
||||
std::vector<float> v(count(x.shape));
|
||||
const float low = n.attrs.contains("low") ? n.attrs.at("low").real : 0,
|
||||
high = n.attrs.contains("high") ? n.attrs.at("high").real : 1,
|
||||
mean = n.attrs.contains("mean") ? n.attrs.at("mean").real : 0,
|
||||
scale = n.attrs.contains("scale") ? n.attrs.at("scale").real : 1;
|
||||
if (!std::isfinite(low) || !std::isfinite(high) || !std::isfinite(mean) || !std::isfinite(scale) ||
|
||||
high < low || scale < 0)
|
||||
throw Failure("Invalid random distribution parameters");
|
||||
for (auto& f : v)
|
||||
f = n.op == "RandomUniformLike" ? static_cast<float>(low + (high - low) * uniform())
|
||||
: static_cast<float>(mean + scale * std::sqrt(-2 * std::log(uniform())) *
|
||||
std::cos(6.283185307179586 * uniform()));
|
||||
return {hold(make(OnnxElement::Float32, x.shape, v), c)};
|
||||
}
|
||||
std::vector<const RuntimeTensor*> tensors;
|
||||
for (const auto& v : in) tensors.push_back(v.borrowed || v.owned ? &v.tensor() : nullptr);
|
||||
auto outputs = execute(n, tensors, c.compute);
|
||||
std::vector<Value> result;
|
||||
for (auto& t : outputs) result.push_back(hold(std::move(t), c));
|
||||
return result;
|
||||
}
|
||||
std::vector<Value> run(const ModelData& g, Values values, const std::vector<std::string>& requested, Context& c,
|
||||
size_t depth) {
|
||||
if (depth > 16) throw Failure("Execution graph nesting limit exceeded");
|
||||
for (const auto& [name, t] : g.constants)
|
||||
if (!g.inputs.contains(name) || !values.contains(name)) values[name] = {&t, {}, {}};
|
||||
auto order = plan(g, requested);
|
||||
std::unordered_map<std::string, size_t> uses;
|
||||
for (auto i : order)
|
||||
for (const auto& dep : dependencies(g.nodes[i])) ++uses[dep];
|
||||
for (const auto& name : requested) ++uses[name];
|
||||
for (auto i : order) {
|
||||
const auto& n = g.nodes[i];
|
||||
if (++c.steps > 1000000) throw Failure("ONNX execution step budget exceeded");
|
||||
try {
|
||||
std::vector<Value> in;
|
||||
for (const auto& name : n.inputs) {
|
||||
if (name.empty()) {
|
||||
in.push_back({});
|
||||
continue;
|
||||
}
|
||||
auto it = values.find(name);
|
||||
if (it == values.end()) throw Failure("Missing feed or lexical capture: " + name);
|
||||
in.push_back(it->second);
|
||||
}
|
||||
auto outputs = control(n, in, values, c, depth);
|
||||
if (outputs.size() < n.outputs.size()) throw Failure("Operator output arity mismatch");
|
||||
for (size_t j = 0; j < n.outputs.size(); ++j)
|
||||
if (!n.outputs[j].empty() && uses[n.outputs[j]]) values[n.outputs[j]] = std::move(outputs[j]);
|
||||
for (const auto& dep : dependencies(n))
|
||||
if (--uses[dep] == 0) values.erase(dep);
|
||||
} catch (const Failure& e) {
|
||||
throw Failure(n.name + ": " + e.what(), e.code, e.path.empty() ? n.name : e.path);
|
||||
}
|
||||
}
|
||||
std::vector<Value> out;
|
||||
for (const auto& name : requested) {
|
||||
auto it = values.find(name);
|
||||
if (it == values.end()) throw Failure("Missing output: " + name, DiagnosticCode::NotFound);
|
||||
out.push_back(it->second);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
} // namespace
|
||||
std::vector<OnnxNamedTensor> evaluate(const ModelData& model, std::span<const OnnxNamedTensor> feeds,
|
||||
const std::vector<std::string>& requested, OnnxCompute* compute,
|
||||
OnnxRunOptions options) {
|
||||
Context c{compute, options.seed, options.requireFinite};
|
||||
Values inputs;
|
||||
for (const auto& f : feeds) inputs[f.name] = hold({f.tensor.element, f.tensor.shape, f.tensor.bytes}, c);
|
||||
auto outputs = run(model, std::move(inputs), requested, c, 0);
|
||||
std::vector<OnnxNamedTensor> result;
|
||||
size_t bytes = c.liveBytes;
|
||||
for (size_t i = 0; i < outputs.size(); ++i) {
|
||||
const auto& t = outputs[i].tensor();
|
||||
if (t.bytes.size() > 512u * 1024u * 1024u - bytes) throw Failure("ONNX output memory limit exceeded");
|
||||
bytes += t.bytes.size();
|
||||
result.push_back({requested[i], {t.element, t.shape, static_cast<std::vector<uint8_t>>(t.bytes)}});
|
||||
}
|
||||
return result;
|
||||
}
|
||||
} // namespace eve::tensor::onnx_detail
|
||||
@@ -0,0 +1,196 @@
|
||||
#include <algorithm>
|
||||
#include <cmath>
|
||||
#include <numeric>
|
||||
#include <set>
|
||||
#include "tensor/OnnxInternal.h"
|
||||
|
||||
namespace eve::tensor::onnx_detail {
|
||||
std::vector<int64_t> broadcast(const std::vector<int64_t>& a, const std::vector<int64_t>& b) {
|
||||
std::vector<int64_t> out(std::max(a.size(), b.size()), 1);
|
||||
for (size_t i = 0; i < out.size(); ++i) {
|
||||
const auto x = i < a.size() ? a[a.size() - 1 - i] : 1, y = i < b.size() ? b[b.size() - 1 - i] : 1;
|
||||
if (x != y && x != 1 && y != 1) throw Failure("Broadcast dimension mismatch");
|
||||
out[out.size() - 1 - i] = x == 1 ? y : x;
|
||||
}
|
||||
count(out);
|
||||
return out;
|
||||
}
|
||||
size_t broadcastIndex(size_t i, const std::vector<int64_t>& shape, const std::vector<int64_t>& output) {
|
||||
size_t index = 0, stride = 1;
|
||||
for (size_t j = 0; j < output.size(); ++j) {
|
||||
const auto coordinate = i % output[output.size() - 1 - j];
|
||||
i /= output[output.size() - 1 - j];
|
||||
if (j < shape.size()) {
|
||||
const auto d = shape[shape.size() - 1 - j];
|
||||
if (d != 1) index += coordinate * stride;
|
||||
stride *= d;
|
||||
}
|
||||
}
|
||||
return index;
|
||||
}
|
||||
std::optional<RuntimeTensor> executeShape(const Node& n, const std::vector<const RuntimeTensor*>& in) {
|
||||
const auto& x = required(in, 0);
|
||||
if (n.op == "Slice") {
|
||||
auto starts = ints(required(in, 1)), ends = ints(required(in, 2));
|
||||
auto axes = in.size() > 3 && in[3] ? ints(*in[3]) : std::vector<int64_t>(starts.size());
|
||||
if (in.size() <= 3 || !in[3]) std::iota(axes.begin(), axes.end(), 0);
|
||||
auto steps = in.size() > 4 && in[4] ? ints(*in[4]) : std::vector<int64_t>(starts.size(), 1);
|
||||
if (starts.size() != ends.size() || starts.size() != axes.size() || starts.size() != steps.size())
|
||||
throw Failure("Slice parameter lengths mismatch");
|
||||
auto shape = x.shape;
|
||||
std::vector<int64_t> begin(shape.size(), 0), step(shape.size(), 1);
|
||||
std::set<int> seen;
|
||||
for (size_t j = 0; j < axes.size(); ++j) {
|
||||
int a = axis(axes[j], shape.size());
|
||||
if (!seen.insert(a).second || !steps[j] || steps[j] == INT64_MIN) throw Failure("Invalid Slice axis/step");
|
||||
const auto d = x.shape[a], st = steps[j];
|
||||
auto norm = [d, st](int64_t v, bool end) {
|
||||
if (v < 0) v = v < -d ? (st < 0 ? -1 : 0) : v + d;
|
||||
return std::clamp(v, st < 0 && end ? int64_t(-1) : int64_t(0),
|
||||
st < 0 ? std::max(int64_t(0), d - 1) : d);
|
||||
};
|
||||
begin[a] = norm(starts[j], false);
|
||||
const auto e = norm(ends[j], true);
|
||||
step[a] = st;
|
||||
const int64_t distance = st > 0 ? e - begin[a] : begin[a] - e, magnitude = st > 0 ? st : -st;
|
||||
shape[a] = d == 0 || distance <= 0 ? 0 : 1 + (distance - 1) / magnitude;
|
||||
}
|
||||
RuntimeTensor out{x.element, shape, std::vector<uint8_t>(count(shape) * elementSize(x.element))};
|
||||
for (size_t i = 0; i < count(shape); ++i) {
|
||||
size_t rest = i, source = 0, stride = 1;
|
||||
for (size_t j = shape.size(); j > 0; --j) {
|
||||
auto c = rest % shape[j - 1];
|
||||
rest /= shape[j - 1];
|
||||
source += (begin[j - 1] + c * step[j - 1]) * stride;
|
||||
stride *= x.shape[j - 1];
|
||||
}
|
||||
std::memcpy(out.bytes.data() + i * elementSize(x.element), x.bytes.data() + source * elementSize(x.element),
|
||||
elementSize(x.element));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
if (n.op == "Concat") {
|
||||
int a = axis(attr(n, "axis", 0), x.shape.size());
|
||||
auto shape = x.shape;
|
||||
shape[a] = 0;
|
||||
for (auto* t : in) {
|
||||
if (!t || t->element != x.element || t->shape.size() != shape.size())
|
||||
throw Failure("Concat type/rank mismatch");
|
||||
for (size_t j = 0; j < shape.size(); ++j)
|
||||
if (j != a && t->shape[j] != shape[j]) throw Failure("Concat dimension mismatch");
|
||||
shape[a] += t->shape[a];
|
||||
}
|
||||
RuntimeTensor out{x.element, shape, std::vector<uint8_t>(count(shape) * elementSize(x.element))};
|
||||
size_t outer = 1, inner = elementSize(x.element);
|
||||
for (int j = 0; j < a; ++j) outer *= shape[j];
|
||||
for (size_t j = a + 1; j < shape.size(); ++j) inner *= shape[j];
|
||||
size_t offset = 0;
|
||||
for (size_t o = 0; o < outer; ++o)
|
||||
for (auto* t : in) {
|
||||
const size_t size = t->shape[a] * inner;
|
||||
if (size) std::memcpy(out.bytes.data() + offset, t->bytes.data() + o * size, size);
|
||||
offset += size;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
if (n.op == "Expand") {
|
||||
auto requested = ints(required(in, 1));
|
||||
count(requested);
|
||||
auto shape = broadcast(x.shape, requested);
|
||||
RuntimeTensor out{x.element, shape, std::vector<uint8_t>(count(shape) * elementSize(x.element))};
|
||||
for (size_t i = 0; i < count(shape); ++i)
|
||||
std::memcpy(out.bytes.data() + i * elementSize(x.element),
|
||||
x.bytes.data() + broadcastIndex(i, x.shape, shape) * elementSize(x.element),
|
||||
elementSize(x.element));
|
||||
return out;
|
||||
}
|
||||
if (n.op == "ConstantOfShape") {
|
||||
auto shape = ints(x);
|
||||
RuntimeTensor value = make(OnnxElement::Float32, {}, std::vector<float>{0});
|
||||
if (n.attrs.contains("value")) {
|
||||
if (!n.attrs.at("value").tensor) throw Failure("Missing fill value");
|
||||
value = *n.attrs.at("value").tensor;
|
||||
}
|
||||
if (count(value.shape) != 1) throw Failure("ConstantOfShape requires one fill value");
|
||||
RuntimeTensor out{value.element, shape, std::vector<uint8_t>(count(shape) * elementSize(value.element))};
|
||||
for (size_t i = 0; i < out.bytes.size(); i += value.bytes.size())
|
||||
std::memcpy(out.bytes.data() + i, value.bytes.data(), value.bytes.size());
|
||||
return out;
|
||||
}
|
||||
if (n.op == "Where") {
|
||||
const auto& a = required(in, 1);
|
||||
const auto& b = required(in, 2);
|
||||
if (x.element != OnnxElement::Bool || a.element != b.element) throw Failure("Where type mismatch");
|
||||
auto shape = broadcast(broadcast(x.shape, a.shape), b.shape);
|
||||
const auto size = elementSize(a.element);
|
||||
RuntimeTensor out{a.element, shape, std::vector<uint8_t>(count(shape) * size)};
|
||||
for (size_t i = 0; i < count(shape); ++i) {
|
||||
const auto& v = x.bytes[broadcastIndex(i, x.shape, shape)] ? a : b;
|
||||
std::memcpy(out.bytes.data() + i * size, v.bytes.data() + broadcastIndex(i, v.shape, shape) * size, size);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
if (n.op == "Not") {
|
||||
if (x.element != OnnxElement::Bool) throw Failure("Not requires boolean");
|
||||
auto out = x;
|
||||
for (auto& b : out.bytes) b = !b;
|
||||
return out;
|
||||
}
|
||||
if (n.op == "Equal" || n.op == "Less" || n.op == "Greater" || n.op == "And") {
|
||||
const auto& y = required(in, 1);
|
||||
if (x.element != y.element || (n.op == "And" && x.element != OnnxElement::Bool))
|
||||
throw Failure("Comparison type mismatch");
|
||||
auto shape = broadcast(x.shape, y.shape);
|
||||
RuntimeTensor out{OnnxElement::Bool, shape, std::vector<uint8_t>(count(shape))};
|
||||
for (size_t i = 0; i < out.bytes.size(); ++i) {
|
||||
auto a = broadcastIndex(i, x.shape, shape), b = broadcastIndex(i, y.shape, shape);
|
||||
auto compare = [&](auto u, auto v) {
|
||||
return n.op == "Equal" ? u == v
|
||||
: n.op == "Less" ? u < v
|
||||
: n.op == "Greater" ? u > v
|
||||
: bool(u) && bool(v);
|
||||
};
|
||||
out.bytes[i] = x.element == OnnxElement::Float32 ? compare(read<float>(x, a), read<float>(y, b))
|
||||
: compare(integer(x, a), integer(y, b));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
if ((n.op == "Add" || n.op == "Sub" || n.op == "Mul" || n.op == "Div") && x.element != OnnxElement::Float32) {
|
||||
const auto& y = required(in, 1);
|
||||
if (x.element != y.element || (x.element != OnnxElement::Int64 && x.element != OnnxElement::Int32))
|
||||
throw Failure("Integer arithmetic dtype mismatch");
|
||||
auto shape = broadcast(x.shape, y.shape);
|
||||
RuntimeTensor out{x.element, shape, std::vector<uint8_t>(count(shape) * elementSize(x.element))};
|
||||
for (size_t i = 0; i < count(shape); ++i) {
|
||||
int64_t a = integer(x, broadcastIndex(i, x.shape, shape)),
|
||||
b = integer(y, broadcastIndex(i, y.shape, shape)), v = 0;
|
||||
if (n.op == "Add") {
|
||||
if ((b > 0 && a > INT64_MAX - b) || (b < 0 && a < INT64_MIN - b))
|
||||
throw Failure("Integer addition overflow");
|
||||
v = a + b;
|
||||
}
|
||||
if (n.op == "Sub") {
|
||||
if ((b < 0 && a > INT64_MAX + b) || (b > 0 && a < INT64_MIN + b))
|
||||
throw Failure("Integer subtraction overflow");
|
||||
v = a - b;
|
||||
}
|
||||
if (n.op == "Mul") {
|
||||
if (a && b &&
|
||||
(a > 0 ? (b > 0 ? a > INT64_MAX / b : b < INT64_MIN / a)
|
||||
: (b > 0 ? a < INT64_MIN / b : a < INT64_MAX / b)))
|
||||
throw Failure("Integer multiplication overflow");
|
||||
v = a * b;
|
||||
}
|
||||
if (n.op == "Div") {
|
||||
if (!b || (a == INT64_MIN && b == -1)) throw Failure("Invalid integer division");
|
||||
v = a / b;
|
||||
}
|
||||
if (x.element == OnnxElement::Int32 && (v < INT32_MIN || v > INT32_MAX))
|
||||
throw Failure("Int32 arithmetic overflow");
|
||||
std::memcpy(out.bytes.data() + i * elementSize(x.element), &v, elementSize(x.element));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
return std::nullopt;
|
||||
}
|
||||
} // namespace eve::tensor::onnx_detail
|
||||
@@ -0,0 +1,33 @@
|
||||
#pragma once
|
||||
#include <cstdint>
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
#include "common/Result.h"
|
||||
namespace eve::tensor {
|
||||
/** @brief Run-scoped device storage. Readback completes pending work on the device thread.
|
||||
* @note Implementations reject access after their run ends; no device handles escape runGpu. */
|
||||
class OnnxDeviceStorage {
|
||||
public:
|
||||
/** @brief Release owning storage on the device thread. */
|
||||
virtual ~OnnxDeviceStorage() = default;
|
||||
/** @brief Return an owning host copy, or an explicit device/lifecycle error. */
|
||||
[[nodiscard]] virtual Result<std::vector<uint8_t>> readback() const = 0;
|
||||
};
|
||||
/** @brief Immutable buffer shared by graph aliases; exactly one storage is authoritative.
|
||||
* @note host owns CPU bytes; device owns GPU bytes. Host inputs may be cached by shared identity
|
||||
* across calls only when persistent marks an immutable model initializer. Consumers use the device thread. */
|
||||
struct OnnxBuffer {
|
||||
std::shared_ptr<const std::vector<uint8_t>> host;
|
||||
std::shared_ptr<OnnxDeviceStorage> device;
|
||||
size_t size = 0;
|
||||
bool persistent = false; // Immutable model initializer, eligible for bounded cross-run caching.
|
||||
};
|
||||
/** @brief Actual transfers and submissions made during one GPU run. */
|
||||
struct OnnxTransferStats {
|
||||
size_t shaderCompilations = 0, shaderCacheHits = 0;
|
||||
size_t uploads = 0, downloads = 0, submissions = 0;
|
||||
size_t uploadedBytes = 0, downloadedBytes = 0;
|
||||
size_t bufferAllocations = 0, bufferReuses = 0;
|
||||
size_t compilerPeakWorkers = 0;
|
||||
};
|
||||
} // namespace eve::tensor
|
||||
@@ -250,6 +250,21 @@ elseif(EVENGINE_IOS_TEST_APP)
|
||||
EVENGINE_IOS_TEST_APP=1)
|
||||
else()
|
||||
target_link_libraries(unit_test ${EVELIBS} zeroerr ${ThirdParty} ${EVENGINE_VULKAN_LIB} ${EVENGINE_WEBGPU_LIB})
|
||||
if(TARGET EVTensor AND NOT EVENGINE_WEBGPU)
|
||||
add_executable(onnx_engine_probe EXCLUDE_FROM_ALL "${CMAKE_SOURCE_DIR}/examples/tensor/onnx/main.cpp")
|
||||
target_compile_definitions(onnx_engine_probe PRIVATE EVE_ONNX_ENGINE_PROVIDER POCO_STATIC POCO_NO_AUTOMATIC_LIBS NOMINMAX)
|
||||
target_link_libraries(onnx_engine_probe eve_engine_includes ${EVELIBS} zeroerr ${ThirdParty} ${EVENGINE_VULKAN_LIB})
|
||||
add_executable(onnx_gpu_contracts EXCLUDE_FROM_ALL main.cpp tensor_onnx_gpu.cpp)
|
||||
target_compile_definitions(onnx_gpu_contracts PRIVATE POCO_STATIC POCO_NO_AUTOMATIC_LIBS NOMINMAX)
|
||||
target_link_libraries(onnx_gpu_contracts eve_engine_includes ${EVELIBS} zeroerr ${ThirdParty} ${EVENGINE_VULKAN_LIB})
|
||||
|
||||
|
||||
if(WIN32)
|
||||
target_link_libraries(onnx_engine_probe EVBacktrace)
|
||||
target_link_libraries(onnx_gpu_contracts EVBacktrace)
|
||||
endif()
|
||||
endif()
|
||||
|
||||
# Scene composition must also run in a trimmed host, where unrelated unit-test
|
||||
# translation units still reference disabled optional domains.
|
||||
if(TARGET EVScene_editor)
|
||||
|
||||
@@ -0,0 +1,231 @@
|
||||
#include "tensor/AffineQuant.h"
|
||||
#include "tensor/OnnxByteStorage.h"
|
||||
#include "tensor/OnnxCompute.h"
|
||||
#include "tensor/OnnxModel.h"
|
||||
#include "zeroerr/unittest.h"
|
||||
|
||||
#include <cmath>
|
||||
#include <cstring>
|
||||
#include <limits>
|
||||
|
||||
namespace {
|
||||
using Bytes = std::vector<uint8_t>;
|
||||
void varint(Bytes& b, uint64_t n) {
|
||||
while (n >= 128) {
|
||||
b.push_back(static_cast<uint8_t>(n) | 128);
|
||||
n >>= 7;
|
||||
}
|
||||
b.push_back(static_cast<uint8_t>(n));
|
||||
}
|
||||
void number(Bytes& b, int id, uint64_t n) {
|
||||
varint(b, id * 8);
|
||||
varint(b, n);
|
||||
}
|
||||
void field(Bytes& b, int id, const Bytes& value) {
|
||||
varint(b, id * 8 + 2);
|
||||
varint(b, value.size());
|
||||
b.insert(b.end(), value.begin(), value.end());
|
||||
}
|
||||
Bytes text(const char* s) { return Bytes(s, s + std::char_traits<char>::length(s)); }
|
||||
Bytes unknownModel() {
|
||||
Bytes node, graph, output, opset, model;
|
||||
field(node, 2, text("y"));
|
||||
field(node, 3, text("unsupported-node"));
|
||||
field(node, 4, text("UnknownOperator"));
|
||||
field(graph, 1, node);
|
||||
field(output, 1, text("y"));
|
||||
field(graph, 12, output);
|
||||
number(opset, 2, 14);
|
||||
number(model, 1, 7);
|
||||
field(model, 8, opset);
|
||||
field(model, 7, graph);
|
||||
return model;
|
||||
}
|
||||
} // namespace
|
||||
|
||||
TEST_CASE("tensor.onnx.affine.roundingAndRange") {
|
||||
using namespace eve::tensor::affine;
|
||||
const std::vector<float> input{-1000, -1.25f, -.75f, -.25f, .25f, .75f, 1.25f, 1000};
|
||||
auto q = quantize(input, .5f, 0, true);
|
||||
REQUIRE(q.ok());
|
||||
CHECK(q.value() == Bytes({128, 254, 254, 0, 0, 2, 2, 127}));
|
||||
auto zero = dynamicQuantize(std::vector<float>{0, 0});
|
||||
REQUIRE(zero.ok());
|
||||
CHECK(zero.value().scale == 1);
|
||||
auto invalid = quantize(input, 0, 0, false);
|
||||
CHECK(!invalid.ok());
|
||||
auto nonfinite = dynamicQuantize(std::vector<float>{std::numeric_limits<float>::infinity()});
|
||||
CHECK(!nonfinite.ok());
|
||||
}
|
||||
|
||||
TEST_CASE("tensor.onnx.affine.exactAccumulators") {
|
||||
using namespace eve::tensor::affine;
|
||||
Bytes a(1025, 255), b(1025, 127);
|
||||
auto product = matmul({a, false}, {b, true}, 1, 1025, 1);
|
||||
REQUIRE(product.ok());
|
||||
CHECK(product.value()[0] == 33194625);
|
||||
auto mismatch = matmul({a, false}, {b, true}, 2, 1025, 1);
|
||||
CHECK(!mismatch.ok());
|
||||
auto zero = matmul({a, false}, {b, true}, 1, 1025, 1, 256);
|
||||
CHECK(!zero.ok());
|
||||
Bytes largeA(40000, 255), largeB(40000, 255);
|
||||
auto overflow = matmul({largeA, false}, {largeB, false}, 1, 40000, 1);
|
||||
CHECK(!overflow.ok());
|
||||
}
|
||||
|
||||
TEST_CASE("tensor.onnx.affine.convolutionValidation") {
|
||||
using namespace eve::tensor::affine;
|
||||
ConvShape shape;
|
||||
shape.width = 3;
|
||||
shape.kernelW = 3;
|
||||
shape.padLeft = 1;
|
||||
shape.padRight = 1;
|
||||
const Bytes x{11, 12, 13}, w{1, 1, 1};
|
||||
const std::vector<int32_t> z{0};
|
||||
auto result = conv({x, false}, {w, false}, shape, 10, z);
|
||||
REQUIRE(result.ok());
|
||||
CHECK(result.value() == std::vector<int32_t>({3, 6, 5}));
|
||||
shape.groups = 2;
|
||||
auto invalid = conv({x, false}, {w, false}, shape, 10, z);
|
||||
CHECK(!invalid.ok());
|
||||
}
|
||||
|
||||
TEST_CASE("tensor.onnx.import.transactionAndDiagnostics") {
|
||||
using eve::tensor::OnnxModel;
|
||||
auto bytes = unknownModel();
|
||||
auto good = OnnxModel::load(bytes);
|
||||
REQUIRE(good.ok());
|
||||
CHECK(good.value()->info().unsupportedNodes.size() == 1);
|
||||
auto rejected = good.value()->run({});
|
||||
REQUIRE(!rejected.ok());
|
||||
REQUIRE(rejected.error() != nullptr);
|
||||
CHECK(rejected.error()->code() == eve::DiagnosticCode::Unsupported);
|
||||
CHECK(rejected.error()->path() == "unsupported-node");
|
||||
bytes.pop_back();
|
||||
auto broken = OnnxModel::load(bytes);
|
||||
CHECK(!broken.ok());
|
||||
CHECK(good.value()->info().nodeCount == 1);
|
||||
auto empty = OnnxModel::load({});
|
||||
CHECK(!empty.ok());
|
||||
auto overflow = OnnxModel::load(Bytes(12, 255));
|
||||
CHECK(!overflow.ok());
|
||||
}
|
||||
|
||||
TEST_CASE("tensor.onnx.import.owningOutputs") {
|
||||
Bytes tensor, attribute, node, graph, output, opset, encoded;
|
||||
number(tensor, 2, 1);
|
||||
const float expected = 1.5f;
|
||||
Bytes payload(sizeof(float));
|
||||
std::memcpy(payload.data(), &expected, sizeof(float));
|
||||
field(tensor, 9, payload);
|
||||
field(attribute, 1, text("value"));
|
||||
number(attribute, 20, 4);
|
||||
field(attribute, 5, tensor);
|
||||
field(node, 2, text("y"));
|
||||
field(node, 4, text("Constant"));
|
||||
field(node, 5, attribute);
|
||||
field(graph, 1, node);
|
||||
field(output, 1, text("y"));
|
||||
field(graph, 12, output);
|
||||
number(opset, 2, 14);
|
||||
number(encoded, 1, 7);
|
||||
field(encoded, 8, opset);
|
||||
field(encoded, 7, graph);
|
||||
auto imported = eve::tensor::OnnxModel::load(encoded);
|
||||
REQUIRE(imported.ok());
|
||||
encoded.clear();
|
||||
auto result = imported.value()->run({});
|
||||
REQUIRE(result.ok());
|
||||
imported.value().reset();
|
||||
REQUIRE(result.value().size() == 1);
|
||||
REQUIRE(result.value()[0].tensor.bytes.size() == 4);
|
||||
float actual;
|
||||
std::memcpy(&actual, result.value()[0].tensor.bytes.data(), 4);
|
||||
CHECK(actual == expected);
|
||||
}
|
||||
|
||||
TEST_CASE("tensor.onnx.gpu.failureAndScope") {
|
||||
class FailedGpu final : public eve::tensor::OnnxCompute {
|
||||
public:
|
||||
int begins = 0, ends = 0, dispatches = 0;
|
||||
eve::Result<std::vector<uint8_t>> dispatch(const eve::tensor::OnnxKernel&) override {
|
||||
++dispatches;
|
||||
return eve::Result<std::vector<uint8_t>>::failure(
|
||||
eve::Diagnostic::error(eve::DiagnosticCode::Failed, "injected GPU failure"));
|
||||
}
|
||||
|
||||
protected:
|
||||
void beginRun() noexcept override { ++begins; }
|
||||
void endRun() noexcept override { ++ends; }
|
||||
} gpu;
|
||||
|
||||
auto unsupported = eve::tensor::OnnxModel::load(unknownModel());
|
||||
REQUIRE(unsupported.ok());
|
||||
|
||||
auto rejected = unsupported.value()->runGpu({}, gpu);
|
||||
|
||||
CHECK(!rejected.ok());
|
||||
CHECK(gpu.dispatches == 0);
|
||||
CHECK(gpu.begins == 1);
|
||||
CHECK(gpu.ends == 1);
|
||||
Bytes tensor, node, graph, output, opset, model;
|
||||
number(tensor, 2, 1);
|
||||
field(tensor, 8, text("x"));
|
||||
float value = -2;
|
||||
Bytes data(4);
|
||||
std::memcpy(data.data(), &value, 4);
|
||||
field(tensor, 9, data);
|
||||
field(node, 1, text("x"));
|
||||
field(node, 2, text("y"));
|
||||
field(node, 3, text("relu-test"));
|
||||
field(node, 4, text("Relu"));
|
||||
field(graph, 1, node);
|
||||
field(graph, 5, tensor);
|
||||
field(output, 1, text("y"));
|
||||
field(graph, 12, output);
|
||||
number(opset, 2, 14);
|
||||
number(model, 1, 7);
|
||||
field(model, 8, opset);
|
||||
field(model, 7, graph);
|
||||
auto imported = eve::tensor::OnnxModel::load(model);
|
||||
REQUIRE(imported.ok());
|
||||
|
||||
auto failed = imported.value()->runGpu({}, gpu);
|
||||
REQUIRE(!failed.ok());
|
||||
CHECK(failed.error()->code() == eve::DiagnosticCode::Failed);
|
||||
CHECK(failed.error()->path() == "relu-test");
|
||||
CHECK(gpu.dispatches == 1);
|
||||
CHECK(gpu.begins == 2);
|
||||
CHECK(gpu.ends == 2);
|
||||
auto cpu = imported.value()->run({});
|
||||
REQUIRE(cpu.ok());
|
||||
float out = -1;
|
||||
std::memcpy(&out, cpu.value()[0].tensor.bytes.data(), 4);
|
||||
CHECK(out == 0);
|
||||
}
|
||||
|
||||
TEST_CASE("tensor.onnx.storage.aliasAndCpuMutation") {
|
||||
using eve::tensor::onnx_detail::ByteStorage;
|
||||
class Deferred final : public eve::tensor::OnnxDeviceStorage {
|
||||
public:
|
||||
mutable int reads = 0;
|
||||
eve::Result<std::vector<uint8_t>> readback() const override {
|
||||
++reads;
|
||||
return eve::Result<std::vector<uint8_t>>::success({1, 2, 3, 4});
|
||||
}
|
||||
};
|
||||
auto deferred = std::make_shared<Deferred>();
|
||||
ByteStorage original(eve::tensor::OnnxBuffer{{}, deferred, 4});
|
||||
ByteStorage alias = original;
|
||||
CHECK(alias.size() == 4);
|
||||
CHECK(deferred->reads == 0);
|
||||
alias.data()[0] = 9;
|
||||
CHECK(deferred->reads == 1);
|
||||
CHECK(static_cast<const ByteStorage&>(alias)[0] == 9);
|
||||
CHECK(static_cast<const ByteStorage&>(original)[0] == 1);
|
||||
CHECK(deferred->reads == 2);
|
||||
ByteStorage copy = alias;
|
||||
copy.data()[1] = 8;
|
||||
CHECK(static_cast<const ByteStorage&>(alias)[1] == 2);
|
||||
CHECK(static_cast<const ByteStorage&>(copy)[1] == 8);
|
||||
}
|
||||
@@ -0,0 +1,297 @@
|
||||
#include "common/config.h"
|
||||
#if !defined(EVENGINE_WEBGPU)
|
||||
#include <cstring>
|
||||
#include <memory>
|
||||
#include "gpgpu/ComputeProgram.h"
|
||||
#include "gpgpu/ComputeShader.h"
|
||||
#include "gpgpu/Gpgpu.h"
|
||||
#include "gpgpu/GpuBuffer.h"
|
||||
#include "gpgpu/Sequence.h"
|
||||
#include "graphics/Graphics.h"
|
||||
#include "graphics/vulkan/Graphics.h"
|
||||
#include "tensor/OnnxCompute.h"
|
||||
#include "tensor/OnnxModel.h"
|
||||
#include "zeroerr/unittest.h"
|
||||
namespace {
|
||||
void initialize() {
|
||||
static bool initialized = false;
|
||||
if (!initialized) {
|
||||
eve::graphics::Graphics::create()->initHeadless(64, 64);
|
||||
initialized = true;
|
||||
}
|
||||
}
|
||||
using Bytes = std::vector<uint8_t>;
|
||||
void number(Bytes& b, uint64_t n) {
|
||||
while (n >= 128) {
|
||||
b.push_back(uint8_t(n) | 128);
|
||||
n >>= 7;
|
||||
}
|
||||
b.push_back(uint8_t(n));
|
||||
}
|
||||
void integer(Bytes& b, int tag, uint64_t n) {
|
||||
number(b, tag * 8);
|
||||
number(b, n);
|
||||
}
|
||||
void field(Bytes& b, int tag, const Bytes& value) {
|
||||
number(b, tag * 8 + 2);
|
||||
number(b, value.size());
|
||||
b.insert(b.end(), value.begin(), value.end());
|
||||
}
|
||||
void text(Bytes& b, int tag, const std::string& s) { field(b, tag, Bytes(s.begin(), s.end())); }
|
||||
Bytes chain(size_t count = 4096, float increment = 1, bool invalidReshape = false, bool mixed = false,
|
||||
int stages = 16) {
|
||||
Bytes graph, tensor, raw(count * sizeof(float));
|
||||
for (size_t i = 0; i < count; ++i) {
|
||||
float f = float(i);
|
||||
std::memcpy(raw.data() + i * 4, &f, 4);
|
||||
}
|
||||
integer(tensor, 1, count);
|
||||
integer(tensor, 2, 1);
|
||||
text(tensor, 8, "x");
|
||||
field(tensor, 9, raw);
|
||||
field(graph, 5, tensor);
|
||||
Bytes bias, one(4);
|
||||
float f = increment;
|
||||
std::memcpy(one.data(), &f, 4);
|
||||
integer(bias, 2, 1);
|
||||
text(bias, 8, "one");
|
||||
field(bias, 9, one);
|
||||
field(graph, 5, bias);
|
||||
std::string previous = "x";
|
||||
for (int i = 0; i < stages; ++i) {
|
||||
Bytes n;
|
||||
text(n, 1, previous);
|
||||
text(n, 1, "one");
|
||||
auto next = "v" + std::to_string(i);
|
||||
text(n, 2, next);
|
||||
text(n, 4, mixed ? std::vector<std::string>{"Add", "Mul", "Sub", "Div"}[i % 4] : "Add");
|
||||
field(graph, 1, n);
|
||||
previous = next;
|
||||
Bytes alias;
|
||||
text(alias, 1, previous);
|
||||
previous += "alias";
|
||||
text(alias, 2, previous);
|
||||
text(alias, 4, "Identity");
|
||||
field(graph, 1, alias);
|
||||
}
|
||||
if (invalidReshape) {
|
||||
Bytes t, raw(8), node;
|
||||
int64_t dimension = int64_t(count) + 1;
|
||||
std::memcpy(raw.data(), &dimension, 8);
|
||||
integer(t, 1, 1);
|
||||
integer(t, 2, 7);
|
||||
text(t, 8, "badShape");
|
||||
field(t, 9, raw);
|
||||
field(graph, 5, t);
|
||||
text(node, 1, previous);
|
||||
text(node, 1, "badShape");
|
||||
text(node, 2, "invalid");
|
||||
text(node, 4, "Reshape");
|
||||
field(graph, 1, node);
|
||||
previous = "invalid";
|
||||
}
|
||||
Bytes out, opset, model;
|
||||
text(out, 1, previous);
|
||||
field(graph, 12, out);
|
||||
integer(opset, 2, 14);
|
||||
integer(model, 1, 7);
|
||||
field(model, 8, opset);
|
||||
field(model, 7, graph);
|
||||
return model;
|
||||
}
|
||||
} // namespace
|
||||
TEST_CASE("tensor.onnx.gpu.residentChain") {
|
||||
initialize();
|
||||
auto provider = eve::tensor::createOnnxGpuCompute();
|
||||
REQUIRE(provider.ok());
|
||||
auto model = eve::tensor::OnnxModel::load(chain());
|
||||
REQUIRE(model.ok());
|
||||
for (int repeat = 0; repeat < 2; ++repeat) {
|
||||
auto r = model.value()->runGpu({}, *provider.value());
|
||||
REQUIRE(r.ok());
|
||||
CHECK(r.value().dispatches == 16);
|
||||
CHECK(r.value().transfers.uploads == (repeat == 0 ? 2 : 0));
|
||||
CHECK(r.value().transfers.shaderCompilations == (repeat == 0 ? 1 : 0));
|
||||
CHECK(r.value().transfers.downloads == 1);
|
||||
CHECK(r.value().transfers.submissions == 1);
|
||||
const auto& raw = r.value().outputs[0].tensor.bytes;
|
||||
for (size_t i = 0; i < 4096; ++i) {
|
||||
float f;
|
||||
std::memcpy(&f, raw.data() + i * 4, 4);
|
||||
REQUIRE(f == float(i + 16));
|
||||
}
|
||||
}
|
||||
}
|
||||
TEST_CASE("gpgpu.sequence.stagingSizeReuse") {
|
||||
initialize();
|
||||
auto* gp = eve::gpgpu::Gpgpu::create();
|
||||
REQUIRE(gp->isAvailable());
|
||||
std::unique_ptr<eve::gpgpu::Sequence> seq(gp->newSequence());
|
||||
for (const auto& sizes :
|
||||
std::vector<std::vector<size_t>>{{4, 4096, 64}, {4096, 4, 64}, {8192, 8, 16384, 4}, {4, 16384, 8, 8192}}) {
|
||||
std::vector<std::unique_ptr<eve::gpgpu::GpuBuffer>> device, staging;
|
||||
std::vector<Bytes> expected;
|
||||
seq->begin();
|
||||
for (size_t i = 0; i < sizes.size(); ++i) {
|
||||
Bytes src(sizes[i], uint8_t(i + 31));
|
||||
expected.push_back(src);
|
||||
device.emplace_back(gp->newBuffer(int(sizes[i])));
|
||||
staging.emplace_back(gp->newBuffer(int(sizes[i]), "staging"));
|
||||
seq->recordUpload(device.back().get(), src.data(), src.size());
|
||||
// recordUpload promises an immediate copy, including in a multi-upload batch.
|
||||
std::fill(src.begin(), src.end(), 0);
|
||||
seq->recordDownload(device.back().get(), staging.back().get(), sizes[i]);
|
||||
}
|
||||
REQUIRE(seq->submitAsync() == eve::gpgpu::SequenceStatus::Submitted);
|
||||
REQUIRE(seq->wait() == eve::gpgpu::SequenceStatus::Complete);
|
||||
for (size_t i = 0; i < sizes.size(); ++i) {
|
||||
Bytes actual(sizes[i]);
|
||||
staging[i]->downloadBytes(actual.data(), actual.size());
|
||||
REQUIRE(actual == expected[i]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST_CASE("gpgpu.sequence.cancelRecordingRetiresDescriptors") {
|
||||
initialize();
|
||||
auto* gp = eve::gpgpu::Gpgpu::create();
|
||||
std::unique_ptr<eve::gpgpu::GpuBuffer> buffer(gp->newBuffer(4));
|
||||
std::unique_ptr<eve::gpgpu::ComputeShader> shader(
|
||||
gp->newShader("#version 450\nlayout(local_size_x=64)in;layout(std430,binding=0)buffer B{float x[];};void "
|
||||
"main(){if(gl_GlobalInvocationID.x==0)x[0]*=2.0;}"));
|
||||
float value = 1;
|
||||
buffer->uploadBytes(&value, 4);
|
||||
for (int i = 0; i < 80; ++i) {
|
||||
std::unique_ptr<eve::gpgpu::Sequence> canceled(gp->newSequence());
|
||||
canceled->begin();
|
||||
shader->bindBuffer(0, buffer.get());
|
||||
canceled->recordDispatch(shader.get(), 1);
|
||||
}
|
||||
std::unique_ptr<eve::gpgpu::Sequence> seq(gp->newSequence());
|
||||
std::unique_ptr<eve::gpgpu::GpuBuffer> staging(gp->newBuffer(4, "staging"));
|
||||
seq->begin();
|
||||
shader->bindBuffer(0, buffer.get());
|
||||
seq->recordDispatch(shader.get(), 1);
|
||||
seq->recordDownload(buffer.get(), staging.get(), 4);
|
||||
seq->submit();
|
||||
staging->downloadBytes(&value, 4);
|
||||
CHECK(value == 2);
|
||||
}
|
||||
|
||||
TEST_CASE("tensor.onnx.gpu.sessionShapesAndLifetime") {
|
||||
auto* previous = eve::graphics::Graphics::create();
|
||||
auto device = std::make_unique<eve::graphics::vulkan::Graphics>();
|
||||
eve::ModuleManager::insert("Graphics", device.get());
|
||||
struct Restore {
|
||||
eve::graphics::Graphics* previous;
|
||||
~Restore() { eve::ModuleManager::insert("Graphics", previous); }
|
||||
} restore{previous};
|
||||
device->initHeadless(64, 64);
|
||||
auto gpu = eve::tensor::createOnnxGpuCompute();
|
||||
REQUIRE(gpu.ok());
|
||||
auto original = eve::tensor::OnnxModel::load(chain());
|
||||
REQUIRE(original.ok());
|
||||
auto alternate = eve::tensor::OnnxModel::load(chain(2048, 2));
|
||||
REQUIRE(alternate.ok());
|
||||
for (int pass = 0; pass < 3; ++pass) {
|
||||
auto& model = pass == 1 ? alternate : original;
|
||||
auto r = model.value()->runGpu({}, *gpu.value());
|
||||
REQUIRE(r.ok());
|
||||
CHECK(r.value().transfers.shaderCompilations == (pass < 2 ? 1 : 0));
|
||||
const size_t count = pass == 1 ? 2048 : 4096;
|
||||
const float added = pass == 1 ? 32.f : 16.f;
|
||||
REQUIRE(r.value().outputs[0].tensor.bytes.size() == count * 4);
|
||||
for (size_t i = 0; i < count; ++i) {
|
||||
float actual;
|
||||
std::memcpy(&actual, r.value().outputs[0].tensor.bytes.data() + i * 4, 4);
|
||||
REQUIRE(actual == float(i) + added);
|
||||
}
|
||||
}
|
||||
auto failing = eve::tensor::OnnxModel::load(chain(4096, 1, true));
|
||||
REQUIRE(failing.ok());
|
||||
auto failure = failing.value()->runGpu({}, *gpu.value());
|
||||
REQUIRE(!failure.ok());
|
||||
const std::vector<std::string> intermediate{"v15alias"};
|
||||
auto recovered = failing.value()->runGpu({}, *gpu.value(), intermediate);
|
||||
REQUIRE(recovered.ok());
|
||||
CHECK(recovered.value().transfers.shaderCompilations == 0);
|
||||
CHECK(recovered.value().transfers.uploads == 2);
|
||||
float recoveredValue = 0;
|
||||
std::memcpy(&recoveredValue, recovered.value().outputs[0].tensor.bytes.data(), 4);
|
||||
CHECK(recoveredValue == 16);
|
||||
// Session-first retirement cancels its observer and releases device resources.
|
||||
{
|
||||
auto second = eve::tensor::createOnnxGpuCompute();
|
||||
REQUIRE(second.ok());
|
||||
auto r = original.value()->runGpu({}, *second.value());
|
||||
REQUIRE(r.ok());
|
||||
}
|
||||
// Device-first retirement clears the surviving session before destroying Vulkan.
|
||||
device.reset();
|
||||
eve::ModuleManager::insert("Graphics", previous);
|
||||
auto stale = original.value()->runGpu({}, *gpu.value());
|
||||
REQUIRE(!stale.ok());
|
||||
CHECK(stale.error()->message().find("retired") != std::string::npos);
|
||||
gpu.value().reset();
|
||||
}
|
||||
TEST_CASE("tensor.onnx.gpu.parallelCompilation") {
|
||||
// Compilation has no dependency on a live Graphics device.
|
||||
auto bytecode = eve::gpgpu::compileComputeSpirv("#version 450\nlayout(local_size_x=64)in;void main(){}");
|
||||
REQUIRE(bytecode.ok());
|
||||
REQUIRE(bytecode.value().front() == 0x07230203);
|
||||
auto invalidHeader = eve::gpgpu::createComputeShader({});
|
||||
REQUIRE(!invalidHeader.ok());
|
||||
for (uint32_t invalid : {0u, 9u}) {
|
||||
auto r = eve::tensor::createOnnxGpuCompute(invalid);
|
||||
REQUIRE(!r.ok());
|
||||
}
|
||||
initialize();
|
||||
Bytes serial;
|
||||
for (uint32_t workers : {1u, 4u}) {
|
||||
auto provider = eve::tensor::createOnnxGpuCompute(workers);
|
||||
REQUIRE(provider.ok());
|
||||
auto model = eve::tensor::OnnxModel::load(chain(4096, 2, false, true, 80));
|
||||
REQUIRE(model.ok());
|
||||
for (int repeat = 0; repeat < 2; ++repeat) {
|
||||
auto r = model.value()->runGpu({}, *provider.value());
|
||||
REQUIRE(r.ok());
|
||||
CHECK(r.value().dispatches == 80);
|
||||
CHECK(r.value().transfers.shaderCompilations == (repeat ? 0 : 4));
|
||||
CHECK(r.value().transfers.compilerPeakWorkers <= workers);
|
||||
if (!repeat)
|
||||
CHECK(r.value().transfers.compilerPeakWorkers >= (workers == 1 ? 1 : 2));
|
||||
else
|
||||
CHECK(r.value().transfers.compilerPeakWorkers == 0);
|
||||
const auto& raw = r.value().outputs[0].tensor.bytes;
|
||||
for (size_t i = 0; i < 4096; ++i) {
|
||||
float actual;
|
||||
std::memcpy(&actual, raw.data() + i * 4, 4);
|
||||
REQUIRE(actual == float(i + 20));
|
||||
}
|
||||
if (serial.empty())
|
||||
serial = raw;
|
||||
else
|
||||
REQUIRE(raw == serial);
|
||||
}
|
||||
// A fresh, abandoned segment has active CPU compile jobs but no submitted
|
||||
// GPU work. Retry must rebuild those programs and re-upload its weights.
|
||||
auto canceled = eve::tensor::OnnxModel::load(chain(4093, 2, true, true));
|
||||
REQUIRE(canceled.ok());
|
||||
auto failed = canceled.value()->runGpu({}, *provider.value());
|
||||
REQUIRE(!failed.ok());
|
||||
const std::vector<std::string> requested{"v15alias"};
|
||||
auto retried = canceled.value()->runGpu({}, *provider.value(), requested);
|
||||
REQUIRE(retried.ok());
|
||||
CHECK(retried.value().transfers.shaderCompilations == 4);
|
||||
CHECK(retried.value().transfers.uploads == 2);
|
||||
// Compilation failure travels through the future and run cleanup;
|
||||
// subsequent valid work on the same session remains usable.
|
||||
eve::tensor::OnnxKernel bad{"#version 450\nvoid main(){missing_function();}", {}, 4, 1};
|
||||
auto rejected = provider.value()->dispatch(bad);
|
||||
REQUIRE(!rejected.ok());
|
||||
auto recovered = model.value()->runGpu({}, *provider.value());
|
||||
REQUIRE(recovered.ok());
|
||||
REQUIRE(recovered.value().outputs[0].tensor.bytes == serial);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
Reference in new issue
Block a user