diff --git a/docs/usr/modules/tensor.md b/docs/usr/modules/tensor.md index f6d31dc8..fa50dd47 100644 --- a/docs/usr/modules/tensor.md +++ b/docs/usr/modules/tensor.md @@ -159,6 +159,26 @@ fp8/fp4 的 block scale 取 `maxAbs / 格式最大幅值`（e4m3=240、e2m1=6） 量化正确性由 `tensor.quant.*` 与 `tensor.llm.quantizedDtypes` 测试覆盖（含 GPU 编译路径），资产缺失时自动跳过。 +## C++ 原生 ONNX 导入与量化 GPU 推理 + +`tensor/OnnxModel.h` 直接导入 ONNX，保留 int8/uint8 权重、仿射 scale/zero-point +及精确 int32/int64 数据。`run()` 执行 CPU 参考路径；`runGpu()` 接收 +`createOnnxGpuCompute()` 返回的引擎 Gpgpu 适配器，在 Vulkan 设备线程上同步执行。 +GPU 负责量化矩阵乘、卷积、LSTM 投影、浮点神经网络计算；形状、索引、控制流、 +LSTM 门控仍由 CPU 处理；动态量化在 GPU 上计算，只回读少量校验标量。 +GPU 错误明确返回，不会自动改用 CPU 重试。 + +已用原版 Kokoro v1.1 INT8 模型跑通中文语音整图，包括 Loop/If/Sequence。 +接口使用 owning Result，输出不依赖模型生命周期，随机激励可指定种子。 +当前 ONNX GPU 执行已使用驻留缓冲区和批量 Sequence 提交；连续 GPU 节点之间不回读， +仅在 CPU 数据依赖边界下载。应在推理循环外创建并保留 `createOnnxGpuCompute()` +返回的 GPU 会话，跨调用复用管线、权重和缓冲区池；Graphics 销毁前自动清理， +失效会话明确拒绝执行。实测同进程第二次合成约 2.60 秒，仍需优化 LSTM CPU 门控等开销。文字前处理、Squirrel +绑定及 dialogue 播放接入不在此示例范围内。 + +完整构建命令、模型资源、数值契约、限制和可重复语音测试见 +[原生 ONNX 示例](../../../examples/tensor/onnx/README.md)。 + ## 常见问题 - 对 symbolic Tensor 调用 `get()`。 @@ -231,6 +251,9 @@ GPU 编译路径），资产缺失时自动跳过。 - 带 `update(dt)` 的系统应在 `eve_update` 调用；绘制方法应在 `eve_render` 调用。 - 参数约束、默认值和返回类型以对应模块头文件及 `addFunc` 绑定为准；本文 API 快查与当前源码同步生成。 - 张量支持 rank 1–6，dtype 为 float32 / int32；二元运算支持广播。 +- 原生 ONNX GPU 会话使用有界 shader 编译队列，`createOnnxGpuCompute(compilerWorkers)` + 可选 1–8 个 CPU 编译线程（默认 4）；同源码去重，编译器上下文和管线缓存均复用。 + Vulkan 管线创建与提交仍在设备线程，`runGpu` 保持同步返回；并发数应按目标构建实测选择。 - GPU 路径：`compile()` 需要已初始化的 Vulkan Graphics（先创建窗口）。Windows 上 GLSL→SPIR-V 由链接进引擎的 shaderc 静态库完成，不需要安装 glslc。 diff --git a/examples/tensor/onnx/CMakeLists.txt b/examples/tensor/onnx/CMakeLists.txt new file mode 100644 index 00000000..09e7ffbe --- /dev/null +++ b/examples/tensor/onnx/CMakeLists.txt @@ -0,0 +1,64 @@ +cmake_minimum_required(VERSION 3.21) +project(EveTensorOnnx LANGUAGES CXX) +get_filename_component(EVE_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/../../.." ABSOLUTE) +set(ZEROERR_SOURCE "${EVE_ROOT}/external/zeroerr" CACHE PATH "Initialized zeroerr source directory") +set(CMAKE_CXX_STANDARD 20) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +add_library(eve_tensor_onnx STATIC + "${EVE_ROOT}/src/modules/tensor/AffineQuant.cpp" + "${EVE_ROOT}/src/modules/tensor/OnnxReader.cpp" + "${EVE_ROOT}/src/modules/tensor/OnnxModel.cpp" + "${EVE_ROOT}/src/modules/tensor/OnnxRuntime.cpp" + "${EVE_ROOT}/src/modules/tensor/OnnxExecute.cpp" + "${EVE_ROOT}/src/modules/tensor/OnnxQuant.cpp" + "${EVE_ROOT}/src/modules/tensor/OnnxGpuKernels.cpp" + "${EVE_ROOT}/src/modules/tensor/OnnxShapeOps.cpp" + "${EVE_ROOT}/src/modules/tensor/OnnxIndexOps.cpp" + "${EVE_ROOT}/src/modules/tensor/OnnxNumeric.cpp" + "${EVE_ROOT}/src/modules/tensor/OnnxNeural.cpp" + "${EVE_ROOT}/src/modules/tensor/OnnxMisc.cpp" + "${EVE_ROOT}/src/modules/tensor/OnnxLstm.cpp" + "${EVE_ROOT}/src/modules/tensor/Tensor.cpp" + "${EVE_ROOT}/src/modules/tensor/CpuKernels.cpp" + "${EVE_ROOT}/src/engine/common/Exception.cpp" + "${EVE_ROOT}/src/engine/common/RenderTrace.cpp") +target_include_directories(eve_tensor_onnx PUBLIC "${EVE_ROOT}/src/modules" "${EVE_ROOT}/src/engine" "${ZEROERR_SOURCE}/include") +if(MSVC) + target_compile_options(eve_tensor_onnx PUBLIC /utf-8 /EHsc) +endif() +add_executable(onnx_probe main.cpp) +target_link_libraries(onnx_probe PRIVATE eve_tensor_onnx) +option(EVE_ONNX_TESTS "Build focused tensor ONNX contract tests" ON) +if(EVE_ONNX_TESTS) + set(BUILD_TEST OFF CACHE BOOL "" FORCE) + set(BUILD_EXAMPLES OFF CACHE BOOL "" FORCE) + add_subdirectory("${ZEROERR_SOURCE}" zeroerr) + target_link_libraries(eve_tensor_onnx PUBLIC zeroerr) + add_executable(onnx_contracts "${EVE_ROOT}/test/tensor_onnx.cpp") + target_link_libraries(onnx_contracts PRIVATE eve_tensor_onnx zeroerr) + enable_testing() + foreach(case roundingAndRange exactAccumulators convolutionValidation) + add_test(NAME "tensor.onnx.affine.${case}" COMMAND onnx_contracts "--testcase=tensor.onnx.affine.${case}") + endforeach() + add_test(NAME tensor.onnx.import.transactionAndDiagnostics COMMAND onnx_contracts "--testcase=tensor.onnx.import.transactionAndDiagnostics") + add_test(NAME tensor.onnx.storage.aliasAndCpuMutation COMMAND onnx_contracts "--testcase=tensor.onnx.storage.aliasAndCpuMutation") + add_test(NAME tensor.onnx.gpu.failureAndScope COMMAND onnx_contracts "--testcase=tensor.onnx.gpu.failureAndScope") + add_test(NAME tensor.onnx.import.owningOutputs COMMAND onnx_contracts "--testcase=tensor.onnx.import.owningOutputs") +endif() + +if(NOT EVE_ONNX_TESTS) + target_compile_definitions(eve_tensor_onnx PRIVATE ZEROERR_NO_ASSERT) + target_compile_definitions(onnx_probe PRIVATE ZEROERR_NO_ASSERT) +endif() + +option(EVE_ONNX_VULKAN_PROBE "Enable standalone hardware parity provider (Vulkan SDK + shaderc)" OFF) +if(EVE_ONNX_VULKAN_PROBE) + find_package(Vulkan REQUIRED) + find_library(ONNX_SHADERC NAMES shaderc_shared HINTS "$ENV{VULKAN_SDK}/Lib" REQUIRED) + target_sources(onnx_probe PRIVATE VulkanProbeCompute.cpp) + target_compile_definitions(onnx_probe PRIVATE EVE_ONNX_VULKAN_PROBE) + target_link_libraries(onnx_probe PRIVATE Vulkan::Vulkan "${ONNX_SHADERC}") + if(WIN32) + add_custom_command(TARGET onnx_probe POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different "$ENV{VULKAN_SDK}/Bin/shaderc_shared.dll" "$") + endif() +endif() diff --git a/examples/tensor/onnx/README.md b/examples/tensor/onnx/README.md new file mode 100644 index 00000000..298c98dd --- /dev/null +++ b/examples/tensor/onnx/README.md @@ -0,0 +1,223 @@ +# Native quantized ONNX inference + +Direct ONNX execution in EVEngine's tensor module, without ONNX Runtime, protobuf +or Python at runtime. Original INT8 weights remain packed. The existing block +quantization format is separate from ONNX affine quantization. + +## Engine GPU path + +After configuring the engine with tensor enabled: + +```powershell +cmake/with-msvc.cmd cmake --build build/win32-debug --target onnx_engine_probe --parallel 4 +$env:EVE_ONNX_GPU='1' +build/win32-debug/test/onnx_engine_probe.exe model.int8.onnx audio audio.f32 0,73,127,172,88,108,169,4,0 style.f32 +``` + +The probe initializes headless Graphics and calls `createOnnxGpuCompute()`. It also +checks device-unavailable behavior before initialization. `style.f32` contains 256 +little-endian floats; providing it also supplies speed=1. The IDs encode “你好。”. + +The current minimal host profile also needs font and devtools to link its closure: +`-DEVENGINE_PROFILE=minimal -DEVENGINE_MODULE_tensor=ON -DEVENGINE_MODULE_font=ON +-DEVENGINE_MODULE_devtools=ON`. No manifest or default profile is changed here. + +C++ callers check Results from `OnnxModel::load`, `createOnnxGpuCompute` and +`model->runGpu(feeds, *compute, requestedNames)`. The result owns `outputs` and +reports successful `dispatches` plus transfer, submission and allocation/reuse counters. `run(feeds, requestedNames)` selects CPU for +reference execution. Empty requestedNames selects all graph outputs. + +GPU kernels cover integer MatMul/Conv and LSTM projections, FP32 MatMul/ConvTranspose, +broadcast arithmetic, activations, Softmax, sum/mean reductions, InstanceNormalization, +Resize and CumSum. Dynamic quantization reduces range on GPU and reads only three +validation scalars; packed activation generation stays on GPU. Shape/index/sequence/control +operations, padding, other quantization variants, RNG and LSTM gates stay on CPU. GPU errors never trigger an automatic CPU retry. +The GLSL adapter explicitly rejects WebGPU. + +The adapter reuses the existing `gpgpu::Sequence` recording and `submitAsync`/`wait` +mechanism used by tensor GPU execution. Consecutive GPU nodes share immutable device +buffers. Identity/reshape aliases do not transfer data. CPU reads materialize host bytes; +CPU writes detach shared storage. Packed initializers and recurrent weights are cached +by owning storage identity across runs for immutable model initializers. Dead device allocations return to a size-bounded +reuse pool. A batch flush occurs at a CPU-read boundary or after a bounded recording +backlog, with no download at a backlog flush. The public runGpu call remains synchronous. + +The static `GpuProgram` compiler is not an ONNX control-flow executor; ONNX uses the same +Gpgpu buffers/sequences directly while retaining dynamic Loop/If scheduling and affine +integer semantics. Device allocations are bounded to 512 MiB, with a 128 MiB host-input +cache and size-compatible buffer reuse. All referenced buffers survive queued work. + +Retain the provider returned by `createOnnxGpuCompute()` outside the dialogue/inference +loop and pass it to every `runGpu` call. It owns a reusable compiled GPU session: +shader pipelines (up to 1024 variants), model initializers (up to 128 MiB) and device +buffer pools (up to 512 MiB) persist across calls. Shader cache keys include the complete +generated source, including shapes, integer interpretation and LSTM direction offsets. +A different shape compiles only missing variants. Dynamic activations remain per-call. +Failed calls discard pending uploads before allowing cache reuse; compiled shaders stay. + +The session subscribes to `Graphics::onResourcesRetiring`: resources are freed before +Vulkan device teardown even if the session outlives Graphics. A retired session rejects +GPU work; construct a new one for the new device. Destroying the session first cancels +the subscription. All of this is device-thread-only. Feeds are borrowed during the call; +outputs outlive the model and provider. Use the device's owning thread without +concurrent provider calls. Independent CPU runs are thread-safe. No global RNG, +background tasks, script callbacks or locks around compute calls are introduced. + +## Standalone build and tests + +```powershell +cmake/with-msvc.cmd cmake -S examples/tensor/onnx -B build/tensor-onnx -G Ninja -DCMAKE_BUILD_TYPE=Release +cmake/with-msvc.cmd cmake --build build/tensor-onnx --parallel 4 +ctest --test-dir build/tensor-onnx --output-on-failure +build/tensor-onnx/onnx_probe.exe model.int8.onnx +``` + +Initialize `external/zeroerr`, or set `ZEROERR_SOURCE` to an initialized checkout. +`EVE_ONNX_TESTS=OFF` removes test linkage. The CPU path needs no Vulkan, SDL or VM. +Tests and their library deliberately use the same assertion policy. + +Optional `-DEVE_ONNX_VULKAN_PROBE=ON` enables a standalone hardware parity provider +using Vulkan SDK and shaderc. It executes the same tensor-generated kernels on a +physical GPU. Production uses the engine's Gpgpu adapter instead. + +## Reproduce Chinese speech + +Download `model.int8.onnx`, `voices.bin` and `tokens.txt` from the +[Kokoro INT8 v1.1 package](https://huggingface.co/csukuangfj/kokoro-int8-multi-lang-v1_1/tree/main). +Tested model: 114299010 bytes, SHA256 +`bda15858163726a492d02a9a727bc263551b86ac77f90812c4b30ff41d380e26`. + +```powershell +python -m pip install numpy onnx onnxruntime +python scripts/run_kokoro_onnx.py --model model.int8.onnx --voices voices.bin --tokens tokens.txt --probe build/win32-debug/test/onnx_engine_probe.exe --work build/kokoro-gpu --gpu +``` + +The utility prepares explicit phonemes and zf_001 voice data, invokes native inference, +and writes `audio.wav`, `native.log` and `result.json`. It does not transform weights. +Style selection follows [sherpa-onnx](https://github.com/k2-fsa/sherpa-onnx/blob/master/sherpa-onnx/csrc/offline-tts-kokoro-model.cc): +index by phoneme length excluding boundary zeros. This is not yet a text normalizer, +phonemizer or dialogue playback integration. + +## Import and numerical contract + +- IR 3–10, default opset 13–17, Microsoft opset 1; FP32, int8/uint8, int32/int64, + bool and internal sequences. External/sparse tensor data and functions are rejected. +- Bounded parsing checks truncation, sizes, duplicates and lexical captures. Nested + Loop/If bodies are parsed; scheduling includes implicit captured values, including + Kokoro captures appearing later in the serialized main graph. +- Unknown operators are reported and required unknown nodes fail before dispatch. + This bounded executor is not a complete ONNX schema validator. +- Affine APIs reject nonfinite values. The ONNX DynamicQuantizeLinear bridge matches + ORT CPU for NaN: exclude it from range estimation and emit byte code zero. Kokoro's + exported STFT phase graph produces 0/0 at zero-amplitude bins. Infinity remains an + error in quantization. Rounding is ties-to-even. +- GPU integer accumulation is exact int32, with a conservative term-count overflow + bound. CPU checks actual int64 sums before narrowing. Per-axis dequantization and + forward/reverse/bidirectional quantized LSTM are supported. +- `OnnxRunOptions.seed` controls per-node RNG streams. Floating CPU/GPU results use + tolerances. `requireFinite` is a strict diagnostic that rejects intentional + intermediate infinity/NaN; keep it false for Kokoro. +- Specialized boundaries: integer MatMul uses rank>=2 A/rank-2 B and scalar zeros; + Conv is 1D/2D NOTSET padding; ConvTranspose is 1D; nearest Resize uses + asymmetric/floor coordinates; linear Resize interpolates the final axis; Loop + supports carried tensors/sequences without scan outputs. Other variants fail. +- Limits: 512 MiB model, 100000 recursive nodes, depth 16, rank 6, 128M elements per + tensor, 10000 loop iterations, 1M execution steps. Live owned intermediates and + copied outputs are bounded to 512 MiB; model/feed data and temporary workspaces + are additional. GPU work is bounded to 65535 groups of 64. + +## Validation and performance + +```powershell +python scripts/test_tensor_onnx.py --probe build/tensor-onnx/onnx_probe.exe --work build/onnx-cpu-fixtures --kokoro model.int8.onnx +python scripts/test_tensor_onnx.py --probe build/tensor-onnx/onnx_probe.exe --work build/onnx-gpu-fixtures --kokoro model.int8.onnx --gpu +``` + +Reference packages are test-only. Coverage includes exact integer results, floating +kernels, shape operations, lexical If captures, Loop/Sequence, neural resampling, +and all six original packed Kokoro LSTMs. Contract tests include malformed inputs, +owning outputs, unsupported preflight, injected GPU failure and scope cleanup. + +The original model generates 26400 finite 24 kHz samples (1.1 s of audio). +The GPU implementation remains synchronous at `runGpu`; CPU boundaries include LSTM +gates, data-dependent shape/index work and quantization validation scalars. + +The hardware residency contract verifies that a 16-node Add chain with aliases uses +only two input uploads, one download and one submission, on the first run. The second run performs zero uploads and zero shader compilations. +Separate hardware tests cover staging-pool growth/reordering and descriptor retirement +when unsubmitted sequences are canceled. Run them with: + +```powershell +cmake/with-msvc.cmd cmake --build build/win32-debug --target onnx_gpu_contracts --parallel 4 +$env:EVENGINE_VULKAN_VALIDATION='1' +$cases = 'tensor.onnx.gpu.residentChain', 'gpgpu.sequence.stagingSizeReuse', 'gpgpu.sequence.cancelRecordingRetiresDescriptors', 'tensor.onnx.gpu.sessionShapesAndLifetime', 'tensor.onnx.gpu.parallelCompilation' +foreach ($case in $cases) { + build/win32-debug/test/onnx_gpu_contracts.exe "--testcase=$case" + if ($LASTEXITCODE -ne 0) { throw "Failed: $case" } +} +``` + + +The session now compiles different GLSL sources on a bounded CPU queue and defers +pipeline creation/dispatch recording until a host-read or segment-budget boundary. +Dispatches retain their buffers and preserve graph order. Identical pending sources +share one future; completed pipelines persist across calls. All Vulkan access stays +on the device thread. Error cleanup drains admitted CPU jobs and discards unsubmitted +commands; compiler workers never capture a device or call the render tracer. + +`createOnnxGpuCompute(compilerWorkers)` accepts 1..8 workers (default 4), with at most +48 waiting compilation jobs. The probe exposes this as `EVE_ONNX_COMPILER_WORKERS`. +This is an upper bound, not a promise of a speedup: dynamic CPU boundaries limit how +many independent sources are available. Shaderc retains a compiler/options context +per thread until that thread exits. Session workers are joined at destruction. +Thread-safety follows the [shaderc C API contract](https://github.com/google/shaderc/blob/main/libshaderc/include/shaderc/shaderc.h). + +Windows CMake now defines `EVE_HAS_SHADERC` on EVGpgpu itself. Previously its +INTERFACE-only definition linked the library but left the implementation launching +`glslc.exe`. A shaderc-enabled build now reports compilation failures directly and +never requires that executable. The build without the shaderc library uses an external compiler path that retains unique +temporary names through output consumption and explicitly restricts child handle +inheritance so concurrent jobs cannot lock each other's temporary files. + +Measured on the same native Windows Debug build and original model/input, with a +fresh process/session (application shader cache empty; driver caches were not cleared): + +- Previous synchronous compilation: first `runGpu` 35.72-36.99 s; repeat 2.60-2.64 s. +- Four external compiler workers: first 19.89 s; repeat 2.69 s. +- Four in-process workers with retained contexts: first 8.23-9.88 s; repeat 2.74-3.26 s. +- One in-process worker with retained context: first 6.81-9.82 s; repeat 2.69-3.22 s. + +Each new session compiled 370 variants; every repeat compiled zero (2608 cache hits). +Audio was byte-identical across the old implementation, external/in-process compilers, +and one/four-worker runs. Thus compiler wiring/context reuse supplies the main benefit; +on this Debug workload, four workers did not consistently beat one. The final paired +run measured 9.82 s cold / 3.22 s warm with one worker, and 9.88 s / 3.26 s with four. +Timing varied between runs, so the ranges above are retained rather than selecting only the best result. Release performance is not +established. Even the warm call exceeds the 1.1 s audio duration, so this is not real-time. + +Set `EVE_ONNX_PROFILE=1` for wall-time buckets. `compile_task_ms` sums durations of +worker tasks (overlapping durations, not elapsed time); `compile_wait_ms` is device-thread +blocking on futures; `pipeline_ms` is Vulkan pipeline creation on that thread. +`shader_ms` comprises the latter two. `compiler_peak_workers` reports observed concurrency. +Allocation, upload/dispatch recording, submit/wait and host readback are also reported; +these are not GPU timestamp-query kernel timings. An earlier four-worker cold measurement reported +7.75 s aggregate compile tasks, 2.07 s compile waits and 2.86 s pipeline creation. +The one-worker control measured 3.07 s aggregate tasks, 1.22 s waits and 2.65 s pipelines. + +Reproduce both configurations with the same model/session in each process: + +```powershell +$env:EVE_ONNX_PROFILE='1' +$env:EVE_ONNX_COMPILER_WORKERS='4' # use 1 for the single-worker control +python scripts/run_kokoro_onnx.py --model model.int8.onnx --voices voices.bin --tokens tokens.txt --probe build/win32-debug/test/onnx_engine_probe.exe --work build/kokoro-repeat --gpu --repeat 2 +``` + +`result.json` includes per-iteration latency, compilation/cache-hit counts, peak workers +and exact repeat verification. Hardware contracts cover serial/parallel dependent chains +across submission boundaries, changed shapes/constants, cold cancellation/retry, +compilation failure/recovery, optional-device absence and both device/session destruction orders. + +Architecture rules applied: checked Results, ownership and device-thread contracts, +transactional publication, optional-device present/absent tests, failure injection, +deterministic RNG and foreign-format bounds. No ECS, Link, module boundary or engine +persistence changes are introduced. diff --git a/examples/tensor/onnx/VulkanProbeCompute.cpp b/examples/tensor/onnx/VulkanProbeCompute.cpp new file mode 100644 index 00000000..adc01b13 --- /dev/null +++ b/examples/tensor/onnx/VulkanProbeCompute.cpp @@ -0,0 +1,249 @@ +#include "VulkanProbeCompute.h" +#include +#include +#include +#include +#include +#include + +namespace { +void check(VkResult r) { + if (r != VK_SUCCESS) throw std::runtime_error("Vulkan failure: " + std::to_string(r)); +} +class VulkanCompute final : public eve::tensor::OnnxCompute { + size_t dispatches = 0; + VkInstance instance = VK_NULL_HANDLE; + VkPhysicalDevice physical = VK_NULL_HANDLE; + VkDevice device = VK_NULL_HANDLE; + VkQueue queue = VK_NULL_HANDLE; + VkCommandPool commands = VK_NULL_HANDLE; + VkPhysicalDeviceMemoryProperties memory{}; + struct Resources { + VkDevice device; + VkCommandPool pool; + std::vector buffers; + std::vector memories; + VkShaderModule shader = VK_NULL_HANDLE; + VkDescriptorSetLayout setLayout = VK_NULL_HANDLE; + VkPipelineLayout layout = VK_NULL_HANDLE; + VkPipeline pipeline = VK_NULL_HANDLE; + VkDescriptorPool descriptors = VK_NULL_HANDLE; + VkCommandBuffer command = VK_NULL_HANDLE; + VkFence fence = VK_NULL_HANDLE; + ~Resources() { + if (fence) vkDestroyFence(device, fence, nullptr); + if (command) vkFreeCommandBuffers(device, pool, 1, &command); + if (descriptors) vkDestroyDescriptorPool(device, descriptors, nullptr); + if (pipeline) vkDestroyPipeline(device, pipeline, nullptr); + if (layout) vkDestroyPipelineLayout(device, layout, nullptr); + if (setLayout) vkDestroyDescriptorSetLayout(device, setLayout, nullptr); + if (shader) vkDestroyShaderModule(device, shader, nullptr); + for (auto b : buffers) vkDestroyBuffer(device, b, nullptr); + for (auto m : memories) vkFreeMemory(device, m, nullptr); + } + }; + +public: + ~VulkanCompute() override { + if (commands) vkDestroyCommandPool(device, commands, nullptr); + if (device) vkDestroyDevice(device, nullptr); + if (instance) vkDestroyInstance(instance, nullptr); + } + void initialize() { + VkApplicationInfo app{VK_STRUCTURE_TYPE_APPLICATION_INFO}; + app.pApplicationName = "EVEngine ONNX GPU parity"; + app.apiVersion = VK_API_VERSION_1_0; + VkInstanceCreateInfo ci{VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO}; + ci.pApplicationInfo = &app; + check(vkCreateInstance(&ci, nullptr, &instance)); + uint32_t count = 0; + check(vkEnumeratePhysicalDevices(instance, &count, nullptr)); + std::vector devices(count); + check(vkEnumeratePhysicalDevices(instance, &count, devices.data())); + uint32_t family = UINT32_MAX; + for (auto d : devices) { + VkPhysicalDeviceProperties p; + vkGetPhysicalDeviceProperties(d, &p); + if (p.deviceType != VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU && + p.deviceType != VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU) + continue; + uint32_t n = 0; + vkGetPhysicalDeviceQueueFamilyProperties(d, &n, nullptr); + std::vector qs(n); + vkGetPhysicalDeviceQueueFamilyProperties(d, &n, qs.data()); + for (uint32_t i = 0; i < n; ++i) + if (qs[i].queueFlags & VK_QUEUE_COMPUTE_BIT) { + physical = d; + family = i; + break; + } + if (physical) { + std::cout << "ONNX_GPU_DEVICE " << p.deviceName << '\n'; + break; + } + } + if (!physical) throw std::runtime_error("No hardware Vulkan compute device"); + vkGetPhysicalDeviceMemoryProperties(physical, &memory); + float priority = 1; + VkDeviceQueueCreateInfo q{VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO}; + q.queueFamilyIndex = family; + q.queueCount = 1; + q.pQueuePriorities = &priority; + VkDeviceCreateInfo dc{VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO}; + dc.queueCreateInfoCount = 1; + dc.pQueueCreateInfos = &q; + check(vkCreateDevice(physical, &dc, nullptr, &device)); + vkGetDeviceQueue(device, family, 0, &queue); + VkCommandPoolCreateInfo pc{VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO}; + pc.queueFamilyIndex = family; + check(vkCreateCommandPool(device, &pc, nullptr, &commands)); + } + eve::Result> dispatch(const eve::tensor::OnnxKernel& k) override { + try { + Resources r{device, commands}; + if (k.inputs.size() >= 8 || !k.outputBytes || !k.workItems || uint64_t(k.workItems) + 63 > 65535u * 64u) + throw std::runtime_error("Invalid probe kernel extent"); + const uint32_t n = static_cast(k.inputs.size() + 1); + std::vector bindings(n); + std::vector infos(n); + for (uint32_t i = 0; i < n; ++i) { + const size_t bytes = i + 1 == n ? k.outputBytes : k.inputs[i].size(); + VkBufferCreateInfo bc{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO}; + bc.size = std::max(size_t(4), (bytes + 3) & ~size_t(3)); + bc.usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT; + VkBuffer b; + check(vkCreateBuffer(device, &bc, nullptr, &b)); + r.buffers.push_back(b); + VkMemoryRequirements req; + vkGetBufferMemoryRequirements(device, b, &req); + uint32_t type = UINT32_MAX; + for (uint32_t j = 0; j < memory.memoryTypeCount; ++j) + if ((req.memoryTypeBits & (1u << j)) && + (memory.memoryTypes[j].propertyFlags & + (VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT)) == + (VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT)) { + type = j; + break; + } + if (type == UINT32_MAX) throw std::runtime_error("No coherent host-visible Vulkan memory"); + VkMemoryAllocateInfo ma{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO}; + ma.allocationSize = req.size; + ma.memoryTypeIndex = type; + VkDeviceMemory m; + check(vkAllocateMemory(device, &ma, nullptr, &m)); + r.memories.push_back(m); + check(vkBindBufferMemory(device, b, m, 0)); + void* mapped; + check(vkMapMemory(device, m, 0, bc.size, 0, &mapped)); + std::memset(mapped, 0, bc.size); + if (i + 1 < n && bytes) std::memcpy(mapped, k.inputs[i].data(), bytes); + vkUnmapMemory(device, m); + bindings[i] = {i, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1, VK_SHADER_STAGE_COMPUTE_BIT, nullptr}; + infos[i] = {b, 0, bc.size}; + } + auto compiler = shaderc_compiler_initialize(); + if (!compiler) throw std::runtime_error("shaderc initialization failed"); + auto compiled = shaderc_compile_into_spv(compiler, k.source.data(), k.source.size(), shaderc_compute_shader, + "onnx.comp", "main", nullptr); + shaderc_compiler_release(compiler); + if (!compiled) throw std::runtime_error("shaderc compilation failed"); + if (shaderc_result_get_compilation_status(compiled) != shaderc_compilation_status_success) { + std::string message = shaderc_result_get_error_message(compiled); + shaderc_result_release(compiled); + throw std::runtime_error(message); + } + std::vector spv(shaderc_result_get_length(compiled) / 4); + std::memcpy(spv.data(), shaderc_result_get_bytes(compiled), spv.size() * 4); + shaderc_result_release(compiled); + VkShaderModuleCreateInfo sm{VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO}; + sm.codeSize = spv.size() * 4; + sm.pCode = spv.data(); + check(vkCreateShaderModule(device, &sm, nullptr, &r.shader)); + VkDescriptorSetLayoutCreateInfo sl{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO}; + sl.bindingCount = n; + sl.pBindings = bindings.data(); + check(vkCreateDescriptorSetLayout(device, &sl, nullptr, &r.setLayout)); + VkPipelineLayoutCreateInfo pl{VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO}; + pl.setLayoutCount = 1; + pl.pSetLayouts = &r.setLayout; + check(vkCreatePipelineLayout(device, &pl, nullptr, &r.layout)); + VkComputePipelineCreateInfo cp{VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO}; + cp.layout = r.layout; + cp.stage = {VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO}; + cp.stage.stage = VK_SHADER_STAGE_COMPUTE_BIT; + cp.stage.module = r.shader; + cp.stage.pName = "main"; + check(vkCreateComputePipelines(device, VK_NULL_HANDLE, 1, &cp, nullptr, &r.pipeline)); + VkDescriptorPoolSize ps{VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, n}; + VkDescriptorPoolCreateInfo dp{VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO}; + dp.maxSets = 1; + dp.poolSizeCount = 1; + dp.pPoolSizes = &ps; + check(vkCreateDescriptorPool(device, &dp, nullptr, &r.descriptors)); + VkDescriptorSetAllocateInfo da{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO}; + da.descriptorPool = r.descriptors; + da.descriptorSetCount = 1; + da.pSetLayouts = &r.setLayout; + VkDescriptorSet set; + check(vkAllocateDescriptorSets(device, &da, &set)); + std::vector writes(n); + for (uint32_t i = 0; i < n; ++i) { + writes[i] = {VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET}; + writes[i].dstSet = set; + writes[i].dstBinding = i; + writes[i].descriptorCount = 1; + writes[i].descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER; + writes[i].pBufferInfo = &infos[i]; + } + vkUpdateDescriptorSets(device, n, writes.data(), 0, nullptr); + VkCommandBufferAllocateInfo ca{VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO}; + ca.commandPool = commands; + ca.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY; + ca.commandBufferCount = 1; + check(vkAllocateCommandBuffers(device, &ca, &r.command)); + VkCommandBufferBeginInfo begin{VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO}; + check(vkBeginCommandBuffer(r.command, &begin)); + VkMemoryBarrier barrier{VK_STRUCTURE_TYPE_MEMORY_BARRIER}; + barrier.srcAccessMask = VK_ACCESS_HOST_WRITE_BIT; + barrier.dstAccessMask = VK_ACCESS_SHADER_READ_BIT; + vkCmdPipelineBarrier(r.command, VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0, 1, + &barrier, 0, nullptr, 0, nullptr); + vkCmdBindPipeline(r.command, VK_PIPELINE_BIND_POINT_COMPUTE, r.pipeline); + vkCmdBindDescriptorSets(r.command, VK_PIPELINE_BIND_POINT_COMPUTE, r.layout, 0, 1, &set, 0, nullptr); + vkCmdDispatch(r.command, (k.workItems + 63) / 64, 1, 1); + barrier.srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT; + barrier.dstAccessMask = VK_ACCESS_HOST_READ_BIT; + vkCmdPipelineBarrier(r.command, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_HOST_BIT, 0, 1, + &barrier, 0, nullptr, 0, nullptr); + check(vkEndCommandBuffer(r.command)); + VkFenceCreateInfo fc{VK_STRUCTURE_TYPE_FENCE_CREATE_INFO}; + check(vkCreateFence(device, &fc, nullptr, &r.fence)); + VkSubmitInfo submit{VK_STRUCTURE_TYPE_SUBMIT_INFO}; + submit.commandBufferCount = 1; + submit.pCommandBuffers = &r.command; + check(vkQueueSubmit(queue, 1, &submit, r.fence)); + check(vkWaitForFences(device, 1, &r.fence, VK_TRUE, UINT64_MAX)); + std::vector result(k.outputBytes); + void* mapped; + check(vkMapMemory(device, r.memories.back(), 0, k.outputBytes, 0, &mapped)); + std::memcpy(result.data(), mapped, k.outputBytes); + vkUnmapMemory(device, r.memories.back()); + if (++dispatches % 100 == 0) std::cout << "GPU_DISPATCHES " << dispatches << std::endl; + return eve::Result>::success(std::move(result)); + } catch (const std::exception& e) { + return eve::Result>::failure( + eve::Diagnostic::error(eve::DiagnosticCode::Failed, e.what())); + } + } +}; +} // namespace +eve::Result> createVulkanProbeCompute() { + try { + auto p = std::make_unique(); + p->initialize(); + return eve::Result>::success(std::move(p)); + } catch (const std::exception& e) { + return eve::Result>::failure( + eve::Diagnostic::error(eve::DiagnosticCode::Failed, e.what())); + } +} diff --git a/examples/tensor/onnx/VulkanProbeCompute.h b/examples/tensor/onnx/VulkanProbeCompute.h new file mode 100644 index 00000000..a6474040 --- /dev/null +++ b/examples/tensor/onnx/VulkanProbeCompute.h @@ -0,0 +1,4 @@ +#pragma once +#include "tensor/OnnxCompute.h" +/** @brief Standalone hardware test provider; production uses createOnnxGpuCompute(). */ +[[nodiscard]] eve::Result> createVulkanProbeCompute(); diff --git a/examples/tensor/onnx/main.cpp b/examples/tensor/onnx/main.cpp new file mode 100644 index 00000000..261d4dc4 --- /dev/null +++ b/examples/tensor/onnx/main.cpp @@ -0,0 +1,156 @@ +#include +#include +#include "tensor/OnnxModel.h" +#ifdef EVE_ONNX_ENGINE_PROVIDER +#include "common/CrashHandler.h" +#include "graphics/Graphics.h" +#include "tensor/OnnxCompute.h" +#elif defined(EVE_ONNX_VULKAN_PROBE) +#include "VulkanProbeCompute.h" +#endif + +#include +#include +#include +#include + +int main(int argc, char** argv) { +#ifdef EVE_ONNX_ENGINE_PROVIDER + eve::installCrashHandler(); +#endif + std::cout.setf(std::ios::unitbuf); + if (argc < 2) { + std::cerr << "Usage: onnx_probe MODEL [OUTPUT_NAME RAW_OUTPUT_FILE [COMMA_SEPARATED_TOKEN_IDS]]\n"; + return 2; + } + std::ifstream file(argv[1], std::ios::binary | std::ios::ate); + if (!file || file.tellg() <= 0 || file.tellg() > 512 * 1024 * 1024) return 2; + std::vector bytes(static_cast(file.tellg())); + file.seekg(0); + if (!file.read(reinterpret_cast(bytes.data()), bytes.size())) return 2; + const auto importStart = std::chrono::steady_clock::now(); + auto imported = eve::tensor::OnnxModel::load(bytes); + std::cout << "ONNX_IMPORT_TIME ms=" + << std::chrono::duration(std::chrono::steady_clock::now() - importStart).count() + << '\n'; + if (!imported.ok()) { + std::cerr << imported.error()->message() << '\n'; + return 1; + } + auto model = std::move(imported.value()); + const auto info = model->info(); + std::cout << "ONNX_IMPORT nodes=" << info.nodeCount << " packed_bytes=" << info.initializerBytes + << " unsupported_nodes=" << info.unsupportedNodes.size() << '\n'; + if (argc < 4) { + for (const auto& node : info.unsupportedNodes) std::cout << node << '\n'; + return 0; + } + std::vector feeds; + if (argc > 4) { + std::vector ids; + std::istringstream text(argv[4]); + std::string item; + try { + while (std::getline(text, item, ',')) ids.push_back(std::stoll(item)); + } catch (const std::exception&) { + return 2; + } + eve::tensor::OnnxTensor tokens{eve::tensor::OnnxElement::Int64, + {1, static_cast(ids.size())}, + std::vector(ids.size() * 8)}; + if (!ids.empty()) std::memcpy(tokens.bytes.data(), ids.data(), tokens.bytes.size()); + feeds.push_back({"tokens", std::move(tokens)}); + } + if (argc > 5) { + eve::tensor::OnnxTensor style{eve::tensor::OnnxElement::Float32, {1, 256}, std::vector(1024)}; + std::ifstream voice(argv[5], std::ios::binary | std::ios::ate); + if (!voice || voice.tellg() != 1024) { + std::cerr << "Style file must contain 256 float32 values\n"; + return 2; + } + voice.seekg(0); + if (!voice.read(reinterpret_cast(style.bytes.data()), 1024)) return 2; + feeds.push_back({"style", std::move(style)}); + eve::tensor::OnnxTensor speed{eve::tensor::OnnxElement::Float32, {1}, std::vector(4)}; + const float rate = 1; + std::memcpy(speed.bytes.data(), &rate, 4); + feeds.push_back({"speed", std::move(speed)}); + } + const std::vector outputs{argv[2]}; + size_t dispatches = 0; + const bool gpu = std::getenv("EVE_ONNX_GPU") && std::string(std::getenv("EVE_ONNX_GPU")) == "1"; + eve::tensor::OnnxRunOptions options; + options.requireFinite = std::getenv("EVE_ONNX_CHECK_FINITE") != nullptr; + auto execute = [&]() -> eve::Result> { + if (!gpu) return model->run(feeds, outputs, options); +#if defined(EVE_ONNX_VULKAN_PROBE) || defined(EVE_ONNX_ENGINE_PROVIDER) +#ifdef EVE_ONNX_ENGINE_PROVIDER + auto unavailable = eve::tensor::createOnnxGpuCompute(); + if (unavailable.ok()) + return eve::Result>::failure(eve::Diagnostic::error( + eve::DiagnosticCode::Failed, "Unexpected device before headless initialization")); + std::cout << "ONNX_GPU_ABSENT_PASS\n"; + auto* graphics = eve::graphics::Graphics::create(); + graphics->initHeadless(64, 64); + uint32_t compilerWorkers = 4; + if (const auto* value = std::getenv("EVE_ONNX_COMPILER_WORKERS")) + compilerWorkers = static_cast(std::stoul(value)); + auto provider = eve::tensor::createOnnxGpuCompute(compilerWorkers); +#else + auto provider = createVulkanProbeCompute(); +#endif + if (!provider.ok()) return eve::Result>::failure(provider.status()); + auto runOnce = [&](int iteration) { + const auto runStart = std::chrono::steady_clock::now(); + auto result = model->runGpu(feeds, *provider.value(), outputs, options); + std::cout << "ONNX_ITERATION index=" << iteration << " ms=" + << std::chrono::duration(std::chrono::steady_clock::now() - runStart).count(); + if (result.ok()) + std::cout << " compilations=" << result.value().transfers.shaderCompilations + << " compiler_peak_workers=" << result.value().transfers.compilerPeakWorkers + << " cache_hits=" << result.value().transfers.shaderCacheHits + << " upload_bytes=" << result.value().transfers.uploadedBytes; + std::cout << '\n'; + return result; + }; + auto result = runOnce(0); + if (const auto* repeat = std::getenv("EVE_ONNX_REPEAT"); repeat && std::string(repeat) == "2") { + if (!result.ok()) return eve::Result>::failure(result.status()); + auto previous = result.value().outputs; + result = runOnce(1); + if (result.ok()) { + if (previous.size() != result.value().outputs.size()) + return eve::Result>::failure( + eve::Diagnostic::error(eve::DiagnosticCode::Failed, "Repeat output count mismatch")); + for (size_t i = 0; i < previous.size(); ++i) + if (previous[i].tensor.bytes != result.value().outputs[i].tensor.bytes) + return eve::Result>::failure( + eve::Diagnostic::error(eve::DiagnosticCode::Failed, "Repeat output bytes differ")); + std::cout << "ONNX_REPEAT_IDENTICAL_PASS\n"; + } + } + if (!result.ok()) return eve::Result>::failure(result.status()); + dispatches = result.value().dispatches; + const auto& t = result.value().transfers; + std::cout << "ONNX_TRANSFERS uploads=" << t.uploads << " downloads=" << t.downloads + << " submissions=" << t.submissions << " upload_bytes=" << t.uploadedBytes + << " download_bytes=" << t.downloadedBytes << " allocations=" << t.bufferAllocations + << " reuses=" << t.bufferReuses << " compilations=" << t.shaderCompilations + << " cache_hits=" << t.shaderCacheHits << '\n'; + return eve::Result>::success(std::move(result.value().outputs)); +#else + return eve::Result>::failure(eve::Diagnostic::error( + eve::DiagnosticCode::Unsupported, "Build EVE_ONNX_VULKAN_PROBE to enable GPU parity")); +#endif + }; + auto generated = execute(); + if (!generated.ok()) { + std::cerr << generated.error()->path() << ": " << generated.error()->message() << '\n'; + return 1; + } + const auto& output = generated.value().front().tensor; + std::ofstream target(argv[3], std::ios::binary); + if (!target.write(reinterpret_cast(output.bytes.data()), output.bytes.size())) return 2; + std::cout << "ONNX_RUN_PASS backend=" << (gpu ? "vulkan_quantized" : "engine_cpu") << " dispatches=" << dispatches + << " bytes=" << output.bytes.size() << '\n'; +} diff --git a/scripts/check_architecture_contracts.py b/scripts/check_architecture_contracts.py index 6b00438e..0a4a5d35 100644 --- a/scripts/check_architecture_contracts.py +++ b/scripts/check_architecture_contracts.py @@ -348,7 +348,7 @@ POINTER_API = re.compile( r"^\s*(?:(?:\[\[nodiscard(?:\([^]]*\))?\]\]\s*)?" r"(?:static|virtual|inline|constexpr|explicit|const)\s+)*" r"(?:[A-Za-z_]\w*(?:::[A-Za-z_]\w*)*(?:\s*<[^;{}()]+>)?\s*)?\*\s*" - r"(?:[A-Za-z_]\w*)?\s*(?:\(|;|=)" + r"[A-Za-z_]\w*\s*(?:\(|;|=)" ) # A bool query is a legitimate API shape when its name follows the project's # query vocabulary. Keep this list deliberately lexical: this checker does diff --git a/scripts/run_kokoro_onnx.py b/scripts/run_kokoro_onnx.py new file mode 100644 index 00000000..b5058c10 --- /dev/null +++ b/scripts/run_kokoro_onnx.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +"""Exercise native Kokoro INT8 inference with explicit phonemes and a selected voice. + +Python/ONNX/NumPy prepare test inputs and WAV output only. Inference runs in the +native probe, without ONNX Runtime. Default phonemes say Chinese 'ni hao'. +""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import re +import subprocess +import time +import wave + +import numpy as np +import onnx + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--model', type=Path, required=True) + parser.add_argument('--voices', type=Path, required=True) + parser.add_argument('--tokens', type=Path, required=True) + parser.add_argument('--probe', type=Path, required=True) + parser.add_argument('--work', type=Path, required=True) + parser.add_argument('--gpu', action='store_true') + parser.add_argument('--repeat', type=int, choices=[1, 2], default=1) + parser.add_argument('--speaker', type=int, default=3) + parser.add_argument('--phonemes', default='ㄋ ㄧ 2 ㄏ ㄠ 3 .') + args = parser.parse_args() + args.work.mkdir(parents=True, exist_ok=True) + model = onnx.load(args.model, load_external_data=False) + metadata = {entry.key: entry.value for entry in model.metadata_props} + dims = [int(d) for d in metadata['style_dim'].split(',')] + speakers = int(metadata['n_speakers']) + if dims[1:] != [1, 256] or not 0 <= args.speaker < speakers: + raise ValueError('Unsupported voice shape or speaker index') + vocab = {line.rsplit(' ', 1)[0]: int(line.rsplit(' ', 1)[1]) + for line in args.tokens.read_text(encoding='utf-8').splitlines()} + phones = args.phonemes.split() + if not 0 < len(phones) < dims[0]: + raise ValueError('Phoneme count exceeds the voice table') + ids = [0] + [vocab[p] for p in phones] + [0] + voices = np.memmap(args.voices, mode='r', dtype='0, (name,result.stdout) + results.append(dict(gpu_dispatches=dispatches, backend='gpu' if args.gpu else 'cpu', case=name, output=output, max_abs_error=error, tolerance=tolerance, reference=reference)) + print(name, output, 'PASS', error) + + for i, x in enumerate([[-1., 0, 1], [0., 0], [2., 3.], [-3., -2.], [-0.01, 0.03], [np.nan,1.,-1.], [np.nan,np.nan]]): + check('dynamic' + str(i), [h.make_node('DynamicQuantizeLinear', ['x'], ['q','s','z'])], + {'x': np.array(x, np.float32)}, ['q','s','z'], reference='ort') + for sign in [False, True]: + dtype = np.int8 if sign else np.uint8 + check('ties' + str(sign), [h.make_node('QuantizeLinear', ['x','s','z'], ['q'])], + {'x':np.array([-1000,-1.25,-0.75,-0.25,0.25,0.75,1.25,1000],np.float32), + 's':np.array(0.5,np.float32),'z':np.array(0 if sign else 128,dtype)}, ['q']) + check('per_axis', [h.make_node('DequantizeLinear',['q','s','z'],['y'],axis=1)], + {'q':np.arange(24,dtype=np.uint8).reshape(2,3,4),'s':np.array([.1,.2,.3],np.float32),'z':np.array([2,5,8],np.uint8)}, ['y']) + check('integer_exact', [h.make_node('MatMulInteger',['a','b'],['y'])], + {'a':np.full((1,1025),255,np.uint8),'b':np.full((1025,1),127,np.int8)}, ['y']) + check('signed_zero', [h.make_node('MatMulInteger',['a','b','az','bz'],['y'])], + {'a':np.array([[-128,127,0]],np.int8),'b':np.array([[3,4],[5,6],[7,8]],np.uint8),'az':np.array(-3,np.int8),'bz':np.array(4,np.uint8)}, ['y']) + check('conv_group', [h.make_node('ConvInteger',['x','w','xz','wz'],['y'],group=2,pads=[1,0,0,1],dilations=[1,2])], + {'x':np.arange(2*4*4,dtype=np.uint8).reshape(1,2,4,4),'w':np.array(range(8),np.uint8).reshape(2,1,2,2),'xz':np.array(7,np.uint8),'wz':np.array(2,np.uint8)}, ['y']) + check('conv1d', [h.make_node('ConvInteger',['x','w'],['y'],pads=[1,2],strides=[2])], + {'x':np.arange(9,dtype=np.uint8).reshape(1,1,9),'w':np.array([1,2,3],np.uint8).reshape(1,1,3)}, ['y']) + check('integer64_shape', [h.make_node('Gather',['x','i'],['y'])], + {'x':np.array([2**60+1,-2**60-3],np.int64),'i':np.array([1,0],np.int64)}, ['y']) + check('reshape_transpose', [h.make_node('Reshape',['x','s'],['r']), h.make_node('Transpose',['r'],['y'],perm=[1,0])], + {'x':np.arange(12,dtype=np.float32),'s':np.array([3,-1],np.int64)}, ['y']) + check('slice_reverse',[h.make_node('Slice',['x','s','e','a','st'],['y'])], + {'x':np.arange(24,dtype=np.int64).reshape(2,3,4),'s':np.array([-1,-1],np.int64),'e':np.array([-100,-100],np.int64),'a':np.array([0,2],np.int64),'st':np.array([-1,-2],np.int64)},['y']) + check('broadcast_empty',[h.make_node('Expand',['x','s'],['y'])],{'x':np.ones((1,3),np.float32),'s':np.array([0,3],np.int64)},['y']) + check('concat',[h.make_node('Concat',['x','b'],['y'],axis=-1)],{'x':np.arange(6,dtype=np.int64).reshape(2,3),'b':np.array([[20],[30]],np.int64)},['y']) + check('where',[h.make_node('Where',['c','a','b'],['y'])],{'c':np.array([[True],[False]]),'a':np.array([2**60+1,2**60+2],np.int64),'b':np.array(0,np.int64)},['y']) + check('range_reverse',[h.make_node('Range',['s','e','d'],['y'])],{'s':np.array(2**60+9,np.int64),'e':np.array(2**60,np.int64),'d':np.array(-2,np.int64)},['y'],reference='reference') + check('topk_ties',[h.make_node('TopK',['x','k'],['y','indices'],axis=0)],{'x':np.array([[2,4],[2,3],[1,5]],np.float32),'k':np.array([2],np.int64)},['y','indices']) + check('scatter_nd',[h.make_node('ScatterND',['x','i','u'],['y'])],{'x':np.zeros((3,2),np.float32),'i':np.array([[-1],[0]],np.int64),'u':np.array([[4,5],[6,7]],np.float32)},['y']) + check('reduce_product',[h.make_node('ReduceProd',['x'],['y'],axes=[0],keepdims=0)],{'x':np.array([[2,3],[4,5]],np.int64)},['y']) + check('broadcast_float',[h.make_node('Add',['a','b'],['y'])],{'a':np.arange(24,dtype=np.float32).reshape(2,3,4),'b':np.array([[.1],[.2],[.3]],np.float32)},['y']) + for name,a,b in [('batched',np.arange(24,dtype=np.float32).reshape(2,3,4)/20,np.arange(20,dtype=np.float32).reshape(4,5)/30),('vector',np.array([1.,2.,3.],np.float32),np.arange(6,dtype=np.float32).reshape(3,2)),('dot',np.array([1.,2.],np.float32),np.array([3.,4.],np.float32))]: + check('matmul_'+name,[h.make_node('MatMul',['a','b'],['y'])],{'a':a,'b':b},['y']) + check('pow_negative',[h.make_node('Pow',['x','p'],['y'])],{'x':np.array([-3.,-2.,-.1,0,2],np.float32),'p':np.array(2,np.float32)},['y']) + check('reduce_mean',[h.make_node('ReduceMean',['x'],['y'],axes=[0,2],keepdims=0)],{'x':np.arange(24,dtype=np.float32).reshape(2,3,4)/7},['y']) + check('softmax_axis',[h.make_node('Softmax',['x'],['y'],axis=1)],{'x':np.arange(24,dtype=np.float32).reshape(2,3,4)/7},['y']) + for op in ['Reciprocal','Floor','Round','Atan','LeakyRelu','Sqrt','Exp','Log','Sin','Cos','Tanh','Sigmoid']: + check('unary_'+op,[h.make_node(op,['x'],['y'])],{'x':np.array([.25,.5,1.,1.5,2.5,3.5],np.float32)},['y'],tolerance=2e-6) + check('atan_infinity',[h.make_node('Atan',['x'],['y'])],{'x':np.array([-np.inf,np.inf],np.float32)},['y'],tolerance=2e-6) + check('instance_norm',[h.make_node('InstanceNormalization',['x','s','b'],['y'],epsilon=1e-5)],{'x':np.arange(24,dtype=np.float32).reshape(2,3,4)/11,'s':np.array([1,2,3],np.float32),'b':np.array([.1,.2,.3],np.float32)},['y'],tolerance=3e-5) + check('transpose_group',[h.make_node('ConvTranspose',['x','w','b'],['y'],group=2,strides=[2],pads=[1,1],output_padding=[1])],{'x':np.arange(12,dtype=np.float32).reshape(1,4,3)/13,'w':np.arange(24,dtype=np.float32).reshape(4,2,3)/7,'b':np.arange(4,dtype=np.float32)},['y'],tolerance=3e-6) + for mode in ['nearest','linear']: + check('resize_'+mode,[h.make_node('Resize',['x','','s'],['y'],mode=mode,coordinate_transformation_mode='asymmetric' if mode=='nearest' else 'half_pixel',nearest_mode='floor')],{'x':np.arange(6,dtype=np.float32).reshape(1,2,3),'s':np.array([1,1,2.5],np.float32)},['y'],tolerance=3e-6) + for mode in ['reflect','edge','constant']: + check('pad_'+mode,[h.make_node('Pad',['x','p'],['y'],mode=mode)],{'x':np.arange(6,dtype=np.float32).reshape(2,3),'p':np.array([0,2,0,1],np.int64)},['y']) + for exclusive in [0,1]: + check('cumsum_'+str(exclusive),[h.make_node('CumSum',['x','a'],['y'],exclusive=exclusive,reverse=1)],{'x':np.arange(24,dtype=np.float32).reshape(2,3,4),'a':np.array(1,np.int64)},['y']) + check('clip',[h.make_node('Clip',['x','lo','hi'],['y'])],{'x':np.arange(6,dtype=np.float32),'lo':np.array(1.,np.float32),'hi':np.array(4.,np.float32)},['y']) + branch=h.make_graph([h.make_node('Identity',['captured'],['branch_result'])],'branch',[],[h.make_tensor_value_info('branch_result',onnx.TensorProto.FLOAT,[2])]) + check('if_lexical_capture',[h.make_node('If',['condition'],['y'],then_branch=branch,else_branch=branch),h.make_node('Identity',['x'],['captured'])],{'condition':np.array(True),'x':np.array([2,3],np.float32)},['y']) + body=h.make_graph([h.make_node('Identity',['condition_in'],['condition_out']),h.make_node('SequenceInsert',['state_in','iteration'],['state_out'])],'body', + [h.make_tensor_value_info('iteration',onnx.TensorProto.INT64,[]),h.make_tensor_value_info('condition_in',onnx.TensorProto.BOOL,[]),h.make_tensor_sequence_value_info('state_in',onnx.TensorProto.INT64,[])], + [h.make_tensor_value_info('condition_out',onnx.TensorProto.BOOL,[]),h.make_tensor_sequence_value_info('state_out',onnx.TensorProto.INT64,[])]) + check('loop_sequence',[h.make_node('SequenceEmpty',[],['empty'],dtype=onnx.TensorProto.INT64),h.make_node('Loop',['trip','condition','empty'],['sequence'],body=body),h.make_node('ConcatFromSequence',['sequence'],['y'],axis=0,new_axis=1)],{'trip':np.array(3,np.int64),'condition':np.array(True)},['y']) + check('split_sequence',[h.make_node('SplitToSequence',['x','split'],['seq'],axis=1),h.make_node('SequenceAt',['seq','index'],['y'])],{'x':np.arange(12,dtype=np.int64).reshape(2,6),'split':np.array([2,1,3],np.int64),'index':np.array(-1,np.int64)},['y']) + rng=np.random.default_rng(3) + for direction in ['forward','reverse','bidirectional']: + d=2 if direction=='bidirectional' else 1 + values={'x':rng.normal(size=(4,2,3)).astype(np.float32), + 'w':rng.integers(-60,60,(d,3,8),dtype=np.int8),'r':rng.integers(-60,60,(d,2,8),dtype=np.int8), + 'b':rng.normal(0,.1,(d,16)).astype(np.float32), + 'ws':np.full((d,8),.01,np.float32),'wz':np.zeros((d,8),np.int8), + 'rs':np.full((d,8),.01,np.float32),'rz':np.zeros((d,8),np.int8)} + check('lstm_'+direction,[h.make_node('DynamicQuantizeLSTM',['x','w','r','b','','','','','ws','wz','rs','rz'],['y','yh','yc'],domain='com.microsoft',hidden_size=2,direction=direction)],values,['y','yh','yc'],reference='ort',tolerance=.004) + values['lengths']=np.array([4,2],np.int32) + values['h']=rng.normal(0,.1,(d,2,2)).astype(np.float32) + values['c']=rng.normal(0,.1,(d,2,2)).astype(np.float32) + check('lstm_state_'+direction,[h.make_node('DynamicQuantizeLSTM',['x','w','r','b','lengths','h','c','','ws','wz','rs','rz'],['y','yh','yc'],domain='com.microsoft',hidden_size=2,direction=direction)],values,['y','yh','yc'],reference='ort',tolerance=.004) + if args.kokoro: + model=onnx.load(args.kokoro) + initializers={x.name:x for x in model.graph.initializer} + for index, lstm in enumerate(n for n in model.graph.node if n.op_type=='DynamicQuantizeLSTM'): + # Keep the original packed weights, scales, and node attributes. + constants={k:nh.to_array(initializers[k]) for k in lstm.input if k in initializers} + weight=constants[lstm.input[1]] + constants[lstm.input[0]]=rng.normal(0,.1,(3,1,weight.shape[1])).astype(np.float32) + for i in [4,5,6,7]: + if i(d);", "n *= count(shape);"): + with self.subTest(text=text): + lines = [contracts.SourceLine("fixture.h", 1, text)] + self.assertEqual([], contracts.lint_api_shapes(lines)) + + def test_named_pointer_apis_still_require_lifetime_contracts(self): + for text in ("Widget* get();", "Widget* value = makeWidget();"): + with self.subTest(text=text): + lines = [contracts.SourceLine("fixture.h", 1, text)] + codes = {finding.code for finding in contracts.lint_api_shapes(lines)} + self.assertIn("undocumented-raw-pointer-api", codes) + def test_new_link_and_system_need_catalogue_coverage(self): metadata = { "entries": [ diff --git a/src/modules/CMakeLists.txt b/src/modules/CMakeLists.txt index 7dc6008e..6473b2f3 100644 --- a/src/modules/CMakeLists.txt +++ b/src/modules/CMakeLists.txt @@ -488,7 +488,9 @@ if(_has_gpgpu) # INTERFACE: EVGpgpu is an OBJECT library; its private link deps would not # reach the final executable, so propagate the SDK archive to consumers. target_link_libraries(EVGpgpu INTERFACE ${EVE_SHADERC_LIB}) - target_compile_definitions(EVGpgpu INTERFACE EVE_HAS_SHADERC=1) + # The implementation itself selects shaderc; INTERFACE would only define + # the macro for consumers and silently leave VulkanUtil spawning glslc. + target_compile_definitions(EVGpgpu PRIVATE EVE_HAS_SHADERC=1) message(STATUS "Gpgpu: in-process shaderc compiler: ${EVE_SHADERC_LIB}") else() message(STATUS "Gpgpu: shaderc lib not found; will spawn glslc at runtime") diff --git a/src/modules/gpgpu/ComputeProgram.h b/src/modules/gpgpu/ComputeProgram.h new file mode 100644 index 00000000..4506a3a4 --- /dev/null +++ b/src/modules/gpgpu/ComputeProgram.h @@ -0,0 +1,23 @@ +#pragma once +#include +#include +#include +#include +#include "common/Result.h" + +namespace eve::gpgpu { +class ComputeShader; +/** @brief Compile GLSL compute source to owning SPIR-V without accessing Graphics or a GPU. + * @param source Borrowed until return; independent calls may execute concurrently on worker threads. + * @return Owning words, compilation diagnostic, or Unsupported on WebGPU. No callbacks; each thread retains its own + * compiler context until thread exit. + */ +[[nodiscard]] Result> compileComputeSpirv(const std::string& source); +/** @brief Create an owning compute pipeline from valid SPIR-V produced by compileComputeSpirv. + * @param words Borrowed only until return; must contain a valid compute entry point named main. + * @return Owning shader or a diagnostic; Unsupported on WebGPU. + * @note Device thread only, no concurrent calls/reentrancy or Graphics retirement during the call. + * The caller must destroy the shader before the active Graphics device retires. + */ +[[nodiscard]] Result> createComputeShader(const std::vector& words); +} // namespace eve::gpgpu diff --git a/src/modules/gpgpu/Gpgpu.cpp b/src/modules/gpgpu/Gpgpu.cpp index 43a22a3f..c66da1aa 100644 --- a/src/modules/gpgpu/Gpgpu.cpp +++ b/src/modules/gpgpu/Gpgpu.cpp @@ -1,4 +1,5 @@ #include "gpgpu/Gpgpu.h" +#include "gpgpu/ComputeProgram.h" #include "gpgpu/ComputeShader.h" #include "gpgpu/EcsScriptPack.h" #include "gpgpu/GpuBuffer.h" @@ -204,6 +205,36 @@ bool Gpgpu::isAvailable() const { #endif } +Result> compileComputeSpirv(const std::string &source) { +#ifdef EVENGINE_WEBGPU + return Result>::failure( + Diagnostic::error(DiagnosticCode::Unsupported, "GLSL compilation requires Vulkan")); +#else + try { + return Result>::success(compileComputeGlsl(source)); + } catch (const std::exception &e) { + return Result>::failure(Diagnostic::error(DiagnosticCode::Failed, e.what())); + } +#endif +} + +Result> createComputeShader(const std::vector &words) { +#ifdef EVENGINE_WEBGPU + return Result>::failure( + Diagnostic::error(DiagnosticCode::Unsupported, "SPIR-V pipelines require Vulkan")); +#else + try { + if (words.size() < 5 || words.front() != 0x07230203) + return Result>::failure( + Diagnostic::error(DiagnosticCode::InvalidArgument, "Invalid SPIR-V header")); + return Result>::success( + std::unique_ptr(vulkanNewShaderFromSpirv(words))); + } catch (const std::exception &e) { + return Result>::failure(Diagnostic::error(DiagnosticCode::Failed, e.what())); + } +#endif +} + ComputeShader *Gpgpu::newShader(const std::string &source) { #ifdef EVENGINE_WEBGPU if (currentGraphicsBackend() != "webgpu") @@ -213,7 +244,11 @@ ComputeShader *Gpgpu::newShader(const std::string &source) { #else if (currentGraphicsBackend() != "vulkan") throw Exception("Gpgpu.newShader: requires vulkan Graphics backend"); - return vulkanNewShaderFromSpirv(compileComputeGlsl(source)); + auto compiled = compileComputeSpirv(source); + if (!compiled.ok()) throw Exception("%s", compiled.error()->message().c_str()); + auto shader = createComputeShader(compiled.value()); + if (!shader.ok()) throw Exception("%s", shader.error()->message().c_str()); + return shader.value().release(); #endif } diff --git a/src/modules/gpgpu/Gpgpu.h b/src/modules/gpgpu/Gpgpu.h index 9d0c1c09..aff34660 100644 --- a/src/modules/gpgpu/Gpgpu.h +++ b/src/modules/gpgpu/Gpgpu.h @@ -26,7 +26,11 @@ public: /** @brief True when the active Graphics backend can run compute (device initialized). */ bool isAvailable() const; - /** @brief Compile compute source for the active backend (Vulkan: GLSL; WebGPU: WGSL). */ + /** @brief Compatibility-only raw-owning shader factory (Vulkan: GLSL; WebGPU: WGSL). + * Vulkan delegates to the + * checked compileComputeSpirv/createComputeShader APIs. + * Device thread only; caller must delete the shader + * before Graphics retires. */ ComputeShader *newShader(const std::string &source); /** @brief Load precompiled compute bytecode from Filesystem path (Vulkan: SPIR-V). */ diff --git a/src/modules/gpgpu/vulkan/VulkanGpgpu.cpp b/src/modules/gpgpu/vulkan/VulkanGpgpu.cpp index 556b447a..36af1d5e 100644 --- a/src/modules/gpgpu/vulkan/VulkanGpgpu.cpp +++ b/src/modules/gpgpu/vulkan/VulkanGpgpu.cpp @@ -14,7 +14,7 @@ ComputeShader *vulkanNewShaderFromSpirv(const std::vector &spv) { auto *vkg = requireVulkanGraphics(); auto &device = vkg->getDevice(); - auto *shader = new VulkanComputeShader(); + auto shader = std::make_unique(); shader->device_ = &device; shader->module_ = vkb::PipelineBuilder::createShaderModule(device.instance, spv); @@ -56,11 +56,10 @@ ComputeShader *vulkanNewShaderFromSpirv(const std::vector &spv) { cpInfo.layout = shader->pipelineLayout_; auto result = device->createComputePipeline(vk::PipelineCache{}, cpInfo, device.allocation_callbacks); if (result.result != vk::Result::eSuccess) { - delete shader; throw Exception("Gpgpu.newShader: createComputePipeline failed"); } shader->pipeline_ = result.value; - return shader; + return shader.release(); } GpuBuffer *vulkanNewBuffer(int byteSize, const std::string &usage) { diff --git a/src/modules/gpgpu/vulkan/VulkanSequence.cpp b/src/modules/gpgpu/vulkan/VulkanSequence.cpp index 2f4e66d6..4531591b 100644 --- a/src/modules/gpgpu/vulkan/VulkanSequence.cpp +++ b/src/modules/gpgpu/vulkan/VulkanSequence.cpp @@ -75,6 +75,16 @@ void VulkanSequence::destroy() { (*dev)->freeCommandBuffers(pool, submittedCommandBuffer); submittedCommandBuffer = vk::CommandBuffer{}; } + // Abandoning an unsubmitted recording must retire its descriptor leases too. + // The caller keeps shaders alive until the Sequence has been destroyed. + for (ComputeShader *shader : usedShaders) { + auto *vs = dynamic_cast(shader); + if (vs) { + vs->endSequence(); + vs->releasePendingDescriptors(*dev); + } + } + usedShaders.clear(); if (fenceReady) { (*dev)->destroyFence(fence, dev->allocation_callbacks); fence = nullptr; @@ -117,15 +127,16 @@ void vulkanSequenceRecordUpload(VulkanSequence *seq, GpuBuffer *dst, vkb::GenericBuffer *staging = nullptr; for (size_t i = seq->stagingUsed; i < seq->stagingPool.size(); ++i) { if (seq->stagingPool[i].capacity >= vk::DeviceSize(nbytes)) { - staging = &seq->stagingPool[i]; std::swap(seq->stagingPool[seq->stagingUsed], seq->stagingPool[i]); + staging = &seq->stagingPool[seq->stagingUsed]; break; } } if (!staging) { seq->stagingPool.emplace_back(device, buf::eTransferSrc, vk::DeviceSize(nbytes), pfb::eHostVisible | pfb::eHostCoherent); - staging = &seq->stagingPool.back(); + std::swap(seq->stagingPool[seq->stagingUsed], seq->stagingPool.back()); + staging = &seq->stagingPool[seq->stagingUsed]; } ++seq->stagingUsed; staging->updateLocal(vkb::FrameSlot::gpuIdle(), src, vk::DeviceSize(nbytes)); diff --git a/src/modules/gpgpu/vulkan/VulkanUtil.cpp b/src/modules/gpgpu/vulkan/VulkanUtil.cpp index b4ce76f6..c1702ee1 100644 --- a/src/modules/gpgpu/vulkan/VulkanUtil.cpp +++ b/src/modules/gpgpu/vulkan/VulkanUtil.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #if defined(_WIN32) @@ -24,12 +25,26 @@ namespace eve::gpgpu { namespace { +// Compiler failures must not call Exception's global render-tracer callback: +// this path also executes on CPU workers without a Graphics lifetime. +std::runtime_error compileError(const char *message) { + return std::runtime_error(message); +} + +template +std::runtime_error compileError(const char *format, Args... args) { + const int count = std::snprintf(nullptr, 0, format, args...); + if (count < 0) return std::runtime_error("Compute compilation failed"); + std::string text(static_cast(count) + 1, '\0'); + std::snprintf(text.data(), text.size(), format, args...); + text.pop_back(); + return std::runtime_error(text); +} + std::vector loadSpirvBytes(const void *data, size_t size) { - if (!data || size < 4 || (size % 4) != 0) - throw Exception("Gpgpu SPIR-V: invalid size %zu", size); + if (!data || size < 4 || (size % 4) != 0) throw compileError("Gpgpu SPIR-V: invalid size %zu", size); const auto *words = static_cast(data); - if (words[0] != 0x07230203) - throw Exception("Gpgpu SPIR-V: bad magic (expected 0x07230203)"); + if (words[0] != 0x07230203) throw compileError("Gpgpu SPIR-V: bad magic (expected 0x07230203)"); return std::vector(words, words + size / 4); } @@ -85,40 +100,54 @@ namespace { #if defined(EVE_HAS_SHADERC) /** In-process GLSL -> SPIR-V via the Vulkan SDK's static shaderc library. */ std::vector compileComputeGlslInProcess(const std::string &glsl) { - shaderc_compiler_t compiler = shaderc_compiler_initialize(); - if (!compiler) throw Exception("Gpgpu.newShader: shaderc_compiler_initialize failed"); - shaderc_compile_options_t options = shaderc_compile_options_initialize(); - if (!options) { - shaderc_compiler_release(compiler); - throw Exception("Gpgpu.newShader: shaderc_compile_options_initialize failed"); - } - shaderc_compile_options_set_target_env(options, shaderc_target_env_vulkan, - shaderc_env_version_vulkan_1_0); - shaderc_compile_options_set_target_spirv(options, shaderc_spirv_version_1_0); + struct CompilerContext { + shaderc_compiler_t compiler = shaderc_compiler_initialize(); + shaderc_compile_options_t options = nullptr; + CompilerContext() { + if (!compiler) throw compileError("Gpgpu.newShader: shaderc_compiler_initialize failed"); + options = shaderc_compile_options_initialize(); + if (!options) { + shaderc_compiler_release(compiler); + throw compileError("Gpgpu.newShader: shaderc_compile_options_initialize failed"); + } + shaderc_compile_options_set_target_env(options, shaderc_target_env_vulkan, shaderc_env_version_vulkan_1_0); + shaderc_compile_options_set_target_spirv(options, shaderc_spirv_version_1_0); + } + ~CompilerContext() { + shaderc_compile_options_release(options); + shaderc_compiler_release(compiler); + } + }; + // Keep glslang's compiler/built-in state alive across jobs and graph segments. + // Each CPU thread owns its context; no device references survive here. + thread_local CompilerContext context; + auto compiler = context.compiler; + auto options = context.options; shaderc_compilation_result_t result = shaderc_compile_into_spv(compiler, glsl.data(), glsl.size(), shaderc_glsl_compute_shader, "eve_compute.comp", "main", options); - shaderc_compile_options_release(options); - shaderc_compiler_release(compiler); + if (!result) throw compileError("Gpgpu.newShader: shaderc returned no compilation result"); + struct CompilationResult { + shaderc_compilation_result_t value; + ~CompilationResult() { shaderc_result_release(value); } + } ownedResult{result}; if (shaderc_result_get_compilation_status(result) != shaderc_compilation_status_success) { const std::string err = shaderc_result_get_error_message(result); - shaderc_result_release(result); - throw Exception("Gpgpu.newShader: shaderc failed:\n%s", err.c_str()); + throw compileError("Gpgpu.newShader: shaderc failed:\n%s", err.c_str()); } const size_t len = shaderc_result_get_length(result); const char *bytes = shaderc_result_get_bytes(result); if (len == 0 || len % 4 != 0) { - shaderc_result_release(result); - throw Exception("Gpgpu.newShader: shaderc returned invalid SPIR-V"); + throw compileError("Gpgpu.newShader: shaderc returned invalid SPIR-V"); } std::vector spv(len / 4); std::memcpy(spv.data(), bytes, len); - shaderc_result_release(result); return spv; } #endif // EVE_HAS_SHADERC +#if !defined(EVE_HAS_SHADERC) /** Locate a usable glslc.exe: VULKAN_SDK, common install roots, then PATH. */ std::string findGlslc() { if (const char *sdk = std::getenv("VULKAN_SDK"); sdk && *sdk) { @@ -157,37 +186,45 @@ std::string findGlslc() { return {}; } +#endif // !EVE_HAS_SHADERC } // namespace #endif std::vector compileComputeGlsl(const std::string &glsl) { - if (glsl.empty()) throw Exception("Gpgpu.newShader: empty GLSL"); + if (glsl.empty()) throw compileError("Gpgpu.newShader: empty GLSL"); #if defined(_WIN32) #if defined(EVE_HAS_SHADERC) - try { - return compileComputeGlslInProcess(glsl); - } catch (...) { - // Fall through to spawning glslc (SDK lib present but compile failed). - } -#endif + // Compile errors remain errors; a packaged shaderc build never needs an SDK executable. + return compileComputeGlslInProcess(glsl); +#else const std::string glslc = findGlslc(); if (glslc.empty()) - throw Exception("Gpgpu.newShader: glslc not found on Windows " - "(install the Vulkan SDK or set VULKAN_SDK)"); + throw compileError( + "Gpgpu.newShader: glslc not found on Windows " + "(install the Vulkan SDK or set VULKAN_SDK)"); char tmpDir[MAX_PATH]; - if (GetTempPathA(MAX_PATH, tmpDir) == 0) - throw Exception("Gpgpu.newShader: GetTempPath failed"); + if (GetTempPathA(MAX_PATH, tmpDir) == 0) throw compileError("Gpgpu.newShader: GetTempPath failed"); char inPath[MAX_PATH]; - if (GetTempFileNameA(tmpDir, "eve", 0, inPath) == 0) - throw Exception("Gpgpu.newShader: GetTempFileName failed"); + if (GetTempFileNameA(tmpDir, "eve", 0, inPath) == 0) throw compileError("Gpgpu.newShader: GetTempFileName failed"); std::string outPath = std::string(inPath) + ".spv"; + const std::string errPath = outPath + ".err"; + struct TemporaryFiles { + const char *input; + const std::string &output; + const std::string &error; + ~TemporaryFiles() { + DeleteFileA(error.c_str()); + DeleteFileA(output.c_str()); + // Reserve the unique basename until both sidecars are consumed. + DeleteFileA(input); + } + } temporary{inPath, outPath, errPath}; { FILE *f = nullptr; if (fopen_s(&f, inPath, "wb") != 0 || !f) { - DeleteFileA(inPath); - throw Exception("Gpgpu.newShader: failed to write temp GLSL"); + throw compileError("Gpgpu.newShader: failed to write temp GLSL"); } fwrite(glsl.data(), 1, glsl.size(), f); fclose(f); @@ -196,64 +233,79 @@ std::vector compileComputeGlsl(const std::string &glsl) { // Spawn glslc directly with CreateProcess: _popen routes through cmd.exe // /c whose quote-stripping rules mangle commands starting with a quoted // program path. Capture stderr/stdout into a sidecar file. - const std::string errPath = outPath + ".err"; std::string cmd = "\"" + glslc + "\" -fshader-stage=comp \"" + inPath + "\" -o \"" + outPath + "\""; - STARTUPINFOA si{}; - si.cb = sizeof(si); + STARTUPINFOEXA si{}; + si.StartupInfo.cb = sizeof(si); PROCESS_INFORMATION pi{}; SECURITY_ATTRIBUTES sa{}; sa.nLength = sizeof(sa); sa.bInheritHandle = TRUE; - HANDLE errFile = CreateFileA(errPath.c_str(), GENERIC_WRITE, FILE_SHARE_READ, &sa, - CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, nullptr); - if (errFile != INVALID_HANDLE_VALUE) { - si.dwFlags = STARTF_USESTDHANDLES; - si.hStdInput = GetStdHandle(STD_INPUT_HANDLE); - si.hStdOutput = errFile; - si.hStdError = errFile; + HANDLE errFile = CreateFileA(errPath.c_str(), GENERIC_WRITE, FILE_SHARE_READ | FILE_SHARE_DELETE, &sa, + CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, nullptr); + HANDLE input = CreateFileA("NUL", GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, &sa, OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL, nullptr); + if (errFile == INVALID_HANDLE_VALUE || input == INVALID_HANDLE_VALUE) { + if (errFile != INVALID_HANDLE_VALUE) CloseHandle(errFile); + if (input != INVALID_HANDLE_VALUE) CloseHandle(input); + throw compileError("Gpgpu.newShader: failed to create compiler standard handles"); } - char *mutableCmd = _strdup(cmd.c_str()); + struct StandardHandles { + HANDLE output, input; + ~StandardHandles() { + CloseHandle(output); + CloseHandle(input); + } + } handles{errFile, input}; + // Without an explicit list, concurrent CreateProcess calls inherit other + // jobs' temporary-file handles and can prevent glslc from opening its input. + SIZE_T attributeBytes = 0; + InitializeProcThreadAttributeList(nullptr, 1, 0, &attributeBytes); + std::vector attributes(attributeBytes); + si.lpAttributeList = reinterpret_cast(attributes.data()); + if (!InitializeProcThreadAttributeList(si.lpAttributeList, 1, 0, &attributeBytes)) + throw compileError("Gpgpu.newShader: failed to initialize process attributes"); + struct AttributeList { + LPPROC_THREAD_ATTRIBUTE_LIST value; + ~AttributeList() { DeleteProcThreadAttributeList(value); } + } attributeList{si.lpAttributeList}; + HANDLE inherited[] = {errFile, input}; + if (!UpdateProcThreadAttribute(si.lpAttributeList, 0, PROC_THREAD_ATTRIBUTE_HANDLE_LIST, inherited, + sizeof(inherited), nullptr, nullptr)) + throw compileError("Gpgpu.newShader: failed to restrict compiler handle inheritance"); + si.StartupInfo.dwFlags = STARTF_USESTDHANDLES; + si.StartupInfo.hStdInput = input; + si.StartupInfo.hStdOutput = errFile; + si.StartupInfo.hStdError = errFile; const BOOL spawned = - CreateProcessA(glslc.c_str(), mutableCmd, nullptr, nullptr, TRUE, CREATE_NO_WINDOW, - nullptr, nullptr, &si, &pi); - free(mutableCmd); - if (errFile != INVALID_HANDLE_VALUE) CloseHandle(errFile); + CreateProcessA(glslc.c_str(), cmd.data(), nullptr, nullptr, TRUE, + CREATE_NO_WINDOW | EXTENDED_STARTUPINFO_PRESENT, nullptr, nullptr, &si.StartupInfo, &pi); if (!spawned) { const DWORD errCode = GetLastError(); - DeleteFileA(inPath); - DeleteFileA(outPath.c_str()); - DeleteFileA(errPath.c_str()); - throw Exception("Gpgpu.newShader: failed to launch glslc (error %lu)", errCode); + throw compileError("Gpgpu.newShader: failed to launch glslc (error %lu)", errCode); } WaitForSingleObject(pi.hProcess, INFINITE); DWORD exitCode = 0; GetExitCodeProcess(pi.hProcess, &exitCode); CloseHandle(pi.hProcess); CloseHandle(pi.hThread); - DeleteFileA(inPath); if (exitCode != 0) { std::string err; - FILE *ef = nullptr; - if (fopen_s(&ef, errPath.c_str(), "rb") == 0 && ef) { - char buf[512]; - size_t n; - while ((n = fread(buf, 1, sizeof(buf) - 1, ef)) > 0) { - buf[n] = '\0'; - err += buf; - } - fclose(ef); + HANDLE diagnostics = + CreateFileA(errPath.c_str(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, nullptr, + OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr); + if (diagnostics != INVALID_HANDLE_VALUE) { + char buffer[512]; + DWORD count = 0; + while (ReadFile(diagnostics, buffer, sizeof(buffer), &count, nullptr) && count) err.append(buffer, count); + CloseHandle(diagnostics); } - DeleteFileA(outPath.c_str()); - DeleteFileA(errPath.c_str()); - throw Exception("Gpgpu.newShader: glslc failed (exit %lu):\n%s", exitCode, err.c_str()); + throw compileError("Gpgpu.newShader: glslc failed (exit %lu):\n%s", exitCode, err.c_str()); } - DeleteFileA(errPath.c_str()); FILE *f = nullptr; if (fopen_s(&f, outPath.c_str(), "rb") != 0 || !f) { - DeleteFileA(outPath.c_str()); - throw Exception("Gpgpu.newShader: failed to open compiled SPIR-V"); + throw compileError("Gpgpu.newShader: failed to open compiled SPIR-V"); } fseek(f, 0, SEEK_END); long sz = ftell(f); @@ -261,23 +313,22 @@ std::vector compileComputeGlsl(const std::string &glsl) { std::vector bytes(static_cast(sz > 0 ? sz : 0)); if (sz > 0 && fread(bytes.data(), 1, static_cast(sz), f) != static_cast(sz)) { fclose(f); - DeleteFileA(outPath.c_str()); - throw Exception("Gpgpu.newShader: failed to read compiled SPIR-V"); + throw compileError("Gpgpu.newShader: failed to read compiled SPIR-V"); } fclose(f); - DeleteFileA(outPath.c_str()); return loadSpirvBytes(bytes.data(), bytes.size()); +#endif // EVE_HAS_SHADERC #else char inPath[] = "/tmp/eve_comp_XXXXXX"; int fd = mkstemp(inPath); - if (fd < 0) throw Exception("Gpgpu.newShader: mkstemp failed"); + if (fd < 0) throw compileError("Gpgpu.newShader: mkstemp failed"); std::string outPath = std::string(inPath) + ".spv"; { ssize_t n = write(fd, glsl.data(), glsl.size()); close(fd); if (n < 0 || size_t(n) != glsl.size()) { unlink(inPath); - throw Exception("Gpgpu.newShader: failed to write temp GLSL"); + throw compileError("Gpgpu.newShader: failed to write temp GLSL"); } } @@ -292,17 +343,17 @@ std::vector compileComputeGlsl(const std::string &glsl) { unlink(inPath); if (status != 0) { unlink(outPath.c_str()); - throw Exception("Gpgpu.newShader: glslc failed:\n%s", err.c_str()); + throw compileError("Gpgpu.newShader: glslc failed:\n%s", err.c_str()); } } else { unlink(inPath); - throw Exception("Gpgpu.newShader: glslc not available (popen failed)"); + throw compileError("Gpgpu.newShader: glslc not available (popen failed)"); } FILE *f = fopen(outPath.c_str(), "rb"); if (!f) { unlink(outPath.c_str()); - throw Exception("Gpgpu.newShader: failed to open compiled SPIR-V"); + throw compileError("Gpgpu.newShader: failed to open compiled SPIR-V"); } fseek(f, 0, SEEK_END); long sz = ftell(f); @@ -311,7 +362,7 @@ std::vector compileComputeGlsl(const std::string &glsl) { if (sz > 0 && fread(bytes.data(), 1, static_cast(sz), f) != static_cast(sz)) { fclose(f); unlink(outPath.c_str()); - throw Exception("Gpgpu.newShader: failed to read compiled SPIR-V"); + throw compileError("Gpgpu.newShader: failed to read compiled SPIR-V"); } fclose(f); unlink(outPath.c_str()); diff --git a/src/modules/graphics/Graphics.h b/src/modules/graphics/Graphics.h index 16850cfb..61aabedf 100644 --- a/src/modules/graphics/Graphics.h +++ b/src/modules/graphics/Graphics.h @@ -31,6 +31,10 @@ struct aiMesh; +namespace eve { +class Subscription; +} + namespace eve::graphics { struct PbrSurface; @@ -188,6 +192,13 @@ public: * callbacks. */ [[nodiscard]] std::weak_ptr resourceLifetime() const; + /** @brief Register cleanup before this provider destroys its GPU device/resources. + * @return Owning subscription, or Conflict if retirement has started. + * @lifetime Retain the subscription until cleanup is no longer needed. Either destruction order is safe. + * @thread Render thread only. Callback must not throw, recreate or destroy Graphics, or start GPU work. + * @note Callbacks run without locks; disposing subscriptions during notification is supported. */ + [[nodiscard]] Result onResourcesRetiring(std::function callback) const; + /** * @brief Whether gbuffer-based post-process shaders (AO, GI) can be created on this diff --git a/src/modules/graphics/GraphicsLifetime.cpp b/src/modules/graphics/GraphicsLifetime.cpp index 3ef0e65a..fd4dfcb6 100644 --- a/src/modules/graphics/GraphicsLifetime.cpp +++ b/src/modules/graphics/GraphicsLifetime.cpp @@ -1,13 +1,16 @@ +#include #include +#include "common/Subscription.h" #include "graphics/Graphics.h" - namespace eve::graphics { namespace { -// Render-thread-only observation registry. Erasing on destruction prevents address reuse -// from making an old observer valid for a new Graphics instance. +struct Lifetime { + eve::Observer<> retiring; + bool retired = false; +}; bool registryAlive = false; struct Registry { - std::unordered_map> tokens; + std::unordered_map> tokens; Registry() { registryAlive = true; } ~Registry() { registryAlive = false; } }; @@ -15,13 +18,31 @@ auto& lifetimes() { static Registry registry; return registry.tokens; } +auto lifetime(const Graphics* graphics) { + auto& value = lifetimes()[graphics]; + if (!value) value = std::make_shared(); + return value; +} } // namespace std::weak_ptr Graphics::resourceLifetime() const { - auto& token = lifetimes()[this]; - if (!token) token = std::make_shared(0); - return token; + auto value = lifetime(this); + return value->retired ? std::weak_ptr{} : value; +} +Result Graphics::onResourcesRetiring(std::function callback) const { + auto value = lifetime(this); + if (value->retired) + return Result::failure( + Diagnostic::error(DiagnosticCode::Conflict, "Graphics resources are retiring")); + return Result::success(value->retiring.subscribe(std::move(callback))); } void Graphics::retireResourceLifetime() const { - if (registryAlive) lifetimes().erase(this); + if (!registryAlive) return; + auto it = lifetimes().find(this); + if (it == lifetimes().end() || it->second->retired) return; + auto value = it->second; + value->retired = true; + const auto failures = value->retiring.notifyChecked([]() noexcept {}); + if (failures) std::fprintf(stderr, "Graphics resource retirement: %zu callback failures\n", failures); + lifetimes().erase(this); } } // namespace eve::graphics diff --git a/src/modules/graphics/vulkan/Graphics.cpp b/src/modules/graphics/vulkan/Graphics.cpp index cbd64fa2..1e9916c7 100644 --- a/src/modules/graphics/vulkan/Graphics.cpp +++ b/src/modules/graphics/vulkan/Graphics.cpp @@ -292,6 +292,7 @@ std::string Graphics::getBackendName() const { return "vulkan"; } Graphics::Graphics() = default; Graphics::~Graphics() { + retireResourceLifetime(); detachGraphicsArtifactProvider(this); if (!initialized) { // A process-level boot warmup may exist even when callers decide not diff --git a/src/modules/graphics/webgpu/Graphics.cpp b/src/modules/graphics/webgpu/Graphics.cpp index cad9d060..6691fe23 100644 --- a/src/modules/graphics/webgpu/Graphics.cpp +++ b/src/modules/graphics/webgpu/Graphics.cpp @@ -99,7 +99,10 @@ Graphics::Graphics() { } } -Graphics::~Graphics() { detachGraphicsArtifactProvider(this); } +Graphics::~Graphics() { + retireResourceLifetime(); + detachGraphicsArtifactProvider(this); +} // --------------------------------------------------------------------------- // Init diff --git a/src/modules/tensor/AffineQuant.cpp b/src/modules/tensor/AffineQuant.cpp new file mode 100644 index 00000000..0ae16173 --- /dev/null +++ b/src/modules/tensor/AffineQuant.cpp @@ -0,0 +1,179 @@ +#include "tensor/AffineQuant.h" +#include "tensor/AffineQuantInternal.h" +#include "tensor/OnnxGpuKernels.h" + +#include +#include +#include + +namespace eve::tensor::affine { +namespace { +template +Result fail(const char* message) { + return Result::failure(Diagnostic::error(DiagnosticCode::InvalidArgument, message)); +} +int value(ByteView v, size_t i) { + const int x = v.bytes[i]; + return v.signedValues && x >= 128 ? x - 256 : x; +} +bool validZero(int z, bool sign) { return sign ? z >= -128 && z <= 127 : z >= 0 && z <= 255; } +double evenRound(double x) { + const double lo = std::floor(x), fraction = x - lo; + return lo + (fraction > 0.5 || (fraction == 0.5 && std::fmod(lo, 2.0) != 0)); +} +bool fits(int64_t n) { return n >= INT32_MIN && n <= INT32_MAX; } +bool product(size_t a, size_t b, size_t limit) { return b == 0 || a <= limit / b; } +} // namespace + +Result> quantize(std::span input, float scale, int zero, bool sign) { + if (!(scale > 0) || !std::isfinite(scale) || !validZero(zero, sign)) + return fail>("Invalid affine scale or zero point"); + std::vector out; + out.reserve(input.size()); + for (float x : input) { + if (!std::isfinite(x)) return fail>("Nonfinite quantization input"); + const double rounded = evenRound(static_cast(x) / scale) + zero; + const int q = static_cast(std::clamp(rounded, sign ? -128.0 : 0.0, sign ? 127.0 : 255.0)); + out.push_back(static_cast(q)); + } + return Result>::success(std::move(out)); +} + +Result dynamicQuantize(std::span input) { + float lo = 0, hi = 0; + for (float x : input) { + if (!std::isfinite(x)) return fail("Nonfinite dynamic quantization input"); + lo = std::min(lo, x); + hi = std::max(hi, x); + } + QuantizedActivation result; + result.scale = hi == lo ? 1.f : static_cast((static_cast(hi) - lo) / 255.0); + if (!(result.scale > 0) || !std::isfinite(result.scale)) + return fail("Dynamic quantization scale is not representable"); + result.zeroPoint = static_cast(std::clamp(evenRound(-static_cast(lo) / result.scale), 0.0, 255.0)); + auto bytes = quantize(input, result.scale, result.zeroPoint, false); + if (!bytes.ok()) return Result::failure(bytes.status()); + result.values = std::move(bytes.value()); + return Result::success(std::move(result)); +} + +Result> dequantize(ByteView input, std::span scales, std::span zeros, + size_t inner) { + if (scales.empty() || scales.size() != zeros.size() || inner == 0 || !product(scales.size(), inner, SIZE_MAX) || + input.bytes.size() % (scales.size() * inner)) + return fail>("Invalid affine channel layout"); + for (size_t i = 0; i < scales.size(); ++i) + if (!(scales[i] > 0) || !std::isfinite(scales[i]) || !validZero(zeros[i], input.signedValues)) + return fail>("Invalid affine scale or zero point"); + std::vector out(input.bytes.size()); + for (size_t i = 0; i < out.size(); ++i) { + const size_t channel = (i / inner) % scales.size(); + out[i] = static_cast(value(input, i) - zeros[channel]) * scales[channel]; + } + return Result>::success(std::move(out)); +} + +Result> matmul(ByteView a, ByteView b, size_t m, size_t k, size_t n, int aZero, int bZero, + OnnxCompute* compute) { + if (!validZero(aZero, a.signedValues) || !validZero(bZero, b.signedValues) || !product(m, k, SIZE_MAX) || + !product(k, n, SIZE_MAX) || !product(m, n, INT32_MAX) || a.bytes.size() != m * k || b.bytes.size() != k * n) + return fail>("Invalid integer matmul dimensions or zero points"); + if (compute && m && n && k) { + try { + const int32_t z = bZero; + return Result>::success( + onnx_detail::gpuMatmul(*compute, a, b, m, k, n, aZero, {&z, 1})); + } catch (const std::exception& e) { + return Result>::failure(Diagnostic::error(DiagnosticCode::Failed, e.what())); + } + } + std::vector out(m * n); + for (size_t r = 0; r < m; ++r) + for (size_t c = 0; c < n; ++c) { + int64_t sum = 0; + for (size_t j = 0; j < k; ++j) + sum += static_cast(value(a, r * k + j) - aZero) * (value(b, j * n + c) - bZero); + if (!fits(sum)) return fail>("Integer matmul accumulator overflow"); + out[r * n + c] = static_cast(sum); + } + return Result>::success(std::move(out)); +} + +Result detail::validateConv(size_t xSize, size_t wSize, bool xSigned, bool wSigned, + const ConvShape& s, int xZero, std::span wZeros) { + for (int d : {s.batch, s.channels, s.height, s.width, s.outputs, s.kernelH, s.kernelW, s.strideH, s.strideW, + s.dilationH, s.dilationW, s.groups}) + if (d <= 0 || d > 65536) return fail("Invalid integer convolution dimensions"); + if (s.channels % s.groups || s.outputs % s.groups || s.padTop < 0 || s.padBottom < 0 || s.padLeft < 0 || + s.padRight < 0 || !validZero(xZero, xSigned) || + (wZeros.size() != 1 && wZeros.size() != static_cast(s.outputs))) + return fail("Invalid convolution groups, padding or zero points"); + for (int z : wZeros) + if (!validZero(z, wSigned)) return fail("Invalid convolution weight zero point"); + const int64_t h = static_cast(s.height) + s.padTop + s.padBottom - + static_cast(s.dilationH) * (s.kernelH - 1) - 1; + const int64_t width = static_cast(s.width) + s.padLeft + s.padRight - + static_cast(s.dilationW) * (s.kernelW - 1) - 1; + if (h < 0 || width < 0) return fail("Convolution kernel exceeds padded input"); + const int64_t oh = h / s.strideH + 1, ow = width / s.strideW + 1; + int64_t outCount = 1; + for (int64_t d : {static_cast(s.batch), static_cast(s.outputs), oh, ow}) { + if (d > INT32_MAX / outCount) return fail("Convolution output too large"); + outCount *= d; + } + const int cg = s.channels / s.groups; + size_t inputCount = 1, weightCount = 1; + for (int d : {s.batch, s.channels, s.height, s.width}) { + if (!product(inputCount, d, INT32_MAX)) return fail("Convolution input too large"); + inputCount *= d; + } + for (int d : {s.outputs, cg, s.kernelH, s.kernelW}) { + if (!product(weightCount, d, INT32_MAX)) return fail("Convolution weights too large"); + weightCount *= d; + } + if (outCount > INT32_MAX || xSize != inputCount || wSize != weightCount) + return fail("Convolution buffer size mismatch or output too large"); + return Result::success({oh, ow, outCount}); +} +Result> conv(ByteView x, ByteView w, const ConvShape& s, int xZero, + std::span wZeros, OnnxCompute* compute) { + auto extent = + detail::validateConv(x.bytes.size(), w.bytes.size(), x.signedValues, w.signedValues, s, xZero, wZeros); + if (!extent.ok()) return Result>::failure(extent.status()); + const auto [oh, ow, outCount] = extent.value(); + const int cg = s.channels / s.groups; + if (compute) { + try { + return Result>::success(onnx_detail::gpuConv(*compute, x, w, s, xZero, wZeros)); + } catch (const std::exception& e) { + return Result>::failure(Diagnostic::error(DiagnosticCode::Failed, e.what())); + } + } + std::vector out(static_cast(outCount)); + for (int b = 0; b < s.batch; ++b) + for (int o = 0; o < s.outputs; ++o) + for (int64_t y = 0; y < oh; ++y) + for (int64_t z = 0; z < ow; ++z) { + int64_t sum = 0; + for (int c = 0; c < cg; ++c) + for (int ky = 0; ky < s.kernelH; ++ky) + for (int kx = 0; kx < s.kernelW; ++kx) { + const int64_t iy = y * s.strideH - s.padTop + static_cast(ky) * s.dilationH; + const int64_t ix = z * s.strideW - s.padLeft + static_cast(kx) * s.dilationW; + if (iy < 0 || ix < 0 || iy >= s.height || ix >= s.width) continue; + const size_t xi = + ((static_cast(b) * s.channels + (o / (s.outputs / s.groups)) * cg + c) * + s.height + + iy) * + s.width + + ix; + const size_t wi = ((static_cast(o) * cg + c) * s.kernelH + ky) * s.kernelW + kx; + sum += static_cast(value(x, xi) - xZero) * + (value(w, wi) - wZeros[wZeros.size() == 1 ? 0 : o]); + } + if (!fits(sum)) return fail>("Integer convolution accumulator overflow"); + out[((static_cast(b) * s.outputs + o) * oh + y) * ow + z] = static_cast(sum); + } + return Result>::success(std::move(out)); +} +} // namespace eve::tensor::affine diff --git a/src/modules/tensor/AffineQuant.h b/src/modules/tensor/AffineQuant.h new file mode 100644 index 00000000..3a0737b4 --- /dev/null +++ b/src/modules/tensor/AffineQuant.h @@ -0,0 +1,77 @@ +#pragma once + +#include "common/Result.h" + +#include +#include +#include + +namespace eve::tensor { +class OnnxCompute; +} +namespace eve::tensor::affine { + +/** @brief ONNX unsigned activation quantization; owns bytes and scalar parameters. */ +struct QuantizedActivation { + std::vector values; + float scale = 1; + uint8_t zeroPoint = 0; +}; + +/** @brief Borrowed packed 8-bit values, valid for the duration of a synchronous call. */ +struct ByteView { + std::span bytes; + bool signedValues = false; +}; + +/** + * @brief Quantize finite FP32 activations with ONNX DynamicQuantizeLinear semantics. + * @param input Borrowed flat input; not retained. + * @return Owning uint8 values, scale, zero point; InvalidArgument for nonfinite input. + * @note CPU, thread-safe, no callbacks; nearest-even rounding independent of fenv. + */ +[[nodiscard]] Result dynamicQuantize(std::span input); + +/** + * @brief Affine quantization to int8/uint8 bytes, saturating and rounding ties to even. + * @return Owning packed bytes; rejects invalid scale, zero point or nonfinite input. + * @note CPU, thread-safe; spans are borrowed only during the call. + */ +[[nodiscard]] Result> quantize(std::span input, float scale, int zeroPoint, + bool signedValues); + +/** + * @brief Affine dequantization using scalar or per-axis scale/zero point. + * @param inner Number of contiguous values following the channel axis. + * @return Owning FP32 values; InvalidArgument on inconsistent channel layout. + * @note CPU, thread-safe; all views are borrowed for the call, no callbacks. + */ +[[nodiscard]] Result> dequantize(ByteView input, std::span scales, + std::span zeros, size_t inner = 1); + +/** + * @brief Integer row-major [M,K] x [K,N], subtracting scalar zero points. + * @return Exact owning int32 accumulators; rejects dimension and int32 overflow. + * @param compute Optional borrowed GPU provider; null selects CPU. No retry on GPU failure. + * @note Synchronous. CPU is thread-safe; GPU uses the provider thread. Weights remain packed. + */ +[[nodiscard]] Result> matmul(ByteView a, ByteView b, size_t m, size_t k, size_t n, int aZero = 0, + int bZero = 0, OnnxCompute* compute = nullptr); + +/** @brief Explicit 1D/2D NCHW convolution geometry; missing 1D height is one. */ +struct ConvShape { + int batch = 1, channels = 1, height = 1, width = 1; + int outputs = 1, kernelH = 1, kernelW = 1; + int strideH = 1, strideW = 1, dilationH = 1, dilationW = 1; + int padTop = 0, padLeft = 0, padBottom = 0, padRight = 0, groups = 1; +}; + +/** + * @brief Integer Conv, supporting groups, asymmetric padding and dilation. + * @return Owning NCHW int32 values; rejects bad geometry or accumulator overflow. + * @param compute Optional borrowed GPU provider; null selects CPU. No retry on GPU failure. + * @note Synchronous. CPU is thread-safe; GPU uses the provider thread. Padding represents real zero. + */ +[[nodiscard]] Result> conv(ByteView x, ByteView w, const ConvShape& shape, int xZero, + std::span wZeros, OnnxCompute* compute = nullptr); +} // namespace eve::tensor::affine diff --git a/src/modules/tensor/AffineQuantInternal.h b/src/modules/tensor/AffineQuantInternal.h new file mode 100644 index 00000000..8576e0bb --- /dev/null +++ b/src/modules/tensor/AffineQuantInternal.h @@ -0,0 +1,9 @@ +#pragma once +#include "tensor/AffineQuant.h" +namespace eve::tensor::affine::detail { +struct ConvExtent { + int64_t height, width, count; +}; +Result validateConv(size_t xSize, size_t wSize, bool xSigned, bool wSigned, const ConvShape&, int, + std::span); +} // namespace eve::tensor::affine::detail diff --git a/src/modules/tensor/OnnxByteStorage.h b/src/modules/tensor/OnnxByteStorage.h new file mode 100644 index 00000000..3f43ca12 --- /dev/null +++ b/src/modules/tensor/OnnxByteStorage.h @@ -0,0 +1,74 @@ +#pragma once +#include +#include +#include "tensor/OnnxModel.h" +#include "tensor/OnnxStorage.h" +namespace eve::tensor::onnx_detail { +// Internal copy-on-write host view. Const reads materialize only at CPU boundaries. +// GPU aliases share immutable storage and never force a readback. +class ByteStorage { + mutable OnnxBuffer buffer_; + std::shared_ptr> writable_; + const std::vector& get() const { + if (!buffer_.host) { + if (!buffer_.device) throw std::runtime_error("Missing ONNX storage"); + auto r = buffer_.device->readback(); + if (!r.ok()) throw std::runtime_error(r.error()->message()); + if (r.value().size() != buffer_.size) throw std::runtime_error("GPU byte count mismatch"); + buffer_.host = std::make_shared>(std::move(r.value())); + } + return *buffer_.host; + } + std::vector& write() { + // CPU mutation creates a new authoritative storage identity. + if (writable_ && buffer_.host.use_count() == 2) return *writable_; + auto v = std::make_shared>(get()); + buffer_ = {v, {}, v->size()}; + writable_ = v; + return *v; + } + +public: + ByteStorage() : ByteStorage(std::vector{}) {} + ByteStorage(std::vector v) : buffer_{std::make_shared>(std::move(v)), {}, 0} { + buffer_.size = buffer_.host->size(); + } + ByteStorage(OnnxBuffer b) : buffer_(std::move(b)) {} + void retainAcrossRuns() { buffer_.persistent = true; } + const OnnxBuffer& buffer() const { return buffer_; } + size_t size() const { return buffer_.size; } + bool empty() const { return !size(); } + /** @brief Borrow CPU data until this storage is mutated or destroyed; device-thread only for lazy data. + * @lifetime Borrowed until mutation or destruction; use the provider thread for GPU readback. + * @throws std::runtime_error On readback failure. Internal executor converts this to Result. */ + const uint8_t* data() const { return get().data(); } + /** @brief Borrow mutable CPU data until mutation/destruction; detaches graph aliases before writing. + * @lifetime Borrowed until mutation or destruction; use the provider thread for GPU readback. + * @throws std::runtime_error On readback failure. Internal executor converts this to Result. */ + uint8_t* data() { return write().data(); } + uint8_t operator[](size_t i) const { return get()[i]; } + uint8_t& operator[](size_t i) { return write()[i]; } + auto begin() { return write().begin(); } + auto end() { return write().end(); } + auto begin() const { return get().begin(); } + auto end() const { return get().end(); } + void resize(size_t n) { + auto& v = write(); + v.resize(n); + buffer_.size = n; + } + /** @brief Copy borrowed iterators into owning CPU storage. + * @lifetime Input iterators are borrowed only for this call; all prior data pointers are invalidated. */ + template + void assign(I a, I b) { + *this = ByteStorage(std::vector(a, b)); + } + operator std::span() const { return get(); } + operator std::vector() const { return get(); } +}; +struct RuntimeTensor { + OnnxElement element = OnnxElement::Float32; + std::vector shape; + ByteStorage bytes; +}; +} // namespace eve::tensor::onnx_detail diff --git a/src/modules/tensor/OnnxCompiler.h b/src/modules/tensor/OnnxCompiler.h new file mode 100644 index 00000000..77771c34 --- /dev/null +++ b/src/modules/tensor/OnnxCompiler.h @@ -0,0 +1,99 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include "gpgpu/ComputeProgram.h" + +namespace eve::tensor::onnx_detail { +// Session-owned CPU workers. Jobs own their source and never capture Graphics, +// Run, buffers or callbacks. Joining the workers needs no device access. +struct CompiledProgram { + std::vector words; + double milliseconds = 0; +}; +class CompilerQueue { + using Job = std::packaged_task; + std::mutex mutex; + std::condition_variable ready; + std::deque jobs; + bool stopping = false; + size_t inFlight = 0; + std::vector workers; + std::atomic running{0}, peak{0}; + void stop() noexcept { + { + std::lock_guard lock(mutex); + stopping = true; + } + ready.notify_all(); + for (auto& worker : workers) + if (worker.joinable()) worker.join(); + } + +public: + explicit CompilerQueue(uint32_t count) { + try { + for (uint32_t i = 0; i < count; ++i) + workers.emplace_back([this] { + for (;;) { + Job job; + { + std::unique_lock lock(mutex); + ready.wait(lock, [this] { return stopping || !jobs.empty(); }); + if (jobs.empty()) return; + job = std::move(jobs.front()); + jobs.pop_front(); + ++inFlight; + } + const auto n = ++running; + auto p = peak.load(); + while (p < n && !peak.compare_exchange_weak(p, n)) { + } + job(); // packaged_task transports failures to the owning device thread. + --running; + { + std::lock_guard lock(mutex); + --inFlight; + } + ready.notify_all(); + } + }); + } catch (...) { + stop(); + throw; + } + } + ~CompilerQueue() { stop(); } + // Called on the device thread. Run bounds each segment to 48 operations; + // this independent queue bound also protects accidental future callers. + std::future enqueue(std::string source) { + Job job([source = std::move(source)] { + const auto start = std::chrono::steady_clock::now(); + auto result = gpgpu::compileComputeSpirv(source); + if (!result.ok()) throw std::runtime_error(result.error()->message() + "\nCompute source:\n" + source); + return CompiledProgram{ + std::move(result.value()), + std::chrono::duration(std::chrono::steady_clock::now() - start).count()}; + }); + auto future = job.get_future(); + { + std::lock_guard lock(mutex); + if (stopping || jobs.size() >= 48) throw std::runtime_error("ONNX compiler queue unavailable"); + jobs.push_back(std::move(job)); + } + ready.notify_one(); + return future; + } + size_t peakWorkers() const noexcept { return peak.load(); } + void waitIdle() noexcept { + std::unique_lock lock(mutex); + ready.wait(lock, [this] { return jobs.empty() && inFlight == 0; }); + } + void resetPeak() noexcept { peak = 0; } +}; +} // namespace eve::tensor::onnx_detail diff --git a/src/modules/tensor/OnnxCompute.h b/src/modules/tensor/OnnxCompute.h new file mode 100644 index 00000000..d78b4499 --- /dev/null +++ b/src/modules/tensor/OnnxCompute.h @@ -0,0 +1,85 @@ +#pragma once +#include +#include +#include +#include +#include +#include "common/Result.h" +#include "tensor/OnnxStorage.h" + +namespace eve::tensor { +/** @brief Synchronous GPU kernel request; all inputs are borrowed only until dispatch returns. */ +struct OnnxKernel { + std::string source; + std::vector> inputs; + size_t outputBytes = 0; + uint32_t workItems = 0; +}; +/** + * @brief GPU execution boundary for native ONNX; retains no model and retains compiled resources for the lifetime of + * the provider. + * @note Use on the device's owning thread, without concurrent calls or reentrancy. + * dispatch completes GPU work before returning host bytes; enqueue may defer completion. Errors never + * trigger CPU execution. Integer kernels require exact int32 results. + */ +class OnnxCompute { +public: + /** @brief Release device resources after all synchronous calls have returned. */ + virtual ~OnnxCompute() = default; + /** @brief Compatibility-only synchronous host-byte entry point; engine dispatch delegates to enqueue. + * @note GLSL 450, local_size_x=64; inputs occupy bindings 0..N-1, output N. */ + [[nodiscard]] virtual Result> dispatch(const OnnxKernel& kernel) = 0; + + /** @brief Queue a kernel using immutable host/device buffers; no readback is required. + * @note Default provider executes synchronously through dispatch. Engine provider records a + * resident graph segment inside runGpu, completing it at a host-read or resource-budget boundary. + * Inputs are retained until completion; returned device storage expires when runGpu ends. */ + [[nodiscard]] virtual Result enqueue(const std::string& source, std::span inputs, + size_t outputBytes, uint32_t workItems) { + OnnxKernel k{source, {}, outputBytes, workItems}; + std::vector> copies; + copies.reserve(inputs.size()); + for (const auto& input : inputs) { + if (input.host) + k.inputs.push_back(*input.host); + else { + if (!input.device) + return Result::failure( + Diagnostic::error(DiagnosticCode::InvalidArgument, "Missing GPU input")); + auto r = input.device->readback(); + if (!r.ok()) return Result::failure(r.status()); + copies.push_back(std::move(r.value())); + k.inputs.push_back(copies.back()); + } + } + auto r = dispatch(k); + if (!r.ok()) return Result::failure(r.status()); + auto bytes = std::make_shared>(std::move(r.value())); + return Result::success({bytes, {}, bytes->size()}); + } + /** @brief Observable transfer counters for the current or most recent run. */ + virtual OnnxTransferStats transferStats() const { return {}; } + +protected: + friend class OnnxModel; + /** @brief Start a synchronous run cache scope; invoked only by OnnxModel on the device thread. */ + virtual void beginRun() noexcept {} + /** @brief Retire temporary activations/recordings; compiled resources may survive for later calls. */ + virtual void endRun() noexcept {} +}; +/** + * @brief Create a reusable GPU session for the active engine Gpgpu Vulkan device, or an error. + * @note Retain and reuse this owning provider across runGpu calls. Pipelines, immutable initializers + * and buffer pools persist until provider destruction or Graphics resource retirement. Cache bounds: + * 1024 shader variants, 128 MiB initializer payload and 512 MiB device buffers. + * CPU shader compilation uses a + * bounded queue; pipeline creation and submission remain on the device thread. + * @param compilerWorkers Number of CPU + * compiler workers (1..8); default 4. Source is owned by each job. + * @lifetime Graphics retirement clears device + * resources before device destruction; either destruction order is supported. A retired session rejects further GPU + * work; create a new session for a new device. + * @thread Graphics thread only, no concurrent calls, reentrancy or device destruction during runGpu. + */ +[[nodiscard]] Result> createOnnxGpuCompute(uint32_t compilerWorkers = 4); +} // namespace eve::tensor diff --git a/src/modules/tensor/OnnxExecute.cpp b/src/modules/tensor/OnnxExecute.cpp new file mode 100644 index 00000000..350cf845 --- /dev/null +++ b/src/modules/tensor/OnnxExecute.cpp @@ -0,0 +1,377 @@ +#include "tensor/CpuKernels.h" +#include "tensor/OnnxInternal.h" +#include "tensor/Tensor.h" + +#include +#include +#include +#include + +namespace eve::tensor::onnx_detail { +bool isSupported(const Node& n) { + if (n.domain == "com.microsoft" && n.op == "DynamicQuantizeLSTM") { + if (n.inputs.size() != 12 || n.outputs.empty() || n.outputs.size() > 3) return false; + for (const auto& [key, a] : n.attrs) { + if (key == "direction") { + if (a.type != 3 || (a.text != "forward" && a.text != "reverse" && a.text != "bidirectional")) + return false; + } else if (key == "hidden_size" || key == "input_forget") { + if (a.type != 2) return false; + } else if (key == "clip") { + if (a.type != 1) return false; + } else + return false; + } + return true; + } + if (!n.domain.empty()) return false; + if (n.op == "If" || n.op == "Loop") { + const std::set keys = + n.op == "If" ? std::set{"then_branch", "else_branch"} : std::set{"body"}; + if (n.attrs.size() != keys.size() || n.outputs.empty() || + (n.op == "If" ? n.inputs.size() != 1 : n.inputs.size() < 2)) + return false; + for (const auto& key : keys) { + auto it = n.attrs.find(key); + if (it == n.attrs.end() || it->second.type != 5 || !it->second.graph) return false; + } + return true; + } + + static const std::map> allowed{ + {"DynamicQuantizeLinear", {}}, + {"QuantizeLinear", {"axis"}}, + {"DequantizeLinear", {"axis"}}, + {"MatMulInteger", {}}, + {"ConvInteger", {"auto_pad", "dilations", "group", "kernel_shape", "pads", "strides"}}, + {"Constant", {"value"}}, + {"Clip", {}}, + {"Pad", {"mode"}}, + {"CumSum", {"exclusive", "reverse"}}, + {"SequenceEmpty", {"dtype"}}, + {"SequenceAt", {}}, + {"SequenceInsert", {}}, + {"SplitToSequence", {"axis", "keepdims"}}, + {"ConcatFromSequence", {"axis", "new_axis"}}, + {"RandomUniformLike", {"dtype", "seed", "low", "high"}}, + {"RandomNormalLike", {"dtype", "seed", "mean", "scale"}}, + {"InstanceNormalization", {"epsilon"}}, + {"ConvTranspose", {"auto_pad", "dilations", "group", "kernel_shape", "output_padding", "pads", "strides"}}, + {"Resize", {"coordinate_transformation_mode", "mode", "nearest_mode", "cubic_coeff_a"}}, + {"Range", {}}, + {"TopK", {"axis", "largest", "sorted"}}, + {"ScatterND", {"reduction"}}, + {"ScatterElements", {"axis", "reduction"}}, + {"ReduceProd", {"axes", "keepdims"}}, + {"ReduceMax", {"axes", "keepdims"}}, + {"Slice", {}}, + {"Concat", {"axis"}}, + {"Expand", {}}, + {"ConstantOfShape", {"value"}}, + {"Where", {}}, + {"Equal", {}}, + {"Less", {}}, + {"Greater", {}}, + {"And", {}}, + {"Not", {}}, + {"Reciprocal", {}}, + {"LeakyRelu", {"alpha"}}, + {"Floor", {}}, + {"Round", {}}, + {"Atan", {}}, + {"Identity", {}}, + {"Cast", {"to"}}, + {"Gather", {"axis"}}, + {"Shape", {"start", "end"}}, + {"Reshape", {"allowzero"}}, + {"Transpose", {"perm"}}, + {"Unsqueeze", {}}, + {"Squeeze", {}}, + {"MatMul", {}}, + {"Add", {}}, + {"Sub", {}}, + {"Mul", {}}, + {"Div", {}}, + {"Sqrt", {}}, + {"Exp", {}}, + {"Log", {}}, + {"Sin", {}}, + {"Cos", {}}, + {"Relu", {}}, + {"Sigmoid", {}}, + {"Tanh", {}}, + {"Neg", {}}, + {"Abs", {}}, + {"Pow", {}}, + {"Softmax", {"axis"}}, + {"ReduceMean", {"axes", "keepdims"}}, + {"ReduceSum", {"keepdims", "noop_with_empty_axes"}}}; + auto op = allowed.find(n.op); + if (op == allowed.end()) return false; + size_t minimum = 1, maximum = 1, outputs = 1; + if (n.op == "Clip") maximum = 3; + if (n.op == "Pad") { + minimum = 2; + maximum = 3; + } + if (n.op == "CumSum") minimum = maximum = 2; + if (n.op == "SequenceEmpty") minimum = maximum = 0; + if (n.op == "SequenceAt") minimum = maximum = 2; + if (n.op == "SequenceInsert") { + minimum = 2; + maximum = 3; + } + if (n.op == "SplitToSequence") maximum = 2; + if (n.op == "InstanceNormalization") minimum = maximum = 3; + if (n.op == "ConvTranspose") { + minimum = 2; + maximum = 3; + } + if (n.op == "Resize") { + minimum = 3; + maximum = 4; + } + if (n.op == "Range" || n.op == "ScatterElements" || n.op == "ScatterND") minimum = maximum = 3; + if (n.op == "TopK") { + minimum = maximum = 2; + outputs = 2; + } + if (n.op == "Slice") { + minimum = 3; + maximum = 5; + } + if (n.op == "Concat") maximum = 100000; + if (n.op == "Where") minimum = maximum = 3; + if (n.op == "Expand" || n.op == "Equal" || n.op == "Less" || n.op == "Greater" || n.op == "And") + minimum = maximum = 2; + if (n.op == "Constant") minimum = maximum = 0; + if (n.op == "DynamicQuantizeLinear") outputs = 3; + if (n.op == "QuantizeLinear" || n.op == "DequantizeLinear") { + minimum = 2; + maximum = 3; + } + if (n.op == "MatMulInteger" || n.op == "ConvInteger") { + minimum = 2; + maximum = 4; + } + if (n.op == "Gather" || n.op == "Reshape" || n.op == "Unsqueeze" || n.op == "MatMul" || n.op == "Add" || + n.op == "Sub" || n.op == "Mul" || n.op == "Div" || n.op == "Pow") + minimum = maximum = 2; + if (n.op == "Squeeze" || n.op == "ReduceSum") maximum = 2; + if (n.inputs.size() < minimum || n.inputs.size() > maximum || n.outputs.size() != outputs) return false; + for (const auto& [key, a] : n.attrs) { + if (!op->second.contains(key)) return false; + const int expected = (key == "alpha" || key == "epsilon" || key == "cubic_coeff_a" || key == "seed" || + key == "low" || key == "high" || key == "mean" || key == "scale") + ? 1 + : key == "value" ? 4 + : (key == "auto_pad" || key == "reduction" || key == "coordinate_transformation_mode" || + key == "mode" || key == "nearest_mode") + ? 3 + : (key == "perm" || key == "axes" || key == "dilations" || key == "kernel_shape" || + key == "pads" || key == "strides" || key == "output_padding") + ? 7 + : 2; + if (a.type != expected) return false; + if (key == "reduction" && a.text != "none") return false; + if (key == "auto_pad" && a.text != "NOTSET") return false; + if (key == "allowzero" && a.integer != 0) return false; + } + if (n.op == "Pad" && n.attrs.contains("mode") && n.attrs.at("mode").text != "constant" && + n.attrs.at("mode").text != "edge" && n.attrs.at("mode").text != "reflect") + return false; + if (n.op == "Resize") { + const auto mode = n.attrs.contains("mode") ? n.attrs.at("mode").text : "nearest"; + const auto coordinate = n.attrs.contains("coordinate_transformation_mode") + ? n.attrs.at("coordinate_transformation_mode").text + : "half_pixel"; + const auto nearest = n.attrs.contains("nearest_mode") ? n.attrs.at("nearest_mode").text : "round_prefer_floor"; + if (mode == "nearest") { + if (coordinate != "asymmetric" || nearest != "floor") return false; + } else if (mode != "linear" || (coordinate != "half_pixel" && coordinate != "asymmetric")) + return false; + } + return true; +} +namespace { +std::vector dims(const RuntimeTensor& x) { + std::vector out; + for (auto d : x.shape) { + if (d <= 0) throw Failure("Empty float kernel input", DiagnosticCode::Unsupported); + out.push_back(static_cast(d)); + } + return out; +} +std::vector single(RuntimeTensor x) { return {std::move(x)}; } +} // namespace +std::vector execute(const Node& n, const std::vector& in, OnnxCompute* compute) { + if (n.domain == "com.microsoft" && n.op == "DynamicQuantizeLSTM") return executeQuantLstm(n, in, compute); + if (n.op == "Constant") { + auto a = n.attrs.find("value"); + if (a == n.attrs.end() || !a->second.tensor) throw Failure("Constant requires tensor value"); + return single(*a->second.tensor); + } + if (n.op.find("Quantize") != std::string::npos || n.op == "DequantizeLinear" || n.op == "MatMulInteger" || + n.op == "ConvInteger") + return executeQuant(n, in, compute); + const auto& x = required(in, 0); + if (n.op == "Identity") return single(x); + if (n.op == "Shape") { + const int64_t rank = x.shape.size(); + auto norm = [rank](int64_t i) { return std::clamp(i < 0 ? i + rank : i, int64_t(0), rank); }; + const auto begin = norm(attr(n, "start", 0)), end = norm(attr(n, "end", rank)); + std::vector shape; + if (begin < end) shape.assign(x.shape.begin() + begin, x.shape.begin() + end); + return single(make(OnnxElement::Int64, {static_cast(shape.size())}, shape)); + } + if (n.op == "Cast") { + const auto target = static_cast(attr(n, "to", 0)); + if (compute && target == OnnxElement::Float32 && x.element == OnnxElement::Int32) + return single(dispatchFloat(*compute, {&x}, x.shape, "y[i]=float(x0[i]);")); + const auto size = elementSize(target); + RuntimeTensor out{target, x.shape, std::vector(count(x.shape) * size)}; + for (size_t i = 0; i < count(x.shape); ++i) { + if (target == OnnxElement::Float32) { + const float v = + x.element == OnnxElement::Float32 ? read(x, i) : static_cast(integer(x, i)); + std::memcpy(out.bytes.data() + i * 4, &v, 4); + } else { + int64_t v; + if (x.element == OnnxElement::Float32) { + const double f = read(x, i); + if (!std::isfinite(f) || f < -9223372036854775808.0 || f >= 9223372036854775808.0) + throw Failure("Cast overflow"); + v = target == OnnxElement::Bool ? (f != 0) : static_cast(f); + } else + v = integer(x, i); + if (target == OnnxElement::Bool) v = v != 0; + std::memcpy(out.bytes.data() + i * size, &v, size); + } + } + return single(std::move(out)); + } + if (n.op == "Gather") { + const auto& idx = required(in, 1); + const int a = axis(attr(n, "axis", 0), x.shape.size()); + if (idx.element != OnnxElement::Int32 && idx.element != OnnxElement::Int64) + throw Failure("Gather indices must be int32/int64"); + size_t inner = 1, outer = 1; + for (size_t i = a + 1; i < x.shape.size(); ++i) inner *= x.shape[i]; + for (int i = 0; i < a; ++i) outer *= x.shape[i]; + std::vector shape(x.shape.begin(), x.shape.begin() + a); + shape.insert(shape.end(), idx.shape.begin(), idx.shape.end()); + shape.insert(shape.end(), x.shape.begin() + a + 1, x.shape.end()); + const size_t stride = inner * elementSize(x.element), ni = count(idx.shape); + RuntimeTensor out{x.element, shape, std::vector(count(shape) * elementSize(x.element))}; + for (size_t o = 0; o < outer; ++o) + for (size_t i = 0; i < ni; ++i) { + auto j = integer(idx, i); + if (j < 0) j += x.shape[a]; + if (j < 0 || j >= x.shape[a]) throw Failure("Gather index out of range"); + if (stride) + std::memcpy(out.bytes.data() + (o * ni + i) * stride, + x.bytes.data() + (o * x.shape[a] + j) * stride, stride); + } + return single(std::move(out)); + } + if (n.op == "Reshape" || n.op == "Unsqueeze" || n.op == "Squeeze") { + auto shape = x.shape; + if (n.op == "Reshape") { + shape = ints(required(in, 1)); + int infer = -1; + size_t known = 1; + for (size_t i = 0; i < shape.size(); ++i) { + if (shape[i] == 0) { + if (i >= x.shape.size()) throw Failure("Reshape zero axis out of range"); + shape[i] = x.shape[i]; + } + if (shape[i] == -1) { + if (infer != -1) throw Failure("Multiple inferred dimensions"); + infer = static_cast(i); + } else { + if (shape[i] < 0 || shape[i] > INT32_MAX || + (shape[i] && known > 128u * 1024u * 1024u / static_cast(shape[i]))) + throw Failure("Reshape overflow"); + known *= static_cast(shape[i]); + } + } + if (infer >= 0) { + if (!known || count(x.shape) % known) throw Failure("Invalid inferred shape"); + shape[infer] = count(x.shape) / known; + } + } else { + std::vector axes; + if (in.size() > 1 && in[1]) + axes = ints(*in[1]); + else if (n.op == "Unsqueeze") + throw Failure("Unsqueeze needs axes"); + else + for (size_t i = 0; i < shape.size(); ++i) + if (shape[i] == 1) axes.push_back(i); + const size_t rank = shape.size() + (n.op == "Unsqueeze" ? axes.size() : 0); + std::set positions; + for (auto a : axes) + if (!positions.insert(axis(a, rank)).second) throw Failure("Duplicate squeeze axis"); + if (n.op == "Unsqueeze") { + for (int a : positions) shape.insert(shape.begin() + a, 1); + } else + for (auto it = positions.rbegin(); it != positions.rend(); ++it) { + if (shape[*it] != 1) throw Failure("Squeeze dimension not one"); + shape.erase(shape.begin() + *it); + } + } + if (count(shape) != count(x.shape)) throw Failure("Reshape element count mismatch"); + RuntimeTensor out = x; + out.shape = std::move(shape); + return single(std::move(out)); + } + if (n.op == "Transpose") { + std::vector order = attrs(n, "perm", {}); + if (order.empty()) + for (size_t i = x.shape.size(); i > 0; --i) order.push_back(i - 1); + if (order.size() != x.shape.size()) throw Failure("Invalid transpose rank"); + std::set used; + std::vector shape; + for (auto a : order) { + if (a < 0 || a >= static_cast(x.shape.size()) || !used.insert(a).second) + throw Failure("Invalid transpose permutation"); + shape.push_back(x.shape[a]); + } + RuntimeTensor out{x.element, shape, std::vector(x.bytes.size())}; + const auto size = elementSize(x.element); + std::vector strides(x.shape.size(), 1); + for (size_t i = x.shape.size(); i > 1; --i) strides[i - 2] = strides[i - 1] * x.shape[i - 1]; + for (size_t i = 0; i < count(shape); ++i) { + size_t rest = i, source = 0; + for (size_t j = shape.size(); j > 0; --j) { + source += (rest % shape[j - 1]) * strides[order[j - 1]]; + rest /= shape[j - 1]; + } + std::memcpy(out.bytes.data() + i * size, x.bytes.data() + source * size, size); + } + return single(std::move(out)); + } + if (auto indexResult = executeIndex(n, in)) return std::move(*indexResult); + if (auto shapeResult = executeShape(n, in)) return single(std::move(*shapeResult)); + if (auto miscResult = executeMisc(n, in, compute)) return single(std::move(*miscResult)); + if (auto neuralResult = executeNeural(n, in, compute)) return single(std::move(*neuralResult)); + if (auto numericResult = executeNumeric(n, in, compute)) return single(std::move(*numericResult)); + const auto a = floats(x); + if (n.op == "Softmax") { + auto xd = dims(x); + std::vector out(a.size()); + kernels::softmax(a.data(), xd.data(), static_cast(xd.size()), axis(attr(n, "axis", -1), xd.size()), false, + out.data()); + return single(make(OnnxElement::Float32, x.shape, out)); + } + static const std::map unary{ + {"Sqrt", OpType::Sqrt}, {"Exp", OpType::Exp}, {"Log", OpType::Log}, {"Sin", OpType::Sin}, + {"Cos", OpType::Cos}, {"Relu", OpType::Relu}, {"Sigmoid", OpType::Sigmoid}, {"Tanh", OpType::Tanh}, + {"Neg", OpType::Neg}, {"Abs", OpType::Abs}}; + auto op = unary.find(n.op); + if (op == unary.end()) throw Failure("Unsupported operator", DiagnosticCode::Unsupported); + std::vector out(a.size()); + kernels::unaryOp(op->second, a.data(), static_cast(a.size()), out.data(), 0, 0); + return single(make(OnnxElement::Float32, x.shape, out)); +} +} // namespace eve::tensor::onnx_detail diff --git a/src/modules/tensor/OnnxGpgpu.cpp b/src/modules/tensor/OnnxGpgpu.cpp new file mode 100644 index 00000000..2ef05cf2 --- /dev/null +++ b/src/modules/tensor/OnnxGpgpu.cpp @@ -0,0 +1,378 @@ +#include +#include +#include +#include +#include +#include +#include "common/Subscription.h" +#include "gpgpu/ComputeShader.h" +#include "gpgpu/Gpgpu.h" +#include "gpgpu/GpuBuffer.h" +#include "gpgpu/Sequence.h" +#include "graphics/Graphics.h" +#include "tensor/OnnxCompiler.h" +#include "tensor/OnnxCompute.h" +namespace eve::tensor { +namespace { +using Clock = std::chrono::steady_clock; +struct Timed { + double& milliseconds; + Clock::time_point start = Clock::now(); + ~Timed() { milliseconds += std::chrono::duration(Clock::now() - start).count(); } +}; +struct Run; +struct DeviceBytes final : OnnxDeviceStorage { + std::weak_ptr run; + std::unique_ptr buffer; + size_t size = 0, capacity = 0; + uint64_t epoch = 0; + ~DeviceBytes() override; + Result> readback() const override; +}; +struct ShaderEntry { + std::future compilation; + std::unique_ptr shader; +}; +struct PendingDispatch { + ShaderEntry* program; + std::vector> bindings; + uint32_t groups; +}; +struct Run : std::enable_shared_from_this { + gpgpu::Gpgpu& gp; + std::unique_ptr sequence; + std::unordered_map shaders; + onnx_detail::CompilerQueue compiler; + std::vector dispatches; + double compileTaskMs = 0, compileWaitMs = 0, pipelineMs = 0; + std::map>, std::shared_ptr, + std::owner_less>>> + uploads, persistentUploads; + std::vector>> uncommittedUploads; + bool active = true; + uint64_t epoch = 1; + std::vector> pending; + std::vector> resources; + OnnxTransferStats stats; + double shaderMs = 0, allocationMs = 0, uploadRecordMs = 0, dispatchRecordMs = 0, waitMs = 0, readbackMs = 0; + size_t compiledShaders = 0; + Clock::time_point started = Clock::now(); + size_t queued = 0, cachedBytes = 0, allocatedBytes = 0; + std::multimap> freeBuffers; + explicit Run(gpgpu::Gpgpu& g, uint32_t workers) : gp(g), sequence(g.newSequence()), compiler(workers) { + sequence->begin(); + } + ~Run() { + // Destroy an unsubmitted command sequence before its referenced resources. + sequence.reset(); + for (auto& [key, entry] : shaders) + if (entry.shader) entry.shader->clearBindings(); + for (auto& weak : resources) + if (auto p = weak.lock()) p->buffer.reset(); + } + void beginCall() noexcept { + stats = {}; + compiler.resetPeak(); + compileTaskMs = compileWaitMs = pipelineMs = 0; + shaderMs = allocationMs = uploadRecordMs = dispatchRecordMs = waitMs = readbackMs = 0; + compiledShaders = 0; + started = Clock::now(); + active = true; + ++epoch; + } + void endCall() noexcept { + compiler.waitIdle(); + // An abandoned graph must not record or submit its deferred commands. + // Unmaterialized jobs contain only CPU data; discard their future (including any + // compilation exception) because the caller already received the graph failure. + dispatches.clear(); + std::erase_if(shaders, [](const auto& item) { return !item.second.shader; }); + if (const auto* profile = std::getenv("EVE_ONNX_PROFILE"); profile && std::string(profile) == "1") + std::cerr << "ONNX_PROFILE shader_ms=" << shaderMs << " allocation_ms=" << allocationMs + << " upload_record_ms=" << uploadRecordMs << " dispatch_record_ms=" << dispatchRecordMs + << " submit_wait_ms=" << waitMs << " readback_host_ms=" << readbackMs + << " compile_task_ms=" << compileTaskMs << " compile_wait_ms=" << compileWaitMs + << " pipeline_ms=" << pipelineMs << " compiler_peak_workers=" << compiler.peakWorkers() + << " compiled_shaders=" << compiledShaders << " resident_scope_ms=" + << std::chrono::duration(Clock::now() - started).count() << '\n'; + sequence.reset(); + for (auto& [key, entry] : shaders) + if (entry.shader) entry.shader->clearBindings(); + // Abandoned recorded uploads have not reached the device and cannot enter a persistent cache. + for (const auto& key : uncommittedUploads) { + if (auto it = persistentUploads.find(key); it != persistentUploads.end()) { + cachedBytes -= key->size(); + persistentUploads.erase(it); + } + } + uncommittedUploads.clear(); + pending.clear(); + uploads.clear(); + queued = 0; + active = false; + std::erase_if(resources, [](const auto& p) { return p.expired(); }); + } + std::shared_ptr allocate(size_t bytes) { + Timed timing{allocationMs}; + if (!bytes || bytes > INT32_MAX - 3) throw std::runtime_error("Invalid resident buffer extent"); + auto p = std::make_shared(); + p->run = shared_from_this(); + p->size = bytes; + p->epoch = epoch; + const auto capacity = std::max(size_t(4), (bytes + 3) & ~size_t(3)); + auto reusable = freeBuffers.lower_bound(capacity); + if (reusable != freeBuffers.end() && reusable->first <= std::max(size_t(4096), capacity * 2)) { + p->capacity = reusable->first; + p->buffer = std::move(reusable->second); + freeBuffers.erase(reusable); + ++stats.bufferReuses; + } else { + if (allocatedBytes + capacity > 512u * 1024u * 1024u) { + for (const auto& [n, b] : freeBuffers) allocatedBytes -= n; + freeBuffers.clear(); + } + if (allocatedBytes + capacity > 512u * 1024u * 1024u) + throw std::runtime_error("ONNX GPU resident memory limit exceeded"); + p->buffer.reset(gp.newBuffer(static_cast(capacity))); + p->capacity = capacity; + allocatedBytes += capacity; + ++stats.bufferAllocations; + } + resources.push_back(p); + return p; + } + void recordDispatches() { + for (auto& command : dispatches) { + auto& entry = *command.program; + if (!entry.shader) { + Timed shaderTiming{shaderMs}; + onnx_detail::CompiledProgram program; + { + Timed waitTiming{compileWaitMs}; + program = entry.compilation.get(); + } + compileTaskMs += program.milliseconds; + { + Timed pipelineTiming{pipelineMs}; + auto result = gpgpu::createComputeShader(program.words); + if (!result.ok()) throw std::runtime_error(result.error()->message()); + entry.shader = std::move(result.value()); + } + } + Timed timing{dispatchRecordMs}; + auto& shader = *entry.shader; + for (size_t i = 0; i < command.bindings.size(); ++i) + shader.bindBuffer(static_cast(i), command.bindings[i]->buffer.get()); + sequence->recordDispatch(&shader, static_cast(command.groups)); + shader.clearBindings(); + } + stats.compilerPeakWorkers = compiler.peakWorkers(); + dispatches.clear(); + } + void flush() { + if (!queued) return; + recordDispatches(); + Timed timing{waitMs}; + // Submit asynchronously through the existing sequence contract; CPU visibility is + // requested only at a graph boundary. Retain every referenced allocation until wait. + if (sequence->submitAsync() != gpgpu::SequenceStatus::Submitted || + sequence->wait() != gpgpu::SequenceStatus::Complete) + throw std::runtime_error("ONNX GPU submission failed"); + uncommittedUploads.clear(); + ++stats.submissions; + pending.clear(); + queued = 0; + sequence->begin(); + } + std::shared_ptr input(const OnnxBuffer& value) { + if (value.device) { + auto d = std::dynamic_pointer_cast(value.device); + if (!d || d->run.lock().get() != this || !d->buffer || d->epoch != epoch || !active) + throw std::runtime_error("Foreign or expired GPU storage"); + return d; + } + if (!value.host || value.host->size() != value.size) throw std::runtime_error("Invalid host input storage"); + auto& cache = value.persistent ? persistentUploads : uploads; + if (auto it = cache.find(value.host); it != cache.end()) { + it->second->epoch = epoch; + return it->second; + } + if (value.persistent && cachedBytes + value.size > 128u * 1024u * 1024u) { + persistentUploads.clear(); + cachedBytes = 0; + } + auto d = allocate(std::max(size_t(1), value.size)); + Timed timing{uploadRecordMs}; + std::vector padded(std::max(size_t(4), (value.size + 3) & ~size_t(3)), 0); + std::copy(value.host->begin(), value.host->end(), padded.begin()); + sequence->recordUpload(d->buffer.get(), padded.data(), padded.size()); + ++stats.uploads; + stats.uploadedBytes += value.size; + ++queued; + cache.emplace(value.host, d); + if (value.persistent) { + cachedBytes += value.size; + uncommittedUploads.push_back(value.host); + } + pending.push_back(d); + return d; + } +}; +DeviceBytes::~DeviceBytes() { + if (buffer) + if (auto owner = run.lock()) owner->freeBuffers.emplace(capacity, std::move(buffer)); +} +Result> DeviceBytes::readback() const { + try { + auto owner = run.lock(); + if (!owner || !buffer || !owner->active || epoch != owner->epoch) + throw std::runtime_error("ONNX GPU storage expired"); + // Dispatches must precede the download in the command stream. Compilation + // waits are accounted separately from host readback and GPU submission. + owner->recordDispatches(); + const auto start = Clock::now(); + const auto priorWait = owner->waitMs; + std::unique_ptr staging(owner->gp.newBuffer(static_cast(size), "staging")); + owner->sequence->recordDownload(buffer.get(), staging.get(), size); + ++owner->queued; + owner->flush(); + std::vector out(size); + staging->downloadBytes(out.data(), size); + owner->readbackMs += + std::chrono::duration(Clock::now() - start).count() - (owner->waitMs - priorWait); + ++owner->stats.downloads; + owner->stats.downloadedBytes += size; + return Result>::success(std::move(out)); + } catch (const std::exception& e) { + return Result>::failure(Diagnostic::error(DiagnosticCode::Failed, e.what())); + } +} +struct SessionState { + std::shared_ptr run; + bool retired = false; + uint32_t compilerWorkers = 4; +}; +class EngineCompute final : public OnnxCompute { + std::shared_ptr state; + eve::Subscription retirement; + OnnxTransferStats last; + bool active = false; + +protected: + void beginRun() noexcept override { + if (state->run) state->run->beginCall(); + last = {}; + active = true; + } + void endRun() noexcept override { + if (state->run) { + last = state->run->stats; + state->run->endCall(); + } + active = false; + } + +public: + EngineCompute(std::shared_ptr s, eve::Subscription token) + : state(std::move(s)), retirement(std::move(token)) {} + OnnxTransferStats transferStats() const override { return active && state->run ? state->run->stats : last; } + Result enqueue(const std::string& source, std::span inputs, size_t bytes, + uint32_t work) override { + try { + auto& run = state->run; + if (state->retired) throw std::runtime_error("ONNX GPU session device has been retired"); + if (!active) throw std::runtime_error("Resident enqueue requires an active runGpu scope"); + if (inputs.size() >= 8 || !bytes || !work || (uint64_t(work) + 63) / 64 > 65535) + throw std::runtime_error("Invalid GPU kernel extent"); + if (!run) { + auto* gp = gpgpu::Gpgpu::create(); + if (!gp || !gp->isAvailable()) throw std::runtime_error("ONNX GPU device unavailable"); + run = std::make_shared(*gp, state->compilerWorkers); + } + if (!run->sequence) { + run->sequence.reset(run->gp.newSequence()); + run->sequence->begin(); + } + // Bound pending resources and the existing shader descriptor pool (64 sets). + if (run->queued >= 48) run->flush(); + auto it = run->shaders.find(source); + if (it == run->shaders.end()) { + if (run->shaders.size() >= 1024) { + run->flush(); + run->shaders.clear(); + } + ++run->stats.shaderCompilations; + ++run->compiledShaders; + it = run->shaders.emplace(source, ShaderEntry{run->compiler.enqueue(source), {}}).first; + } else + ++run->stats.shaderCacheHits; + PendingDispatch command{&it->second, {}, (work + 63) / 64}; + for (const auto& input : inputs) { + auto d = run->input(input); + command.bindings.push_back(d); + run->pending.push_back(d); + } + auto out = run->allocate(bytes); + command.bindings.push_back(out); + run->dispatches.push_back(std::move(command)); + run->pending.push_back(out); + ++run->queued; + return Result::success({{}, out, bytes}); + } catch (const std::exception& e) { + return Result::failure( + Diagnostic::error(DiagnosticCode::Failed, e.what(), {}, {}, "tensor.onnx.gpu")); + } + } + Result> dispatch(const OnnxKernel& k) override { + const bool temporary = !active; + if (temporary) beginRun(); + struct Scope { + EngineCompute& owner; + bool temporary; + ~Scope() { + if (temporary) owner.endRun(); + } + } scope{*this, temporary}; + std::vector inputs; + for (auto s : k.inputs) { + auto p = std::make_shared>(s.begin(), s.end()); + inputs.push_back({p, {}, p->size()}); + } + auto r = enqueue(k.source, inputs, k.outputBytes, k.workItems); + if (!r.ok()) return Result>::failure(r.status()); + return r.value().device->readback(); + } +}; +} // namespace +Result> createOnnxGpuCompute(uint32_t compilerWorkers) { + if (compilerWorkers < 1 || compilerWorkers > 8) + return Result>::failure( + Diagnostic::error(DiagnosticCode::InvalidArgument, "Compiler worker count must be 1..8")); +#ifdef EVENGINE_WEBGPU + return Result>::failure( + Diagnostic::error(DiagnosticCode::Unsupported, "ONNX GLSL kernels require the Vulkan backend")); +#else + try { + auto* gp = gpgpu::Gpgpu::create(); + if (gp && gp->isAvailable()) { + auto state = std::make_shared(); + state->compilerWorkers = compilerWorkers; + std::weak_ptr weak = state; + auto token = graphics::Graphics::create()->onResourcesRetiring([weak]() noexcept { + if (auto s = weak.lock()) { + s->retired = true; + s->run.reset(); + } + }); + if (!token.ok()) return Result>::failure(token.status()); + return Result>::success( + std::make_unique(state, std::move(token.value()))); + } + } catch (const std::exception& e) { + return Result>::failure(Diagnostic::error(DiagnosticCode::Failed, e.what())); + } + return Result>::failure(Diagnostic::error( + DiagnosticCode::Unsupported, "Initialize the engine Vulkan graphics device before ONNX GPU execution")); +#endif +} +} // namespace eve::tensor diff --git a/src/modules/tensor/OnnxGpuKernels.cpp b/src/modules/tensor/OnnxGpuKernels.cpp new file mode 100644 index 00000000..3609d5e5 --- /dev/null +++ b/src/modules/tensor/OnnxGpuKernels.cpp @@ -0,0 +1,102 @@ +#include "tensor/OnnxGpuKernels.h" +#include +#include "tensor/OnnxInternal.h" +namespace eve::tensor::onnx_detail { +namespace { +std::string prefix(bool as, bool bs) { + std::ostringstream s; + s << "#version 450\nlayout(local_size_x=64) in;\n" + "layout(std430,binding=0) readonly buffer A {uint a[];};\n" + "layout(std430,binding=1) readonly buffer B {uint b[];};\n" + "layout(std430,binding=2) readonly buffer Z {int zp[];};\n" + "layout(std430,binding=3) writeonly buffer Y {int y[];};\n" + "int av(uint i){int v=int((a[i/4]>>((i%4)*8))&255u);return " + << (as ? "v>=128?v-256:v" : "v") + << ";}\n" + "int bv(uint i){int v=int((b[i/4]>>((i%4)*8))&255u);return " + << (bs ? "v>=128?v-256:v" : "v") << ";}\n"; + return s.str(); +} +// Compatibility host-byte calls use the same resident kernel generators. The +// adapter below requests completion through the provider's synchronous contract. +class Synchronous final : public OnnxCompute { + OnnxCompute& target; + +public: + explicit Synchronous(OnnxCompute& t) : target(t) {} + Result> dispatch(const OnnxKernel& k) override { return target.dispatch(k); } +}; +OnnxBuffer hostBuffer(std::span bytes) { + auto p = std::make_shared>(bytes.begin(), bytes.end()); + return {p, {}, p->size()}; +} +std::vector integerOutput(const OnnxBuffer& result) { + if (!result.host || result.host->size() % 4) throw Failure("Invalid synchronous integer output"); + std::vector out(result.host->size() / 4); + std::memcpy(out.data(), result.host->data(), result.host->size()); + return out; +} +} // namespace +std::vector gpuMatmul(OnnxCompute& d, affine::ByteView a, affine::ByteView b, size_t m, size_t k, size_t n, + int az, std::span bz) { + Synchronous sync(d); + return integerOutput(gpuMatmulResident(sync, hostBuffer(a.bytes), hostBuffer(b.bytes), a.signedValues, + b.signedValues, m, k, n, az, bz)); +} +std::vector gpuConv(OnnxCompute& d, affine::ByteView x, affine::ByteView w, const affine::ConvShape& c, int xz, + std::span wz) { + Synchronous sync(d); + return integerOutput( + gpuConvResident(sync, hostBuffer(x.bytes), hostBuffer(w.bytes), x.signedValues, w.signedValues, c, xz, wz)); +} +OnnxBuffer enqueueInteger(OnnxCompute& d, const std::string& source, OnnxBuffer a, OnnxBuffer b, + std::span z, size_t size, size_t terms) { + if (terms > INT32_MAX / 65025 || !size || size > 64u * 65535u) + throw Failure("GPU integer kernel exceeds exact accumulator or dispatch limit", DiagnosticCode::Unsupported); + std::vector bytes(z.size_bytes()); + std::memcpy(bytes.data(), z.data(), bytes.size()); + auto host = std::make_shared>(std::move(bytes)); + const std::vector inputs{a, b, {host, {}, host->size()}}; + auto r = d.enqueue(source, inputs, size * 4, static_cast(size)); + if (!r.ok()) throw Failure(r.error()->message(), r.error()->code()); + return std::move(r.value()); +} +OnnxBuffer gpuMatmulResident(OnnxCompute& d, OnnxBuffer a, OnnxBuffer b, bool as, bool bs, size_t m, size_t k, size_t n, + int az, std::span bz, size_t bOffset) { + if (bz.size() != 1 && bz.size() != n) throw Failure("GPU matmul zero-point extent mismatch"); + std::ostringstream s; + s << prefix(as, bs) << "void main(){uint i=gl_GlobalInvocationID.x;if(i>=" << m * n << "u)return;" + << "uint r=i/" << n << "u,c=i%" << n << "u;int v=0;for(uint j=0;j<" << k << "u;++j)" + << "v+=(av(r*" << k << "u+j)-(" << az << "))*(bv(" << bOffset << "u+j*" << n << "u+c)-zp[" + << (bz.size() == 1 ? "0" : "c") << "]);y[i]=v;}"; + return enqueueInteger(d, s.str(), a, b, bz, m * n, k); +} + +OnnxBuffer gpuConvResident(OnnxCompute& d, OnnxBuffer x, OnnxBuffer w, bool xs, bool ws, const affine::ConvShape& c, + int xz, std::span wz) { + const int64_t oh = + (int64_t(c.height) + c.padTop + c.padBottom - int64_t(c.dilationH) * (c.kernelH - 1) - 1) / c.strideH + 1; + const int64_t ow = + (int64_t(c.width) + c.padLeft + c.padRight - int64_t(c.dilationW) * (c.kernelW - 1) - 1) / c.strideW + 1; + if ((oh - 1) * c.strideH > INT32_MAX || (ow - 1) * c.strideW > INT32_MAX || + (oh - 1) * c.strideH - c.padTop + int64_t(c.kernelH - 1) * c.dilationH > INT32_MAX || + (ow - 1) * c.strideW - c.padLeft + int64_t(c.kernelW - 1) * c.dilationW > INT32_MAX) + throw Failure("GPU convolution coordinate overflow", DiagnosticCode::Unsupported); + const size_t size = size_t(c.batch) * c.outputs * oh * ow; + const int cg = c.channels / c.groups; + std::ostringstream s; + s << prefix(xs, ws) << "void main(){uint i=gl_GlobalInvocationID.x;if(i>=" << size << "u)return;" + << "int xx=int(i%" << ow << "u),yy=int(i/" << ow << "u%" << oh << "u),o=int(i/" << ow * oh << "u%" << c.outputs + << "u),batch=int(i/" << ow * oh * c.outputs << "u);int v=0;" + << "for(int ch=0;ch<" << cg << ";++ch)for(int ky=0;ky<" << c.kernelH << ";++ky)for(int kx=0;kx<" << c.kernelW + << ";++kx){" + << "int iy=yy*" << c.strideH << "-" << c.padTop << "+ky*" << c.dilationH << ",ix=xx*" << c.strideW << "-" + << c.padLeft << "+kx*" << c.dilationW << ";" + << "if(iy<0||ix<0||iy>=" << c.height << "||ix>=" << c.width << ")continue;" + << "uint ai=uint(((batch*" << c.channels << "+o/" << c.outputs / c.groups << "*" << cg << "+ch)*" << c.height + << "+iy)*" << c.width << "+ix);" + << "uint bi=uint(((o*" << cg << "+ch)*" << c.kernelH << "+ky)*" << c.kernelW << "+kx);" + << "v+=(av(ai)-(" << xz << "))*(bv(bi)-zp[" << (wz.size() == 1 ? "0" : "o") << "]);}y[i]=v;}"; + return enqueueInteger(d, s.str(), x, w, wz, size, size_t(cg) * c.kernelH * c.kernelW); +} +} // namespace eve::tensor::onnx_detail diff --git a/src/modules/tensor/OnnxGpuKernels.h b/src/modules/tensor/OnnxGpuKernels.h new file mode 100644 index 00000000..7bdbdc56 --- /dev/null +++ b/src/modules/tensor/OnnxGpuKernels.h @@ -0,0 +1,14 @@ +#pragma once +#include "tensor/AffineQuant.h" +#include "tensor/OnnxCompute.h" +namespace eve::tensor::onnx_detail { +// Internal kernels; callers validate geometry and byte extents before dispatch. +std::vector gpuMatmul(OnnxCompute& device, affine::ByteView a, affine::ByteView b, size_t m, size_t k, + size_t n, int az, std::span bz); +std::vector gpuConv(OnnxCompute& device, affine::ByteView x, affine::ByteView w, const affine::ConvShape& s, + int xz, std::span wz); +OnnxBuffer gpuMatmulResident(OnnxCompute&, OnnxBuffer, OnnxBuffer, bool, bool, size_t, size_t, size_t, int, + std::span, size_t bOffset = 0); +OnnxBuffer gpuConvResident(OnnxCompute&, OnnxBuffer, OnnxBuffer, bool, bool, const affine::ConvShape&, int, + std::span); +} // namespace eve::tensor::onnx_detail diff --git a/src/modules/tensor/OnnxIndexOps.cpp b/src/modules/tensor/OnnxIndexOps.cpp new file mode 100644 index 00000000..1646bc9e --- /dev/null +++ b/src/modules/tensor/OnnxIndexOps.cpp @@ -0,0 +1,180 @@ +#include +#include +#include +#include +#include +#include "tensor/OnnxInternal.h" + +namespace eve::tensor::onnx_detail { +std::optional> executeIndex(const Node& n, const std::vector& in) { + const auto& x = required(in, 0); + auto one = [](RuntimeTensor t) { return std::vector{std::move(t)}; }; + if (n.op == "Range") { + const auto& end = required(in, 1); + const auto& delta = required(in, 2); + if (count(x.shape) != 1 || count(end.shape) != 1 || count(delta.shape) != 1 || x.element != end.element || + x.element != delta.element) + throw Failure("Range requires matching scalars"); + if (x.element == OnnxElement::Float32) { + const double start = read(x, 0), stop = read(end, 0), step = read(delta, 0); + if (!std::isfinite(start) || !std::isfinite(stop) || !std::isfinite(step) || step == 0) + throw Failure("Invalid Range"); + const double length = std::max(0., std::ceil((stop - start) / step)); + if (length > 128u * 1024u * 1024u) throw Failure("Range too large"); + std::vector v(static_cast(length)); + for (size_t i = 0; i < v.size(); ++i) v[i] = static_cast(start + i * step); + return one(make(x.element, {static_cast(v.size())}, v)); + } + if (x.element != OnnxElement::Int64 && x.element != OnnxElement::Int32) + throw Failure("Range integer dtype unsupported"); + const int64_t start = integer(x), stop = integer(end), step = integer(delta); + if (!step || step == INT64_MIN) throw Failure("Invalid Range step"); + uint64_t length = 0; + if (step > 0 && stop > start) length = 1 + (uint64_t(stop) - uint64_t(start) - 1) / uint64_t(step); + if (step < 0 && stop < start) length = 1 + (uint64_t(start) - uint64_t(stop) - 1) / uint64_t(-step); + if (length > 128u * 1024u * 1024u) throw Failure("Range too large"); + RuntimeTensor out{ + x.element, {static_cast(length)}, std::vector(length * elementSize(x.element))}; + int64_t v = start; + for (size_t i = 0; i < length; ++i) { + std::memcpy(out.bytes.data() + i * elementSize(x.element), &v, elementSize(x.element)); + if (i + 1 < length) v += step; + } + return one(std::move(out)); + } + if (n.op == "TopK") { + const auto& kt = required(in, 1); + if (count(kt.shape) != 1) throw Failure("TopK requires one K"); + int a = axis(attr(n, "axis", -1), x.shape.size()); + const int64_t k = integer(kt), size = x.shape[a]; + if (k < 0 || k > size) throw Failure("Invalid TopK K"); + size_t outer = 1, inner = 1; + for (int j = 0; j < a; ++j) outer *= x.shape[j]; + for (size_t j = a + 1; j < x.shape.size(); ++j) inner *= x.shape[j]; + auto shape = x.shape; + shape[a] = k; + RuntimeTensor values{x.element, shape, std::vector(count(shape) * elementSize(x.element))}; + std::vector indices(count(shape)); + for (size_t o = 0; o < outer; ++o) + for (size_t i = 0; i < inner; ++i) { + std::vector order(size); + std::iota(order.begin(), order.end(), 0); + std::stable_sort(order.begin(), order.end(), [&](int64_t aa, int64_t bb) { + auto compare = [&](auto u, auto v) { return attr(n, "largest", 1) ? u > v : u < v; }; + const size_t ai = (o * size + aa) * inner + i, bi = (o * size + bb) * inner + i; + return x.element == OnnxElement::Float32 ? compare(read(x, ai), read(x, bi)) + : compare(integer(x, ai), integer(x, bi)); + }); + for (int64_t j = 0; j < k; ++j) { + size_t dst = (o * k + j) * inner + i, src = (o * size + order[j]) * inner + i; + indices[dst] = order[j]; + std::memcpy(values.bytes.data() + dst * elementSize(x.element), + x.bytes.data() + src * elementSize(x.element), elementSize(x.element)); + } + } + return std::vector{std::move(values), make(OnnxElement::Int64, shape, indices)}; + } + if (n.op == "ScatterElements" || n.op == "ScatterND") { + const auto& idx = required(in, 1); + const auto& updates = required(in, 2); + if (updates.element != x.element || (idx.element != OnnxElement::Int64 && idx.element != OnnxElement::Int32)) + throw Failure("Scatter dtype mismatch"); + auto out = x; + const auto bytes = elementSize(x.element); + if (n.op == "ScatterElements") { + if (idx.shape != updates.shape || idx.shape.size() != x.shape.size()) + throw Failure("ScatterElements shape mismatch"); + int a = axis(attr(n, "axis", 0), x.shape.size()); + for (size_t j = 0; j < x.shape.size(); ++j) + if (j != a && idx.shape[j] > x.shape[j]) throw Failure("Scatter index shape exceeds data"); + for (size_t i = 0; i < count(idx.shape); ++i) { + size_t rest = i, source = 0, stride = 1; + for (size_t j = x.shape.size(); j > 0; --j) { + int64_t c = rest % idx.shape[j - 1]; + rest /= idx.shape[j - 1]; + if (j - 1 == a) { + c = integer(idx, i); + if (c < 0) c += x.shape[j - 1]; + } + if (c < 0 || c >= x.shape[j - 1]) throw Failure("Scatter index out of range"); + source += c * stride; + stride *= x.shape[j - 1]; + } + std::memcpy(out.bytes.data() + source * bytes, updates.bytes.data() + i * bytes, bytes); + } + } else { + if (idx.shape.empty() || idx.shape.back() < 1 || static_cast(idx.shape.back()) > x.shape.size()) + throw Failure("ScatterND index rank mismatch"); + const size_t depth = idx.shape.back(); + std::vector expected(idx.shape.begin(), idx.shape.end() - 1); + expected.insert(expected.end(), x.shape.begin() + depth, x.shape.end()); + if (expected != updates.shape) throw Failure("ScatterND updates shape mismatch"); + size_t chunk = 1; + for (size_t j = depth; j < x.shape.size(); ++j) chunk *= x.shape[j]; + for (size_t i = 0; i < count(idx.shape) / depth; ++i) { + size_t target = 0; + for (size_t j = 0; j < depth; ++j) { + auto c = integer(idx, i * depth + j); + if (c < 0) c += x.shape[j]; + if (c < 0 || c >= x.shape[j]) throw Failure("ScatterND index out of range"); + target = target * x.shape[j] + c; + } + if (chunk) + std::memcpy(out.bytes.data() + target * chunk * bytes, updates.bytes.data() + i * chunk * bytes, + chunk * bytes); + } + } + return one(std::move(out)); + } + if (n.op == "ReduceProd" || n.op == "ReduceMax") { + auto axes = attrs(n, "axes", {}); + if (axes.empty()) + for (size_t i = 0; i < x.shape.size(); ++i) axes.push_back(i); + std::set reduced; + for (auto a : axes) + if (!reduced.insert(axis(a, x.shape.size())).second) throw Failure("Duplicate reduction axis"); + auto mapShape = x.shape; + std::vector shape; + for (size_t j = 0; j < x.shape.size(); ++j) { + if (reduced.contains(static_cast(j))) { + mapShape[j] = 1; + if (attr(n, "keepdims", 1)) shape.push_back(1); + } else + shape.push_back(x.shape[j]); + } + if (x.element == OnnxElement::Float32) { + std::vector out(count(shape), n.op == "ReduceProd" ? 1.f : -std::numeric_limits::infinity()); + for (size_t i = 0; i < count(x.shape); ++i) { + size_t t = broadcastIndex(i, mapShape, x.shape); + const float v = read(x, i); + out[t] = n.op == "ReduceProd" ? out[t] * v : std::max(out[t], v); + } + return one(make(x.element, shape, out)); + } + if (x.element != OnnxElement::Int64 && x.element != OnnxElement::Int32) + throw Failure("Integer reduction dtype unsupported"); + std::vector out(count(shape), n.op == "ReduceProd" ? 1 : INT64_MIN); + for (size_t i = 0; i < count(x.shape); ++i) { + auto& a = out[broadcastIndex(i, mapShape, x.shape)]; + const auto b = integer(x, i); + if (n.op == "ReduceMax") + a = std::max(a, b); + else { + if (a && b && + (a > 0 ? (b > 0 ? a > INT64_MAX / b : b < INT64_MIN / a) + : (b > 0 ? a < INT64_MIN / b : a < INT64_MAX / b))) + throw Failure("ReduceProd overflow"); + a *= b; + } + } + if (x.element == OnnxElement::Int64) return one(make(x.element, shape, out)); + std::vector narrow; + for (auto v : out) { + if (v < INT32_MIN || v > INT32_MAX) throw Failure("Int32 reduction overflow"); + narrow.push_back(static_cast(v)); + } + return one(make(x.element, shape, narrow)); + } + return std::nullopt; +} +} // namespace eve::tensor::onnx_detail diff --git a/src/modules/tensor/OnnxInternal.h b/src/modules/tensor/OnnxInternal.h new file mode 100644 index 00000000..241afac0 --- /dev/null +++ b/src/modules/tensor/OnnxInternal.h @@ -0,0 +1,140 @@ +#pragma once +#include +#include +#include +#include +#include +#include "tensor/OnnxByteStorage.h" +#include "tensor/OnnxModel.h" + +namespace eve::tensor::onnx_detail { +struct Failure : std::runtime_error { + DiagnosticCode code; + std::string path; + explicit Failure(std::string message, DiagnosticCode c = DiagnosticCode::InvalidArgument, std::string p = {}) + : std::runtime_error(std::move(message)), code(c), path(std::move(p)) {} +}; +struct ModelData; +struct Attribute { + int type = 0; + int64_t integer = 0; + float real = 0; + std::string text; + std::vector integers; + std::vector reals; + std::vector strings; + std::optional tensor; + std::shared_ptr graph; +}; +struct Node { + std::string name, op, domain; + std::vector inputs, outputs; + std::map attrs; +}; +struct Input { + int type = 0; + std::vector shape; +}; +struct ModelData { + OnnxModelInfo info; + std::vector nodes; + std::unordered_map constants; + std::map inputs; + std::map opsets; +}; +inline size_t elementSize(OnnxElement e) { + switch (e) { + case OnnxElement::Float32: + case OnnxElement::Int32: return 4; + case OnnxElement::Int64: return 8; + case OnnxElement::UInt8: + case OnnxElement::Int8: + case OnnxElement::Bool: return 1; + } + throw Failure("Unsupported ONNX element type", DiagnosticCode::Unsupported); +} +inline size_t count(const std::vector& shape) { + if (shape.size() > 6) throw Failure("ONNX rank exceeds six", DiagnosticCode::Unsupported); + size_t n = 1; + for (int64_t d : shape) { + if (d < 0 || d > INT32_MAX || (d && n > (128u * 1024u * 1024u) / static_cast(d))) + throw Failure("Invalid or excessive tensor shape"); + n *= static_cast(d); + } + return n; +} +inline void validate(const RuntimeTensor& v) { + if (count(v.shape) * elementSize(v.element) != v.bytes.size()) throw Failure("Tensor byte count mismatch"); +} +template +T read(const RuntimeTensor& v, size_t i) { + T x; + std::memcpy(&x, v.bytes.data() + i * sizeof(T), sizeof(T)); + return x; +} +template +RuntimeTensor make(OnnxElement e, std::vector shape, const std::vector& values) { + RuntimeTensor v{e, std::move(shape), std::vector(values.size() * sizeof(T))}; + if (!values.empty()) std::memcpy(v.bytes.data(), values.data(), v.bytes.size()); + validate(v); + return v; +} +inline int64_t integer(const RuntimeTensor& v, size_t i = 0) { + if (i >= count(v.shape)) throw Failure("Integer index out of bounds"); + switch (v.element) { + case OnnxElement::Int64: return read(v, i); + case OnnxElement::Int32: return read(v, i); + case OnnxElement::Int8: return v.bytes[i] >= 128 ? int(v.bytes[i]) - 256 : v.bytes[i]; + case OnnxElement::UInt8: + case OnnxElement::Bool: return v.bytes[i]; + default: throw Failure("Expected integer tensor"); + } +} +inline std::vector floats(const RuntimeTensor& v) { + if (v.element != OnnxElement::Float32) throw Failure("Expected FP32 tensor"); + std::vector out(count(v.shape)); + if (!out.empty()) std::memcpy(out.data(), v.bytes.data(), v.bytes.size()); + return out; +} +inline std::vector ints(const RuntimeTensor& v) { + std::vector out(count(v.shape)); + for (size_t i = 0; i < out.size(); ++i) out[i] = integer(v, i); + return out; +} +inline int64_t attr(const Node& n, const char* key, int64_t fallback) { + auto it = n.attrs.find(key); + return it == n.attrs.end() ? fallback : it->second.integer; +} +inline std::vector attrs(const Node& n, const char* key, std::vector fallback) { + auto it = n.attrs.find(key); + return it == n.attrs.end() ? fallback : it->second.integers; +} +inline const RuntimeTensor& required(const std::vector& in, size_t i) { + if (i >= in.size() || !in[i]) throw Failure("Missing required input"); + return *in[i]; +} +inline int axis(int64_t a, size_t rank) { + if (a < 0) a += rank; + if (a < 0 || a >= static_cast(rank)) throw Failure("Invalid axis"); + return static_cast(a); +} +std::vector broadcast(const std::vector& a, const std::vector& b); +size_t broadcastIndex(size_t i, const std::vector& shape, const std::vector& output); +std::optional> executeIndex(const Node&, const std::vector&); +std::optional executeShape(const Node&, const std::vector&); +RuntimeTensor dispatchFloat(OnnxCompute&, const std::vector&, const std::vector&, + const std::string&, size_t work = 0); +std::optional executeMisc(const Node&, const std::vector&, OnnxCompute*); +std::optional executeNeural(const Node&, const std::vector&, OnnxCompute*); +std::optional executeNumeric(const Node&, const std::vector&, OnnxCompute*); +std::vector evaluate(const ModelData&, std::span, + const std::vector&, OnnxCompute*, OnnxRunOptions options); +ModelData parse(std::span bytes); +bool isSupported(const Node& node); +std::vector execute(const Node& node, const std::vector& inputs, + OnnxCompute* compute = nullptr); +std::vector executeQuant(const Node& node, const std::vector& inputs, + OnnxCompute* compute = nullptr); +std::vector executeQuantLstm(const Node& node, const std::vector& inputs, + OnnxCompute* compute = nullptr); +} // namespace eve::tensor::onnx_detail diff --git a/src/modules/tensor/OnnxLstm.cpp b/src/modules/tensor/OnnxLstm.cpp new file mode 100644 index 00000000..e454e3ab --- /dev/null +++ b/src/modules/tensor/OnnxLstm.cpp @@ -0,0 +1,142 @@ +#include "tensor/AffineQuant.h" +#include "tensor/OnnxGpuKernels.h" +#include "tensor/OnnxInternal.h" + +#include +#include +#include + +namespace eve::tensor::onnx_detail { +namespace { +template +T checked(Result r) { + if (!r.ok()) throw Failure(r.error()->message(), r.error()->code()); + return std::move(r.value()); +} +void shape(const RuntimeTensor& t, const std::vector& expected) { + if (t.shape != expected) throw Failure("LSTM input shape mismatch"); +} +std::vector optionalFloats(const std::vector& in, size_t i, + const std::vector& dims) { + if (i >= in.size() || !in[i]) return std::vector(count(dims), 0); + shape(*in[i], dims); + return floats(*in[i]); +} +struct Weights { + const RuntimeTensor& tensor; + std::vector scales; + std::vector zeros; + size_t directions, gates, rows; + Weights(const RuntimeTensor& t, const RuntimeTensor& s, const RuntimeTensor& z, size_t d, size_t g, size_t r) + : tensor(t), scales(floats(s)), zeros(ints(z)), directions(d), gates(g), rows(r) { + if (t.element != OnnxElement::Int8 && t.element != OnnxElement::UInt8) + throw Failure("LSTM weights must be 8-bit"); + if (t.element != z.element || s.shape != z.shape || + (s.shape != std::vector{static_cast(d)} && + s.shape != std::vector{static_cast(d), static_cast(g)})) + throw Failure("LSTM quantization parameter shape mismatch"); + shape(t, {static_cast(d), static_cast(r), static_cast(g)}); + for (float v : scales) + if (!(v > 0) || !std::isfinite(v)) throw Failure("Invalid LSTM weight scale"); + } + std::vector multiply(const affine::QuantizedActivation& a, size_t batch, size_t direction, + OnnxCompute* compute) const { + std::vector out(batch * gates); + const bool sign = tensor.element == OnnxElement::Int8; + if (compute) { + std::vector z(scales.size() == directions ? 1 : gates); + for (size_t i = 0; i < z.size(); ++i) z[i] = static_cast(zeros[direction * z.size() + i]); + ByteStorage activation(a.values); + ByteStorage accumulator(gpuMatmulResident(*compute, activation.buffer(), tensor.bytes.buffer(), false, sign, + batch, rows, gates, a.zeroPoint, z, direction * rows * gates)); + std::vector sums(batch * gates); + std::memcpy(sums.data(), static_cast(accumulator).data(), + sums.size() * sizeof(int32_t)); + for (size_t b = 0; b < batch; ++b) + for (size_t g = 0; g < gates; ++g) + out[b * gates + g] = static_cast(sums[b * gates + g]) * a.scale * + scales[direction * z.size() + (z.size() == 1 ? 0 : g)]; + return out; + } + // Per-output-channel affine zero points do not require unpacking a weight matrix. + for (size_t b = 0; b < batch; ++b) + for (size_t g = 0; g < gates; ++g) { + const size_t p = scales.size() == directions ? direction : direction * gates + g; + int64_t sum = 0; + for (size_t k = 0; k < rows; ++k) { + const int byte = tensor.bytes[(direction * rows + k) * gates + g]; + const int w = sign && byte >= 128 ? byte - 256 : byte; + sum += static_cast(int(a.values[b * rows + k]) - a.zeroPoint) * (w - zeros[p]); + } + if (sum < INT32_MIN || sum > INT32_MAX) throw Failure("LSTM integer accumulator overflow"); + out[b * gates + g] = static_cast(sum) * a.scale * scales[p]; + } + return out; + } +}; +} // namespace +std::vector executeQuantLstm(const Node& n, const std::vector& in, + OnnxCompute* compute) { + const auto& xt = required(in, 0); + if (xt.shape.size() != 3) throw Failure("LSTM X must be [time,batch,input]"); + const auto x = floats(xt); + const int64_t time = xt.shape[0], batch = xt.shape[1], input = xt.shape[2], hidden = attr(n, "hidden_size", 0); + const auto direction = n.attrs.contains("direction") ? n.attrs.at("direction").text : "forward"; + const int64_t dirs = direction == "bidirectional" ? 2 : 1; + if (time <= 0 || batch <= 0 || input <= 0 || hidden <= 0 || hidden > 16384) + throw Failure("Invalid LSTM dimensions"); + const size_t gates = 4 * hidden; + const size_t projectionSize = count({time, batch, static_cast(gates)}); + if (projectionSize > 128u * 1024u * 1024u) throw Failure("LSTM projection exceeds memory limit"); + Weights w(required(in, 1), required(in, 8), required(in, 9), dirs, gates, input); + Weights r(required(in, 2), required(in, 10), required(in, 11), dirs, gates, hidden); + auto biases = optionalFloats(in, 3, {dirs, 8 * hidden}); + auto h = optionalFloats(in, 5, {dirs, batch, hidden}), c = optionalFloats(in, 6, {dirs, batch, hidden}); + auto p = optionalFloats(in, 7, {dirs, 3 * hidden}); + std::vector lengths(batch, time); + if (in.size() > 4 && in[4]) { + if (in[4]->element != OnnxElement::Int32) throw Failure("LSTM sequence lengths require int32"); + shape(*in[4], {batch}); + lengths = ints(*in[4]); + } + for (auto len : lengths) + if (len < 0 || len > time) throw Failure("Invalid LSTM sequence length"); + const int64_t coupled = attr(n, "input_forget", 0); + if (coupled != 0 && coupled != 1) throw Failure("Invalid coupled gate flag"); + const float clip = n.attrs.contains("clip") ? n.attrs.at("clip").real : std::numeric_limits::max(); + if (!(clip > 0) || !std::isfinite(clip)) throw Failure("Invalid LSTM clipping threshold"); + auto sigmoid = [clip](float z) { return 1.f / (1.f + std::exp(-std::clamp(z, -clip, clip))); }; + auto tanh = [clip](float z) { return std::tanh(std::clamp(z, -clip, clip)); }; + std::vector y(count({time, dirs, batch, hidden}), 0); + const auto qx = checked(affine::dynamicQuantize(x)); + for (int64_t d = 0; d < dirs; ++d) { + const auto projected = w.multiply(qx, time * batch, d, compute); + const bool reverse = direction == "reverse" || d == 1; + for (int64_t step = 0; step < time; ++step) { + const auto qh = checked(affine::dynamicQuantize(std::span(h).subspan(d * batch * hidden, batch * hidden))); + const auto recurrent = r.multiply(qh, batch, d, compute); + for (int64_t b = 0; b < batch; ++b) { + if (step >= lengths[b]) continue; + const int64_t t = reverse ? lengths[b] - 1 - step : step; + const size_t state = (d * batch + b) * hidden; + for (int64_t j = 0; j < hidden; ++j) { + auto gate = [&](int g) { + const size_t channel = g * hidden + j; + return projected[(t * batch + b) * gates + channel] + recurrent[b * gates + channel] + + biases[d * 8 * hidden + channel] + biases[d * 8 * hidden + gates + channel]; + }; + const float i = sigmoid(gate(0) + p[d * 3 * hidden + j] * c[state + j]); + const float f = + coupled ? 1 - i : sigmoid(gate(2) + p[d * 3 * hidden + 2 * hidden + j] * c[state + j]); + c[state + j] = f * c[state + j] + i * tanh(gate(3)); + const float o = sigmoid(gate(1) + p[d * 3 * hidden + hidden + j] * c[state + j]); + h[state + j] = o * tanh(c[state + j]); + y[((t * dirs + d) * batch + b) * hidden + j] = h[state + j]; + } + } + } + } + return {make(OnnxElement::Float32, {time, dirs, batch, hidden}, y), + make(OnnxElement::Float32, {dirs, batch, hidden}, h), make(OnnxElement::Float32, {dirs, batch, hidden}, c)}; +} +} // namespace eve::tensor::onnx_detail diff --git a/src/modules/tensor/OnnxMisc.cpp b/src/modules/tensor/OnnxMisc.cpp new file mode 100644 index 00000000..438c37ac --- /dev/null +++ b/src/modules/tensor/OnnxMisc.cpp @@ -0,0 +1,107 @@ +#include +#include +#include +#include "tensor/OnnxInternal.h" + +namespace eve::tensor::onnx_detail { +std::optional executeMisc(const Node& n, const std::vector& in, + OnnxCompute* compute) { + const auto& x = required(in, 0); + if (n.op == "Clip") { + if (x.element != OnnxElement::Float32) + throw Failure("Clip currently requires FP32", DiagnosticCode::Unsupported); + auto low = make(x.element, {}, std::vector{-std::numeric_limits::infinity()}), + high = make(x.element, {}, std::vector{std::numeric_limits::infinity()}); + const auto& a = in.size() > 1 && in[1] ? *in[1] : low; + const auto& b = in.size() > 2 && in[2] ? *in[2] : high; + if (a.element != x.element || b.element != x.element || count(a.shape) != 1 || count(b.shape) != 1) + throw Failure("Clip bounds must be matching scalars"); + if (compute) return dispatchFloat(*compute, {&x, &a, &b}, x.shape, "y[i]=min(max(x0[i],x1[0]),x2[0]);"); + auto v = floats(x); + for (auto& f : v) f = std::min(std::max(f, read(a, 0)), read(b, 0)); + return make(x.element, x.shape, v); + } + if (n.op == "CumSum") { + const auto& at = required(in, 1); + if (count(at.shape) != 1) throw Failure("CumSum axis must be scalar"); + const int a = axis(integer(at), x.shape.size()); + if (x.element != OnnxElement::Float32) + throw Failure("CumSum currently requires FP32", DiagnosticCode::Unsupported); + const bool reverse = attr(n, "reverse", 0) != 0, exclusive = attr(n, "exclusive", 0) != 0; + size_t outer = 1, inner = 1; + for (int j = 0; j < a; ++j) outer *= x.shape[j]; + for (size_t j = a + 1; j < x.shape.size(); ++j) inner *= x.shape[j]; + const auto width = x.shape[a]; + if (compute && outer && inner && width) { + std::ostringstream s; + s << "uint base=i/" << inner << "u*" << inner * width << "u+i%" << inner + << "u;precise float sum=0.0;for(uint j=0;j<" << width << "u;++j){uint index=base+" + << (reverse ? "(" + std::to_string(width - 1) + "u-j)" : "j") << "*" << inner << "u;" + << (exclusive ? "y[index]=sum;sum+=x0[index];" : "sum+=x0[index];y[index]=sum;") << "}"; + return dispatchFloat(*compute, {&x}, x.shape, s.str(), outer * inner); + } + auto out = floats(x); + for (size_t o = 0; o < outer; ++o) + for (size_t i = 0; i < inner; ++i) { + float sum = 0; + for (int64_t j = 0; j < width; ++j) { + size_t p = (o * width + (reverse ? width - 1 - j : j)) * inner + i; + const float v = out[p]; + if (exclusive) { + out[p] = sum; + sum += v; + } else { + sum += v; + out[p] = sum; + } + } + } + return make(x.element, x.shape, out); + } + if (n.op == "Pad") { + const auto pads = ints(required(in, 1)); + if (pads.size() != 2 * x.shape.size()) throw Failure("Pad rank mismatch"); + auto shape = x.shape; + const auto mode = n.attrs.contains("mode") ? n.attrs.at("mode").text : "constant"; + for (size_t j = 0; j < shape.size(); ++j) { + if (pads[j] < -INT32_MAX || pads[j] > INT32_MAX || pads[j + shape.size()] < -INT32_MAX || + pads[j + shape.size()] > INT32_MAX) + throw Failure("Pad extent out of range"); + shape[j] += pads[j] + pads[j + shape.size()]; + if (mode != "constant" && x.shape[j] <= 0) + throw Failure("Edge/reflect padding requires nonempty dimensions"); + if (mode == "reflect" && (pads[j] >= x.shape[j] || pads[j + shape.size()] >= x.shape[j])) + throw Failure("Reflect padding exceeds input dimension"); + } + const auto size = elementSize(x.element); + std::vector fill(size, 0); + if (in.size() > 2 && in[2]) { + if (in[2]->element != x.element || count(in[2]->shape) != 1) + throw Failure("Pad value dtype/shape mismatch"); + fill = in[2]->bytes; + } + RuntimeTensor out{x.element, shape, std::vector(count(shape) * size)}; + for (size_t i = 0; i < count(shape); ++i) { + size_t rest = i, source = 0, stride = 1; + bool outside = false; + for (size_t j = shape.size(); j > 0; --j) { + int64_t p = static_cast(rest % shape[j - 1]) - pads[j - 1]; + rest /= shape[j - 1]; + if (p < 0 || p >= x.shape[j - 1]) { + if (mode == "constant") + outside = true; + else if (mode == "edge") + p = std::clamp(p, int64_t(0), x.shape[j - 1] - 1); + else + p = p < 0 ? -p : 2 * x.shape[j - 1] - 2 - p; + } + if (!outside) source += p * stride; + stride *= x.shape[j - 1]; + } + std::memcpy(out.bytes.data() + i * size, outside ? fill.data() : x.bytes.data() + source * size, size); + } + return out; + } + return std::nullopt; +} +} // namespace eve::tensor::onnx_detail diff --git a/src/modules/tensor/OnnxModel.cpp b/src/modules/tensor/OnnxModel.cpp new file mode 100644 index 00000000..f8646f53 --- /dev/null +++ b/src/modules/tensor/OnnxModel.cpp @@ -0,0 +1,94 @@ +#include "tensor/OnnxCompute.h" +#include "tensor/OnnxInternal.h" + +#include + +namespace eve::tensor { +struct OnnxModel::Impl { + onnx_detail::ModelData model; +}; +OnnxModel::OnnxModel(std::unique_ptr impl) : impl_(std::move(impl)) {} +OnnxModel::~OnnxModel() = default; +OnnxModelInfo OnnxModel::info() const { return impl_->model.info; } +Result> OnnxModel::load(std::span bytes) { + try { + auto impl = std::make_unique(); + impl->model = onnx_detail::parse(bytes); + return Result>::success(std::unique_ptr(new OnnxModel(std::move(impl)))); + } catch (const onnx_detail::Failure& e) { + return Result>::failure( + Diagnostic::error(e.code, e.what(), {}, {}, "tensor.onnx.import")); + } catch (const std::exception& e) { + return Result>::failure( + Diagnostic::error(DiagnosticCode::Failed, e.what(), {}, {}, "tensor.onnx.import")); + } +} +Result> OnnxModel::run(std::span feeds, + std::span requested, + OnnxRunOptions options) const { + return runInternal(feeds, requested, nullptr, options); +} +Result OnnxModel::runGpu(std::span feeds, OnnxCompute& compute, + std::span requested, OnnxRunOptions options) const { + class Counter final : public OnnxCompute { + public: + OnnxCompute& target; + size_t count = 0; + explicit Counter(OnnxCompute& t) : target(t) {} + Result> dispatch(const OnnxKernel& k) override { + auto r = target.dispatch(k); + if (r.ok()) ++count; + return r; + } + Result enqueue(const std::string& source, std::span inputs, size_t bytes, + uint32_t work) override { + auto r = target.enqueue(source, inputs, bytes, work); + if (r.ok()) ++count; + return r; + } + } counter(compute); + compute.beginRun(); + struct RunScope { + OnnxCompute& device; + ~RunScope() { device.endRun(); } + } scope{compute}; + auto r = runInternal(feeds, requested, &counter, options); + if (!r.ok()) return Result::failure(r.status()); + return Result::success({std::move(r.value()), counter.count, compute.transferStats()}); +} +Result> OnnxModel::runInternal(std::span feeds, + std::span requested, + OnnxCompute* compute, OnnxRunOptions options) const { + using namespace onnx_detail; + std::string nodeName; + try { + const auto& model = impl_->model; + const std::vector targets = + requested.empty() ? model.info.outputs : std::vector(requested.begin(), requested.end()); + std::set seenFeeds; + for (const auto& f : feeds) { + nodeName = f.name; + if (count(f.tensor.shape) * elementSize(f.tensor.element) != f.tensor.bytes.size()) + throw Failure("Tensor byte count mismatch"); + auto input = model.inputs.find(f.name); + if (input == model.inputs.end() || !seenFeeds.insert(f.name).second) + throw Failure("Unknown or duplicate feed"); + if (static_cast(f.tensor.element) != input->second.type || + f.tensor.shape.size() != input->second.shape.size()) + throw Failure("Feed type/rank mismatch"); + for (size_t i = 0; i < f.tensor.shape.size(); ++i) + if (input->second.shape[i] >= 0 && input->second.shape[i] != f.tensor.shape[i]) + throw Failure("Feed dimension mismatch"); + } + nodeName.clear(); + auto result = evaluate(model, feeds, targets, compute, options); + return Result>::success(std::move(result)); + } catch (const Failure& e) { + return Result>::failure( + Diagnostic::error(e.code, e.what(), e.path.empty() ? nodeName : e.path, {}, "tensor.onnx.run")); + } catch (const std::exception& e) { + return Result>::failure( + Diagnostic::error(DiagnosticCode::Failed, e.what(), nodeName, {}, "tensor.onnx.run")); + } +} +} // namespace eve::tensor diff --git a/src/modules/tensor/OnnxModel.h b/src/modules/tensor/OnnxModel.h new file mode 100644 index 00000000..39b1664d --- /dev/null +++ b/src/modules/tensor/OnnxModel.h @@ -0,0 +1,106 @@ +#pragma once + +#include "common/Result.h" +#include "tensor/OnnxStorage.h" + +#include +#include +#include +#include +#include +#include + +namespace eve::tensor { +class OnnxCompute; +/** @brief ONNX wire element types; distinct from block-quantized Tensor storage. */ +enum class OnnxElement : int { Float32 = 1, UInt8 = 2, Int8 = 3, Int32 = 6, Int64 = 7, Bool = 9 }; + +/** + * @brief Owning ONNX boundary tensor: row-major little-endian bytes and exact integer shape. + * @note No borrowed storage. run() validates caller-provided values before execution. + * Scalar shape is empty; zero-sized dimensions are supported. No implicit FP32 integer conversion. + */ +struct OnnxTensor { + OnnxElement element = OnnxElement::Float32; + std::vector shape; + std::vector bytes; +}; +/** @brief Owning named feed/output value. */ +struct OnnxNamedTensor { + std::string name; + OnnxTensor tensor; +}; +/** @brief Per-call deterministic RNG and optional strict finite-output diagnostic. + * @note requireFinite rejects even intentional intermediate infinities/NaNs, so keep it false for Kokoro. */ +struct OnnxRunOptions { + uint64_t seed = 0; + bool requireFinite = false; +}; +/** @brief Owning GPU execution output and completed dispatch count. */ +struct OnnxGpuResult { + std::vector outputs; + size_t dispatches = 0; + OnnxTransferStats transfers; +}; +/** @brief Owning admission report; unsupported nodes remain inspectable but cannot execute. */ +struct OnnxModelInfo { + int64_t irVersion = 0; + size_t nodeCount = 0, initializerBytes = 0; + std::vector inputs, outputs; + std::vector unsupportedNodes; +}; + +/** + * @brief Native ONNX import and CPU/GPU execution using tensor kernels, without ONNX Runtime. + * @note Owns immutable model data. Concurrent CPU run calls are independent; no script callbacks, + * global RNG or background threads. runGpu uses a caller-provided device on its owning thread. + * FP32 results use tolerance comparison. Seeded random excitation is local to a run. + * Loading is transactional. Unknown protobuf metadata is ignored, unknown operators + * are reported; unsupported semantic features fail explicitly, never become identity. + */ +class OnnxModel { +public: + /** @brief Destroy model-owned graph and packed initializers; outputs remain valid. */ + ~OnnxModel(); + /** + * @brief Import bounded in-memory ONNX ModelProto; copies all retained data. + * @param bytes Borrowed input, not retained. Maximum 512 MiB, 100000 recursive nodes, graph depth 16, rank 6. + * @return Owning model or ParseError/Unsupported/UnknownVersion; no partial publication. + * @note Supports ONNX IR 3..10 and default-domain opset 13..17. External data is rejected. + */ + [[nodiscard]] static Result> load(std::span bytes); + /** @brief Return owning graph diagnostics; importing does not imply full operator support. */ + [[nodiscard]] OnnxModelInfo info() const; + /** + * @brief Execute selected named values, or graph outputs when requested is empty. + * @param feeds Borrowed inputs, valid only during this call; never mutated or retained. + * @param requested Borrowed output names; intermediate outputs may be selected for parity tests. + * @return Owning outputs or structured node diagnostic; failed calls publish no outputs. + * @note CPU only. Unsupported dependencies fail before executing the selected subgraph. + */ + [[nodiscard]] Result> run(std::span feeds, + std::span requested = {}, + OnnxRunOptions options = {}) const; + + /** + * @brief Execute with GPU matrix, convolution, normalization, activation and resampling kernels. + * @note Shape/index/control operations, padding and LSTM gates execute on CPU. + * Dynamic quantization reads three validation scalars; activations remain on GPU. + * Integer GEMM/Conv, LSTM projections, FP32 GEMM/ConvTranspose, elementwise math, + * Softmax, sum/mean reductions, normalization, Resize and CumSum execute on GPU. + * The borrowed compute provider must outlive this synchronous call; use its device thread. + * No automatic CPU retry occurs after a GPU error. Outputs own their storage. + */ + [[nodiscard]] Result runGpu(std::span feeds, OnnxCompute& compute, + std::span requested = {}, + OnnxRunOptions options = {}) const; + +private: + [[nodiscard]] Result> runInternal(std::span feeds, + std::span requested, + OnnxCompute* compute, OnnxRunOptions options) const; + struct Impl; + explicit OnnxModel(std::unique_ptr impl); + std::unique_ptr impl_; +}; +} // namespace eve::tensor diff --git a/src/modules/tensor/OnnxNeural.cpp b/src/modules/tensor/OnnxNeural.cpp new file mode 100644 index 00000000..0590a713 --- /dev/null +++ b/src/modules/tensor/OnnxNeural.cpp @@ -0,0 +1,190 @@ +#include +#include +#include +#include "tensor/OnnxInternal.h" + +namespace eve::tensor::onnx_detail { +std::optional executeNeural(const Node& n, const std::vector& in, + OnnxCompute* compute) { + const auto& x = required(in, 0); + if (x.element != OnnxElement::Float32) return std::nullopt; + if (n.op == "InstanceNormalization") { + const auto& scale = required(in, 1); + const auto& bias = required(in, 2); + if (x.shape.size() < 3 || scale.shape != std::vector{x.shape[1]} || bias.shape != scale.shape || + scale.element != x.element || bias.element != x.element) + throw Failure("InstanceNormalization shape/type mismatch"); + const float epsilon = n.attrs.contains("epsilon") ? n.attrs.at("epsilon").real : 1e-5f; + if (!(epsilon > 0) || !std::isfinite(epsilon)) throw Failure("Invalid normalization epsilon"); + size_t spatial = 1; + for (size_t j = 2; j < x.shape.size(); ++j) spatial *= x.shape[j]; + if (!spatial) throw Failure("Empty InstanceNormalization"); + if (compute) { + std::ostringstream s; + s.precision(9); + s << std::scientific; + s << "uint base=i*" << spatial << "u;precise float mean=0.0;for(uint j=0;j<" << spatial + << "u;++j)mean+=x0[base+j];mean/=" << spatial << ".0;precise float var=0.0;for(uint j=0;j<" << spatial + << "u;++j){float d=x0[base+j]-mean;var+=d*d;}float inv=inversesqrt(var/" << spatial << ".0+" << epsilon + << ");for(uint j=0;j<" << spatial << "u;++j)y[base+j]=(x0[base+j]-mean)*inv*x1[i%" << x.shape[1] + << "u]+x2[i%" << x.shape[1] << "u];"; + return dispatchFloat(*compute, {&x, &scale, &bias}, x.shape, s.str(), count(x.shape) / spatial); + } + auto out = floats(x); + for (size_t i = 0; i < out.size() / spatial; ++i) { + float mean = 0, var = 0; + for (size_t j = 0; j < spatial; ++j) mean += out[i * spatial + j]; + mean /= spatial; + for (size_t j = 0; j < spatial; ++j) { + float d = out[i * spatial + j] - mean; + var += d * d; + } + const float multiplier = read(scale, i % x.shape[1]) / std::sqrt(var / spatial + epsilon), + offset = read(bias, i % x.shape[1]); + for (size_t j = 0; j < spatial; ++j) + out[i * spatial + j] = (out[i * spatial + j] - mean) * multiplier + offset; + } + return make(x.element, x.shape, out); + } + if (n.op == "ConvTranspose") { + const auto& w = required(in, 1); + if (x.shape.size() != 3 || w.shape.size() != 3 || w.element != x.element || w.shape[0] != x.shape[1]) + throw Failure("ConvTranspose requires NCL / CIO weights", DiagnosticCode::Unsupported); + const auto strides = attrs(n, "strides", {1}), pads = attrs(n, "pads", {0, 0}), + dilations = attrs(n, "dilations", {1}), extra = attrs(n, "output_padding", {0}); + if (strides.size() != 1 || pads.size() != 2 || dilations.size() != 1 || extra.size() != 1) + throw Failure("ConvTranspose attribute rank mismatch"); + const int64_t stride = strides[0], dilation = dilations[0], groups = attr(n, "group", 1), channels = x.shape[1], + length = x.shape[2], kernel = w.shape[2], cout = w.shape[1]; + if (stride <= 0 || stride > 65536 || dilation <= 0 || dilation > 65536 || groups <= 0 || channels % groups || + pads[0] < 0 || pads[1] < 0 || extra[0] < 0 || extra[0] >= std::max(stride, dilation) || length <= 0 || + kernel <= 0 || cout <= 0) + throw Failure("Invalid ConvTranspose geometry"); + if (n.attrs.contains("kernel_shape") && attrs(n, "kernel_shape", {}) != std::vector{kernel}) + throw Failure("ConvTranspose kernel_shape mismatch"); + const int64_t width = stride * (length - 1) + extra[0] + dilation * (kernel - 1) + 1 - pads[0] - pads[1], + outputs = cout * groups; + if (width <= 0 || width > INT32_MAX) throw Failure("Invalid ConvTranspose output extent"); + std::vector shape{x.shape[0], outputs, width}; + count(shape); + RuntimeTensor zeroBias = make(OnnxElement::Float32, {outputs}, std::vector(outputs, 0)); + const auto& bias = in.size() > 2 && in[2] ? *in[2] : zeroBias; + if (bias.element != x.element || bias.shape != zeroBias.shape) throw Failure("ConvTranspose bias mismatch"); + if (compute) { + std::ostringstream s; + s << "uint pos=i%" << width << "u,o=i/" << width << "u%" << outputs << "u,batch=i/" << width * outputs + << "u;precise float v=x2[o];for(uint c=0;c<" << channels / groups << "u;++c)for(uint j=0;j<" << kernel + << "u;++j){int z=int(pos)+" << pads[0] << "-int(j)*" << dilation << ";if(z<0||z%" << stride << "!=0||z/" + << stride << ">=" << length << ")continue;uint ch=o/" << cout << "u*" << channels / groups + << "u+c;v+=x0[(batch*" << channels << "u+ch)*" << length << "u+uint(z/" << stride << ")]*x1[(ch*" << cout + << "u+o%" << cout << "u)*" << kernel << "u+j];}y[i]=v;"; + return dispatchFloat(*compute, {&x, &w, &bias}, shape, s.str()); + } + std::vector out(count(shape)); + for (int64_t batch = 0; batch < x.shape[0]; ++batch) + for (int64_t o = 0; o < outputs; ++o) + for (int64_t pos = 0; pos < width; ++pos) { + float v = read(bias, o); + for (int64_t c = 0; c < channels / groups; ++c) + for (int64_t j = 0; j < kernel; ++j) { + int64_t z = pos + pads[0] - j * dilation; + if (z < 0 || z % stride || z / stride >= length) continue; + int64_t ch = o / cout * (channels / groups) + c; + v += read(x, (batch * channels + ch) * length + z / stride) * + read(w, (ch * cout + o % cout) * kernel + j); + } + out[(batch * outputs + o) * width + pos] = v; + } + return make(x.element, shape, out); + } + if (n.op == "Resize") { + if (in.size() > 1 && in[1] && !in[1]->bytes.empty()) + throw Failure("Resize ROI unsupported", DiagnosticCode::Unsupported); + std::vector scales; + auto shape = x.shape; + if (in.size() > 2 && in[2] && !in[2]->bytes.empty()) { + auto s = floats(*in[2]); + scales.assign(s.begin(), s.end()); + if (scales.size() != shape.size()) throw Failure("Resize scales rank mismatch"); + for (size_t j = 0; j < shape.size(); ++j) { + if (!(scales[j] > 0) || !std::isfinite(scales[j]) || std::floor(shape[j] * scales[j]) > INT32_MAX) + throw Failure("Invalid Resize scale"); + shape[j] = static_cast(std::floor(shape[j] * scales[j])); + } + } else { + shape = ints(required(in, 3)); + if (shape.size() != x.shape.size()) throw Failure("Resize sizes rank mismatch"); + for (size_t j = 0; j < shape.size(); ++j) { + if (x.shape[j] <= 0) throw Failure("Empty Resize input"); + scales.push_back(double(shape[j]) / x.shape[j]); + } + } + count(shape); + const auto mode = n.attrs.contains("mode") ? n.attrs.at("mode").text : "nearest", + coordinate = n.attrs.contains("coordinate_transformation_mode") + ? n.attrs.at("coordinate_transformation_mode").text + : "half_pixel"; + for (auto d : x.shape) + if (d <= 0) throw Failure("Empty Resize input"); + if (mode == "nearest") { + if (compute) { + std::ostringstream s; + s.precision(17); + s << "uint rest=i,source=0u;"; + size_t stride = 1; + for (size_t j = shape.size(); j > 0; --j) { + s << "source+=uint(min(floor(float(rest%" << shape[j - 1] << "u)/" << std::scientific + << scales[j - 1] << ")," << double(x.shape[j - 1] - 1) << "))*" << stride + << "u;rest/=" << shape[j - 1] << "u;"; + stride *= x.shape[j - 1]; + } + s << "y[i]=x0[source];"; + return dispatchFloat(*compute, {&x}, shape, s.str()); + } + std::vector out(count(shape)); + for (size_t i = 0; i < out.size(); ++i) { + size_t rest = i, source = 0, stride = 1; + for (size_t j = shape.size(); j > 0; --j) { + source += std::min(int64_t(std::floor((rest % shape[j - 1]) / scales[j - 1])), x.shape[j - 1] - 1) * + stride; + rest /= shape[j - 1]; + stride *= x.shape[j - 1]; + } + out[i] = read(x, source); + } + return make(x.element, shape, out); + } + // The speech model interpolates its final axis only. Reject other geometries explicitly. + for (size_t j = 0; j + 1 < shape.size(); ++j) + if (shape[j] != x.shape[j]) + throw Failure("Linear Resize supports final axis only", DiagnosticCode::Unsupported); + if (shape.empty()) throw Failure("Resize scalar unsupported"); + const auto width = shape.back(), old = x.shape.back(); + if (!width) return RuntimeTensor{x.element, shape, {}}; + auto position = [&](int64_t p) { + return coordinate == "half_pixel" ? (p + .5) / scales.back() - .5 : p / scales.back(); + }; + if (compute) { + std::ostringstream s; + s.precision(17); + s << std::scientific; + s << "uint base=i/" << width << "u*" << old + << "u;float p=" << (coordinate == "half_pixel" ? "(float(i%" : "float(i%") << width + << (coordinate == "half_pixel" ? "u)+0.5)/" : "u)/") << scales.back() + << (coordinate == "half_pixel" ? "-0.5" : "") << ";float t=floor(p),f=p-t;uint a=uint(clamp(t,0.0," + << double(old - 1) << ")),b=uint(clamp(t+1.0,0.0," << double(old - 1) + << "));y[i]=x0[base+a]*(1.0-f)+x0[base+b]*f;"; + return dispatchFloat(*compute, {&x}, shape, s.str()); + } + std::vector out(count(shape)); + for (size_t i = 0; i < out.size(); ++i) { + double p = position(i % width), lo = std::floor(p), f = p - lo; + size_t base = i / width * old; + out[i] = static_cast(read(x, base + std::clamp(int64_t(lo), int64_t(0), old - 1)) * (1 - f) + + read(x, base + std::clamp(int64_t(lo + 1), int64_t(0), old - 1)) * f); + } + return make(x.element, shape, out); + } + return std::nullopt; +} +} // namespace eve::tensor::onnx_detail diff --git a/src/modules/tensor/OnnxNumeric.cpp b/src/modules/tensor/OnnxNumeric.cpp new file mode 100644 index 00000000..79b47554 --- /dev/null +++ b/src/modules/tensor/OnnxNumeric.cpp @@ -0,0 +1,257 @@ +#include +#include +#include +#include +#include +#include "tensor/OnnxCompute.h" +#include "tensor/OnnxInternal.h" + +namespace eve::tensor::onnx_detail { +namespace { +std::string literal(float v) { + std::ostringstream s; + s.precision(9); + s << std::scientific << v; + return s.str(); +} +std::string indexCode(const std::vector& input, const std::vector& output, const char* index) { + std::string result = "0u"; + size_t src = 1, dst = 1; + for (size_t j = 0; j < output.size(); ++j) { + if (j < input.size()) { + const auto dim = input[input.size() - 1 - j]; + if (dim != 1) + result += "+(" + std::string(index) + "/" + std::to_string(dst) + "u%" + std::to_string(dim) + "u)*" + + std::to_string(src) + "u"; + src *= dim; + } + dst *= output[output.size() - 1 - j]; + } + return result; +} +} // namespace +RuntimeTensor dispatchFloat(OnnxCompute& compute, const std::vector& inputs, + const std::vector& shape, const std::string& body, size_t work) { + RuntimeTensor out{OnnxElement::Float32, shape, std::vector(count(shape) * 4)}; + if (out.bytes.empty()) return out; + if (!work) work = count(shape); + if (work > 65535u * 64u) throw Failure("GPU float dispatch limit exceeded", DiagnosticCode::Unsupported); + OnnxKernel k; + std::vector buffers; + k.outputBytes = out.bytes.size(); + k.workItems = static_cast(work); + k.source = "#version 450\nlayout(local_size_x=64)in;\n"; + for (size_t i = 0; i < inputs.size(); ++i) { + if (inputs[i]->element != OnnxElement::Float32 && inputs[i]->element != OnnxElement::Int32) + throw Failure("GPU float input type mismatch"); + buffers.push_back(inputs[i]->bytes.buffer()); + k.source += "layout(std430,binding=" + std::to_string(i) + ")readonly buffer B" + std::to_string(i) + + (inputs[i]->element == OnnxElement::Int32 ? "{int x" : "{float x") + std::to_string(i) + "[];};\n"; + } + k.source += + "layout(std430,binding=" + std::to_string(inputs.size()) + + ")writeonly buffer Y{float y[];};\nvoid main(){uint i=gl_GlobalInvocationID.x;if(i>=" + std::to_string(work) + + "u)return;" + body + "}"; + auto r = compute.enqueue(k.source, buffers, k.outputBytes, k.workItems); + if (!r.ok()) throw Failure(r.error()->message(), r.error()->code()); + if (r.value().size != out.bytes.size()) throw Failure("GPU output size mismatch"); + out.bytes = std::move(r.value()); + return out; +} +std::optional executeNumeric(const Node& n, const std::vector& in, + OnnxCompute* compute) { + const auto& x = required(in, 0); + if (x.element != OnnxElement::Float32) return std::nullopt; + if (n.op == "MatMul") { + const auto& b = required(in, 1); + if (b.element != x.element || x.shape.empty() || b.shape.empty()) throw Failure("MatMul type/rank mismatch"); + auto as = x.shape, bs = b.shape; + const bool av = as.size() == 1, bv = bs.size() == 1; + if (av) as.insert(as.begin(), 1); + if (bv) bs.push_back(1); + const auto m = as[as.size() - 2], k = as.back(), cols = bs.back(); + if (k != bs[bs.size() - 2]) throw Failure("MatMul inner dimension mismatch"); + std::vector ab(as.begin(), as.end() - 2), bb(bs.begin(), bs.end() - 2); + auto batches = broadcast(ab, bb); + auto shape = batches; + shape.push_back(m); + shape.push_back(cols); + count(shape); + if (av) shape.erase(shape.end() - 2); + if (bv) shape.pop_back(); + if (compute && m && cols) { + std::string body = "uint batch=i/" + std::to_string(m * cols) + "u,r=i/" + std::to_string(cols) + "u%" + + std::to_string(m) + "u,c=i%" + std::to_string(cols) + + "u;precise float v=0.0;for(uint j=0;j<" + std::to_string(k) + "u;++j)v+=x0[(" + + indexCode(ab, batches, "batch") + ")*" + std::to_string(m * k) + "u+r*" + + std::to_string(k) + "u+j]*x1[(" + indexCode(bb, batches, "batch") + ")*" + + std::to_string(k * cols) + "u+j*" + std::to_string(cols) + "u+c];y[i]=v;"; + return dispatchFloat(*compute, {&x, &b}, shape, body); + } + std::vector out(count(shape), 0); + for (size_t batch = 0; batch < count(batches); ++batch) + for (int64_t r = 0; r < m; ++r) + for (int64_t c = 0; c < cols; ++c) { + float sum = 0; + for (int64_t j = 0; j < k; ++j) + sum += read(x, broadcastIndex(batch, ab, batches) * m * k + r * k + j) * + read(b, broadcastIndex(batch, bb, batches) * k * cols + j * cols + c); + out[(batch * m + r) * cols + c] = sum; + } + return make(OnnxElement::Float32, shape, out); + } + static const std::set binary{"Add", "Sub", "Mul", "Div", "Pow"}; + if (binary.contains(n.op)) { + const auto& b = required(in, 1); + if (b.element != x.element) throw Failure("Float arithmetic dtype mismatch"); + auto shape = broadcast(x.shape, b.shape); + if (compute) { + const std::string a = "x0[" + indexCode(x.shape, shape, "i") + "]", + v = "x1[" + indexCode(b.shape, shape, "i") + "]"; + const std::string e = n.op == "Pow" ? "pow(" + a + "," + v + ")" + : a + + (n.op == "Add" ? "+" + : n.op == "Sub" ? "-" + : n.op == "Mul" ? "*" + : "/") + + v; + if (n.op == "Pow") + return dispatchFloat(*compute, {&x, &b}, shape, + "float a=" + a + ",b=" + v + + ";float value;if(b==0.0)value=1.0;else " + "if(a==0.0)value=b>0.0?0.0:uintBitsToFloat(0x7f800000u);else " + "if(a<0.0)value=b==floor(b)?pow(-a,b)*(mod(abs(b),2.0)==0.0?1.0:-1.0):" + "uintBitsToFloat(0x7fc00000u);else value=pow(a,b);y[i]=value;"); + return dispatchFloat(*compute, {&x, &b}, shape, "y[i]=" + e + ";"); + } + std::vector out(count(shape)); + for (size_t i = 0; i < out.size(); ++i) { + const float a = read(x, broadcastIndex(i, x.shape, shape)), + v = read(b, broadcastIndex(i, b.shape, shape)); + out[i] = n.op == "Add" ? a + v + : n.op == "Sub" ? a - v + : n.op == "Mul" ? a * v + : n.op == "Div" ? a / v + : std::pow(a, v); + } + return make(OnnxElement::Float32, shape, out); + } + if (n.op == "ReduceMean" || n.op == "ReduceSum") { + auto axes = n.op == "ReduceMean" ? attrs(n, "axes", {}) + : in.size() > 1 && in[1] ? ints(*in[1]) + : std::vector{}; + if (axes.empty() && n.op == "ReduceSum" && attr(n, "noop_with_empty_axes", 0)) return x; + if (axes.empty()) + for (size_t i = 0; i < x.shape.size(); ++i) axes.push_back(i); + std::set set; + for (auto a : axes) + if (!set.insert(axis(a, x.shape.size())).second) throw Failure("Duplicate reduction axis"); + std::vector outerShape, reducedShape, shape; + std::vector outerStrides, reducedStrides; + size_t stride = count(x.shape); + for (size_t j = 0; j < x.shape.size(); ++j) { + if (!x.shape[j]) throw Failure("Empty reduction unsupported", DiagnosticCode::Unsupported); + stride /= x.shape[j]; + if (set.contains(static_cast(j))) { + reducedShape.push_back(x.shape[j]); + reducedStrides.push_back(stride); + if (attr(n, "keepdims", 1)) shape.push_back(1); + } else { + outerShape.push_back(x.shape[j]); + outerStrides.push_back(stride); + shape.push_back(x.shape[j]); + } + } + auto offset = [](size_t i, const std::vector& dims, const std::vector& strides) { + size_t p = 0; + for (size_t j = dims.size(); j > 0; --j) { + p += (i % dims[j - 1]) * strides[j - 1]; + i /= dims[j - 1]; + } + return p; + }; + auto code = [](const char* var, const std::vector& dims, const std::vector& strides) { + std::string s = "0u"; + size_t divisor = 1; + for (size_t j = dims.size(); j > 0; --j) { + s += "+(" + std::string(var) + "/" + std::to_string(divisor) + "u%" + std::to_string(dims[j - 1]) + + "u)*" + std::to_string(strides[j - 1]) + "u"; + divisor *= dims[j - 1]; + } + return s; + }; + const size_t terms = count(reducedShape); + if (compute) + return dispatchFloat(*compute, {&x}, shape, + "uint base=" + code("i", outerShape, outerStrides) + + ";precise float v=0.0;for(uint j=0;j<" + std::to_string(terms) + + "u;++j)v+=x0[base+(" + code("j", reducedShape, reducedStrides) + ")];y[i]=v" + + (n.op == "ReduceMean" ? "/" + std::to_string(terms) + ".0" : "") + ";"); + std::vector out(count(shape)); + for (size_t i = 0; i < out.size(); ++i) { + float v = 0; + for (size_t j = 0; j < terms; ++j) + v += read(x, offset(i, outerShape, outerStrides) + offset(j, reducedShape, reducedStrides)); + out[i] = n.op == "ReduceMean" ? v / terms : v; + } + return make(OnnxElement::Float32, shape, out); + } + if (n.op == "Softmax" && compute) { + int a = axis(attr(n, "axis", -1), x.shape.size()); + size_t inner = 1, outer = 1; + for (int j = 0; j < a; ++j) outer *= x.shape[j]; + for (size_t j = a + 1; j < x.shape.size(); ++j) inner *= x.shape[j]; + const auto channels = x.shape[a]; + if (!channels || !inner) return RuntimeTensor{x.element, x.shape, {}}; + auto ch = std::to_string(channels), ins = std::to_string(inner); + return dispatchFloat(*compute, {&x}, x.shape, + "uint base=i/" + ins + "u*" + ins + "u*" + ch + "u+i%" + ins + + "u;float m=-3.402823466e38;for(uint j=0;j<" + ch + "u;++j)m=max(m,x0[base+j*" + ins + + "u]);float sum=0.0;for(uint j=0;j<" + ch + "u;++j)sum+=exp(x0[base+j*" + ins + + "u]-m);for(uint j=0;j<" + ch + "u;++j)y[base+j*" + ins + "u]=exp(x0[base+j*" + ins + + "u]-m)/sum;", + outer * inner); + } + static const std::map unary{{"Sqrt", "sqrt(v)"}, + {"Exp", "exp(v)"}, + {"Log", "log(v)"}, + {"Sin", "sin(v)"}, + {"Cos", "cos(v)"}, + {"Relu", "max(v,0.0)"}, + {"Sigmoid", "1.0/(1.0+exp(-v))"}, + {"Tanh", "isnan(v)?v:tanh(clamp(v,-10.0,10.0))"}, + {"Neg", "-v"}, + {"Abs", "abs(v)"}, + {"Reciprocal", "1.0/v"}, + {"Floor", "floor(v)"}, + {"Round", "roundEven(v)"}, + {"Atan", "isinf(v)?sign(v)*1.5707963267948966:atan(v)"}}; + if (n.op == "LeakyRelu" || unary.contains(n.op)) { + const float alpha = n.attrs.contains("alpha") ? n.attrs.at("alpha").real : 0.01f; + if (!std::isfinite(alpha)) throw Failure("Invalid LeakyRelu alpha"); + if (compute) + return dispatchFloat( + *compute, {&x}, x.shape, + "float v=x0[i];y[i]=" + (n.op == "LeakyRelu" ? "v>=0.0?v:v*" + literal(alpha) : unary.at(n.op)) + ";"); + if (n.op != "LeakyRelu" && n.op != "Reciprocal" && n.op != "Floor" && n.op != "Round" && n.op != "Atan") + return std::nullopt; + auto out = floats(x); + for (auto& v : out) { + if (n.op == "LeakyRelu") + v = v >= 0 ? v : alpha * v; + else if (n.op == "Reciprocal") + v = 1 / v; + else if (n.op == "Floor") + v = std::floor(v); + else if (n.op == "Atan") + v = std::atan(v); + else { + float lo = std::floor(v), f = v - lo; + v = lo + (f > 0.5f || (f == 0.5f && std::fmod(lo, 2.f) != 0)); + } + } + return make(OnnxElement::Float32, x.shape, out); + } + return std::nullopt; +} +} // namespace eve::tensor::onnx_detail diff --git a/src/modules/tensor/OnnxQuant.cpp b/src/modules/tensor/OnnxQuant.cpp new file mode 100644 index 00000000..a015ee2b --- /dev/null +++ b/src/modules/tensor/OnnxQuant.cpp @@ -0,0 +1,196 @@ +#include "tensor/AffineQuant.h" +#include "tensor/AffineQuantInternal.h" +#include "tensor/OnnxGpuKernels.h" +#include "tensor/OnnxInternal.h" + +#include +#include + +namespace eve::tensor::onnx_detail { +namespace { +affine::ByteView bytes(const RuntimeTensor& t) { + if (t.element != OnnxElement::Int8 && t.element != OnnxElement::UInt8) + throw Failure("Expected packed int8/uint8 tensor"); + return {t.bytes, t.element == OnnxElement::Int8}; +} +template +T checked(Result result) { + if (!result.ok()) throw Failure(result.error()->message(), result.error()->code()); + return std::move(result.value()); +} +int zero(const std::vector& in, size_t i, const RuntimeTensor& owner) { + if (i >= in.size() || !in[i]) return 0; + if (in[i]->element != owner.element || count(in[i]->shape) != 1) + throw Failure("Expected matching scalar zero point"); + return static_cast(integer(*in[i])); +} +} // namespace +std::vector executeQuant(const Node& n, const std::vector& in, + OnnxCompute* compute) { + const auto& x = required(in, 0); + if (n.op == "DynamicQuantizeLinear") { + if (compute && count(x.shape)) { + if (x.element != OnnxElement::Float32) throw Failure("Expected FP32 quantization input"); + const size_t total = count(x.shape); + auto stats = dispatchFloat( + *compute, {&x}, {3}, + "float lo=0.0,hi=0.0,invalid=0.0;for(uint j=0;j<" + std::to_string(total) + + "u;++j){float v=x0[j];if(isinf(v))invalid=1.0;if(!isnan(v)){lo=min(lo,v);hi=max(hi,v);}}float " + "s=(hi==lo)?1.0:(hi-lo)/255.0;y[0]=s;y[1]=clamp(roundEven(-lo/s),0.0,255.0);y[2]=invalid;", + 1); + auto values = floats(stats); + if (values[2] != 0 || !std::isfinite(values[0]) || !(values[0] > 0)) + throw Failure("Nonfinite quantization input/range"); + auto scale = make(OnnxElement::Float32, {}, std::vector{values[0]}); + auto zeroPoint = make(OnnxElement::UInt8, {}, std::vector{static_cast(values[1])}); + auto params = make(OnnxElement::Float32, {2}, std::vector{values[0], values[1]}); + std::string source = + "#version 450\nlayout(local_size_x=64)in;layout(std430,binding=0)readonly buffer X{float " + "x[];};layout(std430,binding=1)readonly buffer P{float p[];};layout(std430,binding=2)writeonly buffer " + "Y{uint y[];};void main(){uint word=gl_GlobalInvocationID.x;if(word>=" + + std::to_string((total + 3) / 4) + + "u)return;uint packed=0u;for(uint c=0u;c<4u;++c){uint j=word*4u+c;if(j<" + std::to_string(total) + + "u){float v=x[j];uint " + "q=isnan(v)?0u:uint(clamp(roundEven(v/p[0])+p[1],0.0,255.0));packed|=q<<(c*8u);}}y[word]=packed;}"; + const std::vector inputs{x.bytes.buffer(), params.bytes.buffer()}; + auto q = checked(compute->enqueue(source, inputs, total, static_cast((total + 3) / 4))); + return {{OnnxElement::UInt8, x.shape, std::move(q)}, std::move(scale), std::move(zeroPoint)}; + } + auto input = floats(x); + // ORT CPU DynamicQuantizeLinear ignores NaN for range estimation and stores code 0. + // Exported Kokoro phase graphs produce 0/0 at zero-amplitude STFT bins. + std::vector nanPositions; + for (size_t i = 0; i < input.size(); ++i) + if (std::isnan(input[i])) { + nanPositions.push_back(i); + input[i] = 0; + } + auto q = checked(affine::dynamicQuantize(input)); + for (auto i : nanPositions) q.values[i] = 0; + return {{OnnxElement::UInt8, x.shape, std::move(q.values)}, + make(OnnxElement::Float32, {}, std::vector{q.scale}), + make(OnnxElement::UInt8, {}, std::vector{q.zeroPoint})}; + } + if (n.op == "QuantizeLinear" || n.op == "DequantizeLinear") { + const auto scales = floats(required(in, 1)); + if (required(in, 1).shape.size() > 1) throw Failure("Quantization scale must be scalar or vector"); + if (scales.empty()) throw Failure("Empty quantization scales"); + const RuntimeTensor* zp = in.size() > 2 ? in[2] : nullptr; + const auto type = n.op == "DequantizeLinear" ? x.element : (zp ? zp->element : OnnxElement::UInt8); + if (type != OnnxElement::UInt8 && type != OnnxElement::Int8) + throw Failure("Unsupported quantization dtype", DiagnosticCode::Unsupported); + if (zp && (zp->element != type || zp->shape != required(in, 1).shape)) + throw Failure("Scale/zero point mismatch"); + std::vector zeros(scales.size(), 0); + if (zp) + for (size_t i = 0; i < zeros.size(); ++i) zeros[i] = static_cast(integer(*zp, i)); + size_t inner = 1; + if (scales.size() > 1) { + const int a = axis(attr(n, "axis", 1), x.shape.size()); + if (static_cast(x.shape[a]) != scales.size()) throw Failure("Scale channel count mismatch"); + for (size_t i = a + 1; i < x.shape.size(); ++i) inner *= x.shape[i]; + } + if (n.op == "DequantizeLinear") + return {make(OnnxElement::Float32, x.shape, checked(affine::dequantize(bytes(x), scales, zeros, inner)))}; + const auto values = floats(x); + std::vector out(values.size()); + for (size_t i = 0; i < values.size(); i += inner) { + const size_t channel = (i / inner) % scales.size(); + auto q = checked(affine::quantize(std::span(values).subspan(i, inner), scales[channel], zeros[channel], + type == OnnxElement::Int8)); + std::copy(q.begin(), q.end(), out.begin() + i); + } + return {{type, x.shape, std::move(out)}}; + } + if (n.op == "MatMulInteger") { + const auto& b = required(in, 1); + if (x.shape.size() < 2 || b.shape.size() != 2) + throw Failure("MatMulInteger requires rank >=2 A and rank 2 B", DiagnosticCode::Unsupported); + const size_t k = x.shape.back(), columns = b.shape[1]; + if (k != static_cast(b.shape[0]) || k == 0) throw Failure("Invalid MatMulInteger inner dimension"); + const size_t rows = count(x.shape) / k; + auto shape = x.shape; + shape.back() = columns; + if (compute) { + if ((x.element != OnnxElement::Int8 && x.element != OnnxElement::UInt8) || + (b.element != OnnxElement::Int8 && b.element != OnnxElement::UInt8)) + throw Failure("Expected packed integer matrices"); + std::vector bz{zero(in, 3, b)}; + return {{OnnxElement::Int32, shape, + gpuMatmulResident(*compute, x.bytes.buffer(), b.bytes.buffer(), x.element == OnnxElement::Int8, + b.element == OnnxElement::Int8, rows, k, columns, zero(in, 2, x), bz)}}; + } + return {make( + OnnxElement::Int32, shape, + checked(affine::matmul(bytes(x), bytes(b), rows, k, columns, zero(in, 2, x), zero(in, 3, b), compute)))}; + } + if (n.op == "ConvInteger") { + const auto& w = required(in, 1); + if ((x.shape.size() != 3 && x.shape.size() != 4) || w.shape.size() != x.shape.size()) + throw Failure("ConvInteger supports 1D/2D NCHW", DiagnosticCode::Unsupported); + const size_t spatial = x.shape.size() - 2; + if (n.attrs.contains("kernel_shape") && + n.attrs.at("kernel_shape").integers != std::vector(w.shape.begin() + 2, w.shape.end())) + throw Failure("Convolution kernel_shape mismatch"); + auto strides = attrs(n, "strides", std::vector(spatial, 1)); + auto pads = attrs(n, "pads", std::vector(2 * spatial, 0)); + auto dilations = attrs(n, "dilations", std::vector(spatial, 1)); + if (strides.size() != spatial || pads.size() != 2 * spatial || dilations.size() != spatial) + throw Failure("Invalid convolution attributes"); + auto narrow = [](int64_t v) { + if (v < 0 || v > 65536) throw Failure("Convolution attribute out of range"); + return static_cast(v); + }; + affine::ConvShape s; + s.batch = narrow(x.shape[0]); + s.channels = narrow(x.shape[1]); + s.outputs = narrow(w.shape[0]); + s.width = narrow(x.shape.back()); + s.kernelW = narrow(w.shape.back()); + s.strideW = narrow(strides.back()); + s.dilationW = narrow(dilations.back()); + s.padLeft = narrow(pads[spatial - 1]); + s.padRight = narrow(pads.back()); + s.groups = narrow(attr(n, "group", 1)); + if (spatial == 2) { + s.height = narrow(x.shape[2]); + s.kernelH = narrow(w.shape[2]); + s.strideH = narrow(strides[0]); + s.dilationH = narrow(dilations[0]); + s.padTop = narrow(pads[0]); + s.padBottom = narrow(pads[2]); + } + if (!s.groups || s.channels % s.groups || w.shape[1] != s.channels / s.groups) + throw Failure("Convolution channel mismatch"); + std::vector wz{0}; + if (in.size() > 3 && in[3]) { + if (in[3]->element != w.element) throw Failure("Weight zero-point type mismatch"); + wz.clear(); + for (auto z : ints(*in[3])) wz.push_back(static_cast(z)); + } + if (compute) { + if ((x.element != OnnxElement::Int8 && x.element != OnnxElement::UInt8) || + (w.element != OnnxElement::Int8 && w.element != OnnxElement::UInt8)) + throw Failure("Expected packed convolution inputs"); + const auto extent = + checked(affine::detail::validateConv(x.bytes.size(), w.bytes.size(), x.element == OnnxElement::Int8, + w.element == OnnxElement::Int8, s, zero(in, 2, x), wz)); + std::vector outputShape{s.batch, s.outputs}; + if (spatial == 2) outputShape.push_back(extent.height); + outputShape.push_back(extent.width); + return {{OnnxElement::Int32, outputShape, + gpuConvResident(*compute, x.bytes.buffer(), w.bytes.buffer(), x.element == OnnxElement::Int8, + w.element == OnnxElement::Int8, s, zero(in, 2, x), wz)}}; + } + auto values = checked(affine::conv(bytes(x), bytes(w), s, zero(in, 2, x), wz, compute)); + std::vector shape{s.batch, s.outputs}; + if (spatial == 2) + shape.push_back( + (s.height + s.padTop + s.padBottom - int64_t(s.dilationH) * (s.kernelH - 1) - 1) / s.strideH + 1); + shape.push_back((s.width + s.padLeft + s.padRight - int64_t(s.dilationW) * (s.kernelW - 1) - 1) / s.strideW + + 1); + return {make(OnnxElement::Int32, shape, values)}; + } + throw Failure("Unsupported quantized operation", DiagnosticCode::Unsupported); +} +} // namespace eve::tensor::onnx_detail diff --git a/src/modules/tensor/OnnxReader.cpp b/src/modules/tensor/OnnxReader.cpp new file mode 100644 index 00000000..c8ef581b --- /dev/null +++ b/src/modules/tensor/OnnxReader.cpp @@ -0,0 +1,419 @@ +#include "tensor/OnnxInternal.h" + +#include +#include +#include + +namespace eve::tensor::onnx_detail { +namespace { +struct Field { + int id = 0, wire = 0; + uint64_t number = 0; + std::span bytes; + std::string text() const { return {reinterpret_cast(bytes.data()), bytes.size()}; } +}; +class Reader { +public: + explicit Reader(std::span data) : data_(data) {} + bool empty() const { return offset_ == data_.size(); } + uint64_t varint() { + uint64_t n = 0; + for (int shift = 0; shift < 70; shift += 7) { + if (offset_ == data_.size()) throw Failure("Truncated protobuf varint", DiagnosticCode::ParseError); + const uint8_t b = data_[offset_++]; + if (shift == 63 && b > 1) throw Failure("Overflowing protobuf varint", DiagnosticCode::ParseError); + n |= static_cast(b & 127) << shift; + if (!(b & 128)) return n; + } + throw Failure("Invalid protobuf varint", DiagnosticCode::ParseError); + } + std::span take(uint64_t n) { + if (n > data_.size() - offset_) throw Failure("Truncated protobuf field", DiagnosticCode::ParseError); + auto result = data_.subspan(offset_, static_cast(n)); + offset_ += static_cast(n); + return result; + } + Field next() { + const uint64_t tag = varint(); + if ((tag >> 3) == 0 || (tag >> 3) > 0x1fffffff) + throw Failure("Invalid protobuf tag", DiagnosticCode::ParseError); + Field f; + f.id = static_cast(tag >> 3); + f.wire = tag & 7; + switch (f.wire) { + case 0: f.number = varint(); break; + case 1: f.bytes = take(8); break; + case 2: f.bytes = take(varint()); break; + case 5: f.bytes = take(4); break; + default: throw Failure("Unsupported protobuf wire type", DiagnosticCode::ParseError); + } + return f; + } + +private: + std::span data_; + size_t offset_ = 0; +}; +void wire(const Field& f, int expected) { + if (f.wire != expected) throw Failure("Unexpected protobuf field type", DiagnosticCode::ParseError); +} +void integers(const Field& f, std::vector& out) { + if (f.wire == 0) + out.push_back(std::bit_cast(f.number)); + else { + wire(f, 2); + Reader r(f.bytes); + while (!r.empty()) out.push_back(std::bit_cast(r.varint())); + } +} +void reals(const Field& f, std::vector& out) { + if (f.wire != 2 && f.wire != 5) throw Failure("Invalid float wire type", DiagnosticCode::ParseError); + if (f.bytes.size() % 4) throw Failure("Truncated float payload", DiagnosticCode::ParseError); + for (size_t i = 0; i < f.bytes.size(); i += 4) { + float x; + std::memcpy(&x, f.bytes.data() + i, 4); + out.push_back(x); + } +} +std::pair tensor(std::span data) { + Reader r(data); + RuntimeTensor v; + std::string name; + std::vector iv; + std::vector fv; + bool raw = false, type = false; + while (!r.empty()) { + auto f = r.next(); + switch (f.id) { + case 1: integers(f, v.shape); break; + case 2: + wire(f, 0); + if (f.number > 9) throw Failure("Unsupported tensor element type", DiagnosticCode::Unsupported); + v.element = static_cast(f.number); + type = true; + break; + case 4: reals(f, fv); break; + case 5: + case 7: integers(f, iv); break; + case 8: + wire(f, 2); + name = f.text(); + break; + case 9: + wire(f, 2); + if (raw) throw Failure("Duplicate raw tensor data", DiagnosticCode::ParseError); + v.bytes.assign(f.bytes.begin(), f.bytes.end()); + raw = true; + break; + case 14: + wire(f, 0); + if (f.number != 0) throw Failure("External tensor data unsupported", DiagnosticCode::Unsupported); + break; + case 3: + case 6: + case 10: + case 11: + case 13: throw Failure("External, segmented or unsupported tensor data", DiagnosticCode::Unsupported); + default: break; + } + } + if (!type) throw Failure("Tensor has no dtype", DiagnosticCode::ParseError); + const size_t n = count(v.shape), size = elementSize(v.element); + if (raw && (!iv.empty() || !fv.empty())) throw Failure("Ambiguous tensor payload", DiagnosticCode::ParseError); + if (!raw) { + if (v.element == OnnxElement::Float32) { + if (fv.size() != n || !iv.empty()) + throw Failure("Invalid float tensor payload", DiagnosticCode::ParseError); + v.bytes.resize(n * size); + if (n) std::memcpy(v.bytes.data(), fv.data(), v.bytes.size()); + } else { + if (iv.size() != n || !fv.empty()) + throw Failure("Invalid integer tensor payload", DiagnosticCode::ParseError); + v.bytes.resize(n * size); + for (size_t i = 0; i < n; ++i) { + const auto x = iv[i]; + if ((v.element == OnnxElement::UInt8 && (x < 0 || x > 255)) || + (v.element == OnnxElement::Int8 && (x < -128 || x > 127)) || + (v.element == OnnxElement::Bool && x != 0 && x != 1) || + (v.element == OnnxElement::Int32 && (x < INT32_MIN || x > INT32_MAX))) + throw Failure("Integer initializer out of range", DiagnosticCode::ParseError); + std::memcpy(v.bytes.data() + i * size, &x, size); + } + } + } + validate(v); + return {std::move(name), std::move(v)}; +} +ModelData parseGraph(std::span bytes, size_t depth); +std::pair attribute(std::span bytes, size_t depth) { + Reader r(bytes); + Attribute a; + std::string name; + while (!r.empty()) { + auto f = r.next(); + switch (f.id) { + case 1: + wire(f, 2); + name = f.text(); + break; + case 20: + wire(f, 0); + a.type = static_cast(f.number); + break; + case 2: { + wire(f, 5); + std::memcpy(&a.real, f.bytes.data(), 4); + break; + } + case 3: + wire(f, 0); + a.integer = std::bit_cast(f.number); + break; + case 4: + wire(f, 2); + a.text = f.text(); + break; + case 5: + wire(f, 2); + a.tensor = tensor(f.bytes).second; + break; + case 7: reals(f, a.reals); break; + case 8: integers(f, a.integers); break; + case 9: + wire(f, 2); + a.strings.push_back(f.text()); + break; + case 6: + wire(f, 2); + a.graph = std::make_shared(parseGraph(f.bytes, depth + 1)); + break; + case 11: throw Failure("Multiple graph attributes unsupported", DiagnosticCode::Unsupported); + case 21: throw Failure("Function attribute references unsupported", DiagnosticCode::Unsupported); + default: break; + } + } + if (name.empty() || !a.type) throw Failure("Incomplete attribute", DiagnosticCode::ParseError); + return {std::move(name), std::move(a)}; +} +Node node(std::span bytes, size_t depth) { + Reader r(bytes); + Node n; + while (!r.empty()) { + auto f = r.next(); + if (f.id >= 1 && f.id <= 5) wire(f, 2); + switch (f.id) { + case 1: n.inputs.push_back(f.text()); break; + case 2: n.outputs.push_back(f.text()); break; + case 3: n.name = f.text(); break; + case 4: n.op = f.text(); break; + case 5: + if (!n.attrs.insert(attribute(f.bytes, depth)).second) + throw Failure("Duplicate attribute", DiagnosticCode::ParseError); + break; + case 7: + wire(f, 2); + n.domain = f.text(); + break; + case 8: throw Failure("Function overloads unsupported", DiagnosticCode::Unsupported); + default: break; + } + } + if (n.op.empty() || n.outputs.empty()) throw Failure("Incomplete node", DiagnosticCode::ParseError); + return n; +} +std::pair input(std::span bytes) { + Reader r(bytes); + std::string name; + Input value; + while (!r.empty()) { + auto f = r.next(); + if (f.id == 1) { + wire(f, 2); + name = f.text(); + } + if (f.id == 2) { + wire(f, 2); + Reader type(f.bytes); + while (!type.empty()) { + auto t = type.next(); + if (t.id == 4) { + value.type = 0; + continue; + } + if (t.id != 1) throw Failure("Non-tensor graph input", DiagnosticCode::Unsupported); + wire(t, 2); + Reader tt(t.bytes); + while (!tt.empty()) { + auto p = tt.next(); + if (p.id == 1) { + wire(p, 0); + value.type = static_cast(p.number); + } + if (p.id == 2) { + wire(p, 2); + Reader shape(p.bytes); + while (!shape.empty()) { + auto d = shape.next(); + if (d.id != 1) continue; + wire(d, 2); + Reader dim(d.bytes); + int64_t v = -1; + while (!dim.empty()) { + auto x = dim.next(); + if (x.id == 1) { + wire(x, 0); + v = std::bit_cast(x.number); + } + } + value.shape.push_back(v); + } + } + } + } + } + } + if (name.empty()) throw Failure("Empty value name", DiagnosticCode::ParseError); + if (value.shape.size() > 6) throw Failure("Input rank exceeds six", DiagnosticCode::Unsupported); + for (auto d : value.shape) + if (d < -1 || d > INT32_MAX) throw Failure("Invalid interface dimension", DiagnosticCode::ParseError); + return {name, value}; +} +ModelData parseGraph(std::span bytes, size_t depth) { + ModelData model; + if (depth > 16) throw Failure("ONNX graph nesting limit exceeded", DiagnosticCode::ParseError); + Reader g(bytes); + while (!g.empty()) { + auto f = g.next(); + if (f.id == 1) { + wire(f, 2); + if (model.nodes.size() >= 100000) throw Failure("Node limit exceeded", DiagnosticCode::ParseError); + model.nodes.push_back(node(f.bytes, depth)); + } + if (f.id == 5) { + wire(f, 2); + auto t = tensor(f.bytes); + t.second.bytes.retainAcrossRuns(); + if (t.first.empty() || !model.constants.emplace(std::move(t)).second) + throw Failure("Duplicate/empty initializer", DiagnosticCode::ParseError); + } + if (f.id == 11) { + wire(f, 2); + auto v = input(f.bytes); + model.info.inputs.push_back(v.first); + if (!model.inputs.emplace(std::move(v)).second) + throw Failure("Duplicate graph input", DiagnosticCode::ParseError); + } + if (f.id == 12) { + wire(f, 2); + model.info.outputs.push_back(input(f.bytes).first); + } + if (f.id == 15) throw Failure("Sparse initializers unsupported", DiagnosticCode::Unsupported); + } + return model; +} +} // namespace + +ModelData parse(std::span bytes) { + if constexpr (std::endian::native != std::endian::little) + throw Failure("ONNX requires little-endian host", DiagnosticCode::Unsupported); + if (bytes.empty() || bytes.size() > 512u * 1024u * 1024u) + throw Failure("Model size exceeds import limit", DiagnosticCode::ParseError); + ModelData model; + Reader r(bytes); + std::span graph; + while (!r.empty()) { + auto f = r.next(); + if (f.id == 1) { + wire(f, 0); + model.info.irVersion = static_cast(f.number); + } + if (f.id == 7) { + wire(f, 2); + if (!graph.empty()) throw Failure("Duplicate graph", DiagnosticCode::ParseError); + graph = f.bytes; + } + if (f.id == 8) { + wire(f, 2); + Reader ops(f.bytes); + std::string domain; + int64_t version = 0; + while (!ops.empty()) { + auto x = ops.next(); + if (x.id == 1) { + wire(x, 2); + domain = x.text(); + } + if (x.id == 2) { + wire(x, 0); + version = x.number; + } + } + if (!model.opsets.emplace(domain, version).second) + throw Failure("Duplicate opset", DiagnosticCode::ParseError); + } + if (f.id == 20 || f.id == 25 || f.id == 26) + throw Failure("Training/functions/device configuration unsupported", DiagnosticCode::Unsupported); + } + if (model.info.irVersion < 3 || model.info.irVersion > 10 || !model.opsets.contains("") || + model.opsets.at("") < 13 || model.opsets.at("") > 17) + throw Failure("Unsupported ONNX IR/opset version", DiagnosticCode::UnknownVersion); + if (model.opsets.contains("com.microsoft") && model.opsets.at("com.microsoft") != 1) + throw Failure("Unsupported Microsoft opset", DiagnosticCode::UnknownVersion); + if (graph.empty()) throw Failure("Missing ONNX graph", DiagnosticCode::ParseError); + auto parsed = parseGraph(graph, 0); + model.nodes = std::move(parsed.nodes); + model.constants = std::move(parsed.constants); + model.inputs = std::move(parsed.inputs); + model.info.inputs = std::move(parsed.info.inputs); + model.info.outputs = std::move(parsed.info.outputs); + std::set names; + for (const auto& [name, v] : model.constants) { + names.insert(name); + model.info.initializerBytes += v.bytes.size(); + } + for (const auto& [name, v] : model.inputs) names.insert(name); + for (const auto& n : model.nodes) { + if (!model.opsets.contains(n.domain)) throw Failure("Node domain has no opset", DiagnosticCode::ParseError); + for (const auto& name : n.inputs) + if (!name.empty() && !names.contains(name)) + throw Failure("Input not in topological order: " + name, DiagnosticCode::ParseError); + for (const auto& name : n.outputs) + if (!name.empty() && !names.insert(name).second) + throw Failure("Duplicate graph value: " + name, DiagnosticCode::ParseError); + if (!isSupported(n)) model.info.unsupportedNodes.push_back(n.name + " [" + n.domain + "::" + n.op + "]"); + } + for (const auto& out : model.info.outputs) + if (!names.contains(out)) throw Failure("Undefined graph output", DiagnosticCode::ParseError); + size_t recursiveNodes = 0; + std::function&, bool)> validateNested; + validateNested = [&](const ModelData& g, const std::set& outer, bool root) { + recursiveNodes += g.nodes.size(); + if (recursiveNodes > 100000) throw Failure("Recursive node limit exceeded", DiagnosticCode::ParseError); + auto available = outer; + std::set locals; + for (const auto& [name, t] : g.constants) locals.insert(name); + for (const auto& [name, t] : g.inputs) locals.insert(name); + for (const auto& n : g.nodes) + for (const auto& out : n.outputs) + if (!out.empty() && !locals.insert(out).second) + throw Failure("Duplicate nested graph value: " + out, DiagnosticCode::ParseError); + available.insert(locals.begin(), locals.end()); + for (const auto& n : g.nodes) { + if (!model.opsets.contains(n.domain)) + throw Failure("Nested node domain has no opset", DiagnosticCode::ParseError); + for (const auto& input : n.inputs) + if (!input.empty() && !available.contains(input)) + throw Failure("Undefined lexical capture: " + input, DiagnosticCode::ParseError); + if (!root && !isSupported(n)) + model.info.unsupportedNodes.push_back(n.name + " [" + n.domain + "::" + n.op + "]"); + for (const auto& [key, a] : n.attrs) + if (a.graph) validateNested(*a.graph, available, false); + } + for (const auto& out : g.info.outputs) + if (!available.contains(out)) throw Failure("Undefined nested output: " + out, DiagnosticCode::ParseError); + }; + validateNested(model, {}, true); + model.info.nodeCount = model.nodes.size(); + return model; +} +} // namespace eve::tensor::onnx_detail diff --git a/src/modules/tensor/OnnxRuntime.cpp b/src/modules/tensor/OnnxRuntime.cpp new file mode 100644 index 00000000..f26ecc4a --- /dev/null +++ b/src/modules/tensor/OnnxRuntime.cpp @@ -0,0 +1,360 @@ +#include +#include +#include +#include +#include +#include +#include "tensor/OnnxInternal.h" + +namespace eve::tensor::onnx_detail { +namespace { +struct Value { + const RuntimeTensor* borrowed = nullptr; + std::shared_ptr owned; + std::shared_ptr> sequence; + OnnxElement sequenceElement = OnnxElement::Float32; + const RuntimeTensor& tensor() const { + if (owned) return *owned; + if (borrowed) return *borrowed; + throw Failure("Expected tensor, received sequence"); + } +}; +using Values = std::unordered_map; +struct Context { + OnnxCompute* compute; + uint64_t seed; + bool finite; + size_t steps = 0, liveBytes = 0; + std::unordered_map randomStreams; +}; +Value hold(RuntimeTensor t, Context& c) { + validate(t); + if (c.finite && t.element == OnnxElement::Float32) + for (size_t i = 0; i < count(t.shape); ++i) + if (!std::isfinite(read(t, i))) throw Failure("Nonfinite float output"); + const auto size = t.bytes.size(); + if (size > 512u * 1024u * 1024u - c.liveBytes) throw Failure("ONNX live tensor memory limit exceeded"); + // Context encloses every runtime Value; escaping public outputs are owning copies. + auto* raw = new RuntimeTensor(std::move(t)); + c.liveBytes += size; + return {nullptr, + std::shared_ptr(raw, + [&c, size](const RuntimeTensor* p) { + delete p; + c.liveBytes -= size; + }), + {}}; +} +std::set captures(const ModelData& graph); +std::set dependencies(const Node& n) { + std::set result; + for (const auto& i : n.inputs) + if (!i.empty()) result.insert(i); + for (const auto& [key, a] : n.attrs) + if (a.graph) { + auto names = captures(*a.graph); + result.insert(names.begin(), names.end()); + } + return result; +} +std::set captures(const ModelData& graph) { + std::set locals, result; + for (const auto& [name, t] : graph.constants) locals.insert(name); + for (const auto& [name, t] : graph.inputs) locals.insert(name); + for (const auto& n : graph.nodes) + for (const auto& out : n.outputs) + if (!out.empty()) locals.insert(out); + for (const auto& n : graph.nodes) { + auto deps = dependencies(n); + for (const auto& d : deps) + if (!locals.contains(d)) result.insert(d); + } + for (const auto& out : graph.info.outputs) + if (!locals.contains(out)) result.insert(out); + return result; +} +void admit(const Node& n) { + if (!isSupported(n)) + throw Failure(n.name + ": Unsupported ONNX node " + n.domain + "::" + n.op, DiagnosticCode::Unsupported, + n.name); + for (const auto& [key, a] : n.attrs) + if (a.graph) + for (const auto& child : a.graph->nodes) admit(child); +} +std::vector plan(const ModelData& g, const std::vector& requested) { + std::unordered_map producers; + for (size_t i = 0; i < g.nodes.size(); ++i) + for (const auto& name : g.nodes[i].outputs) + if (!name.empty() && !producers.emplace(name, i).second) throw Failure("Duplicate graph output"); + std::vector selected(g.nodes.size(), false); + std::vector pending = requested; + while (!pending.empty()) { + auto name = std::move(pending.back()); + pending.pop_back(); + auto p = producers.find(name); + if (p == producers.end() || selected[p->second]) continue; + selected[p->second] = true; + admit(g.nodes[p->second]); + auto deps = dependencies(g.nodes[p->second]); + pending.insert(pending.end(), deps.begin(), deps.end()); + } + std::vector indegree(g.nodes.size()); + std::vector> users(g.nodes.size()); + size_t total = 0; + std::priority_queue, std::greater> ready; + for (size_t i = 0; i < g.nodes.size(); ++i) + if (selected[i]) { + ++total; + std::set parents; + for (const auto& name : dependencies(g.nodes[i])) { + auto p = producers.find(name); + if (p != producers.end()) parents.insert(p->second); + } + indegree[i] = parents.size(); + for (auto p : parents) users[p].push_back(i); + if (parents.empty()) ready.push(i); + } + std::vector order; + while (!ready.empty()) { + size_t i = ready.top(); + ready.pop(); + order.push_back(i); + for (auto user : users[i]) + if (--indegree[user] == 0) ready.push(user); + } + if (order.size() != total) throw Failure("ONNX graph has cyclic lexical dependencies", DiagnosticCode::ParseError); + return order; +} +std::vector run(const ModelData&, Values, const std::vector&, Context&, size_t); +std::vector control(const Node& n, const std::vector& in, const Values& values, Context& c, + size_t depth) { + auto scalar = [&](size_t i) { + const auto& t = in.at(i).tensor(); + if (count(t.shape) != 1) throw Failure("Control input must contain one element"); + return integer(t); + }; + auto seq = [&](size_t i) -> const std::vector& { + if (!in.at(i).sequence) throw Failure("Expected sequence input"); + return *in[i].sequence; + }; + auto sequence = [](std::vector v, OnnxElement type) { + return Value{nullptr, {}, std::make_shared>(std::move(v)), type}; + }; + if (n.op == "SequenceEmpty") { + auto type = static_cast(attr(n, "dtype", 1)); + elementSize(type); + return {sequence({}, type)}; + } + if (n.op == "SequenceAt") { + const auto& s = seq(0); + int64_t i = scalar(1); + if (i < 0) i += static_cast(s.size()); + if (i < 0 || static_cast(i) >= s.size()) throw Failure("SequenceAt out of bounds"); + return {s[i]}; + } + if (n.op == "SequenceInsert") { + auto s = seq(0); + if (s.size() >= 100000) throw Failure("Sequence length limit exceeded"); + int64_t i = in.size() > 2 ? scalar(2) : static_cast(s.size()); + if (i < 0) i += static_cast(s.size()); + if (i < 0 || static_cast(i) > s.size()) throw Failure("SequenceInsert out of bounds"); + const auto& t = in.at(1).tensor(); + if (in[0].sequenceElement != t.element) throw Failure("Sequence element dtype mismatch"); + s.insert(s.begin() + i, in[1]); + return {sequence(std::move(s), in[0].sequenceElement)}; + } + if (n.op == "SplitToSequence") { + const auto& x = in.at(0).tensor(); + int a = axis(attr(n, "axis", 0), x.shape.size()); + std::vector lengths; + if (in.size() > 1 && (in[1].borrowed || in[1].owned)) { + const auto& split = in[1].tensor(); + lengths = ints(split); + if (split.shape.empty()) { + if (lengths[0] <= 0) throw Failure("Invalid split size"); + int64_t width = lengths[0]; + lengths.clear(); + for (int64_t i = 0; i < x.shape[a]; i += width) { + if (lengths.size() >= 100000) throw Failure("Sequence length limit exceeded"); + lengths.push_back(std::min(width, x.shape[a] - i)); + } + } + } else + lengths.assign(x.shape[a], 1); + if (lengths.size() > 100000) throw Failure("Sequence length limit exceeded"); + int64_t total = 0; + for (auto size : lengths) { + if (size < 0 || size > x.shape[a] - total) throw Failure("Invalid split lengths"); + total += size; + } + if (total != x.shape[a]) throw Failure("Split lengths do not cover axis"); + size_t outer = 1, inner = elementSize(x.element); + for (int j = 0; j < a; ++j) outer *= x.shape[j]; + for (size_t j = a + 1; j < x.shape.size(); ++j) inner *= x.shape[j]; + std::vector s; + size_t offset = 0; + for (auto length : lengths) { + auto shape = x.shape; + shape[a] = length; + if (!attr(n, "keepdims", 1)) { + if (in.size() > 1 && (in[1].borrowed || in[1].owned)) + throw Failure("keepdims=0 with explicit split unsupported", DiagnosticCode::Unsupported); + shape.erase(shape.begin() + a); + } + RuntimeTensor out{x.element, shape, std::vector(count(shape) * elementSize(x.element))}; + for (size_t o = 0; o < outer; ++o) + if (length) + std::memcpy(out.bytes.data() + o * length * inner, + x.bytes.data() + (o * x.shape[a] + offset) * inner, length * inner); + s.push_back(hold(std::move(out), c)); + offset += length; + } + return {sequence(std::move(s), x.element)}; + } + if (n.op == "ConcatFromSequence") { + const auto& s = seq(0); + if (s.empty()) throw Failure("Cannot concatenate empty sequence"); + std::vector tensors; + std::vector inputs; + for (const auto& v : s) { + tensors.push_back(v.tensor()); + if (attr(n, "new_axis", 0)) { + const int a = axis(attr(n, "axis", 0), tensors.back().shape.size() + 1); + tensors.back().shape.insert(tensors.back().shape.begin() + a, 1); + } + } + for (const auto& t : tensors) inputs.push_back(&t); + Node concat = n; + concat.op = "Concat"; + auto out = executeShape(concat, inputs); + if (!out) throw Failure("ConcatFromSequence dispatch failed"); + return {hold(std::move(*out), c)}; + } + if (n.op == "If") { + if (in.at(0).tensor().element != OnnxElement::Bool) throw Failure("If condition must be boolean"); + auto key = scalar(0) ? "then_branch" : "else_branch"; + const auto& g = n.attrs.at(key).graph; + if (!g) throw Failure("Missing If branch"); + return run(*g, values, g->info.outputs, c, depth + 1); + } + if (n.op == "Loop") { + const auto& body = n.attrs.at("body").graph; + if (!body) throw Failure("Missing Loop body"); + if (in.size() < 2 || body->info.inputs.size() != in.size() || body->info.outputs.size() != in.size() - 1 || + n.outputs.size() != in.size() - 2) + throw Failure("Loop scan outputs unsupported", DiagnosticCode::Unsupported); + const int64_t trips = (in[0].borrowed || in[0].owned) ? scalar(0) : 10000; + if (trips < 0 || trips > 10000) throw Failure("Loop trip budget exceeded"); + bool condition = (in[1].borrowed || in[1].owned) ? scalar(1) != 0 : true; + std::vector state(in.begin() + 2, in.end()); + for (int64_t i = 0; i < trips && condition; ++i) { + auto scope = values; + scope[body->info.inputs[0]] = hold(make(OnnxElement::Int64, {}, std::vector{i}), c); + scope[body->info.inputs[1]] = + hold(make(OnnxElement::Bool, {}, std::vector{uint8_t(condition)}), c); + for (size_t j = 0; j < state.size(); ++j) scope[body->info.inputs[j + 2]] = state[j]; + auto result = run(*body, std::move(scope), body->info.outputs, c, depth + 1); + const auto& cond = result[0].tensor(); + if (cond.element != OnnxElement::Bool || count(cond.shape) != 1) + throw Failure("Loop returned invalid condition"); + condition = integer(cond) != 0; + state.assign(result.begin() + 1, result.end()); + } + return state; + } + if (n.op == "RandomUniformLike" || n.op == "RandomNormalLike") { + const auto& x = in.at(0).tensor(); + if (attr(n, "dtype", static_cast(x.element)) != 1) + throw Failure("Random output requires FP32", DiagnosticCode::Unsupported); + uint64_t seed = c.seed; + for (unsigned char ch : n.name) seed = (seed ^ ch) * 1099511628211ull; + if (n.attrs.contains("seed")) { + uint32_t bits; + std::memcpy(&bits, &n.attrs.at("seed").real, 4); + seed ^= bits; + } + auto [stream, inserted] = c.randomStreams.try_emplace(&n, static_cast(seed ^ (seed >> 32))); + auto& rng = stream->second; + auto uniform = [&]() { return (double(rng()) + .5) / 4294967296.; }; + std::vector v(count(x.shape)); + const float low = n.attrs.contains("low") ? n.attrs.at("low").real : 0, + high = n.attrs.contains("high") ? n.attrs.at("high").real : 1, + mean = n.attrs.contains("mean") ? n.attrs.at("mean").real : 0, + scale = n.attrs.contains("scale") ? n.attrs.at("scale").real : 1; + if (!std::isfinite(low) || !std::isfinite(high) || !std::isfinite(mean) || !std::isfinite(scale) || + high < low || scale < 0) + throw Failure("Invalid random distribution parameters"); + for (auto& f : v) + f = n.op == "RandomUniformLike" ? static_cast(low + (high - low) * uniform()) + : static_cast(mean + scale * std::sqrt(-2 * std::log(uniform())) * + std::cos(6.283185307179586 * uniform())); + return {hold(make(OnnxElement::Float32, x.shape, v), c)}; + } + std::vector tensors; + for (const auto& v : in) tensors.push_back(v.borrowed || v.owned ? &v.tensor() : nullptr); + auto outputs = execute(n, tensors, c.compute); + std::vector result; + for (auto& t : outputs) result.push_back(hold(std::move(t), c)); + return result; +} +std::vector run(const ModelData& g, Values values, const std::vector& requested, Context& c, + size_t depth) { + if (depth > 16) throw Failure("Execution graph nesting limit exceeded"); + for (const auto& [name, t] : g.constants) + if (!g.inputs.contains(name) || !values.contains(name)) values[name] = {&t, {}, {}}; + auto order = plan(g, requested); + std::unordered_map uses; + for (auto i : order) + for (const auto& dep : dependencies(g.nodes[i])) ++uses[dep]; + for (const auto& name : requested) ++uses[name]; + for (auto i : order) { + const auto& n = g.nodes[i]; + if (++c.steps > 1000000) throw Failure("ONNX execution step budget exceeded"); + try { + std::vector in; + for (const auto& name : n.inputs) { + if (name.empty()) { + in.push_back({}); + continue; + } + auto it = values.find(name); + if (it == values.end()) throw Failure("Missing feed or lexical capture: " + name); + in.push_back(it->second); + } + auto outputs = control(n, in, values, c, depth); + if (outputs.size() < n.outputs.size()) throw Failure("Operator output arity mismatch"); + for (size_t j = 0; j < n.outputs.size(); ++j) + if (!n.outputs[j].empty() && uses[n.outputs[j]]) values[n.outputs[j]] = std::move(outputs[j]); + for (const auto& dep : dependencies(n)) + if (--uses[dep] == 0) values.erase(dep); + } catch (const Failure& e) { + throw Failure(n.name + ": " + e.what(), e.code, e.path.empty() ? n.name : e.path); + } + } + std::vector out; + for (const auto& name : requested) { + auto it = values.find(name); + if (it == values.end()) throw Failure("Missing output: " + name, DiagnosticCode::NotFound); + out.push_back(it->second); + } + return out; +} +} // namespace +std::vector evaluate(const ModelData& model, std::span feeds, + const std::vector& requested, OnnxCompute* compute, + OnnxRunOptions options) { + Context c{compute, options.seed, options.requireFinite}; + Values inputs; + for (const auto& f : feeds) inputs[f.name] = hold({f.tensor.element, f.tensor.shape, f.tensor.bytes}, c); + auto outputs = run(model, std::move(inputs), requested, c, 0); + std::vector result; + size_t bytes = c.liveBytes; + for (size_t i = 0; i < outputs.size(); ++i) { + const auto& t = outputs[i].tensor(); + if (t.bytes.size() > 512u * 1024u * 1024u - bytes) throw Failure("ONNX output memory limit exceeded"); + bytes += t.bytes.size(); + result.push_back({requested[i], {t.element, t.shape, static_cast>(t.bytes)}}); + } + return result; +} +} // namespace eve::tensor::onnx_detail diff --git a/src/modules/tensor/OnnxShapeOps.cpp b/src/modules/tensor/OnnxShapeOps.cpp new file mode 100644 index 00000000..093cf161 --- /dev/null +++ b/src/modules/tensor/OnnxShapeOps.cpp @@ -0,0 +1,196 @@ +#include +#include +#include +#include +#include "tensor/OnnxInternal.h" + +namespace eve::tensor::onnx_detail { +std::vector broadcast(const std::vector& a, const std::vector& b) { + std::vector out(std::max(a.size(), b.size()), 1); + for (size_t i = 0; i < out.size(); ++i) { + const auto x = i < a.size() ? a[a.size() - 1 - i] : 1, y = i < b.size() ? b[b.size() - 1 - i] : 1; + if (x != y && x != 1 && y != 1) throw Failure("Broadcast dimension mismatch"); + out[out.size() - 1 - i] = x == 1 ? y : x; + } + count(out); + return out; +} +size_t broadcastIndex(size_t i, const std::vector& shape, const std::vector& output) { + size_t index = 0, stride = 1; + for (size_t j = 0; j < output.size(); ++j) { + const auto coordinate = i % output[output.size() - 1 - j]; + i /= output[output.size() - 1 - j]; + if (j < shape.size()) { + const auto d = shape[shape.size() - 1 - j]; + if (d != 1) index += coordinate * stride; + stride *= d; + } + } + return index; +} +std::optional executeShape(const Node& n, const std::vector& in) { + const auto& x = required(in, 0); + if (n.op == "Slice") { + auto starts = ints(required(in, 1)), ends = ints(required(in, 2)); + auto axes = in.size() > 3 && in[3] ? ints(*in[3]) : std::vector(starts.size()); + if (in.size() <= 3 || !in[3]) std::iota(axes.begin(), axes.end(), 0); + auto steps = in.size() > 4 && in[4] ? ints(*in[4]) : std::vector(starts.size(), 1); + if (starts.size() != ends.size() || starts.size() != axes.size() || starts.size() != steps.size()) + throw Failure("Slice parameter lengths mismatch"); + auto shape = x.shape; + std::vector begin(shape.size(), 0), step(shape.size(), 1); + std::set seen; + for (size_t j = 0; j < axes.size(); ++j) { + int a = axis(axes[j], shape.size()); + if (!seen.insert(a).second || !steps[j] || steps[j] == INT64_MIN) throw Failure("Invalid Slice axis/step"); + const auto d = x.shape[a], st = steps[j]; + auto norm = [d, st](int64_t v, bool end) { + if (v < 0) v = v < -d ? (st < 0 ? -1 : 0) : v + d; + return std::clamp(v, st < 0 && end ? int64_t(-1) : int64_t(0), + st < 0 ? std::max(int64_t(0), d - 1) : d); + }; + begin[a] = norm(starts[j], false); + const auto e = norm(ends[j], true); + step[a] = st; + const int64_t distance = st > 0 ? e - begin[a] : begin[a] - e, magnitude = st > 0 ? st : -st; + shape[a] = d == 0 || distance <= 0 ? 0 : 1 + (distance - 1) / magnitude; + } + RuntimeTensor out{x.element, shape, std::vector(count(shape) * elementSize(x.element))}; + for (size_t i = 0; i < count(shape); ++i) { + size_t rest = i, source = 0, stride = 1; + for (size_t j = shape.size(); j > 0; --j) { + auto c = rest % shape[j - 1]; + rest /= shape[j - 1]; + source += (begin[j - 1] + c * step[j - 1]) * stride; + stride *= x.shape[j - 1]; + } + std::memcpy(out.bytes.data() + i * elementSize(x.element), x.bytes.data() + source * elementSize(x.element), + elementSize(x.element)); + } + return out; + } + if (n.op == "Concat") { + int a = axis(attr(n, "axis", 0), x.shape.size()); + auto shape = x.shape; + shape[a] = 0; + for (auto* t : in) { + if (!t || t->element != x.element || t->shape.size() != shape.size()) + throw Failure("Concat type/rank mismatch"); + for (size_t j = 0; j < shape.size(); ++j) + if (j != a && t->shape[j] != shape[j]) throw Failure("Concat dimension mismatch"); + shape[a] += t->shape[a]; + } + RuntimeTensor out{x.element, shape, std::vector(count(shape) * elementSize(x.element))}; + size_t outer = 1, inner = elementSize(x.element); + for (int j = 0; j < a; ++j) outer *= shape[j]; + for (size_t j = a + 1; j < shape.size(); ++j) inner *= shape[j]; + size_t offset = 0; + for (size_t o = 0; o < outer; ++o) + for (auto* t : in) { + const size_t size = t->shape[a] * inner; + if (size) std::memcpy(out.bytes.data() + offset, t->bytes.data() + o * size, size); + offset += size; + } + return out; + } + if (n.op == "Expand") { + auto requested = ints(required(in, 1)); + count(requested); + auto shape = broadcast(x.shape, requested); + RuntimeTensor out{x.element, shape, std::vector(count(shape) * elementSize(x.element))}; + for (size_t i = 0; i < count(shape); ++i) + std::memcpy(out.bytes.data() + i * elementSize(x.element), + x.bytes.data() + broadcastIndex(i, x.shape, shape) * elementSize(x.element), + elementSize(x.element)); + return out; + } + if (n.op == "ConstantOfShape") { + auto shape = ints(x); + RuntimeTensor value = make(OnnxElement::Float32, {}, std::vector{0}); + if (n.attrs.contains("value")) { + if (!n.attrs.at("value").tensor) throw Failure("Missing fill value"); + value = *n.attrs.at("value").tensor; + } + if (count(value.shape) != 1) throw Failure("ConstantOfShape requires one fill value"); + RuntimeTensor out{value.element, shape, std::vector(count(shape) * elementSize(value.element))}; + for (size_t i = 0; i < out.bytes.size(); i += value.bytes.size()) + std::memcpy(out.bytes.data() + i, value.bytes.data(), value.bytes.size()); + return out; + } + if (n.op == "Where") { + const auto& a = required(in, 1); + const auto& b = required(in, 2); + if (x.element != OnnxElement::Bool || a.element != b.element) throw Failure("Where type mismatch"); + auto shape = broadcast(broadcast(x.shape, a.shape), b.shape); + const auto size = elementSize(a.element); + RuntimeTensor out{a.element, shape, std::vector(count(shape) * size)}; + for (size_t i = 0; i < count(shape); ++i) { + const auto& v = x.bytes[broadcastIndex(i, x.shape, shape)] ? a : b; + std::memcpy(out.bytes.data() + i * size, v.bytes.data() + broadcastIndex(i, v.shape, shape) * size, size); + } + return out; + } + if (n.op == "Not") { + if (x.element != OnnxElement::Bool) throw Failure("Not requires boolean"); + auto out = x; + for (auto& b : out.bytes) b = !b; + return out; + } + if (n.op == "Equal" || n.op == "Less" || n.op == "Greater" || n.op == "And") { + const auto& y = required(in, 1); + if (x.element != y.element || (n.op == "And" && x.element != OnnxElement::Bool)) + throw Failure("Comparison type mismatch"); + auto shape = broadcast(x.shape, y.shape); + RuntimeTensor out{OnnxElement::Bool, shape, std::vector(count(shape))}; + for (size_t i = 0; i < out.bytes.size(); ++i) { + auto a = broadcastIndex(i, x.shape, shape), b = broadcastIndex(i, y.shape, shape); + auto compare = [&](auto u, auto v) { + return n.op == "Equal" ? u == v + : n.op == "Less" ? u < v + : n.op == "Greater" ? u > v + : bool(u) && bool(v); + }; + out.bytes[i] = x.element == OnnxElement::Float32 ? compare(read(x, a), read(y, b)) + : compare(integer(x, a), integer(y, b)); + } + return out; + } + if ((n.op == "Add" || n.op == "Sub" || n.op == "Mul" || n.op == "Div") && x.element != OnnxElement::Float32) { + const auto& y = required(in, 1); + if (x.element != y.element || (x.element != OnnxElement::Int64 && x.element != OnnxElement::Int32)) + throw Failure("Integer arithmetic dtype mismatch"); + auto shape = broadcast(x.shape, y.shape); + RuntimeTensor out{x.element, shape, std::vector(count(shape) * elementSize(x.element))}; + for (size_t i = 0; i < count(shape); ++i) { + int64_t a = integer(x, broadcastIndex(i, x.shape, shape)), + b = integer(y, broadcastIndex(i, y.shape, shape)), v = 0; + if (n.op == "Add") { + if ((b > 0 && a > INT64_MAX - b) || (b < 0 && a < INT64_MIN - b)) + throw Failure("Integer addition overflow"); + v = a + b; + } + if (n.op == "Sub") { + if ((b < 0 && a > INT64_MAX + b) || (b > 0 && a < INT64_MIN + b)) + throw Failure("Integer subtraction overflow"); + v = a - b; + } + if (n.op == "Mul") { + if (a && b && + (a > 0 ? (b > 0 ? a > INT64_MAX / b : b < INT64_MIN / a) + : (b > 0 ? a < INT64_MIN / b : a < INT64_MAX / b))) + throw Failure("Integer multiplication overflow"); + v = a * b; + } + if (n.op == "Div") { + if (!b || (a == INT64_MIN && b == -1)) throw Failure("Invalid integer division"); + v = a / b; + } + if (x.element == OnnxElement::Int32 && (v < INT32_MIN || v > INT32_MAX)) + throw Failure("Int32 arithmetic overflow"); + std::memcpy(out.bytes.data() + i * elementSize(x.element), &v, elementSize(x.element)); + } + return out; + } + return std::nullopt; +} +} // namespace eve::tensor::onnx_detail diff --git a/src/modules/tensor/OnnxStorage.h b/src/modules/tensor/OnnxStorage.h new file mode 100644 index 00000000..8fde4981 --- /dev/null +++ b/src/modules/tensor/OnnxStorage.h @@ -0,0 +1,33 @@ +#pragma once +#include +#include +#include +#include "common/Result.h" +namespace eve::tensor { +/** @brief Run-scoped device storage. Readback completes pending work on the device thread. + * @note Implementations reject access after their run ends; no device handles escape runGpu. */ +class OnnxDeviceStorage { +public: + /** @brief Release owning storage on the device thread. */ + virtual ~OnnxDeviceStorage() = default; + /** @brief Return an owning host copy, or an explicit device/lifecycle error. */ + [[nodiscard]] virtual Result> readback() const = 0; +}; +/** @brief Immutable buffer shared by graph aliases; exactly one storage is authoritative. + * @note host owns CPU bytes; device owns GPU bytes. Host inputs may be cached by shared identity + * across calls only when persistent marks an immutable model initializer. Consumers use the device thread. */ +struct OnnxBuffer { + std::shared_ptr> host; + std::shared_ptr device; + size_t size = 0; + bool persistent = false; // Immutable model initializer, eligible for bounded cross-run caching. +}; +/** @brief Actual transfers and submissions made during one GPU run. */ +struct OnnxTransferStats { + size_t shaderCompilations = 0, shaderCacheHits = 0; + size_t uploads = 0, downloads = 0, submissions = 0; + size_t uploadedBytes = 0, downloadedBytes = 0; + size_t bufferAllocations = 0, bufferReuses = 0; + size_t compilerPeakWorkers = 0; +}; +} // namespace eve::tensor diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 532e9aa0..c12abb18 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -250,6 +250,21 @@ elseif(EVENGINE_IOS_TEST_APP) EVENGINE_IOS_TEST_APP=1) else() target_link_libraries(unit_test ${EVELIBS} zeroerr ${ThirdParty} ${EVENGINE_VULKAN_LIB} ${EVENGINE_WEBGPU_LIB}) + if(TARGET EVTensor AND NOT EVENGINE_WEBGPU) + add_executable(onnx_engine_probe EXCLUDE_FROM_ALL "${CMAKE_SOURCE_DIR}/examples/tensor/onnx/main.cpp") + target_compile_definitions(onnx_engine_probe PRIVATE EVE_ONNX_ENGINE_PROVIDER POCO_STATIC POCO_NO_AUTOMATIC_LIBS NOMINMAX) + target_link_libraries(onnx_engine_probe eve_engine_includes ${EVELIBS} zeroerr ${ThirdParty} ${EVENGINE_VULKAN_LIB}) + add_executable(onnx_gpu_contracts EXCLUDE_FROM_ALL main.cpp tensor_onnx_gpu.cpp) + target_compile_definitions(onnx_gpu_contracts PRIVATE POCO_STATIC POCO_NO_AUTOMATIC_LIBS NOMINMAX) + target_link_libraries(onnx_gpu_contracts eve_engine_includes ${EVELIBS} zeroerr ${ThirdParty} ${EVENGINE_VULKAN_LIB}) + + + if(WIN32) + target_link_libraries(onnx_engine_probe EVBacktrace) + target_link_libraries(onnx_gpu_contracts EVBacktrace) + endif() + endif() + # Scene composition must also run in a trimmed host, where unrelated unit-test # translation units still reference disabled optional domains. if(TARGET EVScene_editor) diff --git a/test/tensor_onnx.cpp b/test/tensor_onnx.cpp new file mode 100644 index 00000000..723c9cf1 --- /dev/null +++ b/test/tensor_onnx.cpp @@ -0,0 +1,231 @@ +#include "tensor/AffineQuant.h" +#include "tensor/OnnxByteStorage.h" +#include "tensor/OnnxCompute.h" +#include "tensor/OnnxModel.h" +#include "zeroerr/unittest.h" + +#include +#include +#include + +namespace { +using Bytes = std::vector; +void varint(Bytes& b, uint64_t n) { + while (n >= 128) { + b.push_back(static_cast(n) | 128); + n >>= 7; + } + b.push_back(static_cast(n)); +} +void number(Bytes& b, int id, uint64_t n) { + varint(b, id * 8); + varint(b, n); +} +void field(Bytes& b, int id, const Bytes& value) { + varint(b, id * 8 + 2); + varint(b, value.size()); + b.insert(b.end(), value.begin(), value.end()); +} +Bytes text(const char* s) { return Bytes(s, s + std::char_traits::length(s)); } +Bytes unknownModel() { + Bytes node, graph, output, opset, model; + field(node, 2, text("y")); + field(node, 3, text("unsupported-node")); + field(node, 4, text("UnknownOperator")); + field(graph, 1, node); + field(output, 1, text("y")); + field(graph, 12, output); + number(opset, 2, 14); + number(model, 1, 7); + field(model, 8, opset); + field(model, 7, graph); + return model; +} +} // namespace + +TEST_CASE("tensor.onnx.affine.roundingAndRange") { + using namespace eve::tensor::affine; + const std::vector input{-1000, -1.25f, -.75f, -.25f, .25f, .75f, 1.25f, 1000}; + auto q = quantize(input, .5f, 0, true); + REQUIRE(q.ok()); + CHECK(q.value() == Bytes({128, 254, 254, 0, 0, 2, 2, 127})); + auto zero = dynamicQuantize(std::vector{0, 0}); + REQUIRE(zero.ok()); + CHECK(zero.value().scale == 1); + auto invalid = quantize(input, 0, 0, false); + CHECK(!invalid.ok()); + auto nonfinite = dynamicQuantize(std::vector{std::numeric_limits::infinity()}); + CHECK(!nonfinite.ok()); +} + +TEST_CASE("tensor.onnx.affine.exactAccumulators") { + using namespace eve::tensor::affine; + Bytes a(1025, 255), b(1025, 127); + auto product = matmul({a, false}, {b, true}, 1, 1025, 1); + REQUIRE(product.ok()); + CHECK(product.value()[0] == 33194625); + auto mismatch = matmul({a, false}, {b, true}, 2, 1025, 1); + CHECK(!mismatch.ok()); + auto zero = matmul({a, false}, {b, true}, 1, 1025, 1, 256); + CHECK(!zero.ok()); + Bytes largeA(40000, 255), largeB(40000, 255); + auto overflow = matmul({largeA, false}, {largeB, false}, 1, 40000, 1); + CHECK(!overflow.ok()); +} + +TEST_CASE("tensor.onnx.affine.convolutionValidation") { + using namespace eve::tensor::affine; + ConvShape shape; + shape.width = 3; + shape.kernelW = 3; + shape.padLeft = 1; + shape.padRight = 1; + const Bytes x{11, 12, 13}, w{1, 1, 1}; + const std::vector z{0}; + auto result = conv({x, false}, {w, false}, shape, 10, z); + REQUIRE(result.ok()); + CHECK(result.value() == std::vector({3, 6, 5})); + shape.groups = 2; + auto invalid = conv({x, false}, {w, false}, shape, 10, z); + CHECK(!invalid.ok()); +} + +TEST_CASE("tensor.onnx.import.transactionAndDiagnostics") { + using eve::tensor::OnnxModel; + auto bytes = unknownModel(); + auto good = OnnxModel::load(bytes); + REQUIRE(good.ok()); + CHECK(good.value()->info().unsupportedNodes.size() == 1); + auto rejected = good.value()->run({}); + REQUIRE(!rejected.ok()); + REQUIRE(rejected.error() != nullptr); + CHECK(rejected.error()->code() == eve::DiagnosticCode::Unsupported); + CHECK(rejected.error()->path() == "unsupported-node"); + bytes.pop_back(); + auto broken = OnnxModel::load(bytes); + CHECK(!broken.ok()); + CHECK(good.value()->info().nodeCount == 1); + auto empty = OnnxModel::load({}); + CHECK(!empty.ok()); + auto overflow = OnnxModel::load(Bytes(12, 255)); + CHECK(!overflow.ok()); +} + +TEST_CASE("tensor.onnx.import.owningOutputs") { + Bytes tensor, attribute, node, graph, output, opset, encoded; + number(tensor, 2, 1); + const float expected = 1.5f; + Bytes payload(sizeof(float)); + std::memcpy(payload.data(), &expected, sizeof(float)); + field(tensor, 9, payload); + field(attribute, 1, text("value")); + number(attribute, 20, 4); + field(attribute, 5, tensor); + field(node, 2, text("y")); + field(node, 4, text("Constant")); + field(node, 5, attribute); + field(graph, 1, node); + field(output, 1, text("y")); + field(graph, 12, output); + number(opset, 2, 14); + number(encoded, 1, 7); + field(encoded, 8, opset); + field(encoded, 7, graph); + auto imported = eve::tensor::OnnxModel::load(encoded); + REQUIRE(imported.ok()); + encoded.clear(); + auto result = imported.value()->run({}); + REQUIRE(result.ok()); + imported.value().reset(); + REQUIRE(result.value().size() == 1); + REQUIRE(result.value()[0].tensor.bytes.size() == 4); + float actual; + std::memcpy(&actual, result.value()[0].tensor.bytes.data(), 4); + CHECK(actual == expected); +} + +TEST_CASE("tensor.onnx.gpu.failureAndScope") { + class FailedGpu final : public eve::tensor::OnnxCompute { + public: + int begins = 0, ends = 0, dispatches = 0; + eve::Result> dispatch(const eve::tensor::OnnxKernel&) override { + ++dispatches; + return eve::Result>::failure( + eve::Diagnostic::error(eve::DiagnosticCode::Failed, "injected GPU failure")); + } + + protected: + void beginRun() noexcept override { ++begins; } + void endRun() noexcept override { ++ends; } + } gpu; + + auto unsupported = eve::tensor::OnnxModel::load(unknownModel()); + REQUIRE(unsupported.ok()); + + auto rejected = unsupported.value()->runGpu({}, gpu); + + CHECK(!rejected.ok()); + CHECK(gpu.dispatches == 0); + CHECK(gpu.begins == 1); + CHECK(gpu.ends == 1); + Bytes tensor, node, graph, output, opset, model; + number(tensor, 2, 1); + field(tensor, 8, text("x")); + float value = -2; + Bytes data(4); + std::memcpy(data.data(), &value, 4); + field(tensor, 9, data); + field(node, 1, text("x")); + field(node, 2, text("y")); + field(node, 3, text("relu-test")); + field(node, 4, text("Relu")); + field(graph, 1, node); + field(graph, 5, tensor); + field(output, 1, text("y")); + field(graph, 12, output); + number(opset, 2, 14); + number(model, 1, 7); + field(model, 8, opset); + field(model, 7, graph); + auto imported = eve::tensor::OnnxModel::load(model); + REQUIRE(imported.ok()); + + auto failed = imported.value()->runGpu({}, gpu); + REQUIRE(!failed.ok()); + CHECK(failed.error()->code() == eve::DiagnosticCode::Failed); + CHECK(failed.error()->path() == "relu-test"); + CHECK(gpu.dispatches == 1); + CHECK(gpu.begins == 2); + CHECK(gpu.ends == 2); + auto cpu = imported.value()->run({}); + REQUIRE(cpu.ok()); + float out = -1; + std::memcpy(&out, cpu.value()[0].tensor.bytes.data(), 4); + CHECK(out == 0); +} + +TEST_CASE("tensor.onnx.storage.aliasAndCpuMutation") { + using eve::tensor::onnx_detail::ByteStorage; + class Deferred final : public eve::tensor::OnnxDeviceStorage { + public: + mutable int reads = 0; + eve::Result> readback() const override { + ++reads; + return eve::Result>::success({1, 2, 3, 4}); + } + }; + auto deferred = std::make_shared(); + ByteStorage original(eve::tensor::OnnxBuffer{{}, deferred, 4}); + ByteStorage alias = original; + CHECK(alias.size() == 4); + CHECK(deferred->reads == 0); + alias.data()[0] = 9; + CHECK(deferred->reads == 1); + CHECK(static_cast(alias)[0] == 9); + CHECK(static_cast(original)[0] == 1); + CHECK(deferred->reads == 2); + ByteStorage copy = alias; + copy.data()[1] = 8; + CHECK(static_cast(alias)[1] == 2); + CHECK(static_cast(copy)[1] == 8); +} diff --git a/test/tensor_onnx_gpu.cpp b/test/tensor_onnx_gpu.cpp new file mode 100644 index 00000000..b2d4d445 --- /dev/null +++ b/test/tensor_onnx_gpu.cpp @@ -0,0 +1,297 @@ +#include "common/config.h" +#if !defined(EVENGINE_WEBGPU) +#include +#include +#include "gpgpu/ComputeProgram.h" +#include "gpgpu/ComputeShader.h" +#include "gpgpu/Gpgpu.h" +#include "gpgpu/GpuBuffer.h" +#include "gpgpu/Sequence.h" +#include "graphics/Graphics.h" +#include "graphics/vulkan/Graphics.h" +#include "tensor/OnnxCompute.h" +#include "tensor/OnnxModel.h" +#include "zeroerr/unittest.h" +namespace { +void initialize() { + static bool initialized = false; + if (!initialized) { + eve::graphics::Graphics::create()->initHeadless(64, 64); + initialized = true; + } +} +using Bytes = std::vector; +void number(Bytes& b, uint64_t n) { + while (n >= 128) { + b.push_back(uint8_t(n) | 128); + n >>= 7; + } + b.push_back(uint8_t(n)); +} +void integer(Bytes& b, int tag, uint64_t n) { + number(b, tag * 8); + number(b, n); +} +void field(Bytes& b, int tag, const Bytes& value) { + number(b, tag * 8 + 2); + number(b, value.size()); + b.insert(b.end(), value.begin(), value.end()); +} +void text(Bytes& b, int tag, const std::string& s) { field(b, tag, Bytes(s.begin(), s.end())); } +Bytes chain(size_t count = 4096, float increment = 1, bool invalidReshape = false, bool mixed = false, + int stages = 16) { + Bytes graph, tensor, raw(count * sizeof(float)); + for (size_t i = 0; i < count; ++i) { + float f = float(i); + std::memcpy(raw.data() + i * 4, &f, 4); + } + integer(tensor, 1, count); + integer(tensor, 2, 1); + text(tensor, 8, "x"); + field(tensor, 9, raw); + field(graph, 5, tensor); + Bytes bias, one(4); + float f = increment; + std::memcpy(one.data(), &f, 4); + integer(bias, 2, 1); + text(bias, 8, "one"); + field(bias, 9, one); + field(graph, 5, bias); + std::string previous = "x"; + for (int i = 0; i < stages; ++i) { + Bytes n; + text(n, 1, previous); + text(n, 1, "one"); + auto next = "v" + std::to_string(i); + text(n, 2, next); + text(n, 4, mixed ? std::vector{"Add", "Mul", "Sub", "Div"}[i % 4] : "Add"); + field(graph, 1, n); + previous = next; + Bytes alias; + text(alias, 1, previous); + previous += "alias"; + text(alias, 2, previous); + text(alias, 4, "Identity"); + field(graph, 1, alias); + } + if (invalidReshape) { + Bytes t, raw(8), node; + int64_t dimension = int64_t(count) + 1; + std::memcpy(raw.data(), &dimension, 8); + integer(t, 1, 1); + integer(t, 2, 7); + text(t, 8, "badShape"); + field(t, 9, raw); + field(graph, 5, t); + text(node, 1, previous); + text(node, 1, "badShape"); + text(node, 2, "invalid"); + text(node, 4, "Reshape"); + field(graph, 1, node); + previous = "invalid"; + } + Bytes out, opset, model; + text(out, 1, previous); + field(graph, 12, out); + integer(opset, 2, 14); + integer(model, 1, 7); + field(model, 8, opset); + field(model, 7, graph); + return model; +} +} // namespace +TEST_CASE("tensor.onnx.gpu.residentChain") { + initialize(); + auto provider = eve::tensor::createOnnxGpuCompute(); + REQUIRE(provider.ok()); + auto model = eve::tensor::OnnxModel::load(chain()); + REQUIRE(model.ok()); + for (int repeat = 0; repeat < 2; ++repeat) { + auto r = model.value()->runGpu({}, *provider.value()); + REQUIRE(r.ok()); + CHECK(r.value().dispatches == 16); + CHECK(r.value().transfers.uploads == (repeat == 0 ? 2 : 0)); + CHECK(r.value().transfers.shaderCompilations == (repeat == 0 ? 1 : 0)); + CHECK(r.value().transfers.downloads == 1); + CHECK(r.value().transfers.submissions == 1); + const auto& raw = r.value().outputs[0].tensor.bytes; + for (size_t i = 0; i < 4096; ++i) { + float f; + std::memcpy(&f, raw.data() + i * 4, 4); + REQUIRE(f == float(i + 16)); + } + } +} +TEST_CASE("gpgpu.sequence.stagingSizeReuse") { + initialize(); + auto* gp = eve::gpgpu::Gpgpu::create(); + REQUIRE(gp->isAvailable()); + std::unique_ptr seq(gp->newSequence()); + for (const auto& sizes : + std::vector>{{4, 4096, 64}, {4096, 4, 64}, {8192, 8, 16384, 4}, {4, 16384, 8, 8192}}) { + std::vector> device, staging; + std::vector expected; + seq->begin(); + for (size_t i = 0; i < sizes.size(); ++i) { + Bytes src(sizes[i], uint8_t(i + 31)); + expected.push_back(src); + device.emplace_back(gp->newBuffer(int(sizes[i]))); + staging.emplace_back(gp->newBuffer(int(sizes[i]), "staging")); + seq->recordUpload(device.back().get(), src.data(), src.size()); + // recordUpload promises an immediate copy, including in a multi-upload batch. + std::fill(src.begin(), src.end(), 0); + seq->recordDownload(device.back().get(), staging.back().get(), sizes[i]); + } + REQUIRE(seq->submitAsync() == eve::gpgpu::SequenceStatus::Submitted); + REQUIRE(seq->wait() == eve::gpgpu::SequenceStatus::Complete); + for (size_t i = 0; i < sizes.size(); ++i) { + Bytes actual(sizes[i]); + staging[i]->downloadBytes(actual.data(), actual.size()); + REQUIRE(actual == expected[i]); + } + } +} + +TEST_CASE("gpgpu.sequence.cancelRecordingRetiresDescriptors") { + initialize(); + auto* gp = eve::gpgpu::Gpgpu::create(); + std::unique_ptr buffer(gp->newBuffer(4)); + std::unique_ptr shader( + gp->newShader("#version 450\nlayout(local_size_x=64)in;layout(std430,binding=0)buffer B{float x[];};void " + "main(){if(gl_GlobalInvocationID.x==0)x[0]*=2.0;}")); + float value = 1; + buffer->uploadBytes(&value, 4); + for (int i = 0; i < 80; ++i) { + std::unique_ptr canceled(gp->newSequence()); + canceled->begin(); + shader->bindBuffer(0, buffer.get()); + canceled->recordDispatch(shader.get(), 1); + } + std::unique_ptr seq(gp->newSequence()); + std::unique_ptr staging(gp->newBuffer(4, "staging")); + seq->begin(); + shader->bindBuffer(0, buffer.get()); + seq->recordDispatch(shader.get(), 1); + seq->recordDownload(buffer.get(), staging.get(), 4); + seq->submit(); + staging->downloadBytes(&value, 4); + CHECK(value == 2); +} + +TEST_CASE("tensor.onnx.gpu.sessionShapesAndLifetime") { + auto* previous = eve::graphics::Graphics::create(); + auto device = std::make_unique(); + eve::ModuleManager::insert("Graphics", device.get()); + struct Restore { + eve::graphics::Graphics* previous; + ~Restore() { eve::ModuleManager::insert("Graphics", previous); } + } restore{previous}; + device->initHeadless(64, 64); + auto gpu = eve::tensor::createOnnxGpuCompute(); + REQUIRE(gpu.ok()); + auto original = eve::tensor::OnnxModel::load(chain()); + REQUIRE(original.ok()); + auto alternate = eve::tensor::OnnxModel::load(chain(2048, 2)); + REQUIRE(alternate.ok()); + for (int pass = 0; pass < 3; ++pass) { + auto& model = pass == 1 ? alternate : original; + auto r = model.value()->runGpu({}, *gpu.value()); + REQUIRE(r.ok()); + CHECK(r.value().transfers.shaderCompilations == (pass < 2 ? 1 : 0)); + const size_t count = pass == 1 ? 2048 : 4096; + const float added = pass == 1 ? 32.f : 16.f; + REQUIRE(r.value().outputs[0].tensor.bytes.size() == count * 4); + for (size_t i = 0; i < count; ++i) { + float actual; + std::memcpy(&actual, r.value().outputs[0].tensor.bytes.data() + i * 4, 4); + REQUIRE(actual == float(i) + added); + } + } + auto failing = eve::tensor::OnnxModel::load(chain(4096, 1, true)); + REQUIRE(failing.ok()); + auto failure = failing.value()->runGpu({}, *gpu.value()); + REQUIRE(!failure.ok()); + const std::vector intermediate{"v15alias"}; + auto recovered = failing.value()->runGpu({}, *gpu.value(), intermediate); + REQUIRE(recovered.ok()); + CHECK(recovered.value().transfers.shaderCompilations == 0); + CHECK(recovered.value().transfers.uploads == 2); + float recoveredValue = 0; + std::memcpy(&recoveredValue, recovered.value().outputs[0].tensor.bytes.data(), 4); + CHECK(recoveredValue == 16); + // Session-first retirement cancels its observer and releases device resources. + { + auto second = eve::tensor::createOnnxGpuCompute(); + REQUIRE(second.ok()); + auto r = original.value()->runGpu({}, *second.value()); + REQUIRE(r.ok()); + } + // Device-first retirement clears the surviving session before destroying Vulkan. + device.reset(); + eve::ModuleManager::insert("Graphics", previous); + auto stale = original.value()->runGpu({}, *gpu.value()); + REQUIRE(!stale.ok()); + CHECK(stale.error()->message().find("retired") != std::string::npos); + gpu.value().reset(); +} +TEST_CASE("tensor.onnx.gpu.parallelCompilation") { + // Compilation has no dependency on a live Graphics device. + auto bytecode = eve::gpgpu::compileComputeSpirv("#version 450\nlayout(local_size_x=64)in;void main(){}"); + REQUIRE(bytecode.ok()); + REQUIRE(bytecode.value().front() == 0x07230203); + auto invalidHeader = eve::gpgpu::createComputeShader({}); + REQUIRE(!invalidHeader.ok()); + for (uint32_t invalid : {0u, 9u}) { + auto r = eve::tensor::createOnnxGpuCompute(invalid); + REQUIRE(!r.ok()); + } + initialize(); + Bytes serial; + for (uint32_t workers : {1u, 4u}) { + auto provider = eve::tensor::createOnnxGpuCompute(workers); + REQUIRE(provider.ok()); + auto model = eve::tensor::OnnxModel::load(chain(4096, 2, false, true, 80)); + REQUIRE(model.ok()); + for (int repeat = 0; repeat < 2; ++repeat) { + auto r = model.value()->runGpu({}, *provider.value()); + REQUIRE(r.ok()); + CHECK(r.value().dispatches == 80); + CHECK(r.value().transfers.shaderCompilations == (repeat ? 0 : 4)); + CHECK(r.value().transfers.compilerPeakWorkers <= workers); + if (!repeat) + CHECK(r.value().transfers.compilerPeakWorkers >= (workers == 1 ? 1 : 2)); + else + CHECK(r.value().transfers.compilerPeakWorkers == 0); + const auto& raw = r.value().outputs[0].tensor.bytes; + for (size_t i = 0; i < 4096; ++i) { + float actual; + std::memcpy(&actual, raw.data() + i * 4, 4); + REQUIRE(actual == float(i + 20)); + } + if (serial.empty()) + serial = raw; + else + REQUIRE(raw == serial); + } + // A fresh, abandoned segment has active CPU compile jobs but no submitted + // GPU work. Retry must rebuild those programs and re-upload its weights. + auto canceled = eve::tensor::OnnxModel::load(chain(4093, 2, true, true)); + REQUIRE(canceled.ok()); + auto failed = canceled.value()->runGpu({}, *provider.value()); + REQUIRE(!failed.ok()); + const std::vector requested{"v15alias"}; + auto retried = canceled.value()->runGpu({}, *provider.value(), requested); + REQUIRE(retried.ok()); + CHECK(retried.value().transfers.shaderCompilations == 4); + CHECK(retried.value().transfers.uploads == 2); + // Compilation failure travels through the future and run cleanup; + // subsequent valid work on the same session remains usable. + eve::tensor::OnnxKernel bad{"#version 450\nvoid main(){missing_function();}", {}, 4, 1}; + auto rejected = provider.value()->dispatch(bad); + REQUIRE(!rejected.ok()); + auto recovered = model.value()->runGpu({}, *provider.value()); + REQUIRE(recovered.ok()); + REQUIRE(recovered.value().outputs[0].tensor.bytes == serial); + } +} +#endif