#include "zeroerr/assert.h" #include "zeroerr/unittest.h" #include "common/Exception.h" #include "tensor/GpuBackend.h" #include "tensor/Graph.h" #include "tensor/KernelGen.h" #include "tensor/Optimizer.h" #include "tensor/TF.h" #include "tensor/Tensor.h" #include "graphics/AmbientOcclusion.h" #include "graphics/AntiAliasing.h" #include "graphics/Canvas.h" #include "graphics/DrawItem2D.h" #include "graphics/Font.h" #include "graphics/GBuffer.h" #include "graphics/GlobalIllumination.h" #include "graphics/Graphics.h" #include "graphics/Grass.h" #include "graphics/Light.h" #include "graphics/Material.h" #include "graphics/Mesh.h" #include "graphics/Outline.h" #include "graphics/Quad.h" #include "graphics/RenderControl.h" #include "graphics/ScreenSpaceReflection.h" #include "graphics/Shader.h" #include "graphics/Texture.h" #include "graphics/Volumetric.h" #include "graphics/Water.h" #include "graphics/Waterfall.h" #include "window/Window.h" #include #include #include #include using namespace eve::tensor; namespace { using Operation = std::function; void compareKernel(const char *name, Tensor *input, const Operation &operation, zeroerr::TestContext *_ZEROERR_TEST_CONTEXT) { std::printf("GPU kernel contract: %s\n", name); auto *tf = TF::create(); std::unique_ptr fn(tf->func()); Tensor *symbolic = nullptr; switch (input->getRank()) { case 1: symbolic = fn->input1(input->getDim0()); break; case 2: symbolic = fn->input2(input->getDim0(), input->getDim1()); break; case 3: symbolic = fn->input3(input->getDim0(), input->getDim1(), input->getDim2()); break; case 4: symbolic = fn->input4(input->getDim0(), input->getDim1(), input->getDim2(), input->getDim3()); break; default: REQUIRE(false); return; } std::unique_ptr traced(operation(symbolic)); fn->setOutput(traced.get()); std::unique_ptr compiled(fn->compile()); REQUIRE_EQ(compiled->getDevice(), std::string("gpu")); // Changing the feed detects stale uploads and arena contents on a reused program. for (int repeat = 0; repeat < 2; ++repeat) { std::unique_ptr feed(input->mulScalar(repeat == 0 ? 1.f : 0.5f)); std::unique_ptr expected(operation(feed.get())); std::unique_ptr actual(compiled->run1(feed.get())); REQUIRE_EQ(actual->getSize(), expected->getSize()); for (int i = 0; i < expected->getSize(); ++i) { REQUIRE(std::isfinite(actual->get(i))); REQUIRE(std::fabs(actual->get(i) - expected->get(i)) < 1e-3f * (1.f + std::fabs(expected->get(i)))); } } } void fillPattern(Tensor *tensor) { for (int i = 0; i < tensor->getSize(); ++i) tensor->set(i, float((i * 7) % 19 - 9) / 10.f); } } // namespace TEST_CASE("tensor.gpu.kernelFamiliesMatchEager") { auto *gfx = eve::graphics::Graphics::create(); REQUIRE(gfx != nullptr); gfx->initHeadless(64, 64); auto *tf = TF::create(); std::unique_ptr matrix(tf->zeros2(4, 3)); fillPattern(matrix.get()); const auto check = [&](const char *name, const Operation &op) { compareKernel(name, matrix.get(), op, _ZEROERR_TEST_CONTEXT); }; check("sum axis 0", [&](Tensor *x) { return tf->sumAxis(x, 0, 0); }); check("mean axis 1", [&](Tensor *x) { return tf->meanAxis(x, 1, 0); }); check("min axis 0", [&](Tensor *x) { return tf->minAxis(x, 0, 0); }); check("max axis 1", [&](Tensor *x) { return tf->maxAxis(x, 1, 0); }); check("argmax", [&](Tensor *x) { return tf->argmax(x, 1, 0); }); check("softmax noncontiguous axis", [&](Tensor *x) { return tf->softmax(x, 0); }); check("log softmax", [&](Tensor *x) { return tf->logSoftmax(x, 1); }); check("rms norm", [&](Tensor *x) { return tf->rmsnorm(x, 1e-5f); }); check("permute", [&](Tensor *x) { return tf->permute2(x, 1, 0); }); check("slice", [&](Tensor *x) { return tf->slice(x, 1, 1, 3); }); std::unique_ptr scale(tf->fill1(3, 1.2f)), bias(tf->fill1(3, -0.2f)); check("scaled rms norm", [&](Tensor *x) { return tf->rmsnormW(x, scale.get(), 1e-5f); }); check("affine layer norm", [&](Tensor *x) { return tf->layernormWB(x, scale.get(), bias.get(), 1e-5f); }); std::unique_ptr other(tf->fill2(1, 3, 0.4f)); check("concat", [&](Tensor *x) { return tf->concat2(x, other.get(), 0); }); check("broadcast addition", [&](Tensor *x) { return tf->add(x, scale.get()); }); std::unique_ptr selected(tf->fill2(4, 3, 1.2f)), rejected(tf->fill2(4, 3, -0.2f)); check("where", [&](Tensor *x) { return tf->where(x, selected.get(), rejected.get()); }); std::unique_ptr indices(tf->zeros1(3)); indices->set(0, -1.f); indices->set(1, 2.f); indices->set(2, 5.f); check("embedding clamps indices", [&](Tensor *x) { return tf->embedding(x, indices.get()); }); std::unique_ptr projection(tf->zeros2(3, 5)), projectionBias(tf->fill1(5, 0.15f)); fillPattern(projection.get()); check("matrix bias activation with edge tiles", [&](Tensor *x) { std::unique_ptr product(tf->matmul(x, projection.get())); std::unique_ptr biased(tf->add(product.get(), projectionBias.get())); return tf->relu(biased.get()); }); std::unique_ptr attention(tf->zeros4(1, 1, 3, 5)), mask(tf->zeros4(1, 1, 3, 3)); fillPattern(attention.get()); mask->set(1, -1000.f); mask->set(2, -1000.f); mask->set(5, -1000.f); compareKernel( "masked attention with aliased inputs", attention.get(), [&](Tensor *x) { return tf->sdpaMasked(x, x, x, mask.get(), 0.4472136f); }, _ZEROERR_TEST_CONTEXT); std::unique_ptr image(tf->zeros4(1, 1, 5, 6)); fillPattern(image.get()); compareKernel( "max pool with padding", image.get(), [&](Tensor *x) { return tf->maxpool2d(x, 2, 1, 1); }, _ZEROERR_TEST_CONTEXT); compareKernel( "average pool with padding", image.get(), [&](Tensor *x) { return tf->avgpool2d(x, 2, 1, 1); }, _ZEROERR_TEST_CONTEXT); for (int mode : {0, 1}) compareKernel( mode == 0 ? "nearest resize" : "bilinear resize", image.get(), [&](Tensor *x) { return tf->resize2d(x, 7, 3, mode); }, _ZEROERR_TEST_CONTEXT); std::unique_ptr signal(tf->zeros3(1, 1, 7)), weights(tf->fill3(2, 1, 3, 0.2f)), convBias(tf->fill1(2, -0.1f)); convBias->set(1, 0.4f); fillPattern(signal.get()); compareKernel( "conv1d bias and padding", signal.get(), [&](Tensor *x) { return tf->conv1dBias(x, weights.get(), convBias.get(), 2, 1); }, _ZEROERR_TEST_CONTEXT); std::unique_ptr spatialWeights(tf->fill4(2, 1, 3, 3, 0.2f)), columnBias(tf->fill1(6, 0.3f)); fillPattern(columnBias.get()); compareKernel( "conv2d channel bias and fused column bias", image.get(), [&](Tensor *x) { std::unique_ptr convolution(tf->conv2dBias(x, spatialWeights.get(), convBias.get(), 1, 1)); std::unique_ptr biased(tf->add(convolution.get(), columnBias.get())); return tf->relu(biased.get()); }, _ZEROERR_TEST_CONTEXT); std::vector values(513); float sum = 0.f; for (size_t i = 0; i < values.size(); ++i) { values[i] = float(int(i % 7) - 3); sum += values[i]; } for (int op : {0, 1, 2}) { float actual = 0.f; const bool executed = gpuReduce(values.data(), int(values.size()), op, actual); REQUIRE(executed); const float expected = op == 0 ? sum : (op == 1 ? -3.f : 3.f); REQUIRE(std::fabs(actual - expected) < 1e-4f); } } TEST_CASE("tensor.gpu.quantizedWeightsMatchEager") { auto *gfx = eve::graphics::Graphics::create(); REQUIRE(gfx != nullptr); gfx->initHeadless(64, 64); auto *tf = TF::create(); std::unique_ptr input(tf->zeros2(3, 5)), weights(tf->zeros2(5, 7)), indices(tf->zeros1(3)); fillPattern(input.get()); fillPattern(weights.get()); indices->set(0, 0.f); indices->set(1, 3.f); indices->set(2, 4.f); for (const char *dtype : {"fp16", "fp8", "fp4", "int8", "int4"}) { std::unique_ptr packed(tf->quantizeWeight(weights.get(), dtype, 8)); compareKernel( dtype, input.get(), [&](Tensor *x) { return tf->matmul(x, packed.get()); }, _ZEROERR_TEST_CONTEXT); compareKernel( dtype, indices.get(), [&](Tensor *x) { return tf->embedding(packed.get(), x); }, _ZEROERR_TEST_CONTEXT); } }