[kernel] Integrate FlashInfer BF16 CuTeDSL Low Latency GEMM (#50572)

Signed-off-by: jiahanc <173873397+jiahanc@users.noreply.github.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
This commit is contained in:
Jiahan Chang (Cyrus)
2026-08-27 22:29:12 +00:00
committed by GitHub
co-authored by OpenAI Codex Jee Jee Li
parent f79a2f5582
commit 7c877062ac
7 changed files with 349 additions and 23 deletions
@@ -0,0 +1,49 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
import torch.nn.functional as F
from vllm.model_executor.layers import utils as layer_utils
from vllm.utils.flashinfer import is_flashinfer_cutedsl_bf16_gemm_supported
@pytest.fixture(scope="module", autouse=True)
def require_flashinfer_bf16_cutedsl() -> None:
if not torch.accelerator.is_available():
pytest.skip("CUDA is required")
if not is_flashinfer_cutedsl_bf16_gemm_supported():
pytest.skip("FlashInfer BF16 cute-dsl backend is unavailable")
@pytest.mark.parametrize("n,k", [(1024, 1024), (2048, 3072)])
@pytest.mark.parametrize(
"m,use_bias,pdl",
[
(1, False, False),
(8, True, False),
(16, False, True),
(32, True, True),
],
)
def test_flashinfer_bf16_cutedsl_correctness(
m: int,
n: int,
k: int,
use_bias: bool,
pdl: bool,
) -> None:
torch.manual_seed(0)
x = torch.randn(m, k, device="cuda", dtype=torch.bfloat16) * 0.1
weight = torch.randn(n, k, device="cuda", dtype=torch.bfloat16) * 0.1
bias = (
torch.randn(n, device="cuda", dtype=torch.bfloat16) * 0.1 if use_bias else None
)
actual = layer_utils.cuda_flashinfer_bf16_gemm_impl(
x, weight, bias, pdl, "flashinfer_cutedsl"
)
expected = F.linear(x, weight, bias)
torch.testing.assert_close(actual, expected, rtol=2e-2, atol=2e-1)