[kernel] Integrate FlashInfer BF16 CuTeDSL Low Latency GEMM (#50572)
Signed-off-by: jiahanc <173873397+jiahanc@users.noreply.github.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
This commit is contained in:
co-authored by
OpenAI Codex
Jee Jee Li
parent
f79a2f5582
commit
7c877062ac
@@ -0,0 +1,49 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm.model_executor.layers import utils as layer_utils
|
||||
from vllm.utils.flashinfer import is_flashinfer_cutedsl_bf16_gemm_supported
|
||||
|
||||
|
||||
@pytest.fixture(scope="module", autouse=True)
|
||||
def require_flashinfer_bf16_cutedsl() -> None:
|
||||
if not torch.accelerator.is_available():
|
||||
pytest.skip("CUDA is required")
|
||||
if not is_flashinfer_cutedsl_bf16_gemm_supported():
|
||||
pytest.skip("FlashInfer BF16 cute-dsl backend is unavailable")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("n,k", [(1024, 1024), (2048, 3072)])
|
||||
@pytest.mark.parametrize(
|
||||
"m,use_bias,pdl",
|
||||
[
|
||||
(1, False, False),
|
||||
(8, True, False),
|
||||
(16, False, True),
|
||||
(32, True, True),
|
||||
],
|
||||
)
|
||||
def test_flashinfer_bf16_cutedsl_correctness(
|
||||
m: int,
|
||||
n: int,
|
||||
k: int,
|
||||
use_bias: bool,
|
||||
pdl: bool,
|
||||
) -> None:
|
||||
torch.manual_seed(0)
|
||||
x = torch.randn(m, k, device="cuda", dtype=torch.bfloat16) * 0.1
|
||||
weight = torch.randn(n, k, device="cuda", dtype=torch.bfloat16) * 0.1
|
||||
bias = (
|
||||
torch.randn(n, device="cuda", dtype=torch.bfloat16) * 0.1 if use_bias else None
|
||||
)
|
||||
|
||||
actual = layer_utils.cuda_flashinfer_bf16_gemm_impl(
|
||||
x, weight, bias, pdl, "flashinfer_cutedsl"
|
||||
)
|
||||
expected = F.linear(x, weight, bias)
|
||||
|
||||
torch.testing.assert_close(actual, expected, rtol=2e-2, atol=2e-1)
|
||||
Reference in New Issue
Block a user