[Kernel][Performance] Add FlashInfer cutedsl NVFP4 GEMM backend (#42235)
Signed-off-by: Mohammad Miadh Angkad <176301910+mmangkad@users.noreply.github.com>
This commit is contained in:
@@ -75,7 +75,7 @@ def get_ref_results(
|
||||
@pytest.mark.parametrize("shape", SHAPES)
|
||||
@pytest.mark.parametrize("seed", SEEDS)
|
||||
@pytest.mark.parametrize("device", CUDA_DEVICES)
|
||||
@pytest.mark.parametrize("backend", ["cutlass", "cudnn", "trtllm", "b12x"])
|
||||
@pytest.mark.parametrize("backend", ["cute-dsl", "cutlass", "cudnn", "trtllm", "b12x"])
|
||||
@pytest.mark.parametrize("autotune", [False, True])
|
||||
@torch.inference_mode()
|
||||
def test_flashinfer_nvfp4_gemm(
|
||||
@@ -88,6 +88,8 @@ def test_flashinfer_nvfp4_gemm(
|
||||
) -> None:
|
||||
if "trtllm" in backend and dtype == torch.float16:
|
||||
pytest.skip("Only torch.bfloat16 is supported for TRTLLM FP4 GEMM operations")
|
||||
if backend == "cute-dsl" and not current_platform.is_device_capability_family(100):
|
||||
pytest.skip("FlashInfer cutedsl backend is only supported on SM10x")
|
||||
if backend == "b12x" and not current_platform.has_device_capability(120):
|
||||
pytest.skip("b12x FP4 GEMM requires SM120+ (CC 12.0+)")
|
||||
if backend == "b12x" and not has_flashinfer_b12x_gemm():
|
||||
|
||||
@@ -90,6 +90,7 @@ def test_models(example_prompts, model_name) -> None:
|
||||
EAGER = [True, False]
|
||||
|
||||
SM_100_NVFP4_BACKENDS = [
|
||||
"flashinfer_cutedsl",
|
||||
"flashinfer_cudnn",
|
||||
"flashinfer_trtllm",
|
||||
"flashinfer_cutlass",
|
||||
@@ -102,12 +103,18 @@ SM_100_NVFP4_BACKENDS = [
|
||||
"backend",
|
||||
[
|
||||
"emulation",
|
||||
"flashinfer_cutedsl",
|
||||
"flashinfer_cudnn",
|
||||
"flashinfer_trtllm", # the small seq_len ensures trtllm_8x4_layout backend is used
|
||||
"flashinfer_cutlass",
|
||||
],
|
||||
)
|
||||
def test_nvfp4(vllm_runner, model, eager, backend):
|
||||
if backend == "flashinfer_cutedsl" and not (
|
||||
current_platform.is_device_capability_family(100)
|
||||
):
|
||||
pytest.skip("The flashinfer_cutedsl backend is only supported on SM10x")
|
||||
|
||||
if (
|
||||
not current_platform.has_device_capability(100)
|
||||
and backend in SM_100_NVFP4_BACKENDS
|
||||
|
||||
Reference in New Issue
Block a user