[kernel] Integrate FlashInfer BF16 CuTeDSL Low Latency GEMM (#50572)

Signed-off-by: jiahanc <173873397+jiahanc@users.noreply.github.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
This commit is contained in:
Jiahan Chang (Cyrus)
2026-08-27 22:29:12 +00:00
committed by GitHub
co-authored by OpenAI Codex Jee Jee Li
parent f79a2f5582
commit 7c877062ac
7 changed files with 349 additions and 23 deletions
@@ -0,0 +1,49 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
import torch.nn.functional as F
from vllm.model_executor.layers import utils as layer_utils
from vllm.utils.flashinfer import is_flashinfer_cutedsl_bf16_gemm_supported
@pytest.fixture(scope="module", autouse=True)
def require_flashinfer_bf16_cutedsl() -> None:
if not torch.accelerator.is_available():
pytest.skip("CUDA is required")
if not is_flashinfer_cutedsl_bf16_gemm_supported():
pytest.skip("FlashInfer BF16 cute-dsl backend is unavailable")
@pytest.mark.parametrize("n,k", [(1024, 1024), (2048, 3072)])
@pytest.mark.parametrize(
"m,use_bias,pdl",
[
(1, False, False),
(8, True, False),
(16, False, True),
(32, True, True),
],
)
def test_flashinfer_bf16_cutedsl_correctness(
m: int,
n: int,
k: int,
use_bias: bool,
pdl: bool,
) -> None:
torch.manual_seed(0)
x = torch.randn(m, k, device="cuda", dtype=torch.bfloat16) * 0.1
weight = torch.randn(n, k, device="cuda", dtype=torch.bfloat16) * 0.1
bias = (
torch.randn(n, device="cuda", dtype=torch.bfloat16) * 0.1 if use_bias else None
)
actual = layer_utils.cuda_flashinfer_bf16_gemm_impl(
x, weight, bias, pdl, "flashinfer_cutedsl"
)
expected = F.linear(x, weight, bias)
torch.testing.assert_close(actual, expected, rtol=2e-2, atol=2e-1)
+6 -2
View File
@@ -267,12 +267,16 @@ class KernelConfig:
"""
linear_backend: LinearBackend = "auto"
"""Backend for quantized linear layer GEMM kernels. Available options:
"""Backend for linear layer GEMM kernels. Available options:
Layer types without an implementation from the requested backend use
automatic selection.
- "auto": Automatically select the best backend based on model and hardware
- "cutlass": Use CUTLASS-based kernels
- "flashinfer_cutlass": Use FlashInfer with CUTLASS kernels
- "flashinfer_cutedsl": Use FlashInfer with CuTe-DSL kernels (NVFP4, MXFP8)
- "flashinfer_cutedsl": Use FlashInfer with CuTe-DSL kernels
(BF16, NVFP4, MXFP8)
- "flashinfer_trtllm": Use FlashInfer with TensorRT-LLM kernels
- "flashinfer_cudnn": Use FlashInfer with cuDNN kernels
- "flashinfer_b12x": Use FlashInfer b12x CuteDSL NVFP4 GEMM (SM120+)
+11 -6
View File
@@ -237,10 +237,7 @@ def _get_linear_backend() -> str:
return "auto"
# Mapping from linear_backend name to the set of kernel classes it covers.
# When a user sets --linear-backend <name>, only kernels in the corresponding
# set are considered candidates. If none can implement the layer config,
# an error is raised to respect the user's explicit intent.
# Kernel classes covered by each --linear-backend value.
_LINEAR_BACKEND_KERNEL_MAP: dict[str, set[type]] = {
"b12x": {
B12xFp8BlockScaledMMKernel,
@@ -345,9 +342,17 @@ def _filter_kernels_by_backend(
backend: str,
kernels: list[type],
) -> list[type]:
"""Filter a kernel priority list to only those matching the backend."""
"""Prefer matching kernels, falling back to auto by layer type."""
backend_kernels = _LINEAR_BACKEND_KERNEL_MAP.get(backend, set())
return [k for k in kernels if k in backend_kernels]
filtered = [kernel for kernel in kernels if kernel in backend_kernels]
if not filtered:
logger.warning_once(
"--linear-backend=%s has no kernel for this linear layer type; "
"using automatic selection for that layer type.",
backend,
)
return kernels
return filtered
def _resolve_backend_kernels(
+9 -2
View File
@@ -10,7 +10,7 @@ from torch.nn.parameter import Parameter
from typing_extensions import TypeIs
import vllm.envs as envs
from vllm.config import get_current_vllm_config
from vllm.config import get_current_vllm_config, get_current_vllm_config_or_none
from vllm.distributed import (
divide,
get_tensor_model_parallel_rank,
@@ -166,6 +166,13 @@ class LinearMethodBase(QuantizeMethodBase):
class UnquantizedLinearMethod(LinearMethodBase):
"""Linear method without quantization."""
def __init__(self) -> None:
config = get_current_vllm_config_or_none()
linear_backend = (
config.kernel_config.linear_backend if config is not None else "auto"
)
self._gemm_impl = dispatch_unquantized_gemm(linear_backend)
def create_weights(
self,
layer: torch.nn.Module,
@@ -215,7 +222,7 @@ class UnquantizedLinearMethod(LinearMethodBase):
) -> torch.Tensor:
if envs.VLLM_BATCH_INVARIANT and current_platform.is_cuda_alike():
return linear_batch_invariant(x, layer.weight, bias)
return dispatch_unquantized_gemm()(layer, x, layer.weight, bias)
return self._gemm_impl(layer, x, layer.weight, bias)
class LinearBase(PluggableLayer):
+172 -2
View File
@@ -2,7 +2,9 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Utility methods for model layers."""
import functools
from collections.abc import Callable
from dataclasses import dataclass
import torch
@@ -11,6 +13,10 @@ from vllm import envs
from vllm._aiter_ops import rocm_aiter_ops
from vllm.logger import init_logger
from vllm.platforms import CpuArchEnum, current_platform
from vllm.utils.flashinfer import (
flashinfer_bf16_mm,
is_flashinfer_cutedsl_bf16_gemm_supported,
)
from vllm.utils.platform_utils import num_compute_units
from vllm.utils.torch_utils import direct_register_custom_op
@@ -92,6 +98,144 @@ def default_unquantized_gemm(
return torch.nn.functional.linear(x, weight, bias)
_FlashInferBf16RuntimeCheck = Callable[
[torch.Tensor, torch.Tensor, torch.Tensor | None], bool
]
@dataclass(frozen=True)
class _FlashInferBf16Backend:
flashinfer_backend: str
is_supported: Callable[[], bool]
can_implement: _FlashInferBf16RuntimeCheck
def _can_use_flashinfer_cutedsl_bf16(
x: torch.Tensor,
weight: torch.Tensor,
bias: torch.Tensor | None,
) -> bool:
if not (
current_platform.is_cuda() and current_platform.is_device_capability_family(100)
):
return False
if x.ndim < 1 or weight.ndim != 2:
return False
if (
not x.is_cuda
or not weight.is_cuda
or x.device != weight.device
or x.dtype != torch.bfloat16
or weight.dtype != torch.bfloat16
or not x.is_contiguous()
or not weight.is_contiguous()
):
return False
k = x.shape[-1]
n = weight.shape[0]
if (
k <= 0
or n <= 0
or weight.shape[1] != k
or k % 128 != 0
or x.data_ptr() % 32 != 0
or weight.data_ptr() % 32 != 0
):
return False
m = x.numel() // k
if not 1 <= m <= 32:
return False
return bias is None or (
bias.is_cuda
and bias.device == x.device
and bias.dtype == torch.bfloat16
and bias.ndim == 1
and bias.shape[0] == n
and bias.is_contiguous()
)
_FLASHINFER_BF16_BACKENDS = {
"flashinfer_cutedsl": _FlashInferBf16Backend(
flashinfer_backend="cute-dsl",
is_supported=is_flashinfer_cutedsl_bf16_gemm_supported,
can_implement=_can_use_flashinfer_cutedsl_bf16,
),
}
def _get_flashinfer_bf16_backend(vllm_backend: str) -> _FlashInferBf16Backend:
backend_spec = _FLASHINFER_BF16_BACKENDS.get(vllm_backend)
if backend_spec is None:
supported = ", ".join(sorted(_FLASHINFER_BF16_BACKENDS))
raise ValueError(
f"Unsupported vLLM FlashInfer BF16 backend {vllm_backend!r}; "
f"supported backends: {supported}"
)
return backend_spec
def cuda_flashinfer_bf16_gemm_impl(
x: torch.Tensor,
weight: torch.Tensor,
bias: torch.Tensor | None,
pdl: bool,
vllm_backend: str,
) -> torch.Tensor:
backend_spec = _get_flashinfer_bf16_backend(vllm_backend)
if not backend_spec.can_implement(x, weight, bias):
return torch.nn.functional.linear(x, weight, bias)
k = x.shape[-1]
n = weight.shape[0]
x_2d = x.view(-1, k)
out_2d = flashinfer_bf16_mm(
x_2d,
weight.t(),
bias,
pdl,
backend_spec.flashinfer_backend,
)
return out_2d.view(*x.shape[:-1], n)
def cuda_flashinfer_bf16_gemm_fake(
x: torch.Tensor,
weight: torch.Tensor,
bias: torch.Tensor | None,
pdl: bool,
vllm_backend: str,
) -> torch.Tensor:
return x.new_empty((*x.shape[:-1], weight.shape[0]))
def cuda_flashinfer_bf16_gemm(
layer: torch.nn.Module,
x: torch.Tensor,
weight: torch.Tensor,
bias: torch.Tensor | None = None,
*,
vllm_backend: str,
pdl: bool,
) -> torch.Tensor:
return torch.ops.vllm.cuda_flashinfer_bf16_gemm(
x,
weight,
bias,
pdl,
vllm_backend,
)
direct_register_custom_op(
op_name="cuda_flashinfer_bf16_gemm",
op_func=cuda_flashinfer_bf16_gemm_impl,
fake_impl=cuda_flashinfer_bf16_gemm_fake,
)
def use_aiter_triton_gemm(n, m, k, dtype):
if (
not rocm_aiter_ops.is_triton_gemm_enabled()
@@ -391,10 +535,36 @@ def cpu_unquantized_gemm(
return layer.cpu_linear(x, weight, bias)
def dispatch_unquantized_gemm() -> Callable[..., torch.Tensor]:
def dispatch_unquantized_gemm(
linear_backend: str = "auto",
) -> Callable[..., torch.Tensor]:
if current_platform.is_rocm():
return rocm_unquantized_gemm
elif current_platform.is_cpu():
return cpu_unquantized_gemm
else:
elif not current_platform.is_cuda():
return default_unquantized_gemm
backend_spec = _FLASHINFER_BF16_BACKENDS.get(linear_backend)
if backend_spec is None:
return default_unquantized_gemm
if not backend_spec.is_supported():
logger.warning_once(
"--linear-backend=%s requested FlashInfer mm_bf16 backend %r, "
"but it is unavailable on the current hardware or environment; "
"using automatic selection for unquantized linear layers.",
linear_backend,
backend_spec.flashinfer_backend,
)
return default_unquantized_gemm
logger.info_once(
"Using FlashInfer %s for eligible unquantized BF16 GEMMs.",
backend_spec.flashinfer_backend,
)
return functools.partial(
cuda_flashinfer_bf16_gemm,
vllm_backend=linear_backend,
pdl=current_platform.is_arch_support_pdl(),
)
+26 -11
View File
@@ -247,6 +247,31 @@ def _flashinfer_autotune_skip_ops(runner: "GPUModelRunner") -> set[str] | None:
return None
_FLASHINFER_BF16_AUTOTUNE_MAX_TOKENS = 32
def _flashinfer_autotune_token_counts(runner: "GPUModelRunner") -> tuple[int, ...]:
max_tokens = runner.scheduler_config.max_num_batched_tokens
linear_backend = runner.vllm_config.kernel_config.linear_backend
if (
linear_backend == "flashinfer_cutedsl"
and max_tokens > _FLASHINFER_BF16_AUTOTUNE_MAX_TOKENS
):
return max_tokens, _FLASHINFER_BF16_AUTOTUNE_MAX_TOKENS
return (max_tokens,)
def _run_flashinfer_autotune_dummy_runs(runner: "GPUModelRunner") -> None:
for num_tokens in _flashinfer_autotune_token_counts(runner):
logger.info("Running FlashInfer autotune with %d tokens.", num_tokens)
runner._dummy_run(
num_tokens=num_tokens,
skip_eplb=True,
is_profile=True,
randomize_inputs=True,
)
def flashinfer_autotune(runner: "GPUModelRunner") -> None:
"""
Autotune FlashInfer operations.
@@ -284,20 +309,10 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
logger.info_once("Using FlashInfer autotune cache file: %s", cache_path)
# We skip EPLB here since we don't want to record dummy metrics.
# When autotuning with number of tokens m, flashinfer will autotune
# operations for all number of tokens up to m, so we only need to
# run with the max number of tokens.
# Randomize inputs to avoid every token pick the same experts,
# which lead to some EP ranks receiving no tokens and skipping their
# MoE kernel entirely, and cause hang due to all-reduce collective
# during synchronized autotuning.
dummy_run_kwargs = dict(
num_tokens=runner.scheduler_config.max_num_batched_tokens,
skip_eplb=True,
is_profile=True,
randomize_inputs=True,
)
# Read cached autotune results and broadcast to all ranks.
cached_results: bytes | None = None
if is_leader and cache_path.exists():
@@ -316,7 +331,7 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
torch.inference_mode(),
fi_utils.autotune(tune_mode=True, **autotune_kwargs),
):
runner._dummy_run(**dummy_run_kwargs)
_run_flashinfer_autotune_dummy_runs(runner)
finally:
set_autotune_process_group(None)
+76
View File
@@ -24,6 +24,26 @@ from vllm.utils.math_utils import cdiv
logger = init_logger(__name__)
def flashinfer_bf16_mm(
a: torch.Tensor,
b: torch.Tensor,
bias: torch.Tensor | None,
pdl: bool,
backend: str,
) -> torch.Tensor:
from flashinfer import mm_bf16
return mm_bf16(
a,
b,
bias=bias,
pdl=pdl,
out_dtype=torch.bfloat16,
backend=backend,
)
# This is the storage path for the cubins, it can be replaced
# with a local path for testing.
# Referenced from https://github.com/flashinfer-ai/flashinfer/blob/0c9a92c3d9a7e043ab6f3f7b2273269caf6ab044/flashinfer/jit/cubin_loader.py#L35 # noqa: E501
@@ -63,6 +83,58 @@ def has_flashinfer() -> bool:
return True
@functools.cache
def has_flashinfer_bf16_gemm() -> bool:
"""Return whether FlashInfer exposes the BF16 dense GEMM API."""
if not has_flashinfer():
return False
mod = _get_submodule("flashinfer")
return mod is not None and callable(getattr(mod, "mm_bf16", None))
@functools.cache
def is_flashinfer_bf16_gemm_supported(
backend: str,
compute_capability: int | None = None,
) -> bool:
"""Return whether an exact FlashInfer BF16 backend is available."""
if not current_platform.is_cuda() or not has_flashinfer_bf16_gemm():
return False
mod = _get_submodule("flashinfer")
mm_bf16 = getattr(mod, "mm_bf16", None) if mod is not None else None
backend_supported = getattr(mm_bf16, "is_backend_supported", None)
if not callable(backend_supported):
return False
if compute_capability is None:
device_capability = current_platform.get_device_capability()
if device_capability is None:
return False
compute_capability = device_capability.to_int()
try:
return bool(backend_supported(backend, compute_capability))
except (AttributeError, ImportError, RuntimeError, TypeError, ValueError):
return False
@functools.cache
def is_flashinfer_cutedsl_bf16_gemm_supported() -> bool:
"""Return whether the CuTeDSL BF16 dense GEMM backend is available."""
if not is_flashinfer_bf16_gemm_supported("cute-dsl"):
return False
try:
from flashinfer.cute_dsl.utils import is_cute_dsl_available
from flashinfer.utils import is_sm100a_supported
except (ImportError, ModuleNotFoundError):
return False
try:
return is_cute_dsl_available() and is_sm100a_supported(torch.device("cuda"))
except (RuntimeError, TypeError, ValueError):
return False
def _missing(*_: Any, **__: Any) -> NoReturn:
"""Placeholder for unavailable FlashInfer backend."""
raise RuntimeError(
@@ -1079,6 +1151,10 @@ def is_flashinfer_cudnn_fp8_prefill_attn_supported() -> bool:
__all__ = [
"has_flashinfer",
"flashinfer_bf16_mm",
"has_flashinfer_bf16_gemm",
"is_flashinfer_bf16_gemm_supported",
"is_flashinfer_cutedsl_bf16_gemm_supported",
"flashinfer_trtllm_fp8_block_scale_moe",
"flashinfer_cutlass_fused_moe",
"flashinfer_cutedsl_grouped_gemm_nt_masked",