[kernel] Integrate FlashInfer BF16 CuTeDSL Low Latency GEMM (#50572)
Signed-off-by: jiahanc <173873397+jiahanc@users.noreply.github.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
This commit is contained in:
co-authored by
OpenAI Codex
Jee Jee Li
parent
f79a2f5582
commit
7c877062ac
@@ -0,0 +1,49 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm.model_executor.layers import utils as layer_utils
|
||||
from vllm.utils.flashinfer import is_flashinfer_cutedsl_bf16_gemm_supported
|
||||
|
||||
|
||||
@pytest.fixture(scope="module", autouse=True)
|
||||
def require_flashinfer_bf16_cutedsl() -> None:
|
||||
if not torch.accelerator.is_available():
|
||||
pytest.skip("CUDA is required")
|
||||
if not is_flashinfer_cutedsl_bf16_gemm_supported():
|
||||
pytest.skip("FlashInfer BF16 cute-dsl backend is unavailable")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("n,k", [(1024, 1024), (2048, 3072)])
|
||||
@pytest.mark.parametrize(
|
||||
"m,use_bias,pdl",
|
||||
[
|
||||
(1, False, False),
|
||||
(8, True, False),
|
||||
(16, False, True),
|
||||
(32, True, True),
|
||||
],
|
||||
)
|
||||
def test_flashinfer_bf16_cutedsl_correctness(
|
||||
m: int,
|
||||
n: int,
|
||||
k: int,
|
||||
use_bias: bool,
|
||||
pdl: bool,
|
||||
) -> None:
|
||||
torch.manual_seed(0)
|
||||
x = torch.randn(m, k, device="cuda", dtype=torch.bfloat16) * 0.1
|
||||
weight = torch.randn(n, k, device="cuda", dtype=torch.bfloat16) * 0.1
|
||||
bias = (
|
||||
torch.randn(n, device="cuda", dtype=torch.bfloat16) * 0.1 if use_bias else None
|
||||
)
|
||||
|
||||
actual = layer_utils.cuda_flashinfer_bf16_gemm_impl(
|
||||
x, weight, bias, pdl, "flashinfer_cutedsl"
|
||||
)
|
||||
expected = F.linear(x, weight, bias)
|
||||
|
||||
torch.testing.assert_close(actual, expected, rtol=2e-2, atol=2e-1)
|
||||
@@ -267,12 +267,16 @@ class KernelConfig:
|
||||
"""
|
||||
|
||||
linear_backend: LinearBackend = "auto"
|
||||
"""Backend for quantized linear layer GEMM kernels. Available options:
|
||||
"""Backend for linear layer GEMM kernels. Available options:
|
||||
|
||||
Layer types without an implementation from the requested backend use
|
||||
automatic selection.
|
||||
|
||||
- "auto": Automatically select the best backend based on model and hardware
|
||||
- "cutlass": Use CUTLASS-based kernels
|
||||
- "flashinfer_cutlass": Use FlashInfer with CUTLASS kernels
|
||||
- "flashinfer_cutedsl": Use FlashInfer with CuTe-DSL kernels (NVFP4, MXFP8)
|
||||
- "flashinfer_cutedsl": Use FlashInfer with CuTe-DSL kernels
|
||||
(BF16, NVFP4, MXFP8)
|
||||
- "flashinfer_trtllm": Use FlashInfer with TensorRT-LLM kernels
|
||||
- "flashinfer_cudnn": Use FlashInfer with cuDNN kernels
|
||||
- "flashinfer_b12x": Use FlashInfer b12x CuteDSL NVFP4 GEMM (SM120+)
|
||||
|
||||
@@ -237,10 +237,7 @@ def _get_linear_backend() -> str:
|
||||
return "auto"
|
||||
|
||||
|
||||
# Mapping from linear_backend name to the set of kernel classes it covers.
|
||||
# When a user sets --linear-backend <name>, only kernels in the corresponding
|
||||
# set are considered candidates. If none can implement the layer config,
|
||||
# an error is raised to respect the user's explicit intent.
|
||||
# Kernel classes covered by each --linear-backend value.
|
||||
_LINEAR_BACKEND_KERNEL_MAP: dict[str, set[type]] = {
|
||||
"b12x": {
|
||||
B12xFp8BlockScaledMMKernel,
|
||||
@@ -345,9 +342,17 @@ def _filter_kernels_by_backend(
|
||||
backend: str,
|
||||
kernels: list[type],
|
||||
) -> list[type]:
|
||||
"""Filter a kernel priority list to only those matching the backend."""
|
||||
"""Prefer matching kernels, falling back to auto by layer type."""
|
||||
backend_kernels = _LINEAR_BACKEND_KERNEL_MAP.get(backend, set())
|
||||
return [k for k in kernels if k in backend_kernels]
|
||||
filtered = [kernel for kernel in kernels if kernel in backend_kernels]
|
||||
if not filtered:
|
||||
logger.warning_once(
|
||||
"--linear-backend=%s has no kernel for this linear layer type; "
|
||||
"using automatic selection for that layer type.",
|
||||
backend,
|
||||
)
|
||||
return kernels
|
||||
return filtered
|
||||
|
||||
|
||||
def _resolve_backend_kernels(
|
||||
|
||||
@@ -10,7 +10,7 @@ from torch.nn.parameter import Parameter
|
||||
from typing_extensions import TypeIs
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.config import get_current_vllm_config, get_current_vllm_config_or_none
|
||||
from vllm.distributed import (
|
||||
divide,
|
||||
get_tensor_model_parallel_rank,
|
||||
@@ -166,6 +166,13 @@ class LinearMethodBase(QuantizeMethodBase):
|
||||
class UnquantizedLinearMethod(LinearMethodBase):
|
||||
"""Linear method without quantization."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
config = get_current_vllm_config_or_none()
|
||||
linear_backend = (
|
||||
config.kernel_config.linear_backend if config is not None else "auto"
|
||||
)
|
||||
self._gemm_impl = dispatch_unquantized_gemm(linear_backend)
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
@@ -215,7 +222,7 @@ class UnquantizedLinearMethod(LinearMethodBase):
|
||||
) -> torch.Tensor:
|
||||
if envs.VLLM_BATCH_INVARIANT and current_platform.is_cuda_alike():
|
||||
return linear_batch_invariant(x, layer.weight, bias)
|
||||
return dispatch_unquantized_gemm()(layer, x, layer.weight, bias)
|
||||
return self._gemm_impl(layer, x, layer.weight, bias)
|
||||
|
||||
|
||||
class LinearBase(PluggableLayer):
|
||||
|
||||
@@ -2,7 +2,9 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Utility methods for model layers."""
|
||||
|
||||
import functools
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
|
||||
import torch
|
||||
|
||||
@@ -11,6 +13,10 @@ from vllm import envs
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import CpuArchEnum, current_platform
|
||||
from vllm.utils.flashinfer import (
|
||||
flashinfer_bf16_mm,
|
||||
is_flashinfer_cutedsl_bf16_gemm_supported,
|
||||
)
|
||||
from vllm.utils.platform_utils import num_compute_units
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
|
||||
@@ -92,6 +98,144 @@ def default_unquantized_gemm(
|
||||
return torch.nn.functional.linear(x, weight, bias)
|
||||
|
||||
|
||||
_FlashInferBf16RuntimeCheck = Callable[
|
||||
[torch.Tensor, torch.Tensor, torch.Tensor | None], bool
|
||||
]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _FlashInferBf16Backend:
|
||||
flashinfer_backend: str
|
||||
is_supported: Callable[[], bool]
|
||||
can_implement: _FlashInferBf16RuntimeCheck
|
||||
|
||||
|
||||
def _can_use_flashinfer_cutedsl_bf16(
|
||||
x: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
bias: torch.Tensor | None,
|
||||
) -> bool:
|
||||
if not (
|
||||
current_platform.is_cuda() and current_platform.is_device_capability_family(100)
|
||||
):
|
||||
return False
|
||||
if x.ndim < 1 or weight.ndim != 2:
|
||||
return False
|
||||
if (
|
||||
not x.is_cuda
|
||||
or not weight.is_cuda
|
||||
or x.device != weight.device
|
||||
or x.dtype != torch.bfloat16
|
||||
or weight.dtype != torch.bfloat16
|
||||
or not x.is_contiguous()
|
||||
or not weight.is_contiguous()
|
||||
):
|
||||
return False
|
||||
|
||||
k = x.shape[-1]
|
||||
n = weight.shape[0]
|
||||
if (
|
||||
k <= 0
|
||||
or n <= 0
|
||||
or weight.shape[1] != k
|
||||
or k % 128 != 0
|
||||
or x.data_ptr() % 32 != 0
|
||||
or weight.data_ptr() % 32 != 0
|
||||
):
|
||||
return False
|
||||
|
||||
m = x.numel() // k
|
||||
if not 1 <= m <= 32:
|
||||
return False
|
||||
return bias is None or (
|
||||
bias.is_cuda
|
||||
and bias.device == x.device
|
||||
and bias.dtype == torch.bfloat16
|
||||
and bias.ndim == 1
|
||||
and bias.shape[0] == n
|
||||
and bias.is_contiguous()
|
||||
)
|
||||
|
||||
|
||||
_FLASHINFER_BF16_BACKENDS = {
|
||||
"flashinfer_cutedsl": _FlashInferBf16Backend(
|
||||
flashinfer_backend="cute-dsl",
|
||||
is_supported=is_flashinfer_cutedsl_bf16_gemm_supported,
|
||||
can_implement=_can_use_flashinfer_cutedsl_bf16,
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _get_flashinfer_bf16_backend(vllm_backend: str) -> _FlashInferBf16Backend:
|
||||
backend_spec = _FLASHINFER_BF16_BACKENDS.get(vllm_backend)
|
||||
if backend_spec is None:
|
||||
supported = ", ".join(sorted(_FLASHINFER_BF16_BACKENDS))
|
||||
raise ValueError(
|
||||
f"Unsupported vLLM FlashInfer BF16 backend {vllm_backend!r}; "
|
||||
f"supported backends: {supported}"
|
||||
)
|
||||
return backend_spec
|
||||
|
||||
|
||||
def cuda_flashinfer_bf16_gemm_impl(
|
||||
x: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
bias: torch.Tensor | None,
|
||||
pdl: bool,
|
||||
vllm_backend: str,
|
||||
) -> torch.Tensor:
|
||||
backend_spec = _get_flashinfer_bf16_backend(vllm_backend)
|
||||
if not backend_spec.can_implement(x, weight, bias):
|
||||
return torch.nn.functional.linear(x, weight, bias)
|
||||
|
||||
k = x.shape[-1]
|
||||
n = weight.shape[0]
|
||||
x_2d = x.view(-1, k)
|
||||
out_2d = flashinfer_bf16_mm(
|
||||
x_2d,
|
||||
weight.t(),
|
||||
bias,
|
||||
pdl,
|
||||
backend_spec.flashinfer_backend,
|
||||
)
|
||||
return out_2d.view(*x.shape[:-1], n)
|
||||
|
||||
|
||||
def cuda_flashinfer_bf16_gemm_fake(
|
||||
x: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
bias: torch.Tensor | None,
|
||||
pdl: bool,
|
||||
vllm_backend: str,
|
||||
) -> torch.Tensor:
|
||||
return x.new_empty((*x.shape[:-1], weight.shape[0]))
|
||||
|
||||
|
||||
def cuda_flashinfer_bf16_gemm(
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
*,
|
||||
vllm_backend: str,
|
||||
pdl: bool,
|
||||
) -> torch.Tensor:
|
||||
return torch.ops.vllm.cuda_flashinfer_bf16_gemm(
|
||||
x,
|
||||
weight,
|
||||
bias,
|
||||
pdl,
|
||||
vllm_backend,
|
||||
)
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="cuda_flashinfer_bf16_gemm",
|
||||
op_func=cuda_flashinfer_bf16_gemm_impl,
|
||||
fake_impl=cuda_flashinfer_bf16_gemm_fake,
|
||||
)
|
||||
|
||||
|
||||
def use_aiter_triton_gemm(n, m, k, dtype):
|
||||
if (
|
||||
not rocm_aiter_ops.is_triton_gemm_enabled()
|
||||
@@ -391,10 +535,36 @@ def cpu_unquantized_gemm(
|
||||
return layer.cpu_linear(x, weight, bias)
|
||||
|
||||
|
||||
def dispatch_unquantized_gemm() -> Callable[..., torch.Tensor]:
|
||||
def dispatch_unquantized_gemm(
|
||||
linear_backend: str = "auto",
|
||||
) -> Callable[..., torch.Tensor]:
|
||||
if current_platform.is_rocm():
|
||||
return rocm_unquantized_gemm
|
||||
elif current_platform.is_cpu():
|
||||
return cpu_unquantized_gemm
|
||||
else:
|
||||
elif not current_platform.is_cuda():
|
||||
return default_unquantized_gemm
|
||||
|
||||
backend_spec = _FLASHINFER_BF16_BACKENDS.get(linear_backend)
|
||||
if backend_spec is None:
|
||||
return default_unquantized_gemm
|
||||
|
||||
if not backend_spec.is_supported():
|
||||
logger.warning_once(
|
||||
"--linear-backend=%s requested FlashInfer mm_bf16 backend %r, "
|
||||
"but it is unavailable on the current hardware or environment; "
|
||||
"using automatic selection for unquantized linear layers.",
|
||||
linear_backend,
|
||||
backend_spec.flashinfer_backend,
|
||||
)
|
||||
return default_unquantized_gemm
|
||||
|
||||
logger.info_once(
|
||||
"Using FlashInfer %s for eligible unquantized BF16 GEMMs.",
|
||||
backend_spec.flashinfer_backend,
|
||||
)
|
||||
return functools.partial(
|
||||
cuda_flashinfer_bf16_gemm,
|
||||
vllm_backend=linear_backend,
|
||||
pdl=current_platform.is_arch_support_pdl(),
|
||||
)
|
||||
|
||||
@@ -247,6 +247,31 @@ def _flashinfer_autotune_skip_ops(runner: "GPUModelRunner") -> set[str] | None:
|
||||
return None
|
||||
|
||||
|
||||
_FLASHINFER_BF16_AUTOTUNE_MAX_TOKENS = 32
|
||||
|
||||
|
||||
def _flashinfer_autotune_token_counts(runner: "GPUModelRunner") -> tuple[int, ...]:
|
||||
max_tokens = runner.scheduler_config.max_num_batched_tokens
|
||||
linear_backend = runner.vllm_config.kernel_config.linear_backend
|
||||
if (
|
||||
linear_backend == "flashinfer_cutedsl"
|
||||
and max_tokens > _FLASHINFER_BF16_AUTOTUNE_MAX_TOKENS
|
||||
):
|
||||
return max_tokens, _FLASHINFER_BF16_AUTOTUNE_MAX_TOKENS
|
||||
return (max_tokens,)
|
||||
|
||||
|
||||
def _run_flashinfer_autotune_dummy_runs(runner: "GPUModelRunner") -> None:
|
||||
for num_tokens in _flashinfer_autotune_token_counts(runner):
|
||||
logger.info("Running FlashInfer autotune with %d tokens.", num_tokens)
|
||||
runner._dummy_run(
|
||||
num_tokens=num_tokens,
|
||||
skip_eplb=True,
|
||||
is_profile=True,
|
||||
randomize_inputs=True,
|
||||
)
|
||||
|
||||
|
||||
def flashinfer_autotune(runner: "GPUModelRunner") -> None:
|
||||
"""
|
||||
Autotune FlashInfer operations.
|
||||
@@ -284,20 +309,10 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
|
||||
logger.info_once("Using FlashInfer autotune cache file: %s", cache_path)
|
||||
|
||||
# We skip EPLB here since we don't want to record dummy metrics.
|
||||
# When autotuning with number of tokens m, flashinfer will autotune
|
||||
# operations for all number of tokens up to m, so we only need to
|
||||
# run with the max number of tokens.
|
||||
# Randomize inputs to avoid every token pick the same experts,
|
||||
# which lead to some EP ranks receiving no tokens and skipping their
|
||||
# MoE kernel entirely, and cause hang due to all-reduce collective
|
||||
# during synchronized autotuning.
|
||||
dummy_run_kwargs = dict(
|
||||
num_tokens=runner.scheduler_config.max_num_batched_tokens,
|
||||
skip_eplb=True,
|
||||
is_profile=True,
|
||||
randomize_inputs=True,
|
||||
)
|
||||
|
||||
# Read cached autotune results and broadcast to all ranks.
|
||||
cached_results: bytes | None = None
|
||||
if is_leader and cache_path.exists():
|
||||
@@ -316,7 +331,7 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
|
||||
torch.inference_mode(),
|
||||
fi_utils.autotune(tune_mode=True, **autotune_kwargs),
|
||||
):
|
||||
runner._dummy_run(**dummy_run_kwargs)
|
||||
_run_flashinfer_autotune_dummy_runs(runner)
|
||||
finally:
|
||||
set_autotune_process_group(None)
|
||||
|
||||
|
||||
@@ -24,6 +24,26 @@ from vllm.utils.math_utils import cdiv
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def flashinfer_bf16_mm(
|
||||
a: torch.Tensor,
|
||||
b: torch.Tensor,
|
||||
bias: torch.Tensor | None,
|
||||
pdl: bool,
|
||||
backend: str,
|
||||
) -> torch.Tensor:
|
||||
from flashinfer import mm_bf16
|
||||
|
||||
return mm_bf16(
|
||||
a,
|
||||
b,
|
||||
bias=bias,
|
||||
pdl=pdl,
|
||||
out_dtype=torch.bfloat16,
|
||||
backend=backend,
|
||||
)
|
||||
|
||||
|
||||
# This is the storage path for the cubins, it can be replaced
|
||||
# with a local path for testing.
|
||||
# Referenced from https://github.com/flashinfer-ai/flashinfer/blob/0c9a92c3d9a7e043ab6f3f7b2273269caf6ab044/flashinfer/jit/cubin_loader.py#L35 # noqa: E501
|
||||
@@ -63,6 +83,58 @@ def has_flashinfer() -> bool:
|
||||
return True
|
||||
|
||||
|
||||
@functools.cache
|
||||
def has_flashinfer_bf16_gemm() -> bool:
|
||||
"""Return whether FlashInfer exposes the BF16 dense GEMM API."""
|
||||
if not has_flashinfer():
|
||||
return False
|
||||
mod = _get_submodule("flashinfer")
|
||||
return mod is not None and callable(getattr(mod, "mm_bf16", None))
|
||||
|
||||
|
||||
@functools.cache
|
||||
def is_flashinfer_bf16_gemm_supported(
|
||||
backend: str,
|
||||
compute_capability: int | None = None,
|
||||
) -> bool:
|
||||
"""Return whether an exact FlashInfer BF16 backend is available."""
|
||||
if not current_platform.is_cuda() or not has_flashinfer_bf16_gemm():
|
||||
return False
|
||||
|
||||
mod = _get_submodule("flashinfer")
|
||||
mm_bf16 = getattr(mod, "mm_bf16", None) if mod is not None else None
|
||||
backend_supported = getattr(mm_bf16, "is_backend_supported", None)
|
||||
if not callable(backend_supported):
|
||||
return False
|
||||
|
||||
if compute_capability is None:
|
||||
device_capability = current_platform.get_device_capability()
|
||||
if device_capability is None:
|
||||
return False
|
||||
compute_capability = device_capability.to_int()
|
||||
|
||||
try:
|
||||
return bool(backend_supported(backend, compute_capability))
|
||||
except (AttributeError, ImportError, RuntimeError, TypeError, ValueError):
|
||||
return False
|
||||
|
||||
|
||||
@functools.cache
|
||||
def is_flashinfer_cutedsl_bf16_gemm_supported() -> bool:
|
||||
"""Return whether the CuTeDSL BF16 dense GEMM backend is available."""
|
||||
if not is_flashinfer_bf16_gemm_supported("cute-dsl"):
|
||||
return False
|
||||
try:
|
||||
from flashinfer.cute_dsl.utils import is_cute_dsl_available
|
||||
from flashinfer.utils import is_sm100a_supported
|
||||
except (ImportError, ModuleNotFoundError):
|
||||
return False
|
||||
try:
|
||||
return is_cute_dsl_available() and is_sm100a_supported(torch.device("cuda"))
|
||||
except (RuntimeError, TypeError, ValueError):
|
||||
return False
|
||||
|
||||
|
||||
def _missing(*_: Any, **__: Any) -> NoReturn:
|
||||
"""Placeholder for unavailable FlashInfer backend."""
|
||||
raise RuntimeError(
|
||||
@@ -1079,6 +1151,10 @@ def is_flashinfer_cudnn_fp8_prefill_attn_supported() -> bool:
|
||||
|
||||
__all__ = [
|
||||
"has_flashinfer",
|
||||
"flashinfer_bf16_mm",
|
||||
"has_flashinfer_bf16_gemm",
|
||||
"is_flashinfer_bf16_gemm_supported",
|
||||
"is_flashinfer_cutedsl_bf16_gemm_supported",
|
||||
"flashinfer_trtllm_fp8_block_scale_moe",
|
||||
"flashinfer_cutlass_fused_moe",
|
||||
"flashinfer_cutedsl_grouped_gemm_nt_masked",
|
||||
|
||||
Reference in New Issue
Block a user