diff --git a/tests/kernels/test_flashinfer_bf16_gemm.py b/tests/kernels/test_flashinfer_bf16_gemm.py new file mode 100644 index 0000000000..63a5a39c2e --- /dev/null +++ b/tests/kernels/test_flashinfer_bf16_gemm.py @@ -0,0 +1,49 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import pytest +import torch +import torch.nn.functional as F + +from vllm.model_executor.layers import utils as layer_utils +from vllm.utils.flashinfer import is_flashinfer_cutedsl_bf16_gemm_supported + + +@pytest.fixture(scope="module", autouse=True) +def require_flashinfer_bf16_cutedsl() -> None: + if not torch.accelerator.is_available(): + pytest.skip("CUDA is required") + if not is_flashinfer_cutedsl_bf16_gemm_supported(): + pytest.skip("FlashInfer BF16 cute-dsl backend is unavailable") + + +@pytest.mark.parametrize("n,k", [(1024, 1024), (2048, 3072)]) +@pytest.mark.parametrize( + "m,use_bias,pdl", + [ + (1, False, False), + (8, True, False), + (16, False, True), + (32, True, True), + ], +) +def test_flashinfer_bf16_cutedsl_correctness( + m: int, + n: int, + k: int, + use_bias: bool, + pdl: bool, +) -> None: + torch.manual_seed(0) + x = torch.randn(m, k, device="cuda", dtype=torch.bfloat16) * 0.1 + weight = torch.randn(n, k, device="cuda", dtype=torch.bfloat16) * 0.1 + bias = ( + torch.randn(n, device="cuda", dtype=torch.bfloat16) * 0.1 if use_bias else None + ) + + actual = layer_utils.cuda_flashinfer_bf16_gemm_impl( + x, weight, bias, pdl, "flashinfer_cutedsl" + ) + expected = F.linear(x, weight, bias) + + torch.testing.assert_close(actual, expected, rtol=2e-2, atol=2e-1) diff --git a/vllm/config/kernel.py b/vllm/config/kernel.py index 2716319e39..56b3546a2f 100644 --- a/vllm/config/kernel.py +++ b/vllm/config/kernel.py @@ -267,12 +267,16 @@ class KernelConfig: """ linear_backend: LinearBackend = "auto" - """Backend for quantized linear layer GEMM kernels. Available options: + """Backend for linear layer GEMM kernels. Available options: + + Layer types without an implementation from the requested backend use + automatic selection. - "auto": Automatically select the best backend based on model and hardware - "cutlass": Use CUTLASS-based kernels - "flashinfer_cutlass": Use FlashInfer with CUTLASS kernels - - "flashinfer_cutedsl": Use FlashInfer with CuTe-DSL kernels (NVFP4, MXFP8) + - "flashinfer_cutedsl": Use FlashInfer with CuTe-DSL kernels + (BF16, NVFP4, MXFP8) - "flashinfer_trtllm": Use FlashInfer with TensorRT-LLM kernels - "flashinfer_cudnn": Use FlashInfer with cuDNN kernels - "flashinfer_b12x": Use FlashInfer b12x CuteDSL NVFP4 GEMM (SM120+) diff --git a/vllm/model_executor/kernels/linear/__init__.py b/vllm/model_executor/kernels/linear/__init__.py index 16d3098df3..e02334b84c 100644 --- a/vllm/model_executor/kernels/linear/__init__.py +++ b/vllm/model_executor/kernels/linear/__init__.py @@ -237,10 +237,7 @@ def _get_linear_backend() -> str: return "auto" -# Mapping from linear_backend name to the set of kernel classes it covers. -# When a user sets --linear-backend , only kernels in the corresponding -# set are considered candidates. If none can implement the layer config, -# an error is raised to respect the user's explicit intent. +# Kernel classes covered by each --linear-backend value. _LINEAR_BACKEND_KERNEL_MAP: dict[str, set[type]] = { "b12x": { B12xFp8BlockScaledMMKernel, @@ -345,9 +342,17 @@ def _filter_kernels_by_backend( backend: str, kernels: list[type], ) -> list[type]: - """Filter a kernel priority list to only those matching the backend.""" + """Prefer matching kernels, falling back to auto by layer type.""" backend_kernels = _LINEAR_BACKEND_KERNEL_MAP.get(backend, set()) - return [k for k in kernels if k in backend_kernels] + filtered = [kernel for kernel in kernels if kernel in backend_kernels] + if not filtered: + logger.warning_once( + "--linear-backend=%s has no kernel for this linear layer type; " + "using automatic selection for that layer type.", + backend, + ) + return kernels + return filtered def _resolve_backend_kernels( diff --git a/vllm/model_executor/layers/linear.py b/vllm/model_executor/layers/linear.py index e723d6fee4..39033f99f2 100644 --- a/vllm/model_executor/layers/linear.py +++ b/vllm/model_executor/layers/linear.py @@ -10,7 +10,7 @@ from torch.nn.parameter import Parameter from typing_extensions import TypeIs import vllm.envs as envs -from vllm.config import get_current_vllm_config +from vllm.config import get_current_vllm_config, get_current_vllm_config_or_none from vllm.distributed import ( divide, get_tensor_model_parallel_rank, @@ -166,6 +166,13 @@ class LinearMethodBase(QuantizeMethodBase): class UnquantizedLinearMethod(LinearMethodBase): """Linear method without quantization.""" + def __init__(self) -> None: + config = get_current_vllm_config_or_none() + linear_backend = ( + config.kernel_config.linear_backend if config is not None else "auto" + ) + self._gemm_impl = dispatch_unquantized_gemm(linear_backend) + def create_weights( self, layer: torch.nn.Module, @@ -215,7 +222,7 @@ class UnquantizedLinearMethod(LinearMethodBase): ) -> torch.Tensor: if envs.VLLM_BATCH_INVARIANT and current_platform.is_cuda_alike(): return linear_batch_invariant(x, layer.weight, bias) - return dispatch_unquantized_gemm()(layer, x, layer.weight, bias) + return self._gemm_impl(layer, x, layer.weight, bias) class LinearBase(PluggableLayer): diff --git a/vllm/model_executor/layers/utils.py b/vllm/model_executor/layers/utils.py index 5cb23fb14b..dc6f6775fc 100644 --- a/vllm/model_executor/layers/utils.py +++ b/vllm/model_executor/layers/utils.py @@ -2,7 +2,9 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """Utility methods for model layers.""" +import functools from collections.abc import Callable +from dataclasses import dataclass import torch @@ -11,6 +13,10 @@ from vllm import envs from vllm._aiter_ops import rocm_aiter_ops from vllm.logger import init_logger from vllm.platforms import CpuArchEnum, current_platform +from vllm.utils.flashinfer import ( + flashinfer_bf16_mm, + is_flashinfer_cutedsl_bf16_gemm_supported, +) from vllm.utils.platform_utils import num_compute_units from vllm.utils.torch_utils import direct_register_custom_op @@ -92,6 +98,144 @@ def default_unquantized_gemm( return torch.nn.functional.linear(x, weight, bias) +_FlashInferBf16RuntimeCheck = Callable[ + [torch.Tensor, torch.Tensor, torch.Tensor | None], bool +] + + +@dataclass(frozen=True) +class _FlashInferBf16Backend: + flashinfer_backend: str + is_supported: Callable[[], bool] + can_implement: _FlashInferBf16RuntimeCheck + + +def _can_use_flashinfer_cutedsl_bf16( + x: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor | None, +) -> bool: + if not ( + current_platform.is_cuda() and current_platform.is_device_capability_family(100) + ): + return False + if x.ndim < 1 or weight.ndim != 2: + return False + if ( + not x.is_cuda + or not weight.is_cuda + or x.device != weight.device + or x.dtype != torch.bfloat16 + or weight.dtype != torch.bfloat16 + or not x.is_contiguous() + or not weight.is_contiguous() + ): + return False + + k = x.shape[-1] + n = weight.shape[0] + if ( + k <= 0 + or n <= 0 + or weight.shape[1] != k + or k % 128 != 0 + or x.data_ptr() % 32 != 0 + or weight.data_ptr() % 32 != 0 + ): + return False + + m = x.numel() // k + if not 1 <= m <= 32: + return False + return bias is None or ( + bias.is_cuda + and bias.device == x.device + and bias.dtype == torch.bfloat16 + and bias.ndim == 1 + and bias.shape[0] == n + and bias.is_contiguous() + ) + + +_FLASHINFER_BF16_BACKENDS = { + "flashinfer_cutedsl": _FlashInferBf16Backend( + flashinfer_backend="cute-dsl", + is_supported=is_flashinfer_cutedsl_bf16_gemm_supported, + can_implement=_can_use_flashinfer_cutedsl_bf16, + ), +} + + +def _get_flashinfer_bf16_backend(vllm_backend: str) -> _FlashInferBf16Backend: + backend_spec = _FLASHINFER_BF16_BACKENDS.get(vllm_backend) + if backend_spec is None: + supported = ", ".join(sorted(_FLASHINFER_BF16_BACKENDS)) + raise ValueError( + f"Unsupported vLLM FlashInfer BF16 backend {vllm_backend!r}; " + f"supported backends: {supported}" + ) + return backend_spec + + +def cuda_flashinfer_bf16_gemm_impl( + x: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor | None, + pdl: bool, + vllm_backend: str, +) -> torch.Tensor: + backend_spec = _get_flashinfer_bf16_backend(vllm_backend) + if not backend_spec.can_implement(x, weight, bias): + return torch.nn.functional.linear(x, weight, bias) + + k = x.shape[-1] + n = weight.shape[0] + x_2d = x.view(-1, k) + out_2d = flashinfer_bf16_mm( + x_2d, + weight.t(), + bias, + pdl, + backend_spec.flashinfer_backend, + ) + return out_2d.view(*x.shape[:-1], n) + + +def cuda_flashinfer_bf16_gemm_fake( + x: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor | None, + pdl: bool, + vllm_backend: str, +) -> torch.Tensor: + return x.new_empty((*x.shape[:-1], weight.shape[0])) + + +def cuda_flashinfer_bf16_gemm( + layer: torch.nn.Module, + x: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor | None = None, + *, + vllm_backend: str, + pdl: bool, +) -> torch.Tensor: + return torch.ops.vllm.cuda_flashinfer_bf16_gemm( + x, + weight, + bias, + pdl, + vllm_backend, + ) + + +direct_register_custom_op( + op_name="cuda_flashinfer_bf16_gemm", + op_func=cuda_flashinfer_bf16_gemm_impl, + fake_impl=cuda_flashinfer_bf16_gemm_fake, +) + + def use_aiter_triton_gemm(n, m, k, dtype): if ( not rocm_aiter_ops.is_triton_gemm_enabled() @@ -391,10 +535,36 @@ def cpu_unquantized_gemm( return layer.cpu_linear(x, weight, bias) -def dispatch_unquantized_gemm() -> Callable[..., torch.Tensor]: +def dispatch_unquantized_gemm( + linear_backend: str = "auto", +) -> Callable[..., torch.Tensor]: if current_platform.is_rocm(): return rocm_unquantized_gemm elif current_platform.is_cpu(): return cpu_unquantized_gemm - else: + elif not current_platform.is_cuda(): return default_unquantized_gemm + + backend_spec = _FLASHINFER_BF16_BACKENDS.get(linear_backend) + if backend_spec is None: + return default_unquantized_gemm + + if not backend_spec.is_supported(): + logger.warning_once( + "--linear-backend=%s requested FlashInfer mm_bf16 backend %r, " + "but it is unavailable on the current hardware or environment; " + "using automatic selection for unquantized linear layers.", + linear_backend, + backend_spec.flashinfer_backend, + ) + return default_unquantized_gemm + + logger.info_once( + "Using FlashInfer %s for eligible unquantized BF16 GEMMs.", + backend_spec.flashinfer_backend, + ) + return functools.partial( + cuda_flashinfer_bf16_gemm, + vllm_backend=linear_backend, + pdl=current_platform.is_arch_support_pdl(), + ) diff --git a/vllm/model_executor/warmup/kernel_warmup.py b/vllm/model_executor/warmup/kernel_warmup.py index 0aad349b45..f4dc088447 100644 --- a/vllm/model_executor/warmup/kernel_warmup.py +++ b/vllm/model_executor/warmup/kernel_warmup.py @@ -247,6 +247,31 @@ def _flashinfer_autotune_skip_ops(runner: "GPUModelRunner") -> set[str] | None: return None +_FLASHINFER_BF16_AUTOTUNE_MAX_TOKENS = 32 + + +def _flashinfer_autotune_token_counts(runner: "GPUModelRunner") -> tuple[int, ...]: + max_tokens = runner.scheduler_config.max_num_batched_tokens + linear_backend = runner.vllm_config.kernel_config.linear_backend + if ( + linear_backend == "flashinfer_cutedsl" + and max_tokens > _FLASHINFER_BF16_AUTOTUNE_MAX_TOKENS + ): + return max_tokens, _FLASHINFER_BF16_AUTOTUNE_MAX_TOKENS + return (max_tokens,) + + +def _run_flashinfer_autotune_dummy_runs(runner: "GPUModelRunner") -> None: + for num_tokens in _flashinfer_autotune_token_counts(runner): + logger.info("Running FlashInfer autotune with %d tokens.", num_tokens) + runner._dummy_run( + num_tokens=num_tokens, + skip_eplb=True, + is_profile=True, + randomize_inputs=True, + ) + + def flashinfer_autotune(runner: "GPUModelRunner") -> None: """ Autotune FlashInfer operations. @@ -284,20 +309,10 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None: logger.info_once("Using FlashInfer autotune cache file: %s", cache_path) # We skip EPLB here since we don't want to record dummy metrics. - # When autotuning with number of tokens m, flashinfer will autotune - # operations for all number of tokens up to m, so we only need to - # run with the max number of tokens. # Randomize inputs to avoid every token pick the same experts, # which lead to some EP ranks receiving no tokens and skipping their # MoE kernel entirely, and cause hang due to all-reduce collective # during synchronized autotuning. - dummy_run_kwargs = dict( - num_tokens=runner.scheduler_config.max_num_batched_tokens, - skip_eplb=True, - is_profile=True, - randomize_inputs=True, - ) - # Read cached autotune results and broadcast to all ranks. cached_results: bytes | None = None if is_leader and cache_path.exists(): @@ -316,7 +331,7 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None: torch.inference_mode(), fi_utils.autotune(tune_mode=True, **autotune_kwargs), ): - runner._dummy_run(**dummy_run_kwargs) + _run_flashinfer_autotune_dummy_runs(runner) finally: set_autotune_process_group(None) diff --git a/vllm/utils/flashinfer.py b/vllm/utils/flashinfer.py index f77b669cc2..ed9cfbd720 100644 --- a/vllm/utils/flashinfer.py +++ b/vllm/utils/flashinfer.py @@ -24,6 +24,26 @@ from vllm.utils.math_utils import cdiv logger = init_logger(__name__) + +def flashinfer_bf16_mm( + a: torch.Tensor, + b: torch.Tensor, + bias: torch.Tensor | None, + pdl: bool, + backend: str, +) -> torch.Tensor: + from flashinfer import mm_bf16 + + return mm_bf16( + a, + b, + bias=bias, + pdl=pdl, + out_dtype=torch.bfloat16, + backend=backend, + ) + + # This is the storage path for the cubins, it can be replaced # with a local path for testing. # Referenced from https://github.com/flashinfer-ai/flashinfer/blob/0c9a92c3d9a7e043ab6f3f7b2273269caf6ab044/flashinfer/jit/cubin_loader.py#L35 # noqa: E501 @@ -63,6 +83,58 @@ def has_flashinfer() -> bool: return True +@functools.cache +def has_flashinfer_bf16_gemm() -> bool: + """Return whether FlashInfer exposes the BF16 dense GEMM API.""" + if not has_flashinfer(): + return False + mod = _get_submodule("flashinfer") + return mod is not None and callable(getattr(mod, "mm_bf16", None)) + + +@functools.cache +def is_flashinfer_bf16_gemm_supported( + backend: str, + compute_capability: int | None = None, +) -> bool: + """Return whether an exact FlashInfer BF16 backend is available.""" + if not current_platform.is_cuda() or not has_flashinfer_bf16_gemm(): + return False + + mod = _get_submodule("flashinfer") + mm_bf16 = getattr(mod, "mm_bf16", None) if mod is not None else None + backend_supported = getattr(mm_bf16, "is_backend_supported", None) + if not callable(backend_supported): + return False + + if compute_capability is None: + device_capability = current_platform.get_device_capability() + if device_capability is None: + return False + compute_capability = device_capability.to_int() + + try: + return bool(backend_supported(backend, compute_capability)) + except (AttributeError, ImportError, RuntimeError, TypeError, ValueError): + return False + + +@functools.cache +def is_flashinfer_cutedsl_bf16_gemm_supported() -> bool: + """Return whether the CuTeDSL BF16 dense GEMM backend is available.""" + if not is_flashinfer_bf16_gemm_supported("cute-dsl"): + return False + try: + from flashinfer.cute_dsl.utils import is_cute_dsl_available + from flashinfer.utils import is_sm100a_supported + except (ImportError, ModuleNotFoundError): + return False + try: + return is_cute_dsl_available() and is_sm100a_supported(torch.device("cuda")) + except (RuntimeError, TypeError, ValueError): + return False + + def _missing(*_: Any, **__: Any) -> NoReturn: """Placeholder for unavailable FlashInfer backend.""" raise RuntimeError( @@ -1079,6 +1151,10 @@ def is_flashinfer_cudnn_fp8_prefill_attn_supported() -> bool: __all__ = [ "has_flashinfer", + "flashinfer_bf16_mm", + "has_flashinfer_bf16_gemm", + "is_flashinfer_bf16_gemm_supported", + "is_flashinfer_cutedsl_bf16_gemm_supported", "flashinfer_trtllm_fp8_block_scale_moe", "flashinfer_cutlass_fused_moe", "flashinfer_cutedsl_grouped_gemm_nt_masked",