diff --git a/tests/kernels/quantization/test_block_fp8.py b/tests/kernels/quantization/test_block_fp8.py index c5eaa2f932..9b41765de6 100644 --- a/tests/kernels/quantization/test_block_fp8.py +++ b/tests/kernels/quantization/test_block_fp8.py @@ -3,6 +3,7 @@ # Adapted from https://github.com/sgl-project/sglang/pull/2575 import itertools +import types import pytest import torch @@ -200,6 +201,65 @@ def test_w8a8_block_fp8_cutlass_matmul(): assert rel_diff < 0.001 +@pytest.mark.skipif( + not (current_platform.is_cuda() and current_platform.has_device_capability(90)), + reason="torch._scaled_mm DeepSeek-style block scaling only supports SM90.", +) +def test_w8a8_block_fp8_torch_scaled_mm_matmul(): + # BlockWiseTorchFP8ScaledMMLinearKernel: 1x128 activation + 128x128 weight + # block scaling routed through torch._scaled_mm. M=83 is not a multiple of + # 4 so this also exercises the M-padding path. + from vllm.model_executor.kernels.linear.scaled_mm.pytorch import ( + BlockWiseTorchFP8ScaledMMLinearKernel, + ) + + M = 83 + N = 576 + K = 7168 + block_size = [128, 128] + out_dtype = torch.bfloat16 + seed = 0 + + torch.manual_seed(seed) + factor_for_scale = 1e-2 + fp8_info = torch.finfo(torch.float8_e4m3fn) + fp8_max, fp8_min = fp8_info.max, fp8_info.min + + A_fp32 = (torch.rand(M, K, dtype=torch.float32) - 0.5) * 2 * fp8_max + B_fp32 = (torch.rand(N, K, dtype=torch.float32) - 0.5) * 2 * fp8_max + B_fp8 = B_fp32.clamp(min=fp8_min, max=fp8_max).to(torch.float8_e4m3fn) + + block_n, block_k = block_size[0], block_size[1] + n_tiles = (N + block_n - 1) // block_n + k_tiles = (K + block_k - 1) // block_k + Bs = torch.rand(n_tiles, k_tiles, dtype=torch.float32) * factor_for_scale + + # Reference uses row-major activation scales. + A_fp8, As = per_token_group_quant_fp8( + A_fp32, block_size[1], column_major_scales=False + ) + ref_out = native_w8a8_block_matmul(A_fp8, B_fp8, As, Bs, block_size, out_dtype) + + # The kernel expects column-major activation scales on CUDA. + A_fp8_cuda, As_cuda = per_token_group_quant_fp8( + A_fp32, block_size[1], column_major_scales=True + ) + + stub = BlockWiseTorchFP8ScaledMMLinearKernel.__new__( + BlockWiseTorchFP8ScaledMMLinearKernel + ) + stub.config = types.SimpleNamespace(out_dtype=out_dtype) + out = stub.apply_block_scaled_mm( + A_fp8_cuda.cuda(), B_fp8.cuda(), As_cuda.cuda(), Bs.cuda() + ) + + ref_out = ref_out.cuda() + rel_diff = torch.mean( + torch.abs(out.to(torch.float32) - ref_out.to(torch.float32)) + ) / torch.mean(torch.abs(ref_out.to(torch.float32))) + assert rel_diff < 0.001 + + @pytest.mark.skipif( current_platform.is_fp8_fnuz(), reason="This platform supports e4m3fnuz, not e4m3fn.", diff --git a/vllm/model_executor/kernels/linear/__init__.py b/vllm/model_executor/kernels/linear/__init__.py index 3d8449a219..c6cc2f2338 100644 --- a/vllm/model_executor/kernels/linear/__init__.py +++ b/vllm/model_executor/kernels/linear/__init__.py @@ -187,6 +187,7 @@ from vllm.model_executor.kernels.linear.scaled_mm.marlin import ( MarlinFP8ScaledMMLinearKernel, ) from vllm.model_executor.kernels.linear.scaled_mm.pytorch import ( + BlockWiseTorchFP8ScaledMMLinearKernel, ChannelWiseTorchFP8ScaledMMLinearKernel, PerTensorTorchFP8ScaledMMLinearKernel, RowWiseTorchFP8ScaledMMLinearKernel, @@ -281,6 +282,7 @@ _LINEAR_BACKEND_KERNEL_MAP: dict[str, set[type]] = { PerTensorTorchFP8ScaledMMLinearKernel, ChannelWiseTorchFP8ScaledMMLinearKernel, RowWiseTorchFP8ScaledMMLinearKernel, + BlockWiseTorchFP8ScaledMMLinearKernel, }, "aiter": { AiterInt8ScaledMMLinearKernel, @@ -380,6 +382,7 @@ _POSSIBLE_FP8_BLOCK_KERNELS: dict[ MarlinFP8ScaledMMLinearKernel, TritonFp8BlockScaledMMKernel, HummingFP8ScaledMMLinearKernel, + BlockWiseTorchFP8ScaledMMLinearKernel, ], PlatformEnum.ROCM: [ AiterFp8BlockScaledMMKernel, diff --git a/vllm/model_executor/kernels/linear/scaled_mm/pytorch.py b/vllm/model_executor/kernels/linear/scaled_mm/pytorch.py index 652e04a0a5..0d36b8079d 100644 --- a/vllm/model_executor/kernels/linear/scaled_mm/pytorch.py +++ b/vllm/model_executor/kernels/linear/scaled_mm/pytorch.py @@ -6,8 +6,11 @@ import math import torch from vllm.config import CompilationMode, get_current_vllm_config +from vllm.model_executor.layers.quantization.input_quant_fp8 import QuantFP8 +from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape from vllm.platforms import current_platform +from .BlockScaledMMLinearKernel import Fp8BlockScaledMMLinearKernel from .ScaledMMLinearKernel import ( FP8ScaledMMLinearKernel, FP8ScaledMMLinearLayerConfig, @@ -244,3 +247,108 @@ class ChannelWiseTorchFP8ScaledMMLinearKernel(TorchFP8ScaledMMLinearKernel): if bias is not None: output = output + bias return output.to(out_dtype).view(*output_shape) + + +class BlockWiseTorchFP8ScaledMMLinearKernel(Fp8BlockScaledMMLinearKernel): + """FP8 block-scaled linear kernel using ``torch._scaled_mm``. + + Implements the block-scaled path of ``torch._scaled_mm``, which + dispatches on the shapes of the scale tensors. For ``A = [M, K]`` and + ``B = [K, N]`` (both fp8) the op's block path requires, with float32 + scales: + * 1x128 activation: ``scale_a = [M, ceil(K / 128)]`` + * 128x128 weight: ``scale_b = [ceil(K / 128), ceil(N / 128)]`` + """ + + def __init__(self, config: FP8ScaledMMLinearLayerConfig) -> None: + super().__init__(config) + act_scale_descriptor = config.activation_quant_key.scale + self.quant_fp8 = QuantFP8( + static=act_scale_descriptor.static, + group_shape=act_scale_descriptor.group_shape, + num_token_padding=self.get_output_padding(), + use_ue8m0=False, + column_major_scales=current_platform.is_cuda_alike(), + ) + + @classmethod + def is_supported( + cls, compute_capability: int | None = None + ) -> tuple[bool, str | None]: + if not (current_platform.is_cuda_alike() or current_platform.is_xpu()): + return False, "requires CUDA, ROCm or XPU." + # torch._scaled_mm DeepSeek-style (1x128, 128x128) block scaling is + # only implemented for SM90 (Hopper) on NVIDIA CUDA. See PyTorch's + # _check_deepseek_support(): + # https://github.com/pytorch/pytorch/blob/33812ece06e2b0d597f73fbe41de03a83f9109f9/aten/src/ATen/native/cuda/ScaledBlas.cpp#L804-L820 # noqa: E501 + if current_platform.is_cuda(): + is_sm90 = ( + compute_capability == 90 + if compute_capability is not None + else current_platform.is_device_capability(90) + ) + if not is_sm90: + return ( + False, + "DeepSeek-style (1x128, 128x128) block scaling on CUDA " + "requires compute capability 90 (Hopper).", + ) + return True, None + + @classmethod + def can_implement( + cls, config: FP8ScaledMMLinearLayerConfig + ) -> tuple[bool, str | None]: + can_implement_base, reason = super().can_implement(config) + if not can_implement_base: + return can_implement_base, reason + + act_group_shape = config.activation_quant_key.scale.group_shape + if act_group_shape != GroupShape(1, 128): + return ( + False, + "requires 1x128 activation quantization.", + ) + weight_group_shape = config.weight_quant_key.scale.group_shape + if weight_group_shape != GroupShape(128, 128): + return ( + False, + "requires 128x128 block weight quantization.", + ) + return True, None + + def apply_block_scaled_mm( + self, + A: torch.Tensor, + B: torch.Tensor, + As: torch.Tensor, + Bs: torch.Tensor, + ) -> torch.Tensor: + # torch._scaled_mm implemented by cuBLASLt requires the M dimension to be a + # multiple of 4; pad A (and its column-major scale) up and slice back. + # https://docs.nvidia.com/cuda/cublas/index.html?highlight=128%2520element%25201D%2520and%2520128x128%25202D#scaling-factors-layouts # noqa: E501 + M = A.shape[0] + pad = (-M) % 4 if current_platform.is_cuda() else 0 + if pad: + Ap = A.new_empty((M + pad, A.shape[1])) + Ap[:M] = A + A = Ap + # As is column-major [M, ceil(K/128)] (stride (1, M)); build a padded + # column-major buffer and copy only the valid rows. Padded rows are left + # uninitialized — they don't affect the sliced-back output. + Asp = As.new_empty((As.shape[1], M + pad)).t() + Asp[:M] = As + As = Asp + + output = torch._scaled_mm( + A, + B.t(), + scale_a=As, + scale_b=Bs.t(), + out_dtype=self.config.out_dtype, + ) + if type(output) is tuple and len(output) == 2: + output = output[0] + if pad: + output = output[:M, :] + return output