[Linear] [Kernel] add block-wise scaled_mm (#49932)
Signed-off-by: Zhu, Zufang <zufang.zhu@intel.com> Co-authored-by: Kunshang Ji <kunshang.ji@intel.com>
This commit is contained in:
@@ -3,6 +3,7 @@
|
||||
|
||||
# Adapted from https://github.com/sgl-project/sglang/pull/2575
|
||||
import itertools
|
||||
import types
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
@@ -200,6 +201,65 @@ def test_w8a8_block_fp8_cutlass_matmul():
|
||||
assert rel_diff < 0.001
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not (current_platform.is_cuda() and current_platform.has_device_capability(90)),
|
||||
reason="torch._scaled_mm DeepSeek-style block scaling only supports SM90.",
|
||||
)
|
||||
def test_w8a8_block_fp8_torch_scaled_mm_matmul():
|
||||
# BlockWiseTorchFP8ScaledMMLinearKernel: 1x128 activation + 128x128 weight
|
||||
# block scaling routed through torch._scaled_mm. M=83 is not a multiple of
|
||||
# 4 so this also exercises the M-padding path.
|
||||
from vllm.model_executor.kernels.linear.scaled_mm.pytorch import (
|
||||
BlockWiseTorchFP8ScaledMMLinearKernel,
|
||||
)
|
||||
|
||||
M = 83
|
||||
N = 576
|
||||
K = 7168
|
||||
block_size = [128, 128]
|
||||
out_dtype = torch.bfloat16
|
||||
seed = 0
|
||||
|
||||
torch.manual_seed(seed)
|
||||
factor_for_scale = 1e-2
|
||||
fp8_info = torch.finfo(torch.float8_e4m3fn)
|
||||
fp8_max, fp8_min = fp8_info.max, fp8_info.min
|
||||
|
||||
A_fp32 = (torch.rand(M, K, dtype=torch.float32) - 0.5) * 2 * fp8_max
|
||||
B_fp32 = (torch.rand(N, K, dtype=torch.float32) - 0.5) * 2 * fp8_max
|
||||
B_fp8 = B_fp32.clamp(min=fp8_min, max=fp8_max).to(torch.float8_e4m3fn)
|
||||
|
||||
block_n, block_k = block_size[0], block_size[1]
|
||||
n_tiles = (N + block_n - 1) // block_n
|
||||
k_tiles = (K + block_k - 1) // block_k
|
||||
Bs = torch.rand(n_tiles, k_tiles, dtype=torch.float32) * factor_for_scale
|
||||
|
||||
# Reference uses row-major activation scales.
|
||||
A_fp8, As = per_token_group_quant_fp8(
|
||||
A_fp32, block_size[1], column_major_scales=False
|
||||
)
|
||||
ref_out = native_w8a8_block_matmul(A_fp8, B_fp8, As, Bs, block_size, out_dtype)
|
||||
|
||||
# The kernel expects column-major activation scales on CUDA.
|
||||
A_fp8_cuda, As_cuda = per_token_group_quant_fp8(
|
||||
A_fp32, block_size[1], column_major_scales=True
|
||||
)
|
||||
|
||||
stub = BlockWiseTorchFP8ScaledMMLinearKernel.__new__(
|
||||
BlockWiseTorchFP8ScaledMMLinearKernel
|
||||
)
|
||||
stub.config = types.SimpleNamespace(out_dtype=out_dtype)
|
||||
out = stub.apply_block_scaled_mm(
|
||||
A_fp8_cuda.cuda(), B_fp8.cuda(), As_cuda.cuda(), Bs.cuda()
|
||||
)
|
||||
|
||||
ref_out = ref_out.cuda()
|
||||
rel_diff = torch.mean(
|
||||
torch.abs(out.to(torch.float32) - ref_out.to(torch.float32))
|
||||
) / torch.mean(torch.abs(ref_out.to(torch.float32)))
|
||||
assert rel_diff < 0.001
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
current_platform.is_fp8_fnuz(),
|
||||
reason="This platform supports e4m3fnuz, not e4m3fn.",
|
||||
|
||||
@@ -187,6 +187,7 @@ from vllm.model_executor.kernels.linear.scaled_mm.marlin import (
|
||||
MarlinFP8ScaledMMLinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.scaled_mm.pytorch import (
|
||||
BlockWiseTorchFP8ScaledMMLinearKernel,
|
||||
ChannelWiseTorchFP8ScaledMMLinearKernel,
|
||||
PerTensorTorchFP8ScaledMMLinearKernel,
|
||||
RowWiseTorchFP8ScaledMMLinearKernel,
|
||||
@@ -281,6 +282,7 @@ _LINEAR_BACKEND_KERNEL_MAP: dict[str, set[type]] = {
|
||||
PerTensorTorchFP8ScaledMMLinearKernel,
|
||||
ChannelWiseTorchFP8ScaledMMLinearKernel,
|
||||
RowWiseTorchFP8ScaledMMLinearKernel,
|
||||
BlockWiseTorchFP8ScaledMMLinearKernel,
|
||||
},
|
||||
"aiter": {
|
||||
AiterInt8ScaledMMLinearKernel,
|
||||
@@ -380,6 +382,7 @@ _POSSIBLE_FP8_BLOCK_KERNELS: dict[
|
||||
MarlinFP8ScaledMMLinearKernel,
|
||||
TritonFp8BlockScaledMMKernel,
|
||||
HummingFP8ScaledMMLinearKernel,
|
||||
BlockWiseTorchFP8ScaledMMLinearKernel,
|
||||
],
|
||||
PlatformEnum.ROCM: [
|
||||
AiterFp8BlockScaledMMKernel,
|
||||
|
||||
@@ -6,8 +6,11 @@ import math
|
||||
import torch
|
||||
|
||||
from vllm.config import CompilationMode, get_current_vllm_config
|
||||
from vllm.model_executor.layers.quantization.input_quant_fp8 import QuantFP8
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from .BlockScaledMMLinearKernel import Fp8BlockScaledMMLinearKernel
|
||||
from .ScaledMMLinearKernel import (
|
||||
FP8ScaledMMLinearKernel,
|
||||
FP8ScaledMMLinearLayerConfig,
|
||||
@@ -244,3 +247,108 @@ class ChannelWiseTorchFP8ScaledMMLinearKernel(TorchFP8ScaledMMLinearKernel):
|
||||
if bias is not None:
|
||||
output = output + bias
|
||||
return output.to(out_dtype).view(*output_shape)
|
||||
|
||||
|
||||
class BlockWiseTorchFP8ScaledMMLinearKernel(Fp8BlockScaledMMLinearKernel):
|
||||
"""FP8 block-scaled linear kernel using ``torch._scaled_mm``.
|
||||
|
||||
Implements the block-scaled path of ``torch._scaled_mm``, which
|
||||
dispatches on the shapes of the scale tensors. For ``A = [M, K]`` and
|
||||
``B = [K, N]`` (both fp8) the op's block path requires, with float32
|
||||
scales:
|
||||
* 1x128 activation: ``scale_a = [M, ceil(K / 128)]``
|
||||
* 128x128 weight: ``scale_b = [ceil(K / 128), ceil(N / 128)]``
|
||||
"""
|
||||
|
||||
def __init__(self, config: FP8ScaledMMLinearLayerConfig) -> None:
|
||||
super().__init__(config)
|
||||
act_scale_descriptor = config.activation_quant_key.scale
|
||||
self.quant_fp8 = QuantFP8(
|
||||
static=act_scale_descriptor.static,
|
||||
group_shape=act_scale_descriptor.group_shape,
|
||||
num_token_padding=self.get_output_padding(),
|
||||
use_ue8m0=False,
|
||||
column_major_scales=current_platform.is_cuda_alike(),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def is_supported(
|
||||
cls, compute_capability: int | None = None
|
||||
) -> tuple[bool, str | None]:
|
||||
if not (current_platform.is_cuda_alike() or current_platform.is_xpu()):
|
||||
return False, "requires CUDA, ROCm or XPU."
|
||||
# torch._scaled_mm DeepSeek-style (1x128, 128x128) block scaling is
|
||||
# only implemented for SM90 (Hopper) on NVIDIA CUDA. See PyTorch's
|
||||
# _check_deepseek_support():
|
||||
# https://github.com/pytorch/pytorch/blob/33812ece06e2b0d597f73fbe41de03a83f9109f9/aten/src/ATen/native/cuda/ScaledBlas.cpp#L804-L820 # noqa: E501
|
||||
if current_platform.is_cuda():
|
||||
is_sm90 = (
|
||||
compute_capability == 90
|
||||
if compute_capability is not None
|
||||
else current_platform.is_device_capability(90)
|
||||
)
|
||||
if not is_sm90:
|
||||
return (
|
||||
False,
|
||||
"DeepSeek-style (1x128, 128x128) block scaling on CUDA "
|
||||
"requires compute capability 90 (Hopper).",
|
||||
)
|
||||
return True, None
|
||||
|
||||
@classmethod
|
||||
def can_implement(
|
||||
cls, config: FP8ScaledMMLinearLayerConfig
|
||||
) -> tuple[bool, str | None]:
|
||||
can_implement_base, reason = super().can_implement(config)
|
||||
if not can_implement_base:
|
||||
return can_implement_base, reason
|
||||
|
||||
act_group_shape = config.activation_quant_key.scale.group_shape
|
||||
if act_group_shape != GroupShape(1, 128):
|
||||
return (
|
||||
False,
|
||||
"requires 1x128 activation quantization.",
|
||||
)
|
||||
weight_group_shape = config.weight_quant_key.scale.group_shape
|
||||
if weight_group_shape != GroupShape(128, 128):
|
||||
return (
|
||||
False,
|
||||
"requires 128x128 block weight quantization.",
|
||||
)
|
||||
return True, None
|
||||
|
||||
def apply_block_scaled_mm(
|
||||
self,
|
||||
A: torch.Tensor,
|
||||
B: torch.Tensor,
|
||||
As: torch.Tensor,
|
||||
Bs: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
# torch._scaled_mm implemented by cuBLASLt requires the M dimension to be a
|
||||
# multiple of 4; pad A (and its column-major scale) up and slice back.
|
||||
# https://docs.nvidia.com/cuda/cublas/index.html?highlight=128%2520element%25201D%2520and%2520128x128%25202D#scaling-factors-layouts # noqa: E501
|
||||
M = A.shape[0]
|
||||
pad = (-M) % 4 if current_platform.is_cuda() else 0
|
||||
if pad:
|
||||
Ap = A.new_empty((M + pad, A.shape[1]))
|
||||
Ap[:M] = A
|
||||
A = Ap
|
||||
# As is column-major [M, ceil(K/128)] (stride (1, M)); build a padded
|
||||
# column-major buffer and copy only the valid rows. Padded rows are left
|
||||
# uninitialized — they don't affect the sliced-back output.
|
||||
Asp = As.new_empty((As.shape[1], M + pad)).t()
|
||||
Asp[:M] = As
|
||||
As = Asp
|
||||
|
||||
output = torch._scaled_mm(
|
||||
A,
|
||||
B.t(),
|
||||
scale_a=As,
|
||||
scale_b=Bs.t(),
|
||||
out_dtype=self.config.out_dtype,
|
||||
)
|
||||
if type(output) is tuple and len(output) == 2:
|
||||
output = output[0]
|
||||
if pad:
|
||||
output = output[:M, :]
|
||||
return output
|
||||
|
||||
Reference in New Issue
Block a user