[Feature] Support batch invariance on ROCm (#52231)
Signed-off-by: Matthew Wong <Matthew.Wong2@amd.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
Andreas Karatzas
parent
50b21a2667
commit
e0317b28ce
@@ -218,6 +218,7 @@ from vllm.model_executor.kernels.linear.scaled_mm.rocm import (
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.scaled_mm.triton import (
|
||||
TritonFp8BlockScaledMMKernel,
|
||||
TritonFP8ScaledMMLinearKernel,
|
||||
TritonInt8ScaledMMLinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.scaled_mm.xpu import (
|
||||
@@ -253,6 +254,20 @@ def _get_linear_backend(*, quantization: str) -> str:
|
||||
return config.kernel_config.linear_backend
|
||||
|
||||
|
||||
def _batch_invariant_fp8_kernel(kernel, config, quantization: str):
|
||||
can_implement, reason = is_supported_and_can_implement_kernel(kernel, config, None)
|
||||
if not can_implement:
|
||||
raise ValueError(f"Batch invariance on ROCm: {reason}")
|
||||
linear_backend = _get_linear_backend(quantization=quantization)
|
||||
if linear_backend not in ("auto", "triton"):
|
||||
logger.warning_once(
|
||||
"VLLM_BATCH_INVARIANT overrides --linear-backend=%s; "
|
||||
"using the Triton backend for deterministic execution.",
|
||||
linear_backend,
|
||||
)
|
||||
return kernel
|
||||
|
||||
|
||||
# Kernel classes covered by each --linear-backend value.
|
||||
_LINEAR_BACKEND_KERNEL_MAP: dict[str, set[type]] = {
|
||||
"b12x": {
|
||||
@@ -723,6 +738,13 @@ def init_fp8_linear_kernel(
|
||||
)
|
||||
|
||||
if activation_quant_key.scale.group_shape.is_per_group():
|
||||
if envs.VLLM_BATCH_INVARIANT and current_platform.is_rocm():
|
||||
force_kernel = _batch_invariant_fp8_kernel(
|
||||
TritonFp8BlockScaledMMKernel,
|
||||
scaled_mm_linear_kernel_config,
|
||||
"fp8_block_w8a8",
|
||||
)
|
||||
|
||||
kernel_type = choose_scaled_mm_linear_kernel(
|
||||
config=scaled_mm_linear_kernel_config,
|
||||
possible_kernels=_POSSIBLE_FP8_BLOCK_KERNELS, # type: ignore[misc]
|
||||
@@ -755,6 +777,13 @@ def init_fp8_linear_kernel(
|
||||
)
|
||||
|
||||
else:
|
||||
if envs.VLLM_BATCH_INVARIANT and current_platform.is_rocm():
|
||||
force_kernel = _batch_invariant_fp8_kernel(
|
||||
TritonFP8ScaledMMLinearKernel,
|
||||
scaled_mm_linear_kernel_config,
|
||||
"fp8_w8a8",
|
||||
)
|
||||
|
||||
kernel_type = choose_scaled_mm_linear_kernel(
|
||||
config=scaled_mm_linear_kernel_config,
|
||||
possible_kernels=_POSSIBLE_FP8_KERNELS, # type: ignore[arg-type]
|
||||
@@ -780,6 +809,19 @@ def init_fp8_linear_kernel(
|
||||
)
|
||||
|
||||
|
||||
def fp8_scaled_mm_is_batch_invariant(
|
||||
kernel: FP8ScaledMMLinearKernel | Fp8BlockScaledMMLinearKernel,
|
||||
) -> bool:
|
||||
"""Whether a row's output from the kernel is independent of the other rows."""
|
||||
if not isinstance(
|
||||
kernel, (CutlassFP8ScaledMMLinearKernel, TritonFP8ScaledMMLinearKernel)
|
||||
):
|
||||
return False
|
||||
# A dynamic per-tensor activation scale is an amax over every row.
|
||||
scale = kernel.config.activation_quant_key.scale
|
||||
return scale.static or not scale.group_shape.is_per_tensor()
|
||||
|
||||
|
||||
def init_int8_linear_kernel(
|
||||
is_channelwise: bool,
|
||||
is_static_input_scheme: bool,
|
||||
@@ -1250,6 +1292,7 @@ def register_linear_kernel(
|
||||
|
||||
|
||||
__all__ = [
|
||||
"fp8_scaled_mm_is_batch_invariant",
|
||||
"init_fp8_linear_kernel",
|
||||
"init_int8_linear_kernel",
|
||||
"init_nvfp4_linear_kernel",
|
||||
|
||||
@@ -62,7 +62,12 @@ if is_aiter_found_and_supported():
|
||||
N = weight.shape[0]
|
||||
K = weight.shape[1]
|
||||
if rocm_use_aiter_fp4_asm_gemm:
|
||||
if M <= 64 and rocm_aiter_ops.is_triton_gemm_afp4wfp4_presh_ws_tuned(N, K):
|
||||
# The preshuffle kernel's tuned configs split K in some M buckets.
|
||||
if (
|
||||
M <= 64
|
||||
and not envs.VLLM_BATCH_INVARIANT
|
||||
and rocm_aiter_ops.is_triton_gemm_afp4wfp4_presh_ws_tuned(N, K)
|
||||
):
|
||||
if x_scales is None:
|
||||
# use hip quant kernel for performance
|
||||
if M >= 32:
|
||||
@@ -116,7 +121,16 @@ if is_aiter_found_and_supported():
|
||||
x_q.shape[0], weight.shape[0], device=x_q.device, dtype=out_dtype
|
||||
)
|
||||
|
||||
gemm_afp4wfp4(x_q, weight, x_s, weight_scale.T, out_dtype, y)
|
||||
config = None
|
||||
if envs.VLLM_BATCH_INVARIANT:
|
||||
# The configs split K in some M buckets (up to M <= 256 for
|
||||
# tuned shapes). Keep the tile, drop the split.
|
||||
from aiter.ops.triton.gemm_afp4wfp4 import _get_config
|
||||
|
||||
config, _ = _get_config(M, N, K)
|
||||
config = dict(config, NUM_KSPLIT=1)
|
||||
|
||||
gemm_afp4wfp4(x_q, weight, x_s, weight_scale.T, out_dtype, y, config)
|
||||
return y
|
||||
|
||||
def gemm_with_dynamic_quant_fake(
|
||||
|
||||
@@ -12,6 +12,7 @@ instead, since ``can_implement`` does not filter on K.
|
||||
import torch
|
||||
from torch.nn.parameter import Parameter
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
|
||||
MXFP8_BLOCK_SIZE,
|
||||
MXFP8_SCALE_DTYPE,
|
||||
@@ -246,7 +247,10 @@ class RocmDotScaledMxfp8LinearKernel(Mxfp8LinearKernel):
|
||||
N, K = weight.shape
|
||||
scale_k = K // MXFP8_BLOCK_SIZE
|
||||
weight_scale = layer.weight_scale.data[:N, :scale_k].contiguous()
|
||||
block_scale = _as_block32_scale(weight_scale)
|
||||
# rocm_mxfp8_block32_gemm's split-K varies with M.
|
||||
block_scale = (
|
||||
None if envs.VLLM_BATCH_INVARIANT else _as_block32_scale(weight_scale)
|
||||
)
|
||||
if block_scale is not None:
|
||||
# Checkpoints with 32x32 scale blocks (DeepSeek V4/V4.1) get them
|
||||
# back from the loader expanded per row; keep the blocks and use
|
||||
|
||||
@@ -39,6 +39,7 @@ from vllm.model_executor.kernels.linear.scaled_mm.ScaledMMLinearKernel import (
|
||||
ScaledMMLinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.scaled_mm.triton import (
|
||||
TritonFP8ScaledMMLinearKernel,
|
||||
TritonInt8ScaledMMLinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.scaled_mm.xpu import (
|
||||
@@ -65,6 +66,7 @@ __all__ = [
|
||||
"PerTensorTorchFP8ScaledMMLinearKernel",
|
||||
"RowWiseTorchFP8ScaledMMLinearKernel",
|
||||
"ROCmFP8ScaledMMLinearKernel",
|
||||
"TritonFP8ScaledMMLinearKernel",
|
||||
"TritonInt8ScaledMMLinearKernel",
|
||||
"ZentorchInt8ScaledMMLinearKernel",
|
||||
"Fp8BlockScaledMMLinearKernel",
|
||||
|
||||
@@ -20,6 +20,8 @@ from .BlockScaledMMLinearKernel import (
|
||||
)
|
||||
from .cutlass import CutlassInt8ScaledMMLinearKernel
|
||||
from .ScaledMMLinearKernel import (
|
||||
FP8ScaledMMLinearKernel,
|
||||
FP8ScaledMMLinearLayerConfig,
|
||||
Int8ScaledMMLinearLayerConfig,
|
||||
)
|
||||
|
||||
@@ -156,6 +158,46 @@ class TritonInt8ScaledMMLinearKernel(CutlassInt8ScaledMMLinearKernel):
|
||||
return out
|
||||
|
||||
|
||||
class TritonFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
|
||||
@classmethod
|
||||
def is_supported(
|
||||
cls, compute_capability: int | None = None
|
||||
) -> tuple[bool, str | None]:
|
||||
if not current_platform.is_cuda_alike():
|
||||
return False, "requires ROCm or CUDA."
|
||||
return True, None
|
||||
|
||||
@classmethod
|
||||
def can_implement(cls, c: FP8ScaledMMLinearLayerConfig) -> tuple[bool, str | None]:
|
||||
act_group = c.activation_quant_key.scale.group_shape
|
||||
weight_group = c.weight_quant_key.scale.group_shape
|
||||
if not (act_group.is_per_tensor() or act_group.is_per_token()):
|
||||
return False, "requires per-tensor or per-token activation scales."
|
||||
if not (
|
||||
weight_group.is_per_tensor()
|
||||
or weight_group.is_per_channel()
|
||||
or weight_group.is_per_token()
|
||||
):
|
||||
return False, "requires per-tensor, per-channel or per-token weight scales."
|
||||
return True, None
|
||||
|
||||
def apply_scaled_mm(
|
||||
self,
|
||||
*,
|
||||
A: torch.Tensor,
|
||||
B: torch.Tensor,
|
||||
out_dtype: torch.dtype,
|
||||
As: torch.Tensor,
|
||||
Bs: torch.Tensor,
|
||||
bias: torch.Tensor | None,
|
||||
output_shape: list,
|
||||
) -> torch.Tensor:
|
||||
out = triton_scaled_mm(
|
||||
A, B, scale_a=As, scale_b=Bs, out_dtype=out_dtype, bias=bias
|
||||
)
|
||||
return out.view(*output_shape)
|
||||
|
||||
|
||||
class TritonFp8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel):
|
||||
@classmethod
|
||||
def is_supported(cls, compute_capability=None):
|
||||
|
||||
Reference in New Issue
Block a user