[ROCm][Bugfix] Fix AITER preshuffled FP8 block-scale kernel (#56433)

Signed-off-by: Matthew Wong <Matthew.Wong2@amd.com>
This commit is contained in:
Matt
2026-09-11 11:44:57 +00:00
committed by GitHub
parent a2bc2ffb2c
commit b4da4d17ae
5 changed files with 120 additions and 36 deletions
@@ -2,6 +2,8 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import ClassVar
import torch
from vllm import _custom_ops as ops
@@ -462,6 +464,12 @@ class AiterFp8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel):
class AiterPreshuffledFp8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel):
"""Aiter FP8 block-scaled GEMM using a pre-shuffled (bpreshuffle) weight."""
# gemm_a8w8_blockscale_bpreshuffle reads the activation scale column-major.
wants_transposed_act_scale: ClassVar[bool] = True
# process_weights_after_loading shuffles layer.weight to layout (16, 16).
preshuffles_weight: ClassVar[bool] = True
@classmethod
def is_supported(
cls, compute_capability: int | None = None
@@ -519,9 +527,25 @@ class AiterPreshuffledFp8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel):
return True, None
@staticmethod
def _reads_weight_directly(layer: torch.nn.Module) -> bool:
"""True when something other than apply_weights consumes layer.weight.
Such a weight must stay in the plain layout: ``is_bmm`` marks a stack
of matrices (wo_a), ``skip_weight_relayout`` marks MLA's kv_b_proj.
Both are stamped after construction, so can_implement cannot see them.
"""
return bool(
getattr(layer, "is_bmm", False)
or getattr(layer, "skip_weight_relayout", False)
)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
super().process_weights_after_loading(layer)
if self._reads_weight_directly(layer):
return
params = FP8BlockParams.from_layer(layer)
if params.weight_scale_inv is not None:
ws, attr = params.weight_scale_inv, params.WEIGHT_SCALE_INV
@@ -565,11 +589,25 @@ class AiterPreshuffledFp8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel):
else params.weight_scale_inv
)
# Left unshuffled above; kv_b_proj still reaches here from MLA's
# prefill-context path.
plain = self._reads_weight_directly(layer)
x_2d = x.view(-1, x.shape[-1])
A, As = rocm_aiter_ops.group_fp8_quant(x_2d, transpose_scale=True)
output = rocm_aiter_ops.gemm_a8w8_blockscale_bpreshuffle(
A, params.weight, As, Bs, output_dtype=self.config.out_dtype
)
A, As = rocm_aiter_ops.group_fp8_quant(x_2d, transpose_scale=not plain)
if plain:
output = rocm_aiter_ops.gemm_a8w8_blockscale(
A,
params.weight,
As,
Bs,
list(self.weight_group_shape),
output_dtype=self.config.out_dtype,
)
else:
output = rocm_aiter_ops.gemm_a8w8_blockscale_bpreshuffle(
A, params.weight, As, Bs, output_dtype=self.config.out_dtype
)
if bias is not None:
output = output + bias
return output.view(*x.shape[:-1], params.weight.shape[0])
@@ -581,7 +619,11 @@ class AiterPreshuffledFp8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel):
As: torch.Tensor,
Bs: torch.Tensor,
) -> torch.Tensor:
raise NotImplementedError(
"AiterPreshuffledFp8BlockScaledMMKernel overrides apply_weights and "
"does not use apply_block_scaled_mm."
"""Block-scaled GEMM for callers that pre-quantize their activations.
``As`` must be column-major; a row-major one will not raise, it just
returns wrong numbers for M > 1.
"""
return rocm_aiter_ops.gemm_a8w8_blockscale_bpreshuffle(
A, B, As, Bs, output_dtype=self.config.out_dtype
)