[ROCm][Bugfix] Revert #56433 + #51692 to fix accuracy breakdown for DeepSeek-V4 (#57132)

Signed-off-by: Shanshan Shen <87969357+shen-shanshan@users.noreply.github.com>
This commit is contained in:
Shanshan Shen
2026-09-16 18:58:23 +00:00
committed by GitHub
parent 403f182c02
commit bc0f47cd03
8 changed files with 34 additions and 402 deletions
@@ -177,7 +177,6 @@ from vllm.model_executor.kernels.linear.scaled_mm.aiter import (
AiterHipbMMPerTokenFp8ScaledMMLinearKernel,
AiterInt8ScaledMMLinearKernel,
AiterPerTokenFp8ScaledMMLinearKernel,
AiterPreshuffledFp8BlockScaledMMKernel,
AiterPreshuffledPerTokenFp8ScaledMMLinearKernel,
)
from vllm.model_executor.kernels.linear.scaled_mm.b12x import (
@@ -322,7 +321,6 @@ _LINEAR_BACKEND_KERNEL_MAP: dict[str, set[type]] = {
"aiter": {
AiterInt8ScaledMMLinearKernel,
AiterFp8BlockScaledMMKernel,
AiterPreshuffledFp8BlockScaledMMKernel,
AiterPerTokenFp8ScaledMMLinearKernel,
AiterPreshuffledPerTokenFp8ScaledMMLinearKernel,
AiterMxfp4LinearKernel,
@@ -466,7 +464,6 @@ _POSSIBLE_FP8_BLOCK_KERNELS: dict[
BlockWiseTorchFP8ScaledMMLinearKernel,
],
PlatformEnum.ROCM: [
AiterPreshuffledFp8BlockScaledMMKernel,
AiterFp8BlockScaledMMKernel,
TritonFp8BlockScaledMMKernel,
],
@@ -1250,7 +1247,6 @@ __all__ = [
"ScaledMMLinearLayerConfig",
"AiterHipbMMPerTokenFp8ScaledMMLinearKernel",
"AiterPreshuffledPerTokenFp8ScaledMMLinearKernel",
"AiterPreshuffledFp8BlockScaledMMKernel",
"AiterPerTokenFp8ScaledMMLinearKernel",
"NvFp4LinearKernel",
"NvFp4LinearLayerConfig",
@@ -2,8 +2,6 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import ClassVar
import torch
from vllm import _custom_ops as ops
@@ -474,171 +472,3 @@ class AiterFp8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel):
return gemm_a8w8_blockscale_op(
A, B, As, Bs, list(self.weight_group_shape), output_dtype=out_dtype
)
class AiterPreshuffledFp8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel):
"""Aiter FP8 block-scaled GEMM using a pre-shuffled (bpreshuffle) weight."""
# gemm_a8w8_blockscale_bpreshuffle reads the activation scale column-major.
wants_transposed_act_scale: ClassVar[bool] = True
# process_weights_after_loading shuffles layer.weight to layout (16, 16).
preshuffles_weight: ClassVar[bool] = True
@classmethod
def is_supported(
cls, compute_capability: int | None = None
) -> tuple[bool, str | None]:
return AiterPreshuffledPerTokenFp8ScaledMMLinearKernel.is_supported(
compute_capability
)
@classmethod
def can_implement(cls, config: FP8ScaledMMLinearLayerConfig):
can_implement_base, reason = super().can_implement(config)
if not can_implement_base:
return can_implement_base, reason
act_quant_desc = config.activation_quant_key.scale
if act_quant_desc.group_shape != GroupShape(1, 128):
return (
False,
(
"Supports only dynamic per token group activation "
"quantization with group_shape=(1,128)."
),
)
# bpreshuffle GEMM requires aiter fp8 linear to be enabled and an fp8
# 2D weight with N and K divisible by 128. Require a tuned config as the
# fallback can trigger faults or numerical errors.
if not rocm_aiter_ops.is_linear_fp8_enabled():
return (
False,
(
"requires setting `VLLM_ROCM_USE_AITER=1` "
"and `VLLM_ROCM_USE_AITER_LINEAR=1`."
),
)
n, k = config.weight_shape
if not (n % 128 == 0 and k % 128 == 0):
return (
False,
(
f"requires N and K dimensions divisible by 128, received "
f"N={n} and K={k}."
),
)
if not rocm_aiter_ops.is_blockscale_bpreshuffle_tuned(n, k):
return (
False,
(
f"requires a tuned aiter blockscale bpreshuffle config for "
f"N={n} and K={k}."
),
)
return True, None
@staticmethod
def _reads_weight_directly(layer: torch.nn.Module) -> bool:
"""True when something other than apply_weights consumes layer.weight.
Such a weight must stay in the plain layout: ``is_bmm`` marks a stack
of matrices (wo_a), ``skip_weight_relayout`` marks MLA's kv_b_proj.
Both are stamped after construction, so can_implement cannot see them.
"""
return bool(
getattr(layer, "is_bmm", False)
or getattr(layer, "skip_weight_relayout", False)
)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
super().process_weights_after_loading(layer)
if self._reads_weight_directly(layer):
return
params = FP8BlockParams.from_layer(layer)
if params.weight_scale_inv is not None:
ws, attr = params.weight_scale_inv, params.WEIGHT_SCALE_INV
else:
ws, attr = params.weight_scale, params.WEIGHT_SCALE
if ws is not None and ws.dtype == torch.float8_e8m0fnu:
replace_parameter(layer, attr, _upcast_e8m0_to_fp32(ws).contiguous())
weight = params.weight
# runtime safety net
assert (
weight.dim() == 2
and weight.dtype == current_platform.fp8_dtype()
and weight.shape[0] % 128 == 0
and weight.shape[1] % 128 == 0
), (
"AiterPreshuffledFp8BlockScaledMMKernel requires a 2D fp8 weight "
"with N and K divisible by 128."
)
shuffled_weight = rocm_aiter_ops.shuffle_weight(
weight.contiguous(), layout=(16, 16)
)
replace_parameter(
layer,
params.WEIGHT,
torch.nn.Parameter(shuffled_weight.data, requires_grad=False),
)
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
**kwargs,
) -> torch.Tensor:
params = self._get_layer_params(layer)
Bs = (
params.weight_scale
if params.weight_scale_inv is None
else params.weight_scale_inv
)
# Left unshuffled above; kv_b_proj still reaches here from MLA's
# prefill-context path.
plain = self._reads_weight_directly(layer)
x_2d = x.view(-1, x.shape[-1])
A, As = rocm_aiter_ops.group_fp8_quant(x_2d, transpose_scale=not plain)
if plain:
output = rocm_aiter_ops.gemm_a8w8_blockscale(
A,
params.weight,
As,
Bs,
list(self.weight_group_shape),
output_dtype=self.config.out_dtype,
)
else:
output = rocm_aiter_ops.gemm_a8w8_blockscale_bpreshuffle(
A, params.weight, As, Bs, output_dtype=self.config.out_dtype
)
if bias is not None:
output = output + bias
return output.view(*x.shape[:-1], params.weight.shape[0])
def apply_block_scaled_mm(
self,
A: torch.Tensor,
B: torch.Tensor,
As: torch.Tensor,
Bs: torch.Tensor,
) -> torch.Tensor:
"""Block-scaled GEMM for callers that pre-quantize their activations.
``As`` must be column-major; a row-major one will not raise, it just
returns wrong numbers for M > 1.
"""
return rocm_aiter_ops.gemm_a8w8_blockscale_bpreshuffle(
A, B, As, Bs, output_dtype=self.config.out_dtype
)
@@ -582,9 +582,10 @@ class MLAAttention(nn.Module, AttentionLayerBase):
)
self.q_pad_num_heads = getattr(self.impl, "q_pad_num_heads", None)
self.is_amx_bmm_enabled = getattr(self.impl, "uses_amx_bmm", False)
# MLA reads this weight directly to build W_UK/W_UV, so a backend must
# not relayout it at load time.
kv_b_proj.skip_weight_relayout = True
# AMX reads kv_b_proj's weight directly and never calls it live; the
# reference CPU MLA backend calls it but isn't perf-critical. Skip
# the packed-kernel dispatch either way.
kv_b_proj._cpu_skip_gemm_dispatch = True
self.use_direct_call = not current_platform.opaque_attention_op()
vllm_config = get_current_vllm_config()
+3 -3
View File
@@ -205,9 +205,9 @@ class UnquantizedLinearMethod(LinearMethodBase):
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
if current_platform.is_cpu():
# MLA's kv_b_proj (see `skip_weight_relayout`): not perf-critical,
# so skip packing and use a plain fallback.
if getattr(layer, "skip_weight_relayout", False):
# MLA's kv_b_proj (see `_cpu_skip_gemm_dispatch`): not
# perf-critical, so skip packing and use a plain fallback.
if getattr(layer, "_cpu_skip_gemm_dispatch", False):
layer.cpu_linear = torch.nn.functional.linear
return