[ROCm][Bugfix] Revert #56433 + #51692 to fix accuracy breakdown for DeepSeek-V4 (#57132)

Signed-off-by: Shanshan Shen <87969357+shen-shanshan@users.noreply.github.com>
This commit is contained in:
Shanshan Shen
2026-09-16 18:58:23 +00:00
committed by GitHub
parent 403f182c02
commit bc0f47cd03
8 changed files with 34 additions and 402 deletions
-125
View File
@@ -15,131 +15,6 @@ pytestmark = pytest.mark.skipif(
)
def _preshuffled_fp8_linear(
holder: str = "quant_method", weight_shape: tuple[int, int] = (256, 128)
) -> nn.Module:
pytest.importorskip("aiter")
from vllm.model_executor.kernels.linear.scaled_mm.aiter import (
AiterPreshuffledFp8BlockScaledMMKernel,
)
from vllm.model_executor.kernels.linear.scaled_mm.ScaledMMLinearKernel import (
FP8ScaledMMLinearLayerConfig,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8Dynamic128Sym,
kFp8Static128BlockSym,
)
config = FP8ScaledMMLinearLayerConfig(
weight_quant_key=kFp8Static128BlockSym,
activation_quant_key=kFp8Dynamic128Sym,
input_dtype=torch.bfloat16,
out_dtype=torch.bfloat16,
weight_shape=weight_shape,
)
layer = nn.Module()
layer.weight = nn.Parameter(
torch.randn(*weight_shape, device="cuda").to(current_platform.fp8_dtype()),
requires_grad=False,
)
layer.register_parameter(
"weight_scale" if holder == "scheme" else "weight_scale_inv",
nn.Parameter(
torch.ones(*(dim // 128 for dim in weight_shape), device="cuda"),
requires_grad=False,
),
)
setattr(
layer,
holder,
SimpleNamespace(fp8_linear=AiterPreshuffledFp8BlockScaledMMKernel(config)),
)
return layer
@pytest.mark.parametrize("holder", ["quant_method", "scheme"])
def test_rocm_wo_a_keeps_row_major_weights(holder: str, default_vllm_config) -> None:
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import _get_cached_wo_a_bf16
layer = _preshuffled_fp8_linear(holder)
layer.is_bmm = True
original = layer.weight.detach().clone()
kernel = getattr(layer, holder).fp8_linear
kernel.process_weights_after_loading(layer)
actual = _get_cached_wo_a_bf16(layer, 2, 128, 128)
torch.testing.assert_close(actual, original.to(torch.bfloat16).view(2, 128, 128))
@pytest.mark.parametrize("holder", ["quant_method", "scheme"])
def test_rocm_gateup_shuffles_once_and_down_proj_keeps_preshuffled_backend(
holder: str, monkeypatch: pytest.MonkeyPatch, default_vllm_config
) -> None:
from vllm._aiter_ops import rocm_aiter_ops
from vllm.models.deepseek_v4.amd import model as rocm_model
gate_up, down = _preshuffled_fp8_linear(holder), _preshuffled_fp8_linear(holder)
original_gate_up = gate_up.weight.detach().clone()
original_down = down.weight.detach().clone()
monkeypatch.setattr(
rocm_model, "MergedColumnParallelLinear", lambda *a, **k: gate_up
)
monkeypatch.setattr(rocm_model, "RowParallelLinear", lambda *a, **k: down)
monkeypatch.setattr(rocm_aiter_ops, "is_enabled", lambda: True)
model = rocm_model.DeepseekV4MLP(128, 128, "silu")
for linear in (gate_up, down):
getattr(linear, holder).fp8_linear.process_weights_after_loading(linear)
model.prepare_gateup_preshuffle()
for linear, original in ((gate_up, original_gate_up), (down, original_down)):
expected = rocm_aiter_ops.shuffle_weight(original, layout=(16, 16))
torch.testing.assert_close(
linear.weight.view(torch.uint8), expected.view(torch.uint8)
)
def test_rocm_fused_qkv_quant_matches_preshuffled_gemm_scale_layout(
monkeypatch: pytest.MonkeyPatch, default_vllm_config
) -> None:
from vllm._aiter_ops import rocm_aiter_ops
from vllm.models.deepseek_v4.amd.rocm import (
DeepseekV4ROCMAiterMLAAttention,
apply_pre_quantized_block_scaled_mm,
)
if not rocm_aiter_ops.is_blockscale_bpreshuffle_tuned(1024, 256):
pytest.skip("Requires a tuned AITER preshuffled FP8 GEMM")
linear = _preshuffled_fp8_linear(weight_shape=(1024, 256))
original = linear.weight.detach().clone()
linear.quant_method.fp8_linear.process_weights_after_loading(linear)
attention = DeepseekV4ROCMAiterMLAAttention.__new__(DeepseekV4ROCMAiterMLAAttention)
nn.Module.__init__(attention)
attention.wq_b = linear
attention.indexer = None
attention.q_lora_rank, attention.head_dim = 256, 128
attention.eps = 1e-5
attention.q_norm = SimpleNamespace(
weight=torch.ones(256, dtype=torch.bfloat16, device="cuda")
)
attention.kv_norm = SimpleNamespace(
weight=torch.ones(128, dtype=torch.bfloat16, device="cuda")
)
monkeypatch.setattr(rocm_aiter_ops, "is_linear_fp8_enabled", lambda: True)
# Multiple rows and groups with different ranges expose byte-order errors.
inputs = torch.randn(16, 384, dtype=torch.bfloat16, device="cuda")
inputs[:, :128] *= 4
quantized, scales, _ = attention._split_qkv_and_norm(inputs)
assert scales is not None
output = apply_pre_quantized_block_scaled_mm(linear, quantized, scales)
q = inputs[:, :256].float()
normalized = q * torch.rsqrt(q.square().mean(dim=-1, keepdim=True) + attention.eps)
reference_scales = normalized.view(16, 2, 128).abs().amax(dim=-1)
reference_scales /= torch.finfo(current_platform.fp8_dtype()).max
dequantized = quantized.float() * reference_scales.repeat_interleave(128, dim=1)
expected = dequantized @ original.float().T
torch.testing.assert_close(output.float(), expected, atol=0.125, rtol=0.01)
def test_rocm_packed_kv_cache_auto_uses_ds_mla_layout() -> None:
from vllm.config import CacheConfig
from vllm.models.deepseek_v4.attention import _resolve_dsv4_kv_cache_dtype