From 0e110f696db450f5645ce1daf9945241212d6ef3 Mon Sep 17 00:00:00 2001 From: yinfengLiu Date: Sun, 20 Sep 2026 17:13:55 +0800 Subject: [PATCH] [ROCm][DSV4][Perf] Use FP8 WO_A output projection (#54894) Signed-off-by: Liuyinfeng01 Co-authored-by: Shanshan Shen <467638484@qq.com> --- tests/models/test_deepseek_v4_rocm_wo_a.py | 44 +++++ vllm/models/deepseek_v4/amd/rocm.py | 184 +++++++++++++++++++-- 2 files changed, 217 insertions(+), 11 deletions(-) create mode 100644 tests/models/test_deepseek_v4_rocm_wo_a.py diff --git a/tests/models/test_deepseek_v4_rocm_wo_a.py b/tests/models/test_deepseek_v4_rocm_wo_a.py new file mode 100644 index 0000000000..69c6ed9530 --- /dev/null +++ b/tests/models/test_deepseek_v4_rocm_wo_a.py @@ -0,0 +1,44 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import pytest +import torch + +from vllm.models.deepseek_v4.amd.rocm import _wo_a_block_scale_to_e8m0 + + +def test_wo_a_block_scale_to_e8m0_from_float(): + scale = torch.tensor([[0.5, 1.0, 2.0, 4.0]], dtype=torch.float32) + + encoded = _wo_a_block_scale_to_e8m0(scale) + + assert encoded is not None + torch.testing.assert_close( + encoded, + torch.tensor([[126, 127, 128, 129]], dtype=torch.uint8), + ) + assert encoded.is_contiguous() + + +def test_wo_a_block_scale_to_e8m0_preserves_encoded_scales(): + raw = torch.tensor([[125, 127, 131]], dtype=torch.uint8) + encoded = raw.view(torch.float8_e8m0fnu) + + converted = _wo_a_block_scale_to_e8m0(encoded) + + assert converted is not None + torch.testing.assert_close(converted, raw) + + +@pytest.mark.parametrize( + "scale", + [ + torch.tensor([[0.0, 1.0]]), + torch.tensor([[-1.0, 1.0]]), + torch.tensor([[0.75, 1.0]]), + torch.tensor([[float("inf"), 1.0]]), + torch.ones(1, dtype=torch.int32), + ], +) +def test_wo_a_block_scale_to_e8m0_rejects_invalid_scales(scale: torch.Tensor): + assert _wo_a_block_scale_to_e8m0(scale) is None diff --git a/vllm/models/deepseek_v4/amd/rocm.py b/vllm/models/deepseek_v4/amd/rocm.py index 3cf26f8767..a70a23e3ab 100644 --- a/vllm/models/deepseek_v4/amd/rocm.py +++ b/vllm/models/deepseek_v4/amd/rocm.py @@ -7,6 +7,7 @@ from typing import cast import torch +from vllm import envs from vllm.distributed import ( get_tensor_model_parallel_world_size, tensor_model_parallel_all_reduce, @@ -42,6 +43,49 @@ from vllm.v1.worker.workspace import current_workspace_manager logger = init_logger(__name__) +def _wo_a_block_scale_to_e8m0(scale: torch.Tensor) -> torch.Tensor | None: + """Normalize checkpoint WO_A scales to raw OCP MX E8M0 bytes. + + E8M0 is an unsigned exponent-only scale format with bias 127. A finite + encoded byte ``b`` in ``[0, 254]`` represents ``2 ** (b - 127)``; + ``0xFF`` is reserved for NaN. This is the vendor-neutral OCP encoding used + by AMD AITER/OPUS, not an NVIDIA-specific convention. Reference: OCP + Microscaling Formats (MX) Specification, section 5.4.1: + https://www.opencompute.org/documents/ocp-microscaling-formats-mx-v1-0-spec-final-pdf + + Loaders may preserve the encoded byte as ``float8_e8m0fnu``/``uint8``, + or decode it to a floating-point power of two. Floating-point inputs are + accepted only when the original E8M0 byte can be recovered losslessly. + This function does not quantize or round arbitrary scales. + """ + if scale.dtype == torch.float8_e8m0fnu: + # Reinterpret the native E8M0 storage. ``to(uint8)`` would perform a + # numeric conversion instead of preserving the encoded exponent byte. + return scale.view(torch.uint8).contiguous() + if scale.dtype == torch.uint8: + # The checkpoint loader already exposed the E8M0 wire representation. + return scale.contiguous() + if not scale.dtype.is_floating_point: + return None + + scale_f32 = scale.detach().float() + if not bool(torch.isfinite(scale_f32).all()) or bool((scale_f32 <= 0).any()): + return None + + # With no mantissa, E8M0 can represent only exact powers of two. Rebuild + # the value before encoding so this adapter never silently quantizes a + # general floating-point checkpoint scale. + exponent = torch.round(torch.log2(scale_f32)) + if not torch.equal(torch.exp2(exponent), scale_f32): + return None + + encoded = exponent.to(torch.int32) + 127 + # 0xFF is NaN in OCP E8M0, not a finite exponent. + if int(encoded.min()) < 0 or int(encoded.max()) > 254: + return None + return encoded.to(torch.uint8).contiguous() + + def _trust_dsv4_extra_cache_nan_free( kv_cache_dtype: str, has_kv_transfer: bool, @@ -555,6 +599,10 @@ class DeepseekV4ROCMAiterMLAAttention(DeepseekV4Attention): # Block scale for the preshuffled weight; None = not preshuffled. self._wqa_wkv_scale: torch.Tensor | None = None self._wo_b_scale: torch.Tensor | None = None + self._wo_a_fp8_weight: torch.Tensor | None = None + self._wo_a_e8m0_scale: torch.Tensor | None = None + self._wo_a_cos_cache: torch.Tensor | None = None + self._wo_a_sin_cache: torch.Tensor | None = None self._fused_compressor_weight: torch.Tensor | None self.register_buffer("_fused_compressor_weight", None, persistent=False) self._fused_compressor_split_sizes: tuple[int, int] | None = None @@ -815,6 +863,93 @@ class DeepseekV4ROCMAiterMLAAttention(DeepseekV4Attention): self._wqa_wkv_scale = _prep(self.fused_wqa_wkv) self._wo_b_scale = _prep(self.wo_b) + if _ON_GFX950 and envs.VLLM_ROCM_USE_AITER_FP8BMM: + self._prepare_fp8_wo_a() + + def _prepare_fp8_wo_a(self) -> None: + try: + from aiter.ops.batched_gemm_op_a8w8 import ( + batched_gemm_a8w8_mxscale as mxscale_op, + ) + from aiter.ops.inverse_rope_group_quant import ( + inverse_rope_group_quant as inverse_quant_op, + ) + except ImportError: + logger.warning_once( + "The DeepSeek V4 FP8 WO_A path requires AITER >= 0.1.20; " + "falling back to BF16 WO_A." + ) + return + del mxscale_op, inverse_quant_op + + from vllm.model_executor.layers.quantization.utils.fp8_utils import ( + get_fp8_block_weight_scale, + is_fp8, + ) + + weight = getattr(self.wo_a, "weight", None) + scale = get_fp8_block_weight_scale(self.wo_a) + if scale is None: + # ModelOpt MXFP8 stores the multiplicative E8M0 scale without the + # historical ``_inv`` suffix. + scale = getattr(self.wo_a, "weight_scale", None) + if weight is None or scale is None: + logger.warning_once( + "DeepSeek V4 FP8 WO_A needs a block-scaled FP8 wo_a weight; " + "the layer exposes no weight/weight scale. Falling back to " + "BF16 WO_A." + ) + return + if weight.dim() != 2 or scale.dim() != 2 or not is_fp8(weight.dtype): + logger.warning_once( + "DeepSeek V4 FP8 WO_A needs a 2-D FP8 wo_a weight with a 2-D " + "block scale, got weight %s%s and scale %s. Falling back to " + "BF16 WO_A.", + weight.dtype, + tuple(weight.shape), + tuple(scale.shape), + ) + return + + groups = self.n_local_groups + out_per_group = self.o_lora_rank + out_features, in_features = weight.shape + if ( + out_features != groups * out_per_group + or out_per_group % 128 != 0 + or in_features % 128 != 0 + or scale.shape != (out_features // 128, in_features // 128) + ): + logger.warning_once( + "DeepSeek V4 FP8 WO_A needs group-128 blocks for %d groups of " + "%d outputs, got weight %s and scale %s. Falling back to BF16 " + "WO_A.", + groups, + out_per_group, + tuple(weight.shape), + tuple(scale.shape), + ) + return + + e8m0_scale = _wo_a_block_scale_to_e8m0(scale) + if e8m0_scale is None: + logger.warning_once( + "DeepSeek V4 FP8 WO_A could not losslessly encode the %s wo_a " + "block scale as OCP E8M0. Falling back to BF16 WO_A.", + scale.dtype, + ) + return + + self._wo_a_fp8_weight = weight.view(groups, out_per_group, in_features) + self._wo_a_e8m0_scale = e8m0_scale.view( + groups, out_per_group // 128, in_features // 128 + ) + cache = getattr(self.rotary_emb, "cos_sin_cache_bf16", None) + if cache is None: + cache = self.rotary_emb.cos_sin_cache.to(dtype=torch.bfloat16) + cos_cache, sin_cache = cache.chunk(2, dim=-1) + self._wo_a_cos_cache = cos_cache.contiguous() + self._wo_a_sin_cache = sin_cache.contiguous() def prepare_compressor_gemm_fusion(self) -> bool: if self._fused_compressor_weight is not None: @@ -973,17 +1108,44 @@ class DeepseekV4ROCMAiterMLAAttention(DeepseekV4Attention): ) def _o_proj(self, o: torch.Tensor, positions: torch.Tensor) -> torch.Tensor: - # ROCm BF16 reference wo_a path (inverse RoPE + einsum) + wo_b. - z = rocm_inv_rope_einsum( - self.rotary_emb, - o, - positions, - self.rope_head_dim, - self.n_local_groups, - self.o_lora_rank, - self.wo_a, - ) - zf = z.flatten(1) + if self._wo_a_fp8_weight is not None: + from aiter.ops.batched_gemm_op_a8w8 import ( + batched_gemm_a8w8_mxscale, + ) + from aiter.ops.inverse_rope_group_quant import ( + inverse_rope_group_quant, + ) + + assert self._wo_a_cos_cache is not None + assert self._wo_a_sin_cache is not None + o_fp8, o_scale = inverse_rope_group_quant( + o.view(o.shape[0], self.n_local_heads, self.head_dim), + positions.to(torch.int64), + self._wo_a_cos_cache, + self._wo_a_sin_cache, + num_groups=self.n_local_groups, + quant_group_size=128, + ) + assert self._wo_a_e8m0_scale is not None + zf = batched_gemm_a8w8_mxscale( + o_fp8, + self._wo_a_fp8_weight, + o_scale, + self._wo_a_e8m0_scale, + dtype=o.dtype, + ).flatten(1) + else: + # ROCm BF16 reference wo_a path (inverse RoPE + einsum) + wo_b. + z = rocm_inv_rope_einsum( + self.rotary_emb, + o, + positions, + self.rope_head_dim, + self.n_local_groups, + self.o_lora_rank, + self.wo_a, + ) + zf = z.flatten(1) if self._wo_b_scale is not None and zf.dim() == 2: return self._bpre_attn_gemm(self.wo_b.weight, self._wo_b_scale, zf, True) return self.wo_b(zf)