[ROCm][DSV4] B-preshuffle the attention fp8 projections (#46720)

Signed-off-by: Mehmet Cagri Kaymak <mehmet.kaymak@amd.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
Mehmet Cagri
2026-07-30 13:59:52 -05:00
committed by GitHub
co-authored by Claude Opus 4.8 Andreas Karatzas
parent 5f8f72866c
commit 12a34a6bc7
5 changed files with 187 additions and 9 deletions
+53 -2
View File
@@ -687,6 +687,32 @@ def _rocm_aiter_gemm_a8w8_blockscale_fake(
return Y
def _rocm_aiter_gemm_a8w8_blockscale_bpreshuffle_impl(
A: torch.Tensor,
B: torch.Tensor,
As: torch.Tensor,
Bs: torch.Tensor,
output_dtype: torch.dtype = torch.float16,
) -> torch.Tensor:
from aiter import gemm_a8w8_blockscale_bpreshuffle
# B preshuffled (shuffle_weight (16,16)); As column-major group scale.
return gemm_a8w8_blockscale_bpreshuffle(A, B, As, Bs, dtype=output_dtype)
def _rocm_aiter_gemm_a8w8_blockscale_bpreshuffle_fake(
A: torch.Tensor,
B: torch.Tensor,
As: torch.Tensor,
Bs: torch.Tensor,
output_dtype: torch.dtype = torch.float16,
) -> torch.Tensor:
m = A.shape[0]
n = B.shape[0]
Y = torch.empty(m, n, dtype=output_dtype, device=A.device)
return Y
def _rocm_aiter_rmsnorm_fused_add_dynamic_quant_impl(
x: torch.Tensor,
residual: torch.Tensor,
@@ -1145,17 +1171,21 @@ def _rocm_aiter_fused_rms_gated_fp8_group_quant_fake(
def _rocm_aiter_group_fp8_quant_impl(
x: torch.Tensor,
group_size: int,
transpose_scale: bool = False,
) -> tuple[torch.Tensor, torch.Tensor]:
assert x.shape[-1] % group_size == 0, "Input shape must be divisible by group size"
from aiter import QuantType, get_hip_quant
aiter_per1x128_quant = get_hip_quant(QuantType.per_1x128)
return aiter_per1x128_quant(x.contiguous(), quant_dtype=FP8_DTYPE)
return aiter_per1x128_quant(
x.contiguous(), quant_dtype=FP8_DTYPE, transpose_scale=transpose_scale
)
def _rocm_aiter_group_fp8_quant_fake(
x: torch.Tensor,
group_size: int,
transpose_scale: bool = False,
) -> tuple[torch.Tensor, torch.Tensor]:
M, N = x.shape
x_fp8 = torch.empty((M, N), dtype=FP8_DTYPE, device=x.device)
@@ -1955,6 +1985,12 @@ class rocm_aiter_ops:
fake_impl=_rocm_aiter_gemm_a8w8_blockscale_fake,
)
direct_register_custom_op(
op_name="rocm_aiter_gemm_a8w8_blockscale_bpreshuffle",
op_func=_rocm_aiter_gemm_a8w8_blockscale_bpreshuffle_impl,
fake_impl=_rocm_aiter_gemm_a8w8_blockscale_bpreshuffle_fake,
)
direct_register_custom_op(
op_name="rocm_aiter_rmsnorm_fused_dynamic_quant",
op_func=_rocm_aiter_rmsnorm_fused_dynamic_quant_impl,
@@ -2209,6 +2245,18 @@ class rocm_aiter_ops:
A, B, As, Bs, output_dtype
)
@staticmethod
def gemm_a8w8_blockscale_bpreshuffle(
A: torch.Tensor,
B: torch.Tensor,
As: torch.Tensor,
Bs: torch.Tensor,
output_dtype: torch.dtype = torch.float16,
) -> torch.Tensor:
return torch.ops.vllm.rocm_aiter_gemm_a8w8_blockscale_bpreshuffle(
A, B, As, Bs, output_dtype
)
@staticmethod
def fused_moe(
hidden_states: torch.Tensor,
@@ -2702,9 +2750,12 @@ class rocm_aiter_ops:
def group_fp8_quant(
input_2d: torch.Tensor,
group_size: int = 128,
transpose_scale: bool = False,
) -> tuple[torch.Tensor, torch.Tensor]:
assert group_size == 128, "Group size must be 128"
return torch.ops.vllm.rocm_aiter_group_fp8_quant(input_2d, group_size)
return torch.ops.vllm.rocm_aiter_group_fp8_quant(
input_2d, group_size, transpose_scale
)
@staticmethod
def is_triton_gemm_w8a8_tuned(n: int, k: int) -> bool:
@@ -142,6 +142,10 @@ def process_weights_after_loading(
if isinstance(module, HpcModule):
module.process_weights_after_loading(model)
# Model-level post-load hook, after the per-layer quant finalize.
if hasattr(model, "process_weights_after_loading"):
model.process_weights_after_loading()
# Needed for torchao model reloading via model.reload_weights
# @kylesayrs @jerryzh168 this can be removed if callers move to `reload_weights`
if model_config.quantization == "torchao":
+53 -3
View File
@@ -9,6 +9,7 @@ import torch
import torch.nn as nn
import vllm.envs as envs
from vllm._aiter_ops import rocm_aiter_ops
from vllm.config import VllmConfig, get_current_vllm_config
from vllm.distributed import (
get_pp_group,
@@ -104,8 +105,50 @@ class DeepseekV4MLP(nn.Module):
else:
self.act_fn = SiluAndMul()
# gate_up_proj B-preshuffle (ColumnParallel -> no all-reduce); set at load.
self._gateup = rocm_aiter_ops.is_enabled()
# Block scale for the preshuffled gate_up weight; None = not preshuffled.
self._gateup_scale: torch.Tensor | None = None
def prepare_gateup_preshuffle(self) -> None:
# B-preshuffle the gate_up_proj weight in place (single weight).
if not self._gateup:
return
from vllm.model_executor.utils import replace_parameter
w = getattr(self.gate_up_proj, "weight", None)
ws = getattr(self.gate_up_proj, "weight_scale_inv", None) # per-block scale
if w is None or ws is None or w.dim() != 2:
return
# K % 128 (group-128 quant) and N % 16 (shuffle_weight) must hold.
if w.shape[-1] % 128 != 0 or w.shape[0] % 16 != 0:
return
if ws.dtype == torch.float8_e8m0fnu:
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
_upcast_e8m0_to_fp32,
)
ws = _upcast_e8m0_to_fp32(ws).contiguous()
replace_parameter(
self.gate_up_proj,
"weight",
rocm_aiter_ops.shuffle_weight(w.data, layout=(16, 16)),
)
self._gateup_scale = ws
def forward(self, x):
gate_up, _ = self.gate_up_proj(x)
if self._gateup_scale is not None and x.dim() == 2:
# gate_up via fp8 group-quant (col-major) + B-preshuffle GEMM.
x_fp8, x_scale = rocm_aiter_ops.group_fp8_quant(x, transpose_scale=True)
gate_up = rocm_aiter_ops.gemm_a8w8_blockscale_bpreshuffle(
x_fp8,
self.gate_up_proj.weight,
x_scale,
self._gateup_scale,
output_dtype=x.dtype,
)
else:
gate_up, _ = self.gate_up_proj(x)
x = self.act_fn(gate_up)
x, _ = self.down_proj(x)
return x
@@ -961,8 +1004,15 @@ class DeepseekV4ForCausalLM(nn.Module, SupportsPP, SupportsEagle3):
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(self, skip_substrs=["mtp."])
loaded_params = loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
return loaded_params
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
def process_weights_after_loading(self) -> None:
# After per-layer quant finalize, so we preshuffle the final fp8 weights.
for module in self.modules():
if isinstance(module, DeepseekV4ROCMAiterMLAAttention):
module.prepare_attn_preshuffle()
elif isinstance(module, DeepseekV4MLP):
module.prepare_gateup_preshuffle()
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
return self.model.get_expert_mapping()
+71 -1
View File
@@ -6,6 +6,10 @@ from typing import cast
import torch
from vllm.distributed import (
get_tensor_model_parallel_world_size,
tensor_model_parallel_all_reduce,
)
from vllm.forward_context import get_forward_context
from vllm.models.deepseek_v4.attention import DeepseekV4Attention
from vllm.models.deepseek_v4.common.ops import dequantize_and_gather_k_cache
@@ -442,10 +446,73 @@ class DeepseekV4ROCMAiterMLAAttention(DeepseekV4Attention):
backend_cls = DeepseekV4ROCMAiterMLASparseBackend
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
# Block scale for the preshuffled weight; None = not preshuffled.
self._wqa_wkv_scale: torch.Tensor | None = None
self._wo_b_scale: torch.Tensor | None = None
@classmethod
def get_padded_num_q_heads(cls, num_heads: int) -> int:
return num_heads
def prepare_attn_preshuffle(self) -> None:
from vllm._aiter_ops import rocm_aiter_ops
if not rocm_aiter_ops.is_enabled():
return
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
_upcast_e8m0_to_fp32,
)
from vllm.model_executor.utils import replace_parameter
def _prep(linear) -> torch.Tensor | None:
w = getattr(linear, "weight", None)
if w is None or w.dim() != 2:
return None
# K % 128 (group-128 quant) and N % 16 (shuffle_weight) must hold.
if w.shape[-1] % 128 != 0 or w.shape[0] % 16 != 0:
return None
ws = getattr(linear, "weight_scale_inv", None) # per-block scale
if ws is None:
return None
if ws.dtype == torch.float8_e8m0fnu:
ws = _upcast_e8m0_to_fp32(ws).contiguous()
# Shuffle the weight in place (single weight, no unshuffled copy).
replace_parameter(
linear,
"weight",
rocm_aiter_ops.shuffle_weight(w.data, layout=(16, 16)),
)
return ws
self._wqa_wkv_scale = _prep(self.fused_wqa_wkv)
self._wo_b_scale = _prep(self.wo_b)
def _bpre_attn_gemm(
self,
weight: torch.Tensor,
scale: torch.Tensor,
x: torch.Tensor,
reduce_tp: bool,
) -> torch.Tensor:
from vllm._aiter_ops import rocm_aiter_ops
x_fp8, x_scale = rocm_aiter_ops.group_fp8_quant(x, transpose_scale=True)
out = rocm_aiter_ops.gemm_a8w8_blockscale_bpreshuffle(
x_fp8, weight, x_scale, scale, output_dtype=x.dtype
)
if reduce_tp and get_tensor_model_parallel_world_size() > 1:
out = tensor_model_parallel_all_reduce(out)
return out
def _fused_wqa_wkv_gemm(self, hidden_states: torch.Tensor) -> torch.Tensor:
if self._wqa_wkv_scale is not None and hidden_states.dim() == 2:
return self._bpre_attn_gemm(
self.fused_wqa_wkv.weight, self._wqa_wkv_scale, hidden_states, False
)
return super()._fused_wqa_wkv_gemm(hidden_states)
def _o_proj(self, o: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
# ROCm BF16 reference wo_a path (inverse RoPE + einsum) + wo_b.
z = rocm_inv_rope_einsum(
@@ -457,7 +524,10 @@ class DeepseekV4ROCMAiterMLAAttention(DeepseekV4Attention):
self.o_lora_rank,
self.wo_a,
)
return self.wo_b(z.flatten(1))
zf = z.flatten(1)
if self._wo_b_scale is not None and zf.dim() == 2:
return self._bpre_attn_gemm(self.wo_b.weight, self._wo_b_scale, zf, True)
return self.wo_b(zf)
def forward_mqa(
self,
+6 -3
View File
@@ -390,6 +390,11 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
# Inverse-RoPE + wo_a + wo_b output projection (platform-specific).
return self._o_proj(o, positions)
def _fused_wqa_wkv_gemm(self, hidden_states: torch.Tensor) -> torch.Tensor:
# MergedColumnParallelLinear returns (output, bias); bias is None.
qr_kv, _ = self.fused_wqa_wkv(hidden_states)
return qr_kv
def attn_gemm_parallel_execute(self, hidden_states) -> tuple[Any, ...]:
aux_streams = self.aux_stream_list
if aux_streams is not None:
@@ -434,9 +439,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
aux_fns[2] = indexer_compressor_kv_score
def fused_wqa_wkv() -> torch.Tensor:
# MergedColumnParallelLinear returns (output, bias); bias is None.
qr_kv, _ = self.fused_wqa_wkv(hidden_states)
return qr_kv
return self._fused_wqa_wkv_gemm(hidden_states)
qr_kv, (kv_score, indexer_weights, indexer_kv_score) = execute_in_parallel(
fused_wqa_wkv,