diff --git a/vllm/_aiter_ops.py b/vllm/_aiter_ops.py index 2a6bdeaa2a..5d70d97b23 100644 --- a/vllm/_aiter_ops.py +++ b/vllm/_aiter_ops.py @@ -687,6 +687,32 @@ def _rocm_aiter_gemm_a8w8_blockscale_fake( return Y +def _rocm_aiter_gemm_a8w8_blockscale_bpreshuffle_impl( + A: torch.Tensor, + B: torch.Tensor, + As: torch.Tensor, + Bs: torch.Tensor, + output_dtype: torch.dtype = torch.float16, +) -> torch.Tensor: + from aiter import gemm_a8w8_blockscale_bpreshuffle + + # B preshuffled (shuffle_weight (16,16)); As column-major group scale. + return gemm_a8w8_blockscale_bpreshuffle(A, B, As, Bs, dtype=output_dtype) + + +def _rocm_aiter_gemm_a8w8_blockscale_bpreshuffle_fake( + A: torch.Tensor, + B: torch.Tensor, + As: torch.Tensor, + Bs: torch.Tensor, + output_dtype: torch.dtype = torch.float16, +) -> torch.Tensor: + m = A.shape[0] + n = B.shape[0] + Y = torch.empty(m, n, dtype=output_dtype, device=A.device) + return Y + + def _rocm_aiter_rmsnorm_fused_add_dynamic_quant_impl( x: torch.Tensor, residual: torch.Tensor, @@ -1145,17 +1171,21 @@ def _rocm_aiter_fused_rms_gated_fp8_group_quant_fake( def _rocm_aiter_group_fp8_quant_impl( x: torch.Tensor, group_size: int, + transpose_scale: bool = False, ) -> tuple[torch.Tensor, torch.Tensor]: assert x.shape[-1] % group_size == 0, "Input shape must be divisible by group size" from aiter import QuantType, get_hip_quant aiter_per1x128_quant = get_hip_quant(QuantType.per_1x128) - return aiter_per1x128_quant(x.contiguous(), quant_dtype=FP8_DTYPE) + return aiter_per1x128_quant( + x.contiguous(), quant_dtype=FP8_DTYPE, transpose_scale=transpose_scale + ) def _rocm_aiter_group_fp8_quant_fake( x: torch.Tensor, group_size: int, + transpose_scale: bool = False, ) -> tuple[torch.Tensor, torch.Tensor]: M, N = x.shape x_fp8 = torch.empty((M, N), dtype=FP8_DTYPE, device=x.device) @@ -1955,6 +1985,12 @@ class rocm_aiter_ops: fake_impl=_rocm_aiter_gemm_a8w8_blockscale_fake, ) + direct_register_custom_op( + op_name="rocm_aiter_gemm_a8w8_blockscale_bpreshuffle", + op_func=_rocm_aiter_gemm_a8w8_blockscale_bpreshuffle_impl, + fake_impl=_rocm_aiter_gemm_a8w8_blockscale_bpreshuffle_fake, + ) + direct_register_custom_op( op_name="rocm_aiter_rmsnorm_fused_dynamic_quant", op_func=_rocm_aiter_rmsnorm_fused_dynamic_quant_impl, @@ -2209,6 +2245,18 @@ class rocm_aiter_ops: A, B, As, Bs, output_dtype ) + @staticmethod + def gemm_a8w8_blockscale_bpreshuffle( + A: torch.Tensor, + B: torch.Tensor, + As: torch.Tensor, + Bs: torch.Tensor, + output_dtype: torch.dtype = torch.float16, + ) -> torch.Tensor: + return torch.ops.vllm.rocm_aiter_gemm_a8w8_blockscale_bpreshuffle( + A, B, As, Bs, output_dtype + ) + @staticmethod def fused_moe( hidden_states: torch.Tensor, @@ -2702,9 +2750,12 @@ class rocm_aiter_ops: def group_fp8_quant( input_2d: torch.Tensor, group_size: int = 128, + transpose_scale: bool = False, ) -> tuple[torch.Tensor, torch.Tensor]: assert group_size == 128, "Group size must be 128" - return torch.ops.vllm.rocm_aiter_group_fp8_quant(input_2d, group_size) + return torch.ops.vllm.rocm_aiter_group_fp8_quant( + input_2d, group_size, transpose_scale + ) @staticmethod def is_triton_gemm_w8a8_tuned(n: int, k: int) -> bool: diff --git a/vllm/model_executor/model_loader/utils.py b/vllm/model_executor/model_loader/utils.py index 7d97a3299d..ecdcc16e9f 100644 --- a/vllm/model_executor/model_loader/utils.py +++ b/vllm/model_executor/model_loader/utils.py @@ -142,6 +142,10 @@ def process_weights_after_loading( if isinstance(module, HpcModule): module.process_weights_after_loading(model) + # Model-level post-load hook, after the per-layer quant finalize. + if hasattr(model, "process_weights_after_loading"): + model.process_weights_after_loading() + # Needed for torchao model reloading via model.reload_weights # @kylesayrs @jerryzh168 this can be removed if callers move to `reload_weights` if model_config.quantization == "torchao": diff --git a/vllm/models/deepseek_v4/amd/model.py b/vllm/models/deepseek_v4/amd/model.py index cdd338d636..ae9d38eb0a 100644 --- a/vllm/models/deepseek_v4/amd/model.py +++ b/vllm/models/deepseek_v4/amd/model.py @@ -9,6 +9,7 @@ import torch import torch.nn as nn import vllm.envs as envs +from vllm._aiter_ops import rocm_aiter_ops from vllm.config import VllmConfig, get_current_vllm_config from vllm.distributed import ( get_pp_group, @@ -104,8 +105,50 @@ class DeepseekV4MLP(nn.Module): else: self.act_fn = SiluAndMul() + # gate_up_proj B-preshuffle (ColumnParallel -> no all-reduce); set at load. + self._gateup = rocm_aiter_ops.is_enabled() + # Block scale for the preshuffled gate_up weight; None = not preshuffled. + self._gateup_scale: torch.Tensor | None = None + + def prepare_gateup_preshuffle(self) -> None: + # B-preshuffle the gate_up_proj weight in place (single weight). + if not self._gateup: + return + from vllm.model_executor.utils import replace_parameter + + w = getattr(self.gate_up_proj, "weight", None) + ws = getattr(self.gate_up_proj, "weight_scale_inv", None) # per-block scale + if w is None or ws is None or w.dim() != 2: + return + # K % 128 (group-128 quant) and N % 16 (shuffle_weight) must hold. + if w.shape[-1] % 128 != 0 or w.shape[0] % 16 != 0: + return + if ws.dtype == torch.float8_e8m0fnu: + from vllm.model_executor.layers.quantization.utils.fp8_utils import ( + _upcast_e8m0_to_fp32, + ) + + ws = _upcast_e8m0_to_fp32(ws).contiguous() + replace_parameter( + self.gate_up_proj, + "weight", + rocm_aiter_ops.shuffle_weight(w.data, layout=(16, 16)), + ) + self._gateup_scale = ws + def forward(self, x): - gate_up, _ = self.gate_up_proj(x) + if self._gateup_scale is not None and x.dim() == 2: + # gate_up via fp8 group-quant (col-major) + B-preshuffle GEMM. + x_fp8, x_scale = rocm_aiter_ops.group_fp8_quant(x, transpose_scale=True) + gate_up = rocm_aiter_ops.gemm_a8w8_blockscale_bpreshuffle( + x_fp8, + self.gate_up_proj.weight, + x_scale, + self._gateup_scale, + output_dtype=x.dtype, + ) + else: + gate_up, _ = self.gate_up_proj(x) x = self.act_fn(gate_up) x, _ = self.down_proj(x) return x @@ -961,8 +1004,15 @@ class DeepseekV4ForCausalLM(nn.Module, SupportsPP, SupportsEagle3): def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: loader = AutoWeightsLoader(self, skip_substrs=["mtp."]) - loaded_params = loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) - return loaded_params + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) + + def process_weights_after_loading(self) -> None: + # After per-layer quant finalize, so we preshuffle the final fp8 weights. + for module in self.modules(): + if isinstance(module, DeepseekV4ROCMAiterMLAAttention): + module.prepare_attn_preshuffle() + elif isinstance(module, DeepseekV4MLP): + module.prepare_gateup_preshuffle() def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: return self.model.get_expert_mapping() diff --git a/vllm/models/deepseek_v4/amd/rocm.py b/vllm/models/deepseek_v4/amd/rocm.py index d4aef6ec9e..7c232fe9e5 100644 --- a/vllm/models/deepseek_v4/amd/rocm.py +++ b/vllm/models/deepseek_v4/amd/rocm.py @@ -6,6 +6,10 @@ from typing import cast import torch +from vllm.distributed import ( + get_tensor_model_parallel_world_size, + tensor_model_parallel_all_reduce, +) from vllm.forward_context import get_forward_context from vllm.models.deepseek_v4.attention import DeepseekV4Attention from vllm.models.deepseek_v4.common.ops import dequantize_and_gather_k_cache @@ -442,10 +446,73 @@ class DeepseekV4ROCMAiterMLAAttention(DeepseekV4Attention): backend_cls = DeepseekV4ROCMAiterMLASparseBackend + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + # Block scale for the preshuffled weight; None = not preshuffled. + self._wqa_wkv_scale: torch.Tensor | None = None + self._wo_b_scale: torch.Tensor | None = None + @classmethod def get_padded_num_q_heads(cls, num_heads: int) -> int: return num_heads + def prepare_attn_preshuffle(self) -> None: + from vllm._aiter_ops import rocm_aiter_ops + + if not rocm_aiter_ops.is_enabled(): + return + from vllm.model_executor.layers.quantization.utils.fp8_utils import ( + _upcast_e8m0_to_fp32, + ) + from vllm.model_executor.utils import replace_parameter + + def _prep(linear) -> torch.Tensor | None: + w = getattr(linear, "weight", None) + if w is None or w.dim() != 2: + return None + # K % 128 (group-128 quant) and N % 16 (shuffle_weight) must hold. + if w.shape[-1] % 128 != 0 or w.shape[0] % 16 != 0: + return None + ws = getattr(linear, "weight_scale_inv", None) # per-block scale + if ws is None: + return None + if ws.dtype == torch.float8_e8m0fnu: + ws = _upcast_e8m0_to_fp32(ws).contiguous() + # Shuffle the weight in place (single weight, no unshuffled copy). + replace_parameter( + linear, + "weight", + rocm_aiter_ops.shuffle_weight(w.data, layout=(16, 16)), + ) + return ws + + self._wqa_wkv_scale = _prep(self.fused_wqa_wkv) + self._wo_b_scale = _prep(self.wo_b) + + def _bpre_attn_gemm( + self, + weight: torch.Tensor, + scale: torch.Tensor, + x: torch.Tensor, + reduce_tp: bool, + ) -> torch.Tensor: + from vllm._aiter_ops import rocm_aiter_ops + + x_fp8, x_scale = rocm_aiter_ops.group_fp8_quant(x, transpose_scale=True) + out = rocm_aiter_ops.gemm_a8w8_blockscale_bpreshuffle( + x_fp8, weight, x_scale, scale, output_dtype=x.dtype + ) + if reduce_tp and get_tensor_model_parallel_world_size() > 1: + out = tensor_model_parallel_all_reduce(out) + return out + + def _fused_wqa_wkv_gemm(self, hidden_states: torch.Tensor) -> torch.Tensor: + if self._wqa_wkv_scale is not None and hidden_states.dim() == 2: + return self._bpre_attn_gemm( + self.fused_wqa_wkv.weight, self._wqa_wkv_scale, hidden_states, False + ) + return super()._fused_wqa_wkv_gemm(hidden_states) + def _o_proj(self, o: torch.Tensor, positions: torch.Tensor) -> torch.Tensor: # ROCm BF16 reference wo_a path (inverse RoPE + einsum) + wo_b. z = rocm_inv_rope_einsum( @@ -457,7 +524,10 @@ class DeepseekV4ROCMAiterMLAAttention(DeepseekV4Attention): self.o_lora_rank, self.wo_a, ) - return self.wo_b(z.flatten(1)) + zf = z.flatten(1) + if self._wo_b_scale is not None and zf.dim() == 2: + return self._bpre_attn_gemm(self.wo_b.weight, self._wo_b_scale, zf, True) + return self.wo_b(zf) def forward_mqa( self, diff --git a/vllm/models/deepseek_v4/attention.py b/vllm/models/deepseek_v4/attention.py index 5d89337ba7..93162f5331 100644 --- a/vllm/models/deepseek_v4/attention.py +++ b/vllm/models/deepseek_v4/attention.py @@ -390,6 +390,11 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC): # Inverse-RoPE + wo_a + wo_b output projection (platform-specific). return self._o_proj(o, positions) + def _fused_wqa_wkv_gemm(self, hidden_states: torch.Tensor) -> torch.Tensor: + # MergedColumnParallelLinear returns (output, bias); bias is None. + qr_kv, _ = self.fused_wqa_wkv(hidden_states) + return qr_kv + def attn_gemm_parallel_execute(self, hidden_states) -> tuple[Any, ...]: aux_streams = self.aux_stream_list if aux_streams is not None: @@ -434,9 +439,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC): aux_fns[2] = indexer_compressor_kv_score def fused_wqa_wkv() -> torch.Tensor: - # MergedColumnParallelLinear returns (output, bias); bias is None. - qr_kv, _ = self.fused_wqa_wkv(hidden_states) - return qr_kv + return self._fused_wqa_wkv_gemm(hidden_states) qr_kv, (kv_score, indexer_weights, indexer_kv_score) = execute_in_parallel( fused_wqa_wkv,