[ROCm][DSV4] B-preshuffle the attention fp8 projections (#46720)
Signed-off-by: Mehmet Cagri Kaymak <mehmet.kaymak@amd.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
Andreas Karatzas
parent
5f8f72866c
commit
12a34a6bc7
+53
-2
@@ -687,6 +687,32 @@ def _rocm_aiter_gemm_a8w8_blockscale_fake(
|
||||
return Y
|
||||
|
||||
|
||||
def _rocm_aiter_gemm_a8w8_blockscale_bpreshuffle_impl(
|
||||
A: torch.Tensor,
|
||||
B: torch.Tensor,
|
||||
As: torch.Tensor,
|
||||
Bs: torch.Tensor,
|
||||
output_dtype: torch.dtype = torch.float16,
|
||||
) -> torch.Tensor:
|
||||
from aiter import gemm_a8w8_blockscale_bpreshuffle
|
||||
|
||||
# B preshuffled (shuffle_weight (16,16)); As column-major group scale.
|
||||
return gemm_a8w8_blockscale_bpreshuffle(A, B, As, Bs, dtype=output_dtype)
|
||||
|
||||
|
||||
def _rocm_aiter_gemm_a8w8_blockscale_bpreshuffle_fake(
|
||||
A: torch.Tensor,
|
||||
B: torch.Tensor,
|
||||
As: torch.Tensor,
|
||||
Bs: torch.Tensor,
|
||||
output_dtype: torch.dtype = torch.float16,
|
||||
) -> torch.Tensor:
|
||||
m = A.shape[0]
|
||||
n = B.shape[0]
|
||||
Y = torch.empty(m, n, dtype=output_dtype, device=A.device)
|
||||
return Y
|
||||
|
||||
|
||||
def _rocm_aiter_rmsnorm_fused_add_dynamic_quant_impl(
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
@@ -1145,17 +1171,21 @@ def _rocm_aiter_fused_rms_gated_fp8_group_quant_fake(
|
||||
def _rocm_aiter_group_fp8_quant_impl(
|
||||
x: torch.Tensor,
|
||||
group_size: int,
|
||||
transpose_scale: bool = False,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
assert x.shape[-1] % group_size == 0, "Input shape must be divisible by group size"
|
||||
from aiter import QuantType, get_hip_quant
|
||||
|
||||
aiter_per1x128_quant = get_hip_quant(QuantType.per_1x128)
|
||||
return aiter_per1x128_quant(x.contiguous(), quant_dtype=FP8_DTYPE)
|
||||
return aiter_per1x128_quant(
|
||||
x.contiguous(), quant_dtype=FP8_DTYPE, transpose_scale=transpose_scale
|
||||
)
|
||||
|
||||
|
||||
def _rocm_aiter_group_fp8_quant_fake(
|
||||
x: torch.Tensor,
|
||||
group_size: int,
|
||||
transpose_scale: bool = False,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
M, N = x.shape
|
||||
x_fp8 = torch.empty((M, N), dtype=FP8_DTYPE, device=x.device)
|
||||
@@ -1955,6 +1985,12 @@ class rocm_aiter_ops:
|
||||
fake_impl=_rocm_aiter_gemm_a8w8_blockscale_fake,
|
||||
)
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="rocm_aiter_gemm_a8w8_blockscale_bpreshuffle",
|
||||
op_func=_rocm_aiter_gemm_a8w8_blockscale_bpreshuffle_impl,
|
||||
fake_impl=_rocm_aiter_gemm_a8w8_blockscale_bpreshuffle_fake,
|
||||
)
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="rocm_aiter_rmsnorm_fused_dynamic_quant",
|
||||
op_func=_rocm_aiter_rmsnorm_fused_dynamic_quant_impl,
|
||||
@@ -2209,6 +2245,18 @@ class rocm_aiter_ops:
|
||||
A, B, As, Bs, output_dtype
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def gemm_a8w8_blockscale_bpreshuffle(
|
||||
A: torch.Tensor,
|
||||
B: torch.Tensor,
|
||||
As: torch.Tensor,
|
||||
Bs: torch.Tensor,
|
||||
output_dtype: torch.dtype = torch.float16,
|
||||
) -> torch.Tensor:
|
||||
return torch.ops.vllm.rocm_aiter_gemm_a8w8_blockscale_bpreshuffle(
|
||||
A, B, As, Bs, output_dtype
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def fused_moe(
|
||||
hidden_states: torch.Tensor,
|
||||
@@ -2702,9 +2750,12 @@ class rocm_aiter_ops:
|
||||
def group_fp8_quant(
|
||||
input_2d: torch.Tensor,
|
||||
group_size: int = 128,
|
||||
transpose_scale: bool = False,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
assert group_size == 128, "Group size must be 128"
|
||||
return torch.ops.vllm.rocm_aiter_group_fp8_quant(input_2d, group_size)
|
||||
return torch.ops.vllm.rocm_aiter_group_fp8_quant(
|
||||
input_2d, group_size, transpose_scale
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def is_triton_gemm_w8a8_tuned(n: int, k: int) -> bool:
|
||||
|
||||
@@ -142,6 +142,10 @@ def process_weights_after_loading(
|
||||
if isinstance(module, HpcModule):
|
||||
module.process_weights_after_loading(model)
|
||||
|
||||
# Model-level post-load hook, after the per-layer quant finalize.
|
||||
if hasattr(model, "process_weights_after_loading"):
|
||||
model.process_weights_after_loading()
|
||||
|
||||
# Needed for torchao model reloading via model.reload_weights
|
||||
# @kylesayrs @jerryzh168 this can be removed if callers move to `reload_weights`
|
||||
if model_config.quantization == "torchao":
|
||||
|
||||
@@ -9,6 +9,7 @@ import torch
|
||||
import torch.nn as nn
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.config import VllmConfig, get_current_vllm_config
|
||||
from vllm.distributed import (
|
||||
get_pp_group,
|
||||
@@ -104,8 +105,50 @@ class DeepseekV4MLP(nn.Module):
|
||||
else:
|
||||
self.act_fn = SiluAndMul()
|
||||
|
||||
# gate_up_proj B-preshuffle (ColumnParallel -> no all-reduce); set at load.
|
||||
self._gateup = rocm_aiter_ops.is_enabled()
|
||||
# Block scale for the preshuffled gate_up weight; None = not preshuffled.
|
||||
self._gateup_scale: torch.Tensor | None = None
|
||||
|
||||
def prepare_gateup_preshuffle(self) -> None:
|
||||
# B-preshuffle the gate_up_proj weight in place (single weight).
|
||||
if not self._gateup:
|
||||
return
|
||||
from vllm.model_executor.utils import replace_parameter
|
||||
|
||||
w = getattr(self.gate_up_proj, "weight", None)
|
||||
ws = getattr(self.gate_up_proj, "weight_scale_inv", None) # per-block scale
|
||||
if w is None or ws is None or w.dim() != 2:
|
||||
return
|
||||
# K % 128 (group-128 quant) and N % 16 (shuffle_weight) must hold.
|
||||
if w.shape[-1] % 128 != 0 or w.shape[0] % 16 != 0:
|
||||
return
|
||||
if ws.dtype == torch.float8_e8m0fnu:
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
_upcast_e8m0_to_fp32,
|
||||
)
|
||||
|
||||
ws = _upcast_e8m0_to_fp32(ws).contiguous()
|
||||
replace_parameter(
|
||||
self.gate_up_proj,
|
||||
"weight",
|
||||
rocm_aiter_ops.shuffle_weight(w.data, layout=(16, 16)),
|
||||
)
|
||||
self._gateup_scale = ws
|
||||
|
||||
def forward(self, x):
|
||||
gate_up, _ = self.gate_up_proj(x)
|
||||
if self._gateup_scale is not None and x.dim() == 2:
|
||||
# gate_up via fp8 group-quant (col-major) + B-preshuffle GEMM.
|
||||
x_fp8, x_scale = rocm_aiter_ops.group_fp8_quant(x, transpose_scale=True)
|
||||
gate_up = rocm_aiter_ops.gemm_a8w8_blockscale_bpreshuffle(
|
||||
x_fp8,
|
||||
self.gate_up_proj.weight,
|
||||
x_scale,
|
||||
self._gateup_scale,
|
||||
output_dtype=x.dtype,
|
||||
)
|
||||
else:
|
||||
gate_up, _ = self.gate_up_proj(x)
|
||||
x = self.act_fn(gate_up)
|
||||
x, _ = self.down_proj(x)
|
||||
return x
|
||||
@@ -961,8 +1004,15 @@ class DeepseekV4ForCausalLM(nn.Module, SupportsPP, SupportsEagle3):
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(self, skip_substrs=["mtp."])
|
||||
loaded_params = loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
return loaded_params
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
def process_weights_after_loading(self) -> None:
|
||||
# After per-layer quant finalize, so we preshuffle the final fp8 weights.
|
||||
for module in self.modules():
|
||||
if isinstance(module, DeepseekV4ROCMAiterMLAAttention):
|
||||
module.prepare_attn_preshuffle()
|
||||
elif isinstance(module, DeepseekV4MLP):
|
||||
module.prepare_gateup_preshuffle()
|
||||
|
||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||
return self.model.get_expert_mapping()
|
||||
|
||||
@@ -6,6 +6,10 @@ from typing import cast
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.distributed import (
|
||||
get_tensor_model_parallel_world_size,
|
||||
tensor_model_parallel_all_reduce,
|
||||
)
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.models.deepseek_v4.attention import DeepseekV4Attention
|
||||
from vllm.models.deepseek_v4.common.ops import dequantize_and_gather_k_cache
|
||||
@@ -442,10 +446,73 @@ class DeepseekV4ROCMAiterMLAAttention(DeepseekV4Attention):
|
||||
|
||||
backend_cls = DeepseekV4ROCMAiterMLASparseBackend
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
# Block scale for the preshuffled weight; None = not preshuffled.
|
||||
self._wqa_wkv_scale: torch.Tensor | None = None
|
||||
self._wo_b_scale: torch.Tensor | None = None
|
||||
|
||||
@classmethod
|
||||
def get_padded_num_q_heads(cls, num_heads: int) -> int:
|
||||
return num_heads
|
||||
|
||||
def prepare_attn_preshuffle(self) -> None:
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
if not rocm_aiter_ops.is_enabled():
|
||||
return
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
_upcast_e8m0_to_fp32,
|
||||
)
|
||||
from vllm.model_executor.utils import replace_parameter
|
||||
|
||||
def _prep(linear) -> torch.Tensor | None:
|
||||
w = getattr(linear, "weight", None)
|
||||
if w is None or w.dim() != 2:
|
||||
return None
|
||||
# K % 128 (group-128 quant) and N % 16 (shuffle_weight) must hold.
|
||||
if w.shape[-1] % 128 != 0 or w.shape[0] % 16 != 0:
|
||||
return None
|
||||
ws = getattr(linear, "weight_scale_inv", None) # per-block scale
|
||||
if ws is None:
|
||||
return None
|
||||
if ws.dtype == torch.float8_e8m0fnu:
|
||||
ws = _upcast_e8m0_to_fp32(ws).contiguous()
|
||||
# Shuffle the weight in place (single weight, no unshuffled copy).
|
||||
replace_parameter(
|
||||
linear,
|
||||
"weight",
|
||||
rocm_aiter_ops.shuffle_weight(w.data, layout=(16, 16)),
|
||||
)
|
||||
return ws
|
||||
|
||||
self._wqa_wkv_scale = _prep(self.fused_wqa_wkv)
|
||||
self._wo_b_scale = _prep(self.wo_b)
|
||||
|
||||
def _bpre_attn_gemm(
|
||||
self,
|
||||
weight: torch.Tensor,
|
||||
scale: torch.Tensor,
|
||||
x: torch.Tensor,
|
||||
reduce_tp: bool,
|
||||
) -> torch.Tensor:
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
x_fp8, x_scale = rocm_aiter_ops.group_fp8_quant(x, transpose_scale=True)
|
||||
out = rocm_aiter_ops.gemm_a8w8_blockscale_bpreshuffle(
|
||||
x_fp8, weight, x_scale, scale, output_dtype=x.dtype
|
||||
)
|
||||
if reduce_tp and get_tensor_model_parallel_world_size() > 1:
|
||||
out = tensor_model_parallel_all_reduce(out)
|
||||
return out
|
||||
|
||||
def _fused_wqa_wkv_gemm(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
if self._wqa_wkv_scale is not None and hidden_states.dim() == 2:
|
||||
return self._bpre_attn_gemm(
|
||||
self.fused_wqa_wkv.weight, self._wqa_wkv_scale, hidden_states, False
|
||||
)
|
||||
return super()._fused_wqa_wkv_gemm(hidden_states)
|
||||
|
||||
def _o_proj(self, o: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
|
||||
# ROCm BF16 reference wo_a path (inverse RoPE + einsum) + wo_b.
|
||||
z = rocm_inv_rope_einsum(
|
||||
@@ -457,7 +524,10 @@ class DeepseekV4ROCMAiterMLAAttention(DeepseekV4Attention):
|
||||
self.o_lora_rank,
|
||||
self.wo_a,
|
||||
)
|
||||
return self.wo_b(z.flatten(1))
|
||||
zf = z.flatten(1)
|
||||
if self._wo_b_scale is not None and zf.dim() == 2:
|
||||
return self._bpre_attn_gemm(self.wo_b.weight, self._wo_b_scale, zf, True)
|
||||
return self.wo_b(zf)
|
||||
|
||||
def forward_mqa(
|
||||
self,
|
||||
|
||||
@@ -390,6 +390,11 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
# Inverse-RoPE + wo_a + wo_b output projection (platform-specific).
|
||||
return self._o_proj(o, positions)
|
||||
|
||||
def _fused_wqa_wkv_gemm(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
# MergedColumnParallelLinear returns (output, bias); bias is None.
|
||||
qr_kv, _ = self.fused_wqa_wkv(hidden_states)
|
||||
return qr_kv
|
||||
|
||||
def attn_gemm_parallel_execute(self, hidden_states) -> tuple[Any, ...]:
|
||||
aux_streams = self.aux_stream_list
|
||||
if aux_streams is not None:
|
||||
@@ -434,9 +439,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
aux_fns[2] = indexer_compressor_kv_score
|
||||
|
||||
def fused_wqa_wkv() -> torch.Tensor:
|
||||
# MergedColumnParallelLinear returns (output, bias); bias is None.
|
||||
qr_kv, _ = self.fused_wqa_wkv(hidden_states)
|
||||
return qr_kv
|
||||
return self._fused_wqa_wkv_gemm(hidden_states)
|
||||
|
||||
qr_kv, (kv_score, indexer_weights, indexer_kv_score) = execute_in_parallel(
|
||||
fused_wqa_wkv,
|
||||
|
||||
Reference in New Issue
Block a user