Signed-off-by: Shanshan Shen <87969357+shen-shanshan@users.noreply.github.com>
This commit is contained in:
@@ -71,10 +71,7 @@ from vllm.model_executor.models.utils import (
|
||||
make_layers,
|
||||
maybe_prefix,
|
||||
)
|
||||
from vllm.models.deepseek_v4.amd.rocm import (
|
||||
DeepseekV4ROCMAiterMLAAttention,
|
||||
weight_already_preshuffled,
|
||||
)
|
||||
from vllm.models.deepseek_v4.amd.rocm import DeepseekV4ROCMAiterMLAAttention
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.platforms.rocm import on_gfx950
|
||||
from vllm.sequence import IntermediateTensors
|
||||
@@ -152,13 +149,11 @@ class DeepseekV4MLP(nn.Module):
|
||||
return
|
||||
if ws.dtype == torch.float8_e8m0fnu:
|
||||
ws = _upcast_e8m0_to_fp32(ws).contiguous()
|
||||
# Skip if the linear's kernel already shuffled it.
|
||||
if not weight_already_preshuffled(self.gate_up_proj):
|
||||
replace_parameter(
|
||||
self.gate_up_proj,
|
||||
"weight",
|
||||
rocm_aiter_ops.shuffle_weight(w.data, layout=(16, 16)),
|
||||
)
|
||||
replace_parameter(
|
||||
self.gate_up_proj,
|
||||
"weight",
|
||||
rocm_aiter_ops.shuffle_weight(w.data, layout=(16, 16)),
|
||||
)
|
||||
self._gateup_scale = ws
|
||||
|
||||
def forward(self, x):
|
||||
|
||||
@@ -62,21 +62,6 @@ def _build_indptr_from_lengths(lengths: torch.Tensor) -> torch.Tensor:
|
||||
return indptr
|
||||
|
||||
|
||||
def weight_already_preshuffled(linear: torch.nn.Module) -> bool:
|
||||
"""True when the linear's kernel already B-preshuffled ``weight``.
|
||||
|
||||
The hand-shuffles below (fused_wqa_wkv, wo_b, gate_up_proj) must be skipped
|
||||
for those, since shuffle_weight is a permutation rather than an involution.
|
||||
"""
|
||||
return any(
|
||||
getattr(getattr(method, "fp8_linear", None), "preshuffles_weight", False)
|
||||
for method in (
|
||||
getattr(linear, "quant_method", None),
|
||||
getattr(linear, "scheme", None),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def apply_pre_quantized_block_scaled_mm(
|
||||
linear: torch.nn.Module,
|
||||
x_fp8: torch.Tensor,
|
||||
@@ -792,13 +777,12 @@ class DeepseekV4ROCMAiterMLAAttention(DeepseekV4Attention):
|
||||
return None
|
||||
if ws.dtype == torch.float8_e8m0fnu:
|
||||
ws = _upcast_e8m0_to_fp32(ws).contiguous()
|
||||
# Skip if the linear's kernel already shuffled it.
|
||||
if not weight_already_preshuffled(linear):
|
||||
replace_parameter(
|
||||
linear,
|
||||
"weight",
|
||||
rocm_aiter_ops.shuffle_weight(w.data, layout=(16, 16)),
|
||||
)
|
||||
# Shuffle the weight in place (single weight, no unshuffled copy).
|
||||
replace_parameter(
|
||||
linear,
|
||||
"weight",
|
||||
rocm_aiter_ops.shuffle_weight(w.data, layout=(16, 16)),
|
||||
)
|
||||
return ws
|
||||
|
||||
self._wqa_wkv_scale = _prep(self.fused_wqa_wkv)
|
||||
@@ -896,26 +880,16 @@ class DeepseekV4ROCMAiterMLAAttention(DeepseekV4Attention):
|
||||
return qr_kv, kv_score, indexer_kv_score, indexer_weights
|
||||
|
||||
@functools.cached_property
|
||||
def _wq_b_act_scale_transpose(self) -> bool | None:
|
||||
"""Activation-scale byte order every wq_b consumer agrees on, else None.
|
||||
|
||||
None means "do not take the fused norm+quant path". Otherwise the value
|
||||
is the ``transpose_scale`` the producer must pass so that the scale it
|
||||
writes matches what the selected GEMM reads.
|
||||
def _wq_b_uses_aiter_block_scaled(self) -> bool:
|
||||
"""True when both wq_b GEMMs run the aiter block-scaled fp8 kernel.
|
||||
|
||||
Cached: the linear kernels and the aiter env gates are fixed once
|
||||
the model is built, so this is evaluated at the first forward
|
||||
only.
|
||||
|
||||
Two conditions, both necessary:
|
||||
|
||||
* The fused norm+quant path is only valid if the quant and GEMM it
|
||||
replaces are exactly the aiter ones; otherwise fall back to the
|
||||
shared path.
|
||||
* One qr_scale feeds both self.wq_b and self.indexer.wq_b, which have
|
||||
different (N, K) and so can resolve to kernels wanting opposite byte
|
||||
orders. A single producer cannot serve both, so refuse the fast path
|
||||
when they disagree rather than guessing.
|
||||
The fused norm+quant path is only valid if the quant and GEMM it
|
||||
replaces are exactly the aiter ones; otherwise fall back to the
|
||||
shared path.
|
||||
"""
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.model_executor.kernels.linear.scaled_mm import (
|
||||
@@ -923,31 +897,16 @@ class DeepseekV4ROCMAiterMLAAttention(DeepseekV4Attention):
|
||||
)
|
||||
|
||||
if not rocm_aiter_ops.is_linear_fp8_enabled():
|
||||
return None
|
||||
return False
|
||||
|
||||
linears = [self.wq_b]
|
||||
if self.indexer is not None:
|
||||
linears.append(self.indexer.wq_b)
|
||||
layouts: set[bool] = set()
|
||||
for linear in linears:
|
||||
kernel = getattr(getattr(linear, "quant_method", None), "fp8_linear", None)
|
||||
if not isinstance(kernel, Fp8BlockScaledMMLinearKernel):
|
||||
return None
|
||||
layouts.add(bool(getattr(kernel, "wants_transposed_act_scale", False)))
|
||||
if len(layouts) != 1:
|
||||
logger.warning_once(
|
||||
"DeepSeek-V4 wq_b consumers disagree on activation-scale layout; "
|
||||
"disabling the fused q/kv norm+quant path.",
|
||||
scope="global",
|
||||
)
|
||||
return None
|
||||
transpose_scale = layouts.pop()
|
||||
logger.debug_once(
|
||||
"DeepSeek-V4 wq_b: emitting %s-major activation scales",
|
||||
"column" if transpose_scale else "row",
|
||||
scope="global",
|
||||
)
|
||||
return transpose_scale
|
||||
return False
|
||||
return True
|
||||
|
||||
def _split_qkv_and_norm(
|
||||
self, qr_kv: torch.Tensor
|
||||
@@ -964,12 +923,11 @@ class DeepseekV4ROCMAiterMLAAttention(DeepseekV4Attention):
|
||||
when the aiter linear path is not active.
|
||||
"""
|
||||
qr, kv = qr_kv.split([self.q_lora_rank, self.head_dim], dim=-1)
|
||||
transpose_scale = self._wq_b_act_scale_transpose
|
||||
if not (
|
||||
qr.dim() == 2
|
||||
and qr.shape[0] > 0
|
||||
and self.q_lora_rank % 128 == 0
|
||||
and transpose_scale is not None
|
||||
and self._wq_b_uses_aiter_block_scaled
|
||||
):
|
||||
return super()._split_qkv_and_norm(qr_kv)
|
||||
|
||||
@@ -983,8 +941,7 @@ class DeepseekV4ROCMAiterMLAAttention(DeepseekV4Attention):
|
||||
kv_weight=self.kv_norm.weight.data,
|
||||
kv_epsilon=self.eps,
|
||||
group_size=128,
|
||||
# Emit the byte order the wq_b GEMMs read.
|
||||
transpose_scale=transpose_scale,
|
||||
transpose_scale=False,
|
||||
)
|
||||
|
||||
def _o_proj(self, o: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
|
||||
|
||||
Reference in New Issue
Block a user