[Model][DeepSeek-V4.1] Permute mega-attention wq_b / wo_a while loading (#60064)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
533d81b4b0
commit
9a1f6fd4cb
@@ -12,6 +12,8 @@ per-step shuffle.
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.model_loader.weight_utils import composed_weight_loader
|
||||
|
||||
HEAD_DIM = 512
|
||||
Q_CHUNK = 16
|
||||
O_CHUNK = 32
|
||||
@@ -60,27 +62,13 @@ def _bytes_view(t: torch.Tensor) -> torch.Tensor:
|
||||
return t.view(torch.uint8) if t.element_size() == 1 else t
|
||||
|
||||
|
||||
def permute_wq_b_(
|
||||
weight: torch.Tensor, weight_scale: torch.Tensor, num_local_heads: int
|
||||
) -> None:
|
||||
"""Permute an MXFP8 ``wq_b`` shard's rows and per-row scales, in place."""
|
||||
head_dim = weight.shape[0] // num_local_heads
|
||||
perm = q_fused_permutation(num_local_heads, head_dim).to(weight.device)
|
||||
for t in (weight, weight_scale):
|
||||
b = _bytes_view(t)
|
||||
b.copy_(b[perm])
|
||||
def permute_on_load(param: torch.Tensor, perm: torch.Tensor, dim: int) -> None:
|
||||
"""Make ``param``'s weight loader gather the loaded shard by ``perm``."""
|
||||
|
||||
def gather(t: torch.Tensor) -> torch.Tensor:
|
||||
return _bytes_view(t).index_select(dim, perm.to(t.device)).view(t.dtype)
|
||||
|
||||
def permute_wo_a_(
|
||||
weight: torch.Tensor,
|
||||
weight_scale: torch.Tensor,
|
||||
heads_per_group: int = WV_GROUP_SIZE,
|
||||
) -> None:
|
||||
"""Permute an MXFP8 ``wo_a`` shard's input columns and scales, in place."""
|
||||
head_dim = weight.shape[1] // heads_per_group
|
||||
perm = o_fused_permutation(heads_per_group, head_dim).to(weight.device)
|
||||
w = _bytes_view(weight)
|
||||
w.copy_(w[:, perm])
|
||||
chunk_perm = o_fused_chunk_permutation(heads_per_group, head_dim)
|
||||
s = _bytes_view(weight_scale)
|
||||
s.copy_(s[:, chunk_perm.to(weight.device)])
|
||||
param.weight_loader = composed_weight_loader( # type: ignore[attr-defined]
|
||||
param.weight_loader, # type: ignore[attr-defined]
|
||||
gather,
|
||||
)
|
||||
|
||||
@@ -500,22 +500,8 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
|
||||
for layer in self.model.layers:
|
||||
layer.ffn.finalize_mega_moe_weights()
|
||||
|
||||
def _finalize_attn(self) -> None:
|
||||
"""Run the attention backend's post-load weight step on the draft layers.
|
||||
|
||||
They are ordinary DeepseekV4DecoderLayers, so they take the engine-wide
|
||||
attention backend and owe it whatever the target model owes it -- mega
|
||||
attention permutes wq_b / wo_a here and refuses to run without it.
|
||||
Idempotent, like the target's.
|
||||
"""
|
||||
for layer in self.model.layers:
|
||||
finalize = getattr(layer.attn, "finalize_loaded_weights", None)
|
||||
if finalize is not None:
|
||||
finalize()
|
||||
|
||||
def process_weights_after_loading(self) -> None:
|
||||
self._finalize_moe()
|
||||
self._finalize_attn()
|
||||
|
||||
def _remap_dspark_name(self, name: str) -> str | None:
|
||||
"""Map a checkpoint ``mtp.{i}.*`` name to this model's parameter path.
|
||||
|
||||
@@ -32,8 +32,10 @@ from vllm.models.deepseek_v41.common.ops import (
|
||||
)
|
||||
from vllm.models.deepseek_v41.common.ops.fused_layout import (
|
||||
WV_GROUP_SIZE,
|
||||
permute_wo_a_,
|
||||
permute_wq_b_,
|
||||
o_fused_chunk_permutation,
|
||||
o_fused_permutation,
|
||||
permute_on_load,
|
||||
q_fused_permutation,
|
||||
)
|
||||
from vllm.models.deepseek_v41.nvidia.flashmla import DeepseekV4FlashMLAAttention
|
||||
from vllm.models.deepseek_v41.sparse_mla import (
|
||||
@@ -182,7 +184,13 @@ class DeepseekV4MegaAttnAttention(DeepseekV4FlashMLAAttention):
|
||||
"wo_a group."
|
||||
)
|
||||
self.n_wv_group = self.padded_heads // WV_GROUP_SIZE
|
||||
self._fused_layouts_ready = False
|
||||
q_perm = q_fused_permutation(self.n_local_heads, self.head_dim)
|
||||
permute_on_load(self.wq_b.weight, q_perm, dim=0)
|
||||
permute_on_load(self.wq_b.weight_scale, q_perm, dim=0)
|
||||
o_perm = o_fused_permutation(WV_GROUP_SIZE, self.head_dim)
|
||||
permute_on_load(self.wo_a.weight, o_perm, dim=1)
|
||||
o_chunk_perm = o_fused_chunk_permutation(WV_GROUP_SIZE, self.head_dim)
|
||||
permute_on_load(self.wo_a.weight_scale, o_chunk_perm, dim=1)
|
||||
|
||||
# ---- interface contract ------------------------------------------------
|
||||
|
||||
@@ -227,25 +235,6 @@ class DeepseekV4MegaAttnAttention(DeepseekV4FlashMLAAttention):
|
||||
)
|
||||
return self._wo_b_proj(z.flatten(1))
|
||||
|
||||
# ---- weights -----------------------------------------------------------
|
||||
|
||||
def finalize_loaded_weights(self) -> None:
|
||||
"""Permute wq_b rows / wo_a columns into the kernel's layouts.
|
||||
|
||||
Idempotent: a second post-load pass must not permute twice.
|
||||
"""
|
||||
if self._fused_layouts_ready:
|
||||
return
|
||||
permute_wq_b_(
|
||||
self.wq_b.weight.data, self.wq_b.weight_scale.data, self.n_local_heads
|
||||
)
|
||||
permute_wo_a_(
|
||||
self.wo_a.weight.data,
|
||||
self.wo_a.weight_scale.data,
|
||||
self.n_local_heads // self.n_local_groups,
|
||||
)
|
||||
self._fused_layouts_ready = True
|
||||
|
||||
# ---- forward -----------------------------------------------------------
|
||||
|
||||
def forward_mqa(
|
||||
@@ -255,11 +244,6 @@ class DeepseekV4MegaAttnAttention(DeepseekV4FlashMLAAttention):
|
||||
positions: torch.Tensor,
|
||||
output: QuantizedActivation,
|
||||
) -> None:
|
||||
if not self._fused_layouts_ready:
|
||||
raise RuntimeError(
|
||||
f"{self.prefix}: wq_b / wo_a were never permuted for the mega "
|
||||
"attention kernel; refusing to run with mismatched layouts."
|
||||
)
|
||||
attn_metadata = get_forward_context().attn_metadata
|
||||
if attn_metadata is None:
|
||||
# Warmup dummy run: reserve the prefill workspace, produce zeros.
|
||||
|
||||
@@ -1364,17 +1364,6 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
for layer in islice(self.layers, self.start_layer, self.end_layer):
|
||||
layer.ffn.finalize_mega_moe_weights()
|
||||
|
||||
def finalize_mega_attn_weights(self) -> None:
|
||||
"""Permute wq_b / wo_a into FlashMLA's mega-attention layouts.
|
||||
|
||||
A no-op for every other attention layer, and idempotent, so a second
|
||||
post-load pass cannot permute twice.
|
||||
"""
|
||||
for layer in islice(self.layers, self.start_layer, self.end_layer):
|
||||
finalize = getattr(layer.attn, "finalize_loaded_weights", None)
|
||||
if finalize is not None:
|
||||
finalize()
|
||||
|
||||
def finalize_mhc_broadcast_weights(self) -> None:
|
||||
if not get_pp_group().is_first_rank or self.start_layer >= self.end_layer:
|
||||
return
|
||||
@@ -1647,7 +1636,6 @@ class DeepseekV41LLMForCausalLM(
|
||||
def process_weights_after_loading(self) -> None:
|
||||
self.model.finalize_mega_moe_weights()
|
||||
self.model.finalize_mhc_broadcast_weights()
|
||||
self.model.finalize_mega_attn_weights()
|
||||
|
||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||
return self.model.get_expert_mapping()
|
||||
|
||||
Reference in New Issue
Block a user