[ROCm][DSv4] Fuse the shared expert into AITER MegaMoE (#61004)
Signed-off-by: Yanfei Wang <yanfei.wang@amd.com> Co-authored-by: aoshen02 <aoshen@inferact.ai>
This commit is contained in:
co-authored by
aoshen02
parent
36f3972812
commit
e3bbaf2369
@@ -10,10 +10,51 @@ import torch.nn as nn
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import get_ep_group
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
get_fp8_block_weight_scale,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import mxfp4_quantize
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
GroupShape,
|
||||
scaled_dequantize,
|
||||
)
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.math_utils import next_power_of_2
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _append_shared_expert_kernel(
|
||||
topk_ids_ptr,
|
||||
topk_weights_ptr,
|
||||
out_ids_ptr,
|
||||
out_weights_ptr,
|
||||
num_tokens,
|
||||
shared_expert_id,
|
||||
TOPK: tl.constexpr,
|
||||
ROUTED_PER_RANK: tl.constexpr,
|
||||
BLOCK_M: tl.constexpr,
|
||||
):
|
||||
rows = tl.program_id(0) * BLOCK_M + tl.arange(0, BLOCK_M)
|
||||
mask = rows < num_tokens
|
||||
for k in tl.static_range(TOPK):
|
||||
e = tl.load(topk_ids_ptr + rows * TOPK + k, mask=mask, other=0)
|
||||
w = tl.load(topk_weights_ptr + rows * TOPK + k, mask=mask, other=0.0)
|
||||
e = tl.where(e >= 0, e + e // ROUTED_PER_RANK, e)
|
||||
tl.store(out_ids_ptr + rows * (TOPK + 1) + k, e, mask=mask)
|
||||
tl.store(out_weights_ptr + rows * (TOPK + 1) + k, w, mask=mask)
|
||||
tl.store(
|
||||
out_ids_ptr + rows * (TOPK + 1) + TOPK,
|
||||
tl.zeros_like(rows) + shared_expert_id,
|
||||
mask=mask,
|
||||
)
|
||||
tl.store(
|
||||
out_weights_ptr + rows * (TOPK + 1) + TOPK,
|
||||
tl.full([BLOCK_M], 1.0, tl.float32),
|
||||
mask=mask,
|
||||
)
|
||||
|
||||
|
||||
class DeepseekV4AiterMegaMoEExperts(nn.Module):
|
||||
# MegaMoEV2 owns large symmetric-heap buffers, so every layer shares one
|
||||
# instance and passes its own weights per call.
|
||||
@@ -28,16 +69,25 @@ class DeepseekV4AiterMegaMoEExperts(nn.Module):
|
||||
hidden_size: int,
|
||||
intermediate_size: int,
|
||||
swiglu_limit: float | None,
|
||||
fuse_shared_expert: bool = False,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
ep_group = get_ep_group()
|
||||
self.ep_rank = ep_group.rank_in_group
|
||||
self.ep_size = ep_group.world_size
|
||||
self.num_experts = num_experts
|
||||
self.num_local_experts = num_experts // self.ep_size
|
||||
self.experts_start_idx = self.ep_rank * self.num_local_experts
|
||||
self.top_k = top_k
|
||||
self.num_routed_local_experts = num_experts // self.ep_size
|
||||
self.experts_start_idx = self.ep_rank * self.num_routed_local_experts
|
||||
# A fused shared expert is one extra local expert per rank and one
|
||||
# extra route per token, to the token's own rank.
|
||||
self.fuse_shared_expert = fuse_shared_expert
|
||||
self.num_local_experts = self.num_routed_local_experts + fuse_shared_expert
|
||||
self.num_experts = self.num_local_experts * self.ep_size
|
||||
self.routed_top_k = top_k
|
||||
self.top_k = top_k + fuse_shared_expert
|
||||
self.shared_expert_id = (
|
||||
self.ep_rank * self.num_local_experts + self.num_routed_local_experts
|
||||
)
|
||||
self.hidden_size = hidden_size
|
||||
self.intermediate_size = intermediate_size
|
||||
self.swiglu_limit = swiglu_limit or 0.0
|
||||
@@ -70,7 +120,7 @@ class DeepseekV4AiterMegaMoEExperts(nn.Module):
|
||||
return_success: bool = False,
|
||||
) -> bool | None:
|
||||
local_expert_id = expert_id - self.experts_start_idx
|
||||
if not 0 <= local_expert_id < self.num_local_experts:
|
||||
if not 0 <= local_expert_id < self.num_routed_local_experts:
|
||||
return False if return_success else None
|
||||
expert_data = param.data[local_expert_id]
|
||||
if shard_id in ("w1", "w3"):
|
||||
@@ -81,6 +131,24 @@ class DeepseekV4AiterMegaMoEExperts(nn.Module):
|
||||
expert_data.copy_(loaded_weight.view(torch.uint8))
|
||||
return True if return_success else None
|
||||
|
||||
def load_shared_expert(self, shared_experts: nn.Module) -> None:
|
||||
slot = self.num_routed_local_experts
|
||||
for linear, weight, weight_scale in (
|
||||
(shared_experts.gate_up_proj, self.w13_weight, self.w13_weight_scale),
|
||||
(shared_experts.down_proj, self.w2_weight, self.w2_weight_scale),
|
||||
):
|
||||
block_scale = get_fp8_block_weight_scale(linear)
|
||||
assert block_scale is not None, "shared expert must be block-FP8"
|
||||
group = GroupShape(
|
||||
linear.weight.shape[0] // block_scale.shape[0],
|
||||
linear.weight.shape[1] // block_scale.shape[1],
|
||||
)
|
||||
packed, scale = mxfp4_quantize(
|
||||
scaled_dequantize(linear.weight, block_scale, group, torch.bfloat16)
|
||||
)
|
||||
weight.data[slot].copy_(packed.view(torch.uint8))
|
||||
weight_scale.data[slot].copy_(scale.view(torch.uint8))
|
||||
|
||||
def finalize_weights(self) -> None:
|
||||
if self._runtime is not None:
|
||||
return
|
||||
@@ -129,6 +197,33 @@ class DeepseekV4AiterMegaMoEExperts(nn.Module):
|
||||
self._runtime_cache[key] = runtime
|
||||
self._runtime = runtime
|
||||
|
||||
def append_shared_expert(
|
||||
self, topk_weights: torch.Tensor, topk_ids: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Remap routed ids to the per-rank (routed + shared) layout and append
|
||||
a weight-1.0 route to this rank's shared slot."""
|
||||
num_tokens = topk_ids.shape[0]
|
||||
out_ids = torch.empty(
|
||||
(num_tokens, self.top_k), dtype=topk_ids.dtype, device=topk_ids.device
|
||||
)
|
||||
out_weights = torch.empty(
|
||||
(num_tokens, self.top_k), dtype=torch.float32, device=topk_ids.device
|
||||
)
|
||||
if num_tokens:
|
||||
block_m = 64
|
||||
_append_shared_expert_kernel[(triton.cdiv(num_tokens, block_m),)](
|
||||
topk_ids.contiguous(),
|
||||
topk_weights.to(torch.float32).contiguous(),
|
||||
out_ids,
|
||||
out_weights,
|
||||
num_tokens,
|
||||
self.shared_expert_id,
|
||||
TOPK=self.routed_top_k,
|
||||
ROUTED_PER_RANK=self.num_routed_local_experts,
|
||||
BLOCK_M=block_m,
|
||||
)
|
||||
return out_weights, out_ids
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
@@ -137,6 +232,8 @@ class DeepseekV4AiterMegaMoEExperts(nn.Module):
|
||||
) -> torch.Tensor:
|
||||
runtime = self._runtime
|
||||
assert runtime is not None
|
||||
if self.fuse_shared_expert:
|
||||
topk_weights, topk_ids = self.append_shared_expert(topk_weights, topk_ids)
|
||||
# MegaMoEV2 reads these per launch; rebind to this layer's experts.
|
||||
runtime._s1_w1 = self.w13_weight
|
||||
runtime._s1_w1_scale = self.w13_weight_scale
|
||||
@@ -149,6 +246,10 @@ DeepseekV4AiterMegaMoEExperts.weight_loader.supports_moe_loading = True # type:
|
||||
|
||||
|
||||
def finalize_mega_moe_weights(model: nn.Module) -> None:
|
||||
for module in model.modules():
|
||||
if isinstance(module, DeepseekV4AiterMegaMoEExperts):
|
||||
module.finalize_weights()
|
||||
for module in list(model.modules()):
|
||||
experts = getattr(module, "experts", None)
|
||||
if isinstance(experts, DeepseekV4AiterMegaMoEExperts):
|
||||
if experts.fuse_shared_expert:
|
||||
experts.load_shared_expert(module.shared_experts)
|
||||
module.shared_experts = None
|
||||
experts.finalize_weights()
|
||||
|
||||
@@ -74,7 +74,10 @@ from vllm.model_executor.models.utils import (
|
||||
make_layers,
|
||||
maybe_prefix,
|
||||
)
|
||||
from vllm.models.deepseek_v4.amd.mega_moe import DeepseekV4AiterMegaMoEExperts
|
||||
from vllm.models.deepseek_v4.amd.mega_moe import (
|
||||
DeepseekV4AiterMegaMoEExperts,
|
||||
finalize_mega_moe_weights,
|
||||
)
|
||||
from vllm.models.deepseek_v4.amd.rocm import DeepseekV4ROCMAiterMLAAttention
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.platforms.rocm import on_gfx950
|
||||
@@ -690,6 +693,10 @@ class DeepseekV4MoE(nn.Module):
|
||||
hidden_size=config.hidden_size,
|
||||
intermediate_size=config.moe_intermediate_size,
|
||||
swiglu_limit=self.swiglu_limit,
|
||||
fuse_shared_expert=(
|
||||
config.n_shared_experts == 1
|
||||
and envs.VLLM_ROCM_AITER_MEGA_MOE_FUSE_SHARED_EXPERTS
|
||||
),
|
||||
prefix=f"{prefix}.experts",
|
||||
)
|
||||
return
|
||||
@@ -1524,6 +1531,8 @@ class DeepseekV4ForCausalLM(nn.Module, SupportsPP, SupportsEagle3):
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
def process_weights_after_loading(self) -> None:
|
||||
# Must precede the gate_up preshuffle of the fused shared experts.
|
||||
finalize_mega_moe_weights(self)
|
||||
# After per-layer quant finalize, so we preshuffle the final fp8 weights.
|
||||
fused_compressor_layers = 0
|
||||
for module in self.modules():
|
||||
@@ -1532,8 +1541,6 @@ class DeepseekV4ForCausalLM(nn.Module, SupportsPP, SupportsEagle3):
|
||||
module.prepare_attn_preshuffle()
|
||||
elif isinstance(module, DeepseekV4MLP):
|
||||
module.prepare_gateup_preshuffle()
|
||||
elif isinstance(module, DeepseekV4AiterMegaMoEExperts):
|
||||
module.finalize_weights()
|
||||
if fused_compressor_layers:
|
||||
logger.info(
|
||||
"Fused the C4 compressor GEMMs in %d DeepSeek V4 layers",
|
||||
|
||||
Reference in New Issue
Block a user