302 lines
12 KiB
Python
302 lines
12 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
"""AITER MegaMoE experts (fused dispatch, MXFP4 GEMMs, combine) for DeepSeek V4."""
|
|
|
|
from typing import Any
|
|
|
|
import torch
|
|
import torch.nn as nn
|
|
|
|
from vllm._aiter_ops import rocm_aiter_ops
|
|
from vllm.config import VllmConfig
|
|
from vllm.distributed import get_ep_group
|
|
from vllm.forward_context import get_forward_context, is_forward_context_available
|
|
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
|
get_fp8_block_weight_scale,
|
|
)
|
|
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import mxfp4_quantize
|
|
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
|
GroupShape,
|
|
scaled_dequantize,
|
|
)
|
|
from vllm.model_executor.utils import set_weight_attrs
|
|
from vllm.triton_utils import tl, triton
|
|
from vllm.utils.math_utils import next_power_of_2, prev_power_of_2
|
|
|
|
|
|
@triton.jit
|
|
def _append_shared_expert_kernel(
|
|
topk_ids_ptr,
|
|
topk_weights_ptr,
|
|
out_ids_ptr,
|
|
out_weights_ptr,
|
|
num_tokens,
|
|
shared_expert_id,
|
|
TOPK: tl.constexpr,
|
|
ROUTED_PER_RANK: tl.constexpr,
|
|
BLOCK_M: tl.constexpr,
|
|
):
|
|
rows = tl.program_id(0) * BLOCK_M + tl.arange(0, BLOCK_M)
|
|
mask = rows < num_tokens
|
|
for k in tl.static_range(TOPK):
|
|
e = tl.load(topk_ids_ptr + rows * TOPK + k, mask=mask, other=0)
|
|
w = tl.load(topk_weights_ptr + rows * TOPK + k, mask=mask, other=0.0)
|
|
e = tl.where(e >= 0, e + e // ROUTED_PER_RANK, e)
|
|
tl.store(out_ids_ptr + rows * (TOPK + 1) + k, e, mask=mask)
|
|
tl.store(out_weights_ptr + rows * (TOPK + 1) + k, w, mask=mask)
|
|
tl.store(
|
|
out_ids_ptr + rows * (TOPK + 1) + TOPK,
|
|
tl.zeros_like(rows) + shared_expert_id,
|
|
mask=mask,
|
|
)
|
|
tl.store(
|
|
out_weights_ptr + rows * (TOPK + 1) + TOPK,
|
|
tl.full([BLOCK_M], 1.0, tl.float32),
|
|
mask=mask,
|
|
)
|
|
|
|
|
|
class DeepseekV4AiterMegaMoEExperts(nn.Module):
|
|
# MegaMoEV2 owns large symmetric-heap buffers, so every layer shares one
|
|
# instance and passes its own weights per call.
|
|
_runtime_cache: dict[tuple, Any] = {}
|
|
|
|
def __init__(
|
|
self,
|
|
vllm_config: VllmConfig,
|
|
*,
|
|
num_experts: int,
|
|
top_k: int,
|
|
hidden_size: int,
|
|
intermediate_size: int,
|
|
swiglu_limit: float | None,
|
|
fuse_shared_expert: bool = False,
|
|
prefix: str = "",
|
|
):
|
|
super().__init__()
|
|
ep_group = get_ep_group()
|
|
self.ep_rank = ep_group.rank_in_group
|
|
self.ep_size = ep_group.world_size
|
|
self.num_routed_local_experts = num_experts // self.ep_size
|
|
self.experts_start_idx = self.ep_rank * self.num_routed_local_experts
|
|
# A fused shared expert is one extra local expert per rank and one
|
|
# extra route per token, to the token's own rank.
|
|
self.fuse_shared_expert = fuse_shared_expert
|
|
self.num_local_experts = self.num_routed_local_experts + fuse_shared_expert
|
|
self.num_experts = self.num_local_experts * self.ep_size
|
|
self.routed_top_k = top_k
|
|
self.top_k = top_k + fuse_shared_expert
|
|
self.shared_expert_id = (
|
|
self.ep_rank * self.num_local_experts + self.num_routed_local_experts
|
|
)
|
|
self.hidden_size = hidden_size
|
|
self.intermediate_size = intermediate_size
|
|
self.swiglu_limit = swiglu_limit or 0.0
|
|
self.max_num_tokens = next_power_of_2(
|
|
vllm_config.scheduler_config.max_num_batched_tokens
|
|
)
|
|
self.prefix = prefix
|
|
|
|
def make_param(rows: int, cols: int) -> nn.Parameter:
|
|
param = nn.Parameter(
|
|
torch.zeros(self.num_local_experts, rows, cols, dtype=torch.uint8),
|
|
requires_grad=False,
|
|
)
|
|
set_weight_attrs(param, {"weight_loader": self.weight_loader})
|
|
return param
|
|
|
|
self.w13_weight = make_param(2 * intermediate_size, hidden_size // 2)
|
|
self.w13_weight_scale = make_param(2 * intermediate_size, hidden_size // 32)
|
|
self.w2_weight = make_param(hidden_size, intermediate_size // 2)
|
|
self.w2_weight_scale = make_param(hidden_size, intermediate_size // 32)
|
|
# (max_tok_per_rank, runtime), ascending by capacity.
|
|
self._runtimes: list[tuple[int, Any]] = []
|
|
|
|
def weight_loader(
|
|
self,
|
|
param: nn.Parameter,
|
|
loaded_weight: torch.Tensor,
|
|
weight_name: str,
|
|
shard_id: str,
|
|
expert_id: int,
|
|
return_success: bool = False,
|
|
) -> bool | None:
|
|
local_expert_id = expert_id - self.experts_start_idx
|
|
if not 0 <= local_expert_id < self.num_routed_local_experts:
|
|
return False if return_success else None
|
|
expert_data = param.data[local_expert_id]
|
|
if shard_id in ("w1", "w3"):
|
|
shard_offset = 0 if shard_id == "w1" else self.intermediate_size
|
|
expert_data = expert_data.narrow(0, shard_offset, self.intermediate_size)
|
|
elif shard_id != "w2":
|
|
raise ValueError(f"Unsupported expert shard id: {shard_id}")
|
|
expert_data.copy_(loaded_weight.view(torch.uint8))
|
|
return True if return_success else None
|
|
|
|
def load_shared_expert(self, shared_experts: nn.Module) -> None:
|
|
slot = self.num_routed_local_experts
|
|
for linear, weight, weight_scale in (
|
|
(shared_experts.gate_up_proj, self.w13_weight, self.w13_weight_scale),
|
|
(shared_experts.down_proj, self.w2_weight, self.w2_weight_scale),
|
|
):
|
|
block_scale = get_fp8_block_weight_scale(linear)
|
|
assert block_scale is not None, "shared expert must be block-FP8"
|
|
group = GroupShape(
|
|
linear.weight.shape[0] // block_scale.shape[0],
|
|
linear.weight.shape[1] // block_scale.shape[1],
|
|
)
|
|
packed, scale = mxfp4_quantize(
|
|
scaled_dequantize(linear.weight, block_scale, group, torch.bfloat16)
|
|
)
|
|
weight.data[slot].copy_(packed.view(torch.uint8))
|
|
weight_scale.data[slot].copy_(scale.view(torch.uint8))
|
|
|
|
def finalize_weights(self) -> None:
|
|
if self._runtimes:
|
|
return
|
|
n = self.num_local_experts
|
|
self.w13_weight.data = rocm_aiter_ops.shuffle_weight_a16w4(
|
|
self.w13_weight.data, 16, True
|
|
)
|
|
self.w13_weight_scale.data = rocm_aiter_ops.shuffle_scale_a16w4(
|
|
self.w13_weight_scale.data.view(-1, self.hidden_size // 32), n, True
|
|
)
|
|
self.w2_weight.data = rocm_aiter_ops.shuffle_weight_a16w4(
|
|
self.w2_weight.data, 16, False
|
|
)
|
|
self.w2_weight_scale.data = rocm_aiter_ops.shuffle_scale_a16w4(
|
|
self.w2_weight_scale.data.view(-1, self.intermediate_size // 32), n, False
|
|
)
|
|
from aiter.ops.flydsl.kernels.mega_moe.mega_moe_config import (
|
|
FIXED_SLOT_MAX_MTPR,
|
|
)
|
|
|
|
# MegaMoEV2 takes fixed-slot dispatch up to FIXED_SLOT_MAX_MTPR (255 by
|
|
# default, raised via AITER_MEGA_FIXED_SLOT_MAX_MTPR); max_tok_per_rank
|
|
# must also be a power of two.
|
|
fixed_slot_max_tokens = prev_power_of_2(FIXED_SLOT_MAX_MTPR)
|
|
capacities = [self.max_num_tokens]
|
|
if self.max_num_tokens > fixed_slot_max_tokens and self._uses_fixed_slot(
|
|
fixed_slot_max_tokens
|
|
):
|
|
capacities.insert(0, fixed_slot_max_tokens)
|
|
self._runtimes = [(c, self.get_runtime(c)) for c in capacities]
|
|
|
|
def _uses_fixed_slot(self, max_tok_per_rank: int) -> bool:
|
|
from aiter.ops.flydsl.kernels.mega_moe.mega_moe_config import (
|
|
build_mega_moe_bundle_plan,
|
|
)
|
|
|
|
return build_mega_moe_bundle_plan(
|
|
max_tok_per_rank,
|
|
experts_per_rank=self.num_local_experts,
|
|
model_dim=self.hidden_size,
|
|
inter_dim=self.intermediate_size,
|
|
world_size=self.ep_size,
|
|
).fixed_slot_dispatch
|
|
|
|
def get_runtime(self, max_tok_per_rank: int) -> Any:
|
|
key = (
|
|
id(get_ep_group().cpu_group),
|
|
torch.accelerator.current_device_index(),
|
|
self.num_experts,
|
|
self.top_k,
|
|
self.hidden_size,
|
|
self.intermediate_size,
|
|
max_tok_per_rank,
|
|
self.swiglu_limit,
|
|
)
|
|
runtime = self._runtime_cache.get(key)
|
|
if runtime is None:
|
|
from aiter.ops.flydsl.kernels.mega_moe import MegaMoEV2
|
|
|
|
runtime = MegaMoEV2(
|
|
rank=self.ep_rank,
|
|
world_size=self.ep_size,
|
|
model_dim=self.hidden_size,
|
|
inter_dim=self.intermediate_size,
|
|
experts=self.num_experts,
|
|
topk=self.top_k,
|
|
quant="a8w4",
|
|
w1=self.w13_weight,
|
|
w1_scale=self.w13_weight_scale,
|
|
w2=self.w2_weight,
|
|
w2_scale=self.w2_weight_scale,
|
|
max_tok_per_rank=max_tok_per_rank,
|
|
swiglu_limit=self.swiglu_limit,
|
|
)
|
|
self._runtime_cache[key] = runtime
|
|
return runtime
|
|
|
|
def select_runtime(self, num_tokens: int) -> Any:
|
|
# Dispatch is collective, so every EP rank must pick the same runtime.
|
|
if is_forward_context_available():
|
|
dp_metadata = get_forward_context().dp_metadata
|
|
if dp_metadata is not None:
|
|
num_tokens = max(
|
|
num_tokens, int(dp_metadata.num_tokens_across_dp_cpu.max())
|
|
)
|
|
for max_tok_per_rank, runtime in self._runtimes:
|
|
if num_tokens <= max_tok_per_rank:
|
|
return runtime
|
|
raise ValueError(
|
|
f"{num_tokens} tokens exceeds MegaMoE capacity {self.max_num_tokens}"
|
|
)
|
|
|
|
def append_shared_expert(
|
|
self, topk_weights: torch.Tensor, topk_ids: torch.Tensor
|
|
) -> tuple[torch.Tensor, torch.Tensor]:
|
|
"""Remap routed ids to the per-rank (routed + shared) layout and append
|
|
a weight-1.0 route to this rank's shared slot."""
|
|
num_tokens = topk_ids.shape[0]
|
|
out_ids = torch.empty(
|
|
(num_tokens, self.top_k), dtype=topk_ids.dtype, device=topk_ids.device
|
|
)
|
|
out_weights = torch.empty(
|
|
(num_tokens, self.top_k), dtype=torch.float32, device=topk_ids.device
|
|
)
|
|
if num_tokens:
|
|
block_m = 64
|
|
_append_shared_expert_kernel[(triton.cdiv(num_tokens, block_m),)](
|
|
topk_ids.contiguous(),
|
|
topk_weights.to(torch.float32).contiguous(),
|
|
out_ids,
|
|
out_weights,
|
|
num_tokens,
|
|
self.shared_expert_id,
|
|
TOPK=self.routed_top_k,
|
|
ROUTED_PER_RANK=self.num_routed_local_experts,
|
|
BLOCK_M=block_m,
|
|
)
|
|
return out_weights, out_ids
|
|
|
|
def forward(
|
|
self,
|
|
hidden_states: torch.Tensor,
|
|
topk_weights: torch.Tensor,
|
|
topk_ids: torch.Tensor,
|
|
) -> torch.Tensor:
|
|
assert self._runtimes
|
|
runtime = self.select_runtime(hidden_states.shape[0])
|
|
if self.fuse_shared_expert:
|
|
topk_weights, topk_ids = self.append_shared_expert(topk_weights, topk_ids)
|
|
# MegaMoEV2 reads these per launch; rebind to this layer's experts.
|
|
runtime._s1_w1 = self.w13_weight
|
|
runtime._s1_w1_scale = self.w13_weight_scale
|
|
runtime.w2 = self.w2_weight
|
|
runtime.w2_scale = self.w2_weight_scale
|
|
return runtime.forward(hidden_states.contiguous(), topk_weights, topk_ids)
|
|
|
|
|
|
DeepseekV4AiterMegaMoEExperts.weight_loader.supports_moe_loading = True # type: ignore[attr-defined]
|
|
|
|
|
|
def finalize_mega_moe_weights(model: nn.Module) -> None:
|
|
for module in list(model.modules()):
|
|
experts = getattr(module, "experts", None)
|
|
if isinstance(experts, DeepseekV4AiterMegaMoEExperts):
|
|
if experts.fuse_shared_expert:
|
|
experts.load_shared_expert(module.shared_experts)
|
|
module.shared_experts = None
|
|
experts.finalize_weights()
|