[MoE] Unify DeepGEMM MegaMoE experts behind one feature-matrix layer; add NVFP4 (#61538)
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
Kevin H. Luu
parent
4d9a8d6a41
commit
4d58fa254a
@@ -16,7 +16,6 @@ from vllm.distributed import (
|
||||
get_pp_group,
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.distributed.eplb.eplb_state import EplbLayerState
|
||||
from vllm.forward_context import get_forward_context, is_forward_context_available
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.kernels.mhc.tilelang import (
|
||||
@@ -32,10 +31,9 @@ from vllm.model_executor.layers.fused_moe import (
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.deep_gemm_mega_moe import (
|
||||
get_deep_gemm_mega_moe_backend,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.router.base_router import (
|
||||
eplb_map_to_physical_and_record,
|
||||
MEGA_MOE_MXFP4,
|
||||
DeepGemmMegaMoEExperts,
|
||||
MegaMoEActivation,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.router.fused_topk_bias_router import (
|
||||
fused_topk_bias,
|
||||
@@ -70,7 +68,6 @@ from vllm.model_executor.models.utils import (
|
||||
maybe_prefix,
|
||||
spec_decode_needs_target_embed,
|
||||
)
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.models.common.ops.sequence_parallel import (
|
||||
sp_all_gather,
|
||||
sp_padding_mask,
|
||||
@@ -84,12 +81,11 @@ from vllm.models.deepseek_v4.nvidia.flashinfer_sparse import (
|
||||
DeepseekV4FlashInferSM120Attention,
|
||||
)
|
||||
from vllm.models.deepseek_v4.nvidia.flashmla import DeepseekV4FlashMLAAttention
|
||||
from vllm.models.deepseek_v4.nvidia.ops.prepare_megamoe import prepare_megamoe_inputs
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.deep_gemm import bf16_mega_gate
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
from vllm.v1.worker.ubatching import dbo_current_ubatch_id
|
||||
|
||||
from ..common.mm_preprocess import IMAGE_SENTINEL_BASE_ID
|
||||
|
||||
@@ -206,326 +202,6 @@ def make_deepseek_v4_expert_params_mapping(
|
||||
]
|
||||
|
||||
|
||||
class DeepseekV4MegaMoEExperts(nn.Module):
|
||||
_symm_buffer_cache: dict[
|
||||
tuple[int, int, int, int, int, int, int, int, str], object
|
||||
] = {}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
*,
|
||||
num_experts: int,
|
||||
num_local_experts: int,
|
||||
experts_start_idx: int,
|
||||
top_k: int,
|
||||
hidden_size: int,
|
||||
intermediate_size: int,
|
||||
num_shared_experts: int = 0,
|
||||
shared_experts: nn.Module | None = None,
|
||||
prefix: str = "",
|
||||
num_logical_experts: int | None = None,
|
||||
):
|
||||
super().__init__()
|
||||
self.prefix = prefix
|
||||
self.capture_fn: Callable[[torch.Tensor], None] | None = None
|
||||
self.num_experts = num_experts
|
||||
self.num_local_experts = num_local_experts
|
||||
self.experts_start_idx = experts_start_idx
|
||||
self.experts_end_idx = experts_start_idx + num_local_experts
|
||||
self.top_k = top_k
|
||||
self.hidden_size = hidden_size
|
||||
self.intermediate_size = intermediate_size
|
||||
self.max_num_tokens = vllm_config.scheduler_config.max_num_batched_tokens
|
||||
|
||||
self.num_logical_experts = (
|
||||
num_logical_experts if num_logical_experts is not None else num_experts
|
||||
)
|
||||
|
||||
self.eplb_state = EplbLayerState()
|
||||
|
||||
device = torch.get_default_device()
|
||||
if device.type != current_platform.device_type: # meta under ipc_cache
|
||||
device = torch.device(
|
||||
current_platform.device_type, torch.accelerator.current_device_index()
|
||||
)
|
||||
self.backend = get_deep_gemm_mega_moe_backend(
|
||||
device, hidden_size, intermediate_size
|
||||
)
|
||||
if num_shared_experts and not self.backend.supports_shared_experts(
|
||||
shared_experts
|
||||
):
|
||||
num_shared_experts = 0
|
||||
self.num_shared_experts = num_shared_experts
|
||||
self.backend.create_weights(
|
||||
self,
|
||||
num_local_experts=num_local_experts,
|
||||
hidden_size=hidden_size,
|
||||
intermediate_size=intermediate_size,
|
||||
num_shared_experts=num_shared_experts,
|
||||
)
|
||||
for name, param in self.named_parameters(recurse=False):
|
||||
shared = name.startswith("shared_")
|
||||
loader = self.shared_weight_loader if shared else self.weight_loader
|
||||
set_weight_attrs(param, {"weight_loader": loader})
|
||||
self.w13_weight_scale.quant_method = "block"
|
||||
self.w2_weight_scale.quant_method = "block"
|
||||
|
||||
# Register in the static forward context so the custom-op wrapper
|
||||
# can look up this module by name from within a torch.compile graph.
|
||||
compilation_config = vllm_config.compilation_config
|
||||
if prefix in compilation_config.static_forward_context:
|
||||
raise ValueError(f"Duplicate layer name: {prefix}")
|
||||
compilation_config.static_forward_context[prefix] = self
|
||||
|
||||
def _map_global_expert_id(self, expert_id: int) -> list[int]:
|
||||
"""Return local (per-rank) slot offsets where logical expert
|
||||
`expert_id` should land on this rank.
|
||||
"""
|
||||
physical_ids: list[int] = []
|
||||
for p in range(self.experts_start_idx, self.experts_end_idx):
|
||||
if p % self.num_logical_experts == expert_id:
|
||||
physical_ids.append(p - self.experts_start_idx)
|
||||
return physical_ids
|
||||
|
||||
def weight_loader(
|
||||
self,
|
||||
param: nn.Parameter,
|
||||
loaded_weight: torch.Tensor,
|
||||
weight_name: str,
|
||||
shard_id: str,
|
||||
expert_id: int,
|
||||
return_success: bool = False,
|
||||
) -> bool | None:
|
||||
local_expert_ids = self._map_global_expert_id(expert_id)
|
||||
param_prefix = "w2_" if shard_id == "w2" else "w13_"
|
||||
if not local_expert_ids or param_prefix not in weight_name:
|
||||
return False if return_success else None
|
||||
is_scale = weight_name.endswith("weight_scale")
|
||||
for local_expert_id in local_expert_ids:
|
||||
self.backend.load_weight(
|
||||
param.data[local_expert_id], loaded_weight, shard_id, is_scale
|
||||
)
|
||||
return True if return_success else None
|
||||
|
||||
def load_shared_expert_weight(self, name: str, loaded_weight: torch.Tensor) -> str:
|
||||
"""Load one shared-expert checkpoint tensor; return the parameter name."""
|
||||
if ".w2." in name or ".down_proj." in name:
|
||||
param_name = "shared_l2_weight"
|
||||
elif ".w1." in name or ".w3." in name:
|
||||
param_name = "shared_l1_weight"
|
||||
else:
|
||||
raise ValueError(f"Unexpected shared-expert tensor: {name}")
|
||||
if "weight_scale" in name:
|
||||
param_name += "_scale"
|
||||
param = getattr(self, param_name)
|
||||
param.weight_loader(param, loaded_weight, name)
|
||||
return f"{name.partition('shared_experts.')[0]}experts.{param_name}"
|
||||
|
||||
def shared_weight_loader(
|
||||
self, param: nn.Parameter, loaded_weight: torch.Tensor, weight_name: str
|
||||
) -> None:
|
||||
shard_id = (
|
||||
"w1" if ".w1." in weight_name else "w3" if ".w3." in weight_name else "w2"
|
||||
)
|
||||
is_scale = "weight_scale" in weight_name
|
||||
self.backend.load_weight(param.data, loaded_weight, shard_id, is_scale)
|
||||
|
||||
@property
|
||||
def _transformed_l1_weights(self) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
return self.w13_weight, self.w13_weight_scale
|
||||
|
||||
@property
|
||||
def _transformed_l2_weights(self) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
return self.w2_weight, self.w2_weight_scale
|
||||
|
||||
@property
|
||||
def has_fused_shared_experts(self) -> bool:
|
||||
return self.num_shared_experts > 0
|
||||
|
||||
def get_symm_buffer(self):
|
||||
from vllm.utils.deep_gemm import _import_deep_gemm
|
||||
|
||||
deep_gemm = _import_deep_gemm()
|
||||
|
||||
backend = self.backend
|
||||
group = get_ep_group().device_group
|
||||
device = torch.accelerator.current_device_index()
|
||||
key = (
|
||||
id(group),
|
||||
device,
|
||||
self.num_experts,
|
||||
self.max_num_tokens,
|
||||
self.top_k,
|
||||
self.hidden_size,
|
||||
self.intermediate_size,
|
||||
self.num_shared_experts if self.has_fused_shared_experts else 0,
|
||||
backend.mma_type,
|
||||
)
|
||||
symm_buffer = self._symm_buffer_cache.get(key)
|
||||
if symm_buffer is None:
|
||||
symm_buffer = deep_gemm.get_symm_buffer_for_mega_moe(
|
||||
group,
|
||||
self.num_experts,
|
||||
self.max_num_tokens,
|
||||
self.top_k,
|
||||
self.hidden_size,
|
||||
self.intermediate_size,
|
||||
num_shared_experts=(
|
||||
self.num_shared_experts if self.has_fused_shared_experts else 0
|
||||
),
|
||||
mma_type=backend.mma_type,
|
||||
)
|
||||
self._symm_buffer_cache[key] = symm_buffer
|
||||
return symm_buffer
|
||||
|
||||
def set_eplb_state(
|
||||
self,
|
||||
moe_layer_idx: int,
|
||||
expert_load_view: torch.Tensor,
|
||||
logical_to_physical_map: torch.Tensor,
|
||||
logical_replica_count: torch.Tensor,
|
||||
) -> None:
|
||||
self.eplb_state.set_layer_state(
|
||||
moe_layer_idx,
|
||||
expert_load_view,
|
||||
logical_to_physical_map,
|
||||
logical_replica_count,
|
||||
)
|
||||
|
||||
def get_expert_weights(self) -> list[torch.Tensor]:
|
||||
def _to_eplb_view(name: str, t: torch.Tensor) -> torch.Tensor:
|
||||
"""Return a (num_local_experts, -1) view with contiguous memory layout."""
|
||||
assert t.shape[0] == self.num_local_experts
|
||||
if t.is_contiguous():
|
||||
return t.view(self.num_local_experts, -1)
|
||||
elif t.dim() == 3 and t.stride(1) == 1 and t.stride(2) == t.shape[1]:
|
||||
# scales have shape (E, M, N) with memory layout (E, N, M)
|
||||
back = torch.transpose(t, 1, 2)
|
||||
assert back.is_contiguous()
|
||||
return back.view(self.num_local_experts, -1)
|
||||
|
||||
raise AssertionError(
|
||||
f"DSv4 EPLB {name}: non-contiguous expert tensor with "
|
||||
f"unexpected layout shape={tuple(t.shape)} "
|
||||
f"stride={tuple(t.stride())} dtype={t.dtype}"
|
||||
)
|
||||
|
||||
return [
|
||||
_to_eplb_view("l1_packed", self._transformed_l1_weights[0]),
|
||||
_to_eplb_view("l1_scale", self._transformed_l1_weights[1]),
|
||||
_to_eplb_view("l2_weight", self._transformed_l2_weights[0]),
|
||||
_to_eplb_view("l2_scale", self._transformed_l2_weights[1]),
|
||||
]
|
||||
|
||||
def update_expert_map(self) -> None:
|
||||
pass
|
||||
|
||||
@property
|
||||
def layer_id(self) -> int:
|
||||
return extract_layer_index(self.prefix)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
*,
|
||||
activation_clamp: float | None,
|
||||
fast_math: bool = True,
|
||||
) -> torch.Tensor:
|
||||
if hidden_states.shape[0] > self.max_num_tokens:
|
||||
raise ValueError(
|
||||
f"DeepSeek V4 MegaMoE got {hidden_states.shape[0]} tokens, "
|
||||
f"but the symmetric buffer was sized for {self.max_num_tokens}."
|
||||
)
|
||||
y = torch.empty_like(hidden_states, dtype=torch.bfloat16)
|
||||
|
||||
from vllm.utils.deep_gemm import _import_deep_gemm
|
||||
|
||||
deep_gemm = _import_deep_gemm()
|
||||
|
||||
num_tokens = hidden_states.shape[0]
|
||||
is_padding = None
|
||||
if envs.VLLM_MOE_SKIP_PADDING and is_forward_context_available():
|
||||
is_padding = get_forward_context().is_padding
|
||||
if is_padding is not None:
|
||||
is_padding = is_padding[:num_tokens]
|
||||
|
||||
if self.capture_fn is not None:
|
||||
self.capture_fn(topk_ids)
|
||||
|
||||
# EPLB: map logical expert IDs to physical replicas and record load.
|
||||
eplb_state = self.eplb_state
|
||||
if eplb_state.logical_to_physical_map is not None:
|
||||
assert eplb_state.expert_load_view is not None
|
||||
assert eplb_state.logical_replica_count is not None
|
||||
assert eplb_state.should_record_tensor is not None
|
||||
if is_padding is not None:
|
||||
topk_ids = torch.where(is_padding.unsqueeze(1), -1, topk_ids)
|
||||
topk_ids = eplb_map_to_physical_and_record(
|
||||
topk_ids=topk_ids,
|
||||
expert_load_view=eplb_state.expert_load_view,
|
||||
logical_to_physical_map=eplb_state.logical_to_physical_map,
|
||||
logical_replica_count=eplb_state.logical_replica_count,
|
||||
record_enabled=eplb_state.should_record_tensor,
|
||||
num_unpadded_tokens=eplb_state.num_unpadded_tokens_tensors[
|
||||
dbo_current_ubatch_id()
|
||||
]
|
||||
if eplb_state.num_unpadded_tokens_tensors is not None
|
||||
else None,
|
||||
)
|
||||
|
||||
backend = self.backend
|
||||
symm_buffer = self.get_symm_buffer()
|
||||
shared_x_sf = None
|
||||
shared_block_m = None
|
||||
if self.has_fused_shared_experts:
|
||||
shared_x_sf = symm_buffer.shared_l1_acts_sf
|
||||
shared_block_m = deep_gemm.get_block_m_for_mega_moe(
|
||||
get_ep_group().world_size,
|
||||
self.num_experts,
|
||||
symm_buffer.num_max_tokens_per_rank,
|
||||
num_tokens,
|
||||
self.top_k,
|
||||
backend.mma_type,
|
||||
)
|
||||
|
||||
prepare_megamoe_inputs(
|
||||
hidden_states,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
symm_buffer.x[:num_tokens],
|
||||
symm_buffer.x_sf[:num_tokens],
|
||||
symm_buffer.topk_idx[:num_tokens],
|
||||
symm_buffer.topk_weights[:num_tokens],
|
||||
is_padding=is_padding,
|
||||
shared_x_sf=shared_x_sf,
|
||||
shared_block_m=shared_block_m,
|
||||
hidden_quant=backend.hidden_quant,
|
||||
)
|
||||
|
||||
fused = self.has_fused_shared_experts
|
||||
backend.run_mega_moe(
|
||||
y=y,
|
||||
l1_weights=self._transformed_l1_weights,
|
||||
l2_weights=self._transformed_l2_weights,
|
||||
symm_buffer=symm_buffer,
|
||||
shared_l1_weights=(
|
||||
(self.shared_l1_weight, self.shared_l1_weight_scale) if fused else None
|
||||
),
|
||||
shared_l2_weights=(
|
||||
(self.shared_l2_weight, self.shared_l2_weight_scale) if fused else None
|
||||
),
|
||||
activation_clamp=activation_clamp,
|
||||
fast_math=fast_math,
|
||||
)
|
||||
return y
|
||||
|
||||
|
||||
DeepseekV4MegaMoEExperts.weight_loader.supports_moe_loading = True # type: ignore[attr-defined]
|
||||
|
||||
|
||||
class DeepseekV4MoE(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
@@ -701,7 +377,8 @@ class DeepseekV4MoE(nn.Module):
|
||||
and (self.use_sequence_parallel or self.tp_size == 1)
|
||||
)
|
||||
|
||||
expert_kwargs: dict[str, typing.Any] = dict(
|
||||
self.experts = DeepGemmMegaMoEExperts(
|
||||
vllm_config,
|
||||
num_experts=self.n_physical_experts,
|
||||
num_local_experts=self.n_local_physical_experts,
|
||||
experts_start_idx=self.physical_expert_start,
|
||||
@@ -709,11 +386,14 @@ class DeepseekV4MoE(nn.Module):
|
||||
top_k=self.n_activated_experts,
|
||||
hidden_size=config.hidden_size,
|
||||
intermediate_size=config.moe_intermediate_size,
|
||||
num_shared_experts=(self.n_shared_experts if fuse_shared_experts else 0),
|
||||
shared_experts=self.shared_experts,
|
||||
weight_format=MEGA_MOE_MXFP4,
|
||||
activation=MegaMoEActivation(
|
||||
"swiglu",
|
||||
clamp=None if self.swiglu_limit is None else float(self.swiglu_limit),
|
||||
),
|
||||
shared_experts=self.shared_experts if fuse_shared_experts else None,
|
||||
prefix=f"{prefix}.experts",
|
||||
)
|
||||
self.experts = DeepseekV4MegaMoEExperts(vllm_config, **expert_kwargs)
|
||||
if self.experts.has_fused_shared_experts:
|
||||
self.shared_experts = None
|
||||
|
||||
@@ -783,13 +463,9 @@ class DeepseekV4MoE(nn.Module):
|
||||
return self._forward_fused_moe(hidden_states, input_ids)
|
||||
|
||||
org_shape = hidden_states.shape
|
||||
mega_backend = self.experts.backend
|
||||
# Small local padded batches favor GateLinear; 128-expert gates cross earlier.
|
||||
gate_threshold = 1 if self.gate.weight.shape[0] == 128 else 16
|
||||
if (
|
||||
not mega_backend.supports_bf16_mega_gate()
|
||||
or hidden_states.shape[0] <= gate_threshold
|
||||
):
|
||||
if hidden_states.shape[0] <= gate_threshold:
|
||||
router_logits, _ = self.gate(hidden_states)
|
||||
topk_weights, topk_ids = fused_topk_bias(
|
||||
hidden_states=hidden_states,
|
||||
@@ -825,10 +501,10 @@ class DeepseekV4MoE(nn.Module):
|
||||
]
|
||||
fix_routing_mask = mega_gate_metadata.hash_token_mask
|
||||
|
||||
topk_weights, topk_ids = mega_backend.bf16_mega_gate(
|
||||
x=hidden_states,
|
||||
weight=self.gate.weight,
|
||||
num_topk=self.n_activated_experts,
|
||||
topk_weights, topk_ids = bf16_mega_gate(
|
||||
hidden_states,
|
||||
self.gate.weight,
|
||||
self.n_activated_experts,
|
||||
scoring_func=self.scoring_func,
|
||||
routed_scaling_factor=self.routed_scaling_factor,
|
||||
ep_rank=self.ep_rank,
|
||||
@@ -840,20 +516,9 @@ class DeepseekV4MoE(nn.Module):
|
||||
fix_routing_mask=fix_routing_mask,
|
||||
unmapped_topk_idx=unmapped_topk_idx,
|
||||
)
|
||||
activation_clamp = (
|
||||
float(self.swiglu_limit) if self.swiglu_limit is not None else None
|
||||
)
|
||||
final_hidden_states = self.experts(
|
||||
hidden_states,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
activation_clamp=activation_clamp,
|
||||
hidden_states, topk_weights, topk_ids, self.shared_experts
|
||||
)
|
||||
|
||||
if self.shared_experts is not None:
|
||||
shared_output = self.shared_experts(hidden_states)
|
||||
final_hidden_states += shared_output
|
||||
|
||||
return final_hidden_states.view(org_shape)
|
||||
|
||||
def _forward_fused_moe(
|
||||
|
||||
@@ -1,323 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Triton input-staging kernel for DeepSeek V4 MegaMoE.
|
||||
|
||||
Quantizes hidden states to fp8 with E8M0 group scales and repacks the
|
||||
routing top-k tensors into the int64/float32 layout that the DeepGEMM
|
||||
MegaMoE kernels consume.
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
FP8_DTYPE,
|
||||
QuantKey,
|
||||
kMxfp8Dynamic,
|
||||
)
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
_E8M0_SCALE_DTYPES = (torch.uint8, torch.float8_e8m0fnu)
|
||||
|
||||
|
||||
@triton.jit(do_not_specialize=["num_tokens"])
|
||||
def _prepare_megamoe_inputs_kernel(
|
||||
hidden_states,
|
||||
x_fp8,
|
||||
x_sf,
|
||||
shared_x_sf,
|
||||
topk_ids,
|
||||
topk_weights,
|
||||
is_padding,
|
||||
topk_idx_out,
|
||||
topk_weights_out,
|
||||
hidden_stride_m: tl.constexpr,
|
||||
hidden_stride_k: tl.constexpr,
|
||||
x_stride_m: tl.constexpr,
|
||||
x_stride_k: tl.constexpr,
|
||||
x_sf_stride_m: tl.constexpr,
|
||||
x_sf_stride_k: tl.constexpr,
|
||||
shared_x_sf_stride_m: tl.constexpr,
|
||||
shared_x_sf_stride_k: tl.constexpr,
|
||||
topk_ids_stride_m: tl.constexpr,
|
||||
topk_ids_stride_k: tl.constexpr,
|
||||
topk_weights_stride_m: tl.constexpr,
|
||||
topk_weights_stride_k: tl.constexpr,
|
||||
is_padding_stride_m: tl.constexpr,
|
||||
topk_idx_stride_m: tl.constexpr,
|
||||
topk_idx_stride_k: tl.constexpr,
|
||||
topk_weights_out_stride_m: tl.constexpr,
|
||||
topk_weights_out_stride_k: tl.constexpr,
|
||||
num_tokens,
|
||||
hidden_size: tl.constexpr,
|
||||
top_k: tl.constexpr,
|
||||
BLOCK_M: tl.constexpr,
|
||||
BLOCK_K: tl.constexpr,
|
||||
GROUP_K: tl.constexpr,
|
||||
BLOCK_TOPK: tl.constexpr,
|
||||
SHARED_BLOCK_M: tl.constexpr,
|
||||
USE_UE8M0: tl.constexpr,
|
||||
) -> None:
|
||||
token_id = tl.program_id(0) * BLOCK_M + tl.arange(0, BLOCK_M)
|
||||
token_mask = token_id < num_tokens
|
||||
k_block_id = tl.program_id(1)
|
||||
|
||||
k_offsets = k_block_id * BLOCK_K + tl.arange(0, BLOCK_K)
|
||||
k_mask = token_mask[:, None] & (k_offsets[None, :] < hidden_size)
|
||||
hidden = tl.load(
|
||||
hidden_states
|
||||
+ token_id[:, None] * hidden_stride_m
|
||||
+ k_offsets[None, :] * hidden_stride_k,
|
||||
mask=k_mask,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
|
||||
num_groups: tl.constexpr = BLOCK_K // GROUP_K
|
||||
hidden_groups = tl.reshape(tl.abs(hidden), [BLOCK_M, num_groups, GROUP_K])
|
||||
amax = tl.max(hidden_groups, axis=2)
|
||||
amax = tl.maximum(amax, 1.0e-4)
|
||||
|
||||
scale = amax / 448.0
|
||||
if USE_UE8M0:
|
||||
scale_bits = scale.to(tl.uint32, bitcast=True)
|
||||
scale_exp = ((scale_bits >> 23) & 0xFF) + ((scale_bits & 0x7FFFFF) != 0).to(
|
||||
tl.uint32
|
||||
)
|
||||
scale_exp = tl.minimum(tl.maximum(scale_exp, 1), 254)
|
||||
quant_scale = (scale_exp << 23).to(tl.float32, bitcast=True)
|
||||
else:
|
||||
quant_scale = scale
|
||||
|
||||
hidden_groups = tl.reshape(hidden, [BLOCK_M, num_groups, GROUP_K])
|
||||
scaled = hidden_groups * (1.0 / quant_scale)[:, :, None]
|
||||
scaled = tl.reshape(scaled, [BLOCK_M, BLOCK_K])
|
||||
fp8 = scaled.to(tl.float8e4nv)
|
||||
tl.store(
|
||||
x_fp8 + token_id[:, None] * x_stride_m + k_offsets[None, :] * x_stride_k,
|
||||
fp8,
|
||||
mask=k_mask,
|
||||
)
|
||||
|
||||
if USE_UE8M0:
|
||||
scale_offsets = tl.arange(0, num_groups)
|
||||
packed_scale = tl.sum(scale_exp << (scale_offsets[None, :] * 8), axis=1).to(
|
||||
tl.int32
|
||||
)
|
||||
tl.store(
|
||||
x_sf + token_id * x_sf_stride_m + k_block_id * x_sf_stride_k,
|
||||
packed_scale,
|
||||
mask=token_mask,
|
||||
)
|
||||
else:
|
||||
sf_offset = tl.arange(0, num_groups)
|
||||
tl.store(
|
||||
x_sf
|
||||
+ token_id[:, None] * x_sf_stride_m
|
||||
+ k_block_id * x_sf_stride_k
|
||||
+ sf_offset[None, :],
|
||||
quant_scale,
|
||||
mask=token_mask[:, None],
|
||||
)
|
||||
|
||||
# DeepGEMM's SM100 shared-expert TMA loads require the activation scales
|
||||
# in an MN-major layout whose row permutation depends on the MegaMoE
|
||||
# scheduler's runtime BLOCK_M. Write that view while the packed UE8M0 scale
|
||||
# is already resident, avoiding another kernel and temporary tensor.
|
||||
if USE_UE8M0 and shared_x_sf is not None:
|
||||
m_block_id = token_id // SHARED_BLOCK_M
|
||||
m_in_block = token_id % SHARED_BLOCK_M
|
||||
aligned_block_m: tl.constexpr = triton.cdiv(SHARED_BLOCK_M, 128) * 128
|
||||
transposed_m = (
|
||||
(m_in_block // 128) * 128 + (m_in_block % 32) * 4 + (m_in_block % 128) // 32
|
||||
)
|
||||
shared_row = m_block_id * aligned_block_m + transposed_m
|
||||
tl.store(
|
||||
shared_x_sf
|
||||
+ shared_row * shared_x_sf_stride_m
|
||||
+ k_block_id * shared_x_sf_stride_k,
|
||||
packed_scale,
|
||||
mask=token_mask,
|
||||
)
|
||||
|
||||
if k_block_id == 0:
|
||||
topk_offsets = tl.arange(0, BLOCK_TOPK)
|
||||
topk_mask = token_mask[:, None] & (topk_offsets[None, :] < top_k)
|
||||
token_is_padding = tl.full((BLOCK_M,), False, tl.int1)
|
||||
if is_padding is not None:
|
||||
token_is_padding = tl.load(
|
||||
is_padding + token_id * is_padding_stride_m,
|
||||
mask=token_mask,
|
||||
other=True,
|
||||
)
|
||||
|
||||
ids = tl.load(
|
||||
topk_ids
|
||||
+ token_id[:, None] * topk_ids_stride_m
|
||||
+ topk_offsets[None, :] * topk_ids_stride_k,
|
||||
mask=topk_mask,
|
||||
other=0,
|
||||
).to(tl.int64)
|
||||
ids = tl.where(token_is_padding[:, None], -1, ids)
|
||||
tl.store(
|
||||
topk_idx_out
|
||||
+ token_id[:, None] * topk_idx_stride_m
|
||||
+ topk_offsets[None, :] * topk_idx_stride_k,
|
||||
ids,
|
||||
mask=topk_mask,
|
||||
)
|
||||
|
||||
weights = tl.load(
|
||||
topk_weights
|
||||
+ token_id[:, None] * topk_weights_stride_m
|
||||
+ topk_offsets[None, :] * topk_weights_stride_k,
|
||||
mask=topk_mask,
|
||||
other=0.0,
|
||||
)
|
||||
weights = tl.where(token_is_padding[:, None], 0.0, weights)
|
||||
tl.store(
|
||||
topk_weights_out
|
||||
+ token_id[:, None] * topk_weights_out_stride_m
|
||||
+ topk_offsets[None, :] * topk_weights_out_stride_k,
|
||||
weights,
|
||||
mask=topk_mask,
|
||||
)
|
||||
|
||||
|
||||
def _resolve_hidden_quant(hidden_quant: QuantKey, block_k: int) -> tuple[int, bool]:
|
||||
"""Map a hidden-state ``QuantKey`` to the kernel's ``(GROUP_K, USE_UE8M0)``."""
|
||||
scale = hidden_quant.scale
|
||||
group_shape = scale.group_shape
|
||||
if (
|
||||
hidden_quant.dtype != FP8_DTYPE
|
||||
or not hidden_quant.symmetric
|
||||
or hidden_quant.scale2 is not None
|
||||
or scale.static
|
||||
or not group_shape.is_per_group()
|
||||
or block_k % group_shape.col != 0
|
||||
or scale.dtype not in (*_E8M0_SCALE_DTYPES, torch.float32)
|
||||
):
|
||||
raise ValueError(
|
||||
"DeepSeek V4 MegaMoE input staging requires symmetric dynamic fp8 "
|
||||
f"per-group quantization with a group size dividing {block_k} and "
|
||||
f"E8M0 or fp32 scales, got {hidden_quant}."
|
||||
)
|
||||
return group_shape.col, scale.dtype in _E8M0_SCALE_DTYPES
|
||||
|
||||
|
||||
def prepare_megamoe_inputs(
|
||||
hidden_states: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
x_fp8: torch.Tensor,
|
||||
x_sf: torch.Tensor,
|
||||
topk_idx_out: torch.Tensor,
|
||||
topk_weights_out: torch.Tensor,
|
||||
is_padding: torch.Tensor | None = None,
|
||||
shared_x_sf: torch.Tensor | None = None,
|
||||
shared_block_m: int | None = None,
|
||||
hidden_quant: QuantKey = kMxfp8Dynamic,
|
||||
) -> None:
|
||||
"""Quantize hidden states and repack top-k routing for DeepGEMM MegaMoE.
|
||||
|
||||
Args:
|
||||
hidden_states: Input activations of shape ``[num_tokens, hidden_dim]``.
|
||||
topk_weights: Router top-k weights of shape ``[num_tokens, top_k]``.
|
||||
topk_ids: Router top-k expert ids of shape ``[num_tokens, top_k]``.
|
||||
x_fp8: Output buffer for the fp8-quantized hidden states.
|
||||
x_sf: Output buffer for the hidden-state scale factors.
|
||||
topk_idx_out: Output buffer for the repacked top-k expert ids.
|
||||
topk_weights_out: Output buffer for the repacked top-k weights.
|
||||
is_padding: Optional per-token mask; padded tokens are not routed.
|
||||
shared_x_sf: Optional output buffer for shared-expert scale factors.
|
||||
Must be given together with ``shared_block_m``.
|
||||
shared_block_m: Block M used to lay out ``shared_x_sf``.
|
||||
hidden_quant: Hidden-state quantization scheme. E8M0 scales are packed
|
||||
four per int32 into ``x_sf``; fp32 scales are stored directly.
|
||||
|
||||
"""
|
||||
num_tokens, hidden_size = hidden_states.shape
|
||||
if num_tokens == 0:
|
||||
return
|
||||
if hidden_size % 128 != 0:
|
||||
raise ValueError(
|
||||
"DeepSeek V4 MegaMoE input staging requires hidden_size to be "
|
||||
"a multiple of 128."
|
||||
)
|
||||
block_k = 128
|
||||
group_k, use_ue8m0 = _resolve_hidden_quant(hidden_quant, block_k)
|
||||
top_k = topk_ids.shape[1]
|
||||
if topk_weights.shape != topk_ids.shape:
|
||||
raise ValueError(
|
||||
"DeepSeek V4 MegaMoE input staging requires topk_weights and "
|
||||
"topk_ids to have the same shape."
|
||||
)
|
||||
if (shared_x_sf is None) != (shared_block_m is None):
|
||||
raise ValueError(
|
||||
"DeepSeek V4 MegaMoE shared input staging requires both "
|
||||
"shared_x_sf and shared_block_m."
|
||||
)
|
||||
if shared_x_sf is not None and not use_ue8m0:
|
||||
raise ValueError(
|
||||
"DeepSeek V4 MegaMoE shared input staging currently requires "
|
||||
"UE8M0-packed hidden scales."
|
||||
)
|
||||
if shared_x_sf is not None:
|
||||
assert shared_block_m is not None
|
||||
if shared_block_m <= 0:
|
||||
raise ValueError("MegaMoE shared_block_m must be positive.")
|
||||
expected_sf_k = hidden_size // 128
|
||||
if shared_x_sf.ndim != 2 or shared_x_sf.shape[1] != expected_sf_k:
|
||||
raise ValueError(
|
||||
"MegaMoE shared_x_sf must have shape "
|
||||
f"(*, {expected_sf_k}), got {tuple(shared_x_sf.shape)}."
|
||||
)
|
||||
aligned_block_m = triton.cdiv(shared_block_m, 128) * 128
|
||||
required_rows = triton.cdiv(num_tokens, shared_block_m) * aligned_block_m
|
||||
if shared_x_sf.shape[0] < required_rows:
|
||||
raise ValueError(
|
||||
"MegaMoE shared_x_sf has insufficient rows: requires "
|
||||
f"{required_rows}, got {shared_x_sf.shape[0]}."
|
||||
)
|
||||
|
||||
# On GB200, eight-row tiles win from 64 tokens; keep smaller batches untiled.
|
||||
block_m = 8 if num_tokens >= 64 else 1
|
||||
grid = (triton.cdiv(num_tokens, block_m), triton.cdiv(hidden_size, block_k))
|
||||
block_topk = triton.next_power_of_2(top_k)
|
||||
padding_stride_m = is_padding.stride(0) if is_padding is not None else 0
|
||||
_prepare_megamoe_inputs_kernel[grid](
|
||||
hidden_states,
|
||||
x_fp8,
|
||||
x_sf,
|
||||
shared_x_sf,
|
||||
topk_ids,
|
||||
topk_weights,
|
||||
is_padding,
|
||||
topk_idx_out,
|
||||
topk_weights_out,
|
||||
hidden_states.stride(0),
|
||||
hidden_states.stride(1),
|
||||
x_fp8.stride(0),
|
||||
x_fp8.stride(1),
|
||||
x_sf.stride(0),
|
||||
x_sf.stride(1),
|
||||
shared_x_sf.stride(0) if shared_x_sf is not None else 0,
|
||||
shared_x_sf.stride(1) if shared_x_sf is not None else 0,
|
||||
topk_ids.stride(0),
|
||||
topk_ids.stride(1),
|
||||
topk_weights.stride(0),
|
||||
topk_weights.stride(1),
|
||||
padding_stride_m,
|
||||
topk_idx_out.stride(0),
|
||||
topk_idx_out.stride(1),
|
||||
topk_weights_out.stride(0),
|
||||
topk_weights_out.stride(1),
|
||||
num_tokens,
|
||||
hidden_size,
|
||||
top_k,
|
||||
BLOCK_M=block_m,
|
||||
BLOCK_K=block_k,
|
||||
GROUP_K=group_k,
|
||||
BLOCK_TOPK=block_topk,
|
||||
SHARED_BLOCK_M=shared_block_m or 1,
|
||||
USE_UE8M0=use_ue8m0,
|
||||
num_warps=4,
|
||||
)
|
||||
@@ -23,8 +23,10 @@ from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoEFactory,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.router.base_router import (
|
||||
eplb_map_to_physical_and_record,
|
||||
from vllm.model_executor.layers.fused_moe.deep_gemm_mega_moe import (
|
||||
MEGA_MOE_MXFP4,
|
||||
DeepGemmMegaMoEExperts,
|
||||
MegaMoEActivation,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.router.gate_linear import GateLinear
|
||||
from vllm.model_executor.layers.fused_moe.router.grouped_topk_router import (
|
||||
@@ -95,8 +97,6 @@ from vllm.models.common.ops.sequence_parallel import (
|
||||
sp_reduce_scatter,
|
||||
sp_shard,
|
||||
)
|
||||
from vllm.models.deepseek_v4.nvidia.model import DeepseekV4MegaMoEExperts
|
||||
from vllm.models.deepseek_v4.nvidia.ops.prepare_megamoe import prepare_megamoe_inputs
|
||||
from vllm.models.kimi_k3.nvidia.kda import KimiK3DeltaAttention
|
||||
from vllm.models.kimi_k3.nvidia.latent_moe_runner import (
|
||||
LatentMoERunner,
|
||||
@@ -115,7 +115,6 @@ from vllm.transformers_utils.configs.kimi_linear import KimiLinearConfig
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.utils.multi_stream_utils import maybe_execute_in_parallel
|
||||
from vllm.utils.torch_utils import aux_stream, is_meta_module
|
||||
from vllm.v1.worker.ubatching import dbo_current_ubatch_id
|
||||
|
||||
from ..common.mm_preprocess import (
|
||||
KimiK3DummyInputsBuilder,
|
||||
@@ -330,143 +329,6 @@ class KimiRoutedOutputTransform(nn.Module):
|
||||
return hidden_states
|
||||
|
||||
|
||||
class KimiK3MegaMoEExperts(DeepseekV4MegaMoEExperts):
|
||||
"""Kimi K3 adapter for the DeepGEMM MegaMoE kernel."""
|
||||
|
||||
_kimi_symm_buffer_cache: dict[tuple[object, ...], object] = {}
|
||||
_synchronized_ep_groups: set[tuple[int, int]] = set()
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*args,
|
||||
activation: str,
|
||||
activation_beta: float | None,
|
||||
activation_linear_beta: float | None,
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.activation = activation
|
||||
self.activation_beta = activation_beta
|
||||
self.activation_linear_beta = activation_linear_beta
|
||||
|
||||
def synchronize_first_launch(self) -> None:
|
||||
ep_group = get_ep_group()
|
||||
device = torch.accelerator.current_device_index()
|
||||
key = (id(ep_group.cpu_group), device)
|
||||
if key in self._synchronized_ep_groups:
|
||||
return
|
||||
torch.accelerator.synchronize()
|
||||
torch.distributed.barrier(group=ep_group.cpu_group)
|
||||
self._synchronized_ep_groups.add(key)
|
||||
|
||||
def get_symm_buffer(self):
|
||||
from vllm.utils.deep_gemm import _import_deep_gemm
|
||||
|
||||
deep_gemm = _import_deep_gemm()
|
||||
backend = self.backend
|
||||
group = get_ep_group().device_group
|
||||
device = torch.accelerator.current_device_index()
|
||||
key = (
|
||||
id(group),
|
||||
device,
|
||||
self.num_experts,
|
||||
self.max_num_tokens,
|
||||
self.top_k,
|
||||
self.hidden_size,
|
||||
self.intermediate_size,
|
||||
self.activation,
|
||||
backend.mma_type,
|
||||
)
|
||||
symm_buffer = self._kimi_symm_buffer_cache.get(key)
|
||||
if symm_buffer is None:
|
||||
symm_buffer = deep_gemm.get_symm_buffer_for_mega_moe(
|
||||
group,
|
||||
self.num_experts,
|
||||
self.max_num_tokens,
|
||||
self.top_k,
|
||||
self.hidden_size,
|
||||
self.intermediate_size,
|
||||
activation=self.activation,
|
||||
mma_type=backend.mma_type,
|
||||
)
|
||||
self._kimi_symm_buffer_cache[key] = symm_buffer
|
||||
return symm_buffer
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
*,
|
||||
activation_clamp: float | None,
|
||||
fast_math: bool = True,
|
||||
) -> torch.Tensor:
|
||||
self.synchronize_first_launch()
|
||||
if hidden_states.shape[0] > self.max_num_tokens:
|
||||
raise ValueError(
|
||||
f"Kimi K3 MegaMoE got {hidden_states.shape[0]} tokens, "
|
||||
f"but its symmetric buffer supports {self.max_num_tokens}."
|
||||
)
|
||||
y = torch.empty_like(hidden_states, dtype=torch.bfloat16)
|
||||
num_tokens = hidden_states.shape[0]
|
||||
is_padding = None
|
||||
if envs.VLLM_MOE_SKIP_PADDING and is_forward_context_available():
|
||||
is_padding = get_forward_context().is_padding
|
||||
if is_padding is not None:
|
||||
is_padding = is_padding[:num_tokens]
|
||||
|
||||
if self.capture_fn is not None:
|
||||
self.capture_fn(topk_ids)
|
||||
|
||||
eplb_state = self.eplb_state
|
||||
if eplb_state.logical_to_physical_map is not None:
|
||||
assert eplb_state.expert_load_view is not None
|
||||
assert eplb_state.logical_replica_count is not None
|
||||
assert eplb_state.should_record_tensor is not None
|
||||
if is_padding is not None:
|
||||
topk_ids = torch.where(is_padding.unsqueeze(1), -1, topk_ids)
|
||||
topk_ids = eplb_map_to_physical_and_record(
|
||||
topk_ids=topk_ids,
|
||||
expert_load_view=eplb_state.expert_load_view,
|
||||
logical_to_physical_map=eplb_state.logical_to_physical_map,
|
||||
logical_replica_count=eplb_state.logical_replica_count,
|
||||
record_enabled=eplb_state.should_record_tensor,
|
||||
num_unpadded_tokens=eplb_state.num_unpadded_tokens_tensors[
|
||||
dbo_current_ubatch_id()
|
||||
]
|
||||
if eplb_state.num_unpadded_tokens_tensors is not None
|
||||
else None,
|
||||
)
|
||||
|
||||
backend = self.backend
|
||||
symm_buffer = self.get_symm_buffer()
|
||||
prepare_megamoe_inputs(
|
||||
hidden_states,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
symm_buffer.x[:num_tokens],
|
||||
symm_buffer.x_sf[:num_tokens],
|
||||
symm_buffer.topk_idx[:num_tokens],
|
||||
symm_buffer.topk_weights[:num_tokens],
|
||||
is_padding=is_padding,
|
||||
hidden_quant=backend.hidden_quant,
|
||||
)
|
||||
backend.run_mega_moe(
|
||||
y=y,
|
||||
l1_weights=self._transformed_l1_weights,
|
||||
l2_weights=self._transformed_l2_weights,
|
||||
symm_buffer=symm_buffer,
|
||||
activation_clamp=activation_clamp,
|
||||
fast_math=fast_math,
|
||||
# DeepGEMM names the SiTU gate tanh scale `activation_alpha` and the
|
||||
# linear/up tanh scale `activation_beta`; beta=0 leaves up untouched.
|
||||
activation=self.activation,
|
||||
activation_alpha=self.activation_beta or 1.0,
|
||||
activation_beta=self.activation_linear_beta or 0.0,
|
||||
)
|
||||
return y
|
||||
|
||||
|
||||
def make_kimi_k3_mega_moe_expert_params_mapping(
|
||||
num_experts: int,
|
||||
) -> list[tuple[str, str, int, str]]:
|
||||
@@ -650,7 +512,7 @@ class KimiMoE(nn.Module):
|
||||
f"EP size {ep_size}."
|
||||
)
|
||||
num_local_experts = num_experts // ep_size
|
||||
self.experts = KimiK3MegaMoEExperts(
|
||||
self.experts = DeepGemmMegaMoEExperts(
|
||||
vllm_config,
|
||||
num_experts=num_experts,
|
||||
num_local_experts=num_local_experts,
|
||||
@@ -658,10 +520,13 @@ class KimiMoE(nn.Module):
|
||||
top_k=num_experts_per_token,
|
||||
hidden_size=self.moe_hidden_size,
|
||||
intermediate_size=self.padded_moe_intermediate_size,
|
||||
activation=MegaMoEActivation(
|
||||
"situ",
|
||||
alpha=activation_situ_beta or 1.0,
|
||||
beta=activation_situ_linear_beta or 0.0,
|
||||
),
|
||||
weight_format=MEGA_MOE_MXFP4,
|
||||
prefix=f"{prefix}.experts",
|
||||
activation="situ",
|
||||
activation_beta=activation_situ_beta,
|
||||
activation_linear_beta=activation_situ_linear_beta,
|
||||
)
|
||||
else:
|
||||
self.experts = FusedMoEFactory(
|
||||
@@ -777,11 +642,10 @@ class KimiMoE(nn.Module):
|
||||
if self.use_mega_moe:
|
||||
assert self.routed_output_transform is not None
|
||||
assert topk_ids is not None
|
||||
# The shared expert reads the un-projected hidden states, so it
|
||||
# stays outside the latent-space MegaMoE launch.
|
||||
final_hidden_states = self.experts(
|
||||
routed_hidden_states,
|
||||
router_output,
|
||||
topk_ids,
|
||||
activation_clamp=None,
|
||||
routed_hidden_states, router_output, topk_ids
|
||||
)
|
||||
# The shared output is folded into the up-projection GEMM's beta-add
|
||||
# epilogue, so combining the two branches costs no extra kernel.
|
||||
|
||||
Reference in New Issue
Block a user