[DSV4][Kernel] Fuse shared experts into MegaMoE (#53040)
Signed-off-by: Canlin Guo <canlinguosdu@gmail.com>
This commit is contained in:
@@ -288,6 +288,7 @@ if TYPE_CHECKING:
|
||||
VLLM_GC_DEBUG: str = ""
|
||||
VLLM_DEBUG_WORKSPACE: bool = False
|
||||
VLLM_DISABLE_SHARED_EXPERTS_STREAM: bool = False
|
||||
VLLM_DISABLE_DSV4_MEGAMOE_SHARED_EXPERT_FUSION: bool = False
|
||||
VLLM_SHARED_EXPERTS_STREAM_TOKEN_THRESHOLD: int = 256
|
||||
VLLM_MULTI_STREAM_GEMM_TOKEN_THRESHOLD: int = 1024
|
||||
VLLM_COMPILE_CACHE_SAVE_FORMAT: Literal["binary", "unpacked"] = "binary"
|
||||
@@ -1993,6 +1994,12 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
"VLLM_DISABLE_SHARED_EXPERTS_STREAM": lambda: bool(
|
||||
int(os.getenv("VLLM_DISABLE_SHARED_EXPERTS_STREAM", "0"))
|
||||
),
|
||||
# Emergency rollback for the DeepSeek-V4 NVIDIA MegaMoE path. By default,
|
||||
# DeepGEMM computes replicated FP8 shared experts in the same persistent
|
||||
# SM100 kernel as the routed FP4 experts.
|
||||
"VLLM_DISABLE_DSV4_MEGAMOE_SHARED_EXPERT_FUSION": lambda: bool(
|
||||
int(os.getenv("VLLM_DISABLE_DSV4_MEGAMOE_SHARED_EXPERT_FUSION", "0"))
|
||||
),
|
||||
# Limits when we run shared_experts in a separate stream.
|
||||
# We found out that for large batch sizes, the separate stream
|
||||
# execution is not beneficial (most likely because of the input clone)
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import typing
|
||||
from collections.abc import Callable, Iterable
|
||||
from inspect import signature
|
||||
from itertools import islice
|
||||
|
||||
import regex as re
|
||||
@@ -18,6 +19,7 @@ from vllm.distributed import (
|
||||
)
|
||||
from vllm.distributed.eplb.eplb_state import EplbLayerState
|
||||
from vllm.forward_context import get_forward_context, is_forward_context_available
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.kernels.mhc.tilelang import (
|
||||
hc_head_fused_kernel_tilelang,
|
||||
mhc_fused_post_pre_tilelang,
|
||||
@@ -84,6 +86,8 @@ from vllm.utils.math_utils import cdiv
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
from vllm.v1.worker.ubatching import dbo_current_ubatch_id
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class DeepseekV4MLP(nn.Module):
|
||||
def __init__(
|
||||
@@ -165,7 +169,7 @@ def make_deepseek_v4_expert_params_mapping(
|
||||
|
||||
|
||||
class DeepseekV4MegaMoEExperts(nn.Module):
|
||||
_symm_buffer_cache: dict[tuple[int, int, int, int, int, int, int], object] = {}
|
||||
_symm_buffer_cache: dict[tuple[int, int, int, int, int, int, int, int], object] = {}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -177,6 +181,7 @@ class DeepseekV4MegaMoEExperts(nn.Module):
|
||||
top_k: int,
|
||||
hidden_size: int,
|
||||
intermediate_size: int,
|
||||
num_shared_experts: int = 0,
|
||||
prefix: str = "",
|
||||
num_logical_experts: int | None = None,
|
||||
):
|
||||
@@ -190,6 +195,7 @@ class DeepseekV4MegaMoEExperts(nn.Module):
|
||||
self.top_k = top_k
|
||||
self.hidden_size = hidden_size
|
||||
self.intermediate_size = intermediate_size
|
||||
self.num_shared_experts = num_shared_experts
|
||||
self.max_num_tokens = vllm_config.scheduler_config.max_num_batched_tokens
|
||||
|
||||
self.num_logical_experts = (
|
||||
@@ -247,6 +253,12 @@ class DeepseekV4MegaMoEExperts(nn.Module):
|
||||
|
||||
self._transformed_l1_weights: tuple[torch.Tensor, torch.Tensor] | None = None
|
||||
self._transformed_l2_weights: tuple[torch.Tensor, torch.Tensor] | None = None
|
||||
self._transformed_shared_l1_weights: (
|
||||
tuple[torch.Tensor, torch.Tensor] | None
|
||||
) = None
|
||||
self._transformed_shared_l2_weights: (
|
||||
tuple[torch.Tensor, torch.Tensor] | None
|
||||
) = None
|
||||
|
||||
# Register in the static forward context so the custom-op wrapper
|
||||
# can look up this module by name from within a torch.compile graph.
|
||||
@@ -321,45 +333,220 @@ class DeepseekV4MegaMoEExperts(nn.Module):
|
||||
"to be multiples of 128."
|
||||
)
|
||||
|
||||
def finalize_weights(self) -> None:
|
||||
if self._transformed_l1_weights is not None:
|
||||
@staticmethod
|
||||
def _deep_gemm_supports_shared_experts(deep_gemm) -> bool:
|
||||
"""Check the Python API before touching a symmetric-memory group.
|
||||
|
||||
This also gives users of an older precompiled vLLM wheel a safe serial
|
||||
fallback instead of failing halfway through multi-rank buffer setup.
|
||||
"""
|
||||
try:
|
||||
buffer_params = signature(deep_gemm.get_symm_buffer_for_mega_moe).parameters
|
||||
kernel_params = signature(deep_gemm.fp8_fp4_mega_moe).parameters
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
return (
|
||||
hasattr(deep_gemm, "get_block_m_for_mega_moe")
|
||||
and hasattr(deep_gemm, "transform_weights_for_mega_moe")
|
||||
and "num_shared_experts" in buffer_params
|
||||
and "shared_l1_weights" in kernel_params
|
||||
and "shared_l2_weights" in kernel_params
|
||||
)
|
||||
|
||||
def _finalize_shared_expert_weights(
|
||||
self, deep_gemm, shared_experts: DeepseekV4MLP
|
||||
) -> None:
|
||||
gate_up = shared_experts.gate_up_proj
|
||||
down = shared_experts.down_proj
|
||||
gate_up_weight = gate_up.weight.data
|
||||
gate_up_scale = gate_up.weight_scale_inv.data
|
||||
down_weight = down.weight.data
|
||||
down_scale = down.weight_scale_inv.data
|
||||
|
||||
# MegaMoE's shared FP8 MMA consumes a 1x32 scale for every weight row,
|
||||
# while the checkpoint uses coarser block-FP8 scales (usually
|
||||
# 128x128). Build a dedicated, numerically equivalent scale view before
|
||||
# the generic linear post-load hook replaces the raw checkpoint scales
|
||||
# with its 128x128 DeepGEMM layout.
|
||||
checkpoint_scale_dtypes = (torch.float8_e8m0fnu, torch.uint8)
|
||||
if (
|
||||
gate_up_scale.dtype in checkpoint_scale_dtypes
|
||||
and down_scale.dtype in checkpoint_scale_dtypes
|
||||
):
|
||||
gate_up_scale = self._prepare_shared_expert_scale(
|
||||
deep_gemm,
|
||||
gate_up,
|
||||
gate_up_scale,
|
||||
gate_up_weight.shape[0],
|
||||
gate_up_weight.shape[1],
|
||||
)
|
||||
down_scale = self._prepare_shared_expert_scale(
|
||||
deep_gemm,
|
||||
down,
|
||||
down_scale,
|
||||
down_weight.shape[0],
|
||||
down_weight.shape[1],
|
||||
)
|
||||
|
||||
if gate_up_scale is None or down_scale is None:
|
||||
self.num_shared_experts = 0
|
||||
return
|
||||
|
||||
self._check_runtime_supported()
|
||||
shared_intermediate_size = self.intermediate_size * self.num_shared_experts
|
||||
expected_gate_up_shape = (
|
||||
2 * shared_intermediate_size,
|
||||
self.hidden_size,
|
||||
)
|
||||
expected_down_shape = (self.hidden_size, shared_intermediate_size)
|
||||
if (
|
||||
gate_up_weight.dtype != torch.float8_e4m3fn
|
||||
or down_weight.dtype != torch.float8_e4m3fn
|
||||
or gate_up_scale.dtype != torch.int32
|
||||
or down_scale.dtype != torch.int32
|
||||
or tuple(gate_up_weight.shape) != expected_gate_up_shape
|
||||
or tuple(down_weight.shape) != expected_down_shape
|
||||
):
|
||||
logger.warning(
|
||||
"Disabling native MegaMoE shared-expert fusion for %s: expected "
|
||||
"replicated block-FP8 weights with gate_up=%s, down=%s, and "
|
||||
"DeepGEMM int32 scales; got gate_up=%s/%s/%s and down=%s/%s/%s.",
|
||||
self.prefix,
|
||||
expected_gate_up_shape,
|
||||
expected_down_shape,
|
||||
tuple(gate_up_weight.shape),
|
||||
gate_up_weight.dtype,
|
||||
gate_up_scale.dtype,
|
||||
tuple(down_weight.shape),
|
||||
down_weight.dtype,
|
||||
down_scale.dtype,
|
||||
)
|
||||
self.num_shared_experts = 0
|
||||
return
|
||||
|
||||
transformed_l1, transformed_l2 = deep_gemm.transform_weights_for_mega_moe(
|
||||
(gate_up_weight, gate_up_scale),
|
||||
(down_weight, down_scale),
|
||||
)
|
||||
# L1 interleaving allocates a full copy. Re-home the loader Parameter on
|
||||
# that storage so the original 2*intermediate*hidden FP8 tensor can be
|
||||
# released instead of adding roughly 0.7 GiB per rank on DSV4-Flash.
|
||||
# The generic linear post-load hook may still repack the serial scales,
|
||||
# but this shared MLP is never called after native fusion is enabled.
|
||||
gate_up.weight.data = transformed_l1[0]
|
||||
self._transformed_shared_l1_weights = (
|
||||
gate_up.weight.data,
|
||||
transformed_l1[1],
|
||||
)
|
||||
self._transformed_shared_l2_weights = transformed_l2
|
||||
|
||||
def _prepare_shared_expert_scale(
|
||||
self,
|
||||
deep_gemm,
|
||||
linear: nn.Module,
|
||||
scale: torch.Tensor,
|
||||
mn: int,
|
||||
k: int,
|
||||
) -> torch.Tensor | None:
|
||||
block_size = getattr(linear, "weight_block_size", None)
|
||||
if block_size is None or len(block_size) != 2:
|
||||
logger.warning(
|
||||
"Disabling native MegaMoE shared-expert fusion for %s: "
|
||||
"shared FP8 weight block size is unavailable.",
|
||||
self.prefix,
|
||||
)
|
||||
return None
|
||||
|
||||
block_m, block_k = block_size
|
||||
expected_shape = (
|
||||
(mn + block_m - 1) // block_m,
|
||||
(k + block_k - 1) // block_k,
|
||||
)
|
||||
if block_k % 32 != 0 or tuple(scale.shape) != expected_shape:
|
||||
logger.warning(
|
||||
"Disabling native MegaMoE shared-expert fusion for %s: "
|
||||
"cannot convert shared scale shape %s with block size %s "
|
||||
"to MegaMoE's 1x32 layout for weight (%d, %d).",
|
||||
self.prefix,
|
||||
tuple(scale.shape),
|
||||
tuple(block_size),
|
||||
mn,
|
||||
k,
|
||||
)
|
||||
return None
|
||||
|
||||
scale_fp32 = self._ue8m0_uint8_to_float(scale.view(torch.uint8))
|
||||
scale_1x32 = (
|
||||
scale_fp32.repeat_interleave(block_m, dim=0)
|
||||
.repeat_interleave(block_k // 32, dim=1)[:mn, : k // 32]
|
||||
.contiguous()
|
||||
)
|
||||
# The grouped API is used with a singleton dimension to request the
|
||||
# MN-major, TMA-aligned packed-UE8M0 strides, then squeezed back to the
|
||||
# 2D layout required for a shared expert.
|
||||
return deep_gemm.transform_sf_into_required_layout(
|
||||
scale_1x32.unsqueeze(0),
|
||||
mn,
|
||||
k,
|
||||
(1, 32),
|
||||
1,
|
||||
).squeeze(0)
|
||||
|
||||
def finalize_weights(self, shared_experts: DeepseekV4MLP | None = None) -> None:
|
||||
from vllm.utils.deep_gemm import _import_deep_gemm
|
||||
|
||||
deep_gemm = _import_deep_gemm()
|
||||
|
||||
w13_scale = deep_gemm.transform_sf_into_required_layout(
|
||||
self._ue8m0_uint8_to_float(self.w13_weight_scale.data).contiguous(),
|
||||
2 * self.intermediate_size,
|
||||
self.hidden_size,
|
||||
(1, 32),
|
||||
self.num_local_experts,
|
||||
)
|
||||
w2_scale = deep_gemm.transform_sf_into_required_layout(
|
||||
self._ue8m0_uint8_to_float(self.w2_weight_scale.data).contiguous(),
|
||||
self.hidden_size,
|
||||
self.intermediate_size,
|
||||
(1, 32),
|
||||
self.num_local_experts,
|
||||
)
|
||||
self._transformed_l1_weights, self._transformed_l2_weights = (
|
||||
deep_gemm.transform_weights_for_mega_moe(
|
||||
(self.w13_weight.data.view(torch.int8).contiguous(), w13_scale),
|
||||
(self.w2_weight.data.view(torch.int8).contiguous(), w2_scale),
|
||||
if self._transformed_l1_weights is None:
|
||||
self._check_runtime_supported()
|
||||
w13_scale = deep_gemm.transform_sf_into_required_layout(
|
||||
self._ue8m0_uint8_to_float(self.w13_weight_scale.data).contiguous(),
|
||||
2 * self.intermediate_size,
|
||||
self.hidden_size,
|
||||
(1, 32),
|
||||
self.num_local_experts,
|
||||
)
|
||||
)
|
||||
# Drop the original loader-side parameters: the MegaMoE kernels only
|
||||
# consume the transformed views above. transform_weights_for_mega_moe
|
||||
# allocates a fresh tensor for the L1 weight (see _interleave_l1_weights)
|
||||
# and fresh SF tensors for L1/L2; the L2 weight is the only tensor that
|
||||
# aliases the original storage, and _transformed_l2_weights still holds
|
||||
# it, so the storage stays live after we drop the Parameter.
|
||||
self.w13_weight = None
|
||||
self.w13_weight_scale = None
|
||||
self.w2_weight = None
|
||||
self.w2_weight_scale = None
|
||||
w2_scale = deep_gemm.transform_sf_into_required_layout(
|
||||
self._ue8m0_uint8_to_float(self.w2_weight_scale.data).contiguous(),
|
||||
self.hidden_size,
|
||||
self.intermediate_size,
|
||||
(1, 32),
|
||||
self.num_local_experts,
|
||||
)
|
||||
self._transformed_l1_weights, self._transformed_l2_weights = (
|
||||
deep_gemm.transform_weights_for_mega_moe(
|
||||
(self.w13_weight.data.view(torch.int8).contiguous(), w13_scale),
|
||||
(self.w2_weight.data.view(torch.int8).contiguous(), w2_scale),
|
||||
)
|
||||
)
|
||||
# Drop the original loader-side parameters: the MegaMoE kernels only
|
||||
# consume the transformed views above. transform_weights_for_mega_moe
|
||||
# allocates a fresh tensor for the L1 weight (see
|
||||
# _interleave_l1_weights) and fresh SF tensors for L1/L2; the L2
|
||||
# weight is the only tensor that aliases the original storage, and
|
||||
# _transformed_l2_weights still holds it, so the storage stays live
|
||||
# after we drop the Parameter.
|
||||
self.w13_weight = None
|
||||
self.w13_weight_scale = None
|
||||
self.w2_weight = None
|
||||
self.w2_weight_scale = None
|
||||
|
||||
if shared_experts is None or self.num_shared_experts == 0:
|
||||
return
|
||||
if self._transformed_shared_l1_weights is not None:
|
||||
return
|
||||
if not self._deep_gemm_supports_shared_experts(deep_gemm):
|
||||
logger.warning_once(
|
||||
"Disabling native MegaMoE shared-expert fusion because the "
|
||||
"installed DeepGEMM Python API is older than the vLLM "
|
||||
"source. Rebuild the vendored _deep_gemm_C extension to enable it.",
|
||||
)
|
||||
self.num_shared_experts = 0
|
||||
return
|
||||
self._finalize_shared_expert_weights(deep_gemm, shared_experts)
|
||||
|
||||
@property
|
||||
def has_fused_shared_experts(self) -> bool:
|
||||
return self._transformed_shared_l1_weights is not None
|
||||
|
||||
def get_symm_buffer(self):
|
||||
from vllm.utils.deep_gemm import _import_deep_gemm
|
||||
@@ -376,6 +563,7 @@ class DeepseekV4MegaMoEExperts(nn.Module):
|
||||
self.top_k,
|
||||
self.hidden_size,
|
||||
self.intermediate_size,
|
||||
self.num_shared_experts if self.has_fused_shared_experts else 0,
|
||||
)
|
||||
symm_buffer = self._symm_buffer_cache.get(key)
|
||||
if symm_buffer is None:
|
||||
@@ -386,6 +574,9 @@ class DeepseekV4MegaMoEExperts(nn.Module):
|
||||
self.top_k,
|
||||
self.hidden_size,
|
||||
self.intermediate_size,
|
||||
num_shared_experts=(
|
||||
self.num_shared_experts if self.has_fused_shared_experts else 0
|
||||
),
|
||||
)
|
||||
self._symm_buffer_cache[key] = symm_buffer
|
||||
return symm_buffer
|
||||
@@ -492,6 +683,19 @@ class DeepseekV4MegaMoEExperts(nn.Module):
|
||||
else None,
|
||||
)
|
||||
|
||||
shared_x_sf = None
|
||||
shared_block_m = None
|
||||
if self.has_fused_shared_experts:
|
||||
shared_x_sf = symm_buffer.shared_l1_acts_sf
|
||||
shared_block_m = deep_gemm.get_block_m_for_mega_moe(
|
||||
get_ep_group().world_size,
|
||||
self.num_experts,
|
||||
symm_buffer.num_max_tokens_per_rank,
|
||||
num_tokens,
|
||||
self.top_k,
|
||||
"fp8xfp4",
|
||||
)
|
||||
|
||||
prepare_megamoe_inputs(
|
||||
hidden_states,
|
||||
topk_weights,
|
||||
@@ -501,18 +705,32 @@ class DeepseekV4MegaMoEExperts(nn.Module):
|
||||
symm_buffer.topk_idx[:num_tokens],
|
||||
symm_buffer.topk_weights[:num_tokens],
|
||||
is_padding=is_padding,
|
||||
shared_x_sf=shared_x_sf,
|
||||
shared_block_m=shared_block_m,
|
||||
)
|
||||
|
||||
assert self._transformed_l1_weights is not None
|
||||
assert self._transformed_l2_weights is not None
|
||||
deep_gemm.fp8_fp4_mega_moe(
|
||||
y,
|
||||
self._transformed_l1_weights,
|
||||
self._transformed_l2_weights,
|
||||
symm_buffer,
|
||||
activation_clamp=activation_clamp,
|
||||
fast_math=fast_math,
|
||||
)
|
||||
if self.has_fused_shared_experts:
|
||||
deep_gemm.fp8_fp4_mega_moe(
|
||||
y,
|
||||
self._transformed_l1_weights,
|
||||
self._transformed_l2_weights,
|
||||
symm_buffer,
|
||||
shared_l1_weights=self._transformed_shared_l1_weights,
|
||||
shared_l2_weights=self._transformed_shared_l2_weights,
|
||||
activation_clamp=activation_clamp,
|
||||
fast_math=fast_math,
|
||||
)
|
||||
else:
|
||||
deep_gemm.fp8_fp4_mega_moe(
|
||||
y,
|
||||
self._transformed_l1_weights,
|
||||
self._transformed_l2_weights,
|
||||
symm_buffer,
|
||||
activation_clamp=activation_clamp,
|
||||
fast_math=fast_math,
|
||||
)
|
||||
return y
|
||||
|
||||
|
||||
@@ -645,6 +863,16 @@ class DeepseekV4MoE(nn.Module):
|
||||
self.experts_start_idx = self.physical_expert_start
|
||||
self.experts_end_idx = self.physical_expert_end
|
||||
|
||||
# Native DeepGEMM fusion requires each EP rank to own the complete
|
||||
# shared MLP. Sequence parallel replicates those weights while sharding
|
||||
# tokens. TP=1 is also naturally replicated. With PP+TP the shared MLP
|
||||
# remains tensor-sharded, so retain the serial path.
|
||||
fuse_shared_experts = bool(
|
||||
self.shared_experts is not None
|
||||
and not envs.VLLM_DISABLE_DSV4_MEGAMOE_SHARED_EXPERT_FUSION
|
||||
and (self.use_sequence_parallel or self.tp_size == 1)
|
||||
)
|
||||
|
||||
self.experts = DeepseekV4MegaMoEExperts(
|
||||
vllm_config,
|
||||
num_experts=self.n_physical_experts,
|
||||
@@ -654,6 +882,7 @@ class DeepseekV4MoE(nn.Module):
|
||||
top_k=config.num_experts_per_tok,
|
||||
hidden_size=config.hidden_size,
|
||||
intermediate_size=config.moe_intermediate_size,
|
||||
num_shared_experts=(self.n_shared_experts if fuse_shared_experts else 0),
|
||||
prefix=f"{prefix}.experts",
|
||||
)
|
||||
|
||||
@@ -739,7 +968,10 @@ class DeepseekV4MoE(nn.Module):
|
||||
activation_clamp=activation_clamp,
|
||||
)
|
||||
|
||||
if self.shared_experts is not None:
|
||||
if (
|
||||
self.shared_experts is not None
|
||||
and not self.experts.has_fused_shared_experts
|
||||
):
|
||||
shared_output = self.shared_experts(hidden_states)
|
||||
final_hidden_states += shared_output
|
||||
|
||||
@@ -759,7 +991,7 @@ class DeepseekV4MoE(nn.Module):
|
||||
|
||||
def finalize_mega_moe_weights(self) -> None:
|
||||
if self.use_mega_moe:
|
||||
self.experts.finalize_weights()
|
||||
self.experts.finalize_weights(self.shared_experts)
|
||||
|
||||
|
||||
def _select_dsv4_attn_cls(vllm_config: VllmConfig) -> type[DeepseekV4Attention]:
|
||||
|
||||
@@ -17,6 +17,7 @@ def _prepare_megamoe_inputs_kernel(
|
||||
hidden_states,
|
||||
x_fp8,
|
||||
x_sf,
|
||||
shared_x_sf,
|
||||
topk_ids,
|
||||
topk_weights,
|
||||
is_padding,
|
||||
@@ -28,6 +29,8 @@ def _prepare_megamoe_inputs_kernel(
|
||||
x_stride_k: tl.constexpr,
|
||||
x_sf_stride_m: tl.constexpr,
|
||||
x_sf_stride_k: tl.constexpr,
|
||||
shared_x_sf_stride_m: tl.constexpr,
|
||||
shared_x_sf_stride_k: tl.constexpr,
|
||||
topk_ids_stride_m: tl.constexpr,
|
||||
topk_ids_stride_k: tl.constexpr,
|
||||
topk_weights_stride_m: tl.constexpr,
|
||||
@@ -42,6 +45,7 @@ def _prepare_megamoe_inputs_kernel(
|
||||
BLOCK_K: tl.constexpr,
|
||||
GROUP_K: tl.constexpr,
|
||||
BLOCK_TOPK: tl.constexpr,
|
||||
SHARED_BLOCK_M: tl.constexpr,
|
||||
) -> None:
|
||||
token_id = tl.program_id(0)
|
||||
k_block_id = tl.program_id(1)
|
||||
@@ -84,6 +88,25 @@ def _prepare_megamoe_inputs_kernel(
|
||||
packed_scale,
|
||||
)
|
||||
|
||||
# DeepGEMM's SM100 shared-expert TMA loads require the activation scales
|
||||
# in an MN-major layout whose row permutation depends on the MegaMoE
|
||||
# scheduler's runtime BLOCK_M. Write that view while the packed UE8M0 scale
|
||||
# is already resident, avoiding another kernel and temporary tensor.
|
||||
if shared_x_sf is not None:
|
||||
m_block_id = token_id // SHARED_BLOCK_M
|
||||
m_in_block = token_id % SHARED_BLOCK_M
|
||||
aligned_block_m: tl.constexpr = triton.cdiv(SHARED_BLOCK_M, 128) * 128
|
||||
transposed_m = (
|
||||
(m_in_block // 128) * 128 + (m_in_block % 32) * 4 + (m_in_block % 128) // 32
|
||||
)
|
||||
shared_row = m_block_id * aligned_block_m + transposed_m
|
||||
tl.store(
|
||||
shared_x_sf
|
||||
+ shared_row * shared_x_sf_stride_m
|
||||
+ k_block_id * shared_x_sf_stride_k,
|
||||
packed_scale,
|
||||
)
|
||||
|
||||
if k_block_id == 0:
|
||||
topk_offsets = tl.arange(0, BLOCK_TOPK)
|
||||
topk_mask = topk_offsets < top_k
|
||||
@@ -131,6 +154,8 @@ def prepare_megamoe_inputs(
|
||||
topk_idx_out: torch.Tensor,
|
||||
topk_weights_out: torch.Tensor,
|
||||
is_padding: torch.Tensor | None = None,
|
||||
shared_x_sf: torch.Tensor | None = None,
|
||||
shared_block_m: int | None = None,
|
||||
) -> None:
|
||||
num_tokens, hidden_size = hidden_states.shape
|
||||
if num_tokens == 0:
|
||||
@@ -146,6 +171,28 @@ def prepare_megamoe_inputs(
|
||||
"DeepSeek V4 MegaMoE input staging requires topk_weights and "
|
||||
"topk_ids to have the same shape."
|
||||
)
|
||||
if (shared_x_sf is None) != (shared_block_m is None):
|
||||
raise ValueError(
|
||||
"DeepSeek V4 MegaMoE shared input staging requires both "
|
||||
"shared_x_sf and shared_block_m."
|
||||
)
|
||||
if shared_x_sf is not None:
|
||||
assert shared_block_m is not None
|
||||
if shared_block_m <= 0:
|
||||
raise ValueError("MegaMoE shared_block_m must be positive.")
|
||||
expected_sf_k = hidden_size // 128
|
||||
if shared_x_sf.ndim != 2 or shared_x_sf.shape[1] != expected_sf_k:
|
||||
raise ValueError(
|
||||
"MegaMoE shared_x_sf must have shape "
|
||||
f"(*, {expected_sf_k}), got {tuple(shared_x_sf.shape)}."
|
||||
)
|
||||
aligned_block_m = triton.cdiv(shared_block_m, 128) * 128
|
||||
required_rows = triton.cdiv(num_tokens, shared_block_m) * aligned_block_m
|
||||
if shared_x_sf.shape[0] < required_rows:
|
||||
raise ValueError(
|
||||
"MegaMoE shared_x_sf has insufficient rows: requires "
|
||||
f"{required_rows}, got {shared_x_sf.shape[0]}."
|
||||
)
|
||||
|
||||
block_k = 128
|
||||
grid = (num_tokens, triton.cdiv(hidden_size, block_k))
|
||||
@@ -155,6 +202,7 @@ def prepare_megamoe_inputs(
|
||||
hidden_states,
|
||||
x_fp8,
|
||||
x_sf,
|
||||
shared_x_sf,
|
||||
topk_ids,
|
||||
topk_weights,
|
||||
is_padding,
|
||||
@@ -166,6 +214,8 @@ def prepare_megamoe_inputs(
|
||||
x_fp8.stride(1),
|
||||
x_sf.stride(0),
|
||||
x_sf.stride(1),
|
||||
shared_x_sf.stride(0) if shared_x_sf is not None else 0,
|
||||
shared_x_sf.stride(1) if shared_x_sf is not None else 0,
|
||||
topk_ids.stride(0),
|
||||
topk_ids.stride(1),
|
||||
topk_weights.stride(0),
|
||||
@@ -180,5 +230,6 @@ def prepare_megamoe_inputs(
|
||||
BLOCK_K=block_k,
|
||||
GROUP_K=32,
|
||||
BLOCK_TOPK=block_topk,
|
||||
SHARED_BLOCK_M=shared_block_m or 1,
|
||||
num_warps=4,
|
||||
)
|
||||
|
||||
@@ -93,7 +93,10 @@ from vllm.models.common.ops.sequence_parallel import (
|
||||
sp_reduce_scatter,
|
||||
sp_shard,
|
||||
)
|
||||
from vllm.models.deepseek_v4.nvidia.model import DeepseekV4MegaMoEExperts
|
||||
from vllm.models.deepseek_v4.nvidia.model import (
|
||||
DeepseekV4MegaMoEExperts,
|
||||
DeepseekV4MLP,
|
||||
)
|
||||
from vllm.models.deepseek_v4.nvidia.ops.prepare_megamoe import prepare_megamoe_inputs
|
||||
from vllm.models.kimi_k3.nvidia.kda import KimiK3DeltaAttention
|
||||
from vllm.models.kimi_k3.nvidia.latent_moe_runner import (
|
||||
@@ -354,7 +357,7 @@ class KimiK3MegaMoEExperts(DeepseekV4MegaMoEExperts):
|
||||
torch.distributed.barrier(group=ep_group.cpu_group)
|
||||
self._synchronized_ep_groups.add(key)
|
||||
|
||||
def finalize_weights(self) -> None:
|
||||
def finalize_weights(self, shared_experts: DeepseekV4MLP | None = None) -> None:
|
||||
if self._transformed_l1_weights is not None:
|
||||
return
|
||||
|
||||
|
||||
Reference in New Issue
Block a user