Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
419 lines
16 KiB
Python
419 lines
16 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
from enum import IntEnum
|
|
from typing import cast
|
|
|
|
import torch
|
|
|
|
import vllm.envs as envs
|
|
from vllm.config import get_current_vllm_config
|
|
from vllm.distributed import (
|
|
get_tensor_model_parallel_rank,
|
|
tensor_model_parallel_all_reduce,
|
|
)
|
|
from vllm.logger import init_logger
|
|
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
|
|
from vllm.model_executor.layers.fused_moe.runner.moe_runner import MoERunner, _unpack
|
|
from vllm.model_executor.layers.layernorm import RMSNorm
|
|
from vllm.platforms import current_platform
|
|
from vllm.utils.multi_stream_utils import maybe_execute_in_parallel
|
|
from vllm.utils.torch_utils import aux_stream
|
|
|
|
logger = init_logger(__name__)
|
|
|
|
|
|
class LatentTailTier(IntEnum):
|
|
"""Which tail implementation the fused path runs, by token count.
|
|
|
|
The tiers share the same replicated up-projection weight, so the choice is
|
|
per batch and needs no weight relayout: tiers 0 and 2 read it as a
|
|
column-parallel row-shard, tier 1 reads it whole.
|
|
"""
|
|
|
|
# ``_small_batch_tail``. Decode-sized (<= the op's max_num_tokens): one
|
|
# CuTeDSL collective fuses the latent reduce, RMSNorm and the shared
|
|
# reduce-scatter, then a sharded up-projection multicasts through a Lamport
|
|
# copy. Requires SM100, TP 8/16, BF16
|
|
TAIL_FUSION = 0
|
|
|
|
# ``_overlap_allreduce_tail``. The portable default, up to
|
|
# VLLM_SHARED_EXPERTS_STREAM_TOKEN_THRESHOLD tokens: reduce the latent,
|
|
# up-project the full hidden dim from the replicated weight, and add the
|
|
# separately reduced shared output, hiding that shared all-reduce behind the
|
|
# up-projection GEMM on the aux stream.
|
|
ALLREDUCE_OVERLAP = 1
|
|
|
|
# ``_shard_up_proj_tail``. Prefill-sized: each rank up-projects only its
|
|
# hidden shard and accumulates into the shared partial, so the shared
|
|
# all-reduce also stitches the routed shards. Same two all-reduces as tier 1
|
|
# at 1/tp of the up-projection FLOPs; it gives up tier 1's aux-stream
|
|
# overlap, since the reduce now has to follow the accumulate.
|
|
COLUMN_PARALLEL = 2
|
|
|
|
|
|
class LatentMoERunner(MoERunner):
|
|
"""MoE runner for latent MoE with a replicated routed up-projection.
|
|
|
|
The fused path (tp>1, un-reduced combine output, shared expert, no SP)
|
|
dispatches over ``LatentTailTier`` by token count; see that enum for what
|
|
each tier does and when it applies.
|
|
|
|
Native path: the replicated up-proj produces the full hidden dim on every
|
|
rank, so the base runner combines routed + shared correctly at any TP size.
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
*args,
|
|
**kwargs,
|
|
) -> None:
|
|
super().__init__(*args, **kwargs)
|
|
|
|
# The tail-fusion kernels are tcgen05-based, so they require an
|
|
# SM100 NVIDIA device; the runner falls back to the default latent
|
|
# MoE path everywhere else.
|
|
self.enable_k3_latent_moe_tail_fusion = (
|
|
current_platform.is_cuda()
|
|
and current_platform.is_device_capability_family(100)
|
|
)
|
|
# Overlap the shared-expert all-reduce with the tier-1 up-projection.
|
|
self._shared_ar_events = (torch.cuda.Event(), torch.cuda.Event())
|
|
use_fused_path = self._use_fused_path()
|
|
if (
|
|
self.enable_k3_latent_moe_tail_fusion
|
|
and use_fused_path
|
|
and self.moe_config.tp_size not in (8, 16)
|
|
):
|
|
logger.warning_once(
|
|
"K3 latent-MoE tail fusion currently supports TP=8 and TP=16, "
|
|
"but TP=%d is configured. Falling back to the default path.",
|
|
self.moe_config.tp_size,
|
|
)
|
|
self.enable_k3_latent_moe_tail_fusion = False
|
|
|
|
if self.enable_k3_latent_moe_tail_fusion and use_fused_path:
|
|
vllm_config = get_current_vllm_config()
|
|
if vllm_config.parallel_config.use_ubatching:
|
|
raise ValueError(
|
|
"K3 latent-MoE tail fusion does not support DBO or ubatching."
|
|
)
|
|
if vllm_config.model_config.enable_sleep_mode:
|
|
raise ValueError(
|
|
"K3 latent-MoE tail fusion does not support sleep mode."
|
|
)
|
|
transform = self.routed_output_transform
|
|
assert transform is not None
|
|
norm = transform.norm
|
|
assert norm is not None
|
|
from vllm.model_executor.layers.fused_moe.experts.trtllm_mxfp4_moe import (
|
|
TrtLlmMxfp4ExpertsMonolithic,
|
|
)
|
|
from vllm.model_executor.layers.fused_moe.experts.trtllm_nvfp4_moe import (
|
|
TrtLlmNvFp4ExpertsMonolithic,
|
|
)
|
|
|
|
experts_cls = getattr(self._quant_method, "experts_cls", None)
|
|
# The kernel instance is built after weight loading, while the tail
|
|
# must register its CuTeDSL warmup units during runner construction.
|
|
if (
|
|
experts_cls is TrtLlmMxfp4ExpertsMonolithic
|
|
or experts_cls is TrtLlmNvFp4ExpertsMonolithic
|
|
) and self.moe_config.hidden_dim == self.moe_config.hidden_dim_unpadded:
|
|
self.moe_config.defer_moe_finalize()
|
|
from vllm.models.kimi_k3.nvidia.ops.latent_moe_tail import (
|
|
KimiK3LatentMoETailOp,
|
|
)
|
|
|
|
op = KimiK3LatentMoETailOp.initialize(
|
|
hidden_size=transform.up_proj.weight.shape[0],
|
|
latent_size=norm.weight.shape[0],
|
|
experts_per_token=(
|
|
self.moe_config.experts_per_token
|
|
if self.moe_config.use_deferred_moe_finalize
|
|
else 0
|
|
),
|
|
dtype=norm.weight.dtype,
|
|
device=current_platform.current_device(),
|
|
rms_eps=norm.variance_epsilon,
|
|
)
|
|
self._k3_latent_moe_tail_op = op
|
|
if self.moe_config.use_deferred_moe_finalize:
|
|
self.moe_config.limit_deferred_moe_finalize(op.contract.max_num_tokens)
|
|
logger.info_once(
|
|
"K3 latent-MoE tail fusion with deferred top-k finalization "
|
|
"is enabled for up to %d tokens.",
|
|
op.contract.max_num_tokens,
|
|
)
|
|
|
|
def _get_zero_residual(
|
|
self,
|
|
hidden_states: torch.Tensor,
|
|
max_token_num: int,
|
|
) -> torch.Tensor:
|
|
"""Read-only zero ``residual_in`` for the fused AR+RMSNorm kernel.
|
|
|
|
flashinfer requires a residual buffer even when there is no residual to
|
|
add.
|
|
"""
|
|
buf = getattr(self, "_zero_residual", None)
|
|
if buf is None:
|
|
buf = torch.zeros(
|
|
max_token_num * hidden_states.shape[-1],
|
|
dtype=hidden_states.dtype,
|
|
device=hidden_states.device,
|
|
)
|
|
self._zero_residual = buf
|
|
|
|
assert buf.dtype == hidden_states.dtype
|
|
assert buf.device == hidden_states.device
|
|
assert hidden_states.numel() <= buf.numel()
|
|
|
|
return buf[: hidden_states.numel()].view_as(hidden_states)
|
|
|
|
def _use_fused_path(self) -> bool:
|
|
# The fused path merges the latent and shared reductions into one
|
|
# all-reduce, so it needs actual TP parallelism, a shared expert (to
|
|
# concat), an un-reduced combine output, and no sequence parallelism.
|
|
return (
|
|
self.moe_config.tp_size > 1
|
|
and self._shared_experts is not None
|
|
and not self._fused_output_is_reduced
|
|
and not self.moe_config.is_sequence_parallel
|
|
)
|
|
|
|
def _select_tail_tier(
|
|
self,
|
|
fused_output: torch.Tensor,
|
|
shared_output: torch.Tensor,
|
|
) -> LatentTailTier:
|
|
num_tokens = fused_output.shape[0]
|
|
# tier 0
|
|
if self.enable_k3_latent_moe_tail_fusion and (
|
|
0 < num_tokens <= self._k3_latent_moe_tail_op.contract.max_num_tokens
|
|
):
|
|
return LatentTailTier.TAIL_FUSION
|
|
|
|
transform = self.routed_output_transform
|
|
assert transform is not None
|
|
# tier 1
|
|
if (
|
|
num_tokens <= envs.VLLM_SHARED_EXPERTS_STREAM_TOKEN_THRESHOLD
|
|
and not envs.VLLM_DISABLE_SHARED_EXPERTS_STREAM
|
|
):
|
|
return LatentTailTier.ALLREDUCE_OVERLAP
|
|
# tier 2
|
|
return LatentTailTier.COLUMN_PARALLEL
|
|
|
|
def _small_batch_tail(
|
|
self,
|
|
fused_output: torch.Tensor | UnfinalizedMoEOutput,
|
|
shared_output: torch.Tensor,
|
|
trunc_size: int | None,
|
|
) -> torch.Tensor:
|
|
"""Tier 0: the CuTeDSL operator fuses the whole tail."""
|
|
transform = self.routed_output_transform
|
|
assert transform is not None
|
|
norm = transform.norm
|
|
assert norm is not None
|
|
|
|
result = self._k3_latent_moe_tail_op(
|
|
fused_output,
|
|
shared_output,
|
|
norm.weight,
|
|
transform.up_proj.weight,
|
|
)
|
|
# The operator already reduced; this only strips padding.
|
|
return self._maybe_reduce_final_output(
|
|
result, trunc_size, output_is_reduced=True
|
|
)
|
|
|
|
def _overlap_allreduce_tail(
|
|
self,
|
|
fused_output: torch.Tensor,
|
|
shared_output: torch.Tensor,
|
|
trunc_size: int | None,
|
|
) -> torch.Tensor:
|
|
"""Tier 1: reduce the latent, up-project the full hidden dim from the
|
|
replicated weight, and add the separately reduced shared output.
|
|
|
|
Small enough batches hide that shared all-reduce behind the up-projection
|
|
GEMM on the aux stream.
|
|
"""
|
|
transform = self.routed_output_transform
|
|
assert transform is not None
|
|
assert shared_output.size(0) <= envs.VLLM_SHARED_EXPERTS_STREAM_TOKEN_THRESHOLD
|
|
if transform.norm is not None:
|
|
fused_latent = self.allreduce_norm_latent_out(fused_output, transform.norm)
|
|
else:
|
|
fused_latent = tensor_model_parallel_all_reduce(fused_output)
|
|
|
|
# Overlap the shared-expert all-reduce with the up-projection GEMM while
|
|
# the batch is small enough for it to pay off.
|
|
result, shared_output = maybe_execute_in_parallel(
|
|
lambda: torch.mm(fused_latent, transform.up_proj.weight.t()),
|
|
lambda: tensor_model_parallel_all_reduce(shared_output),
|
|
self._shared_ar_events[0],
|
|
self._shared_ar_events[1],
|
|
aux_stream(),
|
|
)
|
|
result.add_(shared_output)
|
|
|
|
# Output is already fully reduced; this only strips padding.
|
|
return self._maybe_reduce_final_output(
|
|
result, trunc_size, output_is_reduced=True
|
|
)
|
|
|
|
def _shard_up_proj_tail(
|
|
self,
|
|
fused_output: torch.Tensor,
|
|
shared_output: torch.Tensor,
|
|
trunc_size: int | None,
|
|
) -> torch.Tensor:
|
|
"""Tier 2: column-parallel up-projection folded into the final reduce."""
|
|
transform = self.routed_output_transform
|
|
assert transform is not None
|
|
|
|
if transform.norm is not None:
|
|
latent = self.allreduce_norm_latent_out(fused_output, transform.norm)
|
|
else:
|
|
latent = tensor_model_parallel_all_reduce(fused_output)
|
|
|
|
weight = transform.up_proj.weight
|
|
shard_size = weight.shape[0] // self.moe_config.tp_size
|
|
shard_start = get_tensor_model_parallel_rank() * shard_size
|
|
|
|
# column-parallel
|
|
up_proj_shard = weight.narrow(0, shard_start, shard_size)
|
|
hidden_shard = shared_output.narrow(-1, shard_start, shard_size)
|
|
|
|
# hidden_shard += latent @ up_proj_shard.T, accumulated in the GEMM's
|
|
# beta-add epilogue so folding in the shared partial costs no kernel.
|
|
hidden_shard.addmm_(latent, up_proj_shard.t())
|
|
|
|
return self._maybe_reduce_final_output(
|
|
shared_output, trunc_size, output_is_reduced=False
|
|
)
|
|
|
|
def forward(
|
|
self,
|
|
hidden_states: torch.Tensor,
|
|
router_logits: torch.Tensor,
|
|
input_ids: torch.Tensor | None = None,
|
|
shared_experts_input: torch.Tensor | None = None,
|
|
) -> torch.Tensor:
|
|
if self._use_fused_path():
|
|
return self._fused_forward(
|
|
hidden_states, router_logits, input_ids, shared_experts_input
|
|
)
|
|
return super().forward(
|
|
hidden_states, router_logits, input_ids, shared_experts_input
|
|
)
|
|
|
|
def _fused_forward(
|
|
self,
|
|
hidden_states: torch.Tensor,
|
|
router_logits: torch.Tensor,
|
|
input_ids: torch.Tensor | None = None,
|
|
shared_experts_input: torch.Tensor | None = None,
|
|
) -> torch.Tensor:
|
|
# When the caller pre-applies the routed input transform outside the
|
|
# runner (e.g. to overlap it on a separate stream), it passes the
|
|
# already-transformed routed input as ``hidden_states`` and the original
|
|
# hidden states as ``shared_experts_input``; skip the transform then.
|
|
if shared_experts_input is None:
|
|
hidden_states, shared_experts_input = self.apply_routed_input_transform(
|
|
hidden_states
|
|
)
|
|
hidden_states, og_hidden_dim_pre_xform, og_hidden_dim_post_xform = (
|
|
self._maybe_pad_hidden_states(
|
|
shared_experts_input,
|
|
hidden_states,
|
|
)
|
|
)
|
|
|
|
if (
|
|
og_hidden_dim_pre_xform is None
|
|
and self.moe_config.should_defer_moe_finalize(hidden_states.shape[0])
|
|
):
|
|
unfinalized = self._forward_impl(
|
|
hidden_states,
|
|
router_logits,
|
|
shared_experts_input,
|
|
input_ids,
|
|
)
|
|
shared_output, fused_output = _unpack(unfinalized)
|
|
assert shared_output is not None
|
|
assert isinstance(fused_output, UnfinalizedMoEOutput)
|
|
result = self._small_batch_tail(fused_output, shared_output, None)
|
|
return self._maybe_add_zero_expert_output(result)
|
|
|
|
result = self._forward_entry(
|
|
hidden_states,
|
|
router_logits,
|
|
shared_experts_input,
|
|
input_ids,
|
|
self._encode_layer_name(),
|
|
self.moe_config.hidden_dim_unpadded
|
|
if self._quant_method.has_unpadded_output
|
|
else 0,
|
|
)
|
|
|
|
shared_output, fused_output = cast(tuple[torch.Tensor, torch.Tensor], result)
|
|
|
|
if og_hidden_dim_pre_xform is not None:
|
|
fused_output = fused_output[..., :og_hidden_dim_pre_xform]
|
|
|
|
tier = self._select_tail_tier(fused_output, shared_output)
|
|
if tier is LatentTailTier.TAIL_FUSION:
|
|
latent_tail = self._small_batch_tail
|
|
elif tier is LatentTailTier.ALLREDUCE_OVERLAP:
|
|
latent_tail = self._overlap_allreduce_tail
|
|
else:
|
|
latent_tail = self._shard_up_proj_tail
|
|
|
|
result = latent_tail(fused_output, shared_output, og_hidden_dim_post_xform)
|
|
|
|
return self._maybe_add_zero_expert_output(result)
|
|
|
|
def allreduce_norm_latent_out(
|
|
self,
|
|
hidden_states: torch.Tensor,
|
|
norm: RMSNorm,
|
|
) -> tuple[torch.Tensor, torch.Tensor]:
|
|
"""All-reduce + add residual + (standard) RMSNorm, fused via flashinfer."""
|
|
from vllm.model_executor.layers.fused_allreduce_gemma_rms_norm import (
|
|
_AR_RESIDUAL_RMS_NORM,
|
|
_can_use_flashinfer,
|
|
flashinfer_trtllm_fused_allreduce_norm,
|
|
)
|
|
|
|
if self.moe_config.tp_size == 1:
|
|
return norm(hidden_states)
|
|
|
|
if flashinfer_trtllm_fused_allreduce_norm is not None:
|
|
ok, max_token_num = _can_use_flashinfer(
|
|
hidden_states, self.moe_config.tp_size
|
|
)
|
|
if ok:
|
|
norm_out = torch.empty_like(hidden_states)
|
|
# With norm_out provided, the kernel writes the new residual
|
|
# (all_reduce(hidden_states) + residual) into the hidden_states
|
|
# buffer and the normalized result into norm_out.
|
|
flashinfer_trtllm_fused_allreduce_norm(
|
|
allreduce_in=hidden_states,
|
|
residual=self._get_zero_residual(hidden_states, max_token_num),
|
|
rms_gamma=norm.weight,
|
|
rms_eps=norm.variance_epsilon,
|
|
world_size=self.moe_config.tp_size,
|
|
weight_bias=0.0,
|
|
launch_with_pdl=True,
|
|
fp32_acc=True,
|
|
max_token_num=max_token_num,
|
|
pattern_code=_AR_RESIDUAL_RMS_NORM,
|
|
norm_out=norm_out,
|
|
)
|
|
return norm_out
|
|
|
|
reduced = tensor_model_parallel_all_reduce(hidden_states)
|
|
return norm(reduced)
|