[Kimi-K3] Add option to shard the shared expert instead of replicating (#50656)

Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
This commit is contained in:
Tyler Michael Smith
2026-08-03 11:41:24 -04:00
committed by GitHub
co-authored by Claude Jee Jee Li
parent dd11df04f3
commit 5df9999fcf
3 changed files with 154 additions and 3 deletions
+62 -3
View File
@@ -129,7 +129,42 @@ logger = init_logger(__name__)
_ROUTED_DOWN_PROJ_STREAM_TOKEN_THRESHOLD = 256
def shard_sequence_parallel_mlp(
hidden_size: int,
intermediate_size: int,
use_sequence_parallel: bool,
eligible: bool,
) -> bool:
"""Whether to TP-shard a sequence-parallel MLP instead of replicating it.
Opt-in via ``VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT``; see :class:`KimiMLP` for
the trade-off and :mod:`vllm.envs` for when it is worth enabling.
"""
enabled = envs.VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT
if not (use_sequence_parallel and eligible and enabled):
return False
tp_size = get_tensor_model_parallel_world_size()
return (
tp_size > 1 and intermediate_size % tp_size == 0 and hidden_size % tp_size == 0
)
class KimiMLP(nn.Module):
"""Dense / shared-expert MLP, optionally TP-sharded under sequence parallel.
Under sequence parallelism each rank owns a distinct slice of the tokens, so
by default both projections are replicated (``disable_tp``) and the block
needs no collective. That makes every rank stream the entire weight to serve
its own token shard.
With ``VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT`` the weights are TP-sharded
instead. A rank then holds only a slice of the intermediate dim, so it
cannot finish its own tokens alone: ``forward`` all-gathers the full token
set, computes this rank's partial, and reduce-scatters. The reduce-scatter
sums across TP and restores the sequence sharding in one collective, so the
block still ends with one collective per direction.
"""
def __init__(
self,
hidden_size: int,
@@ -138,18 +173,27 @@ class KimiMLP(nn.Module):
quant_config: QuantizationConfig | None = None,
reduce_results: bool = True,
use_sequence_parallel: bool = False,
can_shard_sequence_parallel: bool = False,
prefix: str = "",
activation_situ_beta: float | None = None,
activation_situ_linear_beta: float | None = None,
) -> None:
super().__init__()
self.shard_sequence_parallel = shard_sequence_parallel_mlp(
hidden_size,
intermediate_size,
use_sequence_parallel,
can_shard_sequence_parallel,
)
replicate = use_sequence_parallel and not self.shard_sequence_parallel
self.gate_up_proj = MergedColumnParallelLinear(
hidden_size,
[intermediate_size] * 2,
bias=False,
quant_config=quant_config,
disable_tp=use_sequence_parallel,
disable_tp=replicate,
prefix=f"{prefix}.gate_up_proj",
)
self.down_proj = RowParallelLinear(
@@ -157,8 +201,10 @@ class KimiMLP(nn.Module):
hidden_size,
bias=False,
quant_config=quant_config,
reduce_results=reduce_results,
disable_tp=use_sequence_parallel,
# Sharded sequence parallel reduces via the reduce-scatter in
# forward(), which also restores the sequence sharding.
reduce_results=False if self.shard_sequence_parallel else reduce_results,
disable_tp=replicate,
prefix=f"{prefix}.down_proj",
)
if hidden_act == "silu":
@@ -175,9 +221,17 @@ class KimiMLP(nn.Module):
)
def forward(self, x):
if self.shard_sequence_parallel:
# Each rank holds a weight shard but only its own tokens, so it
# cannot finish those tokens alone: gather the full token set,
# compute this rank's partial for all of them, then reduce-scatter,
# which sums across TP and restores the sequence sharding.
x = sp_all_gather(x)
gate_up, _ = self.gate_up_proj(x)
x = self.act_fn(gate_up)
x, _ = self.down_proj(x)
if self.shard_sequence_parallel:
x = sp_reduce_scatter(x)
return x
@@ -478,6 +532,10 @@ class KimiMoE(nn.Module):
quant_config=quant_config,
reduce_results=False,
use_sequence_parallel=use_sequence_parallel,
# Only the MegaMoE path calls the shared experts directly; the
# FusedMoE path below hands them to the runner, which fuses
# their reduction and assumes the replicated layout.
can_shard_sequence_parallel=self.use_mega_moe,
prefix=f"{prefix}.shared_experts",
activation_situ_beta=activation_situ_beta,
activation_situ_linear_beta=activation_situ_linear_beta,
@@ -788,6 +846,7 @@ class KimiDecoderLayer(nn.Module):
quant_config=quant_config,
prefix=f"{prefix}.mlp",
use_sequence_parallel=self.use_sequence_parallel,
can_shard_sequence_parallel=True,
activation_situ_beta=config.activation_situ_beta,
activation_situ_linear_beta=config.activation_situ_linear_beta,
)