[Kimi-K3] Add option to shard the shared expert instead of replicating (#50656)

Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
This commit is contained in:
Tyler Michael Smith
2026-08-03 11:41:24 -04:00
committed by GitHub
co-authored by Claude Jee Jee Li
parent dd11df04f3
commit 5df9999fcf
3 changed files with 154 additions and 3 deletions
+13
View File
@@ -195,6 +195,7 @@ if TYPE_CHECKING:
] = "relax"
VLLM_USE_FUSED_MOE_GROUPED_TOPK: bool = True
VLLM_MOE_SKIP_PADDING: bool = True
VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT: bool = False
VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER: bool = True
VLLM_USE_FLASHINFER_MOE_INT4: bool = False
VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR: str | None = None
@@ -1524,6 +1525,18 @@ environment_variables: dict[str, Callable[[], Any]] = {
# ids to -1 so the dispatch and experts drop them. Requires a MoE kernel that
# treats topk_id == -1 as a skip sentinel
"VLLM_MOE_SKIP_PADDING": lambda: bool(int(os.getenv("VLLM_MOE_SKIP_PADDING", "1"))),
# Kimi-K3 only. Under sequence-parallel MoE the dense and shared-expert MLPs
# are replicated on every rank, so each rank streams the whole weight to
# serve its own token shard. Shard them across TP instead: the MLP then
# all-gathers the full token set, computes this rank's partial, and
# reduce-scatters (which both sums across TP and restores the sequence
# sharding). Trades weight bandwidth and resident memory for two collectives
# per layer, so it only wins at low token counts: intended for decode
# instances in a P/D disaggregated deployment, not for prefill or unified
# serving.
"VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT": lambda: bool(
int(os.getenv("VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT", "0"))
),
# Allow use of FlashInfer FP8 block-scale GEMM for linear layers.
# This uses TensorRT-LLM kernels and requires SM90+ (Hopper).
"VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER": lambda: bool(