[Kimi-K3] Add option to shard the shared expert instead of replicating (#50656)
Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
This commit is contained in:
co-authored by
Claude
Jee Jee Li
parent
dd11df04f3
commit
5df9999fcf
@@ -195,6 +195,7 @@ if TYPE_CHECKING:
|
||||
] = "relax"
|
||||
VLLM_USE_FUSED_MOE_GROUPED_TOPK: bool = True
|
||||
VLLM_MOE_SKIP_PADDING: bool = True
|
||||
VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT: bool = False
|
||||
VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER: bool = True
|
||||
VLLM_USE_FLASHINFER_MOE_INT4: bool = False
|
||||
VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR: str | None = None
|
||||
@@ -1524,6 +1525,18 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
# ids to -1 so the dispatch and experts drop them. Requires a MoE kernel that
|
||||
# treats topk_id == -1 as a skip sentinel
|
||||
"VLLM_MOE_SKIP_PADDING": lambda: bool(int(os.getenv("VLLM_MOE_SKIP_PADDING", "1"))),
|
||||
# Kimi-K3 only. Under sequence-parallel MoE the dense and shared-expert MLPs
|
||||
# are replicated on every rank, so each rank streams the whole weight to
|
||||
# serve its own token shard. Shard them across TP instead: the MLP then
|
||||
# all-gathers the full token set, computes this rank's partial, and
|
||||
# reduce-scatters (which both sums across TP and restores the sequence
|
||||
# sharding). Trades weight bandwidth and resident memory for two collectives
|
||||
# per layer, so it only wins at low token counts: intended for decode
|
||||
# instances in a P/D disaggregated deployment, not for prefill or unified
|
||||
# serving.
|
||||
"VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT": lambda: bool(
|
||||
int(os.getenv("VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT", "0"))
|
||||
),
|
||||
# Allow use of FlashInfer FP8 block-scale GEMM for linear layers.
|
||||
# This uses TensorRT-LLM kernels and requires SM90+ (Hopper).
|
||||
"VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER": lambda: bool(
|
||||
|
||||
Reference in New Issue
Block a user