[Optimization] Skip DP padding tokens in MoE (#46428)

Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Woosuk Kwon
2026-06-23 14:54:46 -07:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 84f13374b3
commit 0a3e2dbc09
9 changed files with 154 additions and 1 deletions
+6
View File
@@ -185,6 +185,7 @@ if TYPE_CHECKING:
"relax",
] = "relax"
VLLM_USE_FUSED_MOE_GROUPED_TOPK: bool = True
VLLM_MOE_SKIP_PADDING: bool = False
VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER: bool = True
VLLM_USE_FLASHINFER_MOE_INT4: bool = False
VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR: str | None = None
@@ -1469,6 +1470,11 @@ environment_variables: dict[str, Callable[[], Any]] = {
"VLLM_USE_FUSED_MOE_GROUPED_TOPK": lambda: bool(
int(os.getenv("VLLM_USE_FUSED_MOE_GROUPED_TOPK", "1"))
),
# Skip cudagraph/DP padding tokens in the MoE path by forcing their expert
# ids to -1 so the dispatch and experts drop them. Requires a MoE kernel that
# treats topk_id == -1 as a skip sentinel; off by default because not all
# kernels support it yet.
"VLLM_MOE_SKIP_PADDING": lambda: bool(int(os.getenv("VLLM_MOE_SKIP_PADDING", "0"))),
# Allow use of FlashInfer FP8 block-scale GEMM for linear layers.
# This uses TensorRT-LLM kernels and requires SM90+ (Hopper).
"VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER": lambda: bool(