[Optimization] Skip DP padding tokens in MoE (#46428)
Signed-off-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
84f13374b3
commit
0a3e2dbc09
@@ -185,6 +185,7 @@ if TYPE_CHECKING:
|
||||
"relax",
|
||||
] = "relax"
|
||||
VLLM_USE_FUSED_MOE_GROUPED_TOPK: bool = True
|
||||
VLLM_MOE_SKIP_PADDING: bool = False
|
||||
VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER: bool = True
|
||||
VLLM_USE_FLASHINFER_MOE_INT4: bool = False
|
||||
VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR: str | None = None
|
||||
@@ -1469,6 +1470,11 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
"VLLM_USE_FUSED_MOE_GROUPED_TOPK": lambda: bool(
|
||||
int(os.getenv("VLLM_USE_FUSED_MOE_GROUPED_TOPK", "1"))
|
||||
),
|
||||
# Skip cudagraph/DP padding tokens in the MoE path by forcing their expert
|
||||
# ids to -1 so the dispatch and experts drop them. Requires a MoE kernel that
|
||||
# treats topk_id == -1 as a skip sentinel; off by default because not all
|
||||
# kernels support it yet.
|
||||
"VLLM_MOE_SKIP_PADDING": lambda: bool(int(os.getenv("VLLM_MOE_SKIP_PADDING", "0"))),
|
||||
# Allow use of FlashInfer FP8 block-scale GEMM for linear layers.
|
||||
# This uses TensorRT-LLM kernels and requires SM90+ (Hopper).
|
||||
"VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER": lambda: bool(
|
||||
|
||||
Reference in New Issue
Block a user