[Kernel][DSV4.1] Fuse MXFP8 wo_b GEMM with sequence-parallel reduce-scatter (#57428)
Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg> Signed-off-by: Canlin <canlinguosdu@gmail.com> Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
This commit is contained in:
co-authored by
Thien Tran
OpenAI Codex
Claude Fable 5.1
Jee Jee Li
parent
a6c47fbbf4
commit
f92b78f6ef
+6
-4
@@ -213,7 +213,7 @@ if TYPE_CHECKING:
|
||||
VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT: bool = False
|
||||
VLLM_KIMI_K3_AUX_ATTN_RES_STREAM: bool = False
|
||||
VLLM_KIMI_K3_GEMM_AR: bool = True
|
||||
VLLM_KIMI_K3_GEMM_RS: bool = False
|
||||
VLLM_ENABLE_GEMM_RS: bool = False
|
||||
VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER: bool = True
|
||||
VLLM_USE_FLASHINFER_MOE_INT4: bool = False
|
||||
VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR: str | None = None
|
||||
@@ -1629,9 +1629,11 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
# Use the SM100 BF16 GEMM-AR kernel for eligible Kimi-K3 row-parallel
|
||||
# attention projections. All TP ranks must belong to one NVLink domain.
|
||||
"VLLM_KIMI_K3_GEMM_AR": lambda: bool(int(os.getenv("VLLM_KIMI_K3_GEMM_AR", "1"))),
|
||||
# Use the SM100 BF16 GEMM-RS kernel for eligible Kimi-K3 sequence-parallel
|
||||
# row-parallel projections. All TP ranks must belong to one NVLink domain.
|
||||
"VLLM_KIMI_K3_GEMM_RS": lambda: bool(int(os.getenv("VLLM_KIMI_K3_GEMM_RS", "0"))),
|
||||
# Fuse eligible sequence-parallel row-parallel projections with their TP
|
||||
# reduce-scatter using the SM100 BF16/MXFP8 GEMM-RS kernel (Kimi-K3
|
||||
# attention/shared-expert projections, DeepSeek-V4.1 ``wo_b``). All TP
|
||||
# ranks must belong to one NVLink domain.
|
||||
"VLLM_ENABLE_GEMM_RS": lambda: bool(int(os.getenv("VLLM_ENABLE_GEMM_RS", "0"))),
|
||||
# Allow use of FlashInfer FP8 block-scale GEMM for linear layers.
|
||||
# This uses TensorRT-LLM kernels and requires SM90+ (Hopper).
|
||||
"VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER": lambda: bool(
|
||||
|
||||
Reference in New Issue
Block a user