[Perf] Improve BF16x3 router GEMM accuracy and make it default on sm100 (#55899)

Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
This commit is contained in:
Thien Tran
2026-09-09 07:33:12 -07:00
committed by GitHub
co-authored by OpenAI Codex Jee Jee Li
parent 3116c5d06b
commit cc4210f671
6 changed files with 314 additions and 88 deletions
-7
View File
@@ -695,7 +695,6 @@ class EngineArgs:
enable_flashinfer_autotune: bool = get_field(
KernelConfig, "enable_flashinfer_autotune"
)
enable_bf16x3_router_gemm: bool | None = None
worker_cls: str = ParallelConfig.worker_cls
worker_extension_cls: str = ParallelConfig.worker_extension_cls
@@ -1655,10 +1654,6 @@ class EngineArgs:
"--enable-flashinfer-autotune",
**kernel_kwargs["enable_flashinfer_autotune"],
)
kernel_group.add_argument(
"--enable-bf16x3-router-gemm",
**kernel_kwargs["enable_bf16x3_router_gemm"],
)
moe_backend_kwargs = kernel_kwargs["moe_backend"]
moe_backend_kwargs["type"] = lambda s: s.lower().replace("-", "_")
kernel_group.add_argument("--moe-backend", **moe_backend_kwargs)
@@ -2519,8 +2514,6 @@ class EngineArgs:
"are mutually exclusive"
)
kernel_config.enable_flashinfer_autotune = self.enable_flashinfer_autotune
if self.enable_bf16x3_router_gemm is not None:
kernel_config.enable_bf16x3_router_gemm = self.enable_bf16x3_router_gemm
if self.moe_backend != "auto":
kernel_config.moe_backend = self.moe_backend
if self.linear_backend != "auto":