[Perf] Improve BF16x3 router GEMM accuracy and make it default on sm100 (#55899)
Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
This commit is contained in:
co-authored by
OpenAI Codex
Jee Jee Li
parent
3116c5d06b
commit
cc4210f671
@@ -695,7 +695,6 @@ class EngineArgs:
|
||||
enable_flashinfer_autotune: bool = get_field(
|
||||
KernelConfig, "enable_flashinfer_autotune"
|
||||
)
|
||||
enable_bf16x3_router_gemm: bool | None = None
|
||||
worker_cls: str = ParallelConfig.worker_cls
|
||||
worker_extension_cls: str = ParallelConfig.worker_extension_cls
|
||||
|
||||
@@ -1655,10 +1654,6 @@ class EngineArgs:
|
||||
"--enable-flashinfer-autotune",
|
||||
**kernel_kwargs["enable_flashinfer_autotune"],
|
||||
)
|
||||
kernel_group.add_argument(
|
||||
"--enable-bf16x3-router-gemm",
|
||||
**kernel_kwargs["enable_bf16x3_router_gemm"],
|
||||
)
|
||||
moe_backend_kwargs = kernel_kwargs["moe_backend"]
|
||||
moe_backend_kwargs["type"] = lambda s: s.lower().replace("-", "_")
|
||||
kernel_group.add_argument("--moe-backend", **moe_backend_kwargs)
|
||||
@@ -2519,8 +2514,6 @@ class EngineArgs:
|
||||
"are mutually exclusive"
|
||||
)
|
||||
kernel_config.enable_flashinfer_autotune = self.enable_flashinfer_autotune
|
||||
if self.enable_bf16x3_router_gemm is not None:
|
||||
kernel_config.enable_bf16x3_router_gemm = self.enable_bf16x3_router_gemm
|
||||
if self.moe_backend != "auto":
|
||||
kernel_config.moe_backend = self.moe_backend
|
||||
if self.linear_backend != "auto":
|
||||
|
||||
Reference in New Issue
Block a user