[UX] Add --moe-backend arg for explicit kernel selection (#33807)
Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
This commit is contained in:
co-authored by
Robert Shaw
parent
1976356ee6
commit
de527e1cec
@@ -70,6 +70,7 @@ from vllm.config.cache import (
|
||||
PrefixCachingHashAlgo,
|
||||
)
|
||||
from vllm.config.device import Device
|
||||
from vllm.config.kernel import MoEBackend
|
||||
from vllm.config.lora import MaxLoRARanks
|
||||
from vllm.config.model import (
|
||||
ConvertOption,
|
||||
@@ -416,6 +417,7 @@ class EngineArgs:
|
||||
data_parallel_external_lb: bool = False
|
||||
data_parallel_backend: DataParallelBackend = ParallelConfig.data_parallel_backend
|
||||
enable_expert_parallel: bool = ParallelConfig.enable_expert_parallel
|
||||
moe_backend: MoEBackend = KernelConfig.moe_backend
|
||||
all2all_backend: All2AllBackend = ParallelConfig.all2all_backend
|
||||
enable_dbo: bool = ParallelConfig.enable_dbo
|
||||
ubatch_size: int = ParallelConfig.ubatch_size
|
||||
@@ -1227,6 +1229,9 @@ class EngineArgs:
|
||||
"--enable-flashinfer-autotune",
|
||||
**kernel_kwargs["enable_flashinfer_autotune"],
|
||||
)
|
||||
moe_backend_kwargs = kernel_kwargs["moe_backend"]
|
||||
moe_backend_kwargs["type"] = lambda s: s.lower().replace("-", "_")
|
||||
kernel_group.add_argument("--moe-backend", **moe_backend_kwargs)
|
||||
|
||||
# vLLM arguments
|
||||
vllm_kwargs = get_kwargs(VllmConfig)
|
||||
@@ -1817,6 +1822,8 @@ class EngineArgs:
|
||||
"are mutually exclusive"
|
||||
)
|
||||
kernel_config.enable_flashinfer_autotune = self.enable_flashinfer_autotune
|
||||
if self.moe_backend != "auto":
|
||||
kernel_config.moe_backend = self.moe_backend
|
||||
|
||||
load_config = self.create_load_config()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user