[Kernel][UX] Add --linear-backend arg for linear kernel selection (#39538)
Signed-off-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
@@ -71,7 +71,7 @@ from vllm.config.cache import (
|
||||
PrefixCachingHashAlgo,
|
||||
)
|
||||
from vllm.config.device import Device
|
||||
from vllm.config.kernel import IrOpPriorityConfig, MoEBackend
|
||||
from vllm.config.kernel import IrOpPriorityConfig, LinearBackend, MoEBackend
|
||||
from vllm.config.lora import MaxLoRARanks
|
||||
from vllm.config.mamba import MambaBackendEnum
|
||||
from vllm.config.model import (
|
||||
@@ -477,6 +477,7 @@ class EngineArgs:
|
||||
enable_expert_parallel: bool = ParallelConfig.enable_expert_parallel
|
||||
enable_ep_weight_filter: bool = ParallelConfig.enable_ep_weight_filter
|
||||
moe_backend: MoEBackend = KernelConfig.moe_backend
|
||||
linear_backend: LinearBackend = KernelConfig.linear_backend
|
||||
all2all_backend: All2AllBackend = ParallelConfig.all2all_backend
|
||||
enable_elastic_ep: bool = ParallelConfig.enable_elastic_ep
|
||||
enable_dbo: bool = ParallelConfig.enable_dbo
|
||||
@@ -1412,6 +1413,9 @@ class EngineArgs:
|
||||
moe_backend_kwargs = kernel_kwargs["moe_backend"]
|
||||
moe_backend_kwargs["type"] = lambda s: s.lower().replace("-", "_")
|
||||
kernel_group.add_argument("--moe-backend", **moe_backend_kwargs)
|
||||
linear_backend_kwargs = kernel_kwargs["linear_backend"]
|
||||
linear_backend_kwargs["type"] = lambda s: s.lower().replace("-", "_")
|
||||
kernel_group.add_argument("--linear-backend", **linear_backend_kwargs)
|
||||
|
||||
# vLLM arguments
|
||||
vllm_kwargs = get_kwargs(VllmConfig)
|
||||
@@ -2085,6 +2089,8 @@ class EngineArgs:
|
||||
kernel_config.enable_flashinfer_autotune = self.enable_flashinfer_autotune
|
||||
if self.moe_backend != "auto":
|
||||
kernel_config.moe_backend = self.moe_backend
|
||||
if self.linear_backend != "auto":
|
||||
kernel_config.linear_backend = self.linear_backend
|
||||
|
||||
# Transfer top-level ir_op_priority into KernelConfig.ir_op_priority
|
||||
for op_name, op_priority in asdict(self.ir_op_priority).items():
|
||||
|
||||
Reference in New Issue
Block a user