[Kernel] Add KernelConfig flag to enable/disable FlashInfer autotune (#34006)
Signed-off-by: Mohammad Miadh Angkad <176301910+mmangkad@users.noreply.github.com> Signed-off-by: Luka Govedič <ProExpertProg@users.noreply.github.com> Co-authored-by: Luka Govedič <ProExpertProg@users.noreply.github.com>
This commit is contained in:
co-authored by
Luka Govedič
parent
edb359cce4
commit
dd6a6e1190
@@ -40,6 +40,7 @@ from vllm.config import (
|
||||
DeviceConfig,
|
||||
ECTransferConfig,
|
||||
EPLBConfig,
|
||||
KernelConfig,
|
||||
KVEventsConfig,
|
||||
KVTransferConfig,
|
||||
LoadConfig,
|
||||
@@ -536,6 +537,10 @@ class EngineArgs:
|
||||
pooler_config: PoolerConfig | None = ModelConfig.pooler_config
|
||||
compilation_config: CompilationConfig = get_field(VllmConfig, "compilation_config")
|
||||
attention_config: AttentionConfig = get_field(VllmConfig, "attention_config")
|
||||
kernel_config: KernelConfig = get_field(VllmConfig, "kernel_config")
|
||||
enable_flashinfer_autotune: bool = get_field(
|
||||
KernelConfig, "enable_flashinfer_autotune"
|
||||
)
|
||||
worker_cls: str = ParallelConfig.worker_cls
|
||||
worker_extension_cls: str = ParallelConfig.worker_extension_cls
|
||||
|
||||
@@ -595,6 +600,8 @@ class EngineArgs:
|
||||
self.compilation_config = CompilationConfig(**self.compilation_config)
|
||||
if isinstance(self.attention_config, dict):
|
||||
self.attention_config = AttentionConfig(**self.attention_config)
|
||||
if isinstance(self.kernel_config, dict):
|
||||
self.kernel_config = KernelConfig(**self.kernel_config)
|
||||
if isinstance(self.eplb_config, dict):
|
||||
self.eplb_config = EPLBConfig(**self.eplb_config)
|
||||
if isinstance(self.weight_transfer_config, dict):
|
||||
@@ -1163,6 +1170,17 @@ class EngineArgs:
|
||||
**compilation_kwargs["max_cudagraph_capture_size"],
|
||||
)
|
||||
|
||||
# Kernel arguments
|
||||
kernel_kwargs = get_kwargs(KernelConfig)
|
||||
kernel_group = parser.add_argument_group(
|
||||
title="KernelConfig",
|
||||
description=KernelConfig.__doc__,
|
||||
)
|
||||
kernel_group.add_argument(
|
||||
"--enable-flashinfer-autotune",
|
||||
**kernel_kwargs["enable_flashinfer_autotune"],
|
||||
)
|
||||
|
||||
# vLLM arguments
|
||||
vllm_kwargs = get_kwargs(VllmConfig)
|
||||
vllm_group = parser.add_argument_group(
|
||||
@@ -1189,6 +1207,7 @@ class EngineArgs:
|
||||
vllm_group.add_argument(
|
||||
"--attention-config", "-ac", **vllm_kwargs["attention_config"]
|
||||
)
|
||||
vllm_group.add_argument("--kernel-config", **vllm_kwargs["kernel_config"])
|
||||
vllm_group.add_argument(
|
||||
"--additional-config", **vllm_kwargs["additional_config"]
|
||||
)
|
||||
@@ -1717,6 +1736,17 @@ class EngineArgs:
|
||||
else:
|
||||
attention_config.backend = self.attention_backend
|
||||
|
||||
# Kernel config overrides
|
||||
kernel_config = copy.deepcopy(self.kernel_config)
|
||||
if self.enable_flashinfer_autotune is not None:
|
||||
if kernel_config.enable_flashinfer_autotune is not None:
|
||||
raise ValueError(
|
||||
"enable_flashinfer_autotune and "
|
||||
"kernel_config.enable_flashinfer_autotune "
|
||||
"are mutually exclusive"
|
||||
)
|
||||
kernel_config.enable_flashinfer_autotune = self.enable_flashinfer_autotune
|
||||
|
||||
load_config = self.create_load_config()
|
||||
|
||||
# Pass reasoning_parser into StructuredOutputsConfig
|
||||
@@ -1767,6 +1797,7 @@ class EngineArgs:
|
||||
device_config=device_config,
|
||||
load_config=load_config,
|
||||
attention_config=attention_config,
|
||||
kernel_config=kernel_config,
|
||||
lora_config=lora_config,
|
||||
speculative_config=speculative_config,
|
||||
structured_outputs_config=self.structured_outputs_config,
|
||||
|
||||
Reference in New Issue
Block a user