[Fast Start] Preload the FlashInfer autotune table on the weight cache daemon (#60085)
Signed-off-by: liusy58 <mg21330037@smail.nju.edu.cn> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
co-authored by
Claude Opus 5
Isotr0py
parent
3709632ff2
commit
f2376cdc4d
@@ -39,6 +39,7 @@ from vllm.logger import init_logger
|
||||
from vllm.model_executor.model_loader.weight_cache.protocol import (
|
||||
check_ipc_platform_support,
|
||||
)
|
||||
from vllm.usage.usage_lib import UsageContext
|
||||
from vllm.utils.network_utils import get_distributed_init_method, get_open_port
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
@@ -157,7 +158,13 @@ class PreloadSubcommand(CLISubcommand):
|
||||
)
|
||||
|
||||
engine_args = EngineArgs.from_cli_args(args)
|
||||
vllm_config = engine_args.create_engine_config()
|
||||
# Warmup context for flashinfer autotune.
|
||||
usage_context = (
|
||||
UsageContext.OPENAI_API_SERVER
|
||||
if engine_args.kernel_config.enable_flashinfer_autotune is not False
|
||||
else None
|
||||
)
|
||||
vllm_config = engine_args.create_engine_config(usage_context=usage_context)
|
||||
if vllm_config.load_config.load_format == "ipc_cache":
|
||||
raise ValueError(
|
||||
"The weight cache daemon itself must load from disk; use the "
|
||||
|
||||
Reference in New Issue
Block a user