[Fast Start] Preload the FlashInfer autotune table on the weight cache daemon (#60085)

Signed-off-by: liusy58 <mg21330037@smail.nju.edu.cn>
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
siyu
2026-10-10 15:48:06 +00:00
committed by GitHub
co-authored by Claude Opus 5 Isotr0py
parent 3709632ff2
commit f2376cdc4d
10 changed files with 253 additions and 17 deletions
+11 -4
View File
@@ -5217,9 +5217,15 @@ class GPUModelRunner(
setattr(self, config_name, new_config)
@instrument(span_name="Loading (GPU)")
def load_model(self, load_dummy_weights: bool = False) -> None:
def load_model(
self,
load_dummy_weights: bool = False,
*,
model: nn.Module | None = None,
) -> None:
"""Args:
load_dummy_weights: load dummy weights instead of real weights.
model: a pre-loaded model to skip the model loader.
"""
logger.info_once(
@@ -5235,13 +5241,14 @@ class GPUModelRunner(
try:
with DeviceMemoryProfiler() as m:
time_before_load = time.perf_counter()
if load_dummy_weights:
if model is None and load_dummy_weights:
self.load_config.load_format = "dummy"
model_loader = get_model_loader(self.load_config)
# Capture warmup providers selected while constructing the model.
with self.jit_warmup_registry.activate(): # type: ignore[attr-defined]
self.model = model_loader.load_model(
vllm_config=self.vllm_config, model_config=self.model_config
self.model = model or model_loader.load_model(
vllm_config=self.vllm_config,
model_config=self.model_config,
)
lookback_depth = getattr(self.model, "token_lookback_depth", 0)
if lookback_depth > 0: