[Model Runner V2] Enable CUDA graph for Eagle3 (#35040)

Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>
This commit is contained in:
Woosuk Kwon
2026-02-21 21:42:50 -08:00
committed by GitHub
parent 30132cd144
commit 2cbf9656ce
2 changed files with 44 additions and 9 deletions
+5 -2
View File
@@ -197,7 +197,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
# CUDA graphs.
self.cudagraph_manager = CudaGraphManager(
self.vllm_config, self.uses_mrope, self.device
self.vllm_config,
self.uses_mrope,
self.use_aux_hidden_state_outputs,
self.device,
)
# Structured outputs worker.
self.structured_outputs_worker = StructuredOutputsWorker(
@@ -1044,7 +1047,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
aux_hidden_states,
input_batch,
kv_connector_output,
)
) # type: ignore
return None
@torch.inference_mode()