[Model Runner V2] Skip attention metadata rebuild before draft prefill (#40410)

Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai>
This commit is contained in:
Giancarlo Delfin
2026-04-27 15:38:05 -07:00
committed by GitHub
parent 5e2c37facd
commit 4c7c69b4e0
4 changed files with 88 additions and 40 deletions
+2 -2
View File
@@ -590,7 +590,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
start_free_gpu_memory = torch.cuda.mem_get_info()[0]
with self.maybe_setup_dummy_loras(self.lora_config):
self.cudagraph_manager.capture(
captured_attn_states = self.cudagraph_manager.capture(
self.model,
self.model_state,
self.input_buffers,
@@ -602,7 +602,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
use_aux_hidden_state_outputs=self.use_aux_hidden_state_outputs,
)
if self.speculator is not None:
self.speculator.capture_model()
self.speculator.capture(captured_attn_states)
end_time = time.perf_counter()
end_free_gpu_memory = torch.cuda.mem_get_info()[0]