[2/2][Model Runner V2] FULL CUDA graph capture for microbatched steps (DBO) (#51700)
Signed-off-by: specture724 <specture724@gmail.com> Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
@@ -723,6 +723,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
decode_query_len=self.decode_query_len,
|
||||
lora_capture_cases=self.lora_capture_cases,
|
||||
varlen_decode=self.adaptive_verification is not None,
|
||||
ubatch_runner=self.ubatch_runner,
|
||||
)
|
||||
if self.cache_config.kv_sharing_fast_prefill and self.pcp_manager is None:
|
||||
self.fast_prefill = FastPrefillHelper(
|
||||
@@ -1775,7 +1776,11 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
assert self.ubatch_runner is not None
|
||||
assert block_tables is not None and slot_mappings is not None
|
||||
ubatch_state = self.ubatch_runner.prepare(
|
||||
input_batch, block_tables, slot_mappings
|
||||
input_batch,
|
||||
block_tables,
|
||||
slot_mappings,
|
||||
cg_mode=batch_desc.cg_mode,
|
||||
for_capture=dummy_run and batch_desc.cg_mode == CUDAGraphMode.FULL,
|
||||
)
|
||||
elif not (dummy_run and skip_attn_for_dummy_run):
|
||||
assert slot_mappings is not None
|
||||
|
||||
Reference in New Issue
Block a user