[Feature][PCP] Support decode-only FULL CUDA graphs (#53867)

Signed-off-by: QiuChunshuo <qiuchunshuo@huawei.com>
Signed-off-by: QiuChunshuo <chunshuoq@gmail.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Qiu Chunshuo
2026-09-15 16:27:46 +08:00
committed by GitHub
co-authored by Codex Lucas Wilkinson mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent ca67438c08
commit 676650397f
5 changed files with 283 additions and 42 deletions
+20 -15
View File
@@ -1468,7 +1468,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
input_batch = pcp.maybe_partition_pcp_batch(
self.pcp_manager,
input_batch,
padded_num_tokens=batch_desc.num_tokens,
batch_desc,
)
return input_batch
@@ -1496,7 +1496,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
def prepare_dummy_attn(
self, input_batch: InputBatch, valid_state_slots: bool = False
) -> tuple[tuple[torch.Tensor, ...], torch.Tensor]:
block_tables = self.block_tables.get_dummy_block_tables(input_batch.num_reqs)
block_table_provider = self.pcp_manager or self.block_tables
block_tables = block_table_provider.get_dummy_block_tables(input_batch.num_reqs)
if valid_state_slots:
state_slots = torch.arange(
1,
@@ -1745,25 +1746,29 @@ class GPUModelRunner(LoRAModelRunnerMixin):
self.input_buffers,
max_query_len=batch_desc.max_query_len,
)
if not skip_attn_for_dummy_run:
block_tables, slot_mappings = self.prepare_dummy_attn(
input_batch, valid_dummy_state_slots
)
if context_len:
set_dummy_context(
input_batch,
self.block_tables,
context_len,
self.kv_cache_config.num_blocks,
self.max_model_len,
)
else:
if self.pcp_manager is not None:
input_batch = self.pcp_manager.prepare_inputs_to_capture(input_batch)
if skip_attn_for_dummy_run:
assert batch_desc.cg_mode != CUDAGraphMode.FULL, (
"Attention metadata must be prepared for dummy runs when using "
"FULL cudagraph mode."
)
block_tables = None
slot_mappings = None
else:
block_tables, slot_mappings = self.prepare_dummy_attn(
input_batch, valid_dummy_state_slots
)
if not skip_attn_for_dummy_run and context_len:
assert block_tables is not None
set_dummy_context(
input_batch,
self.block_tables,
context_len,
self.kv_cache_config.num_blocks,
self.max_model_len,
input_block_tables=block_tables,
)
attn_metadata = None
slot_mappings_by_layer = None