[Bugfix][MRV2] Capture DCP FULL cudagraphs with KV context (#61496)

Signed-off-by: Ayushman Singh <40520701+ayush1399@users.noreply.github.com>
This commit is contained in:
Ayushman Singh
2026-10-14 19:20:48 +00:00
committed by GitHub
parent 222e00fc44
commit 9b5c6707a3
3 changed files with 94 additions and 5 deletions
@@ -174,6 +174,52 @@ def test_piecewise_capture_uses_pcp_dummy_slot_mappings():
block_tables.get_dummy_slot_mappings.assert_not_called()
@pytest.mark.parametrize("full_cudagraph,expected_context_len", [(True, 2), (False, 0)])
def test_dcp_full_capture_has_kv_context(
monkeypatch, full_cudagraph, expected_context_len
):
"""DCP skips context attention on the host when no request has context, so a
FULL graph captured without context would ignore the KV cache on replay."""
monkeypatch.setattr(
gpu_cudagraph_utils, "prepare_dcp_local_seq_lens", lambda *a, **kw: None
)
num_tokens = num_reqs = 8
buffers = InputBuffers(num_reqs, num_tokens, torch.device("cpu"))
block_tables = MagicMock()
block_tables.cp_size = 2
block_tables.cp_interleave = 1
block_tables.block_sizes = [16]
block_tables.get_dummy_block_tables.return_value = (
torch.zeros(num_reqs, 4, dtype=torch.int32),
)
block_tables.get_dummy_slot_mappings.return_value = torch.empty(
1, num_tokens, dtype=torch.int64
)
model_state = MagicMock()
model_state.max_model_len = 1024
kv_cache_config = KVCacheConfig(
num_blocks=8, kv_cache_tensors=[], kv_cache_groups=[]
)
gpu_cudagraph_utils.prepare_inputs_to_capture(
num_reqs,
num_tokens,
model_state,
buffers,
block_tables,
[],
kv_cache_config,
full_cudagraph=full_cudagraph,
max_query_len=1,
)
batch = model_state.prepare_attn.call_args.args[0]
context_lens = batch.seq_lens_cpu_upper_bound[:num_reqs] - torch.from_numpy(
batch.num_scheduled_tokens[:num_reqs]
)
assert (context_lens == expected_context_len).all()
@pytest.mark.parametrize(
"full_cudagraph,max_query_len,expected_max_query_len",
[(False, None, None), (True, None, 8), (True, 1, 1), (True, 3, 3)],
+46 -4
View File
@@ -3,7 +3,7 @@
import gc
import itertools
from collections import defaultdict
from collections.abc import Callable, Iterable
from collections.abc import Callable, Iterable, Sequence
from contextlib import ExitStack
from dataclasses import dataclass, replace
from itertools import groupby, product
@@ -44,7 +44,11 @@ from vllm.v1.spec_decode.dynamic.utils import build_dynamic_sd_schedule_lookup
from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer
from vllm.v1.worker.gpu.block_table import BlockTables
from vllm.v1.worker.gpu.cp_utils import prepare_dcp_local_seq_lens
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
from vllm.v1.worker.gpu.input_batch import (
InputBatch,
InputBuffers,
set_dummy_context,
)
from vllm.v1.worker.gpu.model_states.interface import ModelState
from vllm.v1.worker.ubatch_utils import check_ubatch_thresholds, get_num_ubatches
from vllm.v1.worker.utils import AttentionGroup, clear_layer_kv_caches
@@ -694,9 +698,18 @@ class ModelCudaGraphManager(CudaGraphManager):
if desc.num_ubatches > 1:
# Prepare and park threads before capture; finish runs inside it.
assert self.ubatch_runner is not None
input_batch = InputBatch.make_dummy(num_reqs, num_tokens, input_buffers)
input_block_tables = block_tables.get_dummy_block_tables(num_reqs)
set_dcp_capture_context(
input_batch,
model_state,
block_tables,
input_block_tables,
kv_cache_config,
)
ubatch_state = self.ubatch_runner.prepare(
InputBatch.make_dummy(num_reqs, num_tokens, input_buffers),
block_tables.get_dummy_block_tables(num_reqs),
input_batch,
input_block_tables,
block_tables.get_dummy_slot_mappings(num_tokens),
cg_mode=CUDAGraphMode.FULL,
for_capture=True,
@@ -783,6 +796,27 @@ class ModelCudaGraphManager(CudaGraphManager):
return hidden_states, [x[: desc.num_tokens] for x in self.aux_hidden_states]
def set_dcp_capture_context(
input_batch: InputBatch,
model_state: ModelState,
block_tables: BlockTables,
input_block_tables: Sequence[torch.Tensor],
kv_cache_config: KVCacheConfig,
) -> None:
# Capture FULL graphs with KV context so a host-side skip of DCP context
# attention is not baked into the graph.
if block_tables.cp_size == 1:
return
set_dummy_context(
input_batch,
block_tables,
block_tables.cp_size * block_tables.cp_interleave,
kv_cache_config.num_blocks,
model_state.max_model_len,
input_block_tables=input_block_tables,
)
def prepare_inputs_to_capture(
num_reqs: int,
num_tokens: int,
@@ -813,6 +847,14 @@ def prepare_inputs_to_capture(
)
if block_tables.cp_size > 1:
if full_cudagraph:
set_dcp_capture_context(
input_batch,
model_state,
block_tables,
input_block_tables,
kv_cache_config,
)
input_batch.dcp_local_seq_lens = prepare_dcp_local_seq_lens(
input_buffers.dcp_local_seq_lens,
input_batch.seq_lens,
+2 -1
View File
@@ -228,7 +228,8 @@ def set_dummy_context(
max_model_len: int,
input_block_tables: Sequence[torch.Tensor] | None = None,
) -> None:
"""Give each dummy request context_len of context, used when profiling step cost."""
"""Give each dummy request context_len of context, for profiling step cost
and for DCP FULL cudagraph capture."""
if input_block_tables is None:
input_block_tables = block_tables.input_block_tables
if not input_block_tables: