diff --git a/tests/v1/cudagraph/test_cudagraph_manager.py b/tests/v1/cudagraph/test_cudagraph_manager.py index 1e4071bcfb..c2d6e4ef7b 100644 --- a/tests/v1/cudagraph/test_cudagraph_manager.py +++ b/tests/v1/cudagraph/test_cudagraph_manager.py @@ -174,6 +174,52 @@ def test_piecewise_capture_uses_pcp_dummy_slot_mappings(): block_tables.get_dummy_slot_mappings.assert_not_called() +@pytest.mark.parametrize("full_cudagraph,expected_context_len", [(True, 2), (False, 0)]) +def test_dcp_full_capture_has_kv_context( + monkeypatch, full_cudagraph, expected_context_len +): + """DCP skips context attention on the host when no request has context, so a + FULL graph captured without context would ignore the KV cache on replay.""" + monkeypatch.setattr( + gpu_cudagraph_utils, "prepare_dcp_local_seq_lens", lambda *a, **kw: None + ) + num_tokens = num_reqs = 8 + buffers = InputBuffers(num_reqs, num_tokens, torch.device("cpu")) + block_tables = MagicMock() + block_tables.cp_size = 2 + block_tables.cp_interleave = 1 + block_tables.block_sizes = [16] + block_tables.get_dummy_block_tables.return_value = ( + torch.zeros(num_reqs, 4, dtype=torch.int32), + ) + block_tables.get_dummy_slot_mappings.return_value = torch.empty( + 1, num_tokens, dtype=torch.int64 + ) + model_state = MagicMock() + model_state.max_model_len = 1024 + kv_cache_config = KVCacheConfig( + num_blocks=8, kv_cache_tensors=[], kv_cache_groups=[] + ) + + gpu_cudagraph_utils.prepare_inputs_to_capture( + num_reqs, + num_tokens, + model_state, + buffers, + block_tables, + [], + kv_cache_config, + full_cudagraph=full_cudagraph, + max_query_len=1, + ) + + batch = model_state.prepare_attn.call_args.args[0] + context_lens = batch.seq_lens_cpu_upper_bound[:num_reqs] - torch.from_numpy( + batch.num_scheduled_tokens[:num_reqs] + ) + assert (context_lens == expected_context_len).all() + + @pytest.mark.parametrize( "full_cudagraph,max_query_len,expected_max_query_len", [(False, None, None), (True, None, 8), (True, 1, 1), (True, 3, 3)], diff --git a/vllm/v1/worker/gpu/cudagraph_utils.py b/vllm/v1/worker/gpu/cudagraph_utils.py index 3a6409db55..dfbb0aa82c 100644 --- a/vllm/v1/worker/gpu/cudagraph_utils.py +++ b/vllm/v1/worker/gpu/cudagraph_utils.py @@ -3,7 +3,7 @@ import gc import itertools from collections import defaultdict -from collections.abc import Callable, Iterable +from collections.abc import Callable, Iterable, Sequence from contextlib import ExitStack from dataclasses import dataclass, replace from itertools import groupby, product @@ -44,7 +44,11 @@ from vllm.v1.spec_decode.dynamic.utils import build_dynamic_sd_schedule_lookup from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer from vllm.v1.worker.gpu.block_table import BlockTables from vllm.v1.worker.gpu.cp_utils import prepare_dcp_local_seq_lens -from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers +from vllm.v1.worker.gpu.input_batch import ( + InputBatch, + InputBuffers, + set_dummy_context, +) from vllm.v1.worker.gpu.model_states.interface import ModelState from vllm.v1.worker.ubatch_utils import check_ubatch_thresholds, get_num_ubatches from vllm.v1.worker.utils import AttentionGroup, clear_layer_kv_caches @@ -694,9 +698,18 @@ class ModelCudaGraphManager(CudaGraphManager): if desc.num_ubatches > 1: # Prepare and park threads before capture; finish runs inside it. assert self.ubatch_runner is not None + input_batch = InputBatch.make_dummy(num_reqs, num_tokens, input_buffers) + input_block_tables = block_tables.get_dummy_block_tables(num_reqs) + set_dcp_capture_context( + input_batch, + model_state, + block_tables, + input_block_tables, + kv_cache_config, + ) ubatch_state = self.ubatch_runner.prepare( - InputBatch.make_dummy(num_reqs, num_tokens, input_buffers), - block_tables.get_dummy_block_tables(num_reqs), + input_batch, + input_block_tables, block_tables.get_dummy_slot_mappings(num_tokens), cg_mode=CUDAGraphMode.FULL, for_capture=True, @@ -783,6 +796,27 @@ class ModelCudaGraphManager(CudaGraphManager): return hidden_states, [x[: desc.num_tokens] for x in self.aux_hidden_states] +def set_dcp_capture_context( + input_batch: InputBatch, + model_state: ModelState, + block_tables: BlockTables, + input_block_tables: Sequence[torch.Tensor], + kv_cache_config: KVCacheConfig, +) -> None: + # Capture FULL graphs with KV context so a host-side skip of DCP context + # attention is not baked into the graph. + if block_tables.cp_size == 1: + return + set_dummy_context( + input_batch, + block_tables, + block_tables.cp_size * block_tables.cp_interleave, + kv_cache_config.num_blocks, + model_state.max_model_len, + input_block_tables=input_block_tables, + ) + + def prepare_inputs_to_capture( num_reqs: int, num_tokens: int, @@ -813,6 +847,14 @@ def prepare_inputs_to_capture( ) if block_tables.cp_size > 1: + if full_cudagraph: + set_dcp_capture_context( + input_batch, + model_state, + block_tables, + input_block_tables, + kv_cache_config, + ) input_batch.dcp_local_seq_lens = prepare_dcp_local_seq_lens( input_buffers.dcp_local_seq_lens, input_batch.seq_lens, diff --git a/vllm/v1/worker/gpu/input_batch.py b/vllm/v1/worker/gpu/input_batch.py index bd55a43f64..c05bb1a431 100644 --- a/vllm/v1/worker/gpu/input_batch.py +++ b/vllm/v1/worker/gpu/input_batch.py @@ -228,7 +228,8 @@ def set_dummy_context( max_model_len: int, input_block_tables: Sequence[torch.Tensor] | None = None, ) -> None: - """Give each dummy request context_len of context, used when profiling step cost.""" + """Give each dummy request context_len of context, for profiling step cost + and for DCP FULL cudagraph capture.""" if input_block_tables is None: input_block_tables = block_tables.input_block_tables if not input_block_tables: