[Bugfix][MRV2] Capture DCP FULL cudagraphs with KV context (#61496)
Signed-off-by: Ayushman Singh <40520701+ayush1399@users.noreply.github.com>
This commit is contained in:
@@ -174,6 +174,52 @@ def test_piecewise_capture_uses_pcp_dummy_slot_mappings():
|
||||
block_tables.get_dummy_slot_mappings.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("full_cudagraph,expected_context_len", [(True, 2), (False, 0)])
|
||||
def test_dcp_full_capture_has_kv_context(
|
||||
monkeypatch, full_cudagraph, expected_context_len
|
||||
):
|
||||
"""DCP skips context attention on the host when no request has context, so a
|
||||
FULL graph captured without context would ignore the KV cache on replay."""
|
||||
monkeypatch.setattr(
|
||||
gpu_cudagraph_utils, "prepare_dcp_local_seq_lens", lambda *a, **kw: None
|
||||
)
|
||||
num_tokens = num_reqs = 8
|
||||
buffers = InputBuffers(num_reqs, num_tokens, torch.device("cpu"))
|
||||
block_tables = MagicMock()
|
||||
block_tables.cp_size = 2
|
||||
block_tables.cp_interleave = 1
|
||||
block_tables.block_sizes = [16]
|
||||
block_tables.get_dummy_block_tables.return_value = (
|
||||
torch.zeros(num_reqs, 4, dtype=torch.int32),
|
||||
)
|
||||
block_tables.get_dummy_slot_mappings.return_value = torch.empty(
|
||||
1, num_tokens, dtype=torch.int64
|
||||
)
|
||||
model_state = MagicMock()
|
||||
model_state.max_model_len = 1024
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=8, kv_cache_tensors=[], kv_cache_groups=[]
|
||||
)
|
||||
|
||||
gpu_cudagraph_utils.prepare_inputs_to_capture(
|
||||
num_reqs,
|
||||
num_tokens,
|
||||
model_state,
|
||||
buffers,
|
||||
block_tables,
|
||||
[],
|
||||
kv_cache_config,
|
||||
full_cudagraph=full_cudagraph,
|
||||
max_query_len=1,
|
||||
)
|
||||
|
||||
batch = model_state.prepare_attn.call_args.args[0]
|
||||
context_lens = batch.seq_lens_cpu_upper_bound[:num_reqs] - torch.from_numpy(
|
||||
batch.num_scheduled_tokens[:num_reqs]
|
||||
)
|
||||
assert (context_lens == expected_context_len).all()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"full_cudagraph,max_query_len,expected_max_query_len",
|
||||
[(False, None, None), (True, None, 8), (True, 1, 1), (True, 3, 3)],
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
import gc
|
||||
import itertools
|
||||
from collections import defaultdict
|
||||
from collections.abc import Callable, Iterable
|
||||
from collections.abc import Callable, Iterable, Sequence
|
||||
from contextlib import ExitStack
|
||||
from dataclasses import dataclass, replace
|
||||
from itertools import groupby, product
|
||||
@@ -44,7 +44,11 @@ from vllm.v1.spec_decode.dynamic.utils import build_dynamic_sd_schedule_lookup
|
||||
from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer
|
||||
from vllm.v1.worker.gpu.block_table import BlockTables
|
||||
from vllm.v1.worker.gpu.cp_utils import prepare_dcp_local_seq_lens
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
|
||||
from vllm.v1.worker.gpu.input_batch import (
|
||||
InputBatch,
|
||||
InputBuffers,
|
||||
set_dummy_context,
|
||||
)
|
||||
from vllm.v1.worker.gpu.model_states.interface import ModelState
|
||||
from vllm.v1.worker.ubatch_utils import check_ubatch_thresholds, get_num_ubatches
|
||||
from vllm.v1.worker.utils import AttentionGroup, clear_layer_kv_caches
|
||||
@@ -694,9 +698,18 @@ class ModelCudaGraphManager(CudaGraphManager):
|
||||
if desc.num_ubatches > 1:
|
||||
# Prepare and park threads before capture; finish runs inside it.
|
||||
assert self.ubatch_runner is not None
|
||||
input_batch = InputBatch.make_dummy(num_reqs, num_tokens, input_buffers)
|
||||
input_block_tables = block_tables.get_dummy_block_tables(num_reqs)
|
||||
set_dcp_capture_context(
|
||||
input_batch,
|
||||
model_state,
|
||||
block_tables,
|
||||
input_block_tables,
|
||||
kv_cache_config,
|
||||
)
|
||||
ubatch_state = self.ubatch_runner.prepare(
|
||||
InputBatch.make_dummy(num_reqs, num_tokens, input_buffers),
|
||||
block_tables.get_dummy_block_tables(num_reqs),
|
||||
input_batch,
|
||||
input_block_tables,
|
||||
block_tables.get_dummy_slot_mappings(num_tokens),
|
||||
cg_mode=CUDAGraphMode.FULL,
|
||||
for_capture=True,
|
||||
@@ -783,6 +796,27 @@ class ModelCudaGraphManager(CudaGraphManager):
|
||||
return hidden_states, [x[: desc.num_tokens] for x in self.aux_hidden_states]
|
||||
|
||||
|
||||
def set_dcp_capture_context(
|
||||
input_batch: InputBatch,
|
||||
model_state: ModelState,
|
||||
block_tables: BlockTables,
|
||||
input_block_tables: Sequence[torch.Tensor],
|
||||
kv_cache_config: KVCacheConfig,
|
||||
) -> None:
|
||||
# Capture FULL graphs with KV context so a host-side skip of DCP context
|
||||
# attention is not baked into the graph.
|
||||
if block_tables.cp_size == 1:
|
||||
return
|
||||
set_dummy_context(
|
||||
input_batch,
|
||||
block_tables,
|
||||
block_tables.cp_size * block_tables.cp_interleave,
|
||||
kv_cache_config.num_blocks,
|
||||
model_state.max_model_len,
|
||||
input_block_tables=input_block_tables,
|
||||
)
|
||||
|
||||
|
||||
def prepare_inputs_to_capture(
|
||||
num_reqs: int,
|
||||
num_tokens: int,
|
||||
@@ -813,6 +847,14 @@ def prepare_inputs_to_capture(
|
||||
)
|
||||
|
||||
if block_tables.cp_size > 1:
|
||||
if full_cudagraph:
|
||||
set_dcp_capture_context(
|
||||
input_batch,
|
||||
model_state,
|
||||
block_tables,
|
||||
input_block_tables,
|
||||
kv_cache_config,
|
||||
)
|
||||
input_batch.dcp_local_seq_lens = prepare_dcp_local_seq_lens(
|
||||
input_buffers.dcp_local_seq_lens,
|
||||
input_batch.seq_lens,
|
||||
|
||||
@@ -228,7 +228,8 @@ def set_dummy_context(
|
||||
max_model_len: int,
|
||||
input_block_tables: Sequence[torch.Tensor] | None = None,
|
||||
) -> None:
|
||||
"""Give each dummy request context_len of context, used when profiling step cost."""
|
||||
"""Give each dummy request context_len of context, for profiling step cost
|
||||
and for DCP FULL cudagraph capture."""
|
||||
if input_block_tables is None:
|
||||
input_block_tables = block_tables.input_block_tables
|
||||
if not input_block_tables:
|
||||
|
||||
Reference in New Issue
Block a user