[Feature][PCP] Support decode-only FULL CUDA graphs (#53867)

Signed-off-by: QiuChunshuo <qiuchunshuo@huawei.com>
Signed-off-by: QiuChunshuo <chunshuoq@gmail.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Qiu Chunshuo
2026-09-15 16:27:46 +08:00
committed by GitHub
co-authored by Codex Lucas Wilkinson mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent ca67438c08
commit 676650397f
5 changed files with 283 additions and 42 deletions
+155 -4
View File
@@ -1,15 +1,19 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import replace
from types import SimpleNamespace
from unittest.mock import MagicMock
import numpy as np
import pytest
import torch
from vllm.config import CUDAGraphMode
from vllm.v1.attention.backends.utils import get_dcp_local_seq_lens
from vllm.v1.worker.gpu import cp_utils as gpu_cp_utils
from vllm.v1.worker.gpu import pcp_manager as pcp_manager_module
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
from vllm.v1.worker.gpu.cudagraph_utils import BatchExecutionDescriptor
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers, set_dummy_context
from vllm.v1.worker.gpu.pcp_manager import PCPManager
@@ -20,6 +24,58 @@ def _copy_to_cpu(value, out=None, device=None):
return tensor
def _make_config(cudagraph_mode: CUDAGraphMode):
return SimpleNamespace(
parallel_config=SimpleNamespace(
prefill_context_parallel_size=2,
decode_context_parallel_size=1,
pipeline_parallel_size=1,
dcp_comm_backend="ag_rs",
),
model_config=SimpleNamespace(
use_mla=True,
is_encoder_decoder=False,
hf_text_config=SimpleNamespace(),
),
lora_config=None,
speculative_config=None,
compilation_config=SimpleNamespace(cudagraph_mode=cudagraph_mode),
)
def _make_capture_manager(block_table: torch.Tensor):
block_tables = SimpleNamespace(
input_block_tables=(block_table,),
num_kv_cache_groups=1,
kernel_block_sizes=(2,),
blocks_per_kv_block=(1,),
)
manager = PCPManager(
pcp_world_size=2,
pcp_rank=0,
device=torch.device("cpu"),
max_num_reqs=4,
max_num_tokens=8,
block_tables=block_tables,
)
return manager, block_tables
@pytest.mark.parametrize(
"cudagraph_mode",
[CUDAGraphMode.FULL_DECODE_ONLY, CUDAGraphMode.FULL_AND_PIECEWISE],
)
def test_validate_config_accepts_decode_only_full_graphs(cudagraph_mode):
PCPManager.validate_config(_make_config(cudagraph_mode), supports_mm_inputs=False)
def test_validate_config_rejects_full_graph_for_prefills():
with pytest.raises(NotImplementedError, match="decode-only routines"):
PCPManager.validate_config(
_make_config(CUDAGraphMode.FULL), supports_mm_inputs=False
)
def test_replicated_decode_piecewise_graph_padding(monkeypatch):
manager = PCPManager(
pcp_world_size=2,
@@ -114,6 +170,89 @@ def test_graph_padding_cannot_be_smaller_than_largest_pcp_rank(monkeypatch):
)
@pytest.mark.parametrize(
("cg_mode", "num_reqs", "expected_tokens", "expected_reqs"),
[
(CUDAGraphMode.NONE, 4, None, None),
(CUDAGraphMode.PIECEWISE, None, 8, None),
(CUDAGraphMode.FULL, 4, 8, 4),
],
)
def test_partition_padding_is_derived_from_batch_descriptor(
cg_mode, num_reqs, expected_tokens, expected_reqs
):
manager = MagicMock()
input_batch = MagicMock()
manager.partition_batch.return_value = input_batch
batch_desc = BatchExecutionDescriptor(
cg_mode=cg_mode,
num_tokens=8,
num_reqs=num_reqs,
)
result = pcp_manager_module.maybe_partition_pcp_batch(
manager,
input_batch,
batch_desc,
)
assert result is input_batch
manager.partition_batch.assert_called_once_with(
input_batch,
padded_num_tokens=expected_tokens,
padded_num_reqs=expected_reqs,
)
def test_capture_uses_pcp_persistent_inputs():
manager, _ = _make_capture_manager(torch.ones((4, 2), dtype=torch.int32))
dummy_batch = InputBatch.make_dummy(
num_reqs=4,
num_tokens=4,
input_buffers=InputBuffers(4, 8, torch.device("cpu")),
max_query_len=1,
)
dummy_batch.input_ids.copy_(torch.arange(4))
dummy_batch.positions.fill_(7)
input_batch = manager.prepare_inputs_to_capture(dummy_batch)
assert input_batch is not dummy_batch
assert input_batch.req_ids == dummy_batch.req_ids
for name in ("input_ids", "positions", "is_padding", "query_start_loc", "seq_lens"):
actual = getattr(input_batch, name)
torch.testing.assert_close(actual, getattr(dummy_batch, name))
assert actual.data_ptr() == getattr(manager.input_buffers, name).data_ptr()
def test_dummy_context_updates_pcp_local_block_tables():
global_block_table = torch.full((4, 4), -1, dtype=torch.int32)
manager, block_tables = _make_capture_manager(global_block_table)
input_batch = InputBatch.make_dummy(
num_reqs=2,
num_tokens=2,
input_buffers=manager.input_buffers,
max_query_len=1,
)
input_batch = manager.prepare_inputs_to_capture(input_batch)
local_block_tables = manager.get_dummy_block_tables(input_batch.num_reqs)
set_dummy_context(
input_batch,
block_tables,
context_len=3,
num_kv_blocks=16,
max_model_len=16,
input_block_tables=local_block_tables,
)
torch.testing.assert_close(
local_block_tables[0][:2, :2],
torch.tensor([[0, 1], [2, 3]], dtype=torch.int32),
)
assert torch.all(global_block_table == -1)
def _rank_rows(
pcp_rank: int,
pcp_world_size: int,
@@ -331,10 +470,22 @@ def test_partition_defers_dcp_metadata_to_post_partition_batch():
global_batch.dcp_local_seq_lens = global_buffers.dcp_local_seq_lens[:2]
global_batch.dcp_local_seq_lens.fill_(-1)
local_batch = manager.partition_batch(global_batch, padded_num_tokens=2)
local_batch = manager.partition_batch(
global_batch,
padded_num_tokens=4,
padded_num_reqs=4,
)
assert local_batch.dcp_local_seq_lens is None
assert local_batch.seq_lens.tolist() == [17, 25]
assert local_batch.seq_lens.tolist() == [17, 25, 0, 0]
assert local_batch.dcp_local_seq_lens_cpu_upper_bound is not None
expected_dcp_upper_bound = get_dcp_local_seq_lens(
torch.tensor([17, 25], dtype=torch.int32), 2, 0, 1
)
assert torch.equal(
local_batch.dcp_local_seq_lens_cpu_upper_bound,
torch.cat([expected_dcp_upper_bound, torch.zeros(2, dtype=torch.int32)]),
)
# What execute_model does next: derive DCP metadata from the final batch
# on the PCP-owned buffers.
@@ -348,7 +499,7 @@ def test_partition_defers_dcp_metadata_to_post_partition_batch():
num_reqs_padded=local_batch.num_reqs_after_padding,
)
expected = get_dcp_local_seq_lens(
torch.tensor([17, 25], dtype=torch.int32), 2, 0, 1
torch.tensor([17, 25, 0, 0], dtype=torch.int32), 2, 0, 1
)
assert local_batch.dcp_local_seq_lens is not None
assert torch.equal(local_batch.dcp_local_seq_lens.cpu(), expected)