[Feature][PCP] Support decode-only FULL CUDA graphs (#53867)
Signed-off-by: QiuChunshuo <qiuchunshuo@huawei.com> Signed-off-by: QiuChunshuo <chunshuoq@gmail.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Codex
Lucas Wilkinson
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
ca67438c08
commit
676650397f
@@ -1,15 +1,19 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from dataclasses import replace
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import CUDAGraphMode
|
||||
from vllm.v1.attention.backends.utils import get_dcp_local_seq_lens
|
||||
from vllm.v1.worker.gpu import cp_utils as gpu_cp_utils
|
||||
from vllm.v1.worker.gpu import pcp_manager as pcp_manager_module
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
|
||||
from vllm.v1.worker.gpu.cudagraph_utils import BatchExecutionDescriptor
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers, set_dummy_context
|
||||
from vllm.v1.worker.gpu.pcp_manager import PCPManager
|
||||
|
||||
|
||||
@@ -20,6 +24,58 @@ def _copy_to_cpu(value, out=None, device=None):
|
||||
return tensor
|
||||
|
||||
|
||||
def _make_config(cudagraph_mode: CUDAGraphMode):
|
||||
return SimpleNamespace(
|
||||
parallel_config=SimpleNamespace(
|
||||
prefill_context_parallel_size=2,
|
||||
decode_context_parallel_size=1,
|
||||
pipeline_parallel_size=1,
|
||||
dcp_comm_backend="ag_rs",
|
||||
),
|
||||
model_config=SimpleNamespace(
|
||||
use_mla=True,
|
||||
is_encoder_decoder=False,
|
||||
hf_text_config=SimpleNamespace(),
|
||||
),
|
||||
lora_config=None,
|
||||
speculative_config=None,
|
||||
compilation_config=SimpleNamespace(cudagraph_mode=cudagraph_mode),
|
||||
)
|
||||
|
||||
|
||||
def _make_capture_manager(block_table: torch.Tensor):
|
||||
block_tables = SimpleNamespace(
|
||||
input_block_tables=(block_table,),
|
||||
num_kv_cache_groups=1,
|
||||
kernel_block_sizes=(2,),
|
||||
blocks_per_kv_block=(1,),
|
||||
)
|
||||
manager = PCPManager(
|
||||
pcp_world_size=2,
|
||||
pcp_rank=0,
|
||||
device=torch.device("cpu"),
|
||||
max_num_reqs=4,
|
||||
max_num_tokens=8,
|
||||
block_tables=block_tables,
|
||||
)
|
||||
return manager, block_tables
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"cudagraph_mode",
|
||||
[CUDAGraphMode.FULL_DECODE_ONLY, CUDAGraphMode.FULL_AND_PIECEWISE],
|
||||
)
|
||||
def test_validate_config_accepts_decode_only_full_graphs(cudagraph_mode):
|
||||
PCPManager.validate_config(_make_config(cudagraph_mode), supports_mm_inputs=False)
|
||||
|
||||
|
||||
def test_validate_config_rejects_full_graph_for_prefills():
|
||||
with pytest.raises(NotImplementedError, match="decode-only routines"):
|
||||
PCPManager.validate_config(
|
||||
_make_config(CUDAGraphMode.FULL), supports_mm_inputs=False
|
||||
)
|
||||
|
||||
|
||||
def test_replicated_decode_piecewise_graph_padding(monkeypatch):
|
||||
manager = PCPManager(
|
||||
pcp_world_size=2,
|
||||
@@ -114,6 +170,89 @@ def test_graph_padding_cannot_be_smaller_than_largest_pcp_rank(monkeypatch):
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("cg_mode", "num_reqs", "expected_tokens", "expected_reqs"),
|
||||
[
|
||||
(CUDAGraphMode.NONE, 4, None, None),
|
||||
(CUDAGraphMode.PIECEWISE, None, 8, None),
|
||||
(CUDAGraphMode.FULL, 4, 8, 4),
|
||||
],
|
||||
)
|
||||
def test_partition_padding_is_derived_from_batch_descriptor(
|
||||
cg_mode, num_reqs, expected_tokens, expected_reqs
|
||||
):
|
||||
manager = MagicMock()
|
||||
input_batch = MagicMock()
|
||||
manager.partition_batch.return_value = input_batch
|
||||
batch_desc = BatchExecutionDescriptor(
|
||||
cg_mode=cg_mode,
|
||||
num_tokens=8,
|
||||
num_reqs=num_reqs,
|
||||
)
|
||||
|
||||
result = pcp_manager_module.maybe_partition_pcp_batch(
|
||||
manager,
|
||||
input_batch,
|
||||
batch_desc,
|
||||
)
|
||||
|
||||
assert result is input_batch
|
||||
manager.partition_batch.assert_called_once_with(
|
||||
input_batch,
|
||||
padded_num_tokens=expected_tokens,
|
||||
padded_num_reqs=expected_reqs,
|
||||
)
|
||||
|
||||
|
||||
def test_capture_uses_pcp_persistent_inputs():
|
||||
manager, _ = _make_capture_manager(torch.ones((4, 2), dtype=torch.int32))
|
||||
|
||||
dummy_batch = InputBatch.make_dummy(
|
||||
num_reqs=4,
|
||||
num_tokens=4,
|
||||
input_buffers=InputBuffers(4, 8, torch.device("cpu")),
|
||||
max_query_len=1,
|
||||
)
|
||||
dummy_batch.input_ids.copy_(torch.arange(4))
|
||||
dummy_batch.positions.fill_(7)
|
||||
input_batch = manager.prepare_inputs_to_capture(dummy_batch)
|
||||
|
||||
assert input_batch is not dummy_batch
|
||||
assert input_batch.req_ids == dummy_batch.req_ids
|
||||
for name in ("input_ids", "positions", "is_padding", "query_start_loc", "seq_lens"):
|
||||
actual = getattr(input_batch, name)
|
||||
torch.testing.assert_close(actual, getattr(dummy_batch, name))
|
||||
assert actual.data_ptr() == getattr(manager.input_buffers, name).data_ptr()
|
||||
|
||||
|
||||
def test_dummy_context_updates_pcp_local_block_tables():
|
||||
global_block_table = torch.full((4, 4), -1, dtype=torch.int32)
|
||||
manager, block_tables = _make_capture_manager(global_block_table)
|
||||
input_batch = InputBatch.make_dummy(
|
||||
num_reqs=2,
|
||||
num_tokens=2,
|
||||
input_buffers=manager.input_buffers,
|
||||
max_query_len=1,
|
||||
)
|
||||
input_batch = manager.prepare_inputs_to_capture(input_batch)
|
||||
local_block_tables = manager.get_dummy_block_tables(input_batch.num_reqs)
|
||||
|
||||
set_dummy_context(
|
||||
input_batch,
|
||||
block_tables,
|
||||
context_len=3,
|
||||
num_kv_blocks=16,
|
||||
max_model_len=16,
|
||||
input_block_tables=local_block_tables,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(
|
||||
local_block_tables[0][:2, :2],
|
||||
torch.tensor([[0, 1], [2, 3]], dtype=torch.int32),
|
||||
)
|
||||
assert torch.all(global_block_table == -1)
|
||||
|
||||
|
||||
def _rank_rows(
|
||||
pcp_rank: int,
|
||||
pcp_world_size: int,
|
||||
@@ -331,10 +470,22 @@ def test_partition_defers_dcp_metadata_to_post_partition_batch():
|
||||
global_batch.dcp_local_seq_lens = global_buffers.dcp_local_seq_lens[:2]
|
||||
global_batch.dcp_local_seq_lens.fill_(-1)
|
||||
|
||||
local_batch = manager.partition_batch(global_batch, padded_num_tokens=2)
|
||||
local_batch = manager.partition_batch(
|
||||
global_batch,
|
||||
padded_num_tokens=4,
|
||||
padded_num_reqs=4,
|
||||
)
|
||||
|
||||
assert local_batch.dcp_local_seq_lens is None
|
||||
assert local_batch.seq_lens.tolist() == [17, 25]
|
||||
assert local_batch.seq_lens.tolist() == [17, 25, 0, 0]
|
||||
assert local_batch.dcp_local_seq_lens_cpu_upper_bound is not None
|
||||
expected_dcp_upper_bound = get_dcp_local_seq_lens(
|
||||
torch.tensor([17, 25], dtype=torch.int32), 2, 0, 1
|
||||
)
|
||||
assert torch.equal(
|
||||
local_batch.dcp_local_seq_lens_cpu_upper_bound,
|
||||
torch.cat([expected_dcp_upper_bound, torch.zeros(2, dtype=torch.int32)]),
|
||||
)
|
||||
|
||||
# What execute_model does next: derive DCP metadata from the final batch
|
||||
# on the PCP-owned buffers.
|
||||
@@ -348,7 +499,7 @@ def test_partition_defers_dcp_metadata_to_post_partition_batch():
|
||||
num_reqs_padded=local_batch.num_reqs_after_padding,
|
||||
)
|
||||
expected = get_dcp_local_seq_lens(
|
||||
torch.tensor([17, 25], dtype=torch.int32), 2, 0, 1
|
||||
torch.tensor([17, 25, 0, 0], dtype=torch.int32), 2, 0, 1
|
||||
)
|
||||
assert local_batch.dcp_local_seq_lens is not None
|
||||
assert torch.equal(local_batch.dcp_local_seq_lens.cpu(), expected)
|
||||
|
||||
Reference in New Issue
Block a user