BugFix(PCP): use persistent input buffers for PIECEWISE CUDA graphs (#53515)
Signed-off-by: QiuChunshuo <qiuchunshuo@huawei.com> Signed-off-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Lucas Wilkinson
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
46638857fd
commit
b1fbbc2ade
@@ -0,0 +1,109 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.v1.worker.gpu import pcp_manager as pcp_manager_module
|
||||
from vllm.v1.worker.gpu.pcp_manager import PCPManager
|
||||
|
||||
|
||||
def _copy_to_cpu(value, out=None, device=None):
|
||||
tensor = torch.from_numpy(value) if isinstance(value, np.ndarray) else value
|
||||
if out is not None:
|
||||
return out.copy_(tensor)
|
||||
return tensor
|
||||
|
||||
|
||||
def test_replicated_decode_piecewise_graph_padding(monkeypatch):
|
||||
manager = PCPManager(
|
||||
pcp_world_size=2,
|
||||
pcp_rank=0,
|
||||
device=torch.device("cpu"),
|
||||
dcp_world_size=1,
|
||||
)
|
||||
monkeypatch.setattr(pcp_manager_module, "async_copy_to_gpu", _copy_to_cpu)
|
||||
|
||||
segments_by_rank, per_rank_num_tokens = manager._build_batch_layout(
|
||||
num_scheduled_tokens=np.ones(3, dtype=np.int32),
|
||||
num_computed_tokens=np.full(3, 16, dtype=np.int32),
|
||||
is_prefilling=np.zeros(3, dtype=np.bool_),
|
||||
query_start_loc_np=np.arange(4, dtype=np.int32),
|
||||
padded_num_tokens=4,
|
||||
)
|
||||
|
||||
assert per_rank_num_tokens == [3, 3]
|
||||
request_indices = [
|
||||
[segment.global_batch_req_idx for segment in rank] for rank in segments_by_rank
|
||||
]
|
||||
assert request_indices == [[0, 1, 2], [0, 1, 2]]
|
||||
assert torch.equal(manager._hidden_restore_idx, torch.tensor([0, 1, 2]))
|
||||
assert torch.equal(
|
||||
manager._padded_gather_idx,
|
||||
torch.tensor([0, 1, 2, 0, 0, 1, 2, 0]),
|
||||
)
|
||||
assert torch.equal(
|
||||
manager._gathered_kv_write_mask,
|
||||
torch.tensor([True, True, True, False, False, False, False, False]),
|
||||
)
|
||||
|
||||
|
||||
def test_input_buffers_are_exposed_for_cudagraph_capture():
|
||||
manager = PCPManager(
|
||||
pcp_world_size=2,
|
||||
pcp_rank=0,
|
||||
device=torch.device("cpu"),
|
||||
max_num_reqs=4,
|
||||
max_num_tokens=8,
|
||||
)
|
||||
|
||||
assert manager.input_buffers is manager._input_buffers
|
||||
assert manager.input_buffers.input_ids.shape == (8,)
|
||||
assert manager.input_buffers.positions.shape == (8,)
|
||||
assert manager.input_buffers.is_padding.shape == (8,)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("pcp_world_size", "num_scheduled_tokens", "is_prefilling", "expected"),
|
||||
[
|
||||
(2, [8], [True], 4),
|
||||
(2, [7], [True], 4),
|
||||
(2, [3], [False], 3),
|
||||
(2, [3, 8], [False, True], 7),
|
||||
(4, [2, 9], [False, True], 5),
|
||||
],
|
||||
)
|
||||
def test_num_tokens_for_dispatch_uses_largest_pcp_rank(
|
||||
pcp_world_size, num_scheduled_tokens, is_prefilling, expected
|
||||
):
|
||||
manager = PCPManager(
|
||||
pcp_world_size=pcp_world_size,
|
||||
pcp_rank=0,
|
||||
device=torch.device("cpu"),
|
||||
)
|
||||
|
||||
actual = manager.get_num_tokens_for_dispatch(
|
||||
np.asarray(num_scheduled_tokens, dtype=np.int32),
|
||||
np.asarray(is_prefilling, dtype=np.bool_),
|
||||
)
|
||||
|
||||
assert actual == expected
|
||||
|
||||
|
||||
def test_graph_padding_cannot_be_smaller_than_largest_pcp_rank(monkeypatch):
|
||||
manager = PCPManager(
|
||||
pcp_world_size=2,
|
||||
pcp_rank=0,
|
||||
device=torch.device("cpu"),
|
||||
dcp_world_size=1,
|
||||
)
|
||||
monkeypatch.setattr(pcp_manager_module, "async_copy_to_gpu", _copy_to_cpu)
|
||||
|
||||
with pytest.raises(ValueError, match="smaller than the largest rank-local batch"):
|
||||
manager._build_batch_layout(
|
||||
num_scheduled_tokens=np.ones(3, dtype=np.int32),
|
||||
num_computed_tokens=np.full(3, 16, dtype=np.int32),
|
||||
is_prefilling=np.zeros(3, dtype=np.bool_),
|
||||
query_start_loc_np=np.arange(4, dtype=np.int32),
|
||||
padded_num_tokens=2,
|
||||
)
|
||||
Reference in New Issue
Block a user