[Perf][PCP] Shard decode requests across PCP ranks (#52162)

Signed-off-by: QiuChunshuo <qiuchunshuo@huawei.com>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
Qiu Chunshuo
2026-10-01 23:50:13 -04:00
committed by GitHub
co-authored by Lucas Wilkinson
parent a810316c08
commit 8486f1ebfe
13 changed files with 406 additions and 33 deletions
@@ -205,6 +205,7 @@ def test_short_prefill_updates_k_cache_before_scoring_decision(
topk_indices,
False,
False,
False,
MLA_LAYER,
)
@@ -259,6 +260,7 @@ def test_skipped_k_cache_insert_accepts_no_k(
topk_indices,
True,
False,
False,
"",
)
@@ -15,6 +15,7 @@ from vllm.models.deepseek_v41.sparse_mla import (
DeepseekV4SparseMLABackend as DeepseekV41SparseMLABackend,
)
from vllm.v1.attention.backend import CommonAttentionMetadata
from vllm.v1.attention.backends.mla import indexer as indexer_module
from vllm.v1.attention.backends.mla.compressor_utils import (
CompressedSlotMappingKernel,
get_compressed_slot_mapping,
@@ -295,6 +296,78 @@ def test_index_conversion_warmup_uses_physical_block_stride():
assert {key.block_stride_rows for key in keys} == {4096}
def test_zero_token_pcp_rank_participates_in_compressed_mapping_gather(monkeypatch):
builder = object.__new__(DeepseekV32IndexerMetadataBuilder)
builder.compress_ratio = 4
builder.use_pcp = True
builder.pcp_world_size = 4
builder.pcp_rank = 0
builder.kernel_block_size = None
builder.kv_cache_spec = SimpleNamespace(block_size=64, num_states=64)
builder.compressed_slot_mapping_buffer = torch.zeros(8, dtype=torch.int64)
def fake_get_compressed_slot_mapping(
num_tokens,
slot_mapping,
query_start_loc,
seq_lens,
block_table,
block_size,
compress_ratio,
out,
):
assert num_tokens == 0
assert block_size == 64
assert compress_ratio == 4
out.fill_(-1)
return out[:num_tokens]
gathered_slot_mapping = torch.tensor([123, -1, -1, -1], dtype=torch.int64)
class FakePCPGroup:
def __init__(self):
self.calls = 0
def all_gather(self, tensor, dim=0):
self.calls += 1
assert dim == 0
torch.testing.assert_close(tensor, torch.tensor([-1]))
return gathered_slot_mapping
fake_pcp_group = FakePCPGroup()
monkeypatch.setattr(
indexer_module,
"get_compressed_slot_mapping",
fake_get_compressed_slot_mapping,
)
monkeypatch.setattr(indexer_module, "get_pcp_group", lambda: fake_pcp_group)
query_start_loc = torch.tensor([0, 0], dtype=torch.int32)
seq_lens = torch.tensor([16], dtype=torch.int32)
common = CommonAttentionMetadata(
query_start_loc=query_start_loc,
query_start_loc_cpu=query_start_loc,
seq_lens=seq_lens,
seq_lens_cpu_upper_bound=seq_lens,
num_reqs=1,
num_actual_tokens=0,
max_query_len=0,
max_seq_len=16,
block_table_tensor=torch.tensor([[0]], dtype=torch.int32),
slot_mapping=torch.tensor([11, 22, 33, 44], dtype=torch.int64),
causal=True,
)
metadata = builder.build(common_prefix_len=0, common_attn_metadata=common)
assert fake_pcp_group.calls == 1
assert metadata.num_decodes == 0
assert metadata.num_decode_tokens == 0
assert metadata.num_prefills == 0
assert metadata.num_prefill_tokens == 0
torch.testing.assert_close(metadata.slot_mapping, gathered_slot_mapping)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_indexer_builder_deepseek_v4_compressed_slot_mapping_uses_num_states():
"""Regression test: DeepseekV4 compression path must compute slot_mapping from
@@ -2499,6 +2499,7 @@ def test_hisparse_kv_update_writes_resident_and_staging_caches():
layer = SimpleNamespace(
hisparse_cache=cache_handle,
use_pcp=False,
pcp_shard_decode_requests=False,
impl=impl,
)
mla_attention.MLAAttention.update_kv_cache(
@@ -132,6 +132,7 @@ def test_piecewise_capture_uses_pcp_dummy_slot_mappings():
pcp_world_size=pcp_world_size,
pcp_rank=0,
device=torch.device("cpu"),
shard_decode_requests=False,
max_num_reqs=num_reqs,
max_num_tokens=num_tokens,
block_tables=pcp_block_tables,
+220 -10
View File
@@ -7,8 +7,9 @@ import numpy as np
import pytest
import torch
from vllm.config import CUDAGraphMode
from vllm.v1.attention.backends.utils import get_dcp_local_seq_lens
from vllm.config import CUDAGraphMode, ParallelConfig
from vllm.v1.attention.backends.utils import PAD_SLOT_ID, get_dcp_local_seq_lens
from vllm.v1.attention.ops import pcp as attention_pcp
from vllm.v1.worker.gpu import cp_utils as gpu_cp_utils
from vllm.v1.worker.gpu import pcp_manager as pcp_manager_module
from vllm.v1.worker.gpu.cudagraph_utils import BatchExecutionDescriptor
@@ -53,6 +54,7 @@ def _make_capture_manager(block_table: torch.Tensor):
pcp_world_size=2,
pcp_rank=0,
device=torch.device("cpu"),
shard_decode_requests=False,
max_num_reqs=4,
max_num_tokens=8,
block_tables=block_tables,
@@ -93,11 +95,12 @@ def test_validate_config_dcp_for_dspark(draft_uses_mla, dcp_size):
PCPManager.validate_config(config, supports_mm_inputs=False)
def test_replicated_decode_piecewise_graph_padding(monkeypatch):
def test_sharded_decode_piecewise_graph_padding(monkeypatch):
manager = PCPManager(
pcp_world_size=2,
pcp_rank=0,
device=torch.device("cpu"),
shard_decode_requests=True,
dcp_world_size=1,
)
monkeypatch.setattr(pcp_manager_module, "async_tensor_h2d", _copy_to_cpu)
@@ -110,19 +113,19 @@ def test_replicated_decode_piecewise_graph_padding(monkeypatch):
padded_num_tokens=4,
)
assert per_rank_num_tokens == [3, 3]
assert per_rank_num_tokens == [2, 1]
request_indices = [
[segment.global_batch_req_idx for segment in rank] for rank in segments_by_rank
]
assert request_indices == [[0, 1, 2], [0, 1, 2]]
assert torch.equal(manager._hidden_restore_idx, torch.tensor([0, 1, 2]))
assert request_indices == [[0, 2], [1]]
assert torch.equal(manager._hidden_restore_idx, torch.tensor([0, 4, 1]))
assert torch.equal(
manager._padded_gather_idx,
torch.tensor([0, 1, 2, 0, 0, 1, 2, 0]),
torch.tensor([0, 2, 0, 0, 1, 0, 0, 0]),
)
assert torch.equal(
manager._gathered_kv_write_mask,
torch.tensor([True, True, True, False, False, False, False, False]),
torch.tensor([True, True, False, False, True, False, False, False]),
)
@@ -131,6 +134,7 @@ def test_input_buffers_are_exposed_for_cudagraph_capture():
pcp_world_size=2,
pcp_rank=0,
device=torch.device("cpu"),
shard_decode_requests=False,
max_num_reqs=4,
max_num_tokens=8,
)
@@ -148,7 +152,7 @@ def test_input_buffers_are_exposed_for_cudagraph_capture():
(2, [7], [True], 4),
(2, [3], [False], 3),
(2, [3, 8], [False, True], 7),
(4, [2, 9], [False, True], 5),
(4, [2, 9], [False, True], 4),
],
)
def test_num_tokens_for_dispatch_uses_largest_pcp_rank(
@@ -158,6 +162,7 @@ def test_num_tokens_for_dispatch_uses_largest_pcp_rank(
pcp_world_size=pcp_world_size,
pcp_rank=0,
device=torch.device("cpu"),
shard_decode_requests=True,
)
actual = manager.get_num_tokens_for_dispatch(
@@ -173,6 +178,7 @@ def test_graph_padding_cannot_be_smaller_than_largest_pcp_rank(monkeypatch):
pcp_world_size=2,
pcp_rank=0,
device=torch.device("cpu"),
shard_decode_requests=True,
dcp_world_size=1,
)
monkeypatch.setattr(pcp_manager_module, "async_tensor_h2d", _copy_to_cpu)
@@ -183,7 +189,7 @@ def test_graph_padding_cannot_be_smaller_than_largest_pcp_rank(monkeypatch):
num_computed_tokens=np.full(3, 16, dtype=np.int32),
is_prefilling=np.zeros(3, dtype=np.bool_),
query_start_loc_np=np.arange(4, dtype=np.int32),
padded_num_tokens=2,
padded_num_tokens=1,
)
@@ -207,6 +213,7 @@ def test_partition_padding_is_derived_from_batch_descriptor(
pcp_world_size=2,
pcp_rank=0,
device=device,
shard_decode_requests=False,
max_num_reqs=4,
max_num_tokens=8,
)
@@ -288,6 +295,189 @@ def test_dummy_context_updates_pcp_local_block_tables():
assert torch.all(global_block_table == -1)
def _rank_request_ids(
manager: PCPManager,
rank: int,
req_ids: list[str],
*,
is_prefilling: np.ndarray | None = None,
) -> list[str]:
num_reqs = len(req_ids)
if is_prefilling is None:
is_prefilling = np.zeros(num_reqs, dtype=np.bool_)
segments = manager._get_rank_segments(
rank=rank,
num_scheduled_tokens=np.ones(num_reqs, dtype=np.int32),
is_prefilling=is_prefilling,
query_start_loc_np=np.arange(num_reqs + 1, dtype=np.int32),
)
return [req_ids[segment.global_batch_req_idx] for segment in segments]
def test_pcp_only_decode_requests_are_round_robin_balanced_each_step():
manager = PCPManager(
pcp_world_size=4,
pcp_rank=0,
device=torch.device("cpu"),
shard_decode_requests=True,
dcp_world_size=1,
)
req_ids = [f"request-{idx}" for idx in range(18)]
owners: dict[str, int] = {}
for rank in range(manager.pcp_world_size):
for req_id in _rank_request_ids(manager, rank, req_ids):
assert req_id not in owners
owners[req_id] = rank
assert owners == {
req_id: index % manager.pcp_world_size for index, req_id in enumerate(req_ids)
}
counts = [list(owners.values()).count(rank) for rank in range(4)]
assert max(counts) - min(counts) == 1
reordered_req_ids = req_ids[::2] + req_ids[1::2]
reordered_owners = {
req_id: rank
for rank in range(manager.pcp_world_size)
for req_id in _rank_request_ids(manager, rank, reordered_req_ids)
}
assert reordered_owners == {
req_id: index % manager.pcp_world_size
for index, req_id in enumerate(reordered_req_ids)
}
assert reordered_owners != owners
def test_decode_requests_remain_replicated_when_dcp_is_enabled():
manager = PCPManager(
pcp_world_size=2,
pcp_rank=0,
device=torch.device("cpu"),
shard_decode_requests=False,
dcp_world_size=2,
)
req_ids = ["request-a", "request-b", "request-c"]
assert _rank_request_ids(manager, 0, req_ids) == req_ids
assert _rank_request_ids(manager, 1, req_ids) == req_ids
def test_decode_sharding_allows_ranks_with_no_owned_request():
manager = PCPManager(
pcp_world_size=4,
pcp_rank=0,
device=torch.device("cpu"),
shard_decode_requests=True,
dcp_world_size=1,
)
req_ids = ["request-a"]
assert _rank_request_ids(manager, 0, req_ids) == req_ids
assert _rank_request_ids(manager, 1, req_ids) == []
assert _rank_request_ids(manager, 2, req_ids) == []
assert _rank_request_ids(manager, 3, req_ids) == []
def test_prefill_partitioning_is_preserved_with_sharded_decode():
manager = PCPManager(
pcp_world_size=2,
pcp_rank=0,
device=torch.device("cpu"),
shard_decode_requests=True,
dcp_world_size=1,
)
segments_by_rank = [
manager._get_rank_segments(
rank=rank,
num_scheduled_tokens=np.array([8, 1, 0, 1], dtype=np.int32),
is_prefilling=np.array([True, False, False, False]),
query_start_loc_np=np.array([0, 8, 9, 9, 10], dtype=np.int32),
)
for rank in range(manager.pcp_world_size)
]
prefill_tokens = sorted(
token_idx
for segments in segments_by_rank
for segment in segments
if segment.global_batch_req_idx == 0
for token_idx in range(
segment.global_batch_slice.start, segment.global_batch_slice.stop
)
)
decode_owners = {
segment.global_batch_req_idx: rank
for rank, segments in enumerate(segments_by_rank)
for segment in segments
if segment.global_batch_req_idx in (1, 3)
}
assert prefill_tokens == list(range(8))
assert decode_owners == {1: 0, 3: 1}
def test_sharded_decode_layout_selects_owner_kv_for_replication(monkeypatch):
manager = PCPManager(
pcp_world_size=2,
pcp_rank=0,
device=torch.device("cpu"),
shard_decode_requests=True,
dcp_world_size=1,
)
monkeypatch.setattr(pcp_manager_module, "async_tensor_h2d", _copy_to_cpu)
manager._build_batch_layout(
num_scheduled_tokens=np.array([1, 1, 1], dtype=np.int32),
num_computed_tokens=np.array([16, 16, 16], dtype=np.int32),
is_prefilling=np.array([False, False, False]),
query_start_loc_np=np.array([0, 1, 2, 3], dtype=np.int32),
)
gathered_slot_mapping = manager._convert_to_gathered_slot_mappings(
torch.tensor([[123, 456, 789]], dtype=torch.int64)
)
assert torch.equal(
gathered_slot_mapping,
torch.tensor([[123, 789, 456, PAD_SLOT_ID]], dtype=torch.int64),
)
assert torch.equal(manager._hidden_restore_idx, torch.tensor([0, 2, 1]))
class FakePCPGroup:
world_size = 2
def all_gather(self, tensor: torch.Tensor, dim: int = 0) -> torch.Tensor:
assert dim == 0
assert tensor.shape == (2, 1)
return torch.tensor([[11.0], [33.0], [22.0], [0.0]])
monkeypatch.setattr(attention_pcp, "get_pcp_group", FakePCPGroup)
(gathered_kv,), cache_slot_mapping = attention_pcp._gather_prefill_cache_inputs(
(torch.tensor([[11.0], [33.0]]),),
gathered_slot_mapping[0],
num_decode_tokens=0,
shard_decode_requests=True,
)
assert torch.equal(gathered_kv, torch.tensor([[11.0], [33.0], [22.0], [0.0]]))
assert torch.equal(cache_slot_mapping, torch.tensor([123, 789, 456, PAD_SLOT_ID]))
def test_replicated_draft_cache_inputs_keep_unexpanded_slot_mapping(monkeypatch):
def unexpected_gather():
pytest.fail("replicated draft decode must not gather PCP cache inputs")
monkeypatch.setattr(attention_pcp, "get_pcp_group", unexpected_gather)
kv = torch.tensor([[11.0], [22.0], [0.0]])
slot_mapping = torch.tensor([123, 456])
(cache_kv,), cache_slot_mapping = attention_pcp._gather_prefill_cache_inputs(
(kv,), slot_mapping, num_decode_tokens=2, shard_decode_requests=True
)
assert torch.equal(cache_kv, kv[:2])
assert torch.equal(cache_slot_mapping, slot_mapping)
def _rank_rows(
pcp_rank: int,
pcp_world_size: int,
@@ -300,6 +490,7 @@ def _rank_rows(
pcp_world_size=pcp_world_size,
pcp_rank=pcp_rank,
device=torch.device("cpu"),
shard_decode_requests=False,
dcp_world_size=2,
)
query_start_loc_np = np.concatenate([[0], np.cumsum(num_scheduled_tokens)]).astype(
@@ -393,6 +584,7 @@ def test_dcp_replicates_prefills_too_short_to_split(query_len):
pcp_world_size=pcp_world_size,
pcp_rank=rank,
device=torch.device("cpu"),
shard_decode_requests=False,
dcp_world_size=2,
)
assert manager.replicated_requests(num_scheduled_tokens, is_prefilling)[0]
@@ -418,6 +610,7 @@ def test_pcp_first_chunk_row_is_never_short(pcp_world_size, query_len):
pcp_world_size=pcp_world_size,
pcp_rank=rank,
device=torch.device("cpu"),
shard_decode_requests=False,
dcp_world_size=pcp_world_size,
)
chunk_lens = [
@@ -493,6 +686,7 @@ def test_partition_defers_dcp_metadata_to_post_partition_batch():
pcp_world_size=2,
pcp_rank=0,
device=device,
shard_decode_requests=False,
max_num_reqs=4,
max_num_tokens=8,
dcp_world_size=2,
@@ -541,3 +735,19 @@ def test_partition_defers_dcp_metadata_to_post_partition_batch():
)
assert local_batch.dcp_local_seq_lens is not None
assert torch.equal(local_batch.dcp_local_seq_lens.cpu(), expected)
@pytest.mark.parametrize(
("pcp_world_size", "dcp_world_size", "expected"),
[(1, 1, False), (2, 1, True), (2, 2, False)],
)
def test_parallel_config_manages_decode_sharding(
pcp_world_size: int, dcp_world_size: int, expected: bool
):
parallel_config = ParallelConfig(
prefill_context_parallel_size=pcp_world_size,
decode_context_parallel_size=dcp_world_size,
distributed_executor_backend="ray",
)
assert parallel_config.pcp_shard_decode_requests is expected