From 8486f1ebfe6e39a5513a8636e7f07ed9746f8be5 Mon Sep 17 00:00:00 2001 From: Qiu Chunshuo Date: Fri, 2 Oct 2026 09:20:13 +0530 Subject: [PATCH] [Perf][PCP] Shard decode requests across PCP ranks (#52162) Signed-off-by: QiuChunshuo Co-authored-by: Lucas Wilkinson --- .../layers/test_mla_short_prefill_indexer.py | 2 + .../test_indexer_deepseek_v4_slot_mapping.py | 73 ++++++ .../v1/attention/test_sparse_mla_backends.py | 1 + tests/v1/cudagraph/test_cudagraph_manager.py | 1 + tests/v1/worker/test_gpu_pcp_manager.py | 230 +++++++++++++++++- vllm/config/parallel.py | 13 + .../layers/attention/mla_attention.py | 2 + .../layers/sparse_attn_indexer.py | 6 + vllm/models/deepseek_v32/attention.py | 2 + .../attention/backends/mla/flashmla_sparse.py | 9 +- vllm/v1/attention/backends/mla/indexer.py | 39 ++- vllm/v1/attention/ops/pcp.py | 32 ++- vllm/v1/worker/gpu/pcp_manager.py | 29 ++- 13 files changed, 406 insertions(+), 33 deletions(-) diff --git a/tests/model_executor/layers/test_mla_short_prefill_indexer.py b/tests/model_executor/layers/test_mla_short_prefill_indexer.py index 2e4d4d3d67..ba4ff895cb 100644 --- a/tests/model_executor/layers/test_mla_short_prefill_indexer.py +++ b/tests/model_executor/layers/test_mla_short_prefill_indexer.py @@ -205,6 +205,7 @@ def test_short_prefill_updates_k_cache_before_scoring_decision( topk_indices, False, False, + False, MLA_LAYER, ) @@ -259,6 +260,7 @@ def test_skipped_k_cache_insert_accepts_no_k( topk_indices, True, False, + False, "", ) diff --git a/tests/v1/attention/test_indexer_deepseek_v4_slot_mapping.py b/tests/v1/attention/test_indexer_deepseek_v4_slot_mapping.py index bf775078c8..5bb6eb4c32 100644 --- a/tests/v1/attention/test_indexer_deepseek_v4_slot_mapping.py +++ b/tests/v1/attention/test_indexer_deepseek_v4_slot_mapping.py @@ -15,6 +15,7 @@ from vllm.models.deepseek_v41.sparse_mla import ( DeepseekV4SparseMLABackend as DeepseekV41SparseMLABackend, ) from vllm.v1.attention.backend import CommonAttentionMetadata +from vllm.v1.attention.backends.mla import indexer as indexer_module from vllm.v1.attention.backends.mla.compressor_utils import ( CompressedSlotMappingKernel, get_compressed_slot_mapping, @@ -295,6 +296,78 @@ def test_index_conversion_warmup_uses_physical_block_stride(): assert {key.block_stride_rows for key in keys} == {4096} +def test_zero_token_pcp_rank_participates_in_compressed_mapping_gather(monkeypatch): + builder = object.__new__(DeepseekV32IndexerMetadataBuilder) + builder.compress_ratio = 4 + builder.use_pcp = True + builder.pcp_world_size = 4 + builder.pcp_rank = 0 + builder.kernel_block_size = None + builder.kv_cache_spec = SimpleNamespace(block_size=64, num_states=64) + builder.compressed_slot_mapping_buffer = torch.zeros(8, dtype=torch.int64) + + def fake_get_compressed_slot_mapping( + num_tokens, + slot_mapping, + query_start_loc, + seq_lens, + block_table, + block_size, + compress_ratio, + out, + ): + assert num_tokens == 0 + assert block_size == 64 + assert compress_ratio == 4 + out.fill_(-1) + return out[:num_tokens] + + gathered_slot_mapping = torch.tensor([123, -1, -1, -1], dtype=torch.int64) + + class FakePCPGroup: + def __init__(self): + self.calls = 0 + + def all_gather(self, tensor, dim=0): + self.calls += 1 + assert dim == 0 + torch.testing.assert_close(tensor, torch.tensor([-1])) + return gathered_slot_mapping + + fake_pcp_group = FakePCPGroup() + monkeypatch.setattr( + indexer_module, + "get_compressed_slot_mapping", + fake_get_compressed_slot_mapping, + ) + monkeypatch.setattr(indexer_module, "get_pcp_group", lambda: fake_pcp_group) + + query_start_loc = torch.tensor([0, 0], dtype=torch.int32) + seq_lens = torch.tensor([16], dtype=torch.int32) + common = CommonAttentionMetadata( + query_start_loc=query_start_loc, + query_start_loc_cpu=query_start_loc, + seq_lens=seq_lens, + seq_lens_cpu_upper_bound=seq_lens, + num_reqs=1, + num_actual_tokens=0, + max_query_len=0, + max_seq_len=16, + block_table_tensor=torch.tensor([[0]], dtype=torch.int32), + slot_mapping=torch.tensor([11, 22, 33, 44], dtype=torch.int64), + causal=True, + ) + + metadata = builder.build(common_prefix_len=0, common_attn_metadata=common) + + assert fake_pcp_group.calls == 1 + assert metadata.num_decodes == 0 + assert metadata.num_decode_tokens == 0 + assert metadata.num_prefills == 0 + assert metadata.num_prefill_tokens == 0 + torch.testing.assert_close(metadata.slot_mapping, gathered_slot_mapping) + + @pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") def test_indexer_builder_deepseek_v4_compressed_slot_mapping_uses_num_states(): """Regression test: DeepseekV4 compression path must compute slot_mapping from diff --git a/tests/v1/attention/test_sparse_mla_backends.py b/tests/v1/attention/test_sparse_mla_backends.py index 048eb40586..a6517430ad 100644 --- a/tests/v1/attention/test_sparse_mla_backends.py +++ b/tests/v1/attention/test_sparse_mla_backends.py @@ -2499,6 +2499,7 @@ def test_hisparse_kv_update_writes_resident_and_staging_caches(): layer = SimpleNamespace( hisparse_cache=cache_handle, use_pcp=False, + pcp_shard_decode_requests=False, impl=impl, ) mla_attention.MLAAttention.update_kv_cache( diff --git a/tests/v1/cudagraph/test_cudagraph_manager.py b/tests/v1/cudagraph/test_cudagraph_manager.py index 366db7b025..8f58462d6b 100644 --- a/tests/v1/cudagraph/test_cudagraph_manager.py +++ b/tests/v1/cudagraph/test_cudagraph_manager.py @@ -132,6 +132,7 @@ def test_piecewise_capture_uses_pcp_dummy_slot_mappings(): pcp_world_size=pcp_world_size, pcp_rank=0, device=torch.device("cpu"), + shard_decode_requests=False, max_num_reqs=num_reqs, max_num_tokens=num_tokens, block_tables=pcp_block_tables, diff --git a/tests/v1/worker/test_gpu_pcp_manager.py b/tests/v1/worker/test_gpu_pcp_manager.py index 4612d49d92..d4b464e014 100644 --- a/tests/v1/worker/test_gpu_pcp_manager.py +++ b/tests/v1/worker/test_gpu_pcp_manager.py @@ -7,8 +7,9 @@ import numpy as np import pytest import torch -from vllm.config import CUDAGraphMode -from vllm.v1.attention.backends.utils import get_dcp_local_seq_lens +from vllm.config import CUDAGraphMode, ParallelConfig +from vllm.v1.attention.backends.utils import PAD_SLOT_ID, get_dcp_local_seq_lens +from vllm.v1.attention.ops import pcp as attention_pcp from vllm.v1.worker.gpu import cp_utils as gpu_cp_utils from vllm.v1.worker.gpu import pcp_manager as pcp_manager_module from vllm.v1.worker.gpu.cudagraph_utils import BatchExecutionDescriptor @@ -53,6 +54,7 @@ def _make_capture_manager(block_table: torch.Tensor): pcp_world_size=2, pcp_rank=0, device=torch.device("cpu"), + shard_decode_requests=False, max_num_reqs=4, max_num_tokens=8, block_tables=block_tables, @@ -93,11 +95,12 @@ def test_validate_config_dcp_for_dspark(draft_uses_mla, dcp_size): PCPManager.validate_config(config, supports_mm_inputs=False) -def test_replicated_decode_piecewise_graph_padding(monkeypatch): +def test_sharded_decode_piecewise_graph_padding(monkeypatch): manager = PCPManager( pcp_world_size=2, pcp_rank=0, device=torch.device("cpu"), + shard_decode_requests=True, dcp_world_size=1, ) monkeypatch.setattr(pcp_manager_module, "async_tensor_h2d", _copy_to_cpu) @@ -110,19 +113,19 @@ def test_replicated_decode_piecewise_graph_padding(monkeypatch): padded_num_tokens=4, ) - assert per_rank_num_tokens == [3, 3] + assert per_rank_num_tokens == [2, 1] request_indices = [ [segment.global_batch_req_idx for segment in rank] for rank in segments_by_rank ] - assert request_indices == [[0, 1, 2], [0, 1, 2]] - assert torch.equal(manager._hidden_restore_idx, torch.tensor([0, 1, 2])) + assert request_indices == [[0, 2], [1]] + assert torch.equal(manager._hidden_restore_idx, torch.tensor([0, 4, 1])) assert torch.equal( manager._padded_gather_idx, - torch.tensor([0, 1, 2, 0, 0, 1, 2, 0]), + torch.tensor([0, 2, 0, 0, 1, 0, 0, 0]), ) assert torch.equal( manager._gathered_kv_write_mask, - torch.tensor([True, True, True, False, False, False, False, False]), + torch.tensor([True, True, False, False, True, False, False, False]), ) @@ -131,6 +134,7 @@ def test_input_buffers_are_exposed_for_cudagraph_capture(): pcp_world_size=2, pcp_rank=0, device=torch.device("cpu"), + shard_decode_requests=False, max_num_reqs=4, max_num_tokens=8, ) @@ -148,7 +152,7 @@ def test_input_buffers_are_exposed_for_cudagraph_capture(): (2, [7], [True], 4), (2, [3], [False], 3), (2, [3, 8], [False, True], 7), - (4, [2, 9], [False, True], 5), + (4, [2, 9], [False, True], 4), ], ) def test_num_tokens_for_dispatch_uses_largest_pcp_rank( @@ -158,6 +162,7 @@ def test_num_tokens_for_dispatch_uses_largest_pcp_rank( pcp_world_size=pcp_world_size, pcp_rank=0, device=torch.device("cpu"), + shard_decode_requests=True, ) actual = manager.get_num_tokens_for_dispatch( @@ -173,6 +178,7 @@ def test_graph_padding_cannot_be_smaller_than_largest_pcp_rank(monkeypatch): pcp_world_size=2, pcp_rank=0, device=torch.device("cpu"), + shard_decode_requests=True, dcp_world_size=1, ) monkeypatch.setattr(pcp_manager_module, "async_tensor_h2d", _copy_to_cpu) @@ -183,7 +189,7 @@ def test_graph_padding_cannot_be_smaller_than_largest_pcp_rank(monkeypatch): num_computed_tokens=np.full(3, 16, dtype=np.int32), is_prefilling=np.zeros(3, dtype=np.bool_), query_start_loc_np=np.arange(4, dtype=np.int32), - padded_num_tokens=2, + padded_num_tokens=1, ) @@ -207,6 +213,7 @@ def test_partition_padding_is_derived_from_batch_descriptor( pcp_world_size=2, pcp_rank=0, device=device, + shard_decode_requests=False, max_num_reqs=4, max_num_tokens=8, ) @@ -288,6 +295,189 @@ def test_dummy_context_updates_pcp_local_block_tables(): assert torch.all(global_block_table == -1) +def _rank_request_ids( + manager: PCPManager, + rank: int, + req_ids: list[str], + *, + is_prefilling: np.ndarray | None = None, +) -> list[str]: + num_reqs = len(req_ids) + if is_prefilling is None: + is_prefilling = np.zeros(num_reqs, dtype=np.bool_) + segments = manager._get_rank_segments( + rank=rank, + num_scheduled_tokens=np.ones(num_reqs, dtype=np.int32), + is_prefilling=is_prefilling, + query_start_loc_np=np.arange(num_reqs + 1, dtype=np.int32), + ) + return [req_ids[segment.global_batch_req_idx] for segment in segments] + + +def test_pcp_only_decode_requests_are_round_robin_balanced_each_step(): + manager = PCPManager( + pcp_world_size=4, + pcp_rank=0, + device=torch.device("cpu"), + shard_decode_requests=True, + dcp_world_size=1, + ) + req_ids = [f"request-{idx}" for idx in range(18)] + + owners: dict[str, int] = {} + for rank in range(manager.pcp_world_size): + for req_id in _rank_request_ids(manager, rank, req_ids): + assert req_id not in owners + owners[req_id] = rank + + assert owners == { + req_id: index % manager.pcp_world_size for index, req_id in enumerate(req_ids) + } + counts = [list(owners.values()).count(rank) for rank in range(4)] + assert max(counts) - min(counts) == 1 + + reordered_req_ids = req_ids[::2] + req_ids[1::2] + reordered_owners = { + req_id: rank + for rank in range(manager.pcp_world_size) + for req_id in _rank_request_ids(manager, rank, reordered_req_ids) + } + assert reordered_owners == { + req_id: index % manager.pcp_world_size + for index, req_id in enumerate(reordered_req_ids) + } + assert reordered_owners != owners + + +def test_decode_requests_remain_replicated_when_dcp_is_enabled(): + manager = PCPManager( + pcp_world_size=2, + pcp_rank=0, + device=torch.device("cpu"), + shard_decode_requests=False, + dcp_world_size=2, + ) + req_ids = ["request-a", "request-b", "request-c"] + + assert _rank_request_ids(manager, 0, req_ids) == req_ids + assert _rank_request_ids(manager, 1, req_ids) == req_ids + + +def test_decode_sharding_allows_ranks_with_no_owned_request(): + manager = PCPManager( + pcp_world_size=4, + pcp_rank=0, + device=torch.device("cpu"), + shard_decode_requests=True, + dcp_world_size=1, + ) + req_ids = ["request-a"] + + assert _rank_request_ids(manager, 0, req_ids) == req_ids + assert _rank_request_ids(manager, 1, req_ids) == [] + assert _rank_request_ids(manager, 2, req_ids) == [] + assert _rank_request_ids(manager, 3, req_ids) == [] + + +def test_prefill_partitioning_is_preserved_with_sharded_decode(): + manager = PCPManager( + pcp_world_size=2, + pcp_rank=0, + device=torch.device("cpu"), + shard_decode_requests=True, + dcp_world_size=1, + ) + segments_by_rank = [ + manager._get_rank_segments( + rank=rank, + num_scheduled_tokens=np.array([8, 1, 0, 1], dtype=np.int32), + is_prefilling=np.array([True, False, False, False]), + query_start_loc_np=np.array([0, 8, 9, 9, 10], dtype=np.int32), + ) + for rank in range(manager.pcp_world_size) + ] + + prefill_tokens = sorted( + token_idx + for segments in segments_by_rank + for segment in segments + if segment.global_batch_req_idx == 0 + for token_idx in range( + segment.global_batch_slice.start, segment.global_batch_slice.stop + ) + ) + decode_owners = { + segment.global_batch_req_idx: rank + for rank, segments in enumerate(segments_by_rank) + for segment in segments + if segment.global_batch_req_idx in (1, 3) + } + + assert prefill_tokens == list(range(8)) + assert decode_owners == {1: 0, 3: 1} + + +def test_sharded_decode_layout_selects_owner_kv_for_replication(monkeypatch): + manager = PCPManager( + pcp_world_size=2, + pcp_rank=0, + device=torch.device("cpu"), + shard_decode_requests=True, + dcp_world_size=1, + ) + monkeypatch.setattr(pcp_manager_module, "async_tensor_h2d", _copy_to_cpu) + manager._build_batch_layout( + num_scheduled_tokens=np.array([1, 1, 1], dtype=np.int32), + num_computed_tokens=np.array([16, 16, 16], dtype=np.int32), + is_prefilling=np.array([False, False, False]), + query_start_loc_np=np.array([0, 1, 2, 3], dtype=np.int32), + ) + + gathered_slot_mapping = manager._convert_to_gathered_slot_mappings( + torch.tensor([[123, 456, 789]], dtype=torch.int64) + ) + assert torch.equal( + gathered_slot_mapping, + torch.tensor([[123, 789, 456, PAD_SLOT_ID]], dtype=torch.int64), + ) + assert torch.equal(manager._hidden_restore_idx, torch.tensor([0, 2, 1])) + + class FakePCPGroup: + world_size = 2 + + def all_gather(self, tensor: torch.Tensor, dim: int = 0) -> torch.Tensor: + assert dim == 0 + assert tensor.shape == (2, 1) + return torch.tensor([[11.0], [33.0], [22.0], [0.0]]) + + monkeypatch.setattr(attention_pcp, "get_pcp_group", FakePCPGroup) + (gathered_kv,), cache_slot_mapping = attention_pcp._gather_prefill_cache_inputs( + (torch.tensor([[11.0], [33.0]]),), + gathered_slot_mapping[0], + num_decode_tokens=0, + shard_decode_requests=True, + ) + + assert torch.equal(gathered_kv, torch.tensor([[11.0], [33.0], [22.0], [0.0]])) + assert torch.equal(cache_slot_mapping, torch.tensor([123, 789, 456, PAD_SLOT_ID])) + + +def test_replicated_draft_cache_inputs_keep_unexpanded_slot_mapping(monkeypatch): + def unexpected_gather(): + pytest.fail("replicated draft decode must not gather PCP cache inputs") + + monkeypatch.setattr(attention_pcp, "get_pcp_group", unexpected_gather) + kv = torch.tensor([[11.0], [22.0], [0.0]]) + slot_mapping = torch.tensor([123, 456]) + + (cache_kv,), cache_slot_mapping = attention_pcp._gather_prefill_cache_inputs( + (kv,), slot_mapping, num_decode_tokens=2, shard_decode_requests=True + ) + + assert torch.equal(cache_kv, kv[:2]) + assert torch.equal(cache_slot_mapping, slot_mapping) + + def _rank_rows( pcp_rank: int, pcp_world_size: int, @@ -300,6 +490,7 @@ def _rank_rows( pcp_world_size=pcp_world_size, pcp_rank=pcp_rank, device=torch.device("cpu"), + shard_decode_requests=False, dcp_world_size=2, ) query_start_loc_np = np.concatenate([[0], np.cumsum(num_scheduled_tokens)]).astype( @@ -393,6 +584,7 @@ def test_dcp_replicates_prefills_too_short_to_split(query_len): pcp_world_size=pcp_world_size, pcp_rank=rank, device=torch.device("cpu"), + shard_decode_requests=False, dcp_world_size=2, ) assert manager.replicated_requests(num_scheduled_tokens, is_prefilling)[0] @@ -418,6 +610,7 @@ def test_pcp_first_chunk_row_is_never_short(pcp_world_size, query_len): pcp_world_size=pcp_world_size, pcp_rank=rank, device=torch.device("cpu"), + shard_decode_requests=False, dcp_world_size=pcp_world_size, ) chunk_lens = [ @@ -493,6 +686,7 @@ def test_partition_defers_dcp_metadata_to_post_partition_batch(): pcp_world_size=2, pcp_rank=0, device=device, + shard_decode_requests=False, max_num_reqs=4, max_num_tokens=8, dcp_world_size=2, @@ -541,3 +735,19 @@ def test_partition_defers_dcp_metadata_to_post_partition_batch(): ) assert local_batch.dcp_local_seq_lens is not None assert torch.equal(local_batch.dcp_local_seq_lens.cpu(), expected) + + +@pytest.mark.parametrize( + ("pcp_world_size", "dcp_world_size", "expected"), + [(1, 1, False), (2, 1, True), (2, 2, False)], +) +def test_parallel_config_manages_decode_sharding( + pcp_world_size: int, dcp_world_size: int, expected: bool +): + parallel_config = ParallelConfig( + prefill_context_parallel_size=pcp_world_size, + decode_context_parallel_size=dcp_world_size, + distributed_executor_backend="ray", + ) + + assert parallel_config.pcp_shard_decode_requests is expected diff --git a/vllm/config/parallel.py b/vllm/config/parallel.py index f4b825eb33..53a28fa691 100644 --- a/vllm/config/parallel.py +++ b/vllm/config/parallel.py @@ -599,6 +599,19 @@ class ParallelConfig: """Process world size across TP, PCP, PP, and DP.""" return self.world_size * self.data_parallel_size + @property + def pcp_shard_decode_requests(self) -> bool: + """Whether PCP can shard decode requests across its ranks. + + PCP-only execution replicates the KV cache, so each decode request can + have a single PCP owner. DCP shards the KV cache and therefore requires + every decode request to run on every participating DCP rank. + """ + return ( + self.prefill_context_parallel_size > 1 + and self.decode_context_parallel_size == 1 + ) + @property def use_ubatching(self) -> bool: return self.enable_dbo or self.ubatch_size > 1 diff --git a/vllm/model_executor/layers/attention/mla_attention.py b/vllm/model_executor/layers/attention/mla_attention.py index e016e5daff..b952a36821 100644 --- a/vllm/model_executor/layers/attention/mla_attention.py +++ b/vllm/model_executor/layers/attention/mla_attention.py @@ -609,6 +609,7 @@ class MLAAttention(nn.Module, AttentionLayerBase): vllm_config = get_current_vllm_config() parallel_config = vllm_config.parallel_config self.use_pcp = parallel_config.prefill_context_parallel_size > 1 + self.pcp_shard_decode_requests = parallel_config.pcp_shard_decode_requests compilation_config = vllm_config.compilation_config if prefix in compilation_config.static_forward_context: raise ValueError(f"Duplicate layer name: {prefix}") @@ -784,6 +785,7 @@ class MLAAttention(nn.Module, AttentionLayerBase): slot_mapping, attn_metadata.num_decode_tokens if attn_metadata is not None else None, self.use_pcp, + pcp_shard_decode_requests=self.pcp_shard_decode_requests, ) assert slot_mapping is not None if cache is not None: diff --git a/vllm/model_executor/layers/sparse_attn_indexer.py b/vllm/model_executor/layers/sparse_attn_indexer.py index ae45b77dce..f8fedf046a 100644 --- a/vllm/model_executor/layers/sparse_attn_indexer.py +++ b/vllm/model_executor/layers/sparse_attn_indexer.py @@ -335,6 +335,7 @@ def sparse_attn_indexer( topk_indices_buffer: torch.Tensor, skip_k_cache_insert: bool, use_pcp: bool, + pcp_shard_decode_requests: bool, dense_mha_metadata_layer_name: LayerNameType, use_fp4_cache: bool = False, dcp_rank: int = 0, @@ -404,6 +405,7 @@ def sparse_attn_indexer( topk_indices_buffer, skip_k_cache_insert, use_pcp, + pcp_shard_decode_requests, dense_mha_metadata_layer_name, use_fp4_cache, candidate_blocks=candidate_blocks, @@ -442,6 +444,7 @@ def sparse_attn_indexer( slot_mapping, num_decode_tokens, use_pcp, + pcp_shard_decode_requests=pcp_shard_decode_requests, ) # scale_fmt can be None, but the function expects str assert scale_fmt is not None @@ -817,6 +820,7 @@ def sparse_attn_indexer_fake( topk_indices_buffer: torch.Tensor | None, skip_k_cache_insert: bool, use_pcp: bool, + pcp_shard_decode_requests: bool, dense_mha_metadata_layer_name: LayerNameType, use_fp4_cache: bool = False, dcp_rank: int = 0, @@ -898,6 +902,7 @@ class SparseAttnIndexer(CustomOp): self.dcp_world_size = parallel_config.decode_context_parallel_size self.dcp_rank = get_dcp_group().rank_in_group if self.dcp_world_size > 1 else 0 self.use_pcp = parallel_config.prefill_context_parallel_size > 1 + self.pcp_shard_decode_requests = parallel_config.pcp_shard_decode_requests self._cp_kv_cache_interleave_size: int | None = None if current_platform.is_cuda() and not has_deep_gemm(): raise RuntimeError( @@ -994,6 +999,7 @@ class SparseAttnIndexer(CustomOp): self.topk_indices_buffer, self.skip_k_cache_insert, self.use_pcp, + self.pcp_shard_decode_requests, _encode_layer_name(self.dense_mha_metadata_layer_name), self.use_fp4_cache, self.dcp_rank, diff --git a/vllm/models/deepseek_v32/attention.py b/vllm/models/deepseek_v32/attention.py index d352002009..384b96dbf8 100644 --- a/vllm/models/deepseek_v32/attention.py +++ b/vllm/models/deepseek_v32/attention.py @@ -500,6 +500,7 @@ class DeepseekV32Attention(MLAAttention): self.topk_indices_buffer, skip_k_cache_insert=not self.use_pcp, use_pcp=self.use_pcp, + pcp_shard_decode_requests=self.pcp_shard_decode_requests, dense_mha_metadata_layer_name=self._dense_mha_metadata_layer_name, dcp_rank=( self.dcp_manager.group.rank_in_group @@ -535,6 +536,7 @@ class DeepseekV32Attention(MLAAttention): layer_slot_mapping, attn_metadata.num_decode_tokens, True, + pcp_shard_decode_requests=self.pcp_shard_decode_requests, ) ) self.impl.do_kv_cache_update( # type: ignore[attr-defined] diff --git a/vllm/v1/attention/backends/mla/flashmla_sparse.py b/vllm/v1/attention/backends/mla/flashmla_sparse.py index 8371fb8b85..1c3756e6c4 100644 --- a/vllm/v1/attention/backends/mla/flashmla_sparse.py +++ b/vllm/v1/attention/backends/mla/flashmla_sparse.py @@ -489,6 +489,14 @@ class FlashMLASparseMetadataBuilder( num_tokens - metadata.num_decode_tokens, ) + FP8Meta = FlashMLASparseMetadata.FP8SeparatePrefillDecode + # PCP decode sharding can leave a rank with only its collective-padding + # row when the global decode batch is smaller than the PCP world size. + # The row has no actual query tokens and must not be interpreted as a + # zero-length decode request by the sparse FP8 metadata builder. + if num_tokens == 0: + return FP8Meta() + decode_query_len = 0 active_num_decodes = num_decodes if num_decodes > 0: @@ -498,7 +506,6 @@ class FlashMLASparseMetadataBuilder( active_num_decodes = num_decode_tokens // decode_query_len assert active_num_decodes * decode_query_len == num_decode_tokens - FP8Meta = FlashMLASparseMetadata.FP8SeparatePrefillDecode fp8_metadata = FP8Meta( num_decodes=active_num_decodes, num_prefills=num_prefills, diff --git a/vllm/v1/attention/backends/mla/indexer.py b/vllm/v1/attention/backends/mla/indexer.py index afaae5e35f..20a52af82b 100644 --- a/vllm/v1/attention/backends/mla/indexer.py +++ b/vllm/v1/attention/backends/mla/indexer.py @@ -1298,17 +1298,6 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder): slot_mapping = common_attn_metadata.slot_mapping block_table = common_attn_metadata.block_table_tensor dcp_local_seq_lens = common_attn_metadata.dcp_local_seq_lens - num_decodes, num_prefills, num_decode_tokens, num_prefill_tokens = ( - split_decodes_and_prefills( - common_attn_metadata, - decode_threshold=self.decode_threshold, - require_uniform=not (self.use_flattening or self.supports_varlen), - treat_short_extends_as_decodes=not self.use_pcp, - ) - ) - - assert num_decodes + num_prefills == num_reqs - assert num_decode_tokens + num_prefill_tokens == num_tokens compressed_slot_mapping = slot_mapping indexer_block_table = block_table @@ -1347,6 +1336,34 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder): dim=0, ) + # PCP decode sharding keeps a zero-token placeholder on ranks that own + # no request in a step so collectives retain a uniform rank shape. Do + # not turn that placeholder into a zero-length indexer decode request. + if num_tokens == 0: + return DeepseekV32IndexerMetadata( + seq_lens=seq_lens, + max_seq_len=common_attn_metadata.max_seq_len, + slot_mapping=compressed_slot_mapping, + num_decodes=0, + num_decode_tokens=0, + num_prefills=0, + num_prefill_tokens=0, + prefill=None, + decode=None, + ) + + num_decodes, num_prefills, num_decode_tokens, num_prefill_tokens = ( + split_decodes_and_prefills( + common_attn_metadata, + decode_threshold=self.decode_threshold, + require_uniform=not (self.use_flattening or self.supports_varlen), + treat_short_extends_as_decodes=not self.use_pcp, + ) + ) + + assert num_decodes + num_prefills == num_reqs + assert num_decode_tokens + num_prefill_tokens == num_tokens + prefill_metadata = None if num_prefills > 0: compressed_seq_lens = ( diff --git a/vllm/v1/attention/ops/pcp.py b/vllm/v1/attention/ops/pcp.py index 7e7dbe2245..97aeba038d 100644 --- a/vllm/v1/attention/ops/pcp.py +++ b/vllm/v1/attention/ops/pcp.py @@ -12,29 +12,42 @@ def _gather_prefill_cache_inputs( tensors: tuple[torch.Tensor, ...], slot_mapping: torch.Tensor, num_decode_tokens: int, + shard_decode_requests: bool = False, ) -> tuple[tuple[torch.Tensor, ...], torch.Tensor]: - """Keep replicated decode writes local and gather partitioned prefills.""" + """Gather PCP cache inputs while preserving replicated KV-cache state. + + PCP-only execution shards decode requests across ranks. In that mode each + rank must gather the other owners' decode KV as well as partitioned prefill + KV so every PCP rank retains a complete cache replica. DCP execution keeps + decode requests replicated and uses the legacy prefill-only gather. + """ local_num_tokens = tensors[0].shape[0] assert all(tensor.shape[0] == local_num_tokens for tensor in tensors) assert 0 <= num_decode_tokens <= local_num_tokens # Replicated draft decodes use unexpanded slot mappings, even with DP padding. - if ( - num_decode_tokens == local_num_tokens - or slot_mapping.shape[0] <= local_num_tokens - ): + if slot_mapping.shape[0] <= local_num_tokens: return ( tuple(tensor[:num_decode_tokens] for tensor in tensors), slot_mapping[:num_decode_tokens], ) pcp_group = get_pcp_group() + pcp_size = pcp_group.world_size + gathered_slot_mapping = slot_mapping[: pcp_size * local_num_tokens] + if shard_decode_requests: + gathered_inputs = tuple( + pcp_group.all_gather(tensor.contiguous(), dim=0) for tensor in tensors + ) + return gathered_inputs, gathered_slot_mapping + + if num_decode_tokens == local_num_tokens: + return tensors, slot_mapping[:num_decode_tokens] + gathered_prefills = tuple( pcp_group.all_gather(tensor[num_decode_tokens:].contiguous(), dim=0) for tensor in tensors ) - pcp_size = pcp_group.world_size - gathered_slot_mapping = slot_mapping[: pcp_size * local_num_tokens] if num_decode_tokens == 0: return gathered_prefills, gathered_slot_mapping @@ -58,6 +71,7 @@ def maybe_gather_mla_latent_cache_inputs( slot_mapping: torch.Tensor | None, num_decode_tokens: int | None, use_pcp: bool, + pcp_shard_decode_requests: bool = False, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]: if not use_pcp or num_decode_tokens is None: return kv_c_normed, k_pe, slot_mapping @@ -68,6 +82,7 @@ def maybe_gather_mla_latent_cache_inputs( (kv_c_normed, k_pe_flat), slot_mapping, num_decode_tokens, + pcp_shard_decode_requests, ) cache_k_pe = cache_k_pe_flat.view(-1, *k_pe.shape[1:]) return cache_kv_c, cache_k_pe, cache_slot_mapping @@ -78,11 +93,12 @@ def maybe_gather_indexer_k( slot_mapping: torch.Tensor, num_decode_tokens: int, use_pcp: bool, + pcp_shard_decode_requests: bool = False, ) -> tuple[torch.Tensor, torch.Tensor]: if not use_pcp: return k, slot_mapping (cache_k,), cache_slot_mapping = _gather_prefill_cache_inputs( - (k,), slot_mapping, num_decode_tokens + (k,), slot_mapping, num_decode_tokens, pcp_shard_decode_requests ) return cache_k, cache_slot_mapping diff --git a/vllm/v1/worker/gpu/pcp_manager.py b/vllm/v1/worker/gpu/pcp_manager.py index 89e6b1064b..6aec96bfe1 100644 --- a/vllm/v1/worker/gpu/pcp_manager.py +++ b/vllm/v1/worker/gpu/pcp_manager.py @@ -49,6 +49,7 @@ class PCPManager: pcp_world_size: int, pcp_rank: int, device: torch.device, + shard_decode_requests: bool, max_num_reqs: int | None = None, max_num_tokens: int | None = None, block_tables: BlockTables | None = None, @@ -62,6 +63,7 @@ class PCPManager: self.dcp_world_size = dcp_world_size self.dcp_rank = dcp_rank self.cp_interleave = cp_interleave + self.shard_decode_requests = shard_decode_requests self._global_batch: InputBatch | None = None self._local_batch: InputBatch | None = None @@ -248,6 +250,7 @@ class PCPManager: Decodes, and prefills too short to fill all eight, are replicated instead. """ + decode_ordinal = 0 num_chunks = 2 * self.pcp_world_size replicated = self.replicated_requests(num_scheduled_tokens, is_prefilling) for global_batch_req_idx, num_tokens in enumerate(num_scheduled_tokens): @@ -258,7 +261,16 @@ class PCPManager: if not replicated[global_batch_req_idx]: chunk_size = (query_len + num_chunks - 1) // num_chunks chunk_indices = (rank, num_chunks - 1 - rank) - else: # decodes, and short prefills under DCP, are replicated + elif self.shard_decode_requests: + chunk_size = query_len + # KV and hidden states are gathered back to every PCP rank, so + # decode ownership does not need to persist across steps. Use a + # compact decode-only ordinal to keep each step exactly + # balanced even when prefills and zero-token rows are present. + owner_rank = decode_ordinal % self.pcp_world_size + decode_ordinal += 1 + chunk_indices = (0,) if rank == owner_rank else () + else: # DCP requires decode queries on every participating rank. chunk_size = query_len chunk_indices = (0,) @@ -346,8 +358,18 @@ class PCPManager: segment.global_batch_slice.stop, dtype=np.int64, ) - # Cache insertion pairs one slot entry with each rank's local decode. - if replicated[segment.global_batch_req_idx] and rank != 0: + # Replicated rows contain identical KV, so rank 0 is the + # canonical writer. A sharded decode has exactly one owner and + # therefore needs no de-duplication here. + is_sharded_decode = ( + not bool(is_prefilling[segment.global_batch_req_idx]) + and self.shard_decode_requests + ) + if ( + replicated[segment.global_batch_req_idx] + and not is_sharded_decode + and rank != 0 + ): continue gathered_kv_write_mask[padded_gathered_slice] = True hidden_restore_idx[segment.global_batch_slice] = np.arange( @@ -829,6 +851,7 @@ def maybe_build_pcp_manager( pcp_world_size=pcp_size, pcp_rank=pcp_rank, device=device, + shard_decode_requests=parallel_config.pcp_shard_decode_requests, max_num_reqs=vllm_config.scheduler_config.max_num_seqs, max_num_tokens=vllm_config.scheduler_config.max_num_batched_tokens, block_tables=block_tables,