[Perf][PCP] Shard decode requests across PCP ranks (#52162)
Signed-off-by: QiuChunshuo <qiuchunshuo@huawei.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
co-authored by
Lucas Wilkinson
parent
a810316c08
commit
8486f1ebfe
@@ -205,6 +205,7 @@ def test_short_prefill_updates_k_cache_before_scoring_decision(
|
||||
topk_indices,
|
||||
False,
|
||||
False,
|
||||
False,
|
||||
MLA_LAYER,
|
||||
)
|
||||
|
||||
@@ -259,6 +260,7 @@ def test_skipped_k_cache_insert_accepts_no_k(
|
||||
topk_indices,
|
||||
True,
|
||||
False,
|
||||
False,
|
||||
"",
|
||||
)
|
||||
|
||||
|
||||
@@ -15,6 +15,7 @@ from vllm.models.deepseek_v41.sparse_mla import (
|
||||
DeepseekV4SparseMLABackend as DeepseekV41SparseMLABackend,
|
||||
)
|
||||
from vllm.v1.attention.backend import CommonAttentionMetadata
|
||||
from vllm.v1.attention.backends.mla import indexer as indexer_module
|
||||
from vllm.v1.attention.backends.mla.compressor_utils import (
|
||||
CompressedSlotMappingKernel,
|
||||
get_compressed_slot_mapping,
|
||||
@@ -295,6 +296,78 @@ def test_index_conversion_warmup_uses_physical_block_stride():
|
||||
assert {key.block_stride_rows for key in keys} == {4096}
|
||||
|
||||
|
||||
def test_zero_token_pcp_rank_participates_in_compressed_mapping_gather(monkeypatch):
|
||||
builder = object.__new__(DeepseekV32IndexerMetadataBuilder)
|
||||
builder.compress_ratio = 4
|
||||
builder.use_pcp = True
|
||||
builder.pcp_world_size = 4
|
||||
builder.pcp_rank = 0
|
||||
builder.kernel_block_size = None
|
||||
builder.kv_cache_spec = SimpleNamespace(block_size=64, num_states=64)
|
||||
builder.compressed_slot_mapping_buffer = torch.zeros(8, dtype=torch.int64)
|
||||
|
||||
def fake_get_compressed_slot_mapping(
|
||||
num_tokens,
|
||||
slot_mapping,
|
||||
query_start_loc,
|
||||
seq_lens,
|
||||
block_table,
|
||||
block_size,
|
||||
compress_ratio,
|
||||
out,
|
||||
):
|
||||
assert num_tokens == 0
|
||||
assert block_size == 64
|
||||
assert compress_ratio == 4
|
||||
out.fill_(-1)
|
||||
return out[:num_tokens]
|
||||
|
||||
gathered_slot_mapping = torch.tensor([123, -1, -1, -1], dtype=torch.int64)
|
||||
|
||||
class FakePCPGroup:
|
||||
def __init__(self):
|
||||
self.calls = 0
|
||||
|
||||
def all_gather(self, tensor, dim=0):
|
||||
self.calls += 1
|
||||
assert dim == 0
|
||||
torch.testing.assert_close(tensor, torch.tensor([-1]))
|
||||
return gathered_slot_mapping
|
||||
|
||||
fake_pcp_group = FakePCPGroup()
|
||||
monkeypatch.setattr(
|
||||
indexer_module,
|
||||
"get_compressed_slot_mapping",
|
||||
fake_get_compressed_slot_mapping,
|
||||
)
|
||||
monkeypatch.setattr(indexer_module, "get_pcp_group", lambda: fake_pcp_group)
|
||||
|
||||
query_start_loc = torch.tensor([0, 0], dtype=torch.int32)
|
||||
seq_lens = torch.tensor([16], dtype=torch.int32)
|
||||
common = CommonAttentionMetadata(
|
||||
query_start_loc=query_start_loc,
|
||||
query_start_loc_cpu=query_start_loc,
|
||||
seq_lens=seq_lens,
|
||||
seq_lens_cpu_upper_bound=seq_lens,
|
||||
num_reqs=1,
|
||||
num_actual_tokens=0,
|
||||
max_query_len=0,
|
||||
max_seq_len=16,
|
||||
block_table_tensor=torch.tensor([[0]], dtype=torch.int32),
|
||||
slot_mapping=torch.tensor([11, 22, 33, 44], dtype=torch.int64),
|
||||
causal=True,
|
||||
)
|
||||
|
||||
metadata = builder.build(common_prefix_len=0, common_attn_metadata=common)
|
||||
|
||||
assert fake_pcp_group.calls == 1
|
||||
assert metadata.num_decodes == 0
|
||||
assert metadata.num_decode_tokens == 0
|
||||
assert metadata.num_prefills == 0
|
||||
assert metadata.num_prefill_tokens == 0
|
||||
torch.testing.assert_close(metadata.slot_mapping, gathered_slot_mapping)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
def test_indexer_builder_deepseek_v4_compressed_slot_mapping_uses_num_states():
|
||||
"""Regression test: DeepseekV4 compression path must compute slot_mapping from
|
||||
|
||||
@@ -2499,6 +2499,7 @@ def test_hisparse_kv_update_writes_resident_and_staging_caches():
|
||||
layer = SimpleNamespace(
|
||||
hisparse_cache=cache_handle,
|
||||
use_pcp=False,
|
||||
pcp_shard_decode_requests=False,
|
||||
impl=impl,
|
||||
)
|
||||
mla_attention.MLAAttention.update_kv_cache(
|
||||
|
||||
@@ -132,6 +132,7 @@ def test_piecewise_capture_uses_pcp_dummy_slot_mappings():
|
||||
pcp_world_size=pcp_world_size,
|
||||
pcp_rank=0,
|
||||
device=torch.device("cpu"),
|
||||
shard_decode_requests=False,
|
||||
max_num_reqs=num_reqs,
|
||||
max_num_tokens=num_tokens,
|
||||
block_tables=pcp_block_tables,
|
||||
|
||||
@@ -7,8 +7,9 @@ import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import CUDAGraphMode
|
||||
from vllm.v1.attention.backends.utils import get_dcp_local_seq_lens
|
||||
from vllm.config import CUDAGraphMode, ParallelConfig
|
||||
from vllm.v1.attention.backends.utils import PAD_SLOT_ID, get_dcp_local_seq_lens
|
||||
from vllm.v1.attention.ops import pcp as attention_pcp
|
||||
from vllm.v1.worker.gpu import cp_utils as gpu_cp_utils
|
||||
from vllm.v1.worker.gpu import pcp_manager as pcp_manager_module
|
||||
from vllm.v1.worker.gpu.cudagraph_utils import BatchExecutionDescriptor
|
||||
@@ -53,6 +54,7 @@ def _make_capture_manager(block_table: torch.Tensor):
|
||||
pcp_world_size=2,
|
||||
pcp_rank=0,
|
||||
device=torch.device("cpu"),
|
||||
shard_decode_requests=False,
|
||||
max_num_reqs=4,
|
||||
max_num_tokens=8,
|
||||
block_tables=block_tables,
|
||||
@@ -93,11 +95,12 @@ def test_validate_config_dcp_for_dspark(draft_uses_mla, dcp_size):
|
||||
PCPManager.validate_config(config, supports_mm_inputs=False)
|
||||
|
||||
|
||||
def test_replicated_decode_piecewise_graph_padding(monkeypatch):
|
||||
def test_sharded_decode_piecewise_graph_padding(monkeypatch):
|
||||
manager = PCPManager(
|
||||
pcp_world_size=2,
|
||||
pcp_rank=0,
|
||||
device=torch.device("cpu"),
|
||||
shard_decode_requests=True,
|
||||
dcp_world_size=1,
|
||||
)
|
||||
monkeypatch.setattr(pcp_manager_module, "async_tensor_h2d", _copy_to_cpu)
|
||||
@@ -110,19 +113,19 @@ def test_replicated_decode_piecewise_graph_padding(monkeypatch):
|
||||
padded_num_tokens=4,
|
||||
)
|
||||
|
||||
assert per_rank_num_tokens == [3, 3]
|
||||
assert per_rank_num_tokens == [2, 1]
|
||||
request_indices = [
|
||||
[segment.global_batch_req_idx for segment in rank] for rank in segments_by_rank
|
||||
]
|
||||
assert request_indices == [[0, 1, 2], [0, 1, 2]]
|
||||
assert torch.equal(manager._hidden_restore_idx, torch.tensor([0, 1, 2]))
|
||||
assert request_indices == [[0, 2], [1]]
|
||||
assert torch.equal(manager._hidden_restore_idx, torch.tensor([0, 4, 1]))
|
||||
assert torch.equal(
|
||||
manager._padded_gather_idx,
|
||||
torch.tensor([0, 1, 2, 0, 0, 1, 2, 0]),
|
||||
torch.tensor([0, 2, 0, 0, 1, 0, 0, 0]),
|
||||
)
|
||||
assert torch.equal(
|
||||
manager._gathered_kv_write_mask,
|
||||
torch.tensor([True, True, True, False, False, False, False, False]),
|
||||
torch.tensor([True, True, False, False, True, False, False, False]),
|
||||
)
|
||||
|
||||
|
||||
@@ -131,6 +134,7 @@ def test_input_buffers_are_exposed_for_cudagraph_capture():
|
||||
pcp_world_size=2,
|
||||
pcp_rank=0,
|
||||
device=torch.device("cpu"),
|
||||
shard_decode_requests=False,
|
||||
max_num_reqs=4,
|
||||
max_num_tokens=8,
|
||||
)
|
||||
@@ -148,7 +152,7 @@ def test_input_buffers_are_exposed_for_cudagraph_capture():
|
||||
(2, [7], [True], 4),
|
||||
(2, [3], [False], 3),
|
||||
(2, [3, 8], [False, True], 7),
|
||||
(4, [2, 9], [False, True], 5),
|
||||
(4, [2, 9], [False, True], 4),
|
||||
],
|
||||
)
|
||||
def test_num_tokens_for_dispatch_uses_largest_pcp_rank(
|
||||
@@ -158,6 +162,7 @@ def test_num_tokens_for_dispatch_uses_largest_pcp_rank(
|
||||
pcp_world_size=pcp_world_size,
|
||||
pcp_rank=0,
|
||||
device=torch.device("cpu"),
|
||||
shard_decode_requests=True,
|
||||
)
|
||||
|
||||
actual = manager.get_num_tokens_for_dispatch(
|
||||
@@ -173,6 +178,7 @@ def test_graph_padding_cannot_be_smaller_than_largest_pcp_rank(monkeypatch):
|
||||
pcp_world_size=2,
|
||||
pcp_rank=0,
|
||||
device=torch.device("cpu"),
|
||||
shard_decode_requests=True,
|
||||
dcp_world_size=1,
|
||||
)
|
||||
monkeypatch.setattr(pcp_manager_module, "async_tensor_h2d", _copy_to_cpu)
|
||||
@@ -183,7 +189,7 @@ def test_graph_padding_cannot_be_smaller_than_largest_pcp_rank(monkeypatch):
|
||||
num_computed_tokens=np.full(3, 16, dtype=np.int32),
|
||||
is_prefilling=np.zeros(3, dtype=np.bool_),
|
||||
query_start_loc_np=np.arange(4, dtype=np.int32),
|
||||
padded_num_tokens=2,
|
||||
padded_num_tokens=1,
|
||||
)
|
||||
|
||||
|
||||
@@ -207,6 +213,7 @@ def test_partition_padding_is_derived_from_batch_descriptor(
|
||||
pcp_world_size=2,
|
||||
pcp_rank=0,
|
||||
device=device,
|
||||
shard_decode_requests=False,
|
||||
max_num_reqs=4,
|
||||
max_num_tokens=8,
|
||||
)
|
||||
@@ -288,6 +295,189 @@ def test_dummy_context_updates_pcp_local_block_tables():
|
||||
assert torch.all(global_block_table == -1)
|
||||
|
||||
|
||||
def _rank_request_ids(
|
||||
manager: PCPManager,
|
||||
rank: int,
|
||||
req_ids: list[str],
|
||||
*,
|
||||
is_prefilling: np.ndarray | None = None,
|
||||
) -> list[str]:
|
||||
num_reqs = len(req_ids)
|
||||
if is_prefilling is None:
|
||||
is_prefilling = np.zeros(num_reqs, dtype=np.bool_)
|
||||
segments = manager._get_rank_segments(
|
||||
rank=rank,
|
||||
num_scheduled_tokens=np.ones(num_reqs, dtype=np.int32),
|
||||
is_prefilling=is_prefilling,
|
||||
query_start_loc_np=np.arange(num_reqs + 1, dtype=np.int32),
|
||||
)
|
||||
return [req_ids[segment.global_batch_req_idx] for segment in segments]
|
||||
|
||||
|
||||
def test_pcp_only_decode_requests_are_round_robin_balanced_each_step():
|
||||
manager = PCPManager(
|
||||
pcp_world_size=4,
|
||||
pcp_rank=0,
|
||||
device=torch.device("cpu"),
|
||||
shard_decode_requests=True,
|
||||
dcp_world_size=1,
|
||||
)
|
||||
req_ids = [f"request-{idx}" for idx in range(18)]
|
||||
|
||||
owners: dict[str, int] = {}
|
||||
for rank in range(manager.pcp_world_size):
|
||||
for req_id in _rank_request_ids(manager, rank, req_ids):
|
||||
assert req_id not in owners
|
||||
owners[req_id] = rank
|
||||
|
||||
assert owners == {
|
||||
req_id: index % manager.pcp_world_size for index, req_id in enumerate(req_ids)
|
||||
}
|
||||
counts = [list(owners.values()).count(rank) for rank in range(4)]
|
||||
assert max(counts) - min(counts) == 1
|
||||
|
||||
reordered_req_ids = req_ids[::2] + req_ids[1::2]
|
||||
reordered_owners = {
|
||||
req_id: rank
|
||||
for rank in range(manager.pcp_world_size)
|
||||
for req_id in _rank_request_ids(manager, rank, reordered_req_ids)
|
||||
}
|
||||
assert reordered_owners == {
|
||||
req_id: index % manager.pcp_world_size
|
||||
for index, req_id in enumerate(reordered_req_ids)
|
||||
}
|
||||
assert reordered_owners != owners
|
||||
|
||||
|
||||
def test_decode_requests_remain_replicated_when_dcp_is_enabled():
|
||||
manager = PCPManager(
|
||||
pcp_world_size=2,
|
||||
pcp_rank=0,
|
||||
device=torch.device("cpu"),
|
||||
shard_decode_requests=False,
|
||||
dcp_world_size=2,
|
||||
)
|
||||
req_ids = ["request-a", "request-b", "request-c"]
|
||||
|
||||
assert _rank_request_ids(manager, 0, req_ids) == req_ids
|
||||
assert _rank_request_ids(manager, 1, req_ids) == req_ids
|
||||
|
||||
|
||||
def test_decode_sharding_allows_ranks_with_no_owned_request():
|
||||
manager = PCPManager(
|
||||
pcp_world_size=4,
|
||||
pcp_rank=0,
|
||||
device=torch.device("cpu"),
|
||||
shard_decode_requests=True,
|
||||
dcp_world_size=1,
|
||||
)
|
||||
req_ids = ["request-a"]
|
||||
|
||||
assert _rank_request_ids(manager, 0, req_ids) == req_ids
|
||||
assert _rank_request_ids(manager, 1, req_ids) == []
|
||||
assert _rank_request_ids(manager, 2, req_ids) == []
|
||||
assert _rank_request_ids(manager, 3, req_ids) == []
|
||||
|
||||
|
||||
def test_prefill_partitioning_is_preserved_with_sharded_decode():
|
||||
manager = PCPManager(
|
||||
pcp_world_size=2,
|
||||
pcp_rank=0,
|
||||
device=torch.device("cpu"),
|
||||
shard_decode_requests=True,
|
||||
dcp_world_size=1,
|
||||
)
|
||||
segments_by_rank = [
|
||||
manager._get_rank_segments(
|
||||
rank=rank,
|
||||
num_scheduled_tokens=np.array([8, 1, 0, 1], dtype=np.int32),
|
||||
is_prefilling=np.array([True, False, False, False]),
|
||||
query_start_loc_np=np.array([0, 8, 9, 9, 10], dtype=np.int32),
|
||||
)
|
||||
for rank in range(manager.pcp_world_size)
|
||||
]
|
||||
|
||||
prefill_tokens = sorted(
|
||||
token_idx
|
||||
for segments in segments_by_rank
|
||||
for segment in segments
|
||||
if segment.global_batch_req_idx == 0
|
||||
for token_idx in range(
|
||||
segment.global_batch_slice.start, segment.global_batch_slice.stop
|
||||
)
|
||||
)
|
||||
decode_owners = {
|
||||
segment.global_batch_req_idx: rank
|
||||
for rank, segments in enumerate(segments_by_rank)
|
||||
for segment in segments
|
||||
if segment.global_batch_req_idx in (1, 3)
|
||||
}
|
||||
|
||||
assert prefill_tokens == list(range(8))
|
||||
assert decode_owners == {1: 0, 3: 1}
|
||||
|
||||
|
||||
def test_sharded_decode_layout_selects_owner_kv_for_replication(monkeypatch):
|
||||
manager = PCPManager(
|
||||
pcp_world_size=2,
|
||||
pcp_rank=0,
|
||||
device=torch.device("cpu"),
|
||||
shard_decode_requests=True,
|
||||
dcp_world_size=1,
|
||||
)
|
||||
monkeypatch.setattr(pcp_manager_module, "async_tensor_h2d", _copy_to_cpu)
|
||||
manager._build_batch_layout(
|
||||
num_scheduled_tokens=np.array([1, 1, 1], dtype=np.int32),
|
||||
num_computed_tokens=np.array([16, 16, 16], dtype=np.int32),
|
||||
is_prefilling=np.array([False, False, False]),
|
||||
query_start_loc_np=np.array([0, 1, 2, 3], dtype=np.int32),
|
||||
)
|
||||
|
||||
gathered_slot_mapping = manager._convert_to_gathered_slot_mappings(
|
||||
torch.tensor([[123, 456, 789]], dtype=torch.int64)
|
||||
)
|
||||
assert torch.equal(
|
||||
gathered_slot_mapping,
|
||||
torch.tensor([[123, 789, 456, PAD_SLOT_ID]], dtype=torch.int64),
|
||||
)
|
||||
assert torch.equal(manager._hidden_restore_idx, torch.tensor([0, 2, 1]))
|
||||
|
||||
class FakePCPGroup:
|
||||
world_size = 2
|
||||
|
||||
def all_gather(self, tensor: torch.Tensor, dim: int = 0) -> torch.Tensor:
|
||||
assert dim == 0
|
||||
assert tensor.shape == (2, 1)
|
||||
return torch.tensor([[11.0], [33.0], [22.0], [0.0]])
|
||||
|
||||
monkeypatch.setattr(attention_pcp, "get_pcp_group", FakePCPGroup)
|
||||
(gathered_kv,), cache_slot_mapping = attention_pcp._gather_prefill_cache_inputs(
|
||||
(torch.tensor([[11.0], [33.0]]),),
|
||||
gathered_slot_mapping[0],
|
||||
num_decode_tokens=0,
|
||||
shard_decode_requests=True,
|
||||
)
|
||||
|
||||
assert torch.equal(gathered_kv, torch.tensor([[11.0], [33.0], [22.0], [0.0]]))
|
||||
assert torch.equal(cache_slot_mapping, torch.tensor([123, 789, 456, PAD_SLOT_ID]))
|
||||
|
||||
|
||||
def test_replicated_draft_cache_inputs_keep_unexpanded_slot_mapping(monkeypatch):
|
||||
def unexpected_gather():
|
||||
pytest.fail("replicated draft decode must not gather PCP cache inputs")
|
||||
|
||||
monkeypatch.setattr(attention_pcp, "get_pcp_group", unexpected_gather)
|
||||
kv = torch.tensor([[11.0], [22.0], [0.0]])
|
||||
slot_mapping = torch.tensor([123, 456])
|
||||
|
||||
(cache_kv,), cache_slot_mapping = attention_pcp._gather_prefill_cache_inputs(
|
||||
(kv,), slot_mapping, num_decode_tokens=2, shard_decode_requests=True
|
||||
)
|
||||
|
||||
assert torch.equal(cache_kv, kv[:2])
|
||||
assert torch.equal(cache_slot_mapping, slot_mapping)
|
||||
|
||||
|
||||
def _rank_rows(
|
||||
pcp_rank: int,
|
||||
pcp_world_size: int,
|
||||
@@ -300,6 +490,7 @@ def _rank_rows(
|
||||
pcp_world_size=pcp_world_size,
|
||||
pcp_rank=pcp_rank,
|
||||
device=torch.device("cpu"),
|
||||
shard_decode_requests=False,
|
||||
dcp_world_size=2,
|
||||
)
|
||||
query_start_loc_np = np.concatenate([[0], np.cumsum(num_scheduled_tokens)]).astype(
|
||||
@@ -393,6 +584,7 @@ def test_dcp_replicates_prefills_too_short_to_split(query_len):
|
||||
pcp_world_size=pcp_world_size,
|
||||
pcp_rank=rank,
|
||||
device=torch.device("cpu"),
|
||||
shard_decode_requests=False,
|
||||
dcp_world_size=2,
|
||||
)
|
||||
assert manager.replicated_requests(num_scheduled_tokens, is_prefilling)[0]
|
||||
@@ -418,6 +610,7 @@ def test_pcp_first_chunk_row_is_never_short(pcp_world_size, query_len):
|
||||
pcp_world_size=pcp_world_size,
|
||||
pcp_rank=rank,
|
||||
device=torch.device("cpu"),
|
||||
shard_decode_requests=False,
|
||||
dcp_world_size=pcp_world_size,
|
||||
)
|
||||
chunk_lens = [
|
||||
@@ -493,6 +686,7 @@ def test_partition_defers_dcp_metadata_to_post_partition_batch():
|
||||
pcp_world_size=2,
|
||||
pcp_rank=0,
|
||||
device=device,
|
||||
shard_decode_requests=False,
|
||||
max_num_reqs=4,
|
||||
max_num_tokens=8,
|
||||
dcp_world_size=2,
|
||||
@@ -541,3 +735,19 @@ def test_partition_defers_dcp_metadata_to_post_partition_batch():
|
||||
)
|
||||
assert local_batch.dcp_local_seq_lens is not None
|
||||
assert torch.equal(local_batch.dcp_local_seq_lens.cpu(), expected)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("pcp_world_size", "dcp_world_size", "expected"),
|
||||
[(1, 1, False), (2, 1, True), (2, 2, False)],
|
||||
)
|
||||
def test_parallel_config_manages_decode_sharding(
|
||||
pcp_world_size: int, dcp_world_size: int, expected: bool
|
||||
):
|
||||
parallel_config = ParallelConfig(
|
||||
prefill_context_parallel_size=pcp_world_size,
|
||||
decode_context_parallel_size=dcp_world_size,
|
||||
distributed_executor_backend="ray",
|
||||
)
|
||||
|
||||
assert parallel_config.pcp_shard_decode_requests is expected
|
||||
|
||||
@@ -599,6 +599,19 @@ class ParallelConfig:
|
||||
"""Process world size across TP, PCP, PP, and DP."""
|
||||
return self.world_size * self.data_parallel_size
|
||||
|
||||
@property
|
||||
def pcp_shard_decode_requests(self) -> bool:
|
||||
"""Whether PCP can shard decode requests across its ranks.
|
||||
|
||||
PCP-only execution replicates the KV cache, so each decode request can
|
||||
have a single PCP owner. DCP shards the KV cache and therefore requires
|
||||
every decode request to run on every participating DCP rank.
|
||||
"""
|
||||
return (
|
||||
self.prefill_context_parallel_size > 1
|
||||
and self.decode_context_parallel_size == 1
|
||||
)
|
||||
|
||||
@property
|
||||
def use_ubatching(self) -> bool:
|
||||
return self.enable_dbo or self.ubatch_size > 1
|
||||
|
||||
@@ -609,6 +609,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
vllm_config = get_current_vllm_config()
|
||||
parallel_config = vllm_config.parallel_config
|
||||
self.use_pcp = parallel_config.prefill_context_parallel_size > 1
|
||||
self.pcp_shard_decode_requests = parallel_config.pcp_shard_decode_requests
|
||||
compilation_config = vllm_config.compilation_config
|
||||
if prefix in compilation_config.static_forward_context:
|
||||
raise ValueError(f"Duplicate layer name: {prefix}")
|
||||
@@ -784,6 +785,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
slot_mapping,
|
||||
attn_metadata.num_decode_tokens if attn_metadata is not None else None,
|
||||
self.use_pcp,
|
||||
pcp_shard_decode_requests=self.pcp_shard_decode_requests,
|
||||
)
|
||||
assert slot_mapping is not None
|
||||
if cache is not None:
|
||||
|
||||
@@ -335,6 +335,7 @@ def sparse_attn_indexer(
|
||||
topk_indices_buffer: torch.Tensor,
|
||||
skip_k_cache_insert: bool,
|
||||
use_pcp: bool,
|
||||
pcp_shard_decode_requests: bool,
|
||||
dense_mha_metadata_layer_name: LayerNameType,
|
||||
use_fp4_cache: bool = False,
|
||||
dcp_rank: int = 0,
|
||||
@@ -404,6 +405,7 @@ def sparse_attn_indexer(
|
||||
topk_indices_buffer,
|
||||
skip_k_cache_insert,
|
||||
use_pcp,
|
||||
pcp_shard_decode_requests,
|
||||
dense_mha_metadata_layer_name,
|
||||
use_fp4_cache,
|
||||
candidate_blocks=candidate_blocks,
|
||||
@@ -442,6 +444,7 @@ def sparse_attn_indexer(
|
||||
slot_mapping,
|
||||
num_decode_tokens,
|
||||
use_pcp,
|
||||
pcp_shard_decode_requests=pcp_shard_decode_requests,
|
||||
)
|
||||
# scale_fmt can be None, but the function expects str
|
||||
assert scale_fmt is not None
|
||||
@@ -817,6 +820,7 @@ def sparse_attn_indexer_fake(
|
||||
topk_indices_buffer: torch.Tensor | None,
|
||||
skip_k_cache_insert: bool,
|
||||
use_pcp: bool,
|
||||
pcp_shard_decode_requests: bool,
|
||||
dense_mha_metadata_layer_name: LayerNameType,
|
||||
use_fp4_cache: bool = False,
|
||||
dcp_rank: int = 0,
|
||||
@@ -898,6 +902,7 @@ class SparseAttnIndexer(CustomOp):
|
||||
self.dcp_world_size = parallel_config.decode_context_parallel_size
|
||||
self.dcp_rank = get_dcp_group().rank_in_group if self.dcp_world_size > 1 else 0
|
||||
self.use_pcp = parallel_config.prefill_context_parallel_size > 1
|
||||
self.pcp_shard_decode_requests = parallel_config.pcp_shard_decode_requests
|
||||
self._cp_kv_cache_interleave_size: int | None = None
|
||||
if current_platform.is_cuda() and not has_deep_gemm():
|
||||
raise RuntimeError(
|
||||
@@ -994,6 +999,7 @@ class SparseAttnIndexer(CustomOp):
|
||||
self.topk_indices_buffer,
|
||||
self.skip_k_cache_insert,
|
||||
self.use_pcp,
|
||||
self.pcp_shard_decode_requests,
|
||||
_encode_layer_name(self.dense_mha_metadata_layer_name),
|
||||
self.use_fp4_cache,
|
||||
self.dcp_rank,
|
||||
|
||||
@@ -500,6 +500,7 @@ class DeepseekV32Attention(MLAAttention):
|
||||
self.topk_indices_buffer,
|
||||
skip_k_cache_insert=not self.use_pcp,
|
||||
use_pcp=self.use_pcp,
|
||||
pcp_shard_decode_requests=self.pcp_shard_decode_requests,
|
||||
dense_mha_metadata_layer_name=self._dense_mha_metadata_layer_name,
|
||||
dcp_rank=(
|
||||
self.dcp_manager.group.rank_in_group
|
||||
@@ -535,6 +536,7 @@ class DeepseekV32Attention(MLAAttention):
|
||||
layer_slot_mapping,
|
||||
attn_metadata.num_decode_tokens,
|
||||
True,
|
||||
pcp_shard_decode_requests=self.pcp_shard_decode_requests,
|
||||
)
|
||||
)
|
||||
self.impl.do_kv_cache_update( # type: ignore[attr-defined]
|
||||
|
||||
@@ -489,6 +489,14 @@ class FlashMLASparseMetadataBuilder(
|
||||
num_tokens - metadata.num_decode_tokens,
|
||||
)
|
||||
|
||||
FP8Meta = FlashMLASparseMetadata.FP8SeparatePrefillDecode
|
||||
# PCP decode sharding can leave a rank with only its collective-padding
|
||||
# row when the global decode batch is smaller than the PCP world size.
|
||||
# The row has no actual query tokens and must not be interpreted as a
|
||||
# zero-length decode request by the sparse FP8 metadata builder.
|
||||
if num_tokens == 0:
|
||||
return FP8Meta()
|
||||
|
||||
decode_query_len = 0
|
||||
active_num_decodes = num_decodes
|
||||
if num_decodes > 0:
|
||||
@@ -498,7 +506,6 @@ class FlashMLASparseMetadataBuilder(
|
||||
active_num_decodes = num_decode_tokens // decode_query_len
|
||||
assert active_num_decodes * decode_query_len == num_decode_tokens
|
||||
|
||||
FP8Meta = FlashMLASparseMetadata.FP8SeparatePrefillDecode
|
||||
fp8_metadata = FP8Meta(
|
||||
num_decodes=active_num_decodes,
|
||||
num_prefills=num_prefills,
|
||||
|
||||
@@ -1298,17 +1298,6 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
|
||||
slot_mapping = common_attn_metadata.slot_mapping
|
||||
block_table = common_attn_metadata.block_table_tensor
|
||||
dcp_local_seq_lens = common_attn_metadata.dcp_local_seq_lens
|
||||
num_decodes, num_prefills, num_decode_tokens, num_prefill_tokens = (
|
||||
split_decodes_and_prefills(
|
||||
common_attn_metadata,
|
||||
decode_threshold=self.decode_threshold,
|
||||
require_uniform=not (self.use_flattening or self.supports_varlen),
|
||||
treat_short_extends_as_decodes=not self.use_pcp,
|
||||
)
|
||||
)
|
||||
|
||||
assert num_decodes + num_prefills == num_reqs
|
||||
assert num_decode_tokens + num_prefill_tokens == num_tokens
|
||||
|
||||
compressed_slot_mapping = slot_mapping
|
||||
indexer_block_table = block_table
|
||||
@@ -1347,6 +1336,34 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
|
||||
dim=0,
|
||||
)
|
||||
|
||||
# PCP decode sharding keeps a zero-token placeholder on ranks that own
|
||||
# no request in a step so collectives retain a uniform rank shape. Do
|
||||
# not turn that placeholder into a zero-length indexer decode request.
|
||||
if num_tokens == 0:
|
||||
return DeepseekV32IndexerMetadata(
|
||||
seq_lens=seq_lens,
|
||||
max_seq_len=common_attn_metadata.max_seq_len,
|
||||
slot_mapping=compressed_slot_mapping,
|
||||
num_decodes=0,
|
||||
num_decode_tokens=0,
|
||||
num_prefills=0,
|
||||
num_prefill_tokens=0,
|
||||
prefill=None,
|
||||
decode=None,
|
||||
)
|
||||
|
||||
num_decodes, num_prefills, num_decode_tokens, num_prefill_tokens = (
|
||||
split_decodes_and_prefills(
|
||||
common_attn_metadata,
|
||||
decode_threshold=self.decode_threshold,
|
||||
require_uniform=not (self.use_flattening or self.supports_varlen),
|
||||
treat_short_extends_as_decodes=not self.use_pcp,
|
||||
)
|
||||
)
|
||||
|
||||
assert num_decodes + num_prefills == num_reqs
|
||||
assert num_decode_tokens + num_prefill_tokens == num_tokens
|
||||
|
||||
prefill_metadata = None
|
||||
if num_prefills > 0:
|
||||
compressed_seq_lens = (
|
||||
|
||||
@@ -12,29 +12,42 @@ def _gather_prefill_cache_inputs(
|
||||
tensors: tuple[torch.Tensor, ...],
|
||||
slot_mapping: torch.Tensor,
|
||||
num_decode_tokens: int,
|
||||
shard_decode_requests: bool = False,
|
||||
) -> tuple[tuple[torch.Tensor, ...], torch.Tensor]:
|
||||
"""Keep replicated decode writes local and gather partitioned prefills."""
|
||||
"""Gather PCP cache inputs while preserving replicated KV-cache state.
|
||||
|
||||
PCP-only execution shards decode requests across ranks. In that mode each
|
||||
rank must gather the other owners' decode KV as well as partitioned prefill
|
||||
KV so every PCP rank retains a complete cache replica. DCP execution keeps
|
||||
decode requests replicated and uses the legacy prefill-only gather.
|
||||
"""
|
||||
local_num_tokens = tensors[0].shape[0]
|
||||
assert all(tensor.shape[0] == local_num_tokens for tensor in tensors)
|
||||
assert 0 <= num_decode_tokens <= local_num_tokens
|
||||
|
||||
# Replicated draft decodes use unexpanded slot mappings, even with DP padding.
|
||||
if (
|
||||
num_decode_tokens == local_num_tokens
|
||||
or slot_mapping.shape[0] <= local_num_tokens
|
||||
):
|
||||
if slot_mapping.shape[0] <= local_num_tokens:
|
||||
return (
|
||||
tuple(tensor[:num_decode_tokens] for tensor in tensors),
|
||||
slot_mapping[:num_decode_tokens],
|
||||
)
|
||||
|
||||
pcp_group = get_pcp_group()
|
||||
pcp_size = pcp_group.world_size
|
||||
gathered_slot_mapping = slot_mapping[: pcp_size * local_num_tokens]
|
||||
if shard_decode_requests:
|
||||
gathered_inputs = tuple(
|
||||
pcp_group.all_gather(tensor.contiguous(), dim=0) for tensor in tensors
|
||||
)
|
||||
return gathered_inputs, gathered_slot_mapping
|
||||
|
||||
if num_decode_tokens == local_num_tokens:
|
||||
return tensors, slot_mapping[:num_decode_tokens]
|
||||
|
||||
gathered_prefills = tuple(
|
||||
pcp_group.all_gather(tensor[num_decode_tokens:].contiguous(), dim=0)
|
||||
for tensor in tensors
|
||||
)
|
||||
pcp_size = pcp_group.world_size
|
||||
gathered_slot_mapping = slot_mapping[: pcp_size * local_num_tokens]
|
||||
if num_decode_tokens == 0:
|
||||
return gathered_prefills, gathered_slot_mapping
|
||||
|
||||
@@ -58,6 +71,7 @@ def maybe_gather_mla_latent_cache_inputs(
|
||||
slot_mapping: torch.Tensor | None,
|
||||
num_decode_tokens: int | None,
|
||||
use_pcp: bool,
|
||||
pcp_shard_decode_requests: bool = False,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]:
|
||||
if not use_pcp or num_decode_tokens is None:
|
||||
return kv_c_normed, k_pe, slot_mapping
|
||||
@@ -68,6 +82,7 @@ def maybe_gather_mla_latent_cache_inputs(
|
||||
(kv_c_normed, k_pe_flat),
|
||||
slot_mapping,
|
||||
num_decode_tokens,
|
||||
pcp_shard_decode_requests,
|
||||
)
|
||||
cache_k_pe = cache_k_pe_flat.view(-1, *k_pe.shape[1:])
|
||||
return cache_kv_c, cache_k_pe, cache_slot_mapping
|
||||
@@ -78,11 +93,12 @@ def maybe_gather_indexer_k(
|
||||
slot_mapping: torch.Tensor,
|
||||
num_decode_tokens: int,
|
||||
use_pcp: bool,
|
||||
pcp_shard_decode_requests: bool = False,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
if not use_pcp:
|
||||
return k, slot_mapping
|
||||
(cache_k,), cache_slot_mapping = _gather_prefill_cache_inputs(
|
||||
(k,), slot_mapping, num_decode_tokens
|
||||
(k,), slot_mapping, num_decode_tokens, pcp_shard_decode_requests
|
||||
)
|
||||
return cache_k, cache_slot_mapping
|
||||
|
||||
|
||||
@@ -49,6 +49,7 @@ class PCPManager:
|
||||
pcp_world_size: int,
|
||||
pcp_rank: int,
|
||||
device: torch.device,
|
||||
shard_decode_requests: bool,
|
||||
max_num_reqs: int | None = None,
|
||||
max_num_tokens: int | None = None,
|
||||
block_tables: BlockTables | None = None,
|
||||
@@ -62,6 +63,7 @@ class PCPManager:
|
||||
self.dcp_world_size = dcp_world_size
|
||||
self.dcp_rank = dcp_rank
|
||||
self.cp_interleave = cp_interleave
|
||||
self.shard_decode_requests = shard_decode_requests
|
||||
|
||||
self._global_batch: InputBatch | None = None
|
||||
self._local_batch: InputBatch | None = None
|
||||
@@ -248,6 +250,7 @@ class PCPManager:
|
||||
|
||||
Decodes, and prefills too short to fill all eight, are replicated instead.
|
||||
"""
|
||||
decode_ordinal = 0
|
||||
num_chunks = 2 * self.pcp_world_size
|
||||
replicated = self.replicated_requests(num_scheduled_tokens, is_prefilling)
|
||||
for global_batch_req_idx, num_tokens in enumerate(num_scheduled_tokens):
|
||||
@@ -258,7 +261,16 @@ class PCPManager:
|
||||
if not replicated[global_batch_req_idx]:
|
||||
chunk_size = (query_len + num_chunks - 1) // num_chunks
|
||||
chunk_indices = (rank, num_chunks - 1 - rank)
|
||||
else: # decodes, and short prefills under DCP, are replicated
|
||||
elif self.shard_decode_requests:
|
||||
chunk_size = query_len
|
||||
# KV and hidden states are gathered back to every PCP rank, so
|
||||
# decode ownership does not need to persist across steps. Use a
|
||||
# compact decode-only ordinal to keep each step exactly
|
||||
# balanced even when prefills and zero-token rows are present.
|
||||
owner_rank = decode_ordinal % self.pcp_world_size
|
||||
decode_ordinal += 1
|
||||
chunk_indices = (0,) if rank == owner_rank else ()
|
||||
else: # DCP requires decode queries on every participating rank.
|
||||
chunk_size = query_len
|
||||
chunk_indices = (0,)
|
||||
|
||||
@@ -346,8 +358,18 @@ class PCPManager:
|
||||
segment.global_batch_slice.stop,
|
||||
dtype=np.int64,
|
||||
)
|
||||
# Cache insertion pairs one slot entry with each rank's local decode.
|
||||
if replicated[segment.global_batch_req_idx] and rank != 0:
|
||||
# Replicated rows contain identical KV, so rank 0 is the
|
||||
# canonical writer. A sharded decode has exactly one owner and
|
||||
# therefore needs no de-duplication here.
|
||||
is_sharded_decode = (
|
||||
not bool(is_prefilling[segment.global_batch_req_idx])
|
||||
and self.shard_decode_requests
|
||||
)
|
||||
if (
|
||||
replicated[segment.global_batch_req_idx]
|
||||
and not is_sharded_decode
|
||||
and rank != 0
|
||||
):
|
||||
continue
|
||||
gathered_kv_write_mask[padded_gathered_slice] = True
|
||||
hidden_restore_idx[segment.global_batch_slice] = np.arange(
|
||||
@@ -829,6 +851,7 @@ def maybe_build_pcp_manager(
|
||||
pcp_world_size=pcp_size,
|
||||
pcp_rank=pcp_rank,
|
||||
device=device,
|
||||
shard_decode_requests=parallel_config.pcp_shard_decode_requests,
|
||||
max_num_reqs=vllm_config.scheduler_config.max_num_seqs,
|
||||
max_num_tokens=vllm_config.scheduler_config.max_num_batched_tokens,
|
||||
block_tables=block_tables,
|
||||
|
||||
Reference in New Issue
Block a user