From 7f4793eaa335a3667927a0191868d01f36b170af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Lucchesi?= Date: Sat, 29 Aug 2026 18:03:53 +0200 Subject: [PATCH] [Nixl][PD] DCP support for MLA models (#50611) Signed-off-by: NickLucche --- tests/test_config.py | 30 +++ .../kv_connector/unit/test_nixl_connector.py | 1 + .../unit/test_nixl_connector_hma.py | 182 +++++++++++++++++- tests/v1/kv_connector/unit/test_tp_mapping.py | 79 +++++++- tests/v1/worker/test_gpu_model_runner.py | 71 +++++++ tests/v1/worker/test_gpu_model_runner_v2.py | 7 +- vllm/config/kv_transfer.py | 9 + vllm/config/vllm.py | 93 +++++++-- .../kv_transfer/kv_connector/utils.py | 73 ++++++- .../kv_connector/v1/nixl/base_scheduler.py | 9 +- .../kv_connector/v1/nixl/base_worker.py | 126 ++++++++++-- .../kv_connector/v1/nixl/connector.py | 4 + .../kv_connector/v1/nixl/metadata.py | 17 +- .../kv_connector/v1/nixl/pull_scheduler.py | 15 +- .../kv_connector/v1/nixl/pull_worker.py | 140 +++++++++----- .../kv_connector/v1/nixl/push_scheduler.py | 4 +- .../kv_connector/v1/nixl/tp_mapping.py | 28 ++- .../kv_transfer/kv_transfer_state.py | 2 + .../layers/attention/mla_attention.py | 3 - .../layers/sparse_attn_indexer.py | 18 +- .../attention/backends/mla/flashinfer_mla.py | 1 - vllm/v1/worker/gpu/model_runner.py | 2 + vllm/v1/worker/gpu_model_runner.py | 8 + 23 files changed, 818 insertions(+), 104 deletions(-) diff --git a/tests/test_config.py b/tests/test_config.py index df77d33e0c..4c27a6fe69 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -17,8 +17,11 @@ import vllm.config.vllm as vllm_config_module import vllm.envs as envs from vllm.compilation.backends import VllmBackend from vllm.config import ( + CacheConfig, CompilationConfig, + DeviceConfig, KernelConfig, + KVTransferConfig, ModelConfig, ObservabilityConfig, ParallelConfig, @@ -100,6 +103,32 @@ def test_per_request_spec_decode_metrics_requires_spec_decode(): ) +def test_pd_dcp_interleave_size_is_adjusted_to_block_size(caplog): + config = VllmConfig( + cache_config=CacheConfig(block_size=16), + device_config=DeviceConfig(device="cpu"), + parallel_config=ParallelConfig( + tensor_parallel_size=2, + decode_context_parallel_size=2, + cp_kv_cache_interleave_size=3, + distributed_executor_backend="mp", + ), + kv_transfer_config=KVTransferConfig( + kv_connector="NixlConnector", + kv_role="kv_both", + ), + ) + + kv_cache_config = SimpleNamespace( + kv_cache_groups=[SimpleNamespace(kv_cache_spec=SimpleNamespace(block_size=16))] + ) + with caplog.at_level(logging.INFO): + config.adjust_dcp_kv_cache_interleave_size(kv_cache_config) + + assert config.parallel_config.cp_kv_cache_interleave_size == 16 + assert "automatically adjusted from 3 to block_size 16" in caplog.text + + def test_compile_config_repr_succeeds(): # setup: VllmBackend mutates the config object config = VllmConfig() @@ -1730,6 +1759,7 @@ def test_validate_mamba_align_subblock_prefill(): long_prefill_token_threshold=4096, disable_chunked_mm_input=False, ), + kv_transfer_config=None, ) VllmConfig.validate_block_size(config) diff --git a/tests/v1/kv_connector/unit/test_nixl_connector.py b/tests/v1/kv_connector/unit/test_nixl_connector.py index d4064d6f6f..1f9cfc7b25 100644 --- a/tests/v1/kv_connector/unit/test_nixl_connector.py +++ b/tests/v1/kv_connector/unit/test_nixl_connector.py @@ -507,6 +507,7 @@ class FakeNixlConnectorWorker(NixlConnectorWorker): port: int, remote_tp_size: int, expected_engine_id: str, + remote_dcp_size: int = 1, remote_pp_size: int = 1, notif_agents_only: bool = False, ) -> tuple[dict[tuple[int, int], str], float]: diff --git a/tests/v1/kv_connector/unit/test_nixl_connector_hma.py b/tests/v1/kv_connector/unit/test_nixl_connector_hma.py index f5bf9e9be1..1a0f70d288 100644 --- a/tests/v1/kv_connector/unit/test_nixl_connector_hma.py +++ b/tests/v1/kv_connector/unit/test_nixl_connector_hma.py @@ -3,7 +3,7 @@ """Unit tests for NixlConnectorScheduler with HMA and Mamba N-1 prefill.""" import gc -from unittest.mock import patch +from unittest.mock import MagicMock, patch import pytest import torch @@ -63,6 +63,180 @@ def test_sw_sizes(mock_platform, swa_enabled, expected_sw_sizes): ) +@pytest.mark.cpu_test +@pytest.mark.parametrize( + "use_mla,source_ranks,tp_ratio,expected", + [ + pytest.param(True, (0,), -2, False, id="pure_mla_with_remote_dcp"), + pytest.param(True, (0, 1), -2, True, id="hybrid_mla_ssm"), + pytest.param(False, (0,), -2, True, id="full_attention"), + pytest.param(False, (0,), 2, False, id="local_tp_greater"), + ], +) +def test_needs_split_local_xfer_handles(use_mla, source_ranks, tp_ratio, expected): + """Handle creation and selection must use the same TP-mapping predicate. + + In pure MLA, DCP can target several physical remote workers even though + the TP mapping has one replicated attention source. Those workers own + disjoint blocks, so every read uses the whole local-region handle. + """ + from vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker import ( + NixlConnectorWorker, + ) + + worker = object.__new__(NixlConnectorWorker) + worker.use_mla = use_mla + plan = MagicMock() + plan.all_source_ranks = source_ranks + + assert worker._needs_split_local_xfer_handles(tp_ratio, plan) is expected + + +@pytest.mark.cpu_test +def test_update_state_after_alloc_tracks_cached_blocks_per_group(): + """Hybrid SWA+FA requests can have different prefix-cache-hit counts per + KV cache group. Each group's count must be tracked independently rather + than collapsed to a single scalar, or DCP read-phase alignment + (local_num_computed_blocks) would misalign one of the groups.""" + from vllm.distributed.kv_transfer.kv_connector.v1.nixl.pull_scheduler import ( + NixlPullConnectorScheduler, + ) + from vllm.v1.core.kv_cache_manager import KVCacheBlocks + from vllm.v1.core.kv_cache_utils import KVCacheBlock + + scheduler = object.__new__(NixlPullConnectorScheduler) + scheduler._reqs_in_batch = set() + scheduler._reqs_need_save = {} + scheduler._reqs_need_recv = {} + scheduler.use_host_buffer = False + scheduler.is_bidirectional_kv_xfer_enabled = False + scheduler._is_hma_required = False + scheduler.kv_cache_config = MagicMock( + select_transfer_block_ids=lambda block_ids: block_ids + ) + + def cached(block_id): + return KVCacheBlock(block_id=block_id, _block_hash=object()) + + def uncached(block_id): + return KVCacheBlock(block_id=block_id) + + # Group 0 (full-attention): 2 of 3 blocks cached. + # Group 1 (sliding-window): 1 of 3 blocks cached. + blocks = KVCacheBlocks( + blocks=( + [cached(0), cached(1), uncached(2)], + [cached(10), uncached(11), uncached(12)], + ) + ) + + request = create_request(do_remote_prefill=True) + + scheduler.update_state_after_alloc(request, blocks, num_external_tokens=2) + + _, _, local_num_computed_blocks = scheduler._reqs_need_recv[request.request_id] + assert local_num_computed_blocks == (2, 1) + + +@pytest.mark.cpu_test +@pytest.mark.parametrize( + "local_ids,remote_ids,remote_rank,local_dcp_size,local_dcp_rank," + "remote_dcp_size,local_num_computed_blocks,expected_local,expected_remote", + [ + # local more finely sharded than remote: remote's raw slice has an + # extra tail block beyond what local's phase-aligned slice covers. + pytest.param( + [100, 101, 102], + [200, 201], + 1, + 1, + 0, + 2, + 0, + [101], + [200], + id="remote_tail_padding", + ), + # remote more finely sharded than local: local's raw slice has a + # block beyond what remote's phase-aligned slice covers. + pytest.param( + [100], + [200], + 0, + 2, + 1, + 1, + 0, + [], + [], + id="local_tail_padding", + ), + # Asymmetric DCP with a partial prefix cache hit: the cached count + # shifts both the local phase and the remote start position. + pytest.param( + [100], + [200, 201], + 2, + 2, + 0, + 4, + 3, + [100], + [201], + id="prefix_phase", + ), + # Equal (but active) DCP sizes still need to skip the locally cached + # prefix from the front of remote's list. + pytest.param( + [100, 101], + [200, 201, 202, 203], + 1, + 2, + 1, + 2, + 2, + [100, 101], + [202, 203], + id="equal_dcp_with_prefix", + ), + ], +) +def test_apply_dcp_prefix_caching_matches_global_logical_positions( + local_ids, + remote_ids, + remote_rank, + local_dcp_size, + local_dcp_rank, + remote_dcp_size, + local_num_computed_blocks, + expected_local, + expected_remote, +): + """_apply_dcp_prefix_caching must operate on logical block IDs (DCP + interleaves at logical-block granularity) and always return equal-length + local/remote slices, so a later, unconditional _apply_prefix_caching + trim is a guaranteed no-op rather than double-processing the read.""" + from vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker import ( + NixlConnectorWorker, + ) + + worker = object.__new__(NixlConnectorWorker) + + actual_local, actual_remote = worker._apply_dcp_prefix_caching( + local_ids, + remote_ids, + remote_rank=remote_rank, + local_dcp_size=local_dcp_size, + local_dcp_rank=local_dcp_rank, + remote_dcp_size=remote_dcp_size, + local_num_computed_blocks=local_num_computed_blocks, + ) + + assert actual_local == expected_local + assert actual_remote == expected_remote + assert len(actual_local) == len(actual_remote) + + @pytest.mark.cpu_test def test_logical_to_kernel_block_ids_with_hma(): """Test _logical_to_kernel_block_ids expands blocks when HMA is enabled. @@ -211,7 +385,12 @@ def test_read_blocks_for_req_expands_remote_ids( worker = object.__new__(NixlConnectorWorker) worker._physical_blocks_per_logical_kv_block = local_physical_per_logical worker._engine_last_active = {} + worker._recving_transfers = {} worker._bidirectional_kv_xfer_enabled = False + worker.dcp_size = 1 + worker.dcp_rank = 0 + worker._group_spec_types = resolved_types + worker._has_mamba = any(t is MambaSpec for t in resolved_types) has_mamba = any(t is MambaSpec for t in resolved_types) has_swa = any(t is SlidingWindowSpec for t in resolved_types) @@ -227,6 +406,7 @@ def test_read_blocks_for_req_expands_remote_ids( worker.transfer_topo.tp_ratio.return_value = tp_ratio remote_info = MagicMock() remote_info.remote_physical_blocks_per_logical = remote_physical_per_logical + remote_info.remote_dcp_size = 1 worker.transfer_topo.get_engine_info.return_value = remote_info worker.use_mla = False diff --git a/tests/v1/kv_connector/unit/test_tp_mapping.py b/tests/v1/kv_connector/unit/test_tp_mapping.py index d72bbc9ec2..a08da3cd15 100644 --- a/tests/v1/kv_connector/unit/test_tp_mapping.py +++ b/tests/v1/kv_connector/unit/test_tp_mapping.py @@ -14,6 +14,7 @@ from types import SimpleNamespace import numpy as np import pytest +from vllm.distributed.kv_transfer.kv_connector.utils import TransferTopology from vllm.distributed.kv_transfer.kv_connector.v1.nixl.tp_mapping import ( TPMapping, compute_tp_mapping, @@ -35,17 +36,20 @@ def _compute_mapping( is_mla: bool = False, num_kv_heads: int = 8, group_spec_types: tuple[type, ...] = (FullAttentionSpec,), + dcp_size: int = 1, + remote_dcp_size: int = 1, ) -> TPMapping: - transfer_topology = SimpleNamespace( - tp_rank=tp_rank, - tp_size=tp_size, - is_mla=is_mla, - total_num_kv_heads=num_kv_heads, - ) + transfer_topology = object.__new__(TransferTopology) + transfer_topology.tp_rank = tp_rank + transfer_topology.tp_size = tp_size + transfer_topology.is_mla = is_mla + transfer_topology.total_num_kv_heads = num_kv_heads + transfer_topology.dcp_size = dcp_size return compute_tp_mapping( transfer_topology=transfer_topology, remote_tp_size=remote_tp_size, group_spec_types=group_spec_types, + remote_dcp_size=remote_dcp_size, ) @@ -68,6 +72,69 @@ class TestTPMappingStructure: assert m.all_source_ranks == (0, 1) +@pytest.mark.parametrize( + "tp_rank,tp_size,remote_tp_size,dcp_size,remote_dcp_size,expected_ranks", + [ + (0, 4, 4, 1, 4, (0, 1, 2, 3)), + (2, 4, 4, 4, 4, (2,)), + (0, 2, 4, 2, 4, (0, 2)), + (3, 4, 2, 4, 2, (1,)), + ], +) +def test_mla_dcp_source_ranks( + tp_rank, + tp_size, + remote_tp_size, + dcp_size, + remote_dcp_size, + expected_ranks, +): + mapping = _compute_mapping( + tp_rank=tp_rank, + tp_size=tp_size, + remote_tp_size=remote_tp_size, + is_mla=True, + num_kv_heads=1, + dcp_size=dcp_size, + remote_dcp_size=remote_dcp_size, + ) + + assert mapping.all_source_ranks == expected_ranks + + +@pytest.mark.parametrize( + "tp_size,remote_tp_size,dcp_size,remote_dcp_size", + [ + (4, 2, 4, 2), + (4, 4, 4, 4), + (4, 1, 4, 1), + (4, 4, 1, 4), + (2, 4, 2, 4), + ], +) +def test_dcp_consumer_count_matches_readers( + tp_size, remote_tp_size, dcp_size, remote_dcp_size +): + """Each producer waits for exactly the local ranks that read from it.""" + mappings = [ + _compute_mapping( + tp_rank=tp_rank, + tp_size=tp_size, + remote_tp_size=remote_tp_size, + is_mla=True, + num_kv_heads=1, + dcp_size=dcp_size, + remote_dcp_size=remote_dcp_size, + ) + for tp_rank in range(tp_size) + ] + + for remote_rank in range(remote_tp_size): + readers = [m for m in mappings if remote_rank in m.all_source_ranks] + for mapping in readers: + assert mapping.local_consumers == len(readers) + + # ====================================================================== # Split handle tests # ====================================================================== diff --git a/tests/v1/worker/test_gpu_model_runner.py b/tests/v1/worker/test_gpu_model_runner.py index bd8fc7dc01..99331e7b35 100644 --- a/tests/v1/worker/test_gpu_model_runner.py +++ b/tests/v1/worker/test_gpu_model_runner.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import gc +from contextlib import nullcontext from types import SimpleNamespace from unittest.mock import Mock @@ -1380,6 +1381,76 @@ def test_hybrid_attention_mamba_tensor_shapes(): assert torch.equal(actual_ssm, expected_ssm) +def test_input_batch_reinitialized_after_late_interleave_adjustment(monkeypatch): + runner = object.__new__(GPUModelRunner) + runner.vllm_config = SimpleNamespace(reasoning_config=None) + runner.parallel_config = SimpleNamespace(cp_kv_cache_interleave_size=16) + runner.cache_config = SimpleNamespace(use_replayssm=False) + runner.model_config = SimpleNamespace(get_vocab_size=lambda: 32) + runner.max_model_len = 64 + runner.max_encoder_len = 0 + runner.max_num_reqs = 1 + runner.max_num_tokens = 64 + runner.num_spec_tokens = 0 + runner.device = torch.device("cpu") + runner.is_pooling_model = False + runner._init_block_sizes = [16] + runner._init_kernel_block_sizes = [16] + runner._init_max_num_blocks = [4] + runner._init_slot_mapping_modes = [ + gpu_model_runner_module.SlotMappingMode.TOKEN_TO_KV_SLOT + ] + runner.cp_kv_cache_interleave_size = 1 + runner.input_batch = SimpleNamespace( + logitsprocs=None, + logitsprocs_need_output_token_ids=False, + ) + runner.jit_warmup_registry = Mock() + runner.jit_warmup_registry.activate.return_value = nullcontext() + + spec = SimpleNamespace( + block_size=16, + max_num_blocks_per_req=lambda *_: 4, + ) + kv_cache_config = SimpleNamespace( + kv_cache_groups=[SimpleNamespace(kv_cache_spec=spec)] + ) + input_batch_cls = Mock(return_value=SimpleNamespace()) + monkeypatch.setattr(gpu_model_runner_module, "InputBatch", input_batch_cls) + monkeypatch.setattr( + gpu_model_runner_module, + "get_kv_cache_spec_kind", + lambda _: gpu_model_runner_module.KVCacheSpecKind.FULL_ATTENTION, + ) + + runner.may_reinitialize_input_batch(kv_cache_config, [16]) + + assert input_batch_cls.call_count == 1 + assert input_batch_cls.call_args.kwargs["cp_kv_cache_interleave_size"] == 16 + + +def test_v2_runner_snapshots_late_interleave_adjustment(monkeypatch): + from vllm.v1.worker.gpu import model_runner as v2_model_runner_module + + runner = object.__new__(v2_model_runner_module.GPUModelRunner) + runner.parallel_config = SimpleNamespace(cp_kv_cache_interleave_size=16) + runner.cp_interleave = 1 + + class StopInitialization(Exception): + pass + + monkeypatch.setattr( + v2_model_runner_module, + "deepcopy", + Mock(side_effect=StopInitialization), + ) + + with pytest.raises(StopInitialization): + runner.initialize_kv_cache(SimpleNamespace()) + + assert runner.cp_interleave == 16 + + def test_hybrid_block_table_initialization(): """Test hybrid block table with different kernel and kvcache_manager block sizes.""" diff --git a/tests/v1/worker/test_gpu_model_runner_v2.py b/tests/v1/worker/test_gpu_model_runner_v2.py index 59c11e79c4..8b48993c47 100644 --- a/tests/v1/worker/test_gpu_model_runner_v2.py +++ b/tests/v1/worker/test_gpu_model_runner_v2.py @@ -57,14 +57,19 @@ def test_initialize_kv_cache_does_not_dcp_shard_mamba_block_table( KVCacheGroupSpec(["kda"], mamba_spec), ], ) + parallel_config = SimpleNamespace( + decode_context_parallel_size=dcp_size, + cp_kv_cache_interleave_size=1, + ) vllm_config = SimpleNamespace( - parallel_config=SimpleNamespace(decode_context_parallel_size=dcp_size), + parallel_config=parallel_config, cache_config=SimpleNamespace(mamba_cache_mode=mamba_cache_mode), ) runner = SimpleNamespace( max_model_len=max_model_len, is_encoder_decoder=False, vllm_config=vllm_config, + parallel_config=parallel_config, ) class _CapturedWidths(Exception): diff --git a/vllm/config/kv_transfer.py b/vllm/config/kv_transfer.py index 9f206ff5d4..d1344a5211 100644 --- a/vllm/config/kv_transfer.py +++ b/vllm/config/kv_transfer.py @@ -119,3 +119,12 @@ class KVTransferConfig: def get_from_extra_config(self, key, default) -> Any: return self.kv_connector_extra_config.get(key, default) + + def has_connector(self, connector_name: str) -> bool: + """Whether ``connector_name`` is configured, directly or in MultiConnector.""" + if self.kv_connector == connector_name: + return True + return self.kv_connector == "MultiConnector" and any( + child.get("kv_connector") == connector_name + for child in self.kv_connector_extra_config.get("connectors", []) + ) diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index ea03625f30..e0d64443f0 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -1141,6 +1141,30 @@ class VllmConfig: self._verify_sampling_replay_config() self._verify_trace_replay_config() + # A NIXL side is either fully replicated or fully DCP-sharded; MLA only. + if ( + self.kv_transfer_config is not None + and self.kv_transfer_config.has_connector("NixlConnector") + ): + assert self.parallel_config.prefill_context_parallel_size == 1, ( + "NIXL does not support prefill context parallelism." + ) + dcp_size = self.parallel_config.decode_context_parallel_size + tp_size = self.parallel_config.tensor_parallel_size + assert dcp_size in (1, tp_size), ( + f"decode_context_parallel_size={dcp_size} must be 1 or equal " + f"to tensor_parallel_size={tp_size} when using NixlConnector." + ) + if self.model_config is not None: + assert self.model_config.use_mla or dcp_size == 1, ( + "PD with decode_context_parallel_size > 1 is only " + "supported for MLA models." + ) + assert not (self.model_config.is_hybrid and dcp_size > 1), ( + "PD with decode_context_parallel_size > 1 is not " + "supported for hybrid Mamba/SSM models." + ) + if self.lora_config is not None: self.lora_config.verify_with_model_config(self.model_config) @@ -2714,6 +2738,52 @@ class VllmConfig: f"Model Runner V1 does not support: {', '.join(unsupported)}" ) + def adjust_dcp_kv_cache_interleave_size( + self, kv_cache_config: "KVCacheConfig" + ) -> None: + """Normalize DCP interleave size against the resolved block_size for PD. + + Called by each worker (via ensure_kv_transfer_initialized), once it knows its + own final block_size via kv_cache_config. + """ + dcp_size = self.parallel_config.decode_context_parallel_size + if dcp_size <= 1: + return + # Get the kernel block_size, but don't use resolve_kv_cache_block_size to avoid + # scaling by dcp_size (we need the local block_size here). + local_block_size = min( + g.kv_cache_spec.block_size for g in kv_cache_config.kv_cache_groups + ) + if self.parallel_config.dcp_kv_cache_interleave_size > 1 and ( + self.parallel_config.cp_kv_cache_interleave_size + != self.parallel_config.dcp_kv_cache_interleave_size + ): + self.parallel_config.cp_kv_cache_interleave_size = ( + self.parallel_config.dcp_kv_cache_interleave_size + ) + logger.warning_once( + "cp_kv_cache_interleave_size is overridden by dcp_kv_cache" + "_interleave_size. And dcp-kv-cache-interleave-size will be " + "deprecated when PCP is fully supported." + ) + + if ( + self.kv_transfer_config is not None + and self.kv_transfer_config.kv_connector is not None + and self.parallel_config.cp_kv_cache_interleave_size != local_block_size + ): + interleave = self.parallel_config.cp_kv_cache_interleave_size + self.parallel_config.cp_kv_cache_interleave_size = local_block_size + logger.info_once( + "When using PD disaggregation with DCP " + "(decode_context_parallel_size=%d), " + "cp_kv_cache_interleave_size is automatically adjusted " + "from %d to block_size %d for block-level alignment.", + dcp_size, + interleave, + local_block_size, + ) + def validate_block_size(self) -> None: """Validate block_size against DCP and mamba constraints. @@ -2722,20 +2792,14 @@ class VllmConfig: """ block_size = self.cache_config.block_size - # DCP interleave-size compatibility - if self.parallel_config.decode_context_parallel_size > 1: - if self.parallel_config.dcp_kv_cache_interleave_size > 1 and ( - self.parallel_config.cp_kv_cache_interleave_size - != self.parallel_config.dcp_kv_cache_interleave_size - ): - self.parallel_config.cp_kv_cache_interleave_size = ( - self.parallel_config.dcp_kv_cache_interleave_size - ) - logger.warning_once( - "cp_kv_cache_interleave_size is overridden by dcp_kv_cache" - "_interleave_size. And dcp-kv-cache-interleave-size will be " - "deprecated when PCP is fully supported." - ) + # Skip DCP interleave-size compatibility when a KV connector is configured: + # cp_kv_cache_interleave_size is pinned to block_size for PD by each worker + pd_active = ( + self.kv_transfer_config is not None + and self.kv_transfer_config.kv_connector is not None + and self.kv_transfer_config.is_kv_transfer_instance + ) + if self.parallel_config.decode_context_parallel_size > 1 and not pd_active: assert ( self.parallel_config.cp_kv_cache_interleave_size <= block_size and block_size % self.parallel_config.cp_kv_cache_interleave_size == 0 @@ -2744,7 +2808,6 @@ class VllmConfig: "than or equal to and divisible by cp_kv_cache_interleave_size " f"({self.parallel_config.cp_kv_cache_interleave_size})." ) - # Mamba cache align-mode constraints if self.cache_config.mamba_cache_mode == "align": assert not self.scheduler_config.disable_chunked_mm_input, ( diff --git a/vllm/distributed/kv_transfer/kv_connector/utils.py b/vllm/distributed/kv_transfer/kv_connector/utils.py index 2e4899ebdf..39f8d8a1b9 100644 --- a/vllm/distributed/kv_transfer/kv_connector/utils.py +++ b/vllm/distributed/kv_transfer/kv_connector/utils.py @@ -404,6 +404,9 @@ class EngineTransferInfo: end_layer: int = 0 """Exclusive global index after the last layer owned by this PP rank.""" + remote_dcp_size: int = 1 + """Remote decode context parallel size.""" + # ---- Transfer topology ---- @@ -420,6 +423,7 @@ class TransferTopology: is_mamba: bool total_num_kv_heads: int attn_backends: list[type[AttentionBackend]] + dcp_size: int = 1 tensor_shape: torch.Size | None = None def __post_init__(self): @@ -461,6 +465,15 @@ class TransferTopology: for key in [k for k in self._engines if k[0] == remote_engine_id]: del self._engines[key] + @property + def dcp_rank(self) -> int: + """This rank's DCP coverage rank. + + with ``dcp_size in (1, tp_size)`` enforced at the connector boundary, a + rank's DCP identity is always exactly ``tp_rank % dcp_size``. + """ + return self.tp_rank % self.dcp_size + # ============================================================ # Common methods # ============================================================ @@ -514,18 +527,70 @@ class TransferTopology: """Whether the local engine's KV cache is replicated.""" return self.is_mla or self.tp_size > self.total_num_kv_heads - def handshake_target_ranks(self, remote_tp_size: int) -> list[int]: + def dcp_source_ranks(self, remote_tp_size: int, remote_dcp_size: int) -> list[int]: + """Remote ranks whose DCP slice overlaps mine (MLA, ``remote_dcp_size > 1``). + + Shared by ``handshake_target_ranks`` (who to query metadata from) and + ``compute_tp_mapping`` (who to actually read from) — for MLA the two + questions have the identical answer, since DCP sharding is the only + thing keeping a remote rank from being interchangeable with any other. + """ + local_dcp_size = self.dcp_size + local_dcp_rank = self.dcp_rank + if local_dcp_size <= remote_dcp_size: + # Keep every remote rank whose slice sits inside mine. When + # local_dcp_size == 1 (replicated locally), local_dcp_rank == 0 reduces to + # every remote rank, since no single one holds the whole sequence. + return [ + r for r in range(remote_tp_size) if r % local_dcp_size == local_dcp_rank + ] + # Local finer-grained: exactly one remote rank covers my whole slice + return [local_dcp_rank % remote_dcp_size] + + def handshake_target_ranks( + self, remote_tp_size: int, remote_dcp_size: int = 1 + ) -> list[int]: """Pre-registration: compute which remote TP ranks to handshake with. - Pure math based on local/remote TP sizes — does not require - the remote engine to be registered yet. + Pure math based on local/remote TP (and DCP, when the remote shards + its KV cache) sizes — does not require the remote engine to be + registered yet. + + DCP support is scoped to ``dcp_size in (1, tp_size)`` on each side + and DCP sizes that divide one another: neither side ever has a + partially-duplicated, partially-sharded KV cache. When the + remote is not sharded (``remote_dcp_size == 1``) this reduces + exactly to the DTP>=PTP case, since a sharded local side + already has ``tp_size == dcp_size``. """ + if remote_dcp_size > 1: + return self.dcp_source_ranks(remote_tp_size, remote_dcp_size) + tp_ratio = self.tp_ratio(remote_tp_size) if tp_ratio > 0: return [self.tp_rank // tp_ratio] abs_ratio = -tp_ratio return [self.tp_rank * abs_ratio + i for i in range(abs_ratio)] + def dcp_consumer_count(self, remote_tp_size: int, remote_dcp_size: int) -> int: + """How many local ranks (in aggregate) read from a given remote rank. + + Used by the producer side to know how many reader notifications to + wait for before freeing a request's blocks. Reuses ``tp_ratio`` + whenever the remote isn't sharded — a sharded local side already + has ``tp_size == dcp_size``, so the existing TP-ratio formula is + already correct there unmodified. + """ + if remote_dcp_size > 1: + if self.dcp_size == 1: + # Replicated locally: every local rank reads every shard. + return self.tp_size + # Both sharded, different degrees. + return max(1, self.dcp_size // remote_dcp_size) + # Remote replicated: `tp_ratio` local ranks share each remote rank + # when local_tp >= remote_tp, else each remote rank has one reader. + return max(1, self.tp_ratio(remote_tp_size)) + def target_remote_ranks( self, remote_engine_id: EngineId, remote_pp_rank: int = 0 ) -> list[int]: @@ -551,6 +616,8 @@ class TransferTopology: f"local_tp={self.tp_size}, " f"remote_tp={info.remote_tp_size}, " f"remote_pp={remote_pp_rank}, " + f"local_dcp={self.dcp_size}, " + f"remote_dcp={info.remote_dcp_size}, " f"local_rank={self.tp_rank}, " f"remote_block_len={info.remote_block_len})" ) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_scheduler.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_scheduler.py index eb2bf76d95..ddfcf35e03 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_scheduler.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_scheduler.py @@ -109,7 +109,9 @@ class NixlBaseConnectorScheduler: # Requests that need to start recv/send. # New requests are added by update_state_after_alloc in # the scheduler. Used to make metadata passed to Worker. - self._reqs_need_recv: dict[ReqId, tuple[Request, BlockIds]] = {} + self._reqs_need_recv: dict[ + ReqId, tuple[Request, BlockIds, tuple[int, ...]] + ] = {} self._reqs_need_save: dict[ReqId, Request] = {} # Reqs to send and their expiration time self._reqs_need_send: dict[ReqId, float] = {} @@ -207,6 +209,7 @@ class NixlBaseConnectorScheduler: host = params.get("remote_host") port = params.get("remote_port") tp_size = params.get("tp_size") + dcp_size = params.get("dcp_size", 1) pp_size = params.get("pp_size", 1) if ( remote_engine_id is None @@ -222,6 +225,7 @@ class NixlBaseConnectorScheduler: host=host, port=port, tp_size=tp_size, + dcp_size=dcp_size, pp_size=pp_size, ) self._heartbeat_by_engine[remote_engine_id].req_ids.add(remote_request_id) @@ -452,12 +456,13 @@ class NixlBaseConnectorScheduler: meta = NixlConnectorMetadata() # Loop through scheduled reqs and convert to ReqMeta. - for req_id, (req, block_ids) in self._reqs_need_recv.items(): + for req_id, (req, block_ids, cached) in self._reqs_need_recv.items(): assert req.kv_transfer_params is not None meta.add_new_req_to_recv( request_id=req_id, local_block_ids=block_ids, kv_transfer_params=req.kv_transfer_params, + local_num_computed_blocks=cached, ) if self.use_host_buffer: diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py index c8166b2e6a..8cb465b6b1 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py @@ -231,6 +231,17 @@ class NixlBaseConnectorWorker: handle.append((addr + p_idx * chunk, chunk, dev)) yield handle + def _needs_split_local_xfer_handles(self, tp_ratio: int, plan: TPMapping) -> bool: + """Whether reads need per-source slices of the local KV region. + + Pure MLA attention is replicated across TP ranks and writes the whole + local region. Multiple physical remote workers may still participate + because DCP assigns them disjoint blocks, but that does not require + splitting the local region. Hybrid MLA+SSM is different: its mapping + contains multiple source ranks for the sharded SSM state. + """ + return tp_ratio < 0 and (not self.use_mla or len(plan.all_source_ranks) > 1) + def _fa_desc_replicated(self, num_fa_descs: int) -> list[bool]: """Per-FA-descriptor replicate flag, in _build_fa_local emission order (region-major; one desc per block, with K/V packed). Length ``num_fa_descs``. @@ -380,6 +391,13 @@ class NixlBaseConnectorWorker: self.dcp_size = vllm_config.parallel_config.decode_context_parallel_size self.pcp_rank = get_pcp_group().rank_in_group if self.pcp_size > 1 else 0 + # DCP support is scoped to MLA, with dcp_size in (1, tp_size): either fully + # replicated or fully sharded. A DCP rank is always derivable this way. + self.dcp_rank = self.tp_rank % self.dcp_size + if self._has_mamba and self.dcp_size > 1: + # Prefix-cache-aware DCP slicing isn't implemented for the Mamba group. + raise ValueError("DCP is not supported for hybrid MLA+Mamba models.") + self.num_blocks = kv_cache_config.num_blocks self.enable_permute_local_kv = False self.enable_heterogeneous_attn_post_process = False @@ -543,9 +561,10 @@ class NixlBaseConnectorWorker: self.compat_hash: str | None = None self.transfer_topo: TransferTopology | None = None - # With heterogeneous TP, P must wait for all assigned D TP workers to - # finish reading before safely freeing the blocks. + # With heterogeneous TP (or DCP), P must wait for all assigned D + # workers to finish reading before safely freeing the blocks. self.consumer_notification_counts_by_req = defaultdict[ReqId, int](int) + self.expected_consumer_notifications_by_req: dict[ReqId, int] = {} self.xfer_stats = NixlKVConnectorStats() self._physical_blocks_per_logical_kv_block = 1 @@ -556,7 +575,6 @@ class NixlBaseConnectorWorker: get_representative_spec_type(g.kv_cache_spec) for g in self.kv_cache_config.transfer_groups ) - # Per-region MLA flag, 1:1 with block_len_per_layer. True -> REPLICATE # (MLA), False -> SPLIT (head-sharded full-attn). Mixed only for models # combining both (e.g. GQA main + MLA Eagle-3 draft). @@ -620,6 +638,7 @@ class NixlBaseConnectorWorker: port: int, remote_tp_size: int, expected_engine_id: str, + remote_dcp_size: int = 1, remote_pp_size: int = 1, notif_agents_only: bool = False, ) -> tuple[dict[tuple[int, int], str], float]: @@ -643,7 +662,9 @@ class NixlBaseConnectorWorker: # local rank will read from. Note that With homogeneous TP, # this happens to be the same single rank_i. assert self.transfer_topo is not None - p_remote_ranks = self.transfer_topo.handshake_target_ranks(remote_tp_size) + p_remote_ranks = self.transfer_topo.handshake_target_ranks( + remote_tp_size, remote_dcp_size + ) remote_rank_to_agent_name: dict[tuple[int, int], str] = {} path = make_zmq_path("tcp", host, port) # Clock offset to the peer, estimated from the handshake round-trip. @@ -747,11 +768,11 @@ class NixlBaseConnectorWorker: # Register Remote agent. if notif_agents_only: remote_agent_name = self._add_notif_only_remote_agent( - metadata, remote_tp_size + metadata, remote_tp_size, metadata.dcp_size ) else: remote_agent_name = self.add_remote_agent( - metadata, remote_rank, remote_tp_size + metadata, remote_rank, remote_tp_size, metadata.dcp_size ) setup_agent_time = time.perf_counter() logger.debug( @@ -766,7 +787,7 @@ class NixlBaseConnectorWorker: return remote_rank_to_agent_name, best_offset def _add_notif_only_remote_agent( - self, metadata: NixlAgentMetadata, remote_tp_size: int + self, metadata: NixlAgentMetadata, remote_tp_size: int, remote_dcp_size: int = 1 ) -> str: """Load a remote agent for notifs only on the push-mode decode side. @@ -782,6 +803,7 @@ class NixlBaseConnectorWorker: remote_physical_blocks_per_logical=( metadata.physical_blocks_per_logical_kv_block ), + remote_dcp_size=remote_dcp_size, ), ) return self.nixl_wrapper.add_remote_agent(metadata.agent_metadata) @@ -898,6 +920,7 @@ class NixlBaseConnectorWorker: host: str, port: int, tp_size: int, + dcp_size: int = 1, pp_size: int = 1, notif_agents_only: bool = False, ) -> Future[tuple[dict[tuple[int, int], str], float]] | None: @@ -923,6 +946,7 @@ class NixlBaseConnectorWorker: port, tp_size, engine_id, + dcp_size, pp_size, notif_agents_only, ) @@ -960,6 +984,8 @@ class NixlBaseConnectorWorker: meta.remote.host, meta.remote.port, meta.tp_size, + meta.dcp_size, + meta.pp_size, ) if fut is None: # Already handshaked — only happens if caller does not pre-check. @@ -991,8 +1017,11 @@ class NixlBaseConnectorWorker: block_size=self.block_size, engine_id=self.engine_id, is_mla=self.use_mla, - total_num_kv_heads=self.model_config.get_total_num_kv_heads(), + total_num_kv_heads=1 + if self.use_mla + else self.model_config.get_total_num_kv_heads(), attn_backends=self.attn_backends, + dcp_size=self.dcp_size, # SSM States come in tuples (ssm, conv) tensor_shape=next(iter(kv_caches.values())).shape if not self._has_mamba @@ -1469,6 +1498,7 @@ class NixlBaseConnectorWorker: nixl_agent_meta: NixlAgentMetadata, remote_tp_rank: int = 0, remote_tp_size: int = 1, + remote_dcp_size: int = 1, ) -> str: """ Add the remote NIXL agent and prepare the descriptors for reading cache @@ -1553,6 +1583,7 @@ class NixlBaseConnectorWorker: remote_block_size=nixl_agent_meta.block_size, remote_block_len=nixl_agent_meta.block_lens[0], remote_physical_blocks_per_logical=physical_blocks_per_logical, + remote_dcp_size=remote_dcp_size, ) transfer_topo.register_remote_engine(engine_id, transfer_info) logger.info("Transfer plan: %s", transfer_topo.describe(engine_id)) @@ -1561,6 +1592,7 @@ class NixlBaseConnectorWorker: transfer_topology=transfer_topo, remote_tp_size=remote_tp_size, group_spec_types=self._group_spec_types, + remote_dcp_size=remote_dcp_size, ) remote_agent_name = self.nixl_wrapper.add_remote_agent( @@ -1583,7 +1615,9 @@ class NixlBaseConnectorWorker: self.kv_caches_base_addr[engine_id][remote_tp_rank] = ( nixl_agent_meta.kv_caches_base_addr ) - self._validate_remote_agent_handshake(nixl_agent_meta, remote_tp_size) + self._validate_remote_agent_handshake( + nixl_agent_meta, remote_tp_size, remote_dcp_size + ) # This is 1 when P and D `--tensor-parallel-size` match. Otherwise, # this is the ratio between the two sizes. @@ -1613,10 +1647,8 @@ class NixlBaseConnectorWorker: ### (Optional) Register local agent memory regions. MLA is not split. split_key = (tp_ratio, remote_block_size) - if ( - tp_ratio < 0 - and (not self.use_mla or len(plan.all_source_ranks) > 1) - and split_key not in self.src_xfer_handles_by_tp_ratio + if self._needs_split_local_xfer_handles(tp_ratio, plan) and ( + split_key not in self.src_xfer_handles_by_tp_ratio ): # Remote tp_size > local tp_size: read from multiple remote ranks. # Logically "split" own regions into per-source chunks. Hybrid @@ -1674,7 +1706,10 @@ class NixlBaseConnectorWorker: return remote_agent_name def _validate_remote_agent_handshake( - self, nixl_agent_meta: NixlAgentMetadata, remote_tp_size: int + self, + nixl_agent_meta: NixlAgentMetadata, + remote_tp_size: int, + remote_dcp_size: int = 1, ): """ Validate the remote agent handshake metadata ensuring the @@ -1685,6 +1720,15 @@ class NixlBaseConnectorWorker: assert self.transfer_topo is not None remote_info = self.transfer_topo.get_engine_info(remote_engine_id) assert remote_info.remote_tp_size == remote_tp_size + assert remote_info.remote_dcp_size == remote_dcp_size + # DCP sizes must divide one another; this is what keeps the + # read-slicing math in pull_worker a closed form. + assert ( + self.dcp_size % remote_dcp_size == 0 or remote_dcp_size % self.dcp_size == 0 + ), ( + f"DCP sizes must divide one another: local={self.dcp_size}, " + f"remote={remote_dcp_size} (engine {remote_engine_id})." + ) tp_ratio = self.transfer_topo.tp_ratio(remote_tp_size) block_size_ratio = self.transfer_topo.block_size_ratio( @@ -2132,6 +2176,7 @@ class NixlBaseConnectorWorker: if now < expires: break count = self.consumer_notification_counts_by_req.pop(req_id, 0) + self.expected_consumer_notifications_by_req.pop(req_id, None) self.xfer_stats.record_kv_expired_req() logger.warning( "Releasing expired KV blocks for request %s which were " @@ -2267,6 +2312,7 @@ class NixlBaseConnectorWorker: hb_info.host, hb_info.port, hb_info.tp_size, + hb_info.dcp_size, hb_info.pp_size, self._hb_handshake_notif_only and hb_info.pp_size > 1, ) @@ -2352,7 +2398,7 @@ class NixlBaseConnectorWorker: This is required when the logical block size (the one set by the user) does not match the one required by the attn backend. `ratio` is the number of physical blocks per logical block. - We always receive logical blocks from the engine, so we expand them here eg: + We always receive logical blocks from the engine, so we expand them here eg: logical block ids: [(SW-clipped) [1], (FA) [2, 3]], ratio=2 physical block ids: [(SW-clipped) [2, 3], (FA) [4, 5, 6, 7]] """ @@ -2377,6 +2423,56 @@ class NixlBaseConnectorWorker: ) return physical_block_ids + def _apply_dcp_prefix_caching( + self, + local_ids: list[int], + remote_ids: list[int], + remote_rank: int, + local_dcp_size: int, + local_dcp_rank: int, + remote_dcp_size: int, + local_num_computed_blocks: int, + ) -> tuple[list[int], list[int]]: + """Match local and remote block IDs by global DCP position. + + ``local_ids`` excludes blocks already satisfied by the local prefix + cache, while ``remote_ids`` contains the full transferable remote list. + + Example: + local DCP size 2, rank 0 owns *global positions*=[0, 2, 4, 6] + cached_blocks=[0,2,4] + to transfer: [6] (the rest is cached) + + remote DCP size 4 + remote rank 0 owns [0, 4] -> no more match, skip the read + remote rank 2 owns [2, 6] -> match position 6 and read + + An empty result (rank 0 above) takes the notification-only path, allowing the + remote to release its blocks without performing a transfer. + """ + local_size, remote_size = local_dcp_size, remote_dcp_size + + if local_size == remote_size: + local_slice = local_ids + remote_slice = remote_ids[local_num_computed_blocks:] + elif local_size < remote_size: + k = remote_size // local_size + p = (remote_rank - local_dcp_rank) // local_size + start_local = (p - local_num_computed_blocks) % k + start_remote = (local_num_computed_blocks + start_local - p) // k + local_slice = local_ids[start_local::k] + remote_slice = remote_ids[start_remote:] + else: + k = local_size // remote_size + remote_dcp_rank = remote_rank % remote_size + c = (local_dcp_rank - remote_dcp_rank) // remote_size + start_remote = c + local_num_computed_blocks * k + local_slice = local_ids + remote_slice = remote_ids[start_remote::k] + + matched_blocks = min(len(local_slice), len(remote_slice)) + return local_slice[:matched_blocks], remote_slice[:matched_blocks] + def _apply_prefix_caching( self, decode_block_ids: BlockIds, diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/connector.py index fa5537c325..ee7b991799 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/connector.py @@ -369,6 +369,10 @@ class NixlPushConnector(NixlBaseConnector): kv_cache_config: "KVCacheConfig", ): super().__init__(vllm_config, role, kv_cache_config) + if vllm_config.parallel_config.decode_context_parallel_size > 1: + raise ValueError( + "NixlPushConnector does not support decode_context_parallel_size > 1." + ) self.connector_scheduler: NixlPushConnectorScheduler | None = None self.connector_worker: NixlPushConnectorWorker | None = None if role == KVConnectorRole.SCHEDULER: diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/metadata.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/metadata.py index 0fca03c93b..99dae07f97 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/metadata.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/metadata.py @@ -209,6 +209,7 @@ class HeartbeatInfo: host: str port: int tp_size: int + dcp_size: int = 1 pp_size: int = 1 @@ -228,6 +229,12 @@ class ReqMeta: # To be used when logical block size does not match the kernel block size local_physical_block_ids: BlockIds tp_size: int + dcp_size: int = 1 + # Per-KV-cache-group logical blocks this rank already holds, i.e. its + # prefix-cache hit. Fixes where this rank's DCP slice starts relative to + # the remote's; kept per-group since hybrid models (e.g. SWA+FA) can have + # different cache-hit counts per group. + local_num_computed_blocks: tuple[int, ...] = () remote: RemoteMeta | None = None # Remote block size, discovered during NIXL handshake (push mode). remote_block_size: int | None = None @@ -261,14 +268,17 @@ class NixlConnectorMetadata(KVConnectorMetadata): self, local_block_ids: BlockIds, kv_transfer_params: dict[str, Any], + local_num_computed_blocks: tuple[int, ...] = (), ) -> ReqMeta: return ReqMeta( local_block_ids=local_block_ids, local_physical_block_ids=local_block_ids, - # P workers don't need to receive tp_size from proxy here. + # P workers don't need to receive these from proxy here. tp_size=kv_transfer_params.get("tp_size", 1), + dcp_size=kv_transfer_params.get("dcp_size", 1), remote_block_size=kv_transfer_params.get("remote_block_size"), pp_size=kv_transfer_params.get("pp_size", 1), + local_num_computed_blocks=local_num_computed_blocks, ) def add_new_req_to_save( @@ -286,8 +296,11 @@ class NixlConnectorMetadata(KVConnectorMetadata): request_id: ReqId, local_block_ids: BlockIds, kv_transfer_params: dict[str, Any], + local_num_computed_blocks: tuple[int, ...] = (), ): - req = self._add_new_req(local_block_ids, kv_transfer_params) + req = self._add_new_req( + local_block_ids, kv_transfer_params, local_num_computed_blocks + ) req.remote = RemoteMeta( block_ids=kv_transfer_params["remote_block_ids"], engine_id=kv_transfer_params["remote_engine_id"], diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_scheduler.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_scheduler.py index 5393989c9a..9740b518a2 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_scheduler.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_scheduler.py @@ -155,12 +155,23 @@ class NixlPullConnectorScheduler(NixlBaseConnectorScheduler): local_block_ids = self.get_exchange_clipped_blocks( unhashed_local_block_ids ) + # Blocks covered by the local prefix cache, per KV cache group. + # Each count fixes where that group's DCP slice starts, which the + # worker needs to line up with the remote's slice. + local_num_computed_blocks = tuple( + sum( + block.block_hash is not None and not block.is_null + for block in group + ) + for group in blocks.blocks + ) # Get unhashed blocks to pull from remote. Mind that a full prefix # cache hit is indicated with an empty list. self._reqs_need_recv[request.request_id] = ( request, local_block_ids, + local_num_computed_blocks, ) else: @@ -213,7 +224,7 @@ class NixlPullConnectorScheduler(NixlBaseConnectorScheduler): # To avoid stranding the prefill blocks in the prefill instance, # we must add empty block_ids to _reqs_need_recv so that our # worker side will notify and free blocks in the prefill instance. - self._reqs_need_recv[request.request_id] = (request, []) + self._reqs_need_recv[request.request_id] = (request, [], ()) params["do_remote_prefill"] = False return False, None @@ -272,6 +283,8 @@ class NixlPullConnectorScheduler(NixlBaseConnectorScheduler): remote_host=self.side_channel_host, remote_port=self.side_channel_port, tp_size=self.vllm_config.parallel_config.tensor_parallel_size, + dcp_size=self.vllm_config.parallel_config.decode_context_parallel_size, + pp_size=self.vllm_config.parallel_config.pipeline_parallel_size, remote_num_tokens=remote_num_tokens, remote_blocks_expiry_time=blocks_expiry_time, transfer_mode=self._TRANSFER_MODE, diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py index a09fd684e3..dde11548e5 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py @@ -5,6 +5,7 @@ import time from typing import TYPE_CHECKING +from vllm.distributed.kv_transfer.kv_connector.utils import BlockIds from vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker import ( NixlBaseConnectorWorker, ) @@ -143,40 +144,78 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker): self._handle_failed_transfer(req_id, None) return + if any(len(group) > 0 for group in meta.local_block_ids): + # The scheduler waits for finished_recving from *every* worker. + # Under DCP a rank's slice can legitimately come out empty when its + # interleaved positions fall past the end of the sequence. _read_blocks + # then takes the notify-only path without registering a transfer. + # Seed the entry so this rank still reports completion. + self._recving_transfers.setdefault(req_id, []) + plan = self.tp_mappings[engine_id] remote_info = self.transfer_topo.get_engine_info(engine_id) tp_ratio = self.transfer_topo.tp_ratio(remote_info.remote_tp_size) + remote_logical_block_ids = meta.remote.block_ids meta.remote.block_ids = self._logical_to_kernel_block_ids( - meta.remote.block_ids, + remote_logical_block_ids, remote_info.remote_physical_blocks_per_logical, ) - remote_block_ids = meta.remote.block_ids - local_block_ids = meta.local_physical_block_ids - num_groups = len(local_block_ids) - read_specs = [ - ReadSpec( - remote_rank=rank, - local_block_ids=[ - list(local_block_ids[g]) - if rank in plan.source_ranks_per_group[g] - else [] - for g in range(num_groups) - ], - remote_block_ids=[ - list(remote_block_ids[g]) - if rank in plan.source_ranks_per_group[g] - else [] - for g in range(num_groups) - ], + num_groups = len(meta.local_block_ids) + dcp_active = self.dcp_size > 1 or remote_info.remote_dcp_size > 1 + + def group_ids(block_ids: BlockIds, rank: int) -> list[list[int]]: + return [ + list(block_ids[g]) if rank in plan.source_ranks_per_group[g] else [] + for g in range(num_groups) + ] + + read_specs = [] + for rank in plan.all_source_ranks: + if dcp_active: + # DCP interleaves at block granularity, so slicing happens + # here on logical blocks, before kernel-block expansion. + local_ids = group_ids(meta.local_block_ids, rank) + remote_ids = group_ids(remote_logical_block_ids, rank) + for g in range(num_groups): + if not local_ids[g]: + continue + # Prefix cache hit may lead to skip some of the remote reads + # TODO (NickLucche) consider unifying prefix cache handling on + # logical blocks here for both dcp and non-dcp + local_ids[g], remote_ids[g] = self._apply_dcp_prefix_caching( + local_ids[g], + remote_ids[g], + remote_rank=rank, + local_dcp_size=self.dcp_size, + local_dcp_rank=self.dcp_rank, + remote_dcp_size=remote_info.remote_dcp_size, + local_num_computed_blocks=meta.local_num_computed_blocks[g], + ) + local_physical_ids = self._logical_to_kernel_block_ids( + local_ids, self._physical_blocks_per_logical_kv_block + ) + remote_physical_ids = self._logical_to_kernel_block_ids( + remote_ids, remote_info.remote_physical_blocks_per_logical + ) + else: + # No DCP realignment needed: reuse the already-expanded full + # physical lists instead of re-deriving them from logical ids. + local_physical_ids = group_ids(meta.local_physical_block_ids, rank) + remote_physical_ids = group_ids(meta.remote.block_ids, rank) + read_specs.append( + ReadSpec( + remote_rank=rank, + local_block_ids=local_physical_ids, + remote_block_ids=remote_physical_ids, + ) ) - for rank in plan.all_source_ranks - ] # D may have to perform multiple reads from different remote ranks. # Pure MLA reads once because its cache is replicated. Hybrid - # MLA+SSM still needs one read per SSM source rank. - if self.use_mla and tp_ratio < 0 and not self._has_mamba: + # MLA+SSM still needs one read per SSM source rank. With DCP, pure + # MLA may also read from multiple ranks (disjoint token slices). + if self.use_mla and tp_ratio < 0 and not self._has_mamba and not dcp_active: assert len(read_specs) == 1 for i, spec in enumerate(read_specs): @@ -214,12 +253,15 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker): remote_request_id=meta.remote.request_id, local_xfer_side_handle=local_xfer_side_handle, remote_xfer_side_handle=remote_xfer_side_handle, + expected_consumers=plan.local_consumers, ) if self.use_mla and tp_ratio < 0 and len(read_specs) == 1: # ..but we still need to notify the other remote ranks that we # have the blocks we need so they can update the request state. - notif_id = f"{meta.remote.request_id}:{self.world_size}".encode() + # Same thing for DCP (tp_size == dcp_size), so the raw tp_ratio already + # reflects whether any remote replica is left unchosen. + notif_id = f"{meta.remote.request_id}:{plan.local_consumers}".encode() remote_agents = self._remote_agents[meta.remote.engine_id] for rank_to_notify, agent in remote_agents.items(): if rank_to_notify != (0, read_specs[0].remote_rank): @@ -233,6 +275,7 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker): remote_request_id: str, local_xfer_side_handle: int, remote_xfer_side_handle: int, + expected_consumers: int, ): """ Post a READ point-to-point xfer request from a single local worker to @@ -262,13 +305,13 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker): # NOTE(rob): according to nvidia the staging blocks are used to # saturate IB with heterogeneous TP sizes. - # Number of D TP workers that will read from dst P. Propagate info - # on notification so that dst worker can wait before freeing blocks. - notif_id = f"{remote_request_id}:{self.world_size}".encode() + # Number of local workers that will notify this producer worker. + # Propagate on notification so dst worker can wait before freeing. + notif_id = f"{remote_request_id}:{expected_consumers}".encode() # Full prefix cache hit: do not need to read remote blocks, # just notify P worker that we have the blocks we need. - if len(local_block_ids) == 0: + if not any(len(group) > 0 for group in local_block_ids): # A full prefix cache hit is indicated with an empty list. agent_name = self._remote_agents[dst_engine_id][(0, remote_rank)] try: @@ -292,12 +335,19 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker): == len(local_block_ids) == len(self.kv_cache_config.transfer_groups) ) - local_block_ids, remote_block_ids = self._apply_prefix_caching( - decode_block_ids=local_block_ids, - prefill_block_ids=remote_block_ids, - decode_physical_per_logical=self._physical_blocks_per_logical_kv_block, - prefill_physical_per_logical=remote_info.remote_physical_blocks_per_logical, - ) + if not (self.dcp_size > 1 or remote_info.remote_dcp_size > 1): + # DCP-active reads were already trimmed (and DCP-realigned, when + # sizes mismatch) in _read_blocks_for_req, at logical granularity. + local_block_ids, remote_block_ids = self._apply_prefix_caching( + decode_block_ids=local_block_ids, + prefill_block_ids=remote_block_ids, + decode_physical_per_logical=( + self._physical_blocks_per_logical_kv_block + ), + prefill_physical_per_logical=( + remote_info.remote_physical_blocks_per_logical + ), + ) # NOTE (nicolo) With homogeneous TP, each TP worker loads KV from # corresponding rank. With heterogeneous TP, fixing D>P, the D tp @@ -351,8 +401,8 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker): def _get_new_notifs(self) -> set[str]: """ Get req_ids which got a remote xfer message. When multiple consumers - are reading from the same producer (heterogeneous TP scenario), wait - for all consumers to be done pulling. + are reading from the same producer (heterogeneous TP or DCP + scenario), wait for all consumers to be done pulling. Also handles heartbeat notifications ("HB:req1,req2,...") by extending the lease on the referenced requests. @@ -368,7 +418,7 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker): self._handle_heartbeat(msg[3:]) continue - req_id, tp_size = msg.rsplit(":", 1) + req_id, expected_consumers = msg.rsplit(":", 1) if ( req_id not in self._reqs_to_send and req_id not in self._reqs_to_process @@ -381,24 +431,22 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker): ) continue - # NOTE: `tp_ratio` is the opposite when swapping local<>remote - n_consumers = int(tp_size) - tp_ratio = self.transfer_topo.tp_ratio(n_consumers) - - # Number of reads *per producer* to wait for. - # When remote D TP > local P TP we expect `tp_ratio` reads. - consumers_per_producer = ( - -tp_ratio if n_consumers > self.world_size else 1 + # Every reader of this req_id reports the same count (it's + # derived from aggregate topology, not the specific rank), + # so repeated notifications never disagree on it. + self.expected_consumer_notifications_by_req[req_id] = int( + expected_consumers ) self.consumer_notification_counts_by_req[req_id] += 1 # Wait all consumers (D) to be done reading before freeing. if ( self.consumer_notification_counts_by_req[req_id] - == consumers_per_producer + == self.expected_consumer_notifications_by_req[req_id] ): notified_req_ids.add(req_id) del self.consumer_notification_counts_by_req[req_id] + del self.expected_consumer_notifications_by_req[req_id] self._reqs_to_process.remove(req_id) self._reqs_to_send.pop(req_id, None) return notified_req_ids diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/push_scheduler.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/push_scheduler.py index 095d90e7b1..53394cf9c1 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/push_scheduler.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/push_scheduler.py @@ -198,7 +198,7 @@ class NixlPushConnectorScheduler(NixlBaseConnectorScheduler): # ReqMeta without a KeyError — the actual remote block IDs are # learned by P over the NIXL handshake at WRITE time. params["remote_block_ids"] = () - self._reqs_need_recv[request.request_id] = (request, local_block_ids) + self._reqs_need_recv[request.request_id] = (request, local_block_ids, ()) # Mark as processed so a re-entry (e.g. preemption + reschedule) # doesn't re-stage the registration. @@ -240,7 +240,7 @@ class NixlPushConnectorScheduler(NixlBaseConnectorScheduler): # recv so the worker emits a notif that lets P free them. # Seed remote_block_ids so add_new_req_to_recv won't KeyError. params["remote_block_ids"] = () - self._reqs_need_recv[request.request_id] = (request, []) + self._reqs_need_recv[request.request_id] = (request, [], ()) params["do_remote_prefill"] = False return False, None diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/tp_mapping.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/tp_mapping.py index b034b76050..fa3ca6a3c3 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/tp_mapping.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/tp_mapping.py @@ -56,6 +56,10 @@ class TPMapping: # FA head offset factor for hetero-TP (D_TP > P_TP). rank_offset_factor: int + # Local ranks (in aggregate) that read from a given source rank. The producer frees + # a request's blocks only once that many notifications have come in. + local_consumers: int = 1 + # ====================================================================== # TP mapping computation @@ -66,22 +70,31 @@ def compute_tp_mapping( transfer_topology: TransferTopology, remote_tp_size: int, group_spec_types: tuple[type[KVCacheSpec], ...], + remote_dcp_size: int = 1, ) -> TPMapping: """Build the complete local-to-remote TP mapping. Computes source ranks, head slot assignments, and the rank offset factor in a single pass. + + DCP support is scoped to MLA only, with a side is either fully replicated or fully + sharded. DCP-branch reuses the same rank set used at handshake selection. """ tp_rank = transfer_topology.tp_rank tp_size = transfer_topology.tp_size total_num_kv_heads = transfer_topology.total_num_kv_heads # --- Attention source ranks --- if transfer_topology.is_mla or tp_size >= remote_tp_size: - # D (local TP) > P (remote TP): multiple local ranks read different chunks from - # *one* remote rank, corresponding to different kv heads. - # For MLA, we only need one remote since cache is duplicated. When P TP=k*TP k, - # this will spread mla ranks to read from remote k*tp_rank. - attn_ranks = [tp_rank * remote_tp_size // tp_size] + if transfer_topology.is_mla and remote_dcp_size > 1: + attn_ranks = transfer_topology.dcp_source_ranks( + remote_tp_size, remote_dcp_size + ) + else: + # D (local TP) > P (remote TP): multiple local ranks read different chunks + # from *one* remote rank, corresponding to different kv heads. + # For MLA, we only need one remote since cache is duplicated. When + # P TP=k*TP k, this will spread mla ranks to read from remote k*tp_rank. + attn_ranks = [tp_rank * remote_tp_size // tp_size] else: # P (remote TP) > D (local TP): one local rank # reads from multiple remote ranks. @@ -134,9 +147,14 @@ def compute_tp_mapping( # D TP > P TP: we index into remote to read different heads depending on rank. rank_offset_factor = tp_rank % (tp_size // remote_tp_size) + local_consumers = transfer_topology.dcp_consumer_count( + remote_tp_size, remote_dcp_size + ) + return TPMapping( source_ranks_per_group=source_ranks_per_group, all_source_ranks=tuple(all_ranks), rank_to_attention_slot=rank_to_attention_slot, rank_offset_factor=rank_offset_factor, + local_consumers=local_consumers, ) diff --git a/vllm/distributed/kv_transfer/kv_transfer_state.py b/vllm/distributed/kv_transfer/kv_transfer_state.py index f9209dc3e4..33193380f2 100644 --- a/vllm/distributed/kv_transfer/kv_transfer_state.py +++ b/vllm/distributed/kv_transfer/kv_transfer_state.py @@ -85,6 +85,8 @@ def ensure_kv_transfer_initialized( vllm_config.kv_transfer_config.is_kv_transfer_instance and _KV_CONNECTOR_AGENT is None ): + # PD only supports an interleave_size equal to block_size. + vllm_config.adjust_dcp_kv_cache_interleave_size(kv_cache_config) _sync_engine_id_across_tp(vllm_config) _KV_CONNECTOR_AGENT = KVConnectorFactory.create_connector( diff --git a/vllm/model_executor/layers/attention/mla_attention.py b/vllm/model_executor/layers/attention/mla_attention.py index 20b5e6f679..305b402858 100644 --- a/vllm/model_executor/layers/attention/mla_attention.py +++ b/vllm/model_executor/layers/attention/mla_attention.py @@ -3157,9 +3157,6 @@ class MLACommonImpl(MLACommonBaseImpl[M], Generic[M]): parallel_config = get_current_vllm_config().parallel_config # Avoid requiring an initialized DCP group in tests. self.dcp_world_size: int = parallel_config.decode_context_parallel_size - self.cp_kv_cache_interleave_size: int = ( - parallel_config.cp_kv_cache_interleave_size - ) @abstractmethod def forward_mqa( diff --git a/vllm/model_executor/layers/sparse_attn_indexer.py b/vllm/model_executor/layers/sparse_attn_indexer.py index 600094ad8c..a87b313f31 100644 --- a/vllm/model_executor/layers/sparse_attn_indexer.py +++ b/vllm/model_executor/layers/sparse_attn_indexer.py @@ -771,14 +771,30 @@ class SparseAttnIndexer(CustomOp): parallel_config = get_current_vllm_config().parallel_config self.dcp_world_size = parallel_config.decode_context_parallel_size self.dcp_rank = get_dcp_group().rank_in_group if self.dcp_world_size > 1 else 0 - self.cp_kv_cache_interleave_size = parallel_config.cp_kv_cache_interleave_size self.use_pcp = parallel_config.prefill_context_parallel_size > 1 + self._cp_kv_cache_interleave_size: int | None = None if current_platform.is_cuda() and not has_deep_gemm(): raise RuntimeError( "Sparse Attention Indexer CUDA op requires DeepGEMM support in " "the current vLLM environment." ) + @property + def cp_kv_cache_interleave_size(self) -> int: + """With PD+DCP, the real value isn't known until block_size is finalized, + which happens after this layer is built. Safe to cache after the first access, + as long as the adjustment always runs before any forward pass + (it's set up in Worker.initialize_from_config, ahead of warmup/serving). + """ + if self._cp_kv_cache_interleave_size is None: + value = ( + get_current_vllm_config().parallel_config.cp_kv_cache_interleave_size + ) + if isinstance(get_forward_context().attn_metadata, dict): + self._cp_kv_cache_interleave_size = value + return value + return self._cp_kv_cache_interleave_size + def forward_native( self, hidden_states: torch.Tensor, diff --git a/vllm/v1/attention/backends/mla/flashinfer_mla.py b/vllm/v1/attention/backends/mla/flashinfer_mla.py index 7932565197..2517c7de3d 100644 --- a/vllm/v1/attention/backends/mla/flashinfer_mla.py +++ b/vllm/v1/attention/backends/mla/flashinfer_mla.py @@ -324,7 +324,6 @@ class FlashInferMLAImpl(MLACommonImpl[MLACommonMetadata]): extra_kwargs: dict[str, Any] = {} decode_backend: str | None if self.dcp_world_size > 1: - assert self.cp_kv_cache_interleave_size == 1 causal_seqlens_kv_global = attn_metadata.decode.dcp_tot_seq_lens assert causal_seqlens_kv_global is not None if not attn_metadata.causal and query_len > 1: diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index 6bce5061b5..cc2ac442c7 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -531,6 +531,8 @@ class GPUModelRunner(LoRAModelRunnerMixin): is_profiling: bool = False, kv_cache_allocation_context: AbstractContextManager | None = None, ) -> None: + # GPUWorker finalizes the PD interleave before KV cache initialization. + self.cp_interleave = self.parallel_config.cp_kv_cache_interleave_size kv_cache_config = deepcopy(kv_cache_config) self.kv_cache_config = kv_cache_config diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index 19e2e514f5..e6e63e1e98 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -749,6 +749,9 @@ class GPUModelRunner( self._init_kernel_block_sizes = [placeholder_block_size] self._init_max_num_blocks = [placeholder_max_num_blocks] self._init_slot_mapping_modes = [SlotMappingMode.TOKEN_TO_KV_SLOT] + self.cp_kv_cache_interleave_size = ( + self.parallel_config.cp_kv_cache_interleave_size + ) # Capture warmup providers registered by the initial placeholder InputBatch with self.jit_warmup_registry.activate(): self.input_batch = InputBatch( @@ -7350,11 +7353,16 @@ class GPUModelRunner( or kernel_block_sizes != self._init_kernel_block_sizes or max_num_blocks != self._init_max_num_blocks or slot_mapping_modes != self._init_slot_mapping_modes + or self.cp_kv_cache_interleave_size + != self.parallel_config.cp_kv_cache_interleave_size ): self._init_block_sizes = block_sizes self._init_kernel_block_sizes = kernel_block_sizes self._init_max_num_blocks = max_num_blocks self._init_slot_mapping_modes = slot_mapping_modes + self.cp_kv_cache_interleave_size = ( + self.parallel_config.cp_kv_cache_interleave_size + ) # Capture warmup providers registered after final KV-cache geometry is known with self.jit_warmup_registry.activate(): self.input_batch = InputBatch(