[Nixl][PD] DCP support for MLA models (#50611)

Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
This commit is contained in:
Nicolò Lucchesi
2026-08-29 16:03:53 +00:00
committed by GitHub
parent 4fc943b867
commit 7f4793eaa3
23 changed files with 818 additions and 104 deletions
+30
View File
@@ -17,8 +17,11 @@ import vllm.config.vllm as vllm_config_module
import vllm.envs as envs
from vllm.compilation.backends import VllmBackend
from vllm.config import (
CacheConfig,
CompilationConfig,
DeviceConfig,
KernelConfig,
KVTransferConfig,
ModelConfig,
ObservabilityConfig,
ParallelConfig,
@@ -100,6 +103,32 @@ def test_per_request_spec_decode_metrics_requires_spec_decode():
)
def test_pd_dcp_interleave_size_is_adjusted_to_block_size(caplog):
config = VllmConfig(
cache_config=CacheConfig(block_size=16),
device_config=DeviceConfig(device="cpu"),
parallel_config=ParallelConfig(
tensor_parallel_size=2,
decode_context_parallel_size=2,
cp_kv_cache_interleave_size=3,
distributed_executor_backend="mp",
),
kv_transfer_config=KVTransferConfig(
kv_connector="NixlConnector",
kv_role="kv_both",
),
)
kv_cache_config = SimpleNamespace(
kv_cache_groups=[SimpleNamespace(kv_cache_spec=SimpleNamespace(block_size=16))]
)
with caplog.at_level(logging.INFO):
config.adjust_dcp_kv_cache_interleave_size(kv_cache_config)
assert config.parallel_config.cp_kv_cache_interleave_size == 16
assert "automatically adjusted from 3 to block_size 16" in caplog.text
def test_compile_config_repr_succeeds():
# setup: VllmBackend mutates the config object
config = VllmConfig()
@@ -1730,6 +1759,7 @@ def test_validate_mamba_align_subblock_prefill():
long_prefill_token_threshold=4096,
disable_chunked_mm_input=False,
),
kv_transfer_config=None,
)
VllmConfig.validate_block_size(config)
@@ -507,6 +507,7 @@ class FakeNixlConnectorWorker(NixlConnectorWorker):
port: int,
remote_tp_size: int,
expected_engine_id: str,
remote_dcp_size: int = 1,
remote_pp_size: int = 1,
notif_agents_only: bool = False,
) -> tuple[dict[tuple[int, int], str], float]:
@@ -3,7 +3,7 @@
"""Unit tests for NixlConnectorScheduler with HMA and Mamba N-1 prefill."""
import gc
from unittest.mock import patch
from unittest.mock import MagicMock, patch
import pytest
import torch
@@ -63,6 +63,180 @@ def test_sw_sizes(mock_platform, swa_enabled, expected_sw_sizes):
)
@pytest.mark.cpu_test
@pytest.mark.parametrize(
"use_mla,source_ranks,tp_ratio,expected",
[
pytest.param(True, (0,), -2, False, id="pure_mla_with_remote_dcp"),
pytest.param(True, (0, 1), -2, True, id="hybrid_mla_ssm"),
pytest.param(False, (0,), -2, True, id="full_attention"),
pytest.param(False, (0,), 2, False, id="local_tp_greater"),
],
)
def test_needs_split_local_xfer_handles(use_mla, source_ranks, tp_ratio, expected):
"""Handle creation and selection must use the same TP-mapping predicate.
In pure MLA, DCP can target several physical remote workers even though
the TP mapping has one replicated attention source. Those workers own
disjoint blocks, so every read uses the whole local-region handle.
"""
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker import (
NixlConnectorWorker,
)
worker = object.__new__(NixlConnectorWorker)
worker.use_mla = use_mla
plan = MagicMock()
plan.all_source_ranks = source_ranks
assert worker._needs_split_local_xfer_handles(tp_ratio, plan) is expected
@pytest.mark.cpu_test
def test_update_state_after_alloc_tracks_cached_blocks_per_group():
"""Hybrid SWA+FA requests can have different prefix-cache-hit counts per
KV cache group. Each group's count must be tracked independently rather
than collapsed to a single scalar, or DCP read-phase alignment
(local_num_computed_blocks) would misalign one of the groups."""
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.pull_scheduler import (
NixlPullConnectorScheduler,
)
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
from vllm.v1.core.kv_cache_utils import KVCacheBlock
scheduler = object.__new__(NixlPullConnectorScheduler)
scheduler._reqs_in_batch = set()
scheduler._reqs_need_save = {}
scheduler._reqs_need_recv = {}
scheduler.use_host_buffer = False
scheduler.is_bidirectional_kv_xfer_enabled = False
scheduler._is_hma_required = False
scheduler.kv_cache_config = MagicMock(
select_transfer_block_ids=lambda block_ids: block_ids
)
def cached(block_id):
return KVCacheBlock(block_id=block_id, _block_hash=object())
def uncached(block_id):
return KVCacheBlock(block_id=block_id)
# Group 0 (full-attention): 2 of 3 blocks cached.
# Group 1 (sliding-window): 1 of 3 blocks cached.
blocks = KVCacheBlocks(
blocks=(
[cached(0), cached(1), uncached(2)],
[cached(10), uncached(11), uncached(12)],
)
)
request = create_request(do_remote_prefill=True)
scheduler.update_state_after_alloc(request, blocks, num_external_tokens=2)
_, _, local_num_computed_blocks = scheduler._reqs_need_recv[request.request_id]
assert local_num_computed_blocks == (2, 1)
@pytest.mark.cpu_test
@pytest.mark.parametrize(
"local_ids,remote_ids,remote_rank,local_dcp_size,local_dcp_rank,"
"remote_dcp_size,local_num_computed_blocks,expected_local,expected_remote",
[
# local more finely sharded than remote: remote's raw slice has an
# extra tail block beyond what local's phase-aligned slice covers.
pytest.param(
[100, 101, 102],
[200, 201],
1,
1,
0,
2,
0,
[101],
[200],
id="remote_tail_padding",
),
# remote more finely sharded than local: local's raw slice has a
# block beyond what remote's phase-aligned slice covers.
pytest.param(
[100],
[200],
0,
2,
1,
1,
0,
[],
[],
id="local_tail_padding",
),
# Asymmetric DCP with a partial prefix cache hit: the cached count
# shifts both the local phase and the remote start position.
pytest.param(
[100],
[200, 201],
2,
2,
0,
4,
3,
[100],
[201],
id="prefix_phase",
),
# Equal (but active) DCP sizes still need to skip the locally cached
# prefix from the front of remote's list.
pytest.param(
[100, 101],
[200, 201, 202, 203],
1,
2,
1,
2,
2,
[100, 101],
[202, 203],
id="equal_dcp_with_prefix",
),
],
)
def test_apply_dcp_prefix_caching_matches_global_logical_positions(
local_ids,
remote_ids,
remote_rank,
local_dcp_size,
local_dcp_rank,
remote_dcp_size,
local_num_computed_blocks,
expected_local,
expected_remote,
):
"""_apply_dcp_prefix_caching must operate on logical block IDs (DCP
interleaves at logical-block granularity) and always return equal-length
local/remote slices, so a later, unconditional _apply_prefix_caching
trim is a guaranteed no-op rather than double-processing the read."""
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker import (
NixlConnectorWorker,
)
worker = object.__new__(NixlConnectorWorker)
actual_local, actual_remote = worker._apply_dcp_prefix_caching(
local_ids,
remote_ids,
remote_rank=remote_rank,
local_dcp_size=local_dcp_size,
local_dcp_rank=local_dcp_rank,
remote_dcp_size=remote_dcp_size,
local_num_computed_blocks=local_num_computed_blocks,
)
assert actual_local == expected_local
assert actual_remote == expected_remote
assert len(actual_local) == len(actual_remote)
@pytest.mark.cpu_test
def test_logical_to_kernel_block_ids_with_hma():
"""Test _logical_to_kernel_block_ids expands blocks when HMA is enabled.
@@ -211,7 +385,12 @@ def test_read_blocks_for_req_expands_remote_ids(
worker = object.__new__(NixlConnectorWorker)
worker._physical_blocks_per_logical_kv_block = local_physical_per_logical
worker._engine_last_active = {}
worker._recving_transfers = {}
worker._bidirectional_kv_xfer_enabled = False
worker.dcp_size = 1
worker.dcp_rank = 0
worker._group_spec_types = resolved_types
worker._has_mamba = any(t is MambaSpec for t in resolved_types)
has_mamba = any(t is MambaSpec for t in resolved_types)
has_swa = any(t is SlidingWindowSpec for t in resolved_types)
@@ -227,6 +406,7 @@ def test_read_blocks_for_req_expands_remote_ids(
worker.transfer_topo.tp_ratio.return_value = tp_ratio
remote_info = MagicMock()
remote_info.remote_physical_blocks_per_logical = remote_physical_per_logical
remote_info.remote_dcp_size = 1
worker.transfer_topo.get_engine_info.return_value = remote_info
worker.use_mla = False
+73 -6
View File
@@ -14,6 +14,7 @@ from types import SimpleNamespace
import numpy as np
import pytest
from vllm.distributed.kv_transfer.kv_connector.utils import TransferTopology
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.tp_mapping import (
TPMapping,
compute_tp_mapping,
@@ -35,17 +36,20 @@ def _compute_mapping(
is_mla: bool = False,
num_kv_heads: int = 8,
group_spec_types: tuple[type, ...] = (FullAttentionSpec,),
dcp_size: int = 1,
remote_dcp_size: int = 1,
) -> TPMapping:
transfer_topology = SimpleNamespace(
tp_rank=tp_rank,
tp_size=tp_size,
is_mla=is_mla,
total_num_kv_heads=num_kv_heads,
)
transfer_topology = object.__new__(TransferTopology)
transfer_topology.tp_rank = tp_rank
transfer_topology.tp_size = tp_size
transfer_topology.is_mla = is_mla
transfer_topology.total_num_kv_heads = num_kv_heads
transfer_topology.dcp_size = dcp_size
return compute_tp_mapping(
transfer_topology=transfer_topology,
remote_tp_size=remote_tp_size,
group_spec_types=group_spec_types,
remote_dcp_size=remote_dcp_size,
)
@@ -68,6 +72,69 @@ class TestTPMappingStructure:
assert m.all_source_ranks == (0, 1)
@pytest.mark.parametrize(
"tp_rank,tp_size,remote_tp_size,dcp_size,remote_dcp_size,expected_ranks",
[
(0, 4, 4, 1, 4, (0, 1, 2, 3)),
(2, 4, 4, 4, 4, (2,)),
(0, 2, 4, 2, 4, (0, 2)),
(3, 4, 2, 4, 2, (1,)),
],
)
def test_mla_dcp_source_ranks(
tp_rank,
tp_size,
remote_tp_size,
dcp_size,
remote_dcp_size,
expected_ranks,
):
mapping = _compute_mapping(
tp_rank=tp_rank,
tp_size=tp_size,
remote_tp_size=remote_tp_size,
is_mla=True,
num_kv_heads=1,
dcp_size=dcp_size,
remote_dcp_size=remote_dcp_size,
)
assert mapping.all_source_ranks == expected_ranks
@pytest.mark.parametrize(
"tp_size,remote_tp_size,dcp_size,remote_dcp_size",
[
(4, 2, 4, 2),
(4, 4, 4, 4),
(4, 1, 4, 1),
(4, 4, 1, 4),
(2, 4, 2, 4),
],
)
def test_dcp_consumer_count_matches_readers(
tp_size, remote_tp_size, dcp_size, remote_dcp_size
):
"""Each producer waits for exactly the local ranks that read from it."""
mappings = [
_compute_mapping(
tp_rank=tp_rank,
tp_size=tp_size,
remote_tp_size=remote_tp_size,
is_mla=True,
num_kv_heads=1,
dcp_size=dcp_size,
remote_dcp_size=remote_dcp_size,
)
for tp_rank in range(tp_size)
]
for remote_rank in range(remote_tp_size):
readers = [m for m in mappings if remote_rank in m.all_source_ranks]
for mapping in readers:
assert mapping.local_consumers == len(readers)
# ======================================================================
# Split handle tests
# ======================================================================
+71
View File
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import gc
from contextlib import nullcontext
from types import SimpleNamespace
from unittest.mock import Mock
@@ -1380,6 +1381,76 @@ def test_hybrid_attention_mamba_tensor_shapes():
assert torch.equal(actual_ssm, expected_ssm)
def test_input_batch_reinitialized_after_late_interleave_adjustment(monkeypatch):
runner = object.__new__(GPUModelRunner)
runner.vllm_config = SimpleNamespace(reasoning_config=None)
runner.parallel_config = SimpleNamespace(cp_kv_cache_interleave_size=16)
runner.cache_config = SimpleNamespace(use_replayssm=False)
runner.model_config = SimpleNamespace(get_vocab_size=lambda: 32)
runner.max_model_len = 64
runner.max_encoder_len = 0
runner.max_num_reqs = 1
runner.max_num_tokens = 64
runner.num_spec_tokens = 0
runner.device = torch.device("cpu")
runner.is_pooling_model = False
runner._init_block_sizes = [16]
runner._init_kernel_block_sizes = [16]
runner._init_max_num_blocks = [4]
runner._init_slot_mapping_modes = [
gpu_model_runner_module.SlotMappingMode.TOKEN_TO_KV_SLOT
]
runner.cp_kv_cache_interleave_size = 1
runner.input_batch = SimpleNamespace(
logitsprocs=None,
logitsprocs_need_output_token_ids=False,
)
runner.jit_warmup_registry = Mock()
runner.jit_warmup_registry.activate.return_value = nullcontext()
spec = SimpleNamespace(
block_size=16,
max_num_blocks_per_req=lambda *_: 4,
)
kv_cache_config = SimpleNamespace(
kv_cache_groups=[SimpleNamespace(kv_cache_spec=spec)]
)
input_batch_cls = Mock(return_value=SimpleNamespace())
monkeypatch.setattr(gpu_model_runner_module, "InputBatch", input_batch_cls)
monkeypatch.setattr(
gpu_model_runner_module,
"get_kv_cache_spec_kind",
lambda _: gpu_model_runner_module.KVCacheSpecKind.FULL_ATTENTION,
)
runner.may_reinitialize_input_batch(kv_cache_config, [16])
assert input_batch_cls.call_count == 1
assert input_batch_cls.call_args.kwargs["cp_kv_cache_interleave_size"] == 16
def test_v2_runner_snapshots_late_interleave_adjustment(monkeypatch):
from vllm.v1.worker.gpu import model_runner as v2_model_runner_module
runner = object.__new__(v2_model_runner_module.GPUModelRunner)
runner.parallel_config = SimpleNamespace(cp_kv_cache_interleave_size=16)
runner.cp_interleave = 1
class StopInitialization(Exception):
pass
monkeypatch.setattr(
v2_model_runner_module,
"deepcopy",
Mock(side_effect=StopInitialization),
)
with pytest.raises(StopInitialization):
runner.initialize_kv_cache(SimpleNamespace())
assert runner.cp_interleave == 16
def test_hybrid_block_table_initialization():
"""Test hybrid block table with different kernel and kvcache_manager block
sizes."""
+6 -1
View File
@@ -57,14 +57,19 @@ def test_initialize_kv_cache_does_not_dcp_shard_mamba_block_table(
KVCacheGroupSpec(["kda"], mamba_spec),
],
)
parallel_config = SimpleNamespace(
decode_context_parallel_size=dcp_size,
cp_kv_cache_interleave_size=1,
)
vllm_config = SimpleNamespace(
parallel_config=SimpleNamespace(decode_context_parallel_size=dcp_size),
parallel_config=parallel_config,
cache_config=SimpleNamespace(mamba_cache_mode=mamba_cache_mode),
)
runner = SimpleNamespace(
max_model_len=max_model_len,
is_encoder_decoder=False,
vllm_config=vllm_config,
parallel_config=parallel_config,
)
class _CapturedWidths(Exception):
+9
View File
@@ -119,3 +119,12 @@ class KVTransferConfig:
def get_from_extra_config(self, key, default) -> Any:
return self.kv_connector_extra_config.get(key, default)
def has_connector(self, connector_name: str) -> bool:
"""Whether ``connector_name`` is configured, directly or in MultiConnector."""
if self.kv_connector == connector_name:
return True
return self.kv_connector == "MultiConnector" and any(
child.get("kv_connector") == connector_name
for child in self.kv_connector_extra_config.get("connectors", [])
)
+78 -15
View File
@@ -1141,6 +1141,30 @@ class VllmConfig:
self._verify_sampling_replay_config()
self._verify_trace_replay_config()
# A NIXL side is either fully replicated or fully DCP-sharded; MLA only.
if (
self.kv_transfer_config is not None
and self.kv_transfer_config.has_connector("NixlConnector")
):
assert self.parallel_config.prefill_context_parallel_size == 1, (
"NIXL does not support prefill context parallelism."
)
dcp_size = self.parallel_config.decode_context_parallel_size
tp_size = self.parallel_config.tensor_parallel_size
assert dcp_size in (1, tp_size), (
f"decode_context_parallel_size={dcp_size} must be 1 or equal "
f"to tensor_parallel_size={tp_size} when using NixlConnector."
)
if self.model_config is not None:
assert self.model_config.use_mla or dcp_size == 1, (
"PD with decode_context_parallel_size > 1 is only "
"supported for MLA models."
)
assert not (self.model_config.is_hybrid and dcp_size > 1), (
"PD with decode_context_parallel_size > 1 is not "
"supported for hybrid Mamba/SSM models."
)
if self.lora_config is not None:
self.lora_config.verify_with_model_config(self.model_config)
@@ -2714,6 +2738,52 @@ class VllmConfig:
f"Model Runner V1 does not support: {', '.join(unsupported)}"
)
def adjust_dcp_kv_cache_interleave_size(
self, kv_cache_config: "KVCacheConfig"
) -> None:
"""Normalize DCP interleave size against the resolved block_size for PD.
Called by each worker (via ensure_kv_transfer_initialized), once it knows its
own final block_size via kv_cache_config.
"""
dcp_size = self.parallel_config.decode_context_parallel_size
if dcp_size <= 1:
return
# Get the kernel block_size, but don't use resolve_kv_cache_block_size to avoid
# scaling by dcp_size (we need the local block_size here).
local_block_size = min(
g.kv_cache_spec.block_size for g in kv_cache_config.kv_cache_groups
)
if self.parallel_config.dcp_kv_cache_interleave_size > 1 and (
self.parallel_config.cp_kv_cache_interleave_size
!= self.parallel_config.dcp_kv_cache_interleave_size
):
self.parallel_config.cp_kv_cache_interleave_size = (
self.parallel_config.dcp_kv_cache_interleave_size
)
logger.warning_once(
"cp_kv_cache_interleave_size is overridden by dcp_kv_cache"
"_interleave_size. And dcp-kv-cache-interleave-size will be "
"deprecated when PCP is fully supported."
)
if (
self.kv_transfer_config is not None
and self.kv_transfer_config.kv_connector is not None
and self.parallel_config.cp_kv_cache_interleave_size != local_block_size
):
interleave = self.parallel_config.cp_kv_cache_interleave_size
self.parallel_config.cp_kv_cache_interleave_size = local_block_size
logger.info_once(
"When using PD disaggregation with DCP "
"(decode_context_parallel_size=%d), "
"cp_kv_cache_interleave_size is automatically adjusted "
"from %d to block_size %d for block-level alignment.",
dcp_size,
interleave,
local_block_size,
)
def validate_block_size(self) -> None:
"""Validate block_size against DCP and mamba constraints.
@@ -2722,20 +2792,14 @@ class VllmConfig:
"""
block_size = self.cache_config.block_size
# DCP interleave-size compatibility
if self.parallel_config.decode_context_parallel_size > 1:
if self.parallel_config.dcp_kv_cache_interleave_size > 1 and (
self.parallel_config.cp_kv_cache_interleave_size
!= self.parallel_config.dcp_kv_cache_interleave_size
):
self.parallel_config.cp_kv_cache_interleave_size = (
self.parallel_config.dcp_kv_cache_interleave_size
)
logger.warning_once(
"cp_kv_cache_interleave_size is overridden by dcp_kv_cache"
"_interleave_size. And dcp-kv-cache-interleave-size will be "
"deprecated when PCP is fully supported."
)
# Skip DCP interleave-size compatibility when a KV connector is configured:
# cp_kv_cache_interleave_size is pinned to block_size for PD by each worker
pd_active = (
self.kv_transfer_config is not None
and self.kv_transfer_config.kv_connector is not None
and self.kv_transfer_config.is_kv_transfer_instance
)
if self.parallel_config.decode_context_parallel_size > 1 and not pd_active:
assert (
self.parallel_config.cp_kv_cache_interleave_size <= block_size
and block_size % self.parallel_config.cp_kv_cache_interleave_size == 0
@@ -2744,7 +2808,6 @@ class VllmConfig:
"than or equal to and divisible by cp_kv_cache_interleave_size "
f"({self.parallel_config.cp_kv_cache_interleave_size})."
)
# Mamba cache align-mode constraints
if self.cache_config.mamba_cache_mode == "align":
assert not self.scheduler_config.disable_chunked_mm_input, (
@@ -404,6 +404,9 @@ class EngineTransferInfo:
end_layer: int = 0
"""Exclusive global index after the last layer owned by this PP rank."""
remote_dcp_size: int = 1
"""Remote decode context parallel size."""
# ---- Transfer topology ----
@@ -420,6 +423,7 @@ class TransferTopology:
is_mamba: bool
total_num_kv_heads: int
attn_backends: list[type[AttentionBackend]]
dcp_size: int = 1
tensor_shape: torch.Size | None = None
def __post_init__(self):
@@ -461,6 +465,15 @@ class TransferTopology:
for key in [k for k in self._engines if k[0] == remote_engine_id]:
del self._engines[key]
@property
def dcp_rank(self) -> int:
"""This rank's DCP coverage rank.
with ``dcp_size in (1, tp_size)`` enforced at the connector boundary, a
rank's DCP identity is always exactly ``tp_rank % dcp_size``.
"""
return self.tp_rank % self.dcp_size
# ============================================================
# Common methods
# ============================================================
@@ -514,18 +527,70 @@ class TransferTopology:
"""Whether the local engine's KV cache is replicated."""
return self.is_mla or self.tp_size > self.total_num_kv_heads
def handshake_target_ranks(self, remote_tp_size: int) -> list[int]:
def dcp_source_ranks(self, remote_tp_size: int, remote_dcp_size: int) -> list[int]:
"""Remote ranks whose DCP slice overlaps mine (MLA, ``remote_dcp_size > 1``).
Shared by ``handshake_target_ranks`` (who to query metadata from) and
``compute_tp_mapping`` (who to actually read from) — for MLA the two
questions have the identical answer, since DCP sharding is the only
thing keeping a remote rank from being interchangeable with any other.
"""
local_dcp_size = self.dcp_size
local_dcp_rank = self.dcp_rank
if local_dcp_size <= remote_dcp_size:
# Keep every remote rank whose slice sits inside mine. When
# local_dcp_size == 1 (replicated locally), local_dcp_rank == 0 reduces to
# every remote rank, since no single one holds the whole sequence.
return [
r for r in range(remote_tp_size) if r % local_dcp_size == local_dcp_rank
]
# Local finer-grained: exactly one remote rank covers my whole slice
return [local_dcp_rank % remote_dcp_size]
def handshake_target_ranks(
self, remote_tp_size: int, remote_dcp_size: int = 1
) -> list[int]:
"""Pre-registration: compute which remote TP ranks to handshake with.
Pure math based on local/remote TP sizes — does not require
the remote engine to be registered yet.
Pure math based on local/remote TP (and DCP, when the remote shards
its KV cache) sizes — does not require the remote engine to be
registered yet.
DCP support is scoped to ``dcp_size in (1, tp_size)`` on each side
and DCP sizes that divide one another: neither side ever has a
partially-duplicated, partially-sharded KV cache. When the
remote is not sharded (``remote_dcp_size == 1``) this reduces
exactly to the DTP>=PTP case, since a sharded local side
already has ``tp_size == dcp_size``.
"""
if remote_dcp_size > 1:
return self.dcp_source_ranks(remote_tp_size, remote_dcp_size)
tp_ratio = self.tp_ratio(remote_tp_size)
if tp_ratio > 0:
return [self.tp_rank // tp_ratio]
abs_ratio = -tp_ratio
return [self.tp_rank * abs_ratio + i for i in range(abs_ratio)]
def dcp_consumer_count(self, remote_tp_size: int, remote_dcp_size: int) -> int:
"""How many local ranks (in aggregate) read from a given remote rank.
Used by the producer side to know how many reader notifications to
wait for before freeing a request's blocks. Reuses ``tp_ratio``
whenever the remote isn't sharded — a sharded local side already
has ``tp_size == dcp_size``, so the existing TP-ratio formula is
already correct there unmodified.
"""
if remote_dcp_size > 1:
if self.dcp_size == 1:
# Replicated locally: every local rank reads every shard.
return self.tp_size
# Both sharded, different degrees.
return max(1, self.dcp_size // remote_dcp_size)
# Remote replicated: `tp_ratio` local ranks share each remote rank
# when local_tp >= remote_tp, else each remote rank has one reader.
return max(1, self.tp_ratio(remote_tp_size))
def target_remote_ranks(
self, remote_engine_id: EngineId, remote_pp_rank: int = 0
) -> list[int]:
@@ -551,6 +616,8 @@ class TransferTopology:
f"local_tp={self.tp_size}, "
f"remote_tp={info.remote_tp_size}, "
f"remote_pp={remote_pp_rank}, "
f"local_dcp={self.dcp_size}, "
f"remote_dcp={info.remote_dcp_size}, "
f"local_rank={self.tp_rank}, "
f"remote_block_len={info.remote_block_len})"
)
@@ -109,7 +109,9 @@ class NixlBaseConnectorScheduler:
# Requests that need to start recv/send.
# New requests are added by update_state_after_alloc in
# the scheduler. Used to make metadata passed to Worker.
self._reqs_need_recv: dict[ReqId, tuple[Request, BlockIds]] = {}
self._reqs_need_recv: dict[
ReqId, tuple[Request, BlockIds, tuple[int, ...]]
] = {}
self._reqs_need_save: dict[ReqId, Request] = {}
# Reqs to send and their expiration time
self._reqs_need_send: dict[ReqId, float] = {}
@@ -207,6 +209,7 @@ class NixlBaseConnectorScheduler:
host = params.get("remote_host")
port = params.get("remote_port")
tp_size = params.get("tp_size")
dcp_size = params.get("dcp_size", 1)
pp_size = params.get("pp_size", 1)
if (
remote_engine_id is None
@@ -222,6 +225,7 @@ class NixlBaseConnectorScheduler:
host=host,
port=port,
tp_size=tp_size,
dcp_size=dcp_size,
pp_size=pp_size,
)
self._heartbeat_by_engine[remote_engine_id].req_ids.add(remote_request_id)
@@ -452,12 +456,13 @@ class NixlBaseConnectorScheduler:
meta = NixlConnectorMetadata()
# Loop through scheduled reqs and convert to ReqMeta.
for req_id, (req, block_ids) in self._reqs_need_recv.items():
for req_id, (req, block_ids, cached) in self._reqs_need_recv.items():
assert req.kv_transfer_params is not None
meta.add_new_req_to_recv(
request_id=req_id,
local_block_ids=block_ids,
kv_transfer_params=req.kv_transfer_params,
local_num_computed_blocks=cached,
)
if self.use_host_buffer:
@@ -231,6 +231,17 @@ class NixlBaseConnectorWorker:
handle.append((addr + p_idx * chunk, chunk, dev))
yield handle
def _needs_split_local_xfer_handles(self, tp_ratio: int, plan: TPMapping) -> bool:
"""Whether reads need per-source slices of the local KV region.
Pure MLA attention is replicated across TP ranks and writes the whole
local region. Multiple physical remote workers may still participate
because DCP assigns them disjoint blocks, but that does not require
splitting the local region. Hybrid MLA+SSM is different: its mapping
contains multiple source ranks for the sharded SSM state.
"""
return tp_ratio < 0 and (not self.use_mla or len(plan.all_source_ranks) > 1)
def _fa_desc_replicated(self, num_fa_descs: int) -> list[bool]:
"""Per-FA-descriptor replicate flag, in _build_fa_local emission order
(region-major; one desc per block, with K/V packed). Length ``num_fa_descs``.
@@ -380,6 +391,13 @@ class NixlBaseConnectorWorker:
self.dcp_size = vllm_config.parallel_config.decode_context_parallel_size
self.pcp_rank = get_pcp_group().rank_in_group if self.pcp_size > 1 else 0
# DCP support is scoped to MLA, with dcp_size in (1, tp_size): either fully
# replicated or fully sharded. A DCP rank is always derivable this way.
self.dcp_rank = self.tp_rank % self.dcp_size
if self._has_mamba and self.dcp_size > 1:
# Prefix-cache-aware DCP slicing isn't implemented for the Mamba group.
raise ValueError("DCP is not supported for hybrid MLA+Mamba models.")
self.num_blocks = kv_cache_config.num_blocks
self.enable_permute_local_kv = False
self.enable_heterogeneous_attn_post_process = False
@@ -543,9 +561,10 @@ class NixlBaseConnectorWorker:
self.compat_hash: str | None = None
self.transfer_topo: TransferTopology | None = None
# With heterogeneous TP, P must wait for all assigned D TP workers to
# finish reading before safely freeing the blocks.
# With heterogeneous TP (or DCP), P must wait for all assigned D
# workers to finish reading before safely freeing the blocks.
self.consumer_notification_counts_by_req = defaultdict[ReqId, int](int)
self.expected_consumer_notifications_by_req: dict[ReqId, int] = {}
self.xfer_stats = NixlKVConnectorStats()
self._physical_blocks_per_logical_kv_block = 1
@@ -556,7 +575,6 @@ class NixlBaseConnectorWorker:
get_representative_spec_type(g.kv_cache_spec)
for g in self.kv_cache_config.transfer_groups
)
# Per-region MLA flag, 1:1 with block_len_per_layer. True -> REPLICATE
# (MLA), False -> SPLIT (head-sharded full-attn). Mixed only for models
# combining both (e.g. GQA main + MLA Eagle-3 draft).
@@ -620,6 +638,7 @@ class NixlBaseConnectorWorker:
port: int,
remote_tp_size: int,
expected_engine_id: str,
remote_dcp_size: int = 1,
remote_pp_size: int = 1,
notif_agents_only: bool = False,
) -> tuple[dict[tuple[int, int], str], float]:
@@ -643,7 +662,9 @@ class NixlBaseConnectorWorker:
# local rank will read from. Note that With homogeneous TP,
# this happens to be the same single rank_i.
assert self.transfer_topo is not None
p_remote_ranks = self.transfer_topo.handshake_target_ranks(remote_tp_size)
p_remote_ranks = self.transfer_topo.handshake_target_ranks(
remote_tp_size, remote_dcp_size
)
remote_rank_to_agent_name: dict[tuple[int, int], str] = {}
path = make_zmq_path("tcp", host, port)
# Clock offset to the peer, estimated from the handshake round-trip.
@@ -747,11 +768,11 @@ class NixlBaseConnectorWorker:
# Register Remote agent.
if notif_agents_only:
remote_agent_name = self._add_notif_only_remote_agent(
metadata, remote_tp_size
metadata, remote_tp_size, metadata.dcp_size
)
else:
remote_agent_name = self.add_remote_agent(
metadata, remote_rank, remote_tp_size
metadata, remote_rank, remote_tp_size, metadata.dcp_size
)
setup_agent_time = time.perf_counter()
logger.debug(
@@ -766,7 +787,7 @@ class NixlBaseConnectorWorker:
return remote_rank_to_agent_name, best_offset
def _add_notif_only_remote_agent(
self, metadata: NixlAgentMetadata, remote_tp_size: int
self, metadata: NixlAgentMetadata, remote_tp_size: int, remote_dcp_size: int = 1
) -> str:
"""Load a remote agent for notifs only on the push-mode decode side.
@@ -782,6 +803,7 @@ class NixlBaseConnectorWorker:
remote_physical_blocks_per_logical=(
metadata.physical_blocks_per_logical_kv_block
),
remote_dcp_size=remote_dcp_size,
),
)
return self.nixl_wrapper.add_remote_agent(metadata.agent_metadata)
@@ -898,6 +920,7 @@ class NixlBaseConnectorWorker:
host: str,
port: int,
tp_size: int,
dcp_size: int = 1,
pp_size: int = 1,
notif_agents_only: bool = False,
) -> Future[tuple[dict[tuple[int, int], str], float]] | None:
@@ -923,6 +946,7 @@ class NixlBaseConnectorWorker:
port,
tp_size,
engine_id,
dcp_size,
pp_size,
notif_agents_only,
)
@@ -960,6 +984,8 @@ class NixlBaseConnectorWorker:
meta.remote.host,
meta.remote.port,
meta.tp_size,
meta.dcp_size,
meta.pp_size,
)
if fut is None:
# Already handshaked — only happens if caller does not pre-check.
@@ -991,8 +1017,11 @@ class NixlBaseConnectorWorker:
block_size=self.block_size,
engine_id=self.engine_id,
is_mla=self.use_mla,
total_num_kv_heads=self.model_config.get_total_num_kv_heads(),
total_num_kv_heads=1
if self.use_mla
else self.model_config.get_total_num_kv_heads(),
attn_backends=self.attn_backends,
dcp_size=self.dcp_size,
# SSM States come in tuples (ssm, conv)
tensor_shape=next(iter(kv_caches.values())).shape
if not self._has_mamba
@@ -1469,6 +1498,7 @@ class NixlBaseConnectorWorker:
nixl_agent_meta: NixlAgentMetadata,
remote_tp_rank: int = 0,
remote_tp_size: int = 1,
remote_dcp_size: int = 1,
) -> str:
"""
Add the remote NIXL agent and prepare the descriptors for reading cache
@@ -1553,6 +1583,7 @@ class NixlBaseConnectorWorker:
remote_block_size=nixl_agent_meta.block_size,
remote_block_len=nixl_agent_meta.block_lens[0],
remote_physical_blocks_per_logical=physical_blocks_per_logical,
remote_dcp_size=remote_dcp_size,
)
transfer_topo.register_remote_engine(engine_id, transfer_info)
logger.info("Transfer plan: %s", transfer_topo.describe(engine_id))
@@ -1561,6 +1592,7 @@ class NixlBaseConnectorWorker:
transfer_topology=transfer_topo,
remote_tp_size=remote_tp_size,
group_spec_types=self._group_spec_types,
remote_dcp_size=remote_dcp_size,
)
remote_agent_name = self.nixl_wrapper.add_remote_agent(
@@ -1583,7 +1615,9 @@ class NixlBaseConnectorWorker:
self.kv_caches_base_addr[engine_id][remote_tp_rank] = (
nixl_agent_meta.kv_caches_base_addr
)
self._validate_remote_agent_handshake(nixl_agent_meta, remote_tp_size)
self._validate_remote_agent_handshake(
nixl_agent_meta, remote_tp_size, remote_dcp_size
)
# This is 1 when P and D `--tensor-parallel-size` match. Otherwise,
# this is the ratio between the two sizes.
@@ -1613,10 +1647,8 @@ class NixlBaseConnectorWorker:
### (Optional) Register local agent memory regions. MLA is not split.
split_key = (tp_ratio, remote_block_size)
if (
tp_ratio < 0
and (not self.use_mla or len(plan.all_source_ranks) > 1)
and split_key not in self.src_xfer_handles_by_tp_ratio
if self._needs_split_local_xfer_handles(tp_ratio, plan) and (
split_key not in self.src_xfer_handles_by_tp_ratio
):
# Remote tp_size > local tp_size: read from multiple remote ranks.
# Logically "split" own regions into per-source chunks. Hybrid
@@ -1674,7 +1706,10 @@ class NixlBaseConnectorWorker:
return remote_agent_name
def _validate_remote_agent_handshake(
self, nixl_agent_meta: NixlAgentMetadata, remote_tp_size: int
self,
nixl_agent_meta: NixlAgentMetadata,
remote_tp_size: int,
remote_dcp_size: int = 1,
):
"""
Validate the remote agent handshake metadata ensuring the
@@ -1685,6 +1720,15 @@ class NixlBaseConnectorWorker:
assert self.transfer_topo is not None
remote_info = self.transfer_topo.get_engine_info(remote_engine_id)
assert remote_info.remote_tp_size == remote_tp_size
assert remote_info.remote_dcp_size == remote_dcp_size
# DCP sizes must divide one another; this is what keeps the
# read-slicing math in pull_worker a closed form.
assert (
self.dcp_size % remote_dcp_size == 0 or remote_dcp_size % self.dcp_size == 0
), (
f"DCP sizes must divide one another: local={self.dcp_size}, "
f"remote={remote_dcp_size} (engine {remote_engine_id})."
)
tp_ratio = self.transfer_topo.tp_ratio(remote_tp_size)
block_size_ratio = self.transfer_topo.block_size_ratio(
@@ -2132,6 +2176,7 @@ class NixlBaseConnectorWorker:
if now < expires:
break
count = self.consumer_notification_counts_by_req.pop(req_id, 0)
self.expected_consumer_notifications_by_req.pop(req_id, None)
self.xfer_stats.record_kv_expired_req()
logger.warning(
"Releasing expired KV blocks for request %s which were "
@@ -2267,6 +2312,7 @@ class NixlBaseConnectorWorker:
hb_info.host,
hb_info.port,
hb_info.tp_size,
hb_info.dcp_size,
hb_info.pp_size,
self._hb_handshake_notif_only and hb_info.pp_size > 1,
)
@@ -2352,7 +2398,7 @@ class NixlBaseConnectorWorker:
This is required when the logical block size (the one set by the user)
does not match the one required by the attn backend.
`ratio` is the number of physical blocks per logical block.
We always receive logical blocks from the engine, so we expand them here eg:
We always receive logical blocks from the engine, so we expand them here eg:
logical block ids: [(SW-clipped) [1], (FA) [2, 3]], ratio=2
physical block ids: [(SW-clipped) [2, 3], (FA) [4, 5, 6, 7]]
"""
@@ -2377,6 +2423,56 @@ class NixlBaseConnectorWorker:
)
return physical_block_ids
def _apply_dcp_prefix_caching(
self,
local_ids: list[int],
remote_ids: list[int],
remote_rank: int,
local_dcp_size: int,
local_dcp_rank: int,
remote_dcp_size: int,
local_num_computed_blocks: int,
) -> tuple[list[int], list[int]]:
"""Match local and remote block IDs by global DCP position.
``local_ids`` excludes blocks already satisfied by the local prefix
cache, while ``remote_ids`` contains the full transferable remote list.
Example:
local DCP size 2, rank 0 owns *global positions*=[0, 2, 4, 6]
cached_blocks=[0,2,4]
to transfer: [6] (the rest is cached)
remote DCP size 4
remote rank 0 owns [0, 4] -> no more match, skip the read
remote rank 2 owns [2, 6] -> match position 6 and read
An empty result (rank 0 above) takes the notification-only path, allowing the
remote to release its blocks without performing a transfer.
"""
local_size, remote_size = local_dcp_size, remote_dcp_size
if local_size == remote_size:
local_slice = local_ids
remote_slice = remote_ids[local_num_computed_blocks:]
elif local_size < remote_size:
k = remote_size // local_size
p = (remote_rank - local_dcp_rank) // local_size
start_local = (p - local_num_computed_blocks) % k
start_remote = (local_num_computed_blocks + start_local - p) // k
local_slice = local_ids[start_local::k]
remote_slice = remote_ids[start_remote:]
else:
k = local_size // remote_size
remote_dcp_rank = remote_rank % remote_size
c = (local_dcp_rank - remote_dcp_rank) // remote_size
start_remote = c + local_num_computed_blocks * k
local_slice = local_ids
remote_slice = remote_ids[start_remote::k]
matched_blocks = min(len(local_slice), len(remote_slice))
return local_slice[:matched_blocks], remote_slice[:matched_blocks]
def _apply_prefix_caching(
self,
decode_block_ids: BlockIds,
@@ -369,6 +369,10 @@ class NixlPushConnector(NixlBaseConnector):
kv_cache_config: "KVCacheConfig",
):
super().__init__(vllm_config, role, kv_cache_config)
if vllm_config.parallel_config.decode_context_parallel_size > 1:
raise ValueError(
"NixlPushConnector does not support decode_context_parallel_size > 1."
)
self.connector_scheduler: NixlPushConnectorScheduler | None = None
self.connector_worker: NixlPushConnectorWorker | None = None
if role == KVConnectorRole.SCHEDULER:
@@ -209,6 +209,7 @@ class HeartbeatInfo:
host: str
port: int
tp_size: int
dcp_size: int = 1
pp_size: int = 1
@@ -228,6 +229,12 @@ class ReqMeta:
# To be used when logical block size does not match the kernel block size
local_physical_block_ids: BlockIds
tp_size: int
dcp_size: int = 1
# Per-KV-cache-group logical blocks this rank already holds, i.e. its
# prefix-cache hit. Fixes where this rank's DCP slice starts relative to
# the remote's; kept per-group since hybrid models (e.g. SWA+FA) can have
# different cache-hit counts per group.
local_num_computed_blocks: tuple[int, ...] = ()
remote: RemoteMeta | None = None
# Remote block size, discovered during NIXL handshake (push mode).
remote_block_size: int | None = None
@@ -261,14 +268,17 @@ class NixlConnectorMetadata(KVConnectorMetadata):
self,
local_block_ids: BlockIds,
kv_transfer_params: dict[str, Any],
local_num_computed_blocks: tuple[int, ...] = (),
) -> ReqMeta:
return ReqMeta(
local_block_ids=local_block_ids,
local_physical_block_ids=local_block_ids,
# P workers don't need to receive tp_size from proxy here.
# P workers don't need to receive these from proxy here.
tp_size=kv_transfer_params.get("tp_size", 1),
dcp_size=kv_transfer_params.get("dcp_size", 1),
remote_block_size=kv_transfer_params.get("remote_block_size"),
pp_size=kv_transfer_params.get("pp_size", 1),
local_num_computed_blocks=local_num_computed_blocks,
)
def add_new_req_to_save(
@@ -286,8 +296,11 @@ class NixlConnectorMetadata(KVConnectorMetadata):
request_id: ReqId,
local_block_ids: BlockIds,
kv_transfer_params: dict[str, Any],
local_num_computed_blocks: tuple[int, ...] = (),
):
req = self._add_new_req(local_block_ids, kv_transfer_params)
req = self._add_new_req(
local_block_ids, kv_transfer_params, local_num_computed_blocks
)
req.remote = RemoteMeta(
block_ids=kv_transfer_params["remote_block_ids"],
engine_id=kv_transfer_params["remote_engine_id"],
@@ -155,12 +155,23 @@ class NixlPullConnectorScheduler(NixlBaseConnectorScheduler):
local_block_ids = self.get_exchange_clipped_blocks(
unhashed_local_block_ids
)
# Blocks covered by the local prefix cache, per KV cache group.
# Each count fixes where that group's DCP slice starts, which the
# worker needs to line up with the remote's slice.
local_num_computed_blocks = tuple(
sum(
block.block_hash is not None and not block.is_null
for block in group
)
for group in blocks.blocks
)
# Get unhashed blocks to pull from remote. Mind that a full prefix
# cache hit is indicated with an empty list.
self._reqs_need_recv[request.request_id] = (
request,
local_block_ids,
local_num_computed_blocks,
)
else:
@@ -213,7 +224,7 @@ class NixlPullConnectorScheduler(NixlBaseConnectorScheduler):
# To avoid stranding the prefill blocks in the prefill instance,
# we must add empty block_ids to _reqs_need_recv so that our
# worker side will notify and free blocks in the prefill instance.
self._reqs_need_recv[request.request_id] = (request, [])
self._reqs_need_recv[request.request_id] = (request, [], ())
params["do_remote_prefill"] = False
return False, None
@@ -272,6 +283,8 @@ class NixlPullConnectorScheduler(NixlBaseConnectorScheduler):
remote_host=self.side_channel_host,
remote_port=self.side_channel_port,
tp_size=self.vllm_config.parallel_config.tensor_parallel_size,
dcp_size=self.vllm_config.parallel_config.decode_context_parallel_size,
pp_size=self.vllm_config.parallel_config.pipeline_parallel_size,
remote_num_tokens=remote_num_tokens,
remote_blocks_expiry_time=blocks_expiry_time,
transfer_mode=self._TRANSFER_MODE,
@@ -5,6 +5,7 @@
import time
from typing import TYPE_CHECKING
from vllm.distributed.kv_transfer.kv_connector.utils import BlockIds
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker import (
NixlBaseConnectorWorker,
)
@@ -143,40 +144,78 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
self._handle_failed_transfer(req_id, None)
return
if any(len(group) > 0 for group in meta.local_block_ids):
# The scheduler waits for finished_recving from *every* worker.
# Under DCP a rank's slice can legitimately come out empty when its
# interleaved positions fall past the end of the sequence. _read_blocks
# then takes the notify-only path without registering a transfer.
# Seed the entry so this rank still reports completion.
self._recving_transfers.setdefault(req_id, [])
plan = self.tp_mappings[engine_id]
remote_info = self.transfer_topo.get_engine_info(engine_id)
tp_ratio = self.transfer_topo.tp_ratio(remote_info.remote_tp_size)
remote_logical_block_ids = meta.remote.block_ids
meta.remote.block_ids = self._logical_to_kernel_block_ids(
meta.remote.block_ids,
remote_logical_block_ids,
remote_info.remote_physical_blocks_per_logical,
)
remote_block_ids = meta.remote.block_ids
local_block_ids = meta.local_physical_block_ids
num_groups = len(local_block_ids)
read_specs = [
ReadSpec(
remote_rank=rank,
local_block_ids=[
list(local_block_ids[g])
if rank in plan.source_ranks_per_group[g]
else []
for g in range(num_groups)
],
remote_block_ids=[
list(remote_block_ids[g])
if rank in plan.source_ranks_per_group[g]
else []
for g in range(num_groups)
],
num_groups = len(meta.local_block_ids)
dcp_active = self.dcp_size > 1 or remote_info.remote_dcp_size > 1
def group_ids(block_ids: BlockIds, rank: int) -> list[list[int]]:
return [
list(block_ids[g]) if rank in plan.source_ranks_per_group[g] else []
for g in range(num_groups)
]
read_specs = []
for rank in plan.all_source_ranks:
if dcp_active:
# DCP interleaves at block granularity, so slicing happens
# here on logical blocks, before kernel-block expansion.
local_ids = group_ids(meta.local_block_ids, rank)
remote_ids = group_ids(remote_logical_block_ids, rank)
for g in range(num_groups):
if not local_ids[g]:
continue
# Prefix cache hit may lead to skip some of the remote reads
# TODO (NickLucche) consider unifying prefix cache handling on
# logical blocks here for both dcp and non-dcp
local_ids[g], remote_ids[g] = self._apply_dcp_prefix_caching(
local_ids[g],
remote_ids[g],
remote_rank=rank,
local_dcp_size=self.dcp_size,
local_dcp_rank=self.dcp_rank,
remote_dcp_size=remote_info.remote_dcp_size,
local_num_computed_blocks=meta.local_num_computed_blocks[g],
)
local_physical_ids = self._logical_to_kernel_block_ids(
local_ids, self._physical_blocks_per_logical_kv_block
)
remote_physical_ids = self._logical_to_kernel_block_ids(
remote_ids, remote_info.remote_physical_blocks_per_logical
)
else:
# No DCP realignment needed: reuse the already-expanded full
# physical lists instead of re-deriving them from logical ids.
local_physical_ids = group_ids(meta.local_physical_block_ids, rank)
remote_physical_ids = group_ids(meta.remote.block_ids, rank)
read_specs.append(
ReadSpec(
remote_rank=rank,
local_block_ids=local_physical_ids,
remote_block_ids=remote_physical_ids,
)
)
for rank in plan.all_source_ranks
]
# D may have to perform multiple reads from different remote ranks.
# Pure MLA reads once because its cache is replicated. Hybrid
# MLA+SSM still needs one read per SSM source rank.
if self.use_mla and tp_ratio < 0 and not self._has_mamba:
# MLA+SSM still needs one read per SSM source rank. With DCP, pure
# MLA may also read from multiple ranks (disjoint token slices).
if self.use_mla and tp_ratio < 0 and not self._has_mamba and not dcp_active:
assert len(read_specs) == 1
for i, spec in enumerate(read_specs):
@@ -214,12 +253,15 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
remote_request_id=meta.remote.request_id,
local_xfer_side_handle=local_xfer_side_handle,
remote_xfer_side_handle=remote_xfer_side_handle,
expected_consumers=plan.local_consumers,
)
if self.use_mla and tp_ratio < 0 and len(read_specs) == 1:
# ..but we still need to notify the other remote ranks that we
# have the blocks we need so they can update the request state.
notif_id = f"{meta.remote.request_id}:{self.world_size}".encode()
# Same thing for DCP (tp_size == dcp_size), so the raw tp_ratio already
# reflects whether any remote replica is left unchosen.
notif_id = f"{meta.remote.request_id}:{plan.local_consumers}".encode()
remote_agents = self._remote_agents[meta.remote.engine_id]
for rank_to_notify, agent in remote_agents.items():
if rank_to_notify != (0, read_specs[0].remote_rank):
@@ -233,6 +275,7 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
remote_request_id: str,
local_xfer_side_handle: int,
remote_xfer_side_handle: int,
expected_consumers: int,
):
"""
Post a READ point-to-point xfer request from a single local worker to
@@ -262,13 +305,13 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
# NOTE(rob): according to nvidia the staging blocks are used to
# saturate IB with heterogeneous TP sizes.
# Number of D TP workers that will read from dst P. Propagate info
# on notification so that dst worker can wait before freeing blocks.
notif_id = f"{remote_request_id}:{self.world_size}".encode()
# Number of local workers that will notify this producer worker.
# Propagate on notification so dst worker can wait before freeing.
notif_id = f"{remote_request_id}:{expected_consumers}".encode()
# Full prefix cache hit: do not need to read remote blocks,
# just notify P worker that we have the blocks we need.
if len(local_block_ids) == 0:
if not any(len(group) > 0 for group in local_block_ids):
# A full prefix cache hit is indicated with an empty list.
agent_name = self._remote_agents[dst_engine_id][(0, remote_rank)]
try:
@@ -292,12 +335,19 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
== len(local_block_ids)
== len(self.kv_cache_config.transfer_groups)
)
local_block_ids, remote_block_ids = self._apply_prefix_caching(
decode_block_ids=local_block_ids,
prefill_block_ids=remote_block_ids,
decode_physical_per_logical=self._physical_blocks_per_logical_kv_block,
prefill_physical_per_logical=remote_info.remote_physical_blocks_per_logical,
)
if not (self.dcp_size > 1 or remote_info.remote_dcp_size > 1):
# DCP-active reads were already trimmed (and DCP-realigned, when
# sizes mismatch) in _read_blocks_for_req, at logical granularity.
local_block_ids, remote_block_ids = self._apply_prefix_caching(
decode_block_ids=local_block_ids,
prefill_block_ids=remote_block_ids,
decode_physical_per_logical=(
self._physical_blocks_per_logical_kv_block
),
prefill_physical_per_logical=(
remote_info.remote_physical_blocks_per_logical
),
)
# NOTE (nicolo) With homogeneous TP, each TP worker loads KV from
# corresponding rank. With heterogeneous TP, fixing D>P, the D tp
@@ -351,8 +401,8 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
def _get_new_notifs(self) -> set[str]:
"""
Get req_ids which got a remote xfer message. When multiple consumers
are reading from the same producer (heterogeneous TP scenario), wait
for all consumers to be done pulling.
are reading from the same producer (heterogeneous TP or DCP
scenario), wait for all consumers to be done pulling.
Also handles heartbeat notifications ("HB:req1,req2,...") by
extending the lease on the referenced requests.
@@ -368,7 +418,7 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
self._handle_heartbeat(msg[3:])
continue
req_id, tp_size = msg.rsplit(":", 1)
req_id, expected_consumers = msg.rsplit(":", 1)
if (
req_id not in self._reqs_to_send
and req_id not in self._reqs_to_process
@@ -381,24 +431,22 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
)
continue
# NOTE: `tp_ratio` is the opposite when swapping local<>remote
n_consumers = int(tp_size)
tp_ratio = self.transfer_topo.tp_ratio(n_consumers)
# Number of reads *per producer* to wait for.
# When remote D TP > local P TP we expect `tp_ratio` reads.
consumers_per_producer = (
-tp_ratio if n_consumers > self.world_size else 1
# Every reader of this req_id reports the same count (it's
# derived from aggregate topology, not the specific rank),
# so repeated notifications never disagree on it.
self.expected_consumer_notifications_by_req[req_id] = int(
expected_consumers
)
self.consumer_notification_counts_by_req[req_id] += 1
# Wait all consumers (D) to be done reading before freeing.
if (
self.consumer_notification_counts_by_req[req_id]
== consumers_per_producer
== self.expected_consumer_notifications_by_req[req_id]
):
notified_req_ids.add(req_id)
del self.consumer_notification_counts_by_req[req_id]
del self.expected_consumer_notifications_by_req[req_id]
self._reqs_to_process.remove(req_id)
self._reqs_to_send.pop(req_id, None)
return notified_req_ids
@@ -198,7 +198,7 @@ class NixlPushConnectorScheduler(NixlBaseConnectorScheduler):
# ReqMeta without a KeyError — the actual remote block IDs are
# learned by P over the NIXL handshake at WRITE time.
params["remote_block_ids"] = ()
self._reqs_need_recv[request.request_id] = (request, local_block_ids)
self._reqs_need_recv[request.request_id] = (request, local_block_ids, ())
# Mark as processed so a re-entry (e.g. preemption + reschedule)
# doesn't re-stage the registration.
@@ -240,7 +240,7 @@ class NixlPushConnectorScheduler(NixlBaseConnectorScheduler):
# recv so the worker emits a notif that lets P free them.
# Seed remote_block_ids so add_new_req_to_recv won't KeyError.
params["remote_block_ids"] = ()
self._reqs_need_recv[request.request_id] = (request, [])
self._reqs_need_recv[request.request_id] = (request, [], ())
params["do_remote_prefill"] = False
return False, None
@@ -56,6 +56,10 @@ class TPMapping:
# FA head offset factor for hetero-TP (D_TP > P_TP).
rank_offset_factor: int
# Local ranks (in aggregate) that read from a given source rank. The producer frees
# a request's blocks only once that many notifications have come in.
local_consumers: int = 1
# ======================================================================
# TP mapping computation
@@ -66,22 +70,31 @@ def compute_tp_mapping(
transfer_topology: TransferTopology,
remote_tp_size: int,
group_spec_types: tuple[type[KVCacheSpec], ...],
remote_dcp_size: int = 1,
) -> TPMapping:
"""Build the complete local-to-remote TP mapping.
Computes source ranks, head slot assignments, and the rank offset
factor in a single pass.
DCP support is scoped to MLA only, with a side is either fully replicated or fully
sharded. DCP-branch reuses the same rank set used at handshake selection.
"""
tp_rank = transfer_topology.tp_rank
tp_size = transfer_topology.tp_size
total_num_kv_heads = transfer_topology.total_num_kv_heads
# --- Attention source ranks ---
if transfer_topology.is_mla or tp_size >= remote_tp_size:
# D (local TP) > P (remote TP): multiple local ranks read different chunks from
# *one* remote rank, corresponding to different kv heads.
# For MLA, we only need one remote since cache is duplicated. When P TP=k*TP k,
# this will spread mla ranks to read from remote k*tp_rank.
attn_ranks = [tp_rank * remote_tp_size // tp_size]
if transfer_topology.is_mla and remote_dcp_size > 1:
attn_ranks = transfer_topology.dcp_source_ranks(
remote_tp_size, remote_dcp_size
)
else:
# D (local TP) > P (remote TP): multiple local ranks read different chunks
# from *one* remote rank, corresponding to different kv heads.
# For MLA, we only need one remote since cache is duplicated. When
# P TP=k*TP k, this will spread mla ranks to read from remote k*tp_rank.
attn_ranks = [tp_rank * remote_tp_size // tp_size]
else:
# P (remote TP) > D (local TP): one local rank
# reads from multiple remote ranks.
@@ -134,9 +147,14 @@ def compute_tp_mapping(
# D TP > P TP: we index into remote to read different heads depending on rank.
rank_offset_factor = tp_rank % (tp_size // remote_tp_size)
local_consumers = transfer_topology.dcp_consumer_count(
remote_tp_size, remote_dcp_size
)
return TPMapping(
source_ranks_per_group=source_ranks_per_group,
all_source_ranks=tuple(all_ranks),
rank_to_attention_slot=rank_to_attention_slot,
rank_offset_factor=rank_offset_factor,
local_consumers=local_consumers,
)
@@ -85,6 +85,8 @@ def ensure_kv_transfer_initialized(
vllm_config.kv_transfer_config.is_kv_transfer_instance
and _KV_CONNECTOR_AGENT is None
):
# PD only supports an interleave_size equal to block_size.
vllm_config.adjust_dcp_kv_cache_interleave_size(kv_cache_config)
_sync_engine_id_across_tp(vllm_config)
_KV_CONNECTOR_AGENT = KVConnectorFactory.create_connector(
@@ -3157,9 +3157,6 @@ class MLACommonImpl(MLACommonBaseImpl[M], Generic[M]):
parallel_config = get_current_vllm_config().parallel_config
# Avoid requiring an initialized DCP group in tests.
self.dcp_world_size: int = parallel_config.decode_context_parallel_size
self.cp_kv_cache_interleave_size: int = (
parallel_config.cp_kv_cache_interleave_size
)
@abstractmethod
def forward_mqa(
@@ -771,14 +771,30 @@ class SparseAttnIndexer(CustomOp):
parallel_config = get_current_vllm_config().parallel_config
self.dcp_world_size = parallel_config.decode_context_parallel_size
self.dcp_rank = get_dcp_group().rank_in_group if self.dcp_world_size > 1 else 0
self.cp_kv_cache_interleave_size = parallel_config.cp_kv_cache_interleave_size
self.use_pcp = parallel_config.prefill_context_parallel_size > 1
self._cp_kv_cache_interleave_size: int | None = None
if current_platform.is_cuda() and not has_deep_gemm():
raise RuntimeError(
"Sparse Attention Indexer CUDA op requires DeepGEMM support in "
"the current vLLM environment."
)
@property
def cp_kv_cache_interleave_size(self) -> int:
"""With PD+DCP, the real value isn't known until block_size is finalized,
which happens after this layer is built. Safe to cache after the first access,
as long as the adjustment always runs before any forward pass
(it's set up in Worker.initialize_from_config, ahead of warmup/serving).
"""
if self._cp_kv_cache_interleave_size is None:
value = (
get_current_vllm_config().parallel_config.cp_kv_cache_interleave_size
)
if isinstance(get_forward_context().attn_metadata, dict):
self._cp_kv_cache_interleave_size = value
return value
return self._cp_kv_cache_interleave_size
def forward_native(
self,
hidden_states: torch.Tensor,
@@ -324,7 +324,6 @@ class FlashInferMLAImpl(MLACommonImpl[MLACommonMetadata]):
extra_kwargs: dict[str, Any] = {}
decode_backend: str | None
if self.dcp_world_size > 1:
assert self.cp_kv_cache_interleave_size == 1
causal_seqlens_kv_global = attn_metadata.decode.dcp_tot_seq_lens
assert causal_seqlens_kv_global is not None
if not attn_metadata.causal and query_len > 1:
+2
View File
@@ -531,6 +531,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
is_profiling: bool = False,
kv_cache_allocation_context: AbstractContextManager | None = None,
) -> None:
# GPUWorker finalizes the PD interleave before KV cache initialization.
self.cp_interleave = self.parallel_config.cp_kv_cache_interleave_size
kv_cache_config = deepcopy(kv_cache_config)
self.kv_cache_config = kv_cache_config
+8
View File
@@ -749,6 +749,9 @@ class GPUModelRunner(
self._init_kernel_block_sizes = [placeholder_block_size]
self._init_max_num_blocks = [placeholder_max_num_blocks]
self._init_slot_mapping_modes = [SlotMappingMode.TOKEN_TO_KV_SLOT]
self.cp_kv_cache_interleave_size = (
self.parallel_config.cp_kv_cache_interleave_size
)
# Capture warmup providers registered by the initial placeholder InputBatch
with self.jit_warmup_registry.activate():
self.input_batch = InputBatch(
@@ -7350,11 +7353,16 @@ class GPUModelRunner(
or kernel_block_sizes != self._init_kernel_block_sizes
or max_num_blocks != self._init_max_num_blocks
or slot_mapping_modes != self._init_slot_mapping_modes
or self.cp_kv_cache_interleave_size
!= self.parallel_config.cp_kv_cache_interleave_size
):
self._init_block_sizes = block_sizes
self._init_kernel_block_sizes = kernel_block_sizes
self._init_max_num_blocks = max_num_blocks
self._init_slot_mapping_modes = slot_mapping_modes
self.cp_kv_cache_interleave_size = (
self.parallel_config.cp_kv_cache_interleave_size
)
# Capture warmup providers registered after final KV-cache geometry is known
with self.jit_warmup_registry.activate():
self.input_batch = InputBatch(