[Nixl][PD] DCP support for MLA models (#50611)

Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
This commit is contained in:
Nicolò Lucchesi
2026-08-29 16:03:53 +00:00
committed by GitHub
parent 4fc943b867
commit 7f4793eaa3
23 changed files with 818 additions and 104 deletions
+30
View File
@@ -17,8 +17,11 @@ import vllm.config.vllm as vllm_config_module
import vllm.envs as envs
from vllm.compilation.backends import VllmBackend
from vllm.config import (
CacheConfig,
CompilationConfig,
DeviceConfig,
KernelConfig,
KVTransferConfig,
ModelConfig,
ObservabilityConfig,
ParallelConfig,
@@ -100,6 +103,32 @@ def test_per_request_spec_decode_metrics_requires_spec_decode():
)
def test_pd_dcp_interleave_size_is_adjusted_to_block_size(caplog):
config = VllmConfig(
cache_config=CacheConfig(block_size=16),
device_config=DeviceConfig(device="cpu"),
parallel_config=ParallelConfig(
tensor_parallel_size=2,
decode_context_parallel_size=2,
cp_kv_cache_interleave_size=3,
distributed_executor_backend="mp",
),
kv_transfer_config=KVTransferConfig(
kv_connector="NixlConnector",
kv_role="kv_both",
),
)
kv_cache_config = SimpleNamespace(
kv_cache_groups=[SimpleNamespace(kv_cache_spec=SimpleNamespace(block_size=16))]
)
with caplog.at_level(logging.INFO):
config.adjust_dcp_kv_cache_interleave_size(kv_cache_config)
assert config.parallel_config.cp_kv_cache_interleave_size == 16
assert "automatically adjusted from 3 to block_size 16" in caplog.text
def test_compile_config_repr_succeeds():
# setup: VllmBackend mutates the config object
config = VllmConfig()
@@ -1730,6 +1759,7 @@ def test_validate_mamba_align_subblock_prefill():
long_prefill_token_threshold=4096,
disable_chunked_mm_input=False,
),
kv_transfer_config=None,
)
VllmConfig.validate_block_size(config)
@@ -507,6 +507,7 @@ class FakeNixlConnectorWorker(NixlConnectorWorker):
port: int,
remote_tp_size: int,
expected_engine_id: str,
remote_dcp_size: int = 1,
remote_pp_size: int = 1,
notif_agents_only: bool = False,
) -> tuple[dict[tuple[int, int], str], float]:
@@ -3,7 +3,7 @@
"""Unit tests for NixlConnectorScheduler with HMA and Mamba N-1 prefill."""
import gc
from unittest.mock import patch
from unittest.mock import MagicMock, patch
import pytest
import torch
@@ -63,6 +63,180 @@ def test_sw_sizes(mock_platform, swa_enabled, expected_sw_sizes):
)
@pytest.mark.cpu_test
@pytest.mark.parametrize(
"use_mla,source_ranks,tp_ratio,expected",
[
pytest.param(True, (0,), -2, False, id="pure_mla_with_remote_dcp"),
pytest.param(True, (0, 1), -2, True, id="hybrid_mla_ssm"),
pytest.param(False, (0,), -2, True, id="full_attention"),
pytest.param(False, (0,), 2, False, id="local_tp_greater"),
],
)
def test_needs_split_local_xfer_handles(use_mla, source_ranks, tp_ratio, expected):
"""Handle creation and selection must use the same TP-mapping predicate.
In pure MLA, DCP can target several physical remote workers even though
the TP mapping has one replicated attention source. Those workers own
disjoint blocks, so every read uses the whole local-region handle.
"""
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker import (
NixlConnectorWorker,
)
worker = object.__new__(NixlConnectorWorker)
worker.use_mla = use_mla
plan = MagicMock()
plan.all_source_ranks = source_ranks
assert worker._needs_split_local_xfer_handles(tp_ratio, plan) is expected
@pytest.mark.cpu_test
def test_update_state_after_alloc_tracks_cached_blocks_per_group():
"""Hybrid SWA+FA requests can have different prefix-cache-hit counts per
KV cache group. Each group's count must be tracked independently rather
than collapsed to a single scalar, or DCP read-phase alignment
(local_num_computed_blocks) would misalign one of the groups."""
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.pull_scheduler import (
NixlPullConnectorScheduler,
)
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
from vllm.v1.core.kv_cache_utils import KVCacheBlock
scheduler = object.__new__(NixlPullConnectorScheduler)
scheduler._reqs_in_batch = set()
scheduler._reqs_need_save = {}
scheduler._reqs_need_recv = {}
scheduler.use_host_buffer = False
scheduler.is_bidirectional_kv_xfer_enabled = False
scheduler._is_hma_required = False
scheduler.kv_cache_config = MagicMock(
select_transfer_block_ids=lambda block_ids: block_ids
)
def cached(block_id):
return KVCacheBlock(block_id=block_id, _block_hash=object())
def uncached(block_id):
return KVCacheBlock(block_id=block_id)
# Group 0 (full-attention): 2 of 3 blocks cached.
# Group 1 (sliding-window): 1 of 3 blocks cached.
blocks = KVCacheBlocks(
blocks=(
[cached(0), cached(1), uncached(2)],
[cached(10), uncached(11), uncached(12)],
)
)
request = create_request(do_remote_prefill=True)
scheduler.update_state_after_alloc(request, blocks, num_external_tokens=2)
_, _, local_num_computed_blocks = scheduler._reqs_need_recv[request.request_id]
assert local_num_computed_blocks == (2, 1)
@pytest.mark.cpu_test
@pytest.mark.parametrize(
"local_ids,remote_ids,remote_rank,local_dcp_size,local_dcp_rank,"
"remote_dcp_size,local_num_computed_blocks,expected_local,expected_remote",
[
# local more finely sharded than remote: remote's raw slice has an
# extra tail block beyond what local's phase-aligned slice covers.
pytest.param(
[100, 101, 102],
[200, 201],
1,
1,
0,
2,
0,
[101],
[200],
id="remote_tail_padding",
),
# remote more finely sharded than local: local's raw slice has a
# block beyond what remote's phase-aligned slice covers.
pytest.param(
[100],
[200],
0,
2,
1,
1,
0,
[],
[],
id="local_tail_padding",
),
# Asymmetric DCP with a partial prefix cache hit: the cached count
# shifts both the local phase and the remote start position.
pytest.param(
[100],
[200, 201],
2,
2,
0,
4,
3,
[100],
[201],
id="prefix_phase",
),
# Equal (but active) DCP sizes still need to skip the locally cached
# prefix from the front of remote's list.
pytest.param(
[100, 101],
[200, 201, 202, 203],
1,
2,
1,
2,
2,
[100, 101],
[202, 203],
id="equal_dcp_with_prefix",
),
],
)
def test_apply_dcp_prefix_caching_matches_global_logical_positions(
local_ids,
remote_ids,
remote_rank,
local_dcp_size,
local_dcp_rank,
remote_dcp_size,
local_num_computed_blocks,
expected_local,
expected_remote,
):
"""_apply_dcp_prefix_caching must operate on logical block IDs (DCP
interleaves at logical-block granularity) and always return equal-length
local/remote slices, so a later, unconditional _apply_prefix_caching
trim is a guaranteed no-op rather than double-processing the read."""
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker import (
NixlConnectorWorker,
)
worker = object.__new__(NixlConnectorWorker)
actual_local, actual_remote = worker._apply_dcp_prefix_caching(
local_ids,
remote_ids,
remote_rank=remote_rank,
local_dcp_size=local_dcp_size,
local_dcp_rank=local_dcp_rank,
remote_dcp_size=remote_dcp_size,
local_num_computed_blocks=local_num_computed_blocks,
)
assert actual_local == expected_local
assert actual_remote == expected_remote
assert len(actual_local) == len(actual_remote)
@pytest.mark.cpu_test
def test_logical_to_kernel_block_ids_with_hma():
"""Test _logical_to_kernel_block_ids expands blocks when HMA is enabled.
@@ -211,7 +385,12 @@ def test_read_blocks_for_req_expands_remote_ids(
worker = object.__new__(NixlConnectorWorker)
worker._physical_blocks_per_logical_kv_block = local_physical_per_logical
worker._engine_last_active = {}
worker._recving_transfers = {}
worker._bidirectional_kv_xfer_enabled = False
worker.dcp_size = 1
worker.dcp_rank = 0
worker._group_spec_types = resolved_types
worker._has_mamba = any(t is MambaSpec for t in resolved_types)
has_mamba = any(t is MambaSpec for t in resolved_types)
has_swa = any(t is SlidingWindowSpec for t in resolved_types)
@@ -227,6 +406,7 @@ def test_read_blocks_for_req_expands_remote_ids(
worker.transfer_topo.tp_ratio.return_value = tp_ratio
remote_info = MagicMock()
remote_info.remote_physical_blocks_per_logical = remote_physical_per_logical
remote_info.remote_dcp_size = 1
worker.transfer_topo.get_engine_info.return_value = remote_info
worker.use_mla = False
+73 -6
View File
@@ -14,6 +14,7 @@ from types import SimpleNamespace
import numpy as np
import pytest
from vllm.distributed.kv_transfer.kv_connector.utils import TransferTopology
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.tp_mapping import (
TPMapping,
compute_tp_mapping,
@@ -35,17 +36,20 @@ def _compute_mapping(
is_mla: bool = False,
num_kv_heads: int = 8,
group_spec_types: tuple[type, ...] = (FullAttentionSpec,),
dcp_size: int = 1,
remote_dcp_size: int = 1,
) -> TPMapping:
transfer_topology = SimpleNamespace(
tp_rank=tp_rank,
tp_size=tp_size,
is_mla=is_mla,
total_num_kv_heads=num_kv_heads,
)
transfer_topology = object.__new__(TransferTopology)
transfer_topology.tp_rank = tp_rank
transfer_topology.tp_size = tp_size
transfer_topology.is_mla = is_mla
transfer_topology.total_num_kv_heads = num_kv_heads
transfer_topology.dcp_size = dcp_size
return compute_tp_mapping(
transfer_topology=transfer_topology,
remote_tp_size=remote_tp_size,
group_spec_types=group_spec_types,
remote_dcp_size=remote_dcp_size,
)
@@ -68,6 +72,69 @@ class TestTPMappingStructure:
assert m.all_source_ranks == (0, 1)
@pytest.mark.parametrize(
"tp_rank,tp_size,remote_tp_size,dcp_size,remote_dcp_size,expected_ranks",
[
(0, 4, 4, 1, 4, (0, 1, 2, 3)),
(2, 4, 4, 4, 4, (2,)),
(0, 2, 4, 2, 4, (0, 2)),
(3, 4, 2, 4, 2, (1,)),
],
)
def test_mla_dcp_source_ranks(
tp_rank,
tp_size,
remote_tp_size,
dcp_size,
remote_dcp_size,
expected_ranks,
):
mapping = _compute_mapping(
tp_rank=tp_rank,
tp_size=tp_size,
remote_tp_size=remote_tp_size,
is_mla=True,
num_kv_heads=1,
dcp_size=dcp_size,
remote_dcp_size=remote_dcp_size,
)
assert mapping.all_source_ranks == expected_ranks
@pytest.mark.parametrize(
"tp_size,remote_tp_size,dcp_size,remote_dcp_size",
[
(4, 2, 4, 2),
(4, 4, 4, 4),
(4, 1, 4, 1),
(4, 4, 1, 4),
(2, 4, 2, 4),
],
)
def test_dcp_consumer_count_matches_readers(
tp_size, remote_tp_size, dcp_size, remote_dcp_size
):
"""Each producer waits for exactly the local ranks that read from it."""
mappings = [
_compute_mapping(
tp_rank=tp_rank,
tp_size=tp_size,
remote_tp_size=remote_tp_size,
is_mla=True,
num_kv_heads=1,
dcp_size=dcp_size,
remote_dcp_size=remote_dcp_size,
)
for tp_rank in range(tp_size)
]
for remote_rank in range(remote_tp_size):
readers = [m for m in mappings if remote_rank in m.all_source_ranks]
for mapping in readers:
assert mapping.local_consumers == len(readers)
# ======================================================================
# Split handle tests
# ======================================================================
+71
View File
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import gc
from contextlib import nullcontext
from types import SimpleNamespace
from unittest.mock import Mock
@@ -1380,6 +1381,76 @@ def test_hybrid_attention_mamba_tensor_shapes():
assert torch.equal(actual_ssm, expected_ssm)
def test_input_batch_reinitialized_after_late_interleave_adjustment(monkeypatch):
runner = object.__new__(GPUModelRunner)
runner.vllm_config = SimpleNamespace(reasoning_config=None)
runner.parallel_config = SimpleNamespace(cp_kv_cache_interleave_size=16)
runner.cache_config = SimpleNamespace(use_replayssm=False)
runner.model_config = SimpleNamespace(get_vocab_size=lambda: 32)
runner.max_model_len = 64
runner.max_encoder_len = 0
runner.max_num_reqs = 1
runner.max_num_tokens = 64
runner.num_spec_tokens = 0
runner.device = torch.device("cpu")
runner.is_pooling_model = False
runner._init_block_sizes = [16]
runner._init_kernel_block_sizes = [16]
runner._init_max_num_blocks = [4]
runner._init_slot_mapping_modes = [
gpu_model_runner_module.SlotMappingMode.TOKEN_TO_KV_SLOT
]
runner.cp_kv_cache_interleave_size = 1
runner.input_batch = SimpleNamespace(
logitsprocs=None,
logitsprocs_need_output_token_ids=False,
)
runner.jit_warmup_registry = Mock()
runner.jit_warmup_registry.activate.return_value = nullcontext()
spec = SimpleNamespace(
block_size=16,
max_num_blocks_per_req=lambda *_: 4,
)
kv_cache_config = SimpleNamespace(
kv_cache_groups=[SimpleNamespace(kv_cache_spec=spec)]
)
input_batch_cls = Mock(return_value=SimpleNamespace())
monkeypatch.setattr(gpu_model_runner_module, "InputBatch", input_batch_cls)
monkeypatch.setattr(
gpu_model_runner_module,
"get_kv_cache_spec_kind",
lambda _: gpu_model_runner_module.KVCacheSpecKind.FULL_ATTENTION,
)
runner.may_reinitialize_input_batch(kv_cache_config, [16])
assert input_batch_cls.call_count == 1
assert input_batch_cls.call_args.kwargs["cp_kv_cache_interleave_size"] == 16
def test_v2_runner_snapshots_late_interleave_adjustment(monkeypatch):
from vllm.v1.worker.gpu import model_runner as v2_model_runner_module
runner = object.__new__(v2_model_runner_module.GPUModelRunner)
runner.parallel_config = SimpleNamespace(cp_kv_cache_interleave_size=16)
runner.cp_interleave = 1
class StopInitialization(Exception):
pass
monkeypatch.setattr(
v2_model_runner_module,
"deepcopy",
Mock(side_effect=StopInitialization),
)
with pytest.raises(StopInitialization):
runner.initialize_kv_cache(SimpleNamespace())
assert runner.cp_interleave == 16
def test_hybrid_block_table_initialization():
"""Test hybrid block table with different kernel and kvcache_manager block
sizes."""
+6 -1
View File
@@ -57,14 +57,19 @@ def test_initialize_kv_cache_does_not_dcp_shard_mamba_block_table(
KVCacheGroupSpec(["kda"], mamba_spec),
],
)
parallel_config = SimpleNamespace(
decode_context_parallel_size=dcp_size,
cp_kv_cache_interleave_size=1,
)
vllm_config = SimpleNamespace(
parallel_config=SimpleNamespace(decode_context_parallel_size=dcp_size),
parallel_config=parallel_config,
cache_config=SimpleNamespace(mamba_cache_mode=mamba_cache_mode),
)
runner = SimpleNamespace(
max_model_len=max_model_len,
is_encoder_decoder=False,
vllm_config=vllm_config,
parallel_config=parallel_config,
)
class _CapturedWidths(Exception):