[Nixl][PD] DCP support for MLA models (#50611)
Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
This commit is contained in:
@@ -17,8 +17,11 @@ import vllm.config.vllm as vllm_config_module
|
||||
import vllm.envs as envs
|
||||
from vllm.compilation.backends import VllmBackend
|
||||
from vllm.config import (
|
||||
CacheConfig,
|
||||
CompilationConfig,
|
||||
DeviceConfig,
|
||||
KernelConfig,
|
||||
KVTransferConfig,
|
||||
ModelConfig,
|
||||
ObservabilityConfig,
|
||||
ParallelConfig,
|
||||
@@ -100,6 +103,32 @@ def test_per_request_spec_decode_metrics_requires_spec_decode():
|
||||
)
|
||||
|
||||
|
||||
def test_pd_dcp_interleave_size_is_adjusted_to_block_size(caplog):
|
||||
config = VllmConfig(
|
||||
cache_config=CacheConfig(block_size=16),
|
||||
device_config=DeviceConfig(device="cpu"),
|
||||
parallel_config=ParallelConfig(
|
||||
tensor_parallel_size=2,
|
||||
decode_context_parallel_size=2,
|
||||
cp_kv_cache_interleave_size=3,
|
||||
distributed_executor_backend="mp",
|
||||
),
|
||||
kv_transfer_config=KVTransferConfig(
|
||||
kv_connector="NixlConnector",
|
||||
kv_role="kv_both",
|
||||
),
|
||||
)
|
||||
|
||||
kv_cache_config = SimpleNamespace(
|
||||
kv_cache_groups=[SimpleNamespace(kv_cache_spec=SimpleNamespace(block_size=16))]
|
||||
)
|
||||
with caplog.at_level(logging.INFO):
|
||||
config.adjust_dcp_kv_cache_interleave_size(kv_cache_config)
|
||||
|
||||
assert config.parallel_config.cp_kv_cache_interleave_size == 16
|
||||
assert "automatically adjusted from 3 to block_size 16" in caplog.text
|
||||
|
||||
|
||||
def test_compile_config_repr_succeeds():
|
||||
# setup: VllmBackend mutates the config object
|
||||
config = VllmConfig()
|
||||
@@ -1730,6 +1759,7 @@ def test_validate_mamba_align_subblock_prefill():
|
||||
long_prefill_token_threshold=4096,
|
||||
disable_chunked_mm_input=False,
|
||||
),
|
||||
kv_transfer_config=None,
|
||||
)
|
||||
|
||||
VllmConfig.validate_block_size(config)
|
||||
|
||||
@@ -507,6 +507,7 @@ class FakeNixlConnectorWorker(NixlConnectorWorker):
|
||||
port: int,
|
||||
remote_tp_size: int,
|
||||
expected_engine_id: str,
|
||||
remote_dcp_size: int = 1,
|
||||
remote_pp_size: int = 1,
|
||||
notif_agents_only: bool = False,
|
||||
) -> tuple[dict[tuple[int, int], str], float]:
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
"""Unit tests for NixlConnectorScheduler with HMA and Mamba N-1 prefill."""
|
||||
|
||||
import gc
|
||||
from unittest.mock import patch
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
@@ -63,6 +63,180 @@ def test_sw_sizes(mock_platform, swa_enabled, expected_sw_sizes):
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
@pytest.mark.parametrize(
|
||||
"use_mla,source_ranks,tp_ratio,expected",
|
||||
[
|
||||
pytest.param(True, (0,), -2, False, id="pure_mla_with_remote_dcp"),
|
||||
pytest.param(True, (0, 1), -2, True, id="hybrid_mla_ssm"),
|
||||
pytest.param(False, (0,), -2, True, id="full_attention"),
|
||||
pytest.param(False, (0,), 2, False, id="local_tp_greater"),
|
||||
],
|
||||
)
|
||||
def test_needs_split_local_xfer_handles(use_mla, source_ranks, tp_ratio, expected):
|
||||
"""Handle creation and selection must use the same TP-mapping predicate.
|
||||
|
||||
In pure MLA, DCP can target several physical remote workers even though
|
||||
the TP mapping has one replicated attention source. Those workers own
|
||||
disjoint blocks, so every read uses the whole local-region handle.
|
||||
"""
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker import (
|
||||
NixlConnectorWorker,
|
||||
)
|
||||
|
||||
worker = object.__new__(NixlConnectorWorker)
|
||||
worker.use_mla = use_mla
|
||||
plan = MagicMock()
|
||||
plan.all_source_ranks = source_ranks
|
||||
|
||||
assert worker._needs_split_local_xfer_handles(tp_ratio, plan) is expected
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_update_state_after_alloc_tracks_cached_blocks_per_group():
|
||||
"""Hybrid SWA+FA requests can have different prefix-cache-hit counts per
|
||||
KV cache group. Each group's count must be tracked independently rather
|
||||
than collapsed to a single scalar, or DCP read-phase alignment
|
||||
(local_num_computed_blocks) would misalign one of the groups."""
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.pull_scheduler import (
|
||||
NixlPullConnectorScheduler,
|
||||
)
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
|
||||
from vllm.v1.core.kv_cache_utils import KVCacheBlock
|
||||
|
||||
scheduler = object.__new__(NixlPullConnectorScheduler)
|
||||
scheduler._reqs_in_batch = set()
|
||||
scheduler._reqs_need_save = {}
|
||||
scheduler._reqs_need_recv = {}
|
||||
scheduler.use_host_buffer = False
|
||||
scheduler.is_bidirectional_kv_xfer_enabled = False
|
||||
scheduler._is_hma_required = False
|
||||
scheduler.kv_cache_config = MagicMock(
|
||||
select_transfer_block_ids=lambda block_ids: block_ids
|
||||
)
|
||||
|
||||
def cached(block_id):
|
||||
return KVCacheBlock(block_id=block_id, _block_hash=object())
|
||||
|
||||
def uncached(block_id):
|
||||
return KVCacheBlock(block_id=block_id)
|
||||
|
||||
# Group 0 (full-attention): 2 of 3 blocks cached.
|
||||
# Group 1 (sliding-window): 1 of 3 blocks cached.
|
||||
blocks = KVCacheBlocks(
|
||||
blocks=(
|
||||
[cached(0), cached(1), uncached(2)],
|
||||
[cached(10), uncached(11), uncached(12)],
|
||||
)
|
||||
)
|
||||
|
||||
request = create_request(do_remote_prefill=True)
|
||||
|
||||
scheduler.update_state_after_alloc(request, blocks, num_external_tokens=2)
|
||||
|
||||
_, _, local_num_computed_blocks = scheduler._reqs_need_recv[request.request_id]
|
||||
assert local_num_computed_blocks == (2, 1)
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
@pytest.mark.parametrize(
|
||||
"local_ids,remote_ids,remote_rank,local_dcp_size,local_dcp_rank,"
|
||||
"remote_dcp_size,local_num_computed_blocks,expected_local,expected_remote",
|
||||
[
|
||||
# local more finely sharded than remote: remote's raw slice has an
|
||||
# extra tail block beyond what local's phase-aligned slice covers.
|
||||
pytest.param(
|
||||
[100, 101, 102],
|
||||
[200, 201],
|
||||
1,
|
||||
1,
|
||||
0,
|
||||
2,
|
||||
0,
|
||||
[101],
|
||||
[200],
|
||||
id="remote_tail_padding",
|
||||
),
|
||||
# remote more finely sharded than local: local's raw slice has a
|
||||
# block beyond what remote's phase-aligned slice covers.
|
||||
pytest.param(
|
||||
[100],
|
||||
[200],
|
||||
0,
|
||||
2,
|
||||
1,
|
||||
1,
|
||||
0,
|
||||
[],
|
||||
[],
|
||||
id="local_tail_padding",
|
||||
),
|
||||
# Asymmetric DCP with a partial prefix cache hit: the cached count
|
||||
# shifts both the local phase and the remote start position.
|
||||
pytest.param(
|
||||
[100],
|
||||
[200, 201],
|
||||
2,
|
||||
2,
|
||||
0,
|
||||
4,
|
||||
3,
|
||||
[100],
|
||||
[201],
|
||||
id="prefix_phase",
|
||||
),
|
||||
# Equal (but active) DCP sizes still need to skip the locally cached
|
||||
# prefix from the front of remote's list.
|
||||
pytest.param(
|
||||
[100, 101],
|
||||
[200, 201, 202, 203],
|
||||
1,
|
||||
2,
|
||||
1,
|
||||
2,
|
||||
2,
|
||||
[100, 101],
|
||||
[202, 203],
|
||||
id="equal_dcp_with_prefix",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_apply_dcp_prefix_caching_matches_global_logical_positions(
|
||||
local_ids,
|
||||
remote_ids,
|
||||
remote_rank,
|
||||
local_dcp_size,
|
||||
local_dcp_rank,
|
||||
remote_dcp_size,
|
||||
local_num_computed_blocks,
|
||||
expected_local,
|
||||
expected_remote,
|
||||
):
|
||||
"""_apply_dcp_prefix_caching must operate on logical block IDs (DCP
|
||||
interleaves at logical-block granularity) and always return equal-length
|
||||
local/remote slices, so a later, unconditional _apply_prefix_caching
|
||||
trim is a guaranteed no-op rather than double-processing the read."""
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker import (
|
||||
NixlConnectorWorker,
|
||||
)
|
||||
|
||||
worker = object.__new__(NixlConnectorWorker)
|
||||
|
||||
actual_local, actual_remote = worker._apply_dcp_prefix_caching(
|
||||
local_ids,
|
||||
remote_ids,
|
||||
remote_rank=remote_rank,
|
||||
local_dcp_size=local_dcp_size,
|
||||
local_dcp_rank=local_dcp_rank,
|
||||
remote_dcp_size=remote_dcp_size,
|
||||
local_num_computed_blocks=local_num_computed_blocks,
|
||||
)
|
||||
|
||||
assert actual_local == expected_local
|
||||
assert actual_remote == expected_remote
|
||||
assert len(actual_local) == len(actual_remote)
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_logical_to_kernel_block_ids_with_hma():
|
||||
"""Test _logical_to_kernel_block_ids expands blocks when HMA is enabled.
|
||||
@@ -211,7 +385,12 @@ def test_read_blocks_for_req_expands_remote_ids(
|
||||
worker = object.__new__(NixlConnectorWorker)
|
||||
worker._physical_blocks_per_logical_kv_block = local_physical_per_logical
|
||||
worker._engine_last_active = {}
|
||||
worker._recving_transfers = {}
|
||||
worker._bidirectional_kv_xfer_enabled = False
|
||||
worker.dcp_size = 1
|
||||
worker.dcp_rank = 0
|
||||
worker._group_spec_types = resolved_types
|
||||
worker._has_mamba = any(t is MambaSpec for t in resolved_types)
|
||||
|
||||
has_mamba = any(t is MambaSpec for t in resolved_types)
|
||||
has_swa = any(t is SlidingWindowSpec for t in resolved_types)
|
||||
@@ -227,6 +406,7 @@ def test_read_blocks_for_req_expands_remote_ids(
|
||||
worker.transfer_topo.tp_ratio.return_value = tp_ratio
|
||||
remote_info = MagicMock()
|
||||
remote_info.remote_physical_blocks_per_logical = remote_physical_per_logical
|
||||
remote_info.remote_dcp_size = 1
|
||||
worker.transfer_topo.get_engine_info.return_value = remote_info
|
||||
worker.use_mla = False
|
||||
|
||||
|
||||
@@ -14,6 +14,7 @@ from types import SimpleNamespace
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.utils import TransferTopology
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.tp_mapping import (
|
||||
TPMapping,
|
||||
compute_tp_mapping,
|
||||
@@ -35,17 +36,20 @@ def _compute_mapping(
|
||||
is_mla: bool = False,
|
||||
num_kv_heads: int = 8,
|
||||
group_spec_types: tuple[type, ...] = (FullAttentionSpec,),
|
||||
dcp_size: int = 1,
|
||||
remote_dcp_size: int = 1,
|
||||
) -> TPMapping:
|
||||
transfer_topology = SimpleNamespace(
|
||||
tp_rank=tp_rank,
|
||||
tp_size=tp_size,
|
||||
is_mla=is_mla,
|
||||
total_num_kv_heads=num_kv_heads,
|
||||
)
|
||||
transfer_topology = object.__new__(TransferTopology)
|
||||
transfer_topology.tp_rank = tp_rank
|
||||
transfer_topology.tp_size = tp_size
|
||||
transfer_topology.is_mla = is_mla
|
||||
transfer_topology.total_num_kv_heads = num_kv_heads
|
||||
transfer_topology.dcp_size = dcp_size
|
||||
return compute_tp_mapping(
|
||||
transfer_topology=transfer_topology,
|
||||
remote_tp_size=remote_tp_size,
|
||||
group_spec_types=group_spec_types,
|
||||
remote_dcp_size=remote_dcp_size,
|
||||
)
|
||||
|
||||
|
||||
@@ -68,6 +72,69 @@ class TestTPMappingStructure:
|
||||
assert m.all_source_ranks == (0, 1)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"tp_rank,tp_size,remote_tp_size,dcp_size,remote_dcp_size,expected_ranks",
|
||||
[
|
||||
(0, 4, 4, 1, 4, (0, 1, 2, 3)),
|
||||
(2, 4, 4, 4, 4, (2,)),
|
||||
(0, 2, 4, 2, 4, (0, 2)),
|
||||
(3, 4, 2, 4, 2, (1,)),
|
||||
],
|
||||
)
|
||||
def test_mla_dcp_source_ranks(
|
||||
tp_rank,
|
||||
tp_size,
|
||||
remote_tp_size,
|
||||
dcp_size,
|
||||
remote_dcp_size,
|
||||
expected_ranks,
|
||||
):
|
||||
mapping = _compute_mapping(
|
||||
tp_rank=tp_rank,
|
||||
tp_size=tp_size,
|
||||
remote_tp_size=remote_tp_size,
|
||||
is_mla=True,
|
||||
num_kv_heads=1,
|
||||
dcp_size=dcp_size,
|
||||
remote_dcp_size=remote_dcp_size,
|
||||
)
|
||||
|
||||
assert mapping.all_source_ranks == expected_ranks
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"tp_size,remote_tp_size,dcp_size,remote_dcp_size",
|
||||
[
|
||||
(4, 2, 4, 2),
|
||||
(4, 4, 4, 4),
|
||||
(4, 1, 4, 1),
|
||||
(4, 4, 1, 4),
|
||||
(2, 4, 2, 4),
|
||||
],
|
||||
)
|
||||
def test_dcp_consumer_count_matches_readers(
|
||||
tp_size, remote_tp_size, dcp_size, remote_dcp_size
|
||||
):
|
||||
"""Each producer waits for exactly the local ranks that read from it."""
|
||||
mappings = [
|
||||
_compute_mapping(
|
||||
tp_rank=tp_rank,
|
||||
tp_size=tp_size,
|
||||
remote_tp_size=remote_tp_size,
|
||||
is_mla=True,
|
||||
num_kv_heads=1,
|
||||
dcp_size=dcp_size,
|
||||
remote_dcp_size=remote_dcp_size,
|
||||
)
|
||||
for tp_rank in range(tp_size)
|
||||
]
|
||||
|
||||
for remote_rank in range(remote_tp_size):
|
||||
readers = [m for m in mappings if remote_rank in m.all_source_ranks]
|
||||
for mapping in readers:
|
||||
assert mapping.local_consumers == len(readers)
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# Split handle tests
|
||||
# ======================================================================
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import gc
|
||||
from contextlib import nullcontext
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
@@ -1380,6 +1381,76 @@ def test_hybrid_attention_mamba_tensor_shapes():
|
||||
assert torch.equal(actual_ssm, expected_ssm)
|
||||
|
||||
|
||||
def test_input_batch_reinitialized_after_late_interleave_adjustment(monkeypatch):
|
||||
runner = object.__new__(GPUModelRunner)
|
||||
runner.vllm_config = SimpleNamespace(reasoning_config=None)
|
||||
runner.parallel_config = SimpleNamespace(cp_kv_cache_interleave_size=16)
|
||||
runner.cache_config = SimpleNamespace(use_replayssm=False)
|
||||
runner.model_config = SimpleNamespace(get_vocab_size=lambda: 32)
|
||||
runner.max_model_len = 64
|
||||
runner.max_encoder_len = 0
|
||||
runner.max_num_reqs = 1
|
||||
runner.max_num_tokens = 64
|
||||
runner.num_spec_tokens = 0
|
||||
runner.device = torch.device("cpu")
|
||||
runner.is_pooling_model = False
|
||||
runner._init_block_sizes = [16]
|
||||
runner._init_kernel_block_sizes = [16]
|
||||
runner._init_max_num_blocks = [4]
|
||||
runner._init_slot_mapping_modes = [
|
||||
gpu_model_runner_module.SlotMappingMode.TOKEN_TO_KV_SLOT
|
||||
]
|
||||
runner.cp_kv_cache_interleave_size = 1
|
||||
runner.input_batch = SimpleNamespace(
|
||||
logitsprocs=None,
|
||||
logitsprocs_need_output_token_ids=False,
|
||||
)
|
||||
runner.jit_warmup_registry = Mock()
|
||||
runner.jit_warmup_registry.activate.return_value = nullcontext()
|
||||
|
||||
spec = SimpleNamespace(
|
||||
block_size=16,
|
||||
max_num_blocks_per_req=lambda *_: 4,
|
||||
)
|
||||
kv_cache_config = SimpleNamespace(
|
||||
kv_cache_groups=[SimpleNamespace(kv_cache_spec=spec)]
|
||||
)
|
||||
input_batch_cls = Mock(return_value=SimpleNamespace())
|
||||
monkeypatch.setattr(gpu_model_runner_module, "InputBatch", input_batch_cls)
|
||||
monkeypatch.setattr(
|
||||
gpu_model_runner_module,
|
||||
"get_kv_cache_spec_kind",
|
||||
lambda _: gpu_model_runner_module.KVCacheSpecKind.FULL_ATTENTION,
|
||||
)
|
||||
|
||||
runner.may_reinitialize_input_batch(kv_cache_config, [16])
|
||||
|
||||
assert input_batch_cls.call_count == 1
|
||||
assert input_batch_cls.call_args.kwargs["cp_kv_cache_interleave_size"] == 16
|
||||
|
||||
|
||||
def test_v2_runner_snapshots_late_interleave_adjustment(monkeypatch):
|
||||
from vllm.v1.worker.gpu import model_runner as v2_model_runner_module
|
||||
|
||||
runner = object.__new__(v2_model_runner_module.GPUModelRunner)
|
||||
runner.parallel_config = SimpleNamespace(cp_kv_cache_interleave_size=16)
|
||||
runner.cp_interleave = 1
|
||||
|
||||
class StopInitialization(Exception):
|
||||
pass
|
||||
|
||||
monkeypatch.setattr(
|
||||
v2_model_runner_module,
|
||||
"deepcopy",
|
||||
Mock(side_effect=StopInitialization),
|
||||
)
|
||||
|
||||
with pytest.raises(StopInitialization):
|
||||
runner.initialize_kv_cache(SimpleNamespace())
|
||||
|
||||
assert runner.cp_interleave == 16
|
||||
|
||||
|
||||
def test_hybrid_block_table_initialization():
|
||||
"""Test hybrid block table with different kernel and kvcache_manager block
|
||||
sizes."""
|
||||
|
||||
@@ -57,14 +57,19 @@ def test_initialize_kv_cache_does_not_dcp_shard_mamba_block_table(
|
||||
KVCacheGroupSpec(["kda"], mamba_spec),
|
||||
],
|
||||
)
|
||||
parallel_config = SimpleNamespace(
|
||||
decode_context_parallel_size=dcp_size,
|
||||
cp_kv_cache_interleave_size=1,
|
||||
)
|
||||
vllm_config = SimpleNamespace(
|
||||
parallel_config=SimpleNamespace(decode_context_parallel_size=dcp_size),
|
||||
parallel_config=parallel_config,
|
||||
cache_config=SimpleNamespace(mamba_cache_mode=mamba_cache_mode),
|
||||
)
|
||||
runner = SimpleNamespace(
|
||||
max_model_len=max_model_len,
|
||||
is_encoder_decoder=False,
|
||||
vllm_config=vllm_config,
|
||||
parallel_config=parallel_config,
|
||||
)
|
||||
|
||||
class _CapturedWidths(Exception):
|
||||
|
||||
Reference in New Issue
Block a user