nixl refactor: new transfer design (#40731)

Signed-off-by: ZhanqiuHu <zhu@redhat.com>
Signed-off-by: NickLucche <nlucches@redhat.com>
Co-authored-by: NickLucche <nlucches@redhat.com>
This commit is contained in:
zhanqiuhu
2026-05-06 06:16:25 -07:00
committed by GitHub
co-authored by NickLucche
parent 242afc6bf4
commit df8e63f4ed
8 changed files with 753 additions and 877 deletions
@@ -527,6 +527,7 @@ class FakeNixlConnectorWorker(NixlConnectorWorker):
block_size=self.block_size,
ssm_sizes=(0, 0),
attn_backend_name=self.backend_name,
physical_blocks_per_logical_kv_block=1,
),
remote_tp_rank=remote_tp_rank,
remote_tp_size=remote_tp_size,
@@ -726,6 +727,7 @@ class TestNixlHandshake:
worker.num_blocks = 1
worker.dst_num_blocks[worker.engine_id] = worker.num_blocks
worker.src_blocks_data = [(0, worker.block_len_per_layer[0], worker.tp_rank)]
worker.num_descs = len(worker.src_blocks_data)
def check_handshake(remote_tp_size: int):
tp_ratio = remote_tp_size // local_tp_size
@@ -978,6 +980,7 @@ class TestNixlHandshake:
block_size=worker.block_size,
ssm_sizes=(0, 0),
attn_backend_name=worker.backend_name,
physical_blocks_per_logical_kv_block=1,
)
with pytest.raises(RuntimeError):
@@ -1035,6 +1038,7 @@ class TestNixlHandshake:
block_size=worker.block_size,
ssm_sizes=(0, 0),
attn_backend_name=worker.backend_name,
physical_blocks_per_logical_kv_block=1,
)
# We don't check layout for homogeneous TP and MLA for now, as the
@@ -2354,6 +2358,7 @@ def test_compatibility_hash_validation(
block_size=prefill_block_size,
ssm_sizes=(0, 0),
attn_backend_name=decode_worker.backend_name,
physical_blocks_per_logical_kv_block=1,
)
handshake_payload = NixlHandshakePayload(
compatibility_hash=remote_hash,
@@ -93,75 +93,82 @@ def test_logical_to_kernel_block_ids_with_hma():
@pytest.mark.cpu_test
@pytest.mark.parametrize(
"has_mamba,swa_enabled,mamba_enabled,remote_ratio,"
"remote_block_ids,expected_remote_block_ids",
"group_spec_types,expansion_stride,remote_block_ids,expected_remote_block_ids",
[
# Non-mamba (FA+SWA): both groups expanded via _logical_to_kernel_block_ids.
# Regression for https://github.com/vllm-project/vllm/pull/39724
(
False,
True,
False,
1,
pytest.param(
("FullAttentionSpec", "SlidingWindowSpec"),
2,
([0, 1, 2], [3, 4]),
[[0, 1, 2, 3, 4, 5], [6, 7, 8, 9]],
id="dense_fa_swa",
),
# Mamba (FA+Mamba): FA expanded via _logical_to_remote_kernel_block_ids,
# Mamba passed through unchanged.
# remote_ratio=261 (Nemotron 30B TP=1) != local_ratio=2 so that using
# the wrong conversion method produces different FA results.
(
True,
False,
True,
pytest.param(
("FullAttentionSpec", "MambaSpec"),
261,
([0, 1, 2], [10, 11]),
[[0, 1, 261, 262, 522, 523], [10, 11]],
id="mamba_fa_ssm",
),
],
ids=["non_mamba_fa_swa", "mamba_fa_ssm"],
)
def test_read_blocks_for_req_expands_remote_ids(
has_mamba,
swa_enabled,
mamba_enabled,
remote_ratio,
group_spec_types,
expansion_stride,
remote_block_ids,
expected_remote_block_ids,
):
"""_read_blocks_for_req must expand remote logical block IDs to kernel
block IDs when kernel block size != logical block size.
Non-mamba path uses _logical_to_kernel_block_ids (all groups expanded).
Mamba path uses _logical_to_remote_kernel_block_ids (FA expanded, Mamba
passed through).
The hot path always calls _logical_to_remote_kernel_block_ids with
remote_info.remote_physical_blocks_per_logical (model-agnostic).
"""
from unittest.mock import MagicMock
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
NixlConnectorMetadata,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.tp_mapping import (
TPMapping,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker import (
NixlConnectorWorker,
)
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
MambaSpec,
SlidingWindowSpec,
)
spec_name_to_type = {
"FullAttentionSpec": FullAttentionSpec,
"SlidingWindowSpec": SlidingWindowSpec,
"MambaSpec": MambaSpec,
}
resolved_types = tuple(spec_name_to_type[n] for n in group_spec_types)
worker = object.__new__(NixlConnectorWorker)
worker._has_mamba = has_mamba
worker._physical_blocks_per_logical_kv_block = 2
has_mamba = any(t is MambaSpec for t in resolved_types)
has_swa = any(t is SlidingWindowSpec for t in resolved_types)
worker.kv_cache_config = make_kv_cache_config(
block_size=16, swa_enabled=swa_enabled, mamba_enabled=mamba_enabled
block_size=16, swa_enabled=has_swa, mamba_enabled=has_mamba
)
remote_engine_id = "remote-engine"
if has_mamba:
worker._physical_blocks_per_logical = {remote_engine_id: remote_ratio}
# Mock transfer_topo: empty remote ranks skips the transfer machinery
# entirely, isolating the block-ID expansion logic.
worker.transfer_topo = MagicMock()
worker.transfer_topo.target_remote_ranks.return_value = []
worker.transfer_topo.get_engine_info.return_value = MagicMock(remote_tp_size=1)
worker.transfer_topo.tp_ratio.return_value = 1
remote_info = MagicMock()
remote_info.remote_physical_blocks_per_logical = expansion_stride
worker.transfer_topo.get_engine_info.return_value = remote_info
worker.use_mla = False
mock_plan = MagicMock(spec=TPMapping)
mock_plan.all_source_ranks = ()
mock_plan.source_ranks_per_group = ()
worker.tp_mappings = {remote_engine_id: mock_plan}
metadata = NixlConnectorMetadata()
metadata.add_new_req_to_recv(
@@ -306,75 +313,82 @@ def test_nixl_metadata_hma_block_ids_structure():
assert list(req_meta.remote.block_ids[1]) == [18, 19, 20, 21]
@pytest.mark.cpu_test
def test_get_block_descs_ids_hybrid_ssm():
"""Test _get_block_descs_ids uses per-group strides for hybrid FA+SSM
when ratio=1 (no kernel block size mismatch)."""
def _make_mock_worker_for_desc_ids(
num_regions: int,
has_mamba: bool,
group_spec_types: tuple,
block_len_per_layer: list[int] | None = None,
):
"""Build a mock NixlConnectorWorker with attrs needed by _compute_desc_ids."""
from unittest.mock import MagicMock
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker import (
NixlConnectorWorker,
)
worker = object.__new__(NixlConnectorWorker)
worker = MagicMock(spec=NixlConnectorWorker)
worker.num_regions = num_regions
worker._has_mamba = has_mamba
worker._group_spec_types = group_spec_types
worker.block_len_per_layer = block_len_per_layer or [100]
worker._compute_desc_ids = NixlConnectorWorker._compute_desc_ids.__get__(
worker, NixlConnectorWorker
)
return worker
num_blocks = 100
engine_id = "test-engine"
worker.num_regions = 2
worker.dst_num_blocks = {engine_id: num_blocks}
worker._has_mamba = True
worker._is_mamba_group = [False, True]
worker._physical_blocks_per_logical_kv_block = 1
worker._physical_blocks_per_logical = {engine_id: 1}
worker.block_len_per_layer = [100]
# num_descs = num_regions * num_blocks (no blocks_first doubling)
worker.num_descs = 2 * num_blocks
@pytest.mark.cpu_test
def test_get_block_descs_ids_hybrid_ssm():
"""Test _compute_desc_ids uses per-group strides for hybrid
FA+SSM when ratio=1 (no kernel block size mismatch)."""
from vllm.v1.kv_cache_interface import FullAttentionSpec, MambaSpec
worker = _make_mock_worker_for_desc_ids(
num_regions=2,
has_mamba=True,
group_spec_types=(FullAttentionSpec, MambaSpec),
block_len_per_layer=[100],
)
fa_blocks = [3, 5]
ssm_blocks = [1, 2]
result = worker._get_block_descs_ids(engine_id, (fa_blocks, ssm_blocks))
result = worker._compute_desc_ids(
block_ids=(fa_blocks, ssm_blocks),
dst_num_blocks=100,
block_size_ratio=None,
physical_blocks_per_logical=1,
)
# FA group: stride=num_blocks=100, offset=0
# region0: [3, 5], region1: [103, 105]
# SSM group: stride=logical_blocks=100 (=num_blocks/ratio=100/1),
# offset=num_fa_descs=200, 4 regions per Mamba layer (x, B, C, ssm)
# region0: [201, 202], region1: [301, 302],
# region2: [401, 402], region3: [501, 502]
expected = [3, 5, 103, 105, 201, 202, 301, 302, 401, 402, 501, 502]
assert list(result) == expected, f"Expected {expected}, got {list(result)}"
@pytest.mark.cpu_test
def test_get_block_descs_ids_kernel_block_mismatch():
"""Test _get_block_descs_ids uses different strides for FA (kernel blocks)
vs SSM (logical blocks) when ratio > 1."""
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker import (
NixlConnectorWorker,
)
worker = object.__new__(NixlConnectorWorker)
"""Test _compute_desc_ids uses different strides for FA
(kernel blocks) vs SSM (logical blocks) when ratio > 1."""
from vllm.v1.kv_cache_interface import FullAttentionSpec, MambaSpec
ratio = 4
logical_blocks = 100
num_blocks = logical_blocks * ratio # 400 kernel blocks
engine_id = "test-engine"
worker.num_regions = 2
worker.dst_num_blocks = {engine_id: num_blocks}
worker._has_mamba = True
worker._is_mamba_group = [False, True]
worker._physical_blocks_per_logical_kv_block = ratio
worker._physical_blocks_per_logical = {engine_id: ratio}
worker.block_len_per_layer = [100]
worker.num_descs = 2 * num_blocks # 800
fa_blocks = [3, 7] # kernel-level block IDs
ssm_blocks = [1, 2] # logical block IDs
result = worker._get_block_descs_ids(engine_id, (fa_blocks, ssm_blocks))
worker = _make_mock_worker_for_desc_ids(
num_regions=2,
has_mamba=True,
group_spec_types=(FullAttentionSpec, MambaSpec),
block_len_per_layer=[100],
)
fa_blocks = [3, 7]
ssm_blocks = [1, 2]
result = worker._compute_desc_ids(
block_ids=(fa_blocks, ssm_blocks),
dst_num_blocks=num_blocks,
block_size_ratio=None,
physical_blocks_per_logical=ratio,
)
# FA group: stride=num_blocks=400, offset=0
# region0: [3, 7], region1: [403, 407]
# SSM group: stride=logical_blocks=400//4=100, offset=num_fa_descs=800,
# 4 regions per Mamba layer (x, B, C, ssm)
# region0: [801, 802], region1: [901, 902],
# region2: [1001, 1002], region3: [1101, 1102]
expected = [3, 7, 403, 407, 801, 802, 901, 902, 1001, 1002, 1101, 1102]
assert list(result) == expected, f"Expected {expected}, got {list(result)}"
@@ -0,0 +1,141 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for TP mapping and transfer plan utilities.
These tests verify that TP mapping produces correct outputs
(source ranks, split handles, desc IDs).
No GPU or NIXL required.
"""
from __future__ import annotations
import pytest
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.tp_mapping import (
TPMapping,
compute_tp_mapping,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker import (
NixlConnectorWorker,
)
from vllm.v1.kv_cache_interface import FullAttentionSpec, MambaSpec
# ======================================================================
# Test fixtures / helpers
# ======================================================================
def _compute_mapping(
tp_rank: int = 0,
tp_size: int = 1,
remote_tp_size: int = 1,
is_mla: bool = False,
num_kv_heads: int = 8,
group_spec_types: tuple[type, ...] = (FullAttentionSpec,),
) -> TPMapping:
return compute_tp_mapping(
tp_rank=tp_rank,
tp_size=tp_size,
remote_tp_size=remote_tp_size,
is_mla=is_mla,
total_num_kv_heads=num_kv_heads,
group_spec_types=group_spec_types,
)
# ======================================================================
# TP mapping structure tests
# ======================================================================
class TestTPMappingStructure:
def test_source_ranks_homogeneous(self):
m = _compute_mapping(tp_size=2, tp_rank=1, remote_tp_size=2)
assert m.all_source_ranks == (1,)
def test_source_ranks_d_gt_p(self):
m = _compute_mapping(tp_size=4, tp_rank=2, remote_tp_size=2)
assert m.all_source_ranks == (1,)
def test_source_ranks_p_gt_d(self):
m = _compute_mapping(tp_size=1, tp_rank=0, remote_tp_size=2)
assert m.all_source_ranks == (0, 1)
# ======================================================================
# Split handle tests
# ======================================================================
def _make_mock_worker_for_splits(group_spec_types):
"""Build a mock NixlConnectorWorker with _group_spec_types for split tests."""
worker = object.__new__(NixlConnectorWorker)
worker._group_spec_types = group_spec_types
return worker
class TestBuildSrcSplitHandles:
@pytest.mark.parametrize("remote_tp_size", [2, 4])
def test_build_src_split_handles(self, remote_tp_size):
tp_rank = 0
tp_size = 1
plan = _compute_mapping(
tp_rank=tp_rank,
tp_size=tp_size,
remote_tp_size=remote_tp_size,
)
worker = _make_mock_worker_for_splits((FullAttentionSpec,))
src_blocks_data = [(0x2000 + i * 1024, 1024, 0) for i in range(8)]
num_descs = len(src_blocks_data)
splits = list(
worker._build_local_splits_from_plan(
plan,
src_blocks_data,
num_descs,
)
)
assert len(splits) == remote_tp_size
for handle in splits:
assert len(handle) == len(src_blocks_data)
for _, length, _ in handle:
assert length == 1024 // remote_tp_size
class TestMambaPlanSplitHandles:
"""Verify split handles for Mamba with FA/SSM distinction."""
def test_fa_and_ssm_different_split_factors(self):
"""Section 0 split by num_attn_reads, section 1 by abs_tp."""
fa_readers = (0,)
ssm_readers = (0, 1)
plan = TPMapping(
source_ranks_per_group=(fa_readers, ssm_readers),
all_source_ranks=(0, 1),
rank_to_attention_slot={0: 0, 1: 0},
rank_offset_factor=0,
)
worker = _make_mock_worker_for_splits((FullAttentionSpec, MambaSpec))
# 2 FA descs + 1 SSM desc
src_blocks_data = [
(1000, 200, 0), # FA desc 0
(2000, 200, 0), # FA desc 1
(3000, 400, 0), # SSM desc 0
]
splits = list(worker._build_local_splits_from_plan(plan, src_blocks_data, 2))
assert len(splits) == 2 # 2 source ranks
# Rank 0 (FA source, p_idx=0):
# FA: chunk=200//1=200, slot=0 → (1000, 200, 0), (2000, 200, 0)
# SSM: chunk=400//2=200, idx=0 → (3000, 200, 0)
assert splits[0] == [(1000, 200, 0), (2000, 200, 0), (3000, 200, 0)]
# Rank 1 (not FA source, p_idx=1):
# FA: chunk=200//1=200, slot=0 (skip_fa) → (1000, 200, 0), (2000, 200, 0)
# SSM: chunk=400//2=200, idx=1 → (3200, 200, 0)
assert splits[1] == [(1000, 200, 0), (2000, 200, 0), (3200, 200, 0)]