nixl refactor: new transfer design (#40731)
Signed-off-by: ZhanqiuHu <zhu@redhat.com> Signed-off-by: NickLucche <nlucches@redhat.com> Co-authored-by: NickLucche <nlucches@redhat.com>
This commit is contained in:
@@ -527,6 +527,7 @@ class FakeNixlConnectorWorker(NixlConnectorWorker):
|
|||||||
block_size=self.block_size,
|
block_size=self.block_size,
|
||||||
ssm_sizes=(0, 0),
|
ssm_sizes=(0, 0),
|
||||||
attn_backend_name=self.backend_name,
|
attn_backend_name=self.backend_name,
|
||||||
|
physical_blocks_per_logical_kv_block=1,
|
||||||
),
|
),
|
||||||
remote_tp_rank=remote_tp_rank,
|
remote_tp_rank=remote_tp_rank,
|
||||||
remote_tp_size=remote_tp_size,
|
remote_tp_size=remote_tp_size,
|
||||||
@@ -726,6 +727,7 @@ class TestNixlHandshake:
|
|||||||
worker.num_blocks = 1
|
worker.num_blocks = 1
|
||||||
worker.dst_num_blocks[worker.engine_id] = worker.num_blocks
|
worker.dst_num_blocks[worker.engine_id] = worker.num_blocks
|
||||||
worker.src_blocks_data = [(0, worker.block_len_per_layer[0], worker.tp_rank)]
|
worker.src_blocks_data = [(0, worker.block_len_per_layer[0], worker.tp_rank)]
|
||||||
|
worker.num_descs = len(worker.src_blocks_data)
|
||||||
|
|
||||||
def check_handshake(remote_tp_size: int):
|
def check_handshake(remote_tp_size: int):
|
||||||
tp_ratio = remote_tp_size // local_tp_size
|
tp_ratio = remote_tp_size // local_tp_size
|
||||||
@@ -978,6 +980,7 @@ class TestNixlHandshake:
|
|||||||
block_size=worker.block_size,
|
block_size=worker.block_size,
|
||||||
ssm_sizes=(0, 0),
|
ssm_sizes=(0, 0),
|
||||||
attn_backend_name=worker.backend_name,
|
attn_backend_name=worker.backend_name,
|
||||||
|
physical_blocks_per_logical_kv_block=1,
|
||||||
)
|
)
|
||||||
|
|
||||||
with pytest.raises(RuntimeError):
|
with pytest.raises(RuntimeError):
|
||||||
@@ -1035,6 +1038,7 @@ class TestNixlHandshake:
|
|||||||
block_size=worker.block_size,
|
block_size=worker.block_size,
|
||||||
ssm_sizes=(0, 0),
|
ssm_sizes=(0, 0),
|
||||||
attn_backend_name=worker.backend_name,
|
attn_backend_name=worker.backend_name,
|
||||||
|
physical_blocks_per_logical_kv_block=1,
|
||||||
)
|
)
|
||||||
|
|
||||||
# We don't check layout for homogeneous TP and MLA for now, as the
|
# We don't check layout for homogeneous TP and MLA for now, as the
|
||||||
@@ -2354,6 +2358,7 @@ def test_compatibility_hash_validation(
|
|||||||
block_size=prefill_block_size,
|
block_size=prefill_block_size,
|
||||||
ssm_sizes=(0, 0),
|
ssm_sizes=(0, 0),
|
||||||
attn_backend_name=decode_worker.backend_name,
|
attn_backend_name=decode_worker.backend_name,
|
||||||
|
physical_blocks_per_logical_kv_block=1,
|
||||||
)
|
)
|
||||||
handshake_payload = NixlHandshakePayload(
|
handshake_payload = NixlHandshakePayload(
|
||||||
compatibility_hash=remote_hash,
|
compatibility_hash=remote_hash,
|
||||||
|
|||||||
@@ -93,75 +93,82 @@ def test_logical_to_kernel_block_ids_with_hma():
|
|||||||
|
|
||||||
@pytest.mark.cpu_test
|
@pytest.mark.cpu_test
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
"has_mamba,swa_enabled,mamba_enabled,remote_ratio,"
|
"group_spec_types,expansion_stride,remote_block_ids,expected_remote_block_ids",
|
||||||
"remote_block_ids,expected_remote_block_ids",
|
|
||||||
[
|
[
|
||||||
# Non-mamba (FA+SWA): both groups expanded via _logical_to_kernel_block_ids.
|
pytest.param(
|
||||||
# Regression for https://github.com/vllm-project/vllm/pull/39724
|
("FullAttentionSpec", "SlidingWindowSpec"),
|
||||||
(
|
2,
|
||||||
False,
|
|
||||||
True,
|
|
||||||
False,
|
|
||||||
1,
|
|
||||||
([0, 1, 2], [3, 4]),
|
([0, 1, 2], [3, 4]),
|
||||||
[[0, 1, 2, 3, 4, 5], [6, 7, 8, 9]],
|
[[0, 1, 2, 3, 4, 5], [6, 7, 8, 9]],
|
||||||
|
id="dense_fa_swa",
|
||||||
),
|
),
|
||||||
# Mamba (FA+Mamba): FA expanded via _logical_to_remote_kernel_block_ids,
|
pytest.param(
|
||||||
# Mamba passed through unchanged.
|
("FullAttentionSpec", "MambaSpec"),
|
||||||
# remote_ratio=261 (Nemotron 30B TP=1) != local_ratio=2 so that using
|
|
||||||
# the wrong conversion method produces different FA results.
|
|
||||||
(
|
|
||||||
True,
|
|
||||||
False,
|
|
||||||
True,
|
|
||||||
261,
|
261,
|
||||||
([0, 1, 2], [10, 11]),
|
([0, 1, 2], [10, 11]),
|
||||||
[[0, 1, 261, 262, 522, 523], [10, 11]],
|
[[0, 1, 261, 262, 522, 523], [10, 11]],
|
||||||
|
id="mamba_fa_ssm",
|
||||||
),
|
),
|
||||||
],
|
],
|
||||||
ids=["non_mamba_fa_swa", "mamba_fa_ssm"],
|
|
||||||
)
|
)
|
||||||
def test_read_blocks_for_req_expands_remote_ids(
|
def test_read_blocks_for_req_expands_remote_ids(
|
||||||
has_mamba,
|
group_spec_types,
|
||||||
swa_enabled,
|
expansion_stride,
|
||||||
mamba_enabled,
|
|
||||||
remote_ratio,
|
|
||||||
remote_block_ids,
|
remote_block_ids,
|
||||||
expected_remote_block_ids,
|
expected_remote_block_ids,
|
||||||
):
|
):
|
||||||
"""_read_blocks_for_req must expand remote logical block IDs to kernel
|
"""_read_blocks_for_req must expand remote logical block IDs to kernel
|
||||||
block IDs when kernel block size != logical block size.
|
block IDs when kernel block size != logical block size.
|
||||||
|
|
||||||
Non-mamba path uses _logical_to_kernel_block_ids (all groups expanded).
|
The hot path always calls _logical_to_remote_kernel_block_ids with
|
||||||
Mamba path uses _logical_to_remote_kernel_block_ids (FA expanded, Mamba
|
remote_info.remote_physical_blocks_per_logical (model-agnostic).
|
||||||
passed through).
|
|
||||||
"""
|
"""
|
||||||
from unittest.mock import MagicMock
|
from unittest.mock import MagicMock
|
||||||
|
|
||||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
|
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
|
||||||
NixlConnectorMetadata,
|
NixlConnectorMetadata,
|
||||||
)
|
)
|
||||||
|
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.tp_mapping import (
|
||||||
|
TPMapping,
|
||||||
|
)
|
||||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker import (
|
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker import (
|
||||||
NixlConnectorWorker,
|
NixlConnectorWorker,
|
||||||
)
|
)
|
||||||
|
from vllm.v1.kv_cache_interface import (
|
||||||
|
FullAttentionSpec,
|
||||||
|
MambaSpec,
|
||||||
|
SlidingWindowSpec,
|
||||||
|
)
|
||||||
|
|
||||||
|
spec_name_to_type = {
|
||||||
|
"FullAttentionSpec": FullAttentionSpec,
|
||||||
|
"SlidingWindowSpec": SlidingWindowSpec,
|
||||||
|
"MambaSpec": MambaSpec,
|
||||||
|
}
|
||||||
|
resolved_types = tuple(spec_name_to_type[n] for n in group_spec_types)
|
||||||
|
|
||||||
worker = object.__new__(NixlConnectorWorker)
|
worker = object.__new__(NixlConnectorWorker)
|
||||||
worker._has_mamba = has_mamba
|
|
||||||
worker._physical_blocks_per_logical_kv_block = 2
|
worker._physical_blocks_per_logical_kv_block = 2
|
||||||
|
|
||||||
|
has_mamba = any(t is MambaSpec for t in resolved_types)
|
||||||
|
has_swa = any(t is SlidingWindowSpec for t in resolved_types)
|
||||||
worker.kv_cache_config = make_kv_cache_config(
|
worker.kv_cache_config = make_kv_cache_config(
|
||||||
block_size=16, swa_enabled=swa_enabled, mamba_enabled=mamba_enabled
|
block_size=16, swa_enabled=has_swa, mamba_enabled=has_mamba
|
||||||
)
|
)
|
||||||
|
|
||||||
remote_engine_id = "remote-engine"
|
remote_engine_id = "remote-engine"
|
||||||
if has_mamba:
|
|
||||||
worker._physical_blocks_per_logical = {remote_engine_id: remote_ratio}
|
|
||||||
|
|
||||||
# Mock transfer_topo: empty remote ranks skips the transfer machinery
|
|
||||||
# entirely, isolating the block-ID expansion logic.
|
|
||||||
worker.transfer_topo = MagicMock()
|
worker.transfer_topo = MagicMock()
|
||||||
worker.transfer_topo.target_remote_ranks.return_value = []
|
|
||||||
worker.transfer_topo.get_engine_info.return_value = MagicMock(remote_tp_size=1)
|
|
||||||
worker.transfer_topo.tp_ratio.return_value = 1
|
worker.transfer_topo.tp_ratio.return_value = 1
|
||||||
|
remote_info = MagicMock()
|
||||||
|
remote_info.remote_physical_blocks_per_logical = expansion_stride
|
||||||
|
worker.transfer_topo.get_engine_info.return_value = remote_info
|
||||||
|
worker.use_mla = False
|
||||||
|
|
||||||
|
mock_plan = MagicMock(spec=TPMapping)
|
||||||
|
mock_plan.all_source_ranks = ()
|
||||||
|
mock_plan.source_ranks_per_group = ()
|
||||||
|
worker.tp_mappings = {remote_engine_id: mock_plan}
|
||||||
|
|
||||||
metadata = NixlConnectorMetadata()
|
metadata = NixlConnectorMetadata()
|
||||||
metadata.add_new_req_to_recv(
|
metadata.add_new_req_to_recv(
|
||||||
@@ -306,75 +313,82 @@ def test_nixl_metadata_hma_block_ids_structure():
|
|||||||
assert list(req_meta.remote.block_ids[1]) == [18, 19, 20, 21]
|
assert list(req_meta.remote.block_ids[1]) == [18, 19, 20, 21]
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.cpu_test
|
def _make_mock_worker_for_desc_ids(
|
||||||
def test_get_block_descs_ids_hybrid_ssm():
|
num_regions: int,
|
||||||
"""Test _get_block_descs_ids uses per-group strides for hybrid FA+SSM
|
has_mamba: bool,
|
||||||
when ratio=1 (no kernel block size mismatch)."""
|
group_spec_types: tuple,
|
||||||
|
block_len_per_layer: list[int] | None = None,
|
||||||
|
):
|
||||||
|
"""Build a mock NixlConnectorWorker with attrs needed by _compute_desc_ids."""
|
||||||
|
from unittest.mock import MagicMock
|
||||||
|
|
||||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker import (
|
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker import (
|
||||||
NixlConnectorWorker,
|
NixlConnectorWorker,
|
||||||
)
|
)
|
||||||
|
|
||||||
worker = object.__new__(NixlConnectorWorker)
|
worker = MagicMock(spec=NixlConnectorWorker)
|
||||||
|
worker.num_regions = num_regions
|
||||||
|
worker._has_mamba = has_mamba
|
||||||
|
worker._group_spec_types = group_spec_types
|
||||||
|
worker.block_len_per_layer = block_len_per_layer or [100]
|
||||||
|
worker._compute_desc_ids = NixlConnectorWorker._compute_desc_ids.__get__(
|
||||||
|
worker, NixlConnectorWorker
|
||||||
|
)
|
||||||
|
return worker
|
||||||
|
|
||||||
num_blocks = 100
|
|
||||||
engine_id = "test-engine"
|
@pytest.mark.cpu_test
|
||||||
worker.num_regions = 2
|
def test_get_block_descs_ids_hybrid_ssm():
|
||||||
worker.dst_num_blocks = {engine_id: num_blocks}
|
"""Test _compute_desc_ids uses per-group strides for hybrid
|
||||||
worker._has_mamba = True
|
FA+SSM when ratio=1 (no kernel block size mismatch)."""
|
||||||
worker._is_mamba_group = [False, True]
|
from vllm.v1.kv_cache_interface import FullAttentionSpec, MambaSpec
|
||||||
worker._physical_blocks_per_logical_kv_block = 1
|
|
||||||
worker._physical_blocks_per_logical = {engine_id: 1}
|
worker = _make_mock_worker_for_desc_ids(
|
||||||
worker.block_len_per_layer = [100]
|
num_regions=2,
|
||||||
# num_descs = num_regions * num_blocks (no blocks_first doubling)
|
has_mamba=True,
|
||||||
worker.num_descs = 2 * num_blocks
|
group_spec_types=(FullAttentionSpec, MambaSpec),
|
||||||
|
block_len_per_layer=[100],
|
||||||
|
)
|
||||||
|
|
||||||
fa_blocks = [3, 5]
|
fa_blocks = [3, 5]
|
||||||
ssm_blocks = [1, 2]
|
ssm_blocks = [1, 2]
|
||||||
result = worker._get_block_descs_ids(engine_id, (fa_blocks, ssm_blocks))
|
result = worker._compute_desc_ids(
|
||||||
|
block_ids=(fa_blocks, ssm_blocks),
|
||||||
|
dst_num_blocks=100,
|
||||||
|
block_size_ratio=None,
|
||||||
|
physical_blocks_per_logical=1,
|
||||||
|
)
|
||||||
|
|
||||||
# FA group: stride=num_blocks=100, offset=0
|
|
||||||
# region0: [3, 5], region1: [103, 105]
|
|
||||||
# SSM group: stride=logical_blocks=100 (=num_blocks/ratio=100/1),
|
|
||||||
# offset=num_fa_descs=200, 4 regions per Mamba layer (x, B, C, ssm)
|
|
||||||
# region0: [201, 202], region1: [301, 302],
|
|
||||||
# region2: [401, 402], region3: [501, 502]
|
|
||||||
expected = [3, 5, 103, 105, 201, 202, 301, 302, 401, 402, 501, 502]
|
expected = [3, 5, 103, 105, 201, 202, 301, 302, 401, 402, 501, 502]
|
||||||
assert list(result) == expected, f"Expected {expected}, got {list(result)}"
|
assert list(result) == expected, f"Expected {expected}, got {list(result)}"
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.cpu_test
|
@pytest.mark.cpu_test
|
||||||
def test_get_block_descs_ids_kernel_block_mismatch():
|
def test_get_block_descs_ids_kernel_block_mismatch():
|
||||||
"""Test _get_block_descs_ids uses different strides for FA (kernel blocks)
|
"""Test _compute_desc_ids uses different strides for FA
|
||||||
vs SSM (logical blocks) when ratio > 1."""
|
(kernel blocks) vs SSM (logical blocks) when ratio > 1."""
|
||||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker import (
|
from vllm.v1.kv_cache_interface import FullAttentionSpec, MambaSpec
|
||||||
NixlConnectorWorker,
|
|
||||||
)
|
|
||||||
|
|
||||||
worker = object.__new__(NixlConnectorWorker)
|
|
||||||
|
|
||||||
ratio = 4
|
ratio = 4
|
||||||
logical_blocks = 100
|
logical_blocks = 100
|
||||||
num_blocks = logical_blocks * ratio # 400 kernel blocks
|
num_blocks = logical_blocks * ratio # 400 kernel blocks
|
||||||
engine_id = "test-engine"
|
|
||||||
worker.num_regions = 2
|
|
||||||
worker.dst_num_blocks = {engine_id: num_blocks}
|
|
||||||
worker._has_mamba = True
|
|
||||||
worker._is_mamba_group = [False, True]
|
|
||||||
worker._physical_blocks_per_logical_kv_block = ratio
|
|
||||||
worker._physical_blocks_per_logical = {engine_id: ratio}
|
|
||||||
worker.block_len_per_layer = [100]
|
|
||||||
worker.num_descs = 2 * num_blocks # 800
|
|
||||||
|
|
||||||
fa_blocks = [3, 7] # kernel-level block IDs
|
worker = _make_mock_worker_for_desc_ids(
|
||||||
ssm_blocks = [1, 2] # logical block IDs
|
num_regions=2,
|
||||||
result = worker._get_block_descs_ids(engine_id, (fa_blocks, ssm_blocks))
|
has_mamba=True,
|
||||||
|
group_spec_types=(FullAttentionSpec, MambaSpec),
|
||||||
|
block_len_per_layer=[100],
|
||||||
|
)
|
||||||
|
|
||||||
|
fa_blocks = [3, 7]
|
||||||
|
ssm_blocks = [1, 2]
|
||||||
|
result = worker._compute_desc_ids(
|
||||||
|
block_ids=(fa_blocks, ssm_blocks),
|
||||||
|
dst_num_blocks=num_blocks,
|
||||||
|
block_size_ratio=None,
|
||||||
|
physical_blocks_per_logical=ratio,
|
||||||
|
)
|
||||||
|
|
||||||
# FA group: stride=num_blocks=400, offset=0
|
|
||||||
# region0: [3, 7], region1: [403, 407]
|
|
||||||
# SSM group: stride=logical_blocks=400//4=100, offset=num_fa_descs=800,
|
|
||||||
# 4 regions per Mamba layer (x, B, C, ssm)
|
|
||||||
# region0: [801, 802], region1: [901, 902],
|
|
||||||
# region2: [1001, 1002], region3: [1101, 1102]
|
|
||||||
expected = [3, 7, 403, 407, 801, 802, 901, 902, 1001, 1002, 1101, 1102]
|
expected = [3, 7, 403, 407, 801, 802, 901, 902, 1001, 1002, 1101, 1102]
|
||||||
assert list(result) == expected, f"Expected {expected}, got {list(result)}"
|
assert list(result) == expected, f"Expected {expected}, got {list(result)}"
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,141 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
"""Tests for TP mapping and transfer plan utilities.
|
||||||
|
|
||||||
|
These tests verify that TP mapping produces correct outputs
|
||||||
|
(source ranks, split handles, desc IDs).
|
||||||
|
No GPU or NIXL required.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.tp_mapping import (
|
||||||
|
TPMapping,
|
||||||
|
compute_tp_mapping,
|
||||||
|
)
|
||||||
|
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker import (
|
||||||
|
NixlConnectorWorker,
|
||||||
|
)
|
||||||
|
from vllm.v1.kv_cache_interface import FullAttentionSpec, MambaSpec
|
||||||
|
|
||||||
|
# ======================================================================
|
||||||
|
# Test fixtures / helpers
|
||||||
|
# ======================================================================
|
||||||
|
|
||||||
|
|
||||||
|
def _compute_mapping(
|
||||||
|
tp_rank: int = 0,
|
||||||
|
tp_size: int = 1,
|
||||||
|
remote_tp_size: int = 1,
|
||||||
|
is_mla: bool = False,
|
||||||
|
num_kv_heads: int = 8,
|
||||||
|
group_spec_types: tuple[type, ...] = (FullAttentionSpec,),
|
||||||
|
) -> TPMapping:
|
||||||
|
return compute_tp_mapping(
|
||||||
|
tp_rank=tp_rank,
|
||||||
|
tp_size=tp_size,
|
||||||
|
remote_tp_size=remote_tp_size,
|
||||||
|
is_mla=is_mla,
|
||||||
|
total_num_kv_heads=num_kv_heads,
|
||||||
|
group_spec_types=group_spec_types,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ======================================================================
|
||||||
|
# TP mapping structure tests
|
||||||
|
# ======================================================================
|
||||||
|
|
||||||
|
|
||||||
|
class TestTPMappingStructure:
|
||||||
|
def test_source_ranks_homogeneous(self):
|
||||||
|
m = _compute_mapping(tp_size=2, tp_rank=1, remote_tp_size=2)
|
||||||
|
assert m.all_source_ranks == (1,)
|
||||||
|
|
||||||
|
def test_source_ranks_d_gt_p(self):
|
||||||
|
m = _compute_mapping(tp_size=4, tp_rank=2, remote_tp_size=2)
|
||||||
|
assert m.all_source_ranks == (1,)
|
||||||
|
|
||||||
|
def test_source_ranks_p_gt_d(self):
|
||||||
|
m = _compute_mapping(tp_size=1, tp_rank=0, remote_tp_size=2)
|
||||||
|
assert m.all_source_ranks == (0, 1)
|
||||||
|
|
||||||
|
|
||||||
|
# ======================================================================
|
||||||
|
# Split handle tests
|
||||||
|
# ======================================================================
|
||||||
|
|
||||||
|
|
||||||
|
def _make_mock_worker_for_splits(group_spec_types):
|
||||||
|
"""Build a mock NixlConnectorWorker with _group_spec_types for split tests."""
|
||||||
|
worker = object.__new__(NixlConnectorWorker)
|
||||||
|
worker._group_spec_types = group_spec_types
|
||||||
|
return worker
|
||||||
|
|
||||||
|
|
||||||
|
class TestBuildSrcSplitHandles:
|
||||||
|
@pytest.mark.parametrize("remote_tp_size", [2, 4])
|
||||||
|
def test_build_src_split_handles(self, remote_tp_size):
|
||||||
|
tp_rank = 0
|
||||||
|
tp_size = 1
|
||||||
|
|
||||||
|
plan = _compute_mapping(
|
||||||
|
tp_rank=tp_rank,
|
||||||
|
tp_size=tp_size,
|
||||||
|
remote_tp_size=remote_tp_size,
|
||||||
|
)
|
||||||
|
|
||||||
|
worker = _make_mock_worker_for_splits((FullAttentionSpec,))
|
||||||
|
src_blocks_data = [(0x2000 + i * 1024, 1024, 0) for i in range(8)]
|
||||||
|
num_descs = len(src_blocks_data)
|
||||||
|
splits = list(
|
||||||
|
worker._build_local_splits_from_plan(
|
||||||
|
plan,
|
||||||
|
src_blocks_data,
|
||||||
|
num_descs,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
assert len(splits) == remote_tp_size
|
||||||
|
for handle in splits:
|
||||||
|
assert len(handle) == len(src_blocks_data)
|
||||||
|
for _, length, _ in handle:
|
||||||
|
assert length == 1024 // remote_tp_size
|
||||||
|
|
||||||
|
|
||||||
|
class TestMambaPlanSplitHandles:
|
||||||
|
"""Verify split handles for Mamba with FA/SSM distinction."""
|
||||||
|
|
||||||
|
def test_fa_and_ssm_different_split_factors(self):
|
||||||
|
"""Section 0 split by num_attn_reads, section 1 by abs_tp."""
|
||||||
|
fa_readers = (0,)
|
||||||
|
ssm_readers = (0, 1)
|
||||||
|
plan = TPMapping(
|
||||||
|
source_ranks_per_group=(fa_readers, ssm_readers),
|
||||||
|
all_source_ranks=(0, 1),
|
||||||
|
rank_to_attention_slot={0: 0, 1: 0},
|
||||||
|
rank_offset_factor=0,
|
||||||
|
)
|
||||||
|
|
||||||
|
worker = _make_mock_worker_for_splits((FullAttentionSpec, MambaSpec))
|
||||||
|
# 2 FA descs + 1 SSM desc
|
||||||
|
src_blocks_data = [
|
||||||
|
(1000, 200, 0), # FA desc 0
|
||||||
|
(2000, 200, 0), # FA desc 1
|
||||||
|
(3000, 400, 0), # SSM desc 0
|
||||||
|
]
|
||||||
|
|
||||||
|
splits = list(worker._build_local_splits_from_plan(plan, src_blocks_data, 2))
|
||||||
|
|
||||||
|
assert len(splits) == 2 # 2 source ranks
|
||||||
|
|
||||||
|
# Rank 0 (FA source, p_idx=0):
|
||||||
|
# FA: chunk=200//1=200, slot=0 → (1000, 200, 0), (2000, 200, 0)
|
||||||
|
# SSM: chunk=400//2=200, idx=0 → (3000, 200, 0)
|
||||||
|
assert splits[0] == [(1000, 200, 0), (2000, 200, 0), (3000, 200, 0)]
|
||||||
|
|
||||||
|
# Rank 1 (not FA source, p_idx=1):
|
||||||
|
# FA: chunk=200//1=200, slot=0 (skip_fa) → (1000, 200, 0), (2000, 200, 0)
|
||||||
|
# SSM: chunk=400//2=200, idx=1 → (3200, 200, 0)
|
||||||
|
assert splits[1] == [(1000, 200, 0), (2000, 200, 0), (3200, 200, 0)]
|
||||||
@@ -386,38 +386,6 @@ class EngineTransferInfo:
|
|||||||
"""Physical blocks per logical block."""
|
"""Physical blocks per logical block."""
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class MambaEngineTransferInfo(EngineTransferInfo):
|
|
||||||
"""Extends ``EngineTransferInfo`` with Mamba-hybrid transfer geometry.
|
|
||||||
|
|
||||||
For hybrid SSM+Attention models, FA and Mamba layers may require
|
|
||||||
different numbers of reads from different remote ranks. This
|
|
||||||
dataclass captures that per-engine transfer plan.
|
|
||||||
"""
|
|
||||||
|
|
||||||
remote_fa_source_ranks: tuple[int, ...]
|
|
||||||
"""Remote ranks carrying unique FA heads for this local rank."""
|
|
||||||
|
|
||||||
remote_all_source_ranks: tuple[int, ...]
|
|
||||||
"""All remote ranks this local rank reads from (FA + Mamba)."""
|
|
||||||
|
|
||||||
remote_num_fa_reads: int
|
|
||||||
"""Number of distinct remote ranks needed for FA data."""
|
|
||||||
|
|
||||||
remote_num_mamba_reads: int
|
|
||||||
"""Number of distinct remote ranks needed for Mamba data."""
|
|
||||||
|
|
||||||
remote_fa_descriptor_bytes: int
|
|
||||||
"""Byte size of one FA K (or V) descriptor entry."""
|
|
||||||
|
|
||||||
is_remote_replicated: bool
|
|
||||||
"""Whether the remote engine has replicated KV heads
|
|
||||||
(remote_tp_size > total_num_kv_heads)."""
|
|
||||||
|
|
||||||
remote_physical_heads: int
|
|
||||||
"""Physical KV heads stored per remote rank."""
|
|
||||||
|
|
||||||
|
|
||||||
# ---- Transfer topology ----
|
# ---- Transfer topology ----
|
||||||
|
|
||||||
|
|
||||||
@@ -439,8 +407,6 @@ class TransferTopology:
|
|||||||
self.local_physical_heads = max(1, self.total_num_kv_heads // self.tp_size)
|
self.local_physical_heads = max(1, self.total_num_kv_heads // self.tp_size)
|
||||||
|
|
||||||
self._engines: dict[EngineId, EngineTransferInfo] = {}
|
self._engines: dict[EngineId, EngineTransferInfo] = {}
|
||||||
self._fa_source_sets: dict[EngineId, frozenset[int]] = {}
|
|
||||||
self._fa_source_indices: dict[EngineId, dict[int, int]] = {}
|
|
||||||
|
|
||||||
# Figure out whether the first dimension of the cache is K/V
|
# Figure out whether the first dimension of the cache is K/V
|
||||||
# or num_blocks.
|
# or num_blocks.
|
||||||
@@ -487,24 +453,12 @@ class TransferTopology:
|
|||||||
def register_remote_engine(
|
def register_remote_engine(
|
||||||
self,
|
self,
|
||||||
remote_engine_id: EngineId,
|
remote_engine_id: EngineId,
|
||||||
remote_tp_size: int,
|
info: EngineTransferInfo,
|
||||||
remote_block_size: int,
|
|
||||||
remote_block_len: int,
|
|
||||||
remote_physical_blocks_per_logical: int,
|
|
||||||
*,
|
|
||||||
local_block_len: int = 0,
|
|
||||||
) -> EngineTransferInfo:
|
) -> EngineTransferInfo:
|
||||||
"""Register a remote engine, unifying worker dicts state.
|
"""Register a remote engine, unifying worker dicts state.
|
||||||
|
|
||||||
Only remote engines should be registered here — the local engine's
|
The caller (worker) is responsible for computing the info via
|
||||||
identity (tp_size, block_size, etc.) is set via ``__init__`` params.
|
the transfer policy. This method only stores and deduplicates.
|
||||||
|
|
||||||
For Mamba models, also computes the Mamba transfer plan and
|
|
||||||
builds the FA source lookup caches.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
local_block_len: Local representative block_len (bytes).
|
|
||||||
Required for Mamba models to compute ``fa_descriptor_bytes``.
|
|
||||||
"""
|
"""
|
||||||
assert remote_engine_id != self.engine_id, (
|
assert remote_engine_id != self.engine_id, (
|
||||||
f"Cannot register local engine {self.engine_id} as remote. "
|
f"Cannot register local engine {self.engine_id} as remote. "
|
||||||
@@ -512,29 +466,6 @@ class TransferTopology:
|
|||||||
)
|
)
|
||||||
if remote_engine_id in self._engines:
|
if remote_engine_id in self._engines:
|
||||||
return self._engines[remote_engine_id]
|
return self._engines[remote_engine_id]
|
||||||
info: EngineTransferInfo
|
|
||||||
if self.is_mamba:
|
|
||||||
info = self._build_mamba_info(
|
|
||||||
remote_tp_size=remote_tp_size,
|
|
||||||
remote_block_size=remote_block_size,
|
|
||||||
remote_block_len=remote_block_len,
|
|
||||||
remote_physical_blocks_per_logical=(remote_physical_blocks_per_logical),
|
|
||||||
local_block_len=local_block_len,
|
|
||||||
)
|
|
||||||
assert isinstance(info, MambaEngineTransferInfo)
|
|
||||||
self._fa_source_sets[remote_engine_id] = frozenset(
|
|
||||||
info.remote_fa_source_ranks
|
|
||||||
)
|
|
||||||
self._fa_source_indices[remote_engine_id] = {
|
|
||||||
r: i for i, r in enumerate(info.remote_fa_source_ranks)
|
|
||||||
}
|
|
||||||
else:
|
|
||||||
info = EngineTransferInfo(
|
|
||||||
remote_tp_size=remote_tp_size,
|
|
||||||
remote_block_len=remote_block_len,
|
|
||||||
remote_block_size=remote_block_size,
|
|
||||||
remote_physical_blocks_per_logical=(remote_physical_blocks_per_logical),
|
|
||||||
)
|
|
||||||
self._engines[remote_engine_id] = info
|
self._engines[remote_engine_id] = info
|
||||||
return info
|
return info
|
||||||
|
|
||||||
@@ -622,14 +553,8 @@ class TransferTopology:
|
|||||||
"""Get the remote TP rank(s) that the current local TP rank will
|
"""Get the remote TP rank(s) that the current local TP rank will
|
||||||
read from. When remote tp_size > local tp_size, reads from
|
read from. When remote tp_size > local tp_size, reads from
|
||||||
multiple remote ranks.
|
multiple remote ranks.
|
||||||
|
|
||||||
For Mamba models, returns the precomputed ``all_source_ranks``
|
|
||||||
(FA + Mamba union).
|
|
||||||
"""
|
"""
|
||||||
info = self._engines[remote_engine_id]
|
info = self._engines[remote_engine_id]
|
||||||
if isinstance(info, MambaEngineTransferInfo):
|
|
||||||
return list(info.remote_all_source_ranks)
|
|
||||||
|
|
||||||
tp_ratio = self.tp_ratio(info.remote_tp_size)
|
tp_ratio = self.tp_ratio(info.remote_tp_size)
|
||||||
if tp_ratio > 0:
|
if tp_ratio > 0:
|
||||||
return [self.tp_rank // tp_ratio]
|
return [self.tp_rank // tp_ratio]
|
||||||
@@ -662,309 +587,15 @@ class TransferTopology:
|
|||||||
# Regular case: backends like FA register K/V in separate regions
|
# Regular case: backends like FA register K/V in separate regions
|
||||||
return cache if self.split_k_and_v else [cache]
|
return cache if self.split_k_and_v else [cache]
|
||||||
|
|
||||||
# ============================================================
|
|
||||||
# Mamba-specific methods
|
|
||||||
# ============================================================
|
|
||||||
|
|
||||||
def should_skip_fa(self, remote_engine_id: EngineId, remote_rank: int) -> bool:
|
|
||||||
"""Whether to skip FA groups for this remote rank (mamba-only)."""
|
|
||||||
return remote_rank not in self._fa_source_sets[remote_engine_id]
|
|
||||||
|
|
||||||
def fa_head_slot(self, remote_engine_id: EngineId, remote_rank: int) -> int:
|
|
||||||
"""Index into local FA block for this remote rank's head data.
|
|
||||||
|
|
||||||
For remote ranks in ``fa_source_ranks``, returns 0, 1, …, reads-1.
|
|
||||||
For ranks NOT in ``fa_source_ranks`` (replicated duplicates),
|
|
||||||
returns the slot of the matching source rank with the same head.
|
|
||||||
"""
|
|
||||||
fa_index = self._fa_source_indices[remote_engine_id]
|
|
||||||
if remote_rank in fa_index:
|
|
||||||
return fa_index[remote_rank]
|
|
||||||
mamba_info = self._engines[remote_engine_id]
|
|
||||||
assert isinstance(mamba_info, MambaEngineTransferInfo)
|
|
||||||
K = self.total_num_kv_heads
|
|
||||||
remote_tp = mamba_info.remote_tp_size
|
|
||||||
r_head = self._physical_head_range(remote_tp, K, remote_rank)
|
|
||||||
for target in mamba_info.remote_fa_source_ranks:
|
|
||||||
t_head = self._physical_head_range(remote_tp, K, target)
|
|
||||||
if self._range_overlap(r_head, t_head):
|
|
||||||
return fa_index[target]
|
|
||||||
return 0
|
|
||||||
|
|
||||||
def fa_rank_offset(
|
|
||||||
self, remote_engine_id: EngineId, remote_kv_block_len: int
|
|
||||||
) -> int:
|
|
||||||
"""Byte offset into remote FA block for this local rank.
|
|
||||||
|
|
||||||
When local TP is replicated (local_tp > K), multiple local ranks
|
|
||||||
share a head. Computes offset *relative to the target remote
|
|
||||||
rank's first head* so it works regardless of how many heads the
|
|
||||||
remote has. Returns 0 when local does not index into remote.
|
|
||||||
"""
|
|
||||||
mamba_info = self._engines[remote_engine_id]
|
|
||||||
assert isinstance(mamba_info, MambaEngineTransferInfo)
|
|
||||||
tp_ratio = self.tp_ratio(mamba_info.remote_tp_size)
|
|
||||||
if self.is_mla or tp_ratio <= 0:
|
|
||||||
return 0
|
|
||||||
K = self.total_num_kv_heads
|
|
||||||
is_local_replicated = self.tp_size > K
|
|
||||||
if is_local_replicated:
|
|
||||||
local_head = self.tp_rank * K // self.tp_size
|
|
||||||
p_rank = mamba_info.remote_fa_source_ranks[0]
|
|
||||||
p_start = p_rank * K // mamba_info.remote_tp_size
|
|
||||||
return (local_head - p_start) * remote_kv_block_len
|
|
||||||
return self.tp_rank % tp_ratio * remote_kv_block_len
|
|
||||||
|
|
||||||
def needs_split_handles(self, remote_engine_id: EngineId) -> bool:
|
|
||||||
"""Whether per-remote-rank split handles are needed.
|
|
||||||
|
|
||||||
True when FA and mamba have different read counts, requiring
|
|
||||||
different splitting factors in the local handle.
|
|
||||||
"""
|
|
||||||
mamba_info = self._engines[remote_engine_id]
|
|
||||||
assert isinstance(mamba_info, MambaEngineTransferInfo)
|
|
||||||
tp_ratio = self.tp_ratio(mamba_info.remote_tp_size)
|
|
||||||
return (
|
|
||||||
tp_ratio < 0
|
|
||||||
and not self.is_mla
|
|
||||||
and len(mamba_info.remote_all_source_ranks) > 1
|
|
||||||
)
|
|
||||||
|
|
||||||
def compute_split_handle_data(
|
|
||||||
self,
|
|
||||||
remote_engine_id: EngineId,
|
|
||||||
src_blocks_data: list[tuple[int, int, int]],
|
|
||||||
num_fa_descs: int,
|
|
||||||
abs_tp: int,
|
|
||||||
) -> list[list[tuple[int, int, int]]]:
|
|
||||||
"""Per-remote-rank (addr, len, dev) triples for Mamba-HMA split
|
|
||||||
handles.
|
|
||||||
|
|
||||||
FA descriptors (indices < num_fa_descs) are sliced by
|
|
||||||
``remote_num_fa_reads``; mamba descriptors are sliced uniformly
|
|
||||||
by ``abs_tp``.
|
|
||||||
"""
|
|
||||||
mamba_info = self._engines[remote_engine_id]
|
|
||||||
assert isinstance(mamba_info, MambaEngineTransferInfo)
|
|
||||||
all_handle_data: list[list[tuple[int, int, int]]] = []
|
|
||||||
for p_idx, p_rank in enumerate(mamba_info.remote_all_source_ranks):
|
|
||||||
handle_data: list[tuple[int, int, int]] = []
|
|
||||||
skip_fa = self.should_skip_fa(remote_engine_id, p_rank)
|
|
||||||
fa_slot = self.fa_head_slot(remote_engine_id, p_rank) if not skip_fa else 0
|
|
||||||
for j, (addr, local_len, dev) in enumerate(src_blocks_data):
|
|
||||||
if j < num_fa_descs:
|
|
||||||
assert mamba_info.remote_num_fa_reads >= 1
|
|
||||||
fa_chunk = local_len // mamba_info.remote_num_fa_reads
|
|
||||||
handle_data.append((addr + fa_slot * fa_chunk, fa_chunk, dev))
|
|
||||||
else:
|
|
||||||
mamba_chunk = local_len // abs_tp
|
|
||||||
handle_data.append((addr + p_idx * mamba_chunk, mamba_chunk, dev))
|
|
||||||
all_handle_data.append(handle_data)
|
|
||||||
return all_handle_data
|
|
||||||
|
|
||||||
def filter_block_ids_for_rank(
|
|
||||||
self,
|
|
||||||
remote_engine_id: EngineId,
|
|
||||||
remote_rank: int,
|
|
||||||
local_ids: BlockIds,
|
|
||||||
remote_ids: BlockIds,
|
|
||||||
is_mamba_group: list[bool],
|
|
||||||
) -> tuple[BlockIds, BlockIds]:
|
|
||||||
"""Zero out FA groups for remote ranks outside ``fa_source_ranks``.
|
|
||||||
|
|
||||||
Returns (filtered_local_ids, filtered_remote_ids). When the
|
|
||||||
remote rank carries FA data for this local rank, returns the
|
|
||||||
inputs unchanged.
|
|
||||||
"""
|
|
||||||
if not self.should_skip_fa(remote_engine_id, remote_rank):
|
|
||||||
return local_ids, remote_ids
|
|
||||||
num_groups = len(local_ids)
|
|
||||||
filtered_local: list[list[int]] = [
|
|
||||||
[] if not is_mamba_group[g] else local_ids[g] for g in range(num_groups)
|
|
||||||
]
|
|
||||||
filtered_remote: list[list[int]] = [
|
|
||||||
[] if not is_mamba_group[g] else remote_ids[g] for g in range(num_groups)
|
|
||||||
]
|
|
||||||
return filtered_local, filtered_remote
|
|
||||||
|
|
||||||
def describe(self, remote_engine_id: EngineId) -> str:
|
def describe(self, remote_engine_id: EngineId) -> str:
|
||||||
"""One-line summary of transfer config for logging."""
|
"""One-line summary of transfer config for logging."""
|
||||||
info = self._engines[remote_engine_id]
|
info = self._engines[remote_engine_id]
|
||||||
base = (
|
return (
|
||||||
|
f"TransferTopology("
|
||||||
f"tp_ratio={self.tp_ratio(info.remote_tp_size)}, "
|
f"tp_ratio={self.tp_ratio(info.remote_tp_size)}, "
|
||||||
f"K={self.total_num_kv_heads}, "
|
f"K={self.total_num_kv_heads}, "
|
||||||
f"local_tp={self.tp_size}, "
|
f"local_tp={self.tp_size}, "
|
||||||
f"remote_tp={info.remote_tp_size}, "
|
f"remote_tp={info.remote_tp_size}, "
|
||||||
f"local_rank={self.tp_rank}, "
|
f"local_rank={self.tp_rank}, "
|
||||||
f"remote_block_len={info.remote_block_len}"
|
f"remote_block_len={info.remote_block_len})"
|
||||||
)
|
|
||||||
if isinstance(info, MambaEngineTransferInfo):
|
|
||||||
return (
|
|
||||||
f"TransferTopology.mamba({base}, "
|
|
||||||
f"fa_reads={info.remote_num_fa_reads}, "
|
|
||||||
f"mamba_reads={info.remote_num_mamba_reads}, "
|
|
||||||
f"fa_sources={list(info.remote_fa_source_ranks)}, "
|
|
||||||
f"all_sources={list(info.remote_all_source_ranks)}, "
|
|
||||||
f"fa_desc_bytes={info.remote_fa_descriptor_bytes})"
|
|
||||||
)
|
|
||||||
return f"TransferTopology({base})"
|
|
||||||
|
|
||||||
# ============================================================
|
|
||||||
# Private helpers
|
|
||||||
# ============================================================
|
|
||||||
# Mamba-HMA hetero-TP transfer config:
|
|
||||||
# With hetero-TP (P_TP > D_TP), FA KV cache may be replicated across
|
|
||||||
# P ranks (when P_TP > num_kv_heads), but Mamba conv/SSM state is
|
|
||||||
# almost always uniquely sharded per P rank. So the number of P
|
|
||||||
# ranks D must read from can differ between FA and Mamba, and they
|
|
||||||
# must be handled separately.
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _physical_head_range(tp_size: int, num_heads: int, rank: int) -> range:
|
|
||||||
"""Physical KV head range stored in a rank's KV cache tensor.
|
|
||||||
|
|
||||||
When ``tp_size <= num_heads``: sharded, K/TP contiguous heads per rank.
|
|
||||||
When ``tp_size > num_heads``: 1 physical head per rank. Heads are
|
|
||||||
distributed **contiguously** (matching vLLM's GQA weight partitioning):
|
|
||||||
consecutive ranks share a head before moving to the next one.
|
|
||||||
"""
|
|
||||||
if tp_size <= num_heads:
|
|
||||||
assert num_heads % tp_size == 0
|
|
||||||
per_rank = num_heads // tp_size
|
|
||||||
return range(rank * per_rank, (rank + 1) * per_rank)
|
|
||||||
else:
|
|
||||||
h = rank * num_heads // tp_size
|
|
||||||
return range(h, h + 1)
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _range_overlap(a: range, b: range) -> range:
|
|
||||||
start = max(a.start, b.start)
|
|
||||||
stop = min(a.stop, b.stop)
|
|
||||||
return range(start, max(start, stop))
|
|
||||||
|
|
||||||
# ============================================================
|
|
||||||
# Private: build Mamba transfer info
|
|
||||||
# ============================================================
|
|
||||||
|
|
||||||
def _build_mamba_info(
|
|
||||||
self,
|
|
||||||
remote_tp_size: int,
|
|
||||||
remote_block_size: int,
|
|
||||||
remote_block_len: int,
|
|
||||||
remote_physical_blocks_per_logical: int,
|
|
||||||
local_block_len: int,
|
|
||||||
) -> MambaEngineTransferInfo:
|
|
||||||
"""Compute Mamba transfer plan."""
|
|
||||||
K = self.total_num_kv_heads
|
|
||||||
local_tp = self.tp_size
|
|
||||||
local_rank = self.tp_rank
|
|
||||||
|
|
||||||
is_remote_replicated = remote_tp_size > K
|
|
||||||
remote_physical_heads = max(1, K // remote_tp_size)
|
|
||||||
|
|
||||||
if local_tp >= remote_tp_size:
|
|
||||||
assert local_tp % remote_tp_size == 0
|
|
||||||
tp_ratio = local_tp // remote_tp_size
|
|
||||||
else:
|
|
||||||
assert remote_tp_size % local_tp == 0
|
|
||||||
tp_ratio = -(remote_tp_size // local_tp)
|
|
||||||
|
|
||||||
abs_tp = -tp_ratio if tp_ratio < 0 else 1
|
|
||||||
|
|
||||||
mamba_range: range | None = None
|
|
||||||
if tp_ratio < 0:
|
|
||||||
mamba_range = range(local_rank * abs_tp, (local_rank + 1) * abs_tp)
|
|
||||||
|
|
||||||
# ---- FA read targets ----
|
|
||||||
if self.is_mla or tp_ratio >= 0:
|
|
||||||
num_fa_reads = 1
|
|
||||||
fa_source_ranks: list[int] = (
|
|
||||||
[0]
|
|
||||||
if self.is_mla
|
|
||||||
else [local_rank // tp_ratio if tp_ratio > 0 else local_rank]
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
local_needs = self._physical_head_range(local_tp, K, local_rank)
|
|
||||||
search_range = (
|
|
||||||
mamba_range if mamba_range is not None else range(remote_tp_size)
|
|
||||||
)
|
|
||||||
seen: set[tuple[int, int]] = set()
|
|
||||||
fa_source_ranks = []
|
|
||||||
for p in search_range:
|
|
||||||
p_has = self._physical_head_range(remote_tp_size, K, p)
|
|
||||||
ov = self._range_overlap(local_needs, p_has)
|
|
||||||
if len(ov) > 0:
|
|
||||||
key = (ov.start, ov.stop)
|
|
||||||
if key not in seen:
|
|
||||||
seen.add(key)
|
|
||||||
fa_source_ranks.append(p)
|
|
||||||
if not fa_source_ranks:
|
|
||||||
for p in range(remote_tp_size):
|
|
||||||
p_has = self._physical_head_range(remote_tp_size, K, p)
|
|
||||||
ov = self._range_overlap(local_needs, p_has)
|
|
||||||
if len(ov) > 0:
|
|
||||||
key = (ov.start, ov.stop)
|
|
||||||
if key not in seen:
|
|
||||||
seen.add(key)
|
|
||||||
fa_source_ranks.append(p)
|
|
||||||
num_fa_reads = len(fa_source_ranks)
|
|
||||||
|
|
||||||
# ---- All source ranks (mamba + FA) ----
|
|
||||||
if mamba_range is not None and abs_tp > num_fa_reads:
|
|
||||||
num_mamba_reads = abs_tp
|
|
||||||
all_source_ranks = list(mamba_range)
|
|
||||||
else:
|
|
||||||
num_mamba_reads = num_fa_reads
|
|
||||||
all_source_ranks = list(fa_source_ranks)
|
|
||||||
|
|
||||||
# ---- FA descriptor bytes ----
|
|
||||||
effective_block_len = min(local_block_len, remote_block_len)
|
|
||||||
if self.is_kv_layout_blocks_first:
|
|
||||||
fa_descriptor_bytes = effective_block_len // 2
|
|
||||||
else:
|
|
||||||
fa_descriptor_bytes = effective_block_len
|
|
||||||
|
|
||||||
# ---- Validation ----
|
|
||||||
is_local_replicated = local_tp > K
|
|
||||||
if is_local_replicated and is_remote_replicated and tp_ratio > 0:
|
|
||||||
logger.info(
|
|
||||||
"Both-replicated hetero-TP: local_tp=%d > remote_tp=%d > K=%d.",
|
|
||||||
local_tp,
|
|
||||||
remote_tp_size,
|
|
||||||
K,
|
|
||||||
)
|
|
||||||
tt_set = set(all_source_ranks)
|
|
||||||
for t in fa_source_ranks:
|
|
||||||
if t not in tt_set:
|
|
||||||
logger.error(
|
|
||||||
"FA source rank %d NOT in all_source_ranks %s.",
|
|
||||||
t,
|
|
||||||
all_source_ranks,
|
|
||||||
)
|
|
||||||
if self.is_kv_layout_blocks_first and tp_ratio < 0 and num_fa_reads > 0:
|
|
||||||
local_k_half = local_block_len // 2
|
|
||||||
remote_k_half = remote_block_len // 2
|
|
||||||
expected = local_k_half // num_fa_reads
|
|
||||||
if expected != remote_k_half:
|
|
||||||
logger.warning(
|
|
||||||
"FA size mismatch: local_k_half=%d / reads=%d = %d, "
|
|
||||||
"but remote_k_half=%d.",
|
|
||||||
local_k_half,
|
|
||||||
num_fa_reads,
|
|
||||||
expected,
|
|
||||||
remote_k_half,
|
|
||||||
)
|
|
||||||
|
|
||||||
return MambaEngineTransferInfo(
|
|
||||||
remote_tp_size=remote_tp_size,
|
|
||||||
remote_block_len=remote_block_len,
|
|
||||||
remote_block_size=remote_block_size,
|
|
||||||
remote_physical_blocks_per_logical=(remote_physical_blocks_per_logical),
|
|
||||||
remote_fa_source_ranks=tuple(fa_source_ranks),
|
|
||||||
remote_all_source_ranks=tuple(all_source_ranks),
|
|
||||||
remote_num_fa_reads=num_fa_reads,
|
|
||||||
remote_num_mamba_reads=num_mamba_reads,
|
|
||||||
remote_fa_descriptor_bytes=fa_descriptor_bytes,
|
|
||||||
is_remote_replicated=is_remote_replicated,
|
|
||||||
remote_physical_heads=remote_physical_heads,
|
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -32,8 +32,9 @@ GET_META_MSG = b"get_meta_msg"
|
|||||||
# Version History:
|
# Version History:
|
||||||
# 1: Initial version with compatibility checking
|
# 1: Initial version with compatibility checking
|
||||||
# 2: Add remote_request_id to kv_transfer_params
|
# 2: Add remote_request_id to kv_transfer_params
|
||||||
|
# 3: Add physical_blocks_per_logical_kv_block to NixlAgentMetadata
|
||||||
#
|
#
|
||||||
NIXL_CONNECTOR_VERSION: int = 2
|
NIXL_CONNECTOR_VERSION: int = 3
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
@@ -48,6 +49,7 @@ class NixlAgentMetadata:
|
|||||||
block_size: int
|
block_size: int
|
||||||
ssm_sizes: tuple[int, int]
|
ssm_sizes: tuple[int, int]
|
||||||
attn_backend_name: str
|
attn_backend_name: str
|
||||||
|
physical_blocks_per_logical_kv_block: int
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
|
|||||||
@@ -0,0 +1,142 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
"""TP mapping computation for NIXL KV cache transfers."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
|
||||||
|
from vllm.distributed.kv_transfer.kv_connector.utils import (
|
||||||
|
BlockIds,
|
||||||
|
)
|
||||||
|
from vllm.v1.kv_cache_interface import AttentionSpec, KVCacheSpec, MambaSpec
|
||||||
|
|
||||||
|
# ======================================================================
|
||||||
|
# Data structures
|
||||||
|
# ======================================================================
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class ReadSpec:
|
||||||
|
"""Specification for a single remote block read operation."""
|
||||||
|
|
||||||
|
remote_rank: int
|
||||||
|
local_block_ids: BlockIds
|
||||||
|
remote_block_ids: BlockIds
|
||||||
|
|
||||||
|
|
||||||
|
def _is_attention_spec(spec_type: type[KVCacheSpec]) -> bool:
|
||||||
|
return issubclass(spec_type, AttentionSpec)
|
||||||
|
|
||||||
|
|
||||||
|
def _is_ssm_spec(spec_type: type[KVCacheSpec]) -> bool:
|
||||||
|
return issubclass(spec_type, MambaSpec)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class TPMapping:
|
||||||
|
"""Complete local-to-remote TP mapping for one remote engine.
|
||||||
|
|
||||||
|
Generated once per remote engine during handshake.
|
||||||
|
"""
|
||||||
|
|
||||||
|
# Remote TP ranks that this local rank reads from, per group.
|
||||||
|
# Position = local piece index.
|
||||||
|
source_ranks_per_group: tuple[tuple[int, ...], ...]
|
||||||
|
|
||||||
|
# Superset of all source ranks (union of all groups).
|
||||||
|
all_source_ranks: tuple[int, ...]
|
||||||
|
|
||||||
|
# Maps each source rank to its FA head slot index.
|
||||||
|
rank_to_attention_slot: dict[int, int]
|
||||||
|
|
||||||
|
# FA head offset factor for hetero-TP (D_TP > P_TP).
|
||||||
|
rank_offset_factor: int
|
||||||
|
|
||||||
|
|
||||||
|
# ======================================================================
|
||||||
|
# TP mapping computation
|
||||||
|
# ======================================================================
|
||||||
|
|
||||||
|
|
||||||
|
def compute_tp_mapping(
|
||||||
|
tp_rank: int,
|
||||||
|
tp_size: int,
|
||||||
|
remote_tp_size: int,
|
||||||
|
is_mla: bool,
|
||||||
|
total_num_kv_heads: int,
|
||||||
|
group_spec_types: tuple[type[KVCacheSpec], ...],
|
||||||
|
) -> TPMapping:
|
||||||
|
"""Build the complete local-to-remote TP mapping.
|
||||||
|
|
||||||
|
Computes source ranks, head slot assignments, and the rank offset
|
||||||
|
factor in a single pass.
|
||||||
|
"""
|
||||||
|
# --- Attention source ranks ---
|
||||||
|
if is_mla:
|
||||||
|
# All heads replicated across all ranks.
|
||||||
|
attn_ranks = [0]
|
||||||
|
elif tp_size >= remote_tp_size:
|
||||||
|
# D (local TP) > P (remote TP): multiple local ranks read different chunks from
|
||||||
|
# *one* remote rank, corresponding to different kv heads.
|
||||||
|
attn_ranks = [tp_rank * remote_tp_size // tp_size]
|
||||||
|
else:
|
||||||
|
# P (remote TP) > D (local TP): one local rank
|
||||||
|
# reads from multiple remote ranks.
|
||||||
|
# GQA dedup: when K < remote_tp_size, several remote ranks
|
||||||
|
# hold the same KV head. np.unique keeps only the first
|
||||||
|
# rank per unique head so we don't issue redundant reads.
|
||||||
|
abs_tp = remote_tp_size // tp_size
|
||||||
|
start = tp_rank * abs_tp
|
||||||
|
heads = np.arange(start, start + abs_tp) * total_num_kv_heads // remote_tp_size
|
||||||
|
_, unique_idx = np.unique(heads, return_index=True)
|
||||||
|
attn_ranks = (start + np.sort(unique_idx)).tolist()
|
||||||
|
|
||||||
|
# --- SSM source ranks ---
|
||||||
|
has_ssm = any(_is_ssm_spec(t) for t in group_spec_types)
|
||||||
|
if has_ssm:
|
||||||
|
if tp_size < remote_tp_size:
|
||||||
|
abs_tp = remote_tp_size // tp_size
|
||||||
|
ssm_ranks = list(range(tp_rank * abs_tp, (tp_rank + 1) * abs_tp))
|
||||||
|
else:
|
||||||
|
ssm_ranks = list(attn_ranks)
|
||||||
|
else:
|
||||||
|
ssm_ranks = []
|
||||||
|
|
||||||
|
all_ranks = sorted(set(attn_ranks) | set(ssm_ranks))
|
||||||
|
|
||||||
|
# --- Per-group ordered source ranks ---
|
||||||
|
source_ranks_per_group = tuple(
|
||||||
|
tuple(ssm_ranks) if _is_ssm_spec(t) else tuple(attn_ranks)
|
||||||
|
for t in group_spec_types
|
||||||
|
)
|
||||||
|
|
||||||
|
# --- Attention head slots ---
|
||||||
|
head_to_slot: dict[int, int] = {}
|
||||||
|
for i, r in enumerate(attn_ranks):
|
||||||
|
head_to_slot[r * total_num_kv_heads // remote_tp_size] = i
|
||||||
|
rank_to_attention_slot = {
|
||||||
|
r: head_to_slot.get(r * total_num_kv_heads // remote_tp_size, 0)
|
||||||
|
for r in all_ranks
|
||||||
|
}
|
||||||
|
|
||||||
|
# --- Rank offset factor ---
|
||||||
|
if is_mla or tp_size <= remote_tp_size:
|
||||||
|
# We don't index into remote for reading, no offset needed.
|
||||||
|
rank_offset_factor = 0
|
||||||
|
elif tp_size > total_num_kv_heads:
|
||||||
|
local_head = tp_rank * total_num_kv_heads // tp_size
|
||||||
|
p_start = attn_ranks[0] * total_num_kv_heads // remote_tp_size
|
||||||
|
rank_offset_factor = local_head - p_start
|
||||||
|
else:
|
||||||
|
# D TP > P TP: we index into remote to read different heads depending on rank.
|
||||||
|
rank_offset_factor = tp_rank % (tp_size // remote_tp_size)
|
||||||
|
|
||||||
|
return TPMapping(
|
||||||
|
source_ranks_per_group=source_ranks_per_group,
|
||||||
|
all_source_ranks=tuple(all_ranks),
|
||||||
|
rank_to_attention_slot=rank_to_attention_slot,
|
||||||
|
rank_offset_factor=rank_offset_factor,
|
||||||
|
)
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -31,6 +31,7 @@ class MambaConvSplitInfo:
|
|||||||
x_local: int # intermediate_size / TP (columns for x)
|
x_local: int # intermediate_size / TP (columns for x)
|
||||||
b_local: int # groups_ss / TP (columns for B; C is same size)
|
b_local: int # groups_ss / TP (columns for B; C is same size)
|
||||||
conv_dtype_size: int # bytes per element (e.g. 2 for float16)
|
conv_dtype_size: int # bytes per element (e.g. 2 for float16)
|
||||||
|
ssm_sizes: tuple[int, int] # (conv_state_bytes, ssm_state_bytes)
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def conv_dim_local(self) -> int:
|
def conv_dim_local(self) -> int:
|
||||||
@@ -99,8 +100,8 @@ def derive_mamba_conv_split(
|
|||||||
local_tp: this engine's tensor-parallel size.
|
local_tp: this engine's tensor-parallel size.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
MambaConvSplitInfo with per-rank x_local, b_local, conv_rows, and
|
MambaConvSplitInfo with per-rank x_local, b_local, conv_rows,
|
||||||
conv_dtype_size.
|
conv_dtype_size, and ssm_sizes (conv_state_bytes, ssm_state_bytes).
|
||||||
"""
|
"""
|
||||||
if mamba_spec.mamba_type != "mamba2":
|
if mamba_spec.mamba_type != "mamba2":
|
||||||
raise NotImplementedError(
|
raise NotImplementedError(
|
||||||
@@ -142,12 +143,20 @@ def derive_mamba_conv_split(
|
|||||||
dtype=mamba_spec.dtypes[0], # type: ignore[misc]
|
dtype=mamba_spec.dtypes[0], # type: ignore[misc]
|
||||||
).element_size()
|
).element_size()
|
||||||
|
|
||||||
|
ssm_dtype_size = torch.tensor(
|
||||||
|
[],
|
||||||
|
dtype=mamba_spec.dtypes[1], # type: ignore[misc]
|
||||||
|
).element_size()
|
||||||
|
conv_state_bytes = torch.Size(mamba_spec.shapes[0]).numel() * conv_dtype_size
|
||||||
|
ssm_state_bytes = torch.Size(mamba_spec.shapes[1]).numel() * ssm_dtype_size
|
||||||
|
|
||||||
# Divide by TP to get per-rank column counts.
|
# Divide by TP to get per-rank column counts.
|
||||||
return MambaConvSplitInfo(
|
return MambaConvSplitInfo(
|
||||||
conv_rows=conv_rows,
|
conv_rows=conv_rows,
|
||||||
x_local=intermediate_size // local_tp,
|
x_local=intermediate_size // local_tp,
|
||||||
b_local=groups_ss // local_tp,
|
b_local=groups_ss // local_tp,
|
||||||
conv_dtype_size=conv_dtype_size,
|
conv_dtype_size=conv_dtype_size,
|
||||||
|
ssm_sizes=(conv_state_bytes, ssm_state_bytes),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user