[KVConnector][MoRIIO] Transfer hybrid mamba/KDA recurrent state in READ mode (#51052)
Signed-off-by: Yichao Zhu <Yichao.Zhu@amd.com>
This commit is contained in:
@@ -367,6 +367,8 @@ async def handle_request(api: str, request: Request):
|
||||
"remote_block_ids"
|
||||
]
|
||||
req_data["kv_transfer_params"]["transfer_id"] = prefill_kv["transfer_id"]
|
||||
if prefill_kv.get("tp_size") is not None:
|
||||
req_data["kv_transfer_params"]["tp_size"] = prefill_kv["tp_size"]
|
||||
|
||||
req_data["kv_transfer_params"]["remote_dp_size"] = prefill_instance_endpoint[
|
||||
"dp_size"
|
||||
|
||||
@@ -775,6 +775,17 @@ def test_is_hma_required(swa_enabled, disable_hma, expected_is_hma):
|
||||
assert scheduler.get_exchange_clipped_blocks(blocks) == blocks
|
||||
|
||||
|
||||
def test_transfer_disabled_hybrid_group_is_ignored():
|
||||
config = _make_hybrid_kv_cache_config()
|
||||
config.kv_cache_groups[1].enable_kv_transfer = False
|
||||
scheduler = _read_scheduler(config)
|
||||
|
||||
assert scheduler._is_hma_required is False
|
||||
assert scheduler.get_exchange_clipped_blocks([[1, 2, 3], [20, 21, 22]]) == [
|
||||
[1, 2, 3]
|
||||
]
|
||||
|
||||
|
||||
def test_non_sliding_window_hybrid_is_rejected():
|
||||
"""A hybrid group that is not sliding-window (e.g. chunked-local
|
||||
attention) must fail closed rather than be silently mistransferred."""
|
||||
|
||||
@@ -0,0 +1,809 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Pure-CPU unit tests for the MoRIIO hybrid (attention + mamba/KDA) scheduler
|
||||
and the KDA offset-template cache.
|
||||
|
||||
These exercise the highest-risk, otherwise-untested scheduler logic without a
|
||||
GPU or the ``mori`` runtime: per-group block-id splitting, the READ/WRITE
|
||||
``N-1`` token accounting, P-side prompt truncation, and offset-template cache
|
||||
wiring.
|
||||
|
||||
Like ``test_moriio_kv_layout.py`` the whole module is skipped unless it is
|
||||
running on ROCm with ``mori`` installed (importing the connector pulls in
|
||||
``mori``). The authoritative run happens on the MIA recipe image.
|
||||
"""
|
||||
|
||||
import importlib
|
||||
import importlib.util
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
mori_available = importlib.util.find_spec("mori") is not None
|
||||
|
||||
if not (current_platform.is_rocm() and mori_available):
|
||||
pytest.skip(
|
||||
"MoRIIOs are only available on ROCm with mori package installed",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
moriio_connector = importlib.import_module(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector"
|
||||
)
|
||||
moriio_layout = importlib.import_module(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_layout"
|
||||
)
|
||||
ssm_conv_transfer_utils = importlib.import_module(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.ssm_conv_transfer_utils"
|
||||
)
|
||||
MambaConvSplitInfo = ssm_conv_transfer_utils.MambaConvSplitInfo
|
||||
MoRIIOMode = moriio_connector.MoRIIOMode
|
||||
moriio_common = importlib.import_module(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_common"
|
||||
)
|
||||
|
||||
|
||||
class _FakeScheduler(moriio_connector.MoRIIOConnectorScheduler): # type: ignore[name-defined]
|
||||
"""Constructs a scheduler without the heavy real ``__init__``; only the
|
||||
attributes read by the method under test are set."""
|
||||
|
||||
def __init__(self, **attrs):
|
||||
self._mamba_group_ids: list[int] = []
|
||||
self._attn_group_ids: list[int] = [0]
|
||||
self._ssm_state_slots_are_positional = False
|
||||
self._is_hma_required = False
|
||||
self.kv_cache_config = SimpleNamespace(
|
||||
kv_cache_groups=[None, None],
|
||||
select_transfer_block_ids=lambda block_ids: tuple(block_ids),
|
||||
)
|
||||
for k, v in attrs.items():
|
||||
setattr(self, k, v)
|
||||
|
||||
|
||||
class _FakeConnector(moriio_connector.MoRIIOConnector): # type: ignore[name-defined]
|
||||
def __init__(self, connector_scheduler):
|
||||
self.connector_scheduler = connector_scheduler
|
||||
|
||||
|
||||
class _FakeWorker(moriio_connector.MoRIIOConnectorWorker): # type: ignore[name-defined]
|
||||
def __init__(self, **attrs):
|
||||
for k, v in attrs.items():
|
||||
setattr(self, k, v)
|
||||
|
||||
|
||||
class _FakeBlocks:
|
||||
def __init__(self, all_groups):
|
||||
self._all_groups = all_groups
|
||||
|
||||
def get_block_ids(self):
|
||||
return self._all_groups
|
||||
|
||||
|
||||
def _slot_strided(num_slots, per_slot_shape, slot_stride, dtype=torch.bfloat16):
|
||||
inner = 1
|
||||
for d in per_slot_shape:
|
||||
inner *= d
|
||||
assert slot_stride >= inner
|
||||
backing = torch.zeros(num_slots * slot_stride, dtype=dtype)
|
||||
inner_strides: list[int] = []
|
||||
acc = 1
|
||||
for d in reversed(per_slot_shape):
|
||||
inner_strides.insert(0, acc)
|
||||
acc *= d
|
||||
return backing.as_strided(
|
||||
(num_slots, *per_slot_shape), (slot_stride, *inner_strides)
|
||||
)
|
||||
|
||||
|
||||
def _gdn_split_info(conv_rows=3, key_dim=4, value_dim=8, dtype_size=2):
|
||||
conv_dim = 2 * key_dim + value_dim
|
||||
conv_state_bytes = conv_dim * conv_rows * dtype_size
|
||||
ssm_state_bytes = 64
|
||||
return MambaConvSplitInfo(
|
||||
conv_rows=conv_rows,
|
||||
local_proj_dims=(key_dim, key_dim, value_dim),
|
||||
conv_dtype_size=dtype_size,
|
||||
ssm_sizes=(conv_state_bytes, ssm_state_bytes),
|
||||
)
|
||||
|
||||
|
||||
def _mamba_spec(num_states: int = 2):
|
||||
shapes = tuple((1, 1) for _ in range(num_states))
|
||||
return moriio_connector.MambaSpec(
|
||||
block_size=16,
|
||||
shapes=shapes,
|
||||
dtypes=(torch.float32,) * num_states,
|
||||
mamba_cache_mode="all",
|
||||
)
|
||||
|
||||
|
||||
def _gate_vllm_config(*, read_mode: bool = True, speculative_config=None):
|
||||
return SimpleNamespace(
|
||||
kv_transfer_config=SimpleNamespace(
|
||||
kv_connector_extra_config={"read_mode": read_mode}
|
||||
),
|
||||
speculative_config=speculative_config,
|
||||
)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# split_block_groups
|
||||
# --------------------------------------------------------------------------
|
||||
def test_split_block_groups_separates_attention_and_mamba():
|
||||
sched = _FakeScheduler(_has_mamba=True, _attn_group_ids=[0], _mamba_group_ids=[1])
|
||||
block_ids = ([1, 2], [99])
|
||||
attn, mamba = sched.split_block_groups(block_ids)
|
||||
assert attn == [1, 2]
|
||||
assert mamba == [[99]]
|
||||
|
||||
|
||||
def test_split_block_groups_ignores_transfer_disabled_group():
|
||||
mamba_spec = moriio_connector.MambaSpec(
|
||||
block_size=16,
|
||||
shapes=((1, 1),),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="all",
|
||||
)
|
||||
config = SimpleNamespace(
|
||||
kv_cache_groups=[
|
||||
SimpleNamespace(enable_kv_transfer=True, kv_cache_spec=object()),
|
||||
SimpleNamespace(enable_kv_transfer=False, kv_cache_spec=object()),
|
||||
SimpleNamespace(enable_kv_transfer=True, kv_cache_spec=mamba_spec),
|
||||
],
|
||||
)
|
||||
config.transfer_groups = tuple(
|
||||
group for group in config.kv_cache_groups if group.enable_kv_transfer
|
||||
)
|
||||
config.select_transfer_block_ids = lambda block_ids: tuple(
|
||||
block_ids[group_id]
|
||||
for group_id, group in enumerate(config.kv_cache_groups)
|
||||
if group.enable_kv_transfer
|
||||
)
|
||||
attn_groups, mamba_groups = moriio_connector._split_kv_cache_group_kinds(config)
|
||||
assert attn_groups == [0]
|
||||
assert mamba_groups == [1]
|
||||
|
||||
sched = _FakeScheduler(
|
||||
_has_mamba=True,
|
||||
_attn_group_ids=attn_groups,
|
||||
_mamba_group_ids=mamba_groups,
|
||||
kv_cache_config=config,
|
||||
)
|
||||
assert sched.split_block_groups(([1, 2], [70, 71], [99])) == (
|
||||
[1, 2],
|
||||
[[99]],
|
||||
)
|
||||
|
||||
|
||||
def test_exchange_blocks_ignore_transfer_disabled_group():
|
||||
config = SimpleNamespace(
|
||||
kv_cache_groups=[None, None, None],
|
||||
select_transfer_block_ids=lambda block_ids: (block_ids[0], block_ids[2]),
|
||||
)
|
||||
sched = _FakeScheduler(
|
||||
kv_cache_config=config,
|
||||
_is_hma_required=True,
|
||||
blocks_per_sw=[0, 2],
|
||||
)
|
||||
|
||||
assert sched.get_exchange_clipped_blocks(([1, 2], [70, 71], [90, 91, 92])) == [
|
||||
[1, 2],
|
||||
[91, 92],
|
||||
]
|
||||
|
||||
|
||||
def test_scheduler_rejects_multiple_attention_groups_with_mamba():
|
||||
config = SimpleNamespace(
|
||||
kv_cache_groups=[
|
||||
SimpleNamespace(enable_kv_transfer=True, kv_cache_spec=object()),
|
||||
SimpleNamespace(enable_kv_transfer=True, kv_cache_spec=object()),
|
||||
SimpleNamespace(enable_kv_transfer=True, kv_cache_spec=_mamba_spec()),
|
||||
],
|
||||
)
|
||||
config.transfer_groups = tuple(config.kv_cache_groups)
|
||||
|
||||
with pytest.raises(moriio_common.MoRIIOError, match="exactly one transferable"):
|
||||
moriio_connector.MoRIIOConnectorScheduler(_gate_vllm_config(), "engine", config)
|
||||
|
||||
|
||||
def test_split_block_groups_preserves_multiple_mamba_groups():
|
||||
sched = _FakeScheduler(
|
||||
_has_mamba=True,
|
||||
_attn_group_ids=[0],
|
||||
_mamba_group_ids=[1, 2],
|
||||
)
|
||||
|
||||
assert sched.split_block_groups(([1, 2], [40], [70])) == (
|
||||
[1, 2],
|
||||
[[40], [70]],
|
||||
)
|
||||
|
||||
|
||||
def test_scheduler_rejects_incompatible_mamba_groups():
|
||||
incompatible = _mamba_spec()
|
||||
object.__setattr__(incompatible, "page_size_padded", 4096)
|
||||
config = SimpleNamespace(
|
||||
kv_cache_groups=[
|
||||
SimpleNamespace(enable_kv_transfer=True, kv_cache_spec=object()),
|
||||
SimpleNamespace(enable_kv_transfer=True, kv_cache_spec=_mamba_spec()),
|
||||
SimpleNamespace(enable_kv_transfer=True, kv_cache_spec=incompatible),
|
||||
]
|
||||
)
|
||||
config.transfer_groups = tuple(config.kv_cache_groups)
|
||||
|
||||
with pytest.raises(moriio_common.MoRIIOError, match="same cache spec"):
|
||||
moriio_connector.MoRIIOConnectorScheduler(_gate_vllm_config(), "engine", config)
|
||||
|
||||
|
||||
def test_scheduler_rejects_mamba_group_without_two_states():
|
||||
config = SimpleNamespace(
|
||||
kv_cache_groups=[
|
||||
SimpleNamespace(enable_kv_transfer=True, kv_cache_spec=object()),
|
||||
SimpleNamespace(enable_kv_transfer=True, kv_cache_spec=_mamba_spec(1)),
|
||||
],
|
||||
)
|
||||
config.transfer_groups = tuple(config.kv_cache_groups)
|
||||
|
||||
with pytest.raises(moriio_common.MoRIIOError, match="exactly two Mamba states"):
|
||||
moriio_connector.MoRIIOConnectorScheduler(_gate_vllm_config(), "engine", config)
|
||||
|
||||
|
||||
def test_scheduler_rejects_speculative_hybrid_read():
|
||||
config = SimpleNamespace(
|
||||
kv_cache_groups=[
|
||||
SimpleNamespace(enable_kv_transfer=True, kv_cache_spec=object()),
|
||||
SimpleNamespace(enable_kv_transfer=True, kv_cache_spec=_mamba_spec()),
|
||||
],
|
||||
)
|
||||
config.transfer_groups = tuple(config.kv_cache_groups)
|
||||
|
||||
with pytest.raises(moriio_common.MoRIIOError, match="speculative decoding"):
|
||||
moriio_connector.MoRIIOConnectorScheduler(
|
||||
_gate_vllm_config(speculative_config=object()), "engine", config
|
||||
)
|
||||
|
||||
|
||||
def test_scheduler_rejects_hybrid_write():
|
||||
config = SimpleNamespace(
|
||||
kv_cache_groups=[
|
||||
SimpleNamespace(enable_kv_transfer=True, kv_cache_spec=object()),
|
||||
SimpleNamespace(enable_kv_transfer=True, kv_cache_spec=_mamba_spec()),
|
||||
],
|
||||
)
|
||||
config.transfer_groups = tuple(config.kv_cache_groups)
|
||||
|
||||
with pytest.raises(moriio_common.MoRIIOError, match="READ mode only"):
|
||||
moriio_connector.MoRIIOConnectorScheduler(
|
||||
_gate_vllm_config(read_mode=False), "engine", config
|
||||
)
|
||||
|
||||
|
||||
def test_split_block_groups_rejects_non_hybrid_use():
|
||||
sched = _FakeScheduler(_has_mamba=False)
|
||||
|
||||
with pytest.raises(moriio_common.MoRIIOError, match="non-hybrid"):
|
||||
sched.split_block_groups(([5, 6, 7],))
|
||||
|
||||
|
||||
def test_split_block_groups_accepts_empty_abort_payload():
|
||||
sched = _FakeScheduler(_has_mamba=True)
|
||||
|
||||
assert sched.split_block_groups(()) == ([], [])
|
||||
|
||||
|
||||
def test_split_block_groups_keeps_positional_slots_in_all_mode():
|
||||
# mamba_cache_mode="all" keeps a state per block position.
|
||||
sched = _FakeScheduler(
|
||||
_has_mamba=True,
|
||||
_attn_group_ids=[0],
|
||||
_mamba_group_ids=[1],
|
||||
_ssm_state_slots_are_positional=True,
|
||||
)
|
||||
attn, mamba = sched.split_block_groups(([1], [40, 41, 42]))
|
||||
assert attn == [1]
|
||||
assert mamba == [[40, 41, 42]]
|
||||
|
||||
|
||||
def test_split_block_groups_keeps_only_running_state_outside_all_mode():
|
||||
sched = _FakeScheduler(
|
||||
_has_mamba=True,
|
||||
_attn_group_ids=[0],
|
||||
_mamba_group_ids=[1],
|
||||
)
|
||||
assert sched.split_block_groups(([1], [40, 41])) == ([1], [[41]])
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# request_finished_all_groups
|
||||
# --------------------------------------------------------------------------
|
||||
def test_request_finished_all_groups_carries_attn_and_mamba_in_one_field():
|
||||
seen = {}
|
||||
|
||||
def _fake_request_finished(request, attn_block_ids, mamba_block_groups):
|
||||
seen["attn"] = list(attn_block_ids)
|
||||
seen["mamba"] = [list(group) for group in mamba_block_groups]
|
||||
return True, {
|
||||
"do_remote_prefill": True,
|
||||
"remote_block_ids": [attn_block_ids, *mamba_block_groups],
|
||||
}
|
||||
|
||||
sched = _FakeScheduler(_has_mamba=True, _attn_group_ids=[0], _mamba_group_ids=[1])
|
||||
sched.request_finished = _fake_request_finished
|
||||
conn = _FakeConnector(sched)
|
||||
|
||||
request = SimpleNamespace(request_id="r0")
|
||||
block_ids = ([10, 11], [42])
|
||||
delay_free, params = conn.request_finished_all_groups(request, block_ids)
|
||||
|
||||
assert delay_free is True
|
||||
# Attention ids drive request_finished; the mamba slot rides the SAME
|
||||
# remote_block_ids channel as [attn, *mamba_groups], so the proxy/router
|
||||
# needs no KDA-specific field.
|
||||
assert seen["attn"] == [10, 11]
|
||||
assert seen["mamba"] == [[42]]
|
||||
assert params["remote_block_ids"] == [[10, 11], [42]]
|
||||
assert "remote_mamba_block_ids" not in params
|
||||
|
||||
|
||||
def test_request_finished_all_groups_pure_attention_stays_grouped():
|
||||
def _fake_request_finished(request, block_ids):
|
||||
return True, {"remote_block_ids": block_ids}
|
||||
|
||||
sched = _FakeScheduler(_has_mamba=False, _mamba_group_ids=[])
|
||||
sched.request_finished = _fake_request_finished
|
||||
conn = _FakeConnector(sched)
|
||||
|
||||
delay_free, params = conn.request_finished_all_groups(
|
||||
SimpleNamespace(request_id="r1"), ([7, 8],)
|
||||
)
|
||||
# Main's HMA contract keeps pure-attention block ids grouped.
|
||||
assert params["remote_block_ids"] == ([7, 8],)
|
||||
assert "remote_mamba_block_ids" not in params
|
||||
|
||||
|
||||
def test_connector_supports_divergent_local_hybrid_hits():
|
||||
connector = _FakeConnector(None)
|
||||
connector._transfers_mamba_state = True
|
||||
connector.mode = MoRIIOMode.READ
|
||||
assert connector.supports_divergent_local_hybrid_hits is True
|
||||
connector.mode = MoRIIOMode.WRITE
|
||||
assert connector.supports_divergent_local_hybrid_hits is False
|
||||
|
||||
connector.mode = MoRIIOMode.READ
|
||||
connector._transfers_mamba_state = False
|
||||
assert connector.supports_divergent_local_hybrid_hits is False
|
||||
|
||||
|
||||
def _make_read_scheduler():
|
||||
return _FakeScheduler(
|
||||
is_producer=False,
|
||||
mode=MoRIIOMode.READ,
|
||||
_has_mamba=True,
|
||||
_attn_group_ids=[0],
|
||||
_mamba_group_ids=[1],
|
||||
_ssm_state_slots_are_positional=True,
|
||||
request_id_to_transfer_id={},
|
||||
transfer_id_to_request_id={},
|
||||
_reqs_need_recv={},
|
||||
_req_kv_params={},
|
||||
_max_decode_tail_blocks=1,
|
||||
)
|
||||
|
||||
|
||||
def _make_read_request(remote_block_ids):
|
||||
return SimpleNamespace(
|
||||
request_id="req",
|
||||
kv_transfer_params={
|
||||
"do_remote_prefill": True,
|
||||
"transfer_id": "tx",
|
||||
"remote_engine_id": "prefill",
|
||||
"remote_block_ids": remote_block_ids,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def test_update_state_drops_decode_recompute_tail_block():
|
||||
sched = _make_read_scheduler()
|
||||
request = _make_read_request([[10, 11], [90, 91]])
|
||||
blocks = _FakeBlocks(
|
||||
all_groups=([100, 101, 102], [200, 201, 202]),
|
||||
)
|
||||
|
||||
sched.update_state_after_alloc(request, blocks, num_external_tokens=256)
|
||||
|
||||
assert sched._reqs_need_recv["req"][1] == [[100, 101], [200, 201]]
|
||||
assert sched._req_kv_params["req"]["remote_block_ids"] == [
|
||||
[10, 11],
|
||||
[90, 91],
|
||||
]
|
||||
|
||||
|
||||
def test_update_state_pairs_shorter_local_blocks_with_remote_suffix():
|
||||
sched = _make_read_scheduler()
|
||||
request = _make_read_request([[10, 11, 12], [92]])
|
||||
blocks = _FakeBlocks(
|
||||
all_groups=([102], [202]),
|
||||
)
|
||||
|
||||
sched.update_state_after_alloc(request, blocks, num_external_tokens=64)
|
||||
|
||||
assert sched._reqs_need_recv["req"][1] == [[102], [202]]
|
||||
assert sched._req_kv_params["req"]["remote_block_ids"] == [[12], [92]]
|
||||
|
||||
|
||||
def test_update_state_full_attention_hit_still_carries_mamba_state():
|
||||
sched = _make_read_scheduler()
|
||||
request = _make_read_request([[10], [90]])
|
||||
|
||||
sched.update_state_after_alloc(
|
||||
request,
|
||||
_FakeBlocks(all_groups=([100], [200])),
|
||||
num_external_tokens=0,
|
||||
)
|
||||
|
||||
assert sched._reqs_need_recv["req"][1] == [[], [200]]
|
||||
|
||||
|
||||
def test_update_state_preserves_each_mamba_group():
|
||||
sched = _make_read_scheduler()
|
||||
sched._mamba_group_ids = [1, 2]
|
||||
request = _make_read_request([[10], [90], [80]])
|
||||
|
||||
sched.update_state_after_alloc(
|
||||
request,
|
||||
_FakeBlocks(all_groups=([100], [200], [300])),
|
||||
num_external_tokens=64,
|
||||
)
|
||||
|
||||
assert sched._reqs_need_recv["req"][1] == [[100], [200], [300]]
|
||||
assert sched._req_kv_params["req"]["remote_block_ids"] == [
|
||||
[10],
|
||||
[90],
|
||||
[80],
|
||||
]
|
||||
|
||||
|
||||
def test_align_read_blocks_allows_configured_lookahead_tail():
|
||||
assert moriio_connector.MoRIIOConnectorScheduler._align_read_blocks(
|
||||
[100, 101, 102], [10], max_decode_tail_blocks=2
|
||||
) == ([100], [10])
|
||||
with pytest.raises(ValueError, match="allowed local tail"):
|
||||
moriio_connector.MoRIIOConnectorScheduler._align_read_blocks(
|
||||
[100, 101, 102], [10], max_decode_tail_blocks=1
|
||||
)
|
||||
|
||||
|
||||
def test_update_state_aligns_attention_only_decode_tail():
|
||||
sched = _make_read_scheduler()
|
||||
sched._has_mamba = False
|
||||
sched._mamba_group_ids = []
|
||||
request = _make_read_request([[10, 11]])
|
||||
|
||||
sched.update_state_after_alloc(
|
||||
request,
|
||||
_FakeBlocks(all_groups=([100, 101, 102],)),
|
||||
num_external_tokens=256,
|
||||
)
|
||||
|
||||
assert sched._reqs_need_recv["req"][1] == [[100, 101]]
|
||||
assert sched._req_kv_params["req"]["remote_block_ids"] == [[10, 11]]
|
||||
|
||||
|
||||
def test_read_producer_does_not_queue_write_only_save_state():
|
||||
sched = _FakeScheduler(
|
||||
is_producer=True,
|
||||
mode=MoRIIOMode.READ,
|
||||
_has_mamba=True,
|
||||
_attn_group_ids=[0],
|
||||
_mamba_group_ids=[1],
|
||||
request_id_to_transfer_id={},
|
||||
transfer_id_to_request_id={},
|
||||
_reqs_need_save={},
|
||||
_req_kv_params={},
|
||||
)
|
||||
request = SimpleNamespace(
|
||||
request_id="req",
|
||||
kv_transfer_params={"do_remote_decode": True, "transfer_id": "tx"},
|
||||
)
|
||||
|
||||
sched.update_state_after_alloc(
|
||||
request,
|
||||
_FakeBlocks(all_groups=([100, 101], [200])),
|
||||
num_external_tokens=0,
|
||||
)
|
||||
|
||||
assert sched._reqs_need_save == {}
|
||||
assert sched._req_kv_params == {}
|
||||
|
||||
|
||||
def test_write_producer_queues_grouped_attention_blocks():
|
||||
sched = _FakeScheduler(
|
||||
is_producer=True,
|
||||
mode=MoRIIOMode.WRITE,
|
||||
_has_mamba=False,
|
||||
_attn_group_ids=[0],
|
||||
_mamba_group_ids=[],
|
||||
request_id_to_transfer_id={},
|
||||
transfer_id_to_request_id={},
|
||||
_reqs_need_save={},
|
||||
_req_kv_params={},
|
||||
)
|
||||
request = SimpleNamespace(
|
||||
request_id="req",
|
||||
kv_transfer_params={"do_remote_decode": True, "transfer_id": "tx"},
|
||||
)
|
||||
|
||||
sched.update_state_after_alloc(
|
||||
request,
|
||||
_FakeBlocks(all_groups=([100, 101],)),
|
||||
num_external_tokens=0,
|
||||
)
|
||||
|
||||
assert sched._reqs_need_save["req"][1] == ([100, 101],)
|
||||
assert sched._req_kv_params["req"] == request.kv_transfer_params
|
||||
|
||||
|
||||
def test_session_build_rejects_per_layer_region_count_mismatch():
|
||||
worker = _FakeWorker(
|
||||
built_write_session={},
|
||||
layer_name_to_local_kv_cache_metadata={"kda.0": ["conv", "ssm"]},
|
||||
layer_name_to_remote_kv_cache_metadata={"prefill": {"kda.0": ["conv"]}},
|
||||
)
|
||||
|
||||
with pytest.raises(moriio_common.MoRIIOError, match="registered 1 region"):
|
||||
worker._get_built_session("prefill")
|
||||
|
||||
|
||||
def test_register_kv_caches_rejects_hybrid_write_before_registration():
|
||||
worker = _FakeWorker(mode=MoRIIOMode.WRITE, _transfer_layer_names={"kda.0"})
|
||||
worker._is_mamba_layer = lambda _layer_name: True
|
||||
|
||||
with pytest.raises(moriio_common.MoRIIOError, match="READ mode only"):
|
||||
worker.register_kv_caches({"kda.0": object()})
|
||||
|
||||
|
||||
def test_register_kv_caches_rejects_no_transfer_enabled_layers():
|
||||
worker = _FakeWorker(mode=MoRIIOMode.READ, _transfer_layer_names=set())
|
||||
|
||||
with pytest.raises(moriio_common.MoRIIOError, match="no transfer-enabled"):
|
||||
worker.register_kv_caches({"disabled.0": object()})
|
||||
|
||||
|
||||
def test_worker_rejects_incompatible_mamba_specs_before_registration():
|
||||
incompatible = _mamba_spec()
|
||||
object.__setattr__(incompatible, "page_size_padded", 4096)
|
||||
worker = _FakeWorker(
|
||||
mode=MoRIIOMode.READ,
|
||||
world_size=1,
|
||||
_transfer_layer_names={"kda.0", "kda.1"},
|
||||
layer_to_spec={"kda.0": _mamba_spec(), "kda.1": incompatible},
|
||||
)
|
||||
|
||||
with pytest.raises(moriio_common.MoRIIOError, match="same cache spec"):
|
||||
worker.register_kv_caches({"kda.0": object(), "kda.1": object()})
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# _truncate_mamba_request_for_prefill
|
||||
# --------------------------------------------------------------------------
|
||||
def _mk_request(prompt, max_tokens=64, params=None):
|
||||
return SimpleNamespace(
|
||||
kv_transfer_params={} if params is None else params,
|
||||
num_prompt_tokens=len(prompt),
|
||||
prompt_token_ids=list(prompt),
|
||||
prompt_embeds=None,
|
||||
_all_token_ids=list(prompt),
|
||||
max_tokens=max_tokens,
|
||||
)
|
||||
|
||||
|
||||
def test_truncate_mamba_request_pops_last_token_and_caps_tokens():
|
||||
sched = _FakeScheduler()
|
||||
req = _mk_request([10, 11, 12, 13, 14])
|
||||
sched._truncate_mamba_request_for_prefill(req)
|
||||
|
||||
assert req.prompt_token_ids == [10, 11, 12, 13] # last token popped
|
||||
assert req._all_token_ids == [10, 11, 12, 13]
|
||||
assert req.num_prompt_tokens == 4 # N-1 accounting
|
||||
assert req.max_tokens == 1
|
||||
assert req.kv_transfer_params["_p_side_truncated"] is True
|
||||
|
||||
|
||||
def test_truncate_mamba_request_is_idempotent_across_reschedule():
|
||||
sched = _FakeScheduler()
|
||||
req = _mk_request([10, 11, 12])
|
||||
sched._truncate_mamba_request_for_prefill(req)
|
||||
assert req.num_prompt_tokens == 2
|
||||
# A second call (e.g. after a preemption) must not truncate again.
|
||||
sched._truncate_mamba_request_for_prefill(req)
|
||||
assert req.num_prompt_tokens == 2
|
||||
assert req.prompt_token_ids == [10, 11]
|
||||
|
||||
|
||||
def test_truncate_mamba_request_noop_for_single_token_prompt():
|
||||
sched = _FakeScheduler()
|
||||
req = _mk_request([10])
|
||||
sched._truncate_mamba_request_for_prefill(req)
|
||||
assert req.num_prompt_tokens == 1
|
||||
assert req.prompt_token_ids == [10]
|
||||
assert "_p_side_truncated" not in req.kv_transfer_params
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# get_num_new_matched_tokens (hybrid N-1 accounting)
|
||||
# --------------------------------------------------------------------------
|
||||
def test_get_num_new_matched_tokens_read_recomputes_last_token():
|
||||
sched = _FakeScheduler(is_producer=False, mode=MoRIIOMode.READ, _has_mamba=True)
|
||||
req = SimpleNamespace(
|
||||
num_prompt_tokens=10,
|
||||
prompt_token_ids=list(range(10)),
|
||||
kv_transfer_params=None,
|
||||
)
|
||||
n, is_async = sched.get_num_new_matched_tokens(req, num_computed_tokens=0)
|
||||
# READ always recomputes the final token locally: N-1 - computed.
|
||||
assert n == 9
|
||||
assert is_async is False
|
||||
|
||||
|
||||
def test_get_num_new_matched_tokens_write_plain_keeps_all_tokens():
|
||||
sched = _FakeScheduler(is_producer=False, mode=MoRIIOMode.WRITE, _has_mamba=False)
|
||||
req = SimpleNamespace(
|
||||
num_prompt_tokens=10,
|
||||
prompt_token_ids=list(range(10)),
|
||||
kv_transfer_params=None,
|
||||
)
|
||||
n, is_async = sched.get_num_new_matched_tokens(req, num_computed_tokens=2)
|
||||
# Pure-attention WRITE: no N-1 drop; full length minus already-computed.
|
||||
assert n == 8
|
||||
assert is_async is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("mode", "num_computed_tokens", "expected", "is_async"),
|
||||
[
|
||||
(MoRIIOMode.READ, 0, 9, False),
|
||||
(MoRIIOMode.READ, 10, 0, False),
|
||||
(MoRIIOMode.WRITE, 2, 8, True),
|
||||
],
|
||||
)
|
||||
def test_get_num_new_matched_tokens_supports_embeds_only_prompts(
|
||||
mode, num_computed_tokens, expected, is_async
|
||||
):
|
||||
sched = _FakeScheduler(is_producer=False, mode=mode, _has_mamba=True)
|
||||
req = SimpleNamespace(
|
||||
num_prompt_tokens=10,
|
||||
prompt_token_ids=None,
|
||||
prompt_embeds=object(),
|
||||
kv_transfer_params=None,
|
||||
)
|
||||
|
||||
assert sched.get_num_new_matched_tokens(req, num_computed_tokens) == (
|
||||
expected,
|
||||
is_async,
|
||||
)
|
||||
|
||||
|
||||
def test_get_num_new_matched_tokens_producer_returns_zero():
|
||||
sched = _FakeScheduler(is_producer=True, mode=MoRIIOMode.WRITE, _has_mamba=True)
|
||||
req = _mk_request(list(range(5)), params={"do_remote_decode": True})
|
||||
n, is_async = sched.get_num_new_matched_tokens(req, num_computed_tokens=0)
|
||||
assert (n, is_async) == (0, False)
|
||||
# The producer never truncates here: the scheduler has already measured the
|
||||
# prefix-cache hit against the full prompt by this point.
|
||||
assert req.num_prompt_tokens == 5
|
||||
|
||||
|
||||
def test_on_new_request_truncates_producer_prompt():
|
||||
sched = _FakeScheduler(is_producer=True, mode=MoRIIOMode.WRITE, _has_mamba=True)
|
||||
req = _mk_request(list(range(5)), params={"do_remote_decode": True})
|
||||
sched.on_new_request(req)
|
||||
# Producer stops at h(N-1): the last prompt token was dropped.
|
||||
assert req.num_prompt_tokens == 4
|
||||
assert req.max_tokens == 1
|
||||
assert req.kv_transfer_params["_p_side_truncated"] is True
|
||||
|
||||
|
||||
def test_on_new_request_is_noop_without_mamba_or_on_decode():
|
||||
plain = _FakeScheduler(is_producer=True, mode=MoRIIOMode.WRITE, _has_mamba=False)
|
||||
req = _mk_request(list(range(5)), params={"do_remote_decode": True})
|
||||
plain.on_new_request(req)
|
||||
assert req.num_prompt_tokens == 5
|
||||
|
||||
decode_side = _FakeScheduler(
|
||||
is_producer=False, mode=MoRIIOMode.WRITE, _has_mamba=True
|
||||
)
|
||||
req2 = _mk_request(list(range(5)), params={"do_remote_prefill": True})
|
||||
decode_side.on_new_request(req2)
|
||||
assert req2.num_prompt_tokens == 5
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Offset-template cache
|
||||
# --------------------------------------------------------------------------
|
||||
_SAMPLE_SLOT_SETS = [
|
||||
([1, 2], [1, 2]),
|
||||
([0], [3]),
|
||||
([3, 0, 2], [3, 0, 2]),
|
||||
([2, 2], [5, 5]),
|
||||
([], []),
|
||||
]
|
||||
|
||||
|
||||
def test_worker_compute_mamba_offsets_caches_and_matches_template():
|
||||
split = _gdn_split_info()
|
||||
conv_dim = sum(split.local_proj_dims)
|
||||
conv_shape = (conv_dim, split.conv_rows)
|
||||
ssm_shape = (2, 4, 4)
|
||||
page_bytes = 256
|
||||
pages = torch.zeros((8, 1, 1, page_bytes), dtype=torch.uint8)
|
||||
spec = moriio_connector.MambaSpec(
|
||||
block_size=1,
|
||||
shapes=(conv_shape, ssm_shape),
|
||||
dtypes=(torch.bfloat16, torch.bfloat16),
|
||||
page_size_padded=page_bytes,
|
||||
)
|
||||
conv, ssm = moriio_layout.kda_conv_ssm(pages, spec)
|
||||
|
||||
worker = _FakeWorker(
|
||||
kv_caches={"kda.0": pages},
|
||||
layer_to_spec={"kda.0": spec},
|
||||
_conv_decomp=split,
|
||||
_mamba_offset_templates={},
|
||||
tp_rank=0,
|
||||
world_size=1,
|
||||
)
|
||||
|
||||
cached_template = None
|
||||
for local_slots, remote_slots in _SAMPLE_SLOT_SETS:
|
||||
lo, ro, sz, n_conv = worker._compute_mamba_transfer_offsets(
|
||||
"kda.0", local_slots, remote_slots
|
||||
)
|
||||
template = moriio_layout.build_mamba_offset_template(
|
||||
conv, ssm, split, tp_ratio=1
|
||||
)
|
||||
expected = moriio_layout.apply_mamba_offset_template(
|
||||
template, local_slots, remote_slots
|
||||
)
|
||||
assert (lo, ro, sz) == expected
|
||||
assert n_conv == moriio_layout.compute_mamba_conv_split_count(
|
||||
local_slots, split
|
||||
)
|
||||
|
||||
current_template = worker._mamba_offset_templates["kda.0"]
|
||||
if cached_template is None:
|
||||
cached_template = current_template
|
||||
else:
|
||||
assert current_template is cached_template
|
||||
|
||||
assert "kda.0" in worker._mamba_offset_templates
|
||||
|
||||
|
||||
@pytest.mark.parametrize("remote_tp_size", [1, None])
|
||||
def test_mamba_tp_ratio_accepts_only_equal_tp(remote_tp_size):
|
||||
worker = _FakeWorker(world_size=1)
|
||||
assert worker._mamba_tp_ratio(remote_tp_size) == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize("remote_tp_size", [2, 3])
|
||||
def test_mamba_tp_ratio_rejects_heterogeneous_tp(remote_tp_size):
|
||||
worker = _FakeWorker(world_size=1)
|
||||
with pytest.raises(NotImplementedError, match="heterogeneous-TP"):
|
||||
worker._mamba_tp_ratio(remote_tp_size)
|
||||
|
||||
|
||||
def test_worker_selects_each_mamba_layers_own_block_group():
|
||||
worker = _FakeWorker(_mamba_payload_index_by_layer={"kda.0": 1, "kda.1": 2})
|
||||
groups = ([10], [40], [70])
|
||||
|
||||
assert worker._mamba_blocks_for_layer("kda.0", groups) == [40]
|
||||
assert worker._mamba_blocks_for_layer("kda.1", groups) == [70]
|
||||
|
||||
|
||||
def test_worker_rejects_missing_mamba_block_group():
|
||||
worker = _FakeWorker(_mamba_payload_index_by_layer={"kda.0": 2})
|
||||
|
||||
with pytest.raises(moriio_common.MoRIIOError, match="no block group"):
|
||||
worker._mamba_blocks_for_layer("kda.0", ([10], [40]))
|
||||
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import importlib.util
|
||||
import math
|
||||
import threading
|
||||
from collections import OrderedDict, defaultdict
|
||||
from queue import Queue
|
||||
@@ -14,6 +15,7 @@ import torch
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
MambaSpec,
|
||||
MLAAttentionSpec,
|
||||
)
|
||||
|
||||
@@ -36,6 +38,10 @@ moriio_layout = importlib.import_module(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_layout"
|
||||
)
|
||||
msgpack = importlib.import_module("msgpack")
|
||||
ssm_conv_transfer_utils = importlib.import_module(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.ssm_conv_transfer_utils"
|
||||
)
|
||||
MambaConvSplitInfo = ssm_conv_transfer_utils.MambaConvSplitInfo
|
||||
|
||||
ROLE = moriio_common.ROLE
|
||||
MoRIIOError = moriio_common.MoRIIOError
|
||||
@@ -353,6 +359,9 @@ def test_write_transfer_plan_caches_offsets_per_layer_geometry():
|
||||
kv_caches: dict[str, torch.Tensor]
|
||||
layer_name_to_local_kv_cache_metadata: dict[str, list[Any]]
|
||||
|
||||
def _region_session_indices(self, layer_name):
|
||||
return [list(self.kv_caches).index(layer_name)]
|
||||
|
||||
def _compute_block_transfer_offsets(
|
||||
self,
|
||||
layer_name,
|
||||
@@ -673,6 +682,30 @@ def test_local_block_ids_longer_than_remote_raises_value_error():
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("local_block_ids", "remote_block_ids", "match"),
|
||||
[
|
||||
([-1], [0], "local block id -1"),
|
||||
([0], [8], "remote block id 8"),
|
||||
],
|
||||
)
|
||||
def test_block_ids_must_fit_registered_regions(
|
||||
local_block_ids, remote_block_ids, match
|
||||
):
|
||||
cache = torch.empty((8, 2, 4, 2, 3), dtype=torch.bfloat16)
|
||||
worker = _worker({"layer": cache}, {"layer": _full_spec()})
|
||||
|
||||
with pytest.raises(ValueError, match=match):
|
||||
moriio_layout.compute_block_transfer_offsets(
|
||||
"layer",
|
||||
cache,
|
||||
worker.layer_to_spec,
|
||||
local_block_ids,
|
||||
remote_block_ids,
|
||||
_remote_meta(num_blocks=8).num_blocks,
|
||||
)
|
||||
|
||||
|
||||
def test_empty_local_block_ids_is_free_only_noop():
|
||||
cache = torch.empty((8, 2, 4, 2, 3), dtype=torch.bfloat16)
|
||||
worker = _worker({"layer": cache}, {"layer": _full_spec()})
|
||||
@@ -745,3 +778,245 @@ def test_unsupported_shape_raises_value_error():
|
||||
|
||||
with pytest.raises(ValueError, match="Unsupported MoRIIO K/V cache shape"):
|
||||
moriio_layout.get_layer_transfer_geometry("layer", cache, worker.layer_to_spec)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Mamba/KDA (conv + ssm) transfer geometry
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _slot_strided(num_slots, per_slot_shape, slot_stride, dtype=torch.bfloat16):
|
||||
"""Build a slot-strided view: shape (num_slots, *per_slot_shape) whose slot
|
||||
stride (dim 0) exceeds the per-slot element count, mimicking the K3 KDA
|
||||
conv/ssm cache tensors."""
|
||||
inner = 1
|
||||
for d in per_slot_shape:
|
||||
inner *= d
|
||||
assert slot_stride >= inner
|
||||
backing = torch.zeros(num_slots * slot_stride, dtype=dtype)
|
||||
inner_strides: list[int] = []
|
||||
acc = 1
|
||||
for d in reversed(per_slot_shape):
|
||||
inner_strides.insert(0, acc)
|
||||
acc *= d
|
||||
return backing.as_strided(
|
||||
(num_slots, *per_slot_shape), (slot_stride, *inner_strides)
|
||||
)
|
||||
|
||||
|
||||
def _gdn_split_info(conv_rows=3, key_dim=4, value_dim=8, dtype_size=2):
|
||||
"""A GDN-style conv split: sub-projections (key, key, value)."""
|
||||
conv_dim = 2 * key_dim + value_dim
|
||||
conv_state_bytes = conv_dim * conv_rows * dtype_size
|
||||
ssm_state_bytes = 64 # opaque here; ssm size is taken from the tensor
|
||||
return MambaConvSplitInfo(
|
||||
conv_rows=conv_rows,
|
||||
local_proj_dims=(key_dim, key_dim, value_dim),
|
||||
conv_dtype_size=dtype_size,
|
||||
ssm_sizes=(conv_state_bytes, ssm_state_bytes),
|
||||
)
|
||||
|
||||
|
||||
def test_mamba_transfer_geometry_is_slot_strided():
|
||||
split = _gdn_split_info()
|
||||
conv_dim = sum(split.local_proj_dims)
|
||||
conv = _slot_strided(5, (conv_dim, split.conv_rows), slot_stride=200)
|
||||
ssm = _slot_strided(5, (2, 4, 4), slot_stride=64)
|
||||
|
||||
geom = moriio_layout.get_mamba_transfer_geometry(conv, ssm)
|
||||
|
||||
assert geom.conv_slot_stride == 200
|
||||
assert geom.ssm_slot_stride == 64
|
||||
assert geom.ssm_slot_bytes == (2 * 4 * 4) * ssm.element_size()
|
||||
# Region spans the full slot-strided extent (shape[0] * stride(0)), not just
|
||||
# numel(), so the highest-slot transfers stay in-bounds.
|
||||
assert geom.conv_region_len == conv.shape[0] * conv.stride(0) * conv.element_size()
|
||||
assert geom.ssm_region_len == ssm.shape[0] * ssm.stride(0) * ssm.element_size()
|
||||
|
||||
|
||||
def test_kda_conv_ssm_unpacks_padded_pages_as_aliasing_strided_views():
|
||||
num_blocks = 4
|
||||
page_bytes = 128
|
||||
conv_shape = (3, 4)
|
||||
ssm_shape = (2, 2, 4)
|
||||
conv_bytes = math.prod(conv_shape) * torch.bfloat16.itemsize
|
||||
ssm_bytes = math.prod(ssm_shape) * torch.bfloat16.itemsize
|
||||
assert conv_bytes + ssm_bytes < page_bytes
|
||||
|
||||
pages = torch.zeros((num_blocks, 1, 1, page_bytes), dtype=torch.uint8)
|
||||
spec = MambaSpec(
|
||||
block_size=1,
|
||||
shapes=(conv_shape, ssm_shape),
|
||||
dtypes=(torch.bfloat16, torch.bfloat16),
|
||||
page_size_padded=page_bytes,
|
||||
)
|
||||
|
||||
conv, ssm = moriio_layout.kda_conv_ssm(pages, spec)
|
||||
|
||||
assert conv.shape == (num_blocks, *conv_shape)
|
||||
assert ssm.shape == (num_blocks, *ssm_shape)
|
||||
assert conv.stride() == (page_bytes // 2, 4, 1)
|
||||
assert ssm.stride() == (page_bytes // 2, 8, 4, 1)
|
||||
assert conv.untyped_storage().data_ptr() == pages.untyped_storage().data_ptr()
|
||||
assert ssm.untyped_storage().data_ptr() == pages.untyped_storage().data_ptr()
|
||||
assert conv.storage_offset() == 0
|
||||
assert ssm.storage_offset() * ssm.element_size() == conv_bytes
|
||||
|
||||
conv[1].fill_(1)
|
||||
ssm[2].fill_(2)
|
||||
assert torch.all(pages[1, 0, 0, :conv_bytes].view(torch.bfloat16) == 1)
|
||||
assert torch.all(
|
||||
pages[2, 0, 0, conv_bytes : conv_bytes + ssm_bytes].view(torch.bfloat16) == 2
|
||||
)
|
||||
|
||||
|
||||
def test_kda_conv_ssm_accepts_strided_blocks_without_copying():
|
||||
backing = torch.zeros(4 * 256, dtype=torch.uint8)
|
||||
pages = backing.as_strided((4, 1, 1, 128), (256, 128, 128, 1))
|
||||
spec = MambaSpec(
|
||||
block_size=1,
|
||||
shapes=((3, 4), (2, 2, 4)),
|
||||
dtypes=(torch.bfloat16, torch.bfloat16),
|
||||
page_size_padded=128,
|
||||
)
|
||||
|
||||
conv, ssm = moriio_layout.kda_conv_ssm(pages, spec)
|
||||
|
||||
assert not pages.is_contiguous()
|
||||
assert conv.stride(0) == 128
|
||||
assert ssm.stride(0) == 128
|
||||
assert conv.untyped_storage().data_ptr() == backing.data_ptr()
|
||||
assert ssm.untyped_storage().data_ptr() == backing.data_ptr()
|
||||
|
||||
|
||||
def test_kda_conv_ssm_rejects_noncontiguous_page_bytes():
|
||||
pages = torch.zeros((4, 1, 1, 256), dtype=torch.uint8)[..., ::2]
|
||||
spec = MambaSpec(
|
||||
block_size=1,
|
||||
shapes=((3, 4), (2, 2, 4)),
|
||||
dtypes=(torch.bfloat16, torch.bfloat16),
|
||||
page_size_padded=128,
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="contiguous byte dimension"):
|
||||
moriio_layout.kda_conv_ssm(pages, spec)
|
||||
|
||||
|
||||
def test_kda_conv_ssm_rejects_non_byte_packed_cache():
|
||||
pages = torch.zeros((4, 1, 1, 128), dtype=torch.bfloat16)
|
||||
spec = MambaSpec(
|
||||
block_size=1,
|
||||
shapes=((3, 4), (2, 2, 4)),
|
||||
dtypes=(torch.bfloat16, torch.bfloat16),
|
||||
page_size_padded=128,
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="byte-sized dtype"):
|
||||
moriio_layout.kda_conv_ssm(pages, spec)
|
||||
|
||||
|
||||
def test_mamba_transfer_geometry_requires_matching_slot_counts():
|
||||
conv = torch.empty((5, 4, 3), dtype=torch.bfloat16)
|
||||
ssm = torch.empty((4, 2, 4, 4), dtype=torch.bfloat16)
|
||||
|
||||
with pytest.raises(ValueError, match="conv/ssm slot count mismatch"):
|
||||
moriio_layout.get_mamba_transfer_geometry(conv, ssm)
|
||||
|
||||
|
||||
def test_mamba_transfer_geometry_rejects_empty_caches():
|
||||
conv = torch.empty((0, 4, 3), dtype=torch.bfloat16)
|
||||
ssm = torch.empty((0, 2, 4, 4), dtype=torch.bfloat16)
|
||||
|
||||
with pytest.raises(ValueError, match="at least one slot"):
|
||||
moriio_layout.get_mamba_transfer_geometry(conv, ssm)
|
||||
|
||||
|
||||
def test_mamba_conv_ssm_offsets_homogeneous_tp():
|
||||
split = _gdn_split_info()
|
||||
conv_dim = sum(split.local_proj_dims)
|
||||
conv = _slot_strided(6, (conv_dim, split.conv_rows), slot_stride=200)
|
||||
ssm = _slot_strided(6, (2, 4, 4), slot_stride=64)
|
||||
geom = moriio_layout.get_mamba_transfer_geometry(conv, ssm)
|
||||
|
||||
local_slots = [1, 2]
|
||||
remote_slots = [1, 2]
|
||||
template = moriio_layout.build_mamba_offset_template(conv, ssm, split, tp_ratio=1)
|
||||
local_offs, remote_offs, sizes = moriio_layout.apply_mamba_offset_template(
|
||||
template, local_slots, remote_slots
|
||||
)
|
||||
|
||||
n_conv = moriio_layout.compute_mamba_conv_split_count(local_slots, split)
|
||||
assert n_conv == len(local_slots) * len(split.local_proj_dims)
|
||||
assert len(local_offs) == n_conv + len(local_slots)
|
||||
|
||||
# Homogeneous TP: remote conv slices equal local slices; since local==remote
|
||||
# slots, conv local and remote offsets coincide.
|
||||
loc_co = split.local_conv_offsets
|
||||
rem_co = split.remote_conv_offsets(0, 1)
|
||||
k = 0
|
||||
esz = conv.element_size()
|
||||
for ls, rs in zip(local_slots, remote_slots):
|
||||
lbase = ls * geom.conv_slot_stride * esz
|
||||
rbase = rs * geom.conv_slot_stride * esz
|
||||
for (loff, _), (roff, rsz) in zip(loc_co, rem_co):
|
||||
assert local_offs[k] == lbase + loff
|
||||
assert remote_offs[k] == rbase + roff
|
||||
assert sizes[k] == rsz
|
||||
k += 1
|
||||
|
||||
# SSM: whole per-slot block, no head sharding at tp_ratio == 1.
|
||||
essz = ssm.element_size()
|
||||
for j, (ls, rs) in enumerate(zip(local_slots, remote_slots)):
|
||||
assert local_offs[n_conv + j] == ls * geom.ssm_slot_stride * essz
|
||||
assert remote_offs[n_conv + j] == rs * geom.ssm_slot_stride * essz
|
||||
assert sizes[n_conv + j] == geom.ssm_slot_bytes
|
||||
|
||||
|
||||
def test_mamba_conv_ssm_offsets_heterogeneous_tp_is_gated():
|
||||
# Heterogeneous-TP mamba transfer is a documented follow-up (remote slot
|
||||
# stride + multi-rank fan-in gather). Until then the offset helper must fail
|
||||
# loudly rather than silently transfer wrong bytes, since the mamba path
|
||||
# bypasses the attention KV-head validator.
|
||||
split = _gdn_split_info()
|
||||
conv_dim = sum(split.local_proj_dims)
|
||||
conv = _slot_strided(4, (conv_dim, split.conv_rows), slot_stride=200)
|
||||
ssm = _slot_strided(4, (2, 4, 4), slot_stride=64)
|
||||
with pytest.raises(NotImplementedError):
|
||||
moriio_layout.build_mamba_offset_template(conv, ssm, split, tp_ratio=2)
|
||||
|
||||
|
||||
def test_local_mamba_slots_must_fit_registered_regions():
|
||||
split = _gdn_split_info()
|
||||
conv_dim = sum(split.local_proj_dims)
|
||||
conv = _slot_strided(6, (conv_dim, split.conv_rows), slot_stride=200)
|
||||
ssm = _slot_strided(6, (2, 4, 4), slot_stride=64)
|
||||
template = moriio_layout.build_mamba_offset_template(conv, ssm, split, tp_ratio=1)
|
||||
|
||||
with pytest.raises(ValueError, match="local mamba slot 6"):
|
||||
moriio_layout.apply_mamba_offset_template(template, [6], [0])
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("local_slots", "remote_slots"),
|
||||
[([1, 2], [3]), ([1], [2, 3])],
|
||||
)
|
||||
def test_mamba_slot_counts_must_match(local_slots, remote_slots):
|
||||
split = _gdn_split_info()
|
||||
conv_dim = sum(split.local_proj_dims)
|
||||
conv = _slot_strided(6, (conv_dim, split.conv_rows), slot_stride=200)
|
||||
ssm = _slot_strided(6, (2, 4, 4), slot_stride=64)
|
||||
template = moriio_layout.build_mamba_offset_template(conv, ssm, split, tp_ratio=1)
|
||||
|
||||
with pytest.raises(ValueError, match="slot counts must match"):
|
||||
moriio_layout.apply_mamba_offset_template(template, local_slots, remote_slots)
|
||||
|
||||
|
||||
def test_remote_mamba_slots_must_be_non_negative():
|
||||
split = _gdn_split_info()
|
||||
conv_dim = sum(split.local_proj_dims)
|
||||
conv = _slot_strided(6, (conv_dim, split.conv_rows), slot_stride=200)
|
||||
ssm = _slot_strided(6, (2, 4, 4), slot_stride=64)
|
||||
template = moriio_layout.build_mamba_offset_template(conv, ssm, split, tp_ratio=1)
|
||||
|
||||
with pytest.raises(ValueError, match="must be non-negative"):
|
||||
moriio_layout.apply_mamba_offset_template(template, [1], [-1])
|
||||
|
||||
@@ -28,6 +28,7 @@ def _producer(global_dp_rank: int, dp_size: int, dp_size_local: int):
|
||||
s = SimpleNamespace(
|
||||
is_producer=True,
|
||||
_is_hma_required=False,
|
||||
_has_mamba=False,
|
||||
blocks_per_sw=[0],
|
||||
_global_dp_rank=global_dp_rank,
|
||||
engine_id="engine-abc",
|
||||
|
||||
@@ -2,12 +2,14 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import threading
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_common import (
|
||||
MoRIIOMode,
|
||||
MoRIIOTransferAck,
|
||||
TransferError,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector import (
|
||||
MoRIIOConnector,
|
||||
@@ -17,6 +19,9 @@ from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector import
|
||||
resolve_moriio_transfer_ack,
|
||||
validate_moriio_heterogeneous_tp_kv_heads,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_engine import (
|
||||
MoRIIOWrapper,
|
||||
)
|
||||
|
||||
|
||||
def test_remote_tp_rank_same_tp_maps_to_self():
|
||||
@@ -277,6 +282,9 @@ def test_read_completion_sends_structured_release_with_consumer_tp_size():
|
||||
self.lock = threading.Lock()
|
||||
self.sent = []
|
||||
|
||||
# Exercise the real batch verdict rather than restating it here.
|
||||
poll_transfer_batch = MoRIIOWrapper.poll_transfer_batch
|
||||
|
||||
def send_notify(
|
||||
self,
|
||||
transfer_id,
|
||||
@@ -292,13 +300,18 @@ def test_read_completion_sends_structured_release_with_consumer_tp_size():
|
||||
|
||||
worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker)
|
||||
worker.world_size = 8
|
||||
worker.moriio_config = SimpleNamespace(recv_abort_timeout=600.0)
|
||||
worker.moriio_wrapper = FakeWrapper()
|
||||
worker._recving_transfers = {"req": {"layer0": DoneStatus()}}
|
||||
# A layer maps to the list of reads posted for it (a KDA layer posts two).
|
||||
worker._recving_transfers = {"req": {"layer0": [DoneStatus(), DoneStatus()]}}
|
||||
worker._recving_transfers_callback_addr = {
|
||||
"req": ("127.0.0.1", "7000", "tx-release")
|
||||
}
|
||||
# Transfer-timeout reaping state consulted by _pop_done_transfers.
|
||||
worker._recving_transfers_start = {}
|
||||
# Load-error bookkeeping cleared alongside a completed transfer.
|
||||
worker._recving_local_blocks = {}
|
||||
worker._invalid_block_ids = set()
|
||||
|
||||
assert worker._pop_done_transfers() == {"tx-release"}
|
||||
assert worker.moriio_wrapper.sent == [
|
||||
@@ -314,6 +327,133 @@ def test_read_completion_sends_structured_release_with_consumer_tp_size():
|
||||
assert worker._recving_transfers_callback_addr == {}
|
||||
|
||||
|
||||
def test_aborted_read_without_local_blocks_only_releases_remote_blocks():
|
||||
class FakeWrapper:
|
||||
def __init__(self):
|
||||
self.sent = []
|
||||
|
||||
def send_notify(
|
||||
self,
|
||||
transfer_id,
|
||||
host,
|
||||
port,
|
||||
message_type=None,
|
||||
message_fields=None,
|
||||
):
|
||||
self.sent.append((transfer_id, host, port, message_type, message_fields))
|
||||
|
||||
def shutdown(self):
|
||||
pass
|
||||
|
||||
worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker)
|
||||
worker.mode = MoRIIOMode.READ
|
||||
worker.world_size = 8
|
||||
worker.moriio_wrapper = FakeWrapper()
|
||||
|
||||
worker._read_blocks(
|
||||
local_block_ids=[],
|
||||
remote_block_ids=[[10, 11], [90]],
|
||||
dst_engine_id="prefill",
|
||||
request_id="req-aborted",
|
||||
transfer_id="tx-aborted",
|
||||
remote_host="127.0.0.1",
|
||||
remote_notify_port=7000,
|
||||
remote_tp_size=8,
|
||||
remote_dp_rank=0,
|
||||
chosen_tp=2,
|
||||
)
|
||||
|
||||
assert worker.moriio_wrapper.sent == [
|
||||
(
|
||||
"tx-aborted",
|
||||
"127.0.0.1",
|
||||
"7002",
|
||||
"release",
|
||||
{"consumer_tp_size": 8},
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def test_read_completion_waits_for_every_posted_read():
|
||||
class Status:
|
||||
def __init__(self, done: bool):
|
||||
self.done = done
|
||||
|
||||
def Succeeded(self):
|
||||
return self.done
|
||||
|
||||
def Failed(self):
|
||||
return False
|
||||
|
||||
class FakeWrapper:
|
||||
def __init__(self):
|
||||
self.lock = threading.Lock()
|
||||
self.sent = []
|
||||
|
||||
poll_transfer_batch = MoRIIOWrapper.poll_transfer_batch
|
||||
|
||||
def send_notify(self, *args, **kwargs):
|
||||
self.sent.append((args, kwargs))
|
||||
|
||||
def shutdown(self):
|
||||
pass
|
||||
|
||||
worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker)
|
||||
worker.world_size = 8
|
||||
worker.moriio_config = SimpleNamespace(recv_abort_timeout=600.0)
|
||||
worker.moriio_wrapper = FakeWrapper()
|
||||
worker._recving_transfers = {
|
||||
"req": {"kda_layer": [Status(done=True), Status(done=False)]}
|
||||
}
|
||||
worker._recving_transfers_callback_addr = {
|
||||
"req": ("127.0.0.1", "7000", "tx-pending")
|
||||
}
|
||||
worker._recving_transfers_start = {"req": float("inf")}
|
||||
worker._recving_local_blocks = {"req": [1, 2]}
|
||||
worker._invalid_block_ids = set()
|
||||
|
||||
assert worker._pop_done_transfers() == set()
|
||||
assert "req" in worker._recving_transfers
|
||||
assert worker.moriio_wrapper.sent == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("has_mamba", "expected_invalid"),
|
||||
[(False, {1, 2}), (True, set())],
|
||||
)
|
||||
def test_failed_read_reports_blocks_only_without_hma(has_mamba, expected_invalid):
|
||||
worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker)
|
||||
worker._has_mamba = has_mamba
|
||||
worker._recving_local_blocks = {"req": [1, 2]}
|
||||
worker._invalid_block_ids = set()
|
||||
|
||||
worker._record_failed_recv("req")
|
||||
|
||||
assert worker.get_block_ids_with_load_errors() == expected_invalid
|
||||
|
||||
|
||||
def test_hybrid_step_barrier_fails_closed(monkeypatch):
|
||||
class FailingWrapper:
|
||||
def waiting_for_transfer_complete(self, _statuses):
|
||||
raise TransferError("failed")
|
||||
|
||||
def shutdown(self):
|
||||
pass
|
||||
|
||||
worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker)
|
||||
worker._has_mamba = True
|
||||
worker._reads_issued_this_step = [object()]
|
||||
worker._mamba_reads_this_step = [object()]
|
||||
worker.moriio_wrapper = FailingWrapper()
|
||||
monkeypatch.setattr(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector.get_forward_context",
|
||||
lambda: SimpleNamespace(cudagraph_runtime_mode=None),
|
||||
)
|
||||
|
||||
with pytest.raises(TransferError, match="failed"):
|
||||
worker._await_reads_issued_this_step()
|
||||
|
||||
|
||||
def test_requested_cudagraph_mode_is_never_overridden():
|
||||
# The configured cudagraph mode is always honored: the barrier fires when
|
||||
# the operator sets cudagraph_mode=PIECEWISE, and READ mode with full
|
||||
|
||||
@@ -0,0 +1,129 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Transfer-completion semantics of the MoRIIO wrapper.
|
||||
|
||||
Covers the two behaviours the READ barrier depends on:
|
||||
* poll_transfer_batch considers EVERY status, not just the newest one
|
||||
* waiting_for_transfer_complete blocks until terminal and raises on failure.
|
||||
"""
|
||||
|
||||
import importlib
|
||||
import importlib.util
|
||||
import threading
|
||||
import time
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
mori_available = importlib.util.find_spec("mori") is not None
|
||||
|
||||
if not (current_platform.is_rocm() and mori_available):
|
||||
pytest.skip(
|
||||
"MoRIIOs are only available on ROCm with mori package installed",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
moriio_common = importlib.import_module(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_common"
|
||||
)
|
||||
moriio_engine = importlib.import_module(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_engine"
|
||||
)
|
||||
|
||||
MoRIIOWrapper = moriio_engine.MoRIIOWrapper
|
||||
TransferBatchState = moriio_common.TransferBatchState
|
||||
TransferError = moriio_common.TransferError
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", [0, -1, float("inf"), float("nan")])
|
||||
def test_timeout_must_be_positive_and_finite(value):
|
||||
with pytest.raises(ValueError, match="must be finite and greater than zero"):
|
||||
moriio_common._positive_finite_timeout("test_timeout", value)
|
||||
|
||||
|
||||
def test_positive_finite_timeout_is_preserved():
|
||||
assert moriio_common._positive_finite_timeout("test_timeout", "1.5") == 1.5
|
||||
|
||||
|
||||
class FakeStatus:
|
||||
"""Mirrors the mori TransferStatus surface the wrapper uses."""
|
||||
|
||||
def __init__(self, state: str = "in_progress", message: str = ""):
|
||||
self.state = state
|
||||
self.message = message
|
||||
|
||||
def Succeeded(self) -> bool:
|
||||
return self.state == "success"
|
||||
|
||||
def Failed(self) -> bool:
|
||||
return self.state == "failed"
|
||||
|
||||
def Message(self) -> str:
|
||||
return self.message
|
||||
|
||||
def Code(self) -> str:
|
||||
return self.state
|
||||
|
||||
|
||||
def _ok():
|
||||
return FakeStatus("success")
|
||||
|
||||
|
||||
def _pending():
|
||||
return FakeStatus("in_progress")
|
||||
|
||||
|
||||
def _bad():
|
||||
return FakeStatus("failed", "SQ full")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"statuses,expected",
|
||||
[
|
||||
([], TransferBatchState.DONE),
|
||||
([_ok(), _ok()], TransferBatchState.DONE),
|
||||
# Each case below puts the interesting status BEFORE a success, which a
|
||||
# statuses[-1] check would have missed entirely.
|
||||
([_bad(), _ok()], TransferBatchState.FAILED),
|
||||
([_pending(), _ok()], TransferBatchState.PENDING),
|
||||
([_pending(), _bad(), _ok()], TransferBatchState.FAILED),
|
||||
],
|
||||
)
|
||||
def test_poll_transfer_batch_considers_every_status(statuses, expected):
|
||||
wrapper = MoRIIOWrapper(moriio_engine=object(), transfer_timeout=0.05)
|
||||
assert wrapper.poll_transfer_batch(statuses) is expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"statuses_factory,expected_message",
|
||||
[
|
||||
(lambda: [_ok(), _bad()], "SQ full"),
|
||||
(lambda: [_ok(), _pending()], "timed out"),
|
||||
],
|
||||
ids=["failed", "timeout"],
|
||||
)
|
||||
def test_wait_raises_with_detail(statuses_factory, expected_message):
|
||||
wrapper = MoRIIOWrapper(moriio_engine=object(), transfer_timeout=0.05)
|
||||
with pytest.raises(TransferError, match=expected_message):
|
||||
wrapper.waiting_for_transfer_complete(statuses_factory())
|
||||
|
||||
|
||||
def test_wait_returns_immediately_when_nothing_pending():
|
||||
wrapper = MoRIIOWrapper(moriio_engine=object(), transfer_timeout=0.05)
|
||||
wrapper.waiting_for_transfer_complete([])
|
||||
wrapper.waiting_for_transfer_complete([_ok(), _ok()])
|
||||
|
||||
|
||||
def test_wait_blocks_until_terminal():
|
||||
"""The barrier is only worth anything if it actually waits."""
|
||||
wrapper = MoRIIOWrapper(moriio_engine=object(), transfer_timeout=5.0)
|
||||
late = FakeStatus("in_progress")
|
||||
timer = threading.Timer(0.2, lambda: setattr(late, "state", "success"))
|
||||
timer.start()
|
||||
try:
|
||||
start = time.monotonic()
|
||||
wrapper.waiting_for_transfer_complete([late])
|
||||
assert time.monotonic() - start >= 0.15
|
||||
finally:
|
||||
timer.cancel()
|
||||
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import contextlib
|
||||
import math
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
@@ -45,6 +46,13 @@ TransferId = str
|
||||
TransferOffsetsKey = tuple[str, tuple[int, ...], tuple[int, ...], torch.dtype]
|
||||
|
||||
|
||||
def _positive_finite_timeout(name: str, value: Any) -> float:
|
||||
timeout = float(value)
|
||||
if not math.isfinite(timeout) or timeout <= 0:
|
||||
raise ValueError(f"{name} must be finite and greater than zero")
|
||||
return timeout
|
||||
|
||||
|
||||
class MoRIIOTransferAck(NamedTuple):
|
||||
transfer_id: TransferId
|
||||
consumer_tp_size: int = 1
|
||||
@@ -160,6 +168,20 @@ class MoRIIOMode(Enum):
|
||||
WRITE = "write"
|
||||
|
||||
|
||||
class TransferBatchState(Enum):
|
||||
"""Verdict for a group of transfer statuses that belong to one request.
|
||||
|
||||
A request's KV transfer is spread over one status per layer (two for a KDA
|
||||
layer), so a single status never decides the request: PENDING means at
|
||||
least one is still in flight and none has failed, FAILED means at least
|
||||
one failed, DONE means all succeeded.
|
||||
"""
|
||||
|
||||
DONE = "done"
|
||||
FAILED = "failed"
|
||||
PENDING = "pending"
|
||||
|
||||
|
||||
class MoRIIOError(Exception):
|
||||
"""Base exception for MoRIIO operations."""
|
||||
|
||||
@@ -260,6 +282,7 @@ class MoRIIOConfig:
|
||||
tp_size: int
|
||||
transfer_timeout: float
|
||||
defer_timeout: float
|
||||
recv_abort_timeout: float
|
||||
read_mode: bool = False
|
||||
qp_per_transfer: int = 1
|
||||
post_batch_size: int = -1
|
||||
@@ -284,6 +307,8 @@ class MoRIIOConfig:
|
||||
# raising TransferError (sec).
|
||||
# defer_timeout -> Timeout before a deferred send with no finished_sending
|
||||
# notification is reaped and its blocks force-freed (sec).
|
||||
# recv_abort_timeout -> Timeout before an in-flight recv whose RDMA
|
||||
# completion never arrived is aborted (sec).
|
||||
|
||||
# Knobs for RDMA transfers, ignored if on xgmi backend
|
||||
# qp_per_transfer -> Number of RDMA Queue Pairs per KV transfer.
|
||||
@@ -315,13 +340,29 @@ class MoRIIOConfig:
|
||||
"must be one of 'rdma' or 'xgmi'."
|
||||
)
|
||||
|
||||
transfer_timeout = float(
|
||||
transfer_timeout = _positive_finite_timeout(
|
||||
"kv_connector_extra_config.transfer_timeout",
|
||||
extra_config.get(
|
||||
"transfer_timeout", MoRIIOConstants.DEFAULT_TRANSFER_TIMEOUT
|
||||
)
|
||||
),
|
||||
)
|
||||
defer_timeout = float(
|
||||
extra_config.get("defer_timeout", MoRIIOConstants.DEFAULT_DEFER_TIMEOUT)
|
||||
defer_timeout = _positive_finite_timeout(
|
||||
"kv_connector_extra_config.defer_timeout",
|
||||
extra_config.get("defer_timeout", MoRIIOConstants.DEFAULT_DEFER_TIMEOUT),
|
||||
)
|
||||
legacy_recv_abort_timeout = os.environ.get("VLLM_MORIIO_TRANSFER_TIMEOUT_S")
|
||||
if legacy_recv_abort_timeout is not None:
|
||||
logger.warning_once(
|
||||
"The environment variable VLLM_MORIIO_TRANSFER_TIMEOUT_S is "
|
||||
"deprecated. Set 'recv_abort_timeout' inside "
|
||||
"kv_transfer_config.kv_connector_extra_config instead."
|
||||
)
|
||||
recv_abort_timeout = _positive_finite_timeout(
|
||||
"kv_connector_extra_config.recv_abort_timeout",
|
||||
extra_config.get(
|
||||
"recv_abort_timeout",
|
||||
legacy_recv_abort_timeout or MoRIIOConstants.DEFAULT_RECV_ABORT_TIMEOUT,
|
||||
),
|
||||
)
|
||||
|
||||
return cls(
|
||||
@@ -347,6 +388,7 @@ class MoRIIOConfig:
|
||||
backend=backend,
|
||||
transfer_timeout=transfer_timeout,
|
||||
defer_timeout=defer_timeout,
|
||||
recv_abort_timeout=recv_abort_timeout,
|
||||
)
|
||||
|
||||
|
||||
@@ -374,6 +416,10 @@ class MoRIIOConstants:
|
||||
# notification is reaped and its blocks force-freed.
|
||||
# Overridable via kv_connector_extra_config["defer_timeout"].
|
||||
DEFAULT_DEFER_TIMEOUT = 60.0
|
||||
# Timeout (seconds) before an in-flight recv whose RDMA completion was lost
|
||||
# is aborted, so the decode worker does not hang on it forever.
|
||||
# Overridable via kv_connector_extra_config["recv_abort_timeout"].
|
||||
DEFAULT_RECV_ABORT_TIMEOUT = 120.0
|
||||
|
||||
|
||||
# The router embeds both zmq_addresses in the request_id:
|
||||
@@ -567,9 +613,12 @@ class MoRIIOConnectorMetadata(KVConnectorMetadata):
|
||||
remote_handshake_port=int(remote_handshake_port),
|
||||
remote_notify_port=int(remote_notify_port),
|
||||
# Remote peer TP degree (used as remote_tp_size downstream). The
|
||||
# proxy advertises it under "remote_tp_size"; #46332 read "tp_size"
|
||||
# which is absent on WRITE producer requests -> defaulted to 1 ->
|
||||
# rank collapse. Read the right key; 0 == unknown (== homogeneous).
|
||||
# decode side must know the prefiller's TP degree to map producer
|
||||
# ranks onto the correct decode rank (and vice versa). The old plain
|
||||
# "tp_size" key (#46332) is absent on WRITE producer requests and
|
||||
# defaulted to 1, collapsing every producer rank onto decode rank 0
|
||||
# and corrupting the transfer; prefer "remote_tp_size". 0 == unknown
|
||||
# (treated as homogeneous downstream).
|
||||
tp_size=int(
|
||||
kv_transfer_params.get("remote_tp_size")
|
||||
or kv_transfer_params.get("tp_size")
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -29,6 +29,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_common import (
|
||||
MoRIIOError,
|
||||
MoRIIOTransferAck,
|
||||
RemoteAllocInfo,
|
||||
TransferBatchState,
|
||||
TransferError,
|
||||
TransferId,
|
||||
WriteTask,
|
||||
@@ -58,7 +59,6 @@ try:
|
||||
except ImportError:
|
||||
logger.error("MoRIIO is not available")
|
||||
|
||||
|
||||
"""Write task execution logic for MoRIIO connector."""
|
||||
|
||||
|
||||
@@ -395,9 +395,9 @@ class MoRIIOWriter:
|
||||
)
|
||||
request_info.transfer_offsets[key] = offsets
|
||||
|
||||
# Get session index
|
||||
layer_names = list(self.worker.layer_name_to_local_kv_cache_metadata.keys())
|
||||
sess_idx = layer_names.index(task.layer_name)
|
||||
# One session per registered region; attention layers map to a single
|
||||
# index (see MoRIIOConnectorWorker._region_session_indices).
|
||||
sess_idx = self.worker._region_session_indices(task.layer_name)[0]
|
||||
|
||||
local_off, remote_off, sizes = offsets
|
||||
|
||||
@@ -662,6 +662,25 @@ class MoRIIOWrapper:
|
||||
)
|
||||
return transfer_status
|
||||
|
||||
def poll_transfer_batch(self, transfer_statuses: list[Any]) -> TransferBatchState:
|
||||
"""Non-blocking verdict over every status of one request.
|
||||
|
||||
Judging a request by its newest status alone calls it done while an
|
||||
earlier layer is unfinished or already failed; failures do land on
|
||||
layers other than the last, because _post_read_with_backoff returns a
|
||||
failed status when the send queue never drains.
|
||||
|
||||
This stays a Python scan so it does not drive the backend's progress
|
||||
callback from the engine thread alongside MoRIIO's own poller.
|
||||
"""
|
||||
state = TransferBatchState.DONE
|
||||
for status in transfer_statuses:
|
||||
if status.Failed():
|
||||
return TransferBatchState.FAILED
|
||||
if not status.Succeeded():
|
||||
state = TransferBatchState.PENDING
|
||||
return state
|
||||
|
||||
def waiting_for_transfer_complete(self, transfer_statuses: list[Any] | None = None):
|
||||
if transfer_statuses is None:
|
||||
with self.lock:
|
||||
@@ -673,6 +692,15 @@ class MoRIIOWrapper:
|
||||
if not transfers_to_wait:
|
||||
return
|
||||
|
||||
self._poll_transfers_until_done(transfers_to_wait)
|
||||
|
||||
def _poll_transfers_until_done(self, transfers_to_wait: list[Any]) -> None:
|
||||
"""Fallback for mori builds without the batched wait.
|
||||
|
||||
Gives failure precedence over pending: once any status has failed the
|
||||
request is lost, so waiting out the remaining ones would only delay the
|
||||
barrier by up to transfer_timeout.
|
||||
"""
|
||||
timeout = self._transfer_timeout
|
||||
deadline = time.monotonic() + timeout
|
||||
remaining = list(transfers_to_wait)
|
||||
@@ -698,6 +726,8 @@ class MoRIIOWrapper:
|
||||
)
|
||||
else:
|
||||
still_waiting.append(status)
|
||||
if errors:
|
||||
break
|
||||
remaining = still_waiting
|
||||
if remaining:
|
||||
time.sleep(0.001)
|
||||
|
||||
@@ -2,14 +2,18 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Callable, Mapping
|
||||
from typing import NamedTuple
|
||||
from typing import NamedTuple, cast
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.ssm_conv_transfer_utils import (
|
||||
MambaConvSplitInfo,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
AttentionSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheSpec,
|
||||
MambaSpec,
|
||||
MLAAttentionSpec,
|
||||
SlidingWindowMLASpec,
|
||||
UniformTypeKVCacheSpecs,
|
||||
@@ -29,6 +33,85 @@ class LayerTransferGeometry(NamedTuple):
|
||||
split_kv_regions: bool
|
||||
|
||||
|
||||
class MambaTransferGeometry(NamedTuple):
|
||||
"""Geometry of a hybrid (mamba/KDA) layer's recurrent state.
|
||||
|
||||
A KDA layer's packed page is exposed as ``conv_state`` and ``ssm_state``
|
||||
views. Each is slot-strided, so a slot's bytes live at
|
||||
``slot * slot_stride * element_size`` and span ``slot_bytes``. The two
|
||||
tensors are registered as two separate MoRIIO regions per layer.
|
||||
"""
|
||||
|
||||
conv_slot_stride: int
|
||||
ssm_slot_stride: int
|
||||
ssm_slot_bytes: int
|
||||
conv_element_size: int
|
||||
ssm_element_size: int
|
||||
conv_region_len: int
|
||||
ssm_region_len: int
|
||||
|
||||
|
||||
def get_mamba_transfer_geometry(
|
||||
conv: torch.Tensor,
|
||||
ssm: torch.Tensor,
|
||||
) -> MambaTransferGeometry:
|
||||
"""Derive the slot-strided geometry of a KDA layer's conv + ssm state.
|
||||
|
||||
``conv``/``ssm`` are zero-copy views over the layer's packed page. Slot byte
|
||||
offset is ``slot * slot_stride * element_size`` and each
|
||||
slot spans ``subtensor[0].numel() * element_size`` bytes; the whole tensor
|
||||
is registered as one region (``*_region_len`` bytes).
|
||||
"""
|
||||
if conv.ndim == 0 or ssm.ndim == 0 or conv.shape[0] == 0 or ssm.shape[0] == 0:
|
||||
raise ValueError("KDA conv/ssm caches must contain at least one slot")
|
||||
if conv.shape[0] != ssm.shape[0]:
|
||||
raise ValueError(
|
||||
f"KDA conv/ssm slot count mismatch: {conv.shape[0]} != {ssm.shape[0]}"
|
||||
)
|
||||
conv_slot_bytes = conv[0].numel() * conv.element_size()
|
||||
ssm_slot_bytes = ssm[0].numel() * ssm.element_size()
|
||||
conv_available_bytes = (
|
||||
conv.untyped_storage().nbytes() - conv.storage_offset() * conv.element_size()
|
||||
)
|
||||
ssm_available_bytes = (
|
||||
ssm.untyped_storage().nbytes() - ssm.storage_offset() * ssm.element_size()
|
||||
)
|
||||
if conv.shape[0]:
|
||||
conv_required_bytes = (conv.shape[0] - 1) * conv.stride(
|
||||
0
|
||||
) * conv.element_size() + conv_slot_bytes
|
||||
ssm_required_bytes = (ssm.shape[0] - 1) * ssm.stride(
|
||||
0
|
||||
) * ssm.element_size() + ssm_slot_bytes
|
||||
if conv_required_bytes > conv_available_bytes:
|
||||
raise ValueError("KDA conv slots exceed the registered storage extent")
|
||||
if ssm_required_bytes > ssm_available_bytes:
|
||||
raise ValueError("KDA ssm slots exceed the registered storage extent")
|
||||
|
||||
return MambaTransferGeometry(
|
||||
conv_slot_stride=conv.stride(0),
|
||||
ssm_slot_stride=ssm.stride(0),
|
||||
ssm_slot_bytes=ssm_slot_bytes,
|
||||
conv_element_size=conv.element_size(),
|
||||
ssm_element_size=ssm.element_size(),
|
||||
# Registered region must span the full slot-strided extent: stride(0)
|
||||
# may exceed the per-slot element count, so numel() would under-size the
|
||||
# region and truncate the highest-slot transfers. shape[0]*stride(0)
|
||||
# covers every byte a ``slot * slot_stride * elem`` offset can address.
|
||||
# Cap at the bytes remaining from each view's storage offset. A packed
|
||||
# conv+ssm page places ssm at a non-zero intra-buffer offset,
|
||||
# so do not register past the end of the shared backing allocation.
|
||||
conv_region_len=min(
|
||||
conv.shape[0] * conv.stride(0) * conv.element_size(),
|
||||
conv_available_bytes,
|
||||
),
|
||||
ssm_region_len=min(
|
||||
ssm.shape[0] * ssm.stride(0) * ssm.element_size(),
|
||||
ssm_available_bytes,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def build_layer_to_spec(kv_cache_config: KVCacheConfig) -> dict[str, KVCacheSpec]:
|
||||
layer_to_spec: dict[str, KVCacheSpec] = {}
|
||||
for group in kv_cache_config.kv_cache_groups:
|
||||
@@ -99,10 +182,10 @@ def get_layer_transfer_geometry(
|
||||
layer_to_spec: Mapping[str, KVCacheSpec],
|
||||
remote_num_blocks: int | None = None,
|
||||
) -> LayerTransferGeometry:
|
||||
spec = layer_to_spec[layer_name]
|
||||
shape = kv_cache.shape
|
||||
stride = kv_cache.stride()
|
||||
element_size = kv_cache.element_size()
|
||||
spec = layer_to_spec[layer_name]
|
||||
is_mla_cache = is_mla_cache_layer(layer_to_spec, layer_name)
|
||||
|
||||
if not is_mla_cache and len(shape) == 5 and shape[0] == 2:
|
||||
@@ -238,12 +321,60 @@ def get_layer_transfer_geometry(
|
||||
)
|
||||
|
||||
|
||||
def kda_conv_ssm(
|
||||
kv_cache: torch.Tensor,
|
||||
spec: MambaSpec,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Return zero-copy conv and SSM views over a packed two-state page."""
|
||||
from math import prod
|
||||
|
||||
from vllm.utils.torch_utils import get_dtype_size
|
||||
|
||||
if len(spec.shapes) != 2 or len(spec.dtypes) != 2:
|
||||
raise ValueError(
|
||||
"MoRIIO KDA transfer requires exactly two states (conv and SSM)"
|
||||
)
|
||||
if kv_cache.element_size() != 1:
|
||||
raise ValueError("MoRIIO KDA packed cache must use a byte-sized dtype")
|
||||
if kv_cache.ndim != 4 or kv_cache.shape[1:3] != (1, 1) or kv_cache.stride(3) != 1:
|
||||
raise ValueError(
|
||||
"MoRIIO KDA packed cache must have shape [B, 1, 1, C] with "
|
||||
"a contiguous byte dimension"
|
||||
)
|
||||
# Mirror MambaBase.bind_kv_cache: blocks may be strided within a shared
|
||||
# multi-layer allocation, while every individual byte page is contiguous.
|
||||
pages = kv_cache.squeeze(dim=(1, 2))
|
||||
states: list[torch.Tensor] = []
|
||||
offset = 0
|
||||
for shp, dt in zip(spec.shapes, spec.dtypes, strict=True):
|
||||
nbytes = prod(shp) * get_dtype_size(dt)
|
||||
if offset + nbytes > pages.shape[1]:
|
||||
raise ValueError(
|
||||
"MoRIIO KDA state layout exceeds the packed cache page size"
|
||||
)
|
||||
state = pages[:, offset : offset + nbytes].view(dt)
|
||||
states.append(state.view(-1, *shp))
|
||||
offset += nbytes
|
||||
return states[0], states[1]
|
||||
|
||||
|
||||
def iter_layer_registration_regions(
|
||||
layer_name: str,
|
||||
kv_cache: torch.Tensor,
|
||||
layer_to_spec: Mapping[str, KVCacheSpec],
|
||||
) -> list[tuple[torch.Tensor, int]]:
|
||||
geometry = get_layer_transfer_geometry(layer_name, kv_cache, layer_to_spec)
|
||||
spec = layer_to_spec[layer_name]
|
||||
if isinstance(spec, MambaSpec):
|
||||
# KDA layer: register the conv and ssm tensors as two whole-tensor
|
||||
# regions (the conv 3-subprojection split is expressed as transfer
|
||||
# offset triples, not extra registrations).
|
||||
conv, ssm = kda_conv_ssm(kv_cache, spec)
|
||||
geom = get_mamba_transfer_geometry(conv, ssm)
|
||||
return [(conv, geom.conv_region_len), (ssm, geom.ssm_region_len)]
|
||||
geometry = cast(
|
||||
LayerTransferGeometry,
|
||||
get_layer_transfer_geometry(layer_name, kv_cache, layer_to_spec),
|
||||
)
|
||||
region_len = geometry.num_blocks * geometry.regions_per_block * geometry.block_len
|
||||
if geometry.regions_per_block == 1:
|
||||
# With padded or interleaved pages the block stride exceeds the
|
||||
@@ -311,9 +442,36 @@ def compute_block_transfer_offsets(
|
||||
"local_block_ids longer than remote_block_ids: "
|
||||
f"{len(local_block_ids)} > {len(remote_block_ids)}"
|
||||
)
|
||||
geometry = get_layer_transfer_geometry(
|
||||
layer_name, kv_cache, layer_to_spec, remote_num_blocks
|
||||
geometry = cast(
|
||||
LayerTransferGeometry,
|
||||
get_layer_transfer_geometry(
|
||||
layer_name, kv_cache, layer_to_spec, remote_num_blocks
|
||||
),
|
||||
)
|
||||
invalid_local = next(
|
||||
(
|
||||
block_id
|
||||
for block_id in local_block_ids
|
||||
if not 0 <= block_id < geometry.num_blocks
|
||||
),
|
||||
None,
|
||||
)
|
||||
if invalid_local is not None:
|
||||
raise ValueError(
|
||||
f"local block id {invalid_local} is outside [0, {geometry.num_blocks})"
|
||||
)
|
||||
invalid_remote = next(
|
||||
(
|
||||
block_id
|
||||
for block_id in remote_block_ids
|
||||
if not 0 <= block_id < remote_num_blocks
|
||||
),
|
||||
None,
|
||||
)
|
||||
if invalid_remote is not None:
|
||||
raise ValueError(
|
||||
f"remote block id {invalid_remote} is outside [0, {remote_num_blocks})"
|
||||
)
|
||||
element_size = kv_cache.element_size()
|
||||
transfer_size_byte = geometry.block_len
|
||||
per_block = geometry.transfers_per_block
|
||||
@@ -339,3 +497,125 @@ def compute_block_transfer_offsets(
|
||||
w += 1
|
||||
|
||||
return merge_fn(offset_local, offset_remote, sizes)
|
||||
|
||||
|
||||
class MambaOffsetTemplate(NamedTuple):
|
||||
"""Slot-independent conv+ssm offset decomposition for a KDA layer.
|
||||
|
||||
Homogeneous-TP geometry (conv/ssm slot strides, conv sub-projection
|
||||
offsets, ssm per-slot size) is identical across requests and across the
|
||||
homogeneous GDN layers, so it is computed once and reused; only the
|
||||
per-request slot bases vary (see ``apply_mamba_offset_template``).
|
||||
"""
|
||||
|
||||
num_slots: int
|
||||
conv_slot_stride_bytes: int
|
||||
ssm_slot_stride_bytes: int
|
||||
ssm_read_bytes: int
|
||||
conv_subprojs: tuple[tuple[int, int, int], ...]
|
||||
|
||||
|
||||
def build_mamba_offset_template(
|
||||
conv: torch.Tensor,
|
||||
ssm: torch.Tensor,
|
||||
split_info: MambaConvSplitInfo,
|
||||
tp_ratio: int,
|
||||
) -> MambaOffsetTemplate:
|
||||
"""Precompute the slot-independent conv+ssm offset decomposition.
|
||||
|
||||
Only homogeneous TP (``tp_ratio == 1``) is supported; heterogeneous TP is
|
||||
gated with ``NotImplementedError`` (see the design doc). The returned
|
||||
template is a pure function of the layer geometry and ``split_info``, so it
|
||||
can be cached and reused across requests without recomputation.
|
||||
"""
|
||||
if tp_ratio != 1:
|
||||
# Heterogeneous-TP recurrent-state transfer needs the remote page's
|
||||
# slot stride (which differs from the local page) and, for
|
||||
# P_TP > D_TP, a multi-rank gather of conv/ssm slices. Both are
|
||||
# follow-ups. The mamba path bypasses the attention KV-head validator,
|
||||
# so fail loudly here instead of silently transferring wrong bytes.
|
||||
raise NotImplementedError(
|
||||
"heterogeneous-TP mamba/KDA transfer is not supported yet "
|
||||
f"(tp_ratio={tp_ratio}); use equal prefill/decode TP for "
|
||||
"hybrid models"
|
||||
)
|
||||
geom = get_mamba_transfer_geometry(conv, ssm)
|
||||
conv_local_offsets = split_info.local_conv_offsets
|
||||
conv_remote_offsets = split_info.remote_conv_offsets(0, 1)
|
||||
ssm_read_bytes = geom.ssm_slot_bytes
|
||||
conv_subprojs = tuple(
|
||||
(loff, roff, rsz)
|
||||
for (loff, _lsz), (roff, rsz) in zip(conv_local_offsets, conv_remote_offsets)
|
||||
)
|
||||
return MambaOffsetTemplate(
|
||||
num_slots=conv.shape[0],
|
||||
conv_slot_stride_bytes=geom.conv_slot_stride * geom.conv_element_size,
|
||||
ssm_slot_stride_bytes=geom.ssm_slot_stride * geom.ssm_element_size,
|
||||
ssm_read_bytes=ssm_read_bytes,
|
||||
conv_subprojs=conv_subprojs,
|
||||
)
|
||||
|
||||
|
||||
def apply_mamba_offset_template(
|
||||
template: MambaOffsetTemplate,
|
||||
local_slots: list[int],
|
||||
remote_slots: list[int],
|
||||
) -> tuple[list[int], list[int], list[int]]:
|
||||
"""Apply per-request slot bases to a cached ``MambaOffsetTemplate``.
|
||||
|
||||
Conv sub-projection entries (all slots) come first, followed by one ssm
|
||||
entry per slot.
|
||||
"""
|
||||
if len(local_slots) != len(remote_slots):
|
||||
raise ValueError(
|
||||
"local and remote mamba slot counts must match: "
|
||||
f"{len(local_slots)} != {len(remote_slots)}"
|
||||
)
|
||||
invalid_local = next(
|
||||
(slot for slot in local_slots if not 0 <= slot < template.num_slots), None
|
||||
)
|
||||
if invalid_local is not None:
|
||||
raise ValueError(
|
||||
f"local mamba slot {invalid_local} is outside [0, {template.num_slots})"
|
||||
)
|
||||
invalid_remote = next((slot for slot in remote_slots if slot < 0), None)
|
||||
if invalid_remote is not None:
|
||||
raise ValueError(f"remote mamba slot {invalid_remote} must be non-negative")
|
||||
# The peer's KDA slot count is not part of MoRIIOAgentMetadata yet. Do not
|
||||
# compare remote slot IDs against the local pool size: two otherwise
|
||||
# compatible engines may reserve different numbers of slots.
|
||||
local_offs: list[int] = []
|
||||
remote_offs: list[int] = []
|
||||
sizes: list[int] = []
|
||||
|
||||
# Conv sub-projections (region 0), all slots first.
|
||||
conv_slot_stride_bytes = template.conv_slot_stride_bytes
|
||||
for ls, rs in zip(local_slots, remote_slots):
|
||||
lbase = ls * conv_slot_stride_bytes
|
||||
rbase = rs * conv_slot_stride_bytes
|
||||
for loff, roff, rsz in template.conv_subprojs:
|
||||
local_offs.append(lbase + loff)
|
||||
remote_offs.append(rbase + roff)
|
||||
sizes.append(rsz)
|
||||
|
||||
# SSM temporal state (region 1), one entry per slot.
|
||||
ssm_slot_stride_bytes = template.ssm_slot_stride_bytes
|
||||
ssm_read_bytes = template.ssm_read_bytes
|
||||
for ls, rs in zip(local_slots, remote_slots):
|
||||
local_offs.append(ls * ssm_slot_stride_bytes)
|
||||
remote_offs.append(rs * ssm_slot_stride_bytes)
|
||||
sizes.append(ssm_read_bytes)
|
||||
|
||||
return local_offs, remote_offs, sizes
|
||||
|
||||
|
||||
def compute_mamba_conv_split_count(
|
||||
local_slots: list[int],
|
||||
split_info: MambaConvSplitInfo,
|
||||
) -> int:
|
||||
"""Number of leading conv entries in a KDA layer transfer plan.
|
||||
|
||||
Entries ``[:count]`` are conv sub-projections (conv region/session);
|
||||
entries ``[count:]`` are the per-slot ssm state (ssm region/session).
|
||||
"""
|
||||
return len(local_slots) * len(split_info.local_conv_offsets)
|
||||
|
||||
Reference in New Issue
Block a user