[KVConnector][MoRIIO] Transfer hybrid mamba/KDA recurrent state in READ mode (#51052)

Signed-off-by: Yichao Zhu <Yichao.Zhu@amd.com>
This commit is contained in:
YukioZzz
2026-09-20 17:30:34 -04:00
committed by GitHub
parent 01f1f58f10
commit 9679173788
11 changed files with 2747 additions and 185 deletions
@@ -367,6 +367,8 @@ async def handle_request(api: str, request: Request):
"remote_block_ids"
]
req_data["kv_transfer_params"]["transfer_id"] = prefill_kv["transfer_id"]
if prefill_kv.get("tp_size") is not None:
req_data["kv_transfer_params"]["tp_size"] = prefill_kv["tp_size"]
req_data["kv_transfer_params"]["remote_dp_size"] = prefill_instance_endpoint[
"dp_size"
@@ -775,6 +775,17 @@ def test_is_hma_required(swa_enabled, disable_hma, expected_is_hma):
assert scheduler.get_exchange_clipped_blocks(blocks) == blocks
def test_transfer_disabled_hybrid_group_is_ignored():
config = _make_hybrid_kv_cache_config()
config.kv_cache_groups[1].enable_kv_transfer = False
scheduler = _read_scheduler(config)
assert scheduler._is_hma_required is False
assert scheduler.get_exchange_clipped_blocks([[1, 2, 3], [20, 21, 22]]) == [
[1, 2, 3]
]
def test_non_sliding_window_hybrid_is_rejected():
"""A hybrid group that is not sliding-window (e.g. chunked-local
attention) must fail closed rather than be silently mistransferred."""
@@ -0,0 +1,809 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Pure-CPU unit tests for the MoRIIO hybrid (attention + mamba/KDA) scheduler
and the KDA offset-template cache.
These exercise the highest-risk, otherwise-untested scheduler logic without a
GPU or the ``mori`` runtime: per-group block-id splitting, the READ/WRITE
``N-1`` token accounting, P-side prompt truncation, and offset-template cache
wiring.
Like ``test_moriio_kv_layout.py`` the whole module is skipped unless it is
running on ROCm with ``mori`` installed (importing the connector pulls in
``mori``). The authoritative run happens on the MIA recipe image.
"""
import importlib
import importlib.util
from types import SimpleNamespace
import pytest
import torch
from vllm.platforms import current_platform
mori_available = importlib.util.find_spec("mori") is not None
if not (current_platform.is_rocm() and mori_available):
pytest.skip(
"MoRIIOs are only available on ROCm with mori package installed",
allow_module_level=True,
)
moriio_connector = importlib.import_module(
"vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector"
)
moriio_layout = importlib.import_module(
"vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_layout"
)
ssm_conv_transfer_utils = importlib.import_module(
"vllm.distributed.kv_transfer.kv_connector.v1.ssm_conv_transfer_utils"
)
MambaConvSplitInfo = ssm_conv_transfer_utils.MambaConvSplitInfo
MoRIIOMode = moriio_connector.MoRIIOMode
moriio_common = importlib.import_module(
"vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_common"
)
class _FakeScheduler(moriio_connector.MoRIIOConnectorScheduler): # type: ignore[name-defined]
"""Constructs a scheduler without the heavy real ``__init__``; only the
attributes read by the method under test are set."""
def __init__(self, **attrs):
self._mamba_group_ids: list[int] = []
self._attn_group_ids: list[int] = [0]
self._ssm_state_slots_are_positional = False
self._is_hma_required = False
self.kv_cache_config = SimpleNamespace(
kv_cache_groups=[None, None],
select_transfer_block_ids=lambda block_ids: tuple(block_ids),
)
for k, v in attrs.items():
setattr(self, k, v)
class _FakeConnector(moriio_connector.MoRIIOConnector): # type: ignore[name-defined]
def __init__(self, connector_scheduler):
self.connector_scheduler = connector_scheduler
class _FakeWorker(moriio_connector.MoRIIOConnectorWorker): # type: ignore[name-defined]
def __init__(self, **attrs):
for k, v in attrs.items():
setattr(self, k, v)
class _FakeBlocks:
def __init__(self, all_groups):
self._all_groups = all_groups
def get_block_ids(self):
return self._all_groups
def _slot_strided(num_slots, per_slot_shape, slot_stride, dtype=torch.bfloat16):
inner = 1
for d in per_slot_shape:
inner *= d
assert slot_stride >= inner
backing = torch.zeros(num_slots * slot_stride, dtype=dtype)
inner_strides: list[int] = []
acc = 1
for d in reversed(per_slot_shape):
inner_strides.insert(0, acc)
acc *= d
return backing.as_strided(
(num_slots, *per_slot_shape), (slot_stride, *inner_strides)
)
def _gdn_split_info(conv_rows=3, key_dim=4, value_dim=8, dtype_size=2):
conv_dim = 2 * key_dim + value_dim
conv_state_bytes = conv_dim * conv_rows * dtype_size
ssm_state_bytes = 64
return MambaConvSplitInfo(
conv_rows=conv_rows,
local_proj_dims=(key_dim, key_dim, value_dim),
conv_dtype_size=dtype_size,
ssm_sizes=(conv_state_bytes, ssm_state_bytes),
)
def _mamba_spec(num_states: int = 2):
shapes = tuple((1, 1) for _ in range(num_states))
return moriio_connector.MambaSpec(
block_size=16,
shapes=shapes,
dtypes=(torch.float32,) * num_states,
mamba_cache_mode="all",
)
def _gate_vllm_config(*, read_mode: bool = True, speculative_config=None):
return SimpleNamespace(
kv_transfer_config=SimpleNamespace(
kv_connector_extra_config={"read_mode": read_mode}
),
speculative_config=speculative_config,
)
# --------------------------------------------------------------------------
# split_block_groups
# --------------------------------------------------------------------------
def test_split_block_groups_separates_attention_and_mamba():
sched = _FakeScheduler(_has_mamba=True, _attn_group_ids=[0], _mamba_group_ids=[1])
block_ids = ([1, 2], [99])
attn, mamba = sched.split_block_groups(block_ids)
assert attn == [1, 2]
assert mamba == [[99]]
def test_split_block_groups_ignores_transfer_disabled_group():
mamba_spec = moriio_connector.MambaSpec(
block_size=16,
shapes=((1, 1),),
dtypes=(torch.float32,),
mamba_cache_mode="all",
)
config = SimpleNamespace(
kv_cache_groups=[
SimpleNamespace(enable_kv_transfer=True, kv_cache_spec=object()),
SimpleNamespace(enable_kv_transfer=False, kv_cache_spec=object()),
SimpleNamespace(enable_kv_transfer=True, kv_cache_spec=mamba_spec),
],
)
config.transfer_groups = tuple(
group for group in config.kv_cache_groups if group.enable_kv_transfer
)
config.select_transfer_block_ids = lambda block_ids: tuple(
block_ids[group_id]
for group_id, group in enumerate(config.kv_cache_groups)
if group.enable_kv_transfer
)
attn_groups, mamba_groups = moriio_connector._split_kv_cache_group_kinds(config)
assert attn_groups == [0]
assert mamba_groups == [1]
sched = _FakeScheduler(
_has_mamba=True,
_attn_group_ids=attn_groups,
_mamba_group_ids=mamba_groups,
kv_cache_config=config,
)
assert sched.split_block_groups(([1, 2], [70, 71], [99])) == (
[1, 2],
[[99]],
)
def test_exchange_blocks_ignore_transfer_disabled_group():
config = SimpleNamespace(
kv_cache_groups=[None, None, None],
select_transfer_block_ids=lambda block_ids: (block_ids[0], block_ids[2]),
)
sched = _FakeScheduler(
kv_cache_config=config,
_is_hma_required=True,
blocks_per_sw=[0, 2],
)
assert sched.get_exchange_clipped_blocks(([1, 2], [70, 71], [90, 91, 92])) == [
[1, 2],
[91, 92],
]
def test_scheduler_rejects_multiple_attention_groups_with_mamba():
config = SimpleNamespace(
kv_cache_groups=[
SimpleNamespace(enable_kv_transfer=True, kv_cache_spec=object()),
SimpleNamespace(enable_kv_transfer=True, kv_cache_spec=object()),
SimpleNamespace(enable_kv_transfer=True, kv_cache_spec=_mamba_spec()),
],
)
config.transfer_groups = tuple(config.kv_cache_groups)
with pytest.raises(moriio_common.MoRIIOError, match="exactly one transferable"):
moriio_connector.MoRIIOConnectorScheduler(_gate_vllm_config(), "engine", config)
def test_split_block_groups_preserves_multiple_mamba_groups():
sched = _FakeScheduler(
_has_mamba=True,
_attn_group_ids=[0],
_mamba_group_ids=[1, 2],
)
assert sched.split_block_groups(([1, 2], [40], [70])) == (
[1, 2],
[[40], [70]],
)
def test_scheduler_rejects_incompatible_mamba_groups():
incompatible = _mamba_spec()
object.__setattr__(incompatible, "page_size_padded", 4096)
config = SimpleNamespace(
kv_cache_groups=[
SimpleNamespace(enable_kv_transfer=True, kv_cache_spec=object()),
SimpleNamespace(enable_kv_transfer=True, kv_cache_spec=_mamba_spec()),
SimpleNamespace(enable_kv_transfer=True, kv_cache_spec=incompatible),
]
)
config.transfer_groups = tuple(config.kv_cache_groups)
with pytest.raises(moriio_common.MoRIIOError, match="same cache spec"):
moriio_connector.MoRIIOConnectorScheduler(_gate_vllm_config(), "engine", config)
def test_scheduler_rejects_mamba_group_without_two_states():
config = SimpleNamespace(
kv_cache_groups=[
SimpleNamespace(enable_kv_transfer=True, kv_cache_spec=object()),
SimpleNamespace(enable_kv_transfer=True, kv_cache_spec=_mamba_spec(1)),
],
)
config.transfer_groups = tuple(config.kv_cache_groups)
with pytest.raises(moriio_common.MoRIIOError, match="exactly two Mamba states"):
moriio_connector.MoRIIOConnectorScheduler(_gate_vllm_config(), "engine", config)
def test_scheduler_rejects_speculative_hybrid_read():
config = SimpleNamespace(
kv_cache_groups=[
SimpleNamespace(enable_kv_transfer=True, kv_cache_spec=object()),
SimpleNamespace(enable_kv_transfer=True, kv_cache_spec=_mamba_spec()),
],
)
config.transfer_groups = tuple(config.kv_cache_groups)
with pytest.raises(moriio_common.MoRIIOError, match="speculative decoding"):
moriio_connector.MoRIIOConnectorScheduler(
_gate_vllm_config(speculative_config=object()), "engine", config
)
def test_scheduler_rejects_hybrid_write():
config = SimpleNamespace(
kv_cache_groups=[
SimpleNamespace(enable_kv_transfer=True, kv_cache_spec=object()),
SimpleNamespace(enable_kv_transfer=True, kv_cache_spec=_mamba_spec()),
],
)
config.transfer_groups = tuple(config.kv_cache_groups)
with pytest.raises(moriio_common.MoRIIOError, match="READ mode only"):
moriio_connector.MoRIIOConnectorScheduler(
_gate_vllm_config(read_mode=False), "engine", config
)
def test_split_block_groups_rejects_non_hybrid_use():
sched = _FakeScheduler(_has_mamba=False)
with pytest.raises(moriio_common.MoRIIOError, match="non-hybrid"):
sched.split_block_groups(([5, 6, 7],))
def test_split_block_groups_accepts_empty_abort_payload():
sched = _FakeScheduler(_has_mamba=True)
assert sched.split_block_groups(()) == ([], [])
def test_split_block_groups_keeps_positional_slots_in_all_mode():
# mamba_cache_mode="all" keeps a state per block position.
sched = _FakeScheduler(
_has_mamba=True,
_attn_group_ids=[0],
_mamba_group_ids=[1],
_ssm_state_slots_are_positional=True,
)
attn, mamba = sched.split_block_groups(([1], [40, 41, 42]))
assert attn == [1]
assert mamba == [[40, 41, 42]]
def test_split_block_groups_keeps_only_running_state_outside_all_mode():
sched = _FakeScheduler(
_has_mamba=True,
_attn_group_ids=[0],
_mamba_group_ids=[1],
)
assert sched.split_block_groups(([1], [40, 41])) == ([1], [[41]])
# --------------------------------------------------------------------------
# request_finished_all_groups
# --------------------------------------------------------------------------
def test_request_finished_all_groups_carries_attn_and_mamba_in_one_field():
seen = {}
def _fake_request_finished(request, attn_block_ids, mamba_block_groups):
seen["attn"] = list(attn_block_ids)
seen["mamba"] = [list(group) for group in mamba_block_groups]
return True, {
"do_remote_prefill": True,
"remote_block_ids": [attn_block_ids, *mamba_block_groups],
}
sched = _FakeScheduler(_has_mamba=True, _attn_group_ids=[0], _mamba_group_ids=[1])
sched.request_finished = _fake_request_finished
conn = _FakeConnector(sched)
request = SimpleNamespace(request_id="r0")
block_ids = ([10, 11], [42])
delay_free, params = conn.request_finished_all_groups(request, block_ids)
assert delay_free is True
# Attention ids drive request_finished; the mamba slot rides the SAME
# remote_block_ids channel as [attn, *mamba_groups], so the proxy/router
# needs no KDA-specific field.
assert seen["attn"] == [10, 11]
assert seen["mamba"] == [[42]]
assert params["remote_block_ids"] == [[10, 11], [42]]
assert "remote_mamba_block_ids" not in params
def test_request_finished_all_groups_pure_attention_stays_grouped():
def _fake_request_finished(request, block_ids):
return True, {"remote_block_ids": block_ids}
sched = _FakeScheduler(_has_mamba=False, _mamba_group_ids=[])
sched.request_finished = _fake_request_finished
conn = _FakeConnector(sched)
delay_free, params = conn.request_finished_all_groups(
SimpleNamespace(request_id="r1"), ([7, 8],)
)
# Main's HMA contract keeps pure-attention block ids grouped.
assert params["remote_block_ids"] == ([7, 8],)
assert "remote_mamba_block_ids" not in params
def test_connector_supports_divergent_local_hybrid_hits():
connector = _FakeConnector(None)
connector._transfers_mamba_state = True
connector.mode = MoRIIOMode.READ
assert connector.supports_divergent_local_hybrid_hits is True
connector.mode = MoRIIOMode.WRITE
assert connector.supports_divergent_local_hybrid_hits is False
connector.mode = MoRIIOMode.READ
connector._transfers_mamba_state = False
assert connector.supports_divergent_local_hybrid_hits is False
def _make_read_scheduler():
return _FakeScheduler(
is_producer=False,
mode=MoRIIOMode.READ,
_has_mamba=True,
_attn_group_ids=[0],
_mamba_group_ids=[1],
_ssm_state_slots_are_positional=True,
request_id_to_transfer_id={},
transfer_id_to_request_id={},
_reqs_need_recv={},
_req_kv_params={},
_max_decode_tail_blocks=1,
)
def _make_read_request(remote_block_ids):
return SimpleNamespace(
request_id="req",
kv_transfer_params={
"do_remote_prefill": True,
"transfer_id": "tx",
"remote_engine_id": "prefill",
"remote_block_ids": remote_block_ids,
},
)
def test_update_state_drops_decode_recompute_tail_block():
sched = _make_read_scheduler()
request = _make_read_request([[10, 11], [90, 91]])
blocks = _FakeBlocks(
all_groups=([100, 101, 102], [200, 201, 202]),
)
sched.update_state_after_alloc(request, blocks, num_external_tokens=256)
assert sched._reqs_need_recv["req"][1] == [[100, 101], [200, 201]]
assert sched._req_kv_params["req"]["remote_block_ids"] == [
[10, 11],
[90, 91],
]
def test_update_state_pairs_shorter_local_blocks_with_remote_suffix():
sched = _make_read_scheduler()
request = _make_read_request([[10, 11, 12], [92]])
blocks = _FakeBlocks(
all_groups=([102], [202]),
)
sched.update_state_after_alloc(request, blocks, num_external_tokens=64)
assert sched._reqs_need_recv["req"][1] == [[102], [202]]
assert sched._req_kv_params["req"]["remote_block_ids"] == [[12], [92]]
def test_update_state_full_attention_hit_still_carries_mamba_state():
sched = _make_read_scheduler()
request = _make_read_request([[10], [90]])
sched.update_state_after_alloc(
request,
_FakeBlocks(all_groups=([100], [200])),
num_external_tokens=0,
)
assert sched._reqs_need_recv["req"][1] == [[], [200]]
def test_update_state_preserves_each_mamba_group():
sched = _make_read_scheduler()
sched._mamba_group_ids = [1, 2]
request = _make_read_request([[10], [90], [80]])
sched.update_state_after_alloc(
request,
_FakeBlocks(all_groups=([100], [200], [300])),
num_external_tokens=64,
)
assert sched._reqs_need_recv["req"][1] == [[100], [200], [300]]
assert sched._req_kv_params["req"]["remote_block_ids"] == [
[10],
[90],
[80],
]
def test_align_read_blocks_allows_configured_lookahead_tail():
assert moriio_connector.MoRIIOConnectorScheduler._align_read_blocks(
[100, 101, 102], [10], max_decode_tail_blocks=2
) == ([100], [10])
with pytest.raises(ValueError, match="allowed local tail"):
moriio_connector.MoRIIOConnectorScheduler._align_read_blocks(
[100, 101, 102], [10], max_decode_tail_blocks=1
)
def test_update_state_aligns_attention_only_decode_tail():
sched = _make_read_scheduler()
sched._has_mamba = False
sched._mamba_group_ids = []
request = _make_read_request([[10, 11]])
sched.update_state_after_alloc(
request,
_FakeBlocks(all_groups=([100, 101, 102],)),
num_external_tokens=256,
)
assert sched._reqs_need_recv["req"][1] == [[100, 101]]
assert sched._req_kv_params["req"]["remote_block_ids"] == [[10, 11]]
def test_read_producer_does_not_queue_write_only_save_state():
sched = _FakeScheduler(
is_producer=True,
mode=MoRIIOMode.READ,
_has_mamba=True,
_attn_group_ids=[0],
_mamba_group_ids=[1],
request_id_to_transfer_id={},
transfer_id_to_request_id={},
_reqs_need_save={},
_req_kv_params={},
)
request = SimpleNamespace(
request_id="req",
kv_transfer_params={"do_remote_decode": True, "transfer_id": "tx"},
)
sched.update_state_after_alloc(
request,
_FakeBlocks(all_groups=([100, 101], [200])),
num_external_tokens=0,
)
assert sched._reqs_need_save == {}
assert sched._req_kv_params == {}
def test_write_producer_queues_grouped_attention_blocks():
sched = _FakeScheduler(
is_producer=True,
mode=MoRIIOMode.WRITE,
_has_mamba=False,
_attn_group_ids=[0],
_mamba_group_ids=[],
request_id_to_transfer_id={},
transfer_id_to_request_id={},
_reqs_need_save={},
_req_kv_params={},
)
request = SimpleNamespace(
request_id="req",
kv_transfer_params={"do_remote_decode": True, "transfer_id": "tx"},
)
sched.update_state_after_alloc(
request,
_FakeBlocks(all_groups=([100, 101],)),
num_external_tokens=0,
)
assert sched._reqs_need_save["req"][1] == ([100, 101],)
assert sched._req_kv_params["req"] == request.kv_transfer_params
def test_session_build_rejects_per_layer_region_count_mismatch():
worker = _FakeWorker(
built_write_session={},
layer_name_to_local_kv_cache_metadata={"kda.0": ["conv", "ssm"]},
layer_name_to_remote_kv_cache_metadata={"prefill": {"kda.0": ["conv"]}},
)
with pytest.raises(moriio_common.MoRIIOError, match="registered 1 region"):
worker._get_built_session("prefill")
def test_register_kv_caches_rejects_hybrid_write_before_registration():
worker = _FakeWorker(mode=MoRIIOMode.WRITE, _transfer_layer_names={"kda.0"})
worker._is_mamba_layer = lambda _layer_name: True
with pytest.raises(moriio_common.MoRIIOError, match="READ mode only"):
worker.register_kv_caches({"kda.0": object()})
def test_register_kv_caches_rejects_no_transfer_enabled_layers():
worker = _FakeWorker(mode=MoRIIOMode.READ, _transfer_layer_names=set())
with pytest.raises(moriio_common.MoRIIOError, match="no transfer-enabled"):
worker.register_kv_caches({"disabled.0": object()})
def test_worker_rejects_incompatible_mamba_specs_before_registration():
incompatible = _mamba_spec()
object.__setattr__(incompatible, "page_size_padded", 4096)
worker = _FakeWorker(
mode=MoRIIOMode.READ,
world_size=1,
_transfer_layer_names={"kda.0", "kda.1"},
layer_to_spec={"kda.0": _mamba_spec(), "kda.1": incompatible},
)
with pytest.raises(moriio_common.MoRIIOError, match="same cache spec"):
worker.register_kv_caches({"kda.0": object(), "kda.1": object()})
# --------------------------------------------------------------------------
# _truncate_mamba_request_for_prefill
# --------------------------------------------------------------------------
def _mk_request(prompt, max_tokens=64, params=None):
return SimpleNamespace(
kv_transfer_params={} if params is None else params,
num_prompt_tokens=len(prompt),
prompt_token_ids=list(prompt),
prompt_embeds=None,
_all_token_ids=list(prompt),
max_tokens=max_tokens,
)
def test_truncate_mamba_request_pops_last_token_and_caps_tokens():
sched = _FakeScheduler()
req = _mk_request([10, 11, 12, 13, 14])
sched._truncate_mamba_request_for_prefill(req)
assert req.prompt_token_ids == [10, 11, 12, 13] # last token popped
assert req._all_token_ids == [10, 11, 12, 13]
assert req.num_prompt_tokens == 4 # N-1 accounting
assert req.max_tokens == 1
assert req.kv_transfer_params["_p_side_truncated"] is True
def test_truncate_mamba_request_is_idempotent_across_reschedule():
sched = _FakeScheduler()
req = _mk_request([10, 11, 12])
sched._truncate_mamba_request_for_prefill(req)
assert req.num_prompt_tokens == 2
# A second call (e.g. after a preemption) must not truncate again.
sched._truncate_mamba_request_for_prefill(req)
assert req.num_prompt_tokens == 2
assert req.prompt_token_ids == [10, 11]
def test_truncate_mamba_request_noop_for_single_token_prompt():
sched = _FakeScheduler()
req = _mk_request([10])
sched._truncate_mamba_request_for_prefill(req)
assert req.num_prompt_tokens == 1
assert req.prompt_token_ids == [10]
assert "_p_side_truncated" not in req.kv_transfer_params
# --------------------------------------------------------------------------
# get_num_new_matched_tokens (hybrid N-1 accounting)
# --------------------------------------------------------------------------
def test_get_num_new_matched_tokens_read_recomputes_last_token():
sched = _FakeScheduler(is_producer=False, mode=MoRIIOMode.READ, _has_mamba=True)
req = SimpleNamespace(
num_prompt_tokens=10,
prompt_token_ids=list(range(10)),
kv_transfer_params=None,
)
n, is_async = sched.get_num_new_matched_tokens(req, num_computed_tokens=0)
# READ always recomputes the final token locally: N-1 - computed.
assert n == 9
assert is_async is False
def test_get_num_new_matched_tokens_write_plain_keeps_all_tokens():
sched = _FakeScheduler(is_producer=False, mode=MoRIIOMode.WRITE, _has_mamba=False)
req = SimpleNamespace(
num_prompt_tokens=10,
prompt_token_ids=list(range(10)),
kv_transfer_params=None,
)
n, is_async = sched.get_num_new_matched_tokens(req, num_computed_tokens=2)
# Pure-attention WRITE: no N-1 drop; full length minus already-computed.
assert n == 8
assert is_async is True
@pytest.mark.parametrize(
("mode", "num_computed_tokens", "expected", "is_async"),
[
(MoRIIOMode.READ, 0, 9, False),
(MoRIIOMode.READ, 10, 0, False),
(MoRIIOMode.WRITE, 2, 8, True),
],
)
def test_get_num_new_matched_tokens_supports_embeds_only_prompts(
mode, num_computed_tokens, expected, is_async
):
sched = _FakeScheduler(is_producer=False, mode=mode, _has_mamba=True)
req = SimpleNamespace(
num_prompt_tokens=10,
prompt_token_ids=None,
prompt_embeds=object(),
kv_transfer_params=None,
)
assert sched.get_num_new_matched_tokens(req, num_computed_tokens) == (
expected,
is_async,
)
def test_get_num_new_matched_tokens_producer_returns_zero():
sched = _FakeScheduler(is_producer=True, mode=MoRIIOMode.WRITE, _has_mamba=True)
req = _mk_request(list(range(5)), params={"do_remote_decode": True})
n, is_async = sched.get_num_new_matched_tokens(req, num_computed_tokens=0)
assert (n, is_async) == (0, False)
# The producer never truncates here: the scheduler has already measured the
# prefix-cache hit against the full prompt by this point.
assert req.num_prompt_tokens == 5
def test_on_new_request_truncates_producer_prompt():
sched = _FakeScheduler(is_producer=True, mode=MoRIIOMode.WRITE, _has_mamba=True)
req = _mk_request(list(range(5)), params={"do_remote_decode": True})
sched.on_new_request(req)
# Producer stops at h(N-1): the last prompt token was dropped.
assert req.num_prompt_tokens == 4
assert req.max_tokens == 1
assert req.kv_transfer_params["_p_side_truncated"] is True
def test_on_new_request_is_noop_without_mamba_or_on_decode():
plain = _FakeScheduler(is_producer=True, mode=MoRIIOMode.WRITE, _has_mamba=False)
req = _mk_request(list(range(5)), params={"do_remote_decode": True})
plain.on_new_request(req)
assert req.num_prompt_tokens == 5
decode_side = _FakeScheduler(
is_producer=False, mode=MoRIIOMode.WRITE, _has_mamba=True
)
req2 = _mk_request(list(range(5)), params={"do_remote_prefill": True})
decode_side.on_new_request(req2)
assert req2.num_prompt_tokens == 5
# --------------------------------------------------------------------------
# Offset-template cache
# --------------------------------------------------------------------------
_SAMPLE_SLOT_SETS = [
([1, 2], [1, 2]),
([0], [3]),
([3, 0, 2], [3, 0, 2]),
([2, 2], [5, 5]),
([], []),
]
def test_worker_compute_mamba_offsets_caches_and_matches_template():
split = _gdn_split_info()
conv_dim = sum(split.local_proj_dims)
conv_shape = (conv_dim, split.conv_rows)
ssm_shape = (2, 4, 4)
page_bytes = 256
pages = torch.zeros((8, 1, 1, page_bytes), dtype=torch.uint8)
spec = moriio_connector.MambaSpec(
block_size=1,
shapes=(conv_shape, ssm_shape),
dtypes=(torch.bfloat16, torch.bfloat16),
page_size_padded=page_bytes,
)
conv, ssm = moriio_layout.kda_conv_ssm(pages, spec)
worker = _FakeWorker(
kv_caches={"kda.0": pages},
layer_to_spec={"kda.0": spec},
_conv_decomp=split,
_mamba_offset_templates={},
tp_rank=0,
world_size=1,
)
cached_template = None
for local_slots, remote_slots in _SAMPLE_SLOT_SETS:
lo, ro, sz, n_conv = worker._compute_mamba_transfer_offsets(
"kda.0", local_slots, remote_slots
)
template = moriio_layout.build_mamba_offset_template(
conv, ssm, split, tp_ratio=1
)
expected = moriio_layout.apply_mamba_offset_template(
template, local_slots, remote_slots
)
assert (lo, ro, sz) == expected
assert n_conv == moriio_layout.compute_mamba_conv_split_count(
local_slots, split
)
current_template = worker._mamba_offset_templates["kda.0"]
if cached_template is None:
cached_template = current_template
else:
assert current_template is cached_template
assert "kda.0" in worker._mamba_offset_templates
@pytest.mark.parametrize("remote_tp_size", [1, None])
def test_mamba_tp_ratio_accepts_only_equal_tp(remote_tp_size):
worker = _FakeWorker(world_size=1)
assert worker._mamba_tp_ratio(remote_tp_size) == 1
@pytest.mark.parametrize("remote_tp_size", [2, 3])
def test_mamba_tp_ratio_rejects_heterogeneous_tp(remote_tp_size):
worker = _FakeWorker(world_size=1)
with pytest.raises(NotImplementedError, match="heterogeneous-TP"):
worker._mamba_tp_ratio(remote_tp_size)
def test_worker_selects_each_mamba_layers_own_block_group():
worker = _FakeWorker(_mamba_payload_index_by_layer={"kda.0": 1, "kda.1": 2})
groups = ([10], [40], [70])
assert worker._mamba_blocks_for_layer("kda.0", groups) == [40]
assert worker._mamba_blocks_for_layer("kda.1", groups) == [70]
def test_worker_rejects_missing_mamba_block_group():
worker = _FakeWorker(_mamba_payload_index_by_layer={"kda.0": 2})
with pytest.raises(moriio_common.MoRIIOError, match="no block group"):
worker._mamba_blocks_for_layer("kda.0", ([10], [40]))
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import importlib.util
import math
import threading
from collections import OrderedDict, defaultdict
from queue import Queue
@@ -14,6 +15,7 @@ import torch
from vllm.platforms import current_platform
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
MambaSpec,
MLAAttentionSpec,
)
@@ -36,6 +38,10 @@ moriio_layout = importlib.import_module(
"vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_layout"
)
msgpack = importlib.import_module("msgpack")
ssm_conv_transfer_utils = importlib.import_module(
"vllm.distributed.kv_transfer.kv_connector.v1.ssm_conv_transfer_utils"
)
MambaConvSplitInfo = ssm_conv_transfer_utils.MambaConvSplitInfo
ROLE = moriio_common.ROLE
MoRIIOError = moriio_common.MoRIIOError
@@ -353,6 +359,9 @@ def test_write_transfer_plan_caches_offsets_per_layer_geometry():
kv_caches: dict[str, torch.Tensor]
layer_name_to_local_kv_cache_metadata: dict[str, list[Any]]
def _region_session_indices(self, layer_name):
return [list(self.kv_caches).index(layer_name)]
def _compute_block_transfer_offsets(
self,
layer_name,
@@ -673,6 +682,30 @@ def test_local_block_ids_longer_than_remote_raises_value_error():
)
@pytest.mark.parametrize(
("local_block_ids", "remote_block_ids", "match"),
[
([-1], [0], "local block id -1"),
([0], [8], "remote block id 8"),
],
)
def test_block_ids_must_fit_registered_regions(
local_block_ids, remote_block_ids, match
):
cache = torch.empty((8, 2, 4, 2, 3), dtype=torch.bfloat16)
worker = _worker({"layer": cache}, {"layer": _full_spec()})
with pytest.raises(ValueError, match=match):
moriio_layout.compute_block_transfer_offsets(
"layer",
cache,
worker.layer_to_spec,
local_block_ids,
remote_block_ids,
_remote_meta(num_blocks=8).num_blocks,
)
def test_empty_local_block_ids_is_free_only_noop():
cache = torch.empty((8, 2, 4, 2, 3), dtype=torch.bfloat16)
worker = _worker({"layer": cache}, {"layer": _full_spec()})
@@ -745,3 +778,245 @@ def test_unsupported_shape_raises_value_error():
with pytest.raises(ValueError, match="Unsupported MoRIIO K/V cache shape"):
moriio_layout.get_layer_transfer_geometry("layer", cache, worker.layer_to_spec)
# ---------------------------------------------------------------------------
# Mamba/KDA (conv + ssm) transfer geometry
# ---------------------------------------------------------------------------
def _slot_strided(num_slots, per_slot_shape, slot_stride, dtype=torch.bfloat16):
"""Build a slot-strided view: shape (num_slots, *per_slot_shape) whose slot
stride (dim 0) exceeds the per-slot element count, mimicking the K3 KDA
conv/ssm cache tensors."""
inner = 1
for d in per_slot_shape:
inner *= d
assert slot_stride >= inner
backing = torch.zeros(num_slots * slot_stride, dtype=dtype)
inner_strides: list[int] = []
acc = 1
for d in reversed(per_slot_shape):
inner_strides.insert(0, acc)
acc *= d
return backing.as_strided(
(num_slots, *per_slot_shape), (slot_stride, *inner_strides)
)
def _gdn_split_info(conv_rows=3, key_dim=4, value_dim=8, dtype_size=2):
"""A GDN-style conv split: sub-projections (key, key, value)."""
conv_dim = 2 * key_dim + value_dim
conv_state_bytes = conv_dim * conv_rows * dtype_size
ssm_state_bytes = 64 # opaque here; ssm size is taken from the tensor
return MambaConvSplitInfo(
conv_rows=conv_rows,
local_proj_dims=(key_dim, key_dim, value_dim),
conv_dtype_size=dtype_size,
ssm_sizes=(conv_state_bytes, ssm_state_bytes),
)
def test_mamba_transfer_geometry_is_slot_strided():
split = _gdn_split_info()
conv_dim = sum(split.local_proj_dims)
conv = _slot_strided(5, (conv_dim, split.conv_rows), slot_stride=200)
ssm = _slot_strided(5, (2, 4, 4), slot_stride=64)
geom = moriio_layout.get_mamba_transfer_geometry(conv, ssm)
assert geom.conv_slot_stride == 200
assert geom.ssm_slot_stride == 64
assert geom.ssm_slot_bytes == (2 * 4 * 4) * ssm.element_size()
# Region spans the full slot-strided extent (shape[0] * stride(0)), not just
# numel(), so the highest-slot transfers stay in-bounds.
assert geom.conv_region_len == conv.shape[0] * conv.stride(0) * conv.element_size()
assert geom.ssm_region_len == ssm.shape[0] * ssm.stride(0) * ssm.element_size()
def test_kda_conv_ssm_unpacks_padded_pages_as_aliasing_strided_views():
num_blocks = 4
page_bytes = 128
conv_shape = (3, 4)
ssm_shape = (2, 2, 4)
conv_bytes = math.prod(conv_shape) * torch.bfloat16.itemsize
ssm_bytes = math.prod(ssm_shape) * torch.bfloat16.itemsize
assert conv_bytes + ssm_bytes < page_bytes
pages = torch.zeros((num_blocks, 1, 1, page_bytes), dtype=torch.uint8)
spec = MambaSpec(
block_size=1,
shapes=(conv_shape, ssm_shape),
dtypes=(torch.bfloat16, torch.bfloat16),
page_size_padded=page_bytes,
)
conv, ssm = moriio_layout.kda_conv_ssm(pages, spec)
assert conv.shape == (num_blocks, *conv_shape)
assert ssm.shape == (num_blocks, *ssm_shape)
assert conv.stride() == (page_bytes // 2, 4, 1)
assert ssm.stride() == (page_bytes // 2, 8, 4, 1)
assert conv.untyped_storage().data_ptr() == pages.untyped_storage().data_ptr()
assert ssm.untyped_storage().data_ptr() == pages.untyped_storage().data_ptr()
assert conv.storage_offset() == 0
assert ssm.storage_offset() * ssm.element_size() == conv_bytes
conv[1].fill_(1)
ssm[2].fill_(2)
assert torch.all(pages[1, 0, 0, :conv_bytes].view(torch.bfloat16) == 1)
assert torch.all(
pages[2, 0, 0, conv_bytes : conv_bytes + ssm_bytes].view(torch.bfloat16) == 2
)
def test_kda_conv_ssm_accepts_strided_blocks_without_copying():
backing = torch.zeros(4 * 256, dtype=torch.uint8)
pages = backing.as_strided((4, 1, 1, 128), (256, 128, 128, 1))
spec = MambaSpec(
block_size=1,
shapes=((3, 4), (2, 2, 4)),
dtypes=(torch.bfloat16, torch.bfloat16),
page_size_padded=128,
)
conv, ssm = moriio_layout.kda_conv_ssm(pages, spec)
assert not pages.is_contiguous()
assert conv.stride(0) == 128
assert ssm.stride(0) == 128
assert conv.untyped_storage().data_ptr() == backing.data_ptr()
assert ssm.untyped_storage().data_ptr() == backing.data_ptr()
def test_kda_conv_ssm_rejects_noncontiguous_page_bytes():
pages = torch.zeros((4, 1, 1, 256), dtype=torch.uint8)[..., ::2]
spec = MambaSpec(
block_size=1,
shapes=((3, 4), (2, 2, 4)),
dtypes=(torch.bfloat16, torch.bfloat16),
page_size_padded=128,
)
with pytest.raises(ValueError, match="contiguous byte dimension"):
moriio_layout.kda_conv_ssm(pages, spec)
def test_kda_conv_ssm_rejects_non_byte_packed_cache():
pages = torch.zeros((4, 1, 1, 128), dtype=torch.bfloat16)
spec = MambaSpec(
block_size=1,
shapes=((3, 4), (2, 2, 4)),
dtypes=(torch.bfloat16, torch.bfloat16),
page_size_padded=128,
)
with pytest.raises(ValueError, match="byte-sized dtype"):
moriio_layout.kda_conv_ssm(pages, spec)
def test_mamba_transfer_geometry_requires_matching_slot_counts():
conv = torch.empty((5, 4, 3), dtype=torch.bfloat16)
ssm = torch.empty((4, 2, 4, 4), dtype=torch.bfloat16)
with pytest.raises(ValueError, match="conv/ssm slot count mismatch"):
moriio_layout.get_mamba_transfer_geometry(conv, ssm)
def test_mamba_transfer_geometry_rejects_empty_caches():
conv = torch.empty((0, 4, 3), dtype=torch.bfloat16)
ssm = torch.empty((0, 2, 4, 4), dtype=torch.bfloat16)
with pytest.raises(ValueError, match="at least one slot"):
moriio_layout.get_mamba_transfer_geometry(conv, ssm)
def test_mamba_conv_ssm_offsets_homogeneous_tp():
split = _gdn_split_info()
conv_dim = sum(split.local_proj_dims)
conv = _slot_strided(6, (conv_dim, split.conv_rows), slot_stride=200)
ssm = _slot_strided(6, (2, 4, 4), slot_stride=64)
geom = moriio_layout.get_mamba_transfer_geometry(conv, ssm)
local_slots = [1, 2]
remote_slots = [1, 2]
template = moriio_layout.build_mamba_offset_template(conv, ssm, split, tp_ratio=1)
local_offs, remote_offs, sizes = moriio_layout.apply_mamba_offset_template(
template, local_slots, remote_slots
)
n_conv = moriio_layout.compute_mamba_conv_split_count(local_slots, split)
assert n_conv == len(local_slots) * len(split.local_proj_dims)
assert len(local_offs) == n_conv + len(local_slots)
# Homogeneous TP: remote conv slices equal local slices; since local==remote
# slots, conv local and remote offsets coincide.
loc_co = split.local_conv_offsets
rem_co = split.remote_conv_offsets(0, 1)
k = 0
esz = conv.element_size()
for ls, rs in zip(local_slots, remote_slots):
lbase = ls * geom.conv_slot_stride * esz
rbase = rs * geom.conv_slot_stride * esz
for (loff, _), (roff, rsz) in zip(loc_co, rem_co):
assert local_offs[k] == lbase + loff
assert remote_offs[k] == rbase + roff
assert sizes[k] == rsz
k += 1
# SSM: whole per-slot block, no head sharding at tp_ratio == 1.
essz = ssm.element_size()
for j, (ls, rs) in enumerate(zip(local_slots, remote_slots)):
assert local_offs[n_conv + j] == ls * geom.ssm_slot_stride * essz
assert remote_offs[n_conv + j] == rs * geom.ssm_slot_stride * essz
assert sizes[n_conv + j] == geom.ssm_slot_bytes
def test_mamba_conv_ssm_offsets_heterogeneous_tp_is_gated():
# Heterogeneous-TP mamba transfer is a documented follow-up (remote slot
# stride + multi-rank fan-in gather). Until then the offset helper must fail
# loudly rather than silently transfer wrong bytes, since the mamba path
# bypasses the attention KV-head validator.
split = _gdn_split_info()
conv_dim = sum(split.local_proj_dims)
conv = _slot_strided(4, (conv_dim, split.conv_rows), slot_stride=200)
ssm = _slot_strided(4, (2, 4, 4), slot_stride=64)
with pytest.raises(NotImplementedError):
moriio_layout.build_mamba_offset_template(conv, ssm, split, tp_ratio=2)
def test_local_mamba_slots_must_fit_registered_regions():
split = _gdn_split_info()
conv_dim = sum(split.local_proj_dims)
conv = _slot_strided(6, (conv_dim, split.conv_rows), slot_stride=200)
ssm = _slot_strided(6, (2, 4, 4), slot_stride=64)
template = moriio_layout.build_mamba_offset_template(conv, ssm, split, tp_ratio=1)
with pytest.raises(ValueError, match="local mamba slot 6"):
moriio_layout.apply_mamba_offset_template(template, [6], [0])
@pytest.mark.parametrize(
("local_slots", "remote_slots"),
[([1, 2], [3]), ([1], [2, 3])],
)
def test_mamba_slot_counts_must_match(local_slots, remote_slots):
split = _gdn_split_info()
conv_dim = sum(split.local_proj_dims)
conv = _slot_strided(6, (conv_dim, split.conv_rows), slot_stride=200)
ssm = _slot_strided(6, (2, 4, 4), slot_stride=64)
template = moriio_layout.build_mamba_offset_template(conv, ssm, split, tp_ratio=1)
with pytest.raises(ValueError, match="slot counts must match"):
moriio_layout.apply_mamba_offset_template(template, local_slots, remote_slots)
def test_remote_mamba_slots_must_be_non_negative():
split = _gdn_split_info()
conv_dim = sum(split.local_proj_dims)
conv = _slot_strided(6, (conv_dim, split.conv_rows), slot_stride=200)
ssm = _slot_strided(6, (2, 4, 4), slot_stride=64)
template = moriio_layout.build_mamba_offset_template(conv, ssm, split, tp_ratio=1)
with pytest.raises(ValueError, match="must be non-negative"):
moriio_layout.apply_mamba_offset_template(template, [1], [-1])
@@ -28,6 +28,7 @@ def _producer(global_dp_rank: int, dp_size: int, dp_size_local: int):
s = SimpleNamespace(
is_producer=True,
_is_hma_required=False,
_has_mamba=False,
blocks_per_sw=[0],
_global_dp_rank=global_dp_rank,
engine_id="engine-abc",
@@ -2,12 +2,14 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import threading
from types import SimpleNamespace
import pytest
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_common import (
MoRIIOMode,
MoRIIOTransferAck,
TransferError,
)
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector import (
MoRIIOConnector,
@@ -17,6 +19,9 @@ from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector import
resolve_moriio_transfer_ack,
validate_moriio_heterogeneous_tp_kv_heads,
)
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_engine import (
MoRIIOWrapper,
)
def test_remote_tp_rank_same_tp_maps_to_self():
@@ -277,6 +282,9 @@ def test_read_completion_sends_structured_release_with_consumer_tp_size():
self.lock = threading.Lock()
self.sent = []
# Exercise the real batch verdict rather than restating it here.
poll_transfer_batch = MoRIIOWrapper.poll_transfer_batch
def send_notify(
self,
transfer_id,
@@ -292,13 +300,18 @@ def test_read_completion_sends_structured_release_with_consumer_tp_size():
worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker)
worker.world_size = 8
worker.moriio_config = SimpleNamespace(recv_abort_timeout=600.0)
worker.moriio_wrapper = FakeWrapper()
worker._recving_transfers = {"req": {"layer0": DoneStatus()}}
# A layer maps to the list of reads posted for it (a KDA layer posts two).
worker._recving_transfers = {"req": {"layer0": [DoneStatus(), DoneStatus()]}}
worker._recving_transfers_callback_addr = {
"req": ("127.0.0.1", "7000", "tx-release")
}
# Transfer-timeout reaping state consulted by _pop_done_transfers.
worker._recving_transfers_start = {}
# Load-error bookkeeping cleared alongside a completed transfer.
worker._recving_local_blocks = {}
worker._invalid_block_ids = set()
assert worker._pop_done_transfers() == {"tx-release"}
assert worker.moriio_wrapper.sent == [
@@ -314,6 +327,133 @@ def test_read_completion_sends_structured_release_with_consumer_tp_size():
assert worker._recving_transfers_callback_addr == {}
def test_aborted_read_without_local_blocks_only_releases_remote_blocks():
class FakeWrapper:
def __init__(self):
self.sent = []
def send_notify(
self,
transfer_id,
host,
port,
message_type=None,
message_fields=None,
):
self.sent.append((transfer_id, host, port, message_type, message_fields))
def shutdown(self):
pass
worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker)
worker.mode = MoRIIOMode.READ
worker.world_size = 8
worker.moriio_wrapper = FakeWrapper()
worker._read_blocks(
local_block_ids=[],
remote_block_ids=[[10, 11], [90]],
dst_engine_id="prefill",
request_id="req-aborted",
transfer_id="tx-aborted",
remote_host="127.0.0.1",
remote_notify_port=7000,
remote_tp_size=8,
remote_dp_rank=0,
chosen_tp=2,
)
assert worker.moriio_wrapper.sent == [
(
"tx-aborted",
"127.0.0.1",
"7002",
"release",
{"consumer_tp_size": 8},
)
]
def test_read_completion_waits_for_every_posted_read():
class Status:
def __init__(self, done: bool):
self.done = done
def Succeeded(self):
return self.done
def Failed(self):
return False
class FakeWrapper:
def __init__(self):
self.lock = threading.Lock()
self.sent = []
poll_transfer_batch = MoRIIOWrapper.poll_transfer_batch
def send_notify(self, *args, **kwargs):
self.sent.append((args, kwargs))
def shutdown(self):
pass
worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker)
worker.world_size = 8
worker.moriio_config = SimpleNamespace(recv_abort_timeout=600.0)
worker.moriio_wrapper = FakeWrapper()
worker._recving_transfers = {
"req": {"kda_layer": [Status(done=True), Status(done=False)]}
}
worker._recving_transfers_callback_addr = {
"req": ("127.0.0.1", "7000", "tx-pending")
}
worker._recving_transfers_start = {"req": float("inf")}
worker._recving_local_blocks = {"req": [1, 2]}
worker._invalid_block_ids = set()
assert worker._pop_done_transfers() == set()
assert "req" in worker._recving_transfers
assert worker.moriio_wrapper.sent == []
@pytest.mark.parametrize(
("has_mamba", "expected_invalid"),
[(False, {1, 2}), (True, set())],
)
def test_failed_read_reports_blocks_only_without_hma(has_mamba, expected_invalid):
worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker)
worker._has_mamba = has_mamba
worker._recving_local_blocks = {"req": [1, 2]}
worker._invalid_block_ids = set()
worker._record_failed_recv("req")
assert worker.get_block_ids_with_load_errors() == expected_invalid
def test_hybrid_step_barrier_fails_closed(monkeypatch):
class FailingWrapper:
def waiting_for_transfer_complete(self, _statuses):
raise TransferError("failed")
def shutdown(self):
pass
worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker)
worker._has_mamba = True
worker._reads_issued_this_step = [object()]
worker._mamba_reads_this_step = [object()]
worker.moriio_wrapper = FailingWrapper()
monkeypatch.setattr(
"vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector.get_forward_context",
lambda: SimpleNamespace(cudagraph_runtime_mode=None),
)
with pytest.raises(TransferError, match="failed"):
worker._await_reads_issued_this_step()
def test_requested_cudagraph_mode_is_never_overridden():
# The configured cudagraph mode is always honored: the barrier fires when
# the operator sets cudagraph_mode=PIECEWISE, and READ mode with full
@@ -0,0 +1,129 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Transfer-completion semantics of the MoRIIO wrapper.
Covers the two behaviours the READ barrier depends on:
* poll_transfer_batch considers EVERY status, not just the newest one
* waiting_for_transfer_complete blocks until terminal and raises on failure.
"""
import importlib
import importlib.util
import threading
import time
import pytest
from vllm.platforms import current_platform
mori_available = importlib.util.find_spec("mori") is not None
if not (current_platform.is_rocm() and mori_available):
pytest.skip(
"MoRIIOs are only available on ROCm with mori package installed",
allow_module_level=True,
)
moriio_common = importlib.import_module(
"vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_common"
)
moriio_engine = importlib.import_module(
"vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_engine"
)
MoRIIOWrapper = moriio_engine.MoRIIOWrapper
TransferBatchState = moriio_common.TransferBatchState
TransferError = moriio_common.TransferError
@pytest.mark.parametrize("value", [0, -1, float("inf"), float("nan")])
def test_timeout_must_be_positive_and_finite(value):
with pytest.raises(ValueError, match="must be finite and greater than zero"):
moriio_common._positive_finite_timeout("test_timeout", value)
def test_positive_finite_timeout_is_preserved():
assert moriio_common._positive_finite_timeout("test_timeout", "1.5") == 1.5
class FakeStatus:
"""Mirrors the mori TransferStatus surface the wrapper uses."""
def __init__(self, state: str = "in_progress", message: str = ""):
self.state = state
self.message = message
def Succeeded(self) -> bool:
return self.state == "success"
def Failed(self) -> bool:
return self.state == "failed"
def Message(self) -> str:
return self.message
def Code(self) -> str:
return self.state
def _ok():
return FakeStatus("success")
def _pending():
return FakeStatus("in_progress")
def _bad():
return FakeStatus("failed", "SQ full")
@pytest.mark.parametrize(
"statuses,expected",
[
([], TransferBatchState.DONE),
([_ok(), _ok()], TransferBatchState.DONE),
# Each case below puts the interesting status BEFORE a success, which a
# statuses[-1] check would have missed entirely.
([_bad(), _ok()], TransferBatchState.FAILED),
([_pending(), _ok()], TransferBatchState.PENDING),
([_pending(), _bad(), _ok()], TransferBatchState.FAILED),
],
)
def test_poll_transfer_batch_considers_every_status(statuses, expected):
wrapper = MoRIIOWrapper(moriio_engine=object(), transfer_timeout=0.05)
assert wrapper.poll_transfer_batch(statuses) is expected
@pytest.mark.parametrize(
"statuses_factory,expected_message",
[
(lambda: [_ok(), _bad()], "SQ full"),
(lambda: [_ok(), _pending()], "timed out"),
],
ids=["failed", "timeout"],
)
def test_wait_raises_with_detail(statuses_factory, expected_message):
wrapper = MoRIIOWrapper(moriio_engine=object(), transfer_timeout=0.05)
with pytest.raises(TransferError, match=expected_message):
wrapper.waiting_for_transfer_complete(statuses_factory())
def test_wait_returns_immediately_when_nothing_pending():
wrapper = MoRIIOWrapper(moriio_engine=object(), transfer_timeout=0.05)
wrapper.waiting_for_transfer_complete([])
wrapper.waiting_for_transfer_complete([_ok(), _ok()])
def test_wait_blocks_until_terminal():
"""The barrier is only worth anything if it actually waits."""
wrapper = MoRIIOWrapper(moriio_engine=object(), transfer_timeout=5.0)
late = FakeStatus("in_progress")
timer = threading.Timer(0.2, lambda: setattr(late, "state", "success"))
timer.start()
try:
start = time.monotonic()
wrapper.waiting_for_transfer_complete([late])
assert time.monotonic() - start >= 0.15
finally:
timer.cancel()
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import contextlib
import math
import os
import threading
import time
@@ -45,6 +46,13 @@ TransferId = str
TransferOffsetsKey = tuple[str, tuple[int, ...], tuple[int, ...], torch.dtype]
def _positive_finite_timeout(name: str, value: Any) -> float:
timeout = float(value)
if not math.isfinite(timeout) or timeout <= 0:
raise ValueError(f"{name} must be finite and greater than zero")
return timeout
class MoRIIOTransferAck(NamedTuple):
transfer_id: TransferId
consumer_tp_size: int = 1
@@ -160,6 +168,20 @@ class MoRIIOMode(Enum):
WRITE = "write"
class TransferBatchState(Enum):
"""Verdict for a group of transfer statuses that belong to one request.
A request's KV transfer is spread over one status per layer (two for a KDA
layer), so a single status never decides the request: PENDING means at
least one is still in flight and none has failed, FAILED means at least
one failed, DONE means all succeeded.
"""
DONE = "done"
FAILED = "failed"
PENDING = "pending"
class MoRIIOError(Exception):
"""Base exception for MoRIIO operations."""
@@ -260,6 +282,7 @@ class MoRIIOConfig:
tp_size: int
transfer_timeout: float
defer_timeout: float
recv_abort_timeout: float
read_mode: bool = False
qp_per_transfer: int = 1
post_batch_size: int = -1
@@ -284,6 +307,8 @@ class MoRIIOConfig:
# raising TransferError (sec).
# defer_timeout -> Timeout before a deferred send with no finished_sending
# notification is reaped and its blocks force-freed (sec).
# recv_abort_timeout -> Timeout before an in-flight recv whose RDMA
# completion never arrived is aborted (sec).
# Knobs for RDMA transfers, ignored if on xgmi backend
# qp_per_transfer -> Number of RDMA Queue Pairs per KV transfer.
@@ -315,13 +340,29 @@ class MoRIIOConfig:
"must be one of 'rdma' or 'xgmi'."
)
transfer_timeout = float(
transfer_timeout = _positive_finite_timeout(
"kv_connector_extra_config.transfer_timeout",
extra_config.get(
"transfer_timeout", MoRIIOConstants.DEFAULT_TRANSFER_TIMEOUT
)
),
)
defer_timeout = float(
extra_config.get("defer_timeout", MoRIIOConstants.DEFAULT_DEFER_TIMEOUT)
defer_timeout = _positive_finite_timeout(
"kv_connector_extra_config.defer_timeout",
extra_config.get("defer_timeout", MoRIIOConstants.DEFAULT_DEFER_TIMEOUT),
)
legacy_recv_abort_timeout = os.environ.get("VLLM_MORIIO_TRANSFER_TIMEOUT_S")
if legacy_recv_abort_timeout is not None:
logger.warning_once(
"The environment variable VLLM_MORIIO_TRANSFER_TIMEOUT_S is "
"deprecated. Set 'recv_abort_timeout' inside "
"kv_transfer_config.kv_connector_extra_config instead."
)
recv_abort_timeout = _positive_finite_timeout(
"kv_connector_extra_config.recv_abort_timeout",
extra_config.get(
"recv_abort_timeout",
legacy_recv_abort_timeout or MoRIIOConstants.DEFAULT_RECV_ABORT_TIMEOUT,
),
)
return cls(
@@ -347,6 +388,7 @@ class MoRIIOConfig:
backend=backend,
transfer_timeout=transfer_timeout,
defer_timeout=defer_timeout,
recv_abort_timeout=recv_abort_timeout,
)
@@ -374,6 +416,10 @@ class MoRIIOConstants:
# notification is reaped and its blocks force-freed.
# Overridable via kv_connector_extra_config["defer_timeout"].
DEFAULT_DEFER_TIMEOUT = 60.0
# Timeout (seconds) before an in-flight recv whose RDMA completion was lost
# is aborted, so the decode worker does not hang on it forever.
# Overridable via kv_connector_extra_config["recv_abort_timeout"].
DEFAULT_RECV_ABORT_TIMEOUT = 120.0
# The router embeds both zmq_addresses in the request_id:
@@ -567,9 +613,12 @@ class MoRIIOConnectorMetadata(KVConnectorMetadata):
remote_handshake_port=int(remote_handshake_port),
remote_notify_port=int(remote_notify_port),
# Remote peer TP degree (used as remote_tp_size downstream). The
# proxy advertises it under "remote_tp_size"; #46332 read "tp_size"
# which is absent on WRITE producer requests -> defaulted to 1 ->
# rank collapse. Read the right key; 0 == unknown (== homogeneous).
# decode side must know the prefiller's TP degree to map producer
# ranks onto the correct decode rank (and vice versa). The old plain
# "tp_size" key (#46332) is absent on WRITE producer requests and
# defaulted to 1, collapsing every producer rank onto decode rank 0
# and corrupting the transfer; prefer "remote_tp_size". 0 == unknown
# (treated as homogeneous downstream).
tp_size=int(
kv_transfer_params.get("remote_tp_size")
or kv_transfer_params.get("tp_size")
File diff suppressed because it is too large Load Diff
@@ -29,6 +29,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_common import (
MoRIIOError,
MoRIIOTransferAck,
RemoteAllocInfo,
TransferBatchState,
TransferError,
TransferId,
WriteTask,
@@ -58,7 +59,6 @@ try:
except ImportError:
logger.error("MoRIIO is not available")
"""Write task execution logic for MoRIIO connector."""
@@ -395,9 +395,9 @@ class MoRIIOWriter:
)
request_info.transfer_offsets[key] = offsets
# Get session index
layer_names = list(self.worker.layer_name_to_local_kv_cache_metadata.keys())
sess_idx = layer_names.index(task.layer_name)
# One session per registered region; attention layers map to a single
# index (see MoRIIOConnectorWorker._region_session_indices).
sess_idx = self.worker._region_session_indices(task.layer_name)[0]
local_off, remote_off, sizes = offsets
@@ -662,6 +662,25 @@ class MoRIIOWrapper:
)
return transfer_status
def poll_transfer_batch(self, transfer_statuses: list[Any]) -> TransferBatchState:
"""Non-blocking verdict over every status of one request.
Judging a request by its newest status alone calls it done while an
earlier layer is unfinished or already failed; failures do land on
layers other than the last, because _post_read_with_backoff returns a
failed status when the send queue never drains.
This stays a Python scan so it does not drive the backend's progress
callback from the engine thread alongside MoRIIO's own poller.
"""
state = TransferBatchState.DONE
for status in transfer_statuses:
if status.Failed():
return TransferBatchState.FAILED
if not status.Succeeded():
state = TransferBatchState.PENDING
return state
def waiting_for_transfer_complete(self, transfer_statuses: list[Any] | None = None):
if transfer_statuses is None:
with self.lock:
@@ -673,6 +692,15 @@ class MoRIIOWrapper:
if not transfers_to_wait:
return
self._poll_transfers_until_done(transfers_to_wait)
def _poll_transfers_until_done(self, transfers_to_wait: list[Any]) -> None:
"""Fallback for mori builds without the batched wait.
Gives failure precedence over pending: once any status has failed the
request is lost, so waiting out the remaining ones would only delay the
barrier by up to transfer_timeout.
"""
timeout = self._transfer_timeout
deadline = time.monotonic() + timeout
remaining = list(transfers_to_wait)
@@ -698,6 +726,8 @@ class MoRIIOWrapper:
)
else:
still_waiting.append(status)
if errors:
break
remaining = still_waiting
if remaining:
time.sleep(0.001)
@@ -2,14 +2,18 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable, Mapping
from typing import NamedTuple
from typing import NamedTuple, cast
import torch
from vllm.distributed.kv_transfer.kv_connector.v1.ssm_conv_transfer_utils import (
MambaConvSplitInfo,
)
from vllm.v1.kv_cache_interface import (
AttentionSpec,
KVCacheConfig,
KVCacheSpec,
MambaSpec,
MLAAttentionSpec,
SlidingWindowMLASpec,
UniformTypeKVCacheSpecs,
@@ -29,6 +33,85 @@ class LayerTransferGeometry(NamedTuple):
split_kv_regions: bool
class MambaTransferGeometry(NamedTuple):
"""Geometry of a hybrid (mamba/KDA) layer's recurrent state.
A KDA layer's packed page is exposed as ``conv_state`` and ``ssm_state``
views. Each is slot-strided, so a slot's bytes live at
``slot * slot_stride * element_size`` and span ``slot_bytes``. The two
tensors are registered as two separate MoRIIO regions per layer.
"""
conv_slot_stride: int
ssm_slot_stride: int
ssm_slot_bytes: int
conv_element_size: int
ssm_element_size: int
conv_region_len: int
ssm_region_len: int
def get_mamba_transfer_geometry(
conv: torch.Tensor,
ssm: torch.Tensor,
) -> MambaTransferGeometry:
"""Derive the slot-strided geometry of a KDA layer's conv + ssm state.
``conv``/``ssm`` are zero-copy views over the layer's packed page. Slot byte
offset is ``slot * slot_stride * element_size`` and each
slot spans ``subtensor[0].numel() * element_size`` bytes; the whole tensor
is registered as one region (``*_region_len`` bytes).
"""
if conv.ndim == 0 or ssm.ndim == 0 or conv.shape[0] == 0 or ssm.shape[0] == 0:
raise ValueError("KDA conv/ssm caches must contain at least one slot")
if conv.shape[0] != ssm.shape[0]:
raise ValueError(
f"KDA conv/ssm slot count mismatch: {conv.shape[0]} != {ssm.shape[0]}"
)
conv_slot_bytes = conv[0].numel() * conv.element_size()
ssm_slot_bytes = ssm[0].numel() * ssm.element_size()
conv_available_bytes = (
conv.untyped_storage().nbytes() - conv.storage_offset() * conv.element_size()
)
ssm_available_bytes = (
ssm.untyped_storage().nbytes() - ssm.storage_offset() * ssm.element_size()
)
if conv.shape[0]:
conv_required_bytes = (conv.shape[0] - 1) * conv.stride(
0
) * conv.element_size() + conv_slot_bytes
ssm_required_bytes = (ssm.shape[0] - 1) * ssm.stride(
0
) * ssm.element_size() + ssm_slot_bytes
if conv_required_bytes > conv_available_bytes:
raise ValueError("KDA conv slots exceed the registered storage extent")
if ssm_required_bytes > ssm_available_bytes:
raise ValueError("KDA ssm slots exceed the registered storage extent")
return MambaTransferGeometry(
conv_slot_stride=conv.stride(0),
ssm_slot_stride=ssm.stride(0),
ssm_slot_bytes=ssm_slot_bytes,
conv_element_size=conv.element_size(),
ssm_element_size=ssm.element_size(),
# Registered region must span the full slot-strided extent: stride(0)
# may exceed the per-slot element count, so numel() would under-size the
# region and truncate the highest-slot transfers. shape[0]*stride(0)
# covers every byte a ``slot * slot_stride * elem`` offset can address.
# Cap at the bytes remaining from each view's storage offset. A packed
# conv+ssm page places ssm at a non-zero intra-buffer offset,
# so do not register past the end of the shared backing allocation.
conv_region_len=min(
conv.shape[0] * conv.stride(0) * conv.element_size(),
conv_available_bytes,
),
ssm_region_len=min(
ssm.shape[0] * ssm.stride(0) * ssm.element_size(),
ssm_available_bytes,
),
)
def build_layer_to_spec(kv_cache_config: KVCacheConfig) -> dict[str, KVCacheSpec]:
layer_to_spec: dict[str, KVCacheSpec] = {}
for group in kv_cache_config.kv_cache_groups:
@@ -99,10 +182,10 @@ def get_layer_transfer_geometry(
layer_to_spec: Mapping[str, KVCacheSpec],
remote_num_blocks: int | None = None,
) -> LayerTransferGeometry:
spec = layer_to_spec[layer_name]
shape = kv_cache.shape
stride = kv_cache.stride()
element_size = kv_cache.element_size()
spec = layer_to_spec[layer_name]
is_mla_cache = is_mla_cache_layer(layer_to_spec, layer_name)
if not is_mla_cache and len(shape) == 5 and shape[0] == 2:
@@ -238,12 +321,60 @@ def get_layer_transfer_geometry(
)
def kda_conv_ssm(
kv_cache: torch.Tensor,
spec: MambaSpec,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Return zero-copy conv and SSM views over a packed two-state page."""
from math import prod
from vllm.utils.torch_utils import get_dtype_size
if len(spec.shapes) != 2 or len(spec.dtypes) != 2:
raise ValueError(
"MoRIIO KDA transfer requires exactly two states (conv and SSM)"
)
if kv_cache.element_size() != 1:
raise ValueError("MoRIIO KDA packed cache must use a byte-sized dtype")
if kv_cache.ndim != 4 or kv_cache.shape[1:3] != (1, 1) or kv_cache.stride(3) != 1:
raise ValueError(
"MoRIIO KDA packed cache must have shape [B, 1, 1, C] with "
"a contiguous byte dimension"
)
# Mirror MambaBase.bind_kv_cache: blocks may be strided within a shared
# multi-layer allocation, while every individual byte page is contiguous.
pages = kv_cache.squeeze(dim=(1, 2))
states: list[torch.Tensor] = []
offset = 0
for shp, dt in zip(spec.shapes, spec.dtypes, strict=True):
nbytes = prod(shp) * get_dtype_size(dt)
if offset + nbytes > pages.shape[1]:
raise ValueError(
"MoRIIO KDA state layout exceeds the packed cache page size"
)
state = pages[:, offset : offset + nbytes].view(dt)
states.append(state.view(-1, *shp))
offset += nbytes
return states[0], states[1]
def iter_layer_registration_regions(
layer_name: str,
kv_cache: torch.Tensor,
layer_to_spec: Mapping[str, KVCacheSpec],
) -> list[tuple[torch.Tensor, int]]:
geometry = get_layer_transfer_geometry(layer_name, kv_cache, layer_to_spec)
spec = layer_to_spec[layer_name]
if isinstance(spec, MambaSpec):
# KDA layer: register the conv and ssm tensors as two whole-tensor
# regions (the conv 3-subprojection split is expressed as transfer
# offset triples, not extra registrations).
conv, ssm = kda_conv_ssm(kv_cache, spec)
geom = get_mamba_transfer_geometry(conv, ssm)
return [(conv, geom.conv_region_len), (ssm, geom.ssm_region_len)]
geometry = cast(
LayerTransferGeometry,
get_layer_transfer_geometry(layer_name, kv_cache, layer_to_spec),
)
region_len = geometry.num_blocks * geometry.regions_per_block * geometry.block_len
if geometry.regions_per_block == 1:
# With padded or interleaved pages the block stride exceeds the
@@ -311,9 +442,36 @@ def compute_block_transfer_offsets(
"local_block_ids longer than remote_block_ids: "
f"{len(local_block_ids)} > {len(remote_block_ids)}"
)
geometry = get_layer_transfer_geometry(
layer_name, kv_cache, layer_to_spec, remote_num_blocks
geometry = cast(
LayerTransferGeometry,
get_layer_transfer_geometry(
layer_name, kv_cache, layer_to_spec, remote_num_blocks
),
)
invalid_local = next(
(
block_id
for block_id in local_block_ids
if not 0 <= block_id < geometry.num_blocks
),
None,
)
if invalid_local is not None:
raise ValueError(
f"local block id {invalid_local} is outside [0, {geometry.num_blocks})"
)
invalid_remote = next(
(
block_id
for block_id in remote_block_ids
if not 0 <= block_id < remote_num_blocks
),
None,
)
if invalid_remote is not None:
raise ValueError(
f"remote block id {invalid_remote} is outside [0, {remote_num_blocks})"
)
element_size = kv_cache.element_size()
transfer_size_byte = geometry.block_len
per_block = geometry.transfers_per_block
@@ -339,3 +497,125 @@ def compute_block_transfer_offsets(
w += 1
return merge_fn(offset_local, offset_remote, sizes)
class MambaOffsetTemplate(NamedTuple):
"""Slot-independent conv+ssm offset decomposition for a KDA layer.
Homogeneous-TP geometry (conv/ssm slot strides, conv sub-projection
offsets, ssm per-slot size) is identical across requests and across the
homogeneous GDN layers, so it is computed once and reused; only the
per-request slot bases vary (see ``apply_mamba_offset_template``).
"""
num_slots: int
conv_slot_stride_bytes: int
ssm_slot_stride_bytes: int
ssm_read_bytes: int
conv_subprojs: tuple[tuple[int, int, int], ...]
def build_mamba_offset_template(
conv: torch.Tensor,
ssm: torch.Tensor,
split_info: MambaConvSplitInfo,
tp_ratio: int,
) -> MambaOffsetTemplate:
"""Precompute the slot-independent conv+ssm offset decomposition.
Only homogeneous TP (``tp_ratio == 1``) is supported; heterogeneous TP is
gated with ``NotImplementedError`` (see the design doc). The returned
template is a pure function of the layer geometry and ``split_info``, so it
can be cached and reused across requests without recomputation.
"""
if tp_ratio != 1:
# Heterogeneous-TP recurrent-state transfer needs the remote page's
# slot stride (which differs from the local page) and, for
# P_TP > D_TP, a multi-rank gather of conv/ssm slices. Both are
# follow-ups. The mamba path bypasses the attention KV-head validator,
# so fail loudly here instead of silently transferring wrong bytes.
raise NotImplementedError(
"heterogeneous-TP mamba/KDA transfer is not supported yet "
f"(tp_ratio={tp_ratio}); use equal prefill/decode TP for "
"hybrid models"
)
geom = get_mamba_transfer_geometry(conv, ssm)
conv_local_offsets = split_info.local_conv_offsets
conv_remote_offsets = split_info.remote_conv_offsets(0, 1)
ssm_read_bytes = geom.ssm_slot_bytes
conv_subprojs = tuple(
(loff, roff, rsz)
for (loff, _lsz), (roff, rsz) in zip(conv_local_offsets, conv_remote_offsets)
)
return MambaOffsetTemplate(
num_slots=conv.shape[0],
conv_slot_stride_bytes=geom.conv_slot_stride * geom.conv_element_size,
ssm_slot_stride_bytes=geom.ssm_slot_stride * geom.ssm_element_size,
ssm_read_bytes=ssm_read_bytes,
conv_subprojs=conv_subprojs,
)
def apply_mamba_offset_template(
template: MambaOffsetTemplate,
local_slots: list[int],
remote_slots: list[int],
) -> tuple[list[int], list[int], list[int]]:
"""Apply per-request slot bases to a cached ``MambaOffsetTemplate``.
Conv sub-projection entries (all slots) come first, followed by one ssm
entry per slot.
"""
if len(local_slots) != len(remote_slots):
raise ValueError(
"local and remote mamba slot counts must match: "
f"{len(local_slots)} != {len(remote_slots)}"
)
invalid_local = next(
(slot for slot in local_slots if not 0 <= slot < template.num_slots), None
)
if invalid_local is not None:
raise ValueError(
f"local mamba slot {invalid_local} is outside [0, {template.num_slots})"
)
invalid_remote = next((slot for slot in remote_slots if slot < 0), None)
if invalid_remote is not None:
raise ValueError(f"remote mamba slot {invalid_remote} must be non-negative")
# The peer's KDA slot count is not part of MoRIIOAgentMetadata yet. Do not
# compare remote slot IDs against the local pool size: two otherwise
# compatible engines may reserve different numbers of slots.
local_offs: list[int] = []
remote_offs: list[int] = []
sizes: list[int] = []
# Conv sub-projections (region 0), all slots first.
conv_slot_stride_bytes = template.conv_slot_stride_bytes
for ls, rs in zip(local_slots, remote_slots):
lbase = ls * conv_slot_stride_bytes
rbase = rs * conv_slot_stride_bytes
for loff, roff, rsz in template.conv_subprojs:
local_offs.append(lbase + loff)
remote_offs.append(rbase + roff)
sizes.append(rsz)
# SSM temporal state (region 1), one entry per slot.
ssm_slot_stride_bytes = template.ssm_slot_stride_bytes
ssm_read_bytes = template.ssm_read_bytes
for ls, rs in zip(local_slots, remote_slots):
local_offs.append(ls * ssm_slot_stride_bytes)
remote_offs.append(rs * ssm_slot_stride_bytes)
sizes.append(ssm_read_bytes)
return local_offs, remote_offs, sizes
def compute_mamba_conv_split_count(
local_slots: list[int],
split_info: MambaConvSplitInfo,
) -> int:
"""Number of leading conv entries in a KDA layer transfer plan.
Entries ``[:count]`` are conv sub-projections (conv region/session);
entries ``[count:]`` are the per-slot ssm state (ssm region/session).
"""
return len(local_slots) * len(split_info.local_conv_offsets)