diff --git a/examples/disaggregated/disaggregated_serving/moriio_toy_proxy_server.py b/examples/disaggregated/disaggregated_serving/moriio_toy_proxy_server.py index 8b2c46999e..ce4d558fb9 100644 --- a/examples/disaggregated/disaggregated_serving/moriio_toy_proxy_server.py +++ b/examples/disaggregated/disaggregated_serving/moriio_toy_proxy_server.py @@ -367,6 +367,8 @@ async def handle_request(api: str, request: Request): "remote_block_ids" ] req_data["kv_transfer_params"]["transfer_id"] = prefill_kv["transfer_id"] + if prefill_kv.get("tp_size") is not None: + req_data["kv_transfer_params"]["tp_size"] = prefill_kv["tp_size"] req_data["kv_transfer_params"]["remote_dp_size"] = prefill_instance_endpoint[ "dp_size" diff --git a/tests/v1/kv_connector/unit/test_moriio_connector.py b/tests/v1/kv_connector/unit/test_moriio_connector.py index a30b5caf4b..f95e12c601 100644 --- a/tests/v1/kv_connector/unit/test_moriio_connector.py +++ b/tests/v1/kv_connector/unit/test_moriio_connector.py @@ -775,6 +775,17 @@ def test_is_hma_required(swa_enabled, disable_hma, expected_is_hma): assert scheduler.get_exchange_clipped_blocks(blocks) == blocks +def test_transfer_disabled_hybrid_group_is_ignored(): + config = _make_hybrid_kv_cache_config() + config.kv_cache_groups[1].enable_kv_transfer = False + scheduler = _read_scheduler(config) + + assert scheduler._is_hma_required is False + assert scheduler.get_exchange_clipped_blocks([[1, 2, 3], [20, 21, 22]]) == [ + [1, 2, 3] + ] + + def test_non_sliding_window_hybrid_is_rejected(): """A hybrid group that is not sliding-window (e.g. chunked-local attention) must fail closed rather than be silently mistransferred.""" diff --git a/tests/v1/kv_connector/unit/test_moriio_hma_scheduler.py b/tests/v1/kv_connector/unit/test_moriio_hma_scheduler.py new file mode 100644 index 0000000000..13934f896a --- /dev/null +++ b/tests/v1/kv_connector/unit/test_moriio_hma_scheduler.py @@ -0,0 +1,809 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Pure-CPU unit tests for the MoRIIO hybrid (attention + mamba/KDA) scheduler +and the KDA offset-template cache. + +These exercise the highest-risk, otherwise-untested scheduler logic without a +GPU or the ``mori`` runtime: per-group block-id splitting, the READ/WRITE +``N-1`` token accounting, P-side prompt truncation, and offset-template cache +wiring. + +Like ``test_moriio_kv_layout.py`` the whole module is skipped unless it is +running on ROCm with ``mori`` installed (importing the connector pulls in +``mori``). The authoritative run happens on the MIA recipe image. +""" + +import importlib +import importlib.util +from types import SimpleNamespace + +import pytest +import torch + +from vllm.platforms import current_platform + +mori_available = importlib.util.find_spec("mori") is not None + +if not (current_platform.is_rocm() and mori_available): + pytest.skip( + "MoRIIOs are only available on ROCm with mori package installed", + allow_module_level=True, + ) + +moriio_connector = importlib.import_module( + "vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector" +) +moriio_layout = importlib.import_module( + "vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_layout" +) +ssm_conv_transfer_utils = importlib.import_module( + "vllm.distributed.kv_transfer.kv_connector.v1.ssm_conv_transfer_utils" +) +MambaConvSplitInfo = ssm_conv_transfer_utils.MambaConvSplitInfo +MoRIIOMode = moriio_connector.MoRIIOMode +moriio_common = importlib.import_module( + "vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_common" +) + + +class _FakeScheduler(moriio_connector.MoRIIOConnectorScheduler): # type: ignore[name-defined] + """Constructs a scheduler without the heavy real ``__init__``; only the + attributes read by the method under test are set.""" + + def __init__(self, **attrs): + self._mamba_group_ids: list[int] = [] + self._attn_group_ids: list[int] = [0] + self._ssm_state_slots_are_positional = False + self._is_hma_required = False + self.kv_cache_config = SimpleNamespace( + kv_cache_groups=[None, None], + select_transfer_block_ids=lambda block_ids: tuple(block_ids), + ) + for k, v in attrs.items(): + setattr(self, k, v) + + +class _FakeConnector(moriio_connector.MoRIIOConnector): # type: ignore[name-defined] + def __init__(self, connector_scheduler): + self.connector_scheduler = connector_scheduler + + +class _FakeWorker(moriio_connector.MoRIIOConnectorWorker): # type: ignore[name-defined] + def __init__(self, **attrs): + for k, v in attrs.items(): + setattr(self, k, v) + + +class _FakeBlocks: + def __init__(self, all_groups): + self._all_groups = all_groups + + def get_block_ids(self): + return self._all_groups + + +def _slot_strided(num_slots, per_slot_shape, slot_stride, dtype=torch.bfloat16): + inner = 1 + for d in per_slot_shape: + inner *= d + assert slot_stride >= inner + backing = torch.zeros(num_slots * slot_stride, dtype=dtype) + inner_strides: list[int] = [] + acc = 1 + for d in reversed(per_slot_shape): + inner_strides.insert(0, acc) + acc *= d + return backing.as_strided( + (num_slots, *per_slot_shape), (slot_stride, *inner_strides) + ) + + +def _gdn_split_info(conv_rows=3, key_dim=4, value_dim=8, dtype_size=2): + conv_dim = 2 * key_dim + value_dim + conv_state_bytes = conv_dim * conv_rows * dtype_size + ssm_state_bytes = 64 + return MambaConvSplitInfo( + conv_rows=conv_rows, + local_proj_dims=(key_dim, key_dim, value_dim), + conv_dtype_size=dtype_size, + ssm_sizes=(conv_state_bytes, ssm_state_bytes), + ) + + +def _mamba_spec(num_states: int = 2): + shapes = tuple((1, 1) for _ in range(num_states)) + return moriio_connector.MambaSpec( + block_size=16, + shapes=shapes, + dtypes=(torch.float32,) * num_states, + mamba_cache_mode="all", + ) + + +def _gate_vllm_config(*, read_mode: bool = True, speculative_config=None): + return SimpleNamespace( + kv_transfer_config=SimpleNamespace( + kv_connector_extra_config={"read_mode": read_mode} + ), + speculative_config=speculative_config, + ) + + +# -------------------------------------------------------------------------- +# split_block_groups +# -------------------------------------------------------------------------- +def test_split_block_groups_separates_attention_and_mamba(): + sched = _FakeScheduler(_has_mamba=True, _attn_group_ids=[0], _mamba_group_ids=[1]) + block_ids = ([1, 2], [99]) + attn, mamba = sched.split_block_groups(block_ids) + assert attn == [1, 2] + assert mamba == [[99]] + + +def test_split_block_groups_ignores_transfer_disabled_group(): + mamba_spec = moriio_connector.MambaSpec( + block_size=16, + shapes=((1, 1),), + dtypes=(torch.float32,), + mamba_cache_mode="all", + ) + config = SimpleNamespace( + kv_cache_groups=[ + SimpleNamespace(enable_kv_transfer=True, kv_cache_spec=object()), + SimpleNamespace(enable_kv_transfer=False, kv_cache_spec=object()), + SimpleNamespace(enable_kv_transfer=True, kv_cache_spec=mamba_spec), + ], + ) + config.transfer_groups = tuple( + group for group in config.kv_cache_groups if group.enable_kv_transfer + ) + config.select_transfer_block_ids = lambda block_ids: tuple( + block_ids[group_id] + for group_id, group in enumerate(config.kv_cache_groups) + if group.enable_kv_transfer + ) + attn_groups, mamba_groups = moriio_connector._split_kv_cache_group_kinds(config) + assert attn_groups == [0] + assert mamba_groups == [1] + + sched = _FakeScheduler( + _has_mamba=True, + _attn_group_ids=attn_groups, + _mamba_group_ids=mamba_groups, + kv_cache_config=config, + ) + assert sched.split_block_groups(([1, 2], [70, 71], [99])) == ( + [1, 2], + [[99]], + ) + + +def test_exchange_blocks_ignore_transfer_disabled_group(): + config = SimpleNamespace( + kv_cache_groups=[None, None, None], + select_transfer_block_ids=lambda block_ids: (block_ids[0], block_ids[2]), + ) + sched = _FakeScheduler( + kv_cache_config=config, + _is_hma_required=True, + blocks_per_sw=[0, 2], + ) + + assert sched.get_exchange_clipped_blocks(([1, 2], [70, 71], [90, 91, 92])) == [ + [1, 2], + [91, 92], + ] + + +def test_scheduler_rejects_multiple_attention_groups_with_mamba(): + config = SimpleNamespace( + kv_cache_groups=[ + SimpleNamespace(enable_kv_transfer=True, kv_cache_spec=object()), + SimpleNamespace(enable_kv_transfer=True, kv_cache_spec=object()), + SimpleNamespace(enable_kv_transfer=True, kv_cache_spec=_mamba_spec()), + ], + ) + config.transfer_groups = tuple(config.kv_cache_groups) + + with pytest.raises(moriio_common.MoRIIOError, match="exactly one transferable"): + moriio_connector.MoRIIOConnectorScheduler(_gate_vllm_config(), "engine", config) + + +def test_split_block_groups_preserves_multiple_mamba_groups(): + sched = _FakeScheduler( + _has_mamba=True, + _attn_group_ids=[0], + _mamba_group_ids=[1, 2], + ) + + assert sched.split_block_groups(([1, 2], [40], [70])) == ( + [1, 2], + [[40], [70]], + ) + + +def test_scheduler_rejects_incompatible_mamba_groups(): + incompatible = _mamba_spec() + object.__setattr__(incompatible, "page_size_padded", 4096) + config = SimpleNamespace( + kv_cache_groups=[ + SimpleNamespace(enable_kv_transfer=True, kv_cache_spec=object()), + SimpleNamespace(enable_kv_transfer=True, kv_cache_spec=_mamba_spec()), + SimpleNamespace(enable_kv_transfer=True, kv_cache_spec=incompatible), + ] + ) + config.transfer_groups = tuple(config.kv_cache_groups) + + with pytest.raises(moriio_common.MoRIIOError, match="same cache spec"): + moriio_connector.MoRIIOConnectorScheduler(_gate_vllm_config(), "engine", config) + + +def test_scheduler_rejects_mamba_group_without_two_states(): + config = SimpleNamespace( + kv_cache_groups=[ + SimpleNamespace(enable_kv_transfer=True, kv_cache_spec=object()), + SimpleNamespace(enable_kv_transfer=True, kv_cache_spec=_mamba_spec(1)), + ], + ) + config.transfer_groups = tuple(config.kv_cache_groups) + + with pytest.raises(moriio_common.MoRIIOError, match="exactly two Mamba states"): + moriio_connector.MoRIIOConnectorScheduler(_gate_vllm_config(), "engine", config) + + +def test_scheduler_rejects_speculative_hybrid_read(): + config = SimpleNamespace( + kv_cache_groups=[ + SimpleNamespace(enable_kv_transfer=True, kv_cache_spec=object()), + SimpleNamespace(enable_kv_transfer=True, kv_cache_spec=_mamba_spec()), + ], + ) + config.transfer_groups = tuple(config.kv_cache_groups) + + with pytest.raises(moriio_common.MoRIIOError, match="speculative decoding"): + moriio_connector.MoRIIOConnectorScheduler( + _gate_vllm_config(speculative_config=object()), "engine", config + ) + + +def test_scheduler_rejects_hybrid_write(): + config = SimpleNamespace( + kv_cache_groups=[ + SimpleNamespace(enable_kv_transfer=True, kv_cache_spec=object()), + SimpleNamespace(enable_kv_transfer=True, kv_cache_spec=_mamba_spec()), + ], + ) + config.transfer_groups = tuple(config.kv_cache_groups) + + with pytest.raises(moriio_common.MoRIIOError, match="READ mode only"): + moriio_connector.MoRIIOConnectorScheduler( + _gate_vllm_config(read_mode=False), "engine", config + ) + + +def test_split_block_groups_rejects_non_hybrid_use(): + sched = _FakeScheduler(_has_mamba=False) + + with pytest.raises(moriio_common.MoRIIOError, match="non-hybrid"): + sched.split_block_groups(([5, 6, 7],)) + + +def test_split_block_groups_accepts_empty_abort_payload(): + sched = _FakeScheduler(_has_mamba=True) + + assert sched.split_block_groups(()) == ([], []) + + +def test_split_block_groups_keeps_positional_slots_in_all_mode(): + # mamba_cache_mode="all" keeps a state per block position. + sched = _FakeScheduler( + _has_mamba=True, + _attn_group_ids=[0], + _mamba_group_ids=[1], + _ssm_state_slots_are_positional=True, + ) + attn, mamba = sched.split_block_groups(([1], [40, 41, 42])) + assert attn == [1] + assert mamba == [[40, 41, 42]] + + +def test_split_block_groups_keeps_only_running_state_outside_all_mode(): + sched = _FakeScheduler( + _has_mamba=True, + _attn_group_ids=[0], + _mamba_group_ids=[1], + ) + assert sched.split_block_groups(([1], [40, 41])) == ([1], [[41]]) + + +# -------------------------------------------------------------------------- +# request_finished_all_groups +# -------------------------------------------------------------------------- +def test_request_finished_all_groups_carries_attn_and_mamba_in_one_field(): + seen = {} + + def _fake_request_finished(request, attn_block_ids, mamba_block_groups): + seen["attn"] = list(attn_block_ids) + seen["mamba"] = [list(group) for group in mamba_block_groups] + return True, { + "do_remote_prefill": True, + "remote_block_ids": [attn_block_ids, *mamba_block_groups], + } + + sched = _FakeScheduler(_has_mamba=True, _attn_group_ids=[0], _mamba_group_ids=[1]) + sched.request_finished = _fake_request_finished + conn = _FakeConnector(sched) + + request = SimpleNamespace(request_id="r0") + block_ids = ([10, 11], [42]) + delay_free, params = conn.request_finished_all_groups(request, block_ids) + + assert delay_free is True + # Attention ids drive request_finished; the mamba slot rides the SAME + # remote_block_ids channel as [attn, *mamba_groups], so the proxy/router + # needs no KDA-specific field. + assert seen["attn"] == [10, 11] + assert seen["mamba"] == [[42]] + assert params["remote_block_ids"] == [[10, 11], [42]] + assert "remote_mamba_block_ids" not in params + + +def test_request_finished_all_groups_pure_attention_stays_grouped(): + def _fake_request_finished(request, block_ids): + return True, {"remote_block_ids": block_ids} + + sched = _FakeScheduler(_has_mamba=False, _mamba_group_ids=[]) + sched.request_finished = _fake_request_finished + conn = _FakeConnector(sched) + + delay_free, params = conn.request_finished_all_groups( + SimpleNamespace(request_id="r1"), ([7, 8],) + ) + # Main's HMA contract keeps pure-attention block ids grouped. + assert params["remote_block_ids"] == ([7, 8],) + assert "remote_mamba_block_ids" not in params + + +def test_connector_supports_divergent_local_hybrid_hits(): + connector = _FakeConnector(None) + connector._transfers_mamba_state = True + connector.mode = MoRIIOMode.READ + assert connector.supports_divergent_local_hybrid_hits is True + connector.mode = MoRIIOMode.WRITE + assert connector.supports_divergent_local_hybrid_hits is False + + connector.mode = MoRIIOMode.READ + connector._transfers_mamba_state = False + assert connector.supports_divergent_local_hybrid_hits is False + + +def _make_read_scheduler(): + return _FakeScheduler( + is_producer=False, + mode=MoRIIOMode.READ, + _has_mamba=True, + _attn_group_ids=[0], + _mamba_group_ids=[1], + _ssm_state_slots_are_positional=True, + request_id_to_transfer_id={}, + transfer_id_to_request_id={}, + _reqs_need_recv={}, + _req_kv_params={}, + _max_decode_tail_blocks=1, + ) + + +def _make_read_request(remote_block_ids): + return SimpleNamespace( + request_id="req", + kv_transfer_params={ + "do_remote_prefill": True, + "transfer_id": "tx", + "remote_engine_id": "prefill", + "remote_block_ids": remote_block_ids, + }, + ) + + +def test_update_state_drops_decode_recompute_tail_block(): + sched = _make_read_scheduler() + request = _make_read_request([[10, 11], [90, 91]]) + blocks = _FakeBlocks( + all_groups=([100, 101, 102], [200, 201, 202]), + ) + + sched.update_state_after_alloc(request, blocks, num_external_tokens=256) + + assert sched._reqs_need_recv["req"][1] == [[100, 101], [200, 201]] + assert sched._req_kv_params["req"]["remote_block_ids"] == [ + [10, 11], + [90, 91], + ] + + +def test_update_state_pairs_shorter_local_blocks_with_remote_suffix(): + sched = _make_read_scheduler() + request = _make_read_request([[10, 11, 12], [92]]) + blocks = _FakeBlocks( + all_groups=([102], [202]), + ) + + sched.update_state_after_alloc(request, blocks, num_external_tokens=64) + + assert sched._reqs_need_recv["req"][1] == [[102], [202]] + assert sched._req_kv_params["req"]["remote_block_ids"] == [[12], [92]] + + +def test_update_state_full_attention_hit_still_carries_mamba_state(): + sched = _make_read_scheduler() + request = _make_read_request([[10], [90]]) + + sched.update_state_after_alloc( + request, + _FakeBlocks(all_groups=([100], [200])), + num_external_tokens=0, + ) + + assert sched._reqs_need_recv["req"][1] == [[], [200]] + + +def test_update_state_preserves_each_mamba_group(): + sched = _make_read_scheduler() + sched._mamba_group_ids = [1, 2] + request = _make_read_request([[10], [90], [80]]) + + sched.update_state_after_alloc( + request, + _FakeBlocks(all_groups=([100], [200], [300])), + num_external_tokens=64, + ) + + assert sched._reqs_need_recv["req"][1] == [[100], [200], [300]] + assert sched._req_kv_params["req"]["remote_block_ids"] == [ + [10], + [90], + [80], + ] + + +def test_align_read_blocks_allows_configured_lookahead_tail(): + assert moriio_connector.MoRIIOConnectorScheduler._align_read_blocks( + [100, 101, 102], [10], max_decode_tail_blocks=2 + ) == ([100], [10]) + with pytest.raises(ValueError, match="allowed local tail"): + moriio_connector.MoRIIOConnectorScheduler._align_read_blocks( + [100, 101, 102], [10], max_decode_tail_blocks=1 + ) + + +def test_update_state_aligns_attention_only_decode_tail(): + sched = _make_read_scheduler() + sched._has_mamba = False + sched._mamba_group_ids = [] + request = _make_read_request([[10, 11]]) + + sched.update_state_after_alloc( + request, + _FakeBlocks(all_groups=([100, 101, 102],)), + num_external_tokens=256, + ) + + assert sched._reqs_need_recv["req"][1] == [[100, 101]] + assert sched._req_kv_params["req"]["remote_block_ids"] == [[10, 11]] + + +def test_read_producer_does_not_queue_write_only_save_state(): + sched = _FakeScheduler( + is_producer=True, + mode=MoRIIOMode.READ, + _has_mamba=True, + _attn_group_ids=[0], + _mamba_group_ids=[1], + request_id_to_transfer_id={}, + transfer_id_to_request_id={}, + _reqs_need_save={}, + _req_kv_params={}, + ) + request = SimpleNamespace( + request_id="req", + kv_transfer_params={"do_remote_decode": True, "transfer_id": "tx"}, + ) + + sched.update_state_after_alloc( + request, + _FakeBlocks(all_groups=([100, 101], [200])), + num_external_tokens=0, + ) + + assert sched._reqs_need_save == {} + assert sched._req_kv_params == {} + + +def test_write_producer_queues_grouped_attention_blocks(): + sched = _FakeScheduler( + is_producer=True, + mode=MoRIIOMode.WRITE, + _has_mamba=False, + _attn_group_ids=[0], + _mamba_group_ids=[], + request_id_to_transfer_id={}, + transfer_id_to_request_id={}, + _reqs_need_save={}, + _req_kv_params={}, + ) + request = SimpleNamespace( + request_id="req", + kv_transfer_params={"do_remote_decode": True, "transfer_id": "tx"}, + ) + + sched.update_state_after_alloc( + request, + _FakeBlocks(all_groups=([100, 101],)), + num_external_tokens=0, + ) + + assert sched._reqs_need_save["req"][1] == ([100, 101],) + assert sched._req_kv_params["req"] == request.kv_transfer_params + + +def test_session_build_rejects_per_layer_region_count_mismatch(): + worker = _FakeWorker( + built_write_session={}, + layer_name_to_local_kv_cache_metadata={"kda.0": ["conv", "ssm"]}, + layer_name_to_remote_kv_cache_metadata={"prefill": {"kda.0": ["conv"]}}, + ) + + with pytest.raises(moriio_common.MoRIIOError, match="registered 1 region"): + worker._get_built_session("prefill") + + +def test_register_kv_caches_rejects_hybrid_write_before_registration(): + worker = _FakeWorker(mode=MoRIIOMode.WRITE, _transfer_layer_names={"kda.0"}) + worker._is_mamba_layer = lambda _layer_name: True + + with pytest.raises(moriio_common.MoRIIOError, match="READ mode only"): + worker.register_kv_caches({"kda.0": object()}) + + +def test_register_kv_caches_rejects_no_transfer_enabled_layers(): + worker = _FakeWorker(mode=MoRIIOMode.READ, _transfer_layer_names=set()) + + with pytest.raises(moriio_common.MoRIIOError, match="no transfer-enabled"): + worker.register_kv_caches({"disabled.0": object()}) + + +def test_worker_rejects_incompatible_mamba_specs_before_registration(): + incompatible = _mamba_spec() + object.__setattr__(incompatible, "page_size_padded", 4096) + worker = _FakeWorker( + mode=MoRIIOMode.READ, + world_size=1, + _transfer_layer_names={"kda.0", "kda.1"}, + layer_to_spec={"kda.0": _mamba_spec(), "kda.1": incompatible}, + ) + + with pytest.raises(moriio_common.MoRIIOError, match="same cache spec"): + worker.register_kv_caches({"kda.0": object(), "kda.1": object()}) + + +# -------------------------------------------------------------------------- +# _truncate_mamba_request_for_prefill +# -------------------------------------------------------------------------- +def _mk_request(prompt, max_tokens=64, params=None): + return SimpleNamespace( + kv_transfer_params={} if params is None else params, + num_prompt_tokens=len(prompt), + prompt_token_ids=list(prompt), + prompt_embeds=None, + _all_token_ids=list(prompt), + max_tokens=max_tokens, + ) + + +def test_truncate_mamba_request_pops_last_token_and_caps_tokens(): + sched = _FakeScheduler() + req = _mk_request([10, 11, 12, 13, 14]) + sched._truncate_mamba_request_for_prefill(req) + + assert req.prompt_token_ids == [10, 11, 12, 13] # last token popped + assert req._all_token_ids == [10, 11, 12, 13] + assert req.num_prompt_tokens == 4 # N-1 accounting + assert req.max_tokens == 1 + assert req.kv_transfer_params["_p_side_truncated"] is True + + +def test_truncate_mamba_request_is_idempotent_across_reschedule(): + sched = _FakeScheduler() + req = _mk_request([10, 11, 12]) + sched._truncate_mamba_request_for_prefill(req) + assert req.num_prompt_tokens == 2 + # A second call (e.g. after a preemption) must not truncate again. + sched._truncate_mamba_request_for_prefill(req) + assert req.num_prompt_tokens == 2 + assert req.prompt_token_ids == [10, 11] + + +def test_truncate_mamba_request_noop_for_single_token_prompt(): + sched = _FakeScheduler() + req = _mk_request([10]) + sched._truncate_mamba_request_for_prefill(req) + assert req.num_prompt_tokens == 1 + assert req.prompt_token_ids == [10] + assert "_p_side_truncated" not in req.kv_transfer_params + + +# -------------------------------------------------------------------------- +# get_num_new_matched_tokens (hybrid N-1 accounting) +# -------------------------------------------------------------------------- +def test_get_num_new_matched_tokens_read_recomputes_last_token(): + sched = _FakeScheduler(is_producer=False, mode=MoRIIOMode.READ, _has_mamba=True) + req = SimpleNamespace( + num_prompt_tokens=10, + prompt_token_ids=list(range(10)), + kv_transfer_params=None, + ) + n, is_async = sched.get_num_new_matched_tokens(req, num_computed_tokens=0) + # READ always recomputes the final token locally: N-1 - computed. + assert n == 9 + assert is_async is False + + +def test_get_num_new_matched_tokens_write_plain_keeps_all_tokens(): + sched = _FakeScheduler(is_producer=False, mode=MoRIIOMode.WRITE, _has_mamba=False) + req = SimpleNamespace( + num_prompt_tokens=10, + prompt_token_ids=list(range(10)), + kv_transfer_params=None, + ) + n, is_async = sched.get_num_new_matched_tokens(req, num_computed_tokens=2) + # Pure-attention WRITE: no N-1 drop; full length minus already-computed. + assert n == 8 + assert is_async is True + + +@pytest.mark.parametrize( + ("mode", "num_computed_tokens", "expected", "is_async"), + [ + (MoRIIOMode.READ, 0, 9, False), + (MoRIIOMode.READ, 10, 0, False), + (MoRIIOMode.WRITE, 2, 8, True), + ], +) +def test_get_num_new_matched_tokens_supports_embeds_only_prompts( + mode, num_computed_tokens, expected, is_async +): + sched = _FakeScheduler(is_producer=False, mode=mode, _has_mamba=True) + req = SimpleNamespace( + num_prompt_tokens=10, + prompt_token_ids=None, + prompt_embeds=object(), + kv_transfer_params=None, + ) + + assert sched.get_num_new_matched_tokens(req, num_computed_tokens) == ( + expected, + is_async, + ) + + +def test_get_num_new_matched_tokens_producer_returns_zero(): + sched = _FakeScheduler(is_producer=True, mode=MoRIIOMode.WRITE, _has_mamba=True) + req = _mk_request(list(range(5)), params={"do_remote_decode": True}) + n, is_async = sched.get_num_new_matched_tokens(req, num_computed_tokens=0) + assert (n, is_async) == (0, False) + # The producer never truncates here: the scheduler has already measured the + # prefix-cache hit against the full prompt by this point. + assert req.num_prompt_tokens == 5 + + +def test_on_new_request_truncates_producer_prompt(): + sched = _FakeScheduler(is_producer=True, mode=MoRIIOMode.WRITE, _has_mamba=True) + req = _mk_request(list(range(5)), params={"do_remote_decode": True}) + sched.on_new_request(req) + # Producer stops at h(N-1): the last prompt token was dropped. + assert req.num_prompt_tokens == 4 + assert req.max_tokens == 1 + assert req.kv_transfer_params["_p_side_truncated"] is True + + +def test_on_new_request_is_noop_without_mamba_or_on_decode(): + plain = _FakeScheduler(is_producer=True, mode=MoRIIOMode.WRITE, _has_mamba=False) + req = _mk_request(list(range(5)), params={"do_remote_decode": True}) + plain.on_new_request(req) + assert req.num_prompt_tokens == 5 + + decode_side = _FakeScheduler( + is_producer=False, mode=MoRIIOMode.WRITE, _has_mamba=True + ) + req2 = _mk_request(list(range(5)), params={"do_remote_prefill": True}) + decode_side.on_new_request(req2) + assert req2.num_prompt_tokens == 5 + + +# -------------------------------------------------------------------------- +# Offset-template cache +# -------------------------------------------------------------------------- +_SAMPLE_SLOT_SETS = [ + ([1, 2], [1, 2]), + ([0], [3]), + ([3, 0, 2], [3, 0, 2]), + ([2, 2], [5, 5]), + ([], []), +] + + +def test_worker_compute_mamba_offsets_caches_and_matches_template(): + split = _gdn_split_info() + conv_dim = sum(split.local_proj_dims) + conv_shape = (conv_dim, split.conv_rows) + ssm_shape = (2, 4, 4) + page_bytes = 256 + pages = torch.zeros((8, 1, 1, page_bytes), dtype=torch.uint8) + spec = moriio_connector.MambaSpec( + block_size=1, + shapes=(conv_shape, ssm_shape), + dtypes=(torch.bfloat16, torch.bfloat16), + page_size_padded=page_bytes, + ) + conv, ssm = moriio_layout.kda_conv_ssm(pages, spec) + + worker = _FakeWorker( + kv_caches={"kda.0": pages}, + layer_to_spec={"kda.0": spec}, + _conv_decomp=split, + _mamba_offset_templates={}, + tp_rank=0, + world_size=1, + ) + + cached_template = None + for local_slots, remote_slots in _SAMPLE_SLOT_SETS: + lo, ro, sz, n_conv = worker._compute_mamba_transfer_offsets( + "kda.0", local_slots, remote_slots + ) + template = moriio_layout.build_mamba_offset_template( + conv, ssm, split, tp_ratio=1 + ) + expected = moriio_layout.apply_mamba_offset_template( + template, local_slots, remote_slots + ) + assert (lo, ro, sz) == expected + assert n_conv == moriio_layout.compute_mamba_conv_split_count( + local_slots, split + ) + + current_template = worker._mamba_offset_templates["kda.0"] + if cached_template is None: + cached_template = current_template + else: + assert current_template is cached_template + + assert "kda.0" in worker._mamba_offset_templates + + +@pytest.mark.parametrize("remote_tp_size", [1, None]) +def test_mamba_tp_ratio_accepts_only_equal_tp(remote_tp_size): + worker = _FakeWorker(world_size=1) + assert worker._mamba_tp_ratio(remote_tp_size) == 1 + + +@pytest.mark.parametrize("remote_tp_size", [2, 3]) +def test_mamba_tp_ratio_rejects_heterogeneous_tp(remote_tp_size): + worker = _FakeWorker(world_size=1) + with pytest.raises(NotImplementedError, match="heterogeneous-TP"): + worker._mamba_tp_ratio(remote_tp_size) + + +def test_worker_selects_each_mamba_layers_own_block_group(): + worker = _FakeWorker(_mamba_payload_index_by_layer={"kda.0": 1, "kda.1": 2}) + groups = ([10], [40], [70]) + + assert worker._mamba_blocks_for_layer("kda.0", groups) == [40] + assert worker._mamba_blocks_for_layer("kda.1", groups) == [70] + + +def test_worker_rejects_missing_mamba_block_group(): + worker = _FakeWorker(_mamba_payload_index_by_layer={"kda.0": 2}) + + with pytest.raises(moriio_common.MoRIIOError, match="no block group"): + worker._mamba_blocks_for_layer("kda.0", ([10], [40])) diff --git a/tests/v1/kv_connector/unit/test_moriio_kv_layout.py b/tests/v1/kv_connector/unit/test_moriio_kv_layout.py index c9fa434ee9..1cb5e271bf 100644 --- a/tests/v1/kv_connector/unit/test_moriio_kv_layout.py +++ b/tests/v1/kv_connector/unit/test_moriio_kv_layout.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import importlib.util +import math import threading from collections import OrderedDict, defaultdict from queue import Queue @@ -14,6 +15,7 @@ import torch from vllm.platforms import current_platform from vllm.v1.kv_cache_interface import ( FullAttentionSpec, + MambaSpec, MLAAttentionSpec, ) @@ -36,6 +38,10 @@ moriio_layout = importlib.import_module( "vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_layout" ) msgpack = importlib.import_module("msgpack") +ssm_conv_transfer_utils = importlib.import_module( + "vllm.distributed.kv_transfer.kv_connector.v1.ssm_conv_transfer_utils" +) +MambaConvSplitInfo = ssm_conv_transfer_utils.MambaConvSplitInfo ROLE = moriio_common.ROLE MoRIIOError = moriio_common.MoRIIOError @@ -353,6 +359,9 @@ def test_write_transfer_plan_caches_offsets_per_layer_geometry(): kv_caches: dict[str, torch.Tensor] layer_name_to_local_kv_cache_metadata: dict[str, list[Any]] + def _region_session_indices(self, layer_name): + return [list(self.kv_caches).index(layer_name)] + def _compute_block_transfer_offsets( self, layer_name, @@ -673,6 +682,30 @@ def test_local_block_ids_longer_than_remote_raises_value_error(): ) +@pytest.mark.parametrize( + ("local_block_ids", "remote_block_ids", "match"), + [ + ([-1], [0], "local block id -1"), + ([0], [8], "remote block id 8"), + ], +) +def test_block_ids_must_fit_registered_regions( + local_block_ids, remote_block_ids, match +): + cache = torch.empty((8, 2, 4, 2, 3), dtype=torch.bfloat16) + worker = _worker({"layer": cache}, {"layer": _full_spec()}) + + with pytest.raises(ValueError, match=match): + moriio_layout.compute_block_transfer_offsets( + "layer", + cache, + worker.layer_to_spec, + local_block_ids, + remote_block_ids, + _remote_meta(num_blocks=8).num_blocks, + ) + + def test_empty_local_block_ids_is_free_only_noop(): cache = torch.empty((8, 2, 4, 2, 3), dtype=torch.bfloat16) worker = _worker({"layer": cache}, {"layer": _full_spec()}) @@ -745,3 +778,245 @@ def test_unsupported_shape_raises_value_error(): with pytest.raises(ValueError, match="Unsupported MoRIIO K/V cache shape"): moriio_layout.get_layer_transfer_geometry("layer", cache, worker.layer_to_spec) + + +# --------------------------------------------------------------------------- +# Mamba/KDA (conv + ssm) transfer geometry +# --------------------------------------------------------------------------- + + +def _slot_strided(num_slots, per_slot_shape, slot_stride, dtype=torch.bfloat16): + """Build a slot-strided view: shape (num_slots, *per_slot_shape) whose slot + stride (dim 0) exceeds the per-slot element count, mimicking the K3 KDA + conv/ssm cache tensors.""" + inner = 1 + for d in per_slot_shape: + inner *= d + assert slot_stride >= inner + backing = torch.zeros(num_slots * slot_stride, dtype=dtype) + inner_strides: list[int] = [] + acc = 1 + for d in reversed(per_slot_shape): + inner_strides.insert(0, acc) + acc *= d + return backing.as_strided( + (num_slots, *per_slot_shape), (slot_stride, *inner_strides) + ) + + +def _gdn_split_info(conv_rows=3, key_dim=4, value_dim=8, dtype_size=2): + """A GDN-style conv split: sub-projections (key, key, value).""" + conv_dim = 2 * key_dim + value_dim + conv_state_bytes = conv_dim * conv_rows * dtype_size + ssm_state_bytes = 64 # opaque here; ssm size is taken from the tensor + return MambaConvSplitInfo( + conv_rows=conv_rows, + local_proj_dims=(key_dim, key_dim, value_dim), + conv_dtype_size=dtype_size, + ssm_sizes=(conv_state_bytes, ssm_state_bytes), + ) + + +def test_mamba_transfer_geometry_is_slot_strided(): + split = _gdn_split_info() + conv_dim = sum(split.local_proj_dims) + conv = _slot_strided(5, (conv_dim, split.conv_rows), slot_stride=200) + ssm = _slot_strided(5, (2, 4, 4), slot_stride=64) + + geom = moriio_layout.get_mamba_transfer_geometry(conv, ssm) + + assert geom.conv_slot_stride == 200 + assert geom.ssm_slot_stride == 64 + assert geom.ssm_slot_bytes == (2 * 4 * 4) * ssm.element_size() + # Region spans the full slot-strided extent (shape[0] * stride(0)), not just + # numel(), so the highest-slot transfers stay in-bounds. + assert geom.conv_region_len == conv.shape[0] * conv.stride(0) * conv.element_size() + assert geom.ssm_region_len == ssm.shape[0] * ssm.stride(0) * ssm.element_size() + + +def test_kda_conv_ssm_unpacks_padded_pages_as_aliasing_strided_views(): + num_blocks = 4 + page_bytes = 128 + conv_shape = (3, 4) + ssm_shape = (2, 2, 4) + conv_bytes = math.prod(conv_shape) * torch.bfloat16.itemsize + ssm_bytes = math.prod(ssm_shape) * torch.bfloat16.itemsize + assert conv_bytes + ssm_bytes < page_bytes + + pages = torch.zeros((num_blocks, 1, 1, page_bytes), dtype=torch.uint8) + spec = MambaSpec( + block_size=1, + shapes=(conv_shape, ssm_shape), + dtypes=(torch.bfloat16, torch.bfloat16), + page_size_padded=page_bytes, + ) + + conv, ssm = moriio_layout.kda_conv_ssm(pages, spec) + + assert conv.shape == (num_blocks, *conv_shape) + assert ssm.shape == (num_blocks, *ssm_shape) + assert conv.stride() == (page_bytes // 2, 4, 1) + assert ssm.stride() == (page_bytes // 2, 8, 4, 1) + assert conv.untyped_storage().data_ptr() == pages.untyped_storage().data_ptr() + assert ssm.untyped_storage().data_ptr() == pages.untyped_storage().data_ptr() + assert conv.storage_offset() == 0 + assert ssm.storage_offset() * ssm.element_size() == conv_bytes + + conv[1].fill_(1) + ssm[2].fill_(2) + assert torch.all(pages[1, 0, 0, :conv_bytes].view(torch.bfloat16) == 1) + assert torch.all( + pages[2, 0, 0, conv_bytes : conv_bytes + ssm_bytes].view(torch.bfloat16) == 2 + ) + + +def test_kda_conv_ssm_accepts_strided_blocks_without_copying(): + backing = torch.zeros(4 * 256, dtype=torch.uint8) + pages = backing.as_strided((4, 1, 1, 128), (256, 128, 128, 1)) + spec = MambaSpec( + block_size=1, + shapes=((3, 4), (2, 2, 4)), + dtypes=(torch.bfloat16, torch.bfloat16), + page_size_padded=128, + ) + + conv, ssm = moriio_layout.kda_conv_ssm(pages, spec) + + assert not pages.is_contiguous() + assert conv.stride(0) == 128 + assert ssm.stride(0) == 128 + assert conv.untyped_storage().data_ptr() == backing.data_ptr() + assert ssm.untyped_storage().data_ptr() == backing.data_ptr() + + +def test_kda_conv_ssm_rejects_noncontiguous_page_bytes(): + pages = torch.zeros((4, 1, 1, 256), dtype=torch.uint8)[..., ::2] + spec = MambaSpec( + block_size=1, + shapes=((3, 4), (2, 2, 4)), + dtypes=(torch.bfloat16, torch.bfloat16), + page_size_padded=128, + ) + + with pytest.raises(ValueError, match="contiguous byte dimension"): + moriio_layout.kda_conv_ssm(pages, spec) + + +def test_kda_conv_ssm_rejects_non_byte_packed_cache(): + pages = torch.zeros((4, 1, 1, 128), dtype=torch.bfloat16) + spec = MambaSpec( + block_size=1, + shapes=((3, 4), (2, 2, 4)), + dtypes=(torch.bfloat16, torch.bfloat16), + page_size_padded=128, + ) + + with pytest.raises(ValueError, match="byte-sized dtype"): + moriio_layout.kda_conv_ssm(pages, spec) + + +def test_mamba_transfer_geometry_requires_matching_slot_counts(): + conv = torch.empty((5, 4, 3), dtype=torch.bfloat16) + ssm = torch.empty((4, 2, 4, 4), dtype=torch.bfloat16) + + with pytest.raises(ValueError, match="conv/ssm slot count mismatch"): + moriio_layout.get_mamba_transfer_geometry(conv, ssm) + + +def test_mamba_transfer_geometry_rejects_empty_caches(): + conv = torch.empty((0, 4, 3), dtype=torch.bfloat16) + ssm = torch.empty((0, 2, 4, 4), dtype=torch.bfloat16) + + with pytest.raises(ValueError, match="at least one slot"): + moriio_layout.get_mamba_transfer_geometry(conv, ssm) + + +def test_mamba_conv_ssm_offsets_homogeneous_tp(): + split = _gdn_split_info() + conv_dim = sum(split.local_proj_dims) + conv = _slot_strided(6, (conv_dim, split.conv_rows), slot_stride=200) + ssm = _slot_strided(6, (2, 4, 4), slot_stride=64) + geom = moriio_layout.get_mamba_transfer_geometry(conv, ssm) + + local_slots = [1, 2] + remote_slots = [1, 2] + template = moriio_layout.build_mamba_offset_template(conv, ssm, split, tp_ratio=1) + local_offs, remote_offs, sizes = moriio_layout.apply_mamba_offset_template( + template, local_slots, remote_slots + ) + + n_conv = moriio_layout.compute_mamba_conv_split_count(local_slots, split) + assert n_conv == len(local_slots) * len(split.local_proj_dims) + assert len(local_offs) == n_conv + len(local_slots) + + # Homogeneous TP: remote conv slices equal local slices; since local==remote + # slots, conv local and remote offsets coincide. + loc_co = split.local_conv_offsets + rem_co = split.remote_conv_offsets(0, 1) + k = 0 + esz = conv.element_size() + for ls, rs in zip(local_slots, remote_slots): + lbase = ls * geom.conv_slot_stride * esz + rbase = rs * geom.conv_slot_stride * esz + for (loff, _), (roff, rsz) in zip(loc_co, rem_co): + assert local_offs[k] == lbase + loff + assert remote_offs[k] == rbase + roff + assert sizes[k] == rsz + k += 1 + + # SSM: whole per-slot block, no head sharding at tp_ratio == 1. + essz = ssm.element_size() + for j, (ls, rs) in enumerate(zip(local_slots, remote_slots)): + assert local_offs[n_conv + j] == ls * geom.ssm_slot_stride * essz + assert remote_offs[n_conv + j] == rs * geom.ssm_slot_stride * essz + assert sizes[n_conv + j] == geom.ssm_slot_bytes + + +def test_mamba_conv_ssm_offsets_heterogeneous_tp_is_gated(): + # Heterogeneous-TP mamba transfer is a documented follow-up (remote slot + # stride + multi-rank fan-in gather). Until then the offset helper must fail + # loudly rather than silently transfer wrong bytes, since the mamba path + # bypasses the attention KV-head validator. + split = _gdn_split_info() + conv_dim = sum(split.local_proj_dims) + conv = _slot_strided(4, (conv_dim, split.conv_rows), slot_stride=200) + ssm = _slot_strided(4, (2, 4, 4), slot_stride=64) + with pytest.raises(NotImplementedError): + moriio_layout.build_mamba_offset_template(conv, ssm, split, tp_ratio=2) + + +def test_local_mamba_slots_must_fit_registered_regions(): + split = _gdn_split_info() + conv_dim = sum(split.local_proj_dims) + conv = _slot_strided(6, (conv_dim, split.conv_rows), slot_stride=200) + ssm = _slot_strided(6, (2, 4, 4), slot_stride=64) + template = moriio_layout.build_mamba_offset_template(conv, ssm, split, tp_ratio=1) + + with pytest.raises(ValueError, match="local mamba slot 6"): + moriio_layout.apply_mamba_offset_template(template, [6], [0]) + + +@pytest.mark.parametrize( + ("local_slots", "remote_slots"), + [([1, 2], [3]), ([1], [2, 3])], +) +def test_mamba_slot_counts_must_match(local_slots, remote_slots): + split = _gdn_split_info() + conv_dim = sum(split.local_proj_dims) + conv = _slot_strided(6, (conv_dim, split.conv_rows), slot_stride=200) + ssm = _slot_strided(6, (2, 4, 4), slot_stride=64) + template = moriio_layout.build_mamba_offset_template(conv, ssm, split, tp_ratio=1) + + with pytest.raises(ValueError, match="slot counts must match"): + moriio_layout.apply_mamba_offset_template(template, local_slots, remote_slots) + + +def test_remote_mamba_slots_must_be_non_negative(): + split = _gdn_split_info() + conv_dim = sum(split.local_proj_dims) + conv = _slot_strided(6, (conv_dim, split.conv_rows), slot_stride=200) + ssm = _slot_strided(6, (2, 4, 4), slot_stride=64) + template = moriio_layout.build_mamba_offset_template(conv, ssm, split, tp_ratio=1) + + with pytest.raises(ValueError, match="must be non-negative"): + moriio_layout.apply_mamba_offset_template(template, [1], [-1]) diff --git a/tests/v1/kv_connector/unit/test_moriio_routing.py b/tests/v1/kv_connector/unit/test_moriio_routing.py index fc8b323218..b0ac6df3cb 100644 --- a/tests/v1/kv_connector/unit/test_moriio_routing.py +++ b/tests/v1/kv_connector/unit/test_moriio_routing.py @@ -28,6 +28,7 @@ def _producer(global_dp_rank: int, dp_size: int, dp_size_local: int): s = SimpleNamespace( is_producer=True, _is_hma_required=False, + _has_mamba=False, blocks_per_sw=[0], _global_dp_rank=global_dp_rank, engine_id="engine-abc", diff --git a/tests/v1/kv_connector/unit/test_moriio_tp_ack.py b/tests/v1/kv_connector/unit/test_moriio_tp_ack.py index da412fcd92..a35f0a1169 100644 --- a/tests/v1/kv_connector/unit/test_moriio_tp_ack.py +++ b/tests/v1/kv_connector/unit/test_moriio_tp_ack.py @@ -2,12 +2,14 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import threading +from types import SimpleNamespace import pytest from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_common import ( MoRIIOMode, MoRIIOTransferAck, + TransferError, ) from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector import ( MoRIIOConnector, @@ -17,6 +19,9 @@ from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector import resolve_moriio_transfer_ack, validate_moriio_heterogeneous_tp_kv_heads, ) +from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_engine import ( + MoRIIOWrapper, +) def test_remote_tp_rank_same_tp_maps_to_self(): @@ -277,6 +282,9 @@ def test_read_completion_sends_structured_release_with_consumer_tp_size(): self.lock = threading.Lock() self.sent = [] + # Exercise the real batch verdict rather than restating it here. + poll_transfer_batch = MoRIIOWrapper.poll_transfer_batch + def send_notify( self, transfer_id, @@ -292,13 +300,18 @@ def test_read_completion_sends_structured_release_with_consumer_tp_size(): worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker) worker.world_size = 8 + worker.moriio_config = SimpleNamespace(recv_abort_timeout=600.0) worker.moriio_wrapper = FakeWrapper() - worker._recving_transfers = {"req": {"layer0": DoneStatus()}} + # A layer maps to the list of reads posted for it (a KDA layer posts two). + worker._recving_transfers = {"req": {"layer0": [DoneStatus(), DoneStatus()]}} worker._recving_transfers_callback_addr = { "req": ("127.0.0.1", "7000", "tx-release") } # Transfer-timeout reaping state consulted by _pop_done_transfers. worker._recving_transfers_start = {} + # Load-error bookkeeping cleared alongside a completed transfer. + worker._recving_local_blocks = {} + worker._invalid_block_ids = set() assert worker._pop_done_transfers() == {"tx-release"} assert worker.moriio_wrapper.sent == [ @@ -314,6 +327,133 @@ def test_read_completion_sends_structured_release_with_consumer_tp_size(): assert worker._recving_transfers_callback_addr == {} +def test_aborted_read_without_local_blocks_only_releases_remote_blocks(): + class FakeWrapper: + def __init__(self): + self.sent = [] + + def send_notify( + self, + transfer_id, + host, + port, + message_type=None, + message_fields=None, + ): + self.sent.append((transfer_id, host, port, message_type, message_fields)) + + def shutdown(self): + pass + + worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker) + worker.mode = MoRIIOMode.READ + worker.world_size = 8 + worker.moriio_wrapper = FakeWrapper() + + worker._read_blocks( + local_block_ids=[], + remote_block_ids=[[10, 11], [90]], + dst_engine_id="prefill", + request_id="req-aborted", + transfer_id="tx-aborted", + remote_host="127.0.0.1", + remote_notify_port=7000, + remote_tp_size=8, + remote_dp_rank=0, + chosen_tp=2, + ) + + assert worker.moriio_wrapper.sent == [ + ( + "tx-aborted", + "127.0.0.1", + "7002", + "release", + {"consumer_tp_size": 8}, + ) + ] + + +def test_read_completion_waits_for_every_posted_read(): + class Status: + def __init__(self, done: bool): + self.done = done + + def Succeeded(self): + return self.done + + def Failed(self): + return False + + class FakeWrapper: + def __init__(self): + self.lock = threading.Lock() + self.sent = [] + + poll_transfer_batch = MoRIIOWrapper.poll_transfer_batch + + def send_notify(self, *args, **kwargs): + self.sent.append((args, kwargs)) + + def shutdown(self): + pass + + worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker) + worker.world_size = 8 + worker.moriio_config = SimpleNamespace(recv_abort_timeout=600.0) + worker.moriio_wrapper = FakeWrapper() + worker._recving_transfers = { + "req": {"kda_layer": [Status(done=True), Status(done=False)]} + } + worker._recving_transfers_callback_addr = { + "req": ("127.0.0.1", "7000", "tx-pending") + } + worker._recving_transfers_start = {"req": float("inf")} + worker._recving_local_blocks = {"req": [1, 2]} + worker._invalid_block_ids = set() + + assert worker._pop_done_transfers() == set() + assert "req" in worker._recving_transfers + assert worker.moriio_wrapper.sent == [] + + +@pytest.mark.parametrize( + ("has_mamba", "expected_invalid"), + [(False, {1, 2}), (True, set())], +) +def test_failed_read_reports_blocks_only_without_hma(has_mamba, expected_invalid): + worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker) + worker._has_mamba = has_mamba + worker._recving_local_blocks = {"req": [1, 2]} + worker._invalid_block_ids = set() + + worker._record_failed_recv("req") + + assert worker.get_block_ids_with_load_errors() == expected_invalid + + +def test_hybrid_step_barrier_fails_closed(monkeypatch): + class FailingWrapper: + def waiting_for_transfer_complete(self, _statuses): + raise TransferError("failed") + + def shutdown(self): + pass + + worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker) + worker._has_mamba = True + worker._reads_issued_this_step = [object()] + worker._mamba_reads_this_step = [object()] + worker.moriio_wrapper = FailingWrapper() + monkeypatch.setattr( + "vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector.get_forward_context", + lambda: SimpleNamespace(cudagraph_runtime_mode=None), + ) + + with pytest.raises(TransferError, match="failed"): + worker._await_reads_issued_this_step() + + def test_requested_cudagraph_mode_is_never_overridden(): # The configured cudagraph mode is always honored: the barrier fires when # the operator sets cudagraph_mode=PIECEWISE, and READ mode with full diff --git a/tests/v1/kv_connector/unit/test_moriio_transfer_completion.py b/tests/v1/kv_connector/unit/test_moriio_transfer_completion.py new file mode 100644 index 0000000000..2a3092ad5a --- /dev/null +++ b/tests/v1/kv_connector/unit/test_moriio_transfer_completion.py @@ -0,0 +1,129 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Transfer-completion semantics of the MoRIIO wrapper. + +Covers the two behaviours the READ barrier depends on: + * poll_transfer_batch considers EVERY status, not just the newest one + * waiting_for_transfer_complete blocks until terminal and raises on failure. +""" + +import importlib +import importlib.util +import threading +import time + +import pytest + +from vllm.platforms import current_platform + +mori_available = importlib.util.find_spec("mori") is not None + +if not (current_platform.is_rocm() and mori_available): + pytest.skip( + "MoRIIOs are only available on ROCm with mori package installed", + allow_module_level=True, + ) + +moriio_common = importlib.import_module( + "vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_common" +) +moriio_engine = importlib.import_module( + "vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_engine" +) + +MoRIIOWrapper = moriio_engine.MoRIIOWrapper +TransferBatchState = moriio_common.TransferBatchState +TransferError = moriio_common.TransferError + + +@pytest.mark.parametrize("value", [0, -1, float("inf"), float("nan")]) +def test_timeout_must_be_positive_and_finite(value): + with pytest.raises(ValueError, match="must be finite and greater than zero"): + moriio_common._positive_finite_timeout("test_timeout", value) + + +def test_positive_finite_timeout_is_preserved(): + assert moriio_common._positive_finite_timeout("test_timeout", "1.5") == 1.5 + + +class FakeStatus: + """Mirrors the mori TransferStatus surface the wrapper uses.""" + + def __init__(self, state: str = "in_progress", message: str = ""): + self.state = state + self.message = message + + def Succeeded(self) -> bool: + return self.state == "success" + + def Failed(self) -> bool: + return self.state == "failed" + + def Message(self) -> str: + return self.message + + def Code(self) -> str: + return self.state + + +def _ok(): + return FakeStatus("success") + + +def _pending(): + return FakeStatus("in_progress") + + +def _bad(): + return FakeStatus("failed", "SQ full") + + +@pytest.mark.parametrize( + "statuses,expected", + [ + ([], TransferBatchState.DONE), + ([_ok(), _ok()], TransferBatchState.DONE), + # Each case below puts the interesting status BEFORE a success, which a + # statuses[-1] check would have missed entirely. + ([_bad(), _ok()], TransferBatchState.FAILED), + ([_pending(), _ok()], TransferBatchState.PENDING), + ([_pending(), _bad(), _ok()], TransferBatchState.FAILED), + ], +) +def test_poll_transfer_batch_considers_every_status(statuses, expected): + wrapper = MoRIIOWrapper(moriio_engine=object(), transfer_timeout=0.05) + assert wrapper.poll_transfer_batch(statuses) is expected + + +@pytest.mark.parametrize( + "statuses_factory,expected_message", + [ + (lambda: [_ok(), _bad()], "SQ full"), + (lambda: [_ok(), _pending()], "timed out"), + ], + ids=["failed", "timeout"], +) +def test_wait_raises_with_detail(statuses_factory, expected_message): + wrapper = MoRIIOWrapper(moriio_engine=object(), transfer_timeout=0.05) + with pytest.raises(TransferError, match=expected_message): + wrapper.waiting_for_transfer_complete(statuses_factory()) + + +def test_wait_returns_immediately_when_nothing_pending(): + wrapper = MoRIIOWrapper(moriio_engine=object(), transfer_timeout=0.05) + wrapper.waiting_for_transfer_complete([]) + wrapper.waiting_for_transfer_complete([_ok(), _ok()]) + + +def test_wait_blocks_until_terminal(): + """The barrier is only worth anything if it actually waits.""" + wrapper = MoRIIOWrapper(moriio_engine=object(), transfer_timeout=5.0) + late = FakeStatus("in_progress") + timer = threading.Timer(0.2, lambda: setattr(late, "state", "success")) + timer.start() + try: + start = time.monotonic() + wrapper.waiting_for_transfer_complete([late]) + assert time.monotonic() - start >= 0.15 + finally: + timer.cancel() diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_common.py b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_common.py index f25e53e3f0..5f1068562c 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_common.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_common.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import contextlib +import math import os import threading import time @@ -45,6 +46,13 @@ TransferId = str TransferOffsetsKey = tuple[str, tuple[int, ...], tuple[int, ...], torch.dtype] +def _positive_finite_timeout(name: str, value: Any) -> float: + timeout = float(value) + if not math.isfinite(timeout) or timeout <= 0: + raise ValueError(f"{name} must be finite and greater than zero") + return timeout + + class MoRIIOTransferAck(NamedTuple): transfer_id: TransferId consumer_tp_size: int = 1 @@ -160,6 +168,20 @@ class MoRIIOMode(Enum): WRITE = "write" +class TransferBatchState(Enum): + """Verdict for a group of transfer statuses that belong to one request. + + A request's KV transfer is spread over one status per layer (two for a KDA + layer), so a single status never decides the request: PENDING means at + least one is still in flight and none has failed, FAILED means at least + one failed, DONE means all succeeded. + """ + + DONE = "done" + FAILED = "failed" + PENDING = "pending" + + class MoRIIOError(Exception): """Base exception for MoRIIO operations.""" @@ -260,6 +282,7 @@ class MoRIIOConfig: tp_size: int transfer_timeout: float defer_timeout: float + recv_abort_timeout: float read_mode: bool = False qp_per_transfer: int = 1 post_batch_size: int = -1 @@ -284,6 +307,8 @@ class MoRIIOConfig: # raising TransferError (sec). # defer_timeout -> Timeout before a deferred send with no finished_sending # notification is reaped and its blocks force-freed (sec). + # recv_abort_timeout -> Timeout before an in-flight recv whose RDMA + # completion never arrived is aborted (sec). # Knobs for RDMA transfers, ignored if on xgmi backend # qp_per_transfer -> Number of RDMA Queue Pairs per KV transfer. @@ -315,13 +340,29 @@ class MoRIIOConfig: "must be one of 'rdma' or 'xgmi'." ) - transfer_timeout = float( + transfer_timeout = _positive_finite_timeout( + "kv_connector_extra_config.transfer_timeout", extra_config.get( "transfer_timeout", MoRIIOConstants.DEFAULT_TRANSFER_TIMEOUT - ) + ), ) - defer_timeout = float( - extra_config.get("defer_timeout", MoRIIOConstants.DEFAULT_DEFER_TIMEOUT) + defer_timeout = _positive_finite_timeout( + "kv_connector_extra_config.defer_timeout", + extra_config.get("defer_timeout", MoRIIOConstants.DEFAULT_DEFER_TIMEOUT), + ) + legacy_recv_abort_timeout = os.environ.get("VLLM_MORIIO_TRANSFER_TIMEOUT_S") + if legacy_recv_abort_timeout is not None: + logger.warning_once( + "The environment variable VLLM_MORIIO_TRANSFER_TIMEOUT_S is " + "deprecated. Set 'recv_abort_timeout' inside " + "kv_transfer_config.kv_connector_extra_config instead." + ) + recv_abort_timeout = _positive_finite_timeout( + "kv_connector_extra_config.recv_abort_timeout", + extra_config.get( + "recv_abort_timeout", + legacy_recv_abort_timeout or MoRIIOConstants.DEFAULT_RECV_ABORT_TIMEOUT, + ), ) return cls( @@ -347,6 +388,7 @@ class MoRIIOConfig: backend=backend, transfer_timeout=transfer_timeout, defer_timeout=defer_timeout, + recv_abort_timeout=recv_abort_timeout, ) @@ -374,6 +416,10 @@ class MoRIIOConstants: # notification is reaped and its blocks force-freed. # Overridable via kv_connector_extra_config["defer_timeout"]. DEFAULT_DEFER_TIMEOUT = 60.0 + # Timeout (seconds) before an in-flight recv whose RDMA completion was lost + # is aborted, so the decode worker does not hang on it forever. + # Overridable via kv_connector_extra_config["recv_abort_timeout"]. + DEFAULT_RECV_ABORT_TIMEOUT = 120.0 # The router embeds both zmq_addresses in the request_id: @@ -567,9 +613,12 @@ class MoRIIOConnectorMetadata(KVConnectorMetadata): remote_handshake_port=int(remote_handshake_port), remote_notify_port=int(remote_notify_port), # Remote peer TP degree (used as remote_tp_size downstream). The - # proxy advertises it under "remote_tp_size"; #46332 read "tp_size" - # which is absent on WRITE producer requests -> defaulted to 1 -> - # rank collapse. Read the right key; 0 == unknown (== homogeneous). + # decode side must know the prefiller's TP degree to map producer + # ranks onto the correct decode rank (and vice versa). The old plain + # "tp_size" key (#46332) is absent on WRITE producer requests and + # defaulted to 1, collapsing every producer rank onto decode rank 0 + # and corrupting the transfer; prefer "remote_tp_size". 0 == unknown + # (treated as homogeneous downstream). tp_size=int( kv_transfer_params.get("remote_tp_size") or kv_transfer_params.get("tp_size") diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py index f15eee9bd5..dce4a98068 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py @@ -2,14 +2,13 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import logging import math -import os import queue import threading import time from collections import defaultdict from collections.abc import Collection from concurrent.futures import Future, ThreadPoolExecutor -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, cast import msgpack import msgspec @@ -33,10 +32,13 @@ from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_common import ( MoRIIOConfig, MoRIIOConnectorMetadata, MoRIIOConstants, + MoRIIOError, MoRIIOMode, MoRIIOTransferAck, ReqId, ReqMeta, + TransferBatchState, + TransferError, TransferId, WriteTask, fold_local_rank, @@ -56,11 +58,20 @@ from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_engine import ( ) from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_layout import ( LayerTransferGeometry, + MambaOffsetTemplate, + apply_mamba_offset_template, build_layer_to_spec, + build_mamba_offset_template, compute_block_transfer_offsets, + compute_mamba_conv_split_count, get_layer_transfer_geometry, is_mla_cache_layer, iter_layer_registration_regions, + kda_conv_ssm, +) +from vllm.distributed.kv_transfer.kv_connector.v1.ssm_conv_transfer_utils import ( + MambaConvSplitInfo, + derive_mamba_conv_split, ) from vllm.distributed.parallel_state import ( get_tensor_model_parallel_world_size, @@ -79,6 +90,7 @@ from vllm.v1.core.sched.output import SchedulerOutput from vllm.v1.kv_cache_interface import ( FullAttentionSpec, KVCacheConfig, + MambaSpec, SlidingWindowSpec, ) from vllm.v1.outputs import KVConnectorOutput @@ -91,6 +103,13 @@ if TYPE_CHECKING: logger = init_logger(__name__) +# Backoff bounds for re-posting an RDMA READ that the NIC rejected because the +# send queue was full. A separate CQ-poll thread drains completions, so the +# wait only needs to be long enough for it to make progress. +_SQ_FULL_BACKOFF_INITIAL_S = 0.001 +_SQ_FULL_BACKOFF_MAX_S = 0.05 +_MAX_LOCAL_DECODE_TAIL_BLOCKS = 1 + try: from mori.io import ( @@ -197,6 +216,12 @@ def resolve_moriio_transfer_ack( class MoRIIOConnector(KVConnectorBase_V1, SupportsHMA): + @property + def supports_divergent_local_hybrid_hits(self) -> bool: + # The READ path always transfers the recurrent-state slot, including + # when the attention prefix is already present locally. + return self.mode == MoRIIOMode.READ and self._transfers_mamba_state + def __init__( self, vllm_config: VllmConfig, @@ -217,19 +242,10 @@ class MoRIIOConnector(KVConnectorBase_V1, SupportsHMA): + str(self.kv_transfer_config.kv_connector_extra_config["handshake_port"]) ) self.mode = get_moriio_mode(self.kv_transfer_config) - if ( - self.mode == MoRIIOMode.READ - and self.kv_transfer_config.is_kv_consumer - and vllm_config.compilation_config.cudagraph_mode.has_full_cudagraphs() - ): - # warn only; kv-read barrier requires PIECEWISE cudagraph mode - logger.warning_once( - "MoRIIO READ mode is running with %s CUDA graphs: per-layer " - "KV-read barrier can't fire inside full graph; accuracy may " - "degrade at high concurrency. Set cudagraph_mode=PIECEWISE " - "in --compilation-config.", - vllm_config.compilation_config.cudagraph_mode.name, - ) + self._transfers_mamba_state = any( + group.enable_kv_transfer and isinstance(group.kv_cache_spec, MambaSpec) + for group in kv_cache_config.kv_cache_groups + ) if role == KVConnectorRole.SCHEDULER: self.connector_scheduler: MoRIIOConnectorScheduler | None = ( MoRIIOConnectorScheduler(vllm_config, self.engine_id, kv_cache_config) @@ -279,6 +295,11 @@ class MoRIIOConnector(KVConnectorBase_V1, SupportsHMA): request, blocks, num_external_tokens, self.connector_worker ) + def on_new_request(self, request: "Request") -> None: + """Fired when the scheduler admits a request, before any allocation.""" + assert self.connector_scheduler is not None + self.connector_scheduler.on_new_request(request) + def build_connector_meta( self, scheduler_output: SchedulerOutput, @@ -300,6 +321,14 @@ class MoRIIOConnector(KVConnectorBase_V1, SupportsHMA): block_ids: BlockIds, ) -> tuple[bool, dict[str, Any] | None]: assert self.connector_scheduler is not None + if self.connector_scheduler._has_mamba: + attn_block_ids, mamba_block_groups = ( + self.connector_scheduler.split_block_groups(block_ids) + ) + delay_free, params = self.connector_scheduler.request_finished( + request, attn_block_ids, mamba_block_groups + ) + return delay_free, params return self.connector_scheduler.request_finished(request, block_ids) def update_connector_output(self, connector_output: KVConnectorOutput) -> None: @@ -318,6 +347,12 @@ class MoRIIOConnector(KVConnectorBase_V1, SupportsHMA): assert self.connector_worker is not None return self.connector_worker.get_finished() + def get_block_ids_with_load_errors(self) -> set[int]: + """Blocks whose MoRIIO read failed, for the scheduler to recompute.""" + if self.connector_worker is None: + return set() + return self.connector_worker.get_block_ids_with_load_errors() + def start_load_kv(self, forward_context: "ForwardContext", **kwargs) -> None: assert self.connector_worker is not None if self.mode == MoRIIOMode.WRITE and get_role() == ROLE.CONSUMER: @@ -381,6 +416,47 @@ class MoRIIOConnector(KVConnectorBase_V1, SupportsHMA): return False +def _split_kv_cache_group_kinds( + kv_cache_config: "KVCacheConfig", +) -> tuple[list[int], list[int]]: + """Classify kv_cache_groups into (attention_group_indices, + mamba_group_indices) by whether each group's spec is a MambaSpec. + + Groups that opt out of external transfer are absent from + ``KVCacheConfig.transfer_groups``. Returned indices address that filtered + sequence and therefore also the connector's transfer block payload. + """ + attn: list[int] = [] + mamba: list[int] = [] + for gi, group in enumerate(kv_cache_config.transfer_groups): + if isinstance(group.kv_cache_spec, MambaSpec): + mamba.append(gi) + else: + attn.append(gi) + return attn, mamba + + +def _validate_mamba_specs(specs: Collection[MambaSpec]) -> MambaSpec | None: + """Validate the common packed-state contract and return its representative.""" + specs = tuple(specs) + if not specs: + return None + for spec in specs: + if len(spec.shapes) != 2 or len(spec.dtypes) != 2: + raise MoRIIOError( + "MoRIIO hybrid READ supports exactly two Mamba states " + "(conv and SSM), got " + f"{len(spec.shapes)} shapes and {len(spec.dtypes)} dtypes" + ) + reference = specs[0] + if any(spec != reference for spec in specs[1:]): + raise MoRIIOError( + "MoRIIO hybrid READ requires all transferable Mamba groups " + "to use the same cache spec" + ) + return reference + + class MoRIIOConnectorScheduler: """Implementation of Scheduler side methods.""" @@ -391,35 +467,73 @@ class MoRIIOConnectorScheduler: kv_cache_config: "KVCacheConfig", ): self.vllm_config = vllm_config - + self.kv_cache_config = kv_cache_config assert vllm_config.kv_transfer_config is not None, ( "kv_transfer_config must be set for MoRIIOConnector" ) self.kv_transfer_config = vllm_config.kv_transfer_config - self.block_size = vllm_config.cache_config.block_size - self.engine_id: EngineId = engine_id self.mode = get_moriio_mode(self.kv_transfer_config) + self._attn_group_ids, self._mamba_group_ids = _split_kv_cache_group_kinds( + kv_cache_config + ) + self._has_mamba = bool(self._mamba_group_ids) + if self._has_mamba: + if len(self._attn_group_ids) != 1: + raise MoRIIOError( + "MoRIIO hybrid READ requires exactly one transferable " + "attention group, got " + f"{len(self._attn_group_ids)} attention groups" + ) + mamba_specs = [ + cast(MambaSpec, kv_cache_config.transfer_groups[group_id].kv_cache_spec) + for group_id in self._mamba_group_ids + ] + _validate_mamba_specs(mamba_specs) + if vllm_config.speculative_config is not None: + raise MoRIIOError( + "MoRIIO hybrid READ does not support speculative decoding" + ) + if self.mode != MoRIIOMode.READ: + raise MoRIIOError( + "MoRIIO hybrid (mamba/KDA) transfer is implemented for READ " + "mode only; set kv_connector_extra_config.read_mode=true" + ) + # Only "all" mode keeps a state per block position; the other modes keep + # a single running state in the last slot. + self._ssm_state_slots_are_positional = ( + vllm_config.cache_config.mamba_cache_mode == "all" + ) + self.block_size = vllm_config.cache_config.block_size + self._max_decode_tail_blocks = ( + _MAX_LOCAL_DECODE_TAIL_BLOCKS + if self._has_mamba + else math.ceil((vllm_config.num_lookahead_tokens + 1) / self.block_size) + ) + self.engine_id: EngineId = engine_id self._is_hma_required = ( not vllm_config.scheduler_config.disable_hybrid_kv_cache_manager and any( not isinstance(g.kv_cache_spec, FullAttentionSpec) - for g in kv_cache_config.kv_cache_groups + for g in kv_cache_config.transfer_groups ) ) if self._is_hma_required: - # TODO(simondanielsson): support non-sliding window hybrids + # MoRIIO handles sliding-window attention and mamba/KDA groups in + # READ mode. Other hybrid cache specs still fail closed. unsupported = [ type(g.kv_cache_spec).__name__ - for g in kv_cache_config.kv_cache_groups + for g in kv_cache_config.transfer_groups if not isinstance( - g.kv_cache_spec, (FullAttentionSpec, SlidingWindowSpec) + g.kv_cache_spec, + (FullAttentionSpec, SlidingWindowSpec, MambaSpec), ) ] if unsupported: raise NotImplementedError( "MoRIIO only supports sliding-window hybrid models (e.g. " - "Gemma) under the hybrid KV cache manager. Unsupported KV " + "Gemma) and mamba/KDA groups under the hybrid KV cache " + "manager. Unsupported KV " f"cache group spec(s): {sorted(set(unsupported))}. Pass " "--disable-hybrid-kv-cache-manager to run without HMA." ) @@ -435,7 +549,7 @@ class MoRIIOConnectorScheduler: (g.kv_cache_spec.sliding_window, g.kv_cache_spec.block_size) if isinstance(g.kv_cache_spec, SlidingWindowSpec) else (0, self.block_size) - for g in kv_cache_config.kv_cache_groups + for g in kv_cache_config.transfer_groups ] # add 1 to conservatively account for boundary overlap eg window isn't fully # aligned with blocks. @@ -451,8 +565,7 @@ class MoRIIOConnectorScheduler: self._full_attn_group_idx = 0 self._full_attn_block_size = self.block_size for gi, group in enumerate(kv_cache_config.kv_cache_groups): - is_full_attn = self.blocks_per_sw[gi] == 0 - if is_full_attn: + if isinstance(group.kv_cache_spec, FullAttentionSpec): self._full_attn_group_idx = gi self._full_attn_block_size = getattr( group.kv_cache_spec, "block_size", self.block_size @@ -533,7 +646,11 @@ class MoRIIOConnectorScheduler: - sliding window: keep only the last window size blocks - full attention: keep all blocks """ - if len(block_ids) == 0 or not self._is_hma_required: + if len(block_ids) == 0: + return [] + if len(block_ids) == len(self.kv_cache_config.kv_cache_groups): + block_ids = self.kv_cache_config.select_transfer_block_ids(block_ids) + if not self._is_hma_required: return [list(blocks) for blocks in block_ids] return [ blocks[-self.blocks_per_sw[i] :] if self.blocks_per_sw[i] > 0 else blocks @@ -595,21 +712,75 @@ class MoRIIOConnectorScheduler: """ if self.is_producer: + # The P-side prompt truncation for hybrid models happens in + # on_new_request, before the scheduler looks the request up in the + # prefix cache. return 0, False - token_ids = request.prompt_token_ids or [] + num_prompt_tokens = request.num_prompt_tokens + num_external_tokens = max(num_prompt_tokens - num_computed_tokens, 0) if self.mode == MoRIIOMode.WRITE: - # MoriiO in write mode, no remote prefill + # MoriiO in write mode, no remote prefill. Hybrid models never get + # here: register_kv_caches refuses WRITE mode for them, so there is + # no recurrent-state accounting to do. + return num_external_tokens, True - return len(token_ids) - num_computed_tokens, True + # READ mode always recomputes the last token locally on the decoder. + # + # The second element declares who waits for the KV, not whether it has + # arrived. False keeps the wait on this side: the forward runs in the + # same step, and the connector blocks in wait_for_layer_load (or in + # start_load_kv under full-graph capture) until the KV is really there. + return max(num_external_tokens - 1, 0), False - return len(token_ids) - 1 - num_computed_tokens, False + def on_new_request(self, request: "Request") -> None: + """Scheduler hook fired when a request is admitted. + + The hybrid P-side truncation must land here rather than in + ``get_num_new_matched_tokens``: the scheduler runs the local prefix + cache lookup first and only then asks the connector, so truncating + there shortens a prompt whose cache hit was already measured against + the full length. On a full hit that leaves ``num_computed_tokens == + num_tokens`` and trips the scheduler's ``num_new_tokens > 0`` assert. + """ + if not (self.is_producer and self._has_mamba): + return + params = request.kv_transfer_params + if params is not None and params.get("do_remote_decode"): + self._truncate_mamba_request_for_prefill(request) + + def _truncate_mamba_request_for_prefill(self, request: "Request") -> None: + """P-side only: drop the last prompt token so the prefiller computes + h(N-1) instead of h(N). + + The decoder recomputes the last token locally to derive h(N) correctly + (see the READ len-1 accounting in get_num_new_matched_tokens). Guarded + by ``_p_side_truncated`` to avoid repeated truncation across a + preemption/reschedule. Mirrors the NIXL scheduler. + """ + params = request.kv_transfer_params + if ( + params is not None + and not params.get("_p_side_truncated") + and request.num_prompt_tokens > 1 + ): + if request.prompt_token_ids is not None: + request.prompt_token_ids.pop() + elif request.prompt_embeds is not None: + request.prompt_embeds = request.prompt_embeds[:-1] + else: + return + + request._all_token_ids.pop() + request.num_prompt_tokens -= 1 + request.max_tokens = 1 + params["_p_side_truncated"] = True def send_notify_block( self, req_id: ReqId, transfer_id: TransferId, - block_notify_list: list[int], + block_notify_list: list, host=None, port=None, ): @@ -739,7 +910,7 @@ class MoRIIOConnectorScheduler: params.setdefault("transfer_id", transfer_id) request_id = request.request_id self.map_request_id(request_id, transfer_id) - if params.get("do_remote_decode"): + if params.get("do_remote_decode") and self.mode == MoRIIOMode.WRITE: local_block_ids: BlockIds = blocks.get_block_ids() self._reqs_need_save[request.request_id] = (request, local_block_ids) # Snapshot params now so chunked-prefill build_connector_meta @@ -753,24 +924,88 @@ class MoRIIOConnectorScheduler: # remote_engine_id is returned by the prefill's request_finished. # host/ports come from the request_id (parsed in add_new_req). if "remote_engine_id" in params: + adjusted_remote_block_ids = remote_block_ids if num_external_tokens > 0: - # Get unhashed blocks to pull from remote. - local_block_ids = self.get_exchange_clipped_blocks( - blocks.get_block_ids() - ) - # Partial cache hits can lead to fewer local blocks vs - # remote blocks, but never more - assert all( - len(local) <= len(remote) - for local, remote in zip( - local_block_ids, remote_block_ids, strict=True + if self._has_mamba: + # READ is synchronous, so allocate_slots may hash + # destination blocks before this hook runs. Pair + # the attention and recurrent-state suffixes + # independently. + remote_attn = list(remote_block_ids[0]) + remote_mamba_groups = [ + list(group) for group in remote_block_ids[1:] + ] + attn_block_ids, mamba_block_groups = ( + self.split_block_groups(blocks.get_block_ids()) ) - ) + if len(mamba_block_groups) != len(remote_mamba_groups): + raise MoRIIOError( + "MoRIIO hybrid READ Mamba group " + "count mismatch: " + f"local={len(mamba_block_groups)}, " + f"remote={len(remote_mamba_groups)}" + ) + local_attn = attn_block_ids + local_attn, remote_attn = self._align_read_blocks( + local_attn, + remote_attn, + self._max_decode_tail_blocks, + ) + aligned_local_mamba: list[list[int]] = [] + aligned_remote_mamba: list[list[int]] = [] + for local_group, remote_group in zip( + mamba_block_groups, + remote_mamba_groups, + strict=True, + ): + aligned_local, aligned_remote = ( + self._align_read_blocks( + local_group, remote_group, 1 + ) + ) + aligned_local_mamba.append(aligned_local) + aligned_remote_mamba.append(aligned_remote) + local_block_ids = [local_attn, *aligned_local_mamba] + adjusted_remote_block_ids = [ + remote_attn, + *aligned_remote_mamba, + ] + else: + # Preserve main's per-group HMA representation + # for attention-only hybrid models while + # accounting for the locally recomputed tail. + local_block_ids = self.get_exchange_clipped_blocks( + blocks.get_block_ids() + ) + adjusted_remote_block_ids = [] + aligned_local_block_ids: list[list[int]] = [] + for local_group, remote_group in zip( + local_block_ids, + remote_block_ids, + strict=True, + ): + aligned_local, aligned_remote = ( + self._align_read_blocks( + local_group, + remote_group, + self._max_decode_tail_blocks, + ) + ) + aligned_local_block_ids.append(aligned_local) + adjusted_remote_block_ids.append(aligned_remote) + local_block_ids = aligned_local_block_ids else: - # If remote_blocks and num_external_tokens = 0, we have - # a full prefix cache hit on the D worker. We need to call - # send_notify in _read_blocks to free the memory on the P. - local_block_ids = [[] for _ in remote_block_ids] + if self._has_mamba: + # Attention can be a complete local hit, but the + # recurrent state is never prefix-cacheable. + _, mamba_block_groups = self.split_block_groups( + blocks.get_block_ids() + ) + local_block_ids = [[], *mamba_block_groups] + else: + # Nothing to pull, but retain one empty list per + # cache group so the worker can notify P. + local_block_ids = [[] for _ in remote_block_ids] self._reqs_need_recv[request.request_id] = ( request, @@ -778,7 +1013,10 @@ class MoRIIOConnectorScheduler: ) # Snapshot params for chunked prefill consumption # in build_connector_meta (see comment above). - self._req_kv_params[request.request_id] = dict(params) + saved_params = dict(params) + if num_external_tokens > 0: + saved_params["remote_block_ids"] = adjusted_remote_block_ids + self._req_kv_params[request.request_id] = saved_params else: logger.warning( "Got invalid KVTransferParams: %s. This " @@ -1003,13 +1241,78 @@ class MoRIIOConnectorScheduler: logger.warning("Error closing ZMQ socket for path %s: %s", path, e) self.paths.clear() + def split_block_groups( + self, block_ids: list[list[int]] | tuple[list[int], ...] + ) -> tuple[list[int], list[list[int]]]: + """Select transferable attention and Mamba block groups. + + The wire payload stores the attention group first, followed by every + Mamba group in transfer-group order. Outside ``mamba_cache_mode="all"``, + only each group's running-state slot is transferred. + """ + if not block_ids: + return [], [] + if not self._has_mamba: + raise MoRIIOError("Mamba block groups requested for a non-hybrid model") + transfer_block_ids = self.kv_cache_config.select_transfer_block_ids(block_ids) + attn = list(transfer_block_ids[self._attn_group_ids[0]]) + mamba_groups = [ + self._clip_mamba_group(list(transfer_block_ids[group_id])) + for group_id in self._mamba_group_ids + ] + return attn, mamba_groups + + def _clip_mamba_group(self, blocks: list[int]) -> list[int]: + """Keep only the state-bearing slots of one mamba kv cache group.""" + if not blocks: + return blocks + if not self._ssm_state_slots_are_positional: + # Single running state: everything before it is a null placeholder + # or the previous step's superseded state. + blocks = blocks[-1:] + return blocks + + @staticmethod + def _align_read_blocks( + local_block_ids: list[int], + remote_block_ids: list[int], + max_decode_tail_blocks: int, + ) -> tuple[list[int], list[int]]: + """Pair a decode suffix with the corresponding prefill blocks. + + Prefix hits can make the decode's unhashed suffix shorter than the + prefill's full list, so read from the matching remote suffix. The + decoder can instead have one extra tail block for the locally + recomputed final token; that block has no remote source and is not + transferred. + """ + if len(local_block_ids) > len(remote_block_ids): + extra_decode_blocks = len(local_block_ids) - len(remote_block_ids) + if extra_decode_blocks > max_decode_tail_blocks: + raise ValueError( + "decode block ids exceed prefill block ids beyond the " + f"allowed local tail ({max_decode_tail_blocks} blocks): " + f"{len(local_block_ids)} > {len(remote_block_ids)}" + ) + local_block_ids = local_block_ids[: len(remote_block_ids)] + elif len(local_block_ids) < len(remote_block_ids): + remote_block_ids = ( + remote_block_ids[-len(local_block_ids) :] if local_block_ids else [] + ) + return local_block_ids, remote_block_ids + def request_finished( self, request: "Request", - block_ids: BlockIds, + block_ids: BlockIds | list[int], + mamba_block_groups: list[list[int]] | None = None, ) -> tuple[bool, dict[str, Any] | None]: """Once a request is finished, determine whether request blocks should be freed now or will be sent asynchronously and freed later. + + ``block_ids`` normally contains all cache groups. Hybrid callers from + ``request_finished_all_groups`` pass a flat attention list and the + recurrent-state groups separately in ``mamba_block_groups``. """ request_id = request.request_id params = request.kv_transfer_params @@ -1042,8 +1345,25 @@ class MoRIIOConnectorScheduler: if self.mode == MoRIIOMode.WRITE: self._release_write_prefill_blocks(request.request_id, params) else: - empty_groups: BlockIds = [[] for _ in self.blocks_per_sw] - self._reqs_need_recv[request.request_id] = (request, empty_groups) + # Carry the actually-allocated blocks (incl. the per-request KDA + # mamba slot) instead of []: a do_remote_prefill request finishing + # before update_state_after_alloc still has blocks, and dropping the + # mamba slot here would re-trip the KDA guard in _read_blocks. + if mamba_block_groups: + recv_blocks: BlockIds = [ + list(cast(list[int], block_ids)), + *[list(group) for group in mamba_block_groups], + ] + elif block_ids and isinstance(block_ids[0], int): + recv_blocks = [list(cast(list[int], block_ids))] + else: + grouped_block_ids = cast(BlockIds, block_ids) + recv_blocks = ( + [list(group) for group in grouped_block_ids] + if self._has_mamba + else [[] for _ in self.blocks_per_sw] + ) + self._reqs_need_recv[request.request_id] = (request, recv_blocks) params["do_remote_prefill"] = False return False, None @@ -1054,7 +1374,17 @@ class MoRIIOConnectorScheduler: return False, None # On producer - computed_block_ids = self.get_exchange_clipped_blocks(block_ids) + if mamba_block_groups is not None: + computed_block_ids: BlockIds = [ + list(cast(list[int], block_ids)), + *[list(group) for group in mamba_block_groups], + ] + elif self._has_mamba: + raise MoRIIOError("Hybrid request completion requires all KV cache groups") + else: + computed_block_ids = self.get_exchange_clipped_blocks( + cast(BlockIds, block_ids) + ) delay_free_blocks = any(len(group) > 0 for group in computed_block_ids) if delay_free_blocks: @@ -1209,10 +1539,14 @@ class MoRIIOConnectorWorker: self.is_producer = self.kv_transfer_config.is_kv_producer self.layer_to_spec = build_layer_to_spec(kv_cache_config) # we transfer by layer, but store by kv cache group - self.layer_to_group: dict[str, int] = { - layer_name: group_idx - for group_idx, group in enumerate(kv_cache_config.kv_cache_groups) - for layer_name in group.layer_names + self.layer_to_group = dict(kv_cache_config.transfer_group_index_by_layer) + self._transfer_layer_names = set(self.layer_to_group) + _, mamba_transfer_group_ids = _split_kv_cache_group_kinds(kv_cache_config) + self._num_mamba_transfer_groups = len(mamba_transfer_group_ids) + self._mamba_payload_index_by_layer = { + layer_name: payload_index + for payload_index, group_id in enumerate(mamba_transfer_group_ids, start=1) + for layer_name in kv_cache_config.transfer_groups[group_id].layer_names } if self.is_producer: @@ -1353,7 +1687,8 @@ class MoRIIOConnectorWorker: # Map of engine_id -> num_blocks. All ranks in the same deployment will # have the same number of blocks. self.dst_num_blocks: dict[EngineId, int] = {} - # In-progress READ transfers: req_id -> {layer_name: status}. + # In-progress READ transfers: req_id -> {layer_name: [statuses]}. + # A KDA layer posts two reads (conv + ssm), so a layer maps to a list. self._recving_transfers: defaultdict[ReqId, dict] = defaultdict(dict) # Values are (remote_host, remote_notify_port, transfer_id). self._recving_transfers_callback_addr: dict[ReqId, tuple[str, str, str]] = {} @@ -1361,6 +1696,18 @@ class MoRIIOConnectorWorker: # Used by _pop_done_transfers to abort transfers whose RDMA # completion is lost, instead of hanging forever. self._recving_transfers_start: dict[str, float] = {} + # Destination attention blocks per in-flight recv. Failed non-HMA reads + # report these to the scheduler's single-group recovery path. + self._recving_local_blocks: dict[ReqId, list[int]] = {} + # Drained by get_block_ids_with_load_errors. + self._invalid_block_ids: set[int] = set() + # This step's posted reads, awaited before the forward when the + # per-layer barrier cannot run. + self._reads_issued_this_step: list = [] + # This step's posted recurrent-state reads. Mamba layers are not + # wrapped by the attention KV-transfer decorator, so wait_for_layer_load + # never fires for them and these must be awaited in start_load_kv. + self._mamba_reads_this_step: list = [] # Track the expiration time of requests that are waiting to be sent. self._reqs_to_send: dict[ReqId, float] = {} @@ -1386,6 +1733,14 @@ class MoRIIOConnectorWorker: self.block_window_per_layer: list[int | None] = [] self.use_mla = self.model_config.use_mla + # Hybrid (mamba/KDA) recurrent-state transfer. Populated in + # register_kv_caches when the model has a MambaSpec kv_cache group. + self._has_mamba = False + self._conv_decomp: MambaConvSplitInfo | None = None + self._mamba_offset_templates: dict[str, MambaOffsetTemplate] = {} + # layer_name -> flat session indices (one per registered region: + # 1 for attention, 2 (conv, ssm) for a KDA layer). Built once lazily. + self._region_session_index: dict[str, list[int]] | None = None self.built_session = False self.built_write_session: defaultdict[str, list] = defaultdict(list) backend = get_attn_backend( @@ -1460,23 +1815,39 @@ class MoRIIOConnectorWorker: def _get_built_session(self, remote_engine_id): if remote_engine_id not in self.built_write_session: + remote_layer_map = self.layer_name_to_remote_kv_cache_metadata[ + remote_engine_id + ] cur_remote_engine_sessions = [] - for ln, local_meta in self.layer_name_to_local_kv_cache_metadata.items(): - unpacked_local_memory_meta = ( - self.moriio_wrapper.get_unpack_memory_metadata(local_meta[0]) - ) - unpacked_remote_memory_meta = ( - self.moriio_wrapper.get_unpack_memory_metadata( - self.layer_name_to_remote_kv_cache_metadata[remote_engine_id][ - ln - ][0] + for ln, local_metas in self.layer_name_to_local_kv_cache_metadata.items(): + remote_metas = remote_layer_map[ln] + if len(remote_metas) != len(local_metas): + # Sessions are addressed by region index, so a peer that + # registered a different number of regions for this layer + # would not just lose a region to zip(): every later index + # would point into the wrong layer's session. + raise MoRIIOError( + f"peer {remote_engine_id} registered " + f"{len(remote_metas)} region(s) for layer {ln}, this " + f"engine has {len(local_metas)}; the two sides must run " + "the same model and KV cache layout" ) - ) - cur_remote_engine_sessions.append( - self.moriio_wrapper.build_session( - unpacked_local_memory_meta, unpacked_remote_memory_meta + # One session per registered region, in registration order: + # attention layers have a single region; KDA layers have two + # (conv then ssm). This flat layout is indexed via + # _region_session_indices(layer_name). + for local_meta, remote_meta in zip(local_metas, remote_metas): + unpacked_local_memory_meta = ( + self.moriio_wrapper.get_unpack_memory_metadata(local_meta) + ) + unpacked_remote_memory_meta = ( + self.moriio_wrapper.get_unpack_memory_metadata(remote_meta) + ) + cur_remote_engine_sessions.append( + self.moriio_wrapper.build_session( + unpacked_local_memory_meta, unpacked_remote_memory_meta + ) ) - ) self.built_write_session[remote_engine_id] = cur_remote_engine_sessions return self.built_write_session[remote_engine_id], self.remote_moriio_metadata[ remote_engine_id @@ -1788,15 +2159,60 @@ class MoRIIOConnectorWorker: def _is_mla_cache_layer(self, layer_name: str) -> bool: return is_mla_cache_layer(self.layer_to_spec, layer_name) + def _is_mamba_layer(self, layer_name: str) -> bool: + """True for a hybrid/KDA layer backed by a packed state page.""" + return isinstance(self.layer_to_spec.get(layer_name), MambaSpec) + + @staticmethod + def _contiguous_byte_alias(tensor: torch.Tensor) -> torch.Tensor: + """Return a contiguous uint8 view over ``tensor``'s full byte extent. + + KDA conv/ssm caches are slot-strided views (``stride(0)`` may exceed the + per-slot element count), so they are non-contiguous and cannot be passed + to ``register_torch_tensor``. This aliases the same storage (zero-copy, + identical ``data_ptr``) as a flat uint8 tensor spanning + ``shape[0] * stride(0)`` elements -- the full slot-strided extent that + transfer offsets (``slot * stride(0) * elem``) address. + """ + esz = tensor.element_size() + storage_nbytes = tensor.untyped_storage().nbytes() + offset_bytes = tensor.storage_offset() * esz + extent_bytes = tensor.shape[0] * tensor.stride(0) * esz + # Clamp to the bytes remaining from the view's storage offset. In a + # packed page, the ssm view starts after conv, so its slot-strided extent + # would otherwise run past the backing allocation. + extent_bytes = min(extent_bytes, storage_nbytes - offset_bytes) + assert extent_bytes > 0, ( + "KDA byte alias empty: storage_offset=" + f"{tensor.storage_offset()} * esz={esz} >= " + f"storage_nbytes={storage_nbytes}" + ) + alias = torch.empty(0, dtype=torch.uint8, device=tensor.device) + alias.set_( + tensor.untyped_storage(), + tensor.storage_offset() * esz, + (extent_bytes,), + (1,), + ) + return alias + def _get_layer_transfer_geometry( self, layer_name: str, remote_num_blocks: int | None = None ) -> LayerTransferGeometry: - return get_layer_transfer_geometry( + geometry = get_layer_transfer_geometry( layer_name, self.kv_caches[layer_name], self.layer_to_spec, remote_num_blocks, ) + # get_layer_transfer_geometry also serves mamba layers, whose geometry + # is a different NamedTuple with no block fields. Every caller here is + # on the attention path (mamba layers return before reaching it), so + # narrow the type instead of leaking the union. + assert isinstance(geometry, LayerTransferGeometry), ( + f"layer {layer_name} is a mamba layer; use get_mamba_transfer_geometry" + ) + return geometry def _iter_layer_registration_regions( self, layer_name: str @@ -1853,22 +2269,71 @@ class MoRIIOConnectorWorker: def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]): """Register the KV Cache data in moriio.""" + kv_caches = { + layer_name: kv_cache + for layer_name, kv_cache in kv_caches.items() + if layer_name in self._transfer_layer_names + } + if not kv_caches: + raise MoRIIOError("MoRIIO has no transfer-enabled KV cache layers") self.kv_caches = kv_caches # layer name to kv cache + self._has_mamba = any( + self._is_mamba_layer(layer_name) for layer_name in kv_caches + ) + if self._has_mamba and self.mode == MoRIIOMode.WRITE: + # WRITE needs a distinct conv/ssm transfer plan. Reject before + # allocating or registering any regions rather than silently + # serving a zero recurrent state. + raise MoRIIOError( + "MoRIIO hybrid (mamba/KDA) transfer is implemented for READ " + "mode only; set kv_connector_extra_config.read_mode=true " + "for hybrid models" + ) + if self._has_mamba: + mamba_specs = [ + cast(MambaSpec, self.layer_to_spec[layer_name]) + for layer_name in kv_caches + if self._is_mamba_layer(layer_name) + ] + mamba_spec = _validate_mamba_specs(mamba_specs) + assert mamba_spec is not None + + from vllm.model_executor.layers.mamba.mamba_utils import ( + is_conv_state_dim_first, + ) + + if not is_conv_state_dim_first(): + raise MoRIIOError( + "MoRIIO KDA conv transfer requires the DS (dim-first) " + "state layout; set VLLM_SSM_CONV_STATE_LAYOUT=DS" + ) + self._conv_decomp = derive_mamba_conv_split(mamba_spec, self.world_size) + # KDA layers use packed pages with different geometry; only record + # attention tensor shapes here. self.kv_cache_shapes = { - layer_name: kv_cache.shape for layer_name, kv_cache in kv_caches.items() + layer_name: kv_cache.shape + for layer_name, kv_cache in kv_caches.items() + if not self._is_mamba_layer(layer_name) } + # Geometry selection must pick an attention layer (never a KDA tuple): + # prefer a standard 5-D K/V layer, else the first attention layer. + attn_items = [ + (layer_name, kv_cache) + for layer_name, kv_cache in kv_caches.items() + if not self._is_mamba_layer(layer_name) + ] first_layer_name, first_kv_cache = next( ( (layer_name, kv_cache) - for layer_name, kv_cache in kv_caches.items() + for layer_name, kv_cache in attn_items if ( not self._is_mla_cache_layer(layer_name) and len(kv_cache.shape) == 5 and (kv_cache.shape[0] == 2 or kv_cache.shape[1] == 2) ) ), - next(iter(kv_caches.items())), + attn_items[0] if attn_items else next(iter(kv_caches.items())), ) kv_elem_size = first_kv_cache.element_size() @@ -1915,6 +2380,17 @@ class MoRIIOConnectorWorker: ) for layer_name in kv_caches: self.layer_base_addr_index[layer_name] = base_addr_idx + if self._is_mamba_layer(layer_name): + # KDA layer: two whole-tensor regions (conv, ssm); no block-size + # geometry (recurrent state is per-slot, not paged). + for cache, region_len in self._iter_layer_registration_regions( + layer_name + ): + base_addr = cache.data_ptr() + caches_data.append((base_addr, region_len, cache.device.index, "")) + kv_caches_base_addr.append(base_addr) + base_addr_idx += 1 + continue geometry = self._get_layer_transfer_geometry(layer_name) if hma_enabled: if geometry.block_len != self.block_len: @@ -1941,31 +2417,87 @@ class MoRIIOConnectorWorker: kv_caches_base_addr.append(base_addr) base_addr_idx += 1 - shared_backing = len({id(t.untyped_storage()) for t in kv_caches.values()}) == 1 + shared_backing = ( + all(isinstance(t, torch.Tensor) for t in kv_caches.values()) + and len({id(t.untyped_storage()) for t in kv_caches.values()}) == 1 + ) shared_mr = None if shared_backing: reg_tensor, self.kv_layer_mr_offset = self._build_shared_kv_mr(kv_caches) shared_mr = self.moriio_wrapper.register_local_tensor(reg_tensor) + # Say it out loud when the groups disagree. This is expected on a hybrid + # model and used to be fatal, so a run that silently had one page size + # and a run that has three should not look identical in the log. + distinct_block_lens = sorted(set(self.block_lens.values())) + if len(distinct_block_lens) > 1: + logger.info( + "MoRIIO registered %d attention layers with %d distinct block " + "lengths (bytes): %s; transfer sizes and strides are per layer, " + "and %d is the page advertised to the peer.", + len(self.block_lens), + len(distinct_block_lens), + distinct_block_lens, + self.block_len, + ) + for layer_name, kv_cache in kv_caches.items(): if layer_name not in self.layer_name_to_local_kv_cache_metadata: self.layer_name_to_local_kv_cache_metadata[layer_name] = [] - moriio_mem_metadata = shared_mr or ( - self.moriio_wrapper.register_local_tensor(kv_cache) - ) - self.layer_name_to_local_kv_cache_metadata[layer_name].append( - moriio_mem_metadata - ) - - self.local_kv_cache_size.append( - kv_cache.nelement() * kv_cache.element_size() - ) + if self._is_mamba_layer(layer_name): + # Register conv and ssm as two whole-tensor regions, appending + # both metadata blobs (order: conv then ssm) so the session + # layout matches iter_layer_registration_regions. conv/ssm are + # slot-strided views and are not contiguous, which + # register_torch_tensor rejects; register a contiguous uint8 + # alias over each tensor's full byte extent instead (zero-copy: + # same storage and data_ptr). + conv, ssm = kda_conv_ssm( + kv_cache, cast(MambaSpec, self.layer_to_spec[layer_name]) + ) + logger.debug( + "MoRIIO KDA register %s: conv shape=%s stride=%s contig=%s " + "storage_nbytes=%s ; ssm shape=%s stride=%s contig=%s " + "storage_nbytes=%s", + layer_name, + tuple(conv.shape), + tuple(conv.stride()), + conv.is_contiguous(), + conv.untyped_storage().nbytes(), + tuple(ssm.shape), + tuple(ssm.stride()), + ssm.is_contiguous(), + ssm.untyped_storage().nbytes(), + ) + for tensor in (conv, ssm): + alias = self._contiguous_byte_alias(tensor) + meta = self.moriio_wrapper.register_local_tensor(alias) + self.layer_name_to_local_kv_cache_metadata[layer_name].append(meta) + self.local_kv_cache_size.append(alias.numel()) + else: + moriio_mem_metadata = shared_mr or ( + self.moriio_wrapper.register_local_tensor(kv_cache) + ) + self.layer_name_to_local_kv_cache_metadata[layer_name].append( + moriio_mem_metadata + ) + self.local_kv_cache_size.append( + kv_cache.nelement() * kv_cache.element_size() + ) self.kv_caches_base_addr[self.engine_id] = kv_caches_base_addr self.num_regions = len(caches_data) self.num_layers = len(self.kv_caches.keys()) + # Report the prevalidated KDA decomposition after registration. + if self._has_mamba: + assert self._conv_decomp is not None + logger.info( + "MoRIIO registered %d KDA (conv+ssm) layer(s); ssm sizes=%s", + sum(1 for ln in kv_caches if self._is_mamba_layer(ln)), + self._conv_decomp.ssm_sizes, + ) # Optimization for models with local attention (Llama 4) if self.vllm_config.model_config.hf_config.model_type == "llama4": from transformers import Llama4TextConfig @@ -2101,49 +2633,49 @@ class MoRIIOConnectorWorker: if get_forward_context().cudagraph_runtime_mode == CUDAGraphMode.FULL: return - deadline = time.monotonic() + self.moriio_config.transfer_timeout - while True: - with self.moriio_wrapper.lock: - pending = [ - status_by_layer[layer_name] - for status_by_layer in self._recving_transfers.values() - if layer_name in status_by_layer - ] + with self.moriio_wrapper.lock: + pending = [ + status + for status_by_layer in self._recving_transfers.values() + for status in status_by_layer.get(layer_name, ()) + ] - if not pending: - return + if not pending: + return - still_running = False - for status in pending: - # A failed read is dropped in _pop_done_transfers. - if status.Succeeded() or status.Failed(): - continue - still_running = True - - if not still_running: - return - - if time.monotonic() > deadline: - logger.warning( - "MoRIIO READ barrier timed out for layer %s; proceeding " - "(request dropped via get_finished).", + try: + self.moriio_wrapper.waiting_for_transfer_complete(pending) + except TransferError: + if self._has_mamba: + logger.exception( + "MoRIIO hybrid READ failed before layer %s; aborting the " + "forward because HMA load-error recovery is unsupported.", layer_name, ) - return - - time.sleep(0.001) + raise + logger.warning( + "MoRIIO READ barrier did not complete for layer %s; proceeding " + "to transfer cleanup.", + layer_name, + exc_info=True, + ) def _pop_done_transfers(self) -> set[str]: done_req_ids: set[str] = set() - _xfer_timeout = int(os.environ.get("VLLM_MORIIO_TRANSFER_TIMEOUT_S", "120")) + _xfer_timeout = self.moriio_config.recv_abort_timeout with self.moriio_wrapper.lock: to_remove = [] for req_id, status_by_layer in self._recving_transfers.items(): - statuses = list(status_by_layer.values()) - failed_status = next( - (status for status in statuses if status.Failed()), None - ) - if statuses and all(status.Succeeded() for status in statuses): + # Every layer's read must land before the request is complete, + # so the whole batch is polled rather than just the newest + # status. + statuses = [ + status + for layer_statuses in status_by_layer.values() + for status in layer_statuses + ] + state = self.moriio_wrapper.poll_transfer_batch(statuses) + if statuses and state is TransferBatchState.DONE: host, port, xfer_id = self._recving_transfers_callback_addr[req_id] done_req_ids.add(xfer_id) self.moriio_wrapper.send_notify( @@ -2154,15 +2686,19 @@ class MoRIIOConnectorWorker: message_fields={"consumer_tp_size": self.world_size}, ) to_remove.append(req_id) - elif failed_status is not None: + elif state is TransferBatchState.FAILED: + failed_status = next( + (status for status in statuses if status.Failed()), None + ) logger.error( "RDMA transfer failed for request %s: %s (code=%s). " - "Notifying prefill to free blocks; request will be " - "aborted by timeout.", + "Notifying prefill to free blocks and recording the " + "load error where scheduler recovery is supported.", req_id, - failed_status.Message(), - failed_status.Code(), + failed_status.Message() if failed_status else "unknown", + failed_status.Code() if failed_status else "unknown", ) + self._record_failed_recv(req_id) host, port, xfer_id = self._recving_transfers_callback_addr[req_id] try: self.moriio_wrapper.send_notify( @@ -2178,8 +2714,9 @@ class MoRIIOConnectorWorker: req_id, ) to_remove.append(req_id) - # Do NOT add to done_req_ids: decode KV cache is incomplete. - # The request will expire via the normal request timeout. + # Deliberately not in done_req_ids: the decode KV is + # incomplete, so this is a load error rather than a + # completion. elif req_id in self._recving_transfers_start: # Abort still-in-flight transfers that exceed the # configured deadline. Otherwise a lost RDMA @@ -2189,19 +2726,84 @@ class MoRIIOConnectorWorker: if _age > _xfer_timeout: logger.error( "RDMA read TIMED OUT for req %s after %.1fs " - "(VLLM_MORIIO_TRANSFER_TIMEOUT_S=%d)", + "(kv_connector_extra_config.recv_abort_timeout=%.0f)", req_id, _age, _xfer_timeout, ) + self._record_failed_recv(req_id) to_remove.append(req_id) for req_id in to_remove: del self._recving_transfers[req_id] del self._recving_transfers_callback_addr[req_id] self._recving_transfers_start.pop(req_id, None) + self._recving_local_blocks.pop(req_id, None) return done_req_ids + def _await_reads_issued_this_step(self) -> None: + """Step-level READ barrier, for the reads no per-layer barrier covers. + + load_kv_async=False promises the KV is in place before the forward runs. + wait_for_layer_load keeps that promise per layer, letting early layers + overlap later transfers, but it only fires for layers that go through + the attention KV-transfer decorator. Two cases fall through it: + + * mamba/KDA layers have no such hook at all, so their conv+ssm reads + would otherwise be consumed while still in flight; + * under CUDAGraphMode.FULL a host-side blocking wait is illegal + anywhere in the graph, so every read is awaited here instead. + + start_load_kv runs outside the graph, so blocking here is safe. + Attention reads keep their per-layer overlap in the non-FULL case. + + Attention-only failures are reported via + get_block_ids_with_load_errors. Hybrid failures are re-raised because + the scheduler's recovery path does not support multiple cache groups; + this fails the step closed instead of running forward with incomplete + recurrent state. + """ + all_statuses = self._reads_issued_this_step + self._reads_issued_this_step = [] + mamba_statuses = self._mamba_reads_this_step + self._mamba_reads_this_step = [] + if get_forward_context().cudagraph_runtime_mode == CUDAGraphMode.FULL: + statuses = all_statuses + else: + statuses = mamba_statuses + if not statuses: + return + try: + self.moriio_wrapper.waiting_for_transfer_complete(statuses) + except TransferError: + if self._has_mamba: + logger.exception( + "MoRIIO hybrid READ failed before the forward; aborting " + "because HMA load-error recovery is unsupported." + ) + raise + logger.exception( + "MoRIIO reads did not complete before the forward; proceeding " + "to transfer cleanup." + ) + + def _record_failed_recv(self, req_id: ReqId) -> None: + """Hand a failed attention-only read to scheduler recovery. + + The scheduler's invalid-block recovery currently assumes exactly one + KV cache group. Reporting IDs for a hybrid request would crash it while + unpacking the request's groups, so match NIXL and keep HMA failures out + of this API until scheduler recovery carries group identity. + """ + if self._has_mamba: + return + self._invalid_block_ids.update(self._recving_local_blocks.get(req_id, ())) + + def get_block_ids_with_load_errors(self) -> set[int]: + """Drain the blocks whose read failed, for the scheduler to recompute.""" + invalid, self._invalid_block_ids = self._invalid_block_ids, set() + return invalid + def save_kv_layer( self, metadata: MoRIIOConnectorMetadata, @@ -2462,11 +3064,12 @@ class MoRIIOConnectorWorker: continue - # Handshake already completed, start async read xfer. + # Handshake already completed, start the read xfer. self._read_blocks_for_req(req_id, meta) # Start transfers for requests whose handshakes have now finished. if remote_engine_id is None and not wait_handshake_readd_req: + self._await_reads_issued_this_step() return _deadline = time.monotonic() + self.moriio_config.transfer_timeout while True: @@ -2493,6 +3096,7 @@ class MoRIIOConnectorWorker: else: break + self._await_reads_issued_this_step() self._reqs_to_send.update(metadata.reqs_to_send) def wait_for_save(self, metadata: MoRIIOConnectorMetadata): @@ -2717,6 +3321,163 @@ class MoRIIOConnectorWorker: remote = [x + remote_off for x in remote] return local, remote, sizes + def _region_session_indices(self, layer_name: str) -> list[int]: + """Flat session indices for a layer's registered regions. + + Sessions are built one per region in registration order (see + _get_built_session), so an attention layer maps to a single index and a + KDA layer to two indices ([conv, ssm]). Built once and cached. + """ + if self._region_session_index is None: + mapping: dict[str, list[int]] = {} + idx = 0 + for ln, metas in self.layer_name_to_local_kv_cache_metadata.items(): + mapping[ln] = list(range(idx, idx + len(metas))) + idx += len(metas) + self._region_session_index = mapping + return self._region_session_index[layer_name] + + def _mamba_tp_ratio(self, remote_tp_size: int | None) -> int: + """Validate the homogeneous-TP KDA contract.""" + remote = self.world_size if remote_tp_size is None else remote_tp_size + if remote != self.world_size: + raise NotImplementedError( + "heterogeneous-TP mamba/KDA transfer is not supported yet " + f"(local_tp={self.world_size}, remote_tp={remote}); use equal " + "prefill/decode TP for hybrid models" + ) + return 1 + + def _mamba_blocks_for_layer( + self, + layer_name: str, + block_groups: BlockIds, + ) -> list[int]: + payload_index = self._mamba_payload_index_by_layer.get(layer_name) + if payload_index is None or payload_index >= len(block_groups): + raise MoRIIOError( + f"KDA layer {layer_name} has no block group in the transfer payload" + ) + return list(block_groups[payload_index]) + + def _compute_mamba_transfer_offsets( + self, + layer_name: str, + local_slots: list[int], + remote_slots: list[int], + remote_tp_size: int | None = None, + ) -> tuple[list[int], list[int], list[int], int]: + """Compute (local, remote, sizes, n_conv) for a KDA layer. + + Entries [:n_conv] address the conv region/session; entries [n_conv:] + address the ssm region/session. + """ + assert self._conv_decomp is not None, "KDA decomposition not derived" + conv, ssm = kda_conv_ssm( + self.kv_caches[layer_name], + cast(MambaSpec, self.layer_to_spec[layer_name]), + ) + tp_ratio = self._mamba_tp_ratio(remote_tp_size) + # Cache the slot-independent geometry per layer; only the per-request + # slot bases vary, and heterogeneous TP is rejected above. + template = self._mamba_offset_templates.get(layer_name) + if template is None: + template = build_mamba_offset_template( + conv, + ssm, + self._conv_decomp, + tp_ratio, + ) + self._mamba_offset_templates[layer_name] = template + local_offs, remote_offs, sizes = apply_mamba_offset_template( + template, local_slots, remote_slots + ) + n_conv = compute_mamba_conv_split_count(local_slots, self._conv_decomp) + return local_offs, remote_offs, sizes, n_conv + + def _post_mamba_reads( + self, + layer_name: str, + sessions: list, + region_sessions: list[int], + local_slots: list[int], + remote_slots: list[int], + remote_tp_size: int, + request_id: str, + deadline: float, + ) -> list: + """Post one KDA layer's conv and ssm reads; return their statuses. + + The layer owns two registered regions, so the offset list is split at + the conv/ssm boundary and each half goes to its own session. A half + with nothing to move is skipped rather than posted empty. + """ + local, remote, sizes, n_conv = self._compute_mamba_transfer_offsets( + layer_name, + local_slots, + remote_slots, + remote_tp_size=remote_tp_size, + ) + statuses: list = [] + for sess_idx, sl in ( + (region_sessions[0], slice(0, n_conv)), + (region_sessions[1], slice(n_conv, None)), + ): + region_sizes = sizes[sl] + if not region_sizes: + continue + statuses.append( + self._post_read_with_backoff( + sessions[sess_idx], + region_sizes, + local[sl], + remote[sl], + request_id, + layer_name, + deadline, + ) + ) + return statuses + + def _post_read_with_backoff( + self, + session, + sizes: list[int], + local_offsets: list[int], + remote_offsets: list[int], + request_id: str, + layer_name: str, + deadline: float, + ): + """Post one RDMA READ, backing off on transient SQ-full rejection. + + read_remote_data posts synchronously, so a send-queue-full rejection is + a Failed() status on return; a separate CQ-poll thread drains + completions and frees SQ depth, so we back off and re-post until + transfer_timeout, then store the failed status (get_finished notifies + prefill and drops the request non-fatally). + """ + _backoff = _SQ_FULL_BACKOFF_INITIAL_S + while True: + transfer_status = self.moriio_wrapper.read_remote_data( + sizes, local_offsets, remote_offsets, session + ) + if not self._is_sq_full_status(transfer_status): + break + if time.monotonic() > deadline: + logger.warning( + "MoRIIO READ send queue stayed full past " + "transfer_timeout for req %s layer %s; storing failed " + "status (get_finished notifies prefill and drops the " + "request). Raise qp_per_transfer if frequent.", + request_id, + layer_name, + ) + break + time.sleep(_backoff) + _backoff = min(_backoff * 2, _SQ_FULL_BACKOFF_MAX_S) + return transfer_status + @staticmethod def _is_sq_full_status(status) -> bool: """True if a MoRIIO transfer status is a transient RDMA send-queue-full @@ -2766,6 +3527,44 @@ class MoRIIOConnectorWorker: if chosen_tp is not None else self._remote_tp_rank(remote_tp_size) ) + if not any(local_block_ids): + # A request aborted before decode allocation still carries the + # producer's block table. There is no local destination or forward, + # so release the producer blocks without building sessions or + # posting RDMA reads. + self.moriio_wrapper.send_notify( + transfer_id, + remote_host, + str( + remote_notify_port + + get_port_offset( + int(remote_dp_rank), + eff_tp, + remote_tp_size, + ) + ), + message_type="release", + message_fields={"consumer_tp_size": self.world_size}, + ) + return + + if self._has_mamba: + expected_groups = 1 + self._num_mamba_transfer_groups + if ( + len(local_block_ids) != expected_groups + or len(remote_block_ids) != expected_groups + ): + raise MoRIIOError( + "MoRIIO hybrid READ block group count mismatch: " + f"expected={expected_groups}, local={len(local_block_ids)}, " + f"remote={len(remote_block_ids)}" + ) + local_attn = list(local_block_ids[0]) + remote_attn = list(remote_block_ids[0]) + self._mamba_tp_ratio(remote_tp_size) + else: + local_attn = remote_attn = [] + if flexible: remote_dp_engine_id = self.get_engine_name_with_dp_tp( dst_engine_id, int(remote_dp_rank), eff_tp @@ -2778,52 +3577,89 @@ class MoRIIOConnectorWorker: # SQ-full backpressure deadline, shared across this request's layers. _sq_deadline = time.monotonic() + self.moriio_config.transfer_timeout + # TODO: apply multi-session batch-read when MoRIIO supports it. for layer_name in self.layer_name_to_local_kv_cache_metadata: - sess_idx = list(self.layer_name_to_local_kv_cache_metadata.keys()).index( - layer_name - ) - group_idx = self.layer_to_group[layer_name] - offs = self._compute_block_transfer_offsets( - layer_name, - local_block_ids[group_idx], - remote_block_ids[group_idx], - remote_moriio_meta, - remote_tp_size=remote_tp_size, - remote_engine_id=remote_dp_engine_id, - ) - # TODO : apply multi-session batch-read when moriio support it - # - # SQ-full backpressure: read_remote_data posts the RDMA READ - # SYNCHRONOUSLY, so a send-queue-full rejection (per-QP HW cap) comes - # back as a Failed() status right here. A SEPARATE CQ-poll thread - # drains completions and frees SQ depth, so back off and RE-POST - # rather than let a transient rejection abort the request. No - # self-deadlock (the drain is off-thread); the reserve is - # all-or-nothing (nothing posted on a rejected attempt). Bounded by - # transfer_timeout; on sustained overload store the failed status and - # let get_finished handle it non-fatally (notify prefill + drop). - _backoff = 0.001 - while True: - transfer_status = self.moriio_wrapper.read_remote_data( - offs[2], offs[0], offs[1], sessions[sess_idx] + region_sessions = self._region_session_indices(layer_name) + statuses = [] + if self._is_mamba_layer(layer_name): + local_mamba = self._mamba_blocks_for_layer(layer_name, local_block_ids) + remote_mamba = self._mamba_blocks_for_layer( + layer_name, remote_block_ids ) - if not self._is_sq_full_status(transfer_status): - break - if time.monotonic() > _sq_deadline: - logger.warning( - "MoRIIO READ send queue stayed full past " - "transfer_timeout for req %s layer %s; storing failed " - "status (get_finished notifies prefill and drops the " - "request). Raise qp_per_transfer if frequent.", + # KDA layer: pull conv (sub-projection offsets) and ssm at the + # request's recurrent-state slot. Two regions -> two sessions. + if not local_mamba or not remote_mamba: + if not local_attn and not remote_attn: + # Whole-request no-op (e.g. full prefix cache hit): + # nothing to transfer for any layer. + continue + # A KDA layer must have its recurrent-state slot in the + # block-id tuple; missing it (while attention blocks are + # present) means the mamba KV-cache group is absent -> bug. + raise MoRIIOError( + f"KDA layer {layer_name}: missing mamba recurrent-state " + f"block ids (local={local_mamba}, remote={remote_mamba}) " + f"with attention blocks present for request {request_id}; " + "the mamba KV-cache group is absent from the transfer " + "block-id tuple" + ) + statuses = self._post_mamba_reads( + layer_name, + sessions, + region_sessions, + local_mamba, + remote_mamba, + remote_tp_size, + request_id, + _sq_deadline, + ) + else: + if self._has_mamba: + local_layer_blocks = local_attn + remote_layer_blocks = remote_attn + else: + group_idx = self.layer_to_group[layer_name] + local_layer_blocks = local_block_ids[group_idx] + remote_layer_blocks = remote_block_ids[group_idx] + offs = self._compute_block_transfer_offsets( + layer_name, + local_layer_blocks, + remote_layer_blocks, + remote_moriio_meta, + remote_tp_size=remote_tp_size, + remote_engine_id=remote_dp_engine_id, + ) + statuses.append( + self._post_read_with_backoff( + sessions[region_sessions[0]], + offs[2], + offs[0], + offs[1], request_id, layer_name, + _sq_deadline, ) - break - time.sleep(_backoff) - _backoff = min(_backoff * 2, 0.05) + ) with self.moriio_wrapper.lock: - self._recving_transfers[request_id][layer_name] = transfer_status + self._recving_transfers[request_id][layer_name] = statuses self._recving_transfers_start.setdefault(request_id, time.monotonic()) + # Destination blocks, kept for _record_failed_recv. + if self._has_mamba: + failed_recv_blocks = local_attn + else: + failed_recv_blocks = [ + block_id for group in local_block_ids for block_id in group + ] + self._recving_local_blocks.setdefault( + request_id, list(failed_recv_blocks) + ) + # Awaited before the forward when the per-layer barrier cannot + # run (see _await_reads_issued_this_step). + self._reads_issued_this_step.extend(statuses) + if self._is_mamba_layer(layer_name): + # No per-layer barrier exists for mamba layers, so these are + # always awaited in start_load_kv. + self._mamba_reads_this_step.extend(statuses) self._recving_transfers_callback_addr[request_id] = ( remote_host, str( diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_engine.py b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_engine.py index 5c1330521d..a0c6ec1f6d 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_engine.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_engine.py @@ -29,6 +29,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_common import ( MoRIIOError, MoRIIOTransferAck, RemoteAllocInfo, + TransferBatchState, TransferError, TransferId, WriteTask, @@ -58,7 +59,6 @@ try: except ImportError: logger.error("MoRIIO is not available") - """Write task execution logic for MoRIIO connector.""" @@ -395,9 +395,9 @@ class MoRIIOWriter: ) request_info.transfer_offsets[key] = offsets - # Get session index - layer_names = list(self.worker.layer_name_to_local_kv_cache_metadata.keys()) - sess_idx = layer_names.index(task.layer_name) + # One session per registered region; attention layers map to a single + # index (see MoRIIOConnectorWorker._region_session_indices). + sess_idx = self.worker._region_session_indices(task.layer_name)[0] local_off, remote_off, sizes = offsets @@ -662,6 +662,25 @@ class MoRIIOWrapper: ) return transfer_status + def poll_transfer_batch(self, transfer_statuses: list[Any]) -> TransferBatchState: + """Non-blocking verdict over every status of one request. + + Judging a request by its newest status alone calls it done while an + earlier layer is unfinished or already failed; failures do land on + layers other than the last, because _post_read_with_backoff returns a + failed status when the send queue never drains. + + This stays a Python scan so it does not drive the backend's progress + callback from the engine thread alongside MoRIIO's own poller. + """ + state = TransferBatchState.DONE + for status in transfer_statuses: + if status.Failed(): + return TransferBatchState.FAILED + if not status.Succeeded(): + state = TransferBatchState.PENDING + return state + def waiting_for_transfer_complete(self, transfer_statuses: list[Any] | None = None): if transfer_statuses is None: with self.lock: @@ -673,6 +692,15 @@ class MoRIIOWrapper: if not transfers_to_wait: return + self._poll_transfers_until_done(transfers_to_wait) + + def _poll_transfers_until_done(self, transfers_to_wait: list[Any]) -> None: + """Fallback for mori builds without the batched wait. + + Gives failure precedence over pending: once any status has failed the + request is lost, so waiting out the remaining ones would only delay the + barrier by up to transfer_timeout. + """ timeout = self._transfer_timeout deadline = time.monotonic() + timeout remaining = list(transfers_to_wait) @@ -698,6 +726,8 @@ class MoRIIOWrapper: ) else: still_waiting.append(status) + if errors: + break remaining = still_waiting if remaining: time.sleep(0.001) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_layout.py b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_layout.py index 0ee2721139..b00692dc51 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_layout.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_layout.py @@ -2,14 +2,18 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from collections.abc import Callable, Mapping -from typing import NamedTuple +from typing import NamedTuple, cast import torch +from vllm.distributed.kv_transfer.kv_connector.v1.ssm_conv_transfer_utils import ( + MambaConvSplitInfo, +) from vllm.v1.kv_cache_interface import ( AttentionSpec, KVCacheConfig, KVCacheSpec, + MambaSpec, MLAAttentionSpec, SlidingWindowMLASpec, UniformTypeKVCacheSpecs, @@ -29,6 +33,85 @@ class LayerTransferGeometry(NamedTuple): split_kv_regions: bool +class MambaTransferGeometry(NamedTuple): + """Geometry of a hybrid (mamba/KDA) layer's recurrent state. + + A KDA layer's packed page is exposed as ``conv_state`` and ``ssm_state`` + views. Each is slot-strided, so a slot's bytes live at + ``slot * slot_stride * element_size`` and span ``slot_bytes``. The two + tensors are registered as two separate MoRIIO regions per layer. + """ + + conv_slot_stride: int + ssm_slot_stride: int + ssm_slot_bytes: int + conv_element_size: int + ssm_element_size: int + conv_region_len: int + ssm_region_len: int + + +def get_mamba_transfer_geometry( + conv: torch.Tensor, + ssm: torch.Tensor, +) -> MambaTransferGeometry: + """Derive the slot-strided geometry of a KDA layer's conv + ssm state. + + ``conv``/``ssm`` are zero-copy views over the layer's packed page. Slot byte + offset is ``slot * slot_stride * element_size`` and each + slot spans ``subtensor[0].numel() * element_size`` bytes; the whole tensor + is registered as one region (``*_region_len`` bytes). + """ + if conv.ndim == 0 or ssm.ndim == 0 or conv.shape[0] == 0 or ssm.shape[0] == 0: + raise ValueError("KDA conv/ssm caches must contain at least one slot") + if conv.shape[0] != ssm.shape[0]: + raise ValueError( + f"KDA conv/ssm slot count mismatch: {conv.shape[0]} != {ssm.shape[0]}" + ) + conv_slot_bytes = conv[0].numel() * conv.element_size() + ssm_slot_bytes = ssm[0].numel() * ssm.element_size() + conv_available_bytes = ( + conv.untyped_storage().nbytes() - conv.storage_offset() * conv.element_size() + ) + ssm_available_bytes = ( + ssm.untyped_storage().nbytes() - ssm.storage_offset() * ssm.element_size() + ) + if conv.shape[0]: + conv_required_bytes = (conv.shape[0] - 1) * conv.stride( + 0 + ) * conv.element_size() + conv_slot_bytes + ssm_required_bytes = (ssm.shape[0] - 1) * ssm.stride( + 0 + ) * ssm.element_size() + ssm_slot_bytes + if conv_required_bytes > conv_available_bytes: + raise ValueError("KDA conv slots exceed the registered storage extent") + if ssm_required_bytes > ssm_available_bytes: + raise ValueError("KDA ssm slots exceed the registered storage extent") + + return MambaTransferGeometry( + conv_slot_stride=conv.stride(0), + ssm_slot_stride=ssm.stride(0), + ssm_slot_bytes=ssm_slot_bytes, + conv_element_size=conv.element_size(), + ssm_element_size=ssm.element_size(), + # Registered region must span the full slot-strided extent: stride(0) + # may exceed the per-slot element count, so numel() would under-size the + # region and truncate the highest-slot transfers. shape[0]*stride(0) + # covers every byte a ``slot * slot_stride * elem`` offset can address. + # Cap at the bytes remaining from each view's storage offset. A packed + # conv+ssm page places ssm at a non-zero intra-buffer offset, + # so do not register past the end of the shared backing allocation. + conv_region_len=min( + conv.shape[0] * conv.stride(0) * conv.element_size(), + conv_available_bytes, + ), + ssm_region_len=min( + ssm.shape[0] * ssm.stride(0) * ssm.element_size(), + ssm_available_bytes, + ), + ) + + def build_layer_to_spec(kv_cache_config: KVCacheConfig) -> dict[str, KVCacheSpec]: layer_to_spec: dict[str, KVCacheSpec] = {} for group in kv_cache_config.kv_cache_groups: @@ -99,10 +182,10 @@ def get_layer_transfer_geometry( layer_to_spec: Mapping[str, KVCacheSpec], remote_num_blocks: int | None = None, ) -> LayerTransferGeometry: + spec = layer_to_spec[layer_name] shape = kv_cache.shape stride = kv_cache.stride() element_size = kv_cache.element_size() - spec = layer_to_spec[layer_name] is_mla_cache = is_mla_cache_layer(layer_to_spec, layer_name) if not is_mla_cache and len(shape) == 5 and shape[0] == 2: @@ -238,12 +321,60 @@ def get_layer_transfer_geometry( ) +def kda_conv_ssm( + kv_cache: torch.Tensor, + spec: MambaSpec, +) -> tuple[torch.Tensor, torch.Tensor]: + """Return zero-copy conv and SSM views over a packed two-state page.""" + from math import prod + + from vllm.utils.torch_utils import get_dtype_size + + if len(spec.shapes) != 2 or len(spec.dtypes) != 2: + raise ValueError( + "MoRIIO KDA transfer requires exactly two states (conv and SSM)" + ) + if kv_cache.element_size() != 1: + raise ValueError("MoRIIO KDA packed cache must use a byte-sized dtype") + if kv_cache.ndim != 4 or kv_cache.shape[1:3] != (1, 1) or kv_cache.stride(3) != 1: + raise ValueError( + "MoRIIO KDA packed cache must have shape [B, 1, 1, C] with " + "a contiguous byte dimension" + ) + # Mirror MambaBase.bind_kv_cache: blocks may be strided within a shared + # multi-layer allocation, while every individual byte page is contiguous. + pages = kv_cache.squeeze(dim=(1, 2)) + states: list[torch.Tensor] = [] + offset = 0 + for shp, dt in zip(spec.shapes, spec.dtypes, strict=True): + nbytes = prod(shp) * get_dtype_size(dt) + if offset + nbytes > pages.shape[1]: + raise ValueError( + "MoRIIO KDA state layout exceeds the packed cache page size" + ) + state = pages[:, offset : offset + nbytes].view(dt) + states.append(state.view(-1, *shp)) + offset += nbytes + return states[0], states[1] + + def iter_layer_registration_regions( layer_name: str, kv_cache: torch.Tensor, layer_to_spec: Mapping[str, KVCacheSpec], ) -> list[tuple[torch.Tensor, int]]: - geometry = get_layer_transfer_geometry(layer_name, kv_cache, layer_to_spec) + spec = layer_to_spec[layer_name] + if isinstance(spec, MambaSpec): + # KDA layer: register the conv and ssm tensors as two whole-tensor + # regions (the conv 3-subprojection split is expressed as transfer + # offset triples, not extra registrations). + conv, ssm = kda_conv_ssm(kv_cache, spec) + geom = get_mamba_transfer_geometry(conv, ssm) + return [(conv, geom.conv_region_len), (ssm, geom.ssm_region_len)] + geometry = cast( + LayerTransferGeometry, + get_layer_transfer_geometry(layer_name, kv_cache, layer_to_spec), + ) region_len = geometry.num_blocks * geometry.regions_per_block * geometry.block_len if geometry.regions_per_block == 1: # With padded or interleaved pages the block stride exceeds the @@ -311,9 +442,36 @@ def compute_block_transfer_offsets( "local_block_ids longer than remote_block_ids: " f"{len(local_block_ids)} > {len(remote_block_ids)}" ) - geometry = get_layer_transfer_geometry( - layer_name, kv_cache, layer_to_spec, remote_num_blocks + geometry = cast( + LayerTransferGeometry, + get_layer_transfer_geometry( + layer_name, kv_cache, layer_to_spec, remote_num_blocks + ), ) + invalid_local = next( + ( + block_id + for block_id in local_block_ids + if not 0 <= block_id < geometry.num_blocks + ), + None, + ) + if invalid_local is not None: + raise ValueError( + f"local block id {invalid_local} is outside [0, {geometry.num_blocks})" + ) + invalid_remote = next( + ( + block_id + for block_id in remote_block_ids + if not 0 <= block_id < remote_num_blocks + ), + None, + ) + if invalid_remote is not None: + raise ValueError( + f"remote block id {invalid_remote} is outside [0, {remote_num_blocks})" + ) element_size = kv_cache.element_size() transfer_size_byte = geometry.block_len per_block = geometry.transfers_per_block @@ -339,3 +497,125 @@ def compute_block_transfer_offsets( w += 1 return merge_fn(offset_local, offset_remote, sizes) + + +class MambaOffsetTemplate(NamedTuple): + """Slot-independent conv+ssm offset decomposition for a KDA layer. + + Homogeneous-TP geometry (conv/ssm slot strides, conv sub-projection + offsets, ssm per-slot size) is identical across requests and across the + homogeneous GDN layers, so it is computed once and reused; only the + per-request slot bases vary (see ``apply_mamba_offset_template``). + """ + + num_slots: int + conv_slot_stride_bytes: int + ssm_slot_stride_bytes: int + ssm_read_bytes: int + conv_subprojs: tuple[tuple[int, int, int], ...] + + +def build_mamba_offset_template( + conv: torch.Tensor, + ssm: torch.Tensor, + split_info: MambaConvSplitInfo, + tp_ratio: int, +) -> MambaOffsetTemplate: + """Precompute the slot-independent conv+ssm offset decomposition. + + Only homogeneous TP (``tp_ratio == 1``) is supported; heterogeneous TP is + gated with ``NotImplementedError`` (see the design doc). The returned + template is a pure function of the layer geometry and ``split_info``, so it + can be cached and reused across requests without recomputation. + """ + if tp_ratio != 1: + # Heterogeneous-TP recurrent-state transfer needs the remote page's + # slot stride (which differs from the local page) and, for + # P_TP > D_TP, a multi-rank gather of conv/ssm slices. Both are + # follow-ups. The mamba path bypasses the attention KV-head validator, + # so fail loudly here instead of silently transferring wrong bytes. + raise NotImplementedError( + "heterogeneous-TP mamba/KDA transfer is not supported yet " + f"(tp_ratio={tp_ratio}); use equal prefill/decode TP for " + "hybrid models" + ) + geom = get_mamba_transfer_geometry(conv, ssm) + conv_local_offsets = split_info.local_conv_offsets + conv_remote_offsets = split_info.remote_conv_offsets(0, 1) + ssm_read_bytes = geom.ssm_slot_bytes + conv_subprojs = tuple( + (loff, roff, rsz) + for (loff, _lsz), (roff, rsz) in zip(conv_local_offsets, conv_remote_offsets) + ) + return MambaOffsetTemplate( + num_slots=conv.shape[0], + conv_slot_stride_bytes=geom.conv_slot_stride * geom.conv_element_size, + ssm_slot_stride_bytes=geom.ssm_slot_stride * geom.ssm_element_size, + ssm_read_bytes=ssm_read_bytes, + conv_subprojs=conv_subprojs, + ) + + +def apply_mamba_offset_template( + template: MambaOffsetTemplate, + local_slots: list[int], + remote_slots: list[int], +) -> tuple[list[int], list[int], list[int]]: + """Apply per-request slot bases to a cached ``MambaOffsetTemplate``. + + Conv sub-projection entries (all slots) come first, followed by one ssm + entry per slot. + """ + if len(local_slots) != len(remote_slots): + raise ValueError( + "local and remote mamba slot counts must match: " + f"{len(local_slots)} != {len(remote_slots)}" + ) + invalid_local = next( + (slot for slot in local_slots if not 0 <= slot < template.num_slots), None + ) + if invalid_local is not None: + raise ValueError( + f"local mamba slot {invalid_local} is outside [0, {template.num_slots})" + ) + invalid_remote = next((slot for slot in remote_slots if slot < 0), None) + if invalid_remote is not None: + raise ValueError(f"remote mamba slot {invalid_remote} must be non-negative") + # The peer's KDA slot count is not part of MoRIIOAgentMetadata yet. Do not + # compare remote slot IDs against the local pool size: two otherwise + # compatible engines may reserve different numbers of slots. + local_offs: list[int] = [] + remote_offs: list[int] = [] + sizes: list[int] = [] + + # Conv sub-projections (region 0), all slots first. + conv_slot_stride_bytes = template.conv_slot_stride_bytes + for ls, rs in zip(local_slots, remote_slots): + lbase = ls * conv_slot_stride_bytes + rbase = rs * conv_slot_stride_bytes + for loff, roff, rsz in template.conv_subprojs: + local_offs.append(lbase + loff) + remote_offs.append(rbase + roff) + sizes.append(rsz) + + # SSM temporal state (region 1), one entry per slot. + ssm_slot_stride_bytes = template.ssm_slot_stride_bytes + ssm_read_bytes = template.ssm_read_bytes + for ls, rs in zip(local_slots, remote_slots): + local_offs.append(ls * ssm_slot_stride_bytes) + remote_offs.append(rs * ssm_slot_stride_bytes) + sizes.append(ssm_read_bytes) + + return local_offs, remote_offs, sizes + + +def compute_mamba_conv_split_count( + local_slots: list[int], + split_info: MambaConvSplitInfo, +) -> int: + """Number of leading conv entries in a KDA layer transfer plan. + + Entries ``[:count]`` are conv sub-projections (conv region/session); + entries ``[count:]`` are the per-slot ssm state (ssm region/session). + """ + return len(local_slots) * len(split_info.local_conv_offsets)