From 30ae8248b663ea56398d44d30c0d8276a598c4a0 Mon Sep 17 00:00:00 2001 From: YukioZzz Date: Wed, 30 Sep 2026 03:14:24 +0800 Subject: [PATCH] [KVConnector][MoRIIO] Support K3 DSpark hybrid READ (#57700) Signed-off-by: Yichao Zhu Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- .../unit/test_moriio_hma_scheduler.py | 200 +++++++++++++++++- .../unit/test_nixl_connector_hma.py | 9 + .../kv_transfer/kv_connector/utils.py | 12 ++ .../v1/moriio/moriio_connector.py | 103 ++++++--- .../kv_connector/v1/nixl/base_scheduler.py | 9 +- 5 files changed, 288 insertions(+), 45 deletions(-) diff --git a/tests/v1/kv_connector/unit/test_moriio_hma_scheduler.py b/tests/v1/kv_connector/unit/test_moriio_hma_scheduler.py index 13934f896a..87ea53225a 100644 --- a/tests/v1/kv_connector/unit/test_moriio_hma_scheduler.py +++ b/tests/v1/kv_connector/unit/test_moriio_hma_scheduler.py @@ -21,6 +21,9 @@ import pytest import torch from vllm.platforms import current_platform +from vllm.v1.core.block_pool import BlockPool +from vllm.v1.core.single_type_kv_cache_manager import FullAttentionManager +from vllm.v1.kv_cache_interface import FullAttentionSpec, UniformTypeKVCacheSpecs mori_available = importlib.util.find_spec("mori") is not None @@ -44,6 +47,16 @@ MoRIIOMode = moriio_connector.MoRIIOMode moriio_common = importlib.import_module( "vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_common" ) +get_role = moriio_common.get_role +set_role = moriio_common.set_role + + +@pytest.fixture +def restore_moriio_role(): + """Undo the process-global role a full scheduler construction leaves behind.""" + role = get_role() + yield + set_role(role) class _FakeScheduler(moriio_connector.MoRIIOConnectorScheduler): # type: ignore[name-defined] @@ -53,6 +66,7 @@ class _FakeScheduler(moriio_connector.MoRIIOConnectorScheduler): # type: ignore def __init__(self, **attrs): self._mamba_group_ids: list[int] = [] self._attn_group_ids: list[int] = [0] + self._num_ssm_scratch_blocks = 0 self._ssm_state_slots_are_positional = False self._is_hma_required = False self.kv_cache_config = SimpleNamespace( @@ -110,22 +124,56 @@ def _gdn_split_info(conv_rows=3, key_dim=4, value_dim=8, dtype_size=2): ) -def _mamba_spec(num_states: int = 2): +def _mamba_spec(num_states: int = 2, *, num_speculative_blocks: int = 0): shapes = tuple((1, 1) for _ in range(num_states)) return moriio_connector.MambaSpec( block_size=16, shapes=shapes, dtypes=(torch.float32,) * num_states, mamba_cache_mode="all", + num_speculative_blocks=num_speculative_blocks, ) -def _gate_vllm_config(*, read_mode: bool = True, speculative_config=None): +def _spec_config(method: str): + return SimpleNamespace(method=method, use_dspark=lambda: method == "dspark") + + +def _gate_vllm_config( + *, + read_mode: bool = True, + speculative_config=None, + block_size: int = 16, + num_lookahead_tokens: int = 0, + num_speculative_tokens: int = 0, + mamba_cache_mode: str = "all", + dcp_size: int = 1, +): return SimpleNamespace( kv_transfer_config=SimpleNamespace( - kv_connector_extra_config={"read_mode": read_mode} + kv_connector_extra_config={ + "read_mode": read_mode, + "host_ip": "127.0.0.1", + "handshake_port": 6001, + "notify_port": 6002, + }, + kv_role="kv_consumer", ), speculative_config=speculative_config, + cache_config=SimpleNamespace( + block_size=block_size, + mamba_cache_mode=mamba_cache_mode, + ), + scheduler_config=SimpleNamespace(disable_hybrid_kv_cache_manager=False), + num_lookahead_tokens=num_lookahead_tokens, + num_speculative_tokens=num_speculative_tokens, + parallel_config=SimpleNamespace( + tensor_parallel_size=1, + decode_context_parallel_size=dcp_size, + data_parallel_rank=0, + data_parallel_size_local=1, + data_parallel_size=1, + ), ) @@ -196,6 +244,7 @@ def test_exchange_blocks_ignore_transfer_disabled_group(): def test_scheduler_rejects_multiple_attention_groups_with_mamba(): + """A drafter owning its own transfer group is refused, with a hint.""" config = SimpleNamespace( kv_cache_groups=[ SimpleNamespace(enable_kv_transfer=True, kv_cache_spec=object()), @@ -205,7 +254,7 @@ def test_scheduler_rejects_multiple_attention_groups_with_mamba(): ) config.transfer_groups = tuple(config.kv_cache_groups) - with pytest.raises(moriio_common.MoRIIOError, match="exactly one transferable"): + with pytest.raises(moriio_common.MoRIIOError, match="separate attention group"): moriio_connector.MoRIIOConnectorScheduler(_gate_vllm_config(), "engine", config) @@ -251,7 +300,7 @@ def test_scheduler_rejects_mamba_group_without_two_states(): moriio_connector.MoRIIOConnectorScheduler(_gate_vllm_config(), "engine", config) -def test_scheduler_rejects_speculative_hybrid_read(): +def test_scheduler_rejects_non_dspark_speculative_hybrid_read(): config = SimpleNamespace( kv_cache_groups=[ SimpleNamespace(enable_kv_transfer=True, kv_cache_spec=object()), @@ -262,7 +311,105 @@ def test_scheduler_rejects_speculative_hybrid_read(): with pytest.raises(moriio_common.MoRIIOError, match="speculative decoding"): moriio_connector.MoRIIOConnectorScheduler( - _gate_vllm_config(speculative_config=object()), "engine", config + _gate_vllm_config(speculative_config=_spec_config("ngram")), + "engine", + config, + ) + + +def _dspark_wrapped_attention_config(*, dcp_sharded: bool = True): + """Mamba first, so a wrapped full-attention group at index 1 has to be found. + + A bare ``isinstance(FullAttentionSpec)`` misses the wrapper and leaves + ``_full_attn_group_idx`` at its initial 0, which is the Mamba group. + """ + full = FullAttentionSpec( + block_size=4, + num_kv_heads=1, + head_size=1, + dtype=torch.float32, + dcp_sharded=dcp_sharded, + ) + config = SimpleNamespace( + kv_cache_groups=[ + SimpleNamespace( + enable_kv_transfer=True, + kv_cache_spec=_mamba_spec(num_speculative_blocks=2), + ), + SimpleNamespace( + enable_kv_transfer=True, + kv_cache_spec=UniformTypeKVCacheSpecs( + block_size=4, kv_cache_specs={"target": full, "draft": full} + ), + ), + ], + ) + config.transfer_groups = tuple(config.kv_cache_groups) + return config + + +@pytest.mark.usefixtures("restore_moriio_role") +@pytest.mark.parametrize( + "dcp_size,dcp_sharded,expected_tail", [(1, True, 4), (4, True, 1), (4, False, 4)] +) +def test_scheduler_accepts_dspark_with_wrapped_full_attention( + dcp_size, dcp_sharded, expected_tail +): + config = _dspark_wrapped_attention_config(dcp_sharded=dcp_sharded) + vllm_config = _gate_vllm_config( + speculative_config=_spec_config("dspark"), + block_size=4, + num_lookahead_tokens=7, + num_speculative_tokens=7, + mamba_cache_mode="align", + dcp_size=dcp_size, + ) + scheduler = moriio_connector.MoRIIOConnectorScheduler(vllm_config, "engine", config) + + assert scheduler._full_attn_group_idx == 1 + assert scheduler._num_ssm_scratch_blocks == 2 + assert scheduler._max_decode_tail_blocks == expected_tail + + # A first decode step reserves padded target rows AND drafter lookahead. + spec = config.transfer_groups[1].kv_cache_spec + manager = FullAttentionManager( + spec, + block_pool=BlockPool(32, enable_caching=False, hash_block_size=4), + enable_caching=False, + kv_cache_group_id=1, + scheduler_block_size=16, + dcp_world_size=dcp_size, + ) + imported_tokens = manager.block_size + target_tokens = 1 + vllm_config.num_speculative_tokens + manager.allocate_new_blocks( + "decode", + num_tokens=imported_tokens + target_tokens + vllm_config.num_lookahead_tokens, + num_tokens_main_model=imported_tokens + target_tokens, + ) + local = [b.block_id for b in manager.req_to_blocks["decode"]] + remote = [100] + assert scheduler._align_read_blocks( + local, remote, scheduler._max_decode_tail_blocks + ) == (local[:1], remote) + with pytest.raises(ValueError, match="tail"): + scheduler._align_read_blocks( + local + [99], remote, scheduler._max_decode_tail_blocks + ) + + +def test_scheduler_rejects_dspark_with_positional_mamba_cache(): + """Positional mode counts lookahead in the Mamba allocation, so the + one-block recurrent-state tail bound no longer holds.""" + with pytest.raises(moriio_common.MoRIIOError, match="mamba_cache_mode"): + moriio_connector.MoRIIOConnectorScheduler( + _gate_vllm_config( + speculative_config=_spec_config("dspark"), + block_size=4, + mamba_cache_mode="all", + ), + "engine", + _dspark_wrapped_attention_config(), ) @@ -316,6 +463,30 @@ def test_split_block_groups_keeps_only_running_state_outside_all_mode(): assert sched.split_block_groups(([1], [40, 41])) == ([1], [[41]]) +@pytest.mark.parametrize( + "positional,blocks,expected", + [ + # Trailing scratch slots go in either cache mode. + (True, [40, 41, 42], [40]), + # Outside "all" the running state is picked only after they are gone. + (False, [40, 41, 42, 43], [41]), + # Never empty: a list shorter than the scratch count keeps one state. + (True, [40], [40]), + (False, [], []), + ], +) +def test_split_block_groups_strips_dspark_scratch_slots(positional, blocks, expected): + sched = _FakeScheduler( + _has_mamba=True, + _attn_group_ids=[0], + _mamba_group_ids=[1], + _num_ssm_scratch_blocks=2, + _ssm_state_slots_are_positional=positional, + ) + + assert sched.split_block_groups(([1], blocks)) == ([1], [expected]) + + # -------------------------------------------------------------------------- # request_finished_all_groups # -------------------------------------------------------------------------- @@ -434,6 +605,23 @@ def test_update_state_pairs_shorter_local_blocks_with_remote_suffix(): assert sched._req_kv_params["req"]["remote_block_ids"] == [[12], [92]] +def test_update_state_pairs_trimmed_mamba_state_with_remote_state(): + """The DSpark shape end to end: scratch slots are stripped, the running + state is then selected, and the attention list loses its lookahead tail.""" + sched = _make_read_scheduler() + sched._ssm_state_slots_are_positional = False + sched._num_ssm_scratch_blocks = 2 + sched._max_decode_tail_blocks = 2 + request = _make_read_request([[10, 11], [90]]) + # Mamba group: [null, running state, scratch, scratch]. + blocks = _FakeBlocks(all_groups=([100, 101, 102, 103], [200, 201, 202, 203])) + + sched.update_state_after_alloc(request, blocks, num_external_tokens=256) + + assert sched._reqs_need_recv["req"][1] == [[100, 101], [201]] + assert sched._req_kv_params["req"]["remote_block_ids"] == [[10, 11], [90]] + + def test_update_state_full_attention_hit_still_carries_mamba_state(): sched = _make_read_scheduler() request = _make_read_request([[10], [90]]) diff --git a/tests/v1/kv_connector/unit/test_nixl_connector_hma.py b/tests/v1/kv_connector/unit/test_nixl_connector_hma.py index 125dfa6143..629e8c7ecc 100644 --- a/tests/v1/kv_connector/unit/test_nixl_connector_hma.py +++ b/tests/v1/kv_connector/unit/test_nixl_connector_hma.py @@ -2055,6 +2055,15 @@ def test_exchange_clipped_blocks_ssm_single_state(): # Scratch slots not allocated: the state slot still survives. assert sched.get_exchange_clipped_blocks(([1], [5]))[1] == [5] + assert sched.get_exchange_clipped_blocks(([1], [])) == ([1], []) + # Per-step partial lists must not be interpreted as state + scratch slots. + assert sched.get_exchange_clipped_blocks(([1], [7, 8]), clip_ssm=False) == ( + [1], + [7, 8], + ) + sched.blocks_per_sw = [2, 0] + assert sched.get_exchange_clipped_blocks(([1, 2, 3], [7, 8, 9])) == ([2, 3], [7]) + # Non-mamba models pass through unchanged. fa_sched = make_nixl_scheduler(has_mamba=False) assert fa_sched.get_exchange_clipped_blocks(([1, 2],)) == ([1, 2],) diff --git a/vllm/distributed/kv_transfer/kv_connector/utils.py b/vllm/distributed/kv_transfer/kv_connector/utils.py index b645e97c4d..30021cde50 100644 --- a/vllm/distributed/kv_transfer/kv_connector/utils.py +++ b/vllm/distributed/kv_transfer/kv_connector/utils.py @@ -33,6 +33,18 @@ EngineId = str BlockIds = tuple[list[int], ...] | list[list[int]] +def clip_ssm_state_blocks( + blocks: list[int], num_spec_blocks: int, positional: bool +) -> list[int]: + """Drop speculative scratch slots, retaining position-indexed or running state.""" + if not blocks: + return blocks + # Keep at least the running-state slot, even before scratch allocation. + if num_scratch := min(num_spec_blocks, len(blocks) - 1): + blocks = blocks[:-num_scratch] + return blocks if positional else blocks[-1:] + + def get_kv_connector_cache_layout(vllm_config: VllmConfig | None = None): # NOTE (NickLucche) When running disaggregated PD with NIXL, LBHNC layout is # used for faster transfer. diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py index 0a8965a670..b482e6a8c4 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py @@ -17,7 +17,10 @@ import torch import zmq from vllm.config import CUDAGraphMode, VllmConfig -from vllm.distributed.kv_transfer.kv_connector.utils import BlockIds +from vllm.distributed.kv_transfer.kv_connector.utils import ( + BlockIds, + clip_ssm_state_blocks, +) from vllm.distributed.kv_transfer.kv_connector.v1.base import ( KVConnectorBase_V1, KVConnectorMetadata, @@ -88,10 +91,10 @@ from vllm.utils.network_utils import ( from vllm.v1.attention.selector import get_attn_backend from vllm.v1.core.sched.output import SchedulerOutput from vllm.v1.kv_cache_interface import ( - FullAttentionSpec, KVCacheConfig, MambaSpec, SlidingWindowSpec, + is_full_attention_spec, ) from vllm.v1.outputs import KVConnectorOutput from vllm.v1.request import RequestStatus @@ -108,7 +111,7 @@ logger = init_logger(__name__) # wait only needs to be long enough for it to make progress. _SQ_FULL_BACKOFF_INITIAL_S = 0.001 _SQ_FULL_BACKOFF_MAX_S = 0.05 -_MAX_LOCAL_DECODE_TAIL_BLOCKS = 1 +_MAX_LOCAL_MAMBA_TAIL_BLOCKS = 1 try: @@ -436,6 +439,26 @@ def _split_kv_cache_group_kinds( return attn, mamba +def _validate_hybrid_speculation(vllm_config: VllmConfig) -> None: + speculative_config = vllm_config.speculative_config + if speculative_config is None: + return + # DSpark is allowlisted because it is the only hybrid speculative method + # validated end to end with MoRIIO READ. + if not speculative_config.use_dspark(): + raise MoRIIOError( + "MoRIIO hybrid READ supports DSpark speculative decoding only, got " + f"method={speculative_config.method!r}" + ) + if vllm_config.cache_config.mamba_cache_mode == "all": + # Positional allocation counts lookahead, so its local tail can exceed + # the single running-state block supported by hybrid READ. + raise MoRIIOError( + "MoRIIO hybrid READ does not support DSpark with " + "mamba_cache_mode='all'; use 'align'" + ) + + def _validate_mamba_specs(specs: Collection[MambaSpec]) -> MambaSpec | None: """Validate the common packed-state contract and return its representative.""" specs = tuple(specs) @@ -477,22 +500,22 @@ class MoRIIOConnectorScheduler: kv_cache_config ) self._has_mamba = bool(self._mamba_group_ids) + mamba_spec = None if self._has_mamba: if len(self._attn_group_ids) != 1: raise MoRIIOError( "MoRIIO hybrid READ requires exactly one transferable " "attention group, got " - f"{len(self._attn_group_ids)} attention groups" + f"{len(self._attn_group_ids)}; a drafter that owns a " + "separate attention group is not supported, give it the " + "target's group or disable KV transfer for it" ) mamba_specs = [ cast(MambaSpec, kv_cache_config.transfer_groups[group_id].kv_cache_spec) for group_id in self._mamba_group_ids ] - _validate_mamba_specs(mamba_specs) - if vllm_config.speculative_config is not None: - raise MoRIIOError( - "MoRIIO hybrid READ does not support speculative decoding" - ) + mamba_spec = _validate_mamba_specs(mamba_specs) + _validate_hybrid_speculation(vllm_config) if self.mode != MoRIIOMode.READ: raise MoRIIOError( "MoRIIO hybrid (mamba/KDA) transfer is implemented for READ " @@ -503,18 +526,31 @@ class MoRIIOConnectorScheduler: self._ssm_state_slots_are_positional = ( vllm_config.cache_config.mamba_cache_mode == "all" ) - self.block_size = vllm_config.cache_config.block_size - self._max_decode_tail_blocks = ( - _MAX_LOCAL_DECODE_TAIL_BLOCKS - if self._has_mamba - else math.ceil((vllm_config.num_lookahead_tokens + 1) / self.block_size) + self._num_ssm_scratch_blocks = ( + mamba_spec.num_speculative_blocks if mamba_spec is not None else 0 ) + self.block_size = vllm_config.cache_config.block_size + attn_block_size = self.block_size + max_decode_tail_tokens = 1 + vllm_config.num_lookahead_tokens + if self._has_mamba: + attn_spec = kv_cache_config.transfer_groups[ + self._attn_group_ids[0] + ].kv_cache_spec + attn_block_size = attn_spec.block_size + if attn_spec.dcp_sharded: + attn_block_size *= ( + vllm_config.parallel_config.decode_context_parallel_size + ) + # Admission can pad the final prompt token to 1 + num_spec rows; + # allocate_slots reserves drafter lookahead in addition to those. + max_decode_tail_tokens += vllm_config.num_speculative_tokens + self._max_decode_tail_blocks = cdiv(max_decode_tail_tokens, attn_block_size) self.engine_id: EngineId = engine_id self._is_hma_required = ( not vllm_config.scheduler_config.disable_hybrid_kv_cache_manager and any( - not isinstance(g.kv_cache_spec, FullAttentionSpec) + not is_full_attention_spec(g.kv_cache_spec) for g in kv_cache_config.transfer_groups ) ) @@ -524,9 +560,9 @@ class MoRIIOConnectorScheduler: unsupported = [ type(g.kv_cache_spec).__name__ for g in kv_cache_config.transfer_groups - if not isinstance( - g.kv_cache_spec, - (FullAttentionSpec, SlidingWindowSpec, MambaSpec), + if not ( + is_full_attention_spec(g.kv_cache_spec) + or isinstance(g.kv_cache_spec, (SlidingWindowSpec, MambaSpec)) ) ] if unsupported: @@ -565,7 +601,7 @@ class MoRIIOConnectorScheduler: self._full_attn_group_idx = 0 self._full_attn_block_size = self.block_size for gi, group in enumerate(kv_cache_config.kv_cache_groups): - if isinstance(group.kv_cache_spec, FullAttentionSpec): + if is_full_attention_spec(group.kv_cache_spec): self._full_attn_group_idx = gi self._full_attn_block_size = getattr( group.kv_cache_spec, "block_size", self.block_size @@ -953,6 +989,9 @@ class MoRIIOConnectorScheduler: ) aligned_local_mamba: list[list[int]] = [] aligned_remote_mamba: list[list[int]] = [] + # Scratch slots were removed by + # split_block_groups, so only one local running + # state tail can remain. for local_group, remote_group in zip( mamba_block_groups, remote_mamba_groups, @@ -960,7 +999,9 @@ class MoRIIOConnectorScheduler: ): aligned_local, aligned_remote = ( self._align_read_blocks( - local_group, remote_group, 1 + local_group, + remote_group, + _MAX_LOCAL_MAMBA_TAIL_BLOCKS, ) ) aligned_local_mamba.append(aligned_local) @@ -1257,21 +1298,15 @@ class MoRIIOConnectorScheduler: transfer_block_ids = self.kv_cache_config.select_transfer_block_ids(block_ids) attn = list(transfer_block_ids[self._attn_group_ids[0]]) mamba_groups = [ - self._clip_mamba_group(list(transfer_block_ids[group_id])) + clip_ssm_state_blocks( + list(transfer_block_ids[group_id]), + self._num_ssm_scratch_blocks, + self._ssm_state_slots_are_positional, + ) for group_id in self._mamba_group_ids ] return attn, mamba_groups - def _clip_mamba_group(self, blocks: list[int]) -> list[int]: - """Keep only the state-bearing slots of one mamba kv cache group.""" - if not blocks: - return blocks - if not self._ssm_state_slots_are_positional: - # Single running state: everything before it is a null placeholder - # or the previous step's superseded state. - blocks = blocks[-1:] - return blocks - @staticmethod def _align_read_blocks( local_block_ids: list[int], @@ -1282,9 +1317,9 @@ class MoRIIOConnectorScheduler: Prefix hits can make the decode's unhashed suffix shorter than the prefill's full list, so read from the matching remote suffix. The - decoder can instead have one extra tail block for the locally - recomputed final token; that block has no remote source and is not - transferred. + decoder can instead have extra tail blocks for the locally recomputed + final token and speculative slots; these have no remote source and are + not transferred. """ if len(local_block_ids) > len(remote_block_ids): extra_decode_blocks = len(local_block_ids) - len(remote_block_ids) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_scheduler.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_scheduler.py index e264a94aaf..0705f944f6 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_scheduler.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_scheduler.py @@ -13,6 +13,7 @@ from vllm import envs from vllm.distributed.kv_transfer.kv_connector.utils import ( BlockIds, EngineId, + clip_ssm_state_blocks, yield_req_data, ) from vllm.distributed.kv_transfer.kv_connector.v1.base import ( @@ -288,11 +289,9 @@ class NixlBaseConnectorScheduler: and blocks and (n_spec_blocks := self._ssm_spec_blocks[i]) is not None ): - if n_spec := min(n_spec_blocks, len(blocks) - 1): - blocks = blocks[:-n_spec] - if not self._ssm_state_slots_are_positional: - # Never empty: downstream reads that as a full prefix hit. - blocks = blocks[-1:] + blocks = clip_ssm_state_blocks( + blocks, n_spec_blocks, self._ssm_state_slots_are_positional + ) clipped.append(blocks) return tuple(clipped)