[KVConnector][MoRIIO] Support K3 DSpark hybrid READ (#57700)

Signed-off-by: Yichao Zhu <Yichao.Zhu@amd.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
YukioZzz
2026-09-29 19:14:24 +00:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 35cd03ebff
commit 30ae8248b6
5 changed files with 288 additions and 45 deletions
@@ -21,6 +21,9 @@ import pytest
import torch
from vllm.platforms import current_platform
from vllm.v1.core.block_pool import BlockPool
from vllm.v1.core.single_type_kv_cache_manager import FullAttentionManager
from vllm.v1.kv_cache_interface import FullAttentionSpec, UniformTypeKVCacheSpecs
mori_available = importlib.util.find_spec("mori") is not None
@@ -44,6 +47,16 @@ MoRIIOMode = moriio_connector.MoRIIOMode
moriio_common = importlib.import_module(
"vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_common"
)
get_role = moriio_common.get_role
set_role = moriio_common.set_role
@pytest.fixture
def restore_moriio_role():
"""Undo the process-global role a full scheduler construction leaves behind."""
role = get_role()
yield
set_role(role)
class _FakeScheduler(moriio_connector.MoRIIOConnectorScheduler): # type: ignore[name-defined]
@@ -53,6 +66,7 @@ class _FakeScheduler(moriio_connector.MoRIIOConnectorScheduler): # type: ignore
def __init__(self, **attrs):
self._mamba_group_ids: list[int] = []
self._attn_group_ids: list[int] = [0]
self._num_ssm_scratch_blocks = 0
self._ssm_state_slots_are_positional = False
self._is_hma_required = False
self.kv_cache_config = SimpleNamespace(
@@ -110,22 +124,56 @@ def _gdn_split_info(conv_rows=3, key_dim=4, value_dim=8, dtype_size=2):
)
def _mamba_spec(num_states: int = 2):
def _mamba_spec(num_states: int = 2, *, num_speculative_blocks: int = 0):
shapes = tuple((1, 1) for _ in range(num_states))
return moriio_connector.MambaSpec(
block_size=16,
shapes=shapes,
dtypes=(torch.float32,) * num_states,
mamba_cache_mode="all",
num_speculative_blocks=num_speculative_blocks,
)
def _gate_vllm_config(*, read_mode: bool = True, speculative_config=None):
def _spec_config(method: str):
return SimpleNamespace(method=method, use_dspark=lambda: method == "dspark")
def _gate_vllm_config(
*,
read_mode: bool = True,
speculative_config=None,
block_size: int = 16,
num_lookahead_tokens: int = 0,
num_speculative_tokens: int = 0,
mamba_cache_mode: str = "all",
dcp_size: int = 1,
):
return SimpleNamespace(
kv_transfer_config=SimpleNamespace(
kv_connector_extra_config={"read_mode": read_mode}
kv_connector_extra_config={
"read_mode": read_mode,
"host_ip": "127.0.0.1",
"handshake_port": 6001,
"notify_port": 6002,
},
kv_role="kv_consumer",
),
speculative_config=speculative_config,
cache_config=SimpleNamespace(
block_size=block_size,
mamba_cache_mode=mamba_cache_mode,
),
scheduler_config=SimpleNamespace(disable_hybrid_kv_cache_manager=False),
num_lookahead_tokens=num_lookahead_tokens,
num_speculative_tokens=num_speculative_tokens,
parallel_config=SimpleNamespace(
tensor_parallel_size=1,
decode_context_parallel_size=dcp_size,
data_parallel_rank=0,
data_parallel_size_local=1,
data_parallel_size=1,
),
)
@@ -196,6 +244,7 @@ def test_exchange_blocks_ignore_transfer_disabled_group():
def test_scheduler_rejects_multiple_attention_groups_with_mamba():
"""A drafter owning its own transfer group is refused, with a hint."""
config = SimpleNamespace(
kv_cache_groups=[
SimpleNamespace(enable_kv_transfer=True, kv_cache_spec=object()),
@@ -205,7 +254,7 @@ def test_scheduler_rejects_multiple_attention_groups_with_mamba():
)
config.transfer_groups = tuple(config.kv_cache_groups)
with pytest.raises(moriio_common.MoRIIOError, match="exactly one transferable"):
with pytest.raises(moriio_common.MoRIIOError, match="separate attention group"):
moriio_connector.MoRIIOConnectorScheduler(_gate_vllm_config(), "engine", config)
@@ -251,7 +300,7 @@ def test_scheduler_rejects_mamba_group_without_two_states():
moriio_connector.MoRIIOConnectorScheduler(_gate_vllm_config(), "engine", config)
def test_scheduler_rejects_speculative_hybrid_read():
def test_scheduler_rejects_non_dspark_speculative_hybrid_read():
config = SimpleNamespace(
kv_cache_groups=[
SimpleNamespace(enable_kv_transfer=True, kv_cache_spec=object()),
@@ -262,7 +311,105 @@ def test_scheduler_rejects_speculative_hybrid_read():
with pytest.raises(moriio_common.MoRIIOError, match="speculative decoding"):
moriio_connector.MoRIIOConnectorScheduler(
_gate_vllm_config(speculative_config=object()), "engine", config
_gate_vllm_config(speculative_config=_spec_config("ngram")),
"engine",
config,
)
def _dspark_wrapped_attention_config(*, dcp_sharded: bool = True):
"""Mamba first, so a wrapped full-attention group at index 1 has to be found.
A bare ``isinstance(FullAttentionSpec)`` misses the wrapper and leaves
``_full_attn_group_idx`` at its initial 0, which is the Mamba group.
"""
full = FullAttentionSpec(
block_size=4,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
dcp_sharded=dcp_sharded,
)
config = SimpleNamespace(
kv_cache_groups=[
SimpleNamespace(
enable_kv_transfer=True,
kv_cache_spec=_mamba_spec(num_speculative_blocks=2),
),
SimpleNamespace(
enable_kv_transfer=True,
kv_cache_spec=UniformTypeKVCacheSpecs(
block_size=4, kv_cache_specs={"target": full, "draft": full}
),
),
],
)
config.transfer_groups = tuple(config.kv_cache_groups)
return config
@pytest.mark.usefixtures("restore_moriio_role")
@pytest.mark.parametrize(
"dcp_size,dcp_sharded,expected_tail", [(1, True, 4), (4, True, 1), (4, False, 4)]
)
def test_scheduler_accepts_dspark_with_wrapped_full_attention(
dcp_size, dcp_sharded, expected_tail
):
config = _dspark_wrapped_attention_config(dcp_sharded=dcp_sharded)
vllm_config = _gate_vllm_config(
speculative_config=_spec_config("dspark"),
block_size=4,
num_lookahead_tokens=7,
num_speculative_tokens=7,
mamba_cache_mode="align",
dcp_size=dcp_size,
)
scheduler = moriio_connector.MoRIIOConnectorScheduler(vllm_config, "engine", config)
assert scheduler._full_attn_group_idx == 1
assert scheduler._num_ssm_scratch_blocks == 2
assert scheduler._max_decode_tail_blocks == expected_tail
# A first decode step reserves padded target rows AND drafter lookahead.
spec = config.transfer_groups[1].kv_cache_spec
manager = FullAttentionManager(
spec,
block_pool=BlockPool(32, enable_caching=False, hash_block_size=4),
enable_caching=False,
kv_cache_group_id=1,
scheduler_block_size=16,
dcp_world_size=dcp_size,
)
imported_tokens = manager.block_size
target_tokens = 1 + vllm_config.num_speculative_tokens
manager.allocate_new_blocks(
"decode",
num_tokens=imported_tokens + target_tokens + vllm_config.num_lookahead_tokens,
num_tokens_main_model=imported_tokens + target_tokens,
)
local = [b.block_id for b in manager.req_to_blocks["decode"]]
remote = [100]
assert scheduler._align_read_blocks(
local, remote, scheduler._max_decode_tail_blocks
) == (local[:1], remote)
with pytest.raises(ValueError, match="tail"):
scheduler._align_read_blocks(
local + [99], remote, scheduler._max_decode_tail_blocks
)
def test_scheduler_rejects_dspark_with_positional_mamba_cache():
"""Positional mode counts lookahead in the Mamba allocation, so the
one-block recurrent-state tail bound no longer holds."""
with pytest.raises(moriio_common.MoRIIOError, match="mamba_cache_mode"):
moriio_connector.MoRIIOConnectorScheduler(
_gate_vllm_config(
speculative_config=_spec_config("dspark"),
block_size=4,
mamba_cache_mode="all",
),
"engine",
_dspark_wrapped_attention_config(),
)
@@ -316,6 +463,30 @@ def test_split_block_groups_keeps_only_running_state_outside_all_mode():
assert sched.split_block_groups(([1], [40, 41])) == ([1], [[41]])
@pytest.mark.parametrize(
"positional,blocks,expected",
[
# Trailing scratch slots go in either cache mode.
(True, [40, 41, 42], [40]),
# Outside "all" the running state is picked only after they are gone.
(False, [40, 41, 42, 43], [41]),
# Never empty: a list shorter than the scratch count keeps one state.
(True, [40], [40]),
(False, [], []),
],
)
def test_split_block_groups_strips_dspark_scratch_slots(positional, blocks, expected):
sched = _FakeScheduler(
_has_mamba=True,
_attn_group_ids=[0],
_mamba_group_ids=[1],
_num_ssm_scratch_blocks=2,
_ssm_state_slots_are_positional=positional,
)
assert sched.split_block_groups(([1], blocks)) == ([1], [expected])
# --------------------------------------------------------------------------
# request_finished_all_groups
# --------------------------------------------------------------------------
@@ -434,6 +605,23 @@ def test_update_state_pairs_shorter_local_blocks_with_remote_suffix():
assert sched._req_kv_params["req"]["remote_block_ids"] == [[12], [92]]
def test_update_state_pairs_trimmed_mamba_state_with_remote_state():
"""The DSpark shape end to end: scratch slots are stripped, the running
state is then selected, and the attention list loses its lookahead tail."""
sched = _make_read_scheduler()
sched._ssm_state_slots_are_positional = False
sched._num_ssm_scratch_blocks = 2
sched._max_decode_tail_blocks = 2
request = _make_read_request([[10, 11], [90]])
# Mamba group: [null, running state, scratch, scratch].
blocks = _FakeBlocks(all_groups=([100, 101, 102, 103], [200, 201, 202, 203]))
sched.update_state_after_alloc(request, blocks, num_external_tokens=256)
assert sched._reqs_need_recv["req"][1] == [[100, 101], [201]]
assert sched._req_kv_params["req"]["remote_block_ids"] == [[10, 11], [90]]
def test_update_state_full_attention_hit_still_carries_mamba_state():
sched = _make_read_scheduler()
request = _make_read_request([[10], [90]])
@@ -2055,6 +2055,15 @@ def test_exchange_clipped_blocks_ssm_single_state():
# Scratch slots not allocated: the state slot still survives.
assert sched.get_exchange_clipped_blocks(([1], [5]))[1] == [5]
assert sched.get_exchange_clipped_blocks(([1], [])) == ([1], [])
# Per-step partial lists must not be interpreted as state + scratch slots.
assert sched.get_exchange_clipped_blocks(([1], [7, 8]), clip_ssm=False) == (
[1],
[7, 8],
)
sched.blocks_per_sw = [2, 0]
assert sched.get_exchange_clipped_blocks(([1, 2, 3], [7, 8, 9])) == ([2, 3], [7])
# Non-mamba models pass through unchanged.
fa_sched = make_nixl_scheduler(has_mamba=False)
assert fa_sched.get_exchange_clipped_blocks(([1, 2],)) == ([1, 2],)
@@ -33,6 +33,18 @@ EngineId = str
BlockIds = tuple[list[int], ...] | list[list[int]]
def clip_ssm_state_blocks(
blocks: list[int], num_spec_blocks: int, positional: bool
) -> list[int]:
"""Drop speculative scratch slots, retaining position-indexed or running state."""
if not blocks:
return blocks
# Keep at least the running-state slot, even before scratch allocation.
if num_scratch := min(num_spec_blocks, len(blocks) - 1):
blocks = blocks[:-num_scratch]
return blocks if positional else blocks[-1:]
def get_kv_connector_cache_layout(vllm_config: VllmConfig | None = None):
# NOTE (NickLucche) When running disaggregated PD with NIXL, LBHNC layout is
# used for faster transfer.
@@ -17,7 +17,10 @@ import torch
import zmq
from vllm.config import CUDAGraphMode, VllmConfig
from vllm.distributed.kv_transfer.kv_connector.utils import BlockIds
from vllm.distributed.kv_transfer.kv_connector.utils import (
BlockIds,
clip_ssm_state_blocks,
)
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorBase_V1,
KVConnectorMetadata,
@@ -88,10 +91,10 @@ from vllm.utils.network_utils import (
from vllm.v1.attention.selector import get_attn_backend
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheConfig,
MambaSpec,
SlidingWindowSpec,
is_full_attention_spec,
)
from vllm.v1.outputs import KVConnectorOutput
from vllm.v1.request import RequestStatus
@@ -108,7 +111,7 @@ logger = init_logger(__name__)
# wait only needs to be long enough for it to make progress.
_SQ_FULL_BACKOFF_INITIAL_S = 0.001
_SQ_FULL_BACKOFF_MAX_S = 0.05
_MAX_LOCAL_DECODE_TAIL_BLOCKS = 1
_MAX_LOCAL_MAMBA_TAIL_BLOCKS = 1
try:
@@ -436,6 +439,26 @@ def _split_kv_cache_group_kinds(
return attn, mamba
def _validate_hybrid_speculation(vllm_config: VllmConfig) -> None:
speculative_config = vllm_config.speculative_config
if speculative_config is None:
return
# DSpark is allowlisted because it is the only hybrid speculative method
# validated end to end with MoRIIO READ.
if not speculative_config.use_dspark():
raise MoRIIOError(
"MoRIIO hybrid READ supports DSpark speculative decoding only, got "
f"method={speculative_config.method!r}"
)
if vllm_config.cache_config.mamba_cache_mode == "all":
# Positional allocation counts lookahead, so its local tail can exceed
# the single running-state block supported by hybrid READ.
raise MoRIIOError(
"MoRIIO hybrid READ does not support DSpark with "
"mamba_cache_mode='all'; use 'align'"
)
def _validate_mamba_specs(specs: Collection[MambaSpec]) -> MambaSpec | None:
"""Validate the common packed-state contract and return its representative."""
specs = tuple(specs)
@@ -477,22 +500,22 @@ class MoRIIOConnectorScheduler:
kv_cache_config
)
self._has_mamba = bool(self._mamba_group_ids)
mamba_spec = None
if self._has_mamba:
if len(self._attn_group_ids) != 1:
raise MoRIIOError(
"MoRIIO hybrid READ requires exactly one transferable "
"attention group, got "
f"{len(self._attn_group_ids)} attention groups"
f"{len(self._attn_group_ids)}; a drafter that owns a "
"separate attention group is not supported, give it the "
"target's group or disable KV transfer for it"
)
mamba_specs = [
cast(MambaSpec, kv_cache_config.transfer_groups[group_id].kv_cache_spec)
for group_id in self._mamba_group_ids
]
_validate_mamba_specs(mamba_specs)
if vllm_config.speculative_config is not None:
raise MoRIIOError(
"MoRIIO hybrid READ does not support speculative decoding"
)
mamba_spec = _validate_mamba_specs(mamba_specs)
_validate_hybrid_speculation(vllm_config)
if self.mode != MoRIIOMode.READ:
raise MoRIIOError(
"MoRIIO hybrid (mamba/KDA) transfer is implemented for READ "
@@ -503,18 +526,31 @@ class MoRIIOConnectorScheduler:
self._ssm_state_slots_are_positional = (
vllm_config.cache_config.mamba_cache_mode == "all"
)
self.block_size = vllm_config.cache_config.block_size
self._max_decode_tail_blocks = (
_MAX_LOCAL_DECODE_TAIL_BLOCKS
if self._has_mamba
else math.ceil((vllm_config.num_lookahead_tokens + 1) / self.block_size)
self._num_ssm_scratch_blocks = (
mamba_spec.num_speculative_blocks if mamba_spec is not None else 0
)
self.block_size = vllm_config.cache_config.block_size
attn_block_size = self.block_size
max_decode_tail_tokens = 1 + vllm_config.num_lookahead_tokens
if self._has_mamba:
attn_spec = kv_cache_config.transfer_groups[
self._attn_group_ids[0]
].kv_cache_spec
attn_block_size = attn_spec.block_size
if attn_spec.dcp_sharded:
attn_block_size *= (
vllm_config.parallel_config.decode_context_parallel_size
)
# Admission can pad the final prompt token to 1 + num_spec rows;
# allocate_slots reserves drafter lookahead in addition to those.
max_decode_tail_tokens += vllm_config.num_speculative_tokens
self._max_decode_tail_blocks = cdiv(max_decode_tail_tokens, attn_block_size)
self.engine_id: EngineId = engine_id
self._is_hma_required = (
not vllm_config.scheduler_config.disable_hybrid_kv_cache_manager
and any(
not isinstance(g.kv_cache_spec, FullAttentionSpec)
not is_full_attention_spec(g.kv_cache_spec)
for g in kv_cache_config.transfer_groups
)
)
@@ -524,9 +560,9 @@ class MoRIIOConnectorScheduler:
unsupported = [
type(g.kv_cache_spec).__name__
for g in kv_cache_config.transfer_groups
if not isinstance(
g.kv_cache_spec,
(FullAttentionSpec, SlidingWindowSpec, MambaSpec),
if not (
is_full_attention_spec(g.kv_cache_spec)
or isinstance(g.kv_cache_spec, (SlidingWindowSpec, MambaSpec))
)
]
if unsupported:
@@ -565,7 +601,7 @@ class MoRIIOConnectorScheduler:
self._full_attn_group_idx = 0
self._full_attn_block_size = self.block_size
for gi, group in enumerate(kv_cache_config.kv_cache_groups):
if isinstance(group.kv_cache_spec, FullAttentionSpec):
if is_full_attention_spec(group.kv_cache_spec):
self._full_attn_group_idx = gi
self._full_attn_block_size = getattr(
group.kv_cache_spec, "block_size", self.block_size
@@ -953,6 +989,9 @@ class MoRIIOConnectorScheduler:
)
aligned_local_mamba: list[list[int]] = []
aligned_remote_mamba: list[list[int]] = []
# Scratch slots were removed by
# split_block_groups, so only one local running
# state tail can remain.
for local_group, remote_group in zip(
mamba_block_groups,
remote_mamba_groups,
@@ -960,7 +999,9 @@ class MoRIIOConnectorScheduler:
):
aligned_local, aligned_remote = (
self._align_read_blocks(
local_group, remote_group, 1
local_group,
remote_group,
_MAX_LOCAL_MAMBA_TAIL_BLOCKS,
)
)
aligned_local_mamba.append(aligned_local)
@@ -1257,21 +1298,15 @@ class MoRIIOConnectorScheduler:
transfer_block_ids = self.kv_cache_config.select_transfer_block_ids(block_ids)
attn = list(transfer_block_ids[self._attn_group_ids[0]])
mamba_groups = [
self._clip_mamba_group(list(transfer_block_ids[group_id]))
clip_ssm_state_blocks(
list(transfer_block_ids[group_id]),
self._num_ssm_scratch_blocks,
self._ssm_state_slots_are_positional,
)
for group_id in self._mamba_group_ids
]
return attn, mamba_groups
def _clip_mamba_group(self, blocks: list[int]) -> list[int]:
"""Keep only the state-bearing slots of one mamba kv cache group."""
if not blocks:
return blocks
if not self._ssm_state_slots_are_positional:
# Single running state: everything before it is a null placeholder
# or the previous step's superseded state.
blocks = blocks[-1:]
return blocks
@staticmethod
def _align_read_blocks(
local_block_ids: list[int],
@@ -1282,9 +1317,9 @@ class MoRIIOConnectorScheduler:
Prefix hits can make the decode's unhashed suffix shorter than the
prefill's full list, so read from the matching remote suffix. The
decoder can instead have one extra tail block for the locally
recomputed final token; that block has no remote source and is not
transferred.
decoder can instead have extra tail blocks for the locally recomputed
final token and speculative slots; these have no remote source and are
not transferred.
"""
if len(local_block_ids) > len(remote_block_ids):
extra_decode_blocks = len(local_block_ids) - len(remote_block_ids)
@@ -13,6 +13,7 @@ from vllm import envs
from vllm.distributed.kv_transfer.kv_connector.utils import (
BlockIds,
EngineId,
clip_ssm_state_blocks,
yield_req_data,
)
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
@@ -288,11 +289,9 @@ class NixlBaseConnectorScheduler:
and blocks
and (n_spec_blocks := self._ssm_spec_blocks[i]) is not None
):
if n_spec := min(n_spec_blocks, len(blocks) - 1):
blocks = blocks[:-n_spec]
if not self._ssm_state_slots_are_positional:
# Never empty: downstream reads that as a full prefix hit.
blocks = blocks[-1:]
blocks = clip_ssm_state_blocks(
blocks, n_spec_blocks, self._ssm_state_slots_are_positional
)
clipped.append(blocks)
return tuple(clipped)