[KVConnector][MoRIIO] Support K3 DSpark hybrid READ (#57700)

Signed-off-by: Yichao Zhu <Yichao.Zhu@amd.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
YukioZzz
2026-09-29 19:14:24 +00:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 35cd03ebff
commit 30ae8248b6
5 changed files with 288 additions and 45 deletions
@@ -33,6 +33,18 @@ EngineId = str
BlockIds = tuple[list[int], ...] | list[list[int]]
def clip_ssm_state_blocks(
blocks: list[int], num_spec_blocks: int, positional: bool
) -> list[int]:
"""Drop speculative scratch slots, retaining position-indexed or running state."""
if not blocks:
return blocks
# Keep at least the running-state slot, even before scratch allocation.
if num_scratch := min(num_spec_blocks, len(blocks) - 1):
blocks = blocks[:-num_scratch]
return blocks if positional else blocks[-1:]
def get_kv_connector_cache_layout(vllm_config: VllmConfig | None = None):
# NOTE (NickLucche) When running disaggregated PD with NIXL, LBHNC layout is
# used for faster transfer.
@@ -17,7 +17,10 @@ import torch
import zmq
from vllm.config import CUDAGraphMode, VllmConfig
from vllm.distributed.kv_transfer.kv_connector.utils import BlockIds
from vllm.distributed.kv_transfer.kv_connector.utils import (
BlockIds,
clip_ssm_state_blocks,
)
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorBase_V1,
KVConnectorMetadata,
@@ -88,10 +91,10 @@ from vllm.utils.network_utils import (
from vllm.v1.attention.selector import get_attn_backend
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheConfig,
MambaSpec,
SlidingWindowSpec,
is_full_attention_spec,
)
from vllm.v1.outputs import KVConnectorOutput
from vllm.v1.request import RequestStatus
@@ -108,7 +111,7 @@ logger = init_logger(__name__)
# wait only needs to be long enough for it to make progress.
_SQ_FULL_BACKOFF_INITIAL_S = 0.001
_SQ_FULL_BACKOFF_MAX_S = 0.05
_MAX_LOCAL_DECODE_TAIL_BLOCKS = 1
_MAX_LOCAL_MAMBA_TAIL_BLOCKS = 1
try:
@@ -436,6 +439,26 @@ def _split_kv_cache_group_kinds(
return attn, mamba
def _validate_hybrid_speculation(vllm_config: VllmConfig) -> None:
speculative_config = vllm_config.speculative_config
if speculative_config is None:
return
# DSpark is allowlisted because it is the only hybrid speculative method
# validated end to end with MoRIIO READ.
if not speculative_config.use_dspark():
raise MoRIIOError(
"MoRIIO hybrid READ supports DSpark speculative decoding only, got "
f"method={speculative_config.method!r}"
)
if vllm_config.cache_config.mamba_cache_mode == "all":
# Positional allocation counts lookahead, so its local tail can exceed
# the single running-state block supported by hybrid READ.
raise MoRIIOError(
"MoRIIO hybrid READ does not support DSpark with "
"mamba_cache_mode='all'; use 'align'"
)
def _validate_mamba_specs(specs: Collection[MambaSpec]) -> MambaSpec | None:
"""Validate the common packed-state contract and return its representative."""
specs = tuple(specs)
@@ -477,22 +500,22 @@ class MoRIIOConnectorScheduler:
kv_cache_config
)
self._has_mamba = bool(self._mamba_group_ids)
mamba_spec = None
if self._has_mamba:
if len(self._attn_group_ids) != 1:
raise MoRIIOError(
"MoRIIO hybrid READ requires exactly one transferable "
"attention group, got "
f"{len(self._attn_group_ids)} attention groups"
f"{len(self._attn_group_ids)}; a drafter that owns a "
"separate attention group is not supported, give it the "
"target's group or disable KV transfer for it"
)
mamba_specs = [
cast(MambaSpec, kv_cache_config.transfer_groups[group_id].kv_cache_spec)
for group_id in self._mamba_group_ids
]
_validate_mamba_specs(mamba_specs)
if vllm_config.speculative_config is not None:
raise MoRIIOError(
"MoRIIO hybrid READ does not support speculative decoding"
)
mamba_spec = _validate_mamba_specs(mamba_specs)
_validate_hybrid_speculation(vllm_config)
if self.mode != MoRIIOMode.READ:
raise MoRIIOError(
"MoRIIO hybrid (mamba/KDA) transfer is implemented for READ "
@@ -503,18 +526,31 @@ class MoRIIOConnectorScheduler:
self._ssm_state_slots_are_positional = (
vllm_config.cache_config.mamba_cache_mode == "all"
)
self.block_size = vllm_config.cache_config.block_size
self._max_decode_tail_blocks = (
_MAX_LOCAL_DECODE_TAIL_BLOCKS
if self._has_mamba
else math.ceil((vllm_config.num_lookahead_tokens + 1) / self.block_size)
self._num_ssm_scratch_blocks = (
mamba_spec.num_speculative_blocks if mamba_spec is not None else 0
)
self.block_size = vllm_config.cache_config.block_size
attn_block_size = self.block_size
max_decode_tail_tokens = 1 + vllm_config.num_lookahead_tokens
if self._has_mamba:
attn_spec = kv_cache_config.transfer_groups[
self._attn_group_ids[0]
].kv_cache_spec
attn_block_size = attn_spec.block_size
if attn_spec.dcp_sharded:
attn_block_size *= (
vllm_config.parallel_config.decode_context_parallel_size
)
# Admission can pad the final prompt token to 1 + num_spec rows;
# allocate_slots reserves drafter lookahead in addition to those.
max_decode_tail_tokens += vllm_config.num_speculative_tokens
self._max_decode_tail_blocks = cdiv(max_decode_tail_tokens, attn_block_size)
self.engine_id: EngineId = engine_id
self._is_hma_required = (
not vllm_config.scheduler_config.disable_hybrid_kv_cache_manager
and any(
not isinstance(g.kv_cache_spec, FullAttentionSpec)
not is_full_attention_spec(g.kv_cache_spec)
for g in kv_cache_config.transfer_groups
)
)
@@ -524,9 +560,9 @@ class MoRIIOConnectorScheduler:
unsupported = [
type(g.kv_cache_spec).__name__
for g in kv_cache_config.transfer_groups
if not isinstance(
g.kv_cache_spec,
(FullAttentionSpec, SlidingWindowSpec, MambaSpec),
if not (
is_full_attention_spec(g.kv_cache_spec)
or isinstance(g.kv_cache_spec, (SlidingWindowSpec, MambaSpec))
)
]
if unsupported:
@@ -565,7 +601,7 @@ class MoRIIOConnectorScheduler:
self._full_attn_group_idx = 0
self._full_attn_block_size = self.block_size
for gi, group in enumerate(kv_cache_config.kv_cache_groups):
if isinstance(group.kv_cache_spec, FullAttentionSpec):
if is_full_attention_spec(group.kv_cache_spec):
self._full_attn_group_idx = gi
self._full_attn_block_size = getattr(
group.kv_cache_spec, "block_size", self.block_size
@@ -953,6 +989,9 @@ class MoRIIOConnectorScheduler:
)
aligned_local_mamba: list[list[int]] = []
aligned_remote_mamba: list[list[int]] = []
# Scratch slots were removed by
# split_block_groups, so only one local running
# state tail can remain.
for local_group, remote_group in zip(
mamba_block_groups,
remote_mamba_groups,
@@ -960,7 +999,9 @@ class MoRIIOConnectorScheduler:
):
aligned_local, aligned_remote = (
self._align_read_blocks(
local_group, remote_group, 1
local_group,
remote_group,
_MAX_LOCAL_MAMBA_TAIL_BLOCKS,
)
)
aligned_local_mamba.append(aligned_local)
@@ -1257,21 +1298,15 @@ class MoRIIOConnectorScheduler:
transfer_block_ids = self.kv_cache_config.select_transfer_block_ids(block_ids)
attn = list(transfer_block_ids[self._attn_group_ids[0]])
mamba_groups = [
self._clip_mamba_group(list(transfer_block_ids[group_id]))
clip_ssm_state_blocks(
list(transfer_block_ids[group_id]),
self._num_ssm_scratch_blocks,
self._ssm_state_slots_are_positional,
)
for group_id in self._mamba_group_ids
]
return attn, mamba_groups
def _clip_mamba_group(self, blocks: list[int]) -> list[int]:
"""Keep only the state-bearing slots of one mamba kv cache group."""
if not blocks:
return blocks
if not self._ssm_state_slots_are_positional:
# Single running state: everything before it is a null placeholder
# or the previous step's superseded state.
blocks = blocks[-1:]
return blocks
@staticmethod
def _align_read_blocks(
local_block_ids: list[int],
@@ -1282,9 +1317,9 @@ class MoRIIOConnectorScheduler:
Prefix hits can make the decode's unhashed suffix shorter than the
prefill's full list, so read from the matching remote suffix. The
decoder can instead have one extra tail block for the locally
recomputed final token; that block has no remote source and is not
transferred.
decoder can instead have extra tail blocks for the locally recomputed
final token and speculative slots; these have no remote source and are
not transferred.
"""
if len(local_block_ids) > len(remote_block_ids):
extra_decode_blocks = len(local_block_ids) - len(remote_block_ids)
@@ -13,6 +13,7 @@ from vllm import envs
from vllm.distributed.kv_transfer.kv_connector.utils import (
BlockIds,
EngineId,
clip_ssm_state_blocks,
yield_req_data,
)
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
@@ -288,11 +289,9 @@ class NixlBaseConnectorScheduler:
and blocks
and (n_spec_blocks := self._ssm_spec_blocks[i]) is not None
):
if n_spec := min(n_spec_blocks, len(blocks) - 1):
blocks = blocks[:-n_spec]
if not self._ssm_state_slots_are_positional:
# Never empty: downstream reads that as a full prefix hit.
blocks = blocks[-1:]
blocks = clip_ssm_state_blocks(
blocks, n_spec_blocks, self._ssm_state_slots_are_positional
)
clipped.append(blocks)
return tuple(clipped)