[KVConnector][MoRIIO] Support K3 DSpark hybrid READ (#57700)
Signed-off-by: Yichao Zhu <Yichao.Zhu@amd.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
35cd03ebff
commit
30ae8248b6
@@ -33,6 +33,18 @@ EngineId = str
|
||||
BlockIds = tuple[list[int], ...] | list[list[int]]
|
||||
|
||||
|
||||
def clip_ssm_state_blocks(
|
||||
blocks: list[int], num_spec_blocks: int, positional: bool
|
||||
) -> list[int]:
|
||||
"""Drop speculative scratch slots, retaining position-indexed or running state."""
|
||||
if not blocks:
|
||||
return blocks
|
||||
# Keep at least the running-state slot, even before scratch allocation.
|
||||
if num_scratch := min(num_spec_blocks, len(blocks) - 1):
|
||||
blocks = blocks[:-num_scratch]
|
||||
return blocks if positional else blocks[-1:]
|
||||
|
||||
|
||||
def get_kv_connector_cache_layout(vllm_config: VllmConfig | None = None):
|
||||
# NOTE (NickLucche) When running disaggregated PD with NIXL, LBHNC layout is
|
||||
# used for faster transfer.
|
||||
|
||||
@@ -17,7 +17,10 @@ import torch
|
||||
import zmq
|
||||
|
||||
from vllm.config import CUDAGraphMode, VllmConfig
|
||||
from vllm.distributed.kv_transfer.kv_connector.utils import BlockIds
|
||||
from vllm.distributed.kv_transfer.kv_connector.utils import (
|
||||
BlockIds,
|
||||
clip_ssm_state_blocks,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorBase_V1,
|
||||
KVConnectorMetadata,
|
||||
@@ -88,10 +91,10 @@ from vllm.utils.network_utils import (
|
||||
from vllm.v1.attention.selector import get_attn_backend
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheConfig,
|
||||
MambaSpec,
|
||||
SlidingWindowSpec,
|
||||
is_full_attention_spec,
|
||||
)
|
||||
from vllm.v1.outputs import KVConnectorOutput
|
||||
from vllm.v1.request import RequestStatus
|
||||
@@ -108,7 +111,7 @@ logger = init_logger(__name__)
|
||||
# wait only needs to be long enough for it to make progress.
|
||||
_SQ_FULL_BACKOFF_INITIAL_S = 0.001
|
||||
_SQ_FULL_BACKOFF_MAX_S = 0.05
|
||||
_MAX_LOCAL_DECODE_TAIL_BLOCKS = 1
|
||||
_MAX_LOCAL_MAMBA_TAIL_BLOCKS = 1
|
||||
|
||||
|
||||
try:
|
||||
@@ -436,6 +439,26 @@ def _split_kv_cache_group_kinds(
|
||||
return attn, mamba
|
||||
|
||||
|
||||
def _validate_hybrid_speculation(vllm_config: VllmConfig) -> None:
|
||||
speculative_config = vllm_config.speculative_config
|
||||
if speculative_config is None:
|
||||
return
|
||||
# DSpark is allowlisted because it is the only hybrid speculative method
|
||||
# validated end to end with MoRIIO READ.
|
||||
if not speculative_config.use_dspark():
|
||||
raise MoRIIOError(
|
||||
"MoRIIO hybrid READ supports DSpark speculative decoding only, got "
|
||||
f"method={speculative_config.method!r}"
|
||||
)
|
||||
if vllm_config.cache_config.mamba_cache_mode == "all":
|
||||
# Positional allocation counts lookahead, so its local tail can exceed
|
||||
# the single running-state block supported by hybrid READ.
|
||||
raise MoRIIOError(
|
||||
"MoRIIO hybrid READ does not support DSpark with "
|
||||
"mamba_cache_mode='all'; use 'align'"
|
||||
)
|
||||
|
||||
|
||||
def _validate_mamba_specs(specs: Collection[MambaSpec]) -> MambaSpec | None:
|
||||
"""Validate the common packed-state contract and return its representative."""
|
||||
specs = tuple(specs)
|
||||
@@ -477,22 +500,22 @@ class MoRIIOConnectorScheduler:
|
||||
kv_cache_config
|
||||
)
|
||||
self._has_mamba = bool(self._mamba_group_ids)
|
||||
mamba_spec = None
|
||||
if self._has_mamba:
|
||||
if len(self._attn_group_ids) != 1:
|
||||
raise MoRIIOError(
|
||||
"MoRIIO hybrid READ requires exactly one transferable "
|
||||
"attention group, got "
|
||||
f"{len(self._attn_group_ids)} attention groups"
|
||||
f"{len(self._attn_group_ids)}; a drafter that owns a "
|
||||
"separate attention group is not supported, give it the "
|
||||
"target's group or disable KV transfer for it"
|
||||
)
|
||||
mamba_specs = [
|
||||
cast(MambaSpec, kv_cache_config.transfer_groups[group_id].kv_cache_spec)
|
||||
for group_id in self._mamba_group_ids
|
||||
]
|
||||
_validate_mamba_specs(mamba_specs)
|
||||
if vllm_config.speculative_config is not None:
|
||||
raise MoRIIOError(
|
||||
"MoRIIO hybrid READ does not support speculative decoding"
|
||||
)
|
||||
mamba_spec = _validate_mamba_specs(mamba_specs)
|
||||
_validate_hybrid_speculation(vllm_config)
|
||||
if self.mode != MoRIIOMode.READ:
|
||||
raise MoRIIOError(
|
||||
"MoRIIO hybrid (mamba/KDA) transfer is implemented for READ "
|
||||
@@ -503,18 +526,31 @@ class MoRIIOConnectorScheduler:
|
||||
self._ssm_state_slots_are_positional = (
|
||||
vllm_config.cache_config.mamba_cache_mode == "all"
|
||||
)
|
||||
self.block_size = vllm_config.cache_config.block_size
|
||||
self._max_decode_tail_blocks = (
|
||||
_MAX_LOCAL_DECODE_TAIL_BLOCKS
|
||||
if self._has_mamba
|
||||
else math.ceil((vllm_config.num_lookahead_tokens + 1) / self.block_size)
|
||||
self._num_ssm_scratch_blocks = (
|
||||
mamba_spec.num_speculative_blocks if mamba_spec is not None else 0
|
||||
)
|
||||
self.block_size = vllm_config.cache_config.block_size
|
||||
attn_block_size = self.block_size
|
||||
max_decode_tail_tokens = 1 + vllm_config.num_lookahead_tokens
|
||||
if self._has_mamba:
|
||||
attn_spec = kv_cache_config.transfer_groups[
|
||||
self._attn_group_ids[0]
|
||||
].kv_cache_spec
|
||||
attn_block_size = attn_spec.block_size
|
||||
if attn_spec.dcp_sharded:
|
||||
attn_block_size *= (
|
||||
vllm_config.parallel_config.decode_context_parallel_size
|
||||
)
|
||||
# Admission can pad the final prompt token to 1 + num_spec rows;
|
||||
# allocate_slots reserves drafter lookahead in addition to those.
|
||||
max_decode_tail_tokens += vllm_config.num_speculative_tokens
|
||||
self._max_decode_tail_blocks = cdiv(max_decode_tail_tokens, attn_block_size)
|
||||
self.engine_id: EngineId = engine_id
|
||||
|
||||
self._is_hma_required = (
|
||||
not vllm_config.scheduler_config.disable_hybrid_kv_cache_manager
|
||||
and any(
|
||||
not isinstance(g.kv_cache_spec, FullAttentionSpec)
|
||||
not is_full_attention_spec(g.kv_cache_spec)
|
||||
for g in kv_cache_config.transfer_groups
|
||||
)
|
||||
)
|
||||
@@ -524,9 +560,9 @@ class MoRIIOConnectorScheduler:
|
||||
unsupported = [
|
||||
type(g.kv_cache_spec).__name__
|
||||
for g in kv_cache_config.transfer_groups
|
||||
if not isinstance(
|
||||
g.kv_cache_spec,
|
||||
(FullAttentionSpec, SlidingWindowSpec, MambaSpec),
|
||||
if not (
|
||||
is_full_attention_spec(g.kv_cache_spec)
|
||||
or isinstance(g.kv_cache_spec, (SlidingWindowSpec, MambaSpec))
|
||||
)
|
||||
]
|
||||
if unsupported:
|
||||
@@ -565,7 +601,7 @@ class MoRIIOConnectorScheduler:
|
||||
self._full_attn_group_idx = 0
|
||||
self._full_attn_block_size = self.block_size
|
||||
for gi, group in enumerate(kv_cache_config.kv_cache_groups):
|
||||
if isinstance(group.kv_cache_spec, FullAttentionSpec):
|
||||
if is_full_attention_spec(group.kv_cache_spec):
|
||||
self._full_attn_group_idx = gi
|
||||
self._full_attn_block_size = getattr(
|
||||
group.kv_cache_spec, "block_size", self.block_size
|
||||
@@ -953,6 +989,9 @@ class MoRIIOConnectorScheduler:
|
||||
)
|
||||
aligned_local_mamba: list[list[int]] = []
|
||||
aligned_remote_mamba: list[list[int]] = []
|
||||
# Scratch slots were removed by
|
||||
# split_block_groups, so only one local running
|
||||
# state tail can remain.
|
||||
for local_group, remote_group in zip(
|
||||
mamba_block_groups,
|
||||
remote_mamba_groups,
|
||||
@@ -960,7 +999,9 @@ class MoRIIOConnectorScheduler:
|
||||
):
|
||||
aligned_local, aligned_remote = (
|
||||
self._align_read_blocks(
|
||||
local_group, remote_group, 1
|
||||
local_group,
|
||||
remote_group,
|
||||
_MAX_LOCAL_MAMBA_TAIL_BLOCKS,
|
||||
)
|
||||
)
|
||||
aligned_local_mamba.append(aligned_local)
|
||||
@@ -1257,21 +1298,15 @@ class MoRIIOConnectorScheduler:
|
||||
transfer_block_ids = self.kv_cache_config.select_transfer_block_ids(block_ids)
|
||||
attn = list(transfer_block_ids[self._attn_group_ids[0]])
|
||||
mamba_groups = [
|
||||
self._clip_mamba_group(list(transfer_block_ids[group_id]))
|
||||
clip_ssm_state_blocks(
|
||||
list(transfer_block_ids[group_id]),
|
||||
self._num_ssm_scratch_blocks,
|
||||
self._ssm_state_slots_are_positional,
|
||||
)
|
||||
for group_id in self._mamba_group_ids
|
||||
]
|
||||
return attn, mamba_groups
|
||||
|
||||
def _clip_mamba_group(self, blocks: list[int]) -> list[int]:
|
||||
"""Keep only the state-bearing slots of one mamba kv cache group."""
|
||||
if not blocks:
|
||||
return blocks
|
||||
if not self._ssm_state_slots_are_positional:
|
||||
# Single running state: everything before it is a null placeholder
|
||||
# or the previous step's superseded state.
|
||||
blocks = blocks[-1:]
|
||||
return blocks
|
||||
|
||||
@staticmethod
|
||||
def _align_read_blocks(
|
||||
local_block_ids: list[int],
|
||||
@@ -1282,9 +1317,9 @@ class MoRIIOConnectorScheduler:
|
||||
|
||||
Prefix hits can make the decode's unhashed suffix shorter than the
|
||||
prefill's full list, so read from the matching remote suffix. The
|
||||
decoder can instead have one extra tail block for the locally
|
||||
recomputed final token; that block has no remote source and is not
|
||||
transferred.
|
||||
decoder can instead have extra tail blocks for the locally recomputed
|
||||
final token and speculative slots; these have no remote source and are
|
||||
not transferred.
|
||||
"""
|
||||
if len(local_block_ids) > len(remote_block_ids):
|
||||
extra_decode_blocks = len(local_block_ids) - len(remote_block_ids)
|
||||
|
||||
@@ -13,6 +13,7 @@ from vllm import envs
|
||||
from vllm.distributed.kv_transfer.kv_connector.utils import (
|
||||
BlockIds,
|
||||
EngineId,
|
||||
clip_ssm_state_blocks,
|
||||
yield_req_data,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
@@ -288,11 +289,9 @@ class NixlBaseConnectorScheduler:
|
||||
and blocks
|
||||
and (n_spec_blocks := self._ssm_spec_blocks[i]) is not None
|
||||
):
|
||||
if n_spec := min(n_spec_blocks, len(blocks) - 1):
|
||||
blocks = blocks[:-n_spec]
|
||||
if not self._ssm_state_slots_are_positional:
|
||||
# Never empty: downstream reads that as a full prefix hit.
|
||||
blocks = blocks[-1:]
|
||||
blocks = clip_ssm_state_blocks(
|
||||
blocks, n_spec_blocks, self._ssm_state_slots_are_positional
|
||||
)
|
||||
clipped.append(blocks)
|
||||
return tuple(clipped)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user