[KVConnector][MoRIIO] Support K3 DSpark hybrid READ (#57700)
Signed-off-by: Yichao Zhu <Yichao.Zhu@amd.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
35cd03ebff
commit
30ae8248b6
@@ -21,6 +21,9 @@ import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.core.block_pool import BlockPool
|
||||
from vllm.v1.core.single_type_kv_cache_manager import FullAttentionManager
|
||||
from vllm.v1.kv_cache_interface import FullAttentionSpec, UniformTypeKVCacheSpecs
|
||||
|
||||
mori_available = importlib.util.find_spec("mori") is not None
|
||||
|
||||
@@ -44,6 +47,16 @@ MoRIIOMode = moriio_connector.MoRIIOMode
|
||||
moriio_common = importlib.import_module(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_common"
|
||||
)
|
||||
get_role = moriio_common.get_role
|
||||
set_role = moriio_common.set_role
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def restore_moriio_role():
|
||||
"""Undo the process-global role a full scheduler construction leaves behind."""
|
||||
role = get_role()
|
||||
yield
|
||||
set_role(role)
|
||||
|
||||
|
||||
class _FakeScheduler(moriio_connector.MoRIIOConnectorScheduler): # type: ignore[name-defined]
|
||||
@@ -53,6 +66,7 @@ class _FakeScheduler(moriio_connector.MoRIIOConnectorScheduler): # type: ignore
|
||||
def __init__(self, **attrs):
|
||||
self._mamba_group_ids: list[int] = []
|
||||
self._attn_group_ids: list[int] = [0]
|
||||
self._num_ssm_scratch_blocks = 0
|
||||
self._ssm_state_slots_are_positional = False
|
||||
self._is_hma_required = False
|
||||
self.kv_cache_config = SimpleNamespace(
|
||||
@@ -110,22 +124,56 @@ def _gdn_split_info(conv_rows=3, key_dim=4, value_dim=8, dtype_size=2):
|
||||
)
|
||||
|
||||
|
||||
def _mamba_spec(num_states: int = 2):
|
||||
def _mamba_spec(num_states: int = 2, *, num_speculative_blocks: int = 0):
|
||||
shapes = tuple((1, 1) for _ in range(num_states))
|
||||
return moriio_connector.MambaSpec(
|
||||
block_size=16,
|
||||
shapes=shapes,
|
||||
dtypes=(torch.float32,) * num_states,
|
||||
mamba_cache_mode="all",
|
||||
num_speculative_blocks=num_speculative_blocks,
|
||||
)
|
||||
|
||||
|
||||
def _gate_vllm_config(*, read_mode: bool = True, speculative_config=None):
|
||||
def _spec_config(method: str):
|
||||
return SimpleNamespace(method=method, use_dspark=lambda: method == "dspark")
|
||||
|
||||
|
||||
def _gate_vllm_config(
|
||||
*,
|
||||
read_mode: bool = True,
|
||||
speculative_config=None,
|
||||
block_size: int = 16,
|
||||
num_lookahead_tokens: int = 0,
|
||||
num_speculative_tokens: int = 0,
|
||||
mamba_cache_mode: str = "all",
|
||||
dcp_size: int = 1,
|
||||
):
|
||||
return SimpleNamespace(
|
||||
kv_transfer_config=SimpleNamespace(
|
||||
kv_connector_extra_config={"read_mode": read_mode}
|
||||
kv_connector_extra_config={
|
||||
"read_mode": read_mode,
|
||||
"host_ip": "127.0.0.1",
|
||||
"handshake_port": 6001,
|
||||
"notify_port": 6002,
|
||||
},
|
||||
kv_role="kv_consumer",
|
||||
),
|
||||
speculative_config=speculative_config,
|
||||
cache_config=SimpleNamespace(
|
||||
block_size=block_size,
|
||||
mamba_cache_mode=mamba_cache_mode,
|
||||
),
|
||||
scheduler_config=SimpleNamespace(disable_hybrid_kv_cache_manager=False),
|
||||
num_lookahead_tokens=num_lookahead_tokens,
|
||||
num_speculative_tokens=num_speculative_tokens,
|
||||
parallel_config=SimpleNamespace(
|
||||
tensor_parallel_size=1,
|
||||
decode_context_parallel_size=dcp_size,
|
||||
data_parallel_rank=0,
|
||||
data_parallel_size_local=1,
|
||||
data_parallel_size=1,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@@ -196,6 +244,7 @@ def test_exchange_blocks_ignore_transfer_disabled_group():
|
||||
|
||||
|
||||
def test_scheduler_rejects_multiple_attention_groups_with_mamba():
|
||||
"""A drafter owning its own transfer group is refused, with a hint."""
|
||||
config = SimpleNamespace(
|
||||
kv_cache_groups=[
|
||||
SimpleNamespace(enable_kv_transfer=True, kv_cache_spec=object()),
|
||||
@@ -205,7 +254,7 @@ def test_scheduler_rejects_multiple_attention_groups_with_mamba():
|
||||
)
|
||||
config.transfer_groups = tuple(config.kv_cache_groups)
|
||||
|
||||
with pytest.raises(moriio_common.MoRIIOError, match="exactly one transferable"):
|
||||
with pytest.raises(moriio_common.MoRIIOError, match="separate attention group"):
|
||||
moriio_connector.MoRIIOConnectorScheduler(_gate_vllm_config(), "engine", config)
|
||||
|
||||
|
||||
@@ -251,7 +300,7 @@ def test_scheduler_rejects_mamba_group_without_two_states():
|
||||
moriio_connector.MoRIIOConnectorScheduler(_gate_vllm_config(), "engine", config)
|
||||
|
||||
|
||||
def test_scheduler_rejects_speculative_hybrid_read():
|
||||
def test_scheduler_rejects_non_dspark_speculative_hybrid_read():
|
||||
config = SimpleNamespace(
|
||||
kv_cache_groups=[
|
||||
SimpleNamespace(enable_kv_transfer=True, kv_cache_spec=object()),
|
||||
@@ -262,7 +311,105 @@ def test_scheduler_rejects_speculative_hybrid_read():
|
||||
|
||||
with pytest.raises(moriio_common.MoRIIOError, match="speculative decoding"):
|
||||
moriio_connector.MoRIIOConnectorScheduler(
|
||||
_gate_vllm_config(speculative_config=object()), "engine", config
|
||||
_gate_vllm_config(speculative_config=_spec_config("ngram")),
|
||||
"engine",
|
||||
config,
|
||||
)
|
||||
|
||||
|
||||
def _dspark_wrapped_attention_config(*, dcp_sharded: bool = True):
|
||||
"""Mamba first, so a wrapped full-attention group at index 1 has to be found.
|
||||
|
||||
A bare ``isinstance(FullAttentionSpec)`` misses the wrapper and leaves
|
||||
``_full_attn_group_idx`` at its initial 0, which is the Mamba group.
|
||||
"""
|
||||
full = FullAttentionSpec(
|
||||
block_size=4,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
dcp_sharded=dcp_sharded,
|
||||
)
|
||||
config = SimpleNamespace(
|
||||
kv_cache_groups=[
|
||||
SimpleNamespace(
|
||||
enable_kv_transfer=True,
|
||||
kv_cache_spec=_mamba_spec(num_speculative_blocks=2),
|
||||
),
|
||||
SimpleNamespace(
|
||||
enable_kv_transfer=True,
|
||||
kv_cache_spec=UniformTypeKVCacheSpecs(
|
||||
block_size=4, kv_cache_specs={"target": full, "draft": full}
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
config.transfer_groups = tuple(config.kv_cache_groups)
|
||||
return config
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("restore_moriio_role")
|
||||
@pytest.mark.parametrize(
|
||||
"dcp_size,dcp_sharded,expected_tail", [(1, True, 4), (4, True, 1), (4, False, 4)]
|
||||
)
|
||||
def test_scheduler_accepts_dspark_with_wrapped_full_attention(
|
||||
dcp_size, dcp_sharded, expected_tail
|
||||
):
|
||||
config = _dspark_wrapped_attention_config(dcp_sharded=dcp_sharded)
|
||||
vllm_config = _gate_vllm_config(
|
||||
speculative_config=_spec_config("dspark"),
|
||||
block_size=4,
|
||||
num_lookahead_tokens=7,
|
||||
num_speculative_tokens=7,
|
||||
mamba_cache_mode="align",
|
||||
dcp_size=dcp_size,
|
||||
)
|
||||
scheduler = moriio_connector.MoRIIOConnectorScheduler(vllm_config, "engine", config)
|
||||
|
||||
assert scheduler._full_attn_group_idx == 1
|
||||
assert scheduler._num_ssm_scratch_blocks == 2
|
||||
assert scheduler._max_decode_tail_blocks == expected_tail
|
||||
|
||||
# A first decode step reserves padded target rows AND drafter lookahead.
|
||||
spec = config.transfer_groups[1].kv_cache_spec
|
||||
manager = FullAttentionManager(
|
||||
spec,
|
||||
block_pool=BlockPool(32, enable_caching=False, hash_block_size=4),
|
||||
enable_caching=False,
|
||||
kv_cache_group_id=1,
|
||||
scheduler_block_size=16,
|
||||
dcp_world_size=dcp_size,
|
||||
)
|
||||
imported_tokens = manager.block_size
|
||||
target_tokens = 1 + vllm_config.num_speculative_tokens
|
||||
manager.allocate_new_blocks(
|
||||
"decode",
|
||||
num_tokens=imported_tokens + target_tokens + vllm_config.num_lookahead_tokens,
|
||||
num_tokens_main_model=imported_tokens + target_tokens,
|
||||
)
|
||||
local = [b.block_id for b in manager.req_to_blocks["decode"]]
|
||||
remote = [100]
|
||||
assert scheduler._align_read_blocks(
|
||||
local, remote, scheduler._max_decode_tail_blocks
|
||||
) == (local[:1], remote)
|
||||
with pytest.raises(ValueError, match="tail"):
|
||||
scheduler._align_read_blocks(
|
||||
local + [99], remote, scheduler._max_decode_tail_blocks
|
||||
)
|
||||
|
||||
|
||||
def test_scheduler_rejects_dspark_with_positional_mamba_cache():
|
||||
"""Positional mode counts lookahead in the Mamba allocation, so the
|
||||
one-block recurrent-state tail bound no longer holds."""
|
||||
with pytest.raises(moriio_common.MoRIIOError, match="mamba_cache_mode"):
|
||||
moriio_connector.MoRIIOConnectorScheduler(
|
||||
_gate_vllm_config(
|
||||
speculative_config=_spec_config("dspark"),
|
||||
block_size=4,
|
||||
mamba_cache_mode="all",
|
||||
),
|
||||
"engine",
|
||||
_dspark_wrapped_attention_config(),
|
||||
)
|
||||
|
||||
|
||||
@@ -316,6 +463,30 @@ def test_split_block_groups_keeps_only_running_state_outside_all_mode():
|
||||
assert sched.split_block_groups(([1], [40, 41])) == ([1], [[41]])
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"positional,blocks,expected",
|
||||
[
|
||||
# Trailing scratch slots go in either cache mode.
|
||||
(True, [40, 41, 42], [40]),
|
||||
# Outside "all" the running state is picked only after they are gone.
|
||||
(False, [40, 41, 42, 43], [41]),
|
||||
# Never empty: a list shorter than the scratch count keeps one state.
|
||||
(True, [40], [40]),
|
||||
(False, [], []),
|
||||
],
|
||||
)
|
||||
def test_split_block_groups_strips_dspark_scratch_slots(positional, blocks, expected):
|
||||
sched = _FakeScheduler(
|
||||
_has_mamba=True,
|
||||
_attn_group_ids=[0],
|
||||
_mamba_group_ids=[1],
|
||||
_num_ssm_scratch_blocks=2,
|
||||
_ssm_state_slots_are_positional=positional,
|
||||
)
|
||||
|
||||
assert sched.split_block_groups(([1], blocks)) == ([1], [expected])
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# request_finished_all_groups
|
||||
# --------------------------------------------------------------------------
|
||||
@@ -434,6 +605,23 @@ def test_update_state_pairs_shorter_local_blocks_with_remote_suffix():
|
||||
assert sched._req_kv_params["req"]["remote_block_ids"] == [[12], [92]]
|
||||
|
||||
|
||||
def test_update_state_pairs_trimmed_mamba_state_with_remote_state():
|
||||
"""The DSpark shape end to end: scratch slots are stripped, the running
|
||||
state is then selected, and the attention list loses its lookahead tail."""
|
||||
sched = _make_read_scheduler()
|
||||
sched._ssm_state_slots_are_positional = False
|
||||
sched._num_ssm_scratch_blocks = 2
|
||||
sched._max_decode_tail_blocks = 2
|
||||
request = _make_read_request([[10, 11], [90]])
|
||||
# Mamba group: [null, running state, scratch, scratch].
|
||||
blocks = _FakeBlocks(all_groups=([100, 101, 102, 103], [200, 201, 202, 203]))
|
||||
|
||||
sched.update_state_after_alloc(request, blocks, num_external_tokens=256)
|
||||
|
||||
assert sched._reqs_need_recv["req"][1] == [[100, 101], [201]]
|
||||
assert sched._req_kv_params["req"]["remote_block_ids"] == [[10, 11], [90]]
|
||||
|
||||
|
||||
def test_update_state_full_attention_hit_still_carries_mamba_state():
|
||||
sched = _make_read_scheduler()
|
||||
request = _make_read_request([[10], [90]])
|
||||
|
||||
@@ -2055,6 +2055,15 @@ def test_exchange_clipped_blocks_ssm_single_state():
|
||||
# Scratch slots not allocated: the state slot still survives.
|
||||
assert sched.get_exchange_clipped_blocks(([1], [5]))[1] == [5]
|
||||
|
||||
assert sched.get_exchange_clipped_blocks(([1], [])) == ([1], [])
|
||||
# Per-step partial lists must not be interpreted as state + scratch slots.
|
||||
assert sched.get_exchange_clipped_blocks(([1], [7, 8]), clip_ssm=False) == (
|
||||
[1],
|
||||
[7, 8],
|
||||
)
|
||||
sched.blocks_per_sw = [2, 0]
|
||||
assert sched.get_exchange_clipped_blocks(([1, 2, 3], [7, 8, 9])) == ([2, 3], [7])
|
||||
|
||||
# Non-mamba models pass through unchanged.
|
||||
fa_sched = make_nixl_scheduler(has_mamba=False)
|
||||
assert fa_sched.get_exchange_clipped_blocks(([1, 2],)) == ([1, 2],)
|
||||
|
||||
@@ -33,6 +33,18 @@ EngineId = str
|
||||
BlockIds = tuple[list[int], ...] | list[list[int]]
|
||||
|
||||
|
||||
def clip_ssm_state_blocks(
|
||||
blocks: list[int], num_spec_blocks: int, positional: bool
|
||||
) -> list[int]:
|
||||
"""Drop speculative scratch slots, retaining position-indexed or running state."""
|
||||
if not blocks:
|
||||
return blocks
|
||||
# Keep at least the running-state slot, even before scratch allocation.
|
||||
if num_scratch := min(num_spec_blocks, len(blocks) - 1):
|
||||
blocks = blocks[:-num_scratch]
|
||||
return blocks if positional else blocks[-1:]
|
||||
|
||||
|
||||
def get_kv_connector_cache_layout(vllm_config: VllmConfig | None = None):
|
||||
# NOTE (NickLucche) When running disaggregated PD with NIXL, LBHNC layout is
|
||||
# used for faster transfer.
|
||||
|
||||
@@ -17,7 +17,10 @@ import torch
|
||||
import zmq
|
||||
|
||||
from vllm.config import CUDAGraphMode, VllmConfig
|
||||
from vllm.distributed.kv_transfer.kv_connector.utils import BlockIds
|
||||
from vllm.distributed.kv_transfer.kv_connector.utils import (
|
||||
BlockIds,
|
||||
clip_ssm_state_blocks,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorBase_V1,
|
||||
KVConnectorMetadata,
|
||||
@@ -88,10 +91,10 @@ from vllm.utils.network_utils import (
|
||||
from vllm.v1.attention.selector import get_attn_backend
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheConfig,
|
||||
MambaSpec,
|
||||
SlidingWindowSpec,
|
||||
is_full_attention_spec,
|
||||
)
|
||||
from vllm.v1.outputs import KVConnectorOutput
|
||||
from vllm.v1.request import RequestStatus
|
||||
@@ -108,7 +111,7 @@ logger = init_logger(__name__)
|
||||
# wait only needs to be long enough for it to make progress.
|
||||
_SQ_FULL_BACKOFF_INITIAL_S = 0.001
|
||||
_SQ_FULL_BACKOFF_MAX_S = 0.05
|
||||
_MAX_LOCAL_DECODE_TAIL_BLOCKS = 1
|
||||
_MAX_LOCAL_MAMBA_TAIL_BLOCKS = 1
|
||||
|
||||
|
||||
try:
|
||||
@@ -436,6 +439,26 @@ def _split_kv_cache_group_kinds(
|
||||
return attn, mamba
|
||||
|
||||
|
||||
def _validate_hybrid_speculation(vllm_config: VllmConfig) -> None:
|
||||
speculative_config = vllm_config.speculative_config
|
||||
if speculative_config is None:
|
||||
return
|
||||
# DSpark is allowlisted because it is the only hybrid speculative method
|
||||
# validated end to end with MoRIIO READ.
|
||||
if not speculative_config.use_dspark():
|
||||
raise MoRIIOError(
|
||||
"MoRIIO hybrid READ supports DSpark speculative decoding only, got "
|
||||
f"method={speculative_config.method!r}"
|
||||
)
|
||||
if vllm_config.cache_config.mamba_cache_mode == "all":
|
||||
# Positional allocation counts lookahead, so its local tail can exceed
|
||||
# the single running-state block supported by hybrid READ.
|
||||
raise MoRIIOError(
|
||||
"MoRIIO hybrid READ does not support DSpark with "
|
||||
"mamba_cache_mode='all'; use 'align'"
|
||||
)
|
||||
|
||||
|
||||
def _validate_mamba_specs(specs: Collection[MambaSpec]) -> MambaSpec | None:
|
||||
"""Validate the common packed-state contract and return its representative."""
|
||||
specs = tuple(specs)
|
||||
@@ -477,22 +500,22 @@ class MoRIIOConnectorScheduler:
|
||||
kv_cache_config
|
||||
)
|
||||
self._has_mamba = bool(self._mamba_group_ids)
|
||||
mamba_spec = None
|
||||
if self._has_mamba:
|
||||
if len(self._attn_group_ids) != 1:
|
||||
raise MoRIIOError(
|
||||
"MoRIIO hybrid READ requires exactly one transferable "
|
||||
"attention group, got "
|
||||
f"{len(self._attn_group_ids)} attention groups"
|
||||
f"{len(self._attn_group_ids)}; a drafter that owns a "
|
||||
"separate attention group is not supported, give it the "
|
||||
"target's group or disable KV transfer for it"
|
||||
)
|
||||
mamba_specs = [
|
||||
cast(MambaSpec, kv_cache_config.transfer_groups[group_id].kv_cache_spec)
|
||||
for group_id in self._mamba_group_ids
|
||||
]
|
||||
_validate_mamba_specs(mamba_specs)
|
||||
if vllm_config.speculative_config is not None:
|
||||
raise MoRIIOError(
|
||||
"MoRIIO hybrid READ does not support speculative decoding"
|
||||
)
|
||||
mamba_spec = _validate_mamba_specs(mamba_specs)
|
||||
_validate_hybrid_speculation(vllm_config)
|
||||
if self.mode != MoRIIOMode.READ:
|
||||
raise MoRIIOError(
|
||||
"MoRIIO hybrid (mamba/KDA) transfer is implemented for READ "
|
||||
@@ -503,18 +526,31 @@ class MoRIIOConnectorScheduler:
|
||||
self._ssm_state_slots_are_positional = (
|
||||
vllm_config.cache_config.mamba_cache_mode == "all"
|
||||
)
|
||||
self.block_size = vllm_config.cache_config.block_size
|
||||
self._max_decode_tail_blocks = (
|
||||
_MAX_LOCAL_DECODE_TAIL_BLOCKS
|
||||
if self._has_mamba
|
||||
else math.ceil((vllm_config.num_lookahead_tokens + 1) / self.block_size)
|
||||
self._num_ssm_scratch_blocks = (
|
||||
mamba_spec.num_speculative_blocks if mamba_spec is not None else 0
|
||||
)
|
||||
self.block_size = vllm_config.cache_config.block_size
|
||||
attn_block_size = self.block_size
|
||||
max_decode_tail_tokens = 1 + vllm_config.num_lookahead_tokens
|
||||
if self._has_mamba:
|
||||
attn_spec = kv_cache_config.transfer_groups[
|
||||
self._attn_group_ids[0]
|
||||
].kv_cache_spec
|
||||
attn_block_size = attn_spec.block_size
|
||||
if attn_spec.dcp_sharded:
|
||||
attn_block_size *= (
|
||||
vllm_config.parallel_config.decode_context_parallel_size
|
||||
)
|
||||
# Admission can pad the final prompt token to 1 + num_spec rows;
|
||||
# allocate_slots reserves drafter lookahead in addition to those.
|
||||
max_decode_tail_tokens += vllm_config.num_speculative_tokens
|
||||
self._max_decode_tail_blocks = cdiv(max_decode_tail_tokens, attn_block_size)
|
||||
self.engine_id: EngineId = engine_id
|
||||
|
||||
self._is_hma_required = (
|
||||
not vllm_config.scheduler_config.disable_hybrid_kv_cache_manager
|
||||
and any(
|
||||
not isinstance(g.kv_cache_spec, FullAttentionSpec)
|
||||
not is_full_attention_spec(g.kv_cache_spec)
|
||||
for g in kv_cache_config.transfer_groups
|
||||
)
|
||||
)
|
||||
@@ -524,9 +560,9 @@ class MoRIIOConnectorScheduler:
|
||||
unsupported = [
|
||||
type(g.kv_cache_spec).__name__
|
||||
for g in kv_cache_config.transfer_groups
|
||||
if not isinstance(
|
||||
g.kv_cache_spec,
|
||||
(FullAttentionSpec, SlidingWindowSpec, MambaSpec),
|
||||
if not (
|
||||
is_full_attention_spec(g.kv_cache_spec)
|
||||
or isinstance(g.kv_cache_spec, (SlidingWindowSpec, MambaSpec))
|
||||
)
|
||||
]
|
||||
if unsupported:
|
||||
@@ -565,7 +601,7 @@ class MoRIIOConnectorScheduler:
|
||||
self._full_attn_group_idx = 0
|
||||
self._full_attn_block_size = self.block_size
|
||||
for gi, group in enumerate(kv_cache_config.kv_cache_groups):
|
||||
if isinstance(group.kv_cache_spec, FullAttentionSpec):
|
||||
if is_full_attention_spec(group.kv_cache_spec):
|
||||
self._full_attn_group_idx = gi
|
||||
self._full_attn_block_size = getattr(
|
||||
group.kv_cache_spec, "block_size", self.block_size
|
||||
@@ -953,6 +989,9 @@ class MoRIIOConnectorScheduler:
|
||||
)
|
||||
aligned_local_mamba: list[list[int]] = []
|
||||
aligned_remote_mamba: list[list[int]] = []
|
||||
# Scratch slots were removed by
|
||||
# split_block_groups, so only one local running
|
||||
# state tail can remain.
|
||||
for local_group, remote_group in zip(
|
||||
mamba_block_groups,
|
||||
remote_mamba_groups,
|
||||
@@ -960,7 +999,9 @@ class MoRIIOConnectorScheduler:
|
||||
):
|
||||
aligned_local, aligned_remote = (
|
||||
self._align_read_blocks(
|
||||
local_group, remote_group, 1
|
||||
local_group,
|
||||
remote_group,
|
||||
_MAX_LOCAL_MAMBA_TAIL_BLOCKS,
|
||||
)
|
||||
)
|
||||
aligned_local_mamba.append(aligned_local)
|
||||
@@ -1257,21 +1298,15 @@ class MoRIIOConnectorScheduler:
|
||||
transfer_block_ids = self.kv_cache_config.select_transfer_block_ids(block_ids)
|
||||
attn = list(transfer_block_ids[self._attn_group_ids[0]])
|
||||
mamba_groups = [
|
||||
self._clip_mamba_group(list(transfer_block_ids[group_id]))
|
||||
clip_ssm_state_blocks(
|
||||
list(transfer_block_ids[group_id]),
|
||||
self._num_ssm_scratch_blocks,
|
||||
self._ssm_state_slots_are_positional,
|
||||
)
|
||||
for group_id in self._mamba_group_ids
|
||||
]
|
||||
return attn, mamba_groups
|
||||
|
||||
def _clip_mamba_group(self, blocks: list[int]) -> list[int]:
|
||||
"""Keep only the state-bearing slots of one mamba kv cache group."""
|
||||
if not blocks:
|
||||
return blocks
|
||||
if not self._ssm_state_slots_are_positional:
|
||||
# Single running state: everything before it is a null placeholder
|
||||
# or the previous step's superseded state.
|
||||
blocks = blocks[-1:]
|
||||
return blocks
|
||||
|
||||
@staticmethod
|
||||
def _align_read_blocks(
|
||||
local_block_ids: list[int],
|
||||
@@ -1282,9 +1317,9 @@ class MoRIIOConnectorScheduler:
|
||||
|
||||
Prefix hits can make the decode's unhashed suffix shorter than the
|
||||
prefill's full list, so read from the matching remote suffix. The
|
||||
decoder can instead have one extra tail block for the locally
|
||||
recomputed final token; that block has no remote source and is not
|
||||
transferred.
|
||||
decoder can instead have extra tail blocks for the locally recomputed
|
||||
final token and speculative slots; these have no remote source and are
|
||||
not transferred.
|
||||
"""
|
||||
if len(local_block_ids) > len(remote_block_ids):
|
||||
extra_decode_blocks = len(local_block_ids) - len(remote_block_ids)
|
||||
|
||||
@@ -13,6 +13,7 @@ from vllm import envs
|
||||
from vllm.distributed.kv_transfer.kv_connector.utils import (
|
||||
BlockIds,
|
||||
EngineId,
|
||||
clip_ssm_state_blocks,
|
||||
yield_req_data,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
@@ -288,11 +289,9 @@ class NixlBaseConnectorScheduler:
|
||||
and blocks
|
||||
and (n_spec_blocks := self._ssm_spec_blocks[i]) is not None
|
||||
):
|
||||
if n_spec := min(n_spec_blocks, len(blocks) - 1):
|
||||
blocks = blocks[:-n_spec]
|
||||
if not self._ssm_state_slots_are_positional:
|
||||
# Never empty: downstream reads that as a full prefix hit.
|
||||
blocks = blocks[-1:]
|
||||
blocks = clip_ssm_state_blocks(
|
||||
blocks, n_spec_blocks, self._ssm_state_slots_are_positional
|
||||
)
|
||||
clipped.append(blocks)
|
||||
return tuple(clipped)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user