[ROCm][Connector] SWA+HMA-support in MoRI-IO connector (Gemma4) (#53721)

Signed-off-by: simondanielsson <simon.danielsson99@hotmail.com>
This commit is contained in:
Simon Danielsson
2026-09-14 20:52:56 +08:00
committed by GitHub
parent 3f55ad2f07
commit 1d0d1081c4
4 changed files with 408 additions and 46 deletions
@@ -39,10 +39,12 @@ from vllm.utils.network_utils import (
make_zmq_path,
)
from vllm.v1.kv_cache_interface import (
ChunkedLocalAttentionSpec,
FullAttentionSpec,
KVCacheConfig,
KVCacheGroupSpec,
KVCacheTensor,
SlidingWindowSpec,
compute_layer_kv_cache_shape_bytes,
)
@@ -338,7 +340,7 @@ def test_write_mode_saves_local_block_ids():
req_meta = kv_connector_metadata.reqs_to_save[request_id]
for block_id, block in zip(
req_meta.local_block_ids,
req_meta.local_block_ids[0],
scheduler.kv_cache_manager.coordinator.single_type_managers[0].req_to_blocks[
request_id
],
@@ -393,7 +395,7 @@ def test_write_mode_with_chunked_prefill_saves_local_block_ids():
req_meta = kv_connector_metadata.reqs_to_save[request_id]
for block_id, block in zip(
req_meta.local_block_ids,
req_meta.local_block_ids[0],
scheduler.kv_cache_manager.coordinator.single_type_managers[0].req_to_blocks[
request_id
],
@@ -432,7 +434,7 @@ def test_read_mode_loads_remote_block_ids():
].req_to_blocks[request_id]
# Set remote block ids to be fetched.
request.kv_transfer_params["remote_block_ids"] = block_list
request.kv_transfer_params["remote_block_ids"] = [block_list]
# Remote Prefill, triggers MoRIIOConnectorMetadata.
@@ -457,7 +459,7 @@ def test_read_mode_loads_remote_block_ids():
req_meta = kv_connector_metadata.reqs_to_recv[request_id]
for block_id, block in zip(
req_meta.local_block_ids,
req_meta.local_block_ids[0],
scheduler.kv_cache_manager.coordinator.single_type_managers[0].req_to_blocks[
request_id
],
@@ -699,3 +701,225 @@ def test_resolve_host_ip_prefers_extra_config():
fallback = get_ip()
assert resolve_host_ip({}) == fallback
assert resolve_host_ip({"host_ip": ""}) == fallback
def _make_hybrid_kv_cache_config() -> KVCacheConfig:
"""One full-attention group + one sliding-window group (Gemma-like)."""
full_spec = FullAttentionSpec(
block_size=16, num_kv_heads=4, head_size=64, dtype=torch.float16
)
sw_spec = SlidingWindowSpec(
block_size=16,
num_kv_heads=4,
head_size=64,
dtype=torch.float16,
sliding_window=32,
)
num_blocks = 2
page = full_spec.page_size_bytes
return KVCacheConfig(
num_blocks=num_blocks,
kv_cache_tensors=[
KVCacheTensor(
size=2 * num_blocks * page,
layers=["full0", "sw0"],
layer_stride=num_blocks * page,
block_stride=page,
)
],
kv_cache_groups=[
KVCacheGroupSpec(layer_names=["full0"], kv_cache_spec=full_spec),
KVCacheGroupSpec(layer_names=["sw0"], kv_cache_spec=sw_spec),
],
)
def _read_scheduler(
kv_cache_config: KVCacheConfig, disable_hma: bool = False
) -> MoRIIOConnectorScheduler:
vllm_config = create_vllm_config(role="kv_producer", read_mode=True)
vllm_config.scheduler_config.disable_hybrid_kv_cache_manager = disable_hma
with set_current_vllm_config(vllm_config):
connector = MoRIIOConnector(
vllm_config, KVConnectorRole.SCHEDULER, kv_cache_config
)
assert connector.connector_scheduler is not None
return connector.connector_scheduler
def test_hma_blocks_per_sw_two_groups():
"""A Full + SlidingWindow config runs with HMA and computes block budgets
correctly"""
scheduler = _read_scheduler(_make_hybrid_kv_cache_config())
assert scheduler._is_hma_required is True
# cdiv(32, 16) + 1 == 3 for the sliding-window group, 0 for full attention.
assert scheduler.blocks_per_sw == [0, 3]
@pytest.mark.parametrize(
"swa_enabled, disable_hma, expected_is_hma",
[
(True, False, True), # sliding-window group present, HMA enabled
(True, True, False), # sliding-window group present but HMA disabled
(False, False, False), # full-attention only, HMA not needed
],
)
def test_is_hma_required(swa_enabled, disable_hma, expected_is_hma):
"""_is_hma_required tracks both the KV cache groups and the
--disable-hybrid-kv-cache-manager flag. When HMA is off,
get_exchange_clipped_blocks must be a no-op."""
config = (
_make_hybrid_kv_cache_config() if swa_enabled else _make_test_kv_cache_config()
)
scheduler = _read_scheduler(config, disable_hma=disable_hma)
assert scheduler._is_hma_required is expected_is_hma
if not expected_is_hma:
blocks = [[1, 2, 3, 4, 5]]
assert scheduler.get_exchange_clipped_blocks(blocks) == blocks
def test_non_sliding_window_hybrid_is_rejected():
"""A hybrid group that is not sliding-window (e.g. chunked-local
attention) must fail closed rather than be silently mistransferred."""
full_spec = FullAttentionSpec(
block_size=16, num_kv_heads=4, head_size=64, dtype=torch.float16
)
local_spec = ChunkedLocalAttentionSpec(
block_size=16,
num_kv_heads=4,
head_size=64,
dtype=torch.float16,
attention_chunk_size=32,
)
num_blocks = 2
page = full_spec.page_size_bytes
config = KVCacheConfig(
num_blocks=num_blocks,
kv_cache_tensors=[
KVCacheTensor(
size=2 * num_blocks * page,
layers=["full0", "local0"],
layer_stride=num_blocks * page,
block_stride=page,
)
],
kv_cache_groups=[
KVCacheGroupSpec(layer_names=["full0"], kv_cache_spec=full_spec),
KVCacheGroupSpec(layer_names=["local0"], kv_cache_spec=local_spec),
],
)
with pytest.raises(NotImplementedError, match="sliding-window hybrid"):
_read_scheduler(config)
def test_token_count_basis_uses_full_attention_group():
"""Chunked-prefill token counting must use an unclipped full-attention
group (blocks_per_sw == 0), not a clipped sliding-window group."""
scheduler = _read_scheduler(_make_hybrid_kv_cache_config())
# Group 0 is the full-attention group
assert scheduler._full_attn_group_idx == 0
assert scheduler._full_attn_block_size == 16
def test_get_exchange_clipped_blocks_clips_only_sw_group():
"""get_exchange_clipped_blocks keeps the full attn group intact and clips the
sliding-window group to its window tail."""
scheduler = _read_scheduler(_make_hybrid_kv_cache_config())
full = [10, 11, 12, 13, 14]
sw = [20, 21, 22, 23, 24]
clipped = scheduler.get_exchange_clipped_blocks([full, sw])
assert clipped[0] == full
assert clipped[1] == [22, 23, 24]
def test_metadata_hma_block_ids_preserved_per_group():
"""add_new_req stores per-group (BlockIds) block lists unchanged for both
read and write, so the hybrid group structure is retained."""
metadata = MoRIIOConnectorMetadata()
# Assume:
# - Full-attention group (6 blocks) +
# - sliding-window group already clipped to its window tail (3 blocks).
fa_blocks = [0, 1, 2, 3, 4, 5]
sw_blocks = [10, 11, 12]
local_block_ids = [fa_blocks, sw_blocks]
remote_block_ids = [[100, 101, 102, 103, 104, 105], [200, 201, 202]]
base_params = {
"remote_engine_id": "remote-engine",
"remote_host": "127.0.0.1",
"remote_handshake_port": 6301,
"remote_notify_port": 61005,
"remote_block_ids": remote_block_ids,
}
# Read mode: both local and remote ids stay per-group.
metadata.add_new_req(
request_id="recv-req",
local_block_ids=local_block_ids,
kv_transfer_params={**base_params, "transfer_id": "recv-req"},
)
recv_meta = metadata.reqs_to_recv["recv-req"]
assert recv_meta.local_block_ids == [fa_blocks, sw_blocks]
assert recv_meta.remote_block_ids == remote_block_ids
# Write mode: the decode peer allocates its own blocks, so #remote_block_ids may
# be empty, Local group structure is kept.
metadata.add_new_req(
request_id="save-req",
local_block_ids=local_block_ids,
kv_transfer_params={
**base_params,
"transfer_id": "save-req",
"remote_block_ids": [],
},
write_mode=True,
)
save_meta = metadata.reqs_to_save["save-req"]
assert save_meta.local_block_ids == [fa_blocks, sw_blocks]
assert save_meta.remote_block_ids == []
def test_single_group_path_unchanged():
"""A full attn config does not enable HMA and never clips blocks."""
scheduler = _read_scheduler(_make_test_kv_cache_config())
assert scheduler._is_hma_required is False
assert scheduler.blocks_per_sw == [0]
blocks = [[1, 2, 3, 4, 5]]
assert scheduler.get_exchange_clipped_blocks(blocks) == blocks
def test_hybrid_write_mode_rejected():
"""Hybrid KV cache groups are unsupported in WRITE mode and fail closed."""
vllm_config = create_vllm_config(role="kv_producer", read_mode=False)
with (
set_current_vllm_config(vllm_config),
pytest.raises(NotImplementedError),
):
MoRIIOConnector(
vllm_config,
KVConnectorRole.SCHEDULER,
_make_hybrid_kv_cache_config(),
)
def test_worker_layer_to_group_routing(mock_parallel_groups):
"""The worker maps every layer to its KV cache group correctly."""
vllm_config = create_vllm_config(role="kv_consumer", read_mode=True)
# Building the worker directly bypasses MoRIIOConnector._set_port_defaults,
# so provide the ports manually.
vllm_config.kv_transfer_config.kv_connector_extra_config.update(
{
"http_port": 12346,
"handshake_port": 12347,
"notify_port": 12348,
}
)
with set_current_vllm_config(vllm_config):
worker = FakeMoRIIOConnectorWorker(
vllm_config,
"engine0",
hand_shake_latency=0,
kv_cache_config=_make_hybrid_kv_cache_config(),
)
assert worker.layer_to_group == {"full0": 0, "sw0": 1}
@@ -20,12 +20,15 @@ from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector import
from vllm.v1.request import RequestStatus
_request_finished = MoRIIOConnectorScheduler.request_finished
_clip_blocks = MoRIIOConnectorScheduler.get_exchange_clipped_blocks
def _producer(global_dp_rank: int, dp_size: int, dp_size_local: int):
"""Minimal prefill-side (producer) scheduler stand-in for request_finished."""
return SimpleNamespace(
s = SimpleNamespace(
is_producer=True,
_is_hma_required=False,
blocks_per_sw=[0],
_global_dp_rank=global_dp_rank,
engine_id="engine-abc",
host_ip="10.0.0.1",
@@ -39,6 +42,8 @@ def _producer(global_dp_rank: int, dp_size: int, dp_size_local: int):
)
),
)
s.get_exchange_clipped_blocks = _clip_blocks.__get__(s)
return s
def _req(
@@ -14,6 +14,7 @@ import torch
import zmq
from vllm.config import KVTransferConfig, VllmConfig
from vllm.distributed.kv_transfer.kv_connector.utils import BlockIds
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorMetadata,
)
@@ -444,8 +445,8 @@ class ReqMeta:
"""Metadata for a single request."""
transfer_id: TransferId
local_block_ids: list[int]
remote_block_ids: list[int]
local_block_ids: BlockIds
remote_block_ids: BlockIds
remote_host: str
remote_port: int
remote_handshake_port: int
@@ -481,7 +482,7 @@ class MoRIIOConnectorMetadata(KVConnectorMetadata):
def add_new_req(
self,
request_id: ReqId,
local_block_ids: list[int],
local_block_ids: BlockIds,
kv_transfer_params: dict[str, Any],
write_mode=False,
):
@@ -18,10 +18,12 @@ import torch
import zmq
from vllm.config import CUDAGraphMode, VllmConfig
from vllm.distributed.kv_transfer.kv_connector.utils import BlockIds
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorBase_V1,
KVConnectorMetadata,
KVConnectorRole,
SupportsHMA,
)
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_common import (
ROLE,
@@ -67,19 +69,24 @@ from vllm.distributed.parallel_state import (
)
from vllm.forward_context import ForwardContext, get_forward_context
from vllm.logger import init_logger
from vllm.utils.math_utils import cdiv
from vllm.utils.network_utils import (
make_zmq_path,
make_zmq_socket,
)
from vllm.v1.attention.selector import get_attn_backend
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheConfig,
SlidingWindowSpec,
)
from vllm.v1.outputs import KVConnectorOutput
from vllm.v1.request import RequestStatus
if TYPE_CHECKING:
from vllm.v1.attention.backend import AttentionMetadata
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.request import Request
logger = init_logger(__name__)
@@ -189,7 +196,7 @@ def resolve_moriio_transfer_ack(
return transfer_id
class MoRIIOConnector(KVConnectorBase_V1):
class MoRIIOConnector(KVConnectorBase_V1, SupportsHMA):
def __init__(
self,
vllm_config: VllmConfig,
@@ -225,7 +232,7 @@ class MoRIIOConnector(KVConnectorBase_V1):
)
if role == KVConnectorRole.SCHEDULER:
self.connector_scheduler: MoRIIOConnectorScheduler | None = (
MoRIIOConnectorScheduler(vllm_config, self.engine_id)
MoRIIOConnectorScheduler(vllm_config, self.engine_id, kv_cache_config)
)
self.connector_worker: MoRIIOConnectorWorker | None = None
elif role == KVConnectorRole.WORKER:
@@ -283,6 +290,14 @@ class MoRIIOConnector(KVConnectorBase_V1):
self,
request: "Request",
block_ids: list[int],
) -> tuple[bool, dict[str, Any] | None]:
assert self.connector_scheduler is not None
return self.connector_scheduler.request_finished(request, (block_ids,))
def request_finished_all_groups(
self,
request: "Request",
block_ids: BlockIds,
) -> tuple[bool, dict[str, Any] | None]:
assert self.connector_scheduler is not None
return self.connector_scheduler.request_finished(request, block_ids)
@@ -368,7 +383,12 @@ class MoRIIOConnector(KVConnectorBase_V1):
class MoRIIOConnectorScheduler:
"""Implementation of Scheduler side methods"""
def __init__(self, vllm_config: VllmConfig, engine_id: str):
def __init__(
self,
vllm_config: VllmConfig,
engine_id: str,
kv_cache_config: "KVCacheConfig",
):
self.vllm_config = vllm_config
assert vllm_config.kv_transfer_config is not None, (
@@ -378,6 +398,66 @@ class MoRIIOConnectorScheduler:
self.block_size = vllm_config.cache_config.block_size
self.engine_id: EngineId = engine_id
self.mode = get_moriio_mode(self.kv_transfer_config)
self._is_hma_required = (
not vllm_config.scheduler_config.disable_hybrid_kv_cache_manager
and any(
not isinstance(g.kv_cache_spec, FullAttentionSpec)
for g in kv_cache_config.kv_cache_groups
)
)
if self._is_hma_required:
# TODO(simondanielsson): support non-sliding window hybrids
unsupported = [
type(g.kv_cache_spec).__name__
for g in kv_cache_config.kv_cache_groups
if not isinstance(
g.kv_cache_spec, (FullAttentionSpec, SlidingWindowSpec)
)
]
if unsupported:
raise NotImplementedError(
"MoRIIO only supports sliding-window hybrid models (e.g. "
"Gemma) under the hybrid KV cache manager. Unsupported KV "
f"cache group spec(s): {sorted(set(unsupported))}. Pass "
"--disable-hybrid-kv-cache-manager to run without HMA."
)
if self.mode == MoRIIOMode.WRITE:
# TODO(simondanielsson): support HMA in WRITE mode
raise NotImplementedError(
"MoRIIO WRITE mode does not support hybrid KV cache groups "
"(sliding-window attention). Use READ mode, or pass "
"--disable-hybrid-kv-cache-manager."
)
sw_sizes_tokens: list[tuple[int, int]] = [
(g.kv_cache_spec.sliding_window, g.kv_cache_spec.block_size)
if isinstance(g.kv_cache_spec, SlidingWindowSpec)
else (0, self.block_size)
for g in kv_cache_config.kv_cache_groups
]
# add 1 to conservatively account for boundary overlap eg window isn't fully
# aligned with blocks.
self.blocks_per_sw = [
cdiv(n_tokens, block_size) + 1 if n_tokens else 0
for n_tokens, block_size in sw_sizes_tokens
]
# In WRITE mode and chunked prefill, we perform the write after the last chunk.
# Hence we need to track once we are in the last chunk. We do this by computing
# len(block_ids[g]) * block_size[g] for a full attn group g, as hybrid groups
# may be clipped and hence not reflect the full sequence length. Here we store
# g and block_size[g] for later.
self._full_attn_group_idx = 0
self._full_attn_block_size = self.block_size
for gi, group in enumerate(kv_cache_config.kv_cache_groups):
is_full_attn = self.blocks_per_sw[gi] == 0
if is_full_attn:
self._full_attn_group_idx = gi
self._full_attn_block_size = getattr(
group.kv_cache_spec, "block_size", self.block_size
)
break
self.host_ip = resolve_host_ip(
self.kv_transfer_config.kv_connector_extra_config
)
@@ -409,14 +489,14 @@ class MoRIIOConnectorScheduler:
# Requests that need to start recv/send.
# New requests are added by update_state_after_alloc in
# the scheduler. Used to make metadata passed to Worker.
self._reqs_need_recv: dict[ReqId, tuple[Request, list[int]]] = {}
self._reqs_need_save: dict[ReqId, tuple[Request, list[int]]] = {}
self._reqs_need_recv: dict[ReqId, tuple[Request, BlockIds]] = {}
self._reqs_need_save: dict[ReqId, tuple[Request, BlockIds]] = {}
# Snapshot of kv_transfer_params for chunked prefill recovery.
self._req_kv_params: dict[ReqId, dict] = {}
# For chunked prefill, we perform layer-wise access within the final chunk.
# TODO: Perform transfer at end chunk.
self._reqs_need_pending_save: dict[ReqId, tuple[Request, list[int]]] = {}
self._reqs_need_pending_save: dict[ReqId, tuple[Request, BlockIds]] = {}
if self.is_producer:
set_role(ROLE.PRODUCER)
@@ -442,6 +522,23 @@ class MoRIIOConnectorScheduler:
self.transfer_id_to_request_id: dict[TransferId, ReqId] = {}
self.request_id_to_transfer_id: dict[ReqId, TransferId] = {}
def get_exchange_clipped_blocks(
self,
block_ids: BlockIds,
) -> BlockIds:
"""Clip the block list based on the layer type.
Behavior:
- sliding window: keep only the last window size blocks
- full attention: keep all blocks
"""
if len(block_ids) == 0 or not self._is_hma_required:
return [list(blocks) for blocks in block_ids]
return [
blocks[-self.blocks_per_sw[i] :] if self.blocks_per_sw[i] > 0 else blocks
for i, blocks in enumerate(block_ids)
]
def map_request_id(self, request_id: ReqId, transfer_id: TransferId):
self.transfer_id_to_request_id[transfer_id] = request_id
self.request_id_to_transfer_id[request_id] = transfer_id
@@ -642,7 +739,7 @@ class MoRIIOConnectorScheduler:
request_id = request.request_id
self.map_request_id(request_id, transfer_id)
if params.get("do_remote_decode"):
local_block_ids = blocks.get_block_ids()[0]
local_block_ids: BlockIds = blocks.get_block_ids()
self._reqs_need_save[request.request_id] = (request, local_block_ids)
# Snapshot params now so chunked-prefill build_connector_meta
# can recover them on the final chunk even if the live
@@ -657,17 +754,22 @@ class MoRIIOConnectorScheduler:
if "remote_engine_id" in params:
if num_external_tokens > 0:
# Get unhashed blocks to pull from remote.
local_block_ids = blocks.get_block_ids()[0]
assert len(local_block_ids) <= len(remote_block_ids)
if len(local_block_ids) != len(remote_block_ids):
local_block_ids = remote_block_ids[
-len(local_block_ids) :
]
local_block_ids = self.get_exchange_clipped_blocks(
blocks.get_block_ids()
)
# Partial cache hits can lead to fewer local blocks vs
# remote blocks, but never more
assert all(
len(local) <= len(remote)
for local, remote in zip(
local_block_ids, remote_block_ids, strict=True
)
)
else:
# If remote_blocks and num_external_tokens = 0, we have
# a full prefix cache hit on the D worker. We need to call
# send_notify in _read_blocks to free the memory on the P.
local_block_ids = []
local_block_ids = [[] for _ in remote_block_ids]
self._reqs_need_recv[request.request_id] = (
request,
@@ -818,8 +920,6 @@ class MoRIIOConnectorScheduler:
new_block_ids = scheduler_output.scheduled_cached_reqs.new_block_ids[i]
if new_block_ids is not None:
block_ids = new_block_ids[0]
# TODO : hybrid attn, etc
# A non-disagg request (no kv_transfer_params, e.g. smoke
# test) is never registered in _reqs_need_pending_save;
# indexing it unconditionally would KeyError and crash the
@@ -827,12 +927,16 @@ class MoRIIOConnectorScheduler:
if req_id not in self._reqs_need_pending_save:
continue
req, existing_blocks = self._reqs_need_pending_save[req_id]
updated_blocks = list(existing_blocks) + (block_ids)
updated_blocks = [
existing_blocks[g] + new_block_ids[g]
for g in range(len(new_block_ids))
]
self._reqs_need_pending_save[req_id] = (req, updated_blocks)
if (
len(self._reqs_need_pending_save[req_id][1]) * self.block_size
>= req.num_prompt_tokens
):
saved_tokens = (
len(updated_blocks[self._full_attn_group_idx])
* self._full_attn_block_size
)
if saved_tokens >= req.num_prompt_tokens:
# Final chunk: live kv_transfer_params may be cleared,
# so prefer the snapshot from update_state_after_alloc.
kv_params = self._req_kv_params.pop(
@@ -857,7 +961,10 @@ class MoRIIOConnectorScheduler:
for req_id, (req, block_ids) in self._reqs_need_save.items():
kv_params = self._req_kv_params.get(req_id, req.kv_transfer_params or {})
if req.num_prompt_tokens > len(block_ids) * self.block_size:
tokens_covered = (
len(block_ids[self._full_attn_group_idx]) * self._full_attn_block_size
)
if req.num_prompt_tokens > tokens_covered:
# not last chunk prefill
self._reqs_need_pending_save[req_id] = (req, block_ids)
continue
@@ -898,7 +1005,7 @@ class MoRIIOConnectorScheduler:
def request_finished(
self,
request: "Request",
block_ids: list[int],
block_ids: BlockIds,
) -> tuple[bool, dict[str, Any] | None]:
"""
Once a request is finished, determine whether request blocks
@@ -936,7 +1043,8 @@ class MoRIIOConnectorScheduler:
if self.mode == MoRIIOMode.WRITE:
self._release_write_prefill_blocks(request.request_id, params)
else:
self._reqs_need_recv[request.request_id] = (request, [])
empty_groups: BlockIds = [[] for _ in self.blocks_per_sw]
self._reqs_need_recv[request.request_id] = (request, empty_groups)
params["do_remote_prefill"] = False
return False, None
@@ -946,10 +1054,9 @@ class MoRIIOConnectorScheduler:
):
return False, None
# computed_block_ids = block_ids if all_full else block_ids[:-1]
computed_block_ids = block_ids
# If prompt < block_size, no xfer so free blocks immediately.
delay_free_blocks = len(computed_block_ids) > 0
# On producer
computed_block_ids = self.get_exchange_clipped_blocks(block_ids)
delay_free_blocks = any(len(group) > 0 for group in computed_block_ids)
if delay_free_blocks:
# Prefill request on remote. It will be read from D upon completion
@@ -1102,6 +1209,12 @@ class MoRIIOConnectorWorker:
self.kv_transfer_config = vllm_config.kv_transfer_config
self.is_producer = self.kv_transfer_config.is_kv_producer
self.layer_to_spec = build_layer_to_spec(kv_cache_config)
# we transfer by layer, but store by kv cache group
self.layer_to_group: dict[str, int] = {
layer_name: group_idx
for group_idx, group in enumerate(kv_cache_config.kv_cache_groups)
for layer_name in group.layer_names
}
if self.is_producer:
set_role(ROLE.PRODUCER)
@@ -1789,8 +1902,6 @@ class MoRIIOConnectorWorker:
self.block_size,
)
self.block_size = first_geometry.block_size
# TODO(tms): self.block_len needs to be per-layer for sliding window,
# hybrid attn, etc
# block size in bytes
self.block_len = first_geometry.block_len
self.kv_cache_shape = first_kv_cache.shape
@@ -1803,14 +1914,30 @@ class MoRIIOConnectorWorker:
self.layer_base_addr_index = {}
base_addr_idx = 0
hma_enabled = (
not self.vllm_config.scheduler_config.disable_hybrid_kv_cache_manager
)
for layer_name in kv_caches:
self.layer_base_addr_index[layer_name] = base_addr_idx
geometry = self._get_layer_transfer_geometry(layer_name)
if geometry.block_size != self.block_size:
if hma_enabled:
if geometry.block_len != self.block_len:
raise ValueError(
"MoRIIO KV cache block length mismatch for layer "
f"{layer_name}: {geometry.block_len} != {self.block_len}"
)
elif geometry.block_size != self.block_size:
raise ValueError(
"MoRIIO KV cache block size mismatch for layer "
f"{layer_name}: {geometry.block_size} != {self.block_size}"
)
# num_blocks is advertised as a single scalar to the peer, so must it
# be uniform
if geometry.num_blocks != self.num_blocks:
raise ValueError(
"MoRIIO KV cache num_blocks mismatch for layer "
f"{layer_name}: {geometry.num_blocks} != {self.num_blocks}"
)
self.block_lens[layer_name] = geometry.block_len
for cache, region_len in self._iter_layer_registration_regions(layer_name):
base_addr = cache.data_ptr()
@@ -2455,12 +2582,16 @@ class MoRIIOConnectorWorker:
self.remote_dp_size_local = int(meta.remote_dp_size_local)
else:
self.remote_dp_size_local = int(meta.remote_dp_size)
# WRITE does not support HMA, so unwrap blocks into flat lists.
# remote_block_ids can itself be empty so need to be careful when unwrapping.
local_block_ids = meta.local_block_ids[0]
remote_block_ids = meta.remote_block_ids[0] if meta.remote_block_ids else []
self.schedule_write_blocks(
request_id=req_id,
transfer_id=meta.transfer_id,
dst_engine_id=meta.remote_engine_id,
local_block_ids=meta.local_block_ids,
remote_block_ids=meta.remote_block_ids,
local_block_ids=local_block_ids,
remote_block_ids=remote_block_ids,
layer_name=layer_name,
kv_layer=kv_layer,
remote_notify_port=meta.remote_notify_port,
@@ -2607,8 +2738,8 @@ class MoRIIOConnectorWorker:
def _read_blocks(
self,
local_block_ids: list[int],
remote_block_ids: list[int],
local_block_ids: BlockIds,
remote_block_ids: BlockIds,
dst_engine_id: str,
request_id: str,
transfer_id: str,
@@ -2653,10 +2784,11 @@ class MoRIIOConnectorWorker:
sess_idx = list(self.layer_name_to_local_kv_cache_metadata.keys()).index(
layer_name
)
group_idx = self.layer_to_group[layer_name]
offs = self._compute_block_transfer_offsets(
layer_name,
local_block_ids,
remote_block_ids,
local_block_ids[group_idx],
remote_block_ids[group_idx],
remote_moriio_meta,
remote_tp_size=remote_tp_size,
remote_engine_id=remote_dp_engine_id,