diff --git a/tests/v1/kv_connector/unit/test_moriio_connector.py b/tests/v1/kv_connector/unit/test_moriio_connector.py index d169779c5d..836f9407bc 100644 --- a/tests/v1/kv_connector/unit/test_moriio_connector.py +++ b/tests/v1/kv_connector/unit/test_moriio_connector.py @@ -39,10 +39,12 @@ from vllm.utils.network_utils import ( make_zmq_path, ) from vllm.v1.kv_cache_interface import ( + ChunkedLocalAttentionSpec, FullAttentionSpec, KVCacheConfig, KVCacheGroupSpec, KVCacheTensor, + SlidingWindowSpec, compute_layer_kv_cache_shape_bytes, ) @@ -338,7 +340,7 @@ def test_write_mode_saves_local_block_ids(): req_meta = kv_connector_metadata.reqs_to_save[request_id] for block_id, block in zip( - req_meta.local_block_ids, + req_meta.local_block_ids[0], scheduler.kv_cache_manager.coordinator.single_type_managers[0].req_to_blocks[ request_id ], @@ -393,7 +395,7 @@ def test_write_mode_with_chunked_prefill_saves_local_block_ids(): req_meta = kv_connector_metadata.reqs_to_save[request_id] for block_id, block in zip( - req_meta.local_block_ids, + req_meta.local_block_ids[0], scheduler.kv_cache_manager.coordinator.single_type_managers[0].req_to_blocks[ request_id ], @@ -432,7 +434,7 @@ def test_read_mode_loads_remote_block_ids(): ].req_to_blocks[request_id] # Set remote block ids to be fetched. - request.kv_transfer_params["remote_block_ids"] = block_list + request.kv_transfer_params["remote_block_ids"] = [block_list] # Remote Prefill, triggers MoRIIOConnectorMetadata. @@ -457,7 +459,7 @@ def test_read_mode_loads_remote_block_ids(): req_meta = kv_connector_metadata.reqs_to_recv[request_id] for block_id, block in zip( - req_meta.local_block_ids, + req_meta.local_block_ids[0], scheduler.kv_cache_manager.coordinator.single_type_managers[0].req_to_blocks[ request_id ], @@ -699,3 +701,225 @@ def test_resolve_host_ip_prefers_extra_config(): fallback = get_ip() assert resolve_host_ip({}) == fallback assert resolve_host_ip({"host_ip": ""}) == fallback + + +def _make_hybrid_kv_cache_config() -> KVCacheConfig: + """One full-attention group + one sliding-window group (Gemma-like).""" + full_spec = FullAttentionSpec( + block_size=16, num_kv_heads=4, head_size=64, dtype=torch.float16 + ) + sw_spec = SlidingWindowSpec( + block_size=16, + num_kv_heads=4, + head_size=64, + dtype=torch.float16, + sliding_window=32, + ) + num_blocks = 2 + page = full_spec.page_size_bytes + return KVCacheConfig( + num_blocks=num_blocks, + kv_cache_tensors=[ + KVCacheTensor( + size=2 * num_blocks * page, + layers=["full0", "sw0"], + layer_stride=num_blocks * page, + block_stride=page, + ) + ], + kv_cache_groups=[ + KVCacheGroupSpec(layer_names=["full0"], kv_cache_spec=full_spec), + KVCacheGroupSpec(layer_names=["sw0"], kv_cache_spec=sw_spec), + ], + ) + + +def _read_scheduler( + kv_cache_config: KVCacheConfig, disable_hma: bool = False +) -> MoRIIOConnectorScheduler: + vllm_config = create_vllm_config(role="kv_producer", read_mode=True) + vllm_config.scheduler_config.disable_hybrid_kv_cache_manager = disable_hma + with set_current_vllm_config(vllm_config): + connector = MoRIIOConnector( + vllm_config, KVConnectorRole.SCHEDULER, kv_cache_config + ) + assert connector.connector_scheduler is not None + return connector.connector_scheduler + + +def test_hma_blocks_per_sw_two_groups(): + """A Full + SlidingWindow config runs with HMA and computes block budgets + correctly""" + scheduler = _read_scheduler(_make_hybrid_kv_cache_config()) + assert scheduler._is_hma_required is True + # cdiv(32, 16) + 1 == 3 for the sliding-window group, 0 for full attention. + assert scheduler.blocks_per_sw == [0, 3] + + +@pytest.mark.parametrize( + "swa_enabled, disable_hma, expected_is_hma", + [ + (True, False, True), # sliding-window group present, HMA enabled + (True, True, False), # sliding-window group present but HMA disabled + (False, False, False), # full-attention only, HMA not needed + ], +) +def test_is_hma_required(swa_enabled, disable_hma, expected_is_hma): + """_is_hma_required tracks both the KV cache groups and the + --disable-hybrid-kv-cache-manager flag. When HMA is off, + get_exchange_clipped_blocks must be a no-op.""" + config = ( + _make_hybrid_kv_cache_config() if swa_enabled else _make_test_kv_cache_config() + ) + scheduler = _read_scheduler(config, disable_hma=disable_hma) + assert scheduler._is_hma_required is expected_is_hma + if not expected_is_hma: + blocks = [[1, 2, 3, 4, 5]] + assert scheduler.get_exchange_clipped_blocks(blocks) == blocks + + +def test_non_sliding_window_hybrid_is_rejected(): + """A hybrid group that is not sliding-window (e.g. chunked-local + attention) must fail closed rather than be silently mistransferred.""" + full_spec = FullAttentionSpec( + block_size=16, num_kv_heads=4, head_size=64, dtype=torch.float16 + ) + local_spec = ChunkedLocalAttentionSpec( + block_size=16, + num_kv_heads=4, + head_size=64, + dtype=torch.float16, + attention_chunk_size=32, + ) + num_blocks = 2 + page = full_spec.page_size_bytes + config = KVCacheConfig( + num_blocks=num_blocks, + kv_cache_tensors=[ + KVCacheTensor( + size=2 * num_blocks * page, + layers=["full0", "local0"], + layer_stride=num_blocks * page, + block_stride=page, + ) + ], + kv_cache_groups=[ + KVCacheGroupSpec(layer_names=["full0"], kv_cache_spec=full_spec), + KVCacheGroupSpec(layer_names=["local0"], kv_cache_spec=local_spec), + ], + ) + with pytest.raises(NotImplementedError, match="sliding-window hybrid"): + _read_scheduler(config) + + +def test_token_count_basis_uses_full_attention_group(): + """Chunked-prefill token counting must use an unclipped full-attention + group (blocks_per_sw == 0), not a clipped sliding-window group.""" + scheduler = _read_scheduler(_make_hybrid_kv_cache_config()) + # Group 0 is the full-attention group + assert scheduler._full_attn_group_idx == 0 + assert scheduler._full_attn_block_size == 16 + + +def test_get_exchange_clipped_blocks_clips_only_sw_group(): + """get_exchange_clipped_blocks keeps the full attn group intact and clips the + sliding-window group to its window tail.""" + scheduler = _read_scheduler(_make_hybrid_kv_cache_config()) + full = [10, 11, 12, 13, 14] + sw = [20, 21, 22, 23, 24] + clipped = scheduler.get_exchange_clipped_blocks([full, sw]) + assert clipped[0] == full + assert clipped[1] == [22, 23, 24] + + +def test_metadata_hma_block_ids_preserved_per_group(): + """add_new_req stores per-group (BlockIds) block lists unchanged for both + read and write, so the hybrid group structure is retained.""" + metadata = MoRIIOConnectorMetadata() + + # Assume: + # - Full-attention group (6 blocks) + + # - sliding-window group already clipped to its window tail (3 blocks). + fa_blocks = [0, 1, 2, 3, 4, 5] + sw_blocks = [10, 11, 12] + local_block_ids = [fa_blocks, sw_blocks] + remote_block_ids = [[100, 101, 102, 103, 104, 105], [200, 201, 202]] + + base_params = { + "remote_engine_id": "remote-engine", + "remote_host": "127.0.0.1", + "remote_handshake_port": 6301, + "remote_notify_port": 61005, + "remote_block_ids": remote_block_ids, + } + + # Read mode: both local and remote ids stay per-group. + metadata.add_new_req( + request_id="recv-req", + local_block_ids=local_block_ids, + kv_transfer_params={**base_params, "transfer_id": "recv-req"}, + ) + recv_meta = metadata.reqs_to_recv["recv-req"] + assert recv_meta.local_block_ids == [fa_blocks, sw_blocks] + assert recv_meta.remote_block_ids == remote_block_ids + + # Write mode: the decode peer allocates its own blocks, so #remote_block_ids may + # be empty, Local group structure is kept. + metadata.add_new_req( + request_id="save-req", + local_block_ids=local_block_ids, + kv_transfer_params={ + **base_params, + "transfer_id": "save-req", + "remote_block_ids": [], + }, + write_mode=True, + ) + save_meta = metadata.reqs_to_save["save-req"] + assert save_meta.local_block_ids == [fa_blocks, sw_blocks] + assert save_meta.remote_block_ids == [] + + +def test_single_group_path_unchanged(): + """A full attn config does not enable HMA and never clips blocks.""" + scheduler = _read_scheduler(_make_test_kv_cache_config()) + assert scheduler._is_hma_required is False + assert scheduler.blocks_per_sw == [0] + blocks = [[1, 2, 3, 4, 5]] + assert scheduler.get_exchange_clipped_blocks(blocks) == blocks + + +def test_hybrid_write_mode_rejected(): + """Hybrid KV cache groups are unsupported in WRITE mode and fail closed.""" + vllm_config = create_vllm_config(role="kv_producer", read_mode=False) + with ( + set_current_vllm_config(vllm_config), + pytest.raises(NotImplementedError), + ): + MoRIIOConnector( + vllm_config, + KVConnectorRole.SCHEDULER, + _make_hybrid_kv_cache_config(), + ) + + +def test_worker_layer_to_group_routing(mock_parallel_groups): + """The worker maps every layer to its KV cache group correctly.""" + vllm_config = create_vllm_config(role="kv_consumer", read_mode=True) + # Building the worker directly bypasses MoRIIOConnector._set_port_defaults, + # so provide the ports manually. + vllm_config.kv_transfer_config.kv_connector_extra_config.update( + { + "http_port": 12346, + "handshake_port": 12347, + "notify_port": 12348, + } + ) + with set_current_vllm_config(vllm_config): + worker = FakeMoRIIOConnectorWorker( + vllm_config, + "engine0", + hand_shake_latency=0, + kv_cache_config=_make_hybrid_kv_cache_config(), + ) + assert worker.layer_to_group == {"full0": 0, "sw0": 1} diff --git a/tests/v1/kv_connector/unit/test_moriio_routing.py b/tests/v1/kv_connector/unit/test_moriio_routing.py index a9a3521beb..fc8b323218 100644 --- a/tests/v1/kv_connector/unit/test_moriio_routing.py +++ b/tests/v1/kv_connector/unit/test_moriio_routing.py @@ -20,12 +20,15 @@ from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector import from vllm.v1.request import RequestStatus _request_finished = MoRIIOConnectorScheduler.request_finished +_clip_blocks = MoRIIOConnectorScheduler.get_exchange_clipped_blocks def _producer(global_dp_rank: int, dp_size: int, dp_size_local: int): """Minimal prefill-side (producer) scheduler stand-in for request_finished.""" - return SimpleNamespace( + s = SimpleNamespace( is_producer=True, + _is_hma_required=False, + blocks_per_sw=[0], _global_dp_rank=global_dp_rank, engine_id="engine-abc", host_ip="10.0.0.1", @@ -39,6 +42,8 @@ def _producer(global_dp_rank: int, dp_size: int, dp_size_local: int): ) ), ) + s.get_exchange_clipped_blocks = _clip_blocks.__get__(s) + return s def _req( diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_common.py b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_common.py index a3010203ee..7d06b925ba 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_common.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_common.py @@ -14,6 +14,7 @@ import torch import zmq from vllm.config import KVTransferConfig, VllmConfig +from vllm.distributed.kv_transfer.kv_connector.utils import BlockIds from vllm.distributed.kv_transfer.kv_connector.v1.base import ( KVConnectorMetadata, ) @@ -444,8 +445,8 @@ class ReqMeta: """Metadata for a single request.""" transfer_id: TransferId - local_block_ids: list[int] - remote_block_ids: list[int] + local_block_ids: BlockIds + remote_block_ids: BlockIds remote_host: str remote_port: int remote_handshake_port: int @@ -481,7 +482,7 @@ class MoRIIOConnectorMetadata(KVConnectorMetadata): def add_new_req( self, request_id: ReqId, - local_block_ids: list[int], + local_block_ids: BlockIds, kv_transfer_params: dict[str, Any], write_mode=False, ): diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py index 349cf50775..ecdd59fca3 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py @@ -18,10 +18,12 @@ import torch import zmq from vllm.config import CUDAGraphMode, VllmConfig +from vllm.distributed.kv_transfer.kv_connector.utils import BlockIds from vllm.distributed.kv_transfer.kv_connector.v1.base import ( KVConnectorBase_V1, KVConnectorMetadata, KVConnectorRole, + SupportsHMA, ) from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_common import ( ROLE, @@ -67,19 +69,24 @@ from vllm.distributed.parallel_state import ( ) from vllm.forward_context import ForwardContext, get_forward_context from vllm.logger import init_logger +from vllm.utils.math_utils import cdiv from vllm.utils.network_utils import ( make_zmq_path, make_zmq_socket, ) from vllm.v1.attention.selector import get_attn_backend from vllm.v1.core.sched.output import SchedulerOutput +from vllm.v1.kv_cache_interface import ( + FullAttentionSpec, + KVCacheConfig, + SlidingWindowSpec, +) from vllm.v1.outputs import KVConnectorOutput from vllm.v1.request import RequestStatus if TYPE_CHECKING: from vllm.v1.attention.backend import AttentionMetadata from vllm.v1.core.kv_cache_manager import KVCacheBlocks - from vllm.v1.kv_cache_interface import KVCacheConfig from vllm.v1.request import Request logger = init_logger(__name__) @@ -189,7 +196,7 @@ def resolve_moriio_transfer_ack( return transfer_id -class MoRIIOConnector(KVConnectorBase_V1): +class MoRIIOConnector(KVConnectorBase_V1, SupportsHMA): def __init__( self, vllm_config: VllmConfig, @@ -225,7 +232,7 @@ class MoRIIOConnector(KVConnectorBase_V1): ) if role == KVConnectorRole.SCHEDULER: self.connector_scheduler: MoRIIOConnectorScheduler | None = ( - MoRIIOConnectorScheduler(vllm_config, self.engine_id) + MoRIIOConnectorScheduler(vllm_config, self.engine_id, kv_cache_config) ) self.connector_worker: MoRIIOConnectorWorker | None = None elif role == KVConnectorRole.WORKER: @@ -283,6 +290,14 @@ class MoRIIOConnector(KVConnectorBase_V1): self, request: "Request", block_ids: list[int], + ) -> tuple[bool, dict[str, Any] | None]: + assert self.connector_scheduler is not None + return self.connector_scheduler.request_finished(request, (block_ids,)) + + def request_finished_all_groups( + self, + request: "Request", + block_ids: BlockIds, ) -> tuple[bool, dict[str, Any] | None]: assert self.connector_scheduler is not None return self.connector_scheduler.request_finished(request, block_ids) @@ -368,7 +383,12 @@ class MoRIIOConnector(KVConnectorBase_V1): class MoRIIOConnectorScheduler: """Implementation of Scheduler side methods""" - def __init__(self, vllm_config: VllmConfig, engine_id: str): + def __init__( + self, + vllm_config: VllmConfig, + engine_id: str, + kv_cache_config: "KVCacheConfig", + ): self.vllm_config = vllm_config assert vllm_config.kv_transfer_config is not None, ( @@ -378,6 +398,66 @@ class MoRIIOConnectorScheduler: self.block_size = vllm_config.cache_config.block_size self.engine_id: EngineId = engine_id self.mode = get_moriio_mode(self.kv_transfer_config) + + self._is_hma_required = ( + not vllm_config.scheduler_config.disable_hybrid_kv_cache_manager + and any( + not isinstance(g.kv_cache_spec, FullAttentionSpec) + for g in kv_cache_config.kv_cache_groups + ) + ) + if self._is_hma_required: + # TODO(simondanielsson): support non-sliding window hybrids + unsupported = [ + type(g.kv_cache_spec).__name__ + for g in kv_cache_config.kv_cache_groups + if not isinstance( + g.kv_cache_spec, (FullAttentionSpec, SlidingWindowSpec) + ) + ] + if unsupported: + raise NotImplementedError( + "MoRIIO only supports sliding-window hybrid models (e.g. " + "Gemma) under the hybrid KV cache manager. Unsupported KV " + f"cache group spec(s): {sorted(set(unsupported))}. Pass " + "--disable-hybrid-kv-cache-manager to run without HMA." + ) + if self.mode == MoRIIOMode.WRITE: + # TODO(simondanielsson): support HMA in WRITE mode + raise NotImplementedError( + "MoRIIO WRITE mode does not support hybrid KV cache groups " + "(sliding-window attention). Use READ mode, or pass " + "--disable-hybrid-kv-cache-manager." + ) + + sw_sizes_tokens: list[tuple[int, int]] = [ + (g.kv_cache_spec.sliding_window, g.kv_cache_spec.block_size) + if isinstance(g.kv_cache_spec, SlidingWindowSpec) + else (0, self.block_size) + for g in kv_cache_config.kv_cache_groups + ] + # add 1 to conservatively account for boundary overlap eg window isn't fully + # aligned with blocks. + self.blocks_per_sw = [ + cdiv(n_tokens, block_size) + 1 if n_tokens else 0 + for n_tokens, block_size in sw_sizes_tokens + ] + # In WRITE mode and chunked prefill, we perform the write after the last chunk. + # Hence we need to track once we are in the last chunk. We do this by computing + # len(block_ids[g]) * block_size[g] for a full attn group g, as hybrid groups + # may be clipped and hence not reflect the full sequence length. Here we store + # g and block_size[g] for later. + self._full_attn_group_idx = 0 + self._full_attn_block_size = self.block_size + for gi, group in enumerate(kv_cache_config.kv_cache_groups): + is_full_attn = self.blocks_per_sw[gi] == 0 + if is_full_attn: + self._full_attn_group_idx = gi + self._full_attn_block_size = getattr( + group.kv_cache_spec, "block_size", self.block_size + ) + break + self.host_ip = resolve_host_ip( self.kv_transfer_config.kv_connector_extra_config ) @@ -409,14 +489,14 @@ class MoRIIOConnectorScheduler: # Requests that need to start recv/send. # New requests are added by update_state_after_alloc in # the scheduler. Used to make metadata passed to Worker. - self._reqs_need_recv: dict[ReqId, tuple[Request, list[int]]] = {} - self._reqs_need_save: dict[ReqId, tuple[Request, list[int]]] = {} + self._reqs_need_recv: dict[ReqId, tuple[Request, BlockIds]] = {} + self._reqs_need_save: dict[ReqId, tuple[Request, BlockIds]] = {} # Snapshot of kv_transfer_params for chunked prefill recovery. self._req_kv_params: dict[ReqId, dict] = {} # For chunked prefill, we perform layer-wise access within the final chunk. # TODO: Perform transfer at end chunk. - self._reqs_need_pending_save: dict[ReqId, tuple[Request, list[int]]] = {} + self._reqs_need_pending_save: dict[ReqId, tuple[Request, BlockIds]] = {} if self.is_producer: set_role(ROLE.PRODUCER) @@ -442,6 +522,23 @@ class MoRIIOConnectorScheduler: self.transfer_id_to_request_id: dict[TransferId, ReqId] = {} self.request_id_to_transfer_id: dict[ReqId, TransferId] = {} + def get_exchange_clipped_blocks( + self, + block_ids: BlockIds, + ) -> BlockIds: + """Clip the block list based on the layer type. + + Behavior: + - sliding window: keep only the last window size blocks + - full attention: keep all blocks + """ + if len(block_ids) == 0 or not self._is_hma_required: + return [list(blocks) for blocks in block_ids] + return [ + blocks[-self.blocks_per_sw[i] :] if self.blocks_per_sw[i] > 0 else blocks + for i, blocks in enumerate(block_ids) + ] + def map_request_id(self, request_id: ReqId, transfer_id: TransferId): self.transfer_id_to_request_id[transfer_id] = request_id self.request_id_to_transfer_id[request_id] = transfer_id @@ -642,7 +739,7 @@ class MoRIIOConnectorScheduler: request_id = request.request_id self.map_request_id(request_id, transfer_id) if params.get("do_remote_decode"): - local_block_ids = blocks.get_block_ids()[0] + local_block_ids: BlockIds = blocks.get_block_ids() self._reqs_need_save[request.request_id] = (request, local_block_ids) # Snapshot params now so chunked-prefill build_connector_meta # can recover them on the final chunk even if the live @@ -657,17 +754,22 @@ class MoRIIOConnectorScheduler: if "remote_engine_id" in params: if num_external_tokens > 0: # Get unhashed blocks to pull from remote. - local_block_ids = blocks.get_block_ids()[0] - assert len(local_block_ids) <= len(remote_block_ids) - if len(local_block_ids) != len(remote_block_ids): - local_block_ids = remote_block_ids[ - -len(local_block_ids) : - ] + local_block_ids = self.get_exchange_clipped_blocks( + blocks.get_block_ids() + ) + # Partial cache hits can lead to fewer local blocks vs + # remote blocks, but never more + assert all( + len(local) <= len(remote) + for local, remote in zip( + local_block_ids, remote_block_ids, strict=True + ) + ) else: # If remote_blocks and num_external_tokens = 0, we have # a full prefix cache hit on the D worker. We need to call # send_notify in _read_blocks to free the memory on the P. - local_block_ids = [] + local_block_ids = [[] for _ in remote_block_ids] self._reqs_need_recv[request.request_id] = ( request, @@ -818,8 +920,6 @@ class MoRIIOConnectorScheduler: new_block_ids = scheduler_output.scheduled_cached_reqs.new_block_ids[i] if new_block_ids is not None: - block_ids = new_block_ids[0] - # TODO : hybrid attn, etc # A non-disagg request (no kv_transfer_params, e.g. smoke # test) is never registered in _reqs_need_pending_save; # indexing it unconditionally would KeyError and crash the @@ -827,12 +927,16 @@ class MoRIIOConnectorScheduler: if req_id not in self._reqs_need_pending_save: continue req, existing_blocks = self._reqs_need_pending_save[req_id] - updated_blocks = list(existing_blocks) + (block_ids) + updated_blocks = [ + existing_blocks[g] + new_block_ids[g] + for g in range(len(new_block_ids)) + ] self._reqs_need_pending_save[req_id] = (req, updated_blocks) - if ( - len(self._reqs_need_pending_save[req_id][1]) * self.block_size - >= req.num_prompt_tokens - ): + saved_tokens = ( + len(updated_blocks[self._full_attn_group_idx]) + * self._full_attn_block_size + ) + if saved_tokens >= req.num_prompt_tokens: # Final chunk: live kv_transfer_params may be cleared, # so prefer the snapshot from update_state_after_alloc. kv_params = self._req_kv_params.pop( @@ -857,7 +961,10 @@ class MoRIIOConnectorScheduler: for req_id, (req, block_ids) in self._reqs_need_save.items(): kv_params = self._req_kv_params.get(req_id, req.kv_transfer_params or {}) - if req.num_prompt_tokens > len(block_ids) * self.block_size: + tokens_covered = ( + len(block_ids[self._full_attn_group_idx]) * self._full_attn_block_size + ) + if req.num_prompt_tokens > tokens_covered: # not last chunk prefill self._reqs_need_pending_save[req_id] = (req, block_ids) continue @@ -898,7 +1005,7 @@ class MoRIIOConnectorScheduler: def request_finished( self, request: "Request", - block_ids: list[int], + block_ids: BlockIds, ) -> tuple[bool, dict[str, Any] | None]: """ Once a request is finished, determine whether request blocks @@ -936,7 +1043,8 @@ class MoRIIOConnectorScheduler: if self.mode == MoRIIOMode.WRITE: self._release_write_prefill_blocks(request.request_id, params) else: - self._reqs_need_recv[request.request_id] = (request, []) + empty_groups: BlockIds = [[] for _ in self.blocks_per_sw] + self._reqs_need_recv[request.request_id] = (request, empty_groups) params["do_remote_prefill"] = False return False, None @@ -946,10 +1054,9 @@ class MoRIIOConnectorScheduler: ): return False, None - # computed_block_ids = block_ids if all_full else block_ids[:-1] - computed_block_ids = block_ids - # If prompt < block_size, no xfer so free blocks immediately. - delay_free_blocks = len(computed_block_ids) > 0 + # On producer + computed_block_ids = self.get_exchange_clipped_blocks(block_ids) + delay_free_blocks = any(len(group) > 0 for group in computed_block_ids) if delay_free_blocks: # Prefill request on remote. It will be read from D upon completion @@ -1102,6 +1209,12 @@ class MoRIIOConnectorWorker: self.kv_transfer_config = vllm_config.kv_transfer_config self.is_producer = self.kv_transfer_config.is_kv_producer self.layer_to_spec = build_layer_to_spec(kv_cache_config) + # we transfer by layer, but store by kv cache group + self.layer_to_group: dict[str, int] = { + layer_name: group_idx + for group_idx, group in enumerate(kv_cache_config.kv_cache_groups) + for layer_name in group.layer_names + } if self.is_producer: set_role(ROLE.PRODUCER) @@ -1789,8 +1902,6 @@ class MoRIIOConnectorWorker: self.block_size, ) self.block_size = first_geometry.block_size - # TODO(tms): self.block_len needs to be per-layer for sliding window, - # hybrid attn, etc # block size in bytes self.block_len = first_geometry.block_len self.kv_cache_shape = first_kv_cache.shape @@ -1803,14 +1914,30 @@ class MoRIIOConnectorWorker: self.layer_base_addr_index = {} base_addr_idx = 0 + hma_enabled = ( + not self.vllm_config.scheduler_config.disable_hybrid_kv_cache_manager + ) for layer_name in kv_caches: self.layer_base_addr_index[layer_name] = base_addr_idx geometry = self._get_layer_transfer_geometry(layer_name) - if geometry.block_size != self.block_size: + if hma_enabled: + if geometry.block_len != self.block_len: + raise ValueError( + "MoRIIO KV cache block length mismatch for layer " + f"{layer_name}: {geometry.block_len} != {self.block_len}" + ) + elif geometry.block_size != self.block_size: raise ValueError( "MoRIIO KV cache block size mismatch for layer " f"{layer_name}: {geometry.block_size} != {self.block_size}" ) + # num_blocks is advertised as a single scalar to the peer, so must it + # be uniform + if geometry.num_blocks != self.num_blocks: + raise ValueError( + "MoRIIO KV cache num_blocks mismatch for layer " + f"{layer_name}: {geometry.num_blocks} != {self.num_blocks}" + ) self.block_lens[layer_name] = geometry.block_len for cache, region_len in self._iter_layer_registration_regions(layer_name): base_addr = cache.data_ptr() @@ -2455,12 +2582,16 @@ class MoRIIOConnectorWorker: self.remote_dp_size_local = int(meta.remote_dp_size_local) else: self.remote_dp_size_local = int(meta.remote_dp_size) + # WRITE does not support HMA, so unwrap blocks into flat lists. + # remote_block_ids can itself be empty so need to be careful when unwrapping. + local_block_ids = meta.local_block_ids[0] + remote_block_ids = meta.remote_block_ids[0] if meta.remote_block_ids else [] self.schedule_write_blocks( request_id=req_id, transfer_id=meta.transfer_id, dst_engine_id=meta.remote_engine_id, - local_block_ids=meta.local_block_ids, - remote_block_ids=meta.remote_block_ids, + local_block_ids=local_block_ids, + remote_block_ids=remote_block_ids, layer_name=layer_name, kv_layer=kv_layer, remote_notify_port=meta.remote_notify_port, @@ -2607,8 +2738,8 @@ class MoRIIOConnectorWorker: def _read_blocks( self, - local_block_ids: list[int], - remote_block_ids: list[int], + local_block_ids: BlockIds, + remote_block_ids: BlockIds, dst_engine_id: str, request_id: str, transfer_id: str, @@ -2653,10 +2784,11 @@ class MoRIIOConnectorWorker: sess_idx = list(self.layer_name_to_local_kv_cache_metadata.keys()).index( layer_name ) + group_idx = self.layer_to_group[layer_name] offs = self._compute_block_transfer_offsets( layer_name, - local_block_ids, - remote_block_ids, + local_block_ids[group_idx], + remote_block_ids[group_idx], remote_moriio_meta, remote_tp_size=remote_tp_size, remote_engine_id=remote_dp_engine_id,