[Mini-vLLM] Cut dead subsystems: models, entrypoints, backends, utils, tests

- Drop dead model modules, keeping files live imports require
- Remove anthropic, cohere, codegen entrypoints and batch STT modes
- Delete unused attention backends, MoE experts, and NPU/XPU util leaves
- Strip dead third_party trees and ROCm leaves with no CUDA path
- Remove mooncake EC connectors: dependency missing since branch base
- Re-audit verified by full import sweep and Qwen3.5 smoke test

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-19 00:20:48 +07:00
co-authored by Hinata
parent 6a95f62e2f
commit 81074e94a3
940 changed files with 11 additions and 380533 deletions
@@ -150,30 +150,12 @@ class KVConnectorFactory:
# The registration should not be done in each individual file, as we want to
# only load the files corresponding to the current connector.
KVConnectorFactory.register_connector(
"ExampleConnector",
"vllm.distributed.kv_transfer.kv_connector.v1.example_connector",
"ExampleConnector",
)
KVConnectorFactory.register_connector(
"ExampleHiddenStatesConnector",
"vllm.distributed.kv_transfer.kv_connector.v1.example_hidden_states_connector",
"ExampleHiddenStatesConnector",
)
KVConnectorFactory.register_connector(
"LMCacheConnectorV1",
"vllm.distributed.kv_transfer.kv_connector.v1.lmcache_connector",
"LMCacheConnectorV1",
)
KVConnectorFactory.register_connector(
"LMCacheMPConnector",
"vllm.distributed.kv_transfer.kv_connector.v1.lmcache_mp_connector",
"LMCacheMPConnector",
)
KVConnectorFactory.register_connector(
"NixlConnector",
"vllm.distributed.kv_transfer.kv_connector.v1.nixl",
@@ -222,21 +204,6 @@ KVConnectorFactory.register_connector(
"DecodeBenchConnector",
)
KVConnectorFactory.register_connector(
"MooncakeConnector",
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector",
"MooncakeConnector",
)
KVConnectorFactory.register_connector(
"MooncakeStoreConnector",
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.connector",
"MooncakeStoreConnector",
)
KVConnectorFactory.register_connector(
"FlexKVConnectorV1",
"vllm.distributed.kv_transfer.kv_connector.v1.flexkv_connector",
"FlexKVConnectorV1",
)
KVConnectorFactory.register_connector(
"SimpleCPUOffloadConnector",
"vllm.distributed.kv_transfer.kv_connector.v1.simple_cpu_offload_connector",
@@ -1,270 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterable
from typing import TYPE_CHECKING, Any
import torch
from vllm.config import VllmConfig
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorBase_V1,
KVConnectorMetadata,
KVConnectorRole,
)
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import KVConnectorStats
from vllm.logger import init_logger
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.outputs import KVConnectorOutput
if TYPE_CHECKING:
from vllm.distributed.kv_events import KVCacheEvent
from vllm.forward_context import ForwardContext
from vllm.v1.attention.backend import AttentionMetadata
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.request import Request
logger = init_logger(__name__)
# FlexKV is a distributed KV Store and multi-level cache management system for
# ultra-large-scale LLM inference.
# GitHub: https://github.com/taco-project/FlexKV
# Install: git clone git@github.com:taco-project/FlexKV.git \
# && cd FlexKV && bash build.sh
class FlexKVConnectorV1(KVConnectorBase_V1):
"""KV Connector that offloads KV cache to FlexKV.
FlexKV is a distributed KV Store and multi-level cache management system
designed for ultra-large-scale LLM inference. It supports offloading KV
cache to CPU memory, SSD, and remote storage.
Installation:
See https://github.com/taco-project/FlexKV for installation instructions.
Quick start::
git clone git@github.com:taco-project/FlexKV.git
cd FlexKV && bash build.sh
Configuration:
Pass ``kv_connector="FlexKVConnectorV1"`` via ``--kv-transfer-config``::
--kv-transfer-config \
'{"kv_connector":"FlexKVConnectorV1","kv_role":"kv_both"}'
"""
def __init__(
self,
vllm_config: "VllmConfig",
role: KVConnectorRole,
kv_cache_config: "KVCacheConfig",
):
super().__init__(
vllm_config=vllm_config, role=role, kv_cache_config=kv_cache_config
)
try:
from flexkv.integration.vllm.vllm_v1_adapter import FlexKVConnectorV1Impl
except ImportError as e:
raise ImportError(
"FlexKV is not installed. Please install it to use "
"FlexKVConnectorV1. See https://github.com/taco-project/FlexKV "
"for installation instructions."
) from e
self._flexkv_connector = FlexKVConnectorV1Impl(vllm_config, role)
def shutdown(self):
self._flexkv_connector.shutdown()
# ==============================
# Worker-side methods
# ==============================
def start_load_kv(self, forward_context: "ForwardContext", **kwargs) -> None:
"""No-op for FlexKV (currently).
FlexKV manages all KV transfers on the **scheduler side** via
``build_connector_meta`` (which calls ``launch_tasks``) and
``update_connector_output`` (which polls ``query_finished_task``).
KV blocks are transferred directly between the FlexKV server and
vLLM's GPU memory without worker-side intervention during the
forward pass — similar to how NIXL operates.
These worker-side hooks are kept (rather than omitted) to satisfy
the ``KVConnectorBase_V1`` interface contract and to serve as
extension points for a future worker-side layer-pipelining path.
Args:
forward_context (ForwardContext): the forward context.
**kwargs (Any): additional arguments (unused).
"""
self._flexkv_connector.start_load_kv(forward_context, **kwargs)
def wait_for_layer_load(self, layer_name: str) -> None:
"""No-op for FlexKV (currently).
FlexKV manages all KV transfers on the scheduler side.
This hook is retained for ``KVConnectorBase_V1`` API compatibility.
Args:
layer_name: the name of the layer (unused).
"""
self._flexkv_connector.wait_for_layer_load(layer_name)
def save_kv_layer(
self,
layer_name: str,
kv_layer: torch.Tensor,
attn_metadata: "AttentionMetadata",
**kwargs,
) -> None:
"""No-op for FlexKV (currently).
FlexKV offloads KV cache asynchronously from the scheduler side
after a request finishes (see ``request_finished``). It does not
intercept individual layer tensors during the forward pass.
This hook is retained to satisfy ``KVConnectorBase_V1`` and as an
extension point for future per-layer async offload support.
Args:
layer_name (str): the name of the layer (unused).
kv_layer (torch.Tensor): the paged KV buffer (unused).
attn_metadata (AttentionMetadata): the attention metadata (unused).
**kwargs (Any): additional arguments (unused).
"""
self._flexkv_connector.save_kv_layer(
layer_name, kv_layer, attn_metadata, **kwargs
)
def wait_for_save(self):
"""No-op for FlexKV (currently).
KV offload tasks are tracked asynchronously by the scheduler
connector via ``request_finished`` / ``query_finished_task``.
There is no pending worker-side save to wait for at
forward-context exit.
Retained to satisfy ``KVConnectorBase_V1`` and as an extension
point for future worker-side save-completion signalling.
"""
self._flexkv_connector.wait_for_save()
def get_finished(
self, finished_req_ids: set[str]
) -> tuple[set[str] | None, set[str] | None]:
"""Notify worker-side connector of requests that have finished
generating tokens.
Returns:
Tuple of (sending/saving ids, recving/loading ids) for requests
that have finished asynchronous transfer. The finished saves/sends
req ids must belong to a set provided in a call to this method
(this call or a prior one).
"""
return self._flexkv_connector.get_finished(finished_req_ids)
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
"""Initialize with the KV caches. Useful for pre-registering the
KV caches in the KVConnector (e.g. for NIXL).
Args:
kv_caches: dictionary of layer names to kv cache tensors.
"""
self._flexkv_connector.register_kv_caches(kv_caches)
# ==============================
# Scheduler-side methods
# ==============================
def get_num_new_matched_tokens(
self,
request: "Request",
num_computed_tokens: int,
) -> tuple[int, bool]:
"""Get the number of new tokens that can be loaded from the
external KV cache beyond ``num_computed_tokens``.
Args:
request (Request): the request object.
num_computed_tokens (int): the number of locally computed
tokens for this request.
Returns:
Tuple of (num_external_tokens, is_ready) where
num_external_tokens is the number of additional tokens that
can be loaded from the external KV cache.
"""
return self._flexkv_connector.get_num_new_matched_tokens(
request, num_computed_tokens
)
def update_state_after_alloc(
self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int
):
"""Update KVConnector state after block allocation."""
self._flexkv_connector.update_state_after_alloc(
request, blocks, num_external_tokens
)
def build_connector_meta(
self, scheduler_output: SchedulerOutput
) -> KVConnectorMetadata:
"""Build the connector metadata for this step.
This function should NOT modify fields in the scheduler_output.
Also, calling this function will reset the state of the connector.
Args:
scheduler_output (SchedulerOutput): the scheduler output object.
"""
return self._flexkv_connector.build_connector_meta(scheduler_output)
def update_connector_output(self, connector_output: KVConnectorOutput):
"""Update KVConnector state from worker-side connectors output.
Args:
connector_output (KVConnectorOutput): the worker-side
connectors output.
"""
self._flexkv_connector.update_connector_output(connector_output)
def request_finished(
self,
request: "Request",
block_ids: list[int],
) -> tuple[bool, dict[str, Any] | None]:
"""Called when a request has finished, before its blocks are freed.
Returns:
Tuple of (async_save, kv_transfer_params) where async_save is
True if the request is being saved/sent asynchronously and blocks
should not be freed until the request_id is returned from
:meth:`get_finished`. kv_transfer_params is an optional dict of
KVTransferParams to be included in the request outputs.
"""
return self._flexkv_connector.request_finished(request, block_ids)
def take_events(self) -> Iterable["KVCacheEvent"]:
"""Collect buffered KV cache events.
Returns:
New KV cache events since the last call.
"""
return self._flexkv_connector.take_events()
def get_kv_connector_stats(self) -> KVConnectorStats | None:
"""Get the KV connector stats collected during the last interval."""
return self._flexkv_connector.get_kv_connector_stats()
def get_block_ids_with_load_errors(self) -> set[int]:
"""Get the block ids that have failed to load."""
return self._flexkv_connector.get_block_ids_with_load_errors()
@@ -1,358 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterable
from typing import TYPE_CHECKING, Any
import torch
from vllm.config import VllmConfig
from vllm.distributed.kv_events import (
BlockStored,
KVCacheEvent,
KVConnectorKVEvents,
KVEventAggregator,
)
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorBase_V1,
KVConnectorMetadata,
KVConnectorRole,
)
from vllm.logger import init_logger
from vllm.v1.attention.backend import AttentionMetadata
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.outputs import KVConnectorOutput
if TYPE_CHECKING:
from vllm.forward_context import ForwardContext
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.request import Request
logger = init_logger(__name__)
class LMCacheKVEvents(KVConnectorKVEvents):
"""Concrete implementation of KVConnectorKVEvents using KVEventAggregator."""
def __init__(self, num_workers: int) -> None:
self._aggregator = KVEventAggregator(num_workers)
def add_events(self, events: list[KVCacheEvent]) -> None:
self._aggregator.add_events(events)
def aggregate(self) -> "LMCacheKVEvents":
"""Aggregate KV events and retain only common events."""
common_events = self._aggregator.get_common_events()
self._aggregator.clear_events()
self._aggregator.add_events(common_events)
self._aggregator.reset_workers()
return self
def increment_workers(self, count: int = 1) -> None:
self._aggregator.increment_workers(count)
def get_all_events(self) -> list[KVCacheEvent]:
return self._aggregator.get_all_events()
def get_number_of_workers(self) -> int:
return self._aggregator.get_number_of_workers()
def clear_events(self) -> None:
self._aggregator.clear_events()
self._aggregator.reset_workers()
def __repr__(self) -> str:
return f"<LMCacheKVEvents events={self.get_all_events()}>"
class LMCacheConnectorV1(KVConnectorBase_V1):
@classmethod
def requires_piecewise_for_cudagraph(cls, extra_config: dict[str, Any]) -> bool:
"""LMCache requires PIECEWISE CUDA graph mode when layerwise
operations are enabled. The wait_for_layer_load and save_kv_layer
methods perform actual async synchronization that cannot be
captured in CUDA graphs.
"""
return extra_config.get("use_layerwise", False)
def __init__(
self,
vllm_config: "VllmConfig",
role: KVConnectorRole,
kv_cache_config: "KVCacheConfig",
):
super().__init__(
vllm_config=vllm_config, role=role, kv_cache_config=kv_cache_config
)
assert vllm_config.kv_transfer_config is not None
use_native = vllm_config.kv_transfer_config.get_from_extra_config(
"use_native", False
)
if use_native:
logger.info("Initializing native LMCache connector")
# lazy import
from vllm.distributed.kv_transfer.kv_connector.v1 import lmcache_integration
_adapter = lmcache_integration.vllm_v1_adapter
cls = _adapter.LMCacheConnectorV1Impl
else:
logger.info("Initializing latest dev LMCache connector")
# lazy import
from lmcache.integration.vllm.vllm_v1_adapter import (
LMCacheConnectorV1Impl as LMCacheConnectorLatestImpl,
)
cls = LMCacheConnectorLatestImpl
self._lmcache_engine = cls(vllm_config, role, self)
self._kv_cache_events: LMCacheKVEvents | None = None
# ==============================
# Worker-side methods
# ==============================
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
"""Initialize with the KV caches. Useful for pre-registering the
KV Caches in the KVConnector (e.g. for NIXL).
Args:
kv_caches: dictionary of layer names, kv cache
"""
if hasattr(self._lmcache_engine, "register_kv_caches"):
self._lmcache_engine.register_kv_caches(kv_caches)
else:
logger.warning(
"LMCache engine does not support register_kv_caches, "
"please check and use the latest version"
)
def bind_connector_metadata(self, connector_metadata: KVConnectorMetadata) -> None:
super().bind_connector_metadata(connector_metadata)
engine = self._lmcache_engine
if hasattr(engine, "bind_connector_metadata"):
engine.bind_connector_metadata(connector_metadata)
else:
# Compat for engines predating the bind hook: start_load_kv may
# run after the forward (SchedulerOutput.has_sync_kv_loads), so
# the per-step layerwise state it used to reset must be reset
# here. Remove once LMCache ships bind_connector_metadata.
if hasattr(engine, "current_layer"):
engine.current_layer = 0
if hasattr(engine, "layerwise_retrievers"):
engine.layerwise_retrievers = []
engine.layerwise_storers = []
def start_load_kv(self, forward_context: "ForwardContext", **kwargs: Any) -> None:
"""Start loading the KV cache from the connector to vLLM's paged
KV buffer. This is called from the forward context before the
forward pass to enable async loading during model execution.
Args:
forward_context (ForwardContext): the forward context.
**kwargs: additional arguments for the load operation
Note:
The number of elements in kv_caches and layer_names should be
the same.
"""
self._lmcache_engine.start_load_kv(forward_context, **kwargs)
def wait_for_layer_load(self, layer_name: str) -> None:
"""Block until the KV for a specific layer is loaded into vLLM's
paged buffer. This is called from within attention layer to ensure
async copying from start_load_kv is complete.
This interface will be useful for layer-by-layer pipelining.
Args:
layer_name: the name of that layer
"""
self._lmcache_engine.wait_for_layer_load(layer_name)
def save_kv_layer(
self,
layer_name: str,
kv_layer: torch.Tensor,
attn_metadata: AttentionMetadata,
**kwargs: Any,
) -> None:
"""Start saving the a layer of KV cache from vLLM's paged buffer
to the connector. This is called from within attention layer to
enable async copying during execution.
Args:
layer_name (str): the name of the layer.
kv_layer (torch.Tensor): the paged KV buffer of the current
layer in vLLM.
attn_metadata (AttentionMetadata): the attention metadata.
**kwargs: additional arguments for the save operation.
"""
self._lmcache_engine.save_kv_layer(
layer_name, kv_layer, attn_metadata, **kwargs
)
def wait_for_save(self):
"""Block until all the save operations is done. This is called
as the forward context exits to ensure that the async saving
from save_kv_layer is complete before finishing the forward.
This prevents overwrites of paged KV buffer before saving done.
"""
self._lmcache_engine.wait_for_save()
def get_finished(
self, finished_req_ids: set[str]
) -> tuple[set[str] | None, set[str] | None]:
"""Notifies worker-side connector ids of requests that have
finished generating tokens.
Returns:
ids of requests that have finished asynchronous transfer
(requests that previously returned True from request_finished()),
tuple of (sending/saving ids, recving/loading ids).
The finished saves/sends req ids must belong to a set provided in a
call to this method (this call or a prior one).
"""
return self._lmcache_engine.get_finished(finished_req_ids)
def get_block_ids_with_load_errors(self) -> set[int]:
"""Get the set of block IDs that failed to load.
Returns:
Set of block IDs that encountered load errors.
Empty set if no load errors occurred.
"""
method = getattr(self._lmcache_engine, "get_block_ids_with_load_errors", None)
if callable(method):
return method()
# Fallback for older versions that don't support this method
return set()
def get_kv_connector_kv_cache_events(self) -> LMCacheKVEvents | None:
"""Get the KV connector kv cache events collected during the last interval."""
events = self._lmcache_engine.get_kv_events() # type: ignore [attr-defined]
if not events:
return None
blocks: list[BlockStored] = [
BlockStored(
block_hashes=e.block_hashes,
parent_block_hash=e.parent_block_hash,
token_ids=e.token_ids,
lora_id=e.lora_id,
block_size=e.block_size,
medium=e.medium,
lora_name=getattr(e, "lora_name", None),
)
for e in events
]
lmcache_kv_events = LMCacheKVEvents(num_workers=1)
lmcache_kv_events.add_events(blocks)
return lmcache_kv_events
# ==============================
# Scheduler-side methods
# ==============================
def get_num_new_matched_tokens(
self,
request: "Request",
num_computed_tokens: int,
) -> tuple[int | None, bool]:
"""Get number of new tokens that can be loaded from the
external KV cache beyond the num_computed_tokens.
Args:
request (Request): the request object.
num_computed_tokens (int): the number of locally
computed tokens for this request
Returns:
the number of tokens that can be loaded from the
external KV cache beyond what is already computed.
"""
return self._lmcache_engine.get_num_new_matched_tokens(
request, num_computed_tokens
), False
def update_state_after_alloc(
self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int
):
"""Update KVConnector state after block allocation."""
self._lmcache_engine.update_state_after_alloc(request, num_external_tokens)
def build_connector_meta(
self, scheduler_output: SchedulerOutput
) -> KVConnectorMetadata:
"""Build the connector metadata for this step.
This function should NOT modify fields in the scheduler_output.
Also, calling this function will reset the state of the connector.
Args:
scheduler_output (SchedulerOutput): the scheduler output object.
"""
return self._lmcache_engine.build_connector_meta(scheduler_output)
def update_connector_output(self, connector_output: KVConnectorOutput):
"""Update KVConnector state from worker-side connectors output.
Args:
connector_output (KVConnectorOutput): the worker-side
connectors output.
"""
# Get the KV events
kv_cache_events = connector_output.kv_cache_events
if not kv_cache_events or not isinstance(kv_cache_events, LMCacheKVEvents):
return
if self._kv_cache_events is None:
self._kv_cache_events = kv_cache_events
else:
self._kv_cache_events.add_events(kv_cache_events.get_all_events())
self._kv_cache_events.increment_workers(
kv_cache_events.get_number_of_workers()
)
return
def request_finished(
self,
request: "Request",
block_ids: list[int],
) -> tuple[bool, dict[str, Any] | None]:
"""Called when a request has finished, before its blocks are freed.
Returns:
True if the request is being saved/sent asynchronously and blocks
should not be freed until the request_id is returned from
get_finished().
Optional KVTransferParams to be included in the request outputs
returned by the engine.
"""
return self._lmcache_engine.request_finished(request, block_ids)
def take_events(self) -> Iterable["KVCacheEvent"]:
"""Take the KV cache events from the connector.
Yields:
New KV cache events since the last call.
"""
if self._kv_cache_events is not None:
self._kv_cache_events.aggregate()
kv_cache_events = self._kv_cache_events.get_all_events()
yield from kv_cache_events
self._kv_cache_events.clear_events()
self._kv_cache_events = None
@@ -1,20 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from . import multi_process_adapter, vllm_v1_adapter
from .multi_process_adapter import (
LMCacheMPSchedulerAdapter,
LMCacheMPWorkerAdapter,
LoadStoreOp,
ParallelStrategy,
)
__all__ = [
"vllm_v1_adapter",
"multi_process_adapter",
"LMCacheMPSchedulerAdapter",
"LMCacheMPWorkerAdapter",
"LoadStoreOp",
"ParallelStrategy",
]
@@ -1,735 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import os
from collections.abc import Iterable
from dataclasses import dataclass
from itertools import islice
from typing import Any
import torch
import zmq
from lmcache.utils import _lmcache_nvtx_annotate, init_logger
from lmcache.v1.multiprocess.custom_types import (
CudaIPCWrapper,
IPCCacheEngineKey,
KVCache,
)
from lmcache.v1.multiprocess.mq import MessageQueueClient, MessagingFuture
from lmcache.v1.multiprocess.protocol import RequestType, get_response_class
logger = init_logger(__name__)
def wrap_kv_caches(kv_caches: dict[str, torch.Tensor]) -> KVCache:
logger.info("KV caches keys are %s", list(kv_caches.keys()))
return [CudaIPCWrapper(tensor) for tensor in kv_caches.values()]
def striding_block_hashes(
block_hashes: list[bytes], blocks_in_chunk: int
) -> Iterable[bytes]:
"""Extract chunk-level hashes from block hashes by striding.
In hash-based vLLM, each vLLM block has its own hash. LMCache chunks
span ``blocks_in_chunk`` consecutive blocks. The representative hash
for a chunk is the hash of the **last** block in that chunk (because
each block hash already encodes its prefix). So we start at index
``blocks_in_chunk - 1`` and stride by ``blocks_in_chunk``.
"""
return islice(block_hashes, blocks_in_chunk - 1, None, blocks_in_chunk)
def send_lmcache_request(
mq_client: MessageQueueClient,
request_type: RequestType,
payloads: list[Any],
) -> MessagingFuture[Any]:
"""Helper function to send the request to the LMCache multiprocess server.
Args:
mq_client: The LMCache multiprocess mode message queue client
request_type: The request type
payloads: The request payloads
Returns:
A messaging future for the request
"""
future = mq_client.submit_request(
request_type, payloads, get_response_class(request_type)
)
return future
def get_lmcache_chunk_size(
mq_client: MessageQueueClient,
) -> int:
"""Helper function to get the LMCache chunk size from the server.
Args:
mq_client: The LMCache multiprocess mode message queue client
Returns:
An integer representing the LMCache chunk size
"""
future = send_lmcache_request(mq_client, RequestType.GET_CHUNK_SIZE, [])
chunk_size = future.result()
return chunk_size
@dataclass
class ParallelStrategy:
use_mla: bool
"""Whether to use the MLA."""
kv_world_size: int
"""
The kv world size, kv_world_size may not be equal to the actual_world_size,
in the case of mla, it will 'exclude' the effect of TP, the value is
calculated by `extract_world_size_and_kv_rank` in `lmcache_mp_connector.py`.
"""
kv_worker_id: int
"""
The kv worker id of the sub-process, kv_worker_id may not be equal to the
actual_worker_id, in the case of mla, it will 'exclude' the effect of TP,
the value is calculated by `extract_world_size_and_kv_rank` in
`lmcache_mp_connector.py`.
"""
actual_world_size: int
"""The actual world size."""
actual_worker_id: int
"""The actual worker id of the sub-process."""
tp_size: int
"""The tensor parallel size."""
pp_size: int
"""The pipeline parallel size."""
@dataclass
class LoadStoreOp:
block_ids: list[int]
"""Block ids for the load/store operation"""
token_ids: list[int] | None = None
"""Token IDs for the load/store operation (token mode)"""
block_hashes: list[bytes] | None = None
"""Block hashes for the load/store operation (hash mode)"""
start: int = 0
"""Start token index (token mode only)"""
end: int = 0
"""End token index (token mode only)"""
def __len__(self) -> int:
return len(self.block_ids)
StoreResult = bool
RetrieveResult = list[bool]
LookupResult = int
class LMCacheMPSchedulerAdapter:
def __init__(
self,
server_url: str,
context: zmq.Context,
model_name: str,
vllm_block_size: int,
parallel_strategy: ParallelStrategy,
):
"""Args:
server_url: The server URL for the LMCache message queue
context: The ZMQ context
model_name: The model name used for LMCache keys
vllm_block_size: The block size used in vLLM
parallel_strategy:
The parallel strategy, which includes `use_mla`,
`world_size`, `worker_id` and so on
"""
self.mq_client = MessageQueueClient(server_url, context)
# Request futures
self.lookup_futures: dict[str, MessagingFuture[LookupResult]] = {}
self.model_name = model_name
self.parallel_strategy = parallel_strategy
# Read chunk size from lmcache
self.chunk_size = get_lmcache_chunk_size(self.mq_client)
assert self.chunk_size % vllm_block_size == 0, (
"LMCache chunk size should be a multiple of vLLM block size"
)
self.blocks_in_chunk = self.chunk_size // vllm_block_size
@property
def world_size(self) -> int:
"""The world size."""
return self.parallel_strategy.kv_world_size
@property
def worker_id(self) -> int:
"""The worker id."""
return self.parallel_strategy.kv_worker_id
@property
def tp_size(self) -> int:
"""The tensor parallel size."""
return self.parallel_strategy.tp_size
@_lmcache_nvtx_annotate
def maybe_submit_lookup_request(
self,
request_id: str,
block_hashes: list[bytes] | None = None,
token_ids: list[int] | None = None,
) -> None:
"""Submit a new lookup request to LMCache if there is no ongoing request.
Supports both token-based and hash-based vLLM:
- token_ids: token IDs (token-based vLLM) -> single token-mode key
- block_hashes: block hashes (hash-based vLLM) -> strided hash-mode keys
Exactly one of block_hashes or token_ids must be provided.
Args:
request_id: The ID of the lookup request. The same ID indicates it's
from the same request
block_hashes: Block hashes to lookup from LMCache (hash mode)
token_ids: Token IDs to lookup from LMCache (token mode)
Returns:
None
Notes:
This function will have a side-effect: submitting a look up request to
LMCache, which will essentially 'lock' the KV cache chunks in the LMCache
for later retrieve operations.
In the meantime, this function will record the lookup request, and the
status of the look up request can be checked by `check_lookup_result`.
"""
if request_id in self.lookup_futures:
# Skip if there is already a lookup request
return
assert (block_hashes is None) != (token_ids is None), (
"Exactly one of block_hashes or token_ids must be provided"
)
if block_hashes is not None:
# Hash mode: stride block hashes -> N hash-mode keys
chunk_hashes = list(
striding_block_hashes(block_hashes, self.blocks_in_chunk)
)
keys = [
self._create_hash_key(ch, request_id=request_id) for ch in chunk_hashes
]
else:
# Token mode: truncate to chunk-aligned length
assert token_ids is not None
aligned_end = (len(token_ids) // self.chunk_size) * self.chunk_size
if aligned_end == 0:
return
keys = [
self._create_key(
token_ids,
start=0,
end=aligned_end,
request_id=request_id,
).no_worker_id_version()
]
future = send_lmcache_request(
self.mq_client,
RequestType.LOOKUP,
[keys],
)
self.lookup_futures[request_id] = future
@_lmcache_nvtx_annotate
def check_lookup_result(self, request_id: str) -> int | None:
"""Check the result of a previously submitted lookup request.
Args:
request_id: The ID of the lookup request submitted in
`maybe_submit_lookup_request`
Returns:
An integer representing the total number of tokens matched
in LMCache (prefix matching), or
None if the lookup request is not finished yet.
"""
assert request_id in self.lookup_futures, (
f"Lookup request for request_id={request_id} has not been submitted"
)
future = self.lookup_futures[request_id]
if not future.query():
return None
result = future.result()
num_chunks = result
return num_chunks * self.chunk_size
def num_blocks_per_chunk(self) -> int:
"""Returns:
The number of vllm blocks in a LMCache data chunk
"""
return self.blocks_in_chunk
def cleanup_lookup_result(self, request_id: str) -> None:
"""Clean up lookup future for a finished request to prevent memory leak.
Args:
request_id: The ID of the finished request.
"""
self.lookup_futures.pop(request_id, None)
def end_session(self, request_id: str) -> None:
"""Notify LMCache server to remove the session for a finished request.
Args:
request_id: The ID of the finished request.
"""
send_lmcache_request(
self.mq_client,
RequestType.END_SESSION,
[request_id],
)
# Helper functions
def _create_key(
self,
token_ids: list[int],
start: int = 0,
end: int = 0,
request_id: str | None = None,
) -> IPCCacheEngineKey:
"""Convert token IDs to an IPC cache engine key."""
return IPCCacheEngineKey(
model_name=self.model_name,
world_size=self.world_size,
worker_id=self.worker_id,
token_ids=tuple(token_ids),
start=start,
end=end,
request_id=request_id,
tp_size=self.tp_size,
)
def _create_hash_key(
self, chunk_hash: bytes, request_id: str | None = None
) -> IPCCacheEngineKey:
"""Create a hash-mode IPC cache engine key."""
return IPCCacheEngineKey(
model_name=self.model_name,
world_size=self.world_size,
worker_id=None,
chunk_hash=chunk_hash,
request_id=request_id,
tp_size=self.tp_size,
)
class LMCacheMPWorkerAdapter:
def __init__(
self,
server_url: str,
context: zmq.Context,
model_name: str,
vllm_block_size: int,
parallel_strategy: ParallelStrategy,
):
self.mq_client = MessageQueueClient(server_url, context)
# Instance id for GPU worker
self.instance_id = os.getpid()
# Registered kv caches from vLLM
self.kv_caches: dict[str, torch.Tensor] = {}
# Request futures
# request_id -> (future, other merged requests)
self.store_futures: dict[
str, tuple[MessagingFuture[StoreResult], list[str]]
] = {}
self.retrieve_futures: dict[
str, tuple[MessagingFuture[RetrieveResult], list[str]]
] = {}
# The store requests that have finished execution in LMCache
self.finished_stores: set[str] = set()
# The finished request ids that are passed via vLLM and also
# have corresponding store requests submitted to LMCache before
self.previously_finished: set[str] = set()
self.model_name = model_name
self.parallel_strategy = parallel_strategy
# Read chunk size from lmcache
chunk_size = get_lmcache_chunk_size(self.mq_client)
assert chunk_size % vllm_block_size == 0, (
"LMCache chunk size should be a multiple of vLLM block size"
)
self.blocks_in_chunk = chunk_size // vllm_block_size
@property
def world_size(self) -> int:
"""The world size."""
return self.parallel_strategy.kv_world_size
@property
def worker_id(self) -> int:
"""The worker id."""
return self.parallel_strategy.kv_worker_id
@property
def use_mla(self) -> bool:
"""Whether to use MLA."""
return self.parallel_strategy.use_mla
@property
def is_first_rank_of_pp_group(self) -> bool:
"""Is the first rank of the pipeline parallel group."""
return (
self.parallel_strategy.actual_worker_id % self.parallel_strategy.tp_size
== 0
)
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
"""Register the kv caches with LMCache server.
Args:
kv_caches: A dict of kv caches to register. The keys are the
layer names and the values are the corresponding tensors.
"""
# Register kv cache and send the request
self.kv_caches = kv_caches
logger.info("Registering kv caches")
future = send_lmcache_request(
self.mq_client,
RequestType.REGISTER_KV_CACHE,
[self.instance_id, wrap_kv_caches(kv_caches)],
)
future.result()
@_lmcache_nvtx_annotate
def submit_store_request(
self, request_id: str, op: LoadStoreOp, event: torch.cuda.Event
):
"""Submit a KV cache store request to LMCache.
Args:
request_id: The ID of the request
op: The LoadStoreOp describing the store operation.
event: The CUDA event that is recorded after the current
model inference step
"""
if op.block_hashes is not None:
# Hash mode
chunk_hashes = list(
striding_block_hashes(op.block_hashes, self.blocks_in_chunk)
)
keys = [
self._create_hash_key(ch, request_id=request_id) for ch in chunk_hashes
]
else:
# Token mode
assert op.token_ids is not None
keys = [
self._create_key(op.token_ids, op.start, op.end, request_id=request_id)
]
future = send_lmcache_request(
self.mq_client,
RequestType.STORE,
[keys, self.instance_id, op.block_ids, event.ipc_handle()],
).to_cuda_future()
self.store_futures[request_id] = (future, [])
@_lmcache_nvtx_annotate
def submit_retrieve_request(
self, request_id: str, op: LoadStoreOp, event: torch.cuda.Event
):
"""Submit a KV cache retrieve request to LMCache.
Args:
request_id: The ID of the request
op: The LoadStoreOp describing the retrieve operation.
event: The CUDA event that is recorded after the current
model inference step
"""
if op.block_hashes is not None:
# Hash mode
chunk_hashes = list(
striding_block_hashes(op.block_hashes, self.blocks_in_chunk)
)
keys = [
self._create_hash_key(ch, request_id=request_id) for ch in chunk_hashes
]
else:
# Token mode
assert op.token_ids is not None
keys = [
self._create_key(op.token_ids, op.start, op.end, request_id=request_id)
]
future = send_lmcache_request(
self.mq_client,
RequestType.RETRIEVE,
[keys, self.instance_id, op.block_ids, event.ipc_handle()],
).to_cuda_future()
self.retrieve_futures[request_id] = (future, [])
@_lmcache_nvtx_annotate
def batched_submit_store_requests(
self,
request_ids: list[str],
ops: list[LoadStoreOp],
event: torch.cuda.Event,
):
"""Submit a batched store request to LMCache.
Args:
request_ids: The IDs of the requests
ops: The LoadStoreOps describing the store operations. Should have
the same length as request_ids
event: The CUDA event that is recorded after the current
model inference step
"""
all_keys: list[IPCCacheEngineKey] = []
block_ids: list[int] = []
for request_id, op in zip(request_ids, ops, strict=False):
if op.block_hashes is not None:
chunk_hashes = list(
striding_block_hashes(op.block_hashes, self.blocks_in_chunk)
)
keys = [
self._create_hash_key(ch, request_id=request_id)
for ch in chunk_hashes
]
all_keys.extend(keys)
else:
assert op.token_ids is not None
all_keys.append(
self._create_key(
op.token_ids, op.start, op.end, request_id=request_id
)
)
block_ids.extend(op.block_ids)
future = send_lmcache_request(
self.mq_client,
RequestType.STORE,
[
all_keys,
self.instance_id,
block_ids,
event.ipc_handle(),
],
).to_cuda_future()
self.store_futures[request_ids[0]] = (future, list(request_ids[1:]))
@_lmcache_nvtx_annotate
def batched_submit_retrieve_requests(
self,
request_ids: list[str],
ops: list[LoadStoreOp],
event: torch.cuda.Event,
):
"""Submit a batched retrieve request to LMCache.
Args:
request_ids: The IDs of the requests
ops: The LoadStoreOps describing the retrieve operations. Should have
the same length as request_ids
event: The CUDA event that is recorded after the current
model inference step
"""
all_keys: list[IPCCacheEngineKey] = []
block_ids: list[int] = []
for request_id, op in zip(request_ids, ops, strict=False):
if op.block_hashes is not None:
chunk_hashes = list(
striding_block_hashes(op.block_hashes, self.blocks_in_chunk)
)
keys = [
self._create_hash_key(ch, request_id=request_id)
for ch in chunk_hashes
]
all_keys.extend(keys)
else:
assert op.token_ids is not None
all_keys.append(
self._create_key(
op.token_ids, op.start, op.end, request_id=request_id
)
)
block_ids.extend(op.block_ids)
future = send_lmcache_request(
self.mq_client,
RequestType.RETRIEVE,
[
all_keys,
self.instance_id,
block_ids,
event.ipc_handle(),
],
).to_cuda_future()
self.retrieve_futures[request_ids[0]] = (future, list(request_ids[1:]))
@_lmcache_nvtx_annotate
def get_finished(
self, finished_req_ids_from_engine: set[str]
) -> tuple[set[str] | None, set[str] | None]:
"""Check and get the finished store and retrieve requests.
Args:
finished_req_ids_from_engine: the set of request ids that are
reported as finished from the vLLM engine side.
Returns:
A tuple of two sets:
- The first set contains the finished store request ids. The returned
store request ids MUST be seen before in the
`finished_req_ids_from_engine`.
- The second set contains the finished retrieve request ids.
Notes:
When enabling async scheduling in vLLM, the same request ID may appear
multiple times in `finished_req_ids_from_engine`. The adapter should
take care of deduplicating the request IDs and only return the request
IDs that have not been returned before.
"""
finished_stores = set()
finished_retrieves = set()
for request_id, (s_future, other_reqs) in self.store_futures.items():
if not s_future.query():
continue
s_result = s_future.result()
finished_stores.add(request_id)
finished_stores.update(other_reqs)
if not s_result:
# TODO: add error handling here
logger.error(
"Something went wrong when processing the "
"store request for request_id=%s",
request_id,
)
for request_id, (r_future, other_reqs) in self.retrieve_futures.items():
if not r_future.query():
continue
r_result = r_future.result()
finished_retrieves.add(request_id)
finished_retrieves.update(other_reqs)
if not all(r_result):
# TODO: add error handing here
logger.error(
"Something went wrong when processing the "
"retrieve request for request_id=%s, result=%s",
request_id,
r_result,
)
# Remove the finished requests from the tracking dicts
for request_id in finished_stores:
self.store_futures.pop(request_id, None)
for request_id in finished_retrieves:
self.retrieve_futures.pop(request_id, None)
# Update the internal states
self.finished_stores.update(finished_stores)
ret_stores = set()
for req_id in finished_req_ids_from_engine:
if req_id in self.finished_stores or req_id in self.store_futures:
self.previously_finished.add(req_id)
else:
ret_stores.add(req_id)
# Calculate the final finished stores
ret_stores.update(self._update_and_get_finished_store())
return ret_stores, finished_retrieves
def num_blocks_per_chunk(self) -> int:
"""Returns:
The number of vllm blocks in a LMCache data chunk
"""
return self.blocks_in_chunk
def shutdown(self):
"""Shutdown the LMCache MP worker adapter."""
logger.info("Unregistering kv caches")
send_lmcache_request(
self.mq_client, RequestType.UNREGISTER_KV_CACHE, [self.instance_id]
).result()
self.mq_client.close()
# Helper functions
def _update_and_get_finished_store(
self,
) -> set[str]:
"""Converge the internal states about finished stores
and returns the 'safe finished store request ids' back
"""
safe_finished_s = self.finished_stores.intersection(self.previously_finished)
self.finished_stores.difference_update(self.previously_finished)
self.previously_finished.difference_update(safe_finished_s)
return safe_finished_s
def _create_key(
self,
token_ids: list[int],
start: int = 0,
end: int = 0,
request_id: str | None = None,
) -> IPCCacheEngineKey:
"""Convert token IDs to an IPC cache engine key."""
return IPCCacheEngineKey(
model_name=self.model_name,
world_size=self.world_size,
worker_id=self.worker_id,
token_ids=tuple(token_ids),
start=start,
end=end,
request_id=request_id,
)
def _create_hash_key(
self, chunk_hash: bytes, request_id: str | None = None
) -> IPCCacheEngineKey:
"""Create a hash-mode IPC cache engine key."""
return IPCCacheEngineKey(
model_name=self.model_name,
world_size=self.world_size,
worker_id=self.worker_id,
chunk_hash=chunk_hash,
request_id=request_id,
)
@@ -1,134 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Standard
import os
import threading
from typing import TYPE_CHECKING, Union
import torch
from lmcache.logging import init_logger
from lmcache.v1.config import LMCacheEngineConfig as V1Config
if TYPE_CHECKING:
from vllm.config import ModelConfig
from vllm.multimodal.inputs import PlaceholderRange
from vllm.v1.core.sched.output import NewRequestData
from vllm.v1.request import Request
logger = init_logger(__name__)
ENGINE_NAME = "vllm-instance"
# Thread-safe singleton storage
_config_instance: V1Config | None = None
_config_lock = threading.Lock()
def lmcache_get_or_create_config() -> V1Config:
"""Get the LMCache configuration from the environment variable
`LMCACHE_CONFIG_FILE`. If the environment variable is not set, this
function will return the default configuration.
This function is thread-safe and implements singleton pattern,
ensuring the configuration is loaded only once.
"""
global _config_instance
# Double-checked locking for thread-safe singleton
if _config_instance is None:
with _config_lock:
if _config_instance is None: # Check again within lock
LMCacheEngineConfig = V1Config # type: ignore[assignment]
if "LMCACHE_CONFIG_FILE" not in os.environ:
logger.warning(
"No LMCache configuration file is set. Trying to read"
" configurations from the environment variables."
)
logger.warning(
"You can set the configuration file through "
"the environment variable: LMCACHE_CONFIG_FILE"
)
_config_instance = LMCacheEngineConfig.from_env()
else:
config_file = os.environ["LMCACHE_CONFIG_FILE"]
logger.info("Loading LMCache config file %s", config_file)
_config_instance = LMCacheEngineConfig.from_file(config_file)
# Update config from environment variables
_config_instance.update_config_from_env()
return _config_instance
def hex_hash_to_int16(s: str) -> int:
"""Convert a hex hash string to a 16-bit integer."""
return int(s, 16) & 0xFFFF
def apply_mm_hashes_to_token_ids(
token_ids: torch.Tensor,
mm_hashes: list[str],
mm_positions: list["PlaceholderRange"],
) -> torch.Tensor:
"""Overwrite token_ids in-place for multimodal placeholders using
efficient slice assignments.
"""
n = token_ids.size(0)
for hash_str, placeholder in zip(mm_hashes, mm_positions):
start, length = placeholder.offset, placeholder.length
if start >= n:
continue
end = min(start + length, n)
token_ids[start:end] = hex_hash_to_int16(hash_str)
return token_ids
def mla_enabled(model_config: "ModelConfig") -> bool:
return (
hasattr(model_config, "use_mla")
and isinstance(model_config.use_mla, bool)
and model_config.use_mla
)
def extract_mm_features(
request: Union["Request", "NewRequestData"], modify: bool = False
) -> tuple[list[str], list["PlaceholderRange"]]:
"""Normalize multimodal information from a Request into parallel lists.
This helper reads either:
1) `request.mm_features` (objects each exposing `.identifier` and
`.mm_position`), or
2) legacy fields `request.mm_hashes` and `request.mm_positions`.
It returns two equally sized lists: the multimodal hash identifiers and
their corresponding positions. If the request contains no multimodal info,
it returns `([], [])`.
Args:
request (Request): The source object.
modify (bool):
Controls copy semantics for the legacy-path return values.
- If True and legacy fields are used, shallow-copies are returned so
the caller can mutate the lists without affecting `request`.
- If False, the original legacy sequences are returned as-is
(zero-copy); treat them as read-only.
Returns:
tuple[list[str], list[PlaceholderRange]]: (`mm_hashes`, `mm_positions`).
May be `([], [])` when no multimodal data is present.
"""
if getattr(request, "mm_features", None):
mm_hashes, mm_positions = zip(
*((f.identifier, f.mm_position) for f in request.mm_features)
)
return (list(mm_hashes), list(mm_positions))
elif getattr(request, "mm_hashes", None):
if modify:
return (
request.mm_hashes.copy(), # type: ignore
request.mm_positions.copy(), # type: ignore
)
else:
return (request.mm_hashes, request.mm_positions) # type: ignore
else:
return ([], [])
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -1,178 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import socket
import threading
import time
from dataclasses import dataclass
import uvicorn
from fastapi import FastAPI, HTTPException
from pydantic import BaseModel
from vllm.distributed.kv_transfer.kv_connector.utils import EngineId
from vllm.logger import init_logger
WorkerAddr = str
logger = init_logger(__name__)
class RegisterWorkerPayload(BaseModel):
engine_id: EngineId
dp_rank: int
tp_rank: int
pp_rank: int
addr: WorkerAddr
@dataclass
class EngineEntry:
engine_id: EngineId
# {tp_rank: {pp_rank: worker_addr}}
worker_addr: dict[int, dict[int, WorkerAddr]]
class MooncakeBootstrapServer:
"""A centralized registry for prefiller connection info (IP, port, ranks)."""
def __init__(self, host: str, port: int):
self.workers: dict[int, EngineEntry] = {}
self.host = host
self.port = port
self.app = FastAPI()
self._register_routes()
self.server_thread: threading.Thread | None = None
self.server: uvicorn.Server | None = None
self._socket: socket.socket | None = None
self._startup_error: BaseException | None = None
def __del__(self):
self.shutdown()
def _register_routes(self):
# All methods are async. No need to use lock to protect data.
self.app.post("/register")(self.register_worker)
self.app.get("/query", response_model=dict[int, EngineEntry])(self.query)
def start(self, *, timeout: float = 30.0):
if self.server_thread:
if (
self.server_thread.is_alive()
and self.server is not None
and self.server.started
and not self.server.should_exit
and self._socket is not None
and self._socket.fileno() >= 0
):
return
raise RuntimeError(
"Mooncake bootstrap server is still starting or stopping"
)
try:
# Bind in the caller so errors propagate, and retain the listener
# when handing it to Uvicorn, including for automatically chosen ports.
family = socket.AF_INET6 if ":" in self.host else socket.AF_INET
self._socket = socket.create_server((self.host, self.port), family=family)
self.port = self._socket.getsockname()[1]
config = uvicorn.Config(app=self.app, host=self.host, port=self.port)
server = self.server = uvicorn.Server(config=config)
listener = self._socket
self._startup_error = None
def run():
try:
server.run(sockets=[listener])
except BaseException as exc:
self._startup_error = exc
self.server_thread = threading.Thread(
target=run, name="mooncake_bootstrap_server", daemon=True
)
self.server_thread.start()
deadline = time.monotonic() + timeout
while not server.started:
if not self.server_thread.is_alive():
raise RuntimeError(
"Mooncake bootstrap server exited during startup"
) from self._startup_error
if time.monotonic() >= deadline:
raise TimeoutError(
"Mooncake bootstrap server did not start in time"
)
time.sleep(0.01)
except BaseException:
self.shutdown()
raise
logger.info("Mooncake Bootstrap Server started at %s:%d", self.host, self.port)
def shutdown(self):
was_started = self.server is not None and self.server.started
if self.server is not None:
self.server.should_exit = True
if self.server_thread is not None and self.server_thread.ident is not None:
self.server_thread.join(timeout=5)
if self._socket is not None:
self._socket.close()
self._socket = None
if self.server_thread is not None and self.server_thread.is_alive():
logger.warning("Mooncake bootstrap server did not stop in time")
else:
self.server_thread = None
self.server = None
if was_started:
logger.info("Mooncake Bootstrap Server stopped.")
async def register_worker(self, payload: RegisterWorkerPayload):
"""Handles registration of a prefiller worker."""
if payload.dp_rank not in self.workers:
self.workers[payload.dp_rank] = EngineEntry(
engine_id=payload.engine_id,
worker_addr={},
)
dp_entry = self.workers[payload.dp_rank]
if dp_entry.engine_id != payload.engine_id:
raise HTTPException(
status_code=400,
detail=(
f"Engine ID mismatch for dp_rank={payload.dp_rank}: "
f"expected {dp_entry.engine_id}, got {payload.engine_id}"
),
)
if payload.tp_rank not in dp_entry.worker_addr:
dp_entry.worker_addr[payload.tp_rank] = {}
tp_entry = dp_entry.worker_addr[payload.tp_rank]
existing = tp_entry.get(payload.pp_rank)
if existing is not None:
# A client timeout can fire after the server recorded the
# registration so an identical retry must not be an error.
if existing == payload.addr:
return {"status": "ok"}
raise HTTPException(
status_code=400,
detail=(
f"Worker with dp_rank={payload.dp_rank}, "
f"tp_rank={payload.tp_rank}, pp_rank={payload.pp_rank} "
f"is already registered at "
f"{existing}, "
f"but still want to register at {payload.addr}"
),
)
tp_entry[payload.pp_rank] = payload.addr
logger.debug(
"Registered worker: engine_id=%s, dp_rank=%d, tp_rank=%d, pp_rank=%d at %s",
payload.engine_id,
payload.dp_rank,
payload.tp_rank,
payload.pp_rank,
payload.addr,
)
return {"status": "ok"}
async def query(self) -> dict[int, EngineEntry]:
return self.workers
@@ -1,46 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Mooncake requester config helpers."""
from collections.abc import Mapping
from typing import Any
import vllm.envs as envs
from vllm.logger import init_logger
logger = init_logger(__name__)
def normalize_string_override(value: Any) -> str | None:
if not isinstance(value, str):
return None
normalized = value.strip()
return normalized or None
def get_requester_local_hostname(local_ip: str) -> str:
override = normalize_string_override(envs.MOONCAKE_REQUESTER_LOCAL_HOSTNAME)
if override is not None:
return override
return local_ip
def get_configured_preferred_segment(
extra_config: Mapping[str, Any],
) -> str | None:
preferred_segment = normalize_string_override(extra_config.get("preferred_segment"))
if preferred_segment is not None:
return preferred_segment
if extra_config.get("preferred_segment") is not None:
raise ValueError(
"Mooncake preferred_segment override must be a non-empty string"
)
env_value = normalize_string_override(envs.MOONCAKE_PREFERRED_SEGMENT)
if env_value is not None:
logger.info(
"Mooncake preferred_segment from MOONCAKE_PREFERRED_SEGMENT: %s",
env_value,
)
return env_value
return None
@@ -1,146 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Stats container for the Mooncake connector."""
import threading
from dataclasses import dataclass
from typing import Any
import numpy as np
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
KVConnectorStats,
)
# TODO(mooncake-stats): add MooncakePromMetrics (mirror NixlPromMetrics)
# and wire it via MooncakeConnector.build_prom_metrics in a follow-up PR.
@dataclass
class MooncakeKVConnectorStats(KVConnectorStats):
"""Container for Mooncake KV transfer performance metrics.
`_lock` serializes record_* against clone_and_reset so each row's
appends are atomic and column lengths stay aligned. Writers run on
the sender pool / receiver loop / sender loop; reader runs on the
main worker thread.
"""
def __post_init__(self):
self._lock = threading.Lock()
if not self.data:
self.reset()
# threading.Lock is not picklable; strip it from the wire form and
# rebuild a fresh per-process lock on the receiver side.
def __getstate__(self) -> dict[str, Any]:
state = self.__dict__.copy()
state.pop("_lock", None)
return state
def __setstate__(self, state: dict[str, Any]) -> None:
self.__dict__.update(state)
self._lock = threading.Lock()
def reset(self):
self.data: dict[str, list[float | int]] = {
"transfer_duration": [],
"bytes_transferred": [],
"num_descriptors": [],
"num_failed_transfers": [],
"num_failed_recvs": [],
"num_kv_expired_reqs": [],
}
def record_transfer(self, duration_s: float, total_bytes: int, num_descs: int):
with self._lock:
self.data["transfer_duration"].append(duration_s)
self.data["bytes_transferred"].append(total_bytes)
self.data["num_descriptors"].append(num_descs)
# Failure counters store a list of 1s so a future Prom counter can iterate
# with .inc(list_item), mirroring NIXL's NixlPromMetrics.observe.
def record_failed_transfer(self):
with self._lock:
self.data["num_failed_transfers"].append(1)
def record_failed_recv(self):
with self._lock:
self.data["num_failed_recvs"].append(1)
def record_kv_expired_req(self):
with self._lock:
self.data["num_kv_expired_reqs"].append(1)
def clone_and_reset(self) -> "MooncakeKVConnectorStats":
# Copy lists under the lock for length alignment; return a fresh
# instance so the snapshot has its own _lock.
with self._lock:
snapshot_data: dict[str, list[float | int]] = {
k: list(v) for k, v in self.data.items()
}
self.reset()
return MooncakeKVConnectorStats(data=snapshot_data)
def is_empty(self) -> bool:
return (
self.num_successful_transfers == 0
and len(self.data["num_failed_transfers"]) == 0
and len(self.data["num_failed_recvs"]) == 0
and len(self.data["num_kv_expired_reqs"]) == 0
)
def aggregate(self, other: KVConnectorStats) -> KVConnectorStats:
if not other.is_empty():
for k, v in other.data.items():
accumulator = self.data[k]
assert isinstance(accumulator, list)
accumulator.extend(v)
return self
def reduce(self) -> dict[str, int | float]:
num_failed_transfers = len(self.data["num_failed_transfers"])
num_failed_recvs = len(self.data["num_failed_recvs"])
num_kv_expired_reqs = len(self.data["num_kv_expired_reqs"])
if self.num_successful_transfers == 0:
return {
"Num successful transfers": 0,
"Avg xfer time (ms)": 0,
"P90 xfer time (ms)": 0,
"Avg MB per transfer": 0,
"Throughput (MB/s)": 0,
"Avg number of descriptors": 0,
"Num failed transfers": num_failed_transfers,
"Num failed recvs": num_failed_recvs,
"Num KV expired reqs": num_kv_expired_reqs,
}
xfer_time = np.asarray(self.data["transfer_duration"])
mb = np.asarray(self.data["bytes_transferred"]) / 2**20
descs = np.asarray(self.data["num_descriptors"], dtype=np.uint32)
n = len(descs)
assert n == self.num_successful_transfers
total_mb = mb.sum()
avg_mb = total_mb / n
total_time_seconds = xfer_time.sum()
throughput_mb_s = (
(total_mb / total_time_seconds).item() if total_time_seconds > 0 else 0.0
)
return {
"Num successful transfers": n,
"Avg xfer time (ms)": round(xfer_time.mean().item() * 1e3, 3),
"P90 xfer time (ms)": round(np.percentile(xfer_time, 90).item() * 1e3, 3),
"Avg MB per transfer": round(avg_mb.item(), 3),
"Throughput (MB/s)": round(throughput_mb_s, 3),
"Avg number of descriptors": round(descs.mean().item(), 1),
"Num failed transfers": num_failed_transfers,
"Num failed recvs": num_failed_recvs,
"Num KV expired reqs": num_kv_expired_reqs,
}
@property
def num_successful_transfers(self) -> int:
return len(self.data["transfer_duration"])
@@ -1,2 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -1,436 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
#
# Adapted from vllm-project/vllm-ascend
# (vllm_ascend/distributed/kv_transfer/kv_pool/ascend_store/).
"""MooncakeStoreConnector - KV cache connector using MooncakeDistributedStore.
Unlike MooncakeConnector which does direct P2P transfer, this connector
uses MooncakeDistributedStore as a shared KV cache pool. Both producer
and consumer instances read/write KV to/from the store independently,
enabling prefix caching via hash-based deduplication.
"""
from collections import Counter
from collections.abc import Iterable, Sequence
from contextlib import AbstractContextManager
from typing import Any
import torch
from vllm.config import KVTransferConfig, VllmConfig
from vllm.distributed.kv_events import (
BlockStored,
KVCacheEvent,
KVConnectorKVEvents,
)
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorBase_V1,
KVConnectorMetadata,
KVConnectorRole,
KVConnectorTransferResults,
KVConnectorWorkerMetadata,
SupportsHMA,
)
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
KVConnectorPromMetrics,
KVConnectorStats,
PromMetric,
PromMetricT,
)
from vllm.forward_context import ForwardContext
from vllm.logger import init_logger
from vllm.v1.attention.backend import AttentionMetadata
from vllm.v1.core.block_pool import BlockPool
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.outputs import KVConnectorOutput
from vllm.v1.request import Request
from .data import MooncakeStoreConnectorMetadata
from .metrics import MooncakeStoreConnectorStats, MooncakeStorePromMetrics
from .scheduler import MooncakeStoreScheduler
from .worker import MooncakeStoreWorker
logger = init_logger(__name__)
class MooncakeStoreKVEvents(KVConnectorKVEvents):
"""KV event aggregation for MooncakeStoreConnector."""
def __init__(
self,
num_workers: int,
group_tp_replication_factors: Sequence[int] = (1,),
) -> None:
if num_workers <= 0:
raise ValueError("num_workers must be greater than zero.")
if any(factor <= 0 for factor in group_tp_replication_factors):
raise ValueError("TP replication factors must be greater than zero.")
self._event_counter: Counter[KVCacheEvent] = Counter()
self._num_workers = num_workers
self._group_tp_replication_factors = tuple(group_tp_replication_factors)
def add_events(self, events: list[KVCacheEvent]) -> None:
if not isinstance(events, list):
raise TypeError("events must be a list of KVCacheEvent.")
self._event_counter.update(events)
def _replication_factor(self, event: KVCacheEvent) -> int:
if not isinstance(event, BlockStored) or event.group_idx is None:
return 1
return self._group_tp_replication_factors[event.group_idx]
def _is_common_event(self, event: KVCacheEvent, count: int) -> bool:
return count * self._replication_factor(event) >= self._num_workers
def aggregate(self) -> "MooncakeStoreKVEvents":
common_events = self.pop_common_events()
self._event_counter.clear()
self._event_counter.update(common_events)
self._num_workers = 1
return self
def pop_common_events(self) -> list[KVCacheEvent]:
common_events = [
event
for event, count in self._event_counter.items()
if self._is_common_event(event, count)
]
for event in common_events:
del self._event_counter[event]
return common_events
def has_events(self) -> bool:
return bool(self._event_counter)
def increment_workers(self, count: int = 1) -> None:
if count <= 0:
raise ValueError("count must be positive.")
self._num_workers += count
def get_all_events(self) -> list[KVCacheEvent]:
return list(self._event_counter.elements())
def get_number_of_workers(self) -> int:
return self._num_workers
def clear_events(self) -> None:
self._event_counter.clear()
self._num_workers = 1
def __repr__(self) -> str:
return f"<MooncakeStoreKVEvents events={self.get_all_events()}>"
class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA):
"""KV connector using MooncakeDistributedStore as shared KV pool."""
@classmethod
def supports_sleep_mode(cls, kv_transfer_config: KVTransferConfig) -> bool:
return False
@staticmethod
def _validate_kv_cache_config(
vllm_config: VllmConfig, kv_cache_config: KVCacheConfig
) -> None:
from vllm.v1.kv_cache_interface import CrossAttentionSpec, MambaSpec
unsupported: list[str] = []
store_group_ids = kv_cache_config.prefix_cacheable_group_ids
if not store_group_ids:
raise ValueError(
"MooncakeStore requires at least one prefix-cacheable KV cache group"
)
for group_id in store_group_ids:
spec = kv_cache_config.kv_cache_groups[group_id].kv_cache_spec
if isinstance(spec, CrossAttentionSpec):
unsupported.append(f"group {group_id}: CrossAttentionSpec")
if isinstance(spec, MambaSpec) and spec.mamba_cache_mode != "align":
unsupported.append(
f"group {group_id}: mamba_cache_mode="
f"{spec.mamba_cache_mode!r} != 'align'"
)
pcp = vllm_config.parallel_config.prefill_context_parallel_size
if len(store_group_ids) > 1 and pcp > 1:
unsupported.append(f"PCP > 1 (pcp={pcp}) with hybrid attention")
if unsupported:
raise ValueError(
"MooncakeStoreConnector does not support: " + "; ".join(unsupported)
)
def __init__(
self,
vllm_config: VllmConfig,
role: KVConnectorRole,
kv_cache_config: KVCacheConfig | None = None,
):
super().__init__(
vllm_config=vllm_config,
role=role,
kv_cache_config=kv_cache_config, # type: ignore[arg-type]
)
assert vllm_config.kv_transfer_config is not None
assert kv_cache_config is not None, "kv_cache_config is required"
self.kv_role = vllm_config.kv_transfer_config.kv_role
extra_config = vllm_config.kv_transfer_config.kv_connector_extra_config
save_decode_cache = extra_config.get("save_decode_cache", False)
# Capacity-only: contributes its segment to the store pool but transfers
# no KV, so the KV-cache-shape invariants below cannot be reached.
self._capacity_only = (
self.kv_role == "kv_consumer"
and not extra_config.get("enable_lookup", True)
and not save_decode_cache
)
if not self._capacity_only:
self._validate_kv_cache_config(vllm_config, kv_cache_config)
self._kv_cache_config = kv_cache_config
self._kv_cache_events: MooncakeStoreKVEvents | None = None
self.connector_scheduler: MooncakeStoreScheduler | None = None
self.connector_worker: MooncakeStoreWorker | None = None
if role == KVConnectorRole.SCHEDULER:
self.connector_scheduler = MooncakeStoreScheduler(
vllm_config, kv_cache_config
)
else:
self.connector_worker = MooncakeStoreWorker(vllm_config, kv_cache_config)
def shutdown(self):
"""Release connector resources on teardown.
Closes the worker's MooncakeDistributedStore handle so its
TransferEngine and RDMA registrations are released. Invoked from the
engine's explicit shutdown path and as a backstop from ``__del__``;
a no-op on the scheduler role, which holds no store handle.
"""
worker = getattr(self, "connector_worker", None)
if worker is not None:
worker.close()
def __del__(self):
self.shutdown()
# ============================================================
# Scheduler-side methods
# ============================================================
def get_num_new_matched_tokens(
self,
request: Request,
num_computed_tokens: int,
) -> tuple[int | None, bool]:
assert self.connector_scheduler is not None
return self.connector_scheduler.get_num_new_matched_tokens(
request, num_computed_tokens
)
def update_state_after_alloc(
self,
request: Request,
blocks: KVCacheBlocks,
num_external_tokens: int,
):
assert self.connector_scheduler is not None
return self.connector_scheduler.update_state_after_alloc(
request, blocks, num_external_tokens
)
def bind_gpu_block_pool(self, gpu_block_pool: BlockPool) -> None:
assert self.connector_scheduler is not None
self.connector_scheduler.bind_gpu_block_pool(gpu_block_pool)
def has_pending_push_work(self) -> bool:
assert self.connector_scheduler is not None
return self.connector_scheduler.has_pending_push_work()
def build_connector_meta(
self,
scheduler_output: SchedulerOutput,
) -> KVConnectorMetadata:
assert self.connector_scheduler is not None
return self.connector_scheduler.build_connector_meta(scheduler_output)
def build_connector_worker_meta(self) -> KVConnectorWorkerMetadata | None:
assert self.connector_worker is not None
return self.connector_worker.build_connector_worker_meta()
def request_finished(
self,
request: Request,
block_ids: list[int],
) -> tuple[bool, dict[str, Any] | None]:
return self.request_finished_all_groups(request, (block_ids,))
def request_finished_all_groups(
self,
request: Request,
block_ids: tuple[list[int], ...],
) -> tuple[bool, dict[str, Any] | None]:
# An in-flight store job holds its own reference on the blocks it reads,
# so a finishing request never has to defer freeing them.
return False, None
def register_finished_partial_tail(
self,
request: Request,
block_ids: tuple[list[int], ...],
partial_tail_offloads: list[tuple[int, int, int]],
) -> bool:
assert self.connector_scheduler is not None
return self.connector_scheduler.register_finished_partial_tail(
request, block_ids, partial_tail_offloads
)
def reset_cache(self) -> bool | None:
"""Reset the external Mooncake store on prefix-cache reset.
Drains the worker send queue, then runs ``remove_all`` on the
Mooncake master. Caller must first pause generation (e.g.
``pause_generation``) so no new puts are enqueued during drain.
Returns True on ack, False on failure, None for the worker role.
"""
if self.role == KVConnectorRole.SCHEDULER:
assert self.connector_scheduler is not None
# Clear local references to keys we're about to wipe.
self.connector_scheduler.load_specs.clear()
self._kv_cache_events = None
return self.connector_scheduler.reset_store()
return None
def update_connector_output(self, connector_output: KVConnectorOutput):
assert self.connector_scheduler is not None
self.connector_scheduler.update_connector_output(connector_output)
kv_cache_events = connector_output.kv_cache_events
if not kv_cache_events or not isinstance(
kv_cache_events, MooncakeStoreKVEvents
):
return
if self._kv_cache_events is None:
self._kv_cache_events = kv_cache_events
else:
self._kv_cache_events.add_events(kv_cache_events.get_all_events())
def take_events(self) -> Iterable[KVCacheEvent]:
if self._kv_cache_events is not None:
events = self._kv_cache_events.pop_common_events()
if not self._kv_cache_events.has_events():
self._kv_cache_events = None
yield from events
# ============================================================
# Worker-side methods
# ============================================================
def get_mem_pool_context(self) -> AbstractContextManager | None:
"""Return a context manager for the custom MemPool, or None.
Called by the Worker before ``initialize_kv_cache`` so that KV
cache is allocated from the Mooncake-managed pool when
``custom_mem_pool`` is set in ``kv_connector_extra_config``.
"""
if self.connector_worker is None:
return None
return self.connector_worker.get_mem_pool_context()
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
assert self.connector_worker is not None
self.connector_worker.register_kv_caches(kv_caches)
def start_load_kv(self, forward_context: ForwardContext, **kwargs: Any) -> None:
assert self.connector_worker is not None
metadata = self._get_connector_metadata()
assert isinstance(metadata, MooncakeStoreConnectorMetadata)
self.connector_worker.start_load_kv(metadata)
def wait_for_layer_load(self, layer_name: str) -> None:
# No layerwise support - no-op
return
def save_kv_layer(
self,
layer_name: str,
kv_layer: torch.Tensor,
attn_metadata: AttentionMetadata,
**kwargs: Any,
) -> None:
# No layerwise support - no-op
return
def wait_for_save(self):
assert self.connector_worker is not None
metadata = self._get_connector_metadata()
assert isinstance(metadata, MooncakeStoreConnectorMetadata)
self.connector_worker.wait_for_save(metadata)
def get_finished(
self, finished_req_ids: set[str]
) -> tuple[set[str] | None, set[str] | None]:
assert self.connector_worker is not None
metadata = self._get_connector_metadata()
assert isinstance(metadata, MooncakeStoreConnectorMetadata)
return self.connector_worker.get_finished(finished_req_ids, metadata)
def get_transfer_results(
self, finished_req_ids: set[str]
) -> KVConnectorTransferResults:
assert self.connector_worker is not None
metadata = self._get_connector_metadata()
assert isinstance(metadata, MooncakeStoreConnectorMetadata)
return self.connector_worker.get_transfer_results(finished_req_ids, metadata)
def get_block_ids_with_load_errors(self) -> set[int]:
assert self.connector_worker is not None
return self.connector_worker.get_block_ids_with_load_errors()
def get_kv_connector_kv_cache_events(
self,
) -> MooncakeStoreKVEvents | None:
assert self.connector_worker is not None
if (
not self.connector_worker.enable_kv_events
or self.connector_worker.kv_send_thread is None
):
return None
events = self.connector_worker.get_kv_events()
# Empty containers still count this worker toward the poll's quorum.
kv_events = MooncakeStoreKVEvents(
num_workers=1,
group_tp_replication_factors=(
self.connector_worker.group_tp_replication_factors
),
)
kv_events.add_events(events)
return kv_events
def get_kv_connector_stats(self) -> KVConnectorStats | None:
if self.connector_worker is None:
return None
return self.connector_worker.get_kv_connector_stats()
@classmethod
def build_kv_connector_stats(
cls, data: dict[str, Any] | None = None
) -> KVConnectorStats | None:
return (
MooncakeStoreConnectorStats(data=data)
if data is not None
else MooncakeStoreConnectorStats()
)
@classmethod
def build_prom_metrics(
cls,
vllm_config: VllmConfig,
metric_types: dict[type[PromMetric], type[PromMetricT]],
labelnames: list[str],
per_engine_labelvalues: dict[int, list[object]],
) -> KVConnectorPromMetrics:
return MooncakeStorePromMetrics(
vllm_config, metric_types, labelnames, per_engine_labelvalues
)
@@ -1,515 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""External-store cache-hit coordinator for MooncakeStoreConnector."""
from collections.abc import Sequence
from dataclasses import replace
from typing import TYPE_CHECKING, NamedTuple, cast
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
chunk_hashes_for_block_size,
)
from vllm.utils.math_utils import cdiv
from vllm.v1.core.block_pool import BlockPool
from vllm.v1.core.kv_cache_coordinator import replay_boundaries
from vllm.v1.core.kv_cache_utils import (
BlockHash,
KVCacheBlock,
eagle_proof_margin,
resolve_dcp_kv_cache_spec,
)
from vllm.v1.core.single_type_kv_cache_manager import (
SingleTypeKVCacheManager,
)
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheConfig,
KVCacheGroupSpec,
KVCacheSpec,
MambaSpec,
UniformTypeKVCacheSpecs,
)
from vllm.v1.kv_cache_spec_registry import KVCacheSpecRegistry
if TYPE_CHECKING:
from vllm.config import VllmConfig
class StoreSpecGroup(NamedTuple):
spec: KVCacheSpec
group_ids: list[int]
manager_cls: type[SingleTypeKVCacheManager]
use_eagle: bool
class ExternalCachedBlockPool:
"""Duck-typed BlockPool backed by a ``(group_id, hash)`` exists set."""
def __init__(
self,
hash_block_size: int,
exists: set[tuple[int, bytes]] | None = None,
) -> None:
# ``exists=None`` is used on the recv side where hit_length is already
# determined and we just want each spec's manager to apply its own mask.
self._exists = exists
self.hash_block_size = hash_block_size
self.null_block = KVCacheBlock(block_id=0)
# Dummy ID 1 for present block for duck-typing.
self._present_block = KVCacheBlock(block_id=1)
def get_cached_block(
self,
block_hash: BlockHash,
group_ids: list[int],
) -> list[KVCacheBlock] | None:
# Mirrors BlockPool.get_cached_block: hit only when every group_id
# (groups sharing a spec) has the hash cached.
if self._exists is None:
return [self._present_block] * len(group_ids)
h = bytes(block_hash)
if all((g, h) in self._exists for g in group_ids):
return [self._present_block] * len(group_ids)
return None
def contains(self, group_id: int, block_hash: BlockHash) -> bool:
"""Return whether a group has a loadable key for this hash."""
return (
self._exists is not None and (group_id, bytes(block_hash)) in self._exists
)
class MooncakeStoreCoordinator:
"""Mirror of ``HybridKVCacheCoordinator.find_longest_cache_hit`` over an
``ExternalCachedBlockPool``."""
def __init__(
self,
kv_cache_groups: list[KVCacheGroupSpec],
scheduler_block_size: int,
hash_block_size: int,
use_eagle: bool = False,
retention_interval: int | None = None,
enable_partial_hash_hits: bool = False,
) -> None:
# Mirrors core's resolve_kv_cache_block_sizes: the hash unit only has
# to divide groups that participate in prefix caching. Non-shareable
# scratch groups (e.g. GLM-5.3-Flash's kpool tail) are skipped by
# _verify_and_split_kv_cache_groups and never probed for hits.
assert all(
g.kv_cache_spec.block_size % hash_block_size == 0
for g in kv_cache_groups
if g.kv_cache_spec.prefix_cacheable
), "block_size must be divisible by hash_block_size"
assert scheduler_block_size % hash_block_size == 0, (
f"scheduler_block_size ({scheduler_block_size}) must be a multiple of "
f"hash_block_size ({hash_block_size})"
)
assert all(
scheduler_block_size % g.kv_cache_spec.block_size == 0
for g in kv_cache_groups
if g.kv_cache_spec.prefix_cacheable
), "scheduler_block_size must be a multiple of each group's block_size"
self.kv_cache_groups = kv_cache_groups
self.mamba_group_ids = {
group_id
for group_id, group in enumerate(kv_cache_groups)
if isinstance(_unwrap_spec(group.kv_cache_spec), MambaSpec)
}
self.hash_block_size = hash_block_size
self.lcm_block_size = scheduler_block_size
# Core's decision: whether hits land on the hash unit, below the block.
self.enable_partial_hash_hits = enable_partial_hash_hits
self.use_eagle = use_eagle
# Mirror vLLM core's KVCacheCoordinator.retention_interval.
self.retention_interval = retention_interval
self._verify_and_split_kv_cache_groups()
@classmethod
def from_kv_cache_config(
cls,
kv_cache_config: KVCacheConfig,
vllm_config: "VllmConfig",
scheduler_block_size: int,
hash_block_size: int,
) -> "MooncakeStoreCoordinator":
"""Build the coordinator the connector's scheduler and workers share.
Specs are DCP-resolved, so a block spans the tokens of one block-table
entry. Partial hits follow core's hit alignment, which checks every KV
cache group and its manager, not only the groups the store transfers;
unset (a config the engine core did not resolve) keeps hits on the
block.
"""
dcp_world_size = vllm_config.parallel_config.decode_context_parallel_size
spec_config = vllm_config.speculative_config
alignment = kv_cache_config.cache_hit_alignment_tokens
return cls(
[
replace(
group,
kv_cache_spec=resolve_dcp_kv_cache_spec(
group.kv_cache_spec, dcp_world_size
),
)
for group in kv_cache_config.prefix_cacheable_groups
],
scheduler_block_size,
hash_block_size,
use_eagle=spec_config is not None and spec_config.use_eagle_block_drop(),
retention_interval=kv_cache_config.prefix_cache_retention_interval,
enable_partial_hash_hits=(
alignment is not None and alignment < scheduler_block_size
),
)
def align_lookup_length(self, length: int) -> int:
alignment = (
self.hash_block_size
if self.enable_partial_hash_hits
else self.lcm_block_size
)
return length // alignment * alignment
def _verify_and_split_kv_cache_groups(self) -> None:
"""Mirrors KVCacheCoordinator.verify_and_split_kv_cache_groups but
dispatches via spec_manager_map (we don't allocate managers).
"""
attention_groups: list[StoreSpecGroup] = []
for i, g in enumerate(self.kv_cache_groups):
# Skip groups that opt out of prefix caching (e.g. GLM-5.3-Flash
# kpool tail): per-request scratch, never shareable, so they must
# not participate in hit lookup. Mirrors core's
# KVCacheCoordinator.verify_and_split_kv_cache_groups.
if not g.kv_cache_spec.prefix_cacheable:
continue
spec = _unwrap_spec(g.kv_cache_spec)
manager_cls = KVCacheSpecRegistry.get_manager_class(spec)
assert manager_cls is not None, (
f"No manager registered for KVCacheSpec {spec}"
)
for idx, group in enumerate(attention_groups):
if group.spec == spec:
assert manager_cls is group.manager_cls
group.group_ids.append(i)
if g.is_eagle_group and not group.use_eagle:
attention_groups[idx] = group._replace(use_eagle=True)
break
else:
attention_groups.append(
StoreSpecGroup(spec, [i], manager_cls, g.is_eagle_group)
)
# Full attention first (matches upstream convergence ordering).
attention_groups.sort(key=lambda g: not isinstance(g.spec, FullAttentionSpec))
# Conservatively flag all groups when use_eagle is set but none is flagged.
if self.use_eagle and not any(g.use_eagle for g in attention_groups):
attention_groups = [g._replace(use_eagle=True) for g in attention_groups]
self.attention_groups = attention_groups
# Per-group eagle bits. SpecGroup carries use_eagle for the whole
# merged spec group, so the per-group store/lookup masks agree with
# the merged-group hit check, which applies the eagle drop to every
# group sharing the spec.
self.eagle_group_ids = {
gid for g in attention_groups if g.use_eagle for gid in g.group_ids
}
self.eagle_proof_margin_by_group = {
gid: eagle_proof_margin(
group.spec.block_size,
self.hash_block_size,
self.enable_partial_hash_hits
and group.manager_cls.supports_fine_grained_hash_lookup,
)
for group in attention_groups
if group.use_eagle and not isinstance(group.spec, MambaSpec)
for gid in group.group_ids
}
def find_longest_cache_hit(
self,
block_hashes: Sequence[BlockHash],
max_length: int,
cached_block_pool: ExternalCachedBlockPool,
*,
apply_eagle: bool = True,
) -> tuple[tuple[list[bool], ...], int]:
"""Returns ``(load_mask_per_group, hit_length)``. ``mask[g][i]`` is True iff
group ``g`` populates chunk ``i`` locally (e.g. SWA and Mamba tail-only);
recv-side callers skip False slots.
``apply_eagle`` controls whether the per-spec ``use_eagle`` last-block
pop is applied. Lookup callers want it (the drafter requires recomputing
the last block); per-chunk mask callers must not, because ``token_len``
already reflects the eagle-pruned hit length and a second pop would
leave the trailing block unloaded.
"""
blocks_per_group, hit_length = self._find_hit_blocks(
block_hashes, max_length, cached_block_pool, apply_eagle=apply_eagle
)
masks = tuple(
[blk is not cached_block_pool.null_block for blk in blocks]
for blocks in blocks_per_group
)
return masks, hit_length
def load_mask(
self,
block_hashes: Sequence[BlockHash],
token_len: int,
) -> tuple[list[bool], ...]:
"""Per-group load masks: ``mask[g][i]`` is True iff group ``g``'s
spec would populate chunk ``i`` locally at length ``token_len``
(e.g. SWA / Mamba tail-only).
"""
# ``apply_eagle=False`` because ``token_len`` is already the
# eagle-pruned hit length returned by ``client.lookup``. Re-applying
# the pop here would shorten the mask by one extra block; the recv
# thread would then silently skip the trailing chunk yielded by
# ``db.process_tokens`` and leave that block uninitialized in the
# local KV pool.
masks, _ = self.find_longest_cache_hit(
block_hashes,
token_len,
ExternalCachedBlockPool(self.hash_block_size),
apply_eagle=False,
)
return masks
def store_mask(
self,
aligned_token_len: int,
start_token: int = 0,
num_prompt_tokens: int | None = None,
) -> tuple[list[bool] | None, ...]:
"""Per-group store masks for the suffix starting at ``start_token``.
``mask[g][i]`` is True iff the i-th chunk of group ``g`` *after*
``start_token`` should be written to the store so a future cache hit
can consume it. ``None`` is the all-True sentinel for the suffix.
Reuses the engine's ``SingleTypeKVCacheManager.reachable_block_mask``
so the store retains exactly the blocks the local prefix cache would.
Mamba groups are always all-False: the normal save resolves blocks
positionally from the connector's append-only block-ID snapshot, but
an align-mode mamba block table is not append-only (interior state
blocks are nulled/freed, and speculative decoding relocates the spec
blocks in place), so a positional read may hit a null, freed, or live
speculative-state block and persist wrong bytes under a valid prefix
hash. Mamba state is persisted only through the connector-pinned exact
block hand-off path
(``SchedulerOutput.kv_connector_block_state.boundary_state_offloads``).
"""
return self._reachable_masks(
aligned_token_len,
start_token,
retention_interval=self.retention_interval,
num_prompt_tokens=num_prompt_tokens,
exclude_mamba=True,
)
def lookup_mask(
self,
aligned_token_len: int,
) -> tuple[list[bool] | None, ...]:
"""Per-group lookup masks.
``mask[g][i]`` is True iff chunk ``i`` of group ``g`` should be
looked up as an aligned hit boundary. ``None`` is the all-True
sentinel.
"""
return self._reachable_masks(
aligned_token_len,
0,
retention_interval=None,
num_prompt_tokens=None,
)
def _reachable_masks(
self,
aligned_token_len: int,
start_token: int,
*,
retention_interval: int | None,
num_prompt_tokens: int | None,
exclude_mamba: bool = False,
) -> tuple[list[bool] | None, ...]:
mask_alignment = (
self.hash_block_size
if self.enable_partial_hash_hits
else self.lcm_block_size
)
assert aligned_token_len % mask_alignment == 0, (
f"aligned_token_len ({aligned_token_len}) must be a multiple of "
f"{mask_alignment}"
)
reachable_boundaries = (
()
if num_prompt_tokens is None
else replay_boundaries(
num_prompt_tokens,
self.lcm_block_size,
use_eagle=bool(self.eagle_group_ids),
)
)
masks: list[list[bool] | None] = []
for g_idx, g in enumerate(self.kv_cache_groups):
spec = _unwrap_spec(g.kv_cache_spec)
end_chunk = aligned_token_len // spec.block_size
start_chunk = min(end_chunk, max(0, cdiv(start_token, spec.block_size)))
if exclude_mamba and isinstance(spec, MambaSpec):
masks.append([False] * (end_chunk - start_chunk))
continue
manager_cls = KVCacheSpecRegistry.get_manager_class(spec)
assert manager_cls is not None
use_eagle = g_idx in self.eagle_group_ids
mask = manager_cls.reachable_block_mask(
start_block=start_chunk,
end_block=end_chunk,
alignment_tokens=self.lcm_block_size,
kv_cache_spec=spec,
use_eagle=use_eagle,
retention_interval=retention_interval,
reachable_boundaries=reachable_boundaries,
# ``spec`` is already DCP-resolved (worker.py applies
# resolve_dcp_kv_cache_spec) and ``end_chunk`` is indexed in
# that scaled block size, so the mask must not scale again.
dcp_world_size=1,
)
if mask is not None:
assert len(mask) == end_chunk - start_chunk
masks.append(mask)
return tuple(masks)
def block_hashes_for_spec(
self, block_hashes: Sequence[BlockHash], spec: KVCacheSpec
) -> Sequence[BlockHash]:
return chunk_hashes_for_block_size(
block_hashes, self.hash_block_size, spec.block_size
)
def _find_hit_blocks(
self,
block_hashes: Sequence[BlockHash],
max_length: int,
cached_block_pool: ExternalCachedBlockPool,
*,
apply_eagle: bool = True,
) -> tuple[tuple[list[KVCacheBlock], ...], int]:
"""Mirrors HybridKVCacheCoordinator.find_longest_cache_hit but
dispatches via spec_manager_map (we don't allocate managers).
When ``apply_eagle`` is False, ignore each group's ``use_eagle`` —
used by ``load_mask`` to avoid popping a second block on top of the
one already removed by the lookup.
"""
alignment_tokens = (
self.hash_block_size
if self.enable_partial_hash_hits
else self.lcm_block_size
)
if len(self.attention_groups) == 1:
spec, group_ids, manager_cls, group_eagle = self.attention_groups[0]
hit_blocks, hit_length = manager_cls.find_longest_cache_hit(
block_hashes=block_hashes, # type: ignore[arg-type]
max_length=max_length,
kv_cache_group_ids=group_ids,
block_pool=cast(BlockPool, cached_block_pool),
kv_cache_spec=spec,
drop_eagle_block=apply_eagle and group_eagle,
alignment_tokens=alignment_tokens,
)
num_groups = len(self.kv_cache_groups)
blocks_by_group: list[list[KVCacheBlock]] = [[] for _ in range(num_groups)]
for gid, blks in zip(group_ids, hit_blocks, strict=True):
blocks_by_group[gid] = blks
return tuple(blocks_by_group), hit_length
num_groups = len(self.kv_cache_groups)
hit_length = max_length
hit_blocks_by_group: list[list[KVCacheBlock] | None] = [None] * num_groups
hit_length_by_group: list[int] = [0] * num_groups
is_simple_hybrid = len(self.attention_groups) == 2 and isinstance(
self.attention_groups[0].spec, FullAttentionSpec
)
eagle_verified: set[int] = set()
while True:
curr_hit_length = hit_length
for idx, (spec, group_ids, manager_cls, group_eagle) in enumerate(
self.attention_groups
):
first_group_id = group_ids[0]
cached = hit_blocks_by_group[first_group_id]
if isinstance(spec, FullAttentionSpec) and cached is not None:
curr_hit_length = min(
curr_hit_length, hit_length_by_group[first_group_id]
)
continue
drop_eagle_block = (
apply_eagle and group_eagle and idx not in eagle_verified
)
_max_length = curr_hit_length
# No eagle peek margin for a recurrent (Mamba) group: its finder
# never drops a block, so a widened bound would match past the
# attention-verified hit and resume from speculative state (#43559).
if drop_eagle_block and not isinstance(spec, MambaSpec):
eagle_margin = eagle_proof_margin(
spec.block_size,
self.hash_block_size,
self.enable_partial_hash_hits
and manager_cls.supports_fine_grained_hash_lookup,
)
_max_length = min(
curr_hit_length + eagle_margin,
len(block_hashes) * self.hash_block_size,
)
hit_blocks, _new_hit_length = manager_cls.find_longest_cache_hit(
block_hashes=block_hashes, # type: ignore[arg-type]
max_length=_max_length,
kv_cache_group_ids=group_ids,
block_pool=cast(BlockPool, cached_block_pool),
kv_cache_spec=spec,
drop_eagle_block=drop_eagle_block,
alignment_tokens=alignment_tokens,
)
if drop_eagle_block:
eagle_verified.add(idx)
elif _new_hit_length < curr_hit_length:
eagle_verified.clear()
curr_hit_length = _new_hit_length
for gid, blocks in zip(group_ids, hit_blocks, strict=True):
hit_blocks_by_group[gid] = blocks
hit_length_by_group[gid] = _new_hit_length
if curr_hit_length >= hit_length:
break
hit_length = curr_hit_length
if is_simple_hybrid:
break
# Truncate full-attention hit_blocks to final converged length;
# other specs already trim themselves inside their hit logic. cdiv keeps
# the partial tail block when hit_length is not block-aligned.
for group in self.attention_groups:
if not isinstance(group.spec, FullAttentionSpec):
continue
num_blocks = cdiv(hit_length, group.spec.block_size)
for group_id in group.group_ids:
full_blks = hit_blocks_by_group[group_id]
assert full_blks is not None
del full_blks[num_blocks:]
hit_length_by_group[group_id] = hit_length
return (
tuple(blks if blks is not None else [] for blks in hit_blocks_by_group),
hit_length,
)
def _unwrap_spec(spec: KVCacheSpec) -> KVCacheSpec:
if isinstance(spec, UniformTypeKVCacheSpecs):
return next(iter(spec.kv_cache_specs.values()))
return spec
@@ -1,898 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
#
# Adapted from vllm-project/vllm-ascend
# (vllm_ascend/distributed/kv_transfer/kv_pool/ascend_store/).
"""Data classes for MooncakeStoreConnector."""
from collections.abc import Iterable, Sequence
from dataclasses import dataclass, field
from typing import NamedTuple, cast
import numpy as np
import torch
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorMetadata,
KVConnectorWorkerMetadata,
)
from vllm.logger import init_logger
from vllm.utils.math_utils import cdiv
from vllm.utils.torch_utils import is_non_overlapping_and_dense
from vllm.v1.core.kv_cache_utils import (
BlockHash,
BlockHashListWithBlockSize,
)
logger = init_logger(__name__)
class BlobBlockHashes(Sequence[BlockHash]):
"""Lazy view over a flat buffer of fixed-size block hashes to avoid the overhead
of materializing all hashes upfront.
"""
def __init__(self, blob: memoryview, hash_len: int):
self._blob = blob
self._hash_len = hash_len
self._n = len(blob) // hash_len if hash_len else 0
def __len__(self) -> int:
return self._n
def __getitem__(self, idx):
if isinstance(idx, slice):
return [self[i] for i in range(*idx.indices(self._n))]
if idx < 0:
idx += self._n
if not 0 <= idx < self._n:
raise IndexError(idx)
off = idx * self._hash_len
return BlockHash(self._blob[off : off + self._hash_len])
class _CompactChunkHashList(BlockHashListWithBlockSize):
"""View that keys each ``block_size`` chunk by the last constituent
``hash_block_size`` hash instead of concatenating all of them.
The engine chains block hashes (each hash folds in the previous one), so the
final sub-block hash of a chunk already uniquely identifies the whole chunk
and its prefix. Using it keeps a Mooncake key at a single hash digest
regardless of the ``block_size`` / ``hash_block_size`` ratio, instead of
growing the key linearly with it (e.g. 64x for ``block_size=256``,
``hash_block_size=4``).
"""
def __init__(
self,
block_hashes: Sequence[BlockHash],
hash_block_size: int,
target_block_size: int,
):
# Accept any indexable sequence (e.g. the lazy ``BlobBlockHashes``), not
# just ``list``; the base only indexes/sizes it.
assert target_block_size % hash_block_size == 0
self.block_hashes = block_hashes # type: ignore[assignment]
self.scale_factor = target_block_size // hash_block_size
def _get_value_at(self, idx: int) -> BlockHash:
return self.block_hashes[idx * self.scale_factor + self.scale_factor - 1]
def chunk_hashes_for_block_size(
block_hashes: Sequence[BlockHash],
hash_block_size: int,
block_size: int,
) -> Sequence[BlockHash]:
"""Map ``hash_block_size``-granular block hashes to one compact hash per
``block_size`` chunk (the chunk's last sub-hash). Returns ``block_hashes``
unchanged when the two sizes are equal.
"""
if block_size == hash_block_size:
return block_hashes
# Structurally a Sequence[BlockHash] (indexable + sized); the base class
# just isn't declared as one.
return cast(
"Sequence[BlockHash]",
_CompactChunkHashList(block_hashes, hash_block_size, block_size),
)
@dataclass
class KeyMetadata:
"""Metadata for constructing pool keys."""
model_name: str
tp_rank: int
pcp_rank: int
dcp_rank: int
pp_rank: int
group_id: int = 0
# Optional namespace prepended to every key. Lets separate deployments
# share one Mooncake master without colliding on identical block hashes.
# Empty (the default) keeps keys byte-identical to the unprefixed format.
cache_prefix: str = ""
# Complete namespace for an opt-in Store payload format. Empty keeps
# historical keys byte-identical.
store_namespace: str = ""
@dataclass(order=True)
class PoolKey:
"""Key for addressing KV cache blocks in the distributed store."""
key_metadata: KeyMetadata
chunk_hash: str
def __hash__(self):
return hash(
(
self.key_metadata.cache_prefix,
self.key_metadata.model_name,
self.key_metadata.store_namespace,
self.key_metadata.tp_rank,
self.key_metadata.pcp_rank,
self.key_metadata.dcp_rank,
self.key_metadata.pp_rank,
self.key_metadata.group_id,
self.chunk_hash,
)
)
@staticmethod
def build_prefix(
key_metadata: KeyMetadata,
*,
tp_rank: int | None = None,
pcp_rank: int | None = None,
dcp_rank: int | None = None,
pp_rank: int | None = None,
) -> str:
"""Return the stable prefix for a Mooncake pool key."""
prefix = f"{key_metadata.cache_prefix}@" if key_metadata.cache_prefix else ""
return (
f"{prefix}"
f"{key_metadata.model_name}"
f"{key_metadata.store_namespace}"
f"@tp_rank:{key_metadata.tp_rank if tp_rank is None else tp_rank}"
f"@pcp{key_metadata.pcp_rank if pcp_rank is None else pcp_rank}"
f"@dcp{key_metadata.dcp_rank if dcp_rank is None else dcp_rank}"
f"@pp_rank:{key_metadata.pp_rank if pp_rank is None else pp_rank}"
f"@group:{key_metadata.group_id}"
)
@staticmethod
def build_key_string(key_prefix: str, chunk_hash: str) -> str:
return f"{key_prefix}@{chunk_hash}"
def to_string(self) -> str:
return self.build_key_string(
self.build_prefix(self.key_metadata), self.chunk_hash
)
StoreShardId = int
# Physical ranks encoded in a rank-local key: (TP, PCP, DCP, PP).
RankNamespace = tuple[int, int, int, int]
class StoreLayout:
"""Store payload layout for one KV-cache group on one local TP rank."""
def __init__(
self,
metadata: KeyMetadata,
block_size: int,
hash_block_size: int,
) -> None:
self.metadata = metadata
self.block_size = block_size
self.hash_block_size = hash_block_size
@property
def local_shard_ids(self) -> tuple[StoreShardId, ...]:
"""Store objects contributed or consumed by this local rank."""
raise NotImplementedError
def key_for(self, shard_id: StoreShardId, chunk_hash: BlockHash) -> str:
raise NotImplementedError
def lookup_key_prefixes(
self,
rank_namespaces: Sequence[RankNamespace],
) -> tuple[str, ...]:
"""Prefixes that must exist for a logical block to be reusable."""
raise NotImplementedError
def register_kv_caches(
self,
kv_caches: Sequence[torch.Tensor],
num_blocks: int,
) -> None:
"""Build descriptors from the local KV cache tensors."""
raise NotImplementedError
def prepare_values(
self,
chunks: Sequence[tuple[int, int]],
block_ids: list[int],
shard_ids: Sequence[StoreShardId],
) -> tuple[list[list[int]], list[list[int]], list[int]]:
"""Build Store multi-buffer descriptors for logical chunks."""
raise NotImplementedError
class RankLocalStoreLayout(StoreLayout):
"""Historical rank-local Store payload layout."""
def __init__(
self,
metadata: KeyMetadata,
block_size: int,
hash_block_size: int,
) -> None:
super().__init__(metadata, block_size, hash_block_size)
self._key_prefix = PoolKey.build_prefix(metadata)
self.kv_caches_base_addr: list[int] = []
self.block_len: list[int] = []
@property
def local_shard_ids(self) -> tuple[StoreShardId, ...]:
return (0,)
def key_for(self, shard_id: StoreShardId, chunk_hash: BlockHash) -> str:
assert shard_id == 0
return PoolKey.build_key_string(self._key_prefix, chunk_hash.hex())
def lookup_key_prefixes(
self,
rank_namespaces: Sequence[RankNamespace],
) -> tuple[str, ...]:
return tuple(
PoolKey.build_prefix(
self.metadata,
tp_rank=tp_rank,
pcp_rank=pcp_rank,
dcp_rank=dcp_rank,
pp_rank=pp_rank,
)
for tp_rank, pcp_rank, dcp_rank, pp_rank in rank_namespaces
)
def set_kv_caches_base_addr(self, base_addrs: list[int]) -> None:
self.kv_caches_base_addr = base_addrs
def set_block_len(self, block_lens: list[int]) -> None:
self.block_len = block_lens
def register_kv_caches(
self,
kv_caches: Sequence[torch.Tensor],
num_blocks: int,
) -> None:
seen_region_ptrs: set[int] = set()
base_addrs: list[int] = []
block_lens: list[int] = []
for cache in kv_caches:
storage = cache.untyped_storage()
storage_addr = storage.data_ptr()
storage_len = storage.nbytes()
if not is_non_overlapping_and_dense(cache[0]):
for head_idx in range(cache.shape[1]):
head_cache = cache[:, head_idx]
assert is_non_overlapping_and_dense(head_cache[0])
region_addr = head_cache.data_ptr()
if region_addr in seen_region_ptrs:
continue
seen_region_ptrs.add(region_addr)
base_addrs.append(region_addr)
block_lens.append(head_cache.stride(0) * head_cache.element_size())
elif cache.stride(0) * cache.element_size() * num_blocks == storage_len:
if storage_addr in seen_region_ptrs:
continue
seen_region_ptrs.add(storage_addr)
base_addrs.append(storage_addr)
block_lens.append(storage_len // num_blocks)
else:
region_addr = cache.data_ptr()
if region_addr in seen_region_ptrs:
continue
seen_region_ptrs.add(region_addr)
base_addrs.append(region_addr)
block_lens.append(cache.stride(0) * cache.element_size())
self.kv_caches_base_addr = base_addrs
self.block_len = block_lens
def prepare_values(
self,
chunks: Sequence[tuple[int, int]],
block_ids: list[int],
shard_ids: Sequence[StoreShardId],
) -> tuple[list[list[int]], list[list[int]], list[int]]:
if not chunks:
return [], [], []
assert len(chunks) == len(shard_ids) and all(
shard_id == 0 for shard_id in shard_ids
)
base = np.asarray(self.kv_caches_base_addr, dtype=np.int64)
length = len(self.block_len)
blen = np.asarray(
[self.block_len[i % length] for i in range(base.shape[0])],
dtype=np.int64,
)
n = len(chunks)
starts = np.fromiter((c[0] for c in chunks), dtype=np.int64, count=n)
spans = np.fromiter((c[1] for c in chunks), dtype=np.int64, count=n) - starts
assert np.all(spans % self.hash_block_size == 0)
bids = np.fromiter(
(block_ids[i] for i in (starts // self.block_size).tolist()),
dtype=np.int64,
count=n,
)
addrs = base[None, :] + bids[:, None] * blen[None, :]
block_counts = (spans + self.block_size - 1) // self.block_size
sizes = blen[None, :] * block_counts[:, None]
return addrs.tolist(), sizes.tolist(), bids.tolist()
def prepare_value_for_block(self, block_id: int) -> tuple[list[int], list[int]]:
length = len(self.block_len)
return (
[
base_addr + block_id * self.block_len[index % length]
for index, base_addr in enumerate(self.kv_caches_base_addr)
],
[
self.block_len[index % length]
for index in range(len(self.kv_caches_base_addr))
],
)
class TPShardedStoreLayout(StoreLayout):
"""Store layout shared by divisible TP sizes."""
store_format: str
@classmethod
def shared_namespace(cls, store_tp_size: int, pp_size: int) -> str:
return (
f"@store_tp:{store_tp_size}@store_pp:{pp_size}"
f"@store_format:{cls.store_format}"
)
def __init__(
self,
metadata: KeyMetadata,
block_size: int,
hash_block_size: int,
local_tp_size: int,
store_tp_size: int,
tp_rank: int,
num_kv_heads: int,
) -> None:
super().__init__(metadata, block_size, hash_block_size)
self.shards_per_rank = store_tp_size // local_tp_size
first_shard = tp_rank * self.shards_per_rank
self.store_shard_ids = tuple(
range(first_shard, first_shard + self.shards_per_rank)
)
self.store_tp_size = store_tp_size
self.heads_per_store_shard = num_kv_heads // store_tp_size
self.local_num_kv_heads = num_kv_heads // local_tp_size
self._shard_key_prefixes = {
shard_id: PoolKey.build_prefix(metadata, tp_rank=shard_id)
for shard_id in self.store_shard_ids
}
self._shard_addr_bases = np.empty((self.shards_per_rank, 0), dtype=np.uint64)
self._shard_block_strides = np.empty((self.shards_per_rank, 0), dtype=np.uint64)
self._shard_sizes = np.empty((self.shards_per_rank, 0), dtype=np.uint64)
@property
def local_shard_ids(self) -> tuple[StoreShardId, ...]:
return self.store_shard_ids
def key_for(self, shard_id: StoreShardId, chunk_hash: BlockHash) -> str:
return PoolKey.build_key_string(
self._shard_key_prefixes[shard_id], chunk_hash.hex()
)
def lookup_key_prefixes(
self,
rank_namespaces: Sequence[RankNamespace],
) -> tuple[str, ...]:
pp_ranks = sorted({namespace[3] for namespace in rank_namespaces})
return tuple(
PoolKey.build_prefix(self.metadata, tp_rank=shard_id, pp_rank=pp_rank)
for shard_id in range(self.store_tp_size)
for pp_rank in pp_ranks
)
def _set_segment_templates(
self, templates: Sequence[tuple[list[int], list[int], list[int]]]
) -> None:
self._shard_addr_bases = np.asarray(
[template[0] for template in templates], dtype=np.uint64
)
self._shard_block_strides = np.asarray(
[template[1] for template in templates], dtype=np.uint64
)
self._shard_sizes = np.asarray(
[template[2] for template in templates], dtype=np.uint64
)
def _has_packed_strides(
self, head_stride: int, token_stride: int, content_bytes: int
) -> bool:
raise NotImplementedError
def _shard_segments(
self, head_start: int, head_stride: int, token_stride: int, content_bytes: int
) -> Iterable[tuple[int, int]]:
raise NotImplementedError
def register_kv_caches(
self,
kv_caches: Sequence[torch.Tensor],
num_blocks: int,
) -> None:
templates: list[tuple[list[int], list[int], list[int]]] = [
([], [], []) for _ in range(self.shards_per_rank)
]
for cache in kv_caches:
if cache.ndim != 4 or tuple(cache.shape[:3]) != (
num_blocks,
self.local_num_kv_heads,
self.block_size,
):
raise ValueError(
"TP-shared Mooncake store requires packed KV caches with "
"logical shape (num_blocks, local_kv_heads, block_size, content)"
)
element_size = cache.element_size()
block_stride, head_stride, token_stride, content_stride = (
stride * element_size for stride in cache.stride()
)
content_bytes = cache.shape[3] * element_size
if content_stride != element_size or not self._has_packed_strides(
head_stride, token_stride, content_bytes
):
raise ValueError(
"TP-shared Mooncake store requires packed "
f"{self.store_format} KV layout"
)
for local_shard, (addr_bases, block_strides, sizes) in enumerate(templates):
head_start = local_shard * self.heads_per_store_shard
for offset, size in self._shard_segments(
head_start, head_stride, token_stride, content_bytes
):
addr_bases.append(cache.data_ptr() + offset)
block_strides.append(block_stride)
sizes.append(size)
self._set_segment_templates(templates)
def prepare_values(
self,
chunks: Sequence[tuple[int, int]],
block_ids: list[int],
shard_ids: Sequence[StoreShardId],
) -> tuple[list[list[int]], list[list[int]], list[int]]:
if not chunks:
return [], [], []
if len(chunks) != len(shard_ids):
raise ValueError("Each Store chunk must have one shard ID")
count = len(chunks)
starts = np.fromiter(
(chunk[0] for chunk in chunks), dtype=np.int64, count=count
)
ends = np.fromiter((chunk[1] for chunk in chunks), dtype=np.int64, count=count)
if np.any(ends - starts != self.block_size):
raise ValueError("TP-shared Mooncake store requires full KV blocks")
first_shard = self.store_shard_ids[0]
local_shards = np.fromiter(
(shard_id - first_shard for shard_id in shard_ids),
dtype=np.int64,
count=count,
)
if np.any(local_shards < 0) or np.any(local_shards >= self.shards_per_rank):
raise ValueError("Store shard is not owned by this TP rank")
chunk_block_ids = np.fromiter(
(block_ids[index] for index in (starts // self.block_size).tolist()),
dtype=np.uint64,
count=count,
)
addr_bases = self._shard_addr_bases[local_shards]
block_strides = self._shard_block_strides[local_shards]
addrs = addr_bases + chunk_block_ids[:, None] * block_strides
sizes = self._shard_sizes[local_shards]
return addrs.tolist(), sizes.tolist(), chunk_block_ids.tolist()
class LBHNCStoreLayout(TPShardedStoreLayout):
"""Native head-major layout shared by divisible TP sizes."""
store_format = "tp_shared_lbhnc"
def _has_packed_strides(
self, head_stride: int, token_stride: int, content_bytes: int
) -> bool:
return (
token_stride == content_bytes
and head_stride == self.block_size * content_bytes
)
def _shard_segments(
self, head_start: int, head_stride: int, token_stride: int, content_bytes: int
) -> Iterable[tuple[int, int]]:
yield head_start * head_stride, self.heads_per_store_shard * head_stride
class LBNHCStoreLayout(TPShardedStoreLayout):
"""Native token-major layout shared by divisible TP sizes."""
store_format = "tp_shared_lbnhc"
def _has_packed_strides(
self, head_stride: int, token_stride: int, content_bytes: int
) -> bool:
return (
head_stride == content_bytes
and token_stride == self.local_num_kv_heads * content_bytes
)
def _shard_segments(
self, head_start: int, head_stride: int, token_stride: int, content_bytes: int
) -> Iterable[tuple[int, int]]:
for token_idx in range(self.block_size):
yield (
token_idx * token_stride + head_start * head_stride,
self.heads_per_store_shard * content_bytes,
)
class ChunkedTokenDatabase:
"""Enumerates logical token chunks and their hashes."""
def __init__(
self,
metadata: KeyMetadata,
block_size: int,
hash_block_size: int | None = None,
store_layout: StoreLayout | None = None,
):
self.metadata = metadata
self.block_size = block_size
self.hash_block_size = hash_block_size or block_size
if self.block_size % self.hash_block_size != 0:
raise ValueError(
f"block_size ({self.block_size}) must be a multiple of "
f"hash_block_size ({self.hash_block_size})"
)
self.store_layout = store_layout or RankLocalStoreLayout(
metadata, block_size, self.hash_block_size
)
@property
def kv_caches_base_addr(self) -> list[int]:
return self._rank_local_layout().kv_caches_base_addr
@property
def block_len(self) -> list[int]:
return self._rank_local_layout().block_len
def _rank_local_layout(self) -> RankLocalStoreLayout:
if not isinstance(self.store_layout, RankLocalStoreLayout):
raise RuntimeError("This operation requires a rank-local Store layout")
return self.store_layout
def key_for(self, chunk_hash: BlockHash) -> str:
return self._rank_local_layout().key_for(0, chunk_hash)
def set_kv_caches_base_addr(self, kv_caches_base_addr: list[int]):
self._rank_local_layout().set_kv_caches_base_addr(kv_caches_base_addr)
def set_block_len(self, block_len: list[int]):
self._rank_local_layout().set_block_len(block_len)
def prepare_value_for_block(self, block_id: int) -> tuple[list[int], list[int]]:
"""Return addresses and sizes for one physical block slot."""
return self._rank_local_layout().prepare_value_for_block(block_id)
def process_tokens(
self,
token_len: int,
block_hashes: list[BlockHash],
mask_num: int = 0,
*,
chunk_mask: list[bool] | None = None,
put_step: int = 1,
put_step_rank: int = 0,
) -> Iterable[tuple[int, int, BlockHash]]:
"""Process tokens and yield (start_idx, end_idx, block_hash) tuples.
When there are fewer KV heads than TP ranks, chunks are distributed
across TP ranks to avoid duplicate load/store. The assignment keys off
the absolute ``chunk_id`` so a given chunk always lands on the same
rank regardless of where the processed suffix begins.
Args:
token_len: Total number of tokens. Must be hash-block aligned and
covered by ``block_hashes`` when hashes are present.
block_hashes: Block hashes computed at ``hash_block_size`` granularity.
When ``block_size > hash_block_size`` each group's ``block_size`` chunk
is keyed by its last sub-hash via ``chunk_hashes_for_block_size``.
mask_num: Number of tokens to skip from the beginning.
chunk_mask: Optional mask relative to the first chunk after
``mask_num``. False entries are skipped before hash access.
put_step: Stride for distributing chunks across ranks.
put_step_rank: ``chunk_id % put_step`` value this rank stores.
"""
assert put_step > 0
if not block_hashes:
return
assert token_len % self.hash_block_size == 0
assert token_len // self.hash_block_size <= len(block_hashes)
start_chunk = max(0, cdiv(mask_num, self.block_size))
max_chunks = cdiv(token_len, self.block_size)
if chunk_mask is not None:
max_chunks = min(max_chunks, start_chunk + len(chunk_mask))
for chunk_id in range(start_chunk, max_chunks):
if chunk_mask is not None and not chunk_mask[chunk_id - start_chunk]:
continue
if chunk_id % put_step != put_step_rank:
continue
start_idx = chunk_id * self.block_size
end_idx = min(start_idx + self.block_size, token_len)
h = block_hashes[end_idx // self.hash_block_size - 1]
yield start_idx, end_idx, h
@dataclass(frozen=True)
class TailKeyBoundary:
"""Hash boundary used to key a group's tail block in the store.
Attributes:
group_id: KV-cache group containing the tail block.
num_tokens: Token boundary whose prefix hash identifies the matched
stored block. The loader uses
``block_hashes[num_tokens // hash_block_size - 1]`` instead of the
hash implied by ``MooncakeLookupResult.hit_length``. This changes
only the load key, not the reusable prefix.
"""
group_id: int
num_tokens: int
@dataclass
class MooncakeLookupResult:
"""Lookup result used to build the subsequent load request.
Attributes:
hit_length: Longest prefix that every KV-cache group can reuse after
their individual cache hits converge.
tail_key_boundaries: Hash boundary used to store each cache group's
tail block when ``hit_length`` does not identify its store key.
There is one entry per group for every nonzero hit.
"""
hit_length: int
tail_key_boundaries: tuple[TailKeyBoundary, ...] = ()
@dataclass
class LoadSpec:
"""Specification for loading KV cache from external store."""
vllm_cached_tokens: int
kvpool_cached_tokens: int
can_load: bool
token_len: int = 0
tail_key_boundaries: tuple[TailKeyBoundary, ...] = ()
@dataclass
class RequestTracker:
"""Tracks per-request state across scheduler ticks."""
req_id: str
token_len: int
allocated_block_ids: tuple[list[int], ...]
num_saved_tokens: int = 0
token_ids: list[int] | None = None
has_pending_offload: bool = False
# Snapshot of the prefill range length at tracker creation time.
# For a fresh request this is len(prompt). For a resumed-from-preemption
# request it includes previously-generated tokens, which are re-prefilled.
prefill_end_tokens: int = 0
partial_tail_sent: bool = False
def reset(self) -> None:
self.token_len = 0
self.allocated_block_ids = ()
self.num_saved_tokens = 0
self.token_ids = None
self.has_pending_offload = False
self.prefill_end_tokens = 0
self.partial_tail_sent = False
def update(
self,
new_block_ids: tuple[list[int], ...] | list[int],
) -> None:
# Backward-compat: accept a single list (broadcast to single group).
if isinstance(new_block_ids, list):
new_block_ids = (new_block_ids,)
if len(new_block_ids) != len(self.allocated_block_ids):
raise ValueError(
f"Group count mismatch: tracker has "
f"{len(self.allocated_block_ids)} groups, update has "
f"{len(new_block_ids)}"
)
for existing, new in zip(self.allocated_block_ids, new_block_ids, strict=True):
if new:
existing.extend(new)
class BoundaryPut(NamedTuple):
"""A block stored under the hash of the prefix ending at ``num_tokens``."""
group_id: int
block_id: int
num_tokens: int
@dataclass
class ReqMeta:
"""Per-request metadata for store put/get operations."""
req_id: str
token_len_chunk: int
block_ids: tuple[list[int], ...]
block_hashes: list[BlockHash]
can_save: bool | None = None
load_spec: LoadSpec | None = None
current_event: torch.cuda.Event | None = None
token_ids: list[int] | None = None
# Absolute request offset represented by token_ids[0].
token_ids_start: int = 0
num_prompt_tokens: int | None = None
# Total prefill tokens, including generated tokens replayed after preemption.
prefill_end_tokens: int | None = None
# Identifies this store job for the engine's lifetime. A request id cannot
# serve that purpose: it is reused once a preempted request resumes, so it
# would release the wrong job's blocks.
store_job_id: int | None = None
# Blocks stored under an explicit prefix hash rather than by position:
# core-provided mamba "align" states (a block-aligned entry is a committed
# snapshot, a non-aligned one the sub-block CoW prompt tail) and the
# scheduler-resolved non-Mamba blocks of the prompt's partial tail. The
# store-job reference keeps each block alive until every worker rank
# finishes the job.
boundary_puts: list[BoundaryPut] | None = None
# Total computed prefix length at the end of this step.
completed_token_len: int | None = None
# Set on the save that first covers the whole prompt, so the scheduler
# publishes the prompt's partial tail once rather than on every save.
publish_partial_tail: bool = False
@staticmethod
def from_request_tracker(
tracker: RequestTracker,
block_size: int,
load_spec: LoadSpec | None = None,
skip_save: bool | None = False,
block_hashes: list[BlockHash] | None = None,
save_partial_tail: bool = False,
num_prompt_tokens: int | None = None,
) -> "ReqMeta | None":
"""Create ReqMeta from a RequestTracker."""
if save_partial_tail:
assert num_prompt_tokens is not None
if block_hashes is None:
block_hashes = []
input_token_len = tracker.token_len
token_ids_start = tracker.num_saved_tokens
chunk_boundary = cdiv(token_ids_start + 1, block_size) * block_size
num_tokens_to_save = input_token_len // block_size * block_size
publish_tail = (
save_partial_tail
and not tracker.partial_tail_sent
and num_prompt_tokens is not None
and 0 < num_prompt_tokens <= input_token_len
)
skip_save = skip_save or (
num_tokens_to_save < chunk_boundary and not publish_tail
)
# A ReqMeta must never carry both a save AND a load.
# The save would also be wasted work — the bytes are being looked up
# in the store right now. Later cached_reqs steps save new tokens
# normally.
if load_spec is not None and load_spec.can_load:
skip_save = True
if skip_save and load_spec is None:
return None
if not skip_save:
tracker.num_saved_tokens = num_tokens_to_save
tracker.partial_tail_sent |= publish_tail
token_ids = None
if tracker.token_ids and not skip_save:
# Scheduler tracking continues while this job is handled by an
# asynchronous worker, so metadata must own a stable snapshot.
token_ids = tracker.token_ids[token_ids_start:num_tokens_to_save]
if load_spec is not None and load_spec.can_load:
logger.debug(
"Scheduled to load %d tokens for request %s",
load_spec.kvpool_cached_tokens,
tracker.req_id,
)
else:
load_spec = None
logger.debug(
"request:%s, meta save spec:%s, meta load spec:%s",
tracker.req_id,
not skip_save,
load_spec,
)
return ReqMeta(
req_id=tracker.req_id,
token_len_chunk=num_tokens_to_save,
block_ids=tracker.allocated_block_ids,
can_save=not skip_save,
load_spec=load_spec,
block_hashes=block_hashes,
token_ids=token_ids,
token_ids_start=token_ids_start,
num_prompt_tokens=num_prompt_tokens,
prefill_end_tokens=tracker.prefill_end_tokens,
completed_token_len=input_token_len,
publish_partial_tail=publish_tail and not skip_save,
)
@dataclass
class MooncakeStoreWorkerMetadata(KVConnectorWorkerMetadata):
"""Maps ``ReqMeta.store_job_id`` to the number of ranks done with that job."""
completed_saves: dict[int, int] = field(default_factory=dict)
def aggregate(
self, other: "KVConnectorWorkerMetadata"
) -> "MooncakeStoreWorkerMetadata":
assert isinstance(other, MooncakeStoreWorkerMetadata)
for store_job_id, count in other.completed_saves.items():
self.completed_saves[store_job_id] = (
self.completed_saves.get(store_job_id, 0) + count
)
return self
class MooncakeStoreConnectorMetadata(KVConnectorMetadata):
"""Metadata passed from scheduler to worker."""
def __init__(
self,
unfinished_request_ids: set[str],
preempted_req_ids: set[str],
):
self.requests: list[ReqMeta] = []
self.unfinished_request_ids = unfinished_request_ids
self.preempted_req_ids = preempted_req_ids
def add_request(self, req_meta: ReqMeta) -> None:
self.requests.append(req_meta)
@@ -1,189 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Per-operation telemetry for MooncakeStoreConnector.
Records one row per Mooncake RPC (``save_exists``, ``save_put``, ``load_get``,
``lookup_exists``) with duration, key/byte counts, status, and failed-key
count. Exposed to the logger via ``KVConnectorLogging`` and to Prometheus
via ``MooncakeStorePromMetrics``.
"""
from dataclasses import dataclass
from statistics import fmean
from typing import Any
from vllm.config import VllmConfig
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
KVConnectorPromMetrics,
KVConnectorStats,
PromMetric,
PromMetricT,
)
def _nearest_rank_percentile(values: list[float], percentile: float) -> float:
if not values:
return 0.0
sorted_values = sorted(values)
rank = max(
0, min(len(sorted_values) - 1, int(percentile * len(sorted_values) - 1e-12))
)
return sorted_values[rank]
@dataclass
class MooncakeStoreConnectorStats(KVConnectorStats):
"""Serializable Mooncake store communication telemetry."""
def __post_init__(self):
if not self.data:
self.reset()
def reset(self):
self.data: dict[str, list[dict[str, int | float | str]]] = {}
def is_empty(self) -> bool:
return not self.data
def aggregate(self, other: KVConnectorStats) -> KVConnectorStats:
if other.is_empty():
return self
for operation, records in other.data.items():
self.data.setdefault(operation, []).extend(records)
return self
def reduce(self) -> dict[str, int | float]:
reduced: dict[str, int | float] = {}
for operation, records in sorted(self.data.items()):
if not records:
continue
durations = [float(record["duration_seconds"]) for record in records]
reduced[f"{operation}_count"] = len(records)
reduced[f"{operation}_avg_ms"] = round(fmean(durations) * 1e3, 3)
reduced[f"{operation}_p90_ms"] = round(
_nearest_rank_percentile(durations, 0.9) * 1e3, 3
)
reduced[f"{operation}_total_keys"] = sum(
int(record["num_keys"]) for record in records
)
reduced[f"{operation}_total_bytes"] = sum(
int(record["num_bytes"]) for record in records
)
reduced[f"{operation}_failed_keys"] = sum(
int(record["num_failed_keys"]) for record in records
)
reduced[f"{operation}_error_count"] = sum(
1 for record in records if record["status"] == "error"
)
return reduced
def record_operation(
self,
operation: str,
duration_seconds: float,
num_keys: int,
*,
num_bytes: int = 0,
status: str = "ok",
num_failed_keys: int = 0,
) -> None:
self.data.setdefault(operation, []).append(
{
"duration_seconds": duration_seconds,
"num_keys": num_keys,
"num_bytes": num_bytes,
"status": status,
"num_failed_keys": num_failed_keys,
}
)
class MooncakeStorePromMetrics(KVConnectorPromMetrics):
"""Prometheus metrics for Mooncake store communication."""
def __init__(
self,
vllm_config: VllmConfig,
metric_types: dict[type[PromMetric], type[PromMetricT]],
labelnames: list[str],
per_engine_labelvalues: dict[int, list[object]],
):
super().__init__(vllm_config, metric_types, labelnames, per_engine_labelvalues)
metric_labelnames = labelnames + ["operation", "status"]
self._metric_cache: dict[tuple[int, str, str], dict[str, PromMetric]] = {}
self._histogram_operation_time = self._histogram_cls(
name="vllm:mooncake_store_operation_time_seconds",
documentation="Histogram of Mooncake store communication time.",
buckets=[
1e-3,
5e-3,
1e-2,
5e-2,
1e-1,
2e-1,
3e-1,
4e-1,
5e-1,
7.5e-1,
1.0,
1.5,
2.0,
3.0,
4.0,
],
labelnames=metric_labelnames,
)
self._counter_operation_calls = self._counter_cls(
name="vllm:mooncake_store_operation_total",
documentation="Number of Mooncake store communication operations.",
labelnames=metric_labelnames,
)
self._counter_operation_keys = self._counter_cls(
name="vllm:mooncake_store_operation_keys_total",
documentation="Number of Mooncake store keys touched by operations.",
labelnames=metric_labelnames,
)
self._counter_operation_bytes = self._counter_cls(
name="vllm:mooncake_store_operation_bytes_total",
documentation="Number of bytes transferred by Mooncake store operations.",
labelnames=metric_labelnames,
)
self._counter_failed_keys = self._counter_cls(
name="vllm:mooncake_store_operation_failed_keys_total",
documentation="Number of Mooncake store keys that failed in operations.",
labelnames=metric_labelnames,
)
def _get_metrics(
self,
engine_idx: int,
operation: str,
status: str,
) -> dict[str, PromMetric]:
cache_key = (engine_idx, operation, status)
if cache_key not in self._metric_cache:
label_values = self.per_engine_labelvalues[engine_idx] + [operation, status]
self._metric_cache[cache_key] = {
"time": self._histogram_operation_time.labels(*label_values),
"calls": self._counter_operation_calls.labels(*label_values),
"keys": self._counter_operation_keys.labels(*label_values),
"bytes": self._counter_operation_bytes.labels(*label_values),
"failed_keys": self._counter_failed_keys.labels(*label_values),
}
return self._metric_cache[cache_key]
def observe(self, transfer_stats_data: dict[str, Any] | None, engine_idx: int = 0):
if not transfer_stats_data:
return
for operation, records in transfer_stats_data.items():
assert isinstance(records, list)
for record in records:
assert isinstance(record, dict)
status = str(record["status"])
metrics = self._get_metrics(engine_idx, operation, status)
metrics["time"].observe(float(record["duration_seconds"]))
metrics["calls"].inc()
metrics["keys"].inc(int(record["num_keys"]))
metrics["bytes"].inc(int(record["num_bytes"]))
metrics["failed_keys"].inc(int(record["num_failed_keys"]))
@@ -1,81 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Wire-format constants for the LookupKey ZMQ admin channel.
This is the single source of truth shared by ``LookupKeyClient`` and
``LookupKeyServer`` on the scheduler<->worker rank-0 admin channel.
Wire format (REQ/REP over IPC):
Request: [msg_type: bytes] [payload_frames...]
msg_type == LOOKUP_MSG:
frame 1: num_tokens (u32 big-endian, 4 bytes); the worker derives
the aligned lookup length
frame 2: hash_len (u16 big-endian, 2 bytes) — byte length of each
fixed-size block hash (0 when there are no hashes)
frame 3: raw block hashes concatenated back-to-back (each hash_len
bytes); the server splits on hash_len
Response: hit_length (u32 big-endian, first 4 bytes), followed by zero
or more 8-byte tail-key boundaries. Each entry is group_id
(u32), then boundary_tokens (u32).
msg_type == RESET_MSG:
(no payload frames)
Response: [RESP_OK] or [RESP_ERR]
The first frame of every request is a named bytes tag (not a numeric
sentinel that aliases the data field) so the protocol stays
self-describing and extensible: adding new admin commands requires
only a new tag and a new dispatch branch.
Mirrors the named-tag convention used by the NIXL connector (see
``vllm/distributed/kv_transfer/kv_connector/v1/nixl/metadata.py``).
"""
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
MooncakeLookupResult,
TailKeyBoundary,
)
# Request message-type tags. Frame 0 of every request.
LOOKUP_MSG: bytes = b"lookup"
RESET_MSG: bytes = b"reset"
# Single-byte response status codes for admin commands.
RESP_OK: bytes = b"\x01"
RESP_ERR: bytes = b"\x00"
# group_id (u32), boundary_tokens (u32).
TAIL_KEY_BOUNDARY_ENTRY_SIZE: int = 8
def encode_lookup_response(result: MooncakeLookupResult) -> bytes:
hit_length = result.hit_length.to_bytes(4, "big")
if not result.tail_key_boundaries:
return hit_length
payload = bytearray(hit_length)
for boundary in result.tail_key_boundaries:
payload.extend(boundary.group_id.to_bytes(4, "big"))
payload.extend(boundary.num_tokens.to_bytes(4, "big"))
return bytes(payload)
def decode_lookup_response(payload: bytes) -> MooncakeLookupResult:
if len(payload) < 4 or (len(payload) - 4) % TAIL_KEY_BOUNDARY_ENTRY_SIZE:
raise ValueError("Invalid Mooncake lookup response")
hit_length = int.from_bytes(payload[:4], "big")
if len(payload) == 4:
return MooncakeLookupResult(hit_length)
boundaries = []
for offset in range(4, len(payload), TAIL_KEY_BOUNDARY_ENTRY_SIZE):
boundaries.append(
TailKeyBoundary(
group_id=int.from_bytes(payload[offset : offset + 4], "big"),
num_tokens=int.from_bytes(payload[offset + 4 : offset + 8], "big"),
)
)
return MooncakeLookupResult(hit_length, tuple(boundaries))
@@ -1,788 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
#
# Adapted from vllm-project/vllm-ascend
# (vllm_ascend/distributed/kv_transfer/kv_pool/ascend_store/).
"""Scheduler-side logic for MooncakeStoreConnector."""
from collections.abc import Sequence
from vllm.config import VllmConfig
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorMetadata,
)
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.coordinator import (
MooncakeStoreCoordinator,
)
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import ( # noqa: E501
BoundaryPut,
LoadSpec,
MooncakeStoreConnectorMetadata,
MooncakeStoreWorkerMetadata,
ReqMeta,
RequestTracker,
)
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.worker import ( # noqa: E501
LookupKeyClient,
)
from vllm.logger import init_logger
from vllm.utils.math_utils import cdiv
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
from vllm.v1.core.block_pool import BlockPool
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
from vllm.v1.core.kv_cache_utils import (
resolve_kv_cache_block_sizes,
)
from vllm.v1.core.sched.output import NewRequestData, SchedulerOutput
from vllm.v1.kv_cache_interface import (
KVCacheConfig,
MambaSpec,
get_mamba_prefill_checkpoint_position,
)
from vllm.v1.outputs import KVConnectorOutput
from vllm.v1.request import Request
logger = init_logger(__name__)
def _new_req_prefill_tokens(request: NewRequestData) -> list[int]:
"""Tokens this prefill will compute KV for.
Under the v2 model runner, resumed-from-preemption requests appear in
``scheduled_new_reqs`` with ``prefill_token_ids`` set to the request's full
token list (prompt + previously-generated). For all other cases this falls
back to the original prompt.
"""
if request.prefill_token_ids is not None:
return request.prefill_token_ids
assert request.prompt_token_ids is not None
return request.prompt_token_ids
def _partial_tail_non_mamba_puts(
coord: MooncakeStoreCoordinator,
req_meta: ReqMeta,
block_sizes: Sequence[int],
) -> list[BoundaryPut]:
"""Locate the non-Mamba blocks a partial-tail save publishes.
A later request resumes at the prompt's Mamba checkpoint (``boundary``) only
if both pieces of KV are stored:
- Mamba groups: the state at ``boundary``, handed off by the core as a
boundary put of its own, so Mamba groups are not listed here.
- Other groups: the KV from the last LCM-aligned normal save up to
``boundary`` plus the group's EAGLE proof margin (``proof_end``). Each
block is keyed by the hash at its end, clipped to ``proof_end``.
Groups whose proof is not computed yet are omitted. For example, with LCM
16, blocks of 4 tokens, and a checkpoint at 44, the full-attention blocks
are 8-10, keyed at 36, 40 and 44.
A Mamba state handed off at any other position, i.e. a shared-prefix
junction (``--enable-mamba-shared-prefix-checkpoint``), adds no puts here.
``block_sizes`` are the per-group block sizes, indexed like
``req_meta.block_ids``.
"""
mamba_tails = [
put.num_tokens
for put in req_meta.boundary_puts or []
if put.group_id in coord.mamba_group_ids
and put.num_tokens % block_sizes[put.group_id]
]
# The tail is due on the save that first covers the prompt, or when Mamba
# hands off its checkpoint state.
if not mamba_tails and not req_meta.publish_partial_tail:
return []
prompt_tokens = req_meta.num_prompt_tokens
completed = req_meta.completed_token_len
assert prompt_tokens is not None and completed is not None
hash_block_size = coord.hash_block_size
# Core drops the Mamba checkpoint whenever EAGLE block drop is on, whichever
# groups carry the eagle flag.
boundary = get_mamba_prefill_checkpoint_position(
prompt_tokens, hash_block_size, coord.use_eagle
)
# Any other sub-block Mamba hand-off is a shared-prefix junction. Its
# attention KV, EAGLE proof included, ends on an attention block end no later
# than this checkpoint's proof, so the normal and prompt-tail saves store it.
if not req_meta.publish_partial_tail and boundary not in mamba_tails:
return []
num_hashes = len(req_meta.block_hashes)
if boundary == 0 or boundary // hash_block_size > num_hashes:
return []
start = boundary // coord.lcm_block_size * coord.lcm_block_size
puts: list[BoundaryPut] = []
for group_id, block_size in enumerate(block_sizes):
if group_id in coord.mamba_group_ids:
continue
proof_end = boundary + coord.eagle_proof_margin_by_group.get(group_id, 0)
if proof_end > completed or proof_end // hash_block_size > num_hashes:
continue
group_blocks = req_meta.block_ids[group_id]
for block_idx in range(start // block_size, cdiv(proof_end, block_size)):
if block_idx >= len(group_blocks):
break
block_id = group_blocks[block_idx]
if block_id == NULL_BLOCK_ID:
continue
num_tokens = min((block_idx + 1) * block_size, proof_end)
puts.append(BoundaryPut(group_id, block_id, num_tokens))
return puts
class MooncakeStoreScheduler:
"""Scheduler-side component for MooncakeStoreConnector."""
def __init__(
self,
vllm_config: VllmConfig,
kv_cache_config: KVCacheConfig,
):
assert vllm_config.kv_transfer_config is not None
self.kv_role = vllm_config.kv_transfer_config.kv_role
kvc_extra_config = vllm_config.kv_transfer_config.kv_connector_extra_config
self.load_async = kvc_extra_config.get("load_async", True)
self.lookup_async = kvc_extra_config.get("lookup_async", False)
# Skips lookup CPU cost on instances that never load KV from the store.
self.enable_lookup = kvc_extra_config.get("enable_lookup", True)
self.save_decode_cache = kvc_extra_config.get("save_decode_cache", False)
kv_event_config = vllm_config.kv_events_config
self.enable_kv_events = bool(
kv_event_config and kv_event_config.enable_kv_cache_events
)
self.client = LookupKeyClient(vllm_config)
self.kv_cache_config = kv_cache_config
self._store_group_ids = kv_cache_config.prefix_cacheable_group_ids
# Map scheduler group IDs to store group indices. Groups outside the
# store projection, such as the QSA ring, map to None and are skipped.
self._store_group_id_by_kv_cache_group_id = {
group_id: store_group_id
for store_group_id, group_id in enumerate(self._store_group_ids)
}
store_groups = kv_cache_config.prefix_cacheable_groups
# Align with the engine's own scheduler_block_size and hash_block_size.
self._block_size, self._hash_block_size = resolve_kv_cache_block_sizes(
kv_cache_config, vllm_config
)
mamba_groups = {
group_id: group.kv_cache_spec
for group_id, group in enumerate(store_groups)
if isinstance(group.kv_cache_spec, MambaSpec)
}
self._boundary_state_group_ids = frozenset(mamba_groups)
# Mirrors MooncakeStoreConnector._capacity_only.
self._capacity_only = (
self.kv_role == "kv_consumer"
and not self.enable_lookup
and not self.save_decode_cache
)
self._store_coord: MooncakeStoreCoordinator | None = None
self.enable_partial_hash_hits = False
if not self._capacity_only:
assert all(
spec.mamba_cache_mode == "align" for spec in mamba_groups.values()
), "MooncakeStoreScheduler requires mamba_cache_mode='align'"
self._store_coord = MooncakeStoreCoordinator.from_kv_cache_config(
kv_cache_config, vllm_config, self._block_size, self._hash_block_size
)
self.enable_partial_hash_hits = self._store_coord.enable_partial_hash_hits
self._gpu_block_pool: BlockPool | None = None
self._num_workers = vllm_config.parallel_config.world_size
self._next_store_job_id = 0
# store_job_id -> (referenced block ids, ranks yet to report completion)
self._pinned_saves: dict[int, tuple[list[int], int]] = {}
# Per-request state
self.load_specs: dict[str, LoadSpec] = {} # to be loaded
self._request_trackers: dict[str, RequestTracker] = {} # scheduled new requests
self._unfinished_requests: dict[str, tuple[Request, tuple[list[int], ...]]] = {}
self._unfinished_request_ids: set[str] = set()
self._finished_partial_tail_metas: dict[str, ReqMeta] = {}
def bind_gpu_block_pool(self, gpu_block_pool: BlockPool) -> None:
self._gpu_block_pool = gpu_block_pool
def get_num_new_matched_tokens(
self,
request: Request,
num_computed_tokens: int,
) -> tuple[int | None, bool]:
"""Check for external KV cache hit.
Returns ``(None, False)`` when an async lookup is still in flight,
signaling the scheduler to retry this request on a later step.
"""
if not self.enable_lookup:
return 0, False
# Fine-grained hits may land on a hash boundary inside a block; without
# partial hits, prefixes shorter than one physical block are skipped.
align = (
self._hash_block_size if self.enable_partial_hash_hits else self._block_size
)
if request.num_tokens < align:
return 0, False
lookup_result = self.client.lookup(
request.request_id,
request.num_tokens,
request.block_hashes,
non_block=self.lookup_async,
)
if lookup_result is None:
# Lookup not ready yet; scheduler will retry on a later step.
return None, False
num_external_hit_tokens = lookup_result.hit_length
if num_external_hit_tokens < num_computed_tokens:
need_to_allocate = 0
else:
need_to_allocate = num_external_hit_tokens - num_computed_tokens
logger.debug(
"Reqid: %s, Total tokens %d, kvpool hit tokens: %d, need to load: %d",
request.request_id,
request.num_tokens,
num_external_hit_tokens,
need_to_allocate,
)
if need_to_allocate <= 0:
return 0, False
self.load_specs[request.request_id] = LoadSpec(
vllm_cached_tokens=num_computed_tokens,
kvpool_cached_tokens=num_external_hit_tokens,
can_load=False,
tail_key_boundaries=lookup_result.tail_key_boundaries,
)
return need_to_allocate, self.load_async
def update_state_after_alloc(
self,
request: Request,
blocks: KVCacheBlocks,
num_external_tokens: int,
):
"""Update state after block allocation."""
local_block_ids: tuple[list[int], ...] = ()
if num_external_tokens > 0:
local_block_ids = blocks.get_block_ids(
group_ids=self._store_group_ids,
)
self._unfinished_requests[request.request_id] = (request, local_block_ids)
self._unfinished_request_ids.add(request.request_id)
if request.request_id not in self.load_specs:
return
if num_external_tokens == 0:
self.load_specs[request.request_id].can_load = False
return
assert (
num_external_tokens > 0
and num_external_tokens
== self.load_specs[request.request_id].kvpool_cached_tokens
- self.load_specs[request.request_id].vllm_cached_tokens
), (
f"Mismatch in number of tokens: {num_external_tokens} vs "
f"{self.load_specs[request.request_id].kvpool_cached_tokens} - "
f"{self.load_specs[request.request_id].vllm_cached_tokens}"
f" for request {request.request_id}"
)
self.load_specs[request.request_id].can_load = True
def build_connector_meta(
self, scheduler_output: SchedulerOutput
) -> KVConnectorMetadata:
"""Build connector metadata for this scheduler step."""
is_consumer = self.kv_role == "kv_consumer"
can_process_cached = not is_consumer or self.save_decode_cache
for finished_req_id in scheduler_output.finished_req_ids:
self.client.discard(finished_req_id)
self.load_specs.pop(finished_req_id, None)
self._request_trackers.pop(finished_req_id, None)
self._unfinished_requests.pop(finished_req_id, None)
self._unfinished_request_ids.discard(finished_req_id)
preempted_ids = scheduler_output.preempted_req_ids or set()
for req_id in preempted_ids:
self.load_specs.pop(req_id, None)
if request_tracker := self._request_trackers.get(req_id):
request_tracker.reset()
self._unfinished_requests.pop(req_id, None)
meta = MooncakeStoreConnectorMetadata(
self._unfinished_request_ids,
preempted_ids,
)
# Handle new requests
for request in scheduler_output.scheduled_new_reqs:
load_spec = self.load_specs.pop(request.req_id, None)
num_tokens_to_compute = (
request.num_computed_tokens
+ scheduler_output.num_scheduled_tokens[request.req_id]
)
assert request.req_id in self._unfinished_requests
request_tuple = self._unfinished_requests.get(request.req_id)
request_real = request_tuple[0] # type: ignore[index]
unfolded_block_ids = tuple(
request.block_ids[group_id].copy() for group_id in self._store_group_ids
)
prefill_tokens = _new_req_prefill_tokens(request)
request_tracker = RequestTracker(
req_id=request.req_id,
token_len=num_tokens_to_compute,
allocated_block_ids=unfolded_block_ids,
num_saved_tokens=0,
token_ids=(
prefill_tokens[:num_tokens_to_compute]
if self.enable_kv_events
else None
),
prefill_end_tokens=len(prefill_tokens),
)
self._request_trackers[request.req_id] = request_tracker
req_meta = ReqMeta.from_request_tracker(
request_tracker,
self._block_size,
load_spec=load_spec,
# A consumer may write decode KV without becoming a prefill
# producer. Loads are still carried by the same metadata.
skip_save=is_consumer,
block_hashes=request_real.block_hashes,
num_prompt_tokens=request_real.num_prompt_tokens,
save_partial_tail=self.enable_partial_hash_hits,
)
if req_meta is not None:
meta.add_request(req_meta)
# Handle cached (running, or MRV1 resumed-from-preemption) requests
cached_reqs = scheduler_output.scheduled_cached_reqs
if can_process_cached:
for i, req_id in enumerate(cached_reqs.req_ids):
new_block_ids = cached_reqs.new_block_ids[i]
if new_block_ids:
new_block_ids = tuple(
new_block_ids[group_id] for group_id in self._store_group_ids
)
req_meta = None
if req_id in cached_reqs.resumed_req_ids:
# Resumed after preemption
if not new_block_ids:
continue
new_block_ids = tuple(b.copy() for b in new_block_ids)
load_spec = self.load_specs.pop(req_id, None)
request_tuple = self._unfinished_requests.get(req_id)
request_real = request_tuple[0] # type: ignore[index]
num_tokens_to_compute = (
request_real.num_computed_tokens
+ scheduler_output.num_scheduled_tokens[req_id]
)
# On resume, the request re-prefills prompt + previously
# generated tokens (all_token_ids).
prefill_tokens = list(request_real.all_token_ids)
request_tracker = RequestTracker(
req_id=req_id,
token_len=num_tokens_to_compute,
allocated_block_ids=new_block_ids,
num_saved_tokens=0,
token_ids=(
prefill_tokens[:num_tokens_to_compute]
if self.enable_kv_events
else None
),
prefill_end_tokens=len(prefill_tokens),
)
self._request_trackers[req_id] = request_tracker
req_meta = ReqMeta.from_request_tracker(
request_tracker,
self._block_size,
load_spec=load_spec,
skip_save=is_consumer,
block_hashes=request_real.block_hashes,
num_prompt_tokens=request_real.num_prompt_tokens,
save_partial_tail=self.enable_partial_hash_hits,
)
else:
# Decode/chunked request
request_tracker = self._request_trackers[req_id]
num_computed_token = cached_reqs.num_computed_tokens[i]
# Use the tracker's snapshot of the prefill range so resumed
# requests keep saving past the original prompt boundary.
prefill_end = request_tracker.prefill_end_tokens
is_decode = num_computed_token >= prefill_end
if is_decode and not self.save_decode_cache:
continue
num_new_tokens = scheduler_output.num_scheduled_tokens[req_id]
req_tuple = self._unfinished_requests.get(req_id)
if req_tuple:
unfinished_req = req_tuple[0]
num_current_tokens = request_tracker.token_len
new_token_ids = unfinished_req.all_token_ids[
num_current_tokens : num_current_tokens + num_new_tokens
]
request_tracker.token_len += len(new_token_ids)
if request_tracker.token_ids is not None:
request_tracker.token_ids.extend(new_token_ids)
else:
raise ValueError(
f"Request {req_id} is not in _unfinished_requests"
)
# A block is usually allocated before the step that fills
# it, so reaching a save boundary does not imply that this
# step has new block ids.
if new_block_ids:
request_tracker.update(new_block_ids)
if is_consumer and not is_decode:
continue
req_meta = ReqMeta.from_request_tracker(
request_tracker,
self._block_size,
load_spec=None,
skip_save=False,
block_hashes=unfinished_req.block_hashes,
num_prompt_tokens=unfinished_req.num_prompt_tokens,
save_partial_tail=self.enable_partial_hash_hits,
)
if req_meta is not None:
meta.add_request(req_meta)
# Handle requests with pending load specs not yet scheduled
request_ids = [req.req_id for req in scheduler_output.scheduled_new_reqs]
for request_id, (
unfinished_req,
block_ids,
) in self._unfinished_requests.items():
if request_id not in request_ids and request_id not in cached_reqs.req_ids:
load_spec = self.load_specs.pop(request_id, None)
# A load spec may have been proposed by this connector's
# lookup but rejected by MultiConnector in favor of another
# connector. Only the chosen connector may issue the pending
# load; the normal store path gets its blocks later from
# SchedulerOutput once the request is actually scheduled.
if load_spec is None or not load_spec.can_load:
continue
num_tokens_to_compute = load_spec.kvpool_cached_tokens
request_tracker = RequestTracker(
req_id=request_id,
token_len=num_tokens_to_compute,
allocated_block_ids=block_ids,
num_saved_tokens=0,
)
self._request_trackers[request_id] = request_tracker
req_meta = ReqMeta.from_request_tracker(
request_tracker,
self._block_size,
load_spec=load_spec,
skip_save=None,
block_hashes=unfinished_req.block_hashes,
num_prompt_tokens=unfinished_req.num_prompt_tokens,
)
if req_meta is not None:
meta.add_request(req_meta)
block_state = getattr(scheduler_output, "kv_connector_block_state", None)
if (
block_state is not None
and block_state.boundary_state_offloads
and not is_consumer
):
self._handle_boundary_state_offloads(
block_state.boundary_state_offloads, meta
)
self._apply_current_save_block_ids(meta, scheduler_output)
# Finish-time handoffs arrive after the producing step's metadata was
# built. Their exact blocks were pinned when they were registered, so
# they remain valid after request cleanup and need no current snapshot.
for req_meta in self._finished_partial_tail_metas.values():
meta.add_request(req_meta)
self._finished_partial_tail_metas.clear()
if not self._capacity_only:
self._reference_save_blocks(meta)
return meta
def _apply_current_save_block_ids(
self,
meta: MooncakeStoreConnectorMetadata,
scheduler_output: SchedulerOutput,
) -> None:
"""Replace append-only mirrors with the core's current block tables."""
save_metas = [req_meta for req_meta in meta.requests if req_meta.can_save]
if not save_metas:
return
block_state = scheduler_output.kv_connector_block_state
assert block_state is not None, (
"Current block tables are required for Mooncake store jobs"
)
for req_meta in save_metas:
block_ids = block_state.get_block_ids(req_meta.req_id)
assert block_ids is not None, (
f"Missing current block table for store request {req_meta.req_id}"
)
req_meta.block_ids = tuple(
block_ids[group_id] for group_id in self._store_group_ids
)
def _reference_save_blocks(self, meta: MooncakeStoreConnectorMetadata) -> None:
"""Take a GPU block reference for every store job this step emits.
The worker DMAs out of these blocks after the step that scheduled them,
so a reference keeps them out of the free queue even once the request
itself is freed, until every rank reports the job done.
"""
pool = self._gpu_block_pool
coord = self._store_coord
assert coord is not None
block_sizes = [g.kv_cache_spec.block_size for g in coord.kv_cache_groups]
for req_meta in meta.requests:
if not req_meta.can_save:
continue
assert pool is not None, (
"GPU block pool must be bound before any store job is emitted"
)
if req_meta.store_job_id is not None:
assert req_meta.store_job_id in self._pinned_saves
continue
if tail := _partial_tail_non_mamba_puts(coord, req_meta, block_sizes):
req_meta.boundary_puts = [*tail, *(req_meta.boundary_puts or [])]
req_meta.store_job_id = store_job_id = self._next_store_job_id
self._next_store_job_id += 1
block_ids = [put.block_id for put in req_meta.boundary_puts or []]
assert NULL_BLOCK_ID not in block_ids, (
"A null block cannot back a boundary put"
)
if req_meta.token_len_chunk:
# Normal prefix save: pin all non-Mamba sources for retries.
# Every allocated block is referenced, not just the ones covering
# this job's token range: a rank resumes from its own last
# successful offset, which lags the scheduler's whenever a save was
# skipped or failed, so it may read anywhere below the range.
block_ids.extend(
block_id
for group_id, group in enumerate(req_meta.block_ids)
if group_id not in self._boundary_state_group_ids
for block_id in group
if block_id != NULL_BLOCK_ID
)
# An aligned boundary block may also be present in the request's
# block table. Take and release exactly one reference per block.
block_ids = list(dict.fromkeys(block_ids))
assert NULL_BLOCK_ID not in block_ids
if not block_ids:
continue
self._pinned_saves[store_job_id] = (block_ids, self._num_workers)
pool.touch([pool.blocks[block_id] for block_id in block_ids])
def register_finished_partial_tail(
self,
request: Request,
block_ids: tuple[list[int], ...],
partial_tail_offloads: list[tuple[int, int, int]],
) -> bool:
"""Queue and pin a finish-time tail for the next connector step."""
if self.kv_role == "kv_consumer" or not partial_tail_offloads:
return False
tracker = self._request_trackers.get(request.request_id)
if tracker is None or not any(block_ids):
return False
boundaries = {boundary for _, _, boundary in partial_tail_offloads}
if len(boundaries) != 1:
raise ValueError(
"Partial-tail offloads for one request must share a boundary"
)
boundary_tokens = next(iter(boundaries))
if boundary_tokens > tracker.prefill_end_tokens:
return False
mamba_puts: list[BoundaryPut] = []
for group_id, block_id, boundary in partial_tail_offloads:
store_group_id = self._store_group_id_by_kv_cache_group_id.get(group_id)
if store_group_id not in self._boundary_state_group_ids:
return False
if block_id == NULL_BLOCK_ID:
return False
mamba_puts.append(BoundaryPut(store_group_id, block_id, boundary))
req_meta = ReqMeta(
req_id=request.request_id,
token_len_chunk=0,
block_ids=tuple(
block_ids[group_id].copy() for group_id in self._store_group_ids
),
block_hashes=list(request.block_hashes),
can_save=True,
num_prompt_tokens=request.num_prompt_tokens,
prefill_end_tokens=tracker.prefill_end_tokens,
boundary_puts=mamba_puts,
completed_token_len=request.num_computed_tokens,
)
coord = self._store_coord
assert coord is not None
req_meta.boundary_puts = [
*_partial_tail_non_mamba_puts(
coord,
req_meta,
[g.kv_cache_spec.block_size for g in coord.kv_cache_groups],
),
*mamba_puts,
]
pinned_block_ids = list(
dict.fromkeys(put.block_id for put in req_meta.boundary_puts)
)
pool = self._gpu_block_pool
assert pool is not None, (
"GPU block pool must be bound before a finish-time handoff"
)
assert request.request_id not in self._finished_partial_tail_metas
store_job_id = self._next_store_job_id
self._next_store_job_id += 1
self._pinned_saves[store_job_id] = (pinned_block_ids, self._num_workers)
pool.touch([pool.blocks[block_id] for block_id in pinned_block_ids])
req_meta.store_job_id = store_job_id
self._finished_partial_tail_metas[request.request_id] = req_meta
tracker.has_pending_offload = True
# The store job owns exact block refs, so request cleanup need not wait.
return False
def _handle_boundary_state_offloads(
self,
offloads: dict[str, list[tuple[int, int, int]]],
meta: MooncakeStoreConnectorMetadata,
) -> None:
"""Attach exact boundary-state blocks to store jobs for this step.
Flushed in the step they arrive: the CoW copy is enqueued before the
connector event records, so this step's event fences the exact block.
Entries ride the request's save meta when present, else an
offload-only ReqMeta (``token_len_chunk=0`` skips the normal save,
``can_save=True`` takes the normal enqueue and store-job pinning path).
"""
save_metas = {m.req_id: m for m in meta.requests if m.can_save}
for req_id, entries in offloads.items():
tracker = self._request_trackers.get(req_id)
req_tuple = self._unfinished_requests.get(req_id)
if tracker is None or req_tuple is None:
# Request finished/preempted within this step; its blocks are
# going away, so the offload is conservatively dropped.
logger.debug("Dropping boundary-state offload for request %s", req_id)
continue
accepted: list[BoundaryPut] = []
for group_id, block_id, boundary_tokens in entries:
# Every other group stops saving at the end of this prefill, so
# a mamba-only key past it can never complete a joint hybrid
# hit. `prefill_end_tokens` — not the original prompt length —
# is the boundary: a resumed request re-prefills and re-saves
# its previously generated tokens for every group.
if boundary_tokens > tracker.prefill_end_tokens:
continue
if block_id == NULL_BLOCK_ID:
continue
store_group_id = self._store_group_id_by_kv_cache_group_id.get(group_id)
if store_group_id not in self._boundary_state_group_ids:
continue
accepted.append(BoundaryPut(store_group_id, block_id, boundary_tokens))
if not accepted:
continue
tracker.has_pending_offload = True
if (req_meta := save_metas.get(req_id)) is not None:
req_meta.boundary_puts = accepted
continue
meta.add_request(
ReqMeta(
req_id=req_id,
token_len_chunk=0,
block_ids=tracker.allocated_block_ids,
block_hashes=req_tuple[0].block_hashes,
can_save=True,
num_prompt_tokens=req_tuple[0].num_prompt_tokens,
prefill_end_tokens=tracker.prefill_end_tokens,
boundary_puts=accepted,
completed_token_len=tracker.token_len,
)
)
def update_connector_output(self, connector_output: KVConnectorOutput) -> None:
"""Drop the block references of store jobs every rank has finished."""
meta = connector_output.kv_connector_worker_meta
if not isinstance(meta, MooncakeStoreWorkerMetadata):
return
pool = self._gpu_block_pool
assert pool is not None
for store_job_id, count in meta.completed_saves.items():
pinned = self._pinned_saves.get(store_job_id)
if pinned is None:
# The job referenced no blocks, so nothing was recorded for it.
continue
block_ids, remaining = pinned
remaining -= count
if remaining > 0:
self._pinned_saves[store_job_id] = (block_ids, remaining)
continue
assert remaining == 0, (
f"store job {store_job_id} reported by too many ranks"
)
del self._pinned_saves[store_job_id]
# Tail-first, as elsewhere, so the shared prefix is evicted last.
pool.free_blocks(pool.blocks[bid] for bid in reversed(block_ids))
def has_pending_push_work(self) -> bool:
"""Keep the engine stepping while any store job still holds block refs.
Completions only reach the scheduler as worker metadata on a step, so an
engine that quiesced with jobs in flight would leave those references
held indefinitely. Nothing else keeps it alive now that a finishing
request no longer defers its own free.
"""
return bool(self._pinned_saves)
def reset_store(self) -> bool:
"""Trigger a global ``remove_all(force=True)`` on the Mooncake master.
Routes through the existing LookupKey ZMQ admin channel to worker
rank 0, which owns the ``MooncakeDistributedStore`` handle.
Ordering assumption: caller (typically
``Scheduler.reset_connector_cache``, invoked via
``reset_prefix_cache(reset_connector=True)``) MUST ensure no
in-flight Mooncake lookups or transfers. For RL workflows this is
satisfied at the step boundary after weight updates and rollout
drain. Violating this can allow stale KV to be served on the next
request, defeating the hard-reset guarantee.
Returns True on ACK from worker, False on NACK or RPC error.
"""
try:
ok = self.client.reset()
if ok:
logger.info("Mooncake store reset via remove_all succeeded.")
else:
logger.warning("Mooncake store reset returned NACK from worker.")
return ok
except Exception as e:
logger.error("Mooncake reset_store RPC failed: %s", e)
return False
File diff suppressed because it is too large Load Diff
@@ -1,41 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from .connector import UMBPStoreConnector
from .data import (
BlockIdentityCodec,
BlockTransferPlan,
KVLayoutDescriptor,
KVLayoutPlanner,
KVRange,
KVRegion,
LoadSpec,
RankTopology,
RequestTracker,
TransferJobState,
TransferJobStatus,
UMBPConnectorMetadata,
UMBPConnectorWorkerMetadata,
UMBPNamespace,
)
from .runtime import UMBPRuntimeConfig, UMBPRuntimeFactory
__all__ = [
"UMBPStoreConnector",
"BlockIdentityCodec",
"BlockTransferPlan",
"KVLayoutPlanner",
"KVLayoutDescriptor",
"KVRange",
"KVRegion",
"LoadSpec",
"RequestTracker",
"RankTopology",
"TransferJobState",
"TransferJobStatus",
"UMBPConnectorMetadata",
"UMBPConnectorWorkerMetadata",
"UMBPNamespace",
"UMBPRuntimeConfig",
"UMBPRuntimeFactory",
]
@@ -1,220 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""vLLM-facing shared UMBP connector."""
from __future__ import annotations
from typing import TYPE_CHECKING, Any
import torch
from vllm.config import VllmConfig
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorBase_V1,
KVConnectorMetadata,
KVConnectorRole,
KVConnectorTransferResults,
)
from vllm.forward_context import ForwardContext
from vllm.v1.attention.backend import AttentionMetadata
from vllm.v1.core.sched.output import SchedulerOutput
from .data import (
BlockIdentityCodec,
KVLayoutPlanner,
RankTopology,
UMBPConnectorMetadata,
UMBPConnectorWorkerMetadata,
UMBPNamespace,
)
from .runtime import UMBPRuntimeConfig, UMBPRuntimeFactory
from .scheduler import UMBPStoreConnectorScheduler
from .worker import UMBPStoreConnectorWorker
if TYPE_CHECKING:
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.request import Request
class UMBPStoreConnector(KVConnectorBase_V1):
"""vLLM connector lifecycle shared by every UMBP runtime mode."""
def __init__(
self,
vllm_config: VllmConfig,
role: KVConnectorRole,
kv_cache_config: KVCacheConfig,
) -> None:
super().__init__(vllm_config, role, kv_cache_config)
topology = RankTopology.from_vllm_config(vllm_config)
runtime_config = UMBPRuntimeConfig.from_vllm(
vllm_config
).resolve_for_rank_count(topology.rank_count)
if topology.pp_size > 1:
raise NotImplementedError(
f"{runtime_config.mode} UMBP does not support pipeline parallelism"
)
runtime = UMBPRuntimeFactory.build(runtime_config)
namespace = UMBPNamespace.from_vllm_config(vllm_config, kv_cache_config)
codec = BlockIdentityCodec(
namespace=namespace,
tp_rank=topology.tp_rank,
pp_rank=topology.pp_rank,
pcp_rank=topology.pcp_rank,
dcp_rank=topology.dcp_rank,
)
layout = KVLayoutPlanner.from_kv_cache_config(kv_cache_config)
layout_descriptor = layout.describe(topology)
self._runtime = runtime
self._store_submitted = False
self.connector_scheduler: UMBPStoreConnectorScheduler | None = None
self.connector_worker: UMBPStoreConnectorWorker | None = None
if role == KVConnectorRole.SCHEDULER:
self.connector_scheduler = UMBPStoreConnectorScheduler(
vllm_config,
kv_cache_config,
runtime.create_scheduler_handle(
namespace.value, topology, layout_descriptor
),
codec,
topology,
)
else:
self.connector_worker = UMBPStoreConnectorWorker(
runtime.create_worker_handle(
namespace.value, topology, layout_descriptor
),
layout,
codec=codec,
)
def get_num_new_matched_tokens(
self, request: Request, num_computed_tokens: int
) -> tuple[int | None, bool]:
assert self.connector_scheduler is not None
return self.connector_scheduler.get_num_new_matched_tokens(
request, num_computed_tokens
)
def update_state_after_alloc(
self,
request: Request,
blocks: KVCacheBlocks,
num_external_tokens: int,
) -> None:
assert self.connector_scheduler is not None
self.connector_scheduler.update_state_after_alloc(
request, blocks, num_external_tokens
)
def bind_gpu_block_pool(self, gpu_block_pool: Any) -> None:
assert self.connector_scheduler is not None
self.connector_scheduler.bind_gpu_block_pool(gpu_block_pool)
def build_connector_meta(
self, scheduler_output: SchedulerOutput
) -> KVConnectorMetadata:
assert self.connector_scheduler is not None
return self.connector_scheduler.build_connector_meta(scheduler_output)
def request_finished(
self, request: Request, block_ids: list[int]
) -> tuple[bool, dict[str, Any] | None]:
assert self.connector_scheduler is not None
return self.connector_scheduler.request_finished(request, (block_ids,))
def update_connector_output(self, connector_output: Any) -> None:
assert self.connector_scheduler is not None
self.connector_scheduler.update_connector_output(connector_output)
def has_pending_push_work(self) -> bool:
assert self.connector_scheduler is not None
return self.connector_scheduler.has_pending_push_work()
def reset_cache(self) -> bool | None:
if self.connector_scheduler is None:
return None
return self.connector_scheduler.reset_store()
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]) -> None:
assert self.connector_worker is not None
self.connector_worker.register_kv_caches(kv_caches)
def bind_connector_metadata(self, connector_metadata: KVConnectorMetadata) -> None:
super().bind_connector_metadata(connector_metadata)
self._store_submitted = False
def clear_connector_metadata(self) -> None:
# No-forward steps skip the model runner's wait_for_save hook.
if self.connector_worker is not None and self.has_connector_metadata():
self.wait_for_save()
super().clear_connector_metadata()
def handle_preemptions(self, kv_connector_metadata: KVConnectorMetadata) -> None:
assert self.connector_worker is not None
assert isinstance(kv_connector_metadata, UMBPConnectorMetadata)
self.connector_worker.handle_preemptions(kv_connector_metadata)
def start_load_kv(self, forward_context: ForwardContext, **kwargs: Any) -> None:
del kwargs
assert self.connector_worker is not None
metadata = self._get_connector_metadata()
assert isinstance(metadata, UMBPConnectorMetadata)
self.connector_worker.start_load_kv(forward_context, metadata)
def wait_for_layer_load(self, layer_name: str) -> None:
assert self.connector_worker is not None
self.connector_worker.wait_for_layer_load(layer_name)
def save_kv_layer(
self,
layer_name: str,
kv_layer: torch.Tensor,
attn_metadata: AttentionMetadata,
**kwargs: Any,
) -> None:
# Stores are submitted once per step, from wait_for_save.
return
def wait_for_save(self) -> None:
if self._store_submitted:
return
assert self.connector_worker is not None
metadata = self._get_connector_metadata()
assert isinstance(metadata, UMBPConnectorMetadata)
self.connector_worker.enqueue_stores(metadata)
self.connector_worker.wait_for_save()
self._store_submitted = True
def get_finished(
self, finished_req_ids: set[str]
) -> tuple[set[str] | None, set[str] | None]:
assert self.connector_worker is not None
return self.connector_worker.get_finished(finished_req_ids)
def get_transfer_results(
self, finished_req_ids: set[str]
) -> KVConnectorTransferResults:
assert self.connector_worker is not None
finished_sending, finished_recving = self.connector_worker.get_finished(
finished_req_ids
)
return KVConnectorTransferResults(
finished_sending=set(finished_sending or ()),
finished_recving=set(finished_recving or ()),
)
def get_block_ids_with_load_errors(self) -> set[int]:
assert self.connector_worker is not None
return self.connector_worker.get_block_ids_with_load_errors()
def build_connector_worker_meta(self) -> UMBPConnectorWorkerMetadata:
assert self.connector_worker is not None
return self.connector_worker.build_connector_worker_meta()
def shutdown(self) -> None:
if self.connector_scheduler is not None:
self.connector_scheduler.close()
if self.connector_worker is not None:
self.connector_worker.close()
@@ -1,620 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Shared data types for the UMBP KV connector.
The types in this module deliberately do not depend on a UMBP client. They
describe vLLM's logical KV objects and the physical ranges a runtime adapter
will transfer.
"""
from __future__ import annotations
import hashlib
import json
from collections.abc import Sequence
from dataclasses import dataclass, field, replace
from enum import Enum
from typing import Any, NamedTuple
import torch
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorMetadata,
KVConnectorWorkerMetadata,
)
from vllm.v1.kv_cache_interface import KVCacheConfig
@dataclass(frozen=True)
class RankTopology:
"""Model-parallel ranks that participate in one KV object."""
tp_rank: int = 0
tp_size: int = 1
pp_rank: int = 0
pp_size: int = 1
pcp_rank: int = 0
pcp_size: int = 1
dcp_rank: int = 0
dcp_size: int = 1
def __post_init__(self) -> None:
for name in ("tp_size", "pp_size", "pcp_size", "dcp_size"):
if getattr(self, name) <= 0:
raise ValueError(f"{name} must be positive")
for rank, size, name in (
(self.tp_rank, self.tp_size, "tp_rank"),
(self.pp_rank, self.pp_size, "pp_rank"),
(self.pcp_rank, self.pcp_size, "pcp_rank"),
(self.dcp_rank, self.dcp_size, "dcp_rank"),
):
if rank < 0 or rank >= size:
raise ValueError(f"{name}={rank} is outside [0, {size})")
@classmethod
def from_vllm_config(cls, vllm_config: Any) -> RankTopology:
parallel = vllm_config.parallel_config
tp_size = getattr(parallel, "tensor_parallel_size", 1)
pp_size = getattr(parallel, "pipeline_parallel_size", 1)
pcp_size = getattr(parallel, "prefill_context_parallel_size", 1)
# An engine's workers are laid out as [PP, PCP, TP], TP innermost
# (initialize_model_parallel).
rank = getattr(parallel, "rank", 0)
tp_rank = rank % tp_size
dcp_size = getattr(parallel, "decode_context_parallel_size", 1)
return cls(
tp_rank=tp_rank,
tp_size=tp_size,
pp_rank=rank // (tp_size * pcp_size) % pp_size,
pp_size=pp_size,
pcp_rank=rank // tp_size % pcp_size,
pcp_size=pcp_size,
# DCP partitions the TP workers; it does not create a second
# Cartesian worker dimension. ParallelConfig is constructed before
# process groups, so derive this stable identity from TP rank.
dcp_rank=tp_rank % dcp_size,
dcp_size=dcp_size,
)
@property
def local_namespace(self) -> tuple[int, int, int, int]:
return (self.tp_rank, self.pcp_rank, self.dcp_rank, self.pp_rank)
@property
def rank_count(self) -> int:
return len(self.all_namespaces())
def all_namespaces(self) -> tuple[tuple[int, int, int, int], ...]:
if self.dcp_size > 1:
if self.tp_size % self.dcp_size:
raise ValueError("dcp_size must divide tp_size")
return tuple(
(tp, pcp, tp % self.dcp_size, pp)
for pp in range(self.pp_size)
for pcp in range(self.pcp_size)
for tp in range(self.tp_size)
)
return tuple(
(tp, pcp, dcp, pp)
for pp in range(self.pp_size)
for dcp in range(self.dcp_size)
for pcp in range(self.pcp_size)
for tp in range(self.tp_size)
)
@dataclass(frozen=True)
class UMBPNamespace:
"""Compatibility namespace for objects stored outside the engine."""
value: str
@classmethod
def from_vllm_config(
cls,
vllm_config: Any,
kv_cache_config: KVCacheConfig,
) -> UMBPNamespace:
extra = vllm_config.kv_transfer_config.kv_connector_extra_config
configured = extra.get("key_namespace", "auto")
if configured != "auto":
if not isinstance(configured, str) or not configured:
raise ValueError("key_namespace must be a non-empty string or 'auto'")
return cls(configured)
model_config = vllm_config.model_config
parallel_config = vllm_config.parallel_config
# Draft-model KV (e.g. DSpark/EAGLE layers) is stored alongside the
# target's, so a different drafter must not reuse it.
speculative_config = getattr(vllm_config, "speculative_config", None)
draft_model_config = getattr(speculative_config, "draft_model_config", None)
speculative = (
None
if speculative_config is None
else {
"method": getattr(speculative_config, "method", None),
"model": getattr(draft_model_config, "model", None),
"revision": getattr(draft_model_config, "revision", None),
}
)
fields = {
"model": getattr(model_config, "model", ""),
"revision": getattr(model_config, "revision", None),
"block_size": vllm_config.cache_config.block_size,
"cache_layout": getattr(kv_cache_config, "kv_cache_layout", None),
"groups": [
{
"layers": group.layer_names,
"block_size": group.kv_cache_spec.block_size,
"kind": type(group.kv_cache_spec).__name__,
"dtype": str(getattr(group.kv_cache_spec, "dtype", None)),
}
for group in kv_cache_config.kv_cache_groups
],
"tp": parallel_config.tensor_parallel_size,
"pp": parallel_config.pipeline_parallel_size,
"pcp": getattr(parallel_config, "prefill_context_parallel_size", 1),
"dcp": getattr(parallel_config, "decode_context_parallel_size", 1),
"layout_version": 1,
}
if speculative is not None:
# Added only when set, so namespaces without a drafter are unchanged.
fields["speculative"] = speculative
payload = json.dumps(fields, sort_keys=True, separators=(",", ":")).encode()
return cls(hashlib.sha256(payload).hexdigest()[:32])
@dataclass(frozen=True)
class BlockIdentityCodec:
"""Build stable rank-local keys from vLLM block hashes."""
namespace: UMBPNamespace
tp_rank: int = 0
pp_rank: int = 0
pcp_rank: int = 0
dcp_rank: int = 0
def key(self, block_hash: bytes, group_id: int = 0) -> str:
return self.key_for_namespace(
block_hash,
group_id,
(self.tp_rank, self.pcp_rank, self.dcp_rank, self.pp_rank),
)
def key_for_namespace(
self,
block_hash: bytes,
group_id: int,
namespace: tuple[int, int, int, int],
) -> str:
if not isinstance(block_hash, bytes) or not block_hash:
raise ValueError("block_hash must be non-empty bytes")
if group_id < 0:
raise ValueError("group_id must be non-negative")
tp_rank, pcp_rank, dcp_rank, pp_rank = namespace
return (
f"umbp:vllm:v1:{self.namespace.value}:"
f"tp{tp_rank}:pcp{pcp_rank}:dcp{dcp_rank}:"
f"pp{pp_rank}:g{group_id}:"
f"{block_hash.hex()}"
)
def keys_for_topology(
self,
block_hash: bytes,
topology: RankTopology,
group_ids: Sequence[int],
) -> tuple[str, ...]:
return tuple(
self.key_for_namespace(block_hash, group_id, namespace)
for namespace in topology.all_namespaces()
for group_id in group_ids
)
@dataclass(frozen=True)
class KVRegion:
"""One deterministic region in a rank-local KV object."""
layer_name: str
group_id: int
block_stride: int
block_bytes: int
object_offset: int
block_size: int = 1
def __post_init__(self) -> None:
if self.block_stride <= 0 or self.block_bytes <= 0:
raise ValueError("KV region sizes must be positive")
if self.block_size <= 0:
raise ValueError("KV region block_size must be positive")
if self.block_bytes > self.block_stride:
raise ValueError(
f"KV region bytes ({self.block_bytes}) exceed its stride "
f"({self.block_stride}) for {self.layer_name!r}"
)
@dataclass(frozen=True)
class KVLayoutDescriptor:
"""Backend-neutral description of one rank-local KV object layout."""
regions: tuple[KVRegion, ...]
topology: RankTopology
layout_format: str = "rank_local"
def __post_init__(self) -> None:
if not self.layout_format:
raise ValueError("layout_format must not be empty")
names = [region.layer_name for region in self.regions]
if len(names) != len(set(names)):
raise ValueError("KV layout contains duplicate layer regions")
@property
def object_size(self) -> int:
return sum(region.block_bytes for region in self.regions)
class KVLayoutPlanner:
"""Create a deterministic all-layer object layout from KVCacheConfig."""
def __init__(self, regions: Sequence[KVRegion]) -> None:
self.regions = tuple(regions)
self._regions_by_group = {
group_id: tuple(r for r in self.regions if r.group_id == group_id)
for group_id in {r.group_id for r in self.regions}
}
self._base_addresses: dict[str, int] = {}
self._physical_strides: dict[str, int] = {}
@classmethod
def from_kv_cache_config(cls, config: KVCacheConfig) -> KVLayoutPlanner:
group_by_layer = {
layer: group_id
for group_id, group in enumerate(config.kv_cache_groups)
for layer in group.layer_names
if group.enable_kv_transfer
}
regions: list[KVRegion] = []
offset = 0
for tensor in config.kv_cache_tensors:
for layer_name in tensor.layers:
group_id = group_by_layer.get(layer_name)
if group_id is None:
continue
spec = config.kv_cache_groups[group_id].kv_cache_spec
block_bytes = spec.page_size_bytes
regions.append(
KVRegion(
layer_name=layer_name,
group_id=group_id,
block_stride=tensor.block_stride,
block_bytes=block_bytes,
object_offset=offset,
block_size=spec.block_size,
)
)
offset += block_bytes
return cls(regions)
@property
def object_size(self) -> int:
return sum(region.block_bytes for region in self.regions)
def describe(
self,
topology: RankTopology,
layout_format: str = "rank_local",
) -> KVLayoutDescriptor:
return KVLayoutDescriptor(
regions=self.regions,
topology=topology,
layout_format=layout_format,
)
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]) -> None:
"""Capture stable base addresses for the planned KV regions."""
required = {region.layer_name for region in self.regions}
missing = required - kv_caches.keys()
if missing:
raise ValueError(f"missing KV cache layers: {sorted(missing)}")
addresses: dict[str, int] = {}
for layer_name in required:
cache = kv_caches[layer_name]
if cache.ndim == 0:
raise ValueError(f"KV cache {layer_name!r} must be a tensor")
block_stride = cache.stride(0) * cache.element_size()
addresses[layer_name] = cache.data_ptr()
for region in self.regions:
if region.layer_name != layer_name:
continue
if (
block_stride > region.block_stride
or region.block_stride % block_stride != 0
or (
region.block_bytes > block_stride
and region.block_bytes % block_stride != 0
)
):
raise ValueError(
f"KV block stride mismatch for {layer_name!r}: "
f"config={region.block_stride}, tensor={block_stride}"
)
self._physical_strides[layer_name] = block_stride
self._base_addresses = addresses
def plan_registered_block(
self,
key: str,
block_id: int,
*,
group_id: int | None = None,
request_id: str | None = None,
) -> BlockTransferPlan:
if not self._base_addresses:
raise RuntimeError("KV caches must be registered before planning")
return self.plan_for_block(
key,
block_id,
self._base_addresses,
group_id=group_id,
request_id=request_id,
)
def plan_for_block(
self,
key: str,
block_id: int,
base_addresses: dict[str, int],
*,
group_id: int | None = None,
request_id: str | None = None,
) -> BlockTransferPlan:
"""Lower one GPU block into a deterministic scatter/gather plan."""
return BlockTransferPlan(
key=key,
block_id=block_id,
ranges=self._ranges_for_block(block_id, base_addresses, group_id),
request_id=request_id,
)
def materialize(self, plan: BlockTransferPlan) -> BlockTransferPlan:
if not self._base_addresses:
raise RuntimeError("KV caches must be registered before planning")
return replace(
plan,
ranges=self._ranges_for_block(
plan.block_id, self._base_addresses, plan.group_id
),
)
def _ranges_for_block(
self,
block_id: int,
base_addresses: dict[str, int],
group_id: int | None,
) -> tuple[KVRange, ...]:
if block_id < 0:
raise ValueError("block_id must be non-negative")
regions = (
self.regions
if group_id is None
else self._regions_by_group.get(group_id, ())
)
if group_id is not None and not regions:
raise ValueError(f"KV layout does not contain cache group {group_id}")
ranges: list[KVRange] = []
object_offset = 0
for region in regions:
try:
base_address = base_addresses[region.layer_name]
except KeyError as exc:
raise ValueError(
f"missing base address for layer {region.layer_name!r}"
) from exc
physical_stride = self._physical_strides.get(
region.layer_name, region.block_stride
)
if region.block_bytes <= physical_stride:
ranges.append(
KVRange(
layer_name=region.layer_name,
group_id=region.group_id,
block_id=block_id,
base_address=base_address + block_id * physical_stride,
stride=physical_stride,
length=region.block_bytes,
object_offset=(
object_offset
if group_id is not None
else region.object_offset
),
)
)
object_offset += region.block_bytes
continue
physical_parts = max(1, region.block_bytes // physical_stride)
for part in range(physical_parts):
part_start = part * physical_stride
part_end = min(part_start + physical_stride, region.block_bytes)
physical_block_id = block_id * physical_parts + part
ranges.append(
KVRange(
layer_name=region.layer_name,
group_id=region.group_id,
block_id=block_id,
base_address=base_address + physical_block_id * physical_stride,
stride=physical_stride,
length=part_end - part_start,
object_offset=(
object_offset + part_start
if group_id is not None
else region.object_offset + part * physical_stride
),
)
)
object_offset += region.block_bytes
return tuple(ranges)
class KVRange(NamedTuple):
"""A source/destination range in a runtime transfer."""
layer_name: str
group_id: int
block_id: int
base_address: int
stride: int
length: int
object_offset: int
@dataclass(frozen=True)
class BlockTransferPlan:
"""One complete logical block transfer."""
key: str
block_id: int
ranges: tuple[KVRange, ...] = ()
request_id: str | None = None
group_id: int | None = None
@dataclass
class LoadSpec:
"""Scheduler decision for one externally loaded prefix."""
local_tokens: int
external_tokens: int
@property
def num_tokens_to_load(self) -> int:
return max(self.external_tokens - self.local_tokens, 0)
@dataclass
class RequestTracker:
"""Core request state shared by every runtime mode."""
saved_tokens: int = 0
block_ids: tuple[list[int], ...] = ()
load_spec: LoadSpec | None = None
def reset(self) -> None:
self.saved_tokens = 0
self.block_ids = ()
self.load_spec = None
def update_blocks(self, block_ids: tuple[list[int], ...]) -> None:
if not self.block_ids:
self.block_ids = tuple(list(group) for group in block_ids)
return
if len(self.block_ids) != len(block_ids):
raise ValueError("KV cache group count changed for a request")
for current, new in zip(self.block_ids, block_ids, strict=True):
if new[: len(current)] == current:
current.extend(new[len(current) :])
elif not new or current[-len(new) :] == new:
continue
else:
current.extend(new)
def replace_blocks(self, block_ids: tuple[list[int], ...]) -> None:
"""Replace stale pre-preemption tables with resumed full tables."""
self.block_ids = tuple(list(group) for group in block_ids)
self.saved_tokens = 0
def mark_saved(self, token_count: int, block_size: int) -> int:
"""Advance the save watermark only to a complete block boundary."""
complete_tokens = token_count // block_size * block_size
if complete_tokens < self.saved_tokens:
return self.saved_tokens
self.saved_tokens = complete_tokens
return complete_tokens
class TransferJobStatus(str, Enum):
PENDING = "pending"
RUNNING = "running"
COMPLETED = "completed"
FAILED = "failed"
@dataclass
class TransferJobState:
"""Per-key completion state; one failed key does not fail its siblings."""
plans: tuple[BlockTransferPlan, ...]
status: TransferJobStatus = TransferJobStatus.PENDING
completed_keys: set[str] = field(default_factory=set)
failed_keys: set[str] = field(default_factory=set)
error: str | None = None
@property
def keys(self) -> Sequence[str]:
return tuple(plan.key for plan in self.plans)
def start(self) -> None:
if self.status != TransferJobStatus.PENDING:
raise RuntimeError(f"cannot start job in state {self.status}")
self.status = TransferJobStatus.RUNNING
def complete(self, keys: Sequence[str] = ()) -> None:
self.completed_keys.update(keys or self.keys)
self._finish_if_done()
def fail(self, keys: Sequence[str], error: str | None = None) -> None:
self.failed_keys.update(keys)
self.error = error
self._finish_if_done()
@property
def failed_block_ids(self) -> set[int]:
return {plan.block_id for plan in self.plans if plan.key in self.failed_keys}
def _finish_if_done(self) -> None:
keys = set(self.keys)
if self.completed_keys | self.failed_keys >= keys:
self.status = (
TransferJobStatus.FAILED
if self.failed_keys
else TransferJobStatus.COMPLETED
)
@dataclass
class UMBPConnectorMetadata(KVConnectorMetadata):
"""Scheduler-to-worker metadata for one engine step."""
async_load: bool = False
store_plans: list[BlockTransferPlan] = field(default_factory=list)
load_requests: dict[str, list[BlockTransferPlan]] = field(default_factory=dict)
store_requests: dict[str, list[BlockTransferPlan]] = field(default_factory=dict)
preempted_request_ids: set[str] = field(default_factory=set)
store_event: int = -1
@dataclass
class StoreEventResult:
"""Number of workers that finished one store event."""
completed_workers: int = 0
def merge(self, other: StoreEventResult) -> None:
self.completed_workers += other.completed_workers
@dataclass
class UMBPConnectorWorkerMetadata(KVConnectorWorkerMetadata):
"""Worker completion metadata aggregated across ranks."""
store_events: dict[int, StoreEventResult] = field(default_factory=dict)
def aggregate(
self, other: KVConnectorWorkerMetadata
) -> UMBPConnectorWorkerMetadata:
if not isinstance(other, UMBPConnectorWorkerMetadata):
raise TypeError("cannot aggregate incompatible UMBP worker metadata")
for event, result in other.store_events.items():
self.store_events.setdefault(event, StoreEventResult()).merge(result)
return self
@@ -1,13 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from .base import IUMBPRuntime, UMBPSchedulerHandle, UMBPWorkerHandle
from .factory import UMBPRuntimeConfig, UMBPRuntimeFactory
__all__ = [
"IUMBPRuntime",
"UMBPSchedulerHandle",
"UMBPWorkerHandle",
"UMBPRuntimeConfig",
"UMBPRuntimeFactory",
]
@@ -1,69 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Backend-neutral runtime contract for UMBP."""
from __future__ import annotations
from collections.abc import Sequence
from typing import Protocol
import torch
from ..data import (
BlockTransferPlan,
KVLayoutDescriptor,
RankTopology,
TransferJobState,
)
class UMBPSchedulerHandle(Protocol):
def lookup(self, keys: Sequence[str]) -> Sequence[bool]:
"""Return authoritative existence for each key."""
def clear(self) -> bool:
"""Clear all objects visible to this scheduler handle."""
def close(self) -> None:
"""Release scheduler-side resources."""
class UMBPWorkerHandle(Protocol):
def register_buffers(self, kv_caches: dict[str, torch.Tensor]) -> None:
"""Register each unique KV allocation with the runtime."""
def load(self, plans: Sequence[BlockTransferPlan]) -> TransferJobState:
"""Start loading the requested plans."""
def store(self, plans: Sequence[BlockTransferPlan]) -> TransferJobState:
"""Start storing the requested plans."""
def wait(self, job: TransferJobState) -> TransferJobState:
"""Block until the job is final and its buffers are safe to reuse."""
def poll(self, job: TransferJobState) -> TransferJobState | None:
"""Return a finished job without blocking, or None if still pending."""
def publish(self, job: TransferJobState) -> None:
"""Accept a completed store; a runtime may defer visibility until then."""
def close(self) -> None:
"""Release worker-side resources."""
class IUMBPRuntime(Protocol):
def create_scheduler_handle(
self,
namespace: str,
topology: RankTopology,
layout: KVLayoutDescriptor,
) -> UMBPSchedulerHandle:
"""Create a scheduler-side handle."""
def create_worker_handle(
self,
namespace: str,
topology: RankTopology,
layout: KVLayoutDescriptor,
) -> UMBPWorkerHandle:
"""Create a worker-side handle."""
@@ -1,76 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Runtime configuration and adapter factory for UMBP."""
from __future__ import annotations
from collections.abc import Callable
from dataclasses import dataclass, replace
from typing import Any
from vllm.config import VllmConfig
from .base import IUMBPRuntime
@dataclass(frozen=True)
class UMBPRuntimeConfig:
"""Shared connector options passed to the selected runtime adapter."""
mode: str
options: dict[str, Any]
rank_count: int = 1
dp_index: int = 0
@classmethod
def from_vllm(cls, vllm_config: VllmConfig) -> UMBPRuntimeConfig:
transfer_config = vllm_config.kv_transfer_config
if transfer_config is None:
raise ValueError("UMBP requires kv_transfer_config")
options = dict(transfer_config.kv_connector_extra_config)
mode = options.get("mode", "embedded")
if mode not in {"embedded"}:
raise ValueError(f"unknown UMBP mode: {mode!r}")
namespace = options.get("key_namespace", "auto")
if namespace != "auto" and (not isinstance(namespace, str) or not namespace):
raise ValueError("key_namespace must be a non-empty string or 'auto'")
return cls(
mode=mode,
options=options,
dp_index=int(
getattr(vllm_config.parallel_config, "data_parallel_index", 0) or 0
),
)
def resolve_for_rank_count(self, rank_count: int) -> UMBPRuntimeConfig:
"""Pass topology to the adapter without interpreting its capacity options."""
if rank_count <= 0:
raise ValueError("rank_count must be positive")
return replace(self, rank_count=rank_count)
RuntimeBuilder = Callable[[UMBPRuntimeConfig], IUMBPRuntime]
class UMBPRuntimeFactory:
"""Registry used by UMBP adapters without importing MORI in vLLM."""
_builders: dict[str, RuntimeBuilder] = {}
@classmethod
def register(cls, mode: str, builder: RuntimeBuilder) -> None:
if mode not in {"embedded"}:
raise ValueError(f"unknown UMBP mode: {mode!r}")
if mode in cls._builders:
raise ValueError(f"UMBP runtime mode {mode!r} is already registered")
cls._builders[mode] = builder
@classmethod
def build(cls, config: UMBPRuntimeConfig) -> IUMBPRuntime:
try:
builder = cls._builders[config.mode]
except KeyError as exc:
raise RuntimeError(
f"no UMBP runtime adapter is registered for mode {config.mode!r}"
) from exc
return builder(config)
@@ -1,518 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Shared scheduler-side UMBP connector behavior."""
from __future__ import annotations
import logging
from collections.abc import Sequence
from dataclasses import dataclass, field
from functools import lru_cache
from typing import Any
from vllm.config import VllmConfig
from vllm.distributed.kv_transfer.kv_connector.v1.base import KVConnectorMetadata
from vllm.logger import init_logger
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
from vllm.v1.core.block_pool import BlockPool
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
from vllm.v1.core.kv_cache_utils import (
resolve_dcp_kv_block_size,
resolve_kv_cache_block_sizes,
)
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.kv_cache_interface import (
KVCacheConfig,
)
from vllm.v1.outputs import KVConnectorOutput
from vllm.v1.request import Request
from .data import (
BlockIdentityCodec,
BlockTransferPlan,
LoadSpec,
RankTopology,
RequestTracker,
StoreEventResult,
UMBPConnectorMetadata,
UMBPConnectorWorkerMetadata,
)
from .runtime import UMBPSchedulerHandle
logger = init_logger(__name__)
@lru_cache(maxsize=8192)
def _rank_keys(
codec: BlockIdentityCodec,
topology: RankTopology,
block_hash: bytes,
group_id: int,
) -> tuple[str, ...]:
"""Reuse immutable object identities, not runtime residency."""
return codec.keys_for_topology(block_hash, topology, (group_id,))
@dataclass
class _StoreEvent:
plans: tuple[BlockTransferPlan, ...]
result: StoreEventResult = field(default_factory=StoreEventResult)
class UMBPStoreConnectorScheduler:
"""Own vLLM prefix matching while the runtime owns lookup transport."""
def __init__(
self,
vllm_config: VllmConfig,
kv_cache_config: KVCacheConfig,
runtime: UMBPSchedulerHandle,
codec: BlockIdentityCodec,
topology: RankTopology | None = None,
) -> None:
self.block_size, resolved_hash_block_size = resolve_kv_cache_block_sizes(
kv_cache_config, vllm_config
)
self.kv_cache_config = kv_cache_config
dcp_size = vllm_config.parallel_config.decode_context_parallel_size
self.group_block_sizes = {
group_id: resolve_dcp_kv_block_size(group.kv_cache_spec, dcp_size)
for group_id, group in enumerate(kv_cache_config.kv_cache_groups)
}
self.runtime = runtime
self.codec = codec
self.topology = topology or RankTopology()
transfer_config = vllm_config.kv_transfer_config
assert transfer_config is not None
extra = transfer_config.kv_connector_extra_config
self.load_async = bool(extra.get("load_async", True))
self.enable_lookup = bool(extra.get("enable_lookup", True))
self.save_decode_cache = bool(extra.get("save_decode_cache", False))
# A kv_consumer only restores from the pool, as with other store
# connectors; it never offloads its own KV.
self.store_enabled = getattr(transfer_config, "kv_role", None) != "kv_consumer"
# Request.block_hashes are computed at this size; keys index into them.
self.hash_block_size = resolved_hash_block_size
speculative_config = getattr(vllm_config, "speculative_config", None)
self.use_eagle = bool(
speculative_config is not None and speculative_config.use_eagle_block_drop()
)
self._pending_loads: dict[str, list[BlockTransferPlan]] = {}
self._load_specs: dict[str, LoadSpec] = {}
self._lookup_units = {
group_id: self.group_block_sizes[group_id]
for group_id in kv_cache_config.prefix_cacheable_group_ids
}
self._requests: dict[str, Request] = {}
self._request_trackers: dict[str, RequestTracker] = {}
self._gpu_block_pool: BlockPool | None = None
self._store_events: dict[int, _StoreEvent] = {}
self._num_workers = getattr(vllm_config.parallel_config, "world_size", 1)
self._store_event_counter = 0
def bind_gpu_block_pool(self, gpu_block_pool: BlockPool) -> None:
self._gpu_block_pool = gpu_block_pool
def get_num_new_matched_tokens(
self, request: Request, num_computed_tokens: int
) -> tuple[int | None, bool]:
if not self.enable_lookup:
return 0, False
hashes = request.block_hashes
if not hashes or request.num_tokens < self.block_size:
return 0, False
if num_computed_tokens % self.block_size != 0:
return 0, False
group_ids = self.kv_cache_config.prefix_cacheable_group_ids
# vLLM must execute at least the final prompt token. Returning a hit
# for the entire prompt leaves the scheduler with no new token to run.
max_external_token = request.num_tokens - 1
if max_external_token <= num_computed_tokens:
self._load_specs.pop(request.request_id, None)
return 0, False
logical_objects_by_group, keys = self._lookup_objects(
request, num_computed_tokens, hashes
)
try:
hits = list(self.runtime.lookup(keys))
except Exception as exc:
logger.debug("UMBP lookup failed request=%s: %s", request.request_id, exc)
return 0, False
if len(hits) != len(keys):
logger.debug(
"UMBP lookup returned an invalid result length request=%s",
request.request_id,
)
return 0, False
per_rank = self.topology.rank_count
offset = 0
logical_hits_by_group: dict[int, list[bool]] = {}
for group_id in group_ids:
group_size = len(logical_objects_by_group[group_id]) * per_rank
group_hits = hits[offset : offset + group_size]
offset += group_size
logical_hits_by_group[group_id] = (
group_hits
if per_rank == 1
else [
all(group_hits[index : index + per_rank])
for index in range(0, len(group_hits), per_rank)
]
)
group_id = group_ids[0]
matched_blocks = 0
for hit in logical_hits_by_group[group_id]:
if not hit:
break
matched_blocks += 1
if self.use_eagle:
# As core's coordinator: the drafter recomputes the last block.
matched_blocks = max(0, matched_blocks - 1)
need_to_load = matched_blocks * self._lookup_units[group_id]
if logger.isEnabledFor(logging.DEBUG):
logger.debug(
"UMBP lookup result request=%s keys=%s group_hits=%s load_tokens=%d",
request.request_id,
keys,
logical_hits_by_group,
need_to_load,
)
matched_tokens = num_computed_tokens + need_to_load
if need_to_load <= 0:
return 0, False
self._load_specs[request.request_id] = LoadSpec(
local_tokens=num_computed_tokens,
external_tokens=matched_tokens,
)
return need_to_load, self.load_async
def _lookup_objects(
self, request: Request, local_tokens: int, hashes: Sequence[bytes]
) -> tuple[dict[int, list[tuple[bytes, tuple[str, ...]]]], list[str]]:
objects: dict[int, list[tuple[bytes, tuple[str, ...]]]] = {}
keys: list[str] = []
for group_id, unit in self._lookup_units.items():
group_objects = []
for token_end in range(local_tokens + unit, request.num_tokens, unit):
block_hash = self._object_hash_at_token_end(hashes, token_end)
rank_keys = _rank_keys(self.codec, self.topology, block_hash, group_id)
group_objects.append((block_hash, rank_keys))
keys.extend(rank_keys)
objects[group_id] = group_objects
return objects, keys
def update_state_after_alloc(
self,
request: Request,
blocks: KVCacheBlocks,
num_external_tokens: int,
) -> None:
self._requests[request.request_id] = request
block_groups = blocks.get_block_ids(
group_ids=self.kv_cache_config.prefix_cacheable_group_ids
)
if not block_groups:
return
tracker = self._tracker_for_request(request.request_id)
# Core passes the request's whole table on every admission. A request
# whose load failed is recomputed into newly allocated blocks.
tracker.replace_blocks(tuple(list(group) for group in block_groups))
if num_external_tokens <= 0:
self._pending_loads.pop(request.request_id, None)
self._load_specs.pop(request.request_id, None)
return
spec = self._load_specs.get(request.request_id)
if spec is not None:
tracker.load_spec = spec
# A request admitted without its load would wait for it forever.
assert num_external_tokens == spec.num_tokens_to_load, (
f"UMBP matched {spec.num_tokens_to_load} external tokens for "
f"request {request.request_id} but {num_external_tokens} were "
"allocated"
)
plans = self._load_plans_for_external_tokens(
request,
tracker,
block_groups,
num_external_tokens,
)
self._pending_loads[request.request_id] = plans
def build_connector_meta(
self, scheduler_output: SchedulerOutput
) -> KVConnectorMetadata:
meta = UMBPConnectorMetadata(async_load=self.load_async)
for request_id in scheduler_output.finished_req_ids:
self._pending_loads.pop(request_id, None)
self._load_specs.pop(request_id, None)
self._request_trackers.pop(request_id, None)
self._requests.pop(request_id, None)
for request in scheduler_output.scheduled_new_reqs:
load_plans = self._pending_loads.pop(request.req_id, [])
self._load_specs.pop(request.req_id, None)
if load_plans:
meta.load_requests[request.req_id] = load_plans
else:
tracker = self._tracker_for_request(request.req_id)
total_tokens = (
request.num_computed_tokens
+ scheduler_output.num_scheduled_tokens[request.req_id]
)
tracked_request = self._requests.get(request.req_id)
selected_block_ids = tuple(
request.block_ids[group_id]
for group_id in (self.kv_cache_config.prefix_cacheable_group_ids)
)
store_plans = (
[]
if not self.store_enabled or tracked_request is None
else self._store_plans(
tracked_request,
tracker,
total_tokens,
block_ids_override=selected_block_ids,
)
)
logger.debug(
"UMBP store planning request=%s tokens=%d plans=%d keys=%s",
request.req_id,
total_tokens,
len(store_plans),
tuple(plan.key for plan in store_plans),
)
meta.store_plans.extend(store_plans)
if store_plans:
meta.store_requests[request.req_id] = store_plans
block_state = getattr(scheduler_output, "kv_connector_block_state", None)
cached_reqs = scheduler_output.scheduled_cached_reqs
resumed_ids: set[str] = set(getattr(cached_reqs, "resumed_req_ids", ()))
for request_id in cached_reqs.req_ids:
load_plans = self._pending_loads.pop(request_id, [])
self._load_specs.pop(request_id, None)
if load_plans:
meta.load_requests[request_id] = load_plans
elif self.store_enabled:
cached_request = self._requests.get(request_id)
cached_tracker = self._request_trackers.get(request_id)
if cached_request is not None and cached_tracker is not None:
request_index = cached_reqs.req_ids.index(request_id)
num_computed_tokens = cached_reqs.num_computed_tokens[request_index]
# num_tokens counts sampled tokens, so a decode step is still
# short of it; only the prompt bounds the prefill.
is_prefill = num_computed_tokens < cached_request.num_prompt_tokens
if not is_prefill and not self.save_decode_cache:
continue
new_block_ids = cached_reqs.new_block_ids[request_index]
if new_block_ids:
selected = tuple(
new_block_ids[group_id]
for group_id in (
self.kv_cache_config.prefix_cacheable_group_ids
)
)
if request_id in resumed_ids:
cached_tracker.replace_blocks(selected)
else:
cached_tracker.update_blocks(selected)
# The appended table keeps blocks core has since freed, such
# as sliding-window blocks out of the window, which a store
# would read; core's current table does not.
current = (
block_state.get_block_ids(request_id)
if block_state is not None
else None
)
if current is not None:
cached_tracker.block_ids = tuple(
list(current[group_id])
for group_id in (
self.kv_cache_config.prefix_cacheable_group_ids
)
)
total_tokens = (
num_computed_tokens
+ scheduler_output.num_scheduled_tokens[request_id]
)
if not self.save_decode_cache:
total_tokens = min(
total_tokens, cached_request.num_prompt_tokens
)
store_plans = self._store_plans(
cached_request,
cached_tracker,
total_tokens,
block_ids_override=cached_tracker.block_ids,
)
meta.store_plans.extend(store_plans)
if store_plans:
meta.store_requests[request_id] = store_plans
# Async loads are admitted without scheduling model tokens, so those
# requests are absent from both scheduled request collections above.
# Forward their plans explicitly so workers can start the transfer and
# eventually move the request out of WAITING_FOR_REMOTE_KVS.
for request_id, load_plans in list(self._pending_loads.items()):
self._pending_loads.pop(request_id, None)
self._load_specs.pop(request_id, None)
if load_plans:
meta.load_requests[request_id] = load_plans
for request_id in scheduler_output.preempted_req_ids or set():
meta.preempted_request_ids.add(request_id)
self._load_specs.pop(request_id, None)
if preempted_tracker := self._request_trackers.get(request_id):
preempted_tracker.reset()
event_plans = tuple(meta.store_plans)
if event_plans:
event = self._store_event_counter
self._store_event_counter += 1
meta.store_event = event
self._store_events[event] = _StoreEvent(event_plans)
if self._gpu_block_pool is not None:
self._gpu_block_pool.touch(
[
self._gpu_block_pool.blocks[block_id]
for block_id in dict.fromkeys(
plan.block_id for plan in event_plans
)
]
)
return meta
def _tracker_for_request(self, request_id: str) -> RequestTracker:
return self._request_trackers.setdefault(request_id, RequestTracker())
def _load_plans_for_external_tokens(
self,
request: Any,
tracker: RequestTracker,
block_groups: tuple[list[int], ...],
num_external_tokens: int,
) -> list[BlockTransferPlan]:
"""Plan the logical objects selected by the external hit window."""
group_ids = self.kv_cache_config.prefix_cacheable_group_ids
spec = tracker.load_spec
local_tokens = spec.local_tokens if spec is not None else 0
end_tokens = local_tokens + num_external_tokens
plans: list[BlockTransferPlan] = []
for group_index, group_id in enumerate(group_ids):
group_block_size = self.group_block_sizes[group_id]
start_block = local_tokens // group_block_size
num_full_blocks = end_tokens // group_block_size
for block_index in range(start_block, num_full_blocks):
token_end = (block_index + 1) * group_block_size
block_hash = self._object_hash_at_token_end(
request.block_hashes, token_end
)
plans.append(
BlockTransferPlan(
key=self.codec.key(block_hash, group_id=group_id),
block_id=block_groups[group_index][block_index],
request_id=request.request_id,
group_id=group_id,
)
)
return plans
def _object_hash_at_token_end(
self, hashes: Sequence[bytes], token_end: int
) -> bytes:
hash_index = token_end // self.hash_block_size - 1
if hash_index < 0 or hash_index >= len(hashes):
raise ValueError(
f"request has {len(hashes)} hashes, needs index {hash_index}"
)
return hashes[hash_index]
def _store_plans(
self,
request: Any,
tracker: RequestTracker,
token_count: int,
block_ids_override: tuple[list[int], ...] | None = None,
) -> list[BlockTransferPlan]:
"""Describe full blocks produced by a scheduled prefill."""
request_id = getattr(request, "request_id", None) or request.req_id
group_ids = self.kv_cache_config.prefix_cacheable_group_ids
# Scheduled speculative tokens are not yet verified and have no block
# hash; a block is storable only once its hash exists.
token_count = min(token_count, len(request.block_hashes) * self.hash_block_size)
previous_saved_tokens = tracker.saved_tokens
# Hash-aligned rather than block-aligned: a group with smaller blocks
# than the largest must not wait for that group's block to fill.
save_to = tracker.mark_saved(token_count, self.hash_block_size)
plans: list[BlockTransferPlan] = []
block_groups = (
block_ids_override
if block_ids_override is not None
else tuple(request.block_ids[group_id] for group_id in group_ids)
)
block_ranges = {
group_id: range(
previous_saved_tokens // self.group_block_sizes[group_id],
min(save_to // self.group_block_sizes[group_id], len(block_ids)),
)
for group_id, block_ids in zip(group_ids, block_groups)
}
for group_id, block_ids in zip(group_ids, block_groups):
if group_id not in block_ranges:
continue
group_block_size = self.group_block_sizes[group_id]
for index in block_ranges[group_id]:
if block_ids[index] == NULL_BLOCK_ID:
continue
token_end = (index + 1) * group_block_size
block_hash = self._object_hash_at_token_end(
request.block_hashes, token_end
)
plans.append(
BlockTransferPlan(
key=self.codec.key(block_hash, group_id=group_id),
block_id=block_ids[index],
request_id=request_id,
group_id=group_id,
)
)
return plans
def request_finished(
self, request: Request, block_ids: tuple[list[int], ...]
) -> tuple[bool, dict[str, Any] | None]:
self._pending_loads.pop(request.request_id, None)
self._load_specs.pop(request.request_id, None)
self._requests.pop(request.request_id, None)
return False, None
def update_connector_output(self, output: KVConnectorOutput) -> None:
metadata = output.kv_connector_worker_meta
if not isinstance(metadata, UMBPConnectorWorkerMetadata):
return
pool = self._gpu_block_pool
for event_id, result in metadata.store_events.items():
event = self._store_events.get(event_id)
if event is None:
continue
event.result.merge(result)
if event.result.completed_workers < self._num_workers:
continue
del self._store_events[event_id]
if pool is not None:
block_ids = dict.fromkeys(plan.block_id for plan in event.plans)
pool.free_blocks(
pool.blocks[block_id] for block_id in reversed(block_ids)
)
def has_pending_push_work(self) -> bool:
return bool(self._store_events)
def reset_store(self) -> bool:
if self._store_events:
return False
self._load_specs.clear()
self._pending_loads.clear()
return self.runtime.clear()
def close(self) -> None:
self.runtime.close()
@@ -1,240 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Shared worker-side UMBP connector behavior."""
from __future__ import annotations
from dataclasses import dataclass, replace
import torch
from vllm.forward_context import ForwardContext
from vllm.logger import init_logger
from .data import (
BlockIdentityCodec,
BlockTransferPlan,
KVLayoutPlanner,
StoreEventResult,
TransferJobState,
TransferJobStatus,
UMBPConnectorMetadata,
UMBPConnectorWorkerMetadata,
)
from .runtime import UMBPWorkerHandle
logger = init_logger(__name__)
@dataclass
class _StoreBatch:
event_id: int
jobs: dict[str, TransferJobState]
class UMBPStoreConnectorWorker:
"""Translate shared metadata into runtime load/store jobs."""
def __init__(
self,
runtime: UMBPWorkerHandle,
layout: KVLayoutPlanner | None = None,
*,
codec: BlockIdentityCodec | None = None,
) -> None:
self.runtime = runtime
self.layout = layout
self.codec = codec
self._load_jobs: dict[str, dict[str | None, TransferJobState]] = {}
self._store_jobs: dict[str, TransferJobState] = {}
self._pending_stores: list[_StoreBatch] = []
self._worker_meta = UMBPConnectorWorkerMetadata()
self._finished_recving: set[str] = set()
self._load_error_block_ids: set[int] = set()
self._report_load_completions = False
self._active_store_event = -1
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]) -> None:
if self.layout is not None and self.layout.regions:
self.layout.register_kv_caches(kv_caches)
self.runtime.register_buffers(kv_caches)
def _materialize_plans(
self, plans: list[BlockTransferPlan]
) -> list[BlockTransferPlan]:
plans = [self._localize_plan(plan) for plan in plans]
if self.layout is None or not self.layout.regions:
return plans
materialized: list[BlockTransferPlan] = []
for plan in plans:
if plan.ranges:
materialized.append(plan)
continue
materialized.append(self.layout.materialize(plan))
return materialized
def _localize_plan(self, plan: BlockTransferPlan) -> BlockTransferPlan:
if self.codec is None or plan.group_id is None:
return plan
key = self._localize_key(plan.key, plan.group_id)
if key == plan.key:
return plan
return replace(plan, key=key)
def _localize_key(self, key: str, group_id: int) -> str:
if self.codec is None:
return key
try:
block_hash = bytes.fromhex(key.rsplit(":", 1)[1])
except (IndexError, ValueError):
return key
return self.codec.key(block_hash, group_id)
def start_load_kv(
self, forward_context: ForwardContext, metadata: UMBPConnectorMetadata
) -> None:
del forward_context
self._report_load_completions = metadata.async_load
for request_id, plans in metadata.load_requests.items():
if plans:
self._submit_loads(request_id, plans)
def _submit_loads(self, request_id: str, plans: list[BlockTransferPlan]) -> None:
materialized = self._materialize_plans(plans)
logger.debug(
"UMBP load submitted request=%s plans=%d ranges=%d bytes=%d",
request_id,
len(materialized),
sum(len(plan.ranges) for plan in materialized),
sum(item.length for plan in materialized for item in plan.ranges),
)
self._load_jobs[request_id] = {None: self.runtime.load(materialized)}
def _finish_job(
self, request_id: str, result: TransferJobState, *, is_load: bool
) -> bool:
operation = "load" if is_load else "store"
succeeded = result.status == TransferJobStatus.COMPLETED
if not succeeded:
logger.debug(
"UMBP %s failed request=%s status=%s error=%s",
operation,
request_id,
result.status.value,
result.error,
)
if is_load:
self._load_error_block_ids.update(result.failed_block_ids)
elif succeeded:
self.runtime.publish(result)
return succeeded
def wait_for_layer_load(self, layer_name: str) -> None:
if self._report_load_completions:
# Asynchronous loads fill blocks of requests that do not run until
# get_finished reports them, so no forward pass reads those blocks.
return
self._drain_load_jobs(wait=True)
def wait_for_save(self) -> None:
if self._store_jobs or self._active_store_event >= 0:
self._pending_stores.append(
_StoreBatch(self._active_store_event, self._store_jobs)
)
self._store_jobs = {}
self._active_store_event = -1
self._drain_store_jobs(wait=False)
def _drain_store_jobs(self, *, wait: bool) -> None:
for batch in list(self._pending_stores):
for request_id, job in list(batch.jobs.items()):
result = self.runtime.wait(job) if wait else self.runtime.poll(job)
if result is None:
continue
# A partly failed job is not published, including its stored keys.
self._finish_job(request_id, result, is_load=False)
del batch.jobs[request_id]
if not batch.jobs:
if batch.event_id >= 0:
self._worker_meta.store_events[batch.event_id] = StoreEventResult(
completed_workers=1
)
self._pending_stores.remove(batch)
def _submit_store_plans(
self, plans: list[BlockTransferPlan], request_id: str
) -> None:
if not plans:
return
materialized = self._materialize_plans(plans)
self._store_jobs[request_id] = self.runtime.store(materialized)
def enqueue_stores(self, metadata: UMBPConnectorMetadata) -> None:
self._active_store_event = metadata.store_event
store_requests = {
request_id: list(plans)
for request_id, plans in metadata.store_requests.items()
}
grouped = {plan for plans in store_requests.values() for plan in plans}
for plan in metadata.store_plans:
if plan not in grouped:
store_requests.setdefault(
plan.request_id or "__umbp_batch__", []
).append(plan)
grouped.add(plan)
for request_id, plans in store_requests.items():
self._submit_store_plans(plans, request_id)
def handle_preemptions(self, metadata: UMBPConnectorMetadata) -> None:
"""Finish request-local jobs before vLLM reuses their GPU blocks."""
preempted = metadata.preempted_request_ids
if not preempted:
return
for request_id in preempted:
for job in self._load_jobs.pop(request_id, {}).values():
self.runtime.wait(job)
for jobs in [self._store_jobs, *(batch.jobs for batch in self._pending_stores)]:
for request_id, job in jobs.items():
if request_id in preempted or any(
plan.request_id in preempted for plan in job.plans
):
jobs[request_id] = self.runtime.wait(job)
def get_finished(
self, finished_req_ids: set[str]
) -> tuple[set[str] | None, set[str] | None]:
del finished_req_ids
self._drain_load_jobs(wait=False)
self._drain_store_jobs(wait=False)
recving = set(self._finished_recving)
self._finished_recving.difference_update(recving)
return None, recving or None
def _drain_load_jobs(self, *, wait: bool) -> None:
for request_id, jobs in list(self._load_jobs.items()):
for layer, job in list(jobs.items()):
result = self.runtime.wait(job) if wait else self.runtime.poll(job)
if result is None:
continue
del jobs[layer]
self._finish_job(request_id, result, is_load=True)
if not jobs:
del self._load_jobs[request_id]
if self._report_load_completions:
self._finished_recving.add(request_id)
def build_connector_worker_meta(self) -> UMBPConnectorWorkerMetadata:
result = self._worker_meta
self._worker_meta = UMBPConnectorWorkerMetadata()
return result
def get_block_ids_with_load_errors(self) -> set[int]:
failed = self._load_error_block_ids
self._load_error_block_ids = set()
return failed
def close(self) -> None:
self.wait_for_save()
self._drain_store_jobs(wait=True)
self._drain_load_jobs(wait=True)
self.runtime.close()