[Mini-vLLM] Cut dead subsystems: models, entrypoints, backends, utils, tests
- Drop dead model modules, keeping files live imports require - Remove anthropic, cohere, codegen entrypoints and batch STT modes - Delete unused attention backends, MoE experts, and NPU/XPU util leaves - Strip dead third_party trees and ROCm leaves with no CUDA path - Remove mooncake EC connectors: dependency missing since branch base - Re-audit verified by full import sweep and Qwen3.5 smoke test Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -150,30 +150,12 @@ class KVConnectorFactory:
|
||||
# The registration should not be done in each individual file, as we want to
|
||||
# only load the files corresponding to the current connector.
|
||||
|
||||
KVConnectorFactory.register_connector(
|
||||
"ExampleConnector",
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.example_connector",
|
||||
"ExampleConnector",
|
||||
)
|
||||
|
||||
KVConnectorFactory.register_connector(
|
||||
"ExampleHiddenStatesConnector",
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.example_hidden_states_connector",
|
||||
"ExampleHiddenStatesConnector",
|
||||
)
|
||||
|
||||
KVConnectorFactory.register_connector(
|
||||
"LMCacheConnectorV1",
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.lmcache_connector",
|
||||
"LMCacheConnectorV1",
|
||||
)
|
||||
|
||||
KVConnectorFactory.register_connector(
|
||||
"LMCacheMPConnector",
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.lmcache_mp_connector",
|
||||
"LMCacheMPConnector",
|
||||
)
|
||||
|
||||
KVConnectorFactory.register_connector(
|
||||
"NixlConnector",
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.nixl",
|
||||
@@ -222,21 +204,6 @@ KVConnectorFactory.register_connector(
|
||||
"DecodeBenchConnector",
|
||||
)
|
||||
|
||||
KVConnectorFactory.register_connector(
|
||||
"MooncakeConnector",
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector",
|
||||
"MooncakeConnector",
|
||||
)
|
||||
KVConnectorFactory.register_connector(
|
||||
"MooncakeStoreConnector",
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.connector",
|
||||
"MooncakeStoreConnector",
|
||||
)
|
||||
KVConnectorFactory.register_connector(
|
||||
"FlexKVConnectorV1",
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.flexkv_connector",
|
||||
"FlexKVConnectorV1",
|
||||
)
|
||||
KVConnectorFactory.register_connector(
|
||||
"SimpleCPUOffloadConnector",
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.simple_cpu_offload_connector",
|
||||
|
||||
@@ -1,270 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Iterable
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorBase_V1,
|
||||
KVConnectorMetadata,
|
||||
KVConnectorRole,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import KVConnectorStats
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.outputs import KVConnectorOutput
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.distributed.kv_events import KVCacheEvent
|
||||
from vllm.forward_context import ForwardContext
|
||||
from vllm.v1.attention.backend import AttentionMetadata
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.request import Request
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
# FlexKV is a distributed KV Store and multi-level cache management system for
|
||||
# ultra-large-scale LLM inference.
|
||||
# GitHub: https://github.com/taco-project/FlexKV
|
||||
# Install: git clone git@github.com:taco-project/FlexKV.git \
|
||||
# && cd FlexKV && bash build.sh
|
||||
class FlexKVConnectorV1(KVConnectorBase_V1):
|
||||
"""KV Connector that offloads KV cache to FlexKV.
|
||||
|
||||
FlexKV is a distributed KV Store and multi-level cache management system
|
||||
designed for ultra-large-scale LLM inference. It supports offloading KV
|
||||
cache to CPU memory, SSD, and remote storage.
|
||||
|
||||
Installation:
|
||||
See https://github.com/taco-project/FlexKV for installation instructions.
|
||||
Quick start::
|
||||
|
||||
git clone git@github.com:taco-project/FlexKV.git
|
||||
cd FlexKV && bash build.sh
|
||||
|
||||
Configuration:
|
||||
Pass ``kv_connector="FlexKVConnectorV1"`` via ``--kv-transfer-config``::
|
||||
|
||||
--kv-transfer-config \
|
||||
'{"kv_connector":"FlexKVConnectorV1","kv_role":"kv_both"}'
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: "VllmConfig",
|
||||
role: KVConnectorRole,
|
||||
kv_cache_config: "KVCacheConfig",
|
||||
):
|
||||
super().__init__(
|
||||
vllm_config=vllm_config, role=role, kv_cache_config=kv_cache_config
|
||||
)
|
||||
try:
|
||||
from flexkv.integration.vllm.vllm_v1_adapter import FlexKVConnectorV1Impl
|
||||
except ImportError as e:
|
||||
raise ImportError(
|
||||
"FlexKV is not installed. Please install it to use "
|
||||
"FlexKVConnectorV1. See https://github.com/taco-project/FlexKV "
|
||||
"for installation instructions."
|
||||
) from e
|
||||
|
||||
self._flexkv_connector = FlexKVConnectorV1Impl(vllm_config, role)
|
||||
|
||||
def shutdown(self):
|
||||
self._flexkv_connector.shutdown()
|
||||
|
||||
# ==============================
|
||||
# Worker-side methods
|
||||
# ==============================
|
||||
def start_load_kv(self, forward_context: "ForwardContext", **kwargs) -> None:
|
||||
"""No-op for FlexKV (currently).
|
||||
|
||||
FlexKV manages all KV transfers on the **scheduler side** via
|
||||
``build_connector_meta`` (which calls ``launch_tasks``) and
|
||||
``update_connector_output`` (which polls ``query_finished_task``).
|
||||
KV blocks are transferred directly between the FlexKV server and
|
||||
vLLM's GPU memory without worker-side intervention during the
|
||||
forward pass — similar to how NIXL operates.
|
||||
|
||||
These worker-side hooks are kept (rather than omitted) to satisfy
|
||||
the ``KVConnectorBase_V1`` interface contract and to serve as
|
||||
extension points for a future worker-side layer-pipelining path.
|
||||
|
||||
Args:
|
||||
forward_context (ForwardContext): the forward context.
|
||||
**kwargs (Any): additional arguments (unused).
|
||||
|
||||
"""
|
||||
self._flexkv_connector.start_load_kv(forward_context, **kwargs)
|
||||
|
||||
def wait_for_layer_load(self, layer_name: str) -> None:
|
||||
"""No-op for FlexKV (currently).
|
||||
|
||||
FlexKV manages all KV transfers on the scheduler side.
|
||||
This hook is retained for ``KVConnectorBase_V1`` API compatibility.
|
||||
|
||||
Args:
|
||||
layer_name: the name of the layer (unused).
|
||||
|
||||
"""
|
||||
self._flexkv_connector.wait_for_layer_load(layer_name)
|
||||
|
||||
def save_kv_layer(
|
||||
self,
|
||||
layer_name: str,
|
||||
kv_layer: torch.Tensor,
|
||||
attn_metadata: "AttentionMetadata",
|
||||
**kwargs,
|
||||
) -> None:
|
||||
"""No-op for FlexKV (currently).
|
||||
|
||||
FlexKV offloads KV cache asynchronously from the scheduler side
|
||||
after a request finishes (see ``request_finished``). It does not
|
||||
intercept individual layer tensors during the forward pass.
|
||||
|
||||
This hook is retained to satisfy ``KVConnectorBase_V1`` and as an
|
||||
extension point for future per-layer async offload support.
|
||||
|
||||
Args:
|
||||
layer_name (str): the name of the layer (unused).
|
||||
kv_layer (torch.Tensor): the paged KV buffer (unused).
|
||||
attn_metadata (AttentionMetadata): the attention metadata (unused).
|
||||
**kwargs (Any): additional arguments (unused).
|
||||
|
||||
"""
|
||||
self._flexkv_connector.save_kv_layer(
|
||||
layer_name, kv_layer, attn_metadata, **kwargs
|
||||
)
|
||||
|
||||
def wait_for_save(self):
|
||||
"""No-op for FlexKV (currently).
|
||||
|
||||
KV offload tasks are tracked asynchronously by the scheduler
|
||||
connector via ``request_finished`` / ``query_finished_task``.
|
||||
There is no pending worker-side save to wait for at
|
||||
forward-context exit.
|
||||
|
||||
Retained to satisfy ``KVConnectorBase_V1`` and as an extension
|
||||
point for future worker-side save-completion signalling.
|
||||
"""
|
||||
self._flexkv_connector.wait_for_save()
|
||||
|
||||
def get_finished(
|
||||
self, finished_req_ids: set[str]
|
||||
) -> tuple[set[str] | None, set[str] | None]:
|
||||
"""Notify worker-side connector of requests that have finished
|
||||
generating tokens.
|
||||
|
||||
Returns:
|
||||
Tuple of (sending/saving ids, recving/loading ids) for requests
|
||||
that have finished asynchronous transfer. The finished saves/sends
|
||||
req ids must belong to a set provided in a call to this method
|
||||
(this call or a prior one).
|
||||
|
||||
"""
|
||||
return self._flexkv_connector.get_finished(finished_req_ids)
|
||||
|
||||
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
|
||||
"""Initialize with the KV caches. Useful for pre-registering the
|
||||
KV caches in the KVConnector (e.g. for NIXL).
|
||||
|
||||
Args:
|
||||
kv_caches: dictionary of layer names to kv cache tensors.
|
||||
|
||||
"""
|
||||
self._flexkv_connector.register_kv_caches(kv_caches)
|
||||
|
||||
# ==============================
|
||||
# Scheduler-side methods
|
||||
# ==============================
|
||||
def get_num_new_matched_tokens(
|
||||
self,
|
||||
request: "Request",
|
||||
num_computed_tokens: int,
|
||||
) -> tuple[int, bool]:
|
||||
"""Get the number of new tokens that can be loaded from the
|
||||
external KV cache beyond ``num_computed_tokens``.
|
||||
|
||||
Args:
|
||||
request (Request): the request object.
|
||||
num_computed_tokens (int): the number of locally computed
|
||||
tokens for this request.
|
||||
|
||||
Returns:
|
||||
Tuple of (num_external_tokens, is_ready) where
|
||||
num_external_tokens is the number of additional tokens that
|
||||
can be loaded from the external KV cache.
|
||||
|
||||
"""
|
||||
return self._flexkv_connector.get_num_new_matched_tokens(
|
||||
request, num_computed_tokens
|
||||
)
|
||||
|
||||
def update_state_after_alloc(
|
||||
self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int
|
||||
):
|
||||
"""Update KVConnector state after block allocation."""
|
||||
self._flexkv_connector.update_state_after_alloc(
|
||||
request, blocks, num_external_tokens
|
||||
)
|
||||
|
||||
def build_connector_meta(
|
||||
self, scheduler_output: SchedulerOutput
|
||||
) -> KVConnectorMetadata:
|
||||
"""Build the connector metadata for this step.
|
||||
|
||||
This function should NOT modify fields in the scheduler_output.
|
||||
Also, calling this function will reset the state of the connector.
|
||||
|
||||
Args:
|
||||
scheduler_output (SchedulerOutput): the scheduler output object.
|
||||
|
||||
"""
|
||||
return self._flexkv_connector.build_connector_meta(scheduler_output)
|
||||
|
||||
def update_connector_output(self, connector_output: KVConnectorOutput):
|
||||
"""Update KVConnector state from worker-side connectors output.
|
||||
|
||||
Args:
|
||||
connector_output (KVConnectorOutput): the worker-side
|
||||
connectors output.
|
||||
|
||||
"""
|
||||
self._flexkv_connector.update_connector_output(connector_output)
|
||||
|
||||
def request_finished(
|
||||
self,
|
||||
request: "Request",
|
||||
block_ids: list[int],
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
"""Called when a request has finished, before its blocks are freed.
|
||||
|
||||
Returns:
|
||||
Tuple of (async_save, kv_transfer_params) where async_save is
|
||||
True if the request is being saved/sent asynchronously and blocks
|
||||
should not be freed until the request_id is returned from
|
||||
:meth:`get_finished`. kv_transfer_params is an optional dict of
|
||||
KVTransferParams to be included in the request outputs.
|
||||
|
||||
"""
|
||||
return self._flexkv_connector.request_finished(request, block_ids)
|
||||
|
||||
def take_events(self) -> Iterable["KVCacheEvent"]:
|
||||
"""Collect buffered KV cache events.
|
||||
|
||||
Returns:
|
||||
New KV cache events since the last call.
|
||||
|
||||
"""
|
||||
return self._flexkv_connector.take_events()
|
||||
|
||||
def get_kv_connector_stats(self) -> KVConnectorStats | None:
|
||||
"""Get the KV connector stats collected during the last interval."""
|
||||
return self._flexkv_connector.get_kv_connector_stats()
|
||||
|
||||
def get_block_ids_with_load_errors(self) -> set[int]:
|
||||
"""Get the block ids that have failed to load."""
|
||||
return self._flexkv_connector.get_block_ids_with_load_errors()
|
||||
@@ -1,358 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Iterable
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.kv_events import (
|
||||
BlockStored,
|
||||
KVCacheEvent,
|
||||
KVConnectorKVEvents,
|
||||
KVEventAggregator,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorBase_V1,
|
||||
KVConnectorMetadata,
|
||||
KVConnectorRole,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.attention.backend import AttentionMetadata
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.outputs import KVConnectorOutput
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.forward_context import ForwardContext
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.request import Request
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class LMCacheKVEvents(KVConnectorKVEvents):
|
||||
"""Concrete implementation of KVConnectorKVEvents using KVEventAggregator."""
|
||||
|
||||
def __init__(self, num_workers: int) -> None:
|
||||
self._aggregator = KVEventAggregator(num_workers)
|
||||
|
||||
def add_events(self, events: list[KVCacheEvent]) -> None:
|
||||
self._aggregator.add_events(events)
|
||||
|
||||
def aggregate(self) -> "LMCacheKVEvents":
|
||||
"""Aggregate KV events and retain only common events."""
|
||||
common_events = self._aggregator.get_common_events()
|
||||
self._aggregator.clear_events()
|
||||
self._aggregator.add_events(common_events)
|
||||
self._aggregator.reset_workers()
|
||||
return self
|
||||
|
||||
def increment_workers(self, count: int = 1) -> None:
|
||||
self._aggregator.increment_workers(count)
|
||||
|
||||
def get_all_events(self) -> list[KVCacheEvent]:
|
||||
return self._aggregator.get_all_events()
|
||||
|
||||
def get_number_of_workers(self) -> int:
|
||||
return self._aggregator.get_number_of_workers()
|
||||
|
||||
def clear_events(self) -> None:
|
||||
self._aggregator.clear_events()
|
||||
self._aggregator.reset_workers()
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"<LMCacheKVEvents events={self.get_all_events()}>"
|
||||
|
||||
|
||||
class LMCacheConnectorV1(KVConnectorBase_V1):
|
||||
@classmethod
|
||||
def requires_piecewise_for_cudagraph(cls, extra_config: dict[str, Any]) -> bool:
|
||||
"""LMCache requires PIECEWISE CUDA graph mode when layerwise
|
||||
operations are enabled. The wait_for_layer_load and save_kv_layer
|
||||
methods perform actual async synchronization that cannot be
|
||||
captured in CUDA graphs.
|
||||
"""
|
||||
return extra_config.get("use_layerwise", False)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: "VllmConfig",
|
||||
role: KVConnectorRole,
|
||||
kv_cache_config: "KVCacheConfig",
|
||||
):
|
||||
super().__init__(
|
||||
vllm_config=vllm_config, role=role, kv_cache_config=kv_cache_config
|
||||
)
|
||||
assert vllm_config.kv_transfer_config is not None
|
||||
use_native = vllm_config.kv_transfer_config.get_from_extra_config(
|
||||
"use_native", False
|
||||
)
|
||||
if use_native:
|
||||
logger.info("Initializing native LMCache connector")
|
||||
# lazy import
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1 import lmcache_integration
|
||||
|
||||
_adapter = lmcache_integration.vllm_v1_adapter
|
||||
|
||||
cls = _adapter.LMCacheConnectorV1Impl
|
||||
else:
|
||||
logger.info("Initializing latest dev LMCache connector")
|
||||
# lazy import
|
||||
from lmcache.integration.vllm.vllm_v1_adapter import (
|
||||
LMCacheConnectorV1Impl as LMCacheConnectorLatestImpl,
|
||||
)
|
||||
|
||||
cls = LMCacheConnectorLatestImpl
|
||||
|
||||
self._lmcache_engine = cls(vllm_config, role, self)
|
||||
|
||||
self._kv_cache_events: LMCacheKVEvents | None = None
|
||||
|
||||
# ==============================
|
||||
# Worker-side methods
|
||||
# ==============================
|
||||
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
|
||||
"""Initialize with the KV caches. Useful for pre-registering the
|
||||
KV Caches in the KVConnector (e.g. for NIXL).
|
||||
|
||||
Args:
|
||||
kv_caches: dictionary of layer names, kv cache
|
||||
|
||||
"""
|
||||
if hasattr(self._lmcache_engine, "register_kv_caches"):
|
||||
self._lmcache_engine.register_kv_caches(kv_caches)
|
||||
else:
|
||||
logger.warning(
|
||||
"LMCache engine does not support register_kv_caches, "
|
||||
"please check and use the latest version"
|
||||
)
|
||||
|
||||
def bind_connector_metadata(self, connector_metadata: KVConnectorMetadata) -> None:
|
||||
super().bind_connector_metadata(connector_metadata)
|
||||
engine = self._lmcache_engine
|
||||
if hasattr(engine, "bind_connector_metadata"):
|
||||
engine.bind_connector_metadata(connector_metadata)
|
||||
else:
|
||||
# Compat for engines predating the bind hook: start_load_kv may
|
||||
# run after the forward (SchedulerOutput.has_sync_kv_loads), so
|
||||
# the per-step layerwise state it used to reset must be reset
|
||||
# here. Remove once LMCache ships bind_connector_metadata.
|
||||
if hasattr(engine, "current_layer"):
|
||||
engine.current_layer = 0
|
||||
if hasattr(engine, "layerwise_retrievers"):
|
||||
engine.layerwise_retrievers = []
|
||||
engine.layerwise_storers = []
|
||||
|
||||
def start_load_kv(self, forward_context: "ForwardContext", **kwargs: Any) -> None:
|
||||
"""Start loading the KV cache from the connector to vLLM's paged
|
||||
KV buffer. This is called from the forward context before the
|
||||
forward pass to enable async loading during model execution.
|
||||
|
||||
Args:
|
||||
forward_context (ForwardContext): the forward context.
|
||||
**kwargs: additional arguments for the load operation
|
||||
|
||||
Note:
|
||||
The number of elements in kv_caches and layer_names should be
|
||||
the same.
|
||||
|
||||
"""
|
||||
self._lmcache_engine.start_load_kv(forward_context, **kwargs)
|
||||
|
||||
def wait_for_layer_load(self, layer_name: str) -> None:
|
||||
"""Block until the KV for a specific layer is loaded into vLLM's
|
||||
paged buffer. This is called from within attention layer to ensure
|
||||
async copying from start_load_kv is complete.
|
||||
|
||||
This interface will be useful for layer-by-layer pipelining.
|
||||
|
||||
Args:
|
||||
layer_name: the name of that layer
|
||||
|
||||
"""
|
||||
self._lmcache_engine.wait_for_layer_load(layer_name)
|
||||
|
||||
def save_kv_layer(
|
||||
self,
|
||||
layer_name: str,
|
||||
kv_layer: torch.Tensor,
|
||||
attn_metadata: AttentionMetadata,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
"""Start saving the a layer of KV cache from vLLM's paged buffer
|
||||
to the connector. This is called from within attention layer to
|
||||
enable async copying during execution.
|
||||
|
||||
Args:
|
||||
layer_name (str): the name of the layer.
|
||||
kv_layer (torch.Tensor): the paged KV buffer of the current
|
||||
layer in vLLM.
|
||||
attn_metadata (AttentionMetadata): the attention metadata.
|
||||
**kwargs: additional arguments for the save operation.
|
||||
|
||||
"""
|
||||
self._lmcache_engine.save_kv_layer(
|
||||
layer_name, kv_layer, attn_metadata, **kwargs
|
||||
)
|
||||
|
||||
def wait_for_save(self):
|
||||
"""Block until all the save operations is done. This is called
|
||||
as the forward context exits to ensure that the async saving
|
||||
from save_kv_layer is complete before finishing the forward.
|
||||
|
||||
This prevents overwrites of paged KV buffer before saving done.
|
||||
"""
|
||||
self._lmcache_engine.wait_for_save()
|
||||
|
||||
def get_finished(
|
||||
self, finished_req_ids: set[str]
|
||||
) -> tuple[set[str] | None, set[str] | None]:
|
||||
"""Notifies worker-side connector ids of requests that have
|
||||
finished generating tokens.
|
||||
|
||||
Returns:
|
||||
ids of requests that have finished asynchronous transfer
|
||||
(requests that previously returned True from request_finished()),
|
||||
tuple of (sending/saving ids, recving/loading ids).
|
||||
The finished saves/sends req ids must belong to a set provided in a
|
||||
call to this method (this call or a prior one).
|
||||
|
||||
"""
|
||||
return self._lmcache_engine.get_finished(finished_req_ids)
|
||||
|
||||
def get_block_ids_with_load_errors(self) -> set[int]:
|
||||
"""Get the set of block IDs that failed to load.
|
||||
|
||||
Returns:
|
||||
Set of block IDs that encountered load errors.
|
||||
Empty set if no load errors occurred.
|
||||
|
||||
"""
|
||||
method = getattr(self._lmcache_engine, "get_block_ids_with_load_errors", None)
|
||||
if callable(method):
|
||||
return method()
|
||||
|
||||
# Fallback for older versions that don't support this method
|
||||
return set()
|
||||
|
||||
def get_kv_connector_kv_cache_events(self) -> LMCacheKVEvents | None:
|
||||
"""Get the KV connector kv cache events collected during the last interval."""
|
||||
events = self._lmcache_engine.get_kv_events() # type: ignore [attr-defined]
|
||||
if not events:
|
||||
return None
|
||||
|
||||
blocks: list[BlockStored] = [
|
||||
BlockStored(
|
||||
block_hashes=e.block_hashes,
|
||||
parent_block_hash=e.parent_block_hash,
|
||||
token_ids=e.token_ids,
|
||||
lora_id=e.lora_id,
|
||||
block_size=e.block_size,
|
||||
medium=e.medium,
|
||||
lora_name=getattr(e, "lora_name", None),
|
||||
)
|
||||
for e in events
|
||||
]
|
||||
|
||||
lmcache_kv_events = LMCacheKVEvents(num_workers=1)
|
||||
lmcache_kv_events.add_events(blocks)
|
||||
return lmcache_kv_events
|
||||
|
||||
# ==============================
|
||||
# Scheduler-side methods
|
||||
# ==============================
|
||||
def get_num_new_matched_tokens(
|
||||
self,
|
||||
request: "Request",
|
||||
num_computed_tokens: int,
|
||||
) -> tuple[int | None, bool]:
|
||||
"""Get number of new tokens that can be loaded from the
|
||||
external KV cache beyond the num_computed_tokens.
|
||||
|
||||
Args:
|
||||
request (Request): the request object.
|
||||
num_computed_tokens (int): the number of locally
|
||||
computed tokens for this request
|
||||
|
||||
Returns:
|
||||
the number of tokens that can be loaded from the
|
||||
external KV cache beyond what is already computed.
|
||||
|
||||
"""
|
||||
return self._lmcache_engine.get_num_new_matched_tokens(
|
||||
request, num_computed_tokens
|
||||
), False
|
||||
|
||||
def update_state_after_alloc(
|
||||
self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int
|
||||
):
|
||||
"""Update KVConnector state after block allocation."""
|
||||
self._lmcache_engine.update_state_after_alloc(request, num_external_tokens)
|
||||
|
||||
def build_connector_meta(
|
||||
self, scheduler_output: SchedulerOutput
|
||||
) -> KVConnectorMetadata:
|
||||
"""Build the connector metadata for this step.
|
||||
|
||||
This function should NOT modify fields in the scheduler_output.
|
||||
Also, calling this function will reset the state of the connector.
|
||||
|
||||
Args:
|
||||
scheduler_output (SchedulerOutput): the scheduler output object.
|
||||
|
||||
"""
|
||||
return self._lmcache_engine.build_connector_meta(scheduler_output)
|
||||
|
||||
def update_connector_output(self, connector_output: KVConnectorOutput):
|
||||
"""Update KVConnector state from worker-side connectors output.
|
||||
|
||||
Args:
|
||||
connector_output (KVConnectorOutput): the worker-side
|
||||
connectors output.
|
||||
|
||||
"""
|
||||
# Get the KV events
|
||||
kv_cache_events = connector_output.kv_cache_events
|
||||
if not kv_cache_events or not isinstance(kv_cache_events, LMCacheKVEvents):
|
||||
return
|
||||
|
||||
if self._kv_cache_events is None:
|
||||
self._kv_cache_events = kv_cache_events
|
||||
else:
|
||||
self._kv_cache_events.add_events(kv_cache_events.get_all_events())
|
||||
self._kv_cache_events.increment_workers(
|
||||
kv_cache_events.get_number_of_workers()
|
||||
)
|
||||
return
|
||||
|
||||
def request_finished(
|
||||
self,
|
||||
request: "Request",
|
||||
block_ids: list[int],
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
"""Called when a request has finished, before its blocks are freed.
|
||||
|
||||
Returns:
|
||||
True if the request is being saved/sent asynchronously and blocks
|
||||
should not be freed until the request_id is returned from
|
||||
get_finished().
|
||||
Optional KVTransferParams to be included in the request outputs
|
||||
returned by the engine.
|
||||
|
||||
"""
|
||||
return self._lmcache_engine.request_finished(request, block_ids)
|
||||
|
||||
def take_events(self) -> Iterable["KVCacheEvent"]:
|
||||
"""Take the KV cache events from the connector.
|
||||
|
||||
Yields:
|
||||
New KV cache events since the last call.
|
||||
|
||||
"""
|
||||
if self._kv_cache_events is not None:
|
||||
self._kv_cache_events.aggregate()
|
||||
kv_cache_events = self._kv_cache_events.get_all_events()
|
||||
yield from kv_cache_events
|
||||
self._kv_cache_events.clear_events()
|
||||
self._kv_cache_events = None
|
||||
@@ -1,20 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
from . import multi_process_adapter, vllm_v1_adapter
|
||||
from .multi_process_adapter import (
|
||||
LMCacheMPSchedulerAdapter,
|
||||
LMCacheMPWorkerAdapter,
|
||||
LoadStoreOp,
|
||||
ParallelStrategy,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"vllm_v1_adapter",
|
||||
"multi_process_adapter",
|
||||
"LMCacheMPSchedulerAdapter",
|
||||
"LMCacheMPWorkerAdapter",
|
||||
"LoadStoreOp",
|
||||
"ParallelStrategy",
|
||||
]
|
||||
-735
@@ -1,735 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import os
|
||||
from collections.abc import Iterable
|
||||
from dataclasses import dataclass
|
||||
from itertools import islice
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
import zmq
|
||||
from lmcache.utils import _lmcache_nvtx_annotate, init_logger
|
||||
from lmcache.v1.multiprocess.custom_types import (
|
||||
CudaIPCWrapper,
|
||||
IPCCacheEngineKey,
|
||||
KVCache,
|
||||
)
|
||||
from lmcache.v1.multiprocess.mq import MessageQueueClient, MessagingFuture
|
||||
from lmcache.v1.multiprocess.protocol import RequestType, get_response_class
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def wrap_kv_caches(kv_caches: dict[str, torch.Tensor]) -> KVCache:
|
||||
logger.info("KV caches keys are %s", list(kv_caches.keys()))
|
||||
return [CudaIPCWrapper(tensor) for tensor in kv_caches.values()]
|
||||
|
||||
|
||||
def striding_block_hashes(
|
||||
block_hashes: list[bytes], blocks_in_chunk: int
|
||||
) -> Iterable[bytes]:
|
||||
"""Extract chunk-level hashes from block hashes by striding.
|
||||
|
||||
In hash-based vLLM, each vLLM block has its own hash. LMCache chunks
|
||||
span ``blocks_in_chunk`` consecutive blocks. The representative hash
|
||||
for a chunk is the hash of the **last** block in that chunk (because
|
||||
each block hash already encodes its prefix). So we start at index
|
||||
``blocks_in_chunk - 1`` and stride by ``blocks_in_chunk``.
|
||||
"""
|
||||
return islice(block_hashes, blocks_in_chunk - 1, None, blocks_in_chunk)
|
||||
|
||||
|
||||
def send_lmcache_request(
|
||||
mq_client: MessageQueueClient,
|
||||
request_type: RequestType,
|
||||
payloads: list[Any],
|
||||
) -> MessagingFuture[Any]:
|
||||
"""Helper function to send the request to the LMCache multiprocess server.
|
||||
|
||||
Args:
|
||||
mq_client: The LMCache multiprocess mode message queue client
|
||||
request_type: The request type
|
||||
payloads: The request payloads
|
||||
|
||||
Returns:
|
||||
A messaging future for the request
|
||||
|
||||
"""
|
||||
future = mq_client.submit_request(
|
||||
request_type, payloads, get_response_class(request_type)
|
||||
)
|
||||
return future
|
||||
|
||||
|
||||
def get_lmcache_chunk_size(
|
||||
mq_client: MessageQueueClient,
|
||||
) -> int:
|
||||
"""Helper function to get the LMCache chunk size from the server.
|
||||
|
||||
Args:
|
||||
mq_client: The LMCache multiprocess mode message queue client
|
||||
|
||||
Returns:
|
||||
An integer representing the LMCache chunk size
|
||||
|
||||
"""
|
||||
future = send_lmcache_request(mq_client, RequestType.GET_CHUNK_SIZE, [])
|
||||
chunk_size = future.result()
|
||||
return chunk_size
|
||||
|
||||
|
||||
@dataclass
|
||||
class ParallelStrategy:
|
||||
use_mla: bool
|
||||
"""Whether to use the MLA."""
|
||||
|
||||
kv_world_size: int
|
||||
"""
|
||||
The kv world size, kv_world_size may not be equal to the actual_world_size,
|
||||
in the case of mla, it will 'exclude' the effect of TP, the value is
|
||||
calculated by `extract_world_size_and_kv_rank` in `lmcache_mp_connector.py`.
|
||||
"""
|
||||
|
||||
kv_worker_id: int
|
||||
"""
|
||||
The kv worker id of the sub-process, kv_worker_id may not be equal to the
|
||||
actual_worker_id, in the case of mla, it will 'exclude' the effect of TP,
|
||||
the value is calculated by `extract_world_size_and_kv_rank` in
|
||||
`lmcache_mp_connector.py`.
|
||||
"""
|
||||
|
||||
actual_world_size: int
|
||||
"""The actual world size."""
|
||||
|
||||
actual_worker_id: int
|
||||
"""The actual worker id of the sub-process."""
|
||||
|
||||
tp_size: int
|
||||
"""The tensor parallel size."""
|
||||
|
||||
pp_size: int
|
||||
"""The pipeline parallel size."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class LoadStoreOp:
|
||||
block_ids: list[int]
|
||||
"""Block ids for the load/store operation"""
|
||||
|
||||
token_ids: list[int] | None = None
|
||||
"""Token IDs for the load/store operation (token mode)"""
|
||||
|
||||
block_hashes: list[bytes] | None = None
|
||||
"""Block hashes for the load/store operation (hash mode)"""
|
||||
|
||||
start: int = 0
|
||||
"""Start token index (token mode only)"""
|
||||
|
||||
end: int = 0
|
||||
"""End token index (token mode only)"""
|
||||
|
||||
def __len__(self) -> int:
|
||||
return len(self.block_ids)
|
||||
|
||||
|
||||
StoreResult = bool
|
||||
RetrieveResult = list[bool]
|
||||
LookupResult = int
|
||||
|
||||
|
||||
class LMCacheMPSchedulerAdapter:
|
||||
def __init__(
|
||||
self,
|
||||
server_url: str,
|
||||
context: zmq.Context,
|
||||
model_name: str,
|
||||
vllm_block_size: int,
|
||||
parallel_strategy: ParallelStrategy,
|
||||
):
|
||||
"""Args:
|
||||
server_url: The server URL for the LMCache message queue
|
||||
context: The ZMQ context
|
||||
|
||||
model_name: The model name used for LMCache keys
|
||||
vllm_block_size: The block size used in vLLM
|
||||
parallel_strategy:
|
||||
The parallel strategy, which includes `use_mla`,
|
||||
`world_size`, `worker_id` and so on
|
||||
|
||||
"""
|
||||
self.mq_client = MessageQueueClient(server_url, context)
|
||||
|
||||
# Request futures
|
||||
self.lookup_futures: dict[str, MessagingFuture[LookupResult]] = {}
|
||||
|
||||
self.model_name = model_name
|
||||
self.parallel_strategy = parallel_strategy
|
||||
|
||||
# Read chunk size from lmcache
|
||||
self.chunk_size = get_lmcache_chunk_size(self.mq_client)
|
||||
assert self.chunk_size % vllm_block_size == 0, (
|
||||
"LMCache chunk size should be a multiple of vLLM block size"
|
||||
)
|
||||
self.blocks_in_chunk = self.chunk_size // vllm_block_size
|
||||
|
||||
@property
|
||||
def world_size(self) -> int:
|
||||
"""The world size."""
|
||||
return self.parallel_strategy.kv_world_size
|
||||
|
||||
@property
|
||||
def worker_id(self) -> int:
|
||||
"""The worker id."""
|
||||
return self.parallel_strategy.kv_worker_id
|
||||
|
||||
@property
|
||||
def tp_size(self) -> int:
|
||||
"""The tensor parallel size."""
|
||||
return self.parallel_strategy.tp_size
|
||||
|
||||
@_lmcache_nvtx_annotate
|
||||
def maybe_submit_lookup_request(
|
||||
self,
|
||||
request_id: str,
|
||||
block_hashes: list[bytes] | None = None,
|
||||
token_ids: list[int] | None = None,
|
||||
) -> None:
|
||||
"""Submit a new lookup request to LMCache if there is no ongoing request.
|
||||
|
||||
Supports both token-based and hash-based vLLM:
|
||||
- token_ids: token IDs (token-based vLLM) -> single token-mode key
|
||||
- block_hashes: block hashes (hash-based vLLM) -> strided hash-mode keys
|
||||
|
||||
Exactly one of block_hashes or token_ids must be provided.
|
||||
|
||||
Args:
|
||||
request_id: The ID of the lookup request. The same ID indicates it's
|
||||
from the same request
|
||||
block_hashes: Block hashes to lookup from LMCache (hash mode)
|
||||
token_ids: Token IDs to lookup from LMCache (token mode)
|
||||
|
||||
Returns:
|
||||
None
|
||||
|
||||
Notes:
|
||||
This function will have a side-effect: submitting a look up request to
|
||||
LMCache, which will essentially 'lock' the KV cache chunks in the LMCache
|
||||
for later retrieve operations.
|
||||
In the meantime, this function will record the lookup request, and the
|
||||
status of the look up request can be checked by `check_lookup_result`.
|
||||
|
||||
"""
|
||||
if request_id in self.lookup_futures:
|
||||
# Skip if there is already a lookup request
|
||||
return
|
||||
|
||||
assert (block_hashes is None) != (token_ids is None), (
|
||||
"Exactly one of block_hashes or token_ids must be provided"
|
||||
)
|
||||
|
||||
if block_hashes is not None:
|
||||
# Hash mode: stride block hashes -> N hash-mode keys
|
||||
chunk_hashes = list(
|
||||
striding_block_hashes(block_hashes, self.blocks_in_chunk)
|
||||
)
|
||||
keys = [
|
||||
self._create_hash_key(ch, request_id=request_id) for ch in chunk_hashes
|
||||
]
|
||||
else:
|
||||
# Token mode: truncate to chunk-aligned length
|
||||
assert token_ids is not None
|
||||
aligned_end = (len(token_ids) // self.chunk_size) * self.chunk_size
|
||||
if aligned_end == 0:
|
||||
return
|
||||
keys = [
|
||||
self._create_key(
|
||||
token_ids,
|
||||
start=0,
|
||||
end=aligned_end,
|
||||
request_id=request_id,
|
||||
).no_worker_id_version()
|
||||
]
|
||||
|
||||
future = send_lmcache_request(
|
||||
self.mq_client,
|
||||
RequestType.LOOKUP,
|
||||
[keys],
|
||||
)
|
||||
self.lookup_futures[request_id] = future
|
||||
|
||||
@_lmcache_nvtx_annotate
|
||||
def check_lookup_result(self, request_id: str) -> int | None:
|
||||
"""Check the result of a previously submitted lookup request.
|
||||
|
||||
Args:
|
||||
request_id: The ID of the lookup request submitted in
|
||||
`maybe_submit_lookup_request`
|
||||
|
||||
Returns:
|
||||
An integer representing the total number of tokens matched
|
||||
in LMCache (prefix matching), or
|
||||
None if the lookup request is not finished yet.
|
||||
|
||||
"""
|
||||
assert request_id in self.lookup_futures, (
|
||||
f"Lookup request for request_id={request_id} has not been submitted"
|
||||
)
|
||||
|
||||
future = self.lookup_futures[request_id]
|
||||
if not future.query():
|
||||
return None
|
||||
|
||||
result = future.result()
|
||||
num_chunks = result
|
||||
return num_chunks * self.chunk_size
|
||||
|
||||
def num_blocks_per_chunk(self) -> int:
|
||||
"""Returns:
|
||||
The number of vllm blocks in a LMCache data chunk
|
||||
|
||||
"""
|
||||
return self.blocks_in_chunk
|
||||
|
||||
def cleanup_lookup_result(self, request_id: str) -> None:
|
||||
"""Clean up lookup future for a finished request to prevent memory leak.
|
||||
|
||||
Args:
|
||||
request_id: The ID of the finished request.
|
||||
|
||||
"""
|
||||
self.lookup_futures.pop(request_id, None)
|
||||
|
||||
def end_session(self, request_id: str) -> None:
|
||||
"""Notify LMCache server to remove the session for a finished request.
|
||||
|
||||
Args:
|
||||
request_id: The ID of the finished request.
|
||||
|
||||
"""
|
||||
send_lmcache_request(
|
||||
self.mq_client,
|
||||
RequestType.END_SESSION,
|
||||
[request_id],
|
||||
)
|
||||
|
||||
# Helper functions
|
||||
def _create_key(
|
||||
self,
|
||||
token_ids: list[int],
|
||||
start: int = 0,
|
||||
end: int = 0,
|
||||
request_id: str | None = None,
|
||||
) -> IPCCacheEngineKey:
|
||||
"""Convert token IDs to an IPC cache engine key."""
|
||||
return IPCCacheEngineKey(
|
||||
model_name=self.model_name,
|
||||
world_size=self.world_size,
|
||||
worker_id=self.worker_id,
|
||||
token_ids=tuple(token_ids),
|
||||
start=start,
|
||||
end=end,
|
||||
request_id=request_id,
|
||||
tp_size=self.tp_size,
|
||||
)
|
||||
|
||||
def _create_hash_key(
|
||||
self, chunk_hash: bytes, request_id: str | None = None
|
||||
) -> IPCCacheEngineKey:
|
||||
"""Create a hash-mode IPC cache engine key."""
|
||||
return IPCCacheEngineKey(
|
||||
model_name=self.model_name,
|
||||
world_size=self.world_size,
|
||||
worker_id=None,
|
||||
chunk_hash=chunk_hash,
|
||||
request_id=request_id,
|
||||
tp_size=self.tp_size,
|
||||
)
|
||||
|
||||
|
||||
class LMCacheMPWorkerAdapter:
|
||||
def __init__(
|
||||
self,
|
||||
server_url: str,
|
||||
context: zmq.Context,
|
||||
model_name: str,
|
||||
vllm_block_size: int,
|
||||
parallel_strategy: ParallelStrategy,
|
||||
):
|
||||
self.mq_client = MessageQueueClient(server_url, context)
|
||||
|
||||
# Instance id for GPU worker
|
||||
self.instance_id = os.getpid()
|
||||
|
||||
# Registered kv caches from vLLM
|
||||
self.kv_caches: dict[str, torch.Tensor] = {}
|
||||
|
||||
# Request futures
|
||||
# request_id -> (future, other merged requests)
|
||||
self.store_futures: dict[
|
||||
str, tuple[MessagingFuture[StoreResult], list[str]]
|
||||
] = {}
|
||||
self.retrieve_futures: dict[
|
||||
str, tuple[MessagingFuture[RetrieveResult], list[str]]
|
||||
] = {}
|
||||
|
||||
# The store requests that have finished execution in LMCache
|
||||
self.finished_stores: set[str] = set()
|
||||
# The finished request ids that are passed via vLLM and also
|
||||
# have corresponding store requests submitted to LMCache before
|
||||
self.previously_finished: set[str] = set()
|
||||
|
||||
self.model_name = model_name
|
||||
self.parallel_strategy = parallel_strategy
|
||||
|
||||
# Read chunk size from lmcache
|
||||
chunk_size = get_lmcache_chunk_size(self.mq_client)
|
||||
assert chunk_size % vllm_block_size == 0, (
|
||||
"LMCache chunk size should be a multiple of vLLM block size"
|
||||
)
|
||||
self.blocks_in_chunk = chunk_size // vllm_block_size
|
||||
|
||||
@property
|
||||
def world_size(self) -> int:
|
||||
"""The world size."""
|
||||
return self.parallel_strategy.kv_world_size
|
||||
|
||||
@property
|
||||
def worker_id(self) -> int:
|
||||
"""The worker id."""
|
||||
return self.parallel_strategy.kv_worker_id
|
||||
|
||||
@property
|
||||
def use_mla(self) -> bool:
|
||||
"""Whether to use MLA."""
|
||||
return self.parallel_strategy.use_mla
|
||||
|
||||
@property
|
||||
def is_first_rank_of_pp_group(self) -> bool:
|
||||
"""Is the first rank of the pipeline parallel group."""
|
||||
return (
|
||||
self.parallel_strategy.actual_worker_id % self.parallel_strategy.tp_size
|
||||
== 0
|
||||
)
|
||||
|
||||
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
|
||||
"""Register the kv caches with LMCache server.
|
||||
|
||||
Args:
|
||||
kv_caches: A dict of kv caches to register. The keys are the
|
||||
layer names and the values are the corresponding tensors.
|
||||
|
||||
"""
|
||||
# Register kv cache and send the request
|
||||
self.kv_caches = kv_caches
|
||||
logger.info("Registering kv caches")
|
||||
future = send_lmcache_request(
|
||||
self.mq_client,
|
||||
RequestType.REGISTER_KV_CACHE,
|
||||
[self.instance_id, wrap_kv_caches(kv_caches)],
|
||||
)
|
||||
future.result()
|
||||
|
||||
@_lmcache_nvtx_annotate
|
||||
def submit_store_request(
|
||||
self, request_id: str, op: LoadStoreOp, event: torch.cuda.Event
|
||||
):
|
||||
"""Submit a KV cache store request to LMCache.
|
||||
|
||||
Args:
|
||||
request_id: The ID of the request
|
||||
op: The LoadStoreOp describing the store operation.
|
||||
event: The CUDA event that is recorded after the current
|
||||
model inference step
|
||||
|
||||
"""
|
||||
if op.block_hashes is not None:
|
||||
# Hash mode
|
||||
chunk_hashes = list(
|
||||
striding_block_hashes(op.block_hashes, self.blocks_in_chunk)
|
||||
)
|
||||
keys = [
|
||||
self._create_hash_key(ch, request_id=request_id) for ch in chunk_hashes
|
||||
]
|
||||
else:
|
||||
# Token mode
|
||||
assert op.token_ids is not None
|
||||
keys = [
|
||||
self._create_key(op.token_ids, op.start, op.end, request_id=request_id)
|
||||
]
|
||||
future = send_lmcache_request(
|
||||
self.mq_client,
|
||||
RequestType.STORE,
|
||||
[keys, self.instance_id, op.block_ids, event.ipc_handle()],
|
||||
).to_cuda_future()
|
||||
self.store_futures[request_id] = (future, [])
|
||||
|
||||
@_lmcache_nvtx_annotate
|
||||
def submit_retrieve_request(
|
||||
self, request_id: str, op: LoadStoreOp, event: torch.cuda.Event
|
||||
):
|
||||
"""Submit a KV cache retrieve request to LMCache.
|
||||
|
||||
Args:
|
||||
request_id: The ID of the request
|
||||
op: The LoadStoreOp describing the retrieve operation.
|
||||
event: The CUDA event that is recorded after the current
|
||||
model inference step
|
||||
|
||||
"""
|
||||
if op.block_hashes is not None:
|
||||
# Hash mode
|
||||
chunk_hashes = list(
|
||||
striding_block_hashes(op.block_hashes, self.blocks_in_chunk)
|
||||
)
|
||||
keys = [
|
||||
self._create_hash_key(ch, request_id=request_id) for ch in chunk_hashes
|
||||
]
|
||||
else:
|
||||
# Token mode
|
||||
assert op.token_ids is not None
|
||||
keys = [
|
||||
self._create_key(op.token_ids, op.start, op.end, request_id=request_id)
|
||||
]
|
||||
future = send_lmcache_request(
|
||||
self.mq_client,
|
||||
RequestType.RETRIEVE,
|
||||
[keys, self.instance_id, op.block_ids, event.ipc_handle()],
|
||||
).to_cuda_future()
|
||||
self.retrieve_futures[request_id] = (future, [])
|
||||
|
||||
@_lmcache_nvtx_annotate
|
||||
def batched_submit_store_requests(
|
||||
self,
|
||||
request_ids: list[str],
|
||||
ops: list[LoadStoreOp],
|
||||
event: torch.cuda.Event,
|
||||
):
|
||||
"""Submit a batched store request to LMCache.
|
||||
|
||||
Args:
|
||||
request_ids: The IDs of the requests
|
||||
ops: The LoadStoreOps describing the store operations. Should have
|
||||
the same length as request_ids
|
||||
event: The CUDA event that is recorded after the current
|
||||
model inference step
|
||||
|
||||
"""
|
||||
all_keys: list[IPCCacheEngineKey] = []
|
||||
block_ids: list[int] = []
|
||||
for request_id, op in zip(request_ids, ops, strict=False):
|
||||
if op.block_hashes is not None:
|
||||
chunk_hashes = list(
|
||||
striding_block_hashes(op.block_hashes, self.blocks_in_chunk)
|
||||
)
|
||||
keys = [
|
||||
self._create_hash_key(ch, request_id=request_id)
|
||||
for ch in chunk_hashes
|
||||
]
|
||||
all_keys.extend(keys)
|
||||
else:
|
||||
assert op.token_ids is not None
|
||||
all_keys.append(
|
||||
self._create_key(
|
||||
op.token_ids, op.start, op.end, request_id=request_id
|
||||
)
|
||||
)
|
||||
block_ids.extend(op.block_ids)
|
||||
future = send_lmcache_request(
|
||||
self.mq_client,
|
||||
RequestType.STORE,
|
||||
[
|
||||
all_keys,
|
||||
self.instance_id,
|
||||
block_ids,
|
||||
event.ipc_handle(),
|
||||
],
|
||||
).to_cuda_future()
|
||||
self.store_futures[request_ids[0]] = (future, list(request_ids[1:]))
|
||||
|
||||
@_lmcache_nvtx_annotate
|
||||
def batched_submit_retrieve_requests(
|
||||
self,
|
||||
request_ids: list[str],
|
||||
ops: list[LoadStoreOp],
|
||||
event: torch.cuda.Event,
|
||||
):
|
||||
"""Submit a batched retrieve request to LMCache.
|
||||
|
||||
Args:
|
||||
request_ids: The IDs of the requests
|
||||
ops: The LoadStoreOps describing the retrieve operations. Should have
|
||||
the same length as request_ids
|
||||
event: The CUDA event that is recorded after the current
|
||||
model inference step
|
||||
|
||||
"""
|
||||
all_keys: list[IPCCacheEngineKey] = []
|
||||
block_ids: list[int] = []
|
||||
for request_id, op in zip(request_ids, ops, strict=False):
|
||||
if op.block_hashes is not None:
|
||||
chunk_hashes = list(
|
||||
striding_block_hashes(op.block_hashes, self.blocks_in_chunk)
|
||||
)
|
||||
keys = [
|
||||
self._create_hash_key(ch, request_id=request_id)
|
||||
for ch in chunk_hashes
|
||||
]
|
||||
all_keys.extend(keys)
|
||||
else:
|
||||
assert op.token_ids is not None
|
||||
all_keys.append(
|
||||
self._create_key(
|
||||
op.token_ids, op.start, op.end, request_id=request_id
|
||||
)
|
||||
)
|
||||
block_ids.extend(op.block_ids)
|
||||
future = send_lmcache_request(
|
||||
self.mq_client,
|
||||
RequestType.RETRIEVE,
|
||||
[
|
||||
all_keys,
|
||||
self.instance_id,
|
||||
block_ids,
|
||||
event.ipc_handle(),
|
||||
],
|
||||
).to_cuda_future()
|
||||
self.retrieve_futures[request_ids[0]] = (future, list(request_ids[1:]))
|
||||
|
||||
@_lmcache_nvtx_annotate
|
||||
def get_finished(
|
||||
self, finished_req_ids_from_engine: set[str]
|
||||
) -> tuple[set[str] | None, set[str] | None]:
|
||||
"""Check and get the finished store and retrieve requests.
|
||||
|
||||
Args:
|
||||
finished_req_ids_from_engine: the set of request ids that are
|
||||
reported as finished from the vLLM engine side.
|
||||
|
||||
Returns:
|
||||
A tuple of two sets:
|
||||
- The first set contains the finished store request ids. The returned
|
||||
store request ids MUST be seen before in the
|
||||
`finished_req_ids_from_engine`.
|
||||
- The second set contains the finished retrieve request ids.
|
||||
|
||||
Notes:
|
||||
When enabling async scheduling in vLLM, the same request ID may appear
|
||||
multiple times in `finished_req_ids_from_engine`. The adapter should
|
||||
take care of deduplicating the request IDs and only return the request
|
||||
IDs that have not been returned before.
|
||||
|
||||
"""
|
||||
finished_stores = set()
|
||||
finished_retrieves = set()
|
||||
for request_id, (s_future, other_reqs) in self.store_futures.items():
|
||||
if not s_future.query():
|
||||
continue
|
||||
|
||||
s_result = s_future.result()
|
||||
finished_stores.add(request_id)
|
||||
finished_stores.update(other_reqs)
|
||||
|
||||
if not s_result:
|
||||
# TODO: add error handling here
|
||||
logger.error(
|
||||
"Something went wrong when processing the "
|
||||
"store request for request_id=%s",
|
||||
request_id,
|
||||
)
|
||||
|
||||
for request_id, (r_future, other_reqs) in self.retrieve_futures.items():
|
||||
if not r_future.query():
|
||||
continue
|
||||
|
||||
r_result = r_future.result()
|
||||
finished_retrieves.add(request_id)
|
||||
finished_retrieves.update(other_reqs)
|
||||
|
||||
if not all(r_result):
|
||||
# TODO: add error handing here
|
||||
logger.error(
|
||||
"Something went wrong when processing the "
|
||||
"retrieve request for request_id=%s, result=%s",
|
||||
request_id,
|
||||
r_result,
|
||||
)
|
||||
|
||||
# Remove the finished requests from the tracking dicts
|
||||
for request_id in finished_stores:
|
||||
self.store_futures.pop(request_id, None)
|
||||
for request_id in finished_retrieves:
|
||||
self.retrieve_futures.pop(request_id, None)
|
||||
|
||||
# Update the internal states
|
||||
self.finished_stores.update(finished_stores)
|
||||
|
||||
ret_stores = set()
|
||||
for req_id in finished_req_ids_from_engine:
|
||||
if req_id in self.finished_stores or req_id in self.store_futures:
|
||||
self.previously_finished.add(req_id)
|
||||
else:
|
||||
ret_stores.add(req_id)
|
||||
|
||||
# Calculate the final finished stores
|
||||
ret_stores.update(self._update_and_get_finished_store())
|
||||
|
||||
return ret_stores, finished_retrieves
|
||||
|
||||
def num_blocks_per_chunk(self) -> int:
|
||||
"""Returns:
|
||||
The number of vllm blocks in a LMCache data chunk
|
||||
|
||||
"""
|
||||
return self.blocks_in_chunk
|
||||
|
||||
def shutdown(self):
|
||||
"""Shutdown the LMCache MP worker adapter."""
|
||||
logger.info("Unregistering kv caches")
|
||||
send_lmcache_request(
|
||||
self.mq_client, RequestType.UNREGISTER_KV_CACHE, [self.instance_id]
|
||||
).result()
|
||||
|
||||
self.mq_client.close()
|
||||
|
||||
# Helper functions
|
||||
def _update_and_get_finished_store(
|
||||
self,
|
||||
) -> set[str]:
|
||||
"""Converge the internal states about finished stores
|
||||
and returns the 'safe finished store request ids' back
|
||||
"""
|
||||
safe_finished_s = self.finished_stores.intersection(self.previously_finished)
|
||||
self.finished_stores.difference_update(self.previously_finished)
|
||||
self.previously_finished.difference_update(safe_finished_s)
|
||||
|
||||
return safe_finished_s
|
||||
|
||||
def _create_key(
|
||||
self,
|
||||
token_ids: list[int],
|
||||
start: int = 0,
|
||||
end: int = 0,
|
||||
request_id: str | None = None,
|
||||
) -> IPCCacheEngineKey:
|
||||
"""Convert token IDs to an IPC cache engine key."""
|
||||
return IPCCacheEngineKey(
|
||||
model_name=self.model_name,
|
||||
world_size=self.world_size,
|
||||
worker_id=self.worker_id,
|
||||
token_ids=tuple(token_ids),
|
||||
start=start,
|
||||
end=end,
|
||||
request_id=request_id,
|
||||
)
|
||||
|
||||
def _create_hash_key(
|
||||
self, chunk_hash: bytes, request_id: str | None = None
|
||||
) -> IPCCacheEngineKey:
|
||||
"""Create a hash-mode IPC cache engine key."""
|
||||
return IPCCacheEngineKey(
|
||||
model_name=self.model_name,
|
||||
world_size=self.world_size,
|
||||
worker_id=self.worker_id,
|
||||
chunk_hash=chunk_hash,
|
||||
request_id=request_id,
|
||||
)
|
||||
@@ -1,134 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# Standard
|
||||
import os
|
||||
import threading
|
||||
from typing import TYPE_CHECKING, Union
|
||||
|
||||
import torch
|
||||
from lmcache.logging import init_logger
|
||||
from lmcache.v1.config import LMCacheEngineConfig as V1Config
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.multimodal.inputs import PlaceholderRange
|
||||
from vllm.v1.core.sched.output import NewRequestData
|
||||
from vllm.v1.request import Request
|
||||
|
||||
logger = init_logger(__name__)
|
||||
ENGINE_NAME = "vllm-instance"
|
||||
|
||||
# Thread-safe singleton storage
|
||||
_config_instance: V1Config | None = None
|
||||
_config_lock = threading.Lock()
|
||||
|
||||
|
||||
def lmcache_get_or_create_config() -> V1Config:
|
||||
"""Get the LMCache configuration from the environment variable
|
||||
`LMCACHE_CONFIG_FILE`. If the environment variable is not set, this
|
||||
function will return the default configuration.
|
||||
|
||||
This function is thread-safe and implements singleton pattern,
|
||||
ensuring the configuration is loaded only once.
|
||||
"""
|
||||
global _config_instance
|
||||
|
||||
# Double-checked locking for thread-safe singleton
|
||||
if _config_instance is None:
|
||||
with _config_lock:
|
||||
if _config_instance is None: # Check again within lock
|
||||
LMCacheEngineConfig = V1Config # type: ignore[assignment]
|
||||
|
||||
if "LMCACHE_CONFIG_FILE" not in os.environ:
|
||||
logger.warning(
|
||||
"No LMCache configuration file is set. Trying to read"
|
||||
" configurations from the environment variables."
|
||||
)
|
||||
logger.warning(
|
||||
"You can set the configuration file through "
|
||||
"the environment variable: LMCACHE_CONFIG_FILE"
|
||||
)
|
||||
_config_instance = LMCacheEngineConfig.from_env()
|
||||
else:
|
||||
config_file = os.environ["LMCACHE_CONFIG_FILE"]
|
||||
logger.info("Loading LMCache config file %s", config_file)
|
||||
_config_instance = LMCacheEngineConfig.from_file(config_file)
|
||||
# Update config from environment variables
|
||||
_config_instance.update_config_from_env()
|
||||
return _config_instance
|
||||
|
||||
|
||||
def hex_hash_to_int16(s: str) -> int:
|
||||
"""Convert a hex hash string to a 16-bit integer."""
|
||||
return int(s, 16) & 0xFFFF
|
||||
|
||||
|
||||
def apply_mm_hashes_to_token_ids(
|
||||
token_ids: torch.Tensor,
|
||||
mm_hashes: list[str],
|
||||
mm_positions: list["PlaceholderRange"],
|
||||
) -> torch.Tensor:
|
||||
"""Overwrite token_ids in-place for multimodal placeholders using
|
||||
efficient slice assignments.
|
||||
"""
|
||||
n = token_ids.size(0)
|
||||
for hash_str, placeholder in zip(mm_hashes, mm_positions):
|
||||
start, length = placeholder.offset, placeholder.length
|
||||
if start >= n:
|
||||
continue
|
||||
end = min(start + length, n)
|
||||
token_ids[start:end] = hex_hash_to_int16(hash_str)
|
||||
return token_ids
|
||||
|
||||
|
||||
def mla_enabled(model_config: "ModelConfig") -> bool:
|
||||
return (
|
||||
hasattr(model_config, "use_mla")
|
||||
and isinstance(model_config.use_mla, bool)
|
||||
and model_config.use_mla
|
||||
)
|
||||
|
||||
|
||||
def extract_mm_features(
|
||||
request: Union["Request", "NewRequestData"], modify: bool = False
|
||||
) -> tuple[list[str], list["PlaceholderRange"]]:
|
||||
"""Normalize multimodal information from a Request into parallel lists.
|
||||
|
||||
This helper reads either:
|
||||
1) `request.mm_features` (objects each exposing `.identifier` and
|
||||
`.mm_position`), or
|
||||
2) legacy fields `request.mm_hashes` and `request.mm_positions`.
|
||||
|
||||
It returns two equally sized lists: the multimodal hash identifiers and
|
||||
their corresponding positions. If the request contains no multimodal info,
|
||||
it returns `([], [])`.
|
||||
|
||||
Args:
|
||||
request (Request): The source object.
|
||||
modify (bool):
|
||||
Controls copy semantics for the legacy-path return values.
|
||||
- If True and legacy fields are used, shallow-copies are returned so
|
||||
the caller can mutate the lists without affecting `request`.
|
||||
- If False, the original legacy sequences are returned as-is
|
||||
(zero-copy); treat them as read-only.
|
||||
|
||||
Returns:
|
||||
tuple[list[str], list[PlaceholderRange]]: (`mm_hashes`, `mm_positions`).
|
||||
May be `([], [])` when no multimodal data is present.
|
||||
|
||||
"""
|
||||
if getattr(request, "mm_features", None):
|
||||
mm_hashes, mm_positions = zip(
|
||||
*((f.identifier, f.mm_position) for f in request.mm_features)
|
||||
)
|
||||
return (list(mm_hashes), list(mm_positions))
|
||||
elif getattr(request, "mm_hashes", None):
|
||||
if modify:
|
||||
return (
|
||||
request.mm_hashes.copy(), # type: ignore
|
||||
request.mm_positions.copy(), # type: ignore
|
||||
)
|
||||
else:
|
||||
return (request.mm_hashes, request.mm_positions) # type: ignore
|
||||
else:
|
||||
return ([], [])
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,178 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import socket
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
|
||||
import uvicorn
|
||||
from fastapi import FastAPI, HTTPException
|
||||
from pydantic import BaseModel
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.utils import EngineId
|
||||
from vllm.logger import init_logger
|
||||
|
||||
WorkerAddr = str
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class RegisterWorkerPayload(BaseModel):
|
||||
engine_id: EngineId
|
||||
dp_rank: int
|
||||
tp_rank: int
|
||||
pp_rank: int
|
||||
addr: WorkerAddr
|
||||
|
||||
|
||||
@dataclass
|
||||
class EngineEntry:
|
||||
engine_id: EngineId
|
||||
# {tp_rank: {pp_rank: worker_addr}}
|
||||
worker_addr: dict[int, dict[int, WorkerAddr]]
|
||||
|
||||
|
||||
class MooncakeBootstrapServer:
|
||||
"""A centralized registry for prefiller connection info (IP, port, ranks)."""
|
||||
|
||||
def __init__(self, host: str, port: int):
|
||||
self.workers: dict[int, EngineEntry] = {}
|
||||
|
||||
self.host = host
|
||||
self.port = port
|
||||
self.app = FastAPI()
|
||||
self._register_routes()
|
||||
self.server_thread: threading.Thread | None = None
|
||||
self.server: uvicorn.Server | None = None
|
||||
self._socket: socket.socket | None = None
|
||||
self._startup_error: BaseException | None = None
|
||||
|
||||
def __del__(self):
|
||||
self.shutdown()
|
||||
|
||||
def _register_routes(self):
|
||||
# All methods are async. No need to use lock to protect data.
|
||||
self.app.post("/register")(self.register_worker)
|
||||
self.app.get("/query", response_model=dict[int, EngineEntry])(self.query)
|
||||
|
||||
def start(self, *, timeout: float = 30.0):
|
||||
if self.server_thread:
|
||||
if (
|
||||
self.server_thread.is_alive()
|
||||
and self.server is not None
|
||||
and self.server.started
|
||||
and not self.server.should_exit
|
||||
and self._socket is not None
|
||||
and self._socket.fileno() >= 0
|
||||
):
|
||||
return
|
||||
raise RuntimeError(
|
||||
"Mooncake bootstrap server is still starting or stopping"
|
||||
)
|
||||
|
||||
try:
|
||||
# Bind in the caller so errors propagate, and retain the listener
|
||||
# when handing it to Uvicorn, including for automatically chosen ports.
|
||||
family = socket.AF_INET6 if ":" in self.host else socket.AF_INET
|
||||
self._socket = socket.create_server((self.host, self.port), family=family)
|
||||
self.port = self._socket.getsockname()[1]
|
||||
config = uvicorn.Config(app=self.app, host=self.host, port=self.port)
|
||||
server = self.server = uvicorn.Server(config=config)
|
||||
listener = self._socket
|
||||
self._startup_error = None
|
||||
|
||||
def run():
|
||||
try:
|
||||
server.run(sockets=[listener])
|
||||
except BaseException as exc:
|
||||
self._startup_error = exc
|
||||
|
||||
self.server_thread = threading.Thread(
|
||||
target=run, name="mooncake_bootstrap_server", daemon=True
|
||||
)
|
||||
self.server_thread.start()
|
||||
deadline = time.monotonic() + timeout
|
||||
while not server.started:
|
||||
if not self.server_thread.is_alive():
|
||||
raise RuntimeError(
|
||||
"Mooncake bootstrap server exited during startup"
|
||||
) from self._startup_error
|
||||
if time.monotonic() >= deadline:
|
||||
raise TimeoutError(
|
||||
"Mooncake bootstrap server did not start in time"
|
||||
)
|
||||
time.sleep(0.01)
|
||||
except BaseException:
|
||||
self.shutdown()
|
||||
raise
|
||||
logger.info("Mooncake Bootstrap Server started at %s:%d", self.host, self.port)
|
||||
|
||||
def shutdown(self):
|
||||
was_started = self.server is not None and self.server.started
|
||||
if self.server is not None:
|
||||
self.server.should_exit = True
|
||||
if self.server_thread is not None and self.server_thread.ident is not None:
|
||||
self.server_thread.join(timeout=5)
|
||||
if self._socket is not None:
|
||||
self._socket.close()
|
||||
self._socket = None
|
||||
if self.server_thread is not None and self.server_thread.is_alive():
|
||||
logger.warning("Mooncake bootstrap server did not stop in time")
|
||||
else:
|
||||
self.server_thread = None
|
||||
self.server = None
|
||||
if was_started:
|
||||
logger.info("Mooncake Bootstrap Server stopped.")
|
||||
|
||||
async def register_worker(self, payload: RegisterWorkerPayload):
|
||||
"""Handles registration of a prefiller worker."""
|
||||
if payload.dp_rank not in self.workers:
|
||||
self.workers[payload.dp_rank] = EngineEntry(
|
||||
engine_id=payload.engine_id,
|
||||
worker_addr={},
|
||||
)
|
||||
|
||||
dp_entry = self.workers[payload.dp_rank]
|
||||
if dp_entry.engine_id != payload.engine_id:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=(
|
||||
f"Engine ID mismatch for dp_rank={payload.dp_rank}: "
|
||||
f"expected {dp_entry.engine_id}, got {payload.engine_id}"
|
||||
),
|
||||
)
|
||||
if payload.tp_rank not in dp_entry.worker_addr:
|
||||
dp_entry.worker_addr[payload.tp_rank] = {}
|
||||
|
||||
tp_entry = dp_entry.worker_addr[payload.tp_rank]
|
||||
existing = tp_entry.get(payload.pp_rank)
|
||||
if existing is not None:
|
||||
# A client timeout can fire after the server recorded the
|
||||
# registration so an identical retry must not be an error.
|
||||
if existing == payload.addr:
|
||||
return {"status": "ok"}
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=(
|
||||
f"Worker with dp_rank={payload.dp_rank}, "
|
||||
f"tp_rank={payload.tp_rank}, pp_rank={payload.pp_rank} "
|
||||
f"is already registered at "
|
||||
f"{existing}, "
|
||||
f"but still want to register at {payload.addr}"
|
||||
),
|
||||
)
|
||||
|
||||
tp_entry[payload.pp_rank] = payload.addr
|
||||
logger.debug(
|
||||
"Registered worker: engine_id=%s, dp_rank=%d, tp_rank=%d, pp_rank=%d at %s",
|
||||
payload.engine_id,
|
||||
payload.dp_rank,
|
||||
payload.tp_rank,
|
||||
payload.pp_rank,
|
||||
payload.addr,
|
||||
)
|
||||
|
||||
return {"status": "ok"}
|
||||
|
||||
async def query(self) -> dict[int, EngineEntry]:
|
||||
return self.workers
|
||||
@@ -1,46 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Mooncake requester config helpers."""
|
||||
|
||||
from collections.abc import Mapping
|
||||
from typing import Any
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def normalize_string_override(value: Any) -> str | None:
|
||||
if not isinstance(value, str):
|
||||
return None
|
||||
normalized = value.strip()
|
||||
return normalized or None
|
||||
|
||||
|
||||
def get_requester_local_hostname(local_ip: str) -> str:
|
||||
override = normalize_string_override(envs.MOONCAKE_REQUESTER_LOCAL_HOSTNAME)
|
||||
if override is not None:
|
||||
return override
|
||||
return local_ip
|
||||
|
||||
|
||||
def get_configured_preferred_segment(
|
||||
extra_config: Mapping[str, Any],
|
||||
) -> str | None:
|
||||
preferred_segment = normalize_string_override(extra_config.get("preferred_segment"))
|
||||
if preferred_segment is not None:
|
||||
return preferred_segment
|
||||
if extra_config.get("preferred_segment") is not None:
|
||||
raise ValueError(
|
||||
"Mooncake preferred_segment override must be a non-empty string"
|
||||
)
|
||||
|
||||
env_value = normalize_string_override(envs.MOONCAKE_PREFERRED_SEGMENT)
|
||||
if env_value is not None:
|
||||
logger.info(
|
||||
"Mooncake preferred_segment from MOONCAKE_PREFERRED_SEGMENT: %s",
|
||||
env_value,
|
||||
)
|
||||
return env_value
|
||||
return None
|
||||
@@ -1,146 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Stats container for the Mooncake connector."""
|
||||
|
||||
import threading
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
|
||||
KVConnectorStats,
|
||||
)
|
||||
|
||||
# TODO(mooncake-stats): add MooncakePromMetrics (mirror NixlPromMetrics)
|
||||
# and wire it via MooncakeConnector.build_prom_metrics in a follow-up PR.
|
||||
|
||||
|
||||
@dataclass
|
||||
class MooncakeKVConnectorStats(KVConnectorStats):
|
||||
"""Container for Mooncake KV transfer performance metrics.
|
||||
|
||||
`_lock` serializes record_* against clone_and_reset so each row's
|
||||
appends are atomic and column lengths stay aligned. Writers run on
|
||||
the sender pool / receiver loop / sender loop; reader runs on the
|
||||
main worker thread.
|
||||
"""
|
||||
|
||||
def __post_init__(self):
|
||||
self._lock = threading.Lock()
|
||||
if not self.data:
|
||||
self.reset()
|
||||
|
||||
# threading.Lock is not picklable; strip it from the wire form and
|
||||
# rebuild a fresh per-process lock on the receiver side.
|
||||
def __getstate__(self) -> dict[str, Any]:
|
||||
state = self.__dict__.copy()
|
||||
state.pop("_lock", None)
|
||||
return state
|
||||
|
||||
def __setstate__(self, state: dict[str, Any]) -> None:
|
||||
self.__dict__.update(state)
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def reset(self):
|
||||
self.data: dict[str, list[float | int]] = {
|
||||
"transfer_duration": [],
|
||||
"bytes_transferred": [],
|
||||
"num_descriptors": [],
|
||||
"num_failed_transfers": [],
|
||||
"num_failed_recvs": [],
|
||||
"num_kv_expired_reqs": [],
|
||||
}
|
||||
|
||||
def record_transfer(self, duration_s: float, total_bytes: int, num_descs: int):
|
||||
with self._lock:
|
||||
self.data["transfer_duration"].append(duration_s)
|
||||
self.data["bytes_transferred"].append(total_bytes)
|
||||
self.data["num_descriptors"].append(num_descs)
|
||||
|
||||
# Failure counters store a list of 1s so a future Prom counter can iterate
|
||||
# with .inc(list_item), mirroring NIXL's NixlPromMetrics.observe.
|
||||
def record_failed_transfer(self):
|
||||
with self._lock:
|
||||
self.data["num_failed_transfers"].append(1)
|
||||
|
||||
def record_failed_recv(self):
|
||||
with self._lock:
|
||||
self.data["num_failed_recvs"].append(1)
|
||||
|
||||
def record_kv_expired_req(self):
|
||||
with self._lock:
|
||||
self.data["num_kv_expired_reqs"].append(1)
|
||||
|
||||
def clone_and_reset(self) -> "MooncakeKVConnectorStats":
|
||||
# Copy lists under the lock for length alignment; return a fresh
|
||||
# instance so the snapshot has its own _lock.
|
||||
with self._lock:
|
||||
snapshot_data: dict[str, list[float | int]] = {
|
||||
k: list(v) for k, v in self.data.items()
|
||||
}
|
||||
self.reset()
|
||||
return MooncakeKVConnectorStats(data=snapshot_data)
|
||||
|
||||
def is_empty(self) -> bool:
|
||||
return (
|
||||
self.num_successful_transfers == 0
|
||||
and len(self.data["num_failed_transfers"]) == 0
|
||||
and len(self.data["num_failed_recvs"]) == 0
|
||||
and len(self.data["num_kv_expired_reqs"]) == 0
|
||||
)
|
||||
|
||||
def aggregate(self, other: KVConnectorStats) -> KVConnectorStats:
|
||||
if not other.is_empty():
|
||||
for k, v in other.data.items():
|
||||
accumulator = self.data[k]
|
||||
assert isinstance(accumulator, list)
|
||||
accumulator.extend(v)
|
||||
return self
|
||||
|
||||
def reduce(self) -> dict[str, int | float]:
|
||||
num_failed_transfers = len(self.data["num_failed_transfers"])
|
||||
num_failed_recvs = len(self.data["num_failed_recvs"])
|
||||
num_kv_expired_reqs = len(self.data["num_kv_expired_reqs"])
|
||||
|
||||
if self.num_successful_transfers == 0:
|
||||
return {
|
||||
"Num successful transfers": 0,
|
||||
"Avg xfer time (ms)": 0,
|
||||
"P90 xfer time (ms)": 0,
|
||||
"Avg MB per transfer": 0,
|
||||
"Throughput (MB/s)": 0,
|
||||
"Avg number of descriptors": 0,
|
||||
"Num failed transfers": num_failed_transfers,
|
||||
"Num failed recvs": num_failed_recvs,
|
||||
"Num KV expired reqs": num_kv_expired_reqs,
|
||||
}
|
||||
|
||||
xfer_time = np.asarray(self.data["transfer_duration"])
|
||||
mb = np.asarray(self.data["bytes_transferred"]) / 2**20
|
||||
descs = np.asarray(self.data["num_descriptors"], dtype=np.uint32)
|
||||
n = len(descs)
|
||||
assert n == self.num_successful_transfers
|
||||
|
||||
total_mb = mb.sum()
|
||||
avg_mb = total_mb / n
|
||||
total_time_seconds = xfer_time.sum()
|
||||
throughput_mb_s = (
|
||||
(total_mb / total_time_seconds).item() if total_time_seconds > 0 else 0.0
|
||||
)
|
||||
|
||||
return {
|
||||
"Num successful transfers": n,
|
||||
"Avg xfer time (ms)": round(xfer_time.mean().item() * 1e3, 3),
|
||||
"P90 xfer time (ms)": round(np.percentile(xfer_time, 90).item() * 1e3, 3),
|
||||
"Avg MB per transfer": round(avg_mb.item(), 3),
|
||||
"Throughput (MB/s)": round(throughput_mb_s, 3),
|
||||
"Avg number of descriptors": round(descs.mean().item(), 1),
|
||||
"Num failed transfers": num_failed_transfers,
|
||||
"Num failed recvs": num_failed_recvs,
|
||||
"Num KV expired reqs": num_kv_expired_reqs,
|
||||
}
|
||||
|
||||
@property
|
||||
def num_successful_transfers(self) -> int:
|
||||
return len(self.data["transfer_duration"])
|
||||
@@ -1,2 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -1,436 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
#
|
||||
# Adapted from vllm-project/vllm-ascend
|
||||
# (vllm_ascend/distributed/kv_transfer/kv_pool/ascend_store/).
|
||||
"""MooncakeStoreConnector - KV cache connector using MooncakeDistributedStore.
|
||||
|
||||
Unlike MooncakeConnector which does direct P2P transfer, this connector
|
||||
uses MooncakeDistributedStore as a shared KV cache pool. Both producer
|
||||
and consumer instances read/write KV to/from the store independently,
|
||||
enabling prefix caching via hash-based deduplication.
|
||||
"""
|
||||
|
||||
from collections import Counter
|
||||
from collections.abc import Iterable, Sequence
|
||||
from contextlib import AbstractContextManager
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import KVTransferConfig, VllmConfig
|
||||
from vllm.distributed.kv_events import (
|
||||
BlockStored,
|
||||
KVCacheEvent,
|
||||
KVConnectorKVEvents,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorBase_V1,
|
||||
KVConnectorMetadata,
|
||||
KVConnectorRole,
|
||||
KVConnectorTransferResults,
|
||||
KVConnectorWorkerMetadata,
|
||||
SupportsHMA,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
|
||||
KVConnectorPromMetrics,
|
||||
KVConnectorStats,
|
||||
PromMetric,
|
||||
PromMetricT,
|
||||
)
|
||||
from vllm.forward_context import ForwardContext
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.attention.backend import AttentionMetadata
|
||||
from vllm.v1.core.block_pool import BlockPool
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.outputs import KVConnectorOutput
|
||||
from vllm.v1.request import Request
|
||||
|
||||
from .data import MooncakeStoreConnectorMetadata
|
||||
from .metrics import MooncakeStoreConnectorStats, MooncakeStorePromMetrics
|
||||
from .scheduler import MooncakeStoreScheduler
|
||||
from .worker import MooncakeStoreWorker
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class MooncakeStoreKVEvents(KVConnectorKVEvents):
|
||||
"""KV event aggregation for MooncakeStoreConnector."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
num_workers: int,
|
||||
group_tp_replication_factors: Sequence[int] = (1,),
|
||||
) -> None:
|
||||
if num_workers <= 0:
|
||||
raise ValueError("num_workers must be greater than zero.")
|
||||
if any(factor <= 0 for factor in group_tp_replication_factors):
|
||||
raise ValueError("TP replication factors must be greater than zero.")
|
||||
self._event_counter: Counter[KVCacheEvent] = Counter()
|
||||
self._num_workers = num_workers
|
||||
self._group_tp_replication_factors = tuple(group_tp_replication_factors)
|
||||
|
||||
def add_events(self, events: list[KVCacheEvent]) -> None:
|
||||
if not isinstance(events, list):
|
||||
raise TypeError("events must be a list of KVCacheEvent.")
|
||||
self._event_counter.update(events)
|
||||
|
||||
def _replication_factor(self, event: KVCacheEvent) -> int:
|
||||
if not isinstance(event, BlockStored) or event.group_idx is None:
|
||||
return 1
|
||||
return self._group_tp_replication_factors[event.group_idx]
|
||||
|
||||
def _is_common_event(self, event: KVCacheEvent, count: int) -> bool:
|
||||
return count * self._replication_factor(event) >= self._num_workers
|
||||
|
||||
def aggregate(self) -> "MooncakeStoreKVEvents":
|
||||
common_events = self.pop_common_events()
|
||||
self._event_counter.clear()
|
||||
self._event_counter.update(common_events)
|
||||
self._num_workers = 1
|
||||
return self
|
||||
|
||||
def pop_common_events(self) -> list[KVCacheEvent]:
|
||||
common_events = [
|
||||
event
|
||||
for event, count in self._event_counter.items()
|
||||
if self._is_common_event(event, count)
|
||||
]
|
||||
for event in common_events:
|
||||
del self._event_counter[event]
|
||||
return common_events
|
||||
|
||||
def has_events(self) -> bool:
|
||||
return bool(self._event_counter)
|
||||
|
||||
def increment_workers(self, count: int = 1) -> None:
|
||||
if count <= 0:
|
||||
raise ValueError("count must be positive.")
|
||||
self._num_workers += count
|
||||
|
||||
def get_all_events(self) -> list[KVCacheEvent]:
|
||||
return list(self._event_counter.elements())
|
||||
|
||||
def get_number_of_workers(self) -> int:
|
||||
return self._num_workers
|
||||
|
||||
def clear_events(self) -> None:
|
||||
self._event_counter.clear()
|
||||
self._num_workers = 1
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"<MooncakeStoreKVEvents events={self.get_all_events()}>"
|
||||
|
||||
|
||||
class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
"""KV connector using MooncakeDistributedStore as shared KV pool."""
|
||||
|
||||
@classmethod
|
||||
def supports_sleep_mode(cls, kv_transfer_config: KVTransferConfig) -> bool:
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _validate_kv_cache_config(
|
||||
vllm_config: VllmConfig, kv_cache_config: KVCacheConfig
|
||||
) -> None:
|
||||
from vllm.v1.kv_cache_interface import CrossAttentionSpec, MambaSpec
|
||||
|
||||
unsupported: list[str] = []
|
||||
store_group_ids = kv_cache_config.prefix_cacheable_group_ids
|
||||
if not store_group_ids:
|
||||
raise ValueError(
|
||||
"MooncakeStore requires at least one prefix-cacheable KV cache group"
|
||||
)
|
||||
for group_id in store_group_ids:
|
||||
spec = kv_cache_config.kv_cache_groups[group_id].kv_cache_spec
|
||||
if isinstance(spec, CrossAttentionSpec):
|
||||
unsupported.append(f"group {group_id}: CrossAttentionSpec")
|
||||
if isinstance(spec, MambaSpec) and spec.mamba_cache_mode != "align":
|
||||
unsupported.append(
|
||||
f"group {group_id}: mamba_cache_mode="
|
||||
f"{spec.mamba_cache_mode!r} != 'align'"
|
||||
)
|
||||
pcp = vllm_config.parallel_config.prefill_context_parallel_size
|
||||
if len(store_group_ids) > 1 and pcp > 1:
|
||||
unsupported.append(f"PCP > 1 (pcp={pcp}) with hybrid attention")
|
||||
if unsupported:
|
||||
raise ValueError(
|
||||
"MooncakeStoreConnector does not support: " + "; ".join(unsupported)
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
role: KVConnectorRole,
|
||||
kv_cache_config: KVCacheConfig | None = None,
|
||||
):
|
||||
super().__init__(
|
||||
vllm_config=vllm_config,
|
||||
role=role,
|
||||
kv_cache_config=kv_cache_config, # type: ignore[arg-type]
|
||||
)
|
||||
assert vllm_config.kv_transfer_config is not None
|
||||
assert kv_cache_config is not None, "kv_cache_config is required"
|
||||
self.kv_role = vllm_config.kv_transfer_config.kv_role
|
||||
extra_config = vllm_config.kv_transfer_config.kv_connector_extra_config
|
||||
save_decode_cache = extra_config.get("save_decode_cache", False)
|
||||
# Capacity-only: contributes its segment to the store pool but transfers
|
||||
# no KV, so the KV-cache-shape invariants below cannot be reached.
|
||||
self._capacity_only = (
|
||||
self.kv_role == "kv_consumer"
|
||||
and not extra_config.get("enable_lookup", True)
|
||||
and not save_decode_cache
|
||||
)
|
||||
if not self._capacity_only:
|
||||
self._validate_kv_cache_config(vllm_config, kv_cache_config)
|
||||
self._kv_cache_config = kv_cache_config
|
||||
self._kv_cache_events: MooncakeStoreKVEvents | None = None
|
||||
|
||||
self.connector_scheduler: MooncakeStoreScheduler | None = None
|
||||
self.connector_worker: MooncakeStoreWorker | None = None
|
||||
|
||||
if role == KVConnectorRole.SCHEDULER:
|
||||
self.connector_scheduler = MooncakeStoreScheduler(
|
||||
vllm_config, kv_cache_config
|
||||
)
|
||||
else:
|
||||
self.connector_worker = MooncakeStoreWorker(vllm_config, kv_cache_config)
|
||||
|
||||
def shutdown(self):
|
||||
"""Release connector resources on teardown.
|
||||
|
||||
Closes the worker's MooncakeDistributedStore handle so its
|
||||
TransferEngine and RDMA registrations are released. Invoked from the
|
||||
engine's explicit shutdown path and as a backstop from ``__del__``;
|
||||
a no-op on the scheduler role, which holds no store handle.
|
||||
"""
|
||||
worker = getattr(self, "connector_worker", None)
|
||||
if worker is not None:
|
||||
worker.close()
|
||||
|
||||
def __del__(self):
|
||||
self.shutdown()
|
||||
|
||||
# ============================================================
|
||||
# Scheduler-side methods
|
||||
# ============================================================
|
||||
|
||||
def get_num_new_matched_tokens(
|
||||
self,
|
||||
request: Request,
|
||||
num_computed_tokens: int,
|
||||
) -> tuple[int | None, bool]:
|
||||
assert self.connector_scheduler is not None
|
||||
return self.connector_scheduler.get_num_new_matched_tokens(
|
||||
request, num_computed_tokens
|
||||
)
|
||||
|
||||
def update_state_after_alloc(
|
||||
self,
|
||||
request: Request,
|
||||
blocks: KVCacheBlocks,
|
||||
num_external_tokens: int,
|
||||
):
|
||||
assert self.connector_scheduler is not None
|
||||
return self.connector_scheduler.update_state_after_alloc(
|
||||
request, blocks, num_external_tokens
|
||||
)
|
||||
|
||||
def bind_gpu_block_pool(self, gpu_block_pool: BlockPool) -> None:
|
||||
assert self.connector_scheduler is not None
|
||||
self.connector_scheduler.bind_gpu_block_pool(gpu_block_pool)
|
||||
|
||||
def has_pending_push_work(self) -> bool:
|
||||
assert self.connector_scheduler is not None
|
||||
return self.connector_scheduler.has_pending_push_work()
|
||||
|
||||
def build_connector_meta(
|
||||
self,
|
||||
scheduler_output: SchedulerOutput,
|
||||
) -> KVConnectorMetadata:
|
||||
assert self.connector_scheduler is not None
|
||||
return self.connector_scheduler.build_connector_meta(scheduler_output)
|
||||
|
||||
def build_connector_worker_meta(self) -> KVConnectorWorkerMetadata | None:
|
||||
assert self.connector_worker is not None
|
||||
return self.connector_worker.build_connector_worker_meta()
|
||||
|
||||
def request_finished(
|
||||
self,
|
||||
request: Request,
|
||||
block_ids: list[int],
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
return self.request_finished_all_groups(request, (block_ids,))
|
||||
|
||||
def request_finished_all_groups(
|
||||
self,
|
||||
request: Request,
|
||||
block_ids: tuple[list[int], ...],
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
# An in-flight store job holds its own reference on the blocks it reads,
|
||||
# so a finishing request never has to defer freeing them.
|
||||
return False, None
|
||||
|
||||
def register_finished_partial_tail(
|
||||
self,
|
||||
request: Request,
|
||||
block_ids: tuple[list[int], ...],
|
||||
partial_tail_offloads: list[tuple[int, int, int]],
|
||||
) -> bool:
|
||||
assert self.connector_scheduler is not None
|
||||
return self.connector_scheduler.register_finished_partial_tail(
|
||||
request, block_ids, partial_tail_offloads
|
||||
)
|
||||
|
||||
def reset_cache(self) -> bool | None:
|
||||
"""Reset the external Mooncake store on prefix-cache reset.
|
||||
|
||||
Drains the worker send queue, then runs ``remove_all`` on the
|
||||
Mooncake master. Caller must first pause generation (e.g.
|
||||
``pause_generation``) so no new puts are enqueued during drain.
|
||||
|
||||
Returns True on ack, False on failure, None for the worker role.
|
||||
"""
|
||||
if self.role == KVConnectorRole.SCHEDULER:
|
||||
assert self.connector_scheduler is not None
|
||||
# Clear local references to keys we're about to wipe.
|
||||
self.connector_scheduler.load_specs.clear()
|
||||
self._kv_cache_events = None
|
||||
return self.connector_scheduler.reset_store()
|
||||
return None
|
||||
|
||||
def update_connector_output(self, connector_output: KVConnectorOutput):
|
||||
assert self.connector_scheduler is not None
|
||||
self.connector_scheduler.update_connector_output(connector_output)
|
||||
|
||||
kv_cache_events = connector_output.kv_cache_events
|
||||
if not kv_cache_events or not isinstance(
|
||||
kv_cache_events, MooncakeStoreKVEvents
|
||||
):
|
||||
return
|
||||
|
||||
if self._kv_cache_events is None:
|
||||
self._kv_cache_events = kv_cache_events
|
||||
else:
|
||||
self._kv_cache_events.add_events(kv_cache_events.get_all_events())
|
||||
|
||||
def take_events(self) -> Iterable[KVCacheEvent]:
|
||||
if self._kv_cache_events is not None:
|
||||
events = self._kv_cache_events.pop_common_events()
|
||||
if not self._kv_cache_events.has_events():
|
||||
self._kv_cache_events = None
|
||||
yield from events
|
||||
|
||||
# ============================================================
|
||||
# Worker-side methods
|
||||
# ============================================================
|
||||
|
||||
def get_mem_pool_context(self) -> AbstractContextManager | None:
|
||||
"""Return a context manager for the custom MemPool, or None.
|
||||
|
||||
Called by the Worker before ``initialize_kv_cache`` so that KV
|
||||
cache is allocated from the Mooncake-managed pool when
|
||||
``custom_mem_pool`` is set in ``kv_connector_extra_config``.
|
||||
"""
|
||||
if self.connector_worker is None:
|
||||
return None
|
||||
return self.connector_worker.get_mem_pool_context()
|
||||
|
||||
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
|
||||
assert self.connector_worker is not None
|
||||
self.connector_worker.register_kv_caches(kv_caches)
|
||||
|
||||
def start_load_kv(self, forward_context: ForwardContext, **kwargs: Any) -> None:
|
||||
assert self.connector_worker is not None
|
||||
metadata = self._get_connector_metadata()
|
||||
assert isinstance(metadata, MooncakeStoreConnectorMetadata)
|
||||
self.connector_worker.start_load_kv(metadata)
|
||||
|
||||
def wait_for_layer_load(self, layer_name: str) -> None:
|
||||
# No layerwise support - no-op
|
||||
return
|
||||
|
||||
def save_kv_layer(
|
||||
self,
|
||||
layer_name: str,
|
||||
kv_layer: torch.Tensor,
|
||||
attn_metadata: AttentionMetadata,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
# No layerwise support - no-op
|
||||
return
|
||||
|
||||
def wait_for_save(self):
|
||||
assert self.connector_worker is not None
|
||||
metadata = self._get_connector_metadata()
|
||||
assert isinstance(metadata, MooncakeStoreConnectorMetadata)
|
||||
self.connector_worker.wait_for_save(metadata)
|
||||
|
||||
def get_finished(
|
||||
self, finished_req_ids: set[str]
|
||||
) -> tuple[set[str] | None, set[str] | None]:
|
||||
assert self.connector_worker is not None
|
||||
metadata = self._get_connector_metadata()
|
||||
assert isinstance(metadata, MooncakeStoreConnectorMetadata)
|
||||
return self.connector_worker.get_finished(finished_req_ids, metadata)
|
||||
|
||||
def get_transfer_results(
|
||||
self, finished_req_ids: set[str]
|
||||
) -> KVConnectorTransferResults:
|
||||
assert self.connector_worker is not None
|
||||
metadata = self._get_connector_metadata()
|
||||
assert isinstance(metadata, MooncakeStoreConnectorMetadata)
|
||||
return self.connector_worker.get_transfer_results(finished_req_ids, metadata)
|
||||
|
||||
def get_block_ids_with_load_errors(self) -> set[int]:
|
||||
assert self.connector_worker is not None
|
||||
return self.connector_worker.get_block_ids_with_load_errors()
|
||||
|
||||
def get_kv_connector_kv_cache_events(
|
||||
self,
|
||||
) -> MooncakeStoreKVEvents | None:
|
||||
assert self.connector_worker is not None
|
||||
if (
|
||||
not self.connector_worker.enable_kv_events
|
||||
or self.connector_worker.kv_send_thread is None
|
||||
):
|
||||
return None
|
||||
events = self.connector_worker.get_kv_events()
|
||||
# Empty containers still count this worker toward the poll's quorum.
|
||||
kv_events = MooncakeStoreKVEvents(
|
||||
num_workers=1,
|
||||
group_tp_replication_factors=(
|
||||
self.connector_worker.group_tp_replication_factors
|
||||
),
|
||||
)
|
||||
kv_events.add_events(events)
|
||||
return kv_events
|
||||
|
||||
def get_kv_connector_stats(self) -> KVConnectorStats | None:
|
||||
if self.connector_worker is None:
|
||||
return None
|
||||
return self.connector_worker.get_kv_connector_stats()
|
||||
|
||||
@classmethod
|
||||
def build_kv_connector_stats(
|
||||
cls, data: dict[str, Any] | None = None
|
||||
) -> KVConnectorStats | None:
|
||||
return (
|
||||
MooncakeStoreConnectorStats(data=data)
|
||||
if data is not None
|
||||
else MooncakeStoreConnectorStats()
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def build_prom_metrics(
|
||||
cls,
|
||||
vllm_config: VllmConfig,
|
||||
metric_types: dict[type[PromMetric], type[PromMetricT]],
|
||||
labelnames: list[str],
|
||||
per_engine_labelvalues: dict[int, list[object]],
|
||||
) -> KVConnectorPromMetrics:
|
||||
return MooncakeStorePromMetrics(
|
||||
vllm_config, metric_types, labelnames, per_engine_labelvalues
|
||||
)
|
||||
@@ -1,515 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""External-store cache-hit coordinator for MooncakeStoreConnector."""
|
||||
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import replace
|
||||
from typing import TYPE_CHECKING, NamedTuple, cast
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
|
||||
chunk_hashes_for_block_size,
|
||||
)
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.v1.core.block_pool import BlockPool
|
||||
from vllm.v1.core.kv_cache_coordinator import replay_boundaries
|
||||
from vllm.v1.core.kv_cache_utils import (
|
||||
BlockHash,
|
||||
KVCacheBlock,
|
||||
eagle_proof_margin,
|
||||
resolve_dcp_kv_cache_spec,
|
||||
)
|
||||
from vllm.v1.core.single_type_kv_cache_manager import (
|
||||
SingleTypeKVCacheManager,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheSpec,
|
||||
MambaSpec,
|
||||
UniformTypeKVCacheSpecs,
|
||||
)
|
||||
from vllm.v1.kv_cache_spec_registry import KVCacheSpecRegistry
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
|
||||
class StoreSpecGroup(NamedTuple):
|
||||
spec: KVCacheSpec
|
||||
group_ids: list[int]
|
||||
manager_cls: type[SingleTypeKVCacheManager]
|
||||
use_eagle: bool
|
||||
|
||||
|
||||
class ExternalCachedBlockPool:
|
||||
"""Duck-typed BlockPool backed by a ``(group_id, hash)`` exists set."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
hash_block_size: int,
|
||||
exists: set[tuple[int, bytes]] | None = None,
|
||||
) -> None:
|
||||
# ``exists=None`` is used on the recv side where hit_length is already
|
||||
# determined and we just want each spec's manager to apply its own mask.
|
||||
self._exists = exists
|
||||
self.hash_block_size = hash_block_size
|
||||
self.null_block = KVCacheBlock(block_id=0)
|
||||
# Dummy ID 1 for present block for duck-typing.
|
||||
self._present_block = KVCacheBlock(block_id=1)
|
||||
|
||||
def get_cached_block(
|
||||
self,
|
||||
block_hash: BlockHash,
|
||||
group_ids: list[int],
|
||||
) -> list[KVCacheBlock] | None:
|
||||
# Mirrors BlockPool.get_cached_block: hit only when every group_id
|
||||
# (groups sharing a spec) has the hash cached.
|
||||
if self._exists is None:
|
||||
return [self._present_block] * len(group_ids)
|
||||
h = bytes(block_hash)
|
||||
if all((g, h) in self._exists for g in group_ids):
|
||||
return [self._present_block] * len(group_ids)
|
||||
return None
|
||||
|
||||
def contains(self, group_id: int, block_hash: BlockHash) -> bool:
|
||||
"""Return whether a group has a loadable key for this hash."""
|
||||
return (
|
||||
self._exists is not None and (group_id, bytes(block_hash)) in self._exists
|
||||
)
|
||||
|
||||
|
||||
class MooncakeStoreCoordinator:
|
||||
"""Mirror of ``HybridKVCacheCoordinator.find_longest_cache_hit`` over an
|
||||
``ExternalCachedBlockPool``."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
kv_cache_groups: list[KVCacheGroupSpec],
|
||||
scheduler_block_size: int,
|
||||
hash_block_size: int,
|
||||
use_eagle: bool = False,
|
||||
retention_interval: int | None = None,
|
||||
enable_partial_hash_hits: bool = False,
|
||||
) -> None:
|
||||
# Mirrors core's resolve_kv_cache_block_sizes: the hash unit only has
|
||||
# to divide groups that participate in prefix caching. Non-shareable
|
||||
# scratch groups (e.g. GLM-5.3-Flash's kpool tail) are skipped by
|
||||
# _verify_and_split_kv_cache_groups and never probed for hits.
|
||||
assert all(
|
||||
g.kv_cache_spec.block_size % hash_block_size == 0
|
||||
for g in kv_cache_groups
|
||||
if g.kv_cache_spec.prefix_cacheable
|
||||
), "block_size must be divisible by hash_block_size"
|
||||
assert scheduler_block_size % hash_block_size == 0, (
|
||||
f"scheduler_block_size ({scheduler_block_size}) must be a multiple of "
|
||||
f"hash_block_size ({hash_block_size})"
|
||||
)
|
||||
assert all(
|
||||
scheduler_block_size % g.kv_cache_spec.block_size == 0
|
||||
for g in kv_cache_groups
|
||||
if g.kv_cache_spec.prefix_cacheable
|
||||
), "scheduler_block_size must be a multiple of each group's block_size"
|
||||
self.kv_cache_groups = kv_cache_groups
|
||||
self.mamba_group_ids = {
|
||||
group_id
|
||||
for group_id, group in enumerate(kv_cache_groups)
|
||||
if isinstance(_unwrap_spec(group.kv_cache_spec), MambaSpec)
|
||||
}
|
||||
self.hash_block_size = hash_block_size
|
||||
self.lcm_block_size = scheduler_block_size
|
||||
# Core's decision: whether hits land on the hash unit, below the block.
|
||||
self.enable_partial_hash_hits = enable_partial_hash_hits
|
||||
self.use_eagle = use_eagle
|
||||
# Mirror vLLM core's KVCacheCoordinator.retention_interval.
|
||||
self.retention_interval = retention_interval
|
||||
self._verify_and_split_kv_cache_groups()
|
||||
|
||||
@classmethod
|
||||
def from_kv_cache_config(
|
||||
cls,
|
||||
kv_cache_config: KVCacheConfig,
|
||||
vllm_config: "VllmConfig",
|
||||
scheduler_block_size: int,
|
||||
hash_block_size: int,
|
||||
) -> "MooncakeStoreCoordinator":
|
||||
"""Build the coordinator the connector's scheduler and workers share.
|
||||
|
||||
Specs are DCP-resolved, so a block spans the tokens of one block-table
|
||||
entry. Partial hits follow core's hit alignment, which checks every KV
|
||||
cache group and its manager, not only the groups the store transfers;
|
||||
unset (a config the engine core did not resolve) keeps hits on the
|
||||
block.
|
||||
"""
|
||||
dcp_world_size = vllm_config.parallel_config.decode_context_parallel_size
|
||||
spec_config = vllm_config.speculative_config
|
||||
alignment = kv_cache_config.cache_hit_alignment_tokens
|
||||
return cls(
|
||||
[
|
||||
replace(
|
||||
group,
|
||||
kv_cache_spec=resolve_dcp_kv_cache_spec(
|
||||
group.kv_cache_spec, dcp_world_size
|
||||
),
|
||||
)
|
||||
for group in kv_cache_config.prefix_cacheable_groups
|
||||
],
|
||||
scheduler_block_size,
|
||||
hash_block_size,
|
||||
use_eagle=spec_config is not None and spec_config.use_eagle_block_drop(),
|
||||
retention_interval=kv_cache_config.prefix_cache_retention_interval,
|
||||
enable_partial_hash_hits=(
|
||||
alignment is not None and alignment < scheduler_block_size
|
||||
),
|
||||
)
|
||||
|
||||
def align_lookup_length(self, length: int) -> int:
|
||||
alignment = (
|
||||
self.hash_block_size
|
||||
if self.enable_partial_hash_hits
|
||||
else self.lcm_block_size
|
||||
)
|
||||
return length // alignment * alignment
|
||||
|
||||
def _verify_and_split_kv_cache_groups(self) -> None:
|
||||
"""Mirrors KVCacheCoordinator.verify_and_split_kv_cache_groups but
|
||||
dispatches via spec_manager_map (we don't allocate managers).
|
||||
"""
|
||||
attention_groups: list[StoreSpecGroup] = []
|
||||
for i, g in enumerate(self.kv_cache_groups):
|
||||
# Skip groups that opt out of prefix caching (e.g. GLM-5.3-Flash
|
||||
# kpool tail): per-request scratch, never shareable, so they must
|
||||
# not participate in hit lookup. Mirrors core's
|
||||
# KVCacheCoordinator.verify_and_split_kv_cache_groups.
|
||||
if not g.kv_cache_spec.prefix_cacheable:
|
||||
continue
|
||||
spec = _unwrap_spec(g.kv_cache_spec)
|
||||
manager_cls = KVCacheSpecRegistry.get_manager_class(spec)
|
||||
assert manager_cls is not None, (
|
||||
f"No manager registered for KVCacheSpec {spec}"
|
||||
)
|
||||
for idx, group in enumerate(attention_groups):
|
||||
if group.spec == spec:
|
||||
assert manager_cls is group.manager_cls
|
||||
group.group_ids.append(i)
|
||||
if g.is_eagle_group and not group.use_eagle:
|
||||
attention_groups[idx] = group._replace(use_eagle=True)
|
||||
break
|
||||
else:
|
||||
attention_groups.append(
|
||||
StoreSpecGroup(spec, [i], manager_cls, g.is_eagle_group)
|
||||
)
|
||||
# Full attention first (matches upstream convergence ordering).
|
||||
attention_groups.sort(key=lambda g: not isinstance(g.spec, FullAttentionSpec))
|
||||
# Conservatively flag all groups when use_eagle is set but none is flagged.
|
||||
if self.use_eagle and not any(g.use_eagle for g in attention_groups):
|
||||
attention_groups = [g._replace(use_eagle=True) for g in attention_groups]
|
||||
self.attention_groups = attention_groups
|
||||
# Per-group eagle bits. SpecGroup carries use_eagle for the whole
|
||||
# merged spec group, so the per-group store/lookup masks agree with
|
||||
# the merged-group hit check, which applies the eagle drop to every
|
||||
# group sharing the spec.
|
||||
self.eagle_group_ids = {
|
||||
gid for g in attention_groups if g.use_eagle for gid in g.group_ids
|
||||
}
|
||||
self.eagle_proof_margin_by_group = {
|
||||
gid: eagle_proof_margin(
|
||||
group.spec.block_size,
|
||||
self.hash_block_size,
|
||||
self.enable_partial_hash_hits
|
||||
and group.manager_cls.supports_fine_grained_hash_lookup,
|
||||
)
|
||||
for group in attention_groups
|
||||
if group.use_eagle and not isinstance(group.spec, MambaSpec)
|
||||
for gid in group.group_ids
|
||||
}
|
||||
|
||||
def find_longest_cache_hit(
|
||||
self,
|
||||
block_hashes: Sequence[BlockHash],
|
||||
max_length: int,
|
||||
cached_block_pool: ExternalCachedBlockPool,
|
||||
*,
|
||||
apply_eagle: bool = True,
|
||||
) -> tuple[tuple[list[bool], ...], int]:
|
||||
"""Returns ``(load_mask_per_group, hit_length)``. ``mask[g][i]`` is True iff
|
||||
group ``g`` populates chunk ``i`` locally (e.g. SWA and Mamba tail-only);
|
||||
recv-side callers skip False slots.
|
||||
|
||||
``apply_eagle`` controls whether the per-spec ``use_eagle`` last-block
|
||||
pop is applied. Lookup callers want it (the drafter requires recomputing
|
||||
the last block); per-chunk mask callers must not, because ``token_len``
|
||||
already reflects the eagle-pruned hit length and a second pop would
|
||||
leave the trailing block unloaded.
|
||||
"""
|
||||
blocks_per_group, hit_length = self._find_hit_blocks(
|
||||
block_hashes, max_length, cached_block_pool, apply_eagle=apply_eagle
|
||||
)
|
||||
masks = tuple(
|
||||
[blk is not cached_block_pool.null_block for blk in blocks]
|
||||
for blocks in blocks_per_group
|
||||
)
|
||||
return masks, hit_length
|
||||
|
||||
def load_mask(
|
||||
self,
|
||||
block_hashes: Sequence[BlockHash],
|
||||
token_len: int,
|
||||
) -> tuple[list[bool], ...]:
|
||||
"""Per-group load masks: ``mask[g][i]`` is True iff group ``g``'s
|
||||
spec would populate chunk ``i`` locally at length ``token_len``
|
||||
(e.g. SWA / Mamba tail-only).
|
||||
"""
|
||||
# ``apply_eagle=False`` because ``token_len`` is already the
|
||||
# eagle-pruned hit length returned by ``client.lookup``. Re-applying
|
||||
# the pop here would shorten the mask by one extra block; the recv
|
||||
# thread would then silently skip the trailing chunk yielded by
|
||||
# ``db.process_tokens`` and leave that block uninitialized in the
|
||||
# local KV pool.
|
||||
masks, _ = self.find_longest_cache_hit(
|
||||
block_hashes,
|
||||
token_len,
|
||||
ExternalCachedBlockPool(self.hash_block_size),
|
||||
apply_eagle=False,
|
||||
)
|
||||
return masks
|
||||
|
||||
def store_mask(
|
||||
self,
|
||||
aligned_token_len: int,
|
||||
start_token: int = 0,
|
||||
num_prompt_tokens: int | None = None,
|
||||
) -> tuple[list[bool] | None, ...]:
|
||||
"""Per-group store masks for the suffix starting at ``start_token``.
|
||||
|
||||
``mask[g][i]`` is True iff the i-th chunk of group ``g`` *after*
|
||||
``start_token`` should be written to the store so a future cache hit
|
||||
can consume it. ``None`` is the all-True sentinel for the suffix.
|
||||
|
||||
Reuses the engine's ``SingleTypeKVCacheManager.reachable_block_mask``
|
||||
so the store retains exactly the blocks the local prefix cache would.
|
||||
|
||||
Mamba groups are always all-False: the normal save resolves blocks
|
||||
positionally from the connector's append-only block-ID snapshot, but
|
||||
an align-mode mamba block table is not append-only (interior state
|
||||
blocks are nulled/freed, and speculative decoding relocates the spec
|
||||
blocks in place), so a positional read may hit a null, freed, or live
|
||||
speculative-state block and persist wrong bytes under a valid prefix
|
||||
hash. Mamba state is persisted only through the connector-pinned exact
|
||||
block hand-off path
|
||||
(``SchedulerOutput.kv_connector_block_state.boundary_state_offloads``).
|
||||
"""
|
||||
return self._reachable_masks(
|
||||
aligned_token_len,
|
||||
start_token,
|
||||
retention_interval=self.retention_interval,
|
||||
num_prompt_tokens=num_prompt_tokens,
|
||||
exclude_mamba=True,
|
||||
)
|
||||
|
||||
def lookup_mask(
|
||||
self,
|
||||
aligned_token_len: int,
|
||||
) -> tuple[list[bool] | None, ...]:
|
||||
"""Per-group lookup masks.
|
||||
|
||||
``mask[g][i]`` is True iff chunk ``i`` of group ``g`` should be
|
||||
looked up as an aligned hit boundary. ``None`` is the all-True
|
||||
sentinel.
|
||||
"""
|
||||
return self._reachable_masks(
|
||||
aligned_token_len,
|
||||
0,
|
||||
retention_interval=None,
|
||||
num_prompt_tokens=None,
|
||||
)
|
||||
|
||||
def _reachable_masks(
|
||||
self,
|
||||
aligned_token_len: int,
|
||||
start_token: int,
|
||||
*,
|
||||
retention_interval: int | None,
|
||||
num_prompt_tokens: int | None,
|
||||
exclude_mamba: bool = False,
|
||||
) -> tuple[list[bool] | None, ...]:
|
||||
mask_alignment = (
|
||||
self.hash_block_size
|
||||
if self.enable_partial_hash_hits
|
||||
else self.lcm_block_size
|
||||
)
|
||||
assert aligned_token_len % mask_alignment == 0, (
|
||||
f"aligned_token_len ({aligned_token_len}) must be a multiple of "
|
||||
f"{mask_alignment}"
|
||||
)
|
||||
reachable_boundaries = (
|
||||
()
|
||||
if num_prompt_tokens is None
|
||||
else replay_boundaries(
|
||||
num_prompt_tokens,
|
||||
self.lcm_block_size,
|
||||
use_eagle=bool(self.eagle_group_ids),
|
||||
)
|
||||
)
|
||||
masks: list[list[bool] | None] = []
|
||||
for g_idx, g in enumerate(self.kv_cache_groups):
|
||||
spec = _unwrap_spec(g.kv_cache_spec)
|
||||
end_chunk = aligned_token_len // spec.block_size
|
||||
start_chunk = min(end_chunk, max(0, cdiv(start_token, spec.block_size)))
|
||||
if exclude_mamba and isinstance(spec, MambaSpec):
|
||||
masks.append([False] * (end_chunk - start_chunk))
|
||||
continue
|
||||
manager_cls = KVCacheSpecRegistry.get_manager_class(spec)
|
||||
assert manager_cls is not None
|
||||
use_eagle = g_idx in self.eagle_group_ids
|
||||
mask = manager_cls.reachable_block_mask(
|
||||
start_block=start_chunk,
|
||||
end_block=end_chunk,
|
||||
alignment_tokens=self.lcm_block_size,
|
||||
kv_cache_spec=spec,
|
||||
use_eagle=use_eagle,
|
||||
retention_interval=retention_interval,
|
||||
reachable_boundaries=reachable_boundaries,
|
||||
# ``spec`` is already DCP-resolved (worker.py applies
|
||||
# resolve_dcp_kv_cache_spec) and ``end_chunk`` is indexed in
|
||||
# that scaled block size, so the mask must not scale again.
|
||||
dcp_world_size=1,
|
||||
)
|
||||
if mask is not None:
|
||||
assert len(mask) == end_chunk - start_chunk
|
||||
masks.append(mask)
|
||||
return tuple(masks)
|
||||
|
||||
def block_hashes_for_spec(
|
||||
self, block_hashes: Sequence[BlockHash], spec: KVCacheSpec
|
||||
) -> Sequence[BlockHash]:
|
||||
return chunk_hashes_for_block_size(
|
||||
block_hashes, self.hash_block_size, spec.block_size
|
||||
)
|
||||
|
||||
def _find_hit_blocks(
|
||||
self,
|
||||
block_hashes: Sequence[BlockHash],
|
||||
max_length: int,
|
||||
cached_block_pool: ExternalCachedBlockPool,
|
||||
*,
|
||||
apply_eagle: bool = True,
|
||||
) -> tuple[tuple[list[KVCacheBlock], ...], int]:
|
||||
"""Mirrors HybridKVCacheCoordinator.find_longest_cache_hit but
|
||||
dispatches via spec_manager_map (we don't allocate managers).
|
||||
|
||||
When ``apply_eagle`` is False, ignore each group's ``use_eagle`` —
|
||||
used by ``load_mask`` to avoid popping a second block on top of the
|
||||
one already removed by the lookup.
|
||||
"""
|
||||
alignment_tokens = (
|
||||
self.hash_block_size
|
||||
if self.enable_partial_hash_hits
|
||||
else self.lcm_block_size
|
||||
)
|
||||
if len(self.attention_groups) == 1:
|
||||
spec, group_ids, manager_cls, group_eagle = self.attention_groups[0]
|
||||
hit_blocks, hit_length = manager_cls.find_longest_cache_hit(
|
||||
block_hashes=block_hashes, # type: ignore[arg-type]
|
||||
max_length=max_length,
|
||||
kv_cache_group_ids=group_ids,
|
||||
block_pool=cast(BlockPool, cached_block_pool),
|
||||
kv_cache_spec=spec,
|
||||
drop_eagle_block=apply_eagle and group_eagle,
|
||||
alignment_tokens=alignment_tokens,
|
||||
)
|
||||
num_groups = len(self.kv_cache_groups)
|
||||
blocks_by_group: list[list[KVCacheBlock]] = [[] for _ in range(num_groups)]
|
||||
for gid, blks in zip(group_ids, hit_blocks, strict=True):
|
||||
blocks_by_group[gid] = blks
|
||||
return tuple(blocks_by_group), hit_length
|
||||
|
||||
num_groups = len(self.kv_cache_groups)
|
||||
hit_length = max_length
|
||||
hit_blocks_by_group: list[list[KVCacheBlock] | None] = [None] * num_groups
|
||||
hit_length_by_group: list[int] = [0] * num_groups
|
||||
|
||||
is_simple_hybrid = len(self.attention_groups) == 2 and isinstance(
|
||||
self.attention_groups[0].spec, FullAttentionSpec
|
||||
)
|
||||
eagle_verified: set[int] = set()
|
||||
|
||||
while True:
|
||||
curr_hit_length = hit_length
|
||||
|
||||
for idx, (spec, group_ids, manager_cls, group_eagle) in enumerate(
|
||||
self.attention_groups
|
||||
):
|
||||
first_group_id = group_ids[0]
|
||||
cached = hit_blocks_by_group[first_group_id]
|
||||
if isinstance(spec, FullAttentionSpec) and cached is not None:
|
||||
curr_hit_length = min(
|
||||
curr_hit_length, hit_length_by_group[first_group_id]
|
||||
)
|
||||
continue
|
||||
|
||||
drop_eagle_block = (
|
||||
apply_eagle and group_eagle and idx not in eagle_verified
|
||||
)
|
||||
_max_length = curr_hit_length
|
||||
# No eagle peek margin for a recurrent (Mamba) group: its finder
|
||||
# never drops a block, so a widened bound would match past the
|
||||
# attention-verified hit and resume from speculative state (#43559).
|
||||
if drop_eagle_block and not isinstance(spec, MambaSpec):
|
||||
eagle_margin = eagle_proof_margin(
|
||||
spec.block_size,
|
||||
self.hash_block_size,
|
||||
self.enable_partial_hash_hits
|
||||
and manager_cls.supports_fine_grained_hash_lookup,
|
||||
)
|
||||
_max_length = min(
|
||||
curr_hit_length + eagle_margin,
|
||||
len(block_hashes) * self.hash_block_size,
|
||||
)
|
||||
hit_blocks, _new_hit_length = manager_cls.find_longest_cache_hit(
|
||||
block_hashes=block_hashes, # type: ignore[arg-type]
|
||||
max_length=_max_length,
|
||||
kv_cache_group_ids=group_ids,
|
||||
block_pool=cast(BlockPool, cached_block_pool),
|
||||
kv_cache_spec=spec,
|
||||
drop_eagle_block=drop_eagle_block,
|
||||
alignment_tokens=alignment_tokens,
|
||||
)
|
||||
if drop_eagle_block:
|
||||
eagle_verified.add(idx)
|
||||
elif _new_hit_length < curr_hit_length:
|
||||
eagle_verified.clear()
|
||||
curr_hit_length = _new_hit_length
|
||||
for gid, blocks in zip(group_ids, hit_blocks, strict=True):
|
||||
hit_blocks_by_group[gid] = blocks
|
||||
hit_length_by_group[gid] = _new_hit_length
|
||||
|
||||
if curr_hit_length >= hit_length:
|
||||
break
|
||||
hit_length = curr_hit_length
|
||||
if is_simple_hybrid:
|
||||
break
|
||||
|
||||
# Truncate full-attention hit_blocks to final converged length;
|
||||
# other specs already trim themselves inside their hit logic. cdiv keeps
|
||||
# the partial tail block when hit_length is not block-aligned.
|
||||
for group in self.attention_groups:
|
||||
if not isinstance(group.spec, FullAttentionSpec):
|
||||
continue
|
||||
num_blocks = cdiv(hit_length, group.spec.block_size)
|
||||
for group_id in group.group_ids:
|
||||
full_blks = hit_blocks_by_group[group_id]
|
||||
assert full_blks is not None
|
||||
del full_blks[num_blocks:]
|
||||
hit_length_by_group[group_id] = hit_length
|
||||
|
||||
return (
|
||||
tuple(blks if blks is not None else [] for blks in hit_blocks_by_group),
|
||||
hit_length,
|
||||
)
|
||||
|
||||
|
||||
def _unwrap_spec(spec: KVCacheSpec) -> KVCacheSpec:
|
||||
if isinstance(spec, UniformTypeKVCacheSpecs):
|
||||
return next(iter(spec.kv_cache_specs.values()))
|
||||
return spec
|
||||
@@ -1,898 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
#
|
||||
# Adapted from vllm-project/vllm-ascend
|
||||
# (vllm_ascend/distributed/kv_transfer/kv_pool/ascend_store/).
|
||||
"""Data classes for MooncakeStoreConnector."""
|
||||
|
||||
from collections.abc import Iterable, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from typing import NamedTuple, cast
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorMetadata,
|
||||
KVConnectorWorkerMetadata,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.utils.torch_utils import is_non_overlapping_and_dense
|
||||
from vllm.v1.core.kv_cache_utils import (
|
||||
BlockHash,
|
||||
BlockHashListWithBlockSize,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class BlobBlockHashes(Sequence[BlockHash]):
|
||||
"""Lazy view over a flat buffer of fixed-size block hashes to avoid the overhead
|
||||
of materializing all hashes upfront.
|
||||
"""
|
||||
|
||||
def __init__(self, blob: memoryview, hash_len: int):
|
||||
self._blob = blob
|
||||
self._hash_len = hash_len
|
||||
self._n = len(blob) // hash_len if hash_len else 0
|
||||
|
||||
def __len__(self) -> int:
|
||||
return self._n
|
||||
|
||||
def __getitem__(self, idx):
|
||||
if isinstance(idx, slice):
|
||||
return [self[i] for i in range(*idx.indices(self._n))]
|
||||
if idx < 0:
|
||||
idx += self._n
|
||||
if not 0 <= idx < self._n:
|
||||
raise IndexError(idx)
|
||||
off = idx * self._hash_len
|
||||
return BlockHash(self._blob[off : off + self._hash_len])
|
||||
|
||||
|
||||
class _CompactChunkHashList(BlockHashListWithBlockSize):
|
||||
"""View that keys each ``block_size`` chunk by the last constituent
|
||||
``hash_block_size`` hash instead of concatenating all of them.
|
||||
|
||||
The engine chains block hashes (each hash folds in the previous one), so the
|
||||
final sub-block hash of a chunk already uniquely identifies the whole chunk
|
||||
and its prefix. Using it keeps a Mooncake key at a single hash digest
|
||||
regardless of the ``block_size`` / ``hash_block_size`` ratio, instead of
|
||||
growing the key linearly with it (e.g. 64x for ``block_size=256``,
|
||||
``hash_block_size=4``).
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
block_hashes: Sequence[BlockHash],
|
||||
hash_block_size: int,
|
||||
target_block_size: int,
|
||||
):
|
||||
# Accept any indexable sequence (e.g. the lazy ``BlobBlockHashes``), not
|
||||
# just ``list``; the base only indexes/sizes it.
|
||||
assert target_block_size % hash_block_size == 0
|
||||
self.block_hashes = block_hashes # type: ignore[assignment]
|
||||
self.scale_factor = target_block_size // hash_block_size
|
||||
|
||||
def _get_value_at(self, idx: int) -> BlockHash:
|
||||
return self.block_hashes[idx * self.scale_factor + self.scale_factor - 1]
|
||||
|
||||
|
||||
def chunk_hashes_for_block_size(
|
||||
block_hashes: Sequence[BlockHash],
|
||||
hash_block_size: int,
|
||||
block_size: int,
|
||||
) -> Sequence[BlockHash]:
|
||||
"""Map ``hash_block_size``-granular block hashes to one compact hash per
|
||||
``block_size`` chunk (the chunk's last sub-hash). Returns ``block_hashes``
|
||||
unchanged when the two sizes are equal.
|
||||
"""
|
||||
if block_size == hash_block_size:
|
||||
return block_hashes
|
||||
# Structurally a Sequence[BlockHash] (indexable + sized); the base class
|
||||
# just isn't declared as one.
|
||||
return cast(
|
||||
"Sequence[BlockHash]",
|
||||
_CompactChunkHashList(block_hashes, hash_block_size, block_size),
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class KeyMetadata:
|
||||
"""Metadata for constructing pool keys."""
|
||||
|
||||
model_name: str
|
||||
tp_rank: int
|
||||
pcp_rank: int
|
||||
dcp_rank: int
|
||||
pp_rank: int
|
||||
group_id: int = 0
|
||||
# Optional namespace prepended to every key. Lets separate deployments
|
||||
# share one Mooncake master without colliding on identical block hashes.
|
||||
# Empty (the default) keeps keys byte-identical to the unprefixed format.
|
||||
cache_prefix: str = ""
|
||||
# Complete namespace for an opt-in Store payload format. Empty keeps
|
||||
# historical keys byte-identical.
|
||||
store_namespace: str = ""
|
||||
|
||||
|
||||
@dataclass(order=True)
|
||||
class PoolKey:
|
||||
"""Key for addressing KV cache blocks in the distributed store."""
|
||||
|
||||
key_metadata: KeyMetadata
|
||||
chunk_hash: str
|
||||
|
||||
def __hash__(self):
|
||||
return hash(
|
||||
(
|
||||
self.key_metadata.cache_prefix,
|
||||
self.key_metadata.model_name,
|
||||
self.key_metadata.store_namespace,
|
||||
self.key_metadata.tp_rank,
|
||||
self.key_metadata.pcp_rank,
|
||||
self.key_metadata.dcp_rank,
|
||||
self.key_metadata.pp_rank,
|
||||
self.key_metadata.group_id,
|
||||
self.chunk_hash,
|
||||
)
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def build_prefix(
|
||||
key_metadata: KeyMetadata,
|
||||
*,
|
||||
tp_rank: int | None = None,
|
||||
pcp_rank: int | None = None,
|
||||
dcp_rank: int | None = None,
|
||||
pp_rank: int | None = None,
|
||||
) -> str:
|
||||
"""Return the stable prefix for a Mooncake pool key."""
|
||||
prefix = f"{key_metadata.cache_prefix}@" if key_metadata.cache_prefix else ""
|
||||
return (
|
||||
f"{prefix}"
|
||||
f"{key_metadata.model_name}"
|
||||
f"{key_metadata.store_namespace}"
|
||||
f"@tp_rank:{key_metadata.tp_rank if tp_rank is None else tp_rank}"
|
||||
f"@pcp{key_metadata.pcp_rank if pcp_rank is None else pcp_rank}"
|
||||
f"@dcp{key_metadata.dcp_rank if dcp_rank is None else dcp_rank}"
|
||||
f"@pp_rank:{key_metadata.pp_rank if pp_rank is None else pp_rank}"
|
||||
f"@group:{key_metadata.group_id}"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def build_key_string(key_prefix: str, chunk_hash: str) -> str:
|
||||
return f"{key_prefix}@{chunk_hash}"
|
||||
|
||||
def to_string(self) -> str:
|
||||
return self.build_key_string(
|
||||
self.build_prefix(self.key_metadata), self.chunk_hash
|
||||
)
|
||||
|
||||
|
||||
StoreShardId = int
|
||||
# Physical ranks encoded in a rank-local key: (TP, PCP, DCP, PP).
|
||||
RankNamespace = tuple[int, int, int, int]
|
||||
|
||||
|
||||
class StoreLayout:
|
||||
"""Store payload layout for one KV-cache group on one local TP rank."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
metadata: KeyMetadata,
|
||||
block_size: int,
|
||||
hash_block_size: int,
|
||||
) -> None:
|
||||
self.metadata = metadata
|
||||
self.block_size = block_size
|
||||
self.hash_block_size = hash_block_size
|
||||
|
||||
@property
|
||||
def local_shard_ids(self) -> tuple[StoreShardId, ...]:
|
||||
"""Store objects contributed or consumed by this local rank."""
|
||||
raise NotImplementedError
|
||||
|
||||
def key_for(self, shard_id: StoreShardId, chunk_hash: BlockHash) -> str:
|
||||
raise NotImplementedError
|
||||
|
||||
def lookup_key_prefixes(
|
||||
self,
|
||||
rank_namespaces: Sequence[RankNamespace],
|
||||
) -> tuple[str, ...]:
|
||||
"""Prefixes that must exist for a logical block to be reusable."""
|
||||
raise NotImplementedError
|
||||
|
||||
def register_kv_caches(
|
||||
self,
|
||||
kv_caches: Sequence[torch.Tensor],
|
||||
num_blocks: int,
|
||||
) -> None:
|
||||
"""Build descriptors from the local KV cache tensors."""
|
||||
raise NotImplementedError
|
||||
|
||||
def prepare_values(
|
||||
self,
|
||||
chunks: Sequence[tuple[int, int]],
|
||||
block_ids: list[int],
|
||||
shard_ids: Sequence[StoreShardId],
|
||||
) -> tuple[list[list[int]], list[list[int]], list[int]]:
|
||||
"""Build Store multi-buffer descriptors for logical chunks."""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class RankLocalStoreLayout(StoreLayout):
|
||||
"""Historical rank-local Store payload layout."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
metadata: KeyMetadata,
|
||||
block_size: int,
|
||||
hash_block_size: int,
|
||||
) -> None:
|
||||
super().__init__(metadata, block_size, hash_block_size)
|
||||
self._key_prefix = PoolKey.build_prefix(metadata)
|
||||
self.kv_caches_base_addr: list[int] = []
|
||||
self.block_len: list[int] = []
|
||||
|
||||
@property
|
||||
def local_shard_ids(self) -> tuple[StoreShardId, ...]:
|
||||
return (0,)
|
||||
|
||||
def key_for(self, shard_id: StoreShardId, chunk_hash: BlockHash) -> str:
|
||||
assert shard_id == 0
|
||||
return PoolKey.build_key_string(self._key_prefix, chunk_hash.hex())
|
||||
|
||||
def lookup_key_prefixes(
|
||||
self,
|
||||
rank_namespaces: Sequence[RankNamespace],
|
||||
) -> tuple[str, ...]:
|
||||
return tuple(
|
||||
PoolKey.build_prefix(
|
||||
self.metadata,
|
||||
tp_rank=tp_rank,
|
||||
pcp_rank=pcp_rank,
|
||||
dcp_rank=dcp_rank,
|
||||
pp_rank=pp_rank,
|
||||
)
|
||||
for tp_rank, pcp_rank, dcp_rank, pp_rank in rank_namespaces
|
||||
)
|
||||
|
||||
def set_kv_caches_base_addr(self, base_addrs: list[int]) -> None:
|
||||
self.kv_caches_base_addr = base_addrs
|
||||
|
||||
def set_block_len(self, block_lens: list[int]) -> None:
|
||||
self.block_len = block_lens
|
||||
|
||||
def register_kv_caches(
|
||||
self,
|
||||
kv_caches: Sequence[torch.Tensor],
|
||||
num_blocks: int,
|
||||
) -> None:
|
||||
seen_region_ptrs: set[int] = set()
|
||||
base_addrs: list[int] = []
|
||||
block_lens: list[int] = []
|
||||
for cache in kv_caches:
|
||||
storage = cache.untyped_storage()
|
||||
storage_addr = storage.data_ptr()
|
||||
storage_len = storage.nbytes()
|
||||
if not is_non_overlapping_and_dense(cache[0]):
|
||||
for head_idx in range(cache.shape[1]):
|
||||
head_cache = cache[:, head_idx]
|
||||
assert is_non_overlapping_and_dense(head_cache[0])
|
||||
region_addr = head_cache.data_ptr()
|
||||
if region_addr in seen_region_ptrs:
|
||||
continue
|
||||
seen_region_ptrs.add(region_addr)
|
||||
base_addrs.append(region_addr)
|
||||
block_lens.append(head_cache.stride(0) * head_cache.element_size())
|
||||
elif cache.stride(0) * cache.element_size() * num_blocks == storage_len:
|
||||
if storage_addr in seen_region_ptrs:
|
||||
continue
|
||||
seen_region_ptrs.add(storage_addr)
|
||||
base_addrs.append(storage_addr)
|
||||
block_lens.append(storage_len // num_blocks)
|
||||
else:
|
||||
region_addr = cache.data_ptr()
|
||||
if region_addr in seen_region_ptrs:
|
||||
continue
|
||||
seen_region_ptrs.add(region_addr)
|
||||
base_addrs.append(region_addr)
|
||||
block_lens.append(cache.stride(0) * cache.element_size())
|
||||
self.kv_caches_base_addr = base_addrs
|
||||
self.block_len = block_lens
|
||||
|
||||
def prepare_values(
|
||||
self,
|
||||
chunks: Sequence[tuple[int, int]],
|
||||
block_ids: list[int],
|
||||
shard_ids: Sequence[StoreShardId],
|
||||
) -> tuple[list[list[int]], list[list[int]], list[int]]:
|
||||
if not chunks:
|
||||
return [], [], []
|
||||
assert len(chunks) == len(shard_ids) and all(
|
||||
shard_id == 0 for shard_id in shard_ids
|
||||
)
|
||||
base = np.asarray(self.kv_caches_base_addr, dtype=np.int64)
|
||||
length = len(self.block_len)
|
||||
blen = np.asarray(
|
||||
[self.block_len[i % length] for i in range(base.shape[0])],
|
||||
dtype=np.int64,
|
||||
)
|
||||
n = len(chunks)
|
||||
starts = np.fromiter((c[0] for c in chunks), dtype=np.int64, count=n)
|
||||
spans = np.fromiter((c[1] for c in chunks), dtype=np.int64, count=n) - starts
|
||||
assert np.all(spans % self.hash_block_size == 0)
|
||||
bids = np.fromiter(
|
||||
(block_ids[i] for i in (starts // self.block_size).tolist()),
|
||||
dtype=np.int64,
|
||||
count=n,
|
||||
)
|
||||
addrs = base[None, :] + bids[:, None] * blen[None, :]
|
||||
block_counts = (spans + self.block_size - 1) // self.block_size
|
||||
sizes = blen[None, :] * block_counts[:, None]
|
||||
return addrs.tolist(), sizes.tolist(), bids.tolist()
|
||||
|
||||
def prepare_value_for_block(self, block_id: int) -> tuple[list[int], list[int]]:
|
||||
length = len(self.block_len)
|
||||
return (
|
||||
[
|
||||
base_addr + block_id * self.block_len[index % length]
|
||||
for index, base_addr in enumerate(self.kv_caches_base_addr)
|
||||
],
|
||||
[
|
||||
self.block_len[index % length]
|
||||
for index in range(len(self.kv_caches_base_addr))
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
class TPShardedStoreLayout(StoreLayout):
|
||||
"""Store layout shared by divisible TP sizes."""
|
||||
|
||||
store_format: str
|
||||
|
||||
@classmethod
|
||||
def shared_namespace(cls, store_tp_size: int, pp_size: int) -> str:
|
||||
return (
|
||||
f"@store_tp:{store_tp_size}@store_pp:{pp_size}"
|
||||
f"@store_format:{cls.store_format}"
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
metadata: KeyMetadata,
|
||||
block_size: int,
|
||||
hash_block_size: int,
|
||||
local_tp_size: int,
|
||||
store_tp_size: int,
|
||||
tp_rank: int,
|
||||
num_kv_heads: int,
|
||||
) -> None:
|
||||
super().__init__(metadata, block_size, hash_block_size)
|
||||
self.shards_per_rank = store_tp_size // local_tp_size
|
||||
first_shard = tp_rank * self.shards_per_rank
|
||||
self.store_shard_ids = tuple(
|
||||
range(first_shard, first_shard + self.shards_per_rank)
|
||||
)
|
||||
self.store_tp_size = store_tp_size
|
||||
self.heads_per_store_shard = num_kv_heads // store_tp_size
|
||||
self.local_num_kv_heads = num_kv_heads // local_tp_size
|
||||
self._shard_key_prefixes = {
|
||||
shard_id: PoolKey.build_prefix(metadata, tp_rank=shard_id)
|
||||
for shard_id in self.store_shard_ids
|
||||
}
|
||||
self._shard_addr_bases = np.empty((self.shards_per_rank, 0), dtype=np.uint64)
|
||||
self._shard_block_strides = np.empty((self.shards_per_rank, 0), dtype=np.uint64)
|
||||
self._shard_sizes = np.empty((self.shards_per_rank, 0), dtype=np.uint64)
|
||||
|
||||
@property
|
||||
def local_shard_ids(self) -> tuple[StoreShardId, ...]:
|
||||
return self.store_shard_ids
|
||||
|
||||
def key_for(self, shard_id: StoreShardId, chunk_hash: BlockHash) -> str:
|
||||
return PoolKey.build_key_string(
|
||||
self._shard_key_prefixes[shard_id], chunk_hash.hex()
|
||||
)
|
||||
|
||||
def lookup_key_prefixes(
|
||||
self,
|
||||
rank_namespaces: Sequence[RankNamespace],
|
||||
) -> tuple[str, ...]:
|
||||
pp_ranks = sorted({namespace[3] for namespace in rank_namespaces})
|
||||
return tuple(
|
||||
PoolKey.build_prefix(self.metadata, tp_rank=shard_id, pp_rank=pp_rank)
|
||||
for shard_id in range(self.store_tp_size)
|
||||
for pp_rank in pp_ranks
|
||||
)
|
||||
|
||||
def _set_segment_templates(
|
||||
self, templates: Sequence[tuple[list[int], list[int], list[int]]]
|
||||
) -> None:
|
||||
self._shard_addr_bases = np.asarray(
|
||||
[template[0] for template in templates], dtype=np.uint64
|
||||
)
|
||||
self._shard_block_strides = np.asarray(
|
||||
[template[1] for template in templates], dtype=np.uint64
|
||||
)
|
||||
self._shard_sizes = np.asarray(
|
||||
[template[2] for template in templates], dtype=np.uint64
|
||||
)
|
||||
|
||||
def _has_packed_strides(
|
||||
self, head_stride: int, token_stride: int, content_bytes: int
|
||||
) -> bool:
|
||||
raise NotImplementedError
|
||||
|
||||
def _shard_segments(
|
||||
self, head_start: int, head_stride: int, token_stride: int, content_bytes: int
|
||||
) -> Iterable[tuple[int, int]]:
|
||||
raise NotImplementedError
|
||||
|
||||
def register_kv_caches(
|
||||
self,
|
||||
kv_caches: Sequence[torch.Tensor],
|
||||
num_blocks: int,
|
||||
) -> None:
|
||||
templates: list[tuple[list[int], list[int], list[int]]] = [
|
||||
([], [], []) for _ in range(self.shards_per_rank)
|
||||
]
|
||||
for cache in kv_caches:
|
||||
if cache.ndim != 4 or tuple(cache.shape[:3]) != (
|
||||
num_blocks,
|
||||
self.local_num_kv_heads,
|
||||
self.block_size,
|
||||
):
|
||||
raise ValueError(
|
||||
"TP-shared Mooncake store requires packed KV caches with "
|
||||
"logical shape (num_blocks, local_kv_heads, block_size, content)"
|
||||
)
|
||||
|
||||
element_size = cache.element_size()
|
||||
block_stride, head_stride, token_stride, content_stride = (
|
||||
stride * element_size for stride in cache.stride()
|
||||
)
|
||||
content_bytes = cache.shape[3] * element_size
|
||||
if content_stride != element_size or not self._has_packed_strides(
|
||||
head_stride, token_stride, content_bytes
|
||||
):
|
||||
raise ValueError(
|
||||
"TP-shared Mooncake store requires packed "
|
||||
f"{self.store_format} KV layout"
|
||||
)
|
||||
|
||||
for local_shard, (addr_bases, block_strides, sizes) in enumerate(templates):
|
||||
head_start = local_shard * self.heads_per_store_shard
|
||||
for offset, size in self._shard_segments(
|
||||
head_start, head_stride, token_stride, content_bytes
|
||||
):
|
||||
addr_bases.append(cache.data_ptr() + offset)
|
||||
block_strides.append(block_stride)
|
||||
sizes.append(size)
|
||||
|
||||
self._set_segment_templates(templates)
|
||||
|
||||
def prepare_values(
|
||||
self,
|
||||
chunks: Sequence[tuple[int, int]],
|
||||
block_ids: list[int],
|
||||
shard_ids: Sequence[StoreShardId],
|
||||
) -> tuple[list[list[int]], list[list[int]], list[int]]:
|
||||
if not chunks:
|
||||
return [], [], []
|
||||
if len(chunks) != len(shard_ids):
|
||||
raise ValueError("Each Store chunk must have one shard ID")
|
||||
|
||||
count = len(chunks)
|
||||
starts = np.fromiter(
|
||||
(chunk[0] for chunk in chunks), dtype=np.int64, count=count
|
||||
)
|
||||
ends = np.fromiter((chunk[1] for chunk in chunks), dtype=np.int64, count=count)
|
||||
if np.any(ends - starts != self.block_size):
|
||||
raise ValueError("TP-shared Mooncake store requires full KV blocks")
|
||||
|
||||
first_shard = self.store_shard_ids[0]
|
||||
local_shards = np.fromiter(
|
||||
(shard_id - first_shard for shard_id in shard_ids),
|
||||
dtype=np.int64,
|
||||
count=count,
|
||||
)
|
||||
if np.any(local_shards < 0) or np.any(local_shards >= self.shards_per_rank):
|
||||
raise ValueError("Store shard is not owned by this TP rank")
|
||||
chunk_block_ids = np.fromiter(
|
||||
(block_ids[index] for index in (starts // self.block_size).tolist()),
|
||||
dtype=np.uint64,
|
||||
count=count,
|
||||
)
|
||||
addr_bases = self._shard_addr_bases[local_shards]
|
||||
block_strides = self._shard_block_strides[local_shards]
|
||||
addrs = addr_bases + chunk_block_ids[:, None] * block_strides
|
||||
sizes = self._shard_sizes[local_shards]
|
||||
return addrs.tolist(), sizes.tolist(), chunk_block_ids.tolist()
|
||||
|
||||
|
||||
class LBHNCStoreLayout(TPShardedStoreLayout):
|
||||
"""Native head-major layout shared by divisible TP sizes."""
|
||||
|
||||
store_format = "tp_shared_lbhnc"
|
||||
|
||||
def _has_packed_strides(
|
||||
self, head_stride: int, token_stride: int, content_bytes: int
|
||||
) -> bool:
|
||||
return (
|
||||
token_stride == content_bytes
|
||||
and head_stride == self.block_size * content_bytes
|
||||
)
|
||||
|
||||
def _shard_segments(
|
||||
self, head_start: int, head_stride: int, token_stride: int, content_bytes: int
|
||||
) -> Iterable[tuple[int, int]]:
|
||||
yield head_start * head_stride, self.heads_per_store_shard * head_stride
|
||||
|
||||
|
||||
class LBNHCStoreLayout(TPShardedStoreLayout):
|
||||
"""Native token-major layout shared by divisible TP sizes."""
|
||||
|
||||
store_format = "tp_shared_lbnhc"
|
||||
|
||||
def _has_packed_strides(
|
||||
self, head_stride: int, token_stride: int, content_bytes: int
|
||||
) -> bool:
|
||||
return (
|
||||
head_stride == content_bytes
|
||||
and token_stride == self.local_num_kv_heads * content_bytes
|
||||
)
|
||||
|
||||
def _shard_segments(
|
||||
self, head_start: int, head_stride: int, token_stride: int, content_bytes: int
|
||||
) -> Iterable[tuple[int, int]]:
|
||||
for token_idx in range(self.block_size):
|
||||
yield (
|
||||
token_idx * token_stride + head_start * head_stride,
|
||||
self.heads_per_store_shard * content_bytes,
|
||||
)
|
||||
|
||||
|
||||
class ChunkedTokenDatabase:
|
||||
"""Enumerates logical token chunks and their hashes."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
metadata: KeyMetadata,
|
||||
block_size: int,
|
||||
hash_block_size: int | None = None,
|
||||
store_layout: StoreLayout | None = None,
|
||||
):
|
||||
self.metadata = metadata
|
||||
self.block_size = block_size
|
||||
self.hash_block_size = hash_block_size or block_size
|
||||
if self.block_size % self.hash_block_size != 0:
|
||||
raise ValueError(
|
||||
f"block_size ({self.block_size}) must be a multiple of "
|
||||
f"hash_block_size ({self.hash_block_size})"
|
||||
)
|
||||
self.store_layout = store_layout or RankLocalStoreLayout(
|
||||
metadata, block_size, self.hash_block_size
|
||||
)
|
||||
|
||||
@property
|
||||
def kv_caches_base_addr(self) -> list[int]:
|
||||
return self._rank_local_layout().kv_caches_base_addr
|
||||
|
||||
@property
|
||||
def block_len(self) -> list[int]:
|
||||
return self._rank_local_layout().block_len
|
||||
|
||||
def _rank_local_layout(self) -> RankLocalStoreLayout:
|
||||
if not isinstance(self.store_layout, RankLocalStoreLayout):
|
||||
raise RuntimeError("This operation requires a rank-local Store layout")
|
||||
return self.store_layout
|
||||
|
||||
def key_for(self, chunk_hash: BlockHash) -> str:
|
||||
return self._rank_local_layout().key_for(0, chunk_hash)
|
||||
|
||||
def set_kv_caches_base_addr(self, kv_caches_base_addr: list[int]):
|
||||
self._rank_local_layout().set_kv_caches_base_addr(kv_caches_base_addr)
|
||||
|
||||
def set_block_len(self, block_len: list[int]):
|
||||
self._rank_local_layout().set_block_len(block_len)
|
||||
|
||||
def prepare_value_for_block(self, block_id: int) -> tuple[list[int], list[int]]:
|
||||
"""Return addresses and sizes for one physical block slot."""
|
||||
return self._rank_local_layout().prepare_value_for_block(block_id)
|
||||
|
||||
def process_tokens(
|
||||
self,
|
||||
token_len: int,
|
||||
block_hashes: list[BlockHash],
|
||||
mask_num: int = 0,
|
||||
*,
|
||||
chunk_mask: list[bool] | None = None,
|
||||
put_step: int = 1,
|
||||
put_step_rank: int = 0,
|
||||
) -> Iterable[tuple[int, int, BlockHash]]:
|
||||
"""Process tokens and yield (start_idx, end_idx, block_hash) tuples.
|
||||
|
||||
When there are fewer KV heads than TP ranks, chunks are distributed
|
||||
across TP ranks to avoid duplicate load/store. The assignment keys off
|
||||
the absolute ``chunk_id`` so a given chunk always lands on the same
|
||||
rank regardless of where the processed suffix begins.
|
||||
|
||||
Args:
|
||||
token_len: Total number of tokens. Must be hash-block aligned and
|
||||
covered by ``block_hashes`` when hashes are present.
|
||||
block_hashes: Block hashes computed at ``hash_block_size`` granularity.
|
||||
When ``block_size > hash_block_size`` each group's ``block_size`` chunk
|
||||
is keyed by its last sub-hash via ``chunk_hashes_for_block_size``.
|
||||
mask_num: Number of tokens to skip from the beginning.
|
||||
chunk_mask: Optional mask relative to the first chunk after
|
||||
``mask_num``. False entries are skipped before hash access.
|
||||
put_step: Stride for distributing chunks across ranks.
|
||||
put_step_rank: ``chunk_id % put_step`` value this rank stores.
|
||||
|
||||
"""
|
||||
assert put_step > 0
|
||||
if not block_hashes:
|
||||
return
|
||||
assert token_len % self.hash_block_size == 0
|
||||
assert token_len // self.hash_block_size <= len(block_hashes)
|
||||
start_chunk = max(0, cdiv(mask_num, self.block_size))
|
||||
max_chunks = cdiv(token_len, self.block_size)
|
||||
if chunk_mask is not None:
|
||||
max_chunks = min(max_chunks, start_chunk + len(chunk_mask))
|
||||
for chunk_id in range(start_chunk, max_chunks):
|
||||
if chunk_mask is not None and not chunk_mask[chunk_id - start_chunk]:
|
||||
continue
|
||||
if chunk_id % put_step != put_step_rank:
|
||||
continue
|
||||
start_idx = chunk_id * self.block_size
|
||||
end_idx = min(start_idx + self.block_size, token_len)
|
||||
h = block_hashes[end_idx // self.hash_block_size - 1]
|
||||
yield start_idx, end_idx, h
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TailKeyBoundary:
|
||||
"""Hash boundary used to key a group's tail block in the store.
|
||||
|
||||
Attributes:
|
||||
group_id: KV-cache group containing the tail block.
|
||||
num_tokens: Token boundary whose prefix hash identifies the matched
|
||||
stored block. The loader uses
|
||||
``block_hashes[num_tokens // hash_block_size - 1]`` instead of the
|
||||
hash implied by ``MooncakeLookupResult.hit_length``. This changes
|
||||
only the load key, not the reusable prefix.
|
||||
|
||||
"""
|
||||
|
||||
group_id: int
|
||||
num_tokens: int
|
||||
|
||||
|
||||
@dataclass
|
||||
class MooncakeLookupResult:
|
||||
"""Lookup result used to build the subsequent load request.
|
||||
|
||||
Attributes:
|
||||
hit_length: Longest prefix that every KV-cache group can reuse after
|
||||
their individual cache hits converge.
|
||||
tail_key_boundaries: Hash boundary used to store each cache group's
|
||||
tail block when ``hit_length`` does not identify its store key.
|
||||
There is one entry per group for every nonzero hit.
|
||||
|
||||
"""
|
||||
|
||||
hit_length: int
|
||||
tail_key_boundaries: tuple[TailKeyBoundary, ...] = ()
|
||||
|
||||
|
||||
@dataclass
|
||||
class LoadSpec:
|
||||
"""Specification for loading KV cache from external store."""
|
||||
|
||||
vllm_cached_tokens: int
|
||||
kvpool_cached_tokens: int
|
||||
can_load: bool
|
||||
token_len: int = 0
|
||||
tail_key_boundaries: tuple[TailKeyBoundary, ...] = ()
|
||||
|
||||
|
||||
@dataclass
|
||||
class RequestTracker:
|
||||
"""Tracks per-request state across scheduler ticks."""
|
||||
|
||||
req_id: str
|
||||
token_len: int
|
||||
allocated_block_ids: tuple[list[int], ...]
|
||||
num_saved_tokens: int = 0
|
||||
token_ids: list[int] | None = None
|
||||
has_pending_offload: bool = False
|
||||
# Snapshot of the prefill range length at tracker creation time.
|
||||
# For a fresh request this is len(prompt). For a resumed-from-preemption
|
||||
# request it includes previously-generated tokens, which are re-prefilled.
|
||||
prefill_end_tokens: int = 0
|
||||
partial_tail_sent: bool = False
|
||||
|
||||
def reset(self) -> None:
|
||||
self.token_len = 0
|
||||
self.allocated_block_ids = ()
|
||||
self.num_saved_tokens = 0
|
||||
self.token_ids = None
|
||||
self.has_pending_offload = False
|
||||
self.prefill_end_tokens = 0
|
||||
self.partial_tail_sent = False
|
||||
|
||||
def update(
|
||||
self,
|
||||
new_block_ids: tuple[list[int], ...] | list[int],
|
||||
) -> None:
|
||||
# Backward-compat: accept a single list (broadcast to single group).
|
||||
if isinstance(new_block_ids, list):
|
||||
new_block_ids = (new_block_ids,)
|
||||
if len(new_block_ids) != len(self.allocated_block_ids):
|
||||
raise ValueError(
|
||||
f"Group count mismatch: tracker has "
|
||||
f"{len(self.allocated_block_ids)} groups, update has "
|
||||
f"{len(new_block_ids)}"
|
||||
)
|
||||
for existing, new in zip(self.allocated_block_ids, new_block_ids, strict=True):
|
||||
if new:
|
||||
existing.extend(new)
|
||||
|
||||
|
||||
class BoundaryPut(NamedTuple):
|
||||
"""A block stored under the hash of the prefix ending at ``num_tokens``."""
|
||||
|
||||
group_id: int
|
||||
block_id: int
|
||||
num_tokens: int
|
||||
|
||||
|
||||
@dataclass
|
||||
class ReqMeta:
|
||||
"""Per-request metadata for store put/get operations."""
|
||||
|
||||
req_id: str
|
||||
token_len_chunk: int
|
||||
block_ids: tuple[list[int], ...]
|
||||
block_hashes: list[BlockHash]
|
||||
|
||||
can_save: bool | None = None
|
||||
load_spec: LoadSpec | None = None
|
||||
current_event: torch.cuda.Event | None = None
|
||||
|
||||
token_ids: list[int] | None = None
|
||||
# Absolute request offset represented by token_ids[0].
|
||||
token_ids_start: int = 0
|
||||
num_prompt_tokens: int | None = None
|
||||
# Total prefill tokens, including generated tokens replayed after preemption.
|
||||
prefill_end_tokens: int | None = None
|
||||
# Identifies this store job for the engine's lifetime. A request id cannot
|
||||
# serve that purpose: it is reused once a preempted request resumes, so it
|
||||
# would release the wrong job's blocks.
|
||||
store_job_id: int | None = None
|
||||
# Blocks stored under an explicit prefix hash rather than by position:
|
||||
# core-provided mamba "align" states (a block-aligned entry is a committed
|
||||
# snapshot, a non-aligned one the sub-block CoW prompt tail) and the
|
||||
# scheduler-resolved non-Mamba blocks of the prompt's partial tail. The
|
||||
# store-job reference keeps each block alive until every worker rank
|
||||
# finishes the job.
|
||||
boundary_puts: list[BoundaryPut] | None = None
|
||||
# Total computed prefix length at the end of this step.
|
||||
completed_token_len: int | None = None
|
||||
# Set on the save that first covers the whole prompt, so the scheduler
|
||||
# publishes the prompt's partial tail once rather than on every save.
|
||||
publish_partial_tail: bool = False
|
||||
|
||||
@staticmethod
|
||||
def from_request_tracker(
|
||||
tracker: RequestTracker,
|
||||
block_size: int,
|
||||
load_spec: LoadSpec | None = None,
|
||||
skip_save: bool | None = False,
|
||||
block_hashes: list[BlockHash] | None = None,
|
||||
save_partial_tail: bool = False,
|
||||
num_prompt_tokens: int | None = None,
|
||||
) -> "ReqMeta | None":
|
||||
"""Create ReqMeta from a RequestTracker."""
|
||||
if save_partial_tail:
|
||||
assert num_prompt_tokens is not None
|
||||
if block_hashes is None:
|
||||
block_hashes = []
|
||||
input_token_len = tracker.token_len
|
||||
|
||||
token_ids_start = tracker.num_saved_tokens
|
||||
chunk_boundary = cdiv(token_ids_start + 1, block_size) * block_size
|
||||
num_tokens_to_save = input_token_len // block_size * block_size
|
||||
|
||||
publish_tail = (
|
||||
save_partial_tail
|
||||
and not tracker.partial_tail_sent
|
||||
and num_prompt_tokens is not None
|
||||
and 0 < num_prompt_tokens <= input_token_len
|
||||
)
|
||||
skip_save = skip_save or (
|
||||
num_tokens_to_save < chunk_boundary and not publish_tail
|
||||
)
|
||||
# A ReqMeta must never carry both a save AND a load.
|
||||
# The save would also be wasted work — the bytes are being looked up
|
||||
# in the store right now. Later cached_reqs steps save new tokens
|
||||
# normally.
|
||||
if load_spec is not None and load_spec.can_load:
|
||||
skip_save = True
|
||||
if skip_save and load_spec is None:
|
||||
return None
|
||||
|
||||
if not skip_save:
|
||||
tracker.num_saved_tokens = num_tokens_to_save
|
||||
tracker.partial_tail_sent |= publish_tail
|
||||
|
||||
token_ids = None
|
||||
if tracker.token_ids and not skip_save:
|
||||
# Scheduler tracking continues while this job is handled by an
|
||||
# asynchronous worker, so metadata must own a stable snapshot.
|
||||
token_ids = tracker.token_ids[token_ids_start:num_tokens_to_save]
|
||||
|
||||
if load_spec is not None and load_spec.can_load:
|
||||
logger.debug(
|
||||
"Scheduled to load %d tokens for request %s",
|
||||
load_spec.kvpool_cached_tokens,
|
||||
tracker.req_id,
|
||||
)
|
||||
else:
|
||||
load_spec = None
|
||||
|
||||
logger.debug(
|
||||
"request:%s, meta save spec:%s, meta load spec:%s",
|
||||
tracker.req_id,
|
||||
not skip_save,
|
||||
load_spec,
|
||||
)
|
||||
return ReqMeta(
|
||||
req_id=tracker.req_id,
|
||||
token_len_chunk=num_tokens_to_save,
|
||||
block_ids=tracker.allocated_block_ids,
|
||||
can_save=not skip_save,
|
||||
load_spec=load_spec,
|
||||
block_hashes=block_hashes,
|
||||
token_ids=token_ids,
|
||||
token_ids_start=token_ids_start,
|
||||
num_prompt_tokens=num_prompt_tokens,
|
||||
prefill_end_tokens=tracker.prefill_end_tokens,
|
||||
completed_token_len=input_token_len,
|
||||
publish_partial_tail=publish_tail and not skip_save,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class MooncakeStoreWorkerMetadata(KVConnectorWorkerMetadata):
|
||||
"""Maps ``ReqMeta.store_job_id`` to the number of ranks done with that job."""
|
||||
|
||||
completed_saves: dict[int, int] = field(default_factory=dict)
|
||||
|
||||
def aggregate(
|
||||
self, other: "KVConnectorWorkerMetadata"
|
||||
) -> "MooncakeStoreWorkerMetadata":
|
||||
assert isinstance(other, MooncakeStoreWorkerMetadata)
|
||||
for store_job_id, count in other.completed_saves.items():
|
||||
self.completed_saves[store_job_id] = (
|
||||
self.completed_saves.get(store_job_id, 0) + count
|
||||
)
|
||||
return self
|
||||
|
||||
|
||||
class MooncakeStoreConnectorMetadata(KVConnectorMetadata):
|
||||
"""Metadata passed from scheduler to worker."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
unfinished_request_ids: set[str],
|
||||
preempted_req_ids: set[str],
|
||||
):
|
||||
self.requests: list[ReqMeta] = []
|
||||
self.unfinished_request_ids = unfinished_request_ids
|
||||
self.preempted_req_ids = preempted_req_ids
|
||||
|
||||
def add_request(self, req_meta: ReqMeta) -> None:
|
||||
self.requests.append(req_meta)
|
||||
@@ -1,189 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Per-operation telemetry for MooncakeStoreConnector.
|
||||
|
||||
Records one row per Mooncake RPC (``save_exists``, ``save_put``, ``load_get``,
|
||||
``lookup_exists``) with duration, key/byte counts, status, and failed-key
|
||||
count. Exposed to the logger via ``KVConnectorLogging`` and to Prometheus
|
||||
via ``MooncakeStorePromMetrics``.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from statistics import fmean
|
||||
from typing import Any
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
|
||||
KVConnectorPromMetrics,
|
||||
KVConnectorStats,
|
||||
PromMetric,
|
||||
PromMetricT,
|
||||
)
|
||||
|
||||
|
||||
def _nearest_rank_percentile(values: list[float], percentile: float) -> float:
|
||||
if not values:
|
||||
return 0.0
|
||||
sorted_values = sorted(values)
|
||||
rank = max(
|
||||
0, min(len(sorted_values) - 1, int(percentile * len(sorted_values) - 1e-12))
|
||||
)
|
||||
return sorted_values[rank]
|
||||
|
||||
|
||||
@dataclass
|
||||
class MooncakeStoreConnectorStats(KVConnectorStats):
|
||||
"""Serializable Mooncake store communication telemetry."""
|
||||
|
||||
def __post_init__(self):
|
||||
if not self.data:
|
||||
self.reset()
|
||||
|
||||
def reset(self):
|
||||
self.data: dict[str, list[dict[str, int | float | str]]] = {}
|
||||
|
||||
def is_empty(self) -> bool:
|
||||
return not self.data
|
||||
|
||||
def aggregate(self, other: KVConnectorStats) -> KVConnectorStats:
|
||||
if other.is_empty():
|
||||
return self
|
||||
for operation, records in other.data.items():
|
||||
self.data.setdefault(operation, []).extend(records)
|
||||
return self
|
||||
|
||||
def reduce(self) -> dict[str, int | float]:
|
||||
reduced: dict[str, int | float] = {}
|
||||
for operation, records in sorted(self.data.items()):
|
||||
if not records:
|
||||
continue
|
||||
durations = [float(record["duration_seconds"]) for record in records]
|
||||
reduced[f"{operation}_count"] = len(records)
|
||||
reduced[f"{operation}_avg_ms"] = round(fmean(durations) * 1e3, 3)
|
||||
reduced[f"{operation}_p90_ms"] = round(
|
||||
_nearest_rank_percentile(durations, 0.9) * 1e3, 3
|
||||
)
|
||||
reduced[f"{operation}_total_keys"] = sum(
|
||||
int(record["num_keys"]) for record in records
|
||||
)
|
||||
reduced[f"{operation}_total_bytes"] = sum(
|
||||
int(record["num_bytes"]) for record in records
|
||||
)
|
||||
reduced[f"{operation}_failed_keys"] = sum(
|
||||
int(record["num_failed_keys"]) for record in records
|
||||
)
|
||||
reduced[f"{operation}_error_count"] = sum(
|
||||
1 for record in records if record["status"] == "error"
|
||||
)
|
||||
return reduced
|
||||
|
||||
def record_operation(
|
||||
self,
|
||||
operation: str,
|
||||
duration_seconds: float,
|
||||
num_keys: int,
|
||||
*,
|
||||
num_bytes: int = 0,
|
||||
status: str = "ok",
|
||||
num_failed_keys: int = 0,
|
||||
) -> None:
|
||||
self.data.setdefault(operation, []).append(
|
||||
{
|
||||
"duration_seconds": duration_seconds,
|
||||
"num_keys": num_keys,
|
||||
"num_bytes": num_bytes,
|
||||
"status": status,
|
||||
"num_failed_keys": num_failed_keys,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class MooncakeStorePromMetrics(KVConnectorPromMetrics):
|
||||
"""Prometheus metrics for Mooncake store communication."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
metric_types: dict[type[PromMetric], type[PromMetricT]],
|
||||
labelnames: list[str],
|
||||
per_engine_labelvalues: dict[int, list[object]],
|
||||
):
|
||||
super().__init__(vllm_config, metric_types, labelnames, per_engine_labelvalues)
|
||||
metric_labelnames = labelnames + ["operation", "status"]
|
||||
self._metric_cache: dict[tuple[int, str, str], dict[str, PromMetric]] = {}
|
||||
|
||||
self._histogram_operation_time = self._histogram_cls(
|
||||
name="vllm:mooncake_store_operation_time_seconds",
|
||||
documentation="Histogram of Mooncake store communication time.",
|
||||
buckets=[
|
||||
1e-3,
|
||||
5e-3,
|
||||
1e-2,
|
||||
5e-2,
|
||||
1e-1,
|
||||
2e-1,
|
||||
3e-1,
|
||||
4e-1,
|
||||
5e-1,
|
||||
7.5e-1,
|
||||
1.0,
|
||||
1.5,
|
||||
2.0,
|
||||
3.0,
|
||||
4.0,
|
||||
],
|
||||
labelnames=metric_labelnames,
|
||||
)
|
||||
self._counter_operation_calls = self._counter_cls(
|
||||
name="vllm:mooncake_store_operation_total",
|
||||
documentation="Number of Mooncake store communication operations.",
|
||||
labelnames=metric_labelnames,
|
||||
)
|
||||
self._counter_operation_keys = self._counter_cls(
|
||||
name="vllm:mooncake_store_operation_keys_total",
|
||||
documentation="Number of Mooncake store keys touched by operations.",
|
||||
labelnames=metric_labelnames,
|
||||
)
|
||||
self._counter_operation_bytes = self._counter_cls(
|
||||
name="vllm:mooncake_store_operation_bytes_total",
|
||||
documentation="Number of bytes transferred by Mooncake store operations.",
|
||||
labelnames=metric_labelnames,
|
||||
)
|
||||
self._counter_failed_keys = self._counter_cls(
|
||||
name="vllm:mooncake_store_operation_failed_keys_total",
|
||||
documentation="Number of Mooncake store keys that failed in operations.",
|
||||
labelnames=metric_labelnames,
|
||||
)
|
||||
|
||||
def _get_metrics(
|
||||
self,
|
||||
engine_idx: int,
|
||||
operation: str,
|
||||
status: str,
|
||||
) -> dict[str, PromMetric]:
|
||||
cache_key = (engine_idx, operation, status)
|
||||
if cache_key not in self._metric_cache:
|
||||
label_values = self.per_engine_labelvalues[engine_idx] + [operation, status]
|
||||
self._metric_cache[cache_key] = {
|
||||
"time": self._histogram_operation_time.labels(*label_values),
|
||||
"calls": self._counter_operation_calls.labels(*label_values),
|
||||
"keys": self._counter_operation_keys.labels(*label_values),
|
||||
"bytes": self._counter_operation_bytes.labels(*label_values),
|
||||
"failed_keys": self._counter_failed_keys.labels(*label_values),
|
||||
}
|
||||
return self._metric_cache[cache_key]
|
||||
|
||||
def observe(self, transfer_stats_data: dict[str, Any] | None, engine_idx: int = 0):
|
||||
if not transfer_stats_data:
|
||||
return
|
||||
for operation, records in transfer_stats_data.items():
|
||||
assert isinstance(records, list)
|
||||
for record in records:
|
||||
assert isinstance(record, dict)
|
||||
status = str(record["status"])
|
||||
metrics = self._get_metrics(engine_idx, operation, status)
|
||||
metrics["time"].observe(float(record["duration_seconds"]))
|
||||
metrics["calls"].inc()
|
||||
metrics["keys"].inc(int(record["num_keys"]))
|
||||
metrics["bytes"].inc(int(record["num_bytes"]))
|
||||
metrics["failed_keys"].inc(int(record["num_failed_keys"]))
|
||||
@@ -1,81 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Wire-format constants for the LookupKey ZMQ admin channel.
|
||||
|
||||
This is the single source of truth shared by ``LookupKeyClient`` and
|
||||
``LookupKeyServer`` on the scheduler<->worker rank-0 admin channel.
|
||||
|
||||
Wire format (REQ/REP over IPC):
|
||||
|
||||
Request: [msg_type: bytes] [payload_frames...]
|
||||
|
||||
msg_type == LOOKUP_MSG:
|
||||
frame 1: num_tokens (u32 big-endian, 4 bytes); the worker derives
|
||||
the aligned lookup length
|
||||
frame 2: hash_len (u16 big-endian, 2 bytes) — byte length of each
|
||||
fixed-size block hash (0 when there are no hashes)
|
||||
frame 3: raw block hashes concatenated back-to-back (each hash_len
|
||||
bytes); the server splits on hash_len
|
||||
Response: hit_length (u32 big-endian, first 4 bytes), followed by zero
|
||||
or more 8-byte tail-key boundaries. Each entry is group_id
|
||||
(u32), then boundary_tokens (u32).
|
||||
|
||||
msg_type == RESET_MSG:
|
||||
(no payload frames)
|
||||
Response: [RESP_OK] or [RESP_ERR]
|
||||
|
||||
The first frame of every request is a named bytes tag (not a numeric
|
||||
sentinel that aliases the data field) so the protocol stays
|
||||
self-describing and extensible: adding new admin commands requires
|
||||
only a new tag and a new dispatch branch.
|
||||
|
||||
Mirrors the named-tag convention used by the NIXL connector (see
|
||||
``vllm/distributed/kv_transfer/kv_connector/v1/nixl/metadata.py``).
|
||||
"""
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
|
||||
MooncakeLookupResult,
|
||||
TailKeyBoundary,
|
||||
)
|
||||
|
||||
# Request message-type tags. Frame 0 of every request.
|
||||
LOOKUP_MSG: bytes = b"lookup"
|
||||
RESET_MSG: bytes = b"reset"
|
||||
|
||||
# Single-byte response status codes for admin commands.
|
||||
RESP_OK: bytes = b"\x01"
|
||||
RESP_ERR: bytes = b"\x00"
|
||||
|
||||
# group_id (u32), boundary_tokens (u32).
|
||||
TAIL_KEY_BOUNDARY_ENTRY_SIZE: int = 8
|
||||
|
||||
|
||||
def encode_lookup_response(result: MooncakeLookupResult) -> bytes:
|
||||
hit_length = result.hit_length.to_bytes(4, "big")
|
||||
if not result.tail_key_boundaries:
|
||||
return hit_length
|
||||
|
||||
payload = bytearray(hit_length)
|
||||
for boundary in result.tail_key_boundaries:
|
||||
payload.extend(boundary.group_id.to_bytes(4, "big"))
|
||||
payload.extend(boundary.num_tokens.to_bytes(4, "big"))
|
||||
return bytes(payload)
|
||||
|
||||
|
||||
def decode_lookup_response(payload: bytes) -> MooncakeLookupResult:
|
||||
if len(payload) < 4 or (len(payload) - 4) % TAIL_KEY_BOUNDARY_ENTRY_SIZE:
|
||||
raise ValueError("Invalid Mooncake lookup response")
|
||||
|
||||
hit_length = int.from_bytes(payload[:4], "big")
|
||||
if len(payload) == 4:
|
||||
return MooncakeLookupResult(hit_length)
|
||||
|
||||
boundaries = []
|
||||
for offset in range(4, len(payload), TAIL_KEY_BOUNDARY_ENTRY_SIZE):
|
||||
boundaries.append(
|
||||
TailKeyBoundary(
|
||||
group_id=int.from_bytes(payload[offset : offset + 4], "big"),
|
||||
num_tokens=int.from_bytes(payload[offset + 4 : offset + 8], "big"),
|
||||
)
|
||||
)
|
||||
return MooncakeLookupResult(hit_length, tuple(boundaries))
|
||||
@@ -1,788 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
#
|
||||
# Adapted from vllm-project/vllm-ascend
|
||||
# (vllm_ascend/distributed/kv_transfer/kv_pool/ascend_store/).
|
||||
"""Scheduler-side logic for MooncakeStoreConnector."""
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorMetadata,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.coordinator import (
|
||||
MooncakeStoreCoordinator,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import ( # noqa: E501
|
||||
BoundaryPut,
|
||||
LoadSpec,
|
||||
MooncakeStoreConnectorMetadata,
|
||||
MooncakeStoreWorkerMetadata,
|
||||
ReqMeta,
|
||||
RequestTracker,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.worker import ( # noqa: E501
|
||||
LookupKeyClient,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
|
||||
from vllm.v1.core.block_pool import BlockPool
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
|
||||
from vllm.v1.core.kv_cache_utils import (
|
||||
resolve_kv_cache_block_sizes,
|
||||
)
|
||||
from vllm.v1.core.sched.output import NewRequestData, SchedulerOutput
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
KVCacheConfig,
|
||||
MambaSpec,
|
||||
get_mamba_prefill_checkpoint_position,
|
||||
)
|
||||
from vllm.v1.outputs import KVConnectorOutput
|
||||
from vllm.v1.request import Request
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _new_req_prefill_tokens(request: NewRequestData) -> list[int]:
|
||||
"""Tokens this prefill will compute KV for.
|
||||
|
||||
Under the v2 model runner, resumed-from-preemption requests appear in
|
||||
``scheduled_new_reqs`` with ``prefill_token_ids`` set to the request's full
|
||||
token list (prompt + previously-generated). For all other cases this falls
|
||||
back to the original prompt.
|
||||
"""
|
||||
if request.prefill_token_ids is not None:
|
||||
return request.prefill_token_ids
|
||||
assert request.prompt_token_ids is not None
|
||||
return request.prompt_token_ids
|
||||
|
||||
|
||||
def _partial_tail_non_mamba_puts(
|
||||
coord: MooncakeStoreCoordinator,
|
||||
req_meta: ReqMeta,
|
||||
block_sizes: Sequence[int],
|
||||
) -> list[BoundaryPut]:
|
||||
"""Locate the non-Mamba blocks a partial-tail save publishes.
|
||||
|
||||
A later request resumes at the prompt's Mamba checkpoint (``boundary``) only
|
||||
if both pieces of KV are stored:
|
||||
|
||||
- Mamba groups: the state at ``boundary``, handed off by the core as a
|
||||
boundary put of its own, so Mamba groups are not listed here.
|
||||
- Other groups: the KV from the last LCM-aligned normal save up to
|
||||
``boundary`` plus the group's EAGLE proof margin (``proof_end``). Each
|
||||
block is keyed by the hash at its end, clipped to ``proof_end``.
|
||||
|
||||
Groups whose proof is not computed yet are omitted. For example, with LCM
|
||||
16, blocks of 4 tokens, and a checkpoint at 44, the full-attention blocks
|
||||
are 8-10, keyed at 36, 40 and 44.
|
||||
|
||||
A Mamba state handed off at any other position, i.e. a shared-prefix
|
||||
junction (``--enable-mamba-shared-prefix-checkpoint``), adds no puts here.
|
||||
|
||||
``block_sizes`` are the per-group block sizes, indexed like
|
||||
``req_meta.block_ids``.
|
||||
"""
|
||||
mamba_tails = [
|
||||
put.num_tokens
|
||||
for put in req_meta.boundary_puts or []
|
||||
if put.group_id in coord.mamba_group_ids
|
||||
and put.num_tokens % block_sizes[put.group_id]
|
||||
]
|
||||
# The tail is due on the save that first covers the prompt, or when Mamba
|
||||
# hands off its checkpoint state.
|
||||
if not mamba_tails and not req_meta.publish_partial_tail:
|
||||
return []
|
||||
prompt_tokens = req_meta.num_prompt_tokens
|
||||
completed = req_meta.completed_token_len
|
||||
assert prompt_tokens is not None and completed is not None
|
||||
hash_block_size = coord.hash_block_size
|
||||
# Core drops the Mamba checkpoint whenever EAGLE block drop is on, whichever
|
||||
# groups carry the eagle flag.
|
||||
boundary = get_mamba_prefill_checkpoint_position(
|
||||
prompt_tokens, hash_block_size, coord.use_eagle
|
||||
)
|
||||
# Any other sub-block Mamba hand-off is a shared-prefix junction. Its
|
||||
# attention KV, EAGLE proof included, ends on an attention block end no later
|
||||
# than this checkpoint's proof, so the normal and prompt-tail saves store it.
|
||||
if not req_meta.publish_partial_tail and boundary not in mamba_tails:
|
||||
return []
|
||||
num_hashes = len(req_meta.block_hashes)
|
||||
if boundary == 0 or boundary // hash_block_size > num_hashes:
|
||||
return []
|
||||
start = boundary // coord.lcm_block_size * coord.lcm_block_size
|
||||
puts: list[BoundaryPut] = []
|
||||
for group_id, block_size in enumerate(block_sizes):
|
||||
if group_id in coord.mamba_group_ids:
|
||||
continue
|
||||
proof_end = boundary + coord.eagle_proof_margin_by_group.get(group_id, 0)
|
||||
if proof_end > completed or proof_end // hash_block_size > num_hashes:
|
||||
continue
|
||||
group_blocks = req_meta.block_ids[group_id]
|
||||
for block_idx in range(start // block_size, cdiv(proof_end, block_size)):
|
||||
if block_idx >= len(group_blocks):
|
||||
break
|
||||
block_id = group_blocks[block_idx]
|
||||
if block_id == NULL_BLOCK_ID:
|
||||
continue
|
||||
num_tokens = min((block_idx + 1) * block_size, proof_end)
|
||||
puts.append(BoundaryPut(group_id, block_id, num_tokens))
|
||||
return puts
|
||||
|
||||
|
||||
class MooncakeStoreScheduler:
|
||||
"""Scheduler-side component for MooncakeStoreConnector."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
kv_cache_config: KVCacheConfig,
|
||||
):
|
||||
assert vllm_config.kv_transfer_config is not None
|
||||
self.kv_role = vllm_config.kv_transfer_config.kv_role
|
||||
kvc_extra_config = vllm_config.kv_transfer_config.kv_connector_extra_config
|
||||
self.load_async = kvc_extra_config.get("load_async", True)
|
||||
self.lookup_async = kvc_extra_config.get("lookup_async", False)
|
||||
# Skips lookup CPU cost on instances that never load KV from the store.
|
||||
self.enable_lookup = kvc_extra_config.get("enable_lookup", True)
|
||||
self.save_decode_cache = kvc_extra_config.get("save_decode_cache", False)
|
||||
kv_event_config = vllm_config.kv_events_config
|
||||
self.enable_kv_events = bool(
|
||||
kv_event_config and kv_event_config.enable_kv_cache_events
|
||||
)
|
||||
self.client = LookupKeyClient(vllm_config)
|
||||
self.kv_cache_config = kv_cache_config
|
||||
self._store_group_ids = kv_cache_config.prefix_cacheable_group_ids
|
||||
# Map scheduler group IDs to store group indices. Groups outside the
|
||||
# store projection, such as the QSA ring, map to None and are skipped.
|
||||
self._store_group_id_by_kv_cache_group_id = {
|
||||
group_id: store_group_id
|
||||
for store_group_id, group_id in enumerate(self._store_group_ids)
|
||||
}
|
||||
store_groups = kv_cache_config.prefix_cacheable_groups
|
||||
|
||||
# Align with the engine's own scheduler_block_size and hash_block_size.
|
||||
self._block_size, self._hash_block_size = resolve_kv_cache_block_sizes(
|
||||
kv_cache_config, vllm_config
|
||||
)
|
||||
mamba_groups = {
|
||||
group_id: group.kv_cache_spec
|
||||
for group_id, group in enumerate(store_groups)
|
||||
if isinstance(group.kv_cache_spec, MambaSpec)
|
||||
}
|
||||
self._boundary_state_group_ids = frozenset(mamba_groups)
|
||||
|
||||
# Mirrors MooncakeStoreConnector._capacity_only.
|
||||
self._capacity_only = (
|
||||
self.kv_role == "kv_consumer"
|
||||
and not self.enable_lookup
|
||||
and not self.save_decode_cache
|
||||
)
|
||||
self._store_coord: MooncakeStoreCoordinator | None = None
|
||||
self.enable_partial_hash_hits = False
|
||||
if not self._capacity_only:
|
||||
assert all(
|
||||
spec.mamba_cache_mode == "align" for spec in mamba_groups.values()
|
||||
), "MooncakeStoreScheduler requires mamba_cache_mode='align'"
|
||||
self._store_coord = MooncakeStoreCoordinator.from_kv_cache_config(
|
||||
kv_cache_config, vllm_config, self._block_size, self._hash_block_size
|
||||
)
|
||||
self.enable_partial_hash_hits = self._store_coord.enable_partial_hash_hits
|
||||
|
||||
self._gpu_block_pool: BlockPool | None = None
|
||||
self._num_workers = vllm_config.parallel_config.world_size
|
||||
self._next_store_job_id = 0
|
||||
# store_job_id -> (referenced block ids, ranks yet to report completion)
|
||||
self._pinned_saves: dict[int, tuple[list[int], int]] = {}
|
||||
|
||||
# Per-request state
|
||||
self.load_specs: dict[str, LoadSpec] = {} # to be loaded
|
||||
self._request_trackers: dict[str, RequestTracker] = {} # scheduled new requests
|
||||
self._unfinished_requests: dict[str, tuple[Request, tuple[list[int], ...]]] = {}
|
||||
self._unfinished_request_ids: set[str] = set()
|
||||
self._finished_partial_tail_metas: dict[str, ReqMeta] = {}
|
||||
|
||||
def bind_gpu_block_pool(self, gpu_block_pool: BlockPool) -> None:
|
||||
self._gpu_block_pool = gpu_block_pool
|
||||
|
||||
def get_num_new_matched_tokens(
|
||||
self,
|
||||
request: Request,
|
||||
num_computed_tokens: int,
|
||||
) -> tuple[int | None, bool]:
|
||||
"""Check for external KV cache hit.
|
||||
|
||||
Returns ``(None, False)`` when an async lookup is still in flight,
|
||||
signaling the scheduler to retry this request on a later step.
|
||||
"""
|
||||
if not self.enable_lookup:
|
||||
return 0, False
|
||||
|
||||
# Fine-grained hits may land on a hash boundary inside a block; without
|
||||
# partial hits, prefixes shorter than one physical block are skipped.
|
||||
align = (
|
||||
self._hash_block_size if self.enable_partial_hash_hits else self._block_size
|
||||
)
|
||||
if request.num_tokens < align:
|
||||
return 0, False
|
||||
|
||||
lookup_result = self.client.lookup(
|
||||
request.request_id,
|
||||
request.num_tokens,
|
||||
request.block_hashes,
|
||||
non_block=self.lookup_async,
|
||||
)
|
||||
if lookup_result is None:
|
||||
# Lookup not ready yet; scheduler will retry on a later step.
|
||||
return None, False
|
||||
num_external_hit_tokens = lookup_result.hit_length
|
||||
|
||||
if num_external_hit_tokens < num_computed_tokens:
|
||||
need_to_allocate = 0
|
||||
else:
|
||||
need_to_allocate = num_external_hit_tokens - num_computed_tokens
|
||||
|
||||
logger.debug(
|
||||
"Reqid: %s, Total tokens %d, kvpool hit tokens: %d, need to load: %d",
|
||||
request.request_id,
|
||||
request.num_tokens,
|
||||
num_external_hit_tokens,
|
||||
need_to_allocate,
|
||||
)
|
||||
|
||||
if need_to_allocate <= 0:
|
||||
return 0, False
|
||||
|
||||
self.load_specs[request.request_id] = LoadSpec(
|
||||
vllm_cached_tokens=num_computed_tokens,
|
||||
kvpool_cached_tokens=num_external_hit_tokens,
|
||||
can_load=False,
|
||||
tail_key_boundaries=lookup_result.tail_key_boundaries,
|
||||
)
|
||||
|
||||
return need_to_allocate, self.load_async
|
||||
|
||||
def update_state_after_alloc(
|
||||
self,
|
||||
request: Request,
|
||||
blocks: KVCacheBlocks,
|
||||
num_external_tokens: int,
|
||||
):
|
||||
"""Update state after block allocation."""
|
||||
local_block_ids: tuple[list[int], ...] = ()
|
||||
if num_external_tokens > 0:
|
||||
local_block_ids = blocks.get_block_ids(
|
||||
group_ids=self._store_group_ids,
|
||||
)
|
||||
|
||||
self._unfinished_requests[request.request_id] = (request, local_block_ids)
|
||||
self._unfinished_request_ids.add(request.request_id)
|
||||
|
||||
if request.request_id not in self.load_specs:
|
||||
return
|
||||
|
||||
if num_external_tokens == 0:
|
||||
self.load_specs[request.request_id].can_load = False
|
||||
return
|
||||
|
||||
assert (
|
||||
num_external_tokens > 0
|
||||
and num_external_tokens
|
||||
== self.load_specs[request.request_id].kvpool_cached_tokens
|
||||
- self.load_specs[request.request_id].vllm_cached_tokens
|
||||
), (
|
||||
f"Mismatch in number of tokens: {num_external_tokens} vs "
|
||||
f"{self.load_specs[request.request_id].kvpool_cached_tokens} - "
|
||||
f"{self.load_specs[request.request_id].vllm_cached_tokens}"
|
||||
f" for request {request.request_id}"
|
||||
)
|
||||
|
||||
self.load_specs[request.request_id].can_load = True
|
||||
|
||||
def build_connector_meta(
|
||||
self, scheduler_output: SchedulerOutput
|
||||
) -> KVConnectorMetadata:
|
||||
"""Build connector metadata for this scheduler step."""
|
||||
is_consumer = self.kv_role == "kv_consumer"
|
||||
can_process_cached = not is_consumer or self.save_decode_cache
|
||||
|
||||
for finished_req_id in scheduler_output.finished_req_ids:
|
||||
self.client.discard(finished_req_id)
|
||||
self.load_specs.pop(finished_req_id, None)
|
||||
self._request_trackers.pop(finished_req_id, None)
|
||||
self._unfinished_requests.pop(finished_req_id, None)
|
||||
self._unfinished_request_ids.discard(finished_req_id)
|
||||
|
||||
preempted_ids = scheduler_output.preempted_req_ids or set()
|
||||
for req_id in preempted_ids:
|
||||
self.load_specs.pop(req_id, None)
|
||||
if request_tracker := self._request_trackers.get(req_id):
|
||||
request_tracker.reset()
|
||||
self._unfinished_requests.pop(req_id, None)
|
||||
|
||||
meta = MooncakeStoreConnectorMetadata(
|
||||
self._unfinished_request_ids,
|
||||
preempted_ids,
|
||||
)
|
||||
|
||||
# Handle new requests
|
||||
for request in scheduler_output.scheduled_new_reqs:
|
||||
load_spec = self.load_specs.pop(request.req_id, None)
|
||||
num_tokens_to_compute = (
|
||||
request.num_computed_tokens
|
||||
+ scheduler_output.num_scheduled_tokens[request.req_id]
|
||||
)
|
||||
assert request.req_id in self._unfinished_requests
|
||||
request_tuple = self._unfinished_requests.get(request.req_id)
|
||||
request_real = request_tuple[0] # type: ignore[index]
|
||||
|
||||
unfolded_block_ids = tuple(
|
||||
request.block_ids[group_id].copy() for group_id in self._store_group_ids
|
||||
)
|
||||
|
||||
prefill_tokens = _new_req_prefill_tokens(request)
|
||||
request_tracker = RequestTracker(
|
||||
req_id=request.req_id,
|
||||
token_len=num_tokens_to_compute,
|
||||
allocated_block_ids=unfolded_block_ids,
|
||||
num_saved_tokens=0,
|
||||
token_ids=(
|
||||
prefill_tokens[:num_tokens_to_compute]
|
||||
if self.enable_kv_events
|
||||
else None
|
||||
),
|
||||
prefill_end_tokens=len(prefill_tokens),
|
||||
)
|
||||
self._request_trackers[request.req_id] = request_tracker
|
||||
|
||||
req_meta = ReqMeta.from_request_tracker(
|
||||
request_tracker,
|
||||
self._block_size,
|
||||
load_spec=load_spec,
|
||||
# A consumer may write decode KV without becoming a prefill
|
||||
# producer. Loads are still carried by the same metadata.
|
||||
skip_save=is_consumer,
|
||||
block_hashes=request_real.block_hashes,
|
||||
num_prompt_tokens=request_real.num_prompt_tokens,
|
||||
save_partial_tail=self.enable_partial_hash_hits,
|
||||
)
|
||||
if req_meta is not None:
|
||||
meta.add_request(req_meta)
|
||||
|
||||
# Handle cached (running, or MRV1 resumed-from-preemption) requests
|
||||
cached_reqs = scheduler_output.scheduled_cached_reqs
|
||||
if can_process_cached:
|
||||
for i, req_id in enumerate(cached_reqs.req_ids):
|
||||
new_block_ids = cached_reqs.new_block_ids[i]
|
||||
if new_block_ids:
|
||||
new_block_ids = tuple(
|
||||
new_block_ids[group_id] for group_id in self._store_group_ids
|
||||
)
|
||||
|
||||
req_meta = None
|
||||
if req_id in cached_reqs.resumed_req_ids:
|
||||
# Resumed after preemption
|
||||
if not new_block_ids:
|
||||
continue
|
||||
new_block_ids = tuple(b.copy() for b in new_block_ids)
|
||||
load_spec = self.load_specs.pop(req_id, None)
|
||||
request_tuple = self._unfinished_requests.get(req_id)
|
||||
request_real = request_tuple[0] # type: ignore[index]
|
||||
num_tokens_to_compute = (
|
||||
request_real.num_computed_tokens
|
||||
+ scheduler_output.num_scheduled_tokens[req_id]
|
||||
)
|
||||
# On resume, the request re-prefills prompt + previously
|
||||
# generated tokens (all_token_ids).
|
||||
prefill_tokens = list(request_real.all_token_ids)
|
||||
request_tracker = RequestTracker(
|
||||
req_id=req_id,
|
||||
token_len=num_tokens_to_compute,
|
||||
allocated_block_ids=new_block_ids,
|
||||
num_saved_tokens=0,
|
||||
token_ids=(
|
||||
prefill_tokens[:num_tokens_to_compute]
|
||||
if self.enable_kv_events
|
||||
else None
|
||||
),
|
||||
prefill_end_tokens=len(prefill_tokens),
|
||||
)
|
||||
self._request_trackers[req_id] = request_tracker
|
||||
|
||||
req_meta = ReqMeta.from_request_tracker(
|
||||
request_tracker,
|
||||
self._block_size,
|
||||
load_spec=load_spec,
|
||||
skip_save=is_consumer,
|
||||
block_hashes=request_real.block_hashes,
|
||||
num_prompt_tokens=request_real.num_prompt_tokens,
|
||||
save_partial_tail=self.enable_partial_hash_hits,
|
||||
)
|
||||
else:
|
||||
# Decode/chunked request
|
||||
request_tracker = self._request_trackers[req_id]
|
||||
num_computed_token = cached_reqs.num_computed_tokens[i]
|
||||
# Use the tracker's snapshot of the prefill range so resumed
|
||||
# requests keep saving past the original prompt boundary.
|
||||
prefill_end = request_tracker.prefill_end_tokens
|
||||
is_decode = num_computed_token >= prefill_end
|
||||
if is_decode and not self.save_decode_cache:
|
||||
continue
|
||||
|
||||
num_new_tokens = scheduler_output.num_scheduled_tokens[req_id]
|
||||
req_tuple = self._unfinished_requests.get(req_id)
|
||||
if req_tuple:
|
||||
unfinished_req = req_tuple[0]
|
||||
num_current_tokens = request_tracker.token_len
|
||||
new_token_ids = unfinished_req.all_token_ids[
|
||||
num_current_tokens : num_current_tokens + num_new_tokens
|
||||
]
|
||||
request_tracker.token_len += len(new_token_ids)
|
||||
if request_tracker.token_ids is not None:
|
||||
request_tracker.token_ids.extend(new_token_ids)
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Request {req_id} is not in _unfinished_requests"
|
||||
)
|
||||
# A block is usually allocated before the step that fills
|
||||
# it, so reaching a save boundary does not imply that this
|
||||
# step has new block ids.
|
||||
if new_block_ids:
|
||||
request_tracker.update(new_block_ids)
|
||||
if is_consumer and not is_decode:
|
||||
continue
|
||||
|
||||
req_meta = ReqMeta.from_request_tracker(
|
||||
request_tracker,
|
||||
self._block_size,
|
||||
load_spec=None,
|
||||
skip_save=False,
|
||||
block_hashes=unfinished_req.block_hashes,
|
||||
num_prompt_tokens=unfinished_req.num_prompt_tokens,
|
||||
save_partial_tail=self.enable_partial_hash_hits,
|
||||
)
|
||||
|
||||
if req_meta is not None:
|
||||
meta.add_request(req_meta)
|
||||
|
||||
# Handle requests with pending load specs not yet scheduled
|
||||
request_ids = [req.req_id for req in scheduler_output.scheduled_new_reqs]
|
||||
for request_id, (
|
||||
unfinished_req,
|
||||
block_ids,
|
||||
) in self._unfinished_requests.items():
|
||||
if request_id not in request_ids and request_id not in cached_reqs.req_ids:
|
||||
load_spec = self.load_specs.pop(request_id, None)
|
||||
# A load spec may have been proposed by this connector's
|
||||
# lookup but rejected by MultiConnector in favor of another
|
||||
# connector. Only the chosen connector may issue the pending
|
||||
# load; the normal store path gets its blocks later from
|
||||
# SchedulerOutput once the request is actually scheduled.
|
||||
if load_spec is None or not load_spec.can_load:
|
||||
continue
|
||||
num_tokens_to_compute = load_spec.kvpool_cached_tokens
|
||||
request_tracker = RequestTracker(
|
||||
req_id=request_id,
|
||||
token_len=num_tokens_to_compute,
|
||||
allocated_block_ids=block_ids,
|
||||
num_saved_tokens=0,
|
||||
)
|
||||
self._request_trackers[request_id] = request_tracker
|
||||
req_meta = ReqMeta.from_request_tracker(
|
||||
request_tracker,
|
||||
self._block_size,
|
||||
load_spec=load_spec,
|
||||
skip_save=None,
|
||||
block_hashes=unfinished_req.block_hashes,
|
||||
num_prompt_tokens=unfinished_req.num_prompt_tokens,
|
||||
)
|
||||
if req_meta is not None:
|
||||
meta.add_request(req_meta)
|
||||
|
||||
block_state = getattr(scheduler_output, "kv_connector_block_state", None)
|
||||
if (
|
||||
block_state is not None
|
||||
and block_state.boundary_state_offloads
|
||||
and not is_consumer
|
||||
):
|
||||
self._handle_boundary_state_offloads(
|
||||
block_state.boundary_state_offloads, meta
|
||||
)
|
||||
|
||||
self._apply_current_save_block_ids(meta, scheduler_output)
|
||||
|
||||
# Finish-time handoffs arrive after the producing step's metadata was
|
||||
# built. Their exact blocks were pinned when they were registered, so
|
||||
# they remain valid after request cleanup and need no current snapshot.
|
||||
for req_meta in self._finished_partial_tail_metas.values():
|
||||
meta.add_request(req_meta)
|
||||
self._finished_partial_tail_metas.clear()
|
||||
|
||||
if not self._capacity_only:
|
||||
self._reference_save_blocks(meta)
|
||||
return meta
|
||||
|
||||
def _apply_current_save_block_ids(
|
||||
self,
|
||||
meta: MooncakeStoreConnectorMetadata,
|
||||
scheduler_output: SchedulerOutput,
|
||||
) -> None:
|
||||
"""Replace append-only mirrors with the core's current block tables."""
|
||||
save_metas = [req_meta for req_meta in meta.requests if req_meta.can_save]
|
||||
if not save_metas:
|
||||
return
|
||||
|
||||
block_state = scheduler_output.kv_connector_block_state
|
||||
assert block_state is not None, (
|
||||
"Current block tables are required for Mooncake store jobs"
|
||||
)
|
||||
for req_meta in save_metas:
|
||||
block_ids = block_state.get_block_ids(req_meta.req_id)
|
||||
assert block_ids is not None, (
|
||||
f"Missing current block table for store request {req_meta.req_id}"
|
||||
)
|
||||
req_meta.block_ids = tuple(
|
||||
block_ids[group_id] for group_id in self._store_group_ids
|
||||
)
|
||||
|
||||
def _reference_save_blocks(self, meta: MooncakeStoreConnectorMetadata) -> None:
|
||||
"""Take a GPU block reference for every store job this step emits.
|
||||
|
||||
The worker DMAs out of these blocks after the step that scheduled them,
|
||||
so a reference keeps them out of the free queue even once the request
|
||||
itself is freed, until every rank reports the job done.
|
||||
"""
|
||||
pool = self._gpu_block_pool
|
||||
coord = self._store_coord
|
||||
assert coord is not None
|
||||
block_sizes = [g.kv_cache_spec.block_size for g in coord.kv_cache_groups]
|
||||
for req_meta in meta.requests:
|
||||
if not req_meta.can_save:
|
||||
continue
|
||||
assert pool is not None, (
|
||||
"GPU block pool must be bound before any store job is emitted"
|
||||
)
|
||||
if req_meta.store_job_id is not None:
|
||||
assert req_meta.store_job_id in self._pinned_saves
|
||||
continue
|
||||
if tail := _partial_tail_non_mamba_puts(coord, req_meta, block_sizes):
|
||||
req_meta.boundary_puts = [*tail, *(req_meta.boundary_puts or [])]
|
||||
req_meta.store_job_id = store_job_id = self._next_store_job_id
|
||||
self._next_store_job_id += 1
|
||||
block_ids = [put.block_id for put in req_meta.boundary_puts or []]
|
||||
assert NULL_BLOCK_ID not in block_ids, (
|
||||
"A null block cannot back a boundary put"
|
||||
)
|
||||
if req_meta.token_len_chunk:
|
||||
# Normal prefix save: pin all non-Mamba sources for retries.
|
||||
# Every allocated block is referenced, not just the ones covering
|
||||
# this job's token range: a rank resumes from its own last
|
||||
# successful offset, which lags the scheduler's whenever a save was
|
||||
# skipped or failed, so it may read anywhere below the range.
|
||||
block_ids.extend(
|
||||
block_id
|
||||
for group_id, group in enumerate(req_meta.block_ids)
|
||||
if group_id not in self._boundary_state_group_ids
|
||||
for block_id in group
|
||||
if block_id != NULL_BLOCK_ID
|
||||
)
|
||||
# An aligned boundary block may also be present in the request's
|
||||
# block table. Take and release exactly one reference per block.
|
||||
block_ids = list(dict.fromkeys(block_ids))
|
||||
assert NULL_BLOCK_ID not in block_ids
|
||||
if not block_ids:
|
||||
continue
|
||||
self._pinned_saves[store_job_id] = (block_ids, self._num_workers)
|
||||
pool.touch([pool.blocks[block_id] for block_id in block_ids])
|
||||
|
||||
def register_finished_partial_tail(
|
||||
self,
|
||||
request: Request,
|
||||
block_ids: tuple[list[int], ...],
|
||||
partial_tail_offloads: list[tuple[int, int, int]],
|
||||
) -> bool:
|
||||
"""Queue and pin a finish-time tail for the next connector step."""
|
||||
if self.kv_role == "kv_consumer" or not partial_tail_offloads:
|
||||
return False
|
||||
tracker = self._request_trackers.get(request.request_id)
|
||||
if tracker is None or not any(block_ids):
|
||||
return False
|
||||
boundaries = {boundary for _, _, boundary in partial_tail_offloads}
|
||||
if len(boundaries) != 1:
|
||||
raise ValueError(
|
||||
"Partial-tail offloads for one request must share a boundary"
|
||||
)
|
||||
boundary_tokens = next(iter(boundaries))
|
||||
if boundary_tokens > tracker.prefill_end_tokens:
|
||||
return False
|
||||
|
||||
mamba_puts: list[BoundaryPut] = []
|
||||
for group_id, block_id, boundary in partial_tail_offloads:
|
||||
store_group_id = self._store_group_id_by_kv_cache_group_id.get(group_id)
|
||||
if store_group_id not in self._boundary_state_group_ids:
|
||||
return False
|
||||
if block_id == NULL_BLOCK_ID:
|
||||
return False
|
||||
mamba_puts.append(BoundaryPut(store_group_id, block_id, boundary))
|
||||
req_meta = ReqMeta(
|
||||
req_id=request.request_id,
|
||||
token_len_chunk=0,
|
||||
block_ids=tuple(
|
||||
block_ids[group_id].copy() for group_id in self._store_group_ids
|
||||
),
|
||||
block_hashes=list(request.block_hashes),
|
||||
can_save=True,
|
||||
num_prompt_tokens=request.num_prompt_tokens,
|
||||
prefill_end_tokens=tracker.prefill_end_tokens,
|
||||
boundary_puts=mamba_puts,
|
||||
completed_token_len=request.num_computed_tokens,
|
||||
)
|
||||
coord = self._store_coord
|
||||
assert coord is not None
|
||||
req_meta.boundary_puts = [
|
||||
*_partial_tail_non_mamba_puts(
|
||||
coord,
|
||||
req_meta,
|
||||
[g.kv_cache_spec.block_size for g in coord.kv_cache_groups],
|
||||
),
|
||||
*mamba_puts,
|
||||
]
|
||||
pinned_block_ids = list(
|
||||
dict.fromkeys(put.block_id for put in req_meta.boundary_puts)
|
||||
)
|
||||
|
||||
pool = self._gpu_block_pool
|
||||
assert pool is not None, (
|
||||
"GPU block pool must be bound before a finish-time handoff"
|
||||
)
|
||||
assert request.request_id not in self._finished_partial_tail_metas
|
||||
store_job_id = self._next_store_job_id
|
||||
self._next_store_job_id += 1
|
||||
self._pinned_saves[store_job_id] = (pinned_block_ids, self._num_workers)
|
||||
pool.touch([pool.blocks[block_id] for block_id in pinned_block_ids])
|
||||
|
||||
req_meta.store_job_id = store_job_id
|
||||
self._finished_partial_tail_metas[request.request_id] = req_meta
|
||||
tracker.has_pending_offload = True
|
||||
# The store job owns exact block refs, so request cleanup need not wait.
|
||||
return False
|
||||
|
||||
def _handle_boundary_state_offloads(
|
||||
self,
|
||||
offloads: dict[str, list[tuple[int, int, int]]],
|
||||
meta: MooncakeStoreConnectorMetadata,
|
||||
) -> None:
|
||||
"""Attach exact boundary-state blocks to store jobs for this step.
|
||||
|
||||
Flushed in the step they arrive: the CoW copy is enqueued before the
|
||||
connector event records, so this step's event fences the exact block.
|
||||
Entries ride the request's save meta when present, else an
|
||||
offload-only ReqMeta (``token_len_chunk=0`` skips the normal save,
|
||||
``can_save=True`` takes the normal enqueue and store-job pinning path).
|
||||
"""
|
||||
save_metas = {m.req_id: m for m in meta.requests if m.can_save}
|
||||
for req_id, entries in offloads.items():
|
||||
tracker = self._request_trackers.get(req_id)
|
||||
req_tuple = self._unfinished_requests.get(req_id)
|
||||
if tracker is None or req_tuple is None:
|
||||
# Request finished/preempted within this step; its blocks are
|
||||
# going away, so the offload is conservatively dropped.
|
||||
logger.debug("Dropping boundary-state offload for request %s", req_id)
|
||||
continue
|
||||
accepted: list[BoundaryPut] = []
|
||||
for group_id, block_id, boundary_tokens in entries:
|
||||
# Every other group stops saving at the end of this prefill, so
|
||||
# a mamba-only key past it can never complete a joint hybrid
|
||||
# hit. `prefill_end_tokens` — not the original prompt length —
|
||||
# is the boundary: a resumed request re-prefills and re-saves
|
||||
# its previously generated tokens for every group.
|
||||
if boundary_tokens > tracker.prefill_end_tokens:
|
||||
continue
|
||||
if block_id == NULL_BLOCK_ID:
|
||||
continue
|
||||
store_group_id = self._store_group_id_by_kv_cache_group_id.get(group_id)
|
||||
if store_group_id not in self._boundary_state_group_ids:
|
||||
continue
|
||||
accepted.append(BoundaryPut(store_group_id, block_id, boundary_tokens))
|
||||
if not accepted:
|
||||
continue
|
||||
tracker.has_pending_offload = True
|
||||
if (req_meta := save_metas.get(req_id)) is not None:
|
||||
req_meta.boundary_puts = accepted
|
||||
continue
|
||||
meta.add_request(
|
||||
ReqMeta(
|
||||
req_id=req_id,
|
||||
token_len_chunk=0,
|
||||
block_ids=tracker.allocated_block_ids,
|
||||
block_hashes=req_tuple[0].block_hashes,
|
||||
can_save=True,
|
||||
num_prompt_tokens=req_tuple[0].num_prompt_tokens,
|
||||
prefill_end_tokens=tracker.prefill_end_tokens,
|
||||
boundary_puts=accepted,
|
||||
completed_token_len=tracker.token_len,
|
||||
)
|
||||
)
|
||||
|
||||
def update_connector_output(self, connector_output: KVConnectorOutput) -> None:
|
||||
"""Drop the block references of store jobs every rank has finished."""
|
||||
meta = connector_output.kv_connector_worker_meta
|
||||
if not isinstance(meta, MooncakeStoreWorkerMetadata):
|
||||
return
|
||||
pool = self._gpu_block_pool
|
||||
assert pool is not None
|
||||
for store_job_id, count in meta.completed_saves.items():
|
||||
pinned = self._pinned_saves.get(store_job_id)
|
||||
if pinned is None:
|
||||
# The job referenced no blocks, so nothing was recorded for it.
|
||||
continue
|
||||
block_ids, remaining = pinned
|
||||
remaining -= count
|
||||
if remaining > 0:
|
||||
self._pinned_saves[store_job_id] = (block_ids, remaining)
|
||||
continue
|
||||
assert remaining == 0, (
|
||||
f"store job {store_job_id} reported by too many ranks"
|
||||
)
|
||||
del self._pinned_saves[store_job_id]
|
||||
# Tail-first, as elsewhere, so the shared prefix is evicted last.
|
||||
pool.free_blocks(pool.blocks[bid] for bid in reversed(block_ids))
|
||||
|
||||
def has_pending_push_work(self) -> bool:
|
||||
"""Keep the engine stepping while any store job still holds block refs.
|
||||
|
||||
Completions only reach the scheduler as worker metadata on a step, so an
|
||||
engine that quiesced with jobs in flight would leave those references
|
||||
held indefinitely. Nothing else keeps it alive now that a finishing
|
||||
request no longer defers its own free.
|
||||
"""
|
||||
return bool(self._pinned_saves)
|
||||
|
||||
def reset_store(self) -> bool:
|
||||
"""Trigger a global ``remove_all(force=True)`` on the Mooncake master.
|
||||
|
||||
Routes through the existing LookupKey ZMQ admin channel to worker
|
||||
rank 0, which owns the ``MooncakeDistributedStore`` handle.
|
||||
|
||||
Ordering assumption: caller (typically
|
||||
``Scheduler.reset_connector_cache``, invoked via
|
||||
``reset_prefix_cache(reset_connector=True)``) MUST ensure no
|
||||
in-flight Mooncake lookups or transfers. For RL workflows this is
|
||||
satisfied at the step boundary after weight updates and rollout
|
||||
drain. Violating this can allow stale KV to be served on the next
|
||||
request, defeating the hard-reset guarantee.
|
||||
|
||||
Returns True on ACK from worker, False on NACK or RPC error.
|
||||
"""
|
||||
try:
|
||||
ok = self.client.reset()
|
||||
if ok:
|
||||
logger.info("Mooncake store reset via remove_all succeeded.")
|
||||
else:
|
||||
logger.warning("Mooncake store reset returned NACK from worker.")
|
||||
return ok
|
||||
except Exception as e:
|
||||
logger.error("Mooncake reset_store RPC failed: %s", e)
|
||||
return False
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,41 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from .connector import UMBPStoreConnector
|
||||
from .data import (
|
||||
BlockIdentityCodec,
|
||||
BlockTransferPlan,
|
||||
KVLayoutDescriptor,
|
||||
KVLayoutPlanner,
|
||||
KVRange,
|
||||
KVRegion,
|
||||
LoadSpec,
|
||||
RankTopology,
|
||||
RequestTracker,
|
||||
TransferJobState,
|
||||
TransferJobStatus,
|
||||
UMBPConnectorMetadata,
|
||||
UMBPConnectorWorkerMetadata,
|
||||
UMBPNamespace,
|
||||
)
|
||||
from .runtime import UMBPRuntimeConfig, UMBPRuntimeFactory
|
||||
|
||||
__all__ = [
|
||||
"UMBPStoreConnector",
|
||||
"BlockIdentityCodec",
|
||||
"BlockTransferPlan",
|
||||
"KVLayoutPlanner",
|
||||
"KVLayoutDescriptor",
|
||||
"KVRange",
|
||||
"KVRegion",
|
||||
"LoadSpec",
|
||||
"RequestTracker",
|
||||
"RankTopology",
|
||||
"TransferJobState",
|
||||
"TransferJobStatus",
|
||||
"UMBPConnectorMetadata",
|
||||
"UMBPConnectorWorkerMetadata",
|
||||
"UMBPNamespace",
|
||||
"UMBPRuntimeConfig",
|
||||
"UMBPRuntimeFactory",
|
||||
]
|
||||
@@ -1,220 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""vLLM-facing shared UMBP connector."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorBase_V1,
|
||||
KVConnectorMetadata,
|
||||
KVConnectorRole,
|
||||
KVConnectorTransferResults,
|
||||
)
|
||||
from vllm.forward_context import ForwardContext
|
||||
from vllm.v1.attention.backend import AttentionMetadata
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
|
||||
from .data import (
|
||||
BlockIdentityCodec,
|
||||
KVLayoutPlanner,
|
||||
RankTopology,
|
||||
UMBPConnectorMetadata,
|
||||
UMBPConnectorWorkerMetadata,
|
||||
UMBPNamespace,
|
||||
)
|
||||
from .runtime import UMBPRuntimeConfig, UMBPRuntimeFactory
|
||||
from .scheduler import UMBPStoreConnectorScheduler
|
||||
from .worker import UMBPStoreConnectorWorker
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.request import Request
|
||||
|
||||
|
||||
class UMBPStoreConnector(KVConnectorBase_V1):
|
||||
"""vLLM connector lifecycle shared by every UMBP runtime mode."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
role: KVConnectorRole,
|
||||
kv_cache_config: KVCacheConfig,
|
||||
) -> None:
|
||||
super().__init__(vllm_config, role, kv_cache_config)
|
||||
topology = RankTopology.from_vllm_config(vllm_config)
|
||||
runtime_config = UMBPRuntimeConfig.from_vllm(
|
||||
vllm_config
|
||||
).resolve_for_rank_count(topology.rank_count)
|
||||
if topology.pp_size > 1:
|
||||
raise NotImplementedError(
|
||||
f"{runtime_config.mode} UMBP does not support pipeline parallelism"
|
||||
)
|
||||
runtime = UMBPRuntimeFactory.build(runtime_config)
|
||||
namespace = UMBPNamespace.from_vllm_config(vllm_config, kv_cache_config)
|
||||
codec = BlockIdentityCodec(
|
||||
namespace=namespace,
|
||||
tp_rank=topology.tp_rank,
|
||||
pp_rank=topology.pp_rank,
|
||||
pcp_rank=topology.pcp_rank,
|
||||
dcp_rank=topology.dcp_rank,
|
||||
)
|
||||
layout = KVLayoutPlanner.from_kv_cache_config(kv_cache_config)
|
||||
layout_descriptor = layout.describe(topology)
|
||||
self._runtime = runtime
|
||||
self._store_submitted = False
|
||||
self.connector_scheduler: UMBPStoreConnectorScheduler | None = None
|
||||
self.connector_worker: UMBPStoreConnectorWorker | None = None
|
||||
if role == KVConnectorRole.SCHEDULER:
|
||||
self.connector_scheduler = UMBPStoreConnectorScheduler(
|
||||
vllm_config,
|
||||
kv_cache_config,
|
||||
runtime.create_scheduler_handle(
|
||||
namespace.value, topology, layout_descriptor
|
||||
),
|
||||
codec,
|
||||
topology,
|
||||
)
|
||||
else:
|
||||
self.connector_worker = UMBPStoreConnectorWorker(
|
||||
runtime.create_worker_handle(
|
||||
namespace.value, topology, layout_descriptor
|
||||
),
|
||||
layout,
|
||||
codec=codec,
|
||||
)
|
||||
|
||||
def get_num_new_matched_tokens(
|
||||
self, request: Request, num_computed_tokens: int
|
||||
) -> tuple[int | None, bool]:
|
||||
assert self.connector_scheduler is not None
|
||||
return self.connector_scheduler.get_num_new_matched_tokens(
|
||||
request, num_computed_tokens
|
||||
)
|
||||
|
||||
def update_state_after_alloc(
|
||||
self,
|
||||
request: Request,
|
||||
blocks: KVCacheBlocks,
|
||||
num_external_tokens: int,
|
||||
) -> None:
|
||||
assert self.connector_scheduler is not None
|
||||
self.connector_scheduler.update_state_after_alloc(
|
||||
request, blocks, num_external_tokens
|
||||
)
|
||||
|
||||
def bind_gpu_block_pool(self, gpu_block_pool: Any) -> None:
|
||||
assert self.connector_scheduler is not None
|
||||
self.connector_scheduler.bind_gpu_block_pool(gpu_block_pool)
|
||||
|
||||
def build_connector_meta(
|
||||
self, scheduler_output: SchedulerOutput
|
||||
) -> KVConnectorMetadata:
|
||||
assert self.connector_scheduler is not None
|
||||
return self.connector_scheduler.build_connector_meta(scheduler_output)
|
||||
|
||||
def request_finished(
|
||||
self, request: Request, block_ids: list[int]
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
assert self.connector_scheduler is not None
|
||||
return self.connector_scheduler.request_finished(request, (block_ids,))
|
||||
|
||||
def update_connector_output(self, connector_output: Any) -> None:
|
||||
assert self.connector_scheduler is not None
|
||||
self.connector_scheduler.update_connector_output(connector_output)
|
||||
|
||||
def has_pending_push_work(self) -> bool:
|
||||
assert self.connector_scheduler is not None
|
||||
return self.connector_scheduler.has_pending_push_work()
|
||||
|
||||
def reset_cache(self) -> bool | None:
|
||||
if self.connector_scheduler is None:
|
||||
return None
|
||||
return self.connector_scheduler.reset_store()
|
||||
|
||||
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]) -> None:
|
||||
assert self.connector_worker is not None
|
||||
self.connector_worker.register_kv_caches(kv_caches)
|
||||
|
||||
def bind_connector_metadata(self, connector_metadata: KVConnectorMetadata) -> None:
|
||||
super().bind_connector_metadata(connector_metadata)
|
||||
self._store_submitted = False
|
||||
|
||||
def clear_connector_metadata(self) -> None:
|
||||
# No-forward steps skip the model runner's wait_for_save hook.
|
||||
if self.connector_worker is not None and self.has_connector_metadata():
|
||||
self.wait_for_save()
|
||||
super().clear_connector_metadata()
|
||||
|
||||
def handle_preemptions(self, kv_connector_metadata: KVConnectorMetadata) -> None:
|
||||
assert self.connector_worker is not None
|
||||
assert isinstance(kv_connector_metadata, UMBPConnectorMetadata)
|
||||
self.connector_worker.handle_preemptions(kv_connector_metadata)
|
||||
|
||||
def start_load_kv(self, forward_context: ForwardContext, **kwargs: Any) -> None:
|
||||
del kwargs
|
||||
assert self.connector_worker is not None
|
||||
metadata = self._get_connector_metadata()
|
||||
assert isinstance(metadata, UMBPConnectorMetadata)
|
||||
self.connector_worker.start_load_kv(forward_context, metadata)
|
||||
|
||||
def wait_for_layer_load(self, layer_name: str) -> None:
|
||||
assert self.connector_worker is not None
|
||||
self.connector_worker.wait_for_layer_load(layer_name)
|
||||
|
||||
def save_kv_layer(
|
||||
self,
|
||||
layer_name: str,
|
||||
kv_layer: torch.Tensor,
|
||||
attn_metadata: AttentionMetadata,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
# Stores are submitted once per step, from wait_for_save.
|
||||
return
|
||||
|
||||
def wait_for_save(self) -> None:
|
||||
if self._store_submitted:
|
||||
return
|
||||
assert self.connector_worker is not None
|
||||
metadata = self._get_connector_metadata()
|
||||
assert isinstance(metadata, UMBPConnectorMetadata)
|
||||
self.connector_worker.enqueue_stores(metadata)
|
||||
self.connector_worker.wait_for_save()
|
||||
self._store_submitted = True
|
||||
|
||||
def get_finished(
|
||||
self, finished_req_ids: set[str]
|
||||
) -> tuple[set[str] | None, set[str] | None]:
|
||||
assert self.connector_worker is not None
|
||||
return self.connector_worker.get_finished(finished_req_ids)
|
||||
|
||||
def get_transfer_results(
|
||||
self, finished_req_ids: set[str]
|
||||
) -> KVConnectorTransferResults:
|
||||
assert self.connector_worker is not None
|
||||
finished_sending, finished_recving = self.connector_worker.get_finished(
|
||||
finished_req_ids
|
||||
)
|
||||
return KVConnectorTransferResults(
|
||||
finished_sending=set(finished_sending or ()),
|
||||
finished_recving=set(finished_recving or ()),
|
||||
)
|
||||
|
||||
def get_block_ids_with_load_errors(self) -> set[int]:
|
||||
assert self.connector_worker is not None
|
||||
return self.connector_worker.get_block_ids_with_load_errors()
|
||||
|
||||
def build_connector_worker_meta(self) -> UMBPConnectorWorkerMetadata:
|
||||
assert self.connector_worker is not None
|
||||
return self.connector_worker.build_connector_worker_meta()
|
||||
|
||||
def shutdown(self) -> None:
|
||||
if self.connector_scheduler is not None:
|
||||
self.connector_scheduler.close()
|
||||
if self.connector_worker is not None:
|
||||
self.connector_worker.close()
|
||||
@@ -1,620 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Shared data types for the UMBP KV connector.
|
||||
|
||||
The types in this module deliberately do not depend on a UMBP client. They
|
||||
describe vLLM's logical KV objects and the physical ranges a runtime adapter
|
||||
will transfer.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass, field, replace
|
||||
from enum import Enum
|
||||
from typing import Any, NamedTuple
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorMetadata,
|
||||
KVConnectorWorkerMetadata,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RankTopology:
|
||||
"""Model-parallel ranks that participate in one KV object."""
|
||||
|
||||
tp_rank: int = 0
|
||||
tp_size: int = 1
|
||||
pp_rank: int = 0
|
||||
pp_size: int = 1
|
||||
pcp_rank: int = 0
|
||||
pcp_size: int = 1
|
||||
dcp_rank: int = 0
|
||||
dcp_size: int = 1
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
for name in ("tp_size", "pp_size", "pcp_size", "dcp_size"):
|
||||
if getattr(self, name) <= 0:
|
||||
raise ValueError(f"{name} must be positive")
|
||||
for rank, size, name in (
|
||||
(self.tp_rank, self.tp_size, "tp_rank"),
|
||||
(self.pp_rank, self.pp_size, "pp_rank"),
|
||||
(self.pcp_rank, self.pcp_size, "pcp_rank"),
|
||||
(self.dcp_rank, self.dcp_size, "dcp_rank"),
|
||||
):
|
||||
if rank < 0 or rank >= size:
|
||||
raise ValueError(f"{name}={rank} is outside [0, {size})")
|
||||
|
||||
@classmethod
|
||||
def from_vllm_config(cls, vllm_config: Any) -> RankTopology:
|
||||
parallel = vllm_config.parallel_config
|
||||
tp_size = getattr(parallel, "tensor_parallel_size", 1)
|
||||
pp_size = getattr(parallel, "pipeline_parallel_size", 1)
|
||||
pcp_size = getattr(parallel, "prefill_context_parallel_size", 1)
|
||||
# An engine's workers are laid out as [PP, PCP, TP], TP innermost
|
||||
# (initialize_model_parallel).
|
||||
rank = getattr(parallel, "rank", 0)
|
||||
tp_rank = rank % tp_size
|
||||
dcp_size = getattr(parallel, "decode_context_parallel_size", 1)
|
||||
return cls(
|
||||
tp_rank=tp_rank,
|
||||
tp_size=tp_size,
|
||||
pp_rank=rank // (tp_size * pcp_size) % pp_size,
|
||||
pp_size=pp_size,
|
||||
pcp_rank=rank // tp_size % pcp_size,
|
||||
pcp_size=pcp_size,
|
||||
# DCP partitions the TP workers; it does not create a second
|
||||
# Cartesian worker dimension. ParallelConfig is constructed before
|
||||
# process groups, so derive this stable identity from TP rank.
|
||||
dcp_rank=tp_rank % dcp_size,
|
||||
dcp_size=dcp_size,
|
||||
)
|
||||
|
||||
@property
|
||||
def local_namespace(self) -> tuple[int, int, int, int]:
|
||||
return (self.tp_rank, self.pcp_rank, self.dcp_rank, self.pp_rank)
|
||||
|
||||
@property
|
||||
def rank_count(self) -> int:
|
||||
return len(self.all_namespaces())
|
||||
|
||||
def all_namespaces(self) -> tuple[tuple[int, int, int, int], ...]:
|
||||
if self.dcp_size > 1:
|
||||
if self.tp_size % self.dcp_size:
|
||||
raise ValueError("dcp_size must divide tp_size")
|
||||
return tuple(
|
||||
(tp, pcp, tp % self.dcp_size, pp)
|
||||
for pp in range(self.pp_size)
|
||||
for pcp in range(self.pcp_size)
|
||||
for tp in range(self.tp_size)
|
||||
)
|
||||
return tuple(
|
||||
(tp, pcp, dcp, pp)
|
||||
for pp in range(self.pp_size)
|
||||
for dcp in range(self.dcp_size)
|
||||
for pcp in range(self.pcp_size)
|
||||
for tp in range(self.tp_size)
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class UMBPNamespace:
|
||||
"""Compatibility namespace for objects stored outside the engine."""
|
||||
|
||||
value: str
|
||||
|
||||
@classmethod
|
||||
def from_vllm_config(
|
||||
cls,
|
||||
vllm_config: Any,
|
||||
kv_cache_config: KVCacheConfig,
|
||||
) -> UMBPNamespace:
|
||||
extra = vllm_config.kv_transfer_config.kv_connector_extra_config
|
||||
configured = extra.get("key_namespace", "auto")
|
||||
if configured != "auto":
|
||||
if not isinstance(configured, str) or not configured:
|
||||
raise ValueError("key_namespace must be a non-empty string or 'auto'")
|
||||
return cls(configured)
|
||||
|
||||
model_config = vllm_config.model_config
|
||||
parallel_config = vllm_config.parallel_config
|
||||
# Draft-model KV (e.g. DSpark/EAGLE layers) is stored alongside the
|
||||
# target's, so a different drafter must not reuse it.
|
||||
speculative_config = getattr(vllm_config, "speculative_config", None)
|
||||
draft_model_config = getattr(speculative_config, "draft_model_config", None)
|
||||
speculative = (
|
||||
None
|
||||
if speculative_config is None
|
||||
else {
|
||||
"method": getattr(speculative_config, "method", None),
|
||||
"model": getattr(draft_model_config, "model", None),
|
||||
"revision": getattr(draft_model_config, "revision", None),
|
||||
}
|
||||
)
|
||||
fields = {
|
||||
"model": getattr(model_config, "model", ""),
|
||||
"revision": getattr(model_config, "revision", None),
|
||||
"block_size": vllm_config.cache_config.block_size,
|
||||
"cache_layout": getattr(kv_cache_config, "kv_cache_layout", None),
|
||||
"groups": [
|
||||
{
|
||||
"layers": group.layer_names,
|
||||
"block_size": group.kv_cache_spec.block_size,
|
||||
"kind": type(group.kv_cache_spec).__name__,
|
||||
"dtype": str(getattr(group.kv_cache_spec, "dtype", None)),
|
||||
}
|
||||
for group in kv_cache_config.kv_cache_groups
|
||||
],
|
||||
"tp": parallel_config.tensor_parallel_size,
|
||||
"pp": parallel_config.pipeline_parallel_size,
|
||||
"pcp": getattr(parallel_config, "prefill_context_parallel_size", 1),
|
||||
"dcp": getattr(parallel_config, "decode_context_parallel_size", 1),
|
||||
"layout_version": 1,
|
||||
}
|
||||
if speculative is not None:
|
||||
# Added only when set, so namespaces without a drafter are unchanged.
|
||||
fields["speculative"] = speculative
|
||||
payload = json.dumps(fields, sort_keys=True, separators=(",", ":")).encode()
|
||||
return cls(hashlib.sha256(payload).hexdigest()[:32])
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BlockIdentityCodec:
|
||||
"""Build stable rank-local keys from vLLM block hashes."""
|
||||
|
||||
namespace: UMBPNamespace
|
||||
tp_rank: int = 0
|
||||
pp_rank: int = 0
|
||||
pcp_rank: int = 0
|
||||
dcp_rank: int = 0
|
||||
|
||||
def key(self, block_hash: bytes, group_id: int = 0) -> str:
|
||||
return self.key_for_namespace(
|
||||
block_hash,
|
||||
group_id,
|
||||
(self.tp_rank, self.pcp_rank, self.dcp_rank, self.pp_rank),
|
||||
)
|
||||
|
||||
def key_for_namespace(
|
||||
self,
|
||||
block_hash: bytes,
|
||||
group_id: int,
|
||||
namespace: tuple[int, int, int, int],
|
||||
) -> str:
|
||||
if not isinstance(block_hash, bytes) or not block_hash:
|
||||
raise ValueError("block_hash must be non-empty bytes")
|
||||
if group_id < 0:
|
||||
raise ValueError("group_id must be non-negative")
|
||||
tp_rank, pcp_rank, dcp_rank, pp_rank = namespace
|
||||
return (
|
||||
f"umbp:vllm:v1:{self.namespace.value}:"
|
||||
f"tp{tp_rank}:pcp{pcp_rank}:dcp{dcp_rank}:"
|
||||
f"pp{pp_rank}:g{group_id}:"
|
||||
f"{block_hash.hex()}"
|
||||
)
|
||||
|
||||
def keys_for_topology(
|
||||
self,
|
||||
block_hash: bytes,
|
||||
topology: RankTopology,
|
||||
group_ids: Sequence[int],
|
||||
) -> tuple[str, ...]:
|
||||
return tuple(
|
||||
self.key_for_namespace(block_hash, group_id, namespace)
|
||||
for namespace in topology.all_namespaces()
|
||||
for group_id in group_ids
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class KVRegion:
|
||||
"""One deterministic region in a rank-local KV object."""
|
||||
|
||||
layer_name: str
|
||||
group_id: int
|
||||
block_stride: int
|
||||
block_bytes: int
|
||||
object_offset: int
|
||||
block_size: int = 1
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if self.block_stride <= 0 or self.block_bytes <= 0:
|
||||
raise ValueError("KV region sizes must be positive")
|
||||
if self.block_size <= 0:
|
||||
raise ValueError("KV region block_size must be positive")
|
||||
if self.block_bytes > self.block_stride:
|
||||
raise ValueError(
|
||||
f"KV region bytes ({self.block_bytes}) exceed its stride "
|
||||
f"({self.block_stride}) for {self.layer_name!r}"
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class KVLayoutDescriptor:
|
||||
"""Backend-neutral description of one rank-local KV object layout."""
|
||||
|
||||
regions: tuple[KVRegion, ...]
|
||||
topology: RankTopology
|
||||
layout_format: str = "rank_local"
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if not self.layout_format:
|
||||
raise ValueError("layout_format must not be empty")
|
||||
names = [region.layer_name for region in self.regions]
|
||||
if len(names) != len(set(names)):
|
||||
raise ValueError("KV layout contains duplicate layer regions")
|
||||
|
||||
@property
|
||||
def object_size(self) -> int:
|
||||
return sum(region.block_bytes for region in self.regions)
|
||||
|
||||
|
||||
class KVLayoutPlanner:
|
||||
"""Create a deterministic all-layer object layout from KVCacheConfig."""
|
||||
|
||||
def __init__(self, regions: Sequence[KVRegion]) -> None:
|
||||
self.regions = tuple(regions)
|
||||
self._regions_by_group = {
|
||||
group_id: tuple(r for r in self.regions if r.group_id == group_id)
|
||||
for group_id in {r.group_id for r in self.regions}
|
||||
}
|
||||
self._base_addresses: dict[str, int] = {}
|
||||
self._physical_strides: dict[str, int] = {}
|
||||
|
||||
@classmethod
|
||||
def from_kv_cache_config(cls, config: KVCacheConfig) -> KVLayoutPlanner:
|
||||
group_by_layer = {
|
||||
layer: group_id
|
||||
for group_id, group in enumerate(config.kv_cache_groups)
|
||||
for layer in group.layer_names
|
||||
if group.enable_kv_transfer
|
||||
}
|
||||
regions: list[KVRegion] = []
|
||||
offset = 0
|
||||
for tensor in config.kv_cache_tensors:
|
||||
for layer_name in tensor.layers:
|
||||
group_id = group_by_layer.get(layer_name)
|
||||
if group_id is None:
|
||||
continue
|
||||
spec = config.kv_cache_groups[group_id].kv_cache_spec
|
||||
block_bytes = spec.page_size_bytes
|
||||
regions.append(
|
||||
KVRegion(
|
||||
layer_name=layer_name,
|
||||
group_id=group_id,
|
||||
block_stride=tensor.block_stride,
|
||||
block_bytes=block_bytes,
|
||||
object_offset=offset,
|
||||
block_size=spec.block_size,
|
||||
)
|
||||
)
|
||||
offset += block_bytes
|
||||
return cls(regions)
|
||||
|
||||
@property
|
||||
def object_size(self) -> int:
|
||||
return sum(region.block_bytes for region in self.regions)
|
||||
|
||||
def describe(
|
||||
self,
|
||||
topology: RankTopology,
|
||||
layout_format: str = "rank_local",
|
||||
) -> KVLayoutDescriptor:
|
||||
return KVLayoutDescriptor(
|
||||
regions=self.regions,
|
||||
topology=topology,
|
||||
layout_format=layout_format,
|
||||
)
|
||||
|
||||
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]) -> None:
|
||||
"""Capture stable base addresses for the planned KV regions."""
|
||||
required = {region.layer_name for region in self.regions}
|
||||
missing = required - kv_caches.keys()
|
||||
if missing:
|
||||
raise ValueError(f"missing KV cache layers: {sorted(missing)}")
|
||||
addresses: dict[str, int] = {}
|
||||
for layer_name in required:
|
||||
cache = kv_caches[layer_name]
|
||||
if cache.ndim == 0:
|
||||
raise ValueError(f"KV cache {layer_name!r} must be a tensor")
|
||||
block_stride = cache.stride(0) * cache.element_size()
|
||||
addresses[layer_name] = cache.data_ptr()
|
||||
for region in self.regions:
|
||||
if region.layer_name != layer_name:
|
||||
continue
|
||||
if (
|
||||
block_stride > region.block_stride
|
||||
or region.block_stride % block_stride != 0
|
||||
or (
|
||||
region.block_bytes > block_stride
|
||||
and region.block_bytes % block_stride != 0
|
||||
)
|
||||
):
|
||||
raise ValueError(
|
||||
f"KV block stride mismatch for {layer_name!r}: "
|
||||
f"config={region.block_stride}, tensor={block_stride}"
|
||||
)
|
||||
self._physical_strides[layer_name] = block_stride
|
||||
self._base_addresses = addresses
|
||||
|
||||
def plan_registered_block(
|
||||
self,
|
||||
key: str,
|
||||
block_id: int,
|
||||
*,
|
||||
group_id: int | None = None,
|
||||
request_id: str | None = None,
|
||||
) -> BlockTransferPlan:
|
||||
if not self._base_addresses:
|
||||
raise RuntimeError("KV caches must be registered before planning")
|
||||
return self.plan_for_block(
|
||||
key,
|
||||
block_id,
|
||||
self._base_addresses,
|
||||
group_id=group_id,
|
||||
request_id=request_id,
|
||||
)
|
||||
|
||||
def plan_for_block(
|
||||
self,
|
||||
key: str,
|
||||
block_id: int,
|
||||
base_addresses: dict[str, int],
|
||||
*,
|
||||
group_id: int | None = None,
|
||||
request_id: str | None = None,
|
||||
) -> BlockTransferPlan:
|
||||
"""Lower one GPU block into a deterministic scatter/gather plan."""
|
||||
return BlockTransferPlan(
|
||||
key=key,
|
||||
block_id=block_id,
|
||||
ranges=self._ranges_for_block(block_id, base_addresses, group_id),
|
||||
request_id=request_id,
|
||||
)
|
||||
|
||||
def materialize(self, plan: BlockTransferPlan) -> BlockTransferPlan:
|
||||
if not self._base_addresses:
|
||||
raise RuntimeError("KV caches must be registered before planning")
|
||||
return replace(
|
||||
plan,
|
||||
ranges=self._ranges_for_block(
|
||||
plan.block_id, self._base_addresses, plan.group_id
|
||||
),
|
||||
)
|
||||
|
||||
def _ranges_for_block(
|
||||
self,
|
||||
block_id: int,
|
||||
base_addresses: dict[str, int],
|
||||
group_id: int | None,
|
||||
) -> tuple[KVRange, ...]:
|
||||
if block_id < 0:
|
||||
raise ValueError("block_id must be non-negative")
|
||||
regions = (
|
||||
self.regions
|
||||
if group_id is None
|
||||
else self._regions_by_group.get(group_id, ())
|
||||
)
|
||||
if group_id is not None and not regions:
|
||||
raise ValueError(f"KV layout does not contain cache group {group_id}")
|
||||
ranges: list[KVRange] = []
|
||||
object_offset = 0
|
||||
for region in regions:
|
||||
try:
|
||||
base_address = base_addresses[region.layer_name]
|
||||
except KeyError as exc:
|
||||
raise ValueError(
|
||||
f"missing base address for layer {region.layer_name!r}"
|
||||
) from exc
|
||||
physical_stride = self._physical_strides.get(
|
||||
region.layer_name, region.block_stride
|
||||
)
|
||||
if region.block_bytes <= physical_stride:
|
||||
ranges.append(
|
||||
KVRange(
|
||||
layer_name=region.layer_name,
|
||||
group_id=region.group_id,
|
||||
block_id=block_id,
|
||||
base_address=base_address + block_id * physical_stride,
|
||||
stride=physical_stride,
|
||||
length=region.block_bytes,
|
||||
object_offset=(
|
||||
object_offset
|
||||
if group_id is not None
|
||||
else region.object_offset
|
||||
),
|
||||
)
|
||||
)
|
||||
object_offset += region.block_bytes
|
||||
continue
|
||||
physical_parts = max(1, region.block_bytes // physical_stride)
|
||||
for part in range(physical_parts):
|
||||
part_start = part * physical_stride
|
||||
part_end = min(part_start + physical_stride, region.block_bytes)
|
||||
physical_block_id = block_id * physical_parts + part
|
||||
ranges.append(
|
||||
KVRange(
|
||||
layer_name=region.layer_name,
|
||||
group_id=region.group_id,
|
||||
block_id=block_id,
|
||||
base_address=base_address + physical_block_id * physical_stride,
|
||||
stride=physical_stride,
|
||||
length=part_end - part_start,
|
||||
object_offset=(
|
||||
object_offset + part_start
|
||||
if group_id is not None
|
||||
else region.object_offset + part * physical_stride
|
||||
),
|
||||
)
|
||||
)
|
||||
object_offset += region.block_bytes
|
||||
return tuple(ranges)
|
||||
|
||||
|
||||
class KVRange(NamedTuple):
|
||||
"""A source/destination range in a runtime transfer."""
|
||||
|
||||
layer_name: str
|
||||
group_id: int
|
||||
block_id: int
|
||||
base_address: int
|
||||
stride: int
|
||||
length: int
|
||||
object_offset: int
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BlockTransferPlan:
|
||||
"""One complete logical block transfer."""
|
||||
|
||||
key: str
|
||||
block_id: int
|
||||
ranges: tuple[KVRange, ...] = ()
|
||||
request_id: str | None = None
|
||||
group_id: int | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class LoadSpec:
|
||||
"""Scheduler decision for one externally loaded prefix."""
|
||||
|
||||
local_tokens: int
|
||||
external_tokens: int
|
||||
|
||||
@property
|
||||
def num_tokens_to_load(self) -> int:
|
||||
return max(self.external_tokens - self.local_tokens, 0)
|
||||
|
||||
|
||||
@dataclass
|
||||
class RequestTracker:
|
||||
"""Core request state shared by every runtime mode."""
|
||||
|
||||
saved_tokens: int = 0
|
||||
block_ids: tuple[list[int], ...] = ()
|
||||
load_spec: LoadSpec | None = None
|
||||
|
||||
def reset(self) -> None:
|
||||
self.saved_tokens = 0
|
||||
self.block_ids = ()
|
||||
self.load_spec = None
|
||||
|
||||
def update_blocks(self, block_ids: tuple[list[int], ...]) -> None:
|
||||
if not self.block_ids:
|
||||
self.block_ids = tuple(list(group) for group in block_ids)
|
||||
return
|
||||
if len(self.block_ids) != len(block_ids):
|
||||
raise ValueError("KV cache group count changed for a request")
|
||||
for current, new in zip(self.block_ids, block_ids, strict=True):
|
||||
if new[: len(current)] == current:
|
||||
current.extend(new[len(current) :])
|
||||
elif not new or current[-len(new) :] == new:
|
||||
continue
|
||||
else:
|
||||
current.extend(new)
|
||||
|
||||
def replace_blocks(self, block_ids: tuple[list[int], ...]) -> None:
|
||||
"""Replace stale pre-preemption tables with resumed full tables."""
|
||||
self.block_ids = tuple(list(group) for group in block_ids)
|
||||
self.saved_tokens = 0
|
||||
|
||||
def mark_saved(self, token_count: int, block_size: int) -> int:
|
||||
"""Advance the save watermark only to a complete block boundary."""
|
||||
complete_tokens = token_count // block_size * block_size
|
||||
if complete_tokens < self.saved_tokens:
|
||||
return self.saved_tokens
|
||||
self.saved_tokens = complete_tokens
|
||||
return complete_tokens
|
||||
|
||||
|
||||
class TransferJobStatus(str, Enum):
|
||||
PENDING = "pending"
|
||||
RUNNING = "running"
|
||||
COMPLETED = "completed"
|
||||
FAILED = "failed"
|
||||
|
||||
|
||||
@dataclass
|
||||
class TransferJobState:
|
||||
"""Per-key completion state; one failed key does not fail its siblings."""
|
||||
|
||||
plans: tuple[BlockTransferPlan, ...]
|
||||
status: TransferJobStatus = TransferJobStatus.PENDING
|
||||
completed_keys: set[str] = field(default_factory=set)
|
||||
failed_keys: set[str] = field(default_factory=set)
|
||||
error: str | None = None
|
||||
|
||||
@property
|
||||
def keys(self) -> Sequence[str]:
|
||||
return tuple(plan.key for plan in self.plans)
|
||||
|
||||
def start(self) -> None:
|
||||
if self.status != TransferJobStatus.PENDING:
|
||||
raise RuntimeError(f"cannot start job in state {self.status}")
|
||||
self.status = TransferJobStatus.RUNNING
|
||||
|
||||
def complete(self, keys: Sequence[str] = ()) -> None:
|
||||
self.completed_keys.update(keys or self.keys)
|
||||
self._finish_if_done()
|
||||
|
||||
def fail(self, keys: Sequence[str], error: str | None = None) -> None:
|
||||
self.failed_keys.update(keys)
|
||||
self.error = error
|
||||
self._finish_if_done()
|
||||
|
||||
@property
|
||||
def failed_block_ids(self) -> set[int]:
|
||||
return {plan.block_id for plan in self.plans if plan.key in self.failed_keys}
|
||||
|
||||
def _finish_if_done(self) -> None:
|
||||
keys = set(self.keys)
|
||||
if self.completed_keys | self.failed_keys >= keys:
|
||||
self.status = (
|
||||
TransferJobStatus.FAILED
|
||||
if self.failed_keys
|
||||
else TransferJobStatus.COMPLETED
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class UMBPConnectorMetadata(KVConnectorMetadata):
|
||||
"""Scheduler-to-worker metadata for one engine step."""
|
||||
|
||||
async_load: bool = False
|
||||
store_plans: list[BlockTransferPlan] = field(default_factory=list)
|
||||
load_requests: dict[str, list[BlockTransferPlan]] = field(default_factory=dict)
|
||||
store_requests: dict[str, list[BlockTransferPlan]] = field(default_factory=dict)
|
||||
preempted_request_ids: set[str] = field(default_factory=set)
|
||||
store_event: int = -1
|
||||
|
||||
|
||||
@dataclass
|
||||
class StoreEventResult:
|
||||
"""Number of workers that finished one store event."""
|
||||
|
||||
completed_workers: int = 0
|
||||
|
||||
def merge(self, other: StoreEventResult) -> None:
|
||||
self.completed_workers += other.completed_workers
|
||||
|
||||
|
||||
@dataclass
|
||||
class UMBPConnectorWorkerMetadata(KVConnectorWorkerMetadata):
|
||||
"""Worker completion metadata aggregated across ranks."""
|
||||
|
||||
store_events: dict[int, StoreEventResult] = field(default_factory=dict)
|
||||
|
||||
def aggregate(
|
||||
self, other: KVConnectorWorkerMetadata
|
||||
) -> UMBPConnectorWorkerMetadata:
|
||||
if not isinstance(other, UMBPConnectorWorkerMetadata):
|
||||
raise TypeError("cannot aggregate incompatible UMBP worker metadata")
|
||||
for event, result in other.store_events.items():
|
||||
self.store_events.setdefault(event, StoreEventResult()).merge(result)
|
||||
return self
|
||||
@@ -1,13 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from .base import IUMBPRuntime, UMBPSchedulerHandle, UMBPWorkerHandle
|
||||
from .factory import UMBPRuntimeConfig, UMBPRuntimeFactory
|
||||
|
||||
__all__ = [
|
||||
"IUMBPRuntime",
|
||||
"UMBPSchedulerHandle",
|
||||
"UMBPWorkerHandle",
|
||||
"UMBPRuntimeConfig",
|
||||
"UMBPRuntimeFactory",
|
||||
]
|
||||
@@ -1,69 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Backend-neutral runtime contract for UMBP."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Sequence
|
||||
from typing import Protocol
|
||||
|
||||
import torch
|
||||
|
||||
from ..data import (
|
||||
BlockTransferPlan,
|
||||
KVLayoutDescriptor,
|
||||
RankTopology,
|
||||
TransferJobState,
|
||||
)
|
||||
|
||||
|
||||
class UMBPSchedulerHandle(Protocol):
|
||||
def lookup(self, keys: Sequence[str]) -> Sequence[bool]:
|
||||
"""Return authoritative existence for each key."""
|
||||
|
||||
def clear(self) -> bool:
|
||||
"""Clear all objects visible to this scheduler handle."""
|
||||
|
||||
def close(self) -> None:
|
||||
"""Release scheduler-side resources."""
|
||||
|
||||
|
||||
class UMBPWorkerHandle(Protocol):
|
||||
def register_buffers(self, kv_caches: dict[str, torch.Tensor]) -> None:
|
||||
"""Register each unique KV allocation with the runtime."""
|
||||
|
||||
def load(self, plans: Sequence[BlockTransferPlan]) -> TransferJobState:
|
||||
"""Start loading the requested plans."""
|
||||
|
||||
def store(self, plans: Sequence[BlockTransferPlan]) -> TransferJobState:
|
||||
"""Start storing the requested plans."""
|
||||
|
||||
def wait(self, job: TransferJobState) -> TransferJobState:
|
||||
"""Block until the job is final and its buffers are safe to reuse."""
|
||||
|
||||
def poll(self, job: TransferJobState) -> TransferJobState | None:
|
||||
"""Return a finished job without blocking, or None if still pending."""
|
||||
|
||||
def publish(self, job: TransferJobState) -> None:
|
||||
"""Accept a completed store; a runtime may defer visibility until then."""
|
||||
|
||||
def close(self) -> None:
|
||||
"""Release worker-side resources."""
|
||||
|
||||
|
||||
class IUMBPRuntime(Protocol):
|
||||
def create_scheduler_handle(
|
||||
self,
|
||||
namespace: str,
|
||||
topology: RankTopology,
|
||||
layout: KVLayoutDescriptor,
|
||||
) -> UMBPSchedulerHandle:
|
||||
"""Create a scheduler-side handle."""
|
||||
|
||||
def create_worker_handle(
|
||||
self,
|
||||
namespace: str,
|
||||
topology: RankTopology,
|
||||
layout: KVLayoutDescriptor,
|
||||
) -> UMBPWorkerHandle:
|
||||
"""Create a worker-side handle."""
|
||||
@@ -1,76 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Runtime configuration and adapter factory for UMBP."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass, replace
|
||||
from typing import Any
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
from .base import IUMBPRuntime
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class UMBPRuntimeConfig:
|
||||
"""Shared connector options passed to the selected runtime adapter."""
|
||||
|
||||
mode: str
|
||||
options: dict[str, Any]
|
||||
rank_count: int = 1
|
||||
dp_index: int = 0
|
||||
|
||||
@classmethod
|
||||
def from_vllm(cls, vllm_config: VllmConfig) -> UMBPRuntimeConfig:
|
||||
transfer_config = vllm_config.kv_transfer_config
|
||||
if transfer_config is None:
|
||||
raise ValueError("UMBP requires kv_transfer_config")
|
||||
options = dict(transfer_config.kv_connector_extra_config)
|
||||
mode = options.get("mode", "embedded")
|
||||
if mode not in {"embedded"}:
|
||||
raise ValueError(f"unknown UMBP mode: {mode!r}")
|
||||
namespace = options.get("key_namespace", "auto")
|
||||
if namespace != "auto" and (not isinstance(namespace, str) or not namespace):
|
||||
raise ValueError("key_namespace must be a non-empty string or 'auto'")
|
||||
return cls(
|
||||
mode=mode,
|
||||
options=options,
|
||||
dp_index=int(
|
||||
getattr(vllm_config.parallel_config, "data_parallel_index", 0) or 0
|
||||
),
|
||||
)
|
||||
|
||||
def resolve_for_rank_count(self, rank_count: int) -> UMBPRuntimeConfig:
|
||||
"""Pass topology to the adapter without interpreting its capacity options."""
|
||||
if rank_count <= 0:
|
||||
raise ValueError("rank_count must be positive")
|
||||
return replace(self, rank_count=rank_count)
|
||||
|
||||
|
||||
RuntimeBuilder = Callable[[UMBPRuntimeConfig], IUMBPRuntime]
|
||||
|
||||
|
||||
class UMBPRuntimeFactory:
|
||||
"""Registry used by UMBP adapters without importing MORI in vLLM."""
|
||||
|
||||
_builders: dict[str, RuntimeBuilder] = {}
|
||||
|
||||
@classmethod
|
||||
def register(cls, mode: str, builder: RuntimeBuilder) -> None:
|
||||
if mode not in {"embedded"}:
|
||||
raise ValueError(f"unknown UMBP mode: {mode!r}")
|
||||
if mode in cls._builders:
|
||||
raise ValueError(f"UMBP runtime mode {mode!r} is already registered")
|
||||
cls._builders[mode] = builder
|
||||
|
||||
@classmethod
|
||||
def build(cls, config: UMBPRuntimeConfig) -> IUMBPRuntime:
|
||||
try:
|
||||
builder = cls._builders[config.mode]
|
||||
except KeyError as exc:
|
||||
raise RuntimeError(
|
||||
f"no UMBP runtime adapter is registered for mode {config.mode!r}"
|
||||
) from exc
|
||||
return builder(config)
|
||||
@@ -1,518 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Shared scheduler-side UMBP connector behavior."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from functools import lru_cache
|
||||
from typing import Any
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import KVConnectorMetadata
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
|
||||
from vllm.v1.core.block_pool import BlockPool
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
|
||||
from vllm.v1.core.kv_cache_utils import (
|
||||
resolve_dcp_kv_block_size,
|
||||
resolve_kv_cache_block_sizes,
|
||||
)
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
KVCacheConfig,
|
||||
)
|
||||
from vllm.v1.outputs import KVConnectorOutput
|
||||
from vllm.v1.request import Request
|
||||
|
||||
from .data import (
|
||||
BlockIdentityCodec,
|
||||
BlockTransferPlan,
|
||||
LoadSpec,
|
||||
RankTopology,
|
||||
RequestTracker,
|
||||
StoreEventResult,
|
||||
UMBPConnectorMetadata,
|
||||
UMBPConnectorWorkerMetadata,
|
||||
)
|
||||
from .runtime import UMBPSchedulerHandle
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@lru_cache(maxsize=8192)
|
||||
def _rank_keys(
|
||||
codec: BlockIdentityCodec,
|
||||
topology: RankTopology,
|
||||
block_hash: bytes,
|
||||
group_id: int,
|
||||
) -> tuple[str, ...]:
|
||||
"""Reuse immutable object identities, not runtime residency."""
|
||||
return codec.keys_for_topology(block_hash, topology, (group_id,))
|
||||
|
||||
|
||||
@dataclass
|
||||
class _StoreEvent:
|
||||
plans: tuple[BlockTransferPlan, ...]
|
||||
result: StoreEventResult = field(default_factory=StoreEventResult)
|
||||
|
||||
|
||||
class UMBPStoreConnectorScheduler:
|
||||
"""Own vLLM prefix matching while the runtime owns lookup transport."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
kv_cache_config: KVCacheConfig,
|
||||
runtime: UMBPSchedulerHandle,
|
||||
codec: BlockIdentityCodec,
|
||||
topology: RankTopology | None = None,
|
||||
) -> None:
|
||||
self.block_size, resolved_hash_block_size = resolve_kv_cache_block_sizes(
|
||||
kv_cache_config, vllm_config
|
||||
)
|
||||
self.kv_cache_config = kv_cache_config
|
||||
dcp_size = vllm_config.parallel_config.decode_context_parallel_size
|
||||
self.group_block_sizes = {
|
||||
group_id: resolve_dcp_kv_block_size(group.kv_cache_spec, dcp_size)
|
||||
for group_id, group in enumerate(kv_cache_config.kv_cache_groups)
|
||||
}
|
||||
self.runtime = runtime
|
||||
self.codec = codec
|
||||
self.topology = topology or RankTopology()
|
||||
transfer_config = vllm_config.kv_transfer_config
|
||||
assert transfer_config is not None
|
||||
extra = transfer_config.kv_connector_extra_config
|
||||
self.load_async = bool(extra.get("load_async", True))
|
||||
self.enable_lookup = bool(extra.get("enable_lookup", True))
|
||||
self.save_decode_cache = bool(extra.get("save_decode_cache", False))
|
||||
# A kv_consumer only restores from the pool, as with other store
|
||||
# connectors; it never offloads its own KV.
|
||||
self.store_enabled = getattr(transfer_config, "kv_role", None) != "kv_consumer"
|
||||
# Request.block_hashes are computed at this size; keys index into them.
|
||||
self.hash_block_size = resolved_hash_block_size
|
||||
speculative_config = getattr(vllm_config, "speculative_config", None)
|
||||
self.use_eagle = bool(
|
||||
speculative_config is not None and speculative_config.use_eagle_block_drop()
|
||||
)
|
||||
self._pending_loads: dict[str, list[BlockTransferPlan]] = {}
|
||||
self._load_specs: dict[str, LoadSpec] = {}
|
||||
self._lookup_units = {
|
||||
group_id: self.group_block_sizes[group_id]
|
||||
for group_id in kv_cache_config.prefix_cacheable_group_ids
|
||||
}
|
||||
self._requests: dict[str, Request] = {}
|
||||
self._request_trackers: dict[str, RequestTracker] = {}
|
||||
self._gpu_block_pool: BlockPool | None = None
|
||||
self._store_events: dict[int, _StoreEvent] = {}
|
||||
self._num_workers = getattr(vllm_config.parallel_config, "world_size", 1)
|
||||
self._store_event_counter = 0
|
||||
|
||||
def bind_gpu_block_pool(self, gpu_block_pool: BlockPool) -> None:
|
||||
self._gpu_block_pool = gpu_block_pool
|
||||
|
||||
def get_num_new_matched_tokens(
|
||||
self, request: Request, num_computed_tokens: int
|
||||
) -> tuple[int | None, bool]:
|
||||
if not self.enable_lookup:
|
||||
return 0, False
|
||||
hashes = request.block_hashes
|
||||
if not hashes or request.num_tokens < self.block_size:
|
||||
return 0, False
|
||||
if num_computed_tokens % self.block_size != 0:
|
||||
return 0, False
|
||||
group_ids = self.kv_cache_config.prefix_cacheable_group_ids
|
||||
# vLLM must execute at least the final prompt token. Returning a hit
|
||||
# for the entire prompt leaves the scheduler with no new token to run.
|
||||
max_external_token = request.num_tokens - 1
|
||||
if max_external_token <= num_computed_tokens:
|
||||
self._load_specs.pop(request.request_id, None)
|
||||
return 0, False
|
||||
logical_objects_by_group, keys = self._lookup_objects(
|
||||
request, num_computed_tokens, hashes
|
||||
)
|
||||
try:
|
||||
hits = list(self.runtime.lookup(keys))
|
||||
except Exception as exc:
|
||||
logger.debug("UMBP lookup failed request=%s: %s", request.request_id, exc)
|
||||
return 0, False
|
||||
if len(hits) != len(keys):
|
||||
logger.debug(
|
||||
"UMBP lookup returned an invalid result length request=%s",
|
||||
request.request_id,
|
||||
)
|
||||
return 0, False
|
||||
per_rank = self.topology.rank_count
|
||||
offset = 0
|
||||
logical_hits_by_group: dict[int, list[bool]] = {}
|
||||
for group_id in group_ids:
|
||||
group_size = len(logical_objects_by_group[group_id]) * per_rank
|
||||
group_hits = hits[offset : offset + group_size]
|
||||
offset += group_size
|
||||
logical_hits_by_group[group_id] = (
|
||||
group_hits
|
||||
if per_rank == 1
|
||||
else [
|
||||
all(group_hits[index : index + per_rank])
|
||||
for index in range(0, len(group_hits), per_rank)
|
||||
]
|
||||
)
|
||||
group_id = group_ids[0]
|
||||
matched_blocks = 0
|
||||
for hit in logical_hits_by_group[group_id]:
|
||||
if not hit:
|
||||
break
|
||||
matched_blocks += 1
|
||||
if self.use_eagle:
|
||||
# As core's coordinator: the drafter recomputes the last block.
|
||||
matched_blocks = max(0, matched_blocks - 1)
|
||||
need_to_load = matched_blocks * self._lookup_units[group_id]
|
||||
if logger.isEnabledFor(logging.DEBUG):
|
||||
logger.debug(
|
||||
"UMBP lookup result request=%s keys=%s group_hits=%s load_tokens=%d",
|
||||
request.request_id,
|
||||
keys,
|
||||
logical_hits_by_group,
|
||||
need_to_load,
|
||||
)
|
||||
matched_tokens = num_computed_tokens + need_to_load
|
||||
if need_to_load <= 0:
|
||||
return 0, False
|
||||
self._load_specs[request.request_id] = LoadSpec(
|
||||
local_tokens=num_computed_tokens,
|
||||
external_tokens=matched_tokens,
|
||||
)
|
||||
return need_to_load, self.load_async
|
||||
|
||||
def _lookup_objects(
|
||||
self, request: Request, local_tokens: int, hashes: Sequence[bytes]
|
||||
) -> tuple[dict[int, list[tuple[bytes, tuple[str, ...]]]], list[str]]:
|
||||
objects: dict[int, list[tuple[bytes, tuple[str, ...]]]] = {}
|
||||
keys: list[str] = []
|
||||
for group_id, unit in self._lookup_units.items():
|
||||
group_objects = []
|
||||
for token_end in range(local_tokens + unit, request.num_tokens, unit):
|
||||
block_hash = self._object_hash_at_token_end(hashes, token_end)
|
||||
rank_keys = _rank_keys(self.codec, self.topology, block_hash, group_id)
|
||||
group_objects.append((block_hash, rank_keys))
|
||||
keys.extend(rank_keys)
|
||||
objects[group_id] = group_objects
|
||||
return objects, keys
|
||||
|
||||
def update_state_after_alloc(
|
||||
self,
|
||||
request: Request,
|
||||
blocks: KVCacheBlocks,
|
||||
num_external_tokens: int,
|
||||
) -> None:
|
||||
self._requests[request.request_id] = request
|
||||
block_groups = blocks.get_block_ids(
|
||||
group_ids=self.kv_cache_config.prefix_cacheable_group_ids
|
||||
)
|
||||
if not block_groups:
|
||||
return
|
||||
tracker = self._tracker_for_request(request.request_id)
|
||||
# Core passes the request's whole table on every admission. A request
|
||||
# whose load failed is recomputed into newly allocated blocks.
|
||||
tracker.replace_blocks(tuple(list(group) for group in block_groups))
|
||||
if num_external_tokens <= 0:
|
||||
self._pending_loads.pop(request.request_id, None)
|
||||
self._load_specs.pop(request.request_id, None)
|
||||
return
|
||||
spec = self._load_specs.get(request.request_id)
|
||||
if spec is not None:
|
||||
tracker.load_spec = spec
|
||||
# A request admitted without its load would wait for it forever.
|
||||
assert num_external_tokens == spec.num_tokens_to_load, (
|
||||
f"UMBP matched {spec.num_tokens_to_load} external tokens for "
|
||||
f"request {request.request_id} but {num_external_tokens} were "
|
||||
"allocated"
|
||||
)
|
||||
plans = self._load_plans_for_external_tokens(
|
||||
request,
|
||||
tracker,
|
||||
block_groups,
|
||||
num_external_tokens,
|
||||
)
|
||||
self._pending_loads[request.request_id] = plans
|
||||
|
||||
def build_connector_meta(
|
||||
self, scheduler_output: SchedulerOutput
|
||||
) -> KVConnectorMetadata:
|
||||
meta = UMBPConnectorMetadata(async_load=self.load_async)
|
||||
for request_id in scheduler_output.finished_req_ids:
|
||||
self._pending_loads.pop(request_id, None)
|
||||
self._load_specs.pop(request_id, None)
|
||||
self._request_trackers.pop(request_id, None)
|
||||
self._requests.pop(request_id, None)
|
||||
for request in scheduler_output.scheduled_new_reqs:
|
||||
load_plans = self._pending_loads.pop(request.req_id, [])
|
||||
self._load_specs.pop(request.req_id, None)
|
||||
if load_plans:
|
||||
meta.load_requests[request.req_id] = load_plans
|
||||
else:
|
||||
tracker = self._tracker_for_request(request.req_id)
|
||||
total_tokens = (
|
||||
request.num_computed_tokens
|
||||
+ scheduler_output.num_scheduled_tokens[request.req_id]
|
||||
)
|
||||
tracked_request = self._requests.get(request.req_id)
|
||||
selected_block_ids = tuple(
|
||||
request.block_ids[group_id]
|
||||
for group_id in (self.kv_cache_config.prefix_cacheable_group_ids)
|
||||
)
|
||||
store_plans = (
|
||||
[]
|
||||
if not self.store_enabled or tracked_request is None
|
||||
else self._store_plans(
|
||||
tracked_request,
|
||||
tracker,
|
||||
total_tokens,
|
||||
block_ids_override=selected_block_ids,
|
||||
)
|
||||
)
|
||||
logger.debug(
|
||||
"UMBP store planning request=%s tokens=%d plans=%d keys=%s",
|
||||
request.req_id,
|
||||
total_tokens,
|
||||
len(store_plans),
|
||||
tuple(plan.key for plan in store_plans),
|
||||
)
|
||||
meta.store_plans.extend(store_plans)
|
||||
if store_plans:
|
||||
meta.store_requests[request.req_id] = store_plans
|
||||
|
||||
block_state = getattr(scheduler_output, "kv_connector_block_state", None)
|
||||
cached_reqs = scheduler_output.scheduled_cached_reqs
|
||||
resumed_ids: set[str] = set(getattr(cached_reqs, "resumed_req_ids", ()))
|
||||
for request_id in cached_reqs.req_ids:
|
||||
load_plans = self._pending_loads.pop(request_id, [])
|
||||
self._load_specs.pop(request_id, None)
|
||||
if load_plans:
|
||||
meta.load_requests[request_id] = load_plans
|
||||
elif self.store_enabled:
|
||||
cached_request = self._requests.get(request_id)
|
||||
cached_tracker = self._request_trackers.get(request_id)
|
||||
if cached_request is not None and cached_tracker is not None:
|
||||
request_index = cached_reqs.req_ids.index(request_id)
|
||||
num_computed_tokens = cached_reqs.num_computed_tokens[request_index]
|
||||
# num_tokens counts sampled tokens, so a decode step is still
|
||||
# short of it; only the prompt bounds the prefill.
|
||||
is_prefill = num_computed_tokens < cached_request.num_prompt_tokens
|
||||
if not is_prefill and not self.save_decode_cache:
|
||||
continue
|
||||
new_block_ids = cached_reqs.new_block_ids[request_index]
|
||||
if new_block_ids:
|
||||
selected = tuple(
|
||||
new_block_ids[group_id]
|
||||
for group_id in (
|
||||
self.kv_cache_config.prefix_cacheable_group_ids
|
||||
)
|
||||
)
|
||||
if request_id in resumed_ids:
|
||||
cached_tracker.replace_blocks(selected)
|
||||
else:
|
||||
cached_tracker.update_blocks(selected)
|
||||
# The appended table keeps blocks core has since freed, such
|
||||
# as sliding-window blocks out of the window, which a store
|
||||
# would read; core's current table does not.
|
||||
current = (
|
||||
block_state.get_block_ids(request_id)
|
||||
if block_state is not None
|
||||
else None
|
||||
)
|
||||
if current is not None:
|
||||
cached_tracker.block_ids = tuple(
|
||||
list(current[group_id])
|
||||
for group_id in (
|
||||
self.kv_cache_config.prefix_cacheable_group_ids
|
||||
)
|
||||
)
|
||||
total_tokens = (
|
||||
num_computed_tokens
|
||||
+ scheduler_output.num_scheduled_tokens[request_id]
|
||||
)
|
||||
if not self.save_decode_cache:
|
||||
total_tokens = min(
|
||||
total_tokens, cached_request.num_prompt_tokens
|
||||
)
|
||||
store_plans = self._store_plans(
|
||||
cached_request,
|
||||
cached_tracker,
|
||||
total_tokens,
|
||||
block_ids_override=cached_tracker.block_ids,
|
||||
)
|
||||
meta.store_plans.extend(store_plans)
|
||||
if store_plans:
|
||||
meta.store_requests[request_id] = store_plans
|
||||
|
||||
# Async loads are admitted without scheduling model tokens, so those
|
||||
# requests are absent from both scheduled request collections above.
|
||||
# Forward their plans explicitly so workers can start the transfer and
|
||||
# eventually move the request out of WAITING_FOR_REMOTE_KVS.
|
||||
for request_id, load_plans in list(self._pending_loads.items()):
|
||||
self._pending_loads.pop(request_id, None)
|
||||
self._load_specs.pop(request_id, None)
|
||||
if load_plans:
|
||||
meta.load_requests[request_id] = load_plans
|
||||
|
||||
for request_id in scheduler_output.preempted_req_ids or set():
|
||||
meta.preempted_request_ids.add(request_id)
|
||||
self._load_specs.pop(request_id, None)
|
||||
if preempted_tracker := self._request_trackers.get(request_id):
|
||||
preempted_tracker.reset()
|
||||
|
||||
event_plans = tuple(meta.store_plans)
|
||||
if event_plans:
|
||||
event = self._store_event_counter
|
||||
self._store_event_counter += 1
|
||||
meta.store_event = event
|
||||
self._store_events[event] = _StoreEvent(event_plans)
|
||||
if self._gpu_block_pool is not None:
|
||||
self._gpu_block_pool.touch(
|
||||
[
|
||||
self._gpu_block_pool.blocks[block_id]
|
||||
for block_id in dict.fromkeys(
|
||||
plan.block_id for plan in event_plans
|
||||
)
|
||||
]
|
||||
)
|
||||
return meta
|
||||
|
||||
def _tracker_for_request(self, request_id: str) -> RequestTracker:
|
||||
return self._request_trackers.setdefault(request_id, RequestTracker())
|
||||
|
||||
def _load_plans_for_external_tokens(
|
||||
self,
|
||||
request: Any,
|
||||
tracker: RequestTracker,
|
||||
block_groups: tuple[list[int], ...],
|
||||
num_external_tokens: int,
|
||||
) -> list[BlockTransferPlan]:
|
||||
"""Plan the logical objects selected by the external hit window."""
|
||||
group_ids = self.kv_cache_config.prefix_cacheable_group_ids
|
||||
spec = tracker.load_spec
|
||||
local_tokens = spec.local_tokens if spec is not None else 0
|
||||
end_tokens = local_tokens + num_external_tokens
|
||||
plans: list[BlockTransferPlan] = []
|
||||
for group_index, group_id in enumerate(group_ids):
|
||||
group_block_size = self.group_block_sizes[group_id]
|
||||
start_block = local_tokens // group_block_size
|
||||
num_full_blocks = end_tokens // group_block_size
|
||||
for block_index in range(start_block, num_full_blocks):
|
||||
token_end = (block_index + 1) * group_block_size
|
||||
block_hash = self._object_hash_at_token_end(
|
||||
request.block_hashes, token_end
|
||||
)
|
||||
plans.append(
|
||||
BlockTransferPlan(
|
||||
key=self.codec.key(block_hash, group_id=group_id),
|
||||
block_id=block_groups[group_index][block_index],
|
||||
request_id=request.request_id,
|
||||
group_id=group_id,
|
||||
)
|
||||
)
|
||||
return plans
|
||||
|
||||
def _object_hash_at_token_end(
|
||||
self, hashes: Sequence[bytes], token_end: int
|
||||
) -> bytes:
|
||||
hash_index = token_end // self.hash_block_size - 1
|
||||
if hash_index < 0 or hash_index >= len(hashes):
|
||||
raise ValueError(
|
||||
f"request has {len(hashes)} hashes, needs index {hash_index}"
|
||||
)
|
||||
return hashes[hash_index]
|
||||
|
||||
def _store_plans(
|
||||
self,
|
||||
request: Any,
|
||||
tracker: RequestTracker,
|
||||
token_count: int,
|
||||
block_ids_override: tuple[list[int], ...] | None = None,
|
||||
) -> list[BlockTransferPlan]:
|
||||
"""Describe full blocks produced by a scheduled prefill."""
|
||||
request_id = getattr(request, "request_id", None) or request.req_id
|
||||
group_ids = self.kv_cache_config.prefix_cacheable_group_ids
|
||||
# Scheduled speculative tokens are not yet verified and have no block
|
||||
# hash; a block is storable only once its hash exists.
|
||||
token_count = min(token_count, len(request.block_hashes) * self.hash_block_size)
|
||||
previous_saved_tokens = tracker.saved_tokens
|
||||
# Hash-aligned rather than block-aligned: a group with smaller blocks
|
||||
# than the largest must not wait for that group's block to fill.
|
||||
save_to = tracker.mark_saved(token_count, self.hash_block_size)
|
||||
plans: list[BlockTransferPlan] = []
|
||||
block_groups = (
|
||||
block_ids_override
|
||||
if block_ids_override is not None
|
||||
else tuple(request.block_ids[group_id] for group_id in group_ids)
|
||||
)
|
||||
block_ranges = {
|
||||
group_id: range(
|
||||
previous_saved_tokens // self.group_block_sizes[group_id],
|
||||
min(save_to // self.group_block_sizes[group_id], len(block_ids)),
|
||||
)
|
||||
for group_id, block_ids in zip(group_ids, block_groups)
|
||||
}
|
||||
for group_id, block_ids in zip(group_ids, block_groups):
|
||||
if group_id not in block_ranges:
|
||||
continue
|
||||
group_block_size = self.group_block_sizes[group_id]
|
||||
for index in block_ranges[group_id]:
|
||||
if block_ids[index] == NULL_BLOCK_ID:
|
||||
continue
|
||||
token_end = (index + 1) * group_block_size
|
||||
block_hash = self._object_hash_at_token_end(
|
||||
request.block_hashes, token_end
|
||||
)
|
||||
plans.append(
|
||||
BlockTransferPlan(
|
||||
key=self.codec.key(block_hash, group_id=group_id),
|
||||
block_id=block_ids[index],
|
||||
request_id=request_id,
|
||||
group_id=group_id,
|
||||
)
|
||||
)
|
||||
return plans
|
||||
|
||||
def request_finished(
|
||||
self, request: Request, block_ids: tuple[list[int], ...]
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
self._pending_loads.pop(request.request_id, None)
|
||||
self._load_specs.pop(request.request_id, None)
|
||||
self._requests.pop(request.request_id, None)
|
||||
return False, None
|
||||
|
||||
def update_connector_output(self, output: KVConnectorOutput) -> None:
|
||||
metadata = output.kv_connector_worker_meta
|
||||
if not isinstance(metadata, UMBPConnectorWorkerMetadata):
|
||||
return
|
||||
pool = self._gpu_block_pool
|
||||
for event_id, result in metadata.store_events.items():
|
||||
event = self._store_events.get(event_id)
|
||||
if event is None:
|
||||
continue
|
||||
event.result.merge(result)
|
||||
if event.result.completed_workers < self._num_workers:
|
||||
continue
|
||||
del self._store_events[event_id]
|
||||
if pool is not None:
|
||||
block_ids = dict.fromkeys(plan.block_id for plan in event.plans)
|
||||
pool.free_blocks(
|
||||
pool.blocks[block_id] for block_id in reversed(block_ids)
|
||||
)
|
||||
|
||||
def has_pending_push_work(self) -> bool:
|
||||
return bool(self._store_events)
|
||||
|
||||
def reset_store(self) -> bool:
|
||||
if self._store_events:
|
||||
return False
|
||||
self._load_specs.clear()
|
||||
self._pending_loads.clear()
|
||||
return self.runtime.clear()
|
||||
|
||||
def close(self) -> None:
|
||||
self.runtime.close()
|
||||
@@ -1,240 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Shared worker-side UMBP connector behavior."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, replace
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.forward_context import ForwardContext
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from .data import (
|
||||
BlockIdentityCodec,
|
||||
BlockTransferPlan,
|
||||
KVLayoutPlanner,
|
||||
StoreEventResult,
|
||||
TransferJobState,
|
||||
TransferJobStatus,
|
||||
UMBPConnectorMetadata,
|
||||
UMBPConnectorWorkerMetadata,
|
||||
)
|
||||
from .runtime import UMBPWorkerHandle
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class _StoreBatch:
|
||||
event_id: int
|
||||
jobs: dict[str, TransferJobState]
|
||||
|
||||
|
||||
class UMBPStoreConnectorWorker:
|
||||
"""Translate shared metadata into runtime load/store jobs."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
runtime: UMBPWorkerHandle,
|
||||
layout: KVLayoutPlanner | None = None,
|
||||
*,
|
||||
codec: BlockIdentityCodec | None = None,
|
||||
) -> None:
|
||||
self.runtime = runtime
|
||||
self.layout = layout
|
||||
self.codec = codec
|
||||
self._load_jobs: dict[str, dict[str | None, TransferJobState]] = {}
|
||||
self._store_jobs: dict[str, TransferJobState] = {}
|
||||
self._pending_stores: list[_StoreBatch] = []
|
||||
self._worker_meta = UMBPConnectorWorkerMetadata()
|
||||
self._finished_recving: set[str] = set()
|
||||
self._load_error_block_ids: set[int] = set()
|
||||
self._report_load_completions = False
|
||||
self._active_store_event = -1
|
||||
|
||||
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]) -> None:
|
||||
if self.layout is not None and self.layout.regions:
|
||||
self.layout.register_kv_caches(kv_caches)
|
||||
self.runtime.register_buffers(kv_caches)
|
||||
|
||||
def _materialize_plans(
|
||||
self, plans: list[BlockTransferPlan]
|
||||
) -> list[BlockTransferPlan]:
|
||||
plans = [self._localize_plan(plan) for plan in plans]
|
||||
if self.layout is None or not self.layout.regions:
|
||||
return plans
|
||||
materialized: list[BlockTransferPlan] = []
|
||||
for plan in plans:
|
||||
if plan.ranges:
|
||||
materialized.append(plan)
|
||||
continue
|
||||
materialized.append(self.layout.materialize(plan))
|
||||
return materialized
|
||||
|
||||
def _localize_plan(self, plan: BlockTransferPlan) -> BlockTransferPlan:
|
||||
if self.codec is None or plan.group_id is None:
|
||||
return plan
|
||||
key = self._localize_key(plan.key, plan.group_id)
|
||||
if key == plan.key:
|
||||
return plan
|
||||
return replace(plan, key=key)
|
||||
|
||||
def _localize_key(self, key: str, group_id: int) -> str:
|
||||
if self.codec is None:
|
||||
return key
|
||||
try:
|
||||
block_hash = bytes.fromhex(key.rsplit(":", 1)[1])
|
||||
except (IndexError, ValueError):
|
||||
return key
|
||||
return self.codec.key(block_hash, group_id)
|
||||
|
||||
def start_load_kv(
|
||||
self, forward_context: ForwardContext, metadata: UMBPConnectorMetadata
|
||||
) -> None:
|
||||
del forward_context
|
||||
self._report_load_completions = metadata.async_load
|
||||
for request_id, plans in metadata.load_requests.items():
|
||||
if plans:
|
||||
self._submit_loads(request_id, plans)
|
||||
|
||||
def _submit_loads(self, request_id: str, plans: list[BlockTransferPlan]) -> None:
|
||||
materialized = self._materialize_plans(plans)
|
||||
logger.debug(
|
||||
"UMBP load submitted request=%s plans=%d ranges=%d bytes=%d",
|
||||
request_id,
|
||||
len(materialized),
|
||||
sum(len(plan.ranges) for plan in materialized),
|
||||
sum(item.length for plan in materialized for item in plan.ranges),
|
||||
)
|
||||
self._load_jobs[request_id] = {None: self.runtime.load(materialized)}
|
||||
|
||||
def _finish_job(
|
||||
self, request_id: str, result: TransferJobState, *, is_load: bool
|
||||
) -> bool:
|
||||
operation = "load" if is_load else "store"
|
||||
succeeded = result.status == TransferJobStatus.COMPLETED
|
||||
if not succeeded:
|
||||
logger.debug(
|
||||
"UMBP %s failed request=%s status=%s error=%s",
|
||||
operation,
|
||||
request_id,
|
||||
result.status.value,
|
||||
result.error,
|
||||
)
|
||||
if is_load:
|
||||
self._load_error_block_ids.update(result.failed_block_ids)
|
||||
elif succeeded:
|
||||
self.runtime.publish(result)
|
||||
return succeeded
|
||||
|
||||
def wait_for_layer_load(self, layer_name: str) -> None:
|
||||
if self._report_load_completions:
|
||||
# Asynchronous loads fill blocks of requests that do not run until
|
||||
# get_finished reports them, so no forward pass reads those blocks.
|
||||
return
|
||||
self._drain_load_jobs(wait=True)
|
||||
|
||||
def wait_for_save(self) -> None:
|
||||
if self._store_jobs or self._active_store_event >= 0:
|
||||
self._pending_stores.append(
|
||||
_StoreBatch(self._active_store_event, self._store_jobs)
|
||||
)
|
||||
self._store_jobs = {}
|
||||
self._active_store_event = -1
|
||||
self._drain_store_jobs(wait=False)
|
||||
|
||||
def _drain_store_jobs(self, *, wait: bool) -> None:
|
||||
for batch in list(self._pending_stores):
|
||||
for request_id, job in list(batch.jobs.items()):
|
||||
result = self.runtime.wait(job) if wait else self.runtime.poll(job)
|
||||
if result is None:
|
||||
continue
|
||||
# A partly failed job is not published, including its stored keys.
|
||||
self._finish_job(request_id, result, is_load=False)
|
||||
del batch.jobs[request_id]
|
||||
if not batch.jobs:
|
||||
if batch.event_id >= 0:
|
||||
self._worker_meta.store_events[batch.event_id] = StoreEventResult(
|
||||
completed_workers=1
|
||||
)
|
||||
self._pending_stores.remove(batch)
|
||||
|
||||
def _submit_store_plans(
|
||||
self, plans: list[BlockTransferPlan], request_id: str
|
||||
) -> None:
|
||||
if not plans:
|
||||
return
|
||||
materialized = self._materialize_plans(plans)
|
||||
self._store_jobs[request_id] = self.runtime.store(materialized)
|
||||
|
||||
def enqueue_stores(self, metadata: UMBPConnectorMetadata) -> None:
|
||||
self._active_store_event = metadata.store_event
|
||||
store_requests = {
|
||||
request_id: list(plans)
|
||||
for request_id, plans in metadata.store_requests.items()
|
||||
}
|
||||
grouped = {plan for plans in store_requests.values() for plan in plans}
|
||||
for plan in metadata.store_plans:
|
||||
if plan not in grouped:
|
||||
store_requests.setdefault(
|
||||
plan.request_id or "__umbp_batch__", []
|
||||
).append(plan)
|
||||
grouped.add(plan)
|
||||
for request_id, plans in store_requests.items():
|
||||
self._submit_store_plans(plans, request_id)
|
||||
|
||||
def handle_preemptions(self, metadata: UMBPConnectorMetadata) -> None:
|
||||
"""Finish request-local jobs before vLLM reuses their GPU blocks."""
|
||||
preempted = metadata.preempted_request_ids
|
||||
if not preempted:
|
||||
return
|
||||
for request_id in preempted:
|
||||
for job in self._load_jobs.pop(request_id, {}).values():
|
||||
self.runtime.wait(job)
|
||||
for jobs in [self._store_jobs, *(batch.jobs for batch in self._pending_stores)]:
|
||||
for request_id, job in jobs.items():
|
||||
if request_id in preempted or any(
|
||||
plan.request_id in preempted for plan in job.plans
|
||||
):
|
||||
jobs[request_id] = self.runtime.wait(job)
|
||||
|
||||
def get_finished(
|
||||
self, finished_req_ids: set[str]
|
||||
) -> tuple[set[str] | None, set[str] | None]:
|
||||
del finished_req_ids
|
||||
self._drain_load_jobs(wait=False)
|
||||
self._drain_store_jobs(wait=False)
|
||||
recving = set(self._finished_recving)
|
||||
self._finished_recving.difference_update(recving)
|
||||
return None, recving or None
|
||||
|
||||
def _drain_load_jobs(self, *, wait: bool) -> None:
|
||||
for request_id, jobs in list(self._load_jobs.items()):
|
||||
for layer, job in list(jobs.items()):
|
||||
result = self.runtime.wait(job) if wait else self.runtime.poll(job)
|
||||
if result is None:
|
||||
continue
|
||||
del jobs[layer]
|
||||
self._finish_job(request_id, result, is_load=True)
|
||||
if not jobs:
|
||||
del self._load_jobs[request_id]
|
||||
if self._report_load_completions:
|
||||
self._finished_recving.add(request_id)
|
||||
|
||||
def build_connector_worker_meta(self) -> UMBPConnectorWorkerMetadata:
|
||||
result = self._worker_meta
|
||||
self._worker_meta = UMBPConnectorWorkerMetadata()
|
||||
return result
|
||||
|
||||
def get_block_ids_with_load_errors(self) -> set[int]:
|
||||
failed = self._load_error_block_ids
|
||||
self._load_error_block_ids = set()
|
||||
return failed
|
||||
|
||||
def close(self) -> None:
|
||||
self.wait_for_save()
|
||||
self._drain_store_jobs(wait=True)
|
||||
self._drain_load_jobs(wait=True)
|
||||
self.runtime.close()
|
||||
Reference in New Issue
Block a user