Add pydocstyle to the ruff rules (#52136)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Signed-off-by: Harry Mellor <hej.mellor@gmail.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Harry Mellor
2026-09-16 16:44:29 +00:00
committed by GitHub
co-authored by Claude Opus 5
parent 804e5377aa
commit 8c1557a79c
2002 changed files with 15024 additions and 14269 deletions
@@ -87,6 +87,7 @@ class KVConnectorFactory:
Returns:
The connector class.
"""
if connector_name not in cls._registry:
raise ValueError(f"Connector '{connector_name}' is not registered.")
@@ -1,8 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
KV cache helper for store.
"""
"""KV cache helper for store."""
from collections.abc import Iterator
from dataclasses import dataclass
@@ -53,7 +51,8 @@ def get_kv_connector_cache_layout(vllm_config: VllmConfig | None = None):
class KVOutputAggregator:
"""Utility class to aggregate the output of all workers into a single
output corresponding to Rank 0 for scheduler."""
output corresponding to Rank 0 for scheduler.
"""
def __init__(self, expected_finished_count: int):
# Complete transfer tracker. Used to track finished requests
@@ -233,11 +232,10 @@ def copy_kv_blocks(
def kv_postprocess_blksize_on_receive(cache, indices, block_size_ratio):
"""
Transforms the layout of received KV cache blocks to the local block_size.
"""Transforms the layout of received KV cache blocks to the local block_size.
(Only works for local blocksize > remote blocksize)
example:
Example:
local blocksize = 16 tokens, remote blocksize = 4 tokens
local block[0] = remote block[0, 1, 2, 3]
remote is |h0-b0|h1-b0|h2-b0|h3-b0|h0-b1|h1-b1|h2-b1|h3-b1|...
@@ -246,6 +244,7 @@ def kv_postprocess_blksize_on_receive(cache, indices, block_size_ratio):
1. view => view remote as n_blocks * remote_shape(H,remoteN,D)
2. permute => (H, nblocks, remoteN, D)
3. flatten => (H, localN, D)
"""
blocks_to_update = cache.index_select(0, indices)
# use physical order
@@ -293,8 +292,7 @@ def kv_postprocess_layout_on_receive(cache, indices):
def kv_postprocess_blksize_and_layout_on_receive(cache, indices, block_size_ratio):
"""
Transforms the layout of received KV cache to the local block_size and LBHNC.
"""Transforms the layout of received KV cache to the local block_size and LBHNC.
(Only works for local blocksize > remote blocksize)
prefill is LBHNC, smaller block_size
@@ -317,9 +315,9 @@ def kv_postprocess_blksize_and_layout_on_receive(cache, indices, block_size_rati
def yield_req_data(
scheduler_output,
) -> Iterator[tuple[str, tuple[list[int], ...] | None, bool]]:
"""
Yields:
(req_id, new_block_id_groups, preempted)
"""Yields:
(req_id, new_block_id_groups, preempted)
"""
# new requests
for req_data in scheduler_output.scheduled_new_reqs:
@@ -346,6 +344,7 @@ def get_current_attn_backends(
Returns:
Deduplicated list of attention backend classes.
"""
layer_type = cast(type[Any], AttentionLayerBase)
layers = get_layers_from_vllm_config(vllm_config, layer_type, layer_names)
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
KVConnectorBase_V1 Class for Distributed KV Cache & Hidden State
"""KVConnectorBase_V1 Class for Distributed KV Cache & Hidden State
communication in vLLM v1
The class provides the following primitives:
@@ -97,8 +96,7 @@ class KVConnectorTransferResults:
class SupportsHMA(ABC):
"""
The class that indicates the corresponding connector supports hybrid memory
"""The class that indicates the corresponding connector supports hybrid memory
allocator (HMA).
This is required to use the connector together with hybrid memory allocator.
"""
@@ -109,8 +107,7 @@ class SupportsHMA(ABC):
request: "Request",
block_ids: tuple[list[int], ...],
) -> tuple[bool, dict[str, Any] | None]:
"""
Called exactly once when a request has finished for all kv cache groups,
"""Called exactly once when a request has finished for all kv cache groups,
before its blocks are freed for each group.
NOTE(Kuntai): This function is only supported by connectors that support HMA.
@@ -124,6 +121,7 @@ class SupportsHMA(ABC):
get_finished().
Optional KVTransferParams to be included in the request outputs
returned by the engine.
"""
raise NotImplementedError
@@ -144,8 +142,7 @@ class KVConnectorRole(enum.Enum):
class KVConnectorHandshakeMetadata(ABC): # noqa: B024
"""
Metadata used for out of band connector handshake between
"""Metadata used for out of band connector handshake between
P/D workers. This needs to serializable.
"""
@@ -153,8 +150,7 @@ class KVConnectorHandshakeMetadata(ABC): # noqa: B024
class KVConnectorMetadata(ABC): # noqa: B024
"""
Abstract Metadata used to communicate
"""Abstract Metadata used to communicate
Scheduler KVConnector -> Worker KVConnector.
"""
@@ -162,8 +158,7 @@ class KVConnectorMetadata(ABC): # noqa: B024
class KVConnectorWorkerMetadata(ABC):
"""
Abstract Metadata used to communicate back
"""Abstract Metadata used to communicate back
Worker KVConnector -> Scheduler KVConnector.
Each worker can output its own metadata.
@@ -176,16 +171,12 @@ class KVConnectorWorkerMetadata(ABC):
def aggregate(
self, other: "KVConnectorWorkerMetadata"
) -> "KVConnectorWorkerMetadata":
"""
Aggregate metadata with another `KVConnectorWorkerMetadata` object.
"""
"""Aggregate metadata with another `KVConnectorWorkerMetadata` object."""
pass
class KVConnectorBase_V1(ABC):
"""
Base class for KV connectors.
"""
"""Base class for KV connectors."""
@property
def supports_divergent_local_hybrid_hits(self) -> bool:
@@ -245,6 +236,7 @@ class KVConnectorBase_V1(ABC):
Args:
connector_metadata (dict): the connector metadata.
"""
self._connector_metadata = connector_metadata
@@ -263,6 +255,7 @@ class KVConnectorBase_V1(ABC):
Returns:
ConnectorMetadata: the connector metadata.
"""
# Should only be called while set to valid metadata.
assert self._connector_metadata is not None
@@ -273,22 +266,22 @@ class KVConnectorBase_V1(ABC):
Returns:
bool: True if connector metadata exists, False otherwise.
"""
return self._connector_metadata is not None
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
"""
Initialize with the KV caches. Useful for pre-registering the
"""Initialize with the KV caches. Useful for pre-registering the
KV Caches in the KVConnector (e.g. for NIXL).
Args:
kv_caches: dictionary of layer names, kv cache
"""
return
def set_host_xfer_buffer_ops(self, copy_operation: CopyBlocksOp):
"""
Set the xPU-specific ops for copying KV between host and device.
"""Set the xPU-specific ops for copying KV between host and device.
Needed when host buffer is used for kv transfer (e.g., in NixlConnector)
"""
return
@@ -303,16 +296,14 @@ class KVConnectorBase_V1(ABC):
return
def handle_preemptions(self, kv_connector_metadata: KVConnectorMetadata):
"""
Handle preempted requests or evicted blocks BEFORE they are overwritten.
"""Handle preempted requests or evicted blocks BEFORE they are overwritten.
Needed for connectors which use async saves (e.g., OffloadingConnector)
"""
return
@abstractmethod
def start_load_kv(self, forward_context: "ForwardContext", **kwargs: Any) -> None:
"""
Start loading the KV cache from the connector to vLLM's paged
"""Start loading the KV cache from the connector to vLLM's paged
KV buffer. Loads required by the current forward start before it;
independent asynchronous loads may start after it is submitted.
@@ -329,8 +320,7 @@ class KVConnectorBase_V1(ABC):
@abstractmethod
def wait_for_layer_load(self, layer_name: str) -> None:
"""
Block until the KV for a specific layer is loaded into vLLM's
"""Block until the KV for a specific layer is loaded into vLLM's
paged buffer. This is called from within attention layer to ensure
async copying from start_load_kv is complete.
@@ -338,6 +328,7 @@ class KVConnectorBase_V1(ABC):
Args:
layer_name: the name of that layer
"""
pass
@@ -349,8 +340,7 @@ class KVConnectorBase_V1(ABC):
attn_metadata: "AttentionMetadata",
**kwargs: Any,
) -> None:
"""
Start saving a layer of KV cache from vLLM's paged buffer
"""Start saving a layer of KV cache from vLLM's paged buffer
to the connector. This is called from within attention layer to
enable async copying during execution.
@@ -360,13 +350,13 @@ class KVConnectorBase_V1(ABC):
layer in vLLM.
attn_metadata (AttentionMetadata): the attention metadata.
**kwargs: additional arguments for the save operation.
"""
pass
@abstractmethod
def wait_for_save(self):
"""
Block until all the save operations is done. This is called
"""Block until all the save operations is done. This is called
as the forward context exits to ensure that the async saving
from save_kv_layer is complete before finishing the forward.
@@ -377,8 +367,7 @@ class KVConnectorBase_V1(ABC):
def get_finished(
self, finished_req_ids: set[str]
) -> tuple[set[str] | None, set[str] | None]:
"""
Notifies worker-side connector ids of requests that have
"""Notifies worker-side connector ids of requests that have
finished generating tokens on the worker.
The scheduler process (via the Executors) will use this output
to track which workers are done.
@@ -389,6 +378,7 @@ class KVConnectorBase_V1(ABC):
tuple of (sending/saving ids, recving/loading ids).
The finished saves/sends req ids must belong to a set provided in a
call to this method (this call or a prior one).
"""
return None, None
@@ -403,8 +393,7 @@ class KVConnectorBase_V1(ABC):
)
def get_block_ids_with_load_errors(self) -> set[int]:
"""
Get the set of block IDs that failed to load.
"""Get the set of block IDs that failed to load.
Returns:
Set of block IDs that encountered load errors.
@@ -419,50 +408,47 @@ class KVConnectorBase_V1(ABC):
block IDs must appear here no later than that same pass.
- Sync loading: failed blocks should be reported in the forward
pass in which they are detected.
"""
return set()
def shutdown(self):
"""
Shutdown the connector. This is called when the worker process
"""Shutdown the connector. This is called when the worker process
is shutting down to ensure that all the async operations are
completed and the connector is cleaned up properly.
"""
return None
def get_kv_connector_stats(self) -> "KVConnectorStats | None":
"""
Get the KV connector stats collected during the last interval.
"""
"""Get the KV connector stats collected during the last interval."""
return None
def get_kv_connector_kv_cache_events(self) -> "KVConnectorKVEvents | None":
"""
Get the KV connector kv cache events collected during the last interval.
"""Get the KV connector kv cache events collected during the last interval.
This function should be called by the model runner every time after the
model execution and before cleanup.
"""
return None
def get_handshake_metadata(self) -> KVConnectorHandshakeMetadata | None:
"""
Get the KVConnector handshake metadata for this connector.
"""Get the KVConnector handshake metadata for this connector.
This metadata is used for out-of-band connector handshake
between P/D workers.
Returns:
KVConnectorHandshakeMetadata: the handshake metadata.
None if no handshake metadata is available.
"""
return None
def build_connector_worker_meta(self) -> KVConnectorWorkerMetadata | None:
"""
Build the KVConnector worker metadata for this engine step.
"""Build the KVConnector worker metadata for this engine step.
Returns:
KVConnectorWorkerMetadata: the worker metadata.
None if no worker metadata is available.
"""
return None
@@ -476,12 +462,12 @@ class KVConnectorBase_V1(ABC):
self.bind_gpu_block_pool(kv_cache_manager.block_pool)
def bind_gpu_block_pool(self, gpu_block_pool: "BlockPool") -> None:
"""
Bind the GPU block pool to the connector for per-GPU block status tracking.
"""Bind the GPU block pool to the connector for per-GPU block status tracking.
For example, inc/dec ref counts, or iterate over the prefix cache blocks.
Args:
gpu_block_pool: the GPU block pool.
"""
return
@@ -491,8 +477,7 @@ class KVConnectorBase_V1(ABC):
request: "Request",
num_computed_tokens: int,
) -> tuple[int | None, bool]:
"""
Get number of new tokens that can be loaded from the
"""Get number of new tokens that can be loaded from the
external KV cache beyond the num_computed_tokens.
Args:
@@ -517,6 +502,7 @@ class KVConnectorBase_V1(ABC):
call. If the cache cannot be loaded for some tokens (e.g., due to
connectivity issues or eviction), those tokens must not be taken
into account.
"""
pass
@@ -524,8 +510,7 @@ class KVConnectorBase_V1(ABC):
def update_state_after_alloc(
self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int
):
"""
Update KVConnector state after block allocation.
"""Update KVConnector state after block allocation.
If get_num_new_matched_tokens previously returned True for a
request, this function may be called twice for that same request -
@@ -543,6 +528,7 @@ class KVConnectorBase_V1(ABC):
blocks (KVCacheBlocks): the blocks allocated for the request.
num_external_tokens (int): the number of tokens to load from the
external KV cache. 0 means nothing should be loaded.
"""
pass
@@ -550,14 +536,14 @@ class KVConnectorBase_V1(ABC):
def build_connector_meta(
self, scheduler_output: SchedulerOutput
) -> KVConnectorMetadata:
"""
Build the connector metadata for this step.
"""Build the connector metadata for this step.
This function should NOT modify fields in the scheduler_output.
Also, calling this function will reset the state of the connector.
Args:
scheduler_output (SchedulerOutput): the scheduler output object.
"""
pass
@@ -570,12 +556,12 @@ class KVConnectorBase_V1(ABC):
return
def update_connector_output(self, connector_output: KVConnectorOutput):
"""
Update KVConnector state from worker-side connectors output.
"""Update KVConnector state from worker-side connectors output.
Args:
connector_output (KVConnectorOutput): the worker-side
connectors output.
"""
return
@@ -584,8 +570,7 @@ class KVConnectorBase_V1(ABC):
request: "Request",
block_ids: list[int],
) -> tuple[bool, dict[str, Any] | None]:
"""
Called exactly once when a request has finished, before its blocks are
"""Called exactly once when a request has finished, before its blocks are
freed.
The connector may assumes responsibility for freeing the blocks
@@ -597,6 +582,7 @@ class KVConnectorBase_V1(ABC):
get_finished().
Optional KVTransferParams to be included in the request outputs
returned by the engine.
"""
return False, None
@@ -614,11 +600,11 @@ class KVConnectorBase_V1(ABC):
return False
def take_events(self) -> Iterable["KVCacheEvent"]:
"""
Take the KV cache events from the connector.
"""Take the KV cache events from the connector.
Yields:
New KV cache events since the last call.
"""
return ()
@@ -640,16 +626,16 @@ class KVConnectorBase_V1(ABC):
@classmethod
def get_required_kvcache_layout(cls, vllm_config: "VllmConfig") -> str | None:
"""
Get the required KV cache layout for this connector.
"""Get the required KV cache layout for this connector.
Args:
vllm_config (VllmConfig): the vllm config.
Returns:
str: the required KV cache layout. e.g. HND, or NHD.
None if the connector does not require a specific layout.
"""
"""
if cls is KVConnectorBase_V1:
raise TypeError(
"get_required_kvcache_layout should not be called "
@@ -659,8 +645,7 @@ class KVConnectorBase_V1(ABC):
@classmethod
def requires_piecewise_for_cudagraph(cls, extra_config: dict[str, Any]) -> bool:
"""
Check if this connector requires PIECEWISE CUDA graph mode.
"""Check if this connector requires PIECEWISE CUDA graph mode.
Connectors that use asynchronous layer-by-layer operations
(wait_for_layer_load/save_kv_layer) should override this method
@@ -676,27 +661,26 @@ class KVConnectorBase_V1(ABC):
Returns:
True if this connector requires PIECEWISE CUDA graph mode,
False otherwise.
"""
return False
def get_finished_count(self) -> int | None:
"""
Get the count of requests expected to complete send/receive operations
"""Get the count of requests expected to complete send/receive operations
via this connector. This method is used to initialize the
KVOutputAggregator, overwriting the default world_size.
Returns:
int: expected sending or receiving completion count.
"""
"""
return None
@classmethod
def build_kv_connector_stats(
cls, data: dict[str, Any] | None = None
) -> "KVConnectorStats | None":
"""
KVConnectorStats resolution method. This method allows dynamically
"""KVConnectorStats resolution method. This method allows dynamically
registered connectors to return their own KVConnectorStats object,
which can implement custom aggregation logic on the data dict.
"""
@@ -705,19 +689,18 @@ class KVConnectorBase_V1(ABC):
def set_xfer_handshake_metadata(
self, metadata: dict[int, KVConnectorHandshakeMetadata]
) -> None:
"""
Set the KV connector handshake metadata for this connector.
"""Set the KV connector handshake metadata for this connector.
Args:
metadata (KVConnectorHandshakeMetadata): the handshake metadata to set.
"""
return None
def set_xfer_handshake_metadata_pp_aware(
self, metadata: dict[tuple[int, int], KVConnectorHandshakeMetadata]
) -> None:
"""
Set handshake metadata keyed by (pp_rank, tp_rank).
"""Set handshake metadata keyed by (pp_rank, tp_rank).
- Default implementation assumes pp_rank is always 0
- PP-aware connectors override this to consume all PP producer shards.
"""
@@ -738,19 +721,18 @@ class KVConnectorBase_V1(ABC):
labelnames: list[str],
per_engine_labelvalues: dict[int, list[object]],
) -> "KVConnectorPromMetrics | None":
"""
Create a KVConnectorPromMetrics subclass which should register
"""Create a KVConnectorPromMetrics subclass which should register
per-connector Prometheus metrics and implement observe() to
expose connector transfer stats via Prometheus.
"""
return None
def reset_cache(self) -> bool | None:
"""
Reset the connector's internal cache.
"""Reset the connector's internal cache.
Returns:
bool: True if the cache was successfully reset, False otherwise.
"""
logger.debug(
"Connector cache reset requested, but %s does not implement reset_cache().",
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
DecodeBenchConnector: A KV Connector for decode instance performance testing.
"""DecodeBenchConnector: A KV Connector for decode instance performance testing.
This connector emulates a prefill-decode disaggregated setting by filling
the KV cache with dummy values, allowing measurement of decoder performance
@@ -10,7 +9,7 @@ under larger input sequence lengths (ISL) in resource-limited environments.
Usage:
To use this connector for benchmarking, configure it in the kv_transfer_config:
Example:
Example:
vllm serve <model> --kv-transfer-config '{
"kv_connector": "DecodeBenchConnector",
"kv_role": "kv_both",
@@ -29,6 +28,7 @@ Usage:
- fill_mean (float): Mean value for random normal fill (default: 0.015)
- fill_std (float): Standard deviation for random fill (default: 0.0)
Set to 0 for constant values, >0 for random sampling
"""
from dataclasses import dataclass
@@ -80,8 +80,7 @@ class DecodeBenchConnectorMetadata(KVConnectorMetadata):
class DecodeBenchConnector(KVConnectorBase_V1, SupportsHMA):
"""
A KV Connector for decode instance performance testing.
"""A KV Connector for decode instance performance testing.
This connector fills the KV cache with dummy values to emulate a
prefill-decode disaggregated setting, enabling performance testing of the
@@ -219,8 +218,7 @@ class DecodeBenchConnectorScheduler:
request: "Request",
num_computed_tokens: int,
) -> tuple[int, bool]:
"""
For new requests, return the number of tokens that should be filled
"""For new requests, return the number of tokens that should be filled
with dummy KV cache values.
Returns:
@@ -228,6 +226,7 @@ class DecodeBenchConnectorScheduler:
- num_tokens_to_fill: number of uncomputed tokens minus 1
(we fill everything except the last token for decode)
- is_async: False (synchronous filling)
"""
req_id = request.request_id
@@ -251,8 +250,7 @@ class DecodeBenchConnectorScheduler:
def update_state_after_alloc(
self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int
):
"""
Called after blocks are allocated. Store the block IDs so we can
"""Called after blocks are allocated. Store the block IDs so we can
fill them with dummy values.
Supports both single- and multi-group KV cache configurations.
@@ -334,8 +332,7 @@ class DecodeBenchConnectorScheduler:
def build_connector_meta(
self, scheduler_output: "SchedulerOutput"
) -> KVConnectorMetadata:
"""
Build metadata containing information about which blocks to fill
"""Build metadata containing information about which blocks to fill
with dummy KV values.
"""
meta = DecodeBenchConnectorMetadata(reqs_to_fill=self._pending_fills.copy())
@@ -346,9 +343,7 @@ class DecodeBenchConnectorScheduler:
return meta
def request_finished(self, request: "Request"):
"""
Called when a request has finished. Clean up any state.
"""
"""Called when a request has finished. Clean up any state."""
self._filled_requests.discard(request.request_id)
@@ -389,8 +384,7 @@ class DecodeBenchConnectorWorker:
)
def start_fill_kv(self, metadata: DecodeBenchConnectorMetadata):
"""
Fill the allocated KV cache blocks with dummy values.
"""Fill the allocated KV cache blocks with dummy values.
This simulates having a populated KV cache from a prefill phase,
allowing decode performance testing with larger context sizes.
@@ -419,13 +413,13 @@ class DecodeBenchConnectorWorker:
)
def _fill_blocks(self, group_idx: int, block_ids: list[int], num_tokens: int):
"""
Fill specified blocks with dummy values for a specific KV cache group.
"""Fill specified blocks with dummy values for a specific KV cache group.
Args:
group_idx: The KV cache group index to fill
block_ids: List of block IDs to fill in this group
num_tokens: Total number of tokens to fill across these blocks
"""
if not block_ids:
return
@@ -501,6 +495,7 @@ class DecodeBenchConnectorWorker:
tensor's first dim are ignored.
fill_mean: Mean value for the fill.
fill_std: Standard deviation for the fill.
"""
# Convert block_ids to tensor on device
block_ids_tensor = torch.tensor(
@@ -551,6 +546,7 @@ class DecodeBenchConnectorWorker:
kv_cache: A state tensor to fill in its entirety.
fill_mean: Mean value for the fill.
fill_std: Standard deviation for the fill.
"""
if fill_std > 0:
kv_cache.normal_(mean=fill_mean, std=fill_std)
@@ -118,6 +118,7 @@ class ExampleConnector(KVConnectorBase_V1):
Note:
The number of elements in kv_caches and layer_names should be
the same.
"""
def inject_kv_into_layer(
@@ -134,6 +135,9 @@ class ExampleConnector(KVConnectorBase_V1):
src_kv_cache (torch.Tensor): the source KV cache.
slot_mapping (torch.Tensor): the slot mapping. In shape
[num_tokens].
attn_metadata (AttentionMetadata): attention metadata for the
current forward pass.
"""
# Deliberate bulk transfers in this reference connector.
with gpu_sync_allowed():
@@ -201,6 +205,7 @@ class ExampleConnector(KVConnectorBase_V1):
Args:
layer_name: the name of that layer
"""
return
@@ -220,6 +225,7 @@ class ExampleConnector(KVConnectorBase_V1):
layer in vLLM.
attn_metadata (AttentionMetadata): the attention metadata.
**kwargs: additional arguments for the save operation.
"""
def extract_kv_from_layer(
@@ -260,8 +266,7 @@ class ExampleConnector(KVConnectorBase_V1):
request: "Request",
num_computed_tokens: int,
) -> tuple[int | None, bool]:
"""
Get number of new tokens that can be loaded from the
"""Get number of new tokens that can be loaded from the
external KV cache beyond the num_computed_tokens.
Args:
@@ -272,6 +277,7 @@ class ExampleConnector(KVConnectorBase_V1):
Returns:
the number of tokens that can be loaded from the
external KV cache beyond what is already computed.
"""
# NOTE: in this debug implementation, we assume that the prompt is
# cached_prompt + newly_generated_single_token
@@ -295,8 +301,7 @@ class ExampleConnector(KVConnectorBase_V1):
def update_state_after_alloc(
self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int
):
"""
Update KVConnector state after block allocation.
"""Update KVConnector state after block allocation.
If blocks were allocated, add to _requests_need_load,
such that we load the KVs in the next forward pass.
@@ -315,6 +320,7 @@ class ExampleConnector(KVConnectorBase_V1):
Args:
scheduler_output (SchedulerOutput): the scheduler output object.
"""
meta = ExampleConnectorMetadata()
@@ -57,6 +57,7 @@ def load_hidden_states(path: str) -> dict[str, torch.Tensor]:
Returns:
Dict with "hidden_states" and "token_ids" tensors.
"""
lock_path = path + ".lock"
with open(lock_path) as lf:
@@ -96,8 +97,7 @@ class ExampleHiddenStatesConnectorMetadata(KVConnectorMetadata):
class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
"""
Simple debug implementation of a HiddenStatesConnector.
"""Simple debug implementation of a HiddenStatesConnector.
Simply extracts the hidden states from the kv cache and stores them to disk.
Must be used in conjunction with the `extract_hidden_states` spec decoding method.
@@ -278,7 +278,8 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
def _on_write_done(self, req_id: str, future: Future) -> None:
"""Surface any exception from the disk-write thread and drop the
completed future from the in-flight tracking dict."""
completed future from the in-flight tracking dict.
"""
self._req_futures.pop(req_id, None)
exc = future.exception()
if exc is not None:
@@ -436,8 +437,7 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
request: "Request",
num_computed_tokens: int,
) -> tuple[int | None, bool]:
"""
Get number of new tokens that can be loaded from the
"""Get number of new tokens that can be loaded from the
external KV cache beyond the num_computed_tokens.
Args:
@@ -448,6 +448,7 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
Returns:
the number of tokens that can be loaded from the
external KV cache beyond what is already computed.
"""
# This connector is store-only, so we don't need to load any tokens
return 0, False
@@ -471,6 +472,7 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
Args:
scheduler_output (SchedulerOutput): the scheduler output object.
"""
meta = ExampleHiddenStatesConnectorMetadata()
@@ -509,8 +511,7 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
request: "Request",
block_ids: list[int],
) -> tuple[bool, dict[str, Any] | None]:
"""
Called exactly once when a request has finished, before its blocks are
"""Called exactly once when a request has finished, before its blocks are
freed.
Returns True to delay block freeing until get_finished extracts
@@ -587,16 +588,16 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
@classmethod
def get_required_kvcache_layout(cls, vllm_config: "VllmConfig") -> str | None:
"""
Get the required KV cache layout for this connector.
"""Get the required KV cache layout for this connector.
Args:
vllm_config (VllmConfig): the vllm config.
Returns:
str: the required KV cache layout. e.g. HND, or NHD.
None if the connector does not require a specific layout.
"""
"""
if cls is KVConnectorBase_V1:
raise TypeError(
"get_required_kvcache_layout should not be called "
@@ -96,6 +96,7 @@ class FlexKVConnectorV1(KVConnectorBase_V1):
Args:
forward_context (ForwardContext): the forward context.
**kwargs (Any): additional arguments (unused).
"""
self._flexkv_connector.start_load_kv(forward_context, **kwargs)
@@ -107,6 +108,7 @@ class FlexKVConnectorV1(KVConnectorBase_V1):
Args:
layer_name: the name of the layer (unused).
"""
self._flexkv_connector.wait_for_layer_load(layer_name)
@@ -131,6 +133,7 @@ class FlexKVConnectorV1(KVConnectorBase_V1):
kv_layer (torch.Tensor): the paged KV buffer (unused).
attn_metadata (AttentionMetadata): the attention metadata (unused).
**kwargs (Any): additional arguments (unused).
"""
self._flexkv_connector.save_kv_layer(
layer_name, kv_layer, attn_metadata, **kwargs
@@ -160,6 +163,7 @@ class FlexKVConnectorV1(KVConnectorBase_V1):
that have finished asynchronous transfer. The finished saves/sends
req ids must belong to a set provided in a call to this method
(this call or a prior one).
"""
return self._flexkv_connector.get_finished(finished_req_ids)
@@ -169,6 +173,7 @@ class FlexKVConnectorV1(KVConnectorBase_V1):
Args:
kv_caches: dictionary of layer names to kv cache tensors.
"""
self._flexkv_connector.register_kv_caches(kv_caches)
@@ -192,6 +197,7 @@ class FlexKVConnectorV1(KVConnectorBase_V1):
Tuple of (num_external_tokens, is_ready) where
num_external_tokens is the number of additional tokens that
can be loaded from the external KV cache.
"""
return self._flexkv_connector.get_num_new_matched_tokens(
request, num_computed_tokens
@@ -215,6 +221,7 @@ class FlexKVConnectorV1(KVConnectorBase_V1):
Args:
scheduler_output (SchedulerOutput): the scheduler output object.
"""
return self._flexkv_connector.build_connector_meta(scheduler_output)
@@ -224,6 +231,7 @@ class FlexKVConnectorV1(KVConnectorBase_V1):
Args:
connector_output (KVConnectorOutput): the worker-side
connectors output.
"""
self._flexkv_connector.update_connector_output(connector_output)
@@ -240,6 +248,7 @@ class FlexKVConnectorV1(KVConnectorBase_V1):
should not be freed until the request_id is returned from
:meth:`get_finished`. kv_transfer_params is an optional dict of
KVTransferParams to be included in the request outputs.
"""
return self._flexkv_connector.request_finished(request, block_ids)
@@ -248,6 +257,7 @@ class FlexKVConnectorV1(KVConnectorBase_V1):
Returns:
New KV cache events since the last call.
"""
return self._flexkv_connector.take_events()
@@ -67,6 +67,7 @@ class Hf3fsClient:
size: Total size of the storage file in bytes
bytes_per_page: Size of each page in bytes
entries: Maximum number of concurrent operations
"""
if not HF3FS_AVAILABLE:
raise ImportError(
@@ -154,7 +155,7 @@ class Hf3fsClient:
)
def _release_resources(self) -> None:
"""Release all acquired resources safely"""
"""Release all acquired resources safely."""
# iov must be released before ioring and shm
for attr in ("iov_r", "iov_w", "ior_r", "ior_w"):
obj = getattr(self, attr, None)
@@ -192,6 +193,7 @@ class Hf3fsClient:
Returns:
List of operation results (0 for success, non-zero for error)
"""
self.check(offsets, tensors)
assert self.ior_r is not None
@@ -228,11 +230,12 @@ class Hf3fsClient:
Args:
offsets: List of byte offsets to write to
tensors: List of tensors containing data to write
event: CUDA event to synchronize on before reading the tensors
Returns:
List of operation results (0 for success, non-zero for error)
"""
"""
self.check(offsets, tensors)
assert self.ior_w is not None
assert self.iov_w is not None
@@ -282,6 +285,7 @@ class Hf3fsClient:
Returns:
Size of the file in bytes
"""
return self.size
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
HF3FS KV Connector Implementation for vLLM.
"""HF3FS KV Connector Implementation for vLLM.
This module implements a KV connector that uses
the 3FS for storing and retrieving KV cache data.
@@ -99,9 +98,7 @@ logger = init_logger(__name__)
class AsyncOperationManager:
"""
Manages async save/load operations with background threads.
"""
"""Manages async save/load operations with background threads."""
def __init__(self, connector: "HF3FSKVConnector"):
# Store connector reference and extract commonly used attributes
@@ -667,9 +664,7 @@ class HF3FSKVConnector(KVConnectorBase_V1):
return self._async_manager.get_finished_operations(finished_req_ids)
def get_kv_connector_stats(self) -> Optional["KVConnectorStats"]:
"""
Get the KV connector stats collected during the last interval.
"""
"""Get the KV connector stats collected during the last interval."""
# Clear stats for next iteration
if (
hasattr(self, "_async_manager")
@@ -886,8 +881,7 @@ class HF3FSKVConnector(KVConnectorBase_V1):
def build_kv_connector_stats(
cls, data: dict[str, Any] | None = None
) -> Optional["KVConnectorStats"]:
"""
KVConnectorStats resolution method. This method allows dynamically
"""KVConnectorStats resolution method. This method allows dynamically
registered connectors to return their own KVConnectorStats object,
which can implement custom aggregation logic on the data dict.
"""
@@ -1021,7 +1015,7 @@ class HF3FSKVConnector(KVConnectorBase_V1):
@dataclass
class HF3FSKVConnectorStats(KVConnectorStats):
"""Container for transfer performance metrics"""
"""Container for transfer performance metrics."""
def __post_init__(self):
if not self.data:
@@ -1,8 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
HF3FS Metadata Server with key-based organization.
"""
"""HF3FS Metadata Server with key-based organization."""
import argparse
import logging
@@ -119,6 +117,7 @@ class GlobalMetadataState:
Returns:
Dictionary mapping key -> allocated page index
"""
with self.global_lock:
if rank not in self.rank_metadata:
@@ -167,6 +166,7 @@ class GlobalMetadataState:
rank: Rank ID that confirmed the writes
key_confirmations: List of (key, page_index) tuples
pages_to_release: List of page indices to release back to free pool
"""
with self.global_lock:
# Confirm successful writes
@@ -197,6 +197,7 @@ class GlobalMetadataState:
Returns:
List of boolean values indicating key existence and completion
"""
with self.global_lock:
results = []
@@ -218,6 +219,7 @@ class GlobalMetadataState:
Returns:
List of page indices in the same order as input keys (None if key not found)
"""
with self.global_lock:
if rank not in self.rank_metadata:
@@ -128,6 +128,7 @@ def scatter_kv_caches(
num_heads: Size of the H axis
content_size: Size of the C axis
kv_cache_strides: Element strides of each ``[B, H, N, C]`` layer view
"""
num_layers = len(kv_caches_ptrs)
num_tokens_in_block = len(token_indices)
@@ -184,6 +185,7 @@ def gather_kv_caches(
num_heads: Size of the H axis
content_size: Size of the C axis
kv_cache_strides: Element strides of each ``[B, H, N, C]`` layer view
"""
num_layers = kv_caches_ptrs.shape[0]
num_tokens_in_block = len(token_indices)
@@ -32,9 +32,7 @@ logger = init_logger(__name__)
class LMCacheKVEvents(KVConnectorKVEvents):
"""
Concrete implementation of KVConnectorKVEvents using KVEventAggregator.
"""
"""Concrete implementation of KVConnectorKVEvents using KVEventAggregator."""
def __init__(self, num_workers: int) -> None:
self._aggregator = KVEventAggregator(num_workers)
@@ -43,9 +41,7 @@ class LMCacheKVEvents(KVConnectorKVEvents):
self._aggregator.add_events(events)
def aggregate(self) -> "LMCacheKVEvents":
"""
Aggregate KV events and retain only common events.
"""
"""Aggregate KV events and retain only common events."""
common_events = self._aggregator.get_common_events()
self._aggregator.clear_events()
self._aggregator.add_events(common_events)
@@ -72,8 +68,7 @@ class LMCacheKVEvents(KVConnectorKVEvents):
class LMCacheConnectorV1(KVConnectorBase_V1):
@classmethod
def requires_piecewise_for_cudagraph(cls, extra_config: dict[str, Any]) -> bool:
"""
LMCache requires PIECEWISE CUDA graph mode when layerwise
"""LMCache requires PIECEWISE CUDA graph mode when layerwise
operations are enabled. The wait_for_layer_load and save_kv_layer
methods perform actual async synchronization that cannot be
captured in CUDA graphs.
@@ -118,12 +113,12 @@ class LMCacheConnectorV1(KVConnectorBase_V1):
# Worker-side methods
# ==============================
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
"""
Initialize with the KV caches. Useful for pre-registering the
"""Initialize with the KV caches. Useful for pre-registering the
KV Caches in the KVConnector (e.g. for NIXL).
Args:
kv_caches: dictionary of layer names, kv cache
"""
if hasattr(self._lmcache_engine, "register_kv_caches"):
self._lmcache_engine.register_kv_caches(kv_caches)
@@ -149,8 +144,7 @@ class LMCacheConnectorV1(KVConnectorBase_V1):
engine.layerwise_retrievers = []
def start_load_kv(self, forward_context: "ForwardContext", **kwargs: Any) -> None:
"""
Start loading the KV cache from the connector to vLLM's paged
"""Start loading the KV cache from the connector to vLLM's paged
KV buffer. This is called from the forward context before the
forward pass to enable async loading during model execution.
@@ -166,8 +160,7 @@ class LMCacheConnectorV1(KVConnectorBase_V1):
self._lmcache_engine.start_load_kv(forward_context, **kwargs)
def wait_for_layer_load(self, layer_name: str) -> None:
"""
Block until the KV for a specific layer is loaded into vLLM's
"""Block until the KV for a specific layer is loaded into vLLM's
paged buffer. This is called from within attention layer to ensure
async copying from start_load_kv is complete.
@@ -175,6 +168,7 @@ class LMCacheConnectorV1(KVConnectorBase_V1):
Args:
layer_name: the name of that layer
"""
self._lmcache_engine.wait_for_layer_load(layer_name)
@@ -185,8 +179,7 @@ class LMCacheConnectorV1(KVConnectorBase_V1):
attn_metadata: AttentionMetadata,
**kwargs: Any,
) -> None:
"""
Start saving the a layer of KV cache from vLLM's paged buffer
"""Start saving the a layer of KV cache from vLLM's paged buffer
to the connector. This is called from within attention layer to
enable async copying during execution.
@@ -196,14 +189,14 @@ class LMCacheConnectorV1(KVConnectorBase_V1):
layer in vLLM.
attn_metadata (AttentionMetadata): the attention metadata.
**kwargs: additional arguments for the save operation.
"""
self._lmcache_engine.save_kv_layer(
layer_name, kv_layer, attn_metadata, **kwargs
)
def wait_for_save(self):
"""
Block until all the save operations is done. This is called
"""Block until all the save operations is done. This is called
as the forward context exits to ensure that the async saving
from save_kv_layer is complete before finishing the forward.
@@ -214,8 +207,7 @@ class LMCacheConnectorV1(KVConnectorBase_V1):
def get_finished(
self, finished_req_ids: set[str]
) -> tuple[set[str] | None, set[str] | None]:
"""
Notifies worker-side connector ids of requests that have
"""Notifies worker-side connector ids of requests that have
finished generating tokens.
Returns:
@@ -224,16 +216,17 @@ class LMCacheConnectorV1(KVConnectorBase_V1):
tuple of (sending/saving ids, recving/loading ids).
The finished saves/sends req ids must belong to a set provided in a
call to this method (this call or a prior one).
"""
return self._lmcache_engine.get_finished(finished_req_ids)
def get_block_ids_with_load_errors(self) -> set[int]:
"""
Get the set of block IDs that failed to load.
"""Get the set of block IDs that failed to load.
Returns:
Set of block IDs that encountered load errors.
Empty set if no load errors occurred.
"""
method = getattr(self._lmcache_engine, "get_block_ids_with_load_errors", None)
if callable(method):
@@ -243,10 +236,7 @@ class LMCacheConnectorV1(KVConnectorBase_V1):
return set()
def get_kv_connector_kv_cache_events(self) -> LMCacheKVEvents | None:
"""
Get the KV connector kv cache events collected during the last interval.
"""
"""Get the KV connector kv cache events collected during the last interval."""
events = self._lmcache_engine.get_kv_events() # type: ignore [attr-defined]
if not events:
return None
@@ -276,8 +266,7 @@ class LMCacheConnectorV1(KVConnectorBase_V1):
request: "Request",
num_computed_tokens: int,
) -> tuple[int | None, bool]:
"""
Get number of new tokens that can be loaded from the
"""Get number of new tokens that can be loaded from the
external KV cache beyond the num_computed_tokens.
Args:
@@ -288,6 +277,7 @@ class LMCacheConnectorV1(KVConnectorBase_V1):
Returns:
the number of tokens that can be loaded from the
external KV cache beyond what is already computed.
"""
return self._lmcache_engine.get_num_new_matched_tokens(
request, num_computed_tokens
@@ -296,32 +286,30 @@ class LMCacheConnectorV1(KVConnectorBase_V1):
def update_state_after_alloc(
self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int
):
"""
Update KVConnector state after block allocation.
"""
"""Update KVConnector state after block allocation."""
self._lmcache_engine.update_state_after_alloc(request, num_external_tokens)
def build_connector_meta(
self, scheduler_output: SchedulerOutput
) -> KVConnectorMetadata:
"""
Build the connector metadata for this step.
"""Build the connector metadata for this step.
This function should NOT modify fields in the scheduler_output.
Also, calling this function will reset the state of the connector.
Args:
scheduler_output (SchedulerOutput): the scheduler output object.
"""
return self._lmcache_engine.build_connector_meta(scheduler_output)
def update_connector_output(self, connector_output: KVConnectorOutput):
"""
Update KVConnector state from worker-side connectors output.
"""Update KVConnector state from worker-side connectors output.
Args:
connector_output (KVConnectorOutput): the worker-side
connectors output.
"""
# Get the KV events
kv_cache_events = connector_output.kv_cache_events
@@ -342,8 +330,7 @@ class LMCacheConnectorV1(KVConnectorBase_V1):
request: "Request",
block_ids: list[int],
) -> tuple[bool, dict[str, Any] | None]:
"""
Called when a request has finished, before its blocks are freed.
"""Called when a request has finished, before its blocks are freed.
Returns:
True if the request is being saved/sent asynchronously and blocks
@@ -351,15 +338,16 @@ class LMCacheConnectorV1(KVConnectorBase_V1):
get_finished().
Optional KVTransferParams to be included in the request outputs
returned by the engine.
"""
return self._lmcache_engine.request_finished(request, block_ids)
def take_events(self) -> Iterable["KVCacheEvent"]:
"""
Take the KV cache events from the connector.
"""Take the KV cache events from the connector.
Yields:
New KV cache events since the last call.
"""
if self._kv_cache_events is not None:
self._kv_cache_events.aggregate()
@@ -44,8 +44,7 @@ def send_lmcache_request(
request_type: RequestType,
payloads: list[Any],
) -> MessagingFuture[Any]:
"""
Helper function to send the request to the LMCache multiprocess server
"""Helper function to send the request to the LMCache multiprocess server.
Args:
mq_client: The LMCache multiprocess mode message queue client
@@ -54,8 +53,8 @@ def send_lmcache_request(
Returns:
A messaging future for the request
"""
"""
future = mq_client.submit_request(
request_type, payloads, get_response_class(request_type)
)
@@ -65,14 +64,14 @@ def send_lmcache_request(
def get_lmcache_chunk_size(
mq_client: MessageQueueClient,
) -> int:
"""
Helper function to get the LMCache chunk size from the server
"""Helper function to get the LMCache chunk size from the server.
Args:
mq_client: The LMCache multiprocess mode message queue client
Returns:
An integer representing the LMCache chunk size
"""
future = send_lmcache_request(mq_client, RequestType.GET_CHUNK_SIZE, [])
chunk_size = future.result()
@@ -147,16 +146,16 @@ class LMCacheMPSchedulerAdapter:
vllm_block_size: int,
parallel_strategy: ParallelStrategy,
):
"""
Args:
server_url: The server URL for the LMCache message queue
context: The ZMQ context
"""Args:
server_url: The server URL for the LMCache message queue
context: The ZMQ context
model_name: The model name used for LMCache keys
vllm_block_size: The block size used in vLLM
parallel_strategy:
The parallel strategy, which includes `use_mla`,
`world_size`, `worker_id` and so on
model_name: The model name used for LMCache keys
vllm_block_size: The block size used in vLLM
parallel_strategy:
The parallel strategy, which includes `use_mla`,
`world_size`, `worker_id` and so on
"""
self.mq_client = MessageQueueClient(server_url, context)
@@ -195,8 +194,7 @@ class LMCacheMPSchedulerAdapter:
block_hashes: list[bytes] | None = None,
token_ids: list[int] | None = None,
) -> None:
"""
Submit a new lookup request to LMCache if there is no ongoing request.
"""Submit a new lookup request to LMCache if there is no ongoing request.
Supports both token-based and hash-based vLLM:
- token_ids: token IDs (token-based vLLM) -> single token-mode key
@@ -219,6 +217,7 @@ class LMCacheMPSchedulerAdapter:
for later retrieve operations.
In the meantime, this function will record the lookup request, and the
status of the look up request can be checked by `check_lookup_result`.
"""
if request_id in self.lookup_futures:
# Skip if there is already a lookup request
@@ -260,8 +259,7 @@ class LMCacheMPSchedulerAdapter:
@_lmcache_nvtx_annotate
def check_lookup_result(self, request_id: str) -> int | None:
"""
Check the result of a previously submitted lookup request.
"""Check the result of a previously submitted lookup request.
Args:
request_id: The ID of the lookup request submitted in
@@ -271,6 +269,7 @@ class LMCacheMPSchedulerAdapter:
An integer representing the total number of tokens matched
in LMCache (prefix matching), or
None if the lookup request is not finished yet.
"""
assert request_id in self.lookup_futures, (
f"Lookup request for request_id={request_id} has not been submitted"
@@ -285,25 +284,27 @@ class LMCacheMPSchedulerAdapter:
return num_chunks * self.chunk_size
def num_blocks_per_chunk(self) -> int:
"""
Returns:
The number of vllm blocks in a LMCache data chunk
"""Returns:
The number of vllm blocks in a LMCache data chunk
"""
return self.blocks_in_chunk
def cleanup_lookup_result(self, request_id: str) -> None:
"""
Clean up lookup future for a finished request to prevent memory leak.
"""Clean up lookup future for a finished request to prevent memory leak.
Args:
request_id: The ID of the finished request.
"""
self.lookup_futures.pop(request_id, None)
def end_session(self, request_id: str) -> None:
"""
Notify LMCache server to remove the session for a finished request.
"""Notify LMCache server to remove the session for a finished request.
Args:
request_id: The ID of the finished request.
"""
send_lmcache_request(
self.mq_client,
@@ -319,7 +320,7 @@ class LMCacheMPSchedulerAdapter:
end: int = 0,
request_id: str | None = None,
) -> IPCCacheEngineKey:
"""Convert token IDs to an IPC cache engine key"""
"""Convert token IDs to an IPC cache engine key."""
return IPCCacheEngineKey(
model_name=self.model_name,
world_size=self.world_size,
@@ -334,7 +335,7 @@ class LMCacheMPSchedulerAdapter:
def _create_hash_key(
self, chunk_hash: bytes, request_id: str | None = None
) -> IPCCacheEngineKey:
"""Create a hash-mode IPC cache engine key"""
"""Create a hash-mode IPC cache engine key."""
return IPCCacheEngineKey(
model_name=self.model_name,
world_size=self.world_size,
@@ -411,12 +412,12 @@ class LMCacheMPWorkerAdapter:
)
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
"""
Register the kv caches with LMCache server
"""Register the kv caches with LMCache server.
Args:
kv_caches: A dict of kv caches to register. The keys are the
layer names and the values are the corresponding tensors.
"""
# Register kv cache and send the request
self.kv_caches = kv_caches
@@ -432,14 +433,14 @@ class LMCacheMPWorkerAdapter:
def submit_store_request(
self, request_id: str, op: LoadStoreOp, event: torch.cuda.Event
):
"""
Submit a KV cache store request to LMCache
"""Submit a KV cache store request to LMCache.
Args:
request_id: The ID of the request
op: The LoadStoreOp describing the store operation.
event: The CUDA event that is recorded after the current
model inference step
"""
if op.block_hashes is not None:
# Hash mode
@@ -466,14 +467,14 @@ class LMCacheMPWorkerAdapter:
def submit_retrieve_request(
self, request_id: str, op: LoadStoreOp, event: torch.cuda.Event
):
"""
Submit a KV cache retrieve request to LMCache
"""Submit a KV cache retrieve request to LMCache.
Args:
request_id: The ID of the request
op: The LoadStoreOp describing the retrieve operation.
event: The CUDA event that is recorded after the current
model inference step
"""
if op.block_hashes is not None:
# Hash mode
@@ -503,8 +504,7 @@ class LMCacheMPWorkerAdapter:
ops: list[LoadStoreOp],
event: torch.cuda.Event,
):
"""
Submit a batched store request to LMCache
"""Submit a batched store request to LMCache.
Args:
request_ids: The IDs of the requests
@@ -512,6 +512,7 @@ class LMCacheMPWorkerAdapter:
the same length as request_ids
event: The CUDA event that is recorded after the current
model inference step
"""
all_keys: list[IPCCacheEngineKey] = []
block_ids: list[int] = []
@@ -552,8 +553,7 @@ class LMCacheMPWorkerAdapter:
ops: list[LoadStoreOp],
event: torch.cuda.Event,
):
"""
Submit a batched retrieve request to LMCache
"""Submit a batched retrieve request to LMCache.
Args:
request_ids: The IDs of the requests
@@ -561,6 +561,7 @@ class LMCacheMPWorkerAdapter:
the same length as request_ids
event: The CUDA event that is recorded after the current
model inference step
"""
all_keys: list[IPCCacheEngineKey] = []
block_ids: list[int] = []
@@ -598,8 +599,7 @@ class LMCacheMPWorkerAdapter:
def get_finished(
self, finished_req_ids_from_engine: set[str]
) -> tuple[set[str] | None, set[str] | None]:
"""
Check and get the finished store and retrieve requests.
"""Check and get the finished store and retrieve requests.
Args:
finished_req_ids_from_engine: the set of request ids that are
@@ -617,6 +617,7 @@ class LMCacheMPWorkerAdapter:
multiple times in `finished_req_ids_from_engine`. The adapter should
take care of deduplicating the request IDs and only return the request
IDs that have not been returned before.
"""
finished_stores = set()
finished_retrieves = set()
@@ -675,16 +676,14 @@ class LMCacheMPWorkerAdapter:
return ret_stores, finished_retrieves
def num_blocks_per_chunk(self) -> int:
"""
Returns:
The number of vllm blocks in a LMCache data chunk
"""Returns:
The number of vllm blocks in a LMCache data chunk
"""
return self.blocks_in_chunk
def shutdown(self):
"""
Shutdown the LMCache MP worker adapter
"""
"""Shutdown the LMCache MP worker adapter."""
logger.info("Unregistering kv caches")
send_lmcache_request(
self.mq_client, RequestType.UNREGISTER_KV_CACHE, [self.instance_id]
@@ -712,7 +711,7 @@ class LMCacheMPWorkerAdapter:
end: int = 0,
request_id: str | None = None,
) -> IPCCacheEngineKey:
"""Convert token IDs to an IPC cache engine key"""
"""Convert token IDs to an IPC cache engine key."""
return IPCCacheEngineKey(
model_name=self.model_name,
world_size=self.world_size,
@@ -726,7 +725,7 @@ class LMCacheMPWorkerAdapter:
def _create_hash_key(
self, chunk_hash: bytes, request_id: str | None = None
) -> IPCCacheEngineKey:
"""Create a hash-mode IPC cache engine key"""
"""Create a hash-mode IPC cache engine key."""
return IPCCacheEngineKey(
model_name=self.model_name,
world_size=self.world_size,
@@ -59,9 +59,7 @@ def lmcache_get_or_create_config() -> V1Config:
def hex_hash_to_int16(s: str) -> int:
"""
Convert a hex hash string to a 16-bit integer.
"""
"""Convert a hex hash string to a 16-bit integer."""
return int(s, 16) & 0xFFFF
@@ -70,8 +68,7 @@ def apply_mm_hashes_to_token_ids(
mm_hashes: list[str],
mm_positions: list["PlaceholderRange"],
) -> torch.Tensor:
"""
Overwrite token_ids in-place for multimodal placeholders using
"""Overwrite token_ids in-place for multimodal placeholders using
efficient slice assignments.
"""
n = token_ids.size(0)
@@ -95,8 +92,7 @@ def mla_enabled(model_config: "ModelConfig") -> bool:
def extract_mm_features(
request: Union["Request", "NewRequestData"], modify: bool = False
) -> tuple[list[str], list["PlaceholderRange"]]:
"""
Normalize multimodal information from a Request into parallel lists.
"""Normalize multimodal information from a Request into parallel lists.
This helper reads either:
1) `request.mm_features` (objects each exposing `.identifier` and
@@ -119,6 +115,7 @@ def extract_mm_features(
Returns:
tuple[list[str], list[PlaceholderRange]]: (`mm_hashes`, `mm_positions`).
May be `([], [])` when no multimodal data is present.
"""
if getattr(request, "mm_features", None):
mm_hashes, mm_positions = zip(
@@ -171,6 +171,7 @@ class RequestTracker:
lmcache_cached_tokens (int): the number of tokens that are
cached in LMCache.
skip_save (bool): whether the request cache should be saved
"""
# vLLM 0.9.0 update: request.block_ids changed from list[int] to
# list[list[int]]
@@ -219,7 +220,6 @@ class RequestTracker:
"""Update the request tracker when a running request is
scheduled again
"""
self.token_ids.extend(new_token_ids)
if new_block_ids is None:
@@ -288,6 +288,7 @@ class ReqMeta:
Returns:
the request metadata if we need to perform load/save
operations, None otherwise.
"""
input_token_ids = tracker.token_ids
input_token_len = len(input_token_ids)
@@ -445,6 +446,7 @@ def _init_lmcache_engine(
Returns:
The initialized LMCache engine
"""
if curr_engine := LMCacheEngineBuilder.get(ENGINE_NAME):
return curr_engine
@@ -563,6 +565,7 @@ class LMCacheConnectorMetadata(KVConnectorMetadata):
Args:
req_meta (ReqMeta): the request metadata.
"""
self.requests.append(req_meta)
@@ -719,6 +722,7 @@ class LMCacheConnectorV1Impl:
Returns:
dict: Dictionary containing inference information
"""
# Get vLLM config information
vllm_config = self._vllm_config
@@ -766,6 +770,7 @@ class LMCacheConnectorV1Impl:
Returns:
str: vLLM version string
"""
return VLLM_VERSION
@@ -816,6 +821,7 @@ class LMCacheConnectorV1Impl:
Note:
The number of elements in kv_caches and layer_names should be
the same.
"""
if len(self.kv_caches) == 0:
self._init_kv_caches_from_forward_context(forward_context)
@@ -920,6 +926,7 @@ class LMCacheConnectorV1Impl:
Args:
layer_name: the name of that layer
"""
if self.layerwise_retrievers:
logger.debug("Waiting for layer %s to be loaded", self.current_layer)
@@ -951,6 +958,7 @@ class LMCacheConnectorV1Impl:
kv_layer (torch.Tensor): the paged KV buffer of the current
layer in vLLM.
attn_metadata (AttentionMetadata): the attention metadata.
"""
assert self.lmcache_engine is not None
@@ -1039,7 +1047,6 @@ class LMCacheConnectorV1Impl:
@_lmcache_nvtx_annotate
def wait_for_save(self):
"""Blocking until the KV cache is saved to the connector buffer."""
connector_metadata = self._parent._get_connector_metadata()
assert isinstance(connector_metadata, LMCacheConnectorMetadata)
@@ -1150,8 +1157,7 @@ class LMCacheConnectorV1Impl:
request: "Request",
num_computed_tokens: int,
) -> int | None:
"""
Check for external KV cache hit.
"""Check for external KV cache hit.
Args:
request (Request): the request object.
@@ -1161,6 +1167,7 @@ class LMCacheConnectorV1Impl:
Returns:
the number of tokens that can be loaded from the
external KV cache beyond what is already computed.
"""
if self.kv_role == "kv_producer" and not hasattr(
self.lookup_client, "supports_producer_reuse"
@@ -1236,13 +1243,11 @@ class LMCacheConnectorV1Impl:
@_lmcache_nvtx_annotate
def update_state_after_alloc(self, request: "Request", num_external_tokens: int):
"""
Update KVConnector state after temporary buffer alloc.
"""Update KVConnector state after temporary buffer alloc.
For SharedStorageConnector, update _request_needs_load
if the CacheManager this allocated blocks for us.
"""
# Clear local status in lookup client when a new request is
# successfully scheduled.
self.lookup_client.clear_lookup_status(request.request_id)
@@ -1311,8 +1316,8 @@ class LMCacheConnectorV1Impl:
Args:
scheduler_output (SchedulerOutput): the scheduler output object.
"""
"""
force_skip_save = self.kv_role == "kv_consumer" or self.force_skip_save
meta = LMCacheConnectorMetadata()
@@ -88,9 +88,7 @@ def extract_world_size_and_kv_rank(
rank: int,
vllm_config: VllmConfig,
) -> tuple[int, int]:
"""
Convert the rank for the MLA.
"""
"""Convert the rank for the MLA."""
use_mla = mla_enabled(vllm_config.model_config)
if not use_mla:
return world_size, rank
@@ -174,8 +172,7 @@ def create_worker_adapter(
class LMCacheMPRequestState(enum.Enum):
"""
State machine:
"""State machine:
PREFETCHING -- update_state_after_alloc --> WAITING_FOR_LOAD
WAITING_FOR_LOAD -- process_loading_requests --> READY
"""
@@ -234,7 +231,8 @@ class LMCacheMPRequestTracker:
####
def needs_retrieve(self) -> bool:
"""Check whether the current request needs retrieve, will be used
update_stage_after_alloc"""
update_stage_after_alloc
"""
return (
self.num_lmcache_hit_blocks > self.num_vllm_hit_blocks
and self.state != LMCacheMPRequestState.READY
@@ -242,7 +240,8 @@ class LMCacheMPRequestTracker:
def is_ready_for_retrieving(self) -> bool:
"""Check whether the current request is ready for retrieving,
will be used in process_loading_requests"""
will be used in process_loading_requests
"""
return (
self.state == LMCacheMPRequestState.WAITING_FOR_LOAD
and self.needs_retrieve()
@@ -301,13 +300,13 @@ class LMCacheMPRequestMetadata:
blocks_in_chunk: int,
vllm_block_size: int,
) -> "LMCacheMPRequestMetadata | None":
"""
Generate the store metadata for the current request tracker.
"""Generate the store metadata for the current request tracker.
Args:
tracker: The request tracker to generate the metadata from.
blocks_in_chunk: the number of blocks in a LMCache data chunk
vllm_block_size: the block size used in vLLM
"""
# Store the blocks that has block hashes
# NOTE: the invariant here is that `num_stored_blocks` should
@@ -375,13 +374,13 @@ class LMCacheMPRequestMetadata:
blocks_in_chunk: int,
vllm_block_size: int,
) -> "LMCacheMPRequestMetadata | None":
"""
Generate the retrieve metadata for the current request tracker.
"""Generate the retrieve metadata for the current request tracker.
Args:
tracker: The request tracker to generate the metadata from.
blocks_in_chunk: the number of blocks in a LMCache data chunk
vllm_block_size: the block size used in vLLM
"""
if not tracker.is_ready_for_retrieving():
return None
@@ -462,8 +461,7 @@ class LMCacheMPConnectorMetadata(KVConnectorMetadata):
class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
"""
The connector for LMCache multi-process mode.
"""The connector for LMCache multi-process mode.
Extra configs (kv_transfer_config.extra_config):
- lmcache.mp.host: the host of the LMCache server.
@@ -538,27 +536,26 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
Returns:
ConnectorMetadata: the connector metadata.
"""
"""
# Should only be called while set to valid metadata.
assert self._connector_metadata is not None
return self._connector_metadata
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
"""
Initialize with the KV caches. Useful for pre-registering the
"""Initialize with the KV caches. Useful for pre-registering the
KV Caches in the KVConnector (e.g. for NIXL).
Args:
kv_caches: dictionary of layer names, kv cache
"""
logger.info("Registering kv caches!")
self.worker_adapter.register_kv_caches(kv_caches)
return
def start_load_kv(self, forward_context: "ForwardContext", **kwargs: Any) -> None:
"""
Start loading the KV cache from the connector to vLLM's paged
"""Start loading the KV cache from the connector to vLLM's paged
KV buffer. This is called from the forward context before the
forward pass to enable async loading during model execution.
@@ -597,8 +594,7 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
)
def wait_for_layer_load(self, layer_name: str) -> None:
"""
Block until the KV for a specific layer is loaded into vLLM's
"""Block until the KV for a specific layer is loaded into vLLM's
paged buffer. This is called from within attention layer to ensure
async copying from start_load_kv is complete.
@@ -606,6 +602,7 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
Args:
layer_name: the name of that layer
"""
return
@@ -616,8 +613,7 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
attn_metadata: AttentionMetadata,
**kwargs: Any,
) -> None:
"""
Start saving a layer of KV cache from vLLM's paged buffer
"""Start saving a layer of KV cache from vLLM's paged buffer
to the connector. This is called from within attention layer to
enable async copying during execution.
@@ -627,12 +623,12 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
layer in vLLM.
attn_metadata (AttentionMetadata): the attention metadata.
**kwargs: additional arguments for the save operation.
"""
return
def wait_for_save(self):
"""
Block until all the save operations is done. This is called
"""Block until all the save operations is done. This is called
as the forward context exits to ensure that the async saving
from save_kv_layer is complete before finishing the forward.
@@ -673,8 +669,7 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
def get_finished(
self, finished_req_ids: set[str]
) -> tuple[set[str] | None, set[str] | None]:
"""
Notifies worker-side connector ids of requests that have
"""Notifies worker-side connector ids of requests that have
finished generating tokens on the worker.
The scheduler process (via the Executors) will use this output
to track which workers are done.
@@ -685,14 +680,14 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
tuple of (sending/saving ids, recving/loading ids).
The finished saves/sends req ids must belong to a set provided in a
call to this method (this call or a prior one).
"""
val = self.worker_adapter.get_finished(finished_req_ids)
# logger.error("Finished req ids: %s, %s", val[0], val[1])
return val
def get_block_ids_with_load_errors(self) -> set[int]:
"""
Get the set of block IDs that failed to load.
"""Get the set of block IDs that failed to load.
Returns:
Set of block IDs that encountered load errors.
@@ -707,12 +702,12 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
block IDs must appear here no later than that same pass.
- Sync loading: failed blocks should be reported in the forward
pass in which they are detected.
"""
return self.worker_adapter.get_block_ids_with_load_errors()
def shutdown(self):
"""
Shutdown the connector. This is called when the worker process
"""Shutdown the connector. This is called when the worker process
is shutting down to ensure that all the async operations are
completed and the connector is cleaned up properly.
"""
@@ -721,9 +716,7 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
return None
def get_kv_connector_stats(self) -> "KVConnectorStats | None":
"""
Get the KV connector stats collected during the last interval.
"""
"""Get the KV connector stats collected during the last interval."""
return None
# ==============================
@@ -735,8 +728,7 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
request: "Request",
num_computed_tokens: int,
) -> tuple[int | None, bool]:
"""
Get number of new tokens that can be loaded from the
"""Get number of new tokens that can be loaded from the
external KV cache beyond the num_computed_tokens.
Args:
@@ -761,6 +753,7 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
call. If the cache cannot be loaded for some tokens (e.g., due to
connectivity issues or eviction), those tokens must not be taken
into account.
"""
tracker = self._get_or_create_request_tracker(request)
# TODO: support loading KV for preempted requests in the future
@@ -803,8 +796,7 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
def update_state_after_alloc(
self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int
):
"""
Update KVConnector state after block allocation.
"""Update KVConnector state after block allocation.
If get_num_new_matched_tokens previously returned True for a
request, this function may be called twice for that same request -
@@ -817,6 +809,7 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
blocks (KVCacheBlocks): the blocks allocated for the request.
num_external_tokens (int): the number of tokens that will be
loaded from the external KV cache.
"""
# NOTE: `blocks` comes from kv_cache_manager.get_blocks(request_id),
# which returns ALL blocks for the request (not just newly allocated).
@@ -883,14 +876,14 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
def build_connector_meta(
self, scheduler_output: SchedulerOutput
) -> KVConnectorMetadata:
"""
Build the connector metadata for this step.
"""Build the connector metadata for this step.
This function should NOT modify fields in the scheduler_output.
Also, calling this function will reset the state of the connector.
Args:
scheduler_output (SchedulerOutput): the scheduler output object.
"""
metadata = LMCacheMPConnectorMetadata()
@@ -907,12 +900,12 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
return metadata
def update_connector_output(self, connector_output: KVConnectorOutput):
"""
Update KVConnector state from worker-side connectors output.
"""Update KVConnector state from worker-side connectors output.
Args:
connector_output (KVConnectorOutput): the worker-side
connectors output.
"""
return
@@ -921,8 +914,7 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
request: "Request",
block_ids: list[int],
) -> tuple[bool, dict[str, Any] | None]:
"""
Called exactly once when a request has finished, before its blocks are
"""Called exactly once when a request has finished, before its blocks are
freed.
The connector may assumes responsibility for freeing the blocks
@@ -934,8 +926,8 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
get_finished().
Optional KVTransferParams to be included in the request outputs
returned by the engine.
"""
"""
params: dict[str, Any] | None = getattr(request, "kv_transfer_params", None)
return_params: dict[str, Any] | None = {} if params is not None else None
@@ -962,26 +954,26 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
return True, return_params
def take_events(self) -> Iterable["KVCacheEvent"]:
"""
Take the KV cache events from the connector.
"""Take the KV cache events from the connector.
Yields:
New KV cache events since the last call.
"""
return ()
@classmethod
def get_required_kvcache_layout(cls, vllm_config: "VllmConfig") -> str | None:
"""
Get the required KV cache layout for this connector.
"""Get the required KV cache layout for this connector.
Args:
vllm_config (VllmConfig): the vllm config.
Returns:
str: the required KV cache layout. e.g. HND, or NHD.
None if the connector does not require a specific layout.
"""
"""
if cls is KVConnectorBase_V1:
raise TypeError(
"get_required_kvcache_layout should not be called "
@@ -990,13 +982,13 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
return None
def get_finished_count(self) -> int | None:
"""
Get the count of requests expected to complete send/receive operations
"""Get the count of requests expected to complete send/receive operations
via this connector. This method is used to initialize the
KVOutputAggregator, overwriting the default world_size.
Returns:
int: expected sending or receiving completion count.
"""
return None
@@ -1004,8 +996,7 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
def build_kv_connector_stats(
cls, data: dict[str, Any] | None = None
) -> "KVConnectorStats | None":
"""
KVConnectorStats resolution method. This method allows dynamically
"""KVConnectorStats resolution method. This method allows dynamically
registered connectors to return their own KVConnectorStats object,
which can implement custom aggregation logic on the data dict.
"""
@@ -1019,8 +1010,7 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
labelnames: list[str],
per_engine_labelvalues: dict[int, list[object]],
) -> "KVConnectorPromMetrics | None":
"""
Create a KVConnectorPromMetrics subclass which should register
"""Create a KVConnectorPromMetrics subclass which should register
per-connector Prometheus metrics and implement observe() to
expose connector transfer stats via Prometheus.
"""
@@ -1183,8 +1173,7 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
return self.request_trackers[request_id]
def _cleanup_request_tracker(self, request_id: str) -> None:
"""
Clean up request tracker and associated lookup future for a request.
"""Clean up request tracker and associated lookup future for a request.
This should be called when a request is finished to prevent memory leak.
"""
# Clean up request tracker
@@ -17,8 +17,7 @@ logger = init_logger(__name__)
@dataclass
class KVConnectorStats:
"""
Base class for KV Connector Stats, a container for transfer performance
"""Base class for KV Connector Stats, a container for transfer performance
metrics or otherwise important telemetry from the connector.
All sub-classes need to be serializable as stats are sent from worker to
logger process.
@@ -35,14 +34,11 @@ class KVConnectorStats:
raise NotImplementedError
def aggregate(self, other: "KVConnectorStats") -> "KVConnectorStats":
"""
Aggregate stats with another `KVConnectorStats` object.
"""
"""Aggregate stats with another `KVConnectorStats` object."""
raise NotImplementedError
def reduce(self) -> dict[str, int | float]:
"""
Reduce the observations collected during a time interval to one or
"""Reduce the observations collected during a time interval to one or
more representative values (eg avg/median/sum of the series).
This is meant to be called by the logger to produce a summary of the
stats for the last time interval.
@@ -95,7 +91,7 @@ class KVConnectorLogging:
)
def log(self, log_fn=logger.info):
"""Log transfer metrics periodically, similar to throughput logging"""
"""Log transfer metrics periodically, similar to throughput logging."""
if (
self.transfer_stats_accumulator
and not self.transfer_stats_accumulator.is_empty()
@@ -111,8 +107,7 @@ class KVConnectorLogging:
class KVConnectorPromMetrics:
"""
A base class for per-connector Prometheus metric registration
"""A base class for per-connector Prometheus metric registration
and recording.
"""
@@ -131,8 +126,7 @@ class KVConnectorPromMetrics:
self.per_engine_labelvalues = per_engine_labelvalues
def observe(self, transfer_stats_data: dict[str, Any], engine_idx: int = 0):
"""
Record the supplied transfer statistics to Prometheus metrics. These
"""Record the supplied transfer statistics to Prometheus metrics. These
statistics are engine-specific, and should be recorded to a metric
with the appropriate 'engine' label. These metric instances can be
created using the create_metric_per_engine() helper method.
@@ -141,8 +135,7 @@ class KVConnectorPromMetrics:
class KVConnectorProm:
"""
Support for registering per-connector Prometheus metrics, and
"""Support for registering per-connector Prometheus metrics, and
recording transfer statistics to those metrics. Uses
KVConnectorBase.build_prom_metrics().
"""
@@ -643,7 +643,7 @@ class MooncakeConnector(KVConnectorBase_V1, SupportsHMA):
class MooncakeConnectorScheduler:
"""Implementation of Scheduler side methods"""
"""Implementation of Scheduler side methods."""
def __init__(
self,
@@ -718,7 +718,8 @@ class MooncakeConnectorScheduler:
def _get_remote_prefill_token_count(self, num_prompt_tokens: int) -> int:
"""D-side only. Returns N-1 for Mamba models since the decoder
always recomputes the last token and must start from h(N-1)."""
always recomputes the last token and must start from h(N-1).
"""
if self._has_mamba and num_prompt_tokens > 1:
return num_prompt_tokens - 1
return num_prompt_tokens
@@ -729,7 +730,8 @@ class MooncakeConnectorScheduler:
derive h(N) correctly.
Guarded by ``_p_side_truncated`` to avoid repeated truncation if the
request is preempted and rescheduled."""
request is preempted and rescheduled.
"""
params = request.kv_transfer_params
if (
params is not None
@@ -756,8 +758,7 @@ class MooncakeConnectorScheduler:
def get_num_new_matched_tokens(
self, request: "Request", num_computed_tokens: int
) -> tuple[int, bool]:
"""
For remote prefill, pull all prompt blocks from remote
"""For remote prefill, pull all prompt blocks from remote
asynchronously relative to engine execution.
Args:
@@ -769,8 +770,8 @@ class MooncakeConnectorScheduler:
external KV cache beyond what is already computed.
* true if the external KV cache tokens will be loaded
asynchronously (between scheduler steps).
"""
"""
params = request.kv_transfer_params
logger.debug(
"MooncakeConnector get_num_new_matched_tokens: "
@@ -881,11 +882,9 @@ class MooncakeConnectorScheduler:
request: "Request",
block_ids: tuple[list[int], ...],
) -> tuple[bool, dict[str, Any] | None]:
"""
Once a request is finished, determine whether request blocks
"""Once a request is finished, determine whether request blocks
should be freed now or will be sent asynchronously and freed later.
"""
params = request.kv_transfer_params
logger.debug(
"MooncakeConnector request_finished, req_id=%s, request_status=%s, "
@@ -934,7 +933,7 @@ class MooncakeConnectorScheduler:
class MooncakeConnectorWorker:
"""Implementation of Worker side methods"""
"""Implementation of Worker side methods."""
def __init__(
self,
@@ -1176,11 +1175,9 @@ class MooncakeConnectorWorker:
raise e
async def _mooncake_sender_listener(self, ready_event: threading.Event):
"""
Background thread that listens for Mooncake requests, dispatches them
"""Background thread that listens for Mooncake requests, dispatches them
to a thread pool, and sends acknowledgments upon completion.
"""
sock = self.async_zmq_ctx.socket(zmq.ROUTER)
self.side_channel_port = sock.bind_to_random_port(f"tcp://{self.hostname}")
logger.debug(
@@ -1680,7 +1677,8 @@ class MooncakeConnectorWorker:
def _bind_sender_thread_device(self) -> None:
"""ThreadPoolExecutor initializer — binds each pool thread to the
correct CUDA device. CUDA device selection is thread-local, so
without this, NVLink transfers fail for TP ranks > 0."""
without this, NVLink transfers fail for TP ranks > 0.
"""
current_platform.set_device(self.device_id)
def _send_blocks(
@@ -1716,7 +1714,6 @@ class MooncakeConnectorWorker:
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
"""Register the KV Cache data in mooncake."""
logger.info("Registering KV_Caches. use_mla: %s", self.use_mla)
kv_data_ptrs: list[int] = []
@@ -1853,8 +1850,7 @@ class MooncakeConnectorWorker:
return finished_sending_reqs
def get_finished(self) -> tuple[set[str] | None, set[str] | None]:
"""
Get requests that are done sending or recving on this specific worker.
"""Get requests that are done sending or recving on this specific worker.
The scheduler process (via the MultiprocExecutor) will use this output
to track which workers are done.
"""
@@ -1885,8 +1881,7 @@ class MooncakeConnectorWorker:
return finished_sending_reqs or None, finished_recving_reqs or None
def get_transfer_results(self) -> KVConnectorTransferResults:
"""
Get transfers that completed on this specific worker, including
"""Get transfers that completed on this specific worker, including
requests whose remote KV load failed.
The scheduler process (via the MultiprocExecutor) will use this output
@@ -1909,7 +1904,8 @@ class MooncakeConnectorWorker:
def get_kv_connector_stats(self) -> KVConnectorStats | None:
"""Return transfer stats collected since the last call, or None
if nothing has been recorded in this interval."""
if nothing has been recorded in this interval.
"""
if self.xfer_stats.is_empty():
return None
return self.xfer_stats.clone_and_reset()
@@ -2321,8 +2317,7 @@ def should_launch_bootstrap_server(vllm_config: VllmConfig) -> bool:
def get_mooncake_bootstrap_addr(vllm_config: VllmConfig) -> tuple[str, int]:
"""
Returns the address of the Mooncake bootstrap server.
"""Returns the address of the Mooncake bootstrap server.
This is only used by prefillers to register workers.
Decoders should get addr from kv_transfer_params.
"""
@@ -32,8 +32,7 @@ class EngineEntry:
class MooncakeBootstrapServer:
"""
A centralized server running on the global rank 0 prefiller worker.
"""A centralized server running on the global rank 0 prefiller worker.
Prefiller workers register their connection info (IP, port, ranks) here.
"""
@@ -73,7 +73,8 @@ class ExternalCachedBlockPool:
class MooncakeStoreCoordinator:
"""Mirror of ``HybridKVCacheCoordinator.find_longest_cache_hit`` over an
``ExternalCachedBlockPool``."""
``ExternalCachedBlockPool``.
"""
def __init__(
self,
@@ -630,6 +630,7 @@ class ChunkedTokenDatabase:
``mask_num``. False entries are skipped before hash access.
put_step: Stride for distributing chunks across ranks.
put_step_rank: ``chunk_id % put_step`` value this rank stores.
"""
assert put_step > 0
if not block_hashes:
@@ -662,6 +663,7 @@ class TailKeyBoundary:
``block_hashes[num_tokens // hash_block_size - 1]`` instead of the
hash implied by ``MooncakeLookupResult.hit_length``. This changes
only the load key, not the reusable prefix.
"""
group_id: int
@@ -678,6 +680,7 @@ class MooncakeLookupResult:
tail_key_boundaries: Hash boundary used to store each cache group's
tail block when ``hit_length`` does not identify its store key.
There is one entry per group for every nonzero hit.
"""
hit_length: int
@@ -815,6 +815,7 @@ class KVCacheStoreSendingThread(KVTransferThread):
Returns:
True when no put is needed or every put succeeds, False otherwise.
"""
offloads = req_meta.boundary_state_offloads
if not offloads or not req_meta.block_hashes:
@@ -2628,7 +2629,8 @@ class LookupKeyClient:
non_block: bool = False,
) -> MooncakeLookupResult | None:
"""If non_block is True, will return None until the result is ready,
so the caller retries on a later step."""
so the caller retries on a later step.
"""
future = self.futures.get(req_id)
if future is None:
future = self.executor.submit(self._lookup, num_tokens, list(block_hashes))
@@ -588,8 +588,7 @@ class MoRIIOConnectorMetadata(KVConnectorMetadata):
@contextlib.contextmanager
def zmq_ctx(socket_type: Any, addr: str) -> Iterator[zmq.Socket]:
"""Context manager for a ZMQ socket"""
"""Context manager for a ZMQ socket."""
if socket_type not in (zmq.ROUTER, zmq.REQ, zmq.DEALER):
raise ValueError(f"Unexpected socket type: {socket_type}")
@@ -373,6 +373,7 @@ class MoRIIOConnector(KVConnectorBase_V1, SupportsHMA):
Returns:
bool: True if connector metadata exists, False otherwise.
"""
try:
return self._connector_metadata is not None
@@ -381,7 +382,7 @@ class MoRIIOConnector(KVConnectorBase_V1, SupportsHMA):
class MoRIIOConnectorScheduler:
"""Implementation of Scheduler side methods"""
"""Implementation of Scheduler side methods."""
def __init__(
self,
@@ -547,7 +548,8 @@ class MoRIIOConnectorScheduler:
self, request_id: ReqId, transfer_id: TransferId | None = None
):
"""Unmap request_id/transfer_id. Uses transfer_id for lookup if
exact request_id match fails (handles input_processor mutation)."""
exact request_id match fails (handles input_processor mutation).
"""
if request_id in self.request_id_to_transfer_id:
tid = self.request_id_to_transfer_id[request_id]
del self.request_id_to_transfer_id[request_id]
@@ -579,8 +581,7 @@ class MoRIIOConnectorScheduler:
request: "Request",
num_computed_tokens: int,
) -> tuple[int, bool]:
"""
For remote prefill, pull all prompt blocks from remote
"""For remote prefill, pull all prompt blocks from remote
asynchronously relative to engine execution.
Args:
@@ -592,6 +593,7 @@ class MoRIIOConnectorScheduler:
external KV cache beyond what is already computed.
* true if the external KV cache tokens will be loaded
asynchronously (between scheduler steps).
"""
if self.is_producer:
return 0, False
@@ -1007,11 +1009,9 @@ class MoRIIOConnectorScheduler:
request: "Request",
block_ids: BlockIds,
) -> tuple[bool, dict[str, Any] | None]:
"""
Once a request is finished, determine whether request blocks
"""Once a request is finished, determine whether request blocks
should be freed now or will be sent asynchronously and freed later.
"""
request_id = request.request_id
params = request.kv_transfer_params
# Consumer: can unmap transfer_id<->request_id immediately since done_recving
@@ -1177,7 +1177,7 @@ class MoRIIOConnectorScheduler:
class MoRIIOConnectorWorker:
"""Implementation of Worker side methods"""
"""Implementation of Worker side methods."""
def __init__(
self,
@@ -1435,8 +1435,8 @@ class MoRIIOConnectorWorker:
kv_layer: KV cache tensor
remote_notify_port: Port for completion notification
remote_ip: IP address of remote node
"""
"""
# synchronization to prevent dirty reads between
# transfer and attention operations
# we can consider removing this synchronization after ibgda is enabled.
@@ -1581,7 +1581,6 @@ class MoRIIOConnectorWorker:
layer_name_to_local_kv_cache_metadata: dict,
):
"""Background thread for getting new MoRIIO handshakes."""
encoder = msgspec.msgpack.Encoder()
encoded_data = encoder.encode(metadata)
size_in_bytes = len(encoded_data)
@@ -1637,7 +1636,6 @@ class MoRIIOConnectorWorker:
local-rank mapping _remote_tp_rank -- byte-identical for callers not yet
TP-aware.
"""
start_time = time.perf_counter()
# NOTE(rob): we need each rank to have a unique port. This is
@@ -1856,7 +1854,6 @@ class MoRIIOConnectorWorker:
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
"""Register the KV Cache data in moriio."""
self.kv_caches = kv_caches # layer name to kv cache
self.kv_cache_shapes = {
layer_name: kv_cache.shape for layer_name, kv_cache in kv_caches.items()
@@ -2020,12 +2017,10 @@ class MoRIIOConnectorWorker:
self.moriio_wrapper.async_wait_reqid()
def get_finished(self) -> tuple[set[str], set[str]]:
"""
Get requests that are done sending or recving on this specific worker.
"""Get requests that are done sending or recving on this specific worker.
The scheduler process (via the MultiprocExecutor) will use this output
to track which workers are done.
"""
done_sending, done_recving = set(), set()
if self.is_producer:
@@ -2416,8 +2411,7 @@ class MoRIIOConnectorWorker:
self._eager_handshaked_engines.add(remote_engine_id)
def start_load_kv(self, metadata: MoRIIOConnectorMetadata):
"""
Start loading by triggering non-blocking moriio_xfer.
"""Start loading by triggering non-blocking moriio_xfer.
We check for these trnxs to complete in each step().
"""
self.transfer_id_to_request_id = metadata.transfer_id_to_request_id
@@ -2686,8 +2680,13 @@ class MoRIIOConnectorWorker:
local_block_ids: IDs of local blocks
remote_block_ids: IDs of remote blocks
remote_moriio_meta: Metadata of the remote MoRIIO agent
remote_tp_size: Tensor parallel size of the remote engine. Falls
back to the locally recorded value when None.
remote_engine_id: Id of the remote engine the blocks live on.
Returns:
Tuple of (local_offsets, remote_offsets, transfer_sizes)
"""
validate_moriio_heterogeneous_tp_kv_heads(
local_tp_size=self.world_size,
@@ -87,6 +87,7 @@ class MoRIIOWriter:
Args:
worker: Reference to the parent worker
"""
self._worker_ref: weakref_ref[MoRIIOConnectorWorker] = weakref_ref(worker)
self._write_task_q: Queue[WriteTask] = Queue()
@@ -108,6 +109,7 @@ class MoRIIOWriter:
Raises:
RuntimeError: If worker has been garbage collected
"""
worker = self._worker_ref()
if worker is None:
@@ -131,6 +133,7 @@ class MoRIIOWriter:
Args:
task: The write task to schedule
"""
self.ensure_worker_started()
if self._is_transfer_terminal(task.transfer_id):
@@ -178,7 +181,6 @@ class MoRIIOWriter:
def _write_worker_loop(self) -> None:
"""Main loop for the write worker thread."""
while True:
# Process deferred tasks first
self._process_deferred_tasks()
@@ -275,6 +277,7 @@ class MoRIIOWriter:
Returns:
True if remote blocks are ready
"""
return (
task.transfer_id in self.worker.moriio_wrapper.done_remote_allocate_req_dict
@@ -291,6 +294,7 @@ class MoRIIOWriter:
Raises:
KeyError: If allocation info is missing
"""
try:
return self.worker.moriio_wrapper.done_remote_allocate_req_dict[transfer_id]
@@ -372,9 +376,11 @@ class MoRIIOWriter:
Args:
task: The write task
request_info: Remote allocation information
remote_moriio_meta: Metadata of the remote MoRIIO agent
Returns:
The transfer plan
"""
layer_cache = self.worker.kv_caches[task.layer_name]
key = (task.layer_name, *_get_write_geometry_key(layer_cache))
@@ -412,6 +418,7 @@ class MoRIIOWriter:
Args:
plan: The transfer plan
sessions: List of transfer sessions
"""
if plan.use_batch:
return [
@@ -510,6 +517,7 @@ class MoRIIOWrapper:
moriio_engine: MoRIIO engine instance
tp_rank: Tensor parallel rank
dp_rank: Data parallel rank
"""
def __init__(
@@ -71,8 +71,7 @@ class MultiKVConnectorWorkerMetadata(KVConnectorWorkerMetadata):
@dataclass
class MultiKVConnectorStats(KVConnectorStats):
"""
Maintain a dict of KVConnectorStats objects, one for each connector.
"""Maintain a dict of KVConnectorStats objects, one for each connector.
This is used to aggregate the stats from all connectors separately.
"""
@@ -132,8 +131,7 @@ class MultiKVConnectorPromMetrics(KVConnectorPromMetrics):
class MultiConnector(KVConnectorBase_V1, SupportsHMA):
"""
A wrapper for using multiple KVConnectors at the same time.
"""A wrapper for using multiple KVConnectors at the same time.
The current logic is:
- Load KV from the first connector that advertises available tokens from
@@ -143,8 +141,7 @@ class MultiConnector(KVConnectorBase_V1, SupportsHMA):
@classmethod
def requires_piecewise_for_cudagraph(cls, extra_config: dict[str, Any]) -> bool:
"""
MultiConnector requires PIECEWISE CUDA graph mode if any of its
"""MultiConnector requires PIECEWISE CUDA graph mode if any of its
child connectors require it.
"""
connectors_config = extra_config.get("connectors", [])
@@ -480,8 +477,7 @@ class MultiConnector(KVConnectorBase_V1, SupportsHMA):
connector_output.kv_connector_worker_meta = multi_connector_worker_meta
def get_handshake_metadata(self) -> KVConnectorHandshakeMetadata | None:
"""
Get the KVConnector handshake metadata from sub-connectors.
"""Get the KVConnector handshake metadata from sub-connectors.
Returns the first non-None metadata from sub-connectors.
"""
for c in self._connectors:
@@ -493,8 +489,7 @@ class MultiConnector(KVConnectorBase_V1, SupportsHMA):
def set_xfer_handshake_metadata(
self, metadata: dict[int, KVConnectorHandshakeMetadata]
) -> None:
"""
Set the KV connector handshake metadata for all sub-connectors.
"""Set the KV connector handshake metadata for all sub-connectors.
This is needed to start the NIXL listener thread for NixlConnector.
"""
for c in self._connectors:
@@ -590,14 +585,15 @@ class MultiConnector(KVConnectorBase_V1, SupportsHMA):
@classmethod
def get_required_kvcache_layout(cls, vllm_config: "VllmConfig") -> str | None:
"""
Get the required KV cache layout for this connector.
"""Get the required KV cache layout for this connector.
Args:
vllm_config (VllmConfig): the vllm config.
Returns:
str: the required KV cache layout. e.g. HND, or NHD.
None if the connector does not require a specific layout.
"""
assert vllm_config.kv_transfer_config is not None
layouts: set[str] = set()
@@ -299,11 +299,11 @@ class NixlBaseConnectorScheduler:
def set_xfer_handshake_metadata(
self, metadata: dict[tuple[int, int], KVConnectorHandshakeMetadata]
) -> None:
"""
Set the KV connector handshake metadata for this connector.
"""Set the KV connector handshake metadata for this connector.
Args:
metadata (dict): the handshake metadata to set.
"""
encoded_data: dict[tuple[int, int], bytes] = {}
encoder = msgspec.msgpack.Encoder()
@@ -400,7 +400,8 @@ class NixlBaseConnectorScheduler:
def _get_remote_prefill_token_count(self, num_prompt_tokens: int) -> int:
"""D-side only. The number of prompt tokens to load from the prefiller.
Stops short of the trailing ``_prefill_backoff()`` tokens that the decoder
will recompute locally."""
will recompute locally.
"""
backoff = self._prefill_backoff()
if backoff and num_prompt_tokens > backoff:
return num_prompt_tokens - backoff
@@ -413,7 +414,8 @@ class NixlBaseConnectorScheduler:
multi-module MTP it is the drafter's whole lookahead window.
Guarded by ``_p_side_truncated`` to avoid repeated truncation if the
request is preempted and rescheduled."""
request is preempted and rescheduled.
"""
backoff = self._prefill_backoff()
params = request.kv_transfer_params
if (
@@ -1147,8 +1147,7 @@ class NixlBaseConnectorWorker:
return self.nixl_wrapper.add_remote_agent(metadata.agent_metadata)
def initialize_host_xfer_buffer(self, kv_caches: dict[str, torch.Tensor]) -> None:
"""
Initialize transfer buffer in CPU mem for accelerators
"""Initialize transfer buffer in CPU mem for accelerators
NOT directly supported by NIXL (e.g., tpu)
"""
xfer_buffers: dict[str, torch.Tensor] = {}
@@ -1262,8 +1261,7 @@ class NixlBaseConnectorWorker:
pp_size: int = 1,
notif_agents_only: bool = False,
) -> Future[tuple[dict[tuple[int, int], str], float]] | None:
"""
Ensure a handshake is in-flight (or already done) for *engine_id*.
"""Ensure a handshake is in-flight (or already done) for *engine_id*.
Returns the ``Future`` if a handshake is pending (or was just
started), or ``None`` if the handshake already completed
@@ -1348,7 +1346,6 @@ class NixlBaseConnectorWorker:
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
"""Register the KV Cache data in nixl."""
self.transfer_topo = TransferTopology(
tp_rank=self.transfer_tp_rank,
tp_size=self.transfer_tp_size,
@@ -1876,7 +1873,8 @@ class NixlBaseConnectorWorker:
) -> np.ndarray:
"""Build remote desc regions (conv sub-projections + ssm) per layer.
For hetero-TP, each D rank reads only its sub-projection slice from
the P rank. Returns an Nx3 uint64 array."""
the P rank. Returns an Nx3 uint64 array.
"""
assert nixl_agent_meta.kv_caches_base_addr, (
"Remote KV cache base addresses must not be empty."
)
@@ -2034,8 +2032,7 @@ class NixlBaseConnectorWorker:
self,
block_size: int,
) -> tuple[int, np.ndarray]:
"""
Function used for register local xfer handler with local block_size or
"""Function used for register local xfer handler with local block_size or
Remote block_size.
When local block_size is same as remote block_size, we use local block_size
@@ -2124,8 +2121,7 @@ class NixlBaseConnectorWorker:
remote_tp_size: int = 1,
remote_dcp_size: int = 1,
) -> str:
"""
Add the remote NIXL agent and prepare the descriptors for reading cache
"""Add the remote NIXL agent and prepare the descriptors for reading cache
blocks from remote.
In particular, handle both homogeneous and heterogeneous TP. The former
@@ -2395,8 +2391,7 @@ class NixlBaseConnectorWorker:
remote_tp_size: int,
remote_dcp_size: int = 1,
):
"""
Validate the remote agent handshake metadata ensuring the
"""Validate the remote agent handshake metadata ensuring the
invariants hold true.
"""
remote_engine_id = nixl_agent_meta.engine_id
@@ -2607,7 +2602,7 @@ class NixlBaseConnectorWorker:
assert len(nixl_agent_meta.region_mem_types) == num_remote_regions
def sync_recved_kv_to_device(self, req_id: str, meta: ReqMeta):
"""copy recved kv from host buffer to device."""
"""Copy recved kv from host buffer to device."""
assert self.use_host_buffer
assert self.copy_blocks is not None
@@ -2632,7 +2627,7 @@ class NixlBaseConnectorWorker:
)
def save_kv_to_host(self, metadata: NixlConnectorMetadata):
"""copy kv from device to host buffer."""
"""Copy kv from device to host buffer."""
assert self.use_host_buffer
assert self.copy_blocks is not None
@@ -2662,7 +2657,8 @@ class NixlBaseConnectorWorker:
@cached_property
def _attention_kv_caches(self) -> list[torch.Tensor]:
"""Device KV caches of attention layers (mamba states excluded),
as consumed by the receive post-process."""
as consumed by the receive post-process.
"""
assert self.device_kv_caches, (
"_attention_kv_caches accessed before register_kv_caches"
)
@@ -2681,8 +2677,7 @@ class NixlBaseConnectorWorker:
block_ids_list: list[tuple[list[int], int]],
convert: bool = True,
):
"""
Post process device kv cache after receiving from remote.
"""Post process device kv cache after receiving from remote.
3 types of conversion supported (``convert``):
* kv_cache_postprocess_layout => convert from HND to NHD
@@ -2771,8 +2766,7 @@ class NixlBaseConnectorWorker:
def post_process_device_kv_on_receive_heterogeneous_attn(
self, block_ids: list[int]
):
"""
Post process device kv cache after receiving from remote
"""Post process device kv cache after receiving from remote
for heterogeneous attention.
"""
assert self.enable_heterogeneous_attn_post_process
@@ -2805,8 +2799,7 @@ class NixlBaseConnectorWorker:
pages[block].zero_()
def get_transfer_results(self) -> KVConnectorTransferResults:
"""
Get transfers that completed on this specific worker.
"""Get transfers that completed on this specific worker.
The scheduler process (via the MultiprocExecutor) will use this output
to track which workers are done.
@@ -2984,6 +2977,7 @@ class NixlBaseConnectorWorker:
Args:
payload: comma-separated P-side request IDs, e.g.
"req_abc,req_def".
"""
new_expiry = time.perf_counter() + self._lease_extension
for req_id in payload.split(","):
@@ -3010,6 +3004,7 @@ class NixlBaseConnectorWorker:
Returns:
Requests with no outstanding handles and requests with observed
failures. Failed requests may still have outstanding handles.
"""
done_req_ids: set[str] = set()
failed_req_ids: set[str] = set()
@@ -3091,9 +3086,7 @@ class NixlBaseConnectorWorker:
self.xfer_stats.record_failed_notification()
def _send_heartbeats(self, metadata: NixlConnectorMetadata) -> None:
"""
Send heartbeat notifications to remote engines, extending lease on KV blocks.
"""
"""Send heartbeats to remote engines, extending the lease on KV blocks."""
for engine_id, hb_info in metadata.heartbeat_by_engine.items():
# Proactive handshake (this request may still be in waiting queue) so
# the **next** heartbeat for this remote can go through.
@@ -3126,8 +3119,7 @@ class NixlBaseConnectorWorker:
def get_mapped_blocks(
self, block_ids: np.ndarray, block_size_ratio: int
) -> np.ndarray:
"""
Calculates the new set of block IDs by mapping every element
"""Calculates the new set of block IDs by mapping every element
in the (potentially sparse) input array.
Example: block_ids=[0, 2], block_size_ratio=2
get_mapped_blocks 0 1 [2 3] 4 5
@@ -3184,8 +3176,7 @@ class NixlBaseConnectorWorker:
return mapped_local, mapped_remote
def _logical_to_kernel_block_ids(self, block_ids: BlockIds, ratio: int) -> BlockIds:
"""
Convert block ids to kernel physical block ids.
"""Convert block ids to kernel physical block ids.
This is required when the logical block size (the one set by the user)
does not match the one required by the attn backend.
`ratio` is the number of physical blocks per logical block.
@@ -3239,6 +3230,7 @@ class NixlBaseConnectorWorker:
An empty result (rank 0 above) takes the notification-only path, allowing the
remote to release its blocks without performing a transfer.
"""
local_size, remote_size = local_dcp_size, remote_dcp_size
@@ -3417,17 +3409,14 @@ class NixlBaseConnectorWorker:
return decode_block_ids, prefill_block_ids
def get_kv_connector_stats(self) -> KVConnectorStats | None:
"""
Get the KV transfer stats for the connector.
"""
"""Get the KV transfer stats for the connector."""
# Clear stats for next iteration
if not self.xfer_stats.is_empty():
return self.xfer_stats.clone_and_reset()
return None
def get_block_ids_with_load_errors(self) -> set[int]:
"""
Return and clear the set of block IDs that failed to load.
"""Return and clear the set of block IDs that failed to load.
This is called by the scheduler to identify blocks that need
to be retried after a NIXL transfer failure.
@@ -214,12 +214,12 @@ class NixlBaseConnector(KVConnectorBase_V1, SupportsHMA):
def set_xfer_handshake_metadata_pp_aware(
self, metadata: dict[tuple[int, int], KVConnectorHandshakeMetadata]
) -> None:
"""
Set handshake metadata keyed by (pp_rank, tp_rank) so the side
"""Set handshake metadata keyed by (pp_rank, tp_rank) so the side
channel can serve every PP stage's agent metadata.
Args:
metadata (dict): the handshake metadata to set.
"""
assert self.connector_scheduler is not None
self.connector_scheduler.set_xfer_handshake_metadata(metadata)
@@ -317,14 +317,14 @@ class NixlBaseConnector(KVConnectorBase_V1, SupportsHMA):
self.connector_scheduler.shutdown()
def get_handshake_metadata(self) -> KVConnectorHandshakeMetadata | None:
"""
Get the KVConnector handshake metadata for this connector.
"""Get the KVConnector handshake metadata for this connector.
This metadata is used for out-of-band connector handshake
between P/D workers.
Returns:
KVConnectorHandshakeMetadata: the handshake metadata.
None if no handshake metadata is available.
"""
assert self.connector_worker is not None
if (
@@ -78,8 +78,7 @@ class NixlAgentMetadata:
@dataclass
class NixlHandshakePayload(KVConnectorHandshakeMetadata):
"""
Wrapper for NIXL handshake sent over the wire.
"""Wrapper for NIXL handshake sent over the wire.
Enables two-phase decoding for graceful compatibility checking:
1. Decode NixlHandshakePayload to get compatibility_hash
@@ -142,8 +141,7 @@ def compute_nixl_compatibility_hash(
attn_backend_name: str,
transfer_mode: str = "pull",
) -> str:
"""
Compute compatibility hash for NIXL KV transfer.
"""Compute compatibility hash for NIXL KV transfer.
Hash only the factors that affect whether two NIXL instances can
successfully transfer KV cache data.
@@ -169,6 +167,7 @@ def compute_nixl_compatibility_hash(
Returns:
SHA-256 hex digest
"""
from vllm import __version__ as vllm_version
from vllm.config.utils import hash_factors
@@ -34,8 +34,7 @@ class NixlPullConnectorScheduler(NixlBaseConnectorScheduler):
def get_num_new_matched_tokens(
self, request: "Request", num_computed_tokens: int
) -> tuple[int, bool]:
"""
For remote prefill, pull all prompt blocks from remote
"""For remote prefill, pull all prompt blocks from remote
asynchronously relative to engine execution.
Args:
@@ -47,8 +46,8 @@ class NixlPullConnectorScheduler(NixlBaseConnectorScheduler):
external KV cache beyond what is already computed.
* true if the external KV cache tokens will be loaded
asynchronously (between scheduler steps).
"""
"""
params = request.kv_transfer_params
logger.debug(
"NIXLConnector get_num_new_matched_tokens: "
@@ -194,8 +193,7 @@ class NixlPullConnectorScheduler(NixlBaseConnectorScheduler):
request: "Request",
block_ids: "BlockIds",
) -> tuple[bool, dict[str, Any] | None]:
"""
Once a request is finished, determine whether request blocks
"""Once a request is finished, determine whether request blocks
should be freed now or will be sent asynchronously and freed later.
"""
from vllm.v1.request import RequestStatus
@@ -45,8 +45,7 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
super().__init__(vllm_config, engine_id, kv_cache_config)
def start_load_kv(self, metadata: NixlConnectorMetadata):
"""
Start loading by triggering non-blocking nixl_xfer.
"""Start loading by triggering non-blocking nixl_xfer.
We check for these trnxs to complete in each step().
"""
for req_id, meta in metadata.reqs_to_recv.items():
@@ -373,8 +372,7 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
expected_consumers: int,
awaiting_kvs: bool,
) -> bool:
"""
Post a READ point-to-point xfer request from a single local worker to
"""Post a READ point-to-point xfer request from a single local worker to
a single remote worker.
Returns True when the read was posted (or was unnecessary), False
@@ -603,8 +601,7 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
self._recving_transfers[request_id].append(handle)
def _get_new_notifs(self) -> set[str]:
"""
Get req_ids which got a remote xfer message. When multiple consumers
"""Get req_ids which got a remote xfer message. When multiple consumers
are reading from the same producer (heterogeneous TP or DCP
scenario), wait for all consumers to be done pulling.
@@ -315,7 +315,8 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
executes on the writer; the handshake runs on the background executor
and the request is re-queued onto ``_reg_send_inbox`` once it
completes (at which point ``_ensure_handshake`` returns ``None`` and we
send directly)."""
send directly).
"""
remote_pp_size = reg_data.get("remote_pp_size", 1)
fut = self._ensure_handshake(
reg_data["remote_engine_id"],
@@ -507,7 +508,8 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
``BlockIds`` is canonically a tuple of per-group lists, but some
registration payloads collapse a single-group case to a flat
list. Re-wrap that case so downstream group-aware helpers see a
consistent shape."""
consistent shape.
"""
if block_ids and not isinstance(block_ids[0], (list, tuple)):
return (list(block_ids),)
return block_ids
@@ -23,7 +23,7 @@ if TYPE_CHECKING:
@dataclass
class NixlKVConnectorStats(KVConnectorStats):
"""Container for transfer performance metrics"""
"""Container for transfer performance metrics."""
def __post_init__(self):
if not self.data:
@@ -34,8 +34,7 @@ _NIXL_SUPPORTED_DEVICE.update(current_platform.get_nixl_supported_devices())
# TODO: merge with vllm.utils.network_utils.zmq_socket_ctx
@contextlib.contextmanager
def zmq_ctx(socket_type: Any, addr: str) -> Iterator[zmq.Socket]:
"""Context manager for a ZMQ socket"""
"""Context manager for a ZMQ socket."""
if socket_type not in (zmq.ROUTER, zmq.REQ):
raise ValueError(f"Unexpected socket type: {socket_type}")
@@ -37,7 +37,8 @@ def canonical_format_id(kv_cache_layout: str) -> str:
Canonical pages keep the worker's KV layout family, so the id couples the
format version with that family; consumers must match it exactly. The family
keeps its historical NHD/HND spelling so ids stay stable for KV persisted
before the layout enum existed."""
before the layout enum existed.
"""
layout = KVCacheLayout[kv_cache_layout]
legacy = {KVCacheLayout.LBNHC: "nhd", KVCacheLayout.LBHNC: "hnd"}
family = legacy.get(layout, layout.name.lower())
@@ -121,7 +122,8 @@ def _interleave_cp_tokens(
ctx: _RankContext,
) -> tuple[CopyRun, ...]:
"""Place each region's num_tokens rows at their canonical token positions,
one run per chunk of interleaved tokens."""
one run per chunk of interleaved tokens.
"""
runs: list[CopyRun] = []
for region in regions:
if ctx.cp_size == 1:
@@ -256,7 +258,8 @@ def _attention_byte_regions(
cp_size: int,
) -> list[ByteRegion] | None:
"""Byte regions of an attention page, given this rank's head shard.
None when the physical layout is not recognized (fail closed)."""
None when the physical layout is not recognized (fail closed).
"""
bs, heads, head_size = spec.block_size, spec.num_kv_heads, spec.head_size
if tuple(kv_cache.shape) == (num_blocks, heads, bs, 2 * head_size):
return _packed_kv_regions(kv_cache, spec, head_shard, num_head_shards, cp_size)
@@ -370,7 +373,8 @@ def _is_exact_partition(intervals: list[tuple[int, int]], size: int) -> bool:
def _verify_tiling(layer_name: str, per_rank: list[CanonicalPageMapping]) -> None:
"""Whichever ranks a block elects as writers must tile the canonical page
exactly once, and each rank's runs must cover exactly its local page."""
exactly once, and each rank's runs must cover exactly its local page.
"""
size = per_rank[0].canonical_page_size_bytes
num_writers = per_rank[0].num_writers
for mapping in per_rank:
@@ -79,7 +79,8 @@ def get_offloading_event_group_spec(
class _OffloadEventMetadata:
"""BlockStored payload snapshot for one OffloadKey, captured while the
Request is available and kept until the final matching removal event.
``medium`` and ``ownership`` are forwarded from the OffloadingEvent."""
``medium`` and ``ownership`` are forwarded from the OffloadingEvent.
"""
# The chunk's constituent block hashes; the last one is the OffloadKey.
block_hashes: tuple[BlockHash, ...]
@@ -245,6 +246,7 @@ class OffloadingEventsTracker:
Yields:
``BlockStored`` or ``BlockRemoved`` events corresponding to
the underlying :class:`OffloadingEvent` stream.
"""
for event in events:
if event.removed:
@@ -254,7 +256,8 @@ class OffloadingEventsTracker:
def reset(self) -> None:
"""Drop all tracked state; pending payloads are stale after a
manager cache reset."""
manager cache reset.
"""
self._pending_event_metadata.clear()
def _build_event_metadata(
@@ -265,7 +268,8 @@ class OffloadingEventsTracker:
) -> _OffloadEventMetadata:
"""Build the payload snapshot for one offloaded chunk: its
constituent per-block hashes, the whole chunk's tokens, and the
per-block ``block_size``."""
per-block ``block_size``.
"""
hashes_per_chunk = group_config.hashes_per_chunk
assert hashes_per_chunk > 0
assert chunk_idx >= 0
@@ -170,8 +170,7 @@ class _StatsKey:
@dataclass
class OffloadingConnectorStats(KVConnectorStats):
"""
Offloading connector stats use flat metric names as keys.
"""Offloading connector stats use flat metric names as keys.
The ``data`` dict is structured using ``_StatsKey`` / ``_MetricType``::
@@ -243,8 +242,7 @@ class OffloadingConnectorStats(KVConnectorStats):
return self
def reduce(self) -> dict[str, int | float]:
"""
Reduce the observations collected during a time interval to one or
"""Reduce the observations collected during a time interval to one or
more representative values (eg avg/median/sum of the series).
This is meant to be called by the logger to produce a summary of the
stats for the last time interval.
@@ -676,7 +676,8 @@ class OffloadingConnectorScheduler:
start_chunk_idx: int,
) -> int | None:
"""Return the number of consecutive offloaded chunks from the start,
or None if the backend deferred a lookup."""
or None if the backend deferred a lookup.
"""
hit_count = 0
defer_lookup = False
for local_idx, key in enumerate(keys):
@@ -711,7 +712,8 @@ class OffloadingConnectorScheduler:
"""Return the end index (in `keys`) of the last run of
`sliding_window_size` consecutive hits, scanning from the end.
The first run may need a larger window for a partial rightmost chunk.
Returns 0 on miss, None if the backend deferred a lookup."""
Returns 0 on miss, None if the backend deferred a lookup.
"""
defer_lookup = False
pending_in_window = False
consecutive_hits = 0
@@ -750,8 +752,7 @@ class OffloadingConnectorScheduler:
req_status: RequestOffloadState,
max_num_new_tokens: int | None = None,
) -> int | None:
"""
Find how many tokens beyond num_locally_computed_tokens can be loaded.
"""Find how many tokens beyond num_locally_computed_tokens can be loaded.
Iterates full-attention groups first (prefix lookup), then sliding-window
groups (suffix lookup). Each group may tighten max_hit_size_tokens, which
@@ -1039,14 +1040,15 @@ class OffloadingConnectorScheduler:
num_computed_tokens: int,
max_num_new_tokens: int | None = None,
) -> tuple[int | None, bool]:
"""
Get number of new tokens that can be loaded beyond the
"""Get number of new tokens that can be loaded beyond the
num_computed_tokens.
Args:
request (Request): the request object.
num_computed_tokens (int): the number of locally
computed tokens for this request
max_num_new_tokens (int | None): cap on the number of tokens that
may be loaded beyond `num_computed_tokens`, if any.
Returns:
A tuple with the following elements:
@@ -1057,6 +1059,7 @@ class OffloadingConnectorScheduler:
should query for this request again later.
- `True` if tokens will be loaded asynchronously
(between scheduler steps).
"""
req_status = self._req_status[request.request_id]
for group_state in req_status.group_states:
@@ -1208,10 +1211,7 @@ class OffloadingConnectorScheduler:
req_status.partial_tail_boundary = None
def _update_req_states(self, scheduler_output: SchedulerOutput) -> None:
"""
Update request states from the Scheduler's output.
"""
"""Update request states from the Scheduler's output."""
# new_block_ids_end[req_id][i] = end of pre-existing block_ids for
# the i-th sliding window group (before this step's extend).
# Used to detect sliding window blocks that got re-allocated.
@@ -1840,12 +1840,12 @@ class OffloadingConnectorScheduler:
return bool(self._jobs) or self.manager.has_pending_work()
def update_connector_output(self, connector_output: KVConnectorOutput):
"""
Update KVConnector state from worker-side connectors output.
"""Update KVConnector state from worker-side connectors output.
Args:
connector_output (KVConnectorOutput): the worker-side
connectors output.
"""
meta = connector_output.kv_connector_worker_meta
if not isinstance(meta, OffloadingWorkerMetadata):
@@ -1938,8 +1938,7 @@ class OffloadingConnectorScheduler:
self,
request: Request,
) -> tuple[bool, dict[str, Any] | None]:
"""
Called when a request has finished, before its blocks are freed.
"""Called when a request has finished, before its blocks are freed.
Returns:
True if the request is being saved/sent asynchronously and blocks
@@ -1947,6 +1946,7 @@ class OffloadingConnectorScheduler:
get_finished().
Optional KVTransferParams to be included in the request outputs
returned by the engine.
"""
req_status = self._req_status.get(request.request_id)
@@ -1985,12 +1985,12 @@ class OffloadingConnectorScheduler:
Yields:
``BlockStored`` or ``BlockRemoved`` events corresponding to
the underlying :class:`OffloadingEvent` stream.
"""
yield from self._events_tracker.take_events(self.manager.take_events())
def reset_cache(self) -> None:
"""Reset the offloading manager cache, evicting all stored chunks."""
# reset_cache cannot be called in the middle of a schedule step
assert not self._current_batch_load_jobs
assert not self._current_batch_jobs_to_flush
@@ -39,7 +39,7 @@ logger = init_logger(__name__)
class OffloadingConnectorWorker:
"""Implementation of Worker side methods"""
"""Implementation of Worker side methods."""
def __init__(
self,
@@ -268,14 +268,14 @@ class OffloadingConnectorWorker:
)
def get_finished(self, finished_req_ids: set[str]) -> tuple[set[str], set[str]]:
"""
Returns:
tuple of (finished_sending, finished_recving). Stores never
emit finished_sending — the scheduler tracks store completion
via kv_connector_worker_meta.completed_jobs and fences any
block reuse via jobs_to_flush. Loads still emit
finished_recving so the base scheduler can resume requests
blocked on remote KV (and free aborted-during-load reqs).
"""Returns:
tuple of (finished_sending, finished_recving). Stores never
emit finished_sending — the scheduler tracks store completion
via kv_connector_worker_meta.completed_jobs and fences any
block reuse via jobs_to_flush. Loads still emit
finished_recving so the base scheduler can resume requests
blocked on remote KV (and free aborted-during-load reqs).
"""
assert self.worker is not None
finished_recving: set[str] = set()
@@ -83,6 +83,7 @@ class MambaConvSplitInfo:
< 0: P_TP > D_TP — P pages are smaller, D reads entire
P page. Local dims are scaled down by |tp_ratio|
to get P-sized offsets.
"""
offsets: list[tuple[int, int]] = []
if tp_ratio >= 1:
@@ -121,6 +122,7 @@ def derive_mamba_conv_split(
Returns:
MambaConvSplitInfo with per-rank sub-projection dims, conv_rows,
conv_dtype_size, and ssm_sizes (conv_state_bytes, ssm_state_bytes).
"""
_supported = (
MambaAttentionBackendEnum.MAMBA1,
@@ -228,5 +230,6 @@ def compute_physical_blocks_per_logical(
Args:
ssm_sizes: (conv_state_bytes, ssm_state_bytes) from NixlAgentMetadata.
block_len: the engine's block_len in bytes (from block_lens[0]).
"""
return math.ceil((ssm_sizes[0] + ssm_sizes[1]) / block_len)
@@ -38,6 +38,7 @@ def is_v1_kv_transfer_group(connector: KVConnectorBaseType | None = None) -> boo
Note:
This function will no-longer be needed after the v1 KV connector
becomes the default.
"""
if connector is None:
connector = _KV_CONNECTOR_AGENT
@@ -72,10 +73,7 @@ def _sync_engine_id_across_tp(vllm_config: "VllmConfig") -> None:
def ensure_kv_transfer_initialized(
vllm_config: "VllmConfig", kv_cache_config: "KVCacheConfig"
) -> None:
"""
Initialize KV cache transfer parallel group.
"""
"""Initialize KV cache transfer parallel group."""
global _KV_CONNECTOR_AGENT
if _KV_CONNECTOR_AGENT is not None: