Add pydocstyle to the ruff rules (#52136)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Signed-off-by: Harry Mellor <hej.mellor@gmail.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
804e5377aa
commit
8c1557a79c
@@ -87,6 +87,7 @@ class KVConnectorFactory:
|
||||
|
||||
Returns:
|
||||
The connector class.
|
||||
|
||||
"""
|
||||
if connector_name not in cls._registry:
|
||||
raise ValueError(f"Connector '{connector_name}' is not registered.")
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
KV cache helper for store.
|
||||
"""
|
||||
"""KV cache helper for store."""
|
||||
|
||||
from collections.abc import Iterator
|
||||
from dataclasses import dataclass
|
||||
@@ -53,7 +51,8 @@ def get_kv_connector_cache_layout(vllm_config: VllmConfig | None = None):
|
||||
|
||||
class KVOutputAggregator:
|
||||
"""Utility class to aggregate the output of all workers into a single
|
||||
output corresponding to Rank 0 for scheduler."""
|
||||
output corresponding to Rank 0 for scheduler.
|
||||
"""
|
||||
|
||||
def __init__(self, expected_finished_count: int):
|
||||
# Complete transfer tracker. Used to track finished requests
|
||||
@@ -233,11 +232,10 @@ def copy_kv_blocks(
|
||||
|
||||
|
||||
def kv_postprocess_blksize_on_receive(cache, indices, block_size_ratio):
|
||||
"""
|
||||
Transforms the layout of received KV cache blocks to the local block_size.
|
||||
"""Transforms the layout of received KV cache blocks to the local block_size.
|
||||
(Only works for local blocksize > remote blocksize)
|
||||
|
||||
example:
|
||||
Example:
|
||||
local blocksize = 16 tokens, remote blocksize = 4 tokens
|
||||
local block[0] = remote block[0, 1, 2, 3]
|
||||
remote is |h0-b0|h1-b0|h2-b0|h3-b0|h0-b1|h1-b1|h2-b1|h3-b1|...
|
||||
@@ -246,6 +244,7 @@ def kv_postprocess_blksize_on_receive(cache, indices, block_size_ratio):
|
||||
1. view => view remote as n_blocks * remote_shape(H,remoteN,D)
|
||||
2. permute => (H, nblocks, remoteN, D)
|
||||
3. flatten => (H, localN, D)
|
||||
|
||||
"""
|
||||
blocks_to_update = cache.index_select(0, indices)
|
||||
# use physical order
|
||||
@@ -293,8 +292,7 @@ def kv_postprocess_layout_on_receive(cache, indices):
|
||||
|
||||
|
||||
def kv_postprocess_blksize_and_layout_on_receive(cache, indices, block_size_ratio):
|
||||
"""
|
||||
Transforms the layout of received KV cache to the local block_size and LBHNC.
|
||||
"""Transforms the layout of received KV cache to the local block_size and LBHNC.
|
||||
(Only works for local blocksize > remote blocksize)
|
||||
|
||||
prefill is LBHNC, smaller block_size
|
||||
@@ -317,9 +315,9 @@ def kv_postprocess_blksize_and_layout_on_receive(cache, indices, block_size_rati
|
||||
def yield_req_data(
|
||||
scheduler_output,
|
||||
) -> Iterator[tuple[str, tuple[list[int], ...] | None, bool]]:
|
||||
"""
|
||||
Yields:
|
||||
(req_id, new_block_id_groups, preempted)
|
||||
"""Yields:
|
||||
(req_id, new_block_id_groups, preempted)
|
||||
|
||||
"""
|
||||
# new requests
|
||||
for req_data in scheduler_output.scheduled_new_reqs:
|
||||
@@ -346,6 +344,7 @@ def get_current_attn_backends(
|
||||
|
||||
Returns:
|
||||
Deduplicated list of attention backend classes.
|
||||
|
||||
"""
|
||||
layer_type = cast(type[Any], AttentionLayerBase)
|
||||
layers = get_layers_from_vllm_config(vllm_config, layer_type, layer_names)
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
KVConnectorBase_V1 Class for Distributed KV Cache & Hidden State
|
||||
"""KVConnectorBase_V1 Class for Distributed KV Cache & Hidden State
|
||||
communication in vLLM v1
|
||||
|
||||
The class provides the following primitives:
|
||||
@@ -97,8 +96,7 @@ class KVConnectorTransferResults:
|
||||
|
||||
|
||||
class SupportsHMA(ABC):
|
||||
"""
|
||||
The class that indicates the corresponding connector supports hybrid memory
|
||||
"""The class that indicates the corresponding connector supports hybrid memory
|
||||
allocator (HMA).
|
||||
This is required to use the connector together with hybrid memory allocator.
|
||||
"""
|
||||
@@ -109,8 +107,7 @@ class SupportsHMA(ABC):
|
||||
request: "Request",
|
||||
block_ids: tuple[list[int], ...],
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
"""
|
||||
Called exactly once when a request has finished for all kv cache groups,
|
||||
"""Called exactly once when a request has finished for all kv cache groups,
|
||||
before its blocks are freed for each group.
|
||||
|
||||
NOTE(Kuntai): This function is only supported by connectors that support HMA.
|
||||
@@ -124,6 +121,7 @@ class SupportsHMA(ABC):
|
||||
get_finished().
|
||||
Optional KVTransferParams to be included in the request outputs
|
||||
returned by the engine.
|
||||
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -144,8 +142,7 @@ class KVConnectorRole(enum.Enum):
|
||||
|
||||
|
||||
class KVConnectorHandshakeMetadata(ABC): # noqa: B024
|
||||
"""
|
||||
Metadata used for out of band connector handshake between
|
||||
"""Metadata used for out of band connector handshake between
|
||||
P/D workers. This needs to serializable.
|
||||
"""
|
||||
|
||||
@@ -153,8 +150,7 @@ class KVConnectorHandshakeMetadata(ABC): # noqa: B024
|
||||
|
||||
|
||||
class KVConnectorMetadata(ABC): # noqa: B024
|
||||
"""
|
||||
Abstract Metadata used to communicate
|
||||
"""Abstract Metadata used to communicate
|
||||
Scheduler KVConnector -> Worker KVConnector.
|
||||
"""
|
||||
|
||||
@@ -162,8 +158,7 @@ class KVConnectorMetadata(ABC): # noqa: B024
|
||||
|
||||
|
||||
class KVConnectorWorkerMetadata(ABC):
|
||||
"""
|
||||
Abstract Metadata used to communicate back
|
||||
"""Abstract Metadata used to communicate back
|
||||
Worker KVConnector -> Scheduler KVConnector.
|
||||
|
||||
Each worker can output its own metadata.
|
||||
@@ -176,16 +171,12 @@ class KVConnectorWorkerMetadata(ABC):
|
||||
def aggregate(
|
||||
self, other: "KVConnectorWorkerMetadata"
|
||||
) -> "KVConnectorWorkerMetadata":
|
||||
"""
|
||||
Aggregate metadata with another `KVConnectorWorkerMetadata` object.
|
||||
"""
|
||||
"""Aggregate metadata with another `KVConnectorWorkerMetadata` object."""
|
||||
pass
|
||||
|
||||
|
||||
class KVConnectorBase_V1(ABC):
|
||||
"""
|
||||
Base class for KV connectors.
|
||||
"""
|
||||
"""Base class for KV connectors."""
|
||||
|
||||
@property
|
||||
def supports_divergent_local_hybrid_hits(self) -> bool:
|
||||
@@ -245,6 +236,7 @@ class KVConnectorBase_V1(ABC):
|
||||
|
||||
Args:
|
||||
connector_metadata (dict): the connector metadata.
|
||||
|
||||
"""
|
||||
self._connector_metadata = connector_metadata
|
||||
|
||||
@@ -263,6 +255,7 @@ class KVConnectorBase_V1(ABC):
|
||||
|
||||
Returns:
|
||||
ConnectorMetadata: the connector metadata.
|
||||
|
||||
"""
|
||||
# Should only be called while set to valid metadata.
|
||||
assert self._connector_metadata is not None
|
||||
@@ -273,22 +266,22 @@ class KVConnectorBase_V1(ABC):
|
||||
|
||||
Returns:
|
||||
bool: True if connector metadata exists, False otherwise.
|
||||
|
||||
"""
|
||||
return self._connector_metadata is not None
|
||||
|
||||
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
|
||||
"""
|
||||
Initialize with the KV caches. Useful for pre-registering the
|
||||
"""Initialize with the KV caches. Useful for pre-registering the
|
||||
KV Caches in the KVConnector (e.g. for NIXL).
|
||||
|
||||
Args:
|
||||
kv_caches: dictionary of layer names, kv cache
|
||||
|
||||
"""
|
||||
return
|
||||
|
||||
def set_host_xfer_buffer_ops(self, copy_operation: CopyBlocksOp):
|
||||
"""
|
||||
Set the xPU-specific ops for copying KV between host and device.
|
||||
"""Set the xPU-specific ops for copying KV between host and device.
|
||||
Needed when host buffer is used for kv transfer (e.g., in NixlConnector)
|
||||
"""
|
||||
return
|
||||
@@ -303,16 +296,14 @@ class KVConnectorBase_V1(ABC):
|
||||
return
|
||||
|
||||
def handle_preemptions(self, kv_connector_metadata: KVConnectorMetadata):
|
||||
"""
|
||||
Handle preempted requests or evicted blocks BEFORE they are overwritten.
|
||||
"""Handle preempted requests or evicted blocks BEFORE they are overwritten.
|
||||
Needed for connectors which use async saves (e.g., OffloadingConnector)
|
||||
"""
|
||||
return
|
||||
|
||||
@abstractmethod
|
||||
def start_load_kv(self, forward_context: "ForwardContext", **kwargs: Any) -> None:
|
||||
"""
|
||||
Start loading the KV cache from the connector to vLLM's paged
|
||||
"""Start loading the KV cache from the connector to vLLM's paged
|
||||
KV buffer. Loads required by the current forward start before it;
|
||||
independent asynchronous loads may start after it is submitted.
|
||||
|
||||
@@ -329,8 +320,7 @@ class KVConnectorBase_V1(ABC):
|
||||
|
||||
@abstractmethod
|
||||
def wait_for_layer_load(self, layer_name: str) -> None:
|
||||
"""
|
||||
Block until the KV for a specific layer is loaded into vLLM's
|
||||
"""Block until the KV for a specific layer is loaded into vLLM's
|
||||
paged buffer. This is called from within attention layer to ensure
|
||||
async copying from start_load_kv is complete.
|
||||
|
||||
@@ -338,6 +328,7 @@ class KVConnectorBase_V1(ABC):
|
||||
|
||||
Args:
|
||||
layer_name: the name of that layer
|
||||
|
||||
"""
|
||||
pass
|
||||
|
||||
@@ -349,8 +340,7 @@ class KVConnectorBase_V1(ABC):
|
||||
attn_metadata: "AttentionMetadata",
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
"""
|
||||
Start saving a layer of KV cache from vLLM's paged buffer
|
||||
"""Start saving a layer of KV cache from vLLM's paged buffer
|
||||
to the connector. This is called from within attention layer to
|
||||
enable async copying during execution.
|
||||
|
||||
@@ -360,13 +350,13 @@ class KVConnectorBase_V1(ABC):
|
||||
layer in vLLM.
|
||||
attn_metadata (AttentionMetadata): the attention metadata.
|
||||
**kwargs: additional arguments for the save operation.
|
||||
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def wait_for_save(self):
|
||||
"""
|
||||
Block until all the save operations is done. This is called
|
||||
"""Block until all the save operations is done. This is called
|
||||
as the forward context exits to ensure that the async saving
|
||||
from save_kv_layer is complete before finishing the forward.
|
||||
|
||||
@@ -377,8 +367,7 @@ class KVConnectorBase_V1(ABC):
|
||||
def get_finished(
|
||||
self, finished_req_ids: set[str]
|
||||
) -> tuple[set[str] | None, set[str] | None]:
|
||||
"""
|
||||
Notifies worker-side connector ids of requests that have
|
||||
"""Notifies worker-side connector ids of requests that have
|
||||
finished generating tokens on the worker.
|
||||
The scheduler process (via the Executors) will use this output
|
||||
to track which workers are done.
|
||||
@@ -389,6 +378,7 @@ class KVConnectorBase_V1(ABC):
|
||||
tuple of (sending/saving ids, recving/loading ids).
|
||||
The finished saves/sends req ids must belong to a set provided in a
|
||||
call to this method (this call or a prior one).
|
||||
|
||||
"""
|
||||
return None, None
|
||||
|
||||
@@ -403,8 +393,7 @@ class KVConnectorBase_V1(ABC):
|
||||
)
|
||||
|
||||
def get_block_ids_with_load_errors(self) -> set[int]:
|
||||
"""
|
||||
Get the set of block IDs that failed to load.
|
||||
"""Get the set of block IDs that failed to load.
|
||||
|
||||
Returns:
|
||||
Set of block IDs that encountered load errors.
|
||||
@@ -419,50 +408,47 @@ class KVConnectorBase_V1(ABC):
|
||||
block IDs must appear here no later than that same pass.
|
||||
- Sync loading: failed blocks should be reported in the forward
|
||||
pass in which they are detected.
|
||||
|
||||
"""
|
||||
return set()
|
||||
|
||||
def shutdown(self):
|
||||
"""
|
||||
Shutdown the connector. This is called when the worker process
|
||||
"""Shutdown the connector. This is called when the worker process
|
||||
is shutting down to ensure that all the async operations are
|
||||
completed and the connector is cleaned up properly.
|
||||
"""
|
||||
return None
|
||||
|
||||
def get_kv_connector_stats(self) -> "KVConnectorStats | None":
|
||||
"""
|
||||
Get the KV connector stats collected during the last interval.
|
||||
"""
|
||||
"""Get the KV connector stats collected during the last interval."""
|
||||
return None
|
||||
|
||||
def get_kv_connector_kv_cache_events(self) -> "KVConnectorKVEvents | None":
|
||||
"""
|
||||
Get the KV connector kv cache events collected during the last interval.
|
||||
"""Get the KV connector kv cache events collected during the last interval.
|
||||
This function should be called by the model runner every time after the
|
||||
model execution and before cleanup.
|
||||
"""
|
||||
return None
|
||||
|
||||
def get_handshake_metadata(self) -> KVConnectorHandshakeMetadata | None:
|
||||
"""
|
||||
Get the KVConnector handshake metadata for this connector.
|
||||
"""Get the KVConnector handshake metadata for this connector.
|
||||
This metadata is used for out-of-band connector handshake
|
||||
between P/D workers.
|
||||
|
||||
Returns:
|
||||
KVConnectorHandshakeMetadata: the handshake metadata.
|
||||
None if no handshake metadata is available.
|
||||
|
||||
"""
|
||||
return None
|
||||
|
||||
def build_connector_worker_meta(self) -> KVConnectorWorkerMetadata | None:
|
||||
"""
|
||||
Build the KVConnector worker metadata for this engine step.
|
||||
"""Build the KVConnector worker metadata for this engine step.
|
||||
|
||||
Returns:
|
||||
KVConnectorWorkerMetadata: the worker metadata.
|
||||
None if no worker metadata is available.
|
||||
|
||||
"""
|
||||
return None
|
||||
|
||||
@@ -476,12 +462,12 @@ class KVConnectorBase_V1(ABC):
|
||||
self.bind_gpu_block_pool(kv_cache_manager.block_pool)
|
||||
|
||||
def bind_gpu_block_pool(self, gpu_block_pool: "BlockPool") -> None:
|
||||
"""
|
||||
Bind the GPU block pool to the connector for per-GPU block status tracking.
|
||||
"""Bind the GPU block pool to the connector for per-GPU block status tracking.
|
||||
For example, inc/dec ref counts, or iterate over the prefix cache blocks.
|
||||
|
||||
Args:
|
||||
gpu_block_pool: the GPU block pool.
|
||||
|
||||
"""
|
||||
return
|
||||
|
||||
@@ -491,8 +477,7 @@ class KVConnectorBase_V1(ABC):
|
||||
request: "Request",
|
||||
num_computed_tokens: int,
|
||||
) -> tuple[int | None, bool]:
|
||||
"""
|
||||
Get number of new tokens that can be loaded from the
|
||||
"""Get number of new tokens that can be loaded from the
|
||||
external KV cache beyond the num_computed_tokens.
|
||||
|
||||
Args:
|
||||
@@ -517,6 +502,7 @@ class KVConnectorBase_V1(ABC):
|
||||
call. If the cache cannot be loaded for some tokens (e.g., due to
|
||||
connectivity issues or eviction), those tokens must not be taken
|
||||
into account.
|
||||
|
||||
"""
|
||||
pass
|
||||
|
||||
@@ -524,8 +510,7 @@ class KVConnectorBase_V1(ABC):
|
||||
def update_state_after_alloc(
|
||||
self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int
|
||||
):
|
||||
"""
|
||||
Update KVConnector state after block allocation.
|
||||
"""Update KVConnector state after block allocation.
|
||||
|
||||
If get_num_new_matched_tokens previously returned True for a
|
||||
request, this function may be called twice for that same request -
|
||||
@@ -543,6 +528,7 @@ class KVConnectorBase_V1(ABC):
|
||||
blocks (KVCacheBlocks): the blocks allocated for the request.
|
||||
num_external_tokens (int): the number of tokens to load from the
|
||||
external KV cache. 0 means nothing should be loaded.
|
||||
|
||||
"""
|
||||
pass
|
||||
|
||||
@@ -550,14 +536,14 @@ class KVConnectorBase_V1(ABC):
|
||||
def build_connector_meta(
|
||||
self, scheduler_output: SchedulerOutput
|
||||
) -> KVConnectorMetadata:
|
||||
"""
|
||||
Build the connector metadata for this step.
|
||||
"""Build the connector metadata for this step.
|
||||
|
||||
This function should NOT modify fields in the scheduler_output.
|
||||
Also, calling this function will reset the state of the connector.
|
||||
|
||||
Args:
|
||||
scheduler_output (SchedulerOutput): the scheduler output object.
|
||||
|
||||
"""
|
||||
pass
|
||||
|
||||
@@ -570,12 +556,12 @@ class KVConnectorBase_V1(ABC):
|
||||
return
|
||||
|
||||
def update_connector_output(self, connector_output: KVConnectorOutput):
|
||||
"""
|
||||
Update KVConnector state from worker-side connectors output.
|
||||
"""Update KVConnector state from worker-side connectors output.
|
||||
|
||||
Args:
|
||||
connector_output (KVConnectorOutput): the worker-side
|
||||
connectors output.
|
||||
|
||||
"""
|
||||
return
|
||||
|
||||
@@ -584,8 +570,7 @@ class KVConnectorBase_V1(ABC):
|
||||
request: "Request",
|
||||
block_ids: list[int],
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
"""
|
||||
Called exactly once when a request has finished, before its blocks are
|
||||
"""Called exactly once when a request has finished, before its blocks are
|
||||
freed.
|
||||
|
||||
The connector may assumes responsibility for freeing the blocks
|
||||
@@ -597,6 +582,7 @@ class KVConnectorBase_V1(ABC):
|
||||
get_finished().
|
||||
Optional KVTransferParams to be included in the request outputs
|
||||
returned by the engine.
|
||||
|
||||
"""
|
||||
return False, None
|
||||
|
||||
@@ -614,11 +600,11 @@ class KVConnectorBase_V1(ABC):
|
||||
return False
|
||||
|
||||
def take_events(self) -> Iterable["KVCacheEvent"]:
|
||||
"""
|
||||
Take the KV cache events from the connector.
|
||||
"""Take the KV cache events from the connector.
|
||||
|
||||
Yields:
|
||||
New KV cache events since the last call.
|
||||
|
||||
"""
|
||||
return ()
|
||||
|
||||
@@ -640,16 +626,16 @@ class KVConnectorBase_V1(ABC):
|
||||
|
||||
@classmethod
|
||||
def get_required_kvcache_layout(cls, vllm_config: "VllmConfig") -> str | None:
|
||||
"""
|
||||
Get the required KV cache layout for this connector.
|
||||
"""Get the required KV cache layout for this connector.
|
||||
|
||||
Args:
|
||||
vllm_config (VllmConfig): the vllm config.
|
||||
|
||||
Returns:
|
||||
str: the required KV cache layout. e.g. HND, or NHD.
|
||||
None if the connector does not require a specific layout.
|
||||
"""
|
||||
|
||||
"""
|
||||
if cls is KVConnectorBase_V1:
|
||||
raise TypeError(
|
||||
"get_required_kvcache_layout should not be called "
|
||||
@@ -659,8 +645,7 @@ class KVConnectorBase_V1(ABC):
|
||||
|
||||
@classmethod
|
||||
def requires_piecewise_for_cudagraph(cls, extra_config: dict[str, Any]) -> bool:
|
||||
"""
|
||||
Check if this connector requires PIECEWISE CUDA graph mode.
|
||||
"""Check if this connector requires PIECEWISE CUDA graph mode.
|
||||
|
||||
Connectors that use asynchronous layer-by-layer operations
|
||||
(wait_for_layer_load/save_kv_layer) should override this method
|
||||
@@ -676,27 +661,26 @@ class KVConnectorBase_V1(ABC):
|
||||
Returns:
|
||||
True if this connector requires PIECEWISE CUDA graph mode,
|
||||
False otherwise.
|
||||
|
||||
"""
|
||||
return False
|
||||
|
||||
def get_finished_count(self) -> int | None:
|
||||
"""
|
||||
Get the count of requests expected to complete send/receive operations
|
||||
"""Get the count of requests expected to complete send/receive operations
|
||||
via this connector. This method is used to initialize the
|
||||
KVOutputAggregator, overwriting the default world_size.
|
||||
|
||||
Returns:
|
||||
int: expected sending or receiving completion count.
|
||||
"""
|
||||
|
||||
"""
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def build_kv_connector_stats(
|
||||
cls, data: dict[str, Any] | None = None
|
||||
) -> "KVConnectorStats | None":
|
||||
"""
|
||||
KVConnectorStats resolution method. This method allows dynamically
|
||||
"""KVConnectorStats resolution method. This method allows dynamically
|
||||
registered connectors to return their own KVConnectorStats object,
|
||||
which can implement custom aggregation logic on the data dict.
|
||||
"""
|
||||
@@ -705,19 +689,18 @@ class KVConnectorBase_V1(ABC):
|
||||
def set_xfer_handshake_metadata(
|
||||
self, metadata: dict[int, KVConnectorHandshakeMetadata]
|
||||
) -> None:
|
||||
"""
|
||||
Set the KV connector handshake metadata for this connector.
|
||||
"""Set the KV connector handshake metadata for this connector.
|
||||
|
||||
Args:
|
||||
metadata (KVConnectorHandshakeMetadata): the handshake metadata to set.
|
||||
|
||||
"""
|
||||
return None
|
||||
|
||||
def set_xfer_handshake_metadata_pp_aware(
|
||||
self, metadata: dict[tuple[int, int], KVConnectorHandshakeMetadata]
|
||||
) -> None:
|
||||
"""
|
||||
Set handshake metadata keyed by (pp_rank, tp_rank).
|
||||
"""Set handshake metadata keyed by (pp_rank, tp_rank).
|
||||
- Default implementation assumes pp_rank is always 0
|
||||
- PP-aware connectors override this to consume all PP producer shards.
|
||||
"""
|
||||
@@ -738,19 +721,18 @@ class KVConnectorBase_V1(ABC):
|
||||
labelnames: list[str],
|
||||
per_engine_labelvalues: dict[int, list[object]],
|
||||
) -> "KVConnectorPromMetrics | None":
|
||||
"""
|
||||
Create a KVConnectorPromMetrics subclass which should register
|
||||
"""Create a KVConnectorPromMetrics subclass which should register
|
||||
per-connector Prometheus metrics and implement observe() to
|
||||
expose connector transfer stats via Prometheus.
|
||||
"""
|
||||
return None
|
||||
|
||||
def reset_cache(self) -> bool | None:
|
||||
"""
|
||||
Reset the connector's internal cache.
|
||||
"""Reset the connector's internal cache.
|
||||
|
||||
Returns:
|
||||
bool: True if the cache was successfully reset, False otherwise.
|
||||
|
||||
"""
|
||||
logger.debug(
|
||||
"Connector cache reset requested, but %s does not implement reset_cache().",
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
DecodeBenchConnector: A KV Connector for decode instance performance testing.
|
||||
"""DecodeBenchConnector: A KV Connector for decode instance performance testing.
|
||||
|
||||
This connector emulates a prefill-decode disaggregated setting by filling
|
||||
the KV cache with dummy values, allowing measurement of decoder performance
|
||||
@@ -10,7 +9,7 @@ under larger input sequence lengths (ISL) in resource-limited environments.
|
||||
Usage:
|
||||
To use this connector for benchmarking, configure it in the kv_transfer_config:
|
||||
|
||||
Example:
|
||||
Example:
|
||||
vllm serve <model> --kv-transfer-config '{
|
||||
"kv_connector": "DecodeBenchConnector",
|
||||
"kv_role": "kv_both",
|
||||
@@ -29,6 +28,7 @@ Usage:
|
||||
- fill_mean (float): Mean value for random normal fill (default: 0.015)
|
||||
- fill_std (float): Standard deviation for random fill (default: 0.0)
|
||||
Set to 0 for constant values, >0 for random sampling
|
||||
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
@@ -80,8 +80,7 @@ class DecodeBenchConnectorMetadata(KVConnectorMetadata):
|
||||
|
||||
|
||||
class DecodeBenchConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
"""
|
||||
A KV Connector for decode instance performance testing.
|
||||
"""A KV Connector for decode instance performance testing.
|
||||
|
||||
This connector fills the KV cache with dummy values to emulate a
|
||||
prefill-decode disaggregated setting, enabling performance testing of the
|
||||
@@ -219,8 +218,7 @@ class DecodeBenchConnectorScheduler:
|
||||
request: "Request",
|
||||
num_computed_tokens: int,
|
||||
) -> tuple[int, bool]:
|
||||
"""
|
||||
For new requests, return the number of tokens that should be filled
|
||||
"""For new requests, return the number of tokens that should be filled
|
||||
with dummy KV cache values.
|
||||
|
||||
Returns:
|
||||
@@ -228,6 +226,7 @@ class DecodeBenchConnectorScheduler:
|
||||
- num_tokens_to_fill: number of uncomputed tokens minus 1
|
||||
(we fill everything except the last token for decode)
|
||||
- is_async: False (synchronous filling)
|
||||
|
||||
"""
|
||||
req_id = request.request_id
|
||||
|
||||
@@ -251,8 +250,7 @@ class DecodeBenchConnectorScheduler:
|
||||
def update_state_after_alloc(
|
||||
self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int
|
||||
):
|
||||
"""
|
||||
Called after blocks are allocated. Store the block IDs so we can
|
||||
"""Called after blocks are allocated. Store the block IDs so we can
|
||||
fill them with dummy values.
|
||||
|
||||
Supports both single- and multi-group KV cache configurations.
|
||||
@@ -334,8 +332,7 @@ class DecodeBenchConnectorScheduler:
|
||||
def build_connector_meta(
|
||||
self, scheduler_output: "SchedulerOutput"
|
||||
) -> KVConnectorMetadata:
|
||||
"""
|
||||
Build metadata containing information about which blocks to fill
|
||||
"""Build metadata containing information about which blocks to fill
|
||||
with dummy KV values.
|
||||
"""
|
||||
meta = DecodeBenchConnectorMetadata(reqs_to_fill=self._pending_fills.copy())
|
||||
@@ -346,9 +343,7 @@ class DecodeBenchConnectorScheduler:
|
||||
return meta
|
||||
|
||||
def request_finished(self, request: "Request"):
|
||||
"""
|
||||
Called when a request has finished. Clean up any state.
|
||||
"""
|
||||
"""Called when a request has finished. Clean up any state."""
|
||||
self._filled_requests.discard(request.request_id)
|
||||
|
||||
|
||||
@@ -389,8 +384,7 @@ class DecodeBenchConnectorWorker:
|
||||
)
|
||||
|
||||
def start_fill_kv(self, metadata: DecodeBenchConnectorMetadata):
|
||||
"""
|
||||
Fill the allocated KV cache blocks with dummy values.
|
||||
"""Fill the allocated KV cache blocks with dummy values.
|
||||
|
||||
This simulates having a populated KV cache from a prefill phase,
|
||||
allowing decode performance testing with larger context sizes.
|
||||
@@ -419,13 +413,13 @@ class DecodeBenchConnectorWorker:
|
||||
)
|
||||
|
||||
def _fill_blocks(self, group_idx: int, block_ids: list[int], num_tokens: int):
|
||||
"""
|
||||
Fill specified blocks with dummy values for a specific KV cache group.
|
||||
"""Fill specified blocks with dummy values for a specific KV cache group.
|
||||
|
||||
Args:
|
||||
group_idx: The KV cache group index to fill
|
||||
block_ids: List of block IDs to fill in this group
|
||||
num_tokens: Total number of tokens to fill across these blocks
|
||||
|
||||
"""
|
||||
if not block_ids:
|
||||
return
|
||||
@@ -501,6 +495,7 @@ class DecodeBenchConnectorWorker:
|
||||
tensor's first dim are ignored.
|
||||
fill_mean: Mean value for the fill.
|
||||
fill_std: Standard deviation for the fill.
|
||||
|
||||
"""
|
||||
# Convert block_ids to tensor on device
|
||||
block_ids_tensor = torch.tensor(
|
||||
@@ -551,6 +546,7 @@ class DecodeBenchConnectorWorker:
|
||||
kv_cache: A state tensor to fill in its entirety.
|
||||
fill_mean: Mean value for the fill.
|
||||
fill_std: Standard deviation for the fill.
|
||||
|
||||
"""
|
||||
if fill_std > 0:
|
||||
kv_cache.normal_(mean=fill_mean, std=fill_std)
|
||||
|
||||
@@ -118,6 +118,7 @@ class ExampleConnector(KVConnectorBase_V1):
|
||||
Note:
|
||||
The number of elements in kv_caches and layer_names should be
|
||||
the same.
|
||||
|
||||
"""
|
||||
|
||||
def inject_kv_into_layer(
|
||||
@@ -134,6 +135,9 @@ class ExampleConnector(KVConnectorBase_V1):
|
||||
src_kv_cache (torch.Tensor): the source KV cache.
|
||||
slot_mapping (torch.Tensor): the slot mapping. In shape
|
||||
[num_tokens].
|
||||
attn_metadata (AttentionMetadata): attention metadata for the
|
||||
current forward pass.
|
||||
|
||||
"""
|
||||
# Deliberate bulk transfers in this reference connector.
|
||||
with gpu_sync_allowed():
|
||||
@@ -201,6 +205,7 @@ class ExampleConnector(KVConnectorBase_V1):
|
||||
|
||||
Args:
|
||||
layer_name: the name of that layer
|
||||
|
||||
"""
|
||||
return
|
||||
|
||||
@@ -220,6 +225,7 @@ class ExampleConnector(KVConnectorBase_V1):
|
||||
layer in vLLM.
|
||||
attn_metadata (AttentionMetadata): the attention metadata.
|
||||
**kwargs: additional arguments for the save operation.
|
||||
|
||||
"""
|
||||
|
||||
def extract_kv_from_layer(
|
||||
@@ -260,8 +266,7 @@ class ExampleConnector(KVConnectorBase_V1):
|
||||
request: "Request",
|
||||
num_computed_tokens: int,
|
||||
) -> tuple[int | None, bool]:
|
||||
"""
|
||||
Get number of new tokens that can be loaded from the
|
||||
"""Get number of new tokens that can be loaded from the
|
||||
external KV cache beyond the num_computed_tokens.
|
||||
|
||||
Args:
|
||||
@@ -272,6 +277,7 @@ class ExampleConnector(KVConnectorBase_V1):
|
||||
Returns:
|
||||
the number of tokens that can be loaded from the
|
||||
external KV cache beyond what is already computed.
|
||||
|
||||
"""
|
||||
# NOTE: in this debug implementation, we assume that the prompt is
|
||||
# cached_prompt + newly_generated_single_token
|
||||
@@ -295,8 +301,7 @@ class ExampleConnector(KVConnectorBase_V1):
|
||||
def update_state_after_alloc(
|
||||
self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int
|
||||
):
|
||||
"""
|
||||
Update KVConnector state after block allocation.
|
||||
"""Update KVConnector state after block allocation.
|
||||
|
||||
If blocks were allocated, add to _requests_need_load,
|
||||
such that we load the KVs in the next forward pass.
|
||||
@@ -315,6 +320,7 @@ class ExampleConnector(KVConnectorBase_V1):
|
||||
|
||||
Args:
|
||||
scheduler_output (SchedulerOutput): the scheduler output object.
|
||||
|
||||
"""
|
||||
meta = ExampleConnectorMetadata()
|
||||
|
||||
|
||||
@@ -57,6 +57,7 @@ def load_hidden_states(path: str) -> dict[str, torch.Tensor]:
|
||||
|
||||
Returns:
|
||||
Dict with "hidden_states" and "token_ids" tensors.
|
||||
|
||||
"""
|
||||
lock_path = path + ".lock"
|
||||
with open(lock_path) as lf:
|
||||
@@ -96,8 +97,7 @@ class ExampleHiddenStatesConnectorMetadata(KVConnectorMetadata):
|
||||
|
||||
|
||||
class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
"""
|
||||
Simple debug implementation of a HiddenStatesConnector.
|
||||
"""Simple debug implementation of a HiddenStatesConnector.
|
||||
|
||||
Simply extracts the hidden states from the kv cache and stores them to disk.
|
||||
Must be used in conjunction with the `extract_hidden_states` spec decoding method.
|
||||
@@ -278,7 +278,8 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
|
||||
def _on_write_done(self, req_id: str, future: Future) -> None:
|
||||
"""Surface any exception from the disk-write thread and drop the
|
||||
completed future from the in-flight tracking dict."""
|
||||
completed future from the in-flight tracking dict.
|
||||
"""
|
||||
self._req_futures.pop(req_id, None)
|
||||
exc = future.exception()
|
||||
if exc is not None:
|
||||
@@ -436,8 +437,7 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
request: "Request",
|
||||
num_computed_tokens: int,
|
||||
) -> tuple[int | None, bool]:
|
||||
"""
|
||||
Get number of new tokens that can be loaded from the
|
||||
"""Get number of new tokens that can be loaded from the
|
||||
external KV cache beyond the num_computed_tokens.
|
||||
|
||||
Args:
|
||||
@@ -448,6 +448,7 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
Returns:
|
||||
the number of tokens that can be loaded from the
|
||||
external KV cache beyond what is already computed.
|
||||
|
||||
"""
|
||||
# This connector is store-only, so we don't need to load any tokens
|
||||
return 0, False
|
||||
@@ -471,6 +472,7 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
|
||||
Args:
|
||||
scheduler_output (SchedulerOutput): the scheduler output object.
|
||||
|
||||
"""
|
||||
meta = ExampleHiddenStatesConnectorMetadata()
|
||||
|
||||
@@ -509,8 +511,7 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
request: "Request",
|
||||
block_ids: list[int],
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
"""
|
||||
Called exactly once when a request has finished, before its blocks are
|
||||
"""Called exactly once when a request has finished, before its blocks are
|
||||
freed.
|
||||
|
||||
Returns True to delay block freeing until get_finished extracts
|
||||
@@ -587,16 +588,16 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
|
||||
@classmethod
|
||||
def get_required_kvcache_layout(cls, vllm_config: "VllmConfig") -> str | None:
|
||||
"""
|
||||
Get the required KV cache layout for this connector.
|
||||
"""Get the required KV cache layout for this connector.
|
||||
|
||||
Args:
|
||||
vllm_config (VllmConfig): the vllm config.
|
||||
|
||||
Returns:
|
||||
str: the required KV cache layout. e.g. HND, or NHD.
|
||||
None if the connector does not require a specific layout.
|
||||
"""
|
||||
|
||||
"""
|
||||
if cls is KVConnectorBase_V1:
|
||||
raise TypeError(
|
||||
"get_required_kvcache_layout should not be called "
|
||||
|
||||
@@ -96,6 +96,7 @@ class FlexKVConnectorV1(KVConnectorBase_V1):
|
||||
Args:
|
||||
forward_context (ForwardContext): the forward context.
|
||||
**kwargs (Any): additional arguments (unused).
|
||||
|
||||
"""
|
||||
self._flexkv_connector.start_load_kv(forward_context, **kwargs)
|
||||
|
||||
@@ -107,6 +108,7 @@ class FlexKVConnectorV1(KVConnectorBase_V1):
|
||||
|
||||
Args:
|
||||
layer_name: the name of the layer (unused).
|
||||
|
||||
"""
|
||||
self._flexkv_connector.wait_for_layer_load(layer_name)
|
||||
|
||||
@@ -131,6 +133,7 @@ class FlexKVConnectorV1(KVConnectorBase_V1):
|
||||
kv_layer (torch.Tensor): the paged KV buffer (unused).
|
||||
attn_metadata (AttentionMetadata): the attention metadata (unused).
|
||||
**kwargs (Any): additional arguments (unused).
|
||||
|
||||
"""
|
||||
self._flexkv_connector.save_kv_layer(
|
||||
layer_name, kv_layer, attn_metadata, **kwargs
|
||||
@@ -160,6 +163,7 @@ class FlexKVConnectorV1(KVConnectorBase_V1):
|
||||
that have finished asynchronous transfer. The finished saves/sends
|
||||
req ids must belong to a set provided in a call to this method
|
||||
(this call or a prior one).
|
||||
|
||||
"""
|
||||
return self._flexkv_connector.get_finished(finished_req_ids)
|
||||
|
||||
@@ -169,6 +173,7 @@ class FlexKVConnectorV1(KVConnectorBase_V1):
|
||||
|
||||
Args:
|
||||
kv_caches: dictionary of layer names to kv cache tensors.
|
||||
|
||||
"""
|
||||
self._flexkv_connector.register_kv_caches(kv_caches)
|
||||
|
||||
@@ -192,6 +197,7 @@ class FlexKVConnectorV1(KVConnectorBase_V1):
|
||||
Tuple of (num_external_tokens, is_ready) where
|
||||
num_external_tokens is the number of additional tokens that
|
||||
can be loaded from the external KV cache.
|
||||
|
||||
"""
|
||||
return self._flexkv_connector.get_num_new_matched_tokens(
|
||||
request, num_computed_tokens
|
||||
@@ -215,6 +221,7 @@ class FlexKVConnectorV1(KVConnectorBase_V1):
|
||||
|
||||
Args:
|
||||
scheduler_output (SchedulerOutput): the scheduler output object.
|
||||
|
||||
"""
|
||||
return self._flexkv_connector.build_connector_meta(scheduler_output)
|
||||
|
||||
@@ -224,6 +231,7 @@ class FlexKVConnectorV1(KVConnectorBase_V1):
|
||||
Args:
|
||||
connector_output (KVConnectorOutput): the worker-side
|
||||
connectors output.
|
||||
|
||||
"""
|
||||
self._flexkv_connector.update_connector_output(connector_output)
|
||||
|
||||
@@ -240,6 +248,7 @@ class FlexKVConnectorV1(KVConnectorBase_V1):
|
||||
should not be freed until the request_id is returned from
|
||||
:meth:`get_finished`. kv_transfer_params is an optional dict of
|
||||
KVTransferParams to be included in the request outputs.
|
||||
|
||||
"""
|
||||
return self._flexkv_connector.request_finished(request, block_ids)
|
||||
|
||||
@@ -248,6 +257,7 @@ class FlexKVConnectorV1(KVConnectorBase_V1):
|
||||
|
||||
Returns:
|
||||
New KV cache events since the last call.
|
||||
|
||||
"""
|
||||
return self._flexkv_connector.take_events()
|
||||
|
||||
|
||||
@@ -67,6 +67,7 @@ class Hf3fsClient:
|
||||
size: Total size of the storage file in bytes
|
||||
bytes_per_page: Size of each page in bytes
|
||||
entries: Maximum number of concurrent operations
|
||||
|
||||
"""
|
||||
if not HF3FS_AVAILABLE:
|
||||
raise ImportError(
|
||||
@@ -154,7 +155,7 @@ class Hf3fsClient:
|
||||
)
|
||||
|
||||
def _release_resources(self) -> None:
|
||||
"""Release all acquired resources safely"""
|
||||
"""Release all acquired resources safely."""
|
||||
# iov must be released before ioring and shm
|
||||
for attr in ("iov_r", "iov_w", "ior_r", "ior_w"):
|
||||
obj = getattr(self, attr, None)
|
||||
@@ -192,6 +193,7 @@ class Hf3fsClient:
|
||||
|
||||
Returns:
|
||||
List of operation results (0 for success, non-zero for error)
|
||||
|
||||
"""
|
||||
self.check(offsets, tensors)
|
||||
assert self.ior_r is not None
|
||||
@@ -228,11 +230,12 @@ class Hf3fsClient:
|
||||
Args:
|
||||
offsets: List of byte offsets to write to
|
||||
tensors: List of tensors containing data to write
|
||||
event: CUDA event to synchronize on before reading the tensors
|
||||
|
||||
Returns:
|
||||
List of operation results (0 for success, non-zero for error)
|
||||
"""
|
||||
|
||||
"""
|
||||
self.check(offsets, tensors)
|
||||
assert self.ior_w is not None
|
||||
assert self.iov_w is not None
|
||||
@@ -282,6 +285,7 @@ class Hf3fsClient:
|
||||
|
||||
Returns:
|
||||
Size of the file in bytes
|
||||
|
||||
"""
|
||||
return self.size
|
||||
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
HF3FS KV Connector Implementation for vLLM.
|
||||
"""HF3FS KV Connector Implementation for vLLM.
|
||||
|
||||
This module implements a KV connector that uses
|
||||
the 3FS for storing and retrieving KV cache data.
|
||||
@@ -99,9 +98,7 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
class AsyncOperationManager:
|
||||
"""
|
||||
Manages async save/load operations with background threads.
|
||||
"""
|
||||
"""Manages async save/load operations with background threads."""
|
||||
|
||||
def __init__(self, connector: "HF3FSKVConnector"):
|
||||
# Store connector reference and extract commonly used attributes
|
||||
@@ -667,9 +664,7 @@ class HF3FSKVConnector(KVConnectorBase_V1):
|
||||
return self._async_manager.get_finished_operations(finished_req_ids)
|
||||
|
||||
def get_kv_connector_stats(self) -> Optional["KVConnectorStats"]:
|
||||
"""
|
||||
Get the KV connector stats collected during the last interval.
|
||||
"""
|
||||
"""Get the KV connector stats collected during the last interval."""
|
||||
# Clear stats for next iteration
|
||||
if (
|
||||
hasattr(self, "_async_manager")
|
||||
@@ -886,8 +881,7 @@ class HF3FSKVConnector(KVConnectorBase_V1):
|
||||
def build_kv_connector_stats(
|
||||
cls, data: dict[str, Any] | None = None
|
||||
) -> Optional["KVConnectorStats"]:
|
||||
"""
|
||||
KVConnectorStats resolution method. This method allows dynamically
|
||||
"""KVConnectorStats resolution method. This method allows dynamically
|
||||
registered connectors to return their own KVConnectorStats object,
|
||||
which can implement custom aggregation logic on the data dict.
|
||||
"""
|
||||
@@ -1021,7 +1015,7 @@ class HF3FSKVConnector(KVConnectorBase_V1):
|
||||
|
||||
@dataclass
|
||||
class HF3FSKVConnectorStats(KVConnectorStats):
|
||||
"""Container for transfer performance metrics"""
|
||||
"""Container for transfer performance metrics."""
|
||||
|
||||
def __post_init__(self):
|
||||
if not self.data:
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
HF3FS Metadata Server with key-based organization.
|
||||
"""
|
||||
"""HF3FS Metadata Server with key-based organization."""
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
@@ -119,6 +117,7 @@ class GlobalMetadataState:
|
||||
|
||||
Returns:
|
||||
Dictionary mapping key -> allocated page index
|
||||
|
||||
"""
|
||||
with self.global_lock:
|
||||
if rank not in self.rank_metadata:
|
||||
@@ -167,6 +166,7 @@ class GlobalMetadataState:
|
||||
rank: Rank ID that confirmed the writes
|
||||
key_confirmations: List of (key, page_index) tuples
|
||||
pages_to_release: List of page indices to release back to free pool
|
||||
|
||||
"""
|
||||
with self.global_lock:
|
||||
# Confirm successful writes
|
||||
@@ -197,6 +197,7 @@ class GlobalMetadataState:
|
||||
|
||||
Returns:
|
||||
List of boolean values indicating key existence and completion
|
||||
|
||||
"""
|
||||
with self.global_lock:
|
||||
results = []
|
||||
@@ -218,6 +219,7 @@ class GlobalMetadataState:
|
||||
|
||||
Returns:
|
||||
List of page indices in the same order as input keys (None if key not found)
|
||||
|
||||
"""
|
||||
with self.global_lock:
|
||||
if rank not in self.rank_metadata:
|
||||
|
||||
@@ -128,6 +128,7 @@ def scatter_kv_caches(
|
||||
num_heads: Size of the H axis
|
||||
content_size: Size of the C axis
|
||||
kv_cache_strides: Element strides of each ``[B, H, N, C]`` layer view
|
||||
|
||||
"""
|
||||
num_layers = len(kv_caches_ptrs)
|
||||
num_tokens_in_block = len(token_indices)
|
||||
@@ -184,6 +185,7 @@ def gather_kv_caches(
|
||||
num_heads: Size of the H axis
|
||||
content_size: Size of the C axis
|
||||
kv_cache_strides: Element strides of each ``[B, H, N, C]`` layer view
|
||||
|
||||
"""
|
||||
num_layers = kv_caches_ptrs.shape[0]
|
||||
num_tokens_in_block = len(token_indices)
|
||||
|
||||
@@ -32,9 +32,7 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
class LMCacheKVEvents(KVConnectorKVEvents):
|
||||
"""
|
||||
Concrete implementation of KVConnectorKVEvents using KVEventAggregator.
|
||||
"""
|
||||
"""Concrete implementation of KVConnectorKVEvents using KVEventAggregator."""
|
||||
|
||||
def __init__(self, num_workers: int) -> None:
|
||||
self._aggregator = KVEventAggregator(num_workers)
|
||||
@@ -43,9 +41,7 @@ class LMCacheKVEvents(KVConnectorKVEvents):
|
||||
self._aggregator.add_events(events)
|
||||
|
||||
def aggregate(self) -> "LMCacheKVEvents":
|
||||
"""
|
||||
Aggregate KV events and retain only common events.
|
||||
"""
|
||||
"""Aggregate KV events and retain only common events."""
|
||||
common_events = self._aggregator.get_common_events()
|
||||
self._aggregator.clear_events()
|
||||
self._aggregator.add_events(common_events)
|
||||
@@ -72,8 +68,7 @@ class LMCacheKVEvents(KVConnectorKVEvents):
|
||||
class LMCacheConnectorV1(KVConnectorBase_V1):
|
||||
@classmethod
|
||||
def requires_piecewise_for_cudagraph(cls, extra_config: dict[str, Any]) -> bool:
|
||||
"""
|
||||
LMCache requires PIECEWISE CUDA graph mode when layerwise
|
||||
"""LMCache requires PIECEWISE CUDA graph mode when layerwise
|
||||
operations are enabled. The wait_for_layer_load and save_kv_layer
|
||||
methods perform actual async synchronization that cannot be
|
||||
captured in CUDA graphs.
|
||||
@@ -118,12 +113,12 @@ class LMCacheConnectorV1(KVConnectorBase_V1):
|
||||
# Worker-side methods
|
||||
# ==============================
|
||||
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
|
||||
"""
|
||||
Initialize with the KV caches. Useful for pre-registering the
|
||||
"""Initialize with the KV caches. Useful for pre-registering the
|
||||
KV Caches in the KVConnector (e.g. for NIXL).
|
||||
|
||||
Args:
|
||||
kv_caches: dictionary of layer names, kv cache
|
||||
|
||||
"""
|
||||
if hasattr(self._lmcache_engine, "register_kv_caches"):
|
||||
self._lmcache_engine.register_kv_caches(kv_caches)
|
||||
@@ -149,8 +144,7 @@ class LMCacheConnectorV1(KVConnectorBase_V1):
|
||||
engine.layerwise_retrievers = []
|
||||
|
||||
def start_load_kv(self, forward_context: "ForwardContext", **kwargs: Any) -> None:
|
||||
"""
|
||||
Start loading the KV cache from the connector to vLLM's paged
|
||||
"""Start loading the KV cache from the connector to vLLM's paged
|
||||
KV buffer. This is called from the forward context before the
|
||||
forward pass to enable async loading during model execution.
|
||||
|
||||
@@ -166,8 +160,7 @@ class LMCacheConnectorV1(KVConnectorBase_V1):
|
||||
self._lmcache_engine.start_load_kv(forward_context, **kwargs)
|
||||
|
||||
def wait_for_layer_load(self, layer_name: str) -> None:
|
||||
"""
|
||||
Block until the KV for a specific layer is loaded into vLLM's
|
||||
"""Block until the KV for a specific layer is loaded into vLLM's
|
||||
paged buffer. This is called from within attention layer to ensure
|
||||
async copying from start_load_kv is complete.
|
||||
|
||||
@@ -175,6 +168,7 @@ class LMCacheConnectorV1(KVConnectorBase_V1):
|
||||
|
||||
Args:
|
||||
layer_name: the name of that layer
|
||||
|
||||
"""
|
||||
self._lmcache_engine.wait_for_layer_load(layer_name)
|
||||
|
||||
@@ -185,8 +179,7 @@ class LMCacheConnectorV1(KVConnectorBase_V1):
|
||||
attn_metadata: AttentionMetadata,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
"""
|
||||
Start saving the a layer of KV cache from vLLM's paged buffer
|
||||
"""Start saving the a layer of KV cache from vLLM's paged buffer
|
||||
to the connector. This is called from within attention layer to
|
||||
enable async copying during execution.
|
||||
|
||||
@@ -196,14 +189,14 @@ class LMCacheConnectorV1(KVConnectorBase_V1):
|
||||
layer in vLLM.
|
||||
attn_metadata (AttentionMetadata): the attention metadata.
|
||||
**kwargs: additional arguments for the save operation.
|
||||
|
||||
"""
|
||||
self._lmcache_engine.save_kv_layer(
|
||||
layer_name, kv_layer, attn_metadata, **kwargs
|
||||
)
|
||||
|
||||
def wait_for_save(self):
|
||||
"""
|
||||
Block until all the save operations is done. This is called
|
||||
"""Block until all the save operations is done. This is called
|
||||
as the forward context exits to ensure that the async saving
|
||||
from save_kv_layer is complete before finishing the forward.
|
||||
|
||||
@@ -214,8 +207,7 @@ class LMCacheConnectorV1(KVConnectorBase_V1):
|
||||
def get_finished(
|
||||
self, finished_req_ids: set[str]
|
||||
) -> tuple[set[str] | None, set[str] | None]:
|
||||
"""
|
||||
Notifies worker-side connector ids of requests that have
|
||||
"""Notifies worker-side connector ids of requests that have
|
||||
finished generating tokens.
|
||||
|
||||
Returns:
|
||||
@@ -224,16 +216,17 @@ class LMCacheConnectorV1(KVConnectorBase_V1):
|
||||
tuple of (sending/saving ids, recving/loading ids).
|
||||
The finished saves/sends req ids must belong to a set provided in a
|
||||
call to this method (this call or a prior one).
|
||||
|
||||
"""
|
||||
return self._lmcache_engine.get_finished(finished_req_ids)
|
||||
|
||||
def get_block_ids_with_load_errors(self) -> set[int]:
|
||||
"""
|
||||
Get the set of block IDs that failed to load.
|
||||
"""Get the set of block IDs that failed to load.
|
||||
|
||||
Returns:
|
||||
Set of block IDs that encountered load errors.
|
||||
Empty set if no load errors occurred.
|
||||
|
||||
"""
|
||||
method = getattr(self._lmcache_engine, "get_block_ids_with_load_errors", None)
|
||||
if callable(method):
|
||||
@@ -243,10 +236,7 @@ class LMCacheConnectorV1(KVConnectorBase_V1):
|
||||
return set()
|
||||
|
||||
def get_kv_connector_kv_cache_events(self) -> LMCacheKVEvents | None:
|
||||
"""
|
||||
Get the KV connector kv cache events collected during the last interval.
|
||||
"""
|
||||
|
||||
"""Get the KV connector kv cache events collected during the last interval."""
|
||||
events = self._lmcache_engine.get_kv_events() # type: ignore [attr-defined]
|
||||
if not events:
|
||||
return None
|
||||
@@ -276,8 +266,7 @@ class LMCacheConnectorV1(KVConnectorBase_V1):
|
||||
request: "Request",
|
||||
num_computed_tokens: int,
|
||||
) -> tuple[int | None, bool]:
|
||||
"""
|
||||
Get number of new tokens that can be loaded from the
|
||||
"""Get number of new tokens that can be loaded from the
|
||||
external KV cache beyond the num_computed_tokens.
|
||||
|
||||
Args:
|
||||
@@ -288,6 +277,7 @@ class LMCacheConnectorV1(KVConnectorBase_V1):
|
||||
Returns:
|
||||
the number of tokens that can be loaded from the
|
||||
external KV cache beyond what is already computed.
|
||||
|
||||
"""
|
||||
return self._lmcache_engine.get_num_new_matched_tokens(
|
||||
request, num_computed_tokens
|
||||
@@ -296,32 +286,30 @@ class LMCacheConnectorV1(KVConnectorBase_V1):
|
||||
def update_state_after_alloc(
|
||||
self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int
|
||||
):
|
||||
"""
|
||||
Update KVConnector state after block allocation.
|
||||
"""
|
||||
"""Update KVConnector state after block allocation."""
|
||||
self._lmcache_engine.update_state_after_alloc(request, num_external_tokens)
|
||||
|
||||
def build_connector_meta(
|
||||
self, scheduler_output: SchedulerOutput
|
||||
) -> KVConnectorMetadata:
|
||||
"""
|
||||
Build the connector metadata for this step.
|
||||
"""Build the connector metadata for this step.
|
||||
|
||||
This function should NOT modify fields in the scheduler_output.
|
||||
Also, calling this function will reset the state of the connector.
|
||||
|
||||
Args:
|
||||
scheduler_output (SchedulerOutput): the scheduler output object.
|
||||
|
||||
"""
|
||||
return self._lmcache_engine.build_connector_meta(scheduler_output)
|
||||
|
||||
def update_connector_output(self, connector_output: KVConnectorOutput):
|
||||
"""
|
||||
Update KVConnector state from worker-side connectors output.
|
||||
"""Update KVConnector state from worker-side connectors output.
|
||||
|
||||
Args:
|
||||
connector_output (KVConnectorOutput): the worker-side
|
||||
connectors output.
|
||||
|
||||
"""
|
||||
# Get the KV events
|
||||
kv_cache_events = connector_output.kv_cache_events
|
||||
@@ -342,8 +330,7 @@ class LMCacheConnectorV1(KVConnectorBase_V1):
|
||||
request: "Request",
|
||||
block_ids: list[int],
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
"""
|
||||
Called when a request has finished, before its blocks are freed.
|
||||
"""Called when a request has finished, before its blocks are freed.
|
||||
|
||||
Returns:
|
||||
True if the request is being saved/sent asynchronously and blocks
|
||||
@@ -351,15 +338,16 @@ class LMCacheConnectorV1(KVConnectorBase_V1):
|
||||
get_finished().
|
||||
Optional KVTransferParams to be included in the request outputs
|
||||
returned by the engine.
|
||||
|
||||
"""
|
||||
return self._lmcache_engine.request_finished(request, block_ids)
|
||||
|
||||
def take_events(self) -> Iterable["KVCacheEvent"]:
|
||||
"""
|
||||
Take the KV cache events from the connector.
|
||||
"""Take the KV cache events from the connector.
|
||||
|
||||
Yields:
|
||||
New KV cache events since the last call.
|
||||
|
||||
"""
|
||||
if self._kv_cache_events is not None:
|
||||
self._kv_cache_events.aggregate()
|
||||
|
||||
+46
-47
@@ -44,8 +44,7 @@ def send_lmcache_request(
|
||||
request_type: RequestType,
|
||||
payloads: list[Any],
|
||||
) -> MessagingFuture[Any]:
|
||||
"""
|
||||
Helper function to send the request to the LMCache multiprocess server
|
||||
"""Helper function to send the request to the LMCache multiprocess server.
|
||||
|
||||
Args:
|
||||
mq_client: The LMCache multiprocess mode message queue client
|
||||
@@ -54,8 +53,8 @@ def send_lmcache_request(
|
||||
|
||||
Returns:
|
||||
A messaging future for the request
|
||||
"""
|
||||
|
||||
"""
|
||||
future = mq_client.submit_request(
|
||||
request_type, payloads, get_response_class(request_type)
|
||||
)
|
||||
@@ -65,14 +64,14 @@ def send_lmcache_request(
|
||||
def get_lmcache_chunk_size(
|
||||
mq_client: MessageQueueClient,
|
||||
) -> int:
|
||||
"""
|
||||
Helper function to get the LMCache chunk size from the server
|
||||
"""Helper function to get the LMCache chunk size from the server.
|
||||
|
||||
Args:
|
||||
mq_client: The LMCache multiprocess mode message queue client
|
||||
|
||||
Returns:
|
||||
An integer representing the LMCache chunk size
|
||||
|
||||
"""
|
||||
future = send_lmcache_request(mq_client, RequestType.GET_CHUNK_SIZE, [])
|
||||
chunk_size = future.result()
|
||||
@@ -147,16 +146,16 @@ class LMCacheMPSchedulerAdapter:
|
||||
vllm_block_size: int,
|
||||
parallel_strategy: ParallelStrategy,
|
||||
):
|
||||
"""
|
||||
Args:
|
||||
server_url: The server URL for the LMCache message queue
|
||||
context: The ZMQ context
|
||||
"""Args:
|
||||
server_url: The server URL for the LMCache message queue
|
||||
context: The ZMQ context
|
||||
|
||||
model_name: The model name used for LMCache keys
|
||||
vllm_block_size: The block size used in vLLM
|
||||
parallel_strategy:
|
||||
The parallel strategy, which includes `use_mla`,
|
||||
`world_size`, `worker_id` and so on
|
||||
|
||||
model_name: The model name used for LMCache keys
|
||||
vllm_block_size: The block size used in vLLM
|
||||
parallel_strategy:
|
||||
The parallel strategy, which includes `use_mla`,
|
||||
`world_size`, `worker_id` and so on
|
||||
"""
|
||||
self.mq_client = MessageQueueClient(server_url, context)
|
||||
|
||||
@@ -195,8 +194,7 @@ class LMCacheMPSchedulerAdapter:
|
||||
block_hashes: list[bytes] | None = None,
|
||||
token_ids: list[int] | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
Submit a new lookup request to LMCache if there is no ongoing request.
|
||||
"""Submit a new lookup request to LMCache if there is no ongoing request.
|
||||
|
||||
Supports both token-based and hash-based vLLM:
|
||||
- token_ids: token IDs (token-based vLLM) -> single token-mode key
|
||||
@@ -219,6 +217,7 @@ class LMCacheMPSchedulerAdapter:
|
||||
for later retrieve operations.
|
||||
In the meantime, this function will record the lookup request, and the
|
||||
status of the look up request can be checked by `check_lookup_result`.
|
||||
|
||||
"""
|
||||
if request_id in self.lookup_futures:
|
||||
# Skip if there is already a lookup request
|
||||
@@ -260,8 +259,7 @@ class LMCacheMPSchedulerAdapter:
|
||||
|
||||
@_lmcache_nvtx_annotate
|
||||
def check_lookup_result(self, request_id: str) -> int | None:
|
||||
"""
|
||||
Check the result of a previously submitted lookup request.
|
||||
"""Check the result of a previously submitted lookup request.
|
||||
|
||||
Args:
|
||||
request_id: The ID of the lookup request submitted in
|
||||
@@ -271,6 +269,7 @@ class LMCacheMPSchedulerAdapter:
|
||||
An integer representing the total number of tokens matched
|
||||
in LMCache (prefix matching), or
|
||||
None if the lookup request is not finished yet.
|
||||
|
||||
"""
|
||||
assert request_id in self.lookup_futures, (
|
||||
f"Lookup request for request_id={request_id} has not been submitted"
|
||||
@@ -285,25 +284,27 @@ class LMCacheMPSchedulerAdapter:
|
||||
return num_chunks * self.chunk_size
|
||||
|
||||
def num_blocks_per_chunk(self) -> int:
|
||||
"""
|
||||
Returns:
|
||||
The number of vllm blocks in a LMCache data chunk
|
||||
"""Returns:
|
||||
The number of vllm blocks in a LMCache data chunk
|
||||
|
||||
"""
|
||||
return self.blocks_in_chunk
|
||||
|
||||
def cleanup_lookup_result(self, request_id: str) -> None:
|
||||
"""
|
||||
Clean up lookup future for a finished request to prevent memory leak.
|
||||
"""Clean up lookup future for a finished request to prevent memory leak.
|
||||
|
||||
Args:
|
||||
request_id: The ID of the finished request.
|
||||
|
||||
"""
|
||||
self.lookup_futures.pop(request_id, None)
|
||||
|
||||
def end_session(self, request_id: str) -> None:
|
||||
"""
|
||||
Notify LMCache server to remove the session for a finished request.
|
||||
"""Notify LMCache server to remove the session for a finished request.
|
||||
|
||||
Args:
|
||||
request_id: The ID of the finished request.
|
||||
|
||||
"""
|
||||
send_lmcache_request(
|
||||
self.mq_client,
|
||||
@@ -319,7 +320,7 @@ class LMCacheMPSchedulerAdapter:
|
||||
end: int = 0,
|
||||
request_id: str | None = None,
|
||||
) -> IPCCacheEngineKey:
|
||||
"""Convert token IDs to an IPC cache engine key"""
|
||||
"""Convert token IDs to an IPC cache engine key."""
|
||||
return IPCCacheEngineKey(
|
||||
model_name=self.model_name,
|
||||
world_size=self.world_size,
|
||||
@@ -334,7 +335,7 @@ class LMCacheMPSchedulerAdapter:
|
||||
def _create_hash_key(
|
||||
self, chunk_hash: bytes, request_id: str | None = None
|
||||
) -> IPCCacheEngineKey:
|
||||
"""Create a hash-mode IPC cache engine key"""
|
||||
"""Create a hash-mode IPC cache engine key."""
|
||||
return IPCCacheEngineKey(
|
||||
model_name=self.model_name,
|
||||
world_size=self.world_size,
|
||||
@@ -411,12 +412,12 @@ class LMCacheMPWorkerAdapter:
|
||||
)
|
||||
|
||||
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
|
||||
"""
|
||||
Register the kv caches with LMCache server
|
||||
"""Register the kv caches with LMCache server.
|
||||
|
||||
Args:
|
||||
kv_caches: A dict of kv caches to register. The keys are the
|
||||
layer names and the values are the corresponding tensors.
|
||||
|
||||
"""
|
||||
# Register kv cache and send the request
|
||||
self.kv_caches = kv_caches
|
||||
@@ -432,14 +433,14 @@ class LMCacheMPWorkerAdapter:
|
||||
def submit_store_request(
|
||||
self, request_id: str, op: LoadStoreOp, event: torch.cuda.Event
|
||||
):
|
||||
"""
|
||||
Submit a KV cache store request to LMCache
|
||||
"""Submit a KV cache store request to LMCache.
|
||||
|
||||
Args:
|
||||
request_id: The ID of the request
|
||||
op: The LoadStoreOp describing the store operation.
|
||||
event: The CUDA event that is recorded after the current
|
||||
model inference step
|
||||
|
||||
"""
|
||||
if op.block_hashes is not None:
|
||||
# Hash mode
|
||||
@@ -466,14 +467,14 @@ class LMCacheMPWorkerAdapter:
|
||||
def submit_retrieve_request(
|
||||
self, request_id: str, op: LoadStoreOp, event: torch.cuda.Event
|
||||
):
|
||||
"""
|
||||
Submit a KV cache retrieve request to LMCache
|
||||
"""Submit a KV cache retrieve request to LMCache.
|
||||
|
||||
Args:
|
||||
request_id: The ID of the request
|
||||
op: The LoadStoreOp describing the retrieve operation.
|
||||
event: The CUDA event that is recorded after the current
|
||||
model inference step
|
||||
|
||||
"""
|
||||
if op.block_hashes is not None:
|
||||
# Hash mode
|
||||
@@ -503,8 +504,7 @@ class LMCacheMPWorkerAdapter:
|
||||
ops: list[LoadStoreOp],
|
||||
event: torch.cuda.Event,
|
||||
):
|
||||
"""
|
||||
Submit a batched store request to LMCache
|
||||
"""Submit a batched store request to LMCache.
|
||||
|
||||
Args:
|
||||
request_ids: The IDs of the requests
|
||||
@@ -512,6 +512,7 @@ class LMCacheMPWorkerAdapter:
|
||||
the same length as request_ids
|
||||
event: The CUDA event that is recorded after the current
|
||||
model inference step
|
||||
|
||||
"""
|
||||
all_keys: list[IPCCacheEngineKey] = []
|
||||
block_ids: list[int] = []
|
||||
@@ -552,8 +553,7 @@ class LMCacheMPWorkerAdapter:
|
||||
ops: list[LoadStoreOp],
|
||||
event: torch.cuda.Event,
|
||||
):
|
||||
"""
|
||||
Submit a batched retrieve request to LMCache
|
||||
"""Submit a batched retrieve request to LMCache.
|
||||
|
||||
Args:
|
||||
request_ids: The IDs of the requests
|
||||
@@ -561,6 +561,7 @@ class LMCacheMPWorkerAdapter:
|
||||
the same length as request_ids
|
||||
event: The CUDA event that is recorded after the current
|
||||
model inference step
|
||||
|
||||
"""
|
||||
all_keys: list[IPCCacheEngineKey] = []
|
||||
block_ids: list[int] = []
|
||||
@@ -598,8 +599,7 @@ class LMCacheMPWorkerAdapter:
|
||||
def get_finished(
|
||||
self, finished_req_ids_from_engine: set[str]
|
||||
) -> tuple[set[str] | None, set[str] | None]:
|
||||
"""
|
||||
Check and get the finished store and retrieve requests.
|
||||
"""Check and get the finished store and retrieve requests.
|
||||
|
||||
Args:
|
||||
finished_req_ids_from_engine: the set of request ids that are
|
||||
@@ -617,6 +617,7 @@ class LMCacheMPWorkerAdapter:
|
||||
multiple times in `finished_req_ids_from_engine`. The adapter should
|
||||
take care of deduplicating the request IDs and only return the request
|
||||
IDs that have not been returned before.
|
||||
|
||||
"""
|
||||
finished_stores = set()
|
||||
finished_retrieves = set()
|
||||
@@ -675,16 +676,14 @@ class LMCacheMPWorkerAdapter:
|
||||
return ret_stores, finished_retrieves
|
||||
|
||||
def num_blocks_per_chunk(self) -> int:
|
||||
"""
|
||||
Returns:
|
||||
The number of vllm blocks in a LMCache data chunk
|
||||
"""Returns:
|
||||
The number of vllm blocks in a LMCache data chunk
|
||||
|
||||
"""
|
||||
return self.blocks_in_chunk
|
||||
|
||||
def shutdown(self):
|
||||
"""
|
||||
Shutdown the LMCache MP worker adapter
|
||||
"""
|
||||
"""Shutdown the LMCache MP worker adapter."""
|
||||
logger.info("Unregistering kv caches")
|
||||
send_lmcache_request(
|
||||
self.mq_client, RequestType.UNREGISTER_KV_CACHE, [self.instance_id]
|
||||
@@ -712,7 +711,7 @@ class LMCacheMPWorkerAdapter:
|
||||
end: int = 0,
|
||||
request_id: str | None = None,
|
||||
) -> IPCCacheEngineKey:
|
||||
"""Convert token IDs to an IPC cache engine key"""
|
||||
"""Convert token IDs to an IPC cache engine key."""
|
||||
return IPCCacheEngineKey(
|
||||
model_name=self.model_name,
|
||||
world_size=self.world_size,
|
||||
@@ -726,7 +725,7 @@ class LMCacheMPWorkerAdapter:
|
||||
def _create_hash_key(
|
||||
self, chunk_hash: bytes, request_id: str | None = None
|
||||
) -> IPCCacheEngineKey:
|
||||
"""Create a hash-mode IPC cache engine key"""
|
||||
"""Create a hash-mode IPC cache engine key."""
|
||||
return IPCCacheEngineKey(
|
||||
model_name=self.model_name,
|
||||
world_size=self.world_size,
|
||||
|
||||
@@ -59,9 +59,7 @@ def lmcache_get_or_create_config() -> V1Config:
|
||||
|
||||
|
||||
def hex_hash_to_int16(s: str) -> int:
|
||||
"""
|
||||
Convert a hex hash string to a 16-bit integer.
|
||||
"""
|
||||
"""Convert a hex hash string to a 16-bit integer."""
|
||||
return int(s, 16) & 0xFFFF
|
||||
|
||||
|
||||
@@ -70,8 +68,7 @@ def apply_mm_hashes_to_token_ids(
|
||||
mm_hashes: list[str],
|
||||
mm_positions: list["PlaceholderRange"],
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Overwrite token_ids in-place for multimodal placeholders using
|
||||
"""Overwrite token_ids in-place for multimodal placeholders using
|
||||
efficient slice assignments.
|
||||
"""
|
||||
n = token_ids.size(0)
|
||||
@@ -95,8 +92,7 @@ def mla_enabled(model_config: "ModelConfig") -> bool:
|
||||
def extract_mm_features(
|
||||
request: Union["Request", "NewRequestData"], modify: bool = False
|
||||
) -> tuple[list[str], list["PlaceholderRange"]]:
|
||||
"""
|
||||
Normalize multimodal information from a Request into parallel lists.
|
||||
"""Normalize multimodal information from a Request into parallel lists.
|
||||
|
||||
This helper reads either:
|
||||
1) `request.mm_features` (objects each exposing `.identifier` and
|
||||
@@ -119,6 +115,7 @@ def extract_mm_features(
|
||||
Returns:
|
||||
tuple[list[str], list[PlaceholderRange]]: (`mm_hashes`, `mm_positions`).
|
||||
May be `([], [])` when no multimodal data is present.
|
||||
|
||||
"""
|
||||
if getattr(request, "mm_features", None):
|
||||
mm_hashes, mm_positions = zip(
|
||||
|
||||
+13
-8
@@ -171,6 +171,7 @@ class RequestTracker:
|
||||
lmcache_cached_tokens (int): the number of tokens that are
|
||||
cached in LMCache.
|
||||
skip_save (bool): whether the request cache should be saved
|
||||
|
||||
"""
|
||||
# vLLM 0.9.0 update: request.block_ids changed from list[int] to
|
||||
# list[list[int]]
|
||||
@@ -219,7 +220,6 @@ class RequestTracker:
|
||||
"""Update the request tracker when a running request is
|
||||
scheduled again
|
||||
"""
|
||||
|
||||
self.token_ids.extend(new_token_ids)
|
||||
|
||||
if new_block_ids is None:
|
||||
@@ -288,6 +288,7 @@ class ReqMeta:
|
||||
Returns:
|
||||
the request metadata if we need to perform load/save
|
||||
operations, None otherwise.
|
||||
|
||||
"""
|
||||
input_token_ids = tracker.token_ids
|
||||
input_token_len = len(input_token_ids)
|
||||
@@ -445,6 +446,7 @@ def _init_lmcache_engine(
|
||||
|
||||
Returns:
|
||||
The initialized LMCache engine
|
||||
|
||||
"""
|
||||
if curr_engine := LMCacheEngineBuilder.get(ENGINE_NAME):
|
||||
return curr_engine
|
||||
@@ -563,6 +565,7 @@ class LMCacheConnectorMetadata(KVConnectorMetadata):
|
||||
|
||||
Args:
|
||||
req_meta (ReqMeta): the request metadata.
|
||||
|
||||
"""
|
||||
self.requests.append(req_meta)
|
||||
|
||||
@@ -719,6 +722,7 @@ class LMCacheConnectorV1Impl:
|
||||
|
||||
Returns:
|
||||
dict: Dictionary containing inference information
|
||||
|
||||
"""
|
||||
# Get vLLM config information
|
||||
vllm_config = self._vllm_config
|
||||
@@ -766,6 +770,7 @@ class LMCacheConnectorV1Impl:
|
||||
|
||||
Returns:
|
||||
str: vLLM version string
|
||||
|
||||
"""
|
||||
return VLLM_VERSION
|
||||
|
||||
@@ -816,6 +821,7 @@ class LMCacheConnectorV1Impl:
|
||||
Note:
|
||||
The number of elements in kv_caches and layer_names should be
|
||||
the same.
|
||||
|
||||
"""
|
||||
if len(self.kv_caches) == 0:
|
||||
self._init_kv_caches_from_forward_context(forward_context)
|
||||
@@ -920,6 +926,7 @@ class LMCacheConnectorV1Impl:
|
||||
|
||||
Args:
|
||||
layer_name: the name of that layer
|
||||
|
||||
"""
|
||||
if self.layerwise_retrievers:
|
||||
logger.debug("Waiting for layer %s to be loaded", self.current_layer)
|
||||
@@ -951,6 +958,7 @@ class LMCacheConnectorV1Impl:
|
||||
kv_layer (torch.Tensor): the paged KV buffer of the current
|
||||
layer in vLLM.
|
||||
attn_metadata (AttentionMetadata): the attention metadata.
|
||||
|
||||
"""
|
||||
assert self.lmcache_engine is not None
|
||||
|
||||
@@ -1039,7 +1047,6 @@ class LMCacheConnectorV1Impl:
|
||||
@_lmcache_nvtx_annotate
|
||||
def wait_for_save(self):
|
||||
"""Blocking until the KV cache is saved to the connector buffer."""
|
||||
|
||||
connector_metadata = self._parent._get_connector_metadata()
|
||||
assert isinstance(connector_metadata, LMCacheConnectorMetadata)
|
||||
|
||||
@@ -1150,8 +1157,7 @@ class LMCacheConnectorV1Impl:
|
||||
request: "Request",
|
||||
num_computed_tokens: int,
|
||||
) -> int | None:
|
||||
"""
|
||||
Check for external KV cache hit.
|
||||
"""Check for external KV cache hit.
|
||||
|
||||
Args:
|
||||
request (Request): the request object.
|
||||
@@ -1161,6 +1167,7 @@ class LMCacheConnectorV1Impl:
|
||||
Returns:
|
||||
the number of tokens that can be loaded from the
|
||||
external KV cache beyond what is already computed.
|
||||
|
||||
"""
|
||||
if self.kv_role == "kv_producer" and not hasattr(
|
||||
self.lookup_client, "supports_producer_reuse"
|
||||
@@ -1236,13 +1243,11 @@ class LMCacheConnectorV1Impl:
|
||||
|
||||
@_lmcache_nvtx_annotate
|
||||
def update_state_after_alloc(self, request: "Request", num_external_tokens: int):
|
||||
"""
|
||||
Update KVConnector state after temporary buffer alloc.
|
||||
"""Update KVConnector state after temporary buffer alloc.
|
||||
|
||||
For SharedStorageConnector, update _request_needs_load
|
||||
if the CacheManager this allocated blocks for us.
|
||||
"""
|
||||
|
||||
# Clear local status in lookup client when a new request is
|
||||
# successfully scheduled.
|
||||
self.lookup_client.clear_lookup_status(request.request_id)
|
||||
@@ -1311,8 +1316,8 @@ class LMCacheConnectorV1Impl:
|
||||
|
||||
Args:
|
||||
scheduler_output (SchedulerOutput): the scheduler output object.
|
||||
"""
|
||||
|
||||
"""
|
||||
force_skip_save = self.kv_role == "kv_consumer" or self.force_skip_save
|
||||
|
||||
meta = LMCacheConnectorMetadata()
|
||||
|
||||
@@ -88,9 +88,7 @@ def extract_world_size_and_kv_rank(
|
||||
rank: int,
|
||||
vllm_config: VllmConfig,
|
||||
) -> tuple[int, int]:
|
||||
"""
|
||||
Convert the rank for the MLA.
|
||||
"""
|
||||
"""Convert the rank for the MLA."""
|
||||
use_mla = mla_enabled(vllm_config.model_config)
|
||||
if not use_mla:
|
||||
return world_size, rank
|
||||
@@ -174,8 +172,7 @@ def create_worker_adapter(
|
||||
|
||||
|
||||
class LMCacheMPRequestState(enum.Enum):
|
||||
"""
|
||||
State machine:
|
||||
"""State machine:
|
||||
PREFETCHING -- update_state_after_alloc --> WAITING_FOR_LOAD
|
||||
WAITING_FOR_LOAD -- process_loading_requests --> READY
|
||||
"""
|
||||
@@ -234,7 +231,8 @@ class LMCacheMPRequestTracker:
|
||||
####
|
||||
def needs_retrieve(self) -> bool:
|
||||
"""Check whether the current request needs retrieve, will be used
|
||||
update_stage_after_alloc"""
|
||||
update_stage_after_alloc
|
||||
"""
|
||||
return (
|
||||
self.num_lmcache_hit_blocks > self.num_vllm_hit_blocks
|
||||
and self.state != LMCacheMPRequestState.READY
|
||||
@@ -242,7 +240,8 @@ class LMCacheMPRequestTracker:
|
||||
|
||||
def is_ready_for_retrieving(self) -> bool:
|
||||
"""Check whether the current request is ready for retrieving,
|
||||
will be used in process_loading_requests"""
|
||||
will be used in process_loading_requests
|
||||
"""
|
||||
return (
|
||||
self.state == LMCacheMPRequestState.WAITING_FOR_LOAD
|
||||
and self.needs_retrieve()
|
||||
@@ -301,13 +300,13 @@ class LMCacheMPRequestMetadata:
|
||||
blocks_in_chunk: int,
|
||||
vllm_block_size: int,
|
||||
) -> "LMCacheMPRequestMetadata | None":
|
||||
"""
|
||||
Generate the store metadata for the current request tracker.
|
||||
"""Generate the store metadata for the current request tracker.
|
||||
|
||||
Args:
|
||||
tracker: The request tracker to generate the metadata from.
|
||||
blocks_in_chunk: the number of blocks in a LMCache data chunk
|
||||
vllm_block_size: the block size used in vLLM
|
||||
|
||||
"""
|
||||
# Store the blocks that has block hashes
|
||||
# NOTE: the invariant here is that `num_stored_blocks` should
|
||||
@@ -375,13 +374,13 @@ class LMCacheMPRequestMetadata:
|
||||
blocks_in_chunk: int,
|
||||
vllm_block_size: int,
|
||||
) -> "LMCacheMPRequestMetadata | None":
|
||||
"""
|
||||
Generate the retrieve metadata for the current request tracker.
|
||||
"""Generate the retrieve metadata for the current request tracker.
|
||||
|
||||
Args:
|
||||
tracker: The request tracker to generate the metadata from.
|
||||
blocks_in_chunk: the number of blocks in a LMCache data chunk
|
||||
vllm_block_size: the block size used in vLLM
|
||||
|
||||
"""
|
||||
if not tracker.is_ready_for_retrieving():
|
||||
return None
|
||||
@@ -462,8 +461,7 @@ class LMCacheMPConnectorMetadata(KVConnectorMetadata):
|
||||
|
||||
|
||||
class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
|
||||
"""
|
||||
The connector for LMCache multi-process mode.
|
||||
"""The connector for LMCache multi-process mode.
|
||||
|
||||
Extra configs (kv_transfer_config.extra_config):
|
||||
- lmcache.mp.host: the host of the LMCache server.
|
||||
@@ -538,27 +536,26 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
|
||||
|
||||
Returns:
|
||||
ConnectorMetadata: the connector metadata.
|
||||
"""
|
||||
|
||||
"""
|
||||
# Should only be called while set to valid metadata.
|
||||
assert self._connector_metadata is not None
|
||||
return self._connector_metadata
|
||||
|
||||
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
|
||||
"""
|
||||
Initialize with the KV caches. Useful for pre-registering the
|
||||
"""Initialize with the KV caches. Useful for pre-registering the
|
||||
KV Caches in the KVConnector (e.g. for NIXL).
|
||||
|
||||
Args:
|
||||
kv_caches: dictionary of layer names, kv cache
|
||||
|
||||
"""
|
||||
logger.info("Registering kv caches!")
|
||||
self.worker_adapter.register_kv_caches(kv_caches)
|
||||
return
|
||||
|
||||
def start_load_kv(self, forward_context: "ForwardContext", **kwargs: Any) -> None:
|
||||
"""
|
||||
Start loading the KV cache from the connector to vLLM's paged
|
||||
"""Start loading the KV cache from the connector to vLLM's paged
|
||||
KV buffer. This is called from the forward context before the
|
||||
forward pass to enable async loading during model execution.
|
||||
|
||||
@@ -597,8 +594,7 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
|
||||
)
|
||||
|
||||
def wait_for_layer_load(self, layer_name: str) -> None:
|
||||
"""
|
||||
Block until the KV for a specific layer is loaded into vLLM's
|
||||
"""Block until the KV for a specific layer is loaded into vLLM's
|
||||
paged buffer. This is called from within attention layer to ensure
|
||||
async copying from start_load_kv is complete.
|
||||
|
||||
@@ -606,6 +602,7 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
|
||||
|
||||
Args:
|
||||
layer_name: the name of that layer
|
||||
|
||||
"""
|
||||
return
|
||||
|
||||
@@ -616,8 +613,7 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
|
||||
attn_metadata: AttentionMetadata,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
"""
|
||||
Start saving a layer of KV cache from vLLM's paged buffer
|
||||
"""Start saving a layer of KV cache from vLLM's paged buffer
|
||||
to the connector. This is called from within attention layer to
|
||||
enable async copying during execution.
|
||||
|
||||
@@ -627,12 +623,12 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
|
||||
layer in vLLM.
|
||||
attn_metadata (AttentionMetadata): the attention metadata.
|
||||
**kwargs: additional arguments for the save operation.
|
||||
|
||||
"""
|
||||
return
|
||||
|
||||
def wait_for_save(self):
|
||||
"""
|
||||
Block until all the save operations is done. This is called
|
||||
"""Block until all the save operations is done. This is called
|
||||
as the forward context exits to ensure that the async saving
|
||||
from save_kv_layer is complete before finishing the forward.
|
||||
|
||||
@@ -673,8 +669,7 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
|
||||
def get_finished(
|
||||
self, finished_req_ids: set[str]
|
||||
) -> tuple[set[str] | None, set[str] | None]:
|
||||
"""
|
||||
Notifies worker-side connector ids of requests that have
|
||||
"""Notifies worker-side connector ids of requests that have
|
||||
finished generating tokens on the worker.
|
||||
The scheduler process (via the Executors) will use this output
|
||||
to track which workers are done.
|
||||
@@ -685,14 +680,14 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
|
||||
tuple of (sending/saving ids, recving/loading ids).
|
||||
The finished saves/sends req ids must belong to a set provided in a
|
||||
call to this method (this call or a prior one).
|
||||
|
||||
"""
|
||||
val = self.worker_adapter.get_finished(finished_req_ids)
|
||||
# logger.error("Finished req ids: %s, %s", val[0], val[1])
|
||||
return val
|
||||
|
||||
def get_block_ids_with_load_errors(self) -> set[int]:
|
||||
"""
|
||||
Get the set of block IDs that failed to load.
|
||||
"""Get the set of block IDs that failed to load.
|
||||
|
||||
Returns:
|
||||
Set of block IDs that encountered load errors.
|
||||
@@ -707,12 +702,12 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
|
||||
block IDs must appear here no later than that same pass.
|
||||
- Sync loading: failed blocks should be reported in the forward
|
||||
pass in which they are detected.
|
||||
|
||||
"""
|
||||
return self.worker_adapter.get_block_ids_with_load_errors()
|
||||
|
||||
def shutdown(self):
|
||||
"""
|
||||
Shutdown the connector. This is called when the worker process
|
||||
"""Shutdown the connector. This is called when the worker process
|
||||
is shutting down to ensure that all the async operations are
|
||||
completed and the connector is cleaned up properly.
|
||||
"""
|
||||
@@ -721,9 +716,7 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
|
||||
return None
|
||||
|
||||
def get_kv_connector_stats(self) -> "KVConnectorStats | None":
|
||||
"""
|
||||
Get the KV connector stats collected during the last interval.
|
||||
"""
|
||||
"""Get the KV connector stats collected during the last interval."""
|
||||
return None
|
||||
|
||||
# ==============================
|
||||
@@ -735,8 +728,7 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
|
||||
request: "Request",
|
||||
num_computed_tokens: int,
|
||||
) -> tuple[int | None, bool]:
|
||||
"""
|
||||
Get number of new tokens that can be loaded from the
|
||||
"""Get number of new tokens that can be loaded from the
|
||||
external KV cache beyond the num_computed_tokens.
|
||||
|
||||
Args:
|
||||
@@ -761,6 +753,7 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
|
||||
call. If the cache cannot be loaded for some tokens (e.g., due to
|
||||
connectivity issues or eviction), those tokens must not be taken
|
||||
into account.
|
||||
|
||||
"""
|
||||
tracker = self._get_or_create_request_tracker(request)
|
||||
# TODO: support loading KV for preempted requests in the future
|
||||
@@ -803,8 +796,7 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
|
||||
def update_state_after_alloc(
|
||||
self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int
|
||||
):
|
||||
"""
|
||||
Update KVConnector state after block allocation.
|
||||
"""Update KVConnector state after block allocation.
|
||||
|
||||
If get_num_new_matched_tokens previously returned True for a
|
||||
request, this function may be called twice for that same request -
|
||||
@@ -817,6 +809,7 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
|
||||
blocks (KVCacheBlocks): the blocks allocated for the request.
|
||||
num_external_tokens (int): the number of tokens that will be
|
||||
loaded from the external KV cache.
|
||||
|
||||
"""
|
||||
# NOTE: `blocks` comes from kv_cache_manager.get_blocks(request_id),
|
||||
# which returns ALL blocks for the request (not just newly allocated).
|
||||
@@ -883,14 +876,14 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
|
||||
def build_connector_meta(
|
||||
self, scheduler_output: SchedulerOutput
|
||||
) -> KVConnectorMetadata:
|
||||
"""
|
||||
Build the connector metadata for this step.
|
||||
"""Build the connector metadata for this step.
|
||||
|
||||
This function should NOT modify fields in the scheduler_output.
|
||||
Also, calling this function will reset the state of the connector.
|
||||
|
||||
Args:
|
||||
scheduler_output (SchedulerOutput): the scheduler output object.
|
||||
|
||||
"""
|
||||
metadata = LMCacheMPConnectorMetadata()
|
||||
|
||||
@@ -907,12 +900,12 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
|
||||
return metadata
|
||||
|
||||
def update_connector_output(self, connector_output: KVConnectorOutput):
|
||||
"""
|
||||
Update KVConnector state from worker-side connectors output.
|
||||
"""Update KVConnector state from worker-side connectors output.
|
||||
|
||||
Args:
|
||||
connector_output (KVConnectorOutput): the worker-side
|
||||
connectors output.
|
||||
|
||||
"""
|
||||
return
|
||||
|
||||
@@ -921,8 +914,7 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
|
||||
request: "Request",
|
||||
block_ids: list[int],
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
"""
|
||||
Called exactly once when a request has finished, before its blocks are
|
||||
"""Called exactly once when a request has finished, before its blocks are
|
||||
freed.
|
||||
|
||||
The connector may assumes responsibility for freeing the blocks
|
||||
@@ -934,8 +926,8 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
|
||||
get_finished().
|
||||
Optional KVTransferParams to be included in the request outputs
|
||||
returned by the engine.
|
||||
"""
|
||||
|
||||
"""
|
||||
params: dict[str, Any] | None = getattr(request, "kv_transfer_params", None)
|
||||
return_params: dict[str, Any] | None = {} if params is not None else None
|
||||
|
||||
@@ -962,26 +954,26 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
|
||||
return True, return_params
|
||||
|
||||
def take_events(self) -> Iterable["KVCacheEvent"]:
|
||||
"""
|
||||
Take the KV cache events from the connector.
|
||||
"""Take the KV cache events from the connector.
|
||||
|
||||
Yields:
|
||||
New KV cache events since the last call.
|
||||
|
||||
"""
|
||||
return ()
|
||||
|
||||
@classmethod
|
||||
def get_required_kvcache_layout(cls, vllm_config: "VllmConfig") -> str | None:
|
||||
"""
|
||||
Get the required KV cache layout for this connector.
|
||||
"""Get the required KV cache layout for this connector.
|
||||
|
||||
Args:
|
||||
vllm_config (VllmConfig): the vllm config.
|
||||
|
||||
Returns:
|
||||
str: the required KV cache layout. e.g. HND, or NHD.
|
||||
None if the connector does not require a specific layout.
|
||||
"""
|
||||
|
||||
"""
|
||||
if cls is KVConnectorBase_V1:
|
||||
raise TypeError(
|
||||
"get_required_kvcache_layout should not be called "
|
||||
@@ -990,13 +982,13 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
|
||||
return None
|
||||
|
||||
def get_finished_count(self) -> int | None:
|
||||
"""
|
||||
Get the count of requests expected to complete send/receive operations
|
||||
"""Get the count of requests expected to complete send/receive operations
|
||||
via this connector. This method is used to initialize the
|
||||
KVOutputAggregator, overwriting the default world_size.
|
||||
|
||||
Returns:
|
||||
int: expected sending or receiving completion count.
|
||||
|
||||
"""
|
||||
return None
|
||||
|
||||
@@ -1004,8 +996,7 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
|
||||
def build_kv_connector_stats(
|
||||
cls, data: dict[str, Any] | None = None
|
||||
) -> "KVConnectorStats | None":
|
||||
"""
|
||||
KVConnectorStats resolution method. This method allows dynamically
|
||||
"""KVConnectorStats resolution method. This method allows dynamically
|
||||
registered connectors to return their own KVConnectorStats object,
|
||||
which can implement custom aggregation logic on the data dict.
|
||||
"""
|
||||
@@ -1019,8 +1010,7 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
|
||||
labelnames: list[str],
|
||||
per_engine_labelvalues: dict[int, list[object]],
|
||||
) -> "KVConnectorPromMetrics | None":
|
||||
"""
|
||||
Create a KVConnectorPromMetrics subclass which should register
|
||||
"""Create a KVConnectorPromMetrics subclass which should register
|
||||
per-connector Prometheus metrics and implement observe() to
|
||||
expose connector transfer stats via Prometheus.
|
||||
"""
|
||||
@@ -1183,8 +1173,7 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
|
||||
return self.request_trackers[request_id]
|
||||
|
||||
def _cleanup_request_tracker(self, request_id: str) -> None:
|
||||
"""
|
||||
Clean up request tracker and associated lookup future for a request.
|
||||
"""Clean up request tracker and associated lookup future for a request.
|
||||
This should be called when a request is finished to prevent memory leak.
|
||||
"""
|
||||
# Clean up request tracker
|
||||
|
||||
@@ -17,8 +17,7 @@ logger = init_logger(__name__)
|
||||
|
||||
@dataclass
|
||||
class KVConnectorStats:
|
||||
"""
|
||||
Base class for KV Connector Stats, a container for transfer performance
|
||||
"""Base class for KV Connector Stats, a container for transfer performance
|
||||
metrics or otherwise important telemetry from the connector.
|
||||
All sub-classes need to be serializable as stats are sent from worker to
|
||||
logger process.
|
||||
@@ -35,14 +34,11 @@ class KVConnectorStats:
|
||||
raise NotImplementedError
|
||||
|
||||
def aggregate(self, other: "KVConnectorStats") -> "KVConnectorStats":
|
||||
"""
|
||||
Aggregate stats with another `KVConnectorStats` object.
|
||||
"""
|
||||
"""Aggregate stats with another `KVConnectorStats` object."""
|
||||
raise NotImplementedError
|
||||
|
||||
def reduce(self) -> dict[str, int | float]:
|
||||
"""
|
||||
Reduce the observations collected during a time interval to one or
|
||||
"""Reduce the observations collected during a time interval to one or
|
||||
more representative values (eg avg/median/sum of the series).
|
||||
This is meant to be called by the logger to produce a summary of the
|
||||
stats for the last time interval.
|
||||
@@ -95,7 +91,7 @@ class KVConnectorLogging:
|
||||
)
|
||||
|
||||
def log(self, log_fn=logger.info):
|
||||
"""Log transfer metrics periodically, similar to throughput logging"""
|
||||
"""Log transfer metrics periodically, similar to throughput logging."""
|
||||
if (
|
||||
self.transfer_stats_accumulator
|
||||
and not self.transfer_stats_accumulator.is_empty()
|
||||
@@ -111,8 +107,7 @@ class KVConnectorLogging:
|
||||
|
||||
|
||||
class KVConnectorPromMetrics:
|
||||
"""
|
||||
A base class for per-connector Prometheus metric registration
|
||||
"""A base class for per-connector Prometheus metric registration
|
||||
and recording.
|
||||
"""
|
||||
|
||||
@@ -131,8 +126,7 @@ class KVConnectorPromMetrics:
|
||||
self.per_engine_labelvalues = per_engine_labelvalues
|
||||
|
||||
def observe(self, transfer_stats_data: dict[str, Any], engine_idx: int = 0):
|
||||
"""
|
||||
Record the supplied transfer statistics to Prometheus metrics. These
|
||||
"""Record the supplied transfer statistics to Prometheus metrics. These
|
||||
statistics are engine-specific, and should be recorded to a metric
|
||||
with the appropriate 'engine' label. These metric instances can be
|
||||
created using the create_metric_per_engine() helper method.
|
||||
@@ -141,8 +135,7 @@ class KVConnectorPromMetrics:
|
||||
|
||||
|
||||
class KVConnectorProm:
|
||||
"""
|
||||
Support for registering per-connector Prometheus metrics, and
|
||||
"""Support for registering per-connector Prometheus metrics, and
|
||||
recording transfer statistics to those metrics. Uses
|
||||
KVConnectorBase.build_prom_metrics().
|
||||
"""
|
||||
|
||||
@@ -643,7 +643,7 @@ class MooncakeConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
|
||||
|
||||
class MooncakeConnectorScheduler:
|
||||
"""Implementation of Scheduler side methods"""
|
||||
"""Implementation of Scheduler side methods."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -718,7 +718,8 @@ class MooncakeConnectorScheduler:
|
||||
|
||||
def _get_remote_prefill_token_count(self, num_prompt_tokens: int) -> int:
|
||||
"""D-side only. Returns N-1 for Mamba models since the decoder
|
||||
always recomputes the last token and must start from h(N-1)."""
|
||||
always recomputes the last token and must start from h(N-1).
|
||||
"""
|
||||
if self._has_mamba and num_prompt_tokens > 1:
|
||||
return num_prompt_tokens - 1
|
||||
return num_prompt_tokens
|
||||
@@ -729,7 +730,8 @@ class MooncakeConnectorScheduler:
|
||||
derive h(N) correctly.
|
||||
|
||||
Guarded by ``_p_side_truncated`` to avoid repeated truncation if the
|
||||
request is preempted and rescheduled."""
|
||||
request is preempted and rescheduled.
|
||||
"""
|
||||
params = request.kv_transfer_params
|
||||
if (
|
||||
params is not None
|
||||
@@ -756,8 +758,7 @@ class MooncakeConnectorScheduler:
|
||||
def get_num_new_matched_tokens(
|
||||
self, request: "Request", num_computed_tokens: int
|
||||
) -> tuple[int, bool]:
|
||||
"""
|
||||
For remote prefill, pull all prompt blocks from remote
|
||||
"""For remote prefill, pull all prompt blocks from remote
|
||||
asynchronously relative to engine execution.
|
||||
|
||||
Args:
|
||||
@@ -769,8 +770,8 @@ class MooncakeConnectorScheduler:
|
||||
external KV cache beyond what is already computed.
|
||||
* true if the external KV cache tokens will be loaded
|
||||
asynchronously (between scheduler steps).
|
||||
"""
|
||||
|
||||
"""
|
||||
params = request.kv_transfer_params
|
||||
logger.debug(
|
||||
"MooncakeConnector get_num_new_matched_tokens: "
|
||||
@@ -881,11 +882,9 @@ class MooncakeConnectorScheduler:
|
||||
request: "Request",
|
||||
block_ids: tuple[list[int], ...],
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
"""
|
||||
Once a request is finished, determine whether request blocks
|
||||
"""Once a request is finished, determine whether request blocks
|
||||
should be freed now or will be sent asynchronously and freed later.
|
||||
"""
|
||||
|
||||
params = request.kv_transfer_params
|
||||
logger.debug(
|
||||
"MooncakeConnector request_finished, req_id=%s, request_status=%s, "
|
||||
@@ -934,7 +933,7 @@ class MooncakeConnectorScheduler:
|
||||
|
||||
|
||||
class MooncakeConnectorWorker:
|
||||
"""Implementation of Worker side methods"""
|
||||
"""Implementation of Worker side methods."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -1176,11 +1175,9 @@ class MooncakeConnectorWorker:
|
||||
raise e
|
||||
|
||||
async def _mooncake_sender_listener(self, ready_event: threading.Event):
|
||||
"""
|
||||
Background thread that listens for Mooncake requests, dispatches them
|
||||
"""Background thread that listens for Mooncake requests, dispatches them
|
||||
to a thread pool, and sends acknowledgments upon completion.
|
||||
"""
|
||||
|
||||
sock = self.async_zmq_ctx.socket(zmq.ROUTER)
|
||||
self.side_channel_port = sock.bind_to_random_port(f"tcp://{self.hostname}")
|
||||
logger.debug(
|
||||
@@ -1680,7 +1677,8 @@ class MooncakeConnectorWorker:
|
||||
def _bind_sender_thread_device(self) -> None:
|
||||
"""ThreadPoolExecutor initializer — binds each pool thread to the
|
||||
correct CUDA device. CUDA device selection is thread-local, so
|
||||
without this, NVLink transfers fail for TP ranks > 0."""
|
||||
without this, NVLink transfers fail for TP ranks > 0.
|
||||
"""
|
||||
current_platform.set_device(self.device_id)
|
||||
|
||||
def _send_blocks(
|
||||
@@ -1716,7 +1714,6 @@ class MooncakeConnectorWorker:
|
||||
|
||||
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
|
||||
"""Register the KV Cache data in mooncake."""
|
||||
|
||||
logger.info("Registering KV_Caches. use_mla: %s", self.use_mla)
|
||||
|
||||
kv_data_ptrs: list[int] = []
|
||||
@@ -1853,8 +1850,7 @@ class MooncakeConnectorWorker:
|
||||
return finished_sending_reqs
|
||||
|
||||
def get_finished(self) -> tuple[set[str] | None, set[str] | None]:
|
||||
"""
|
||||
Get requests that are done sending or recving on this specific worker.
|
||||
"""Get requests that are done sending or recving on this specific worker.
|
||||
The scheduler process (via the MultiprocExecutor) will use this output
|
||||
to track which workers are done.
|
||||
"""
|
||||
@@ -1885,8 +1881,7 @@ class MooncakeConnectorWorker:
|
||||
return finished_sending_reqs or None, finished_recving_reqs or None
|
||||
|
||||
def get_transfer_results(self) -> KVConnectorTransferResults:
|
||||
"""
|
||||
Get transfers that completed on this specific worker, including
|
||||
"""Get transfers that completed on this specific worker, including
|
||||
requests whose remote KV load failed.
|
||||
|
||||
The scheduler process (via the MultiprocExecutor) will use this output
|
||||
@@ -1909,7 +1904,8 @@ class MooncakeConnectorWorker:
|
||||
|
||||
def get_kv_connector_stats(self) -> KVConnectorStats | None:
|
||||
"""Return transfer stats collected since the last call, or None
|
||||
if nothing has been recorded in this interval."""
|
||||
if nothing has been recorded in this interval.
|
||||
"""
|
||||
if self.xfer_stats.is_empty():
|
||||
return None
|
||||
return self.xfer_stats.clone_and_reset()
|
||||
@@ -2321,8 +2317,7 @@ def should_launch_bootstrap_server(vllm_config: VllmConfig) -> bool:
|
||||
|
||||
|
||||
def get_mooncake_bootstrap_addr(vllm_config: VllmConfig) -> tuple[str, int]:
|
||||
"""
|
||||
Returns the address of the Mooncake bootstrap server.
|
||||
"""Returns the address of the Mooncake bootstrap server.
|
||||
This is only used by prefillers to register workers.
|
||||
Decoders should get addr from kv_transfer_params.
|
||||
"""
|
||||
|
||||
@@ -32,8 +32,7 @@ class EngineEntry:
|
||||
|
||||
|
||||
class MooncakeBootstrapServer:
|
||||
"""
|
||||
A centralized server running on the global rank 0 prefiller worker.
|
||||
"""A centralized server running on the global rank 0 prefiller worker.
|
||||
Prefiller workers register their connection info (IP, port, ranks) here.
|
||||
"""
|
||||
|
||||
|
||||
@@ -73,7 +73,8 @@ class ExternalCachedBlockPool:
|
||||
|
||||
class MooncakeStoreCoordinator:
|
||||
"""Mirror of ``HybridKVCacheCoordinator.find_longest_cache_hit`` over an
|
||||
``ExternalCachedBlockPool``."""
|
||||
``ExternalCachedBlockPool``.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
@@ -630,6 +630,7 @@ class ChunkedTokenDatabase:
|
||||
``mask_num``. False entries are skipped before hash access.
|
||||
put_step: Stride for distributing chunks across ranks.
|
||||
put_step_rank: ``chunk_id % put_step`` value this rank stores.
|
||||
|
||||
"""
|
||||
assert put_step > 0
|
||||
if not block_hashes:
|
||||
@@ -662,6 +663,7 @@ class TailKeyBoundary:
|
||||
``block_hashes[num_tokens // hash_block_size - 1]`` instead of the
|
||||
hash implied by ``MooncakeLookupResult.hit_length``. This changes
|
||||
only the load key, not the reusable prefix.
|
||||
|
||||
"""
|
||||
|
||||
group_id: int
|
||||
@@ -678,6 +680,7 @@ class MooncakeLookupResult:
|
||||
tail_key_boundaries: Hash boundary used to store each cache group's
|
||||
tail block when ``hit_length`` does not identify its store key.
|
||||
There is one entry per group for every nonzero hit.
|
||||
|
||||
"""
|
||||
|
||||
hit_length: int
|
||||
|
||||
@@ -815,6 +815,7 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
|
||||
Returns:
|
||||
True when no put is needed or every put succeeds, False otherwise.
|
||||
|
||||
"""
|
||||
offloads = req_meta.boundary_state_offloads
|
||||
if not offloads or not req_meta.block_hashes:
|
||||
@@ -2628,7 +2629,8 @@ class LookupKeyClient:
|
||||
non_block: bool = False,
|
||||
) -> MooncakeLookupResult | None:
|
||||
"""If non_block is True, will return None until the result is ready,
|
||||
so the caller retries on a later step."""
|
||||
so the caller retries on a later step.
|
||||
"""
|
||||
future = self.futures.get(req_id)
|
||||
if future is None:
|
||||
future = self.executor.submit(self._lookup, num_tokens, list(block_hashes))
|
||||
|
||||
@@ -588,8 +588,7 @@ class MoRIIOConnectorMetadata(KVConnectorMetadata):
|
||||
|
||||
@contextlib.contextmanager
|
||||
def zmq_ctx(socket_type: Any, addr: str) -> Iterator[zmq.Socket]:
|
||||
"""Context manager for a ZMQ socket"""
|
||||
|
||||
"""Context manager for a ZMQ socket."""
|
||||
if socket_type not in (zmq.ROUTER, zmq.REQ, zmq.DEALER):
|
||||
raise ValueError(f"Unexpected socket type: {socket_type}")
|
||||
|
||||
|
||||
@@ -373,6 +373,7 @@ class MoRIIOConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
|
||||
Returns:
|
||||
bool: True if connector metadata exists, False otherwise.
|
||||
|
||||
"""
|
||||
try:
|
||||
return self._connector_metadata is not None
|
||||
@@ -381,7 +382,7 @@ class MoRIIOConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
|
||||
|
||||
class MoRIIOConnectorScheduler:
|
||||
"""Implementation of Scheduler side methods"""
|
||||
"""Implementation of Scheduler side methods."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -547,7 +548,8 @@ class MoRIIOConnectorScheduler:
|
||||
self, request_id: ReqId, transfer_id: TransferId | None = None
|
||||
):
|
||||
"""Unmap request_id/transfer_id. Uses transfer_id for lookup if
|
||||
exact request_id match fails (handles input_processor mutation)."""
|
||||
exact request_id match fails (handles input_processor mutation).
|
||||
"""
|
||||
if request_id in self.request_id_to_transfer_id:
|
||||
tid = self.request_id_to_transfer_id[request_id]
|
||||
del self.request_id_to_transfer_id[request_id]
|
||||
@@ -579,8 +581,7 @@ class MoRIIOConnectorScheduler:
|
||||
request: "Request",
|
||||
num_computed_tokens: int,
|
||||
) -> tuple[int, bool]:
|
||||
"""
|
||||
For remote prefill, pull all prompt blocks from remote
|
||||
"""For remote prefill, pull all prompt blocks from remote
|
||||
asynchronously relative to engine execution.
|
||||
|
||||
Args:
|
||||
@@ -592,6 +593,7 @@ class MoRIIOConnectorScheduler:
|
||||
external KV cache beyond what is already computed.
|
||||
* true if the external KV cache tokens will be loaded
|
||||
asynchronously (between scheduler steps).
|
||||
|
||||
"""
|
||||
if self.is_producer:
|
||||
return 0, False
|
||||
@@ -1007,11 +1009,9 @@ class MoRIIOConnectorScheduler:
|
||||
request: "Request",
|
||||
block_ids: BlockIds,
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
"""
|
||||
Once a request is finished, determine whether request blocks
|
||||
"""Once a request is finished, determine whether request blocks
|
||||
should be freed now or will be sent asynchronously and freed later.
|
||||
"""
|
||||
|
||||
request_id = request.request_id
|
||||
params = request.kv_transfer_params
|
||||
# Consumer: can unmap transfer_id<->request_id immediately since done_recving
|
||||
@@ -1177,7 +1177,7 @@ class MoRIIOConnectorScheduler:
|
||||
|
||||
|
||||
class MoRIIOConnectorWorker:
|
||||
"""Implementation of Worker side methods"""
|
||||
"""Implementation of Worker side methods."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -1435,8 +1435,8 @@ class MoRIIOConnectorWorker:
|
||||
kv_layer: KV cache tensor
|
||||
remote_notify_port: Port for completion notification
|
||||
remote_ip: IP address of remote node
|
||||
"""
|
||||
|
||||
"""
|
||||
# synchronization to prevent dirty reads between
|
||||
# transfer and attention operations
|
||||
# we can consider removing this synchronization after ibgda is enabled.
|
||||
@@ -1581,7 +1581,6 @@ class MoRIIOConnectorWorker:
|
||||
layer_name_to_local_kv_cache_metadata: dict,
|
||||
):
|
||||
"""Background thread for getting new MoRIIO handshakes."""
|
||||
|
||||
encoder = msgspec.msgpack.Encoder()
|
||||
encoded_data = encoder.encode(metadata)
|
||||
size_in_bytes = len(encoded_data)
|
||||
@@ -1637,7 +1636,6 @@ class MoRIIOConnectorWorker:
|
||||
local-rank mapping _remote_tp_rank -- byte-identical for callers not yet
|
||||
TP-aware.
|
||||
"""
|
||||
|
||||
start_time = time.perf_counter()
|
||||
|
||||
# NOTE(rob): we need each rank to have a unique port. This is
|
||||
@@ -1856,7 +1854,6 @@ class MoRIIOConnectorWorker:
|
||||
|
||||
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
|
||||
"""Register the KV Cache data in moriio."""
|
||||
|
||||
self.kv_caches = kv_caches # layer name to kv cache
|
||||
self.kv_cache_shapes = {
|
||||
layer_name: kv_cache.shape for layer_name, kv_cache in kv_caches.items()
|
||||
@@ -2020,12 +2017,10 @@ class MoRIIOConnectorWorker:
|
||||
self.moriio_wrapper.async_wait_reqid()
|
||||
|
||||
def get_finished(self) -> tuple[set[str], set[str]]:
|
||||
"""
|
||||
Get requests that are done sending or recving on this specific worker.
|
||||
"""Get requests that are done sending or recving on this specific worker.
|
||||
The scheduler process (via the MultiprocExecutor) will use this output
|
||||
to track which workers are done.
|
||||
"""
|
||||
|
||||
done_sending, done_recving = set(), set()
|
||||
|
||||
if self.is_producer:
|
||||
@@ -2416,8 +2411,7 @@ class MoRIIOConnectorWorker:
|
||||
self._eager_handshaked_engines.add(remote_engine_id)
|
||||
|
||||
def start_load_kv(self, metadata: MoRIIOConnectorMetadata):
|
||||
"""
|
||||
Start loading by triggering non-blocking moriio_xfer.
|
||||
"""Start loading by triggering non-blocking moriio_xfer.
|
||||
We check for these trnxs to complete in each step().
|
||||
"""
|
||||
self.transfer_id_to_request_id = metadata.transfer_id_to_request_id
|
||||
@@ -2686,8 +2680,13 @@ class MoRIIOConnectorWorker:
|
||||
local_block_ids: IDs of local blocks
|
||||
remote_block_ids: IDs of remote blocks
|
||||
remote_moriio_meta: Metadata of the remote MoRIIO agent
|
||||
remote_tp_size: Tensor parallel size of the remote engine. Falls
|
||||
back to the locally recorded value when None.
|
||||
remote_engine_id: Id of the remote engine the blocks live on.
|
||||
|
||||
Returns:
|
||||
Tuple of (local_offsets, remote_offsets, transfer_sizes)
|
||||
|
||||
"""
|
||||
validate_moriio_heterogeneous_tp_kv_heads(
|
||||
local_tp_size=self.world_size,
|
||||
|
||||
@@ -87,6 +87,7 @@ class MoRIIOWriter:
|
||||
|
||||
Args:
|
||||
worker: Reference to the parent worker
|
||||
|
||||
"""
|
||||
self._worker_ref: weakref_ref[MoRIIOConnectorWorker] = weakref_ref(worker)
|
||||
self._write_task_q: Queue[WriteTask] = Queue()
|
||||
@@ -108,6 +109,7 @@ class MoRIIOWriter:
|
||||
|
||||
Raises:
|
||||
RuntimeError: If worker has been garbage collected
|
||||
|
||||
"""
|
||||
worker = self._worker_ref()
|
||||
if worker is None:
|
||||
@@ -131,6 +133,7 @@ class MoRIIOWriter:
|
||||
|
||||
Args:
|
||||
task: The write task to schedule
|
||||
|
||||
"""
|
||||
self.ensure_worker_started()
|
||||
if self._is_transfer_terminal(task.transfer_id):
|
||||
@@ -178,7 +181,6 @@ class MoRIIOWriter:
|
||||
|
||||
def _write_worker_loop(self) -> None:
|
||||
"""Main loop for the write worker thread."""
|
||||
|
||||
while True:
|
||||
# Process deferred tasks first
|
||||
self._process_deferred_tasks()
|
||||
@@ -275,6 +277,7 @@ class MoRIIOWriter:
|
||||
|
||||
Returns:
|
||||
True if remote blocks are ready
|
||||
|
||||
"""
|
||||
return (
|
||||
task.transfer_id in self.worker.moriio_wrapper.done_remote_allocate_req_dict
|
||||
@@ -291,6 +294,7 @@ class MoRIIOWriter:
|
||||
|
||||
Raises:
|
||||
KeyError: If allocation info is missing
|
||||
|
||||
"""
|
||||
try:
|
||||
return self.worker.moriio_wrapper.done_remote_allocate_req_dict[transfer_id]
|
||||
@@ -372,9 +376,11 @@ class MoRIIOWriter:
|
||||
Args:
|
||||
task: The write task
|
||||
request_info: Remote allocation information
|
||||
remote_moriio_meta: Metadata of the remote MoRIIO agent
|
||||
|
||||
Returns:
|
||||
The transfer plan
|
||||
|
||||
"""
|
||||
layer_cache = self.worker.kv_caches[task.layer_name]
|
||||
key = (task.layer_name, *_get_write_geometry_key(layer_cache))
|
||||
@@ -412,6 +418,7 @@ class MoRIIOWriter:
|
||||
Args:
|
||||
plan: The transfer plan
|
||||
sessions: List of transfer sessions
|
||||
|
||||
"""
|
||||
if plan.use_batch:
|
||||
return [
|
||||
@@ -510,6 +517,7 @@ class MoRIIOWrapper:
|
||||
moriio_engine: MoRIIO engine instance
|
||||
tp_rank: Tensor parallel rank
|
||||
dp_rank: Data parallel rank
|
||||
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
|
||||
@@ -71,8 +71,7 @@ class MultiKVConnectorWorkerMetadata(KVConnectorWorkerMetadata):
|
||||
|
||||
@dataclass
|
||||
class MultiKVConnectorStats(KVConnectorStats):
|
||||
"""
|
||||
Maintain a dict of KVConnectorStats objects, one for each connector.
|
||||
"""Maintain a dict of KVConnectorStats objects, one for each connector.
|
||||
This is used to aggregate the stats from all connectors separately.
|
||||
"""
|
||||
|
||||
@@ -132,8 +131,7 @@ class MultiKVConnectorPromMetrics(KVConnectorPromMetrics):
|
||||
|
||||
|
||||
class MultiConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
"""
|
||||
A wrapper for using multiple KVConnectors at the same time.
|
||||
"""A wrapper for using multiple KVConnectors at the same time.
|
||||
|
||||
The current logic is:
|
||||
- Load KV from the first connector that advertises available tokens from
|
||||
@@ -143,8 +141,7 @@ class MultiConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
|
||||
@classmethod
|
||||
def requires_piecewise_for_cudagraph(cls, extra_config: dict[str, Any]) -> bool:
|
||||
"""
|
||||
MultiConnector requires PIECEWISE CUDA graph mode if any of its
|
||||
"""MultiConnector requires PIECEWISE CUDA graph mode if any of its
|
||||
child connectors require it.
|
||||
"""
|
||||
connectors_config = extra_config.get("connectors", [])
|
||||
@@ -480,8 +477,7 @@ class MultiConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
connector_output.kv_connector_worker_meta = multi_connector_worker_meta
|
||||
|
||||
def get_handshake_metadata(self) -> KVConnectorHandshakeMetadata | None:
|
||||
"""
|
||||
Get the KVConnector handshake metadata from sub-connectors.
|
||||
"""Get the KVConnector handshake metadata from sub-connectors.
|
||||
Returns the first non-None metadata from sub-connectors.
|
||||
"""
|
||||
for c in self._connectors:
|
||||
@@ -493,8 +489,7 @@ class MultiConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
def set_xfer_handshake_metadata(
|
||||
self, metadata: dict[int, KVConnectorHandshakeMetadata]
|
||||
) -> None:
|
||||
"""
|
||||
Set the KV connector handshake metadata for all sub-connectors.
|
||||
"""Set the KV connector handshake metadata for all sub-connectors.
|
||||
This is needed to start the NIXL listener thread for NixlConnector.
|
||||
"""
|
||||
for c in self._connectors:
|
||||
@@ -590,14 +585,15 @@ class MultiConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
|
||||
@classmethod
|
||||
def get_required_kvcache_layout(cls, vllm_config: "VllmConfig") -> str | None:
|
||||
"""
|
||||
Get the required KV cache layout for this connector.
|
||||
"""Get the required KV cache layout for this connector.
|
||||
|
||||
Args:
|
||||
vllm_config (VllmConfig): the vllm config.
|
||||
|
||||
Returns:
|
||||
str: the required KV cache layout. e.g. HND, or NHD.
|
||||
None if the connector does not require a specific layout.
|
||||
|
||||
"""
|
||||
assert vllm_config.kv_transfer_config is not None
|
||||
layouts: set[str] = set()
|
||||
|
||||
@@ -299,11 +299,11 @@ class NixlBaseConnectorScheduler:
|
||||
def set_xfer_handshake_metadata(
|
||||
self, metadata: dict[tuple[int, int], KVConnectorHandshakeMetadata]
|
||||
) -> None:
|
||||
"""
|
||||
Set the KV connector handshake metadata for this connector.
|
||||
"""Set the KV connector handshake metadata for this connector.
|
||||
|
||||
Args:
|
||||
metadata (dict): the handshake metadata to set.
|
||||
|
||||
"""
|
||||
encoded_data: dict[tuple[int, int], bytes] = {}
|
||||
encoder = msgspec.msgpack.Encoder()
|
||||
@@ -400,7 +400,8 @@ class NixlBaseConnectorScheduler:
|
||||
def _get_remote_prefill_token_count(self, num_prompt_tokens: int) -> int:
|
||||
"""D-side only. The number of prompt tokens to load from the prefiller.
|
||||
Stops short of the trailing ``_prefill_backoff()`` tokens that the decoder
|
||||
will recompute locally."""
|
||||
will recompute locally.
|
||||
"""
|
||||
backoff = self._prefill_backoff()
|
||||
if backoff and num_prompt_tokens > backoff:
|
||||
return num_prompt_tokens - backoff
|
||||
@@ -413,7 +414,8 @@ class NixlBaseConnectorScheduler:
|
||||
multi-module MTP it is the drafter's whole lookahead window.
|
||||
|
||||
Guarded by ``_p_side_truncated`` to avoid repeated truncation if the
|
||||
request is preempted and rescheduled."""
|
||||
request is preempted and rescheduled.
|
||||
"""
|
||||
backoff = self._prefill_backoff()
|
||||
params = request.kv_transfer_params
|
||||
if (
|
||||
|
||||
@@ -1147,8 +1147,7 @@ class NixlBaseConnectorWorker:
|
||||
return self.nixl_wrapper.add_remote_agent(metadata.agent_metadata)
|
||||
|
||||
def initialize_host_xfer_buffer(self, kv_caches: dict[str, torch.Tensor]) -> None:
|
||||
"""
|
||||
Initialize transfer buffer in CPU mem for accelerators
|
||||
"""Initialize transfer buffer in CPU mem for accelerators
|
||||
NOT directly supported by NIXL (e.g., tpu)
|
||||
"""
|
||||
xfer_buffers: dict[str, torch.Tensor] = {}
|
||||
@@ -1262,8 +1261,7 @@ class NixlBaseConnectorWorker:
|
||||
pp_size: int = 1,
|
||||
notif_agents_only: bool = False,
|
||||
) -> Future[tuple[dict[tuple[int, int], str], float]] | None:
|
||||
"""
|
||||
Ensure a handshake is in-flight (or already done) for *engine_id*.
|
||||
"""Ensure a handshake is in-flight (or already done) for *engine_id*.
|
||||
|
||||
Returns the ``Future`` if a handshake is pending (or was just
|
||||
started), or ``None`` if the handshake already completed
|
||||
@@ -1348,7 +1346,6 @@ class NixlBaseConnectorWorker:
|
||||
|
||||
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
|
||||
"""Register the KV Cache data in nixl."""
|
||||
|
||||
self.transfer_topo = TransferTopology(
|
||||
tp_rank=self.transfer_tp_rank,
|
||||
tp_size=self.transfer_tp_size,
|
||||
@@ -1876,7 +1873,8 @@ class NixlBaseConnectorWorker:
|
||||
) -> np.ndarray:
|
||||
"""Build remote desc regions (conv sub-projections + ssm) per layer.
|
||||
For hetero-TP, each D rank reads only its sub-projection slice from
|
||||
the P rank. Returns an Nx3 uint64 array."""
|
||||
the P rank. Returns an Nx3 uint64 array.
|
||||
"""
|
||||
assert nixl_agent_meta.kv_caches_base_addr, (
|
||||
"Remote KV cache base addresses must not be empty."
|
||||
)
|
||||
@@ -2034,8 +2032,7 @@ class NixlBaseConnectorWorker:
|
||||
self,
|
||||
block_size: int,
|
||||
) -> tuple[int, np.ndarray]:
|
||||
"""
|
||||
Function used for register local xfer handler with local block_size or
|
||||
"""Function used for register local xfer handler with local block_size or
|
||||
Remote block_size.
|
||||
|
||||
When local block_size is same as remote block_size, we use local block_size
|
||||
@@ -2124,8 +2121,7 @@ class NixlBaseConnectorWorker:
|
||||
remote_tp_size: int = 1,
|
||||
remote_dcp_size: int = 1,
|
||||
) -> str:
|
||||
"""
|
||||
Add the remote NIXL agent and prepare the descriptors for reading cache
|
||||
"""Add the remote NIXL agent and prepare the descriptors for reading cache
|
||||
blocks from remote.
|
||||
|
||||
In particular, handle both homogeneous and heterogeneous TP. The former
|
||||
@@ -2395,8 +2391,7 @@ class NixlBaseConnectorWorker:
|
||||
remote_tp_size: int,
|
||||
remote_dcp_size: int = 1,
|
||||
):
|
||||
"""
|
||||
Validate the remote agent handshake metadata ensuring the
|
||||
"""Validate the remote agent handshake metadata ensuring the
|
||||
invariants hold true.
|
||||
"""
|
||||
remote_engine_id = nixl_agent_meta.engine_id
|
||||
@@ -2607,7 +2602,7 @@ class NixlBaseConnectorWorker:
|
||||
assert len(nixl_agent_meta.region_mem_types) == num_remote_regions
|
||||
|
||||
def sync_recved_kv_to_device(self, req_id: str, meta: ReqMeta):
|
||||
"""copy recved kv from host buffer to device."""
|
||||
"""Copy recved kv from host buffer to device."""
|
||||
assert self.use_host_buffer
|
||||
assert self.copy_blocks is not None
|
||||
|
||||
@@ -2632,7 +2627,7 @@ class NixlBaseConnectorWorker:
|
||||
)
|
||||
|
||||
def save_kv_to_host(self, metadata: NixlConnectorMetadata):
|
||||
"""copy kv from device to host buffer."""
|
||||
"""Copy kv from device to host buffer."""
|
||||
assert self.use_host_buffer
|
||||
assert self.copy_blocks is not None
|
||||
|
||||
@@ -2662,7 +2657,8 @@ class NixlBaseConnectorWorker:
|
||||
@cached_property
|
||||
def _attention_kv_caches(self) -> list[torch.Tensor]:
|
||||
"""Device KV caches of attention layers (mamba states excluded),
|
||||
as consumed by the receive post-process."""
|
||||
as consumed by the receive post-process.
|
||||
"""
|
||||
assert self.device_kv_caches, (
|
||||
"_attention_kv_caches accessed before register_kv_caches"
|
||||
)
|
||||
@@ -2681,8 +2677,7 @@ class NixlBaseConnectorWorker:
|
||||
block_ids_list: list[tuple[list[int], int]],
|
||||
convert: bool = True,
|
||||
):
|
||||
"""
|
||||
Post process device kv cache after receiving from remote.
|
||||
"""Post process device kv cache after receiving from remote.
|
||||
|
||||
3 types of conversion supported (``convert``):
|
||||
* kv_cache_postprocess_layout => convert from HND to NHD
|
||||
@@ -2771,8 +2766,7 @@ class NixlBaseConnectorWorker:
|
||||
def post_process_device_kv_on_receive_heterogeneous_attn(
|
||||
self, block_ids: list[int]
|
||||
):
|
||||
"""
|
||||
Post process device kv cache after receiving from remote
|
||||
"""Post process device kv cache after receiving from remote
|
||||
for heterogeneous attention.
|
||||
"""
|
||||
assert self.enable_heterogeneous_attn_post_process
|
||||
@@ -2805,8 +2799,7 @@ class NixlBaseConnectorWorker:
|
||||
pages[block].zero_()
|
||||
|
||||
def get_transfer_results(self) -> KVConnectorTransferResults:
|
||||
"""
|
||||
Get transfers that completed on this specific worker.
|
||||
"""Get transfers that completed on this specific worker.
|
||||
|
||||
The scheduler process (via the MultiprocExecutor) will use this output
|
||||
to track which workers are done.
|
||||
@@ -2984,6 +2977,7 @@ class NixlBaseConnectorWorker:
|
||||
Args:
|
||||
payload: comma-separated P-side request IDs, e.g.
|
||||
"req_abc,req_def".
|
||||
|
||||
"""
|
||||
new_expiry = time.perf_counter() + self._lease_extension
|
||||
for req_id in payload.split(","):
|
||||
@@ -3010,6 +3004,7 @@ class NixlBaseConnectorWorker:
|
||||
Returns:
|
||||
Requests with no outstanding handles and requests with observed
|
||||
failures. Failed requests may still have outstanding handles.
|
||||
|
||||
"""
|
||||
done_req_ids: set[str] = set()
|
||||
failed_req_ids: set[str] = set()
|
||||
@@ -3091,9 +3086,7 @@ class NixlBaseConnectorWorker:
|
||||
self.xfer_stats.record_failed_notification()
|
||||
|
||||
def _send_heartbeats(self, metadata: NixlConnectorMetadata) -> None:
|
||||
"""
|
||||
Send heartbeat notifications to remote engines, extending lease on KV blocks.
|
||||
"""
|
||||
"""Send heartbeats to remote engines, extending the lease on KV blocks."""
|
||||
for engine_id, hb_info in metadata.heartbeat_by_engine.items():
|
||||
# Proactive handshake (this request may still be in waiting queue) so
|
||||
# the **next** heartbeat for this remote can go through.
|
||||
@@ -3126,8 +3119,7 @@ class NixlBaseConnectorWorker:
|
||||
def get_mapped_blocks(
|
||||
self, block_ids: np.ndarray, block_size_ratio: int
|
||||
) -> np.ndarray:
|
||||
"""
|
||||
Calculates the new set of block IDs by mapping every element
|
||||
"""Calculates the new set of block IDs by mapping every element
|
||||
in the (potentially sparse) input array.
|
||||
Example: block_ids=[0, 2], block_size_ratio=2
|
||||
get_mapped_blocks 0 1 [2 3] 4 5
|
||||
@@ -3184,8 +3176,7 @@ class NixlBaseConnectorWorker:
|
||||
return mapped_local, mapped_remote
|
||||
|
||||
def _logical_to_kernel_block_ids(self, block_ids: BlockIds, ratio: int) -> BlockIds:
|
||||
"""
|
||||
Convert block ids to kernel physical block ids.
|
||||
"""Convert block ids to kernel physical block ids.
|
||||
This is required when the logical block size (the one set by the user)
|
||||
does not match the one required by the attn backend.
|
||||
`ratio` is the number of physical blocks per logical block.
|
||||
@@ -3239,6 +3230,7 @@ class NixlBaseConnectorWorker:
|
||||
|
||||
An empty result (rank 0 above) takes the notification-only path, allowing the
|
||||
remote to release its blocks without performing a transfer.
|
||||
|
||||
"""
|
||||
local_size, remote_size = local_dcp_size, remote_dcp_size
|
||||
|
||||
@@ -3417,17 +3409,14 @@ class NixlBaseConnectorWorker:
|
||||
return decode_block_ids, prefill_block_ids
|
||||
|
||||
def get_kv_connector_stats(self) -> KVConnectorStats | None:
|
||||
"""
|
||||
Get the KV transfer stats for the connector.
|
||||
"""
|
||||
"""Get the KV transfer stats for the connector."""
|
||||
# Clear stats for next iteration
|
||||
if not self.xfer_stats.is_empty():
|
||||
return self.xfer_stats.clone_and_reset()
|
||||
return None
|
||||
|
||||
def get_block_ids_with_load_errors(self) -> set[int]:
|
||||
"""
|
||||
Return and clear the set of block IDs that failed to load.
|
||||
"""Return and clear the set of block IDs that failed to load.
|
||||
|
||||
This is called by the scheduler to identify blocks that need
|
||||
to be retried after a NIXL transfer failure.
|
||||
|
||||
@@ -214,12 +214,12 @@ class NixlBaseConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
def set_xfer_handshake_metadata_pp_aware(
|
||||
self, metadata: dict[tuple[int, int], KVConnectorHandshakeMetadata]
|
||||
) -> None:
|
||||
"""
|
||||
Set handshake metadata keyed by (pp_rank, tp_rank) so the side
|
||||
"""Set handshake metadata keyed by (pp_rank, tp_rank) so the side
|
||||
channel can serve every PP stage's agent metadata.
|
||||
|
||||
Args:
|
||||
metadata (dict): the handshake metadata to set.
|
||||
|
||||
"""
|
||||
assert self.connector_scheduler is not None
|
||||
self.connector_scheduler.set_xfer_handshake_metadata(metadata)
|
||||
@@ -317,14 +317,14 @@ class NixlBaseConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
self.connector_scheduler.shutdown()
|
||||
|
||||
def get_handshake_metadata(self) -> KVConnectorHandshakeMetadata | None:
|
||||
"""
|
||||
Get the KVConnector handshake metadata for this connector.
|
||||
"""Get the KVConnector handshake metadata for this connector.
|
||||
This metadata is used for out-of-band connector handshake
|
||||
between P/D workers.
|
||||
|
||||
Returns:
|
||||
KVConnectorHandshakeMetadata: the handshake metadata.
|
||||
None if no handshake metadata is available.
|
||||
|
||||
"""
|
||||
assert self.connector_worker is not None
|
||||
if (
|
||||
|
||||
@@ -78,8 +78,7 @@ class NixlAgentMetadata:
|
||||
|
||||
@dataclass
|
||||
class NixlHandshakePayload(KVConnectorHandshakeMetadata):
|
||||
"""
|
||||
Wrapper for NIXL handshake sent over the wire.
|
||||
"""Wrapper for NIXL handshake sent over the wire.
|
||||
|
||||
Enables two-phase decoding for graceful compatibility checking:
|
||||
1. Decode NixlHandshakePayload to get compatibility_hash
|
||||
@@ -142,8 +141,7 @@ def compute_nixl_compatibility_hash(
|
||||
attn_backend_name: str,
|
||||
transfer_mode: str = "pull",
|
||||
) -> str:
|
||||
"""
|
||||
Compute compatibility hash for NIXL KV transfer.
|
||||
"""Compute compatibility hash for NIXL KV transfer.
|
||||
|
||||
Hash only the factors that affect whether two NIXL instances can
|
||||
successfully transfer KV cache data.
|
||||
@@ -169,6 +167,7 @@ def compute_nixl_compatibility_hash(
|
||||
|
||||
Returns:
|
||||
SHA-256 hex digest
|
||||
|
||||
"""
|
||||
from vllm import __version__ as vllm_version
|
||||
from vllm.config.utils import hash_factors
|
||||
|
||||
@@ -34,8 +34,7 @@ class NixlPullConnectorScheduler(NixlBaseConnectorScheduler):
|
||||
def get_num_new_matched_tokens(
|
||||
self, request: "Request", num_computed_tokens: int
|
||||
) -> tuple[int, bool]:
|
||||
"""
|
||||
For remote prefill, pull all prompt blocks from remote
|
||||
"""For remote prefill, pull all prompt blocks from remote
|
||||
asynchronously relative to engine execution.
|
||||
|
||||
Args:
|
||||
@@ -47,8 +46,8 @@ class NixlPullConnectorScheduler(NixlBaseConnectorScheduler):
|
||||
external KV cache beyond what is already computed.
|
||||
* true if the external KV cache tokens will be loaded
|
||||
asynchronously (between scheduler steps).
|
||||
"""
|
||||
|
||||
"""
|
||||
params = request.kv_transfer_params
|
||||
logger.debug(
|
||||
"NIXLConnector get_num_new_matched_tokens: "
|
||||
@@ -194,8 +193,7 @@ class NixlPullConnectorScheduler(NixlBaseConnectorScheduler):
|
||||
request: "Request",
|
||||
block_ids: "BlockIds",
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
"""
|
||||
Once a request is finished, determine whether request blocks
|
||||
"""Once a request is finished, determine whether request blocks
|
||||
should be freed now or will be sent asynchronously and freed later.
|
||||
"""
|
||||
from vllm.v1.request import RequestStatus
|
||||
|
||||
@@ -45,8 +45,7 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
|
||||
super().__init__(vllm_config, engine_id, kv_cache_config)
|
||||
|
||||
def start_load_kv(self, metadata: NixlConnectorMetadata):
|
||||
"""
|
||||
Start loading by triggering non-blocking nixl_xfer.
|
||||
"""Start loading by triggering non-blocking nixl_xfer.
|
||||
We check for these trnxs to complete in each step().
|
||||
"""
|
||||
for req_id, meta in metadata.reqs_to_recv.items():
|
||||
@@ -373,8 +372,7 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
|
||||
expected_consumers: int,
|
||||
awaiting_kvs: bool,
|
||||
) -> bool:
|
||||
"""
|
||||
Post a READ point-to-point xfer request from a single local worker to
|
||||
"""Post a READ point-to-point xfer request from a single local worker to
|
||||
a single remote worker.
|
||||
|
||||
Returns True when the read was posted (or was unnecessary), False
|
||||
@@ -603,8 +601,7 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
|
||||
self._recving_transfers[request_id].append(handle)
|
||||
|
||||
def _get_new_notifs(self) -> set[str]:
|
||||
"""
|
||||
Get req_ids which got a remote xfer message. When multiple consumers
|
||||
"""Get req_ids which got a remote xfer message. When multiple consumers
|
||||
are reading from the same producer (heterogeneous TP or DCP
|
||||
scenario), wait for all consumers to be done pulling.
|
||||
|
||||
|
||||
@@ -315,7 +315,8 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
|
||||
executes on the writer; the handshake runs on the background executor
|
||||
and the request is re-queued onto ``_reg_send_inbox`` once it
|
||||
completes (at which point ``_ensure_handshake`` returns ``None`` and we
|
||||
send directly)."""
|
||||
send directly).
|
||||
"""
|
||||
remote_pp_size = reg_data.get("remote_pp_size", 1)
|
||||
fut = self._ensure_handshake(
|
||||
reg_data["remote_engine_id"],
|
||||
@@ -507,7 +508,8 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
|
||||
``BlockIds`` is canonically a tuple of per-group lists, but some
|
||||
registration payloads collapse a single-group case to a flat
|
||||
list. Re-wrap that case so downstream group-aware helpers see a
|
||||
consistent shape."""
|
||||
consistent shape.
|
||||
"""
|
||||
if block_ids and not isinstance(block_ids[0], (list, tuple)):
|
||||
return (list(block_ids),)
|
||||
return block_ids
|
||||
|
||||
@@ -23,7 +23,7 @@ if TYPE_CHECKING:
|
||||
|
||||
@dataclass
|
||||
class NixlKVConnectorStats(KVConnectorStats):
|
||||
"""Container for transfer performance metrics"""
|
||||
"""Container for transfer performance metrics."""
|
||||
|
||||
def __post_init__(self):
|
||||
if not self.data:
|
||||
|
||||
@@ -34,8 +34,7 @@ _NIXL_SUPPORTED_DEVICE.update(current_platform.get_nixl_supported_devices())
|
||||
# TODO: merge with vllm.utils.network_utils.zmq_socket_ctx
|
||||
@contextlib.contextmanager
|
||||
def zmq_ctx(socket_type: Any, addr: str) -> Iterator[zmq.Socket]:
|
||||
"""Context manager for a ZMQ socket"""
|
||||
|
||||
"""Context manager for a ZMQ socket."""
|
||||
if socket_type not in (zmq.ROUTER, zmq.REQ):
|
||||
raise ValueError(f"Unexpected socket type: {socket_type}")
|
||||
|
||||
|
||||
@@ -37,7 +37,8 @@ def canonical_format_id(kv_cache_layout: str) -> str:
|
||||
Canonical pages keep the worker's KV layout family, so the id couples the
|
||||
format version with that family; consumers must match it exactly. The family
|
||||
keeps its historical NHD/HND spelling so ids stay stable for KV persisted
|
||||
before the layout enum existed."""
|
||||
before the layout enum existed.
|
||||
"""
|
||||
layout = KVCacheLayout[kv_cache_layout]
|
||||
legacy = {KVCacheLayout.LBNHC: "nhd", KVCacheLayout.LBHNC: "hnd"}
|
||||
family = legacy.get(layout, layout.name.lower())
|
||||
@@ -121,7 +122,8 @@ def _interleave_cp_tokens(
|
||||
ctx: _RankContext,
|
||||
) -> tuple[CopyRun, ...]:
|
||||
"""Place each region's num_tokens rows at their canonical token positions,
|
||||
one run per chunk of interleaved tokens."""
|
||||
one run per chunk of interleaved tokens.
|
||||
"""
|
||||
runs: list[CopyRun] = []
|
||||
for region in regions:
|
||||
if ctx.cp_size == 1:
|
||||
@@ -256,7 +258,8 @@ def _attention_byte_regions(
|
||||
cp_size: int,
|
||||
) -> list[ByteRegion] | None:
|
||||
"""Byte regions of an attention page, given this rank's head shard.
|
||||
None when the physical layout is not recognized (fail closed)."""
|
||||
None when the physical layout is not recognized (fail closed).
|
||||
"""
|
||||
bs, heads, head_size = spec.block_size, spec.num_kv_heads, spec.head_size
|
||||
if tuple(kv_cache.shape) == (num_blocks, heads, bs, 2 * head_size):
|
||||
return _packed_kv_regions(kv_cache, spec, head_shard, num_head_shards, cp_size)
|
||||
@@ -370,7 +373,8 @@ def _is_exact_partition(intervals: list[tuple[int, int]], size: int) -> bool:
|
||||
|
||||
def _verify_tiling(layer_name: str, per_rank: list[CanonicalPageMapping]) -> None:
|
||||
"""Whichever ranks a block elects as writers must tile the canonical page
|
||||
exactly once, and each rank's runs must cover exactly its local page."""
|
||||
exactly once, and each rank's runs must cover exactly its local page.
|
||||
"""
|
||||
size = per_rank[0].canonical_page_size_bytes
|
||||
num_writers = per_rank[0].num_writers
|
||||
for mapping in per_rank:
|
||||
|
||||
@@ -79,7 +79,8 @@ def get_offloading_event_group_spec(
|
||||
class _OffloadEventMetadata:
|
||||
"""BlockStored payload snapshot for one OffloadKey, captured while the
|
||||
Request is available and kept until the final matching removal event.
|
||||
``medium`` and ``ownership`` are forwarded from the OffloadingEvent."""
|
||||
``medium`` and ``ownership`` are forwarded from the OffloadingEvent.
|
||||
"""
|
||||
|
||||
# The chunk's constituent block hashes; the last one is the OffloadKey.
|
||||
block_hashes: tuple[BlockHash, ...]
|
||||
@@ -245,6 +246,7 @@ class OffloadingEventsTracker:
|
||||
Yields:
|
||||
``BlockStored`` or ``BlockRemoved`` events corresponding to
|
||||
the underlying :class:`OffloadingEvent` stream.
|
||||
|
||||
"""
|
||||
for event in events:
|
||||
if event.removed:
|
||||
@@ -254,7 +256,8 @@ class OffloadingEventsTracker:
|
||||
|
||||
def reset(self) -> None:
|
||||
"""Drop all tracked state; pending payloads are stale after a
|
||||
manager cache reset."""
|
||||
manager cache reset.
|
||||
"""
|
||||
self._pending_event_metadata.clear()
|
||||
|
||||
def _build_event_metadata(
|
||||
@@ -265,7 +268,8 @@ class OffloadingEventsTracker:
|
||||
) -> _OffloadEventMetadata:
|
||||
"""Build the payload snapshot for one offloaded chunk: its
|
||||
constituent per-block hashes, the whole chunk's tokens, and the
|
||||
per-block ``block_size``."""
|
||||
per-block ``block_size``.
|
||||
"""
|
||||
hashes_per_chunk = group_config.hashes_per_chunk
|
||||
assert hashes_per_chunk > 0
|
||||
assert chunk_idx >= 0
|
||||
|
||||
@@ -170,8 +170,7 @@ class _StatsKey:
|
||||
|
||||
@dataclass
|
||||
class OffloadingConnectorStats(KVConnectorStats):
|
||||
"""
|
||||
Offloading connector stats use flat metric names as keys.
|
||||
"""Offloading connector stats use flat metric names as keys.
|
||||
|
||||
The ``data`` dict is structured using ``_StatsKey`` / ``_MetricType``::
|
||||
|
||||
@@ -243,8 +242,7 @@ class OffloadingConnectorStats(KVConnectorStats):
|
||||
return self
|
||||
|
||||
def reduce(self) -> dict[str, int | float]:
|
||||
"""
|
||||
Reduce the observations collected during a time interval to one or
|
||||
"""Reduce the observations collected during a time interval to one or
|
||||
more representative values (eg avg/median/sum of the series).
|
||||
This is meant to be called by the logger to produce a summary of the
|
||||
stats for the last time interval.
|
||||
|
||||
@@ -676,7 +676,8 @@ class OffloadingConnectorScheduler:
|
||||
start_chunk_idx: int,
|
||||
) -> int | None:
|
||||
"""Return the number of consecutive offloaded chunks from the start,
|
||||
or None if the backend deferred a lookup."""
|
||||
or None if the backend deferred a lookup.
|
||||
"""
|
||||
hit_count = 0
|
||||
defer_lookup = False
|
||||
for local_idx, key in enumerate(keys):
|
||||
@@ -711,7 +712,8 @@ class OffloadingConnectorScheduler:
|
||||
"""Return the end index (in `keys`) of the last run of
|
||||
`sliding_window_size` consecutive hits, scanning from the end.
|
||||
The first run may need a larger window for a partial rightmost chunk.
|
||||
Returns 0 on miss, None if the backend deferred a lookup."""
|
||||
Returns 0 on miss, None if the backend deferred a lookup.
|
||||
"""
|
||||
defer_lookup = False
|
||||
pending_in_window = False
|
||||
consecutive_hits = 0
|
||||
@@ -750,8 +752,7 @@ class OffloadingConnectorScheduler:
|
||||
req_status: RequestOffloadState,
|
||||
max_num_new_tokens: int | None = None,
|
||||
) -> int | None:
|
||||
"""
|
||||
Find how many tokens beyond num_locally_computed_tokens can be loaded.
|
||||
"""Find how many tokens beyond num_locally_computed_tokens can be loaded.
|
||||
|
||||
Iterates full-attention groups first (prefix lookup), then sliding-window
|
||||
groups (suffix lookup). Each group may tighten max_hit_size_tokens, which
|
||||
@@ -1039,14 +1040,15 @@ class OffloadingConnectorScheduler:
|
||||
num_computed_tokens: int,
|
||||
max_num_new_tokens: int | None = None,
|
||||
) -> tuple[int | None, bool]:
|
||||
"""
|
||||
Get number of new tokens that can be loaded beyond the
|
||||
"""Get number of new tokens that can be loaded beyond the
|
||||
num_computed_tokens.
|
||||
|
||||
Args:
|
||||
request (Request): the request object.
|
||||
num_computed_tokens (int): the number of locally
|
||||
computed tokens for this request
|
||||
max_num_new_tokens (int | None): cap on the number of tokens that
|
||||
may be loaded beyond `num_computed_tokens`, if any.
|
||||
|
||||
Returns:
|
||||
A tuple with the following elements:
|
||||
@@ -1057,6 +1059,7 @@ class OffloadingConnectorScheduler:
|
||||
should query for this request again later.
|
||||
- `True` if tokens will be loaded asynchronously
|
||||
(between scheduler steps).
|
||||
|
||||
"""
|
||||
req_status = self._req_status[request.request_id]
|
||||
for group_state in req_status.group_states:
|
||||
@@ -1208,10 +1211,7 @@ class OffloadingConnectorScheduler:
|
||||
req_status.partial_tail_boundary = None
|
||||
|
||||
def _update_req_states(self, scheduler_output: SchedulerOutput) -> None:
|
||||
"""
|
||||
Update request states from the Scheduler's output.
|
||||
"""
|
||||
|
||||
"""Update request states from the Scheduler's output."""
|
||||
# new_block_ids_end[req_id][i] = end of pre-existing block_ids for
|
||||
# the i-th sliding window group (before this step's extend).
|
||||
# Used to detect sliding window blocks that got re-allocated.
|
||||
@@ -1840,12 +1840,12 @@ class OffloadingConnectorScheduler:
|
||||
return bool(self._jobs) or self.manager.has_pending_work()
|
||||
|
||||
def update_connector_output(self, connector_output: KVConnectorOutput):
|
||||
"""
|
||||
Update KVConnector state from worker-side connectors output.
|
||||
"""Update KVConnector state from worker-side connectors output.
|
||||
|
||||
Args:
|
||||
connector_output (KVConnectorOutput): the worker-side
|
||||
connectors output.
|
||||
|
||||
"""
|
||||
meta = connector_output.kv_connector_worker_meta
|
||||
if not isinstance(meta, OffloadingWorkerMetadata):
|
||||
@@ -1938,8 +1938,7 @@ class OffloadingConnectorScheduler:
|
||||
self,
|
||||
request: Request,
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
"""
|
||||
Called when a request has finished, before its blocks are freed.
|
||||
"""Called when a request has finished, before its blocks are freed.
|
||||
|
||||
Returns:
|
||||
True if the request is being saved/sent asynchronously and blocks
|
||||
@@ -1947,6 +1946,7 @@ class OffloadingConnectorScheduler:
|
||||
get_finished().
|
||||
Optional KVTransferParams to be included in the request outputs
|
||||
returned by the engine.
|
||||
|
||||
"""
|
||||
req_status = self._req_status.get(request.request_id)
|
||||
|
||||
@@ -1985,12 +1985,12 @@ class OffloadingConnectorScheduler:
|
||||
Yields:
|
||||
``BlockStored`` or ``BlockRemoved`` events corresponding to
|
||||
the underlying :class:`OffloadingEvent` stream.
|
||||
|
||||
"""
|
||||
yield from self._events_tracker.take_events(self.manager.take_events())
|
||||
|
||||
def reset_cache(self) -> None:
|
||||
"""Reset the offloading manager cache, evicting all stored chunks."""
|
||||
|
||||
# reset_cache cannot be called in the middle of a schedule step
|
||||
assert not self._current_batch_load_jobs
|
||||
assert not self._current_batch_jobs_to_flush
|
||||
|
||||
@@ -39,7 +39,7 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
class OffloadingConnectorWorker:
|
||||
"""Implementation of Worker side methods"""
|
||||
"""Implementation of Worker side methods."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -268,14 +268,14 @@ class OffloadingConnectorWorker:
|
||||
)
|
||||
|
||||
def get_finished(self, finished_req_ids: set[str]) -> tuple[set[str], set[str]]:
|
||||
"""
|
||||
Returns:
|
||||
tuple of (finished_sending, finished_recving). Stores never
|
||||
emit finished_sending — the scheduler tracks store completion
|
||||
via kv_connector_worker_meta.completed_jobs and fences any
|
||||
block reuse via jobs_to_flush. Loads still emit
|
||||
finished_recving so the base scheduler can resume requests
|
||||
blocked on remote KV (and free aborted-during-load reqs).
|
||||
"""Returns:
|
||||
tuple of (finished_sending, finished_recving). Stores never
|
||||
emit finished_sending — the scheduler tracks store completion
|
||||
via kv_connector_worker_meta.completed_jobs and fences any
|
||||
block reuse via jobs_to_flush. Loads still emit
|
||||
finished_recving so the base scheduler can resume requests
|
||||
blocked on remote KV (and free aborted-during-load reqs).
|
||||
|
||||
"""
|
||||
assert self.worker is not None
|
||||
finished_recving: set[str] = set()
|
||||
|
||||
@@ -83,6 +83,7 @@ class MambaConvSplitInfo:
|
||||
< 0: P_TP > D_TP — P pages are smaller, D reads entire
|
||||
P page. Local dims are scaled down by |tp_ratio|
|
||||
to get P-sized offsets.
|
||||
|
||||
"""
|
||||
offsets: list[tuple[int, int]] = []
|
||||
if tp_ratio >= 1:
|
||||
@@ -121,6 +122,7 @@ def derive_mamba_conv_split(
|
||||
Returns:
|
||||
MambaConvSplitInfo with per-rank sub-projection dims, conv_rows,
|
||||
conv_dtype_size, and ssm_sizes (conv_state_bytes, ssm_state_bytes).
|
||||
|
||||
"""
|
||||
_supported = (
|
||||
MambaAttentionBackendEnum.MAMBA1,
|
||||
@@ -228,5 +230,6 @@ def compute_physical_blocks_per_logical(
|
||||
Args:
|
||||
ssm_sizes: (conv_state_bytes, ssm_state_bytes) from NixlAgentMetadata.
|
||||
block_len: the engine's block_len in bytes (from block_lens[0]).
|
||||
|
||||
"""
|
||||
return math.ceil((ssm_sizes[0] + ssm_sizes[1]) / block_len)
|
||||
|
||||
@@ -38,6 +38,7 @@ def is_v1_kv_transfer_group(connector: KVConnectorBaseType | None = None) -> boo
|
||||
Note:
|
||||
This function will no-longer be needed after the v1 KV connector
|
||||
becomes the default.
|
||||
|
||||
"""
|
||||
if connector is None:
|
||||
connector = _KV_CONNECTOR_AGENT
|
||||
@@ -72,10 +73,7 @@ def _sync_engine_id_across_tp(vllm_config: "VllmConfig") -> None:
|
||||
def ensure_kv_transfer_initialized(
|
||||
vllm_config: "VllmConfig", kv_cache_config: "KVCacheConfig"
|
||||
) -> None:
|
||||
"""
|
||||
Initialize KV cache transfer parallel group.
|
||||
"""
|
||||
|
||||
"""Initialize KV cache transfer parallel group."""
|
||||
global _KV_CONNECTOR_AGENT
|
||||
|
||||
if _KV_CONNECTOR_AGENT is not None:
|
||||
|
||||
Reference in New Issue
Block a user