Add pydocstyle to the ruff rules (#52136)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Signed-off-by: Harry Mellor <hej.mellor@gmail.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
804e5377aa
commit
8c1557a79c
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
ECConnectorBase Class for Distributed Encoder Cache &
|
||||
"""ECConnectorBase Class for Distributed Encoder Cache &
|
||||
P2P Encoder cache communication in V1
|
||||
|
||||
The class provides the following primitives:
|
||||
@@ -50,8 +49,7 @@ class ECConnectorRole(enum.Enum):
|
||||
|
||||
|
||||
class ECConnectorMetadata(ABC): # noqa: B024
|
||||
"""
|
||||
Abstract Metadata used to communicate between the
|
||||
"""Abstract Metadata used to communicate between the
|
||||
Scheduler ECConnector and Worker ECConnector.
|
||||
"""
|
||||
|
||||
@@ -59,8 +57,7 @@ class ECConnectorMetadata(ABC): # noqa: B024
|
||||
|
||||
|
||||
class ECConnectorWorkerMetadata(ABC):
|
||||
"""
|
||||
Abstract Metadata used to communicate back
|
||||
"""Abstract Metadata used to communicate back
|
||||
Worker ECConnector -> Scheduler ECConnector.
|
||||
|
||||
Each worker can output its own metadata.
|
||||
@@ -73,9 +70,7 @@ class ECConnectorWorkerMetadata(ABC):
|
||||
def aggregate(
|
||||
self, other: "ECConnectorWorkerMetadata"
|
||||
) -> "ECConnectorWorkerMetadata":
|
||||
"""
|
||||
Aggregate metadata with another `ECConnectorWorkerMetadata` object.
|
||||
"""
|
||||
"""Aggregate metadata with another `ECConnectorWorkerMetadata` object."""
|
||||
pass
|
||||
|
||||
|
||||
@@ -103,8 +98,7 @@ class ECConnectorBase(ABC):
|
||||
return self._is_consumer
|
||||
|
||||
def shutdown(self) -> None:
|
||||
"""
|
||||
Shutdown the connector. This is called when the process
|
||||
"""Shutdown the connector. This is called when the process
|
||||
is shutting down to ensure that all the async operations are
|
||||
completed and the connector is cleaned up properly.
|
||||
"""
|
||||
@@ -123,6 +117,7 @@ class ECConnectorBase(ABC):
|
||||
|
||||
Args:
|
||||
connector_metadata (dict): the connector metadata.
|
||||
|
||||
"""
|
||||
self._connector_metadata = connector_metadata
|
||||
|
||||
@@ -141,8 +136,8 @@ class ECConnectorBase(ABC):
|
||||
|
||||
Returns:
|
||||
ConnectorMetadata: the connector metadata.
|
||||
"""
|
||||
|
||||
"""
|
||||
# Should only be called while set to valid metadata.
|
||||
assert self._connector_metadata is not None
|
||||
return self._connector_metadata
|
||||
@@ -151,10 +146,11 @@ class ECConnectorBase(ABC):
|
||||
self,
|
||||
ec_caches: dict[str, torch.Tensor],
|
||||
):
|
||||
"""
|
||||
Initialize with the EC caches.
|
||||
"""Initialize with the EC caches.
|
||||
|
||||
Args:
|
||||
ec_caches: dictionary of encoder cache
|
||||
|
||||
"""
|
||||
# TODO: Implement this later for P2P feature
|
||||
return
|
||||
@@ -163,8 +159,7 @@ class ECConnectorBase(ABC):
|
||||
def start_load_caches(
|
||||
self, encoder_cache: dict[str, torch.Tensor], **kwargs
|
||||
) -> None:
|
||||
"""
|
||||
Start loading the cache from the connector into vLLM's encoder cache.
|
||||
"""Start loading the cache from the connector into vLLM's encoder cache.
|
||||
|
||||
This method loads the encoder cache based on metadata provided by the scheduler.
|
||||
It is called before `_gather_mm_embeddings` for the EC Connector. For EC,
|
||||
@@ -174,6 +169,7 @@ class ECConnectorBase(ABC):
|
||||
encoder_cache (dict[str, torch.Tensor]): A dictionary mapping multimodal
|
||||
data hashes (`mm_hash`) to encoder cache tensors.
|
||||
kwargs (dict): Additional keyword arguments for the connector.
|
||||
|
||||
"""
|
||||
pass
|
||||
|
||||
@@ -181,8 +177,7 @@ class ECConnectorBase(ABC):
|
||||
def save_caches(
|
||||
self, encoder_cache: dict[str, torch.Tensor], mm_hash: str, **kwargs
|
||||
) -> None:
|
||||
"""
|
||||
Save the encoder cache to the connector.
|
||||
"""Save the encoder cache to the connector.
|
||||
|
||||
This method saves the encoder cache from the worker's local storage
|
||||
to shared storage or another external connector.
|
||||
@@ -192,14 +187,14 @@ class ECConnectorBase(ABC):
|
||||
data hashes (`mm_hash`) to encoder cache tensors.
|
||||
mm_hash (str): The hash of the multimodal data whose cache is being saved.
|
||||
kwargs (dict): Additional keyword arguments for the connector.
|
||||
|
||||
"""
|
||||
pass
|
||||
|
||||
def get_finished(
|
||||
self, finished_req_ids: set[str]
|
||||
) -> tuple[set[str] | None, set[str] | None]:
|
||||
"""
|
||||
Notifies worker-side connector ids of requests that have
|
||||
"""Notifies worker-side connector ids of requests that have
|
||||
finished generating tokens on the worker.
|
||||
The scheduler process (via the Executors) will use this output
|
||||
to track which workers are done.
|
||||
@@ -210,16 +205,17 @@ class ECConnectorBase(ABC):
|
||||
tuple of (sending/saving ids, recving/loading ids).
|
||||
The finished saves/sends req ids must belong to a set provided in a
|
||||
call to this method (this call or a prior one).
|
||||
|
||||
"""
|
||||
return None, None
|
||||
|
||||
def build_connector_worker_meta(self) -> ECConnectorWorkerMetadata | None:
|
||||
"""
|
||||
Build the ECConnector worker metadata for this engine step.
|
||||
"""Build the ECConnector worker metadata for this engine step.
|
||||
|
||||
Returns:
|
||||
ECConnectorWorkerMetadata: the worker metadata.
|
||||
None if no worker metadata is available.
|
||||
|
||||
"""
|
||||
return None
|
||||
|
||||
@@ -232,8 +228,7 @@ class ECConnectorBase(ABC):
|
||||
self,
|
||||
identifier: str,
|
||||
) -> bool:
|
||||
"""
|
||||
Check if a single encoder cache exists
|
||||
"""Check if a single encoder cache exists.
|
||||
|
||||
Args:
|
||||
identifier (str): the identifier of the media.
|
||||
@@ -241,14 +236,14 @@ class ECConnectorBase(ABC):
|
||||
Returns:
|
||||
A bool where value is True if cache exist for
|
||||
the media
|
||||
|
||||
"""
|
||||
pass
|
||||
|
||||
def ensure_cache_available(
|
||||
self, request: "Request", num_computed_tokens: int
|
||||
) -> bool:
|
||||
"""
|
||||
Ensure encoder cache items are available for the given request.
|
||||
"""Ensure encoder cache items are available for the given request.
|
||||
May initiate asynchronous transfers for items not yet local.
|
||||
|
||||
Args:
|
||||
@@ -258,6 +253,7 @@ class ECConnectorBase(ABC):
|
||||
Returns:
|
||||
True if all items are ready or no transfer is needed.
|
||||
False if any items are still in transit (request should be deferred).
|
||||
|
||||
"""
|
||||
return True
|
||||
|
||||
@@ -282,11 +278,12 @@ class ECConnectorBase(ABC):
|
||||
|
||||
@abstractmethod
|
||||
def update_state_after_alloc(self, request: "Request", index: int):
|
||||
"""
|
||||
Update ECConnector state to decide allocate cache for requests
|
||||
"""Update ECConnector state to decide allocate cache for requests.
|
||||
|
||||
Args:
|
||||
request (Request): the request object.
|
||||
index (int): index of the encoder cache entry being allocated.
|
||||
|
||||
"""
|
||||
pass
|
||||
|
||||
@@ -294,37 +291,37 @@ class ECConnectorBase(ABC):
|
||||
def build_connector_meta(
|
||||
self, scheduler_output: SchedulerOutput
|
||||
) -> ECConnectorMetadata:
|
||||
"""
|
||||
Build the connector metadata for this step.
|
||||
"""Build the connector metadata for this step.
|
||||
|
||||
This function should NOT modify fields in the scheduler_output.
|
||||
Also, calling this function will reset the state of the connector.
|
||||
|
||||
Args:
|
||||
scheduler_output (SchedulerOutput): the scheduler output object.
|
||||
|
||||
"""
|
||||
pass
|
||||
|
||||
def update_connector_output(self, connector_output: ECConnectorOutput):
|
||||
"""
|
||||
Update ECConnector state from worker-side connectors output.
|
||||
"""Update ECConnector state from worker-side connectors output.
|
||||
|
||||
Args:
|
||||
connector_output (ECConnectorOutput): the worker-side
|
||||
connectors output.
|
||||
|
||||
"""
|
||||
return
|
||||
|
||||
def request_finished(
|
||||
self, request: "Request"
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
"""
|
||||
Called when a request has finished, before its encoder cache is freed.
|
||||
"""Called when a request has finished, before its encoder cache is freed.
|
||||
|
||||
Returns:
|
||||
True if the request is being saved/sent asynchronously and cached
|
||||
should not be freed until the request_id is returned from
|
||||
get_finished().
|
||||
|
||||
"""
|
||||
return False, None
|
||||
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Lightweight mmap-backed shared memory region for encoder cache (EC) data.
|
||||
"""Lightweight mmap-backed shared memory region for encoder cache (EC) data.
|
||||
|
||||
Modeled after SharedOffloadRegion (vllm/v1/kv_offload/cpu/) but simplified
|
||||
for EC: flat shared layout, no multi-tensor cursor, no block_size_factor.
|
||||
|
||||
@@ -229,6 +229,7 @@ class ECCPUScheduler:
|
||||
Returns:
|
||||
True if no item needs to wait for a remote encoding. False if any
|
||||
item is pending or the request must be aborted.
|
||||
|
||||
"""
|
||||
if not request.ec_transfer_params:
|
||||
return True
|
||||
@@ -259,6 +260,7 @@ class ECCPUScheduler:
|
||||
Raises:
|
||||
_RemoteUnavailable: The remote source is invalid, the read failed,
|
||||
or the item's total wait budget expired.
|
||||
|
||||
"""
|
||||
pos = feature.mm_position
|
||||
mm_hash = feature.identifier
|
||||
@@ -334,6 +336,7 @@ class ECCPUScheduler:
|
||||
item's remote announcement so subsequent steps do not retry it.
|
||||
False if only placeholder metadata (or no input) is available,
|
||||
recording the request for take_unavailable_requests() to drain.
|
||||
|
||||
"""
|
||||
mm_hash = feature.identifier
|
||||
self._deferred_since.pop((request.request_id, mm_hash), None)
|
||||
|
||||
@@ -15,7 +15,8 @@ from collections import OrderedDict
|
||||
|
||||
class CacheEntry:
|
||||
"""A single cache entry. Read `.block_ids` and `.ready` freely;
|
||||
mutations only through EmbeddingCache methods."""
|
||||
mutations only through EmbeddingCache methods.
|
||||
"""
|
||||
|
||||
__slots__ = ("block_ids", "_pin_count")
|
||||
|
||||
|
||||
@@ -67,8 +67,7 @@ class ECExampleConnector(ECConnectorBase):
|
||||
raise ValueError("ec_transfer_config must be set for ECConnectorBase")
|
||||
|
||||
def start_load_caches(self, encoder_cache, **kwargs) -> None:
|
||||
"""
|
||||
Start loading the cache from the connector into vLLM's encoder cache.
|
||||
"""Start loading the cache from the connector into vLLM's encoder cache.
|
||||
|
||||
This method loads the encoder cache based on metadata provided by the scheduler.
|
||||
It is called before `_gather_mm_embeddings` for the EC Connector. For EC,
|
||||
@@ -78,6 +77,7 @@ class ECExampleConnector(ECConnectorBase):
|
||||
encoder_cache (dict[str, torch.Tensor]): A dictionary mapping multimodal
|
||||
data hashes (`mm_hash`) to encoder cache tensors.
|
||||
kwargs (dict): Additional keyword arguments for the connector.
|
||||
|
||||
"""
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
@@ -108,8 +108,7 @@ class ECExampleConnector(ECConnectorBase):
|
||||
logger.debug("Success load encoder cache for hash %s", mm_data.mm_hash)
|
||||
|
||||
def save_caches(self, encoder_cache, mm_hash, **kwargs) -> None:
|
||||
"""
|
||||
Save the encoder cache to the connector.
|
||||
"""Save the encoder cache to the connector.
|
||||
|
||||
This method saves the encoder cache from the worker's local storage
|
||||
to shared storage or another external connector.
|
||||
@@ -119,6 +118,7 @@ class ECExampleConnector(ECConnectorBase):
|
||||
data hashes (`mm_hash`) to encoder cache tensors.
|
||||
mm_hash (str): The hash of the multimodal data whose cache is being saved.
|
||||
kwargs (dict): Additional keyword arguments for the connector.
|
||||
|
||||
"""
|
||||
# Return if it is PD Instance
|
||||
if not self.is_producer:
|
||||
@@ -134,14 +134,14 @@ class ECExampleConnector(ECConnectorBase):
|
||||
self,
|
||||
identifier: str,
|
||||
) -> bool:
|
||||
"""
|
||||
Check if cache exist externally for the media
|
||||
"""Check if cache exist externally for the media.
|
||||
|
||||
Args:
|
||||
identifier (str): the identifier of the media.
|
||||
|
||||
Returns:
|
||||
Bool indicate that media exists in cache or not
|
||||
|
||||
"""
|
||||
return self._found_match_for_mm_data(identifier)
|
||||
|
||||
@@ -150,9 +150,7 @@ class ECExampleConnector(ECConnectorBase):
|
||||
request: "Request",
|
||||
index: int,
|
||||
) -> None:
|
||||
"""
|
||||
Update ECConnector state after encoder cache allocation.
|
||||
"""
|
||||
"""Update ECConnector state after encoder cache allocation."""
|
||||
mm_hash = request.mm_features[index].identifier
|
||||
# Only load cache if it is consumer and cache exists
|
||||
if not self.is_consumer or not self.has_cache_item(mm_hash):
|
||||
@@ -212,8 +210,7 @@ class ECExampleConnector(ECConnectorBase):
|
||||
mm_hash: str,
|
||||
create_folder: bool = True, # <- now defaults to True
|
||||
) -> str:
|
||||
"""
|
||||
Return the folder in which the cache for this mm_hash lives.
|
||||
"""Return the folder in which the cache for this mm_hash lives.
|
||||
If `create_folder` is True (default) the directory is created
|
||||
recursively the first time it is needed.
|
||||
"""
|
||||
@@ -223,8 +220,7 @@ class ECExampleConnector(ECConnectorBase):
|
||||
return foldername
|
||||
|
||||
def _generate_filename_debug(self, mm_hash: str) -> str:
|
||||
"""
|
||||
Return the full path of the safetensors file for this mm_hash.
|
||||
"""Return the full path of the safetensors file for this mm_hash.
|
||||
Ensures the parent directory exists because
|
||||
`_generate_foldername_debug` is called with its default
|
||||
(`create_folder=True`).
|
||||
|
||||
@@ -24,10 +24,7 @@ def has_ec_transfer() -> bool:
|
||||
|
||||
|
||||
def ensure_ec_transfer_initialized(vllm_config: "VllmConfig") -> None:
|
||||
"""
|
||||
Initialize EC cache connector.
|
||||
"""
|
||||
|
||||
"""Initialize EC cache connector."""
|
||||
global _EC_CONNECTOR_AGENT
|
||||
|
||||
if vllm_config.ec_transfer_config is None:
|
||||
|
||||
Reference in New Issue
Block a user