Add pydocstyle to the ruff rules (#52136)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Signed-off-by: Harry Mellor <hej.mellor@gmail.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
804e5377aa
commit
8c1557a79c
@@ -80,8 +80,7 @@ def compute_sub_block_ptrs(
|
||||
tensor: torch.Tensor,
|
||||
skip_count: int = 0,
|
||||
):
|
||||
"""
|
||||
Compute byte pointers for sub-blocks of the given block IDs.
|
||||
"""Compute byte pointers for sub-blocks of the given block IDs.
|
||||
|
||||
Each block in block_ids contains blocks_per_chunk sub-blocks.
|
||||
The pointer for sub-block j of block b is:
|
||||
@@ -98,6 +97,7 @@ def compute_sub_block_ptrs(
|
||||
output: pre-allocated pointer array to write pointers into.
|
||||
tensor: the source or destination tensor.
|
||||
skip_count: sub-blocks to skip in the first block.
|
||||
|
||||
"""
|
||||
assert skip_count < blocks_per_chunk
|
||||
|
||||
@@ -126,7 +126,8 @@ def compute_sub_block_ptrs(
|
||||
class CopyPlan(NamedTuple):
|
||||
"""Precomputed fragment-copy template for one data ref under the canonical
|
||||
CPU layout, unrolled from the ref's mapped runs. Offsets are relative to
|
||||
the per-block base pointers on each side."""
|
||||
the per-block base pointers on each side.
|
||||
"""
|
||||
|
||||
frag_offsets_src: np.ndarray
|
||||
frag_offsets_dst: np.ndarray
|
||||
@@ -164,7 +165,8 @@ def _canonical_page_ids(
|
||||
) -> np.ndarray:
|
||||
"""Global canonical page ids matching compute_sub_block_ptrs' enumeration.
|
||||
These identify canonical pages consistently across ranks, so they key
|
||||
CanonicalPageMapping.is_writer rotation."""
|
||||
CanonicalPageMapping.is_writer rotation.
|
||||
"""
|
||||
if blocks_per_chunk == 1:
|
||||
return block_ids[:count]
|
||||
flat = (
|
||||
@@ -177,7 +179,8 @@ def _canonical_block_sizes(
|
||||
layer_refs_per_group: list[list[CanonicalKVCacheRef]], num_tensors: int
|
||||
) -> list[int]:
|
||||
"""Canonical CPU bytes per GPU block for each tensor, taken from the refs'
|
||||
mappings. Requires every ref to carry a mapping."""
|
||||
mappings. Requires every ref to carry a mapping.
|
||||
"""
|
||||
canonical_bytes_per_block = [0] * num_tensors
|
||||
for layer_refs in layer_refs_per_group:
|
||||
for ref in layer_refs:
|
||||
@@ -234,8 +237,7 @@ def _new_descriptor_buffers(
|
||||
|
||||
|
||||
class SingleDirectionOffloadingHandler:
|
||||
"""
|
||||
Handles transfers for a single direction, either CPU->GPU or GPU->CPU.
|
||||
"""Handles transfers for a single direction, either CPU->GPU or GPU->CPU.
|
||||
Transfers are guaranteed to be executed in order of their submission.
|
||||
Each transfer uses a unique CUDA stream, and its stream will start
|
||||
executing only after the streams of previous transfers have finished.
|
||||
@@ -250,8 +252,7 @@ class SingleDirectionOffloadingHandler:
|
||||
gpu_to_cpu: bool,
|
||||
canonical_layout: bool = False,
|
||||
):
|
||||
"""
|
||||
Initialize a SingleDirectionOffloadingHandler.
|
||||
"""Initialize a SingleDirectionOffloadingHandler.
|
||||
|
||||
Args:
|
||||
gpu_tensors: list of GPU KV cache tensors.
|
||||
@@ -259,10 +260,12 @@ class SingleDirectionOffloadingHandler:
|
||||
cpu_tensors: list of CPU KV cache tensors.
|
||||
Each of shape (num_cpu_chunks, cpu_page_size_bytes) with dtype int8.
|
||||
Order should match gpu_tensors.
|
||||
blocks_per_chunk: number of blocks transferred per chunk.
|
||||
layer_refs_per_group: list of CanonicalKVCacheRef per group.
|
||||
gpu_to_cpu: if True, transfer from GPU to CPU; otherwise CPU to GPU.
|
||||
canonical_layout: if True, CPU pages use the canonical layout
|
||||
described by the refs' mappings.
|
||||
|
||||
"""
|
||||
assert len(gpu_tensors) == len(cpu_tensors)
|
||||
assert len(gpu_tensors) > 0
|
||||
@@ -340,7 +343,8 @@ class SingleDirectionOffloadingHandler:
|
||||
"""Upper bound on the number of copy descriptors for a transfer.
|
||||
|
||||
Exact for the direct layout. The canonical path may fill fewer:
|
||||
writer rotation later drops the blocks this rank does not write."""
|
||||
writer rotation later drops the blocks this rank does not write.
|
||||
"""
|
||||
num_copy_ops = 0
|
||||
for g_idx, (group_size, layer_refs) in enumerate(
|
||||
zip(group_sizes, self.layer_refs_per_group)
|
||||
@@ -369,7 +373,8 @@ class SingleDirectionOffloadingHandler:
|
||||
"""Fill one group's copy descriptors for the direct (worker-private)
|
||||
layout: one whole-page copy per (block, ref).
|
||||
|
||||
Returns (op_idx past the filled descriptors, bytes added)."""
|
||||
Returns (op_idx past the filled descriptors, bytes added).
|
||||
"""
|
||||
num_bytes = 0
|
||||
for data_ref in self.layer_refs_per_group[g_idx]:
|
||||
t_idx = data_ref.tensor_idx
|
||||
@@ -412,7 +417,8 @@ class SingleDirectionOffloadingHandler:
|
||||
scatter each block through the ref's precomputed CopyPlan, keeping
|
||||
only the blocks this rank writes.
|
||||
|
||||
Returns (op_idx past the filled descriptors, bytes added)."""
|
||||
Returns (op_idx past the filled descriptors, bytes added).
|
||||
"""
|
||||
assert self._canonical_copy_plans is not None
|
||||
# Zero-copy reinterpretation for pointer arithmetic: uint64 and the
|
||||
# buffers' int64 are bit-equivalent for addresses
|
||||
@@ -499,7 +505,8 @@ class SingleDirectionOffloadingHandler:
|
||||
) -> tuple[np.ndarray, np.ndarray]:
|
||||
"""Keep only the blocks this rank writes: replicated ranks take turns
|
||||
writing shared canonical pages, keyed by the rank-consistent CPU-side
|
||||
canonical page id."""
|
||||
canonical page id.
|
||||
"""
|
||||
cpu_page_ids = _canonical_page_ids(
|
||||
group_dst,
|
||||
self.dst_blocks_per_chunk,
|
||||
|
||||
@@ -44,8 +44,7 @@ class _RequestCacheAccess:
|
||||
|
||||
|
||||
class CPUOffloadingManager(OffloadingManager):
|
||||
"""
|
||||
An OffloadingManager with a pluggable CachePolicy, resolved by name via
|
||||
"""An OffloadingManager with a pluggable CachePolicy, resolved by name via
|
||||
CachePolicyFactory (built in: "lru", "arc"; external policies can either
|
||||
register their own or be loaded out-of-tree via cache_policy_module_path).
|
||||
|
||||
|
||||
@@ -14,8 +14,7 @@ from vllm.v1.kv_offload.cpu.policies.base import (
|
||||
|
||||
|
||||
class ARCCachePolicy(CachePolicy):
|
||||
"""
|
||||
ARC (Adaptive Replacement Cache) cache policy.
|
||||
"""ARC (Adaptive Replacement Cache) cache policy.
|
||||
|
||||
Data Structures:
|
||||
T1: Recent cache containing chunks accessed once.
|
||||
|
||||
@@ -23,8 +23,7 @@ def order_request_keys(
|
||||
|
||||
|
||||
class ChunkStatus(ctypes.Structure):
|
||||
"""
|
||||
Offloading status for a single chunk of KV data.
|
||||
"""Offloading status for a single chunk of KV data.
|
||||
Holds the following information:
|
||||
|
||||
ref_cnt - the current number of transfers using this chunk as a source.
|
||||
@@ -42,15 +41,12 @@ class ChunkStatus(ctypes.Structure):
|
||||
|
||||
@property
|
||||
def is_ready(self) -> bool:
|
||||
"""
|
||||
Returns whether the chunk is ready to be read.
|
||||
"""
|
||||
"""Returns whether the chunk is ready to be read."""
|
||||
return self.ref_cnt >= 0
|
||||
|
||||
|
||||
class CachePolicy(ABC):
|
||||
"""
|
||||
Encapsulates both chunk organization (data structures) and replacement
|
||||
"""Encapsulates both chunk organization (data structures) and replacement
|
||||
decisions (which chunk to evict). LRU and ARC differ in both dimensions —
|
||||
ARC's ghost lists and target_t1_size live at the intersection of storage
|
||||
and eviction, so they cannot be separated cleanly.
|
||||
@@ -73,12 +69,12 @@ class CachePolicy(ABC):
|
||||
|
||||
@abstractmethod
|
||||
def touch(self, keys: Iterable[OffloadKey], req_context: ReqContext) -> None:
|
||||
"""
|
||||
Mark chunks as recently used.
|
||||
"""Mark chunks as recently used.
|
||||
|
||||
Args:
|
||||
keys: Chunks to mark as recently used.
|
||||
req_context: Per-request context for the request touching these chunks.
|
||||
|
||||
"""
|
||||
|
||||
def on_store_miss(
|
||||
@@ -120,8 +116,7 @@ class CachePolicy(ABC):
|
||||
def evict(
|
||||
self, n: int, protected: set[OffloadKey]
|
||||
) -> list[tuple[OffloadKey, ChunkStatus]] | None:
|
||||
"""
|
||||
Evict exactly n chunks, skipping any in protected.
|
||||
"""Evict exactly n chunks, skipping any in protected.
|
||||
|
||||
Returns a list of (key, chunk) for the evicted chunks,
|
||||
or None if n evictions cannot be satisfied. The operation is atomic:
|
||||
@@ -133,8 +128,7 @@ class CachePolicy(ABC):
|
||||
|
||||
@abstractmethod
|
||||
def clear(self) -> None:
|
||||
"""
|
||||
Remove ALL chunks regardless of ref_cnt.
|
||||
"""Remove ALL chunks regardless of ref_cnt.
|
||||
|
||||
Ghost lists and adaptive state are also reset.
|
||||
"""
|
||||
|
||||
@@ -57,6 +57,7 @@ class CachePolicyFactory:
|
||||
|
||||
Raises ValueError if the cache policy is neither registered nor
|
||||
resolvable via `module_path`.
|
||||
|
||||
"""
|
||||
if name in cls._registry:
|
||||
return cls._registry[name]()
|
||||
|
||||
@@ -14,8 +14,7 @@ from vllm.v1.kv_offload.cpu.policies.base import (
|
||||
|
||||
|
||||
class LRUCachePolicy(CachePolicy):
|
||||
"""
|
||||
LRU caching policy with logical recency independent of transfer pinning.
|
||||
"""LRU caching policy with logical recency independent of transfer pinning.
|
||||
|
||||
Evictable chunks live in a lazy-invalidating min-heap. A chunk's recency
|
||||
can therefore be updated while it is pinned; when it later becomes
|
||||
|
||||
@@ -69,8 +69,7 @@ def _get_populate_write_fn(
|
||||
|
||||
|
||||
class SharedOffloadRegion:
|
||||
"""
|
||||
Single mmap-backed memory region shared across all workers for a
|
||||
"""Single mmap-backed memory region shared across all workers for a
|
||||
vLLM instance. Workers coordinate via the filesystem: the first worker
|
||||
to open the file with O_EXCL becomes the creator and calls ftruncate;
|
||||
the rest open the existing file and wait until it reaches the expected
|
||||
@@ -264,6 +263,7 @@ class SharedOffloadRegion:
|
||||
|
||||
Args:
|
||||
tensor_page_size: Bytes per chunk for this tensor.
|
||||
|
||||
"""
|
||||
assert self.rank is not None
|
||||
new_offset = self._worker_offset + tensor_page_size
|
||||
@@ -313,6 +313,7 @@ class SharedOffloadRegion:
|
||||
|
||||
Args:
|
||||
tensor_page_size: Canonical bytes per chunk for this tensor.
|
||||
|
||||
"""
|
||||
new_offset = self._canonical_offset + tensor_page_size
|
||||
assert new_offset <= self._row_stride
|
||||
|
||||
@@ -28,7 +28,8 @@ def _all_workers_barrier() -> None:
|
||||
"""Block until every worker rank has reached this point (gloo cpu group).
|
||||
|
||||
A superset of the node-local mmap openers suffices: once the barrier
|
||||
releases, every worker sharing the region file has mapped it."""
|
||||
releases, every worker sharing the region file has mapped it.
|
||||
"""
|
||||
from vllm.distributed.parallel_state import (
|
||||
get_inner_dp_world_group,
|
||||
get_world_group,
|
||||
@@ -161,7 +162,8 @@ class CPUOffloadingSpec(OffloadingSpec):
|
||||
|
||||
def _uses_shared_region(self) -> bool:
|
||||
"""Whether the worker CPU buffer is the shared mmap region (vs a private
|
||||
per-rank tensor); replicated-layout dedup is gated on this being True."""
|
||||
per-rank tensor); replicated-layout dedup is gated on this being True.
|
||||
"""
|
||||
return current_platform.is_cuda_alike()
|
||||
|
||||
def create_worker(self, kv_caches: CanonicalKVCaches) -> CPUOffloadingWorker:
|
||||
|
||||
Reference in New Issue
Block a user