Add pydocstyle to the ruff rules (#52136)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Signed-off-by: Harry Mellor <hej.mellor@gmail.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Harry Mellor
2026-09-16 16:44:29 +00:00
committed by GitHub
co-authored by Claude Opus 5
parent 804e5377aa
commit 8c1557a79c
2002 changed files with 15024 additions and 14269 deletions
+21 -18
View File
@@ -31,8 +31,7 @@ logger = init_logger(__name__)
class BlockHashToBlockMap:
"""
Cache of blocks that are used for prefix caching. It caches blocks
"""Cache of blocks that are used for prefix caching. It caches blocks
from hash directly to a block or multiple blocks
(i.e. {block_hash: KVCacheBlocks})
- Mostly block_hash maps to a single KVCacheBlock, and KVCacheBlocks
@@ -59,9 +58,7 @@ class BlockHashToBlockMap:
] = {}
def get_one_block(self, key: BlockHashWithGroupId) -> KVCacheBlock | None:
"""
Gets any block with the given block hash key.
"""
"""Gets any block with the given block hash key."""
blocks = self._cache.get(key)
if blocks is not None:
if isinstance(blocks, KVCacheBlock):
@@ -72,9 +69,7 @@ class BlockHashToBlockMap:
return None
def contain(self, key: BlockHashWithGroupId, block_id: int) -> bool:
"""
Checks whether the key maps to the given block ID.
"""
"""Checks whether the key maps to the given block ID."""
blocks = self._cache.get(key)
if blocks is None:
return False
@@ -86,9 +81,7 @@ class BlockHashToBlockMap:
return False
def insert(self, key: BlockHashWithGroupId, block: KVCacheBlock) -> None:
"""
Inserts the KVCacheBlock to the cache
"""
"""Inserts the KVCacheBlock to the cache"""
blocks = self._cache.get(key)
if blocks is None:
# When key is not found, attach a single block to the key
@@ -104,9 +97,7 @@ class BlockHashToBlockMap:
self._unexpected_blocks_type(blocks)
def pop(self, key: BlockHashWithGroupId, block_id: int) -> KVCacheBlock | None:
"""
Checks if block_hash exists and pop block_id from the cache
"""
"""Checks if block_hash exists and pop block_id from the cache"""
blocks = self._cache.pop(key, None)
if blocks is None:
# block_hash not found in the cache
@@ -158,6 +149,7 @@ class BlockPool:
enable_kv_cache_events: Whether to enable kv cache events.
metrics_collector: Optional metrics collector for tracking block residency.
medium: Storage medium reported in KV cache events.
"""
def __init__(
@@ -214,6 +206,7 @@ class BlockPool:
Returns:
The cached blocks if exists, or None.
"""
cached_blocks = []
for group_id in kv_cache_group_ids:
@@ -261,6 +254,7 @@ class BlockPool:
consults a subset of blocks (e.g. SWA tail-window), so blocks
that can never serve a hit stay out of the prefix-cache hash
map.
"""
if num_cached_blocks >= num_full_blocks:
return
@@ -395,6 +389,7 @@ class BlockPool:
num_cached_blocks: Number of blocks that were cache hits.
block_size: Number of tokens per block.
kv_cache_group_id: The KV cache group ID.
"""
if not self.enable_kv_cache_events or num_cached_blocks == 0:
return
@@ -491,6 +486,7 @@ class BlockPool:
Returns:
The hash key with group ID if a partial entry can be registered;
otherwise ``None`` for null blocks.
"""
if block.is_null:
return None
@@ -671,6 +667,7 @@ class BlockPool:
Returns:
A list of new block.
"""
if num_blocks > self.get_num_free_blocks():
raise ValueError(f"Cannot get {num_blocks} free blocks from the pool")
@@ -724,8 +721,7 @@ class BlockPool:
self.free_block_queue.append_n(released)
def _maybe_evict_cached_block(self, block: KVCacheBlock) -> bool:
"""
If a block is cached in `cached_block_hash_to_block`, we reset its hash
"""If a block is cached in `cached_block_hash_to_block`, we reset its hash
metadata and evict it from the cache.
Args:
@@ -733,6 +729,7 @@ class BlockPool:
Returns:
True if the block is evicted, False otherwise.
"""
# Clean up metrics tracking first to prevent leaks
if self.metrics_collector:
@@ -753,6 +750,7 @@ class BlockPool:
Args:
blocks: A list of blocks to touch.
"""
for block in blocks:
# ref_cnt=0 means this block is in the free list (i.e. eviction
@@ -774,6 +772,7 @@ class BlockPool:
Args:
ordered_blocks: A list of blocks to free ordered by their eviction
priority.
"""
# Identify blocks with hash (LRU cache) and without it (never match APC)
blocks_to_evict_last = []
@@ -800,7 +799,7 @@ class BlockPool:
pool.free_blocks(blocks)
def evict_blocks(self, block_ids: set[int]) -> None:
"""evict blocks from the prefix cache by their block IDs.
"""Evict blocks from the prefix cache by their block IDs.
only evicts blocks that are currently cached (have a hash). blocks
with ref_cnt > 0 are not freed from the block pool, only evicted
@@ -808,6 +807,7 @@ class BlockPool:
Args:
block_ids: Set of block IDs to evict from cache.
"""
for block_id in block_ids:
assert block_id < len(self.blocks), (
@@ -826,6 +826,7 @@ class BlockPool:
Returns:
bool: True if the prefix cache is successfully reset,
False otherwise.
"""
num_used_blocks = self.num_gpu_blocks - self.get_num_free_blocks()
if num_used_blocks != 1: # The null block is always marked as used
@@ -863,6 +864,7 @@ class BlockPool:
Returns:
The number of free blocks.
"""
return self.free_block_queue.num_free_blocks
@@ -871,8 +873,8 @@ class BlockPool:
Returns:
The KV cache usage (between 0.0 and 1.0).
"""
"""
# Subtract 1 to account for null block.
total_gpu_blocks = self.num_gpu_blocks - 1
if not total_gpu_blocks:
@@ -884,6 +886,7 @@ class BlockPool:
Returns:
A list of KV cache events.
"""
if not self.enable_kv_cache_events:
return []
+7 -3
View File
@@ -64,6 +64,7 @@ class EncoderCacheManager:
make space when needed.
freed: List of mm_hash strings that were actually evicted since the
last call to get_freed_mm_hashes(). This list is cleared on return.
"""
@classmethod
@@ -113,6 +114,7 @@ class EncoderCacheManager:
Returns:
True if the encoder output for this input is already cached
"""
mm_hash = request.mm_features[input_id].identifier
# Not cached at all
@@ -162,6 +164,7 @@ class EncoderCacheManager:
Note: This method does not allocate physical memory for the encoder
output but only the state of EncoderCacheManager.
"""
num_embeds = request.get_num_encoder_embeds(input_id)
@@ -198,8 +201,8 @@ class EncoderCacheManager:
Note:
This method assumes can_allocate() returned True for the same input.
"""
"""
mm_hash = request.mm_features[input_id].identifier
request_id = request.request_id
if mm_hash not in self.cached:
@@ -222,7 +225,7 @@ class EncoderCacheManager:
return self.request_cached_ids.get(request.request_id, set())
def free_encoder_input(self, request: Request, input_id: int) -> None:
"""Free the request's reference to the encoder input (`mm_data`)
"""Free the request's reference to the encoder input (`mm_data`).
When the reference set for the corresponding `mm_hash` becomes empty,
the entry is appended to `freeable` and `num_freeable_slots` is
@@ -279,6 +282,7 @@ class EncoderCacheManager:
call to be used by the scheduler to notify workers about which
encoder outputs can be removed from their caches. The internal
list is cleared after this call.
"""
# An entry evicted early in the scheduling pass can be allocated again
# later in the same pass. Keep its worker-side tensor in that case.
@@ -307,8 +311,8 @@ def compute_mm_encoder_budget(
from the input sequence.
- Space budget for encoder cache size, measured in number of tokens
from the input sequence.
"""
"""
if not mm_max_toks_per_item:
logger.warning(
"All non-text modalities supported by the model have been "
+27 -35
View File
@@ -64,9 +64,7 @@ def _validate_prefix_cache_retention_interval(
class KVCacheCoordinator(ABC):
"""
Coordinate the KV cache of different KV cache groups.
"""
"""Coordinate the KV cache of different KV cache groups."""
enable_partial_hash_hits = False
@@ -179,8 +177,7 @@ class KVCacheCoordinator(ABC):
num_tokens_main_model: int,
apply_admission_cap: bool = False,
) -> int:
"""
Get the number of device blocks needed to be allocated for the request.
"""Get the number of device blocks needed to be allocated for the request.
Args:
request_id: The request ID.
@@ -203,6 +200,7 @@ class KVCacheCoordinator(ABC):
Returns:
The number of blocks to allocate.
"""
num_blocks_to_allocate = 0
for i, manager in enumerate(self.single_type_managers):
@@ -237,8 +235,7 @@ class KVCacheCoordinator(ABC):
num_local_computed_tokens: int,
num_external_computed_tokens: int,
) -> None:
"""
Add the new computed blocks to the request. Optionally allocate new
"""Add the new computed blocks to the request. Optionally allocate new
blocks for external computed tokens (if any).
Args:
@@ -247,6 +244,7 @@ class KVCacheCoordinator(ABC):
prefix cache.
num_local_computed_tokens: The number of local computed tokens.
num_external_computed_tokens: The number of external computed tokens.
"""
# A running request is already tracked in num_cached_block and won't
# have new prefix-cache hits, so this is a no-op for it.
@@ -283,8 +281,7 @@ class KVCacheCoordinator(ABC):
num_tokens_main_model: int,
num_encoder_tokens: int = 0,
) -> tuple[list[KVCacheBlock], ...]:
"""
Allocate new blocks for the request to give it at least `num_tokens`
"""Allocate new blocks for the request to give it at least `num_tokens`
token slots.
Args:
@@ -299,6 +296,7 @@ class KVCacheCoordinator(ABC):
Returns:
The new allocated blocks.
"""
return tuple(
manager.allocate_new_blocks(
@@ -333,14 +331,14 @@ class KVCacheCoordinator(ABC):
return tuple(sorted({max(resend - block, 0), max(extension - block, 0)}))
def cache_blocks(self, request: Request, num_computed_tokens: int) -> None:
"""
Cache the blocks for the request.
"""Cache the blocks for the request.
Args:
request: The request.
num_computed_tokens: The total number of tokens
that need to be cached
(including tokens that are already cached).
"""
boundaries = self.get_replay_boundaries(request)
for manager in self.single_type_managers:
@@ -380,18 +378,17 @@ class KVCacheCoordinator(ABC):
return all([pool.reset_prefix_cache() for pool in pools])
def free(self, request_id: str) -> None:
"""
Free the blocks for the request.
"""Free the blocks for the request.
Args:
request_id: The request ID.
"""
for manager in self.single_type_managers:
manager.free(request_id)
def pop_blocks_for_free(self, request_id: str) -> list[KVCacheBlock]:
"""
Pop the request's bookkeeping from all single-type managers and
"""Pop the request's bookkeeping from all single-type managers and
return its blocks without returning them to the block pool. The
caller must eventually pass the returned blocks to
`block_pool.free_blocks`, freeing them in reverse order (so that
@@ -402,6 +399,7 @@ class KVCacheCoordinator(ABC):
Returns:
The request's blocks in allocation order.
"""
blocks: list[KVCacheBlock] = []
for manager in self.single_type_managers:
@@ -409,8 +407,7 @@ class KVCacheCoordinator(ABC):
return blocks
def get_num_common_prefix_blocks(self, running_request_id: str) -> list[int]:
"""
Get the number of common prefix blocks for all requests with allocated
"""Get the number of common prefix blocks for all requests with allocated
KV cache for each kv cache group.
Args:
@@ -419,6 +416,7 @@ class KVCacheCoordinator(ABC):
Returns:
list[int]: The number of common prefix blocks for each kv cache group.
"""
return [
manager.get_num_common_prefix_blocks(running_request_id)
@@ -431,8 +429,7 @@ class KVCacheCoordinator(ABC):
processed_computed_tokens: int,
num_prompt_tokens: int | None = None,
) -> None:
"""
Remove the blocks that are no longer needed from `blocks` and replace
"""Remove the blocks that are no longer needed from `blocks` and replace
the removed blocks with null_block.
Args:
@@ -442,6 +439,7 @@ class KVCacheCoordinator(ABC):
num_prompt_tokens: Optional prompt length. R-SWA managers use this to
free gap blocks between the prefill tail and decode window; other
manager types ignore it.
"""
for manager in self.single_type_managers:
manager.remove_skipped_blocks(
@@ -449,9 +447,7 @@ class KVCacheCoordinator(ABC):
)
def get_blocks(self, request_id: str) -> tuple[list[KVCacheBlock], ...]:
"""
Get the blocks for the request.
"""
"""Get the blocks for the request."""
return tuple(
manager.req_to_blocks.get(request_id) or []
for manager in self.single_type_managers
@@ -465,7 +461,8 @@ class KVCacheCoordinator(ABC):
) -> tuple[tuple[list[KVCacheBlock], ...], int, int]:
"""Returns the per-group hit blocks, the hit length, and the number of
``num_uncached_common_prefix_tokens`` (a shared prefix that a
sparse-retention group has not cached yet; 0 unless hybrid)."""
sparse-retention group has not cached yet; 0 unless hybrid).
"""
pass
def new_step_starts(self) -> None:
@@ -475,8 +472,7 @@ class KVCacheCoordinator(ABC):
class KVCacheCoordinatorNoPrefixCache(KVCacheCoordinator):
"""
KV cache coordinator to use if prefix caching is disabled or unsupported.
"""KV cache coordinator to use if prefix caching is disabled or unsupported.
In contrast to UnitaryKVCacheCoordinator and HybridKVCacheCoordinator,
supports arbitrary numbers of KV cache groups (including 0 groups).
Does not implement any features related to prefix caching.
@@ -527,8 +523,7 @@ class KVCacheCoordinatorNoPrefixCache(KVCacheCoordinator):
class UnitaryKVCacheCoordinator(KVCacheCoordinator):
"""
KV cache coordinator for models with only one KV cache group. This is the
"""KV cache coordinator for models with only one KV cache group. This is the
case for models with only one KV cache type, e.g., all attention layers use
full attention or all attention layers use sliding window attention.
"""
@@ -613,8 +608,7 @@ class SpecGroup(NamedTuple):
class HybridKVCacheCoordinator(KVCacheCoordinator):
"""
KV cache coordinator for hybrid models with multiple KV cache types, and
"""KV cache coordinator for hybrid models with multiple KV cache types, and
thus multiple kv cache groups.
"""
@@ -733,8 +727,7 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
)
def verify_and_split_kv_cache_groups(self) -> None:
"""
Groups KV cache groups by their spec type for efficient batch processing
"""Groups KV cache groups by their spec type for efficient batch processing
during cache hit lookup.
"""
self.attention_groups: list[SpecGroup] = []
@@ -843,8 +836,7 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
block_hashes: list[BlockHash],
max_cache_hit_length: int,
) -> tuple[tuple[list[KVCacheBlock], ...], int, int]:
"""
Find the longest cache hit using an iterative fixed-point algorithm.
"""Find the longest cache hit using an iterative fixed-point algorithm.
Each attention type either accepts the current candidate length or
reduces it. If any type reduces the length, restart checks over all
@@ -861,8 +853,8 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
- The number of tokens of the reconciled (combined) cache hit.
- ``num_uncached_common_prefix_tokens``: a shared prefix that a
sparse-retention group has not cached yet (0 unless hybrid).
"""
"""
num_groups = len(self.kv_cache_config.kv_cache_groups)
hit_length = max_cache_hit_length
longest_hit_length = 0
@@ -985,8 +977,8 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
Returns:
(blocks_per_group, hit_lengths_per_group)
"""
"""
num_groups = len(self.kv_cache_config.kv_cache_groups)
hit_blocks: list[list[KVCacheBlock]] = [[] for _ in range(num_groups)]
hit_lengths: list[int] = [0] * num_groups
+20 -9
View File
@@ -33,8 +33,7 @@ logger = init_logger(__name__)
@dataclass
class KVCacheBlocks:
"""
The allocation result of KVCacheManager, work as the interface between
"""The allocation result of KVCacheManager, work as the interface between
Scheduler and KVCacheManager, to hide KVCacheManager's internal data
structure from the Scheduler.
"""
@@ -85,8 +84,7 @@ class KVCacheBlocks:
*,
group_ids: Sequence[int] | None = None,
) -> tuple[list[int], ...] | None:
"""
Converts the KVCacheBlocks instance to block_ids.
"""Converts the KVCacheBlocks instance to block_ids.
Args:
allow_none: Return None when every selected group is empty.
@@ -97,6 +95,7 @@ class KVCacheBlocks:
- the outer tuple corresponds to KV cache groups
- each inner list contains the block_ids of the blocks in that
group
"""
groups = (
self.blocks
@@ -125,9 +124,7 @@ class KVCacheBlocks:
]
def new_empty(self) -> "KVCacheBlocks":
"""
Creates a new KVCacheBlocks instance with no blocks.
"""
"""Creates a new KVCacheBlocks instance with no blocks."""
return KVCacheBlocks(tuple(() for _ in range(len(self.blocks))))
@@ -232,6 +229,7 @@ class KVCacheManager:
Returns:
The KV cache usage (between 0.0 and 1.0).
"""
return self.block_pool.get_usage()
@@ -240,6 +238,7 @@ class KVCacheManager:
Returns:
The current prefix caching stats, or None if logging is disabled.
"""
if not self.log_stats:
return None
@@ -278,6 +277,7 @@ class KVCacheManager:
window) has not cached yet (Marconi-style APC), or 0 if none.
Pinned so sparse prefix-cache retention does not drop
the junction and defeat cross-request reuse.
"""
# We skip finding the prefix cache hit when prefix caching is
# disabled or the request is marked as skipping kv cache read
@@ -340,6 +340,7 @@ class KVCacheManager:
Returns:
The ``get_computed_blocks`` triple (blocks, number of local computed
tokens, shared-prefix boundary) plus ``hit_diverged``.
"""
coordinator = self.coordinator
if not (
@@ -462,6 +463,7 @@ class KVCacheManager:
Returns:
A list of new allocated blocks.
"""
# When loading KV data asynchronously, we may have zero new tokens to
# compute while still allocating slots for externally computed tokens.
@@ -607,6 +609,7 @@ class KVCacheManager:
Args:
request: The request to free the blocks.
"""
self.coordinator.free(request.request_id)
@@ -624,6 +627,7 @@ class KVCacheManager:
processed_computed_tokens: Computed-token prefix length covering
fully processed and committed tokens only (safe to free).
num_prompt_tokens: Optional prompt length for R-SWA gap eviction.
"""
self.coordinator.remove_skipped_blocks(
request_id, processed_computed_tokens, num_prompt_tokens
@@ -639,14 +643,16 @@ class KVCacheManager:
Returns:
The request's blocks in allocation order.
"""
return self.coordinator.pop_blocks_for_free(request.request_id)
def evict_blocks(self, block_ids: set[int]) -> None:
"""evict blocks from the prefix cache by their block IDs.
"""Evict blocks from the prefix cache by their block IDs.
Args:
block_ids: Set of block IDs to evict from cache.
"""
self.block_pool.evict_blocks(block_ids)
@@ -658,6 +664,7 @@ class KVCacheManager:
Returns:
bool: True if the prefix cache is successfully reset,
False otherwise.
"""
if not self.coordinator.reset_prefix_cache():
return False
@@ -697,6 +704,7 @@ class KVCacheManager:
Returns:
list[int]: The number of common prefix blocks for each kv cache
group.
"""
return self.coordinator.get_num_common_prefix_blocks(running_request_id)
@@ -705,6 +713,7 @@ class KVCacheManager:
Returns:
A list of KV cache events.
"""
events = self.block_pool.take_events()
for event in events:
@@ -792,6 +801,7 @@ class KVCacheManager:
request: The request to cache the blocks.
num_computed_tokens: The number of computed tokens, including tokens
that are already cached and tokens to be cached.
"""
if self.enable_caching:
self.coordinator.cache_blocks(request, num_computed_tokens)
@@ -846,7 +856,8 @@ class KVCacheManager:
self, request_id: str, start_token: int, end_token: int
) -> list[int]:
"""The request's block ids covering [start_token, end_token), from
the groups whose new blocks are zeroed by the worker."""
the groups whose new blocks are zeroed by the worker.
"""
ids: list[int] = []
for mgr in self.coordinator.single_type_managers:
if mgr.records_new_block_ids:
+57 -60
View File
@@ -263,6 +263,7 @@ class FreeKVCacheBlockQueue:
Args:
blocks: A list of KVCacheBlock objects.
"""
def __init__(self, blocks: list[KVCacheBlock]) -> None:
@@ -300,6 +301,7 @@ class FreeKVCacheBlockQueue:
Returns:
The first free block.
"""
if (
self.fake_free_list_head.next_free_block is self.fake_free_list_tail
@@ -340,6 +342,7 @@ class FreeKVCacheBlockQueue:
Returns:
A list of n free blocks.
"""
if n == 0:
return []
@@ -370,6 +373,7 @@ class FreeKVCacheBlockQueue:
Args:
block: The block to remove.
"""
if block.prev_free_block is None or block.next_free_block is None:
# This should not happen if the block is from the free list.
@@ -391,6 +395,7 @@ class FreeKVCacheBlockQueue:
Args:
block: The block to append.
"""
if self.fake_free_list_tail.prev_free_block is None:
raise RuntimeError(
@@ -434,6 +439,7 @@ class FreeKVCacheBlockQueue:
Args:
blocks: The blocks to append.
"""
if len(blocks) == 0:
return
@@ -459,6 +465,7 @@ class FreeKVCacheBlockQueue:
Returns:
A list of free blocks.
"""
ret = []
if self.fake_free_list_head.next_free_block is None:
@@ -505,6 +512,7 @@ def _gen_mm_extra_hash_keys(
Returns:
A tuple of extra keys and the next multi-modal index.
"""
extra_keys: list[Any] = []
@@ -565,6 +573,7 @@ def _gen_lora_extra_hash_keys(request: Request) -> list[str]:
Returns:
Return LoRA name of the request if it is a LoRA request. Return empty
list otherwise.
"""
if not request.lora_request:
return []
@@ -584,6 +593,7 @@ def _gen_prompt_embeds_extra_hash_keys(
Returns:
Return a stable hash of the block prompt embeddings if prompt embeds
are present. Return empty list otherwise.
"""
if request.prompt_embeds is None:
return []
@@ -612,6 +622,7 @@ def generate_block_hash_extra_keys(
Returns:
A tuple of extra keys and the next multi-modal index.
"""
mm_extra_keys: list[Any]
mm_extra_keys, new_start_mm_idx = _gen_mm_extra_hash_keys(
@@ -645,6 +656,7 @@ def hash_block_tokens(
the contents of the preceding block(s). The hash value is used for
prefix caching. We use LRU cache for this function to avoid recomputing
hash values for the same block contents.
Args:
hash_function: The hash function used to compute block hash.
parent_block_hash: The hash of the parent block. None
@@ -652,9 +664,11 @@ def hash_block_tokens(
curr_block_token_ids: A list of token ids in the current
block. The current block is assumed to be full.
extra_keys: Extra keys for the block.
Returns:
The hash value of the block and the token ids in the block.
The entire tuple is used as the hash key of the block.
"""
if not parent_block_hash:
parent_block_hash = NONE_HASH
@@ -813,8 +827,7 @@ def get_request_block_hasher(
hash_block_size: int,
caching_hash_fn: Callable[[Any], bytes],
) -> Callable[[Request], list[BlockHash]]:
"""
Returns a function which computes the list of un-computed block hashes
"""Returns a function which computes the list of un-computed block hashes
of a request.
Hashes are computed at ``hash_block_size`` granularity and chained over the
@@ -912,9 +925,7 @@ def _check_enough_kv_cache_memory(
def max_memory_usage_bytes(
vllm_config: VllmConfig, kv_cache_specs: Iterable[KVCacheSpec]
) -> int:
"""
Get the maximum memory usage in bytes for the given KV cache specs.
"""
"""Get the maximum memory usage in bytes for the given KV cache specs."""
return sum(spec.max_memory_usage_bytes(vllm_config) for spec in kv_cache_specs)
@@ -923,8 +934,7 @@ def estimate_max_model_len(
kv_cache_spec: dict[str, KVCacheSpec],
available_memory: int,
) -> int:
"""
Estimates the maximum model length that can fit in the available memory
"""Estimates the maximum model length that can fit in the available memory
using binary search.
This function temporarily modifies max_model_len during estimation but
@@ -937,6 +947,7 @@ def estimate_max_model_len(
Returns:
The estimated maximum model length that can fit in the available memory.
"""
# Save the original max_model_len to restore after estimation
original_max_model_len = vllm_config.model_config.max_model_len
@@ -977,8 +988,7 @@ def check_enough_kv_cache_memory(
kv_cache_spec: dict[str, KVCacheSpec],
available_memory: int,
):
"""
Checks whether `available_memory` is enough for the KV cache to hold at
"""Checks whether `available_memory` is enough for the KV cache to hold at
least one request with the model's max_model_len.
Args:
@@ -988,8 +998,8 @@ def check_enough_kv_cache_memory(
Raises:
ValueError: If there is not enough memory available for the KV cache.
"""
"""
# No need to check for available memory if the kv_cache_spec is empty
if kv_cache_spec:
# Reserve the null block BlockPool permanently holds back, so the check
@@ -1012,8 +1022,7 @@ def check_enough_kv_cache_memory(
def create_kv_cache_group_specs(
kv_cache_spec: dict[str, KVCacheSpec], grouped_layer_names: list[list[str]]
) -> list[KVCacheGroupSpec]:
"""
Create KVCacheGroupSpec object for each kv cache group layer.
"""Create KVCacheGroupSpec object for each kv cache group layer.
The layers in the same group should share the same
KVCacheSpec.
@@ -1024,8 +1033,10 @@ def create_kv_cache_group_specs(
A list of kv cache groups, where each element is a list of layer
names that belong to the same group and should share the same
KVCacheSpec.
Returns:
A list of KVCacheGroupSpec objects, one for each group.
"""
kv_cache_groups = []
for layer_names_one_group in grouped_layer_names:
@@ -1040,8 +1051,7 @@ def create_kv_cache_group_specs(
def is_kv_cache_spec_uniform(kv_cache_spec: dict[str, KVCacheSpec]) -> bool:
"""
Whether all layers in the given KVCacheSpec have the same KV cache spec.
"""Whether all layers in the given KVCacheSpec have the same KV cache spec.
Note that we regard FullAttentionSpec with and without sliding window as
the same type.
@@ -1050,8 +1060,8 @@ def is_kv_cache_spec_uniform(kv_cache_spec: dict[str, KVCacheSpec]) -> bool:
Returns:
True if all layers have the same type, False otherwise.
"""
"""
if not kv_cache_spec:
# Encoder-only models do not have KV cache, kv_cache_type can be
# regarded as uniform.
@@ -1067,8 +1077,7 @@ def is_kv_cache_spec_uniform(kv_cache_spec: dict[str, KVCacheSpec]) -> bool:
def get_max_concurrency_for_kv_cache_config(
vllm_config: VllmConfig, kv_cache_config: KVCacheConfig
) -> float:
"""
Get the maximum concurrency for the given KV cache configuration.
"""Get the maximum concurrency for the given KV cache configuration.
A request at max_model_len consumes whole blocks from each group's block
table — cdiv(per-request bytes, page bytes) of the group's spec — and all
@@ -1101,8 +1110,7 @@ def get_max_concurrency_for_kv_cache_config(
def may_override_num_blocks(vllm_config: VllmConfig, num_blocks: int) -> int:
"""
Override the number of kv cache blocks if `num_gpu_blocks_override` is set.
"""Override the number of kv cache blocks if `num_gpu_blocks_override` is set.
The override is logged once, at the call site in `get_kv_cache_configs`.
"""
if vllm_config.cache_config.num_gpu_blocks_override is not None:
@@ -1111,8 +1119,7 @@ def may_override_num_blocks(vllm_config: VllmConfig, num_blocks: int) -> int:
def _pool_bytes_per_block(kv_cache_groups: list[KVCacheGroupSpec]) -> int:
"""
Bytes consumed by one block in the worker's shared KV cache pool, mirroring
"""Bytes consumed by one block in the worker's shared KV cache pool, mirroring
the divisor used by `get_kv_cache_config_from_groups` to convert
`available_memory` into `num_blocks`. Used to compute the effective KV cache
capacity once `num_gpu_blocks_override` is applied.
@@ -1121,9 +1128,7 @@ def _pool_bytes_per_block(kv_cache_groups: list[KVCacheGroupSpec]) -> int:
def get_uniform_page_size(kv_cache_specs: Iterable[KVCacheSpec]) -> int:
"""
Get the page size of the KV cache.
"""
"""Get the page size of the KV cache."""
page_sizes = {layer.page_size_bytes for layer in kv_cache_specs}
assert len(page_sizes) == 1
return page_sizes.pop()
@@ -1132,8 +1137,7 @@ def get_uniform_page_size(kv_cache_specs: Iterable[KVCacheSpec]) -> int:
def _get_kv_cache_groups_uniform_spec(
kv_cache_specs: dict[str, KVCacheSpec],
) -> list[KVCacheGroupSpec]:
"""
Generates the KV cache configuration for a model with the same KV cache
"""Generates the KV cache configuration for a model with the same KV cache
spec for all layers.
Args:
@@ -1141,16 +1145,15 @@ def _get_kv_cache_groups_uniform_spec(
Returns:
The generated KVCacheGroupSpecs
"""
"""
return create_kv_cache_group_specs(kv_cache_specs, [list(kv_cache_specs.keys())])
def _get_kv_cache_groups_uniform_type(
spec: UniformTypeKVCacheSpecs,
) -> list[KVCacheGroupSpec]:
"""
Generates the KV cache configuration for a model with one type of KV cache
"""Generates the KV cache configuration for a model with one type of KV cache
but different hidden sizes. All layers are merged into one group.
Args:
@@ -1158,8 +1161,8 @@ def _get_kv_cache_groups_uniform_type(
Returns:
The generated KVCacheGroupSpecs
"""
"""
return [KVCacheGroupSpec(list(spec.kv_cache_specs.keys()), spec)]
@@ -1365,8 +1368,7 @@ def _glm5_next_tensor_layout(
def unify_kv_cache_spec_page_size(
kv_cache_spec: dict[str, KVCacheSpec],
) -> dict[str, KVCacheSpec]:
"""
Unify the page size of the given KVCacheSpec. If the page size of all layers
"""Unify the page size of the given KVCacheSpec. If the page size of all layers
are the same, return the original KVCacheSpec. If not same, unify the page
size by increasing the block size of layers with smaller page size. Two
cases cannot be unified by block size alone and pad their physical page to
@@ -1385,6 +1387,7 @@ def unify_kv_cache_spec_page_size(
Returns:
The updated KVCacheSpec with the same page_size_bytes.
"""
page_sizes = {layer.page_size_bytes for layer in kv_cache_spec.values()}
if len(page_sizes) <= 1:
@@ -1435,8 +1438,7 @@ def is_kv_cache_type_attention_free(kv_cache_spec: dict[str, KVCacheSpec]) -> bo
def _get_kv_cache_groups_uniform_page_size(
kv_cache_spec: dict[str, KVCacheSpec],
) -> list[KVCacheGroupSpec]:
"""
Generates the KV cache groups for hybrid models with multiple
"""Generates the KV cache groups for hybrid models with multiple
attention types but still with a uniform page size (physical memory per
block per layer) for all layers.
@@ -1496,6 +1498,7 @@ def _get_kv_cache_groups_uniform_page_size(
kv_cache_spec: The KVCacheSpec of each attention layer in the model
Returns:
The generated KVCacheGroupSpecs
"""
# Group all layers by kv_cache_spec.
# E.g., 2 full attention layers and 3 sliding window attention layers,
@@ -1659,8 +1662,7 @@ def get_kv_cache_config_from_groups(
kv_cache_groups: list[KVCacheGroupSpec],
available_memory: int,
) -> KVCacheConfig:
"""
Generate the KV cache configuration from the KV cache groups and spec
"""Generate the KV cache configuration from the KV cache groups and spec
of each layer.
Args:
@@ -1669,6 +1671,7 @@ def get_kv_cache_config_from_groups(
available_memory: Memory available for KV cache in bytes
Returns:
The generated KVCacheConfig
"""
if len(kv_cache_groups) == 0:
# Attention free models do not have KV cache.
@@ -1926,15 +1929,14 @@ def _try_get_full_allocation_fallback_groups(
def unify_hybrid_kv_cache_specs(kv_cache_spec: dict[str, KVCacheSpec]):
"""
This function tries to convert the KV cache specs to one type if the model
"""This function tries to convert the KV cache specs to one type if the model
is a hybrid model with multiple type of KV cache. It will convert all
SlidingWindowSpec to FullAttentionSpec if both types are present.
Args:
kv_cache_spec: The kv cache spec of each attention layer in the model
"""
"""
if is_kv_cache_spec_uniform(
kv_cache_spec
) or UniformTypeKVCacheSpecs.is_uniform_type(kv_cache_spec):
@@ -2166,6 +2168,7 @@ def _annotate_eagle_groups(
kv_cache_groups: Groups to annotate in place.
use_deepseek_v4_fallback: Enable rule 2 for a DeepseekV4/V4.1 packed
group.
"""
spec_config = vllm_config.speculative_config
if spec_config is None or not spec_config.use_eagle_block_drop():
@@ -2196,6 +2199,7 @@ def _warn_if_unannotated_eagle_mamba(
Args:
vllm_config: Config supplying the speculative method, if any.
kv_cache_groups: Groups as they will be handed to consumers.
"""
spec_config = vllm_config.speculative_config
if spec_config is None or not spec_config.use_eagle():
@@ -2262,8 +2266,7 @@ def get_kv_cache_groups(
vllm_config: VllmConfig,
kv_cache_spec: dict[str, KVCacheSpec],
) -> list[KVCacheGroupSpec]:
"""
Split the layers in the model into groups with the same KV cache spec.
"""Split the layers in the model into groups with the same KV cache spec.
Args:
vllm_config: The global VllmConfig
@@ -2271,6 +2274,7 @@ def get_kv_cache_groups(
Returns:
The generated KVCacheGroups
"""
if vllm_config.scheduler_config.disable_hybrid_kv_cache_manager:
unify_hybrid_kv_cache_specs(kv_cache_spec)
@@ -2356,9 +2360,7 @@ def get_kv_cache_groups(
def generate_scheduler_kv_cache_config(
kv_cache_configs: list[KVCacheConfig],
) -> KVCacheConfig:
"""
Generate the KV cache configuration for the scheduler.
"""
"""Generate the KV cache configuration for the scheduler."""
assert all(
[cfg.num_blocks == kv_cache_configs[0].num_blocks for cfg in kv_cache_configs]
)
@@ -2382,9 +2384,7 @@ def generate_scheduler_kv_cache_config(
def get_kv_cache_capacity(
vllm_config: VllmConfig, kv_cache_config: KVCacheConfig
) -> tuple[int, float]:
"""
Get the group-aware KV cache token capacity and max concurrency.
"""
"""Get the group-aware KV cache token capacity and max concurrency."""
max_model_len = vllm_config.model_config.max_model_len
max_concurrency = get_max_concurrency_for_kv_cache_config(
vllm_config, kv_cache_config
@@ -2415,8 +2415,7 @@ def _max_memory_usage_bytes_from_groups(
vllm_config: VllmConfig,
kv_cache_groups: list[KVCacheGroupSpec],
) -> int:
"""
Calculate maximum memory usage in bytes from KV cache groups.
"""Calculate maximum memory usage in bytes from KV cache groups.
This correctly accounts for padding in hybrid models. For example, if a
model has 8 full attention layers and 9 sliding window layers, they will
@@ -2475,8 +2474,7 @@ def _estimate_max_model_len_from_groups(
kv_cache_groups: list[KVCacheGroupSpec],
available_memory: int,
) -> int:
"""
Binary search for the maximum model length that fits in available memory.
"""Binary search for the maximum model length that fits in available memory.
Returns 0 if even 1 token doesn't fit.
"""
original_max = vllm_config.model_config.max_model_len
@@ -2522,8 +2520,7 @@ def _auto_fit_max_model_len(
projected_groups_per_worker: list[list[KVCacheGroupSpec]],
available_memory: list[int],
) -> None:
"""
When max_model_len is set to -1, this function estimates the largest
"""When max_model_len is set to -1, this function estimates the largest
context length that can be supported with the available GPU memory.
It uses binary search to find the maximum length that fits across all
workers.
@@ -2533,6 +2530,7 @@ def _auto_fit_max_model_len(
projected_groups_per_worker: KV cache groups projected to each worker.
available_memory: Memory available for KV cache in bytes for each
worker.
"""
original_max = vllm_config.model_config.max_model_len
@@ -2585,8 +2583,7 @@ def _project_kv_cache_groups_to_worker(
global_kv_cache_groups: list[KVCacheGroupSpec],
worker_spec: dict[str, KVCacheSpec],
) -> list[KVCacheGroupSpec]:
"""
Projects global KV cache groups onto a single worker's assigned layers.
"""Projects global KV cache groups onto a single worker's assigned layers.
In pipeline parallelism, each worker only owns a subset of layers. This
function filters the global groups to include only layers present on the
@@ -2598,6 +2595,7 @@ def _project_kv_cache_groups_to_worker(
Returns:
The projected KV cache groups containing only this worker's layers.
"""
projected_groups: list[KVCacheGroupSpec] = []
for group in global_kv_cache_groups:
@@ -2628,8 +2626,7 @@ def get_kv_cache_configs(
kv_cache_specs: list[dict[str, KVCacheSpec]],
available_memory: list[int],
) -> list[KVCacheConfig]:
"""
Generates the KV cache configurations for a model.
"""Generates the KV cache configurations for a model.
Since we use a shared centralized controller for all workers, we need the
`kv_cache_config` to be consistent across all workers to make sure
the KV cache allocation can be applied to all workers. However, different
@@ -2656,8 +2653,8 @@ def get_kv_cache_configs(
Returns:
The generated KVCacheConfigs for each worker.
"""
"""
# Merge the KV cache specs of all workers. Different PP stages may have
# different layer names, and different TP ranks of the same PP stage should
# have the same KV cache spec.
@@ -2778,8 +2775,7 @@ def get_kv_cache_configs(
class BlockHashListWithBlockSize:
"""
Convert block-hash granularity from `hash_block_size` to `target_block_size`.
"""Convert block-hash granularity from `hash_block_size` to `target_block_size`.
Used when KV cache groups have different block sizes: `hash_block_size`
is the size used to compute the original `block_hashes`; `target_block_size`
is the group's actual block size.
@@ -2809,6 +2805,7 @@ class BlockHashListWithBlockSize:
block_hashes: Block hashes to convert, computed at `hash_block_size`.
hash_block_size: Block size at which `block_hashes` were computed.
target_block_size: Desired block size; must be a multiple of `hash_block_size`.
"""
def __init__(
+12 -2
View File
@@ -79,6 +79,7 @@ class SchedulerInterface(ABC):
Returns:
A SchedulerOutput object containing information about the scheduled
requests.
"""
raise NotImplementedError
@@ -105,6 +106,7 @@ class SchedulerInterface(ABC):
Returns:
A dict of client index to EngineCoreOutputs object containing the
outputs for each request originating from that client.
"""
raise NotImplementedError
@@ -115,6 +117,7 @@ class SchedulerInterface(ABC):
Args:
draft_token_ids: The input draft token ids for each request.
"""
raise NotImplementedError
@@ -129,6 +132,7 @@ class SchedulerInterface(ABC):
draft_token_ids: The input draft token ids for each request.
scheduler_output: Update the given scheduler_output
with the corresponding draft token ids.
"""
raise NotImplementedError
@@ -138,6 +142,7 @@ class SchedulerInterface(ABC):
Args:
request: The new request being added.
"""
raise NotImplementedError
@@ -162,6 +167,7 @@ class SchedulerInterface(ABC):
Returns:
List of requests that were aborted. Will not include any that were
already finished.
"""
raise NotImplementedError
@@ -172,7 +178,8 @@ class SchedulerInterface(ABC):
def has_unfinished_requests(self) -> bool:
"""Returns True if there are unfinished requests in the scheduler's
internal queue."""
internal queue.
"""
return self.get_num_unfinished_requests() > 0
@abstractmethod
@@ -192,7 +199,8 @@ class SchedulerInterface(ABC):
def has_requests(self) -> bool:
"""Returns True if there are unfinished requests, or finished requests
not yet returned in SchedulerOutputs."""
not yet returned in SchedulerOutputs.
"""
return self.has_unfinished_requests() or self.has_finished_requests()
@property
@@ -218,6 +226,8 @@ class SchedulerInterface(ABC):
preempted and moved to the waiting queue. Otherwise, this method
will only reset the KV prefix cache when there is no running request
taking KV cache.
reset_connector: If True, also reset any KV connector state.
"""
raise NotImplementedError
+7 -5
View File
@@ -43,7 +43,8 @@ class RequestQueue(ABC):
@abstractmethod
def prepend_requests(self, requests: "RequestQueue") -> None:
"""Prepend all requests from another queue to the front of this
queue."""
queue.
"""
pass
@abstractmethod
@@ -129,8 +130,7 @@ class FCFSRequestQueue(deque[Request], RequestQueue):
class PriorityRequestQueue(RequestQueue):
"""
A priority queue that supports heap operations.
"""A priority queue that supports heap operations.
Respects the ordering defined in the Request class, where
requests with a smaller value of `priority` are processed first.
@@ -161,14 +161,16 @@ class PriorityRequestQueue(RequestQueue):
"""Add a request to the queue according to priority policy.
Note: In a priority queue, there is no concept of prepending to the
front. Requests are ordered by (priority, arrival_time)."""
front. Requests are ordered by (priority, arrival_time).
"""
self.add_request(request)
def prepend_requests(self, requests: RequestQueue) -> None:
"""Add all requests from another queue according to priority policy.
Note: In a priority queue, there is no concept of prepending to the
front. Requests are ordered by (priority, arrival_time)."""
front. Requests are ordered by (priority, arrival_time).
"""
for request in requests:
self.add_request(request)
+11 -20
View File
@@ -1558,12 +1558,10 @@ class Scheduler(SchedulerInterface):
def _update_request_as_session(
self, session: Request, update: StreamingUpdate
) -> None:
"""
Updates the waiting session with the next streaming update.
"""Updates the waiting session with the next streaming update.
Discards the last sampled output token from the prior input chunk.
"""
# Current streaming input behaviour: Keep only computed output tokens
# (discard final sampled output token).
num_computed_tokens = session.num_computed_tokens
@@ -1665,8 +1663,7 @@ class Scheduler(SchedulerInterface):
encoder_compute_budget: int,
shift_computed_tokens: int = 0,
) -> tuple[list[int], int, int, list[int]]:
"""
Determine which encoder inputs need to be scheduled in the current step,
"""Determine which encoder inputs need to be scheduled in the current step,
and update `num_new_tokens` and encoder token budget accordingly.
An encoder input will be scheduled if:
@@ -2507,6 +2504,7 @@ class Scheduler(SchedulerInterface):
Returns:
List of requests that were aborted. Will not include any that were
already finished.
"""
assert RequestStatus.is_finished(finished_status)
if isinstance(request_ids, str):
@@ -2841,8 +2839,7 @@ class Scheduler(SchedulerInterface):
def _connector_finished(
self, request: Request
) -> tuple[bool, dict[str, Any] | None]:
"""
Invoke the KV connector request_finished() method if applicable.
"""Invoke the KV connector request_finished() method if applicable.
Returns optional kv transfer parameters to be included with the
request outputs.
@@ -2910,14 +2907,12 @@ class Scheduler(SchedulerInterface):
def _inflight_prefill_reserved_blocks(self) -> int:
"""Num blocks in-flight prefills still need to finish (their reservation)."""
return sum(
self._request_remaining_blocks(req) for req in self._inflight_prefills
)
def _update_waiting_for_remote_kv(self, request: Request) -> None:
"""
KV Connector: update request state after async recv is finished.
"""KV Connector: update request state after async recv is finished.
When the kv transfer is ready, we cache the blocks
and the request state will be moved back to WAITING from
@@ -2959,9 +2954,7 @@ class Scheduler(SchedulerInterface):
self.finished_recving_kv_req_ids.remove(request.request_id)
def _try_promote_blocked_waiting_request(self, request: Request) -> bool:
"""
Try to promote a blocked waiting request back to schedulable states.
"""
"""Try to promote a blocked waiting request back to schedulable states."""
if request.status == RequestStatus.WAITING_FOR_REMOTE_KVS:
# finished_recving_kv_req_ids is populated during
# update_from_output(), based on worker-side connector signals
@@ -2995,8 +2988,7 @@ class Scheduler(SchedulerInterface):
)
def _update_from_kv_xfer_finished(self, kv_connector_output: KVConnectorOutput):
"""
KV Connector: update the scheduler state based on the output.
"""KV Connector: update the scheduler state based on the output.
The Worker side connectors add finished_recving and
finished_sending reqs to the output.
@@ -3004,7 +2996,6 @@ class Scheduler(SchedulerInterface):
# if finished_recving: add to state so we can
schedule the request during the next step.
"""
if self.connector is not None:
self.connector.update_connector_output(kv_connector_output)
@@ -3030,8 +3021,7 @@ class Scheduler(SchedulerInterface):
num_scheduled_tokens: dict[str, int],
evict_blocks: bool = True,
) -> tuple[set[str], int, set[int]]:
"""
Identify and update requests affected by invalid KV cache blocks.
"""Identify and update requests affected by invalid KV cache blocks.
This method scans the given requests, detects those with invalid blocks
and adjusts their `num_computed_tokens` to the longest valid prefix.
@@ -3053,6 +3043,7 @@ class Scheduler(SchedulerInterface):
be recomputed across all affected requests.
- blocks_to_evict (set[int]): Block IDs to evict from cache,
including invalid blocks and downstream dependent blocks.
"""
affected_req_ids: set[str] = set()
total_affected_tokens = 0
@@ -3145,11 +3136,11 @@ class Scheduler(SchedulerInterface):
def _handle_invalid_blocks(
self, invalid_block_ids: set[int], num_scheduled_tokens: dict[str, int]
) -> set[str]:
"""
Handle requests affected by invalid KV cache blocks.
"""Handle requests affected by invalid KV cache blocks.
Returns:
Set of affected request IDs to skip in update_from_output main loop.
"""
if len(self.kv_cache_config.kv_cache_groups) > 1:
# Block IDs are only unique within a group, so a flat set of
+4
View File
@@ -30,11 +30,14 @@ def check_sequence_repetition(
params: RepetitionDetectionParams,
) -> bool:
"""Check if a sequence of token IDs has a repetition pattern.
Args:
token_ids: List of token IDs
params: Repetition detection parameters.
Returns:
True if a repetition pattern is found, False otherwise.
"""
max_pattern_size = params.max_pattern_size
min_pattern_size = params.min_pattern_size
@@ -77,6 +80,7 @@ def remove_all(lst: list, items_to_remove: set) -> list:
Note:
For single item removal, this modifies the original list in-place
and returns it. For multiple items, it creates and returns a new list.
"""
if not items_to_remove:
return lst
+45 -47
View File
@@ -49,8 +49,7 @@ logger = init_logger(__name__)
class SingleTypeKVCacheManager(ABC):
"""
An abstract base class for a manager that handle the kv cache management
"""An abstract base class for a manager that handle the kv cache management
logic of one specific type of attention layer.
"""
@@ -81,14 +80,17 @@ class SingleTypeKVCacheManager(ABC):
needs_kv_cache_zeroing: bool = False,
max_admission_blocks_per_request: int | None = None,
) -> None:
"""
Initializes the SingleTypeKVCacheManager.
"""Initializes the SingleTypeKVCacheManager.
Args:
kv_cache_spec: The kv_cache_spec for this manager.
block_pool: The block pool.
enable_caching: Whether prefix caching is enabled.
kv_cache_group_id: The id of the kv cache group of this manager.
scheduler_block_size: The scheduling granularity (LCM of all group
block sizes); a multiple of this manager's ``block_size``.
dcp_world_size: Decode context parallel world size.
pcp_world_size: Prefill context parallel world size.
needs_kv_cache_zeroing: Whether worker-side KV cache zeroing needs
newly allocated block IDs from this manager.
max_admission_blocks_per_request: Recycling-aware per-request
@@ -97,6 +99,7 @@ class SingleTypeKVCacheManager(ABC):
chunked-local); `None` (the default) means no cap, which is
correct for full-attention-style specs that hold every
block until the request finishes.
"""
self.scheduler_block_size = scheduler_block_size
# Hybrid fine-grained lookup may lower this after all participating
@@ -184,8 +187,7 @@ class SingleTypeKVCacheManager(ABC):
num_tokens_main_model: int,
apply_admission_cap: bool = False,
) -> int:
"""
Get the number of blocks needed to be allocated for the request.
"""Get the number of blocks needed to be allocated for the request.
Args:
request_id: The request ID.
@@ -206,8 +208,8 @@ class SingleTypeKVCacheManager(ABC):
Returns:
The number of blocks to allocate.
"""
"""
num_required_blocks = cdiv(num_tokens, self.block_size)
if apply_admission_cap and self._max_admission_blocks_per_request is not None:
# Recycling-aware specs (SWA, chunked-local) cap the per-request
@@ -269,8 +271,7 @@ class SingleTypeKVCacheManager(ABC):
num_local_computed_tokens: int,
num_external_computed_tokens: int,
) -> None:
"""
Add the locally cached (prefix-hit) blocks to the request:
"""Add the locally cached (prefix-hit) blocks to the request:
1. Touch the computed blocks (paired with adding them to `req_blocks`)
so their ref_cnt exactly tracks the referencing requests.
1.5. (Optional) For sliding window, skipped blocks are padded with nulls.
@@ -282,6 +283,7 @@ class SingleTypeKVCacheManager(ABC):
prefix cache.
num_local_computed_tokens: The number of local computed tokens.
num_external_computed_tokens: The number of external computed tokens.
"""
# The coordinator only calls this for first-time allocations (running
# requests are short-circuited there), so the request has no blocks yet.
@@ -327,8 +329,7 @@ class SingleTypeKVCacheManager(ABC):
num_local_computed_tokens: int,
num_external_computed_tokens: int,
) -> None:
"""
Allocate new blocks for external (KV-connector) computed tokens.
"""Allocate new blocks for external (KV-connector) computed tokens.
Must run only after every group's local blocks have been touched via
`add_local_computed_blocks`, so this group's `get_new_blocks` cannot
@@ -338,6 +339,7 @@ class SingleTypeKVCacheManager(ABC):
request_id: The request ID.
num_local_computed_tokens: The number of local computed tokens.
num_external_computed_tokens: The number of external computed tokens.
"""
num_total_computed_tokens = (
num_local_computed_tokens + num_external_computed_tokens
@@ -364,8 +366,7 @@ class SingleTypeKVCacheManager(ABC):
def allocate_new_blocks(
self, request_id: str, num_tokens: int, num_tokens_main_model: int
) -> list[KVCacheBlock]:
"""
Allocate new blocks for the request to give it at least `num_tokens`
"""Allocate new blocks for the request to give it at least `num_tokens`
token slots.
Args:
@@ -375,8 +376,10 @@ class SingleTypeKVCacheManager(ABC):
num_tokens_main_model: The number of tokens for the main model (aka target
model in spec decode). w/o spec decode, it is num_tokens;
with spec decode, it is num_tokens - num_lookahead_tokens.
Returns:
The new allocated blocks.
"""
cow_blocks: list[KVCacheBlock] = []
if request_id in self._partial_hit_reqs:
@@ -475,8 +478,7 @@ class SingleTypeKVCacheManager(ABC):
*,
replay_boundaries: Sequence[int],
) -> None:
"""
Cache the blocks for the request.
"""Cache the blocks for the request.
Args:
request: The request.
@@ -488,6 +490,7 @@ class SingleTypeKVCacheManager(ABC):
a tail once per that-sized segment. Only SWA acts on it.
replay_boundaries: Positions a later request replaying this prompt
can resume at, from ``get_replay_boundaries``.
"""
num_cached_blocks = self.num_cached_block.get(request.request_id, 0)
num_full_blocks = num_tokens // self.block_size
@@ -552,8 +555,7 @@ class SingleTypeKVCacheManager(ABC):
return None
def pop_blocks_for_free(self, request_id: str) -> list[KVCacheBlock]:
"""
Pop the request's bookkeeping and return its blocks without yet
"""Pop the request's bookkeeping and return its blocks without yet
returning them to the block pool. The caller is responsible for
eventually passing the returned blocks to `block_pool.free_blocks`,
freeing them in reverse order (so that tail blocks are evicted first).
@@ -563,6 +565,7 @@ class SingleTypeKVCacheManager(ABC):
Returns:
The request's blocks in allocation order.
"""
# Default to [] in case a request is freed (aborted) before alloc.
req_blocks = self.req_to_blocks.pop(request_id, [])
@@ -571,19 +574,18 @@ class SingleTypeKVCacheManager(ABC):
return req_blocks
def free(self, request_id: str) -> None:
"""
Free the blocks for the request.
"""Free the blocks for the request.
Args:
request_id: The request ID.
"""
# Free blocks in reverse order so that the tail blocks are freed first.
self.block_pool.free_blocks(reversed(self.pop_blocks_for_free(request_id)))
@abstractmethod
def get_num_common_prefix_blocks(self, running_request_id: str) -> int:
"""
Get the number of common prefix blocks for all requests with allocated
"""Get the number of common prefix blocks for all requests with allocated
KV cache.
Args:
@@ -592,8 +594,8 @@ class SingleTypeKVCacheManager(ABC):
Returns:
The number of common prefix blocks for all requests with allocated
KV cache.
"""
"""
raise NotImplementedError
@classmethod
@@ -610,8 +612,7 @@ class SingleTypeKVCacheManager(ABC):
dcp_world_size: int = 1,
pcp_world_size: int = 1,
) -> tuple[tuple[list[KVCacheBlock], ...], int]:
"""
Get the longest cache hit prefix of the blocks that is not longer than
"""Get the longest cache hit prefix of the blocks that is not longer than
`max_length`. The prefix should be a common prefix hit for all the
kv cache groups in `kv_cache_group_ids`. If no cache hit is found,
return an empty list.
@@ -647,8 +648,8 @@ class SingleTypeKVCacheManager(ABC):
For example, sliding window manager should return a list like
([NULL, NULL, KVCacheBlock(7), KVCacheBlock(8)]) for block size 4
and sliding window 8 and len(kv_cache_group_ids) = 1.
"""
"""
raise NotImplementedError
def _remove_blocks_in_range(
@@ -684,8 +685,8 @@ class SingleTypeKVCacheManager(ABC):
processed_computed_tokens: int,
num_prompt_tokens: int | None = None,
) -> None:
"""
Remove and free the blocks that are no longer needed for attention computation.
"""Remove and free blocks no longer needed for attention computation.
The removed blocks should be replaced by null_block.
This function depends on `get_num_skipped_tokens`, which need to be implemented
@@ -698,6 +699,7 @@ class SingleTypeKVCacheManager(ABC):
num_prompt_tokens: Optional prompt length for attention types (e.g.
R-SWA) that evict a middle gap rather than a head prefix. Ignored
by the default implementation.
"""
del num_prompt_tokens
# Remove the blocks that will be skipped during attention computation.
@@ -718,14 +720,14 @@ class SingleTypeKVCacheManager(ABC):
self._remove_blocks_in_range(request_id, 0, num_skipped_blocks)
def get_num_skipped_tokens(self, num_computed_tokens: int) -> int:
"""
Get the number of tokens that will be skipped for attention computation.
"""Get the number of tokens that will be skipped for attention computation.
Args:
num_computed_tokens: The number of tokens that have been computed.
Returns:
The number of tokens that will be skipped for attention computation.
"""
# The default behavior is to not skip any tokens.
return 0
@@ -1144,8 +1146,7 @@ class SlidingWindowManager(SingleTypeKVCacheManager):
return mask
def get_num_skipped_tokens(self, num_computed_tokens: int) -> int:
"""
Get the number of tokens that will be skipped for attention computation.
"""Get the number of tokens that will be skipped for attention computation.
For sliding window, this corresponds to the tokens that are prior to
the current sliding window.
@@ -1174,6 +1175,7 @@ class SlidingWindowManager(SingleTypeKVCacheManager):
Returns:
The number of tokens that will be skipped for attention computation.
"""
return max(
0,
@@ -1181,8 +1183,7 @@ class SlidingWindowManager(SingleTypeKVCacheManager):
)
def get_num_common_prefix_blocks(self, running_request_id: str) -> int:
"""
NOTE(Chen): The prefix blocks are null blocks for sliding window layers.
"""NOTE(Chen): The prefix blocks are null blocks for sliding window layers.
So it's not correct to count ref_cnt like FullAttentionManager. Return
0 here for correctness. Need to support cascade attention + sliding
window in the future.
@@ -1301,8 +1302,7 @@ class ChunkedLocalAttentionManager(SingleTypeKVCacheManager):
dcp_world_size: int = 1,
pcp_world_size: int = 1,
) -> tuple[tuple[list[KVCacheBlock], ...], int]:
"""
For chunked local attention, we need to find the longest cache hit
"""For chunked local attention, we need to find the longest cache hit
prefix of the blocks that is not longer than `max_length`. The prefix
should be a common prefix hit for all the kv cache groups in
`kv_cache_group_ids`. If no cache hit is found, return an empty list.
@@ -1335,6 +1335,7 @@ class ChunkedLocalAttentionManager(SingleTypeKVCacheManager):
Returns:
A list of cached blocks
"""
assert isinstance(kv_cache_spec, ChunkedLocalAttentionSpec), (
"ChunkedLocalAttentionManager can only be used for "
@@ -1389,8 +1390,7 @@ class ChunkedLocalAttentionManager(SingleTypeKVCacheManager):
return computed_blocks, hit_length
def get_num_skipped_tokens(self, num_computed_tokens: int) -> int:
"""
Get the number of tokens that will be skipped for attention computation.
"""Get the number of tokens that will be skipped for attention computation.
For chunked local attention, this corresponds to the tokens that are on
the left side of the current chunk.
@@ -1428,6 +1428,7 @@ class ChunkedLocalAttentionManager(SingleTypeKVCacheManager):
Returns:
The number of tokens that will be skipped for attention computation.
"""
num_skipped_tokens = (
num_computed_tokens // self.attention_chunk_size
@@ -1435,9 +1436,7 @@ class ChunkedLocalAttentionManager(SingleTypeKVCacheManager):
return num_skipped_tokens
def get_num_common_prefix_blocks(self, running_request_id: str) -> int:
"""
cascade attention is not supported by chunked local attention.
"""
"""Cascade attention is not supported by chunked local attention."""
return 0
@@ -1674,9 +1673,7 @@ class MambaManager(SingleTypeKVCacheManager):
blocks[last_state_block_idx] = self._null_block
def get_num_common_prefix_blocks(self, running_request_id: str) -> int:
"""
cascade attention is not supported by mamba
"""
"""Cascade attention is not supported by mamba"""
return 0
def _needs_internal_checkpoint(
@@ -1979,8 +1976,7 @@ class MambaManager(SingleTypeKVCacheManager):
return super().pop_blocks_for_free(request_id)
def get_num_skipped_tokens(self, num_computed_tokens: int) -> int:
"""
Get the number of tokens whose mamba state are not needed anymore. Mamba only
"""Get the number of tokens whose mamba state are not needed anymore. Mamba only
need to keep the state of the last computed token, so we return
num_computed_tokens - 1.
"""
@@ -2600,8 +2596,7 @@ def get_manager_for_kv_cache_spec(
role: str | None = None,
**kwargs,
) -> SingleTypeKVCacheManager:
"""
Get the appropriate manager for a given KVCacheSpec.
"""Get the appropriate manager for a given KVCacheSpec.
Uses the KVCacheSpecRegistry to look up the manager class, supporting
both built-in and custom specs registered via @register_kv_cache_spec
@@ -2612,8 +2607,11 @@ def get_manager_for_kv_cache_spec(
max_in_flight_tokens: The max tokens scheduled but not yet settled
(one batch per concurrent step); see `VllmConfig.max_in_flight_tokens`
max_model_len: The maximum context length the model could serve
role: Optional KV cache group role used to select a role-specific
manager
Returns:
An instance of the appropriate SingleTypeKVCacheManager subclass
"""
manager_class = KVCacheSpecRegistry.get_manager_class(kv_cache_spec, role)
assert manager_class is not None, (