Add pydocstyle to the ruff rules (#52136)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Signed-off-by: Harry Mellor <hej.mellor@gmail.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
804e5377aa
commit
8c1557a79c
+21
-18
@@ -31,8 +31,7 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
class BlockHashToBlockMap:
|
||||
"""
|
||||
Cache of blocks that are used for prefix caching. It caches blocks
|
||||
"""Cache of blocks that are used for prefix caching. It caches blocks
|
||||
from hash directly to a block or multiple blocks
|
||||
(i.e. {block_hash: KVCacheBlocks})
|
||||
- Mostly block_hash maps to a single KVCacheBlock, and KVCacheBlocks
|
||||
@@ -59,9 +58,7 @@ class BlockHashToBlockMap:
|
||||
] = {}
|
||||
|
||||
def get_one_block(self, key: BlockHashWithGroupId) -> KVCacheBlock | None:
|
||||
"""
|
||||
Gets any block with the given block hash key.
|
||||
"""
|
||||
"""Gets any block with the given block hash key."""
|
||||
blocks = self._cache.get(key)
|
||||
if blocks is not None:
|
||||
if isinstance(blocks, KVCacheBlock):
|
||||
@@ -72,9 +69,7 @@ class BlockHashToBlockMap:
|
||||
return None
|
||||
|
||||
def contain(self, key: BlockHashWithGroupId, block_id: int) -> bool:
|
||||
"""
|
||||
Checks whether the key maps to the given block ID.
|
||||
"""
|
||||
"""Checks whether the key maps to the given block ID."""
|
||||
blocks = self._cache.get(key)
|
||||
if blocks is None:
|
||||
return False
|
||||
@@ -86,9 +81,7 @@ class BlockHashToBlockMap:
|
||||
return False
|
||||
|
||||
def insert(self, key: BlockHashWithGroupId, block: KVCacheBlock) -> None:
|
||||
"""
|
||||
Inserts the KVCacheBlock to the cache
|
||||
"""
|
||||
"""Inserts the KVCacheBlock to the cache"""
|
||||
blocks = self._cache.get(key)
|
||||
if blocks is None:
|
||||
# When key is not found, attach a single block to the key
|
||||
@@ -104,9 +97,7 @@ class BlockHashToBlockMap:
|
||||
self._unexpected_blocks_type(blocks)
|
||||
|
||||
def pop(self, key: BlockHashWithGroupId, block_id: int) -> KVCacheBlock | None:
|
||||
"""
|
||||
Checks if block_hash exists and pop block_id from the cache
|
||||
"""
|
||||
"""Checks if block_hash exists and pop block_id from the cache"""
|
||||
blocks = self._cache.pop(key, None)
|
||||
if blocks is None:
|
||||
# block_hash not found in the cache
|
||||
@@ -158,6 +149,7 @@ class BlockPool:
|
||||
enable_kv_cache_events: Whether to enable kv cache events.
|
||||
metrics_collector: Optional metrics collector for tracking block residency.
|
||||
medium: Storage medium reported in KV cache events.
|
||||
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -214,6 +206,7 @@ class BlockPool:
|
||||
|
||||
Returns:
|
||||
The cached blocks if exists, or None.
|
||||
|
||||
"""
|
||||
cached_blocks = []
|
||||
for group_id in kv_cache_group_ids:
|
||||
@@ -261,6 +254,7 @@ class BlockPool:
|
||||
consults a subset of blocks (e.g. SWA tail-window), so blocks
|
||||
that can never serve a hit stay out of the prefix-cache hash
|
||||
map.
|
||||
|
||||
"""
|
||||
if num_cached_blocks >= num_full_blocks:
|
||||
return
|
||||
@@ -395,6 +389,7 @@ class BlockPool:
|
||||
num_cached_blocks: Number of blocks that were cache hits.
|
||||
block_size: Number of tokens per block.
|
||||
kv_cache_group_id: The KV cache group ID.
|
||||
|
||||
"""
|
||||
if not self.enable_kv_cache_events or num_cached_blocks == 0:
|
||||
return
|
||||
@@ -491,6 +486,7 @@ class BlockPool:
|
||||
Returns:
|
||||
The hash key with group ID if a partial entry can be registered;
|
||||
otherwise ``None`` for null blocks.
|
||||
|
||||
"""
|
||||
if block.is_null:
|
||||
return None
|
||||
@@ -671,6 +667,7 @@ class BlockPool:
|
||||
|
||||
Returns:
|
||||
A list of new block.
|
||||
|
||||
"""
|
||||
if num_blocks > self.get_num_free_blocks():
|
||||
raise ValueError(f"Cannot get {num_blocks} free blocks from the pool")
|
||||
@@ -724,8 +721,7 @@ class BlockPool:
|
||||
self.free_block_queue.append_n(released)
|
||||
|
||||
def _maybe_evict_cached_block(self, block: KVCacheBlock) -> bool:
|
||||
"""
|
||||
If a block is cached in `cached_block_hash_to_block`, we reset its hash
|
||||
"""If a block is cached in `cached_block_hash_to_block`, we reset its hash
|
||||
metadata and evict it from the cache.
|
||||
|
||||
Args:
|
||||
@@ -733,6 +729,7 @@ class BlockPool:
|
||||
|
||||
Returns:
|
||||
True if the block is evicted, False otherwise.
|
||||
|
||||
"""
|
||||
# Clean up metrics tracking first to prevent leaks
|
||||
if self.metrics_collector:
|
||||
@@ -753,6 +750,7 @@ class BlockPool:
|
||||
|
||||
Args:
|
||||
blocks: A list of blocks to touch.
|
||||
|
||||
"""
|
||||
for block in blocks:
|
||||
# ref_cnt=0 means this block is in the free list (i.e. eviction
|
||||
@@ -774,6 +772,7 @@ class BlockPool:
|
||||
Args:
|
||||
ordered_blocks: A list of blocks to free ordered by their eviction
|
||||
priority.
|
||||
|
||||
"""
|
||||
# Identify blocks with hash (LRU cache) and without it (never match APC)
|
||||
blocks_to_evict_last = []
|
||||
@@ -800,7 +799,7 @@ class BlockPool:
|
||||
pool.free_blocks(blocks)
|
||||
|
||||
def evict_blocks(self, block_ids: set[int]) -> None:
|
||||
"""evict blocks from the prefix cache by their block IDs.
|
||||
"""Evict blocks from the prefix cache by their block IDs.
|
||||
|
||||
only evicts blocks that are currently cached (have a hash). blocks
|
||||
with ref_cnt > 0 are not freed from the block pool, only evicted
|
||||
@@ -808,6 +807,7 @@ class BlockPool:
|
||||
|
||||
Args:
|
||||
block_ids: Set of block IDs to evict from cache.
|
||||
|
||||
"""
|
||||
for block_id in block_ids:
|
||||
assert block_id < len(self.blocks), (
|
||||
@@ -826,6 +826,7 @@ class BlockPool:
|
||||
Returns:
|
||||
bool: True if the prefix cache is successfully reset,
|
||||
False otherwise.
|
||||
|
||||
"""
|
||||
num_used_blocks = self.num_gpu_blocks - self.get_num_free_blocks()
|
||||
if num_used_blocks != 1: # The null block is always marked as used
|
||||
@@ -863,6 +864,7 @@ class BlockPool:
|
||||
|
||||
Returns:
|
||||
The number of free blocks.
|
||||
|
||||
"""
|
||||
return self.free_block_queue.num_free_blocks
|
||||
|
||||
@@ -871,8 +873,8 @@ class BlockPool:
|
||||
|
||||
Returns:
|
||||
The KV cache usage (between 0.0 and 1.0).
|
||||
"""
|
||||
|
||||
"""
|
||||
# Subtract 1 to account for null block.
|
||||
total_gpu_blocks = self.num_gpu_blocks - 1
|
||||
if not total_gpu_blocks:
|
||||
@@ -884,6 +886,7 @@ class BlockPool:
|
||||
|
||||
Returns:
|
||||
A list of KV cache events.
|
||||
|
||||
"""
|
||||
if not self.enable_kv_cache_events:
|
||||
return []
|
||||
|
||||
@@ -64,6 +64,7 @@ class EncoderCacheManager:
|
||||
make space when needed.
|
||||
freed: List of mm_hash strings that were actually evicted since the
|
||||
last call to get_freed_mm_hashes(). This list is cleared on return.
|
||||
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
@@ -113,6 +114,7 @@ class EncoderCacheManager:
|
||||
|
||||
Returns:
|
||||
True if the encoder output for this input is already cached
|
||||
|
||||
"""
|
||||
mm_hash = request.mm_features[input_id].identifier
|
||||
# Not cached at all
|
||||
@@ -162,6 +164,7 @@ class EncoderCacheManager:
|
||||
|
||||
Note: This method does not allocate physical memory for the encoder
|
||||
output but only the state of EncoderCacheManager.
|
||||
|
||||
"""
|
||||
num_embeds = request.get_num_encoder_embeds(input_id)
|
||||
|
||||
@@ -198,8 +201,8 @@ class EncoderCacheManager:
|
||||
|
||||
Note:
|
||||
This method assumes can_allocate() returned True for the same input.
|
||||
"""
|
||||
|
||||
"""
|
||||
mm_hash = request.mm_features[input_id].identifier
|
||||
request_id = request.request_id
|
||||
if mm_hash not in self.cached:
|
||||
@@ -222,7 +225,7 @@ class EncoderCacheManager:
|
||||
return self.request_cached_ids.get(request.request_id, set())
|
||||
|
||||
def free_encoder_input(self, request: Request, input_id: int) -> None:
|
||||
"""Free the request's reference to the encoder input (`mm_data`)
|
||||
"""Free the request's reference to the encoder input (`mm_data`).
|
||||
|
||||
When the reference set for the corresponding `mm_hash` becomes empty,
|
||||
the entry is appended to `freeable` and `num_freeable_slots` is
|
||||
@@ -279,6 +282,7 @@ class EncoderCacheManager:
|
||||
call to be used by the scheduler to notify workers about which
|
||||
encoder outputs can be removed from their caches. The internal
|
||||
list is cleared after this call.
|
||||
|
||||
"""
|
||||
# An entry evicted early in the scheduling pass can be allocated again
|
||||
# later in the same pass. Keep its worker-side tensor in that case.
|
||||
@@ -307,8 +311,8 @@ def compute_mm_encoder_budget(
|
||||
from the input sequence.
|
||||
- Space budget for encoder cache size, measured in number of tokens
|
||||
from the input sequence.
|
||||
"""
|
||||
|
||||
"""
|
||||
if not mm_max_toks_per_item:
|
||||
logger.warning(
|
||||
"All non-text modalities supported by the model have been "
|
||||
|
||||
@@ -64,9 +64,7 @@ def _validate_prefix_cache_retention_interval(
|
||||
|
||||
|
||||
class KVCacheCoordinator(ABC):
|
||||
"""
|
||||
Coordinate the KV cache of different KV cache groups.
|
||||
"""
|
||||
"""Coordinate the KV cache of different KV cache groups."""
|
||||
|
||||
enable_partial_hash_hits = False
|
||||
|
||||
@@ -179,8 +177,7 @@ class KVCacheCoordinator(ABC):
|
||||
num_tokens_main_model: int,
|
||||
apply_admission_cap: bool = False,
|
||||
) -> int:
|
||||
"""
|
||||
Get the number of device blocks needed to be allocated for the request.
|
||||
"""Get the number of device blocks needed to be allocated for the request.
|
||||
|
||||
Args:
|
||||
request_id: The request ID.
|
||||
@@ -203,6 +200,7 @@ class KVCacheCoordinator(ABC):
|
||||
|
||||
Returns:
|
||||
The number of blocks to allocate.
|
||||
|
||||
"""
|
||||
num_blocks_to_allocate = 0
|
||||
for i, manager in enumerate(self.single_type_managers):
|
||||
@@ -237,8 +235,7 @@ class KVCacheCoordinator(ABC):
|
||||
num_local_computed_tokens: int,
|
||||
num_external_computed_tokens: int,
|
||||
) -> None:
|
||||
"""
|
||||
Add the new computed blocks to the request. Optionally allocate new
|
||||
"""Add the new computed blocks to the request. Optionally allocate new
|
||||
blocks for external computed tokens (if any).
|
||||
|
||||
Args:
|
||||
@@ -247,6 +244,7 @@ class KVCacheCoordinator(ABC):
|
||||
prefix cache.
|
||||
num_local_computed_tokens: The number of local computed tokens.
|
||||
num_external_computed_tokens: The number of external computed tokens.
|
||||
|
||||
"""
|
||||
# A running request is already tracked in num_cached_block and won't
|
||||
# have new prefix-cache hits, so this is a no-op for it.
|
||||
@@ -283,8 +281,7 @@ class KVCacheCoordinator(ABC):
|
||||
num_tokens_main_model: int,
|
||||
num_encoder_tokens: int = 0,
|
||||
) -> tuple[list[KVCacheBlock], ...]:
|
||||
"""
|
||||
Allocate new blocks for the request to give it at least `num_tokens`
|
||||
"""Allocate new blocks for the request to give it at least `num_tokens`
|
||||
token slots.
|
||||
|
||||
Args:
|
||||
@@ -299,6 +296,7 @@ class KVCacheCoordinator(ABC):
|
||||
|
||||
Returns:
|
||||
The new allocated blocks.
|
||||
|
||||
"""
|
||||
return tuple(
|
||||
manager.allocate_new_blocks(
|
||||
@@ -333,14 +331,14 @@ class KVCacheCoordinator(ABC):
|
||||
return tuple(sorted({max(resend - block, 0), max(extension - block, 0)}))
|
||||
|
||||
def cache_blocks(self, request: Request, num_computed_tokens: int) -> None:
|
||||
"""
|
||||
Cache the blocks for the request.
|
||||
"""Cache the blocks for the request.
|
||||
|
||||
Args:
|
||||
request: The request.
|
||||
num_computed_tokens: The total number of tokens
|
||||
that need to be cached
|
||||
(including tokens that are already cached).
|
||||
|
||||
"""
|
||||
boundaries = self.get_replay_boundaries(request)
|
||||
for manager in self.single_type_managers:
|
||||
@@ -380,18 +378,17 @@ class KVCacheCoordinator(ABC):
|
||||
return all([pool.reset_prefix_cache() for pool in pools])
|
||||
|
||||
def free(self, request_id: str) -> None:
|
||||
"""
|
||||
Free the blocks for the request.
|
||||
"""Free the blocks for the request.
|
||||
|
||||
Args:
|
||||
request_id: The request ID.
|
||||
|
||||
"""
|
||||
for manager in self.single_type_managers:
|
||||
manager.free(request_id)
|
||||
|
||||
def pop_blocks_for_free(self, request_id: str) -> list[KVCacheBlock]:
|
||||
"""
|
||||
Pop the request's bookkeeping from all single-type managers and
|
||||
"""Pop the request's bookkeeping from all single-type managers and
|
||||
return its blocks without returning them to the block pool. The
|
||||
caller must eventually pass the returned blocks to
|
||||
`block_pool.free_blocks`, freeing them in reverse order (so that
|
||||
@@ -402,6 +399,7 @@ class KVCacheCoordinator(ABC):
|
||||
|
||||
Returns:
|
||||
The request's blocks in allocation order.
|
||||
|
||||
"""
|
||||
blocks: list[KVCacheBlock] = []
|
||||
for manager in self.single_type_managers:
|
||||
@@ -409,8 +407,7 @@ class KVCacheCoordinator(ABC):
|
||||
return blocks
|
||||
|
||||
def get_num_common_prefix_blocks(self, running_request_id: str) -> list[int]:
|
||||
"""
|
||||
Get the number of common prefix blocks for all requests with allocated
|
||||
"""Get the number of common prefix blocks for all requests with allocated
|
||||
KV cache for each kv cache group.
|
||||
|
||||
Args:
|
||||
@@ -419,6 +416,7 @@ class KVCacheCoordinator(ABC):
|
||||
|
||||
Returns:
|
||||
list[int]: The number of common prefix blocks for each kv cache group.
|
||||
|
||||
"""
|
||||
return [
|
||||
manager.get_num_common_prefix_blocks(running_request_id)
|
||||
@@ -431,8 +429,7 @@ class KVCacheCoordinator(ABC):
|
||||
processed_computed_tokens: int,
|
||||
num_prompt_tokens: int | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
Remove the blocks that are no longer needed from `blocks` and replace
|
||||
"""Remove the blocks that are no longer needed from `blocks` and replace
|
||||
the removed blocks with null_block.
|
||||
|
||||
Args:
|
||||
@@ -442,6 +439,7 @@ class KVCacheCoordinator(ABC):
|
||||
num_prompt_tokens: Optional prompt length. R-SWA managers use this to
|
||||
free gap blocks between the prefill tail and decode window; other
|
||||
manager types ignore it.
|
||||
|
||||
"""
|
||||
for manager in self.single_type_managers:
|
||||
manager.remove_skipped_blocks(
|
||||
@@ -449,9 +447,7 @@ class KVCacheCoordinator(ABC):
|
||||
)
|
||||
|
||||
def get_blocks(self, request_id: str) -> tuple[list[KVCacheBlock], ...]:
|
||||
"""
|
||||
Get the blocks for the request.
|
||||
"""
|
||||
"""Get the blocks for the request."""
|
||||
return tuple(
|
||||
manager.req_to_blocks.get(request_id) or []
|
||||
for manager in self.single_type_managers
|
||||
@@ -465,7 +461,8 @@ class KVCacheCoordinator(ABC):
|
||||
) -> tuple[tuple[list[KVCacheBlock], ...], int, int]:
|
||||
"""Returns the per-group hit blocks, the hit length, and the number of
|
||||
``num_uncached_common_prefix_tokens`` (a shared prefix that a
|
||||
sparse-retention group has not cached yet; 0 unless hybrid)."""
|
||||
sparse-retention group has not cached yet; 0 unless hybrid).
|
||||
"""
|
||||
pass
|
||||
|
||||
def new_step_starts(self) -> None:
|
||||
@@ -475,8 +472,7 @@ class KVCacheCoordinator(ABC):
|
||||
|
||||
|
||||
class KVCacheCoordinatorNoPrefixCache(KVCacheCoordinator):
|
||||
"""
|
||||
KV cache coordinator to use if prefix caching is disabled or unsupported.
|
||||
"""KV cache coordinator to use if prefix caching is disabled or unsupported.
|
||||
In contrast to UnitaryKVCacheCoordinator and HybridKVCacheCoordinator,
|
||||
supports arbitrary numbers of KV cache groups (including 0 groups).
|
||||
Does not implement any features related to prefix caching.
|
||||
@@ -527,8 +523,7 @@ class KVCacheCoordinatorNoPrefixCache(KVCacheCoordinator):
|
||||
|
||||
|
||||
class UnitaryKVCacheCoordinator(KVCacheCoordinator):
|
||||
"""
|
||||
KV cache coordinator for models with only one KV cache group. This is the
|
||||
"""KV cache coordinator for models with only one KV cache group. This is the
|
||||
case for models with only one KV cache type, e.g., all attention layers use
|
||||
full attention or all attention layers use sliding window attention.
|
||||
"""
|
||||
@@ -613,8 +608,7 @@ class SpecGroup(NamedTuple):
|
||||
|
||||
|
||||
class HybridKVCacheCoordinator(KVCacheCoordinator):
|
||||
"""
|
||||
KV cache coordinator for hybrid models with multiple KV cache types, and
|
||||
"""KV cache coordinator for hybrid models with multiple KV cache types, and
|
||||
thus multiple kv cache groups.
|
||||
"""
|
||||
|
||||
@@ -733,8 +727,7 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
|
||||
)
|
||||
|
||||
def verify_and_split_kv_cache_groups(self) -> None:
|
||||
"""
|
||||
Groups KV cache groups by their spec type for efficient batch processing
|
||||
"""Groups KV cache groups by their spec type for efficient batch processing
|
||||
during cache hit lookup.
|
||||
"""
|
||||
self.attention_groups: list[SpecGroup] = []
|
||||
@@ -843,8 +836,7 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
|
||||
block_hashes: list[BlockHash],
|
||||
max_cache_hit_length: int,
|
||||
) -> tuple[tuple[list[KVCacheBlock], ...], int, int]:
|
||||
"""
|
||||
Find the longest cache hit using an iterative fixed-point algorithm.
|
||||
"""Find the longest cache hit using an iterative fixed-point algorithm.
|
||||
|
||||
Each attention type either accepts the current candidate length or
|
||||
reduces it. If any type reduces the length, restart checks over all
|
||||
@@ -861,8 +853,8 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
|
||||
- The number of tokens of the reconciled (combined) cache hit.
|
||||
- ``num_uncached_common_prefix_tokens``: a shared prefix that a
|
||||
sparse-retention group has not cached yet (0 unless hybrid).
|
||||
"""
|
||||
|
||||
"""
|
||||
num_groups = len(self.kv_cache_config.kv_cache_groups)
|
||||
hit_length = max_cache_hit_length
|
||||
longest_hit_length = 0
|
||||
@@ -985,8 +977,8 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
|
||||
|
||||
Returns:
|
||||
(blocks_per_group, hit_lengths_per_group)
|
||||
"""
|
||||
|
||||
"""
|
||||
num_groups = len(self.kv_cache_config.kv_cache_groups)
|
||||
hit_blocks: list[list[KVCacheBlock]] = [[] for _ in range(num_groups)]
|
||||
hit_lengths: list[int] = [0] * num_groups
|
||||
|
||||
@@ -33,8 +33,7 @@ logger = init_logger(__name__)
|
||||
|
||||
@dataclass
|
||||
class KVCacheBlocks:
|
||||
"""
|
||||
The allocation result of KVCacheManager, work as the interface between
|
||||
"""The allocation result of KVCacheManager, work as the interface between
|
||||
Scheduler and KVCacheManager, to hide KVCacheManager's internal data
|
||||
structure from the Scheduler.
|
||||
"""
|
||||
@@ -85,8 +84,7 @@ class KVCacheBlocks:
|
||||
*,
|
||||
group_ids: Sequence[int] | None = None,
|
||||
) -> tuple[list[int], ...] | None:
|
||||
"""
|
||||
Converts the KVCacheBlocks instance to block_ids.
|
||||
"""Converts the KVCacheBlocks instance to block_ids.
|
||||
|
||||
Args:
|
||||
allow_none: Return None when every selected group is empty.
|
||||
@@ -97,6 +95,7 @@ class KVCacheBlocks:
|
||||
- the outer tuple corresponds to KV cache groups
|
||||
- each inner list contains the block_ids of the blocks in that
|
||||
group
|
||||
|
||||
"""
|
||||
groups = (
|
||||
self.blocks
|
||||
@@ -125,9 +124,7 @@ class KVCacheBlocks:
|
||||
]
|
||||
|
||||
def new_empty(self) -> "KVCacheBlocks":
|
||||
"""
|
||||
Creates a new KVCacheBlocks instance with no blocks.
|
||||
"""
|
||||
"""Creates a new KVCacheBlocks instance with no blocks."""
|
||||
return KVCacheBlocks(tuple(() for _ in range(len(self.blocks))))
|
||||
|
||||
|
||||
@@ -232,6 +229,7 @@ class KVCacheManager:
|
||||
|
||||
Returns:
|
||||
The KV cache usage (between 0.0 and 1.0).
|
||||
|
||||
"""
|
||||
return self.block_pool.get_usage()
|
||||
|
||||
@@ -240,6 +238,7 @@ class KVCacheManager:
|
||||
|
||||
Returns:
|
||||
The current prefix caching stats, or None if logging is disabled.
|
||||
|
||||
"""
|
||||
if not self.log_stats:
|
||||
return None
|
||||
@@ -278,6 +277,7 @@ class KVCacheManager:
|
||||
window) has not cached yet (Marconi-style APC), or 0 if none.
|
||||
Pinned so sparse prefix-cache retention does not drop
|
||||
the junction and defeat cross-request reuse.
|
||||
|
||||
"""
|
||||
# We skip finding the prefix cache hit when prefix caching is
|
||||
# disabled or the request is marked as skipping kv cache read
|
||||
@@ -340,6 +340,7 @@ class KVCacheManager:
|
||||
Returns:
|
||||
The ``get_computed_blocks`` triple (blocks, number of local computed
|
||||
tokens, shared-prefix boundary) plus ``hit_diverged``.
|
||||
|
||||
"""
|
||||
coordinator = self.coordinator
|
||||
if not (
|
||||
@@ -462,6 +463,7 @@ class KVCacheManager:
|
||||
|
||||
Returns:
|
||||
A list of new allocated blocks.
|
||||
|
||||
"""
|
||||
# When loading KV data asynchronously, we may have zero new tokens to
|
||||
# compute while still allocating slots for externally computed tokens.
|
||||
@@ -607,6 +609,7 @@ class KVCacheManager:
|
||||
|
||||
Args:
|
||||
request: The request to free the blocks.
|
||||
|
||||
"""
|
||||
self.coordinator.free(request.request_id)
|
||||
|
||||
@@ -624,6 +627,7 @@ class KVCacheManager:
|
||||
processed_computed_tokens: Computed-token prefix length covering
|
||||
fully processed and committed tokens only (safe to free).
|
||||
num_prompt_tokens: Optional prompt length for R-SWA gap eviction.
|
||||
|
||||
"""
|
||||
self.coordinator.remove_skipped_blocks(
|
||||
request_id, processed_computed_tokens, num_prompt_tokens
|
||||
@@ -639,14 +643,16 @@ class KVCacheManager:
|
||||
|
||||
Returns:
|
||||
The request's blocks in allocation order.
|
||||
|
||||
"""
|
||||
return self.coordinator.pop_blocks_for_free(request.request_id)
|
||||
|
||||
def evict_blocks(self, block_ids: set[int]) -> None:
|
||||
"""evict blocks from the prefix cache by their block IDs.
|
||||
"""Evict blocks from the prefix cache by their block IDs.
|
||||
|
||||
Args:
|
||||
block_ids: Set of block IDs to evict from cache.
|
||||
|
||||
"""
|
||||
self.block_pool.evict_blocks(block_ids)
|
||||
|
||||
@@ -658,6 +664,7 @@ class KVCacheManager:
|
||||
Returns:
|
||||
bool: True if the prefix cache is successfully reset,
|
||||
False otherwise.
|
||||
|
||||
"""
|
||||
if not self.coordinator.reset_prefix_cache():
|
||||
return False
|
||||
@@ -697,6 +704,7 @@ class KVCacheManager:
|
||||
Returns:
|
||||
list[int]: The number of common prefix blocks for each kv cache
|
||||
group.
|
||||
|
||||
"""
|
||||
return self.coordinator.get_num_common_prefix_blocks(running_request_id)
|
||||
|
||||
@@ -705,6 +713,7 @@ class KVCacheManager:
|
||||
|
||||
Returns:
|
||||
A list of KV cache events.
|
||||
|
||||
"""
|
||||
events = self.block_pool.take_events()
|
||||
for event in events:
|
||||
@@ -792,6 +801,7 @@ class KVCacheManager:
|
||||
request: The request to cache the blocks.
|
||||
num_computed_tokens: The number of computed tokens, including tokens
|
||||
that are already cached and tokens to be cached.
|
||||
|
||||
"""
|
||||
if self.enable_caching:
|
||||
self.coordinator.cache_blocks(request, num_computed_tokens)
|
||||
@@ -846,7 +856,8 @@ class KVCacheManager:
|
||||
self, request_id: str, start_token: int, end_token: int
|
||||
) -> list[int]:
|
||||
"""The request's block ids covering [start_token, end_token), from
|
||||
the groups whose new blocks are zeroed by the worker."""
|
||||
the groups whose new blocks are zeroed by the worker.
|
||||
"""
|
||||
ids: list[int] = []
|
||||
for mgr in self.coordinator.single_type_managers:
|
||||
if mgr.records_new_block_ids:
|
||||
|
||||
@@ -263,6 +263,7 @@ class FreeKVCacheBlockQueue:
|
||||
|
||||
Args:
|
||||
blocks: A list of KVCacheBlock objects.
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self, blocks: list[KVCacheBlock]) -> None:
|
||||
@@ -300,6 +301,7 @@ class FreeKVCacheBlockQueue:
|
||||
|
||||
Returns:
|
||||
The first free block.
|
||||
|
||||
"""
|
||||
if (
|
||||
self.fake_free_list_head.next_free_block is self.fake_free_list_tail
|
||||
@@ -340,6 +342,7 @@ class FreeKVCacheBlockQueue:
|
||||
|
||||
Returns:
|
||||
A list of n free blocks.
|
||||
|
||||
"""
|
||||
if n == 0:
|
||||
return []
|
||||
@@ -370,6 +373,7 @@ class FreeKVCacheBlockQueue:
|
||||
|
||||
Args:
|
||||
block: The block to remove.
|
||||
|
||||
"""
|
||||
if block.prev_free_block is None or block.next_free_block is None:
|
||||
# This should not happen if the block is from the free list.
|
||||
@@ -391,6 +395,7 @@ class FreeKVCacheBlockQueue:
|
||||
|
||||
Args:
|
||||
block: The block to append.
|
||||
|
||||
"""
|
||||
if self.fake_free_list_tail.prev_free_block is None:
|
||||
raise RuntimeError(
|
||||
@@ -434,6 +439,7 @@ class FreeKVCacheBlockQueue:
|
||||
|
||||
Args:
|
||||
blocks: The blocks to append.
|
||||
|
||||
"""
|
||||
if len(blocks) == 0:
|
||||
return
|
||||
@@ -459,6 +465,7 @@ class FreeKVCacheBlockQueue:
|
||||
|
||||
Returns:
|
||||
A list of free blocks.
|
||||
|
||||
"""
|
||||
ret = []
|
||||
if self.fake_free_list_head.next_free_block is None:
|
||||
@@ -505,6 +512,7 @@ def _gen_mm_extra_hash_keys(
|
||||
|
||||
Returns:
|
||||
A tuple of extra keys and the next multi-modal index.
|
||||
|
||||
"""
|
||||
extra_keys: list[Any] = []
|
||||
|
||||
@@ -565,6 +573,7 @@ def _gen_lora_extra_hash_keys(request: Request) -> list[str]:
|
||||
Returns:
|
||||
Return LoRA name of the request if it is a LoRA request. Return empty
|
||||
list otherwise.
|
||||
|
||||
"""
|
||||
if not request.lora_request:
|
||||
return []
|
||||
@@ -584,6 +593,7 @@ def _gen_prompt_embeds_extra_hash_keys(
|
||||
Returns:
|
||||
Return a stable hash of the block prompt embeddings if prompt embeds
|
||||
are present. Return empty list otherwise.
|
||||
|
||||
"""
|
||||
if request.prompt_embeds is None:
|
||||
return []
|
||||
@@ -612,6 +622,7 @@ def generate_block_hash_extra_keys(
|
||||
|
||||
Returns:
|
||||
A tuple of extra keys and the next multi-modal index.
|
||||
|
||||
"""
|
||||
mm_extra_keys: list[Any]
|
||||
mm_extra_keys, new_start_mm_idx = _gen_mm_extra_hash_keys(
|
||||
@@ -645,6 +656,7 @@ def hash_block_tokens(
|
||||
the contents of the preceding block(s). The hash value is used for
|
||||
prefix caching. We use LRU cache for this function to avoid recomputing
|
||||
hash values for the same block contents.
|
||||
|
||||
Args:
|
||||
hash_function: The hash function used to compute block hash.
|
||||
parent_block_hash: The hash of the parent block. None
|
||||
@@ -652,9 +664,11 @@ def hash_block_tokens(
|
||||
curr_block_token_ids: A list of token ids in the current
|
||||
block. The current block is assumed to be full.
|
||||
extra_keys: Extra keys for the block.
|
||||
|
||||
Returns:
|
||||
The hash value of the block and the token ids in the block.
|
||||
The entire tuple is used as the hash key of the block.
|
||||
|
||||
"""
|
||||
if not parent_block_hash:
|
||||
parent_block_hash = NONE_HASH
|
||||
@@ -813,8 +827,7 @@ def get_request_block_hasher(
|
||||
hash_block_size: int,
|
||||
caching_hash_fn: Callable[[Any], bytes],
|
||||
) -> Callable[[Request], list[BlockHash]]:
|
||||
"""
|
||||
Returns a function which computes the list of un-computed block hashes
|
||||
"""Returns a function which computes the list of un-computed block hashes
|
||||
of a request.
|
||||
|
||||
Hashes are computed at ``hash_block_size`` granularity and chained over the
|
||||
@@ -912,9 +925,7 @@ def _check_enough_kv_cache_memory(
|
||||
def max_memory_usage_bytes(
|
||||
vllm_config: VllmConfig, kv_cache_specs: Iterable[KVCacheSpec]
|
||||
) -> int:
|
||||
"""
|
||||
Get the maximum memory usage in bytes for the given KV cache specs.
|
||||
"""
|
||||
"""Get the maximum memory usage in bytes for the given KV cache specs."""
|
||||
return sum(spec.max_memory_usage_bytes(vllm_config) for spec in kv_cache_specs)
|
||||
|
||||
|
||||
@@ -923,8 +934,7 @@ def estimate_max_model_len(
|
||||
kv_cache_spec: dict[str, KVCacheSpec],
|
||||
available_memory: int,
|
||||
) -> int:
|
||||
"""
|
||||
Estimates the maximum model length that can fit in the available memory
|
||||
"""Estimates the maximum model length that can fit in the available memory
|
||||
using binary search.
|
||||
|
||||
This function temporarily modifies max_model_len during estimation but
|
||||
@@ -937,6 +947,7 @@ def estimate_max_model_len(
|
||||
|
||||
Returns:
|
||||
The estimated maximum model length that can fit in the available memory.
|
||||
|
||||
"""
|
||||
# Save the original max_model_len to restore after estimation
|
||||
original_max_model_len = vllm_config.model_config.max_model_len
|
||||
@@ -977,8 +988,7 @@ def check_enough_kv_cache_memory(
|
||||
kv_cache_spec: dict[str, KVCacheSpec],
|
||||
available_memory: int,
|
||||
):
|
||||
"""
|
||||
Checks whether `available_memory` is enough for the KV cache to hold at
|
||||
"""Checks whether `available_memory` is enough for the KV cache to hold at
|
||||
least one request with the model's max_model_len.
|
||||
|
||||
Args:
|
||||
@@ -988,8 +998,8 @@ def check_enough_kv_cache_memory(
|
||||
|
||||
Raises:
|
||||
ValueError: If there is not enough memory available for the KV cache.
|
||||
"""
|
||||
|
||||
"""
|
||||
# No need to check for available memory if the kv_cache_spec is empty
|
||||
if kv_cache_spec:
|
||||
# Reserve the null block BlockPool permanently holds back, so the check
|
||||
@@ -1012,8 +1022,7 @@ def check_enough_kv_cache_memory(
|
||||
def create_kv_cache_group_specs(
|
||||
kv_cache_spec: dict[str, KVCacheSpec], grouped_layer_names: list[list[str]]
|
||||
) -> list[KVCacheGroupSpec]:
|
||||
"""
|
||||
Create KVCacheGroupSpec object for each kv cache group layer.
|
||||
"""Create KVCacheGroupSpec object for each kv cache group layer.
|
||||
The layers in the same group should share the same
|
||||
KVCacheSpec.
|
||||
|
||||
@@ -1024,8 +1033,10 @@ def create_kv_cache_group_specs(
|
||||
A list of kv cache groups, where each element is a list of layer
|
||||
names that belong to the same group and should share the same
|
||||
KVCacheSpec.
|
||||
|
||||
Returns:
|
||||
A list of KVCacheGroupSpec objects, one for each group.
|
||||
|
||||
"""
|
||||
kv_cache_groups = []
|
||||
for layer_names_one_group in grouped_layer_names:
|
||||
@@ -1040,8 +1051,7 @@ def create_kv_cache_group_specs(
|
||||
|
||||
|
||||
def is_kv_cache_spec_uniform(kv_cache_spec: dict[str, KVCacheSpec]) -> bool:
|
||||
"""
|
||||
Whether all layers in the given KVCacheSpec have the same KV cache spec.
|
||||
"""Whether all layers in the given KVCacheSpec have the same KV cache spec.
|
||||
Note that we regard FullAttentionSpec with and without sliding window as
|
||||
the same type.
|
||||
|
||||
@@ -1050,8 +1060,8 @@ def is_kv_cache_spec_uniform(kv_cache_spec: dict[str, KVCacheSpec]) -> bool:
|
||||
|
||||
Returns:
|
||||
True if all layers have the same type, False otherwise.
|
||||
"""
|
||||
|
||||
"""
|
||||
if not kv_cache_spec:
|
||||
# Encoder-only models do not have KV cache, kv_cache_type can be
|
||||
# regarded as uniform.
|
||||
@@ -1067,8 +1077,7 @@ def is_kv_cache_spec_uniform(kv_cache_spec: dict[str, KVCacheSpec]) -> bool:
|
||||
def get_max_concurrency_for_kv_cache_config(
|
||||
vllm_config: VllmConfig, kv_cache_config: KVCacheConfig
|
||||
) -> float:
|
||||
"""
|
||||
Get the maximum concurrency for the given KV cache configuration.
|
||||
"""Get the maximum concurrency for the given KV cache configuration.
|
||||
|
||||
A request at max_model_len consumes whole blocks from each group's block
|
||||
table — cdiv(per-request bytes, page bytes) of the group's spec — and all
|
||||
@@ -1101,8 +1110,7 @@ def get_max_concurrency_for_kv_cache_config(
|
||||
|
||||
|
||||
def may_override_num_blocks(vllm_config: VllmConfig, num_blocks: int) -> int:
|
||||
"""
|
||||
Override the number of kv cache blocks if `num_gpu_blocks_override` is set.
|
||||
"""Override the number of kv cache blocks if `num_gpu_blocks_override` is set.
|
||||
The override is logged once, at the call site in `get_kv_cache_configs`.
|
||||
"""
|
||||
if vllm_config.cache_config.num_gpu_blocks_override is not None:
|
||||
@@ -1111,8 +1119,7 @@ def may_override_num_blocks(vllm_config: VllmConfig, num_blocks: int) -> int:
|
||||
|
||||
|
||||
def _pool_bytes_per_block(kv_cache_groups: list[KVCacheGroupSpec]) -> int:
|
||||
"""
|
||||
Bytes consumed by one block in the worker's shared KV cache pool, mirroring
|
||||
"""Bytes consumed by one block in the worker's shared KV cache pool, mirroring
|
||||
the divisor used by `get_kv_cache_config_from_groups` to convert
|
||||
`available_memory` into `num_blocks`. Used to compute the effective KV cache
|
||||
capacity once `num_gpu_blocks_override` is applied.
|
||||
@@ -1121,9 +1128,7 @@ def _pool_bytes_per_block(kv_cache_groups: list[KVCacheGroupSpec]) -> int:
|
||||
|
||||
|
||||
def get_uniform_page_size(kv_cache_specs: Iterable[KVCacheSpec]) -> int:
|
||||
"""
|
||||
Get the page size of the KV cache.
|
||||
"""
|
||||
"""Get the page size of the KV cache."""
|
||||
page_sizes = {layer.page_size_bytes for layer in kv_cache_specs}
|
||||
assert len(page_sizes) == 1
|
||||
return page_sizes.pop()
|
||||
@@ -1132,8 +1137,7 @@ def get_uniform_page_size(kv_cache_specs: Iterable[KVCacheSpec]) -> int:
|
||||
def _get_kv_cache_groups_uniform_spec(
|
||||
kv_cache_specs: dict[str, KVCacheSpec],
|
||||
) -> list[KVCacheGroupSpec]:
|
||||
"""
|
||||
Generates the KV cache configuration for a model with the same KV cache
|
||||
"""Generates the KV cache configuration for a model with the same KV cache
|
||||
spec for all layers.
|
||||
|
||||
Args:
|
||||
@@ -1141,16 +1145,15 @@ def _get_kv_cache_groups_uniform_spec(
|
||||
|
||||
Returns:
|
||||
The generated KVCacheGroupSpecs
|
||||
"""
|
||||
|
||||
"""
|
||||
return create_kv_cache_group_specs(kv_cache_specs, [list(kv_cache_specs.keys())])
|
||||
|
||||
|
||||
def _get_kv_cache_groups_uniform_type(
|
||||
spec: UniformTypeKVCacheSpecs,
|
||||
) -> list[KVCacheGroupSpec]:
|
||||
"""
|
||||
Generates the KV cache configuration for a model with one type of KV cache
|
||||
"""Generates the KV cache configuration for a model with one type of KV cache
|
||||
but different hidden sizes. All layers are merged into one group.
|
||||
|
||||
Args:
|
||||
@@ -1158,8 +1161,8 @@ def _get_kv_cache_groups_uniform_type(
|
||||
|
||||
Returns:
|
||||
The generated KVCacheGroupSpecs
|
||||
"""
|
||||
|
||||
"""
|
||||
return [KVCacheGroupSpec(list(spec.kv_cache_specs.keys()), spec)]
|
||||
|
||||
|
||||
@@ -1365,8 +1368,7 @@ def _glm5_next_tensor_layout(
|
||||
def unify_kv_cache_spec_page_size(
|
||||
kv_cache_spec: dict[str, KVCacheSpec],
|
||||
) -> dict[str, KVCacheSpec]:
|
||||
"""
|
||||
Unify the page size of the given KVCacheSpec. If the page size of all layers
|
||||
"""Unify the page size of the given KVCacheSpec. If the page size of all layers
|
||||
are the same, return the original KVCacheSpec. If not same, unify the page
|
||||
size by increasing the block size of layers with smaller page size. Two
|
||||
cases cannot be unified by block size alone and pad their physical page to
|
||||
@@ -1385,6 +1387,7 @@ def unify_kv_cache_spec_page_size(
|
||||
|
||||
Returns:
|
||||
The updated KVCacheSpec with the same page_size_bytes.
|
||||
|
||||
"""
|
||||
page_sizes = {layer.page_size_bytes for layer in kv_cache_spec.values()}
|
||||
if len(page_sizes) <= 1:
|
||||
@@ -1435,8 +1438,7 @@ def is_kv_cache_type_attention_free(kv_cache_spec: dict[str, KVCacheSpec]) -> bo
|
||||
def _get_kv_cache_groups_uniform_page_size(
|
||||
kv_cache_spec: dict[str, KVCacheSpec],
|
||||
) -> list[KVCacheGroupSpec]:
|
||||
"""
|
||||
Generates the KV cache groups for hybrid models with multiple
|
||||
"""Generates the KV cache groups for hybrid models with multiple
|
||||
attention types but still with a uniform page size (physical memory per
|
||||
block per layer) for all layers.
|
||||
|
||||
@@ -1496,6 +1498,7 @@ def _get_kv_cache_groups_uniform_page_size(
|
||||
kv_cache_spec: The KVCacheSpec of each attention layer in the model
|
||||
Returns:
|
||||
The generated KVCacheGroupSpecs
|
||||
|
||||
"""
|
||||
# Group all layers by kv_cache_spec.
|
||||
# E.g., 2 full attention layers and 3 sliding window attention layers,
|
||||
@@ -1659,8 +1662,7 @@ def get_kv_cache_config_from_groups(
|
||||
kv_cache_groups: list[KVCacheGroupSpec],
|
||||
available_memory: int,
|
||||
) -> KVCacheConfig:
|
||||
"""
|
||||
Generate the KV cache configuration from the KV cache groups and spec
|
||||
"""Generate the KV cache configuration from the KV cache groups and spec
|
||||
of each layer.
|
||||
|
||||
Args:
|
||||
@@ -1669,6 +1671,7 @@ def get_kv_cache_config_from_groups(
|
||||
available_memory: Memory available for KV cache in bytes
|
||||
Returns:
|
||||
The generated KVCacheConfig
|
||||
|
||||
"""
|
||||
if len(kv_cache_groups) == 0:
|
||||
# Attention free models do not have KV cache.
|
||||
@@ -1926,15 +1929,14 @@ def _try_get_full_allocation_fallback_groups(
|
||||
|
||||
|
||||
def unify_hybrid_kv_cache_specs(kv_cache_spec: dict[str, KVCacheSpec]):
|
||||
"""
|
||||
This function tries to convert the KV cache specs to one type if the model
|
||||
"""This function tries to convert the KV cache specs to one type if the model
|
||||
is a hybrid model with multiple type of KV cache. It will convert all
|
||||
SlidingWindowSpec to FullAttentionSpec if both types are present.
|
||||
|
||||
Args:
|
||||
kv_cache_spec: The kv cache spec of each attention layer in the model
|
||||
"""
|
||||
|
||||
"""
|
||||
if is_kv_cache_spec_uniform(
|
||||
kv_cache_spec
|
||||
) or UniformTypeKVCacheSpecs.is_uniform_type(kv_cache_spec):
|
||||
@@ -2166,6 +2168,7 @@ def _annotate_eagle_groups(
|
||||
kv_cache_groups: Groups to annotate in place.
|
||||
use_deepseek_v4_fallback: Enable rule 2 for a DeepseekV4/V4.1 packed
|
||||
group.
|
||||
|
||||
"""
|
||||
spec_config = vllm_config.speculative_config
|
||||
if spec_config is None or not spec_config.use_eagle_block_drop():
|
||||
@@ -2196,6 +2199,7 @@ def _warn_if_unannotated_eagle_mamba(
|
||||
Args:
|
||||
vllm_config: Config supplying the speculative method, if any.
|
||||
kv_cache_groups: Groups as they will be handed to consumers.
|
||||
|
||||
"""
|
||||
spec_config = vllm_config.speculative_config
|
||||
if spec_config is None or not spec_config.use_eagle():
|
||||
@@ -2262,8 +2266,7 @@ def get_kv_cache_groups(
|
||||
vllm_config: VllmConfig,
|
||||
kv_cache_spec: dict[str, KVCacheSpec],
|
||||
) -> list[KVCacheGroupSpec]:
|
||||
"""
|
||||
Split the layers in the model into groups with the same KV cache spec.
|
||||
"""Split the layers in the model into groups with the same KV cache spec.
|
||||
|
||||
Args:
|
||||
vllm_config: The global VllmConfig
|
||||
@@ -2271,6 +2274,7 @@ def get_kv_cache_groups(
|
||||
|
||||
Returns:
|
||||
The generated KVCacheGroups
|
||||
|
||||
"""
|
||||
if vllm_config.scheduler_config.disable_hybrid_kv_cache_manager:
|
||||
unify_hybrid_kv_cache_specs(kv_cache_spec)
|
||||
@@ -2356,9 +2360,7 @@ def get_kv_cache_groups(
|
||||
def generate_scheduler_kv_cache_config(
|
||||
kv_cache_configs: list[KVCacheConfig],
|
||||
) -> KVCacheConfig:
|
||||
"""
|
||||
Generate the KV cache configuration for the scheduler.
|
||||
"""
|
||||
"""Generate the KV cache configuration for the scheduler."""
|
||||
assert all(
|
||||
[cfg.num_blocks == kv_cache_configs[0].num_blocks for cfg in kv_cache_configs]
|
||||
)
|
||||
@@ -2382,9 +2384,7 @@ def generate_scheduler_kv_cache_config(
|
||||
def get_kv_cache_capacity(
|
||||
vllm_config: VllmConfig, kv_cache_config: KVCacheConfig
|
||||
) -> tuple[int, float]:
|
||||
"""
|
||||
Get the group-aware KV cache token capacity and max concurrency.
|
||||
"""
|
||||
"""Get the group-aware KV cache token capacity and max concurrency."""
|
||||
max_model_len = vllm_config.model_config.max_model_len
|
||||
max_concurrency = get_max_concurrency_for_kv_cache_config(
|
||||
vllm_config, kv_cache_config
|
||||
@@ -2415,8 +2415,7 @@ def _max_memory_usage_bytes_from_groups(
|
||||
vllm_config: VllmConfig,
|
||||
kv_cache_groups: list[KVCacheGroupSpec],
|
||||
) -> int:
|
||||
"""
|
||||
Calculate maximum memory usage in bytes from KV cache groups.
|
||||
"""Calculate maximum memory usage in bytes from KV cache groups.
|
||||
|
||||
This correctly accounts for padding in hybrid models. For example, if a
|
||||
model has 8 full attention layers and 9 sliding window layers, they will
|
||||
@@ -2475,8 +2474,7 @@ def _estimate_max_model_len_from_groups(
|
||||
kv_cache_groups: list[KVCacheGroupSpec],
|
||||
available_memory: int,
|
||||
) -> int:
|
||||
"""
|
||||
Binary search for the maximum model length that fits in available memory.
|
||||
"""Binary search for the maximum model length that fits in available memory.
|
||||
Returns 0 if even 1 token doesn't fit.
|
||||
"""
|
||||
original_max = vllm_config.model_config.max_model_len
|
||||
@@ -2522,8 +2520,7 @@ def _auto_fit_max_model_len(
|
||||
projected_groups_per_worker: list[list[KVCacheGroupSpec]],
|
||||
available_memory: list[int],
|
||||
) -> None:
|
||||
"""
|
||||
When max_model_len is set to -1, this function estimates the largest
|
||||
"""When max_model_len is set to -1, this function estimates the largest
|
||||
context length that can be supported with the available GPU memory.
|
||||
It uses binary search to find the maximum length that fits across all
|
||||
workers.
|
||||
@@ -2533,6 +2530,7 @@ def _auto_fit_max_model_len(
|
||||
projected_groups_per_worker: KV cache groups projected to each worker.
|
||||
available_memory: Memory available for KV cache in bytes for each
|
||||
worker.
|
||||
|
||||
"""
|
||||
original_max = vllm_config.model_config.max_model_len
|
||||
|
||||
@@ -2585,8 +2583,7 @@ def _project_kv_cache_groups_to_worker(
|
||||
global_kv_cache_groups: list[KVCacheGroupSpec],
|
||||
worker_spec: dict[str, KVCacheSpec],
|
||||
) -> list[KVCacheGroupSpec]:
|
||||
"""
|
||||
Projects global KV cache groups onto a single worker's assigned layers.
|
||||
"""Projects global KV cache groups onto a single worker's assigned layers.
|
||||
|
||||
In pipeline parallelism, each worker only owns a subset of layers. This
|
||||
function filters the global groups to include only layers present on the
|
||||
@@ -2598,6 +2595,7 @@ def _project_kv_cache_groups_to_worker(
|
||||
|
||||
Returns:
|
||||
The projected KV cache groups containing only this worker's layers.
|
||||
|
||||
"""
|
||||
projected_groups: list[KVCacheGroupSpec] = []
|
||||
for group in global_kv_cache_groups:
|
||||
@@ -2628,8 +2626,7 @@ def get_kv_cache_configs(
|
||||
kv_cache_specs: list[dict[str, KVCacheSpec]],
|
||||
available_memory: list[int],
|
||||
) -> list[KVCacheConfig]:
|
||||
"""
|
||||
Generates the KV cache configurations for a model.
|
||||
"""Generates the KV cache configurations for a model.
|
||||
Since we use a shared centralized controller for all workers, we need the
|
||||
`kv_cache_config` to be consistent across all workers to make sure
|
||||
the KV cache allocation can be applied to all workers. However, different
|
||||
@@ -2656,8 +2653,8 @@ def get_kv_cache_configs(
|
||||
|
||||
Returns:
|
||||
The generated KVCacheConfigs for each worker.
|
||||
"""
|
||||
|
||||
"""
|
||||
# Merge the KV cache specs of all workers. Different PP stages may have
|
||||
# different layer names, and different TP ranks of the same PP stage should
|
||||
# have the same KV cache spec.
|
||||
@@ -2778,8 +2775,7 @@ def get_kv_cache_configs(
|
||||
|
||||
|
||||
class BlockHashListWithBlockSize:
|
||||
"""
|
||||
Convert block-hash granularity from `hash_block_size` to `target_block_size`.
|
||||
"""Convert block-hash granularity from `hash_block_size` to `target_block_size`.
|
||||
Used when KV cache groups have different block sizes: `hash_block_size`
|
||||
is the size used to compute the original `block_hashes`; `target_block_size`
|
||||
is the group's actual block size.
|
||||
@@ -2809,6 +2805,7 @@ class BlockHashListWithBlockSize:
|
||||
block_hashes: Block hashes to convert, computed at `hash_block_size`.
|
||||
hash_block_size: Block size at which `block_hashes` were computed.
|
||||
target_block_size: Desired block size; must be a multiple of `hash_block_size`.
|
||||
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
|
||||
@@ -79,6 +79,7 @@ class SchedulerInterface(ABC):
|
||||
Returns:
|
||||
A SchedulerOutput object containing information about the scheduled
|
||||
requests.
|
||||
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -105,6 +106,7 @@ class SchedulerInterface(ABC):
|
||||
Returns:
|
||||
A dict of client index to EngineCoreOutputs object containing the
|
||||
outputs for each request originating from that client.
|
||||
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -115,6 +117,7 @@ class SchedulerInterface(ABC):
|
||||
|
||||
Args:
|
||||
draft_token_ids: The input draft token ids for each request.
|
||||
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -129,6 +132,7 @@ class SchedulerInterface(ABC):
|
||||
draft_token_ids: The input draft token ids for each request.
|
||||
scheduler_output: Update the given scheduler_output
|
||||
with the corresponding draft token ids.
|
||||
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -138,6 +142,7 @@ class SchedulerInterface(ABC):
|
||||
|
||||
Args:
|
||||
request: The new request being added.
|
||||
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -162,6 +167,7 @@ class SchedulerInterface(ABC):
|
||||
Returns:
|
||||
List of requests that were aborted. Will not include any that were
|
||||
already finished.
|
||||
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -172,7 +178,8 @@ class SchedulerInterface(ABC):
|
||||
|
||||
def has_unfinished_requests(self) -> bool:
|
||||
"""Returns True if there are unfinished requests in the scheduler's
|
||||
internal queue."""
|
||||
internal queue.
|
||||
"""
|
||||
return self.get_num_unfinished_requests() > 0
|
||||
|
||||
@abstractmethod
|
||||
@@ -192,7 +199,8 @@ class SchedulerInterface(ABC):
|
||||
|
||||
def has_requests(self) -> bool:
|
||||
"""Returns True if there are unfinished requests, or finished requests
|
||||
not yet returned in SchedulerOutputs."""
|
||||
not yet returned in SchedulerOutputs.
|
||||
"""
|
||||
return self.has_unfinished_requests() or self.has_finished_requests()
|
||||
|
||||
@property
|
||||
@@ -218,6 +226,8 @@ class SchedulerInterface(ABC):
|
||||
preempted and moved to the waiting queue. Otherwise, this method
|
||||
will only reset the KV prefix cache when there is no running request
|
||||
taking KV cache.
|
||||
reset_connector: If True, also reset any KV connector state.
|
||||
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
@@ -43,7 +43,8 @@ class RequestQueue(ABC):
|
||||
@abstractmethod
|
||||
def prepend_requests(self, requests: "RequestQueue") -> None:
|
||||
"""Prepend all requests from another queue to the front of this
|
||||
queue."""
|
||||
queue.
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
@@ -129,8 +130,7 @@ class FCFSRequestQueue(deque[Request], RequestQueue):
|
||||
|
||||
|
||||
class PriorityRequestQueue(RequestQueue):
|
||||
"""
|
||||
A priority queue that supports heap operations.
|
||||
"""A priority queue that supports heap operations.
|
||||
|
||||
Respects the ordering defined in the Request class, where
|
||||
requests with a smaller value of `priority` are processed first.
|
||||
@@ -161,14 +161,16 @@ class PriorityRequestQueue(RequestQueue):
|
||||
"""Add a request to the queue according to priority policy.
|
||||
|
||||
Note: In a priority queue, there is no concept of prepending to the
|
||||
front. Requests are ordered by (priority, arrival_time)."""
|
||||
front. Requests are ordered by (priority, arrival_time).
|
||||
"""
|
||||
self.add_request(request)
|
||||
|
||||
def prepend_requests(self, requests: RequestQueue) -> None:
|
||||
"""Add all requests from another queue according to priority policy.
|
||||
|
||||
Note: In a priority queue, there is no concept of prepending to the
|
||||
front. Requests are ordered by (priority, arrival_time)."""
|
||||
front. Requests are ordered by (priority, arrival_time).
|
||||
"""
|
||||
for request in requests:
|
||||
self.add_request(request)
|
||||
|
||||
|
||||
@@ -1558,12 +1558,10 @@ class Scheduler(SchedulerInterface):
|
||||
def _update_request_as_session(
|
||||
self, session: Request, update: StreamingUpdate
|
||||
) -> None:
|
||||
"""
|
||||
Updates the waiting session with the next streaming update.
|
||||
"""Updates the waiting session with the next streaming update.
|
||||
|
||||
Discards the last sampled output token from the prior input chunk.
|
||||
"""
|
||||
|
||||
# Current streaming input behaviour: Keep only computed output tokens
|
||||
# (discard final sampled output token).
|
||||
num_computed_tokens = session.num_computed_tokens
|
||||
@@ -1665,8 +1663,7 @@ class Scheduler(SchedulerInterface):
|
||||
encoder_compute_budget: int,
|
||||
shift_computed_tokens: int = 0,
|
||||
) -> tuple[list[int], int, int, list[int]]:
|
||||
"""
|
||||
Determine which encoder inputs need to be scheduled in the current step,
|
||||
"""Determine which encoder inputs need to be scheduled in the current step,
|
||||
and update `num_new_tokens` and encoder token budget accordingly.
|
||||
|
||||
An encoder input will be scheduled if:
|
||||
@@ -2507,6 +2504,7 @@ class Scheduler(SchedulerInterface):
|
||||
Returns:
|
||||
List of requests that were aborted. Will not include any that were
|
||||
already finished.
|
||||
|
||||
"""
|
||||
assert RequestStatus.is_finished(finished_status)
|
||||
if isinstance(request_ids, str):
|
||||
@@ -2841,8 +2839,7 @@ class Scheduler(SchedulerInterface):
|
||||
def _connector_finished(
|
||||
self, request: Request
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
"""
|
||||
Invoke the KV connector request_finished() method if applicable.
|
||||
"""Invoke the KV connector request_finished() method if applicable.
|
||||
|
||||
Returns optional kv transfer parameters to be included with the
|
||||
request outputs.
|
||||
@@ -2910,14 +2907,12 @@ class Scheduler(SchedulerInterface):
|
||||
|
||||
def _inflight_prefill_reserved_blocks(self) -> int:
|
||||
"""Num blocks in-flight prefills still need to finish (their reservation)."""
|
||||
|
||||
return sum(
|
||||
self._request_remaining_blocks(req) for req in self._inflight_prefills
|
||||
)
|
||||
|
||||
def _update_waiting_for_remote_kv(self, request: Request) -> None:
|
||||
"""
|
||||
KV Connector: update request state after async recv is finished.
|
||||
"""KV Connector: update request state after async recv is finished.
|
||||
|
||||
When the kv transfer is ready, we cache the blocks
|
||||
and the request state will be moved back to WAITING from
|
||||
@@ -2959,9 +2954,7 @@ class Scheduler(SchedulerInterface):
|
||||
self.finished_recving_kv_req_ids.remove(request.request_id)
|
||||
|
||||
def _try_promote_blocked_waiting_request(self, request: Request) -> bool:
|
||||
"""
|
||||
Try to promote a blocked waiting request back to schedulable states.
|
||||
"""
|
||||
"""Try to promote a blocked waiting request back to schedulable states."""
|
||||
if request.status == RequestStatus.WAITING_FOR_REMOTE_KVS:
|
||||
# finished_recving_kv_req_ids is populated during
|
||||
# update_from_output(), based on worker-side connector signals
|
||||
@@ -2995,8 +2988,7 @@ class Scheduler(SchedulerInterface):
|
||||
)
|
||||
|
||||
def _update_from_kv_xfer_finished(self, kv_connector_output: KVConnectorOutput):
|
||||
"""
|
||||
KV Connector: update the scheduler state based on the output.
|
||||
"""KV Connector: update the scheduler state based on the output.
|
||||
|
||||
The Worker side connectors add finished_recving and
|
||||
finished_sending reqs to the output.
|
||||
@@ -3004,7 +2996,6 @@ class Scheduler(SchedulerInterface):
|
||||
# if finished_recving: add to state so we can
|
||||
schedule the request during the next step.
|
||||
"""
|
||||
|
||||
if self.connector is not None:
|
||||
self.connector.update_connector_output(kv_connector_output)
|
||||
|
||||
@@ -3030,8 +3021,7 @@ class Scheduler(SchedulerInterface):
|
||||
num_scheduled_tokens: dict[str, int],
|
||||
evict_blocks: bool = True,
|
||||
) -> tuple[set[str], int, set[int]]:
|
||||
"""
|
||||
Identify and update requests affected by invalid KV cache blocks.
|
||||
"""Identify and update requests affected by invalid KV cache blocks.
|
||||
|
||||
This method scans the given requests, detects those with invalid blocks
|
||||
and adjusts their `num_computed_tokens` to the longest valid prefix.
|
||||
@@ -3053,6 +3043,7 @@ class Scheduler(SchedulerInterface):
|
||||
be recomputed across all affected requests.
|
||||
- blocks_to_evict (set[int]): Block IDs to evict from cache,
|
||||
including invalid blocks and downstream dependent blocks.
|
||||
|
||||
"""
|
||||
affected_req_ids: set[str] = set()
|
||||
total_affected_tokens = 0
|
||||
@@ -3145,11 +3136,11 @@ class Scheduler(SchedulerInterface):
|
||||
def _handle_invalid_blocks(
|
||||
self, invalid_block_ids: set[int], num_scheduled_tokens: dict[str, int]
|
||||
) -> set[str]:
|
||||
"""
|
||||
Handle requests affected by invalid KV cache blocks.
|
||||
"""Handle requests affected by invalid KV cache blocks.
|
||||
|
||||
Returns:
|
||||
Set of affected request IDs to skip in update_from_output main loop.
|
||||
|
||||
"""
|
||||
if len(self.kv_cache_config.kv_cache_groups) > 1:
|
||||
# Block IDs are only unique within a group, so a flat set of
|
||||
|
||||
@@ -30,11 +30,14 @@ def check_sequence_repetition(
|
||||
params: RepetitionDetectionParams,
|
||||
) -> bool:
|
||||
"""Check if a sequence of token IDs has a repetition pattern.
|
||||
|
||||
Args:
|
||||
token_ids: List of token IDs
|
||||
params: Repetition detection parameters.
|
||||
|
||||
Returns:
|
||||
True if a repetition pattern is found, False otherwise.
|
||||
|
||||
"""
|
||||
max_pattern_size = params.max_pattern_size
|
||||
min_pattern_size = params.min_pattern_size
|
||||
@@ -77,6 +80,7 @@ def remove_all(lst: list, items_to_remove: set) -> list:
|
||||
Note:
|
||||
For single item removal, this modifies the original list in-place
|
||||
and returns it. For multiple items, it creates and returns a new list.
|
||||
|
||||
"""
|
||||
if not items_to_remove:
|
||||
return lst
|
||||
|
||||
@@ -49,8 +49,7 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
class SingleTypeKVCacheManager(ABC):
|
||||
"""
|
||||
An abstract base class for a manager that handle the kv cache management
|
||||
"""An abstract base class for a manager that handle the kv cache management
|
||||
logic of one specific type of attention layer.
|
||||
"""
|
||||
|
||||
@@ -81,14 +80,17 @@ class SingleTypeKVCacheManager(ABC):
|
||||
needs_kv_cache_zeroing: bool = False,
|
||||
max_admission_blocks_per_request: int | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
Initializes the SingleTypeKVCacheManager.
|
||||
"""Initializes the SingleTypeKVCacheManager.
|
||||
|
||||
Args:
|
||||
kv_cache_spec: The kv_cache_spec for this manager.
|
||||
block_pool: The block pool.
|
||||
enable_caching: Whether prefix caching is enabled.
|
||||
kv_cache_group_id: The id of the kv cache group of this manager.
|
||||
scheduler_block_size: The scheduling granularity (LCM of all group
|
||||
block sizes); a multiple of this manager's ``block_size``.
|
||||
dcp_world_size: Decode context parallel world size.
|
||||
pcp_world_size: Prefill context parallel world size.
|
||||
needs_kv_cache_zeroing: Whether worker-side KV cache zeroing needs
|
||||
newly allocated block IDs from this manager.
|
||||
max_admission_blocks_per_request: Recycling-aware per-request
|
||||
@@ -97,6 +99,7 @@ class SingleTypeKVCacheManager(ABC):
|
||||
chunked-local); `None` (the default) means no cap, which is
|
||||
correct for full-attention-style specs that hold every
|
||||
block until the request finishes.
|
||||
|
||||
"""
|
||||
self.scheduler_block_size = scheduler_block_size
|
||||
# Hybrid fine-grained lookup may lower this after all participating
|
||||
@@ -184,8 +187,7 @@ class SingleTypeKVCacheManager(ABC):
|
||||
num_tokens_main_model: int,
|
||||
apply_admission_cap: bool = False,
|
||||
) -> int:
|
||||
"""
|
||||
Get the number of blocks needed to be allocated for the request.
|
||||
"""Get the number of blocks needed to be allocated for the request.
|
||||
|
||||
Args:
|
||||
request_id: The request ID.
|
||||
@@ -206,8 +208,8 @@ class SingleTypeKVCacheManager(ABC):
|
||||
|
||||
Returns:
|
||||
The number of blocks to allocate.
|
||||
"""
|
||||
|
||||
"""
|
||||
num_required_blocks = cdiv(num_tokens, self.block_size)
|
||||
if apply_admission_cap and self._max_admission_blocks_per_request is not None:
|
||||
# Recycling-aware specs (SWA, chunked-local) cap the per-request
|
||||
@@ -269,8 +271,7 @@ class SingleTypeKVCacheManager(ABC):
|
||||
num_local_computed_tokens: int,
|
||||
num_external_computed_tokens: int,
|
||||
) -> None:
|
||||
"""
|
||||
Add the locally cached (prefix-hit) blocks to the request:
|
||||
"""Add the locally cached (prefix-hit) blocks to the request:
|
||||
1. Touch the computed blocks (paired with adding them to `req_blocks`)
|
||||
so their ref_cnt exactly tracks the referencing requests.
|
||||
1.5. (Optional) For sliding window, skipped blocks are padded with nulls.
|
||||
@@ -282,6 +283,7 @@ class SingleTypeKVCacheManager(ABC):
|
||||
prefix cache.
|
||||
num_local_computed_tokens: The number of local computed tokens.
|
||||
num_external_computed_tokens: The number of external computed tokens.
|
||||
|
||||
"""
|
||||
# The coordinator only calls this for first-time allocations (running
|
||||
# requests are short-circuited there), so the request has no blocks yet.
|
||||
@@ -327,8 +329,7 @@ class SingleTypeKVCacheManager(ABC):
|
||||
num_local_computed_tokens: int,
|
||||
num_external_computed_tokens: int,
|
||||
) -> None:
|
||||
"""
|
||||
Allocate new blocks for external (KV-connector) computed tokens.
|
||||
"""Allocate new blocks for external (KV-connector) computed tokens.
|
||||
|
||||
Must run only after every group's local blocks have been touched via
|
||||
`add_local_computed_blocks`, so this group's `get_new_blocks` cannot
|
||||
@@ -338,6 +339,7 @@ class SingleTypeKVCacheManager(ABC):
|
||||
request_id: The request ID.
|
||||
num_local_computed_tokens: The number of local computed tokens.
|
||||
num_external_computed_tokens: The number of external computed tokens.
|
||||
|
||||
"""
|
||||
num_total_computed_tokens = (
|
||||
num_local_computed_tokens + num_external_computed_tokens
|
||||
@@ -364,8 +366,7 @@ class SingleTypeKVCacheManager(ABC):
|
||||
def allocate_new_blocks(
|
||||
self, request_id: str, num_tokens: int, num_tokens_main_model: int
|
||||
) -> list[KVCacheBlock]:
|
||||
"""
|
||||
Allocate new blocks for the request to give it at least `num_tokens`
|
||||
"""Allocate new blocks for the request to give it at least `num_tokens`
|
||||
token slots.
|
||||
|
||||
Args:
|
||||
@@ -375,8 +376,10 @@ class SingleTypeKVCacheManager(ABC):
|
||||
num_tokens_main_model: The number of tokens for the main model (aka target
|
||||
model in spec decode). w/o spec decode, it is num_tokens;
|
||||
with spec decode, it is num_tokens - num_lookahead_tokens.
|
||||
|
||||
Returns:
|
||||
The new allocated blocks.
|
||||
|
||||
"""
|
||||
cow_blocks: list[KVCacheBlock] = []
|
||||
if request_id in self._partial_hit_reqs:
|
||||
@@ -475,8 +478,7 @@ class SingleTypeKVCacheManager(ABC):
|
||||
*,
|
||||
replay_boundaries: Sequence[int],
|
||||
) -> None:
|
||||
"""
|
||||
Cache the blocks for the request.
|
||||
"""Cache the blocks for the request.
|
||||
|
||||
Args:
|
||||
request: The request.
|
||||
@@ -488,6 +490,7 @@ class SingleTypeKVCacheManager(ABC):
|
||||
a tail once per that-sized segment. Only SWA acts on it.
|
||||
replay_boundaries: Positions a later request replaying this prompt
|
||||
can resume at, from ``get_replay_boundaries``.
|
||||
|
||||
"""
|
||||
num_cached_blocks = self.num_cached_block.get(request.request_id, 0)
|
||||
num_full_blocks = num_tokens // self.block_size
|
||||
@@ -552,8 +555,7 @@ class SingleTypeKVCacheManager(ABC):
|
||||
return None
|
||||
|
||||
def pop_blocks_for_free(self, request_id: str) -> list[KVCacheBlock]:
|
||||
"""
|
||||
Pop the request's bookkeeping and return its blocks without yet
|
||||
"""Pop the request's bookkeeping and return its blocks without yet
|
||||
returning them to the block pool. The caller is responsible for
|
||||
eventually passing the returned blocks to `block_pool.free_blocks`,
|
||||
freeing them in reverse order (so that tail blocks are evicted first).
|
||||
@@ -563,6 +565,7 @@ class SingleTypeKVCacheManager(ABC):
|
||||
|
||||
Returns:
|
||||
The request's blocks in allocation order.
|
||||
|
||||
"""
|
||||
# Default to [] in case a request is freed (aborted) before alloc.
|
||||
req_blocks = self.req_to_blocks.pop(request_id, [])
|
||||
@@ -571,19 +574,18 @@ class SingleTypeKVCacheManager(ABC):
|
||||
return req_blocks
|
||||
|
||||
def free(self, request_id: str) -> None:
|
||||
"""
|
||||
Free the blocks for the request.
|
||||
"""Free the blocks for the request.
|
||||
|
||||
Args:
|
||||
request_id: The request ID.
|
||||
|
||||
"""
|
||||
# Free blocks in reverse order so that the tail blocks are freed first.
|
||||
self.block_pool.free_blocks(reversed(self.pop_blocks_for_free(request_id)))
|
||||
|
||||
@abstractmethod
|
||||
def get_num_common_prefix_blocks(self, running_request_id: str) -> int:
|
||||
"""
|
||||
Get the number of common prefix blocks for all requests with allocated
|
||||
"""Get the number of common prefix blocks for all requests with allocated
|
||||
KV cache.
|
||||
|
||||
Args:
|
||||
@@ -592,8 +594,8 @@ class SingleTypeKVCacheManager(ABC):
|
||||
Returns:
|
||||
The number of common prefix blocks for all requests with allocated
|
||||
KV cache.
|
||||
"""
|
||||
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@classmethod
|
||||
@@ -610,8 +612,7 @@ class SingleTypeKVCacheManager(ABC):
|
||||
dcp_world_size: int = 1,
|
||||
pcp_world_size: int = 1,
|
||||
) -> tuple[tuple[list[KVCacheBlock], ...], int]:
|
||||
"""
|
||||
Get the longest cache hit prefix of the blocks that is not longer than
|
||||
"""Get the longest cache hit prefix of the blocks that is not longer than
|
||||
`max_length`. The prefix should be a common prefix hit for all the
|
||||
kv cache groups in `kv_cache_group_ids`. If no cache hit is found,
|
||||
return an empty list.
|
||||
@@ -647,8 +648,8 @@ class SingleTypeKVCacheManager(ABC):
|
||||
For example, sliding window manager should return a list like
|
||||
([NULL, NULL, KVCacheBlock(7), KVCacheBlock(8)]) for block size 4
|
||||
and sliding window 8 and len(kv_cache_group_ids) = 1.
|
||||
"""
|
||||
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def _remove_blocks_in_range(
|
||||
@@ -684,8 +685,8 @@ class SingleTypeKVCacheManager(ABC):
|
||||
processed_computed_tokens: int,
|
||||
num_prompt_tokens: int | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
Remove and free the blocks that are no longer needed for attention computation.
|
||||
"""Remove and free blocks no longer needed for attention computation.
|
||||
|
||||
The removed blocks should be replaced by null_block.
|
||||
|
||||
This function depends on `get_num_skipped_tokens`, which need to be implemented
|
||||
@@ -698,6 +699,7 @@ class SingleTypeKVCacheManager(ABC):
|
||||
num_prompt_tokens: Optional prompt length for attention types (e.g.
|
||||
R-SWA) that evict a middle gap rather than a head prefix. Ignored
|
||||
by the default implementation.
|
||||
|
||||
"""
|
||||
del num_prompt_tokens
|
||||
# Remove the blocks that will be skipped during attention computation.
|
||||
@@ -718,14 +720,14 @@ class SingleTypeKVCacheManager(ABC):
|
||||
self._remove_blocks_in_range(request_id, 0, num_skipped_blocks)
|
||||
|
||||
def get_num_skipped_tokens(self, num_computed_tokens: int) -> int:
|
||||
"""
|
||||
Get the number of tokens that will be skipped for attention computation.
|
||||
"""Get the number of tokens that will be skipped for attention computation.
|
||||
|
||||
Args:
|
||||
num_computed_tokens: The number of tokens that have been computed.
|
||||
|
||||
Returns:
|
||||
The number of tokens that will be skipped for attention computation.
|
||||
|
||||
"""
|
||||
# The default behavior is to not skip any tokens.
|
||||
return 0
|
||||
@@ -1144,8 +1146,7 @@ class SlidingWindowManager(SingleTypeKVCacheManager):
|
||||
return mask
|
||||
|
||||
def get_num_skipped_tokens(self, num_computed_tokens: int) -> int:
|
||||
"""
|
||||
Get the number of tokens that will be skipped for attention computation.
|
||||
"""Get the number of tokens that will be skipped for attention computation.
|
||||
|
||||
For sliding window, this corresponds to the tokens that are prior to
|
||||
the current sliding window.
|
||||
@@ -1174,6 +1175,7 @@ class SlidingWindowManager(SingleTypeKVCacheManager):
|
||||
|
||||
Returns:
|
||||
The number of tokens that will be skipped for attention computation.
|
||||
|
||||
"""
|
||||
return max(
|
||||
0,
|
||||
@@ -1181,8 +1183,7 @@ class SlidingWindowManager(SingleTypeKVCacheManager):
|
||||
)
|
||||
|
||||
def get_num_common_prefix_blocks(self, running_request_id: str) -> int:
|
||||
"""
|
||||
NOTE(Chen): The prefix blocks are null blocks for sliding window layers.
|
||||
"""NOTE(Chen): The prefix blocks are null blocks for sliding window layers.
|
||||
So it's not correct to count ref_cnt like FullAttentionManager. Return
|
||||
0 here for correctness. Need to support cascade attention + sliding
|
||||
window in the future.
|
||||
@@ -1301,8 +1302,7 @@ class ChunkedLocalAttentionManager(SingleTypeKVCacheManager):
|
||||
dcp_world_size: int = 1,
|
||||
pcp_world_size: int = 1,
|
||||
) -> tuple[tuple[list[KVCacheBlock], ...], int]:
|
||||
"""
|
||||
For chunked local attention, we need to find the longest cache hit
|
||||
"""For chunked local attention, we need to find the longest cache hit
|
||||
prefix of the blocks that is not longer than `max_length`. The prefix
|
||||
should be a common prefix hit for all the kv cache groups in
|
||||
`kv_cache_group_ids`. If no cache hit is found, return an empty list.
|
||||
@@ -1335,6 +1335,7 @@ class ChunkedLocalAttentionManager(SingleTypeKVCacheManager):
|
||||
|
||||
Returns:
|
||||
A list of cached blocks
|
||||
|
||||
"""
|
||||
assert isinstance(kv_cache_spec, ChunkedLocalAttentionSpec), (
|
||||
"ChunkedLocalAttentionManager can only be used for "
|
||||
@@ -1389,8 +1390,7 @@ class ChunkedLocalAttentionManager(SingleTypeKVCacheManager):
|
||||
return computed_blocks, hit_length
|
||||
|
||||
def get_num_skipped_tokens(self, num_computed_tokens: int) -> int:
|
||||
"""
|
||||
Get the number of tokens that will be skipped for attention computation.
|
||||
"""Get the number of tokens that will be skipped for attention computation.
|
||||
|
||||
For chunked local attention, this corresponds to the tokens that are on
|
||||
the left side of the current chunk.
|
||||
@@ -1428,6 +1428,7 @@ class ChunkedLocalAttentionManager(SingleTypeKVCacheManager):
|
||||
|
||||
Returns:
|
||||
The number of tokens that will be skipped for attention computation.
|
||||
|
||||
"""
|
||||
num_skipped_tokens = (
|
||||
num_computed_tokens // self.attention_chunk_size
|
||||
@@ -1435,9 +1436,7 @@ class ChunkedLocalAttentionManager(SingleTypeKVCacheManager):
|
||||
return num_skipped_tokens
|
||||
|
||||
def get_num_common_prefix_blocks(self, running_request_id: str) -> int:
|
||||
"""
|
||||
cascade attention is not supported by chunked local attention.
|
||||
"""
|
||||
"""Cascade attention is not supported by chunked local attention."""
|
||||
return 0
|
||||
|
||||
|
||||
@@ -1674,9 +1673,7 @@ class MambaManager(SingleTypeKVCacheManager):
|
||||
blocks[last_state_block_idx] = self._null_block
|
||||
|
||||
def get_num_common_prefix_blocks(self, running_request_id: str) -> int:
|
||||
"""
|
||||
cascade attention is not supported by mamba
|
||||
"""
|
||||
"""Cascade attention is not supported by mamba"""
|
||||
return 0
|
||||
|
||||
def _needs_internal_checkpoint(
|
||||
@@ -1979,8 +1976,7 @@ class MambaManager(SingleTypeKVCacheManager):
|
||||
return super().pop_blocks_for_free(request_id)
|
||||
|
||||
def get_num_skipped_tokens(self, num_computed_tokens: int) -> int:
|
||||
"""
|
||||
Get the number of tokens whose mamba state are not needed anymore. Mamba only
|
||||
"""Get the number of tokens whose mamba state are not needed anymore. Mamba only
|
||||
need to keep the state of the last computed token, so we return
|
||||
num_computed_tokens - 1.
|
||||
"""
|
||||
@@ -2600,8 +2596,7 @@ def get_manager_for_kv_cache_spec(
|
||||
role: str | None = None,
|
||||
**kwargs,
|
||||
) -> SingleTypeKVCacheManager:
|
||||
"""
|
||||
Get the appropriate manager for a given KVCacheSpec.
|
||||
"""Get the appropriate manager for a given KVCacheSpec.
|
||||
|
||||
Uses the KVCacheSpecRegistry to look up the manager class, supporting
|
||||
both built-in and custom specs registered via @register_kv_cache_spec
|
||||
@@ -2612,8 +2607,11 @@ def get_manager_for_kv_cache_spec(
|
||||
max_in_flight_tokens: The max tokens scheduled but not yet settled
|
||||
(one batch per concurrent step); see `VllmConfig.max_in_flight_tokens`
|
||||
max_model_len: The maximum context length the model could serve
|
||||
role: Optional KV cache group role used to select a role-specific
|
||||
manager
|
||||
Returns:
|
||||
An instance of the appropriate SingleTypeKVCacheManager subclass
|
||||
|
||||
"""
|
||||
manager_class = KVCacheSpecRegistry.get_manager_class(kv_cache_spec, role)
|
||||
assert manager_class is not None, (
|
||||
|
||||
Reference in New Issue
Block a user