Convert formatting to use ruff instead of yapf + isort (#26247)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Harry Mellor
2025-10-05 07:06:22 -07:00
committed by GitHub
parent 17edd8a807
commit d6953beb91
1508 changed files with 115205 additions and 94107 deletions
+58 -48
View File
@@ -3,16 +3,24 @@
from collections.abc import Iterable
from typing import Any, Optional, Union
from vllm.distributed.kv_events import (MEDIUM_GPU, AllBlocksCleared,
BlockRemoved, BlockStored,
KVCacheEvent)
from vllm.distributed.kv_events import (
MEDIUM_GPU,
AllBlocksCleared,
BlockRemoved,
BlockStored,
KVCacheEvent,
)
from vllm.logger import init_logger
from vllm.v1.core.kv_cache_utils import (BlockHash, BlockHashWithGroupId,
ExternalBlockHash,
FreeKVCacheBlockQueue, KVCacheBlock,
get_block_hash,
make_block_hash_with_group_id,
maybe_convert_block_hash)
from vllm.v1.core.kv_cache_utils import (
BlockHash,
BlockHashWithGroupId,
ExternalBlockHash,
FreeKVCacheBlockQueue,
KVCacheBlock,
get_block_hash,
make_block_hash_with_group_id,
maybe_convert_block_hash,
)
from vllm.v1.request import Request
logger = init_logger(__name__)
@@ -20,7 +28,7 @@ logger = init_logger(__name__)
class BlockHashToBlockMap:
"""
Cache of blocks that are used for prefix caching. It caches blocks
Cache of blocks that are used for prefix caching. It caches blocks
from hash directly to a block or multiple blocks
(i.e. {block_hash: KVCacheBlocks})
- Mostly block_hash maps to a single KVCacheBlock, and KVCacheBlocks
@@ -42,11 +50,11 @@ class BlockHashToBlockMap:
"""
def __init__(self):
self._cache: dict[BlockHashWithGroupId,
Union[KVCacheBlock, dict[int, KVCacheBlock]]] = {}
self._cache: dict[
BlockHashWithGroupId, Union[KVCacheBlock, dict[int, KVCacheBlock]]
] = {}
def get_one_block(self,
key: BlockHashWithGroupId) -> Optional[KVCacheBlock]:
def get_one_block(self, key: BlockHashWithGroupId) -> Optional[KVCacheBlock]:
"""
Gets any block with the given block hash key.
"""
@@ -77,8 +85,7 @@ class BlockHashToBlockMap:
else:
self._unexpected_blocks_type(blocks)
def pop(self, key: BlockHashWithGroupId,
block_id: int) -> Optional[KVCacheBlock]:
def pop(self, key: BlockHashWithGroupId, block_id: int) -> Optional[KVCacheBlock]:
"""
Checks if block_hash exists and pop block_id from the cache
"""
@@ -148,8 +155,7 @@ class BlockPool:
self.free_block_queue = FreeKVCacheBlockQueue(self.blocks)
# Cache for block lookup
self.cached_block_hash_to_block: BlockHashToBlockMap = \
BlockHashToBlockMap()
self.cached_block_hash_to_block: BlockHashToBlockMap = BlockHashToBlockMap()
# To represent a placeholder block with block_id=0.
# The ref_cnt of null_block is not maintained, needs special care to
@@ -161,9 +167,9 @@ class BlockPool:
self.kv_event_queue: list[KVCacheEvent] = []
def get_cached_block(
self, block_hash: BlockHash,
kv_cache_group_ids: list[int]) -> Optional[list[KVCacheBlock]]:
"""Get the cached block by the block hash for each group in
self, block_hash: BlockHash, kv_cache_group_ids: list[int]
) -> Optional[list[KVCacheBlock]]:
"""Get the cached block by the block hash for each group in
`kv_cache_group_ids`, or None if cache miss for any group.
If there are duplicated blocks, we return the first block in the cache.
@@ -177,9 +183,11 @@ class BlockPool:
cached_blocks = []
for group_id in kv_cache_group_ids:
block_hash_with_group_id = make_block_hash_with_group_id(
block_hash, group_id)
block_hash, group_id
)
block = self.cached_block_hash_to_block.get_one_block(
block_hash_with_group_id)
block_hash_with_group_id
)
if not block:
return None
cached_blocks.append(block)
@@ -218,17 +226,18 @@ class BlockPool:
new_block_hashes = request.block_hashes[num_cached_blocks:]
new_hashes: Optional[list[ExternalBlockHash]] = (
[] if self.enable_kv_cache_events else None)
[] if self.enable_kv_cache_events else None
)
for i, blk in enumerate(new_full_blocks):
assert blk.block_hash is None
block_hash = new_block_hashes[i]
# Update and added the full block to the cache.
block_hash_with_group_id = make_block_hash_with_group_id(
block_hash, kv_cache_group_id)
block_hash, kv_cache_group_id
)
blk.block_hash = block_hash_with_group_id
self.cached_block_hash_to_block.insert(block_hash_with_group_id,
blk)
self.cached_block_hash_to_block.insert(block_hash_with_group_id, blk)
if new_hashes is not None:
new_hashes.append(maybe_convert_block_hash(block_hash))
@@ -239,20 +248,21 @@ class BlockPool:
parent_block = blocks[num_cached_blocks - 1]
assert parent_block.block_hash is not None
parent_block_hash = maybe_convert_block_hash(
get_block_hash(parent_block.block_hash))
get_block_hash(parent_block.block_hash)
)
self.kv_event_queue.append(
BlockStored(
block_hashes=new_hashes,
parent_block_hash=parent_block_hash,
token_ids=request.
all_token_ids[num_cached_blocks *
block_size:num_full_blocks * block_size],
token_ids=request.all_token_ids[
num_cached_blocks * block_size : num_full_blocks * block_size
],
block_size=block_size,
lora_id=request.lora_request.id
if request.lora_request else None,
lora_id=request.lora_request.id if request.lora_request else None,
medium=MEDIUM_GPU,
))
)
)
def get_new_blocks(self, num_blocks: int) -> list[KVCacheBlock]:
"""Get new blocks from the free block pool.
@@ -266,8 +276,7 @@ class BlockPool:
A list of new block.
"""
if num_blocks > self.get_num_free_blocks():
raise ValueError(
f"Cannot get {num_blocks} free blocks from the pool")
raise ValueError(f"Cannot get {num_blocks} free blocks from the pool")
ret: list[KVCacheBlock] = self.free_block_queue.popleft_n(num_blocks)
@@ -299,8 +308,7 @@ class BlockPool:
# The block doesn't have hash, eviction is not needed
return False
if self.cached_block_hash_to_block.pop(block_hash,
block.block_id) is None:
if self.cached_block_hash_to_block.pop(block_hash, block.block_id) is None:
# block not found in cached_block_hash_to_block,
# eviction is not needed
return False
@@ -313,10 +321,11 @@ class BlockPool:
# we disable hybrid kv cache manager when kv cache event is
# enabled, so there is only one group.
self.kv_event_queue.append(
BlockRemoved(block_hashes=[
maybe_convert_block_hash(get_block_hash(block_hash))
],
medium=MEDIUM_GPU))
BlockRemoved(
block_hashes=[maybe_convert_block_hash(get_block_hash(block_hash))],
medium=MEDIUM_GPU,
)
)
return True
def touch(self, blocks: tuple[list[KVCacheBlock], ...]) -> None:
@@ -347,10 +356,9 @@ class BlockPool:
blocks_list = list(ordered_blocks)
for block in blocks_list:
block.ref_cnt -= 1
self.free_block_queue.append_n([
block for block in blocks_list
if block.ref_cnt == 0 and not block.is_null
])
self.free_block_queue.append_n(
[block for block in blocks_list if block.ref_cnt == 0 and not block.is_null]
)
def reset_prefix_cache(self) -> bool:
"""Reset prefix cache. This function may be used in RLHF
@@ -365,7 +373,9 @@ class BlockPool:
if num_used_blocks != 1: # The null block is always marked as used
logger.warning(
"Failed to reset prefix cache because some "
"blocks (%d) are not freed yet", num_used_blocks - 1)
"blocks (%d) are not freed yet",
num_used_blocks - 1,
)
return False
# Remove all hashes so that no new blocks will hit.
@@ -405,7 +415,7 @@ class BlockPool:
def take_events(self) -> list[KVCacheEvent]:
"""Atomically takes all events and clears the queue.
Returns:
A list of KV cache events.
"""
+47 -37
View File
@@ -33,12 +33,12 @@ class EncoderCacheManager:
within requests, allowing for fine-grained memory management and enabling
chunked processing of multimodal inputs.
Cache is enabled to share embeddings of same multimodal data
item (identified by their hash value) between different requests,
and eviction takes place at allocation time when there's no free
Cache is enabled to share embeddings of same multimodal data
item (identified by their hash value) between different requests,
and eviction takes place at allocation time when there's no free
space for new embeddings.
Oldest cached embeddings with no request referenced will be first evicted.
Args:
cache_size: Limit the size of the cache, measured by the number of
tokens from the input sequence.
@@ -99,27 +99,31 @@ class EncoderCacheManager:
self.cached[mm_hash].add(request.request_id)
return True
def can_allocate(self, request: Request, input_id: int,
encoder_compute_budget: int,
num_tokens_to_schedule: int) -> bool:
"""Check if there's sufficient cache space for a multimodal input.
def can_allocate(
self,
request: Request,
input_id: int,
encoder_compute_budget: int,
num_tokens_to_schedule: int,
) -> bool:
"""Check if there's sufficient cache space for a multimodal input.
If there is, return True and update EncoderCacheManager state.
If there is not enough free space in `num_free_slots` but there is
enough reclaimable space in `num_freeable_slots`, entries will be
evicted from `freeable` (their mm_hash appended to `freed`) until
enough space is available, and then this method returns True.
enough space is available, and then this method returns True.
Older entries are evicted first.
Returns False only if the requested number of tokens exceeds both
Returns False only if the requested number of tokens exceeds both
the free and reclaimable capacities combined.
Args:
request: The request containing the multimodal input.
input_id: Index of the multimodal input within the request.
encoder_compute_budget: Number of encoder tokens allowed to be
encoder_compute_budget: Number of encoder tokens allowed to be
computed when this method is invoked.
num_tokens_to_schedule: Number of tokens already scheduled to be
num_tokens_to_schedule: Number of tokens already scheduled to be
allocated with cache space when this method is invoked.
Returns:
@@ -127,7 +131,7 @@ class EncoderCacheManager:
input (possibly after reclaiming `freeable` entries); otherwise
False.
Note: This method does not allocate physical memory for the encoder
Note: This method does not allocate physical memory for the encoder
output but only the state of EncoderCacheManager.
"""
num_tokens = request.get_num_encoder_tokens(input_id)
@@ -202,7 +206,7 @@ class EncoderCacheManager:
When the reference set for the corresponding `mm_hash` becomes empty,
the entry is appended to `freeable` and `num_freeable_slots` is
increased by the number of encoder tokens for that input.
increased by the number of encoder tokens for that input.
The entry is NOT physically freed until capacity is needed (e.g., by
`can_allocate`).
@@ -221,8 +225,8 @@ class EncoderCacheManager:
def free(self, request: Request) -> None:
"""Free all encoder input cache reference held by *request*.
For each cached input ID, `free_encoder_input` is invoked.
The data stays in memory until eviction is triggered by a future
For each cached input ID, `free_encoder_input` is invoked.
The data stays in memory until eviction is triggered by a future
attempt allocation called by 'can_allocate'.
Typically called when a request is finished, cancelled, or aborted.
@@ -236,9 +240,9 @@ class EncoderCacheManager:
Returns:
List of mm_hash strings that were actually evicted since the last
call to be used by the scheduler to notify workers about which
encoder outputs can be removed from their caches. The internal
list is cleared after this call.
call to be used by the scheduler to notify workers about which
encoder outputs can be removed from their caches. The internal
list is cleared after this call.
"""
freed = self.freed
self.freed = []
@@ -250,7 +254,7 @@ def compute_encoder_budget(
scheduler_config: "SchedulerConfig",
mm_registry: MultiModalRegistry,
) -> tuple[int, int]:
"""Compute the encoder cache budget based on the model and scheduler
"""Compute the encoder cache budget based on the model and scheduler
configurations.
Returns:
@@ -260,8 +264,9 @@ def compute_encoder_budget(
from the input sequence.
"""
if mm_registry.supports_multimodal_inputs(model_config):
max_tokens_by_modality = mm_registry \
.get_max_tokens_per_item_by_nonzero_modality(model_config)
max_tokens_by_modality = (
mm_registry.get_max_tokens_per_item_by_nonzero_modality(model_config)
)
return compute_mm_encoder_budget(
scheduler_config,
@@ -271,18 +276,17 @@ def compute_encoder_budget(
return compute_text_encoder_budget(scheduler_config)
def compute_text_encoder_budget(
scheduler_config: "SchedulerConfig") -> tuple[int, int]:
"""Compute the encoder cache budget based on the model and scheduler
def compute_text_encoder_budget(scheduler_config: "SchedulerConfig") -> tuple[int, int]:
"""Compute the encoder cache budget based on the model and scheduler
configurations for a text-only model.
Args:
scheduler_config: Scheduler configuration.
Returns:
- Compute budget for encoder execution, in unit of number of tokens
- Compute budget for encoder execution, in unit of number of tokens
in the input sequence.
- Space budget for encoder cache size, in unit of number of tokens
- Space budget for encoder cache size, in unit of number of tokens
in the input sequence.
"""
# Currently text-only encoder-decoder models are not supported
@@ -293,7 +297,7 @@ def compute_mm_encoder_budget(
scheduler_config: "SchedulerConfig",
max_tokens_by_modality: Mapping[str, int],
) -> tuple[int, int]:
"""Compute the encoder cache budget based on the model and scheduler
"""Compute the encoder cache budget based on the model and scheduler
configurations for a multimodal model.
Args:
@@ -312,22 +316,28 @@ def compute_mm_encoder_budget(
logger.warning(
"All non-text modalities supported by the model have been "
"explicitly disabled via limit_mm_per_prompt. Encoder cache will "
"not be initialized.")
"not be initialized."
)
return 0, 0
max_tokens_per_mm_item = max(max_tokens_by_modality.values())
if (scheduler_config.disable_chunked_mm_input and max_tokens_per_mm_item
> scheduler_config.max_num_batched_tokens):
if (
scheduler_config.disable_chunked_mm_input
and max_tokens_per_mm_item > scheduler_config.max_num_batched_tokens
):
raise ValueError(
"Chunked MM input disabled but max_tokens_per_mm_item "
f"({max_tokens_per_mm_item}) is larger than max_num_batched_tokens"
f" ({scheduler_config.max_num_batched_tokens}). Please increase "
"max_num_batched_tokens.")
"max_num_batched_tokens."
)
encoder_compute_budget = max(scheduler_config.max_num_encoder_input_tokens,
max_tokens_per_mm_item)
encoder_cache_size = max(scheduler_config.encoder_cache_size,
max_tokens_per_mm_item)
encoder_compute_budget = max(
scheduler_config.max_num_encoder_input_tokens, max_tokens_per_mm_item
)
encoder_cache_size = max(
scheduler_config.encoder_cache_size, max_tokens_per_mm_item
)
return encoder_compute_budget, encoder_cache_size
+168 -121
View File
@@ -6,9 +6,11 @@ from typing import Optional
from vllm.v1.core.block_pool import BlockPool
from vllm.v1.core.kv_cache_utils import BlockHash, KVCacheBlock
from vllm.v1.core.single_type_kv_cache_manager import (
CrossAttentionManager, FullAttentionManager, get_manager_for_kv_cache_spec)
from vllm.v1.kv_cache_interface import (FullAttentionSpec, KVCacheConfig,
KVCacheSpec)
CrossAttentionManager,
FullAttentionManager,
get_manager_for_kv_cache_spec,
)
from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheConfig, KVCacheSpec
from vllm.v1.request import Request
@@ -30,8 +32,9 @@ class KVCacheCoordinator(ABC):
self.max_model_len = max_model_len
self.enable_caching = enable_caching
self.block_pool = BlockPool(kv_cache_config.num_blocks, enable_caching,
enable_kv_cache_events)
self.block_pool = BlockPool(
kv_cache_config.num_blocks, enable_caching, enable_kv_cache_events
)
# Needs special handling for find_longest_cache_hit if eagle is enabled
self.use_eagle = use_eagle
@@ -41,19 +44,23 @@ class KVCacheCoordinator(ABC):
block_pool=self.block_pool,
kv_cache_group_id=i,
dcp_world_size=dcp_world_size,
) for i, kv_cache_group in enumerate(
self.kv_cache_config.kv_cache_groups))
)
for i, kv_cache_group in enumerate(self.kv_cache_config.kv_cache_groups)
)
def get_num_blocks_to_allocate(self, request_id: str, num_tokens: int,
new_computed_blocks: tuple[
list[KVCacheBlock], ...],
num_encoder_tokens: int) -> int:
def get_num_blocks_to_allocate(
self,
request_id: str,
num_tokens: int,
new_computed_blocks: tuple[list[KVCacheBlock], ...],
num_encoder_tokens: int,
) -> int:
"""
Get the number of blocks needed to be allocated for the request.
Args:
request_id: The request ID.
num_tokens: The total number of tokens that need a slot (including
num_tokens: The total number of tokens that need a slot (including
tokens that are already allocated).
new_computed_blocks: The new computed blocks just hitting the
prefix caching.
@@ -69,15 +76,17 @@ class KVCacheCoordinator(ABC):
# For cross-attention, we issue a single static allocation
# of blocks based on the number of encoder input tokens.
num_blocks_to_allocate += manager.get_num_blocks_to_allocate(
request_id, num_encoder_tokens, [])
request_id, num_encoder_tokens, []
)
else:
num_blocks_to_allocate += manager.get_num_blocks_to_allocate(
request_id, num_tokens, new_computed_blocks[i])
request_id, num_tokens, new_computed_blocks[i]
)
return num_blocks_to_allocate
def save_new_computed_blocks(
self, request_id: str,
new_computed_blocks: tuple[list[KVCacheBlock], ...]) -> None:
self, request_id: str, new_computed_blocks: tuple[list[KVCacheBlock], ...]
) -> None:
"""
Add the new computed blocks to the request.
@@ -87,21 +96,18 @@ class KVCacheCoordinator(ABC):
prefix cache.
"""
for i, manager in enumerate(self.single_type_managers):
manager.save_new_computed_blocks(request_id,
new_computed_blocks[i])
manager.save_new_computed_blocks(request_id, new_computed_blocks[i])
def allocate_new_blocks(
self,
request_id: str,
num_tokens: int,
num_encoder_tokens: int = 0) -> tuple[list[KVCacheBlock], ...]:
self, request_id: str, num_tokens: int, num_encoder_tokens: int = 0
) -> tuple[list[KVCacheBlock], ...]:
"""
Allocate new blocks for the request to give it at least `num_tokens`
Allocate new blocks for the request to give it at least `num_tokens`
token slots.
Args:
request_id: The request ID.
num_tokens: The total number of tokens that need a slot (including
num_tokens: The total number of tokens that need a slot (including
tokens that are already allocated).
num_encoder_tokens: The number of encoder tokens for allocating
blocks for cross-attention.
@@ -111,9 +117,13 @@ class KVCacheCoordinator(ABC):
"""
return tuple(
manager.allocate_new_blocks(
request_id, num_encoder_tokens if isinstance(
manager, CrossAttentionManager) else num_tokens)
for manager in self.single_type_managers)
request_id,
num_encoder_tokens
if isinstance(manager, CrossAttentionManager)
else num_tokens,
)
for manager in self.single_type_managers
)
def cache_blocks(self, request: Request, num_computed_tokens: int) -> None:
"""
@@ -138,8 +148,9 @@ class KVCacheCoordinator(ABC):
for manager in self.single_type_managers:
manager.free(request_id)
def get_num_common_prefix_blocks(self, request_id: str,
num_running_requests: int) -> list[int]:
def get_num_common_prefix_blocks(
self, request_id: str, num_running_requests: int
) -> list[int]:
"""
Get the number of common prefix blocks for all requests in the RUNNING
state for each kv cache group.
@@ -154,16 +165,14 @@ class KVCacheCoordinator(ABC):
the RUNNING state for each kv cache group.
"""
num_blocks_per_group = [
manager.get_num_common_prefix_blocks(request_id,
num_running_requests)
manager.get_num_common_prefix_blocks(request_id, num_running_requests)
for manager in self.single_type_managers
]
return num_blocks_per_group
def remove_skipped_blocks(self, request_id: str,
num_computed_tokens: int) -> None:
def remove_skipped_blocks(self, request_id: str, num_computed_tokens: int) -> None:
"""
Remove the blocks that are no longer needed from `blocks` and replace
Remove the blocks that are no longer needed from `blocks` and replace
the removed blocks with null_block.
Args:
@@ -179,7 +188,8 @@ class KVCacheCoordinator(ABC):
"""
return tuple(
manager.req_to_blocks.get(request_id) or []
for manager in self.single_type_managers)
for manager in self.single_type_managers
)
@abstractmethod
def find_longest_cache_hit(
@@ -198,19 +208,27 @@ class KVCacheCoordinatorNoPrefixCache(KVCacheCoordinator):
Does not implement any features related to prefix caching.
"""
def __init__(self, kv_cache_config: KVCacheConfig, max_model_len: int,
use_eagle: bool, enable_kv_cache_events: bool,
dcp_world_size: int):
super().__init__(kv_cache_config,
max_model_len,
use_eagle,
False,
enable_kv_cache_events,
dcp_world_size=dcp_world_size)
def __init__(
self,
kv_cache_config: KVCacheConfig,
max_model_len: int,
use_eagle: bool,
enable_kv_cache_events: bool,
dcp_world_size: int,
):
super().__init__(
kv_cache_config,
max_model_len,
use_eagle,
False,
enable_kv_cache_events,
dcp_world_size=dcp_world_size,
)
self.num_single_type_manager = len(self.single_type_managers)
def get_num_common_prefix_blocks(self, request_id: str,
num_running_requests: int) -> list[int]:
def get_num_common_prefix_blocks(
self, request_id: str, num_running_requests: int
) -> list[int]:
return [0] * self.num_single_type_manager
def find_longest_cache_hit(
@@ -219,7 +237,8 @@ class KVCacheCoordinatorNoPrefixCache(KVCacheCoordinator):
max_cache_hit_length: int,
) -> tuple[tuple[list[KVCacheBlock], ...], int]:
blocks: tuple[list[KVCacheBlock], ...] = tuple(
[] for _ in range(self.num_single_type_manager))
[] for _ in range(self.num_single_type_manager)
)
return blocks, 0
@@ -230,23 +249,31 @@ class UnitaryKVCacheCoordinator(KVCacheCoordinator):
full attention or all attention layers use sliding window attention.
"""
def __init__(self, kv_cache_config: KVCacheConfig, max_model_len: int,
use_eagle: bool, enable_caching: bool,
enable_kv_cache_events: bool, dcp_world_size: int):
super().__init__(kv_cache_config,
max_model_len,
use_eagle,
enable_caching,
enable_kv_cache_events,
dcp_world_size=dcp_world_size)
self.kv_cache_spec = self.kv_cache_config.kv_cache_groups[
0].kv_cache_spec
def __init__(
self,
kv_cache_config: KVCacheConfig,
max_model_len: int,
use_eagle: bool,
enable_caching: bool,
enable_kv_cache_events: bool,
dcp_world_size: int,
):
super().__init__(
kv_cache_config,
max_model_len,
use_eagle,
enable_caching,
enable_kv_cache_events,
dcp_world_size=dcp_world_size,
)
self.kv_cache_spec = self.kv_cache_config.kv_cache_groups[0].kv_cache_spec
self.block_size = self.kv_cache_spec.block_size
self.dcp_world_size = dcp_world_size
if dcp_world_size > 1:
self.block_size *= dcp_world_size
assert len(self.kv_cache_config.kv_cache_groups) == 1, (
"UnitaryKVCacheCoordinator assumes only one kv cache group")
"UnitaryKVCacheCoordinator assumes only one kv cache group"
)
def find_longest_cache_hit(
self,
@@ -269,26 +296,34 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
"""
KV cache coordinator for hybrid models with multiple KV cache types, and
thus multiple kv cache groups.
To simplify `find_longest_cache_hit`, it only supports the combination of
To simplify `find_longest_cache_hit`, it only supports the combination of
two types of KV cache groups, and one of them must be full attention.
May extend to more general cases in the future.
"""
def __init__(self, kv_cache_config: KVCacheConfig, max_model_len: int,
use_eagle: bool, enable_caching: bool,
enable_kv_cache_events: bool, dcp_world_size: int):
super().__init__(kv_cache_config,
max_model_len,
use_eagle,
enable_caching,
enable_kv_cache_events,
dcp_world_size=dcp_world_size)
def __init__(
self,
kv_cache_config: KVCacheConfig,
max_model_len: int,
use_eagle: bool,
enable_caching: bool,
enable_kv_cache_events: bool,
dcp_world_size: int,
):
super().__init__(
kv_cache_config,
max_model_len,
use_eagle,
enable_caching,
enable_kv_cache_events,
dcp_world_size=dcp_world_size,
)
assert dcp_world_size == 1, "DCP not support hybrid attn now."
self.verify_and_split_kv_cache_groups()
def verify_and_split_kv_cache_groups(self) -> None:
"""
Verifies that the model has exactly two types of KV cache groups, and
Verifies that the model has exactly two types of KV cache groups, and
one of them is full attention. Then, split the kv cache groups into full
attention groups and other groups.
"""
@@ -303,7 +338,8 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
else:
assert full_attention_spec == g.kv_cache_spec, (
"HybridKVCacheCoordinator assumes exactly one type of "
"full attention groups now.")
"full attention groups now."
)
self.full_attention_group_ids.append(i)
else:
if other_spec is None:
@@ -311,19 +347,22 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
else:
assert other_spec == g.kv_cache_spec, (
"HybridKVCacheCoordinator assumes "
"exactly one other type of groups now.")
"exactly one other type of groups now."
)
self.other_group_ids.append(i)
assert full_attention_spec is not None, (
"HybridKVCacheCoordinator assumes exactly one type of full "
"attention groups now.")
"attention groups now."
)
assert other_spec is not None, (
"HybridKVCacheCoordinator assumes exactly one type of other "
"groups now.")
"HybridKVCacheCoordinator assumes exactly one type of other groups now."
)
self.full_attention_manager_cls = FullAttentionManager
self.other_attention_cls = self.single_type_managers[
self.other_group_ids[0]].__class__
self.other_group_ids[0]
].__class__
self.full_attention_spec = full_attention_spec
self.other_spec = other_spec
self.full_attention_block_size = self.full_attention_spec.block_size
@@ -334,7 +373,8 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
divisible = self.other_block_size % self.full_attention_block_size
assert divisible == 0, (
"KVCacheCoordinator assumes the block_size of full "
"attention layers is divisible by other layers now.")
"attention layers is divisible by other layers now."
)
if max(self.full_attention_group_ids) < min(self.other_group_ids):
self.full_attn_first = True
@@ -347,7 +387,8 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
"do not interleave, either full attention group ids "
"are before other attention group ids or vice versa."
"This is for simplifying merging hit_blocks_full_attn and "
"hit_blocks_other_attn to hit_blocks.")
"hit_blocks_other_attn to hit_blocks."
)
def find_longest_cache_hit(
self,
@@ -367,29 +408,26 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
- The number of tokens of the longest cache hit.
"""
# First, find the longest cache hit for full attention.
hit_blocks_full_attn = (
self.full_attention_manager_cls.find_longest_cache_hit(
block_hashes=block_hashes,
max_length=max_cache_hit_length,
kv_cache_group_ids=self.full_attention_group_ids,
block_pool=self.block_pool,
kv_cache_spec=self.full_attention_spec,
use_eagle=self.use_eagle,
))
hit_length = len(
hit_blocks_full_attn[0]) * self.full_attention_block_size
hit_blocks_full_attn = self.full_attention_manager_cls.find_longest_cache_hit(
block_hashes=block_hashes,
max_length=max_cache_hit_length,
kv_cache_group_ids=self.full_attention_group_ids,
block_pool=self.block_pool,
kv_cache_spec=self.full_attention_spec,
use_eagle=self.use_eagle,
)
hit_length = len(hit_blocks_full_attn[0]) * self.full_attention_block_size
# Next, find the cache hit for the other attention WITHIN
# the cache hit of full attention.
hit_blocks_other_attn = (
self.other_attention_cls.find_longest_cache_hit(
block_hashes=block_hashes,
max_length=hit_length,
kv_cache_group_ids=self.other_group_ids,
block_pool=self.block_pool,
kv_cache_spec=self.other_spec,
use_eagle=self.use_eagle,
))
hit_blocks_other_attn = self.other_attention_cls.find_longest_cache_hit(
block_hashes=block_hashes,
max_length=hit_length,
kv_cache_group_ids=self.other_group_ids,
block_pool=self.block_pool,
kv_cache_spec=self.other_spec,
use_eagle=self.use_eagle,
)
hit_length = len(hit_blocks_other_attn[0]) * self.other_block_size
# NOTE: the prefix cache hit length must be a multiple of block_size as
@@ -404,7 +442,7 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
# Truncate the full attention cache hit to the length of the
# cache hit of the other attention.
for group_hit_blocks in hit_blocks_full_attn:
del group_hit_blocks[hit_length // self.full_attention_block_size:]
del group_hit_blocks[hit_length // self.full_attention_block_size :]
# Merge the hit blocks of full attention and other attention.
if self.full_attn_first:
@@ -414,27 +452,36 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
return hit_blocks, hit_length
def get_kv_cache_coordinator(kv_cache_config: KVCacheConfig,
max_model_len: int, use_eagle: bool,
enable_caching: bool,
enable_kv_cache_events: bool,
dcp_world_size: int) -> KVCacheCoordinator:
def get_kv_cache_coordinator(
kv_cache_config: KVCacheConfig,
max_model_len: int,
use_eagle: bool,
enable_caching: bool,
enable_kv_cache_events: bool,
dcp_world_size: int,
) -> KVCacheCoordinator:
if not enable_caching:
return KVCacheCoordinatorNoPrefixCache(kv_cache_config,
max_model_len,
use_eagle,
enable_kv_cache_events,
dcp_world_size=dcp_world_size)
return KVCacheCoordinatorNoPrefixCache(
kv_cache_config,
max_model_len,
use_eagle,
enable_kv_cache_events,
dcp_world_size=dcp_world_size,
)
if len(kv_cache_config.kv_cache_groups) == 1:
return UnitaryKVCacheCoordinator(kv_cache_config,
max_model_len,
use_eagle,
enable_caching,
enable_kv_cache_events,
dcp_world_size=dcp_world_size)
return HybridKVCacheCoordinator(kv_cache_config,
max_model_len,
use_eagle,
enable_caching,
enable_kv_cache_events,
dcp_world_size=dcp_world_size)
return UnitaryKVCacheCoordinator(
kv_cache_config,
max_model_len,
use_eagle,
enable_caching,
enable_kv_cache_events,
dcp_world_size=dcp_world_size,
)
return HybridKVCacheCoordinator(
kv_cache_config,
max_model_len,
use_eagle,
enable_caching,
enable_kv_cache_events,
dcp_world_size=dcp_world_size,
)
+48 -41
View File
@@ -22,6 +22,7 @@ class KVCacheBlocks:
Scheduler and KVCacheManager, to hide KVCacheManager's internal data
structure from the Scheduler.
"""
blocks: tuple[list[KVCacheBlock], ...]
"""
`blocks[i][j]` refers to the i-th kv_cache_group
@@ -35,22 +36,20 @@ class KVCacheBlocks:
def __add__(self, other: "KVCacheBlocks") -> "KVCacheBlocks":
"""Adds two KVCacheBlocks instances."""
return KVCacheBlocks(
tuple(blk1 + blk2
for blk1, blk2 in zip(self.blocks, other.blocks)))
tuple(blk1 + blk2 for blk1, blk2 in zip(self.blocks, other.blocks))
)
@overload
def get_block_ids(
self,
allow_none: Literal[False] = False,
) -> tuple[list[int], ...]:
...
) -> tuple[list[int], ...]: ...
@overload
def get_block_ids(
self,
allow_none: Literal[True] = True,
) -> Optional[tuple[list[int], ...]]:
...
) -> Optional[tuple[list[int], ...]]: ...
def get_block_ids(
self,
@@ -72,10 +71,7 @@ class KVCacheBlocks:
def get_unhashed_block_ids(self) -> list[int]:
"""Get block_ids of unhashed blocks from KVCacheBlocks instance."""
assert len(self.blocks) == 1, "Only one group is supported"
return [
block.block_id for block in self.blocks[0]
if block.block_hash is None
]
return [block.block_id for block in self.blocks[0] if block.block_hash is None]
def new_empty(self) -> "KVCacheBlocks":
"""Creates a new KVCacheBlocks instance with no blocks."""
@@ -83,7 +79,6 @@ class KVCacheBlocks:
class KVCacheManager:
def __init__(
self,
kv_cache_config: KVCacheConfig,
@@ -104,12 +99,18 @@ class KVCacheManager:
self.block_size: Optional[int] = None
if self.enable_caching:
assert len(
set(g.kv_cache_spec.block_size
for g in kv_cache_config.kv_cache_groups)
) == 1, "Only one block size is supported for now"
assert (
len(
set(
g.kv_cache_spec.block_size
for g in kv_cache_config.kv_cache_groups
)
)
== 1
), "Only one block size is supported for now"
self.block_size = kv_cache_config.kv_cache_groups[
0].kv_cache_spec.block_size
0
].kv_cache_spec.block_size
if dcp_world_size > 1:
assert len(kv_cache_config.kv_cache_groups) == 1
@@ -151,8 +152,7 @@ class KVCacheManager:
self.prefix_cache_stats = PrefixCacheStats()
return stats
def get_computed_blocks(self,
request: Request) -> tuple[KVCacheBlocks, int]:
def get_computed_blocks(self, request: Request) -> tuple[KVCacheBlocks, int]:
"""Get the computed (cached) blocks for the request.
Note that the computed blocks must be full.
@@ -166,9 +166,10 @@ class KVCacheManager:
"""
# Prefix caching is disabled or
# When the request requires prompt logprobs, we skip prefix caching.
if (not self.enable_caching
or (request.sampling_params is not None
and request.sampling_params.prompt_logprobs is not None)):
if not self.enable_caching or (
request.sampling_params is not None
and request.sampling_params.prompt_logprobs is not None
):
return self.create_empty_block_list(), 0
# NOTE: When all tokens hit the cache, we must recompute the last token
@@ -179,8 +180,10 @@ class KVCacheManager:
# could slightly improve performance in the future.
max_cache_hit_length = request.num_tokens - 1
computed_blocks, num_new_computed_tokens = (
self.coordinator.find_longest_cache_hit(request.block_hashes,
max_cache_hit_length))
self.coordinator.find_longest_cache_hit(
request.block_hashes, max_cache_hit_length
)
)
if self.log_stats:
assert self.prefix_cache_stats is not None
@@ -188,8 +191,7 @@ class KVCacheManager:
# Previously preempted request
self.prefix_cache_stats.preempted_requests += 1
self.prefix_cache_stats.preempted_queries += request.num_tokens
self.prefix_cache_stats.preempted_hits += (
num_new_computed_tokens)
self.prefix_cache_stats.preempted_hits += num_new_computed_tokens
else:
# New request
self.prefix_cache_stats.requests += 1
@@ -250,7 +252,8 @@ class KVCacheManager:
new_computed_block_list = new_computed_blocks.blocks
else:
new_computed_block_list = tuple(
[] for _ in range(len(self.kv_cache_config.kv_cache_groups)))
[] for _ in range(len(self.kv_cache_config.kv_cache_groups))
)
# Free the blocks that are skipped during the attention computation
# (e.g., tokens outside the sliding window).
@@ -258,16 +261,17 @@ class KVCacheManager:
# insufficient free blocks.
# Should call this function before allocating new blocks to reduce
# the number of evicted blocks.
self.coordinator.remove_skipped_blocks(request.request_id,
request.num_computed_tokens)
self.coordinator.remove_skipped_blocks(
request.request_id, request.num_computed_tokens
)
# The number of computed tokens is the number of computed tokens plus
# the new prefix caching hits
num_computed_tokens = (request.num_computed_tokens +
num_new_computed_tokens)
num_computed_tokens = request.num_computed_tokens + num_new_computed_tokens
num_tokens_need_slot = min(
num_computed_tokens + num_new_tokens + num_lookahead_tokens,
self.max_model_len)
self.max_model_len,
)
num_blocks_to_allocate = self.coordinator.get_num_blocks_to_allocate(
request_id=request.request_id,
@@ -285,16 +289,18 @@ class KVCacheManager:
self.block_pool.touch(new_computed_block_list)
else:
assert not any(new_computed_block_list), (
"Computed blocks should be empty when "
"prefix caching is disabled")
"Computed blocks should be empty when prefix caching is disabled"
)
# Append the new computed blocks to the request blocks until now to
# avoid the case where the new blocks cannot be allocated.
self.coordinator.save_new_computed_blocks(request.request_id,
new_computed_block_list)
self.coordinator.save_new_computed_blocks(
request.request_id, new_computed_block_list
)
new_blocks = self.coordinator.allocate_new_blocks(
request.request_id, num_tokens_need_slot, num_encoder_tokens)
request.request_id, num_tokens_need_slot, num_encoder_tokens
)
# P/D: delay caching blocks if we have to recv from
# remote. Update state for locally cached blocks.
@@ -305,8 +311,9 @@ class KVCacheManager:
# num_new_tokens, but must exclude "non-committable" tokens (e.g.,
# draft tokens that could be rejected). Therefore, we cap the number
# at `request.num_tokens`, ensuring only "finalized" tokens are cached.
num_tokens_to_cache = min(num_computed_tokens + num_new_tokens,
request.num_tokens)
num_tokens_to_cache = min(
num_computed_tokens + num_new_tokens, request.num_tokens
)
self.coordinator.cache_blocks(request, num_tokens_to_cache)
return KVCacheBlocks(new_blocks)
@@ -378,7 +385,8 @@ class KVCacheManager:
"""
assert request.status == RequestStatus.RUNNING
return self.coordinator.get_num_common_prefix_blocks(
request.request_id, num_running_requests)
request.request_id, num_running_requests
)
def take_events(self) -> list[KVCacheEvent]:
"""Take the KV cache events from the block pool.
@@ -403,5 +411,4 @@ class KVCacheManager:
def create_empty_block_list(self) -> KVCacheBlocks:
"""Creates a new KVCacheBlocks instance with no blocks."""
return KVCacheBlocks(tuple([]
for _ in range(self.num_kv_cache_groups)))
return KVCacheBlocks(tuple([] for _ in range(self.num_kv_cache_groups)))
+260 -198
View File
@@ -13,11 +13,16 @@ from vllm import envs
from vllm.config import VllmConfig
from vllm.logger import init_logger
from vllm.utils import GiB_bytes, cdiv, sha256_cbor
from vllm.v1.kv_cache_interface import (ChunkedLocalAttentionSpec,
FullAttentionSpec, KVCacheConfig,
KVCacheGroupSpec, KVCacheSpec,
KVCacheTensor, SlidingWindowSpec,
UniformTypeKVCacheSpecs)
from vllm.v1.kv_cache_interface import (
ChunkedLocalAttentionSpec,
FullAttentionSpec,
KVCacheConfig,
KVCacheGroupSpec,
KVCacheSpec,
KVCacheTensor,
SlidingWindowSpec,
UniformTypeKVCacheSpecs,
)
from vllm.v1.metrics.stats import PrefixCacheStats
from vllm.v1.request import Request
@@ -37,16 +42,16 @@ BlockHashWithGroupId = NewType("BlockHashWithGroupId", bytes)
ExternalBlockHash = Union[bytes, int]
def make_block_hash_with_group_id(block_hash: BlockHash,
group_id: int) -> BlockHashWithGroupId:
def make_block_hash_with_group_id(
block_hash: BlockHash, group_id: int
) -> BlockHashWithGroupId:
"""Pack a ``BlockHash`` and group id into a ``BlockHashWithGroupId``.
The group id is encoded using 4 bytes in big-endian order and appended to
the block hash bytes. This representation avoids creating tuples while
still allowing us to recover both components when needed.
"""
return BlockHashWithGroupId(block_hash +
group_id.to_bytes(4, "big", signed=False))
return BlockHashWithGroupId(block_hash + group_id.to_bytes(4, "big", signed=False))
def get_block_hash(key: BlockHashWithGroupId) -> BlockHash:
@@ -87,7 +92,8 @@ def init_none_hash(hash_fn: Callable[[Any], bytes]):
"PYTHONHASHSEED is not set. This will lead to non-reproducible "
"block-hashes when using sha256_cbor as the hash function."
"Consider setting PYTHONHASHSEED to a fixed value for "
"reproducibility.")
"reproducibility."
)
if hash_seed is None:
NONE_HASH = BlockHash(os.urandom(32))
@@ -143,9 +149,10 @@ class PrefixCachingMetrics:
# Remove the oldest stats until number of requests does not exceed
# the limit.
# NOTE: We preserve the latest added stats regardless.
while len(
self.query_queue
) > 1 and self.aggregated_requests > self.max_recent_requests:
while (
len(self.query_queue) > 1
and self.aggregated_requests > self.max_recent_requests
):
old_requests, old_queries, old_hits = self.query_queue.popleft()
self.aggregated_requests -= old_requests
self.aggregated_query_total -= old_queries
@@ -169,6 +176,7 @@ class PrefixCachingMetrics:
@dataclass
class KVCacheBlock:
"""KV-cache block metadata."""
# Block ID, ranging from 0 to num_gpu_blocks - 1.
block_id: int
# Reference count.
@@ -192,7 +200,8 @@ class KVCacheBlock:
@block_hash.setter
def block_hash(self, block_hash: BlockHashWithGroupId):
assert self.block_hash is None, (
"The block already has a hash. This should not happen.")
"The block already has a hash. This should not happen."
)
self._block_hash = block_hash
def reset_hash(self):
@@ -202,15 +211,15 @@ class KVCacheBlock:
def __repr__(self) -> str:
# Use block_id instead of KVCacheBlock object to avoid calling __repr__
# on KVCacheBlock object recursively.
prev_block_id = (self.prev_free_block.block_id
if self.prev_free_block else None)
next_block_id = (self.next_free_block.block_id
if self.next_free_block else None)
return (f"KVCacheBlock(block_id={self.block_id}, "
f"ref_cnt={self.ref_cnt}, "
f"_block_hash={self._block_hash!r}, "
f"prev_free_block={prev_block_id}, "
f"next_free_block={next_block_id})")
prev_block_id = self.prev_free_block.block_id if self.prev_free_block else None
next_block_id = self.next_free_block.block_id if self.next_free_block else None
return (
f"KVCacheBlock(block_id={self.block_id}, "
f"ref_cnt={self.ref_cnt}, "
f"_block_hash={self._block_hash!r}, "
f"prev_free_block={prev_block_id}, "
f"next_free_block={next_block_id})"
)
class FreeKVCacheBlockQueue:
@@ -271,12 +280,14 @@ class FreeKVCacheBlockQueue:
Returns:
The first free block.
"""
if (self.fake_free_list_head.next_free_block
is self.fake_free_list_tail
or self.fake_free_list_head.next_free_block is None):
if (
self.fake_free_list_head.next_free_block is self.fake_free_list_tail
or self.fake_free_list_head.next_free_block is None
):
assert self.num_free_blocks == 0, (
f"num_free_blocks ({self.num_free_blocks}) is out of sync "
"with the free list.")
"with the free list."
)
raise ValueError("No free blocks available")
first_block: KVCacheBlock = self.fake_free_list_head.next_free_block
@@ -284,8 +295,10 @@ class FreeKVCacheBlockQueue:
if first_block.next_free_block is None:
# This should not happen if the block is from the free list.
# It indicates a bug in the caller's logic.
raise RuntimeError("Invalid block found in popleft() "
"which doesn't have a valid next_free_block")
raise RuntimeError(
"Invalid block found in popleft() "
"which doesn't have a valid next_free_block"
)
# Connect fake_head and the next block of first_block (i.e. second block
# or fake tail).
@@ -360,7 +373,8 @@ class FreeKVCacheBlockQueue:
"""
if self.fake_free_list_tail.prev_free_block is None:
raise RuntimeError(
"prev_free_block of fake_free_list_tail should always exist")
"prev_free_block of fake_free_list_tail should always exist"
)
last_block: KVCacheBlock = self.fake_free_list_tail.prev_free_block
# Connect the new block after the last block.
@@ -384,7 +398,8 @@ class FreeKVCacheBlockQueue:
last_block = self.fake_free_list_tail.prev_free_block
assert last_block is not None, (
"prev_free_block of fake_free_list_tail should always exist")
"prev_free_block of fake_free_list_tail should always exist"
)
# Add inter-connections between consecutive blocks
for block in blocks:
block.prev_free_block = last_block
@@ -406,7 +421,8 @@ class FreeKVCacheBlockQueue:
ret = []
if self.fake_free_list_head.next_free_block is None:
raise RuntimeError(
"next_free_block of fake_free_list_head should always exist")
"next_free_block of fake_free_list_head should always exist"
)
# Start from the first block
curr_block: KVCacheBlock = self.fake_free_list_head.next_free_block
# As long as next_free_block is available, we haven't reached to
@@ -430,14 +446,16 @@ def need_extra_keys(request: Request) -> bool:
# Multimodal requests need to include the MM hash.
# LoRA requests need to include the LoRA ID.
# Request with provided cache salt need to include the salt.
return bool(request.mm_features) or (request.lora_request
is not None) or (request.cache_salt
is not None)
return (
bool(request.mm_features)
or (request.lora_request is not None)
or (request.cache_salt is not None)
)
def _gen_mm_extra_hash_keys(request: Request, start_token_idx: int,
end_token_idx: int,
start_mm_idx: int) -> tuple[list[Any], int]:
def _gen_mm_extra_hash_keys(
request: Request, start_token_idx: int, end_token_idx: int, start_mm_idx: int
) -> tuple[list[Any], int]:
"""Generate extra keys related to MultiModal request for block hash
computation. For multi-modal inputs, the extra keys are
(mm_hash, start_offset) that indicate a mm input contained in the
@@ -515,8 +533,8 @@ def _gen_lora_extra_hash_keys(request: Request) -> list[int]:
def generate_block_hash_extra_keys(
request: Request, start_token_idx: int, end_token_idx: int,
start_mm_idx: int) -> tuple[Optional[tuple[Any, ...]], int]:
request: Request, start_token_idx: int, end_token_idx: int, start_mm_idx: int
) -> tuple[Optional[tuple[Any, ...]], int]:
"""Generate extra keys for the block hash. The extra keys can come from
the multi-modal inputs and request specific metadata (e.g., LoRA ID).
@@ -531,10 +549,12 @@ def generate_block_hash_extra_keys(
"""
mm_extra_keys: list[Any]
mm_extra_keys, new_start_mm_idx = _gen_mm_extra_hash_keys(
request, start_token_idx, end_token_idx, start_mm_idx)
request, start_token_idx, end_token_idx, start_mm_idx
)
lora_extra_keys: list[int] = _gen_lora_extra_hash_keys(request)
cache_salt_keys: list[str] = [request.cache_salt] if (
start_token_idx == 0 and request.cache_salt) else []
cache_salt_keys: list[str] = (
[request.cache_salt] if (start_token_idx == 0 and request.cache_salt) else []
)
extra_keys: list[Any] = lora_extra_keys + mm_extra_keys + cache_salt_keys
@@ -545,10 +565,11 @@ def generate_block_hash_extra_keys(
def hash_block_tokens(
hash_function: Callable[[Any], bytes],
parent_block_hash: Optional[BlockHash],
curr_block_token_ids: Sequence[int],
extra_keys: Optional[tuple[Any, ...]] = None) -> BlockHash:
hash_function: Callable[[Any], bytes],
parent_block_hash: Optional[BlockHash],
curr_block_token_ids: Sequence[int],
extra_keys: Optional[tuple[Any, ...]] = None,
) -> BlockHash:
"""Computes a hash value corresponding to the contents of a block and
the contents of the preceding block(s). The hash value is used for
prefix caching. We use LRU cache for this function to avoid recomputing
@@ -569,8 +590,8 @@ def hash_block_tokens(
curr_block_token_ids_tuple = tuple(curr_block_token_ids)
return BlockHash(
hash_function(
(parent_block_hash, curr_block_token_ids_tuple, extra_keys)))
hash_function((parent_block_hash, curr_block_token_ids_tuple, extra_keys))
)
def get_request_block_hasher(
@@ -597,8 +618,9 @@ def get_request_block_hasher(
# last mm input.
curr_mm_idx = -1
prev_block_hash_value = (request.block_hashes[-1]
if request.block_hashes else None)
prev_block_hash_value = (
request.block_hashes[-1] if request.block_hashes else None
)
new_block_hashes: list[BlockHash] = []
while True:
end_token_idx = start_token_idx + block_size
@@ -608,13 +630,14 @@ def get_request_block_hasher(
# MM and LoRA requests need extra keys for block-hash computation.
extra_keys, curr_mm_idx = generate_block_hash_extra_keys(
request, start_token_idx, end_token_idx, curr_mm_idx)
request, start_token_idx, end_token_idx, curr_mm_idx
)
# Compute the hash of the current block
block_tokens = request.all_token_ids[start_token_idx:end_token_idx]
block_hash = hash_block_tokens(caching_hash_fn,
prev_block_hash_value, block_tokens,
extra_keys)
block_hash = hash_block_tokens(
caching_hash_fn, prev_block_hash_value, block_tokens, extra_keys
)
new_block_hashes.append(block_hash)
start_token_idx += block_size
@@ -625,18 +648,20 @@ def get_request_block_hasher(
return request_block_hasher
def max_memory_usage_bytes(vllm_config: VllmConfig,
kv_cache_specs: Iterable[KVCacheSpec]) -> int:
def max_memory_usage_bytes(
vllm_config: VllmConfig, kv_cache_specs: Iterable[KVCacheSpec]
) -> int:
"""
Get the maximum memory usage in bytes for the given KV cache specs.
"""
return sum(
spec.max_memory_usage_bytes(vllm_config) for spec in kv_cache_specs)
return sum(spec.max_memory_usage_bytes(vllm_config) for spec in kv_cache_specs)
def estimate_max_model_len(vllm_config: VllmConfig,
kv_cache_spec: dict[str, KVCacheSpec],
available_memory: int) -> int:
def estimate_max_model_len(
vllm_config: VllmConfig,
kv_cache_spec: dict[str, KVCacheSpec],
available_memory: int,
) -> int:
"""
Estimates the maximum model length that can fit in the available memory
using binary search.
@@ -655,8 +680,7 @@ def estimate_max_model_len(vllm_config: VllmConfig,
# Modify the max_model_len for this calculation
vllm_config.model_config.max_model_len = model_len
# Calculate memory needed for the given model length
memory_needed = max_memory_usage_bytes(vllm_config,
kv_cache_spec.values())
memory_needed = max_memory_usage_bytes(vllm_config, kv_cache_spec.values())
return memory_needed <= available_memory
# Binary search for the maximum model length
@@ -679,9 +703,11 @@ def estimate_max_model_len(vllm_config: VllmConfig,
return result
def check_enough_kv_cache_memory(vllm_config: VllmConfig,
kv_cache_spec: dict[str, KVCacheSpec],
available_memory: int):
def check_enough_kv_cache_memory(
vllm_config: VllmConfig,
kv_cache_spec: dict[str, KVCacheSpec],
available_memory: int,
):
"""
Checks whether `available_memory` is enough for the KV cache to hold at
least one request with the model's max_model_len.
@@ -700,36 +726,41 @@ def check_enough_kv_cache_memory(vllm_config: VllmConfig,
return
if available_memory <= 0:
raise ValueError("No available memory for the cache blocks. "
"Try increasing `gpu_memory_utilization` when "
"initializing the engine.")
raise ValueError(
"No available memory for the cache blocks. "
"Try increasing `gpu_memory_utilization` when "
"initializing the engine."
)
max_model_len = vllm_config.model_config.max_model_len
needed_memory = max_memory_usage_bytes(vllm_config, kv_cache_spec.values())
if needed_memory > available_memory:
# Estimate the maximum model length that can fit in the available memory
estimated_max_len = estimate_max_model_len(vllm_config, kv_cache_spec,
available_memory)
estimated_max_len = estimate_max_model_len(
vllm_config, kv_cache_spec, available_memory
)
estimated_msg = ""
if estimated_max_len > 0:
estimated_msg = (
"Based on the available memory, "
f"the estimated maximum model length is {estimated_max_len}.")
f"the estimated maximum model length is {estimated_max_len}."
)
raise ValueError(
f"To serve at least one request with the models's max seq len "
f"({max_model_len}), ({needed_memory/GiB_bytes:.2f} GiB KV "
f"({max_model_len}), ({needed_memory / GiB_bytes:.2f} GiB KV "
f"cache is needed, which is larger than the available KV cache "
f"memory ({available_memory/GiB_bytes:.2f} GiB). "
f"memory ({available_memory / GiB_bytes:.2f} GiB). "
f"{estimated_msg} "
f"Try increasing `gpu_memory_utilization` or decreasing "
f"`max_model_len` when initializing the engine.")
f"`max_model_len` when initializing the engine."
)
def create_kv_cache_group_specs(
kv_cache_spec: dict[str, KVCacheSpec],
grouped_layer_names: list[list[str]]) -> list[KVCacheGroupSpec]:
kv_cache_spec: dict[str, KVCacheSpec], grouped_layer_names: list[list[str]]
) -> list[KVCacheGroupSpec]:
"""
Create KVCacheGroupSpec object for each kv cache group layer.
The layers in the same group should share the same
@@ -752,7 +783,8 @@ def create_kv_cache_group_specs(
]
merged_layer_spec = layer_specs[0].merge(layer_specs)
kv_cache_groups.append(
KVCacheGroupSpec(layer_names_one_group, merged_layer_spec))
KVCacheGroupSpec(layer_names_one_group, merged_layer_spec)
)
return kv_cache_groups
@@ -782,19 +814,22 @@ def is_kv_cache_spec_uniform(kv_cache_spec: dict[str, KVCacheSpec]) -> bool:
def get_max_concurrency_for_kv_cache_config(
vllm_config: VllmConfig, kv_cache_config: KVCacheConfig) -> float:
vllm_config: VllmConfig, kv_cache_config: KVCacheConfig
) -> float:
"""
Get the maximum concurrency for the given KV cache configuration.
"""
num_layer_per_group = max(
len(group.layer_names) for group in kv_cache_config.kv_cache_groups)
len(group.layer_names) for group in kv_cache_config.kv_cache_groups
)
max_memory_usage_per_request = num_layer_per_group * max_memory_usage_bytes(
vllm_config,
(group.kv_cache_spec for group in kv_cache_config.kv_cache_groups))
memory_per_block = kv_cache_config.kv_cache_groups[
0].kv_cache_spec.page_size_bytes * num_layer_per_group
num_block_per_request = cdiv(max_memory_usage_per_request,
memory_per_block)
vllm_config, (group.kv_cache_spec for group in kv_cache_config.kv_cache_groups)
)
memory_per_block = (
kv_cache_config.kv_cache_groups[0].kv_cache_spec.page_size_bytes
* num_layer_per_group
)
num_block_per_request = cdiv(max_memory_usage_per_request, memory_per_block)
max_concurrency = kv_cache_config.num_blocks / num_block_per_request
return max_concurrency
@@ -804,18 +839,20 @@ def may_override_num_blocks(vllm_config: VllmConfig, num_blocks: int) -> int:
Override the number of kv cache blocks if `num_gpu_blocks_override` is set.
"""
if vllm_config.cache_config.num_gpu_blocks_override is not None:
num_gpu_blocks_override = \
vllm_config.cache_config.num_gpu_blocks_override
num_gpu_blocks_override = vllm_config.cache_config.num_gpu_blocks_override
logger.info(
"Overriding num_gpu_blocks=%d with "
"num_gpu_blocks_override=%d", num_blocks, num_gpu_blocks_override)
"Overriding num_gpu_blocks=%d with num_gpu_blocks_override=%d",
num_blocks,
num_gpu_blocks_override,
)
num_blocks = num_gpu_blocks_override
return num_blocks
def get_num_blocks(vllm_config: VllmConfig, num_layers: int,
available_memory: int, page_size: int) -> int:
def get_num_blocks(
vllm_config: VllmConfig, num_layers: int, available_memory: int, page_size: int
) -> int:
"""
Get the number of kv cache blocks.
@@ -841,9 +878,10 @@ def get_uniform_page_size(kv_cache_spec: dict[str, KVCacheSpec]) -> int:
def _get_kv_cache_groups_uniform_spec(
kv_cache_specs: dict[str, KVCacheSpec]) -> list[KVCacheGroupSpec]:
kv_cache_specs: dict[str, KVCacheSpec],
) -> list[KVCacheGroupSpec]:
"""
Generates the KV cache configuration for a model with the same KV cache
Generates the KV cache configuration for a model with the same KV cache
spec for all layers.
Args:
@@ -853,12 +891,12 @@ def _get_kv_cache_groups_uniform_spec(
The generated KVCacheGroupSpecs
"""
return create_kv_cache_group_specs(kv_cache_specs,
[list(kv_cache_specs.keys())])
return create_kv_cache_group_specs(kv_cache_specs, [list(kv_cache_specs.keys())])
def _get_kv_cache_groups_uniform_type(
spec: UniformTypeKVCacheSpecs) -> list[KVCacheGroupSpec]:
spec: UniformTypeKVCacheSpecs,
) -> list[KVCacheGroupSpec]:
"""
Generates the KV cache configuration for a model with one type of KV cache
but different hidden sizes. All layers are merged into one group.
@@ -873,8 +911,7 @@ def _get_kv_cache_groups_uniform_type(
return [KVCacheGroupSpec(list(spec.kv_cache_specs.keys()), spec)]
def is_kv_cache_page_size_uniform(
kv_cache_spec: dict[str, KVCacheSpec]) -> bool:
def is_kv_cache_page_size_uniform(kv_cache_spec: dict[str, KVCacheSpec]) -> bool:
"""
Whether all layers in the given KVCacheSpec have the same page size.
Args:
@@ -888,70 +925,69 @@ def is_kv_cache_page_size_uniform(
return len(page_sizes) == 1
def is_kv_cache_type_attention_free(
kv_cache_spec: dict[str, KVCacheSpec]) -> bool:
def is_kv_cache_type_attention_free(kv_cache_spec: dict[str, KVCacheSpec]) -> bool:
# kv_cache_spec is an empty dict for attention free models
return not kv_cache_spec
def _get_kv_cache_groups_uniform_page_size(
kv_cache_spec: dict[str, KVCacheSpec]) -> list[KVCacheGroupSpec]:
kv_cache_spec: dict[str, KVCacheSpec],
) -> list[KVCacheGroupSpec]:
"""
Generates the KV cache groups for hybrid models with multiple
attention types but still with a uniform page size (physical memory per
Generates the KV cache groups for hybrid models with multiple
attention types but still with a uniform page size (physical memory per
block per layer) for all layers.
Detailed explanation about kv cache management of hybrid models:
The layers in the models are repeated with some patterns, e.g., a model
with 10 full attention layers and 20 sliding window attention layers can be
regarded as repeating the pattern (1 * full, 2 * sw) 10 times.
regarded as repeating the pattern (1 * full, 2 * sw) 10 times.
The KVCacheManager allocates different block tables for each of the 3 layers
in the pattern, and repeats each of them 10 times to generate the
in the pattern, and repeats each of them 10 times to generate the
block_table for the 30 layers in the model.
Therefore, we can group the layers in the model into 3 kv_cache_groups, each
of which contains 10 layers in the model.
The KVCacheManager allocates the block_table for each group based on its
kv_cache spec, and the model runner applies the block table to each layer
kv_cache spec, and the model runner applies the block table to each layer
in the group.
For example:
1. A model only uses full attention. The pattern is
(num_hidden_layers * full), so there is only one group and the block table
is shared by all layers. It is already handled by
1. A model only uses full attention. The pattern is
(num_hidden_layers * full), so there is only one group and the block table
is shared by all layers. It is already handled by
`_get_kv_cache_config_uniform_type`.
2. A model with 10 full attention layers and 20 sliding window
attention layers. There are 3 layers in the pattern (1 * full, 2 * sw), so
2. A model with 10 full attention layers and 20 sliding window
attention layers. There are 3 layers in the pattern (1 * full, 2 * sw), so
there are 3 kv_cache_groups, each of which represents 10 layers.
To simplify the implementation, we make the following assumptions:
1. Physical memory per block: Must be the same across all KV cache groups.
1. Physical memory per block: Must be the same across all KV cache groups.
Breaking this assumption is non-trivial due to memory fragmentation concerns
when allocating blocks of different sizes.
2. Tokens per block (block_size): Currently, we directly use
`CacheConfig.block_size` for all layers. It can be extended to vary by KV
cache group, but within each KV cache group, all layers must share the same
2. Tokens per block (block_size): Currently, we directly use
`CacheConfig.block_size` for all layers. It can be extended to vary by KV
cache group, but within each KV cache group, all layers must share the same
block size.
3. Physical memory per token per layer: This property is decided by model
config. Currently we only support models that have the same physical memory
per token per layer for all layers. Can be relaxed with a simple extension,
3. Physical memory per token per layer: This property is decided by model
config. Currently we only support models that have the same physical memory
per token per layer for all layers. Can be relaxed with a simple extension,
but still need to keep physical memory per block the same for all groups.
4. Number of layers per group: Currently assumed the same for all layers.
Can be relaxed with a simple extension, but still need to keep physical
4. Number of layers per group: Currently assumed the same for all layers.
Can be relaxed with a simple extension, but still need to keep physical
memory per block the same for all groups.
5. Attention type within groups: All layers in a group must share the same
attention type. One exception is that, when
`--disable-hybrid-kv-cache-manager` is true, the single group for full
attention layers may also include attention layers using sliding window or
attention type. One exception is that, when
`--disable-hybrid-kv-cache-manager` is true, the single group for full
attention layers may also include attention layers using sliding window or
LLaMA 4 local attention. See `unify_hybrid_kv_cache_specs` for more details.
6. Support for multiple attention types: The design for most components is
general to an arbitrary number of attention types. But
`find_longest_cache_hit` only supports one attention type or two
6. Support for multiple attention types: The design for most components is
general to an arbitrary number of attention types. But
`find_longest_cache_hit` only supports one attention type or two
types of full-attention plus exactly one another type. The general
implementation of this function is feasible but we don't know how to
implementation of this function is feasible but we don't know how to
implement it cleanly yet.
As we assume tokens per block, physical memory per token per layer, and
number of layers per group are the same now, we can ensure that physical
As we assume tokens per block, physical memory per token per layer, and
number of layers per group are the same now, we can ensure that physical
memory per block is the same for all groups.
Args:
@@ -1005,10 +1041,12 @@ def _get_kv_cache_groups_uniform_page_size(
return create_kv_cache_group_specs(kv_cache_spec, grouped_layers)
def get_kv_cache_config_from_groups(vllm_config: VllmConfig,
kv_cache_groups: list[KVCacheGroupSpec],
kv_cache_specs: dict[str, KVCacheSpec],
available_memory: int) -> KVCacheConfig:
def get_kv_cache_config_from_groups(
vllm_config: VllmConfig,
kv_cache_groups: list[KVCacheGroupSpec],
kv_cache_specs: dict[str, KVCacheSpec],
available_memory: int,
) -> KVCacheConfig:
"""
Generate the KV cache configuration from the KV cache groups and spec
of each layer.
@@ -1031,19 +1069,22 @@ def get_kv_cache_config_from_groups(vllm_config: VllmConfig,
)
# Determine how model runners should initialize the KV cache tensors.
if len(kv_cache_groups) == 1 and \
isinstance(kv_cache_groups[0].kv_cache_spec, UniformTypeKVCacheSpecs):
if len(kv_cache_groups) == 1 and isinstance(
kv_cache_groups[0].kv_cache_spec, UniformTypeKVCacheSpecs
):
# Special case: all layers have the same type of KV cache but with
# different hidden size. Allocate different amount of memory for each
# layer based on its hidden size.
num_blocks = available_memory // kv_cache_groups[
0].kv_cache_spec.page_size_bytes
num_blocks = (
available_memory // kv_cache_groups[0].kv_cache_spec.page_size_bytes
)
num_blocks = may_override_num_blocks(vllm_config, num_blocks)
per_layer_specs = kv_cache_groups[0].kv_cache_spec.kv_cache_specs
kv_cache_tensors = [
KVCacheTensor(size=per_layer_specs[layer_name].page_size_bytes *
num_blocks,
shared_by=[layer_name])
KVCacheTensor(
size=per_layer_specs[layer_name].page_size_bytes * num_blocks,
shared_by=[layer_name],
)
for layer_name in kv_cache_groups[0].layer_names
]
else:
@@ -1059,8 +1100,9 @@ def get_kv_cache_config_from_groups(vllm_config: VllmConfig,
page_size = get_uniform_page_size(kv_cache_specs)
assert group_size > 0, "group_size must be greater than 0"
num_blocks = get_num_blocks(vllm_config, group_size, available_memory,
page_size)
num_blocks = get_num_blocks(
vllm_config, group_size, available_memory, page_size
)
kv_cache_tensors = []
for i in range(group_size):
shared_by = []
@@ -1068,8 +1110,8 @@ def get_kv_cache_config_from_groups(vllm_config: VllmConfig,
if i < len(kv_cache_groups[j].layer_names):
shared_by.append(kv_cache_groups[j].layer_names[i])
kv_cache_tensors.append(
KVCacheTensor(size=page_size * num_blocks,
shared_by=shared_by))
KVCacheTensor(size=page_size * num_blocks, shared_by=shared_by)
)
kv_cache_config = KVCacheConfig(
num_blocks=num_blocks,
@@ -1077,8 +1119,7 @@ def get_kv_cache_config_from_groups(vllm_config: VllmConfig,
kv_cache_groups=kv_cache_groups,
)
min_block_size = min(
[group.kv_cache_spec.block_size for group in kv_cache_groups])
min_block_size = min([group.kv_cache_spec.block_size for group in kv_cache_groups])
# Print the KV cache size and maximum concurrency.
num_tokens = num_blocks // len(kv_cache_groups) * min_block_size
@@ -1086,14 +1127,19 @@ def get_kv_cache_config_from_groups(vllm_config: VllmConfig,
num_tokens *= vllm_config.parallel_config.decode_context_parallel_size
logger.info(
"Multiplying the GPU KV cache size by the dcp_world_size %d.",
vllm_config.parallel_config.decode_context_parallel_size)
vllm_config.parallel_config.decode_context_parallel_size,
)
num_tokens_str = f"{num_tokens:,}"
logger.info("GPU KV cache size: %s tokens", num_tokens_str)
max_model_len_str = f"{vllm_config.model_config.max_model_len:,}"
max_concurrency = get_max_concurrency_for_kv_cache_config(
vllm_config, kv_cache_config)
logger.info("Maximum concurrency for %s tokens per request: %.2fx",
max_model_len_str, max_concurrency)
vllm_config, kv_cache_config
)
logger.info(
"Maximum concurrency for %s tokens per request: %.2fx",
max_model_len_str,
max_concurrency,
)
return kv_cache_config
@@ -1108,25 +1154,27 @@ def unify_hybrid_kv_cache_specs(kv_cache_spec: dict[str, KVCacheSpec]):
"""
if is_kv_cache_spec_uniform(
kv_cache_spec) or UniformTypeKVCacheSpecs.is_uniform_type(
kv_cache_spec):
kv_cache_spec
) or UniformTypeKVCacheSpecs.is_uniform_type(kv_cache_spec):
return
logger.warning(
"Hybrid KV cache manager is disabled for this hybrid model, "
"This means we do not enable any optimizations for saving KV cache "
"memory (e.g., dropping the KV cache outside the sliding window). "
"The compute of layers like sliding window is still saved.")
"The compute of layers like sliding window is still saved."
)
has_full_attention = any(
isinstance(spec, FullAttentionSpec) for spec in kv_cache_spec.values())
isinstance(spec, FullAttentionSpec) for spec in kv_cache_spec.values()
)
has_sliding_window = any(
isinstance(spec, SlidingWindowSpec) for spec in kv_cache_spec.values())
isinstance(spec, SlidingWindowSpec) for spec in kv_cache_spec.values()
)
has_chunked_local_attention = any(
isinstance(spec, ChunkedLocalAttentionSpec)
for spec in kv_cache_spec.values())
if has_full_attention and (has_sliding_window
or has_chunked_local_attention):
isinstance(spec, ChunkedLocalAttentionSpec) for spec in kv_cache_spec.values()
)
if has_full_attention and (has_sliding_window or has_chunked_local_attention):
for layer_name, spec in kv_cache_spec.items():
if isinstance(spec, SlidingWindowSpec):
kv_cache_spec[layer_name] = FullAttentionSpec(
@@ -1145,15 +1193,19 @@ def unify_hybrid_kv_cache_specs(kv_cache_spec: dict[str, KVCacheSpec]):
attention_chunk_size=spec.attention_chunk_size,
)
if not (is_kv_cache_spec_uniform(kv_cache_spec)
or UniformTypeKVCacheSpecs.is_uniform_type(kv_cache_spec)):
raise ValueError("Hybrid KV cache manager is disabled but failed to "
"convert the KV cache specs to one unified type.")
if not (
is_kv_cache_spec_uniform(kv_cache_spec)
or UniformTypeKVCacheSpecs.is_uniform_type(kv_cache_spec)
):
raise ValueError(
"Hybrid KV cache manager is disabled but failed to "
"convert the KV cache specs to one unified type."
)
def get_kv_cache_groups(
vllm_config: VllmConfig,
kv_cache_spec: dict[str, KVCacheSpec]) -> list[KVCacheGroupSpec]:
vllm_config: VllmConfig, kv_cache_spec: dict[str, KVCacheSpec]
) -> list[KVCacheGroupSpec]:
"""
Split the layers in the model into groups with the same KV cache spec.
@@ -1192,14 +1244,14 @@ def get_kv_cache_groups(
def generate_scheduler_kv_cache_config(
kv_cache_configs: list[KVCacheConfig]) -> KVCacheConfig:
kv_cache_configs: list[KVCacheConfig],
) -> KVCacheConfig:
"""
Generate the KV cache configuration for the scheduler.
"""
assert all([
cfg.num_blocks == kv_cache_configs[0].num_blocks
for cfg in kv_cache_configs
])
assert all(
[cfg.num_blocks == kv_cache_configs[0].num_blocks for cfg in kv_cache_configs]
)
# All workers have the same kv_cache_config except layer names, so use
# an arbitrary one to initialize the scheduler.
cfg = copy.deepcopy(kv_cache_configs[0])
@@ -1208,15 +1260,18 @@ def generate_scheduler_kv_cache_config(
# All layers in the UniformTypeKVCacheSpecs have the same type,
# so use an arbitrary one to initialize the scheduler.
group.kv_cache_spec = next(
iter(group.kv_cache_spec.kv_cache_specs.values()))
iter(group.kv_cache_spec.kv_cache_specs.values())
)
return cfg
def get_kv_cache_configs(vllm_config: VllmConfig,
kv_cache_specs: list[dict[str, KVCacheSpec]],
available_memory: list[int]) -> list[KVCacheConfig]:
def get_kv_cache_configs(
vllm_config: VllmConfig,
kv_cache_specs: list[dict[str, KVCacheSpec]],
available_memory: list[int],
) -> list[KVCacheConfig]:
"""
Generates the KV cache configurations for a model.
Generates the KV cache configurations for a model.
Since we use a shared centralized controller for all workers, we need the
`kv_cache_config` to be consistent across all workers to make sure
the KV cache allocation can be applied to all workers. However, different
@@ -1235,7 +1290,7 @@ def get_kv_cache_configs(vllm_config: VllmConfig,
vllm_config: The global VllmConfig
kv_cache_specs: List of dict[layer_name, KVCacheSpec] for each worker.
available_memory: Memory available for KV cache in bytes for each
worker.
worker.
Returns:
The generated KVCacheConfigs for each worker.
@@ -1243,9 +1298,11 @@ def get_kv_cache_configs(vllm_config: VllmConfig,
# Check if the available memory is enough for each worker.
for kv_cache_spec_one_worker, available_memory_one_worker in zip(
kv_cache_specs, available_memory):
check_enough_kv_cache_memory(vllm_config, kv_cache_spec_one_worker,
available_memory_one_worker)
kv_cache_specs, available_memory
):
check_enough_kv_cache_memory(
vllm_config, kv_cache_spec_one_worker, available_memory_one_worker
)
# Merge the KV cache specs of all workers. Different PP stages may have
# different layer names, and different TP ranks of the same PP stage should
@@ -1258,37 +1315,42 @@ def get_kv_cache_configs(vllm_config: VllmConfig,
else:
assert merged_kv_cache_specs[layer_name] == layer_spec, (
"The KV cache specs for the same layer are different "
"across workers. This is not supported yet.")
global_kv_cache_groups = get_kv_cache_groups(vllm_config,
merged_kv_cache_specs)
"across workers. This is not supported yet."
)
global_kv_cache_groups = get_kv_cache_groups(vllm_config, merged_kv_cache_specs)
kv_cache_configs: list[KVCacheConfig] = []
for kv_cache_spec_one_worker, available_memory_one_worker in zip(
kv_cache_specs, available_memory):
kv_cache_specs, available_memory
):
kv_cache_groups_one_worker: list[KVCacheGroupSpec] = []
for group in global_kv_cache_groups:
group_layer_names_one_worker = [
layer_name for layer_name in group.layer_names
layer_name
for layer_name in group.layer_names
if layer_name in kv_cache_spec_one_worker
]
kv_cache_groups_one_worker.append(
KVCacheGroupSpec(group_layer_names_one_worker,
group.kv_cache_spec))
KVCacheGroupSpec(group_layer_names_one_worker, group.kv_cache_spec)
)
assert sum(
len(group.layer_names) for group in
kv_cache_groups_one_worker) == len(kv_cache_spec_one_worker), (
"Some layers are not assigned to any group.")
len(group.layer_names) for group in kv_cache_groups_one_worker
) == len(kv_cache_spec_one_worker), "Some layers are not assigned to any group."
kv_cache_configs.append(
get_kv_cache_config_from_groups(vllm_config,
kv_cache_groups_one_worker,
kv_cache_spec_one_worker,
available_memory_one_worker))
get_kv_cache_config_from_groups(
vllm_config,
kv_cache_groups_one_worker,
kv_cache_spec_one_worker,
available_memory_one_worker,
)
)
# Change the num_blocks of each rank to the smallest among all ranks. We
# do not need to shrink the tensor size because it is valid to only use the
# first `num_blocks` blocks of the tensor.
min_num_blocks = min(kv_cache_config.num_blocks
for kv_cache_config in kv_cache_configs)
min_num_blocks = min(
kv_cache_config.num_blocks for kv_cache_config in kv_cache_configs
)
for kv_cache_config in kv_cache_configs:
kv_cache_config.num_blocks = min_num_blocks
+8 -6
View File
@@ -12,7 +12,6 @@ logger = init_logger(__name__)
class AsyncScheduler(Scheduler):
def _update_after_schedule(
self,
scheduler_output: SchedulerOutput,
@@ -20,8 +19,10 @@ class AsyncScheduler(Scheduler):
super()._update_after_schedule(scheduler_output)
for req_id in scheduler_output.num_scheduled_tokens:
request = self.requests[req_id]
if (request.num_computed_tokens == request.num_tokens +
request.num_output_placeholders):
if (
request.num_computed_tokens
== request.num_tokens + request.num_output_placeholders
):
# The request will generate a new token in this scheduling step.
# TODO(woosuk): Support speculative decoding.
request.num_output_placeholders += 1
@@ -33,7 +34,8 @@ class AsyncScheduler(Scheduler):
) -> tuple[list[int], bool]:
status_before_update = request.status
new_token_ids, stopped = super()._update_request_with_output(
request, new_token_ids)
request, new_token_ids
)
# Update the number of output placeholders.
request.num_output_placeholders -= len(new_token_ids)
@@ -42,6 +44,6 @@ class AsyncScheduler(Scheduler):
# Cache the new tokens. Preempted requests should be skipped.
if status_before_update == RequestStatus.RUNNING:
self.kv_cache_manager.cache_blocks(
request,
request.num_computed_tokens - request.num_output_placeholders)
request, request.num_computed_tokens - request.num_output_placeholders
)
return new_token_ids, stopped
+2 -3
View File
@@ -14,7 +14,6 @@ if TYPE_CHECKING:
class SchedulerInterface(ABC):
@abstractmethod
def schedule(self) -> "SchedulerOutput":
"""Schedule the requests to process in this scheduling step.
@@ -72,7 +71,7 @@ class SchedulerInterface(ABC):
@abstractmethod
def add_request(self, request: "Request") -> None:
"""Add a new request to the scheduler's internal queue.
Args:
request: The new request being added.
"""
@@ -91,7 +90,7 @@ class SchedulerInterface(ABC):
1. When the request is aborted by the client.
2. When the frontend process detects a stop string of the request after
de-tokenizing its generated tokens.
Args:
request_ids: A single or a list of request IDs.
finished_status: The finished status of the given requests.
+30 -32
View File
@@ -13,8 +13,7 @@ if TYPE_CHECKING:
import numpy.typing as npt
import torch
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorMetadata)
from vllm.distributed.kv_transfer.kv_connector.v1.base import KVConnectorMetadata
from vllm.lora.request import LoRARequest
from vllm.multimodal.inputs import MultiModalFeatureSpec
from vllm.pooling_params import PoolingParams
@@ -25,7 +24,6 @@ if TYPE_CHECKING:
@bc_linter_include
@dataclass
class NewRequestData:
req_id: str
prompt_token_ids: Optional[list[int]]
mm_features: list[MultiModalFeatureSpec]
@@ -55,42 +53,43 @@ class NewRequestData:
)
def __repr__(self) -> str:
prompt_embeds_shape = (self.prompt_embeds.shape
if self.prompt_embeds else None)
return (f"NewRequestData("
f"req_id={self.req_id},"
f"prompt_token_ids={self.prompt_token_ids},"
f"mm_features={self.mm_features},"
f"sampling_params={self.sampling_params},"
f"block_ids={self.block_ids},"
f"num_computed_tokens={self.num_computed_tokens},"
f"lora_request={self.lora_request},"
f"prompt_embeds_shape={prompt_embeds_shape}"
")")
prompt_embeds_shape = self.prompt_embeds.shape if self.prompt_embeds else None
return (
f"NewRequestData("
f"req_id={self.req_id},"
f"prompt_token_ids={self.prompt_token_ids},"
f"mm_features={self.mm_features},"
f"sampling_params={self.sampling_params},"
f"block_ids={self.block_ids},"
f"num_computed_tokens={self.num_computed_tokens},"
f"lora_request={self.lora_request},"
f"prompt_embeds_shape={prompt_embeds_shape}"
")"
)
# Version of __repr__ with the prompt data obfuscated
def anon_repr(self) -> str:
prompt_token_ids_len = len(
self.prompt_token_ids
) if self.prompt_token_ids is not None else None
prompt_embeds_shape = (self.prompt_embeds.shape
if self.prompt_embeds else None)
return (f"NewRequestData("
f"req_id={self.req_id},"
f"prompt_token_ids_len={prompt_token_ids_len},"
f"mm_features={self.mm_features},"
f"sampling_params={self.sampling_params},"
f"block_ids={self.block_ids},"
f"num_computed_tokens={self.num_computed_tokens},"
f"lora_request={self.lora_request},"
f"prompt_embeds_shape={prompt_embeds_shape}"
")")
prompt_token_ids_len = (
len(self.prompt_token_ids) if self.prompt_token_ids is not None else None
)
prompt_embeds_shape = self.prompt_embeds.shape if self.prompt_embeds else None
return (
f"NewRequestData("
f"req_id={self.req_id},"
f"prompt_token_ids_len={prompt_token_ids_len},"
f"mm_features={self.mm_features},"
f"sampling_params={self.sampling_params},"
f"block_ids={self.block_ids},"
f"num_computed_tokens={self.num_computed_tokens},"
f"lora_request={self.lora_request},"
f"prompt_embeds_shape={prompt_embeds_shape}"
")"
)
@bc_linter_include
@dataclass
class CachedRequestData:
req_ids: list[str]
# If resumed_from_preemption is False, new_block_ids will be appended to
# the request's block IDs. If True, new_block_ids will be used as the
@@ -122,7 +121,6 @@ class CachedRequestData:
@bc_linter_include
@dataclass
class SchedulerOutput:
# list of the requests that are scheduled for the first time.
# We cache the request's data in each worker process, so that we don't
# need to re-send it every scheduling step.
+10 -11
View File
@@ -14,6 +14,7 @@ from vllm.v1.request import Request
class SchedulingPolicy(Enum):
"""Enum for scheduling policies."""
FCFS = "fcfs"
PRIORITY = "priority"
@@ -111,9 +112,7 @@ class FCFSRequestQueue(deque[Request], RequestQueue):
def remove_requests(self, requests: Iterable[Request]) -> None:
"""Remove multiple specific requests from the queue."""
requests_to_remove = set(requests)
filtered_requests = [
req for req in self if req not in requests_to_remove
]
filtered_requests = [req for req in self if req not in requests_to_remove]
# deque does not support in-place filtering, so we need to clear
# and extend
self.clear()
@@ -150,8 +149,7 @@ class PriorityRequestQueue(RequestQueue):
def add_request(self, request: Request) -> None:
"""Add a request to the queue according to priority policy."""
heapq.heappush(self._heap,
(request.priority, request.arrival_time, request))
heapq.heappush(self._heap, (request.priority, request.arrival_time, request))
def pop_request(self) -> Request:
"""Pop a request from the queue according to priority policy."""
@@ -169,15 +167,15 @@ class PriorityRequestQueue(RequestQueue):
def prepend_request(self, request: Request) -> None:
"""Add a request to the queue according to priority policy.
Note: In a priority queue, there is no concept of prepending to the
Note: In a priority queue, there is no concept of prepending to the
front. Requests are ordered by (priority, arrival_time)."""
self.add_request(request)
def prepend_requests(self, requests: RequestQueue) -> None:
"""Add all requests from another queue according to priority policy.
Note: In a priority queue, there is no concept of prepending to the
Note: In a priority queue, there is no concept of prepending to the
front. Requests are ordered by (priority, arrival_time)."""
for request in requests:
self.add_request(request)
@@ -190,8 +188,9 @@ class PriorityRequestQueue(RequestQueue):
def remove_requests(self, requests: Iterable[Request]) -> None:
"""Remove multiple specific requests from the queue."""
requests_to_remove = set(requests)
self._heap = [(p, t, r) for p, t, r in self._heap
if r not in requests_to_remove]
self._heap = [
(p, t, r) for p, t, r in self._heap if r not in requests_to_remove
]
heapq.heapify(self._heap)
def __bool__(self) -> bool:
+246 -186
View File
@@ -11,25 +11,24 @@ from typing import Any, Optional, Union
from vllm.config import VllmConfig
from vllm.distributed.kv_events import EventPublisherFactory, KVEventBatch
from vllm.distributed.kv_transfer.kv_connector.factory import (
KVConnectorFactory)
from vllm.distributed.kv_transfer.kv_connector.v1 import (KVConnectorBase_V1,
KVConnectorRole)
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
KVConnectorStats)
from vllm.distributed.kv_transfer.kv_connector.factory import KVConnectorFactory
from vllm.distributed.kv_transfer.kv_connector.v1 import (
KVConnectorBase_V1,
KVConnectorRole,
)
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import KVConnectorStats
from vllm.logger import init_logger
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry
from vllm.v1.core.encoder_cache_manager import (EncoderCacheManager,
compute_encoder_budget)
from vllm.v1.core.encoder_cache_manager import (
EncoderCacheManager,
compute_encoder_budget,
)
from vllm.v1.core.kv_cache_manager import KVCacheBlocks, KVCacheManager
from vllm.v1.core.sched.interface import SchedulerInterface
from vllm.v1.core.sched.output import (CachedRequestData, NewRequestData,
SchedulerOutput)
from vllm.v1.core.sched.request_queue import (SchedulingPolicy,
create_request_queue)
from vllm.v1.core.sched.output import CachedRequestData, NewRequestData, SchedulerOutput
from vllm.v1.core.sched.request_queue import SchedulingPolicy, create_request_queue
from vllm.v1.core.sched.utils import check_stop, remove_all
from vllm.v1.engine import (EngineCoreEventType, EngineCoreOutput,
EngineCoreOutputs)
from vllm.v1.engine import EngineCoreEventType, EngineCoreOutput, EngineCoreOutputs
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.metrics.stats import SchedulerStats
from vllm.v1.outputs import DraftTokenIds, KVConnectorOutput, ModelRunnerOutput
@@ -41,7 +40,6 @@ logger = init_logger(__name__)
class Scheduler(SchedulerInterface):
def __init__(
self,
vllm_config: VllmConfig,
@@ -67,16 +65,17 @@ class Scheduler(SchedulerInterface):
# by update_from_outputs(). This is currently used in the multi-engine
# case to track request lifetimes efficiently.
self.finished_req_ids_dict: Optional[dict[int, set[str]]] = (
defaultdict(set) if include_finished_set else None)
defaultdict(set) if include_finished_set else None
)
# Scheduling constraints.
self.max_num_running_reqs = self.scheduler_config.max_num_seqs
self.max_num_scheduled_tokens = \
self.scheduler_config.max_num_batched_tokens
self.max_num_scheduled_tokens = self.scheduler_config.max_num_batched_tokens
self.max_model_len = self.scheduler_config.max_model_len
self.enable_kv_cache_events = (
self.kv_events_config is not None
and self.kv_events_config.enable_kv_cache_events)
and self.kv_events_config.enable_kv_cache_events
)
# Create KVConnector for the Scheduler. Note that each Worker
# will have a corresponding KVConnector with Role=WORKER.
@@ -85,12 +84,14 @@ class Scheduler(SchedulerInterface):
if self.vllm_config.kv_transfer_config is not None:
assert len(self.kv_cache_config.kv_cache_groups) == 1, (
"Multiple KV cache groups are not currently supported "
"with KV connectors")
"with KV connectors"
)
assert not self.is_encoder_decoder, (
"Encoder-decoder models are not currently supported "
"with KV connectors")
"Encoder-decoder models are not currently supported with KV connectors"
)
self.connector = KVConnectorFactory.create_connector(
config=self.vllm_config, role=KVConnectorRole.SCHEDULER)
config=self.vllm_config, role=KVConnectorRole.SCHEDULER
)
self.kv_event_publisher = EventPublisherFactory.create(
self.kv_events_config,
@@ -102,8 +103,7 @@ class Scheduler(SchedulerInterface):
self.block_size = self.cache_config.block_size
self.dcp_world_size = \
vllm_config.parallel_config.decode_context_parallel_size
self.dcp_world_size = vllm_config.parallel_config.decode_context_parallel_size
# Note(hc): The scheduler’s block_size must be multiplied
# by dcp_world_size, since block hashes are computed on the
# original full token sequence at a granularity of
@@ -120,7 +120,8 @@ class Scheduler(SchedulerInterface):
self.policy = SchedulingPolicy.FCFS
else:
raise ValueError(
f"Unknown scheduling policy: {self.scheduler_config.policy}")
f"Unknown scheduling policy: {self.scheduler_config.policy}"
)
# Priority queues for requests.
self.waiting = create_request_queue(self.policy)
self.running: list[Request] = []
@@ -153,8 +154,7 @@ class Scheduler(SchedulerInterface):
# NOTE: For the models without encoder (e.g., text-only models),
# the encoder cache will not be initialized because cache size is 0
# for these models.
self.encoder_cache_manager = EncoderCacheManager(
cache_size=encoder_cache_size)
self.encoder_cache_manager = EncoderCacheManager(cache_size=encoder_cache_size)
speculative_config = vllm_config.speculative_config
self.use_eagle = False
@@ -211,30 +211,35 @@ class Scheduler(SchedulerInterface):
while req_index < len(self.running) and token_budget > 0:
request = self.running[req_index]
num_new_tokens = (request.num_tokens_with_spec +
request.num_output_placeholders -
request.num_computed_tokens)
if (0 < self.scheduler_config.long_prefill_token_threshold <
num_new_tokens):
num_new_tokens = (
self.scheduler_config.long_prefill_token_threshold)
num_new_tokens = (
request.num_tokens_with_spec
+ request.num_output_placeholders
- request.num_computed_tokens
)
if 0 < self.scheduler_config.long_prefill_token_threshold < num_new_tokens:
num_new_tokens = self.scheduler_config.long_prefill_token_threshold
num_new_tokens = min(num_new_tokens, token_budget)
# Make sure the input position does not exceed the max model len.
# This is necessary when using spec decoding.
num_new_tokens = min(
num_new_tokens,
self.max_model_len - request.num_computed_tokens)
num_new_tokens, self.max_model_len - request.num_computed_tokens
)
# Schedule encoder inputs.
encoder_inputs_to_schedule = None
new_encoder_compute_budget = encoder_compute_budget
if request.has_encoder_inputs:
(encoder_inputs_to_schedule, num_new_tokens,
new_encoder_compute_budget
) = self._try_schedule_encoder_inputs(
request, request.num_computed_tokens, num_new_tokens,
encoder_compute_budget)
(
encoder_inputs_to_schedule,
num_new_tokens,
new_encoder_compute_budget,
) = self._try_schedule_encoder_inputs(
request,
request.num_computed_tokens,
num_new_tokens,
encoder_compute_budget,
)
if num_new_tokens == 0:
# The request cannot be scheduled because one of the following
@@ -257,7 +262,8 @@ class Scheduler(SchedulerInterface):
new_blocks = self.kv_cache_manager.allocate_slots(
request,
num_new_tokens,
num_lookahead_tokens=self.num_lookahead_tokens)
num_lookahead_tokens=self.num_lookahead_tokens,
)
if new_blocks is not None:
# The request can be scheduled.
@@ -282,8 +288,9 @@ class Scheduler(SchedulerInterface):
preempted_req.num_computed_tokens = 0
preempted_req.num_preemptions += 1
if self.log_stats:
preempted_req.record_event(EngineCoreEventType.PREEMPTED,
scheduled_timestamp)
preempted_req.record_event(
EngineCoreEventType.PREEMPTED, scheduled_timestamp
)
self.waiting.prepend_request(preempted_req)
preempted_reqs.append(preempted_req)
@@ -304,19 +311,21 @@ class Scheduler(SchedulerInterface):
# Speculative decode related.
if request.spec_token_ids:
num_scheduled_spec_tokens = (num_new_tokens +
request.num_computed_tokens -
request.num_tokens)
num_scheduled_spec_tokens = (
num_new_tokens + request.num_computed_tokens - request.num_tokens
)
if num_scheduled_spec_tokens > 0:
# Trim spec_token_ids list to num_scheduled_spec_tokens.
del request.spec_token_ids[num_scheduled_spec_tokens:]
scheduled_spec_decode_tokens[request.request_id] = (
request.spec_token_ids)
request.spec_token_ids
)
# Encoder-related.
if encoder_inputs_to_schedule:
scheduled_encoder_inputs[request.request_id] = (
encoder_inputs_to_schedule)
encoder_inputs_to_schedule
)
# Allocate the encoder cache.
for i in encoder_inputs_to_schedule:
self.encoder_cache_manager.allocate(request, i)
@@ -326,8 +335,10 @@ class Scheduler(SchedulerInterface):
scheduled_loras: set[int] = set()
if self.lora_config:
scheduled_loras = set(
req.lora_request.lora_int_id for req in scheduled_running_reqs
if req.lora_request and req.lora_request.lora_int_id > 0)
req.lora_request.lora_int_id
for req in scheduled_running_reqs
if req.lora_request and req.lora_request.lora_int_id > 0
)
assert len(scheduled_loras) <= self.lora_config.max_loras
# Use a temporary RequestQueue to collect requests that need to be
@@ -350,7 +361,8 @@ class Scheduler(SchedulerInterface):
else:
logger.debug(
"%s is still in WAITING_FOR_REMOTE_KVS state.",
request.request_id)
request.request_id,
)
self.waiting.pop_request()
skipped_waiting_requests.prepend_request(request)
continue
@@ -368,9 +380,14 @@ class Scheduler(SchedulerInterface):
# Check that adding the request still respects the max_loras
# constraint.
if (self.lora_config and request.lora_request and
(len(scheduled_loras) == self.lora_config.max_loras and
request.lora_request.lora_int_id not in scheduled_loras)):
if (
self.lora_config
and request.lora_request
and (
len(scheduled_loras) == self.lora_config.max_loras
and request.lora_request.lora_int_id not in scheduled_loras
)
):
# Scheduling would exceed max_loras, skip.
self.waiting.pop_request()
skipped_waiting_requests.prepend_request(request)
@@ -382,15 +399,17 @@ class Scheduler(SchedulerInterface):
# Get already-cached tokens.
if request.num_computed_tokens == 0:
# Get locally-cached tokens.
new_computed_blocks, num_new_local_computed_tokens = \
self.kv_cache_manager.get_computed_blocks(
request)
new_computed_blocks, num_new_local_computed_tokens = (
self.kv_cache_manager.get_computed_blocks(request)
)
# Get externally-cached tokens if using a KVConnector.
if self.connector is not None:
num_external_computed_tokens, load_kv_async = (
self.connector.get_num_new_matched_tokens(
request, num_new_local_computed_tokens))
request, num_new_local_computed_tokens
)
)
if num_external_computed_tokens is None:
# The request cannot be scheduled because
@@ -401,13 +420,15 @@ class Scheduler(SchedulerInterface):
continue
# Total computed tokens (local + external).
num_computed_tokens = (num_new_local_computed_tokens +
num_external_computed_tokens)
num_computed_tokens = (
num_new_local_computed_tokens + num_external_computed_tokens
)
# KVTransfer: WAITING reqs have num_computed_tokens > 0
# after async KV recvs are completed.
else:
new_computed_blocks = (
self.kv_cache_manager.create_empty_block_list())
self.kv_cache_manager.create_empty_block_list()
)
num_new_local_computed_tokens = 0
num_computed_tokens = request.num_computed_tokens
@@ -424,15 +445,21 @@ class Scheduler(SchedulerInterface):
# `request.num_prompt_tokens` to consider the resumed
# requests, which have output tokens.
num_new_tokens = request.num_tokens - num_computed_tokens
if (0 < self.scheduler_config.long_prefill_token_threshold
< num_new_tokens):
if (
0
< self.scheduler_config.long_prefill_token_threshold
< num_new_tokens
):
num_new_tokens = (
self.scheduler_config.long_prefill_token_threshold)
self.scheduler_config.long_prefill_token_threshold
)
# chunked prefill has to be enabled explicitly to allow
# pooling requests to be chunked
if not self.scheduler_config.chunked_prefill_enabled and \
num_new_tokens > token_budget:
if (
not self.scheduler_config.chunked_prefill_enabled
and num_new_tokens > token_budget
):
self.waiting.pop_request()
skipped_waiting_requests.prepend_request(request)
continue
@@ -442,11 +469,16 @@ class Scheduler(SchedulerInterface):
# Schedule encoder inputs.
if request.has_encoder_inputs:
(encoder_inputs_to_schedule, num_new_tokens,
new_encoder_compute_budget
) = self._try_schedule_encoder_inputs(
request, num_computed_tokens, num_new_tokens,
encoder_compute_budget)
(
encoder_inputs_to_schedule,
num_new_tokens,
new_encoder_compute_budget,
) = self._try_schedule_encoder_inputs(
request,
num_computed_tokens,
num_new_tokens,
encoder_compute_budget,
)
if num_new_tokens == 0:
# The request cannot be scheduled.
break
@@ -456,9 +488,9 @@ class Scheduler(SchedulerInterface):
# extra block gets allocated which
# creates a mismatch between the number
# of local and remote blocks.
effective_lookahead_tokens = (0 if request.num_computed_tokens
== 0 else
self.num_lookahead_tokens)
effective_lookahead_tokens = (
0 if request.num_computed_tokens == 0 else self.num_lookahead_tokens
)
# Determine if we need to allocate cross-attention blocks.
if self.is_encoder_decoder and request.has_encoder_inputs:
@@ -466,8 +498,9 @@ class Scheduler(SchedulerInterface):
# always padded to the maximum length. If we support other
# encoder-decoder models, this will need to be updated if we
# want to only allocate what is needed.
num_encoder_tokens =\
num_encoder_tokens = (
self.scheduler_config.max_num_encoder_input_tokens
)
else:
num_encoder_tokens = 0
@@ -509,20 +542,21 @@ class Scheduler(SchedulerInterface):
req_index += 1
self.running.append(request)
if self.log_stats:
request.record_event(EngineCoreEventType.SCHEDULED,
scheduled_timestamp)
request.record_event(
EngineCoreEventType.SCHEDULED, scheduled_timestamp
)
if request.status == RequestStatus.WAITING:
scheduled_new_reqs.append(request)
elif request.status == RequestStatus.PREEMPTED:
scheduled_resumed_reqs.append(request)
else:
raise RuntimeError(
f"Invalid request status: {request.status}")
raise RuntimeError(f"Invalid request status: {request.status}")
if self.lora_config and request.lora_request:
scheduled_loras.add(request.lora_request.lora_int_id)
req_to_new_blocks[request.request_id] = (
self.kv_cache_manager.get_blocks(request.request_id))
self.kv_cache_manager.get_blocks(request.request_id)
)
num_scheduled_tokens[request.request_id] = num_new_tokens
token_budget -= num_new_tokens
request.status = RequestStatus.RUNNING
@@ -533,7 +567,8 @@ class Scheduler(SchedulerInterface):
# Encoder-related.
if encoder_inputs_to_schedule:
scheduled_encoder_inputs[request.request_id] = (
encoder_inputs_to_schedule)
encoder_inputs_to_schedule
)
# Allocate the encoder cache.
for i in encoder_inputs_to_schedule:
self.encoder_cache_manager.allocate(request, i)
@@ -551,23 +586,26 @@ class Scheduler(SchedulerInterface):
# Since some requests in the RUNNING queue may not be scheduled in
# this step, the total number of scheduled requests can be smaller than
# len(self.running).
assert (len(scheduled_new_reqs) + len(scheduled_resumed_reqs) +
len(scheduled_running_reqs) <= len(self.running))
assert len(scheduled_new_reqs) + len(scheduled_resumed_reqs) + len(
scheduled_running_reqs
) <= len(self.running)
# Get the longest common prefix among all requests in the running queue.
# This can be potentially used for cascade attention.
num_common_prefix_blocks = [0] * len(
self.kv_cache_config.kv_cache_groups)
num_common_prefix_blocks = [0] * len(self.kv_cache_config.kv_cache_groups)
if self.running:
any_request = self.running[0]
num_common_prefix_blocks = (
self.kv_cache_manager.get_num_common_prefix_blocks(
any_request, len(self.running)))
any_request, len(self.running)
)
)
# Construct the scheduler output.
new_reqs_data = [
NewRequestData.from_request(
req, req_to_new_blocks[req.request_id].get_block_ids())
req, req_to_new_blocks[req.request_id].get_block_ids()
)
for req in scheduled_new_reqs
]
cached_reqs_data = self._make_cached_request_data(
@@ -577,11 +615,12 @@ class Scheduler(SchedulerInterface):
scheduled_spec_decode_tokens,
req_to_new_blocks,
)
scheduled_requests = (scheduled_new_reqs + scheduled_running_reqs +
scheduled_resumed_reqs)
structured_output_request_ids, grammar_bitmask = (
self.get_grammar_bitmask(scheduled_requests,
scheduled_spec_decode_tokens))
scheduled_requests = (
scheduled_new_reqs + scheduled_running_reqs + scheduled_resumed_reqs
)
structured_output_request_ids, grammar_bitmask = self.get_grammar_bitmask(
scheduled_requests, scheduled_spec_decode_tokens
)
scheduler_output = SchedulerOutput(
scheduled_new_reqs=new_reqs_data,
scheduled_cached_reqs=cached_reqs_data,
@@ -595,8 +634,7 @@ class Scheduler(SchedulerInterface):
# It contains the request IDs that are finished in between
# the previous and the current steps.
finished_req_ids=self.finished_req_ids,
free_encoder_mm_hashes=self.encoder_cache_manager.
get_freed_mm_hashes(),
free_encoder_mm_hashes=self.encoder_cache_manager.get_freed_mm_hashes(),
structured_output_request_ids=structured_output_request_ids,
grammar_bitmask=grammar_bitmask,
)
@@ -678,16 +716,18 @@ class Scheduler(SchedulerInterface):
for req in itertools.chain(running_reqs, resumed_reqs):
req_id = req.request_id
req_ids.append(req_id)
num_tokens = (num_scheduled_tokens[req_id] -
len(spec_decode_tokens.get(req_id, ())))
num_tokens = num_scheduled_tokens[req_id] - len(
spec_decode_tokens.get(req_id, ())
)
if self.use_pp:
# When using PP, the scheduler sends the sampled tokens back,
# because there's no direct communication between the first-
# stage worker and the last-stage worker. Otherwise, we don't
# need to send the sampled tokens back because the model runner
# will cache them.
token_ids = req.all_token_ids[req.num_computed_tokens:req.
num_computed_tokens + num_tokens]
token_ids = req.all_token_ids[
req.num_computed_tokens : req.num_computed_tokens + num_tokens
]
new_token_ids.append(token_ids)
elif use_connector:
# When using a KVConnector, we add a placeholder to avoid index
@@ -695,7 +735,8 @@ class Scheduler(SchedulerInterface):
# is updated to handle token IDs properly.
new_token_ids.append([])
new_block_ids.append(
req_to_new_blocks[req_id].get_block_ids(allow_none=True))
req_to_new_blocks[req_id].get_block_ids(allow_none=True)
)
num_computed_tokens.append(req.num_computed_tokens)
num_output_tokens.append(len(req.output_token_ids))
# Because resumed_reqs is usually empty, it is more efficient to do
@@ -764,7 +805,8 @@ class Scheduler(SchedulerInterface):
if self.is_encoder_decoder and num_computed_tokens > 0:
assert start_pos == 0, (
"Encoder input should be processed at the beginning of "
"the sequence when encoder-decoder models are used.")
"the sequence when encoder-decoder models are used."
)
# Encoder input has already been computed
# The calculation here is a bit different. We don't turn encoder
# output into tokens that get processed by the decoder and
@@ -788,8 +830,7 @@ class Scheduler(SchedulerInterface):
# current step.
continue
if self.encoder_cache_manager.check_and_update_cache(
request, i):
if self.encoder_cache_manager.check_and_update_cache(request, i):
# The encoder input is already computed and cached from a
# previous step.
continue
@@ -797,16 +838,18 @@ class Scheduler(SchedulerInterface):
# If no encoder input chunking is allowed, we do not want to
# partially schedule a multimodal item. If the scheduled range would
# only cover part of the mm input, roll back to before the mm item.
if (self.scheduler_config.disable_chunked_mm_input
and num_computed_tokens < start_pos
and (num_computed_tokens + num_new_tokens)
< (start_pos + num_encoder_tokens)):
if (
self.scheduler_config.disable_chunked_mm_input
and num_computed_tokens < start_pos
and (num_computed_tokens + num_new_tokens)
< (start_pos + num_encoder_tokens)
):
num_new_tokens = start_pos - num_computed_tokens
break
if not self.encoder_cache_manager.can_allocate(
request, i, encoder_compute_budget,
num_tokens_to_schedule):
request, i, encoder_compute_budget, num_tokens_to_schedule
):
# The encoder cache is full or the encoder budget is exhausted.
# NOTE(woosuk): We assume that the encoder input tokens should
# be processed altogether, as the encoder usually uses
@@ -879,8 +922,9 @@ class Scheduler(SchedulerInterface):
outputs: dict[int, list[EngineCoreOutput]] = defaultdict(list)
spec_decoding_stats: Optional[SpecDecodingStats] = None
kv_connector_stats = (kv_connector_output.kv_connector_stats
if kv_connector_output else None)
kv_connector_stats = (
kv_connector_output.kv_connector_stats if kv_connector_output else None
)
failed_kv_load_req_ids = None
if kv_connector_output and kv_connector_output.invalid_block_ids:
@@ -888,7 +932,8 @@ class Scheduler(SchedulerInterface):
# load. Identify affected requests and adjust their computed token
# count to trigger recomputation of the invalid blocks.
failed_kv_load_req_ids = self._handle_invalid_blocks(
kv_connector_output.invalid_block_ids)
kv_connector_output.invalid_block_ids
)
# NOTE(woosuk): As len(num_scheduled_tokens) can be up to 1K or more,
# the below loop can be a performance bottleneck. We should do our best
@@ -908,11 +953,13 @@ class Scheduler(SchedulerInterface):
continue
req_index = model_runner_output.req_id_to_index[req_id]
generated_token_ids = sampled_token_ids[
req_index] if sampled_token_ids else []
generated_token_ids = (
sampled_token_ids[req_index] if sampled_token_ids else []
)
scheduled_spec_token_ids = (
scheduler_output.scheduled_spec_decode_tokens.get(req_id))
scheduler_output.scheduled_spec_decode_tokens.get(req_id)
)
if scheduled_spec_token_ids:
num_draft_tokens = len(scheduled_spec_token_ids)
num_accepted = len(generated_token_ids) - 1
@@ -926,7 +973,8 @@ class Scheduler(SchedulerInterface):
spec_decoding_stats = self.make_spec_decoding_stats(
spec_decoding_stats,
num_draft_tokens=num_draft_tokens,
num_accepted_tokens=num_accepted)
num_accepted_tokens=num_accepted,
)
stopped = False
new_logprobs = None
@@ -937,14 +985,14 @@ class Scheduler(SchedulerInterface):
# Check for stop and update request status.
if new_token_ids:
new_token_ids, stopped = self._update_request_with_output(
request, new_token_ids)
request, new_token_ids
)
# Stop checking for pooler models.
pooler_output = None
if pooler_outputs:
pooler_output = pooler_outputs[req_index]
stopped = check_stop(request, self.max_model_len,
pooler_output)
stopped = check_stop(request, self.max_model_len, pooler_output)
if stopped:
kv_transfer_params = self._free_request(request)
@@ -954,28 +1002,29 @@ class Scheduler(SchedulerInterface):
stopped_preempted_reqs.add(request)
# Extract sample logprobs if needed.
if request.sampling_params is not None \
and request.sampling_params.logprobs is not None and logprobs:
if (
request.sampling_params is not None
and request.sampling_params.logprobs is not None
and logprobs
):
# NOTE: once we support N tokens per step (spec decode),
# the outer lists can be of length > 1.
new_logprobs = logprobs.slice(req_index, req_index + 1)
if new_token_ids and self.structured_output_manager.should_advance(
request):
if new_token_ids and self.structured_output_manager.should_advance(request):
# NOTE: structured_output_request
# should not be None if use_structured_output, we have
# checked above, so safe to ignore type warning
request.structured_output_request.grammar.accept_tokens( # type: ignore[union-attr]
req_id, new_token_ids)
req_id, new_token_ids
)
if num_nans_in_logits is not None and req_id in num_nans_in_logits:
request.num_nans_in_logits = num_nans_in_logits[req_id]
# Get prompt logprobs for this request.
prompt_logprobs_tensors = prompt_logprobs_dict.get(req_id)
if new_token_ids or pooler_output is not None \
or kv_transfer_params:
if new_token_ids or pooler_output is not None or kv_transfer_params:
# Add EngineCoreOutput for this Request.
outputs[request.client_index].append(
EngineCoreOutput(
@@ -990,7 +1039,8 @@ class Scheduler(SchedulerInterface):
kv_transfer_params=kv_transfer_params,
trace_headers=request.trace_headers,
num_cached_tokens=request.num_cached_tokens,
))
)
)
else:
# Invariant: EngineCore returns no partial prefill outputs.
assert not prompt_logprobs_tensors
@@ -1023,11 +1073,13 @@ class Scheduler(SchedulerInterface):
eco.finished_requests = finished_set
else:
engine_core_outputs[client_index] = EngineCoreOutputs(
finished_requests=finished_set)
finished_requests=finished_set
)
finished_req_ids.clear()
if (stats := self.make_stats(spec_decoding_stats,
kv_connector_stats)) is not None:
if (
stats := self.make_stats(spec_decoding_stats, kv_connector_stats)
) is not None:
# Return stats to only one of the front-ends.
if (eco := next(iter(engine_core_outputs.values()), None)) is None:
# We must return the stats even if there are no request
@@ -1058,8 +1110,9 @@ class Scheduler(SchedulerInterface):
return new_token_ids, stopped
def _free_encoder_inputs(self, request: Request) -> None:
cached_encoder_input_ids = (
self.encoder_cache_manager.get_cached_input_ids(request))
cached_encoder_input_ids = self.encoder_cache_manager.get_cached_input_ids(
request
)
# OPTIMIZATION: Avoid list(set) if the set is empty.
if not cached_encoder_input_ids:
return
@@ -1074,21 +1127,19 @@ class Scheduler(SchedulerInterface):
# With Whisper, as soon as we've generated a single token,
# we know we're done with the encoder input. Cross Attention
# KVs have been calculated and cached already.
self.encoder_cache_manager.free_encoder_input(
request, input_id)
self.encoder_cache_manager.free_encoder_input(request, input_id)
elif start_pos + num_tokens <= request.num_computed_tokens:
# The encoder output is already processed and stored
# in the decoder's KV cache.
self.encoder_cache_manager.free_encoder_input(
request, input_id)
self.encoder_cache_manager.free_encoder_input(request, input_id)
def update_draft_token_ids(
self,
draft_token_ids: DraftTokenIds,
) -> None:
for req_id, spec_token_ids in zip(
draft_token_ids.req_ids,
draft_token_ids.draft_token_ids,
draft_token_ids.req_ids,
draft_token_ids.draft_token_ids,
):
request = self.requests.get(req_id)
if request is None or request.is_finished():
@@ -1102,7 +1153,8 @@ class Scheduler(SchedulerInterface):
elif self.structured_output_manager.should_advance(request):
metadata = request.structured_output_request
request.spec_token_ids = metadata.grammar.validate_tokens( # type: ignore[union-attr]
spec_token_ids)
spec_token_ids
)
else:
request.spec_token_ids = spec_token_ids
@@ -1128,7 +1180,7 @@ class Scheduler(SchedulerInterface):
"""
assert RequestStatus.is_finished(finished_status)
if isinstance(request_ids, str):
request_ids = (request_ids, )
request_ids = (request_ids,)
else:
request_ids = set(request_ids)
@@ -1198,15 +1250,15 @@ class Scheduler(SchedulerInterface):
return None
prefix_cache_stats = self.kv_cache_manager.make_prefix_cache_stats()
assert prefix_cache_stats is not None
return SchedulerStats(num_running_reqs=len(self.running),
num_waiting_reqs=len(self.waiting),
kv_cache_usage=self.kv_cache_manager.usage,
prefix_cache_stats=prefix_cache_stats,
spec_decoding_stats=spec_decoding_stats,
num_corrupted_reqs=sum(req.is_output_corrupted
for req in self.running),
kv_connector_stats=kv_connector_stats.data
if kv_connector_stats else None)
return SchedulerStats(
num_running_reqs=len(self.running),
num_waiting_reqs=len(self.waiting),
kv_cache_usage=self.kv_cache_manager.usage,
prefix_cache_stats=prefix_cache_stats,
spec_decoding_stats=spec_decoding_stats,
num_corrupted_reqs=sum(req.is_output_corrupted for req in self.running),
kv_connector_stats=kv_connector_stats.data if kv_connector_stats else None,
)
def make_spec_decoding_stats(
self,
@@ -1219,8 +1271,8 @@ class Scheduler(SchedulerInterface):
if spec_decoding_stats is None:
spec_decoding_stats = SpecDecodingStats.new(self.num_spec_tokens)
spec_decoding_stats.observe_draft(
num_draft_tokens=num_draft_tokens,
num_accepted_tokens=num_accepted_tokens)
num_draft_tokens=num_draft_tokens, num_accepted_tokens=num_accepted_tokens
)
return spec_decoding_stats
def shutdown(self) -> None:
@@ -1237,7 +1289,8 @@ class Scheduler(SchedulerInterface):
return self.connector
def _connector_finished(
self, request: Request) -> tuple[bool, Optional[dict[str, Any]]]:
self, request: Request
) -> tuple[bool, Optional[dict[str, Any]]]:
"""
Invoke the KV connector request_finished() method if applicable.
@@ -1247,7 +1300,7 @@ class Scheduler(SchedulerInterface):
if self.connector is None:
return False, None
(block_ids, ) = self.kv_cache_manager.get_block_ids(request.request_id)
(block_ids,) = self.kv_cache_manager.get_block_ids(request.request_id)
return self.connector.request_finished(request, block_ids)
def _update_waiting_for_remote_kv(self, request: Request) -> bool:
@@ -1271,8 +1324,7 @@ class Scheduler(SchedulerInterface):
# updated in _update_requests_with_invalid_blocks
if request.num_computed_tokens:
# Cache any valid computed tokens.
self.kv_cache_manager.cache_blocks(request,
request.num_computed_tokens)
self.kv_cache_manager.cache_blocks(request, request.num_computed_tokens)
else:
# No valid computed tokens, release allocated blocks.
# There may be a local cache hit on retry.
@@ -1281,8 +1333,7 @@ class Scheduler(SchedulerInterface):
self.failed_recving_kv_req_ids.remove(request.request_id)
else:
# Now that the blocks are ready, actually cache them.
(block_ids, ) = self.kv_cache_manager.get_block_ids(
request.request_id)
(block_ids,) = self.kv_cache_manager.get_block_ids(request.request_id)
num_computed_tokens = len(block_ids) * self.block_size
# Handle the case where num request tokens less than one block.
num_computed_tokens = min(num_computed_tokens, request.num_tokens)
@@ -1298,8 +1349,7 @@ class Scheduler(SchedulerInterface):
self.finished_recving_kv_req_ids.remove(request.request_id)
return True
def _update_from_kv_xfer_finished(self,
kv_connector_output: KVConnectorOutput):
def _update_from_kv_xfer_finished(self, kv_connector_output: KVConnectorOutput):
"""
KV Connector: update the scheduler state based on the output.
@@ -1314,21 +1364,23 @@ class Scheduler(SchedulerInterface):
self.connector.update_connector_output(kv_connector_output)
# KV Connector:: update recv and send status from last step.
for req_id in (kv_connector_output.finished_recving or ()):
for req_id in kv_connector_output.finished_recving or ():
logger.debug("Finished recving KV transfer for request %s", req_id)
self.finished_recving_kv_req_ids.add(req_id)
for req_id in (kv_connector_output.finished_sending or ()):
for req_id in kv_connector_output.finished_sending or ():
logger.debug("Finished sending KV transfer for request %s", req_id)
if req_id not in self.requests:
logger.warning(
"Got finished sending KV transfer for request %s,"
"but the request is already freed.", req_id)
"but the request is already freed.",
req_id,
)
else:
self._free_blocks(self.requests[req_id])
def _update_requests_with_invalid_blocks(
self, requests: Iterable[Request],
invalid_block_ids: set[int]) -> tuple[set[str], int]:
self, requests: Iterable[Request], invalid_block_ids: set[int]
) -> tuple[set[str], int]:
"""
Identify and update requests affected by invalid KV cache blocks.
@@ -1359,25 +1411,25 @@ class Scheduler(SchedulerInterface):
marked_invalid_block = False
req_id = request.request_id
# TODO (davidb): add support for hybrid memory allocator
(req_block_ids, ) = self.kv_cache_manager.get_block_ids(req_id)
(req_block_ids,) = self.kv_cache_manager.get_block_ids(req_id)
# We iterate only over blocks that may contain externally computed
# tokens
if request.status == RequestStatus.WAITING_FOR_REMOTE_KVS:
# Async loading. If num_computed_tokens is set it implies we
# already processed some block failures for it in a prior step
req_num_computed_tokens = (
request.num_computed_tokens if req_id
in self.failed_recving_kv_req_ids else len(req_block_ids) *
self.block_size)
request.num_computed_tokens
if req_id in self.failed_recving_kv_req_ids
else len(req_block_ids) * self.block_size
)
else:
# Sync loading. num_computed_tokens includes new tokens
req_num_computed_tokens = request.num_cached_tokens
req_num_computed_blocks = (req_num_computed_tokens +
self.block_size - 1) // self.block_size
for idx, block_id in zip(range(req_num_computed_blocks),
req_block_ids):
req_num_computed_blocks = (
req_num_computed_tokens + self.block_size - 1
) // self.block_size
for idx, block_id in zip(range(req_num_computed_blocks), req_block_ids):
if block_id not in invalid_block_ids:
continue
@@ -1402,8 +1454,9 @@ class Scheduler(SchedulerInterface):
marked_invalid_block = True
# Truncate the computed tokens at the first failed block
request.num_computed_tokens = idx * self.block_size
total_affected_tokens += (req_num_computed_tokens -
request.num_computed_tokens)
total_affected_tokens += (
req_num_computed_tokens - request.num_computed_tokens
)
if is_affected:
if not marked_invalid_block:
@@ -1412,8 +1465,9 @@ class Scheduler(SchedulerInterface):
# Revert to considering only cached tokens as computed.
# Currently this only applies to sync loading; Async
# loading does not yet support block sharing
total_affected_tokens += (request.num_computed_tokens -
request.num_cached_tokens)
total_affected_tokens += (
request.num_computed_tokens - request.num_cached_tokens
)
request.num_computed_tokens = request.num_cached_tokens
affected_req_ids.add(request.request_id)
@@ -1426,11 +1480,15 @@ class Scheduler(SchedulerInterface):
# --- Handle async KV loads (WAITING_FOR_REMOTE_KVS) ---
async_load_reqs = (
req for req in self.waiting
if req.status == RequestStatus.WAITING_FOR_REMOTE_KVS)
req
for req in self.waiting
if req.status == RequestStatus.WAITING_FOR_REMOTE_KVS
)
async_affected_req_ids, num_tokens_to_reschedule = (
self._update_requests_with_invalid_blocks(async_load_reqs,
invalid_block_ids))
self._update_requests_with_invalid_blocks(
async_load_reqs, invalid_block_ids
)
)
total_requests_to_reschedule += len(async_affected_req_ids)
total_tokens_to_reschedule += num_tokens_to_reschedule
@@ -1441,8 +1499,8 @@ class Scheduler(SchedulerInterface):
# --- Handle sync KV loads (running requests) ---
sync_affected_req_ids, num_tokens_to_reschedule = (
self._update_requests_with_invalid_blocks(self.running,
invalid_block_ids))
self._update_requests_with_invalid_blocks(self.running, invalid_block_ids)
)
total_requests_to_reschedule += len(sync_affected_req_ids)
total_tokens_to_reschedule += num_tokens_to_reschedule
@@ -1451,7 +1509,9 @@ class Scheduler(SchedulerInterface):
logger.warning(
"Recovered from KV load failure: "
"%d request(s) rescheduled (%d tokens affected).",
total_requests_to_reschedule, total_tokens_to_reschedule)
total_requests_to_reschedule,
total_tokens_to_reschedule,
)
# Return the IDs of affected running requests to skip in
# update_from_output.
+8 -7
View File
@@ -40,11 +40,13 @@ def remove_all(lst: list, items_to_remove: set) -> list:
return [item for item in lst if item not in items_to_remove]
def check_stop(request: Request,
max_model_len: int,
pooler_output: Optional[torch.Tensor] = None) -> bool:
if (request.num_tokens > max_model_len
or request.num_output_tokens >= request.max_tokens):
def check_stop(
request: Request, max_model_len: int, pooler_output: Optional[torch.Tensor] = None
) -> bool:
if (
request.num_tokens > max_model_len
or request.num_output_tokens >= request.max_tokens
):
request.status = RequestStatus.FINISHED_LENGTH_CAPPED
return True
@@ -57,8 +59,7 @@ def check_stop(request: Request,
sampling_params = request.sampling_params
assert sampling_params is not None
last_token_id = request.output_token_ids[-1]
if (not sampling_params.ignore_eos
and last_token_id == request.eos_token_id):
if not sampling_params.ignore_eos and last_token_id == request.eos_token_id:
request.status = RequestStatus.FINISHED_STOPPED
return True
+145 -114
View File
@@ -7,16 +7,21 @@ from collections import defaultdict
from vllm.utils import cdiv
from vllm.v1.core.block_pool import BlockPool
from vllm.v1.core.kv_cache_utils import BlockHash, KVCacheBlock
from vllm.v1.kv_cache_interface import (ChunkedLocalAttentionSpec,
CrossAttentionSpec, FullAttentionSpec,
KVCacheSpec, MambaSpec,
MLAAttentionSpec, SlidingWindowSpec)
from vllm.v1.kv_cache_interface import (
ChunkedLocalAttentionSpec,
CrossAttentionSpec,
FullAttentionSpec,
KVCacheSpec,
MambaSpec,
MLAAttentionSpec,
SlidingWindowSpec,
)
from vllm.v1.request import Request
class SingleTypeKVCacheManager(ABC):
"""
An abstract base class for a manager that handle the kv cache management
An abstract base class for a manager that handle the kv cache management
logic of one specific type of attention layer.
"""
@@ -44,8 +49,7 @@ class SingleTypeKVCacheManager(ABC):
# Mapping from request ID to blocks to track the blocks allocated
# for each request, so that we can free the blocks when the request
# is finished.
self.req_to_blocks: defaultdict[str,
list[KVCacheBlock]] = defaultdict(list)
self.req_to_blocks: defaultdict[str, list[KVCacheBlock]] = defaultdict(list)
# {req_id: The number of cached blocks for this given request}
# This is used to track the number of cached blocks for each request.
@@ -57,14 +61,14 @@ class SingleTypeKVCacheManager(ABC):
self._null_block = block_pool.null_block
def get_num_blocks_to_allocate(
self, request_id: str, num_tokens: int,
new_computed_blocks: list[KVCacheBlock]) -> int:
self, request_id: str, num_tokens: int, new_computed_blocks: list[KVCacheBlock]
) -> int:
"""
Get the number of blocks needed to be allocated for the request.
Args:
request_id: The request ID.
num_tokens: The total number of tokens that need a slot (including
num_tokens: The total number of tokens that need a slot (including
tokens that are already allocated).
new_computed_blocks: The new computed blocks just hitting the
prefix caching.
@@ -74,20 +78,23 @@ class SingleTypeKVCacheManager(ABC):
"""
num_required_blocks = cdiv(num_tokens, self.block_size)
num_new_blocks = (num_required_blocks - len(new_computed_blocks) -
len(self.req_to_blocks[request_id]))
num_new_blocks = (
num_required_blocks
- len(new_computed_blocks)
- len(self.req_to_blocks[request_id])
)
# If a computed block of a request is an eviction candidate (in the
# free queue and ref_cnt == 0), it will be changed from a free block
# to a computed block when the request is allocated, so we also count
# it as needed to be allocated.
num_evictable_computed_blocks = sum(
blk.ref_cnt == 0 and not blk.is_null
for blk in new_computed_blocks)
blk.ref_cnt == 0 and not blk.is_null for blk in new_computed_blocks
)
return num_new_blocks + num_evictable_computed_blocks
def save_new_computed_blocks(
self, request_id: str,
new_computed_blocks: list[KVCacheBlock]) -> None:
self, request_id: str, new_computed_blocks: list[KVCacheBlock]
) -> None:
"""
Add the new computed blocks to the request.
@@ -106,15 +113,16 @@ class SingleTypeKVCacheManager(ABC):
# A running request. Should not have new computed blocks.
assert len(new_computed_blocks) == 0
def allocate_new_blocks(self, request_id: str,
num_tokens: int) -> list[KVCacheBlock]:
def allocate_new_blocks(
self, request_id: str, num_tokens: int
) -> list[KVCacheBlock]:
"""
Allocate new blocks for the request to give it at least `num_tokens`
Allocate new blocks for the request to give it at least `num_tokens`
token slots.
Args:
request_id: The request ID.
num_tokens: The total number of tokens that need a slot (including
num_tokens: The total number of tokens that need a slot (including
tokens that are already allocated).
Returns:
@@ -136,7 +144,7 @@ class SingleTypeKVCacheManager(ABC):
Args:
request: The request.
num_tokens: The total number of tokens that need to be cached
num_tokens: The total number of tokens that need to be cached
(including tokens that are already cached).
"""
num_cached_blocks = self.num_cached_block[request.request_id]
@@ -174,8 +182,9 @@ class SingleTypeKVCacheManager(ABC):
self.num_cached_block.pop(request_id, None)
@abstractmethod
def get_num_common_prefix_blocks(self, request_id: str,
num_running_requests: int) -> int:
def get_num_common_prefix_blocks(
self, request_id: str, num_running_requests: int
) -> int:
"""
Get the number of common prefix blocks for all requests in the RUNNING
state.
@@ -205,12 +214,12 @@ class SingleTypeKVCacheManager(ABC):
dcp_world_size: int = 1,
) -> tuple[list[KVCacheBlock], ...]:
"""
Get the longest cache hit prefix of the blocks that is not longer than
`max_length`. The prefix should be a common prefix hit for all the
kv cache groups in `kv_cache_group_ids`. If no cache hit is found,
return an empty list.
If eagle is enabled, drop the last matched block to force recompute the
last block to get the required hidden states for eagle drafting head.
Get the longest cache hit prefix of the blocks that is not longer than
`max_length`. The prefix should be a common prefix hit for all the
kv cache groups in `kv_cache_group_ids`. If no cache hit is found,
return an empty list.
If eagle is enabled, drop the last matched block to force recompute the
last block to get the required hidden states for eagle drafting head.
Need to be customized for each attention type.
Args:
@@ -235,10 +244,9 @@ class SingleTypeKVCacheManager(ABC):
raise NotImplementedError
@abstractmethod
def remove_skipped_blocks(self, request_id: str,
num_computed_tokens: int) -> None:
def remove_skipped_blocks(self, request_id: str, num_computed_tokens: int) -> None:
"""
Remove the blocks that are no longer needed from `blocks` and free the
Remove the blocks that are no longer needed from `blocks` and free the
blocks. The removed blocks should be replaced by null_block.
Need to be customized for each attention type.
@@ -250,7 +258,6 @@ class SingleTypeKVCacheManager(ABC):
class FullAttentionManager(SingleTypeKVCacheManager):
@classmethod
def find_longest_cache_hit(
cls,
@@ -264,10 +271,13 @@ class FullAttentionManager(SingleTypeKVCacheManager):
) -> tuple[list[KVCacheBlock], ...]:
assert isinstance(
kv_cache_spec, (FullAttentionSpec, ChunkedLocalAttentionSpec)
), "FullAttentionManager can only be used for full attention " \
), (
"FullAttentionManager can only be used for full attention "
"and chunked local attention groups"
)
computed_blocks: tuple[list[KVCacheBlock], ...] = tuple(
[] for _ in range(len(kv_cache_group_ids)))
[] for _ in range(len(kv_cache_group_ids))
)
block_size = kv_cache_spec.block_size
if dcp_world_size > 1:
block_size *= dcp_world_size
@@ -277,7 +287,8 @@ class FullAttentionManager(SingleTypeKVCacheManager):
# in the cached_block_hash_to_id, the following block hashes are
# not computed yet for sure.
if cached_block := block_pool.get_cached_block(
block_hash, kv_cache_group_ids):
block_hash, kv_cache_group_ids
):
for computed, cached in zip(computed_blocks, cached_block):
computed.append(cached)
else:
@@ -287,13 +298,13 @@ class FullAttentionManager(SingleTypeKVCacheManager):
computed.pop()
return computed_blocks
def remove_skipped_blocks(self, request_id: str,
num_computed_tokens: int) -> None:
def remove_skipped_blocks(self, request_id: str, num_computed_tokens: int) -> None:
# No need to remove blocks for full attention.
pass
def get_num_common_prefix_blocks(self, request_id: str,
num_running_requests: int) -> int:
def get_num_common_prefix_blocks(
self, request_id: str, num_running_requests: int
) -> int:
blocks = self.req_to_blocks[request_id]
num_common_blocks = 0
for block in blocks:
@@ -305,9 +316,9 @@ class FullAttentionManager(SingleTypeKVCacheManager):
class SlidingWindowManager(SingleTypeKVCacheManager):
def __init__(self, kv_cache_spec: SlidingWindowSpec, block_pool: BlockPool,
**kwargs) -> None:
def __init__(
self, kv_cache_spec: SlidingWindowSpec, block_pool: BlockPool, **kwargs
) -> None:
super().__init__(kv_cache_spec, block_pool, **kwargs)
self.sliding_window = kv_cache_spec.sliding_window
self._null_block = block_pool.null_block
@@ -324,13 +335,15 @@ class SlidingWindowManager(SingleTypeKVCacheManager):
dcp_world_size: int = 1,
) -> tuple[list[KVCacheBlock], ...]:
assert isinstance(kv_cache_spec, SlidingWindowSpec), (
"SlidingWindowManager can only be used for sliding window groups")
"SlidingWindowManager can only be used for sliding window groups"
)
assert dcp_world_size == 1, "DCP not support sliding window attn now."
# The number of contiguous blocks needed for prefix cache hit.
# -1 since the input token itself is also included in the window
sliding_window_contiguous_blocks = cdiv(
kv_cache_spec.sliding_window - 1, kv_cache_spec.block_size)
kv_cache_spec.sliding_window - 1, kv_cache_spec.block_size
)
if use_eagle:
# Need to drop the last matched block if eagle is enabled. For
# sliding window layer, we achieve this by increasing the number of
@@ -344,14 +357,17 @@ class SlidingWindowManager(SingleTypeKVCacheManager):
# sliding_window_contiguous_blocks),
# which is good for low cache hit rate scenarios.
max_num_blocks = max_length // kv_cache_spec.block_size
computed_blocks = tuple([block_pool.null_block] * max_num_blocks
for _ in range(len(kv_cache_group_ids)))
computed_blocks = tuple(
[block_pool.null_block] * max_num_blocks
for _ in range(len(kv_cache_group_ids))
)
num_contiguous_blocks = 0
match_found = False
# Search from right to left and early stop when a match is found.
for i in range(max_num_blocks - 1, -1, -1):
if cached_block := block_pool.get_cached_block(
block_hashes[i], kv_cache_group_ids):
block_hashes[i], kv_cache_group_ids
):
for computed, cached in zip(computed_blocks, cached_block):
computed[i] = cached
num_contiguous_blocks += 1
@@ -360,7 +376,7 @@ class SlidingWindowManager(SingleTypeKVCacheManager):
# E.g., [NULL, NULL, 8, 3, NULL, 9] -> [NULL, NULL, 8, 3]
# when sliding_window_contiguous_blocks=2.
for computed in computed_blocks:
del computed[i + num_contiguous_blocks:]
del computed[i + num_contiguous_blocks :]
match_found = True
break
else:
@@ -375,8 +391,7 @@ class SlidingWindowManager(SingleTypeKVCacheManager):
computed.pop()
return computed_blocks
def remove_skipped_blocks(self, request_id: str,
num_computed_tokens: int) -> None:
def remove_skipped_blocks(self, request_id: str, num_computed_tokens: int) -> None:
# Remove the blocks that are no longer be in the sliding window and
# skipped during the attention computation.
last_useful_token = num_computed_tokens - self.sliding_window + 1
@@ -393,21 +408,22 @@ class SlidingWindowManager(SingleTypeKVCacheManager):
blocks[i] = self._null_block
self.block_pool.free_blocks(removed_blocks)
def get_num_common_prefix_blocks(self, request_id: str,
num_running_requests: int) -> int:
def get_num_common_prefix_blocks(
self, request_id: str, num_running_requests: int
) -> int:
"""
NOTE(Chen): The prefix blocks are null blocks for sliding window layers.
So it's not correct to count ref_cnt like FullAttentionManager. Return
0 here for correctness. Need to support cascade attention + sliding
So it's not correct to count ref_cnt like FullAttentionManager. Return
0 here for correctness. Need to support cascade attention + sliding
window in the future.
"""
return 0
class ChunkedLocalAttentionManager(SingleTypeKVCacheManager):
def __init__(self, kv_cache_spec: ChunkedLocalAttentionSpec,
block_pool: BlockPool, **kwargs) -> None:
def __init__(
self, kv_cache_spec: ChunkedLocalAttentionSpec, block_pool: BlockPool, **kwargs
) -> None:
super().__init__(kv_cache_spec, block_pool, **kwargs)
self.attention_chunk_size = kv_cache_spec.attention_chunk_size
self._null_block = block_pool.null_block
@@ -428,19 +444,19 @@ class ChunkedLocalAttentionManager(SingleTypeKVCacheManager):
prefix of the blocks that is not longer than `max_length`. The prefix
should be a common prefix hit for all the kv cache groups in
`kv_cache_group_ids`. If no cache hit is found, return an empty list.
note we mark as computed if the whole block is outside of the local
note we mark as computed if the whole block is outside of the local
window, and set the block as null. Examples:
1. Attention chunk size of 8, block size of 4, max length of 15
for next token at 15th (zero-indexed), 8th - 14th tokens are in
the window(needs lookup), 0th - 7th are not in the window,
so they are already marked as computed. We check the complete
block3 (8th - 11th tokens), Assume block 3 is hit, we will return
for next token at 15th (zero-indexed), 8th - 14th tokens are in
the window(needs lookup), 0th - 7th are not in the window,
so they are already marked as computed. We check the complete
block3 (8th - 11th tokens), Assume block 3 is hit, we will return
[null, null, block 3], otherwise, we return [null, null]
2. Attention chunk size of 8, block size of 4, max length of 16
for next token at 16th (zero-indexed), 0th - 15th tokens are not
in the window, so they are already marked as computed.
for next token at 16th (zero-indexed), 0th - 15th tokens are not
in the window, so they are already marked as computed.
we return 4 blocks[null, null, null, null]
Args:
@@ -455,39 +471,45 @@ class ChunkedLocalAttentionManager(SingleTypeKVCacheManager):
A list of cached blocks
"""
assert isinstance(kv_cache_spec, ChunkedLocalAttentionSpec), (
"ChunkedLocalAttentionManager can only be used for " +
"chunked local attention groups")
assert use_eagle is False, ("Hybrid KV cache is not supported for " +
"eagle + chunked local attention.")
"ChunkedLocalAttentionManager can only be used for "
+ "chunked local attention groups"
)
assert use_eagle is False, (
"Hybrid KV cache is not supported for " + "eagle + chunked local attention."
)
assert dcp_world_size == 1, "DCP not support chunked local attn now."
max_num_blocks = max_length // kv_cache_spec.block_size
if max_length > 0:
local_attention_start_idx = (max_length //
kv_cache_spec.attention_chunk_size *
kv_cache_spec.attention_chunk_size)
local_attention_start_idx = (
max_length
// kv_cache_spec.attention_chunk_size
* kv_cache_spec.attention_chunk_size
)
else:
local_attention_start_idx = 0
# we marked blocks out of window as computed
# with null blocks, and blocks inside window based on cache lookup
# result [null] [null] ... [null] [hit block 1 (1st block contain
# last window)] [hit block 2] ... [hit block x]
local_attention_start_block_idx = (local_attention_start_idx //
kv_cache_spec.block_size)
local_attention_start_block_idx = (
local_attention_start_idx // kv_cache_spec.block_size
)
computed_blocks: tuple[list[KVCacheBlock], ...] = tuple(
[block_pool.null_block] * local_attention_start_block_idx
for _ in range(len(kv_cache_group_ids)))
for _ in range(len(kv_cache_group_ids))
)
for i in range(local_attention_start_block_idx, max_num_blocks):
block_hash = block_hashes[i]
if cached_block := block_pool.get_cached_block(
block_hash, kv_cache_group_ids):
block_hash, kv_cache_group_ids
):
for computed, cached in zip(computed_blocks, cached_block):
computed.append(cached)
else:
break
return computed_blocks
def remove_skipped_blocks(self, request_id: str,
num_computed_tokens: int) -> None:
def remove_skipped_blocks(self, request_id: str, num_computed_tokens: int) -> None:
# Remove the blocks that are no longer be in the chunked attention
# window and skipped during the attention computation.
@@ -499,13 +521,14 @@ class ChunkedLocalAttentionManager(SingleTypeKVCacheManager):
# is 1024. for 1023, it will be 0.
num_cached_block = self.num_cached_block.get(request_id, 0)
local_attention_start_idx = (
num_computed_tokens
) // self.attention_chunk_size * self.attention_chunk_size
(num_computed_tokens)
// self.attention_chunk_size
* self.attention_chunk_size
)
first_useful_block_idx = local_attention_start_idx // self.block_size
if num_cached_block > 0:
# Make sure we don't delete the last cached block
first_useful_block_idx = min(first_useful_block_idx,
num_cached_block - 1)
first_useful_block_idx = min(first_useful_block_idx, num_cached_block - 1)
# if block size = 128, 0 -> block 0, 1024 (= 128 * 8) ->
# block 8, 372 (= 128 * 2 + 116) -> block 2
blocks = self.req_to_blocks[request_id]
@@ -521,8 +544,9 @@ class ChunkedLocalAttentionManager(SingleTypeKVCacheManager):
blocks[i] = self._null_block
self.block_pool.free_blocks(removed_blocks)
def get_num_common_prefix_blocks(self, request_id: str,
num_running_requests: int) -> int:
def get_num_common_prefix_blocks(
self, request_id: str, num_running_requests: int
) -> int:
"""
cascade attention is not supported by chunked local attention.
"""
@@ -530,7 +554,6 @@ class ChunkedLocalAttentionManager(SingleTypeKVCacheManager):
class MambaManager(SingleTypeKVCacheManager):
@classmethod
def find_longest_cache_hit(
cls,
@@ -542,18 +565,20 @@ class MambaManager(SingleTypeKVCacheManager):
use_eagle: bool,
dcp_world_size: int = 1,
) -> tuple[list[KVCacheBlock], ...]:
assert isinstance(
kv_cache_spec,
MambaSpec), ("MambaManager can only be used for mamba groups")
assert isinstance(kv_cache_spec, MambaSpec), (
"MambaManager can only be used for mamba groups"
)
assert dcp_world_size == 1, "DCP not support mamba now."
computed_blocks: tuple[list[KVCacheBlock], ...] = tuple(
[] for _ in range(len(kv_cache_group_ids)))
[] for _ in range(len(kv_cache_group_ids))
)
max_num_blocks = max_length // kv_cache_spec.block_size
# Search from right to left and early stop when a match is found.
for i in range(max_num_blocks - 1, -1, -1):
if cached_block := block_pool.get_cached_block(
block_hashes[i], kv_cache_group_ids):
block_hashes[i], kv_cache_group_ids
):
for computed, cached in zip(computed_blocks, cached_block):
# the hit length logic later assumes:
# hit_length = len(hit_blocks_other_attn[0])
@@ -566,40 +591,46 @@ class MambaManager(SingleTypeKVCacheManager):
return computed_blocks
def remove_skipped_blocks(self, request_id: str,
num_computed_tokens: int) -> None:
def remove_skipped_blocks(self, request_id: str, num_computed_tokens: int) -> None:
# Here unused blocks may be freed up for running requests.
# TODO(@s3woz) Free up all blocks that aren't needed by Mamba2
# (for which find_longest_cache_hit returns block_pool.null_block)
pass
def get_num_common_prefix_blocks(self, request_id: str,
num_running_requests: int) -> int:
def get_num_common_prefix_blocks(
self, request_id: str, num_running_requests: int
) -> int:
"""
cascade attention is not supported by mamba
"""
return 0
def get_num_blocks_to_allocate(
self, request_id: str, num_tokens: int,
new_computed_blocks: list[KVCacheBlock]) -> int:
self, request_id: str, num_tokens: int, new_computed_blocks: list[KVCacheBlock]
) -> int:
# Allocate extra `num_speculative_blocks` blocks for
# speculative decoding (MTP/EAGLE) with linear attention.
assert isinstance(self.kv_cache_spec, MambaSpec)
if self.kv_cache_spec.num_speculative_blocks > 0:
num_tokens += (self.kv_cache_spec.block_size *
self.kv_cache_spec.num_speculative_blocks)
return super().get_num_blocks_to_allocate(request_id, num_tokens,
new_computed_blocks)
num_tokens += (
self.kv_cache_spec.block_size
* self.kv_cache_spec.num_speculative_blocks
)
return super().get_num_blocks_to_allocate(
request_id, num_tokens, new_computed_blocks
)
def allocate_new_blocks(self, request_id: str,
num_tokens: int) -> list[KVCacheBlock]:
def allocate_new_blocks(
self, request_id: str, num_tokens: int
) -> list[KVCacheBlock]:
# Allocate extra `num_speculative_blocks` blocks for
# speculative decoding (MTP/EAGLE) with linear attention.
assert isinstance(self.kv_cache_spec, MambaSpec)
if self.kv_cache_spec.num_speculative_blocks > 0:
num_tokens += (self.kv_cache_spec.block_size *
self.kv_cache_spec.num_speculative_blocks)
num_tokens += (
self.kv_cache_spec.block_size
* self.kv_cache_spec.num_speculative_blocks
)
return super().allocate_new_blocks(request_id, num_tokens)
@@ -607,8 +638,8 @@ class CrossAttentionManager(SingleTypeKVCacheManager):
"""Manager for cross-attention KV cache in encoder-decoder models."""
def save_new_computed_blocks(
self, request_id: str,
new_computed_blocks: list[KVCacheBlock]) -> None:
self, request_id: str, new_computed_blocks: list[KVCacheBlock]
) -> None:
# We do not cache blocks for cross-attention to be shared between
# requests, so `new_computed_blocks` should always be empty.
assert len(new_computed_blocks) == 0
@@ -618,8 +649,9 @@ class CrossAttentionManager(SingleTypeKVCacheManager):
# requests, so this method is not relevant.
raise ValueError("Should not be called as prefix caching is disabled.")
def get_num_common_prefix_blocks(self, request_id: str,
num_running_requests: int) -> int:
def get_num_common_prefix_blocks(
self, request_id: str, num_running_requests: int
) -> int:
# Cross-attention blocks contain request-specific encoder states
# and are not shared between different requests
return 0
@@ -644,11 +676,9 @@ class CrossAttentionManager(SingleTypeKVCacheManager):
# 2. Encoder states are computed once per request, not incrementally
# 3. No reusable prefix exists between different multimodal inputs
# Return empty blocks to indicate no cache hits
raise NotImplementedError(
"CrossAttentionManager does not support caching")
raise NotImplementedError("CrossAttentionManager does not support caching")
def remove_skipped_blocks(self, request_id: str,
num_computed_tokens: int) -> None:
def remove_skipped_blocks(self, request_id: str, num_computed_tokens: int) -> None:
# Cross-attention blocks represent encoder states which are needed
# for the entire decoding process, so no blocks should be skipped
pass
@@ -664,8 +694,9 @@ spec_manager_map: dict[type[KVCacheSpec], type[SingleTypeKVCacheManager]] = {
}
def get_manager_for_kv_cache_spec(kv_cache_spec: KVCacheSpec,
**kwargs) -> SingleTypeKVCacheManager:
def get_manager_for_kv_cache_spec(
kv_cache_spec: KVCacheSpec, **kwargs
) -> SingleTypeKVCacheManager:
manager_class = spec_manager_map[type(kv_cache_spec)]
manager = manager_class(kv_cache_spec, **kwargs)
return manager