Convert formatting to use ruff instead of yapf + isort (#26247)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
+58
-48
@@ -3,16 +3,24 @@
|
||||
from collections.abc import Iterable
|
||||
from typing import Any, Optional, Union
|
||||
|
||||
from vllm.distributed.kv_events import (MEDIUM_GPU, AllBlocksCleared,
|
||||
BlockRemoved, BlockStored,
|
||||
KVCacheEvent)
|
||||
from vllm.distributed.kv_events import (
|
||||
MEDIUM_GPU,
|
||||
AllBlocksCleared,
|
||||
BlockRemoved,
|
||||
BlockStored,
|
||||
KVCacheEvent,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.core.kv_cache_utils import (BlockHash, BlockHashWithGroupId,
|
||||
ExternalBlockHash,
|
||||
FreeKVCacheBlockQueue, KVCacheBlock,
|
||||
get_block_hash,
|
||||
make_block_hash_with_group_id,
|
||||
maybe_convert_block_hash)
|
||||
from vllm.v1.core.kv_cache_utils import (
|
||||
BlockHash,
|
||||
BlockHashWithGroupId,
|
||||
ExternalBlockHash,
|
||||
FreeKVCacheBlockQueue,
|
||||
KVCacheBlock,
|
||||
get_block_hash,
|
||||
make_block_hash_with_group_id,
|
||||
maybe_convert_block_hash,
|
||||
)
|
||||
from vllm.v1.request import Request
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -20,7 +28,7 @@ logger = init_logger(__name__)
|
||||
|
||||
class BlockHashToBlockMap:
|
||||
"""
|
||||
Cache of blocks that are used for prefix caching. It caches blocks
|
||||
Cache of blocks that are used for prefix caching. It caches blocks
|
||||
from hash directly to a block or multiple blocks
|
||||
(i.e. {block_hash: KVCacheBlocks})
|
||||
- Mostly block_hash maps to a single KVCacheBlock, and KVCacheBlocks
|
||||
@@ -42,11 +50,11 @@ class BlockHashToBlockMap:
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self._cache: dict[BlockHashWithGroupId,
|
||||
Union[KVCacheBlock, dict[int, KVCacheBlock]]] = {}
|
||||
self._cache: dict[
|
||||
BlockHashWithGroupId, Union[KVCacheBlock, dict[int, KVCacheBlock]]
|
||||
] = {}
|
||||
|
||||
def get_one_block(self,
|
||||
key: BlockHashWithGroupId) -> Optional[KVCacheBlock]:
|
||||
def get_one_block(self, key: BlockHashWithGroupId) -> Optional[KVCacheBlock]:
|
||||
"""
|
||||
Gets any block with the given block hash key.
|
||||
"""
|
||||
@@ -77,8 +85,7 @@ class BlockHashToBlockMap:
|
||||
else:
|
||||
self._unexpected_blocks_type(blocks)
|
||||
|
||||
def pop(self, key: BlockHashWithGroupId,
|
||||
block_id: int) -> Optional[KVCacheBlock]:
|
||||
def pop(self, key: BlockHashWithGroupId, block_id: int) -> Optional[KVCacheBlock]:
|
||||
"""
|
||||
Checks if block_hash exists and pop block_id from the cache
|
||||
"""
|
||||
@@ -148,8 +155,7 @@ class BlockPool:
|
||||
self.free_block_queue = FreeKVCacheBlockQueue(self.blocks)
|
||||
|
||||
# Cache for block lookup
|
||||
self.cached_block_hash_to_block: BlockHashToBlockMap = \
|
||||
BlockHashToBlockMap()
|
||||
self.cached_block_hash_to_block: BlockHashToBlockMap = BlockHashToBlockMap()
|
||||
|
||||
# To represent a placeholder block with block_id=0.
|
||||
# The ref_cnt of null_block is not maintained, needs special care to
|
||||
@@ -161,9 +167,9 @@ class BlockPool:
|
||||
self.kv_event_queue: list[KVCacheEvent] = []
|
||||
|
||||
def get_cached_block(
|
||||
self, block_hash: BlockHash,
|
||||
kv_cache_group_ids: list[int]) -> Optional[list[KVCacheBlock]]:
|
||||
"""Get the cached block by the block hash for each group in
|
||||
self, block_hash: BlockHash, kv_cache_group_ids: list[int]
|
||||
) -> Optional[list[KVCacheBlock]]:
|
||||
"""Get the cached block by the block hash for each group in
|
||||
`kv_cache_group_ids`, or None if cache miss for any group.
|
||||
If there are duplicated blocks, we return the first block in the cache.
|
||||
|
||||
@@ -177,9 +183,11 @@ class BlockPool:
|
||||
cached_blocks = []
|
||||
for group_id in kv_cache_group_ids:
|
||||
block_hash_with_group_id = make_block_hash_with_group_id(
|
||||
block_hash, group_id)
|
||||
block_hash, group_id
|
||||
)
|
||||
block = self.cached_block_hash_to_block.get_one_block(
|
||||
block_hash_with_group_id)
|
||||
block_hash_with_group_id
|
||||
)
|
||||
if not block:
|
||||
return None
|
||||
cached_blocks.append(block)
|
||||
@@ -218,17 +226,18 @@ class BlockPool:
|
||||
new_block_hashes = request.block_hashes[num_cached_blocks:]
|
||||
|
||||
new_hashes: Optional[list[ExternalBlockHash]] = (
|
||||
[] if self.enable_kv_cache_events else None)
|
||||
[] if self.enable_kv_cache_events else None
|
||||
)
|
||||
for i, blk in enumerate(new_full_blocks):
|
||||
assert blk.block_hash is None
|
||||
block_hash = new_block_hashes[i]
|
||||
|
||||
# Update and added the full block to the cache.
|
||||
block_hash_with_group_id = make_block_hash_with_group_id(
|
||||
block_hash, kv_cache_group_id)
|
||||
block_hash, kv_cache_group_id
|
||||
)
|
||||
blk.block_hash = block_hash_with_group_id
|
||||
self.cached_block_hash_to_block.insert(block_hash_with_group_id,
|
||||
blk)
|
||||
self.cached_block_hash_to_block.insert(block_hash_with_group_id, blk)
|
||||
if new_hashes is not None:
|
||||
new_hashes.append(maybe_convert_block_hash(block_hash))
|
||||
|
||||
@@ -239,20 +248,21 @@ class BlockPool:
|
||||
parent_block = blocks[num_cached_blocks - 1]
|
||||
assert parent_block.block_hash is not None
|
||||
parent_block_hash = maybe_convert_block_hash(
|
||||
get_block_hash(parent_block.block_hash))
|
||||
get_block_hash(parent_block.block_hash)
|
||||
)
|
||||
|
||||
self.kv_event_queue.append(
|
||||
BlockStored(
|
||||
block_hashes=new_hashes,
|
||||
parent_block_hash=parent_block_hash,
|
||||
token_ids=request.
|
||||
all_token_ids[num_cached_blocks *
|
||||
block_size:num_full_blocks * block_size],
|
||||
token_ids=request.all_token_ids[
|
||||
num_cached_blocks * block_size : num_full_blocks * block_size
|
||||
],
|
||||
block_size=block_size,
|
||||
lora_id=request.lora_request.id
|
||||
if request.lora_request else None,
|
||||
lora_id=request.lora_request.id if request.lora_request else None,
|
||||
medium=MEDIUM_GPU,
|
||||
))
|
||||
)
|
||||
)
|
||||
|
||||
def get_new_blocks(self, num_blocks: int) -> list[KVCacheBlock]:
|
||||
"""Get new blocks from the free block pool.
|
||||
@@ -266,8 +276,7 @@ class BlockPool:
|
||||
A list of new block.
|
||||
"""
|
||||
if num_blocks > self.get_num_free_blocks():
|
||||
raise ValueError(
|
||||
f"Cannot get {num_blocks} free blocks from the pool")
|
||||
raise ValueError(f"Cannot get {num_blocks} free blocks from the pool")
|
||||
|
||||
ret: list[KVCacheBlock] = self.free_block_queue.popleft_n(num_blocks)
|
||||
|
||||
@@ -299,8 +308,7 @@ class BlockPool:
|
||||
# The block doesn't have hash, eviction is not needed
|
||||
return False
|
||||
|
||||
if self.cached_block_hash_to_block.pop(block_hash,
|
||||
block.block_id) is None:
|
||||
if self.cached_block_hash_to_block.pop(block_hash, block.block_id) is None:
|
||||
# block not found in cached_block_hash_to_block,
|
||||
# eviction is not needed
|
||||
return False
|
||||
@@ -313,10 +321,11 @@ class BlockPool:
|
||||
# we disable hybrid kv cache manager when kv cache event is
|
||||
# enabled, so there is only one group.
|
||||
self.kv_event_queue.append(
|
||||
BlockRemoved(block_hashes=[
|
||||
maybe_convert_block_hash(get_block_hash(block_hash))
|
||||
],
|
||||
medium=MEDIUM_GPU))
|
||||
BlockRemoved(
|
||||
block_hashes=[maybe_convert_block_hash(get_block_hash(block_hash))],
|
||||
medium=MEDIUM_GPU,
|
||||
)
|
||||
)
|
||||
return True
|
||||
|
||||
def touch(self, blocks: tuple[list[KVCacheBlock], ...]) -> None:
|
||||
@@ -347,10 +356,9 @@ class BlockPool:
|
||||
blocks_list = list(ordered_blocks)
|
||||
for block in blocks_list:
|
||||
block.ref_cnt -= 1
|
||||
self.free_block_queue.append_n([
|
||||
block for block in blocks_list
|
||||
if block.ref_cnt == 0 and not block.is_null
|
||||
])
|
||||
self.free_block_queue.append_n(
|
||||
[block for block in blocks_list if block.ref_cnt == 0 and not block.is_null]
|
||||
)
|
||||
|
||||
def reset_prefix_cache(self) -> bool:
|
||||
"""Reset prefix cache. This function may be used in RLHF
|
||||
@@ -365,7 +373,9 @@ class BlockPool:
|
||||
if num_used_blocks != 1: # The null block is always marked as used
|
||||
logger.warning(
|
||||
"Failed to reset prefix cache because some "
|
||||
"blocks (%d) are not freed yet", num_used_blocks - 1)
|
||||
"blocks (%d) are not freed yet",
|
||||
num_used_blocks - 1,
|
||||
)
|
||||
return False
|
||||
|
||||
# Remove all hashes so that no new blocks will hit.
|
||||
@@ -405,7 +415,7 @@ class BlockPool:
|
||||
|
||||
def take_events(self) -> list[KVCacheEvent]:
|
||||
"""Atomically takes all events and clears the queue.
|
||||
|
||||
|
||||
Returns:
|
||||
A list of KV cache events.
|
||||
"""
|
||||
|
||||
@@ -33,12 +33,12 @@ class EncoderCacheManager:
|
||||
within requests, allowing for fine-grained memory management and enabling
|
||||
chunked processing of multimodal inputs.
|
||||
|
||||
Cache is enabled to share embeddings of same multimodal data
|
||||
item (identified by their hash value) between different requests,
|
||||
and eviction takes place at allocation time when there's no free
|
||||
Cache is enabled to share embeddings of same multimodal data
|
||||
item (identified by their hash value) between different requests,
|
||||
and eviction takes place at allocation time when there's no free
|
||||
space for new embeddings.
|
||||
Oldest cached embeddings with no request referenced will be first evicted.
|
||||
|
||||
|
||||
Args:
|
||||
cache_size: Limit the size of the cache, measured by the number of
|
||||
tokens from the input sequence.
|
||||
@@ -99,27 +99,31 @@ class EncoderCacheManager:
|
||||
self.cached[mm_hash].add(request.request_id)
|
||||
return True
|
||||
|
||||
def can_allocate(self, request: Request, input_id: int,
|
||||
encoder_compute_budget: int,
|
||||
num_tokens_to_schedule: int) -> bool:
|
||||
"""Check if there's sufficient cache space for a multimodal input.
|
||||
def can_allocate(
|
||||
self,
|
||||
request: Request,
|
||||
input_id: int,
|
||||
encoder_compute_budget: int,
|
||||
num_tokens_to_schedule: int,
|
||||
) -> bool:
|
||||
"""Check if there's sufficient cache space for a multimodal input.
|
||||
If there is, return True and update EncoderCacheManager state.
|
||||
|
||||
If there is not enough free space in `num_free_slots` but there is
|
||||
enough reclaimable space in `num_freeable_slots`, entries will be
|
||||
evicted from `freeable` (their mm_hash appended to `freed`) until
|
||||
enough space is available, and then this method returns True.
|
||||
enough space is available, and then this method returns True.
|
||||
Older entries are evicted first.
|
||||
|
||||
Returns False only if the requested number of tokens exceeds both
|
||||
|
||||
Returns False only if the requested number of tokens exceeds both
|
||||
the free and reclaimable capacities combined.
|
||||
|
||||
Args:
|
||||
request: The request containing the multimodal input.
|
||||
input_id: Index of the multimodal input within the request.
|
||||
encoder_compute_budget: Number of encoder tokens allowed to be
|
||||
encoder_compute_budget: Number of encoder tokens allowed to be
|
||||
computed when this method is invoked.
|
||||
num_tokens_to_schedule: Number of tokens already scheduled to be
|
||||
num_tokens_to_schedule: Number of tokens already scheduled to be
|
||||
allocated with cache space when this method is invoked.
|
||||
|
||||
Returns:
|
||||
@@ -127,7 +131,7 @@ class EncoderCacheManager:
|
||||
input (possibly after reclaiming `freeable` entries); otherwise
|
||||
False.
|
||||
|
||||
Note: This method does not allocate physical memory for the encoder
|
||||
Note: This method does not allocate physical memory for the encoder
|
||||
output but only the state of EncoderCacheManager.
|
||||
"""
|
||||
num_tokens = request.get_num_encoder_tokens(input_id)
|
||||
@@ -202,7 +206,7 @@ class EncoderCacheManager:
|
||||
|
||||
When the reference set for the corresponding `mm_hash` becomes empty,
|
||||
the entry is appended to `freeable` and `num_freeable_slots` is
|
||||
increased by the number of encoder tokens for that input.
|
||||
increased by the number of encoder tokens for that input.
|
||||
|
||||
The entry is NOT physically freed until capacity is needed (e.g., by
|
||||
`can_allocate`).
|
||||
@@ -221,8 +225,8 @@ class EncoderCacheManager:
|
||||
def free(self, request: Request) -> None:
|
||||
"""Free all encoder input cache reference held by *request*.
|
||||
|
||||
For each cached input ID, `free_encoder_input` is invoked.
|
||||
The data stays in memory until eviction is triggered by a future
|
||||
For each cached input ID, `free_encoder_input` is invoked.
|
||||
The data stays in memory until eviction is triggered by a future
|
||||
attempt allocation called by 'can_allocate'.
|
||||
|
||||
Typically called when a request is finished, cancelled, or aborted.
|
||||
@@ -236,9 +240,9 @@ class EncoderCacheManager:
|
||||
|
||||
Returns:
|
||||
List of mm_hash strings that were actually evicted since the last
|
||||
call to be used by the scheduler to notify workers about which
|
||||
encoder outputs can be removed from their caches. The internal
|
||||
list is cleared after this call.
|
||||
call to be used by the scheduler to notify workers about which
|
||||
encoder outputs can be removed from their caches. The internal
|
||||
list is cleared after this call.
|
||||
"""
|
||||
freed = self.freed
|
||||
self.freed = []
|
||||
@@ -250,7 +254,7 @@ def compute_encoder_budget(
|
||||
scheduler_config: "SchedulerConfig",
|
||||
mm_registry: MultiModalRegistry,
|
||||
) -> tuple[int, int]:
|
||||
"""Compute the encoder cache budget based on the model and scheduler
|
||||
"""Compute the encoder cache budget based on the model and scheduler
|
||||
configurations.
|
||||
|
||||
Returns:
|
||||
@@ -260,8 +264,9 @@ def compute_encoder_budget(
|
||||
from the input sequence.
|
||||
"""
|
||||
if mm_registry.supports_multimodal_inputs(model_config):
|
||||
max_tokens_by_modality = mm_registry \
|
||||
.get_max_tokens_per_item_by_nonzero_modality(model_config)
|
||||
max_tokens_by_modality = (
|
||||
mm_registry.get_max_tokens_per_item_by_nonzero_modality(model_config)
|
||||
)
|
||||
|
||||
return compute_mm_encoder_budget(
|
||||
scheduler_config,
|
||||
@@ -271,18 +276,17 @@ def compute_encoder_budget(
|
||||
return compute_text_encoder_budget(scheduler_config)
|
||||
|
||||
|
||||
def compute_text_encoder_budget(
|
||||
scheduler_config: "SchedulerConfig") -> tuple[int, int]:
|
||||
"""Compute the encoder cache budget based on the model and scheduler
|
||||
def compute_text_encoder_budget(scheduler_config: "SchedulerConfig") -> tuple[int, int]:
|
||||
"""Compute the encoder cache budget based on the model and scheduler
|
||||
configurations for a text-only model.
|
||||
|
||||
Args:
|
||||
scheduler_config: Scheduler configuration.
|
||||
|
||||
Returns:
|
||||
- Compute budget for encoder execution, in unit of number of tokens
|
||||
- Compute budget for encoder execution, in unit of number of tokens
|
||||
in the input sequence.
|
||||
- Space budget for encoder cache size, in unit of number of tokens
|
||||
- Space budget for encoder cache size, in unit of number of tokens
|
||||
in the input sequence.
|
||||
"""
|
||||
# Currently text-only encoder-decoder models are not supported
|
||||
@@ -293,7 +297,7 @@ def compute_mm_encoder_budget(
|
||||
scheduler_config: "SchedulerConfig",
|
||||
max_tokens_by_modality: Mapping[str, int],
|
||||
) -> tuple[int, int]:
|
||||
"""Compute the encoder cache budget based on the model and scheduler
|
||||
"""Compute the encoder cache budget based on the model and scheduler
|
||||
configurations for a multimodal model.
|
||||
|
||||
Args:
|
||||
@@ -312,22 +316,28 @@ def compute_mm_encoder_budget(
|
||||
logger.warning(
|
||||
"All non-text modalities supported by the model have been "
|
||||
"explicitly disabled via limit_mm_per_prompt. Encoder cache will "
|
||||
"not be initialized.")
|
||||
"not be initialized."
|
||||
)
|
||||
return 0, 0
|
||||
|
||||
max_tokens_per_mm_item = max(max_tokens_by_modality.values())
|
||||
|
||||
if (scheduler_config.disable_chunked_mm_input and max_tokens_per_mm_item
|
||||
> scheduler_config.max_num_batched_tokens):
|
||||
if (
|
||||
scheduler_config.disable_chunked_mm_input
|
||||
and max_tokens_per_mm_item > scheduler_config.max_num_batched_tokens
|
||||
):
|
||||
raise ValueError(
|
||||
"Chunked MM input disabled but max_tokens_per_mm_item "
|
||||
f"({max_tokens_per_mm_item}) is larger than max_num_batched_tokens"
|
||||
f" ({scheduler_config.max_num_batched_tokens}). Please increase "
|
||||
"max_num_batched_tokens.")
|
||||
"max_num_batched_tokens."
|
||||
)
|
||||
|
||||
encoder_compute_budget = max(scheduler_config.max_num_encoder_input_tokens,
|
||||
max_tokens_per_mm_item)
|
||||
encoder_cache_size = max(scheduler_config.encoder_cache_size,
|
||||
max_tokens_per_mm_item)
|
||||
encoder_compute_budget = max(
|
||||
scheduler_config.max_num_encoder_input_tokens, max_tokens_per_mm_item
|
||||
)
|
||||
encoder_cache_size = max(
|
||||
scheduler_config.encoder_cache_size, max_tokens_per_mm_item
|
||||
)
|
||||
|
||||
return encoder_compute_budget, encoder_cache_size
|
||||
|
||||
@@ -6,9 +6,11 @@ from typing import Optional
|
||||
from vllm.v1.core.block_pool import BlockPool
|
||||
from vllm.v1.core.kv_cache_utils import BlockHash, KVCacheBlock
|
||||
from vllm.v1.core.single_type_kv_cache_manager import (
|
||||
CrossAttentionManager, FullAttentionManager, get_manager_for_kv_cache_spec)
|
||||
from vllm.v1.kv_cache_interface import (FullAttentionSpec, KVCacheConfig,
|
||||
KVCacheSpec)
|
||||
CrossAttentionManager,
|
||||
FullAttentionManager,
|
||||
get_manager_for_kv_cache_spec,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheConfig, KVCacheSpec
|
||||
from vllm.v1.request import Request
|
||||
|
||||
|
||||
@@ -30,8 +32,9 @@ class KVCacheCoordinator(ABC):
|
||||
self.max_model_len = max_model_len
|
||||
self.enable_caching = enable_caching
|
||||
|
||||
self.block_pool = BlockPool(kv_cache_config.num_blocks, enable_caching,
|
||||
enable_kv_cache_events)
|
||||
self.block_pool = BlockPool(
|
||||
kv_cache_config.num_blocks, enable_caching, enable_kv_cache_events
|
||||
)
|
||||
|
||||
# Needs special handling for find_longest_cache_hit if eagle is enabled
|
||||
self.use_eagle = use_eagle
|
||||
@@ -41,19 +44,23 @@ class KVCacheCoordinator(ABC):
|
||||
block_pool=self.block_pool,
|
||||
kv_cache_group_id=i,
|
||||
dcp_world_size=dcp_world_size,
|
||||
) for i, kv_cache_group in enumerate(
|
||||
self.kv_cache_config.kv_cache_groups))
|
||||
)
|
||||
for i, kv_cache_group in enumerate(self.kv_cache_config.kv_cache_groups)
|
||||
)
|
||||
|
||||
def get_num_blocks_to_allocate(self, request_id: str, num_tokens: int,
|
||||
new_computed_blocks: tuple[
|
||||
list[KVCacheBlock], ...],
|
||||
num_encoder_tokens: int) -> int:
|
||||
def get_num_blocks_to_allocate(
|
||||
self,
|
||||
request_id: str,
|
||||
num_tokens: int,
|
||||
new_computed_blocks: tuple[list[KVCacheBlock], ...],
|
||||
num_encoder_tokens: int,
|
||||
) -> int:
|
||||
"""
|
||||
Get the number of blocks needed to be allocated for the request.
|
||||
|
||||
Args:
|
||||
request_id: The request ID.
|
||||
num_tokens: The total number of tokens that need a slot (including
|
||||
num_tokens: The total number of tokens that need a slot (including
|
||||
tokens that are already allocated).
|
||||
new_computed_blocks: The new computed blocks just hitting the
|
||||
prefix caching.
|
||||
@@ -69,15 +76,17 @@ class KVCacheCoordinator(ABC):
|
||||
# For cross-attention, we issue a single static allocation
|
||||
# of blocks based on the number of encoder input tokens.
|
||||
num_blocks_to_allocate += manager.get_num_blocks_to_allocate(
|
||||
request_id, num_encoder_tokens, [])
|
||||
request_id, num_encoder_tokens, []
|
||||
)
|
||||
else:
|
||||
num_blocks_to_allocate += manager.get_num_blocks_to_allocate(
|
||||
request_id, num_tokens, new_computed_blocks[i])
|
||||
request_id, num_tokens, new_computed_blocks[i]
|
||||
)
|
||||
return num_blocks_to_allocate
|
||||
|
||||
def save_new_computed_blocks(
|
||||
self, request_id: str,
|
||||
new_computed_blocks: tuple[list[KVCacheBlock], ...]) -> None:
|
||||
self, request_id: str, new_computed_blocks: tuple[list[KVCacheBlock], ...]
|
||||
) -> None:
|
||||
"""
|
||||
Add the new computed blocks to the request.
|
||||
|
||||
@@ -87,21 +96,18 @@ class KVCacheCoordinator(ABC):
|
||||
prefix cache.
|
||||
"""
|
||||
for i, manager in enumerate(self.single_type_managers):
|
||||
manager.save_new_computed_blocks(request_id,
|
||||
new_computed_blocks[i])
|
||||
manager.save_new_computed_blocks(request_id, new_computed_blocks[i])
|
||||
|
||||
def allocate_new_blocks(
|
||||
self,
|
||||
request_id: str,
|
||||
num_tokens: int,
|
||||
num_encoder_tokens: int = 0) -> tuple[list[KVCacheBlock], ...]:
|
||||
self, request_id: str, num_tokens: int, num_encoder_tokens: int = 0
|
||||
) -> tuple[list[KVCacheBlock], ...]:
|
||||
"""
|
||||
Allocate new blocks for the request to give it at least `num_tokens`
|
||||
Allocate new blocks for the request to give it at least `num_tokens`
|
||||
token slots.
|
||||
|
||||
Args:
|
||||
request_id: The request ID.
|
||||
num_tokens: The total number of tokens that need a slot (including
|
||||
num_tokens: The total number of tokens that need a slot (including
|
||||
tokens that are already allocated).
|
||||
num_encoder_tokens: The number of encoder tokens for allocating
|
||||
blocks for cross-attention.
|
||||
@@ -111,9 +117,13 @@ class KVCacheCoordinator(ABC):
|
||||
"""
|
||||
return tuple(
|
||||
manager.allocate_new_blocks(
|
||||
request_id, num_encoder_tokens if isinstance(
|
||||
manager, CrossAttentionManager) else num_tokens)
|
||||
for manager in self.single_type_managers)
|
||||
request_id,
|
||||
num_encoder_tokens
|
||||
if isinstance(manager, CrossAttentionManager)
|
||||
else num_tokens,
|
||||
)
|
||||
for manager in self.single_type_managers
|
||||
)
|
||||
|
||||
def cache_blocks(self, request: Request, num_computed_tokens: int) -> None:
|
||||
"""
|
||||
@@ -138,8 +148,9 @@ class KVCacheCoordinator(ABC):
|
||||
for manager in self.single_type_managers:
|
||||
manager.free(request_id)
|
||||
|
||||
def get_num_common_prefix_blocks(self, request_id: str,
|
||||
num_running_requests: int) -> list[int]:
|
||||
def get_num_common_prefix_blocks(
|
||||
self, request_id: str, num_running_requests: int
|
||||
) -> list[int]:
|
||||
"""
|
||||
Get the number of common prefix blocks for all requests in the RUNNING
|
||||
state for each kv cache group.
|
||||
@@ -154,16 +165,14 @@ class KVCacheCoordinator(ABC):
|
||||
the RUNNING state for each kv cache group.
|
||||
"""
|
||||
num_blocks_per_group = [
|
||||
manager.get_num_common_prefix_blocks(request_id,
|
||||
num_running_requests)
|
||||
manager.get_num_common_prefix_blocks(request_id, num_running_requests)
|
||||
for manager in self.single_type_managers
|
||||
]
|
||||
return num_blocks_per_group
|
||||
|
||||
def remove_skipped_blocks(self, request_id: str,
|
||||
num_computed_tokens: int) -> None:
|
||||
def remove_skipped_blocks(self, request_id: str, num_computed_tokens: int) -> None:
|
||||
"""
|
||||
Remove the blocks that are no longer needed from `blocks` and replace
|
||||
Remove the blocks that are no longer needed from `blocks` and replace
|
||||
the removed blocks with null_block.
|
||||
|
||||
Args:
|
||||
@@ -179,7 +188,8 @@ class KVCacheCoordinator(ABC):
|
||||
"""
|
||||
return tuple(
|
||||
manager.req_to_blocks.get(request_id) or []
|
||||
for manager in self.single_type_managers)
|
||||
for manager in self.single_type_managers
|
||||
)
|
||||
|
||||
@abstractmethod
|
||||
def find_longest_cache_hit(
|
||||
@@ -198,19 +208,27 @@ class KVCacheCoordinatorNoPrefixCache(KVCacheCoordinator):
|
||||
Does not implement any features related to prefix caching.
|
||||
"""
|
||||
|
||||
def __init__(self, kv_cache_config: KVCacheConfig, max_model_len: int,
|
||||
use_eagle: bool, enable_kv_cache_events: bool,
|
||||
dcp_world_size: int):
|
||||
super().__init__(kv_cache_config,
|
||||
max_model_len,
|
||||
use_eagle,
|
||||
False,
|
||||
enable_kv_cache_events,
|
||||
dcp_world_size=dcp_world_size)
|
||||
def __init__(
|
||||
self,
|
||||
kv_cache_config: KVCacheConfig,
|
||||
max_model_len: int,
|
||||
use_eagle: bool,
|
||||
enable_kv_cache_events: bool,
|
||||
dcp_world_size: int,
|
||||
):
|
||||
super().__init__(
|
||||
kv_cache_config,
|
||||
max_model_len,
|
||||
use_eagle,
|
||||
False,
|
||||
enable_kv_cache_events,
|
||||
dcp_world_size=dcp_world_size,
|
||||
)
|
||||
self.num_single_type_manager = len(self.single_type_managers)
|
||||
|
||||
def get_num_common_prefix_blocks(self, request_id: str,
|
||||
num_running_requests: int) -> list[int]:
|
||||
def get_num_common_prefix_blocks(
|
||||
self, request_id: str, num_running_requests: int
|
||||
) -> list[int]:
|
||||
return [0] * self.num_single_type_manager
|
||||
|
||||
def find_longest_cache_hit(
|
||||
@@ -219,7 +237,8 @@ class KVCacheCoordinatorNoPrefixCache(KVCacheCoordinator):
|
||||
max_cache_hit_length: int,
|
||||
) -> tuple[tuple[list[KVCacheBlock], ...], int]:
|
||||
blocks: tuple[list[KVCacheBlock], ...] = tuple(
|
||||
[] for _ in range(self.num_single_type_manager))
|
||||
[] for _ in range(self.num_single_type_manager)
|
||||
)
|
||||
return blocks, 0
|
||||
|
||||
|
||||
@@ -230,23 +249,31 @@ class UnitaryKVCacheCoordinator(KVCacheCoordinator):
|
||||
full attention or all attention layers use sliding window attention.
|
||||
"""
|
||||
|
||||
def __init__(self, kv_cache_config: KVCacheConfig, max_model_len: int,
|
||||
use_eagle: bool, enable_caching: bool,
|
||||
enable_kv_cache_events: bool, dcp_world_size: int):
|
||||
super().__init__(kv_cache_config,
|
||||
max_model_len,
|
||||
use_eagle,
|
||||
enable_caching,
|
||||
enable_kv_cache_events,
|
||||
dcp_world_size=dcp_world_size)
|
||||
self.kv_cache_spec = self.kv_cache_config.kv_cache_groups[
|
||||
0].kv_cache_spec
|
||||
def __init__(
|
||||
self,
|
||||
kv_cache_config: KVCacheConfig,
|
||||
max_model_len: int,
|
||||
use_eagle: bool,
|
||||
enable_caching: bool,
|
||||
enable_kv_cache_events: bool,
|
||||
dcp_world_size: int,
|
||||
):
|
||||
super().__init__(
|
||||
kv_cache_config,
|
||||
max_model_len,
|
||||
use_eagle,
|
||||
enable_caching,
|
||||
enable_kv_cache_events,
|
||||
dcp_world_size=dcp_world_size,
|
||||
)
|
||||
self.kv_cache_spec = self.kv_cache_config.kv_cache_groups[0].kv_cache_spec
|
||||
self.block_size = self.kv_cache_spec.block_size
|
||||
self.dcp_world_size = dcp_world_size
|
||||
if dcp_world_size > 1:
|
||||
self.block_size *= dcp_world_size
|
||||
assert len(self.kv_cache_config.kv_cache_groups) == 1, (
|
||||
"UnitaryKVCacheCoordinator assumes only one kv cache group")
|
||||
"UnitaryKVCacheCoordinator assumes only one kv cache group"
|
||||
)
|
||||
|
||||
def find_longest_cache_hit(
|
||||
self,
|
||||
@@ -269,26 +296,34 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
|
||||
"""
|
||||
KV cache coordinator for hybrid models with multiple KV cache types, and
|
||||
thus multiple kv cache groups.
|
||||
To simplify `find_longest_cache_hit`, it only supports the combination of
|
||||
To simplify `find_longest_cache_hit`, it only supports the combination of
|
||||
two types of KV cache groups, and one of them must be full attention.
|
||||
May extend to more general cases in the future.
|
||||
"""
|
||||
|
||||
def __init__(self, kv_cache_config: KVCacheConfig, max_model_len: int,
|
||||
use_eagle: bool, enable_caching: bool,
|
||||
enable_kv_cache_events: bool, dcp_world_size: int):
|
||||
super().__init__(kv_cache_config,
|
||||
max_model_len,
|
||||
use_eagle,
|
||||
enable_caching,
|
||||
enable_kv_cache_events,
|
||||
dcp_world_size=dcp_world_size)
|
||||
def __init__(
|
||||
self,
|
||||
kv_cache_config: KVCacheConfig,
|
||||
max_model_len: int,
|
||||
use_eagle: bool,
|
||||
enable_caching: bool,
|
||||
enable_kv_cache_events: bool,
|
||||
dcp_world_size: int,
|
||||
):
|
||||
super().__init__(
|
||||
kv_cache_config,
|
||||
max_model_len,
|
||||
use_eagle,
|
||||
enable_caching,
|
||||
enable_kv_cache_events,
|
||||
dcp_world_size=dcp_world_size,
|
||||
)
|
||||
assert dcp_world_size == 1, "DCP not support hybrid attn now."
|
||||
self.verify_and_split_kv_cache_groups()
|
||||
|
||||
def verify_and_split_kv_cache_groups(self) -> None:
|
||||
"""
|
||||
Verifies that the model has exactly two types of KV cache groups, and
|
||||
Verifies that the model has exactly two types of KV cache groups, and
|
||||
one of them is full attention. Then, split the kv cache groups into full
|
||||
attention groups and other groups.
|
||||
"""
|
||||
@@ -303,7 +338,8 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
|
||||
else:
|
||||
assert full_attention_spec == g.kv_cache_spec, (
|
||||
"HybridKVCacheCoordinator assumes exactly one type of "
|
||||
"full attention groups now.")
|
||||
"full attention groups now."
|
||||
)
|
||||
self.full_attention_group_ids.append(i)
|
||||
else:
|
||||
if other_spec is None:
|
||||
@@ -311,19 +347,22 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
|
||||
else:
|
||||
assert other_spec == g.kv_cache_spec, (
|
||||
"HybridKVCacheCoordinator assumes "
|
||||
"exactly one other type of groups now.")
|
||||
"exactly one other type of groups now."
|
||||
)
|
||||
self.other_group_ids.append(i)
|
||||
|
||||
assert full_attention_spec is not None, (
|
||||
"HybridKVCacheCoordinator assumes exactly one type of full "
|
||||
"attention groups now.")
|
||||
"attention groups now."
|
||||
)
|
||||
assert other_spec is not None, (
|
||||
"HybridKVCacheCoordinator assumes exactly one type of other "
|
||||
"groups now.")
|
||||
"HybridKVCacheCoordinator assumes exactly one type of other groups now."
|
||||
)
|
||||
|
||||
self.full_attention_manager_cls = FullAttentionManager
|
||||
self.other_attention_cls = self.single_type_managers[
|
||||
self.other_group_ids[0]].__class__
|
||||
self.other_group_ids[0]
|
||||
].__class__
|
||||
self.full_attention_spec = full_attention_spec
|
||||
self.other_spec = other_spec
|
||||
self.full_attention_block_size = self.full_attention_spec.block_size
|
||||
@@ -334,7 +373,8 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
|
||||
divisible = self.other_block_size % self.full_attention_block_size
|
||||
assert divisible == 0, (
|
||||
"KVCacheCoordinator assumes the block_size of full "
|
||||
"attention layers is divisible by other layers now.")
|
||||
"attention layers is divisible by other layers now."
|
||||
)
|
||||
|
||||
if max(self.full_attention_group_ids) < min(self.other_group_ids):
|
||||
self.full_attn_first = True
|
||||
@@ -347,7 +387,8 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
|
||||
"do not interleave, either full attention group ids "
|
||||
"are before other attention group ids or vice versa."
|
||||
"This is for simplifying merging hit_blocks_full_attn and "
|
||||
"hit_blocks_other_attn to hit_blocks.")
|
||||
"hit_blocks_other_attn to hit_blocks."
|
||||
)
|
||||
|
||||
def find_longest_cache_hit(
|
||||
self,
|
||||
@@ -367,29 +408,26 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
|
||||
- The number of tokens of the longest cache hit.
|
||||
"""
|
||||
# First, find the longest cache hit for full attention.
|
||||
hit_blocks_full_attn = (
|
||||
self.full_attention_manager_cls.find_longest_cache_hit(
|
||||
block_hashes=block_hashes,
|
||||
max_length=max_cache_hit_length,
|
||||
kv_cache_group_ids=self.full_attention_group_ids,
|
||||
block_pool=self.block_pool,
|
||||
kv_cache_spec=self.full_attention_spec,
|
||||
use_eagle=self.use_eagle,
|
||||
))
|
||||
hit_length = len(
|
||||
hit_blocks_full_attn[0]) * self.full_attention_block_size
|
||||
hit_blocks_full_attn = self.full_attention_manager_cls.find_longest_cache_hit(
|
||||
block_hashes=block_hashes,
|
||||
max_length=max_cache_hit_length,
|
||||
kv_cache_group_ids=self.full_attention_group_ids,
|
||||
block_pool=self.block_pool,
|
||||
kv_cache_spec=self.full_attention_spec,
|
||||
use_eagle=self.use_eagle,
|
||||
)
|
||||
hit_length = len(hit_blocks_full_attn[0]) * self.full_attention_block_size
|
||||
|
||||
# Next, find the cache hit for the other attention WITHIN
|
||||
# the cache hit of full attention.
|
||||
hit_blocks_other_attn = (
|
||||
self.other_attention_cls.find_longest_cache_hit(
|
||||
block_hashes=block_hashes,
|
||||
max_length=hit_length,
|
||||
kv_cache_group_ids=self.other_group_ids,
|
||||
block_pool=self.block_pool,
|
||||
kv_cache_spec=self.other_spec,
|
||||
use_eagle=self.use_eagle,
|
||||
))
|
||||
hit_blocks_other_attn = self.other_attention_cls.find_longest_cache_hit(
|
||||
block_hashes=block_hashes,
|
||||
max_length=hit_length,
|
||||
kv_cache_group_ids=self.other_group_ids,
|
||||
block_pool=self.block_pool,
|
||||
kv_cache_spec=self.other_spec,
|
||||
use_eagle=self.use_eagle,
|
||||
)
|
||||
hit_length = len(hit_blocks_other_attn[0]) * self.other_block_size
|
||||
|
||||
# NOTE: the prefix cache hit length must be a multiple of block_size as
|
||||
@@ -404,7 +442,7 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
|
||||
# Truncate the full attention cache hit to the length of the
|
||||
# cache hit of the other attention.
|
||||
for group_hit_blocks in hit_blocks_full_attn:
|
||||
del group_hit_blocks[hit_length // self.full_attention_block_size:]
|
||||
del group_hit_blocks[hit_length // self.full_attention_block_size :]
|
||||
|
||||
# Merge the hit blocks of full attention and other attention.
|
||||
if self.full_attn_first:
|
||||
@@ -414,27 +452,36 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
|
||||
return hit_blocks, hit_length
|
||||
|
||||
|
||||
def get_kv_cache_coordinator(kv_cache_config: KVCacheConfig,
|
||||
max_model_len: int, use_eagle: bool,
|
||||
enable_caching: bool,
|
||||
enable_kv_cache_events: bool,
|
||||
dcp_world_size: int) -> KVCacheCoordinator:
|
||||
def get_kv_cache_coordinator(
|
||||
kv_cache_config: KVCacheConfig,
|
||||
max_model_len: int,
|
||||
use_eagle: bool,
|
||||
enable_caching: bool,
|
||||
enable_kv_cache_events: bool,
|
||||
dcp_world_size: int,
|
||||
) -> KVCacheCoordinator:
|
||||
if not enable_caching:
|
||||
return KVCacheCoordinatorNoPrefixCache(kv_cache_config,
|
||||
max_model_len,
|
||||
use_eagle,
|
||||
enable_kv_cache_events,
|
||||
dcp_world_size=dcp_world_size)
|
||||
return KVCacheCoordinatorNoPrefixCache(
|
||||
kv_cache_config,
|
||||
max_model_len,
|
||||
use_eagle,
|
||||
enable_kv_cache_events,
|
||||
dcp_world_size=dcp_world_size,
|
||||
)
|
||||
if len(kv_cache_config.kv_cache_groups) == 1:
|
||||
return UnitaryKVCacheCoordinator(kv_cache_config,
|
||||
max_model_len,
|
||||
use_eagle,
|
||||
enable_caching,
|
||||
enable_kv_cache_events,
|
||||
dcp_world_size=dcp_world_size)
|
||||
return HybridKVCacheCoordinator(kv_cache_config,
|
||||
max_model_len,
|
||||
use_eagle,
|
||||
enable_caching,
|
||||
enable_kv_cache_events,
|
||||
dcp_world_size=dcp_world_size)
|
||||
return UnitaryKVCacheCoordinator(
|
||||
kv_cache_config,
|
||||
max_model_len,
|
||||
use_eagle,
|
||||
enable_caching,
|
||||
enable_kv_cache_events,
|
||||
dcp_world_size=dcp_world_size,
|
||||
)
|
||||
return HybridKVCacheCoordinator(
|
||||
kv_cache_config,
|
||||
max_model_len,
|
||||
use_eagle,
|
||||
enable_caching,
|
||||
enable_kv_cache_events,
|
||||
dcp_world_size=dcp_world_size,
|
||||
)
|
||||
|
||||
@@ -22,6 +22,7 @@ class KVCacheBlocks:
|
||||
Scheduler and KVCacheManager, to hide KVCacheManager's internal data
|
||||
structure from the Scheduler.
|
||||
"""
|
||||
|
||||
blocks: tuple[list[KVCacheBlock], ...]
|
||||
"""
|
||||
`blocks[i][j]` refers to the i-th kv_cache_group
|
||||
@@ -35,22 +36,20 @@ class KVCacheBlocks:
|
||||
def __add__(self, other: "KVCacheBlocks") -> "KVCacheBlocks":
|
||||
"""Adds two KVCacheBlocks instances."""
|
||||
return KVCacheBlocks(
|
||||
tuple(blk1 + blk2
|
||||
for blk1, blk2 in zip(self.blocks, other.blocks)))
|
||||
tuple(blk1 + blk2 for blk1, blk2 in zip(self.blocks, other.blocks))
|
||||
)
|
||||
|
||||
@overload
|
||||
def get_block_ids(
|
||||
self,
|
||||
allow_none: Literal[False] = False,
|
||||
) -> tuple[list[int], ...]:
|
||||
...
|
||||
) -> tuple[list[int], ...]: ...
|
||||
|
||||
@overload
|
||||
def get_block_ids(
|
||||
self,
|
||||
allow_none: Literal[True] = True,
|
||||
) -> Optional[tuple[list[int], ...]]:
|
||||
...
|
||||
) -> Optional[tuple[list[int], ...]]: ...
|
||||
|
||||
def get_block_ids(
|
||||
self,
|
||||
@@ -72,10 +71,7 @@ class KVCacheBlocks:
|
||||
def get_unhashed_block_ids(self) -> list[int]:
|
||||
"""Get block_ids of unhashed blocks from KVCacheBlocks instance."""
|
||||
assert len(self.blocks) == 1, "Only one group is supported"
|
||||
return [
|
||||
block.block_id for block in self.blocks[0]
|
||||
if block.block_hash is None
|
||||
]
|
||||
return [block.block_id for block in self.blocks[0] if block.block_hash is None]
|
||||
|
||||
def new_empty(self) -> "KVCacheBlocks":
|
||||
"""Creates a new KVCacheBlocks instance with no blocks."""
|
||||
@@ -83,7 +79,6 @@ class KVCacheBlocks:
|
||||
|
||||
|
||||
class KVCacheManager:
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
kv_cache_config: KVCacheConfig,
|
||||
@@ -104,12 +99,18 @@ class KVCacheManager:
|
||||
|
||||
self.block_size: Optional[int] = None
|
||||
if self.enable_caching:
|
||||
assert len(
|
||||
set(g.kv_cache_spec.block_size
|
||||
for g in kv_cache_config.kv_cache_groups)
|
||||
) == 1, "Only one block size is supported for now"
|
||||
assert (
|
||||
len(
|
||||
set(
|
||||
g.kv_cache_spec.block_size
|
||||
for g in kv_cache_config.kv_cache_groups
|
||||
)
|
||||
)
|
||||
== 1
|
||||
), "Only one block size is supported for now"
|
||||
self.block_size = kv_cache_config.kv_cache_groups[
|
||||
0].kv_cache_spec.block_size
|
||||
0
|
||||
].kv_cache_spec.block_size
|
||||
|
||||
if dcp_world_size > 1:
|
||||
assert len(kv_cache_config.kv_cache_groups) == 1
|
||||
@@ -151,8 +152,7 @@ class KVCacheManager:
|
||||
self.prefix_cache_stats = PrefixCacheStats()
|
||||
return stats
|
||||
|
||||
def get_computed_blocks(self,
|
||||
request: Request) -> tuple[KVCacheBlocks, int]:
|
||||
def get_computed_blocks(self, request: Request) -> tuple[KVCacheBlocks, int]:
|
||||
"""Get the computed (cached) blocks for the request.
|
||||
Note that the computed blocks must be full.
|
||||
|
||||
@@ -166,9 +166,10 @@ class KVCacheManager:
|
||||
"""
|
||||
# Prefix caching is disabled or
|
||||
# When the request requires prompt logprobs, we skip prefix caching.
|
||||
if (not self.enable_caching
|
||||
or (request.sampling_params is not None
|
||||
and request.sampling_params.prompt_logprobs is not None)):
|
||||
if not self.enable_caching or (
|
||||
request.sampling_params is not None
|
||||
and request.sampling_params.prompt_logprobs is not None
|
||||
):
|
||||
return self.create_empty_block_list(), 0
|
||||
|
||||
# NOTE: When all tokens hit the cache, we must recompute the last token
|
||||
@@ -179,8 +180,10 @@ class KVCacheManager:
|
||||
# could slightly improve performance in the future.
|
||||
max_cache_hit_length = request.num_tokens - 1
|
||||
computed_blocks, num_new_computed_tokens = (
|
||||
self.coordinator.find_longest_cache_hit(request.block_hashes,
|
||||
max_cache_hit_length))
|
||||
self.coordinator.find_longest_cache_hit(
|
||||
request.block_hashes, max_cache_hit_length
|
||||
)
|
||||
)
|
||||
|
||||
if self.log_stats:
|
||||
assert self.prefix_cache_stats is not None
|
||||
@@ -188,8 +191,7 @@ class KVCacheManager:
|
||||
# Previously preempted request
|
||||
self.prefix_cache_stats.preempted_requests += 1
|
||||
self.prefix_cache_stats.preempted_queries += request.num_tokens
|
||||
self.prefix_cache_stats.preempted_hits += (
|
||||
num_new_computed_tokens)
|
||||
self.prefix_cache_stats.preempted_hits += num_new_computed_tokens
|
||||
else:
|
||||
# New request
|
||||
self.prefix_cache_stats.requests += 1
|
||||
@@ -250,7 +252,8 @@ class KVCacheManager:
|
||||
new_computed_block_list = new_computed_blocks.blocks
|
||||
else:
|
||||
new_computed_block_list = tuple(
|
||||
[] for _ in range(len(self.kv_cache_config.kv_cache_groups)))
|
||||
[] for _ in range(len(self.kv_cache_config.kv_cache_groups))
|
||||
)
|
||||
|
||||
# Free the blocks that are skipped during the attention computation
|
||||
# (e.g., tokens outside the sliding window).
|
||||
@@ -258,16 +261,17 @@ class KVCacheManager:
|
||||
# insufficient free blocks.
|
||||
# Should call this function before allocating new blocks to reduce
|
||||
# the number of evicted blocks.
|
||||
self.coordinator.remove_skipped_blocks(request.request_id,
|
||||
request.num_computed_tokens)
|
||||
self.coordinator.remove_skipped_blocks(
|
||||
request.request_id, request.num_computed_tokens
|
||||
)
|
||||
|
||||
# The number of computed tokens is the number of computed tokens plus
|
||||
# the new prefix caching hits
|
||||
num_computed_tokens = (request.num_computed_tokens +
|
||||
num_new_computed_tokens)
|
||||
num_computed_tokens = request.num_computed_tokens + num_new_computed_tokens
|
||||
num_tokens_need_slot = min(
|
||||
num_computed_tokens + num_new_tokens + num_lookahead_tokens,
|
||||
self.max_model_len)
|
||||
self.max_model_len,
|
||||
)
|
||||
|
||||
num_blocks_to_allocate = self.coordinator.get_num_blocks_to_allocate(
|
||||
request_id=request.request_id,
|
||||
@@ -285,16 +289,18 @@ class KVCacheManager:
|
||||
self.block_pool.touch(new_computed_block_list)
|
||||
else:
|
||||
assert not any(new_computed_block_list), (
|
||||
"Computed blocks should be empty when "
|
||||
"prefix caching is disabled")
|
||||
"Computed blocks should be empty when prefix caching is disabled"
|
||||
)
|
||||
|
||||
# Append the new computed blocks to the request blocks until now to
|
||||
# avoid the case where the new blocks cannot be allocated.
|
||||
self.coordinator.save_new_computed_blocks(request.request_id,
|
||||
new_computed_block_list)
|
||||
self.coordinator.save_new_computed_blocks(
|
||||
request.request_id, new_computed_block_list
|
||||
)
|
||||
|
||||
new_blocks = self.coordinator.allocate_new_blocks(
|
||||
request.request_id, num_tokens_need_slot, num_encoder_tokens)
|
||||
request.request_id, num_tokens_need_slot, num_encoder_tokens
|
||||
)
|
||||
|
||||
# P/D: delay caching blocks if we have to recv from
|
||||
# remote. Update state for locally cached blocks.
|
||||
@@ -305,8 +311,9 @@ class KVCacheManager:
|
||||
# num_new_tokens, but must exclude "non-committable" tokens (e.g.,
|
||||
# draft tokens that could be rejected). Therefore, we cap the number
|
||||
# at `request.num_tokens`, ensuring only "finalized" tokens are cached.
|
||||
num_tokens_to_cache = min(num_computed_tokens + num_new_tokens,
|
||||
request.num_tokens)
|
||||
num_tokens_to_cache = min(
|
||||
num_computed_tokens + num_new_tokens, request.num_tokens
|
||||
)
|
||||
self.coordinator.cache_blocks(request, num_tokens_to_cache)
|
||||
|
||||
return KVCacheBlocks(new_blocks)
|
||||
@@ -378,7 +385,8 @@ class KVCacheManager:
|
||||
"""
|
||||
assert request.status == RequestStatus.RUNNING
|
||||
return self.coordinator.get_num_common_prefix_blocks(
|
||||
request.request_id, num_running_requests)
|
||||
request.request_id, num_running_requests
|
||||
)
|
||||
|
||||
def take_events(self) -> list[KVCacheEvent]:
|
||||
"""Take the KV cache events from the block pool.
|
||||
@@ -403,5 +411,4 @@ class KVCacheManager:
|
||||
|
||||
def create_empty_block_list(self) -> KVCacheBlocks:
|
||||
"""Creates a new KVCacheBlocks instance with no blocks."""
|
||||
return KVCacheBlocks(tuple([]
|
||||
for _ in range(self.num_kv_cache_groups)))
|
||||
return KVCacheBlocks(tuple([] for _ in range(self.num_kv_cache_groups)))
|
||||
|
||||
+260
-198
@@ -13,11 +13,16 @@ from vllm import envs
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils import GiB_bytes, cdiv, sha256_cbor
|
||||
from vllm.v1.kv_cache_interface import (ChunkedLocalAttentionSpec,
|
||||
FullAttentionSpec, KVCacheConfig,
|
||||
KVCacheGroupSpec, KVCacheSpec,
|
||||
KVCacheTensor, SlidingWindowSpec,
|
||||
UniformTypeKVCacheSpecs)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
ChunkedLocalAttentionSpec,
|
||||
FullAttentionSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheSpec,
|
||||
KVCacheTensor,
|
||||
SlidingWindowSpec,
|
||||
UniformTypeKVCacheSpecs,
|
||||
)
|
||||
from vllm.v1.metrics.stats import PrefixCacheStats
|
||||
from vllm.v1.request import Request
|
||||
|
||||
@@ -37,16 +42,16 @@ BlockHashWithGroupId = NewType("BlockHashWithGroupId", bytes)
|
||||
ExternalBlockHash = Union[bytes, int]
|
||||
|
||||
|
||||
def make_block_hash_with_group_id(block_hash: BlockHash,
|
||||
group_id: int) -> BlockHashWithGroupId:
|
||||
def make_block_hash_with_group_id(
|
||||
block_hash: BlockHash, group_id: int
|
||||
) -> BlockHashWithGroupId:
|
||||
"""Pack a ``BlockHash`` and group id into a ``BlockHashWithGroupId``.
|
||||
|
||||
The group id is encoded using 4 bytes in big-endian order and appended to
|
||||
the block hash bytes. This representation avoids creating tuples while
|
||||
still allowing us to recover both components when needed.
|
||||
"""
|
||||
return BlockHashWithGroupId(block_hash +
|
||||
group_id.to_bytes(4, "big", signed=False))
|
||||
return BlockHashWithGroupId(block_hash + group_id.to_bytes(4, "big", signed=False))
|
||||
|
||||
|
||||
def get_block_hash(key: BlockHashWithGroupId) -> BlockHash:
|
||||
@@ -87,7 +92,8 @@ def init_none_hash(hash_fn: Callable[[Any], bytes]):
|
||||
"PYTHONHASHSEED is not set. This will lead to non-reproducible "
|
||||
"block-hashes when using sha256_cbor as the hash function."
|
||||
"Consider setting PYTHONHASHSEED to a fixed value for "
|
||||
"reproducibility.")
|
||||
"reproducibility."
|
||||
)
|
||||
|
||||
if hash_seed is None:
|
||||
NONE_HASH = BlockHash(os.urandom(32))
|
||||
@@ -143,9 +149,10 @@ class PrefixCachingMetrics:
|
||||
# Remove the oldest stats until number of requests does not exceed
|
||||
# the limit.
|
||||
# NOTE: We preserve the latest added stats regardless.
|
||||
while len(
|
||||
self.query_queue
|
||||
) > 1 and self.aggregated_requests > self.max_recent_requests:
|
||||
while (
|
||||
len(self.query_queue) > 1
|
||||
and self.aggregated_requests > self.max_recent_requests
|
||||
):
|
||||
old_requests, old_queries, old_hits = self.query_queue.popleft()
|
||||
self.aggregated_requests -= old_requests
|
||||
self.aggregated_query_total -= old_queries
|
||||
@@ -169,6 +176,7 @@ class PrefixCachingMetrics:
|
||||
@dataclass
|
||||
class KVCacheBlock:
|
||||
"""KV-cache block metadata."""
|
||||
|
||||
# Block ID, ranging from 0 to num_gpu_blocks - 1.
|
||||
block_id: int
|
||||
# Reference count.
|
||||
@@ -192,7 +200,8 @@ class KVCacheBlock:
|
||||
@block_hash.setter
|
||||
def block_hash(self, block_hash: BlockHashWithGroupId):
|
||||
assert self.block_hash is None, (
|
||||
"The block already has a hash. This should not happen.")
|
||||
"The block already has a hash. This should not happen."
|
||||
)
|
||||
self._block_hash = block_hash
|
||||
|
||||
def reset_hash(self):
|
||||
@@ -202,15 +211,15 @@ class KVCacheBlock:
|
||||
def __repr__(self) -> str:
|
||||
# Use block_id instead of KVCacheBlock object to avoid calling __repr__
|
||||
# on KVCacheBlock object recursively.
|
||||
prev_block_id = (self.prev_free_block.block_id
|
||||
if self.prev_free_block else None)
|
||||
next_block_id = (self.next_free_block.block_id
|
||||
if self.next_free_block else None)
|
||||
return (f"KVCacheBlock(block_id={self.block_id}, "
|
||||
f"ref_cnt={self.ref_cnt}, "
|
||||
f"_block_hash={self._block_hash!r}, "
|
||||
f"prev_free_block={prev_block_id}, "
|
||||
f"next_free_block={next_block_id})")
|
||||
prev_block_id = self.prev_free_block.block_id if self.prev_free_block else None
|
||||
next_block_id = self.next_free_block.block_id if self.next_free_block else None
|
||||
return (
|
||||
f"KVCacheBlock(block_id={self.block_id}, "
|
||||
f"ref_cnt={self.ref_cnt}, "
|
||||
f"_block_hash={self._block_hash!r}, "
|
||||
f"prev_free_block={prev_block_id}, "
|
||||
f"next_free_block={next_block_id})"
|
||||
)
|
||||
|
||||
|
||||
class FreeKVCacheBlockQueue:
|
||||
@@ -271,12 +280,14 @@ class FreeKVCacheBlockQueue:
|
||||
Returns:
|
||||
The first free block.
|
||||
"""
|
||||
if (self.fake_free_list_head.next_free_block
|
||||
is self.fake_free_list_tail
|
||||
or self.fake_free_list_head.next_free_block is None):
|
||||
if (
|
||||
self.fake_free_list_head.next_free_block is self.fake_free_list_tail
|
||||
or self.fake_free_list_head.next_free_block is None
|
||||
):
|
||||
assert self.num_free_blocks == 0, (
|
||||
f"num_free_blocks ({self.num_free_blocks}) is out of sync "
|
||||
"with the free list.")
|
||||
"with the free list."
|
||||
)
|
||||
raise ValueError("No free blocks available")
|
||||
|
||||
first_block: KVCacheBlock = self.fake_free_list_head.next_free_block
|
||||
@@ -284,8 +295,10 @@ class FreeKVCacheBlockQueue:
|
||||
if first_block.next_free_block is None:
|
||||
# This should not happen if the block is from the free list.
|
||||
# It indicates a bug in the caller's logic.
|
||||
raise RuntimeError("Invalid block found in popleft() "
|
||||
"which doesn't have a valid next_free_block")
|
||||
raise RuntimeError(
|
||||
"Invalid block found in popleft() "
|
||||
"which doesn't have a valid next_free_block"
|
||||
)
|
||||
|
||||
# Connect fake_head and the next block of first_block (i.e. second block
|
||||
# or fake tail).
|
||||
@@ -360,7 +373,8 @@ class FreeKVCacheBlockQueue:
|
||||
"""
|
||||
if self.fake_free_list_tail.prev_free_block is None:
|
||||
raise RuntimeError(
|
||||
"prev_free_block of fake_free_list_tail should always exist")
|
||||
"prev_free_block of fake_free_list_tail should always exist"
|
||||
)
|
||||
last_block: KVCacheBlock = self.fake_free_list_tail.prev_free_block
|
||||
|
||||
# Connect the new block after the last block.
|
||||
@@ -384,7 +398,8 @@ class FreeKVCacheBlockQueue:
|
||||
|
||||
last_block = self.fake_free_list_tail.prev_free_block
|
||||
assert last_block is not None, (
|
||||
"prev_free_block of fake_free_list_tail should always exist")
|
||||
"prev_free_block of fake_free_list_tail should always exist"
|
||||
)
|
||||
# Add inter-connections between consecutive blocks
|
||||
for block in blocks:
|
||||
block.prev_free_block = last_block
|
||||
@@ -406,7 +421,8 @@ class FreeKVCacheBlockQueue:
|
||||
ret = []
|
||||
if self.fake_free_list_head.next_free_block is None:
|
||||
raise RuntimeError(
|
||||
"next_free_block of fake_free_list_head should always exist")
|
||||
"next_free_block of fake_free_list_head should always exist"
|
||||
)
|
||||
# Start from the first block
|
||||
curr_block: KVCacheBlock = self.fake_free_list_head.next_free_block
|
||||
# As long as next_free_block is available, we haven't reached to
|
||||
@@ -430,14 +446,16 @@ def need_extra_keys(request: Request) -> bool:
|
||||
# Multimodal requests need to include the MM hash.
|
||||
# LoRA requests need to include the LoRA ID.
|
||||
# Request with provided cache salt need to include the salt.
|
||||
return bool(request.mm_features) or (request.lora_request
|
||||
is not None) or (request.cache_salt
|
||||
is not None)
|
||||
return (
|
||||
bool(request.mm_features)
|
||||
or (request.lora_request is not None)
|
||||
or (request.cache_salt is not None)
|
||||
)
|
||||
|
||||
|
||||
def _gen_mm_extra_hash_keys(request: Request, start_token_idx: int,
|
||||
end_token_idx: int,
|
||||
start_mm_idx: int) -> tuple[list[Any], int]:
|
||||
def _gen_mm_extra_hash_keys(
|
||||
request: Request, start_token_idx: int, end_token_idx: int, start_mm_idx: int
|
||||
) -> tuple[list[Any], int]:
|
||||
"""Generate extra keys related to MultiModal request for block hash
|
||||
computation. For multi-modal inputs, the extra keys are
|
||||
(mm_hash, start_offset) that indicate a mm input contained in the
|
||||
@@ -515,8 +533,8 @@ def _gen_lora_extra_hash_keys(request: Request) -> list[int]:
|
||||
|
||||
|
||||
def generate_block_hash_extra_keys(
|
||||
request: Request, start_token_idx: int, end_token_idx: int,
|
||||
start_mm_idx: int) -> tuple[Optional[tuple[Any, ...]], int]:
|
||||
request: Request, start_token_idx: int, end_token_idx: int, start_mm_idx: int
|
||||
) -> tuple[Optional[tuple[Any, ...]], int]:
|
||||
"""Generate extra keys for the block hash. The extra keys can come from
|
||||
the multi-modal inputs and request specific metadata (e.g., LoRA ID).
|
||||
|
||||
@@ -531,10 +549,12 @@ def generate_block_hash_extra_keys(
|
||||
"""
|
||||
mm_extra_keys: list[Any]
|
||||
mm_extra_keys, new_start_mm_idx = _gen_mm_extra_hash_keys(
|
||||
request, start_token_idx, end_token_idx, start_mm_idx)
|
||||
request, start_token_idx, end_token_idx, start_mm_idx
|
||||
)
|
||||
lora_extra_keys: list[int] = _gen_lora_extra_hash_keys(request)
|
||||
cache_salt_keys: list[str] = [request.cache_salt] if (
|
||||
start_token_idx == 0 and request.cache_salt) else []
|
||||
cache_salt_keys: list[str] = (
|
||||
[request.cache_salt] if (start_token_idx == 0 and request.cache_salt) else []
|
||||
)
|
||||
|
||||
extra_keys: list[Any] = lora_extra_keys + mm_extra_keys + cache_salt_keys
|
||||
|
||||
@@ -545,10 +565,11 @@ def generate_block_hash_extra_keys(
|
||||
|
||||
|
||||
def hash_block_tokens(
|
||||
hash_function: Callable[[Any], bytes],
|
||||
parent_block_hash: Optional[BlockHash],
|
||||
curr_block_token_ids: Sequence[int],
|
||||
extra_keys: Optional[tuple[Any, ...]] = None) -> BlockHash:
|
||||
hash_function: Callable[[Any], bytes],
|
||||
parent_block_hash: Optional[BlockHash],
|
||||
curr_block_token_ids: Sequence[int],
|
||||
extra_keys: Optional[tuple[Any, ...]] = None,
|
||||
) -> BlockHash:
|
||||
"""Computes a hash value corresponding to the contents of a block and
|
||||
the contents of the preceding block(s). The hash value is used for
|
||||
prefix caching. We use LRU cache for this function to avoid recomputing
|
||||
@@ -569,8 +590,8 @@ def hash_block_tokens(
|
||||
|
||||
curr_block_token_ids_tuple = tuple(curr_block_token_ids)
|
||||
return BlockHash(
|
||||
hash_function(
|
||||
(parent_block_hash, curr_block_token_ids_tuple, extra_keys)))
|
||||
hash_function((parent_block_hash, curr_block_token_ids_tuple, extra_keys))
|
||||
)
|
||||
|
||||
|
||||
def get_request_block_hasher(
|
||||
@@ -597,8 +618,9 @@ def get_request_block_hasher(
|
||||
# last mm input.
|
||||
curr_mm_idx = -1
|
||||
|
||||
prev_block_hash_value = (request.block_hashes[-1]
|
||||
if request.block_hashes else None)
|
||||
prev_block_hash_value = (
|
||||
request.block_hashes[-1] if request.block_hashes else None
|
||||
)
|
||||
new_block_hashes: list[BlockHash] = []
|
||||
while True:
|
||||
end_token_idx = start_token_idx + block_size
|
||||
@@ -608,13 +630,14 @@ def get_request_block_hasher(
|
||||
|
||||
# MM and LoRA requests need extra keys for block-hash computation.
|
||||
extra_keys, curr_mm_idx = generate_block_hash_extra_keys(
|
||||
request, start_token_idx, end_token_idx, curr_mm_idx)
|
||||
request, start_token_idx, end_token_idx, curr_mm_idx
|
||||
)
|
||||
|
||||
# Compute the hash of the current block
|
||||
block_tokens = request.all_token_ids[start_token_idx:end_token_idx]
|
||||
block_hash = hash_block_tokens(caching_hash_fn,
|
||||
prev_block_hash_value, block_tokens,
|
||||
extra_keys)
|
||||
block_hash = hash_block_tokens(
|
||||
caching_hash_fn, prev_block_hash_value, block_tokens, extra_keys
|
||||
)
|
||||
|
||||
new_block_hashes.append(block_hash)
|
||||
start_token_idx += block_size
|
||||
@@ -625,18 +648,20 @@ def get_request_block_hasher(
|
||||
return request_block_hasher
|
||||
|
||||
|
||||
def max_memory_usage_bytes(vllm_config: VllmConfig,
|
||||
kv_cache_specs: Iterable[KVCacheSpec]) -> int:
|
||||
def max_memory_usage_bytes(
|
||||
vllm_config: VllmConfig, kv_cache_specs: Iterable[KVCacheSpec]
|
||||
) -> int:
|
||||
"""
|
||||
Get the maximum memory usage in bytes for the given KV cache specs.
|
||||
"""
|
||||
return sum(
|
||||
spec.max_memory_usage_bytes(vllm_config) for spec in kv_cache_specs)
|
||||
return sum(spec.max_memory_usage_bytes(vllm_config) for spec in kv_cache_specs)
|
||||
|
||||
|
||||
def estimate_max_model_len(vllm_config: VllmConfig,
|
||||
kv_cache_spec: dict[str, KVCacheSpec],
|
||||
available_memory: int) -> int:
|
||||
def estimate_max_model_len(
|
||||
vllm_config: VllmConfig,
|
||||
kv_cache_spec: dict[str, KVCacheSpec],
|
||||
available_memory: int,
|
||||
) -> int:
|
||||
"""
|
||||
Estimates the maximum model length that can fit in the available memory
|
||||
using binary search.
|
||||
@@ -655,8 +680,7 @@ def estimate_max_model_len(vllm_config: VllmConfig,
|
||||
# Modify the max_model_len for this calculation
|
||||
vllm_config.model_config.max_model_len = model_len
|
||||
# Calculate memory needed for the given model length
|
||||
memory_needed = max_memory_usage_bytes(vllm_config,
|
||||
kv_cache_spec.values())
|
||||
memory_needed = max_memory_usage_bytes(vllm_config, kv_cache_spec.values())
|
||||
return memory_needed <= available_memory
|
||||
|
||||
# Binary search for the maximum model length
|
||||
@@ -679,9 +703,11 @@ def estimate_max_model_len(vllm_config: VllmConfig,
|
||||
return result
|
||||
|
||||
|
||||
def check_enough_kv_cache_memory(vllm_config: VllmConfig,
|
||||
kv_cache_spec: dict[str, KVCacheSpec],
|
||||
available_memory: int):
|
||||
def check_enough_kv_cache_memory(
|
||||
vllm_config: VllmConfig,
|
||||
kv_cache_spec: dict[str, KVCacheSpec],
|
||||
available_memory: int,
|
||||
):
|
||||
"""
|
||||
Checks whether `available_memory` is enough for the KV cache to hold at
|
||||
least one request with the model's max_model_len.
|
||||
@@ -700,36 +726,41 @@ def check_enough_kv_cache_memory(vllm_config: VllmConfig,
|
||||
return
|
||||
|
||||
if available_memory <= 0:
|
||||
raise ValueError("No available memory for the cache blocks. "
|
||||
"Try increasing `gpu_memory_utilization` when "
|
||||
"initializing the engine.")
|
||||
raise ValueError(
|
||||
"No available memory for the cache blocks. "
|
||||
"Try increasing `gpu_memory_utilization` when "
|
||||
"initializing the engine."
|
||||
)
|
||||
|
||||
max_model_len = vllm_config.model_config.max_model_len
|
||||
needed_memory = max_memory_usage_bytes(vllm_config, kv_cache_spec.values())
|
||||
|
||||
if needed_memory > available_memory:
|
||||
# Estimate the maximum model length that can fit in the available memory
|
||||
estimated_max_len = estimate_max_model_len(vllm_config, kv_cache_spec,
|
||||
available_memory)
|
||||
estimated_max_len = estimate_max_model_len(
|
||||
vllm_config, kv_cache_spec, available_memory
|
||||
)
|
||||
estimated_msg = ""
|
||||
if estimated_max_len > 0:
|
||||
estimated_msg = (
|
||||
"Based on the available memory, "
|
||||
f"the estimated maximum model length is {estimated_max_len}.")
|
||||
f"the estimated maximum model length is {estimated_max_len}."
|
||||
)
|
||||
|
||||
raise ValueError(
|
||||
f"To serve at least one request with the models's max seq len "
|
||||
f"({max_model_len}), ({needed_memory/GiB_bytes:.2f} GiB KV "
|
||||
f"({max_model_len}), ({needed_memory / GiB_bytes:.2f} GiB KV "
|
||||
f"cache is needed, which is larger than the available KV cache "
|
||||
f"memory ({available_memory/GiB_bytes:.2f} GiB). "
|
||||
f"memory ({available_memory / GiB_bytes:.2f} GiB). "
|
||||
f"{estimated_msg} "
|
||||
f"Try increasing `gpu_memory_utilization` or decreasing "
|
||||
f"`max_model_len` when initializing the engine.")
|
||||
f"`max_model_len` when initializing the engine."
|
||||
)
|
||||
|
||||
|
||||
def create_kv_cache_group_specs(
|
||||
kv_cache_spec: dict[str, KVCacheSpec],
|
||||
grouped_layer_names: list[list[str]]) -> list[KVCacheGroupSpec]:
|
||||
kv_cache_spec: dict[str, KVCacheSpec], grouped_layer_names: list[list[str]]
|
||||
) -> list[KVCacheGroupSpec]:
|
||||
"""
|
||||
Create KVCacheGroupSpec object for each kv cache group layer.
|
||||
The layers in the same group should share the same
|
||||
@@ -752,7 +783,8 @@ def create_kv_cache_group_specs(
|
||||
]
|
||||
merged_layer_spec = layer_specs[0].merge(layer_specs)
|
||||
kv_cache_groups.append(
|
||||
KVCacheGroupSpec(layer_names_one_group, merged_layer_spec))
|
||||
KVCacheGroupSpec(layer_names_one_group, merged_layer_spec)
|
||||
)
|
||||
return kv_cache_groups
|
||||
|
||||
|
||||
@@ -782,19 +814,22 @@ def is_kv_cache_spec_uniform(kv_cache_spec: dict[str, KVCacheSpec]) -> bool:
|
||||
|
||||
|
||||
def get_max_concurrency_for_kv_cache_config(
|
||||
vllm_config: VllmConfig, kv_cache_config: KVCacheConfig) -> float:
|
||||
vllm_config: VllmConfig, kv_cache_config: KVCacheConfig
|
||||
) -> float:
|
||||
"""
|
||||
Get the maximum concurrency for the given KV cache configuration.
|
||||
"""
|
||||
num_layer_per_group = max(
|
||||
len(group.layer_names) for group in kv_cache_config.kv_cache_groups)
|
||||
len(group.layer_names) for group in kv_cache_config.kv_cache_groups
|
||||
)
|
||||
max_memory_usage_per_request = num_layer_per_group * max_memory_usage_bytes(
|
||||
vllm_config,
|
||||
(group.kv_cache_spec for group in kv_cache_config.kv_cache_groups))
|
||||
memory_per_block = kv_cache_config.kv_cache_groups[
|
||||
0].kv_cache_spec.page_size_bytes * num_layer_per_group
|
||||
num_block_per_request = cdiv(max_memory_usage_per_request,
|
||||
memory_per_block)
|
||||
vllm_config, (group.kv_cache_spec for group in kv_cache_config.kv_cache_groups)
|
||||
)
|
||||
memory_per_block = (
|
||||
kv_cache_config.kv_cache_groups[0].kv_cache_spec.page_size_bytes
|
||||
* num_layer_per_group
|
||||
)
|
||||
num_block_per_request = cdiv(max_memory_usage_per_request, memory_per_block)
|
||||
max_concurrency = kv_cache_config.num_blocks / num_block_per_request
|
||||
return max_concurrency
|
||||
|
||||
@@ -804,18 +839,20 @@ def may_override_num_blocks(vllm_config: VllmConfig, num_blocks: int) -> int:
|
||||
Override the number of kv cache blocks if `num_gpu_blocks_override` is set.
|
||||
"""
|
||||
if vllm_config.cache_config.num_gpu_blocks_override is not None:
|
||||
num_gpu_blocks_override = \
|
||||
vllm_config.cache_config.num_gpu_blocks_override
|
||||
num_gpu_blocks_override = vllm_config.cache_config.num_gpu_blocks_override
|
||||
logger.info(
|
||||
"Overriding num_gpu_blocks=%d with "
|
||||
"num_gpu_blocks_override=%d", num_blocks, num_gpu_blocks_override)
|
||||
"Overriding num_gpu_blocks=%d with num_gpu_blocks_override=%d",
|
||||
num_blocks,
|
||||
num_gpu_blocks_override,
|
||||
)
|
||||
num_blocks = num_gpu_blocks_override
|
||||
|
||||
return num_blocks
|
||||
|
||||
|
||||
def get_num_blocks(vllm_config: VllmConfig, num_layers: int,
|
||||
available_memory: int, page_size: int) -> int:
|
||||
def get_num_blocks(
|
||||
vllm_config: VllmConfig, num_layers: int, available_memory: int, page_size: int
|
||||
) -> int:
|
||||
"""
|
||||
Get the number of kv cache blocks.
|
||||
|
||||
@@ -841,9 +878,10 @@ def get_uniform_page_size(kv_cache_spec: dict[str, KVCacheSpec]) -> int:
|
||||
|
||||
|
||||
def _get_kv_cache_groups_uniform_spec(
|
||||
kv_cache_specs: dict[str, KVCacheSpec]) -> list[KVCacheGroupSpec]:
|
||||
kv_cache_specs: dict[str, KVCacheSpec],
|
||||
) -> list[KVCacheGroupSpec]:
|
||||
"""
|
||||
Generates the KV cache configuration for a model with the same KV cache
|
||||
Generates the KV cache configuration for a model with the same KV cache
|
||||
spec for all layers.
|
||||
|
||||
Args:
|
||||
@@ -853,12 +891,12 @@ def _get_kv_cache_groups_uniform_spec(
|
||||
The generated KVCacheGroupSpecs
|
||||
"""
|
||||
|
||||
return create_kv_cache_group_specs(kv_cache_specs,
|
||||
[list(kv_cache_specs.keys())])
|
||||
return create_kv_cache_group_specs(kv_cache_specs, [list(kv_cache_specs.keys())])
|
||||
|
||||
|
||||
def _get_kv_cache_groups_uniform_type(
|
||||
spec: UniformTypeKVCacheSpecs) -> list[KVCacheGroupSpec]:
|
||||
spec: UniformTypeKVCacheSpecs,
|
||||
) -> list[KVCacheGroupSpec]:
|
||||
"""
|
||||
Generates the KV cache configuration for a model with one type of KV cache
|
||||
but different hidden sizes. All layers are merged into one group.
|
||||
@@ -873,8 +911,7 @@ def _get_kv_cache_groups_uniform_type(
|
||||
return [KVCacheGroupSpec(list(spec.kv_cache_specs.keys()), spec)]
|
||||
|
||||
|
||||
def is_kv_cache_page_size_uniform(
|
||||
kv_cache_spec: dict[str, KVCacheSpec]) -> bool:
|
||||
def is_kv_cache_page_size_uniform(kv_cache_spec: dict[str, KVCacheSpec]) -> bool:
|
||||
"""
|
||||
Whether all layers in the given KVCacheSpec have the same page size.
|
||||
Args:
|
||||
@@ -888,70 +925,69 @@ def is_kv_cache_page_size_uniform(
|
||||
return len(page_sizes) == 1
|
||||
|
||||
|
||||
def is_kv_cache_type_attention_free(
|
||||
kv_cache_spec: dict[str, KVCacheSpec]) -> bool:
|
||||
|
||||
def is_kv_cache_type_attention_free(kv_cache_spec: dict[str, KVCacheSpec]) -> bool:
|
||||
# kv_cache_spec is an empty dict for attention free models
|
||||
return not kv_cache_spec
|
||||
|
||||
|
||||
def _get_kv_cache_groups_uniform_page_size(
|
||||
kv_cache_spec: dict[str, KVCacheSpec]) -> list[KVCacheGroupSpec]:
|
||||
kv_cache_spec: dict[str, KVCacheSpec],
|
||||
) -> list[KVCacheGroupSpec]:
|
||||
"""
|
||||
Generates the KV cache groups for hybrid models with multiple
|
||||
attention types but still with a uniform page size (physical memory per
|
||||
Generates the KV cache groups for hybrid models with multiple
|
||||
attention types but still with a uniform page size (physical memory per
|
||||
block per layer) for all layers.
|
||||
|
||||
Detailed explanation about kv cache management of hybrid models:
|
||||
The layers in the models are repeated with some patterns, e.g., a model
|
||||
with 10 full attention layers and 20 sliding window attention layers can be
|
||||
regarded as repeating the pattern (1 * full, 2 * sw) 10 times.
|
||||
regarded as repeating the pattern (1 * full, 2 * sw) 10 times.
|
||||
The KVCacheManager allocates different block tables for each of the 3 layers
|
||||
in the pattern, and repeats each of them 10 times to generate the
|
||||
in the pattern, and repeats each of them 10 times to generate the
|
||||
block_table for the 30 layers in the model.
|
||||
Therefore, we can group the layers in the model into 3 kv_cache_groups, each
|
||||
of which contains 10 layers in the model.
|
||||
The KVCacheManager allocates the block_table for each group based on its
|
||||
kv_cache spec, and the model runner applies the block table to each layer
|
||||
kv_cache spec, and the model runner applies the block table to each layer
|
||||
in the group.
|
||||
For example:
|
||||
1. A model only uses full attention. The pattern is
|
||||
(num_hidden_layers * full), so there is only one group and the block table
|
||||
is shared by all layers. It is already handled by
|
||||
1. A model only uses full attention. The pattern is
|
||||
(num_hidden_layers * full), so there is only one group and the block table
|
||||
is shared by all layers. It is already handled by
|
||||
`_get_kv_cache_config_uniform_type`.
|
||||
2. A model with 10 full attention layers and 20 sliding window
|
||||
attention layers. There are 3 layers in the pattern (1 * full, 2 * sw), so
|
||||
2. A model with 10 full attention layers and 20 sliding window
|
||||
attention layers. There are 3 layers in the pattern (1 * full, 2 * sw), so
|
||||
there are 3 kv_cache_groups, each of which represents 10 layers.
|
||||
|
||||
To simplify the implementation, we make the following assumptions:
|
||||
1. Physical memory per block: Must be the same across all KV cache groups.
|
||||
1. Physical memory per block: Must be the same across all KV cache groups.
|
||||
Breaking this assumption is non-trivial due to memory fragmentation concerns
|
||||
when allocating blocks of different sizes.
|
||||
2. Tokens per block (block_size): Currently, we directly use
|
||||
`CacheConfig.block_size` for all layers. It can be extended to vary by KV
|
||||
cache group, but within each KV cache group, all layers must share the same
|
||||
2. Tokens per block (block_size): Currently, we directly use
|
||||
`CacheConfig.block_size` for all layers. It can be extended to vary by KV
|
||||
cache group, but within each KV cache group, all layers must share the same
|
||||
block size.
|
||||
3. Physical memory per token per layer: This property is decided by model
|
||||
config. Currently we only support models that have the same physical memory
|
||||
per token per layer for all layers. Can be relaxed with a simple extension,
|
||||
3. Physical memory per token per layer: This property is decided by model
|
||||
config. Currently we only support models that have the same physical memory
|
||||
per token per layer for all layers. Can be relaxed with a simple extension,
|
||||
but still need to keep physical memory per block the same for all groups.
|
||||
4. Number of layers per group: Currently assumed the same for all layers.
|
||||
Can be relaxed with a simple extension, but still need to keep physical
|
||||
4. Number of layers per group: Currently assumed the same for all layers.
|
||||
Can be relaxed with a simple extension, but still need to keep physical
|
||||
memory per block the same for all groups.
|
||||
5. Attention type within groups: All layers in a group must share the same
|
||||
attention type. One exception is that, when
|
||||
`--disable-hybrid-kv-cache-manager` is true, the single group for full
|
||||
attention layers may also include attention layers using sliding window or
|
||||
attention type. One exception is that, when
|
||||
`--disable-hybrid-kv-cache-manager` is true, the single group for full
|
||||
attention layers may also include attention layers using sliding window or
|
||||
LLaMA 4 local attention. See `unify_hybrid_kv_cache_specs` for more details.
|
||||
6. Support for multiple attention types: The design for most components is
|
||||
general to an arbitrary number of attention types. But
|
||||
`find_longest_cache_hit` only supports one attention type or two
|
||||
6. Support for multiple attention types: The design for most components is
|
||||
general to an arbitrary number of attention types. But
|
||||
`find_longest_cache_hit` only supports one attention type or two
|
||||
types of full-attention plus exactly one another type. The general
|
||||
implementation of this function is feasible but we don't know how to
|
||||
implementation of this function is feasible but we don't know how to
|
||||
implement it cleanly yet.
|
||||
|
||||
As we assume tokens per block, physical memory per token per layer, and
|
||||
number of layers per group are the same now, we can ensure that physical
|
||||
As we assume tokens per block, physical memory per token per layer, and
|
||||
number of layers per group are the same now, we can ensure that physical
|
||||
memory per block is the same for all groups.
|
||||
|
||||
Args:
|
||||
@@ -1005,10 +1041,12 @@ def _get_kv_cache_groups_uniform_page_size(
|
||||
return create_kv_cache_group_specs(kv_cache_spec, grouped_layers)
|
||||
|
||||
|
||||
def get_kv_cache_config_from_groups(vllm_config: VllmConfig,
|
||||
kv_cache_groups: list[KVCacheGroupSpec],
|
||||
kv_cache_specs: dict[str, KVCacheSpec],
|
||||
available_memory: int) -> KVCacheConfig:
|
||||
def get_kv_cache_config_from_groups(
|
||||
vllm_config: VllmConfig,
|
||||
kv_cache_groups: list[KVCacheGroupSpec],
|
||||
kv_cache_specs: dict[str, KVCacheSpec],
|
||||
available_memory: int,
|
||||
) -> KVCacheConfig:
|
||||
"""
|
||||
Generate the KV cache configuration from the KV cache groups and spec
|
||||
of each layer.
|
||||
@@ -1031,19 +1069,22 @@ def get_kv_cache_config_from_groups(vllm_config: VllmConfig,
|
||||
)
|
||||
|
||||
# Determine how model runners should initialize the KV cache tensors.
|
||||
if len(kv_cache_groups) == 1 and \
|
||||
isinstance(kv_cache_groups[0].kv_cache_spec, UniformTypeKVCacheSpecs):
|
||||
if len(kv_cache_groups) == 1 and isinstance(
|
||||
kv_cache_groups[0].kv_cache_spec, UniformTypeKVCacheSpecs
|
||||
):
|
||||
# Special case: all layers have the same type of KV cache but with
|
||||
# different hidden size. Allocate different amount of memory for each
|
||||
# layer based on its hidden size.
|
||||
num_blocks = available_memory // kv_cache_groups[
|
||||
0].kv_cache_spec.page_size_bytes
|
||||
num_blocks = (
|
||||
available_memory // kv_cache_groups[0].kv_cache_spec.page_size_bytes
|
||||
)
|
||||
num_blocks = may_override_num_blocks(vllm_config, num_blocks)
|
||||
per_layer_specs = kv_cache_groups[0].kv_cache_spec.kv_cache_specs
|
||||
kv_cache_tensors = [
|
||||
KVCacheTensor(size=per_layer_specs[layer_name].page_size_bytes *
|
||||
num_blocks,
|
||||
shared_by=[layer_name])
|
||||
KVCacheTensor(
|
||||
size=per_layer_specs[layer_name].page_size_bytes * num_blocks,
|
||||
shared_by=[layer_name],
|
||||
)
|
||||
for layer_name in kv_cache_groups[0].layer_names
|
||||
]
|
||||
else:
|
||||
@@ -1059,8 +1100,9 @@ def get_kv_cache_config_from_groups(vllm_config: VllmConfig,
|
||||
|
||||
page_size = get_uniform_page_size(kv_cache_specs)
|
||||
assert group_size > 0, "group_size must be greater than 0"
|
||||
num_blocks = get_num_blocks(vllm_config, group_size, available_memory,
|
||||
page_size)
|
||||
num_blocks = get_num_blocks(
|
||||
vllm_config, group_size, available_memory, page_size
|
||||
)
|
||||
kv_cache_tensors = []
|
||||
for i in range(group_size):
|
||||
shared_by = []
|
||||
@@ -1068,8 +1110,8 @@ def get_kv_cache_config_from_groups(vllm_config: VllmConfig,
|
||||
if i < len(kv_cache_groups[j].layer_names):
|
||||
shared_by.append(kv_cache_groups[j].layer_names[i])
|
||||
kv_cache_tensors.append(
|
||||
KVCacheTensor(size=page_size * num_blocks,
|
||||
shared_by=shared_by))
|
||||
KVCacheTensor(size=page_size * num_blocks, shared_by=shared_by)
|
||||
)
|
||||
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=num_blocks,
|
||||
@@ -1077,8 +1119,7 @@ def get_kv_cache_config_from_groups(vllm_config: VllmConfig,
|
||||
kv_cache_groups=kv_cache_groups,
|
||||
)
|
||||
|
||||
min_block_size = min(
|
||||
[group.kv_cache_spec.block_size for group in kv_cache_groups])
|
||||
min_block_size = min([group.kv_cache_spec.block_size for group in kv_cache_groups])
|
||||
|
||||
# Print the KV cache size and maximum concurrency.
|
||||
num_tokens = num_blocks // len(kv_cache_groups) * min_block_size
|
||||
@@ -1086,14 +1127,19 @@ def get_kv_cache_config_from_groups(vllm_config: VllmConfig,
|
||||
num_tokens *= vllm_config.parallel_config.decode_context_parallel_size
|
||||
logger.info(
|
||||
"Multiplying the GPU KV cache size by the dcp_world_size %d.",
|
||||
vllm_config.parallel_config.decode_context_parallel_size)
|
||||
vllm_config.parallel_config.decode_context_parallel_size,
|
||||
)
|
||||
num_tokens_str = f"{num_tokens:,}"
|
||||
logger.info("GPU KV cache size: %s tokens", num_tokens_str)
|
||||
max_model_len_str = f"{vllm_config.model_config.max_model_len:,}"
|
||||
max_concurrency = get_max_concurrency_for_kv_cache_config(
|
||||
vllm_config, kv_cache_config)
|
||||
logger.info("Maximum concurrency for %s tokens per request: %.2fx",
|
||||
max_model_len_str, max_concurrency)
|
||||
vllm_config, kv_cache_config
|
||||
)
|
||||
logger.info(
|
||||
"Maximum concurrency for %s tokens per request: %.2fx",
|
||||
max_model_len_str,
|
||||
max_concurrency,
|
||||
)
|
||||
return kv_cache_config
|
||||
|
||||
|
||||
@@ -1108,25 +1154,27 @@ def unify_hybrid_kv_cache_specs(kv_cache_spec: dict[str, KVCacheSpec]):
|
||||
"""
|
||||
|
||||
if is_kv_cache_spec_uniform(
|
||||
kv_cache_spec) or UniformTypeKVCacheSpecs.is_uniform_type(
|
||||
kv_cache_spec):
|
||||
kv_cache_spec
|
||||
) or UniformTypeKVCacheSpecs.is_uniform_type(kv_cache_spec):
|
||||
return
|
||||
|
||||
logger.warning(
|
||||
"Hybrid KV cache manager is disabled for this hybrid model, "
|
||||
"This means we do not enable any optimizations for saving KV cache "
|
||||
"memory (e.g., dropping the KV cache outside the sliding window). "
|
||||
"The compute of layers like sliding window is still saved.")
|
||||
"The compute of layers like sliding window is still saved."
|
||||
)
|
||||
|
||||
has_full_attention = any(
|
||||
isinstance(spec, FullAttentionSpec) for spec in kv_cache_spec.values())
|
||||
isinstance(spec, FullAttentionSpec) for spec in kv_cache_spec.values()
|
||||
)
|
||||
has_sliding_window = any(
|
||||
isinstance(spec, SlidingWindowSpec) for spec in kv_cache_spec.values())
|
||||
isinstance(spec, SlidingWindowSpec) for spec in kv_cache_spec.values()
|
||||
)
|
||||
has_chunked_local_attention = any(
|
||||
isinstance(spec, ChunkedLocalAttentionSpec)
|
||||
for spec in kv_cache_spec.values())
|
||||
if has_full_attention and (has_sliding_window
|
||||
or has_chunked_local_attention):
|
||||
isinstance(spec, ChunkedLocalAttentionSpec) for spec in kv_cache_spec.values()
|
||||
)
|
||||
if has_full_attention and (has_sliding_window or has_chunked_local_attention):
|
||||
for layer_name, spec in kv_cache_spec.items():
|
||||
if isinstance(spec, SlidingWindowSpec):
|
||||
kv_cache_spec[layer_name] = FullAttentionSpec(
|
||||
@@ -1145,15 +1193,19 @@ def unify_hybrid_kv_cache_specs(kv_cache_spec: dict[str, KVCacheSpec]):
|
||||
attention_chunk_size=spec.attention_chunk_size,
|
||||
)
|
||||
|
||||
if not (is_kv_cache_spec_uniform(kv_cache_spec)
|
||||
or UniformTypeKVCacheSpecs.is_uniform_type(kv_cache_spec)):
|
||||
raise ValueError("Hybrid KV cache manager is disabled but failed to "
|
||||
"convert the KV cache specs to one unified type.")
|
||||
if not (
|
||||
is_kv_cache_spec_uniform(kv_cache_spec)
|
||||
or UniformTypeKVCacheSpecs.is_uniform_type(kv_cache_spec)
|
||||
):
|
||||
raise ValueError(
|
||||
"Hybrid KV cache manager is disabled but failed to "
|
||||
"convert the KV cache specs to one unified type."
|
||||
)
|
||||
|
||||
|
||||
def get_kv_cache_groups(
|
||||
vllm_config: VllmConfig,
|
||||
kv_cache_spec: dict[str, KVCacheSpec]) -> list[KVCacheGroupSpec]:
|
||||
vllm_config: VllmConfig, kv_cache_spec: dict[str, KVCacheSpec]
|
||||
) -> list[KVCacheGroupSpec]:
|
||||
"""
|
||||
Split the layers in the model into groups with the same KV cache spec.
|
||||
|
||||
@@ -1192,14 +1244,14 @@ def get_kv_cache_groups(
|
||||
|
||||
|
||||
def generate_scheduler_kv_cache_config(
|
||||
kv_cache_configs: list[KVCacheConfig]) -> KVCacheConfig:
|
||||
kv_cache_configs: list[KVCacheConfig],
|
||||
) -> KVCacheConfig:
|
||||
"""
|
||||
Generate the KV cache configuration for the scheduler.
|
||||
"""
|
||||
assert all([
|
||||
cfg.num_blocks == kv_cache_configs[0].num_blocks
|
||||
for cfg in kv_cache_configs
|
||||
])
|
||||
assert all(
|
||||
[cfg.num_blocks == kv_cache_configs[0].num_blocks for cfg in kv_cache_configs]
|
||||
)
|
||||
# All workers have the same kv_cache_config except layer names, so use
|
||||
# an arbitrary one to initialize the scheduler.
|
||||
cfg = copy.deepcopy(kv_cache_configs[0])
|
||||
@@ -1208,15 +1260,18 @@ def generate_scheduler_kv_cache_config(
|
||||
# All layers in the UniformTypeKVCacheSpecs have the same type,
|
||||
# so use an arbitrary one to initialize the scheduler.
|
||||
group.kv_cache_spec = next(
|
||||
iter(group.kv_cache_spec.kv_cache_specs.values()))
|
||||
iter(group.kv_cache_spec.kv_cache_specs.values())
|
||||
)
|
||||
return cfg
|
||||
|
||||
|
||||
def get_kv_cache_configs(vllm_config: VllmConfig,
|
||||
kv_cache_specs: list[dict[str, KVCacheSpec]],
|
||||
available_memory: list[int]) -> list[KVCacheConfig]:
|
||||
def get_kv_cache_configs(
|
||||
vllm_config: VllmConfig,
|
||||
kv_cache_specs: list[dict[str, KVCacheSpec]],
|
||||
available_memory: list[int],
|
||||
) -> list[KVCacheConfig]:
|
||||
"""
|
||||
Generates the KV cache configurations for a model.
|
||||
Generates the KV cache configurations for a model.
|
||||
Since we use a shared centralized controller for all workers, we need the
|
||||
`kv_cache_config` to be consistent across all workers to make sure
|
||||
the KV cache allocation can be applied to all workers. However, different
|
||||
@@ -1235,7 +1290,7 @@ def get_kv_cache_configs(vllm_config: VllmConfig,
|
||||
vllm_config: The global VllmConfig
|
||||
kv_cache_specs: List of dict[layer_name, KVCacheSpec] for each worker.
|
||||
available_memory: Memory available for KV cache in bytes for each
|
||||
worker.
|
||||
worker.
|
||||
|
||||
Returns:
|
||||
The generated KVCacheConfigs for each worker.
|
||||
@@ -1243,9 +1298,11 @@ def get_kv_cache_configs(vllm_config: VllmConfig,
|
||||
|
||||
# Check if the available memory is enough for each worker.
|
||||
for kv_cache_spec_one_worker, available_memory_one_worker in zip(
|
||||
kv_cache_specs, available_memory):
|
||||
check_enough_kv_cache_memory(vllm_config, kv_cache_spec_one_worker,
|
||||
available_memory_one_worker)
|
||||
kv_cache_specs, available_memory
|
||||
):
|
||||
check_enough_kv_cache_memory(
|
||||
vllm_config, kv_cache_spec_one_worker, available_memory_one_worker
|
||||
)
|
||||
|
||||
# Merge the KV cache specs of all workers. Different PP stages may have
|
||||
# different layer names, and different TP ranks of the same PP stage should
|
||||
@@ -1258,37 +1315,42 @@ def get_kv_cache_configs(vllm_config: VllmConfig,
|
||||
else:
|
||||
assert merged_kv_cache_specs[layer_name] == layer_spec, (
|
||||
"The KV cache specs for the same layer are different "
|
||||
"across workers. This is not supported yet.")
|
||||
global_kv_cache_groups = get_kv_cache_groups(vllm_config,
|
||||
merged_kv_cache_specs)
|
||||
"across workers. This is not supported yet."
|
||||
)
|
||||
global_kv_cache_groups = get_kv_cache_groups(vllm_config, merged_kv_cache_specs)
|
||||
|
||||
kv_cache_configs: list[KVCacheConfig] = []
|
||||
for kv_cache_spec_one_worker, available_memory_one_worker in zip(
|
||||
kv_cache_specs, available_memory):
|
||||
kv_cache_specs, available_memory
|
||||
):
|
||||
kv_cache_groups_one_worker: list[KVCacheGroupSpec] = []
|
||||
for group in global_kv_cache_groups:
|
||||
group_layer_names_one_worker = [
|
||||
layer_name for layer_name in group.layer_names
|
||||
layer_name
|
||||
for layer_name in group.layer_names
|
||||
if layer_name in kv_cache_spec_one_worker
|
||||
]
|
||||
kv_cache_groups_one_worker.append(
|
||||
KVCacheGroupSpec(group_layer_names_one_worker,
|
||||
group.kv_cache_spec))
|
||||
KVCacheGroupSpec(group_layer_names_one_worker, group.kv_cache_spec)
|
||||
)
|
||||
assert sum(
|
||||
len(group.layer_names) for group in
|
||||
kv_cache_groups_one_worker) == len(kv_cache_spec_one_worker), (
|
||||
"Some layers are not assigned to any group.")
|
||||
len(group.layer_names) for group in kv_cache_groups_one_worker
|
||||
) == len(kv_cache_spec_one_worker), "Some layers are not assigned to any group."
|
||||
kv_cache_configs.append(
|
||||
get_kv_cache_config_from_groups(vllm_config,
|
||||
kv_cache_groups_one_worker,
|
||||
kv_cache_spec_one_worker,
|
||||
available_memory_one_worker))
|
||||
get_kv_cache_config_from_groups(
|
||||
vllm_config,
|
||||
kv_cache_groups_one_worker,
|
||||
kv_cache_spec_one_worker,
|
||||
available_memory_one_worker,
|
||||
)
|
||||
)
|
||||
|
||||
# Change the num_blocks of each rank to the smallest among all ranks. We
|
||||
# do not need to shrink the tensor size because it is valid to only use the
|
||||
# first `num_blocks` blocks of the tensor.
|
||||
min_num_blocks = min(kv_cache_config.num_blocks
|
||||
for kv_cache_config in kv_cache_configs)
|
||||
min_num_blocks = min(
|
||||
kv_cache_config.num_blocks for kv_cache_config in kv_cache_configs
|
||||
)
|
||||
for kv_cache_config in kv_cache_configs:
|
||||
kv_cache_config.num_blocks = min_num_blocks
|
||||
|
||||
|
||||
@@ -12,7 +12,6 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
class AsyncScheduler(Scheduler):
|
||||
|
||||
def _update_after_schedule(
|
||||
self,
|
||||
scheduler_output: SchedulerOutput,
|
||||
@@ -20,8 +19,10 @@ class AsyncScheduler(Scheduler):
|
||||
super()._update_after_schedule(scheduler_output)
|
||||
for req_id in scheduler_output.num_scheduled_tokens:
|
||||
request = self.requests[req_id]
|
||||
if (request.num_computed_tokens == request.num_tokens +
|
||||
request.num_output_placeholders):
|
||||
if (
|
||||
request.num_computed_tokens
|
||||
== request.num_tokens + request.num_output_placeholders
|
||||
):
|
||||
# The request will generate a new token in this scheduling step.
|
||||
# TODO(woosuk): Support speculative decoding.
|
||||
request.num_output_placeholders += 1
|
||||
@@ -33,7 +34,8 @@ class AsyncScheduler(Scheduler):
|
||||
) -> tuple[list[int], bool]:
|
||||
status_before_update = request.status
|
||||
new_token_ids, stopped = super()._update_request_with_output(
|
||||
request, new_token_ids)
|
||||
request, new_token_ids
|
||||
)
|
||||
|
||||
# Update the number of output placeholders.
|
||||
request.num_output_placeholders -= len(new_token_ids)
|
||||
@@ -42,6 +44,6 @@ class AsyncScheduler(Scheduler):
|
||||
# Cache the new tokens. Preempted requests should be skipped.
|
||||
if status_before_update == RequestStatus.RUNNING:
|
||||
self.kv_cache_manager.cache_blocks(
|
||||
request,
|
||||
request.num_computed_tokens - request.num_output_placeholders)
|
||||
request, request.num_computed_tokens - request.num_output_placeholders
|
||||
)
|
||||
return new_token_ids, stopped
|
||||
|
||||
@@ -14,7 +14,6 @@ if TYPE_CHECKING:
|
||||
|
||||
|
||||
class SchedulerInterface(ABC):
|
||||
|
||||
@abstractmethod
|
||||
def schedule(self) -> "SchedulerOutput":
|
||||
"""Schedule the requests to process in this scheduling step.
|
||||
@@ -72,7 +71,7 @@ class SchedulerInterface(ABC):
|
||||
@abstractmethod
|
||||
def add_request(self, request: "Request") -> None:
|
||||
"""Add a new request to the scheduler's internal queue.
|
||||
|
||||
|
||||
Args:
|
||||
request: The new request being added.
|
||||
"""
|
||||
@@ -91,7 +90,7 @@ class SchedulerInterface(ABC):
|
||||
1. When the request is aborted by the client.
|
||||
2. When the frontend process detects a stop string of the request after
|
||||
de-tokenizing its generated tokens.
|
||||
|
||||
|
||||
Args:
|
||||
request_ids: A single or a list of request IDs.
|
||||
finished_status: The finished status of the given requests.
|
||||
|
||||
@@ -13,8 +13,7 @@ if TYPE_CHECKING:
|
||||
import numpy.typing as npt
|
||||
import torch
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorMetadata)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import KVConnectorMetadata
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.multimodal.inputs import MultiModalFeatureSpec
|
||||
from vllm.pooling_params import PoolingParams
|
||||
@@ -25,7 +24,6 @@ if TYPE_CHECKING:
|
||||
@bc_linter_include
|
||||
@dataclass
|
||||
class NewRequestData:
|
||||
|
||||
req_id: str
|
||||
prompt_token_ids: Optional[list[int]]
|
||||
mm_features: list[MultiModalFeatureSpec]
|
||||
@@ -55,42 +53,43 @@ class NewRequestData:
|
||||
)
|
||||
|
||||
def __repr__(self) -> str:
|
||||
prompt_embeds_shape = (self.prompt_embeds.shape
|
||||
if self.prompt_embeds else None)
|
||||
return (f"NewRequestData("
|
||||
f"req_id={self.req_id},"
|
||||
f"prompt_token_ids={self.prompt_token_ids},"
|
||||
f"mm_features={self.mm_features},"
|
||||
f"sampling_params={self.sampling_params},"
|
||||
f"block_ids={self.block_ids},"
|
||||
f"num_computed_tokens={self.num_computed_tokens},"
|
||||
f"lora_request={self.lora_request},"
|
||||
f"prompt_embeds_shape={prompt_embeds_shape}"
|
||||
")")
|
||||
prompt_embeds_shape = self.prompt_embeds.shape if self.prompt_embeds else None
|
||||
return (
|
||||
f"NewRequestData("
|
||||
f"req_id={self.req_id},"
|
||||
f"prompt_token_ids={self.prompt_token_ids},"
|
||||
f"mm_features={self.mm_features},"
|
||||
f"sampling_params={self.sampling_params},"
|
||||
f"block_ids={self.block_ids},"
|
||||
f"num_computed_tokens={self.num_computed_tokens},"
|
||||
f"lora_request={self.lora_request},"
|
||||
f"prompt_embeds_shape={prompt_embeds_shape}"
|
||||
")"
|
||||
)
|
||||
|
||||
# Version of __repr__ with the prompt data obfuscated
|
||||
def anon_repr(self) -> str:
|
||||
prompt_token_ids_len = len(
|
||||
self.prompt_token_ids
|
||||
) if self.prompt_token_ids is not None else None
|
||||
prompt_embeds_shape = (self.prompt_embeds.shape
|
||||
if self.prompt_embeds else None)
|
||||
return (f"NewRequestData("
|
||||
f"req_id={self.req_id},"
|
||||
f"prompt_token_ids_len={prompt_token_ids_len},"
|
||||
f"mm_features={self.mm_features},"
|
||||
f"sampling_params={self.sampling_params},"
|
||||
f"block_ids={self.block_ids},"
|
||||
f"num_computed_tokens={self.num_computed_tokens},"
|
||||
f"lora_request={self.lora_request},"
|
||||
f"prompt_embeds_shape={prompt_embeds_shape}"
|
||||
")")
|
||||
prompt_token_ids_len = (
|
||||
len(self.prompt_token_ids) if self.prompt_token_ids is not None else None
|
||||
)
|
||||
prompt_embeds_shape = self.prompt_embeds.shape if self.prompt_embeds else None
|
||||
return (
|
||||
f"NewRequestData("
|
||||
f"req_id={self.req_id},"
|
||||
f"prompt_token_ids_len={prompt_token_ids_len},"
|
||||
f"mm_features={self.mm_features},"
|
||||
f"sampling_params={self.sampling_params},"
|
||||
f"block_ids={self.block_ids},"
|
||||
f"num_computed_tokens={self.num_computed_tokens},"
|
||||
f"lora_request={self.lora_request},"
|
||||
f"prompt_embeds_shape={prompt_embeds_shape}"
|
||||
")"
|
||||
)
|
||||
|
||||
|
||||
@bc_linter_include
|
||||
@dataclass
|
||||
class CachedRequestData:
|
||||
|
||||
req_ids: list[str]
|
||||
# If resumed_from_preemption is False, new_block_ids will be appended to
|
||||
# the request's block IDs. If True, new_block_ids will be used as the
|
||||
@@ -122,7 +121,6 @@ class CachedRequestData:
|
||||
@bc_linter_include
|
||||
@dataclass
|
||||
class SchedulerOutput:
|
||||
|
||||
# list of the requests that are scheduled for the first time.
|
||||
# We cache the request's data in each worker process, so that we don't
|
||||
# need to re-send it every scheduling step.
|
||||
|
||||
@@ -14,6 +14,7 @@ from vllm.v1.request import Request
|
||||
|
||||
class SchedulingPolicy(Enum):
|
||||
"""Enum for scheduling policies."""
|
||||
|
||||
FCFS = "fcfs"
|
||||
PRIORITY = "priority"
|
||||
|
||||
@@ -111,9 +112,7 @@ class FCFSRequestQueue(deque[Request], RequestQueue):
|
||||
def remove_requests(self, requests: Iterable[Request]) -> None:
|
||||
"""Remove multiple specific requests from the queue."""
|
||||
requests_to_remove = set(requests)
|
||||
filtered_requests = [
|
||||
req for req in self if req not in requests_to_remove
|
||||
]
|
||||
filtered_requests = [req for req in self if req not in requests_to_remove]
|
||||
# deque does not support in-place filtering, so we need to clear
|
||||
# and extend
|
||||
self.clear()
|
||||
@@ -150,8 +149,7 @@ class PriorityRequestQueue(RequestQueue):
|
||||
|
||||
def add_request(self, request: Request) -> None:
|
||||
"""Add a request to the queue according to priority policy."""
|
||||
heapq.heappush(self._heap,
|
||||
(request.priority, request.arrival_time, request))
|
||||
heapq.heappush(self._heap, (request.priority, request.arrival_time, request))
|
||||
|
||||
def pop_request(self) -> Request:
|
||||
"""Pop a request from the queue according to priority policy."""
|
||||
@@ -169,15 +167,15 @@ class PriorityRequestQueue(RequestQueue):
|
||||
|
||||
def prepend_request(self, request: Request) -> None:
|
||||
"""Add a request to the queue according to priority policy.
|
||||
|
||||
Note: In a priority queue, there is no concept of prepending to the
|
||||
|
||||
Note: In a priority queue, there is no concept of prepending to the
|
||||
front. Requests are ordered by (priority, arrival_time)."""
|
||||
self.add_request(request)
|
||||
|
||||
def prepend_requests(self, requests: RequestQueue) -> None:
|
||||
"""Add all requests from another queue according to priority policy.
|
||||
|
||||
Note: In a priority queue, there is no concept of prepending to the
|
||||
|
||||
Note: In a priority queue, there is no concept of prepending to the
|
||||
front. Requests are ordered by (priority, arrival_time)."""
|
||||
for request in requests:
|
||||
self.add_request(request)
|
||||
@@ -190,8 +188,9 @@ class PriorityRequestQueue(RequestQueue):
|
||||
def remove_requests(self, requests: Iterable[Request]) -> None:
|
||||
"""Remove multiple specific requests from the queue."""
|
||||
requests_to_remove = set(requests)
|
||||
self._heap = [(p, t, r) for p, t, r in self._heap
|
||||
if r not in requests_to_remove]
|
||||
self._heap = [
|
||||
(p, t, r) for p, t, r in self._heap if r not in requests_to_remove
|
||||
]
|
||||
heapq.heapify(self._heap)
|
||||
|
||||
def __bool__(self) -> bool:
|
||||
|
||||
+246
-186
@@ -11,25 +11,24 @@ from typing import Any, Optional, Union
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.kv_events import EventPublisherFactory, KVEventBatch
|
||||
from vllm.distributed.kv_transfer.kv_connector.factory import (
|
||||
KVConnectorFactory)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1 import (KVConnectorBase_V1,
|
||||
KVConnectorRole)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
|
||||
KVConnectorStats)
|
||||
from vllm.distributed.kv_transfer.kv_connector.factory import KVConnectorFactory
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1 import (
|
||||
KVConnectorBase_V1,
|
||||
KVConnectorRole,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import KVConnectorStats
|
||||
from vllm.logger import init_logger
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry
|
||||
from vllm.v1.core.encoder_cache_manager import (EncoderCacheManager,
|
||||
compute_encoder_budget)
|
||||
from vllm.v1.core.encoder_cache_manager import (
|
||||
EncoderCacheManager,
|
||||
compute_encoder_budget,
|
||||
)
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks, KVCacheManager
|
||||
from vllm.v1.core.sched.interface import SchedulerInterface
|
||||
from vllm.v1.core.sched.output import (CachedRequestData, NewRequestData,
|
||||
SchedulerOutput)
|
||||
from vllm.v1.core.sched.request_queue import (SchedulingPolicy,
|
||||
create_request_queue)
|
||||
from vllm.v1.core.sched.output import CachedRequestData, NewRequestData, SchedulerOutput
|
||||
from vllm.v1.core.sched.request_queue import SchedulingPolicy, create_request_queue
|
||||
from vllm.v1.core.sched.utils import check_stop, remove_all
|
||||
from vllm.v1.engine import (EngineCoreEventType, EngineCoreOutput,
|
||||
EngineCoreOutputs)
|
||||
from vllm.v1.engine import EngineCoreEventType, EngineCoreOutput, EngineCoreOutputs
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.metrics.stats import SchedulerStats
|
||||
from vllm.v1.outputs import DraftTokenIds, KVConnectorOutput, ModelRunnerOutput
|
||||
@@ -41,7 +40,6 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
class Scheduler(SchedulerInterface):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
@@ -67,16 +65,17 @@ class Scheduler(SchedulerInterface):
|
||||
# by update_from_outputs(). This is currently used in the multi-engine
|
||||
# case to track request lifetimes efficiently.
|
||||
self.finished_req_ids_dict: Optional[dict[int, set[str]]] = (
|
||||
defaultdict(set) if include_finished_set else None)
|
||||
defaultdict(set) if include_finished_set else None
|
||||
)
|
||||
|
||||
# Scheduling constraints.
|
||||
self.max_num_running_reqs = self.scheduler_config.max_num_seqs
|
||||
self.max_num_scheduled_tokens = \
|
||||
self.scheduler_config.max_num_batched_tokens
|
||||
self.max_num_scheduled_tokens = self.scheduler_config.max_num_batched_tokens
|
||||
self.max_model_len = self.scheduler_config.max_model_len
|
||||
self.enable_kv_cache_events = (
|
||||
self.kv_events_config is not None
|
||||
and self.kv_events_config.enable_kv_cache_events)
|
||||
and self.kv_events_config.enable_kv_cache_events
|
||||
)
|
||||
|
||||
# Create KVConnector for the Scheduler. Note that each Worker
|
||||
# will have a corresponding KVConnector with Role=WORKER.
|
||||
@@ -85,12 +84,14 @@ class Scheduler(SchedulerInterface):
|
||||
if self.vllm_config.kv_transfer_config is not None:
|
||||
assert len(self.kv_cache_config.kv_cache_groups) == 1, (
|
||||
"Multiple KV cache groups are not currently supported "
|
||||
"with KV connectors")
|
||||
"with KV connectors"
|
||||
)
|
||||
assert not self.is_encoder_decoder, (
|
||||
"Encoder-decoder models are not currently supported "
|
||||
"with KV connectors")
|
||||
"Encoder-decoder models are not currently supported with KV connectors"
|
||||
)
|
||||
self.connector = KVConnectorFactory.create_connector(
|
||||
config=self.vllm_config, role=KVConnectorRole.SCHEDULER)
|
||||
config=self.vllm_config, role=KVConnectorRole.SCHEDULER
|
||||
)
|
||||
|
||||
self.kv_event_publisher = EventPublisherFactory.create(
|
||||
self.kv_events_config,
|
||||
@@ -102,8 +103,7 @@ class Scheduler(SchedulerInterface):
|
||||
|
||||
self.block_size = self.cache_config.block_size
|
||||
|
||||
self.dcp_world_size = \
|
||||
vllm_config.parallel_config.decode_context_parallel_size
|
||||
self.dcp_world_size = vllm_config.parallel_config.decode_context_parallel_size
|
||||
# Note(hc): The scheduler’s block_size must be multiplied
|
||||
# by dcp_world_size, since block hashes are computed on the
|
||||
# original full token sequence at a granularity of
|
||||
@@ -120,7 +120,8 @@ class Scheduler(SchedulerInterface):
|
||||
self.policy = SchedulingPolicy.FCFS
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Unknown scheduling policy: {self.scheduler_config.policy}")
|
||||
f"Unknown scheduling policy: {self.scheduler_config.policy}"
|
||||
)
|
||||
# Priority queues for requests.
|
||||
self.waiting = create_request_queue(self.policy)
|
||||
self.running: list[Request] = []
|
||||
@@ -153,8 +154,7 @@ class Scheduler(SchedulerInterface):
|
||||
# NOTE: For the models without encoder (e.g., text-only models),
|
||||
# the encoder cache will not be initialized because cache size is 0
|
||||
# for these models.
|
||||
self.encoder_cache_manager = EncoderCacheManager(
|
||||
cache_size=encoder_cache_size)
|
||||
self.encoder_cache_manager = EncoderCacheManager(cache_size=encoder_cache_size)
|
||||
|
||||
speculative_config = vllm_config.speculative_config
|
||||
self.use_eagle = False
|
||||
@@ -211,30 +211,35 @@ class Scheduler(SchedulerInterface):
|
||||
while req_index < len(self.running) and token_budget > 0:
|
||||
request = self.running[req_index]
|
||||
|
||||
num_new_tokens = (request.num_tokens_with_spec +
|
||||
request.num_output_placeholders -
|
||||
request.num_computed_tokens)
|
||||
if (0 < self.scheduler_config.long_prefill_token_threshold <
|
||||
num_new_tokens):
|
||||
num_new_tokens = (
|
||||
self.scheduler_config.long_prefill_token_threshold)
|
||||
num_new_tokens = (
|
||||
request.num_tokens_with_spec
|
||||
+ request.num_output_placeholders
|
||||
- request.num_computed_tokens
|
||||
)
|
||||
if 0 < self.scheduler_config.long_prefill_token_threshold < num_new_tokens:
|
||||
num_new_tokens = self.scheduler_config.long_prefill_token_threshold
|
||||
num_new_tokens = min(num_new_tokens, token_budget)
|
||||
|
||||
# Make sure the input position does not exceed the max model len.
|
||||
# This is necessary when using spec decoding.
|
||||
num_new_tokens = min(
|
||||
num_new_tokens,
|
||||
self.max_model_len - request.num_computed_tokens)
|
||||
num_new_tokens, self.max_model_len - request.num_computed_tokens
|
||||
)
|
||||
|
||||
# Schedule encoder inputs.
|
||||
encoder_inputs_to_schedule = None
|
||||
new_encoder_compute_budget = encoder_compute_budget
|
||||
if request.has_encoder_inputs:
|
||||
(encoder_inputs_to_schedule, num_new_tokens,
|
||||
new_encoder_compute_budget
|
||||
) = self._try_schedule_encoder_inputs(
|
||||
request, request.num_computed_tokens, num_new_tokens,
|
||||
encoder_compute_budget)
|
||||
(
|
||||
encoder_inputs_to_schedule,
|
||||
num_new_tokens,
|
||||
new_encoder_compute_budget,
|
||||
) = self._try_schedule_encoder_inputs(
|
||||
request,
|
||||
request.num_computed_tokens,
|
||||
num_new_tokens,
|
||||
encoder_compute_budget,
|
||||
)
|
||||
|
||||
if num_new_tokens == 0:
|
||||
# The request cannot be scheduled because one of the following
|
||||
@@ -257,7 +262,8 @@ class Scheduler(SchedulerInterface):
|
||||
new_blocks = self.kv_cache_manager.allocate_slots(
|
||||
request,
|
||||
num_new_tokens,
|
||||
num_lookahead_tokens=self.num_lookahead_tokens)
|
||||
num_lookahead_tokens=self.num_lookahead_tokens,
|
||||
)
|
||||
|
||||
if new_blocks is not None:
|
||||
# The request can be scheduled.
|
||||
@@ -282,8 +288,9 @@ class Scheduler(SchedulerInterface):
|
||||
preempted_req.num_computed_tokens = 0
|
||||
preempted_req.num_preemptions += 1
|
||||
if self.log_stats:
|
||||
preempted_req.record_event(EngineCoreEventType.PREEMPTED,
|
||||
scheduled_timestamp)
|
||||
preempted_req.record_event(
|
||||
EngineCoreEventType.PREEMPTED, scheduled_timestamp
|
||||
)
|
||||
|
||||
self.waiting.prepend_request(preempted_req)
|
||||
preempted_reqs.append(preempted_req)
|
||||
@@ -304,19 +311,21 @@ class Scheduler(SchedulerInterface):
|
||||
|
||||
# Speculative decode related.
|
||||
if request.spec_token_ids:
|
||||
num_scheduled_spec_tokens = (num_new_tokens +
|
||||
request.num_computed_tokens -
|
||||
request.num_tokens)
|
||||
num_scheduled_spec_tokens = (
|
||||
num_new_tokens + request.num_computed_tokens - request.num_tokens
|
||||
)
|
||||
if num_scheduled_spec_tokens > 0:
|
||||
# Trim spec_token_ids list to num_scheduled_spec_tokens.
|
||||
del request.spec_token_ids[num_scheduled_spec_tokens:]
|
||||
scheduled_spec_decode_tokens[request.request_id] = (
|
||||
request.spec_token_ids)
|
||||
request.spec_token_ids
|
||||
)
|
||||
|
||||
# Encoder-related.
|
||||
if encoder_inputs_to_schedule:
|
||||
scheduled_encoder_inputs[request.request_id] = (
|
||||
encoder_inputs_to_schedule)
|
||||
encoder_inputs_to_schedule
|
||||
)
|
||||
# Allocate the encoder cache.
|
||||
for i in encoder_inputs_to_schedule:
|
||||
self.encoder_cache_manager.allocate(request, i)
|
||||
@@ -326,8 +335,10 @@ class Scheduler(SchedulerInterface):
|
||||
scheduled_loras: set[int] = set()
|
||||
if self.lora_config:
|
||||
scheduled_loras = set(
|
||||
req.lora_request.lora_int_id for req in scheduled_running_reqs
|
||||
if req.lora_request and req.lora_request.lora_int_id > 0)
|
||||
req.lora_request.lora_int_id
|
||||
for req in scheduled_running_reqs
|
||||
if req.lora_request and req.lora_request.lora_int_id > 0
|
||||
)
|
||||
assert len(scheduled_loras) <= self.lora_config.max_loras
|
||||
|
||||
# Use a temporary RequestQueue to collect requests that need to be
|
||||
@@ -350,7 +361,8 @@ class Scheduler(SchedulerInterface):
|
||||
else:
|
||||
logger.debug(
|
||||
"%s is still in WAITING_FOR_REMOTE_KVS state.",
|
||||
request.request_id)
|
||||
request.request_id,
|
||||
)
|
||||
self.waiting.pop_request()
|
||||
skipped_waiting_requests.prepend_request(request)
|
||||
continue
|
||||
@@ -368,9 +380,14 @@ class Scheduler(SchedulerInterface):
|
||||
|
||||
# Check that adding the request still respects the max_loras
|
||||
# constraint.
|
||||
if (self.lora_config and request.lora_request and
|
||||
(len(scheduled_loras) == self.lora_config.max_loras and
|
||||
request.lora_request.lora_int_id not in scheduled_loras)):
|
||||
if (
|
||||
self.lora_config
|
||||
and request.lora_request
|
||||
and (
|
||||
len(scheduled_loras) == self.lora_config.max_loras
|
||||
and request.lora_request.lora_int_id not in scheduled_loras
|
||||
)
|
||||
):
|
||||
# Scheduling would exceed max_loras, skip.
|
||||
self.waiting.pop_request()
|
||||
skipped_waiting_requests.prepend_request(request)
|
||||
@@ -382,15 +399,17 @@ class Scheduler(SchedulerInterface):
|
||||
# Get already-cached tokens.
|
||||
if request.num_computed_tokens == 0:
|
||||
# Get locally-cached tokens.
|
||||
new_computed_blocks, num_new_local_computed_tokens = \
|
||||
self.kv_cache_manager.get_computed_blocks(
|
||||
request)
|
||||
new_computed_blocks, num_new_local_computed_tokens = (
|
||||
self.kv_cache_manager.get_computed_blocks(request)
|
||||
)
|
||||
|
||||
# Get externally-cached tokens if using a KVConnector.
|
||||
if self.connector is not None:
|
||||
num_external_computed_tokens, load_kv_async = (
|
||||
self.connector.get_num_new_matched_tokens(
|
||||
request, num_new_local_computed_tokens))
|
||||
request, num_new_local_computed_tokens
|
||||
)
|
||||
)
|
||||
|
||||
if num_external_computed_tokens is None:
|
||||
# The request cannot be scheduled because
|
||||
@@ -401,13 +420,15 @@ class Scheduler(SchedulerInterface):
|
||||
continue
|
||||
|
||||
# Total computed tokens (local + external).
|
||||
num_computed_tokens = (num_new_local_computed_tokens +
|
||||
num_external_computed_tokens)
|
||||
num_computed_tokens = (
|
||||
num_new_local_computed_tokens + num_external_computed_tokens
|
||||
)
|
||||
# KVTransfer: WAITING reqs have num_computed_tokens > 0
|
||||
# after async KV recvs are completed.
|
||||
else:
|
||||
new_computed_blocks = (
|
||||
self.kv_cache_manager.create_empty_block_list())
|
||||
self.kv_cache_manager.create_empty_block_list()
|
||||
)
|
||||
num_new_local_computed_tokens = 0
|
||||
num_computed_tokens = request.num_computed_tokens
|
||||
|
||||
@@ -424,15 +445,21 @@ class Scheduler(SchedulerInterface):
|
||||
# `request.num_prompt_tokens` to consider the resumed
|
||||
# requests, which have output tokens.
|
||||
num_new_tokens = request.num_tokens - num_computed_tokens
|
||||
if (0 < self.scheduler_config.long_prefill_token_threshold
|
||||
< num_new_tokens):
|
||||
if (
|
||||
0
|
||||
< self.scheduler_config.long_prefill_token_threshold
|
||||
< num_new_tokens
|
||||
):
|
||||
num_new_tokens = (
|
||||
self.scheduler_config.long_prefill_token_threshold)
|
||||
self.scheduler_config.long_prefill_token_threshold
|
||||
)
|
||||
|
||||
# chunked prefill has to be enabled explicitly to allow
|
||||
# pooling requests to be chunked
|
||||
if not self.scheduler_config.chunked_prefill_enabled and \
|
||||
num_new_tokens > token_budget:
|
||||
if (
|
||||
not self.scheduler_config.chunked_prefill_enabled
|
||||
and num_new_tokens > token_budget
|
||||
):
|
||||
self.waiting.pop_request()
|
||||
skipped_waiting_requests.prepend_request(request)
|
||||
continue
|
||||
@@ -442,11 +469,16 @@ class Scheduler(SchedulerInterface):
|
||||
|
||||
# Schedule encoder inputs.
|
||||
if request.has_encoder_inputs:
|
||||
(encoder_inputs_to_schedule, num_new_tokens,
|
||||
new_encoder_compute_budget
|
||||
) = self._try_schedule_encoder_inputs(
|
||||
request, num_computed_tokens, num_new_tokens,
|
||||
encoder_compute_budget)
|
||||
(
|
||||
encoder_inputs_to_schedule,
|
||||
num_new_tokens,
|
||||
new_encoder_compute_budget,
|
||||
) = self._try_schedule_encoder_inputs(
|
||||
request,
|
||||
num_computed_tokens,
|
||||
num_new_tokens,
|
||||
encoder_compute_budget,
|
||||
)
|
||||
if num_new_tokens == 0:
|
||||
# The request cannot be scheduled.
|
||||
break
|
||||
@@ -456,9 +488,9 @@ class Scheduler(SchedulerInterface):
|
||||
# extra block gets allocated which
|
||||
# creates a mismatch between the number
|
||||
# of local and remote blocks.
|
||||
effective_lookahead_tokens = (0 if request.num_computed_tokens
|
||||
== 0 else
|
||||
self.num_lookahead_tokens)
|
||||
effective_lookahead_tokens = (
|
||||
0 if request.num_computed_tokens == 0 else self.num_lookahead_tokens
|
||||
)
|
||||
|
||||
# Determine if we need to allocate cross-attention blocks.
|
||||
if self.is_encoder_decoder and request.has_encoder_inputs:
|
||||
@@ -466,8 +498,9 @@ class Scheduler(SchedulerInterface):
|
||||
# always padded to the maximum length. If we support other
|
||||
# encoder-decoder models, this will need to be updated if we
|
||||
# want to only allocate what is needed.
|
||||
num_encoder_tokens =\
|
||||
num_encoder_tokens = (
|
||||
self.scheduler_config.max_num_encoder_input_tokens
|
||||
)
|
||||
else:
|
||||
num_encoder_tokens = 0
|
||||
|
||||
@@ -509,20 +542,21 @@ class Scheduler(SchedulerInterface):
|
||||
req_index += 1
|
||||
self.running.append(request)
|
||||
if self.log_stats:
|
||||
request.record_event(EngineCoreEventType.SCHEDULED,
|
||||
scheduled_timestamp)
|
||||
request.record_event(
|
||||
EngineCoreEventType.SCHEDULED, scheduled_timestamp
|
||||
)
|
||||
if request.status == RequestStatus.WAITING:
|
||||
scheduled_new_reqs.append(request)
|
||||
elif request.status == RequestStatus.PREEMPTED:
|
||||
scheduled_resumed_reqs.append(request)
|
||||
else:
|
||||
raise RuntimeError(
|
||||
f"Invalid request status: {request.status}")
|
||||
raise RuntimeError(f"Invalid request status: {request.status}")
|
||||
|
||||
if self.lora_config and request.lora_request:
|
||||
scheduled_loras.add(request.lora_request.lora_int_id)
|
||||
req_to_new_blocks[request.request_id] = (
|
||||
self.kv_cache_manager.get_blocks(request.request_id))
|
||||
self.kv_cache_manager.get_blocks(request.request_id)
|
||||
)
|
||||
num_scheduled_tokens[request.request_id] = num_new_tokens
|
||||
token_budget -= num_new_tokens
|
||||
request.status = RequestStatus.RUNNING
|
||||
@@ -533,7 +567,8 @@ class Scheduler(SchedulerInterface):
|
||||
# Encoder-related.
|
||||
if encoder_inputs_to_schedule:
|
||||
scheduled_encoder_inputs[request.request_id] = (
|
||||
encoder_inputs_to_schedule)
|
||||
encoder_inputs_to_schedule
|
||||
)
|
||||
# Allocate the encoder cache.
|
||||
for i in encoder_inputs_to_schedule:
|
||||
self.encoder_cache_manager.allocate(request, i)
|
||||
@@ -551,23 +586,26 @@ class Scheduler(SchedulerInterface):
|
||||
# Since some requests in the RUNNING queue may not be scheduled in
|
||||
# this step, the total number of scheduled requests can be smaller than
|
||||
# len(self.running).
|
||||
assert (len(scheduled_new_reqs) + len(scheduled_resumed_reqs) +
|
||||
len(scheduled_running_reqs) <= len(self.running))
|
||||
assert len(scheduled_new_reqs) + len(scheduled_resumed_reqs) + len(
|
||||
scheduled_running_reqs
|
||||
) <= len(self.running)
|
||||
|
||||
# Get the longest common prefix among all requests in the running queue.
|
||||
# This can be potentially used for cascade attention.
|
||||
num_common_prefix_blocks = [0] * len(
|
||||
self.kv_cache_config.kv_cache_groups)
|
||||
num_common_prefix_blocks = [0] * len(self.kv_cache_config.kv_cache_groups)
|
||||
if self.running:
|
||||
any_request = self.running[0]
|
||||
num_common_prefix_blocks = (
|
||||
self.kv_cache_manager.get_num_common_prefix_blocks(
|
||||
any_request, len(self.running)))
|
||||
any_request, len(self.running)
|
||||
)
|
||||
)
|
||||
|
||||
# Construct the scheduler output.
|
||||
new_reqs_data = [
|
||||
NewRequestData.from_request(
|
||||
req, req_to_new_blocks[req.request_id].get_block_ids())
|
||||
req, req_to_new_blocks[req.request_id].get_block_ids()
|
||||
)
|
||||
for req in scheduled_new_reqs
|
||||
]
|
||||
cached_reqs_data = self._make_cached_request_data(
|
||||
@@ -577,11 +615,12 @@ class Scheduler(SchedulerInterface):
|
||||
scheduled_spec_decode_tokens,
|
||||
req_to_new_blocks,
|
||||
)
|
||||
scheduled_requests = (scheduled_new_reqs + scheduled_running_reqs +
|
||||
scheduled_resumed_reqs)
|
||||
structured_output_request_ids, grammar_bitmask = (
|
||||
self.get_grammar_bitmask(scheduled_requests,
|
||||
scheduled_spec_decode_tokens))
|
||||
scheduled_requests = (
|
||||
scheduled_new_reqs + scheduled_running_reqs + scheduled_resumed_reqs
|
||||
)
|
||||
structured_output_request_ids, grammar_bitmask = self.get_grammar_bitmask(
|
||||
scheduled_requests, scheduled_spec_decode_tokens
|
||||
)
|
||||
scheduler_output = SchedulerOutput(
|
||||
scheduled_new_reqs=new_reqs_data,
|
||||
scheduled_cached_reqs=cached_reqs_data,
|
||||
@@ -595,8 +634,7 @@ class Scheduler(SchedulerInterface):
|
||||
# It contains the request IDs that are finished in between
|
||||
# the previous and the current steps.
|
||||
finished_req_ids=self.finished_req_ids,
|
||||
free_encoder_mm_hashes=self.encoder_cache_manager.
|
||||
get_freed_mm_hashes(),
|
||||
free_encoder_mm_hashes=self.encoder_cache_manager.get_freed_mm_hashes(),
|
||||
structured_output_request_ids=structured_output_request_ids,
|
||||
grammar_bitmask=grammar_bitmask,
|
||||
)
|
||||
@@ -678,16 +716,18 @@ class Scheduler(SchedulerInterface):
|
||||
for req in itertools.chain(running_reqs, resumed_reqs):
|
||||
req_id = req.request_id
|
||||
req_ids.append(req_id)
|
||||
num_tokens = (num_scheduled_tokens[req_id] -
|
||||
len(spec_decode_tokens.get(req_id, ())))
|
||||
num_tokens = num_scheduled_tokens[req_id] - len(
|
||||
spec_decode_tokens.get(req_id, ())
|
||||
)
|
||||
if self.use_pp:
|
||||
# When using PP, the scheduler sends the sampled tokens back,
|
||||
# because there's no direct communication between the first-
|
||||
# stage worker and the last-stage worker. Otherwise, we don't
|
||||
# need to send the sampled tokens back because the model runner
|
||||
# will cache them.
|
||||
token_ids = req.all_token_ids[req.num_computed_tokens:req.
|
||||
num_computed_tokens + num_tokens]
|
||||
token_ids = req.all_token_ids[
|
||||
req.num_computed_tokens : req.num_computed_tokens + num_tokens
|
||||
]
|
||||
new_token_ids.append(token_ids)
|
||||
elif use_connector:
|
||||
# When using a KVConnector, we add a placeholder to avoid index
|
||||
@@ -695,7 +735,8 @@ class Scheduler(SchedulerInterface):
|
||||
# is updated to handle token IDs properly.
|
||||
new_token_ids.append([])
|
||||
new_block_ids.append(
|
||||
req_to_new_blocks[req_id].get_block_ids(allow_none=True))
|
||||
req_to_new_blocks[req_id].get_block_ids(allow_none=True)
|
||||
)
|
||||
num_computed_tokens.append(req.num_computed_tokens)
|
||||
num_output_tokens.append(len(req.output_token_ids))
|
||||
# Because resumed_reqs is usually empty, it is more efficient to do
|
||||
@@ -764,7 +805,8 @@ class Scheduler(SchedulerInterface):
|
||||
if self.is_encoder_decoder and num_computed_tokens > 0:
|
||||
assert start_pos == 0, (
|
||||
"Encoder input should be processed at the beginning of "
|
||||
"the sequence when encoder-decoder models are used.")
|
||||
"the sequence when encoder-decoder models are used."
|
||||
)
|
||||
# Encoder input has already been computed
|
||||
# The calculation here is a bit different. We don't turn encoder
|
||||
# output into tokens that get processed by the decoder and
|
||||
@@ -788,8 +830,7 @@ class Scheduler(SchedulerInterface):
|
||||
# current step.
|
||||
continue
|
||||
|
||||
if self.encoder_cache_manager.check_and_update_cache(
|
||||
request, i):
|
||||
if self.encoder_cache_manager.check_and_update_cache(request, i):
|
||||
# The encoder input is already computed and cached from a
|
||||
# previous step.
|
||||
continue
|
||||
@@ -797,16 +838,18 @@ class Scheduler(SchedulerInterface):
|
||||
# If no encoder input chunking is allowed, we do not want to
|
||||
# partially schedule a multimodal item. If the scheduled range would
|
||||
# only cover part of the mm input, roll back to before the mm item.
|
||||
if (self.scheduler_config.disable_chunked_mm_input
|
||||
and num_computed_tokens < start_pos
|
||||
and (num_computed_tokens + num_new_tokens)
|
||||
< (start_pos + num_encoder_tokens)):
|
||||
if (
|
||||
self.scheduler_config.disable_chunked_mm_input
|
||||
and num_computed_tokens < start_pos
|
||||
and (num_computed_tokens + num_new_tokens)
|
||||
< (start_pos + num_encoder_tokens)
|
||||
):
|
||||
num_new_tokens = start_pos - num_computed_tokens
|
||||
break
|
||||
|
||||
if not self.encoder_cache_manager.can_allocate(
|
||||
request, i, encoder_compute_budget,
|
||||
num_tokens_to_schedule):
|
||||
request, i, encoder_compute_budget, num_tokens_to_schedule
|
||||
):
|
||||
# The encoder cache is full or the encoder budget is exhausted.
|
||||
# NOTE(woosuk): We assume that the encoder input tokens should
|
||||
# be processed altogether, as the encoder usually uses
|
||||
@@ -879,8 +922,9 @@ class Scheduler(SchedulerInterface):
|
||||
|
||||
outputs: dict[int, list[EngineCoreOutput]] = defaultdict(list)
|
||||
spec_decoding_stats: Optional[SpecDecodingStats] = None
|
||||
kv_connector_stats = (kv_connector_output.kv_connector_stats
|
||||
if kv_connector_output else None)
|
||||
kv_connector_stats = (
|
||||
kv_connector_output.kv_connector_stats if kv_connector_output else None
|
||||
)
|
||||
|
||||
failed_kv_load_req_ids = None
|
||||
if kv_connector_output and kv_connector_output.invalid_block_ids:
|
||||
@@ -888,7 +932,8 @@ class Scheduler(SchedulerInterface):
|
||||
# load. Identify affected requests and adjust their computed token
|
||||
# count to trigger recomputation of the invalid blocks.
|
||||
failed_kv_load_req_ids = self._handle_invalid_blocks(
|
||||
kv_connector_output.invalid_block_ids)
|
||||
kv_connector_output.invalid_block_ids
|
||||
)
|
||||
|
||||
# NOTE(woosuk): As len(num_scheduled_tokens) can be up to 1K or more,
|
||||
# the below loop can be a performance bottleneck. We should do our best
|
||||
@@ -908,11 +953,13 @@ class Scheduler(SchedulerInterface):
|
||||
continue
|
||||
|
||||
req_index = model_runner_output.req_id_to_index[req_id]
|
||||
generated_token_ids = sampled_token_ids[
|
||||
req_index] if sampled_token_ids else []
|
||||
generated_token_ids = (
|
||||
sampled_token_ids[req_index] if sampled_token_ids else []
|
||||
)
|
||||
|
||||
scheduled_spec_token_ids = (
|
||||
scheduler_output.scheduled_spec_decode_tokens.get(req_id))
|
||||
scheduler_output.scheduled_spec_decode_tokens.get(req_id)
|
||||
)
|
||||
if scheduled_spec_token_ids:
|
||||
num_draft_tokens = len(scheduled_spec_token_ids)
|
||||
num_accepted = len(generated_token_ids) - 1
|
||||
@@ -926,7 +973,8 @@ class Scheduler(SchedulerInterface):
|
||||
spec_decoding_stats = self.make_spec_decoding_stats(
|
||||
spec_decoding_stats,
|
||||
num_draft_tokens=num_draft_tokens,
|
||||
num_accepted_tokens=num_accepted)
|
||||
num_accepted_tokens=num_accepted,
|
||||
)
|
||||
|
||||
stopped = False
|
||||
new_logprobs = None
|
||||
@@ -937,14 +985,14 @@ class Scheduler(SchedulerInterface):
|
||||
# Check for stop and update request status.
|
||||
if new_token_ids:
|
||||
new_token_ids, stopped = self._update_request_with_output(
|
||||
request, new_token_ids)
|
||||
request, new_token_ids
|
||||
)
|
||||
|
||||
# Stop checking for pooler models.
|
||||
pooler_output = None
|
||||
if pooler_outputs:
|
||||
pooler_output = pooler_outputs[req_index]
|
||||
stopped = check_stop(request, self.max_model_len,
|
||||
pooler_output)
|
||||
stopped = check_stop(request, self.max_model_len, pooler_output)
|
||||
|
||||
if stopped:
|
||||
kv_transfer_params = self._free_request(request)
|
||||
@@ -954,28 +1002,29 @@ class Scheduler(SchedulerInterface):
|
||||
stopped_preempted_reqs.add(request)
|
||||
|
||||
# Extract sample logprobs if needed.
|
||||
if request.sampling_params is not None \
|
||||
and request.sampling_params.logprobs is not None and logprobs:
|
||||
if (
|
||||
request.sampling_params is not None
|
||||
and request.sampling_params.logprobs is not None
|
||||
and logprobs
|
||||
):
|
||||
# NOTE: once we support N tokens per step (spec decode),
|
||||
# the outer lists can be of length > 1.
|
||||
new_logprobs = logprobs.slice(req_index, req_index + 1)
|
||||
|
||||
if new_token_ids and self.structured_output_manager.should_advance(
|
||||
request):
|
||||
if new_token_ids and self.structured_output_manager.should_advance(request):
|
||||
# NOTE: structured_output_request
|
||||
# should not be None if use_structured_output, we have
|
||||
# checked above, so safe to ignore type warning
|
||||
request.structured_output_request.grammar.accept_tokens( # type: ignore[union-attr]
|
||||
req_id, new_token_ids)
|
||||
req_id, new_token_ids
|
||||
)
|
||||
|
||||
if num_nans_in_logits is not None and req_id in num_nans_in_logits:
|
||||
request.num_nans_in_logits = num_nans_in_logits[req_id]
|
||||
|
||||
# Get prompt logprobs for this request.
|
||||
prompt_logprobs_tensors = prompt_logprobs_dict.get(req_id)
|
||||
if new_token_ids or pooler_output is not None \
|
||||
or kv_transfer_params:
|
||||
|
||||
if new_token_ids or pooler_output is not None or kv_transfer_params:
|
||||
# Add EngineCoreOutput for this Request.
|
||||
outputs[request.client_index].append(
|
||||
EngineCoreOutput(
|
||||
@@ -990,7 +1039,8 @@ class Scheduler(SchedulerInterface):
|
||||
kv_transfer_params=kv_transfer_params,
|
||||
trace_headers=request.trace_headers,
|
||||
num_cached_tokens=request.num_cached_tokens,
|
||||
))
|
||||
)
|
||||
)
|
||||
else:
|
||||
# Invariant: EngineCore returns no partial prefill outputs.
|
||||
assert not prompt_logprobs_tensors
|
||||
@@ -1023,11 +1073,13 @@ class Scheduler(SchedulerInterface):
|
||||
eco.finished_requests = finished_set
|
||||
else:
|
||||
engine_core_outputs[client_index] = EngineCoreOutputs(
|
||||
finished_requests=finished_set)
|
||||
finished_requests=finished_set
|
||||
)
|
||||
finished_req_ids.clear()
|
||||
|
||||
if (stats := self.make_stats(spec_decoding_stats,
|
||||
kv_connector_stats)) is not None:
|
||||
if (
|
||||
stats := self.make_stats(spec_decoding_stats, kv_connector_stats)
|
||||
) is not None:
|
||||
# Return stats to only one of the front-ends.
|
||||
if (eco := next(iter(engine_core_outputs.values()), None)) is None:
|
||||
# We must return the stats even if there are no request
|
||||
@@ -1058,8 +1110,9 @@ class Scheduler(SchedulerInterface):
|
||||
return new_token_ids, stopped
|
||||
|
||||
def _free_encoder_inputs(self, request: Request) -> None:
|
||||
cached_encoder_input_ids = (
|
||||
self.encoder_cache_manager.get_cached_input_ids(request))
|
||||
cached_encoder_input_ids = self.encoder_cache_manager.get_cached_input_ids(
|
||||
request
|
||||
)
|
||||
# OPTIMIZATION: Avoid list(set) if the set is empty.
|
||||
if not cached_encoder_input_ids:
|
||||
return
|
||||
@@ -1074,21 +1127,19 @@ class Scheduler(SchedulerInterface):
|
||||
# With Whisper, as soon as we've generated a single token,
|
||||
# we know we're done with the encoder input. Cross Attention
|
||||
# KVs have been calculated and cached already.
|
||||
self.encoder_cache_manager.free_encoder_input(
|
||||
request, input_id)
|
||||
self.encoder_cache_manager.free_encoder_input(request, input_id)
|
||||
elif start_pos + num_tokens <= request.num_computed_tokens:
|
||||
# The encoder output is already processed and stored
|
||||
# in the decoder's KV cache.
|
||||
self.encoder_cache_manager.free_encoder_input(
|
||||
request, input_id)
|
||||
self.encoder_cache_manager.free_encoder_input(request, input_id)
|
||||
|
||||
def update_draft_token_ids(
|
||||
self,
|
||||
draft_token_ids: DraftTokenIds,
|
||||
) -> None:
|
||||
for req_id, spec_token_ids in zip(
|
||||
draft_token_ids.req_ids,
|
||||
draft_token_ids.draft_token_ids,
|
||||
draft_token_ids.req_ids,
|
||||
draft_token_ids.draft_token_ids,
|
||||
):
|
||||
request = self.requests.get(req_id)
|
||||
if request is None or request.is_finished():
|
||||
@@ -1102,7 +1153,8 @@ class Scheduler(SchedulerInterface):
|
||||
elif self.structured_output_manager.should_advance(request):
|
||||
metadata = request.structured_output_request
|
||||
request.spec_token_ids = metadata.grammar.validate_tokens( # type: ignore[union-attr]
|
||||
spec_token_ids)
|
||||
spec_token_ids
|
||||
)
|
||||
else:
|
||||
request.spec_token_ids = spec_token_ids
|
||||
|
||||
@@ -1128,7 +1180,7 @@ class Scheduler(SchedulerInterface):
|
||||
"""
|
||||
assert RequestStatus.is_finished(finished_status)
|
||||
if isinstance(request_ids, str):
|
||||
request_ids = (request_ids, )
|
||||
request_ids = (request_ids,)
|
||||
else:
|
||||
request_ids = set(request_ids)
|
||||
|
||||
@@ -1198,15 +1250,15 @@ class Scheduler(SchedulerInterface):
|
||||
return None
|
||||
prefix_cache_stats = self.kv_cache_manager.make_prefix_cache_stats()
|
||||
assert prefix_cache_stats is not None
|
||||
return SchedulerStats(num_running_reqs=len(self.running),
|
||||
num_waiting_reqs=len(self.waiting),
|
||||
kv_cache_usage=self.kv_cache_manager.usage,
|
||||
prefix_cache_stats=prefix_cache_stats,
|
||||
spec_decoding_stats=spec_decoding_stats,
|
||||
num_corrupted_reqs=sum(req.is_output_corrupted
|
||||
for req in self.running),
|
||||
kv_connector_stats=kv_connector_stats.data
|
||||
if kv_connector_stats else None)
|
||||
return SchedulerStats(
|
||||
num_running_reqs=len(self.running),
|
||||
num_waiting_reqs=len(self.waiting),
|
||||
kv_cache_usage=self.kv_cache_manager.usage,
|
||||
prefix_cache_stats=prefix_cache_stats,
|
||||
spec_decoding_stats=spec_decoding_stats,
|
||||
num_corrupted_reqs=sum(req.is_output_corrupted for req in self.running),
|
||||
kv_connector_stats=kv_connector_stats.data if kv_connector_stats else None,
|
||||
)
|
||||
|
||||
def make_spec_decoding_stats(
|
||||
self,
|
||||
@@ -1219,8 +1271,8 @@ class Scheduler(SchedulerInterface):
|
||||
if spec_decoding_stats is None:
|
||||
spec_decoding_stats = SpecDecodingStats.new(self.num_spec_tokens)
|
||||
spec_decoding_stats.observe_draft(
|
||||
num_draft_tokens=num_draft_tokens,
|
||||
num_accepted_tokens=num_accepted_tokens)
|
||||
num_draft_tokens=num_draft_tokens, num_accepted_tokens=num_accepted_tokens
|
||||
)
|
||||
return spec_decoding_stats
|
||||
|
||||
def shutdown(self) -> None:
|
||||
@@ -1237,7 +1289,8 @@ class Scheduler(SchedulerInterface):
|
||||
return self.connector
|
||||
|
||||
def _connector_finished(
|
||||
self, request: Request) -> tuple[bool, Optional[dict[str, Any]]]:
|
||||
self, request: Request
|
||||
) -> tuple[bool, Optional[dict[str, Any]]]:
|
||||
"""
|
||||
Invoke the KV connector request_finished() method if applicable.
|
||||
|
||||
@@ -1247,7 +1300,7 @@ class Scheduler(SchedulerInterface):
|
||||
if self.connector is None:
|
||||
return False, None
|
||||
|
||||
(block_ids, ) = self.kv_cache_manager.get_block_ids(request.request_id)
|
||||
(block_ids,) = self.kv_cache_manager.get_block_ids(request.request_id)
|
||||
return self.connector.request_finished(request, block_ids)
|
||||
|
||||
def _update_waiting_for_remote_kv(self, request: Request) -> bool:
|
||||
@@ -1271,8 +1324,7 @@ class Scheduler(SchedulerInterface):
|
||||
# updated in _update_requests_with_invalid_blocks
|
||||
if request.num_computed_tokens:
|
||||
# Cache any valid computed tokens.
|
||||
self.kv_cache_manager.cache_blocks(request,
|
||||
request.num_computed_tokens)
|
||||
self.kv_cache_manager.cache_blocks(request, request.num_computed_tokens)
|
||||
else:
|
||||
# No valid computed tokens, release allocated blocks.
|
||||
# There may be a local cache hit on retry.
|
||||
@@ -1281,8 +1333,7 @@ class Scheduler(SchedulerInterface):
|
||||
self.failed_recving_kv_req_ids.remove(request.request_id)
|
||||
else:
|
||||
# Now that the blocks are ready, actually cache them.
|
||||
(block_ids, ) = self.kv_cache_manager.get_block_ids(
|
||||
request.request_id)
|
||||
(block_ids,) = self.kv_cache_manager.get_block_ids(request.request_id)
|
||||
num_computed_tokens = len(block_ids) * self.block_size
|
||||
# Handle the case where num request tokens less than one block.
|
||||
num_computed_tokens = min(num_computed_tokens, request.num_tokens)
|
||||
@@ -1298,8 +1349,7 @@ class Scheduler(SchedulerInterface):
|
||||
self.finished_recving_kv_req_ids.remove(request.request_id)
|
||||
return True
|
||||
|
||||
def _update_from_kv_xfer_finished(self,
|
||||
kv_connector_output: KVConnectorOutput):
|
||||
def _update_from_kv_xfer_finished(self, kv_connector_output: KVConnectorOutput):
|
||||
"""
|
||||
KV Connector: update the scheduler state based on the output.
|
||||
|
||||
@@ -1314,21 +1364,23 @@ class Scheduler(SchedulerInterface):
|
||||
self.connector.update_connector_output(kv_connector_output)
|
||||
|
||||
# KV Connector:: update recv and send status from last step.
|
||||
for req_id in (kv_connector_output.finished_recving or ()):
|
||||
for req_id in kv_connector_output.finished_recving or ():
|
||||
logger.debug("Finished recving KV transfer for request %s", req_id)
|
||||
self.finished_recving_kv_req_ids.add(req_id)
|
||||
for req_id in (kv_connector_output.finished_sending or ()):
|
||||
for req_id in kv_connector_output.finished_sending or ():
|
||||
logger.debug("Finished sending KV transfer for request %s", req_id)
|
||||
if req_id not in self.requests:
|
||||
logger.warning(
|
||||
"Got finished sending KV transfer for request %s,"
|
||||
"but the request is already freed.", req_id)
|
||||
"but the request is already freed.",
|
||||
req_id,
|
||||
)
|
||||
else:
|
||||
self._free_blocks(self.requests[req_id])
|
||||
|
||||
def _update_requests_with_invalid_blocks(
|
||||
self, requests: Iterable[Request],
|
||||
invalid_block_ids: set[int]) -> tuple[set[str], int]:
|
||||
self, requests: Iterable[Request], invalid_block_ids: set[int]
|
||||
) -> tuple[set[str], int]:
|
||||
"""
|
||||
Identify and update requests affected by invalid KV cache blocks.
|
||||
|
||||
@@ -1359,25 +1411,25 @@ class Scheduler(SchedulerInterface):
|
||||
marked_invalid_block = False
|
||||
req_id = request.request_id
|
||||
# TODO (davidb): add support for hybrid memory allocator
|
||||
(req_block_ids, ) = self.kv_cache_manager.get_block_ids(req_id)
|
||||
(req_block_ids,) = self.kv_cache_manager.get_block_ids(req_id)
|
||||
# We iterate only over blocks that may contain externally computed
|
||||
# tokens
|
||||
if request.status == RequestStatus.WAITING_FOR_REMOTE_KVS:
|
||||
# Async loading. If num_computed_tokens is set it implies we
|
||||
# already processed some block failures for it in a prior step
|
||||
req_num_computed_tokens = (
|
||||
request.num_computed_tokens if req_id
|
||||
in self.failed_recving_kv_req_ids else len(req_block_ids) *
|
||||
self.block_size)
|
||||
request.num_computed_tokens
|
||||
if req_id in self.failed_recving_kv_req_ids
|
||||
else len(req_block_ids) * self.block_size
|
||||
)
|
||||
else:
|
||||
# Sync loading. num_computed_tokens includes new tokens
|
||||
req_num_computed_tokens = request.num_cached_tokens
|
||||
|
||||
req_num_computed_blocks = (req_num_computed_tokens +
|
||||
self.block_size - 1) // self.block_size
|
||||
for idx, block_id in zip(range(req_num_computed_blocks),
|
||||
req_block_ids):
|
||||
|
||||
req_num_computed_blocks = (
|
||||
req_num_computed_tokens + self.block_size - 1
|
||||
) // self.block_size
|
||||
for idx, block_id in zip(range(req_num_computed_blocks), req_block_ids):
|
||||
if block_id not in invalid_block_ids:
|
||||
continue
|
||||
|
||||
@@ -1402,8 +1454,9 @@ class Scheduler(SchedulerInterface):
|
||||
marked_invalid_block = True
|
||||
# Truncate the computed tokens at the first failed block
|
||||
request.num_computed_tokens = idx * self.block_size
|
||||
total_affected_tokens += (req_num_computed_tokens -
|
||||
request.num_computed_tokens)
|
||||
total_affected_tokens += (
|
||||
req_num_computed_tokens - request.num_computed_tokens
|
||||
)
|
||||
|
||||
if is_affected:
|
||||
if not marked_invalid_block:
|
||||
@@ -1412,8 +1465,9 @@ class Scheduler(SchedulerInterface):
|
||||
# Revert to considering only cached tokens as computed.
|
||||
# Currently this only applies to sync loading; Async
|
||||
# loading does not yet support block sharing
|
||||
total_affected_tokens += (request.num_computed_tokens -
|
||||
request.num_cached_tokens)
|
||||
total_affected_tokens += (
|
||||
request.num_computed_tokens - request.num_cached_tokens
|
||||
)
|
||||
request.num_computed_tokens = request.num_cached_tokens
|
||||
|
||||
affected_req_ids.add(request.request_id)
|
||||
@@ -1426,11 +1480,15 @@ class Scheduler(SchedulerInterface):
|
||||
|
||||
# --- Handle async KV loads (WAITING_FOR_REMOTE_KVS) ---
|
||||
async_load_reqs = (
|
||||
req for req in self.waiting
|
||||
if req.status == RequestStatus.WAITING_FOR_REMOTE_KVS)
|
||||
req
|
||||
for req in self.waiting
|
||||
if req.status == RequestStatus.WAITING_FOR_REMOTE_KVS
|
||||
)
|
||||
async_affected_req_ids, num_tokens_to_reschedule = (
|
||||
self._update_requests_with_invalid_blocks(async_load_reqs,
|
||||
invalid_block_ids))
|
||||
self._update_requests_with_invalid_blocks(
|
||||
async_load_reqs, invalid_block_ids
|
||||
)
|
||||
)
|
||||
|
||||
total_requests_to_reschedule += len(async_affected_req_ids)
|
||||
total_tokens_to_reschedule += num_tokens_to_reschedule
|
||||
@@ -1441,8 +1499,8 @@ class Scheduler(SchedulerInterface):
|
||||
|
||||
# --- Handle sync KV loads (running requests) ---
|
||||
sync_affected_req_ids, num_tokens_to_reschedule = (
|
||||
self._update_requests_with_invalid_blocks(self.running,
|
||||
invalid_block_ids))
|
||||
self._update_requests_with_invalid_blocks(self.running, invalid_block_ids)
|
||||
)
|
||||
|
||||
total_requests_to_reschedule += len(sync_affected_req_ids)
|
||||
total_tokens_to_reschedule += num_tokens_to_reschedule
|
||||
@@ -1451,7 +1509,9 @@ class Scheduler(SchedulerInterface):
|
||||
logger.warning(
|
||||
"Recovered from KV load failure: "
|
||||
"%d request(s) rescheduled (%d tokens affected).",
|
||||
total_requests_to_reschedule, total_tokens_to_reschedule)
|
||||
total_requests_to_reschedule,
|
||||
total_tokens_to_reschedule,
|
||||
)
|
||||
|
||||
# Return the IDs of affected running requests to skip in
|
||||
# update_from_output.
|
||||
|
||||
@@ -40,11 +40,13 @@ def remove_all(lst: list, items_to_remove: set) -> list:
|
||||
return [item for item in lst if item not in items_to_remove]
|
||||
|
||||
|
||||
def check_stop(request: Request,
|
||||
max_model_len: int,
|
||||
pooler_output: Optional[torch.Tensor] = None) -> bool:
|
||||
if (request.num_tokens > max_model_len
|
||||
or request.num_output_tokens >= request.max_tokens):
|
||||
def check_stop(
|
||||
request: Request, max_model_len: int, pooler_output: Optional[torch.Tensor] = None
|
||||
) -> bool:
|
||||
if (
|
||||
request.num_tokens > max_model_len
|
||||
or request.num_output_tokens >= request.max_tokens
|
||||
):
|
||||
request.status = RequestStatus.FINISHED_LENGTH_CAPPED
|
||||
return True
|
||||
|
||||
@@ -57,8 +59,7 @@ def check_stop(request: Request,
|
||||
sampling_params = request.sampling_params
|
||||
assert sampling_params is not None
|
||||
last_token_id = request.output_token_ids[-1]
|
||||
if (not sampling_params.ignore_eos
|
||||
and last_token_id == request.eos_token_id):
|
||||
if not sampling_params.ignore_eos and last_token_id == request.eos_token_id:
|
||||
request.status = RequestStatus.FINISHED_STOPPED
|
||||
return True
|
||||
|
||||
|
||||
@@ -7,16 +7,21 @@ from collections import defaultdict
|
||||
from vllm.utils import cdiv
|
||||
from vllm.v1.core.block_pool import BlockPool
|
||||
from vllm.v1.core.kv_cache_utils import BlockHash, KVCacheBlock
|
||||
from vllm.v1.kv_cache_interface import (ChunkedLocalAttentionSpec,
|
||||
CrossAttentionSpec, FullAttentionSpec,
|
||||
KVCacheSpec, MambaSpec,
|
||||
MLAAttentionSpec, SlidingWindowSpec)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
ChunkedLocalAttentionSpec,
|
||||
CrossAttentionSpec,
|
||||
FullAttentionSpec,
|
||||
KVCacheSpec,
|
||||
MambaSpec,
|
||||
MLAAttentionSpec,
|
||||
SlidingWindowSpec,
|
||||
)
|
||||
from vllm.v1.request import Request
|
||||
|
||||
|
||||
class SingleTypeKVCacheManager(ABC):
|
||||
"""
|
||||
An abstract base class for a manager that handle the kv cache management
|
||||
An abstract base class for a manager that handle the kv cache management
|
||||
logic of one specific type of attention layer.
|
||||
"""
|
||||
|
||||
@@ -44,8 +49,7 @@ class SingleTypeKVCacheManager(ABC):
|
||||
# Mapping from request ID to blocks to track the blocks allocated
|
||||
# for each request, so that we can free the blocks when the request
|
||||
# is finished.
|
||||
self.req_to_blocks: defaultdict[str,
|
||||
list[KVCacheBlock]] = defaultdict(list)
|
||||
self.req_to_blocks: defaultdict[str, list[KVCacheBlock]] = defaultdict(list)
|
||||
|
||||
# {req_id: The number of cached blocks for this given request}
|
||||
# This is used to track the number of cached blocks for each request.
|
||||
@@ -57,14 +61,14 @@ class SingleTypeKVCacheManager(ABC):
|
||||
self._null_block = block_pool.null_block
|
||||
|
||||
def get_num_blocks_to_allocate(
|
||||
self, request_id: str, num_tokens: int,
|
||||
new_computed_blocks: list[KVCacheBlock]) -> int:
|
||||
self, request_id: str, num_tokens: int, new_computed_blocks: list[KVCacheBlock]
|
||||
) -> int:
|
||||
"""
|
||||
Get the number of blocks needed to be allocated for the request.
|
||||
|
||||
Args:
|
||||
request_id: The request ID.
|
||||
num_tokens: The total number of tokens that need a slot (including
|
||||
num_tokens: The total number of tokens that need a slot (including
|
||||
tokens that are already allocated).
|
||||
new_computed_blocks: The new computed blocks just hitting the
|
||||
prefix caching.
|
||||
@@ -74,20 +78,23 @@ class SingleTypeKVCacheManager(ABC):
|
||||
"""
|
||||
|
||||
num_required_blocks = cdiv(num_tokens, self.block_size)
|
||||
num_new_blocks = (num_required_blocks - len(new_computed_blocks) -
|
||||
len(self.req_to_blocks[request_id]))
|
||||
num_new_blocks = (
|
||||
num_required_blocks
|
||||
- len(new_computed_blocks)
|
||||
- len(self.req_to_blocks[request_id])
|
||||
)
|
||||
# If a computed block of a request is an eviction candidate (in the
|
||||
# free queue and ref_cnt == 0), it will be changed from a free block
|
||||
# to a computed block when the request is allocated, so we also count
|
||||
# it as needed to be allocated.
|
||||
num_evictable_computed_blocks = sum(
|
||||
blk.ref_cnt == 0 and not blk.is_null
|
||||
for blk in new_computed_blocks)
|
||||
blk.ref_cnt == 0 and not blk.is_null for blk in new_computed_blocks
|
||||
)
|
||||
return num_new_blocks + num_evictable_computed_blocks
|
||||
|
||||
def save_new_computed_blocks(
|
||||
self, request_id: str,
|
||||
new_computed_blocks: list[KVCacheBlock]) -> None:
|
||||
self, request_id: str, new_computed_blocks: list[KVCacheBlock]
|
||||
) -> None:
|
||||
"""
|
||||
Add the new computed blocks to the request.
|
||||
|
||||
@@ -106,15 +113,16 @@ class SingleTypeKVCacheManager(ABC):
|
||||
# A running request. Should not have new computed blocks.
|
||||
assert len(new_computed_blocks) == 0
|
||||
|
||||
def allocate_new_blocks(self, request_id: str,
|
||||
num_tokens: int) -> list[KVCacheBlock]:
|
||||
def allocate_new_blocks(
|
||||
self, request_id: str, num_tokens: int
|
||||
) -> list[KVCacheBlock]:
|
||||
"""
|
||||
Allocate new blocks for the request to give it at least `num_tokens`
|
||||
Allocate new blocks for the request to give it at least `num_tokens`
|
||||
token slots.
|
||||
|
||||
Args:
|
||||
request_id: The request ID.
|
||||
num_tokens: The total number of tokens that need a slot (including
|
||||
num_tokens: The total number of tokens that need a slot (including
|
||||
tokens that are already allocated).
|
||||
|
||||
Returns:
|
||||
@@ -136,7 +144,7 @@ class SingleTypeKVCacheManager(ABC):
|
||||
|
||||
Args:
|
||||
request: The request.
|
||||
num_tokens: The total number of tokens that need to be cached
|
||||
num_tokens: The total number of tokens that need to be cached
|
||||
(including tokens that are already cached).
|
||||
"""
|
||||
num_cached_blocks = self.num_cached_block[request.request_id]
|
||||
@@ -174,8 +182,9 @@ class SingleTypeKVCacheManager(ABC):
|
||||
self.num_cached_block.pop(request_id, None)
|
||||
|
||||
@abstractmethod
|
||||
def get_num_common_prefix_blocks(self, request_id: str,
|
||||
num_running_requests: int) -> int:
|
||||
def get_num_common_prefix_blocks(
|
||||
self, request_id: str, num_running_requests: int
|
||||
) -> int:
|
||||
"""
|
||||
Get the number of common prefix blocks for all requests in the RUNNING
|
||||
state.
|
||||
@@ -205,12 +214,12 @@ class SingleTypeKVCacheManager(ABC):
|
||||
dcp_world_size: int = 1,
|
||||
) -> tuple[list[KVCacheBlock], ...]:
|
||||
"""
|
||||
Get the longest cache hit prefix of the blocks that is not longer than
|
||||
`max_length`. The prefix should be a common prefix hit for all the
|
||||
kv cache groups in `kv_cache_group_ids`. If no cache hit is found,
|
||||
return an empty list.
|
||||
If eagle is enabled, drop the last matched block to force recompute the
|
||||
last block to get the required hidden states for eagle drafting head.
|
||||
Get the longest cache hit prefix of the blocks that is not longer than
|
||||
`max_length`. The prefix should be a common prefix hit for all the
|
||||
kv cache groups in `kv_cache_group_ids`. If no cache hit is found,
|
||||
return an empty list.
|
||||
If eagle is enabled, drop the last matched block to force recompute the
|
||||
last block to get the required hidden states for eagle drafting head.
|
||||
Need to be customized for each attention type.
|
||||
|
||||
Args:
|
||||
@@ -235,10 +244,9 @@ class SingleTypeKVCacheManager(ABC):
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def remove_skipped_blocks(self, request_id: str,
|
||||
num_computed_tokens: int) -> None:
|
||||
def remove_skipped_blocks(self, request_id: str, num_computed_tokens: int) -> None:
|
||||
"""
|
||||
Remove the blocks that are no longer needed from `blocks` and free the
|
||||
Remove the blocks that are no longer needed from `blocks` and free the
|
||||
blocks. The removed blocks should be replaced by null_block.
|
||||
Need to be customized for each attention type.
|
||||
|
||||
@@ -250,7 +258,6 @@ class SingleTypeKVCacheManager(ABC):
|
||||
|
||||
|
||||
class FullAttentionManager(SingleTypeKVCacheManager):
|
||||
|
||||
@classmethod
|
||||
def find_longest_cache_hit(
|
||||
cls,
|
||||
@@ -264,10 +271,13 @@ class FullAttentionManager(SingleTypeKVCacheManager):
|
||||
) -> tuple[list[KVCacheBlock], ...]:
|
||||
assert isinstance(
|
||||
kv_cache_spec, (FullAttentionSpec, ChunkedLocalAttentionSpec)
|
||||
), "FullAttentionManager can only be used for full attention " \
|
||||
), (
|
||||
"FullAttentionManager can only be used for full attention "
|
||||
"and chunked local attention groups"
|
||||
)
|
||||
computed_blocks: tuple[list[KVCacheBlock], ...] = tuple(
|
||||
[] for _ in range(len(kv_cache_group_ids)))
|
||||
[] for _ in range(len(kv_cache_group_ids))
|
||||
)
|
||||
block_size = kv_cache_spec.block_size
|
||||
if dcp_world_size > 1:
|
||||
block_size *= dcp_world_size
|
||||
@@ -277,7 +287,8 @@ class FullAttentionManager(SingleTypeKVCacheManager):
|
||||
# in the cached_block_hash_to_id, the following block hashes are
|
||||
# not computed yet for sure.
|
||||
if cached_block := block_pool.get_cached_block(
|
||||
block_hash, kv_cache_group_ids):
|
||||
block_hash, kv_cache_group_ids
|
||||
):
|
||||
for computed, cached in zip(computed_blocks, cached_block):
|
||||
computed.append(cached)
|
||||
else:
|
||||
@@ -287,13 +298,13 @@ class FullAttentionManager(SingleTypeKVCacheManager):
|
||||
computed.pop()
|
||||
return computed_blocks
|
||||
|
||||
def remove_skipped_blocks(self, request_id: str,
|
||||
num_computed_tokens: int) -> None:
|
||||
def remove_skipped_blocks(self, request_id: str, num_computed_tokens: int) -> None:
|
||||
# No need to remove blocks for full attention.
|
||||
pass
|
||||
|
||||
def get_num_common_prefix_blocks(self, request_id: str,
|
||||
num_running_requests: int) -> int:
|
||||
def get_num_common_prefix_blocks(
|
||||
self, request_id: str, num_running_requests: int
|
||||
) -> int:
|
||||
blocks = self.req_to_blocks[request_id]
|
||||
num_common_blocks = 0
|
||||
for block in blocks:
|
||||
@@ -305,9 +316,9 @@ class FullAttentionManager(SingleTypeKVCacheManager):
|
||||
|
||||
|
||||
class SlidingWindowManager(SingleTypeKVCacheManager):
|
||||
|
||||
def __init__(self, kv_cache_spec: SlidingWindowSpec, block_pool: BlockPool,
|
||||
**kwargs) -> None:
|
||||
def __init__(
|
||||
self, kv_cache_spec: SlidingWindowSpec, block_pool: BlockPool, **kwargs
|
||||
) -> None:
|
||||
super().__init__(kv_cache_spec, block_pool, **kwargs)
|
||||
self.sliding_window = kv_cache_spec.sliding_window
|
||||
self._null_block = block_pool.null_block
|
||||
@@ -324,13 +335,15 @@ class SlidingWindowManager(SingleTypeKVCacheManager):
|
||||
dcp_world_size: int = 1,
|
||||
) -> tuple[list[KVCacheBlock], ...]:
|
||||
assert isinstance(kv_cache_spec, SlidingWindowSpec), (
|
||||
"SlidingWindowManager can only be used for sliding window groups")
|
||||
"SlidingWindowManager can only be used for sliding window groups"
|
||||
)
|
||||
assert dcp_world_size == 1, "DCP not support sliding window attn now."
|
||||
|
||||
# The number of contiguous blocks needed for prefix cache hit.
|
||||
# -1 since the input token itself is also included in the window
|
||||
sliding_window_contiguous_blocks = cdiv(
|
||||
kv_cache_spec.sliding_window - 1, kv_cache_spec.block_size)
|
||||
kv_cache_spec.sliding_window - 1, kv_cache_spec.block_size
|
||||
)
|
||||
if use_eagle:
|
||||
# Need to drop the last matched block if eagle is enabled. For
|
||||
# sliding window layer, we achieve this by increasing the number of
|
||||
@@ -344,14 +357,17 @@ class SlidingWindowManager(SingleTypeKVCacheManager):
|
||||
# sliding_window_contiguous_blocks),
|
||||
# which is good for low cache hit rate scenarios.
|
||||
max_num_blocks = max_length // kv_cache_spec.block_size
|
||||
computed_blocks = tuple([block_pool.null_block] * max_num_blocks
|
||||
for _ in range(len(kv_cache_group_ids)))
|
||||
computed_blocks = tuple(
|
||||
[block_pool.null_block] * max_num_blocks
|
||||
for _ in range(len(kv_cache_group_ids))
|
||||
)
|
||||
num_contiguous_blocks = 0
|
||||
match_found = False
|
||||
# Search from right to left and early stop when a match is found.
|
||||
for i in range(max_num_blocks - 1, -1, -1):
|
||||
if cached_block := block_pool.get_cached_block(
|
||||
block_hashes[i], kv_cache_group_ids):
|
||||
block_hashes[i], kv_cache_group_ids
|
||||
):
|
||||
for computed, cached in zip(computed_blocks, cached_block):
|
||||
computed[i] = cached
|
||||
num_contiguous_blocks += 1
|
||||
@@ -360,7 +376,7 @@ class SlidingWindowManager(SingleTypeKVCacheManager):
|
||||
# E.g., [NULL, NULL, 8, 3, NULL, 9] -> [NULL, NULL, 8, 3]
|
||||
# when sliding_window_contiguous_blocks=2.
|
||||
for computed in computed_blocks:
|
||||
del computed[i + num_contiguous_blocks:]
|
||||
del computed[i + num_contiguous_blocks :]
|
||||
match_found = True
|
||||
break
|
||||
else:
|
||||
@@ -375,8 +391,7 @@ class SlidingWindowManager(SingleTypeKVCacheManager):
|
||||
computed.pop()
|
||||
return computed_blocks
|
||||
|
||||
def remove_skipped_blocks(self, request_id: str,
|
||||
num_computed_tokens: int) -> None:
|
||||
def remove_skipped_blocks(self, request_id: str, num_computed_tokens: int) -> None:
|
||||
# Remove the blocks that are no longer be in the sliding window and
|
||||
# skipped during the attention computation.
|
||||
last_useful_token = num_computed_tokens - self.sliding_window + 1
|
||||
@@ -393,21 +408,22 @@ class SlidingWindowManager(SingleTypeKVCacheManager):
|
||||
blocks[i] = self._null_block
|
||||
self.block_pool.free_blocks(removed_blocks)
|
||||
|
||||
def get_num_common_prefix_blocks(self, request_id: str,
|
||||
num_running_requests: int) -> int:
|
||||
def get_num_common_prefix_blocks(
|
||||
self, request_id: str, num_running_requests: int
|
||||
) -> int:
|
||||
"""
|
||||
NOTE(Chen): The prefix blocks are null blocks for sliding window layers.
|
||||
So it's not correct to count ref_cnt like FullAttentionManager. Return
|
||||
0 here for correctness. Need to support cascade attention + sliding
|
||||
So it's not correct to count ref_cnt like FullAttentionManager. Return
|
||||
0 here for correctness. Need to support cascade attention + sliding
|
||||
window in the future.
|
||||
"""
|
||||
return 0
|
||||
|
||||
|
||||
class ChunkedLocalAttentionManager(SingleTypeKVCacheManager):
|
||||
|
||||
def __init__(self, kv_cache_spec: ChunkedLocalAttentionSpec,
|
||||
block_pool: BlockPool, **kwargs) -> None:
|
||||
def __init__(
|
||||
self, kv_cache_spec: ChunkedLocalAttentionSpec, block_pool: BlockPool, **kwargs
|
||||
) -> None:
|
||||
super().__init__(kv_cache_spec, block_pool, **kwargs)
|
||||
self.attention_chunk_size = kv_cache_spec.attention_chunk_size
|
||||
self._null_block = block_pool.null_block
|
||||
@@ -428,19 +444,19 @@ class ChunkedLocalAttentionManager(SingleTypeKVCacheManager):
|
||||
prefix of the blocks that is not longer than `max_length`. The prefix
|
||||
should be a common prefix hit for all the kv cache groups in
|
||||
`kv_cache_group_ids`. If no cache hit is found, return an empty list.
|
||||
note we mark as computed if the whole block is outside of the local
|
||||
note we mark as computed if the whole block is outside of the local
|
||||
window, and set the block as null. Examples:
|
||||
|
||||
1. Attention chunk size of 8, block size of 4, max length of 15
|
||||
for next token at 15th (zero-indexed), 8th - 14th tokens are in
|
||||
the window(needs lookup), 0th - 7th are not in the window,
|
||||
so they are already marked as computed. We check the complete
|
||||
block3 (8th - 11th tokens), Assume block 3 is hit, we will return
|
||||
for next token at 15th (zero-indexed), 8th - 14th tokens are in
|
||||
the window(needs lookup), 0th - 7th are not in the window,
|
||||
so they are already marked as computed. We check the complete
|
||||
block3 (8th - 11th tokens), Assume block 3 is hit, we will return
|
||||
[null, null, block 3], otherwise, we return [null, null]
|
||||
|
||||
2. Attention chunk size of 8, block size of 4, max length of 16
|
||||
for next token at 16th (zero-indexed), 0th - 15th tokens are not
|
||||
in the window, so they are already marked as computed.
|
||||
for next token at 16th (zero-indexed), 0th - 15th tokens are not
|
||||
in the window, so they are already marked as computed.
|
||||
we return 4 blocks[null, null, null, null]
|
||||
|
||||
Args:
|
||||
@@ -455,39 +471,45 @@ class ChunkedLocalAttentionManager(SingleTypeKVCacheManager):
|
||||
A list of cached blocks
|
||||
"""
|
||||
assert isinstance(kv_cache_spec, ChunkedLocalAttentionSpec), (
|
||||
"ChunkedLocalAttentionManager can only be used for " +
|
||||
"chunked local attention groups")
|
||||
assert use_eagle is False, ("Hybrid KV cache is not supported for " +
|
||||
"eagle + chunked local attention.")
|
||||
"ChunkedLocalAttentionManager can only be used for "
|
||||
+ "chunked local attention groups"
|
||||
)
|
||||
assert use_eagle is False, (
|
||||
"Hybrid KV cache is not supported for " + "eagle + chunked local attention."
|
||||
)
|
||||
assert dcp_world_size == 1, "DCP not support chunked local attn now."
|
||||
max_num_blocks = max_length // kv_cache_spec.block_size
|
||||
if max_length > 0:
|
||||
local_attention_start_idx = (max_length //
|
||||
kv_cache_spec.attention_chunk_size *
|
||||
kv_cache_spec.attention_chunk_size)
|
||||
local_attention_start_idx = (
|
||||
max_length
|
||||
// kv_cache_spec.attention_chunk_size
|
||||
* kv_cache_spec.attention_chunk_size
|
||||
)
|
||||
else:
|
||||
local_attention_start_idx = 0
|
||||
# we marked blocks out of window as computed
|
||||
# with null blocks, and blocks inside window based on cache lookup
|
||||
# result [null] [null] ... [null] [hit block 1 (1st block contain
|
||||
# last window)] [hit block 2] ... [hit block x]
|
||||
local_attention_start_block_idx = (local_attention_start_idx //
|
||||
kv_cache_spec.block_size)
|
||||
local_attention_start_block_idx = (
|
||||
local_attention_start_idx // kv_cache_spec.block_size
|
||||
)
|
||||
computed_blocks: tuple[list[KVCacheBlock], ...] = tuple(
|
||||
[block_pool.null_block] * local_attention_start_block_idx
|
||||
for _ in range(len(kv_cache_group_ids)))
|
||||
for _ in range(len(kv_cache_group_ids))
|
||||
)
|
||||
for i in range(local_attention_start_block_idx, max_num_blocks):
|
||||
block_hash = block_hashes[i]
|
||||
if cached_block := block_pool.get_cached_block(
|
||||
block_hash, kv_cache_group_ids):
|
||||
block_hash, kv_cache_group_ids
|
||||
):
|
||||
for computed, cached in zip(computed_blocks, cached_block):
|
||||
computed.append(cached)
|
||||
else:
|
||||
break
|
||||
return computed_blocks
|
||||
|
||||
def remove_skipped_blocks(self, request_id: str,
|
||||
num_computed_tokens: int) -> None:
|
||||
def remove_skipped_blocks(self, request_id: str, num_computed_tokens: int) -> None:
|
||||
# Remove the blocks that are no longer be in the chunked attention
|
||||
# window and skipped during the attention computation.
|
||||
|
||||
@@ -499,13 +521,14 @@ class ChunkedLocalAttentionManager(SingleTypeKVCacheManager):
|
||||
# is 1024. for 1023, it will be 0.
|
||||
num_cached_block = self.num_cached_block.get(request_id, 0)
|
||||
local_attention_start_idx = (
|
||||
num_computed_tokens
|
||||
) // self.attention_chunk_size * self.attention_chunk_size
|
||||
(num_computed_tokens)
|
||||
// self.attention_chunk_size
|
||||
* self.attention_chunk_size
|
||||
)
|
||||
first_useful_block_idx = local_attention_start_idx // self.block_size
|
||||
if num_cached_block > 0:
|
||||
# Make sure we don't delete the last cached block
|
||||
first_useful_block_idx = min(first_useful_block_idx,
|
||||
num_cached_block - 1)
|
||||
first_useful_block_idx = min(first_useful_block_idx, num_cached_block - 1)
|
||||
# if block size = 128, 0 -> block 0, 1024 (= 128 * 8) ->
|
||||
# block 8, 372 (= 128 * 2 + 116) -> block 2
|
||||
blocks = self.req_to_blocks[request_id]
|
||||
@@ -521,8 +544,9 @@ class ChunkedLocalAttentionManager(SingleTypeKVCacheManager):
|
||||
blocks[i] = self._null_block
|
||||
self.block_pool.free_blocks(removed_blocks)
|
||||
|
||||
def get_num_common_prefix_blocks(self, request_id: str,
|
||||
num_running_requests: int) -> int:
|
||||
def get_num_common_prefix_blocks(
|
||||
self, request_id: str, num_running_requests: int
|
||||
) -> int:
|
||||
"""
|
||||
cascade attention is not supported by chunked local attention.
|
||||
"""
|
||||
@@ -530,7 +554,6 @@ class ChunkedLocalAttentionManager(SingleTypeKVCacheManager):
|
||||
|
||||
|
||||
class MambaManager(SingleTypeKVCacheManager):
|
||||
|
||||
@classmethod
|
||||
def find_longest_cache_hit(
|
||||
cls,
|
||||
@@ -542,18 +565,20 @@ class MambaManager(SingleTypeKVCacheManager):
|
||||
use_eagle: bool,
|
||||
dcp_world_size: int = 1,
|
||||
) -> tuple[list[KVCacheBlock], ...]:
|
||||
assert isinstance(
|
||||
kv_cache_spec,
|
||||
MambaSpec), ("MambaManager can only be used for mamba groups")
|
||||
assert isinstance(kv_cache_spec, MambaSpec), (
|
||||
"MambaManager can only be used for mamba groups"
|
||||
)
|
||||
assert dcp_world_size == 1, "DCP not support mamba now."
|
||||
computed_blocks: tuple[list[KVCacheBlock], ...] = tuple(
|
||||
[] for _ in range(len(kv_cache_group_ids)))
|
||||
[] for _ in range(len(kv_cache_group_ids))
|
||||
)
|
||||
|
||||
max_num_blocks = max_length // kv_cache_spec.block_size
|
||||
# Search from right to left and early stop when a match is found.
|
||||
for i in range(max_num_blocks - 1, -1, -1):
|
||||
if cached_block := block_pool.get_cached_block(
|
||||
block_hashes[i], kv_cache_group_ids):
|
||||
block_hashes[i], kv_cache_group_ids
|
||||
):
|
||||
for computed, cached in zip(computed_blocks, cached_block):
|
||||
# the hit length logic later assumes:
|
||||
# hit_length = len(hit_blocks_other_attn[0])
|
||||
@@ -566,40 +591,46 @@ class MambaManager(SingleTypeKVCacheManager):
|
||||
|
||||
return computed_blocks
|
||||
|
||||
def remove_skipped_blocks(self, request_id: str,
|
||||
num_computed_tokens: int) -> None:
|
||||
def remove_skipped_blocks(self, request_id: str, num_computed_tokens: int) -> None:
|
||||
# Here unused blocks may be freed up for running requests.
|
||||
# TODO(@s3woz) Free up all blocks that aren't needed by Mamba2
|
||||
# (for which find_longest_cache_hit returns block_pool.null_block)
|
||||
pass
|
||||
|
||||
def get_num_common_prefix_blocks(self, request_id: str,
|
||||
num_running_requests: int) -> int:
|
||||
def get_num_common_prefix_blocks(
|
||||
self, request_id: str, num_running_requests: int
|
||||
) -> int:
|
||||
"""
|
||||
cascade attention is not supported by mamba
|
||||
"""
|
||||
return 0
|
||||
|
||||
def get_num_blocks_to_allocate(
|
||||
self, request_id: str, num_tokens: int,
|
||||
new_computed_blocks: list[KVCacheBlock]) -> int:
|
||||
self, request_id: str, num_tokens: int, new_computed_blocks: list[KVCacheBlock]
|
||||
) -> int:
|
||||
# Allocate extra `num_speculative_blocks` blocks for
|
||||
# speculative decoding (MTP/EAGLE) with linear attention.
|
||||
assert isinstance(self.kv_cache_spec, MambaSpec)
|
||||
if self.kv_cache_spec.num_speculative_blocks > 0:
|
||||
num_tokens += (self.kv_cache_spec.block_size *
|
||||
self.kv_cache_spec.num_speculative_blocks)
|
||||
return super().get_num_blocks_to_allocate(request_id, num_tokens,
|
||||
new_computed_blocks)
|
||||
num_tokens += (
|
||||
self.kv_cache_spec.block_size
|
||||
* self.kv_cache_spec.num_speculative_blocks
|
||||
)
|
||||
return super().get_num_blocks_to_allocate(
|
||||
request_id, num_tokens, new_computed_blocks
|
||||
)
|
||||
|
||||
def allocate_new_blocks(self, request_id: str,
|
||||
num_tokens: int) -> list[KVCacheBlock]:
|
||||
def allocate_new_blocks(
|
||||
self, request_id: str, num_tokens: int
|
||||
) -> list[KVCacheBlock]:
|
||||
# Allocate extra `num_speculative_blocks` blocks for
|
||||
# speculative decoding (MTP/EAGLE) with linear attention.
|
||||
assert isinstance(self.kv_cache_spec, MambaSpec)
|
||||
if self.kv_cache_spec.num_speculative_blocks > 0:
|
||||
num_tokens += (self.kv_cache_spec.block_size *
|
||||
self.kv_cache_spec.num_speculative_blocks)
|
||||
num_tokens += (
|
||||
self.kv_cache_spec.block_size
|
||||
* self.kv_cache_spec.num_speculative_blocks
|
||||
)
|
||||
return super().allocate_new_blocks(request_id, num_tokens)
|
||||
|
||||
|
||||
@@ -607,8 +638,8 @@ class CrossAttentionManager(SingleTypeKVCacheManager):
|
||||
"""Manager for cross-attention KV cache in encoder-decoder models."""
|
||||
|
||||
def save_new_computed_blocks(
|
||||
self, request_id: str,
|
||||
new_computed_blocks: list[KVCacheBlock]) -> None:
|
||||
self, request_id: str, new_computed_blocks: list[KVCacheBlock]
|
||||
) -> None:
|
||||
# We do not cache blocks for cross-attention to be shared between
|
||||
# requests, so `new_computed_blocks` should always be empty.
|
||||
assert len(new_computed_blocks) == 0
|
||||
@@ -618,8 +649,9 @@ class CrossAttentionManager(SingleTypeKVCacheManager):
|
||||
# requests, so this method is not relevant.
|
||||
raise ValueError("Should not be called as prefix caching is disabled.")
|
||||
|
||||
def get_num_common_prefix_blocks(self, request_id: str,
|
||||
num_running_requests: int) -> int:
|
||||
def get_num_common_prefix_blocks(
|
||||
self, request_id: str, num_running_requests: int
|
||||
) -> int:
|
||||
# Cross-attention blocks contain request-specific encoder states
|
||||
# and are not shared between different requests
|
||||
return 0
|
||||
@@ -644,11 +676,9 @@ class CrossAttentionManager(SingleTypeKVCacheManager):
|
||||
# 2. Encoder states are computed once per request, not incrementally
|
||||
# 3. No reusable prefix exists between different multimodal inputs
|
||||
# Return empty blocks to indicate no cache hits
|
||||
raise NotImplementedError(
|
||||
"CrossAttentionManager does not support caching")
|
||||
raise NotImplementedError("CrossAttentionManager does not support caching")
|
||||
|
||||
def remove_skipped_blocks(self, request_id: str,
|
||||
num_computed_tokens: int) -> None:
|
||||
def remove_skipped_blocks(self, request_id: str, num_computed_tokens: int) -> None:
|
||||
# Cross-attention blocks represent encoder states which are needed
|
||||
# for the entire decoding process, so no blocks should be skipped
|
||||
pass
|
||||
@@ -664,8 +694,9 @@ spec_manager_map: dict[type[KVCacheSpec], type[SingleTypeKVCacheManager]] = {
|
||||
}
|
||||
|
||||
|
||||
def get_manager_for_kv_cache_spec(kv_cache_spec: KVCacheSpec,
|
||||
**kwargs) -> SingleTypeKVCacheManager:
|
||||
def get_manager_for_kv_cache_spec(
|
||||
kv_cache_spec: KVCacheSpec, **kwargs
|
||||
) -> SingleTypeKVCacheManager:
|
||||
manager_class = spec_manager_map[type(kv_cache_spec)]
|
||||
manager = manager_class(kv_cache_spec, **kwargs)
|
||||
return manager
|
||||
|
||||
Reference in New Issue
Block a user