[V1][Prefix Cache] Move the logic of num_computed_tokens into KVCacheManager (#12003)

This commit is contained in:
Chen Zhang
2025-01-15 07:55:30 +00:00
committed by GitHub
parent 3f9b7ab9f5
commit 994fc655b7
3 changed files with 61 additions and 35 deletions
+12 -5
View File
@@ -1,5 +1,5 @@
from collections import defaultdict
from typing import Dict, Iterable, List, Optional
from typing import Dict, Iterable, List, Optional, Tuple
from vllm.logger import init_logger
from vllm.utils import cdiv
@@ -69,7 +69,8 @@ class KVCacheManager:
# is finished.
self.req_to_blocks: Dict[str, List[KVCacheBlock]] = {}
def get_computed_blocks(self, request: Request) -> List[KVCacheBlock]:
def get_computed_blocks(
self, request: Request) -> Tuple[List[KVCacheBlock], int]:
"""Get the computed (cached) blocks for the request.
Note that the computed blocks must be full.
@@ -77,11 +78,13 @@ class KVCacheManager:
request: The request to get the computed blocks.
Returns:
A list of blocks that are computed for the request.
A tuple containing:
- A list of blocks that are computed for the request.
- The number of computed tokens.
"""
if not self.enable_caching:
# Prefix caching is disabled.
return []
return [], 0
computed_blocks = []
@@ -101,7 +104,11 @@ class KVCacheManager:
else:
break
return computed_blocks
# NOTE(woosuk): Since incomplete blocks are not eligible for
# sharing, `num_computed_tokens` is always a multiple of
# `block_size`.
num_computed_tokens = len(computed_blocks) * self.block_size
return computed_blocks, num_computed_tokens
def append_slots(
self,
+2 -6
View File
@@ -184,12 +184,8 @@ class Scheduler:
request = self.waiting[0]
# Get already-cached tokens.
computed_blocks = self.kv_cache_manager.get_computed_blocks(
request)
# NOTE(woosuk): Since incomplete blocks are not eligible for
# sharing, `num_computed_tokens` is always a multiple of
# `block_size`.
num_computed_tokens = len(computed_blocks) * self.block_size
computed_blocks, num_computed_tokens = \
self.kv_cache_manager.get_computed_blocks(request)
# Number of tokens to be scheduled.
# We use `request.num_tokens` instead of
# `request.num_prompt_tokens` to consider the resumed requests,