[v1][KVCacheManager] pass num_new_computed_tokens to kv cache manager (#18001)
Signed-off-by: Chen Zhang <zhangch99@outlook.com>
This commit is contained in:
@@ -121,13 +121,6 @@ class KVCacheManager:
|
||||
- A list of blocks that are computed for the request.
|
||||
- The number of computed tokens.
|
||||
"""
|
||||
|
||||
# Request already has blocks from async load via KVConnector.
|
||||
num_existing_blocks = len(
|
||||
self.single_type_manager.req_to_blocks[request.request_id])
|
||||
if num_existing_blocks > 0:
|
||||
return KVCacheBlocks.create_empty(), request.num_computed_tokens
|
||||
|
||||
# Prefix caching is disabled or
|
||||
# When the request requires prompt logprobs, we skip prefix caching.
|
||||
if (not self.enable_caching
|
||||
@@ -172,6 +165,7 @@ class KVCacheManager:
|
||||
self,
|
||||
request: Request,
|
||||
num_new_tokens: int,
|
||||
num_new_computed_tokens: int = 0,
|
||||
new_computed_blocks: Optional[KVCacheBlocks] = None,
|
||||
num_lookahead_tokens: int = 0,
|
||||
delay_cache_blocks: bool = False,
|
||||
@@ -183,8 +177,10 @@ class KVCacheManager:
|
||||
num_new_tokens: The number of tokens to allocate, including external
|
||||
tokens. Note that this does not include tokens that have
|
||||
already been computed locally (i.e. new_computed_blocks).
|
||||
new_computed_blocks: The new computed blocks just hitting the
|
||||
prefix caching.
|
||||
num_new_computed_tokens: The number of new computed tokens just
|
||||
hitting the prefix caching, excluding external tokens.
|
||||
new_computed_blocks: The cached blocks for the above new computed
|
||||
tokens.
|
||||
num_lookahead_tokens: The number of speculative tokens to allocate.
|
||||
This is used by spec decode proposers with kv-cache such
|
||||
as eagle.
|
||||
@@ -229,7 +225,7 @@ class KVCacheManager:
|
||||
# The number of computed tokens is the number of computed tokens plus
|
||||
# the new prefix caching hits
|
||||
num_computed_tokens = (request.num_computed_tokens +
|
||||
len(new_computed_block_list) * self.block_size)
|
||||
num_new_computed_tokens)
|
||||
num_tokens_need_slot = min(
|
||||
num_computed_tokens + num_new_tokens + num_lookahead_tokens,
|
||||
self.max_model_len)
|
||||
|
||||
@@ -18,7 +18,7 @@ from vllm.logger import init_logger
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry
|
||||
from vllm.v1.core.encoder_cache_manager import (EncoderCacheManager,
|
||||
compute_encoder_budget)
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheManager
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks, KVCacheManager
|
||||
from vllm.v1.core.sched.interface import SchedulerInterface
|
||||
from vllm.v1.core.sched.output import (CachedRequestData, NewRequestData,
|
||||
SchedulerOutput)
|
||||
@@ -311,12 +311,14 @@ class Scheduler(SchedulerInterface):
|
||||
break
|
||||
|
||||
request = self.waiting[0]
|
||||
|
||||
num_prealloc_computed_tokens = 0
|
||||
# P/D: skip request if still waiting for remote kvs.
|
||||
if request.status == RequestStatus.WAITING_FOR_REMOTE_KVS:
|
||||
is_ready = self._update_waiting_for_remote_kv(request)
|
||||
if is_ready:
|
||||
request.status = RequestStatus.WAITING
|
||||
num_prealloc_computed_tokens = (
|
||||
request.num_computed_tokens)
|
||||
else:
|
||||
self.waiting.popleft()
|
||||
skipped_waiting_requests.appendleft(request)
|
||||
@@ -345,18 +347,25 @@ class Scheduler(SchedulerInterface):
|
||||
continue
|
||||
|
||||
# Get already-cached tokens.
|
||||
new_computed_blocks, num_computed_tokens = \
|
||||
self.kv_cache_manager.get_computed_blocks(
|
||||
request)
|
||||
if num_prealloc_computed_tokens == 0:
|
||||
new_computed_blocks, num_native_computed_tokens = \
|
||||
self.kv_cache_manager.get_computed_blocks(
|
||||
request)
|
||||
else:
|
||||
# P/D: skip checking prefix cache if loaded from remote kvs.
|
||||
new_computed_blocks = KVCacheBlocks.create_empty()
|
||||
num_native_computed_tokens = 0
|
||||
|
||||
# Get externally-cached tokens if using a KVConnector.
|
||||
num_external_tokens, load_kv_async = (
|
||||
num_external_computed_tokens, load_kv_async = (
|
||||
(0, False) if self.connector is None else
|
||||
self.connector.get_num_new_matched_tokens(
|
||||
request, num_computed_tokens))
|
||||
request, num_native_computed_tokens))
|
||||
|
||||
# Total computed tokens (local + external).
|
||||
num_computed_tokens += num_external_tokens
|
||||
num_computed_tokens = (num_native_computed_tokens +
|
||||
num_external_computed_tokens +
|
||||
num_prealloc_computed_tokens)
|
||||
|
||||
encoder_inputs_to_schedule = None
|
||||
new_encoder_budget = encoder_budget
|
||||
@@ -390,7 +399,8 @@ class Scheduler(SchedulerInterface):
|
||||
|
||||
new_blocks = self.kv_cache_manager.allocate_slots(
|
||||
request,
|
||||
num_new_tokens + num_external_tokens,
|
||||
num_new_tokens + num_external_computed_tokens,
|
||||
num_native_computed_tokens,
|
||||
new_computed_blocks,
|
||||
num_lookahead_tokens=self.num_lookahead_tokens,
|
||||
delay_cache_blocks=load_kv_async,
|
||||
@@ -406,7 +416,7 @@ class Scheduler(SchedulerInterface):
|
||||
self.connector.update_state_after_alloc(
|
||||
request,
|
||||
new_computed_blocks + new_blocks,
|
||||
num_external_tokens,
|
||||
num_external_computed_tokens,
|
||||
)
|
||||
|
||||
self.waiting.popleft()
|
||||
|
||||
Reference in New Issue
Block a user