[V1][Spec Decode] Ngram Spec Decode (#12193)
Signed-off-by: LiuXiaoxuanPKU <lilyliupku@gmail.com>
This commit is contained in:
@@ -82,6 +82,11 @@ class KVCacheManager:
|
||||
self.req_to_block_hashes: DefaultDict[
|
||||
str, List[BlockHashType]] = defaultdict(list)
|
||||
|
||||
# {req_id: The number of cached blocks for this given request}
|
||||
# This is used to track the number of cached blocks for each request.
|
||||
# This is only used to track the RUNNING requests, we do not track the
|
||||
# data for reempted ones.
|
||||
self.num_cached_block: Dict[str, int] = defaultdict(int)
|
||||
self.prefix_cache_stats = PrefixCacheStats()
|
||||
|
||||
@property
|
||||
@@ -241,23 +246,25 @@ class KVCacheManager:
|
||||
if not self.enable_caching:
|
||||
return new_blocks
|
||||
|
||||
# NOTE(rickyx): We are assuming the `num_tokens` are actual
|
||||
# tokens rather than lookahead slots (e.g. for speculative decoding).
|
||||
# TODO(rickyx): When supporting speculative decoding, we will need to
|
||||
# differentiate between them so that we can know how many blocks are
|
||||
# full after appending the actual tokens.
|
||||
num_full_blocks = (num_computed_tokens + num_tokens) // self.block_size
|
||||
num_computed_full_blocks = num_computed_tokens // self.block_size
|
||||
new_full_blocks = req_blocks[num_computed_full_blocks:num_full_blocks]
|
||||
num_cached_blocks = self.num_cached_block[request.request_id]
|
||||
# Speculated tokens might be rejected in the future, so we does
|
||||
# not cache any speculated tokens. We only cache blocks with
|
||||
# generated (accepted) tokens.
|
||||
num_full_blocks_after_append = (num_computed_tokens + num_tokens - len(
|
||||
request.spec_token_ids)) // self.block_size
|
||||
new_full_blocks = req_blocks[
|
||||
num_cached_blocks:num_full_blocks_after_append]
|
||||
|
||||
if new_full_blocks:
|
||||
self._cache_full_blocks(
|
||||
request=request,
|
||||
blk_start_idx=num_computed_full_blocks,
|
||||
blk_start_idx=num_cached_blocks,
|
||||
# The new full blocks are the full blocks that are not computed.
|
||||
full_blocks=new_full_blocks,
|
||||
prev_block=(req_blocks[num_computed_full_blocks - 1]
|
||||
if num_computed_full_blocks > 0 else None))
|
||||
|
||||
prev_block=(req_blocks[num_cached_blocks -
|
||||
1] if num_cached_blocks > 0 else None))
|
||||
self.num_cached_block[
|
||||
request.request_id] = num_full_blocks_after_append
|
||||
return new_blocks
|
||||
|
||||
def free(self, request: Request) -> None:
|
||||
@@ -281,6 +288,8 @@ class KVCacheManager:
|
||||
if block.ref_cnt == 0:
|
||||
self.free_block_queue.append(block)
|
||||
|
||||
self.num_cached_block.pop(request.request_id, None)
|
||||
|
||||
def reset_prefix_cache(self) -> bool:
|
||||
"""Reset prefix cache. This function may be used in RLHF
|
||||
flows to invalid prefix caching after the weights are updated,
|
||||
|
||||
+64
-29
@@ -4,7 +4,8 @@ import time
|
||||
from collections import deque
|
||||
from typing import Deque, Dict, Iterable, List, Optional, Set, Tuple, Union
|
||||
|
||||
from vllm.config import CacheConfig, LoRAConfig, ModelConfig, SchedulerConfig
|
||||
from vllm.config import (CacheConfig, LoRAConfig, ModelConfig, SchedulerConfig,
|
||||
SpeculativeConfig)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.core.encoder_cache_manager import (EncoderCacheManager,
|
||||
compute_encoder_budget)
|
||||
@@ -28,11 +29,13 @@ class Scheduler:
|
||||
model_config: ModelConfig,
|
||||
cache_config: CacheConfig,
|
||||
lora_config: Optional[LoRAConfig],
|
||||
speculative_config: Optional[SpeculativeConfig],
|
||||
log_stats: bool,
|
||||
) -> None:
|
||||
self.scheduler_config = scheduler_config
|
||||
self.cache_config = cache_config
|
||||
self.lora_config = lora_config
|
||||
self.speculative_config = speculative_config
|
||||
self.log_stats = log_stats
|
||||
|
||||
# Scheduling constraints.
|
||||
@@ -96,12 +99,14 @@ class Scheduler:
|
||||
def schedule(self) -> "SchedulerOutput":
|
||||
# NOTE(woosuk) on the scheduling algorithm:
|
||||
# There's no "decoding phase" nor "prefill phase" in the scheduler.
|
||||
# Each request just has the num_computed_tokens and num_tokens,
|
||||
# which is equal to len(prompt_token_ids) + len(output_token_ids).
|
||||
# Each request just has the num_computed_tokens and
|
||||
# num_tokens_with_spec. num_tokens_with_spec =
|
||||
# len(prompt_token_ids) + len(output_token_ids) + len(spec_token_ids).
|
||||
# At each step, the scheduler tries to assign tokens to the requests
|
||||
# so that each request's num_computed_tokens can catch up its
|
||||
# num_tokens. This is general enough to cover chunked prefills,
|
||||
# prefix caching, and the "jump decoding" optimization in the future.
|
||||
# num_tokens_with_spec. This is general enough to cover
|
||||
# chunked prefills, prefix caching, speculative decoding,
|
||||
# and the "jump decoding" optimization in the future.
|
||||
|
||||
scheduled_new_reqs: List[Request] = []
|
||||
scheduled_resumed_reqs: List[Request] = []
|
||||
@@ -114,7 +119,8 @@ class Scheduler:
|
||||
# Encoder-related.
|
||||
scheduled_encoder_inputs: Dict[str, List[int]] = {}
|
||||
encoder_budget = self.max_num_encoder_input_tokens
|
||||
|
||||
# Spec decode-related.
|
||||
scheduled_spec_decode_tokens: Dict[str, List[int]] = {}
|
||||
scheduled_timestamp = time.monotonic()
|
||||
|
||||
# First, schedule the RUNNING requests.
|
||||
@@ -126,7 +132,8 @@ class Scheduler:
|
||||
req_index += 1
|
||||
continue
|
||||
|
||||
num_new_tokens = request.num_tokens - request.num_computed_tokens
|
||||
num_new_tokens = (request.num_tokens_with_spec -
|
||||
request.num_computed_tokens)
|
||||
num_new_tokens = min(num_new_tokens, token_budget)
|
||||
assert num_new_tokens > 0
|
||||
|
||||
@@ -189,6 +196,11 @@ class Scheduler:
|
||||
self.encoder_cache_manager.allocate(request, i)
|
||||
encoder_budget = new_encoder_budget
|
||||
|
||||
# Speculative decode related.
|
||||
if request.spec_token_ids:
|
||||
scheduled_spec_decode_tokens[
|
||||
request.request_id] = request.spec_token_ids
|
||||
|
||||
# Record the LoRAs in scheduled_running_reqs
|
||||
requested_loras: Set[int] = set()
|
||||
if self.lora_config:
|
||||
@@ -338,6 +350,7 @@ class Scheduler:
|
||||
num_scheduled_tokens=num_scheduled_tokens,
|
||||
total_num_scheduled_tokens=total_num_scheduled_tokens,
|
||||
scheduled_encoder_inputs=scheduled_encoder_inputs,
|
||||
scheduled_spec_decode_tokens=scheduled_spec_decode_tokens,
|
||||
num_common_prefix_blocks=num_common_prefix_blocks,
|
||||
# finished_req_ids is an existing state in the scheduler,
|
||||
# instead of being newly scheduled in this step.
|
||||
@@ -447,11 +460,11 @@ class Scheduler:
|
||||
scheduler_output: "SchedulerOutput",
|
||||
model_runner_output: "ModelRunnerOutput",
|
||||
) -> EngineCoreOutputs:
|
||||
# NOTE(woosuk): This method doesn't consider speculative decoding.
|
||||
sampled_token_ids = model_runner_output.sampled_token_ids
|
||||
logprobs = model_runner_output.logprobs
|
||||
prompt_logprobs_dict = model_runner_output.prompt_logprobs_dict
|
||||
num_scheduled_tokens = scheduler_output.num_scheduled_tokens
|
||||
|
||||
new_running: List[Request] = []
|
||||
outputs: List[EngineCoreOutput] = []
|
||||
|
||||
@@ -466,11 +479,30 @@ class Scheduler:
|
||||
new_running.append(request)
|
||||
continue
|
||||
|
||||
request.num_computed_tokens += num_tokens_scheduled
|
||||
# When the request's num_computed_tokens catches up its num_tokens,
|
||||
# the request generates output tokens. Otherwise, we ignore the
|
||||
# sampler output for the request.
|
||||
assert request.num_computed_tokens <= request.num_tokens
|
||||
req_index = model_runner_output.req_id_to_index[req_id]
|
||||
generated_token_ids = sampled_token_ids[req_index]
|
||||
if req_id not in scheduler_output.scheduled_spec_decode_tokens:
|
||||
# When the request's num_computed_tokens catches up
|
||||
# its num_tokens, the request generates output tokens.
|
||||
# Otherwise, we ignore the sampler output for the request.
|
||||
request.num_computed_tokens += num_tokens_scheduled
|
||||
assert request.num_computed_tokens <= request.num_tokens
|
||||
else:
|
||||
# num_computed_tokens_step represents the number of tokens
|
||||
# processed in the current step, considering scheduled
|
||||
# tokens and rejections.
|
||||
# It is calculated as:
|
||||
# num_computed_tokens_step = num_scheduled_tokens -
|
||||
# num_tokens_rejected,
|
||||
# where num_tokens_rejected is given by:
|
||||
# len(scheduled_spec_token_ids) + 1 - len(generated_token_ids).
|
||||
scheduled_spec_token_ids = (
|
||||
scheduler_output.scheduled_spec_decode_tokens[req_id])
|
||||
|
||||
num_computed_tokens_step = num_scheduled_tokens[req_id] - (
|
||||
len(scheduled_spec_token_ids) + 1 -
|
||||
len(generated_token_ids))
|
||||
request.num_computed_tokens += num_computed_tokens_step
|
||||
|
||||
cached_encoder_input_ids = (
|
||||
self.encoder_cache_manager.get_cached_input_ids(request))
|
||||
@@ -485,27 +517,32 @@ class Scheduler:
|
||||
self.encoder_cache_manager.free_encoder_input(
|
||||
request, input_id)
|
||||
|
||||
if request.num_computed_tokens >= request.num_tokens:
|
||||
# Clear the spec tokens as the request has generated
|
||||
# a new token. Here, We assume all spec tokens are verified
|
||||
# if we perform speculative decoding for this request.
|
||||
# Therefore, we can clear all spec tokens after
|
||||
# the generation step.
|
||||
request.clear_spec_tokens()
|
||||
|
||||
# Get prompt logprobs for this request.
|
||||
prompt_logprobs_tensors = prompt_logprobs_dict.get(req_id)
|
||||
|
||||
stopped = False
|
||||
new_logprobs = None
|
||||
new_token_ids = None
|
||||
new_token_ids: List[int] = []
|
||||
|
||||
if request.num_computed_tokens == request.num_tokens:
|
||||
req_index = model_runner_output.req_id_to_index[req_id]
|
||||
# NOTE(woosuk): Currently, we assume that each request
|
||||
# generates at most one token at each step.
|
||||
token_id = sampled_token_ids[req_index]
|
||||
request.append_output_token_ids(token_id)
|
||||
num_new_tokens = 1
|
||||
# TODO: Update the KV cache manager for prefix caching.
|
||||
if request.num_computed_tokens >= request.num_tokens:
|
||||
for output_token_id in generated_token_ids:
|
||||
request.append_output_token_ids(output_token_id)
|
||||
new_token_ids.append(output_token_id)
|
||||
|
||||
# Check for stop and update request state.
|
||||
# This must be called before we make the EngineCoreOutput.
|
||||
stopped = self._check_stop(request)
|
||||
if stopped:
|
||||
self._free_request(request)
|
||||
# Check for stop and update request state.
|
||||
# This must be called before we make the EngineCoreOutput.
|
||||
stopped = self._check_stop(request)
|
||||
if stopped:
|
||||
self._free_request(request)
|
||||
break
|
||||
|
||||
# Extract sample logprobs if needed.
|
||||
if request.sampling_params.logprobs is not None:
|
||||
@@ -514,8 +551,6 @@ class Scheduler:
|
||||
# the outer lists can be of length > 1.
|
||||
new_logprobs = logprobs.slice(req_index, req_index + 1)
|
||||
|
||||
new_token_ids = request.output_token_ids[-num_new_tokens:]
|
||||
|
||||
# Transmit partial if chunked prefill & prompt logprobs is enabled
|
||||
if new_token_ids or prompt_logprobs_tensors is not None:
|
||||
# Add EngineCoreOutput for this Request.
|
||||
|
||||
@@ -91,6 +91,10 @@ class SchedulerOutput:
|
||||
# Total number of tokens scheduled for all requests.
|
||||
# Equal to sum(num_scheduled_tokens.values())
|
||||
total_num_scheduled_tokens: int
|
||||
# req_id -> spec_decode_tokens
|
||||
# If a request does not have any spec decode tokens, it will
|
||||
# not be included in the dictionary.
|
||||
scheduled_spec_decode_tokens: Dict[str, List[int]]
|
||||
# req_id -> encoder input indices that need processing.
|
||||
# E.g., if a request has [0, 1], it could mean the vision encoder needs
|
||||
# to process that the request's 0-th and 1-th images in the current step.
|
||||
|
||||
Reference in New Issue
Block a user