[V1][Spec Decode] Add Dynamic SD (#32374)
Signed-off-by: Ekagra Ranjan <3116519+ekagra-ranjan@users.noreply.github.com> Signed-off-by: Benjamin Chislett <chislett.ben@gmail.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Benjamin Chislett <chislett.ben@gmail.com>
This commit is contained in:
co-authored by
gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Harry Mellor
Benjamin Chislett
parent
78e7293bb1
commit
4ef4492e9b
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -0,0 +1,148 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
DynamicSDSchedule = list[tuple[int, int, int]]
|
||||
|
||||
|
||||
def validate_and_normalize_dynamic_sd_schedule(
|
||||
num_speculative_tokens_per_batch_size: object,
|
||||
) -> DynamicSDSchedule:
|
||||
"""Validate and normalize a Dynamic SD batch-size schedule.
|
||||
|
||||
The schedule is expressed as a list of inclusive ranges:
|
||||
|
||||
``[(range_start, range_end, num_speculative_tokens), ...]``
|
||||
"""
|
||||
if num_speculative_tokens_per_batch_size is None:
|
||||
raise ValueError(
|
||||
"num_speculative_tokens_per_batch_size is required for "
|
||||
"dynamic speculative decoding."
|
||||
)
|
||||
if not isinstance(num_speculative_tokens_per_batch_size, list):
|
||||
raise ValueError(
|
||||
"num_speculative_tokens_per_batch_size must be a non-empty list of "
|
||||
"(range_start, range_end, num_speculative_tokens) entries."
|
||||
)
|
||||
if not num_speculative_tokens_per_batch_size:
|
||||
raise ValueError("num_speculative_tokens_per_batch_size must not be empty.")
|
||||
|
||||
parsed_schedule: DynamicSDSchedule = []
|
||||
for entry in num_speculative_tokens_per_batch_size:
|
||||
if not isinstance(entry, list | tuple) or len(entry) != 3:
|
||||
raise ValueError(
|
||||
"Each num_speculative_tokens_per_batch_size entry must be a "
|
||||
"3-item sequence: (range_start, range_end, num_speculative_tokens)."
|
||||
)
|
||||
|
||||
range_start, range_end, num_speculative_tokens = (
|
||||
int(entry[0]),
|
||||
int(entry[1]),
|
||||
int(entry[2]),
|
||||
)
|
||||
|
||||
if range_start <= 0 or range_end <= 0:
|
||||
raise ValueError(
|
||||
f"Batch-size range ({range_start}, {range_end}) must be positive."
|
||||
)
|
||||
if range_start > range_end:
|
||||
raise ValueError(
|
||||
"Batch-size range start must be <= end for "
|
||||
f"({range_start}, {range_end}, {num_speculative_tokens})."
|
||||
)
|
||||
if num_speculative_tokens < 0:
|
||||
raise ValueError(
|
||||
"num_speculative_tokens_per_batch_size values must be >= 0."
|
||||
)
|
||||
|
||||
parsed_schedule.append((range_start, range_end, num_speculative_tokens))
|
||||
|
||||
parsed_schedule.sort(key=lambda entry: entry[0])
|
||||
|
||||
previous_end = 0
|
||||
for range_start, range_end, _ in parsed_schedule:
|
||||
if range_start <= previous_end:
|
||||
raise ValueError("Batch-size ranges must be non-overlapping and sorted.")
|
||||
previous_end = range_end
|
||||
|
||||
first_range_start = parsed_schedule[0][0]
|
||||
if first_range_start != 1:
|
||||
raise ValueError(
|
||||
"The first batch-size range must start at 1 so every runtime "
|
||||
"batch size has a defined schedule."
|
||||
)
|
||||
|
||||
return parsed_schedule
|
||||
|
||||
|
||||
def build_dynamic_sd_schedule_lookup(
|
||||
num_speculative_tokens_per_batch_size: object,
|
||||
vllm_max_batch_size: int,
|
||||
vllm_num_speculative_tokens: int,
|
||||
) -> list[int]:
|
||||
"""Expand the configured schedule into a dense batch_size -> K lookup.
|
||||
|
||||
"dense_schedule" means a 1-indexed lookup table where index ``batch_size``
|
||||
stores the exact K to use for that runtime batch size. This lets the
|
||||
scheduler do a simple array lookup instead of searching the configured
|
||||
ranges on every scheduling step.
|
||||
"""
|
||||
if vllm_max_batch_size <= 0:
|
||||
raise ValueError("vllm_max_batch_size must be > 0.")
|
||||
if vllm_num_speculative_tokens <= 0:
|
||||
raise ValueError("vllm_num_speculative_tokens must be > 0.")
|
||||
|
||||
parsed_schedule = validate_and_normalize_dynamic_sd_schedule(
|
||||
num_speculative_tokens_per_batch_size
|
||||
)
|
||||
|
||||
# Index 0 is intentionally unused so that valid runtime batch sizes can be
|
||||
# looked up directly as dense_schedule[batch_size].
|
||||
dense_schedule = [0] * (vllm_max_batch_size + 1)
|
||||
next_batch_size = 1
|
||||
last_num_speculative_tokens: int | None = None
|
||||
|
||||
for range_start, range_end, num_speculative_tokens in parsed_schedule:
|
||||
if range_start > next_batch_size and last_num_speculative_tokens is not None:
|
||||
# Fill any gap before the next configured range by carrying forward
|
||||
# the previous K. For example, [(1, 16, 3), (32, 128, 2)] should map
|
||||
# batch sizes 17-31 to K=3.
|
||||
for batch_size in range(
|
||||
next_batch_size,
|
||||
min(range_start, vllm_max_batch_size + 1),
|
||||
):
|
||||
dense_schedule[batch_size] = min(
|
||||
vllm_num_speculative_tokens,
|
||||
last_num_speculative_tokens,
|
||||
)
|
||||
|
||||
# Fill the current configured inclusive range with its K value.
|
||||
for batch_size in range(
|
||||
max(range_start, next_batch_size),
|
||||
min(range_end, vllm_max_batch_size) + 1,
|
||||
):
|
||||
dense_schedule[batch_size] = min(
|
||||
vllm_num_speculative_tokens,
|
||||
num_speculative_tokens,
|
||||
)
|
||||
|
||||
next_batch_size = max(next_batch_size, range_end + 1)
|
||||
last_num_speculative_tokens = num_speculative_tokens
|
||||
|
||||
if next_batch_size > vllm_max_batch_size:
|
||||
break
|
||||
|
||||
if last_num_speculative_tokens is None:
|
||||
raise ValueError(
|
||||
"num_speculative_tokens_per_batch_size must contain at least "
|
||||
"one valid batch-size range."
|
||||
)
|
||||
|
||||
# Fill the tail after the final configured range by carrying forward the
|
||||
# last K through vllm_max_batch_size.
|
||||
for batch_size in range(next_batch_size, vllm_max_batch_size + 1):
|
||||
dense_schedule[batch_size] = min(
|
||||
vllm_num_speculative_tokens,
|
||||
last_num_speculative_tokens,
|
||||
)
|
||||
|
||||
return dense_schedule
|
||||
@@ -29,7 +29,10 @@ class ExtractHiddenStatesProposer:
|
||||
def __init__(self, vllm_config: VllmConfig, device):
|
||||
assert vllm_config.speculative_config is not None
|
||||
|
||||
assert vllm_config.speculative_config.num_speculative_tokens == 1
|
||||
self.num_speculative_tokens = (
|
||||
vllm_config.speculative_config.num_speculative_tokens
|
||||
)
|
||||
assert self.num_speculative_tokens == 1
|
||||
if vllm_config.speculative_config.disable_padded_drafter_batch:
|
||||
raise ValueError(
|
||||
"disable_padded_drafter_batch is not supported with "
|
||||
@@ -82,6 +85,7 @@ class ExtractHiddenStatesProposer:
|
||||
|
||||
def propose(
|
||||
self,
|
||||
num_speculative_tokens: int,
|
||||
sampled_token_ids: torch.Tensor,
|
||||
target_hidden_states: list[torch.Tensor],
|
||||
common_attn_metadata: CommonAttentionMetadata,
|
||||
@@ -112,6 +116,7 @@ class ExtractHiddenStatesProposer:
|
||||
- Draft tokens matching sampled tokens, shape [batch_size, 1]
|
||||
- KV connector output (if KV transfer is active), else None
|
||||
"""
|
||||
assert num_speculative_tokens == self.num_speculative_tokens
|
||||
assert self.model is not None and isinstance(target_hidden_states, list)
|
||||
|
||||
# target_hidden_states is a list of tensors (one per layer)
|
||||
|
||||
@@ -434,6 +434,7 @@ class SpecDecodeBaseProposer:
|
||||
|
||||
def propose(
|
||||
self,
|
||||
num_speculative_tokens,
|
||||
# [num_tokens]
|
||||
target_token_ids: torch.Tensor,
|
||||
# [num_tokens] or [3, num_tokens] when M-RoPE is enabled
|
||||
@@ -451,6 +452,7 @@ class SpecDecodeBaseProposer:
|
||||
| list[dict[str, torch.Tensor]]
|
||||
| None = None,
|
||||
) -> torch.Tensor:
|
||||
self.num_speculative_tokens = num_speculative_tokens
|
||||
self._last_draft_probs = None
|
||||
batch_size = common_attn_metadata.batch_size()
|
||||
|
||||
@@ -521,6 +523,17 @@ class SpecDecodeBaseProposer:
|
||||
|
||||
sample_hidden_states = last_hidden_states[token_indices_to_sample]
|
||||
|
||||
# No draft tokens requested (e.g. Dynamic SD decided K=0).
|
||||
# The prefill forward pass above already ran to keep the drafter
|
||||
# KV cache in sync, so just return an empty tensor.
|
||||
if self.num_speculative_tokens == 0:
|
||||
return torch.empty(
|
||||
batch_size,
|
||||
0,
|
||||
device=sample_hidden_states.device,
|
||||
dtype=torch.int64,
|
||||
)
|
||||
|
||||
# Early exit if there is only one draft token to be generated.
|
||||
if self.num_speculative_tokens == 1 or self.parallel_drafting:
|
||||
draft_token_ids, draft_probs = self._sample_draft_tokens(
|
||||
|
||||
@@ -35,15 +35,18 @@ class MedusaProposer:
|
||||
self.max_num_tokens = vllm_config.scheduler_config.max_num_batched_tokens
|
||||
self.hidden_size = self.spec_config.draft_model_config.get_hidden_size()
|
||||
self.dtype = vllm_config.model_config.dtype
|
||||
self.num_speculative_tokens = self.spec_config.num_speculative_tokens
|
||||
|
||||
def propose(
|
||||
self,
|
||||
num_speculative_tokens: int,
|
||||
target_hidden_states: torch.Tensor,
|
||||
sampling_metadata: SamplingMetadata,
|
||||
slot_mappings: dict[str, torch.Tensor]
|
||||
| list[dict[str, torch.Tensor]]
|
||||
| None = None, # unused
|
||||
) -> torch.Tensor:
|
||||
assert num_speculative_tokens == self.num_speculative_tokens
|
||||
# Generate blocks and compute logits
|
||||
blocks = self.model(target_hidden_states)
|
||||
logits = self.model.compute_logits(blocks)
|
||||
|
||||
@@ -28,12 +28,14 @@ class SpecDecodingStats:
|
||||
num_draft_tokens: int = 0
|
||||
num_accepted_tokens: int = 0
|
||||
num_accepted_tokens_per_pos: list[int] = field(default_factory=list)
|
||||
num_draft_tokens_per_pos: list[int] = field(default_factory=list)
|
||||
|
||||
@classmethod
|
||||
def new(cls, num_spec_tokens: int) -> "SpecDecodingStats":
|
||||
return cls(
|
||||
num_spec_tokens=num_spec_tokens,
|
||||
num_accepted_tokens_per_pos=[0] * num_spec_tokens,
|
||||
num_draft_tokens_per_pos=[0] * num_spec_tokens,
|
||||
)
|
||||
|
||||
def observe_draft(self, num_draft_tokens: int, num_accepted_tokens: int):
|
||||
@@ -43,6 +45,8 @@ class SpecDecodingStats:
|
||||
assert num_accepted_tokens <= self.num_spec_tokens
|
||||
for i in range(num_accepted_tokens):
|
||||
self.num_accepted_tokens_per_pos[i] += 1
|
||||
for i in range(num_draft_tokens):
|
||||
self.num_draft_tokens_per_pos[i] += 1
|
||||
|
||||
|
||||
class SpecDecodingLogging:
|
||||
|
||||
@@ -55,6 +55,7 @@ class NgramProposer:
|
||||
# Trigger Numba JIT compilation for N-gram proposer.
|
||||
# This usually takes less than 1 second.
|
||||
self.propose(
|
||||
self.k,
|
||||
[[]] * 1024,
|
||||
np.zeros(1024, dtype=np.int32),
|
||||
np.zeros((1024, self.max_model_len), dtype=np.int32),
|
||||
@@ -66,6 +67,7 @@ class NgramProposer:
|
||||
valid_ngram_requests: list,
|
||||
num_tokens_no_spec: np.ndarray,
|
||||
token_ids_cpu: np.ndarray,
|
||||
k: int,
|
||||
) -> list[list[int]]:
|
||||
"""Batch version of ngram proposer using numba for acceleration.
|
||||
|
||||
@@ -78,6 +80,8 @@ class NgramProposer:
|
||||
token_ids_cpu:
|
||||
Numpy array of shape (batch_size, max_model_len)
|
||||
representing the token IDs for each request.
|
||||
k:
|
||||
Number of speculative tokens to propose.
|
||||
|
||||
Returns:
|
||||
list[list[int]]:
|
||||
@@ -110,7 +114,7 @@ class NgramProposer:
|
||||
self.min_n,
|
||||
self.max_n,
|
||||
self.max_model_len,
|
||||
self.k,
|
||||
k,
|
||||
self.valid_ngram_draft,
|
||||
self.valid_ngram_num_drafts,
|
||||
)
|
||||
@@ -130,6 +134,7 @@ class NgramProposer:
|
||||
|
||||
def propose(
|
||||
self,
|
||||
num_speculative_tokens: int,
|
||||
sampled_token_ids: list[list[int]],
|
||||
num_tokens_no_spec: np.ndarray,
|
||||
token_ids_cpu: np.ndarray,
|
||||
@@ -137,6 +142,8 @@ class NgramProposer:
|
||||
| list[dict[str, torch.Tensor]]
|
||||
| None = None, # unused
|
||||
) -> list[list[int]]:
|
||||
assert num_speculative_tokens <= self.k
|
||||
|
||||
# find which requests need ngram proposals
|
||||
valid_ngram_requests = []
|
||||
for i, sampled_ids in enumerate(sampled_token_ids):
|
||||
@@ -157,6 +164,7 @@ class NgramProposer:
|
||||
valid_ngram_requests,
|
||||
num_tokens_no_spec,
|
||||
token_ids_cpu,
|
||||
num_speculative_tokens,
|
||||
)
|
||||
|
||||
return draft_token_ids
|
||||
|
||||
@@ -314,6 +314,7 @@ class NgramProposerGPU:
|
||||
|
||||
def propose(
|
||||
self,
|
||||
num_speculative_tokens: int,
|
||||
num_tokens_no_spec: torch.Tensor, # [batch_size]
|
||||
token_ids_gpu: torch.Tensor, # [batch_size, max_len]
|
||||
valid_sampled_token_ids_gpu: torch.Tensor, # [batch_size, num_spec_tokens + 1]
|
||||
@@ -326,6 +327,7 @@ class NgramProposerGPU:
|
||||
updated lengths, then run the kernel.
|
||||
|
||||
Args:
|
||||
num_speculative_tokens: Number of speculative tokens to propose.
|
||||
num_tokens_no_spec: Number of tokens per sequence (read-only)
|
||||
token_ids_gpu: Token IDs tensor (modified in-place with new tokens)
|
||||
valid_sampled_token_ids_gpu: Newly sampled tokens to scatter
|
||||
@@ -336,6 +338,7 @@ class NgramProposerGPU:
|
||||
num_valid_draft_tokens: Count of leading valid draft tokens
|
||||
per request [batch_size]
|
||||
"""
|
||||
assert num_speculative_tokens == self.k
|
||||
assert token_ids_gpu.device == self.device
|
||||
assert num_tokens_no_spec.device == self.device
|
||||
|
||||
|
||||
@@ -273,6 +273,7 @@ class Step3p5MTPProposer(EagleProposer):
|
||||
|
||||
def propose(
|
||||
self,
|
||||
num_speculative_tokens: int,
|
||||
target_token_ids: torch.Tensor,
|
||||
target_positions: torch.Tensor,
|
||||
target_hidden_states: torch.Tensor,
|
||||
@@ -286,6 +287,7 @@ class Step3p5MTPProposer(EagleProposer):
|
||||
| list[dict[str, torch.Tensor]]
|
||||
| None = None,
|
||||
) -> torch.Tensor:
|
||||
self.num_speculative_tokens = num_speculative_tokens
|
||||
self._last_draft_probs = None
|
||||
batch_size = common_attn_metadata.batch_size()
|
||||
|
||||
|
||||
@@ -34,12 +34,14 @@ class SuffixDecodingProposer:
|
||||
|
||||
def propose(
|
||||
self,
|
||||
num_speculative_tokens: int,
|
||||
input_batch: InputBatch,
|
||||
sampled_token_ids: list[list[int]],
|
||||
slot_mappings: dict[str, torch.Tensor]
|
||||
| list[dict[str, torch.Tensor]]
|
||||
| None = None, # unused
|
||||
) -> list[list[int]]:
|
||||
assert num_speculative_tokens == self.num_speculative_tokens
|
||||
"""
|
||||
Propose speculative tokens for each request in the input batch. Suffix Decoding
|
||||
will speculate a dynamic number of tokens for each request every decoding step,
|
||||
|
||||
Reference in New Issue
Block a user