[V1][Spec Decode] Add Dynamic SD (#32374)

Signed-off-by: Ekagra Ranjan <3116519+ekagra-ranjan@users.noreply.github.com>
Signed-off-by: Benjamin Chislett <chislett.ben@gmail.com>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Benjamin Chislett <chislett.ben@gmail.com>
This commit is contained in:
Ekagra Ranjan
2026-06-14 00:14:27 -07:00
committed by GitHub
co-authored by gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Harry Mellor Benjamin Chislett
parent 78e7293bb1
commit 4ef4492e9b
23 changed files with 586 additions and 7 deletions
+2
View File
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+148
View File
@@ -0,0 +1,148 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
DynamicSDSchedule = list[tuple[int, int, int]]
def validate_and_normalize_dynamic_sd_schedule(
num_speculative_tokens_per_batch_size: object,
) -> DynamicSDSchedule:
"""Validate and normalize a Dynamic SD batch-size schedule.
The schedule is expressed as a list of inclusive ranges:
``[(range_start, range_end, num_speculative_tokens), ...]``
"""
if num_speculative_tokens_per_batch_size is None:
raise ValueError(
"num_speculative_tokens_per_batch_size is required for "
"dynamic speculative decoding."
)
if not isinstance(num_speculative_tokens_per_batch_size, list):
raise ValueError(
"num_speculative_tokens_per_batch_size must be a non-empty list of "
"(range_start, range_end, num_speculative_tokens) entries."
)
if not num_speculative_tokens_per_batch_size:
raise ValueError("num_speculative_tokens_per_batch_size must not be empty.")
parsed_schedule: DynamicSDSchedule = []
for entry in num_speculative_tokens_per_batch_size:
if not isinstance(entry, list | tuple) or len(entry) != 3:
raise ValueError(
"Each num_speculative_tokens_per_batch_size entry must be a "
"3-item sequence: (range_start, range_end, num_speculative_tokens)."
)
range_start, range_end, num_speculative_tokens = (
int(entry[0]),
int(entry[1]),
int(entry[2]),
)
if range_start <= 0 or range_end <= 0:
raise ValueError(
f"Batch-size range ({range_start}, {range_end}) must be positive."
)
if range_start > range_end:
raise ValueError(
"Batch-size range start must be <= end for "
f"({range_start}, {range_end}, {num_speculative_tokens})."
)
if num_speculative_tokens < 0:
raise ValueError(
"num_speculative_tokens_per_batch_size values must be >= 0."
)
parsed_schedule.append((range_start, range_end, num_speculative_tokens))
parsed_schedule.sort(key=lambda entry: entry[0])
previous_end = 0
for range_start, range_end, _ in parsed_schedule:
if range_start <= previous_end:
raise ValueError("Batch-size ranges must be non-overlapping and sorted.")
previous_end = range_end
first_range_start = parsed_schedule[0][0]
if first_range_start != 1:
raise ValueError(
"The first batch-size range must start at 1 so every runtime "
"batch size has a defined schedule."
)
return parsed_schedule
def build_dynamic_sd_schedule_lookup(
num_speculative_tokens_per_batch_size: object,
vllm_max_batch_size: int,
vllm_num_speculative_tokens: int,
) -> list[int]:
"""Expand the configured schedule into a dense batch_size -> K lookup.
"dense_schedule" means a 1-indexed lookup table where index ``batch_size``
stores the exact K to use for that runtime batch size. This lets the
scheduler do a simple array lookup instead of searching the configured
ranges on every scheduling step.
"""
if vllm_max_batch_size <= 0:
raise ValueError("vllm_max_batch_size must be > 0.")
if vllm_num_speculative_tokens <= 0:
raise ValueError("vllm_num_speculative_tokens must be > 0.")
parsed_schedule = validate_and_normalize_dynamic_sd_schedule(
num_speculative_tokens_per_batch_size
)
# Index 0 is intentionally unused so that valid runtime batch sizes can be
# looked up directly as dense_schedule[batch_size].
dense_schedule = [0] * (vllm_max_batch_size + 1)
next_batch_size = 1
last_num_speculative_tokens: int | None = None
for range_start, range_end, num_speculative_tokens in parsed_schedule:
if range_start > next_batch_size and last_num_speculative_tokens is not None:
# Fill any gap before the next configured range by carrying forward
# the previous K. For example, [(1, 16, 3), (32, 128, 2)] should map
# batch sizes 17-31 to K=3.
for batch_size in range(
next_batch_size,
min(range_start, vllm_max_batch_size + 1),
):
dense_schedule[batch_size] = min(
vllm_num_speculative_tokens,
last_num_speculative_tokens,
)
# Fill the current configured inclusive range with its K value.
for batch_size in range(
max(range_start, next_batch_size),
min(range_end, vllm_max_batch_size) + 1,
):
dense_schedule[batch_size] = min(
vllm_num_speculative_tokens,
num_speculative_tokens,
)
next_batch_size = max(next_batch_size, range_end + 1)
last_num_speculative_tokens = num_speculative_tokens
if next_batch_size > vllm_max_batch_size:
break
if last_num_speculative_tokens is None:
raise ValueError(
"num_speculative_tokens_per_batch_size must contain at least "
"one valid batch-size range."
)
# Fill the tail after the final configured range by carrying forward the
# last K through vllm_max_batch_size.
for batch_size in range(next_batch_size, vllm_max_batch_size + 1):
dense_schedule[batch_size] = min(
vllm_num_speculative_tokens,
last_num_speculative_tokens,
)
return dense_schedule
+6 -1
View File
@@ -29,7 +29,10 @@ class ExtractHiddenStatesProposer:
def __init__(self, vllm_config: VllmConfig, device):
assert vllm_config.speculative_config is not None
assert vllm_config.speculative_config.num_speculative_tokens == 1
self.num_speculative_tokens = (
vllm_config.speculative_config.num_speculative_tokens
)
assert self.num_speculative_tokens == 1
if vllm_config.speculative_config.disable_padded_drafter_batch:
raise ValueError(
"disable_padded_drafter_batch is not supported with "
@@ -82,6 +85,7 @@ class ExtractHiddenStatesProposer:
def propose(
self,
num_speculative_tokens: int,
sampled_token_ids: torch.Tensor,
target_hidden_states: list[torch.Tensor],
common_attn_metadata: CommonAttentionMetadata,
@@ -112,6 +116,7 @@ class ExtractHiddenStatesProposer:
- Draft tokens matching sampled tokens, shape [batch_size, 1]
- KV connector output (if KV transfer is active), else None
"""
assert num_speculative_tokens == self.num_speculative_tokens
assert self.model is not None and isinstance(target_hidden_states, list)
# target_hidden_states is a list of tensors (one per layer)
+13
View File
@@ -434,6 +434,7 @@ class SpecDecodeBaseProposer:
def propose(
self,
num_speculative_tokens,
# [num_tokens]
target_token_ids: torch.Tensor,
# [num_tokens] or [3, num_tokens] when M-RoPE is enabled
@@ -451,6 +452,7 @@ class SpecDecodeBaseProposer:
| list[dict[str, torch.Tensor]]
| None = None,
) -> torch.Tensor:
self.num_speculative_tokens = num_speculative_tokens
self._last_draft_probs = None
batch_size = common_attn_metadata.batch_size()
@@ -521,6 +523,17 @@ class SpecDecodeBaseProposer:
sample_hidden_states = last_hidden_states[token_indices_to_sample]
# No draft tokens requested (e.g. Dynamic SD decided K=0).
# The prefill forward pass above already ran to keep the drafter
# KV cache in sync, so just return an empty tensor.
if self.num_speculative_tokens == 0:
return torch.empty(
batch_size,
0,
device=sample_hidden_states.device,
dtype=torch.int64,
)
# Early exit if there is only one draft token to be generated.
if self.num_speculative_tokens == 1 or self.parallel_drafting:
draft_token_ids, draft_probs = self._sample_draft_tokens(
+3
View File
@@ -35,15 +35,18 @@ class MedusaProposer:
self.max_num_tokens = vllm_config.scheduler_config.max_num_batched_tokens
self.hidden_size = self.spec_config.draft_model_config.get_hidden_size()
self.dtype = vllm_config.model_config.dtype
self.num_speculative_tokens = self.spec_config.num_speculative_tokens
def propose(
self,
num_speculative_tokens: int,
target_hidden_states: torch.Tensor,
sampling_metadata: SamplingMetadata,
slot_mappings: dict[str, torch.Tensor]
| list[dict[str, torch.Tensor]]
| None = None, # unused
) -> torch.Tensor:
assert num_speculative_tokens == self.num_speculative_tokens
# Generate blocks and compute logits
blocks = self.model(target_hidden_states)
logits = self.model.compute_logits(blocks)
+4
View File
@@ -28,12 +28,14 @@ class SpecDecodingStats:
num_draft_tokens: int = 0
num_accepted_tokens: int = 0
num_accepted_tokens_per_pos: list[int] = field(default_factory=list)
num_draft_tokens_per_pos: list[int] = field(default_factory=list)
@classmethod
def new(cls, num_spec_tokens: int) -> "SpecDecodingStats":
return cls(
num_spec_tokens=num_spec_tokens,
num_accepted_tokens_per_pos=[0] * num_spec_tokens,
num_draft_tokens_per_pos=[0] * num_spec_tokens,
)
def observe_draft(self, num_draft_tokens: int, num_accepted_tokens: int):
@@ -43,6 +45,8 @@ class SpecDecodingStats:
assert num_accepted_tokens <= self.num_spec_tokens
for i in range(num_accepted_tokens):
self.num_accepted_tokens_per_pos[i] += 1
for i in range(num_draft_tokens):
self.num_draft_tokens_per_pos[i] += 1
class SpecDecodingLogging:
+9 -1
View File
@@ -55,6 +55,7 @@ class NgramProposer:
# Trigger Numba JIT compilation for N-gram proposer.
# This usually takes less than 1 second.
self.propose(
self.k,
[[]] * 1024,
np.zeros(1024, dtype=np.int32),
np.zeros((1024, self.max_model_len), dtype=np.int32),
@@ -66,6 +67,7 @@ class NgramProposer:
valid_ngram_requests: list,
num_tokens_no_spec: np.ndarray,
token_ids_cpu: np.ndarray,
k: int,
) -> list[list[int]]:
"""Batch version of ngram proposer using numba for acceleration.
@@ -78,6 +80,8 @@ class NgramProposer:
token_ids_cpu:
Numpy array of shape (batch_size, max_model_len)
representing the token IDs for each request.
k:
Number of speculative tokens to propose.
Returns:
list[list[int]]:
@@ -110,7 +114,7 @@ class NgramProposer:
self.min_n,
self.max_n,
self.max_model_len,
self.k,
k,
self.valid_ngram_draft,
self.valid_ngram_num_drafts,
)
@@ -130,6 +134,7 @@ class NgramProposer:
def propose(
self,
num_speculative_tokens: int,
sampled_token_ids: list[list[int]],
num_tokens_no_spec: np.ndarray,
token_ids_cpu: np.ndarray,
@@ -137,6 +142,8 @@ class NgramProposer:
| list[dict[str, torch.Tensor]]
| None = None, # unused
) -> list[list[int]]:
assert num_speculative_tokens <= self.k
# find which requests need ngram proposals
valid_ngram_requests = []
for i, sampled_ids in enumerate(sampled_token_ids):
@@ -157,6 +164,7 @@ class NgramProposer:
valid_ngram_requests,
num_tokens_no_spec,
token_ids_cpu,
num_speculative_tokens,
)
return draft_token_ids
@@ -314,6 +314,7 @@ class NgramProposerGPU:
def propose(
self,
num_speculative_tokens: int,
num_tokens_no_spec: torch.Tensor, # [batch_size]
token_ids_gpu: torch.Tensor, # [batch_size, max_len]
valid_sampled_token_ids_gpu: torch.Tensor, # [batch_size, num_spec_tokens + 1]
@@ -326,6 +327,7 @@ class NgramProposerGPU:
updated lengths, then run the kernel.
Args:
num_speculative_tokens: Number of speculative tokens to propose.
num_tokens_no_spec: Number of tokens per sequence (read-only)
token_ids_gpu: Token IDs tensor (modified in-place with new tokens)
valid_sampled_token_ids_gpu: Newly sampled tokens to scatter
@@ -336,6 +338,7 @@ class NgramProposerGPU:
num_valid_draft_tokens: Count of leading valid draft tokens
per request [batch_size]
"""
assert num_speculative_tokens == self.k
assert token_ids_gpu.device == self.device
assert num_tokens_no_spec.device == self.device
+2
View File
@@ -273,6 +273,7 @@ class Step3p5MTPProposer(EagleProposer):
def propose(
self,
num_speculative_tokens: int,
target_token_ids: torch.Tensor,
target_positions: torch.Tensor,
target_hidden_states: torch.Tensor,
@@ -286,6 +287,7 @@ class Step3p5MTPProposer(EagleProposer):
| list[dict[str, torch.Tensor]]
| None = None,
) -> torch.Tensor:
self.num_speculative_tokens = num_speculative_tokens
self._last_draft_probs = None
batch_size = common_attn_metadata.batch_size()
+2
View File
@@ -34,12 +34,14 @@ class SuffixDecodingProposer:
def propose(
self,
num_speculative_tokens: int,
input_batch: InputBatch,
sampled_token_ids: list[list[int]],
slot_mappings: dict[str, torch.Tensor]
| list[dict[str, torch.Tensor]]
| None = None, # unused
) -> list[list[int]]:
assert num_speculative_tokens == self.num_speculative_tokens
"""
Propose speculative tokens for each request in the input batch. Suffix Decoding
will speculate a dynamic number of tokens for each request every decoding step,