[V1][Spec Decode] Add Dynamic SD (#32374)

Signed-off-by: Ekagra Ranjan <3116519+ekagra-ranjan@users.noreply.github.com>
Signed-off-by: Benjamin Chislett <chislett.ben@gmail.com>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Benjamin Chislett <chislett.ben@gmail.com>
This commit is contained in:
Ekagra Ranjan
2026-06-14 00:14:27 -07:00
committed by GitHub
co-authored by gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Harry Mellor Benjamin Chislett
parent 78e7293bb1
commit 4ef4492e9b
23 changed files with 586 additions and 7 deletions
+218
View File
@@ -0,0 +1,218 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Regression tests for the Dynamic SD batch-size schedule helpers."""
import pytest
from tests.v1.core.utils import create_requests, create_scheduler
from vllm.v1.core.sched.scheduler import Scheduler
from vllm.v1.spec_decode.dynamic.utils import build_dynamic_sd_schedule_lookup
from vllm.v1.structured_output import StructuredOutputManager
def _make_lookup(
num_speculative_tokens_per_batch_size: list[tuple[int, int, int]],
*,
max_batch_size: int = 256,
runtime_num_speculative_tokens: int = 3,
) -> list[int]:
return build_dynamic_sd_schedule_lookup(
num_speculative_tokens_per_batch_size=num_speculative_tokens_per_batch_size,
vllm_max_batch_size=max_batch_size,
vllm_num_speculative_tokens=runtime_num_speculative_tokens,
)
def _make_scheduler_with_dynamic_sd(
schedule: list[tuple[int, int, int]],
*,
max_num_seqs: int = 16,
max_num_batched_tokens: int = 8192,
runtime_num_speculative_tokens: int = 3,
) -> Scheduler:
base_scheduler = create_scheduler(
max_num_seqs=max_num_seqs,
max_num_batched_tokens=max_num_batched_tokens,
num_speculative_tokens=runtime_num_speculative_tokens,
)
speculative_config = base_scheduler.vllm_config.speculative_config
assert speculative_config is not None
speculative_config.num_speculative_tokens_per_batch_size = schedule
return Scheduler(
vllm_config=base_scheduler.vllm_config,
kv_cache_config=base_scheduler.kv_cache_config,
block_size=base_scheduler.block_size,
log_stats=True,
structured_output_manager=StructuredOutputManager(base_scheduler.vllm_config),
)
def _add_requests_and_schedule(
scheduler: Scheduler, num_requests: int, *, num_tokens: int = 10
):
requests = create_requests(num_requests=num_requests, num_tokens=num_tokens)
for request in requests:
scheduler.add_request(request)
return scheduler.schedule()
def test_dynamic_sd_uses_batch_size_schedule():
dynamic_sd_lookup = _make_lookup(
[
(1, 16, 3),
(32, 128, 2),
(256, 2048, 0),
]
)
assert dynamic_sd_lookup[1] == 3
assert dynamic_sd_lookup[16] == 3
assert dynamic_sd_lookup[17] == 3
assert dynamic_sd_lookup[31] == 3
assert dynamic_sd_lookup[32] == 2
assert dynamic_sd_lookup[128] == 2
assert dynamic_sd_lookup[129] == 2
assert dynamic_sd_lookup[255] == 2
assert dynamic_sd_lookup[256] == 0
def test_dynamic_sd_requires_schedule_starting_at_batch_size_one():
with pytest.raises(ValueError, match="must start at 1"):
_make_lookup([(2, 16, 3)])
def test_dynamic_sd_clamps_k_to_runtime_max():
dynamic_sd_lookup = _make_lookup(
[(1, 256, 4)],
runtime_num_speculative_tokens=3,
)
assert dynamic_sd_lookup[1] == 3
assert dynamic_sd_lookup[256] == 3
def test_dynamic_sd_rejects_invalid_schedule_entry():
with pytest.raises(ValueError, match="3-item sequence"):
_make_lookup([(1, 16, 3), (32, 64)]) # type: ignore[list-item]
def test_dynamic_sd_rejects_overlapping_ranges():
with pytest.raises(ValueError, match="non-overlapping and sorted"):
_make_lookup([(1, 16, 3), (16, 32, 2)])
def test_dynamic_sd_rejects_negative_k():
with pytest.raises(ValueError, match="values must be >= 0"):
_make_lookup([(1, 16, -1)])
def test_dynamic_sd_rejects_empty_schedule():
with pytest.raises(ValueError, match="must not be empty"):
_make_lookup([])
def test_dynamic_sd_requires_schedule_config():
with pytest.raises(
ValueError, match="num_speculative_tokens_per_batch_size is required"
):
build_dynamic_sd_schedule_lookup(
None,
vllm_max_batch_size=256,
vllm_num_speculative_tokens=3,
)
def test_dynamic_sd_lookup_rejects_invalid_batch_size_queries():
dynamic_sd_lookup = _make_lookup([(1, 256, 3)])
assert dynamic_sd_lookup[0] == 0
with pytest.raises(IndexError):
_ = dynamic_sd_lookup[257]
def test_scheduler_initializes_dynamic_sd_lookup_from_speculative_config():
scheduler = _make_scheduler_with_dynamic_sd(
[(1, 16, 3), (64, 128, 2), (256, 4096, 0)],
runtime_num_speculative_tokens=3,
)
assert scheduler.dynamic_sd_lookup is not None
assert scheduler.num_spec_tokens == 3
def test_scheduler_uses_dsd_k_based_on_number_of_scheduled_requests():
test_cases = [
(4, 3),
(64, 2),
(256, 0),
]
for num_requests, expected_k in test_cases:
scheduler = _make_scheduler_with_dynamic_sd(
[(1, 16, 3), (64, 128, 2), (256, 4096, 0)],
max_num_seqs=num_requests,
max_num_batched_tokens=num_requests * 10,
runtime_num_speculative_tokens=3,
)
output = _add_requests_and_schedule(scheduler, num_requests)
assert len(output.num_scheduled_tokens) == num_requests
assert output.num_spec_tokens_to_schedule == expected_k
def test_scheduler_clamps_dsd_k_to_runtime_num_speculative_tokens():
scheduler = _make_scheduler_with_dynamic_sd(
[(1, 256, 5)],
max_num_seqs=16,
max_num_batched_tokens=160,
runtime_num_speculative_tokens=3,
)
output = _add_requests_and_schedule(scheduler, 16)
assert len(output.num_scheduled_tokens) == 16
assert output.num_spec_tokens_to_schedule == 3
def test_scheduler_falls_back_to_static_k_when_dsd_not_configured():
scheduler = create_scheduler(
max_num_seqs=4,
max_num_batched_tokens=40,
num_speculative_tokens=3,
)
output = _add_requests_and_schedule(scheduler, 4)
assert scheduler.dynamic_sd_lookup is None
assert output.num_spec_tokens_to_schedule == 3
def test_scheduler_uses_static_k_when_no_requests_are_scheduled():
scheduler = _make_scheduler_with_dynamic_sd(
[(1, 16, 3), (64, 128, 2), (256, 4096, 0)],
runtime_num_speculative_tokens=3,
)
output = scheduler.schedule()
assert len(output.num_scheduled_tokens) == 0
assert output.num_spec_tokens_to_schedule == 3
def test_scheduler_rejects_bad_dsd_config_at_construction():
with pytest.raises(ValueError, match="must start at 1"):
_make_scheduler_with_dynamic_sd([(2, 16, 3)])
def test_scheduler_passes_max_num_seqs_as_dsd_runtime_batch_limit():
scheduler = _make_scheduler_with_dynamic_sd(
[(1, 16, 3), (64, 128, 2), (256, 4096, 0)],
max_num_seqs=16,
max_num_batched_tokens=160,
runtime_num_speculative_tokens=3,
)
output = _add_requests_and_schedule(scheduler, 16)
assert scheduler.dynamic_sd_lookup is not None
assert len(scheduler.dynamic_sd_lookup) == 17
assert len(output.num_scheduled_tokens) == 16
assert output.num_spec_tokens_to_schedule == 3
+2
View File
@@ -969,6 +969,7 @@ def test_propose(method, attn_backend, num_speculative_tokens, monkeypatch):
proposer.draft_attn_groups = [mock_attn_group]
result = proposer.propose(
num_speculative_tokens=num_speculative_tokens,
target_token_ids=target_token_ids,
target_positions=target_positions,
target_hidden_states=target_hidden_states,
@@ -1071,6 +1072,7 @@ def test_propose_stores_probabilistic_draft_probs(monkeypatch):
sampling_metadata.all_greedy = False
result = proposer.propose(
num_speculative_tokens=num_speculative_tokens,
target_token_ids=torch.randint(0, vocab_size, (total_tokens,), device=device),
target_positions=torch.cat(
[
@@ -255,6 +255,7 @@ def test_propose():
# Call propose
draft_tokens = proposer.propose(
num_speculative_tokens=1,
sampled_token_ids=sampled_token_ids,
target_hidden_states=target_hidden_states,
common_attn_metadata=common_attn_metadata,
@@ -321,6 +322,7 @@ def test_propose_different_layer_counts(num_hidden_layers):
).unsqueeze(-1)
draft_tokens = proposer.propose(
num_speculative_tokens=1,
sampled_token_ids=sampled_token_ids,
target_hidden_states=target_hidden_states,
common_attn_metadata=common_attn_metadata,
+1
View File
@@ -205,6 +205,7 @@ def test_mtp_propose(num_speculative_tokens, monkeypatch):
# Run propose
result = proposer.propose(
num_speculative_tokens=num_speculative_tokens,
target_token_ids=target_token_ids,
target_positions=target_positions,
target_hidden_states=target_hidden_states,
+10
View File
@@ -81,6 +81,7 @@ def test_ngram_proposer():
# No match.
token_ids_cpu = np.array([[1, 2, 3, 4, 5]])
result = get_ngram_proposer(min_n=2, max_n=2, k=2).propose(
num_speculative_tokens=2,
sampled_token_ids=[[0]],
num_tokens_no_spec=np.array([len(c) for c in token_ids_cpu]),
token_ids_cpu=token_ids_cpu,
@@ -90,6 +91,7 @@ def test_ngram_proposer():
# No match for 4-gram.
token_ids_cpu = np.array([[1, 2, 3, 4, 1, 2, 3]])
result = get_ngram_proposer(min_n=4, max_n=4, k=2).propose(
num_speculative_tokens=2,
sampled_token_ids=[[0]],
num_tokens_no_spec=np.array([len(c) for c in token_ids_cpu]),
token_ids_cpu=token_ids_cpu,
@@ -99,6 +101,7 @@ def test_ngram_proposer():
# No match for 4-gram but match for 3-gram.
token_ids_cpu = np.array([[1, 2, 3, 4, 1, 2, 3]])
result = get_ngram_proposer(min_n=3, max_n=4, k=2).propose(
num_speculative_tokens=2,
sampled_token_ids=[[0]],
num_tokens_no_spec=np.array([len(c) for c in token_ids_cpu]),
token_ids_cpu=token_ids_cpu,
@@ -109,6 +112,7 @@ def test_ngram_proposer():
# In this case, the proposer should return the 4-gram match.
token_ids_cpu = np.array([[2, 3, 4, 5, 1, 2, 3, 4, 1, 2, 3, 4]])
result = get_ngram_proposer(min_n=3, max_n=4, k=2).propose(
num_speculative_tokens=2,
sampled_token_ids=[[0]],
num_tokens_no_spec=np.array([len(c) for c in token_ids_cpu]),
token_ids_cpu=token_ids_cpu,
@@ -118,6 +122,7 @@ def test_ngram_proposer():
# Match for 2-gram and 3-gram, but not 4-gram.
token_ids_cpu = np.array([[3, 4, 5, 2, 3, 4, 1, 2, 3, 4]])
result = get_ngram_proposer(min_n=2, max_n=4, k=2).propose(
num_speculative_tokens=2,
sampled_token_ids=[[0]],
num_tokens_no_spec=np.array([len(c) for c in token_ids_cpu]),
token_ids_cpu=token_ids_cpu,
@@ -127,6 +132,7 @@ def test_ngram_proposer():
# Multiple 3-gram matched, but always pick the first one.
token_ids_cpu = np.array([[1, 2, 3, 100, 1, 2, 3, 200, 1, 2, 3, 300, 1, 2, 3]])
result = get_ngram_proposer(min_n=3, max_n=3, k=2).propose(
num_speculative_tokens=2,
sampled_token_ids=[[0]],
num_tokens_no_spec=np.array([len(c) for c in token_ids_cpu]),
token_ids_cpu=token_ids_cpu,
@@ -136,6 +142,7 @@ def test_ngram_proposer():
# check empty input
token_ids_cpu = np.array([[]])
result = get_ngram_proposer(min_n=2, max_n=2, k=2).propose(
num_speculative_tokens=2,
sampled_token_ids=[[0]],
num_tokens_no_spec=np.array([len(c) for c in token_ids_cpu]),
token_ids_cpu=token_ids_cpu,
@@ -147,6 +154,7 @@ def test_ngram_proposer():
# second request has 3 tokens and no match. Padded with -1 for max len 5
token_ids_cpu = np.array([[1, 2, 3, 1, 2], [4, 5, 6, -1, -1]])
result = get_ngram_proposer(min_n=2, max_n=2, k=2).propose(
num_speculative_tokens=2,
sampled_token_ids=[[0], [1]],
num_tokens_no_spec=np.array([5, 3]),
token_ids_cpu=token_ids_cpu,
@@ -166,6 +174,7 @@ def test_ngram_proposer():
num_tokens_no_spec = np.array([5, 3, 5], dtype=np.int32)
sampled_token_ids = [[2], [], [8]] # Empty list for request 1 simulates prefill
result = proposer.propose(
num_speculative_tokens=2,
sampled_token_ids=sampled_token_ids,
num_tokens_no_spec=num_tokens_no_spec,
token_ids_cpu=token_ids_cpu,
@@ -195,6 +204,7 @@ def test_ngram_proposer():
input_2[:3] = [4, 5, 6]
token_ids_cpu = np.array([input_1, input_2])
result = ngram_proposer.propose(
num_speculative_tokens=2,
sampled_token_ids=[[0], [1]],
num_tokens_no_spec=np.array([len(input_1), 3]),
token_ids_cpu=token_ids_cpu,