[V1][Spec Decode] Add Dynamic SD (#32374)
Signed-off-by: Ekagra Ranjan <3116519+ekagra-ranjan@users.noreply.github.com> Signed-off-by: Benjamin Chislett <chislett.ben@gmail.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Benjamin Chislett <chislett.ben@gmail.com>
This commit is contained in:
co-authored by
gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Harry Mellor
Benjamin Chislett
parent
78e7293bb1
commit
4ef4492e9b
@@ -0,0 +1,218 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Regression tests for the Dynamic SD batch-size schedule helpers."""
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.v1.core.utils import create_requests, create_scheduler
|
||||
from vllm.v1.core.sched.scheduler import Scheduler
|
||||
from vllm.v1.spec_decode.dynamic.utils import build_dynamic_sd_schedule_lookup
|
||||
from vllm.v1.structured_output import StructuredOutputManager
|
||||
|
||||
|
||||
def _make_lookup(
|
||||
num_speculative_tokens_per_batch_size: list[tuple[int, int, int]],
|
||||
*,
|
||||
max_batch_size: int = 256,
|
||||
runtime_num_speculative_tokens: int = 3,
|
||||
) -> list[int]:
|
||||
return build_dynamic_sd_schedule_lookup(
|
||||
num_speculative_tokens_per_batch_size=num_speculative_tokens_per_batch_size,
|
||||
vllm_max_batch_size=max_batch_size,
|
||||
vllm_num_speculative_tokens=runtime_num_speculative_tokens,
|
||||
)
|
||||
|
||||
|
||||
def _make_scheduler_with_dynamic_sd(
|
||||
schedule: list[tuple[int, int, int]],
|
||||
*,
|
||||
max_num_seqs: int = 16,
|
||||
max_num_batched_tokens: int = 8192,
|
||||
runtime_num_speculative_tokens: int = 3,
|
||||
) -> Scheduler:
|
||||
base_scheduler = create_scheduler(
|
||||
max_num_seqs=max_num_seqs,
|
||||
max_num_batched_tokens=max_num_batched_tokens,
|
||||
num_speculative_tokens=runtime_num_speculative_tokens,
|
||||
)
|
||||
|
||||
speculative_config = base_scheduler.vllm_config.speculative_config
|
||||
assert speculative_config is not None
|
||||
speculative_config.num_speculative_tokens_per_batch_size = schedule
|
||||
|
||||
return Scheduler(
|
||||
vllm_config=base_scheduler.vllm_config,
|
||||
kv_cache_config=base_scheduler.kv_cache_config,
|
||||
block_size=base_scheduler.block_size,
|
||||
log_stats=True,
|
||||
structured_output_manager=StructuredOutputManager(base_scheduler.vllm_config),
|
||||
)
|
||||
|
||||
|
||||
def _add_requests_and_schedule(
|
||||
scheduler: Scheduler, num_requests: int, *, num_tokens: int = 10
|
||||
):
|
||||
requests = create_requests(num_requests=num_requests, num_tokens=num_tokens)
|
||||
for request in requests:
|
||||
scheduler.add_request(request)
|
||||
return scheduler.schedule()
|
||||
|
||||
|
||||
def test_dynamic_sd_uses_batch_size_schedule():
|
||||
dynamic_sd_lookup = _make_lookup(
|
||||
[
|
||||
(1, 16, 3),
|
||||
(32, 128, 2),
|
||||
(256, 2048, 0),
|
||||
]
|
||||
)
|
||||
|
||||
assert dynamic_sd_lookup[1] == 3
|
||||
assert dynamic_sd_lookup[16] == 3
|
||||
assert dynamic_sd_lookup[17] == 3
|
||||
assert dynamic_sd_lookup[31] == 3
|
||||
assert dynamic_sd_lookup[32] == 2
|
||||
assert dynamic_sd_lookup[128] == 2
|
||||
assert dynamic_sd_lookup[129] == 2
|
||||
assert dynamic_sd_lookup[255] == 2
|
||||
assert dynamic_sd_lookup[256] == 0
|
||||
|
||||
|
||||
def test_dynamic_sd_requires_schedule_starting_at_batch_size_one():
|
||||
with pytest.raises(ValueError, match="must start at 1"):
|
||||
_make_lookup([(2, 16, 3)])
|
||||
|
||||
|
||||
def test_dynamic_sd_clamps_k_to_runtime_max():
|
||||
dynamic_sd_lookup = _make_lookup(
|
||||
[(1, 256, 4)],
|
||||
runtime_num_speculative_tokens=3,
|
||||
)
|
||||
|
||||
assert dynamic_sd_lookup[1] == 3
|
||||
assert dynamic_sd_lookup[256] == 3
|
||||
|
||||
|
||||
def test_dynamic_sd_rejects_invalid_schedule_entry():
|
||||
with pytest.raises(ValueError, match="3-item sequence"):
|
||||
_make_lookup([(1, 16, 3), (32, 64)]) # type: ignore[list-item]
|
||||
|
||||
|
||||
def test_dynamic_sd_rejects_overlapping_ranges():
|
||||
with pytest.raises(ValueError, match="non-overlapping and sorted"):
|
||||
_make_lookup([(1, 16, 3), (16, 32, 2)])
|
||||
|
||||
|
||||
def test_dynamic_sd_rejects_negative_k():
|
||||
with pytest.raises(ValueError, match="values must be >= 0"):
|
||||
_make_lookup([(1, 16, -1)])
|
||||
|
||||
|
||||
def test_dynamic_sd_rejects_empty_schedule():
|
||||
with pytest.raises(ValueError, match="must not be empty"):
|
||||
_make_lookup([])
|
||||
|
||||
|
||||
def test_dynamic_sd_requires_schedule_config():
|
||||
with pytest.raises(
|
||||
ValueError, match="num_speculative_tokens_per_batch_size is required"
|
||||
):
|
||||
build_dynamic_sd_schedule_lookup(
|
||||
None,
|
||||
vllm_max_batch_size=256,
|
||||
vllm_num_speculative_tokens=3,
|
||||
)
|
||||
|
||||
|
||||
def test_dynamic_sd_lookup_rejects_invalid_batch_size_queries():
|
||||
dynamic_sd_lookup = _make_lookup([(1, 256, 3)])
|
||||
|
||||
assert dynamic_sd_lookup[0] == 0
|
||||
with pytest.raises(IndexError):
|
||||
_ = dynamic_sd_lookup[257]
|
||||
|
||||
|
||||
def test_scheduler_initializes_dynamic_sd_lookup_from_speculative_config():
|
||||
scheduler = _make_scheduler_with_dynamic_sd(
|
||||
[(1, 16, 3), (64, 128, 2), (256, 4096, 0)],
|
||||
runtime_num_speculative_tokens=3,
|
||||
)
|
||||
|
||||
assert scheduler.dynamic_sd_lookup is not None
|
||||
assert scheduler.num_spec_tokens == 3
|
||||
|
||||
|
||||
def test_scheduler_uses_dsd_k_based_on_number_of_scheduled_requests():
|
||||
test_cases = [
|
||||
(4, 3),
|
||||
(64, 2),
|
||||
(256, 0),
|
||||
]
|
||||
|
||||
for num_requests, expected_k in test_cases:
|
||||
scheduler = _make_scheduler_with_dynamic_sd(
|
||||
[(1, 16, 3), (64, 128, 2), (256, 4096, 0)],
|
||||
max_num_seqs=num_requests,
|
||||
max_num_batched_tokens=num_requests * 10,
|
||||
runtime_num_speculative_tokens=3,
|
||||
)
|
||||
output = _add_requests_and_schedule(scheduler, num_requests)
|
||||
|
||||
assert len(output.num_scheduled_tokens) == num_requests
|
||||
assert output.num_spec_tokens_to_schedule == expected_k
|
||||
|
||||
|
||||
def test_scheduler_clamps_dsd_k_to_runtime_num_speculative_tokens():
|
||||
scheduler = _make_scheduler_with_dynamic_sd(
|
||||
[(1, 256, 5)],
|
||||
max_num_seqs=16,
|
||||
max_num_batched_tokens=160,
|
||||
runtime_num_speculative_tokens=3,
|
||||
)
|
||||
output = _add_requests_and_schedule(scheduler, 16)
|
||||
|
||||
assert len(output.num_scheduled_tokens) == 16
|
||||
assert output.num_spec_tokens_to_schedule == 3
|
||||
|
||||
|
||||
def test_scheduler_falls_back_to_static_k_when_dsd_not_configured():
|
||||
scheduler = create_scheduler(
|
||||
max_num_seqs=4,
|
||||
max_num_batched_tokens=40,
|
||||
num_speculative_tokens=3,
|
||||
)
|
||||
output = _add_requests_and_schedule(scheduler, 4)
|
||||
|
||||
assert scheduler.dynamic_sd_lookup is None
|
||||
assert output.num_spec_tokens_to_schedule == 3
|
||||
|
||||
|
||||
def test_scheduler_uses_static_k_when_no_requests_are_scheduled():
|
||||
scheduler = _make_scheduler_with_dynamic_sd(
|
||||
[(1, 16, 3), (64, 128, 2), (256, 4096, 0)],
|
||||
runtime_num_speculative_tokens=3,
|
||||
)
|
||||
output = scheduler.schedule()
|
||||
|
||||
assert len(output.num_scheduled_tokens) == 0
|
||||
assert output.num_spec_tokens_to_schedule == 3
|
||||
|
||||
|
||||
def test_scheduler_rejects_bad_dsd_config_at_construction():
|
||||
with pytest.raises(ValueError, match="must start at 1"):
|
||||
_make_scheduler_with_dynamic_sd([(2, 16, 3)])
|
||||
|
||||
|
||||
def test_scheduler_passes_max_num_seqs_as_dsd_runtime_batch_limit():
|
||||
scheduler = _make_scheduler_with_dynamic_sd(
|
||||
[(1, 16, 3), (64, 128, 2), (256, 4096, 0)],
|
||||
max_num_seqs=16,
|
||||
max_num_batched_tokens=160,
|
||||
runtime_num_speculative_tokens=3,
|
||||
)
|
||||
output = _add_requests_and_schedule(scheduler, 16)
|
||||
|
||||
assert scheduler.dynamic_sd_lookup is not None
|
||||
assert len(scheduler.dynamic_sd_lookup) == 17
|
||||
assert len(output.num_scheduled_tokens) == 16
|
||||
assert output.num_spec_tokens_to_schedule == 3
|
||||
@@ -969,6 +969,7 @@ def test_propose(method, attn_backend, num_speculative_tokens, monkeypatch):
|
||||
proposer.draft_attn_groups = [mock_attn_group]
|
||||
|
||||
result = proposer.propose(
|
||||
num_speculative_tokens=num_speculative_tokens,
|
||||
target_token_ids=target_token_ids,
|
||||
target_positions=target_positions,
|
||||
target_hidden_states=target_hidden_states,
|
||||
@@ -1071,6 +1072,7 @@ def test_propose_stores_probabilistic_draft_probs(monkeypatch):
|
||||
sampling_metadata.all_greedy = False
|
||||
|
||||
result = proposer.propose(
|
||||
num_speculative_tokens=num_speculative_tokens,
|
||||
target_token_ids=torch.randint(0, vocab_size, (total_tokens,), device=device),
|
||||
target_positions=torch.cat(
|
||||
[
|
||||
|
||||
@@ -255,6 +255,7 @@ def test_propose():
|
||||
|
||||
# Call propose
|
||||
draft_tokens = proposer.propose(
|
||||
num_speculative_tokens=1,
|
||||
sampled_token_ids=sampled_token_ids,
|
||||
target_hidden_states=target_hidden_states,
|
||||
common_attn_metadata=common_attn_metadata,
|
||||
@@ -321,6 +322,7 @@ def test_propose_different_layer_counts(num_hidden_layers):
|
||||
).unsqueeze(-1)
|
||||
|
||||
draft_tokens = proposer.propose(
|
||||
num_speculative_tokens=1,
|
||||
sampled_token_ids=sampled_token_ids,
|
||||
target_hidden_states=target_hidden_states,
|
||||
common_attn_metadata=common_attn_metadata,
|
||||
|
||||
@@ -205,6 +205,7 @@ def test_mtp_propose(num_speculative_tokens, monkeypatch):
|
||||
|
||||
# Run propose
|
||||
result = proposer.propose(
|
||||
num_speculative_tokens=num_speculative_tokens,
|
||||
target_token_ids=target_token_ids,
|
||||
target_positions=target_positions,
|
||||
target_hidden_states=target_hidden_states,
|
||||
|
||||
@@ -81,6 +81,7 @@ def test_ngram_proposer():
|
||||
# No match.
|
||||
token_ids_cpu = np.array([[1, 2, 3, 4, 5]])
|
||||
result = get_ngram_proposer(min_n=2, max_n=2, k=2).propose(
|
||||
num_speculative_tokens=2,
|
||||
sampled_token_ids=[[0]],
|
||||
num_tokens_no_spec=np.array([len(c) for c in token_ids_cpu]),
|
||||
token_ids_cpu=token_ids_cpu,
|
||||
@@ -90,6 +91,7 @@ def test_ngram_proposer():
|
||||
# No match for 4-gram.
|
||||
token_ids_cpu = np.array([[1, 2, 3, 4, 1, 2, 3]])
|
||||
result = get_ngram_proposer(min_n=4, max_n=4, k=2).propose(
|
||||
num_speculative_tokens=2,
|
||||
sampled_token_ids=[[0]],
|
||||
num_tokens_no_spec=np.array([len(c) for c in token_ids_cpu]),
|
||||
token_ids_cpu=token_ids_cpu,
|
||||
@@ -99,6 +101,7 @@ def test_ngram_proposer():
|
||||
# No match for 4-gram but match for 3-gram.
|
||||
token_ids_cpu = np.array([[1, 2, 3, 4, 1, 2, 3]])
|
||||
result = get_ngram_proposer(min_n=3, max_n=4, k=2).propose(
|
||||
num_speculative_tokens=2,
|
||||
sampled_token_ids=[[0]],
|
||||
num_tokens_no_spec=np.array([len(c) for c in token_ids_cpu]),
|
||||
token_ids_cpu=token_ids_cpu,
|
||||
@@ -109,6 +112,7 @@ def test_ngram_proposer():
|
||||
# In this case, the proposer should return the 4-gram match.
|
||||
token_ids_cpu = np.array([[2, 3, 4, 5, 1, 2, 3, 4, 1, 2, 3, 4]])
|
||||
result = get_ngram_proposer(min_n=3, max_n=4, k=2).propose(
|
||||
num_speculative_tokens=2,
|
||||
sampled_token_ids=[[0]],
|
||||
num_tokens_no_spec=np.array([len(c) for c in token_ids_cpu]),
|
||||
token_ids_cpu=token_ids_cpu,
|
||||
@@ -118,6 +122,7 @@ def test_ngram_proposer():
|
||||
# Match for 2-gram and 3-gram, but not 4-gram.
|
||||
token_ids_cpu = np.array([[3, 4, 5, 2, 3, 4, 1, 2, 3, 4]])
|
||||
result = get_ngram_proposer(min_n=2, max_n=4, k=2).propose(
|
||||
num_speculative_tokens=2,
|
||||
sampled_token_ids=[[0]],
|
||||
num_tokens_no_spec=np.array([len(c) for c in token_ids_cpu]),
|
||||
token_ids_cpu=token_ids_cpu,
|
||||
@@ -127,6 +132,7 @@ def test_ngram_proposer():
|
||||
# Multiple 3-gram matched, but always pick the first one.
|
||||
token_ids_cpu = np.array([[1, 2, 3, 100, 1, 2, 3, 200, 1, 2, 3, 300, 1, 2, 3]])
|
||||
result = get_ngram_proposer(min_n=3, max_n=3, k=2).propose(
|
||||
num_speculative_tokens=2,
|
||||
sampled_token_ids=[[0]],
|
||||
num_tokens_no_spec=np.array([len(c) for c in token_ids_cpu]),
|
||||
token_ids_cpu=token_ids_cpu,
|
||||
@@ -136,6 +142,7 @@ def test_ngram_proposer():
|
||||
# check empty input
|
||||
token_ids_cpu = np.array([[]])
|
||||
result = get_ngram_proposer(min_n=2, max_n=2, k=2).propose(
|
||||
num_speculative_tokens=2,
|
||||
sampled_token_ids=[[0]],
|
||||
num_tokens_no_spec=np.array([len(c) for c in token_ids_cpu]),
|
||||
token_ids_cpu=token_ids_cpu,
|
||||
@@ -147,6 +154,7 @@ def test_ngram_proposer():
|
||||
# second request has 3 tokens and no match. Padded with -1 for max len 5
|
||||
token_ids_cpu = np.array([[1, 2, 3, 1, 2], [4, 5, 6, -1, -1]])
|
||||
result = get_ngram_proposer(min_n=2, max_n=2, k=2).propose(
|
||||
num_speculative_tokens=2,
|
||||
sampled_token_ids=[[0], [1]],
|
||||
num_tokens_no_spec=np.array([5, 3]),
|
||||
token_ids_cpu=token_ids_cpu,
|
||||
@@ -166,6 +174,7 @@ def test_ngram_proposer():
|
||||
num_tokens_no_spec = np.array([5, 3, 5], dtype=np.int32)
|
||||
sampled_token_ids = [[2], [], [8]] # Empty list for request 1 simulates prefill
|
||||
result = proposer.propose(
|
||||
num_speculative_tokens=2,
|
||||
sampled_token_ids=sampled_token_ids,
|
||||
num_tokens_no_spec=num_tokens_no_spec,
|
||||
token_ids_cpu=token_ids_cpu,
|
||||
@@ -195,6 +204,7 @@ def test_ngram_proposer():
|
||||
input_2[:3] = [4, 5, 6]
|
||||
token_ids_cpu = np.array([input_1, input_2])
|
||||
result = ngram_proposer.propose(
|
||||
num_speculative_tokens=2,
|
||||
sampled_token_ids=[[0], [1]],
|
||||
num_tokens_no_spec=np.array([len(input_1), 3]),
|
||||
token_ids_cpu=token_ids_cpu,
|
||||
|
||||
Reference in New Issue
Block a user