[Spec Decode] DSpark confidence-scheduled verification (#47808)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com> Signed-off-by: Benjamin Chislett <chislett.ben@gmail.com> Signed-off-by: Lucas Wilkinson <wilkinson.lucas@gmail.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Benjamin Chislett <chislett.ben@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
Claude Opus 5
Benjamin Chislett
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Nick Hill
OpenAI Codex
parent
fe889ac925
commit
7f7a32cfec
@@ -0,0 +1,170 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import numpy as np
|
||||
|
||||
from vllm.v1.worker.gpu.async_utils import StepTimingSample
|
||||
from vllm.v1.worker.gpu.spec_decode.adaptive_verification import (
|
||||
AdaptiveVerificationManager,
|
||||
)
|
||||
|
||||
|
||||
def make_manager(
|
||||
confidences: np.ndarray, verify_cost_ms: np.ndarray
|
||||
) -> AdaptiveVerificationManager:
|
||||
num_reqs, num_steps = confidences.shape
|
||||
manager = AdaptiveVerificationManager.__new__(AdaptiveVerificationManager)
|
||||
manager.num_speculative_steps = num_steps
|
||||
manager._stale_confidences = [SimpleNamespace(np=confidences)]
|
||||
manager._stale_idx = 0
|
||||
manager.req_states = SimpleNamespace(
|
||||
req_id_to_index={"low": 0, "high": 1},
|
||||
num_computed_tokens_np=np.ones(num_reqs, dtype=np.int32),
|
||||
prefill_len=SimpleNamespace(np=np.ones(num_reqs, dtype=np.int32)),
|
||||
)
|
||||
manager.cost_tables = (np.zeros(num_reqs + 1), verify_cost_ms)
|
||||
manager._max_total_logits = 1 << 30
|
||||
manager.num_bonus_tokens = 1
|
||||
return manager
|
||||
|
||||
|
||||
def test_budget_stops_where_marginal_drafts_stop_paying_for_themselves():
|
||||
# Verification is cheap up to two extra tokens, then jumps 100x; only the
|
||||
# highest-confidence draft is worth the cheap slot.
|
||||
manager = make_manager(
|
||||
np.array([[0.1, 0.1], [0.9, 0.9]], dtype=np.float32),
|
||||
np.array([1.0, 1.0, 1.0, 1.0, 100.0, 100.0, 100.0]),
|
||||
)
|
||||
|
||||
manager.get_num_tokens(
|
||||
{"low": 3, "high": 3},
|
||||
{"low": [1, 2], "high": [3, 4]},
|
||||
)
|
||||
valid_drafts, num_non_draft_tokens, draft_budget = manager._batch_budget
|
||||
|
||||
assert draft_budget == 1
|
||||
assert valid_drafts == {"low": 2, "high": 2}
|
||||
assert num_non_draft_tokens == {"low": 1, "high": 1}
|
||||
|
||||
|
||||
def test_profiled_batches_seed_cost_curves_via_consumer():
|
||||
manager = AdaptiveVerificationManager.__new__(AdaptiveVerificationManager)
|
||||
manager.req_states = SimpleNamespace(max_num_batched_tokens=4096, max_num_reqs=64)
|
||||
manager.num_speculative_steps = 7
|
||||
manager.num_bonus_tokens = 1
|
||||
curves: dict[str, list[tuple[int, float]]] = {}
|
||||
manager.set_cost_curves = lambda draft, verify: curves.update(
|
||||
draft=draft, verify=verify
|
||||
)
|
||||
|
||||
timings = [
|
||||
StepTimingSample(
|
||||
forward_ms=float(batch["num_tokens"]),
|
||||
drafter_ms=1.0,
|
||||
num_target_tokens=batch["num_tokens"],
|
||||
num_reqs=batch["num_tokens"] // 8,
|
||||
# Only the captured sizes replay a graph; the tail sizes run eager.
|
||||
full_cudagraph=batch["num_tokens"] <= 1024,
|
||||
)
|
||||
for batch in manager.batches_to_profile([8, 1024])
|
||||
]
|
||||
manager.set_initial_cost_curves(timings)
|
||||
|
||||
# Tail beyond the last capture size: 1.5x then doubling to the max.
|
||||
assert curves["verify"] == [
|
||||
(8, 8.0),
|
||||
(1024, 1024.0),
|
||||
(1536, 1536.0),
|
||||
(2048, 2048.0),
|
||||
(4096, 4096.0),
|
||||
]
|
||||
# Eager batches must not contribute to the draft curve: keyed by request
|
||||
# count they would land inside the captured range and, once made monotonic,
|
||||
# smear that eager cost across every larger request count.
|
||||
assert curves["draft"] == [(1, 1.0), (128, 1.0)]
|
||||
|
||||
|
||||
def test_compact_batch_preserves_totals_and_bounds():
|
||||
# The CPU placeholder layout must keep the batch total equal to the GPU
|
||||
# total and every verification row within decode_query_len, or downstream
|
||||
# CPU metadata desyncs from the reallocated GPU boundaries.
|
||||
manager = make_manager(
|
||||
np.array([[0.9, 0.9], [0.9, 0.9], [1.0, 1.0]], dtype=np.float32),
|
||||
np.array([1.0] * 44 + [100.0] * 3),
|
||||
)
|
||||
manager.req_states.req_id_to_index["prefill"] = 2
|
||||
manager.req_states.num_computed_tokens_np = np.zeros(3, dtype=np.int32)
|
||||
manager.req_states.prefill_len.np = np.array([0, 0, 60], dtype=np.int32)
|
||||
num_tokens = manager.get_num_tokens(
|
||||
{"low": 3, "high": 3, "prefill": 40},
|
||||
{"low": [1, 2], "high": [3, 4]},
|
||||
)
|
||||
scheduled = np.array([3, 3, 40], dtype=np.int32)
|
||||
drafts = np.array([2, 2, 0], dtype=np.int32)
|
||||
cu_num_logits_np = np.array([0, 3, 6, 7], dtype=np.int32)
|
||||
compacted, _ = manager.compact_batch(drafts, scheduled, cu_num_logits_np)
|
||||
|
||||
assert int(compacted.sum()) == num_tokens
|
||||
num_steps = manager.num_speculative_steps
|
||||
assert (compacted[:2] <= 1 + num_steps).all()
|
||||
assert compacted[2] == 40
|
||||
|
||||
|
||||
def test_budget_caps_at_one_rejection_sampler_chunk():
|
||||
# The chunked verification path cannot address the compacted logits
|
||||
# layout, so the budget must keep total logits within a single chunk.
|
||||
manager = make_manager(
|
||||
np.array([[0.9, 0.9], [0.9, 0.9]], dtype=np.float32),
|
||||
np.ones(7),
|
||||
)
|
||||
manager._max_total_logits = 3 # 2 bonus logits + at most 1 draft
|
||||
manager.get_num_tokens(
|
||||
{"low": 3, "high": 3},
|
||||
{"low": [1, 2], "high": [3, 4]},
|
||||
)
|
||||
_, _, draft_budget = manager._batch_budget
|
||||
assert draft_budget <= 1
|
||||
|
||||
|
||||
def test_zero_budget_rebuilds_cpu_cu_num_logits():
|
||||
# When one bonus row per request already overflows a verification chunk, the
|
||||
# budget clamps to zero but the batch still needs chunking. Every capacity is
|
||||
# zeroed on device, so the CPU can name that layout exactly -- and must, since
|
||||
# _iter_request_chunks slices the compacted logits with these offsets.
|
||||
#
|
||||
# The third request is a chunked prefill (no drafts, still mid-prompt). The
|
||||
# runner gives *every* request num_bonus_tokens logits rows regardless
|
||||
# (num_logits = num_draft_tokens_per_req + num_bonus_tokens), so the rebuilt
|
||||
# offsets stay uniform rather than skipping non-verification rows.
|
||||
manager = make_manager(
|
||||
np.array([[0.9, 0.9], [0.9, 0.9], [1.0, 1.0]], dtype=np.float32),
|
||||
np.ones(64),
|
||||
)
|
||||
manager.req_states.req_id_to_index["prefill"] = 2
|
||||
manager.req_states.num_computed_tokens_np = np.zeros(3, dtype=np.int32)
|
||||
manager.req_states.prefill_len.np = np.array([0, 0, 60], dtype=np.int32)
|
||||
manager._max_total_logits = 2 # < 3 requests * 1 bonus token
|
||||
|
||||
manager.get_num_tokens(
|
||||
{"low": 3, "high": 3, "prefill": 40},
|
||||
{"low": [1, 2], "high": [3, 4]},
|
||||
)
|
||||
_, _, draft_budget = manager._batch_budget
|
||||
assert draft_budget == 0
|
||||
|
||||
scheduled = np.array([3, 3, 40], dtype=np.int32)
|
||||
drafts = np.array([2, 2, 0], dtype=np.int32)
|
||||
scheduled_cu_num_logits = np.array([0, 3, 6, 7], dtype=np.int32)
|
||||
compacted, cu_num_logits_np = manager.compact_batch(
|
||||
drafts, scheduled, scheduled_cu_num_logits
|
||||
)
|
||||
|
||||
# One bonus row per request, matching cumsum(capacities + num_bonus_tokens)
|
||||
# with every capacity zeroed -- the prefill row included.
|
||||
expected = np.arange(4, dtype=np.int32) * manager.num_bonus_tokens
|
||||
assert np.array_equal(cu_num_logits_np, expected)
|
||||
assert cu_num_logits_np.dtype == scheduled_cu_num_logits.dtype
|
||||
# The prefill keeps its scheduled tokens; only drafts are dropped.
|
||||
assert np.array_equal(compacted, np.array([1, 1, 40], dtype=np.int32))
|
||||
Reference in New Issue
Block a user