[Spec Decode] DSpark confidence-scheduled verification (#47808)

Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
Signed-off-by: Benjamin Chislett <chislett.ben@gmail.com>
Signed-off-by: Lucas Wilkinson <wilkinson.lucas@gmail.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: Benjamin Chislett <chislett.ben@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Lucas Wilkinson
2026-08-12 12:27:48 -07:00
committed by GitHub
co-authored by OpenAI Codex Claude Opus 5 Benjamin Chislett mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Nick Hill OpenAI Codex
parent fe889ac925
commit 7f7a32cfec
40 changed files with 1643 additions and 111 deletions
+134 -16
View File
@@ -72,7 +72,11 @@ from vllm.v1.outputs import (
from vllm.v1.worker.block_table import get_block_table_width
from vllm.v1.worker.cp_utils import check_attention_cp_compatibility
from vllm.v1.worker.gpu import pcp_manager as pcp
from vllm.v1.worker.gpu.async_utils import AsyncOutput, AsyncPoolingOutput
from vllm.v1.worker.gpu.async_utils import (
AsyncOutput,
AsyncPoolingOutput,
StepTimingCollector,
)
from vllm.v1.worker.gpu.attn_utils import (
build_slot_mappings_by_layer,
get_kv_cache_spec,
@@ -101,6 +105,7 @@ from vllm.v1.worker.gpu.input_batch import (
post_update_num_computed_tokens,
prepare_pos_seq_lens,
prepare_prefill_inputs,
set_dummy_context,
)
from vllm.v1.worker.gpu.kv_connector import (
NO_OP_KV_CONNECTOR,
@@ -123,10 +128,17 @@ from vllm.v1.worker.gpu.sample.prompt_logprob import PromptLogprobsWorker
from vllm.v1.worker.gpu.sample.sampler import Sampler
from vllm.v1.worker.gpu.shutdown import free_before_shutdown
from vllm.v1.worker.gpu.spec_decode import init_speculator
from vllm.v1.worker.gpu.spec_decode.adaptive_verification import (
AdaptiveVerificationManager,
maybe_create_adaptive_verification_manager,
)
from vllm.v1.worker.gpu.spec_decode.eagle.eagle3_utils import (
set_eagle3_aux_hidden_state_layers,
)
from vllm.v1.worker.gpu.spec_decode.rejection_sampler import RejectionSampler
from vllm.v1.worker.gpu.spec_decode.rejection_sampler import (
RejectionSampler,
get_max_chunk_logits,
)
from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator
from vllm.v1.worker.gpu.spec_decode.utils import DraftTokensHandler
from vllm.v1.worker.gpu.states import RequestState
@@ -252,6 +264,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
and self.speculative_config.use_multi_module_mtp()
else 1
)
self.step_timing = StepTimingCollector()
# General request states.
self.req_states = RequestState(
max_num_reqs=self.max_num_reqs,
@@ -262,6 +277,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
device=self.device,
num_prefill_lookahead=num_prefill_lookahead,
)
self.adaptive_verification: AdaptiveVerificationManager | None = None
self.input_buffers = InputBuffers(
max_num_reqs=self.max_num_reqs,
max_num_tokens=self.max_num_tokens,
@@ -401,6 +417,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
max_num_logits=self.max_num_reqs * self.decode_query_len,
vocab_size=self.vocab_size,
device=self.device,
mask_stride=self.decode_query_len,
)
if self.is_pooling_model and self.is_last_pp_rank:
@@ -510,6 +527,20 @@ class GPUModelRunner(LoRAModelRunnerMixin):
attn_cg_support = attn_cg_support.narrow(
*self.model_state.get_additional_cg_support()
)
# The speculator clears the flag at load time when the checkpoint has
# no confidence head, so it holds the effective value.
self.adaptive_verification = maybe_create_adaptive_verification_manager(
enable_adaptive_verification=getattr(
self.speculator, "enable_adaptive_verification", False
),
attn_groups=self.attn_groups,
attn_cg_support=attn_cg_support,
req_states=self.req_states,
query_start_loc=self.input_buffers.query_start_loc,
num_bonus_tokens=self.model_state.num_new_sampled_tokens_per_step,
max_total_logits=get_max_chunk_logits(self.vocab_size),
)
self.block_tables = BlockTables(
block_sizes=block_sizes,
max_num_reqs=self.max_num_reqs,
@@ -532,6 +563,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
initialize_mamba_ssu_backend(
self.vllm_config.mamba_config, self.kv_cache_config
)
if self.adaptive_verification is not None:
self.compilation_config.cudagraph_mode = CUDAGraphMode.FULL_AND_PIECEWISE
cudagraph_mode = self.compilation_config.resolve_cudagraph_mode_and_sizes(
attn_cg_support.min_cg_support,
attn_cg_support.min_cg_attn_backend,
@@ -547,6 +580,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
cudagraph_mode,
decode_query_len=self.decode_query_len,
lora_capture_cases=self.lora_capture_cases,
varlen_decode=self.adaptive_verification is not None,
)
check_attention_cp_compatibility(self.vllm_config)
if isinstance(self.speculator, DraftModelSpeculator):
@@ -594,6 +628,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
*args,
skip_attn: bool = False,
uniform_decode: bool = False,
context_len: int = 0,
skip_eplb: bool = False,
is_profile: bool = False,
**kwargs,
@@ -655,6 +690,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
dummy_run=True,
skip_attn_for_dummy_run=skip_attn,
is_profile=is_profile,
context_len=context_len,
)
self.kv_connector.set_disabled(False)
@@ -670,9 +706,12 @@ class GPUModelRunner(LoRAModelRunnerMixin):
aux_hidden_states = self.execute_model_state.aux_hidden_states
self.execute_model_state = None
self.step_timing.forward_end()
# dummy run the eagle speculator's propose to ensure DP/EP sync.
if self.speculator is not None:
assert self.sampler is not None
self.step_timing.drafter_start()
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None
if self.speculator.supports_mm_inputs:
mm_inputs = (
@@ -713,6 +752,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
mm_inputs=mm_inputs,
is_profile=is_profile,
)
self.step_timing.drafter_end()
assert hidden_states is not None # Last PP rank always has hidden_states
sample_hidden_states = hidden_states[input_batch.logits_indices]
@@ -830,6 +870,13 @@ class GPUModelRunner(LoRAModelRunnerMixin):
)
if self.speculator is not None:
self.speculator.capture()
if self.adaptive_verification is not None:
with self.step_timing.collect() as timings:
for batch in self.adaptive_verification.batches_to_profile(
self.cudagraph_manager.captured_token_counts()
):
self._dummy_run(**batch)
self.adaptive_verification.set_initial_cost_curves(timings)
end_time = time.perf_counter()
end_free_gpu_memory = torch.accelerator.get_memory_info()[0]
@@ -906,6 +953,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
max_tokens=sampling_params.max_tokens if sampling_params else 1, # type: ignore[arg-type]
)
req_index = self.req_states.req_id_to_index[req_id]
if self.adaptive_verification is not None:
self.adaptive_verification.add_request(req_index)
if self.pooling_runner is not None:
assert new_req_data.pooling_params is not None
@@ -994,8 +1043,12 @@ class GPUModelRunner(LoRAModelRunnerMixin):
num_reqs, num_toks, max_query_len, has_prefill=False
)
draft_tokens = scheduler_output.scheduled_spec_decode_tokens
# batch_idx -> req_id
req_ids = sort_batch_req_ids(num_tokens_per_req, self.decode_query_len)
req_ids = sort_batch_req_ids(
num_tokens_per_req, draft_tokens, self.decode_query_len
)
numtoks_iter = map(num_tokens_per_req.__getitem__, req_ids)
num_scheduled_tokens = np.fromiter(numtoks_iter, dtype=np.int32, count=num_reqs)
@@ -1007,9 +1060,15 @@ class GPUModelRunner(LoRAModelRunnerMixin):
]
is_prefilling_np = num_computed_prefill_tokens_np < prefill_len_np
if self.adaptive_verification is not None and draft_tokens:
num_toks = self.adaptive_verification.get_num_tokens(
num_tokens_per_req, draft_tokens
)
batch_state = BatchReqState(
req_ids=req_ids,
num_scheduled_tokens=num_scheduled_tokens,
num_tokens=num_toks,
idx_mapping_np=idx_mapping_np,
prefill_len_np=prefill_len_np,
num_computed_prefill_tokens_np=num_computed_prefill_tokens_np,
@@ -1026,7 +1085,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
batch_req_state: "BatchReqState",
batch_desc: BatchExecutionDescriptor,
) -> InputBatch:
num_tokens = scheduler_output.total_num_scheduled_tokens
num_tokens = batch_req_state.num_tokens
num_tokens_after_padding = batch_desc.num_tokens
assert num_tokens > 0
if envs.VLLM_MOE_SKIP_PADDING:
@@ -1072,9 +1131,20 @@ class GPUModelRunner(LoRAModelRunnerMixin):
np.cumsum(num_logits, out=cu_num_logits_np[1:])
cu_num_logits = async_copy_to_gpu(cu_num_logits_np, device=self.device)
max_expand_len = self.decode_query_len
expanded_idx_mapping, expanded_local_pos = expand_idx_mapping(
idx_mapping, total_num_logits, cu_num_logits, max_expand_len
adaptive_verification = (
self.adaptive_verification if num_draft_tokens_per_req is not None else None
)
num_scheduled_tokens_upper_bound = num_scheduled_tokens_np
if adaptive_verification is not None:
# num_scheduled_tokens represents the draft budget evenly distributed across
# all verification requests, `reallocate_drafts` will unevenly assign the
# draft budget to requests on the GPU side only.
num_scheduled_tokens_np, cu_num_logits_np = (
adaptive_verification.compact_batch(
num_draft_tokens_per_req,
num_scheduled_tokens_np,
cu_num_logits_np,
)
)
# Get query_start_loc.
@@ -1086,9 +1156,19 @@ class GPUModelRunner(LoRAModelRunnerMixin):
# Pad for full CUDA graph mode.
# Some attention backends like FA3 require query_start_loc to be non-decreasing.
query_start_loc_np[num_reqs + 1 :] = num_tokens
async_copy_to_gpu(query_start_loc_np, out=self.input_buffers.query_start_loc)
query_start_loc = self.input_buffers.query_start_loc
async_copy_to_gpu(query_start_loc_np, out=query_start_loc)
if adaptive_verification is not None:
cu_num_logits, query_start_loc, total_num_draft_tokens = (
adaptive_verification.reallocate_drafts(req_ids, idx_mapping)
)
total_num_logits = num_reqs * num_bonus_tokens + total_num_draft_tokens
if draft_tokens:
expanded_idx_mapping, expanded_local_pos = expand_idx_mapping(
idx_mapping, total_num_logits, cu_num_logits, self.decode_query_len
)
query_start_loc_np = query_start_loc_np[: num_reqs_padded + 1]
query_start_loc = self.input_buffers.query_start_loc[: num_reqs_padded + 1]
query_start_loc = query_start_loc[: num_reqs_padded + 1]
# Get prefill tokens if any.
if batch_req_state.has_prefill:
@@ -1145,7 +1225,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
seq_lens_cpu_upper_bound_np = np.zeros(num_reqs_padded, dtype=np.int32)
np.add(
num_computed_tokens_np,
num_scheduled_tokens_np,
num_scheduled_tokens_upper_bound,
out=seq_lens_cpu_upper_bound_np[:num_reqs],
)
seq_lens_cpu_upper_bound = torch.from_numpy(seq_lens_cpu_upper_bound_np)
@@ -1168,7 +1248,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
idx_mapping_np=idx_mapping_np,
expanded_idx_mapping=expanded_idx_mapping,
expanded_local_pos=expanded_local_pos,
num_scheduled_tokens=num_scheduled_tokens_np,
num_scheduled_tokens=num_scheduled_tokens_upper_bound,
num_tokens=num_tokens,
num_tokens_after_padding=num_tokens_after_padding,
num_draft_tokens=total_num_draft_tokens,
@@ -1192,6 +1272,11 @@ class GPUModelRunner(LoRAModelRunnerMixin):
cu_num_logits_np=cu_num_logits_np,
has_structured_output_reqs=scheduler_output.has_structured_output_requests,
prompt_lens=prompt_lens,
max_query_len=(
int(num_scheduled_tokens_upper_bound.max())
if adaptive_verification is not None
else None
),
)
return pcp.maybe_partition_pcp_batch(self.pcp_manager, input_batch)
@@ -1298,6 +1383,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
dummy_run: bool = False,
skip_attn_for_dummy_run: bool = False,
is_profile: bool = False,
context_len: int = 0,
) -> ModelRunnerOutput | IntermediateTensors | None:
if not dummy_run:
# Update the request states.
@@ -1315,9 +1401,12 @@ class GPUModelRunner(LoRAModelRunnerMixin):
# Get batch descriptor and sync across DP ranks.
num_reqs = len(scheduler_output.num_scheduled_tokens)
num_toks = scheduler_output.total_num_scheduled_tokens
max_query_len = max(scheduler_output.num_scheduled_tokens.values())
batch_req_state, uniform_tok_count = self.gather_batch_req_state(
scheduler_output, dummy_run
)
if batch_req_state is not None:
num_toks = batch_req_state.num_tokens
num_active_loras = 0
if self.lora_config:
@@ -1340,6 +1429,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
uniform_tok_count,
self.dp_size,
self.dp_rank,
max_query_len=max_query_len,
need_eager=is_profile or skip_compiled,
num_active_loras=num_active_loras,
)
@@ -1378,13 +1468,23 @@ class GPUModelRunner(LoRAModelRunnerMixin):
self._set_active_loras(*lora_inputs)
else:
# No actual tokens to run. A dummy run for DP or memory profiling.
dummy_num_reqs = batch_desc.num_reqs or num_reqs
input_batch = InputBatch.make_dummy(
batch_desc.num_reqs or num_reqs,
dummy_num_reqs,
batch_desc.num_tokens,
self.input_buffers,
max_query_len=batch_desc.max_query_len,
)
if not skip_attn_for_dummy_run:
block_tables, slot_mappings = self.prepare_dummy_attn(input_batch)
if context_len:
set_dummy_context(
input_batch,
self.block_tables,
context_len,
self.kv_cache_config.num_blocks,
self.max_model_len,
)
else:
assert batch_desc.cg_mode != CUDAGraphMode.FULL, (
"Attention metadata must be prepared for dummy runs when using "
@@ -1488,6 +1588,11 @@ class GPUModelRunner(LoRAModelRunnerMixin):
# Update the EPLB meta.
self.eplb.prepare_forward(self.model_config, input_batch.num_tokens)
self.step_timing.record_batch(
input_batch, batch_desc.cg_mode == CUDAGraphMode.FULL
)
self.step_timing.forward_start()
# Run model.
if batch_desc.cg_mode == CUDAGraphMode.FULL:
# Use explicit cudagraph replay for FULL mode.
@@ -1698,6 +1803,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
mm_inputs=mm_inputs,
)
self.req_states.draft_tokens[input_batch.idx_mapping] = draft_tokens
if self.adaptive_verification is not None:
self.adaptive_verification.record_confidences(
self.speculator.draft_token_confidence_probs, input_batch
)
if self.num_speculative_steps > 0:
# Spec-decode and diffusion LLMs both use draft tokens but the latter does
@@ -1839,6 +1948,9 @@ class BatchReqState(NamedTuple):
req_ids: list[str]
num_scheduled_tokens: np.ndarray # [num_reqs]
# May be less than scheduler_output.total_num_scheduled_tokens:
# adaptive verification trims the draft budget before running.
num_tokens: int
idx_mapping_np: np.ndarray # [num_reqs]
prefill_len_np: np.ndarray # [num_reqs]
num_computed_prefill_tokens_np: np.ndarray # [num_reqs]
@@ -1847,9 +1959,15 @@ class BatchReqState(NamedTuple):
def sort_batch_req_ids(
num_tokens_per_req: dict[str, int], decode_query_len: int
num_tokens_per_req: dict[str, int],
draft_tokens: dict[str, list[int]],
decode_query_len: int,
) -> list[str]:
# Order decode -> short_extend -> prefill; split_decodes_and_prefills
# relies on uniform decodes (query_len == decode_query_len) leading.
key = lambda r: ((num := num_tokens_per_req[r]) != decode_query_len, num)
# Order verification/decode -> short_extend -> prefill;
# split_decodes_and_prefills relies on decode-like requests leading.
key = lambda r: (
not draft_tokens.get(r),
(num := num_tokens_per_req[r]) != decode_query_len,
num,
)
return sorted(num_tokens_per_req, key=key)