[Spec Decode] DSpark confidence-scheduled verification (#47808)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com> Signed-off-by: Benjamin Chislett <chislett.ben@gmail.com> Signed-off-by: Lucas Wilkinson <wilkinson.lucas@gmail.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Benjamin Chislett <chislett.ben@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
Claude Opus 5
Benjamin Chislett
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Nick Hill
OpenAI Codex
parent
fe889ac925
commit
7f7a32cfec
@@ -72,7 +72,11 @@ from vllm.v1.outputs import (
|
||||
from vllm.v1.worker.block_table import get_block_table_width
|
||||
from vllm.v1.worker.cp_utils import check_attention_cp_compatibility
|
||||
from vllm.v1.worker.gpu import pcp_manager as pcp
|
||||
from vllm.v1.worker.gpu.async_utils import AsyncOutput, AsyncPoolingOutput
|
||||
from vllm.v1.worker.gpu.async_utils import (
|
||||
AsyncOutput,
|
||||
AsyncPoolingOutput,
|
||||
StepTimingCollector,
|
||||
)
|
||||
from vllm.v1.worker.gpu.attn_utils import (
|
||||
build_slot_mappings_by_layer,
|
||||
get_kv_cache_spec,
|
||||
@@ -101,6 +105,7 @@ from vllm.v1.worker.gpu.input_batch import (
|
||||
post_update_num_computed_tokens,
|
||||
prepare_pos_seq_lens,
|
||||
prepare_prefill_inputs,
|
||||
set_dummy_context,
|
||||
)
|
||||
from vllm.v1.worker.gpu.kv_connector import (
|
||||
NO_OP_KV_CONNECTOR,
|
||||
@@ -123,10 +128,17 @@ from vllm.v1.worker.gpu.sample.prompt_logprob import PromptLogprobsWorker
|
||||
from vllm.v1.worker.gpu.sample.sampler import Sampler
|
||||
from vllm.v1.worker.gpu.shutdown import free_before_shutdown
|
||||
from vllm.v1.worker.gpu.spec_decode import init_speculator
|
||||
from vllm.v1.worker.gpu.spec_decode.adaptive_verification import (
|
||||
AdaptiveVerificationManager,
|
||||
maybe_create_adaptive_verification_manager,
|
||||
)
|
||||
from vllm.v1.worker.gpu.spec_decode.eagle.eagle3_utils import (
|
||||
set_eagle3_aux_hidden_state_layers,
|
||||
)
|
||||
from vllm.v1.worker.gpu.spec_decode.rejection_sampler import RejectionSampler
|
||||
from vllm.v1.worker.gpu.spec_decode.rejection_sampler import (
|
||||
RejectionSampler,
|
||||
get_max_chunk_logits,
|
||||
)
|
||||
from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator
|
||||
from vllm.v1.worker.gpu.spec_decode.utils import DraftTokensHandler
|
||||
from vllm.v1.worker.gpu.states import RequestState
|
||||
@@ -252,6 +264,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
and self.speculative_config.use_multi_module_mtp()
|
||||
else 1
|
||||
)
|
||||
|
||||
self.step_timing = StepTimingCollector()
|
||||
|
||||
# General request states.
|
||||
self.req_states = RequestState(
|
||||
max_num_reqs=self.max_num_reqs,
|
||||
@@ -262,6 +277,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
device=self.device,
|
||||
num_prefill_lookahead=num_prefill_lookahead,
|
||||
)
|
||||
self.adaptive_verification: AdaptiveVerificationManager | None = None
|
||||
self.input_buffers = InputBuffers(
|
||||
max_num_reqs=self.max_num_reqs,
|
||||
max_num_tokens=self.max_num_tokens,
|
||||
@@ -401,6 +417,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
max_num_logits=self.max_num_reqs * self.decode_query_len,
|
||||
vocab_size=self.vocab_size,
|
||||
device=self.device,
|
||||
mask_stride=self.decode_query_len,
|
||||
)
|
||||
|
||||
if self.is_pooling_model and self.is_last_pp_rank:
|
||||
@@ -510,6 +527,20 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
attn_cg_support = attn_cg_support.narrow(
|
||||
*self.model_state.get_additional_cg_support()
|
||||
)
|
||||
# The speculator clears the flag at load time when the checkpoint has
|
||||
# no confidence head, so it holds the effective value.
|
||||
self.adaptive_verification = maybe_create_adaptive_verification_manager(
|
||||
enable_adaptive_verification=getattr(
|
||||
self.speculator, "enable_adaptive_verification", False
|
||||
),
|
||||
attn_groups=self.attn_groups,
|
||||
attn_cg_support=attn_cg_support,
|
||||
req_states=self.req_states,
|
||||
query_start_loc=self.input_buffers.query_start_loc,
|
||||
num_bonus_tokens=self.model_state.num_new_sampled_tokens_per_step,
|
||||
max_total_logits=get_max_chunk_logits(self.vocab_size),
|
||||
)
|
||||
|
||||
self.block_tables = BlockTables(
|
||||
block_sizes=block_sizes,
|
||||
max_num_reqs=self.max_num_reqs,
|
||||
@@ -532,6 +563,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
initialize_mamba_ssu_backend(
|
||||
self.vllm_config.mamba_config, self.kv_cache_config
|
||||
)
|
||||
if self.adaptive_verification is not None:
|
||||
self.compilation_config.cudagraph_mode = CUDAGraphMode.FULL_AND_PIECEWISE
|
||||
cudagraph_mode = self.compilation_config.resolve_cudagraph_mode_and_sizes(
|
||||
attn_cg_support.min_cg_support,
|
||||
attn_cg_support.min_cg_attn_backend,
|
||||
@@ -547,6 +580,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
cudagraph_mode,
|
||||
decode_query_len=self.decode_query_len,
|
||||
lora_capture_cases=self.lora_capture_cases,
|
||||
varlen_decode=self.adaptive_verification is not None,
|
||||
)
|
||||
check_attention_cp_compatibility(self.vllm_config)
|
||||
if isinstance(self.speculator, DraftModelSpeculator):
|
||||
@@ -594,6 +628,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
*args,
|
||||
skip_attn: bool = False,
|
||||
uniform_decode: bool = False,
|
||||
context_len: int = 0,
|
||||
skip_eplb: bool = False,
|
||||
is_profile: bool = False,
|
||||
**kwargs,
|
||||
@@ -655,6 +690,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
dummy_run=True,
|
||||
skip_attn_for_dummy_run=skip_attn,
|
||||
is_profile=is_profile,
|
||||
context_len=context_len,
|
||||
)
|
||||
self.kv_connector.set_disabled(False)
|
||||
|
||||
@@ -670,9 +706,12 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
aux_hidden_states = self.execute_model_state.aux_hidden_states
|
||||
self.execute_model_state = None
|
||||
|
||||
self.step_timing.forward_end()
|
||||
|
||||
# dummy run the eagle speculator's propose to ensure DP/EP sync.
|
||||
if self.speculator is not None:
|
||||
assert self.sampler is not None
|
||||
self.step_timing.drafter_start()
|
||||
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None
|
||||
if self.speculator.supports_mm_inputs:
|
||||
mm_inputs = (
|
||||
@@ -713,6 +752,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
mm_inputs=mm_inputs,
|
||||
is_profile=is_profile,
|
||||
)
|
||||
self.step_timing.drafter_end()
|
||||
|
||||
assert hidden_states is not None # Last PP rank always has hidden_states
|
||||
sample_hidden_states = hidden_states[input_batch.logits_indices]
|
||||
@@ -830,6 +870,13 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
)
|
||||
if self.speculator is not None:
|
||||
self.speculator.capture()
|
||||
if self.adaptive_verification is not None:
|
||||
with self.step_timing.collect() as timings:
|
||||
for batch in self.adaptive_verification.batches_to_profile(
|
||||
self.cudagraph_manager.captured_token_counts()
|
||||
):
|
||||
self._dummy_run(**batch)
|
||||
self.adaptive_verification.set_initial_cost_curves(timings)
|
||||
|
||||
end_time = time.perf_counter()
|
||||
end_free_gpu_memory = torch.accelerator.get_memory_info()[0]
|
||||
@@ -906,6 +953,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
max_tokens=sampling_params.max_tokens if sampling_params else 1, # type: ignore[arg-type]
|
||||
)
|
||||
req_index = self.req_states.req_id_to_index[req_id]
|
||||
if self.adaptive_verification is not None:
|
||||
self.adaptive_verification.add_request(req_index)
|
||||
|
||||
if self.pooling_runner is not None:
|
||||
assert new_req_data.pooling_params is not None
|
||||
@@ -994,8 +1043,12 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
num_reqs, num_toks, max_query_len, has_prefill=False
|
||||
)
|
||||
|
||||
draft_tokens = scheduler_output.scheduled_spec_decode_tokens
|
||||
# batch_idx -> req_id
|
||||
req_ids = sort_batch_req_ids(num_tokens_per_req, self.decode_query_len)
|
||||
req_ids = sort_batch_req_ids(
|
||||
num_tokens_per_req, draft_tokens, self.decode_query_len
|
||||
)
|
||||
|
||||
numtoks_iter = map(num_tokens_per_req.__getitem__, req_ids)
|
||||
num_scheduled_tokens = np.fromiter(numtoks_iter, dtype=np.int32, count=num_reqs)
|
||||
|
||||
@@ -1007,9 +1060,15 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
]
|
||||
is_prefilling_np = num_computed_prefill_tokens_np < prefill_len_np
|
||||
|
||||
if self.adaptive_verification is not None and draft_tokens:
|
||||
num_toks = self.adaptive_verification.get_num_tokens(
|
||||
num_tokens_per_req, draft_tokens
|
||||
)
|
||||
|
||||
batch_state = BatchReqState(
|
||||
req_ids=req_ids,
|
||||
num_scheduled_tokens=num_scheduled_tokens,
|
||||
num_tokens=num_toks,
|
||||
idx_mapping_np=idx_mapping_np,
|
||||
prefill_len_np=prefill_len_np,
|
||||
num_computed_prefill_tokens_np=num_computed_prefill_tokens_np,
|
||||
@@ -1026,7 +1085,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
batch_req_state: "BatchReqState",
|
||||
batch_desc: BatchExecutionDescriptor,
|
||||
) -> InputBatch:
|
||||
num_tokens = scheduler_output.total_num_scheduled_tokens
|
||||
num_tokens = batch_req_state.num_tokens
|
||||
num_tokens_after_padding = batch_desc.num_tokens
|
||||
assert num_tokens > 0
|
||||
if envs.VLLM_MOE_SKIP_PADDING:
|
||||
@@ -1072,9 +1131,20 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
np.cumsum(num_logits, out=cu_num_logits_np[1:])
|
||||
cu_num_logits = async_copy_to_gpu(cu_num_logits_np, device=self.device)
|
||||
|
||||
max_expand_len = self.decode_query_len
|
||||
expanded_idx_mapping, expanded_local_pos = expand_idx_mapping(
|
||||
idx_mapping, total_num_logits, cu_num_logits, max_expand_len
|
||||
adaptive_verification = (
|
||||
self.adaptive_verification if num_draft_tokens_per_req is not None else None
|
||||
)
|
||||
num_scheduled_tokens_upper_bound = num_scheduled_tokens_np
|
||||
if adaptive_verification is not None:
|
||||
# num_scheduled_tokens represents the draft budget evenly distributed across
|
||||
# all verification requests, `reallocate_drafts` will unevenly assign the
|
||||
# draft budget to requests on the GPU side only.
|
||||
num_scheduled_tokens_np, cu_num_logits_np = (
|
||||
adaptive_verification.compact_batch(
|
||||
num_draft_tokens_per_req,
|
||||
num_scheduled_tokens_np,
|
||||
cu_num_logits_np,
|
||||
)
|
||||
)
|
||||
|
||||
# Get query_start_loc.
|
||||
@@ -1086,9 +1156,19 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
# Pad for full CUDA graph mode.
|
||||
# Some attention backends like FA3 require query_start_loc to be non-decreasing.
|
||||
query_start_loc_np[num_reqs + 1 :] = num_tokens
|
||||
async_copy_to_gpu(query_start_loc_np, out=self.input_buffers.query_start_loc)
|
||||
query_start_loc = self.input_buffers.query_start_loc
|
||||
async_copy_to_gpu(query_start_loc_np, out=query_start_loc)
|
||||
if adaptive_verification is not None:
|
||||
cu_num_logits, query_start_loc, total_num_draft_tokens = (
|
||||
adaptive_verification.reallocate_drafts(req_ids, idx_mapping)
|
||||
)
|
||||
total_num_logits = num_reqs * num_bonus_tokens + total_num_draft_tokens
|
||||
if draft_tokens:
|
||||
expanded_idx_mapping, expanded_local_pos = expand_idx_mapping(
|
||||
idx_mapping, total_num_logits, cu_num_logits, self.decode_query_len
|
||||
)
|
||||
query_start_loc_np = query_start_loc_np[: num_reqs_padded + 1]
|
||||
query_start_loc = self.input_buffers.query_start_loc[: num_reqs_padded + 1]
|
||||
query_start_loc = query_start_loc[: num_reqs_padded + 1]
|
||||
|
||||
# Get prefill tokens if any.
|
||||
if batch_req_state.has_prefill:
|
||||
@@ -1145,7 +1225,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
seq_lens_cpu_upper_bound_np = np.zeros(num_reqs_padded, dtype=np.int32)
|
||||
np.add(
|
||||
num_computed_tokens_np,
|
||||
num_scheduled_tokens_np,
|
||||
num_scheduled_tokens_upper_bound,
|
||||
out=seq_lens_cpu_upper_bound_np[:num_reqs],
|
||||
)
|
||||
seq_lens_cpu_upper_bound = torch.from_numpy(seq_lens_cpu_upper_bound_np)
|
||||
@@ -1168,7 +1248,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
idx_mapping_np=idx_mapping_np,
|
||||
expanded_idx_mapping=expanded_idx_mapping,
|
||||
expanded_local_pos=expanded_local_pos,
|
||||
num_scheduled_tokens=num_scheduled_tokens_np,
|
||||
num_scheduled_tokens=num_scheduled_tokens_upper_bound,
|
||||
num_tokens=num_tokens,
|
||||
num_tokens_after_padding=num_tokens_after_padding,
|
||||
num_draft_tokens=total_num_draft_tokens,
|
||||
@@ -1192,6 +1272,11 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
cu_num_logits_np=cu_num_logits_np,
|
||||
has_structured_output_reqs=scheduler_output.has_structured_output_requests,
|
||||
prompt_lens=prompt_lens,
|
||||
max_query_len=(
|
||||
int(num_scheduled_tokens_upper_bound.max())
|
||||
if adaptive_verification is not None
|
||||
else None
|
||||
),
|
||||
)
|
||||
return pcp.maybe_partition_pcp_batch(self.pcp_manager, input_batch)
|
||||
|
||||
@@ -1298,6 +1383,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
dummy_run: bool = False,
|
||||
skip_attn_for_dummy_run: bool = False,
|
||||
is_profile: bool = False,
|
||||
context_len: int = 0,
|
||||
) -> ModelRunnerOutput | IntermediateTensors | None:
|
||||
if not dummy_run:
|
||||
# Update the request states.
|
||||
@@ -1315,9 +1401,12 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
# Get batch descriptor and sync across DP ranks.
|
||||
num_reqs = len(scheduler_output.num_scheduled_tokens)
|
||||
num_toks = scheduler_output.total_num_scheduled_tokens
|
||||
max_query_len = max(scheduler_output.num_scheduled_tokens.values())
|
||||
batch_req_state, uniform_tok_count = self.gather_batch_req_state(
|
||||
scheduler_output, dummy_run
|
||||
)
|
||||
if batch_req_state is not None:
|
||||
num_toks = batch_req_state.num_tokens
|
||||
|
||||
num_active_loras = 0
|
||||
if self.lora_config:
|
||||
@@ -1340,6 +1429,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
uniform_tok_count,
|
||||
self.dp_size,
|
||||
self.dp_rank,
|
||||
max_query_len=max_query_len,
|
||||
need_eager=is_profile or skip_compiled,
|
||||
num_active_loras=num_active_loras,
|
||||
)
|
||||
@@ -1378,13 +1468,23 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self._set_active_loras(*lora_inputs)
|
||||
else:
|
||||
# No actual tokens to run. A dummy run for DP or memory profiling.
|
||||
dummy_num_reqs = batch_desc.num_reqs or num_reqs
|
||||
input_batch = InputBatch.make_dummy(
|
||||
batch_desc.num_reqs or num_reqs,
|
||||
dummy_num_reqs,
|
||||
batch_desc.num_tokens,
|
||||
self.input_buffers,
|
||||
max_query_len=batch_desc.max_query_len,
|
||||
)
|
||||
if not skip_attn_for_dummy_run:
|
||||
block_tables, slot_mappings = self.prepare_dummy_attn(input_batch)
|
||||
if context_len:
|
||||
set_dummy_context(
|
||||
input_batch,
|
||||
self.block_tables,
|
||||
context_len,
|
||||
self.kv_cache_config.num_blocks,
|
||||
self.max_model_len,
|
||||
)
|
||||
else:
|
||||
assert batch_desc.cg_mode != CUDAGraphMode.FULL, (
|
||||
"Attention metadata must be prepared for dummy runs when using "
|
||||
@@ -1488,6 +1588,11 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
# Update the EPLB meta.
|
||||
self.eplb.prepare_forward(self.model_config, input_batch.num_tokens)
|
||||
|
||||
self.step_timing.record_batch(
|
||||
input_batch, batch_desc.cg_mode == CUDAGraphMode.FULL
|
||||
)
|
||||
self.step_timing.forward_start()
|
||||
|
||||
# Run model.
|
||||
if batch_desc.cg_mode == CUDAGraphMode.FULL:
|
||||
# Use explicit cudagraph replay for FULL mode.
|
||||
@@ -1698,6 +1803,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
mm_inputs=mm_inputs,
|
||||
)
|
||||
self.req_states.draft_tokens[input_batch.idx_mapping] = draft_tokens
|
||||
if self.adaptive_verification is not None:
|
||||
self.adaptive_verification.record_confidences(
|
||||
self.speculator.draft_token_confidence_probs, input_batch
|
||||
)
|
||||
|
||||
if self.num_speculative_steps > 0:
|
||||
# Spec-decode and diffusion LLMs both use draft tokens but the latter does
|
||||
@@ -1839,6 +1948,9 @@ class BatchReqState(NamedTuple):
|
||||
|
||||
req_ids: list[str]
|
||||
num_scheduled_tokens: np.ndarray # [num_reqs]
|
||||
# May be less than scheduler_output.total_num_scheduled_tokens:
|
||||
# adaptive verification trims the draft budget before running.
|
||||
num_tokens: int
|
||||
idx_mapping_np: np.ndarray # [num_reqs]
|
||||
prefill_len_np: np.ndarray # [num_reqs]
|
||||
num_computed_prefill_tokens_np: np.ndarray # [num_reqs]
|
||||
@@ -1847,9 +1959,15 @@ class BatchReqState(NamedTuple):
|
||||
|
||||
|
||||
def sort_batch_req_ids(
|
||||
num_tokens_per_req: dict[str, int], decode_query_len: int
|
||||
num_tokens_per_req: dict[str, int],
|
||||
draft_tokens: dict[str, list[int]],
|
||||
decode_query_len: int,
|
||||
) -> list[str]:
|
||||
# Order decode -> short_extend -> prefill; split_decodes_and_prefills
|
||||
# relies on uniform decodes (query_len == decode_query_len) leading.
|
||||
key = lambda r: ((num := num_tokens_per_req[r]) != decode_query_len, num)
|
||||
# Order verification/decode -> short_extend -> prefill;
|
||||
# split_decodes_and_prefills relies on decode-like requests leading.
|
||||
key = lambda r: (
|
||||
not draft_tokens.get(r),
|
||||
(num := num_tokens_per_req[r]) != decode_query_len,
|
||||
num,
|
||||
)
|
||||
return sorted(num_tokens_per_req, key=key)
|
||||
|
||||
Reference in New Issue
Block a user