[Spec Decode] DSpark confidence-scheduled verification (#47808)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com> Signed-off-by: Benjamin Chislett <chislett.ben@gmail.com> Signed-off-by: Lucas Wilkinson <wilkinson.lucas@gmail.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Benjamin Chislett <chislett.ben@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
Claude Opus 5
Benjamin Chislett
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Nick Hill
OpenAI Codex
parent
fe889ac925
commit
7f7a32cfec
@@ -135,11 +135,10 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
|
||||
def capture(self) -> None:
|
||||
logger.info("Capturing model for %s speculator...", self._speculator_name)
|
||||
# Reset sampling indices to zero to prevent stale values from prior
|
||||
# dummy runs from being baked into the captured graph.
|
||||
# Padded sample rows must not scatter into a live request during capture.
|
||||
self.sample_indices.zero_()
|
||||
self.sample_pos.zero_()
|
||||
self.sample_idx_mapping.zero_()
|
||||
self.sample_idx_mapping.fill_(-1)
|
||||
assert self.query_cudagraph_manager is not None
|
||||
self.query_cudagraph_manager.capture(
|
||||
self._generate_draft,
|
||||
@@ -589,7 +588,10 @@ def _prepare_dflash_inputs_kernel(
|
||||
# seq_lens is the absolute sequence length the draft attention
|
||||
# reads up to (context + query), not just the count of accepted
|
||||
# tokens this step.
|
||||
tl.store(out_seq_lens_ptr + req_idx, last_valid_pos + 1 + num_query_per_req)
|
||||
tl.store(
|
||||
out_seq_lens_ptr + req_idx,
|
||||
tl.minimum(last_valid_pos + 1 + num_query_per_req, max_model_len),
|
||||
)
|
||||
# Copy sampling state.
|
||||
tl.store(
|
||||
out_temperature_ptr + req_state_idx,
|
||||
|
||||
Reference in New Issue
Block a user