[Spec Decode] DSpark confidence-scheduled verification (#47808)

Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
Signed-off-by: Benjamin Chislett <chislett.ben@gmail.com>
Signed-off-by: Lucas Wilkinson <wilkinson.lucas@gmail.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: Benjamin Chislett <chislett.ben@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Lucas Wilkinson
2026-08-12 12:27:48 -07:00
committed by GitHub
co-authored by OpenAI Codex Claude Opus 5 Benjamin Chislett mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Nick Hill OpenAI Codex
parent fe889ac925
commit 7f7a32cfec
40 changed files with 1643 additions and 111 deletions
+13 -2
View File
@@ -553,7 +553,10 @@ def fp8_fp4_mqa_logits(
def get_paged_mqa_logits_metadata(
context_lens: torch.Tensor, block_size: int, num_sms: int
context_lens: torch.Tensor,
block_size: int,
num_sms: int,
indices: torch.Tensor | None = None,
) -> torch.Tensor:
"""Build scheduling metadata for paged MQA logits.
@@ -562,6 +565,7 @@ def get_paged_mqa_logits_metadata(
per batch element.
block_size: KV-cache block size in tokens (e.g., 64).
num_sms: Number of SMs available. 132 for Hopper
indices: Optional request index for each varlen row.
Returns:
Backend-specific tensor consumed by `fp8_fp4_paged_mqa_logits` to
@@ -570,7 +574,10 @@ def get_paged_mqa_logits_metadata(
_lazy_init()
if _get_paged_mqa_logits_metadata_impl is None:
return _missing()
return _get_paged_mqa_logits_metadata_impl(context_lens, block_size, num_sms)
kwargs = {} if indices is None else {"indices": indices}
return _get_paged_mqa_logits_metadata_impl(
context_lens, block_size, num_sms, **kwargs
)
def fp8_fp4_paged_mqa_logits(
@@ -582,6 +589,7 @@ def fp8_fp4_paged_mqa_logits(
schedule_metadata: torch.Tensor,
max_model_len: int,
clean_logits: bool,
indices: torch.Tensor | None = None,
) -> torch.Tensor:
"""Compute MQA logits using a paged KV-cache.
@@ -606,6 +614,7 @@ def fp8_fp4_paged_mqa_logits(
used to distribute work across SMs.
max_model_len: Maximum sequence length used to size the logits output.
clean_logits: Whether to clean the unfilled logits into `-inf`.
indices: Optional request index for each varlen row.
Returns:
Logits tensor of shape [B * next_n, max_model_len], dtype
@@ -614,6 +623,7 @@ def fp8_fp4_paged_mqa_logits(
_lazy_init()
if _fp8_fp4_paged_mqa_logits_impl is None:
return _missing()
kwargs = {} if indices is None else {"indices": indices}
return _fp8_fp4_paged_mqa_logits_impl(
q,
kv_cache,
@@ -623,6 +633,7 @@ def fp8_fp4_paged_mqa_logits(
schedule_metadata,
max_model_len,
clean_logits=clean_logits,
**kwargs,
)