[Spec Decode] DSpark confidence-scheduled verification (#47808)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com> Signed-off-by: Benjamin Chislett <chislett.ben@gmail.com> Signed-off-by: Lucas Wilkinson <wilkinson.lucas@gmail.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Benjamin Chislett <chislett.ben@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
Claude Opus 5
Benjamin Chislett
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Nick Hill
OpenAI Codex
parent
fe889ac925
commit
7f7a32cfec
+13
-2
@@ -553,7 +553,10 @@ def fp8_fp4_mqa_logits(
|
||||
|
||||
|
||||
def get_paged_mqa_logits_metadata(
|
||||
context_lens: torch.Tensor, block_size: int, num_sms: int
|
||||
context_lens: torch.Tensor,
|
||||
block_size: int,
|
||||
num_sms: int,
|
||||
indices: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Build scheduling metadata for paged MQA logits.
|
||||
|
||||
@@ -562,6 +565,7 @@ def get_paged_mqa_logits_metadata(
|
||||
per batch element.
|
||||
block_size: KV-cache block size in tokens (e.g., 64).
|
||||
num_sms: Number of SMs available. 132 for Hopper
|
||||
indices: Optional request index for each varlen row.
|
||||
|
||||
Returns:
|
||||
Backend-specific tensor consumed by `fp8_fp4_paged_mqa_logits` to
|
||||
@@ -570,7 +574,10 @@ def get_paged_mqa_logits_metadata(
|
||||
_lazy_init()
|
||||
if _get_paged_mqa_logits_metadata_impl is None:
|
||||
return _missing()
|
||||
return _get_paged_mqa_logits_metadata_impl(context_lens, block_size, num_sms)
|
||||
kwargs = {} if indices is None else {"indices": indices}
|
||||
return _get_paged_mqa_logits_metadata_impl(
|
||||
context_lens, block_size, num_sms, **kwargs
|
||||
)
|
||||
|
||||
|
||||
def fp8_fp4_paged_mqa_logits(
|
||||
@@ -582,6 +589,7 @@ def fp8_fp4_paged_mqa_logits(
|
||||
schedule_metadata: torch.Tensor,
|
||||
max_model_len: int,
|
||||
clean_logits: bool,
|
||||
indices: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Compute MQA logits using a paged KV-cache.
|
||||
|
||||
@@ -606,6 +614,7 @@ def fp8_fp4_paged_mqa_logits(
|
||||
used to distribute work across SMs.
|
||||
max_model_len: Maximum sequence length used to size the logits output.
|
||||
clean_logits: Whether to clean the unfilled logits into `-inf`.
|
||||
indices: Optional request index for each varlen row.
|
||||
|
||||
Returns:
|
||||
Logits tensor of shape [B * next_n, max_model_len], dtype
|
||||
@@ -614,6 +623,7 @@ def fp8_fp4_paged_mqa_logits(
|
||||
_lazy_init()
|
||||
if _fp8_fp4_paged_mqa_logits_impl is None:
|
||||
return _missing()
|
||||
kwargs = {} if indices is None else {"indices": indices}
|
||||
return _fp8_fp4_paged_mqa_logits_impl(
|
||||
q,
|
||||
kv_cache,
|
||||
@@ -623,6 +633,7 @@ def fp8_fp4_paged_mqa_logits(
|
||||
schedule_metadata,
|
||||
max_model_len,
|
||||
clean_logits=clean_logits,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user