[SpecDecode] Add LiLiCorr drafter (#57934)

Signed-off-by: Andrii Skliar <askliar@nvidia.com>
Signed-off-by: Andrii Skliar <andreyws96@gmail.com>
Co-authored-by: Andrii Skliar <askliar@nvidia.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: Benjamin Chislett <chislett.ben@gmail.com>
This commit is contained in:
Andrii Skliar
2026-09-25 10:08:28 -04:00
committed by GitHub
co-authored by Andrii Skliar Codex Benjamin Chislett
parent 25b0add7b8
commit 73a78e6f1f
21 changed files with 1944 additions and 110 deletions
+21 -1
View File
@@ -256,6 +256,8 @@ class LogitsProcessor(PluggableLayer):
hidden_states: torch.Tensor,
k: int,
embedding_bias: torch.Tensor | None = None,
*,
return_log_probs: bool = False,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Vocab-parallel top-k without all-gathering full logits.
@@ -263,7 +265,9 @@ class LogitsProcessor(PluggableLayer):
the values as well as the global ids. Communication is
O(batch * 2k * tp_size) rather than O(batch * vocab_size).
Scale and soft cap are applied to the k selected values rather than
With return_log_probs, values use the full-vocabulary log-partition.
Otherwise scale and soft cap are applied to the k selected values rather than
the whole vocabulary; both are monotonic, so the selection is the same
and only k entries are touched.
"""
@@ -274,12 +278,26 @@ class LogitsProcessor(PluggableLayer):
)
logits = self._apply_head(lm_head, hidden_states, embedding_bias)
if return_log_probs:
if self.soft_cap is not None:
logits = torch.tanh(logits / self.soft_cap) * self.soft_cap
if self.scale != 1.0:
logits = logits * self.scale
# Mask out padding entries beyond org_vocab_size on this shard.
num_pad = lm_head.shard_indices.num_org_vocab_padding
if num_pad > 0:
logits[..., -num_pad:] = -float("inf")
if return_log_probs:
log_partition = torch.logsumexp(logits.float(), dim=-1, keepdim=True)
if lm_head.tp_size > 1:
log_partition = torch.logsumexp(
tensor_model_parallel_all_gather(log_partition, dim=-1),
dim=-1,
keepdim=True,
)
values, ids = _topk(logits, k)
# Convert shard-local indices to global vocab indices.
ids = ids.to(torch.int64) + lm_head.shard_indices.org_vocab_start_index
@@ -291,6 +309,8 @@ class LogitsProcessor(PluggableLayer):
ids = ids.gather(-1, selected)
values = values.float()
if return_log_probs:
return ids, values - log_partition
if self.scale != 1.0:
values = values * self.scale
if self.soft_cap is not None: