[SpecDecode] Add LiLiCorr drafter (#57934)

Signed-off-by: Andrii Skliar <askliar@nvidia.com>
Signed-off-by: Andrii Skliar <andreyws96@gmail.com>
Co-authored-by: Andrii Skliar <askliar@nvidia.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: Benjamin Chislett <chislett.ben@gmail.com>
This commit is contained in:
Andrii Skliar
2026-09-25 10:08:28 -04:00
committed by GitHub
co-authored by Andrii Skliar Codex Benjamin Chislett
parent 25b0add7b8
commit 73a78e6f1f
21 changed files with 1944 additions and 110 deletions
@@ -322,6 +322,11 @@ class DFlashSpeculator(DraftModelSpeculator):
context_slots,
)
def prepare_context_anchor(
self, input_batch: InputBatch, num_rejected: torch.Tensor
) -> None:
"""Publish context features required by a draft's candidate head."""
@torch.inference_mode()
def propose(
self,
@@ -369,6 +374,7 @@ class DFlashSpeculator(DraftModelSpeculator):
else:
hidden_states = last_hidden_states
self.hidden_states[:num_target_tokens].copy_(hidden_states[:num_target_tokens])
self.prepare_context_anchor(input_batch, num_rejected)
if dummy_run and skip_attn_for_dummy_run:
# Memory profiling path: block_tables / kv_cache_config are not initialized.
@@ -61,8 +61,12 @@ def load_dflash_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mo
target_lm_head = get_target_lm_head(target_model, target_language_model)
draft_lm_head = getattr(dflash_model, "lm_head", None)
if target_lm_head is not None and _should_share(
dflash_model, "has_own_lm_head", draft_lm_head, target_lm_head
if (
target_lm_head is not None
and not getattr(dflash_model.config, "has_own_lm_head", False)
and _should_share(
dflash_model, "has_own_lm_head", draft_lm_head, target_lm_head
)
):
if draft_lm_head is not None:
del dflash_model.lm_head