[SpecDecode] Add LiLiCorr drafter (#57934)
Signed-off-by: Andrii Skliar <askliar@nvidia.com> Signed-off-by: Andrii Skliar <andreyws96@gmail.com> Co-authored-by: Andrii Skliar <askliar@nvidia.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: Benjamin Chislett <chislett.ben@gmail.com>
This commit is contained in:
co-authored by
Andrii Skliar
Codex
Benjamin Chislett
parent
25b0add7b8
commit
73a78e6f1f
@@ -322,6 +322,11 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
context_slots,
|
||||
)
|
||||
|
||||
def prepare_context_anchor(
|
||||
self, input_batch: InputBatch, num_rejected: torch.Tensor
|
||||
) -> None:
|
||||
"""Publish context features required by a draft's candidate head."""
|
||||
|
||||
@torch.inference_mode()
|
||||
def propose(
|
||||
self,
|
||||
@@ -369,6 +374,7 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
else:
|
||||
hidden_states = last_hidden_states
|
||||
self.hidden_states[:num_target_tokens].copy_(hidden_states[:num_target_tokens])
|
||||
self.prepare_context_anchor(input_batch, num_rejected)
|
||||
|
||||
if dummy_run and skip_attn_for_dummy_run:
|
||||
# Memory profiling path: block_tables / kv_cache_config are not initialized.
|
||||
|
||||
@@ -61,8 +61,12 @@ def load_dflash_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mo
|
||||
|
||||
target_lm_head = get_target_lm_head(target_model, target_language_model)
|
||||
draft_lm_head = getattr(dflash_model, "lm_head", None)
|
||||
if target_lm_head is not None and _should_share(
|
||||
dflash_model, "has_own_lm_head", draft_lm_head, target_lm_head
|
||||
if (
|
||||
target_lm_head is not None
|
||||
and not getattr(dflash_model.config, "has_own_lm_head", False)
|
||||
and _should_share(
|
||||
dflash_model, "has_own_lm_head", draft_lm_head, target_lm_head
|
||||
)
|
||||
):
|
||||
if draft_lm_head is not None:
|
||||
del dflash_model.lm_head
|
||||
|
||||
Reference in New Issue
Block a user