[SpecDecode] Add LiLiCorr drafter (#57934)

Signed-off-by: Andrii Skliar <askliar@nvidia.com>
Signed-off-by: Andrii Skliar <andreyws96@gmail.com>
Co-authored-by: Andrii Skliar <askliar@nvidia.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: Benjamin Chislett <chislett.ben@gmail.com>
This commit is contained in:
Andrii Skliar
2026-09-25 10:08:28 -04:00
committed by GitHub
co-authored by Andrii Skliar Codex Benjamin Chislett
parent 25b0add7b8
commit 73a78e6f1f
21 changed files with 1944 additions and 110 deletions
@@ -15,6 +15,12 @@ def init_speculator(vllm_config: VllmConfig, device: torch.device):
return ExtractHiddenStatesSpeculator(vllm_config, device)
elif speculative_config.method == "dflash":
if "LiLiCorrDraftModel" in speculative_config.draft_model_config.architectures:
from vllm.v1.worker.gpu.spec_decode.lilicorr.speculator import (
LiLiCorrSpeculator,
)
return LiLiCorrSpeculator(vllm_config, device)
if "DFlash2DraftModel" in speculative_config.draft_model_config.architectures:
from vllm.v1.worker.gpu.spec_decode.dflash2.speculator import (
DFlash2Speculator,
@@ -322,6 +322,11 @@ class DFlashSpeculator(DraftModelSpeculator):
context_slots,
)
def prepare_context_anchor(
self, input_batch: InputBatch, num_rejected: torch.Tensor
) -> None:
"""Publish context features required by a draft's candidate head."""
@torch.inference_mode()
def propose(
self,
@@ -369,6 +374,7 @@ class DFlashSpeculator(DraftModelSpeculator):
else:
hidden_states = last_hidden_states
self.hidden_states[:num_target_tokens].copy_(hidden_states[:num_target_tokens])
self.prepare_context_anchor(input_batch, num_rejected)
if dummy_run and skip_attn_for_dummy_run:
# Memory profiling path: block_tables / kv_cache_config are not initialized.
@@ -61,8 +61,12 @@ def load_dflash_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mo
target_lm_head = get_target_lm_head(target_model, target_language_model)
draft_lm_head = getattr(dflash_model, "lm_head", None)
if target_lm_head is not None and _should_share(
dflash_model, "has_own_lm_head", draft_lm_head, target_lm_head
if (
target_lm_head is not None
and not getattr(dflash_model.config, "has_own_lm_head", False)
and _should_share(
dflash_model, "has_own_lm_head", draft_lm_head, target_lm_head
)
):
if draft_lm_head is not None:
del dflash_model.lm_head
@@ -108,26 +108,93 @@ def _cache_draft_logits_kernel(
tl.store(cached_candidate_ptr + cache_base + offsets, token_ids, mask=mask)
class CandidateSampler:
"""The shared DFlash2/LiLiCorr candidate walk and realized proposal cache."""
def __init__(
self, max_num_reqs: int, num_steps: int, top_k: int, device: torch.device
):
self.num_steps = num_steps
self.top_k = top_k
self.scores = torch.empty(
max_num_reqs, num_steps, top_k, dtype=torch.float32, device=device
)
self.cached_candidate_ids = torch.zeros(
self.scores.shape, dtype=torch.int64, device=device
)
def sample(
self,
candidate_ids: torch.Tensor,
scores: torch.Tensor,
num_reqs: int,
sample_pos: torch.Tensor,
idx_mapping: torch.Tensor,
temperature: torch.Tensor,
seeds: torch.Tensor,
draft_tokens: torch.Tensor,
draft_logits: torch.Tensor | None,
use_fp64: bool,
) -> None:
candidate_ids = candidate_ids.contiguous()
sample_pos = sample_pos.contiguous()
idx_mapping = idx_mapping.contiguous()
block_k = triton.next_power_of_2(self.top_k)
_selector_walk_kernel[(num_reqs,)](
scores.contiguous(),
candidate_ids.contiguous(),
sample_pos,
idx_mapping,
temperature,
seeds,
draft_tokens,
self.scores,
num_steps=self.num_steps,
top_k=self.top_k,
BLOCK_K=block_k,
SAMPLE_PROBABILISTIC=draft_logits is not None,
USE_FP64=use_fp64,
num_warps=1,
)
if draft_logits is not None:
self._cache_draft_logits(
candidate_ids, num_reqs * self.num_steps, idx_mapping, draft_logits
)
def _cache_draft_logits(
self,
candidate_ids: torch.Tensor,
num_sample: int,
idx_mapping: torch.Tensor,
draft_logits: torch.Tensor,
) -> None:
block_k = triton.next_power_of_2(self.top_k)
_cache_draft_logits_kernel[(num_sample,)](
draft_logits,
self.cached_candidate_ids,
candidate_ids,
self.scores,
idx_mapping,
draft_logits.stride(0),
draft_logits.stride(1),
num_steps=self.num_steps,
top_k=self.top_k,
BLOCK_K=block_k,
num_warps=1,
)
class DFlash2Speculator(DFlashSpeculator):
_speculator_name = "DFlash2"
_candidate_top_k_key = "selector_top_k"
def __init__(self, vllm_config: VllmConfig, device: torch.device):
super().__init__(vllm_config, device)
draft_config = self.draft_model_config.hf_config.dflash_config
self.selector_top_k = int(draft_config["selector_top_k"])
self._anchor_indices = (
torch.arange(self.max_num_reqs, dtype=torch.int64, device=device)
* self.num_query_per_req
)
self._selector_scores = torch.empty(
self.max_num_reqs,
self.num_speculative_steps,
self.selector_top_k,
dtype=torch.float32,
device=device,
)
self._cached_candidate_ids = torch.zeros(
self._selector_scores.shape, dtype=torch.int64, device=device
self.top_k = int(draft_config[self._candidate_top_k_key])
self.candidate_sampler = CandidateSampler(
self.max_num_reqs, self.num_speculative_steps, self.top_k, device
)
def draft_logits_spec(self, vllm_config: VllmConfig) -> tuple[torch.dtype, float]:
@@ -136,48 +203,6 @@ class DFlash2Speculator(DFlashSpeculator):
# candidates.
return torch.float32, -float("inf")
def _sample_path(
self,
candidate_ids: torch.Tensor,
scores: torch.Tensor,
num_reqs: int,
) -> None:
block_k = triton.next_power_of_2(self.selector_top_k)
_selector_walk_kernel[(num_reqs,)](
scores.contiguous(),
candidate_ids.contiguous(),
self.sample_pos,
self.sample_idx_mapping,
self.temperature,
self.seeds,
self.draft_tokens,
self._selector_scores,
num_steps=self.num_speculative_steps,
top_k=self.selector_top_k,
BLOCK_K=block_k,
SAMPLE_PROBABILISTIC=self.draft_logits is not None,
USE_FP64=self.use_fp64_gumbel,
num_warps=1,
)
def _cache_draft_logits(self, candidate_ids: torch.Tensor, num_sample: int) -> None:
draft_logits = self.draft_logits
assert draft_logits is not None
block_k = triton.next_power_of_2(self.selector_top_k)
_cache_draft_logits_kernel[(num_sample,)](
draft_logits,
self._cached_candidate_ids,
candidate_ids,
self._selector_scores,
self.sample_idx_mapping,
draft_logits.stride(0),
draft_logits.stride(1),
num_steps=self.num_speculative_steps,
top_k=self.selector_top_k,
BLOCK_K=block_k,
num_warps=1,
)
def _generate_draft(
self,
num_reqs: int,
@@ -202,22 +227,39 @@ class DFlash2Speculator(DFlashSpeculator):
hidden_states.flatten(0, 1)
)
candidate_ids = candidate_ids.view(
num_reqs, self.num_speculative_steps, self.selector_top_k
num_reqs, self.num_speculative_steps, self.top_k
)
unary_logits = unary_logits.view_as(candidate_ids)
anchor_token_ids = self.input_buffers.input_ids[self._anchor_indices[:num_reqs]]
scores = self.model.model.candidate_selector(
scores = self._score_candidates(candidate_ids, unary_logits, hidden_states)
self.candidate_sampler.sample(
candidate_ids,
unary_logits,
hidden_states,
anchor_token_ids,
scores,
num_reqs,
self.sample_pos,
self.sample_idx_mapping,
self.temperature,
self.seeds,
self.draft_tokens,
self.draft_logits,
self.use_fp64_gumbel,
)
self._sample_path(candidate_ids, scores, num_reqs)
if self.enable_adaptive_verification:
self._maybe_predict_acceptance(
self._selector_scores[:num_reqs].flatten(0, 1),
self.candidate_sampler.scores[:num_reqs].flatten(0, 1),
self.sample_idx_mapping[:num_sample],
self.sample_col[:num_sample],
)
if self.draft_logits is not None:
self._cache_draft_logits(candidate_ids, num_sample)
def _score_candidates(
self,
candidate_ids: torch.Tensor,
unary_logits: torch.Tensor,
hidden_states: torch.Tensor,
) -> torch.Tensor:
num_reqs = candidate_ids.shape[0]
anchor_token_ids = self.input_buffers.input_ids[
: num_reqs * self.num_query_per_req : self.num_query_per_req
]
return self.model.model.candidate_selector(
candidate_ids, unary_logits, hidden_states, anchor_token_ids
)
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -0,0 +1,97 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from torch import nn
from vllm.config import VllmConfig
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
from vllm.v1.worker.gpu.input_batch import InputBatch
from vllm.v1.worker.gpu.spec_decode.dflash2.speculator import DFlash2Speculator
from vllm.v1.worker.gpu.spec_decode.eagle.utils import get_target_lm_head
class LiLiCorrSpeculator(DFlash2Speculator):
_speculator_name = "LiLiCorr"
_candidate_top_k_key = "lilicorr_candidate_topk"
def __init__(self, vllm_config: VllmConfig, device: torch.device):
super().__init__(vllm_config, device)
config = self.draft_model_config.hf_config
self.anchor_hidden = torch.zeros(
self.max_num_reqs, config.hidden_size, dtype=self.dtype, device=device
)
self.anchor_valid = torch.zeros(
self.max_num_reqs, dtype=torch.bool, device=device
)
def load_draft_model(
self, target_model: nn.Module, target_attn_layer_names: set[str]
) -> nn.Module:
model = super().load_draft_model(target_model, target_attn_layer_names)
language_model = (
target_model.get_language_model()
if hasattr(target_model, "get_language_model")
else target_model
)
inner = getattr(language_model, "model", language_model)
embedding = getattr(inner, "embed_tokens", None) or getattr(
inner, "embedding", None
)
lm_head = (
model.lm_head
if model.has_own_lm_head
else get_target_lm_head(target_model, language_model)
)
if not isinstance(embedding, VocabParallelEmbedding) or not isinstance(
lm_head, VocabParallelEmbedding
):
raise ValueError(
"LiLiCorr requires the target input embedding and its selected "
"LM head on the draft rank."
)
if embedding.embedding_dim != model.config.hidden_size:
raise ValueError(
"LiLiCorr target embedding width must match its trained "
"token projection input."
)
if (
embedding.shard_indices.num_added_elements
or lm_head.shard_indices.num_added_elements
):
raise ValueError("LiLiCorr does not support added vocabulary entries.")
# Candidate embeddings always come from the target; the checkpoint may
# supply its own candidate LM head.
self.target_embeddings = embedding
model.lm_head = lm_head
return model
def prepare_context_anchor(
self, input_batch: InputBatch, num_rejected: torch.Tensor
) -> None:
num_reqs = input_batch.num_reqs
starts = input_batch.query_start_loc[:num_reqs]
ends = input_batch.query_start_loc[1 : num_reqs + 1] - num_rejected[:num_reqs]
valid = ends > starts
indices = (ends - 1).clamp_min(0).long()
self.anchor_hidden.zero_()
self.anchor_valid.zero_()
self.anchor_hidden[:num_reqs].copy_(
self.model.model.hidden_norm(self.hidden_states[indices]) * valid[:, None]
)
self.anchor_valid[:num_reqs].copy_(valid)
def _score_candidates(
self,
candidate_ids: torch.Tensor,
unary_logits: torch.Tensor,
hidden_states: torch.Tensor,
) -> torch.Tensor:
num_reqs = candidate_ids.shape[0]
return self.model.model.lilicorr(
self.target_embeddings(candidate_ids),
unary_logits,
hidden_states,
self.anchor_hidden[:num_reqs],
self.anchor_valid[:num_reqs],
)