[SpecDecode] Add LiLiCorr drafter (#57934)
Signed-off-by: Andrii Skliar <askliar@nvidia.com> Signed-off-by: Andrii Skliar <andreyws96@gmail.com> Co-authored-by: Andrii Skliar <askliar@nvidia.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: Benjamin Chislett <chislett.ben@gmail.com>
This commit is contained in:
co-authored by
Andrii Skliar
Codex
Benjamin Chislett
parent
25b0add7b8
commit
73a78e6f1f
@@ -15,6 +15,12 @@ def init_speculator(vllm_config: VllmConfig, device: torch.device):
|
||||
|
||||
return ExtractHiddenStatesSpeculator(vllm_config, device)
|
||||
elif speculative_config.method == "dflash":
|
||||
if "LiLiCorrDraftModel" in speculative_config.draft_model_config.architectures:
|
||||
from vllm.v1.worker.gpu.spec_decode.lilicorr.speculator import (
|
||||
LiLiCorrSpeculator,
|
||||
)
|
||||
|
||||
return LiLiCorrSpeculator(vllm_config, device)
|
||||
if "DFlash2DraftModel" in speculative_config.draft_model_config.architectures:
|
||||
from vllm.v1.worker.gpu.spec_decode.dflash2.speculator import (
|
||||
DFlash2Speculator,
|
||||
|
||||
@@ -322,6 +322,11 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
context_slots,
|
||||
)
|
||||
|
||||
def prepare_context_anchor(
|
||||
self, input_batch: InputBatch, num_rejected: torch.Tensor
|
||||
) -> None:
|
||||
"""Publish context features required by a draft's candidate head."""
|
||||
|
||||
@torch.inference_mode()
|
||||
def propose(
|
||||
self,
|
||||
@@ -369,6 +374,7 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
else:
|
||||
hidden_states = last_hidden_states
|
||||
self.hidden_states[:num_target_tokens].copy_(hidden_states[:num_target_tokens])
|
||||
self.prepare_context_anchor(input_batch, num_rejected)
|
||||
|
||||
if dummy_run and skip_attn_for_dummy_run:
|
||||
# Memory profiling path: block_tables / kv_cache_config are not initialized.
|
||||
|
||||
@@ -61,8 +61,12 @@ def load_dflash_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mo
|
||||
|
||||
target_lm_head = get_target_lm_head(target_model, target_language_model)
|
||||
draft_lm_head = getattr(dflash_model, "lm_head", None)
|
||||
if target_lm_head is not None and _should_share(
|
||||
dflash_model, "has_own_lm_head", draft_lm_head, target_lm_head
|
||||
if (
|
||||
target_lm_head is not None
|
||||
and not getattr(dflash_model.config, "has_own_lm_head", False)
|
||||
and _should_share(
|
||||
dflash_model, "has_own_lm_head", draft_lm_head, target_lm_head
|
||||
)
|
||||
):
|
||||
if draft_lm_head is not None:
|
||||
del dflash_model.lm_head
|
||||
|
||||
@@ -108,26 +108,93 @@ def _cache_draft_logits_kernel(
|
||||
tl.store(cached_candidate_ptr + cache_base + offsets, token_ids, mask=mask)
|
||||
|
||||
|
||||
class CandidateSampler:
|
||||
"""The shared DFlash2/LiLiCorr candidate walk and realized proposal cache."""
|
||||
|
||||
def __init__(
|
||||
self, max_num_reqs: int, num_steps: int, top_k: int, device: torch.device
|
||||
):
|
||||
self.num_steps = num_steps
|
||||
self.top_k = top_k
|
||||
self.scores = torch.empty(
|
||||
max_num_reqs, num_steps, top_k, dtype=torch.float32, device=device
|
||||
)
|
||||
self.cached_candidate_ids = torch.zeros(
|
||||
self.scores.shape, dtype=torch.int64, device=device
|
||||
)
|
||||
|
||||
def sample(
|
||||
self,
|
||||
candidate_ids: torch.Tensor,
|
||||
scores: torch.Tensor,
|
||||
num_reqs: int,
|
||||
sample_pos: torch.Tensor,
|
||||
idx_mapping: torch.Tensor,
|
||||
temperature: torch.Tensor,
|
||||
seeds: torch.Tensor,
|
||||
draft_tokens: torch.Tensor,
|
||||
draft_logits: torch.Tensor | None,
|
||||
use_fp64: bool,
|
||||
) -> None:
|
||||
candidate_ids = candidate_ids.contiguous()
|
||||
sample_pos = sample_pos.contiguous()
|
||||
idx_mapping = idx_mapping.contiguous()
|
||||
block_k = triton.next_power_of_2(self.top_k)
|
||||
_selector_walk_kernel[(num_reqs,)](
|
||||
scores.contiguous(),
|
||||
candidate_ids.contiguous(),
|
||||
sample_pos,
|
||||
idx_mapping,
|
||||
temperature,
|
||||
seeds,
|
||||
draft_tokens,
|
||||
self.scores,
|
||||
num_steps=self.num_steps,
|
||||
top_k=self.top_k,
|
||||
BLOCK_K=block_k,
|
||||
SAMPLE_PROBABILISTIC=draft_logits is not None,
|
||||
USE_FP64=use_fp64,
|
||||
num_warps=1,
|
||||
)
|
||||
|
||||
if draft_logits is not None:
|
||||
self._cache_draft_logits(
|
||||
candidate_ids, num_reqs * self.num_steps, idx_mapping, draft_logits
|
||||
)
|
||||
|
||||
def _cache_draft_logits(
|
||||
self,
|
||||
candidate_ids: torch.Tensor,
|
||||
num_sample: int,
|
||||
idx_mapping: torch.Tensor,
|
||||
draft_logits: torch.Tensor,
|
||||
) -> None:
|
||||
block_k = triton.next_power_of_2(self.top_k)
|
||||
_cache_draft_logits_kernel[(num_sample,)](
|
||||
draft_logits,
|
||||
self.cached_candidate_ids,
|
||||
candidate_ids,
|
||||
self.scores,
|
||||
idx_mapping,
|
||||
draft_logits.stride(0),
|
||||
draft_logits.stride(1),
|
||||
num_steps=self.num_steps,
|
||||
top_k=self.top_k,
|
||||
BLOCK_K=block_k,
|
||||
num_warps=1,
|
||||
)
|
||||
|
||||
|
||||
class DFlash2Speculator(DFlashSpeculator):
|
||||
_speculator_name = "DFlash2"
|
||||
_candidate_top_k_key = "selector_top_k"
|
||||
|
||||
def __init__(self, vllm_config: VllmConfig, device: torch.device):
|
||||
super().__init__(vllm_config, device)
|
||||
draft_config = self.draft_model_config.hf_config.dflash_config
|
||||
self.selector_top_k = int(draft_config["selector_top_k"])
|
||||
self._anchor_indices = (
|
||||
torch.arange(self.max_num_reqs, dtype=torch.int64, device=device)
|
||||
* self.num_query_per_req
|
||||
)
|
||||
self._selector_scores = torch.empty(
|
||||
self.max_num_reqs,
|
||||
self.num_speculative_steps,
|
||||
self.selector_top_k,
|
||||
dtype=torch.float32,
|
||||
device=device,
|
||||
)
|
||||
self._cached_candidate_ids = torch.zeros(
|
||||
self._selector_scores.shape, dtype=torch.int64, device=device
|
||||
self.top_k = int(draft_config[self._candidate_top_k_key])
|
||||
self.candidate_sampler = CandidateSampler(
|
||||
self.max_num_reqs, self.num_speculative_steps, self.top_k, device
|
||||
)
|
||||
|
||||
def draft_logits_spec(self, vllm_config: VllmConfig) -> tuple[torch.dtype, float]:
|
||||
@@ -136,48 +203,6 @@ class DFlash2Speculator(DFlashSpeculator):
|
||||
# candidates.
|
||||
return torch.float32, -float("inf")
|
||||
|
||||
def _sample_path(
|
||||
self,
|
||||
candidate_ids: torch.Tensor,
|
||||
scores: torch.Tensor,
|
||||
num_reqs: int,
|
||||
) -> None:
|
||||
block_k = triton.next_power_of_2(self.selector_top_k)
|
||||
_selector_walk_kernel[(num_reqs,)](
|
||||
scores.contiguous(),
|
||||
candidate_ids.contiguous(),
|
||||
self.sample_pos,
|
||||
self.sample_idx_mapping,
|
||||
self.temperature,
|
||||
self.seeds,
|
||||
self.draft_tokens,
|
||||
self._selector_scores,
|
||||
num_steps=self.num_speculative_steps,
|
||||
top_k=self.selector_top_k,
|
||||
BLOCK_K=block_k,
|
||||
SAMPLE_PROBABILISTIC=self.draft_logits is not None,
|
||||
USE_FP64=self.use_fp64_gumbel,
|
||||
num_warps=1,
|
||||
)
|
||||
|
||||
def _cache_draft_logits(self, candidate_ids: torch.Tensor, num_sample: int) -> None:
|
||||
draft_logits = self.draft_logits
|
||||
assert draft_logits is not None
|
||||
block_k = triton.next_power_of_2(self.selector_top_k)
|
||||
_cache_draft_logits_kernel[(num_sample,)](
|
||||
draft_logits,
|
||||
self._cached_candidate_ids,
|
||||
candidate_ids,
|
||||
self._selector_scores,
|
||||
self.sample_idx_mapping,
|
||||
draft_logits.stride(0),
|
||||
draft_logits.stride(1),
|
||||
num_steps=self.num_speculative_steps,
|
||||
top_k=self.selector_top_k,
|
||||
BLOCK_K=block_k,
|
||||
num_warps=1,
|
||||
)
|
||||
|
||||
def _generate_draft(
|
||||
self,
|
||||
num_reqs: int,
|
||||
@@ -202,22 +227,39 @@ class DFlash2Speculator(DFlashSpeculator):
|
||||
hidden_states.flatten(0, 1)
|
||||
)
|
||||
candidate_ids = candidate_ids.view(
|
||||
num_reqs, self.num_speculative_steps, self.selector_top_k
|
||||
num_reqs, self.num_speculative_steps, self.top_k
|
||||
)
|
||||
unary_logits = unary_logits.view_as(candidate_ids)
|
||||
anchor_token_ids = self.input_buffers.input_ids[self._anchor_indices[:num_reqs]]
|
||||
scores = self.model.model.candidate_selector(
|
||||
scores = self._score_candidates(candidate_ids, unary_logits, hidden_states)
|
||||
self.candidate_sampler.sample(
|
||||
candidate_ids,
|
||||
unary_logits,
|
||||
hidden_states,
|
||||
anchor_token_ids,
|
||||
scores,
|
||||
num_reqs,
|
||||
self.sample_pos,
|
||||
self.sample_idx_mapping,
|
||||
self.temperature,
|
||||
self.seeds,
|
||||
self.draft_tokens,
|
||||
self.draft_logits,
|
||||
self.use_fp64_gumbel,
|
||||
)
|
||||
self._sample_path(candidate_ids, scores, num_reqs)
|
||||
if self.enable_adaptive_verification:
|
||||
self._maybe_predict_acceptance(
|
||||
self._selector_scores[:num_reqs].flatten(0, 1),
|
||||
self.candidate_sampler.scores[:num_reqs].flatten(0, 1),
|
||||
self.sample_idx_mapping[:num_sample],
|
||||
self.sample_col[:num_sample],
|
||||
)
|
||||
if self.draft_logits is not None:
|
||||
self._cache_draft_logits(candidate_ids, num_sample)
|
||||
|
||||
def _score_candidates(
|
||||
self,
|
||||
candidate_ids: torch.Tensor,
|
||||
unary_logits: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
num_reqs = candidate_ids.shape[0]
|
||||
anchor_token_ids = self.input_buffers.input_ids[
|
||||
: num_reqs * self.num_query_per_req : self.num_query_per_req
|
||||
]
|
||||
return self.model.model.candidate_selector(
|
||||
candidate_ids, unary_logits, hidden_states, anchor_token_ids
|
||||
)
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -0,0 +1,97 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch
|
||||
from vllm.v1.worker.gpu.spec_decode.dflash2.speculator import DFlash2Speculator
|
||||
from vllm.v1.worker.gpu.spec_decode.eagle.utils import get_target_lm_head
|
||||
|
||||
|
||||
class LiLiCorrSpeculator(DFlash2Speculator):
|
||||
_speculator_name = "LiLiCorr"
|
||||
_candidate_top_k_key = "lilicorr_candidate_topk"
|
||||
|
||||
def __init__(self, vllm_config: VllmConfig, device: torch.device):
|
||||
super().__init__(vllm_config, device)
|
||||
config = self.draft_model_config.hf_config
|
||||
self.anchor_hidden = torch.zeros(
|
||||
self.max_num_reqs, config.hidden_size, dtype=self.dtype, device=device
|
||||
)
|
||||
self.anchor_valid = torch.zeros(
|
||||
self.max_num_reqs, dtype=torch.bool, device=device
|
||||
)
|
||||
|
||||
def load_draft_model(
|
||||
self, target_model: nn.Module, target_attn_layer_names: set[str]
|
||||
) -> nn.Module:
|
||||
model = super().load_draft_model(target_model, target_attn_layer_names)
|
||||
language_model = (
|
||||
target_model.get_language_model()
|
||||
if hasattr(target_model, "get_language_model")
|
||||
else target_model
|
||||
)
|
||||
inner = getattr(language_model, "model", language_model)
|
||||
embedding = getattr(inner, "embed_tokens", None) or getattr(
|
||||
inner, "embedding", None
|
||||
)
|
||||
lm_head = (
|
||||
model.lm_head
|
||||
if model.has_own_lm_head
|
||||
else get_target_lm_head(target_model, language_model)
|
||||
)
|
||||
if not isinstance(embedding, VocabParallelEmbedding) or not isinstance(
|
||||
lm_head, VocabParallelEmbedding
|
||||
):
|
||||
raise ValueError(
|
||||
"LiLiCorr requires the target input embedding and its selected "
|
||||
"LM head on the draft rank."
|
||||
)
|
||||
if embedding.embedding_dim != model.config.hidden_size:
|
||||
raise ValueError(
|
||||
"LiLiCorr target embedding width must match its trained "
|
||||
"token projection input."
|
||||
)
|
||||
if (
|
||||
embedding.shard_indices.num_added_elements
|
||||
or lm_head.shard_indices.num_added_elements
|
||||
):
|
||||
raise ValueError("LiLiCorr does not support added vocabulary entries.")
|
||||
# Candidate embeddings always come from the target; the checkpoint may
|
||||
# supply its own candidate LM head.
|
||||
self.target_embeddings = embedding
|
||||
model.lm_head = lm_head
|
||||
return model
|
||||
|
||||
def prepare_context_anchor(
|
||||
self, input_batch: InputBatch, num_rejected: torch.Tensor
|
||||
) -> None:
|
||||
num_reqs = input_batch.num_reqs
|
||||
starts = input_batch.query_start_loc[:num_reqs]
|
||||
ends = input_batch.query_start_loc[1 : num_reqs + 1] - num_rejected[:num_reqs]
|
||||
valid = ends > starts
|
||||
indices = (ends - 1).clamp_min(0).long()
|
||||
self.anchor_hidden.zero_()
|
||||
self.anchor_valid.zero_()
|
||||
self.anchor_hidden[:num_reqs].copy_(
|
||||
self.model.model.hidden_norm(self.hidden_states[indices]) * valid[:, None]
|
||||
)
|
||||
self.anchor_valid[:num_reqs].copy_(valid)
|
||||
|
||||
def _score_candidates(
|
||||
self,
|
||||
candidate_ids: torch.Tensor,
|
||||
unary_logits: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
num_reqs = candidate_ids.shape[0]
|
||||
return self.model.model.lilicorr(
|
||||
self.target_embeddings(candidate_ids),
|
||||
unary_logits,
|
||||
hidden_states,
|
||||
self.anchor_hidden[:num_reqs],
|
||||
self.anchor_valid[:num_reqs],
|
||||
)
|
||||
Reference in New Issue
Block a user