[SpecDecode] Add LiLiCorr drafter (#57934)
Signed-off-by: Andrii Skliar <askliar@nvidia.com> Signed-off-by: Andrii Skliar <andreyws96@gmail.com> Co-authored-by: Andrii Skliar <askliar@nvidia.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: Benjamin Chislett <chislett.ben@gmail.com>
This commit is contained in:
co-authored by
Andrii Skliar
Codex
Benjamin Chislett
parent
25b0add7b8
commit
73a78e6f1f
@@ -256,6 +256,8 @@ class LogitsProcessor(PluggableLayer):
|
||||
hidden_states: torch.Tensor,
|
||||
k: int,
|
||||
embedding_bias: torch.Tensor | None = None,
|
||||
*,
|
||||
return_log_probs: bool = False,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Vocab-parallel top-k without all-gathering full logits.
|
||||
|
||||
@@ -263,7 +265,9 @@ class LogitsProcessor(PluggableLayer):
|
||||
the values as well as the global ids. Communication is
|
||||
O(batch * 2k * tp_size) rather than O(batch * vocab_size).
|
||||
|
||||
Scale and soft cap are applied to the k selected values rather than
|
||||
With return_log_probs, values use the full-vocabulary log-partition.
|
||||
|
||||
Otherwise scale and soft cap are applied to the k selected values rather than
|
||||
the whole vocabulary; both are monotonic, so the selection is the same
|
||||
and only k entries are touched.
|
||||
"""
|
||||
@@ -274,12 +278,26 @@ class LogitsProcessor(PluggableLayer):
|
||||
)
|
||||
|
||||
logits = self._apply_head(lm_head, hidden_states, embedding_bias)
|
||||
if return_log_probs:
|
||||
if self.soft_cap is not None:
|
||||
logits = torch.tanh(logits / self.soft_cap) * self.soft_cap
|
||||
if self.scale != 1.0:
|
||||
logits = logits * self.scale
|
||||
|
||||
# Mask out padding entries beyond org_vocab_size on this shard.
|
||||
num_pad = lm_head.shard_indices.num_org_vocab_padding
|
||||
if num_pad > 0:
|
||||
logits[..., -num_pad:] = -float("inf")
|
||||
|
||||
if return_log_probs:
|
||||
log_partition = torch.logsumexp(logits.float(), dim=-1, keepdim=True)
|
||||
if lm_head.tp_size > 1:
|
||||
log_partition = torch.logsumexp(
|
||||
tensor_model_parallel_all_gather(log_partition, dim=-1),
|
||||
dim=-1,
|
||||
keepdim=True,
|
||||
)
|
||||
|
||||
values, ids = _topk(logits, k)
|
||||
# Convert shard-local indices to global vocab indices.
|
||||
ids = ids.to(torch.int64) + lm_head.shard_indices.org_vocab_start_index
|
||||
@@ -291,6 +309,8 @@ class LogitsProcessor(PluggableLayer):
|
||||
ids = ids.gather(-1, selected)
|
||||
|
||||
values = values.float()
|
||||
if return_log_probs:
|
||||
return ids, values - log_partition
|
||||
if self.scale != 1.0:
|
||||
values = values * self.scale
|
||||
if self.soft_cap is not None:
|
||||
|
||||
@@ -0,0 +1,546 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# LiLiCorr head adapted from sgl-project/sglang PR #37462 (Apache-2.0).
|
||||
"""DFlash backbone with the LiLiCorr candidate-lattice correlator."""
|
||||
|
||||
from collections.abc import Iterable
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from torch import nn
|
||||
|
||||
from vllm.compilation.backends import set_model_tag
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.model_executor.layers.linear import (
|
||||
LinearBase,
|
||||
ReplicatedLinear,
|
||||
UnquantizedLinearMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
|
||||
from vllm.model_executor.layers.quantization.modelopt import ModelOptLinearMethod
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import kNvfp4Static
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
|
||||
|
||||
from .nemotron_h import NemotronHMLP
|
||||
from .qwen3_dflash import DFlashQwen3ForCausalLM, DFlashQwen3Model
|
||||
from .qwen3_dflash2 import DFlash2Qwen3DecoderLayer
|
||||
from .utils import AutoWeightsLoader, maybe_prefix
|
||||
|
||||
|
||||
class LiLiCorrLatticeAttention(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
hidden_size: int,
|
||||
num_heads: int,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
if num_heads <= 0 or hidden_size % num_heads != 0:
|
||||
raise ValueError(
|
||||
f"LiLiCorr hidden_size={hidden_size} must be divisible "
|
||||
f"by num_heads={num_heads}."
|
||||
)
|
||||
self.hidden_size = hidden_size
|
||||
self.num_heads = num_heads
|
||||
self.head_dim = self.hidden_size // self.num_heads
|
||||
# Exported lattice QKV parameters stay in the model's floating dtype.
|
||||
self.in_proj_weight = nn.Parameter(
|
||||
torch.empty(3 * hidden_size, hidden_size), requires_grad=False
|
||||
)
|
||||
self.in_proj_bias = nn.Parameter(
|
||||
torch.empty(3 * hidden_size), requires_grad=False
|
||||
)
|
||||
self.out_proj = ReplicatedLinear(
|
||||
hidden_size,
|
||||
hidden_size,
|
||||
return_bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=maybe_prefix(prefix, "out_proj"),
|
||||
)
|
||||
|
||||
def forward(
|
||||
self, hidden_states: torch.Tensor, attention_bias: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
bsz, seq_len, _ = hidden_states.shape
|
||||
qkv = F.linear(hidden_states, self.in_proj_weight, self.in_proj_bias)
|
||||
q, k, v = qkv.chunk(3, dim=-1)
|
||||
shape = (bsz, seq_len, self.num_heads, self.head_dim)
|
||||
q = q.view(shape).transpose(1, 2)
|
||||
k = k.view(shape).transpose(1, 2)
|
||||
v = v.view(shape).transpose(1, 2)
|
||||
# Attention and MMEncoderAttention cannot accept this learned per-head
|
||||
# additive bias. The full candidate lattice has no KV-cache state.
|
||||
out = F.scaled_dot_product_attention(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
attn_mask=attention_bias,
|
||||
)
|
||||
return self.out_proj(
|
||||
out.transpose(1, 2).reshape(bsz, seq_len, self.hidden_size)
|
||||
)
|
||||
|
||||
|
||||
class LiLiCorrLayer(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
hidden_size: int,
|
||||
num_heads: int,
|
||||
mlp_ratio: float,
|
||||
rms_norm_eps: float,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.attn_norm = nn.RMSNorm(hidden_size, eps=rms_norm_eps)
|
||||
self.attn = LiLiCorrLatticeAttention(
|
||||
hidden_size, num_heads, quant_config, maybe_prefix(prefix, "attn")
|
||||
)
|
||||
self.mlp_norm = nn.RMSNorm(hidden_size, eps=rms_norm_eps)
|
||||
mlp_hidden_size = int(hidden_size * mlp_ratio)
|
||||
self.mlp = NemotronHMLP(
|
||||
config=None,
|
||||
hidden_size=hidden_size,
|
||||
intermediate_size=mlp_hidden_size,
|
||||
hidden_act="silu",
|
||||
bias=True,
|
||||
quant_config=quant_config,
|
||||
prefix=maybe_prefix(prefix, "mlp"),
|
||||
)
|
||||
|
||||
def forward(
|
||||
self, hidden_states: torch.Tensor, attention_bias: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
hidden_states = hidden_states + self.attn(
|
||||
self.attn_norm(hidden_states), attention_bias
|
||||
)
|
||||
return hidden_states + self.mlp(self.mlp_norm(hidden_states))
|
||||
|
||||
|
||||
@support_torch_compile
|
||||
class LiLiCorrHead(nn.Module):
|
||||
num_candidate_features = 5
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
model_hidden_size: int,
|
||||
block_size: int,
|
||||
rms_norm_eps: float,
|
||||
config: dict[str, Any],
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
hidden_size = config["lilicorr_hidden_size"] or model_hidden_size
|
||||
self.block_size = block_size
|
||||
self.num_candidate_slots = block_size - 1
|
||||
self.candidate_topk = config["lilicorr_candidate_topk"]
|
||||
if (
|
||||
self.candidate_topk <= 0
|
||||
or self.candidate_topk > 16
|
||||
or self.candidate_topk & (self.candidate_topk - 1)
|
||||
):
|
||||
raise ValueError("LiLiCorr candidate_topk must be a power of two <= 16.")
|
||||
self.hidden_size = hidden_size
|
||||
self.num_heads = config["lilicorr_num_heads"]
|
||||
self.mlp_ratio = config["lilicorr_mlp_ratio"]
|
||||
self.factor_dim = config["lilicorr_factor_dim"]
|
||||
self.vector_eps = config["lilicorr_vector_eps"]
|
||||
self.logit_scale = config["lilicorr_logit_scale"]
|
||||
# Candidate IDs and embeddings are replicated across TP ranks, so each
|
||||
# rank scores the same lattice; the MLPs use standard tensor parallelism.
|
||||
self.token_proj = (
|
||||
nn.Identity()
|
||||
if model_hidden_size == hidden_size
|
||||
else ReplicatedLinear(
|
||||
model_hidden_size,
|
||||
hidden_size,
|
||||
return_bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=maybe_prefix(prefix, "token_proj"),
|
||||
)
|
||||
)
|
||||
self.pass_hidden_proj = ReplicatedLinear(
|
||||
model_hidden_size,
|
||||
hidden_size,
|
||||
return_bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=maybe_prefix(prefix, "pass_hidden_proj"),
|
||||
)
|
||||
self.feature_norm = nn.LayerNorm(self.num_candidate_features)
|
||||
self.feature_mlp = NemotronHMLP(
|
||||
config=None,
|
||||
hidden_size=self.num_candidate_features,
|
||||
intermediate_size=hidden_size,
|
||||
hidden_act="silu",
|
||||
output_size=hidden_size,
|
||||
bias=True,
|
||||
quant_config=quant_config,
|
||||
prefix=maybe_prefix(prefix, "feature_mlp"),
|
||||
)
|
||||
self.slot_embedding = nn.Parameter(
|
||||
torch.zeros(1, 1, self.num_candidate_slots, 1, hidden_size)
|
||||
)
|
||||
self.rank_embedding = nn.Parameter(
|
||||
torch.zeros(1, 1, 1, self.candidate_topk, hidden_size)
|
||||
)
|
||||
self.relative_slot_bias = nn.Parameter(
|
||||
torch.zeros(self.num_heads, 2 * self.block_size - 1)
|
||||
)
|
||||
self.same_slot_bias = nn.Parameter(torch.zeros(self.num_heads))
|
||||
self.context_proj = ReplicatedLinear(
|
||||
model_hidden_size,
|
||||
hidden_size,
|
||||
return_bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=maybe_prefix(prefix, "context_proj"),
|
||||
)
|
||||
self.layers = nn.ModuleList(
|
||||
[
|
||||
LiLiCorrLayer(
|
||||
hidden_size=hidden_size,
|
||||
num_heads=self.num_heads,
|
||||
mlp_ratio=self.mlp_ratio,
|
||||
rms_norm_eps=rms_norm_eps,
|
||||
quant_config=quant_config,
|
||||
prefix=maybe_prefix(prefix, f"layers.{i}"),
|
||||
)
|
||||
for i in range(config["lilicorr_num_layers"])
|
||||
]
|
||||
)
|
||||
self.output_norm = nn.RMSNorm(hidden_size, eps=rms_norm_eps)
|
||||
self.anchor_norm = nn.RMSNorm(hidden_size, eps=rms_norm_eps)
|
||||
self.factor_input_proj = ReplicatedLinear(
|
||||
hidden_size * 3,
|
||||
hidden_size,
|
||||
return_bias=False,
|
||||
quant_config=None,
|
||||
prefix=maybe_prefix(prefix, "factor_input_proj"),
|
||||
)
|
||||
# Each candidate gets outgoing and incoming vectors. After normalization,
|
||||
# their dot product scores a transition from slot s to slot s+1:
|
||||
# pair_score[s, i, j] = dot(out[s, i], incoming[s+1, j])
|
||||
# The anchor supplies the outgoing vector for the first slot.
|
||||
self.out_head = ReplicatedLinear(
|
||||
hidden_size,
|
||||
self.factor_dim,
|
||||
return_bias=False,
|
||||
quant_config=None,
|
||||
prefix=maybe_prefix(prefix, "out_head"),
|
||||
)
|
||||
self.in_head = ReplicatedLinear(
|
||||
hidden_size,
|
||||
self.factor_dim,
|
||||
return_bias=False,
|
||||
quant_config=None,
|
||||
prefix=maybe_prefix(prefix, "in_head"),
|
||||
)
|
||||
self.anchor_out_head = ReplicatedLinear(
|
||||
hidden_size,
|
||||
self.factor_dim,
|
||||
return_bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=maybe_prefix(prefix, "anchor_out_head"),
|
||||
)
|
||||
self._fused_edge_weight: torch.Tensor | None = None
|
||||
self._fused_edge_bias: torch.Tensor | None = None
|
||||
self._factor_input_splits: tuple[torch.Tensor, ...]
|
||||
self._attn_bias: torch.Tensor | None = None
|
||||
self._rank_frac_col: torch.Tensor
|
||||
self._is_top1_col: torch.Tensor
|
||||
|
||||
@torch.no_grad()
|
||||
def materialize_inference_buffers(
|
||||
self, device: torch.device, dtype: torch.dtype
|
||||
) -> None:
|
||||
"""Build inference-only views/copies after checkpoint weights are loaded."""
|
||||
topk = self.candidate_topk
|
||||
self._attn_bias = self._build_attention_bias(device=device, dtype=dtype)
|
||||
# Both projections consume the same candidate features f:
|
||||
# out = W_out @ f + b_out
|
||||
# incoming = W_in @ f + b_in
|
||||
# Stack weights and biases to compute both in one linear call. score()
|
||||
# splits the result and normalizes the two vectors separately.
|
||||
self._fused_edge_weight = (
|
||||
torch.cat([self.out_head.weight, self.in_head.weight], dim=0)
|
||||
.to(device=device, dtype=dtype)
|
||||
.contiguous()
|
||||
)
|
||||
self._fused_edge_bias = (
|
||||
torch.cat([self.out_head.bias, self.in_head.bias], dim=0)
|
||||
.to(device=device, dtype=dtype)
|
||||
.contiguous()
|
||||
)
|
||||
# For candidate state x and request anchor a, the trained projection is:
|
||||
# W @ concat(x, a, x*a) + b
|
||||
# Split W into input blocks to evaluate the same expression as:
|
||||
# W_x @ x + W_a @ a + W_cross @ (x*a) + b
|
||||
# Here x*a is elementwise. This avoids concatenating inputs and reuses
|
||||
# W_a @ a across candidates, adding the original bias only once.
|
||||
weight = self.factor_input_proj.weight
|
||||
hdim = self.hidden_size
|
||||
self._factor_input_splits = (
|
||||
weight[:, :hdim].contiguous(),
|
||||
weight[:, hdim : 2 * hdim].contiguous(),
|
||||
weight[:, 2 * hdim :].contiguous(),
|
||||
)
|
||||
if topk > 1:
|
||||
rank_frac = torch.arange(topk, device=device, dtype=torch.float32).view(
|
||||
1, 1, topk
|
||||
) / float(topk - 1)
|
||||
else:
|
||||
rank_frac = torch.zeros(1, 1, topk, device=device, dtype=torch.float32)
|
||||
is_top1 = torch.zeros(1, 1, topk, device=device, dtype=torch.float32)
|
||||
is_top1[..., 0] = 1.0
|
||||
self._rank_frac_col = rank_frac.contiguous()
|
||||
self._is_top1_col = is_top1.contiguous()
|
||||
|
||||
def _build_attention_bias(
|
||||
self, *, device: torch.device, dtype: torch.dtype
|
||||
) -> torch.Tensor:
|
||||
topk = self.candidate_topk
|
||||
slot_ids = torch.arange(
|
||||
self.num_candidate_slots, device=device, dtype=torch.long
|
||||
).repeat_interleave(topk)
|
||||
# Attention flattens [slot, candidate] into one sequence. With topk=2,
|
||||
# the candidate slot IDs are [0, 0, 1, 1, ...]. Each head learns:
|
||||
# bias(q, k) = relative_bias[slot(q) - slot(k) + offset]
|
||||
# + same_slot_bias * (slot(q) == slot(k))
|
||||
# The same-slot term includes distinct candidates at the same position.
|
||||
# This is an additive preference, not a causal mask. Keep the trained
|
||||
# offset = block_size-1 when score() selects a shorter draft's prefix,
|
||||
# so each distance continues to use the same learned weight.
|
||||
rel = slot_ids.view(-1, 1) - slot_ids.view(1, -1)
|
||||
bias = self.relative_slot_bias[:, rel + self.block_size - 1]
|
||||
same_slot = slot_ids.view(-1, 1) == slot_ids.view(1, -1)
|
||||
bias = bias + same_slot.unsqueeze(0).to(
|
||||
dtype=bias.dtype
|
||||
) * self.same_slot_bias.view(-1, 1, 1)
|
||||
return bias.to(device=device, dtype=dtype).contiguous()
|
||||
|
||||
def score(
|
||||
self,
|
||||
*,
|
||||
token_embeddings: torch.Tensor,
|
||||
candidate_log_probs: torch.Tensor,
|
||||
pass_hidden: torch.Tensor,
|
||||
anchor_hidden: torch.Tensor,
|
||||
anchor_valid: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
if self._attn_bias is None:
|
||||
raise RuntimeError(
|
||||
"Call materialize_inference_buffers() after loading the LiLiCorr head."
|
||||
)
|
||||
bsz, n_slots, topk = candidate_log_probs.shape
|
||||
if not 1 <= n_slots <= self.num_candidate_slots:
|
||||
raise ValueError(
|
||||
f"LiLiCorr supports 1..{self.num_candidate_slots} candidate slots, "
|
||||
f"got {n_slots}."
|
||||
)
|
||||
if topk != self.candidate_topk:
|
||||
raise ValueError(
|
||||
f"LiLiCorr expects candidate_topk={self.candidate_topk}, got {topk}."
|
||||
)
|
||||
proj_dtype = self.slot_embedding.dtype
|
||||
if token_embeddings.dtype != proj_dtype:
|
||||
token_embeddings = token_embeddings.to(proj_dtype)
|
||||
if pass_hidden.dtype != proj_dtype:
|
||||
pass_hidden = pass_hidden.to(proj_dtype)
|
||||
token_states = self.token_proj(token_embeddings)
|
||||
pass_states = self.pass_hidden_proj(pass_hidden).unsqueeze(-2)
|
||||
log_probs = candidate_log_probs.float()
|
||||
features = torch.stack(
|
||||
[
|
||||
log_probs,
|
||||
log_probs.exp(),
|
||||
log_probs - log_probs.max(dim=-1, keepdim=True).values,
|
||||
self._rank_frac_col.expand_as(log_probs),
|
||||
self._is_top1_col.expand_as(log_probs),
|
||||
],
|
||||
dim=-1,
|
||||
)
|
||||
hidden_states = token_states + pass_states
|
||||
hidden_states = hidden_states + self.feature_mlp(
|
||||
self.feature_norm(features.to(dtype=token_states.dtype))
|
||||
)
|
||||
hidden_states = hidden_states + self.slot_embedding[:, 0, :n_slots]
|
||||
hidden_states = hidden_states + self.rank_embedding[:, 0]
|
||||
hidden_states = hidden_states.reshape(bsz, n_slots * topk, self.hidden_size)
|
||||
anchor_state = self.context_proj(anchor_hidden)
|
||||
anchor_state = anchor_state * anchor_valid[:, None].to(anchor_state.dtype)
|
||||
# Shorter drafts use the learned prefix; keep checkpoint parameter shapes.
|
||||
lattice = n_slots * topk
|
||||
attention_bias = self._attn_bias[None, :, :lattice, :lattice]
|
||||
for layer in self.layers:
|
||||
hidden_states = layer(hidden_states, attention_bias)
|
||||
hidden_states = self.output_norm(hidden_states).reshape(
|
||||
bsz, n_slots, topk, self.hidden_size
|
||||
)
|
||||
anchor_state = self.anchor_norm(anchor_state)
|
||||
w_self, w_anchor, w_cross = self._factor_input_splits
|
||||
anchor_row = anchor_state[:, None, None, :]
|
||||
pre = F.linear(hidden_states, w_self, self.factor_input_proj.bias)
|
||||
pre = pre + F.linear(anchor_row, w_anchor)
|
||||
pre = pre + F.linear(hidden_states * anchor_row, w_cross)
|
||||
factor_hidden = F.silu(pre)
|
||||
edges = F.linear(factor_hidden, self._fused_edge_weight, self._fused_edge_bias)
|
||||
out_vec, in_vec = F.normalize(
|
||||
edges.unflatten(-1, (2, self.factor_dim)), dim=-1, eps=self.vector_eps
|
||||
).unbind(-2)
|
||||
anchor_out = F.normalize(
|
||||
self.anchor_out_head(anchor_state), dim=-1, eps=self.vector_eps
|
||||
)
|
||||
# start_scores[b, j] scores anchor -> candidate j in slot zero.
|
||||
# pair_scores[b, s, i, j] scores candidate i in slot s -> candidate j
|
||||
# in slot s+1. Normalized factor dot products encode learned compatibility;
|
||||
# forward() scales them into logits for the candidate-path selector.
|
||||
start_scores = (anchor_out[:, None, :] * in_vec[:, 0, :, :]).sum(dim=-1)
|
||||
pair_scores = torch.matmul(out_vec[:, :-1], in_vec[:, 1:].transpose(-1, -2))
|
||||
return (start_scores, pair_scores)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
token_embeddings: torch.Tensor,
|
||||
candidate_log_probs: torch.Tensor,
|
||||
pass_hidden: torch.Tensor,
|
||||
anchor_hidden: torch.Tensor,
|
||||
anchor_valid: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
start, pairs = self.score(
|
||||
token_embeddings=token_embeddings,
|
||||
candidate_log_probs=candidate_log_probs,
|
||||
pass_hidden=pass_hidden,
|
||||
anchor_hidden=anchor_hidden,
|
||||
anchor_valid=anchor_valid,
|
||||
)
|
||||
start = self.logit_scale * start.float()
|
||||
pairs = self.logit_scale * pairs.float()
|
||||
# The shared walk starts at predecessor index zero; all first rows agree.
|
||||
first = start[:, None, None, :].expand(-1, 1, self.candidate_topk, -1)
|
||||
return torch.cat((first, pairs), dim=1)
|
||||
|
||||
|
||||
class LiLiCorr(DFlashQwen3Model):
|
||||
def __init__(
|
||||
self, *, vllm_config: VllmConfig, start_layer_id: int = 0, prefix: str = ""
|
||||
):
|
||||
spec = vllm_config.speculative_config
|
||||
assert spec is not None
|
||||
config = spec.draft_model_config.hf_config
|
||||
draft_config = config.dflash_config
|
||||
if not draft_config.get("lilicorr_enabled", True):
|
||||
raise ValueError("LiLiCorr requires lilicorr_enabled=True.")
|
||||
block_size = draft_config.get("block_size", getattr(config, "block_size", 0))
|
||||
if not 1 <= spec.num_speculative_tokens < block_size:
|
||||
raise ValueError(
|
||||
"LiLiCorr requires 1 <= num_speculative_tokens < trained block_size."
|
||||
)
|
||||
taps = draft_config.get("conv_kernel_size", 0)
|
||||
groups = draft_config.get("conv_group_size", 0)
|
||||
if taps < 0 or groups < 0 or bool(taps) != bool(groups):
|
||||
raise ValueError(
|
||||
"LiLiCorr convolution geometry must enable both taps and group size."
|
||||
)
|
||||
if taps:
|
||||
self.decoder_layer_cls = DFlash2Qwen3DecoderLayer
|
||||
super().__init__(
|
||||
vllm_config=vllm_config, start_layer_id=start_layer_id, prefix=prefix
|
||||
)
|
||||
with set_model_tag("lilicorr_head"):
|
||||
self.lilicorr = LiLiCorrHead(
|
||||
model_hidden_size=config.hidden_size,
|
||||
block_size=block_size,
|
||||
rms_norm_eps=config.rms_norm_eps,
|
||||
config=draft_config,
|
||||
quant_config=self.quant_config,
|
||||
prefix=maybe_prefix(prefix, "lilicorr"),
|
||||
)
|
||||
|
||||
|
||||
class LiLiCorrForCausalLM(DFlashQwen3ForCausalLM):
|
||||
model_cls = LiLiCorr
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
spec = vllm_config.speculative_config
|
||||
assert spec is not None
|
||||
super().__init__(vllm_config=vllm_config, prefix=prefix)
|
||||
if self.draft_id_to_target_id is not None:
|
||||
raise ValueError("LiLiCorr candidates require the full target vocabulary.")
|
||||
self.has_own_lm_head = bool(getattr(self.config, "has_own_lm_head", False))
|
||||
# The correlator consumes raw candidate-head log-softmax features.
|
||||
self.candidate_logits_processor = LogitsProcessor(self.config.vocab_size)
|
||||
|
||||
def compute_candidates(
|
||||
self, hidden_states: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
return self.candidate_logits_processor.get_top_k_tokens(
|
||||
self.lm_head,
|
||||
hidden_states,
|
||||
self.model.lilicorr.candidate_topk,
|
||||
return_log_probs=True,
|
||||
)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
|
||||
quantized_metadata: set[str] = set()
|
||||
for name, module in self.named_modules():
|
||||
name = name.removeprefix("model.")
|
||||
if not isinstance(module, (LinearBase, ParallelLMHead)) or isinstance(
|
||||
module.quant_method, UnquantizedLinearMethod
|
||||
):
|
||||
continue
|
||||
required = {"weight", "bias"}
|
||||
method = module.quant_method
|
||||
if (
|
||||
isinstance(method, ModelOptLinearMethod)
|
||||
and method.spec.weight == kNvfp4Static
|
||||
):
|
||||
# NVFP4 scales are trained checkpoint data. Only W4A16's
|
||||
# deprecated, unused input_scale may be absent.
|
||||
required.update(("weight_scale", "weight_scale_2"))
|
||||
if method.spec.activation is not None:
|
||||
required.add("input_scale")
|
||||
quantized_metadata.update(
|
||||
f"{name}.{parameter}"
|
||||
for parameter, _ in module.named_parameters(recurse=False)
|
||||
if parameter not in required
|
||||
)
|
||||
expected = {
|
||||
name.removeprefix("model.")
|
||||
for name, _ in self.named_parameters()
|
||||
if (
|
||||
name.startswith("model.lilicorr.")
|
||||
or ".attention_conv." in name
|
||||
or ".mlp_conv." in name
|
||||
or (self.has_own_lm_head and name.startswith("lm_head."))
|
||||
)
|
||||
and name.removeprefix("model.") not in quantized_metadata
|
||||
}
|
||||
seen: set[str] = set()
|
||||
head_weights: list[tuple[str, torch.Tensor]] = []
|
||||
|
||||
def normalized_weights():
|
||||
for name, value in weights:
|
||||
name = name.removeprefix("model.")
|
||||
seen.add(name)
|
||||
# The backbone stacks up_proj into gate_up_proj. Load the
|
||||
# nongated correlator separately to avoid that transformation.
|
||||
if name.startswith("lilicorr."):
|
||||
head_weights.append((name.removeprefix("lilicorr."), value))
|
||||
else:
|
||||
yield name, value
|
||||
|
||||
super().load_weights(normalized_weights())
|
||||
AutoWeightsLoader(self.model.lilicorr).load_weights(head_weights)
|
||||
missing = expected - seen
|
||||
if missing:
|
||||
raise ValueError(
|
||||
f"LiLiCorr checkpoint coverage mismatch: missing={sorted(missing)}"
|
||||
)
|
||||
head = self.model.lilicorr
|
||||
parameter = head.slot_embedding
|
||||
head.materialize_inference_buffers(parameter.device, parameter.dtype)
|
||||
@@ -34,6 +34,7 @@ from vllm.model_executor.models.interfaces import (
|
||||
HasInnerState,
|
||||
IsHybrid,
|
||||
MultiModalEmbeddings,
|
||||
SupportsEagle3,
|
||||
SupportsLoRA,
|
||||
SupportsMultiModal,
|
||||
SupportsMultiModalPruning,
|
||||
@@ -928,6 +929,7 @@ class NemotronH_Nano_VL_V2(
|
||||
nn.Module,
|
||||
HasInnerState,
|
||||
IsHybrid,
|
||||
SupportsEagle3,
|
||||
SupportsMultiModal,
|
||||
SupportsMultiModalPruning,
|
||||
SupportsLoRA,
|
||||
|
||||
@@ -34,7 +34,7 @@ from vllm.config.parallel import ParallelConfig
|
||||
from vllm.distributed import get_ep_group, get_tensor_model_parallel_world_size
|
||||
from vllm.distributed.communication_op import tensor_model_parallel_all_gather
|
||||
from vllm.distributed.parallel_state import get_pp_group
|
||||
from vllm.model_executor.layers.activation import ReLUSquaredActivation
|
||||
from vllm.model_executor.layers.activation import get_act_fn
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoEFactory,
|
||||
@@ -93,7 +93,7 @@ from vllm.transformers_utils.configs.nemotron_h import NemotronHConfig
|
||||
class NemotronHMLP(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
config: NemotronHConfig,
|
||||
config: NemotronHConfig | None,
|
||||
hidden_size: int,
|
||||
intermediate_size: int,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
@@ -101,6 +101,9 @@ class NemotronHMLP(nn.Module):
|
||||
reduce_results: bool = True,
|
||||
is_sequence_parallel: bool = False,
|
||||
prefix: str = "",
|
||||
*,
|
||||
hidden_act: str = "relu2",
|
||||
output_size: int | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
|
||||
@@ -114,14 +117,14 @@ class NemotronHMLP(nn.Module):
|
||||
)
|
||||
self.down_proj = RowParallelLinear(
|
||||
input_size=intermediate_size,
|
||||
output_size=hidden_size,
|
||||
output_size=hidden_size if output_size is None else output_size,
|
||||
bias=bias,
|
||||
quant_config=quant_config,
|
||||
reduce_results=reduce_results,
|
||||
disable_tp=is_sequence_parallel,
|
||||
prefix=f"{prefix}.down_proj",
|
||||
)
|
||||
self.act_fn = ReLUSquaredActivation()
|
||||
self.act_fn = get_act_fn(hidden_act)
|
||||
|
||||
def forward(self, x: torch.Tensor):
|
||||
x, _ = self.up_proj(x)
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
import io
|
||||
from collections.abc import Iterable
|
||||
|
||||
import regex as re
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from torch import nn
|
||||
@@ -23,6 +24,7 @@ from vllm.model_executor.layers.linear import (
|
||||
QKVParallelLinear,
|
||||
ReplicatedLinear,
|
||||
RowParallelLinear,
|
||||
UnquantizedLinearMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
|
||||
@@ -54,6 +56,31 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
_SLIDING_ATTENTION = "sliding_attention"
|
||||
_DRAFT_LAYER_PATTERN = re.compile(r"(?<![A-Za-z0-9_.])layers\.(\d+)")
|
||||
|
||||
|
||||
def _add_global_draft_layer_exclusions(
|
||||
quant_config: QuantizationConfig | None,
|
||||
start_layer_id: int,
|
||||
num_hidden_layers: int,
|
||||
) -> None:
|
||||
"""Add runtime layer aliases for checkpoint-local quant exclusions."""
|
||||
if quant_config is None or start_layer_id == 0:
|
||||
return
|
||||
exclusions = getattr(quant_config, "exclude_modules", None)
|
||||
if not isinstance(exclusions, list):
|
||||
return
|
||||
|
||||
def offset_local_layer(match: re.Match[str]) -> str:
|
||||
layer_idx = int(match.group(1))
|
||||
if layer_idx >= num_hidden_layers:
|
||||
return match.group(0)
|
||||
return f"layers.{layer_idx + start_layer_id}"
|
||||
|
||||
for exclusion in tuple(exclusions):
|
||||
global_exclusion = _DRAFT_LAYER_PATTERN.sub(offset_local_layer, exclusion)
|
||||
if global_exclusion != exclusion and global_exclusion not in exclusions:
|
||||
exclusions.append(global_exclusion)
|
||||
|
||||
|
||||
def _dflash_layer_causal(config: Qwen3Config, layer_idx: int) -> bool:
|
||||
@@ -394,6 +421,9 @@ class DFlashQwen3Model(nn.Module):
|
||||
self.config = vllm_config.speculative_config.draft_model_config.hf_config
|
||||
self.vocab_size = self.config.vocab_size
|
||||
self.quant_config = get_draft_quant_config(vllm_config)
|
||||
_add_global_draft_layer_exclusions(
|
||||
self.quant_config, start_layer_id, self.config.num_hidden_layers
|
||||
)
|
||||
|
||||
drafter_config = getattr(self.config, "eagle_config", {})
|
||||
drafter_config.update(getattr(self.config, "dflash_config", {}))
|
||||
@@ -473,14 +503,25 @@ class DFlashQwen3Model(nn.Module):
|
||||
has_bias: bool,
|
||||
) -> None:
|
||||
self._hidden_norm_weight = self.hidden_norm.weight.data
|
||||
self._context_qkv_projs = [a.qkv_proj for a in layers_attn]
|
||||
self._context_q_sizes = [a.q_size for a in layers_attn]
|
||||
|
||||
# KV projection weights: [num_layers * 2 * kv_size, hidden_size]
|
||||
kv_weights = [a.qkv_proj.weight[a.q_size :] for a in layers_attn]
|
||||
self._fused_kv_weight = torch.cat(kv_weights, dim=0)
|
||||
if has_bias:
|
||||
kv_biases = [a.qkv_proj.bias[a.q_size :] for a in layers_attn]
|
||||
self._fused_kv_bias: torch.Tensor | None = torch.cat(kv_biases, dim=0)
|
||||
if all(
|
||||
isinstance(proj.quant_method, UnquantizedLinearMethod)
|
||||
for proj in self._context_qkv_projs
|
||||
):
|
||||
# KV projection weights: [num_layers * 2 * kv_size, hidden_size]
|
||||
kv_weights = [a.qkv_proj.weight[a.q_size :] for a in layers_attn]
|
||||
self._fused_kv_weight: torch.Tensor | None = torch.cat(kv_weights, dim=0)
|
||||
if has_bias:
|
||||
kv_biases = [a.qkv_proj.bias[a.q_size :] for a in layers_attn]
|
||||
self._fused_kv_bias: torch.Tensor | None = torch.cat(kv_biases, dim=0)
|
||||
else:
|
||||
self._fused_kv_bias = None
|
||||
else:
|
||||
# Quantized linear weights may use packed storage that cannot be
|
||||
# consumed by F.linear. Run each projection through its quant method.
|
||||
self._fused_kv_weight = None
|
||||
self._fused_kv_bias = None
|
||||
|
||||
# K-norm weights stacked into one contiguous [num_layers, head_dim]
|
||||
@@ -548,17 +589,26 @@ class DFlashQwen3Model(nn.Module):
|
||||
self._hidden_norm_weight,
|
||||
self._rms_norm_eps,
|
||||
)
|
||||
all_kv_flat = F.linear(
|
||||
normed_context_states, self._fused_kv_weight, self._fused_kv_bias
|
||||
)
|
||||
if self._fused_kv_weight is not None:
|
||||
all_kv_flat = F.linear(
|
||||
normed_context_states, self._fused_kv_weight, self._fused_kv_bias
|
||||
)
|
||||
all_kv = all_kv_flat.view(num_ctx, num_layers, 2, num_kv_heads, head_dim)
|
||||
else:
|
||||
layer_kv = []
|
||||
for proj, q_size in zip(
|
||||
self._context_qkv_projs, self._context_q_sizes, strict=True
|
||||
):
|
||||
qkv, _ = proj(normed_context_states)
|
||||
layer_kv.append(
|
||||
qkv[:, q_size:].view(num_ctx, 2, num_kv_heads, head_dim)
|
||||
)
|
||||
all_kv = torch.stack(layer_kv, dim=1)
|
||||
|
||||
# Single contiguous copy that separates K/V and transposes to
|
||||
# layer-major layout. Result: [2, L, num_ctx, nkv, hd] contiguous.
|
||||
# layer-major layout. Result: [2, L, num_ctx, nkv, hd] contiguous.
|
||||
# Indexing dim-0 gives contiguous [L, num_ctx, nkv, hd] for K and V.
|
||||
all_kv = (
|
||||
all_kv_flat.view(num_ctx, num_layers, 2, num_kv_heads, head_dim)
|
||||
.permute(2, 1, 0, 3, 4)
|
||||
.contiguous()
|
||||
)
|
||||
all_kv = all_kv.permute(2, 1, 0, 3, 4).contiguous()
|
||||
all_k = all_kv[0] # [L, num_ctx, nkv, hd], contiguous
|
||||
all_v = all_kv[1] # [L, num_ctx, nkv, hd], contiguous
|
||||
return all_k, all_v
|
||||
@@ -724,6 +774,11 @@ class DFlashQwen3ForCausalLM(Qwen3ForCausalLM):
|
||||
self.lm_head = ParallelLMHead(
|
||||
self.config.draft_vocab_size,
|
||||
self.config.hidden_size,
|
||||
quant_config=(
|
||||
get_draft_quant_config(vllm_config)
|
||||
if getattr(self.config, "has_own_lm_head", False)
|
||||
else None
|
||||
),
|
||||
prefix=maybe_prefix(prefix, "lm_head"),
|
||||
)
|
||||
self.logits_processor = LogitsProcessor(
|
||||
|
||||
@@ -176,6 +176,7 @@ class DFlashGroupedConv(nn.Module):
|
||||
block_size: int,
|
||||
params_dtype: torch.dtype,
|
||||
prefix: str,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
if hidden_size % group_size:
|
||||
@@ -195,7 +196,7 @@ class DFlashGroupedConv(nn.Module):
|
||||
2 * taps * self.num_groups,
|
||||
bias=False,
|
||||
params_dtype=params_dtype,
|
||||
quant_config=None,
|
||||
quant_config=quant_config,
|
||||
prefix=maybe_prefix(prefix, "kernel_projection"),
|
||||
return_bias=False,
|
||||
)
|
||||
@@ -254,6 +255,7 @@ class DFlash2Qwen3DecoderLayer(DFlashQwen3DecoderLayer):
|
||||
# Query tokens per request: the bonus token plus the mask tokens.
|
||||
block_size=1 + speculative_config.num_speculative_tokens,
|
||||
params_dtype=vllm_config.model_config.dtype,
|
||||
quant_config=quant_config,
|
||||
)
|
||||
self.attention_conv = DFlashGroupedConv(
|
||||
**conv_args, prefix=maybe_prefix(prefix, "attention_conv")
|
||||
|
||||
@@ -633,6 +633,7 @@ _SPECULATIVE_DECODING_MODELS = {
|
||||
"EagleLlama4ForCausalLM": ("llama4_eagle", "EagleLlama4ForCausalLM"),
|
||||
"EagleMiniCPMForCausalLM": ("minicpm_eagle", "EagleMiniCPMForCausalLM"),
|
||||
"DFlashDraftModel": ("qwen3_dflash", "DFlashQwen3ForCausalLM"),
|
||||
"LiLiCorrDraftModel": ("lilicorr", "LiLiCorrForCausalLM"),
|
||||
"DFlash2DraftModel": ("qwen3_dflash2", "DFlash2Qwen3ForCausalLM"),
|
||||
# Muse Glimmer's DFlash draft head, reusing the generic qwen3_dflash
|
||||
# implementation. EAGLEConfig rewrites a dflash draft's architecture to
|
||||
|
||||
Reference in New Issue
Block a user