From 73a78e6f1f38e280986b81e0f2a9aa5e1ee6fe47 Mon Sep 17 00:00:00 2001 From: Andrii Skliar Date: Fri, 25 Sep 2026 16:08:28 +0200 Subject: [PATCH] [SpecDecode] Add LiLiCorr drafter (#57934) Signed-off-by: Andrii Skliar Signed-off-by: Andrii Skliar Co-authored-by: Andrii Skliar Co-authored-by: Codex Co-authored-by: Benjamin Chislett --- docs/features/speculative_decoding/README.md | 1 + .../features/speculative_decoding/lilicorr.md | 77 ++ tests/models/registry.py | 7 + tests/test_config.py | 23 +- tests/v1/spec_decode/test_dflash2.py | 185 ++++- tests/v1/spec_decode/test_lilicorr.py | 777 ++++++++++++++++++ vllm/config/vllm.py | 16 +- .../model_executor/layers/logits_processor.py | 22 +- vllm/model_executor/models/lilicorr.py | 546 ++++++++++++ .../model_executor/models/nano_nemotron_vl.py | 2 + vllm/model_executor/models/nemotron_h.py | 11 +- vllm/model_executor/models/qwen3_dflash.py | 85 +- vllm/model_executor/models/qwen3_dflash2.py | 4 +- vllm/model_executor/models/registry.py | 1 + vllm/transformers_utils/configs/eagle.py | 4 +- vllm/v1/worker/gpu/spec_decode/__init__.py | 6 + .../gpu/spec_decode/dflash/speculator.py | 6 + .../v1/worker/gpu/spec_decode/dflash/utils.py | 8 +- .../gpu/spec_decode/dflash2/speculator.py | 174 ++-- .../gpu/spec_decode/lilicorr/__init__.py | 2 + .../gpu/spec_decode/lilicorr/speculator.py | 97 +++ 21 files changed, 1944 insertions(+), 110 deletions(-) create mode 100644 docs/features/speculative_decoding/lilicorr.md create mode 100644 tests/v1/spec_decode/test_lilicorr.py create mode 100644 vllm/model_executor/models/lilicorr.py create mode 100644 vllm/v1/worker/gpu/spec_decode/lilicorr/__init__.py create mode 100644 vllm/v1/worker/gpu/spec_decode/lilicorr/speculator.py diff --git a/docs/features/speculative_decoding/README.md b/docs/features/speculative_decoding/README.md index f4417c7fde..7e2c36d293 100644 --- a/docs/features/speculative_decoding/README.md +++ b/docs/features/speculative_decoding/README.md @@ -9,6 +9,7 @@ To train your own draft models for optimized speculative decoding, see [vllm-pro vLLM supports a variety of methods of speculative decoding. Model-based methods such as EAGLE, MTP, draft models, PARD and MLP provide the best latency reduction, while simpler methods such as n-gram and suffix decoding provide modest speedups without increasing workload during peak traffic. - [EAGLE](eagle.md) +- [LiLiCorr](lilicorr.md) - [Multi-Token Prediction (MTP)](mtp.md) - [Draft Model](draft_model.md) - [Parallel Draft Model (PARD)](parallel_draft_model.md) diff --git a/docs/features/speculative_decoding/lilicorr.md b/docs/features/speculative_decoding/lilicorr.md new file mode 100644 index 0000000000..5d45d3c51f --- /dev/null +++ b/docs/features/speculative_decoding/lilicorr.md @@ -0,0 +1,77 @@ +# LiLiCorr + +LiLiCorr uses a DFlash backbone to produce per-position candidates, then scores +candidate combinations with a learned correlator. It requires a compatible +trained checkpoint and Model Runner V2, which is selected automatically. +Both plain and grouped-convolution checkpoints are supported. + +## Usage + +For a checkpoint trained with block size 16: + +```bash +vllm serve /path/to/target --dtype bfloat16 \ + --speculative-config '{"method":"dflash","model":"/path/to/lilicorr","num_speculative_tokens":15,"draft_sample_method":"probabilistic"}' +``` + +We strongly recommend explicitly setting `num_speculative_tokens` to the +checkpoint's trained `block_size - 1` (for example, `15` for `block_size=16`). +The trained block includes one anchor position. Other supported lengths are +available, but may degrade acceptance length unpredictably; benchmark them with +your checkpoint and workload before using them. + +Shorter drafts, down to one token, use the corresponding prefix of the learned +slot embeddings and attention bias without changing checkpoint parameter shapes. +Lengths above `block_size - 1` are unsupported because the checkpoint has no +learned slot embeddings for those positions. + +Fixed-length drafting with shorter verification is a separate option provided by +adaptive verification (`enable_adaptive_verification`). LiLiCorr shares the +candidate-sampling and acceptance-estimation path, but this combination has not +been validated end-to-end. Adaptive verification also requires compatible target +attention backends and variable-length CUDA graph support; do not assume it works +with Mamba2 targets. + +## Proposal sampling + +- `"draft_sample_method":"greedy"` selects the highest-scoring candidate at each + step of the correlator's conditional walk. +- `"draft_sample_method":"probabilistic"` samples from those conditional scores + using the request's temperature and supplies the realized proposal + distributions to the rejection sampler. + +These settings control proposals; they do not replace the target model's +sampling settings. With request temperature zero, probabilistic proposals also +use argmax. For nonzero temperatures, benchmark both modes with your checkpoint +and workload: compare acceptance length and end-to-end throughput or latency. +Neither mode is universally faster. + +The default rejection method is `"rejection_sample_method":"standard"`. +The shared Model Runner V2 rejection sampler also exposes `"block"`, but +LiLiCorr-specific GPU correctness and performance validation for that combination +is still pending. Do not assume that changing the rejection method improves +throughput; evaluate it separately from the proposal sampling mode. + +## Checkpoint requirements + +The checkpoint must declare `LiLiCorrDraftModel` and include all `lilicorr_*` +geometry fields and trained head weights. Geometry is read from `dflash_config`. +Convolution tensors must match the configured `conv_kernel_size` and +`conv_group_size`. Target input embeddings must be available on the draft rank. +By default, candidates use the target LM head, which must also be available there. +A checkpoint with top-level `"has_own_lm_head": true` instead uses its own +`lm_head` with the draft quantization configuration and exclusions. Its weights +must be present; for ModelOpt NVFP4 this includes `lm_head.weight`, +`lm_head.weight_scale`, and `lm_head.weight_scale_2`. The owned head is preserved +and is not replaced by the target head. Candidate token embeddings still come +from the target. + +Correlator linear layers and convolution kernel projections use the draft +quantization configuration and its module exclusions. The following parameters +must remain in floating-point model dtype: + +- Lattice QKV `in_proj_weight` and `in_proj_bias`, which retain the exported + parameter layout. +- `lilicorr.factor_input_proj`, `lilicorr.out_head`, and `lilicorr.in_head`, which + are used by the split factor and fused edge projections. Exclude these modules + when exporting a quantized checkpoint. diff --git a/tests/models/registry.py b/tests/models/registry.py index 0511d7ef7d..45d1a0a4f5 100644 --- a/tests/models/registry.py +++ b/tests/models/registry.py @@ -1502,6 +1502,13 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = { max_model_len=8192, # Reduce max len to ensure test runs in low-VRAM CI env max_num_seqs=32, ), + # LiLiCorr checkpoints are not published yet. + "LiLiCorrDraftModel": _HfExamplesInfo( + "Qwen/Qwen3-8B", + speculative_model="LiLiCorrDraftModel", + is_available_online=False, + use_original_num_layers=True, + ), "DFlash2DraftModel": _HfExamplesInfo( "Qwen/Qwen3.8-27B", speculative_model="z-lab/Qwen3.8-27B-DFlash2", diff --git a/tests/test_config.py b/tests/test_config.py index aff9758a45..fe4d8ef343 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -582,7 +582,7 @@ def test_rocm_mrv1_default_yields_to_v1_unsupported_config(monkeypatch): speculative_config=None, ) config._dflash_needs_multi_kv_group = lambda: False - config._is_dflash2_draft = lambda: False + config._is_dflash_candidate_draft = lambda: False config._get_v2_model_runner_unsupported_features = lambda: [] # The real predicate, so the test also pins where dspark lands in it. config._get_v1_model_runner_unsupported_features = lambda: ( @@ -813,7 +813,8 @@ def test_v2_model_runner_supports_custom_logits_processors(): assert config._get_v2_model_runner_unsupported_features() == [] -def test_dflash2_draft_forces_v2_model_runner(): +@pytest.mark.parametrize("architecture", ["DFlash2DraftModel", "LiLiCorrDraftModel"]) +def test_dflash_candidate_draft_forces_v2_model_runner(architecture): """A DFlash2 draft must reach the V2 speculator, the only one that runs its candidate selector; on V1 it would draft as DFlash1 without raising.""" @@ -825,11 +826,15 @@ def test_dflash2_draft_forces_v2_model_runner(): ) ) - assert VllmConfig._is_dflash2_draft(config("dflash", ["DFlash2DraftModel"])) - assert not VllmConfig._is_dflash2_draft(config("dflash", ["DFlashDraftModel"])) - assert not VllmConfig._is_dflash2_draft(config("eagle", ["DFlash2DraftModel"])) - assert not VllmConfig._is_dflash2_draft(SimpleNamespace(speculative_config=None)) - assert not VllmConfig._is_dflash2_draft( + assert VllmConfig._is_dflash_candidate_draft(config("dflash", [architecture])) + assert not VllmConfig._is_dflash_candidate_draft( + config("dflash", ["DFlashDraftModel"]) + ) + assert not VllmConfig._is_dflash_candidate_draft(config("eagle", [architecture])) + assert not VllmConfig._is_dflash_candidate_draft( + SimpleNamespace(speculative_config=None) + ) + assert not VllmConfig._is_dflash_candidate_draft( SimpleNamespace( speculative_config=SimpleNamespace(method="dflash", draft_model_config=None) ) @@ -1243,7 +1248,7 @@ def test_v1_model_runner_rejects_v2_only_features(): model_config=None, ) config._dflash_needs_multi_kv_group = lambda: False - config._is_dflash2_draft = lambda: False + config._is_dflash_candidate_draft = lambda: False config._get_v1_model_runner_unsupported_features = lambda: ( VllmConfig._get_v1_model_runner_unsupported_features(config) ) @@ -1430,7 +1435,7 @@ def test_v1_model_runner_rejects_pipeline_parallelism_with_async_scheduling(): model_config=None, ) config._dflash_needs_multi_kv_group = lambda: False - config._is_dflash2_draft = lambda: False + config._is_dflash_candidate_draft = lambda: False assert VllmConfig._get_v1_model_runner_unsupported_features(config) == [] diff --git a/tests/v1/spec_decode/test_dflash2.py b/tests/v1/spec_decode/test_dflash2.py index 72379fa7d7..c508768b39 100644 --- a/tests/v1/spec_decode/test_dflash2.py +++ b/tests/v1/spec_decode/test_dflash2.py @@ -6,6 +6,9 @@ from types import SimpleNamespace import pytest import torch +from vllm.model_executor.models.qwen3_dflash import ( + _add_global_draft_layer_exclusions, +) from vllm.model_executor.models.qwen3_dflash2 import _grouped_conv, _score_edges from vllm.platforms import current_platform from vllm.v1.worker.gpu.spec_decode.dflash.speculator import DFlashSpeculator @@ -85,6 +88,24 @@ def test_grouped_conv_triton_matches_reference( ) +def test_draft_quant_exclusions_include_global_layer_indices(): + quant_config = SimpleNamespace( + exclude_modules=[ + "layers.0.mlp_conv*", + "*layers.4.self_attn.q_proj", + "layers.88.already_global", + "lilicorr.layers.0.mlp.0", + ] + ) + + _add_global_draft_layer_exclusions(quant_config, 88, 5) + + assert "layers.88.mlp_conv*" in quant_config.exclude_modules + assert "*layers.92.self_attn.q_proj" in quant_config.exclude_modules + assert quant_config.exclude_modules.count("layers.88.already_global") == 1 + assert "lilicorr.layers.88.mlp.0" not in quant_config.exclude_modules + + def test_selector_edges_match_sequential_reference(): torch.manual_seed(1) batch, steps, top_k, rank = 2, 4, 3, 5 @@ -169,10 +190,13 @@ def test_selector_asks_for_fp32_proposal_logits(): @pytest.mark.skip_global_cleanup -def test_dflash2_model_decoder_layer_cls(monkeypatch): +@pytest.mark.parametrize("variant", ["dflash2", "lilicorr", "lilicorr_plain"]) +def test_candidate_model_decoder_layer_cls(monkeypatch, variant): from types import SimpleNamespace from vllm.config import set_current_vllm_config + from vllm.model_executor.models.lilicorr import LiLiCorr + from vllm.model_executor.models.qwen3_dflash import DFlashQwen3DecoderLayer from vllm.model_executor.models.qwen3_dflash2 import ( DFlash2Qwen3DecoderLayer, DFlash2Qwen3Model, @@ -249,8 +273,17 @@ def test_dflash2_model_decoder_layer_cls(monkeypatch): dflash_config={ "selector_rank": 4, "selector_top_k": 3, - "conv_kernel_size": 3, - "conv_group_size": 2, + "conv_kernel_size": 0 if variant == "lilicorr_plain" else 3, + "conv_group_size": 0 if variant == "lilicorr_plain" else 2, + "block_size": 5, + "lilicorr_candidate_topk": 4, + "lilicorr_hidden_size": 8, + "lilicorr_num_layers": 2, + "lilicorr_num_heads": 2, + "lilicorr_mlp_ratio": 2.0, + "lilicorr_factor_dim": 4, + "lilicorr_vector_eps": 1e-6, + "lilicorr_logit_scale": 3.0, "use_aux_hidden_state": False, }, ) @@ -277,8 +310,150 @@ def test_dflash2_model_decoder_layer_cls(monkeypatch): # 3. Instantiate the model under meta device to avoid parameter allocation issues with set_current_vllm_config(mock_current_vllm_config), torch.device("meta"): - model = DFlash2Qwen3Model(vllm_config=vllm_config) + model_cls = DFlash2Qwen3Model if variant == "dflash2" else LiLiCorr + model = model_cls(vllm_config=vllm_config) # 4. Assert that the layers are DFlash2Qwen3DecoderLayer (the subclass) assert len(model.layers) == 2 - assert isinstance(model.layers[0], DFlash2Qwen3DecoderLayer) + expected = ( + DFlashQwen3DecoderLayer + if variant == "lilicorr_plain" + else DFlash2Qwen3DecoderLayer + ) + assert type(model.layers[0]) is expected + + +def test_conv_projections_use_draft_quant_config(monkeypatch): + from torch import nn + + from vllm.distributed import parallel_state + from vllm.model_executor.layers.quantization import modelopt + from vllm.model_executor.models.qwen3_dflash import DFlashQwen3DecoderLayer + from vllm.model_executor.models.qwen3_dflash2 import DFlash2Qwen3DecoderLayer + from vllm.model_executor.models.utils import AutoWeightsLoader + + monkeypatch.setattr( + parallel_state, "_TP", SimpleNamespace(rank_in_group=0, world_size=1) + ) + monkeypatch.setattr( + DFlashQwen3DecoderLayer, + "__init__", + lambda self, *a, **kw: nn.Module.__init__(self), + ) + # Exercise the real quantized parameter allocation without selecting a GPU kernel. + monkeypatch.setattr( + modelopt, + "select_linear_kernel", + lambda *a, **kw: SimpleNamespace(input_quant_key=lambda: None), + ) + quant_config = modelopt.ModelOptNvFp4Config( + quant_method="W4A16_NVFP4", is_checkpoint_nvfp4_serialized=True + ) + layer = DFlash2Qwen3DecoderLayer( + SimpleNamespace( + speculative_config=SimpleNamespace(num_speculative_tokens=7), + model_config=SimpleNamespace(dtype=torch.bfloat16), + ), + config=SimpleNamespace( + hidden_size=16, + dflash_config={"conv_kernel_size": 2, "conv_group_size": 2}, + ), + layer_idx=0, + prefix="model.layers.0", + quant_config=quant_config, + ) + for name in ("attention_conv", "mlp_conv"): + module = getattr(layer, name) + projection = module.kernel_projection + assert projection.weight.dtype == torch.uint8 + assert projection.weight.shape == (32, 8) + weight = torch.ones_like(projection.weight) + AutoWeightsLoader(module).load_weights([("kernel_projection.weight", weight)]) + torch.testing.assert_close(projection.weight, weight) + assert module.base_kernel.dtype == torch.bfloat16 + + +def test_context_kv_uses_quantized_projection_fallback(monkeypatch): + from torch import nn + + from vllm.model_executor.models import qwen3_dflash + + class Projection(nn.Module): + def __init__(self, packed_weight): + super().__init__() + self.register_buffer("packed_weight", packed_weight) + self.quant_method = object() + self.calls = 0 + + def forward(self, hidden_states): + self.calls += 1 + return torch.nn.functional.linear(hidden_states, self.packed_weight), None + + monkeypatch.setattr( + qwen3_dflash.ops, + "rms_norm", + lambda output, hidden_states, weight, eps: output.copy_(hidden_states), + ) + context_states = torch.tensor([[1.0, 2.0], [3.0, 4.0]]) + projections = [ + Projection( + torch.tensor( + [ + [0.0, 0.0], + [0.0, 0.0], + [1.0, 0.0], + [0.0, 1.0], + [1.0, 1.0], + [1.0, -1.0], + ] + ) + ), + Projection( + torch.tensor( + [ + [0.0, 0.0], + [0.0, 0.0], + [2.0, 0.0], + [0.0, 2.0], + [-1.0, 0.0], + [0.0, -1.0], + ] + ) + ), + ] + model = SimpleNamespace( + hidden_norm=SimpleNamespace(weight=nn.Parameter(torch.ones(2))), + _rms_norm_eps=1e-6, + ) + layers_attn = [ + SimpleNamespace( + qkv_proj=projection, + q_size=2, + k_norm=SimpleNamespace(weight=nn.Parameter(torch.ones(2))), + ) + for projection in projections + ] + qwen3_dflash.DFlashQwen3Model._build_context_kv_buffers( + model, layers_attn, has_bias=False + ) + assert model._fused_kv_weight is None + assert all(not hasattr(projection, "weight") for projection in projections) + + actual_k, actual_v = qwen3_dflash.DFlashQwen3Model._project_context_kv( + model, + context_states, + num_ctx=2, + num_layers=2, + num_kv_heads=1, + head_dim=2, + ) + + expected_k = torch.tensor( + [[[1.0, 2.0], [3.0, 4.0]], [[2.0, 4.0], [6.0, 8.0]]] + ).unsqueeze(2) + expected_v = torch.tensor( + [[[3.0, -1.0], [7.0, -1.0]], [[-1.0, -2.0], [-3.0, -4.0]]] + ).unsqueeze(2) + torch.testing.assert_close(actual_k, expected_k) + torch.testing.assert_close(actual_v, expected_v) + assert [projection.calls for projection in projections] == [1, 1] diff --git a/tests/v1/spec_decode/test_lilicorr.py b/tests/v1/spec_decode/test_lilicorr.py new file mode 100644 index 0000000000..14bad45f6b --- /dev/null +++ b/tests/v1/spec_decode/test_lilicorr.py @@ -0,0 +1,777 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from types import SimpleNamespace +from typing import Any + +import pytest +import torch +import torch.nn.functional as F +from torch import nn + +from vllm.config import ( + CompilationConfig, + DeviceConfig, + LoadConfig, + VllmConfig, + set_current_vllm_config, +) +from vllm.distributed import parallel_state +from vllm.model_executor.layers import logits_processor +from vllm.model_executor.layers.linear import LinearBase +from vllm.model_executor.models.lilicorr import LiLiCorrHead +from vllm.transformers_utils.configs.eagle import EAGLEConfig +from vllm.v1.worker.gpu.spec_decode.dflash2.speculator import CandidateSampler +from vllm.v1.worker.gpu.spec_decode.lilicorr.speculator import LiLiCorrSpeculator + + +@pytest.fixture(autouse=True) +def head_tp_metadata(monkeypatch): + # Single-rank unit tests only need TP metadata. + monkeypatch.setattr( + parallel_state, "_TP", SimpleNamespace(rank_in_group=0, world_size=1) + ) + + +def _config(**overrides): + config = dict( + candidate_topk=4, + hidden_size=8, + num_layers=2, + num_heads=2, + mlp_ratio=2.0, + factor_dim=4, + vector_eps=1e-6, + logit_scale=3.0, + ) + config.update(overrides) + return {f"lilicorr_{key}": value for key, value in config.items()} + + +def _reference_scores(head, embeddings, log_probs, hidden, anchor, valid): + """Unfused exported head: MHA attention, concatenated factors, separate edges.""" + batch, slots, top_k = log_probs.shape + rank = torch.arange(top_k, device=log_probs.device) / max(top_k - 1, 1) + top1 = (torch.arange(top_k, device=log_probs.device) == 0).float() + features = torch.stack( + ( + log_probs, + log_probs.exp(), + log_probs - log_probs.amax(-1, keepdim=True), + rank.expand_as(log_probs), + top1.expand_as(log_probs), + ), + -1, + ) + x = head.token_proj(embeddings) + head.pass_hidden_proj(hidden).unsqueeze(-2) + features = head.feature_norm(features.to(x.dtype)) + x = x + head.feature_mlp.down_proj(F.silu(head.feature_mlp.up_proj(features)[0]))[0] + x = x + head.slot_embedding[:, 0, :slots] + head.rank_embedding[:, 0] + x = x.flatten(1, 2) + positions = torch.arange(slots, device=x.device).repeat_interleave(top_k) + relative = positions[:, None] - positions[None, :] + bias = head.relative_slot_bias[:, relative + head.block_size - 1] + bias = bias + (relative == 0) * head.same_slot_bias[:, None, None] + bias = bias.repeat(batch, 1, 1) + for layer in head.layers: + attention = nn.MultiheadAttention( + head.hidden_size, + head.num_heads, + batch_first=True, + dtype=x.dtype, + device=x.device, + ) + attention.load_state_dict(layer.attn.state_dict()) + normalized = layer.attn_norm(x) + x = ( + x + + attention( + normalized, normalized, normalized, attn_mask=bias, need_weights=False + )[0] + ) + x = x + layer.mlp.down_proj(F.silu(layer.mlp.up_proj(layer.mlp_norm(x))[0]))[0] + x = head.output_norm(x).view(batch, slots, top_k, -1) + a = head.anchor_norm(head.context_proj(anchor) * valid[:, None]) + expanded = a[:, None, None].expand_as(x) + factors = F.silu(head.factor_input_proj(torch.cat((x, expanded, x * expanded), -1))) + incoming = F.normalize(head.in_head(factors), dim=-1, eps=head.vector_eps) + outgoing = F.normalize(head.out_head(factors), dim=-1, eps=head.vector_eps) + a = F.normalize(head.anchor_out_head(a), dim=-1, eps=head.vector_eps) + start = (a[:, None] * incoming[:, 0]).sum(-1) + pairs = outgoing[:, :-1] @ incoming[:, 1:].transpose(-1, -2) + return ( + head.logit_scale + * torch.cat((start[:, None, None].expand(-1, 1, top_k, -1), pairs), 1).float() + ) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA or ROCm") +@pytest.mark.parametrize( + "head_width,slots,dtype", + [ + pytest.param(8, 3, torch.float32, id="projected-multi-slot"), + pytest.param(16, 1, torch.float32, id="identity-shortened-single-slot"), + pytest.param(8, 2, torch.bfloat16, id="bf16-shortened-multi-slot"), + ], +) +def test_lilicorr_matches_exported_head(head_width, slots, dtype): + device = torch.device("cuda") + torch.manual_seed(7) + with ( + torch.device(device), + set_current_vllm_config( + VllmConfig( + device_config=DeviceConfig("cuda"), + compilation_config=CompilationConfig(mode=0), + ) + ), + ): + head = LiLiCorrHead( + model_hidden_size=16, + block_size=4, + rms_norm_eps=1e-6, + config=_config(hidden_size=head_width), + ).to(dtype) + # vLLM linear parameters are initialized by the checkpoint loader. + with torch.no_grad(), torch.device(device): + for module in head.modules(): + if isinstance(module, LinearBase): + for parameter in module.parameters(): + parameter.weight_loader( + parameter, torch.randn_like(parameter) * 0.1 + ) + for layer in head.layers: + layer.attn.in_proj_weight.normal_(std=0.1) + layer.attn.in_proj_bias.normal_(std=0.1) + head.relative_slot_bias.normal_(std=0.1) + head.same_slot_bias.normal_(std=0.1) + head.slot_embedding.normal_(std=0.1) + head.rank_embedding.normal_(std=0.1) + head.materialize_inference_buffers(device, dtype) + inputs = ( + torch.randn(2, slots, 4, 16, dtype=dtype), + torch.randn(2, slots, 23).log_softmax(-1).topk(4).values, + torch.randn(2, slots, 16, dtype=dtype), + torch.randn(2, 16, dtype=dtype), + torch.tensor([True, False]), + ) + actual = head(*inputs) + expected = _reference_scores(head, *inputs) + tolerance = 0.04 if dtype == torch.bfloat16 else 2e-6 + torch.testing.assert_close(actual, expected, atol=tolerance, rtol=tolerance) + + +def test_runtime_length_keeps_trained_checkpoint_geometry(monkeypatch): + from vllm.model_executor.models.lilicorr import LiLiCorr + from vllm.model_executor.models.qwen3_dflash import DFlashQwen3Model + + def init_backbone(self, **kwargs): + nn.Module.__init__(self) + self.quant_config = None + + monkeypatch.setattr(DFlashQwen3Model, "__init__", init_backbone) + spec = SimpleNamespace( + num_speculative_tokens=1, + draft_model_config=SimpleNamespace( + hf_config=SimpleNamespace( + hidden_size=16, + rms_norm_eps=1e-6, + dflash_config=_config() | {"block_size": 4}, + ) + ), + ) + with set_current_vllm_config( + VllmConfig( + device_config=DeviceConfig("cpu"), + compilation_config=CompilationConfig(mode=0), + ) + ): + for slots in (1, 3): + spec.num_speculative_tokens = slots + head = LiLiCorr( + vllm_config=SimpleNamespace(speculative_config=spec) + ).lilicorr + assert head.slot_embedding.shape == (1, 1, 3, 1, 8) + assert head.relative_slot_bias.shape == (2, 7) + spec.draft_model_config.hf_config.dflash_config["lilicorr_enabled"] = False + with pytest.raises(ValueError, match="lilicorr_enabled"): + LiLiCorr(vllm_config=SimpleNamespace(speculative_config=spec)) + spec.draft_model_config.hf_config.dflash_config["lilicorr_enabled"] = True + spec.num_speculative_tokens = 4 + with pytest.raises(ValueError, match="trained block_size"): + LiLiCorr(vllm_config=SimpleNamespace(speculative_config=spec)) + + +@pytest.mark.parametrize("tp_size", [1, 2]) +def test_candidates_use_global_partition_and_exclude_padding(monkeypatch, tp_size): + """Candidate features must include mass outside top-k and on other TP ranks.""" + + class Gather: + def __init__(self, calls): + self.calls = calls + self.index = 0 + + def __call__(self, value, dim=-1): + expected_input, result = self.calls[self.index] + assert dim == -1 + torch.testing.assert_close(value, expected_input) + self.index += 1 + return result + + shards = [ + torch.tensor([[1.0, 3.0, 2.0, 100.0], [4.0, 1.0, 2.0, 100.0]]), + torch.tensor([[6.0, 5.0, 0.0, 100.0], [3.0, 6.0, 5.0, 100.0]]), + ][:tp_size] + for rank in range(tp_size): + fake = SimpleNamespace( + scale=1.0, + soft_cap=None, + _apply_head=lambda *args, rank=rank: shards[rank].clone(), + ) + lm_head = SimpleNamespace( + tp_size=tp_size, + shard_indices=SimpleNamespace( + num_org_vocab_padding=1, org_vocab_start_index=rank * 3 + ), + ) + valid_shards = [shard[:, :3] for shard in shards] + local_topk = [shard.topk(2) for shard in valid_shards] + gather_calls = [ + ( + valid_shards[rank].logsumexp(-1, keepdim=True), + torch.cat( + [shard.logsumexp(-1, keepdim=True) for shard in valid_shards], + -1, + ), + ), + ( + local_topk[rank].values, + torch.cat([topk.values for topk in local_topk], -1), + ), + ( + local_topk[rank].indices + rank * 3, + torch.cat( + [ + topk.indices + shard_rank * 3 + for shard_rank, topk in enumerate(local_topk) + ], + -1, + ), + ), + ] + all_gather = Gather(gather_calls) + + monkeypatch.setattr( + logits_processor, + "tensor_model_parallel_all_gather", + all_gather, + ) + monkeypatch.setattr(logits_processor, "_topk", lambda x, k: x.topk(k, dim=-1)) + ids, values = logits_processor.LogitsProcessor.get_top_k_tokens( + fake, lm_head, torch.empty(2, 1), 2, return_log_probs=True + ) + expected = torch.cat(valid_shards, -1).log_softmax(-1).topk(2) + torch.testing.assert_close(ids, expected.indices) + torch.testing.assert_close(values, expected.values) + assert all_gather.index == (3 if tp_size > 1 else 0) + + +def test_context_anchor_is_last_committed_normalized_feature(): + hidden = torch.arange(1, 25, dtype=torch.float32).view(6, 4) + norm = nn.RMSNorm(4) + spec = SimpleNamespace( + hidden_states=hidden, + model=SimpleNamespace(model=SimpleNamespace(hidden_norm=norm)), + anchor_hidden=torch.full((4, 4), 123.0), + anchor_valid=torch.ones(4, dtype=torch.bool), + ) + batch = SimpleNamespace(num_reqs=2, query_start_loc=torch.tensor([0, 2, 6])) + LiLiCorrSpeculator.prepare_context_anchor(spec, batch, torch.tensor([0, 2])) + torch.testing.assert_close(spec.anchor_hidden[:2], norm(hidden[[1, 3]])) + assert spec.anchor_valid.tolist() == [True, True, False, False] + assert not spec.anchor_hidden[2:].any() + # A smaller reordered batch must not retain an old request's anchor. + batch = SimpleNamespace(num_reqs=1, query_start_loc=torch.tensor([0, 3])) + LiLiCorrSpeculator.prepare_context_anchor(spec, batch, torch.tensor([1])) + torch.testing.assert_close(spec.anchor_hidden[0], norm(hidden[1])) + assert not spec.anchor_hidden[1:].any() + + +@pytest.mark.parametrize("lilicorr", [False, True]) +def test_candidate_generation_routes_scores_and_adaptive_inputs(monkeypatch, lilicorr): + """Both heads share the walk, including a padded request and adaptive scores.""" + from vllm.v1.worker.gpu.spec_decode.dflash.speculator import DFlashSpeculator + from vllm.v1.worker.gpu.spec_decode.dflash2.speculator import DFlash2Speculator + + key = "lilicorr_candidate_topk" if lilicorr else "selector_top_k" + + def init_base(self, config, device): + self.draft_model_config = SimpleNamespace( + hf_config=SimpleNamespace(hidden_size=4, dflash_config={key: 4}) + ) + self.max_num_reqs = 3 + self.num_speculative_steps = 2 + self.num_query_per_req = 3 + self.dtype = torch.float32 + + monkeypatch.setattr(DFlashSpeculator, "__init__", init_base) + cls = LiLiCorrSpeculator if lilicorr else DFlash2Speculator + spec = cls(None, torch.device("cpu")) + hidden = torch.arange(36).float().view(9, 4) + spec._run_model = lambda *args: hidden + spec.sample_indices = torch.tensor([1, 2, 4, 5, 7, 8]) + spec.sample_pos = torch.arange(6) + spec.sample_idx_mapping = torch.tensor([1, 1, 0, 0, -1, -1]) + spec.sample_col = torch.tensor([0, 1, 0, 1, 0, 1]) + spec.temperature = torch.ones(3) + spec.seeds = torch.arange(3) + spec.draft_tokens = torch.empty(3, 2, dtype=torch.long) + spec.draft_logits = None + spec.use_fp64_gumbel = False + spec.enable_adaptive_verification = True + spec.input_buffers = SimpleNamespace(input_ids=torch.arange(9)) + candidates = torch.arange(24).view(6, 4) + log_probs = candidates.float().log_softmax(-1) + scores = torch.randn(3, 2, 4, 4) + scoring_inputs: list[torch.Tensor] = [] + sampling_inputs: list[Any] = [] + adaptive_inputs: list[torch.Tensor] = [] + + def score(*args): + scoring_inputs.extend(args) + return scores + + def sample(*args): + sampling_inputs.extend(args) + spec.candidate_sampler.scores.copy_(scores[:, :, 0]) + + spec.model = SimpleNamespace( + compute_candidates=lambda h: (candidates, log_probs), + model=SimpleNamespace(lilicorr=score, candidate_selector=score), + ) + spec.target_embeddings = lambda ids: ids.float().unsqueeze(-1).expand(*ids.shape, 4) + if lilicorr: + spec.anchor_hidden.copy_(torch.arange(12).view(3, 4)) + spec.anchor_valid[:2] = True + spec.candidate_sampler.sample = sample + spec._maybe_predict_acceptance = lambda *args: adaptive_inputs.extend(args) + spec._generate_draft(3, 9, None, None, None) + + expected_ids = candidates.view(3, 2, 4) + expected_first = spec.target_embeddings(expected_ids) if lilicorr else expected_ids + torch.testing.assert_close(scoring_inputs[0], expected_first) + torch.testing.assert_close(scoring_inputs[1], log_probs.view(3, 2, 4)) + torch.testing.assert_close( + scoring_inputs[2], hidden[spec.sample_indices].view(3, 2, 4) + ) + if lilicorr: + torch.testing.assert_close(scoring_inputs[3], spec.anchor_hidden) + torch.testing.assert_close(scoring_inputs[4], spec.anchor_valid) + else: + torch.testing.assert_close(scoring_inputs[3], torch.tensor([0, 3, 6])) + torch.testing.assert_close(sampling_inputs[0], expected_ids) + assert sampling_inputs[1] is scores + assert sampling_inputs[4] is spec.sample_idx_mapping + torch.testing.assert_close(adaptive_inputs[0], scores[:, :, 0].flatten(0, 1)) + torch.testing.assert_close(adaptive_inputs[1], spec.sample_idx_mapping) + torch.testing.assert_close(adaptive_inputs[2], spec.sample_col) + + +def test_lilicorr_config_preserves_architecture(): + from transformers import Qwen3Config + + config = Qwen3Config(architectures=["LiLiCorrDraftModel"]) + wrapped = EAGLEConfig(config, method="dflash") + assert wrapped.architectures == ["LiLiCorrDraftModel"] + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") +@pytest.mark.parametrize("probabilistic", [False, True]) +def test_candidate_walk_preserves_conditional_scores_and_padding(probabilistic): + """Check the shared sampler's actual q, stale support, and graph padding.""" + device = torch.device("cuda") + batch, steps, top_k, vocab = 3, 3, 4, 32 + sampler = CandidateSampler(batch, steps, top_k, device) + candidates = ( + torch.arange(steps * top_k, device=device) + .view(1, steps, top_k) + .repeat(batch, 1, 1) + ) + scores = torch.randn(batch, steps, top_k, top_k, device=device) + tokens = torch.full((batch, steps), -1, dtype=torch.long, device=device) + cache = ( + torch.full((batch, steps, vocab), -float("inf"), device=device) + if probabilistic + else None + ) + positions = torch.arange(batch * steps * 2, device=device)[::2] + mappings = torch.tensor( + [2] * steps + [0] * steps + [-1] * steps, device=device + ).repeat_interleave(2)[::2] + temperature = torch.tensor([0.0, 1.0, 0.8], device=device) + seeds = torch.tensor([11, 22, 33], device=device) + + def run(): + sampler.sample( + candidates, + scores, + batch, + positions, + mappings, + temperature, + seeds, + tokens, + cache, + False, + ) + + run() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + run() + old_candidates = candidates.clone() + candidates.add_(12) + graph.replay() + for row, state in ((0, 2), (1, 0)): + previous = 0 + for step in range(steps): + realized = scores[row, step, previous] + torch.testing.assert_close(sampler.scores[row, step], realized) + index = (candidates[row, step] == tokens[row, step]).nonzero().item() + if not probabilistic or temperature[state] == 0: + assert index == realized.argmax().item() + if cache is not None: + torch.testing.assert_close( + cache[state, step, candidates[row, step]], realized + ) + assert cache[state, step, old_candidates[row, step]].isneginf().all() + previous = index + assert (tokens[2] == -1).all() + if cache is not None: + assert cache[1].isneginf().all() + + +@pytest.mark.parametrize( + ("convolution", "mismatch", "quantized"), + [ + pytest.param(False, None, False, id="valid-plain"), + pytest.param(True, None, False, id="valid-convolution"), + pytest.param(False, "missing_head", False, id="missing-head"), + pytest.param(False, "extra_head", False, id="extra-head"), + pytest.param(True, "missing_conv", False, id="missing-convolution"), + pytest.param(False, "extra_conv", False, id="unexpected-convolution"), + pytest.param(False, None, True, id="optional-w4a16-input-scale"), + pytest.param(False, "missing_weight", True, id="missing-quantized-weight"), + pytest.param(False, "missing_bias", True, id="missing-quantized-bias"), + pytest.param(False, "missing_weight_scale", True, id="missing-block-scale"), + pytest.param(False, "missing_weight_scale_2", True, id="missing-global-scale"), + ], +) +def test_checkpoint_coverage_rejects_incomplete_or_wrong_heads( + monkeypatch, convolution, mismatch, quantized +): + from vllm.model_executor.models.lilicorr import LiLiCorr, LiLiCorrForCausalLM + + wrapper = LiLiCorrForCausalLM.__new__(LiLiCorrForCausalLM) + nn.Module.__init__(wrapper) + wrapper.model = LiLiCorr.__new__(LiLiCorr) + nn.Module.__init__(wrapper.model) + wrapper.has_own_lm_head = False + with set_current_vllm_config( + VllmConfig( + device_config=DeviceConfig("cpu"), + compilation_config=CompilationConfig(mode=0), + ) + ): + wrapper.model.lilicorr = LiLiCorrHead( + model_hidden_size=16, block_size=4, rms_norm_eps=1e-6, config=_config() + ) + layer = nn.Module() + if convolution: + layer.attention_conv = nn.Linear(2, 2, bias=False) + wrapper.model.layers = nn.ModuleList([layer]) + if quantized: + from vllm.config.quantization import QuantSpec + from vllm.model_executor.layers.quantization.modelopt import ( + CkptCtx, + ModelOptLinearMethod, + ) + from vllm.model_executor.layers.quantization.utils.quant_utils import ( + kNvfp4Static, + ) + + # Match the W4A16 NVFP4 checkpoint contract without GPU kernel setup. + projection = wrapper.model.lilicorr.pass_hidden_proj + projection.quant_method = ModelOptLinearMethod( + QuantSpec(weight=kNvfp4Static, activation=None), CkptCtx(group_size=16) + ) + projection.weight = nn.Parameter( + torch.zeros(8, 8, dtype=torch.uint8), requires_grad=False + ) + for name, value in ( + ("weight_scale", torch.ones(8, 1).to(torch.float8_e4m3fn)), + ("weight_scale_2", torch.ones(1)), + ("input_scale", torch.full((1,), torch.nan)), + ): + projection.register_parameter( + name, nn.Parameter(value, requires_grad=False) + ) + weights = dict(wrapper.model.state_dict()) + if quantized: + del weights["lilicorr.pass_hidden_proj.input_scale"] + if mismatch == "missing_head": + del weights["lilicorr.slot_embedding"] + elif mismatch in ( + "missing_weight", + "missing_bias", + "missing_weight_scale", + "missing_weight_scale_2", + ): + del weights[f"lilicorr.pass_hidden_proj.{mismatch.removeprefix('missing_')}"] + elif mismatch == "extra_head": + weights["lilicorr.untrained.weight"] = torch.zeros(1) + elif mismatch == "missing_conv": + del weights["layers.0.attention_conv.weight"] + elif mismatch == "extra_conv": + weights["layers.0.mlp_conv.untrained"] = torch.zeros(1) + + # Keep production loading and mapping, bypass only backbone GPU KV setup. + wrapper.model.use_aux_hidden_state = True + wrapper.model.has_separate_mask_embedding = False + monkeypatch.setattr(wrapper, "_read_mask_embedding", lambda: None) + monkeypatch.setattr( + wrapper.model, "_build_fused_kv_buffers", lambda: None, raising=False + ) + supplied = [("model." + name, value) for name, value in weights.items()] + if mismatch: + message = ( + "no module or parameter" + if mismatch.startswith("extra") + else "coverage mismatch" + ) + with pytest.raises(ValueError, match=message): + wrapper.load_weights(supplied) + else: + wrapper.load_weights(supplied) + assert wrapper.model.lilicorr._attn_bias is not None + + +@pytest.fixture +def owned_head_model(monkeypatch): + from vllm.model_executor.layers.quantization import modelopt + from vllm.model_executor.models import qwen3_dflash + from vllm.model_executor.models.lilicorr import LiLiCorrForCausalLM + + quant_config = modelopt.ModelOptNvFp4Config( + quant_method="W4A16_NVFP4", is_checkpoint_nvfp4_serialized=True + ) + monkeypatch.setattr( + modelopt, + "select_linear_kernel", + lambda *a, **kw: SimpleNamespace(input_quant_key=lambda: None), + ) + monkeypatch.setattr(qwen3_dflash, "get_draft_quant_config", lambda _: quant_config) + + class Backbone(nn.Module): + def __init__(self, **kwargs): + super().__init__() + self.quant_config = quant_config + self.use_aux_hidden_state = True + self.has_separate_mask_embedding = False + self.lilicorr = LiLiCorrHead( + model_hidden_size=16, block_size=4, rms_norm_eps=1e-6, config=_config() + ) + + def _build_fused_kv_buffers(self): + pass + + monkeypatch.setattr(LiLiCorrForCausalLM, "model_cls", Backbone) + monkeypatch.setattr(LiLiCorrForCausalLM, "_read_mask_embedding", lambda _: None) + + def build(owned=True): + config = SimpleNamespace( + vocab_size=64, + draft_vocab_size=64, + hidden_size=16, + has_own_lm_head=owned, + num_hidden_layers=0, + dflash_config={}, + ) + vllm_config = SimpleNamespace( + speculative_config=SimpleNamespace( + draft_model_config=SimpleNamespace(hf_config=config), + draft_load_config=None, + attention_backend=None, + kv_cache_dtype=None, + ), + model_config=SimpleNamespace( + get_total_num_hidden_layers=lambda: 0, + get_vocab_size=lambda: 64, + ), + attention_config=SimpleNamespace(), + cache_config=SimpleNamespace(), + load_config=LoadConfig(), + ) + with set_current_vllm_config( + VllmConfig( + device_config=DeviceConfig("cpu"), + compilation_config=CompilationConfig(mode=0), + ) + ): + model = LiLiCorrForCausalLM(vllm_config=vllm_config) + return model, vllm_config + + return build + + +@pytest.mark.parametrize("missing", [None, "weight", "weight_scale", "weight_scale_2"]) +def test_owned_nvfp4_lm_head_loads_required_checkpoint_tensors( + owned_head_model, missing +): + model, _ = owned_head_model() + assert model.lm_head.weight.dtype == torch.uint8 + assert model.lm_head.weight.shape == (64, 8) + weights = { + name: torch.ones_like(value) for name, value in model.state_dict().items() + } + del weights["lm_head.input_scale"] # Deprecated and unused for W4A16. + if missing: + del weights[f"lm_head.{missing}"] + with pytest.raises(ValueError, match=f"lm_head.{missing}"): + model.load_weights(weights.items()) + else: + model.load_weights(weights.items()) + assert model.has_own_lm_head + for name in ("weight", "weight_scale", "weight_scale_2"): + torch.testing.assert_close( + getattr(model.lm_head, name), weights[f"lm_head.{name}"] + ) + + +@pytest.mark.parametrize("owned", [False, True]) +def test_selected_lm_head_survives_dflash_and_lilicorr_sharing( + monkeypatch, owned_head_model, owned +): + from vllm.model_executor.layers.vocab_parallel_embedding import ( + VocabParallelEmbedding, + ) + from vllm.v1.worker.gpu.spec_decode.dflash import utils as dflash_utils + from vllm.v1.worker.gpu.spec_decode.eagle import utils as eagle_utils + + model, config = owned_head_model(owned) + own_head = model.lm_head + target_head = owned_head_model(owned)[0].lm_head + # Identical packed bytes do not imply identical quantized heads. + with torch.no_grad(): + own_head.weight.fill_(1) + target_head.weight.fill_(1) + if owned: + own_head.weight_scale_2.fill_(1) + target_head.weight_scale_2.fill_(2) + embedding = VocabParallelEmbedding(64, 16) + target = SimpleNamespace( + model=SimpleNamespace(embed_tokens=embedding), lm_head=target_head + ) + monkeypatch.setattr( + eagle_utils, "get_pp_group", lambda: SimpleNamespace(world_size=1) + ) + monkeypatch.setattr( + dflash_utils, "replace", lambda obj, **kw: SimpleNamespace(**(vars(obj) | kw)) + ) + monkeypatch.setattr(dflash_utils, "get_pp_safe_draft_load_config", lambda c: c) + monkeypatch.setattr(dflash_utils, "get_model", lambda **kw: model) + if owned: + + def no_weight_comparison(*args): + raise AssertionError("Explicit ownership must bypass raw-weight comparison") + + monkeypatch.setattr(dflash_utils, "_should_share", no_weight_comparison) + spec = LiLiCorrSpeculator.__new__(LiLiCorrSpeculator) + spec.vllm_config = config + assert spec.load_draft_model(target, set()) is model + assert model.lm_head is (own_head if owned else target_head) + assert spec.target_embeddings is embedding + if owned: + target.lm_head = None + assert spec.load_draft_model(target, set()).lm_head is own_head + + +def test_quantized_head_calls_methods_without_reading_packed_weights(monkeypatch): + from vllm.model_executor.layers import linear + from vllm.model_executor.layers.linear import UnquantizedLinearMethod + + calls = [] + configured = [] + quant_config = object() + + class PackedMethod(UnquantizedLinearMethod): + def apply(self, layer, x, bias=None): + calls.append(layer.prefix) + # Packed storage has a different name/dtype from the activations. + assert x.dtype == torch.float32 + return F.linear(x, layer.packed_weight.float() * layer.weight_scale, bias) + + def resolve(config, layer, prefix): + assert config is quant_config + configured.append(prefix) + return PackedMethod() + + monkeypatch.setattr(linear, "resolve_quant_method", resolve) + with set_current_vllm_config( + VllmConfig( + device_config=DeviceConfig("cpu"), + compilation_config=CompilationConfig(mode=0), + ) + ): + head = LiLiCorrHead( + model_hidden_size=16, + block_size=4, + rms_norm_eps=1e-6, + config=_config(hidden_size=8), + quant_config=quant_config, + prefix="model.lilicorr", + ) + with torch.no_grad(): + for module in head.modules(): + if isinstance(module, LinearBase): + if module.quant_config is None: + module.weight.normal_(std=0.1) + module.bias.zero_() + continue + module.register_parameter( + "packed_weight", + nn.Parameter( + torch.randint(-4, 5, module.weight.shape, dtype=torch.int8), + requires_grad=False, + ), + ) + module.register_parameter( + "weight_scale", + nn.Parameter( + torch.tensor(0.05), + requires_grad=False, + ), + ) + del module.weight + module.bias.zero_() + for layer in head.layers: + layer.attn.in_proj_weight.normal_(std=0.1) + layer.attn.in_proj_bias.zero_() + head.materialize_inference_buffers(torch.device("cpu"), torch.float32) + result = head( + torch.randn(2, 3, 4, 16), + torch.randn(2, 3, 4).log_softmax(-1), + torch.randn(2, 3, 16), + torch.randn(2, 16), + torch.tensor([True, False]), + ) + assert result.shape == (2, 3, 4, 4) + assert torch.isfinite(result).all() + for name in ("factor_input_proj", "out_head", "in_head"): + assert getattr(head, name).quant_config is None + assert f"model.lilicorr.{name}" not in configured + assert set(calls) == set(configured) + assert len(configured) == len(set(configured)) + for name, module in head.named_modules(): + if isinstance(module, LinearBase): + assert module.prefix == f"model.lilicorr.{name}" diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index 283cb50198..ada1506c20 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -760,8 +760,8 @@ class VllmConfig: return True - def _is_dflash2_draft(self) -> bool: - """Whether the DFlash draft is a DFlash2 one, by the architecture the + def _is_dflash_candidate_draft(self) -> bool: + """Whether the DFlash draft has a candidate head, by the architecture the speculator selects on (v1/worker/gpu/spec_decode/__init__.py).""" spec = self.speculative_config if spec is None or spec.method != "dflash": @@ -769,7 +769,11 @@ class VllmConfig: draft_config = getattr(spec, "draft_model_config", None) if draft_config is None: return False - return "DFlash2DraftModel" in (draft_config.architectures or []) + return bool( + {"DFlash2DraftModel", "LiLiCorrDraftModel"}.intersection( + draft_config.architectures or [] + ) + ) def _dflash_needs_multi_kv_group(self) -> bool: """Whether a DFlash draft mixes sliding-window and full attention.""" @@ -3097,11 +3101,11 @@ class VllmConfig: if self._dflash_needs_multi_kv_group(): unsupported.append("mixed sliding/full dflash drafts") - # The DFlash2 candidate selector exists only in the V2 speculator. On + # DFlash candidate heads exist only in the V2 speculator. On # V1 the same checkpoint drafts through DFlashProposer, which never # calls it, so the draft would degrade to DFlash1 silently. - if self._is_dflash2_draft(): - unsupported.append("dflash2 drafts") + if self._is_dflash_candidate_draft(): + unsupported.append("DFlash candidate-head drafts") if self.model_config is not None and self.model_config.is_diffusion: unsupported.append("diffusion models") diff --git a/vllm/model_executor/layers/logits_processor.py b/vllm/model_executor/layers/logits_processor.py index 10f425a503..04541212fe 100644 --- a/vllm/model_executor/layers/logits_processor.py +++ b/vllm/model_executor/layers/logits_processor.py @@ -256,6 +256,8 @@ class LogitsProcessor(PluggableLayer): hidden_states: torch.Tensor, k: int, embedding_bias: torch.Tensor | None = None, + *, + return_log_probs: bool = False, ) -> tuple[torch.Tensor, torch.Tensor]: """Vocab-parallel top-k without all-gathering full logits. @@ -263,7 +265,9 @@ class LogitsProcessor(PluggableLayer): the values as well as the global ids. Communication is O(batch * 2k * tp_size) rather than O(batch * vocab_size). - Scale and soft cap are applied to the k selected values rather than + With return_log_probs, values use the full-vocabulary log-partition. + + Otherwise scale and soft cap are applied to the k selected values rather than the whole vocabulary; both are monotonic, so the selection is the same and only k entries are touched. """ @@ -274,12 +278,26 @@ class LogitsProcessor(PluggableLayer): ) logits = self._apply_head(lm_head, hidden_states, embedding_bias) + if return_log_probs: + if self.soft_cap is not None: + logits = torch.tanh(logits / self.soft_cap) * self.soft_cap + if self.scale != 1.0: + logits = logits * self.scale # Mask out padding entries beyond org_vocab_size on this shard. num_pad = lm_head.shard_indices.num_org_vocab_padding if num_pad > 0: logits[..., -num_pad:] = -float("inf") + if return_log_probs: + log_partition = torch.logsumexp(logits.float(), dim=-1, keepdim=True) + if lm_head.tp_size > 1: + log_partition = torch.logsumexp( + tensor_model_parallel_all_gather(log_partition, dim=-1), + dim=-1, + keepdim=True, + ) + values, ids = _topk(logits, k) # Convert shard-local indices to global vocab indices. ids = ids.to(torch.int64) + lm_head.shard_indices.org_vocab_start_index @@ -291,6 +309,8 @@ class LogitsProcessor(PluggableLayer): ids = ids.gather(-1, selected) values = values.float() + if return_log_probs: + return ids, values - log_partition if self.scale != 1.0: values = values * self.scale if self.soft_cap is not None: diff --git a/vllm/model_executor/models/lilicorr.py b/vllm/model_executor/models/lilicorr.py new file mode 100644 index 0000000000..9901a68615 --- /dev/null +++ b/vllm/model_executor/models/lilicorr.py @@ -0,0 +1,546 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# LiLiCorr head adapted from sgl-project/sglang PR #37462 (Apache-2.0). +"""DFlash backbone with the LiLiCorr candidate-lattice correlator.""" + +from collections.abc import Iterable +from typing import Any + +import torch +import torch.nn.functional as F +from torch import nn + +from vllm.compilation.backends import set_model_tag +from vllm.compilation.decorators import support_torch_compile +from vllm.config import VllmConfig +from vllm.model_executor.layers.linear import ( + LinearBase, + ReplicatedLinear, + UnquantizedLinearMethod, +) +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.quantization.base_config import QuantizationConfig +from vllm.model_executor.layers.quantization.modelopt import ModelOptLinearMethod +from vllm.model_executor.layers.quantization.utils.quant_utils import kNvfp4Static +from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead + +from .nemotron_h import NemotronHMLP +from .qwen3_dflash import DFlashQwen3ForCausalLM, DFlashQwen3Model +from .qwen3_dflash2 import DFlash2Qwen3DecoderLayer +from .utils import AutoWeightsLoader, maybe_prefix + + +class LiLiCorrLatticeAttention(nn.Module): + def __init__( + self, + hidden_size: int, + num_heads: int, + quant_config: QuantizationConfig | None = None, + prefix: str = "", + ) -> None: + super().__init__() + if num_heads <= 0 or hidden_size % num_heads != 0: + raise ValueError( + f"LiLiCorr hidden_size={hidden_size} must be divisible " + f"by num_heads={num_heads}." + ) + self.hidden_size = hidden_size + self.num_heads = num_heads + self.head_dim = self.hidden_size // self.num_heads + # Exported lattice QKV parameters stay in the model's floating dtype. + self.in_proj_weight = nn.Parameter( + torch.empty(3 * hidden_size, hidden_size), requires_grad=False + ) + self.in_proj_bias = nn.Parameter( + torch.empty(3 * hidden_size), requires_grad=False + ) + self.out_proj = ReplicatedLinear( + hidden_size, + hidden_size, + return_bias=False, + quant_config=quant_config, + prefix=maybe_prefix(prefix, "out_proj"), + ) + + def forward( + self, hidden_states: torch.Tensor, attention_bias: torch.Tensor + ) -> torch.Tensor: + bsz, seq_len, _ = hidden_states.shape + qkv = F.linear(hidden_states, self.in_proj_weight, self.in_proj_bias) + q, k, v = qkv.chunk(3, dim=-1) + shape = (bsz, seq_len, self.num_heads, self.head_dim) + q = q.view(shape).transpose(1, 2) + k = k.view(shape).transpose(1, 2) + v = v.view(shape).transpose(1, 2) + # Attention and MMEncoderAttention cannot accept this learned per-head + # additive bias. The full candidate lattice has no KV-cache state. + out = F.scaled_dot_product_attention( + q, + k, + v, + attn_mask=attention_bias, + ) + return self.out_proj( + out.transpose(1, 2).reshape(bsz, seq_len, self.hidden_size) + ) + + +class LiLiCorrLayer(nn.Module): + def __init__( + self, + hidden_size: int, + num_heads: int, + mlp_ratio: float, + rms_norm_eps: float, + quant_config: QuantizationConfig | None = None, + prefix: str = "", + ) -> None: + super().__init__() + self.attn_norm = nn.RMSNorm(hidden_size, eps=rms_norm_eps) + self.attn = LiLiCorrLatticeAttention( + hidden_size, num_heads, quant_config, maybe_prefix(prefix, "attn") + ) + self.mlp_norm = nn.RMSNorm(hidden_size, eps=rms_norm_eps) + mlp_hidden_size = int(hidden_size * mlp_ratio) + self.mlp = NemotronHMLP( + config=None, + hidden_size=hidden_size, + intermediate_size=mlp_hidden_size, + hidden_act="silu", + bias=True, + quant_config=quant_config, + prefix=maybe_prefix(prefix, "mlp"), + ) + + def forward( + self, hidden_states: torch.Tensor, attention_bias: torch.Tensor + ) -> torch.Tensor: + hidden_states = hidden_states + self.attn( + self.attn_norm(hidden_states), attention_bias + ) + return hidden_states + self.mlp(self.mlp_norm(hidden_states)) + + +@support_torch_compile +class LiLiCorrHead(nn.Module): + num_candidate_features = 5 + + def __init__( + self, + *, + model_hidden_size: int, + block_size: int, + rms_norm_eps: float, + config: dict[str, Any], + quant_config: QuantizationConfig | None = None, + prefix: str = "", + ) -> None: + super().__init__() + hidden_size = config["lilicorr_hidden_size"] or model_hidden_size + self.block_size = block_size + self.num_candidate_slots = block_size - 1 + self.candidate_topk = config["lilicorr_candidate_topk"] + if ( + self.candidate_topk <= 0 + or self.candidate_topk > 16 + or self.candidate_topk & (self.candidate_topk - 1) + ): + raise ValueError("LiLiCorr candidate_topk must be a power of two <= 16.") + self.hidden_size = hidden_size + self.num_heads = config["lilicorr_num_heads"] + self.mlp_ratio = config["lilicorr_mlp_ratio"] + self.factor_dim = config["lilicorr_factor_dim"] + self.vector_eps = config["lilicorr_vector_eps"] + self.logit_scale = config["lilicorr_logit_scale"] + # Candidate IDs and embeddings are replicated across TP ranks, so each + # rank scores the same lattice; the MLPs use standard tensor parallelism. + self.token_proj = ( + nn.Identity() + if model_hidden_size == hidden_size + else ReplicatedLinear( + model_hidden_size, + hidden_size, + return_bias=False, + quant_config=quant_config, + prefix=maybe_prefix(prefix, "token_proj"), + ) + ) + self.pass_hidden_proj = ReplicatedLinear( + model_hidden_size, + hidden_size, + return_bias=False, + quant_config=quant_config, + prefix=maybe_prefix(prefix, "pass_hidden_proj"), + ) + self.feature_norm = nn.LayerNorm(self.num_candidate_features) + self.feature_mlp = NemotronHMLP( + config=None, + hidden_size=self.num_candidate_features, + intermediate_size=hidden_size, + hidden_act="silu", + output_size=hidden_size, + bias=True, + quant_config=quant_config, + prefix=maybe_prefix(prefix, "feature_mlp"), + ) + self.slot_embedding = nn.Parameter( + torch.zeros(1, 1, self.num_candidate_slots, 1, hidden_size) + ) + self.rank_embedding = nn.Parameter( + torch.zeros(1, 1, 1, self.candidate_topk, hidden_size) + ) + self.relative_slot_bias = nn.Parameter( + torch.zeros(self.num_heads, 2 * self.block_size - 1) + ) + self.same_slot_bias = nn.Parameter(torch.zeros(self.num_heads)) + self.context_proj = ReplicatedLinear( + model_hidden_size, + hidden_size, + return_bias=False, + quant_config=quant_config, + prefix=maybe_prefix(prefix, "context_proj"), + ) + self.layers = nn.ModuleList( + [ + LiLiCorrLayer( + hidden_size=hidden_size, + num_heads=self.num_heads, + mlp_ratio=self.mlp_ratio, + rms_norm_eps=rms_norm_eps, + quant_config=quant_config, + prefix=maybe_prefix(prefix, f"layers.{i}"), + ) + for i in range(config["lilicorr_num_layers"]) + ] + ) + self.output_norm = nn.RMSNorm(hidden_size, eps=rms_norm_eps) + self.anchor_norm = nn.RMSNorm(hidden_size, eps=rms_norm_eps) + self.factor_input_proj = ReplicatedLinear( + hidden_size * 3, + hidden_size, + return_bias=False, + quant_config=None, + prefix=maybe_prefix(prefix, "factor_input_proj"), + ) + # Each candidate gets outgoing and incoming vectors. After normalization, + # their dot product scores a transition from slot s to slot s+1: + # pair_score[s, i, j] = dot(out[s, i], incoming[s+1, j]) + # The anchor supplies the outgoing vector for the first slot. + self.out_head = ReplicatedLinear( + hidden_size, + self.factor_dim, + return_bias=False, + quant_config=None, + prefix=maybe_prefix(prefix, "out_head"), + ) + self.in_head = ReplicatedLinear( + hidden_size, + self.factor_dim, + return_bias=False, + quant_config=None, + prefix=maybe_prefix(prefix, "in_head"), + ) + self.anchor_out_head = ReplicatedLinear( + hidden_size, + self.factor_dim, + return_bias=False, + quant_config=quant_config, + prefix=maybe_prefix(prefix, "anchor_out_head"), + ) + self._fused_edge_weight: torch.Tensor | None = None + self._fused_edge_bias: torch.Tensor | None = None + self._factor_input_splits: tuple[torch.Tensor, ...] + self._attn_bias: torch.Tensor | None = None + self._rank_frac_col: torch.Tensor + self._is_top1_col: torch.Tensor + + @torch.no_grad() + def materialize_inference_buffers( + self, device: torch.device, dtype: torch.dtype + ) -> None: + """Build inference-only views/copies after checkpoint weights are loaded.""" + topk = self.candidate_topk + self._attn_bias = self._build_attention_bias(device=device, dtype=dtype) + # Both projections consume the same candidate features f: + # out = W_out @ f + b_out + # incoming = W_in @ f + b_in + # Stack weights and biases to compute both in one linear call. score() + # splits the result and normalizes the two vectors separately. + self._fused_edge_weight = ( + torch.cat([self.out_head.weight, self.in_head.weight], dim=0) + .to(device=device, dtype=dtype) + .contiguous() + ) + self._fused_edge_bias = ( + torch.cat([self.out_head.bias, self.in_head.bias], dim=0) + .to(device=device, dtype=dtype) + .contiguous() + ) + # For candidate state x and request anchor a, the trained projection is: + # W @ concat(x, a, x*a) + b + # Split W into input blocks to evaluate the same expression as: + # W_x @ x + W_a @ a + W_cross @ (x*a) + b + # Here x*a is elementwise. This avoids concatenating inputs and reuses + # W_a @ a across candidates, adding the original bias only once. + weight = self.factor_input_proj.weight + hdim = self.hidden_size + self._factor_input_splits = ( + weight[:, :hdim].contiguous(), + weight[:, hdim : 2 * hdim].contiguous(), + weight[:, 2 * hdim :].contiguous(), + ) + if topk > 1: + rank_frac = torch.arange(topk, device=device, dtype=torch.float32).view( + 1, 1, topk + ) / float(topk - 1) + else: + rank_frac = torch.zeros(1, 1, topk, device=device, dtype=torch.float32) + is_top1 = torch.zeros(1, 1, topk, device=device, dtype=torch.float32) + is_top1[..., 0] = 1.0 + self._rank_frac_col = rank_frac.contiguous() + self._is_top1_col = is_top1.contiguous() + + def _build_attention_bias( + self, *, device: torch.device, dtype: torch.dtype + ) -> torch.Tensor: + topk = self.candidate_topk + slot_ids = torch.arange( + self.num_candidate_slots, device=device, dtype=torch.long + ).repeat_interleave(topk) + # Attention flattens [slot, candidate] into one sequence. With topk=2, + # the candidate slot IDs are [0, 0, 1, 1, ...]. Each head learns: + # bias(q, k) = relative_bias[slot(q) - slot(k) + offset] + # + same_slot_bias * (slot(q) == slot(k)) + # The same-slot term includes distinct candidates at the same position. + # This is an additive preference, not a causal mask. Keep the trained + # offset = block_size-1 when score() selects a shorter draft's prefix, + # so each distance continues to use the same learned weight. + rel = slot_ids.view(-1, 1) - slot_ids.view(1, -1) + bias = self.relative_slot_bias[:, rel + self.block_size - 1] + same_slot = slot_ids.view(-1, 1) == slot_ids.view(1, -1) + bias = bias + same_slot.unsqueeze(0).to( + dtype=bias.dtype + ) * self.same_slot_bias.view(-1, 1, 1) + return bias.to(device=device, dtype=dtype).contiguous() + + def score( + self, + *, + token_embeddings: torch.Tensor, + candidate_log_probs: torch.Tensor, + pass_hidden: torch.Tensor, + anchor_hidden: torch.Tensor, + anchor_valid: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor]: + if self._attn_bias is None: + raise RuntimeError( + "Call materialize_inference_buffers() after loading the LiLiCorr head." + ) + bsz, n_slots, topk = candidate_log_probs.shape + if not 1 <= n_slots <= self.num_candidate_slots: + raise ValueError( + f"LiLiCorr supports 1..{self.num_candidate_slots} candidate slots, " + f"got {n_slots}." + ) + if topk != self.candidate_topk: + raise ValueError( + f"LiLiCorr expects candidate_topk={self.candidate_topk}, got {topk}." + ) + proj_dtype = self.slot_embedding.dtype + if token_embeddings.dtype != proj_dtype: + token_embeddings = token_embeddings.to(proj_dtype) + if pass_hidden.dtype != proj_dtype: + pass_hidden = pass_hidden.to(proj_dtype) + token_states = self.token_proj(token_embeddings) + pass_states = self.pass_hidden_proj(pass_hidden).unsqueeze(-2) + log_probs = candidate_log_probs.float() + features = torch.stack( + [ + log_probs, + log_probs.exp(), + log_probs - log_probs.max(dim=-1, keepdim=True).values, + self._rank_frac_col.expand_as(log_probs), + self._is_top1_col.expand_as(log_probs), + ], + dim=-1, + ) + hidden_states = token_states + pass_states + hidden_states = hidden_states + self.feature_mlp( + self.feature_norm(features.to(dtype=token_states.dtype)) + ) + hidden_states = hidden_states + self.slot_embedding[:, 0, :n_slots] + hidden_states = hidden_states + self.rank_embedding[:, 0] + hidden_states = hidden_states.reshape(bsz, n_slots * topk, self.hidden_size) + anchor_state = self.context_proj(anchor_hidden) + anchor_state = anchor_state * anchor_valid[:, None].to(anchor_state.dtype) + # Shorter drafts use the learned prefix; keep checkpoint parameter shapes. + lattice = n_slots * topk + attention_bias = self._attn_bias[None, :, :lattice, :lattice] + for layer in self.layers: + hidden_states = layer(hidden_states, attention_bias) + hidden_states = self.output_norm(hidden_states).reshape( + bsz, n_slots, topk, self.hidden_size + ) + anchor_state = self.anchor_norm(anchor_state) + w_self, w_anchor, w_cross = self._factor_input_splits + anchor_row = anchor_state[:, None, None, :] + pre = F.linear(hidden_states, w_self, self.factor_input_proj.bias) + pre = pre + F.linear(anchor_row, w_anchor) + pre = pre + F.linear(hidden_states * anchor_row, w_cross) + factor_hidden = F.silu(pre) + edges = F.linear(factor_hidden, self._fused_edge_weight, self._fused_edge_bias) + out_vec, in_vec = F.normalize( + edges.unflatten(-1, (2, self.factor_dim)), dim=-1, eps=self.vector_eps + ).unbind(-2) + anchor_out = F.normalize( + self.anchor_out_head(anchor_state), dim=-1, eps=self.vector_eps + ) + # start_scores[b, j] scores anchor -> candidate j in slot zero. + # pair_scores[b, s, i, j] scores candidate i in slot s -> candidate j + # in slot s+1. Normalized factor dot products encode learned compatibility; + # forward() scales them into logits for the candidate-path selector. + start_scores = (anchor_out[:, None, :] * in_vec[:, 0, :, :]).sum(dim=-1) + pair_scores = torch.matmul(out_vec[:, :-1], in_vec[:, 1:].transpose(-1, -2)) + return (start_scores, pair_scores) + + def forward( + self, + token_embeddings: torch.Tensor, + candidate_log_probs: torch.Tensor, + pass_hidden: torch.Tensor, + anchor_hidden: torch.Tensor, + anchor_valid: torch.Tensor, + ) -> torch.Tensor: + start, pairs = self.score( + token_embeddings=token_embeddings, + candidate_log_probs=candidate_log_probs, + pass_hidden=pass_hidden, + anchor_hidden=anchor_hidden, + anchor_valid=anchor_valid, + ) + start = self.logit_scale * start.float() + pairs = self.logit_scale * pairs.float() + # The shared walk starts at predecessor index zero; all first rows agree. + first = start[:, None, None, :].expand(-1, 1, self.candidate_topk, -1) + return torch.cat((first, pairs), dim=1) + + +class LiLiCorr(DFlashQwen3Model): + def __init__( + self, *, vllm_config: VllmConfig, start_layer_id: int = 0, prefix: str = "" + ): + spec = vllm_config.speculative_config + assert spec is not None + config = spec.draft_model_config.hf_config + draft_config = config.dflash_config + if not draft_config.get("lilicorr_enabled", True): + raise ValueError("LiLiCorr requires lilicorr_enabled=True.") + block_size = draft_config.get("block_size", getattr(config, "block_size", 0)) + if not 1 <= spec.num_speculative_tokens < block_size: + raise ValueError( + "LiLiCorr requires 1 <= num_speculative_tokens < trained block_size." + ) + taps = draft_config.get("conv_kernel_size", 0) + groups = draft_config.get("conv_group_size", 0) + if taps < 0 or groups < 0 or bool(taps) != bool(groups): + raise ValueError( + "LiLiCorr convolution geometry must enable both taps and group size." + ) + if taps: + self.decoder_layer_cls = DFlash2Qwen3DecoderLayer + super().__init__( + vllm_config=vllm_config, start_layer_id=start_layer_id, prefix=prefix + ) + with set_model_tag("lilicorr_head"): + self.lilicorr = LiLiCorrHead( + model_hidden_size=config.hidden_size, + block_size=block_size, + rms_norm_eps=config.rms_norm_eps, + config=draft_config, + quant_config=self.quant_config, + prefix=maybe_prefix(prefix, "lilicorr"), + ) + + +class LiLiCorrForCausalLM(DFlashQwen3ForCausalLM): + model_cls = LiLiCorr + + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): + spec = vllm_config.speculative_config + assert spec is not None + super().__init__(vllm_config=vllm_config, prefix=prefix) + if self.draft_id_to_target_id is not None: + raise ValueError("LiLiCorr candidates require the full target vocabulary.") + self.has_own_lm_head = bool(getattr(self.config, "has_own_lm_head", False)) + # The correlator consumes raw candidate-head log-softmax features. + self.candidate_logits_processor = LogitsProcessor(self.config.vocab_size) + + def compute_candidates( + self, hidden_states: torch.Tensor + ) -> tuple[torch.Tensor, torch.Tensor]: + return self.candidate_logits_processor.get_top_k_tokens( + self.lm_head, + hidden_states, + self.model.lilicorr.candidate_topk, + return_log_probs=True, + ) + + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]): + quantized_metadata: set[str] = set() + for name, module in self.named_modules(): + name = name.removeprefix("model.") + if not isinstance(module, (LinearBase, ParallelLMHead)) or isinstance( + module.quant_method, UnquantizedLinearMethod + ): + continue + required = {"weight", "bias"} + method = module.quant_method + if ( + isinstance(method, ModelOptLinearMethod) + and method.spec.weight == kNvfp4Static + ): + # NVFP4 scales are trained checkpoint data. Only W4A16's + # deprecated, unused input_scale may be absent. + required.update(("weight_scale", "weight_scale_2")) + if method.spec.activation is not None: + required.add("input_scale") + quantized_metadata.update( + f"{name}.{parameter}" + for parameter, _ in module.named_parameters(recurse=False) + if parameter not in required + ) + expected = { + name.removeprefix("model.") + for name, _ in self.named_parameters() + if ( + name.startswith("model.lilicorr.") + or ".attention_conv." in name + or ".mlp_conv." in name + or (self.has_own_lm_head and name.startswith("lm_head.")) + ) + and name.removeprefix("model.") not in quantized_metadata + } + seen: set[str] = set() + head_weights: list[tuple[str, torch.Tensor]] = [] + + def normalized_weights(): + for name, value in weights: + name = name.removeprefix("model.") + seen.add(name) + # The backbone stacks up_proj into gate_up_proj. Load the + # nongated correlator separately to avoid that transformation. + if name.startswith("lilicorr."): + head_weights.append((name.removeprefix("lilicorr."), value)) + else: + yield name, value + + super().load_weights(normalized_weights()) + AutoWeightsLoader(self.model.lilicorr).load_weights(head_weights) + missing = expected - seen + if missing: + raise ValueError( + f"LiLiCorr checkpoint coverage mismatch: missing={sorted(missing)}" + ) + head = self.model.lilicorr + parameter = head.slot_embedding + head.materialize_inference_buffers(parameter.device, parameter.dtype) diff --git a/vllm/model_executor/models/nano_nemotron_vl.py b/vllm/model_executor/models/nano_nemotron_vl.py index 0f4dc8b110..074a89abb9 100644 --- a/vllm/model_executor/models/nano_nemotron_vl.py +++ b/vllm/model_executor/models/nano_nemotron_vl.py @@ -34,6 +34,7 @@ from vllm.model_executor.models.interfaces import ( HasInnerState, IsHybrid, MultiModalEmbeddings, + SupportsEagle3, SupportsLoRA, SupportsMultiModal, SupportsMultiModalPruning, @@ -928,6 +929,7 @@ class NemotronH_Nano_VL_V2( nn.Module, HasInnerState, IsHybrid, + SupportsEagle3, SupportsMultiModal, SupportsMultiModalPruning, SupportsLoRA, diff --git a/vllm/model_executor/models/nemotron_h.py b/vllm/model_executor/models/nemotron_h.py index e5451d16ba..3f97deebe4 100644 --- a/vllm/model_executor/models/nemotron_h.py +++ b/vllm/model_executor/models/nemotron_h.py @@ -34,7 +34,7 @@ from vllm.config.parallel import ParallelConfig from vllm.distributed import get_ep_group, get_tensor_model_parallel_world_size from vllm.distributed.communication_op import tensor_model_parallel_all_gather from vllm.distributed.parallel_state import get_pp_group -from vllm.model_executor.layers.activation import ReLUSquaredActivation +from vllm.model_executor.layers.activation import get_act_fn from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.fused_moe import ( FusedMoEFactory, @@ -93,7 +93,7 @@ from vllm.transformers_utils.configs.nemotron_h import NemotronHConfig class NemotronHMLP(nn.Module): def __init__( self, - config: NemotronHConfig, + config: NemotronHConfig | None, hidden_size: int, intermediate_size: int, quant_config: QuantizationConfig | None = None, @@ -101,6 +101,9 @@ class NemotronHMLP(nn.Module): reduce_results: bool = True, is_sequence_parallel: bool = False, prefix: str = "", + *, + hidden_act: str = "relu2", + output_size: int | None = None, ) -> None: super().__init__() @@ -114,14 +117,14 @@ class NemotronHMLP(nn.Module): ) self.down_proj = RowParallelLinear( input_size=intermediate_size, - output_size=hidden_size, + output_size=hidden_size if output_size is None else output_size, bias=bias, quant_config=quant_config, reduce_results=reduce_results, disable_tp=is_sequence_parallel, prefix=f"{prefix}.down_proj", ) - self.act_fn = ReLUSquaredActivation() + self.act_fn = get_act_fn(hidden_act) def forward(self, x: torch.Tensor): x, _ = self.up_proj(x) diff --git a/vllm/model_executor/models/qwen3_dflash.py b/vllm/model_executor/models/qwen3_dflash.py index 1636e3852c..0f02b98233 100644 --- a/vllm/model_executor/models/qwen3_dflash.py +++ b/vllm/model_executor/models/qwen3_dflash.py @@ -4,6 +4,7 @@ import io from collections.abc import Iterable +import regex as re import torch import torch.nn.functional as F from torch import nn @@ -23,6 +24,7 @@ from vllm.model_executor.layers.linear import ( QKVParallelLinear, ReplicatedLinear, RowParallelLinear, + UnquantizedLinearMethod, ) from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.quantization.base_config import QuantizationConfig @@ -54,6 +56,31 @@ logger = init_logger(__name__) _SLIDING_ATTENTION = "sliding_attention" +_DRAFT_LAYER_PATTERN = re.compile(r"(? None: + """Add runtime layer aliases for checkpoint-local quant exclusions.""" + if quant_config is None or start_layer_id == 0: + return + exclusions = getattr(quant_config, "exclude_modules", None) + if not isinstance(exclusions, list): + return + + def offset_local_layer(match: re.Match[str]) -> str: + layer_idx = int(match.group(1)) + if layer_idx >= num_hidden_layers: + return match.group(0) + return f"layers.{layer_idx + start_layer_id}" + + for exclusion in tuple(exclusions): + global_exclusion = _DRAFT_LAYER_PATTERN.sub(offset_local_layer, exclusion) + if global_exclusion != exclusion and global_exclusion not in exclusions: + exclusions.append(global_exclusion) def _dflash_layer_causal(config: Qwen3Config, layer_idx: int) -> bool: @@ -394,6 +421,9 @@ class DFlashQwen3Model(nn.Module): self.config = vllm_config.speculative_config.draft_model_config.hf_config self.vocab_size = self.config.vocab_size self.quant_config = get_draft_quant_config(vllm_config) + _add_global_draft_layer_exclusions( + self.quant_config, start_layer_id, self.config.num_hidden_layers + ) drafter_config = getattr(self.config, "eagle_config", {}) drafter_config.update(getattr(self.config, "dflash_config", {})) @@ -473,14 +503,25 @@ class DFlashQwen3Model(nn.Module): has_bias: bool, ) -> None: self._hidden_norm_weight = self.hidden_norm.weight.data + self._context_qkv_projs = [a.qkv_proj for a in layers_attn] + self._context_q_sizes = [a.q_size for a in layers_attn] - # KV projection weights: [num_layers * 2 * kv_size, hidden_size] - kv_weights = [a.qkv_proj.weight[a.q_size :] for a in layers_attn] - self._fused_kv_weight = torch.cat(kv_weights, dim=0) - if has_bias: - kv_biases = [a.qkv_proj.bias[a.q_size :] for a in layers_attn] - self._fused_kv_bias: torch.Tensor | None = torch.cat(kv_biases, dim=0) + if all( + isinstance(proj.quant_method, UnquantizedLinearMethod) + for proj in self._context_qkv_projs + ): + # KV projection weights: [num_layers * 2 * kv_size, hidden_size] + kv_weights = [a.qkv_proj.weight[a.q_size :] for a in layers_attn] + self._fused_kv_weight: torch.Tensor | None = torch.cat(kv_weights, dim=0) + if has_bias: + kv_biases = [a.qkv_proj.bias[a.q_size :] for a in layers_attn] + self._fused_kv_bias: torch.Tensor | None = torch.cat(kv_biases, dim=0) + else: + self._fused_kv_bias = None else: + # Quantized linear weights may use packed storage that cannot be + # consumed by F.linear. Run each projection through its quant method. + self._fused_kv_weight = None self._fused_kv_bias = None # K-norm weights stacked into one contiguous [num_layers, head_dim] @@ -548,17 +589,26 @@ class DFlashQwen3Model(nn.Module): self._hidden_norm_weight, self._rms_norm_eps, ) - all_kv_flat = F.linear( - normed_context_states, self._fused_kv_weight, self._fused_kv_bias - ) + if self._fused_kv_weight is not None: + all_kv_flat = F.linear( + normed_context_states, self._fused_kv_weight, self._fused_kv_bias + ) + all_kv = all_kv_flat.view(num_ctx, num_layers, 2, num_kv_heads, head_dim) + else: + layer_kv = [] + for proj, q_size in zip( + self._context_qkv_projs, self._context_q_sizes, strict=True + ): + qkv, _ = proj(normed_context_states) + layer_kv.append( + qkv[:, q_size:].view(num_ctx, 2, num_kv_heads, head_dim) + ) + all_kv = torch.stack(layer_kv, dim=1) + # Single contiguous copy that separates K/V and transposes to - # layer-major layout. Result: [2, L, num_ctx, nkv, hd] contiguous. + # layer-major layout. Result: [2, L, num_ctx, nkv, hd] contiguous. # Indexing dim-0 gives contiguous [L, num_ctx, nkv, hd] for K and V. - all_kv = ( - all_kv_flat.view(num_ctx, num_layers, 2, num_kv_heads, head_dim) - .permute(2, 1, 0, 3, 4) - .contiguous() - ) + all_kv = all_kv.permute(2, 1, 0, 3, 4).contiguous() all_k = all_kv[0] # [L, num_ctx, nkv, hd], contiguous all_v = all_kv[1] # [L, num_ctx, nkv, hd], contiguous return all_k, all_v @@ -724,6 +774,11 @@ class DFlashQwen3ForCausalLM(Qwen3ForCausalLM): self.lm_head = ParallelLMHead( self.config.draft_vocab_size, self.config.hidden_size, + quant_config=( + get_draft_quant_config(vllm_config) + if getattr(self.config, "has_own_lm_head", False) + else None + ), prefix=maybe_prefix(prefix, "lm_head"), ) self.logits_processor = LogitsProcessor( diff --git a/vllm/model_executor/models/qwen3_dflash2.py b/vllm/model_executor/models/qwen3_dflash2.py index 2043307d02..861a7c5412 100644 --- a/vllm/model_executor/models/qwen3_dflash2.py +++ b/vllm/model_executor/models/qwen3_dflash2.py @@ -176,6 +176,7 @@ class DFlashGroupedConv(nn.Module): block_size: int, params_dtype: torch.dtype, prefix: str, + quant_config: QuantizationConfig | None = None, ) -> None: super().__init__() if hidden_size % group_size: @@ -195,7 +196,7 @@ class DFlashGroupedConv(nn.Module): 2 * taps * self.num_groups, bias=False, params_dtype=params_dtype, - quant_config=None, + quant_config=quant_config, prefix=maybe_prefix(prefix, "kernel_projection"), return_bias=False, ) @@ -254,6 +255,7 @@ class DFlash2Qwen3DecoderLayer(DFlashQwen3DecoderLayer): # Query tokens per request: the bonus token plus the mask tokens. block_size=1 + speculative_config.num_speculative_tokens, params_dtype=vllm_config.model_config.dtype, + quant_config=quant_config, ) self.attention_conv = DFlashGroupedConv( **conv_args, prefix=maybe_prefix(prefix, "attention_conv") diff --git a/vllm/model_executor/models/registry.py b/vllm/model_executor/models/registry.py index cdb13df8f5..b435d77ad7 100644 --- a/vllm/model_executor/models/registry.py +++ b/vllm/model_executor/models/registry.py @@ -633,6 +633,7 @@ _SPECULATIVE_DECODING_MODELS = { "EagleLlama4ForCausalLM": ("llama4_eagle", "EagleLlama4ForCausalLM"), "EagleMiniCPMForCausalLM": ("minicpm_eagle", "EagleMiniCPMForCausalLM"), "DFlashDraftModel": ("qwen3_dflash", "DFlashQwen3ForCausalLM"), + "LiLiCorrDraftModel": ("lilicorr", "LiLiCorrForCausalLM"), "DFlash2DraftModel": ("qwen3_dflash2", "DFlash2Qwen3ForCausalLM"), # Muse Glimmer's DFlash draft head, reusing the generic qwen3_dflash # implementation. EAGLEConfig rewrites a dflash draft's architecture to diff --git a/vllm/transformers_utils/configs/eagle.py b/vllm/transformers_utils/configs/eagle.py index 6b218cd0f3..4a5efead72 100644 --- a/vllm/transformers_utils/configs/eagle.py +++ b/vllm/transformers_utils/configs/eagle.py @@ -68,7 +68,9 @@ class EAGLEConfig(PreTrainedConfig): ) kwargs["architectures"] = [ arch - if arch.startswith("DFlash") or arch.endswith("DFlash") + if arch.startswith("DFlash") + or arch.endswith("DFlash") + or arch == "LiLiCorrDraftModel" else f"DFlash{arch}" for arch in self.model.architectures ] diff --git a/vllm/v1/worker/gpu/spec_decode/__init__.py b/vllm/v1/worker/gpu/spec_decode/__init__.py index b527c9f21b..ede733a755 100644 --- a/vllm/v1/worker/gpu/spec_decode/__init__.py +++ b/vllm/v1/worker/gpu/spec_decode/__init__.py @@ -15,6 +15,12 @@ def init_speculator(vllm_config: VllmConfig, device: torch.device): return ExtractHiddenStatesSpeculator(vllm_config, device) elif speculative_config.method == "dflash": + if "LiLiCorrDraftModel" in speculative_config.draft_model_config.architectures: + from vllm.v1.worker.gpu.spec_decode.lilicorr.speculator import ( + LiLiCorrSpeculator, + ) + + return LiLiCorrSpeculator(vllm_config, device) if "DFlash2DraftModel" in speculative_config.draft_model_config.architectures: from vllm.v1.worker.gpu.spec_decode.dflash2.speculator import ( DFlash2Speculator, diff --git a/vllm/v1/worker/gpu/spec_decode/dflash/speculator.py b/vllm/v1/worker/gpu/spec_decode/dflash/speculator.py index e3fd89a82a..cdcdd5fcba 100644 --- a/vllm/v1/worker/gpu/spec_decode/dflash/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/dflash/speculator.py @@ -322,6 +322,11 @@ class DFlashSpeculator(DraftModelSpeculator): context_slots, ) + def prepare_context_anchor( + self, input_batch: InputBatch, num_rejected: torch.Tensor + ) -> None: + """Publish context features required by a draft's candidate head.""" + @torch.inference_mode() def propose( self, @@ -369,6 +374,7 @@ class DFlashSpeculator(DraftModelSpeculator): else: hidden_states = last_hidden_states self.hidden_states[:num_target_tokens].copy_(hidden_states[:num_target_tokens]) + self.prepare_context_anchor(input_batch, num_rejected) if dummy_run and skip_attn_for_dummy_run: # Memory profiling path: block_tables / kv_cache_config are not initialized. diff --git a/vllm/v1/worker/gpu/spec_decode/dflash/utils.py b/vllm/v1/worker/gpu/spec_decode/dflash/utils.py index 2e58263623..d32587d266 100644 --- a/vllm/v1/worker/gpu/spec_decode/dflash/utils.py +++ b/vllm/v1/worker/gpu/spec_decode/dflash/utils.py @@ -61,8 +61,12 @@ def load_dflash_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mo target_lm_head = get_target_lm_head(target_model, target_language_model) draft_lm_head = getattr(dflash_model, "lm_head", None) - if target_lm_head is not None and _should_share( - dflash_model, "has_own_lm_head", draft_lm_head, target_lm_head + if ( + target_lm_head is not None + and not getattr(dflash_model.config, "has_own_lm_head", False) + and _should_share( + dflash_model, "has_own_lm_head", draft_lm_head, target_lm_head + ) ): if draft_lm_head is not None: del dflash_model.lm_head diff --git a/vllm/v1/worker/gpu/spec_decode/dflash2/speculator.py b/vllm/v1/worker/gpu/spec_decode/dflash2/speculator.py index 7db5284497..bb14621129 100644 --- a/vllm/v1/worker/gpu/spec_decode/dflash2/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/dflash2/speculator.py @@ -108,26 +108,93 @@ def _cache_draft_logits_kernel( tl.store(cached_candidate_ptr + cache_base + offsets, token_ids, mask=mask) +class CandidateSampler: + """The shared DFlash2/LiLiCorr candidate walk and realized proposal cache.""" + + def __init__( + self, max_num_reqs: int, num_steps: int, top_k: int, device: torch.device + ): + self.num_steps = num_steps + self.top_k = top_k + self.scores = torch.empty( + max_num_reqs, num_steps, top_k, dtype=torch.float32, device=device + ) + self.cached_candidate_ids = torch.zeros( + self.scores.shape, dtype=torch.int64, device=device + ) + + def sample( + self, + candidate_ids: torch.Tensor, + scores: torch.Tensor, + num_reqs: int, + sample_pos: torch.Tensor, + idx_mapping: torch.Tensor, + temperature: torch.Tensor, + seeds: torch.Tensor, + draft_tokens: torch.Tensor, + draft_logits: torch.Tensor | None, + use_fp64: bool, + ) -> None: + candidate_ids = candidate_ids.contiguous() + sample_pos = sample_pos.contiguous() + idx_mapping = idx_mapping.contiguous() + block_k = triton.next_power_of_2(self.top_k) + _selector_walk_kernel[(num_reqs,)]( + scores.contiguous(), + candidate_ids.contiguous(), + sample_pos, + idx_mapping, + temperature, + seeds, + draft_tokens, + self.scores, + num_steps=self.num_steps, + top_k=self.top_k, + BLOCK_K=block_k, + SAMPLE_PROBABILISTIC=draft_logits is not None, + USE_FP64=use_fp64, + num_warps=1, + ) + + if draft_logits is not None: + self._cache_draft_logits( + candidate_ids, num_reqs * self.num_steps, idx_mapping, draft_logits + ) + + def _cache_draft_logits( + self, + candidate_ids: torch.Tensor, + num_sample: int, + idx_mapping: torch.Tensor, + draft_logits: torch.Tensor, + ) -> None: + block_k = triton.next_power_of_2(self.top_k) + _cache_draft_logits_kernel[(num_sample,)]( + draft_logits, + self.cached_candidate_ids, + candidate_ids, + self.scores, + idx_mapping, + draft_logits.stride(0), + draft_logits.stride(1), + num_steps=self.num_steps, + top_k=self.top_k, + BLOCK_K=block_k, + num_warps=1, + ) + + class DFlash2Speculator(DFlashSpeculator): _speculator_name = "DFlash2" + _candidate_top_k_key = "selector_top_k" def __init__(self, vllm_config: VllmConfig, device: torch.device): super().__init__(vllm_config, device) draft_config = self.draft_model_config.hf_config.dflash_config - self.selector_top_k = int(draft_config["selector_top_k"]) - self._anchor_indices = ( - torch.arange(self.max_num_reqs, dtype=torch.int64, device=device) - * self.num_query_per_req - ) - self._selector_scores = torch.empty( - self.max_num_reqs, - self.num_speculative_steps, - self.selector_top_k, - dtype=torch.float32, - device=device, - ) - self._cached_candidate_ids = torch.zeros( - self._selector_scores.shape, dtype=torch.int64, device=device + self.top_k = int(draft_config[self._candidate_top_k_key]) + self.candidate_sampler = CandidateSampler( + self.max_num_reqs, self.num_speculative_steps, self.top_k, device ) def draft_logits_spec(self, vllm_config: VllmConfig) -> tuple[torch.dtype, float]: @@ -136,48 +203,6 @@ class DFlash2Speculator(DFlashSpeculator): # candidates. return torch.float32, -float("inf") - def _sample_path( - self, - candidate_ids: torch.Tensor, - scores: torch.Tensor, - num_reqs: int, - ) -> None: - block_k = triton.next_power_of_2(self.selector_top_k) - _selector_walk_kernel[(num_reqs,)]( - scores.contiguous(), - candidate_ids.contiguous(), - self.sample_pos, - self.sample_idx_mapping, - self.temperature, - self.seeds, - self.draft_tokens, - self._selector_scores, - num_steps=self.num_speculative_steps, - top_k=self.selector_top_k, - BLOCK_K=block_k, - SAMPLE_PROBABILISTIC=self.draft_logits is not None, - USE_FP64=self.use_fp64_gumbel, - num_warps=1, - ) - - def _cache_draft_logits(self, candidate_ids: torch.Tensor, num_sample: int) -> None: - draft_logits = self.draft_logits - assert draft_logits is not None - block_k = triton.next_power_of_2(self.selector_top_k) - _cache_draft_logits_kernel[(num_sample,)]( - draft_logits, - self._cached_candidate_ids, - candidate_ids, - self._selector_scores, - self.sample_idx_mapping, - draft_logits.stride(0), - draft_logits.stride(1), - num_steps=self.num_speculative_steps, - top_k=self.selector_top_k, - BLOCK_K=block_k, - num_warps=1, - ) - def _generate_draft( self, num_reqs: int, @@ -202,22 +227,39 @@ class DFlash2Speculator(DFlashSpeculator): hidden_states.flatten(0, 1) ) candidate_ids = candidate_ids.view( - num_reqs, self.num_speculative_steps, self.selector_top_k + num_reqs, self.num_speculative_steps, self.top_k ) unary_logits = unary_logits.view_as(candidate_ids) - anchor_token_ids = self.input_buffers.input_ids[self._anchor_indices[:num_reqs]] - scores = self.model.model.candidate_selector( + scores = self._score_candidates(candidate_ids, unary_logits, hidden_states) + self.candidate_sampler.sample( candidate_ids, - unary_logits, - hidden_states, - anchor_token_ids, + scores, + num_reqs, + self.sample_pos, + self.sample_idx_mapping, + self.temperature, + self.seeds, + self.draft_tokens, + self.draft_logits, + self.use_fp64_gumbel, ) - self._sample_path(candidate_ids, scores, num_reqs) if self.enable_adaptive_verification: self._maybe_predict_acceptance( - self._selector_scores[:num_reqs].flatten(0, 1), + self.candidate_sampler.scores[:num_reqs].flatten(0, 1), self.sample_idx_mapping[:num_sample], self.sample_col[:num_sample], ) - if self.draft_logits is not None: - self._cache_draft_logits(candidate_ids, num_sample) + + def _score_candidates( + self, + candidate_ids: torch.Tensor, + unary_logits: torch.Tensor, + hidden_states: torch.Tensor, + ) -> torch.Tensor: + num_reqs = candidate_ids.shape[0] + anchor_token_ids = self.input_buffers.input_ids[ + : num_reqs * self.num_query_per_req : self.num_query_per_req + ] + return self.model.model.candidate_selector( + candidate_ids, unary_logits, hidden_states, anchor_token_ids + ) diff --git a/vllm/v1/worker/gpu/spec_decode/lilicorr/__init__.py b/vllm/v1/worker/gpu/spec_decode/lilicorr/__init__.py new file mode 100644 index 0000000000..208f01a7cb --- /dev/null +++ b/vllm/v1/worker/gpu/spec_decode/lilicorr/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project diff --git a/vllm/v1/worker/gpu/spec_decode/lilicorr/speculator.py b/vllm/v1/worker/gpu/spec_decode/lilicorr/speculator.py new file mode 100644 index 0000000000..213dd962c6 --- /dev/null +++ b/vllm/v1/worker/gpu/spec_decode/lilicorr/speculator.py @@ -0,0 +1,97 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import torch +from torch import nn + +from vllm.config import VllmConfig +from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding +from vllm.v1.worker.gpu.input_batch import InputBatch +from vllm.v1.worker.gpu.spec_decode.dflash2.speculator import DFlash2Speculator +from vllm.v1.worker.gpu.spec_decode.eagle.utils import get_target_lm_head + + +class LiLiCorrSpeculator(DFlash2Speculator): + _speculator_name = "LiLiCorr" + _candidate_top_k_key = "lilicorr_candidate_topk" + + def __init__(self, vllm_config: VllmConfig, device: torch.device): + super().__init__(vllm_config, device) + config = self.draft_model_config.hf_config + self.anchor_hidden = torch.zeros( + self.max_num_reqs, config.hidden_size, dtype=self.dtype, device=device + ) + self.anchor_valid = torch.zeros( + self.max_num_reqs, dtype=torch.bool, device=device + ) + + def load_draft_model( + self, target_model: nn.Module, target_attn_layer_names: set[str] + ) -> nn.Module: + model = super().load_draft_model(target_model, target_attn_layer_names) + language_model = ( + target_model.get_language_model() + if hasattr(target_model, "get_language_model") + else target_model + ) + inner = getattr(language_model, "model", language_model) + embedding = getattr(inner, "embed_tokens", None) or getattr( + inner, "embedding", None + ) + lm_head = ( + model.lm_head + if model.has_own_lm_head + else get_target_lm_head(target_model, language_model) + ) + if not isinstance(embedding, VocabParallelEmbedding) or not isinstance( + lm_head, VocabParallelEmbedding + ): + raise ValueError( + "LiLiCorr requires the target input embedding and its selected " + "LM head on the draft rank." + ) + if embedding.embedding_dim != model.config.hidden_size: + raise ValueError( + "LiLiCorr target embedding width must match its trained " + "token projection input." + ) + if ( + embedding.shard_indices.num_added_elements + or lm_head.shard_indices.num_added_elements + ): + raise ValueError("LiLiCorr does not support added vocabulary entries.") + # Candidate embeddings always come from the target; the checkpoint may + # supply its own candidate LM head. + self.target_embeddings = embedding + model.lm_head = lm_head + return model + + def prepare_context_anchor( + self, input_batch: InputBatch, num_rejected: torch.Tensor + ) -> None: + num_reqs = input_batch.num_reqs + starts = input_batch.query_start_loc[:num_reqs] + ends = input_batch.query_start_loc[1 : num_reqs + 1] - num_rejected[:num_reqs] + valid = ends > starts + indices = (ends - 1).clamp_min(0).long() + self.anchor_hidden.zero_() + self.anchor_valid.zero_() + self.anchor_hidden[:num_reqs].copy_( + self.model.model.hidden_norm(self.hidden_states[indices]) * valid[:, None] + ) + self.anchor_valid[:num_reqs].copy_(valid) + + def _score_candidates( + self, + candidate_ids: torch.Tensor, + unary_logits: torch.Tensor, + hidden_states: torch.Tensor, + ) -> torch.Tensor: + num_reqs = candidate_ids.shape[0] + return self.model.model.lilicorr( + self.target_embeddings(candidate_ids), + unary_logits, + hidden_states, + self.anchor_hidden[:num_reqs], + self.anchor_valid[:num_reqs], + )