[MRV2][Spec Decode] Quantized draft lm_head for drafters that share the target head (#59973)

Signed-off-by: Stefano Castagnetta <scastagnetta@nvidia.com>
Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
This commit is contained in:
Stefano Castagnetta
2026-10-15 22:06:40 +00:00
committed by GitHub
co-authored by Kevin H. Luu
parent d92a4f25f2
commit 784d1fff0c
11 changed files with 613 additions and 3 deletions
@@ -74,3 +74,12 @@ vllm serve XiaomiMiMo/MiMo-7B-Base \
is a good default to start with.
- If your model does not support MTP, use another method such as EAGLE or draft
model speculation.
- When the MTP head shares the target's unquantized lm_head (most MTP models),
`"draft_lm_head_quantization": "fp8"` or `"nvfp4"` in `--speculative-config`
gives the drafter a quantized copy of that head (Model Runner V2). The target
still verifies with its own head, so outputs are unchanged; only the
acceptance length can move. It helps most on bandwidth-bound GPUs with a
large vocabulary. The copy costs about one byte (FP8) or 0.56 bytes (NVFP4)
per lm_head weight, e.g. 0.6 or 0.35 GiB for a 248k x 2560 head; it is
counted in the model weights, so with `--kv-cache-memory-bytes` leave room
for it.
@@ -5,7 +5,13 @@ import pytest
from tests.utils import single_gpu_only
from vllm import SamplingParams
from vllm.platforms import current_platform
from ...utils import (
assert_request_outputs_match,
compute_acceptance_len,
get_test_prompts,
)
from .._correctness import check_mtp_correctness
@@ -37,3 +43,107 @@ def test_mtp_correctness(
expected_accuracy_threshold,
vllm_runner,
)
@single_gpu_only
def test_mtp_draft_lm_head_quantization(
monkeypatch: pytest.MonkeyPatch,
sampling_config: SamplingParams,
vllm_runner,
):
"""A quantized draft copy of the shared lm_head only changes drafting:
greedy outputs match stock MTP and acceptance stays within 10%."""
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
prompts = get_test_prompts(mm_enabled=False)
results = {}
# NVFP4 needs a CUDA W4A16 kernel.
quants = (None, "fp8", "nvfp4") if current_platform.is_cuda() else (None, "fp8")
for quant in quants:
with vllm_runner(
"Qwen/Qwen3.5-0.8B-Base",
block_size=None,
enable_chunked_prefill=None,
max_model_len=2048,
limit_mm_per_prompt={"image": 0, "video": 0},
speculative_config={
"method": "mtp",
"num_speculative_tokens": 3,
"draft_lm_head_quantization": quant,
},
disable_log_stats=False,
) as runner:
outputs = runner.llm.chat(prompts, sampling_config)
results[quant] = (outputs, compute_acceptance_len(runner.llm.get_metrics()))
ref_outputs, ref_al = results.pop(None)
for quant, (outputs, al) in results.items():
assert_request_outputs_match(
ref_outputs,
outputs,
# Not bit-exact: a different acceptance length changes the target's
# verify batch shapes, which can flip a near-tied greedy token.
required_matches=int(0.8 * len(ref_outputs)) + 1,
context=f"draft_lm_head_quantization={quant}",
)
assert al >= 0.9 * ref_al, (
f"{quant}: acceptance length {al:.3f} vs {ref_al:.3f}"
)
@single_gpu_only
@pytest.mark.skipif(
not current_platform.is_cuda(), reason="nvfp4 needs a CUDA W4A16 kernel"
)
def test_mtp_draft_lm_head_quantization_survives_sleep_and_reload(
monkeypatch: pytest.MonkeyPatch,
sampling_config: SamplingParams,
vllm_runner,
):
"""The quantized draft head is derived from the target head: it must come
back intact after a level 1 sleep and be re-derived after a target weight
reload, so outputs and acceptance match the run before."""
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
prompts = get_test_prompts(mm_enabled=False)
with vllm_runner(
"Qwen/Qwen3.5-0.8B-Base",
block_size=None,
enable_chunked_prefill=None,
max_model_len=2048,
limit_mm_per_prompt={"image": 0, "video": 0},
speculative_config={
"method": "mtp",
"num_speculative_tokens": 3,
"draft_lm_head_quantization": "nvfp4",
},
enable_sleep_mode=True,
disable_log_stats=False,
) as runner:
llm = runner.llm
def run():
before = llm.get_metrics()
outputs = llm.chat(prompts, sampling_config)
return outputs, compute_acceptance_len(llm.get_metrics(), before)
ref_outputs, ref_al = run()
llm.sleep(level=1)
llm.wake_up()
outputs, al = run()
assert_request_outputs_match(
ref_outputs,
outputs,
required_matches=int(0.8 * len(ref_outputs)) + 1,
context="after level 1 sleep",
)
assert al == pytest.approx(ref_al, rel=0.03)
llm.collective_rpc("reload_weights")
outputs, al = run()
assert_request_outputs_match(
ref_outputs,
outputs,
required_matches=int(0.8 * len(ref_outputs)) + 1,
context="after weight reload",
)
assert al == pytest.approx(ref_al, rel=0.03)
+217
View File
@@ -0,0 +1,217 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import gc
from types import SimpleNamespace
import pytest
import torch
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.quantization.online.lm_head import (
quantized_lm_head_copy,
refresh_quantized_lm_heads,
)
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
from vllm.model_executor.model_loader.reload import (
finalize_layerwise_reload,
initialize_layerwise_reload,
record_metadata_for_reloading,
)
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.platforms import current_platform
from vllm.v1.worker.gpu_worker import Worker
# Not a multiple of the vocab padding, so the head has padded rows.
VOCAB, HIDDEN = 4000, 512
requires_cuda = pytest.mark.skipif(
not torch.cuda.is_available(), reason="Marlin needs CUDA"
)
# NVFP4 needs a CUDA W4A16 kernel; ROCm only runs the fp8 cases.
requires_nvfp4 = pytest.mark.skipif(
not current_platform.is_cuda(), reason="nvfp4 needs a CUDA W4A16 kernel"
)
NVFP4 = pytest.param("nvfp4", marks=requires_nvfp4)
def _target_head(monkeypatch, seed: int = 0) -> ParallelLMHead:
# Online methods read the model dtype from the current config.
model_config = SimpleNamespace(dtype=torch.bfloat16)
monkeypatch.setattr(
"vllm.model_executor.layers.quantization.online.fp8.get_current_vllm_config",
lambda: SimpleNamespace(model_config=model_config),
)
head = ParallelLMHead(VOCAB, HIDDEN, disable_tp=True)
gen = torch.Generator().manual_seed(seed)
head.weight.data.copy_(torch.randn(head.weight.shape, generator=gen) * 0.02)
return head.to("cuda", torch.bfloat16)
def _tensors(module: torch.nn.Module) -> dict[str, torch.Tensor]:
return dict(module.named_parameters()) | dict(module.named_buffers())
@requires_cuda
@pytest.mark.parametrize(
("quantization", "max_rel_err", "min_top1"),
[("fp8", 0.05, 0.9), pytest.param("nvfp4", 0.15, 0.75, marks=requires_nvfp4)],
)
def test_tracks_bf16_head_through_logits_processor(
default_vllm_config, dist_init, monkeypatch, quantization, max_rel_err, min_top1
):
head = _target_head(monkeypatch)
target_weight = head.weight.detach().clone()
draft_head = quantized_lm_head_copy(head, quantization)
assert isinstance(draft_head, ParallelLMHead)
# The source head keeps its bf16 weight, and no bf16 copy is kept around.
assert torch.equal(head.weight, target_weight)
assert draft_head.weight.dtype != torch.bfloat16
logits_processor = LogitsProcessor(VOCAB)
hidden = torch.randn(2048, HIDDEN, device="cuda", dtype=torch.bfloat16)
ref = logits_processor(head, hidden).float()
out = logits_processor(draft_head, hidden).float()
assert out.shape == ref.shape == (2048, VOCAB)
rel_err = ((out - ref).norm() / ref.norm()).item()
top1 = (out.argmax(-1) == ref.argmax(-1)).float().mean().item()
assert rel_err < max_rel_err
assert top1 > min_top1
@requires_cuda
@pytest.mark.parametrize("quantization", ["fp8", NVFP4])
def test_refresh_follows_target_update(
default_vllm_config, dist_init, monkeypatch, quantization
):
"""After the target's weights change, the copy is re-derived in place: it
equals a fresh quantization of the new target, keeps its storage, and
frees the temporary copy without waiting for the cycle collector."""
head = _target_head(monkeypatch)
drafter = torch.nn.Module()
drafter.lm_head = quantized_lm_head_copy(head, quantization)
ptrs = {k: t.data_ptr() for k, t in _tensors(drafter.lm_head).items()}
new_weight = _target_head(monkeypatch, seed=1).weight.detach()
head.weight.data.copy_(new_weight)
gc.disable()
try:
allocated = torch.accelerator.memory_allocated()
refresh_quantized_lm_heads(drafter)
assert torch.accelerator.memory_allocated() == allocated
finally:
gc.enable()
expected = _tensors(quantized_lm_head_copy(head, quantization))
for name, tensor in _tensors(drafter.lm_head).items():
assert tensor.data_ptr() == ptrs[name]
if name != "workspace":
assert torch.equal(tensor, expected[name]), name
assert torch.equal(head.weight, new_weight)
def _draft_update_worker(vllm_config, drafter: torch.nn.Module, **engine) -> Worker:
worker = object.__new__(Worker)
worker.vllm_config = vllm_config
worker.weight_transfer_engine = SimpleNamespace(
supports_draft_weight_update=True,
reset_weight_update_target=lambda: None,
**engine,
)
worker.model_runner = SimpleNamespace(get_draft_model=lambda: drafter)
worker._weight_update_active = False
worker._set_draft_weight_update_target = lambda: None
return worker
@requires_cuda
def test_draft_weight_update_refreshes_copy(
default_vllm_config, dist_init, monkeypatch
):
"""A draft weight update can write a target weight the drafter shares, such
as an embedding tied to the lm_head; the copy must follow it."""
head = _target_head(monkeypatch)
drafter = torch.nn.Module()
drafter.lm_head = quantized_lm_head_copy(head, "fp8")
worker = _draft_update_worker(
default_vllm_config,
drafter,
start_weight_update=lambda: None,
finish_weight_update=lambda: None,
)
worker.start_draft_weight_update()
head.weight.data.copy_(_target_head(monkeypatch, seed=1).weight)
worker.finish_weight_update()
expected = _tensors(quantized_lm_head_copy(head, "fp8"))
for name, tensor in _tensors(drafter.lm_head).items():
assert torch.equal(tensor, expected[name]), name
@requires_cuda
@pytest.mark.parametrize("quantization", ["fp8", NVFP4])
def test_draft_weight_update_loads_lm_head(
default_vllm_config, dist_init, monkeypatch, quantization
):
"""A draft update carrying lm_head.weight loads the shared target head, as
without quantization, and the copy is re-derived from it in place."""
head = _target_head(monkeypatch)
record_metadata_for_reloading(head)
drafter = torch.nn.Module()
draft_head = quantized_lm_head_copy(head, quantization)
drafter.lm_head = draft_head
ptrs = {k: t.data_ptr() for k, t in _tensors(draft_head).items()}
new_weight = torch.randn(VOCAB, HIDDEN, device="cuda", dtype=torch.bfloat16)
def load(update_info):
param = drafter.get_parameter("lm_head.weight")
getattr(param, "weight_loader", default_weight_loader)(param, new_weight)
worker = _draft_update_worker(
default_vllm_config,
drafter,
start_weight_update=lambda: initialize_layerwise_reload(drafter),
update_weights=load,
finish_weight_update=lambda: finalize_layerwise_reload(drafter, None),
)
worker.start_draft_weight_update()
worker.update_weights({})
worker.finish_weight_update()
assert drafter.lm_head is draft_head
assert torch.equal(head.weight[:VOCAB], new_weight)
expected = _tensors(quantized_lm_head_copy(head, quantization))
for name, tensor in _tensors(draft_head).items():
assert tensor.data_ptr() == ptrs[name]
if name != "workspace":
assert torch.equal(tensor, expected[name]), name
hidden = torch.randn(64, HIDDEN, device="cuda", dtype=torch.bfloat16)
assert LogitsProcessor(VOCAB)(draft_head, hidden).isfinite().all()
@requires_cuda
@requires_nvfp4
def test_layerwise_reload_of_drafter_keeps_copy(
default_vllm_config, dist_init, monkeypatch
):
"""A drafter weight update reloads the drafter layerwise; the derived copy
must come back unchanged, not be re-quantized."""
head = _target_head(monkeypatch)
drafter = torch.nn.Module()
record_metadata_for_reloading(drafter)
drafter.lm_head = quantized_lm_head_copy(head, "nvfp4")
before = {k: t.clone() for k, t in _tensors(drafter.lm_head).items()}
def fail(*args, **kwargs):
raise AssertionError("the copy was re-quantized")
monkeypatch.setattr(
drafter.lm_head.quant_method, "process_weights_after_loading", fail
)
initialize_layerwise_reload(drafter)
finalize_layerwise_reload(drafter, None)
after = _tensors(drafter.lm_head)
assert after.keys() == before.keys()
for name, tensor in before.items():
assert torch.equal(after[name], tensor), name
@@ -65,6 +65,9 @@ class _RecordingModelRunner:
def reset_lora_state(self) -> None:
self.reset_lora_calls += 1
def get_draft_model(self) -> nn.Module | None:
return None
def _make_worker(engine: _RecordingEngine | None) -> Worker:
worker = object.__new__(Worker)
+20
View File
@@ -452,6 +452,12 @@ class SpeculativeConfig:
speculative methods. This is an experimental option for measuring the
acceptance-rate impact of reusing that block. It does not disable the
speculative drafter itself."""
draft_lm_head_quantization: Literal["fp8", "nvfp4"] | None = None
"""Quantize the drafter's copy of a target lm_head it shares (EAGLE and
MTP). "fp8" uses per-row weight and per-token activation scales on the
platform's FP8 linear kernel; "nvfp4" is weight-only on the platform's
W4A16 kernel. The target verifies with its own head, so only the
acceptance length can change. Model Runner V2 only."""
use_local_argmax_reduction: bool = False
"""Use vocab-parallel local argmax instead of all-gathering full logits
for draft token generation. Reduces communication from O(vocab_size) to
@@ -1780,6 +1786,20 @@ class SpeculativeConfig:
"n_predict parameter."
)
if self.draft_lm_head_quantization is not None:
# Gemma-4 MTP scores tied embedding rows directly; no lm_head to share.
if self.method not in ("eagle", "eagle3", "mtp") or self.use_gemma4_mtp():
raise ValueError(
"draft_lm_head_quantization is not supported for method "
f"{self.method!r}."
)
draft = self.draft_model_config
if draft is not None and draft.head_dtype != draft.dtype:
raise ValueError(
"draft_lm_head_quantization needs the lm_head to run in the "
f"model dtype, got head_dtype={draft.head_dtype}."
)
if self.num_speculative_tokens <= 0:
raise ValueError(
"Expected num_speculative_tokens to be greater "
+2
View File
@@ -3048,6 +3048,8 @@ class VllmConfig:
unsupported.append("dspark speculative decoding")
if self.speculative_config.enable_adaptive_verification:
unsupported.append("adaptive draft verification")
if self.speculative_config.draft_lm_head_quantization is not None:
unsupported.append("draft lm_head quantization")
# Mixed sliding/full DFlash drafts need multiple KV groups (V2 only).
if self._dflash_needs_multi_kv_group():
@@ -0,0 +1,204 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Online-quantized copies of an already-loaded lm_head."""
from itertools import chain
from typing import Literal
from weakref import WeakKeyDictionary
import torch
from torch.nn import Module, Parameter
from vllm.logger import init_logger
from vllm.model_executor.kernels.linear import init_nvfp4_linear_kernel
from vllm.model_executor.layers.linear import UnquantizedLinearMethod
from vllm.model_executor.layers.quantization.online.fp8 import (
Fp8PtpcOnlineLinearMethod,
OnlineLinearBase,
)
from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import (
FLOAT4_E2M1_MAX,
ref_nvfp4_quant,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import weight_amax
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
UnquantizedEmbeddingMethod,
)
from vllm.model_executor.model_loader.reload.layerwise import get_layerwise_info
from vllm.model_executor.utils import replace_parameter
from vllm.utils.torch_utils import set_default_torch_dtype
logger = init_logger(__name__)
def _quantize_nvfp4(
weight: torch.Tensor, chunk_rows: int = 8192
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""NVFP4-quantize the rows of `weight` without the SM100+ quant kernel.
Returns packed E2M1 codes (low nibble first), E4M3 scales per 16 values and
the fp32 dequantization global scale, in the checkpoint layout.
"""
fp8_max = torch.finfo(torch.float8_e4m3fn).max
global_scale = (weight_amax(weight).float() / (FLOAT4_E2M1_MAX * fp8_max)).clamp(
min=1e-12
)
inv_global_scale = (1.0 / global_scale).reshape(())
# E2M1 magnitudes 0, 0.5, ..., 6 doubled to integers, mapped to their codes.
code_of = torch.zeros(13, dtype=torch.uint8, device=weight.device)
code_of[torch.tensor([0, 1, 2, 3, 4, 6, 8, 12])] = torch.arange(
8, dtype=torch.uint8
).to(weight.device)
n, k = weight.shape
packed = torch.empty(n, k // 2, dtype=torch.uint8, device=weight.device)
scales = torch.empty(n, k // 16, dtype=torch.float8_e4m3fn, device=weight.device)
for start in range(0, n, chunk_rows):
rows = slice(start, start + chunk_rows)
values, block_scales = ref_nvfp4_quant(weight[rows], inv_global_scale, 16)
codes = code_of[(values.abs() * 2).long()] | ((values < 0).to(torch.uint8) << 3)
packed[rows] = codes[:, 0::2] | (codes[:, 1::2] << 4)
scales[rows] = block_scales.to(torch.float8_e4m3fn)
return packed, scales, global_scale
class _LMHeadNvfp4Method(OnlineLinearBase):
"""Weight-only NVFP4 lm_head copy on the platform's W4A16 kernel;
activations stay in the model dtype."""
def process_weights_after_loading(self, layer: Module) -> None:
packed, scales, global_scale = _quantize_nvfp4(layer.weight)
replace_parameter(layer, "weight", packed)
layer.weight_scale = Parameter(scales, requires_grad=False)
layer.weight_global_scale = Parameter(global_scale, requires_grad=False)
self.kernel = init_nvfp4_linear_kernel(use_a16=True)
self.kernel.process_weights_after_loading(layer)
def apply(
self, layer: Module, x: torch.Tensor, bias: torch.Tensor | None = None
) -> torch.Tensor:
return self.kernel.apply_weights(layer, x, bias)
_LM_HEAD_METHODS: dict[str, type[OnlineLinearBase]] = {
"fp8": Fp8PtpcOnlineLinearMethod,
"nvfp4": _LMHeadNvfp4Method,
}
def _quantize_copy(lm_head: ParallelLMHead, quantization: str) -> ParallelLMHead:
if (
not isinstance(
lm_head.quant_method, (UnquantizedEmbeddingMethod, UnquantizedLinearMethod)
)
or getattr(lm_head, "bias", None) is not None
or lm_head.weight.dtype not in (torch.float16, torch.bfloat16)
):
raise ValueError(
"Quantizing an lm_head copy needs an unquantized fp16/bf16 lm_head "
"without bias."
)
# Online methods take their output dtype from the default dtype, which is
# only the model dtype while the model itself is being built.
dtype = lm_head.weight.dtype
with set_default_torch_dtype(dtype):
head = ParallelLMHead(
lm_head.num_embeddings,
lm_head.embedding_dim,
params_dtype=dtype,
org_num_embeddings=lm_head.org_vocab_size,
padding_size=lm_head.padding_size,
prefix=lm_head.prefix,
disable_tp=lm_head.disable_tp,
quant_method=_LM_HEAD_METHODS[quantization](),
)
replace_parameter(head, "weight", lm_head.weight.data)
head.quant_method.process_weights_after_loading(head)
# The copy is never loaded. The online loader wrapper captures the layer,
# so keeping it would leave each copy to the cycle collector.
for param in head.parameters(recurse=False):
if hasattr(param, "weight_loader"):
del param.weight_loader
return head
# Each quantized copy maps to the head it is derived from. Weak keys, and no
# module attribute, so the copy never owns the source as a submodule.
_SOURCES: WeakKeyDictionary[Module, tuple[ParallelLMHead, str]] = WeakKeyDictionary()
def quantized_lm_head_copy(
lm_head: ParallelLMHead, quantization: Literal["fp8", "nvfp4"]
) -> ParallelLMHead:
"""A ParallelLMHead holding this rank's shard of `lm_head`, quantized.
"fp8" uses per-row weight and per-token activation scales on the platform's
FP8 linear kernel; "nvfp4" is weight-only on the
platform's W4A16 kernel. `lm_head` is left
unchanged and no bf16 copy of its weight is made. The copy is derived state:
`refresh_quantized_lm_heads` re-derives it after `lm_head` changes.
"""
head = _quantize_copy(lm_head, quantization)
# Keep it out of layerwise reload, which would re-run
# process_weights_after_loading on the already-quantized weight.
get_layerwise_info(head).reset()
_SOURCES[head] = (lm_head, quantization)
logger.info(
"Quantized a %s copy of the lm_head (%d x %d per rank).",
quantization,
head.output_size_per_partition,
head.input_size_per_partition,
)
return head
@torch.no_grad()
def refresh_quantized_lm_heads(model: Module) -> None:
"""Re-derive the quantized lm_head copies in `model` from their source heads.
Copies in place, so CUDA graphs keep pointing at valid tensors.
"""
for module in model.modules():
if module not in _SOURCES:
continue
fresh = _quantize_copy(*_SOURCES[module])
for name, tensor in chain(
fresh.named_parameters(recurse=False), fresh.named_buffers(recurse=False)
):
getattr(module, name).copy_(tensor)
# (parent, attribute, copy) for each copy swapped out by share_source_lm_heads.
_SWAPPED: dict[Module, list[tuple[Module, str, Module]]] = {}
def share_source_lm_heads(model: Module) -> None:
"""Replace each quantized copy in `model` with its source head.
`model` then loads weights like a drafter sharing the unquantized head,
until `restore_quantized_lm_heads` swaps the copies back in.
"""
swapped = [
(module, name, child)
for module in model.modules()
for name, child in module.named_children()
if child in _SOURCES
]
for module, name, child in swapped:
setattr(module, name, _SOURCES[child][0])
_SWAPPED[model] = swapped
def restore_quantized_lm_heads(model: Module, refresh: bool = True) -> None:
"""Undo `share_source_lm_heads`, then re-derive the copies in `model`.
Args:
model: The model passed to `share_source_lm_heads`, if any.
refresh: Whether to re-derive the copies; False when the source heads
may be mid-reload.
"""
for module, name, head in _SWAPPED.pop(model, ()):
setattr(module, name, head)
if refresh:
refresh_quantized_lm_heads(model)
@@ -607,6 +607,7 @@ class ParallelLMHead(VocabParallelEmbedding):
prefix: str = "",
*,
disable_tp: bool = False,
quant_method: QuantizeMethodBase | None = None,
):
self.has_bias = bias
super().__init__(
@@ -618,6 +619,7 @@ class ParallelLMHead(VocabParallelEmbedding):
quant_config,
prefix,
disable_tp=disable_tp,
quant_method=quant_method,
)
self.quant_config = quant_config
if bias:
+13 -1
View File
@@ -33,7 +33,7 @@ import vllm.envs as envs
from vllm.compilation.counter import compilation_counter
from vllm.compilation.cuda_graph import CUDAGraphStat
from vllm.compilation.wrapper import compile_model_with_stock_torch
from vllm.config import VllmConfig
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.config.compilation import CompilationMode, CUDAGraphMode
from vllm.distributed.aux_output_connector.worker import (
AuxOutputWorkerConnector,
@@ -46,6 +46,9 @@ from vllm.model_executor.layers.fused_moe.all2all_utils import get_ep_all2all_ma
from vllm.model_executor.layers.mamba.ops.ssu_dispatch import (
initialize_mamba_ssu_backend,
)
from vllm.model_executor.layers.quantization.online.lm_head import (
refresh_quantized_lm_heads,
)
from vllm.model_executor.model_loader import get_model_loader
from vllm.model_executor.model_loader.reload.inplace import reload_weights
from vllm.model_executor.models.interfaces import requires_raw_input_tokens
@@ -569,6 +572,15 @@ class GPUModelRunner(LoRAModelRunnerMixin):
return None
return speculator.model
def reset_lora_state(self) -> None:
super().reset_lora_state()
# Called whenever the target's weights are replaced (reload, weight
# transfer), so draft weights derived from them must follow.
draft_model = self.get_draft_model()
if draft_model is not None:
with set_current_vllm_config(self.vllm_config):
refresh_quantized_lm_heads(draft_model)
def reload_weights(
self,
weights_iterator: Iterable[tuple[str, torch.Tensor]] | None = None,
+14 -2
View File
@@ -6,6 +6,9 @@ import torch.nn as nn
from vllm.config import VllmConfig, replace
from vllm.distributed.parallel_state import get_pp_group
from vllm.lora.layers.base import BaseLayerWithLoRA
from vllm.model_executor.layers.quantization.online.lm_head import (
quantized_lm_head_copy,
)
from vllm.model_executor.model_loader import get_model
from vllm.model_executor.model_loader.utils import get_draft_load_config
from vllm.model_executor.models.utils import PPMissingLayer
@@ -106,9 +109,14 @@ def load_eagle_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mod
if target_lm_head is not None and _should_share(
eagle_model, "has_own_lm_head", draft_lm_head, target_lm_head
):
draft_head = target_lm_head
if speculative_config.draft_lm_head_quantization is not None:
draft_head = quantized_lm_head_copy(
target_lm_head, speculative_config.draft_lm_head_quantization
)
if draft_lm_head is not None:
del eagle_model.lm_head
eagle_model.lm_head = target_lm_head
eagle_model.lm_head = draft_head
# MTP layers route logits through layer.shared_head.head, not
# eagle_model.lm_head, so the per-layer copies need fixing up too.
@@ -119,7 +127,11 @@ def load_eagle_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mod
sh = getattr(layer, "shared_head", None)
if sh is not None and hasattr(sh, "head"):
del sh.head
sh.head = target_lm_head
sh.head = draft_head
elif speculative_config.draft_lm_head_quantization is not None:
raise ValueError(
"draft_lm_head_quantization needs a drafter that shares the target lm_head."
)
# MTP shares topk_indices_buffer with the target model. We update
# every module in the draft that holds a buffer reference so that
+19
View File
@@ -62,6 +62,10 @@ from vllm.distributed.weight_transfer import (
)
from vllm.logger import init_logger
from vllm.lora.request import LoRARequest
from vllm.model_executor.layers.quantization.online.lm_head import (
restore_quantized_lm_heads,
share_source_lm_heads,
)
from vllm.model_executor.model_loader import get_model_loader
from vllm.model_executor.model_loader.weight_checksum import (
compute_tensor_digests,
@@ -1544,12 +1548,19 @@ class Worker(WorkerBase):
"active. Call finish_weight_update first."
)
draft_model = self.get_draft_model() if is_draft else None
try:
if is_draft:
self._set_draft_weight_update_target()
if draft_model is not None:
# Load through the unquantized target head, as without
# draft_lm_head_quantization; finish re-derives the copies.
share_source_lm_heads(draft_model)
self.weight_transfer_engine.start_weight_update()
except BaseException:
self.weight_transfer_engine.reset_weight_update_target()
if draft_model is not None:
restore_quantized_lm_heads(draft_model, refresh=False)
raise
self._weight_update_active = True
self._weight_update_is_draft = is_draft
@@ -1589,6 +1600,10 @@ class Worker(WorkerBase):
except BaseException:
self._weight_update_active = False
self.weight_transfer_engine.reset_weight_update_target()
if self._weight_update_is_draft and (
(draft_model := self.get_draft_model()) is not None
):
restore_quantized_lm_heads(draft_model, refresh=False)
raise
def finish_weight_update(self) -> None:
@@ -1605,6 +1620,10 @@ class Worker(WorkerBase):
self.weight_transfer_engine.finish_weight_update()
self.weight_transfer_engine.reset_weight_update_target()
self._weight_update_active = False
if self._weight_update_is_draft and (
(draft_model := self.get_draft_model()) is not None
):
restore_quantized_lm_heads(draft_model)
# Weight transfer bypasses GPUModelRunner.reload_weights().
if not self._weight_update_is_draft: