[ModelRunner V2] Share identical MTP weights (#42538)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nick Hill
2026-05-13 18:57:04 +00:00
committed by GitHub
parent 40330967ab
commit a505cf807e
+54 -36
View File
@@ -1,11 +1,29 @@
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
import torch.nn as nn import torch.nn as nn
from vllm.config import VllmConfig from vllm.config import VllmConfig
from vllm.distributed.parallel_state import get_pp_group
from vllm.model_executor.model_loader import get_model from vllm.model_executor.model_loader import get_model
def _should_share(eagle: nn.Module, flag: str, draft, target) -> bool:
"""Share when the draft has no own copy, or its copy matches the target."""
if not getattr(eagle, flag, False) or draft is None:
return True
if target is None:
return False
# torch.equal on GPU allocates a bool mask the size of the input.
# Use the faster GPU path when there is plenty of headroom;
# otherwise compare on CPU.
w = draft.weight
if w.is_cuda and torch.cuda.mem_get_info(w.device)[0] < w.numel() * 2:
return torch.equal(w.cpu(), target.weight.cpu())
return torch.equal(w, target.weight)
def load_eagle_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Module: def load_eagle_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Module:
from vllm.compilation.backends import set_model_tag from vllm.compilation.backends import set_model_tag
@@ -17,51 +35,51 @@ def load_eagle_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mod
vllm_config=vllm_config, model_config=draft_model_config vllm_config=vllm_config, model_config=draft_model_config
) )
# Share target embeddings when the draft checkpoint does not include target_language_model = (
# its own vocab embedding table. target_model.get_language_model()
share_embeddings = True if hasattr(target_model, "get_language_model")
if hasattr(eagle_model, "has_own_embed_tokens"): else target_model
share_embeddings = not eagle_model.has_own_embed_tokens )
if share_embeddings: target_inner = target_language_model.model
target_language_model = ( draft_inner = eagle_model.model
target_model.get_language_model()
if hasattr(target_model, "get_language_model") # Skip embedding sharing under PP — each rank owns its own embedding.
else target_model if get_pp_group().world_size == 1:
target_embed = getattr(target_inner, "embed_tokens", None) or getattr(
target_inner, "embedding", None
) )
inner_model = getattr(target_language_model, "model", None) draft_embed = getattr(draft_inner, "embed_tokens", None)
target_embed_tokens = None if target_embed is not None and _should_share(
if inner_model is not None: eagle_model, "has_own_embed_tokens", draft_embed, target_embed
if hasattr(inner_model, "embed_tokens"): ):
target_embed_tokens = inner_model.embed_tokens if draft_embed is not None:
elif hasattr(inner_model, "embedding"): del draft_inner.embed_tokens
target_embed_tokens = inner_model.embedding draft_inner.embed_tokens = target_embed
if target_embed_tokens is not None and hasattr(eagle_model, "model"):
if hasattr(eagle_model.model, "embed_tokens"):
del eagle_model.model.embed_tokens
eagle_model.model.embed_tokens = target_embed_tokens
# Only share target lm_head when the draft model does not own one. target_lm_head = getattr(target_model, "lm_head", None)
share_lm_head = True draft_lm_head = getattr(eagle_model, "lm_head", None)
if hasattr(eagle_model, "has_own_lm_head"): if target_lm_head is not None and _should_share(
share_lm_head = not eagle_model.has_own_lm_head eagle_model, "has_own_lm_head", draft_lm_head, target_lm_head
if share_lm_head and hasattr(target_model, "lm_head"): ):
if hasattr(eagle_model, "lm_head"): if draft_lm_head is not None:
del eagle_model.lm_head del eagle_model.lm_head
eagle_model.lm_head = target_model.lm_head eagle_model.lm_head = target_lm_head
# MTP models call compute_logits via shared_head.head (a # MTP layers route logits through layer.shared_head.head, not
# ParallelLMHead inside each MTP layer), not self.model.lm_head. # eagle_model.lm_head, so the per-layer copies need fixing up too.
# If the checkpoint omits a copy of the lm_head weights at the layers = getattr(draft_inner, "layers", None)
# MTP layer path, shared_head.head stays uninitialised and
# produces zero/NaN logits. Share it explicitly from the target.
inner = getattr(eagle_model, "model", None)
layers = getattr(inner, "layers", None) if inner is not None else None
if layers is not None: if layers is not None:
items = layers.values() if isinstance(layers, nn.ModuleDict) else layers items = layers.values() if isinstance(layers, nn.ModuleDict) else layers
for layer in items: for layer in items:
sh = getattr(layer, "shared_head", None) sh = getattr(layer, "shared_head", None)
if sh is not None and hasattr(sh, "head"): if sh is not None and hasattr(sh, "head"):
del sh.head del sh.head
sh.head = target_model.lm_head sh.head = target_lm_head
# MTP also shares a topk_indices_buffer between target and draft.
if hasattr(target_inner, "topk_indices_buffer"):
if hasattr(draft_inner, "topk_indices_buffer"):
del draft_inner.topk_indices_buffer
draft_inner.topk_indices_buffer = target_inner.topk_indices_buffer
return eagle_model return eagle_model