Add Muse Glimmer model support (#51655)

Signed-off-by: Tiezhen Wang <tiezhen@inferact.ai>
Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai>
Signed-off-by: zixi-qi <zixi@inferact.ai>
Co-authored-by: Beto de Paola <betodepaola@meta.com>
Co-authored-by: Giancarlo Delfin <gdelfin@inferact.ai>
Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
Co-authored-by: zixi-qi <zixi@inferact.ai>
This commit is contained in:
Tiezhen WANG
2026-08-14 12:28:56 +08:00
committed by GitHub
co-authored by Beto de Paola Giancarlo Delfin Jee Jee Li Claude Opus 5 Jee Jee Li zixi-qi
parent 59d1af5698
commit 6adad08767
21 changed files with 4333 additions and 16 deletions
+11 -10
View File
@@ -1567,13 +1567,15 @@ class SupportsEagle3(SupportsEagleBase, Protocol):
parent_ref = self.get_language_model()
elif hasattr(self, "language_model"):
parent_ref = self.language_model
assert hasattr(parent_ref, "model"), (
"Model instance must have 'model' attribute to set number of layers"
)
assert isinstance(parent_ref.model, EagleModelMixin), (
# A multimodal model that builds its decoder inside
# `_mark_language_model` has get_language_model() return the inner
# decoder itself, which IS the EagleModelMixin and has no further
# `.model`. Unwrap only when there is something to unwrap.
holder = getattr(parent_ref, "model", parent_ref)
assert isinstance(holder, EagleModelMixin), (
"Model instance must inherit from EagleModelMixin to set auxiliary layers"
)
parent_ref.model._set_aux_hidden_state_layers(layers)
holder._set_aux_hidden_state_layers(layers)
def get_eagle3_default_aux_hidden_state_layers(self) -> tuple[int, ...]:
"""
@@ -1590,13 +1592,12 @@ class SupportsEagle3(SupportsEagleBase, Protocol):
parent_ref = self.get_language_model()
elif hasattr(self, "language_model"):
parent_ref = self.language_model
assert hasattr(parent_ref, "model"), (
"Model instance must have 'model' attribute to get number of layers"
)
assert hasattr(parent_ref.model, "layers"), (
# Same unwrap-only-if-needed rule as set_aux_hidden_state_layers.
holder = getattr(parent_ref, "model", parent_ref)
assert hasattr(holder, "layers"), (
"Model instance must have 'layers' attribute to get number of layers"
)
num_layers = len(parent_ref.model.layers)
num_layers = len(holder.layers)
return (2, num_layers // 2, num_layers - 3)
File diff suppressed because it is too large Load Diff
+38 -1
View File
@@ -72,6 +72,26 @@ def dflash_has_any_non_causal(config: Qwen3Config) -> bool:
)
def dflash_target_rope_is_neox_style(target_model: nn.Module) -> bool | None:
"""The target's RoPE layout, from its first attention layer.
A DFlash head must rotate Q/K the way the target it was distilled against
does, and a mismatch is silent — acceptance collapses but nothing errors and
the output stays correct. Draft checkpoints do not carry this, so take it
from the target. None if the target uses no RoPE.
"""
language_model = (
target_model.get_language_model()
if hasattr(target_model, "get_language_model")
else target_model
)
for module in language_model.modules():
style = getattr(module, "is_neox_style", None)
if isinstance(style, bool):
return style
return None
def _get_dflash_fc_input_size(vllm_config: VllmConfig) -> int:
spec_config = vllm_config.speculative_config
config = spec_config.draft_model_config.hf_config
@@ -166,6 +186,7 @@ class DFlashQwen3Attention(nn.Module):
add_swa_attention_sink_bias: bool = False,
sliding_window: int | None = None,
causal: bool = False,
is_neox_style: bool = True,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
@@ -209,6 +230,7 @@ class DFlashQwen3Attention(nn.Module):
self.rotary_emb = get_rope(
self.head_dim,
max_position=max_position,
is_neox_style=is_neox_style,
rope_parameters=rope_parameters,
)
@@ -291,6 +313,13 @@ class DFlashQwen3DecoderLayer(nn.Module):
# non-causal) from the draft config.
sliding_window, causal = _resolve_layer_attention(config, layer_idx)
# RoPE layout, copied off the target at load time by the draft loader
# (see `dflash_target_rope_is_neox_style`). Checkpoints do not carry it:
# a head distilled from an interleaved-RoPE target must rotate the way
# that target does, or every drafted Q/K is wrong and acceptance
# collapses with no error raised.
is_neox_style = getattr(config, "is_neox_style", True)
self.self_attn = DFlashQwen3Attention(
hidden_size=self.hidden_size,
num_heads=config.num_attention_heads,
@@ -301,6 +330,7 @@ class DFlashQwen3DecoderLayer(nn.Module):
add_swa_attention_sink_bias=add_swa_attention_sink_bias,
sliding_window=sliding_window,
causal=causal,
is_neox_style=is_neox_style,
head_dim=getattr(config, "head_dim", None),
cache_config=cache_config,
quant_config=quant_config,
@@ -345,7 +375,14 @@ class DFlashQwen3DecoderLayer(nn.Module):
@support_torch_compile
class DFlashQwen3Model(nn.Module):
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_substr={"midlayer.": "layers.0."},
orig_to_new_substr={
"midlayer.": "layers.0.",
# Muse-Glimmer-30B-assistant names the aux-hidden-state encoder
# `encoder.fc` / `encoder.output_norm_enc`; this head calls them
# `fc` / `hidden_norm`. Same tensors and shapes, different names.
"encoder.output_norm_enc.": "hidden_norm.",
"encoder.fc.": "fc.",
},
orig_to_new_stacked={
".q_proj": (".qkv_proj", "q"),
".k_proj": (".qkv_proj", "k"),
+10
View File
@@ -183,6 +183,7 @@ _TEXT_GENERATION_MODELS = {
"NemotronHForCausalLM": ("nemotron_h", "NemotronHForCausalLM"),
"NemotronHPuzzleForCausalLM": ("nemotron_h", "NemotronHForCausalLM"),
"Olmo3ForCausalLM": ("olmo3", "Olmo3ForCausalLM"),
"MuseGlimmerForCausalLM": ("muse_glimmer", "MuseGlimmerForCausalLM"),
"OlmoHybridForCausalLM": ("olmo_hybrid", "OlmoHybridForCausalLM"),
"OlmoeForCausalLM": ("olmoe", "OlmoeForCausalLM"),
"OPTForCausalLM": ("opt", "OPTForCausalLM"),
@@ -530,6 +531,7 @@ _MULTIMODAL_MODELS = {
"NemotronH_Nano_Omni_Reasoning_V3": ("nano_nemotron_vl", "NemotronH_Nano_VL_V2"),
"NemotronH_Super_Omni_Reasoning_V3": ("nano_nemotron_vl", "NemotronH_Nano_VL_V2"),
"NVLM_D": ("nvlm_d", "NVLM_D_Model"),
"MuseGlimmerForConditionalGeneration": ("muse_glimmer", "MuseGlimmerForCausalLM"),
"OpenCUAForConditionalGeneration": ("opencua", "OpenCUAForConditionalGeneration"),
"OpenPanguVLForConditionalGeneration": (
"openpangu_vl",
@@ -623,6 +625,14 @@ _SPECULATIVE_DECODING_MODELS = {
"EagleLlama4ForCausalLM": ("llama4_eagle", "EagleLlama4ForCausalLM"),
"EagleMiniCPMForCausalLM": ("minicpm_eagle", "EagleMiniCPMForCausalLM"),
"DFlashDraftModel": ("qwen3_dflash", "DFlashQwen3ForCausalLM"),
# Muse Glimmer's DFlash draft head, reusing the generic qwen3_dflash
# implementation. EAGLEConfig rewrites a dflash draft's architecture to
# DFlash{arch} unless it already starts or ends with "DFlash" (see
# transformers_utils/configs/eagle.py), so the name the registry is asked
# for is DFlashMuseGlimmerAssistantModel -- same convention as
# DFlashLagunaForCausalLM. The bare name is kept as a defensive alias.
"MuseGlimmerAssistantModel": ("qwen3_dflash", "DFlashQwen3ForCausalLM"),
"DFlashMuseGlimmerAssistantModel": ("qwen3_dflash", "DFlashQwen3ForCausalLM"),
"DSparkDraftModel": ("vllm.models.deepseek_v4", "DSparkDeepseekV4ForCausalLM"),
"Qwen3DSparkModel": ("qwen3_dspark", "Qwen3DSparkForCausalLM"),
"K3DSparkModel": (