Add Muse Glimmer model support (#51655)
Signed-off-by: Tiezhen Wang <tiezhen@inferact.ai> Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai> Signed-off-by: zixi-qi <zixi@inferact.ai> Co-authored-by: Beto de Paola <betodepaola@meta.com> Co-authored-by: Giancarlo Delfin <gdelfin@inferact.ai> Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com> Co-authored-by: zixi-qi <zixi@inferact.ai>
This commit is contained in:
co-authored by
Beto de Paola
Giancarlo Delfin
Jee Jee Li
Claude Opus 5
Jee Jee Li
zixi-qi
parent
59d1af5698
commit
6adad08767
@@ -1567,13 +1567,15 @@ class SupportsEagle3(SupportsEagleBase, Protocol):
|
||||
parent_ref = self.get_language_model()
|
||||
elif hasattr(self, "language_model"):
|
||||
parent_ref = self.language_model
|
||||
assert hasattr(parent_ref, "model"), (
|
||||
"Model instance must have 'model' attribute to set number of layers"
|
||||
)
|
||||
assert isinstance(parent_ref.model, EagleModelMixin), (
|
||||
# A multimodal model that builds its decoder inside
|
||||
# `_mark_language_model` has get_language_model() return the inner
|
||||
# decoder itself, which IS the EagleModelMixin and has no further
|
||||
# `.model`. Unwrap only when there is something to unwrap.
|
||||
holder = getattr(parent_ref, "model", parent_ref)
|
||||
assert isinstance(holder, EagleModelMixin), (
|
||||
"Model instance must inherit from EagleModelMixin to set auxiliary layers"
|
||||
)
|
||||
parent_ref.model._set_aux_hidden_state_layers(layers)
|
||||
holder._set_aux_hidden_state_layers(layers)
|
||||
|
||||
def get_eagle3_default_aux_hidden_state_layers(self) -> tuple[int, ...]:
|
||||
"""
|
||||
@@ -1590,13 +1592,12 @@ class SupportsEagle3(SupportsEagleBase, Protocol):
|
||||
parent_ref = self.get_language_model()
|
||||
elif hasattr(self, "language_model"):
|
||||
parent_ref = self.language_model
|
||||
assert hasattr(parent_ref, "model"), (
|
||||
"Model instance must have 'model' attribute to get number of layers"
|
||||
)
|
||||
assert hasattr(parent_ref.model, "layers"), (
|
||||
# Same unwrap-only-if-needed rule as set_aux_hidden_state_layers.
|
||||
holder = getattr(parent_ref, "model", parent_ref)
|
||||
assert hasattr(holder, "layers"), (
|
||||
"Model instance must have 'layers' attribute to get number of layers"
|
||||
)
|
||||
num_layers = len(parent_ref.model.layers)
|
||||
num_layers = len(holder.layers)
|
||||
return (2, num_layers // 2, num_layers - 3)
|
||||
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -72,6 +72,26 @@ def dflash_has_any_non_causal(config: Qwen3Config) -> bool:
|
||||
)
|
||||
|
||||
|
||||
def dflash_target_rope_is_neox_style(target_model: nn.Module) -> bool | None:
|
||||
"""The target's RoPE layout, from its first attention layer.
|
||||
|
||||
A DFlash head must rotate Q/K the way the target it was distilled against
|
||||
does, and a mismatch is silent — acceptance collapses but nothing errors and
|
||||
the output stays correct. Draft checkpoints do not carry this, so take it
|
||||
from the target. None if the target uses no RoPE.
|
||||
"""
|
||||
language_model = (
|
||||
target_model.get_language_model()
|
||||
if hasattr(target_model, "get_language_model")
|
||||
else target_model
|
||||
)
|
||||
for module in language_model.modules():
|
||||
style = getattr(module, "is_neox_style", None)
|
||||
if isinstance(style, bool):
|
||||
return style
|
||||
return None
|
||||
|
||||
|
||||
def _get_dflash_fc_input_size(vllm_config: VllmConfig) -> int:
|
||||
spec_config = vllm_config.speculative_config
|
||||
config = spec_config.draft_model_config.hf_config
|
||||
@@ -166,6 +186,7 @@ class DFlashQwen3Attention(nn.Module):
|
||||
add_swa_attention_sink_bias: bool = False,
|
||||
sliding_window: int | None = None,
|
||||
causal: bool = False,
|
||||
is_neox_style: bool = True,
|
||||
cache_config: CacheConfig | None = None,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
@@ -209,6 +230,7 @@ class DFlashQwen3Attention(nn.Module):
|
||||
self.rotary_emb = get_rope(
|
||||
self.head_dim,
|
||||
max_position=max_position,
|
||||
is_neox_style=is_neox_style,
|
||||
rope_parameters=rope_parameters,
|
||||
)
|
||||
|
||||
@@ -291,6 +313,13 @@ class DFlashQwen3DecoderLayer(nn.Module):
|
||||
# non-causal) from the draft config.
|
||||
sliding_window, causal = _resolve_layer_attention(config, layer_idx)
|
||||
|
||||
# RoPE layout, copied off the target at load time by the draft loader
|
||||
# (see `dflash_target_rope_is_neox_style`). Checkpoints do not carry it:
|
||||
# a head distilled from an interleaved-RoPE target must rotate the way
|
||||
# that target does, or every drafted Q/K is wrong and acceptance
|
||||
# collapses with no error raised.
|
||||
is_neox_style = getattr(config, "is_neox_style", True)
|
||||
|
||||
self.self_attn = DFlashQwen3Attention(
|
||||
hidden_size=self.hidden_size,
|
||||
num_heads=config.num_attention_heads,
|
||||
@@ -301,6 +330,7 @@ class DFlashQwen3DecoderLayer(nn.Module):
|
||||
add_swa_attention_sink_bias=add_swa_attention_sink_bias,
|
||||
sliding_window=sliding_window,
|
||||
causal=causal,
|
||||
is_neox_style=is_neox_style,
|
||||
head_dim=getattr(config, "head_dim", None),
|
||||
cache_config=cache_config,
|
||||
quant_config=quant_config,
|
||||
@@ -345,7 +375,14 @@ class DFlashQwen3DecoderLayer(nn.Module):
|
||||
@support_torch_compile
|
||||
class DFlashQwen3Model(nn.Module):
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_substr={"midlayer.": "layers.0."},
|
||||
orig_to_new_substr={
|
||||
"midlayer.": "layers.0.",
|
||||
# Muse-Glimmer-30B-assistant names the aux-hidden-state encoder
|
||||
# `encoder.fc` / `encoder.output_norm_enc`; this head calls them
|
||||
# `fc` / `hidden_norm`. Same tensors and shapes, different names.
|
||||
"encoder.output_norm_enc.": "hidden_norm.",
|
||||
"encoder.fc.": "fc.",
|
||||
},
|
||||
orig_to_new_stacked={
|
||||
".q_proj": (".qkv_proj", "q"),
|
||||
".k_proj": (".qkv_proj", "k"),
|
||||
|
||||
@@ -183,6 +183,7 @@ _TEXT_GENERATION_MODELS = {
|
||||
"NemotronHForCausalLM": ("nemotron_h", "NemotronHForCausalLM"),
|
||||
"NemotronHPuzzleForCausalLM": ("nemotron_h", "NemotronHForCausalLM"),
|
||||
"Olmo3ForCausalLM": ("olmo3", "Olmo3ForCausalLM"),
|
||||
"MuseGlimmerForCausalLM": ("muse_glimmer", "MuseGlimmerForCausalLM"),
|
||||
"OlmoHybridForCausalLM": ("olmo_hybrid", "OlmoHybridForCausalLM"),
|
||||
"OlmoeForCausalLM": ("olmoe", "OlmoeForCausalLM"),
|
||||
"OPTForCausalLM": ("opt", "OPTForCausalLM"),
|
||||
@@ -530,6 +531,7 @@ _MULTIMODAL_MODELS = {
|
||||
"NemotronH_Nano_Omni_Reasoning_V3": ("nano_nemotron_vl", "NemotronH_Nano_VL_V2"),
|
||||
"NemotronH_Super_Omni_Reasoning_V3": ("nano_nemotron_vl", "NemotronH_Nano_VL_V2"),
|
||||
"NVLM_D": ("nvlm_d", "NVLM_D_Model"),
|
||||
"MuseGlimmerForConditionalGeneration": ("muse_glimmer", "MuseGlimmerForCausalLM"),
|
||||
"OpenCUAForConditionalGeneration": ("opencua", "OpenCUAForConditionalGeneration"),
|
||||
"OpenPanguVLForConditionalGeneration": (
|
||||
"openpangu_vl",
|
||||
@@ -623,6 +625,14 @@ _SPECULATIVE_DECODING_MODELS = {
|
||||
"EagleLlama4ForCausalLM": ("llama4_eagle", "EagleLlama4ForCausalLM"),
|
||||
"EagleMiniCPMForCausalLM": ("minicpm_eagle", "EagleMiniCPMForCausalLM"),
|
||||
"DFlashDraftModel": ("qwen3_dflash", "DFlashQwen3ForCausalLM"),
|
||||
# Muse Glimmer's DFlash draft head, reusing the generic qwen3_dflash
|
||||
# implementation. EAGLEConfig rewrites a dflash draft's architecture to
|
||||
# DFlash{arch} unless it already starts or ends with "DFlash" (see
|
||||
# transformers_utils/configs/eagle.py), so the name the registry is asked
|
||||
# for is DFlashMuseGlimmerAssistantModel -- same convention as
|
||||
# DFlashLagunaForCausalLM. The bare name is kept as a defensive alias.
|
||||
"MuseGlimmerAssistantModel": ("qwen3_dflash", "DFlashQwen3ForCausalLM"),
|
||||
"DFlashMuseGlimmerAssistantModel": ("qwen3_dflash", "DFlashQwen3ForCausalLM"),
|
||||
"DSparkDraftModel": ("vllm.models.deepseek_v4", "DSparkDeepseekV4ForCausalLM"),
|
||||
"Qwen3DSparkModel": ("qwen3_dspark", "Qwen3DSparkForCausalLM"),
|
||||
"K3DSparkModel": (
|
||||
|
||||
Reference in New Issue
Block a user