[Model] LongCat-Flash: scale MLA norms while loading and drop the post-load sweep (#60080)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
aoshen02
2026-10-06 08:47:02 +00:00
committed by GitHub
co-authored by Claude Opus 5.5
parent e11962fc1c
commit 31e2443c90
2 changed files with 30 additions and 104 deletions
+28 -59
View File
@@ -58,13 +58,17 @@ from vllm.model_executor.layers.linear import (
)
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.quantization.utils.int8_utils import block_dequant
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.model_executor.model_loader.weight_utils import (
LoaderFunction,
composed_weight_loader,
default_weight_loader,
)
from vllm.model_executor.models.deepseek_v2 import DeepseekV2MLAAttention
from vllm.model_executor.utils import set_weight_attrs
from vllm.sequence import IntermediateTensors
from .interfaces import SupportsLoRA, SupportsPP
@@ -80,6 +84,23 @@ from .utils import (
logger = init_logger(__name__)
def _scaled_weight_loader(scale: float) -> LoaderFunction:
return composed_weight_loader(default_weight_loader, lambda x: x * scale)
def scale_mla_lora_norms_on_load(self_attn: nn.Module, config) -> None:
"""Fold LongCat's MLA LoRA scaling into the q/kv_a_layernorm weight loaders."""
for norm, rank, enabled in (
(self_attn.q_a_layernorm, config.q_lora_rank, config.mla_scale_q_lora),
(self_attn.kv_a_layernorm, config.kv_lora_rank, config.mla_scale_kv_lora),
):
if enabled:
scale = (config.hidden_size / rank) ** 0.5
set_weight_attrs(
norm.weight, {"weight_loader": _scaled_weight_loader(scale)}
)
class FlashMLP(nn.Module):
"""Flash MLP layer."""
@@ -265,6 +286,8 @@ class FlashDecoderLayer(nn.Module):
for i in range(2)
]
)
for self_attn in self.self_attn:
scale_mla_lora_norms_on_load(self_attn, config)
self.input_layernorm = nn.ModuleList(
[RMSNorm(config.hidden_size, eps=config.rms_norm_eps) for i in range(2)]
)
@@ -450,6 +473,9 @@ class FlashModel(nn.Module):
for name, loaded_weight in weights:
if "rotary_emb.inv_freq" in name:
continue
# The MTP layers are loaded by LongCatFlashMTP.
if name.startswith("mtp."):
continue
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
@@ -461,9 +487,6 @@ class FlashModel(nn.Module):
name.endswith(".bias") or name.endswith("_bias")
) and name not in params_dict:
continue
# Skip mtp
if ".mtp." in name:
continue
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
@@ -478,9 +501,6 @@ class FlashModel(nn.Module):
continue
is_expert_weight = True
name_mapped = name.replace(weight_name, param_name)
# Skip mtp
if ".mtp." in name_mapped:
continue
if (
name_mapped.endswith(".bias") or name_mapped.endswith("_bias")
) and name not in params_dict:
@@ -515,9 +535,6 @@ class FlashModel(nn.Module):
# Skip loading kv_scale from ckpts towards new design.
if name.endswith(".kv_scale") and name not in params_dict:
continue
# Skip mtp
if ".mtp." in name:
continue
if name is None:
continue
if is_pp_missing_parameter(name, self):
@@ -528,54 +545,6 @@ class FlashModel(nn.Module):
)
weight_loader(param, loaded_weight)
loaded_params.add(name)
for layer_id in range(self.config.num_layers):
for i in range(2):
if isinstance(self.layers[layer_id], PPMissingLayer):
continue
self_attn = self.layers[layer_id].self_attn[i]
if (
self.quant_config is not None
and hasattr(self.quant_config, "weight_block_size")
and self_attn.kv_b_proj.weight.dtype
in (
torch.float8_e4m3fn,
torch.float8_e4m3fnuz,
)
):
weight_block_size = self.quant_config.weight_block_size
if weight_block_size is not None:
assert hasattr(self_attn.kv_b_proj, "weight_scale_inv")
dtype = torch.get_default_dtype()
w = block_dequant(
self_attn.kv_b_proj.weight,
self_attn.kv_b_proj.weight_scale_inv,
weight_block_size,
).to(dtype)
else:
w = self_attn.kv_b_proj.weight
w_kc, w_vc = w.unflatten(
0, (-1, self_attn.qk_nope_head_dim + self_attn.v_head_dim)
).split([self_attn.qk_nope_head_dim, self_attn.v_head_dim], dim=1)
self_attn.w_kc = w_kc.transpose(1, 2).contiguous().transpose(1, 2)
self_attn.w_vc = w_vc.contiguous().transpose(1, 2)
# Guard against compounding on incremental load_weights calls:
# the in-place ``*=`` would otherwise re-apply the MLA LoRA
# scaling to the layernorm weights on each pass.
if self.config.mla_scale_q_lora and not getattr(
self_attn, "_mla_q_lora_scaled", False
):
self_attn.q_a_layernorm.weight.data *= (
self.config.hidden_size / self.config.q_lora_rank
) ** 0.5
self_attn._mla_q_lora_scaled = True
if self.config.mla_scale_kv_lora and not getattr(
self_attn, "_mla_kv_lora_scaled", False
):
self_attn.kv_a_layernorm.weight.data *= (
self.config.hidden_size / self.config.kv_lora_rank
) ** 0.5
self_attn._mla_kv_lora_scaled = True
return loaded_params
@@ -14,7 +14,6 @@ from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import ReplicatedLinear
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.quantization.utils.int8_utils import block_dequant
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
@@ -23,6 +22,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.sequence import IntermediateTensors
from .deepseek_v2 import DeepseekV2DecoderLayer
from .longcat_flash import scale_mla_lora_norms_on_load
from .utils import maybe_prefix
@@ -45,6 +45,7 @@ class LongCatMultiTokenPredictorLayer(nn.Module):
prefix="eh_proj",
)
self.mtp_block = DeepseekV2DecoderLayer(vllm_config, prefix)
scale_mla_lora_norms_on_load(self.mtp_block.self_attn, config)
self.final_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
def forward(
@@ -263,50 +264,6 @@ class LongCatFlashMTP(nn.Module):
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
spec_layer_id = self.model.mtp_start_layer_idx
self_attn = self.model.layers[str(spec_layer_id)].mtp_block.self_attn
if (
self.quant_config is not None
and hasattr(self.quant_config, "weight_block_size")
and self_attn.kv_b_proj.weight.dtype
in (
torch.float8_e4m3fn,
torch.float8_e4m3fnuz,
)
):
weight_block_size = self.quant_config.weight_block_size
if weight_block_size is not None:
dtype = torch.get_default_dtype()
w = block_dequant(
self_attn.kv_b_proj.weight,
self_attn.kv_b_proj.weight_scale_inv,
weight_block_size,
).to(dtype)
else:
w = self_attn.kv_b_proj.weight
else:
w = self_attn.kv_b_proj.weight
w_kc, w_vc = w.unflatten(
0, (-1, self_attn.qk_nope_head_dim + self_attn.v_head_dim)
).split([self_attn.qk_nope_head_dim, self_attn.v_head_dim], dim=1)
self_attn.w_kc = w_kc.transpose(1, 2).contiguous().transpose(1, 2)
self_attn.w_vc = w_vc.contiguous().transpose(1, 2)
# Guard against compounding on incremental load_weights calls (the
# in-place *= would otherwise double-apply the LoRA scaling).
if self.config.mla_scale_q_lora and not getattr(
self_attn, "_mla_q_lora_scaled", False
):
self_attn.q_a_layernorm.weight.data *= (
self.config.hidden_size / self.config.q_lora_rank
) ** 0.5
self_attn._mla_q_lora_scaled = True
if self.config.mla_scale_kv_lora and not getattr(
self_attn, "_mla_kv_lora_scaled", False
):
self_attn.kv_a_layernorm.weight.data *= (
self.config.hidden_size / self.config.kv_lora_rank
) ** 0.5
self_attn._mla_kv_lora_scaled = True
return loaded_params
def _rewrite_spec_layer_name(