[Model][Quantization] Add Ling-3.0-flash-fp8 support (#51265)

Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
zexplorerhj
2026-08-09 23:09:08 -07:00
committed by GitHub
co-authored by Isotr0py mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 3b4c86e489
commit ba1cdcfcf0
6 changed files with 298 additions and 36 deletions
+23 -2
View File
@@ -701,11 +701,32 @@ def test_non_fused_layer_unaffected():
def test_substr_match_on_fused_name(): def test_substr_match_on_fused_name():
# skip_with_substr=True path: fused-name substring match should also # Substring matching: a fused-name match should also
# short-circuit before shard expansion. # short-circuit before shard expansion.
assert is_layer_skipped( assert is_layer_skipped(
prefix="model.layers.0.self_attn.qkv_proj", prefix="model.layers.0.self_attn.qkv_proj",
ignored_layers=["self_attn.qkv_proj"], ignored_layers=["self_attn.qkv_proj"],
fused_mapping=FUSED_MAPPING, fused_mapping=FUSED_MAPPING,
skip_with_substr=True, match_mode="substring",
)
@pytest.mark.parametrize(
("prefix", "ignored_layer", "expected"),
[
("model.layers.0.self_attn.b_proj", "b_proj", True),
("model.layers.0.self_attn.q_b_proj", "b_proj", False),
("model.layers.0.self_attn.kv_b_proj", "b_proj", False),
("model.layers.5.self_attn.g_proj", "5.self_attn.g_proj", True),
("model.layers.6.self_attn.g_proj", "5.self_attn.g_proj", False),
],
)
def test_suffix_match_at_module_boundary(prefix, ignored_layer, expected):
assert (
is_layer_skipped(
prefix=prefix,
ignored_layers=[ignored_layer],
match_mode="suffix",
)
is expected
) )
@@ -292,7 +292,7 @@ class AutoAWQConfig(QuantizationConfig):
prefix, prefix,
self.modules_to_not_convert, self.modules_to_not_convert,
self.packed_modules_mapping, self.packed_modules_mapping,
skip_with_substr=True, match_mode="substring",
): ):
return UnquantizedLinearMethod() return UnquantizedLinearMethod()
@@ -335,7 +335,7 @@ class AutoAWQConfig(QuantizationConfig):
if is_layer_skipped( if is_layer_skipped(
prefix, prefix,
getattr(self, "modules_to_not_convert", []), getattr(self, "modules_to_not_convert", []),
skip_with_substr=True, match_mode="substring",
): ):
return UnquantizedFusedMoEMethod(layer.moe_config) return UnquantizedFusedMoEMethod(layer.moe_config)
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TYPE_CHECKING, Any from typing import TYPE_CHECKING, Any, Literal
import torch import torch
@@ -108,6 +108,9 @@ class Fp8Config(QuantizationConfig):
raise ValueError(f"Unsupported activation scheme {activation_scheme}") raise ValueError(f"Unsupported activation scheme {activation_scheme}")
self.activation_scheme = activation_scheme self.activation_scheme = activation_scheme
self.ignored_layers = ignored_layers or [] self.ignored_layers = ignored_layers or []
self.ignored_layers_match_mode: Literal["exact", "substring", "suffix"] = (
"exact"
)
self.store_dtype = store_dtype self.store_dtype = store_dtype
if weight_block_size is not None: if weight_block_size is not None:
if not is_checkpoint_fp8_serialized: if not is_checkpoint_fp8_serialized:
@@ -177,6 +180,7 @@ class Fp8Config(QuantizationConfig):
prefix=prefix, prefix=prefix,
ignored_layers=self.ignored_layers, ignored_layers=self.ignored_layers,
fused_mapping=self.packed_modules_mapping, fused_mapping=self.packed_modules_mapping,
match_mode=self.ignored_layers_match_mode,
): ):
return UnquantizedLinearMethod() return UnquantizedLinearMethod()
if not self.is_checkpoint_fp8_serialized: if not self.is_checkpoint_fp8_serialized:
@@ -196,6 +200,7 @@ class Fp8Config(QuantizationConfig):
prefix=prefix, prefix=prefix,
ignored_layers=self.ignored_layers, ignored_layers=self.ignored_layers,
fused_mapping=self.packed_modules_mapping, fused_mapping=self.packed_modules_mapping,
match_mode=self.ignored_layers_match_mode,
): ):
return UnquantizedFusedMoEMethod(layer.moe_config) return UnquantizedFusedMoEMethod(layer.moe_config)
if self.store_dtype == "mxfp4": if self.store_dtype == "mxfp4":
@@ -5,7 +5,7 @@
from collections.abc import Callable, Mapping from collections.abc import Callable, Mapping
from dataclasses import dataclass from dataclasses import dataclass
from types import MappingProxyType from types import MappingProxyType
from typing import TYPE_CHECKING, ClassVar, NamedTuple from typing import TYPE_CHECKING, ClassVar, Literal, NamedTuple
import numpy import numpy
import torch import torch
@@ -604,7 +604,7 @@ def is_layer_skipped(
ignored_layers: list[str], ignored_layers: list[str],
fused_mapping: Mapping[str, list[str]] = MappingProxyType({}), fused_mapping: Mapping[str, list[str]] = MappingProxyType({}),
*, *,
skip_with_substr: bool = False, match_mode: Literal["exact", "substring", "suffix"] = "exact",
) -> bool: ) -> bool:
def prefix_full_match(prefix: str, ignored_layers: list[str]) -> bool: def prefix_full_match(prefix: str, ignored_layers: list[str]) -> bool:
return prefix in ignored_layers return prefix in ignored_layers
@@ -613,7 +613,22 @@ def is_layer_skipped(
def substr_match(prefix: str, ignored_layers: list[str]) -> bool: def substr_match(prefix: str, ignored_layers: list[str]) -> bool:
return any(layer in prefix for layer in ignored_layers) return any(layer in prefix for layer in ignored_layers)
match_func = substr_match if skip_with_substr else prefix_full_match # Match abbreviated module paths at a component boundary. For example,
# ``b_proj`` matches ``model.layers.0.self_attn.b_proj`` but not
# ``model.layers.0.self_attn.q_b_proj``.
def suffix_match(prefix: str, ignored_layers: list[str]) -> bool:
return any(
prefix == layer or prefix.endswith(f".{layer}") for layer in ignored_layers
)
if match_mode == "exact":
match_func = prefix_full_match
elif match_mode == "substring":
match_func = substr_match
elif match_mode == "suffix":
match_func = suffix_match
else:
raise ValueError(f"Unsupported layer skip match mode: {match_mode}")
# prefix: model.layers.0.self_attn.q_proj # prefix: model.layers.0.self_attn.q_proj
# proj_name: q_proj # proj_name: q_proj
@@ -649,14 +664,11 @@ def is_layer_skipped(
"are quantized. All shards of fused layers " "are quantized. All shards of fused layers "
"to have the same precision." "to have the same precision."
) )
elif "experts" in prefix and not skip_with_substr: elif "experts" in prefix and match_mode == "exact":
expert_ignore_layers = filter( expert_ignore_layers = filter(
lambda layer_name: "experts" in layer_name, ignored_layers lambda layer_name: "experts" in layer_name, ignored_layers
) )
return any( return any(prefix in layer_name for layer_name in expert_ignore_layers)
prefix in layer_name if not skip_with_substr else layer_name in prefix
for layer_name in expert_ignore_layers
)
else: else:
is_skipped = match_func(prefix, ignored_layers) is_skipped = match_func(prefix, ignored_layers)
+224 -21
View File
@@ -10,6 +10,8 @@ vLLM's parallel linear layers, MLA kernel, KDA kernel and fused MoE loader.
import copy import copy
from collections.abc import Callable, Iterable from collections.abc import Callable, Iterable
from math import lcm
from typing import TypeGuard
import torch import torch
import torch.nn as nn import torch.nn as nn
@@ -55,6 +57,8 @@ from vllm.model_executor.layers.mla import (
MultiHeadLatentAttentionWrapper, MultiHeadLatentAttentionWrapper,
) )
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
from vllm.model_executor.layers.quantization.utils.quant_utils import is_layer_skipped
from vllm.model_executor.layers.rotary_embedding import get_rope from vllm.model_executor.layers.rotary_embedding import get_rope
from vllm.model_executor.layers.vocab_parallel_embedding import ( from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead, ParallelLMHead,
@@ -79,7 +83,13 @@ from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadata
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
from .interfaces import HasInnerState, IsHybrid, SupportsPP from .interfaces import HasInnerState, IsHybrid, SupportsPP
from .utils import PPMissingLayer, is_pp_missing_parameter, make_layers, maybe_prefix from .utils import (
PPMissingLayer,
WeightsMapper,
is_pp_missing_parameter,
make_layers,
maybe_prefix,
)
def bailing_v3_kda_attention( def bailing_v3_kda_attention(
@@ -193,6 +203,155 @@ def _load_a_log(param: torch.nn.Parameter, loaded_weight: torch.Tensor) -> None:
sharded_weight_loader(2)(param, loaded_weight) sharded_weight_loader(2)(param, loaded_weight)
def _is_block_fp8_config(
quant_config: QuantizationConfig | None,
) -> TypeGuard[Fp8Config]:
return (
isinstance(quant_config, Fp8Config)
and quant_config.weight_block_size is not None
)
def _configure_ling_fp8_quant_config(
quant_config: QuantizationConfig | None,
) -> None:
if _is_block_fp8_config(quant_config):
quant_config.ignored_layers_match_mode = "suffix"
def _is_fp8_module_excluded(
quant_config: QuantizationConfig | None,
prefix: str,
) -> bool:
"""Match Ling's abbreviated FP8 exclusions against mapped vLLM prefixes."""
if not _is_block_fp8_config(quant_config):
return False
return is_layer_skipped(
prefix=prefix,
ignored_layers=quant_config.ignored_layers,
fused_mapping=quant_config.packed_modules_mapping,
match_mode=quant_config.ignored_layers_match_mode,
)
def _get_block_fp8_mlp_padded_intermediate_size(
quant_config: QuantizationConfig | None,
intermediate_size: int,
prefix: str,
) -> int:
"""Pad a block-FP8 MLP so each TP shard contains whole quant blocks."""
if not _is_block_fp8_config(quant_config):
return intermediate_size
block_size = quant_config.weight_block_size
assert block_size is not None
alignments: list[int] = []
if not _is_fp8_module_excluded(quant_config, f"{prefix}.gate_up_proj"):
alignments.append(int(block_size[0]))
if not _is_fp8_module_excluded(quant_config, f"{prefix}.down_proj"):
alignments.append(int(block_size[1]))
if not alignments:
return intermediate_size
tp_size = get_tensor_model_parallel_world_size()
tp_alignment = tp_size * lcm(*alignments)
return (intermediate_size + tp_alignment - 1) // tp_alignment * tp_alignment
def _pad_block_fp8_mlp_checkpoint_tensor(
quant_config: QuantizationConfig,
name: str,
loaded_weight: torch.Tensor,
intermediate_size: int,
padded_intermediate_size: int,
) -> torch.Tensor:
"""Zero-pad an MLP checkpoint tensor on its intermediate dimension."""
if padded_intermediate_size == intermediate_size:
return loaded_weight
block_size = getattr(quant_config, "weight_block_size", None)
assert block_size is not None
if ".down_proj." in name:
if name.endswith(".bias"):
return loaded_weight
dim = 1
block = int(block_size[1])
logical_shards = 1
elif any(
projection in name
for projection in (".gate_proj.", ".up_proj.", ".gate_up_proj.")
):
dim = 0
block = int(block_size[0])
logical_shards = 2 if ".gate_up_proj." in name else 1
else:
return loaded_weight
if name.endswith(".weight_scale_inv"):
expected_shard_size = (intermediate_size + block - 1) // block
target_shard_size = (padded_intermediate_size + block - 1) // block
elif name.endswith((".weight", ".bias")):
expected_shard_size = intermediate_size
target_shard_size = padded_intermediate_size
else:
return loaded_weight
current_size = loaded_weight.shape[dim]
if current_size % logical_shards != 0:
raise ValueError(
f"Cannot split {name} dimension {current_size} into "
f"{logical_shards} logical shards."
)
current_shard_size = current_size // logical_shards
if current_shard_size == target_shard_size:
return loaded_weight
if current_shard_size != expected_shard_size:
raise ValueError(
f"Cannot pad {name}: expected each logical intermediate dimension "
f"to be {expected_shard_size}, but got {current_shard_size}."
)
padded_shards: list[torch.Tensor] = []
for shard in loaded_weight.split(current_shard_size, dim=dim):
pad_shape = list(shard.shape)
pad_shape[dim] = target_shard_size - current_shard_size
padded_shards.extend([shard, shard.new_zeros(pad_shape)])
return torch.cat(padded_shards, dim=dim)
def _maybe_pad_block_fp8_shared_expert_checkpoint_tensor(
quant_config: QuantizationConfig | None,
config: PretrainedConfig,
name: str,
loaded_weight: torch.Tensor,
) -> torch.Tensor:
if ".mlp.shared_experts." not in name:
return loaded_weight
shared_prefix = name.split(".shared_experts.", 1)[0] + ".shared_experts"
shared_intermediate = (
config.moe_shared_expert_intermediate_size * config.num_shared_experts
)
padded_shared_intermediate = _get_block_fp8_mlp_padded_intermediate_size(
quant_config,
shared_intermediate,
shared_prefix,
)
if padded_shared_intermediate == shared_intermediate:
return loaded_weight
assert quant_config is not None
return _pad_block_fp8_mlp_checkpoint_tensor(
quant_config,
name,
loaded_weight,
shared_intermediate,
padded_shared_intermediate,
)
class _IdentityProjection(nn.Module): class _IdentityProjection(nn.Module):
def forward(self, x: torch.Tensor) -> tuple[torch.Tensor, None]: def forward(self, x: torch.Tensor) -> tuple[torch.Tensor, None]:
return x, None return x, None
@@ -446,13 +605,36 @@ class BailingMoeV3KimiDeltaAttention(PluggableLayer, MambaBase):
projection_size = self.head_dim * self.num_heads projection_size = self.head_dim * self.num_heads
self.projection_size_per_partition = projection_size // self.tp_size self.projection_size_per_partition = projection_size // self.tp_size
self.qkvb_proj = MergedColumnParallelLinear( self.separate_b_proj = _is_fp8_module_excluded(quant_config, f"{prefix}.b_proj")
self.hidden_size, self.qkv_proj: MergedColumnParallelLinear | None
[projection_size, projection_size, projection_size, self.num_heads], self.b_proj: ColumnParallelLinear | None
bias=False, self.qkvb_proj: MergedColumnParallelLinear | None
quant_config=quant_config, if self.separate_b_proj:
prefix=f"{prefix}.qkvb_proj", self.qkv_proj = MergedColumnParallelLinear(
) self.hidden_size,
[projection_size, projection_size, projection_size],
bias=False,
quant_config=quant_config,
prefix=f"{prefix}.qkv_proj",
)
self.b_proj = ColumnParallelLinear(
self.hidden_size,
self.num_heads,
bias=False,
quant_config=quant_config,
prefix=f"{prefix}.b_proj",
)
self.qkvb_proj = None
else:
self.qkvb_proj = MergedColumnParallelLinear(
self.hidden_size,
[projection_size, projection_size, projection_size, self.num_heads],
bias=False,
quant_config=quant_config,
prefix=f"{prefix}.qkvb_proj",
)
self.qkv_proj = None
self.b_proj = None
self.f_proj = ColumnParallelLinear( self.f_proj = ColumnParallelLinear(
self.hidden_size, self.hidden_size,
projection_size, projection_size,
@@ -532,16 +714,23 @@ class BailingMoeV3KimiDeltaAttention(PluggableLayer, MambaBase):
) -> None: ) -> None:
del positions del positions
num_tokens = hidden_states.size(0) num_tokens = hidden_states.size(0)
qkvb = self.qkvb_proj(hidden_states)[0] if self.separate_b_proj:
q, k, v, beta_logits = qkvb.split( assert self.qkv_proj is not None and self.b_proj is not None
[ qkv = self.qkv_proj(hidden_states)[0]
self.projection_size_per_partition, q, k, v = qkv.split(self.projection_size_per_partition, dim=-1)
self.projection_size_per_partition, beta_logits = self.b_proj(hidden_states)[0]
self.projection_size_per_partition, else:
self.local_num_heads, assert self.qkvb_proj is not None
], qkvb = self.qkvb_proj(hidden_states)[0]
dim=-1, q, k, v, beta_logits = qkvb.split(
) [
self.projection_size_per_partition,
self.projection_size_per_partition,
self.projection_size_per_partition,
self.local_num_heads,
],
dim=-1,
)
beta = beta_logits.float().sigmoid().unsqueeze(0) beta = beta_logits.float().sigmoid().unsqueeze(0)
g1 = self.f_proj(hidden_states)[0] g1 = self.f_proj(hidden_states)[0]
g1 = fused_kda_gate( g1 = fused_kda_gate(
@@ -910,7 +1099,6 @@ class BailingMoeV3MoE(nn.Module):
prefix: str = "", prefix: str = "",
) -> None: ) -> None:
super().__init__() super().__init__()
self.tp_size = get_tensor_model_parallel_world_size()
self.num_experts = config.num_experts self.num_experts = config.num_experts
self.top_k = config.num_experts_per_tok self.top_k = config.num_experts_per_tok
self.hidden_size = config.hidden_size self.hidden_size = config.hidden_size
@@ -926,7 +1114,9 @@ class BailingMoeV3MoE(nn.Module):
config.moe_shared_expert_intermediate_size * config.num_shared_experts config.moe_shared_expert_intermediate_size * config.num_shared_experts
) )
self.shared_experts = BailingMoeV3MLP( self.shared_experts = BailingMoeV3MLP(
intermediate_size=shared_intermediate, intermediate_size=_get_block_fp8_mlp_padded_intermediate_size(
quant_config, shared_intermediate, f"{prefix}.shared_experts"
),
config=config, config=config,
quant_config=quant_config, quant_config=quant_config,
reduce_results=False, reduce_results=False,
@@ -1149,8 +1339,11 @@ class BailingMoeV3Model(nn.Module):
class BailingMoeV3ForCausalLM(nn.Module, HasInnerState, IsHybrid, SupportsPP): class BailingMoeV3ForCausalLM(nn.Module, HasInnerState, IsHybrid, SupportsPP):
hf_to_vllm_mapper = WeightsMapper(orig_to_new_substr={".attention.": ".self_attn."})
packed_modules_mapping = { packed_modules_mapping = {
"gate_up_proj": ["gate_proj", "up_proj"], "gate_up_proj": ["gate_proj", "up_proj"],
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
"qkvb_proj": ["q_proj", "k_proj", "v_proj", "b_proj"], "qkvb_proj": ["q_proj", "k_proj", "v_proj", "b_proj"],
} }
@@ -1158,6 +1351,7 @@ class BailingMoeV3ForCausalLM(nn.Module, HasInnerState, IsHybrid, SupportsPP):
super().__init__() super().__init__()
config = vllm_config.model_config.hf_config config = vllm_config.model_config.hf_config
quant_config = vllm_config.quant_config quant_config = vllm_config.quant_config
_configure_ling_fp8_quant_config(quant_config)
self.config = config self.config = config
self.quant_config = quant_config self.quant_config = quant_config
self.model = BailingMoeV3Model( self.model = BailingMoeV3Model(
@@ -1226,11 +1420,15 @@ class BailingMoeV3ForCausalLM(nn.Module, HasInnerState, IsHybrid, SupportsPP):
return self.model.get_expert_mapping() return self.model.get_expert_mapping()
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
weights = self.hf_to_vllm_mapper.apply(weights)
params_dict = dict(self.named_parameters(remove_duplicate=False)) params_dict = dict(self.named_parameters(remove_duplicate=False))
loaded_params: set[str] = set() loaded_params: set[str] = set()
stacked_mappings = [ stacked_mappings = [
(".gate_up_proj", ".gate_proj", 0), (".gate_up_proj", ".gate_proj", 0),
(".gate_up_proj", ".up_proj", 1), (".gate_up_proj", ".up_proj", 1),
(".qkv_proj", ".q_proj", 0),
(".qkv_proj", ".k_proj", 1),
(".qkv_proj", ".v_proj", 2),
(".qkvb_proj", ".q_proj", 0), (".qkvb_proj", ".q_proj", 0),
(".qkvb_proj", ".k_proj", 1), (".qkvb_proj", ".k_proj", 1),
(".qkvb_proj", ".v_proj", 2), (".qkvb_proj", ".v_proj", 2),
@@ -1263,13 +1461,18 @@ class BailingMoeV3ForCausalLM(nn.Module, HasInnerState, IsHybrid, SupportsPP):
layer_idx = int(name.split("model.layers.")[1].split(".")[0]) layer_idx = int(name.split("model.layers.")[1].split(".")[0])
if layer_idx >= self.config.num_hidden_layers: if layer_idx >= self.config.num_hidden_layers:
return None return None
name = name.replace("attention.", "self_attn.")
return maybe_remap_kv_scale_name(name, params_dict) return maybe_remap_kv_scale_name(name, params_dict)
for orig_name, weight in weights: for orig_name, weight in weights:
name = normalize_name(orig_name) name = normalize_name(orig_name)
if name is None: if name is None:
continue continue
weight = _maybe_pad_block_fp8_shared_expert_checkpoint_tensor(
self.quant_config,
self.config,
name,
weight,
)
loaded = False loaded = False
for param_suf, weight_suf, stacked_shard_id in stacked_mappings: for param_suf, weight_suf, stacked_shard_id in stacked_mappings:
if weight_suf in name: if weight_suf in name:
@@ -15,6 +15,7 @@ from vllm.model_executor.layers.fused_moe import (
fused_moe_make_expert_params_mapping, fused_moe_make_expert_params_mapping,
) )
from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import ReplicatedLinear
from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.vocab_parallel_embedding import ( from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead, ParallelLMHead,
@@ -25,8 +26,11 @@ from vllm.model_executor.model_loader.weight_utils import (
maybe_remap_kv_scale_name, maybe_remap_kv_scale_name,
) )
from vllm.model_executor.models.bailing_moe_v3 import ( from vllm.model_executor.models.bailing_moe_v3 import (
BailingMoeV3ForCausalLM,
BailingMoeV3MLAAttention, BailingMoeV3MLAAttention,
BailingMoeV3MoE, BailingMoeV3MoE,
_configure_ling_fp8_quant_config,
_maybe_pad_block_fp8_shared_expert_checkpoint_tensor,
) )
from vllm.model_executor.models.interfaces import SupportsPP from vllm.model_executor.models.interfaces import SupportsPP
from vllm.model_executor.models.utils import ( from vllm.model_executor.models.utils import (
@@ -79,7 +83,14 @@ class BailingMoeV3MultiTokenPredictorLayer(nn.Module):
self.layer_id = layer_id self.layer_id = layer_id
self.enorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) self.enorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.hnorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) self.hnorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.eh_proj = nn.Linear(config.hidden_size * 2, config.hidden_size, bias=False) self.eh_proj = ReplicatedLinear(
config.hidden_size * 2,
config.hidden_size,
bias=False,
quant_config=vllm_config.quant_config,
prefix=maybe_prefix(prefix, "eh_proj"),
return_bias=False,
)
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.self_attn = BailingMoeV3MLAAttention( self.self_attn = BailingMoeV3MLAAttention(
config, config,
@@ -204,6 +215,8 @@ class BailingMoeV3MultiTokenPredictor(nn.Module):
@support_torch_compile @support_torch_compile
class BailingMoeV3MTPModel(nn.Module, SupportsPP): class BailingMoeV3MTPModel(nn.Module, SupportsPP):
hf_to_vllm_mapper = BailingMoeV3ForCausalLM.hf_to_vllm_mapper
packed_modules_mapping = { packed_modules_mapping = {
"gate_up_proj": ["gate_proj", "up_proj"], "gate_up_proj": ["gate_proj", "up_proj"],
"fused_qkv_a_proj": ["q_a_proj", "kv_a_proj_with_mqa"], "fused_qkv_a_proj": ["q_a_proj", "kv_a_proj_with_mqa"],
@@ -217,6 +230,8 @@ class BailingMoeV3MTPModel(nn.Module, SupportsPP):
) -> None: ) -> None:
super().__init__() super().__init__()
self.config = _get_draft_hf_config(vllm_config) self.config = _get_draft_hf_config(vllm_config)
self.quant_config = vllm_config.quant_config
_configure_ling_fp8_quant_config(self.quant_config)
self.model = BailingMoeV3MultiTokenPredictor( self.model = BailingMoeV3MultiTokenPredictor(
vllm_config=vllm_config, vllm_config=vllm_config,
prefix=maybe_prefix(prefix, "model"), prefix=maybe_prefix(prefix, "model"),
@@ -265,6 +280,7 @@ class BailingMoeV3MTPModel(nn.Module, SupportsPP):
) )
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
weights = self.hf_to_vllm_mapper.apply(weights)
stacked_params_mapping = [ stacked_params_mapping = [
(".fused_qkv_a_proj", ".q_a_proj", 0), (".fused_qkv_a_proj", ".q_a_proj", 0),
(".fused_qkv_a_proj", ".kv_a_proj_with_mqa", 1), (".fused_qkv_a_proj", ".kv_a_proj_with_mqa", 1),
@@ -320,7 +336,6 @@ class BailingMoeV3MTPModel(nn.Module, SupportsPP):
return None return None
def normalize_name(name: str) -> str: def normalize_name(name: str) -> str:
name = name.replace(".attention.", ".self_attn.")
return name.replace( return name.replace(
"mlp.gate.e_score_correction_bias", "mlp.gate.e_score_correction_bias",
"mlp.gate.expert_bias", "mlp.gate.expert_bias",
@@ -349,6 +364,12 @@ class BailingMoeV3MTPModel(nn.Module, SupportsPP):
if spec_layer is None: if spec_layer is None:
continue continue
name = normalize_name(name) name = normalize_name(name)
loaded_weight = _maybe_pad_block_fp8_shared_expert_checkpoint_tensor(
self.quant_config,
self.config,
name,
loaded_weight,
)
loaded = False loaded = False
for param_name, weight_name, stacked_shard_id in stacked_params_mapping: for param_name, weight_name, stacked_shard_id in stacked_params_mapping: