[Model][Quantization] Add Ling-3.0-flash-fp8 support (#51265)

Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
zexplorerhj
2026-08-09 23:09:08 -07:00
committed by GitHub
co-authored by Isotr0py mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 3b4c86e489
commit ba1cdcfcf0
6 changed files with 298 additions and 36 deletions
@@ -292,7 +292,7 @@ class AutoAWQConfig(QuantizationConfig):
prefix,
self.modules_to_not_convert,
self.packed_modules_mapping,
skip_with_substr=True,
match_mode="substring",
):
return UnquantizedLinearMethod()
@@ -335,7 +335,7 @@ class AutoAWQConfig(QuantizationConfig):
if is_layer_skipped(
prefix,
getattr(self, "modules_to_not_convert", []),
skip_with_substr=True,
match_mode="substring",
):
return UnquantizedFusedMoEMethod(layer.moe_config)
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TYPE_CHECKING, Any
from typing import TYPE_CHECKING, Any, Literal
import torch
@@ -108,6 +108,9 @@ class Fp8Config(QuantizationConfig):
raise ValueError(f"Unsupported activation scheme {activation_scheme}")
self.activation_scheme = activation_scheme
self.ignored_layers = ignored_layers or []
self.ignored_layers_match_mode: Literal["exact", "substring", "suffix"] = (
"exact"
)
self.store_dtype = store_dtype
if weight_block_size is not None:
if not is_checkpoint_fp8_serialized:
@@ -177,6 +180,7 @@ class Fp8Config(QuantizationConfig):
prefix=prefix,
ignored_layers=self.ignored_layers,
fused_mapping=self.packed_modules_mapping,
match_mode=self.ignored_layers_match_mode,
):
return UnquantizedLinearMethod()
if not self.is_checkpoint_fp8_serialized:
@@ -196,6 +200,7 @@ class Fp8Config(QuantizationConfig):
prefix=prefix,
ignored_layers=self.ignored_layers,
fused_mapping=self.packed_modules_mapping,
match_mode=self.ignored_layers_match_mode,
):
return UnquantizedFusedMoEMethod(layer.moe_config)
if self.store_dtype == "mxfp4":
@@ -5,7 +5,7 @@
from collections.abc import Callable, Mapping
from dataclasses import dataclass
from types import MappingProxyType
from typing import TYPE_CHECKING, ClassVar, NamedTuple
from typing import TYPE_CHECKING, ClassVar, Literal, NamedTuple
import numpy
import torch
@@ -604,7 +604,7 @@ def is_layer_skipped(
ignored_layers: list[str],
fused_mapping: Mapping[str, list[str]] = MappingProxyType({}),
*,
skip_with_substr: bool = False,
match_mode: Literal["exact", "substring", "suffix"] = "exact",
) -> bool:
def prefix_full_match(prefix: str, ignored_layers: list[str]) -> bool:
return prefix in ignored_layers
@@ -613,7 +613,22 @@ def is_layer_skipped(
def substr_match(prefix: str, ignored_layers: list[str]) -> bool:
return any(layer in prefix for layer in ignored_layers)
match_func = substr_match if skip_with_substr else prefix_full_match
# Match abbreviated module paths at a component boundary. For example,
# ``b_proj`` matches ``model.layers.0.self_attn.b_proj`` but not
# ``model.layers.0.self_attn.q_b_proj``.
def suffix_match(prefix: str, ignored_layers: list[str]) -> bool:
return any(
prefix == layer or prefix.endswith(f".{layer}") for layer in ignored_layers
)
if match_mode == "exact":
match_func = prefix_full_match
elif match_mode == "substring":
match_func = substr_match
elif match_mode == "suffix":
match_func = suffix_match
else:
raise ValueError(f"Unsupported layer skip match mode: {match_mode}")
# prefix: model.layers.0.self_attn.q_proj
# proj_name: q_proj
@@ -649,14 +664,11 @@ def is_layer_skipped(
"are quantized. All shards of fused layers "
"to have the same precision."
)
elif "experts" in prefix and not skip_with_substr:
elif "experts" in prefix and match_mode == "exact":
expert_ignore_layers = filter(
lambda layer_name: "experts" in layer_name, ignored_layers
)
return any(
prefix in layer_name if not skip_with_substr else layer_name in prefix
for layer_name in expert_ignore_layers
)
return any(prefix in layer_name for layer_name in expert_ignore_layers)
else:
is_skipped = match_func(prefix, ignored_layers)
+224 -21
View File
@@ -10,6 +10,8 @@ vLLM's parallel linear layers, MLA kernel, KDA kernel and fused MoE loader.
import copy
from collections.abc import Callable, Iterable
from math import lcm
from typing import TypeGuard
import torch
import torch.nn as nn
@@ -55,6 +57,8 @@ from vllm.model_executor.layers.mla import (
MultiHeadLatentAttentionWrapper,
)
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
from vllm.model_executor.layers.quantization.utils.quant_utils import is_layer_skipped
from vllm.model_executor.layers.rotary_embedding import get_rope
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
@@ -79,7 +83,13 @@ from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadata
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
from .interfaces import HasInnerState, IsHybrid, SupportsPP
from .utils import PPMissingLayer, is_pp_missing_parameter, make_layers, maybe_prefix
from .utils import (
PPMissingLayer,
WeightsMapper,
is_pp_missing_parameter,
make_layers,
maybe_prefix,
)
def bailing_v3_kda_attention(
@@ -193,6 +203,155 @@ def _load_a_log(param: torch.nn.Parameter, loaded_weight: torch.Tensor) -> None:
sharded_weight_loader(2)(param, loaded_weight)
def _is_block_fp8_config(
quant_config: QuantizationConfig | None,
) -> TypeGuard[Fp8Config]:
return (
isinstance(quant_config, Fp8Config)
and quant_config.weight_block_size is not None
)
def _configure_ling_fp8_quant_config(
quant_config: QuantizationConfig | None,
) -> None:
if _is_block_fp8_config(quant_config):
quant_config.ignored_layers_match_mode = "suffix"
def _is_fp8_module_excluded(
quant_config: QuantizationConfig | None,
prefix: str,
) -> bool:
"""Match Ling's abbreviated FP8 exclusions against mapped vLLM prefixes."""
if not _is_block_fp8_config(quant_config):
return False
return is_layer_skipped(
prefix=prefix,
ignored_layers=quant_config.ignored_layers,
fused_mapping=quant_config.packed_modules_mapping,
match_mode=quant_config.ignored_layers_match_mode,
)
def _get_block_fp8_mlp_padded_intermediate_size(
quant_config: QuantizationConfig | None,
intermediate_size: int,
prefix: str,
) -> int:
"""Pad a block-FP8 MLP so each TP shard contains whole quant blocks."""
if not _is_block_fp8_config(quant_config):
return intermediate_size
block_size = quant_config.weight_block_size
assert block_size is not None
alignments: list[int] = []
if not _is_fp8_module_excluded(quant_config, f"{prefix}.gate_up_proj"):
alignments.append(int(block_size[0]))
if not _is_fp8_module_excluded(quant_config, f"{prefix}.down_proj"):
alignments.append(int(block_size[1]))
if not alignments:
return intermediate_size
tp_size = get_tensor_model_parallel_world_size()
tp_alignment = tp_size * lcm(*alignments)
return (intermediate_size + tp_alignment - 1) // tp_alignment * tp_alignment
def _pad_block_fp8_mlp_checkpoint_tensor(
quant_config: QuantizationConfig,
name: str,
loaded_weight: torch.Tensor,
intermediate_size: int,
padded_intermediate_size: int,
) -> torch.Tensor:
"""Zero-pad an MLP checkpoint tensor on its intermediate dimension."""
if padded_intermediate_size == intermediate_size:
return loaded_weight
block_size = getattr(quant_config, "weight_block_size", None)
assert block_size is not None
if ".down_proj." in name:
if name.endswith(".bias"):
return loaded_weight
dim = 1
block = int(block_size[1])
logical_shards = 1
elif any(
projection in name
for projection in (".gate_proj.", ".up_proj.", ".gate_up_proj.")
):
dim = 0
block = int(block_size[0])
logical_shards = 2 if ".gate_up_proj." in name else 1
else:
return loaded_weight
if name.endswith(".weight_scale_inv"):
expected_shard_size = (intermediate_size + block - 1) // block
target_shard_size = (padded_intermediate_size + block - 1) // block
elif name.endswith((".weight", ".bias")):
expected_shard_size = intermediate_size
target_shard_size = padded_intermediate_size
else:
return loaded_weight
current_size = loaded_weight.shape[dim]
if current_size % logical_shards != 0:
raise ValueError(
f"Cannot split {name} dimension {current_size} into "
f"{logical_shards} logical shards."
)
current_shard_size = current_size // logical_shards
if current_shard_size == target_shard_size:
return loaded_weight
if current_shard_size != expected_shard_size:
raise ValueError(
f"Cannot pad {name}: expected each logical intermediate dimension "
f"to be {expected_shard_size}, but got {current_shard_size}."
)
padded_shards: list[torch.Tensor] = []
for shard in loaded_weight.split(current_shard_size, dim=dim):
pad_shape = list(shard.shape)
pad_shape[dim] = target_shard_size - current_shard_size
padded_shards.extend([shard, shard.new_zeros(pad_shape)])
return torch.cat(padded_shards, dim=dim)
def _maybe_pad_block_fp8_shared_expert_checkpoint_tensor(
quant_config: QuantizationConfig | None,
config: PretrainedConfig,
name: str,
loaded_weight: torch.Tensor,
) -> torch.Tensor:
if ".mlp.shared_experts." not in name:
return loaded_weight
shared_prefix = name.split(".shared_experts.", 1)[0] + ".shared_experts"
shared_intermediate = (
config.moe_shared_expert_intermediate_size * config.num_shared_experts
)
padded_shared_intermediate = _get_block_fp8_mlp_padded_intermediate_size(
quant_config,
shared_intermediate,
shared_prefix,
)
if padded_shared_intermediate == shared_intermediate:
return loaded_weight
assert quant_config is not None
return _pad_block_fp8_mlp_checkpoint_tensor(
quant_config,
name,
loaded_weight,
shared_intermediate,
padded_shared_intermediate,
)
class _IdentityProjection(nn.Module):
def forward(self, x: torch.Tensor) -> tuple[torch.Tensor, None]:
return x, None
@@ -446,13 +605,36 @@ class BailingMoeV3KimiDeltaAttention(PluggableLayer, MambaBase):
projection_size = self.head_dim * self.num_heads
self.projection_size_per_partition = projection_size // self.tp_size
self.qkvb_proj = MergedColumnParallelLinear(
self.hidden_size,
[projection_size, projection_size, projection_size, self.num_heads],
bias=False,
quant_config=quant_config,
prefix=f"{prefix}.qkvb_proj",
)
self.separate_b_proj = _is_fp8_module_excluded(quant_config, f"{prefix}.b_proj")
self.qkv_proj: MergedColumnParallelLinear | None
self.b_proj: ColumnParallelLinear | None
self.qkvb_proj: MergedColumnParallelLinear | None
if self.separate_b_proj:
self.qkv_proj = MergedColumnParallelLinear(
self.hidden_size,
[projection_size, projection_size, projection_size],
bias=False,
quant_config=quant_config,
prefix=f"{prefix}.qkv_proj",
)
self.b_proj = ColumnParallelLinear(
self.hidden_size,
self.num_heads,
bias=False,
quant_config=quant_config,
prefix=f"{prefix}.b_proj",
)
self.qkvb_proj = None
else:
self.qkvb_proj = MergedColumnParallelLinear(
self.hidden_size,
[projection_size, projection_size, projection_size, self.num_heads],
bias=False,
quant_config=quant_config,
prefix=f"{prefix}.qkvb_proj",
)
self.qkv_proj = None
self.b_proj = None
self.f_proj = ColumnParallelLinear(
self.hidden_size,
projection_size,
@@ -532,16 +714,23 @@ class BailingMoeV3KimiDeltaAttention(PluggableLayer, MambaBase):
) -> None:
del positions
num_tokens = hidden_states.size(0)
qkvb = self.qkvb_proj(hidden_states)[0]
q, k, v, beta_logits = qkvb.split(
[
self.projection_size_per_partition,
self.projection_size_per_partition,
self.projection_size_per_partition,
self.local_num_heads,
],
dim=-1,
)
if self.separate_b_proj:
assert self.qkv_proj is not None and self.b_proj is not None
qkv = self.qkv_proj(hidden_states)[0]
q, k, v = qkv.split(self.projection_size_per_partition, dim=-1)
beta_logits = self.b_proj(hidden_states)[0]
else:
assert self.qkvb_proj is not None
qkvb = self.qkvb_proj(hidden_states)[0]
q, k, v, beta_logits = qkvb.split(
[
self.projection_size_per_partition,
self.projection_size_per_partition,
self.projection_size_per_partition,
self.local_num_heads,
],
dim=-1,
)
beta = beta_logits.float().sigmoid().unsqueeze(0)
g1 = self.f_proj(hidden_states)[0]
g1 = fused_kda_gate(
@@ -910,7 +1099,6 @@ class BailingMoeV3MoE(nn.Module):
prefix: str = "",
) -> None:
super().__init__()
self.tp_size = get_tensor_model_parallel_world_size()
self.num_experts = config.num_experts
self.top_k = config.num_experts_per_tok
self.hidden_size = config.hidden_size
@@ -926,7 +1114,9 @@ class BailingMoeV3MoE(nn.Module):
config.moe_shared_expert_intermediate_size * config.num_shared_experts
)
self.shared_experts = BailingMoeV3MLP(
intermediate_size=shared_intermediate,
intermediate_size=_get_block_fp8_mlp_padded_intermediate_size(
quant_config, shared_intermediate, f"{prefix}.shared_experts"
),
config=config,
quant_config=quant_config,
reduce_results=False,
@@ -1149,8 +1339,11 @@ class BailingMoeV3Model(nn.Module):
class BailingMoeV3ForCausalLM(nn.Module, HasInnerState, IsHybrid, SupportsPP):
hf_to_vllm_mapper = WeightsMapper(orig_to_new_substr={".attention.": ".self_attn."})
packed_modules_mapping = {
"gate_up_proj": ["gate_proj", "up_proj"],
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
"qkvb_proj": ["q_proj", "k_proj", "v_proj", "b_proj"],
}
@@ -1158,6 +1351,7 @@ class BailingMoeV3ForCausalLM(nn.Module, HasInnerState, IsHybrid, SupportsPP):
super().__init__()
config = vllm_config.model_config.hf_config
quant_config = vllm_config.quant_config
_configure_ling_fp8_quant_config(quant_config)
self.config = config
self.quant_config = quant_config
self.model = BailingMoeV3Model(
@@ -1226,11 +1420,15 @@ class BailingMoeV3ForCausalLM(nn.Module, HasInnerState, IsHybrid, SupportsPP):
return self.model.get_expert_mapping()
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
weights = self.hf_to_vllm_mapper.apply(weights)
params_dict = dict(self.named_parameters(remove_duplicate=False))
loaded_params: set[str] = set()
stacked_mappings = [
(".gate_up_proj", ".gate_proj", 0),
(".gate_up_proj", ".up_proj", 1),
(".qkv_proj", ".q_proj", 0),
(".qkv_proj", ".k_proj", 1),
(".qkv_proj", ".v_proj", 2),
(".qkvb_proj", ".q_proj", 0),
(".qkvb_proj", ".k_proj", 1),
(".qkvb_proj", ".v_proj", 2),
@@ -1263,13 +1461,18 @@ class BailingMoeV3ForCausalLM(nn.Module, HasInnerState, IsHybrid, SupportsPP):
layer_idx = int(name.split("model.layers.")[1].split(".")[0])
if layer_idx >= self.config.num_hidden_layers:
return None
name = name.replace("attention.", "self_attn.")
return maybe_remap_kv_scale_name(name, params_dict)
for orig_name, weight in weights:
name = normalize_name(orig_name)
if name is None:
continue
weight = _maybe_pad_block_fp8_shared_expert_checkpoint_tensor(
self.quant_config,
self.config,
name,
weight,
)
loaded = False
for param_suf, weight_suf, stacked_shard_id in stacked_mappings:
if weight_suf in name:
@@ -15,6 +15,7 @@ from vllm.model_executor.layers.fused_moe import (
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import ReplicatedLinear
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
@@ -25,8 +26,11 @@ from vllm.model_executor.model_loader.weight_utils import (
maybe_remap_kv_scale_name,
)
from vllm.model_executor.models.bailing_moe_v3 import (
BailingMoeV3ForCausalLM,
BailingMoeV3MLAAttention,
BailingMoeV3MoE,
_configure_ling_fp8_quant_config,
_maybe_pad_block_fp8_shared_expert_checkpoint_tensor,
)
from vllm.model_executor.models.interfaces import SupportsPP
from vllm.model_executor.models.utils import (
@@ -79,7 +83,14 @@ class BailingMoeV3MultiTokenPredictorLayer(nn.Module):
self.layer_id = layer_id
self.enorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.hnorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.eh_proj = nn.Linear(config.hidden_size * 2, config.hidden_size, bias=False)
self.eh_proj = ReplicatedLinear(
config.hidden_size * 2,
config.hidden_size,
bias=False,
quant_config=vllm_config.quant_config,
prefix=maybe_prefix(prefix, "eh_proj"),
return_bias=False,
)
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.self_attn = BailingMoeV3MLAAttention(
config,
@@ -204,6 +215,8 @@ class BailingMoeV3MultiTokenPredictor(nn.Module):
@support_torch_compile
class BailingMoeV3MTPModel(nn.Module, SupportsPP):
hf_to_vllm_mapper = BailingMoeV3ForCausalLM.hf_to_vllm_mapper
packed_modules_mapping = {
"gate_up_proj": ["gate_proj", "up_proj"],
"fused_qkv_a_proj": ["q_a_proj", "kv_a_proj_with_mqa"],
@@ -217,6 +230,8 @@ class BailingMoeV3MTPModel(nn.Module, SupportsPP):
) -> None:
super().__init__()
self.config = _get_draft_hf_config(vllm_config)
self.quant_config = vllm_config.quant_config
_configure_ling_fp8_quant_config(self.quant_config)
self.model = BailingMoeV3MultiTokenPredictor(
vllm_config=vllm_config,
prefix=maybe_prefix(prefix, "model"),
@@ -265,6 +280,7 @@ class BailingMoeV3MTPModel(nn.Module, SupportsPP):
)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
weights = self.hf_to_vllm_mapper.apply(weights)
stacked_params_mapping = [
(".fused_qkv_a_proj", ".q_a_proj", 0),
(".fused_qkv_a_proj", ".kv_a_proj_with_mqa", 1),
@@ -320,7 +336,6 @@ class BailingMoeV3MTPModel(nn.Module, SupportsPP):
return None
def normalize_name(name: str) -> str:
name = name.replace(".attention.", ".self_attn.")
return name.replace(
"mlp.gate.e_score_correction_bias",
"mlp.gate.expert_bias",
@@ -349,6 +364,12 @@ class BailingMoeV3MTPModel(nn.Module, SupportsPP):
if spec_layer is None:
continue
name = normalize_name(name)
loaded_weight = _maybe_pad_block_fp8_shared_expert_checkpoint_tensor(
self.quant_config,
self.config,
name,
loaded_weight,
)
loaded = False
for param_name, weight_name, stacked_shard_id in stacked_params_mapping: