[Model][Quantization] Add Ling-3.0-flash-fp8 support (#51265)

Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
zexplorerhj
2026-08-09 23:09:08 -07:00
committed by GitHub
co-authored by Isotr0py mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 3b4c86e489
commit ba1cdcfcf0
6 changed files with 298 additions and 36 deletions
+224 -21
View File
@@ -10,6 +10,8 @@ vLLM's parallel linear layers, MLA kernel, KDA kernel and fused MoE loader.
import copy
from collections.abc import Callable, Iterable
from math import lcm
from typing import TypeGuard
import torch
import torch.nn as nn
@@ -55,6 +57,8 @@ from vllm.model_executor.layers.mla import (
MultiHeadLatentAttentionWrapper,
)
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
from vllm.model_executor.layers.quantization.utils.quant_utils import is_layer_skipped
from vllm.model_executor.layers.rotary_embedding import get_rope
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
@@ -79,7 +83,13 @@ from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadata
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
from .interfaces import HasInnerState, IsHybrid, SupportsPP
from .utils import PPMissingLayer, is_pp_missing_parameter, make_layers, maybe_prefix
from .utils import (
PPMissingLayer,
WeightsMapper,
is_pp_missing_parameter,
make_layers,
maybe_prefix,
)
def bailing_v3_kda_attention(
@@ -193,6 +203,155 @@ def _load_a_log(param: torch.nn.Parameter, loaded_weight: torch.Tensor) -> None:
sharded_weight_loader(2)(param, loaded_weight)
def _is_block_fp8_config(
quant_config: QuantizationConfig | None,
) -> TypeGuard[Fp8Config]:
return (
isinstance(quant_config, Fp8Config)
and quant_config.weight_block_size is not None
)
def _configure_ling_fp8_quant_config(
quant_config: QuantizationConfig | None,
) -> None:
if _is_block_fp8_config(quant_config):
quant_config.ignored_layers_match_mode = "suffix"
def _is_fp8_module_excluded(
quant_config: QuantizationConfig | None,
prefix: str,
) -> bool:
"""Match Ling's abbreviated FP8 exclusions against mapped vLLM prefixes."""
if not _is_block_fp8_config(quant_config):
return False
return is_layer_skipped(
prefix=prefix,
ignored_layers=quant_config.ignored_layers,
fused_mapping=quant_config.packed_modules_mapping,
match_mode=quant_config.ignored_layers_match_mode,
)
def _get_block_fp8_mlp_padded_intermediate_size(
quant_config: QuantizationConfig | None,
intermediate_size: int,
prefix: str,
) -> int:
"""Pad a block-FP8 MLP so each TP shard contains whole quant blocks."""
if not _is_block_fp8_config(quant_config):
return intermediate_size
block_size = quant_config.weight_block_size
assert block_size is not None
alignments: list[int] = []
if not _is_fp8_module_excluded(quant_config, f"{prefix}.gate_up_proj"):
alignments.append(int(block_size[0]))
if not _is_fp8_module_excluded(quant_config, f"{prefix}.down_proj"):
alignments.append(int(block_size[1]))
if not alignments:
return intermediate_size
tp_size = get_tensor_model_parallel_world_size()
tp_alignment = tp_size * lcm(*alignments)
return (intermediate_size + tp_alignment - 1) // tp_alignment * tp_alignment
def _pad_block_fp8_mlp_checkpoint_tensor(
quant_config: QuantizationConfig,
name: str,
loaded_weight: torch.Tensor,
intermediate_size: int,
padded_intermediate_size: int,
) -> torch.Tensor:
"""Zero-pad an MLP checkpoint tensor on its intermediate dimension."""
if padded_intermediate_size == intermediate_size:
return loaded_weight
block_size = getattr(quant_config, "weight_block_size", None)
assert block_size is not None
if ".down_proj." in name:
if name.endswith(".bias"):
return loaded_weight
dim = 1
block = int(block_size[1])
logical_shards = 1
elif any(
projection in name
for projection in (".gate_proj.", ".up_proj.", ".gate_up_proj.")
):
dim = 0
block = int(block_size[0])
logical_shards = 2 if ".gate_up_proj." in name else 1
else:
return loaded_weight
if name.endswith(".weight_scale_inv"):
expected_shard_size = (intermediate_size + block - 1) // block
target_shard_size = (padded_intermediate_size + block - 1) // block
elif name.endswith((".weight", ".bias")):
expected_shard_size = intermediate_size
target_shard_size = padded_intermediate_size
else:
return loaded_weight
current_size = loaded_weight.shape[dim]
if current_size % logical_shards != 0:
raise ValueError(
f"Cannot split {name} dimension {current_size} into "
f"{logical_shards} logical shards."
)
current_shard_size = current_size // logical_shards
if current_shard_size == target_shard_size:
return loaded_weight
if current_shard_size != expected_shard_size:
raise ValueError(
f"Cannot pad {name}: expected each logical intermediate dimension "
f"to be {expected_shard_size}, but got {current_shard_size}."
)
padded_shards: list[torch.Tensor] = []
for shard in loaded_weight.split(current_shard_size, dim=dim):
pad_shape = list(shard.shape)
pad_shape[dim] = target_shard_size - current_shard_size
padded_shards.extend([shard, shard.new_zeros(pad_shape)])
return torch.cat(padded_shards, dim=dim)
def _maybe_pad_block_fp8_shared_expert_checkpoint_tensor(
quant_config: QuantizationConfig | None,
config: PretrainedConfig,
name: str,
loaded_weight: torch.Tensor,
) -> torch.Tensor:
if ".mlp.shared_experts." not in name:
return loaded_weight
shared_prefix = name.split(".shared_experts.", 1)[0] + ".shared_experts"
shared_intermediate = (
config.moe_shared_expert_intermediate_size * config.num_shared_experts
)
padded_shared_intermediate = _get_block_fp8_mlp_padded_intermediate_size(
quant_config,
shared_intermediate,
shared_prefix,
)
if padded_shared_intermediate == shared_intermediate:
return loaded_weight
assert quant_config is not None
return _pad_block_fp8_mlp_checkpoint_tensor(
quant_config,
name,
loaded_weight,
shared_intermediate,
padded_shared_intermediate,
)
class _IdentityProjection(nn.Module):
def forward(self, x: torch.Tensor) -> tuple[torch.Tensor, None]:
return x, None
@@ -446,13 +605,36 @@ class BailingMoeV3KimiDeltaAttention(PluggableLayer, MambaBase):
projection_size = self.head_dim * self.num_heads
self.projection_size_per_partition = projection_size // self.tp_size
self.qkvb_proj = MergedColumnParallelLinear(
self.hidden_size,
[projection_size, projection_size, projection_size, self.num_heads],
bias=False,
quant_config=quant_config,
prefix=f"{prefix}.qkvb_proj",
)
self.separate_b_proj = _is_fp8_module_excluded(quant_config, f"{prefix}.b_proj")
self.qkv_proj: MergedColumnParallelLinear | None
self.b_proj: ColumnParallelLinear | None
self.qkvb_proj: MergedColumnParallelLinear | None
if self.separate_b_proj:
self.qkv_proj = MergedColumnParallelLinear(
self.hidden_size,
[projection_size, projection_size, projection_size],
bias=False,
quant_config=quant_config,
prefix=f"{prefix}.qkv_proj",
)
self.b_proj = ColumnParallelLinear(
self.hidden_size,
self.num_heads,
bias=False,
quant_config=quant_config,
prefix=f"{prefix}.b_proj",
)
self.qkvb_proj = None
else:
self.qkvb_proj = MergedColumnParallelLinear(
self.hidden_size,
[projection_size, projection_size, projection_size, self.num_heads],
bias=False,
quant_config=quant_config,
prefix=f"{prefix}.qkvb_proj",
)
self.qkv_proj = None
self.b_proj = None
self.f_proj = ColumnParallelLinear(
self.hidden_size,
projection_size,
@@ -532,16 +714,23 @@ class BailingMoeV3KimiDeltaAttention(PluggableLayer, MambaBase):
) -> None:
del positions
num_tokens = hidden_states.size(0)
qkvb = self.qkvb_proj(hidden_states)[0]
q, k, v, beta_logits = qkvb.split(
[
self.projection_size_per_partition,
self.projection_size_per_partition,
self.projection_size_per_partition,
self.local_num_heads,
],
dim=-1,
)
if self.separate_b_proj:
assert self.qkv_proj is not None and self.b_proj is not None
qkv = self.qkv_proj(hidden_states)[0]
q, k, v = qkv.split(self.projection_size_per_partition, dim=-1)
beta_logits = self.b_proj(hidden_states)[0]
else:
assert self.qkvb_proj is not None
qkvb = self.qkvb_proj(hidden_states)[0]
q, k, v, beta_logits = qkvb.split(
[
self.projection_size_per_partition,
self.projection_size_per_partition,
self.projection_size_per_partition,
self.local_num_heads,
],
dim=-1,
)
beta = beta_logits.float().sigmoid().unsqueeze(0)
g1 = self.f_proj(hidden_states)[0]
g1 = fused_kda_gate(
@@ -910,7 +1099,6 @@ class BailingMoeV3MoE(nn.Module):
prefix: str = "",
) -> None:
super().__init__()
self.tp_size = get_tensor_model_parallel_world_size()
self.num_experts = config.num_experts
self.top_k = config.num_experts_per_tok
self.hidden_size = config.hidden_size
@@ -926,7 +1114,9 @@ class BailingMoeV3MoE(nn.Module):
config.moe_shared_expert_intermediate_size * config.num_shared_experts
)
self.shared_experts = BailingMoeV3MLP(
intermediate_size=shared_intermediate,
intermediate_size=_get_block_fp8_mlp_padded_intermediate_size(
quant_config, shared_intermediate, f"{prefix}.shared_experts"
),
config=config,
quant_config=quant_config,
reduce_results=False,
@@ -1149,8 +1339,11 @@ class BailingMoeV3Model(nn.Module):
class BailingMoeV3ForCausalLM(nn.Module, HasInnerState, IsHybrid, SupportsPP):
hf_to_vllm_mapper = WeightsMapper(orig_to_new_substr={".attention.": ".self_attn."})
packed_modules_mapping = {
"gate_up_proj": ["gate_proj", "up_proj"],
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
"qkvb_proj": ["q_proj", "k_proj", "v_proj", "b_proj"],
}
@@ -1158,6 +1351,7 @@ class BailingMoeV3ForCausalLM(nn.Module, HasInnerState, IsHybrid, SupportsPP):
super().__init__()
config = vllm_config.model_config.hf_config
quant_config = vllm_config.quant_config
_configure_ling_fp8_quant_config(quant_config)
self.config = config
self.quant_config = quant_config
self.model = BailingMoeV3Model(
@@ -1226,11 +1420,15 @@ class BailingMoeV3ForCausalLM(nn.Module, HasInnerState, IsHybrid, SupportsPP):
return self.model.get_expert_mapping()
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
weights = self.hf_to_vllm_mapper.apply(weights)
params_dict = dict(self.named_parameters(remove_duplicate=False))
loaded_params: set[str] = set()
stacked_mappings = [
(".gate_up_proj", ".gate_proj", 0),
(".gate_up_proj", ".up_proj", 1),
(".qkv_proj", ".q_proj", 0),
(".qkv_proj", ".k_proj", 1),
(".qkv_proj", ".v_proj", 2),
(".qkvb_proj", ".q_proj", 0),
(".qkvb_proj", ".k_proj", 1),
(".qkvb_proj", ".v_proj", 2),
@@ -1263,13 +1461,18 @@ class BailingMoeV3ForCausalLM(nn.Module, HasInnerState, IsHybrid, SupportsPP):
layer_idx = int(name.split("model.layers.")[1].split(".")[0])
if layer_idx >= self.config.num_hidden_layers:
return None
name = name.replace("attention.", "self_attn.")
return maybe_remap_kv_scale_name(name, params_dict)
for orig_name, weight in weights:
name = normalize_name(orig_name)
if name is None:
continue
weight = _maybe_pad_block_fp8_shared_expert_checkpoint_tensor(
self.quant_config,
self.config,
name,
weight,
)
loaded = False
for param_suf, weight_suf, stacked_shard_id in stacked_mappings:
if weight_suf in name:
@@ -15,6 +15,7 @@ from vllm.model_executor.layers.fused_moe import (
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import ReplicatedLinear
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
@@ -25,8 +26,11 @@ from vllm.model_executor.model_loader.weight_utils import (
maybe_remap_kv_scale_name,
)
from vllm.model_executor.models.bailing_moe_v3 import (
BailingMoeV3ForCausalLM,
BailingMoeV3MLAAttention,
BailingMoeV3MoE,
_configure_ling_fp8_quant_config,
_maybe_pad_block_fp8_shared_expert_checkpoint_tensor,
)
from vllm.model_executor.models.interfaces import SupportsPP
from vllm.model_executor.models.utils import (
@@ -79,7 +83,14 @@ class BailingMoeV3MultiTokenPredictorLayer(nn.Module):
self.layer_id = layer_id
self.enorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.hnorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.eh_proj = nn.Linear(config.hidden_size * 2, config.hidden_size, bias=False)
self.eh_proj = ReplicatedLinear(
config.hidden_size * 2,
config.hidden_size,
bias=False,
quant_config=vllm_config.quant_config,
prefix=maybe_prefix(prefix, "eh_proj"),
return_bias=False,
)
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.self_attn = BailingMoeV3MLAAttention(
config,
@@ -204,6 +215,8 @@ class BailingMoeV3MultiTokenPredictor(nn.Module):
@support_torch_compile
class BailingMoeV3MTPModel(nn.Module, SupportsPP):
hf_to_vllm_mapper = BailingMoeV3ForCausalLM.hf_to_vllm_mapper
packed_modules_mapping = {
"gate_up_proj": ["gate_proj", "up_proj"],
"fused_qkv_a_proj": ["q_a_proj", "kv_a_proj_with_mqa"],
@@ -217,6 +230,8 @@ class BailingMoeV3MTPModel(nn.Module, SupportsPP):
) -> None:
super().__init__()
self.config = _get_draft_hf_config(vllm_config)
self.quant_config = vllm_config.quant_config
_configure_ling_fp8_quant_config(self.quant_config)
self.model = BailingMoeV3MultiTokenPredictor(
vllm_config=vllm_config,
prefix=maybe_prefix(prefix, "model"),
@@ -265,6 +280,7 @@ class BailingMoeV3MTPModel(nn.Module, SupportsPP):
)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
weights = self.hf_to_vllm_mapper.apply(weights)
stacked_params_mapping = [
(".fused_qkv_a_proj", ".q_a_proj", 0),
(".fused_qkv_a_proj", ".kv_a_proj_with_mqa", 1),
@@ -320,7 +336,6 @@ class BailingMoeV3MTPModel(nn.Module, SupportsPP):
return None
def normalize_name(name: str) -> str:
name = name.replace(".attention.", ".self_attn.")
return name.replace(
"mlp.gate.e_score_correction_bias",
"mlp.gate.expert_bias",
@@ -349,6 +364,12 @@ class BailingMoeV3MTPModel(nn.Module, SupportsPP):
if spec_layer is None:
continue
name = normalize_name(name)
loaded_weight = _maybe_pad_block_fp8_shared_expert_checkpoint_tensor(
self.quant_config,
self.config,
name,
loaded_weight,
)
loaded = False
for param_name, weight_name, stacked_shard_id in stacked_params_mapping: