From ba1cdcfcf05f581d1f58cc911644fed62317c8c6 Mon Sep 17 00:00:00 2001 From: zexplorerhj Date: Mon, 10 Aug 2026 14:09:08 +0800 Subject: [PATCH] `[Model][Quantization] Add Ling-3.0-flash-fp8 support` (#51265) Signed-off-by: Isotr0py Co-authored-by: Isotr0py Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- tests/quantization/test_quark.py | 25 +- .../layers/quantization/auto_awq.py | 4 +- .../model_executor/layers/quantization/fp8.py | 7 +- .../layers/quantization/utils/quant_utils.py | 28 +- vllm/model_executor/models/bailing_moe_v3.py | 245 ++++++++++++++++-- .../models/bailing_moe_v3_mtp.py | 25 +- 6 files changed, 298 insertions(+), 36 deletions(-) diff --git a/tests/quantization/test_quark.py b/tests/quantization/test_quark.py index c59ef11f65..c64b1f2d7b 100644 --- a/tests/quantization/test_quark.py +++ b/tests/quantization/test_quark.py @@ -701,11 +701,32 @@ def test_non_fused_layer_unaffected(): def test_substr_match_on_fused_name(): - # skip_with_substr=True path: fused-name substring match should also + # Substring matching: a fused-name match should also # short-circuit before shard expansion. assert is_layer_skipped( prefix="model.layers.0.self_attn.qkv_proj", ignored_layers=["self_attn.qkv_proj"], fused_mapping=FUSED_MAPPING, - skip_with_substr=True, + match_mode="substring", + ) + + +@pytest.mark.parametrize( + ("prefix", "ignored_layer", "expected"), + [ + ("model.layers.0.self_attn.b_proj", "b_proj", True), + ("model.layers.0.self_attn.q_b_proj", "b_proj", False), + ("model.layers.0.self_attn.kv_b_proj", "b_proj", False), + ("model.layers.5.self_attn.g_proj", "5.self_attn.g_proj", True), + ("model.layers.6.self_attn.g_proj", "5.self_attn.g_proj", False), + ], +) +def test_suffix_match_at_module_boundary(prefix, ignored_layer, expected): + assert ( + is_layer_skipped( + prefix=prefix, + ignored_layers=[ignored_layer], + match_mode="suffix", + ) + is expected ) diff --git a/vllm/model_executor/layers/quantization/auto_awq.py b/vllm/model_executor/layers/quantization/auto_awq.py index 83ef60b41f..ef81b0b876 100644 --- a/vllm/model_executor/layers/quantization/auto_awq.py +++ b/vllm/model_executor/layers/quantization/auto_awq.py @@ -292,7 +292,7 @@ class AutoAWQConfig(QuantizationConfig): prefix, self.modules_to_not_convert, self.packed_modules_mapping, - skip_with_substr=True, + match_mode="substring", ): return UnquantizedLinearMethod() @@ -335,7 +335,7 @@ class AutoAWQConfig(QuantizationConfig): if is_layer_skipped( prefix, getattr(self, "modules_to_not_convert", []), - skip_with_substr=True, + match_mode="substring", ): return UnquantizedFusedMoEMethod(layer.moe_config) diff --git a/vllm/model_executor/layers/quantization/fp8.py b/vllm/model_executor/layers/quantization/fp8.py index 7c04a654fb..9c69ff93d3 100644 --- a/vllm/model_executor/layers/quantization/fp8.py +++ b/vllm/model_executor/layers/quantization/fp8.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, Literal import torch @@ -108,6 +108,9 @@ class Fp8Config(QuantizationConfig): raise ValueError(f"Unsupported activation scheme {activation_scheme}") self.activation_scheme = activation_scheme self.ignored_layers = ignored_layers or [] + self.ignored_layers_match_mode: Literal["exact", "substring", "suffix"] = ( + "exact" + ) self.store_dtype = store_dtype if weight_block_size is not None: if not is_checkpoint_fp8_serialized: @@ -177,6 +180,7 @@ class Fp8Config(QuantizationConfig): prefix=prefix, ignored_layers=self.ignored_layers, fused_mapping=self.packed_modules_mapping, + match_mode=self.ignored_layers_match_mode, ): return UnquantizedLinearMethod() if not self.is_checkpoint_fp8_serialized: @@ -196,6 +200,7 @@ class Fp8Config(QuantizationConfig): prefix=prefix, ignored_layers=self.ignored_layers, fused_mapping=self.packed_modules_mapping, + match_mode=self.ignored_layers_match_mode, ): return UnquantizedFusedMoEMethod(layer.moe_config) if self.store_dtype == "mxfp4": diff --git a/vllm/model_executor/layers/quantization/utils/quant_utils.py b/vllm/model_executor/layers/quantization/utils/quant_utils.py index b127f2dd61..7ec658a375 100644 --- a/vllm/model_executor/layers/quantization/utils/quant_utils.py +++ b/vllm/model_executor/layers/quantization/utils/quant_utils.py @@ -5,7 +5,7 @@ from collections.abc import Callable, Mapping from dataclasses import dataclass from types import MappingProxyType -from typing import TYPE_CHECKING, ClassVar, NamedTuple +from typing import TYPE_CHECKING, ClassVar, Literal, NamedTuple import numpy import torch @@ -604,7 +604,7 @@ def is_layer_skipped( ignored_layers: list[str], fused_mapping: Mapping[str, list[str]] = MappingProxyType({}), *, - skip_with_substr: bool = False, + match_mode: Literal["exact", "substring", "suffix"] = "exact", ) -> bool: def prefix_full_match(prefix: str, ignored_layers: list[str]) -> bool: return prefix in ignored_layers @@ -613,7 +613,22 @@ def is_layer_skipped( def substr_match(prefix: str, ignored_layers: list[str]) -> bool: return any(layer in prefix for layer in ignored_layers) - match_func = substr_match if skip_with_substr else prefix_full_match + # Match abbreviated module paths at a component boundary. For example, + # ``b_proj`` matches ``model.layers.0.self_attn.b_proj`` but not + # ``model.layers.0.self_attn.q_b_proj``. + def suffix_match(prefix: str, ignored_layers: list[str]) -> bool: + return any( + prefix == layer or prefix.endswith(f".{layer}") for layer in ignored_layers + ) + + if match_mode == "exact": + match_func = prefix_full_match + elif match_mode == "substring": + match_func = substr_match + elif match_mode == "suffix": + match_func = suffix_match + else: + raise ValueError(f"Unsupported layer skip match mode: {match_mode}") # prefix: model.layers.0.self_attn.q_proj # proj_name: q_proj @@ -649,14 +664,11 @@ def is_layer_skipped( "are quantized. All shards of fused layers " "to have the same precision." ) - elif "experts" in prefix and not skip_with_substr: + elif "experts" in prefix and match_mode == "exact": expert_ignore_layers = filter( lambda layer_name: "experts" in layer_name, ignored_layers ) - return any( - prefix in layer_name if not skip_with_substr else layer_name in prefix - for layer_name in expert_ignore_layers - ) + return any(prefix in layer_name for layer_name in expert_ignore_layers) else: is_skipped = match_func(prefix, ignored_layers) diff --git a/vllm/model_executor/models/bailing_moe_v3.py b/vllm/model_executor/models/bailing_moe_v3.py index 2085a192a9..a2b105572d 100644 --- a/vllm/model_executor/models/bailing_moe_v3.py +++ b/vllm/model_executor/models/bailing_moe_v3.py @@ -10,6 +10,8 @@ vLLM's parallel linear layers, MLA kernel, KDA kernel and fused MoE loader. import copy from collections.abc import Callable, Iterable +from math import lcm +from typing import TypeGuard import torch import torch.nn as nn @@ -55,6 +57,8 @@ from vllm.model_executor.layers.mla import ( MultiHeadLatentAttentionWrapper, ) from vllm.model_executor.layers.quantization.base_config import QuantizationConfig +from vllm.model_executor.layers.quantization.fp8 import Fp8Config +from vllm.model_executor.layers.quantization.utils.quant_utils import is_layer_skipped from vllm.model_executor.layers.rotary_embedding import get_rope from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, @@ -79,7 +83,13 @@ from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadata from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum from .interfaces import HasInnerState, IsHybrid, SupportsPP -from .utils import PPMissingLayer, is_pp_missing_parameter, make_layers, maybe_prefix +from .utils import ( + PPMissingLayer, + WeightsMapper, + is_pp_missing_parameter, + make_layers, + maybe_prefix, +) def bailing_v3_kda_attention( @@ -193,6 +203,155 @@ def _load_a_log(param: torch.nn.Parameter, loaded_weight: torch.Tensor) -> None: sharded_weight_loader(2)(param, loaded_weight) +def _is_block_fp8_config( + quant_config: QuantizationConfig | None, +) -> TypeGuard[Fp8Config]: + return ( + isinstance(quant_config, Fp8Config) + and quant_config.weight_block_size is not None + ) + + +def _configure_ling_fp8_quant_config( + quant_config: QuantizationConfig | None, +) -> None: + if _is_block_fp8_config(quant_config): + quant_config.ignored_layers_match_mode = "suffix" + + +def _is_fp8_module_excluded( + quant_config: QuantizationConfig | None, + prefix: str, +) -> bool: + """Match Ling's abbreviated FP8 exclusions against mapped vLLM prefixes.""" + if not _is_block_fp8_config(quant_config): + return False + + return is_layer_skipped( + prefix=prefix, + ignored_layers=quant_config.ignored_layers, + fused_mapping=quant_config.packed_modules_mapping, + match_mode=quant_config.ignored_layers_match_mode, + ) + + +def _get_block_fp8_mlp_padded_intermediate_size( + quant_config: QuantizationConfig | None, + intermediate_size: int, + prefix: str, +) -> int: + """Pad a block-FP8 MLP so each TP shard contains whole quant blocks.""" + if not _is_block_fp8_config(quant_config): + return intermediate_size + block_size = quant_config.weight_block_size + assert block_size is not None + + alignments: list[int] = [] + if not _is_fp8_module_excluded(quant_config, f"{prefix}.gate_up_proj"): + alignments.append(int(block_size[0])) + if not _is_fp8_module_excluded(quant_config, f"{prefix}.down_proj"): + alignments.append(int(block_size[1])) + if not alignments: + return intermediate_size + + tp_size = get_tensor_model_parallel_world_size() + tp_alignment = tp_size * lcm(*alignments) + return (intermediate_size + tp_alignment - 1) // tp_alignment * tp_alignment + + +def _pad_block_fp8_mlp_checkpoint_tensor( + quant_config: QuantizationConfig, + name: str, + loaded_weight: torch.Tensor, + intermediate_size: int, + padded_intermediate_size: int, +) -> torch.Tensor: + """Zero-pad an MLP checkpoint tensor on its intermediate dimension.""" + if padded_intermediate_size == intermediate_size: + return loaded_weight + + block_size = getattr(quant_config, "weight_block_size", None) + assert block_size is not None + + if ".down_proj." in name: + if name.endswith(".bias"): + return loaded_weight + dim = 1 + block = int(block_size[1]) + logical_shards = 1 + elif any( + projection in name + for projection in (".gate_proj.", ".up_proj.", ".gate_up_proj.") + ): + dim = 0 + block = int(block_size[0]) + logical_shards = 2 if ".gate_up_proj." in name else 1 + else: + return loaded_weight + + if name.endswith(".weight_scale_inv"): + expected_shard_size = (intermediate_size + block - 1) // block + target_shard_size = (padded_intermediate_size + block - 1) // block + elif name.endswith((".weight", ".bias")): + expected_shard_size = intermediate_size + target_shard_size = padded_intermediate_size + else: + return loaded_weight + + current_size = loaded_weight.shape[dim] + if current_size % logical_shards != 0: + raise ValueError( + f"Cannot split {name} dimension {current_size} into " + f"{logical_shards} logical shards." + ) + current_shard_size = current_size // logical_shards + if current_shard_size == target_shard_size: + return loaded_weight + if current_shard_size != expected_shard_size: + raise ValueError( + f"Cannot pad {name}: expected each logical intermediate dimension " + f"to be {expected_shard_size}, but got {current_shard_size}." + ) + + padded_shards: list[torch.Tensor] = [] + for shard in loaded_weight.split(current_shard_size, dim=dim): + pad_shape = list(shard.shape) + pad_shape[dim] = target_shard_size - current_shard_size + padded_shards.extend([shard, shard.new_zeros(pad_shape)]) + return torch.cat(padded_shards, dim=dim) + + +def _maybe_pad_block_fp8_shared_expert_checkpoint_tensor( + quant_config: QuantizationConfig | None, + config: PretrainedConfig, + name: str, + loaded_weight: torch.Tensor, +) -> torch.Tensor: + if ".mlp.shared_experts." not in name: + return loaded_weight + + shared_prefix = name.split(".shared_experts.", 1)[0] + ".shared_experts" + shared_intermediate = ( + config.moe_shared_expert_intermediate_size * config.num_shared_experts + ) + padded_shared_intermediate = _get_block_fp8_mlp_padded_intermediate_size( + quant_config, + shared_intermediate, + shared_prefix, + ) + if padded_shared_intermediate == shared_intermediate: + return loaded_weight + + assert quant_config is not None + return _pad_block_fp8_mlp_checkpoint_tensor( + quant_config, + name, + loaded_weight, + shared_intermediate, + padded_shared_intermediate, + ) + + class _IdentityProjection(nn.Module): def forward(self, x: torch.Tensor) -> tuple[torch.Tensor, None]: return x, None @@ -446,13 +605,36 @@ class BailingMoeV3KimiDeltaAttention(PluggableLayer, MambaBase): projection_size = self.head_dim * self.num_heads self.projection_size_per_partition = projection_size // self.tp_size - self.qkvb_proj = MergedColumnParallelLinear( - self.hidden_size, - [projection_size, projection_size, projection_size, self.num_heads], - bias=False, - quant_config=quant_config, - prefix=f"{prefix}.qkvb_proj", - ) + self.separate_b_proj = _is_fp8_module_excluded(quant_config, f"{prefix}.b_proj") + self.qkv_proj: MergedColumnParallelLinear | None + self.b_proj: ColumnParallelLinear | None + self.qkvb_proj: MergedColumnParallelLinear | None + if self.separate_b_proj: + self.qkv_proj = MergedColumnParallelLinear( + self.hidden_size, + [projection_size, projection_size, projection_size], + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.qkv_proj", + ) + self.b_proj = ColumnParallelLinear( + self.hidden_size, + self.num_heads, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.b_proj", + ) + self.qkvb_proj = None + else: + self.qkvb_proj = MergedColumnParallelLinear( + self.hidden_size, + [projection_size, projection_size, projection_size, self.num_heads], + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.qkvb_proj", + ) + self.qkv_proj = None + self.b_proj = None self.f_proj = ColumnParallelLinear( self.hidden_size, projection_size, @@ -532,16 +714,23 @@ class BailingMoeV3KimiDeltaAttention(PluggableLayer, MambaBase): ) -> None: del positions num_tokens = hidden_states.size(0) - qkvb = self.qkvb_proj(hidden_states)[0] - q, k, v, beta_logits = qkvb.split( - [ - self.projection_size_per_partition, - self.projection_size_per_partition, - self.projection_size_per_partition, - self.local_num_heads, - ], - dim=-1, - ) + if self.separate_b_proj: + assert self.qkv_proj is not None and self.b_proj is not None + qkv = self.qkv_proj(hidden_states)[0] + q, k, v = qkv.split(self.projection_size_per_partition, dim=-1) + beta_logits = self.b_proj(hidden_states)[0] + else: + assert self.qkvb_proj is not None + qkvb = self.qkvb_proj(hidden_states)[0] + q, k, v, beta_logits = qkvb.split( + [ + self.projection_size_per_partition, + self.projection_size_per_partition, + self.projection_size_per_partition, + self.local_num_heads, + ], + dim=-1, + ) beta = beta_logits.float().sigmoid().unsqueeze(0) g1 = self.f_proj(hidden_states)[0] g1 = fused_kda_gate( @@ -910,7 +1099,6 @@ class BailingMoeV3MoE(nn.Module): prefix: str = "", ) -> None: super().__init__() - self.tp_size = get_tensor_model_parallel_world_size() self.num_experts = config.num_experts self.top_k = config.num_experts_per_tok self.hidden_size = config.hidden_size @@ -926,7 +1114,9 @@ class BailingMoeV3MoE(nn.Module): config.moe_shared_expert_intermediate_size * config.num_shared_experts ) self.shared_experts = BailingMoeV3MLP( - intermediate_size=shared_intermediate, + intermediate_size=_get_block_fp8_mlp_padded_intermediate_size( + quant_config, shared_intermediate, f"{prefix}.shared_experts" + ), config=config, quant_config=quant_config, reduce_results=False, @@ -1149,8 +1339,11 @@ class BailingMoeV3Model(nn.Module): class BailingMoeV3ForCausalLM(nn.Module, HasInnerState, IsHybrid, SupportsPP): + hf_to_vllm_mapper = WeightsMapper(orig_to_new_substr={".attention.": ".self_attn."}) + packed_modules_mapping = { "gate_up_proj": ["gate_proj", "up_proj"], + "qkv_proj": ["q_proj", "k_proj", "v_proj"], "qkvb_proj": ["q_proj", "k_proj", "v_proj", "b_proj"], } @@ -1158,6 +1351,7 @@ class BailingMoeV3ForCausalLM(nn.Module, HasInnerState, IsHybrid, SupportsPP): super().__init__() config = vllm_config.model_config.hf_config quant_config = vllm_config.quant_config + _configure_ling_fp8_quant_config(quant_config) self.config = config self.quant_config = quant_config self.model = BailingMoeV3Model( @@ -1226,11 +1420,15 @@ class BailingMoeV3ForCausalLM(nn.Module, HasInnerState, IsHybrid, SupportsPP): return self.model.get_expert_mapping() def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + weights = self.hf_to_vllm_mapper.apply(weights) params_dict = dict(self.named_parameters(remove_duplicate=False)) loaded_params: set[str] = set() stacked_mappings = [ (".gate_up_proj", ".gate_proj", 0), (".gate_up_proj", ".up_proj", 1), + (".qkv_proj", ".q_proj", 0), + (".qkv_proj", ".k_proj", 1), + (".qkv_proj", ".v_proj", 2), (".qkvb_proj", ".q_proj", 0), (".qkvb_proj", ".k_proj", 1), (".qkvb_proj", ".v_proj", 2), @@ -1263,13 +1461,18 @@ class BailingMoeV3ForCausalLM(nn.Module, HasInnerState, IsHybrid, SupportsPP): layer_idx = int(name.split("model.layers.")[1].split(".")[0]) if layer_idx >= self.config.num_hidden_layers: return None - name = name.replace("attention.", "self_attn.") return maybe_remap_kv_scale_name(name, params_dict) for orig_name, weight in weights: name = normalize_name(orig_name) if name is None: continue + weight = _maybe_pad_block_fp8_shared_expert_checkpoint_tensor( + self.quant_config, + self.config, + name, + weight, + ) loaded = False for param_suf, weight_suf, stacked_shard_id in stacked_mappings: if weight_suf in name: diff --git a/vllm/model_executor/models/bailing_moe_v3_mtp.py b/vllm/model_executor/models/bailing_moe_v3_mtp.py index 48c8c9d7a9..ffddaa8480 100644 --- a/vllm/model_executor/models/bailing_moe_v3_mtp.py +++ b/vllm/model_executor/models/bailing_moe_v3_mtp.py @@ -15,6 +15,7 @@ from vllm.model_executor.layers.fused_moe import ( fused_moe_make_expert_params_mapping, ) from vllm.model_executor.layers.layernorm import RMSNorm +from vllm.model_executor.layers.linear import ReplicatedLinear from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, @@ -25,8 +26,11 @@ from vllm.model_executor.model_loader.weight_utils import ( maybe_remap_kv_scale_name, ) from vllm.model_executor.models.bailing_moe_v3 import ( + BailingMoeV3ForCausalLM, BailingMoeV3MLAAttention, BailingMoeV3MoE, + _configure_ling_fp8_quant_config, + _maybe_pad_block_fp8_shared_expert_checkpoint_tensor, ) from vllm.model_executor.models.interfaces import SupportsPP from vllm.model_executor.models.utils import ( @@ -79,7 +83,14 @@ class BailingMoeV3MultiTokenPredictorLayer(nn.Module): self.layer_id = layer_id self.enorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) self.hnorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) - self.eh_proj = nn.Linear(config.hidden_size * 2, config.hidden_size, bias=False) + self.eh_proj = ReplicatedLinear( + config.hidden_size * 2, + config.hidden_size, + bias=False, + quant_config=vllm_config.quant_config, + prefix=maybe_prefix(prefix, "eh_proj"), + return_bias=False, + ) self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) self.self_attn = BailingMoeV3MLAAttention( config, @@ -204,6 +215,8 @@ class BailingMoeV3MultiTokenPredictor(nn.Module): @support_torch_compile class BailingMoeV3MTPModel(nn.Module, SupportsPP): + hf_to_vllm_mapper = BailingMoeV3ForCausalLM.hf_to_vllm_mapper + packed_modules_mapping = { "gate_up_proj": ["gate_proj", "up_proj"], "fused_qkv_a_proj": ["q_a_proj", "kv_a_proj_with_mqa"], @@ -217,6 +230,8 @@ class BailingMoeV3MTPModel(nn.Module, SupportsPP): ) -> None: super().__init__() self.config = _get_draft_hf_config(vllm_config) + self.quant_config = vllm_config.quant_config + _configure_ling_fp8_quant_config(self.quant_config) self.model = BailingMoeV3MultiTokenPredictor( vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model"), @@ -265,6 +280,7 @@ class BailingMoeV3MTPModel(nn.Module, SupportsPP): ) def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + weights = self.hf_to_vllm_mapper.apply(weights) stacked_params_mapping = [ (".fused_qkv_a_proj", ".q_a_proj", 0), (".fused_qkv_a_proj", ".kv_a_proj_with_mqa", 1), @@ -320,7 +336,6 @@ class BailingMoeV3MTPModel(nn.Module, SupportsPP): return None def normalize_name(name: str) -> str: - name = name.replace(".attention.", ".self_attn.") return name.replace( "mlp.gate.e_score_correction_bias", "mlp.gate.expert_bias", @@ -349,6 +364,12 @@ class BailingMoeV3MTPModel(nn.Module, SupportsPP): if spec_layer is None: continue name = normalize_name(name) + loaded_weight = _maybe_pad_block_fp8_shared_expert_checkpoint_tensor( + self.quant_config, + self.config, + name, + loaded_weight, + ) loaded = False for param_name, weight_name, stacked_shard_id in stacked_params_mapping: