[Model][Quantization] Add Ling-3.0-flash-fp8 support (#51265)

Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
zexplorerhj
2026-08-09 23:09:08 -07:00
committed by GitHub
co-authored by Isotr0py mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 3b4c86e489
commit ba1cdcfcf0
6 changed files with 298 additions and 36 deletions
@@ -292,7 +292,7 @@ class AutoAWQConfig(QuantizationConfig):
prefix,
self.modules_to_not_convert,
self.packed_modules_mapping,
skip_with_substr=True,
match_mode="substring",
):
return UnquantizedLinearMethod()
@@ -335,7 +335,7 @@ class AutoAWQConfig(QuantizationConfig):
if is_layer_skipped(
prefix,
getattr(self, "modules_to_not_convert", []),
skip_with_substr=True,
match_mode="substring",
):
return UnquantizedFusedMoEMethod(layer.moe_config)
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TYPE_CHECKING, Any
from typing import TYPE_CHECKING, Any, Literal
import torch
@@ -108,6 +108,9 @@ class Fp8Config(QuantizationConfig):
raise ValueError(f"Unsupported activation scheme {activation_scheme}")
self.activation_scheme = activation_scheme
self.ignored_layers = ignored_layers or []
self.ignored_layers_match_mode: Literal["exact", "substring", "suffix"] = (
"exact"
)
self.store_dtype = store_dtype
if weight_block_size is not None:
if not is_checkpoint_fp8_serialized:
@@ -177,6 +180,7 @@ class Fp8Config(QuantizationConfig):
prefix=prefix,
ignored_layers=self.ignored_layers,
fused_mapping=self.packed_modules_mapping,
match_mode=self.ignored_layers_match_mode,
):
return UnquantizedLinearMethod()
if not self.is_checkpoint_fp8_serialized:
@@ -196,6 +200,7 @@ class Fp8Config(QuantizationConfig):
prefix=prefix,
ignored_layers=self.ignored_layers,
fused_mapping=self.packed_modules_mapping,
match_mode=self.ignored_layers_match_mode,
):
return UnquantizedFusedMoEMethod(layer.moe_config)
if self.store_dtype == "mxfp4":
@@ -5,7 +5,7 @@
from collections.abc import Callable, Mapping
from dataclasses import dataclass
from types import MappingProxyType
from typing import TYPE_CHECKING, ClassVar, NamedTuple
from typing import TYPE_CHECKING, ClassVar, Literal, NamedTuple
import numpy
import torch
@@ -604,7 +604,7 @@ def is_layer_skipped(
ignored_layers: list[str],
fused_mapping: Mapping[str, list[str]] = MappingProxyType({}),
*,
skip_with_substr: bool = False,
match_mode: Literal["exact", "substring", "suffix"] = "exact",
) -> bool:
def prefix_full_match(prefix: str, ignored_layers: list[str]) -> bool:
return prefix in ignored_layers
@@ -613,7 +613,22 @@ def is_layer_skipped(
def substr_match(prefix: str, ignored_layers: list[str]) -> bool:
return any(layer in prefix for layer in ignored_layers)
match_func = substr_match if skip_with_substr else prefix_full_match
# Match abbreviated module paths at a component boundary. For example,
# ``b_proj`` matches ``model.layers.0.self_attn.b_proj`` but not
# ``model.layers.0.self_attn.q_b_proj``.
def suffix_match(prefix: str, ignored_layers: list[str]) -> bool:
return any(
prefix == layer or prefix.endswith(f".{layer}") for layer in ignored_layers
)
if match_mode == "exact":
match_func = prefix_full_match
elif match_mode == "substring":
match_func = substr_match
elif match_mode == "suffix":
match_func = suffix_match
else:
raise ValueError(f"Unsupported layer skip match mode: {match_mode}")
# prefix: model.layers.0.self_attn.q_proj
# proj_name: q_proj
@@ -649,14 +664,11 @@ def is_layer_skipped(
"are quantized. All shards of fused layers "
"to have the same precision."
)
elif "experts" in prefix and not skip_with_substr:
elif "experts" in prefix and match_mode == "exact":
expert_ignore_layers = filter(
lambda layer_name: "experts" in layer_name, ignored_layers
)
return any(
prefix in layer_name if not skip_with_substr else layer_name in prefix
for layer_name in expert_ignore_layers
)
return any(prefix in layer_name for layer_name in expert_ignore_layers)
else:
is_skipped = match_func(prefix, ignored_layers)