[Model][Quantization] Add Ling-3.0-flash-fp8 support (#51265)
Signed-off-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Isotr0py
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
3b4c86e489
commit
ba1cdcfcf0
@@ -292,7 +292,7 @@ class AutoAWQConfig(QuantizationConfig):
|
||||
prefix,
|
||||
self.modules_to_not_convert,
|
||||
self.packed_modules_mapping,
|
||||
skip_with_substr=True,
|
||||
match_mode="substring",
|
||||
):
|
||||
return UnquantizedLinearMethod()
|
||||
|
||||
@@ -335,7 +335,7 @@ class AutoAWQConfig(QuantizationConfig):
|
||||
if is_layer_skipped(
|
||||
prefix,
|
||||
getattr(self, "modules_to_not_convert", []),
|
||||
skip_with_substr=True,
|
||||
match_mode="substring",
|
||||
):
|
||||
return UnquantizedFusedMoEMethod(layer.moe_config)
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import TYPE_CHECKING, Any
|
||||
from typing import TYPE_CHECKING, Any, Literal
|
||||
|
||||
import torch
|
||||
|
||||
@@ -108,6 +108,9 @@ class Fp8Config(QuantizationConfig):
|
||||
raise ValueError(f"Unsupported activation scheme {activation_scheme}")
|
||||
self.activation_scheme = activation_scheme
|
||||
self.ignored_layers = ignored_layers or []
|
||||
self.ignored_layers_match_mode: Literal["exact", "substring", "suffix"] = (
|
||||
"exact"
|
||||
)
|
||||
self.store_dtype = store_dtype
|
||||
if weight_block_size is not None:
|
||||
if not is_checkpoint_fp8_serialized:
|
||||
@@ -177,6 +180,7 @@ class Fp8Config(QuantizationConfig):
|
||||
prefix=prefix,
|
||||
ignored_layers=self.ignored_layers,
|
||||
fused_mapping=self.packed_modules_mapping,
|
||||
match_mode=self.ignored_layers_match_mode,
|
||||
):
|
||||
return UnquantizedLinearMethod()
|
||||
if not self.is_checkpoint_fp8_serialized:
|
||||
@@ -196,6 +200,7 @@ class Fp8Config(QuantizationConfig):
|
||||
prefix=prefix,
|
||||
ignored_layers=self.ignored_layers,
|
||||
fused_mapping=self.packed_modules_mapping,
|
||||
match_mode=self.ignored_layers_match_mode,
|
||||
):
|
||||
return UnquantizedFusedMoEMethod(layer.moe_config)
|
||||
if self.store_dtype == "mxfp4":
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
from collections.abc import Callable, Mapping
|
||||
from dataclasses import dataclass
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, ClassVar, NamedTuple
|
||||
from typing import TYPE_CHECKING, ClassVar, Literal, NamedTuple
|
||||
|
||||
import numpy
|
||||
import torch
|
||||
@@ -604,7 +604,7 @@ def is_layer_skipped(
|
||||
ignored_layers: list[str],
|
||||
fused_mapping: Mapping[str, list[str]] = MappingProxyType({}),
|
||||
*,
|
||||
skip_with_substr: bool = False,
|
||||
match_mode: Literal["exact", "substring", "suffix"] = "exact",
|
||||
) -> bool:
|
||||
def prefix_full_match(prefix: str, ignored_layers: list[str]) -> bool:
|
||||
return prefix in ignored_layers
|
||||
@@ -613,7 +613,22 @@ def is_layer_skipped(
|
||||
def substr_match(prefix: str, ignored_layers: list[str]) -> bool:
|
||||
return any(layer in prefix for layer in ignored_layers)
|
||||
|
||||
match_func = substr_match if skip_with_substr else prefix_full_match
|
||||
# Match abbreviated module paths at a component boundary. For example,
|
||||
# ``b_proj`` matches ``model.layers.0.self_attn.b_proj`` but not
|
||||
# ``model.layers.0.self_attn.q_b_proj``.
|
||||
def suffix_match(prefix: str, ignored_layers: list[str]) -> bool:
|
||||
return any(
|
||||
prefix == layer or prefix.endswith(f".{layer}") for layer in ignored_layers
|
||||
)
|
||||
|
||||
if match_mode == "exact":
|
||||
match_func = prefix_full_match
|
||||
elif match_mode == "substring":
|
||||
match_func = substr_match
|
||||
elif match_mode == "suffix":
|
||||
match_func = suffix_match
|
||||
else:
|
||||
raise ValueError(f"Unsupported layer skip match mode: {match_mode}")
|
||||
|
||||
# prefix: model.layers.0.self_attn.q_proj
|
||||
# proj_name: q_proj
|
||||
@@ -649,14 +664,11 @@ def is_layer_skipped(
|
||||
"are quantized. All shards of fused layers "
|
||||
"to have the same precision."
|
||||
)
|
||||
elif "experts" in prefix and not skip_with_substr:
|
||||
elif "experts" in prefix and match_mode == "exact":
|
||||
expert_ignore_layers = filter(
|
||||
lambda layer_name: "experts" in layer_name, ignored_layers
|
||||
)
|
||||
return any(
|
||||
prefix in layer_name if not skip_with_substr else layer_name in prefix
|
||||
for layer_name in expert_ignore_layers
|
||||
)
|
||||
return any(prefix in layer_name for layer_name in expert_ignore_layers)
|
||||
else:
|
||||
is_skipped = match_func(prefix, ignored_layers)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user