[Quantization] Enable shared expert fusion compatibility with online shared_expert quantization (showcase: along Quark MXFP4 routed experts) (#55686)
Signed-off-by: Felix Marty <Felix.Marty@amd.com> Signed-off-by: fxmarty-amd <felmarty@amd.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Misha Goin <mgoin64@gmail.com>
This commit is contained in:
co-authored by
OpenAI Codex
Misha Goin
parent
e2414ec1db
commit
9f3d70e081
@@ -8,9 +8,13 @@ from typing import TYPE_CHECKING
|
||||
|
||||
import regex as re
|
||||
|
||||
from vllm.logger import init_logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def get_quark_ocp_mx_group_size(
|
||||
quant_config: "QuantizationConfig | None",
|
||||
@@ -70,18 +74,100 @@ def is_shared_expert_quant_fse_compatible(
|
||||
A compatibility flag and, when incompatible, the reason.
|
||||
|
||||
"""
|
||||
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
|
||||
from vllm.model_executor.layers.quantization.online.fp8 import OnlineLinearBase
|
||||
from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
is_layer_skipped,
|
||||
)
|
||||
from vllm.models.deepseek_v4.quant_config import DeepseekV4FP8Config
|
||||
|
||||
if projection_names is None:
|
||||
projection_names = ["gate_up_proj", "down_proj"]
|
||||
|
||||
if quant_config is None:
|
||||
return True, None
|
||||
|
||||
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
|
||||
from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
is_layer_skipped,
|
||||
)
|
||||
from vllm.models.deepseek_v4.quant_config import DeepseekV4FP8Config
|
||||
online_quant_config = quant_config.online_quantization_config
|
||||
shared_expert_targets = None
|
||||
if online_quant_config is not None:
|
||||
from vllm.model_executor.layers.linear import LinearBase
|
||||
|
||||
online_quant_config.packed_modules_mapping = quant_config.packed_modules_mapping
|
||||
shared_expert_targets = [
|
||||
online_quant_config.resolve_quant_method_cls(
|
||||
LinearBase, f"{shared_expert_prefix}.{projection_name}"
|
||||
)
|
||||
for projection_name in projection_names
|
||||
]
|
||||
if all(target is None for target in shared_expert_targets):
|
||||
shared_expert_targets = None
|
||||
|
||||
if shared_expert_targets is not None:
|
||||
from vllm.model_executor.layers.fused_moe import RoutedExperts
|
||||
from vllm.model_executor.layers.linear import UnquantizedLinearMethod
|
||||
from vllm.model_executor.layers.quantization.online.base import (
|
||||
ONLINE_SHARED_EXPERT_QUANTIZERS,
|
||||
)
|
||||
|
||||
# NOTE: online shared experts quantization check is only implemented for quark
|
||||
# quant method at the moment. This can be extended here for other quant methods.
|
||||
# Requirement is:
|
||||
# - Retrieve routed weight quant key
|
||||
# - Retrieve routed activation quant key
|
||||
# and ensure the online quantization set on shared expert match them.
|
||||
if isinstance(quant_config, QuarkConfig):
|
||||
routed_weight_key, routed_activation_key, _ = (
|
||||
quant_config.get_quant_method_target(expert_prefix, RoutedExperts)
|
||||
)
|
||||
else:
|
||||
reason = (
|
||||
"online quantization FSE compatibility check is not implemented for "
|
||||
f"{type(quant_config).__name__}"
|
||||
)
|
||||
logger.warning_once(reason)
|
||||
return False, reason
|
||||
|
||||
if routed_weight_key not in ONLINE_SHARED_EXPERT_QUANTIZERS:
|
||||
return (
|
||||
False,
|
||||
"online shared-expert quantization is not supported for routed "
|
||||
f"expert weight key {routed_weight_key}",
|
||||
)
|
||||
|
||||
for shared_expert_target in shared_expert_targets:
|
||||
if shared_expert_target is None:
|
||||
return (
|
||||
False,
|
||||
"online quantization targets only part of the shared expert at "
|
||||
f"{shared_expert_prefix}",
|
||||
)
|
||||
|
||||
_, _, _, shared_quant_spec, shared_method_cls = shared_expert_target
|
||||
if shared_method_cls in (None, UnquantizedLinearMethod):
|
||||
return (
|
||||
False,
|
||||
"online quantization excludes FSE weights at "
|
||||
f"{shared_expert_prefix}",
|
||||
)
|
||||
assert issubclass(shared_method_cls, OnlineLinearBase)
|
||||
|
||||
# NOTE: We can not rely on shared_quant_spec.activation_quant_key
|
||||
# as _ONLINE_SHORTHANDS: dict[str, QuantSpec] in quantization.py
|
||||
# does not define the default activation quant key.
|
||||
# `online_quant_config.resolve_quant_method_cls` should return the
|
||||
# activation quant key as well. Only some online methods define it
|
||||
# on the class, so compare the weight key first.
|
||||
if (
|
||||
shared_quant_spec.weight != routed_weight_key
|
||||
or shared_method_cls.activation_quant_key != routed_activation_key
|
||||
):
|
||||
return (
|
||||
False,
|
||||
"online shared-expert quantization keys do not match the "
|
||||
"routed expert keys",
|
||||
)
|
||||
return True, None
|
||||
|
||||
if isinstance(quant_config, DeepseekV4FP8Config):
|
||||
from vllm.config import get_current_vllm_config
|
||||
|
||||
Reference in New Issue
Block a user