[Quantization] Enable shared expert fusion compatibility with online shared_expert quantization (showcase: along Quark MXFP4 routed experts) (#55686)

Signed-off-by: Felix Marty <Felix.Marty@amd.com>
Signed-off-by: fxmarty-amd <felmarty@amd.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Misha Goin <mgoin64@gmail.com>
This commit is contained in:
fxmarty-amd
2026-10-08 12:04:46 -04:00
committed by GitHub
co-authored by OpenAI Codex Misha Goin
parent e2414ec1db
commit 9f3d70e081
24 changed files with 984 additions and 73 deletions
@@ -8,9 +8,13 @@ from typing import TYPE_CHECKING
import regex as re
from vllm.logger import init_logger
if TYPE_CHECKING:
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
logger = init_logger(__name__)
def get_quark_ocp_mx_group_size(
quant_config: "QuantizationConfig | None",
@@ -70,18 +74,100 @@ def is_shared_expert_quant_fse_compatible(
A compatibility flag and, when incompatible, the reason.
"""
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
from vllm.model_executor.layers.quantization.online.fp8 import OnlineLinearBase
from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig
from vllm.model_executor.layers.quantization.utils.quant_utils import (
is_layer_skipped,
)
from vllm.models.deepseek_v4.quant_config import DeepseekV4FP8Config
if projection_names is None:
projection_names = ["gate_up_proj", "down_proj"]
if quant_config is None:
return True, None
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig
from vllm.model_executor.layers.quantization.utils.quant_utils import (
is_layer_skipped,
)
from vllm.models.deepseek_v4.quant_config import DeepseekV4FP8Config
online_quant_config = quant_config.online_quantization_config
shared_expert_targets = None
if online_quant_config is not None:
from vllm.model_executor.layers.linear import LinearBase
online_quant_config.packed_modules_mapping = quant_config.packed_modules_mapping
shared_expert_targets = [
online_quant_config.resolve_quant_method_cls(
LinearBase, f"{shared_expert_prefix}.{projection_name}"
)
for projection_name in projection_names
]
if all(target is None for target in shared_expert_targets):
shared_expert_targets = None
if shared_expert_targets is not None:
from vllm.model_executor.layers.fused_moe import RoutedExperts
from vllm.model_executor.layers.linear import UnquantizedLinearMethod
from vllm.model_executor.layers.quantization.online.base import (
ONLINE_SHARED_EXPERT_QUANTIZERS,
)
# NOTE: online shared experts quantization check is only implemented for quark
# quant method at the moment. This can be extended here for other quant methods.
# Requirement is:
# - Retrieve routed weight quant key
# - Retrieve routed activation quant key
# and ensure the online quantization set on shared expert match them.
if isinstance(quant_config, QuarkConfig):
routed_weight_key, routed_activation_key, _ = (
quant_config.get_quant_method_target(expert_prefix, RoutedExperts)
)
else:
reason = (
"online quantization FSE compatibility check is not implemented for "
f"{type(quant_config).__name__}"
)
logger.warning_once(reason)
return False, reason
if routed_weight_key not in ONLINE_SHARED_EXPERT_QUANTIZERS:
return (
False,
"online shared-expert quantization is not supported for routed "
f"expert weight key {routed_weight_key}",
)
for shared_expert_target in shared_expert_targets:
if shared_expert_target is None:
return (
False,
"online quantization targets only part of the shared expert at "
f"{shared_expert_prefix}",
)
_, _, _, shared_quant_spec, shared_method_cls = shared_expert_target
if shared_method_cls in (None, UnquantizedLinearMethod):
return (
False,
"online quantization excludes FSE weights at "
f"{shared_expert_prefix}",
)
assert issubclass(shared_method_cls, OnlineLinearBase)
# NOTE: We can not rely on shared_quant_spec.activation_quant_key
# as _ONLINE_SHORTHANDS: dict[str, QuantSpec] in quantization.py
# does not define the default activation quant key.
# `online_quant_config.resolve_quant_method_cls` should return the
# activation quant key as well. Only some online methods define it
# on the class, so compare the weight key first.
if (
shared_quant_spec.weight != routed_weight_key
or shared_method_cls.activation_quant_key != routed_activation_key
):
return (
False,
"online shared-expert quantization keys do not match the "
"routed expert keys",
)
return True, None
if isinstance(quant_config, DeepseekV4FP8Config):
from vllm.config import get_current_vllm_config