[Quantization] Enable shared expert fusion compatibility with online shared_expert quantization (showcase: along Quark MXFP4 routed experts) (#55686)

Signed-off-by: Felix Marty <Felix.Marty@amd.com>
Signed-off-by: fxmarty-amd <felmarty@amd.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Misha Goin <mgoin64@gmail.com>
This commit is contained in:
fxmarty-amd
2026-10-08 12:04:46 -04:00
committed by GitHub
co-authored by OpenAI Codex Misha Goin
parent e2414ec1db
commit 9f3d70e081
24 changed files with 984 additions and 73 deletions
@@ -320,7 +320,7 @@ def resolve_quant_method(
base_quant_method, (UnquantizedLinearMethod, UnquantizedFusedMoEMethod)
)
online_target = quant_config.online_quantization_config.resolve_quant_method_cls(
layer, prefix
type(layer), prefix
)
if checkpoint_is_quantized:
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Mapping
from collections.abc import Callable, Mapping
from enum import Enum
from types import MappingProxyType
from typing import Any
@@ -60,6 +60,7 @@ from vllm.model_executor.layers.quantization.utils.config_utils import (
find_matching_patterns,
get_layer_name_after_index,
)
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import mxfp4_quantize
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kFp8Static128BlockSym,
@@ -104,6 +105,15 @@ _ONLINE_MOE_METHODS: dict[QuantKey, type] = {
kNvfp4Static: Nvfp4OnlineMoEMethod,
}
# Quantizers for full-precision fused shared-expert weights, keyed by the routed
# experts' weight QuantKey. Each shard is quantized to the checkpoint layout on
# arrival, so only block-local schemes are supported.
ONLINE_SHARED_EXPERT_QUANTIZERS: dict[
QuantKey, Callable[[torch.Tensor], tuple[torch.Tensor, torch.Tensor]]
] = {
kMxfp4Static: mxfp4_quantize,
}
def _find_matching_targets(
prefix: str,
@@ -217,14 +227,14 @@ class OnlineQuantizationConfig(QuantizationConfig):
self,
spec: QuantSpec | None,
table: dict[QuantKey, type],
layer: torch.nn.Module,
layer_cls: type[torch.nn.Module],
) -> type | None:
"""Resolve the online method class for a layer's quantization spec.
Args:
spec: Quantization specification to resolve.
table: Mapping from weight quantization keys to method classes.
layer: Layer that will use the resolved method.
layer_cls: Type of the layer that will use the resolved method.
Returns:
The matching method class, or None when ``spec`` has no weight
@@ -239,7 +249,7 @@ class OnlineQuantizationConfig(QuantizationConfig):
cls = table.get(spec.weight)
if cls is None:
raise ValueError(
f"online quantization for {type(layer).__name__} with "
f"online quantization for {layer_cls.__name__} with "
f"weight={spec.weight} is not supported; supported weight "
f"keys: {sorted(str(k) for k in table)}"
)
@@ -254,12 +264,12 @@ class OnlineQuantizationConfig(QuantizationConfig):
return cls
def resolve_quant_method_cls(
self, layer: torch.nn.Module, prefix: str
self, layer_cls: type[torch.nn.Module], prefix: str
) -> tuple[OnlineQuantizationSource, str, str | None, QuantSpec, type] | None:
"""Resolve quantization metadata and method class without instantiating it.
Args:
layer: Layer for which to resolve online quantization.
layer_cls: Type of layer for which to resolve online quantization.
prefix: Fully qualified layer name.
Returns:
@@ -271,17 +281,17 @@ class OnlineQuantizationConfig(QuantizationConfig):
quant_spec: QuantSpec | None
if self.args.targets is not None:
resolved_pattern = self._resolve_targets_quant_method_metadata(
prefix, layer
prefix, layer_cls
)
if resolved_pattern is None:
return None
source, quant_key_str, target_pattern, quant_spec, table = resolved_pattern
else:
if isinstance(layer, LinearBase):
if issubclass(layer_cls, LinearBase):
source = OnlineQuantizationSource.linear
quant_spec = self.args.linear
table = _ONLINE_LINEAR_METHODS
elif isinstance(layer, RoutedExperts):
elif issubclass(layer_cls, RoutedExperts):
source = OnlineQuantizationSource.moe
quant_spec = self.args.moe
table = _ONLINE_MOE_METHODS
@@ -298,14 +308,14 @@ class OnlineQuantizationConfig(QuantizationConfig):
quant_key_str = str(quant_spec)
target_pattern = None
quant_method_cls = self._get_method_cls(quant_spec, table, layer)
quant_method_cls = self._get_method_cls(quant_spec, table, layer_cls)
if quant_method_cls is None:
return None
assert quant_spec is not None
return source, quant_key_str, target_pattern, quant_spec, quant_method_cls
def _resolve_targets_quant_method_metadata(
self, prefix: str, layer: torch.nn.Module
self, prefix: str, layer_cls: type[torch.nn.Module]
) -> (
tuple[OnlineQuantizationSource, str, str, QuantSpec, dict[QuantKey, type]]
| None
@@ -314,7 +324,7 @@ class OnlineQuantizationConfig(QuantizationConfig):
Args:
prefix: Fully qualified layer name.
layer: Layer matched against configured target patterns.
layer_cls: Type of layer matched against configured target patterns.
Returns:
A tuple of source, quantization key string, target pattern, spec,
@@ -349,22 +359,22 @@ class OnlineQuantizationConfig(QuantizationConfig):
target_pattern = matches[0]
quant_key_str = self.args.targets[target_pattern]
shorthand = _ONLINE_SHORTHANDS[quant_key_str]
if isinstance(layer, LinearBase):
if issubclass(layer_cls, LinearBase):
quant_spec = shorthand.linear
table = _ONLINE_LINEAR_METHODS
elif isinstance(layer, RoutedExperts):
elif issubclass(layer_cls, RoutedExperts):
quant_spec = shorthand.moe
table = _ONLINE_MOE_METHODS
else:
raise ValueError(
f"Layer {prefix} was matched by quantization_config.targets "
f"({target_pattern}), but online quantization is not supported for "
f"{type(layer).__name__}."
f"{layer_cls.__name__}."
)
if quant_spec is None:
raise ValueError(
f"targets pattern {target_pattern} = {quant_key_str} does "
f"not define a QuantSpec for {type(layer).__name__} layers "
f"not define a QuantSpec for {layer_cls.__name__} layers "
f"(matched at {prefix})."
)
return (
@@ -379,7 +389,7 @@ class OnlineQuantizationConfig(QuantizationConfig):
self, layer: torch.nn.Module, prefix: str
) -> "QuantizeMethodBase | None":
# `targets` takes precedence over `moe` and `linear` and is exclusive.
resolved = self.resolve_quant_method_cls(layer, prefix)
resolved = self.resolve_quant_method_cls(type(layer), prefix)
if resolved is not None:
source, quant_key_str, target_pattern, _, quant_method_cls = resolved
self.quantized_layers[prefix] = (
@@ -12,8 +12,6 @@ if TYPE_CHECKING:
FusedMoEQuantConfig,
)
from vllm.model_executor.layers.fused_moe.oracle.fp8 import Fp8MoeBackend
from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey
import vllm.envs as envs
from vllm import _custom_ops as ops
from vllm.config import get_current_vllm_config
@@ -40,6 +38,7 @@ from vllm.model_executor.layers.quantization.online.moe_base import (
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
GroupShape,
QuantKey,
amax_for_moe_weight_quant,
amax_for_tp_weight_quant,
create_fp8_quant_key,
@@ -122,6 +121,7 @@ class OnlineLinearBase(LinearMethodBase):
weights onto meta device and materializes them just-in-time."""
uses_meta_device: bool = True
activation_quant_key: QuantKey | None
def __init__(self):
self.out_dtype = torch.get_default_dtype()
@@ -203,6 +203,10 @@ class Fp8PerTensorOnlineLinearMethod(OnlineLinearBase):
"""Online tensorwise FP8 linear quantization.
Loads fp16/bf16 weights and quantizes them per-tensor during loading."""
activation_quant_key = (
kFp8DynamicTokenSym if cutlass_fp8_supported() else kFp8DynamicTensorSym
)
def __init__(self):
super().__init__()
@@ -211,11 +215,6 @@ class Fp8PerTensorOnlineLinearMethod(OnlineLinearBase):
self.use_marlin = False
self.marlin_input_dtype = None
self.weight_quant_key = kFp8StaticTensorSym
# Use per-token quantization for better perf if dynamic and cutlass
if cutlass_fp8_supported():
self.activation_quant_key = kFp8DynamicTokenSym
else:
self.activation_quant_key = kFp8DynamicTensorSym
def create_weights(
self,
@@ -237,6 +236,8 @@ class Fp8PerTensorOnlineLinearMethod(OnlineLinearBase):
**extra_weight_attrs,
)
# TODO: init_fp8_linear_kernel should support activation_quant_key=None
assert self.activation_quant_key is not None
self.fp8_linear = init_fp8_linear_kernel(
activation_quant_key=self.activation_quant_key,
weight_quant_key=self.weight_quant_key,
@@ -305,13 +306,11 @@ class Fp8PerBlockOnlineLinearMethod(OnlineLinearBase):
"""Online blockwise FP8 linear quantization.
Loads fp16/bf16 weights and quantizes them per-block during loading."""
activation_quant_key = kFp8Dynamic128Sym
def __init__(self):
super().__init__()
self.weight_block_size = [128, 128]
self.activation_quant_key = create_fp8_quant_key(
static=False,
group_shape=GroupShape(1, self.weight_block_size[0]),
)
self.weight_quant_key = create_fp8_quant_key(
static=True, group_shape=GroupShape(*self.weight_block_size)
)
@@ -337,6 +336,8 @@ class Fp8PerBlockOnlineLinearMethod(OnlineLinearBase):
)
layer.weight_block_size = self.weight_block_size
# TODO: init_fp8_linear_kernel should support activation_quant_key=None
assert self.activation_quant_key is not None
self.fp8_linear = init_fp8_linear_kernel(
activation_quant_key=self.activation_quant_key,
weight_quant_key=self.weight_quant_key,
@@ -9,6 +9,7 @@ import torch
from torch.nn import Module
from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig
from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey
if TYPE_CHECKING:
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
@@ -42,6 +43,8 @@ class Mxfp8OnlineLinearMethod(OnlineLinearBase):
FP8 with block-32 scales) during weight loading.
"""
activation_quant_key: QuantKey | None = None
def create_weights(
self,
layer: torch.nn.Module,
@@ -8,9 +8,13 @@ from typing import TYPE_CHECKING
import regex as re
from vllm.logger import init_logger
if TYPE_CHECKING:
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
logger = init_logger(__name__)
def get_quark_ocp_mx_group_size(
quant_config: "QuantizationConfig | None",
@@ -70,18 +74,100 @@ def is_shared_expert_quant_fse_compatible(
A compatibility flag and, when incompatible, the reason.
"""
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
from vllm.model_executor.layers.quantization.online.fp8 import OnlineLinearBase
from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig
from vllm.model_executor.layers.quantization.utils.quant_utils import (
is_layer_skipped,
)
from vllm.models.deepseek_v4.quant_config import DeepseekV4FP8Config
if projection_names is None:
projection_names = ["gate_up_proj", "down_proj"]
if quant_config is None:
return True, None
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig
from vllm.model_executor.layers.quantization.utils.quant_utils import (
is_layer_skipped,
)
from vllm.models.deepseek_v4.quant_config import DeepseekV4FP8Config
online_quant_config = quant_config.online_quantization_config
shared_expert_targets = None
if online_quant_config is not None:
from vllm.model_executor.layers.linear import LinearBase
online_quant_config.packed_modules_mapping = quant_config.packed_modules_mapping
shared_expert_targets = [
online_quant_config.resolve_quant_method_cls(
LinearBase, f"{shared_expert_prefix}.{projection_name}"
)
for projection_name in projection_names
]
if all(target is None for target in shared_expert_targets):
shared_expert_targets = None
if shared_expert_targets is not None:
from vllm.model_executor.layers.fused_moe import RoutedExperts
from vllm.model_executor.layers.linear import UnquantizedLinearMethod
from vllm.model_executor.layers.quantization.online.base import (
ONLINE_SHARED_EXPERT_QUANTIZERS,
)
# NOTE: online shared experts quantization check is only implemented for quark
# quant method at the moment. This can be extended here for other quant methods.
# Requirement is:
# - Retrieve routed weight quant key
# - Retrieve routed activation quant key
# and ensure the online quantization set on shared expert match them.
if isinstance(quant_config, QuarkConfig):
routed_weight_key, routed_activation_key, _ = (
quant_config.get_quant_method_target(expert_prefix, RoutedExperts)
)
else:
reason = (
"online quantization FSE compatibility check is not implemented for "
f"{type(quant_config).__name__}"
)
logger.warning_once(reason)
return False, reason
if routed_weight_key not in ONLINE_SHARED_EXPERT_QUANTIZERS:
return (
False,
"online shared-expert quantization is not supported for routed "
f"expert weight key {routed_weight_key}",
)
for shared_expert_target in shared_expert_targets:
if shared_expert_target is None:
return (
False,
"online quantization targets only part of the shared expert at "
f"{shared_expert_prefix}",
)
_, _, _, shared_quant_spec, shared_method_cls = shared_expert_target
if shared_method_cls in (None, UnquantizedLinearMethod):
return (
False,
"online quantization excludes FSE weights at "
f"{shared_expert_prefix}",
)
assert issubclass(shared_method_cls, OnlineLinearBase)
# NOTE: We can not rely on shared_quant_spec.activation_quant_key
# as _ONLINE_SHORTHANDS: dict[str, QuantSpec] in quantization.py
# does not define the default activation quant key.
# `online_quant_config.resolve_quant_method_cls` should return the
# activation quant key as well. Only some online methods define it
# on the class, so compare the weight key first.
if (
shared_quant_spec.weight != routed_weight_key
or shared_method_cls.activation_quant_key != routed_activation_key
):
return (
False,
"online shared-expert quantization keys do not match the "
"routed expert keys",
)
return True, None
if isinstance(quant_config, DeepseekV4FP8Config):
from vllm.config import get_current_vllm_config