[Quantization] Enable shared expert fusion compatibility with online shared_expert quantization (showcase: along Quark MXFP4 routed experts) (#55686)
Signed-off-by: Felix Marty <Felix.Marty@amd.com> Signed-off-by: fxmarty-amd <felmarty@amd.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Misha Goin <mgoin64@gmail.com>
This commit is contained in:
co-authored by
OpenAI Codex
Misha Goin
parent
e2414ec1db
commit
9f3d70e081
@@ -1293,6 +1293,7 @@ class FusedMoEConfig:
|
||||
has_bias: bool = False
|
||||
is_lora_enabled: bool = False
|
||||
has_hash_routing: bool = False
|
||||
shared_expert_prefix: str | None = None
|
||||
|
||||
# When True, the MoE skips its final cross-rank all-reduce (and the separate
|
||||
# shared-expert reduce), returning the partial per-rank sum. The caller is
|
||||
|
||||
@@ -122,6 +122,7 @@ def FusedMoEFactory(
|
||||
is_fused_checkpoint_transposed: bool = False,
|
||||
n_shared_experts: int | None = None,
|
||||
fuse_shared_experts: bool = False,
|
||||
shared_expert_prefix: str | None = None,
|
||||
router_logits_dtype: torch.dtype | None = None,
|
||||
gate: torch.nn.Module | None = None,
|
||||
shared_experts: torch.nn.Module | None = None,
|
||||
@@ -197,6 +198,7 @@ def FusedMoEFactory(
|
||||
n_shared_experts: Number of shared experts to fuse into the routed
|
||||
grouped GEMM (ROCm; requires aiter FSE or the router-append path)
|
||||
fuse_shared_experts: Whether to enable shared-expert fusion.
|
||||
shared_expert_prefix: Checkpoint prefix for the fused shared expert.
|
||||
router_logits_dtype: Data type for router logits buffers
|
||||
gate: Pre-configured gate module
|
||||
shared_experts: Pre-configured shared experts module
|
||||
@@ -360,6 +362,7 @@ def FusedMoEFactory(
|
||||
elastic_ep_max_dp_size=vllm_config.parallel_config.elastic_ep_max_dp_size,
|
||||
has_bias=has_bias,
|
||||
is_lora_enabled=vllm_config.lora_config is not None,
|
||||
shared_expert_prefix=shared_expert_prefix,
|
||||
activation=moe_activation,
|
||||
device=vllm_config.device_config.device,
|
||||
routing_method=router.routing_method_type, # Not ideal
|
||||
|
||||
@@ -26,6 +26,7 @@ from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
|
||||
from vllm.model_executor.layers.fused_moe.unquantized_fused_moe_method import (
|
||||
UnquantizedFusedMoEMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import LinearBase
|
||||
from vllm.model_executor.layers.quantization.base_config import (
|
||||
QuantizationConfig,
|
||||
resolve_quant_method,
|
||||
@@ -153,6 +154,7 @@ class RoutedExperts(PluggableLayer):
|
||||
self.apply_router_weight_on_input = apply_router_weight_on_input
|
||||
# End random parameters
|
||||
self._loaded_expert_biases: set[str] = set()
|
||||
self._fused_shared_expert_quantizer = self._get_fused_shared_expert_quantizer()
|
||||
|
||||
self.quant_method = self._get_quant_method(
|
||||
self.layer_name,
|
||||
@@ -630,6 +632,31 @@ class RoutedExperts(PluggableLayer):
|
||||
# _to_scalar's reshape(()) would reject the size-2 weight_shape.
|
||||
param_data[expert_id] = loaded_weight
|
||||
|
||||
def _get_fused_shared_expert_quantizer(
|
||||
self,
|
||||
) -> Callable[[torch.Tensor], tuple[torch.Tensor, torch.Tensor]] | None:
|
||||
"""Return the quantizer for an online-quantized fused shared expert."""
|
||||
online_config = getattr(self.quant_config, "online_quantization_config", None)
|
||||
prefix = self.moe_config.shared_expert_prefix
|
||||
if (
|
||||
online_config is None
|
||||
or prefix is None
|
||||
or self.expert_map_manager.num_fused_shared_experts == 0
|
||||
):
|
||||
return None
|
||||
# FSE compatibility checks require every shared-expert projection to
|
||||
# use the routed experts' weight key.
|
||||
resolved = online_config.resolve_quant_method_cls(
|
||||
LinearBase, f"{prefix}.down_proj"
|
||||
)
|
||||
if resolved is None:
|
||||
return None
|
||||
from vllm.model_executor.layers.quantization.online.base import (
|
||||
ONLINE_SHARED_EXPERT_QUANTIZERS,
|
||||
)
|
||||
|
||||
return ONLINE_SHARED_EXPERT_QUANTIZERS[resolved[3].weight]
|
||||
|
||||
@overload
|
||||
def weight_loader(
|
||||
self,
|
||||
@@ -687,6 +714,35 @@ class RoutedExperts(PluggableLayer):
|
||||
return False if return_success else None
|
||||
# Hereafter, `expert_id` is local physical id
|
||||
|
||||
# A full-precision fused shared expert is quantized into the checkpoint
|
||||
# layout, then loaded like a pre-quantized expert weight and scale.
|
||||
if (
|
||||
self._fused_shared_expert_quantizer is not None
|
||||
and self.moe_config.num_logical_experts
|
||||
<= global_expert_id
|
||||
< self.moe_config.num_logical_experts
|
||||
+ self.expert_map_manager.num_fused_shared_experts
|
||||
and weight_name.endswith("_weight")
|
||||
and loaded_weight.is_floating_point()
|
||||
and loaded_weight.dtype != param.dtype
|
||||
):
|
||||
weight, weight_scale = self._fused_shared_expert_quantizer(
|
||||
loaded_weight.to(self.moe_config.device)
|
||||
)
|
||||
stem = "w2" if shard_id == "w2" else "w13"
|
||||
self.weight_loader(
|
||||
getattr(self, f"{stem}_weight_scale"),
|
||||
weight_scale,
|
||||
f"{weight_name}_scale",
|
||||
shard_id,
|
||||
global_expert_id,
|
||||
return_success=True,
|
||||
)
|
||||
loaded = self.weight_loader(
|
||||
param, weight, weight_name, shard_id, global_expert_id, True
|
||||
)
|
||||
return loaded if return_success else None
|
||||
|
||||
# is_transposed: if the dim to shard the weight
|
||||
# should be flipped. Required by GPTQ/AWQ (K-first format).
|
||||
# should be whatever dimension intermediate_size_per_partition is
|
||||
|
||||
@@ -84,6 +84,40 @@ def resolve_layer_fused_shared_expert(
|
||||
else (True, None)
|
||||
)
|
||||
is_fused_shared_expert_enabled = fse_requested and fse_compatible
|
||||
|
||||
# In online shared_expert quantization is used, register it to
|
||||
# `online_quantization_config.quantized_layers` ahead of weight loading.
|
||||
if is_fused_shared_expert_enabled and quant_config is not None:
|
||||
online_quantization_config = quant_config.online_quantization_config
|
||||
if online_quantization_config is not None:
|
||||
from vllm.model_executor.layers.linear import (
|
||||
LinearBase,
|
||||
UnquantizedLinearMethod,
|
||||
)
|
||||
|
||||
for projection_name in ("gate_up_proj", "down_proj"):
|
||||
projection_prefix = f"{prefix}.{shared_expert_name}.{projection_name}"
|
||||
quant_method_metadata = (
|
||||
online_quantization_config.resolve_quant_method_cls(
|
||||
LinearBase, projection_prefix
|
||||
)
|
||||
)
|
||||
|
||||
if quant_method_metadata is None:
|
||||
continue
|
||||
|
||||
source, quant_key_str, target_pattern, _, quant_method_cls = (
|
||||
quant_method_metadata
|
||||
)
|
||||
|
||||
if quant_method_cls in (None, UnquantizedLinearMethod):
|
||||
continue
|
||||
|
||||
online_quantization_config.quantized_layers[projection_prefix] = (
|
||||
source.value,
|
||||
quant_key_str,
|
||||
target_pattern,
|
||||
)
|
||||
if fse_requested and not is_fused_shared_expert_enabled:
|
||||
logger.warning(
|
||||
"VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS is enabled but "
|
||||
|
||||
@@ -320,7 +320,7 @@ def resolve_quant_method(
|
||||
base_quant_method, (UnquantizedLinearMethod, UnquantizedFusedMoEMethod)
|
||||
)
|
||||
online_target = quant_config.online_quantization_config.resolve_quant_method_cls(
|
||||
layer, prefix
|
||||
type(layer), prefix
|
||||
)
|
||||
|
||||
if checkpoint_is_quantized:
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Mapping
|
||||
from collections.abc import Callable, Mapping
|
||||
from enum import Enum
|
||||
from types import MappingProxyType
|
||||
from typing import Any
|
||||
@@ -60,6 +60,7 @@ from vllm.model_executor.layers.quantization.utils.config_utils import (
|
||||
find_matching_patterns,
|
||||
get_layer_name_after_index,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import mxfp4_quantize
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kFp8Static128BlockSym,
|
||||
@@ -104,6 +105,15 @@ _ONLINE_MOE_METHODS: dict[QuantKey, type] = {
|
||||
kNvfp4Static: Nvfp4OnlineMoEMethod,
|
||||
}
|
||||
|
||||
# Quantizers for full-precision fused shared-expert weights, keyed by the routed
|
||||
# experts' weight QuantKey. Each shard is quantized to the checkpoint layout on
|
||||
# arrival, so only block-local schemes are supported.
|
||||
ONLINE_SHARED_EXPERT_QUANTIZERS: dict[
|
||||
QuantKey, Callable[[torch.Tensor], tuple[torch.Tensor, torch.Tensor]]
|
||||
] = {
|
||||
kMxfp4Static: mxfp4_quantize,
|
||||
}
|
||||
|
||||
|
||||
def _find_matching_targets(
|
||||
prefix: str,
|
||||
@@ -217,14 +227,14 @@ class OnlineQuantizationConfig(QuantizationConfig):
|
||||
self,
|
||||
spec: QuantSpec | None,
|
||||
table: dict[QuantKey, type],
|
||||
layer: torch.nn.Module,
|
||||
layer_cls: type[torch.nn.Module],
|
||||
) -> type | None:
|
||||
"""Resolve the online method class for a layer's quantization spec.
|
||||
|
||||
Args:
|
||||
spec: Quantization specification to resolve.
|
||||
table: Mapping from weight quantization keys to method classes.
|
||||
layer: Layer that will use the resolved method.
|
||||
layer_cls: Type of the layer that will use the resolved method.
|
||||
|
||||
Returns:
|
||||
The matching method class, or None when ``spec`` has no weight
|
||||
@@ -239,7 +249,7 @@ class OnlineQuantizationConfig(QuantizationConfig):
|
||||
cls = table.get(spec.weight)
|
||||
if cls is None:
|
||||
raise ValueError(
|
||||
f"online quantization for {type(layer).__name__} with "
|
||||
f"online quantization for {layer_cls.__name__} with "
|
||||
f"weight={spec.weight} is not supported; supported weight "
|
||||
f"keys: {sorted(str(k) for k in table)}"
|
||||
)
|
||||
@@ -254,12 +264,12 @@ class OnlineQuantizationConfig(QuantizationConfig):
|
||||
return cls
|
||||
|
||||
def resolve_quant_method_cls(
|
||||
self, layer: torch.nn.Module, prefix: str
|
||||
self, layer_cls: type[torch.nn.Module], prefix: str
|
||||
) -> tuple[OnlineQuantizationSource, str, str | None, QuantSpec, type] | None:
|
||||
"""Resolve quantization metadata and method class without instantiating it.
|
||||
|
||||
Args:
|
||||
layer: Layer for which to resolve online quantization.
|
||||
layer_cls: Type of layer for which to resolve online quantization.
|
||||
prefix: Fully qualified layer name.
|
||||
|
||||
Returns:
|
||||
@@ -271,17 +281,17 @@ class OnlineQuantizationConfig(QuantizationConfig):
|
||||
quant_spec: QuantSpec | None
|
||||
if self.args.targets is not None:
|
||||
resolved_pattern = self._resolve_targets_quant_method_metadata(
|
||||
prefix, layer
|
||||
prefix, layer_cls
|
||||
)
|
||||
if resolved_pattern is None:
|
||||
return None
|
||||
source, quant_key_str, target_pattern, quant_spec, table = resolved_pattern
|
||||
else:
|
||||
if isinstance(layer, LinearBase):
|
||||
if issubclass(layer_cls, LinearBase):
|
||||
source = OnlineQuantizationSource.linear
|
||||
quant_spec = self.args.linear
|
||||
table = _ONLINE_LINEAR_METHODS
|
||||
elif isinstance(layer, RoutedExperts):
|
||||
elif issubclass(layer_cls, RoutedExperts):
|
||||
source = OnlineQuantizationSource.moe
|
||||
quant_spec = self.args.moe
|
||||
table = _ONLINE_MOE_METHODS
|
||||
@@ -298,14 +308,14 @@ class OnlineQuantizationConfig(QuantizationConfig):
|
||||
quant_key_str = str(quant_spec)
|
||||
target_pattern = None
|
||||
|
||||
quant_method_cls = self._get_method_cls(quant_spec, table, layer)
|
||||
quant_method_cls = self._get_method_cls(quant_spec, table, layer_cls)
|
||||
if quant_method_cls is None:
|
||||
return None
|
||||
assert quant_spec is not None
|
||||
return source, quant_key_str, target_pattern, quant_spec, quant_method_cls
|
||||
|
||||
def _resolve_targets_quant_method_metadata(
|
||||
self, prefix: str, layer: torch.nn.Module
|
||||
self, prefix: str, layer_cls: type[torch.nn.Module]
|
||||
) -> (
|
||||
tuple[OnlineQuantizationSource, str, str, QuantSpec, dict[QuantKey, type]]
|
||||
| None
|
||||
@@ -314,7 +324,7 @@ class OnlineQuantizationConfig(QuantizationConfig):
|
||||
|
||||
Args:
|
||||
prefix: Fully qualified layer name.
|
||||
layer: Layer matched against configured target patterns.
|
||||
layer_cls: Type of layer matched against configured target patterns.
|
||||
|
||||
Returns:
|
||||
A tuple of source, quantization key string, target pattern, spec,
|
||||
@@ -349,22 +359,22 @@ class OnlineQuantizationConfig(QuantizationConfig):
|
||||
target_pattern = matches[0]
|
||||
quant_key_str = self.args.targets[target_pattern]
|
||||
shorthand = _ONLINE_SHORTHANDS[quant_key_str]
|
||||
if isinstance(layer, LinearBase):
|
||||
if issubclass(layer_cls, LinearBase):
|
||||
quant_spec = shorthand.linear
|
||||
table = _ONLINE_LINEAR_METHODS
|
||||
elif isinstance(layer, RoutedExperts):
|
||||
elif issubclass(layer_cls, RoutedExperts):
|
||||
quant_spec = shorthand.moe
|
||||
table = _ONLINE_MOE_METHODS
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Layer {prefix} was matched by quantization_config.targets "
|
||||
f"({target_pattern}), but online quantization is not supported for "
|
||||
f"{type(layer).__name__}."
|
||||
f"{layer_cls.__name__}."
|
||||
)
|
||||
if quant_spec is None:
|
||||
raise ValueError(
|
||||
f"targets pattern {target_pattern} = {quant_key_str} does "
|
||||
f"not define a QuantSpec for {type(layer).__name__} layers "
|
||||
f"not define a QuantSpec for {layer_cls.__name__} layers "
|
||||
f"(matched at {prefix})."
|
||||
)
|
||||
return (
|
||||
@@ -379,7 +389,7 @@ class OnlineQuantizationConfig(QuantizationConfig):
|
||||
self, layer: torch.nn.Module, prefix: str
|
||||
) -> "QuantizeMethodBase | None":
|
||||
# `targets` takes precedence over `moe` and `linear` and is exclusive.
|
||||
resolved = self.resolve_quant_method_cls(layer, prefix)
|
||||
resolved = self.resolve_quant_method_cls(type(layer), prefix)
|
||||
if resolved is not None:
|
||||
source, quant_key_str, target_pattern, _, quant_method_cls = resolved
|
||||
self.quantized_layers[prefix] = (
|
||||
|
||||
@@ -12,8 +12,6 @@ if TYPE_CHECKING:
|
||||
FusedMoEQuantConfig,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.fp8 import Fp8MoeBackend
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.config import get_current_vllm_config
|
||||
@@ -40,6 +38,7 @@ from vllm.model_executor.layers.quantization.online.moe_base import (
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
GroupShape,
|
||||
QuantKey,
|
||||
amax_for_moe_weight_quant,
|
||||
amax_for_tp_weight_quant,
|
||||
create_fp8_quant_key,
|
||||
@@ -122,6 +121,7 @@ class OnlineLinearBase(LinearMethodBase):
|
||||
weights onto meta device and materializes them just-in-time."""
|
||||
|
||||
uses_meta_device: bool = True
|
||||
activation_quant_key: QuantKey | None
|
||||
|
||||
def __init__(self):
|
||||
self.out_dtype = torch.get_default_dtype()
|
||||
@@ -203,6 +203,10 @@ class Fp8PerTensorOnlineLinearMethod(OnlineLinearBase):
|
||||
"""Online tensorwise FP8 linear quantization.
|
||||
Loads fp16/bf16 weights and quantizes them per-tensor during loading."""
|
||||
|
||||
activation_quant_key = (
|
||||
kFp8DynamicTokenSym if cutlass_fp8_supported() else kFp8DynamicTensorSym
|
||||
)
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
|
||||
@@ -211,11 +215,6 @@ class Fp8PerTensorOnlineLinearMethod(OnlineLinearBase):
|
||||
self.use_marlin = False
|
||||
self.marlin_input_dtype = None
|
||||
self.weight_quant_key = kFp8StaticTensorSym
|
||||
# Use per-token quantization for better perf if dynamic and cutlass
|
||||
if cutlass_fp8_supported():
|
||||
self.activation_quant_key = kFp8DynamicTokenSym
|
||||
else:
|
||||
self.activation_quant_key = kFp8DynamicTensorSym
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
@@ -237,6 +236,8 @@ class Fp8PerTensorOnlineLinearMethod(OnlineLinearBase):
|
||||
**extra_weight_attrs,
|
||||
)
|
||||
|
||||
# TODO: init_fp8_linear_kernel should support activation_quant_key=None
|
||||
assert self.activation_quant_key is not None
|
||||
self.fp8_linear = init_fp8_linear_kernel(
|
||||
activation_quant_key=self.activation_quant_key,
|
||||
weight_quant_key=self.weight_quant_key,
|
||||
@@ -305,13 +306,11 @@ class Fp8PerBlockOnlineLinearMethod(OnlineLinearBase):
|
||||
"""Online blockwise FP8 linear quantization.
|
||||
Loads fp16/bf16 weights and quantizes them per-block during loading."""
|
||||
|
||||
activation_quant_key = kFp8Dynamic128Sym
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.weight_block_size = [128, 128]
|
||||
self.activation_quant_key = create_fp8_quant_key(
|
||||
static=False,
|
||||
group_shape=GroupShape(1, self.weight_block_size[0]),
|
||||
)
|
||||
self.weight_quant_key = create_fp8_quant_key(
|
||||
static=True, group_shape=GroupShape(*self.weight_block_size)
|
||||
)
|
||||
@@ -337,6 +336,8 @@ class Fp8PerBlockOnlineLinearMethod(OnlineLinearBase):
|
||||
)
|
||||
layer.weight_block_size = self.weight_block_size
|
||||
|
||||
# TODO: init_fp8_linear_kernel should support activation_quant_key=None
|
||||
assert self.activation_quant_key is not None
|
||||
self.fp8_linear = init_fp8_linear_kernel(
|
||||
activation_quant_key=self.activation_quant_key,
|
||||
weight_quant_key=self.weight_quant_key,
|
||||
|
||||
@@ -9,6 +9,7 @@ import torch
|
||||
from torch.nn import Module
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
@@ -42,6 +43,8 @@ class Mxfp8OnlineLinearMethod(OnlineLinearBase):
|
||||
FP8 with block-32 scales) during weight loading.
|
||||
"""
|
||||
|
||||
activation_quant_key: QuantKey | None = None
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
|
||||
@@ -8,9 +8,13 @@ from typing import TYPE_CHECKING
|
||||
|
||||
import regex as re
|
||||
|
||||
from vllm.logger import init_logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def get_quark_ocp_mx_group_size(
|
||||
quant_config: "QuantizationConfig | None",
|
||||
@@ -70,18 +74,100 @@ def is_shared_expert_quant_fse_compatible(
|
||||
A compatibility flag and, when incompatible, the reason.
|
||||
|
||||
"""
|
||||
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
|
||||
from vllm.model_executor.layers.quantization.online.fp8 import OnlineLinearBase
|
||||
from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
is_layer_skipped,
|
||||
)
|
||||
from vllm.models.deepseek_v4.quant_config import DeepseekV4FP8Config
|
||||
|
||||
if projection_names is None:
|
||||
projection_names = ["gate_up_proj", "down_proj"]
|
||||
|
||||
if quant_config is None:
|
||||
return True, None
|
||||
|
||||
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
|
||||
from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
is_layer_skipped,
|
||||
)
|
||||
from vllm.models.deepseek_v4.quant_config import DeepseekV4FP8Config
|
||||
online_quant_config = quant_config.online_quantization_config
|
||||
shared_expert_targets = None
|
||||
if online_quant_config is not None:
|
||||
from vllm.model_executor.layers.linear import LinearBase
|
||||
|
||||
online_quant_config.packed_modules_mapping = quant_config.packed_modules_mapping
|
||||
shared_expert_targets = [
|
||||
online_quant_config.resolve_quant_method_cls(
|
||||
LinearBase, f"{shared_expert_prefix}.{projection_name}"
|
||||
)
|
||||
for projection_name in projection_names
|
||||
]
|
||||
if all(target is None for target in shared_expert_targets):
|
||||
shared_expert_targets = None
|
||||
|
||||
if shared_expert_targets is not None:
|
||||
from vllm.model_executor.layers.fused_moe import RoutedExperts
|
||||
from vllm.model_executor.layers.linear import UnquantizedLinearMethod
|
||||
from vllm.model_executor.layers.quantization.online.base import (
|
||||
ONLINE_SHARED_EXPERT_QUANTIZERS,
|
||||
)
|
||||
|
||||
# NOTE: online shared experts quantization check is only implemented for quark
|
||||
# quant method at the moment. This can be extended here for other quant methods.
|
||||
# Requirement is:
|
||||
# - Retrieve routed weight quant key
|
||||
# - Retrieve routed activation quant key
|
||||
# and ensure the online quantization set on shared expert match them.
|
||||
if isinstance(quant_config, QuarkConfig):
|
||||
routed_weight_key, routed_activation_key, _ = (
|
||||
quant_config.get_quant_method_target(expert_prefix, RoutedExperts)
|
||||
)
|
||||
else:
|
||||
reason = (
|
||||
"online quantization FSE compatibility check is not implemented for "
|
||||
f"{type(quant_config).__name__}"
|
||||
)
|
||||
logger.warning_once(reason)
|
||||
return False, reason
|
||||
|
||||
if routed_weight_key not in ONLINE_SHARED_EXPERT_QUANTIZERS:
|
||||
return (
|
||||
False,
|
||||
"online shared-expert quantization is not supported for routed "
|
||||
f"expert weight key {routed_weight_key}",
|
||||
)
|
||||
|
||||
for shared_expert_target in shared_expert_targets:
|
||||
if shared_expert_target is None:
|
||||
return (
|
||||
False,
|
||||
"online quantization targets only part of the shared expert at "
|
||||
f"{shared_expert_prefix}",
|
||||
)
|
||||
|
||||
_, _, _, shared_quant_spec, shared_method_cls = shared_expert_target
|
||||
if shared_method_cls in (None, UnquantizedLinearMethod):
|
||||
return (
|
||||
False,
|
||||
"online quantization excludes FSE weights at "
|
||||
f"{shared_expert_prefix}",
|
||||
)
|
||||
assert issubclass(shared_method_cls, OnlineLinearBase)
|
||||
|
||||
# NOTE: We can not rely on shared_quant_spec.activation_quant_key
|
||||
# as _ONLINE_SHORTHANDS: dict[str, QuantSpec] in quantization.py
|
||||
# does not define the default activation quant key.
|
||||
# `online_quant_config.resolve_quant_method_cls` should return the
|
||||
# activation quant key as well. Only some online methods define it
|
||||
# on the class, so compare the weight key first.
|
||||
if (
|
||||
shared_quant_spec.weight != routed_weight_key
|
||||
or shared_method_cls.activation_quant_key != routed_activation_key
|
||||
):
|
||||
return (
|
||||
False,
|
||||
"online shared-expert quantization keys do not match the "
|
||||
"routed expert keys",
|
||||
)
|
||||
return True, None
|
||||
|
||||
if isinstance(quant_config, DeepseekV4FP8Config):
|
||||
from vllm.config import get_current_vllm_config
|
||||
|
||||
@@ -199,6 +199,7 @@ class AXK1MoE(nn.Module):
|
||||
if self.is_fused_shared_expert_enabled
|
||||
else None,
|
||||
fuse_shared_experts=self.is_fused_shared_expert_enabled,
|
||||
shared_expert_prefix=f"{prefix}.shared_experts",
|
||||
)
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
|
||||
@@ -396,6 +396,7 @@ class DeepseekV2MoE(nn.Module):
|
||||
if self.is_fused_shared_expert_enabled
|
||||
else None,
|
||||
fuse_shared_experts=self.is_fused_shared_expert_enabled,
|
||||
shared_expert_prefix=f"{prefix}.shared_experts",
|
||||
router_logits_dtype=self.gate.out_dtype,
|
||||
)
|
||||
|
||||
|
||||
@@ -204,6 +204,7 @@ class Glm4MoE(nn.Module):
|
||||
config.n_shared_experts if self.is_fused_shared_expert_enabled else None
|
||||
),
|
||||
fuse_shared_experts=self.is_fused_shared_expert_enabled,
|
||||
shared_expert_prefix=f"{prefix}.shared_experts",
|
||||
)
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
|
||||
@@ -232,6 +232,7 @@ class Qwen3NextSparseMoeBlock(nn.Module):
|
||||
is_sequence_parallel=self.is_sequence_parallel,
|
||||
n_shared_experts=1 if self.shared_expert is None else None,
|
||||
fuse_shared_experts=self.is_fused_shared_expert_enabled,
|
||||
shared_expert_prefix=f"{prefix}.shared_expert",
|
||||
shared_expert_gate=self.shared_expert_gate
|
||||
if self.shared_expert is None
|
||||
else None,
|
||||
|
||||
Reference in New Issue
Block a user