diff --git a/tests/quantization/test_quark.py b/tests/quantization/test_quark.py index 39481456fd..f2f8ed6bd6 100644 --- a/tests/quantization/test_quark.py +++ b/tests/quantization/test_quark.py @@ -855,7 +855,7 @@ def test_quant_method_dispatch_ignored(default_vllm_config): UnquantizedFusedMoEMethod, ) - dynamic_mxfp4_config = _make_qtensor_config( + mxfp4_config = _make_qtensor_config( { "dtype": "fp4", "qscheme": "per_group", @@ -866,24 +866,23 @@ def test_quant_method_dispatch_ignored(default_vllm_config): None, exclude=["self_attn.q_proj", "mlp.down_proj"], ) - dynamic_mxfp4_config.dynamic_mxfp4_quant = True - - assert dynamic_mxfp4_config.get_quant_method_target( - "self_attn.q_proj", LinearBase - ) == (kMxfp4Static, None, QuarkLinearMethod) - attention_proj = TestLinear() - assert isinstance( - dynamic_mxfp4_config.get_quant_method(attention_proj, "self_attn.q_proj"), - QuarkLinearMethod, + assert mxfp4_config.get_quant_method_target("self_attn.q_proj", LinearBase) == ( + None, + None, + UnquantizedLinearMethod, ) - assert isinstance(attention_proj.scheme, QuarkOCP_MX) - assert attention_proj.scheme.dynamic_mxfp4_quant - - assert dynamic_mxfp4_config.get_quant_method_target( - "mlp.down_proj", LinearBase - ) == (None, None, UnquantizedLinearMethod) assert isinstance( - dynamic_mxfp4_config.get_quant_method(TestLinear(), "mlp.down_proj"), + mxfp4_config.get_quant_method(TestLinear(), "self_attn.q_proj"), + UnquantizedLinearMethod, + ) + + assert mxfp4_config.get_quant_method_target("mlp.down_proj", LinearBase) == ( + None, + None, + UnquantizedLinearMethod, + ) + assert isinstance( + mxfp4_config.get_quant_method(TestLinear(), "mlp.down_proj"), UnquantizedLinearMethod, ) diff --git a/vllm/model_executor/layers/quantization/quark/quark.py b/vllm/model_executor/layers/quantization/quark/quark.py index a3a3a36f00..2be85d8176 100644 --- a/vllm/model_executor/layers/quantization/quark/quark.py +++ b/vllm/model_executor/layers/quantization/quark/quark.py @@ -5,7 +5,6 @@ import fnmatch from typing import TYPE_CHECKING, Any, NamedTuple, cast import torch -from transformers import PretrainedConfig from vllm.logger import init_logger from vllm.model_executor.layers.attention import Attention @@ -82,10 +81,6 @@ __all__ = ["QuarkLinearMethod"] logger = init_logger(__name__) -# model_type values that use dynamic MXFP4 re-quantization for -# OCP MX fp4 Quark checkpoints -_DEEPSEEK_V3_FAMILY_MODEL_TYPES = frozenset({"deepseek_v3", "deepseek_v32"}) - class QuantKeyMatch(NamedTuple): matches: bool @@ -119,37 +114,6 @@ class QuarkConfig(QuantizationConfig): self.kv_cache_group = kv_cache_group self.kv_cache_config = kv_cache_config self.pack_method = pack_method - # Note : this flag is kept disabled because the overhead of - # dynamic mxfp4 quantization negates the performance gains - # that come from shifting to mxfp4. It is left here in case - # we want to re-enable it in the future. - self.dynamic_mxfp4_quant = False - - def maybe_update_config( - self, - model_name: str, - hf_config: PretrainedConfig | None = None, - revision: str | None = None, - ): - """Enable dynamic MXFP4 only for DeepSeek-V3-family fp4 checkpoints.""" - if hf_config is None: - return - - if ( - getattr(hf_config, "model_type", None) - not in _DEEPSEEK_V3_FAMILY_MODEL_TYPES - ): - return - - quant_config = getattr(hf_config, "quantization_config", None) - if isinstance(quant_config, dict): - quant_dtype = ( - quant_config.get("global_quant_config", {}) - .get("weight", {}) - .get("dtype") - ) - if quant_dtype == "fp4": - self.dynamic_mxfp4_quant = True def get_linear_method(self) -> "QuarkLinearMethod": return QuarkLinearMethod(self) @@ -201,17 +165,6 @@ class QuarkConfig(QuantizationConfig): self.get_quant_method_target(prefix, type(layer)) ) - exclude_layers = cast(list[str], self.quant_config.get("exclude")) - is_ignored = should_ignore_layer( - prefix, - ignore=exclude_layers, - fused_mapping=self.packed_modules_mapping, - check_children=isinstance(layer, RoutedExperts), - ) - dynamic_mxfp4_quant = ( - is_ignored and "self_attn" in prefix and self.dynamic_mxfp4_quant - ) - if method_cls is UnquantizedFusedMoEMethod: return UnquantizedFusedMoEMethod(layer.moe_config) if method_cls is UnquantizedLinearMethod: @@ -222,7 +175,6 @@ class QuarkConfig(QuantizationConfig): scheme_cls, weight_quant_key=weight_quant_key, activation_quant_key=activation_quant_key, - dynamic_mxfp4_quant=dynamic_mxfp4_quant, weight_config=self._find_matched_config(prefix, type(layer)).get( "weight" ), @@ -274,11 +226,6 @@ class QuarkConfig(QuantizationConfig): if is_routed_experts: return None, None, UnquantizedFusedMoEMethod if issubclass(layer_type, LinearBase): - if "self_attn" in prefix and self.dynamic_mxfp4_quant: - weight_quant_key, activation_key, _ = self.get_scheme_cls( - layer_type, prefix - ) - return weight_quant_key, activation_key, QuarkLinearMethod return None, None, UnquantizedLinearMethod return None, None, None if issubclass(layer_type, LinearBase): @@ -956,7 +903,6 @@ class QuarkConfig(QuantizationConfig): scheme_cls: type["QuarkScheme"], weight_quant_key: QuantKey | None, activation_quant_key: QuantKey | None, - dynamic_mxfp4_quant: bool = False, weight_config: dict[str, Any] | None = None, ) -> "QuarkScheme": """Construct a Quark scheme selected by get_scheme_cls.""" @@ -996,9 +942,6 @@ class QuarkConfig(QuantizationConfig): ): kwargs["weight_quant_key"] = weight_quant_key - if scheme_cls is QuarkOCP_MX: - kwargs["dynamic_mxfp4_quant"] = dynamic_mxfp4_quant - scheme = scheme_cls(**kwargs) # Raise error if device does not support the scheme diff --git a/vllm/model_executor/layers/quantization/quark/schemes/quark_ocp_mx.py b/vllm/model_executor/layers/quantization/quark/schemes/quark_ocp_mx.py index 30758782c3..50263863d8 100644 --- a/vllm/model_executor/layers/quantization/quark/schemes/quark_ocp_mx.py +++ b/vllm/model_executor/layers/quantization/quark/schemes/quark_ocp_mx.py @@ -26,10 +26,8 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import ( ) from vllm.model_executor.parameter import ( GroupQuantScaleParameter, - ModelWeightParameter, PackedvLLMParameter, ) -from vllm.model_executor.utils import set_weight_attrs from vllm.platforms import current_platform from .quark_scheme import QuarkScheme @@ -46,10 +44,9 @@ class QuarkOCP_MX(QuarkScheme): self, weight_quant_key: QuantKey, activation_quant_key: QuantKey | None, - dynamic_mxfp4_quant: bool = False, ): super().__init__(weight_quant_key, activation_quant_key) - self.dynamic_mxfp4_quant = dynamic_mxfp4_quant + self.weight_dtype = next( dtype for dtype, quant_key in _WEIGHT_QUANT_KEY_MAP.items() @@ -109,21 +106,9 @@ class QuarkOCP_MX(QuarkScheme): def get_min_capability(cls) -> int: return 70 - def process_dynamic_mxfp4_weights_after_loading( - self, layer: torch.nn.Module - ) -> None: - from aiter.ops.triton.quant import dynamic_mxfp4_quant - - w_q, w_s = dynamic_mxfp4_quant(layer.weight) - layer.weight_scale = torch.nn.Parameter(w_s, requires_grad=False) - layer.weight = torch.nn.Parameter(w_q, requires_grad=False) - def process_weights_after_loading(self, layer: torch.nn.Module) -> None: layer.weight = torch.nn.Parameter(layer.weight.data, requires_grad=False) - if self.dynamic_mxfp4_quant: - self.process_dynamic_mxfp4_weights_after_loading(layer) - self.ocp_mx_linear.process_weights_after_loading(layer) def create_weights( @@ -145,51 +130,36 @@ class QuarkOCP_MX(QuarkScheme): "tensor-parallel sharding for this layer." ) - if self.dynamic_mxfp4_quant: - weight = ModelWeightParameter( - data=torch.empty( - sum(output_partition_sizes), - input_size_per_partition, - dtype=params_dtype, - ), - input_dim=1, - output_dim=0, - weight_loader=weight_loader, - ) + output_size_per_partition = sum(output_partition_sizes) + layer.logical_widths = output_partition_sizes - layer.register_parameter("weight", weight) - set_weight_attrs(weight, kwargs) - else: - output_size_per_partition = sum(output_partition_sizes) - layer.logical_widths = output_partition_sizes + # WEIGHT + weight = PackedvLLMParameter( + data=torch.empty( + output_size_per_partition, + self.get_packed_dim(input_size_per_partition, self.weight_dtype), + dtype=torch.uint8, + ), + input_dim=1, + output_dim=0, + packed_dim=1, + packed_factor=self.packed_factor, + weight_loader=weight_loader, + ) + layer.register_parameter("weight", weight) - # WEIGHT - weight = PackedvLLMParameter( - data=torch.empty( - output_size_per_partition, - self.get_packed_dim(input_size_per_partition, self.weight_dtype), - dtype=torch.uint8, - ), - input_dim=1, - output_dim=0, - packed_dim=1, - packed_factor=self.packed_factor, - weight_loader=weight_loader, - ) - layer.register_parameter("weight", weight) - - # WEIGHT SCALE - weight_scale = GroupQuantScaleParameter( - data=torch.empty( - output_size_per_partition, - input_size_per_partition // OCP_MX_BLOCK_SIZE, - dtype=torch.uint8, - ), - input_dim=1, - output_dim=0, - weight_loader=weight_loader, - ) - layer.register_parameter("weight_scale", weight_scale) + # WEIGHT SCALE + weight_scale = GroupQuantScaleParameter( + data=torch.empty( + output_size_per_partition, + input_size_per_partition // OCP_MX_BLOCK_SIZE, + dtype=torch.uint8, + ), + input_dim=1, + output_dim=0, + weight_loader=weight_loader, + ) + layer.register_parameter("weight_scale", weight_scale) if self.weight_quant_key == kMxfp4Static: self.ocp_mx_linear = init_mxfp4_linear_kernel(