[Quark] Remove quark-specific silent online quantization (#51800)

Signed-off-by: Felix Marty <Felix.Marty@amd.com>
Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
fxmarty-amd
2026-09-23 11:35:30 -04:00
committed by GitHub
co-authored by mgoin
parent f69cc7c178
commit 0549e8d0ab
3 changed files with 45 additions and 133 deletions
+16 -17
View File
@@ -855,7 +855,7 @@ def test_quant_method_dispatch_ignored(default_vllm_config):
UnquantizedFusedMoEMethod,
)
dynamic_mxfp4_config = _make_qtensor_config(
mxfp4_config = _make_qtensor_config(
{
"dtype": "fp4",
"qscheme": "per_group",
@@ -866,24 +866,23 @@ def test_quant_method_dispatch_ignored(default_vllm_config):
None,
exclude=["self_attn.q_proj", "mlp.down_proj"],
)
dynamic_mxfp4_config.dynamic_mxfp4_quant = True
assert dynamic_mxfp4_config.get_quant_method_target(
"self_attn.q_proj", LinearBase
) == (kMxfp4Static, None, QuarkLinearMethod)
attention_proj = TestLinear()
assert isinstance(
dynamic_mxfp4_config.get_quant_method(attention_proj, "self_attn.q_proj"),
QuarkLinearMethod,
assert mxfp4_config.get_quant_method_target("self_attn.q_proj", LinearBase) == (
None,
None,
UnquantizedLinearMethod,
)
assert isinstance(attention_proj.scheme, QuarkOCP_MX)
assert attention_proj.scheme.dynamic_mxfp4_quant
assert dynamic_mxfp4_config.get_quant_method_target(
"mlp.down_proj", LinearBase
) == (None, None, UnquantizedLinearMethod)
assert isinstance(
dynamic_mxfp4_config.get_quant_method(TestLinear(), "mlp.down_proj"),
mxfp4_config.get_quant_method(TestLinear(), "self_attn.q_proj"),
UnquantizedLinearMethod,
)
assert mxfp4_config.get_quant_method_target("mlp.down_proj", LinearBase) == (
None,
None,
UnquantizedLinearMethod,
)
assert isinstance(
mxfp4_config.get_quant_method(TestLinear(), "mlp.down_proj"),
UnquantizedLinearMethod,
)
@@ -5,7 +5,6 @@ import fnmatch
from typing import TYPE_CHECKING, Any, NamedTuple, cast
import torch
from transformers import PretrainedConfig
from vllm.logger import init_logger
from vllm.model_executor.layers.attention import Attention
@@ -82,10 +81,6 @@ __all__ = ["QuarkLinearMethod"]
logger = init_logger(__name__)
# model_type values that use dynamic MXFP4 re-quantization for
# OCP MX fp4 Quark checkpoints
_DEEPSEEK_V3_FAMILY_MODEL_TYPES = frozenset({"deepseek_v3", "deepseek_v32"})
class QuantKeyMatch(NamedTuple):
matches: bool
@@ -119,37 +114,6 @@ class QuarkConfig(QuantizationConfig):
self.kv_cache_group = kv_cache_group
self.kv_cache_config = kv_cache_config
self.pack_method = pack_method
# Note : this flag is kept disabled because the overhead of
# dynamic mxfp4 quantization negates the performance gains
# that come from shifting to mxfp4. It is left here in case
# we want to re-enable it in the future.
self.dynamic_mxfp4_quant = False
def maybe_update_config(
self,
model_name: str,
hf_config: PretrainedConfig | None = None,
revision: str | None = None,
):
"""Enable dynamic MXFP4 only for DeepSeek-V3-family fp4 checkpoints."""
if hf_config is None:
return
if (
getattr(hf_config, "model_type", None)
not in _DEEPSEEK_V3_FAMILY_MODEL_TYPES
):
return
quant_config = getattr(hf_config, "quantization_config", None)
if isinstance(quant_config, dict):
quant_dtype = (
quant_config.get("global_quant_config", {})
.get("weight", {})
.get("dtype")
)
if quant_dtype == "fp4":
self.dynamic_mxfp4_quant = True
def get_linear_method(self) -> "QuarkLinearMethod":
return QuarkLinearMethod(self)
@@ -201,17 +165,6 @@ class QuarkConfig(QuantizationConfig):
self.get_quant_method_target(prefix, type(layer))
)
exclude_layers = cast(list[str], self.quant_config.get("exclude"))
is_ignored = should_ignore_layer(
prefix,
ignore=exclude_layers,
fused_mapping=self.packed_modules_mapping,
check_children=isinstance(layer, RoutedExperts),
)
dynamic_mxfp4_quant = (
is_ignored and "self_attn" in prefix and self.dynamic_mxfp4_quant
)
if method_cls is UnquantizedFusedMoEMethod:
return UnquantizedFusedMoEMethod(layer.moe_config)
if method_cls is UnquantizedLinearMethod:
@@ -222,7 +175,6 @@ class QuarkConfig(QuantizationConfig):
scheme_cls,
weight_quant_key=weight_quant_key,
activation_quant_key=activation_quant_key,
dynamic_mxfp4_quant=dynamic_mxfp4_quant,
weight_config=self._find_matched_config(prefix, type(layer)).get(
"weight"
),
@@ -274,11 +226,6 @@ class QuarkConfig(QuantizationConfig):
if is_routed_experts:
return None, None, UnquantizedFusedMoEMethod
if issubclass(layer_type, LinearBase):
if "self_attn" in prefix and self.dynamic_mxfp4_quant:
weight_quant_key, activation_key, _ = self.get_scheme_cls(
layer_type, prefix
)
return weight_quant_key, activation_key, QuarkLinearMethod
return None, None, UnquantizedLinearMethod
return None, None, None
if issubclass(layer_type, LinearBase):
@@ -956,7 +903,6 @@ class QuarkConfig(QuantizationConfig):
scheme_cls: type["QuarkScheme"],
weight_quant_key: QuantKey | None,
activation_quant_key: QuantKey | None,
dynamic_mxfp4_quant: bool = False,
weight_config: dict[str, Any] | None = None,
) -> "QuarkScheme":
"""Construct a Quark scheme selected by get_scheme_cls."""
@@ -996,9 +942,6 @@ class QuarkConfig(QuantizationConfig):
):
kwargs["weight_quant_key"] = weight_quant_key
if scheme_cls is QuarkOCP_MX:
kwargs["dynamic_mxfp4_quant"] = dynamic_mxfp4_quant
scheme = scheme_cls(**kwargs)
# Raise error if device does not support the scheme
@@ -26,10 +26,8 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
)
from vllm.model_executor.parameter import (
GroupQuantScaleParameter,
ModelWeightParameter,
PackedvLLMParameter,
)
from vllm.model_executor.utils import set_weight_attrs
from vllm.platforms import current_platform
from .quark_scheme import QuarkScheme
@@ -46,10 +44,9 @@ class QuarkOCP_MX(QuarkScheme):
self,
weight_quant_key: QuantKey,
activation_quant_key: QuantKey | None,
dynamic_mxfp4_quant: bool = False,
):
super().__init__(weight_quant_key, activation_quant_key)
self.dynamic_mxfp4_quant = dynamic_mxfp4_quant
self.weight_dtype = next(
dtype
for dtype, quant_key in _WEIGHT_QUANT_KEY_MAP.items()
@@ -109,21 +106,9 @@ class QuarkOCP_MX(QuarkScheme):
def get_min_capability(cls) -> int:
return 70
def process_dynamic_mxfp4_weights_after_loading(
self, layer: torch.nn.Module
) -> None:
from aiter.ops.triton.quant import dynamic_mxfp4_quant
w_q, w_s = dynamic_mxfp4_quant(layer.weight)
layer.weight_scale = torch.nn.Parameter(w_s, requires_grad=False)
layer.weight = torch.nn.Parameter(w_q, requires_grad=False)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
layer.weight = torch.nn.Parameter(layer.weight.data, requires_grad=False)
if self.dynamic_mxfp4_quant:
self.process_dynamic_mxfp4_weights_after_loading(layer)
self.ocp_mx_linear.process_weights_after_loading(layer)
def create_weights(
@@ -145,51 +130,36 @@ class QuarkOCP_MX(QuarkScheme):
"tensor-parallel sharding for this layer."
)
if self.dynamic_mxfp4_quant:
weight = ModelWeightParameter(
data=torch.empty(
sum(output_partition_sizes),
input_size_per_partition,
dtype=params_dtype,
),
input_dim=1,
output_dim=0,
weight_loader=weight_loader,
)
output_size_per_partition = sum(output_partition_sizes)
layer.logical_widths = output_partition_sizes
layer.register_parameter("weight", weight)
set_weight_attrs(weight, kwargs)
else:
output_size_per_partition = sum(output_partition_sizes)
layer.logical_widths = output_partition_sizes
# WEIGHT
weight = PackedvLLMParameter(
data=torch.empty(
output_size_per_partition,
self.get_packed_dim(input_size_per_partition, self.weight_dtype),
dtype=torch.uint8,
),
input_dim=1,
output_dim=0,
packed_dim=1,
packed_factor=self.packed_factor,
weight_loader=weight_loader,
)
layer.register_parameter("weight", weight)
# WEIGHT
weight = PackedvLLMParameter(
data=torch.empty(
output_size_per_partition,
self.get_packed_dim(input_size_per_partition, self.weight_dtype),
dtype=torch.uint8,
),
input_dim=1,
output_dim=0,
packed_dim=1,
packed_factor=self.packed_factor,
weight_loader=weight_loader,
)
layer.register_parameter("weight", weight)
# WEIGHT SCALE
weight_scale = GroupQuantScaleParameter(
data=torch.empty(
output_size_per_partition,
input_size_per_partition // OCP_MX_BLOCK_SIZE,
dtype=torch.uint8,
),
input_dim=1,
output_dim=0,
weight_loader=weight_loader,
)
layer.register_parameter("weight_scale", weight_scale)
# WEIGHT SCALE
weight_scale = GroupQuantScaleParameter(
data=torch.empty(
output_size_per_partition,
input_size_per_partition // OCP_MX_BLOCK_SIZE,
dtype=torch.uint8,
),
input_dim=1,
output_dim=0,
weight_loader=weight_loader,
)
layer.register_parameter("weight_scale", weight_scale)
if self.weight_quant_key == kMxfp4Static:
self.ocp_mx_linear = init_mxfp4_linear_kernel(