[Quark] Remove quark-specific silent online quantization (#51800)
Signed-off-by: Felix Marty <Felix.Marty@amd.com> Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
@@ -855,7 +855,7 @@ def test_quant_method_dispatch_ignored(default_vllm_config):
|
||||
UnquantizedFusedMoEMethod,
|
||||
)
|
||||
|
||||
dynamic_mxfp4_config = _make_qtensor_config(
|
||||
mxfp4_config = _make_qtensor_config(
|
||||
{
|
||||
"dtype": "fp4",
|
||||
"qscheme": "per_group",
|
||||
@@ -866,24 +866,23 @@ def test_quant_method_dispatch_ignored(default_vllm_config):
|
||||
None,
|
||||
exclude=["self_attn.q_proj", "mlp.down_proj"],
|
||||
)
|
||||
dynamic_mxfp4_config.dynamic_mxfp4_quant = True
|
||||
|
||||
assert dynamic_mxfp4_config.get_quant_method_target(
|
||||
"self_attn.q_proj", LinearBase
|
||||
) == (kMxfp4Static, None, QuarkLinearMethod)
|
||||
attention_proj = TestLinear()
|
||||
assert isinstance(
|
||||
dynamic_mxfp4_config.get_quant_method(attention_proj, "self_attn.q_proj"),
|
||||
QuarkLinearMethod,
|
||||
assert mxfp4_config.get_quant_method_target("self_attn.q_proj", LinearBase) == (
|
||||
None,
|
||||
None,
|
||||
UnquantizedLinearMethod,
|
||||
)
|
||||
assert isinstance(attention_proj.scheme, QuarkOCP_MX)
|
||||
assert attention_proj.scheme.dynamic_mxfp4_quant
|
||||
|
||||
assert dynamic_mxfp4_config.get_quant_method_target(
|
||||
"mlp.down_proj", LinearBase
|
||||
) == (None, None, UnquantizedLinearMethod)
|
||||
assert isinstance(
|
||||
dynamic_mxfp4_config.get_quant_method(TestLinear(), "mlp.down_proj"),
|
||||
mxfp4_config.get_quant_method(TestLinear(), "self_attn.q_proj"),
|
||||
UnquantizedLinearMethod,
|
||||
)
|
||||
|
||||
assert mxfp4_config.get_quant_method_target("mlp.down_proj", LinearBase) == (
|
||||
None,
|
||||
None,
|
||||
UnquantizedLinearMethod,
|
||||
)
|
||||
assert isinstance(
|
||||
mxfp4_config.get_quant_method(TestLinear(), "mlp.down_proj"),
|
||||
UnquantizedLinearMethod,
|
||||
)
|
||||
|
||||
|
||||
@@ -5,7 +5,6 @@ import fnmatch
|
||||
from typing import TYPE_CHECKING, Any, NamedTuple, cast
|
||||
|
||||
import torch
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
@@ -82,10 +81,6 @@ __all__ = ["QuarkLinearMethod"]
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# model_type values that use dynamic MXFP4 re-quantization for
|
||||
# OCP MX fp4 Quark checkpoints
|
||||
_DEEPSEEK_V3_FAMILY_MODEL_TYPES = frozenset({"deepseek_v3", "deepseek_v32"})
|
||||
|
||||
|
||||
class QuantKeyMatch(NamedTuple):
|
||||
matches: bool
|
||||
@@ -119,37 +114,6 @@ class QuarkConfig(QuantizationConfig):
|
||||
self.kv_cache_group = kv_cache_group
|
||||
self.kv_cache_config = kv_cache_config
|
||||
self.pack_method = pack_method
|
||||
# Note : this flag is kept disabled because the overhead of
|
||||
# dynamic mxfp4 quantization negates the performance gains
|
||||
# that come from shifting to mxfp4. It is left here in case
|
||||
# we want to re-enable it in the future.
|
||||
self.dynamic_mxfp4_quant = False
|
||||
|
||||
def maybe_update_config(
|
||||
self,
|
||||
model_name: str,
|
||||
hf_config: PretrainedConfig | None = None,
|
||||
revision: str | None = None,
|
||||
):
|
||||
"""Enable dynamic MXFP4 only for DeepSeek-V3-family fp4 checkpoints."""
|
||||
if hf_config is None:
|
||||
return
|
||||
|
||||
if (
|
||||
getattr(hf_config, "model_type", None)
|
||||
not in _DEEPSEEK_V3_FAMILY_MODEL_TYPES
|
||||
):
|
||||
return
|
||||
|
||||
quant_config = getattr(hf_config, "quantization_config", None)
|
||||
if isinstance(quant_config, dict):
|
||||
quant_dtype = (
|
||||
quant_config.get("global_quant_config", {})
|
||||
.get("weight", {})
|
||||
.get("dtype")
|
||||
)
|
||||
if quant_dtype == "fp4":
|
||||
self.dynamic_mxfp4_quant = True
|
||||
|
||||
def get_linear_method(self) -> "QuarkLinearMethod":
|
||||
return QuarkLinearMethod(self)
|
||||
@@ -201,17 +165,6 @@ class QuarkConfig(QuantizationConfig):
|
||||
self.get_quant_method_target(prefix, type(layer))
|
||||
)
|
||||
|
||||
exclude_layers = cast(list[str], self.quant_config.get("exclude"))
|
||||
is_ignored = should_ignore_layer(
|
||||
prefix,
|
||||
ignore=exclude_layers,
|
||||
fused_mapping=self.packed_modules_mapping,
|
||||
check_children=isinstance(layer, RoutedExperts),
|
||||
)
|
||||
dynamic_mxfp4_quant = (
|
||||
is_ignored and "self_attn" in prefix and self.dynamic_mxfp4_quant
|
||||
)
|
||||
|
||||
if method_cls is UnquantizedFusedMoEMethod:
|
||||
return UnquantizedFusedMoEMethod(layer.moe_config)
|
||||
if method_cls is UnquantizedLinearMethod:
|
||||
@@ -222,7 +175,6 @@ class QuarkConfig(QuantizationConfig):
|
||||
scheme_cls,
|
||||
weight_quant_key=weight_quant_key,
|
||||
activation_quant_key=activation_quant_key,
|
||||
dynamic_mxfp4_quant=dynamic_mxfp4_quant,
|
||||
weight_config=self._find_matched_config(prefix, type(layer)).get(
|
||||
"weight"
|
||||
),
|
||||
@@ -274,11 +226,6 @@ class QuarkConfig(QuantizationConfig):
|
||||
if is_routed_experts:
|
||||
return None, None, UnquantizedFusedMoEMethod
|
||||
if issubclass(layer_type, LinearBase):
|
||||
if "self_attn" in prefix and self.dynamic_mxfp4_quant:
|
||||
weight_quant_key, activation_key, _ = self.get_scheme_cls(
|
||||
layer_type, prefix
|
||||
)
|
||||
return weight_quant_key, activation_key, QuarkLinearMethod
|
||||
return None, None, UnquantizedLinearMethod
|
||||
return None, None, None
|
||||
if issubclass(layer_type, LinearBase):
|
||||
@@ -956,7 +903,6 @@ class QuarkConfig(QuantizationConfig):
|
||||
scheme_cls: type["QuarkScheme"],
|
||||
weight_quant_key: QuantKey | None,
|
||||
activation_quant_key: QuantKey | None,
|
||||
dynamic_mxfp4_quant: bool = False,
|
||||
weight_config: dict[str, Any] | None = None,
|
||||
) -> "QuarkScheme":
|
||||
"""Construct a Quark scheme selected by get_scheme_cls."""
|
||||
@@ -996,9 +942,6 @@ class QuarkConfig(QuantizationConfig):
|
||||
):
|
||||
kwargs["weight_quant_key"] = weight_quant_key
|
||||
|
||||
if scheme_cls is QuarkOCP_MX:
|
||||
kwargs["dynamic_mxfp4_quant"] = dynamic_mxfp4_quant
|
||||
|
||||
scheme = scheme_cls(**kwargs)
|
||||
|
||||
# Raise error if device does not support the scheme
|
||||
|
||||
@@ -26,10 +26,8 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
)
|
||||
from vllm.model_executor.parameter import (
|
||||
GroupQuantScaleParameter,
|
||||
ModelWeightParameter,
|
||||
PackedvLLMParameter,
|
||||
)
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from .quark_scheme import QuarkScheme
|
||||
@@ -46,10 +44,9 @@ class QuarkOCP_MX(QuarkScheme):
|
||||
self,
|
||||
weight_quant_key: QuantKey,
|
||||
activation_quant_key: QuantKey | None,
|
||||
dynamic_mxfp4_quant: bool = False,
|
||||
):
|
||||
super().__init__(weight_quant_key, activation_quant_key)
|
||||
self.dynamic_mxfp4_quant = dynamic_mxfp4_quant
|
||||
|
||||
self.weight_dtype = next(
|
||||
dtype
|
||||
for dtype, quant_key in _WEIGHT_QUANT_KEY_MAP.items()
|
||||
@@ -109,21 +106,9 @@ class QuarkOCP_MX(QuarkScheme):
|
||||
def get_min_capability(cls) -> int:
|
||||
return 70
|
||||
|
||||
def process_dynamic_mxfp4_weights_after_loading(
|
||||
self, layer: torch.nn.Module
|
||||
) -> None:
|
||||
from aiter.ops.triton.quant import dynamic_mxfp4_quant
|
||||
|
||||
w_q, w_s = dynamic_mxfp4_quant(layer.weight)
|
||||
layer.weight_scale = torch.nn.Parameter(w_s, requires_grad=False)
|
||||
layer.weight = torch.nn.Parameter(w_q, requires_grad=False)
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
layer.weight = torch.nn.Parameter(layer.weight.data, requires_grad=False)
|
||||
|
||||
if self.dynamic_mxfp4_quant:
|
||||
self.process_dynamic_mxfp4_weights_after_loading(layer)
|
||||
|
||||
self.ocp_mx_linear.process_weights_after_loading(layer)
|
||||
|
||||
def create_weights(
|
||||
@@ -145,51 +130,36 @@ class QuarkOCP_MX(QuarkScheme):
|
||||
"tensor-parallel sharding for this layer."
|
||||
)
|
||||
|
||||
if self.dynamic_mxfp4_quant:
|
||||
weight = ModelWeightParameter(
|
||||
data=torch.empty(
|
||||
sum(output_partition_sizes),
|
||||
input_size_per_partition,
|
||||
dtype=params_dtype,
|
||||
),
|
||||
input_dim=1,
|
||||
output_dim=0,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
output_size_per_partition = sum(output_partition_sizes)
|
||||
layer.logical_widths = output_partition_sizes
|
||||
|
||||
layer.register_parameter("weight", weight)
|
||||
set_weight_attrs(weight, kwargs)
|
||||
else:
|
||||
output_size_per_partition = sum(output_partition_sizes)
|
||||
layer.logical_widths = output_partition_sizes
|
||||
# WEIGHT
|
||||
weight = PackedvLLMParameter(
|
||||
data=torch.empty(
|
||||
output_size_per_partition,
|
||||
self.get_packed_dim(input_size_per_partition, self.weight_dtype),
|
||||
dtype=torch.uint8,
|
||||
),
|
||||
input_dim=1,
|
||||
output_dim=0,
|
||||
packed_dim=1,
|
||||
packed_factor=self.packed_factor,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
layer.register_parameter("weight", weight)
|
||||
|
||||
# WEIGHT
|
||||
weight = PackedvLLMParameter(
|
||||
data=torch.empty(
|
||||
output_size_per_partition,
|
||||
self.get_packed_dim(input_size_per_partition, self.weight_dtype),
|
||||
dtype=torch.uint8,
|
||||
),
|
||||
input_dim=1,
|
||||
output_dim=0,
|
||||
packed_dim=1,
|
||||
packed_factor=self.packed_factor,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
layer.register_parameter("weight", weight)
|
||||
|
||||
# WEIGHT SCALE
|
||||
weight_scale = GroupQuantScaleParameter(
|
||||
data=torch.empty(
|
||||
output_size_per_partition,
|
||||
input_size_per_partition // OCP_MX_BLOCK_SIZE,
|
||||
dtype=torch.uint8,
|
||||
),
|
||||
input_dim=1,
|
||||
output_dim=0,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
layer.register_parameter("weight_scale", weight_scale)
|
||||
# WEIGHT SCALE
|
||||
weight_scale = GroupQuantScaleParameter(
|
||||
data=torch.empty(
|
||||
output_size_per_partition,
|
||||
input_size_per_partition // OCP_MX_BLOCK_SIZE,
|
||||
dtype=torch.uint8,
|
||||
),
|
||||
input_dim=1,
|
||||
output_dim=0,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
layer.register_parameter("weight_scale", weight_scale)
|
||||
|
||||
if self.weight_quant_key == kMxfp4Static:
|
||||
self.ocp_mx_linear = init_mxfp4_linear_kernel(
|
||||
|
||||
Reference in New Issue
Block a user