[ROCm] Add env flags to disable dynamic MXFP4 quant and enable AITER tuned GEMMs for Attention Projection Layers (#39987)

Signed-off-by: Hemanth Acharya <heachary@amd.com>
This commit is contained in:
Hemanth Acharya
2026-04-29 16:07:46 -07:00
committed by GitHub
parent c2fb013312
commit 6841f5dc77
5 changed files with 27 additions and 102 deletions
@@ -5,7 +5,6 @@ import fnmatch
from typing import TYPE_CHECKING, Any, cast
import torch
from transformers import PretrainedConfig
from vllm.logger import init_logger
from vllm.model_executor.layers.attention import Attention
@@ -45,10 +44,6 @@ __all__ = ["QuarkLinearMethod"]
logger = init_logger(__name__)
# model_type values that use dynamic MXFP4 re-quantization for
# OCP MX fp4 Quark checkpoints
_DEEPSEEK_V3_FAMILY_MODEL_TYPES = frozenset({"deepseek_v3"})
class QuarkConfig(QuantizationConfig):
def __init__(
@@ -65,32 +60,12 @@ class QuarkConfig(QuantizationConfig):
self.kv_cache_group = kv_cache_group
self.kv_cache_config = kv_cache_config
self.pack_method = pack_method
# Note : this flag is kept disabled because the overhead of
# dynamic mxfp4 quantization negates the performance gains
# that come from shifting to mxfp4. It is left here in case
# we want to re-enable it in the future.
self.dynamic_mxfp4_quant = False
def maybe_update_config(
self,
model_name: str,
hf_config: PretrainedConfig | None = None,
revision: str | None = None,
):
"""Enable dynamic MXFP4 only for DeepSeek-V3-family + fp4 Quark checkpoints."""
if (
getattr(hf_config, "model_type", None)
not in _DEEPSEEK_V3_FAMILY_MODEL_TYPES
):
return
quant_config = getattr(hf_config, "quantization_config", None)
if quant_config is not None:
quant_dtype = (
quant_config.get("global_quant_config", {})
.get("weight", {})
.get("dtype")
)
if quant_dtype == "fp4":
self.dynamic_mxfp4_quant = True
def get_linear_method(self) -> "QuarkLinearMethod":
return QuarkLinearMethod(self)