[ROCm] Add env flags to disable dynamic MXFP4 quant and enable AITER tuned GEMMs for Attention Projection Layers (#39987)
Signed-off-by: Hemanth Acharya <heachary@amd.com>
This commit is contained in:
@@ -5,7 +5,6 @@ import fnmatch
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
|
||||
import torch
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
@@ -45,10 +44,6 @@ __all__ = ["QuarkLinearMethod"]
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# model_type values that use dynamic MXFP4 re-quantization for
|
||||
# OCP MX fp4 Quark checkpoints
|
||||
_DEEPSEEK_V3_FAMILY_MODEL_TYPES = frozenset({"deepseek_v3"})
|
||||
|
||||
|
||||
class QuarkConfig(QuantizationConfig):
|
||||
def __init__(
|
||||
@@ -65,32 +60,12 @@ class QuarkConfig(QuantizationConfig):
|
||||
self.kv_cache_group = kv_cache_group
|
||||
self.kv_cache_config = kv_cache_config
|
||||
self.pack_method = pack_method
|
||||
# Note : this flag is kept disabled because the overhead of
|
||||
# dynamic mxfp4 quantization negates the performance gains
|
||||
# that come from shifting to mxfp4. It is left here in case
|
||||
# we want to re-enable it in the future.
|
||||
self.dynamic_mxfp4_quant = False
|
||||
|
||||
def maybe_update_config(
|
||||
self,
|
||||
model_name: str,
|
||||
hf_config: PretrainedConfig | None = None,
|
||||
revision: str | None = None,
|
||||
):
|
||||
"""Enable dynamic MXFP4 only for DeepSeek-V3-family + fp4 Quark checkpoints."""
|
||||
|
||||
if (
|
||||
getattr(hf_config, "model_type", None)
|
||||
not in _DEEPSEEK_V3_FAMILY_MODEL_TYPES
|
||||
):
|
||||
return
|
||||
|
||||
quant_config = getattr(hf_config, "quantization_config", None)
|
||||
if quant_config is not None:
|
||||
quant_dtype = (
|
||||
quant_config.get("global_quant_config", {})
|
||||
.get("weight", {})
|
||||
.get("dtype")
|
||||
)
|
||||
if quant_dtype == "fp4":
|
||||
self.dynamic_mxfp4_quant = True
|
||||
|
||||
def get_linear_method(self) -> "QuarkLinearMethod":
|
||||
return QuarkLinearMethod(self)
|
||||
|
||||
|
||||
@@ -159,6 +159,7 @@ def rocm_unquantized_gemm_impl(
|
||||
and weight.is_contiguous()
|
||||
)
|
||||
)
|
||||
|
||||
if use_skinny_reduce_counting:
|
||||
return ops.wvSplitKrc(x, weight, cu_count, bias)
|
||||
|
||||
@@ -174,17 +175,21 @@ def rocm_unquantized_gemm_impl(
|
||||
and k % 8 == 0
|
||||
)
|
||||
|
||||
if not use_skinny:
|
||||
return torch.nn.functional.linear(x, weight, bias)
|
||||
if use_skinny:
|
||||
x_view = x.reshape(-1, x.size(-1))
|
||||
if m > 8 and 0 < n <= 4:
|
||||
cu_count = num_compute_units()
|
||||
out = ops.wvSplitK(weight, x_view, cu_count, bias)
|
||||
return out.reshape(*x.shape[:-1], weight.shape[0])
|
||||
elif m % 4 == 0 and n == 1 and k <= 8192 and bias is None:
|
||||
out = ops.LLMM1(weight, x_view, 4)
|
||||
return out.reshape(*x.shape[:-1], weight.shape[0])
|
||||
|
||||
if rocm_aiter_ops.is_tgemm_enabled():
|
||||
from aiter.tuned_gemm import tgemm
|
||||
|
||||
return tgemm.mm(x, weight, bias)
|
||||
|
||||
x_view = x.reshape(-1, x.size(-1))
|
||||
if m > 8 and 0 < n <= 4:
|
||||
cu_count = num_compute_units()
|
||||
out = ops.wvSplitK(weight, x_view, cu_count, bias)
|
||||
return out.reshape(*x.shape[:-1], weight.shape[0])
|
||||
elif m % 4 == 0 and n == 1 and k <= 8192 and bias is None:
|
||||
out = ops.LLMM1(weight, x_view, 4)
|
||||
return out.reshape(*x.shape[:-1], weight.shape[0])
|
||||
return torch.nn.functional.linear(x, weight, bias)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user