[ROCm][Quark][7/N] Use MXFP4 linear kernel abstraction for emulation backend (#48949)
Signed-off-by: Felix Marty <Felix.Marty@amd.com>
This commit is contained in:
@@ -0,0 +1,93 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Callable
|
||||
from functools import partial
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from torch.nn.parameter import Parameter
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
|
||||
quant_dequant_mxfp4,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp6_utils import (
|
||||
dequant_mxfp6,
|
||||
quant_dequant_mxfp6,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kMxfp4Dynamic,
|
||||
kMxfp6E2M3Dynamic,
|
||||
kMxfp6E2M3Static,
|
||||
kMxfp6E3M2Dynamic,
|
||||
kMxfp6E3M2Static,
|
||||
)
|
||||
|
||||
from .base import MxFp6LinearKernel, MxFp6LinearLayerConfig
|
||||
|
||||
_WEIGHT_DEQUANT_FUNCS: dict[QuantKey, Callable[..., torch.Tensor]] = {
|
||||
kMxfp6E3M2Static: partial(dequant_mxfp6, quant_dtype="fp6_e3m2"),
|
||||
kMxfp6E2M3Static: partial(dequant_mxfp6, quant_dtype="fp6_e2m3"),
|
||||
}
|
||||
|
||||
_ACTIVATION_QUANT_DEQUANT_FUNCS: dict[
|
||||
QuantKey, Callable[[torch.Tensor], torch.Tensor]
|
||||
] = {
|
||||
kMxfp4Dynamic: quant_dequant_mxfp4,
|
||||
kMxfp6E3M2Dynamic: partial(quant_dequant_mxfp6, quant_dtype="fp6_e3m2"),
|
||||
kMxfp6E2M3Dynamic: partial(quant_dequant_mxfp6, quant_dtype="fp6_e2m3"),
|
||||
}
|
||||
|
||||
|
||||
class EmulationMxfp6LinearKernel(MxFp6LinearKernel):
|
||||
"""Software emulation fallback for OCP MXFP4/MXFP6 (dequant + F.linear)."""
|
||||
|
||||
def __init__(self, config: MxFp6LinearLayerConfig) -> None:
|
||||
super().__init__(config)
|
||||
self.dequant_func = _WEIGHT_DEQUANT_FUNCS[config.weight_quant_key]
|
||||
if config.activation_quant_key is None:
|
||||
# no input Q/DQ for weight-only
|
||||
self.quant_dequant_func: Callable[[torch.Tensor], torch.Tensor] = (
|
||||
lambda x: x
|
||||
)
|
||||
else:
|
||||
self.quant_dequant_func = _ACTIVATION_QUANT_DEQUANT_FUNCS[
|
||||
config.activation_quant_key
|
||||
]
|
||||
|
||||
@classmethod
|
||||
def is_supported(
|
||||
cls, compute_capability: int | None = None
|
||||
) -> tuple[bool, str | None]:
|
||||
return True, None
|
||||
|
||||
@classmethod
|
||||
def can_implement(cls, config: MxFp6LinearLayerConfig) -> tuple[bool, str | None]:
|
||||
if config.weight_quant_key not in (
|
||||
kMxfp6E2M3Static,
|
||||
kMxfp6E3M2Static,
|
||||
):
|
||||
return False, "only supports MXFP6 weights"
|
||||
if config.activation_quant_key not in (
|
||||
None,
|
||||
kMxfp4Dynamic,
|
||||
kMxfp6E3M2Dynamic,
|
||||
kMxfp6E2M3Dynamic,
|
||||
):
|
||||
return False, "only supports MXFP4 or MXFP6 or unquantized activations"
|
||||
|
||||
return True, None
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
layer.weight_scale = Parameter(layer.weight_scale.data, requires_grad=False)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
dq_w = self.dequant_func(layer.weight, layer.weight_scale, x.dtype)
|
||||
qdq_x = self.quant_dequant_func(x)
|
||||
return F.linear(qdq_x, dq_w, bias)
|
||||
Reference in New Issue
Block a user