[ROCm][Quark][7/N] Use MXFP4 linear kernel abstraction for emulation backend (#48949)

Signed-off-by: Felix Marty <Felix.Marty@amd.com>
This commit is contained in:
fxmarty-amd
2026-07-31 10:07:15 -07:00
committed by GitHub
parent d87d2ca747
commit e67a2e0a56
17 changed files with 824 additions and 90 deletions
@@ -0,0 +1,12 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from vllm.model_executor.kernels.linear.mxfp6.base import (
MxFp6LinearKernel,
MxFp6LinearLayerConfig,
)
__all__ = [
"MxFp6LinearKernel",
"MxFp6LinearLayerConfig",
]
@@ -0,0 +1,76 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from abc import ABC, abstractmethod
from dataclasses import dataclass
import torch
from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey
@dataclass
class MxFp6LinearLayerConfig:
"""Configuration for an MXFP6 linear layer.
All MXFP6 layers share the same structure: packed uint8 weights (2 FP4 values per
byte) and per-block weight scales (group size 32).
Attributes:
weight_quant_key: Identifies the weight quantization format. Can be
kMxfp6E2M3Static or kMxfp6E3M2Static.
activation_quant_key: Identifies the activation quantization format,
or `None` when activations must not be quantized.
"""
weight_quant_key: QuantKey
activation_quant_key: QuantKey | None = None
class MxFp6LinearKernel(ABC):
"""Base class for MXFP6 quantized linear kernels.
Each subclass implements a specific GEMM backend (CUTLASS, Marlin, etc).
The kernel selection mechanism iterates over registered subclasses in
priority order,calling ``is_supported`` and ``can_implement`` to find the best
match for the current hardware.
"""
def __init__(self, config: MxFp6LinearLayerConfig) -> None:
assert self.can_implement(config)[0]
assert self.is_supported()[0]
self.config = config
@classmethod
@abstractmethod
def is_supported(
cls, compute_capability: int | None = None
) -> tuple[bool, str | None]:
"""Return whether this kernel can run on the current platform."""
raise NotImplementedError
@classmethod
@abstractmethod
def can_implement(cls, config: MxFp6LinearLayerConfig) -> tuple[bool, str | None]:
"""Return whether this kernel can handle *config*."""
raise NotImplementedError
@abstractmethod
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
"""Transform weights into the format required by this kernel.
Called once after checkpoint weights have been loaded onto the
device. Implementations should repack / swizzle / pad weights
and scales in-place on *layer*.
"""
raise NotImplementedError
@abstractmethod
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
"""Run the quantized GEMM."""
raise NotImplementedError
@@ -0,0 +1,93 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable
from functools import partial
import torch
import torch.nn.functional as F
from torch.nn.parameter import Parameter
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
quant_dequant_mxfp4,
)
from vllm.model_executor.layers.quantization.utils.mxfp6_utils import (
dequant_mxfp6,
quant_dequant_mxfp6,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kMxfp4Dynamic,
kMxfp6E2M3Dynamic,
kMxfp6E2M3Static,
kMxfp6E3M2Dynamic,
kMxfp6E3M2Static,
)
from .base import MxFp6LinearKernel, MxFp6LinearLayerConfig
_WEIGHT_DEQUANT_FUNCS: dict[QuantKey, Callable[..., torch.Tensor]] = {
kMxfp6E3M2Static: partial(dequant_mxfp6, quant_dtype="fp6_e3m2"),
kMxfp6E2M3Static: partial(dequant_mxfp6, quant_dtype="fp6_e2m3"),
}
_ACTIVATION_QUANT_DEQUANT_FUNCS: dict[
QuantKey, Callable[[torch.Tensor], torch.Tensor]
] = {
kMxfp4Dynamic: quant_dequant_mxfp4,
kMxfp6E3M2Dynamic: partial(quant_dequant_mxfp6, quant_dtype="fp6_e3m2"),
kMxfp6E2M3Dynamic: partial(quant_dequant_mxfp6, quant_dtype="fp6_e2m3"),
}
class EmulationMxfp6LinearKernel(MxFp6LinearKernel):
"""Software emulation fallback for OCP MXFP4/MXFP6 (dequant + F.linear)."""
def __init__(self, config: MxFp6LinearLayerConfig) -> None:
super().__init__(config)
self.dequant_func = _WEIGHT_DEQUANT_FUNCS[config.weight_quant_key]
if config.activation_quant_key is None:
# no input Q/DQ for weight-only
self.quant_dequant_func: Callable[[torch.Tensor], torch.Tensor] = (
lambda x: x
)
else:
self.quant_dequant_func = _ACTIVATION_QUANT_DEQUANT_FUNCS[
config.activation_quant_key
]
@classmethod
def is_supported(
cls, compute_capability: int | None = None
) -> tuple[bool, str | None]:
return True, None
@classmethod
def can_implement(cls, config: MxFp6LinearLayerConfig) -> tuple[bool, str | None]:
if config.weight_quant_key not in (
kMxfp6E2M3Static,
kMxfp6E3M2Static,
):
return False, "only supports MXFP6 weights"
if config.activation_quant_key not in (
None,
kMxfp4Dynamic,
kMxfp6E3M2Dynamic,
kMxfp6E2M3Dynamic,
):
return False, "only supports MXFP4 or MXFP6 or unquantized activations"
return True, None
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
layer.weight_scale = Parameter(layer.weight_scale.data, requires_grad=False)
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
dq_w = self.dequant_func(layer.weight, layer.weight_scale, x.dtype)
qdq_x = self.quant_dequant_func(x)
return F.linear(qdq_x, dq_w, bias)