[Quantization][INC]Add MXFP8 Linear Support (#47514)

Signed-off-by: Zhenzhong1 <zhenzhong.xu@intel.com>
Signed-off-by: Zhenzhong Xu <zhenzhong.xu@intel.com>
Co-authored-by: Yi Liu <yi4.liu@intel.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Zhenzhong Xu
2026-07-27 14:26:31 +08:00
committed by GitHub
co-authored by Yi Liu mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent d742856610
commit 5d07e268b1
7 changed files with 322 additions and 9 deletions
@@ -2,12 +2,14 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from .factory import resolve_scheme
from .inc_mxfp8_scheme import INCMxfp8Scheme
from .inc_scheme import INCLinearScheme, INCScheme
from .inc_wna16_scheme import INCWna16Scheme
__all__ = [
"INCScheme",
"INCLinearScheme",
"INCMxfp8Scheme",
"INCWna16Scheme",
"resolve_scheme",
]
@@ -9,9 +9,11 @@ if TYPE_CHECKING:
def resolve_scheme(layer_config: "INCLayerConfig") -> "INCScheme":
from .inc_mxfp8_scheme import INCMxfp8Scheme
from .inc_wna16_scheme import INCWna16Scheme
scheme_list: list[type[INCScheme]] = [
INCMxfp8Scheme,
INCWna16Scheme,
]
@@ -0,0 +1,84 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.model_executor.kernels.linear import init_mxfp8_linear_kernel
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
MXFP8_BLOCK_SIZE,
MXFP8_SCALE_DTYPE,
MXFP8_VALUE_DTYPE,
)
from vllm.model_executor.parameter import (
GroupQuantScaleParameter,
ModelWeightParameter,
)
from .inc_scheme import INCLinearScheme
class INCMxfp8LinearScheme(INCLinearScheme):
def __init__(self) -> None:
self.kernel = init_mxfp8_linear_kernel()
@classmethod
def get_min_capability(cls) -> int:
return 75
def create_weights(
self,
layer: torch.nn.Module,
input_size_per_partition: int,
output_partition_sizes: list[int],
input_size: int,
output_size: int,
params_dtype: torch.dtype,
**extra_weight_attrs,
) -> None:
del input_size, output_size
if input_size_per_partition % MXFP8_BLOCK_SIZE != 0:
raise ValueError(
"INC MXFP8 requires input_size_per_partition "
f"({input_size_per_partition}) to be divisible by "
f"{MXFP8_BLOCK_SIZE}."
)
output_size_per_partition = sum(output_partition_sizes)
layer.logical_widths = output_partition_sizes
layer.input_size_per_partition = input_size_per_partition
layer.output_size_per_partition = output_size_per_partition
layer.params_dtype = params_dtype
weight = ModelWeightParameter(
data=torch.empty(
output_size_per_partition,
input_size_per_partition,
dtype=MXFP8_VALUE_DTYPE,
),
input_dim=1,
output_dim=0,
weight_loader=extra_weight_attrs.get("weight_loader"),
)
layer.register_parameter("weight", weight)
weight_scale = GroupQuantScaleParameter(
data=torch.empty(
output_size_per_partition,
input_size_per_partition // MXFP8_BLOCK_SIZE,
dtype=MXFP8_SCALE_DTYPE,
),
input_dim=1,
output_dim=0,
weight_loader=extra_weight_attrs.get("weight_loader"),
)
layer.register_parameter("weight_scale", weight_scale)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
self.kernel.process_weights_after_loading(layer)
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
return self.kernel.apply_weights(layer, x, bias)
@@ -0,0 +1,31 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TYPE_CHECKING
from ..inc_linear import INCLinearMethod
from .inc_scheme import INCScheme
if TYPE_CHECKING:
import torch
from ..config_parser import INCLayerConfig
from ..inc import INCConfig
class INCMxfp8Scheme(INCScheme):
@staticmethod
def can_handle(layer_config: "INCLayerConfig") -> bool:
return layer_config.is_mxfp8
def get_linear_method(
self,
config: "INCConfig",
layer: "torch.nn.Module",
prefix: str,
layer_config: "INCLayerConfig",
):
del config, layer, prefix, layer_config
from .inc_mxfp8_linear import INCMxfp8LinearScheme
return INCLinearMethod(INCMxfp8LinearScheme())
@@ -166,6 +166,7 @@ class INCXPULinearBase(INCLinearScheme):
def __init__(self, layer_config: "INCLayerConfig") -> None:
self.weight_bits = layer_config.bits
self.group_size = layer_config.group_size
self.sym = layer_config.sym
self.pack_factor = 32 // self.weight_bits
self.is_awq_packed = layer_config.is_awq