[AutoRound] Support AutoRound Format Block-Wise FP8 in vLLM (#47434)

Signed-off-by: Zhenzhong1 <zhenzhong.xu@intel.com>
Signed-off-by: Zhenzhong Xu <zhenzhong.xu@intel.com>
Co-authored-by: Kunshang Ji <kunshang.ji@intel.com>
This commit is contained in:
Zhenzhong Xu
2026-08-31 22:56:20 +08:00
committed by GitHub
co-authored by Kunshang Ji
parent dafbef15a1
commit bd575a0d0b
10 changed files with 503 additions and 62 deletions
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from .factory import resolve_scheme
from .inc_fp8_scheme import INCFp8Scheme
from .inc_mxfp4_scheme import INCMxfp4Scheme
from .inc_mxfp8_scheme import INCMxfp8Scheme
from .inc_scheme import INCLinearScheme, INCScheme
@@ -11,6 +12,7 @@ __all__ = [
"INCScheme",
"INCLinearScheme",
"INCMxfp8Scheme",
"INCFp8Scheme",
"INCWna16Scheme",
"INCMxfp4Scheme",
"resolve_scheme",
@@ -9,11 +9,13 @@ if TYPE_CHECKING:
def resolve_scheme(layer_config: "INCLayerConfig") -> "INCScheme":
from .inc_fp8_scheme import INCFp8Scheme
from .inc_mxfp4_scheme import INCMxfp4Scheme
from .inc_mxfp8_scheme import INCMxfp8Scheme
from .inc_wna16_scheme import INCWna16Scheme
scheme_list: list[type[INCScheme]] = [
INCFp8Scheme,
INCMxfp8Scheme,
INCWna16Scheme,
INCMxfp4Scheme,
@@ -0,0 +1,57 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.model_executor.layers.quantization.fp8 import Fp8Config, Fp8LinearMethod
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
get_marlin_input_dtype,
)
from .inc_scheme import INCLinearScheme
class INCFp8LinearScheme(INCLinearScheme):
def __init__(self, prefix: str, weight_block_size: tuple[int, int]) -> None:
self.quant_config = Fp8Config(
is_checkpoint_fp8_serialized=True,
activation_scheme="dynamic",
weight_block_size=list(weight_block_size),
)
self.linear_method = Fp8LinearMethod(self.quant_config)
self.linear_method.marlin_input_dtype = get_marlin_input_dtype(prefix)
@classmethod
def get_min_capability(cls) -> int:
return Fp8Config.get_min_capability()
def create_weights(
self,
layer: torch.nn.Module,
input_size_per_partition: int,
output_partition_sizes: list[int],
input_size: int,
output_size: int,
params_dtype: torch.dtype,
**extra_weight_attrs,
) -> None:
self.linear_method.create_weights(
layer=layer,
input_size_per_partition=input_size_per_partition,
output_partition_sizes=output_partition_sizes,
input_size=input_size,
output_size=output_size,
params_dtype=params_dtype,
**extra_weight_attrs,
)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
self.linear_method.process_weights_after_loading(layer)
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
return self.linear_method.apply(layer, x, bias)
@@ -0,0 +1,39 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TYPE_CHECKING
from ..inc_linear import INCLinearMethod
from .inc_scheme import INCScheme
if TYPE_CHECKING:
import torch
from ..config_parser import INCLayerConfig
from ..inc import INCConfig
class INCFp8Scheme(INCScheme):
@staticmethod
def can_handle(layer_config: "INCLayerConfig") -> bool:
return layer_config.is_fp8_block
def get_linear_method(
self,
config: "INCConfig",
layer: "torch.nn.Module",
prefix: str,
layer_config: "INCLayerConfig",
):
del config, layer
assert isinstance(layer_config.group_size, tuple)
from .inc_fp8_linear import INCFp8LinearScheme
return INCLinearMethod(
INCFp8LinearScheme(
prefix=prefix,
weight_block_size=layer_config.group_size,
)
)
@@ -29,6 +29,11 @@ class INCMxfp4LinearMethod(INCLinearScheme):
"""
def __init__(self, layer_config: "INCLayerConfig") -> None:
if not isinstance(layer_config.group_size, int):
raise ValueError(
"INC MXFP4 requires scalar group_size, "
f"but found group_size={layer_config.group_size!r}."
)
self.group_size = layer_config.group_size or 32
self.kernel = init_mxfp4_linear_kernel(activation_quant_key=kMxfp4Dynamic)
@@ -27,7 +27,7 @@ if TYPE_CHECKING:
class INCWNA16LinearScheme(INCLinearScheme):
def __init__(self, layer_config: "INCLayerConfig") -> None:
self.layer_config = layer_config
self.inner_method = self._build_inner_method()
self.linear_method = self._build_inner_method()
@classmethod
def get_min_capability(cls) -> int:
@@ -43,6 +43,10 @@ class INCWNA16LinearScheme(INCLinearScheme):
)
def _build_gptq_method(self):
assert isinstance(self.layer_config.group_size, int), (
"WNA16 only supports integer group_size."
)
gptq_type_map = {
(4, True): scalar_types.uint4b8,
(8, True): scalar_types.uint8b128,
@@ -82,6 +86,10 @@ class INCWNA16LinearScheme(INCLinearScheme):
)
def _build_awq_method(self):
assert isinstance(self.layer_config.group_size, int), (
"WNA16 only supports integer group_size."
)
awq_type_map = {
4: scalar_types.uint4,
8: scalar_types.uint8,
@@ -135,7 +143,7 @@ class INCWNA16LinearScheme(INCLinearScheme):
params_dtype: "torch.dtype",
**extra_weight_attrs,
) -> None:
return self.inner_method.create_weights(
return self.linear_method.create_weights(
layer=layer,
input_size_per_partition=input_size_per_partition,
output_partition_sizes=output_partition_sizes,
@@ -146,7 +154,7 @@ class INCWNA16LinearScheme(INCLinearScheme):
)
def process_weights_after_loading(self, layer: "torch.nn.Module") -> None:
return self.inner_method.process_weights_after_loading(layer)
return self.linear_method.process_weights_after_loading(layer)
def apply_weights(
self,
@@ -154,7 +162,7 @@ class INCWNA16LinearScheme(INCLinearScheme):
x: "torch.Tensor",
bias: "torch.Tensor | None" = None,
) -> "torch.Tensor":
return self.inner_method.apply(layer, x, bias)
return self.linear_method.apply(layer, x, bias)
class INCXPULinearBase(INCLinearScheme):
@@ -165,6 +173,9 @@ class INCXPULinearBase(INCLinearScheme):
def __init__(self, layer_config: "INCLayerConfig") -> None:
self.weight_bits = layer_config.bits
assert isinstance(layer_config.group_size, int), (
"INCXPULinearBase requires integer group_size."
)
self.group_size = layer_config.group_size
self.sym = layer_config.sym
@@ -41,6 +41,9 @@ def _check_xpu_w4a8_supported(layer_config: "INCLayerConfig", prefix: str) -> No
"which this build of vllm-xpu-kernels does not provide. "
f"Layer: {prefix}."
)
assert isinstance(layer_config.group_size, int), (
"WNA16 only supports integer group_size."
)
if layer_config.group_size <= 0 or layer_config.group_size % 32 != 0:
raise NotImplementedError(
"VLLM_XPU_INC_WNA16_BACKEND=w4a8 requires a group size that is a "
@@ -168,6 +171,10 @@ def _resolve_gptq_moe(layer: "torch.nn.Module", layer_config: "INCLayerConfig"):
check_moe_marlin_supports_layer,
)
assert isinstance(layer_config.group_size, int), (
"WNA16 only supports integer group_size."
)
# AutoGPTQMoEMethod selects its fused-MoE backend through the WNA16 oracle
# (Marlin on CUDA, XPUExpertsWNA16 on XPU). Gate only on the layer-shape
# check like compressed-tensors does; the capability-based
@@ -213,6 +220,10 @@ def _resolve_awq_moe(layer: "torch.nn.Module", layer_config: "INCLayerConfig"):
check_moe_marlin_supports_layer,
)
assert isinstance(layer_config.group_size, int), (
"WNA16 only supports integer group_size."
)
use_marlin = layer_config.bits in (4, 8) and check_moe_marlin_supports_layer(
layer, layer_config.group_size
)