[AutoRound] Support AutoRound Format Block-Wise FP8 in vLLM (#47434)
Signed-off-by: Zhenzhong1 <zhenzhong.xu@intel.com> Signed-off-by: Zhenzhong Xu <zhenzhong.xu@intel.com> Co-authored-by: Kunshang Ji <kunshang.ji@intel.com>
This commit is contained in:
co-authored by
Kunshang Ji
parent
dafbef15a1
commit
bd575a0d0b
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from .factory import resolve_scheme
|
||||
from .inc_fp8_scheme import INCFp8Scheme
|
||||
from .inc_mxfp4_scheme import INCMxfp4Scheme
|
||||
from .inc_mxfp8_scheme import INCMxfp8Scheme
|
||||
from .inc_scheme import INCLinearScheme, INCScheme
|
||||
@@ -11,6 +12,7 @@ __all__ = [
|
||||
"INCScheme",
|
||||
"INCLinearScheme",
|
||||
"INCMxfp8Scheme",
|
||||
"INCFp8Scheme",
|
||||
"INCWna16Scheme",
|
||||
"INCMxfp4Scheme",
|
||||
"resolve_scheme",
|
||||
|
||||
@@ -9,11 +9,13 @@ if TYPE_CHECKING:
|
||||
|
||||
|
||||
def resolve_scheme(layer_config: "INCLayerConfig") -> "INCScheme":
|
||||
from .inc_fp8_scheme import INCFp8Scheme
|
||||
from .inc_mxfp4_scheme import INCMxfp4Scheme
|
||||
from .inc_mxfp8_scheme import INCMxfp8Scheme
|
||||
from .inc_wna16_scheme import INCWna16Scheme
|
||||
|
||||
scheme_list: list[type[INCScheme]] = [
|
||||
INCFp8Scheme,
|
||||
INCMxfp8Scheme,
|
||||
INCWna16Scheme,
|
||||
INCMxfp4Scheme,
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.quantization.fp8 import Fp8Config, Fp8LinearMethod
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
|
||||
get_marlin_input_dtype,
|
||||
)
|
||||
|
||||
from .inc_scheme import INCLinearScheme
|
||||
|
||||
|
||||
class INCFp8LinearScheme(INCLinearScheme):
|
||||
def __init__(self, prefix: str, weight_block_size: tuple[int, int]) -> None:
|
||||
self.quant_config = Fp8Config(
|
||||
is_checkpoint_fp8_serialized=True,
|
||||
activation_scheme="dynamic",
|
||||
weight_block_size=list(weight_block_size),
|
||||
)
|
||||
self.linear_method = Fp8LinearMethod(self.quant_config)
|
||||
self.linear_method.marlin_input_dtype = get_marlin_input_dtype(prefix)
|
||||
|
||||
@classmethod
|
||||
def get_min_capability(cls) -> int:
|
||||
return Fp8Config.get_min_capability()
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
input_size_per_partition: int,
|
||||
output_partition_sizes: list[int],
|
||||
input_size: int,
|
||||
output_size: int,
|
||||
params_dtype: torch.dtype,
|
||||
**extra_weight_attrs,
|
||||
) -> None:
|
||||
self.linear_method.create_weights(
|
||||
layer=layer,
|
||||
input_size_per_partition=input_size_per_partition,
|
||||
output_partition_sizes=output_partition_sizes,
|
||||
input_size=input_size,
|
||||
output_size=output_size,
|
||||
params_dtype=params_dtype,
|
||||
**extra_weight_attrs,
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
self.linear_method.process_weights_after_loading(layer)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
return self.linear_method.apply(layer, x, bias)
|
||||
@@ -0,0 +1,39 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from ..inc_linear import INCLinearMethod
|
||||
from .inc_scheme import INCScheme
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import torch
|
||||
|
||||
from ..config_parser import INCLayerConfig
|
||||
from ..inc import INCConfig
|
||||
|
||||
|
||||
class INCFp8Scheme(INCScheme):
|
||||
@staticmethod
|
||||
def can_handle(layer_config: "INCLayerConfig") -> bool:
|
||||
return layer_config.is_fp8_block
|
||||
|
||||
def get_linear_method(
|
||||
self,
|
||||
config: "INCConfig",
|
||||
layer: "torch.nn.Module",
|
||||
prefix: str,
|
||||
layer_config: "INCLayerConfig",
|
||||
):
|
||||
del config, layer
|
||||
|
||||
assert isinstance(layer_config.group_size, tuple)
|
||||
|
||||
from .inc_fp8_linear import INCFp8LinearScheme
|
||||
|
||||
return INCLinearMethod(
|
||||
INCFp8LinearScheme(
|
||||
prefix=prefix,
|
||||
weight_block_size=layer_config.group_size,
|
||||
)
|
||||
)
|
||||
@@ -29,6 +29,11 @@ class INCMxfp4LinearMethod(INCLinearScheme):
|
||||
"""
|
||||
|
||||
def __init__(self, layer_config: "INCLayerConfig") -> None:
|
||||
if not isinstance(layer_config.group_size, int):
|
||||
raise ValueError(
|
||||
"INC MXFP4 requires scalar group_size, "
|
||||
f"but found group_size={layer_config.group_size!r}."
|
||||
)
|
||||
self.group_size = layer_config.group_size or 32
|
||||
self.kernel = init_mxfp4_linear_kernel(activation_quant_key=kMxfp4Dynamic)
|
||||
|
||||
|
||||
@@ -27,7 +27,7 @@ if TYPE_CHECKING:
|
||||
class INCWNA16LinearScheme(INCLinearScheme):
|
||||
def __init__(self, layer_config: "INCLayerConfig") -> None:
|
||||
self.layer_config = layer_config
|
||||
self.inner_method = self._build_inner_method()
|
||||
self.linear_method = self._build_inner_method()
|
||||
|
||||
@classmethod
|
||||
def get_min_capability(cls) -> int:
|
||||
@@ -43,6 +43,10 @@ class INCWNA16LinearScheme(INCLinearScheme):
|
||||
)
|
||||
|
||||
def _build_gptq_method(self):
|
||||
assert isinstance(self.layer_config.group_size, int), (
|
||||
"WNA16 only supports integer group_size."
|
||||
)
|
||||
|
||||
gptq_type_map = {
|
||||
(4, True): scalar_types.uint4b8,
|
||||
(8, True): scalar_types.uint8b128,
|
||||
@@ -82,6 +86,10 @@ class INCWNA16LinearScheme(INCLinearScheme):
|
||||
)
|
||||
|
||||
def _build_awq_method(self):
|
||||
assert isinstance(self.layer_config.group_size, int), (
|
||||
"WNA16 only supports integer group_size."
|
||||
)
|
||||
|
||||
awq_type_map = {
|
||||
4: scalar_types.uint4,
|
||||
8: scalar_types.uint8,
|
||||
@@ -135,7 +143,7 @@ class INCWNA16LinearScheme(INCLinearScheme):
|
||||
params_dtype: "torch.dtype",
|
||||
**extra_weight_attrs,
|
||||
) -> None:
|
||||
return self.inner_method.create_weights(
|
||||
return self.linear_method.create_weights(
|
||||
layer=layer,
|
||||
input_size_per_partition=input_size_per_partition,
|
||||
output_partition_sizes=output_partition_sizes,
|
||||
@@ -146,7 +154,7 @@ class INCWNA16LinearScheme(INCLinearScheme):
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer: "torch.nn.Module") -> None:
|
||||
return self.inner_method.process_weights_after_loading(layer)
|
||||
return self.linear_method.process_weights_after_loading(layer)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
@@ -154,7 +162,7 @@ class INCWNA16LinearScheme(INCLinearScheme):
|
||||
x: "torch.Tensor",
|
||||
bias: "torch.Tensor | None" = None,
|
||||
) -> "torch.Tensor":
|
||||
return self.inner_method.apply(layer, x, bias)
|
||||
return self.linear_method.apply(layer, x, bias)
|
||||
|
||||
|
||||
class INCXPULinearBase(INCLinearScheme):
|
||||
@@ -165,6 +173,9 @@ class INCXPULinearBase(INCLinearScheme):
|
||||
|
||||
def __init__(self, layer_config: "INCLayerConfig") -> None:
|
||||
self.weight_bits = layer_config.bits
|
||||
assert isinstance(layer_config.group_size, int), (
|
||||
"INCXPULinearBase requires integer group_size."
|
||||
)
|
||||
self.group_size = layer_config.group_size
|
||||
|
||||
self.sym = layer_config.sym
|
||||
|
||||
@@ -41,6 +41,9 @@ def _check_xpu_w4a8_supported(layer_config: "INCLayerConfig", prefix: str) -> No
|
||||
"which this build of vllm-xpu-kernels does not provide. "
|
||||
f"Layer: {prefix}."
|
||||
)
|
||||
assert isinstance(layer_config.group_size, int), (
|
||||
"WNA16 only supports integer group_size."
|
||||
)
|
||||
if layer_config.group_size <= 0 or layer_config.group_size % 32 != 0:
|
||||
raise NotImplementedError(
|
||||
"VLLM_XPU_INC_WNA16_BACKEND=w4a8 requires a group size that is a "
|
||||
@@ -168,6 +171,10 @@ def _resolve_gptq_moe(layer: "torch.nn.Module", layer_config: "INCLayerConfig"):
|
||||
check_moe_marlin_supports_layer,
|
||||
)
|
||||
|
||||
assert isinstance(layer_config.group_size, int), (
|
||||
"WNA16 only supports integer group_size."
|
||||
)
|
||||
|
||||
# AutoGPTQMoEMethod selects its fused-MoE backend through the WNA16 oracle
|
||||
# (Marlin on CUDA, XPUExpertsWNA16 on XPU). Gate only on the layer-shape
|
||||
# check like compressed-tensors does; the capability-based
|
||||
@@ -213,6 +220,10 @@ def _resolve_awq_moe(layer: "torch.nn.Module", layer_config: "INCLayerConfig"):
|
||||
check_moe_marlin_supports_layer,
|
||||
)
|
||||
|
||||
assert isinstance(layer_config.group_size, int), (
|
||||
"WNA16 only supports integer group_size."
|
||||
)
|
||||
|
||||
use_marlin = layer_config.bits in (4, 8) and check_moe_marlin_supports_layer(
|
||||
layer, layer_config.group_size
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user