diff --git a/tests/quantization/test_auto_round.py b/tests/quantization/test_auto_round.py index f3e1eea1d7..df409740fc 100644 --- a/tests/quantization/test_auto_round.py +++ b/tests/quantization/test_auto_round.py @@ -22,11 +22,15 @@ from vllm.model_executor.layers.quantization.inc import INCConfig from vllm.model_executor.layers.quantization.inc.config_parser import INCLayerConfig from vllm.model_executor.layers.quantization.inc.inc_linear import INCLinearMethod from vllm.model_executor.layers.quantization.inc.schemes import ( + INCFp8Scheme, INCMxfp4Scheme, INCMxfp8Scheme, INCWna16Scheme, resolve_scheme, ) +from vllm.model_executor.layers.quantization.inc.schemes.inc_fp8_linear import ( + INCFp8LinearScheme, +) from vllm.model_executor.layers.quantization.inc.schemes.inc_mxfp8_linear import ( INCMxfp8LinearScheme, ) @@ -77,6 +81,14 @@ MODELS = [ ), id="auto_round:llm_compressor_mxfp8", ), + pytest.param( + "INC4AI/Qwen3-8B-fp-w8g128x128-for-ut", + marks=pytest.mark.skipif( + not (current_platform.is_cuda()), + reason="Block-wise AutoRound model only supports CUDA backend for now.", + ), + id="auto_round:block_wise_fp8_on_cuda", + ), ] QWEN3_AUTOROUND_MODELS = [ @@ -128,6 +140,10 @@ MODEL_RUNNER_KWARGS: dict[str, dict[str, Any]] = { "gpu_memory_utilization": 0.8, "max_model_len": 512, }, + "INC4AI/Qwen3-8B-fp-w8g128x128-for-ut": { + "gpu_memory_utilization": 0.8, + "max_model_len": 512, + }, } @@ -190,6 +206,27 @@ def make_layer_config(**overrides) -> INCLayerConfig: return INCLayerConfig(**kwargs) +def make_block_fp8_raw_config(**overrides) -> dict[str, object]: + kwargs = { + "bits": 8, + "group_size": [128, 128], + "sym": True, + "packing_format": "auto_round:fp8", + "quant_method": "auto-round", + "data_type": "fp", + "act_bits": 8, + "act_data_type": "fp", + "act_group_size": 128, + "act_sym": True, + "act_dynamic": True, + "activation_scheme": "dynamic", + "fmt": "e4m3", + "weight_block_size": [128, 128], + } + kwargs.update(overrides) + return kwargs + + def make_qwen3_autoround_config(kind: str) -> INCConfig: configs = { "qwen3_1p7b_mxfp4": { @@ -480,6 +517,52 @@ def test_inc_layer_config_mx_fp_helpers() -> None: assert layer_config.is_mxfp8 is False +def test_inc_config_accepts_block_fp8() -> None: + config = INCConfig.from_config(make_block_fp8_raw_config()) + + assert config.weight_bits == 8 + assert config.group_size == (128, 128) + assert config.sym is True + assert config.data_type == "fp" + assert config.packing_format == "auto_round:fp8" + + +def test_inc_config_rejects_invalid_block_fp8_activation_config() -> None: + with pytest.raises(AssertionError, match="act_dynamic=True"): + INCConfig.from_config(make_block_fp8_raw_config(act_dynamic=False)) + + +def test_inc_layer_config_fp8_block_helper() -> None: + layer_config = INCLayerConfig( + bits=8, + group_size=(128, 128), + sym=True, + packing_format="auto_round:fp8", + backend="auto", + data_type="fp", + quantized=True, + ) + + assert layer_config.is_fp8_block is True + assert layer_config.is_wna16_int is False + + +def test_inc_resolve_scheme_selects_fp8_block() -> None: + layer_config = INCLayerConfig( + bits=8, + group_size=(128, 128), + sym=True, + packing_format="auto_round:fp8", + backend="auto", + data_type="fp", + quantized=True, + ) + + scheme = resolve_scheme(layer_config) + + assert isinstance(scheme, INCFp8Scheme) + + def test_inc_resolve_scheme_selects_wna16() -> None: layer_config = INCLayerConfig( bits=4, @@ -1053,6 +1136,89 @@ def test_inc_linear_method_delegates() -> None: ] +def test_inc_fp8_linear_scheme_delegates_to_fp8_linear_method( + monkeypatch, +) -> None: + captured = {} + + class DummyMethod: + def __init__(self, quant_config) -> None: + captured["quant_config"] = quant_config + captured["method"] = self + self.calls: list[tuple] = [] + self.marlin_input_dtype = None + + def create_weights( + self, + layer, + input_size_per_partition, + output_partition_sizes, + input_size, + output_size, + params_dtype, + **extra_weight_attrs, + ) -> None: + self.calls.append( + ( + "create_weights", + layer, + input_size_per_partition, + output_partition_sizes, + input_size, + output_size, + params_dtype, + extra_weight_attrs, + ) + ) + + def process_weights_after_loading(self, layer) -> None: + self.calls.append(("process_weights_after_loading", layer)) + + def apply(self, layer, x, bias=None): + self.calls.append(("apply", layer, x, bias)) + return "applied" + + monkeypatch.setattr( + "vllm.model_executor.layers.quantization.inc.schemes.inc_fp8_linear." + "Fp8LinearMethod", + DummyMethod, + ) + monkeypatch.setattr( + "vllm.model_executor.layers.quantization.inc.schemes.inc_fp8_linear." + "get_marlin_input_dtype", + lambda _prefix: "fp8-marlin", + ) + + scheme = INCFp8LinearScheme( + prefix="layers.0.mlp.down_proj", + weight_block_size=(128, 128), + ) + layer = torch.nn.Module() + + scheme.create_weights( + layer=layer, + input_size_per_partition=128, + output_partition_sizes=[64, 64], + input_size=128, + output_size=128, + params_dtype=torch.bfloat16, + weight_loader=lambda *args, **kwargs: None, + ) + scheme.process_weights_after_loading(layer) + result = scheme.apply_weights(layer, torch.randn(1, 128), None) + + assert captured["quant_config"].is_checkpoint_fp8_serialized is True + assert captured["quant_config"].activation_scheme == "dynamic" + assert captured["quant_config"].weight_block_size == [128, 128] + assert captured["method"].marlin_input_dtype == "fp8-marlin" + assert result == "applied" + assert [call[0] for call in captured["method"].calls] == [ + "create_weights", + "process_weights_after_loading", + "apply", + ] + + def test_wna16_xpu_prefers_ark_when_available(monkeypatch) -> None: class DummyQuantLinear: pass @@ -1284,7 +1450,7 @@ def test_wna16_linear_gptq_uses_auto_gptq_when_supported(monkeypatch) -> None: scheme = INCWNA16LinearScheme(make_layer_config()) - assert isinstance(scheme.inner_method, DummyMethod) + assert isinstance(scheme.linear_method, DummyMethod) assert isinstance(captured["cfg"], AutoGPTQConfig) assert captured["cfg"].weight_bits == 4 assert captured["cfg"].group_size == 128 diff --git a/vllm/model_executor/layers/quantization/inc/config_parser.py b/vllm/model_executor/layers/quantization/inc/config_parser.py index 7a0d8496b8..ce3e99bb8d 100644 --- a/vllm/model_executor/layers/quantization/inc/config_parser.py +++ b/vllm/model_executor/layers/quantization/inc/config_parser.py @@ -17,7 +17,7 @@ if TYPE_CHECKING: @dataclass(frozen=True) class INCLayerConfig: bits: int - group_size: int + group_size: int | tuple[int, int] sym: bool packing_format: str backend: str @@ -44,6 +44,18 @@ class INCLayerConfig: def is_mxfp8(self) -> bool: return "mx_fp" in self.data_type and self.bits == 8 + @property + def is_fp8_block(self) -> bool: + return ( + self.data_type == "fp" + and self.bits == 8 + and self.sym + and self.quantized + and self.packing_format == "auto_round:fp8" + and isinstance(self.group_size, tuple) + and len(self.group_size) == 2 + ) + class INCConfigParser: def __init__(self, config: "INCConfig") -> None: @@ -63,13 +75,13 @@ class INCConfigParser: def get_layer_config( self, layer: "torch.nn.Module", layer_name: str - ) -> tuple[int, int, bool]: + ) -> tuple[int, int | tuple[int, int], bool]: layer_config = self.resolve(layer, layer_name) return layer_config.bits, layer_config.group_size, layer_config.sym def _resolve_raw( self, layer: "torch.nn.Module", layer_name: str - ) -> tuple[int, int, bool]: + ) -> tuple[int, int | tuple[int, int], bool]: REGEX_SPECIAL_CHARS = set(r"*+?^$()[]{}|\\") def is_explicitly_configured(name: str) -> bool: @@ -91,7 +103,9 @@ class INCConfigParser: continue return False - def get_config(name: str, quantized: bool = True) -> tuple[int, int, bool]: + def get_config( + name: str, quantized: bool = True + ) -> tuple[int, int | tuple[int, int], bool]: if not self._config.extra_config: return ( self._config.weight_bits if quantized else 16, @@ -103,9 +117,11 @@ class INCConfigParser: cfg = self._config.extra_config[name] return ( cfg.get("bits", self._config.weight_bits if quantized else 16), - cfg.get( - "group_size", - self._config.group_size if quantized else -1, + self._normalize_group_size( + cfg.get( + "group_size", + self._config.group_size if quantized else -1, + ) ), cfg.get("sym", self._config.sym if quantized else True), ) @@ -124,9 +140,11 @@ class INCConfigParser: "bits", self._config.weight_bits if quantized else 16, ), - cfg.get( - "group_size", - self._config.group_size if quantized else -1, + self._normalize_group_size( + cfg.get( + "group_size", + self._config.group_size if quantized else -1, + ) ), cfg.get("sym", self._config.sym if quantized else True), ) @@ -194,3 +212,26 @@ class INCConfigParser: ) return get_config(layer_name, quantized) + + @staticmethod + def _normalize_group_size( + group_size: int | list[int] | tuple[int, int], + ) -> int | tuple[int, int]: + """Normalize INC group_size into either an int or a 2-D int tuple.""" + if isinstance(group_size, (list, tuple)): + if len(group_size) != 2 or not all( + isinstance(value, int) for value in group_size + ): + raise ValueError( + "INC block-wise FP8 requires group_size to be a 2-D " + f"integer sequence, but found {group_size!r}." + ) + return (group_size[0], group_size[1]) + + if not isinstance(group_size, int): + raise ValueError( + "INC group_size must be an int or a 2-D integer sequence, " + f"but found {type(group_size).__name__}." + ) + + return group_size diff --git a/vllm/model_executor/layers/quantization/inc/inc.py b/vllm/model_executor/layers/quantization/inc/inc.py index ff8eae3a46..8e5fd49c57 100644 --- a/vllm/model_executor/layers/quantization/inc/inc.py +++ b/vllm/model_executor/layers/quantization/inc/inc.py @@ -34,11 +34,14 @@ class INCConfig(QuantizationConfig): Repo: https://github.com/intel/neural-compressor """ + DEFAULT_INT_PACKING_FORMAT = "auto_round:auto_gptq" + SUPPORTED_BITS = {2, 3, 4, 8} - SUPPORTED_DTYPES = {"int", "mx_fp"} + SUPPORTED_DTYPES = {"int", "mx_fp", "fp"} SUPPORTED_FORMATS = { "auto_round:auto_gptq", "auto_round:auto_awq", + "auto_round:fp8", "auto_round:llm_compressor", } SUPPORTED_BACKENDS = { @@ -49,23 +52,39 @@ class INCConfig(QuantizationConfig): "awq:marlin", "marlin", } + + FP8_BLOCK_BITS = 8 + FP8_BLOCK_DATA_TYPE = "fp" + FP8_BLOCK_PACKING_FORMAT = "auto_round:fp8" + FP8_BLOCK_FMT = "e4m3" + FP8_BLOCK_ACTIVATION_SCHEME = "dynamic" + MXFP8_BITS = 8 MXFP8_GROUP_SIZE = 32 MXFP8_DATA_TYPE = "mx_fp" MXFP8_PACKING_FORMAT = "auto_round:llm_compressor" + @staticmethod + def _normalize_data_type(data_type: str) -> str: + # Keep a small alias for branches that already experimented with "fp8". + return "fp" if data_type == "fp8" else data_type + def __init__( self, weight_bits: int, - group_size: int, + group_size: int | tuple[int, int], sym: bool = True, - packing_format: str = "auto_round:auto_gptq", + packing_format: str = DEFAULT_INT_PACKING_FORMAT, block_name_to_quantize: str | list[str] | None = None, extra_config: dict[str, Any] | None = None, data_type: str = "int", backend: str = "auto", ) -> None: super().__init__() + + data_type = self._normalize_data_type(data_type) + group_size = INCConfigParser._normalize_group_size(group_size) + if weight_bits not in self.SUPPORTED_BITS: raise ValueError( f"Unsupported weight_bits: {weight_bits}, " @@ -124,48 +143,118 @@ class INCConfig(QuantizationConfig): # MXFP4 and MXFP8 share data_type "mx_fp" and differ only by bit width. return self.is_mxfp and self.weight_bits == self.MXFP8_BITS - def _validate_supported_quantization(self) -> None: - if self.is_mxfp8: - assert self.group_size == self.MXFP8_GROUP_SIZE, ( - "INC MXFP8 only supports group_size=32, " - f"but found group_size={self.group_size}." - ) - assert self.sym, "INC MXFP8 only supports symmetric weights." - assert self.packing_format == self.MXFP8_PACKING_FORMAT, ( - "INC MXFP8 only supports " - f"packing_format={self.MXFP8_PACKING_FORMAT!r}, " - f"but found {self.packing_format!r}." - ) - assert self.backend == "auto", ( - "INC MXFP8 only supports backend='auto', " - f"but found backend={self.backend!r}." - ) - elif self.packing_format == self.MXFP8_PACKING_FORMAT and not self.is_mxfp: - raise ValueError( - f"packing_format={self.MXFP8_PACKING_FORMAT!r} requires " - f"an {self.MXFP8_DATA_TYPE!r} data_type." - ) - - def _validate_raw_config(self, config: dict[str, Any]) -> None: - if not self.is_mxfp8: - return - - expected_fields = { - "act_bits": self.MXFP8_BITS, - "act_data_type": self.MXFP8_DATA_TYPE, - "act_group_size": self.MXFP8_GROUP_SIZE, - "act_sym": True, - "act_dynamic": True, - "enable_quanted_input": False, - } + @classmethod + def _check_expected( + cls, + values: dict[str, Any], + expected_fields: dict[str, Any], + quantization_name: str, + ) -> None: for field_name, expected_value in expected_fields.items(): - actual_value = self.get_from_keys_or(config, [field_name], expected_value) + actual_value = cls.get_from_keys_or(values, [field_name], expected_value) assert actual_value == expected_value, ( - "INC MXFP8 only supports " + f"{quantization_name} only supports " f"{field_name}={expected_value!r}, " f"but found {field_name}={actual_value!r}." ) + def _validate_int_quantization(self) -> None: + int_packing_formats = { + self.DEFAULT_INT_PACKING_FORMAT, + "auto_round:auto_awq", + } + assert isinstance(self.group_size, int), ( + "INC int quantization requires scalar group_size, " + f"but found group_size={self.group_size!r}." + ) + assert self.packing_format in int_packing_formats, ( + "INC int quantization only supports " + f"packing_format in {sorted(int_packing_formats)!r}, " + f"but found packing_format={self.packing_format!r}." + ) + + def _validate_fp8_block_quantization(self) -> None: + assert self.weight_bits == self.FP8_BLOCK_BITS, ( + "INC block-wise FP8 only supports bits=8, " + f"but found bits={self.weight_bits}." + ) + assert isinstance(self.group_size, tuple), ( + "INC block-wise FP8 requires a 2-D group_size, " + f"but found group_size={self.group_size!r}." + ) + assert self.sym, "INC block-wise FP8 only supports symmetric weights." + assert self.packing_format == self.FP8_BLOCK_PACKING_FORMAT, ( + "INC block-wise FP8 only supports " + f"packing_format={self.FP8_BLOCK_PACKING_FORMAT!r}, " + f"but found {self.packing_format!r}." + ) + assert self.backend == "auto", ( + "INC block-wise FP8 only supports backend='auto', " + f"but found backend={self.backend!r}." + ) + + def _validate_mxfp_quantization(self) -> None: + assert self.weight_bits in {4, self.MXFP8_BITS}, ( + "INC MXFP only supports bits=4 or bits=8, " + f"but found bits={self.weight_bits}." + ) + assert self.group_size == self.MXFP8_GROUP_SIZE, ( + "INC MXFP only supports group_size=32, " + f"but found group_size={self.group_size}." + ) + assert self.sym, "INC MXFP only supports symmetric weights." + assert self.packing_format == self.MXFP8_PACKING_FORMAT, ( + "INC MXFP only supports " + f"packing_format={self.MXFP8_PACKING_FORMAT!r}, " + f"but found {self.packing_format!r}." + ) + assert self.backend == "auto", ( + "INC MXFP only supports backend='auto', " + f"but found backend={self.backend!r}." + ) + + def _validate_supported_quantization(self) -> None: + if self.data_type == "int": + self._validate_int_quantization() + elif self.data_type == self.FP8_BLOCK_DATA_TYPE: + self._validate_fp8_block_quantization() + elif self.data_type == self.MXFP8_DATA_TYPE: + self._validate_mxfp_quantization() + + def _validate_raw_config(self, config: dict[str, Any]) -> None: + if self.data_type == self.FP8_BLOCK_DATA_TYPE: + assert isinstance(self.group_size, tuple), ( + "INC block-wise FP8 group_size must be a tuple." + ) + + expected_fields = { + "act_bits": self.FP8_BLOCK_BITS, + "act_group_size": self.group_size[0], + "act_data_type": self.FP8_BLOCK_DATA_TYPE, + "act_sym": True, + "act_dynamic": True, + "activation_scheme": self.FP8_BLOCK_ACTIVATION_SCHEME, + "fmt": self.FP8_BLOCK_FMT, + } + self._check_expected( + config, + expected_fields, + "INC block-wise FP8", + ) + elif self.is_mxfp8: + expected_fields = { + "act_bits": self.MXFP8_BITS, + "act_data_type": self.MXFP8_DATA_TYPE, + "act_group_size": self.MXFP8_GROUP_SIZE, + "act_sym": True, + "act_dynamic": True, + } + self._check_expected( + config, + expected_fields, + "INC MXFP8", + ) + @classmethod def get_name(cls) -> QuantizationMethods: return "inc" @@ -184,18 +273,37 @@ class INCConfig(QuantizationConfig): @classmethod def from_config(cls, config: dict[str, Any]) -> "INCConfig": + data_type = cls._normalize_data_type( + cls.get_from_keys_or(config, ["data_type"], "int") + ) + weight_bits = cls.get_from_keys(config, ["bits"]) + group_size = INCConfigParser._normalize_group_size( + cls.get_from_keys(config, ["group_size"]) + ) + packing_format = cls.get_from_keys_or(config, ["packing_format"], None) + + if packing_format is None: + if ( + data_type == cls.FP8_BLOCK_DATA_TYPE + and weight_bits == cls.FP8_BLOCK_BITS + and isinstance(group_size, tuple) + ): + packing_format = cls.FP8_BLOCK_PACKING_FORMAT + elif "mx_fp" in data_type: + packing_format = cls.MXFP8_PACKING_FORMAT + else: + packing_format = cls.DEFAULT_INT_PACKING_FORMAT + quant_config = cls( - weight_bits=cls.get_from_keys(config, ["bits"]), - group_size=cls.get_from_keys(config, ["group_size"]), - sym=cls.get_from_keys(config, ["sym"]), - packing_format=cls.get_from_keys_or( - config, ["packing_format"], "auto_round:auto_gptq" - ), + weight_bits=weight_bits, + group_size=group_size, + sym=cls.get_from_keys_or(config, ["sym"], True), + packing_format=packing_format, block_name_to_quantize=cls.get_from_keys_or( config, ["block_name_to_quantize", "to_quant_block_names"], None ), extra_config=cls.get_from_keys_or(config, ["extra_config"], None), - data_type=cls.get_from_keys_or(config, ["data_type"], "int"), + data_type=data_type, backend=cls.get_from_keys_or(config, ["backend", "vllm_backend"], "auto"), ) quant_config._validate_raw_config(config) @@ -215,7 +323,6 @@ class INCConfig(QuantizationConfig): def get_quant_method(self, layer: torch.nn.Module, prefix: str): from .schemes.factory import resolve_scheme - # Match original: check model.-prefixed names for unquantized layers if prefix and self.extra_config: for layer_name in self.extra_config: if ( diff --git a/vllm/model_executor/layers/quantization/inc/schemes/__init__.py b/vllm/model_executor/layers/quantization/inc/schemes/__init__.py index 1f2d5f4401..9e0c23c95f 100644 --- a/vllm/model_executor/layers/quantization/inc/schemes/__init__.py +++ b/vllm/model_executor/layers/quantization/inc/schemes/__init__.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from .factory import resolve_scheme +from .inc_fp8_scheme import INCFp8Scheme from .inc_mxfp4_scheme import INCMxfp4Scheme from .inc_mxfp8_scheme import INCMxfp8Scheme from .inc_scheme import INCLinearScheme, INCScheme @@ -11,6 +12,7 @@ __all__ = [ "INCScheme", "INCLinearScheme", "INCMxfp8Scheme", + "INCFp8Scheme", "INCWna16Scheme", "INCMxfp4Scheme", "resolve_scheme", diff --git a/vllm/model_executor/layers/quantization/inc/schemes/factory.py b/vllm/model_executor/layers/quantization/inc/schemes/factory.py index 61335e602f..2ee27c116e 100644 --- a/vllm/model_executor/layers/quantization/inc/schemes/factory.py +++ b/vllm/model_executor/layers/quantization/inc/schemes/factory.py @@ -9,11 +9,13 @@ if TYPE_CHECKING: def resolve_scheme(layer_config: "INCLayerConfig") -> "INCScheme": + from .inc_fp8_scheme import INCFp8Scheme from .inc_mxfp4_scheme import INCMxfp4Scheme from .inc_mxfp8_scheme import INCMxfp8Scheme from .inc_wna16_scheme import INCWna16Scheme scheme_list: list[type[INCScheme]] = [ + INCFp8Scheme, INCMxfp8Scheme, INCWna16Scheme, INCMxfp4Scheme, diff --git a/vllm/model_executor/layers/quantization/inc/schemes/inc_fp8_linear.py b/vllm/model_executor/layers/quantization/inc/schemes/inc_fp8_linear.py new file mode 100644 index 0000000000..060c0a82f4 --- /dev/null +++ b/vllm/model_executor/layers/quantization/inc/schemes/inc_fp8_linear.py @@ -0,0 +1,57 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import torch + +from vllm.model_executor.layers.quantization.fp8 import Fp8Config, Fp8LinearMethod +from vllm.model_executor.layers.quantization.utils.marlin_utils import ( + get_marlin_input_dtype, +) + +from .inc_scheme import INCLinearScheme + + +class INCFp8LinearScheme(INCLinearScheme): + def __init__(self, prefix: str, weight_block_size: tuple[int, int]) -> None: + self.quant_config = Fp8Config( + is_checkpoint_fp8_serialized=True, + activation_scheme="dynamic", + weight_block_size=list(weight_block_size), + ) + self.linear_method = Fp8LinearMethod(self.quant_config) + self.linear_method.marlin_input_dtype = get_marlin_input_dtype(prefix) + + @classmethod + def get_min_capability(cls) -> int: + return Fp8Config.get_min_capability() + + def create_weights( + self, + layer: torch.nn.Module, + input_size_per_partition: int, + output_partition_sizes: list[int], + input_size: int, + output_size: int, + params_dtype: torch.dtype, + **extra_weight_attrs, + ) -> None: + self.linear_method.create_weights( + layer=layer, + input_size_per_partition=input_size_per_partition, + output_partition_sizes=output_partition_sizes, + input_size=input_size, + output_size=output_size, + params_dtype=params_dtype, + **extra_weight_attrs, + ) + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + self.linear_method.process_weights_after_loading(layer) + + def apply_weights( + self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: torch.Tensor | None = None, + ) -> torch.Tensor: + return self.linear_method.apply(layer, x, bias) diff --git a/vllm/model_executor/layers/quantization/inc/schemes/inc_fp8_scheme.py b/vllm/model_executor/layers/quantization/inc/schemes/inc_fp8_scheme.py new file mode 100644 index 0000000000..f89048375e --- /dev/null +++ b/vllm/model_executor/layers/quantization/inc/schemes/inc_fp8_scheme.py @@ -0,0 +1,39 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from typing import TYPE_CHECKING + +from ..inc_linear import INCLinearMethod +from .inc_scheme import INCScheme + +if TYPE_CHECKING: + import torch + + from ..config_parser import INCLayerConfig + from ..inc import INCConfig + + +class INCFp8Scheme(INCScheme): + @staticmethod + def can_handle(layer_config: "INCLayerConfig") -> bool: + return layer_config.is_fp8_block + + def get_linear_method( + self, + config: "INCConfig", + layer: "torch.nn.Module", + prefix: str, + layer_config: "INCLayerConfig", + ): + del config, layer + + assert isinstance(layer_config.group_size, tuple) + + from .inc_fp8_linear import INCFp8LinearScheme + + return INCLinearMethod( + INCFp8LinearScheme( + prefix=prefix, + weight_block_size=layer_config.group_size, + ) + ) diff --git a/vllm/model_executor/layers/quantization/inc/schemes/inc_mxfp4_linear.py b/vllm/model_executor/layers/quantization/inc/schemes/inc_mxfp4_linear.py index 04b3cd65cf..a4902fae60 100644 --- a/vllm/model_executor/layers/quantization/inc/schemes/inc_mxfp4_linear.py +++ b/vllm/model_executor/layers/quantization/inc/schemes/inc_mxfp4_linear.py @@ -29,6 +29,11 @@ class INCMxfp4LinearMethod(INCLinearScheme): """ def __init__(self, layer_config: "INCLayerConfig") -> None: + if not isinstance(layer_config.group_size, int): + raise ValueError( + "INC MXFP4 requires scalar group_size, " + f"but found group_size={layer_config.group_size!r}." + ) self.group_size = layer_config.group_size or 32 self.kernel = init_mxfp4_linear_kernel(activation_quant_key=kMxfp4Dynamic) diff --git a/vllm/model_executor/layers/quantization/inc/schemes/inc_wna16_linear.py b/vllm/model_executor/layers/quantization/inc/schemes/inc_wna16_linear.py index 42f053b471..936a155be6 100644 --- a/vllm/model_executor/layers/quantization/inc/schemes/inc_wna16_linear.py +++ b/vllm/model_executor/layers/quantization/inc/schemes/inc_wna16_linear.py @@ -27,7 +27,7 @@ if TYPE_CHECKING: class INCWNA16LinearScheme(INCLinearScheme): def __init__(self, layer_config: "INCLayerConfig") -> None: self.layer_config = layer_config - self.inner_method = self._build_inner_method() + self.linear_method = self._build_inner_method() @classmethod def get_min_capability(cls) -> int: @@ -43,6 +43,10 @@ class INCWNA16LinearScheme(INCLinearScheme): ) def _build_gptq_method(self): + assert isinstance(self.layer_config.group_size, int), ( + "WNA16 only supports integer group_size." + ) + gptq_type_map = { (4, True): scalar_types.uint4b8, (8, True): scalar_types.uint8b128, @@ -82,6 +86,10 @@ class INCWNA16LinearScheme(INCLinearScheme): ) def _build_awq_method(self): + assert isinstance(self.layer_config.group_size, int), ( + "WNA16 only supports integer group_size." + ) + awq_type_map = { 4: scalar_types.uint4, 8: scalar_types.uint8, @@ -135,7 +143,7 @@ class INCWNA16LinearScheme(INCLinearScheme): params_dtype: "torch.dtype", **extra_weight_attrs, ) -> None: - return self.inner_method.create_weights( + return self.linear_method.create_weights( layer=layer, input_size_per_partition=input_size_per_partition, output_partition_sizes=output_partition_sizes, @@ -146,7 +154,7 @@ class INCWNA16LinearScheme(INCLinearScheme): ) def process_weights_after_loading(self, layer: "torch.nn.Module") -> None: - return self.inner_method.process_weights_after_loading(layer) + return self.linear_method.process_weights_after_loading(layer) def apply_weights( self, @@ -154,7 +162,7 @@ class INCWNA16LinearScheme(INCLinearScheme): x: "torch.Tensor", bias: "torch.Tensor | None" = None, ) -> "torch.Tensor": - return self.inner_method.apply(layer, x, bias) + return self.linear_method.apply(layer, x, bias) class INCXPULinearBase(INCLinearScheme): @@ -165,6 +173,9 @@ class INCXPULinearBase(INCLinearScheme): def __init__(self, layer_config: "INCLayerConfig") -> None: self.weight_bits = layer_config.bits + assert isinstance(layer_config.group_size, int), ( + "INCXPULinearBase requires integer group_size." + ) self.group_size = layer_config.group_size self.sym = layer_config.sym diff --git a/vllm/model_executor/layers/quantization/inc/schemes/inc_wna16_scheme.py b/vllm/model_executor/layers/quantization/inc/schemes/inc_wna16_scheme.py index 77888db1eb..b9e93ad657 100644 --- a/vllm/model_executor/layers/quantization/inc/schemes/inc_wna16_scheme.py +++ b/vllm/model_executor/layers/quantization/inc/schemes/inc_wna16_scheme.py @@ -41,6 +41,9 @@ def _check_xpu_w4a8_supported(layer_config: "INCLayerConfig", prefix: str) -> No "which this build of vllm-xpu-kernels does not provide. " f"Layer: {prefix}." ) + assert isinstance(layer_config.group_size, int), ( + "WNA16 only supports integer group_size." + ) if layer_config.group_size <= 0 or layer_config.group_size % 32 != 0: raise NotImplementedError( "VLLM_XPU_INC_WNA16_BACKEND=w4a8 requires a group size that is a " @@ -168,6 +171,10 @@ def _resolve_gptq_moe(layer: "torch.nn.Module", layer_config: "INCLayerConfig"): check_moe_marlin_supports_layer, ) + assert isinstance(layer_config.group_size, int), ( + "WNA16 only supports integer group_size." + ) + # AutoGPTQMoEMethod selects its fused-MoE backend through the WNA16 oracle # (Marlin on CUDA, XPUExpertsWNA16 on XPU). Gate only on the layer-shape # check like compressed-tensors does; the capability-based @@ -213,6 +220,10 @@ def _resolve_awq_moe(layer: "torch.nn.Module", layer_config: "INCLayerConfig"): check_moe_marlin_supports_layer, ) + assert isinstance(layer_config.group_size, int), ( + "WNA16 only supports integer group_size." + ) + use_marlin = layer_config.bits in (4, 8) and check_moe_marlin_supports_layer( layer, layer_config.group_size )