[Mini-vLLM] SM86-only cut: models, backends, Rust, tests

- Keep only Qwen3.5, Qwen3.5-MTP and Qwen4Exp model paths; trim the model registry to 41 entries
- Delete idle attention backends (MLA, FlashInfer, ROCm-only, quant backends) and dead GDN CuTeDSL kernels
- Remove inc/quark quant loaders; keep compressed-tensors and Marlin
- Drop the Rust frontend, pooling entrypoints, and idle test suites
- Rebuild kernels for sm_80;sm_86 and strip the extensions
- Verified: full import sweep clean (optional deps only) and a dummy-load Qwen3.5 engine run on RTX 3090 via the Triton GDN path

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-18 19:19:56 +07:00
co-authored by Hinata
parent c5ab42bc0f
commit e61dcb9143
2728 changed files with 159 additions and 950401 deletions
@@ -29,13 +29,10 @@ QuantizationMethods = Literal[
"humming",
"compressed-tensors",
"experts_int8",
"quark",
"moe_wna16",
"torchao",
"inc",
"mxfp4",
"gpt_oss_mxfp4",
"deepseek_v4_fp8",
"online",
# Below are online quant shorthand names (see vllm.config.quantization).
# Listed here as strings to avoid a circular import; kept in sync with
@@ -115,21 +112,6 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]:
# lazy import to avoid triggering `torch.compile` too early
from vllm.config.quantization import _ONLINE_SHORTHANDS
from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig
if current_platform.is_cuda() or current_platform.is_rocm():
# The v4.1 class is the v4 one extended to accept model_type
# "deepseek_v41" and its 32x32 MXFP8 linear layout. V4.1 has
# platform-specific implementations for both CUDA and ROCm.
from vllm.models.deepseek_v41 import (
DeepseekV4FP8Config as DeepseekV41FP8Config,
)
deepseek_config: type[QuantizationConfig] = DeepseekV41FP8Config
else:
from vllm.models.deepseek_v4 import DeepseekV4FP8Config
deepseek_config = DeepseekV4FP8Config
from .auto_awq import AutoAWQConfig
from .auto_gptq import AutoGPTQConfig
@@ -141,7 +123,6 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]:
from .fp8 import Fp8Config
from .fp_quant import FPQuantConfig
from .humming import HummingConfig
from .inc import INCConfig
from .modelopt import (
ModelOptFp8Config,
ModelOptMixedPrecisionConfig,
@@ -169,13 +150,10 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]:
"gptq_marlin": AutoGPTQConfig,
"compressed-tensors": CompressedTensorsConfig,
"experts_int8": ExpertsInt8Config,
"quark": QuarkConfig,
"moe_wna16": MoeWNA16Config,
"torchao": TorchAOConfig,
"inc": INCConfig,
"mxfp4": Mxfp4Config,
"gpt_oss_mxfp4": GptOssMxfp4Config,
"deepseek_v4_fp8": deepseek_config,
"humming": HummingConfig,
"online": OnlineQuantizationConfig,
# MiniMax-style checkpoints tag `quant_method: "mxfp8"`; load with the
@@ -1,6 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from .inc import INCConfig
__all__ = ["INCConfig"]
@@ -1,237 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass
from typing import TYPE_CHECKING
import regex as re
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
if TYPE_CHECKING:
import torch
from .inc import INCConfig
@dataclass(frozen=True)
class INCLayerConfig:
bits: int
group_size: int | tuple[int, int]
sym: bool
packing_format: str
backend: str
data_type: str
quantized: bool
@property
def is_gptq(self) -> bool:
return "gptq" in self.packing_format or "gptq" in self.backend
@property
def is_awq(self) -> bool:
return "awq" in self.packing_format or "awq" in self.backend
@property
def is_wna16_int(self) -> bool:
return self.data_type == "int" and self.quantized
@property
def is_mxfp4(self) -> bool:
return "mx_fp" in self.data_type and self.bits == 4
@property
def is_mxfp8(self) -> bool:
return "mx_fp" in self.data_type and self.bits == 8
@property
def is_fp8_block(self) -> bool:
return (
self.data_type == "fp"
and self.bits == 8
and self.sym
and self.quantized
and self.packing_format == "auto_round:fp8"
and isinstance(self.group_size, tuple)
and len(self.group_size) == 2
)
class INCConfigParser:
def __init__(self, config: "INCConfig") -> None:
self._config = config
def resolve(self, layer: "torch.nn.Module", layer_name: str) -> INCLayerConfig:
bits, group_size, sym = self._resolve_raw(layer, layer_name)
return INCLayerConfig(
bits=bits,
group_size=group_size,
sym=sym,
packing_format=self._config.packing_format,
backend=self._config.backend,
data_type=self._config.data_type,
quantized=bits < 16,
)
def get_layer_config(
self, layer: "torch.nn.Module", layer_name: str
) -> tuple[int, int | tuple[int, int], bool]:
layer_config = self.resolve(layer, layer_name)
return layer_config.bits, layer_config.group_size, layer_config.sym
def _resolve_raw(
self, layer: "torch.nn.Module", layer_name: str
) -> tuple[int, int | tuple[int, int], bool]:
REGEX_SPECIAL_CHARS = set(r"*+?^$()[]{}|\\")
def is_explicitly_configured(name: str) -> bool:
"""Return True if *name* has an explicit entry in extra_config,
either via exact key match or via a regex pattern key."""
if not self._config.extra_config:
return False
if name in self._config.extra_config:
return True
for pattern in self._config.extra_config:
if not isinstance(pattern, str) or not any(
c in REGEX_SPECIAL_CHARS for c in pattern
):
continue
try:
if re.search(re.compile(pattern), name) is not None:
return True
except re.error:
continue
return False
def get_config(
name: str, quantized: bool = True
) -> tuple[int, int | tuple[int, int], bool]:
if not self._config.extra_config:
return (
self._config.weight_bits if quantized else 16,
self._config.group_size if quantized else -1,
self._config.sym if quantized else True,
)
if name in self._config.extra_config:
cfg = self._config.extra_config[name]
return (
cfg.get("bits", self._config.weight_bits if quantized else 16),
self._normalize_group_size(
cfg.get(
"group_size",
self._config.group_size if quantized else -1,
)
),
cfg.get("sym", self._config.sym if quantized else True),
)
regex_special_chars = set(r"*+?^$()[]{}|\\")
for pattern, cfg in self._config.extra_config.items():
if not isinstance(pattern, str) or not any(
c in regex_special_chars for c in pattern
):
continue
try:
if re.search(re.compile(pattern), name) is not None:
return (
cfg.get(
"bits",
self._config.weight_bits if quantized else 16,
),
self._normalize_group_size(
cfg.get(
"group_size",
self._config.group_size if quantized else -1,
)
),
cfg.get("sym", self._config.sym if quantized else True),
)
except re.error:
continue
return (
self._config.weight_bits if quantized else 16,
self._config.group_size if quantized else -1,
self._config.sym if quantized else True,
)
if self._config.extra_config and layer_name in self._config.extra_config:
return get_config(layer_name)
# Suffix match: handle cases where extra_config keys use short names
# (e.g. "lm_head") but the layer_name is fully qualified
# (e.g. "model.language_model.lm_head") due to model nesting.
if self._config.extra_config:
for cfg_key in self._config.extra_config:
if layer_name.endswith(f".{cfg_key}"):
return get_config(cfg_key)
quantized = not isinstance(layer, ParallelLMHead)
if self._config.block_name_to_quantize:
quantized = any(
layer_name.startswith(name)
for name in self._config.block_name_to_quantize
)
if self._config.extra_config and "fusedmoe" in layer.__class__.__name__.lower():
moe_configs = [
get_config(name, quantized)
for name in self._config.extra_config
if name.startswith(layer_name)
]
if moe_configs:
if len(set(moe_configs)) == 1:
return moe_configs[0]
raise ValueError(
f"Fused MoE layer '{layer_name}' requires "
f"consistent quant config for all sub-layers"
)
if self._config.extra_config:
for fusion_key, sub_keys in self._config.packed_modules_mapping.items():
if fusion_key in layer_name and layer_name.count(fusion_key) == 1:
sub_names = [
layer_name.replace(fusion_key, sub_key) for sub_key in sub_keys
]
# Only trigger if at least one sub_name is explicitly
# configured in extra_config (via exact match or regex).
# This prevents false matches when a short fusion_key
# (e.g. "qkv") is merely a substring of a longer layer
# name (e.g. "in_proj_qkvz") and none of the generated
# sub_names are actually configured.
if not any(is_explicitly_configured(n) for n in sub_names):
continue
sub_configs = [get_config(name, quantized) for name in sub_names]
if len(set(sub_configs)) == 1:
return sub_configs[0]
raise ValueError(
f"Fused module '{layer_name}' requires "
f"consistent quant config for {sub_names}"
)
return get_config(layer_name, quantized)
@staticmethod
def _normalize_group_size(
group_size: int | list[int] | tuple[int, int],
) -> int | tuple[int, int]:
"""Normalize INC group_size into either an int or a 2-D int tuple."""
if isinstance(group_size, (list, tuple)):
if len(group_size) != 2 or not all(
isinstance(value, int) for value in group_size
):
raise ValueError(
"INC block-wise FP8 requires group_size to be a 2-D "
f"integer sequence, but found {group_size!r}."
)
return (group_size[0], group_size[1])
if not isinstance(group_size, int):
raise ValueError(
"INC group_size must be an int or a 2-D integer sequence, "
f"but found {type(group_size).__name__}."
)
return group_size
@@ -1,368 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from fractions import Fraction
from typing import TYPE_CHECKING, Any
import torch
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import (
RoutedExperts,
UnquantizedFusedMoEMethod,
)
from vllm.model_executor.layers.linear import (
LinearBase,
UnquantizedLinearMethod,
)
from vllm.model_executor.layers.quantization import (
QuantizationConfig,
QuantizationMethods,
)
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
from .config_parser import INCConfigParser
if TYPE_CHECKING:
from vllm.model_executor.models.utils import WeightsMapper
logger = init_logger(__name__)
class INCConfig(QuantizationConfig):
"""Config class for Intel Neural Compressor (INC).
Repo: https://github.com/intel/neural-compressor
"""
SUPPORTED_BITS = {2, 3, 4, 5, 6, 7, 8}
DEFAULT_INT_PACKING_FORMAT = "auto_round:auto_gptq"
SUPPORTED_DTYPES = {"int", "mx_fp", "fp"}
SUPPORTED_FORMATS = {
"auto_round:auto_gptq",
"auto_round:auto_awq",
"auto_round:fp8",
"auto_round:llm_compressor",
}
SUPPORTED_BACKENDS = {
"auto",
"gptq",
"gptq:marlin",
"awq",
"awq:marlin",
"marlin",
}
FP8_BLOCK_BITS = 8
FP8_BLOCK_DATA_TYPE = "fp"
FP8_BLOCK_PACKING_FORMAT = "auto_round:fp8"
FP8_BLOCK_FMT = "e4m3"
FP8_BLOCK_ACTIVATION_SCHEME = "dynamic"
MXFP8_BITS = 8
MXFP8_GROUP_SIZE = 32
MXFP8_DATA_TYPE = "mx_fp"
MXFP8_PACKING_FORMAT = "auto_round:llm_compressor"
@staticmethod
def _normalize_data_type(data_type: str) -> str:
# Keep a small alias for branches that already experimented with "fp8".
return "fp" if data_type == "fp8" else data_type
def __init__(
self,
weight_bits: int,
group_size: int | tuple[int, int],
sym: bool = True,
packing_format: str = DEFAULT_INT_PACKING_FORMAT,
block_name_to_quantize: str | list[str] | None = None,
extra_config: dict[str, Any] | None = None,
data_type: str = "int",
backend: str = "auto",
) -> None:
super().__init__()
data_type = self._normalize_data_type(data_type)
group_size = INCConfigParser._normalize_group_size(group_size)
if weight_bits not in self.SUPPORTED_BITS:
raise ValueError(
f"Unsupported weight_bits: {weight_bits}, "
f"currently only support {self.SUPPORTED_BITS}."
)
# auto-round mxfp data_type is e.g. "mx_fp4" / "mx_fp4e2m1"; match the
# "mx_fp" family by substring like auto_round.compressors.is_mx_fp.
is_mxfp = "mx_fp" in data_type
if data_type not in self.SUPPORTED_DTYPES and not is_mxfp:
raise ValueError(
f"Unsupported data_type: {data_type}, "
f"currently only support {self.SUPPORTED_DTYPES}."
)
if packing_format not in self.SUPPORTED_FORMATS:
raise ValueError(
f"Unsupported packing_format: {packing_format}, "
f"currently only support {self.SUPPORTED_FORMATS}."
)
if backend not in self.SUPPORTED_BACKENDS:
raise ValueError(
f"Unsupported backend: {backend}, "
f"currently only support {self.SUPPORTED_BACKENDS}."
)
self.weight_bits = weight_bits
self.group_size = group_size
self.sym = sym
self.packing_format = packing_format
self.block_name_to_quantize = (
block_name_to_quantize.split(",")
if isinstance(block_name_to_quantize, str)
else block_name_to_quantize
)
self.extra_config = extra_config
self.data_type = data_type
self.backend = backend
self.pack_factor = Fraction(32, weight_bits)
self.config_parser = INCConfigParser(self)
self._validate_supported_quantization()
def __repr__(self) -> str:
return (
f"INCConfig(weight_bits={self.weight_bits}, "
f"group_size={self.group_size}, sym={self.sym})"
)
@property
def is_mxfp(self) -> bool:
# auto-round mxfp data_type is the "mx_fp" family (e.g. "mx_fp",
# "mx_fp4e2m1"); match by substring like auto_round.compressors.is_mx_fp.
return "mx_fp" in self.data_type
@property
def is_mxfp8(self) -> bool:
# MXFP4 and MXFP8 share data_type "mx_fp" and differ only by bit width.
return self.is_mxfp and self.weight_bits == self.MXFP8_BITS
@classmethod
def _check_expected(
cls,
values: dict[str, Any],
expected_fields: dict[str, Any],
quantization_name: str,
) -> None:
for field_name, expected_value in expected_fields.items():
actual_value = cls.get_from_keys_or(values, [field_name], expected_value)
assert actual_value == expected_value, (
f"{quantization_name} only supports "
f"{field_name}={expected_value!r}, "
f"but found {field_name}={actual_value!r}."
)
def _validate_int_quantization(self) -> None:
int_packing_formats = {
self.DEFAULT_INT_PACKING_FORMAT,
"auto_round:auto_awq",
}
assert isinstance(self.group_size, int), (
"INC int quantization requires scalar group_size, "
f"but found group_size={self.group_size!r}."
)
assert self.packing_format in int_packing_formats, (
"INC int quantization only supports "
f"packing_format in {sorted(int_packing_formats)!r}, "
f"but found packing_format={self.packing_format!r}."
)
def _validate_fp8_block_quantization(self) -> None:
assert self.weight_bits == self.FP8_BLOCK_BITS, (
"INC block-wise FP8 only supports bits=8, "
f"but found bits={self.weight_bits}."
)
assert isinstance(self.group_size, tuple), (
"INC block-wise FP8 requires a 2-D group_size, "
f"but found group_size={self.group_size!r}."
)
assert self.sym, "INC block-wise FP8 only supports symmetric weights."
assert self.packing_format == self.FP8_BLOCK_PACKING_FORMAT, (
"INC block-wise FP8 only supports "
f"packing_format={self.FP8_BLOCK_PACKING_FORMAT!r}, "
f"but found {self.packing_format!r}."
)
assert self.backend == "auto", (
"INC block-wise FP8 only supports backend='auto', "
f"but found backend={self.backend!r}."
)
def _validate_mxfp_quantization(self) -> None:
assert self.weight_bits in {4, self.MXFP8_BITS}, (
"INC MXFP only supports bits=4 or bits=8, "
f"but found bits={self.weight_bits}."
)
assert self.group_size == self.MXFP8_GROUP_SIZE, (
"INC MXFP only supports group_size=32, "
f"but found group_size={self.group_size}."
)
assert self.sym, "INC MXFP only supports symmetric weights."
assert self.packing_format == self.MXFP8_PACKING_FORMAT, (
"INC MXFP only supports "
f"packing_format={self.MXFP8_PACKING_FORMAT!r}, "
f"but found {self.packing_format!r}."
)
assert self.backend == "auto", (
"INC MXFP only supports backend='auto', "
f"but found backend={self.backend!r}."
)
def _validate_supported_quantization(self) -> None:
if self.data_type == "int":
self._validate_int_quantization()
elif self.data_type == self.FP8_BLOCK_DATA_TYPE:
self._validate_fp8_block_quantization()
elif self.data_type == self.MXFP8_DATA_TYPE:
self._validate_mxfp_quantization()
def _validate_raw_config(self, config: dict[str, Any]) -> None:
if self.data_type == self.FP8_BLOCK_DATA_TYPE:
assert isinstance(self.group_size, tuple), (
"INC block-wise FP8 group_size must be a tuple."
)
expected_fields = {
"act_bits": self.FP8_BLOCK_BITS,
"act_group_size": self.group_size[0],
"act_data_type": self.FP8_BLOCK_DATA_TYPE,
"act_sym": True,
"act_dynamic": True,
"activation_scheme": self.FP8_BLOCK_ACTIVATION_SCHEME,
"fmt": self.FP8_BLOCK_FMT,
}
self._check_expected(
config,
expected_fields,
"INC block-wise FP8",
)
elif self.is_mxfp8:
expected_fields = {
"act_bits": self.MXFP8_BITS,
"act_data_type": self.MXFP8_DATA_TYPE,
"act_group_size": self.MXFP8_GROUP_SIZE,
"act_sym": True,
"act_dynamic": True,
}
self._check_expected(
config,
expected_fields,
"INC MXFP8",
)
@classmethod
def get_name(cls) -> QuantizationMethods:
return "inc"
@classmethod
def get_supported_act_dtypes(cls) -> list[torch.dtype]:
return [torch.half, torch.bfloat16]
@classmethod
def get_min_capability(cls) -> int:
return 60
@classmethod
def get_config_filenames(cls) -> list[str]:
return ["quantization_config.json"]
@classmethod
def from_config(cls, config: dict[str, Any]) -> "INCConfig":
data_type = cls._normalize_data_type(
cls.get_from_keys_or(config, ["data_type"], "int")
)
weight_bits = cls.get_from_keys(config, ["bits"])
group_size = INCConfigParser._normalize_group_size(
cls.get_from_keys(config, ["group_size"])
)
packing_format = cls.get_from_keys_or(config, ["packing_format"], None)
if packing_format is None:
if (
data_type == cls.FP8_BLOCK_DATA_TYPE
and weight_bits == cls.FP8_BLOCK_BITS
and isinstance(group_size, tuple)
):
packing_format = cls.FP8_BLOCK_PACKING_FORMAT
elif "mx_fp" in data_type:
packing_format = cls.MXFP8_PACKING_FORMAT
else:
packing_format = cls.DEFAULT_INT_PACKING_FORMAT
quant_config = cls(
weight_bits=weight_bits,
group_size=group_size,
sym=cls.get_from_keys_or(config, ["sym"], True),
packing_format=packing_format,
block_name_to_quantize=cls.get_from_keys_or(
config, ["block_name_to_quantize", "to_quant_block_names"], None
),
extra_config=cls.get_from_keys_or(config, ["extra_config"], None),
data_type=data_type,
backend=cls.get_from_keys_or(config, ["backend", "vllm_backend"], "auto"),
)
quant_config._validate_raw_config(config)
return quant_config
def get_layer_config(self, layer, layer_name: str):
return self.config_parser.get_layer_config(layer, layer_name)
def apply_vllm_mapper(self, hf_to_vllm_mapper: "WeightsMapper"):
if self.block_name_to_quantize is not None:
self.block_name_to_quantize = hf_to_vllm_mapper.apply_list(
self.block_name_to_quantize
)
if self.extra_config is not None:
self.extra_config = hf_to_vllm_mapper.apply_dict(self.extra_config)
def get_quant_method(self, layer: torch.nn.Module, prefix: str):
from .schemes.factory import resolve_scheme
if prefix and self.extra_config:
for layer_name in self.extra_config:
if (
layer_name == prefix or layer_name == f"model.{prefix}"
) and self.extra_config[layer_name].get("bits", 16) >= 16:
if isinstance(layer, (LinearBase, ParallelLMHead)):
return UnquantizedLinearMethod()
if isinstance(layer, RoutedExperts):
return UnquantizedFusedMoEMethod(layer.moe_config)
return None
layer_config = self.config_parser.resolve(layer, prefix)
if not layer_config.quantized:
if isinstance(layer, (LinearBase, ParallelLMHead)):
return UnquantizedLinearMethod()
if isinstance(layer, RoutedExperts):
return UnquantizedFusedMoEMethod(layer.moe_config)
return None
logger.debug(
"[%s] Type: %s, Bits: %s, Group Size: %s, Sym: %s",
prefix,
layer.__class__.__name__,
layer_config.bits,
layer_config.group_size,
layer_config.sym,
)
scheme = resolve_scheme(layer_config)
if isinstance(layer, (LinearBase, ParallelLMHead)):
return scheme.get_linear_method(self, layer, prefix, layer_config)
if isinstance(layer, RoutedExperts):
return scheme.get_moe_method(self, layer, prefix, layer_config)
return None
@classmethod
def override_quantization_method(
cls, hf_quant_cfg, user_quant, hf_config=None
) -> "QuantizationMethods | None":
"""Override the `auto-round` method to `inc`."""
is_auto_round_format = hf_quant_cfg.get("quant_method", None) == "auto-round"
if is_auto_round_format:
return cls.get_name()
return None
@@ -1,47 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TYPE_CHECKING
import torch
from vllm.model_executor.layers.linear import LinearMethodBase
if TYPE_CHECKING:
from .schemes.inc_scheme import INCLinearScheme
class INCLinearMethod(LinearMethodBase):
def __init__(self, scheme: "INCLinearScheme") -> None:
self.scheme = scheme
def create_weights(
self,
layer: torch.nn.Module,
input_size_per_partition: int,
output_partition_sizes: list[int],
input_size: int,
output_size: int,
params_dtype: torch.dtype,
**extra_weight_attrs,
):
return self.scheme.create_weights(
layer=layer,
input_size_per_partition=input_size_per_partition,
output_partition_sizes=output_partition_sizes,
input_size=input_size,
output_size=output_size,
params_dtype=params_dtype,
**extra_weight_attrs,
)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
return self.scheme.process_weights_after_loading(layer)
def apply(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
return self.scheme.apply_weights(layer, x, bias)
@@ -1,19 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from .factory import resolve_scheme
from .inc_fp8_scheme import INCFp8Scheme
from .inc_mxfp4_scheme import INCMxfp4Scheme
from .inc_mxfp8_scheme import INCMxfp8Scheme
from .inc_scheme import INCLinearScheme, INCScheme
from .inc_wna16_scheme import INCWna16Scheme
__all__ = [
"INCScheme",
"INCLinearScheme",
"INCMxfp8Scheme",
"INCFp8Scheme",
"INCWna16Scheme",
"INCMxfp4Scheme",
"resolve_scheme",
]
@@ -1,28 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from ..config_parser import INCLayerConfig
from .inc_scheme import INCScheme
def resolve_scheme(layer_config: "INCLayerConfig") -> "INCScheme":
from .inc_fp8_scheme import INCFp8Scheme
from .inc_mxfp4_scheme import INCMxfp4Scheme
from .inc_mxfp8_scheme import INCMxfp8Scheme
from .inc_wna16_scheme import INCWna16Scheme
scheme_list: list[type[INCScheme]] = [
INCFp8Scheme,
INCMxfp8Scheme,
INCWna16Scheme,
INCMxfp4Scheme,
]
for scheme_cls in scheme_list:
if scheme_cls.can_handle(layer_config):
return scheme_cls()
raise NotImplementedError(f"No INC scheme found for layer config: {layer_config}")
@@ -1,123 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from functools import lru_cache
from typing import Any
import torch
from vllm.logger import init_logger
from vllm.platforms import current_platform
from vllm.utils.torch_utils import direct_register_custom_op
logger = init_logger(__name__)
_OPS_REGISTERED = False
@lru_cache(maxsize=1)
def get_ark_state() -> tuple[bool, str | None, Any | None, Any | None]:
"""Return ARK availability, error details, cached module, and QuantLinear."""
try:
import auto_round_kernel as ark
from auto_round_kernel.qlinear import QuantLinear
logger.info("Successfully imported auto_round_kernel.")
except ImportError as error:
return False, str(error), None, None
if getattr(ark, "cpu_lib", None) is None and getattr(ark, "xpu_lib", None) is None:
return (
False,
"No ARK backend library is available.",
None,
None,
)
logger.info("Successfully loaded auto_round_kernel backend library.")
return True, None, ark, QuantLinear
def _inc_ark_woq_linear_impl(
x: torch.Tensor,
qweight: torch.Tensor,
bias: torch.Tensor | None,
out_features: int,
in_features: int,
group_size: int,
compute_type: str,
weight_type: str,
scale_type: str,
asym: bool,
) -> torch.Tensor:
ark = get_ark_state()[2]
assert ark is not None
return ark.woqgemm_linear(
x,
qweight,
bias,
out_features,
in_features,
group_size,
compute_type,
weight_type,
scale_type,
asym,
)
def _inc_ark_woq_linear_fake(
x: torch.Tensor,
qweight: torch.Tensor,
bias: torch.Tensor | None,
out_features: int,
in_features: int,
group_size: int,
compute_type: str,
weight_type: str,
scale_type: str,
asym: bool,
) -> torch.Tensor:
del qweight
del bias
del in_features
del group_size
del compute_type
del weight_type
del scale_type
del asym
return torch.empty(
(*x.shape[:-1], out_features),
dtype=x.dtype,
device=x.device,
)
class ark_ops:
@staticmethod
def register_ops_once() -> None:
global _OPS_REGISTERED
if _OPS_REGISTERED:
return
is_available, error_str, _, _ = get_ark_state()
if not is_available:
logger.debug(
"Skip registering ark op because ARK is unavailable: %s",
error_str or "unknown error",
)
return
direct_register_custom_op(
op_name="inc_ark_woq_linear",
op_func=_inc_ark_woq_linear_impl,
fake_impl=_inc_ark_woq_linear_fake,
dispatch_key=current_platform.dispatch_key,
)
_OPS_REGISTERED = True
ark_ops.register_ops_once()
__all__ = ["get_ark_state"]
@@ -1,57 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.model_executor.layers.quantization.fp8 import Fp8Config, Fp8LinearMethod
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
get_marlin_input_dtype,
)
from .inc_scheme import INCLinearScheme
class INCFp8LinearScheme(INCLinearScheme):
def __init__(self, prefix: str, weight_block_size: tuple[int, int]) -> None:
self.quant_config = Fp8Config(
is_checkpoint_fp8_serialized=True,
activation_scheme="dynamic",
weight_block_size=list(weight_block_size),
)
self.linear_method = Fp8LinearMethod(self.quant_config)
self.linear_method.marlin_input_dtype = get_marlin_input_dtype(prefix)
@classmethod
def get_min_capability(cls) -> int:
return Fp8Config.get_min_capability()
def create_weights(
self,
layer: torch.nn.Module,
input_size_per_partition: int,
output_partition_sizes: list[int],
input_size: int,
output_size: int,
params_dtype: torch.dtype,
**extra_weight_attrs,
) -> None:
self.linear_method.create_weights(
layer=layer,
input_size_per_partition=input_size_per_partition,
output_partition_sizes=output_partition_sizes,
input_size=input_size,
output_size=output_size,
params_dtype=params_dtype,
**extra_weight_attrs,
)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
self.linear_method.process_weights_after_loading(layer)
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
return self.linear_method.apply(layer, x, bias)
@@ -1,39 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TYPE_CHECKING
from ..inc_linear import INCLinearMethod
from .inc_scheme import INCScheme
if TYPE_CHECKING:
import torch
from ..config_parser import INCLayerConfig
from ..inc import INCConfig
class INCFp8Scheme(INCScheme):
@staticmethod
def can_handle(layer_config: "INCLayerConfig") -> bool:
return layer_config.is_fp8_block
def get_linear_method(
self,
config: "INCConfig",
layer: "torch.nn.Module",
prefix: str,
layer_config: "INCLayerConfig",
):
del config, layer
assert isinstance(layer_config.group_size, tuple)
from .inc_fp8_linear import INCFp8LinearScheme
return INCLinearMethod(
INCFp8LinearScheme(
prefix=prefix,
weight_block_size=layer_config.group_size,
)
)
@@ -1,97 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TYPE_CHECKING, Any
import torch
from torch.nn.parameter import Parameter
from vllm.model_executor.kernels.linear import init_mxfp4_linear_kernel
from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp4Dynamic
from vllm.model_executor.parameter import (
GroupQuantScaleParameter,
ModelWeightParameter,
)
from .inc_scheme import INCLinearScheme
if TYPE_CHECKING:
from ..config_parser import INCLayerConfig
class INCMxfp4LinearMethod(INCLinearScheme):
"""MXFP4 (W4A4) linear method for AutoRound checkpoints.
E2M1 weights packed two per byte with per-group E8M0 scales
(group_size=32, no global scale). The platform kernel is selected by
``init_mxfp4_linear_kernel`` (FlashInfer / Marlin on CUDA, ``fp4_gemm``
on XPU).
"""
def __init__(self, layer_config: "INCLayerConfig") -> None:
if not isinstance(layer_config.group_size, int):
raise ValueError(
"INC MXFP4 requires scalar group_size, "
f"but found group_size={layer_config.group_size!r}."
)
self.group_size = layer_config.group_size or 32
self.kernel = init_mxfp4_linear_kernel(activation_quant_key=kMxfp4Dynamic)
@classmethod
def get_min_capability(cls) -> int:
return 80
def create_weights(
self,
layer: torch.nn.Module,
input_size_per_partition: int,
output_partition_sizes: list[int],
input_size: int,
output_size: int,
params_dtype: torch.dtype,
**extra_weight_attrs: Any,
) -> None:
del input_size, output_size
output_size_per_partition = sum(output_partition_sizes)
layer.logical_widths = output_partition_sizes
layer.input_size_per_partition = input_size_per_partition
layer.output_size_per_partition = output_size_per_partition
layer.params_dtype = params_dtype
weight_loader = extra_weight_attrs.get("weight_loader")
weight = ModelWeightParameter(
data=torch.empty(
output_size_per_partition,
input_size_per_partition // 2,
dtype=torch.uint8,
),
input_dim=1,
output_dim=0,
weight_loader=weight_loader,
)
layer.register_parameter("weight_packed", weight)
weight_scale = GroupQuantScaleParameter(
data=torch.empty(
output_size_per_partition,
input_size_per_partition // self.group_size,
dtype=torch.uint8,
),
input_dim=1,
output_dim=0,
weight_loader=weight_loader,
)
layer.register_parameter("weight_scale", weight_scale)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
layer.weight = Parameter(layer.weight_packed.data, requires_grad=False)
del layer.weight_packed
self.kernel.process_weights_after_loading(layer)
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
return self.kernel.apply_weights(layer, x, bias)
@@ -1,252 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm.config.kernel import FLASHINFER_MOE_EP_BACKENDS
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import (
FusedMoeWeightScaleSupported,
RoutedExperts,
SharedExperts,
)
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEQuantConfig,
mxfp4_moe_quant_config,
)
from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import (
CutlassExpertsMxfp4,
)
from vllm.model_executor.layers.fused_moe.experts.marlin_moe import (
MarlinExperts,
)
from vllm.model_executor.layers.fused_moe.fused_moe_method_base import (
FusedMoEMethodBase,
)
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
B12X_BACKENDS,
Mxfp4MoeBackend,
make_mxfp4_moe_kernel,
make_mxfp4_moe_quant_config,
select_mxfp4_moe_backend,
)
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp4 import (
prepare_moe_fp4_layer_for_marlin,
)
from vllm.model_executor.utils import set_weight_attrs
from vllm.platforms import current_platform
logger = init_logger(__name__)
class INCMxfp4MoEMethod(FusedMoEMethodBase):
"""W4A4 MXFP4 group MoE for AutoRound ``auto_round:llm_compressor`` exports.
Registers the packed MXFP4 layout (uint8 ``weight_packed`` + uint8 E8M0
``weight_scale``, ``group_size=32``) and dispatches the fused MoE to b12x
when requested; otherwise it uses CUTLASS W4A4, XPU W4A4, or Marlin W4A16.
The per-expert ``gate_proj`` / ``up_proj`` / ``down_proj`` tensors are
folded into the stacked ``w13`` / ``w2`` parameters by
``make_expert_params_mapping``.
"""
def __init__(self, moe) -> None:
super().__init__(moe)
if moe.moe_backend in FLASHINFER_MOE_EP_BACKENDS:
raise ValueError(
f"moe_backend={moe.moe_backend!r} is not supported for AutoRound "
"(INC) MXFP4 MoE checkpoints"
)
self.group_size = 32
# Backend selection must stay consistent with the weight preparation in
# process_weights_after_loading / get_fused_moe_quant_config, which use
# three layouts: CUTLASS swizzle (true W4A4), b12x and XPU packed
# passthrough, and Marlin weight-only. XPU dispatch is deferred to the
# shared oracle; b12x dispatch is explicit; every remaining non-CUTLASS
# device falls back to Marlin.
self.use_cutlass_mxfp4 = CutlassExpertsMxfp4._supports_current_device()
self.mxfp4_backend = Mxfp4MoeBackend.MARLIN
self.experts_cls: type[mk.FusedMoEExperts] | None = None
if moe.moe_backend == "b12x":
self.mxfp4_backend, self.experts_cls = select_mxfp4_moe_backend(moe)
self.use_cutlass_mxfp4 = False
elif self.use_cutlass_mxfp4:
self.experts_cls = CutlassExpertsMxfp4
logger.info_once("Using CutlassExpertsMxfp4 for AutoRound MXFP4 MoE")
elif current_platform.is_xpu():
self.mxfp4_backend, self.experts_cls = select_mxfp4_moe_backend(moe)
else:
self.experts_cls = MarlinExperts
logger.info_once(
"Using MarlinExperts (weight-only FP4) for AutoRound MXFP4 MoE"
)
def create_weights(
self,
layer: torch.nn.Module,
num_experts: int,
hidden_size: int,
intermediate_size_per_partition: int,
params_dtype: torch.dtype,
**extra_weight_attrs,
) -> None:
layer.num_experts = num_experts
layer.params_dtype = params_dtype
# gate + up fused on the output dim; two FP4 packed per input byte.
w13_weight = torch.nn.Parameter(
torch.empty(
num_experts,
self.moe.w13_num_shards * intermediate_size_per_partition,
hidden_size // 2,
dtype=torch.uint8,
),
requires_grad=False,
)
layer.register_parameter("w13_weight_packed", w13_weight)
set_weight_attrs(w13_weight, extra_weight_attrs)
w2_weight = torch.nn.Parameter(
torch.empty(
num_experts,
hidden_size,
intermediate_size_per_partition // 2,
dtype=torch.uint8,
),
requires_grad=False,
)
layer.register_parameter("w2_weight_packed", w2_weight)
set_weight_attrs(w2_weight, extra_weight_attrs)
# Per-group E8M0 block scales (group_size=32), stored as uint8.
w13_weight_scale = torch.nn.Parameter(
torch.empty(
num_experts,
self.moe.w13_num_shards * intermediate_size_per_partition,
hidden_size // self.group_size,
dtype=torch.uint8,
),
requires_grad=False,
)
layer.register_parameter("w13_weight_scale", w13_weight_scale)
extra_weight_attrs.update(
{"quant_method": FusedMoeWeightScaleSupported.GROUP.value}
)
set_weight_attrs(w13_weight_scale, extra_weight_attrs)
w2_weight_scale = torch.nn.Parameter(
torch.empty(
num_experts,
hidden_size,
intermediate_size_per_partition // self.group_size,
dtype=torch.uint8,
),
requires_grad=False,
)
layer.register_parameter("w2_weight_scale", w2_weight_scale)
set_weight_attrs(w2_weight_scale, extra_weight_attrs)
def get_fused_moe_quant_config(
self, layer: torch.nn.Module
) -> FusedMoEQuantConfig | None:
if self.use_cutlass_mxfp4:
# W4A4: both weights and activations quantized to MXFP4.
return mxfp4_moe_quant_config(
w1_scale=layer.w13_weight_scale,
w2_scale=layer.w2_weight_scale,
)
# b12x selects W4A8 or W4A16; XPU uses W4A4; Marlin uses W4A16.
return make_mxfp4_moe_quant_config(
mxfp4_backend=self.mxfp4_backend,
w1_scale=layer.w13_weight_scale,
w2_scale=layer.w2_weight_scale,
)
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
layer.w13_weight = torch.nn.Parameter(
layer.w13_weight_packed.data, requires_grad=False
)
delattr(layer, "w13_weight_packed")
layer.w2_weight = torch.nn.Parameter(
layer.w2_weight_packed.data, requires_grad=False
)
delattr(layer, "w2_weight_packed")
if self.use_cutlass_mxfp4:
# Swizzle weight scales from flat checkpoint layout [E, N, K//32]
# to the CUTLASS tiled layout.
from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import (
swizzle_mxfp4_scales,
)
E = layer.w13_weight_scale.shape[0]
w13_N = layer.w13_weight_scale.shape[1]
w13_scale_K = layer.w13_weight_scale.shape[2]
w13_K = w13_scale_K * 32
w2_M = layer.w2_weight_scale.shape[1]
w2_scale_N = layer.w2_weight_scale.shape[2]
w2_N = w2_scale_N * 32
swizzled_w13 = []
swizzled_w2 = []
for e_idx in range(E):
s13 = layer.w13_weight_scale[e_idx]
sw13 = swizzle_mxfp4_scales(s13, w13_N, w13_K)
swizzled_w13.append(sw13.reshape(w13_N, w13_scale_K))
s2 = layer.w2_weight_scale[e_idx]
sw2 = swizzle_mxfp4_scales(s2, w2_M, w2_N)
swizzled_w2.append(sw2.reshape(w2_M, w2_scale_N))
layer.w13_weight_scale = torch.nn.Parameter(
torch.stack(swizzled_w13), requires_grad=False
)
layer.w2_weight_scale = torch.nn.Parameter(
torch.stack(swizzled_w2), requires_grad=False
)
elif self.mxfp4_backend in B12X_BACKENDS or current_platform.is_xpu():
pass
else:
logger.warning_once(
"This device lacks native FP4 compute; using weight-only FP4 "
"via the Marlin kernel, which may reduce performance for "
"compute-heavy workloads."
)
prepare_moe_fp4_layer_for_marlin(layer)
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
if self.moe_quant_config is not None:
assert self.experts_cls is not None
self.moe_kernel = make_mxfp4_moe_kernel(
moe_quant_config=self.moe_quant_config,
moe_config=self.moe,
experts_cls=self.experts_cls,
mxfp4_backend=self.mxfp4_backend,
routing_tables=layer._expert_routing_tables(),
)
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
def apply(
self,
layer: RoutedExperts,
x: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
shared_experts: SharedExperts | None,
shared_experts_input: torch.Tensor | None,
) -> torch.Tensor | UnfinalizedMoEOutput:
assert self.moe_kernel is not None
return self.moe_kernel.apply(
x,
layer.w13_weight,
layer.w2_weight,
topk_weights,
topk_ids,
activation=layer.activation,
global_num_experts=layer.global_num_experts,
expert_map=layer.expert_map,
apply_router_weight_on_input=layer.apply_router_weight_on_input,
shared_experts=shared_experts,
shared_experts_input=shared_experts_input,
)
@@ -1,54 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TYPE_CHECKING
from vllm.logger import init_logger
from ..inc_linear import INCLinearMethod
from .inc_scheme import INCScheme
if TYPE_CHECKING:
import torch
from ..config_parser import INCLayerConfig
from ..inc import INCConfig
logger = init_logger(__name__)
class INCMxfp4Scheme(INCScheme):
"""MXFP4 (W4A4) scheme for AutoRound checkpoints.
Dispatches to :class:`INCMxfp4LinearMethod` for linear layers and
:class:`INCMxfp4MoEMethod` for fused MoE layers; see those classes for the
per-module weight layout and kernel-selection details.
"""
@staticmethod
def can_handle(layer_config: "INCLayerConfig") -> bool:
return layer_config.is_mxfp4
def get_linear_method(
self,
config: "INCConfig",
layer: "torch.nn.Module",
prefix: str,
layer_config: "INCLayerConfig",
):
del config, layer, prefix
from .inc_mxfp4_linear import INCMxfp4LinearMethod
return INCLinearMethod(INCMxfp4LinearMethod(layer_config))
def get_moe_method(
self,
config: "INCConfig",
layer: "torch.nn.Module",
prefix: str,
layer_config: "INCLayerConfig",
):
del config, prefix, layer_config
from .inc_mxfp4_moe import INCMxfp4MoEMethod
return INCMxfp4MoEMethod(layer.moe_config)
@@ -1,83 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.model_executor.kernels.linear import init_mxfp8_linear_kernel
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
MXFP8_BLOCK_SIZE,
MXFP8_SCALE_DTYPE,
MXFP8_VALUE_DTYPE,
)
from vllm.model_executor.parameter import (
GroupQuantScaleParameter,
ModelWeightParameter,
)
from .inc_scheme import INCLinearScheme
class INCMxfp8LinearScheme(INCLinearScheme):
@classmethod
def get_min_capability(cls) -> int:
return 75
def create_weights(
self,
layer: torch.nn.Module,
input_size_per_partition: int,
output_partition_sizes: list[int],
input_size: int,
output_size: int,
params_dtype: torch.dtype,
**extra_weight_attrs,
) -> None:
del input_size, output_size
if input_size_per_partition % MXFP8_BLOCK_SIZE != 0:
raise ValueError(
"INC MXFP8 requires input_size_per_partition "
f"({input_size_per_partition}) to be divisible by "
f"{MXFP8_BLOCK_SIZE}."
)
output_size_per_partition = sum(output_partition_sizes)
layer.logical_widths = output_partition_sizes
layer.input_size_per_partition = input_size_per_partition
layer.output_size_per_partition = output_size_per_partition
layer.params_dtype = params_dtype
weight = ModelWeightParameter(
data=torch.empty(
output_size_per_partition,
input_size_per_partition,
dtype=MXFP8_VALUE_DTYPE,
),
input_dim=1,
output_dim=0,
weight_loader=extra_weight_attrs.get("weight_loader"),
)
layer.register_parameter("weight", weight)
weight_scale = GroupQuantScaleParameter(
data=torch.empty(
output_size_per_partition,
input_size_per_partition // MXFP8_BLOCK_SIZE,
dtype=MXFP8_SCALE_DTYPE,
),
input_dim=1,
output_dim=0,
weight_loader=extra_weight_attrs.get("weight_loader"),
)
layer.register_parameter("weight_scale", weight_scale)
self.kernel = init_mxfp8_linear_kernel(weight_shape=layer.weight.shape)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
self.kernel.process_weights_after_loading(layer)
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
return self.kernel.apply_weights(layer, x, bias)
@@ -1,197 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.model_executor.layers.fused_moe import (
FusedMoEQuantConfig,
FusedMoeWeightScaleSupported,
RoutedExperts,
SharedExperts,
)
from vllm.model_executor.layers.fused_moe.fused_moe_method_base import (
FusedMoEMethodBase,
)
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
convert_to_fp8_moe_kernel_format,
make_fp8_moe_kernel,
make_fp8_moe_quant_config,
)
from vllm.model_executor.layers.fused_moe.oracle.mxfp8 import (
select_mxfp8_moe_backend,
)
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
MXFP8_BLOCK_SIZE,
MXFP8_SCALE_DTYPE,
MXFP8_VALUE_DTYPE,
)
from vllm.model_executor.parameter import ModelWeightParameter
from vllm.model_executor.utils import replace_parameter, set_weight_attrs
class INCMxfp8MoEMethod(FusedMoEMethodBase):
"""W8A8 MXFP8 MoE method for serialized AutoRound checkpoints."""
def __init__(self, moe) -> None:
super().__init__(moe)
self.weight_block_size = [1, MXFP8_BLOCK_SIZE]
self.mxfp8_backend, self.experts_cls = select_mxfp8_moe_backend(config=self.moe)
def create_weights(
self,
layer: RoutedExperts,
num_experts: int,
hidden_size: int,
intermediate_size_per_partition: int,
params_dtype: torch.dtype,
**extra_weight_attrs,
) -> None:
assert layer.intermediate_size_per_partition == intermediate_size_per_partition
assert layer.hidden_size == hidden_size
layer.orig_dtype = params_dtype
if hidden_size % MXFP8_BLOCK_SIZE != 0:
raise ValueError(
f"MXFP8 MoE requires hidden_size divisible by {MXFP8_BLOCK_SIZE}, "
f"got {hidden_size}."
)
if intermediate_size_per_partition % MXFP8_BLOCK_SIZE != 0:
raise ValueError(
"MXFP8 MoE requires intermediate_size_per_partition divisible by "
f"{MXFP8_BLOCK_SIZE}, got {intermediate_size_per_partition}."
)
layer.num_experts = num_experts
weight_loader = extra_weight_attrs.get("weight_loader")
w13_num_shards = 2 if self.moe.is_act_and_mul else 1
w13_weight = ModelWeightParameter(
data=torch.empty(
num_experts,
w13_num_shards * intermediate_size_per_partition,
hidden_size,
dtype=MXFP8_VALUE_DTYPE,
),
input_dim=2,
output_dim=1,
weight_loader=weight_loader,
)
layer.register_parameter("w13_weight", w13_weight)
w2_weight = ModelWeightParameter(
data=torch.empty(
num_experts,
hidden_size,
intermediate_size_per_partition,
dtype=MXFP8_VALUE_DTYPE,
),
input_dim=2,
output_dim=1,
weight_loader=weight_loader,
)
layer.register_parameter("w2_weight", w2_weight)
w13_weight_scale = ModelWeightParameter(
data=torch.empty(
num_experts,
w13_num_shards * intermediate_size_per_partition,
hidden_size // MXFP8_BLOCK_SIZE,
dtype=MXFP8_SCALE_DTYPE,
),
input_dim=2,
output_dim=1,
weight_loader=weight_loader,
)
layer.register_parameter("w13_weight_scale", w13_weight_scale)
w2_weight_scale = ModelWeightParameter(
data=torch.empty(
num_experts,
hidden_size,
intermediate_size_per_partition // MXFP8_BLOCK_SIZE,
dtype=MXFP8_SCALE_DTYPE,
),
input_dim=2,
output_dim=1,
weight_loader=weight_loader,
)
layer.register_parameter("w2_weight_scale", w2_weight_scale)
set_weight_attrs(
layer.w13_weight_scale,
{"quant_method": FusedMoeWeightScaleSupported.BLOCK.value},
)
set_weight_attrs(
layer.w2_weight_scale,
{"quant_method": FusedMoeWeightScaleSupported.BLOCK.value},
)
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
layer.weight_block_size = self.weight_block_size
w13, w2, w13_scale, w2_scale = convert_to_fp8_moe_kernel_format(
fp8_backend=self.mxfp8_backend,
layer=layer,
w13=layer.w13_weight,
w2=layer.w2_weight,
w13_scale=layer.w13_weight_scale,
w2_scale=layer.w2_weight_scale,
w13_input_scale=None,
w2_input_scale=None,
)
replace_parameter(layer, "w13_weight", w13)
replace_parameter(layer, "w2_weight", w2)
replace_parameter(layer, "w13_weight_scale", w13_scale)
replace_parameter(layer, "w2_weight_scale", w2_scale)
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
assert self.moe_quant_config is not None
self.moe_kernel = make_fp8_moe_kernel(
moe_quant_config=self.moe_quant_config,
moe_config=self.moe,
fp8_backend=self.mxfp8_backend,
experts_cls=self.experts_cls,
routing_tables=layer._expert_routing_tables(),
)
def get_fused_moe_quant_config(
self, layer: RoutedExperts
) -> FusedMoEQuantConfig | None:
return make_fp8_moe_quant_config(
fp8_backend=self.mxfp8_backend,
w1_scale=layer.w13_weight_scale,
w2_scale=layer.w2_weight_scale,
a1_scale=None,
a2_scale=None,
block_shape=self.weight_block_size,
swiglu_limit=getattr(layer, "swiglu_limit", None),
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
gemm1_beta=getattr(layer, "swiglu_beta", None),
layer=layer,
)
def apply(
self,
layer: RoutedExperts,
x: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
shared_experts: SharedExperts | None,
shared_experts_input: torch.Tensor | None,
) -> torch.Tensor | UnfinalizedMoEOutput:
assert self.moe_kernel is not None
return self.moe_kernel.apply(
x,
layer.w13_weight,
layer.w2_weight,
topk_weights,
topk_ids,
activation=layer.activation,
global_num_experts=layer.global_num_experts,
expert_map=layer.expert_map,
apply_router_weight_on_input=layer.apply_router_weight_on_input,
shared_experts=shared_experts,
shared_experts_input=shared_experts_input,
)
@@ -1,43 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TYPE_CHECKING
from ..inc_linear import INCLinearMethod
from .inc_scheme import INCScheme
if TYPE_CHECKING:
import torch
from ..config_parser import INCLayerConfig
from ..inc import INCConfig
class INCMxfp8Scheme(INCScheme):
@staticmethod
def can_handle(layer_config: "INCLayerConfig") -> bool:
return layer_config.is_mxfp8
def get_linear_method(
self,
config: "INCConfig",
layer: "torch.nn.Module",
prefix: str,
layer_config: "INCLayerConfig",
):
del config, layer, prefix, layer_config
from .inc_mxfp8_linear import INCMxfp8LinearScheme
return INCLinearMethod(INCMxfp8LinearScheme())
def get_moe_method(
self,
config: "INCConfig",
layer: "torch.nn.Module",
prefix: str,
layer_config: "INCLayerConfig",
):
del config, prefix, layer_config
from .inc_mxfp8_moe import INCMxfp8MoEMethod
return INCMxfp8MoEMethod(layer.moe_config)
@@ -1,88 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from abc import ABC, abstractmethod
from typing import TYPE_CHECKING
if TYPE_CHECKING:
import torch
from vllm.model_executor.layers.fused_moe import FusedMoEMethodBase
from vllm.model_executor.layers.linear import LinearMethodBase
from ..config_parser import INCLayerConfig
from ..inc import INCConfig
class INCScheme(ABC):
"""One class per quant type. Single registration point for the factory.
Each subclass defines:
- can_handle(): when does this scheme apply?
- get_linear_method(): required — how to quantize Linear layers
- get_moe_method(): optional — how to quantize MoE layers
Schemes that don't support MoE/KVCache inherit the default raise.
"""
@staticmethod
@abstractmethod
def can_handle(layer_config: "INCLayerConfig") -> bool:
raise NotImplementedError
@abstractmethod
def get_linear_method(
self,
config: "INCConfig",
layer: "torch.nn.Module",
prefix: str,
layer_config: "INCLayerConfig",
) -> "LinearMethodBase":
raise NotImplementedError
def get_moe_method(
self,
config: "INCConfig",
layer: "torch.nn.Module",
prefix: str,
layer_config: "INCLayerConfig",
) -> "FusedMoEMethodBase | None":
"""Optional. Override if this scheme supports MoE.
Default raises NotImplementedError."""
raise NotImplementedError(
f"{type(self).__name__} does not support MoE layers. "
f"Layer config: {layer_config}"
)
class INCLinearScheme(ABC):
@classmethod
@abstractmethod
def get_min_capability(cls) -> int:
raise NotImplementedError
@abstractmethod
def create_weights(
self,
layer: "torch.nn.Module",
input_size_per_partition: int,
output_partition_sizes: list[int],
input_size: int,
output_size: int,
params_dtype: "torch.dtype",
**extra_weight_attrs,
) -> None:
raise NotImplementedError
@abstractmethod
def process_weights_after_loading(self, layer: "torch.nn.Module") -> None:
raise NotImplementedError
@abstractmethod
def apply_weights(
self,
layer: "torch.nn.Module",
x: "torch.Tensor",
bias: "torch.Tensor | None" = None,
) -> "torch.Tensor":
raise NotImplementedError
@@ -1,112 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from torch.nn.parameter import Parameter
from .inc_wna16_linear import INCXPULinearMethod
class INCXPUW4A8LinearMethod(INCXPULinearMethod):
"""XPU linear method for INC int4 weights with dynamic int8 activations.
Uses the same GPTQ-packed "NT" qweight layout as ``INCXPULinearMethod`` —
``int4_gemm_w4a8`` and ``int4_gemm_w4a16`` accept bit-identical weights, so
no extra repacking is needed. Activations are dynamically quantized
per-token to symmetric int8, which keeps the GEMM on the int8 datapath
instead of upconverting the weights to the activation dtype. That is a win
for compute-bound shapes (large token counts, e.g. diffusion), where w4a16
is dominated by dequantization rather than by weight bandwidth.
Both scale tensors must be float16: the kernel reads them as fp16
regardless of the activation dtype, so bf16 scales are silently
misinterpreted. The activations themselves may be bf16 or fp16.
Opt in with ``VLLM_XPU_INC_WNA16_BACKEND=w4a8``. Whether this beats ARK is
device-dependent, so it is not the default: it was measured faster on B70
(Xe2), where ARK cannot use its XMX int8 path and falls back to fp16.
"""
# int8 activations only pay off once the GEMM is compute-bound. Below this
# token count the per-token quantization overhead dominates and w4a16 is
# faster, so fall back to it per-call.
_MIN_TOKENS_FOR_INT8 = 512
# int4_gemm_w4a8 requires both GEMM dims to be multiples of 8. Tensor
# parallelism shards these, so an aligned model dimension can still yield an
# unaligned partition.
_DIM_ALIGNMENT = 8
def create_weights(
self,
layer: torch.nn.Module,
input_size_per_partition: int,
output_partition_sizes: list[int],
input_size: int,
output_size: int,
params_dtype: torch.dtype,
**extra_weight_attrs,
) -> None:
output_size_per_partition = sum(output_partition_sizes)
unaligned = [
(name, size)
for name, size in (
("input", input_size_per_partition),
("output", output_size_per_partition),
)
if size % self._DIM_ALIGNMENT != 0
]
if unaligned:
raise NotImplementedError(
"VLLM_XPU_INC_WNA16_BACKEND=w4a8 requires partitioned in/out "
f"sizes that are multiples of {self._DIM_ALIGNMENT}, got "
+ ", ".join(f"{name}={size}" for name, size in unaligned)
+ f". Partition shape: ({input_size_per_partition}, "
f"{output_size_per_partition})."
)
super().create_weights(
layer=layer,
input_size_per_partition=input_size_per_partition,
output_partition_sizes=output_partition_sizes,
input_size=input_size,
output_size=output_size,
params_dtype=params_dtype,
**extra_weight_attrs,
)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
super().process_weights_after_loading(layer)
# The kernel reads both scale tensors as fp16; keep an fp16 copy for the
# w4a8 path and leave ``scales`` untouched for the w4a16 fallback.
layer.scales_fp16 = Parameter(
layer.scales.data.to(torch.float16), requires_grad=False
)
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
reshaped_x = x.reshape(-1, x.shape[-1])
if reshaped_x.shape[0] < self._MIN_TOKENS_FOR_INT8:
return super().apply_weights(layer, x, bias)
from vllm._xpu_ops import xpu_ops as ops
out_shape = x.shape[:-1] + (layer.qweight.shape[1],)
quant_x, x_scale, x_zero = ops.dynamic_per_token_int8_quant_ref(
reshaped_x, True, 8
)
out = torch.ops._xpu_C.int4_gemm_w4a8(
quant_x,
x_scale.to(torch.float16),
x_zero,
layer.qweight,
layer.scales_fp16,
layer.qzeros,
self.group_size,
None, # Retained by the external XPU op ABI.
bias,
)
return out.to(x.dtype).reshape(out_shape)
@@ -1,460 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TYPE_CHECKING, Any
import torch
from torch.nn.parameter import Parameter
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQConfig
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
check_marlin_supported,
)
from vllm.model_executor.parameter import (
GroupQuantScaleParameter,
PackedvLLMParameter,
)
from vllm.scalar_type import scalar_types
from .inc_scheme import INCLinearScheme
if TYPE_CHECKING:
from ..config_parser import INCLayerConfig
class INCWNA16LinearScheme(INCLinearScheme):
def __init__(self, layer_config: "INCLayerConfig") -> None:
self.layer_config = layer_config
self.linear_method = self._build_inner_method()
@classmethod
def get_min_capability(cls) -> int:
return 60
def _build_inner_method(self):
if self.layer_config.is_gptq:
return self._build_gptq_method()
if self.layer_config.is_awq:
return self._build_awq_method()
raise NotImplementedError(
f"WNA16 linear scheme does not support {self.layer_config}"
)
def _build_gptq_method(self):
assert isinstance(self.layer_config.group_size, int), (
"WNA16 only supports integer group_size."
)
gptq_type_map = {
(4, True): scalar_types.uint4b8,
(8, True): scalar_types.uint8b128,
}
use_marlin = (
self.layer_config.backend == "auto" or "marlin" in self.layer_config.backend
) and (self.layer_config.bits, self.layer_config.sym) in gptq_type_map
if use_marlin:
use_marlin = check_marlin_supported(
gptq_type_map[(self.layer_config.bits, self.layer_config.sym)],
self.layer_config.group_size,
has_zp=not self.layer_config.sym,
)
if use_marlin:
from vllm.model_executor.layers.quantization.auto_gptq import (
AutoGPTQLinearMethod,
)
return AutoGPTQLinearMethod(
AutoGPTQConfig(
weight_bits=self.layer_config.bits,
group_size=self.layer_config.group_size,
desc_act=False,
is_sym=self.layer_config.sym,
lm_head_quantized=False,
dynamic={},
full_config={},
)
)
raise NotImplementedError(
f"INC quantization with bits={self.layer_config.bits}, "
f"sym={self.layer_config.sym} is not supported. "
"Only 4-bit and 8-bit symmetric quantization is supported "
"with Marlin kernels."
)
def _build_awq_method(self):
assert isinstance(self.layer_config.group_size, int), (
"WNA16 only supports integer group_size."
)
awq_type_map = {
4: scalar_types.uint4,
8: scalar_types.uint8,
}
use_marlin = (
self.layer_config.backend == "auto" or "marlin" in self.layer_config.backend
) and self.layer_config.bits in awq_type_map
if use_marlin:
use_marlin = check_marlin_supported(
awq_type_map[self.layer_config.bits],
self.layer_config.group_size,
not self.layer_config.sym,
)
if use_marlin:
from vllm.model_executor.layers.quantization.auto_awq import (
AutoAWQMarlinLinearMethod,
)
return AutoAWQMarlinLinearMethod(
AutoAWQConfig(
weight_bits=self.layer_config.bits,
group_size=self.layer_config.group_size,
zero_point=not self.layer_config.sym,
lm_head_quantized=False,
modules_to_not_convert=[],
full_config={},
)
)
from vllm.model_executor.layers.quantization.auto_awq import (
AutoAWQLinearMethod,
)
return AutoAWQLinearMethod(
AutoAWQConfig(
weight_bits=self.layer_config.bits,
group_size=self.layer_config.group_size,
zero_point=not self.layer_config.sym,
lm_head_quantized=False,
)
)
def create_weights(
self,
layer: "torch.nn.Module",
input_size_per_partition: int,
output_partition_sizes: list[int],
input_size: int,
output_size: int,
params_dtype: "torch.dtype",
**extra_weight_attrs,
) -> None:
return self.linear_method.create_weights(
layer=layer,
input_size_per_partition=input_size_per_partition,
output_partition_sizes=output_partition_sizes,
input_size=input_size,
output_size=output_size,
params_dtype=params_dtype,
**extra_weight_attrs,
)
def process_weights_after_loading(self, layer: "torch.nn.Module") -> None:
return self.linear_method.process_weights_after_loading(layer)
def apply_weights(
self,
layer: "torch.nn.Module",
x: "torch.Tensor",
bias: "torch.Tensor | None" = None,
) -> "torch.Tensor":
return self.linear_method.apply(layer, x, bias)
class INCXPULinearBase(INCLinearScheme):
# AWQ packs nibbles within each int32 in the order [0, 2, 4, 6, 1, 3, 5, 7];
# this permutation undoes that ordering so values can be repacked in
# standard sequential (GPTQ) order.
_REVERSE_AWQ_PACK_ORDER = [0, 4, 1, 5, 2, 6, 3, 7]
def __init__(self, layer_config: "INCLayerConfig") -> None:
self.weight_bits = layer_config.bits
assert isinstance(layer_config.group_size, int), (
"INCXPULinearBase requires integer group_size."
)
self.group_size = layer_config.group_size
self.sym = layer_config.sym
self.pack_factor = 32 // self.weight_bits
self.is_awq_packed = layer_config.is_awq
@classmethod
def get_min_capability(cls) -> int:
return 0
def _create_inc_weights(
self,
layer: torch.nn.Module,
input_size_per_partition: int,
output_partition_sizes: list[int],
params_dtype: torch.dtype,
weight_loader: Any,
) -> None:
output_size_per_partition = sum(output_partition_sizes)
scales_and_zp_size = input_size_per_partition // self.group_size
if self.is_awq_packed:
# AWQ: qweight [in, out // pack_factor] packed along output dim
qweight = PackedvLLMParameter(
data=torch.empty(
input_size_per_partition,
output_size_per_partition // self.pack_factor,
dtype=torch.int32,
),
input_dim=0,
output_dim=1,
packed_dim=1,
packed_factor=self.pack_factor,
weight_loader=weight_loader,
)
else:
# GPTQ: qweight [in // pack_factor, out] packed along input dim
qweight = PackedvLLMParameter(
data=torch.empty(
input_size_per_partition // self.pack_factor,
output_size_per_partition,
dtype=torch.int32,
),
input_dim=0,
output_dim=1,
packed_dim=0,
packed_factor=self.pack_factor,
weight_loader=weight_loader,
)
scales = GroupQuantScaleParameter(
data=torch.empty(
scales_and_zp_size,
output_size_per_partition,
dtype=params_dtype,
),
input_dim=0,
output_dim=1,
weight_loader=weight_loader,
)
# Both AWQ and GPTQ checkpoints store qzeros with this shape; for
# symmetric quantization the values are ignored downstream.
qzeros = PackedvLLMParameter(
data=torch.empty(
scales_and_zp_size,
output_size_per_partition // self.pack_factor,
dtype=torch.int32,
),
input_dim=0,
output_dim=1,
packed_dim=1,
packed_factor=self.pack_factor,
weight_loader=weight_loader,
)
layer.register_parameter("qweight", qweight)
layer.register_parameter("scales", scales)
layer.register_parameter("qzeros", qzeros)
def _convert_awq_qweight_to_gptq(self, qw: torch.Tensor) -> torch.Tensor:
"""Convert AWQ qweight [K, N // pf] to GPTQ qweight [K // pf, N].
AWQ packs along the output dim with a non-standard nibble order; GPTQ
packs along the input dim with sequential nibble order. The conversion
is lossless — it only reshuffles bits.
"""
size_bits = self.weight_bits
pack_factor = self.pack_factor
mask = (1 << size_bits) - 1
device = qw.device
reverse_order = torch.tensor(
self._REVERSE_AWQ_PACK_ORDER, dtype=torch.long, device=device
)
shifts = torch.arange(0, 32, size_bits, dtype=torch.int32, device=device)
K, N_packed = qw.shape
N = N_packed * pack_factor
# Unpack int32 → individual values, fix AWQ nibble ordering
unpacked = (qw.unsqueeze(-1) >> shifts) & mask # (K, N_packed, pf)
unpacked = unpacked[:, :, reverse_order]
unpacked = unpacked.reshape(K, N) # (K, N)
# Repack along input dim (dim 0) in sequential nibble order
unpacked = unpacked.reshape(K // pack_factor, pack_factor, N)
new_qw = (unpacked.to(torch.int32) << shifts[None, :, None]).sum(
dim=1, dtype=torch.int32
)
return new_qw.contiguous()
def create_weights(
self,
layer: torch.nn.Module,
input_size_per_partition: int,
output_partition_sizes: list[int],
input_size: int,
output_size: int,
params_dtype: torch.dtype,
**extra_weight_attrs,
) -> None:
del input_size, output_size
self._create_inc_weights(
layer=layer,
input_size_per_partition=input_size_per_partition,
output_partition_sizes=output_partition_sizes,
params_dtype=params_dtype,
weight_loader=extra_weight_attrs.get("weight_loader"),
)
class INCXPULinearMethod(INCXPULinearBase):
"""XPU linear method for INC w4a16 quantization (symmetric only).
Supports both GPTQ-packed (``auto_round:auto_gptq``) and AWQ-packed
(``auto_round:auto_awq``) AutoRound checkpoints. AWQ-packed qweights are
losslessly repacked into the GPTQ-style nibble layout during
``process_weights_after_loading``, before the final oneDNN "NT" transpose
that ``torch.ops._xpu_C.int4_gemm_w4a16`` expects.
"""
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
device = layer.qweight.data.device
qweight_data = layer.qweight.data
if self.is_awq_packed:
# Lossless repack: AWQ [K, N // pf] → GPTQ [K // pf, N]
qweight_data = self._convert_awq_qweight_to_gptq(qweight_data)
qweight_ct = qweight_data.t().contiguous()
layer.qweight = Parameter(qweight_ct.t(), requires_grad=False)
layer.scales = Parameter(layer.scales.data, requires_grad=False)
layer.qzeros = Parameter(
torch.tensor([8], dtype=torch.int8, device=device),
requires_grad=False,
)
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
out_shape = x.shape[:-1] + (layer.qweight.shape[1],)
reshaped_x = x.reshape(-1, x.shape[-1])
out = torch.ops._xpu_C.int4_gemm_w4a16(
reshaped_x,
layer.qweight,
bias,
layer.scales,
layer.qzeros,
self.group_size,
None, # Retained by the external XPU op ABI.
)
return out.reshape(out_shape)
class INCARKLinearMethod(INCXPULinearBase):
def __init__(self, layer_config: "INCLayerConfig") -> None:
super().__init__(layer_config)
from .inc_ark_ops import get_ark_state
is_available, error_str, _, quant_linear_cls = get_ark_state()
if not is_available or quant_linear_cls is None:
reason = error_str or "unknown error"
raise ImportError(f"Failed to import auto_round_kernel. {reason}")
self.quant_linear_cls = quant_linear_cls
def create_weights(
self,
layer: torch.nn.Module,
input_size_per_partition: int,
output_partition_sizes: list[int],
input_size: int,
output_size: int,
params_dtype: torch.dtype,
**extra_weight_attrs,
) -> None:
super().create_weights(
layer=layer,
input_size_per_partition=input_size_per_partition,
output_partition_sizes=output_partition_sizes,
input_size=input_size,
output_size=output_size,
params_dtype=params_dtype,
**extra_weight_attrs,
)
layer.in_features = input_size_per_partition
layer.out_features = sum(output_partition_sizes)
layer.params_dtype = params_dtype
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
if hasattr(layer, "input_size_per_partition"):
in_features = layer.input_size_per_partition
elif hasattr(layer, "input_size"):
in_features = layer.input_size
else:
raise AttributeError("Cannot determine in_features for layer.")
if hasattr(layer, "output_partition_sizes"):
out_features = sum(layer.output_partition_sizes)
elif hasattr(layer, "output_size_per_partition"):
out_features = layer.output_size_per_partition
elif hasattr(layer, "output_size"):
out_features = layer.output_size
else:
out_features = layer.scales.shape[-1]
ark_linear = self.quant_linear_cls(
bits=self.weight_bits,
group_size=self.group_size,
sym=self.sym,
in_features=in_features,
out_features=out_features,
bias=layer.bias is not None,
weight_dtype=layer.params_dtype,
)
ark_linear.to(layer.qweight.device)
with torch.no_grad():
qweight_src = layer.qweight.detach()
if self.is_awq_packed:
# ARK consumes GPTQ-style packed nibbles; convert AWQ losslessly.
qweight_src = self._convert_awq_qweight_to_gptq(qweight_src)
ark_linear.qweight.copy_(qweight_src)
if hasattr(layer, "qzeros") and layer.qzeros is not None:
ark_linear.qzeros.copy_(layer.qzeros.detach())
else:
ark_linear.qzeros = None
ark_linear.scales.copy_(layer.scales.detach())
if hasattr(layer, "bias") and layer.bias is not None:
ark_linear.bias.copy_(layer.bias.detach())
ark_linear.post_init()
layer.qweight = Parameter(ark_linear.qweight.detach(), requires_grad=False)
layer.ark_bias = ark_linear.bias
layer.ark_compute_type = ark_linear.cdt
layer.ark_weight_type = ark_linear.wdt
layer.ark_scale_type = ark_linear.sdt
if hasattr(layer, "qzeros"):
del layer.qzeros
del layer.scales
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
return torch.ops.vllm.inc_ark_woq_linear.default(
x,
layer.qweight,
layer.ark_bias,
layer.out_features,
layer.in_features,
self.group_size,
layer.ark_compute_type,
layer.ark_weight_type,
layer.ark_scale_type,
not self.sym,
)
@@ -1,318 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TYPE_CHECKING
from vllm import envs
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQConfig
from vllm.platforms import current_platform
from ..inc_linear import INCLinearMethod
from .inc_scheme import INCScheme
if TYPE_CHECKING:
import torch
from ..config_parser import INCLayerConfig
from ..inc import INCConfig
logger = init_logger(__name__)
XPU_WNA16_SUPPORTED_BITS = {2, 4}
# On CUDA the Marlin/GPTQ/AWQ kernels only cover 4/8-bit. The remaining
# widths (2/3/5/6/7) have no dedicated CUDA kernel, so they are dispatched to
# the humming kernel instead. This mirrors compressed-tensors WNA16, whose
# 5/6/7-bit biased scalar types also fall back to humming at kernel selection.
CUDA_HUMMING_SUPPORTED_BITS = {2, 3, 5, 6, 7}
# Backends selectable through VLLM_XPU_INC_WNA16_BACKEND that are served by the
# oneDNN int4 GEMMs rather than by ARK. These only cover int4.
XPU_ONEDNN_BACKENDS = ("w4a16", "w4a8")
def _check_xpu_w4a8_supported(layer_config: "INCLayerConfig", prefix: str) -> None:
"""Raise unless ``int4_gemm_w4a8`` can serve this layer.
The backend is requested explicitly, so an unusable configuration is an
error rather than something to silently fall back from.
"""
import torch
if not hasattr(torch.ops._xpu_C, "int4_gemm_w4a8"):
raise NotImplementedError(
"VLLM_XPU_INC_WNA16_BACKEND=w4a8 requires the int4_gemm_w4a8 op, "
"which this build of vllm-xpu-kernels does not provide. "
f"Layer: {prefix}."
)
assert isinstance(layer_config.group_size, int), (
"WNA16 only supports integer group_size."
)
if layer_config.group_size <= 0 or layer_config.group_size % 32 != 0:
raise NotImplementedError(
"VLLM_XPU_INC_WNA16_BACKEND=w4a8 requires a group size that is a "
f"positive multiple of 32, got {layer_config.group_size}. "
f"Layer: {prefix}."
)
class INCWna16Scheme(INCScheme):
@staticmethod
def can_handle(layer_config: "INCLayerConfig") -> bool:
return layer_config.is_wna16_int
def get_linear_method(
self,
config: "INCConfig",
layer: "torch.nn.Module",
prefix: str,
layer_config: "INCLayerConfig",
):
del config, layer
if current_platform.is_xpu():
if layer_config.bits in XPU_WNA16_SUPPORTED_BITS and layer_config.sym:
from .inc_ark_ops import get_ark_state
from .inc_w4a8_linear import INCXPUW4A8LinearMethod
from .inc_wna16_linear import (
INCARKLinearMethod,
INCXPULinearMethod,
)
backend = envs.VLLM_XPU_INC_WNA16_BACKEND
if backend in XPU_ONEDNN_BACKENDS:
if layer_config.bits != 4:
raise NotImplementedError(
f"VLLM_XPU_INC_WNA16_BACKEND={backend} only supports "
f"int4, got int{layer_config.bits}. Layer: {prefix}."
)
if backend == "w4a8":
_check_xpu_w4a8_supported(layer_config, prefix)
return INCLinearMethod(INCXPUW4A8LinearMethod(layer_config))
return INCLinearMethod(INCXPULinearMethod(layer_config))
is_ark_available, ark_error, _, _ = get_ark_state()
if backend == "ark" and not is_ark_available:
raise NotImplementedError(
"VLLM_XPU_INC_WNA16_BACKEND=ark was requested but "
f"auto_round_kernel is unavailable: "
f"{ark_error or 'unknown error'}. Layer: {prefix}."
)
if is_ark_available:
return INCLinearMethod(INCARKLinearMethod(layer_config))
elif layer_config.bits == 2:
raise NotImplementedError(
"INC int2 on XPU requires the ARK backend. "
f"Layer: {prefix}. "
f"auto_round_kernel unavailable: "
f"{ark_error or 'unknown error'}"
)
logger.debug(
"ARK backend is unavailable for layer %s; "
"falling back to the default XPU INC path. Error: %s",
prefix,
ark_error or "unknown error",
)
return INCLinearMethod(INCXPULinearMethod(layer_config))
raise NotImplementedError(f"INC on XPU: unsupported config {layer_config}")
if current_platform.is_cpu() and layer_config.is_gptq:
if layer_config.bits == 4 and layer_config.sym:
from .inc_ark_ops import get_ark_state
from .inc_wna16_linear import (
INCARKLinearMethod,
INCWNA16LinearScheme,
)
is_ark_available, ark_error, _, _ = get_ark_state()
if is_ark_available:
return INCLinearMethod(INCARKLinearMethod(layer_config))
logger.debug(
"ARK backend is unavailable for layer %s; "
"falling back to the default CPU INC path. Error: %s",
prefix,
ark_error or "unknown error",
)
return INCLinearMethod(INCWNA16LinearScheme(layer_config))
raise NotImplementedError(f"INC on CPU: unsupported config {layer_config}")
# CUDA low-bit (2/3/5/6/7): no Marlin/GPTQ/AWQ kernel, route to humming
# so a single model can mix 4/8-bit (Marlin) and 2/3/5/6/7-bit (humming)
# layers.
if (
current_platform.is_cuda()
and layer_config.bits in CUDA_HUMMING_SUPPORTED_BITS
):
return _build_humming_linear_method(layer_config)
from .inc_wna16_linear import INCWNA16LinearScheme
return INCLinearMethod(INCWNA16LinearScheme(layer_config))
def get_moe_method(
self,
config: "INCConfig",
layer: "torch.nn.Module",
prefix: str,
layer_config: "INCLayerConfig",
):
del config, prefix
# CPU does not support quantized MoE yet.
if current_platform.is_cpu():
from vllm.model_executor.layers.fused_moe import (
UnquantizedFusedMoEMethod,
)
return UnquantizedFusedMoEMethod(layer.moe_config)
# CUDA low-bit (2/3/5/6/7): route to the humming MoE kernel (see above).
if (
current_platform.is_cuda()
and layer_config.bits in CUDA_HUMMING_SUPPORTED_BITS
):
return _build_humming_moe_method(layer, layer_config)
if layer_config.is_gptq:
return _resolve_gptq_moe(layer, layer_config)
if layer_config.is_awq:
return _resolve_awq_moe(layer, layer_config)
raise NotImplementedError(f"WNA16 MoE does not support config {layer_config}")
def _resolve_gptq_moe(layer: "torch.nn.Module", layer_config: "INCLayerConfig"):
from vllm.model_executor.layers.quantization.auto_gptq import (
AutoGPTQMoEMethod,
)
from vllm.model_executor.layers.quantization.moe_wna16 import (
MoeWNA16Config,
MoeWNA16Method,
)
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
check_moe_marlin_supports_layer,
)
assert isinstance(layer_config.group_size, int), (
"WNA16 only supports integer group_size."
)
# AutoGPTQMoEMethod selects its fused-MoE backend through the WNA16 oracle
# (Marlin on CUDA, XPUExpertsWNA16 on XPU). Gate only on the layer-shape
# check like compressed-tensors does; the capability-based
# check_marlin_supported is skipped so the XPU path is reachable.
use_marlin = (layer_config.bits, layer_config.sym) in {
(4, True),
(8, True),
} and check_moe_marlin_supports_layer(layer, layer_config.group_size)
if use_marlin:
return AutoGPTQMoEMethod(
AutoGPTQConfig(
weight_bits=layer_config.bits,
group_size=layer_config.group_size,
desc_act=False,
is_sym=layer_config.sym,
lm_head_quantized=False,
dynamic={},
full_config={},
),
layer.moe_config,
)
moe_config = MoeWNA16Config.from_config(
{
"quant_method": "gptq",
"bits": layer_config.bits,
"group_size": layer_config.group_size,
"sym": layer_config.sym,
"lm_head": False,
}
)
return MoeWNA16Method(moe_config, layer.moe_config)
def _resolve_awq_moe(layer: "torch.nn.Module", layer_config: "INCLayerConfig"):
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQMoEMethod
from vllm.model_executor.layers.quantization.moe_wna16 import (
MoeWNA16Config,
MoeWNA16Method,
)
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
check_moe_marlin_supports_layer,
)
assert isinstance(layer_config.group_size, int), (
"WNA16 only supports integer group_size."
)
use_marlin = layer_config.bits in (4, 8) and check_moe_marlin_supports_layer(
layer, layer_config.group_size
)
if use_marlin:
return AutoAWQMoEMethod(
AutoAWQConfig(
weight_bits=layer_config.bits,
group_size=layer_config.group_size,
zero_point=not layer_config.sym,
lm_head_quantized=False,
modules_to_not_convert=[],
full_config={},
),
layer.moe_config,
)
moe_config = MoeWNA16Config.from_config(
{
"quant_method": "awq",
"bits": layer_config.bits,
"group_size": layer_config.group_size,
"zero_point": not layer_config.sym,
"lm_head": False,
}
)
return MoeWNA16Method(moe_config, layer.moe_config)
def _humming_weight_config(layer_config: "INCLayerConfig") -> dict:
"""Build the humming weight-schema config for a WNA16 int checkpoint."""
if layer_config.is_gptq:
return {
"quant_method": "gptq",
"bits": layer_config.bits,
"group_size": layer_config.group_size,
"desc_act": False,
"sym": layer_config.sym,
}
if layer_config.is_awq:
return {
"quant_method": "awq",
"bits": layer_config.bits,
"group_size": layer_config.group_size,
"zero_point": not layer_config.sym,
}
raise NotImplementedError(
"INC humming dispatch only supports gptq/awq packed int checkpoints, "
f"but found {layer_config}."
)
def _build_humming_quant_config(layer_config: "INCLayerConfig"):
from vllm.model_executor.layers.quantization.humming import (
HummingLayerQuantizationConfig,
)
from vllm.utils.humming import BaseWeightSchema
weight_schema = BaseWeightSchema.from_config(_humming_weight_config(layer_config))
return HummingLayerQuantizationConfig(weight_schema=weight_schema)
def _build_humming_linear_method(layer_config: "INCLayerConfig"):
from vllm.model_executor.layers.quantization.humming import HummingLinearMethod
return HummingLinearMethod(_build_humming_quant_config(layer_config))
def _build_humming_moe_method(layer: "torch.nn.Module", layer_config: "INCLayerConfig"):
from vllm.model_executor.layers.quantization.humming import HummingMoEMethod
return HummingMoEMethod(_build_humming_quant_config(layer_config), layer.moe_config)
@@ -19,7 +19,7 @@ from vllm.model_executor.kernels.linear import (
init_mxfp8_linear_kernel,
init_nvfp4_linear_kernel,
)
from vllm.model_executor.layers.attention import Attention, MLAAttention
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoEConfig,
FusedMoEMethodBase,
@@ -228,6 +228,8 @@ class ModelOptQuantConfigBase(QuantizationConfig):
self, layer: torch.nn.Module, prefix: str
) -> "QuantizeMethodBase | None":
# handle kv-cache first so we can focus only on weight quantization thereafter
from vllm.model_executor.layers.attention.mla_attention import MLAAttention
if isinstance(layer, (Attention, MLAAttention)):
return self.KVCacheMethodCls(self)
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -1,21 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from .quark_nvfp4 import QuarkNVFP4
from .quark_ocp_mx import QuarkOCP_MX
from .quark_scheme import QuarkScheme
from .quark_w4a8_mxfp4_fp8 import QuarkW4A8_MXFP4_FP8
from .quark_w4a16_int4 import QuarkW4A16Int4
from .quark_w8a8_fp8 import QuarkW8A8Fp8, QuarkW8A8Fp8PerBlock
from .quark_w8a8_int8 import QuarkW8A8Int8
__all__ = [
"QuarkScheme",
"QuarkW8A8Fp8",
"QuarkW8A8Fp8PerBlock",
"QuarkW8A8Int8",
"QuarkOCP_MX",
"QuarkW4A16Int4",
"QuarkW4A8_MXFP4_FP8",
"QuarkNVFP4",
]
@@ -1,163 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable
import torch
from torch.nn.parameter import Parameter
from vllm.logger import init_logger
from vllm.model_executor.kernels.linear import init_nvfp4_linear_kernel
from vllm.model_executor.kernels.linear.nvfp4.emulation import (
EmulationNvFp4LinearKernel,
)
from vllm.model_executor.layers.quantization.quark.schemes.quark_scheme import (
QuarkScheme,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kNvfp4Dynamic,
kNvfp4Static,
)
from vllm.model_executor.parameter import (
GroupQuantScaleParameter,
ModelWeightParameter,
PerTensorScaleParameter,
)
__all__ = ["QuarkNVFP4"]
logger = init_logger(__name__)
class QuarkNVFP4(QuarkScheme):
"""Quark NVFP4 quantization scheme.
Supports loading NVFP4 checkpoints with the following structure:
- weight: uint8, shape [out_features, in_features // 2] (packed FP4)
- weight_scale: float8_e4m3fn, shape [out_features, in_features // group_size]
- weight_scale_2: bfloat16/float32, scalar (global weight scale)
- input_scale_2: bfloat16/float32, scalar (global input scale)
"""
supported_activation_quant_keys = [kNvfp4Dynamic]
supported_weight_quant_keys = [kNvfp4Static]
def __init__(
self,
activation_quant_key: QuantKey | None,
):
super().__init__(kNvfp4Static, activation_quant_key)
self.kernel = init_nvfp4_linear_kernel()
self.group_size = 16
if not isinstance(self.kernel, EmulationNvFp4LinearKernel):
logger.warning_once(
"Only EmulationNvFp4LinearKernel NVFP4 dense implementation is "
"tested with QuarkNVFP4, got kernel=%s. Correctness is not validated.",
type(self.kernel).__name__,
)
@classmethod
def get_min_capability(cls) -> int:
# FP4 requires Turing (75) or newer
return 75
def create_weights(
self,
layer: torch.nn.Module,
output_partition_sizes: list[int],
input_size_per_partition: int,
params_dtype: torch.dtype,
weight_loader: Callable,
**kwargs,
):
output_size_per_partition = sum(output_partition_sizes)
layer.logical_widths = output_partition_sizes
layer.input_size_per_partition = input_size_per_partition
layer.output_size_per_partition = output_size_per_partition
if input_size_per_partition % self.group_size != 0:
raise ValueError(
f"Input size per partition ({input_size_per_partition}) must be "
f"divisible by group size ({self.group_size})"
)
# Weight: FP4 packed as uint8 (2 FP4 values per uint8)
weight = ModelWeightParameter(
data=torch.empty(
output_size_per_partition,
input_size_per_partition // 2,
dtype=torch.uint8,
),
input_dim=1,
output_dim=0,
weight_loader=weight_loader,
)
layer.register_parameter("weight", weight)
# Per-group weight scale (FP8 E4M3)
weight_scale = GroupQuantScaleParameter(
data=torch.empty(
output_size_per_partition,
input_size_per_partition // self.group_size,
dtype=torch.float8_e4m3fn,
),
input_dim=1,
output_dim=0,
weight_loader=weight_loader,
)
layer.register_parameter("weight_scale", weight_scale)
# Global weight scale (scalar, per partition)
weight_scale_2 = PerTensorScaleParameter(
data=torch.empty(len(output_partition_sizes), dtype=torch.float32),
weight_loader=weight_loader,
)
layer.register_parameter("weight_scale_2", weight_scale_2)
# Global input scale (scalar, per partition)
input_scale_2 = PerTensorScaleParameter(
data=torch.empty(len(output_partition_sizes), dtype=torch.float32),
weight_loader=weight_loader,
)
layer.register_parameter("input_scale_2", input_scale_2)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
input_global_scale = layer.input_scale_2.max().to(torch.float32)
layer.input_global_scale = Parameter(input_global_scale, requires_grad=False)
del layer.input_scale_2
weight_global_scale = layer.weight_scale_2.to(torch.float32)
if torch.unique(weight_global_scale).numel() != 1:
logger.warning_once(
"In NVFP4 linear, the global scale for weight are different"
" for parallel layers (e.g. q_proj, k_proj, v_proj). This"
" will likely result in reduced accuracy. Please verify the"
" model accuracy. Consider using a checkpoint with a shared"
" global NVFP4 scale for fused layers."
)
weight_global_scale = weight_global_scale.max()
layer.weight_global_scale = Parameter(weight_global_scale, requires_grad=False)
del layer.weight_scale_2
layer.alpha = Parameter(
layer.input_global_scale * layer.weight_global_scale, requires_grad=False
)
layer.input_global_scale_inv = Parameter(
(1.0 / layer.input_global_scale).to(torch.float32), requires_grad=False
)
# Convert layer to NVFP4 linear kernel format
self.kernel.process_weights_after_loading(layer)
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
return self.kernel.apply_weights(layer=layer, x=x, bias=bias)
@@ -1,264 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable
from fractions import Fraction
import torch
from vllm.logger import init_logger
from vllm.model_executor.kernels.linear import (
MxFp4LinearKernel,
MxFp6LinearKernel,
Mxfp8LinearKernel,
init_mxfp4_linear_kernel,
init_mxfp6_linear_kernel,
init_mxfp8_linear_kernel,
)
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
MXFP8_SCALE_DTYPE,
MXFP8_VALUE_DTYPE,
)
from vllm.model_executor.layers.quantization.utils.ocp_mx_utils import (
_ACTIVATION_QUANT_KEY_MAP,
_WEIGHT_QUANT_KEY_MAP,
OCP_MX_BLOCK_SIZE,
OCP_MX_UNPACKED_DTYPES,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kMxfp4Static,
kMxfp6E2M3Static,
kMxfp6E3M2Static,
kMxfp8Static,
)
from vllm.model_executor.parameter import (
GroupQuantScaleParameter,
ModelWeightParameter,
PackedvLLMParameter,
)
from vllm.platforms import current_platform
from .quark_scheme import QuarkScheme
logger = init_logger(__name__)
class QuarkOCP_MX(QuarkScheme):
ocp_mx_linear: MxFp6LinearKernel | MxFp4LinearKernel | Mxfp8LinearKernel
supported_activation_quant_keys = [*_ACTIVATION_QUANT_KEY_MAP.values(), None]
supported_weight_quant_keys = [*_WEIGHT_QUANT_KEY_MAP.values()]
def __init__(
self,
weight_quant_key: QuantKey,
activation_quant_key: QuantKey | None,
scale_block_rows: int = 1,
):
super().__init__(weight_quant_key, activation_quant_key)
self.scale_block_rows = scale_block_rows
self.weight_dtype = next(
dtype
for dtype, quant_key in _WEIGHT_QUANT_KEY_MAP.items()
if quant_key == weight_quant_key
)
self.input_dtype = (
next(
dtype
for dtype, quant_key in _ACTIVATION_QUANT_KEY_MAP.items()
if quant_key == activation_quant_key
)
if activation_quant_key is not None
else None
)
self.is_unpacked = self.weight_dtype in OCP_MX_UNPACKED_DTYPES
if self.is_unpacked:
self.packed_factor: int | Fraction = 1
elif self.weight_dtype == "mxfp4":
self.packed_factor = 2
else:
self.packed_factor = Fraction(numerator=8, denominator=6)
if self.is_unpacked:
# MXFP8 runs on the shared kernel abstraction, which picks a native
# backend where one exists and emulates otherwise. The MXFP4/MXFP6
# warnings below do not apply.
return
if not current_platform.supports_mx():
logger.warning_once(
"The current platform does not support native MXFP4/MXFP6 "
"computation. Simulated weight dequantization and activation "
"QDQ (quantize and dequantize) will be used, with the linear "
"layers computed in high precision."
)
if current_platform.supports_mx() and (
self.input_dtype != "mxfp4" or self.weight_dtype != "mxfp4"
):
logger.warning_once(
"The current platform supports native MXFP4/MXFP6 "
f"computation, but kernels for input_dtype={self.input_dtype} "
f"and weight_dtype={self.weight_dtype} are not yet integrated "
"in vLLM. Simulated weight dequantization and activation "
"QDQ (quantize and dequantize) will be used, with the linear "
"layers computed in high precision."
)
def get_packed_dim(self, dim: int, quant_dtype: str):
if quant_dtype in OCP_MX_UNPACKED_DTYPES:
return dim
elif quant_dtype == "mxfp4":
assert dim % 2 == 0
return dim // 2
elif quant_dtype in {"mxfp6_e3m2", "mxfp6_e2m3"}:
# FP6 packs 4 * 6 = 24 bits on 3 bytes.
assert (dim * 3) % 4 == 0
return (dim * 3) // 4
else:
raise NotImplementedError(
"Unsupported quant_dtype in QuarkOCP_MX.get_packed_dim, "
f"got quant_dtype={quant_dtype}. Something is wrong, please "
"open an issue."
)
def _expanding_scale_loader(self, weight_loader: Callable) -> Callable:
"""Expand a 2-D block scale to one row per weight row.
A ``block_size=[R, 32]`` export carries one e8m0 value per R weight
rows; the kernels want one per row. At ``R == 1`` the layouts already
coincide, so the loader is passed through untouched.
"""
if self.scale_block_rows == 1:
return weight_loader
rows = self.scale_block_rows
def expanding_loader(param, loaded_weight, *args, **kwargs):
loaded_weight = loaded_weight.view(torch.uint8).repeat_interleave(
rows, dim=0
)
return weight_loader(param, loaded_weight, *args, **kwargs)
return expanding_loader
@classmethod
def get_min_capability(cls) -> int:
return 70
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
layer.weight = torch.nn.Parameter(layer.weight.data, requires_grad=False)
self.ocp_mx_linear.process_weights_after_loading(layer)
def create_weights(
self,
layer: torch.nn.Module,
output_partition_sizes: list[int],
input_size_per_partition: int,
params_dtype: torch.dtype,
weight_loader: Callable,
**kwargs,
):
if input_size_per_partition % OCP_MX_BLOCK_SIZE != 0:
layer_name = getattr(layer, "prefix", "") or type(layer).__name__
raise ValueError(
f"OCP MX linear layer {layer_name!r} has an input size per "
f"partition of {input_size_per_partition}, which must be "
f"divisible by the OCP MX group size {OCP_MX_BLOCK_SIZE}. "
"Choose a compatible tensor-parallel size or avoid "
"tensor-parallel sharding for this layer."
)
if self.is_unpacked:
output_size_per_partition = sum(output_partition_sizes)
layer.logical_widths = output_partition_sizes
layer.input_size_per_partition = input_size_per_partition
layer.output_size_per_partition = output_size_per_partition
layer.params_dtype = params_dtype
weight = ModelWeightParameter(
data=torch.empty(
output_size_per_partition,
input_size_per_partition,
dtype=MXFP8_VALUE_DTYPE,
),
input_dim=1,
output_dim=0,
weight_loader=weight_loader,
)
layer.register_parameter("weight", weight)
weight_scale = GroupQuantScaleParameter(
data=torch.empty(
output_size_per_partition,
input_size_per_partition // OCP_MX_BLOCK_SIZE,
dtype=MXFP8_SCALE_DTYPE,
),
input_dim=1,
output_dim=0,
weight_loader=self._expanding_scale_loader(weight_loader),
)
layer.register_parameter("weight_scale", weight_scale)
layer.weight_block_size = [1, OCP_MX_BLOCK_SIZE]
else:
output_size_per_partition = sum(output_partition_sizes)
layer.logical_widths = output_partition_sizes
# WEIGHT
weight = PackedvLLMParameter(
data=torch.empty(
output_size_per_partition,
self.get_packed_dim(input_size_per_partition, self.weight_dtype),
dtype=torch.uint8,
),
input_dim=1,
output_dim=0,
packed_dim=1,
packed_factor=self.packed_factor,
weight_loader=weight_loader,
)
layer.register_parameter("weight", weight)
# WEIGHT SCALE
weight_scale = GroupQuantScaleParameter(
data=torch.empty(
output_size_per_partition,
input_size_per_partition // OCP_MX_BLOCK_SIZE,
dtype=torch.uint8,
),
input_dim=1,
output_dim=0,
weight_loader=weight_loader,
)
layer.register_parameter("weight_scale", weight_scale)
if self.weight_quant_key == kMxfp4Static:
self.ocp_mx_linear = init_mxfp4_linear_kernel(
activation_quant_key=self.activation_quant_key,
)
elif self.weight_quant_key in [kMxfp6E2M3Static, kMxfp6E3M2Static]:
self.ocp_mx_linear = init_mxfp6_linear_kernel(
weight_quant_key=self.weight_quant_key,
activation_quant_key=self.activation_quant_key,
)
elif self.weight_quant_key == kMxfp8Static:
self.ocp_mx_linear = init_mxfp8_linear_kernel(
weight_shape=layer.weight.shape
)
else:
raise NotImplementedError(
"No OCP MX linear kernel for weight_quant_key="
f"{self.weight_quant_key}. Something is wrong, please open an "
"issue."
)
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
return self.ocp_mx_linear.apply_weights(layer, x, bias)
@@ -1,67 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from abc import ABC, abstractmethod
import torch
from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey
__all__ = ["QuarkScheme"]
class QuarkScheme(ABC):
"""Abstract class used to describe the weight creation and forward pass
of different quantization schemes supported by Quark.
"""
supported_activation_quant_keys: list[QuantKey | None] = []
supported_weight_quant_keys: list[QuantKey] = []
def __init__(
self,
weight_quant_key: QuantKey,
activation_quant_key: QuantKey | None,
):
if activation_quant_key not in self.supported_activation_quant_keys:
raise ValueError(
f"Unsupported activation quant key: {activation_quant_key}"
)
if weight_quant_key not in self.supported_weight_quant_keys:
raise ValueError(f"Unsupported weight quant key: {weight_quant_key}")
self.activation_quant_key = activation_quant_key
self.weight_quant_key = weight_quant_key
@classmethod
@abstractmethod
def get_min_capability(cls) -> int:
"""Get minimum device capability."""
raise NotImplementedError
@abstractmethod
def create_weights(self, *args, **kwargs):
"""Weight creation for the particular scheme. Inputs to this function."""
raise NotImplementedError
@abstractmethod
def apply_weights(
self, layer: torch.nn.Module, x: torch.Tensor, bias: torch.Tensor | None
):
"""Run the forward pass for the particular scheme. This is where
scheme-specific dequant/quant steps/kernels should be applied.
Args:
layer: torch.nn.Module with the registered weights and
other parameters relevant to the particular scheme.
x: input to the layer
bias: bias parameter
"""
raise NotImplementedError
@abstractmethod
def process_weights_after_loading(self, layer: torch.nn.Module):
"""Called after weight loading is complete for any cleanup that
needs to occur.
"""
raise NotImplementedError
@@ -1,212 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import math
from collections.abc import Callable
from typing import Any
import torch
from vllm.model_executor.kernels.linear import (
MPLinearKernel,
MPLinearLayerConfig,
choose_mp_linear_kernel,
)
from vllm.model_executor.layers.quantization.auto_awq import (
_convert_awq_to_standard_format,
)
from vllm.model_executor.layers.quantization.quark.schemes.quark_scheme import (
QuarkScheme,
)
from vllm.model_executor.layers.quantization.quark.utils import (
canonicalize_quark_packed_int4,
parse_w4a16_int4_weight_config,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kInt4Static,
kInt4Static32,
kInt4Static32Asym,
kInt4StaticAsym,
)
from vllm.model_executor.parameter import (
GroupQuantScaleParameter,
PackedvLLMParameter,
)
from vllm.scalar_type import scalar_types
class QuarkW4A16Int4(QuarkScheme):
"""Quark packed INT4 weight-only linear scheme via MPLinearKernel."""
supported_activation_quant_keys: list[QuantKey | None] = [None]
supported_weight_quant_keys: list[QuantKey] = [
kInt4Static,
kInt4Static32,
kInt4StaticAsym,
kInt4Static32Asym,
]
def __init__(
self,
weight_quant_key: QuantKey,
activation_quant_key: QuantKey | None,
pack_method: str,
weight_config: dict[str, Any],
):
super().__init__(weight_quant_key, activation_quant_key)
# group_size is read from the Quark config rather than weight_quant_key:
# the int4 QuantKeys only encode group_size 32 or -1, but Quark int4
# checkpoints commonly use group_size 128, which the QuantKey cannot
# represent. Symmetry is unambiguous, so it comes from the key.
self.group_size = parse_w4a16_int4_weight_config(weight_config)[0]
self.pack_factor = 8
self.pack_reorder = pack_method == "reorder"
self.is_symmetric = weight_quant_key.symmetric
self.quant_type = (
scalar_types.uint4b8 if self.is_symmetric else scalar_types.uint4
)
self.kernel: MPLinearKernel | None = None
@classmethod
def get_min_capability(cls) -> int:
return 70
def create_weights(
self,
layer: torch.nn.Module,
output_partition_sizes: list[int],
input_size_per_partition: int,
params_dtype: torch.dtype,
weight_loader: Callable,
**kwargs,
):
input_size = kwargs["input_size"]
output_size = kwargs["output_size"]
group_size = (
self.group_size if self.group_size != -1 else input_size_per_partition
)
if input_size_per_partition % group_size != 0:
raise ValueError(
"The input size is not aligned with the quantized weight shape. "
"This can be caused by too large tensor parallel size. "
f"input_size_per_partition={input_size_per_partition}, "
f"group_size={group_size}."
)
output_size_per_partition = sum(output_partition_sizes)
packed_output_size_per_partition = math.ceil(
output_size_per_partition / self.pack_factor
)
layer.output_size_per_partition = output_size_per_partition
layer.packed_output_size_per_partition = packed_output_size_per_partition
mp_linear_kernel_config = MPLinearLayerConfig(
full_weight_shape=(input_size, output_size),
partition_weight_shape=(
input_size_per_partition,
output_size_per_partition,
),
weight_type=self.quant_type,
act_type=params_dtype,
group_size=group_size,
zero_points=not self.is_symmetric,
)
kernel_type = choose_mp_linear_kernel(mp_linear_kernel_config)
def weight_scale_loader(
param: torch.nn.Parameter,
loaded_weight: torch.Tensor,
*args,
**loader_kwargs,
) -> None:
if loaded_weight.shape[1] < param.data.shape[1]:
padded_weight = loaded_weight.new_zeros(param.data.shape)
padded_weight[:, : loaded_weight.shape[1]] = loaded_weight
loaded_weight = padded_weight
weight_loader(param, loaded_weight, *args, **loader_kwargs)
weight = PackedvLLMParameter(
data=torch.empty(
input_size_per_partition,
packed_output_size_per_partition,
dtype=torch.int32,
),
input_dim=0,
output_dim=1,
packed_dim=1,
packed_factor=self.pack_factor,
weight_loader=weight_loader,
)
num_groups = input_size_per_partition // group_size
weight_zero_point = PackedvLLMParameter(
data=torch.zeros(
num_groups,
packed_output_size_per_partition,
dtype=torch.int32,
),
input_dim=0,
output_dim=1,
packed_dim=1,
packed_factor=self.pack_factor,
weight_loader=weight_loader,
)
weight_scale = GroupQuantScaleParameter(
data=torch.empty(
num_groups,
packed_output_size_per_partition * self.pack_factor,
dtype=params_dtype,
),
input_dim=0,
output_dim=1,
weight_loader=weight_scale_loader,
)
layer.register_parameter("weight", weight)
layer.register_parameter("weight_zero_point", weight_zero_point)
layer.register_parameter("weight_scale", weight_scale)
self.kernel = kernel_type(
mp_linear_kernel_config,
w_q_param_name="weight",
w_s_param_name="weight_scale",
w_zp_param_name="weight_zero_point" if not self.is_symmetric else None,
)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
layer.weight.data = canonicalize_quark_packed_int4(
layer.weight.data,
pack_reorder=self.pack_reorder,
is_symmetric=self.is_symmetric,
pack_factor=self.pack_factor,
)
if not self.is_symmetric:
layer.weight_zero_point.data = canonicalize_quark_packed_int4(
layer.weight_zero_point.data,
pack_reorder=self.pack_reorder,
is_symmetric=self.is_symmetric,
pack_factor=self.pack_factor,
)
output_size = layer.output_size_per_partition
packed_output_size = layer.packed_output_size_per_partition * self.pack_factor
if output_size < packed_output_size:
layer.weight_scale.data[:, output_size:].zero_()
_convert_awq_to_standard_format(
layer,
"weight",
"weight_zero_point" if not self.is_symmetric else None,
self.quant_type.size_bits,
)
assert self.kernel is not None
self.kernel.process_weights_after_loading(layer)
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
):
assert self.kernel is not None
return self.kernel.apply_weights(layer, x, bias)
@@ -1,204 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable
from fractions import Fraction
import torch
import torch.nn.functional as F
from vllm._aiter_ops import is_aiter_found_and_supported, rocm_aiter_ops
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
get_fp8_min_max,
kFp8StaticTensorSym,
kMxfp4Static,
)
from vllm.model_executor.parameter import (
GroupQuantScaleParameter,
PackedvLLMParameter,
PerTensorScaleParameter,
)
from vllm.platforms import current_platform
from .quark_scheme import QuarkScheme
logger = init_logger(__name__)
__all__ = ["QuarkW4A8_MXFP4_FP8"]
OCP_MX_BLOCK_SIZE = 32
class QuarkW4A8_MXFP4_FP8(QuarkScheme):
"""- Weights: MXFP4 with E8M0 scales per block of 32
- Activations: FP8 E4M3 (static per-tensor quantization)
Uses the AITER Triton kernel and falls back to emulation if AITER not available.
"""
supported_activation_quant_keys = [kFp8StaticTensorSym]
supported_weight_quant_keys = [kMxfp4Static]
def __init__(
self,
activation_quant_key: QuantKey | None,
):
super().__init__(kMxfp4Static, activation_quant_key)
self.out_dtype = None
self.weight_dtype = "mxfp4"
self.packed_factor: Fraction = Fraction(2, 1) # 2 FP4 values per byte
self.weight_block_size = OCP_MX_BLOCK_SIZE
self.fp8_min, self.fp8_max = get_fp8_min_max()
self.fp8_dtype = current_platform.fp8_dtype()
kernel_supported_gpu = False
if current_platform.is_rocm():
kernel_supported_gpu = current_platform.supports_mx()
self.use_aiter_kernel = is_aiter_found_and_supported() and kernel_supported_gpu
if not self.use_aiter_kernel:
logger.warning_once(
"[W4A8 MXFP4+FP8] Aiter Triton kernel not found. Using emulation mode."
)
@classmethod
def get_min_capability(cls) -> int:
return 70
def get_packed_dim(self, dim: int) -> int:
assert dim % 2 == 0, f"Dimension {dim} must be even for MXFP4 packing"
return dim // 2
def create_weights(
self,
layer: torch.nn.Module,
output_partition_sizes: list[int],
input_size_per_partition: int,
params_dtype: torch.dtype,
weight_loader: Callable,
**kwargs,
):
output_size_per_partition = sum(output_partition_sizes)
layer.logical_widths = output_partition_sizes
layer.input_size_per_partition = input_size_per_partition
layer.output_size_per_partition = output_size_per_partition
# MXFP4 WEIGHT (packed, 2 values per byte)
weight = PackedvLLMParameter(
data=torch.empty(
output_size_per_partition,
self.get_packed_dim(input_size_per_partition),
dtype=torch.uint8,
),
input_dim=1,
output_dim=0,
packed_dim=1,
packed_factor=self.packed_factor,
weight_loader=weight_loader,
)
layer.register_parameter("weight", weight)
# WEIGHT SCALE (E8M0 format, per block of 32)
weight_scale = GroupQuantScaleParameter(
data=torch.empty(
output_size_per_partition,
input_size_per_partition // self.weight_block_size,
dtype=torch.uint8,
),
input_dim=1,
output_dim=0,
weight_loader=weight_loader,
)
layer.register_parameter("weight_scale", weight_scale)
# INPUT SCALE (FP8 per-tensor static scale)
input_scale = PerTensorScaleParameter(
data=torch.empty(
len(output_partition_sizes),
dtype=torch.float32,
),
weight_loader=weight_loader,
)
# Initialize to avoid NaN
input_scale[:] = torch.finfo(torch.float32).min
layer.register_parameter("input_scale", input_scale)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
# Ensuring weights & scales are non-trainable
layer.weight = torch.nn.Parameter(layer.weight.data, requires_grad=False)
layer.weight_scale = torch.nn.Parameter(
layer.weight_scale.data, requires_grad=False
)
input_scale = layer.input_scale.data
# For fused modules (QKV), take the max scale
if input_scale.numel() != 1:
input_scale = input_scale.max()
layer.input_scale = torch.nn.Parameter(
torch.tensor(input_scale, dtype=torch.float32),
requires_grad=False,
)
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
if self.use_aiter_kernel:
return self._apply_aiter_kernel(layer, x, bias)
else:
return self._apply_emulation(layer, x, bias)
def _apply_aiter_kernel(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
M = x.shape[0]
out_dtype = x.dtype if self.out_dtype is None else self.out_dtype
input_scale = layer.input_scale
x_fp8 = (x / input_scale).clamp(self.fp8_min, self.fp8_max).to(self.fp8_dtype)
# Broadcast per-tensor scale to per-row (M, 1) for Aiter kernel
x_scales = input_scale.expand(M, 1).to(dtype=torch.float32, device=x.device)
y = rocm_aiter_ops.gemm_a8wfp4(
x_fp8, layer.weight, x_scales, layer.weight_scale, out_dtype
)
if bias is not None:
y = y + bias
return y
def _apply_emulation(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
dequant_mxfp4,
)
weight_dq = dequant_mxfp4(
layer.weight,
layer.weight_scale,
x.dtype,
)
input_scale = layer.input_scale
x_fp8 = (x / input_scale).clamp(self.fp8_min, self.fp8_max).to(self.fp8_dtype)
x_dq = (x_fp8.to(x.dtype) * input_scale).to(x.dtype)
return F.linear(x_dq, weight_dq, bias)
@@ -1,317 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable
import torch
from torch.nn import Parameter
from vllm.config import get_current_vllm_config
from vllm.logger import init_logger
from vllm.model_executor.kernels.linear import init_fp8_linear_kernel
from vllm.model_executor.layers.fusion.quant_activation import (
QuantizedActivation,
expose_input_quant_key,
)
from vllm.model_executor.layers.quantization.quark.schemes import QuarkScheme
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
create_fp8_scale_parameter,
create_fp8_weight_parameter,
validate_fp8_block_shape,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
GroupShape,
QuantKey,
kFp8Dynamic128Sym,
kFp8DynamicTensorSym,
kFp8DynamicTokenSym,
kFp8Static128BlockE8M0Sym,
kFp8Static128BlockSym,
kFp8StaticChannelSym,
kFp8StaticTensorSym,
)
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
normalize_e4m3fn_to_e4m3fnuz,
requantize_with_max_scale,
)
from vllm.model_executor.parameter import (
BlockQuantScaleParameter,
ChannelQuantScaleParameter,
ModelWeightParameter,
PerTensorScaleParameter,
)
from vllm.platforms import current_platform
__all__ = ["QuarkW8A8Fp8", "QuarkW8A8Fp8PerBlock"]
logger = init_logger(__name__)
class QuarkW8A8Fp8(QuarkScheme):
supported_activation_quant_keys: list[QuantKey | None] = [
kFp8DynamicTensorSym,
kFp8DynamicTokenSym,
kFp8StaticTensorSym,
]
supported_weight_quant_keys: list[QuantKey] = [
kFp8StaticChannelSym,
kFp8StaticTensorSym,
]
def __init__(
self,
weight_quant_key: QuantKey,
activation_quant_key: QuantKey | None,
):
super().__init__(weight_quant_key, activation_quant_key)
self.weight_qscheme = (
"per_channel" if weight_quant_key == kFp8StaticChannelSym else "per_tensor"
)
self.is_static_input_scheme = activation_quant_key == kFp8StaticTensorSym
self.out_dtype = torch.get_default_dtype()
self.input_dtype = get_current_vllm_config().model_config.dtype
@classmethod
def get_min_capability(cls) -> int:
# lovelace and up
return 89
def process_weights_after_loading(self, layer) -> None:
# If per tensor, when we have a fused module (e.g. QKV) with per
# tensor scales (thus N scales being passed to the kernel),
# requantize so we can always run per tensor
if self.weight_qscheme == "per_tensor":
if current_platform.is_fp8_fnuz():
input_scale = getattr(layer, "input_scale", None)
weight, max_w_scale, input_scale = normalize_e4m3fn_to_e4m3fnuz(
weight=layer.weight,
weight_scale=layer.weight_scale,
input_scale=input_scale,
)
if input_scale is not None:
layer.input_scale = Parameter(input_scale, requires_grad=False)
else:
max_w_scale = layer.weight_scale
weight = layer.weight
max_w_scale, weight = requantize_with_max_scale(
weight=weight,
weight_scale=max_w_scale,
logical_widths=layer.logical_widths,
)
layer.weight = Parameter(weight.t(), requires_grad=False)
layer.weight_scale = Parameter(max_w_scale, requires_grad=False)
# If channelwise, scales are already lined up, so just transpose.
elif self.weight_qscheme == "per_channel":
weight = layer.weight
if current_platform.is_fp8_fnuz():
input_scale = getattr(layer, "input_scale", None)
weight, weight_scale, input_scale = normalize_e4m3fn_to_e4m3fnuz(
weight=weight,
weight_scale=layer.weight_scale,
input_scale=input_scale,
)
if input_scale is not None:
layer.input_scale = Parameter(input_scale, requires_grad=False)
else:
weight_scale = layer.weight_scale.data
assert self.activation_quant_key is not None
if self.activation_quant_key.scale.group_shape == GroupShape.PER_TOKEN:
weight_scale = weight_scale.view(-1, 1)
layer.weight = Parameter(weight.t(), requires_grad=False)
# required by torch.compile to be torch.nn.Parameter
layer.weight_scale = Parameter(weight_scale, requires_grad=False)
else:
raise ValueError(f"Unknown quantization scheme {self.weight_qscheme}")
# INPUT SCALE
if self.is_static_input_scheme:
layer.input_scale = Parameter(layer.input_scale.max(), requires_grad=False)
self.fp8_linear.process_weights_after_loading(layer)
def create_weights(
self,
layer: torch.nn.Module,
output_partition_sizes: list[int],
input_size_per_partition: int,
params_dtype: torch.dtype,
weight_loader: Callable,
**kwargs,
):
output_size_per_partition = sum(output_partition_sizes)
layer.logical_widths = output_partition_sizes
# WEIGHT
weight = ModelWeightParameter(
data=torch.empty(
output_size_per_partition,
input_size_per_partition,
dtype=torch.float8_e4m3fn,
),
input_dim=1,
output_dim=0,
weight_loader=weight_loader,
)
layer.register_parameter("weight", weight)
# WEIGHT SCALE
# TODO: update create_xxx_parameter functions to return
# the newly added parameters
if self.weight_qscheme == "per_channel":
weight_scale = ChannelQuantScaleParameter(
data=torch.empty((sum(output_partition_sizes)), dtype=torch.float32),
output_dim=0,
weight_loader=weight_loader,
)
else:
assert self.weight_qscheme == "per_tensor"
weight_scale = PerTensorScaleParameter(
data=torch.empty(len(output_partition_sizes), dtype=torch.float32),
weight_loader=weight_loader,
)
# min requirement for fp8 kernels
weight_scale[:] = torch.finfo(torch.float32).min
layer.register_parameter("weight_scale", weight_scale)
# INPUT SCALE
if self.is_static_input_scheme:
input_scale = PerTensorScaleParameter(
data=torch.empty(len(output_partition_sizes), dtype=torch.float32),
weight_loader=weight_loader,
)
input_scale[:] = torch.finfo(torch.float32).min
layer.register_parameter("input_scale", input_scale)
assert self.activation_quant_key is not None
self.fp8_linear = init_fp8_linear_kernel(
activation_quant_key=self.activation_quant_key,
weight_quant_key=self.weight_quant_key,
weight_shape=layer.weight.shape,
input_dtype=self.input_dtype,
out_dtype=self.out_dtype,
module_name=self.__class__.__name__,
)
expose_input_quant_key(layer, self.fp8_linear)
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor | QuantizedActivation,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
return self.fp8_linear.apply_weights(layer, x, bias)
class QuarkW8A8Fp8PerBlock(QuarkScheme):
supported_activation_quant_keys = [kFp8Dynamic128Sym]
supported_weight_quant_keys = [
kFp8Static128BlockSym,
kFp8Static128BlockE8M0Sym,
]
def __init__(
self,
weight_quant_key: QuantKey,
activation_quant_key: QuantKey | None,
):
super().__init__(weight_quant_key, activation_quant_key)
self.weight_scale_dtype = weight_quant_key.scale.dtype
if self.weight_quant_key == kFp8Static128BlockE8M0Sym:
# TODO: oracle needs to support kFp8Static128BlockE8M0Sym.
self.weight_quant_key = kFp8Static128BlockSym
self.weight_block_size = list(weight_quant_key.scale.group_shape)
self.out_dtype = torch.get_default_dtype()
self.input_dtype = get_current_vllm_config().model_config.dtype
@classmethod
def get_min_capability(cls) -> int:
return QuarkW8A8Fp8.get_min_capability()
def process_weights_after_loading(self, layer) -> None:
# Quark exports the dequant multiplier as ``weight_scale`` (the same
# numerical convention as DeepSeek's ``weight_scale_inv``). Kernels
# multiply ``weight * scale``; do not invert here.
self.fp8_linear.process_weights_after_loading(layer)
def create_weights(
self,
layer: torch.nn.Module,
output_partition_sizes: list[int],
input_size_per_partition: int,
params_dtype: torch.dtype,
weight_loader: Callable,
**kwargs,
):
input_size = kwargs.get("input_size", input_size_per_partition)
output_size = kwargs.get("output_size", sum(output_partition_sizes))
output_size_per_partition = sum(output_partition_sizes)
layer.logical_widths = output_partition_sizes
layer.input_size_per_partition = input_size_per_partition
layer.output_size_per_partition = output_size_per_partition
layer.orig_dtype = params_dtype
layer.weight_block_size = self.weight_block_size
validate_fp8_block_shape(
layer,
input_size,
output_size,
input_size_per_partition,
output_partition_sizes,
self.weight_block_size,
)
weight = create_fp8_weight_parameter(
output_size_per_partition,
input_size_per_partition,
weight_loader,
)
layer.register_parameter("weight", weight)
scale_dtype = (
self.weight_scale_dtype
if self.weight_scale_dtype != torch.float32
else None
)
weight_scale = create_fp8_scale_parameter(
BlockQuantScaleParameter,
output_partition_sizes,
input_size_per_partition,
self.weight_block_size,
weight_loader,
scale_dtype=scale_dtype,
)
# Match Quark-exported checkpoints (``.weight_scale``).
# ``Fp8LinearMethod.create_weights`` registers weight_scale_inv, which has the
# same semantic meaning as Quark's `weight_scale`.
layer.register_parameter("weight_scale", weight_scale)
assert self.activation_quant_key is not None
self.fp8_linear = init_fp8_linear_kernel(
activation_quant_key=self.activation_quant_key,
weight_quant_key=self.weight_quant_key,
weight_shape=layer.weight.shape,
input_dtype=self.input_dtype,
out_dtype=self.out_dtype,
module_name=self.__class__.__name__,
)
logger.info_once(
"Selected %s for QuarkW8A8Fp8PerBlock",
type(self.fp8_linear).__name__,
)
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
return self.fp8_linear.apply_weights(layer, x, bias)
@@ -1,165 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable
import torch
from vllm.logger import init_logger
from vllm.model_executor.kernels.linear import (
init_int8_linear_kernel,
)
from vllm.model_executor.layers.quantization.quark.schemes import QuarkScheme
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kInt8DynamicTensorAsym,
kInt8DynamicTensorSym,
kInt8DynamicTokenAsym,
kInt8DynamicTokenSym,
kInt8StaticChannelSym,
kInt8StaticTensorAsym,
kInt8StaticTensorSym,
)
from vllm.model_executor.parameter import (
BasevLLMParameter,
ChannelQuantScaleParameter,
ModelWeightParameter,
PerTensorScaleParameter,
)
logger = init_logger(__name__)
class QuarkW8A8Int8(QuarkScheme):
supported_activation_quant_keys = [
kInt8StaticTensorSym,
kInt8StaticTensorAsym,
kInt8DynamicTensorSym,
kInt8DynamicTensorAsym,
kInt8DynamicTokenSym,
kInt8DynamicTokenAsym,
]
supported_weight_quant_keys = [
kInt8StaticChannelSym,
kInt8StaticTensorSym,
]
def __init__(
self,
weight_quant_key: QuantKey,
activation_quant_key: QuantKey | None,
):
super().__init__(weight_quant_key, activation_quant_key)
self.qscheme = (
"per_channel" if weight_quant_key == kInt8StaticChannelSym else "per_tensor"
)
assert activation_quant_key is not None
self.is_static_input_scheme = activation_quant_key.scale.static
self.input_symmetric = activation_quant_key.symmetric
@classmethod
def get_min_capability(cls) -> int:
# turing and up
return 75
def create_weights(
self,
layer: torch.nn.Module,
output_partition_sizes: list[int],
input_size_per_partition: int,
params_dtype: torch.dtype,
weight_loader: Callable,
**kwargs,
):
layer.logical_widths = output_partition_sizes
# Quark stores per-channel weight_scale as 1D [N]; reshape to [N, 1].
def _scale_weight_loader(
param: torch.nn.Parameter,
loaded_weight: torch.Tensor,
*args,
**kwargs,
):
if loaded_weight.dim() == 1:
loaded_weight = loaded_weight.unsqueeze(-1)
return weight_loader(param, loaded_weight, *args, **kwargs)
self.kernel = init_int8_linear_kernel(
is_channelwise=(self.qscheme == "per_channel"),
is_static_input_scheme=(self.is_static_input_scheme is True),
input_symmetric=(self.input_symmetric is True),
module_name=self.__class__.__name__,
)
# WEIGHT
weight = ModelWeightParameter(
data=torch.empty(
sum(output_partition_sizes), input_size_per_partition, dtype=torch.int8
),
input_dim=1,
output_dim=0,
weight_loader=weight_loader,
)
layer.register_parameter("weight", weight)
# WEIGHT SCALE
if self.qscheme == "per_channel":
weight_scale = ChannelQuantScaleParameter(
data=torch.empty((sum(output_partition_sizes), 1), dtype=torch.float32),
output_dim=0,
weight_loader=_scale_weight_loader,
)
ChannelQuantZPParameter = ChannelQuantScaleParameter
weight_zero_point = ChannelQuantZPParameter(
data=torch.empty((sum(output_partition_sizes), 1), dtype=torch.int8),
output_dim=0,
weight_loader=_scale_weight_loader,
)
else:
assert self.qscheme == "per_tensor"
weight_scale = PerTensorScaleParameter(
data=torch.empty(len(output_partition_sizes), dtype=torch.float32),
weight_loader=weight_loader,
)
PerTensorZPParameter = PerTensorScaleParameter
weight_zero_point = PerTensorZPParameter(
data=torch.empty(len(output_partition_sizes), dtype=torch.int8),
weight_loader=weight_loader,
)
layer.register_parameter("weight_scale", weight_scale)
layer.register_parameter("weight_zero_point", weight_zero_point)
# INPUT SCALE
input_zero_point = None
input_scale = None
if self.is_static_input_scheme:
input_scale = BasevLLMParameter(
data=torch.empty(1, dtype=torch.float32), weight_loader=weight_loader
)
input_zero_point = BasevLLMParameter(
data=torch.empty(1, dtype=torch.int8), weight_loader=weight_loader
)
layer.register_parameter("input_scale", input_scale)
layer.register_parameter("input_zero_point", input_zero_point)
if not hasattr(layer, "azp_adj"):
layer.register_parameter("azp_adj", None)
# Checkpoints are serialized in quark format, which is
# different from the format the kernel may want. Handle repacking here.
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
layer.register_parameter("weight_zero_point", None)
delattr(layer, "weight_zero_point")
if self.input_symmetric:
layer.register_parameter("input_zero_point", None)
delattr(layer, "input_zero_point")
self.kernel.process_weights_after_loading(layer)
def apply_weights(
self, layer: torch.nn.Module, x: torch.Tensor, bias: torch.Tensor | None
) -> torch.Tensor:
return self.kernel.apply_weights(layer, x, bias)
@@ -1,136 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterable, Mapping
from types import MappingProxyType
from typing import Any
import torch
from vllm.model_executor.layers.quantization.utils.config_utils import (
find_matching_patterns,
)
QuarkQTensorHint = dict[str, Any] | list[dict[str, Any]] | None
def deep_compare(dict1: Any, dict2: Any) -> bool:
if type(dict1) is not type(dict2):
return False
if isinstance(dict1, dict):
if dict1.keys() != dict2.keys():
return False
return all(deep_compare(dict1[k], dict2[k]) for k in dict1)
elif isinstance(dict1, list):
# `dict1` may be a list of dict.
return all(deep_compare(dict1[i], dict2[i]) for i in range(len(dict1)))
else:
return dict1 == dict2
def should_ignore_layer(
layer_name: str | None,
ignore: Iterable[str],
fused_mapping: Mapping[str, list[str]] = MappingProxyType({}),
*,
check_children: bool = False,
) -> bool:
if layer_name is None:
return False
# MoE layers are currently all-or-nothing: if any child is ignored,
# the parent layer must be ignored as well. For example, the
# amd/GLM-5.2-MXFP4 config ignores children like
# model.layers.78.mlp.experts.*.down_proj, while the layer checked
# here is the parent model.layers.N.mlp.experts.
# See:
# https://huggingface.co/amd/GLM-5.2-MXFP4/blob/main/config.json#L793-L795
if check_children and any(
target == layer_name or target.startswith(layer_name + ".")
for target in ignore
if not target.startswith("re:")
):
return True
# A direct fused-layer pattern takes precedence over expansion. For
# model.layers.0.self_attn.qkv_proj,
# ignore=["re:.*qkv_proj.*"] yields [{"re:.*qkv_proj.*"}]. In contrast,
# ignore=["re:.*[qkv]_proj"] yields one matching set per expanded shard.
per_shard_matches = find_matching_patterns(layer_name, ignore, fused_mapping)
shards_ignored = [len(matches) > 0 for matches in per_shard_matches]
if any(shards_ignored) and not all(shards_ignored):
raise ValueError(
f"Found different quantization schemes for the shards of "
f"{layer_name}. vLLM requires all to use the same scheme."
)
return all(shards_ignored)
def parse_w4a16_int4_weight_config(
weight_config: Mapping[str, Any],
) -> tuple[int, bool]:
"""Parse required W4A16 INT4/UINT4 weight fields from Quark config."""
if "group_size" not in weight_config:
raise ValueError(
"Quark W4A16 INT4/UINT4 configs must specify weight.group_size"
)
if "symmetric" not in weight_config:
raise ValueError("Quark W4A16 INT4/UINT4 configs must specify weight.symmetric")
group_size = weight_config["group_size"]
is_symmetric = weight_config["symmetric"]
if not isinstance(group_size, int) or group_size <= 0:
raise ValueError(
f"Quark W4A16 weight.group_size must be a positive int, got {group_size!r}"
)
if not isinstance(is_symmetric, bool):
raise ValueError(
f"Quark W4A16 weight.symmetric must be a bool, got {is_symmetric!r}"
)
return group_size, is_symmetric
def canonicalize_quark_packed_int4(
packed_weight: torch.Tensor,
*,
pack_reorder: bool,
is_symmetric: bool,
pack_factor: int = 8,
) -> torch.Tensor:
"""Convert Quark export nibble layout to AWQ checkpoint layout."""
from vllm.model_executor.layers.quantization.auto_awq import (
_REVERSE_AWQ_PACK_ORDER,
)
if pack_reorder:
source_order = torch.tensor(
_REVERSE_AWQ_PACK_ORDER, device=packed_weight.device, dtype=torch.int32
)
else:
source_order = torch.arange(
pack_factor, device=packed_weight.device, dtype=torch.int32
)
target_order = torch.tensor(
_REVERSE_AWQ_PACK_ORDER, device=packed_weight.device, dtype=torch.int32
)
source_shifts = source_order * 4
target_shifts = target_order * 4
values = (packed_weight.to(torch.int32)[..., None] >> source_shifts) & 0xF
if is_symmetric:
values = values ^ 0x8
packed = (values.to(torch.int64) << target_shifts.to(torch.int64)).sum(dim=-1)
return packed.to(torch.int32)
# utility for tensor dims > 2 cases
def quark_quantize_weight_to_mxfp4(w: torch.Tensor):
assert w.dtype == torch.bfloat16, (
"Quark dynamic quantization is supported only for fp16 weights and only to MXF4"
)
from aiter.ops.triton.quant import dynamic_mxfp4_quant
*dims, d = w.shape
w, w_scales = dynamic_mxfp4_quant(w.reshape(-1, d))
return w.view(*dims, d // 2), w_scales.view(*dims, d // 32)
@@ -20,40 +20,8 @@ def get_quark_ocp_mx_group_size(
quant_config: "QuantizationConfig | None",
layer_name: str,
) -> int | None:
"""Return the OCP MX group size for a quantized Quark linear layer."""
if quant_config is None:
return None
from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig
from vllm.model_executor.layers.quantization.quark.utils import should_ignore_layer
if not isinstance(quant_config, QuarkConfig):
return None
if should_ignore_layer(
layer_name,
ignore=quant_config.quant_config.get("exclude") or [],
fused_mapping=quant_config.packed_modules_mapping,
):
return None
layer_quant_config = (
quant_config.get_layer_quant_config_from_name(layer_name)
or quant_config.quant_config.get("global_quant_config")
or {}
)
weight_quant = layer_quant_config.get("weight")
input_quant = layer_quant_config.get("input_tensors")
if not quant_config._is_w_ocp_mx_a_x(weight_quant, input_quant):
return None
assert weight_quant is not None
group_size = weight_quant.get("group_size")
if group_size is None:
block_size = weight_quant.get("block_size")
if isinstance(block_size, list | tuple) and len(block_size) >= 2:
group_size = block_size[1]
return int(group_size) if group_size is not None else None
"""Quark support was removed; kept for callers that probe MX group size."""
return None
def is_shared_expert_quant_fse_compatible(
@@ -75,12 +43,9 @@ def is_shared_expert_quant_fse_compatible(
"""
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
from vllm.model_executor.layers.quantization.online.fp8 import OnlineLinearBase
from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig
from vllm.model_executor.layers.quantization.utils.quant_utils import (
is_layer_skipped,
)
from vllm.models.deepseek_v4.quant_config import DeepseekV4FP8Config
if projection_names is None:
projection_names = ["gate_up_proj", "down_proj"]
@@ -104,191 +69,12 @@ def is_shared_expert_quant_fse_compatible(
shared_expert_targets = None
if shared_expert_targets is not None:
from vllm.model_executor.layers.fused_moe import RoutedExperts
from vllm.model_executor.layers.linear import UnquantizedLinearMethod
from vllm.model_executor.layers.quantization.online.base import (
ONLINE_SHARED_EXPERT_QUANTIZERS,
)
# NOTE: online shared experts quantization check is only implemented for quark
# quant method at the moment. This can be extended here for other quant methods.
# Requirement is:
# - Retrieve routed weight quant key
# - Retrieve routed activation quant key
# and ensure the online quantization set on shared expert match them.
if isinstance(quant_config, QuarkConfig):
routed_weight_key, routed_activation_key, _ = (
quant_config.get_quant_method_target(expert_prefix, RoutedExperts)
)
else:
reason = (
"online quantization FSE compatibility check is not implemented for "
f"{type(quant_config).__name__}"
)
logger.warning_once(reason)
return False, reason
if routed_weight_key not in ONLINE_SHARED_EXPERT_QUANTIZERS:
return (
False,
"online shared-expert quantization is not supported for routed "
f"expert weight key {routed_weight_key}",
)
for shared_expert_target in shared_expert_targets:
if shared_expert_target is None:
return (
False,
"online quantization targets only part of the shared expert at "
f"{shared_expert_prefix}",
)
_, _, _, shared_quant_spec, shared_method_cls = shared_expert_target
if shared_method_cls in (None, UnquantizedLinearMethod):
return (
False,
"online quantization excludes FSE weights at "
f"{shared_expert_prefix}",
)
assert issubclass(shared_method_cls, OnlineLinearBase)
# NOTE: We can not rely on shared_quant_spec.activation_quant_key
# as _ONLINE_SHORTHANDS: dict[str, QuantSpec] in quantization.py
# does not define the default activation quant key.
# `online_quant_config.resolve_quant_method_cls` should return the
# activation quant key as well. Only some online methods define it
# on the class, so compare the weight key first.
if (
shared_quant_spec.weight != routed_weight_key
or shared_method_cls.activation_quant_key != routed_activation_key
):
return (
False,
"online shared-expert quantization keys do not match the "
"routed expert keys",
)
return True, None
if isinstance(quant_config, DeepseekV4FP8Config):
from vllm.config import get_current_vllm_config
from vllm.model_executor.models.utils import extract_layer_index
if quant_config.expert_dtype != "fp4":
return False, "DeepSeek-V4 routed experts are not MXFP4"
hf_config = get_current_vllm_config().model_config.hf_config
# TODO: This is adapted from former `_shared_experts_are_fp4`, and
# needs to be cleaned up this . There should not be Quark-specific
# logic in DeepseekV4FP8Config.
quantization_config = getattr(hf_config, "quantization_config", None)
if quantization_config is None:
return False, "DeepSeek-V4 has no quantization configuration"
if not quant_config._is_quark_mxfp4_ocp(quantization_config):
return False, "DeepSeek-v4 FSE is only implemented/tested with Quark MXFP4"
layer_idx = extract_layer_index(shared_expert_prefix)
if layer_idx >= hf_config.num_hidden_layers:
shared_expert_prefix = (
f"mtp.{layer_idx - hf_config.num_hidden_layers}.ffn.shared_experts"
)
else:
shared_expert_prefix = f"layers.{layer_idx}.ffn.shared_experts"
if any(
entry.startswith(shared_expert_prefix)
for entry in quantization_config.get("exclude") or []
if isinstance(entry, str)
):
return (
False,
f"DeepSeek-V4 excludes shared experts at {shared_expert_prefix}",
)
shared_expert_weight_name = f"{shared_expert_prefix}.w1"
layer_quant_config = quantization_config.get("layer_quant_config") or {}
layer_config = layer_quant_config.get(shared_expert_weight_name)
if layer_config is None:
layer_config = next(
(
config
for pattern, config in layer_quant_config.items()
if isinstance(pattern, str)
and pattern.startswith("re:")
and re.fullmatch(
pattern.removeprefix("re:"), shared_expert_weight_name
)
),
None,
)
shared_weight_config = (
layer_config or quantization_config.get("global_quant_config") or {}
).get("weight") or {}
if shared_weight_config.get("dtype") == "fp4":
return True, None
return (
False,
f"DeepSeek-V4 shared experts at {shared_expert_prefix} are not MXFP4",
)
if isinstance(quant_config, QuarkConfig):
from vllm.model_executor.layers.quantization.quark.utils import (
should_ignore_layer,
)
# TODO: layer_type_quant_config is not taken into account here.
assert "exclude" in quant_config.quant_config
assert "global_quant_config" in quant_config.quant_config
exclude = quant_config.quant_config["exclude"]
# should_ignore_layer raises a rightful ValueError in case different shards
# have a different ignore policy, as unsupported.
is_excluded = any(
should_ignore_layer(
f"{shared_expert_prefix}.{projection_name}",
ignore=exclude,
fused_mapping=quant_config.packed_modules_mapping,
)
for projection_name in projection_names
)
if is_excluded:
return False, f"Quark excludes shared experts at {shared_expert_prefix}"
global_quant_config = quant_config.quant_config["global_quant_config"]
def get_projection_quant_configs(layer_name: str) -> list[object]:
module_prefix, _, projection_name = layer_name.rpartition(".")
packed_projection_names = quant_config.packed_modules_mapping.get(
projection_name, [projection_name]
)
return [
quant_config.get_layer_quant_config_from_name(
f"{module_prefix}.{packed_projection_name}"
)
or global_quant_config
for packed_projection_name in packed_projection_names
]
expert_quant_config = (
quant_config.get_layer_quant_config_from_name(expert_prefix)
or global_quant_config
)
shared_expert_quant_configs = [
config
for projection_name in projection_names
for config in get_projection_quant_configs(
f"{shared_expert_prefix}.{projection_name}"
)
]
if all(config == expert_quant_config for config in shared_expert_quant_configs):
return True, None
return (
False,
"Quark uses different quantization configurations for routed and "
f"shared experts at {shared_expert_prefix}",
reason = (
"online quantization FSE compatibility check is not implemented for "
f"{type(quant_config).__name__}"
)
logger.warning_once(reason)
return False, reason
if isinstance(quant_config, Fp8Config):
if quant_config.store_dtype is not None:
@@ -52,16 +52,6 @@ def is_fp8(x: torch.dtype | torch.Tensor) -> bool:
def get_fp8_block_weight_scale(layer: torch.nn.Module) -> torch.Tensor | None:
"""Return the block-FP8 weight scale for supported quant methods."""
# Local import avoids a circular import: quark_w8a8_fp8 imports this module.
from vllm.model_executor.layers.quantization.quark.schemes.quark_w8a8_fp8 import (
QuarkW8A8Fp8PerBlock,
)
if isinstance(
getattr(layer, "scheme", None),
QuarkW8A8Fp8PerBlock,
):
return getattr(layer, "weight_scale", None)
return getattr(layer, "weight_scale_inv", None)
@@ -549,15 +549,6 @@ def mxfp4_quantize(x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
if current_platform.is_xpu():
return xpu_mxfp4_quantize(x)
from vllm._aiter_ops import is_aiter_found_and_supported
if is_aiter_found_and_supported() and x.dtype == torch.bfloat16:
from vllm.model_executor.layers.quantization.quark.utils import (
quark_quantize_weight_to_mxfp4,
)
return quark_quantize_weight_to_mxfp4(x)
quant_tensor, scale, _ = downcast_to_mxfp(x, axis=-1)
return quant_tensor, scale