diff --git a/tests/kernels/quantization/test_block_int8.py b/tests/kernels/quantization/test_block_int8.py deleted file mode 100644 index 4a12ecd50d..0000000000 --- a/tests/kernels/quantization/test_block_int8.py +++ /dev/null @@ -1,75 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -# Adapted from https://github.com/sgl-project/sglang/blob/main/test/srt/test_block_int8.py -import itertools - -import pytest -import torch - -from tests.kernels.quant_utils import native_w8a8_block_matmul -from vllm.config import VllmConfig -from vllm.model_executor.layers.quantization.utils.int8_utils import ( - w8a8_block_int8_matmul, -) -from vllm.platforms import current_platform - -if not (current_platform.is_cuda_alike() or current_platform.is_xpu()): - pytest.skip( - "INT8 Triton kernels require a CUDA-alike or XPU device", - allow_module_level=True, - ) - -if current_platform.is_cuda_alike() and not current_platform.has_device_capability( - (7, 0) -): - pytest.skip("INT8 Triton requires CUDA 7.0 or higher", allow_module_level=True) - -vllm_config = VllmConfig() - -DTYPES = [torch.half, torch.bfloat16] -M = [1, 33, 64, 222] -N = [128, 1024] -K = [256, 4096] -# BLOCK_SIZE = [[64, 64], [64, 128], [128, 64], [128, 128]] -BLOCK_SIZE = [[128, 128]] -SEEDS = [0] - - -@pytest.mark.parametrize( - "M,N,K,block_size,out_dtype,seed", - itertools.product(M, N, K, BLOCK_SIZE, DTYPES, SEEDS), -) -@torch.inference_mode() -def test_w8a8_block_int8_matmul(M, N, K, block_size, out_dtype, seed): - torch.manual_seed(seed) - device = current_platform.device_type - factor_for_scale = 1e-2 - int8_info = torch.iinfo(torch.int8) - int8_max, int8_min = int8_info.max, int8_info.min - - A_fp32 = torch.rand(M, K, dtype=torch.float32, device=device) - A_fp32 = (A_fp32 - 0.5) * 2 * int8_max - A_fp8 = A_fp32.clamp(min=int8_min, max=int8_max).to(torch.float8_e4m3fn) - - B_fp32 = torch.rand(N, K, dtype=torch.float32, device=device) - B_fp32 = (B_fp32 - 0.5) * 2 * int8_max - B_fp8 = B_fp32.clamp(min=int8_min, max=int8_max).to(torch.float8_e4m3fn) - - block_n, block_k = block_size[0], block_size[1] - n_tiles = (N + block_n - 1) // block_n - k_tiles = (K + block_k - 1) // block_k - - As = torch.rand(M, k_tiles, dtype=torch.float32, device=device) * factor_for_scale - Bs = ( - torch.rand(n_tiles, k_tiles, dtype=torch.float32, device=device) - * factor_for_scale - ) - - ref_out = native_w8a8_block_matmul(A_fp8, B_fp8, As, Bs, block_size, out_dtype) - out = w8a8_block_int8_matmul(A_fp8, B_fp8, As, Bs, block_size, out_dtype) - - rel_diff = torch.mean( - torch.abs(out.to(torch.float32) - ref_out.to(torch.float32)) - ) / torch.mean(torch.abs(ref_out.to(torch.float32))) - assert rel_diff < 0.001 diff --git a/vllm/model_executor/kernels/linear/base.py b/vllm/model_executor/kernels/linear/base.py index 416b6ea1c1..036b30b7c4 100644 --- a/vllm/model_executor/kernels/linear/base.py +++ b/vllm/model_executor/kernels/linear/base.py @@ -89,28 +89,6 @@ class FP8Params(Params): ) -@dataclass -class Int8Params(Params): - """Int8 layer parameters with typed fields""" - - input_zero_point: torch.Tensor | None - azp_adj: torch.Tensor | None - - INPUT_ZERO_POINT: ClassVar[str] = "input_zero_point" - AZP_ADJ: ClassVar[str] = "azp_adj" - - @classmethod - def from_layer(cls, layer: torch.nn.Module) -> "Int8Params": - """Extract parameters from layer""" - return cls( - weight=getattr(layer, cls.WEIGHT), - weight_scale=getattr(layer, cls.WEIGHT_SCALE), - input_scale=getattr(layer, cls.INPUT_SCALE, None), - input_zero_point=getattr(layer, cls.INPUT_ZERO_POINT, None), - azp_adj=getattr(layer, cls.AZP_ADJ, None), - ) - - _ParamsT = TypeVar("_ParamsT", bound=Params) _ConfigT = TypeVar("_ConfigT", bound=MMLinearLayerConfig) @@ -130,7 +108,7 @@ class MMLinearKernel(ABC, Generic[_ConfigT, _ParamsT]): Typical Usage: 1. Define a config dataclass inheriting from MMLinearLayerConfig - 2. Define a params dataclass inheriting from Params (or FP8Params/Int8Params) + 2. Define a params dataclass inheriting from Params 3. Subclass MMLinearKernel with your config and params types 4. Implement all abstract methods 5. Register the kernel with the quantization method diff --git a/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py b/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py index d09587b6d5..e118331970 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py +++ b/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py @@ -130,14 +130,6 @@ class Mxfp4MoeBackend(Enum): HUMMING = "HUMMING" -# AITER backends group -AITER_BACKENDS = ( - Mxfp4MoeBackend.AITER_MXFP4_BF16, - Mxfp4MoeBackend.AITER_MXFP4_FP8, - Mxfp4MoeBackend.AITER_MXFP4_MXFP4, -) - - # Backends that share the same TRTLLM weight format TRTLLM_BACKENDS = ( Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_BF16, diff --git a/vllm/model_executor/layers/quantization/auto_awq.py b/vllm/model_executor/layers/quantization/auto_awq.py index ef81b0b876..713bd7bd5e 100644 --- a/vllm/model_executor/layers/quantization/auto_awq.py +++ b/vllm/model_executor/layers/quantization/auto_awq.py @@ -358,31 +358,6 @@ class AutoAWQConfig(QuantizationConfig): return None - @classmethod - def is_awq_marlin_compatible(cls, quant_config: dict[str, Any]): - # Extract data from quant config. - quant_method = quant_config.get("quant_method", "").lower() - num_bits = quant_config.get("bits") - group_size = quant_config.get("group_size") - zero_point = quant_config.get("zero_point") - - if not (current_platform.is_cuda_alike() or current_platform.is_cpu()): - return False - - if quant_method != "awq": - return False - - # If we cannot find the info needed in the config, cannot convert. - if num_bits is None or group_size is None or zero_point is None: - return False - - if num_bits not in cls.TYPE_MAP: - return False - - return check_marlin_supported( - quant_type=cls.TYPE_MAP[num_bits], group_size=group_size, has_zp=zero_point - ) - def apply_vllm_mapper(self, hf_to_vllm_mapper: "WeightsMapper"): if self.modules_to_not_convert: self.modules_to_not_convert = hf_to_vllm_mapper.apply_list( diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w4a8_fp8.py b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w4a8_fp8.py index 22c3539e9a..2c121c6edf 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w4a8_fp8.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w4a8_fp8.py @@ -32,7 +32,6 @@ __all__ = ["CompressedTensorsW4A8Fp8"] W4A8_SUPPORTED_TYPES_MAP = { 4: scalar_types.int4, } -W4A8_SUPPORTED_BITS = list(W4A8_SUPPORTED_TYPES_MAP.keys()) class CompressedTensorsW4A8Fp8(CompressedTensorsScheme): diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w4a8_int.py b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w4a8_int.py index 77933ea2c7..a26a35f4e3 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w4a8_int.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w4a8_int.py @@ -27,7 +27,6 @@ __all__ = ["CompressedTensorsW4A8Int"] W4A8_SUPPORTED_TYPES_MAP = { 4: scalar_types.int4, } -W4A8_SUPPORTED_BITS = list(W4A8_SUPPORTED_TYPES_MAP.keys()) class CompressedTensorsW4A8Int(CompressedTensorsScheme): diff --git a/vllm/model_executor/layers/quantization/humming.py b/vllm/model_executor/layers/quantization/humming.py index b814205922..ea00c0f3c4 100644 --- a/vllm/model_executor/layers/quantization/humming.py +++ b/vllm/model_executor/layers/quantization/humming.py @@ -1,6 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -import math from collections.abc import Callable from typing import TYPE_CHECKING, Any @@ -61,11 +60,6 @@ if TYPE_CHECKING: ) -def prepare_padded_shape(shape, x): - padded_shape = math.ceil(shape / x) * x - return padded_shape, padded_shape - shape - - def prepare_param(tensor, name, extra_attrs): extra_attrs = extra_attrs.copy() scale_type = extra_attrs.pop("scale_type", None) @@ -129,22 +123,6 @@ def prepare_moe_param(tensor: torch.Tensor, name: str, extra_attrs: dict[str, An return param -def may_pad_loaded_weight(param, loaded_weight): - pad_shape = getattr(param, "pad_shape", None) - if pad_shape is None: - return loaded_weight - value = 1 if loaded_weight.dtype == torch.float8_e8m0fnu else 0 - padding = [] - for x in pad_shape[::-1][: loaded_weight.ndim]: - padding += [0, x] - loaded_weight = torch.nn.functional.pad( - input=loaded_weight, - pad=padding, - value=value, - ) - return loaded_weight - - def compressed_tensors_get_config(config: dict[str, Any], key: str): assert key in ["weights", "input_activations"] target_group_config = None diff --git a/vllm/model_executor/layers/quantization/inc/inc.py b/vllm/model_executor/layers/quantization/inc/inc.py index 219fc2b046..ff8eae3a46 100644 --- a/vllm/model_executor/layers/quantization/inc/inc.py +++ b/vllm/model_executor/layers/quantization/inc/inc.py @@ -53,7 +53,6 @@ class INCConfig(QuantizationConfig): MXFP8_GROUP_SIZE = 32 MXFP8_DATA_TYPE = "mx_fp" MXFP8_PACKING_FORMAT = "auto_round:llm_compressor" - MXFP8_SUPPORTED_ACT_DTYPES = {"mx_fp", "mx_fp_rceil"} def __init__( self, diff --git a/vllm/model_executor/layers/quantization/inc/schemes/inc_wna16_linear.py b/vllm/model_executor/layers/quantization/inc/schemes/inc_wna16_linear.py index 5c99fd98b5..42f053b471 100644 --- a/vllm/model_executor/layers/quantization/inc/schemes/inc_wna16_linear.py +++ b/vllm/model_executor/layers/quantization/inc/schemes/inc_wna16_linear.py @@ -458,7 +458,3 @@ class INCARKLinearMethod(INCXPULinearBase): layer.ark_scale_type, not self.sym, ) - - -class INCXPUW4A16LinearScheme(INCXPULinearMethod): - pass diff --git a/vllm/model_executor/layers/quantization/modelopt.py b/vllm/model_executor/layers/quantization/modelopt.py index db98b76b3f..e24c5656ad 100644 --- a/vllm/model_executor/layers/quantization/modelopt.py +++ b/vllm/model_executor/layers/quantization/modelopt.py @@ -112,7 +112,6 @@ QUANT_ALGOS = [ # MIXED_PRECISION, "MIXED_PRECISION", ] -KV_CACHE_QUANT_ALGOS = ["FP8", "NVFP4"] class ModelOptKVCacheMethod(BaseKVCacheMethod): diff --git a/vllm/model_executor/layers/quantization/torchao.py b/vllm/model_executor/layers/quantization/torchao.py index 15399cfd39..8862578d6f 100644 --- a/vllm/model_executor/layers/quantization/torchao.py +++ b/vllm/model_executor/layers/quantization/torchao.py @@ -283,9 +283,6 @@ class TorchAOConfig(QuantizationConfig): return TorchAOLinearMethod(self) - def get_scaled_act_names(self) -> list[str]: - return [] - def torchao_quantize_param_data( param: torch.Tensor, torchao_config: Any diff --git a/vllm/model_executor/layers/quantization/utils/int8_utils.py b/vllm/model_executor/layers/quantization/utils/int8_utils.py index 4f624cf496..db95d3588a 100644 --- a/vllm/model_executor/layers/quantization/utils/int8_utils.py +++ b/vllm/model_executor/layers/quantization/utils/int8_utils.py @@ -2,17 +2,12 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project # Adapted from https://github.com/sgl-project/sglang/blob/4cb53ecd0cffceb6dee5c011a58f65997a86f151/python/sglang/srt/layers/quantization/int8_kernel.py -import functools -import json import logging -import os -from typing import Any import torch from vllm.platforms import current_platform from vllm.triton_utils import tl, triton -from vllm.utils.platform_utils import get_device_name_as_file_name logger = logging.getLogger(__name__) @@ -228,221 +223,3 @@ def per_token_group_quant_int8( ) return x_q, x_s - - -@triton.jit -def _w8a8_block_int8_matmul( - # Pointers to inputs and output - A, - B, - C, - As, - Bs, - # Shape for matmul - M, - N, - K, - # Block size for block-wise quantization - group_n, - group_k, - # Stride for inputs and output - stride_am, - stride_ak, - stride_bk, - stride_bn, - stride_cm, - stride_cn, - stride_As_m, - stride_As_k, - stride_Bs_k, - stride_Bs_n, - # Meta-parameters - BLOCK_SIZE_M: tl.constexpr, - BLOCK_SIZE_N: tl.constexpr, - BLOCK_SIZE_K: tl.constexpr, - GROUP_SIZE_M: tl.constexpr, -): - """Triton-accelerated function used to perform linear operations (dot - product) on input tensors `A` and `B` with block-wise quantization, and - store the result in output tensor `C`. - """ - - pid = tl.program_id(axis=0) - num_pid_m = tl.cdiv(M, BLOCK_SIZE_M) - num_pid_n = tl.cdiv(N, BLOCK_SIZE_N) - num_pid_in_group = GROUP_SIZE_M * num_pid_n - group_id = pid // num_pid_in_group - first_pid_m = group_id * GROUP_SIZE_M - group_size_m = min(num_pid_m - first_pid_m, GROUP_SIZE_M) - pid_m = first_pid_m + (pid % group_size_m) - pid_n = (pid % num_pid_in_group) // group_size_m - - offs_am = (pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M)) % M - offs_bn = (pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N)) % N - offs_k = tl.arange(0, BLOCK_SIZE_K) - a_ptrs = A + (offs_am[:, None] * stride_am + offs_k[None, :] * stride_ak) - b_ptrs = B + (offs_k[:, None] * stride_bk + offs_bn[None, :] * stride_bn) - - As_ptrs = As + offs_am * stride_As_m - offs_bsn = offs_bn // group_n - Bs_ptrs = Bs + offs_bsn * stride_Bs_n - - accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32) - for k in range(0, tl.cdiv(K, BLOCK_SIZE_K)): - a = tl.load(a_ptrs, mask=offs_k[None, :] < K - k * BLOCK_SIZE_K, other=0.0) - b = tl.load(b_ptrs, mask=offs_k[:, None] < K - k * BLOCK_SIZE_K, other=0.0) - - k_start = k * BLOCK_SIZE_K - offs_ks = k_start // group_k - a_s = tl.load(As_ptrs + offs_ks * stride_As_k) - b_s = tl.load(Bs_ptrs + offs_ks * stride_Bs_k) - - accumulator += tl.dot(a, b).to(tl.float32) * a_s[:, None] * b_s[None, :] - a_ptrs += BLOCK_SIZE_K * stride_ak - b_ptrs += BLOCK_SIZE_K * stride_bk - - if C.dtype.element_ty == tl.bfloat16: - c = accumulator.to(tl.bfloat16) - elif C.dtype.element_ty == tl.float16: - c = accumulator.to(tl.float16) - else: - c = accumulator.to(tl.float32) - - offs_cm = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M) - offs_cn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N) - c_ptrs = C + stride_cm * offs_cm[:, None] + stride_cn * offs_cn[None, :] - c_mask = (offs_cm[:, None] < M) & (offs_cn[None, :] < N) - tl.store(c_ptrs, c, mask=c_mask) - - -@functools.lru_cache -def get_w8a8_block_int8_configs( - N: int, K: int, block_n: int, block_k: int -) -> dict[int, Any] | None: - """ - Return optimized configurations for the w8a8 block fp8 kernel. - - The return value will be a dictionary that maps an irregular grid of - batch sizes to configurations of the w8a8 block fp8 kernel. To evaluate the - kernel on a given batch size bs, the closest batch size in the grid should - be picked and the associated configuration chosen to invoke the kernel. - """ - - # First look up if an optimized configuration is available in the configs - # directory - device_name = get_device_name_as_file_name() - json_file_name = f"N={N},K={K},device_name={device_name},dtype=int8_w8a8,block_shape=[{block_n}, {block_k}].json" # noqa: E501 - - config_file_path = os.path.join( - os.path.dirname(os.path.realpath(__file__)), "configs", json_file_name - ) - if os.path.exists(config_file_path): - with open(config_file_path) as f: - logger.info( - "Using configuration from %s for W8A8 Block INT8 kernel.", - config_file_path, - ) - # If a configuration has been found, return it - return {int(key): val for key, val in json.load(f).items()} - - # If no optimized configuration is available, we will use the default - # configuration - logger.warning( - ( - "Using default W8A8 Block INT8 kernel config. Performance might " - "be sub-optimal! Config file not found at %s" - ), - config_file_path, - ) - return None - - -def w8a8_block_int8_matmul( - A: torch.Tensor, - B: torch.Tensor, - As: torch.Tensor, - Bs: torch.Tensor, - block_size: list[int], - output_dtype: torch.dtype = torch.float16, -) -> torch.Tensor: - """This function performs matrix multiplication with block-wise - quantization. - - It takes two input tensors `A` and `B` with scales `As` and `Bs`. - The output is returned in the specified `output_dtype`. - - Args: - A: The input tensor, e.g., activation. - B: The input tensor, e.g., weight. - As: The per-token-group quantization scale for `A`. - Bs: The per-block quantization scale for `B`. - block_size: The block size for per-block quantization. It should be - 2-dim, e.g., [128, 128]. - output_dtype: The dtype of the returned tensor. - - Returns: - torch.Tensor: The result of matmul. - """ - assert len(block_size) == 2 - block_n, block_k = block_size[0], block_size[1] - - assert A.shape[-1] == B.shape[-1] - assert A.shape[:-1] == As.shape[:-1] and A.is_contiguous() - assert triton.cdiv(A.shape[-1], block_k) == As.shape[-1] - M = A.numel() // A.shape[-1] - - assert B.ndim == 2 and B.is_contiguous() and Bs.ndim == 2 - N, K = B.shape - assert triton.cdiv(N, block_n) == Bs.shape[0] - assert triton.cdiv(K, block_k) == Bs.shape[1] - - C_shape = A.shape[:-1] + (N,) - C = A.new_empty(C_shape, dtype=output_dtype) - - configs = get_w8a8_block_int8_configs(N, K, block_size[0], block_size[1]) - if configs: - # If an optimal configuration map has been found, look up the - # optimal config - config = configs[min(configs.keys(), key=lambda x: abs(x - M))] - else: - # Default config - # Block-wise quant: BLOCK_SIZE_K must be divisible by block_size[1] - config = { - "BLOCK_SIZE_M": 64, - "BLOCK_SIZE_N": block_size[0], - "BLOCK_SIZE_K": block_size[1], - "GROUP_SIZE_M": 32, - "num_warps": 4, - "num_stages": 3, - } - - def grid(META): - return ( - triton.cdiv(M, META["BLOCK_SIZE_M"]) * triton.cdiv(N, META["BLOCK_SIZE_N"]), - ) - - _w8a8_block_int8_matmul[grid]( - A, - B, - C, - As, - Bs, - M, - N, - K, - block_n, - block_k, - A.stride(-2), - A.stride(-1), - B.stride(1), - B.stride(0), - C.stride(-2), - C.stride(-1), - As.stride(-2), - As.stride(-1), - Bs.stride(1), - Bs.stride(0), - **config, - ) - - return C diff --git a/vllm/model_executor/layers/quantization/utils/mxfp4_utils.py b/vllm/model_executor/layers/quantization/utils/mxfp4_utils.py index 65348a8222..2d0a4fa48c 100644 --- a/vllm/model_executor/layers/quantization/utils/mxfp4_utils.py +++ b/vllm/model_executor/layers/quantization/utils/mxfp4_utils.py @@ -12,13 +12,6 @@ from vllm.utils.torch_utils import direct_register_custom_op, is_torch_equal_or_ logger = init_logger(__name__) -# CK's pre-compiled MXFP4 MoE GEMM kernel instances require the -# intermediate_size (after TP split) to be a multiple of this value. -# This arises from FP4 packing (2 values per byte) combined with CK -# tile size constraints. When violated, AITER raises: -# "device_gemm ... does not support this GEMM problem". -CK_MXFP4_MOE_DIM_ALIGNMENT = 256 - def should_use_cdna4_mx_scale_swizzle() -> bool: """Whether to use the CDNA4 swizzled scale layout for mxfp4 on gfx950. diff --git a/vllm/model_executor/layers/quantization/utils/ocp_mx_utils.py b/vllm/model_executor/layers/quantization/utils/ocp_mx_utils.py index a9157cbfb0..0d56f179b9 100644 --- a/vllm/model_executor/layers/quantization/utils/ocp_mx_utils.py +++ b/vllm/model_executor/layers/quantization/utils/ocp_mx_utils.py @@ -16,7 +16,6 @@ OCP_MX_DTYPES = { "mxfp8_e5m2", "mxint8", } -SUPPORTED_OCP_MX_DTYPES = {"mxfp4", "mxfp6_e3m2", "mxfp6_e2m3"} class OCP_MX_Scheme(str, Enum):