From 6af03f2394b54e95440945d2659a8120288b21ea Mon Sep 17 00:00:00 2001 From: BadrBasowid <61441185+BadrBasowid@users.noreply.github.com> Date: Tue, 24 Feb 2026 14:47:22 +0800 Subject: [PATCH] [Refactor] [1/N] Reorganize kernel abstraction directory (#34055) Signed-off-by: BadrBasowid Co-authored-by: vllmellm Co-authored-by: TJian --- tests/compile/passes/test_fusion.py | 20 +- .../passes/test_silu_mul_quant_fusion.py | 14 +- .../test_scaled_mm_kernel_selection.py | 10 +- tests/utils.py | 6 +- .../quantization => }/kernels/__init__.py | 0 .../scaled_mm => kernels/linear}/__init__.py | 197 ++++++++++++++++-- .../linear}/mixed_precision/MPLinearKernel.py | 0 .../linear/mixed_precision/__init__.py | 48 +++++ .../linear}/mixed_precision/allspark.py | 0 .../linear}/mixed_precision/conch.py | 0 .../linear}/mixed_precision/cpu.py | 0 .../linear}/mixed_precision/cutlass.py | 0 .../linear}/mixed_precision/dynamic_4bit.py | 0 .../linear}/mixed_precision/exllama.py | 0 .../linear}/mixed_precision/machete.py | 0 .../linear}/mixed_precision/marlin.py | 0 .../linear}/mixed_precision/xpu.py | 0 .../linear}/scaled_mm/ScaledMMLinearKernel.py | 0 .../kernels/linear/scaled_mm/__init__.py | 54 +++++ .../linear}/scaled_mm/aiter.py | 0 .../linear}/scaled_mm/cpu.py | 0 .../linear}/scaled_mm/cutlass.py | 0 .../linear}/scaled_mm/flashinfer.py | 0 .../linear}/scaled_mm/pytorch.py | 0 .../linear}/scaled_mm/rocm.py | 0 .../linear}/scaled_mm/triton.py | 0 .../linear}/scaled_mm/xpu.py | 2 +- .../schemes/compressed_tensors_w4a8_fp8.py | 8 +- .../schemes/compressed_tensors_w4a8_int.py | 8 +- .../schemes/compressed_tensors_w8a8_fp8.py | 6 +- .../schemes/compressed_tensors_w8a8_int8.py | 6 +- .../schemes/compressed_tensors_wNa16.py | 10 +- .../layers/quantization/fbgemm_fp8.py | 6 +- .../model_executor/layers/quantization/fp8.py | 6 +- .../layers/quantization/gptq_marlin.py | 8 +- .../kernels/mixed_precision/__init__.py | 119 ----------- .../layers/quantization/modelopt.py | 6 +- .../layers/quantization/ptpc_fp8.py | 6 +- .../quark/schemes/quark_w8a8_fp8.py | 2 +- .../quark/schemes/quark_w8a8_int8.py | 2 +- 40 files changed, 331 insertions(+), 213 deletions(-) rename vllm/model_executor/{layers/quantization => }/kernels/__init__.py (100%) rename vllm/model_executor/{layers/quantization/kernels/scaled_mm => kernels/linear}/__init__.py (54%) rename vllm/model_executor/{layers/quantization/kernels => kernels/linear}/mixed_precision/MPLinearKernel.py (100%) create mode 100644 vllm/model_executor/kernels/linear/mixed_precision/__init__.py rename vllm/model_executor/{layers/quantization/kernels => kernels/linear}/mixed_precision/allspark.py (100%) rename vllm/model_executor/{layers/quantization/kernels => kernels/linear}/mixed_precision/conch.py (100%) rename vllm/model_executor/{layers/quantization/kernels => kernels/linear}/mixed_precision/cpu.py (100%) rename vllm/model_executor/{layers/quantization/kernels => kernels/linear}/mixed_precision/cutlass.py (100%) rename vllm/model_executor/{layers/quantization/kernels => kernels/linear}/mixed_precision/dynamic_4bit.py (100%) rename vllm/model_executor/{layers/quantization/kernels => kernels/linear}/mixed_precision/exllama.py (100%) rename vllm/model_executor/{layers/quantization/kernels => kernels/linear}/mixed_precision/machete.py (100%) rename vllm/model_executor/{layers/quantization/kernels => kernels/linear}/mixed_precision/marlin.py (100%) rename vllm/model_executor/{layers/quantization/kernels => kernels/linear}/mixed_precision/xpu.py (100%) rename vllm/model_executor/{layers/quantization/kernels => kernels/linear}/scaled_mm/ScaledMMLinearKernel.py (100%) create mode 100644 vllm/model_executor/kernels/linear/scaled_mm/__init__.py rename vllm/model_executor/{layers/quantization/kernels => kernels/linear}/scaled_mm/aiter.py (100%) rename vllm/model_executor/{layers/quantization/kernels => kernels/linear}/scaled_mm/cpu.py (100%) rename vllm/model_executor/{layers/quantization/kernels => kernels/linear}/scaled_mm/cutlass.py (100%) rename vllm/model_executor/{layers/quantization/kernels => kernels/linear}/scaled_mm/flashinfer.py (100%) rename vllm/model_executor/{layers/quantization/kernels => kernels/linear}/scaled_mm/pytorch.py (100%) rename vllm/model_executor/{layers/quantization/kernels => kernels/linear}/scaled_mm/rocm.py (100%) rename vllm/model_executor/{layers/quantization/kernels => kernels/linear}/scaled_mm/triton.py (100%) rename vllm/model_executor/{layers/quantization/kernels => kernels/linear}/scaled_mm/xpu.py (94%) delete mode 100644 vllm/model_executor/layers/quantization/kernels/mixed_precision/__init__.py diff --git a/tests/compile/passes/test_fusion.py b/tests/compile/passes/test_fusion.py index a2128150f7..5df9424a50 100644 --- a/tests/compile/passes/test_fusion.py +++ b/tests/compile/passes/test_fusion.py @@ -26,24 +26,16 @@ from vllm.config import ( PassConfig, VllmConfig, ) -from vllm.model_executor.layers.layernorm import RMSNorm -from vllm.model_executor.layers.quantization.kernels.scaled_mm.cutlass import ( - CutlassFP8ScaledMMLinearKernel, -) -from vllm.model_executor.layers.quantization.kernels.scaled_mm.flashinfer import ( - FlashInferFP8ScaledMMLinearKernel, -) -from vllm.model_executor.layers.quantization.kernels.scaled_mm.pytorch import ( +from vllm.model_executor.kernels.linear import ( ChannelWiseTorchFP8ScaledMMLinearKernel, + CutlassFP8ScaledMMLinearKernel, + FlashInferFP8ScaledMMLinearKernel, + FP8ScaledMMLinearKernel, PerTensorTorchFP8ScaledMMLinearKernel, + ROCmFP8ScaledMMLinearKernel, RowWiseTorchFP8ScaledMMLinearKernel, ) -from vllm.model_executor.layers.quantization.kernels.scaled_mm.rocm import ( - ROCmFP8ScaledMMLinearKernel, -) -from vllm.model_executor.layers.quantization.kernels.scaled_mm.ScaledMMLinearKernel import ( # noqa: E501 - FP8ScaledMMLinearKernel, -) +from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.quantization.utils.quant_utils import ( GroupShape, QuantKey, diff --git a/tests/compile/passes/test_silu_mul_quant_fusion.py b/tests/compile/passes/test_silu_mul_quant_fusion.py index c5ef015015..cc06208ea7 100644 --- a/tests/compile/passes/test_silu_mul_quant_fusion.py +++ b/tests/compile/passes/test_silu_mul_quant_fusion.py @@ -26,22 +26,14 @@ from vllm.config import ( VllmConfig, set_current_vllm_config, ) -from vllm.model_executor.layers.activation import SiluAndMul -from vllm.model_executor.layers.quantization.kernels.scaled_mm.cutlass import ( +from vllm.model_executor.kernels.linear import ( CutlassFP8ScaledMMLinearKernel, -) -from vllm.model_executor.layers.quantization.kernels.scaled_mm.flashinfer import ( FlashInferFP8ScaledMMLinearKernel, -) -from vllm.model_executor.layers.quantization.kernels.scaled_mm.pytorch import ( + FP8ScaledMMLinearKernel, PerTensorTorchFP8ScaledMMLinearKernel, -) -from vllm.model_executor.layers.quantization.kernels.scaled_mm.rocm import ( ROCmFP8ScaledMMLinearKernel, ) -from vllm.model_executor.layers.quantization.kernels.scaled_mm.ScaledMMLinearKernel import ( # noqa: E501 - FP8ScaledMMLinearKernel, -) +from vllm.model_executor.layers.activation import SiluAndMul from vllm.model_executor.layers.quantization.utils.fp8_utils import W8A8BlockFp8LinearOp from vllm.model_executor.layers.quantization.utils.quant_utils import ( GroupShape, diff --git a/tests/kernels/quantization/test_scaled_mm_kernel_selection.py b/tests/kernels/quantization/test_scaled_mm_kernel_selection.py index 1de8c444cf..1ac663ff6d 100644 --- a/tests/kernels/quantization/test_scaled_mm_kernel_selection.py +++ b/tests/kernels/quantization/test_scaled_mm_kernel_selection.py @@ -10,16 +10,10 @@ from abc import ABC import pytest -from vllm.model_executor.layers.quantization.kernels.scaled_mm import ( - Int8ScaledMMLinearLayerConfig, -) -from vllm.model_executor.layers.quantization.kernels.scaled_mm.aiter import ( +from vllm.model_executor.kernels.linear import ( AiterInt8ScaledMMLinearKernel, -) -from vllm.model_executor.layers.quantization.kernels.scaled_mm.cpu import ( CPUInt8ScaledMMLinearKernel, -) -from vllm.model_executor.layers.quantization.kernels.scaled_mm.ScaledMMLinearKernel import ( # noqa: E501 + Int8ScaledMMLinearLayerConfig, ScaledMMLinearKernel, ) diff --git a/tests/utils.py b/tests/utils.py index c12b235fa1..75d33e5095 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -42,11 +42,9 @@ from vllm.distributed import ( ) from vllm.engine.arg_utils import AsyncEngineArgs from vllm.entrypoints.cli.serve import ServeSubcommand -from vllm.model_executor.layers.quantization.kernels.scaled_mm import ( - init_fp8_linear_kernel, -) -from vllm.model_executor.layers.quantization.kernels.scaled_mm.ScaledMMLinearKernel import ( # noqa: E501 +from vllm.model_executor.kernels.linear import ( FP8ScaledMMLinearKernel, + init_fp8_linear_kernel, ) from vllm.model_executor.layers.quantization.utils.fp8_utils import W8A8BlockFp8LinearOp from vllm.model_executor.layers.quantization.utils.quant_utils import ( diff --git a/vllm/model_executor/layers/quantization/kernels/__init__.py b/vllm/model_executor/kernels/__init__.py similarity index 100% rename from vllm/model_executor/layers/quantization/kernels/__init__.py rename to vllm/model_executor/kernels/__init__.py diff --git a/vllm/model_executor/layers/quantization/kernels/scaled_mm/__init__.py b/vllm/model_executor/kernels/linear/__init__.py similarity index 54% rename from vllm/model_executor/layers/quantization/kernels/scaled_mm/__init__.py rename to vllm/model_executor/kernels/linear/__init__.py index bbd43dd108..1b4b7dc88a 100644 --- a/vllm/model_executor/layers/quantization/kernels/scaled_mm/__init__.py +++ b/vllm/model_executor/kernels/linear/__init__.py @@ -1,34 +1,57 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +""" +This module re-exports linear kernel implementations to provide a +stable import interface during an ongoing reorganization. Upcoming +PRs will remove the scaled_mm and mixed_precision subdirectories +and reorganize kernels by provider (aiter, cutlass, flashinfer, etc.) +rather than by precision type. By centralizing exports here, we +minimize the need to update imports across other modules when the +internal structure changes. If you are adding a new kernel selector +or kernel implementation, add it to this __init__.py to maintain +import stability. +""" + import os from typing import TypeVar import torch +import vllm.envs as envs from vllm.logger import init_logger -from vllm.model_executor.layers.quantization.kernels.scaled_mm.aiter import ( - AiterInt8ScaledMMLinearKernel, +from vllm.model_executor.kernels.linear.mixed_precision import ( + MPLinearKernel, + MPLinearLayerConfig, ) -from vllm.model_executor.layers.quantization.kernels.scaled_mm.cpu import ( - CPUInt8ScaledMMLinearKernel, +from vllm.model_executor.kernels.linear.mixed_precision.allspark import ( + AllSparkLinearKernel, ) -from vllm.model_executor.layers.quantization.kernels.scaled_mm.cutlass import ( - CutlassFP8ScaledMMLinearKernel, - CutlassInt8ScaledMMLinearKernel, +from vllm.model_executor.kernels.linear.mixed_precision.conch import ( + ConchLinearKernel, ) -from vllm.model_executor.layers.quantization.kernels.scaled_mm.flashinfer import ( - FlashInferFP8ScaledMMLinearKernel, +from vllm.model_executor.kernels.linear.mixed_precision.cpu import ( + CPUWNA16LinearKernel, ) -from vllm.model_executor.layers.quantization.kernels.scaled_mm.pytorch import ( - ChannelWiseTorchFP8ScaledMMLinearKernel, - PerTensorTorchFP8ScaledMMLinearKernel, - RowWiseTorchFP8ScaledMMLinearKernel, +from vllm.model_executor.kernels.linear.mixed_precision.cutlass import ( + CutlassW4A8LinearKernel, ) -from vllm.model_executor.layers.quantization.kernels.scaled_mm.rocm import ( - ROCmFP8ScaledMMLinearKernel, +from vllm.model_executor.kernels.linear.mixed_precision.dynamic_4bit import ( + Dynamic4bitLinearKernel, ) -from vllm.model_executor.layers.quantization.kernels.scaled_mm.ScaledMMLinearKernel import ( # noqa: E501 +from vllm.model_executor.kernels.linear.mixed_precision.exllama import ( + ExllamaLinearKernel, +) +from vllm.model_executor.kernels.linear.mixed_precision.machete import ( + MacheteLinearKernel, +) +from vllm.model_executor.kernels.linear.mixed_precision.marlin import ( + MarlinLinearKernel, +) +from vllm.model_executor.kernels.linear.mixed_precision.xpu import ( + XPUwNa16LinearKernel, +) +from vllm.model_executor.kernels.linear.scaled_mm import ( FP8ScaledMMLinearKernel, FP8ScaledMMLinearLayerConfig, Int8ScaledMMLinearKernel, @@ -36,10 +59,31 @@ from vllm.model_executor.layers.quantization.kernels.scaled_mm.ScaledMMLinearKer ScaledMMLinearKernel, ScaledMMLinearLayerConfig, ) -from vllm.model_executor.layers.quantization.kernels.scaled_mm.triton import ( +from vllm.model_executor.kernels.linear.scaled_mm.aiter import ( + AiterInt8ScaledMMLinearKernel, +) +from vllm.model_executor.kernels.linear.scaled_mm.cpu import ( + CPUInt8ScaledMMLinearKernel, +) +from vllm.model_executor.kernels.linear.scaled_mm.cutlass import ( + CutlassFP8ScaledMMLinearKernel, + CutlassInt8ScaledMMLinearKernel, +) +from vllm.model_executor.kernels.linear.scaled_mm.flashinfer import ( + FlashInferFP8ScaledMMLinearKernel, +) +from vllm.model_executor.kernels.linear.scaled_mm.pytorch import ( + ChannelWiseTorchFP8ScaledMMLinearKernel, + PerTensorTorchFP8ScaledMMLinearKernel, + RowWiseTorchFP8ScaledMMLinearKernel, +) +from vllm.model_executor.kernels.linear.scaled_mm.rocm import ( + ROCmFP8ScaledMMLinearKernel, +) +from vllm.model_executor.kernels.linear.scaled_mm.triton import ( TritonInt8ScaledMMLinearKernel, ) -from vllm.model_executor.layers.quantization.kernels.scaled_mm.xpu import ( +from vllm.model_executor.kernels.linear.scaled_mm.xpu import ( XPUFP8ScaledMMLinearKernel, ) from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey @@ -80,6 +124,29 @@ _POSSIBLE_FP8_KERNELS: dict[PlatformEnum, list[type[FP8ScaledMMLinearKernel]]] = ], } +# in priority/performance order (when available) +_POSSIBLE_KERNELS: dict[PlatformEnum, list[type[MPLinearKernel]]] = { + PlatformEnum.CUDA: [ + CutlassW4A8LinearKernel, + MacheteLinearKernel, + AllSparkLinearKernel, + MarlinLinearKernel, + ConchLinearKernel, + ExllamaLinearKernel, + ], + PlatformEnum.ROCM: [ + ConchLinearKernel, + ExllamaLinearKernel, + ], + PlatformEnum.XPU: [ + XPUwNa16LinearKernel, + ], + PlatformEnum.CPU: [ + Dynamic4bitLinearKernel, + CPUWNA16LinearKernel, + ], +} + _KernelT = TypeVar("_KernelT", bound=ScaledMMLinearKernel) _KernelConfigT = TypeVar("_KernelConfigT", bound=ScaledMMLinearLayerConfig) @@ -234,3 +301,97 @@ def init_int8_linear_kernel( "azp_adj", ], ) + + +def choose_mp_linear_kernel( + config: MPLinearLayerConfig, compute_capability: int | None = None +) -> type[MPLinearKernel]: + """ + Choose an MPLinearKernel that can implement the given config for the given + compute capability. Attempts to choose the best kernel in terms of + performance. + + Args: + config (MPLinearLayerConfig): Description of the linear layer to be + implemented. + compute_capability (Optional[int], optional): The compute capability of + the target device, if None uses `current_platform` to get + the compute capability. Defaults to None. + + Raises: + ValueError: If no kernel can implement the given config. + + Returns: + type[MPLinearKernel]: Chosen kernel. + """ + if compute_capability is None: + if current_platform is None: + raise ValueError("Cannot determine compute capability") + _cc = current_platform.get_device_capability() + if _cc is not None: + compute_capability = _cc[0] * 10 + _cc[1] + + failure_reasons = [] + for kernel in _POSSIBLE_KERNELS[current_platform._enum]: + if kernel.__name__ in envs.VLLM_DISABLED_KERNELS: + failure_reasons.append( + f" {kernel.__name__} disabled by environment variable" + ) + continue + if ( + compute_capability is not None + and kernel.get_min_capability() > compute_capability + ): + failure_reasons.append( + f"{kernel.__name__} requires capability " + f"{kernel.get_min_capability()}, current compute " + f" capability is {compute_capability}" + ) + continue + + can_implement, failure_reason = kernel.can_implement(config) + if can_implement: + return kernel + else: + failure_reasons.append( + f" {kernel.__name__} cannot implement due to: {failure_reason}" + ) + + raise ValueError( + "Failed to find a kernel that can implement the " + "WNA16 linear layer. Reasons: \n" + "\n".join(failure_reasons) + ) + + +__all__ = [ + "init_fp8_linear_kernel", + "init_int8_linear_kernel", + "choose_mp_linear_kernel", + "FP8ScaledMMLinearKernel", + "Int8ScaledMMLinearKernel", + "ScaledMMLinearKernel", + "FP8ScaledMMLinearLayerConfig", + "Int8ScaledMMLinearLayerConfig", + "ScaledMMLinearLayerConfig", + "AiterInt8ScaledMMLinearKernel", + "CPUInt8ScaledMMLinearKernel", + "CutlassFP8ScaledMMLinearKernel", + "CutlassInt8ScaledMMLinearKernel", + "FlashInferFP8ScaledMMLinearKernel", + "ChannelWiseTorchFP8ScaledMMLinearKernel", + "PerTensorTorchFP8ScaledMMLinearKernel", + "RowWiseTorchFP8ScaledMMLinearKernel", + "ROCmFP8ScaledMMLinearKernel", + "TritonInt8ScaledMMLinearKernel", + "MPLinearKernel", + "MPLinearLayerConfig", + "AllSparkLinearKernel", + "ConchLinearKernel", + "CPUWNA16LinearKernel", + "CutlassW4A8LinearKernel", + "Dynamic4bitLinearKernel", + "ExllamaLinearKernel", + "MacheteLinearKernel", + "MarlinLinearKernel", + "XPUwNa16LinearKernel", +] diff --git a/vllm/model_executor/layers/quantization/kernels/mixed_precision/MPLinearKernel.py b/vllm/model_executor/kernels/linear/mixed_precision/MPLinearKernel.py similarity index 100% rename from vllm/model_executor/layers/quantization/kernels/mixed_precision/MPLinearKernel.py rename to vllm/model_executor/kernels/linear/mixed_precision/MPLinearKernel.py diff --git a/vllm/model_executor/kernels/linear/mixed_precision/__init__.py b/vllm/model_executor/kernels/linear/mixed_precision/__init__.py new file mode 100644 index 0000000000..32f9afcceb --- /dev/null +++ b/vllm/model_executor/kernels/linear/mixed_precision/__init__.py @@ -0,0 +1,48 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from vllm.model_executor.kernels.linear.mixed_precision.allspark import ( + AllSparkLinearKernel, +) +from vllm.model_executor.kernels.linear.mixed_precision.conch import ( + ConchLinearKernel, +) +from vllm.model_executor.kernels.linear.mixed_precision.cpu import ( + CPUWNA16LinearKernel, +) +from vllm.model_executor.kernels.linear.mixed_precision.cutlass import ( + CutlassW4A8LinearKernel, +) +from vllm.model_executor.kernels.linear.mixed_precision.dynamic_4bit import ( + Dynamic4bitLinearKernel, +) +from vllm.model_executor.kernels.linear.mixed_precision.exllama import ( + ExllamaLinearKernel, +) +from vllm.model_executor.kernels.linear.mixed_precision.machete import ( + MacheteLinearKernel, +) +from vllm.model_executor.kernels.linear.mixed_precision.marlin import ( + MarlinLinearKernel, +) +from vllm.model_executor.kernels.linear.mixed_precision.MPLinearKernel import ( + MPLinearKernel, + MPLinearLayerConfig, +) +from vllm.model_executor.kernels.linear.mixed_precision.xpu import ( + XPUwNa16LinearKernel, +) + +__all__ = [ + "MPLinearKernel", + "MPLinearLayerConfig", + "AllSparkLinearKernel", + "ConchLinearKernel", + "CPUWNA16LinearKernel", + "CutlassW4A8LinearKernel", + "Dynamic4bitLinearKernel", + "ExllamaLinearKernel", + "MacheteLinearKernel", + "MarlinLinearKernel", + "XPUwNa16LinearKernel", +] diff --git a/vllm/model_executor/layers/quantization/kernels/mixed_precision/allspark.py b/vllm/model_executor/kernels/linear/mixed_precision/allspark.py similarity index 100% rename from vllm/model_executor/layers/quantization/kernels/mixed_precision/allspark.py rename to vllm/model_executor/kernels/linear/mixed_precision/allspark.py diff --git a/vllm/model_executor/layers/quantization/kernels/mixed_precision/conch.py b/vllm/model_executor/kernels/linear/mixed_precision/conch.py similarity index 100% rename from vllm/model_executor/layers/quantization/kernels/mixed_precision/conch.py rename to vllm/model_executor/kernels/linear/mixed_precision/conch.py diff --git a/vllm/model_executor/layers/quantization/kernels/mixed_precision/cpu.py b/vllm/model_executor/kernels/linear/mixed_precision/cpu.py similarity index 100% rename from vllm/model_executor/layers/quantization/kernels/mixed_precision/cpu.py rename to vllm/model_executor/kernels/linear/mixed_precision/cpu.py diff --git a/vllm/model_executor/layers/quantization/kernels/mixed_precision/cutlass.py b/vllm/model_executor/kernels/linear/mixed_precision/cutlass.py similarity index 100% rename from vllm/model_executor/layers/quantization/kernels/mixed_precision/cutlass.py rename to vllm/model_executor/kernels/linear/mixed_precision/cutlass.py diff --git a/vllm/model_executor/layers/quantization/kernels/mixed_precision/dynamic_4bit.py b/vllm/model_executor/kernels/linear/mixed_precision/dynamic_4bit.py similarity index 100% rename from vllm/model_executor/layers/quantization/kernels/mixed_precision/dynamic_4bit.py rename to vllm/model_executor/kernels/linear/mixed_precision/dynamic_4bit.py diff --git a/vllm/model_executor/layers/quantization/kernels/mixed_precision/exllama.py b/vllm/model_executor/kernels/linear/mixed_precision/exllama.py similarity index 100% rename from vllm/model_executor/layers/quantization/kernels/mixed_precision/exllama.py rename to vllm/model_executor/kernels/linear/mixed_precision/exllama.py diff --git a/vllm/model_executor/layers/quantization/kernels/mixed_precision/machete.py b/vllm/model_executor/kernels/linear/mixed_precision/machete.py similarity index 100% rename from vllm/model_executor/layers/quantization/kernels/mixed_precision/machete.py rename to vllm/model_executor/kernels/linear/mixed_precision/machete.py diff --git a/vllm/model_executor/layers/quantization/kernels/mixed_precision/marlin.py b/vllm/model_executor/kernels/linear/mixed_precision/marlin.py similarity index 100% rename from vllm/model_executor/layers/quantization/kernels/mixed_precision/marlin.py rename to vllm/model_executor/kernels/linear/mixed_precision/marlin.py diff --git a/vllm/model_executor/layers/quantization/kernels/mixed_precision/xpu.py b/vllm/model_executor/kernels/linear/mixed_precision/xpu.py similarity index 100% rename from vllm/model_executor/layers/quantization/kernels/mixed_precision/xpu.py rename to vllm/model_executor/kernels/linear/mixed_precision/xpu.py diff --git a/vllm/model_executor/layers/quantization/kernels/scaled_mm/ScaledMMLinearKernel.py b/vllm/model_executor/kernels/linear/scaled_mm/ScaledMMLinearKernel.py similarity index 100% rename from vllm/model_executor/layers/quantization/kernels/scaled_mm/ScaledMMLinearKernel.py rename to vllm/model_executor/kernels/linear/scaled_mm/ScaledMMLinearKernel.py diff --git a/vllm/model_executor/kernels/linear/scaled_mm/__init__.py b/vllm/model_executor/kernels/linear/scaled_mm/__init__.py new file mode 100644 index 0000000000..3056d5d0f7 --- /dev/null +++ b/vllm/model_executor/kernels/linear/scaled_mm/__init__.py @@ -0,0 +1,54 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from vllm.model_executor.kernels.linear.scaled_mm.aiter import ( + AiterInt8ScaledMMLinearKernel, +) +from vllm.model_executor.kernels.linear.scaled_mm.cpu import ( + CPUInt8ScaledMMLinearKernel, +) +from vllm.model_executor.kernels.linear.scaled_mm.cutlass import ( + CutlassFP8ScaledMMLinearKernel, + CutlassInt8ScaledMMLinearKernel, +) +from vllm.model_executor.kernels.linear.scaled_mm.flashinfer import ( + FlashInferFP8ScaledMMLinearKernel, +) +from vllm.model_executor.kernels.linear.scaled_mm.pytorch import ( + ChannelWiseTorchFP8ScaledMMLinearKernel, + PerTensorTorchFP8ScaledMMLinearKernel, + RowWiseTorchFP8ScaledMMLinearKernel, +) +from vllm.model_executor.kernels.linear.scaled_mm.rocm import ( + ROCmFP8ScaledMMLinearKernel, +) +from vllm.model_executor.kernels.linear.scaled_mm.ScaledMMLinearKernel import ( + FP8ScaledMMLinearKernel, + FP8ScaledMMLinearLayerConfig, + Int8ScaledMMLinearKernel, + Int8ScaledMMLinearLayerConfig, + ScaledMMLinearKernel, + ScaledMMLinearLayerConfig, +) +from vllm.model_executor.kernels.linear.scaled_mm.triton import ( + TritonInt8ScaledMMLinearKernel, +) + +__all__ = [ + "FP8ScaledMMLinearKernel", + "FP8ScaledMMLinearLayerConfig", + "Int8ScaledMMLinearKernel", + "Int8ScaledMMLinearLayerConfig", + "ScaledMMLinearKernel", + "ScaledMMLinearLayerConfig", + "AiterInt8ScaledMMLinearKernel", + "CPUInt8ScaledMMLinearKernel", + "CutlassFP8ScaledMMLinearKernel", + "CutlassInt8ScaledMMLinearKernel", + "FlashInferFP8ScaledMMLinearKernel", + "ChannelWiseTorchFP8ScaledMMLinearKernel", + "PerTensorTorchFP8ScaledMMLinearKernel", + "RowWiseTorchFP8ScaledMMLinearKernel", + "ROCmFP8ScaledMMLinearKernel", + "TritonInt8ScaledMMLinearKernel", +] diff --git a/vllm/model_executor/layers/quantization/kernels/scaled_mm/aiter.py b/vllm/model_executor/kernels/linear/scaled_mm/aiter.py similarity index 100% rename from vllm/model_executor/layers/quantization/kernels/scaled_mm/aiter.py rename to vllm/model_executor/kernels/linear/scaled_mm/aiter.py diff --git a/vllm/model_executor/layers/quantization/kernels/scaled_mm/cpu.py b/vllm/model_executor/kernels/linear/scaled_mm/cpu.py similarity index 100% rename from vllm/model_executor/layers/quantization/kernels/scaled_mm/cpu.py rename to vllm/model_executor/kernels/linear/scaled_mm/cpu.py diff --git a/vllm/model_executor/layers/quantization/kernels/scaled_mm/cutlass.py b/vllm/model_executor/kernels/linear/scaled_mm/cutlass.py similarity index 100% rename from vllm/model_executor/layers/quantization/kernels/scaled_mm/cutlass.py rename to vllm/model_executor/kernels/linear/scaled_mm/cutlass.py diff --git a/vllm/model_executor/layers/quantization/kernels/scaled_mm/flashinfer.py b/vllm/model_executor/kernels/linear/scaled_mm/flashinfer.py similarity index 100% rename from vllm/model_executor/layers/quantization/kernels/scaled_mm/flashinfer.py rename to vllm/model_executor/kernels/linear/scaled_mm/flashinfer.py diff --git a/vllm/model_executor/layers/quantization/kernels/scaled_mm/pytorch.py b/vllm/model_executor/kernels/linear/scaled_mm/pytorch.py similarity index 100% rename from vllm/model_executor/layers/quantization/kernels/scaled_mm/pytorch.py rename to vllm/model_executor/kernels/linear/scaled_mm/pytorch.py diff --git a/vllm/model_executor/layers/quantization/kernels/scaled_mm/rocm.py b/vllm/model_executor/kernels/linear/scaled_mm/rocm.py similarity index 100% rename from vllm/model_executor/layers/quantization/kernels/scaled_mm/rocm.py rename to vllm/model_executor/kernels/linear/scaled_mm/rocm.py diff --git a/vllm/model_executor/layers/quantization/kernels/scaled_mm/triton.py b/vllm/model_executor/kernels/linear/scaled_mm/triton.py similarity index 100% rename from vllm/model_executor/layers/quantization/kernels/scaled_mm/triton.py rename to vllm/model_executor/kernels/linear/scaled_mm/triton.py diff --git a/vllm/model_executor/layers/quantization/kernels/scaled_mm/xpu.py b/vllm/model_executor/kernels/linear/scaled_mm/xpu.py similarity index 94% rename from vllm/model_executor/layers/quantization/kernels/scaled_mm/xpu.py rename to vllm/model_executor/kernels/linear/scaled_mm/xpu.py index 5b816a3f5d..b16ee16997 100644 --- a/vllm/model_executor/layers/quantization/kernels/scaled_mm/xpu.py +++ b/vllm/model_executor/kernels/linear/scaled_mm/xpu.py @@ -5,7 +5,7 @@ from collections.abc import Sequence import torch -from vllm.model_executor.layers.quantization.kernels.scaled_mm.ScaledMMLinearKernel import ( # noqa: E501 +from vllm.model_executor.kernels.linear import ( # noqa: E501 FP8ScaledMMLinearKernel, FP8ScaledMMLinearLayerConfig, ) diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w4a8_fp8.py b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w4a8_fp8.py index 9a25e08cba..cf64cc180d 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w4a8_fp8.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w4a8_fp8.py @@ -7,13 +7,13 @@ import torch from compressed_tensors.quantization import ActivationOrdering from vllm.logger import init_logger -from vllm.model_executor.layers.quantization.compressed_tensors.schemes import ( - CompressedTensorsScheme, -) -from vllm.model_executor.layers.quantization.kernels.mixed_precision import ( +from vllm.model_executor.kernels.linear import ( MPLinearLayerConfig, choose_mp_linear_kernel, ) +from vllm.model_executor.layers.quantization.compressed_tensors.schemes import ( + CompressedTensorsScheme, +) from vllm.model_executor.layers.quantization.utils.marlin_utils import ( marlin_repeat_scales_on_all_ranks, ) diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w4a8_int.py b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w4a8_int.py index aa0c52beda..1822df5697 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w4a8_int.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w4a8_int.py @@ -6,13 +6,13 @@ from collections.abc import Callable import torch from vllm.logger import init_logger -from vllm.model_executor.layers.quantization.compressed_tensors.schemes import ( - CompressedTensorsScheme, -) -from vllm.model_executor.layers.quantization.kernels.mixed_precision import ( +from vllm.model_executor.kernels.linear import ( MPLinearLayerConfig, choose_mp_linear_kernel, ) +from vllm.model_executor.layers.quantization.compressed_tensors.schemes import ( + CompressedTensorsScheme, +) from vllm.model_executor.parameter import ( ChannelQuantScaleParameter, GroupQuantScaleParameter, diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_fp8.py b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_fp8.py index 1120202f29..23a8413523 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_fp8.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_fp8.py @@ -9,12 +9,12 @@ from torch.nn import Parameter from vllm._aiter_ops import rocm_aiter_ops from vllm.logger import init_logger +from vllm.model_executor.kernels.linear import ( + init_fp8_linear_kernel, +) from vllm.model_executor.layers.quantization.compressed_tensors.schemes import ( CompressedTensorsScheme, ) -from vllm.model_executor.layers.quantization.kernels.scaled_mm import ( - init_fp8_linear_kernel, -) from vllm.model_executor.layers.quantization.utils.fp8_utils import ( W8A8BlockFp8LinearOp, create_fp8_input_scale, diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_int8.py b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_int8.py index 652feb1964..833e3172c0 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_int8.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_int8.py @@ -7,12 +7,12 @@ import torch from compressed_tensors.quantization import QuantizationStrategy from vllm.logger import init_logger +from vllm.model_executor.kernels.linear import ( + init_int8_linear_kernel, +) from vllm.model_executor.layers.quantization.compressed_tensors.schemes import ( CompressedTensorsScheme, ) -from vllm.model_executor.layers.quantization.kernels.scaled_mm import ( - init_int8_linear_kernel, -) from vllm.model_executor.parameter import ( BasevLLMParameter, ChannelQuantScaleParameter, diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa16.py b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa16.py index f8b29041ee..1883d4ae32 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa16.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa16.py @@ -7,15 +7,13 @@ import torch from compressed_tensors.quantization import ActivationOrdering from vllm.logger import init_logger -from vllm.model_executor.layers.quantization.compressed_tensors.schemes import ( - CompressedTensorsScheme, -) -from vllm.model_executor.layers.quantization.kernels.mixed_precision import ( +from vllm.model_executor.kernels.linear import ( + MarlinLinearKernel, MPLinearLayerConfig, choose_mp_linear_kernel, ) -from vllm.model_executor.layers.quantization.kernels.mixed_precision.marlin import ( - MarlinLinearKernel, +from vllm.model_executor.layers.quantization.compressed_tensors.schemes import ( + CompressedTensorsScheme, ) from vllm.model_executor.layers.quantization.utils.marlin_utils import ( get_marlin_input_dtype, diff --git a/vllm/model_executor/layers/quantization/fbgemm_fp8.py b/vllm/model_executor/layers/quantization/fbgemm_fp8.py index 03a2d786a7..cca3b58eb6 100644 --- a/vllm/model_executor/layers/quantization/fbgemm_fp8.py +++ b/vllm/model_executor/layers/quantization/fbgemm_fp8.py @@ -8,6 +8,9 @@ from torch.nn import Module from torch.nn.parameter import Parameter from vllm.logger import init_logger +from vllm.model_executor.kernels.linear import ( + init_fp8_linear_kernel, +) from vllm.model_executor.layers.linear import ( LinearBase, LinearMethodBase, @@ -18,9 +21,6 @@ from vllm.model_executor.layers.quantization.base_config import ( QuantizationConfig, QuantizeMethodBase, ) -from vllm.model_executor.layers.quantization.kernels.scaled_mm import ( - init_fp8_linear_kernel, -) from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import ( apply_fp8_marlin_linear, prepare_fp8_layer_for_marlin, diff --git a/vllm/model_executor/layers/quantization/fp8.py b/vllm/model_executor/layers/quantization/fp8.py index e9a75f9d11..e3174ba995 100644 --- a/vllm/model_executor/layers/quantization/fp8.py +++ b/vllm/model_executor/layers/quantization/fp8.py @@ -13,6 +13,9 @@ from vllm import _custom_ops as ops from vllm._aiter_ops import rocm_aiter_ops from vllm.distributed import get_tensor_model_parallel_world_size from vllm.logger import init_logger +from vllm.model_executor.kernels.linear import ( + init_fp8_linear_kernel, +) from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.batch_invariant import ( vllm_is_batch_invariant, @@ -46,9 +49,6 @@ from vllm.model_executor.layers.quantization.base_config import ( QuantizationConfig, QuantizeMethodBase, ) -from vllm.model_executor.layers.quantization.kernels.scaled_mm import ( - init_fp8_linear_kernel, -) from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod from vllm.model_executor.layers.quantization.utils.flashinfer_utils import ( apply_fi_trtllm_fp8_per_tensor_moe, diff --git a/vllm/model_executor/layers/quantization/gptq_marlin.py b/vllm/model_executor/layers/quantization/gptq_marlin.py index 4c175fddb4..d7b2a366e1 100644 --- a/vllm/model_executor/layers/quantization/gptq_marlin.py +++ b/vllm/model_executor/layers/quantization/gptq_marlin.py @@ -10,6 +10,10 @@ from safetensors.torch import _TYPES as _SAFETENSORS_TO_TORCH_DTYPE import vllm.model_executor.layers.fused_moe # noqa from vllm import _custom_ops as ops from vllm.logger import init_logger +from vllm.model_executor.kernels.linear import ( + MPLinearLayerConfig, + choose_mp_linear_kernel, +) from vllm.model_executor.layers.fused_moe.config import ( FusedMoEConfig, FusedMoEQuantConfig, @@ -27,10 +31,6 @@ from vllm.model_executor.layers.quantization.base_config import ( QuantizationConfig, QuantizeMethodBase, ) -from vllm.model_executor.layers.quantization.kernels.mixed_precision import ( - MPLinearLayerConfig, - choose_mp_linear_kernel, -) from vllm.model_executor.layers.quantization.utils import replace_parameter from vllm.model_executor.layers.quantization.utils.gptq_utils import ( get_dynamic_override, diff --git a/vllm/model_executor/layers/quantization/kernels/mixed_precision/__init__.py b/vllm/model_executor/layers/quantization/kernels/mixed_precision/__init__.py deleted file mode 100644 index 93706e0b14..0000000000 --- a/vllm/model_executor/layers/quantization/kernels/mixed_precision/__init__.py +++ /dev/null @@ -1,119 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import vllm.envs as envs -from vllm.model_executor.layers.quantization.kernels.mixed_precision.allspark import ( # noqa: E501 - AllSparkLinearKernel, -) -from vllm.model_executor.layers.quantization.kernels.mixed_precision.conch import ( # noqa: E501 - ConchLinearKernel, -) -from vllm.model_executor.layers.quantization.kernels.mixed_precision.cpu import ( # noqa: E501 - CPUWNA16LinearKernel, -) -from vllm.model_executor.layers.quantization.kernels.mixed_precision.cutlass import ( # noqa: E501 - CutlassW4A8LinearKernel, -) -from vllm.model_executor.layers.quantization.kernels.mixed_precision.dynamic_4bit import ( # noqa: E501 - Dynamic4bitLinearKernel, -) -from vllm.model_executor.layers.quantization.kernels.mixed_precision.exllama import ( # noqa: E501 - ExllamaLinearKernel, -) -from vllm.model_executor.layers.quantization.kernels.mixed_precision.machete import ( # noqa: E501 - MacheteLinearKernel, -) -from vllm.model_executor.layers.quantization.kernels.mixed_precision.marlin import ( # noqa: E501 - MarlinLinearKernel, -) -from vllm.model_executor.layers.quantization.kernels.mixed_precision.MPLinearKernel import ( # noqa: E501 - MPLinearKernel, - MPLinearLayerConfig, -) -from vllm.model_executor.layers.quantization.kernels.mixed_precision.xpu import ( # noqa: E501 - XPUwNa16LinearKernel, -) -from vllm.platforms import PlatformEnum, current_platform - -# in priority/performance order (when available) -_POSSIBLE_KERNELS: dict[PlatformEnum, list[type[MPLinearKernel]]] = { - PlatformEnum.CUDA: [ - CutlassW4A8LinearKernel, - MacheteLinearKernel, - AllSparkLinearKernel, - MarlinLinearKernel, - ConchLinearKernel, - ExllamaLinearKernel, - ], - PlatformEnum.ROCM: [ - ConchLinearKernel, - ExllamaLinearKernel, - ], - PlatformEnum.XPU: [ - XPUwNa16LinearKernel, - ], - PlatformEnum.CPU: [ - Dynamic4bitLinearKernel, - CPUWNA16LinearKernel, - ], -} - - -def choose_mp_linear_kernel( - config: MPLinearLayerConfig, compute_capability: int | None = None -) -> type[MPLinearKernel]: - """ - Choose an MPLinearKernel that can implement the given config for the given - compute capability. Attempts to choose the best kernel in terms of - performance. - - Args: - config (MPLinearLayerConfig): Description of the linear layer to be - implemented. - compute_capability (Optional[int], optional): The compute capability of - the target device, if None uses `current_platform` to get - the compute capability. Defaults to None. - - Raises: - ValueError: If no kernel can implement the given config. - - Returns: - type[MPLinearKernel]: Chosen kernel. - """ - if compute_capability is None: - if current_platform is None: - raise ValueError("Cannot determine compute capability") - _cc = current_platform.get_device_capability() - if _cc is not None: - compute_capability = _cc[0] * 10 + _cc[1] - - failure_reasons = [] - for kernel in _POSSIBLE_KERNELS[current_platform._enum]: - if kernel.__name__ in envs.VLLM_DISABLED_KERNELS: - failure_reasons.append( - f" {kernel.__name__} disabled by environment variable" - ) - continue - if ( - compute_capability is not None - and kernel.get_min_capability() > compute_capability - ): - failure_reasons.append( - f"{kernel.__name__} requires capability " - f"{kernel.get_min_capability()}, current compute " - f" capability is {compute_capability}" - ) - continue - - can_implement, failure_reason = kernel.can_implement(config) - if can_implement: - return kernel - else: - failure_reasons.append( - f" {kernel.__name__} cannot implement due to: {failure_reason}" - ) - - raise ValueError( - "Failed to find a kernel that can implement the " - "WNA16 linear layer. Reasons: \n" + "\n".join(failure_reasons) - ) diff --git a/vllm/model_executor/layers/quantization/modelopt.py b/vllm/model_executor/layers/quantization/modelopt.py index 1991c6935d..5178060625 100644 --- a/vllm/model_executor/layers/quantization/modelopt.py +++ b/vllm/model_executor/layers/quantization/modelopt.py @@ -9,6 +9,9 @@ from torch.nn.parameter import Parameter import vllm.model_executor.layers.fused_moe.modular_kernel as mk from vllm.logger import init_logger +from vllm.model_executor.kernels.linear import ( + init_fp8_linear_kernel, +) from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.fused_moe.activation import MoEActivation from vllm.model_executor.layers.fused_moe.config import ( @@ -45,9 +48,6 @@ from vllm.model_executor.layers.quantization.base_config import ( QuantizationConfig, QuantizeMethodBase, ) -from vllm.model_executor.layers.quantization.kernels.scaled_mm import ( - init_fp8_linear_kernel, -) from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod from vllm.model_executor.layers.quantization.utils.flashinfer_fp4_moe import ( flashinfer_trtllm_fp4_moe, diff --git a/vllm/model_executor/layers/quantization/ptpc_fp8.py b/vllm/model_executor/layers/quantization/ptpc_fp8.py index 7ae732513c..76410f2e4d 100644 --- a/vllm/model_executor/layers/quantization/ptpc_fp8.py +++ b/vllm/model_executor/layers/quantization/ptpc_fp8.py @@ -8,6 +8,9 @@ from torch.nn.parameter import Parameter from vllm import _custom_ops as ops from vllm.logger import init_logger +from vllm.model_executor.kernels.linear import ( + init_fp8_linear_kernel, +) from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.linear import LinearBase, UnquantizedLinearMethod from vllm.model_executor.layers.quantization import QuantizationMethods @@ -17,9 +20,6 @@ from vllm.model_executor.layers.quantization.fp8 import ( Fp8KVCacheMethod, Fp8LinearMethod, ) -from vllm.model_executor.layers.quantization.kernels.scaled_mm import ( - init_fp8_linear_kernel, -) from vllm.model_executor.layers.quantization.utils.quant_utils import ( is_layer_skipped, kFp8DynamicTokenSym, diff --git a/vllm/model_executor/layers/quantization/quark/schemes/quark_w8a8_fp8.py b/vllm/model_executor/layers/quantization/quark/schemes/quark_w8a8_fp8.py index 635b5cf894..72f050a124 100644 --- a/vllm/model_executor/layers/quantization/quark/schemes/quark_w8a8_fp8.py +++ b/vllm/model_executor/layers/quantization/quark/schemes/quark_w8a8_fp8.py @@ -8,7 +8,7 @@ import torch from torch.nn import Parameter from vllm.logger import init_logger -from vllm.model_executor.layers.quantization.kernels.scaled_mm import ( +from vllm.model_executor.kernels.linear import ( init_fp8_linear_kernel, ) from vllm.model_executor.layers.quantization.quark.schemes import QuarkScheme diff --git a/vllm/model_executor/layers/quantization/quark/schemes/quark_w8a8_int8.py b/vllm/model_executor/layers/quantization/quark/schemes/quark_w8a8_int8.py index a7a7726bae..2afbe521c4 100644 --- a/vllm/model_executor/layers/quantization/quark/schemes/quark_w8a8_int8.py +++ b/vllm/model_executor/layers/quantization/quark/schemes/quark_w8a8_int8.py @@ -6,7 +6,7 @@ from collections.abc import Callable import torch from vllm.logger import init_logger -from vllm.model_executor.layers.quantization.kernels.scaled_mm import ( +from vllm.model_executor.kernels.linear import ( init_int8_linear_kernel, ) from vllm.model_executor.layers.quantization.quark.schemes import QuarkScheme