[Mini-vLLM] Cut dead subsystems: models, entrypoints, backends, utils, tests
- Drop dead model modules, keeping files live imports require - Remove anthropic, cohere, codegen entrypoints and batch STT modes - Delete unused attention backends, MoE experts, and NPU/XPU util leaves - Strip dead third_party trees and ROCm leaves with no CUDA path - Remove mooncake EC connectors: dependency missing since branch base - Re-audit verified by full import sweep and Qwen3.5 smoke test Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -59,10 +59,6 @@ from vllm.model_executor.kernels.linear.mixed_precision.rdna_hybrid_w4a16 import
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.triton_w4a16 import (
|
||||
TritonW4A16LinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.xpu import (
|
||||
XPUW4A8IntLinearKernel,
|
||||
XPUwNa16LinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.zentorch import (
|
||||
ZentorchWNA16LinearKernel,
|
||||
)
|
||||
@@ -91,9 +87,6 @@ from vllm.model_executor.kernels.linear.mxfp4.marlin import (
|
||||
from vllm.model_executor.kernels.linear.mxfp4.pytorch import (
|
||||
TorchMxFp4LinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mxfp4.xpu import (
|
||||
XPUMxFp4LinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mxfp6 import (
|
||||
MxFp6LinearKernel,
|
||||
MxFp6LinearLayerConfig,
|
||||
@@ -132,9 +125,6 @@ from vllm.model_executor.kernels.linear.mxfp8.pytorch import (
|
||||
from vllm.model_executor.kernels.linear.mxfp8.rocm_native import (
|
||||
RocmDotScaledMxfp8LinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mxfp8.xpu import (
|
||||
XPUMxFp8LinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.nvfp4 import (
|
||||
NvFp4LinearKernel,
|
||||
NvFp4LinearLayerConfig,
|
||||
@@ -228,11 +218,6 @@ from vllm.model_executor.kernels.linear.scaled_mm.triton import (
|
||||
TritonFP8ScaledMMLinearKernel,
|
||||
TritonInt8ScaledMMLinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.scaled_mm.xpu import (
|
||||
XPUFp8BlockScaledMMKernel,
|
||||
XPUW8A8FP8LinearKernel,
|
||||
XPUW8A16FP8LinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.scaled_mm.zentorch import (
|
||||
ZentorchInt8ScaledMMLinearKernel,
|
||||
)
|
||||
@@ -373,13 +358,6 @@ _LINEAR_BACKEND_KERNEL_MAP: dict[str, set[type]] = {
|
||||
EmulationMxfp6LinearKernel,
|
||||
EmulationMxfp4LinearKernel,
|
||||
},
|
||||
"xpu": {
|
||||
XPUW8A8FP8LinearKernel,
|
||||
XPUFp8BlockScaledMMKernel,
|
||||
},
|
||||
"xpu_woq": {
|
||||
XPUW8A16FP8LinearKernel,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@@ -444,7 +422,6 @@ _POSSIBLE_INT8_KERNELS: dict[PlatformEnum, list[type[Int8ScaledMMLinearKernel]]]
|
||||
HummingInt8ScaledMMLinearKernel,
|
||||
],
|
||||
PlatformEnum.ROCM: [AiterInt8ScaledMMLinearKernel, TritonInt8ScaledMMLinearKernel],
|
||||
PlatformEnum.XPU: [TritonInt8ScaledMMLinearKernel],
|
||||
}
|
||||
|
||||
# in priority/performance order (when available)
|
||||
@@ -473,12 +450,6 @@ _POSSIBLE_FP8_KERNELS: dict[PlatformEnum, list[type[FP8ScaledMMLinearKernel]]] =
|
||||
PerTensorTorchFP8ScaledMMLinearKernel,
|
||||
ChannelWiseTorchFP8ScaledMMLinearKernel,
|
||||
],
|
||||
PlatformEnum.XPU: [
|
||||
XPUW8A16FP8LinearKernel,
|
||||
XPUW8A8FP8LinearKernel,
|
||||
PerTensorTorchFP8ScaledMMLinearKernel,
|
||||
ChannelWiseTorchFP8ScaledMMLinearKernel,
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
@@ -504,11 +475,6 @@ _POSSIBLE_FP8_BLOCK_KERNELS: dict[
|
||||
CPUFp8W8A8BlockScaledMMKernel, # W8A8 preferred; falls back to W8A16 below
|
||||
CPUFp8BlockScaledMMKernel,
|
||||
],
|
||||
PlatformEnum.XPU: [
|
||||
XPUFp8BlockScaledMMKernel,
|
||||
TritonFp8BlockScaledMMKernel,
|
||||
BlockWiseTorchFP8ScaledMMLinearKernel,
|
||||
],
|
||||
}
|
||||
|
||||
_POSSIBLE_WFP8A16_KERNELS: dict[PlatformEnum, list[type[FP8ScaledMMLinearKernel]]] = {
|
||||
@@ -522,9 +488,6 @@ _POSSIBLE_WFP8A16_KERNELS: dict[PlatformEnum, list[type[FP8ScaledMMLinearKernel]
|
||||
PlatformEnum.CPU: [
|
||||
# To be added
|
||||
],
|
||||
PlatformEnum.XPU: [
|
||||
XPUW8A16FP8LinearKernel,
|
||||
],
|
||||
}
|
||||
|
||||
# in priority/performance order (when available)
|
||||
@@ -545,10 +508,6 @@ _POSSIBLE_KERNELS: dict[PlatformEnum, list[type[MPLinearKernel]]] = {
|
||||
ConchLinearKernel,
|
||||
ExllamaLinearKernel,
|
||||
],
|
||||
PlatformEnum.XPU: [
|
||||
XPUW4A8IntLinearKernel,
|
||||
XPUwNa16LinearKernel,
|
||||
],
|
||||
PlatformEnum.CPU: [
|
||||
Dynamic4bitLinearKernel,
|
||||
ZentorchWNA16LinearKernel,
|
||||
@@ -574,11 +533,6 @@ _POSSIBLE_MXFP8_KERNELS: dict[PlatformEnum, list[type[Mxfp8LinearKernel]]] = {
|
||||
RocmDotScaledMxfp8LinearKernel,
|
||||
EmulationMxfp8LinearKernel,
|
||||
],
|
||||
PlatformEnum.XPU: [
|
||||
XPUMxFp8LinearKernel,
|
||||
TorchMxFp8LinearKernel,
|
||||
EmulationMxfp8LinearKernel,
|
||||
],
|
||||
}
|
||||
|
||||
_POSSIBLE_NVFP4_KERNELS: dict[PlatformEnum, list[type[NvFp4LinearKernel]]] = {
|
||||
@@ -626,10 +580,6 @@ _POSSIBLE_MXFP4_KERNELS: dict[PlatformEnum, list[type[MxFp4LinearKernel]]] = {
|
||||
AiterMxfp4LinearKernel,
|
||||
EmulationMxfp4LinearKernel,
|
||||
],
|
||||
PlatformEnum.XPU: [
|
||||
XPUMxFp4LinearKernel,
|
||||
TorchMxFp4LinearKernel,
|
||||
],
|
||||
}
|
||||
|
||||
# TODO make all kernels inherit from MMLinearKernel
|
||||
@@ -1348,7 +1298,6 @@ __all__ = [
|
||||
"MacheteLinearKernel",
|
||||
"MarlinLinearKernel",
|
||||
"TritonW4A16LinearKernel",
|
||||
"XPUW4A8IntLinearKernel",
|
||||
"XPUwNa16LinearKernel",
|
||||
"init_mxfp8_linear_kernel",
|
||||
"Mxfp8LinearKernel",
|
||||
@@ -1373,7 +1322,6 @@ __all__ = [
|
||||
"FlashInferCutlassMxfp8LinearKernel",
|
||||
"FlashInferTrtllmMxfp8LinearKernel",
|
||||
"MarlinMxfp8LinearKernel",
|
||||
"XPUMxFp8LinearKernel",
|
||||
"TorchMxFp8LinearKernel",
|
||||
"EmulationMxfp8LinearKernel",
|
||||
"CutlassNvFp4LinearKernel",
|
||||
|
||||
@@ -38,10 +38,6 @@ from vllm.model_executor.kernels.linear.mixed_precision.rdna_hybrid_w4a16 import
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.triton_w4a16 import (
|
||||
TritonW4A16LinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.xpu import (
|
||||
XPUW4A8IntLinearKernel,
|
||||
XPUwNa16LinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.zentorch import (
|
||||
ZentorchWNA16LinearKernel,
|
||||
)
|
||||
@@ -60,7 +56,6 @@ __all__ = [
|
||||
"RDNA3W4A16LinearKernel",
|
||||
"RDNAHybridW4A16LinearKernel",
|
||||
"TritonW4A16LinearKernel",
|
||||
"XPUW4A8IntLinearKernel",
|
||||
"XPUwNa16LinearKernel",
|
||||
"ZentorchWNA16LinearKernel",
|
||||
]
|
||||
|
||||
@@ -1,218 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
import torch
|
||||
from torch.nn.parameter import Parameter
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization.utils import replace_parameter
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.scalar_type import scalar_types
|
||||
|
||||
from .MPLinearKernel import MPLinearKernel, MPLinearLayerConfig
|
||||
|
||||
_XPUWNA16_SUPPORTED_QUANT_TYPES = (scalar_types.uint4, scalar_types.uint4b8)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class XPUwNa16LinearKernel(MPLinearKernel):
|
||||
@classmethod
|
||||
def get_min_capability(cls) -> int:
|
||||
return -1
|
||||
|
||||
@classmethod
|
||||
def can_implement(cls, c: MPLinearLayerConfig) -> tuple[bool, str | None]:
|
||||
if not current_platform.is_xpu():
|
||||
return False, "XPUwNa16 only supported on XPU"
|
||||
|
||||
if c.act_type != torch.bfloat16 and c.act_type != torch.float16:
|
||||
return False, "XPUwNa16 only supports BF16/FP16 activations"
|
||||
|
||||
if c.weight_type not in _XPUWNA16_SUPPORTED_QUANT_TYPES:
|
||||
return (
|
||||
False,
|
||||
f"Quant type ({c.weight_type}) not supported by "
|
||||
"XPUwNa16, supported types are: "
|
||||
f"{_XPUWNA16_SUPPORTED_QUANT_TYPES}",
|
||||
)
|
||||
if c.group_size != -1 and c.group_size % 32 != 0:
|
||||
return (
|
||||
False,
|
||||
f"Group size ({c.group_size}) not supported by "
|
||||
"XPUwNa16, supported group sizes are multiples of 32",
|
||||
)
|
||||
|
||||
if c.partition_weight_shape[0] % 32 != 0:
|
||||
return (
|
||||
False,
|
||||
f"Input size ({c.partition_weight_shape[0]}) not supported by "
|
||||
"XPUwNa16, supported sizes are multiples of 32",
|
||||
)
|
||||
|
||||
return True, None
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module):
|
||||
# Default name since marlin requires empty parameter for zp,
|
||||
# TODO: remove this requirement from marlin (allow optional tensors)
|
||||
if self.w_zp_name is None:
|
||||
self.w_zp_name = "w_zp"
|
||||
|
||||
need_transpose = False
|
||||
qweight_shape = getattr(layer, self.w_q_name).shape
|
||||
scale_shape = getattr(layer, self.w_s_name).shape
|
||||
# gptq marlin and compressed tensors wna16 expect different default
|
||||
# layouts for weight and scale, so we check the shapes to determine
|
||||
# if we need to transpose
|
||||
if qweight_shape[0] != scale_shape[0]:
|
||||
need_transpose = True
|
||||
|
||||
if need_transpose:
|
||||
getattr(layer, self.w_q_name).data = (
|
||||
getattr(layer, self.w_q_name).data.t().contiguous()
|
||||
)
|
||||
getattr(layer, self.w_s_name).data = getattr(layer, self.w_s_name).data
|
||||
else:
|
||||
getattr(layer, self.w_s_name).data = (
|
||||
getattr(layer, self.w_s_name).data.t().contiguous()
|
||||
)
|
||||
|
||||
if self.config.zero_points:
|
||||
# (FIXME): maybe zero points should also be transposed.
|
||||
getattr(layer, self.w_zp_name).data = (
|
||||
getattr(layer, self.w_zp_name).data.t().contiguous()
|
||||
)
|
||||
else:
|
||||
weight_zero_point = torch.Tensor([8]).to(torch.int8).to("xpu")
|
||||
setattr(
|
||||
layer, self.w_zp_name, Parameter(weight_zero_point, requires_grad=False)
|
||||
)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
reshaped_x = x.reshape(-1, x.shape[-1])
|
||||
w_q, w_s, w_zp = self._get_weight_params(layer)
|
||||
out = torch.ops._xpu_C.int4_gemm_w4a16(
|
||||
reshaped_x,
|
||||
w_q.t(),
|
||||
bias if bias is not None else None,
|
||||
w_s,
|
||||
w_zp,
|
||||
self.config.group_size,
|
||||
None, # Retained by the external XPU op ABI.
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
class XPUW4A8IntLinearKernel(MPLinearKernel):
|
||||
"""XPU kernel for W4A8 integer quantization using oneDNN int4_gemm_w4a8.
|
||||
|
||||
Weights are symmetric group-quantized int4 packed as uint4.
|
||||
Activations are dynamically quantized per-token to symmetric int8.
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def get_min_capability(cls) -> int:
|
||||
return -1
|
||||
|
||||
@classmethod
|
||||
def can_implement(cls, c: MPLinearLayerConfig) -> tuple[bool, str | None]:
|
||||
if not current_platform.is_xpu():
|
||||
return False, "XPUW4A8Int only supported on XPU"
|
||||
if c.act_type not in (torch.bfloat16, torch.float16):
|
||||
return False, "XPUW4A8Int requires BF16/FP16 activations"
|
||||
if c.weight_type != scalar_types.int4:
|
||||
return (
|
||||
False,
|
||||
f"XPUW4A8Int requires int4 weights, got {c.weight_type}",
|
||||
)
|
||||
if c.zero_points:
|
||||
return False, "XPUW4A8Int only supports symmetric weight quantization"
|
||||
if c.group_size != -1 and c.group_size % 32 != 0:
|
||||
return (
|
||||
False,
|
||||
f"Group size ({c.group_size}) not supported by XPUW4A8Int, "
|
||||
"must be a multiple of 32",
|
||||
)
|
||||
in_size, out_size = c.partition_weight_shape
|
||||
if in_size % 8 != 0 or out_size % 8 != 0:
|
||||
return (
|
||||
False,
|
||||
f"in/out sizes ({in_size}, {out_size}) must be multiples of 8",
|
||||
)
|
||||
|
||||
if c.act_type != torch.float16:
|
||||
logger.warning_once(
|
||||
"XPUW4A8IntLinearKernel is running with model dtype %s, "
|
||||
"but int4_gemm_w4a8 produces float16 output. Recommend "
|
||||
"setting --dtype float16 for best performance.",
|
||||
c.act_type,
|
||||
)
|
||||
|
||||
return True, None
|
||||
|
||||
def _pack_int4_weight(self, w: torch.Tensor) -> torch.Tensor:
|
||||
# w is [N, K] int8 with values in [-8, 7]
|
||||
w_u4 = w.to(torch.int32) + 8 # shift to [0, 15]
|
||||
w_u4 = w_u4.reshape(w.shape[0], w.shape[1] // 8, 8) # [N, K/8, 8]
|
||||
shifts = torch.arange(0, 32, 4, dtype=torch.int32, device=w.device)
|
||||
packed = ((w_u4 & 0xF) << shifts[None, None, :]).sum(dim=2).to(torch.int32)
|
||||
return packed
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
layer.weight_scale.data = layer.weight_scale.data.t().contiguous()
|
||||
|
||||
device = layer.weight_packed.device
|
||||
# TODO: support asymmetric quantization
|
||||
weight_zero_point = torch.tensor([8], dtype=torch.int8, device=device)
|
||||
layer.weight_zero_point = Parameter(weight_zero_point, requires_grad=False)
|
||||
|
||||
# weight_packed is [out, in] int8, signed int4 values in [-8, 7]
|
||||
w = layer.weight_packed.data # [out, in]
|
||||
|
||||
# TODO: implement asym case
|
||||
packed = self._pack_int4_weight(w) # [out, in/8] packed uint4
|
||||
|
||||
replace_parameter(
|
||||
layer,
|
||||
self.w_q_name,
|
||||
torch.nn.Parameter(packed, requires_grad=False),
|
||||
)
|
||||
|
||||
# Free the original unpacked int8 weight (still registered as "weight")
|
||||
# to avoid double-storing both int8 [N, K] and int32 [N, K/8] in memory.
|
||||
delattr(layer, "weight")
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
reshaped_x = x.reshape(-1, x.shape[-1]) # [M, K]
|
||||
from vllm._xpu_ops import xpu_ops as ops
|
||||
|
||||
# TODO: static and asymmetric quantization case
|
||||
# Common code for CompressedTensorsW4A8Int does not read act symmetry data
|
||||
quant_x, x_scale, x_zero = ops.dynamic_per_token_int8_quant_ref(
|
||||
reshaped_x, True, 8
|
||||
)
|
||||
|
||||
out = torch.ops._xpu_C.int4_gemm_w4a8(
|
||||
quant_x,
|
||||
x_scale,
|
||||
x_zero,
|
||||
layer.weight_packed.t(),
|
||||
layer.weight_scale,
|
||||
layer.weight_zero_point,
|
||||
self.config.group_size,
|
||||
None, # Retained by the external XPU op ABI.
|
||||
bias,
|
||||
)
|
||||
|
||||
return out.to(x.dtype)
|
||||
@@ -1,58 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
|
||||
xpu_mxfp4_quantize as quant_mxfp4,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kMxfp4Dynamic,
|
||||
)
|
||||
from vllm.model_executor.utils import replace_parameter
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from .base import MxFp4LinearKernel, MxFp4LinearLayerConfig
|
||||
|
||||
|
||||
class XPUMxFp4LinearKernel(MxFp4LinearKernel):
|
||||
"""MXFP4 W4A4 GEMM on XPU."""
|
||||
|
||||
@classmethod
|
||||
def is_supported(
|
||||
cls, compute_capability: int | None = None
|
||||
) -> tuple[bool, str | None]:
|
||||
if not current_platform.is_xpu():
|
||||
return False, "XPUMxFp4 only support on XPU"
|
||||
return True, None
|
||||
|
||||
@classmethod
|
||||
def can_implement(cls, config: MxFp4LinearLayerConfig) -> tuple[bool, str | None]:
|
||||
if config.activation_quant_key != kMxfp4Dynamic:
|
||||
return False, "only supports MXFP4 dynamic activation"
|
||||
return True, None
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
weight = layer.weight.view(torch.float4_e2m1fn_x2)
|
||||
replace_parameter(layer, "weight", weight.data.t())
|
||||
|
||||
weight_scale = layer.weight_scale.view(torch.float8_e8m0fnu)
|
||||
weight_scale = weight_scale.t().contiguous()
|
||||
replace_parameter(layer, "weight_scale", weight_scale.data)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
out_dtype = x.dtype
|
||||
x_fp4, x_blockscale = quant_mxfp4(x)
|
||||
return torch.ops._xpu_C.fp4_gemm(
|
||||
x_fp4,
|
||||
layer.weight,
|
||||
x_blockscale,
|
||||
layer.weight_scale,
|
||||
out_dtype,
|
||||
bias,
|
||||
)
|
||||
@@ -1,82 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
|
||||
xpu_mxfp8_quantize as quant_mxfp8,
|
||||
)
|
||||
from vllm.model_executor.utils import replace_parameter
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from .Mxfp8LinearKernel import Mxfp8LinearKernel, Mxfp8LinearLayerConfig
|
||||
|
||||
|
||||
class XPUMxFp8LinearKernel(Mxfp8LinearKernel):
|
||||
"""MXFP8 W8A8 GEMM on XPU."""
|
||||
|
||||
@classmethod
|
||||
def is_supported(
|
||||
cls, compute_capability: int | None = None
|
||||
) -> tuple[bool, str | None]:
|
||||
if not current_platform.is_xpu():
|
||||
return False, "XPUMxFp8 only support on XPU"
|
||||
return True, None
|
||||
|
||||
@classmethod
|
||||
def can_implement(cls, c: Mxfp8LinearLayerConfig) -> tuple[bool, str | None]:
|
||||
return True, None
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
# Checkpoint scale is [N, K//32] (one E8M0 scale per 32 elements).
|
||||
# oneDNN fp8_gemm requires contiguous [K//32, N] layout. Store the
|
||||
# transposed contiguous buffer as a .t() view so that:
|
||||
# - dequantization consumers still see the checkpoint shape
|
||||
# - apply_weights recovers the oneDNN layout via .t() at zero cost
|
||||
weight_scale = layer.weight_scale.view(torch.float8_e8m0fnu)
|
||||
scale_kn = weight_scale.data.t().contiguous()
|
||||
replace_parameter(layer, "weight_scale", scale_kn.t())
|
||||
|
||||
if getattr(layer, "is_bmm", False):
|
||||
self._prepare_bmm_params(layer, scale_kn)
|
||||
|
||||
def _prepare_bmm_params(
|
||||
self, layer: torch.nn.Module, scale_kn: torch.Tensor
|
||||
) -> None:
|
||||
"""Precompute batched weight and scale for grouped fp8_bmm (e.g. wo_a).
|
||||
|
||||
Splits scale [K//32, N_total] into [G, K//32, N_per_group] and weight
|
||||
[N_total, K] into contiguous [G, K, N_per_group] for batch GEMM.
|
||||
"""
|
||||
batch = layer.bmm_batch_size
|
||||
k_blocks, n_blocks = scale_kn.shape
|
||||
layer.bmm_scale = (
|
||||
scale_kn.reshape(k_blocks, batch, n_blocks // batch)
|
||||
.permute(1, 0, 2)
|
||||
.contiguous()
|
||||
)
|
||||
w = layer.weight
|
||||
n_total, k = w.shape
|
||||
layer.bmm_weight = (
|
||||
w.reshape(batch, n_total // batch, k).permute(0, 2, 1).contiguous()
|
||||
)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
out_dtype = x.dtype
|
||||
x_fp8, x_scale = quant_mxfp8(x)
|
||||
# Weight is [N, K]; .t() gives a [K, N] view without copying.
|
||||
# Scale is stored as [N, K//32]; .t() recovers the contiguous
|
||||
# [K//32, N] buffer that oneDNN expects.
|
||||
return torch.ops._xpu_C.fp8_gemm(
|
||||
x_fp8,
|
||||
layer.weight.t(),
|
||||
out_dtype,
|
||||
x_scale,
|
||||
layer.weight_scale.t(),
|
||||
bias,
|
||||
)
|
||||
@@ -42,9 +42,6 @@ from vllm.model_executor.kernels.linear.scaled_mm.triton import (
|
||||
TritonFP8ScaledMMLinearKernel,
|
||||
TritonInt8ScaledMMLinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.scaled_mm.xpu import (
|
||||
XPUFp8BlockScaledMMKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.scaled_mm.zentorch import (
|
||||
ZentorchInt8ScaledMMLinearKernel,
|
||||
)
|
||||
@@ -73,5 +70,4 @@ __all__ = [
|
||||
"CPUFp8BlockScaledMMKernel",
|
||||
"CPUFp8PerTensorScaledMMLinearKernel",
|
||||
"CPUFP8W8A8ScaledMMLinearKernel",
|
||||
"XPUFp8BlockScaledMMKernel",
|
||||
]
|
||||
|
||||
@@ -1,284 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import math
|
||||
from collections.abc import Sequence
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kFp8DynamicTensorSym,
|
||||
kFp8DynamicTokenSym,
|
||||
kFp8StaticChannelSym,
|
||||
kFp8StaticTensorSym,
|
||||
kFp8StaticTokenSym,
|
||||
)
|
||||
from vllm.model_executor.utils import replace_parameter
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from .BlockScaledMMLinearKernel import Fp8BlockScaledMMLinearKernel
|
||||
from .ScaledMMLinearKernel import FP8ScaledMMLinearKernel, FP8ScaledMMLinearLayerConfig
|
||||
|
||||
|
||||
class XPUW8A8FP8LinearKernel(FP8ScaledMMLinearKernel):
|
||||
_SUPPORTED_ACT_QUANT_KEYS = {
|
||||
kFp8DynamicTensorSym,
|
||||
kFp8DynamicTokenSym,
|
||||
kFp8StaticTensorSym,
|
||||
kFp8StaticTokenSym,
|
||||
}
|
||||
_SUPPORTED_WEIGHT_QUANT_KEYS = {
|
||||
kFp8StaticChannelSym,
|
||||
kFp8StaticTensorSym,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def is_supported(
|
||||
cls, compute_capability: int | None = None
|
||||
) -> tuple[bool, str | None]:
|
||||
if not current_platform.is_xpu():
|
||||
return False, "XPUW8A8FP8Linear only support on XPU"
|
||||
return True, None
|
||||
|
||||
@classmethod
|
||||
def can_implement(cls, c: FP8ScaledMMLinearLayerConfig) -> tuple[bool, str | None]:
|
||||
if c.weight_quant_key not in cls._SUPPORTED_WEIGHT_QUANT_KEYS:
|
||||
return (
|
||||
False,
|
||||
"XPUW8A8FP8Linear only support per-channel and per-tensor quantization",
|
||||
)
|
||||
if c.activation_quant_key not in cls._SUPPORTED_ACT_QUANT_KEYS:
|
||||
return (
|
||||
False,
|
||||
"XPUW8A8FP8Linear only support per-tensor and per-token activation "
|
||||
"quantization",
|
||||
)
|
||||
if c.weight_quant_key.dtype not in {torch.float8_e5m2, torch.float8_e4m3fn}:
|
||||
return False, "XPUW8A8FP8Linear only support FP8 weight dtype"
|
||||
if c.activation_quant_key.dtype not in {
|
||||
torch.float8_e5m2,
|
||||
torch.float8_e4m3fn,
|
||||
}:
|
||||
return False, "XPUW8A8FP8Linear only support FP8 activation dtype"
|
||||
return True, None
|
||||
|
||||
def __init__(
|
||||
self, c: FP8ScaledMMLinearLayerConfig, layer_param_names: Sequence[str]
|
||||
) -> None:
|
||||
super().__init__(c, layer_param_names)
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
"""Ensure weight is stored as C-contiguous [K, N] (KN layout).
|
||||
|
||||
Checkpoints store weight as [N, K]; fp8_gemm requires [K, N],
|
||||
C-contiguous. Three incoming layouts are possible:
|
||||
• [N, K] C-contiguous ← direct checkpoint → .t().contiguous()
|
||||
• [K, N] Fortran-order ← fp8.py's weight.t() → .contiguous()
|
||||
• [K, N] C-contiguous ← already correct → no-op
|
||||
|
||||
For square weights (K == N) the shape is ambiguous; contiguity is used
|
||||
as a proxy: C-contiguous ≡ checkpoint [N, K] (needs transpose);
|
||||
Fortran-order ≡ fp8.py already transposed (needs only contiguous).
|
||||
"""
|
||||
K = getattr(layer, "input_size_per_partition", self.config.weight_shape[1])
|
||||
N = getattr(layer, "output_size_per_partition", self.config.weight_shape[0])
|
||||
w = layer.weight
|
||||
|
||||
if w.shape not in {(K, N), (N, K)}:
|
||||
raise ValueError(
|
||||
f"XPUFP8ScaledMM expects weight shape ({K},{N}) or ({N},{K}), "
|
||||
f"but got {tuple(w.shape)}"
|
||||
)
|
||||
|
||||
needs_transpose = w.shape == (N, K) if K != N else w.is_contiguous()
|
||||
layer_weight = w.t() if needs_transpose else w
|
||||
replace_parameter(layer, "weight", layer_weight)
|
||||
ws = layer.weight_scale
|
||||
if ws.numel() == 1:
|
||||
replace_parameter(layer, "weight_scale", ws.reshape(1))
|
||||
|
||||
def apply_scaled_mm(
|
||||
self,
|
||||
*,
|
||||
A: torch.Tensor,
|
||||
B: torch.Tensor,
|
||||
out_dtype: torch.dtype,
|
||||
As: torch.Tensor,
|
||||
Bs: torch.Tensor,
|
||||
bias: torch.Tensor | None,
|
||||
output_shape: list,
|
||||
) -> torch.Tensor:
|
||||
# B is C-contiguous [K, N] from process_weights_after_loading.
|
||||
# fp8_gemm routes on scale dtype (float32) and numel:
|
||||
# As [1] → per-tensor (numel==1 branch)
|
||||
# As [M,1] → per-token (group={1,K} branch, broadcast across K)
|
||||
# Bs [1] → per-tensor
|
||||
# Bs [N] → per-channel (mask=bit1 branch)
|
||||
# No shape manipulation needed here.
|
||||
output = torch.ops._xpu_C.fp8_gemm(A, B, out_dtype, As, Bs, bias)
|
||||
return output.view(*output_shape)
|
||||
|
||||
|
||||
class XPUW8A16FP8LinearKernel(FP8ScaledMMLinearKernel):
|
||||
@classmethod
|
||||
def is_supported(
|
||||
cls, compute_capability: int | None = None
|
||||
) -> tuple[bool, str | None]:
|
||||
if not current_platform.is_xpu():
|
||||
return False, "XPUW8A16FP8Linear only support on XPU"
|
||||
return True, None
|
||||
|
||||
@classmethod
|
||||
def can_implement(cls, c: FP8ScaledMMLinearLayerConfig) -> tuple[bool, str | None]:
|
||||
if c.weight_quant_key not in {kFp8StaticChannelSym, kFp8StaticTensorSym}:
|
||||
return (
|
||||
False,
|
||||
"XPUW8A16FP8Linear only support per-channel and per-tensor "
|
||||
"quantization",
|
||||
)
|
||||
if c.weight_quant_key.dtype not in {torch.float8_e5m2, torch.float8_e4m3fn}:
|
||||
return False, "XPUW8A16FP8Linear only support FP8 weight dtype"
|
||||
return True, None
|
||||
|
||||
def __init__(
|
||||
self, c: FP8ScaledMMLinearLayerConfig, layer_param_names: Sequence[str]
|
||||
) -> None:
|
||||
assert self.can_implement(c)[0]
|
||||
assert self.is_supported()[0]
|
||||
self.config = c
|
||||
self.layer_param_names = layer_param_names
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
# fp8_gemm_w8a16 expects weight in [in, out] layout.
|
||||
# Transpose if weight is still in [out, in] layout.
|
||||
# For square matrices, use contiguity as tie-breaker:
|
||||
# checkpoint weights are contiguous, .t() views are not.
|
||||
weight = layer.weight
|
||||
out_features, in_features = self.config.weight_shape
|
||||
|
||||
if weight.shape == (out_features, in_features) and (
|
||||
in_features != out_features or weight.is_contiguous()
|
||||
):
|
||||
replace_parameter(layer, "weight", weight.data.t())
|
||||
# else: already in [in, out] layout — no-op
|
||||
|
||||
weight_scale = layer.weight_scale.t().contiguous()
|
||||
replace_parameter(layer, "weight_scale", weight_scale.data)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
weight = layer.weight
|
||||
weight_scale = layer.weight_scale
|
||||
return torch.ops._xpu_C.fp8_gemm_w8a16(x, weight, weight_scale, bias)
|
||||
|
||||
def apply_scaled_mm(
|
||||
self,
|
||||
*,
|
||||
A: torch.Tensor,
|
||||
B: torch.Tensor,
|
||||
out_dtype: torch.dtype,
|
||||
As: torch.Tensor,
|
||||
Bs: torch.Tensor,
|
||||
bias: torch.Tensor | None,
|
||||
output_shape: list,
|
||||
) -> torch.Tensor:
|
||||
pass
|
||||
|
||||
|
||||
class XPUFp8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel):
|
||||
@classmethod
|
||||
def is_supported(
|
||||
cls, compute_capability: int | None = None
|
||||
) -> tuple[bool, str | None]:
|
||||
if not current_platform.is_xpu():
|
||||
return False, "XPUFp8BlockScaledMM only support on XPU"
|
||||
return True, None
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module):
|
||||
super().process_weights_after_loading(layer)
|
||||
scale_attr = (
|
||||
"weight_scale_inv" if hasattr(layer, "weight_scale_inv") else "weight_scale"
|
||||
)
|
||||
scale = getattr(layer, scale_attr)
|
||||
|
||||
# Ragged N (N % block_n != 0): oneDNN needs n_blocks to divide N.
|
||||
# Weight untouched; only repeat scale rows to a finer N-group gn that
|
||||
# divides both N and block_n (gn = gcd(N, block_n)):
|
||||
# scale [ceil(N/block_n), K/block_k] --> [N/gn, K/block_k]
|
||||
# oneDNN only accepts gn that is a multiple of 16, and gcd(N, block_n)
|
||||
# is a power of two (block_n=128), so gn must be >= 16. No-op when
|
||||
# N % block_n == 0.
|
||||
block_n, block_k = self.weight_group_shape
|
||||
N, K = layer.weight.shape
|
||||
if N % block_n != 0:
|
||||
gn = math.gcd(N, block_n)
|
||||
assert gn % 16 == 0, (
|
||||
f"XPU block-scaled FP8: N ({N}) yields group width {gn}, but "
|
||||
f"oneDNN only supports multiples of 16; this weight shape is "
|
||||
f"unsupported."
|
||||
)
|
||||
col_start = torch.arange(N // gn, device=scale.device) * gn
|
||||
src_idx = torch.div(col_start, block_n, rounding_mode="floor")
|
||||
scale = scale.index_select(0, src_idx).contiguous()
|
||||
|
||||
# Ragged K needs the runtime activation scale expanded too, which we
|
||||
# don't handle; DeepSeek/GLM keep K block-aligned, so fail loudly.
|
||||
assert K % block_k == 0, (
|
||||
f"XPU block-scaled FP8 requires K ({K}) to be a multiple of the "
|
||||
f"weight block size ({block_k}); ragged-K weights are unsupported."
|
||||
)
|
||||
|
||||
# Checkpoint scale is [n_blocks, k_blocks] (one value per block tile).
|
||||
# oneDNN fp8_gemm requires contiguous [k_blocks, n_blocks] layout.
|
||||
# We store the transposed contiguous buffer as a .t() view so that:
|
||||
# - MLA's scaled_dequantize still sees [n_blocks, k_blocks] shape
|
||||
# - apply_block_scaled_mm recovers the contiguous buffer via .t()
|
||||
scale_kn = scale.data.t().contiguous() # [k_blocks, n_blocks]
|
||||
replace_parameter(layer, scale_attr, scale_kn.t()) # view: [n_blocks, k_blocks]
|
||||
|
||||
if getattr(layer, "is_bmm", False):
|
||||
self._prepare_bmm_params(layer, scale_kn)
|
||||
|
||||
def _prepare_bmm_params(
|
||||
self, layer: torch.nn.Module, scale_kn: torch.Tensor
|
||||
) -> None:
|
||||
"""Precompute batched weight and scale for grouped fp8_bmm (e.g. wo_a).
|
||||
|
||||
Splits scale [k_blocks, n_blocks] into [G, k_blocks, n_blocks_per_group]
|
||||
and weight [N_total, K] into [G, K, N_per_group] for batch GEMM.
|
||||
"""
|
||||
batch = layer.bmm_batch_size
|
||||
k_blocks, n_blocks = scale_kn.shape
|
||||
layer.bmm_scale = (
|
||||
scale_kn.reshape(k_blocks, batch, n_blocks // batch)
|
||||
.permute(1, 0, 2)
|
||||
.contiguous()
|
||||
)
|
||||
w = layer.weight
|
||||
N_total, K = w.shape
|
||||
layer.bmm_weight = w.reshape(batch, N_total // batch, K).permute(
|
||||
0, 2, 1
|
||||
) # [G, K, N_per_group]
|
||||
|
||||
def apply_block_scaled_mm(
|
||||
self,
|
||||
A: torch.Tensor,
|
||||
B: torch.Tensor,
|
||||
As: torch.Tensor,
|
||||
Bs: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
# B is [N, K]; .t() gives [K, N] view (no copy).
|
||||
# Bs is stored as [n_blocks, k_blocks] view; .t() recovers the
|
||||
# contiguous [k_blocks, n_blocks] buffer that oneDNN expects.
|
||||
return torch.ops._xpu_C.fp8_gemm(
|
||||
A,
|
||||
B.t(),
|
||||
self.config.out_dtype,
|
||||
As,
|
||||
Bs.t(),
|
||||
torch.Tensor(),
|
||||
)
|
||||
Reference in New Issue
Block a user