[XPU]feat: enable FP8 block-scaled quantization on XPU (#42952)
Signed-off-by: Ma Jian <jian1.ma@intel.com> Co-authored-by: Kunshang Ji <kunshang.ji@intel.com>
This commit is contained in:
@@ -307,6 +307,9 @@ _POSSIBLE_FP8_BLOCK_KERNELS: dict[
|
||||
PlatformEnum.CPU: [
|
||||
CPUFp8BlockScaledMMKernel,
|
||||
],
|
||||
PlatformEnum.XPU: [
|
||||
TritonFp8BlockScaledMMKernel,
|
||||
],
|
||||
}
|
||||
|
||||
_POSSIBLE_WFP8A16_KERNELS: dict[PlatformEnum, list[type[FP8ScaledMMLinearKernel]]] = {
|
||||
|
||||
@@ -159,8 +159,8 @@ class TritonInt8ScaledMMLinearKernel(CutlassInt8ScaledMMLinearKernel):
|
||||
class TritonFp8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel):
|
||||
@classmethod
|
||||
def is_supported(cls, compute_capability=None):
|
||||
if not current_platform.is_cuda_alike():
|
||||
return False, "only cuda like devices are supported."
|
||||
if not (current_platform.is_cuda_alike() or current_platform.is_xpu()):
|
||||
return False, "only cuda-like and xpu devices are supported."
|
||||
return True, None
|
||||
|
||||
def apply_block_scaled_mm(
|
||||
|
||||
@@ -173,7 +173,16 @@ class QuantFP8(CustomOp):
|
||||
scale_ub: torch.Tensor | None = None,
|
||||
use_triton: bool = False,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
# XPU can use same code path as CUDA.
|
||||
if self.is_group_quant and not self.static:
|
||||
from vllm.model_executor.layers.quantization.utils import fp8_utils
|
||||
|
||||
return fp8_utils.per_token_group_quant_fp8(
|
||||
x,
|
||||
group_size=self.group_size,
|
||||
column_major_scales=self.column_major_scales,
|
||||
dtype=_FP8_DTYPE,
|
||||
use_ue8m0=self.use_ue8m0,
|
||||
)
|
||||
return self.forward_cuda(x, scale, scale_ub, use_triton)
|
||||
|
||||
def forward_native(
|
||||
|
||||
@@ -573,7 +573,7 @@ def per_token_group_quant_fp8(
|
||||
shape = x.shape[:-1] + (x.shape[-1] // group_size,)
|
||||
x_s = torch.empty(shape, device=x.device, dtype=torch.float32)
|
||||
|
||||
# prefer CUDA kernel if available
|
||||
# prefer CUDA/XPU kernel if available
|
||||
# TODO(bnell): this causes some fp8 moe test to fail.
|
||||
if current_platform.is_cuda() and x.is_contiguous():
|
||||
torch.ops._C.per_token_group_fp8_quant(
|
||||
@@ -590,6 +590,12 @@ def per_token_group_quant_fp8(
|
||||
)
|
||||
return x_q, x_s
|
||||
|
||||
if current_platform.is_xpu() and x.is_contiguous():
|
||||
torch.ops._C.per_token_group_fp8_quant(
|
||||
x, x_q, x_s, group_size, eps, fp8_min, fp8_max, use_ue8m0
|
||||
)
|
||||
return x_q, x_s
|
||||
|
||||
# TRITON FALLBACK
|
||||
M = x.numel() // group_size
|
||||
N = group_size
|
||||
@@ -861,7 +867,7 @@ def w8a8_triton_block_scaled_mm(
|
||||
# Triton cannot currently bind E8M0 scale tensors directly. On ROCm,
|
||||
# DeepSeek-V4 checkpoints store block scales in exponent-only E8M0 format,
|
||||
# so decode them to fp32 before launching the kernel.
|
||||
if current_platform.is_rocm():
|
||||
if current_platform.is_rocm() or current_platform.is_xpu():
|
||||
if As.dtype == torch.float8_e8m0fnu:
|
||||
As = _upcast_e8m0_to_fp32(As).contiguous()
|
||||
if Bs.dtype == torch.float8_e8m0fnu:
|
||||
|
||||
Reference in New Issue
Block a user