[CPU] Add CPU FP8 W8A8 linear/MoE support (#49942)
Signed-off-by: yuwenzho <yuwen.zhou@intel.com> Co-authored-by: Li, Jiang <jiang1.li@intel.com>
This commit is contained in:
co-authored by
Li, Jiang <jiang1.li@intel.com>
parent
6df2b1a8c6
commit
b346479065
@@ -186,6 +186,8 @@ from vllm.model_executor.kernels.linear.scaled_mm.b12x import (
|
||||
from vllm.model_executor.kernels.linear.scaled_mm.cpu import (
|
||||
CPUFp8BlockScaledMMKernel,
|
||||
CPUFp8PerTensorScaledMMLinearKernel,
|
||||
CPUFp8W8A8BlockScaledMMKernel,
|
||||
CPUFP8W8A8ScaledMMLinearKernel,
|
||||
CPUInt8ScaledMMLinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.scaled_mm.cutlass import (
|
||||
@@ -436,6 +438,7 @@ _POSSIBLE_FP8_KERNELS: dict[PlatformEnum, list[type[FP8ScaledMMLinearKernel]]] =
|
||||
ChannelWiseTorchFP8ScaledMMLinearKernel,
|
||||
],
|
||||
PlatformEnum.CPU: [
|
||||
CPUFP8W8A8ScaledMMLinearKernel,
|
||||
CPUFp8PerTensorScaledMMLinearKernel,
|
||||
PerTensorTorchFP8ScaledMMLinearKernel,
|
||||
ChannelWiseTorchFP8ScaledMMLinearKernel,
|
||||
@@ -468,6 +471,7 @@ _POSSIBLE_FP8_BLOCK_KERNELS: dict[
|
||||
TritonFp8BlockScaledMMKernel,
|
||||
],
|
||||
PlatformEnum.CPU: [
|
||||
CPUFp8W8A8BlockScaledMMKernel, # W8A8 preferred; falls back to W8A16 below
|
||||
CPUFp8BlockScaledMMKernel,
|
||||
],
|
||||
PlatformEnum.XPU: [
|
||||
|
||||
@@ -10,6 +10,7 @@ from vllm.model_executor.kernels.linear.scaled_mm.BlockScaledMMLinearKernel impo
|
||||
from vllm.model_executor.kernels.linear.scaled_mm.cpu import (
|
||||
CPUFp8BlockScaledMMKernel,
|
||||
CPUFp8PerTensorScaledMMLinearKernel,
|
||||
CPUFP8W8A8ScaledMMLinearKernel,
|
||||
CPUInt8ScaledMMLinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.scaled_mm.cutlass import (
|
||||
@@ -69,5 +70,6 @@ __all__ = [
|
||||
"Fp8BlockScaledMMLinearKernel",
|
||||
"CPUFp8BlockScaledMMKernel",
|
||||
"CPUFp8PerTensorScaledMMLinearKernel",
|
||||
"CPUFP8W8A8ScaledMMLinearKernel",
|
||||
"XPUFp8BlockScaledMMKernel",
|
||||
]
|
||||
|
||||
@@ -270,12 +270,17 @@ class CPUFp8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel):
|
||||
# Skip the base class process (FP8 padding / fnuz normalization)
|
||||
# which is GPU-oriented. Instead, VNNI-prepack weights for AMX.
|
||||
params = self._get_layer_params(layer)
|
||||
packed_weight = torch.ops._C.convert_weight_packed(params.weight)
|
||||
replace_parameter(
|
||||
layer,
|
||||
params.WEIGHT,
|
||||
torch.nn.Parameter(packed_weight, requires_grad=False),
|
||||
)
|
||||
|
||||
# MLA's kv_b_proj (see `_cpu_skip_gemm_dispatch`): MLA reads this
|
||||
# weight to derive W_UK/W_UV, so keep the checkpoint [N, K] layout and
|
||||
# let the kernel pack on the fly instead.
|
||||
if not getattr(layer, "_cpu_skip_gemm_dispatch", False):
|
||||
packed_weight = torch.ops._C.convert_weight_packed(params.weight)
|
||||
replace_parameter(
|
||||
layer,
|
||||
params.WEIGHT,
|
||||
torch.nn.Parameter(packed_weight, requires_grad=False),
|
||||
)
|
||||
|
||||
# Re-wrap scale as a plain Parameter so the kernel can read it
|
||||
# without weight-loader metadata interfering.
|
||||
@@ -320,7 +325,7 @@ class CPUFp8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel):
|
||||
list(self.weight_group_shape),
|
||||
bias,
|
||||
x.dtype,
|
||||
True, # is_vnni (weight already prepacked)
|
||||
not getattr(layer, "_cpu_skip_gemm_dispatch", False), # is_vnni
|
||||
)
|
||||
return out.reshape(x.shape[:-1] + (out.size(-1),)) if x.dim() > 2 else out
|
||||
|
||||
@@ -430,3 +435,294 @@ class CPUFp8PerTensorScaledMMLinearKernel(FP8ScaledMMLinearKernel):
|
||||
raise NotImplementedError(
|
||||
"CPUFp8PerTensorScaledMMLinearKernel overrides apply_weights directly."
|
||||
)
|
||||
|
||||
|
||||
class CPUFp8W8A8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel):
|
||||
"""FP8 W8A8 block-quantized GEMM with dynamic per-token activation quant on CPU."""
|
||||
|
||||
# Activation quant is done inside fp8_scaled_mm_with_quant;
|
||||
# the base class must not apply a second quant step.
|
||||
apply_input_quant = False
|
||||
|
||||
@classmethod
|
||||
def is_supported(
|
||||
cls, compute_capability: int | None = None
|
||||
) -> tuple[bool, str | None]:
|
||||
if not current_platform.is_cpu():
|
||||
return False, "requires CPU platform."
|
||||
if not torch.cpu._is_amx_tile_supported():
|
||||
return False, "requires AMX tile support (Sapphire Rapids or newer)."
|
||||
if not ops.cpu_has_amx_fp8():
|
||||
return False, (
|
||||
"requires native AMX-FP8 hardware; falls back "
|
||||
"to CPUFp8BlockScaledMMKernel (W8A16) otherwise."
|
||||
)
|
||||
return True, None
|
||||
|
||||
@classmethod
|
||||
def can_implement(
|
||||
cls, config: FP8ScaledMMLinearLayerConfig
|
||||
) -> tuple[bool, str | None]:
|
||||
# Same block-shape requirements as the W8A16 sibling kernel.
|
||||
weight_gs = config.weight_quant_key.scale.group_shape
|
||||
if weight_gs.col <= 0 or weight_gs.col != 128:
|
||||
return False, (
|
||||
"CPU FP8 W8A8 block kernel requires K-dimension block size of 128, "
|
||||
f"got {weight_gs.col}."
|
||||
)
|
||||
if weight_gs.row <= 0 or weight_gs.row % 32 != 0:
|
||||
return False, (
|
||||
"CPU FP8 W8A8 block kernel requires N-dimension block size to be "
|
||||
f"a positive multiple of 32, got {weight_gs.row}."
|
||||
)
|
||||
if config.out_dtype not in (torch.bfloat16, torch.float32):
|
||||
return False, "Only bfloat16/float32 output dtype supported."
|
||||
n = config.weight_shape[0]
|
||||
if n % 32 != 0:
|
||||
# float8_linear_prepack_cpu tiles N in chunks of 32 and cannot
|
||||
# handle a remainder tile smaller than that.
|
||||
return False, f"requires weight output dim (N={n}) to be a multiple of 32."
|
||||
return True, None
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
"""Prepack block-quantized FP8 weight for W8A8 AMX BRGEMM.
|
||||
|
||||
For block-quant FP8 models weight is stored as [N, K] (NOT transposed,
|
||||
unlike non-block FP8 where vLLM stores [K, N]).
|
||||
|
||||
float8_linear_prepack_cpu expects:
|
||||
weight: [N, K]
|
||||
scales: [N, G] where G = K / block_k
|
||||
|
||||
weight_scale_inv from the checkpoint has shape [ceil(N/block_n), G],
|
||||
so we expand rows from ceil(N/block_n) to N via repeat_interleave.
|
||||
"""
|
||||
params = self._get_layer_params(layer)
|
||||
weight = params.weight # [N, K]
|
||||
|
||||
# MLA's kv_b_proj (see `_cpu_skip_gemm_dispatch`): MLA reads this
|
||||
# weight to derive W_UK/W_UV, so keep the checkpoint [N, K] layout and
|
||||
# its block scales; apply_weights falls back to the W8A16 op.
|
||||
if getattr(layer, "_cpu_skip_gemm_dispatch", False):
|
||||
return
|
||||
|
||||
weight_scale = (
|
||||
params.weight_scale_inv
|
||||
if params.weight_scale_inv is not None
|
||||
else params.weight_scale
|
||||
) # [ceil(N/block_n), ceil(K/block_k)]
|
||||
assert weight_scale is not None, "FP8 W8A8 block kernel requires weight scale."
|
||||
|
||||
N = weight.shape[0]
|
||||
block_n = self.weight_group_shape.row # e.g. 128
|
||||
|
||||
# Expand scale rows: [ceil(N/block_n), G] → [N, G]
|
||||
if weight_scale.size(0) != N:
|
||||
weight_scale_expanded = weight_scale.repeat_interleave(block_n, dim=0)[
|
||||
:N, :
|
||||
].contiguous()
|
||||
else:
|
||||
weight_scale_expanded = weight_scale.contiguous()
|
||||
|
||||
packed_weight, packed_scales = torch.ops._C.float8_linear_prepack_cpu(
|
||||
weight, weight_scale_expanded.to(torch.float32)
|
||||
)
|
||||
|
||||
scale_attr = (
|
||||
params.WEIGHT_SCALE_INV
|
||||
if params.weight_scale_inv is not None
|
||||
else params.WEIGHT_SCALE
|
||||
)
|
||||
replace_parameter(
|
||||
layer,
|
||||
params.WEIGHT,
|
||||
torch.nn.Parameter(packed_weight, requires_grad=False),
|
||||
)
|
||||
replace_parameter(
|
||||
layer,
|
||||
scale_attr,
|
||||
torch.nn.Parameter(packed_scales, requires_grad=False),
|
||||
)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
**kwargs,
|
||||
) -> torch.Tensor:
|
||||
params = self._get_layer_params(layer)
|
||||
weight_scale = (
|
||||
params.weight_scale_inv
|
||||
if params.weight_scale_inv is not None
|
||||
else params.weight_scale
|
||||
)
|
||||
|
||||
x_2d = x.reshape(-1, x.shape[-1]) if x.dim() > 2 else x
|
||||
if getattr(layer, "_cpu_skip_gemm_dispatch", False):
|
||||
# Unpacked weight: fp8_scaled_mm_with_quant has no is_vnni option,
|
||||
# so fall back to the W8A16 op (BF16 activation, block-scaled FP8
|
||||
# weight) for this layer.
|
||||
out = torch.ops._C.fp8_scaled_mm_cpu(
|
||||
x_2d,
|
||||
params.weight,
|
||||
weight_scale,
|
||||
list(self.weight_group_shape),
|
||||
bias,
|
||||
x.dtype,
|
||||
False, # is_vnni
|
||||
)
|
||||
return out.reshape(x.shape[:-1] + (out.size(-1),)) if x.dim() > 2 else out
|
||||
|
||||
# act_scales=None + channelwise=True → dynamic per-token activation quant
|
||||
out = ops.fp8_scaled_mm_with_quant(
|
||||
x_2d,
|
||||
None,
|
||||
True,
|
||||
params.weight,
|
||||
weight_scale,
|
||||
bias,
|
||||
x.dtype,
|
||||
)
|
||||
return out.reshape(x.shape[:-1] + (out.size(-1),)) if x.dim() > 2 else out
|
||||
|
||||
def apply_block_scaled_mm(
|
||||
self,
|
||||
A: torch.Tensor,
|
||||
B: torch.Tensor,
|
||||
As: torch.Tensor,
|
||||
Bs: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
raise NotImplementedError(
|
||||
"CPUFp8W8A8BlockScaledMMKernel overrides apply_weights directly."
|
||||
)
|
||||
|
||||
|
||||
class CPUFP8W8A8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
|
||||
"""FP8 W8A8 GEMM with dynamic per-token activation quantization on CPU."""
|
||||
|
||||
@classmethod
|
||||
def is_supported(
|
||||
cls, compute_capability: int | None = None
|
||||
) -> tuple[bool, str | None]:
|
||||
if not current_platform.is_cpu():
|
||||
return False, "requires CPU platform."
|
||||
if not torch.cpu._is_amx_tile_supported():
|
||||
return False, "requires AMX tile support (Sapphire Rapids or newer)."
|
||||
if not ops.cpu_has_amx_fp8():
|
||||
return False, (
|
||||
"requires native AMX-FP8 hardware (Diamond Rapids); falls back "
|
||||
"to the Torch FP8 kernels otherwise."
|
||||
)
|
||||
return True, None
|
||||
|
||||
@classmethod
|
||||
def can_implement(
|
||||
cls, config: FP8ScaledMMLinearLayerConfig
|
||||
) -> tuple[bool, str | None]:
|
||||
if config.out_dtype not in (torch.bfloat16, torch.float16, torch.float32):
|
||||
return False, "Only bfloat16/float16/float32 output dtype supported."
|
||||
# Reject block-quantized weights — those go to CPUFp8BlockScaledMMKernel
|
||||
weight_gs = config.weight_quant_key.scale.group_shape
|
||||
if weight_gs.col > 0 and weight_gs.row > 0:
|
||||
return False, "Block-quantized weights not handled by FP8 W8A8 kernel."
|
||||
n = config.weight_shape[0]
|
||||
if n % 32 != 0:
|
||||
# float8_linear_prepack_cpu tiles N in chunks of 32 and cannot
|
||||
# handle a remainder tile smaller than that.
|
||||
return False, f"requires weight output dim (N={n}) to be a multiple of 32."
|
||||
return True, None
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
"""Prepack weights with float8_linear_prepack_cpu (VNNI + block layout)."""
|
||||
w_name, ws_name, xs_name, _ = self.layer_param_names
|
||||
weight = getattr(layer, w_name)
|
||||
weight_scale = getattr(layer, ws_name)
|
||||
|
||||
# vLLM stores linear weight as [K, N], while the CPU prepack op expects [N, K].
|
||||
N = weight.shape[-1]
|
||||
|
||||
# Ensure scale is 2D [N, G]
|
||||
ws = weight_scale
|
||||
if ws is None:
|
||||
raise RuntimeError("FP8 W8A8 kernel requires weight scale.")
|
||||
ws = ws.to(torch.float32)
|
||||
|
||||
# Fused modules may carry per-shard scales (e.g. QKV/MLP shards).
|
||||
# Expand those to per-output-channel scales before prepack.
|
||||
if (
|
||||
len(getattr(layer, "logical_widths", [])) > 1
|
||||
and ws.dim() <= 1
|
||||
and ws.numel() == len(layer.logical_widths)
|
||||
):
|
||||
ws = convert_to_channelwise(ws.reshape(-1), layer.logical_widths)
|
||||
|
||||
if ws.dim() == 0:
|
||||
ws = ws.view(1, 1).expand(N, 1)
|
||||
elif ws.dim() == 1:
|
||||
if ws.numel() == 1:
|
||||
ws = ws.view(1, 1).expand(N, 1)
|
||||
elif ws.numel() == N:
|
||||
ws = ws.unsqueeze(1) # [N] -> [N, 1]
|
||||
else:
|
||||
raise RuntimeError(
|
||||
"FP8 W8A8 kernel got unsupported 1D weight scale shape: "
|
||||
f"{tuple(ws.shape)} for N={N}."
|
||||
)
|
||||
|
||||
# Align scale rows to output channels N.
|
||||
if ws.size(0) != N:
|
||||
if ws.size(0) == 1:
|
||||
ws = ws.expand(N, ws.size(1)).contiguous()
|
||||
else:
|
||||
raise RuntimeError(
|
||||
"FP8 W8A8 kernel got mismatched weight scale rows: "
|
||||
f"ws_rows={ws.size(0)}, expected N={N}."
|
||||
)
|
||||
|
||||
w_2d = weight.t().contiguous() # [N, K]
|
||||
packed_weight, packed_scales = torch.ops._C.float8_linear_prepack_cpu(w_2d, ws)
|
||||
|
||||
replace_parameter(
|
||||
layer,
|
||||
w_name,
|
||||
torch.nn.Parameter(packed_weight, requires_grad=False),
|
||||
)
|
||||
replace_parameter(
|
||||
layer,
|
||||
ws_name,
|
||||
torch.nn.Parameter(packed_scales, requires_grad=False),
|
||||
)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
**kwargs,
|
||||
) -> torch.Tensor:
|
||||
w_name, ws_name, xs_name, _ = self.layer_param_names
|
||||
weight = getattr(layer, w_name)
|
||||
weight_scale = getattr(layer, ws_name)
|
||||
# For static quantization, use pre-computed activation scale
|
||||
act_scales = getattr(layer, xs_name, None) if xs_name else None
|
||||
|
||||
x_2d = x.reshape(-1, x.shape[-1]) if x.dim() > 2 else x
|
||||
# channelwise=True: per-token (per-row) quantization when act_scales is None
|
||||
# channelwise=False: per-tensor quantization when act_scales is provided
|
||||
channelwise = act_scales is None
|
||||
out = ops.fp8_scaled_mm_with_quant(
|
||||
x_2d,
|
||||
act_scales,
|
||||
channelwise,
|
||||
weight,
|
||||
weight_scale,
|
||||
bias,
|
||||
x.dtype,
|
||||
)
|
||||
return out.reshape(x.shape[:-1] + (out.size(-1),)) if x.dim() > 2 else out
|
||||
|
||||
def apply_scaled_mm(self, A, B, out_dtype, As, Bs, bias=None, **kwargs):
|
||||
raise NotImplementedError(
|
||||
"CPUFP8W8A8ScaledMMLinearKernel uses apply_weights directly."
|
||||
)
|
||||
|
||||
@@ -1055,7 +1055,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
mqa_q_nope,
|
||||
self.impl._w_uk_packed, # type: ignore[attr-defined]
|
||||
True,
|
||||
None,
|
||||
self.impl._w_scale, # type: ignore[attr-defined]
|
||||
)
|
||||
mqa_ql_nope = mqa_ql_nope.transpose(0, 1)
|
||||
else:
|
||||
@@ -1401,7 +1401,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
x,
|
||||
self.impl._w_uv_packed, # type: ignore[attr-defined]
|
||||
True,
|
||||
None,
|
||||
self.impl._w_scale, # type: ignore[attr-defined]
|
||||
)
|
||||
else:
|
||||
# Multiply + Transpose (N, B, L) x (N, L, V)->(N, B, V)->(B, N, V)
|
||||
|
||||
@@ -15,6 +15,7 @@ from vllm._custom_ops import (
|
||||
convert_weight_packed_scale_zp,
|
||||
cpu_fused_moe,
|
||||
cpu_fused_moe_int8,
|
||||
cpu_has_amx_fp8,
|
||||
cpu_prepack_moe_weight,
|
||||
cpu_prepack_moe_weight_int8,
|
||||
fused_experts_cpu,
|
||||
@@ -467,6 +468,7 @@ class CPUExpertsFp8(mk.FusedMoEExpertsModular):
|
||||
RoutingMethodType.Default,
|
||||
RoutingMethodType.Renormalize,
|
||||
RoutingMethodType.RenormalizeNaive,
|
||||
RoutingMethodType.DeepSeekV3,
|
||||
RoutingMethodType.DeepseekV4,
|
||||
RoutingMethodType.DeepSeekV3,
|
||||
]
|
||||
@@ -1612,3 +1614,218 @@ class ZenCPUExpertsInt8(mk.FusedMoEExpertsModular):
|
||||
self._w1_scale_bf16,
|
||||
self._w2_scale_bf16,
|
||||
)
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# FP8 W8A8 MoE
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
def prepare_fp8_w8a8_moe_layer_for_cpu(
|
||||
w13: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
w13_scale: torch.Tensor,
|
||||
w2_scale: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""Prepack FP8 W8A8 MoE weights for CPU kernel."""
|
||||
E = w13.size(0)
|
||||
|
||||
packed_w13_list = []
|
||||
packed_w13_scale_list = []
|
||||
|
||||
for i in range(E):
|
||||
w1_exp = w13[i] # [2N, K] FP8
|
||||
ws_exp = w13_scale[i] # [2N, K/128] or [2N_blocks, K/128]
|
||||
|
||||
# Align scale rows to weight rows if necessary (for block-128 checkpoints
|
||||
# where scale may have fewer rows: [2N/128, K/128] → [2N, K/128])
|
||||
if ws_exp.size(0) < w1_exp.size(0):
|
||||
repeat_factor = w1_exp.size(0) // ws_exp.size(0)
|
||||
ws_exp = torch.repeat_interleave(ws_exp, repeat_factor, dim=0)
|
||||
ws_exp = ws_exp[: w1_exp.size(0), :].contiguous()
|
||||
|
||||
pw, ps = torch.ops._C.float8_linear_prepack_cpu(w1_exp, ws_exp)
|
||||
packed_w13_list.append(pw)
|
||||
packed_w13_scale_list.append(ps)
|
||||
|
||||
packed_w13 = torch.stack(packed_w13_list)
|
||||
packed_w13_scale = torch.stack(packed_w13_scale_list)
|
||||
|
||||
# w2 uses the W8A16 path (BF16 activation × FP8 weight), VNNI packed
|
||||
packed_w2 = torch.ops._C.convert_weight_packed(w2)
|
||||
return packed_w13, packed_w13_scale, packed_w2, w2_scale
|
||||
|
||||
|
||||
class CPUExpertsFp8W8A8(mk.FusedMoEExpertsModular):
|
||||
"""CPU FP8 W8A8 block-quantized modular MoE experts."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
moe_config: FusedMoEConfig,
|
||||
quant_config: FusedMoEQuantConfig,
|
||||
):
|
||||
super().__init__(moe_config, quant_config)
|
||||
|
||||
@property
|
||||
def expects_unquantized_inputs(self) -> bool:
|
||||
# Receives BF16 hidden states; quantization to FP8 is done inside apply()
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def activation_format() -> mk.FusedMoEActivationFormat:
|
||||
return mk.FusedMoEActivationFormat.Standard
|
||||
|
||||
@staticmethod
|
||||
def _supports_current_device() -> bool:
|
||||
# Requires native AMX-FP8 hardware.
|
||||
return current_platform.is_cpu() and cpu_has_amx_fp8()
|
||||
|
||||
@staticmethod
|
||||
def _supports_no_act_and_mul() -> bool:
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _supports_activation(activation: MoEActivation) -> bool:
|
||||
return activation == MoEActivation.SILU
|
||||
|
||||
@staticmethod
|
||||
def _supports_parallel_config(
|
||||
moe_parallel_config: FusedMoEParallelConfig,
|
||||
) -> bool:
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def _supports_quant_scheme(
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
) -> bool:
|
||||
# Block-quantized FP8.
|
||||
return (weight_key, activation_key) == (
|
||||
kFp8Static128BlockSym,
|
||||
kFp8Dynamic128Sym,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _supports_routing_method(
|
||||
routing_method: RoutingMethodType,
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
) -> bool:
|
||||
return routing_method in [
|
||||
RoutingMethodType.Default,
|
||||
RoutingMethodType.Renormalize,
|
||||
RoutingMethodType.RenormalizeNaive,
|
||||
RoutingMethodType.DeepSeekV3,
|
||||
RoutingMethodType.DeepseekV4,
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def _supports_router_logits_dtype(
|
||||
router_logits_dtype: torch.dtype | None,
|
||||
routing_method: RoutingMethodType,
|
||||
) -> bool:
|
||||
return True
|
||||
|
||||
def moe_problem_size(
|
||||
self,
|
||||
a1: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
) -> tuple[int, int, int, int, int]:
|
||||
# float8_linear_prepack_cpu blocks w1 into [E, Nc, Kc, block_k, block_n],
|
||||
# not the (E, N, K) layout the base implementation assumes -- N isn't
|
||||
# recoverable from that shape, so read it from moe_config instead.
|
||||
E = w1.shape[0]
|
||||
K = a1.size(-1)
|
||||
N = (
|
||||
self.moe_config.intermediate_size_per_partition
|
||||
* self.moe_config.w13_num_shards
|
||||
)
|
||||
M = a1.size(0) if a1.dim() == 2 else a1.size(1)
|
||||
topk = topk_ids.size(1)
|
||||
return E, M, N, K, topk
|
||||
|
||||
def workspace_shapes(
|
||||
self,
|
||||
M: int,
|
||||
N: int,
|
||||
K: int,
|
||||
topk: int,
|
||||
global_num_experts: int,
|
||||
local_num_experts: int,
|
||||
expert_tokens_meta: mk.ExpertTokensMetadata | None,
|
||||
activation: MoEActivation,
|
||||
) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]:
|
||||
# fused_experts_cpu manages its own scratch space.
|
||||
return (0,), (0,), (M, K)
|
||||
|
||||
def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce:
|
||||
return TopKWeightAndReduceNoOP()
|
||||
|
||||
def apply(
|
||||
self,
|
||||
output: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
activation: MoEActivation,
|
||||
global_num_experts: int,
|
||||
expert_map: torch.Tensor | None,
|
||||
a1q_scale: torch.Tensor | None,
|
||||
a2_scale: torch.Tensor | None,
|
||||
workspace13: torch.Tensor,
|
||||
workspace2: torch.Tensor,
|
||||
expert_tokens_meta: mk.ExpertTokensMetadata | None,
|
||||
apply_router_weight_on_input: bool,
|
||||
) -> None:
|
||||
if apply_router_weight_on_input:
|
||||
# fused_experts_cpu always applies topk_weights internally on
|
||||
# combine; MoEPrepareAndFinalizeNoDPEPModular.prepare() would
|
||||
# also pre-apply it to hidden_states, double-weighting the
|
||||
# output. Not needed by any CPU FP8 model today.
|
||||
raise NotImplementedError(
|
||||
"CPUExpertsFp8W8A8 does not support apply_router_weight_on_input=True."
|
||||
)
|
||||
|
||||
block_shape = (
|
||||
list(self.quant_config.block_shape)
|
||||
if self.quant_config.block_shape
|
||||
else (
|
||||
[
|
||||
self.quant_config._w1.shape.row,
|
||||
self.quant_config._w1.shape.col,
|
||||
]
|
||||
if self.quant_config._w1.shape is not None
|
||||
else None
|
||||
)
|
||||
)
|
||||
|
||||
# Quantize hidden_states (BF16) → FP8 with per-token scales
|
||||
hidden_states_2d = hidden_states.reshape(-1, hidden_states.shape[-1])
|
||||
x_fp8, x_scales = torch.ops._C.quantize_fp8e4m3_vec(
|
||||
hidden_states_2d, True, None
|
||||
)
|
||||
|
||||
fused_experts_cpu(
|
||||
output,
|
||||
x_fp8,
|
||||
w1,
|
||||
w2,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
CPUQuantMethod.FP8_W8A8,
|
||||
self.w1_scale, # w1_scale
|
||||
self.w2_scale, # w2_scale
|
||||
None, # w1_zero
|
||||
None, # w2_zero
|
||||
block_shape, # block_size
|
||||
None, # w1_bias
|
||||
None, # w2_bias
|
||||
None, # alpha
|
||||
None, # limit
|
||||
True, # is_vnni
|
||||
x_scales, # a1_scale: per-token FP8 activation scales
|
||||
)
|
||||
|
||||
@@ -57,7 +57,9 @@ class Fp8MoeBackend(Enum):
|
||||
VLLM_CUTLASS = "VLLM_CUTLASS"
|
||||
BATCHED_VLLM_CUTLASS = "BATCHED_VLLM_CUTLASS"
|
||||
XPU = "XPU"
|
||||
# CPU FP8 W8A16 (BF16 activations); CPU_W8A8 needs native AMX-FP8.
|
||||
CPU = "CPU"
|
||||
CPU_W8A8 = "CPU_W8A8"
|
||||
HPC = "HPC"
|
||||
# Dequantize-to-BF16 emulation for MXFP8 on devices without a native
|
||||
# MXFP8 MoE kernel (e.g. ROCm). Weights pass through unchanged here.
|
||||
@@ -92,6 +94,7 @@ def _get_priority_backends(
|
||||
Fp8MoeBackend.BATCHED_VLLM_CUTLASS,
|
||||
Fp8MoeBackend.BATCHED_TRITON,
|
||||
Fp8MoeBackend.XPU,
|
||||
Fp8MoeBackend.CPU_W8A8,
|
||||
Fp8MoeBackend.CPU,
|
||||
Fp8MoeBackend.HPC,
|
||||
]
|
||||
@@ -129,8 +132,10 @@ def _get_priority_backends(
|
||||
_move_to_front(_AVAILABLE_BACKENDS, Fp8MoeBackend.XPU)
|
||||
|
||||
if current_platform.is_cpu():
|
||||
# CPU platform uses FP8 W8A16 fused MoE kernel.
|
||||
# W8A8 first: it falls through to the W8A16 backend whenever the
|
||||
# hardware (AMX-FP8) or the config isn't supported.
|
||||
_move_to_front(_AVAILABLE_BACKENDS, Fp8MoeBackend.CPU)
|
||||
_move_to_front(_AVAILABLE_BACKENDS, Fp8MoeBackend.CPU_W8A8)
|
||||
|
||||
return _AVAILABLE_BACKENDS
|
||||
|
||||
@@ -238,6 +243,13 @@ def backend_to_kernel_cls(
|
||||
|
||||
return [CPUExpertsFp8]
|
||||
|
||||
elif backend == Fp8MoeBackend.CPU_W8A8:
|
||||
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
|
||||
CPUExpertsFp8W8A8,
|
||||
)
|
||||
|
||||
return [CPUExpertsFp8W8A8]
|
||||
|
||||
elif backend == Fp8MoeBackend.HPC:
|
||||
from vllm.model_executor.layers.fused_moe.hpc_moe import (
|
||||
HPCExperts,
|
||||
@@ -652,6 +664,14 @@ def convert_to_fp8_moe_kernel_format(
|
||||
)
|
||||
|
||||
w13, w2 = prepare_fp8_moe_layer_for_cpu(w13, w2)
|
||||
elif fp8_backend == Fp8MoeBackend.CPU_W8A8:
|
||||
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
|
||||
prepare_fp8_w8a8_moe_layer_for_cpu,
|
||||
)
|
||||
|
||||
w13, w13_scale, w2, w2_scale = prepare_fp8_w8a8_moe_layer_for_cpu(
|
||||
w13, w2, w13_scale, w2_scale
|
||||
)
|
||||
else:
|
||||
if fp8_backend not in [
|
||||
Fp8MoeBackend.TRITON,
|
||||
@@ -697,7 +717,15 @@ def make_fp8_moe_quant_config(
|
||||
In a future PR, we will have this function should be
|
||||
a method of the modular kernel itself.
|
||||
"""
|
||||
# MARLIN and CPU are mixed precision W8A16 config.
|
||||
if fp8_backend == Fp8MoeBackend.CPU_W8A8:
|
||||
return fp8_w8a8_moe_quant_config(
|
||||
w1_scale=w1_scale,
|
||||
w2_scale=w2_scale,
|
||||
a1_scale=a1_scale,
|
||||
block_shape=block_shape,
|
||||
)
|
||||
|
||||
# MARLIN and CPU (W8A16) are mixed precision W8A16 configs.
|
||||
if fp8_backend == Fp8MoeBackend.MARLIN or fp8_backend == Fp8MoeBackend.CPU:
|
||||
return fp8_w8a16_moe_quant_config(
|
||||
w1_scale=w1_scale,
|
||||
|
||||
@@ -737,11 +737,13 @@ class Fp8PtpcOnlineMoEMethod(_Fp8OnlineMoEBase):
|
||||
)
|
||||
# Reject backends whose make_fp8_moe_quant_config branch silently
|
||||
# drops per_act_token_quant / per_out_ch_quant or collapses scales:
|
||||
# MARLIN / CPU route through fp8_w8a16_moe_quant_config; FLASHINFER_*
|
||||
# fold scales into a per-tensor alpha (oracle/fp8.py).
|
||||
# MARLIN / CPU route through fp8_w8a16_moe_quant_config, CPU_W8A8
|
||||
# ignores both flags, and FLASHINFER_* fold scales into a per-tensor
|
||||
# alpha (oracle/fp8.py).
|
||||
if self.fp8_backend in (
|
||||
Fp8MoeBackend.MARLIN,
|
||||
Fp8MoeBackend.CPU,
|
||||
Fp8MoeBackend.CPU_W8A8,
|
||||
Fp8MoeBackend.FLASHINFER_CUTLASS,
|
||||
Fp8MoeBackend.FLASHINFER_TRTLLM,
|
||||
):
|
||||
|
||||
Reference in New Issue
Block a user