[CPU] Add FP8 W8A16 MoE support (#41314)
Signed-off-by: yuwenzho <yuwen.zhou@intel.com>
This commit is contained in:
@@ -12,6 +12,7 @@ steps:
|
||||
- vllm/_custom_ops.py
|
||||
- tests/kernels/attention/test_cpu_attn.py
|
||||
- tests/kernels/moe/test_cpu_fused_moe.py
|
||||
- tests/kernels/moe/test_cpu_fp8_fused_moe.py
|
||||
- tests/kernels/test_onednn.py
|
||||
- tests/kernels/test_awq_int4_to_int8.py
|
||||
- tests/kernels/quantization/test_cpu_fp8_scaled_mm.py
|
||||
@@ -20,6 +21,7 @@ steps:
|
||||
bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 30m "
|
||||
pytest -x -v -s tests/kernels/attention/test_cpu_attn.py
|
||||
pytest -x -v -s tests/kernels/moe/test_cpu_fused_moe.py
|
||||
pytest -x -v -s tests/kernels/moe/test_cpu_fp8_fused_moe.py
|
||||
pytest -x -v -s tests/kernels/test_onednn.py
|
||||
pytest -x -v -s tests/kernels/test_awq_int4_to_int8.py
|
||||
pytest -x -v -s tests/kernels/quantization/test_cpu_fp8_scaled_mm.py"
|
||||
|
||||
@@ -0,0 +1,258 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for CPU FP8 W8A16 block-scaled fused MoE kernel."""
|
||||
|
||||
import math
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
if not current_platform.is_cpu():
|
||||
pytest.skip("skipping CPU-only tests", allow_module_level=True)
|
||||
|
||||
import vllm._custom_ops as ops # noqa: E402
|
||||
|
||||
if not hasattr(torch.ops._C, "fused_experts_cpu"):
|
||||
pytest.skip("fused_experts_cpu op not available", allow_module_level=True)
|
||||
|
||||
|
||||
BLOCK_SIZE = [128, 128] # [block_n, block_k]
|
||||
|
||||
# FP8 weight generation parameters
|
||||
_FP8_INFO = torch.finfo(torch.float8_e4m3fn)
|
||||
FP8_SCALE = _FP8_INFO.max # 448.0
|
||||
FACTOR_FOR_SCALE = 1e-3
|
||||
|
||||
# Tolerance for FP8 W8A16
|
||||
FP8_W8A16_ATOL = 1e-2
|
||||
FP8_W8A16_RTOL = 1e-2
|
||||
|
||||
|
||||
def _silu_and_mul(x: torch.Tensor) -> torch.Tensor:
|
||||
d = x.shape[-1] // 2
|
||||
return F.silu(x[..., :d]) * x[..., d:]
|
||||
|
||||
|
||||
def _block_dequant_weight(
|
||||
weight: torch.Tensor,
|
||||
scales: torch.Tensor,
|
||||
block_size: list[int],
|
||||
) -> torch.Tensor:
|
||||
"""Block-dequantize FP8 weight [E, N, K] → float [E, N, K].
|
||||
|
||||
Each (block_n × block_k) tile is multiplied by its per-block scale.
|
||||
"""
|
||||
E, N, K = weight.shape
|
||||
block_n, block_k = block_size
|
||||
pad_N = (block_n - N % block_n) % block_n
|
||||
pad_K = (block_k - K % block_k) % block_k
|
||||
|
||||
if pad_N > 0 or pad_K > 0:
|
||||
weight = F.pad(weight, (0, pad_K, 0, pad_N))
|
||||
|
||||
n_tiles = math.ceil(N / block_n)
|
||||
k_tiles = math.ceil(K / block_k)
|
||||
|
||||
weight_block = (
|
||||
weight.view(E, n_tiles, block_n, k_tiles, block_k)
|
||||
.permute(0, 1, 3, 2, 4)
|
||||
.float()
|
||||
.contiguous()
|
||||
)
|
||||
weight_scaled = (
|
||||
(weight_block * scales.view(E, n_tiles, k_tiles, 1, 1))
|
||||
.permute(0, 1, 3, 2, 4)
|
||||
.contiguous()
|
||||
)
|
||||
if pad_N > 0 or pad_K > 0:
|
||||
weight_scaled = weight_scaled.view(E, N + pad_N, K + pad_K)
|
||||
weight_scaled = weight_scaled[..., :N, :K].contiguous()
|
||||
else:
|
||||
weight_scaled = weight_scaled.view(E, N, K)
|
||||
return weight_scaled
|
||||
|
||||
|
||||
def ref_w8a16_block_fp8_moe(
|
||||
a: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
w1_s: torch.Tensor,
|
||||
w2_s: torch.Tensor,
|
||||
topk_weight: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
block_size: list[int],
|
||||
) -> torch.Tensor:
|
||||
"""Reference FP8 W8A16 block-scaled fused MoE in pure torch.
|
||||
|
||||
Steps:
|
||||
1. Block-dequant FP8 weights → float
|
||||
2. For each expert: matmul → SiLU+Mul → matmul
|
||||
3. Weighted sum across top-k experts
|
||||
"""
|
||||
B, D = a.shape
|
||||
topk = topk_ids.size(1)
|
||||
|
||||
w1_dq = _block_dequant_weight(w1, w1_s, block_size)
|
||||
w2_dq = _block_dequant_weight(w2, w2_s, block_size)
|
||||
|
||||
a_exp = a.view(B, -1, D).repeat(1, topk, 1).reshape(-1, D).float()
|
||||
out = torch.zeros(B * topk, w2_dq.shape[1], dtype=torch.float32)
|
||||
|
||||
topk_weight_flat = topk_weight.view(-1)
|
||||
topk_ids_flat = topk_ids.view(-1)
|
||||
|
||||
for i in range(w1_dq.shape[0]):
|
||||
mask = topk_ids_flat == i
|
||||
if mask.sum():
|
||||
ic0 = torch.matmul(a_exp[mask], w1_dq[i].transpose(0, 1))
|
||||
ic1 = _silu_and_mul(ic0)
|
||||
out[mask] = torch.matmul(ic1, w2_dq[i].transpose(0, 1))
|
||||
|
||||
return (
|
||||
(out.view(B, -1, w2_dq.shape[1]) * topk_weight_flat.view(B, -1, 1))
|
||||
.sum(dim=1)
|
||||
.to(a.dtype)
|
||||
)
|
||||
|
||||
|
||||
def _make_fp8_moe_weights(
|
||||
E: int,
|
||||
N: int,
|
||||
K: int,
|
||||
block_size: list[int],
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""Generate random FP8 MoE weights with random block scales.
|
||||
|
||||
Weight generation follows SGLang: ``randn * FP8_SCALE`` → clamp → cast.
|
||||
Scales are small random values (``FACTOR_FOR_SCALE``), independent of
|
||||
the actual weight magnitudes — this is sufficient to test the kernel's
|
||||
block-dequant + matmul correctness.
|
||||
|
||||
Returns: (w1, w2, w1_s, w2_s)
|
||||
"""
|
||||
block_n, block_k = block_size
|
||||
|
||||
w1 = (
|
||||
(torch.randn(E, 2 * N, K) * FP8_SCALE)
|
||||
.clamp(min=-FP8_SCALE, max=FP8_SCALE)
|
||||
.to(torch.float8_e4m3fn)
|
||||
)
|
||||
w2 = (
|
||||
(torch.randn(E, K, N) * FP8_SCALE)
|
||||
.clamp(min=-FP8_SCALE, max=FP8_SCALE)
|
||||
.to(torch.float8_e4m3fn)
|
||||
)
|
||||
|
||||
w1_s = (
|
||||
torch.randn(E, math.ceil(2 * N / block_n), math.ceil(K / block_k))
|
||||
* FACTOR_FOR_SCALE
|
||||
)
|
||||
w2_s = (
|
||||
torch.randn(E, math.ceil(K / block_n), math.ceil(N / block_k))
|
||||
* FACTOR_FOR_SCALE
|
||||
)
|
||||
return w1, w2, w1_s, w2_s
|
||||
|
||||
|
||||
def _prepack_experts(w: torch.Tensor) -> torch.Tensor:
|
||||
"""VNNI-prepack each expert's weight via ``convert_weight_packed``."""
|
||||
return torch.stack(
|
||||
[torch.ops._C.convert_weight_packed(w[e]) for e in range(w.shape[0])]
|
||||
)
|
||||
|
||||
|
||||
NUM_TOKENS = [1, 2, 64, 121]
|
||||
# (M, intermediate_size N, hidden_size K, num_experts E, topk)
|
||||
MoE_CONFIGS = [
|
||||
(256, 512, 8, 2),
|
||||
(256, 512, 8, 4),
|
||||
(512, 256, 8, 2),
|
||||
(512, 256, 8, 4),
|
||||
(512, 512, 8, 2),
|
||||
(512, 512, 8, 4),
|
||||
(768, 2048, 8, 2),
|
||||
(768, 2048, 8, 4),
|
||||
(768, 2048, 128, 8),
|
||||
]
|
||||
SEEDS = [0]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("M", NUM_TOKENS)
|
||||
@pytest.mark.parametrize("N,K,E,topk", MoE_CONFIGS)
|
||||
@pytest.mark.parametrize("seed", SEEDS)
|
||||
def test_w8a16_block_fp8_cpu_fused_moe(M, N, K, E, topk, seed):
|
||||
"""Test fused_experts_cpu FP8 W8A16 against dequantised torch reference."""
|
||||
set_random_seed(seed)
|
||||
|
||||
a = torch.randn(M, K, dtype=torch.bfloat16) / math.sqrt(K)
|
||||
w1, w2, w1_s, w2_s = _make_fp8_moe_weights(E, N, K, BLOCK_SIZE)
|
||||
|
||||
score = torch.randn(M, E, dtype=torch.bfloat16)
|
||||
score = torch.softmax(score, dim=-1, dtype=torch.float32)
|
||||
topk_weight, topk_ids = torch.topk(score, topk)
|
||||
topk_ids = topk_ids.to(torch.int32)
|
||||
|
||||
ref_out = ref_w8a16_block_fp8_moe(
|
||||
a,
|
||||
w1,
|
||||
w2,
|
||||
w1_s,
|
||||
w2_s,
|
||||
topk_weight,
|
||||
topk_ids,
|
||||
BLOCK_SIZE,
|
||||
)
|
||||
|
||||
pw1, pw2 = _prepack_experts(w1), _prepack_experts(w2)
|
||||
|
||||
# Test inplace=False against reference
|
||||
out = ops.fused_experts_cpu(
|
||||
a.clone(),
|
||||
pw1,
|
||||
pw2,
|
||||
topk_weight,
|
||||
topk_ids,
|
||||
False,
|
||||
False,
|
||||
True,
|
||||
w1_s,
|
||||
w2_s,
|
||||
BLOCK_SIZE,
|
||||
None,
|
||||
None,
|
||||
True,
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
ref_out.bfloat16(),
|
||||
out,
|
||||
atol=FP8_W8A16_ATOL,
|
||||
rtol=FP8_W8A16_RTOL,
|
||||
)
|
||||
|
||||
# Test inplace=True produces identical output
|
||||
out_inplace = ops.fused_experts_cpu(
|
||||
a.clone(),
|
||||
pw1,
|
||||
pw2,
|
||||
topk_weight,
|
||||
topk_ids,
|
||||
True,
|
||||
False,
|
||||
True,
|
||||
w1_s,
|
||||
w2_s,
|
||||
BLOCK_SIZE,
|
||||
None,
|
||||
None,
|
||||
True,
|
||||
)
|
||||
torch.testing.assert_close(out_inplace, out, atol=0, rtol=0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(pytest.main([__file__, "-v"]))
|
||||
@@ -13,7 +13,8 @@ MODELS = [
|
||||
"Qwen/Qwen1.5-0.5B-Chat-GPTQ-Int4", # without g_idx
|
||||
"RedHatAI/Qwen3-1.7B-quantized.w4a16", # with zp
|
||||
"OPEA/Qwen2.5-0.5B-Instruct-int4-sym-inc",
|
||||
"Qwen/Qwen3-0.6B-FP8", # FP8 W8A16 block-quantized
|
||||
"Qwen/Qwen3-0.6B-FP8", # FP8 W8A16 block-quantized linear
|
||||
"Qwen/Qwen3-30B-A3B-FP8", # FP8 W8A16 block-quantized MoE
|
||||
]
|
||||
DTYPE = ["bfloat16"]
|
||||
|
||||
|
||||
@@ -3172,6 +3172,40 @@ if hasattr(torch.ops._C, "fused_experts_cpu"):
|
||||
return torch.empty_like(hidden_states)
|
||||
|
||||
|
||||
def fused_experts_cpu(
|
||||
hidden_states: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
inplace: bool,
|
||||
use_int8_w8a8: bool,
|
||||
use_fp8_w8a16: bool,
|
||||
w1_scale: torch.Tensor | None,
|
||||
w2_scale: torch.Tensor | None,
|
||||
block_size: list[int] | None,
|
||||
a1_scale: torch.Tensor | None,
|
||||
a2_scale: torch.Tensor | None,
|
||||
is_vnni: bool,
|
||||
) -> torch.Tensor:
|
||||
return torch.ops._C.fused_experts_cpu(
|
||||
hidden_states,
|
||||
w1,
|
||||
w2,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
inplace,
|
||||
use_int8_w8a8,
|
||||
use_fp8_w8a16,
|
||||
w1_scale,
|
||||
w2_scale,
|
||||
block_size,
|
||||
a1_scale,
|
||||
a2_scale,
|
||||
is_vnni,
|
||||
)
|
||||
|
||||
|
||||
if hasattr(torch.ops._C, "int8_scaled_mm_with_quant"):
|
||||
|
||||
@register_fake("_C::int8_scaled_mm_with_quant")
|
||||
|
||||
@@ -0,0 +1,175 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""CPU FP8 W8A16 block-quantized fused MoE experts."""
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEParallelConfig,
|
||||
FusedMoEQuantConfig,
|
||||
RoutingMethodType,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kFp8Dynamic128Sym,
|
||||
kFp8Static128BlockSym,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
def prepare_fp8_moe_layer_for_cpu(
|
||||
w13: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""VNNI-prepack FP8 MoE weights for CPU kernel."""
|
||||
num_experts = w13.size(0)
|
||||
packed_w13_list = []
|
||||
packed_w2_list = []
|
||||
for i in range(num_experts):
|
||||
packed_w13_list.append(torch.ops._C.convert_weight_packed(w13[i]))
|
||||
packed_w2_list.append(torch.ops._C.convert_weight_packed(w2[i]))
|
||||
packed_w13 = torch.stack(packed_w13_list)
|
||||
packed_w2 = torch.stack(packed_w2_list)
|
||||
return packed_w13, packed_w2
|
||||
|
||||
|
||||
class CPUExpertsFp8(mk.FusedMoEExpertsMonolithic):
|
||||
"""CPU FP8 W8A16 block-quantized monolithic MoE experts."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
moe_config: FusedMoEConfig,
|
||||
quant_config: FusedMoEQuantConfig,
|
||||
):
|
||||
super().__init__(
|
||||
moe_config,
|
||||
quant_config,
|
||||
)
|
||||
|
||||
@property
|
||||
def expects_unquantized_inputs(self) -> bool:
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def activation_format() -> mk.FusedMoEActivationFormat:
|
||||
return mk.FusedMoEActivationFormat.Standard
|
||||
|
||||
@staticmethod
|
||||
def _supports_current_device() -> bool:
|
||||
return current_platform.is_cpu()
|
||||
|
||||
@staticmethod
|
||||
def _supports_no_act_and_mul() -> bool:
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _supports_activation(activation: MoEActivation) -> bool:
|
||||
return activation == MoEActivation.SILU
|
||||
|
||||
@staticmethod
|
||||
def _supports_parallel_config(
|
||||
moe_parallel_config: FusedMoEParallelConfig,
|
||||
) -> bool:
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def _supports_quant_scheme(
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
) -> bool:
|
||||
SUPPORTED_W_A = [
|
||||
(kFp8Static128BlockSym, kFp8Dynamic128Sym),
|
||||
]
|
||||
return (weight_key, activation_key) in SUPPORTED_W_A
|
||||
|
||||
@staticmethod
|
||||
def _supports_routing_method(
|
||||
routing_method: RoutingMethodType,
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
) -> bool:
|
||||
return routing_method in [
|
||||
RoutingMethodType.Default,
|
||||
RoutingMethodType.Renormalize,
|
||||
RoutingMethodType.RenormalizeNaive,
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def _supports_router_logits_dtype(
|
||||
router_logits_dtype: torch.dtype | None,
|
||||
routing_method: RoutingMethodType,
|
||||
) -> bool:
|
||||
return True
|
||||
|
||||
def supports_expert_map(self) -> bool:
|
||||
return False
|
||||
|
||||
def apply(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
activation: MoEActivation,
|
||||
global_num_experts: int,
|
||||
expert_map: torch.Tensor | None,
|
||||
a1q_scale: torch.Tensor | None,
|
||||
apply_router_weight_on_input: bool,
|
||||
# grouped topk + fused topk bias parameters
|
||||
num_expert_group: int | None = None,
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
routed_scaling_factor: float | None = None,
|
||||
topk_group: int | None = None,
|
||||
) -> torch.Tensor:
|
||||
from vllm.model_executor.layers.fused_moe.cpu_fused_moe import (
|
||||
select_experts,
|
||||
)
|
||||
|
||||
topk_weights, topk_ids = select_experts(
|
||||
hidden_states=hidden_states,
|
||||
router_logits=router_logits,
|
||||
use_grouped_topk=num_expert_group is not None,
|
||||
top_k=self.moe_config.experts_per_token,
|
||||
renormalize=self.moe_config.routing_method
|
||||
in (
|
||||
RoutingMethodType.Renormalize,
|
||||
RoutingMethodType.RenormalizeNaive,
|
||||
),
|
||||
topk_group=topk_group,
|
||||
num_expert_group=num_expert_group,
|
||||
scoring_func="softmax",
|
||||
routed_scaling_factor=(
|
||||
routed_scaling_factor if routed_scaling_factor is not None else 1.0
|
||||
),
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
)
|
||||
|
||||
block_shape = (
|
||||
list(self.quant_config.block_shape)
|
||||
if self.quant_config.block_shape
|
||||
else (
|
||||
[self.quant_config._w1.shape.row, self.quant_config._w1.shape.col]
|
||||
if self.quant_config._w1.shape is not None
|
||||
else None
|
||||
)
|
||||
)
|
||||
|
||||
return torch.ops._C.fused_experts_cpu(
|
||||
hidden_states,
|
||||
w1,
|
||||
w2,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
False, # inplace
|
||||
False, # use_int8_w8a8
|
||||
True, # use_fp8_w8a16
|
||||
self.w1_scale, # w1_scale
|
||||
self.w2_scale, # w2_scale
|
||||
block_shape, # block_size
|
||||
None, # a1_scale (W8A16: no activation quantization)
|
||||
None, # a2_scale
|
||||
True, # is_vnni
|
||||
)
|
||||
@@ -55,6 +55,7 @@ class Fp8MoeBackend(Enum):
|
||||
VLLM_CUTLASS = "VLLM_CUTLASS"
|
||||
BATCHED_VLLM_CUTLASS = "BATCHED_VLLM_CUTLASS"
|
||||
XPU = "XPU"
|
||||
CPU = "CPU"
|
||||
|
||||
|
||||
def _get_priority_backends(
|
||||
@@ -80,6 +81,7 @@ def _get_priority_backends(
|
||||
Fp8MoeBackend.BATCHED_VLLM_CUTLASS,
|
||||
Fp8MoeBackend.BATCHED_TRITON,
|
||||
Fp8MoeBackend.XPU,
|
||||
Fp8MoeBackend.CPU,
|
||||
]
|
||||
|
||||
def _move_to_front(backends: list[Fp8MoeBackend], backend: Fp8MoeBackend) -> None:
|
||||
@@ -102,6 +104,10 @@ def _get_priority_backends(
|
||||
# move XPU backend to the front.
|
||||
_move_to_front(_AVAILABLE_BACKENDS, Fp8MoeBackend.XPU)
|
||||
|
||||
if current_platform.is_cpu():
|
||||
# CPU platform uses FP8 W8A16 fused MoE kernel.
|
||||
_move_to_front(_AVAILABLE_BACKENDS, Fp8MoeBackend.CPU)
|
||||
|
||||
return _AVAILABLE_BACKENDS
|
||||
|
||||
|
||||
@@ -186,6 +192,13 @@ def backend_to_kernel_cls(
|
||||
|
||||
return [XPUExpertsFp8]
|
||||
|
||||
elif backend == Fp8MoeBackend.CPU:
|
||||
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
|
||||
CPUExpertsFp8,
|
||||
)
|
||||
|
||||
return [CPUExpertsFp8]
|
||||
|
||||
else:
|
||||
raise ValueError(f"Unknown FP8 MoE backend: {backend.value}")
|
||||
|
||||
@@ -472,6 +485,12 @@ def convert_to_fp8_moe_kernel_format(
|
||||
)
|
||||
|
||||
w13, w2 = prepare_fp8_moe_layer_for_xpu(w13, w2)
|
||||
elif fp8_backend == Fp8MoeBackend.CPU:
|
||||
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
|
||||
prepare_fp8_moe_layer_for_cpu,
|
||||
)
|
||||
|
||||
w13, w2 = prepare_fp8_moe_layer_for_cpu(w13, w2)
|
||||
else:
|
||||
if fp8_backend not in [
|
||||
Fp8MoeBackend.TRITON,
|
||||
@@ -516,6 +535,14 @@ def make_fp8_moe_quant_config(
|
||||
block_shape=block_shape,
|
||||
)
|
||||
|
||||
# CPU is mixed precision W8A16 config.
|
||||
if fp8_backend == Fp8MoeBackend.CPU:
|
||||
return fp8_w8a16_moe_quant_config(
|
||||
w1_scale=w1_scale,
|
||||
w2_scale=w2_scale,
|
||||
block_shape=block_shape,
|
||||
)
|
||||
|
||||
# Flashinfer CUTLASS per-tensor uses single dq scale
|
||||
# (alpha = w_scale * a_scale) and inverse a2 scale.
|
||||
if fp8_backend == Fp8MoeBackend.FLASHINFER_CUTLASS and block_shape is None:
|
||||
|
||||
Reference in New Issue
Block a user